panic: mtx_lock() of spin mutex (null) @ /syzkaller/managers/i386/kernel/sys/modules/tcp/bbr/../../../netinet/tcp_stacks/bbr.c:12388 cpuid = 0 time = 408 KDB: stack backtrace: db_trace_self_wrapper() at db_trace_self_wrapper+0x47/frame 0xfffffe00516b25b0 vpanic() at vpanic+0x1cc/frame 0xfffffe00516b2610 panic() at panic+0x43/frame 0xfffffe00516b2670 __mtx_lock_flags() at __mtx_lock_flags+0x1d7/frame 0xfffffe00516b26d0 bbr_output_wtime() at bbr_output_wtime+0x4fa8/frame 0xfffffe00516b2990 bbr_output() at bbr_output+0x67/frame 0xfffffe00516b29c0 tcp_hpts_thread() at tcp_hpts_thread+0xe50/frame 0xfffffe00516b2b10 ithread_loop() at ithread_loop+0x33f/frame 0xfffffe00516b2bb0 fork_exit() at fork_exit+0xb3/frame 0xfffffe00516b2bf0 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe00516b2bf0 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- KDB: enter: panic [ thread pid 12 tid 100084 ] Stopped at kdb_enter+0x67: movq $0,0x164017e(%rip) db> db> set $lines = 0 db> set $maxwidth = 0 db> show registers cs 0x20 ds 0x3b es 0x3b fs 0x13 gs 0x1b ss 0 rax 0x12 rcx 0xffffffff811326f0 vprintf+0x140 rdx 0x1 rbx 0 rsp 0xfffffe00516b2590 rbp 0xfffffe00516b25b0 rsi 0 rdi 0xffffffff81132726 vprintf+0x176 r8 0 r9 0x8080808080808080 r10 0xfffffe00516b2480 r11 0x1ffaefff59c r12 0xffffffff82267d30 ddb_dbbe r13 0xffffffff81a6e600 r14 0xffffffff81a74fa7 r15 0xffffffff81a74fa7 rip 0xffffffff81129787 kdb_enter+0x67 rflags 0x86 kdb_enter+0x67: movq $0,0x164017e(%rip) db> show proc Process 12 (intr) at 0xfffff80004c74a60: state: NORMAL uid: 0 gids: 0 parent: pid 0 at 0xffffffff8271ce70 ABI: null flag: 0x10000284 flag2: 0 reaper: 0xffffffff8271ce70 reapsubtree: 12 sigparent: 20 vmspace: 0xffffffff8271dad0 (map 0xffffffff8271dad0) (map.pmap 0xffffffff8271db90) (pmap 0xffffffff8271dbf0) threads: 23 100011 I [swi5: fast taskq] 100014 I [swi6: task queue] 100016 I [swi6: Giant taskq] 100030 I [swi3: vm] 100031 I [swi4: clock (0)] 100032 I [swi4: clock (1)] 100033 I [swi1: netisr 0] 100046 I [irq24: virtio_pci0] 100047 I [irq25: virtio_pci0] 100048 I [irq26: virtio_pci0] 100049 I [irq27: virtio_pci0] 100050 I [irq28: virtio_pci1] 100051 I [irq29: virtio_pci1] 100052 I [irq30: virtio_pci1] 100053 I [irq31: virtio_pci1] 100054 I [irq32: virtio_pci1] 100059 I [irq10: virtio_pci2] 100061 I [irq1: atkbd0] 100062 I [irq12: psm0] 100063 I [swi0: uart uart++] 100071 I [swi1: pf send] 100084 Run CPU 0 [swi1: hpts] 100085 I [swi1: hpts] db> ps pid ppid pgrp uid state wmesg wchan cmd 49644 784 784 0 R (threaded) syz-executor.2 152884 RunQ syz-executor.2 155091 S uwait 0xfffff8002f6c3100 syz-executor.2 49643 785 785 0 R (threaded) syz-executor.1 155059 RunQ syz-executor.1 155090 S connec 0xfffff800140e3bf0 syz-executor.1 49640 797 797 0 R (threaded) syz-executor.0 147664 RunQ syz-executor.0 155084 S connec 0xfffff80014053840 syz-executor.0 155088 S uwait 0xfffff80029351d00 syz-executor.0 49465 1 797 0 S uwait 0xfffff8002f6c3900 syz-executor.0 49464 1 797 0 S uwait 0xfffff80029c74b00 syz-executor.0 49167 49164 784 0 SV uwait 0xfffff80029351b00 syz-executor.2 49164 1 784 0 DV ppwait 0xfffff8002b998a10 syz-executor.2 47843 1 797 0 S uwait 0xfffff80029aff100 syz-executor.0 47842 1 797 0 S uwait 0xfffff8002f694f00 syz-executor.0 45332 45331 797 0 SV uwait 0xfffff8002b6a9b00 syz-executor.0 45331 1 797 0 DV ppwait 0xfffff8002f83da10 syz-executor.0 41651 1 784 0 S uwait 0xfffff80029429d00 syz-executor.2 41650 1 784 0 S uwait 0xfffff80029e5e500 syz-executor.2 41599 41598 784 0 SV uwait 0xfffff8002b6a9500 syz-executor.2 41598 1 784 0 DV ppwait 0xfffff8002ffb6a10 syz-executor.2 40372 40371 784 0 SV uwait 0xfffff80029e9e100 syz-executor.2 40371 1 784 0 DV ppwait 0xfffff8002b5cef40 syz-executor.2 36411 1 797 0 S uwait 0xfffff8002b257300 syz-executor.0 36405 1 797 0 S uwait 0xfffff80029c74300 syz-executor.0 35300 1 797 0 S uwait 0xfffff80029b01100 syz-executor.0 35297 1 797 0 S uwait 0xfffff80029429b00 syz-executor.0 32146 1 786 0 S uwait 0xfffff80029c74500 syz-executor.3 32145 1 786 0 S uwait 0xfffff80029351900 syz-executor.3 26789 1 786 0 S uwait 0xfffff80029aff900 syz-executor.3 26785 1 786 0 S uwait 0xfffff80029351f00 syz-executor.3 25171 1 784 0 S uwait 0xfffff80029b01d00 syz-executor.2 25166 1 784 0 S uwait 0xfffff80029b01700 syz-executor.2 25123 1 786 0 S uwait 0xfffff80029c74100 syz-executor.3 25121 1 786 0 S uwait 0xfffff80029e9ed00 syz-executor.3 21592 21591 784 0 SV uwait 0xfffff80029e5ef00 syz-executor.2 21591 1 784 0 DV ppwait 0xfffff8002b5cea10 syz-executor.2 18479 18474 797 0 SV uwait 0xfffff80029351300 syz-executor.0 18474 1 797 0 DV ppwait 0xfffff80029d184e0 syz-executor.0 14664 1 14664 65 Ss select 0xfffff8002f0b25c0 dhclient 9667 0 0 0 DL aiordy 0xfffff80029c13530 [aiod4] 9666 0 0 0 DL aiordy 0xfffff80029da2530 [aiod3] 9665 0 0 0 DL aiordy 0xfffff80029d3ea60 [aiod2] 9664 0 0 0 DL aiordy 0xfffff80029c13a60 [aiod1] 9179 1 9179 0 Ss select 0xfffff8002f0b2bc0 dhclient 9172 1 9172 0 Ss select 0xfffff8002bf95440 dhclient 9131 1 9131 65 Ss select 0xfffff8002f0c85c0 dhclient 5098 1 5098 0 Ss select 0xfffff8002bb275c0 dhclient 5095 1 5095 0 Ss select 0xfffff8002ba55a40 dhclient 5076 1 5076 65 Ss select 0xfffff8002bb27440 dhclient 2719 1 2719 0 Ss select 0xfffff8002b5a7a40 dhclient 2716 1 2716 0 Ss select 0xfffff8002b57ebc0 dhclient 2697 1 2697 65 Ss select 0xfffff8002b524a40 dhclient 1407 1 1407 0 Ss select 0xfffff80029e97bc0 dhclient 1399 1 1399 0 Ss select 0xfffff80029f4a140 dhclient 797 780 797 0 Ss nanslp 0xffffffff8273d0a0 syz-executor.0 786 780 786 0 Rs syz-executor.3 785 780 785 0 Ss nanslp 0xffffffff8273d0a0 syz-executor.1 784 780 784 0 Rs syz-executor.2 780 778 778 0 R (threaded) syz-fuzzer 100108 S uwait 0xfffff80015e46500 syz-fuzzer 100115 RunQ syz-fuzzer 100116 S uwait 0xfffff80029508500 syz-fuzzer 100117 S uwait 0xfffff80029508700 syz-fuzzer 100118 S uwait 0xfffff80029508900 syz-fuzzer 100119 S uwait 0xfffff80029429500 syz-fuzzer 100120 S uwait 0xfffff80029429700 syz-fuzzer 100121 RunQ syz-fuzzer 100122 Run CPU 1 syz-fuzzer 100124 S uwait 0xfffff80029b01500 syz-fuzzer 778 776 778 0 Ss pause 0xfffff800159de0b0 csh 776 694 776 0 Ss select 0xfffff800295d0740 sshd 760 1 760 0 Ss+ ttyin 0xfffff800154a5cb0 getty 759 1 759 0 Ss+ ttyin 0xfffff80015acb4b0 getty 758 1 758 0 Ss+ ttyin 0xfffff80015acbcb0 getty 757 1 757 0 Ss+ ttyin 0xfffff80015ace4b0 getty 756 1 756 0 Ss+ ttyin 0xfffff80015acecb0 getty 755 1 755 0 Ss+ ttyin 0xfffff80015ad14b0 getty 754 1 754 0 Ss+ ttyin 0xfffff80015ad1cb0 getty 753 1 753 0 Ss+ ttyin 0xfffff80015ad64b0 getty 752 1 752 0 Ss+ ttyin 0xfffff80015ad6cb0 getty 698 1 698 0 Ss nanslp 0xffffffff8273d0a0 cron 694 1 694 0 Ss select 0xfffff80015cb9440 sshd 507 1 507 0 Ss select 0xfffff80015d728c0 syslogd 436 1 436 0 Ss select 0xfffff80015ef22c0 devd 435 1 435 65 Ss select 0xfffff800293dbd40 dhclient 350 1 350 0 Ss select 0xfffff80015fad440 dhclient 347 1 347 0 Ss select 0xfffff80029311d40 dhclient 23 0 0 0 DL vlruwt 0xfffff80015c6aa60 [vnlru] 22 0 0 0 DL syncer 0xffffffff8282c450 [syncer] 21 0 0 0 DL (threaded) [bufdaemon] 100081 D qsleep 0xffffffff8282b520 [bufdaemon] 100086 D - 0xffffffff8220aec0 [bufspacedaemon-0] 100099 D sdflush 0xfffff80004ddd4e8 [/ worker] 20 0 0 0 DL psleep 0xffffffff82853388 [vmdaemon] 19 0 0 0 DL (threaded) [pagedaemon] 100079 D psleep 0xffffffff828477f8 [dom0] 100087 D launds 0xffffffff82847804 [laundry: dom0] 100088 D umarcl 0xffffffff815c5090 [uma] 18 0 0 0 DL - 0xffffffff82571458 [rand_harvestq] 17 0 0 0 DL waiting 0xffffffff8305b8b8 [sctp_iterator] 16 0 0 0 DL pftm 0xffffffff82d90410 [pf purge] 15 0 0 0 DL - 0xffffffff82828bdc [soaiod4] 9 0 0 0 DL - 0xffffffff82828bdc [soaiod3] 8 0 0 0 DL - 0xffffffff82828bdc [soaiod2] 7 0 0 0 DL - 0xffffffff82828bdc [soaiod1] 6 0 0 0 DL (threaded) [cam] 100044 D - 0xffffffff82448b40 [doneq0] 100045 D - 0xffffffff82448ac0 [async] 100078 D - 0xffffffff82448990 [scanner] 14 0 0 0 DL seqstat 0xfffff80004d8a488 [sequencer 00] 5 0 0 0 DL crypto_ 0xfffff80004d53d80 [crypto returns 1] 4 0 0 0 DL crypto_ 0xfffff80004d53d30 [crypto returns 0] 3 0 0 0 DL crypto_ 0xffffffff82844ce0 [crypto] 13 0 0 0 DL (threaded) [geom] 100035 D - 0xffffffff8271c8e0 [g_event] 100036 D - 0xffffffff8271c8e8 [g_up] 100037 D - 0xffffffff8271c8f0 [g_down] 2 0 0 0 DL (threaded) [KTLS] 100028 D - 0xfffff80004c69100 [thr_0] 100029 D - 0xfffff80004c69180 [thr_1] 12 0 0 0 RL (threaded) [intr] 100011 I [swi5: fast taskq] 100014 I [swi6: task queue] 100016 I [swi6: Giant taskq] 100030 I [swi3: vm] 100031 I [swi4: clock (0)] 100032 I [swi4: clock (1)] 100033 I [swi1: netisr 0] 100046 I [irq24: virtio_pci0] 100047 I [irq25: virtio_pci0] 100048 I [irq26: virtio_pci0] 100049 I [irq27: virtio_pci0] 100050 I [irq28: virtio_pci1] 100051 I [irq29: virtio_pci1] 100052 I [irq30: virtio_pci1] 100053 I [irq31: virtio_pci1] 100054 I [irq32: virtio_pci1] 100059 I [irq10: virtio_pci2] 100061 I [irq1: atkbd0] 100062 I [irq12: psm0] 100063 I [swi0: uart uart++] 100071 I [swi1: pf send] 100084 Run CPU 0 [swi1: hpts] 100085 I [swi1: hpts] 11 0 0 0 RL (threaded) [idle] 100003 CanRun [idle: cpu0] 100004 CanRun [idle: cpu1] 1 0 1 0 SLs wait 0xfffff80004bda530 [init] 10 0 0 0 DL audit_w 0xffffffff828451f0 [audit] 0 0 0 0 DLs (threaded) [kernel] 100000 D swapin 0xffffffff8271ce70 [swapper] 100005 D - 0xfffff8000443b500 [if_io_tqg_0] 100006 D - 0xfffff8000443b100 [if_io_tqg_1] 100007 D - 0xfffff80004c78d00 [if_config_tqg_0] 100008 D - 0xfffff80004c78900 [softirq_0] 100009 D - 0xfffff80004c78500 [softirq_1] 100010 D - 0xfffff80004c78100 [aiod_kick taskq] 100012 D - 0xfffff80004c75900 [kqueue_ctx taskq] 100013 D - 0xfffff80004c75500 [pci_hp taskq] 100015 D - 0xfffff80004c6ed00 [inm_free taskq] 100017 D - 0xfffff80004c6e500 [linuxkpi_irq_wq] 100018 D - 0xfffff80004c6e100 [thread taskq] 100019 D - 0xfffff80004c69d00 [in6m_free taskq] 100020 D - 0xfffff80004c69900 [linuxkpi_short_wq_0] 100021 D - 0xfffff80004c69900 [linuxkpi_short_wq_1] 100022 D - 0xfffff80004c69900 [linuxkpi_short_wq_2] 100023 D - 0xfffff80004c69900 [linuxkpi_short_wq_3] 100024 D - 0xfffff80004c69500 [linuxkpi_long_wq_0] 100025 D - 0xfffff80004c69500 [linuxkpi_long_wq_1] 100026 D - 0xfffff80004c69500 [linuxkpi_long_wq_2] 100027 D - 0xfffff80004c69500 [linuxkpi_long_wq_3] 100034 D - 0xfffff80004cd3d00 [firmware taskq] 100038 D - 0xfffff80004cd3900 [crypto_0] 100039 D - 0xfffff80004cd3900 [crypto_1] 100055 D - 0xfffff800153ecd00 [vtnet0 rxq 0] 100056 D - 0xfffff800153ec900 [vtnet0 txq 0] 100057 D - 0xfffff800153ec500 [vtnet0 rxq 1] 100058 D - 0xfffff800153ec100 [vtnet0 txq 1] 100060 D vtbslp 0xfffff80015463700 [virtio_balloon] 100064 D - 0xfffff80015469900 [mca taskq] 100066 D - 0xffffffff81e217c1 [deadlkres] 100074 D - 0xfffff80015a5d900 [acpi_task_0] 100075 D - 0xfffff80015a5d900 [acpi_task_1] 100076 D - 0xfffff80015a5d900 [acpi_task_2] 100077 D - 0xfffff80004cd3500 [CAM taskq] db> show all locks Process 49640 (syz-executor.0) thread 0xfffffe009559cc80 (147664) exclusive sleep mutex pmap (pmap) r = 0 (0xfffffe009ec6b120) locked @ /syzkaller/managers/i386/kernel/sys/amd64/amd64/pmap.c:7367 shared rw vm object (vm object) r = 0 (0xfffff80029305840) locked @ /syzkaller/managers/i386/kernel/sys/vm/vm_fault.c:1332 shared sx vm map (user) (vm map (user)) r = 0 (0xfffffe009ec6b060) locked @ /syzkaller/managers/i386/kernel/sys/vm/vm_map.c:4935 Process 12 (intr) thread 0xfffffe005537a020 (100084) exclusive rw tcpinp (tcpinp) r = 0 (0xfffff8002f69fb90) locked @ /syzkaller/managers/i386/kernel/sys/netinet/tcp_hpts.c:1536 db> show malloc Type InUse MemUse Requests sysctloid 34288 12858K 34355 pf_hash 5 11560K 5 devbuf 4216 6982K 4244 pcb 831 4215K 84946 tcp_hpts 5 3219K 5 kobj 326 2608K 486 sctp_stro 396 2084K 17549 vtbuf 24 2064K 46 filedesc 72 1150K 97140 newblk 11 1037K 66377 vfscache 3 1035K 3 inodedep 507 1026K 51494 subproc 205 877K 49757 sctp_atcl 797 797K 55728 acpica 1674 649K 54806 callout 2 528K 2 ufs_quota 1 520K 1 vfs_hash 1 520K 1 intr 4 480K 4 sctp_atky 1201 451K 73935 bus 986 377K 3469 BPF 46 337K 91 pf_rule 76 304K 568 sctp_athm 797 299K 56243 sctp_map 792 297K 34614 linker 348 270K 481 dirrem 498 249K 49151 kdtrace 284 224K 104739 DEVFS1 107 214K 124 module 510 192K 510 freefile 495 186K 49132 umtx 418 183K 418 vnet_data 1 176K 1 tidhash 3 164K 3 pagedep 14 143K 49297 tfo_ccache 1 136K 1 sem 4 120K 4 vmem 3 104K 6 UMA 270 102K 270 sctp_timw 178 89K 178 audit_evclass 236 89K 294 temp 36 81K 10932 mtx_pool 2 80K 2 syncache 1 76K 1 ksem 34 70K 2604 routetbl 126 69K 475 msg 4 68K 4 ifaddr 70 66K 72 acpitask 1 64K 1 ddb_capture 1 64K 1 DEVFS3 126 63K 136 ether_multi 152 57K 162 gtaskqueue 18 57K 18 DEVFS_RULE 56 54K 56 kenv 95 52K 95 eventhandler 133 50K 133 kqueue 98 45K 49868 ithread 99 43K 99 rman 84 42K 425 lltable 46 40K 410 KTRACE 100 38K 100 pwddesc 96 36K 49611 DEVFSP 95 36K 22284 taskqueue 60 36K 60 proc 3 34K 3 ufs_mount 5 34K 6 bus-sc 33 34K 1684 devstat 4 33K 4 hostcache 1 32K 1 tty 16 32K 16 shm 1 32K 121 GEOM 60 29K 489 in6_multi 65 29K 65 kbdmux 6 28K 6 cred 27 27K 1233 ifnet 7 25K 7 plimit 24 24K 961 kcovinfo 64 24K 68 CAM queue 5 21K 1528 selfd 55 21K 647583 pfs_nodes 20 20K 20 proc-args 52 20K 722 UART 12 18K 12 session 35 18K 64 crypto 31 17K 22018 bmsafemap 2 17K 50176 rpc 2 16K 2 shmfd 1 16K 89 pfs_vncache 1 16K 1 inpcbpolicy 39 15K 17444 ufs_dirhash 24 14K 24 xform 35 14K 44845 sglist 5 13K 5 lockf 33 13K 111 MCA 32 12K 32 CAM DEV 3 12K 510 Unitno 31 12K 51 vt 11 11K 11 acpisem 28 11K 28 CAM XPT 22 11K 543 osd 26 10K 14793 pf_ifnet 12 10K 196 uidinfo 3 9K 84 memdesc 1 8K 1 ipsec-saq 2 8K 2 evdev 4 8K 4 acpidev 20 8K 20 hhook 15 8K 17 select 19 8K 739 mount 16 7K 90 fpukern_ctx 3 6K 3 terminal 11 6K 11 ip6ndp 12 6K 14 sctp_ifa 13 5K 14 ipsecpolicy 2 5K 2 encap_export_host 12 5K 12 clone 9 5K 9 local_apic 1 4K 1 io_apic 1 4K 1 sahead 1 4K 1 secasvar 1 4K 1 nhops 6 4K 8 pci_link 10 4K 10 msi 9 4K 9 DEVFS 9 4K 10 newdirblk 9 4K 48549 tun 7 4K 7 ipsec 3 3K 3 mkdir 8 3K 97098 in_multi 6 3K 8 nexusdev 7 3K 7 feeder 7 3K 7 procdesc 5 3K 18 freework 5 3K 55868 sctp_ifn 6 3K 14 mld 6 3K 6 igmp 6 3K 6 soname 6 3K 55718 toponodes 6 3K 6 prison 6 3K 6 isadev 6 3K 6 softdep 1 2K 1 freeblks 4 2K 49596 vnodemarker 2 2K 2686 NFSD session 1 2K 1 linux 5 2K 6 CAM periph 4 2K 271 filecaps 5 2K 122 diradd 4 2K 49192 pfil 4 2K 4 CAM path 4 2K 1034 dctcp data 3 2K 265 tcpfunc 3 2K 3 loginclass 3 2K 8 sctp_stri 1 1K 1208 chacha20random 1 1K 1 vnodes 1 1K 1 CAM SIM 2 1K 2 ktls 1 1K 1 cdev 2 1K 2 aesni_data 2 1K 2 cpus 2 1K 2 atkbddev 2 1K 2 CAM dev queue 2 1K 2 entropy 2 1K 48 NFSD lckfile 1 1K 1 NFSD V4client 1 1K 1 pmchooks 1 1K 1 sctp_vrf 1 1K 1 apmdev 1 1K 1 CAM I/O Scheduler 1 1K 1 vnet_data_free 1 1K 1 vnet 1 1K 1 Per-cpu 1 1K 1 p1003.1b 1 1K 1 acpiintr 1 1K 1 pmc 1 1K 1 sctp_mcore 0 0K 0 sctp_socko 0 0K 23630 sctp_iter 0 0K 58 sctp_mvrf 0 0K 0 sctp_cpal 0 0K 50 sctp_cmsg 0 0K 0 sctp_stre 0 0K 0 sctp_athi 0 0K 0 sctp_a_it 0 0K 9 sctp_aadr 0 0K 65 tcp_do 0 0K 0 tcp_fsb 0 0K 2887 mqdata 0 0K 0 chd data 0 0K 493 cubic data 0 0K 294 htcp data 0 0K 639 pf_table 0 0K 18 pf_altq 0 0K 0 pf_osfp 0 0K 0 pf_temp 0 0K 0 cdg data 0 0K 573 vegas data 0 0K 74 NFSCL lckown 0 0K 0 NFSCL client 0 0K 0 madt_table 0 0K 2 smartpqi 0 0K 0 NFSCL deleg 0 0K 0 NFSCL open 0 0K 0 NFSCL owner 0 0K 0 NFS fh 0 0K 0 NFS req 0 0K 0 NFSD usrgroup 0 0K 0 NFSD string 0 0K 0 iavf 0 0K 0 ixl 0 0K 0 NFSD V4lock 0 0K 0 NFSD V4state 0 0K 0 NFSD srvcache 0 0K 0 msdosfs_fat 0 0K 0 msdosfs_mount 0 0K 0 msdosfs_node 0 0K 0 ice-resmgr 0 0K 0 ice-osdep 0 0K 0 ice 0 0K 0 axgbe 0 0K 0 DEVFS4 0 0K 0 DEVFS2 0 0K 0 gntdev 0 0K 0 privcmd_dev 0 0K 0 evtchn_dev 0 0K 0 xen_intr 0 0K 0 xenstore 0 0K 0 ciss_data 0 0K 0 xen_hvm 0 0K 0 legacydrv 0 0K 0 qpidrv 0 0K 0 BACKLIGHT 0 0K 0 xnb 0 0K 0 dmar_idpgtbl 0 0K 0 dmar_dom 0 0K 0 dmar_ctx 0 0K 0 xbbd 0 0K 0 xbd 0 0K 0 Balloon 0 0K 0 isci 0 0K 0 iommu_dmamap 0 0K 0 sysmouse 0 0K 0 hyperv_socket 0 0K 0 bxe_ilt 0 0K 0 xenbus 0 0K 0 vtfont 0 0K 0 vm_fictitious 0 0K 0 ath_hal 0 0K 0 athdev 0 0K 0 ata_pci 0 0K 0 ata_dma 0 0K 0 ata_generic 0 0K 0 amr 0 0K 0 pvscsi 0 0K 0 scsi_da 0 0K 69 ata_da 0 0K 0 scsi_ch 0 0K 0 UMAHash 0 0K 0 scsi_cd 0 0K 0 AHCI driver 0 0K 0 vm_pgdata 0 0K 0 jblocks 0 0K 0 savedino 0 0K 38516 sentinel 0 0K 0 jfsync 0 0K 0 jtrunc 0 0K 0 sbdep 0 0K 1348 jsegdep 0 0K 0 jseg 0 0K 0 jfreefrag 0 0K 0 jfreeblk 0 0K 0 jnewblk 0 0K 0 jmvref 0 0K 0 jremref 0 0K 0 jaddref 0 0K 0 freedep 0 0K 0 freefrag 0 0K 6 allocindir 0 0K 0 indirdep 0 0K 10443 allocdirect 0 0K 0 ufs_trim 0 0K 0 mactemp 0 0K 0 audit_trigger 0 0K 0 audit_pipe_presel 0 0K 0 audit_pipeent 0 0K 0 audit_pipe 0 0K 0 audit_evname 0 0K 0 audit_bsm 0 0K 0 audit_gidset 0 0K 0 audit_text 0 0K 0 audit_path 0 0K 0 audit_data 0 0K 0 audit_cred 0 0K 0 USBdev 0 0K 0 USB 0 0K 0 agp 0 0K 0 nvme_da 0 0K 0 acpipwr 0 0K 0 acpi_perf 0 0K 0 ktls_ocf 0 0K 0 twsbuf 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5E_TLS 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EN 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5DUMP 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 seq_file 0 0K 0 radix 0 0K 0 idr 0 0K 0 lkpifw 0 0K 0 NLM 0 0K 0 ipsec-spdcache 0 0K 0 ipsec-reg 0 0K 0 ipsec-misc 0 0K 0 ipsecrequest 0 0K 0 ip6opt 0 0K 21 ip6_msource 0 0K 0 ip6_moptions 0 0K 2 in6_mfilter 0 0K 0 frag6 0 0K 0 tcplog 0 0K 0 tcp_hwpace 0 0K 0 twe_commands 0 0K 0 LRO 0 0K 0 newreno data 0 0K 10 ip_msource 0 0K 0 ip_moptions 0 0K 0 in_mfilter 0 0K 0 ipid 0 0K 0 80211scan 0 0K 0 80211ratectl 0 0K 0 80211power 0 0K 0 80211nodeie 0 0K 0 80211node 0 0K 0 80211mesh_gt 0 0K 0 80211mesh_rt 0 0K 0 80211perr 0 0K 0 80211prep 0 0K 0 80211preq 0 0K 0 80211dfs 0 0K 0 80211crypto 0 0K 0 80211vap 0 0K 0 iflib 0 0K 0 vlan 0 0K 0 gif 0 0K 0 ifdescr 0 0K 0 zlib 0 0K 0 fadvise 0 0K 0 VN POLL 0 0K 0 twa_commands 0 0K 0 statfs 0 0K 48821 namei_tracker 0 0K 8 export_host 0 0K 0 cl_savebuf 0 0K 4 tcp_log_dev 0 0K 1978 midi buffers 0 0K 0 mixer 0 0K 0 ac97 0 0K 0 hdacc 0 0K 0 hdac 0 0K 0 hdaa 0 0K 0 acpicmbat 0 0K 0 SIIS driver 0 0K 0 CAM CCB 0 0K 52389 PUC 0 0K 0 ppbusdev 0 0K 0 agtiapi_MemAlloc malloc 0 0K 0 osti_cacheable 0 0K 0 tempbuff 0 0K 0 biobuf 0 0K 0 aios 0 0K 22 lio 0 0K 44 acl 0 0K 0 tempbuff 0 0K 0 mbuf_tag 0 0K 360 ag_tgt_map_t malloc 0 0K 0 ag_slr_map_t malloc 0 0K 0 lDevFlags * malloc 0 0K 0 tiDeviceHandle_t * malloc 0 0K 0 ag_portal_data_t malloc 0 0K 0 ag_device_t malloc 0 0K 0 STLock malloc 0 0K 0 CCB List 0 0K 0 sr_iov 0 0K 0 OCS 0 0K 0 OCS 0 0K 0 nvme 0 0K 0 nvd 0 0K 0 netmap 0 0K 0 mwldev 0 0K 0 MVS driver 0 0K 0 CAM ccb queue 0 0K 0 mrsasbuf 0 0K 0 mpt_user 0 0K 0 mps_user 0 0K 0 accf 0 0K 0 pts 0 0K 0 iov 0 0K 37180 ioctlops 0 0K 2958 eventfd 0 0K 0 Witness 0 0K 0 stack 0 0K 0 MPSSAS 0 0K 0 mps 0 0K 0 mpr_user 0 0K 0 MPRSAS 0 0K 0 mpr 0 0K 0 mfibuf 0 0K 0 sbuf 0 0K 288 md_sectors 0 0K 0 firmware 0 0K 0 compressor 0 0K 0 md_disk 0 0K 0 SWAP 0 0K 0 malodev 0 0K 0 LED 0 0K 0 sysctltmp 0 0K 871 sysctl 0 0K 3 ekcd 0 0K 0 dumper 0 0K 0 sendfile 0 0K 0 rctl 0 0K 0 ix_sriov 0 0K 0 aacraidcam 0 0K 0 aacraid_buf 0 0K 0 ix 0 0K 0 ipsbuf 0 0K 0 cache 0 0K 0 iirbuf 0 0K 0 prison_racct 0 0K 0 Fail Points 0 0K 0 sigio 0 0K 3 filedesc_to_leader 0 0K 56 pwd 0 0K 0 tty console 0 0K 0 aaccam 0 0K 0 aacbuf 0 0K 0 zstd 0 0K 0 XZ_DEC 0 0K 0 nvlist 0 0K 0 SCSI ENC 0 0K 0 SCSI sa 0 0K 0 scsi_pass 0 0K 0 isofs_node 0 0K 0 isofs_mount 0 0K 0 tr_raid5_data 0 0K 0 tr_raid1e_data 0 0K 0 tr_raid1_data 0 0K 0 tr_raid0_data 0 0K 0 tr_concat_data 0 0K 0 md_sii_data 0 0K 0 md_promise_data 0 0K 0 md_nvidia_data 0 0K 0 md_jmicron_data 0 0K 0 md_intel_data 0 0K 0 md_ddf_data 0 0K 0 raid_data 0 0K 72 geom_flashmap 0 0K 0 tmpfs dir 0 0K 0 tmpfs name 0 0K 0 tmpfs mount 0 0K 0 NFS FHA 0 0K 0 newnfsmnt 0 0K 0 newnfsclient_req 0 0K 0 NFSCL layrecall 0 0K 0 NFSCL session 0 0K 0 NFSCL sockreq 0 0K 0 NFSCL devinfo 0 0K 0 NFSCL flayout 0 0K 0 NFSCL layout 0 0K 0 NFSD rollback 0 0K 0 NFSCL diroff 0 0K 0 NEWdirectio 0 0K 0 NEWNFSnode 0 0K 0 NFSCL lck 0 0K 0 db> show uma Zone Size Used Free Requests Sleeps Bucket Total Mem XFree mbuf_jumbo_page 4096 8324 984 952566 0 254 38125568 0 malloc-384 384 36295 1785 75751 0 30 14622720 0 malloc-8192 8192 402 872 37697 0 1 10436608 0 malloc-1024 1024 4174 10 4224 0 16 4284416 0 malloc-1024 1024 593 3187 55123 0 16 3870720 0 mbuf_cluster 2048 1500 2 1500 0 254 3076096 0 tcp_log 416 1848 5253 197254 0 254 2954016 0 sctp_asoc 2288 396 894 17307 0 254 2951520 0 malloc-384 384 3127 4353 849262 0 30 2872320 0 malloc-8192 8192 328 2 1099 0 1 2703360 0 mbuf 256 9224 1186 3196766 0 254 2664960 0 malloc-1024 1024 803 1761 58121 0 16 2625536 0 malloc-2048 2048 396 884 18977 0 8 2621440 0 pbuf 2624 0 973 0 0 2 2553152 0 malloc-1024 1024 134 2286 41105 0 16 2478080 0 malloc-512 512 636 3204 49765 0 30 1966080 0 UMA Slabs 0 112 17480 13 17482 0 126 1959216 0 BUF TRIE 144 192 13276 27934 0 62 1939392 0 RADIX NODE 144 12497 184 1205300 0 62 1826064 0 sctp_ep 1280 400 929 37692 0 254 1701120 0 malloc-384 384 1077 3163 212684 0 30 1628160 0 malloc-1024 1024 597 883 95580 0 16 1515520 0 malloc-32768 32768 37 5 48574 0 1 1376256 0 sctp_raddr 736 398 900 19507 0 254 955328 0 malloc-8192 8192 98 4 49646 0 1 835584 0 malloc-384 384 1731 69 55409 0 30 691200 0 malloc-65536 65536 3 7 9914 0 1 655360 0 FFS inode 1160 535 25 49670 0 8 649600 0 tcp_bbr_map 128 24 4285 126057 0 126 551552 0 ertt_txseginfo 40 1 13230 1532442 0 254 529240 0 VM OBJECT 264 1751 64 703890 0 30 479160 0 256 Bucket 2048 184 26 23660 0 8 430080 0 malloc-16384 16384 24 0 25 0 1 393216 0 mbuf_packet 256 188 1312 85417 0 254 384000 0 THREAD 1808 187 22 55092 0 8 377872 0 lkpicurr 160 2 2348 2 0 62 376000 0 lkpimm 160 1 2324 1 0 62 372000 0 malloc-4096 4096 83 2 529 0 2 348160 0 socket 944 72 288 57074 0 254 339840 0 malloc-65536 65536 5 0 5 0 1 327680 0 MAP ENTRY 96 2930 304 2685054 0 126 310464 0 malloc-512 512 33 551 72062 0 30 299008 0 VNODE 448 573 93 49710 0 30 298368 0 malloc-384 384 650 60 29697 0 30 272640 0 malloc-65536 65536 2 2 1119 0 1 262144 0 malloc-2048 2048 125 3 146 0 8 262144 0 tcp_inpcb 488 25 511 14781 0 254 261568 0 vmem btag 56 4411 41 4411 0 254 249312 0 DEVCTL 1024 0 216 132 0 0 221184 0 malloc-16384 16384 11 2 247 0 1 212992 0 malloc-4096 4096 7 44 53368 0 3 208896 0 malloc-8192 8192 23 2 53 0 1 204800 0 sctp_chunk 152 365 935 4744 0 254 197600 0 UMA Zones 768 242 2 242 0 16 187392 0 VMSPACE 2544 63 9 49613 0 4 183168 0 32 Bucket 256 497 193 41944 0 62 176640 0 malloc-384 384 388 62 205939 0 30 172800 0 malloc-384 384 369 71 94398 0 30 168960 0 FFS2 dinode 256 535 110 49668 0 62 165120 0 malloc-512 512 196 108 108620 0 30 155648 0 tcpcb 1080 22 118 14781 0 254 151200 0 malloc-16384 16384 4 5 73 0 1 147456 0 malloc-512 512 232 56 49868 0 30 147456 0 S VFS Cache 104 1063 341 52949 0 126 146016 0 PROC 1328 95 13 49645 0 8 143424 0 malloc-65536 65536 0 2 176 0 1 131072 0 malloc-1024 1024 121 3 130 0 16 126976 0 filedesc0 1072 96 16 49611 0 8 120064 0 ksiginfo 112 97 947 2215 0 126 116928 0 16 Bucket 144 542 242 10328 0 62 112896 0 128 Bucket 1024 69 34 4716 0 16 105472 0 malloc-512 512 192 8 1357 0 30 102400 0 malloc-32768 32768 1 2 4 0 1 98304 0 malloc-32768 32768 1 2 34 0 1 98304 0 malloc-32768 32768 1 2 121 0 1 98304 0 malloc-16384 16384 2 4 64 0 1 98304 0 malloc-512 512 130 62 51844 0 30 98304 0 UMA Kegs 384 227 6 227 0 30 89472 0 g_bio 408 0 210 207021 0 30 85680 0 clpbuf 2624 0 32 20 0 16 83968 0 malloc-16384 16384 3 2 8 0 1 81920 0 malloc-8192 8192 3 6 409 0 1 73728 0 malloc-4096 4096 13 4 1467 0 2 69632 0 malloc-65536 65536 0 1 1 0 1 65536 0 malloc-16384 16384 2 2 7 0 1 65536 0 malloc-4096 4096 14 2 1282 0 2 65536 0 sctp_laddr 48 17 1327 5943 0 254 64512 0 malloc-2048 2048 10 20 523 0 8 61440 0 malloc-384 384 97 63 70068 0 30 61440 0 malloc-8192 8192 2 5 49474 0 1 57344 0 malloc-512 512 45 67 9210 0 30 57344 0 64 Bucket 512 54 50 34912 0 30 53248 0 sctp_stream_msg_out 112 41 427 3892 0 254 52416 0 malloc-16384 16384 3 0 3 0 1 49152 0 pcpu-8 8 5341 803 12088 0 254 49152 0 pipe 744 23 42 3778 0 16 48360 0 malloc-2048 2048 16 6 77 0 8 45056 0 Files 80 280 270 155992 0 126 44000 0 DIRHASH 1024 34 6 34 0 16 40960 0 malloc-512 512 27 53 1617 0 30 40960 0 tcp_bbr_pcb 832 12 33 6169 0 16 37440 0 NAMEI 1024 0 36 245028 0 16 36864 0 malloc-2048 2048 2 16 417 0 8 36864 0 malloc-32768 32768 1 0 1 0 1 32768 0 malloc-32768 32768 1 0 1 0 1 32768 0 malloc-4096 4096 6 2 31 0 2 32768 0 pcpu-64 64 487 25 487 0 254 32768 0 2 Bucket 32 522 486 12459 0 254 32256 0 TURNSTILE 136 210 21 210 0 62 31416 0 tcp_rack_pcb 832 0 36 2887 0 16 29952 0 4 Bucket 48 19 569 5669 0 254 28224 0 KNOTE 160 27 148 504937 0 62 28000 0 8 Bucket 80 153 197 9214 0 126 28000 0 udplite_inpcb 488 2 54 1312 0 254 27328 0 malloc-8192 8192 0 3 13 0 1 24576 0 malloc-8192 8192 1 2 60 0 1 24576 0 malloc-4096 4096 1 5 10 0 2 24576 0 malloc-4096 4096 2 4 2002 0 2 24576 0 PWD 32 36 720 48634 0 254 24192 0 ttyinq 160 135 15 300 0 62 24000 0 unpcb 256 24 66 1883 0 254 23040 0 ttyoutq 256 72 18 160 0 62 23040 0 SLEEPQUEUE 88 210 14 210 0 126 19712 0 udp_inpcb 488 10 30 850 0 254 19520 0 Mountpoints 2752 2 5 2 0 4 19264 0 malloc-16384 16384 1 0 1 0 1 16384 0 vtnet_tx_hdr 24 0 668 544823 0 254 16032 0 ripcb 488 5 27 501 0 254 15616 0 malloc-4096 4096 1 2 142 0 2 12288 0 malloc-2048 2048 3 3 163 0 8 12288 0 malloc-2048 2048 2 4 6 0 8 12288 0 tcp_rack_map 112 0 108 4481 0 126 12096 0 ertt 72 22 146 14781 0 126 12096 0 kenv 258 15 30 1042 0 30 11610 0 routing nhops 256 27 18 34 0 62 11520 0 malloc-2048 2048 4 0 4 0 8 8192 0 malloc-1024 1024 1 7 10 0 16 8192 0 malloc-1024 1024 0 8 18 0 16 8192 0 pf tags 104 2 76 19 0 126 8112 0 rtentry 176 30 16 34 0 62 8096 0 PGRP 88 35 57 70 0 126 8096 0 rl_entry 40 98 104 98 0 254 8080 0 udpcb 32 8 244 2162 0 254 8064 0 AIOP 32 4 248 4 0 254 8064 0 sctp_readq 152 2 50 329 0 254 7904 0 AIO 208 2 36 114 0 62 7904 0 cryptop 280 0 28 2851 0 30 7840 0 itimer 352 0 22 8 0 30 7744 0 AIOCB 552 0 14 203 0 16 7728 0 AIOLIO 272 0 28 22 0 30 7616 0 FPU_save_area 832 1 8 1 0 16 7488 0 cpuset 104 7 55 21 0 126 6448 0 domainset 40 0 126 12 0 254 5040 0 epoch_record pcpu 256 4 12 4 0 62 4096 0 pcpu-16 16 7 249 7 0 254 4096 0 hostcache 64 1 62 1 0 254 4032 0 syncache 168 0 24 4 0 254 4032 0 UMA Slabs 1 176 11 11 11 0 62 3872 0 mqnode 416 3 6 3 0 30 3744 0 KMAP ENTRY 96 12 27 12 0 0 3744 0 vmem 1856 1 1 1 0 8 3712 0 SMR CPU 32 3 60 3 0 254 2016 0 SMR SHARED 24 3 60 3 0 254 1512 0 FFS1 dinode 128 0 0 0 0 126 0 0 da_ccb 544 0 0 0 0 16 0 0 ada_ccb 272 0 0 0 0 30 0 0 swblk 136 0 0 0 0 62 0 0 swpctrie 144 0 0 0 0 62 0 0 sctp_asconf_ack 48 0 0 0 0 254 0 0 sctp_asconf 40 0 0 0 0 254 0 0 cdg_qdiffsample 16 0 0 0 0 254 0 0 pf state scrubs 40 0 0 0 0 254 0 0 pf frag entries 40 0 0 0 0 254 0 0 pf frags 248 0 0 0 0 62 0 0 pf table entries 160 0 0 0 0 62 0 0 pf table entry counters 64 0 0 0 0 254 0 0 pf source nodes 136 0 0 0 0 254 0 0 pf state keys 88 0 0 0 0 126 0 0 pf states 304 0 0 0 0 0 0 0 pf mtags 48 0 0 0 0 254 0 0 tcp_log_node 120 0 0 0 0 126 0 0 tcp_log_bucket 176 0 0 0 0 62 0 0 tcpreass 48 0 0 0 0 254 0 0 tfo_ccache_entries 80 0 0 0 0 126 0 0 tfo 4 0 0 0 0 254 0 0 sackhole 32 0 0 0 0 254 0 0 tcptw 88 0 0 0 0 254 0 0 ipq 56 0 0 0 0 254 0 0 IPsec SA lft_c 16 0 0 0 0 254 0 0 NCLNODE 584 0 0 0 0 16 0 0 mqnotifier 216 0 0 0 0 62 0 0 mvdata 64 0 0 0 0 254 0 0 mqueue 248 0 0 0 0 62 0 0 TMPFS node 224 0 0 0 0 62 0 0 LTS VFS Cache 360 0 0 0 0 30 0 0 L VFS Cache 320 0 0 0 0 30 0 0 STS VFS Cache 144 0 0 0 0 62 0 0 linux_dma_object 24 0 0 0 0 254 0 0 linux_dma_pctrie 144 0 0 0 0 62 0 0 IOMMU_MAP_ENTRY 120 0 0 0 0 126 0 0 ktls_session 128 0 0 0 0 126 0 0 mbuf_jumbo_16k 16384 0 0 0 0 254 0 0 mbuf_jumbo_9k 9216 0 0 0 0 254 0 0 audit_record 1280 0 0 0 0 8 0 0 MAC labels 40 0 0 0 0 254 0 0 vnpbuf 2624 0 0 0 0 64 0 0 mdpbuf 2624 0 0 0 0 3 0 0 nfspbuf 2624 0 0 0 0 16 0 0 swwbuf 2624 0 0 0 0 8 0 0 swrbuf 2624 0 0 0 0 16 0 0 umtx_shm 88 0 0 0 0 126 0 0 umtx pi 96 0 0 0 0 126 0 0 rangeset pctrie nodes 144 0 0 0 0 62 0 0 malloc-65536 65536 0 0 0 0 1 0 0 malloc-65536 65536 0 0 0 0 1 0 0 malloc-65536 65536 0 0 0 0 1 0 0 malloc-32768 32768 0 0 0 0 1 0 0 malloc-32768 32768 0 0 0 0 1 0 0 malloc-256 256 0 0 0 0 62 0 0 malloc-256 256 0 0 0 0 62 0 0 malloc-256 256 0 0 0 0 62 0 0 malloc-256 256 0 0 0 0 62 0 0 malloc-256 256 0 0 0 0 62 0 0 malloc-256 256 0 0 0 0 62 0 0 malloc-256 256 0 0 0 0 62 0 0 malloc-256 256 0 0 0 0 62 0 0 malloc-128 128 0 0 0 0 126 0 0 malloc-128 128 0 0 0 0 126 0 0 malloc-128 128 0 0 0 0 126 0 0 malloc-128 128 0 0 0 0 126 0 0 malloc-128 128 0 0 0 0 126 0 0 malloc-128 128 0 0 0 0 126 0 0 malloc-128 128 0 0 0 0 126 0 0 malloc-128 128 0 0 0 0 126 0 0 malloc-64 64 0 0 0 0 254 0 0 malloc-64 64 0 0 0 0 254 0 0 malloc-64 64 0 0 0 0 254 0 0 malloc-64 64 0 0 0 0 254 0 0 malloc-64 64 0 0 0 0 254 0 0 malloc-64 64 0 0 0 0 254 0 0 malloc-64 64 0 0 0 0 254 0 0 malloc-64 64 0 0 0 0 254 0 0 malloc-32 32 0 0 0 0 254 0 0 malloc-32 32 0 0 0 0 254 0 0 malloc-32 32 0 0 0 0 254 0 0 malloc-32 32 0 0 0 0 254 0 0 malloc-32 32 0 0 0 0 254 0 0 malloc-32 32 0 0 0 0 254 0 0 malloc-32 32 0 0 0 0 254 0 0 malloc-32 32 0 0 0 0 254 0 0 malloc-16 16 0 0 0 0 254 0 0 malloc-16 16 0 0 0 0 254 0 0 malloc-16 16 0 0 0 0 254 0 0 malloc-16 16 0 0 0 0 254 0 0 malloc-16 16 0 0 0 0 254 0 0 malloc-16 16 0 0 0 0 254 0 0 malloc-16 16 0 0 0 0 254 0 0 malloc-16 16 0 0 0 0 254 0 0 pcpu-32 32 0 0 0 0 254 0 0 pcpu-4 4 0 0 0 0 254 0 0 fakepg 104 0 0 0 0 126 0 0 UMA Hash 256 0 0 0 0 62 0 0