panic: mtx_lock() of spin mutex (null) @ /syzkaller/managers/i386/kernel/sys/modules/tcp/rack/../../../netinet/tcp_stacks/rack.c:16472 cpuid = 0 time = 1624459906 KDB: stack backtrace: db_trace_self_wrapper() at db_trace_self_wrapper+0x47/frame 0xfffffe0088779560 vpanic() at vpanic+0x1cc/frame 0xfffffe00887795c0 panic() at panic+0x43/frame 0xfffffe0088779620 __mtx_lock_flags() at __mtx_lock_flags+0x1d7/frame 0xfffffe0088779680 rack_output() at rack_output+0x1ed7/frame 0xfffffe00887799c0 tcp_hpts_thread() at tcp_hpts_thread+0xe50/frame 0xfffffe0088779b10 ithread_loop() at ithread_loop+0x33f/frame 0xfffffe0088779bb0 fork_exit() at fork_exit+0xb3/frame 0xfffffe0088779bf0 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe0088779bf0 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- KDB: enter: panic [ thread pid 12 tid 100085 ] Stopped at kdb_enter+0x67: movq $0,0x16409ae(%rip) db> db> set $lines = 0 db> set $maxwidth = 0 db> show registers cs 0x20 ds 0x3b es 0x3b fs 0x13 gs 0x1b ss 0 rax 0x12 rcx 0xffffffff81131ec0 vprintf+0x140 rdx 0x1 rbx 0 rsp 0xfffffe0088779540 rbp 0xfffffe0088779560 rsi 0 rdi 0xffffffff81131ef6 vprintf+0x176 r8 0 r9 0x8080808080808080 r10 0xfffffe0088779430 r11 0x1ff77fff59c r12 0xffffffff82267d30 ddb_dbbe r13 0xffffffff81a6d600 r14 0xffffffff81a73f85 r15 0xffffffff81a73f85 rip 0xffffffff81128f57 kdb_enter+0x67 rflags 0x86 kdb_enter+0x67: movq $0,0x16409ae(%rip) db> show proc Process 12 (intr) at 0xfffff80004c74a60: state: NORMAL uid: 0 gids: 0 parent: pid 0 at 0xffffffff8271ce70 ABI: null flag: 0x10000284 flag2: 0 reaper: 0xffffffff8271ce70 reapsubtree: 12 sigparent: 20 vmspace: 0xffffffff8271dad0 (map 0xffffffff8271dad0) (map.pmap 0xffffffff8271db90) (pmap 0xffffffff8271dbf0) threads: 23 100011 I [swi5: fast taskq] 100014 I [swi6: task queue] 100016 I [swi6: Giant taskq] 100030 I [swi3: vm] 100031 I [swi4: clock (0)] 100032 I [swi4: clock (1)] 100033 I [swi1: netisr 0] 100046 I [irq24: virtio_pci0] 100047 I [irq25: virtio_pci0] 100048 I [irq26: virtio_pci0] 100049 I [irq27: virtio_pci0] 100050 I [irq28: virtio_pci1] 100051 I [irq29: virtio_pci1] 100052 I [irq30: virtio_pci1] 100053 I [irq31: virtio_pci1] 100054 I [irq32: virtio_pci1] 100059 I [irq10: virtio_pci2] 100061 I [irq1: atkbd0] 100062 I [irq12: psm0] 100063 I [swi0: uart uart++] 100071 I [swi1: pf send] 100084 Run CPU 1 [swi1: hpts] 100085 Run CPU 0 [swi1: hpts] db> ps pid ppid pgrp uid state wmesg wchan cmd 35448 788 788 0 R (threaded) syz-executor.1 136345 RunQ syz-executor.1 138461 RunQ syz-executor.1 35441 811 811 0 S (threaded) syz-executor.3 138233 S nanslp 0xffffffff8273d0a0 syz-executor.3 138451 S connec 0xfffff80038d4fbf0 syz-executor.3 138456 S uwait 0xfffff800383c5b00 syz-executor.3 33020 1 787 0 S uwait 0xfffff80038aef100 syz-executor.0 33018 1 787 0 S uwait 0xfffff80038aef700 syz-executor.0 32705 1 811 0 SV uwait 0xfffff80044f7df00 syz-executor.3 32701 1 811 0 SV sigwait 0xfffffe00952f6000 syz-executor.3 30806 1 811 0 S uwait 0xfffff80044c2dd00 syz-executor.3 30797 1 788 0 S uwait 0xfffff800443dd500 syz-executor.1 30795 1 788 0 S uwait 0xfffff80038f38900 syz-executor.1 30667 1 788 0 S uwait 0xfffff80015e5cd00 syz-executor.1 30660 1 788 0 S uwait 0xfffff80038a87f00 syz-executor.1 30253 1 788 0 SV uwait 0xfffff80038d13b00 syz-executor.1 30249 1 788 0 SV uwait 0xfffff80044c2d100 syz-executor.1 28438 1 789 0 SV uwait 0xfffff80038eee300 syz-executor.2 28435 1 789 0 SV uwait 0xfffff80044c2d500 syz-executor.2 27142 1 787 0 S uwait 0xfffff80044c2db00 syz-executor.0 27139 1 787 0 S uwait 0xfffff80038ed8300 syz-executor.0 25629 1 787 0 S uwait 0xfffff80038a87900 syz-executor.0 25627 1 787 0 S uwait 0xfffff80038d13100 syz-executor.0 24232 24228 811 0 SV uwait 0xfffff80044c2d900 syz-executor.3 24228 1 811 0 DV ppwait 0xfffff80038f3fa10 syz-executor.3 24131 24128 788 0 SV uwait 0xfffff80038cb1f00 syz-executor.1 24128 1 788 0 DV ppwait 0xfffff80044fe14e0 syz-executor.1 24104 1 788 0 S uwait 0xfffff80038f38d00 syz-executor.1 24103 1 788 0 S uwait 0xfffff80015e5c900 syz-executor.1 22916 1 811 0 S uwait 0xfffff80044c2df00 syz-executor.3 22909 1 811 0 S uwait 0xfffff80038f38f00 syz-executor.3 18718 1 789 0 S uwait 0xfffff80038a87500 syz-executor.2 18709 1 789 0 S uwait 0xfffff80038a87d00 syz-executor.2 18698 1 789 0 S uwait 0xfffff80038d13f00 syz-executor.2 18655 1 789 0 S uwait 0xfffff80038ed8b00 syz-executor.2 18647 1 789 0 S uwait 0xfffff80038f38700 syz-executor.2 16973 1 16973 65 Ss select 0xfffff800386caa40 dhclient 15263 1 788 0 S uwait 0xfffff80038d13500 syz-executor.1 15262 1 788 0 S uwait 0xfffff80038a8b300 syz-executor.1 15071 1 15071 0 Ss select 0xfffff80044c07d40 dhclient 15068 1 15068 0 Ss select 0xfffff80044101ec0 dhclient 15049 1 15049 65 Ss select 0xfffff80044c07ec0 dhclient 10000 1 10000 0 Ss select 0xfffff80015f79ec0 dhclient 9997 1 9997 0 Ss select 0xfffff800440f6ec0 dhclient 9978 1 9978 65 Ss select 0xfffff800386672c0 dhclient 6881 1 6881 0 Ss select 0xfffff80038e7da40 dhclient 6878 1 6878 0 Ss select 0xfffff80038eb45c0 dhclient 6853 1 6853 65 Ss select 0xfffff80038eb4a40 dhclient 3148 0 0 0 DL aiordy 0xfffff80038bc0a60 [aiod4] 3147 0 0 0 DL aiordy 0xfffff800384cb530 [aiod3] 3146 0 0 0 DL aiordy 0xfffff80038e26530 [aiod2] 3145 0 0 0 DL aiordy 0xfffff80038ca0530 [aiod1] 1935 1 1935 0 Ss select 0xfffff80038eb4440 dhclient 1932 1 1932 0 Ss select 0xfffff80038f25bc0 dhclient 811 780 811 0 Ss nanslp 0xffffffff8273d0a0 syz-executor.3 789 780 789 0 Ss piperd 0xfffff80038a745d0 syz-executor.2 788 780 788 0 Ss nanslp 0xffffffff8273d0a0 syz-executor.1 787 780 787 0 Ss piperd 0xfffff800384b9ba0 syz-executor.0 780 778 778 0 S (threaded) syz-fuzzer 100110 S uwait 0xfffff8003836b500 syz-fuzzer 100117 S nanslp 0xffffffff8273d0a1 syz-fuzzer 100118 S uwait 0xfffff80015af8900 syz-fuzzer 100119 S uwait 0xfffff800383c5d00 syz-fuzzer 100120 S uwait 0xfffff80038a87100 syz-fuzzer 100121 S uwait 0xfffff800383c5f00 syz-fuzzer 100122 S uwait 0xfffff8003836b100 syz-fuzzer 100123 S uwait 0xfffff80038a8bb00 syz-fuzzer 100194 S uwait 0xfffff80015d04100 syz-fuzzer 135979 S kqread 0xfffff80038a8b900 syz-fuzzer 778 776 778 0 Ss pause 0xfffff80015fafb10 csh 776 694 776 0 Ss select 0xfffff800383e2440 sshd 760 1 760 0 Ss+ ttyin 0xfffff8001540dcb0 getty 759 1 759 0 Ss+ ttyin 0xfffff80015b654b0 getty 758 1 758 0 Ss+ ttyin 0xfffff80015b65cb0 getty 757 1 757 0 Ss+ ttyin 0xfffff80015b694b0 getty 756 1 756 0 Ss+ ttyin 0xfffff80015b69cb0 getty 755 1 755 0 Ss+ ttyin 0xfffff80015b6c4b0 getty 754 1 754 0 Ss+ ttyin 0xfffff80015b6ccb0 getty 753 1 753 0 Ss+ ttyin 0xfffff80015b6f4b0 getty 752 1 752 0 Ss+ ttyin 0xfffff80015b6fcb0 getty 698 1 698 0 Ss nanslp 0xffffffff8273d0a1 cron 694 1 694 0 Ss select 0xfffff80015f61740 sshd 507 1 507 0 Ss select 0xfffff80015f61d40 syslogd 436 1 436 0 Ss select 0xfffff80015f69ec0 devd 435 1 435 65 Ss select 0xfffff80038460bc0 dhclient 350 1 350 0 Ss select 0xfffff80015f615c0 dhclient 347 1 347 0 Ss select 0xfffff80015e11440 dhclient 23 0 0 0 DL vlruwt 0xfffff80015d06530 [vnlru] 22 0 0 0 DL syncer 0xffffffff8282c450 [syncer] 21 0 0 0 DL (threaded) [bufdaemon] 100081 D qsleep 0xffffffff8282b520 [bufdaemon] 100088 D - 0xffffffff8220aec0 [bufspacedaemon-0] 100099 D sdflush 0xfffff80004de04e8 [/ worker] 20 0 0 0 DL psleep 0xffffffff82853388 [vmdaemon] 19 0 0 0 DL (threaded) [pagedaemon] 100079 D psleep 0xffffffff828477f8 [dom0] 100086 D launds 0xffffffff82847804 [laundry: dom0] 100087 D umarcl 0xffffffff815c4960 [uma] 18 0 0 0 DL - 0xffffffff82571458 [rand_harvestq] 17 0 0 0 DL waiting 0xffffffff830dd8b8 [sctp_iterator] 16 0 0 0 DL pftm 0xffffffff82dfe410 [pf purge] 15 0 0 0 DL - 0xffffffff82828bdc [soaiod4] 9 0 0 0 DL - 0xffffffff82828bdc [soaiod3] 8 0 0 0 DL - 0xffffffff82828bdc [soaiod2] 7 0 0 0 DL - 0xffffffff82828bdc [soaiod1] 6 0 0 0 DL (threaded) [cam] 100044 D - 0xffffffff82448b40 [doneq0] 100045 D - 0xffffffff82448ac0 [async] 100078 D - 0xffffffff82448990 [scanner] 14 0 0 0 DL seqstat 0xfffff80004d8d488 [sequencer 00] 5 0 0 0 DL crypto_ 0xfffff80004d55d80 [crypto returns 1] 4 0 0 0 DL crypto_ 0xfffff80004d55d30 [crypto returns 0] 3 0 0 0 DL crypto_ 0xffffffff82844ce0 [crypto] 13 0 0 0 DL (threaded) [geom] 100035 D - 0xffffffff8271c8e0 [g_event] 100036 D - 0xffffffff8271c8e8 [g_up] 100037 D - 0xffffffff8271c8f0 [g_down] 2 0 0 0 DL (threaded) [KTLS] 100028 D - 0xfffff80004c69100 [thr_0] 100029 D - 0xfffff80004c69180 [thr_1] 12 0 0 0 RL (threaded) [intr] 100011 I [swi5: fast taskq] 100014 I [swi6: task queue] 100016 I [swi6: Giant taskq] 100030 I [swi3: vm] 100031 I [swi4: clock (0)] 100032 I [swi4: clock (1)] 100033 I [swi1: netisr 0] 100046 I [irq24: virtio_pci0] 100047 I [irq25: virtio_pci0] 100048 I [irq26: virtio_pci0] 100049 I [irq27: virtio_pci0] 100050 I [irq28: virtio_pci1] 100051 I [irq29: virtio_pci1] 100052 I [irq30: virtio_pci1] 100053 I [irq31: virtio_pci1] 100054 I [irq32: virtio_pci1] 100059 I [irq10: virtio_pci2] 100061 I [irq1: atkbd0] 100062 I [irq12: psm0] 100063 I [swi0: uart uart++] 100071 I [swi1: pf send] 100084 Run CPU 1 [swi1: hpts] 100085 Run CPU 0 [swi1: hpts] 11 0 0 0 RL (threaded) [idle] 100003 CanRun [idle: cpu0] 100004 CanRun [idle: cpu1] 1 0 1 0 SLs wait 0xfffff80004bda530 [init] 10 0 0 0 DL audit_w 0xffffffff828451f0 [audit] 0 0 0 0 DLs (threaded) [kernel] 100000 D swapin 0xffffffff8271ce70 [swapper] 100005 D - 0xfffff8000443b500 [if_io_tqg_0] 100006 D - 0xfffff8000443b100 [if_io_tqg_1] 100007 D - 0xfffff80004c78d00 [if_config_tqg_0] 100008 D - 0xfffff80004c78900 [softirq_0] 100009 D - 0xfffff80004c78500 [softirq_1] 100010 D - 0xfffff80004c78100 [aiod_kick taskq] 100012 D - 0xfffff80004c75900 [kqueue_ctx taskq] 100013 D - 0xfffff80004c75500 [pci_hp taskq] 100015 D - 0xfffff80004c6ed00 [inm_free taskq] 100017 D - 0xfffff80004c6e500 [linuxkpi_irq_wq] 100018 D - 0xfffff80004c6e100 [thread taskq] 100019 D - 0xfffff80004c69d00 [in6m_free taskq] 100020 D - 0xfffff80004c69900 [linuxkpi_short_wq_0] 100021 D - 0xfffff80004c69900 [linuxkpi_short_wq_1] 100022 D - 0xfffff80004c69900 [linuxkpi_short_wq_2] 100023 D - 0xfffff80004c69900 [linuxkpi_short_wq_3] 100024 D - 0xfffff80004c69500 [linuxkpi_long_wq_0] 100025 D - 0xfffff80004c69500 [linuxkpi_long_wq_1] 100026 D - 0xfffff80004c69500 [linuxkpi_long_wq_2] 100027 D - 0xfffff80004c69500 [linuxkpi_long_wq_3] 100034 D - 0xfffff80004cbed00 [firmware taskq] 100038 D - 0xfffff80004cbe900 [crypto_0] 100039 D - 0xfffff80004cbe900 [crypto_1] 100055 D - 0xfffff800153efd00 [vtnet0 rxq 0] 100056 D - 0xfffff800153ef900 [vtnet0 txq 0] 100057 D - 0xfffff800153ef500 [vtnet0 rxq 1] 100058 D - 0xfffff800153ef100 [vtnet0 txq 1] 100060 D vtbslp 0xfffff800154a6700 [virtio_balloon] 100064 D - 0xfffff800153d0900 [mca taskq] 100069 D - 0xffffffff81e20781 [deadlkres] 100074 D - 0xfffff80015aee100 [acpi_task_0] 100075 D - 0xfffff80015aee100 [acpi_task_1] 100076 D - 0xfffff80015aee100 [acpi_task_2] 100077 D - 0xfffff80004cbe500 [CAM taskq] db> show all locks Process 35448 (syz-executor.1) thread 0xfffffe009541c720 (136345) exclusive sleep mutex umtxql (umtxql) r = 0 (0xffffffff82762830) locked @ /syzkaller/managers/i386/kernel/sys/kern/kern_umtx.c:932 Process 12 (intr) thread 0xfffffe0055726ac0 (100084) exclusive rw tcpinp (tcpinp) r = 0 (0xfffff8005d40ab90) locked @ /syzkaller/managers/i386/kernel/sys/netinet/tcp_hpts.c:1536 Process 12 (intr) thread 0xfffffe00557263a0 (100085) exclusive rw tcpinp (tcpinp) r = 0 (0xfffff8005d4205d8) locked @ /syzkaller/managers/i386/kernel/sys/netinet/tcp_hpts.c:1536 db> show malloc Type InUse MemUse Requests sysctloid 34288 12858K 34355 pf_hash 5 11560K 5 devbuf 4216 6982K 4244 tcp_hpts 5 3219K 5 kobj 326 2608K 486 vtbuf 24 2064K 46 filedesc 78 1247K 68873 newblk 9 1036K 36513 vfscache 3 1035K 3 subproc 212 891K 35588 inodedep 304 823K 34589 pcb 329 777K 24846 acpica 1674 649K 54806 callout 2 528K 2 ufs_quota 1 520K 1 vfs_hash 1 520K 1 intr 4 480K 4 bus 986 377K 3469 BPF 46 337K 109 linker 348 270K 511 kdtrace 290 230K 73911 DEVFS1 107 214K 124 umtx 440 193K 440 module 510 192K 510 vnet_data 1 176K 1 tidhash 3 164K 3 dirrem 295 148K 34491 pagedep 9 140K 34428 tfo_ccache 1 136K 1 inpcbpolicy 321 121K 17854 sem 4 120K 4 osd 314 118K 16658 freefile 295 111K 34470 UMA 270 102K 270 audit_evclass 236 89K 294 tcp_fsb 227 86K 4927 mtx_pool 2 80K 2 temp 35 77K 11035 syncache 1 76K 1 routetbl 137 73K 817 vmem 3 72K 5 msg 4 68K 4 ifaddr 71 67K 73 acpitask 1 64K 1 ddb_capture 1 64K 1 DEVFS3 126 63K 136 ether_multi 157 59K 167 gtaskqueue 18 57K 18 shm 4 56K 320 DEVFS_RULE 56 54K 56 kenv 95 52K 95 eventhandler 133 50K 133 pf_rule 12 48K 238 kqueue 100 46K 35513 lltable 50 44K 234 ithread 99 43K 99 rman 84 42K 425 KTRACE 100 38K 100 DEVFSP 99 38K 30745 pwddesc 97 37K 35449 taskqueue 60 36K 60 proc 3 34K 3 ufs_mount 5 34K 6 bus-sc 33 34K 1684 devstat 4 33K 4 hostcache 1 32K 1 tty 16 32K 16 in6_multi 71 31K 71 pf_osfp 80 30K 80 GEOM 60 29K 489 kbdmux 6 28K 6 cred 26 26K 308 sctp_timw 51 26K 51 ifnet 7 25K 7 plimit 24 24K 496 kcovinfo 64 24K 68 CAM queue 5 21K 1528 selfd 55 21K 491652 pfs_nodes 20 20K 20 proc-args 52 20K 717 UART 12 18K 12 session 35 18K 61 ksem 2 17K 790 bmsafemap 2 17K 34609 rpc 2 16K 2 shmfd 1 16K 92 pfs_vncache 1 16K 1 xform 41 16K 56627 ufs_dirhash 24 14K 24 sglist 5 13K 5 lockf 33 13K 89 MCA 32 12K 32 CAM DEV 3 12K 510 Unitno 30 12K 51 vt 11 11K 11 crypto 24 11K 28477 acpisem 28 11K 28 CAM XPT 22 11K 543 sctp_atcl 10 10K 8008 uidinfo 3 9K 13 pf_ifnet 10 9K 83 sctp_stro 4 8K 2044 memdesc 1 8K 1 ipsec-saq 2 8K 2 evdev 4 8K 4 acpidev 20 8K 20 hhook 15 8K 17 select 19 8K 94 mount 16 7K 144 fpukern_ctx 3 6K 3 terminal 11 6K 11 sctp_ifa 14 6K 15 sctp_atky 14 6K 10078 ip6ndp 12 6K 15 ipsecpolicy 2 5K 2 encap_export_host 12 5K 12 clone 9 5K 9 local_apic 1 4K 1 io_apic 1 4K 1 sahead 1 4K 1 secasvar 1 4K 1 nhops 6 4K 8 sctp_athm 10 4K 8020 pci_link 10 4K 10 msi 9 4K 9 DEVFS 9 4K 10 tun 7 4K 7 sctp_map 8 3K 4086 ipsec 3 3K 3 mkdir 8 3K 68812 in_multi 6 3K 8 nexusdev 7 3K 7 feeder 7 3K 7 procdesc 5 3K 18 freework 5 3K 35054 sctp_ifn 6 3K 15 mld 6 3K 6 igmp 6 3K 6 toponodes 6 3K 6 prison 6 3K 6 isadev 6 3K 6 softdep 1 2K 1 freeblks 4 2K 34470 vnodemarker 2 2K 106 NFSD session 1 2K 1 linux 5 2K 6 CAM periph 4 2K 271 soname 5 2K 44296 filecaps 5 2K 120 newdirblk 4 2K 34406 diradd 4 2K 34532 pfil 4 2K 4 CAM path 4 2K 1034 tcpfunc 3 2K 3 loginclass 3 2K 6 chacha20random 1 1K 1 vnodes 1 1K 1 CAM SIM 2 1K 2 ktls 1 1K 1 cdev 2 1K 2 aesni_data 2 1K 2 cpus 2 1K 2 atkbddev 2 1K 2 CAM dev queue 2 1K 2 entropy 2 1K 48 NFSD lckfile 1 1K 1 NFSD V4client 1 1K 1 pmchooks 1 1K 1 sctp_vrf 1 1K 1 apmdev 1 1K 1 CAM I/O Scheduler 1 1K 1 vnet_data_free 1 1K 1 vnet 1 1K 1 Per-cpu 1 1K 1 p1003.1b 1 1K 1 acpiintr 1 1K 1 pmc 1 1K 1 sctp_mcore 0 0K 0 sctp_socko 0 0K 5445 sctp_iter 0 0K 14 sctp_mvrf 0 0K 0 sctp_cpal 0 0K 3 sctp_cmsg 0 0K 0 sctp_stre 0 0K 0 sctp_athi 0 0K 0 sctp_a_it 0 0K 11 sctp_aadr 0 0K 2 sctp_stri 0 0K 15 tcp_do 0 0K 0 htcp data 0 0K 0 chd data 0 0K 17 dctcp data 0 0K 56 pf_table 0 0K 96 pf_altq 0 0K 0 pf_temp 0 0K 0 cdg data 0 0K 20 cubic data 0 0K 15 mqdata 0 0K 0 vegas data 0 0K 20 NFSCL lckown 0 0K 0 NFSCL client 0 0K 0 madt_table 0 0K 2 smartpqi 0 0K 0 NFSCL deleg 0 0K 0 NFSCL open 0 0K 0 NFSCL owner 0 0K 0 NFS fh 0 0K 0 NFS req 0 0K 0 NFSD usrgroup 0 0K 0 NFSD string 0 0K 0 iavf 0 0K 0 ixl 0 0K 0 NFSD V4lock 0 0K 0 NFSD V4state 0 0K 0 NFSD srvcache 0 0K 0 msdosfs_fat 0 0K 0 msdosfs_mount 0 0K 0 msdosfs_node 0 0K 0 ice-resmgr 0 0K 0 ice-osdep 0 0K 0 ice 0 0K 0 axgbe 0 0K 0 DEVFS4 0 0K 0 DEVFS2 0 0K 0 gntdev 0 0K 0 privcmd_dev 0 0K 0 evtchn_dev 0 0K 0 xen_intr 0 0K 0 xenstore 0 0K 0 ciss_data 0 0K 0 xen_hvm 0 0K 0 legacydrv 0 0K 0 qpidrv 0 0K 0 BACKLIGHT 0 0K 0 xnb 0 0K 0 dmar_idpgtbl 0 0K 0 dmar_dom 0 0K 0 dmar_ctx 0 0K 0 xbbd 0 0K 0 xbd 0 0K 0 Balloon 0 0K 0 isci 0 0K 0 iommu_dmamap 0 0K 0 sysmouse 0 0K 0 hyperv_socket 0 0K 0 bxe_ilt 0 0K 0 xenbus 0 0K 0 vtfont 0 0K 0 vm_fictitious 0 0K 0 ath_hal 0 0K 0 athdev 0 0K 0 ata_pci 0 0K 0 ata_dma 0 0K 0 ata_generic 0 0K 0 amr 0 0K 0 pvscsi 0 0K 0 scsi_da 0 0K 69 ata_da 0 0K 0 scsi_ch 0 0K 0 UMAHash 0 0K 0 scsi_cd 0 0K 0 AHCI driver 0 0K 0 vm_pgdata 0 0K 0 jblocks 0 0K 0 savedino 0 0K 28268 sentinel 0 0K 0 jfsync 0 0K 0 jtrunc 0 0K 0 sbdep 0 0K 51 jsegdep 0 0K 0 jseg 0 0K 0 jfreefrag 0 0K 0 jfreeblk 0 0K 0 jnewblk 0 0K 0 jmvref 0 0K 0 jremref 0 0K 0 jaddref 0 0K 0 freedep 0 0K 0 freefrag 0 0K 6 allocindir 0 0K 0 indirdep 0 0K 851 allocdirect 0 0K 0 ufs_trim 0 0K 0 mactemp 0 0K 0 audit_trigger 0 0K 0 audit_pipe_presel 0 0K 0 audit_pipeent 0 0K 0 audit_pipe 0 0K 0 audit_evname 0 0K 0 audit_bsm 0 0K 0 audit_gidset 0 0K 0 audit_text 0 0K 0 audit_path 0 0K 0 audit_data 0 0K 0 audit_cred 0 0K 0 USBdev 0 0K 0 USB 0 0K 0 agp 0 0K 0 nvme_da 0 0K 0 acpipwr 0 0K 0 acpi_perf 0 0K 0 ktls_ocf 0 0K 0 twsbuf 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5E_TLS 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EN 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5DUMP 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 seq_file 0 0K 0 radix 0 0K 0 idr 0 0K 0 lkpifw 0 0K 0 NLM 0 0K 0 ipsec-spdcache 0 0K 0 ipsec-reg 0 0K 0 ipsec-misc 0 0K 0 ipsecrequest 0 0K 0 ip6opt 0 0K 347 ip6_msource 0 0K 0 ip6_moptions 0 0K 1 in6_mfilter 0 0K 0 frag6 0 0K 0 tcplog 0 0K 0 tcp_hwpace 0 0K 0 twe_commands 0 0K 0 LRO 0 0K 0 newreno data 0 0K 10 ip_msource 0 0K 0 ip_moptions 0 0K 12 in_mfilter 0 0K 0 ipid 0 0K 0 80211scan 0 0K 0 80211ratectl 0 0K 0 80211power 0 0K 0 80211nodeie 0 0K 0 80211node 0 0K 0 80211mesh_gt 0 0K 0 80211mesh_rt 0 0K 0 80211perr 0 0K 0 80211prep 0 0K 0 80211preq 0 0K 0 80211dfs 0 0K 0 80211crypto 0 0K 0 80211vap 0 0K 0 iflib 0 0K 0 vlan 0 0K 0 gif 0 0K 0 ifdescr 0 0K 0 zlib 0 0K 0 fadvise 0 0K 0 VN POLL 0 0K 0 twa_commands 0 0K 0 statfs 0 0K 34679 namei_tracker 0 0K 2 export_host 0 0K 0 cl_savebuf 0 0K 6 tcp_log_dev 0 0K 2616 midi buffers 0 0K 0 mixer 0 0K 0 ac97 0 0K 0 hdacc 0 0K 0 hdac 0 0K 0 hdaa 0 0K 0 acpicmbat 0 0K 0 SIIS driver 0 0K 0 CAM CCB 0 0K 5959 PUC 0 0K 0 ppbusdev 0 0K 0 agtiapi_MemAlloc malloc 0 0K 0 osti_cacheable 0 0K 0 tempbuff 0 0K 0 biobuf 0 0K 0 aios 0 0K 28 lio 0 0K 22 acl 0 0K 0 tempbuff 0 0K 0 mbuf_tag 0 0K 240 ag_tgt_map_t malloc 0 0K 0 ag_slr_map_t malloc 0 0K 0 lDevFlags * malloc 0 0K 0 tiDeviceHandle_t * malloc 0 0K 0 ag_portal_data_t malloc 0 0K 0 ag_device_t malloc 0 0K 0 STLock malloc 0 0K 0 CCB List 0 0K 0 sr_iov 0 0K 0 OCS 0 0K 0 OCS 0 0K 0 nvme 0 0K 0 nvd 0 0K 0 netmap 0 0K 0 mwldev 0 0K 0 MVS driver 0 0K 0 CAM ccb queue 0 0K 0 mrsasbuf 0 0K 0 mpt_user 0 0K 0 mps_user 0 0K 0 accf 0 0K 0 pts 0 0K 0 iov 0 0K 30967 ioctlops 0 0K 4215 eventfd 0 0K 0 Witness 0 0K 0 stack 0 0K 0 MPSSAS 0 0K 0 mps 0 0K 0 mpr_user 0 0K 0 MPRSAS 0 0K 0 mpr 0 0K 0 mfibuf 0 0K 0 sbuf 0 0K 288 md_sectors 0 0K 0 firmware 0 0K 0 compressor 0 0K 0 md_disk 0 0K 0 SWAP 0 0K 0 malodev 0 0K 0 LED 0 0K 0 sysctltmp 0 0K 822 sysctl 0 0K 3 ekcd 0 0K 0 dumper 0 0K 0 sendfile 0 0K 0 rctl 0 0K 0 ix_sriov 0 0K 0 aacraidcam 0 0K 0 aacraid_buf 0 0K 0 ix 0 0K 0 ipsbuf 0 0K 0 cache 0 0K 0 iirbuf 0 0K 0 prison_racct 0 0K 0 Fail Points 0 0K 0 sigio 0 0K 6 filedesc_to_leader 0 0K 0 pwd 0 0K 0 tty console 0 0K 0 aaccam 0 0K 0 aacbuf 0 0K 0 zstd 0 0K 0 XZ_DEC 0 0K 0 nvlist 0 0K 0 SCSI ENC 0 0K 0 SCSI sa 0 0K 0 scsi_pass 0 0K 0 isofs_node 0 0K 0 isofs_mount 0 0K 0 tr_raid5_data 0 0K 0 tr_raid1e_data 0 0K 0 tr_raid1_data 0 0K 0 tr_raid0_data 0 0K 0 tr_concat_data 0 0K 0 md_sii_data 0 0K 0 md_promise_data 0 0K 0 md_nvidia_data 0 0K 0 md_jmicron_data 0 0K 0 md_intel_data 0 0K 0 md_ddf_data 0 0K 0 raid_data 0 0K 72 geom_flashmap 0 0K 0 tmpfs dir 0 0K 0 tmpfs name 0 0K 0 tmpfs mount 0 0K 0 NFS FHA 0 0K 0 newnfsmnt 0 0K 0 newnfsclient_req 0 0K 0 NFSCL layrecall 0 0K 0 NFSCL session 0 0K 0 NFSCL sockreq 0 0K 0 NFSCL devinfo 0 0K 0 NFSCL flayout 0 0K 0 NFSCL layout 0 0K 0 NFSD rollback 0 0K 0 NFSCL diroff 0 0K 0 NEWdirectio 0 0K 0 NEWNFSnode 0 0K 0 NFSCL lck 0 0K 0 db> show uma Zone Size Used Free Requests Sleeps Bucket Total Mem XFree mbuf_jumbo_page 4096 8322 949 1120590 0 254 37974016 0 malloc-384 384 35501 99 43226 0 30 13670400 0 malloc-1024 1024 4174 14 4256 0 16 4288512 0 malloc-1024 1024 133 3399 29883 0 16 3616768 0 malloc-1024 1024 389 3095 37175 0 16 3567616 0 tcp_log 416 1298 5236 96964 0 254 2718144 0 malloc-8192 8192 328 2 1099 0 1 2703360 0 pbuf 2624 0 973 0 0 2 2553152 0 mbuf 256 8884 656 4563249 0 254 2442240 0 RADIX NODE 144 13596 401 859322 0 64 2015568 0 BUF TRIE 144 194 13274 2991 0 62 1939392 0 malloc-512 512 433 3103 35272 0 30 1810432 0 UMA Slabs 0 112 14254 35 14254 0 126 1600368 0 malloc-32768 32768 41 6 34437 0 1 1540096 0 malloc-384 384 888 3062 174992 0 30 1516800 0 mbuf_cluster 2048 585 1 585 0 254 1200128 0 malloc-8192 8192 99 6 35451 0 1 860160 0 tcp_bbr_map 128 1078 4285 100152 0 126 686464 0 malloc-384 384 1728 32 53929 0 30 675840 0 FFS inode 1160 537 30 35008 0 8 657720 0 malloc-65536 65536 3 6 794 0 1 589824 0 malloc-384 384 1442 58 565450 0 30 576000 0 VM OBJECT 264 1670 85 501583 0 30 463320 0 malloc-16384 16384 24 1 27 0 1 409600 0 socket 944 350 74 25295 0 254 400256 0 THREAD 1808 192 28 38461 0 8 397760 0 ertt_txseginfo 40 1078 8719 426236 0 254 391880 0 lkpimm 160 1 2324 1 0 62 372000 0 lkpicurr 160 2 2323 2 0 62 372000 0 tcpcb 1080 310 33 16646 0 254 370440 0 malloc-384 384 936 24 32127 0 30 368640 0 tcp_inpcb 488 311 433 16646 0 254 363072 0 MAP ENTRY 96 3053 391 1908699 0 126 330624 0 256 Bucket 2048 141 19 18473 0 8 327680 0 malloc-512 512 31 577 37347 0 30 311296 0 malloc-8192 8192 8 29 5961 0 1 303104 0 VNODE 448 575 91 35048 0 30 298368 0 malloc-384 384 686 24 152238 0 30 272640 0 malloc-1024 1024 208 52 44792 0 16 266240 0 malloc-65536 65536 4 0 4 0 1 262144 0 malloc-65536 65536 2 2 38 0 1 262144 0 malloc-2048 2048 125 3 150 0 8 262144 0 DEVCTL 1024 0 216 132 0 0 221184 0 malloc-16384 16384 11 2 247 0 1 212992 0 malloc-8192 8192 23 3 85 0 1 212992 0 tcp_rack_pcb 832 227 16 4927 0 16 202176 0 malloc-4096 4096 7 42 7016 0 2 200704 0 VMSPACE 2544 65 10 35418 0 4 190800 0 UMA Zones 768 242 2 242 0 16 187392 0 sctp_asoc 2288 4 76 2043 0 254 183040 0 malloc-384 384 387 73 93262 0 30 176640 0 vmem btag 56 2903 37 2903 0 254 164640 0 malloc-384 384 399 11 45031 0 30 157440 0 PROC 1328 96 18 35448 0 8 151392 0 tcp_bbr_pcb 832 3 177 9188 0 16 149760 0 mbuf_packet 256 66 519 40464 0 254 149760 0 malloc-512 512 197 91 73331 0 30 147456 0 S VFS Cache 104 1053 351 37631 0 126 146016 0 FFS2 dinode 256 537 33 35007 0 62 145920 0 sctp_ep 1280 6 99 5955 0 254 134400 0 malloc-65536 65536 0 2 176 0 1 131072 0 malloc-1024 1024 127 1 156 0 16 131072 0 malloc-512 512 239 17 34958 0 30 131072 0 filedesc0 1072 97 22 35449 0 8 127568 0 ksiginfo 112 99 945 3969 0 126 116928 0 32 Bucket 256 311 139 28819 0 62 115200 0 malloc-16384 16384 2 5 36 0 1 114688 0 128 Bucket 1024 72 39 6243 0 16 113664 0 malloc-32768 32768 1 2 121 0 1 98304 0 malloc-16384 16384 4 2 6 0 1 98304 0 malloc-1024 1024 15 73 12700 0 16 90112 0 UMA Kegs 384 227 6 227 0 30 89472 0 16 Bucket 144 479 137 7233 0 62 88704 0 malloc-4096 4096 19 2 145 0 2 86016 0 malloc-2048 2048 6 36 3373 0 8 86016 0 malloc-512 512 120 48 35586 0 30 86016 0 clpbuf 2624 0 32 29 0 16 83968 0 g_bio 408 0 200 21250 0 30 81600 0 malloc-4096 4096 12 7 1882 0 2 77824 0 malloc-8192 8192 6 3 325 0 1 73728 0 malloc-65536 65536 0 1 1 0 1 65536 0 malloc-65536 65536 0 1 1 0 1 65536 0 malloc-32768 32768 1 1 16 0 1 65536 0 malloc-16384 16384 2 2 7 0 1 65536 0 sctp_raddr 736 4 84 2081 0 254 64768 0 malloc-2048 2048 10 20 520 0 8 61440 0 malloc-2048 2048 16 14 273 0 8 61440 0 malloc-8192 8192 2 5 35156 0 1 57344 0 tcp_rack_map 112 454 50 8706 0 126 56448 0 malloc-512 512 44 60 14211 0 30 53248 0 malloc-16384 16384 3 0 3 0 1 49152 0 malloc-16384 16384 3 0 3 0 1 49152 0 pcpu-8 8 4801 831 7686 0 254 45056 0 64 Bucket 512 45 43 18169 0 30 45056 0 Files 80 281 269 111505 0 126 44000 0 malloc-512 512 65 15 519 0 30 40960 0 malloc-512 512 25 55 1478 0 30 40960 0 pipe 744 23 27 1308 0 16 37200 0 DIRHASH 1024 34 2 34 0 16 36864 0 NAMEI 1024 0 36 194523 0 16 36864 0 malloc-4096 4096 6 3 29 0 2 36864 0 malloc-2048 2048 2 16 157 0 8 36864 0 TURNSTILE 136 221 31 221 0 62 34272 0 malloc-32768 32768 1 0 1 0 1 32768 0 malloc-32768 32768 1 0 1 0 1 32768 0 malloc-32768 32768 1 0 1 0 1 32768 0 pcpu-64 64 487 25 487 0 254 32768 0 ertt 72 310 82 16646 0 126 28224 0 KNOTE 160 28 147 431967 0 62 28000 0 malloc-8192 8192 0 3 13 0 1 24576 0 malloc-8192 8192 1 2 60 0 1 24576 0 malloc-4096 4096 1 5 10 0 2 24576 0 malloc-4096 4096 3 3 166 0 2 24576 0 ttyinq 160 135 15 300 0 62 24000 0 8 Bucket 80 51 249 6071 0 126 24000 0 ttyoutq 256 72 18 160 0 62 23040 0 SLEEPQUEUE 88 221 35 221 0 126 22528 0 PWD 32 38 592 34494 0 254 20160 0 2 Bucket 32 122 508 6724 0 254 20160 0 Mountpoints 2752 2 5 2 0 4 19264 0 malloc-16384 16384 1 0 1 0 1 16384 0 malloc-4096 4096 2 2 2549 0 2 16384 0 vtnet_tx_hdr 24 0 668 817292 0 254 16032 0 ripcb 488 5 27 501 0 254 15616 0 udp_inpcb 488 6 26 452 0 254 15616 0 unpcb 256 20 40 1438 0 254 15360 0 malloc-2048 2048 3 3 163 0 8 12288 0 malloc-2048 2048 2 4 6 0 8 12288 0 sctp_chunk 152 2 76 78 0 254 11856 0 udplite_inpcb 488 0 24 255 0 254 11712 0 kenv 258 15 30 1042 0 30 11610 0 malloc-4096 4096 1 1 3 0 2 8192 0 malloc-2048 2048 4 0 4 0 8 8192 0 malloc-1024 1024 1 7 11 0 16 8192 0 malloc-1024 1024 0 8 36 0 16 8192 0 rtentry 176 31 15 35 0 62 8096 0 tcptw 88 0 92 4 0 254 8096 0 PGRP 88 35 57 74 0 126 8096 0 rl_entry 40 91 111 91 0 254 8080 0 sctp_stream_msg_out 112 2 70 273 0 254 8064 0 sctp_laddr 48 4 164 188 0 254 8064 0 udpcb 32 6 246 707 0 254 8064 0 AIOP 32 4 248 4 0 254 8064 0 4 Bucket 48 7 161 742 0 254 8064 0 sctp_readq 152 0 52 10 0 254 7904 0 AIO 208 3 35 144 0 62 7904 0 cryptop 280 0 28 298 0 30 7840 0 itimer 352 0 22 34 0 30 7744 0 AIOCB 552 0 14 214 0 16 7728 0 routing nhops 256 27 3 35 0 62 7680 0 AIOLIO 272 0 28 11 0 30 7616 0 FPU_save_area 832 1 8 1 0 16 7488 0 cpuset 104 7 55 15 0 126 6448 0 tfo_ccache_entries 80 2 62 2 0 126 5120 0 epoch_record pcpu 256 4 12 4 0 62 4096 0 pcpu-16 16 7 249 7 0 254 4096 0 hostcache 64 1 62 1 0 254 4032 0 syncache 168 0 24 4 0 254 4032 0 UMA Slabs 1 176 10 12 10 0 62 3872 0 mqnode 416 3 6 3 0 30 3744 0 KMAP ENTRY 96 12 27 12 0 0 3744 0 vmem 1856 1 1 1 0 8 3712 0 domainset 40 0 63 6 0 254 2520 0 SMR CPU 32 3 60 3 0 254 2016 0 SMR SHARED 24 3 60 3 0 254 1512 0 FFS1 dinode 128 0 0 0 0 126 0 0 da_ccb 544 0 0 0 0 16 0 0 ada_ccb 272 0 0 0 0 30 0 0 swblk 136 0 0 0 0 62 0 0 swpctrie 144 0 0 0 0 62 0 0 sctp_asconf_ack 48 0 0 0 0 254 0 0 sctp_asconf 40 0 0 0 0 254 0 0 cdg_qdiffsample 16 0 0 0 0 254 0 0 pf state scrubs 40 0 0 0 0 254 0 0 pf frag entries 40 0 0 0 0 254 0 0 pf frags 248 0 0 0 0 62 0 0 pf table entries 160 0 0 0 0 0 0 0 pf table entry counters 64 0 0 0 0 254 0 0 pf source nodes 136 0 0 0 0 254 0 0 pf state keys 88 0 0 0 0 126 0 0 pf states 304 0 0 0 0 0 0 0 pf tags 104 0 0 0 0 126 0 0 pf mtags 48 0 0 0 0 254 0 0 tcp_log_node 120 0 0 0 0 126 0 0 tcp_log_bucket 176 0 0 0 0 62 0 0 tcpreass 48 0 0 0 0 254 0 0 tfo 4 0 0 0 0 254 0 0 sackhole 32 0 0 0 0 254 0 0 ipq 56 0 0 0 0 254 0 0 IPsec SA lft_c 16 0 0 0 0 254 0 0 NCLNODE 584 0 0 0 0 16 0 0 TMPFS node 224 0 0 0 0 62 0 0 mqnotifier 216 0 0 0 0 62 0 0 mvdata 64 0 0 0 0 254 0 0 mqueue 248 0 0 0 0 62 0 0 LTS VFS Cache 360 0 0 0 0 30 0 0 L VFS Cache 320 0 0 0 0 30 0 0 STS VFS Cache 144 0 0 0 0 62 0 0 linux_dma_object 24 0 0 0 0 254 0 0 linux_dma_pctrie 144 0 0 0 0 62 0 0 IOMMU_MAP_ENTRY 120 0 0 0 0 126 0 0 ktls_session 128 0 0 0 0 126 0 0 mbuf_jumbo_16k 16384 0 0 0 0 254 0 0 mbuf_jumbo_9k 9216 0 0 0 0 254 0 0 audit_record 1280 0 0 0 0 8 0 0 MAC labels 40 0 0 0 0 254 0 0 vnpbuf 2624 0 0 0 0 64 0 0 mdpbuf 2624 0 0 0 0 3 0 0 nfspbuf 2624 0 0 0 0 16 0 0 swwbuf 2624 0 0 0 0 8 0 0 swrbuf 2624 0 0 0 0 16 0 0 umtx_shm 88 0 0 0 0 126 0 0 umtx pi 96 0 0 0 0 126 0 0 rangeset pctrie nodes 144 0 0 0 0 62 0 0 malloc-65536 65536 0 0 0 0 1 0 0 malloc-65536 65536 0 0 0 0 1 0 0 malloc-32768 32768 0 0 0 0 1 0 0 malloc-32768 32768 0 0 0 0 1 0 0 malloc-256 256 0 0 0 0 62 0 0 malloc-256 256 0 0 0 0 62 0 0 malloc-256 256 0 0 0 0 62 0 0 malloc-256 256 0 0 0 0 62 0 0 malloc-256 256 0 0 0 0 62 0 0 malloc-256 256 0 0 0 0 62 0 0 malloc-256 256 0 0 0 0 62 0 0 malloc-256 256 0 0 0 0 62 0 0 malloc-128 128 0 0 0 0 126 0 0 malloc-128 128 0 0 0 0 126 0 0 malloc-128 128 0 0 0 0 126 0 0 malloc-128 128 0 0 0 0 126 0 0 malloc-128 128 0 0 0 0 126 0 0 malloc-128 128 0 0 0 0 126 0 0 malloc-128 128 0 0 0 0 126 0 0 malloc-128 128 0 0 0 0 126 0 0 malloc-64 64 0 0 0 0 254 0 0 malloc-64 64 0 0 0 0 254 0 0 malloc-64 64 0 0 0 0 254 0 0 malloc-64 64 0 0 0 0 254 0 0 malloc-64 64 0 0 0 0 254 0 0 malloc-64 64 0 0 0 0 254 0 0 malloc-64 64 0 0 0 0 254 0 0 malloc-64 64 0 0 0 0 254 0 0 malloc-32 32 0 0 0 0 254 0 0 malloc-32 32 0 0 0 0 254 0 0 malloc-32 32 0 0 0 0 254 0 0 malloc-32 32 0 0 0 0 254 0 0 malloc-32 32 0 0 0 0 254 0 0 malloc-32 32 0 0 0 0 254 0 0 malloc-32 32 0 0 0 0 254 0 0 malloc-32 32 0 0 0 0 254 0 0 malloc-16 16 0 0 0 0 254 0 0 malloc-16 16 0 0 0 0 254 0 0 malloc-16 16 0 0 0 0 254 0 0 malloc-16 16 0 0 0 0 254 0 0 malloc-16 16 0 0 0 0 254 0 0 malloc-16 16 0 0 0 0 254 0 0 malloc-16 16 0 0 0 0 254 0 0 malloc-16 16 0 0 0 0 254 0 0 pcpu-32 32 0 0 0 0 254 0 0 pcpu-4 4 0 0 0 0 254 0 0 fakepg 104 0 0 0 0 126 0 0 UMA Hash 256 0 0 0 0 62 0 0