panic: mtx_lock() of spin mutex (null) @ /syzkaller/managers/i386/kernel/sys/modules/tcp/rack/../../../netinet/tcp_stacks/rack.c:16470 cpuid = 1 time = 1623374303 KDB: stack backtrace: db_trace_self_wrapper() at db_trace_self_wrapper+0x47/frame 0xfffffe00516f9560 vpanic() at vpanic+0x1c7/frame 0xfffffe00516f95c0 panic() at panic+0x43/frame 0xfffffe00516f9620 __mtx_lock_flags() at __mtx_lock_flags+0x202/frame 0xfffffe00516f9680 rack_output() at rack_output+0x2594/frame 0xfffffe00516f99c0 tcp_hpts_thread() at tcp_hpts_thread+0xe34/frame 0xfffffe00516f9b10 ithread_loop() at ithread_loop+0x33f/frame 0xfffffe00516f9bb0 fork_exit() at fork_exit+0xb3/frame 0xfffffe00516f9bf0 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe00516f9bf0 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- KDB: enter: panic [ thread pid 12 tid 100084 ] Stopped at kdb_enter+0x67: movq $0,0x1635afe(%rip) db> db> set $lines = 0 db> set $maxwidth = 0 db> show registers cs 0x20 ds 0x3b es 0x3b fs 0x13 gs 0x1b ss 0 rax 0x12 rcx 0xffffffff8113c660 vprintf+0x140 rdx 0x1 rbx 0 rsp 0xfffffe00516f9540 rbp 0xfffffe00516f9560 rsi 0 rdi 0xffffffff8113c696 vprintf+0x176 r8 0 r9 0x8080808080808080 r10 0xfffffe00516f9430 r11 0x1ffaefff59c r12 0xffffffff82267ba0 ddb_dbbe r13 0 r14 0xffffffff81a7ca0b r15 0xffffffff81a7ca0b rip 0xffffffff81133687 kdb_enter+0x67 rflags 0x86 kdb_enter+0x67: movq $0,0x1635afe(%rip) db> show proc Process 12 (intr) at 0xfffff80004c6ea60: state: NORMAL uid: 0 gids: 0 parent: pid 0 at 0xffffffff8271c6b0 ABI: null flag: 0x10000284 flag2: 0 reaper: 0xffffffff8271c6b0 reapsubtree: 12 sigparent: 20 vmspace: 0xffffffff8271d310 (map 0xffffffff8271d310) (map.pmap 0xffffffff8271d3d0) (pmap 0xffffffff8271d430) threads: 23 100011 I [swi5: fast taskq] 100014 I [swi6: task queue] 100016 I [swi6: Giant taskq] 100030 I [swi3: vm] 100031 I [swi4: clock (0)] 100032 I [swi4: clock (1)] 100033 I [swi1: netisr 0] 100046 I [irq24: virtio_pci0] 100047 I [irq25: virtio_pci0] 100048 I [irq26: virtio_pci0] 100049 I [irq27: virtio_pci0] 100050 I [irq28: virtio_pci1] 100051 I [irq29: virtio_pci1] 100052 I [irq30: virtio_pci1] 100053 I [irq31: virtio_pci1] 100054 I [irq32: virtio_pci1] 100059 I [irq10: virtio_pci2] 100061 I [irq1: atkbd0] 100062 I [irq12: psm0] 100063 I [swi0: uart uart++] 100071 I [swi1: pf send] 100084 Run CPU 1 [swi1: hpts] 100085 I [swi1: hpts] db> ps pid ppid pgrp uid state wmesg wchan cmd 5186 784 784 0 R (threaded) syz-executor.1 104587 RunQ syz-executor.1 104625 RunQ syz-executor.1 5185 783 783 0 R (threaded) syz-executor.0 104483 RunQ syz-executor.0 104622 S uwait 0xfffff8005bde7300 syz-executor.0 104624 RunQ syz-executor.0 5181 853 853 0 R (threaded) syz-executor.3 104528 Run CPU 0 syz-executor.3 104615 S connec 0xfffff8000802c490 syz-executor.3 104623 RunQ syz-executor.3 5083 1 808 0 S uwait 0xfffff800084fe300 syz-executor.2 5076 1 808 0 S uwait 0xfffff8005bf1a500 syz-executor.2 5071 1 808 0 S uwait 0xfffff80015dad500 syz-executor.2 5066 1 808 0 S uwait 0xfffff8005bf1a900 syz-executor.2 5061 1 808 0 S uwait 0xfffff8005bde7900 syz-executor.2 5057 1 808 0 S uwait 0xfffff8005bf1af00 syz-executor.2 5052 1 808 0 S uwait 0xfffff8005b1b6100 syz-executor.2 5049 1 808 0 S umtxn 0xfffff80015dadd00 syz-executor.2 5043 1 808 0 S uwait 0xfffff80015d5db00 syz-executor.2 5039 1 808 0 S uwait 0xfffff8005bde7500 syz-executor.2 5037 1 808 0 S uwait 0xfffff8005bc61d00 syz-executor.2 5033 1 808 0 S uwait 0xfffff8000869f500 syz-executor.2 5027 1 808 0 S uwait 0xfffff8005bc61f00 syz-executor.2 5023 1 808 0 S uwait 0xfffff8005b377d00 syz-executor.2 5009 1 808 0 S uwait 0xfffff8005bde7700 syz-executor.2 5002 1 808 0 S uwait 0xfffff8005b377300 syz-executor.2 5000 1 808 0 S uwait 0xfffff800084fed00 syz-executor.2 4995 1 808 0 S uwait 0xfffff8005bf1ab00 syz-executor.2 4982 1 783 0 S uwait 0xfffff800084feb00 syz-executor.0 4981 1 783 0 S uwait 0xfffff80015d5dd00 syz-executor.0 4570 1 784 0 S uwait 0xfffff8005bc61700 syz-executor.1 4565 1 784 0 S uwait 0xfffff80004ca6900 syz-executor.1 4560 1 784 0 S uwait 0xfffff8005bf1ad00 syz-executor.1 4554 1 784 0 S uwait 0xfffff80015ddab00 syz-executor.1 4545 1 784 0 S uwait 0xfffff8000869f700 syz-executor.1 4535 1 784 0 S uwait 0xfffff80015f10b00 syz-executor.1 4533 1 784 0 S uwait 0xfffff80015dadf00 syz-executor.1 4526 1 784 0 S uwait 0xfffff8005bc61b00 syz-executor.1 2688 1 808 0 S uwait 0xfffff8005bf1d500 syz-executor.2 2686 1 808 0 S uwait 0xfffff80015dda900 syz-executor.2 1175 1169 1175 0 Ss select 0xfffff8005bd4fd40 dhclient 1172 1 1172 0 Ss select 0xfffff8005bf5b5c0 dhclient 1169 1162 436 65 S select 0xfffff8005bd4fec0 dhclient 1162 436 436 0 S wait 0xfffff80015e88530 sh 958 0 0 0 DL aiordy 0xfffff80015d7b530 [aiod4] 957 0 0 0 DL aiordy 0xfffff8005bbf7000 [aiod3] 956 0 0 0 DL aiordy 0xfffff8005bbf7a60 [aiod2] 955 0 0 0 DL aiordy 0xfffff8005bbf7530 [aiod1] 853 780 853 0 Rs syz-executor.3 808 780 808 0 Rs syz-executor.2 784 780 784 0 Rs syz-executor.1 783 780 783 0 Rs syz-executor.0 780 778 778 0 S (threaded) syz-fuzzer 100107 S uwait 0xfffff80015d1b700 syz-fuzzer 100117 S uwait 0xfffff80015d1bf00 syz-fuzzer 100118 S uwait 0xfffff80015a3f100 syz-fuzzer 100119 S uwait 0xfffff80015a3f300 syz-fuzzer 100120 S uwait 0xfffff8005b2eb300 syz-fuzzer 100121 S uwait 0xfffff8005b2eb500 syz-fuzzer 100122 S uwait 0xfffff8005b3a2d00 syz-fuzzer 100124 S kqread 0xfffff8005b377900 syz-fuzzer 778 776 778 0 Ss pause 0xfffff8005b38b0b0 csh 776 694 776 0 Ss select 0xfffff80015ed55c0 sshd 762 1 762 0 Ss+ ttyin 0xfffff8001538acb0 getty 761 1 761 0 Ss+ ttyin 0xfffff80015aab4b0 getty 760 1 760 0 Ss+ ttyin 0xfffff80015aabcb0 getty 759 1 759 0 Ss+ ttyin 0xfffff80015aaf4b0 getty 758 1 758 0 Ss+ ttyin 0xfffff80015aafcb0 getty 757 1 757 0 Ss+ ttyin 0xfffff80015a354b0 getty 756 1 756 0 Ss+ ttyin 0xfffff80015a35cb0 getty 755 1 755 0 Ss+ ttyin 0xfffff800158a74b0 getty 754 1 754 0 Ss+ ttyin 0xfffff800158a7cb0 getty 698 1 698 0 Ss nanslp 0xffffffff8273c8e0 cron 694 1 694 0 Ss select 0xfffff80015e5e740 sshd 507 1 507 0 Ss select 0xfffff80015ed5ec0 syslogd 436 1 436 0 Ss wait 0xfffff80015e85000 devd 435 1 435 65 Ss select 0xfffff80015ee4740 dhclient 350 1 350 0 Ss select 0xfffff8005b306a40 dhclient 347 1 347 0 Ss select 0xfffff8005b30ba40 dhclient 23 0 0 0 DL vlruwt 0xfffff80015c4c530 [vnlru] 22 0 0 0 DL syncer 0xffffffff8282bcd0 [syncer] 21 0 0 0 DL (threaded) [bufdaemon] 100081 D qsleep 0xffffffff8282ada0 [bufdaemon] 100088 D - 0xffffffff8220aec0 [bufspacedaemon-0] 100099 D sdflush 0xfffff80015e09ce8 [/ worker] 20 0 0 0 DL psleep 0xffffffff82852c08 [vmdaemon] 19 0 0 0 DL (threaded) [pagedaemon] 100079 D psleep 0xffffffff82847078 [dom0] 100086 D launds 0xffffffff82847084 [laundry: dom0] 100087 D umarcl 0xffffffff815d11b0 [uma] 18 0 0 0 DL - 0xffffffff82570c88 [rand_harvestq] 17 0 0 0 DL waiting 0xffffffff82e34828 [sctp_iterator] 16 0 0 0 DL pftm 0xffffffff8304c3c0 [pf purge] 15 0 0 0 DL - 0xffffffff8282845c [soaiod4] 9 0 0 0 DL - 0xffffffff8282845c [soaiod3] 8 0 0 0 DL - 0xffffffff8282845c [soaiod2] 7 0 0 0 DL - 0xffffffff8282845c [soaiod1] 6 0 0 0 DL (threaded) [cam] 100044 D - 0xffffffff82448140 [doneq0] 100045 D - 0xffffffff824480c0 [async] 100078 D - 0xffffffff82447f90 [scanner] 14 0 0 0 DL seqstat 0xfffff80004d50488 [sequencer 00] 5 0 0 0 DL crypto_ 0xfffff80004db7f80 [crypto returns 1] 4 0 0 0 DL crypto_ 0xfffff80004db7f30 [crypto returns 0] 3 0 0 0 DL crypto_ 0xffffffff82844560 [crypto] 13 0 0 0 DL (threaded) [geom] 100035 D - 0xffffffff8271c120 [g_event] 100036 D - 0xffffffff8271c128 [g_up] 100037 D - 0xffffffff8271c130 [g_down] 2 0 0 0 DL (threaded) [KTLS] 100028 D - 0xfffff80004c6d500 [thr_0] 100029 D - 0xfffff80004c6d580 [thr_1] 12 0 0 0 RL (threaded) [intr] 100011 I [swi5: fast taskq] 100014 I [swi6: task queue] 100016 I [swi6: Giant taskq] 100030 I [swi3: vm] 100031 I [swi4: clock (0)] 100032 I [swi4: clock (1)] 100033 I [swi1: netisr 0] 100046 I [irq24: virtio_pci0] 100047 I [irq25: virtio_pci0] 100048 I [irq26: virtio_pci0] 100049 I [irq27: virtio_pci0] 100050 I [irq28: virtio_pci1] 100051 I [irq29: virtio_pci1] 100052 I [irq30: virtio_pci1] 100053 I [irq31: virtio_pci1] 100054 I [irq32: virtio_pci1] 100059 I [irq10: virtio_pci2] 100061 I [irq1: atkbd0] 100062 I [irq12: psm0] 100063 I [swi0: uart uart++] 100071 I [swi1: pf send] 100084 Run CPU 1 [swi1: hpts] 100085 I [swi1: hpts] 11 0 0 0 RL (threaded) [idle] 100003 CanRun [idle: cpu0] 100004 CanRun [idle: cpu1] 1 0 1 0 SLs wait 0xfffff80004bd6530 [init] 10 0 0 0 DL audit_w 0xffffffff82844a70 [audit] 0 0 0 0 DLs (threaded) [kernel] 100000 D swapin 0xffffffff8271c6b0 [swapper] 100005 D - 0xfffff80004c73d00 [if_io_tqg_0] 100006 D - 0xfffff80004c73900 [if_io_tqg_1] 100007 D - 0xfffff80004c73500 [if_config_tqg_0] 100008 D - 0xfffff80004c73100 [softirq_0] 100009 D - 0xfffff80004c71d00 [softirq_1] 100010 D - 0xfffff80004c72500 [aiod_kick taskq] 100012 D - 0xfffff80004c6f500 [kqueue_ctx taskq] 100013 D - 0xfffff80004c67d00 [pci_hp taskq] 100015 D - 0xfffff80004c65d00 [inm_free taskq] 100017 D - 0xfffff80004c64d00 [linuxkpi_irq_wq] 100018 D - 0xfffff80004c64500 [thread taskq] 100019 D - 0xfffff80004c62d00 [in6m_free taskq] 100020 D - 0xfffff80004c62500 [linuxkpi_short_wq_0] 100021 D - 0xfffff80004c62500 [linuxkpi_short_wq_1] 100022 D - 0xfffff80004c62500 [linuxkpi_short_wq_2] 100023 D - 0xfffff80004c62500 [linuxkpi_short_wq_3] 100024 D - 0xfffff80004c61100 [linuxkpi_long_wq_0] 100025 D - 0xfffff80004c61100 [linuxkpi_long_wq_1] 100026 D - 0xfffff80004c61100 [linuxkpi_long_wq_2] 100027 D - 0xfffff80004c61100 [linuxkpi_long_wq_3] 100034 D - 0xfffff80004cf0500 [firmware taskq] 100038 D - 0xfffff80004d3fd00 [crypto_0] 100039 D - 0xfffff80004d3fd00 [crypto_1] 100055 D - 0xfffff80015382500 [vtnet0 rxq 0] 100056 D - 0xfffff80015382100 [vtnet0 txq 0] 100057 D - 0xfffff80015366d00 [vtnet0 rxq 1] 100058 D - 0xfffff80015366900 [vtnet0 txq 1] 100060 D vtbslp 0xfffff80015376b00 [virtio_balloon] 100064 D - 0xfffff80015463d00 [mca taskq] 100066 D - 0xffffffff81e29a21 [deadlkres] 100074 D - 0xfffff80015a42900 [acpi_task_0] 100075 D - 0xfffff80015a42900 [acpi_task_1] 100076 D - 0xfffff80015a42900 [acpi_task_2] 100077 D - 0xfffff80004db6100 [CAM taskq] db> show all locks Process 5186 (syz-executor.1) thread 0xfffffe009527b720 (104587) exclusive sleep mutex umtxql (umtxql) r = 0 (0xffffffff82764870) locked @ /syzkaller/managers/i386/kernel/sys/kern/kern_umtx.c:946 Process 12 (intr) thread 0xfffffe0055374000 (100084) exclusive rw tcpinp (tcpinp) r = 0 (0xfffff80008add5d8) locked @ /syzkaller/managers/i386/kernel/sys/netinet/tcp_hpts.c:1536 db> show malloc Type InUse MemUse Requests sysctloid 34288 12858K 34355 pf_hash 5 11560K 5 devbuf 4216 6982K 4244 tcp_hpts 5 3219K 5 kobj 328 2624K 488 vtbuf 24 2064K 46 filedesc 76 1214K 8595 inodedep 551 1070K 4330 newblk 45 1054K 4900 vfscache 3 1035K 3 subproc 179 795K 5278 pcb 71 721K 3032 acpica 1674 649K 54692 callout 2 528K 2 ufs_quota 1 520K 1 vfs_hash 1 520K 1 intr 4 480K 4 bus 990 378K 3485 linker 348 270K 409 dirrem 528 264K 4264 DEVFS1 107 214K 124 kdtrace 256 201K 9813 module 512 192K 512 freefile 501 188K 4234 vnet_data 1 176K 1 tidhash 3 164K 3 umtx 374 164K 374 pagedep 43 157K 4266 tfo_ccache 1 136K 1 BPF 22 136K 26 sem 4 120K 4 UMA 270 102K 270 audit_evclass 236 89K 294 mtx_pool 2 80K 2 temp 35 77K 2386 syncache 1 76K 1 vmem 3 72K 5 routetbl 127 69K 920 msg 4 68K 4 acpitask 1 64K 1 ddb_capture 1 64K 1 DEVFS3 126 63K 136 ifaddr 67 63K 69 ether_multi 152 57K 162 gtaskqueue 18 57K 18 DEVFS_RULE 56 54K 56 kenv 95 52K 95 eventhandler 133 50K 133 lltable 50 44K 52 ithread 99 43K 99 kqueue 90 42K 5195 rman 84 42K 425 KTRACE 100 38K 100 taskqueue 60 36K 60 proc 3 34K 3 ufs_mount 5 34K 6 bus-sc 33 34K 1696 pwddesc 88 33K 5187 devstat 4 33K 4 hostcache 1 32K 1 tty 16 32K 16 shm 1 32K 1 GEOM 60 29K 489 in6_multi 65 29K 65 DEVFSP 74 28K 185 kbdmux 6 28K 6 ifnet 7 25K 7 kcovinfo 64 24K 68 ksem 15 23K 112 cred 23 23K 204 sctp_atcl 21 21K 996 plimit 21 21K 349 CAM queue 5 21K 1528 pfs_nodes 20 20K 20 sctp_stro 9 18K 181 UART 12 18K 12 proc-args 44 17K 588 bmsafemap 2 17K 4302 sctp_timw 32 16K 32 rpc 2 16K 2 shmfd 1 16K 325 pfs_vncache 1 16K 1 newdirblk 40 15K 4253 inpcbpolicy 38 15K 3068 ufs_dirhash 24 14K 24 sglist 5 13K 5 session 25 13K 40 osd 33 13K 2774 MCA 32 12K 32 CAM DEV 3 12K 510 selfd 31 12K 82172 sctp_atky 30 12K 1518 vt 11 11K 11 acpisem 28 11K 28 CAM XPT 22 11K 543 Unitno 27 11K 47 uidinfo 3 9K 7 pf_ifnet 10 9K 171 memdesc 1 8K 1 ipsec-saq 2 8K 2 evdev 4 8K 4 sctp_athm 21 8K 1052 acpidev 20 8K 20 hhook 15 8K 17 lockf 19 8K 29 sctp_map 18 7K 362 mount 16 7K 90 fpukern_ctx 3 6K 3 mkdir 16 6K 8506 terminal 11 6K 11 tcp_fsb 14 6K 1076 ip6ndp 12 6K 14 sctp_ifa 13 5K 14 diradd 13 5K 4297 ipsecpolicy 2 5K 2 encap_export_host 12 5K 12 clone 9 5K 9 local_apic 1 4K 1 io_apic 1 4K 1 sahead 1 4K 1 secasvar 1 4K 1 nhops 6 4K 8 pci_link 10 4K 10 select 10 4K 50 msi 9 4K 9 DEVFS 9 4K 10 tun 7 4K 7 ipsec 3 3K 3 in_multi 6 3K 8 nexusdev 7 3K 7 feeder 7 3K 7 freework 5 3K 4259 sctp_ifn 6 3K 14 mld 6 3K 6 igmp 6 3K 6 toponodes 6 3K 6 prison 6 3K 6 isadev 6 3K 6 softdep 1 2K 1 freeblks 4 2K 4234 vnodemarker 2 2K 18 NFSD session 1 2K 1 linux 5 2K 6 CAM periph 4 2K 271 soname 5 2K 7185 crypto 4 2K 4 pfil 4 2K 4 CAM path 4 2K 1034 tcpfunc 3 2K 3 loginclass 3 2K 6 sctp_stri 1 1K 112 chacha20random 1 1K 1 vnodes 1 1K 1 CAM SIM 2 1K 2 ktls 1 1K 1 procdesc 2 1K 12 cdev 2 1K 2 aesni_data 2 1K 2 cpus 2 1K 2 atkbddev 2 1K 2 CAM dev queue 2 1K 2 xform 2 1K 141 entropy 2 1K 39 filecaps 2 1K 90 NFSD lckfile 1 1K 1 NFSD V4client 1 1K 1 pmchooks 1 1K 1 sctp_vrf 1 1K 1 apmdev 1 1K 1 CAM I/O Scheduler 1 1K 1 vnet_data_free 1 1K 1 vnet 1 1K 1 Per-cpu 1 1K 1 p1003.1b 1 1K 1 acpiintr 1 1K 1 pmc 1 1K 1 cdg data 0 0K 0 tcp_do 0 0K 0 pf_table 0 0K 129 pf_rule 0 0K 251 pf_altq 0 0K 0 pf_osfp 0 0K 0 pf_temp 0 0K 0 htcp data 0 0K 0 dctcp data 0 0K 0 mqdata 0 0K 0 cubic data 0 0K 0 vegas data 0 0K 0 chd data 0 0K 0 sctp_mcore 0 0K 0 sctp_socko 0 0K 644 sctp_iter 0 0K 11 sctp_mvrf 0 0K 0 sctp_cpal 0 0K 0 sctp_cmsg 0 0K 0 sctp_stre 0 0K 0 sctp_athi 0 0K 0 sctp_a_it 0 0K 11 sctp_aadr 0 0K 0 NFSCL lckown 0 0K 0 NFSCL client 0 0K 0 madt_table 0 0K 2 smartpqi 0 0K 0 NFSCL deleg 0 0K 0 NFSCL open 0 0K 0 NFSCL owner 0 0K 0 NFS fh 0 0K 0 NFS req 0 0K 0 NFSD usrgroup 0 0K 0 NFSD string 0 0K 0 iavf 0 0K 0 ixl 0 0K 0 NFSD V4lock 0 0K 0 NFSD V4state 0 0K 0 NFSD srvcache 0 0K 0 msdosfs_fat 0 0K 0 msdosfs_mount 0 0K 0 msdosfs_node 0 0K 0 ice-resmgr 0 0K 0 ice-osdep 0 0K 0 ice 0 0K 0 axgbe 0 0K 0 DEVFS4 0 0K 0 DEVFS2 0 0K 0 gntdev 0 0K 0 privcmd_dev 0 0K 0 evtchn_dev 0 0K 0 xen_intr 0 0K 0 xenstore 0 0K 0 ciss_data 0 0K 0 xen_hvm 0 0K 0 legacydrv 0 0K 0 qpidrv 0 0K 0 BACKLIGHT 0 0K 0 xnb 0 0K 0 dmar_idpgtbl 0 0K 0 dmar_dom 0 0K 0 dmar_ctx 0 0K 0 xbbd 0 0K 0 xbd 0 0K 0 Balloon 0 0K 0 isci 0 0K 0 iommu_dmamap 0 0K 0 sysmouse 0 0K 0 hyperv_socket 0 0K 0 bxe_ilt 0 0K 0 xenbus 0 0K 0 vtfont 0 0K 0 vm_fictitious 0 0K 0 ath_hal 0 0K 0 athdev 0 0K 0 ata_pci 0 0K 0 ata_dma 0 0K 0 ata_generic 0 0K 0 amr 0 0K 0 pvscsi 0 0K 0 scsi_da 0 0K 69 ata_da 0 0K 0 scsi_ch 0 0K 0 UMAHash 0 0K 0 scsi_cd 0 0K 0 AHCI driver 0 0K 0 vm_pgdata 0 0K 0 jblocks 0 0K 0 savedino 0 0K 2476 sentinel 0 0K 0 jfsync 0 0K 0 jtrunc 0 0K 0 sbdep 0 0K 7 jsegdep 0 0K 0 jseg 0 0K 0 jfreefrag 0 0K 0 jfreeblk 0 0K 0 jnewblk 0 0K 0 jmvref 0 0K 0 jremref 0 0K 0 jaddref 0 0K 0 freedep 0 0K 0 freefrag 0 0K 6 allocindir 0 0K 0 indirdep 0 0K 52 allocdirect 0 0K 0 ufs_trim 0 0K 0 mactemp 0 0K 0 audit_trigger 0 0K 0 audit_pipe_presel 0 0K 0 audit_pipeent 0 0K 0 audit_pipe 0 0K 0 audit_evname 0 0K 0 audit_bsm 0 0K 0 audit_gidset 0 0K 0 audit_text 0 0K 0 audit_path 0 0K 0 audit_data 0 0K 0 audit_cred 0 0K 0 USBdev 0 0K 0 USB 0 0K 0 agp 0 0K 0 nvme_da 0 0K 0 acpipwr 0 0K 0 acpi_perf 0 0K 0 ktls_ocf 0 0K 0 twsbuf 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5E_TLS 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EN 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5DUMP 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 seq_file 0 0K 0 radix 0 0K 0 idr 0 0K 0 lkpifw 0 0K 0 NLM 0 0K 0 ipsec-spdcache 0 0K 0 ipsec-reg 0 0K 0 ipsec-misc 0 0K 0 ipsecrequest 0 0K 0 ip6opt 0 0K 16 ip6_msource 0 0K 0 ip6_moptions 0 0K 0 in6_mfilter 0 0K 0 frag6 0 0K 0 tcplog 0 0K 0 tcp_hwpace 0 0K 0 twe_commands 0 0K 0 LRO 0 0K 0 newreno data 0 0K 0 ip_msource 0 0K 0 ip_moptions 0 0K 5 in_mfilter 0 0K 0 ipid 0 0K 0 80211scan 0 0K 0 80211ratectl 0 0K 0 80211power 0 0K 0 80211nodeie 0 0K 0 80211node 0 0K 0 80211mesh_gt 0 0K 0 80211mesh_rt 0 0K 0 80211perr 0 0K 0 80211prep 0 0K 0 80211preq 0 0K 0 80211dfs 0 0K 0 80211crypto 0 0K 0 80211vap 0 0K 0 iflib 0 0K 0 vlan 0 0K 0 gif 0 0K 0 ifdescr 0 0K 0 zlib 0 0K 0 fadvise 0 0K 0 VN POLL 0 0K 0 twa_commands 0 0K 0 statfs 0 0K 4436 namei_tracker 0 0K 0 export_host 0 0K 0 cl_savebuf 0 0K 4 tcp_log_dev 0 0K 825 midi buffers 0 0K 0 mixer 0 0K 0 ac97 0 0K 0 hdacc 0 0K 0 hdac 0 0K 0 hdaa 0 0K 0 acpicmbat 0 0K 0 SIIS driver 0 0K 0 CAM CCB 0 0K 2176 PUC 0 0K 0 ppbusdev 0 0K 0 agtiapi_MemAlloc malloc 0 0K 0 osti_cacheable 0 0K 0 tempbuff 0 0K 0 biobuf 0 0K 0 aios 0 0K 6 lio 0 0K 8 acl 0 0K 0 tempbuff 0 0K 0 mbuf_tag 0 0K 97 ag_tgt_map_t malloc 0 0K 0 ag_slr_map_t malloc 0 0K 0 lDevFlags * malloc 0 0K 0 tiDeviceHandle_t * malloc 0 0K 0 ag_portal_data_t malloc 0 0K 0 ag_device_t malloc 0 0K 0 STLock malloc 0 0K 0 CCB List 0 0K 0 sr_iov 0 0K 0 OCS 0 0K 0 OCS 0 0K 0 nvme 0 0K 0 nvd 0 0K 0 netmap 0 0K 0 mwldev 0 0K 0 MVS driver 0 0K 0 CAM ccb queue 0 0K 0 mrsasbuf 0 0K 0 mpt_user 0 0K 0 mps_user 0 0K 0 accf 0 0K 0 pts 0 0K 0 iov 0 0K 14878 ioctlops 0 0K 2485 eventfd 0 0K 0 Witness 0 0K 0 stack 0 0K 0 MPSSAS 0 0K 0 mps 0 0K 0 mpr_user 0 0K 0 MPRSAS 0 0K 0 mpr 0 0K 0 mfibuf 0 0K 0 sbuf 0 0K 288 md_sectors 0 0K 0 firmware 0 0K 0 compressor 0 0K 0 md_disk 0 0K 0 SWAP 0 0K 0 malodev 0 0K 0 LED 0 0K 0 sysctltmp 0 0K 637 sysctl 0 0K 3 ekcd 0 0K 0 dumper 0 0K 0 sendfile 0 0K 0 rctl 0 0K 0 ix_sriov 0 0K 0 aacraidcam 0 0K 0 aacraid_buf 0 0K 0 ix 0 0K 0 ipsbuf 0 0K 0 cache 0 0K 0 iirbuf 0 0K 0 prison_racct 0 0K 0 Fail Points 0 0K 0 sigio 0 0K 1 filedesc_to_leader 0 0K 0 pwd 0 0K 0 tty console 0 0K 0 aaccam 0 0K 0 aacbuf 0 0K 0 zstd 0 0K 0 XZ_DEC 0 0K 0 nvlist 0 0K 0 SCSI ENC 0 0K 0 SCSI sa 0 0K 0 scsi_pass 0 0K 0 isofs_node 0 0K 0 isofs_mount 0 0K 0 tr_raid5_data 0 0K 0 tr_raid1e_data 0 0K 0 tr_raid1_data 0 0K 0 tr_raid0_data 0 0K 0 tr_concat_data 0 0K 0 md_sii_data 0 0K 0 md_promise_data 0 0K 0 md_nvidia_data 0 0K 0 md_jmicron_data 0 0K 0 md_intel_data 0 0K 0 md_ddf_data 0 0K 0 raid_data 0 0K 72 geom_flashmap 0 0K 0 tmpfs dir 0 0K 0 tmpfs name 0 0K 0 tmpfs mount 0 0K 0 NFS FHA 0 0K 0 newnfsmnt 0 0K 0 newnfsclient_req 0 0K 0 NFSCL layrecall 0 0K 0 NFSCL session 0 0K 0 NFSCL sockreq 0 0K 0 NFSCL devinfo 0 0K 0 NFSCL flayout 0 0K 0 NFSCL layout 0 0K 0 NFSD rollback 0 0K 0 NFSCL diroff 0 0K 0 NEWdirectio 0 0K 0 NEWNFSnode 0 0K 0 NFSCL lck 0 0K 0 db> show uma Zone Size Used Free Requests Sleeps Bucket Total Mem XFree mbuf_jumbo_page 4096 8320 983 1010962 0 254 38105088 0 malloc-384 384 34363 7 34528 0 30 13198080 0 malloc-1024 1024 4958 2510 14134 0 16 7647232 0 malloc-8192 8192 429 7 6517 0 1 3571712 0 pbuf 2624 0 973 0 0 2 2553152 0 mbuf 256 8819 751 1572974 0 254 2449920 0 malloc-1024 1024 75 1881 2699 0 16 2002944 0 BUF TRIE 144 182 13286 656 0 62 1939392 0 RADIX NODE 144 12191 182 157777 0 62 1781712 0 malloc-512 512 857 2519 4974 0 30 1728512 0 UMA Slabs 0 112 13605 0 13605 0 126 1523760 0 malloc-32768 32768 38 4 4311 0 1 1376256 0 mbuf_cluster 2048 585 1 585 0 254 1200128 0 malloc-384 384 551 2559 4353 0 30 1194240 0 malloc-384 384 2489 61 148780 0 30 979200 0 malloc-384 384 2297 23 31647 0 30 890880 0 tcp_log 416 68 1939 26864 0 254 834912 0 FFS inode 1160 533 20 4768 0 8 641480 0 VM OBJECT 264 1472 58 74487 0 30 403920 0 malloc-384 384 971 29 14418 0 30 384000 0 lkpimm 160 1 2324 1 0 62 372000 0 lkpicurr 160 2 2323 2 0 62 372000 0 malloc-512 512 146 526 5658 0 30 344064 0 THREAD 1808 167 20 4625 0 8 338096 0 malloc-65536 65536 0 5 40 0 1 327680 0 MAP ENTRY 96 2901 291 274590 0 126 306432 0 VNODE 448 571 68 4808 0 30 286272 0 malloc-65536 65536 4 0 4 0 1 262144 0 256 Bucket 2048 103 17 15774 0 8 245760 0 malloc-2048 2048 109 3 285 0 8 229376 0 DEVCTL 1024 12 204 132 0 0 221184 0 malloc-16384 16384 10 2 245 0 1 196608 0 UMA Zones 768 242 2 242 0 16 187392 0 malloc-1024 1024 151 29 2464 0 16 184320 0 malloc-16384 16384 11 0 11 0 1 180224 0 VMSPACE 2544 61 8 5166 0 4 175536 0 tcp_bbr_map 128 0 1333 19673 0 126 170624 0 vmem btag 56 2693 103 2693 0 254 156576 0 malloc-4096 4096 0 37 2176 0 2 151552 0 mbuf_packet 256 45 540 4963 0 254 149760 0 tcp_inpcb 488 33 263 2762 0 254 144448 0 FFS2 dinode 256 533 22 4767 0 62 142080 0 malloc-8192 8192 13 4 4460 0 1 139264 0 S VFS Cache 104 1011 315 5730 0 126 137904 0 malloc-65536 65536 0 2 176 0 1 131072 0 malloc-65536 65536 2 0 2 0 1 131072 0 malloc-65536 65536 2 0 2 0 1 131072 0 malloc-8192 8192 12 3 548 0 1 122880 0 PROC 1328 87 3 5186 0 8 119520 0 ksiginfo 112 74 970 1055 0 126 116928 0 malloc-16384 16384 3 4 33 0 1 114688 0 filedesc0 1072 88 17 5187 0 8 112560 0 ertt_txseginfo 40 1 2726 33705 0 254 109080 0 socket 944 52 60 5571 0 254 105728 0 malloc-32768 32768 1 2 131 0 1 98304 0 malloc-32768 32768 3 0 3 0 1 98304 0 malloc-16384 16384 6 0 6 0 1 98304 0 malloc-2048 2048 30 16 542 0 8 94208 0 malloc-512 512 166 10 1594 0 30 90112 0 UMA Kegs 384 227 6 227 0 30 89472 0 malloc-384 384 174 56 18812 0 30 88320 0 clpbuf 2624 0 32 20 0 16 83968 0 tcpcb 1080 29 48 2762 0 254 83160 0 malloc-16384 16384 5 0 5 0 1 81920 0 32 Bucket 256 260 40 9863 0 62 76800 0 128 Bucket 1024 40 31 695 0 16 72704 0 g_bio 408 0 170 6169 0 30 69360 0 malloc-65536 65536 1 0 2 0 1 65536 0 malloc-32768 32768 2 0 2 0 1 65536 0 malloc-4096 4096 10 6 55 0 2 65536 0 sctp_ep 1280 11 40 759 0 254 65280 0 malloc-4096 4096 13 2 568 0 2 61440 0 malloc-4096 4096 9 6 138 0 2 61440 0 16 Bucket 144 330 90 3009 0 62 60480 0 malloc-8192 8192 3 4 99 0 1 57344 0 malloc-512 512 39 73 174 0 30 57344 0 sctp_asoc 2288 9 16 181 0 254 57200 0 malloc-512 512 81 23 4663 0 30 53248 0 malloc-16384 16384 2 1 4 0 1 49152 0 malloc-512 512 29 67 5594 0 30 49152 0 pcpu-8 8 4647 1497 8381 0 254 49152 0 malloc-4096 4096 4 7 512 0 2 45056 0 malloc-2048 2048 9 13 181 0 8 45056 0 tcp_bbr_pcb 832 0 54 1463 0 16 44928 0 malloc-2048 2048 16 4 1087 0 8 40960 0 64 Bucket 512 45 35 5408 0 30 40960 0 sctp_raddr 736 15 40 517 0 254 40480 0 malloc-384 384 42 58 2253 0 30 38400 0 pipe 744 21 29 431 0 16 37200 0 DIRHASH 1024 34 2 34 0 16 36864 0 NAMEI 1024 0 36 33761 0 16 36864 0 malloc-512 512 58 14 4288 0 30 36864 0 malloc-32768 32768 1 0 1 0 1 32768 0 malloc-16384 16384 1 1 4 0 1 32768 0 malloc-8192 8192 1 3 15 0 1 32768 0 malloc-8192 8192 4 0 4 0 1 32768 0 malloc-4096 4096 1 7 2712 0 2 32768 0 malloc-2048 2048 2 14 151 0 8 32768 0 malloc-1024 1024 27 5 371 0 16 32768 0 pcpu-64 64 487 25 487 0 254 32768 0 tcp_rack_pcb 832 14 22 1076 0 16 29952 0 malloc-512 512 14 42 4348 0 30 28672 0 TURNSTILE 136 188 22 188 0 62 28560 0 KNOTE 160 28 147 115470 0 62 28000 0 Files 80 231 119 19690 0 126 28000 0 malloc-4096 4096 1 5 10 0 2 24576 0 PWD 32 46 710 4342 0 254 24192 0 ttyinq 160 135 15 300 0 62 24000 0 8 Bucket 80 39 261 1633 0 126 24000 0 ttyoutq 256 72 18 160 0 62 23040 0 malloc-384 384 37 23 1100 0 30 23040 0 2 Bucket 32 97 533 2210 0 254 20160 0 SLEEPQUEUE 88 188 36 188 0 126 19712 0 Mountpoints 2752 2 5 2 0 4 19264 0 malloc-1024 1024 14 2 14 0 16 16384 0 malloc-1024 1024 10 6 121 0 16 16384 0 vtnet_tx_hdr 24 0 668 523157 0 254 16032 0 unpcb 256 11 49 1717 0 254 15360 0 malloc-1024 1024 5 7 14 0 16 12288 0 sctp_chunk 152 13 65 288 0 254 11856 0 udp_inpcb 488 6 18 207 0 254 11712 0 kenv 258 15 30 1039 0 30 11610 0 routing nhops 256 27 18 34 0 62 11520 0 malloc-2048 2048 3 1 3 0 8 8192 0 malloc-2048 2048 3 1 3 0 8 8192 0 malloc-1024 1024 6 2 10 0 16 8192 0 rtentry 176 30 16 34 0 62 8096 0 PGRP 88 25 67 46 0 126 8096 0 rl_entry 40 43 159 43 0 254 8080 0 sctp_stream_msg_out 112 0 72 22 0 254 8064 0 sctp_laddr 48 8 160 72 0 254 8064 0 tcp_rack_map 112 27 45 949 0 126 8064 0 udpcb 32 6 246 239 0 254 8064 0 ertt 72 29 83 2762 0 126 8064 0 4 Bucket 48 6 162 142 0 254 8064 0 sctp_readq 152 1 51 51 0 254 7904 0 AIO 208 0 38 26 0 62 7904 0 ripcb 488 2 14 67 0 254 7808 0 udplite_inpcb 488 0 16 32 0 254 7808 0 AIOCB 552 0 14 39 0 16 7728 0 AIOLIO 272 0 28 4 0 30 7616 0 FPU_save_area 832 1 8 1 0 16 7488 0 cpuset 104 7 55 7 0 126 6448 0 epoch_record pcpu 256 4 12 4 0 62 4096 0 malloc-4096 4096 1 0 1 0 2 4096 0 malloc-2048 2048 1 1 1 0 8 4096 0 pcpu-16 16 7 249 7 0 254 4096 0 hostcache 64 1 62 1 0 254 4032 0 syncache 168 0 24 4 0 254 4032 0 AIOP 32 4 122 4 0 254 4032 0 UMA Slabs 1 176 11 11 11 0 62 3872 0 mqnode 416 3 6 3 0 30 3744 0 KMAP ENTRY 96 12 27 12 0 0 3744 0 vmem 1856 1 1 1 0 8 3712 0 SMR CPU 32 3 60 3 0 254 2016 0 SMR SHARED 24 3 60 3 0 254 1512 0 FFS1 dinode 128 0 0 0 0 126 0 0 da_ccb 544 0 0 0 0 16 0 0 ada_ccb 272 0 0 0 0 30 0 0 swblk 136 0 0 0 0 62 0 0 swpctrie 144 0 0 0 0 62 0 0 cdg_qdiffsample 16 0 0 0 0 254 0 0 sctp_asconf_ack 48 0 0 0 0 254 0 0 sctp_asconf 40 0 0 0 0 254 0 0 pf state scrubs 40 0 0 0 0 254 0 0 pf frag entries 40 0 0 0 0 254 0 0 pf frags 248 0 0 0 0 62 0 0 pf table entries 160 0 0 0 0 62 0 0 pf table entry counters 64 0 0 0 0 254 0 0 pf source nodes 136 0 0 0 0 254 0 0 pf state keys 88 0 0 0 0 126 0 0 pf states 304 0 0 0 0 0 0 0 pf tags 104 0 0 0 0 126 0 0 pf mtags 48 0 0 0 0 254 0 0 tcp_log_node 120 0 0 0 0 126 0 0 tcp_log_bucket 176 0 0 0 0 62 0 0 tcpreass 48 0 0 0 0 254 0 0 tfo_ccache_entries 80 0 0 0 0 126 0 0 tfo 4 0 0 0 0 254 0 0 sackhole 32 0 0 0 0 254 0 0 tcptw 88 0 0 0 0 254 0 0 ipq 56 0 0 0 0 254 0 0 IPsec SA lft_c 16 0 0 0 0 254 0 0 itimer 352 0 0 0 0 30 0 0 NCLNODE 584 0 0 0 0 16 0 0 mqnotifier 216 0 0 0 0 62 0 0 mvdata 64 0 0 0 0 254 0 0 mqueue 248 0 0 0 0 62 0 0 TMPFS node 224 0 0 0 0 62 0 0 LTS VFS Cache 360 0 0 0 0 30 0 0 L VFS Cache 320 0 0 0 0 30 0 0 STS VFS Cache 144 0 0 0 0 62 0 0 cryptop 280 0 0 0 0 30 0 0 linux_dma_object 24 0 0 0 0 254 0 0 linux_dma_pctrie 144 0 0 0 0 62 0 0 IOMMU_MAP_ENTRY 120 0 0 0 0 126 0 0 ktls_session 128 0 0 0 0 126 0 0 mbuf_jumbo_16k 16384 0 0 0 0 254 0 0 mbuf_jumbo_9k 9216 0 0 0 0 254 0 0 audit_record 1280 0 0 0 0 8 0 0 domainset 40 0 0 0 0 254 0 0 MAC labels 40 0 0 0 0 254 0 0 vnpbuf 2624 0 0 0 0 64 0 0 mdpbuf 2624 0 0 0 0 3 0 0 nfspbuf 2624 0 0 0 0 16 0 0 swwbuf 2624 0 0 0 0 8 0 0 swrbuf 2624 0 0 0 0 16 0 0 umtx_shm 88 0 0 0 0 126 0 0 umtx pi 96 0 0 0 0 126 0 0 rangeset pctrie nodes 144 0 0 0 0 62 0 0 malloc-65536 65536 0 0 0 0 1 0 0 malloc-65536 65536 0 0 0 0 1 0 0 malloc-32768 32768 0 0 0 0 1 0 0 malloc-32768 32768 0 0 0 0 1 0 0 malloc-32768 32768 0 0 0 0 1 0 0 malloc-16384 16384 0 0 0 0 1 0 0 malloc-8192 8192 0 0 0 0 1 0 0 malloc-8192 8192 0 0 0 0 1 0 0 malloc-256 256 0 0 0 0 62 0 0 malloc-256 256 0 0 0 0 62 0 0 malloc-256 256 0 0 0 0 62 0 0 malloc-256 256 0 0 0 0 62 0 0 malloc-256 256 0 0 0 0 62 0 0 malloc-256 256 0 0 0 0 62 0 0 malloc-256 256 0 0 0 0 62 0 0 malloc-256 256 0 0 0 0 62 0 0 malloc-128 128 0 0 0 0 126 0 0 malloc-128 128 0 0 0 0 126 0 0 malloc-128 128 0 0 0 0 126 0 0 malloc-128 128 0 0 0 0 126 0 0 malloc-128 128 0 0 0 0 126 0 0 malloc-128 128 0 0 0 0 126 0 0 malloc-128 128 0 0 0 0 126 0 0 malloc-128 128 0 0 0 0 126 0 0 malloc-64 64 0 0 0 0 254 0 0 malloc-64 64 0 0 0 0 254 0 0 malloc-64 64 0 0 0 0 254 0 0 malloc-64 64 0 0 0 0 254 0 0 malloc-64 64 0 0 0 0 254 0 0 malloc-64 64 0 0 0 0 254 0 0 malloc-64 64 0 0 0 0 254 0 0 malloc-64 64 0 0 0 0 254 0 0 malloc-32 32 0 0 0 0 254 0 0 malloc-32 32 0 0 0 0 254 0 0 malloc-32 32 0 0 0 0 254 0 0 malloc-32 32 0 0 0 0 254 0 0 malloc-32 32 0 0 0 0 254 0 0 malloc-32 32 0 0 0 0 254 0 0 malloc-32 32 0 0 0 0 254 0 0 malloc-32 32 0 0 0 0 254 0 0 malloc-16 16 0 0 0 0 254 0 0 malloc-16 16 0 0 0 0 254 0 0 malloc-16 16 0 0 0 0 254 0 0 malloc-16 16 0 0 0 0 254 0 0 malloc-16 16 0 0 0 0 254 0 0 malloc-16 16 0 0 0 0 254 0 0 malloc-16 16 0 0 0 0 254 0 0 malloc-16 16 0 0 0 0 254 0 0 pcpu-32 32 0 0 0 0 254 0 0 pcpu-4 4 0 0 0 0 254 0 0 fakepg 104 0 0 0 0 126 0 0 UMA Hash 256 0 0 0 0 62 0 0