panic: Queues are not empty when handling SHUTDOWN-ACK cpuid = 0 time = 1621123832 KDB: stack backtrace: db_trace_self_wrapper() at db_trace_self_wrapper+0x47/frame 0xfffffe00037831f0 vpanic() at vpanic+0x1c7/frame 0xfffffe0003783250 panic() at panic+0x43/frame 0xfffffe00037832b0 sctp_process_control() at sctp_process_control+0xa28f/frame 0xfffffe0003783740 sctp_common_input_processing() at sctp_common_input_processing+0x7d0/frame 0xfffffe00037838c0 sctp_input_with_port() at sctp_input_with_port+0x30a/frame 0xfffffe00037839b0 sctp_input() at sctp_input+0x1f/frame 0xfffffe00037839d0 ip_input() at ip_input+0x38b/frame 0xfffffe0003783a90 swi_net() at swi_net+0x20d/frame 0xfffffe0003783b10 ithread_loop() at ithread_loop+0x33f/frame 0xfffffe0003783bb0 fork_exit() at fork_exit+0xb3/frame 0xfffffe0003783bf0 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe0003783bf0 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- KDB: enter: panic [ thread pid 12 tid 100030 ] Stopped at kdb_enter+0x67: movq $0,0x163a0ee(%rip) db> db> set $lines = 0 db> set $maxwidth = 0 db> show registers cs 0x20 ds 0x3b es 0x3b fs 0x13 gs 0x1b ss 0x28 rax 0x12 rcx 0xffffffff81138070 vprintf+0x140 rdx 0x1 rbx 0 rsp 0xfffffe00037831d0 rbp 0xfffffe00037831f0 rsi 0 rdi 0xffffffff811380a6 vprintf+0x176 r8 0 r9 0x8080808080808080 r10 0xfffffe00037830c0 r11 0x1fffcfff59c r12 0xffffffff82267ac0 ddb_dbbe r13 0 r14 0xffffffff81a75f09 r15 0xffffffff81a75f09 rip 0xffffffff8112f097 kdb_enter+0x67 rflags 0x86 kdb_enter+0x67: movq $0,0x163a0ee(%rip) db> show proc Process 12 (intr) at 0xfffff80004c5ea70: state: NORMAL uid: 0 gids: 0 parent: pid 0 at 0xffffffff8271c6b0 ABI: null flag: 0x10000284 flag2: 0 reaper: 0xffffffff8271c6b0 reapsubtree: 12 sigparent: 20 vmspace: 0xffffffff8271d330 (map 0xffffffff8271d330) (map.pmap 0xffffffff8271d3f0) (pmap 0xffffffff8271d450) threads: 23 100011 I [swi5: fast taskq] 100014 I [swi6: task queue] 100016 I [swi6: Giant taskq] 100030 Run CPU 0 [swi1: netisr 0] 100031 I [swi3: vm] 100032 I [swi4: clock (0)] 100033 I [swi4: clock (1)] 100046 I [irq24: virtio_pci0] 100047 I [irq25: virtio_pci0] 100048 I [irq26: virtio_pci0] 100049 I [irq27: virtio_pci0] 100050 I [irq28: virtio_pci1] 100051 I [irq29: virtio_pci1] 100052 I [irq30: virtio_pci1] 100053 I [irq31: virtio_pci1] 100054 I [irq32: virtio_pci1] 100059 I [irq10: virtio_pci2] 100061 I [irq1: atkbd0] 100062 I [irq12: psm0] 100063 I [swi0: uart uart++] 100071 I [swi1: pf send] 100084 I [swi1: hpts] 100085 I [swi1: hpts] db> ps pid ppid pgrp uid state wmesg wchan cmd 31043 823 823 0 S (threaded) syz-executor.3 128071 S nanslp 0xffffffff8273c8e0 syz-executor.3 134686 S connec 0xfffff80058be60e0 syz-executor.3 134688 S uwait 0xfffff80036de0900 syz-executor.3 31037 786 786 0 R (threaded) syz-executor.2 103198 Run CPU 1 syz-executor.2 134678 RunQ syz-executor.2 134682 S uwait 0xfffff80058d0dd00 syz-executor.2 134689 S uwait 0xfffff80058bbe700 syz-executor.2 25716 1 823 0 S uwait 0xfffff80036d23700 syz-executor.3 25585 1 823 0 SV uwait 0xfffff80050ae4d00 syz-executor.3 25578 1 823 0 SV accept 0xfffff80058c7fc68 syz-executor.3 25499 1 782 0 S uwait 0xfffff80036ebcd00 syz-executor.0 25497 1 782 0 S uwait 0xfffff80036de0100 syz-executor.0 25492 1 782 0 S uwait 0xfffff80036e9ad00 syz-executor.0 25484 1 782 0 S uwait 0xfffff80015d60700 syz-executor.0 25479 1 782 0 S uwait 0xfffff80050ae4500 syz-executor.0 25475 1 782 0 S uwait 0xfffff80036e9a700 syz-executor.0 25470 1 782 0 S uwait 0xfffff80050a4bf00 syz-executor.0 25464 1 782 0 S uwait 0xfffff80036e9a100 syz-executor.0 25409 1 25398 0 S uwait 0xfffff80036bf0500 syz-executor.3 25408 1 823 0 S uwait 0xfffff80015d87b00 syz-executor.3 25336 1 823 0 S uwait 0xfffff80036ebc700 syz-executor.3 25332 1 823 0 S uwait 0xfffff80036ebcb00 syz-executor.3 25093 1 823 0 S uwait 0xfffff80015c9dd00 syz-executor.3 25091 1 823 0 S uwait 0xfffff800154b2900 syz-executor.3 21791 1 823 0 S uwait 0xfffff80036d23d00 syz-executor.3 21788 1 823 0 S uwait 0xfffff80015d81d00 syz-executor.3 21592 1 782 0 S uwait 0xfffff80036cf4500 syz-executor.0 21590 1 782 0 S uwait 0xfffff80015d8a500 syz-executor.0 14798 1 786 0 S uwait 0xfffff80015d81500 syz-executor.2 14795 1 786 0 S uwait 0xfffff80015d7f500 syz-executor.2 14581 1 14578 0 S uwait 0xfffff80036de0500 syz-executor.0 14580 1 14578 0 S uwait 0xfffff80036de0f00 syz-executor.0 8868 1 786 0 S uwait 0xfffff80036bf0f00 syz-executor.2 8867 1 786 0 S uwait 0xfffff80036bf0900 syz-executor.2 8528 1 823 0 S uwait 0xfffff80036e9af00 syz-executor.3 8076 1 782 0 S uwait 0xfffff80036bf0700 syz-executor.0 8071 1 782 0 S uwait 0xfffff80036e9a500 syz-executor.0 7982 0 0 0 DL - 0xffffffff8271de24 [accounting] 6089 1 6089 65 Ss select 0xfffff800506a9740 dhclient 4936 1 4936 0 Ss select 0xfffff8005078d8c0 dhclient 4933 1 4933 0 Ss select 0xfffff800506f7140 dhclient 4914 1 4914 65 Ss select 0xfffff80050226ec0 dhclient 4369 1 4369 0 Ss select 0xfffff800507865c0 dhclient 4365 1 4365 0 Ss select 0xfffff800506b2bc0 dhclient 4343 1 4343 65 Ss select 0xfffff800506dda40 dhclient 3542 1 3542 0 Ss select 0xfffff800364e52c0 dhclient 3539 1 3539 0 Ss select 0xfffff80050226740 dhclient 3520 1 3520 65 Ss select 0xfffff80050299ec0 dhclient 1233 1 1233 0 Ss select 0xfffff800363edd40 dhclient 1230 1 1230 0 Ss select 0xfffff80036d14a40 dhclient 1079 1 823 0 S uwait 0xfffff80015d8a700 syz-executor.3 1078 1 823 0 S uwait 0xfffff80015d60f00 syz-executor.3 823 780 823 0 Ss nanslp 0xffffffff8273c8e1 syz-executor.3 786 780 786 0 Ss nanslp 0xffffffff8273c8e1 syz-executor.2 783 780 783 0 Ss piperd 0xfffff800364072e8 syz-executor.1 782 780 782 0 Ss piperd 0xfffff80015df58b8 syz-executor.0 780 778 778 0 S (threaded) syz-fuzzer 100103 S uwait 0xfffff80015a97d00 syz-fuzzer 100121 S uwait 0xfffff80015d81100 syz-fuzzer 100122 S uwait 0xfffff80015d87100 syz-fuzzer 100123 S uwait 0xfffff80015d81f00 syz-fuzzer 100124 S uwait 0xfffff80015d8ab00 syz-fuzzer 100125 S uwait 0xfffff80015d81900 syz-fuzzer 100126 S uwait 0xfffff80015d81b00 syz-fuzzer 100127 S uwait 0xfffff80015d87300 syz-fuzzer 100157 S uwait 0xfffff80036d23f00 syz-fuzzer 105216 S kqread 0xfffff80015d7b900 syz-fuzzer 778 776 778 0 Ss pause 0xfffff80015df4b20 csh 776 694 776 0 Ss select 0xfffff80036432ec0 sshd 760 1 760 0 Ss+ ttyin 0xfffff80015465cb0 getty 759 1 759 0 Ss+ ttyin 0xfffff80015afd4b0 getty 758 1 758 0 Ss+ ttyin 0xfffff80015afdcb0 getty 757 1 757 0 Ss+ ttyin 0xfffff80015a874b0 getty 756 1 756 0 Ss+ ttyin 0xfffff80015a87cb0 getty 755 1 755 0 Ss+ ttyin 0xfffff80015a894b0 getty 754 1 754 0 Ss+ ttyin 0xfffff80015a89cb0 getty 753 1 753 0 Ss+ ttyin 0xfffff80015a8e4b0 getty 752 1 752 0 Ss+ ttyin 0xfffff80015a8ecb0 getty 698 1 698 0 Ss nanslp 0xffffffff8273c8e1 cron 694 1 694 0 Ss select 0xfffff80015d855c0 sshd 507 1 507 0 Ss select 0xfffff80015d80ec0 syslogd 436 1 436 0 Ss select 0xfffff80015e2c2c0 devd 435 1 435 65 Ss select 0xfffff80015d858c0 dhclient 350 1 350 0 Ss select 0xfffff80015e2c440 dhclient 347 1 347 0 Ss select 0xfffff80015e8f2c0 dhclient 23 0 0 0 DL vlruwt 0xfffff80015ca1538 [vnlru] 22 0 0 0 DL syncer 0xffffffff8282bd50 [syncer] 21 0 0 0 DL (threaded) [bufdaemon] 100081 D qsleep 0xffffffff8282ae00 [bufdaemon] 100086 D - 0xffffffff8220ae80 [bufspacedaemon-0] 100099 D sdflush 0xfffff80015debce8 [/ worker] 20 0 0 0 DL psleep 0xffffffff82852c48 [vmdaemon] 19 0 0 0 DL (threaded) [pagedaemon] 100079 D psleep 0xffffffff828470b8 [dom0] 100087 D launds 0xffffffff828470c4 [laundry: dom0] 100088 D umarcl 0xffffffff815caad0 [uma] 18 0 0 0 DL - 0xffffffff82570c78 [rand_harvestq] 17 0 0 0 DL waiting 0xffffffff8309b828 [sctp_iterator] 16 0 0 0 DL pftm 0xffffffff82dfc3c0 [pf purge] 15 0 0 0 DL - 0xffffffff8282845c [soaiod4] 9 0 0 0 DL - 0xffffffff8282845c [soaiod3] 8 0 0 0 DL - 0xffffffff8282845c [soaiod2] 7 0 0 0 DL - 0xffffffff8282845c [soaiod1] 6 0 0 0 DL (threaded) [cam] 100044 D - 0xffffffff82448140 [doneq0] 100045 D - 0xffffffff824480c0 [async] 100078 D - 0xffffffff82447f90 [scanner] 14 0 0 0 DL seqstat 0xfffff80004dccc88 [sequencer 00] 5 0 0 0 DL crypto_ 0xfffff80004d9bd80 [crypto returns 1] 4 0 0 0 DL crypto_ 0xfffff80004d9bd30 [crypto returns 0] 3 0 0 0 DL crypto_ 0xffffffff828445a0 [crypto] 13 0 0 0 DL (threaded) [geom] 100035 D - 0xffffffff8271c120 [g_event] 100036 D - 0xffffffff8271c128 [g_up] 100037 D - 0xffffffff8271c130 [g_down] 2 0 0 0 DL (threaded) [KTLS] 100028 D - 0xfffff80004c3ad00 [thr_0] 100029 D - 0xfffff80004c3ad80 [thr_1] 12 0 0 0 RL (threaded) [intr] 100011 I [swi5: fast taskq] 100014 I [swi6: task queue] 100016 I [swi6: Giant taskq] 100030 Run CPU 0 [swi1: netisr 0] 100031 I [swi3: vm] 100032 I [swi4: clock (0)] 100033 I [swi4: clock (1)] 100046 I [irq24: virtio_pci0] 100047 I [irq25: virtio_pci0] 100048 I [irq26: virtio_pci0] 100049 I [irq27: virtio_pci0] 100050 I [irq28: virtio_pci1] 100051 I [irq29: virtio_pci1] 100052 I [irq30: virtio_pci1] 100053 I [irq31: virtio_pci1] 100054 I [irq32: virtio_pci1] 100059 I [irq10: virtio_pci2] 100061 I [irq1: atkbd0] 100062 I [irq12: psm0] 100063 I [swi0: uart uart++] 100071 I [swi1: pf send] 100084 I [swi1: hpts] 100085 I [swi1: hpts] 11 0 0 0 RL (threaded) [idle] 100003 CanRun [idle: cpu0] 100004 CanRun [idle: cpu1] 1 0 1 0 SLs wait 0xfffff80004bc9538 [init] 10 0 0 0 DL audit_w 0xffffffff82844ab0 [audit] 0 0 0 0 DLs (threaded) [kernel] 100000 D swapin 0xffffffff8271c6b0 [swapper] 100005 D - 0xfffff80004c65d00 [softirq_0] 100006 D - 0xfffff80004c65900 [softirq_1] 100007 D - 0xfffff80004c65500 [if_io_tqg_0] 100008 D - 0xfffff80004c65100 [if_io_tqg_1] 100009 D - 0xfffff80004c62d00 [if_config_tqg_0] 100010 D - 0xfffff80004c61d00 [aiod_kick taskq] 100012 D - 0xfffff80004c61500 [kqueue_ctx taskq] 100013 D - 0xfffff80004c61100 [pci_hp taskq] 100015 D - 0xfffff80004c5a900 [inm_free taskq] 100017 D - 0xfffff80004c5a100 [linuxkpi_irq_wq] 100018 D - 0xfffff80004c55d00 [thread taskq] 100019 D - 0xfffff80004c55900 [in6m_free taskq] 100020 D - 0xfffff80004c55500 [linuxkpi_short_wq_0] 100021 D - 0xfffff80004c55500 [linuxkpi_short_wq_1] 100022 D - 0xfffff80004c55500 [linuxkpi_short_wq_2] 100023 D - 0xfffff80004c55500 [linuxkpi_short_wq_3] 100024 D - 0xfffff80004c55100 [linuxkpi_long_wq_0] 100025 D - 0xfffff80004c55100 [linuxkpi_long_wq_1] 100026 D - 0xfffff80004c55100 [linuxkpi_long_wq_2] 100027 D - 0xfffff80004c55100 [linuxkpi_long_wq_3] 100034 D - 0xfffff80004c3a900 [firmware taskq] 100038 D - 0xfffff80004c3a500 [crypto_0] 100039 D - 0xfffff80004c3a500 [crypto_1] 100055 D - 0xfffff800153dd900 [vtnet0 rxq 0] 100056 D - 0xfffff800153dd500 [vtnet0 txq 0] 100057 D - 0xfffff800153dd100 [vtnet0 rxq 1] 100058 D - 0xfffff800153c9d00 [vtnet0 txq 1] 100060 D vtbslp 0xfffff8001542b500 [virtio_balloon] 100064 D - 0xfffff800153c9900 [mca taskq] 100069 D - 0xffffffff81e22871 [deadlkres] 100074 D - 0xfffff80015a1d500 [acpi_task_0] 100075 D - 0xfffff80015a1d500 [acpi_task_1] 100076 D - 0xfffff80015a1d500 [acpi_task_2] 100077 D - 0xfffff80004c3a100 [CAM taskq] db> show all locks Process 31037 (syz-executor.2) thread 0xfffffe0095089c80 (103198) exclusive sleep mutex umtxql (umtxql) r = 0 (0xffffffff82761c60) locked @ /syzkaller/managers/main/kernel/sys/kern/kern_umtx.c:511 Process 12 (intr) thread 0xfffffe00042b9000 (100030) exclusive sleep mutex sctp-tcb (tcb) r = 0 (0xfffffe009e053380) locked @ /syzkaller/managers/main/kernel/sys/netinet/sctp_pcb.c:2143 db> show malloc Type InUse MemUse Requests sysctloid 34060 12773K 34127 pf_hash 5 11560K 5 devbuf 4216 6982K 4244 tcp_hpts 5 3219K 5 kobj 332 2656K 491 pcb 470 2134K 93702 vtbuf 24 2064K 46 filedesc 78 1247K 59999 newblk 9 1036K 30785 vfscache 3 1035K 3 subproc 206 864K 31161 sctp_atcl 767 767K 71878 acpica 1674 649K 55203 inodedep 127 646K 30151 callout 2 528K 2 ufs_quota 1 520K 1 vfs_hash 1 520K 1 sctp_stro 245 490K 16327 intr 4 480K 4 bus 995 380K 3503 sctp_atky 1011 380K 88558 BPF 46 337K 46 sctp_athm 767 288K 72013 linker 350 271K 467 sctp_stri 235 235K 6842 kdtrace 286 228K 65735 DEVFS1 107 214K 124 module 516 194K 516 umtx 440 193K 440 sctp_map 490 184K 32654 vnet_data 1 176K 1 tidhash 3 164K 3 pagedep 9 140K 29988 tfo_ccache 1 136K 1 sem 4 120K 4 vmem 3 104K 6 UMA 268 101K 268 audit_evclass 236 89K 294 mtx_pool 2 80K 2 temp 35 77K 2994 syncache 1 76K 1 routetbl 134 72K 418 msg 4 68K 4 ifaddr 71 67K 73 acpitask 1 64K 1 ddb_capture 1 64K 1 sctp_timw 126 63K 126 DEVFS3 126 63K 136 dirrem 118 59K 30046 ether_multi 157 59K 167 gtaskqueue 18 57K 18 DEVFS_RULE 56 54K 56 kenv 94 51K 94 eventhandler 133 50K 133 freefile 118 45K 30014 kqueue 96 44K 31150 ithread 99 43K 99 rman 84 42K 425 lltable 47 41K 124 KTRACE 100 38K 100 taskqueue 60 36K 60 pwddesc 94 36K 31045 proc 3 34K 3 ufs_mount 5 34K 6 bus-sc 33 34K 1713 devstat 4 33K 4 hostcache 1 32K 1 tty 16 32K 16 shm 1 32K 53 in6_multi 71 31K 71 DEVFSP 80 30K 390 GEOM 60 29K 487 kbdmux 6 28K 6 cred 27 27K 2287 plimit 25 25K 563 ifnet 7 25K 7 kcovinfo 64 24K 68 CAM queue 5 21K 1528 selfd 55 21K 403810 pfs_nodes 20 20K 20 proc-args 52 20K 757 session 37 19K 143 UART 12 18K 12 bmsafemap 2 17K 30141 ksem 1 16K 27 rpc 2 16K 2 shmfd 1 16K 40 pfs_vncache 1 16K 1 ufs_dirhash 24 14K 24 sglist 5 13K 5 MCA 32 12K 32 CAM DEV 3 12K 510 lockf 32 12K 127 vt 11 11K 11 Unitno 29 11K 55 acpisem 28 11K 28 CAM XPT 22 11K 543 inpcbpolicy 24 9K 5517 uidinfo 3 9K 24 pf_ifnet 10 9K 19 memdesc 1 8K 1 ipsec-saq 2 8K 2 evdev 4 8K 4 acpidev 20 8K 20 hhook 15 8K 17 select 19 8K 89 mount 16 7K 90 fpukern_ctx 3 6K 3 terminal 11 6K 11 osd 14 6K 4500 sctp_ifa 14 6K 15 ip6ndp 12 6K 15 ipsecpolicy 2 5K 2 encap_export_host 12 5K 12 clone 9 5K 9 local_apic 1 4K 1 io_apic 1 4K 1 sahead 1 4K 1 secasvar 1 4K 1 nhops 6 4K 8 pci_link 10 4K 10 msi 9 4K 9 DEVFS 9 4K 10 tun 7 4K 7 ipsec 3 3K 3 mkdir 8 3K 59932 in_multi 6 3K 8 nexusdev 7 3K 7 feeder 7 3K 7 procdesc 5 3K 18 freework 5 3K 29979 sctp_ifn 6 3K 15 mld 6 3K 6 igmp 6 3K 6 toponodes 6 3K 6 prison 6 3K 6 isadev 6 3K 6 softdep 1 2K 1 freeblks 4 2K 29964 vnodemarker 2 2K 120 NFSD session 1 2K 1 linux 5 2K 6 CAM periph 4 2K 271 soname 5 2K 40901 filecaps 5 2K 126 crypto 4 2K 85 newdirblk 4 2K 29966 diradd 4 2K 30090 pfil 4 2K 4 CAM path 4 2K 1034 tcpfunc 3 2K 3 loginclass 3 2K 6 chacha20random 1 1K 1 vnodes 1 1K 3 CAM SIM 2 1K 2 ktls 1 1K 1 cdev 2 1K 2 aesni_data 2 1K 2 cpus 2 1K 2 atkbddev 2 1K 2 CAM dev queue 2 1K 2 xform 2 1K 522 entropy 2 1K 50 NFSD lckfile 1 1K 1 NFSD V4client 1 1K 1 pmchooks 1 1K 1 sctp_vrf 1 1K 1 apmdev 1 1K 1 CAM I/O Scheduler 1 1K 1 ip_msource 1 1K 1 vnet_data_free 1 1K 1 vnet 1 1K 1 Per-cpu 1 1K 1 p1003.1b 1 1K 1 acpiintr 1 1K 1 pmc 1 1K 1 sctp_mcore 0 0K 0 sctp_socko 0 0K 22539 sctp_iter 0 0K 50 sctp_mvrf 0 0K 0 sctp_cpal 0 0K 40 sctp_cmsg 0 0K 0 sctp_stre 0 0K 0 sctp_athi 0 0K 0 sctp_a_it 0 0K 10 sctp_aadr 0 0K 0 tcp_do 0 0K 0 tcp_fsb 0 0K 502 cubic data 0 0K 0 mqdata 0 0K 0 htcp data 0 0K 0 vegas data 0 0K 81 cdg data 0 0K 0 chd data 0 0K 0 dctcp data 0 0K 81 pf_table 0 0K 0 pf_rule 0 0K 0 pf_altq 0 0K 0 pf_osfp 0 0K 0 pf_temp 0 0K 0 NFSD string 0 0K 0 NFSD V4lock 0 0K 0 madt_table 0 0K 2 smartpqi 0 0K 0 NFSD V4state 0 0K 0 NFSD srvcache 0 0K 0 msdosfs_fat 0 0K 0 msdosfs_mount 0 0K 0 msdosfs_node 0 0K 0 iavf 0 0K 0 ixl 0 0K 0 DEVFS4 0 0K 0 DEVFS2 0 0K 0 gntdev 0 0K 0 privcmd_dev 0 0K 0 ice-resmgr 0 0K 0 ice-osdep 0 0K 0 ice 0 0K 0 axgbe 0 0K 0 evtchn_dev 0 0K 0 xenstore 0 0K 0 ciss_data 0 0K 0 BACKLIGHT 0 0K 0 xnb 0 0K 0 xbbd 0 0K 0 xbd 0 0K 0 Balloon 0 0K 0 sysmouse 0 0K 0 vtfont 0 0K 0 xen_intr 0 0K 0 xen_hvm 0 0K 0 legacydrv 0 0K 0 qpidrv 0 0K 0 ath_hal 0 0K 0 athdev 0 0K 0 dmar_idpgtbl 0 0K 0 dmar_dom 0 0K 0 dmar_ctx 0 0K 0 ata_pci 0 0K 0 ata_dma 0 0K 0 ata_generic 0 0K 0 isci 0 0K 0 iommu_dmamap 0 0K 0 amr 0 0K 0 hyperv_socket 0 0K 0 bxe_ilt 0 0K 0 xenbus 0 0K 0 pvscsi 0 0K 0 scsi_da 0 0K 69 vm_fictitious 0 0K 0 ata_da 0 0K 0 scsi_ch 0 0K 0 scsi_cd 0 0K 0 AHCI driver 0 0K 0 USBdev 0 0K 0 USB 0 0K 0 agp 0 0K 0 nvme_da 0 0K 0 UMAHash 0 0K 0 acpipwr 0 0K 0 acpi_perf 0 0K 0 vm_pgdata 0 0K 0 jblocks 0 0K 0 savedino 0 0K 25118 sentinel 0 0K 0 jfsync 0 0K 0 jtrunc 0 0K 0 sbdep 0 0K 57 jsegdep 0 0K 0 jseg 0 0K 0 jfreefrag 0 0K 0 jfreeblk 0 0K 0 jnewblk 0 0K 0 jmvref 0 0K 0 jremref 0 0K 0 jaddref 0 0K 0 freedep 0 0K 0 freefrag 0 0K 50 allocindir 0 0K 0 indirdep 0 0K 25 allocdirect 0 0K 0 ufs_trim 0 0K 0 mactemp 0 0K 0 audit_trigger 0 0K 0 audit_pipe_presel 0 0K 0 audit_pipeent 0 0K 0 audit_pipe 0 0K 0 audit_evname 0 0K 0 audit_bsm 0 0K 0 audit_gidset 0 0K 0 audit_text 0 0K 0 audit_path 0 0K 0 audit_data 0 0K 0 audit_cred 0 0K 0 twsbuf 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5E_TLS 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EN 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5DUMP 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 seq_file 0 0K 0 radix 0 0K 0 idr 0 0K 0 lkpifw 0 0K 0 NLM 0 0K 0 ipsec-spdcache 0 0K 0 ipsec-reg 0 0K 0 ipsec-misc 0 0K 0 ipsecrequest 0 0K 0 ip6opt 0 0K 23 ip6_msource 0 0K 0 ip6_moptions 0 0K 0 in6_mfilter 0 0K 0 frag6 0 0K 3 tcplog 0 0K 0 tcp_hwpace 0 0K 0 twe_commands 0 0K 0 LRO 0 0K 0 newreno data 0 0K 0 ip_moptions 0 0K 6 in_mfilter 0 0K 4 ipid 0 0K 0 80211scan 0 0K 0 80211ratectl 0 0K 0 80211power 0 0K 0 80211nodeie 0 0K 0 80211node 0 0K 0 80211mesh_gt 0 0K 0 80211mesh_rt 0 0K 0 80211perr 0 0K 0 80211prep 0 0K 0 80211preq 0 0K 0 80211dfs 0 0K 0 80211crypto 0 0K 0 80211vap 0 0K 0 iflib 0 0K 0 vlan 0 0K 0 gif 0 0K 0 ifdescr 0 0K 0 zlib 0 0K 0 fadvise 0 0K 0 VN POLL 0 0K 0 twa_commands 0 0K 0 statfs 0 0K 30352 namei_tracker 0 0K 0 export_host 0 0K 0 cl_savebuf 0 0K 13 tcp_log_dev 0 0K 0 midi buffers 0 0K 0 mixer 0 0K 0 ac97 0 0K 0 hdacc 0 0K 0 hdac 0 0K 0 hdaa 0 0K 0 acpicmbat 0 0K 0 SIIS driver 0 0K 0 CAM CCB 0 0K 3255 PUC 0 0K 0 ppbusdev 0 0K 0 agtiapi_MemAlloc malloc 0 0K 0 osti_cacheable 0 0K 0 tempbuff 0 0K 0 biobuf 0 0K 0 aios 0 0K 0 lio 0 0K 0 acl 0 0K 0 tempbuff 0 0K 0 mbuf_tag 0 0K 149 ag_tgt_map_t malloc 0 0K 0 ag_slr_map_t malloc 0 0K 0 lDevFlags * malloc 0 0K 0 tiDeviceHandle_t * malloc 0 0K 0 ag_portal_data_t malloc 0 0K 0 ag_device_t malloc 0 0K 0 STLock malloc 0 0K 0 CCB List 0 0K 0 sr_iov 0 0K 0 OCS 0 0K 0 OCS 0 0K 0 nvme 0 0K 0 nvd 0 0K 0 netmap 0 0K 0 mwldev 0 0K 0 MVS driver 0 0K 0 CAM ccb queue 0 0K 0 mrsasbuf 0 0K 0 mpt_user 0 0K 0 mps_user 0 0K 0 accf 0 0K 0 pts 0 0K 0 iov 0 0K 42726 ioctlops 0 0K 376 eventfd 0 0K 0 Witness 0 0K 0 stack 0 0K 0 MPSSAS 0 0K 0 mps 0 0K 0 mpr_user 0 0K 0 MPRSAS 0 0K 0 mpr 0 0K 0 mfibuf 0 0K 0 sbuf 0 0K 288 md_sectors 0 0K 0 firmware 0 0K 0 compressor 0 0K 0 md_disk 0 0K 0 SWAP 0 0K 0 malodev 0 0K 0 LED 0 0K 0 sysctltmp 0 0K 1004 sysctl 0 0K 3 ekcd 0 0K 0 dumper 0 0K 0 sendfile 0 0K 0 rctl 0 0K 0 ix_sriov 0 0K 0 aacraidcam 0 0K 0 aacraid_buf 0 0K 0 ix 0 0K 0 ipsbuf 0 0K 0 cache 0 0K 0 iirbuf 0 0K 0 prison_racct 0 0K 0 Fail Points 0 0K 0 sigio 0 0K 74 filedesc_to_leader 0 0K 0 pwd 0 0K 0 tty console 0 0K 0 aaccam 0 0K 0 aacbuf 0 0K 0 zstd 0 0K 0 XZ_DEC 0 0K 0 nvlist 0 0K 0 SCSI ENC 0 0K 0 SCSI sa 0 0K 0 scsi_pass 0 0K 0 isofs_node 0 0K 0 isofs_mount 0 0K 0 tr_raid5_data 0 0K 0 tr_raid1e_data 0 0K 0 tr_raid1_data 0 0K 0 tr_raid0_data 0 0K 0 tr_concat_data 0 0K 0 md_sii_data 0 0K 0 md_promise_data 0 0K 0 md_nvidia_data 0 0K 0 md_jmicron_data 0 0K 0 md_intel_data 0 0K 0 md_ddf_data 0 0K 0 raid_data 0 0K 72 geom_flashmap 0 0K 0 tmpfs dir 0 0K 0 tmpfs name 0 0K 0 tmpfs mount 0 0K 0 NFS FHA 0 0K 0 newnfsmnt 0 0K 0 newnfsclient_req 0 0K 0 NFSCL layrecall 0 0K 0 NFSCL session 0 0K 0 NFSCL sockreq 0 0K 0 NFSCL devinfo 0 0K 0 NFSCL flayout 0 0K 0 NFSCL layout 0 0K 0 NFSD rollback 0 0K 0 NFSCL diroff 0 0K 0 NEWdirectio 0 0K 0 NEWNFSnode 0 0K 0 NFSCL lck 0 0K 0 NFSCL lckown 0 0K 0 NFSCL client 0 0K 0 NFSCL deleg 0 0K 0 NFSCL open 0 0K 0 NFSCL owner 0 0K 0 NFS fh 0 0K 0 NFS req 0 0K 0 NFSD usrgroup 0 0K 0 db> show uma Zone Size Used Free Requests Sleeps Bucket Total Mem XFree mbuf_jumbo_page 4096 8321 1169 876227 0 254 38871040 0 malloc-384 384 36270 5600 227365 0 30 16078080 0 malloc-8192 8192 157 1106 46050 0 3 10346496 0 malloc-1024 1024 4155 21 6533 0 16 4276224 0 malloc-1024 1024 1155 2225 79857 0 16 3461120 0 sctp_asoc 2288 244 1176 16327 0 254 3248960 0 mbuf 256 9499 2816 1943659 0 254 3152640 0 malloc-2048 2048 253 1129 16843 0 8 2830336 0 malloc-8192 8192 333 2 523 0 1 2744320 0 malloc-1024 1024 193 2403 30288 0 16 2658304 0 pbuf 2624 0 973 0 0 2 2553152 0 mbuf_cluster 2048 1245 1 1245 0 254 2551808 0 malloc-1024 1024 2 2190 25205 0 16 2244608 0 sctp_raddr 736 1419 1617 50016 0 254 2234496 0 BUF TRIE 144 190 13278 1623 0 62 1939392 0 UMA Slabs 0 112 17086 11 17086 0 126 1914864 0 sctp_ep 1280 288 1167 46047 0 254 1862400 0 malloc-384 384 2176 2404 263431 0 30 1758720 0 RADIX NODE 144 11639 398 742412 0 63 1733328 0 malloc-384 384 2010 2200 127307 0 30 1616640 0 malloc-32768 32768 41 6 30146 0 1 1540096 0 malloc-1024 1024 320 1116 46089 0 16 1470464 0 malloc-512 512 188 2420 91106 0 30 1335296 0 malloc-8192 8192 103 7 31106 0 1 901120 0 FFS inode 1160 545 29 30561 0 8 665840 0 socket 944 228 388 53233 0 254 581504 0 VM OBJECT 264 1782 93 439206 0 30 495000 0 malloc-16384 16384 26 1 30 0 1 442368 0 malloc-384 384 1009 41 4367 0 30 403200 0 THREAD 1808 191 29 34689 0 8 397760 0 lkpimm 160 1 2324 1 0 62 372000 0 lkpicurr 160 2 2323 2 0 62 372000 0 ertt_txseginfo 40 1 8786 279661 0 254 351480 0 MAP ENTRY 96 3256 356 1678249 0 126 346752 0 256 Bucket 2048 153 15 17858 0 8 344064 0 mbuf_packet 256 387 858 57874 0 254 318720 0 malloc-1024 1024 261 31 35804 0 16 299008 0 sctp_chunk 152 899 1051 30828 0 254 296400 0 VNODE 448 582 75 30601 0 30 294336 0 tcp_bbr_map 128 0 2263 47956 0 126 289664 0 malloc-384 384 684 26 1003 0 30 272640 0 malloc-65536 65536 4 0 4 0 1 262144 0 vmem btag 56 4573 23 4573 0 254 257376 0 malloc-16384 16384 10 5 273 0 1 245760 0 malloc-2048 2048 107 3 124 0 8 225280 0 DEVCTL 1024 0 216 132 0 0 221184 0 VMSPACE 2544 69 12 31020 0 4 206064 0 malloc-65536 65536 2 1 179 0 1 196608 0 malloc-65536 65536 0 3 4 0 1 196608 0 UMA Zones 768 240 4 240 0 16 187392 0 malloc-512 512 350 10 1229 0 30 184320 0 malloc-4096 4096 2 40 3258 0 2 172032 0 malloc-8192 8192 16 4 209 0 1 163840 0 32 Bucket 256 495 120 44364 0 62 157440 0 S VFS Cache 104 1107 336 31663 0 126 150072 0 PROC 1336 93 18 31044 0 8 148296 0 FFS2 dinode 256 545 25 30560 0 62 145920 0 malloc-65536 65536 2 0 2 0 1 131072 0 malloc-16384 16384 6 2 8 0 1 131072 0 malloc-384 384 266 74 434363 0 30 130560 0 sctp_stream_msg_out 112 77 1003 4696 0 254 120960 0 filedesc0 1072 94 18 31045 0 8 120064 0 tcpcb 1064 10 102 4488 0 254 119168 0 ksiginfo 112 96 948 11605 0 126 116928 0 malloc-512 512 199 17 329 0 30 110592 0 sctp_readq 152 92 532 2346 0 254 94848 0 16 Bucket 144 554 90 7908 0 62 92736 0 UMA Kegs 384 225 8 225 0 30 89472 0 128 Bucket 1024 56 31 7091 0 16 89088 0 clpbuf 2624 0 32 37 0 16 83968 0 tcp_inpcb 488 10 158 4488 0 254 81984 0 malloc-512 512 139 21 316 0 30 81920 0 malloc-512 512 93 59 30434 0 30 77824 0 g_bio 408 0 190 10460 0 30 77520 0 malloc-65536 65536 1 0 1 0 1 65536 0 malloc-65536 65536 1 0 1 0 1 65536 0 malloc-32768 32768 2 0 2 0 1 65536 0 malloc-32768 32768 2 0 2 0 1 65536 0 malloc-16384 16384 1 3 12 0 1 65536 0 malloc-16384 16384 2 2 6 0 1 65536 0 malloc-8192 8192 2 5 30359 0 1 57344 0 malloc-1024 1024 53 3 57 0 16 57344 0 malloc-384 384 87 53 22048 0 30 53760 0 malloc-16384 16384 3 0 3 0 1 49152 0 malloc-8192 8192 4 2 534 0 1 49152 0 malloc-4096 4096 5 7 554 0 2 49152 0 malloc-4096 4096 10 2 87 0 2 49152 0 malloc-512 512 21 75 33108 0 30 49152 0 malloc-2048 2048 9 13 1681 0 8 45056 0 malloc-2048 2048 18 4 22 0 8 45056 0 Files 80 293 257 95451 0 126 44000 0 DIRHASH 1024 34 6 34 0 16 40960 0 malloc-4096 4096 8 2 213 0 2 40960 0 malloc-512 512 19 61 30075 0 30 40960 0 pcpu-8 8 4687 433 4843 0 254 40960 0 64 Bucket 512 43 37 13370 0 30 40960 0 malloc-384 384 36 64 4560 0 30 38400 0 NAMEI 1024 0 36 138269 0 16 36864 0 TURNSTILE 136 221 31 221 0 62 34272 0 pipe 744 23 22 687 0 16 33480 0 malloc-4096 4096 6 2 23 0 2 32768 0 pcpu-64 64 486 26 486 0 254 32768 0 2 Bucket 32 638 370 11795 0 254 32256 0 tcp_bbr_pcb 832 1 35 942 0 16 29952 0 malloc-4096 4096 1 6 280 0 2 28672 0 malloc-4096 4096 7 0 7 0 2 28672 0 KNOTE 160 28 147 372137 0 62 28000 0 malloc-2048 2048 9 3 11 0 8 24576 0 malloc-2048 2048 4 8 85 0 8 24576 0 malloc-512 512 33 15 2469 0 30 24576 0 sctp_laddr 48 8 496 4538 0 254 24192 0 PWD 32 41 715 30065 0 254 24192 0 ttyinq 160 135 15 300 0 62 24000 0 8 Bucket 80 184 116 6197 0 126 24000 0 udp_inpcb 488 9 39 445 0 254 23424 0 ttyoutq 256 72 18 160 0 62 23040 0 SLEEPQUEUE 88 221 35 221 0 126 22528 0 tcp_rack_pcb 832 0 27 502 0 16 22464 0 malloc-2048 2048 9 1 9 0 8 20480 0 vtnet_tx_hdr 24 0 835 672382 0 254 20040 0 Mountpoints 2752 2 5 2 0 4 19264 0 unpcb 256 24 51 1596 0 254 19200 0 malloc-16384 16384 1 0 1 0 1 16384 0 malloc-16384 16384 1 0 1 0 1 16384 0 ripcb 488 5 27 385 0 254 15616 0 udplite_inpcb 488 0 32 199 0 254 15616 0 ertt 72 10 158 4488 0 126 12096 0 4 Bucket 48 10 242 702 0 254 12096 0 itimer 352 0 33 20 0 30 11616 0 kenv 258 15 30 1059 0 30 11610 0 malloc-8192 8192 0 1 2 0 1 8192 0 malloc-4096 4096 0 2 163 0 2 8192 0 malloc-2048 2048 0 4 9 0 8 8192 0 malloc-1024 1024 0 8 102 0 16 8192 0 rtentry 176 31 15 35 0 62 8096 0 PGRP 88 37 55 143 0 126 8096 0 rl_entry 40 99 103 99 0 254 8080 0 tcp_rack_map 112 0 72 54 0 126 8064 0 udpcb 32 9 243 644 0 254 8064 0 routing nhops 256 27 3 35 0 62 7680 0 tcp_log 416 0 18 4 0 254 7488 0 FPU_save_area 832 1 8 1 0 16 7488 0 cpuset 104 7 55 24 0 126 6448 0 domainset 40 0 126 13 0 254 5040 0 epoch_record pcpu 256 4 12 4 0 62 4096 0 pcpu-16 16 7 249 7 0 254 4096 0 tcptw 88 0 46 2 0 254 4048 0 sackhole 32 0 126 1 0 254 4032 0 hostcache 64 1 62 1 0 254 4032 0 syncache 168 0 24 4 0 254 4032 0 UMA Slabs 1 176 9 13 9 0 62 3872 0 mqnode 416 3 6 3 0 30 3744 0 KMAP ENTRY 96 12 27 12 0 0 3744 0 vmem 1856 1 1 1 0 8 3712 0 SMR CPU 32 3 60 3 0 254 2016 0 SMR SHARED 24 3 60 3 0 254 1512 0 FFS1 dinode 128 0 0 0 0 126 0 0 swblk 136 0 0 0 0 62 0 0 swpctrie 144 0 0 0 0 62 0 0 sctp_asconf_ack 48 0 0 0 0 254 0 0 sctp_asconf 40 0 0 0 0 254 0 0 cdg_qdiffsample 16 0 0 0 0 254 0 0 pf state scrubs 40 0 0 0 0 254 0 0 pf frag entries 40 0 0 0 0 254 0 0 pf frags 248 0 0 0 0 62 0 0 pf table entries 160 0 0 0 0 62 0 0 pf table entry counters 64 0 0 0 0 254 0 0 pf source nodes 136 0 0 0 0 254 0 0 pf state keys 88 0 0 0 0 126 0 0 pf states 296 0 0 0 0 254 0 0 pf tags 104 0 0 0 0 126 0 0 pf mtags 48 0 0 0 0 254 0 0 IPsec SA lft_c 16 0 0 0 0 254 0 0 tcp_log_node 120 0 0 0 0 126 0 0 tcp_log_bucket 176 0 0 0 0 62 0 0 tcpreass 48 0 0 0 0 254 0 0 tfo_ccache_entries 80 0 0 0 0 126 0 0 tfo 4 0 0 0 0 254 0 0 ipq 56 0 0 0 0 254 0 0 AIOLIO 272 0 0 0 0 30 0 0 AIOCB 552 0 0 0 0 16 0 0 AIOP 32 0 0 0 0 254 0 0 AIO 208 0 0 0 0 62 0 0 NCLNODE 584 0 0 0 0 16 0 0 mqnotifier 216 0 0 0 0 62 0 0 mvdata 64 0 0 0 0 254 0 0 mqueue 248 0 0 0 0 62 0 0 TMPFS node 224 0 0 0 0 62 0 0 LTS VFS Cache 360 0 0 0 0 30 0 0 L VFS Cache 320 0 0 0 0 30 0 0 STS VFS Cache 144 0 0 0 0 62 0 0 cryptop 280 0 0 0 0 30 0 0 linux_dma_object 24 0 0 0 0 254 0 0 linux_dma_pctrie 144 0 0 0 0 62 0 0 IOMMU_MAP_ENTRY 120 0 0 0 0 126 0 0 ktls_session 192 0 0 0 0 62 0 0 mbuf_jumbo_16k 16384 0 0 0 0 254 0 0 mbuf_jumbo_9k 9216 0 0 0 0 254 0 0 audit_record 1280 0 0 0 0 8 0 0 MAC labels 40 0 0 0 0 254 0 0 vnpbuf 2624 0 0 0 0 64 0 0 mdpbuf 2624 0 0 0 0 3 0 0 nfspbuf 2624 0 0 0 0 16 0 0 swwbuf 2624 0 0 0 0 8 0 0 swrbuf 2624 0 0 0 0 16 0 0 umtx_shm 88 0 0 0 0 126 0 0 umtx pi 96 0 0 0 0 126 0 0 rangeset pctrie nodes 144 0 0 0 0 62 0 0 malloc-65536 65536 0 0 0 0 1 0 0 malloc-65536 65536 0 0 0 0 1 0 0 malloc-32768 32768 0 0 0 0 1 0 0 malloc-32768 32768 0 0 0 0 1 0 0 malloc-32768 32768 0 0 0 0 1 0 0 malloc-32768 32768 0 0 0 0 1 0 0 malloc-32768 32768 0 0 0 0 1 0 0 malloc-8192 8192 0 0 0 0 1 0 0 malloc-256 256 0 0 0 0 62 0 0 malloc-256 256 0 0 0 0 62 0 0 malloc-256 256 0 0 0 0 62 0 0 malloc-256 256 0 0 0 0 62 0 0 malloc-256 256 0 0 0 0 62 0 0 malloc-256 256 0 0 0 0 62 0 0 malloc-256 256 0 0 0 0 62 0 0 malloc-256 256 0 0 0 0 62 0 0 malloc-128 128 0 0 0 0 126 0 0 malloc-128 128 0 0 0 0 126 0 0 malloc-128 128 0 0 0 0 126 0 0 malloc-128 128 0 0 0 0 126 0 0 malloc-128 128 0 0 0 0 126 0 0 malloc-128 128 0 0 0 0 126 0 0 malloc-128 128 0 0 0 0 126 0 0 malloc-128 128 0 0 0 0 126 0 0 malloc-64 64 0 0 0 0 254 0 0 malloc-64 64 0 0 0 0 254 0 0 malloc-64 64 0 0 0 0 254 0 0 malloc-64 64 0 0 0 0 254 0 0 malloc-64 64 0 0 0 0 254 0 0 malloc-64 64 0 0 0 0 254 0 0 malloc-64 64 0 0 0 0 254 0 0 malloc-64 64 0 0 0 0 254 0 0 malloc-32 32 0 0 0 0 254 0 0 malloc-32 32 0 0 0 0 254 0 0 malloc-32 32 0 0 0 0 254 0 0 malloc-32 32 0 0 0 0 254 0 0 malloc-32 32 0 0 0 0 254 0 0 malloc-32 32 0 0 0 0 254 0 0 malloc-32 32 0 0 0 0 254 0 0 malloc-32 32 0 0 0 0 254 0 0 malloc-16 16 0 0 0 0 254 0 0 malloc-16 16 0 0 0 0 254 0 0 malloc-16 16 0 0 0 0 254 0 0 malloc-16 16 0 0 0 0 254 0 0 malloc-16 16 0 0 0 0 254 0 0 malloc-16 16 0 0 0 0 254 0 0 malloc-16 16 0 0 0 0 254 0 0 malloc-16 16 0 0 0 0 254 0 0 pcpu-32 32 0 0 0 0 254 0 0 pcpu-4 4 0 0 0 0 254 0 0 fakepg 104 0 0 0 0 126 0 0 UMA Hash 256 0 0 0 0 62 0 0