panic: lock (sleep mutex) sctp-inp not locked @ /syzkaller/managers/main/kernel/sys/netinet/sctp_usrreq.c:945 cpuid = 1 time = 1705466981 KDB: stack backtrace: db_trace_self_wrapper() at db_trace_self_wrapper+0xc6/frame 0xfffffe007420d5b0 kdb_backtrace() at kdb_backtrace+0xd0/frame 0xfffffe007420d710 vpanic() at vpanic+0x271/frame 0xfffffe007420d8b0 panic() at panic+0xb5/frame 0xfffffe007420d970 witness_unlock() at witness_unlock+0x4fd/frame 0xfffffe007420d9f0 __mtx_unlock_flags() at __mtx_unlock_flags+0xec/frame 0xfffffe007420dab0 sctp_shutdown() at sctp_shutdown+0x1057/frame 0xfffffe007420dc10 soshutdown() at soshutdown+0xb3/frame 0xfffffe007420dc50 kern_shutdown() at kern_shutdown+0x154/frame 0xfffffe007420dd10 amd64_syscall() at amd64_syscall+0x473/frame 0xfffffe007420df30 fast_syscall_common() at fast_syscall_common+0xf8/frame 0xfffffe007420df30 --- syscall (198, FreeBSD ELF64, __syscall), rip = 0x2aeb2a, rsp = 0x825bccf08, rbp = 0x825bccf80 --- KDB: enter: panic [ thread pid 3087 tid 103349 ] Stopped at kdb_enter+0x6e: movq $0,0x2194267(%rip) db> db> set $lines = 0 db> set $maxwidth = 0 db> show registers cs 0x20 ds 0x3b es 0x3b fs 0x13 gs 0x1b ss 0x28 rax 0x12 rcx 0xfffffe006ec00000 rdx 0x3ffff rbx 0xffffffff826f86c0 .str.28 rsp 0xfffffe007420d6f0 rbp 0xfffffe007420d710 rsi 0x40001 rdi 0xffffffff815de326 printf+0x156 r8 0 r9 0xffffffff r10 0x1 r11 0 r12 0 r13 0xfffffe0058ad4000 r14 0xffffffff826f86c0 .str.28 r15 0 rip 0xffffffff815cadae kdb_enter+0x6e rflags 0x46 kdb_enter+0x6e: movq $0,0x2194267(%rip) db> show proc Process 3087 (syz-executor.2) at 0xfffffe0074b6a5a0: state: NORMAL uid: 0 gids: 0, 0, 5 parent: pid 776 at 0xfffffe006cd7b5a0 ABI: FreeBSD ELF64 flag: 0x10000080 flag2: 0 arguments: /root/syz-executor.2 exec reaper: 0xfffffe00541d1040 reapsubtree: 1 sigparent: 20 vmspace: 0xfffffe0074bad268 (map 0xfffffe0074bad268) (map.pmap 0xfffffe0074bad328) (pmap 0xfffffe0074bad398) threads: 2 103278 RunQ syz-executor.2 103349 Run CPU 1 syz-executor.2 db> ps pid ppid pgrp uid state wmesg wchan cmd 3087 776 776 0 R (threaded) syz-executor.2 103278 RunQ syz-executor.2 103349 Run CPU 1 syz-executor.2 3086 802 802 0 R (threaded) syz-executor.3 103074 RunQ syz-executor.3 103348 S uwait 0xfffffe0057a77800 syz-executor.3 3085 771 771 0 R (threaded) syz-executor.1 103079 Run CPU 0 syz-executor.1 103347 S sbwait 0xfffffe006a312a1c syz-executor.1 3083 770 770 0 S (threaded) syz-executor.0 103293 S nanslp 0xffffffff83740401 syz-executor.0 103344 S connec 0xfffffe0058ac5c1a syz-executor.0 103346 S uwait 0xfffffe006cd4b300 syz-executor.0 2801 2794 2801 0 Ss select 0xfffffe0057a92bc0 dhclient 2797 1 2797 0 Ss select 0xfffffe0057a92c40 dhclient 2794 2786 424 65 S select 0xfffffe0057a92d40 dhclient 2786 424 424 0 S wait 0xfffffe006cb07580 sh 2776 1 2776 65 Ss select 0xfffffe0057a92e40 dhclient 2769 1 802 0 S uwait 0xfffffe005750b800 syz-executor.3 2768 1 771 0 S uwait 0xfffffe0073ae9f00 syz-executor.1 2766 1 802 0 S uwait 0xfffffe0074a69900 syz-executor.3 2762 1 771 0 S uwait 0xfffffe005750d680 syz-executor.1 2760 1 802 0 S uwait 0xfffffe005750b900 syz-executor.3 2759 1 771 0 S uwait 0xfffffe006cd4b600 syz-executor.1 2756 1 771 0 S uwait 0xfffffe0074a68a80 syz-executor.1 2406 1 802 0 S uwait 0xfffffe006cd4bb00 syz-executor.3 2403 1 771 0 S uwait 0xfffffe0057a76580 syz-executor.1 2401 1 771 0 S uwait 0xfffffe0057a77580 syz-executor.1 2399 1 771 0 S uwait 0xfffffe006cd4c480 syz-executor.1 2397 1 770 0 S uwait 0xfffffe006cd4c800 syz-executor.0 2396 1 771 0 S uwait 0xfffffe0073aea780 syz-executor.1 2382 1 802 0 S uwait 0xfffffe0073aea480 syz-executor.3 2374 1 770 0 S uwait 0xfffffe006cd4d000 syz-executor.0 2369 1 802 0 S uwait 0xfffffe006cd4c080 syz-executor.3 2368 1 770 0 S uwait 0xfffffe0073aea380 syz-executor.0 2365 1 771 0 S uwait 0xfffffe006cd4c280 syz-executor.1 2363 1 776 0 S uwait 0xfffffe005750d780 syz-executor.2 2361 1 770 0 S uwait 0xfffffe006cd4ca80 syz-executor.0 2359 1 802 0 S uwait 0xfffffe0073aea580 syz-executor.3 2357 1 771 0 S uwait 0xfffffe006cd4dd00 syz-executor.1 2355 1 776 0 S uwait 0xfffffe006cd4b800 syz-executor.2 2353 1 770 0 S uwait 0xfffffe0057a76080 syz-executor.0 2351 1 802 0 S uwait 0xfffffe006cd4b700 syz-executor.3 2350 1 771 0 S uwait 0xfffffe006cd4dd80 syz-executor.1 2347 1 776 0 S uwait 0xfffffe0057a76480 syz-executor.2 2346 1 770 0 S uwait 0xfffffe006cd4c680 syz-executor.0 2339 1 802 0 S uwait 0xfffffe005750aa00 syz-executor.3 2332 1 802 0 S uwait 0xfffffe006cd4cb80 syz-executor.3 2317 0 0 0 DL - 0xffffffff83850240 [soaiod4] 2316 0 0 0 DL - 0xffffffff83850240 [soaiod3] 2315 0 0 0 DL - 0xffffffff83850240 [soaiod2] 2314 0 0 0 DL - 0xffffffff83850240 [soaiod1] 2150 1 2150 0 Ss select 0xfffffe0057a93540 dhclient 2147 1 2147 0 Ss select 0xfffffe0057a934c0 dhclient 2124 1 2124 65 Ss select 0xfffffe00571b1f40 dhclient 1847 0 0 0 DL aiordy 0xfffffe006cd79560 [aiod4] 1846 0 0 0 DL aiordy 0xfffffe006cd7b040 [aiod3] 1845 0 0 0 DL aiordy 0xfffffe006d191ac0 [aiod2] 1844 0 0 0 DL aiordy 0xfffffe007407d020 [aiod1] 1750 1 771 0 S uwait 0xfffffe0073aea680 syz-executor.1 1748 1 771 0 S uwait 0xfffffe005750a500 syz-executor.1 1746 1 771 0 S uwait 0xfffffe006cd4d200 syz-executor.1 1744 1 771 0 S uwait 0xfffffe005750a080 syz-executor.1 1648 1 1647 0 S uwait 0xfffffe0057a76280 syz-executor.3 1646 1 1645 0 S uwait 0xfffffe0073ae9b00 syz-executor.3 1643 1 1642 0 S uwait 0xfffffe0073ae9e00 syz-executor.3 1641 1 1640 0 S uwait 0xfffffe0073ae9a00 syz-executor.3 1111 1 1111 0 Ss select 0xfffffe00571b1ec0 dhclient 1107 1 1107 0 Ss select 0xfffffe00571b1e40 dhclient 802 768 802 0 Ss nanslp 0xffffffff83740400 syz-executor.3 776 768 776 0 Ss nanslp 0xffffffff83740400 syz-executor.2 771 768 771 0 Ss nanslp 0xffffffff83740401 syz-executor.1 770 768 770 0 Ss nanslp 0xffffffff83740401 syz-executor.0 768 766 766 0 S (threaded) syz-fuzzer 100113 S uwait 0xfffffe0057a76d00 syz-fuzzer 100114 S uwait 0xfffffe006cd4de80 syz-fuzzer 100115 S uwait 0xfffffe006cd4e000 syz-fuzzer 100116 S uwait 0xfffffe006cd4e100 syz-fuzzer 100117 S uwait 0xfffffe006cd4e200 syz-fuzzer 100118 S uwait 0xfffffe0057a76e00 syz-fuzzer 100119 S wait 0xfffffe006d191000 syz-fuzzer 100120 S wait 0xfffffe006d191000 syz-fuzzer 100123 S wait 0xfffffe006d191000 syz-fuzzer 100126 S uwait 0xfffffe006cd4e400 syz-fuzzer 100131 S wait 0xfffffe006d191000 syz-fuzzer 100144 S uwait 0xfffffe006cd4e600 syz-fuzzer 100630 S kqread 0xfffffe0058aecb00 syz-fuzzer 766 764 766 0 Ss pause 0xfffffe0057b9b630 csh 764 682 764 0 Ss select 0xfffffe00571b60c0 sshd 748 1 748 0 Ss+ ttyin 0xfffffe005766fcb0 getty 747 1 747 0 Ss+ ttyin 0xfffffe00587670b0 getty 746 1 746 0 Ss+ ttyin 0xfffffe00587674b0 getty 745 1 745 0 Ss+ ttyin 0xfffffe00587678b0 getty 744 1 744 0 Ss+ ttyin 0xfffffe0058767cb0 getty 743 1 743 0 Ss+ ttyin 0xfffffe00587680b0 getty 742 1 742 0 Ss+ ttyin 0xfffffe00587684b0 getty 741 1 741 0 Ss+ ttyin 0xfffffe00587688b0 getty 740 1 740 0 Ss+ ttyin 0xfffffe0058768cb0 getty 686 1 686 0 Ss nanslp 0xffffffff83740400 cron 682 1 682 0 Ss select 0xfffffe00571b63c0 sshd 495 1 495 0 Ss select 0xfffffe00571b62c0 syslogd 424 1 424 0 Ss wait 0xfffffe0057b9bae0 devd 423 1 423 65 Ss select 0xfffffe0057a93b40 dhclient 338 1 338 0 Ss select 0xfffffe00571b6340 dhclient 335 1 335 0 Ss select 0xfffffe0057a93940 dhclient 17 0 0 0 DL syncer 0xffffffff8385db20 [syncer] 16 0 0 0 DL vlruwt 0xfffffe0057181040 [vnlru] 15 0 0 0 DL (threaded) [bufdaemon] 100079 D psleep 0xffffffff8385c100 [bufdaemon] 100080 D - 0xffffffff82c0a140 [bufspacedaemon-0] 100092 D sdflush 0xfffffe00588a24e8 [/ worker] 9 0 0 0 DL psleep 0xffffffff838a55c0 [vmdaemon] 8 0 0 0 DL (threaded) [pagedaemon] 100077 D psleep 0xffffffff8388d438 [dom0] 100081 D launds 0xffffffff8388d444 [laundry: dom0] 100082 D umarcl 0xffffffff81d61000 [uma] 7 0 0 0 DL - 0xffffffff834bdc10 [rand_harvestq] 6 0 0 0 DL pftm 0xffffffff843a3ea0 [pf purge] 5 0 0 0 DL waiting 0xffffffff840a85a0 [sctp_iterator] 4 0 0 0 DL (threaded) [cam] 100044 D - 0xffffffff83488340 [doneq0] 100045 D - 0xffffffff834882c0 [async] 100076 D - 0xffffffff83488140 [scanner] 3 0 0 0 DL (threaded) [crypto] 100041 D crypto_ 0xffffffff83888c60 [crypto] 100042 D crypto_ 0xfffffe00542d5730 [crypto returns 0] 100043 D crypto_ 0xfffffe00542d5780 [crypto returns 1] 14 0 0 0 DL seqstat 0xfffffe000796e888 [sequencer 00] 13 0 0 0 DL (threaded) [geom] 100035 D - 0xffffffff836e8d40 [g_event] 100036 D - 0xffffffff836e8d60 [g_up] 100037 D - 0xffffffff836e8d80 [g_down] 2 0 0 0 WL (threaded) [clock] 100029 I [clock (0)] 100030 I [clock (1)] 12 0 0 0 WL (threaded) [intr] 100012 I [swi6: task queue] 100013 I [swi6: Giant taskq] 100015 I [swi5: fast taskq] 100031 I [swi1: netisr 0] 100032 I [swi1: hpts] 100033 I [swi1: hpts] 100046 I [irq24: virtio_pci0] 100047 I [irq25: virtio_pci0] 100048 I [irq26: virtio_pci0] 100049 I [irq27: virtio_pci0] 100050 I [irq28: virtio_pci1] 100051 I [irq29: virtio_pci1] 100052 I [irq30: virtio_pci1] 100053 I [irq31: virtio_pci1] 100054 I [irq32: virtio_pci1] 100059 I [irq33: virtio_pci2] 100060 I [irq34: virtio_pci2] 100061 I [irq35: virtio_pci2] 100063 I [irq1: atkbd0] 100064 I [irq12: psm0] 100065 I [swi0: uart uart++] 100069 I [swi1: pf send] 11 0 0 0 RL (threaded) [idle] 100003 CanRun [idle: cpu0] 100004 CanRun [idle: cpu1] 1 0 1 0 SLs wait 0xfffffe00541d1040 [init] 10 0 0 0 DL audit_w 0xffffffff838896c0 [audit] 0 0 0 0 DLs (threaded) [kernel] 100000 D swapin 0xffffffff836e9720 [swapper] 100005 D - 0xfffffe0054282100 [softirq_0] 100006 D - 0xfffffe0054282000 [softirq_1] 100007 D - 0xfffffe0054281e00 [if_io_tqg_0] 100008 D - 0xfffffe0054281d00 [if_io_tqg_1] 100009 D - 0xfffffe0054281c00 [if_config_tqg_0] 100010 D - 0xfffffe0007977900 [pci_hp taskq] 100011 D - 0xfffffe0007977800 [kqueue_ctx taskq] 100014 D - 0xfffffe0007977500 [thread taskq] 100016 D - 0xfffffe0007977300 [aiod_kick taskq] 100017 D - 0xfffffe0007977200 [deferred_unmount ta] 100018 D - 0xfffffe0007977100 [inm_free taskq] 100019 D - 0xfffffe0007977000 [in6m_free taskq] 100020 D - 0xfffffe0007976e00 [linuxkpi_irq_wq] 100021 D - 0xfffffe0007976d00 [linuxkpi_short_wq_0] 100022 D - 0xfffffe0007976d00 [linuxkpi_short_wq_1] 100023 D - 0xfffffe0007976d00 [linuxkpi_short_wq_2] 100024 D - 0xfffffe0007976d00 [linuxkpi_short_wq_3] 100025 D - 0xfffffe0007976c00 [linuxkpi_long_wq_0] 100026 D - 0xfffffe0007976c00 [linuxkpi_long_wq_1] 100027 D - 0xfffffe0007976c00 [linuxkpi_long_wq_2] 100028 D - 0xfffffe0007976c00 [linuxkpi_long_wq_3] 100034 D - 0xfffffe0007976a00 [firmware taskq] 100039 D - 0xfffffe0007976700 [crypto_0] 100040 D - 0xfffffe0007976700 [crypto_1] 100055 D - 0xfffffe00571c4b00 [vtnet0 rxq 0] 100056 D - 0xfffffe00571c4a00 [vtnet0 txq 0] 100057 D - 0xfffffe00571c4900 [vtnet0 rxq 1] 100058 D - 0xfffffe00571c4800 [vtnet0 txq 1] 100062 D vtbslp 0xfffffe00571b6580 [virtio_balloon] 100066 D - 0xffffffff826fd920 [deadlkres] 100070 D - 0xfffffe0057bbd100 [acpi_task_0] 100071 D - 0xfffffe0057bbd100 [acpi_task_1] 100072 D - 0xfffffe0057bbd100 [acpi_task_2] 100074 D - 0xfffffe0007977c00 [mca taskq] 100075 D - 0xfffffe0007976500 [CAM taskq] db> show all locks Process 3087 (syz-executor.2) thread 0xfffffe0058ad4000 (103349) exclusive sleep mutex sctp-tcb (tcb) r = 0 (0xfffffe0073c2aa50) locked @ /syzkaller/managers/main/kernel/sys/netinet/sctp_usrreq.c:888 Process 3085 (syz-executor.1) thread 0xfffffe0074a64740 (103079) exclusive sleep mutex umtxql (umtxql) r = 0 (0xffffffff837527c0) locked @ /syzkaller/managers/main/kernel/sys/kern/kern_umtx.c:1303 Process 3085 (syz-executor.1) thread 0xfffffe0074a40740 (103347) exclusive sx so_snd_sx (so_snd_sx) r = 0 (0xfffffe006a3128c0) locked @ /syzkaller/managers/main/kernel/sys/kern/uipc_socket.c:3992 db> show malloc Type InUse MemUse Requests pf_hash 5 11524K 5 tcp_hpts 7 4801K 7 devbuf 4192 4324K 4220 sysctloid 34817 2052K 34888 vtbuf 24 1968K 46 kobj 326 1304K 488 pcb 374 1060K 6263 newblk 120 1054K 12487 vfscache 3 1025K 3 filedesc 94 746K 4301 inodedep 181 580K 2728 ufs_quota 1 512K 1 vfs_hash 1 512K 1 callout 2 512K 2 intr 4 472K 4 subproc 223 450K 3214 vmem 3 266K 6 vnet_data 2 224K 2 acpitask 1 224K 1 filemon 23 184K 56 acpica 1674 184K 60830 sctp_stro 150 150K 1120 tidhash 3 141K 3 pagedep 17 132K 2187 linker 352 130K 386 tfo_ccache 1 128K 1 IP reass 1 128K 1 sctp_atcl 326 123K 4398 DEVFS1 109 109K 126 sem 4 106K 4 gtaskqueue 18 98K 18 bus 985 81K 5155 mtx_pool 2 72K 2 BPF 38 71K 169 syncache 1 68K 1 NFSD srvcache 3 68K 3 module 513 65K 513 ddb_capture 1 64K 1 kdtrace 302 57K 6438 umtx 448 56K 448 dirrem 159 40K 2540 temp 36 39K 2271 hostcache 1 32K 1 shm 1 32K 15 DEVFS3 128 32K 138 msg 4 30K 4 kbdmux 6 28K 6 LRO 22 23K 22 freefile 163 21K 2460 DEVFS_RULE 56 20K 56 sctp_atky 476 20K 5775 ifaddr 69 20K 71 routetbl 138 19K 494 ufs_mount 4 17K 5 proc 3 17K 3 tty 16 16K 16 ithread 97 16K 97 freework 60 15K 6315 sctp_timw 59 15K 59 lltable 46 15K 82 bus-sc 34 15K 1687 eventhandler 161 14K 161 KTRACE 101 13K 764 ifnet 7 13K 7 ether_multi 152 13K 162 kenv 95 12K 95 GEOM 61 11K 481 rman 86 11K 431 CAM queue 5 11K 1528 in6_multi 65 9K 65 kqueue 103 9K 3109 bmsafemap 2 9K 2743 rpc 4 9K 4 devstat 4 9K 4 UART 12 9K 12 ksem 1 8K 3 shmfd 1 8K 1 pfs_vncache 1 8K 1 audit_evclass 238 8K 300 taskqueue 63 7K 63 pwddesc 101 7K 3088 cred 25 7K 283 sglist 6 7K 6 CAM DEV 3 6K 510 plimit 23 6K 379 DEVFSP 90 6K 338 sctp_athm 326 6K 4600 pfs_nodes 20 5K 20 hhook 15 5K 17 sctp_map 300 5K 2408 ufs_dirhash 24 5K 24 pf_ifnet 12 5K 33 session 35 5K 74 UMA 267 5K 267 tcp_fsb 2 5K 842 vt 11 5K 11 pf_table 2 4K 21 memdesc 1 4K 1 MCA 32 4K 32 evdev 4 4K 4 proc-args 126 4K 4314 acpisem 28 4K 28 CC Mem 13 4K 1197 kcovinfo 52 4K 52 lockf 27 3K 59 selfd 45 3K 46518 freeblks 11 3K 2626 terminal 11 3K 11 clone 9 3K 9 uidinfo 3 3K 18 local_apic 1 2K 1 io_apic 1 2K 1 ipsec-saq 2 2K 2 select 16 2K 101 ip6ndp 12 2K 13 Unitno 30 2K 65 sctp_ifa 13 2K 14 tun 7 2K 7 CAM XPT 22 2K 543 msi 12 2K 12 in_multi 6 2K 8 toponodes 6 2K 6 newdirblk 11 2K 2152 ipsecpolicy 2 2K 2 acpidev 20 2K 20 sctp_stri 2 1K 396 softdep 1 1K 1 mkdir 8 1K 4304 indirdep 4 1K 5562 sahead 1 1K 1 secasvar 1 1K 1 ip6opt 4 1K 15 nhops 6 1K 8 vnodemarker 2 1K 36 NFSD session 1 1K 1 CAM periph 4 1K 271 ipsec 3 1K 3 sctp_ifn 6 1K 14 inpcbpolicy 24 1K 1528 mld 6 1K 6 igmp 6 1K 6 pfil 6 1K 6 isadev 6 1K 6 mount 16 1K 89 pci_link 10 1K 10 diradd 5 1K 2577 crypto 4 1K 26 encap_export_host 12 1K 12 osd 18 1K 1196 procdesc 4 1K 16 cdev 2 1K 2 lkpikmalloc 8 1K 9 netlink 2 1K 64 chacha20random 1 1K 1 biobuf 1 1K 1 vnodes 1 1K 1 NFSD lckfile 1 1K 1 NFSD V4client 1 1K 1 DEVFS 9 1K 10 CAM SIM 2 1K 2 feeder 7 1K 7 cryptodev 3 1K 196 tcpfunc 3 1K 3 loginclass 3 1K 6 prison 6 1K 6 pf_rule 1 1K 15 nexusdev 8 1K 8 apmdev 1 1K 1 atkbddev 2 1K 2 aio 4 1K 14 soname 5 1K 7626 pmchooks 1 1K 1 CAM path 4 1K 1034 CAM dev queue 2 1K 2 CAM I/O Scheduler 1 1K 1 sctp_vrf 1 1K 1 vnet 1 1K 1 pmc 1 1K 1 filecaps 4 1K 104 entropy 2 1K 41 acpiintr 1 1K 1 cpus 2 1K 2 vnet_data_free 1 1K 1 Per-cpu 1 1K 1 p1003.1b 1 1K 1 mqdata 0 0K 0 ipcomp 0 0K 0 esp 0 0K 0 ah 0 0K 0 pf_altq 0 0K 0 pf_osfp 0 0K 0 pf_krule_item 0 0K 0 pf_temp 0 0K 0 tcp_do 0 0K 0 sctp_mcore 0 0K 0 sctp_socko 0 0K 1945 sctp_iter 0 0K 12 sctp_mvrf 0 0K 0 sctp_cpal 0 0K 0 sctp_cmsg 0 0K 0 sctp_stre 0 0K 0 sctp_athi 0 0K 0 sctp_a_it 0 0K 12 sctp_aadr 0 0K 0 madt_table 0 0K 2 smartpqi 0 0K 0 ixl 0 0K 0 ice-resmgr 0 0K 0 ice-osdep 0 0K 0 ice 0 0K 0 iavf 0 0K 0 axgbe 0 0K 0 fpukern_ctx 0 0K 0 xen_intr 0 0K 0 xen_hvm 0 0K 0 legacydrv 0 0K 0 bounce 0 0K 0 busdma 0 0K 0 qpidrv 0 0K 0 dmar_idpgtbl 0 0K 0 dmar_dom 0 0K 0 dmar_ctx 0 0K 0 isci 0 0K 0 iommu_dmamap 0 0K 0 hyperv_socket 0 0K 0 bxe_ilt 0 0K 0 aesni_data 0 0K 0 xenbus 0 0K 0 vm_fictitious 0 0K 0 UMAHash 0 0K 0 vm_pgdata 0 0K 0 jblocks 0 0K 0 savedino 0 0K 3999 sentinel 0 0K 0 jfsync 0 0K 0 jtrunc 0 0K 0 sbdep 0 0K 17 jsegdep 0 0K 0 jseg 0 0K 0 jfreefrag 0 0K 0 jfreeblk 0 0K 0 jnewblk 0 0K 0 jmvref 0 0K 0 jremref 0 0K 0 jaddref 0 0K 0 freedep 0 0K 0 freefrag 0 0K 5 allocindir 0 0K 0 allocdirect 0 0K 0 ufs_trim 0 0K 0 mactemp 0 0K 0 audit_trigger 0 0K 0 audit_pipe_presel 0 0K 0 audit_pipeent 0 0K 0 audit_pipe 0 0K 0 audit_evname 0 0K 0 audit_bsm 0 0K 0 audit_gidset 0 0K 0 audit_text 0 0K 0 audit_path 0 0K 0 audit_data 0 0K 0 audit_cred 0 0K 0 ktls_ocf 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5E_TLS_RX 0 0K 0 MLX5EEPROM 0 0K 0 MLX5E_TLS 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EN 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5DUMP 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 simple_attr 0 0K 0 seq_file 0 0K 0 lkpiskb 0 0K 0 radix 0 0K 0 idr 0 0K 0 lkpindev 0 0K 0 lkpimhi 0 0K 0 lkpifw 0 0K 0 lkpi80211 0 0K 0 NLM 0 0K 0 ipsec-spdcache 0 0K 0 ipsec-reg 0 0K 0 ipsec-misc 0 0K 0 ipsecrequest 0 0K 0 ip6_msource 0 0K 0 ip6_moptions 0 0K 0 in6_mfilter 0 0K 0 frag6 0 0K 0 tcplog 0 0K 0 tcp_hwpace 0 0K 0 ip_msource 0 0K 0 ip_moptions 0 0K 0 in_mfilter 0 0K 0 ipid 0 0K 0 80211scan 0 0K 0 80211ratectl 0 0K 0 80211power 0 0K 0 80211nodeie 0 0K 0 80211node 0 0K 0 80211mesh_gt 0 0K 0 80211mesh_rt 0 0K 0 80211perr 0 0K 0 80211prep 0 0K 0 80211preq 0 0K 0 80211dfs 0 0K 0 80211crypto 0 0K 0 80211vap 0 0K 0 iflib 0 0K 0 vlan 0 0K 0 gif 0 0K 0 ifdescr 0 0K 0 zlib 0 0K 0 fadvise 0 0K 0 VN POLL 0 0K 0 statfs 0 0K 215 namei_tracker 0 0K 0 export_host 0 0K 0 cl_savebuf 0 0K 2 lio 0 0K 5 acl 0 0K 0 mbuf_tag 0 0K 0 ktls 0 0K 0 accf 0 0K 0 pts 0 0K 0 timerfd 0 0K 0 iov 0 0K 17648 ioctlops 0 0K 213 eventfd 0 0K 21 Witness 0 0K 0 stack 0 0K 0 sbuf 0 0K 288 firmware 0 0K 0 compressor 0 0K 0 SWAP 0 0K 0 sysctltmp 0 0K 746 sysctl 0 0K 3 ekcd 0 0K 0 dumper 0 0K 0 sendfile 0 0K 0 rctl 0 0K 0 cache 0 0K 0 prison_racct 0 0K 0 Fail Points 0 0K 0 sigio 0 0K 1 filedesc_to_leader 0 0K 0 pwd 0 0K 0 tty console 0 0K 0 boottrace 0 0K 0 isofs_node 0 0K 0 isofs_mount 0 0K 0 tr_raid5_data 0 0K 0 tr_raid1e_data 0 0K 0 tr_raid1_data 0 0K 0 tr_raid0_data 0 0K 0 tr_concat_data 0 0K 0 md_sii_data 0 0K 0 md_promise_data 0 0K 0 md_nvidia_data 0 0K 0 md_jmicron_data 0 0K 0 md_intel_data 0 0K 0 md_ddf_data 0 0K 0 raid_data 0 0K 72 geom_flashmap 0 0K 0 tmpfs dir 0 0K 0 tmpfs name 0 0K 0 tmpfs mount 0 0K 0 tmpfs extattr 0 0K 0 NFS FHA 0 0K 0 newnfsmnt 0 0K 0 newnfsclient_req 0 0K 0 NFSCL layrecall 0 0K 0 NFSCL session 0 0K 0 NFSCL sockreq 0 0K 0 NFSCL devinfo 0 0K 0 NFSCL flayout 0 0K 0 NFSCL layout 0 0K 0 NFSD rollback 0 0K 0 NFSCL diroff 0 0K 0 NEWdirectio 0 0K 0 NEWNFSnode 0 0K 0 NFSCL lck 0 0K 0 NFSCL lckown 0 0K 0 NFSCL client 0 0K 0 NFSCL deleg 0 0K 0 NFSCL open 0 0K 0 NFSCL owner 0 0K 0 NFS fh 0 0K 0 NFS req 0 0K 0 NFSD usrgroup 0 0K 0 NFSD string 0 0K 0 NFSD V4lock 0 0K 0 NFSD V4state 0 0K 0 msdosfs_fat 0 0K 0 msdosfs_mount 0 0K 0 msdosfs_node 0 0K 0 DEVFS4 0 0K 0 DEVFS2 0 0K 0 gntdev 0 0K 0 privcmd_dev 0 0K 0 evtchn_dev 0 0K 0 xenstore 0 0K 0 xnb 0 0K 0 xen_acpi 0 0K 0 xbbd 0 0K 0 xbd 0 0K 0 Balloon 0 0K 0 sysmouse 0 0K 0 vtfont 0 0K 0 pvscsi 0 0K 0 USBdev 0 0K 0 USB 0 0K 0 twsbuf 0 0K 0 tcp_log_dev 0 0K 19 midi buffers 0 0K 0 mixer 0 0K 0 ac97 0 0K 0 hdacc 0 0K 0 hdac 0 0K 0 hdaa 0 0K 0 SIIS driver 0 0K 0 PUC 0 0K 0 ppbusdev 0 0K 0 sr_iov 0 0K 0 OCS 0 0K 0 OCS 0 0K 0 nvme 0 0K 0 nvd 0 0K 0 netmap 0 0K 0 mwldev 0 0K 0 MVS driver 0 0K 0 mrsasbuf 0 0K 0 mpt_user 0 0K 0 mps_user 0 0K 0 MPSSAS 0 0K 0 mps 0 0K 0 mpr_user 0 0K 0 MPRSAS 0 0K 0 mpr 0 0K 0 mfibuf 0 0K 0 md_sectors 0 0K 0 md_disk 0 0K 4 malodev 0 0K 0 LED 0 0K 0 ix_sriov 0 0K 0 ix 0 0K 0 ipsbuf 0 0K 0 ciss_data 0 0K 0 BACKLIGHT 0 0K 0 ath_hal 0 0K 0 athdev 0 0K 0 ata_pci 0 0K 0 ata_dma 0 0K 0 ata_generic 0 0K 0 AHCI driver 0 0K 0 agp 0 0K 0 acpipwr 0 0K 0 acpi_perf 0 0K 0 acpicmbat 0 0K 0 aacraidcam 0 0K 0 aacraid_buf 0 0K 0 aaccam 0 0K 0 aacbuf 0 0K 0 zstd 0 0K 0 XZ_DEC 0 0K 0 nvlist 0 0K 0 SCSI ENC 0 0K 0 SCSI sa 0 0K 0 scsi_pass 0 0K 0 scsi_da 0 0K 69 ata_da 0 0K 0 scsi_ch 0 0K 0 scsi_cd 0 0K 0 nvme_da 0 0K 0 CAM CCB 0 0K 523 CAM ccb queue 0 0K 0 db> show uma Zone Size Used Free Requests Sleeps Bucket Total Mem XFree mbuf_jumbo_page 4096 8328 1070 235501 0 254 38494208 0 mbuf 256 9070 11768 1588383 0 254 5334528 0 tcp_log 416 0 8640 50977 0 254 3594240 0 BUF TRIE 144 200 11616 10112 0 62 1701504 0 malloc-384 384 4273 17 4276 0 30 1647360 0 mbuf_cluster 2048 762 0 762 0 254 1560576 0 malloc-128 128 11504 90 12123 0 126 1484032 0 malloc-4096 4096 326 2 488 0 2 1343488 0 UMA Slabs 0 112 11212 14 11212 0 126 1257312 0 RADIX NODE 144 8021 682 98256 0 62 1253232 0 sctp_asoc 2256 150 360 1120 0 254 1150560 0 vmem btag 56 19042 77 19042 0 254 1070664 0 malloc-16384 16384 46 4 2148 0 1 819200 0 FFS inode 1168 548 47 3008 0 8 694960 0 256 Bucket 2048 286 18 5834 0 8 622592 0 sctp_ep 1176 174 336 3080 0 254 599760 0 lkpicurr 176 2 3080 2 0 62 542432 0 pbuf 2624 0 198 0 0 2 519552 0 ertt_txseginfo 40 2112 10412 590250 0 254 500960 0 malloc-4096 4096 115 5 3585 0 2 491520 0 socket 960 73 435 5876 0 254 487680 0 sctp_chunk 152 137 2931 64447 0 254 466336 0 tcp_bbr_map 128 343 3191 67216 0 126 452352 0 VM OBJECT 264 1631 79 42263 0 30 451440 0 malloc-2048 2048 181 27 3388 0 8 425984 0 THREAD 1824 200 24 3349 0 8 408576 0 malloc-65536 65536 4 2 152 0 1 393216 0 sctp_raddr 736 150 367 1127 0 254 380512 0 malloc-256 256 443 802 16311 0 62 318720 0 VNODE 448 588 105 3050 0 30 310464 0 MAP ENTRY 96 2495 277 129914 0 126 266112 0 malloc-64 64 3792 303 3793 0 254 262080 0 malloc-16 16 14379 371 14452 0 254 236000 0 malloc-32768 32768 0 7 5259 0 1 229376 0 DEVCTL 1024 11 209 150 0 0 225280 0 malloc-8192 8192 24 2 57 0 1 212992 0 FPU_save_area 832 202 41 4151 0 16 202176 0 mbuf_packet 256 173 589 43531 0 254 195072 0 malloc-128 128 1320 199 29115 0 126 194432 0 malloc-256 256 462 288 14118 0 62 192000 0 UMA Zones 768 239 0 239 0 16 183552 0 malloc-32 32 5281 389 5302 0 254 181440 0 malloc-1024 1024 158 18 1077 0 16 180224 0 FFS2 dinode 256 548 142 3008 0 62 176640 0 lkpimm 56 1 3095 1 0 254 173376 0 malloc-384 384 180 270 2728 0 30 172800 0 PROC 1376 100 21 3087 0 8 166496 0 malloc-128 128 1062 209 5819 0 126 162688 0 malloc-1024 1024 120 24 364 0 16 147456 0 S VFS Cache 104 1011 393 3648 0 126 146016 0 malloc-384 384 327 33 4776 0 30 138240 0 filedesc0 1072 101 25 3088 0 8 135072 0 malloc-65536 65536 2 0 2 0 1 131072 0 malloc-65536 65536 0 2 110 0 1 131072 0 malloc-65536 65536 2 0 2 0 1 131072 0 malloc-32768 32768 2 2 122 0 1 131072 0 malloc-2048 2048 10 54 1042 0 8 131072 0 malloc-128 128 654 369 11452 0 126 130944 0 unpcb 256 17 493 1235 0 254 130560 0 tcp_inpcb 1312 13 83 1183 0 8 125952 0 128 Bucket 1024 82 33 526 0 16 117760 0 ksiginfo 112 106 938 985 0 126 116928 0 malloc-32768 32768 3 0 3 0 1 98304 0 malloc-256 256 3 372 4002 0 62 96000 0 malloc-256 256 326 49 4073 0 62 96000 0 64 Bucket 512 109 67 5015 0 30 90112 0 UMA Kegs 384 226 7 226 0 30 89472 0 syncache 168 0 528 11 0 254 88704 0 g_bio 408 0 210 80587 0 30 85680 0 malloc-64 64 957 366 4938 0 254 84672 0 malloc-16384 16384 5 0 5 0 1 81920 0 malloc-8192 8192 10 0 12 0 1 81920 0 sctp_readq 152 2 518 165 0 254 79040 0 malloc-8192 8192 7 2 108 0 1 73728 0 malloc-4096 4096 18 0 18 0 2 73728 0 VMSPACE 616 76 38 3060 0 16 70224 0 malloc-64 64 834 237 50288 0 254 68544 0 malloc-65536 65536 0 1 8 0 1 65536 0 malloc-32768 32768 2 0 2 0 1 65536 0 malloc-4096 4096 12 4 238 0 2 65536 0 malloc-1024 1024 20 44 1703 0 16 65536 0 32 Bucket 256 92 163 3694 0 62 65280 0 sctp_stream_msg_out 112 40 500 437 0 254 60480 0 malloc-384 384 96 54 138 0 30 57600 0 malloc-4096 4096 3 11 517 0 2 57344 0 udp_inpcb 424 6 120 217 0 30 53424 0 malloc-256 256 46 149 5568 0 62 49920 0 tcp_rack_pcb 1024 1 47 421 0 16 49152 0 DIRHASH 1024 35 13 35 0 16 49152 0 NAMEI 1024 0 48 24900 0 16 49152 0 malloc-16384 16384 1 2 161 0 1 49152 0 malloc-2048 2048 4 20 35 0 8 49152 0 malloc-2048 2048 12 12 135 0 8 49152 0 malloc-1024 1024 16 32 30 0 16 49152 0 da_ccb 544 0 84 20278 0 16 45696 0 pcpu-8 8 4828 804 6275 0 254 45056 0 tcp_bbr_pcb 832 7 47 484 0 16 44928 0 Files 80 265 285 15952 0 126 44000 0 malloc-8192 8192 3 2 7 0 1 40960 0 pipe 728 22 33 431 0 16 40040 0 malloc-64 64 181 386 19475 0 254 36288 0 malloc-64 64 32 535 56 0 254 36288 0 malloc-64 64 137 430 3347 0 254 36288 0 malloc-64 64 2 565 2 0 254 36288 0 malloc-64 64 1 566 5 0 254 36288 0 16 Bucket 144 82 170 1136 0 62 36288 0 tcp_rack_map 128 0 279 1936 0 126 35712 0 malloc-128 128 23 256 234 0 126 35712 0 malloc-128 128 9 270 39 0 126 35712 0 malloc-128 128 2 277 2 0 126 35712 0 malloc-128 128 11 268 11 0 126 35712 0 ripcb 392 4 86 107 0 30 35280 0 routing nhops 256 27 108 34 0 62 34560 0 ttyoutq 256 72 63 160 0 62 34560 0 malloc-256 256 46 89 916 0 62 34560 0 malloc-256 256 10 125 14 0 62 34560 0 TURNSTILE 136 225 27 225 0 62 34272 0 SLEEPQUEUE 88 225 159 225 0 126 33792 0 malloc-32768 32768 1 0 1 0 1 32768 0 malloc-2048 2048 2 14 10 0 8 32768 0 malloc-2048 2048 2 14 2 0 8 32768 0 malloc-2048 2048 11 5 11 0 8 32768 0 malloc-1024 1024 5 27 45 0 16 32768 0 malloc-1024 1024 4 28 4 0 16 32768 0 malloc-512 512 1 63 119 0 30 32768 0 malloc-512 512 8 56 81 0 30 32768 0 malloc-512 512 3 61 70 0 30 32768 0 malloc-512 512 10 54 404 0 30 32768 0 malloc-512 512 2 62 37 0 30 32768 0 malloc-512 512 8 56 135 0 30 32768 0 pcpu-64 64 487 25 487 0 254 32768 0 PWD 40 59 749 2208 0 254 32320 0 KNOTE 160 30 170 25444 0 62 32000 0 ttyinq 160 135 65 300 0 62 32000 0 PGRP 120 35 229 93 0 126 31680 0 clpbuf 2624 0 12 58 0 4 31488 0 sctp_laddr 48 113 475 1313 0 254 28224 0 tcp_inpcb ports 32 10 872 776 0 254 28224 0 malloc-32 32 301 581 2101 0 254 28224 0 4 Bucket 48 7 581 10 0 254 28224 0 2 Bucket 32 59 823 1715 0 254 28224 0 udplite_inpcb 424 1 62 21 0 30 26712 0 cpuset 200 7 121 7 0 62 25600 0 malloc-8192 8192 2 1 28 0 1 24576 0 malloc-4096 4096 3 3 10 0 2 24576 0 rl_entry 40 103 503 103 0 254 24240 0 rtentry 168 30 114 34 0 62 24192 0 8 Bucket 80 67 233 1459 0 126 24000 0 itimer 352 0 66 68 0 30 23232 0 Mountpoints 2816 2 6 2 0 4 22528 0 hostcache 64 3 312 3 0 254 20160 0 udp_inpcb ports 32 3 627 53 0 254 20160 0 ertt 72 13 267 1183 0 126 20160 0 malloc-32 32 81 549 126 0 254 20160 0 malloc-32 32 260 370 4692 0 254 20160 0 malloc-32 32 110 520 2274 0 254 20160 0 malloc-32 32 365 265 7663 0 254 20160 0 malloc-32 32 2 628 9 0 254 20160 0 cryptop 280 0 70 7 0 30 19600 0 AIOCB 552 0 35 14 0 16 19320 0 L VFS Cache 320 0 60 26 0 30 19200 0 malloc-256 256 11 64 11 0 62 19200 0 AIOLIO 272 0 70 5 0 30 19040 0 epoch_record pcpu 256 4 60 4 0 62 16384 0 malloc-16384 16384 1 0 1 0 1 16384 0 malloc-2048 2048 0 8 14 0 8 16384 0 malloc-512 512 1 31 1 0 30 16384 0 SMR CPU 32 7 504 7 0 254 16352 0 vtnet_tx_hdr 24 2 666 168985 0 254 16032 0 malloc-16 16 525 475 5862 0 254 16000 0 AIO 208 0 76 12 0 62 15808 0 kenv 258 17 43 1082 0 30 15480 0 mqnode 416 3 33 3 0 30 14976 0 vmem 1856 1 7 1 0 8 14848 0 SMR SHARED 24 7 504 7 0 254 12264 0 malloc-32 32 5 373 5 0 254 12096 0 malloc-16 16 92 658 4583 0 254 12000 0 malloc-16 16 317 433 2268 0 254 12000 0 malloc-16 16 40 710 30765 0 254 12000 0 malloc-16 16 22 728 22 0 254 12000 0 malloc-16 16 339 411 4615 0 254 12000 0 malloc-384 384 11 19 11 0 30 11520 0 malloc-384 384 1 29 1 0 30 11520 0 malloc-384 384 13 17 13 0 30 11520 0 malloc-8192 8192 1 0 1 0 1 8192 0 pcpu-16 16 8 504 8 0 254 8192 0 malloc-16 16 10 490 11 0 254 8000 0 UMA Slabs 1 176 10 12 10 0 62 3872 0 KMAP ENTRY 96 12 27 14 0 0 3744 0 FFS1 dinode 128 0 0 0 0 126 0 0 ada_ccb 272 0 0 0 0 30 0 0 swblk 136 0 0 0 0 62 0 0 swpctrie 144 0 0 0 0 62 0 0 cdg_qdiffsample 16 0 0 0 0 254 0 0 pf state scrubs 40 0 0 0 0 254 0 0 pf frag entries 40 0 0 0 0 254 0 0 pf frags 248 0 0 0 0 62 0 0 pf table entries 160 0 0 0 0 254 0 0 pf table entry counters 64 0 0 0 0 254 0 0 pf source nodes 152 0 0 0 0 254 0 0 pf state keys 88 0 0 0 0 126 0 0 pf states 360 0 0 0 0 254 0 0 pf tags 104 0 0 0 0 126 0 0 pf mtags 184 0 0 0 0 62 0 0 tfo_ccache_entries 80 0 0 0 0 126 0 0 tfo 4 0 0 0 0 254 0 0 sackhole 32 0 0 0 0 254 0 0 ipq 56 0 0 0 0 254 0 0 sctp_asconf_ack 48 0 0 0 0 254 0 0 sctp_asconf 40 0 0 0 0 254 0 0 tcp_log_id_node 120 0 0 0 0 126 0 0 tcp_log_id_bucket 176 0 0 0 0 62 0 0 tcpreass 48 0 0 0 0 254 0 0 udplite_inpcb ports 32 0 0 0 0 254 0 0 ripcb ports 32 0 0 0 0 254 0 0 IPsec SA lft_c 16 0 0 0 0 254 0 0 mqnotifier 216 0 0 0 0 62 0 0 mvdata 64 0 0 0 0 254 0 0 mqueue 248 0 0 0 0 62 0 0 TMPFS node 232 0 0 0 0 62 0 0 NCLNODE 608 0 0 0 0 16 0 0 LTS VFS Cache 360 0 0 0 0 30 0 0 STS VFS Cache 144 0 0 0 0 62 0 0 linux_dma_object 32 0 0 0 0 254 0 0 linux_dma_pctrie 144 0 0 0 0 62 0 0 IOMMU_MAP_ENTRY 104 0 0 0 0 126 0 0 mbuf_jumbo_16k 16384 0 0 0 0 254 0 0 mbuf_jumbo_9k 9216 0 0 0 0 254 0 0 audit_record 1280 0 0 0 0 8 0 0 domainset 40 0 0 0 0 254 0 0 MAC labels 40 0 0 0 0 254 0 0 vnpbuf 2624 0 0 0 0 16 0 0 nfspbuf 2624 0 0 0 0 4 0 0 swwbuf 2624 0 0 0 0 2 0 0 swrbuf 2624 0 0 0 0 4 0 0 umtx_shm 88 0 0 0 0 126 0 0 umtx pi 96 0 0 0 0 126 0 0 rangeset pctrie nodes 144 0 0 0 0 62 0 0 malloc-65536 65536 0 0 0 0 1 0 0 malloc-65536 65536 0 0 0 0 1 0 0 malloc-65536 65536 0 0 0 0 1 0 0 malloc-32768 32768 0 0 0 0 1 0 0 malloc-32768 32768 0 0 0 0 1 0 0 malloc-32768 32768 0 0 0 0 1 0 0 malloc-16384 16384 0 0 0 0 1 0 0 malloc-16384 16384 0 0 0 0 1 0 0 malloc-16384 16384 0 0 0 0 1 0 0 malloc-16384 16384 0 0 0 0 1 0 0 malloc-8192 8192 0 0 0 0 1 0 0 malloc-8192 8192 0 0 0 0 1 0 0 malloc-4096 4096 0 0 0 0 2 0 0 malloc-4096 4096 0 0 0 0 2 0 0 malloc-1024 1024 0 0 0 0 16 0 0 malloc-1024 1024 0 0 0 0 16 0 0 malloc-512 512 0 0 0 0 30 0 0 malloc-384 384 0 0 0 0 30 0 0 pcpu-32 32 0 0 0 0 254 0 0 pcpu-4 4 0 0 0 0 254 0 0 fakepg 104 0 0 0 0 126 0 0 UMA Hash 256 0 0 0 0 62 0 0