panic: lock (sleep mutex) sctp-inp not locked @ /syzkaller/managers/main/kernel/sys/netinet/sctp_usrreq.c:958 cpuid = 1 time = 1705471601 KDB: stack backtrace: db_trace_self_wrapper() at db_trace_self_wrapper+0xc6/frame 0xfffffe00746c35b0 kdb_backtrace() at kdb_backtrace+0xd0/frame 0xfffffe00746c3710 vpanic() at vpanic+0x271/frame 0xfffffe00746c38b0 panic() at panic+0xb5/frame 0xfffffe00746c3970 witness_unlock() at witness_unlock+0x4fd/frame 0xfffffe00746c39f0 __mtx_unlock_flags() at __mtx_unlock_flags+0xec/frame 0xfffffe00746c3ab0 sctp_shutdown() at sctp_shutdown+0xf35/frame 0xfffffe00746c3c10 soshutdown() at soshutdown+0xb3/frame 0xfffffe00746c3c50 kern_shutdown() at kern_shutdown+0x154/frame 0xfffffe00746c3d10 amd64_syscall() at amd64_syscall+0x473/frame 0xfffffe00746c3f30 fast_syscall_common() at fast_syscall_common+0xf8/frame 0xfffffe00746c3f30 --- syscall (198, FreeBSD ELF64, __syscall), rip = 0x2aeb2a, rsp = 0x826983f08, rbp = 0x826983f80 --- KDB: enter: panic [ thread pid 3350 tid 104228 ] Stopped at kdb_enter+0x6e: movq $0,0x2194267(%rip) db> db> set $lines = 0 db> set $maxwidth = 0 db> show registers cs 0x20 ds 0x3b es 0x3b fs 0x13 gs 0x1b ss 0x28 rax 0x12 rcx 0xfffffe006d200000 rdx 0x3ffff rbx 0xffffffff826f86c0 .str.28 rsp 0xfffffe00746c36f0 rbp 0xfffffe00746c3710 rsi 0x40001 rdi 0xffffffff815de326 printf+0x156 r8 0 r9 0xffffffff r10 0 r11 0 r12 0 r13 0xfffffe0073f78000 r14 0xffffffff826f86c0 .str.28 r15 0 rip 0xffffffff815cadae kdb_enter+0x6e rflags 0x46 kdb_enter+0x6e: movq $0,0x2194267(%rip) db> show proc Process 3350 (syz-executor.1) at 0xfffffe007495f060: state: NORMAL uid: 0 gids: 0, 0, 5 parent: pid 772 at 0xfffffe0057b9d5c0 ABI: FreeBSD ELF64 flag: 0x10000080 flag2: 0 arguments: /root/syz-executor.1 exec reaper: 0xfffffe00541d1040 reapsubtree: 1 sigparent: 20 vmspace: 0xfffffe00749a1268 (map 0xfffffe00749a1268) (map.pmap 0xfffffe00749a1328) (pmap 0xfffffe00749a1398) threads: 2 104156 RunQ syz-executor.1 104228 Run CPU 1 syz-executor.1 db> ps pid ppid pgrp uid state wmesg wchan cmd 3350 772 772 0 R (threaded) syz-executor.1 104156 RunQ syz-executor.1 104228 Run CPU 1 syz-executor.1 3345 780 780 0 R (threaded) syz-executor.3 102839 RunQ syz-executor.3 104221 S connec 0xfffffe0058cf885a syz-executor.3 104224 S connec 0xfffffe0058a9d49a syz-executor.3 104225 S uwait 0xfffffe0074899300 syz-executor.3 3339 771 771 0 R (threaded) syz-executor.0 103975 Run nanslp 0xffffffff83740400 syz-executor.0 104214 RunQ syz-executor.0 104218 S uwait 0xfffffe0074551480 syz-executor.0 3301 3294 3301 0 Ss select 0xfffffe00571d7140 dhclient 3297 1 3297 0 Ss select 0xfffffe00571d7bc0 dhclient 3294 3286 424 65 S select 0xfffffe00571d7240 dhclient 3286 424 424 0 S wait 0xfffffe0074459580 sh 3274 1 3274 65 Ss select 0xfffffe00571d72c0 dhclient 3080 1 772 0 S uwait 0xfffffe0007989a00 syz-executor.1 3077 1 772 0 S uwait 0xfffffe006d1b9380 syz-executor.1 3075 1 772 0 S uwait 0xfffffe006d1bb880 syz-executor.1 3054 1 773 0 S uwait 0xfffffe0074899780 syz-executor.2 3052 1 773 0 S uwait 0xfffffe006d1b9c80 syz-executor.2 3050 1 773 0 S uwait 0xfffffe0057a76900 syz-executor.2 3048 1 773 0 S uwait 0xfffffe006d1bb900 syz-executor.2 2895 1 772 0 S uwait 0xfffffe0074551880 syz-executor.1 2894 1 780 0 S uwait 0xfffffe0057a77300 syz-executor.3 2889 1 772 0 S uwait 0xfffffe006d1b9480 syz-executor.1 2888 1 780 0 S uwait 0xfffffe0057a79e80 syz-executor.3 2882 1 780 0 S uwait 0xfffffe005750db00 syz-executor.3 2880 1 772 0 S uwait 0xfffffe0057a77600 syz-executor.1 2875 1 772 0 S uwait 0xfffffe0057a76780 syz-executor.1 2867 1 772 0 S uwait 0xfffffe005750a400 syz-executor.1 2862 1 772 0 S uwait 0xfffffe006d1bc500 syz-executor.1 2854 1 772 0 S uwait 0xfffffe0007989500 syz-executor.1 2536 1 780 0 S uwait 0xfffffe006d1bad80 syz-executor.3 2528 1 780 0 S uwait 0xfffffe0057a79c80 syz-executor.3 2527 1 2527 0 Ss select 0xfffffe00571b1d40 dhclient 2522 1 2522 0 Ss select 0xfffffe00571d7c40 dhclient 2509 1 772 0 S uwait 0xfffffe0057a77900 syz-executor.1 2497 1 2497 65 Ss select 0xfffffe00571b6040 dhclient 2493 1 780 0 S uwait 0xfffffe006d1baa80 syz-executor.3 2490 1 772 0 S uwait 0xfffffe006d1bb380 syz-executor.1 2487 1 772 0 S uwait 0xfffffe006d1bb580 syz-executor.1 2484 1 772 0 S uwait 0xfffffe005750a480 syz-executor.1 2482 1 772 0 S uwait 0xfffffe006d1b9c00 syz-executor.1 2480 1 772 0 S uwait 0xfffffe006d1bc300 syz-executor.1 2477 1 772 0 S uwait 0xfffffe006d1b9680 syz-executor.1 2476 1 773 0 S uwait 0xfffffe006d1bbe80 syz-executor.2 2475 1 780 0 S uwait 0xfffffe006d1bb680 syz-executor.3 2473 1 772 0 S uwait 0xfffffe005750da80 syz-executor.1 2469 1 772 0 S uwait 0xfffffe006d1b9780 syz-executor.1 2466 1 772 0 S uwait 0xfffffe006d1bb780 syz-executor.1 2463 1 772 0 S uwait 0xfffffe0057a77380 syz-executor.1 2460 1 773 0 S uwait 0xfffffe006d1b9280 syz-executor.2 2459 1 780 0 S uwait 0xfffffe006d1b9080 syz-executor.3 2456 1 772 0 S uwait 0xfffffe006d1bc400 syz-executor.1 2452 1 780 0 S uwait 0xfffffe0057a76680 syz-executor.3 2451 1 773 0 S uwait 0xfffffe005750b680 syz-executor.2 2420 1 780 0 S uwait 0xfffffe005750c500 syz-executor.3 2028 0 0 0 DL aiordy 0xfffffe006cbd1b00 [aiod4] 2027 0 0 0 DL aiordy 0xfffffe006cbd25c0 [aiod3] 2026 0 0 0 DL aiordy 0xfffffe006cd96ae0 [aiod2] 2025 0 0 0 DL aiordy 0xfffffe0057b9bae0 [aiod1] 1569 1 773 0 S uwait 0xfffffe005750de80 syz-executor.2 1451 1 1451 0 Ss select 0xfffffe00571b6dc0 dhclient 1443 1 1443 0 Ss select 0xfffffe0057a93a40 dhclient 1416 1 1416 65 Ss select 0xfffffe00571b1dc0 dhclient 810 1 810 0 Ss select 0xfffffe00571b6140 dhclient 805 1 805 0 Ss select 0xfffffe0057a93ac0 dhclient 780 768 780 0 Rs syz-executor.3 773 768 773 0 Rs syz-executor.2 772 768 772 0 Ss nanslp 0xffffffff83740401 syz-executor.1 771 768 771 0 Rs syz-executor.0 768 766 766 0 S (threaded) syz-fuzzer 100110 S uwait 0xfffffe0057a76c00 syz-fuzzer 100114 S uwait 0xfffffe0057a77080 syz-fuzzer 100115 S uwait 0xfffffe0057a77180 syz-fuzzer 100116 S uwait 0xfffffe0057a76180 syz-fuzzer 100117 S uwait 0xfffffe0057a76280 syz-fuzzer 100118 S uwait 0xfffffe005750ac80 syz-fuzzer 100119 S wait 0xfffffe006cde2000 syz-fuzzer 100120 S wait 0xfffffe006cde2000 syz-fuzzer 100121 S kqread 0xfffffe0057bbd500 syz-fuzzer 100122 S wait 0xfffffe006cde2000 syz-fuzzer 100123 S wait 0xfffffe006cde2000 syz-fuzzer 100153 S uwait 0xfffffe006d1bb280 syz-fuzzer 100261 S uwait 0xfffffe005750a180 syz-fuzzer 766 764 766 0 Ss pause 0xfffffe006cd97bb0 csh 764 682 764 0 Ss select 0xfffffe00571b1f40 sshd 748 1 748 0 Ss+ ttyin 0xfffffe005766fcb0 getty 747 1 747 0 Ss+ ttyin 0xfffffe00587670b0 getty 746 1 746 0 Ss+ ttyin 0xfffffe00587674b0 getty 745 1 745 0 Ss+ ttyin 0xfffffe00587678b0 getty 744 1 744 0 Ss+ ttyin 0xfffffe0058767cb0 getty 743 1 743 0 Ss+ ttyin 0xfffffe00587680b0 getty 742 1 742 0 Ss+ ttyin 0xfffffe00587684b0 getty 741 1 741 0 Ss+ ttyin 0xfffffe00587688b0 getty 740 1 740 0 Ss+ ttyin 0xfffffe0058768cb0 getty 686 1 686 0 Ss nanslp 0xffffffff83740400 cron 682 1 682 0 Ss select 0xfffffe00571b62c0 sshd 495 1 495 0 Ss select 0xfffffe00571b61c0 syslogd 424 1 424 0 Ss wait 0xfffffe006cbd0ae0 devd 423 1 423 65 Ss select 0xfffffe00571b6240 dhclient 338 1 338 0 Ss select 0xfffffe0057a93b40 dhclient 335 1 335 0 Ss select 0xfffffe00571b63c0 dhclient 17 0 0 0 DL syncer 0xffffffff8385db20 [syncer] 16 0 0 0 DL vlruwt 0xfffffe0057182060 [vnlru] 15 0 0 0 DL (threaded) [bufdaemon] 100079 D psleep 0xffffffff8385c100 [bufdaemon] 100080 D - 0xffffffff82c0a140 [bufspacedaemon-0] 100093 D sdflush 0xfffffe00588a14e8 [/ worker] 9 0 0 0 DL psleep 0xffffffff838a55c0 [vmdaemon] 8 0 0 0 DL (threaded) [pagedaemon] 100077 D psleep 0xffffffff8388d438 [dom0] 100081 D launds 0xffffffff8388d444 [laundry: dom0] 100082 D umarcl 0xffffffff81d61000 [uma] 7 0 0 0 DL - 0xffffffff834bdc10 [rand_harvestq] 6 0 0 0 DL pftm 0xffffffff8427cea0 [pf purge] 5 0 0 0 DL waiting 0xffffffff845b55a0 [sctp_iterator] 4 0 0 0 DL (threaded) [cam] 100044 D - 0xffffffff83488340 [doneq0] 100045 D - 0xffffffff834882c0 [async] 100076 D - 0xffffffff83488140 [scanner] 3 0 0 0 DL (threaded) [crypto] 100041 D crypto_ 0xffffffff83888c60 [crypto] 100042 D crypto_ 0xfffffe00542d5730 [crypto returns 0] 100043 D crypto_ 0xfffffe00542d5780 [crypto returns 1] 14 0 0 0 DL seqstat 0xfffffe000796e888 [sequencer 00] 13 0 0 0 DL (threaded) [geom] 100035 D - 0xffffffff836e8d40 [g_event] 100036 D - 0xffffffff836e8d60 [g_up] 100037 D - 0xffffffff836e8d80 [g_down] 2 0 0 0 WL (threaded) [clock] 100029 I [clock (0)] 100030 I [clock (1)] 12 0 0 0 WL (threaded) [intr] 100012 I [swi6: task queue] 100013 I [swi6: Giant taskq] 100015 I [swi5: fast taskq] 100031 I [swi1: netisr 0] 100032 I [swi1: hpts] 100033 I [swi1: hpts] 100046 I [irq24: virtio_pci0] 100047 I [irq25: virtio_pci0] 100048 I [irq26: virtio_pci0] 100049 I [irq27: virtio_pci0] 100050 I [irq28: virtio_pci1] 100051 I [irq29: virtio_pci1] 100052 I [irq30: virtio_pci1] 100053 I [irq31: virtio_pci1] 100054 I [irq32: virtio_pci1] 100059 I [irq33: virtio_pci2] 100060 I [irq34: virtio_pci2] 100061 I [irq35: virtio_pci2] 100063 I [irq1: atkbd0] 100064 I [irq12: psm0] 100065 I [swi0: uart uart++] 100069 I [swi1: pf send] 11 0 0 0 RL (threaded) [idle] 100003 CanRun [idle: cpu0] 100004 CanRun [idle: cpu1] 1 0 1 0 SLs wait 0xfffffe00541d1040 [init] 10 0 0 0 DL audit_w 0xffffffff838896c0 [audit] 0 0 0 0 DLs (threaded) [kernel] 100000 D swapin 0xffffffff836e9720 [swapper] 100005 D - 0xfffffe0054282100 [softirq_0] 100006 D - 0xfffffe0054282000 [softirq_1] 100007 D - 0xfffffe0054281e00 [if_io_tqg_0] 100008 D - 0xfffffe0054281d00 [if_io_tqg_1] 100009 D - 0xfffffe0054281c00 [if_config_tqg_0] 100010 D - 0xfffffe0007977900 [pci_hp taskq] 100011 D - 0xfffffe0007977800 [kqueue_ctx taskq] 100014 D - 0xfffffe0007977500 [thread taskq] 100016 D - 0xfffffe0007977300 [aiod_kick taskq] 100017 D - 0xfffffe0007977200 [deferred_unmount ta] 100018 D - 0xfffffe0007977100 [inm_free taskq] 100019 D - 0xfffffe0007977000 [in6m_free taskq] 100020 D - 0xfffffe0007976e00 [linuxkpi_irq_wq] 100021 D - 0xfffffe0007976d00 [linuxkpi_short_wq_0] 100022 D - 0xfffffe0007976d00 [linuxkpi_short_wq_1] 100023 D - 0xfffffe0007976d00 [linuxkpi_short_wq_2] 100024 D - 0xfffffe0007976d00 [linuxkpi_short_wq_3] 100025 D - 0xfffffe0007976c00 [linuxkpi_long_wq_0] 100026 D - 0xfffffe0007976c00 [linuxkpi_long_wq_1] 100027 D - 0xfffffe0007976c00 [linuxkpi_long_wq_2] 100028 D - 0xfffffe0007976c00 [linuxkpi_long_wq_3] 100034 D - 0xfffffe0007976a00 [firmware taskq] 100039 D - 0xfffffe0007976700 [crypto_0] 100040 D - 0xfffffe0007976700 [crypto_1] 100055 D - 0xfffffe00571c4b00 [vtnet0 rxq 0] 100056 D - 0xfffffe00571c4a00 [vtnet0 txq 0] 100057 D - 0xfffffe00571c4900 [vtnet0 rxq 1] 100058 D - 0xfffffe00571c4800 [vtnet0 txq 1] 100062 D vtbslp 0xfffffe00571b6580 [virtio_balloon] 100066 D - 0xffffffff826fd921 [deadlkres] 100070 D - 0xfffffe0057bbd100 [acpi_task_0] 100071 D - 0xfffffe0057bbd100 [acpi_task_1] 100072 D - 0xfffffe0057bbd100 [acpi_task_2] 100074 D - 0xfffffe0007977c00 [mca taskq] 100075 D - 0xfffffe0007976500 [CAM taskq] db> show all locks Process 3339 (syz-executor.0) thread 0xfffffe0074965740 (104214) exclusive lockmgr bufwait (bufwait) r = 0 (0xfffffe0007e02948) locked @ /syzkaller/managers/main/kernel/sys/kern/vfs_bio.c:4010 exclusive lockmgr ufs (ufs) r = 0 (0xfffffe0074519070) locked @ /syzkaller/managers/main/kernel/sys/kern/vfs_vnops.c:1175 db> show malloc Type InUse MemUse Requests pf_hash 5 11524K 5 tcp_hpts 7 4801K 7 devbuf 4192 4324K 4220 sysctloid 34817 2052K 34888 vtbuf 24 1968K 46 kobj 326 1304K 488 newblk 36 1033K 27138 vfscache 3 1025K 3 pcb 323 1002K 6747 filedesc 96 762K 4953 inodedep 11 516K 4639 ufs_quota 1 512K 1 vfs_hash 1 512K 1 callout 2 512K 2 intr 4 472K 4 subproc 223 450K 3480 vmem 3 266K 6 vnet_data 2 224K 2 acpitask 1 224K 1 filemon 25 200K 38 acpica 1674 184K 60830 tidhash 3 141K 3 pagedep 9 130K 2540 linker 352 130K 386 tfo_ccache 1 128K 1 IP reass 1 128K 1 DEVFS1 109 109K 126 sem 4 106K 4 gtaskqueue 18 98K 18 tcp_fsb 46 95K 1344 BPF 46 88K 102 bus 985 81K 5155 sctp_atcl 205 77K 4652 mtx_pool 2 72K 2 syncache 1 68K 1 NFSD srvcache 3 68K 3 module 513 65K 513 ddb_capture 1 64K 1 sctp_stro 57 57K 1075 kdtrace 302 57K 7580 umtx 448 56K 448 temp 36 39K 2280 hostcache 1 32K 1 shm 1 32K 19 DEVFS3 128 32K 138 msg 4 30K 4 kbdmux 6 28K 6 LRO 24 25K 24 ifaddr 70 20K 72 DEVFS_RULE 56 20K 56 routetbl 136 19K 495 ufs_mount 4 17K 5 proc 3 17K 3 tty 16 16K 16 ithread 97 16K 97 lltable 49 16K 75 bus-sc 34 15K 1687 eventhandler 161 14K 161 KTRACE 101 13K 2608 ifnet 7 13K 7 ether_multi 152 13K 162 sctp_timw 46 12K 46 kenv 95 12K 95 sctp_atky 290 11K 6106 GEOM 61 11K 481 rman 86 11K 431 CAM queue 5 11K 1528 crypto 48 9K 130 in6_multi 65 9K 65 kqueue 103 9K 3394 DEVFSP 137 9K 467 bmsafemap 2 9K 4131 rpc 4 9K 4 devstat 4 9K 4 UART 12 9K 12 ksem 1 8K 31 shmfd 1 8K 410 pfs_vncache 1 8K 1 CC Mem 30 8K 1405 audit_evclass 238 8K 300 taskqueue 63 7K 63 cred 26 7K 266 pwddesc 101 7K 3364 pf_ifnet 19 7K 47 sglist 6 7K 6 plimit 24 6K 409 CAM DEV 3 6K 510 cryptodev 68 6K 495 pfs_nodes 20 5K 20 hhook 15 5K 17 ufs_dirhash 24 5K 24 UMA 267 5K 267 session 34 5K 49 proc-args 133 5K 4589 vt 11 5K 11 pf_table 2 4K 21 memdesc 1 4K 1 MCA 32 4K 32 evdev 4 4K 4 pf_osfp 32 4K 32 acpisem 28 4K 28 lockf 32 4K 187 selfd 52 4K 56017 kcovinfo 52 4K 52 sctp_athm 205 4K 4912 terminal 11 3K 11 select 19 3K 57 clone 9 3K 9 uidinfo 3 3K 13 local_apic 1 2K 1 io_apic 1 2K 1 ipsec-saq 2 2K 2 ip6ndp 12 2K 13 sctp_map 114 2K 2446 dirrem 7 2K 3472 Unitno 30 2K 89 sctp_ifa 13 2K 14 tun 7 2K 7 CAM XPT 22 2K 543 msi 12 2K 12 inpcbpolicy 48 2K 1728 in_multi 6 2K 8 toponodes 6 2K 6 ipsecpolicy 2 2K 2 acpidev 20 2K 20 softdep 1 1K 1 sahead 1 1K 1 secasvar 1 1K 1 nhops 6 1K 8 vnodemarker 2 1K 418 NFSD session 1 1K 1 CAM periph 4 1K 271 osd 35 1K 1396 sctp_ifn 6 1K 14 ipsec 3 1K 3 diradd 6 1K 3513 mld 6 1K 6 igmp 6 1K 6 pfil 6 1K 6 isadev 6 1K 6 mount 16 1K 89 pci_link 10 1K 10 freefile 5 1K 3427 encap_export_host 12 1K 12 procdesc 5 1K 14 newdirblk 4 1K 2414 mkdir 4 1K 4828 cdev 2 1K 2 lkpikmalloc 8 1K 9 netlink 2 1K 54 chacha20random 1 1K 1 biobuf 1 1K 1 pf_rule 2 1K 40 indirdep 1 1K 14232 vnodes 1 1K 1 NFSD lckfile 1 1K 1 NFSD V4client 1 1K 1 DEVFS 9 1K 10 CAM SIM 2 1K 2 feeder 7 1K 7 tcpfunc 3 1K 3 loginclass 3 1K 6 prison 6 1K 6 soname 6 1K 9081 nexusdev 8 1K 8 apmdev 1 1K 1 atkbddev 2 1K 2 aio 4 1K 4 iov 2 1K 21707 pmchooks 1 1K 1 CAM path 4 1K 1034 CAM dev queue 2 1K 2 CAM I/O Scheduler 1 1K 1 filecaps 5 1K 93 sctp_vrf 1 1K 1 freework 1 1K 12620 vnet 1 1K 1 pmc 1 1K 1 entropy 2 1K 41 acpiintr 1 1K 1 cpus 2 1K 2 vnet_data_free 1 1K 1 Per-cpu 1 1K 1 p1003.1b 1 1K 1 sctp_mcore 0 0K 0 sctp_socko 0 0K 1726 sctp_iter 0 0K 12 sctp_mvrf 0 0K 0 sctp_cpal 0 0K 0 sctp_cmsg 0 0K 0 sctp_stre 0 0K 0 sctp_athi 0 0K 0 sctp_a_it 0 0K 12 sctp_aadr 0 0K 0 sctp_stri 0 0K 506 pf_altq 0 0K 0 pf_krule_item 0 0K 0 pf_temp 0 0K 0 tcp_do 0 0K 0 mqdata 0 0K 0 ipcomp 0 0K 0 esp 0 0K 0 ah 0 0K 0 madt_table 0 0K 2 smartpqi 0 0K 0 ixl 0 0K 0 ice-resmgr 0 0K 0 ice-osdep 0 0K 0 ice 0 0K 0 iavf 0 0K 0 axgbe 0 0K 0 fpukern_ctx 0 0K 0 xen_intr 0 0K 0 xen_hvm 0 0K 0 legacydrv 0 0K 0 bounce 0 0K 0 busdma 0 0K 0 qpidrv 0 0K 0 dmar_idpgtbl 0 0K 0 dmar_dom 0 0K 0 dmar_ctx 0 0K 0 isci 0 0K 0 iommu_dmamap 0 0K 0 hyperv_socket 0 0K 0 bxe_ilt 0 0K 0 aesni_data 0 0K 0 xenbus 0 0K 0 vm_fictitious 0 0K 0 UMAHash 0 0K 0 vm_pgdata 0 0K 0 jblocks 0 0K 0 savedino 0 0K 8996 sentinel 0 0K 0 jfsync 0 0K 0 jtrunc 0 0K 0 sbdep 0 0K 206 jsegdep 0 0K 0 jseg 0 0K 0 jfreefrag 0 0K 0 jfreeblk 0 0K 0 jnewblk 0 0K 0 jmvref 0 0K 0 jremref 0 0K 0 jaddref 0 0K 0 freedep 0 0K 0 freeblks 0 0K 3684 freefrag 0 0K 36 allocindir 0 0K 0 allocdirect 0 0K 0 ufs_trim 0 0K 0 mactemp 0 0K 0 audit_trigger 0 0K 0 audit_pipe_presel 0 0K 0 audit_pipeent 0 0K 0 audit_pipe 0 0K 0 audit_evname 0 0K 0 audit_bsm 0 0K 0 audit_gidset 0 0K 0 audit_text 0 0K 0 audit_path 0 0K 0 audit_data 0 0K 0 audit_cred 0 0K 0 ktls_ocf 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5E_TLS_RX 0 0K 0 MLX5EEPROM 0 0K 0 MLX5E_TLS 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EN 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5DUMP 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 simple_attr 0 0K 0 seq_file 0 0K 0 lkpiskb 0 0K 0 radix 0 0K 0 idr 0 0K 0 lkpindev 0 0K 0 lkpimhi 0 0K 0 lkpifw 0 0K 0 lkpi80211 0 0K 0 NLM 0 0K 0 ipsec-spdcache 0 0K 0 ipsec-reg 0 0K 0 ipsec-misc 0 0K 0 ipsecrequest 0 0K 0 ip6opt 0 0K 26 ip6_msource 0 0K 0 ip6_moptions 0 0K 0 in6_mfilter 0 0K 0 frag6 0 0K 0 tcplog 0 0K 0 tcp_hwpace 0 0K 0 ip_msource 0 0K 0 ip_moptions 0 0K 0 in_mfilter 0 0K 0 ipid 0 0K 0 80211scan 0 0K 0 80211ratectl 0 0K 0 80211power 0 0K 0 80211nodeie 0 0K 0 80211node 0 0K 0 80211mesh_gt 0 0K 0 80211mesh_rt 0 0K 0 80211perr 0 0K 0 80211prep 0 0K 0 80211preq 0 0K 0 80211dfs 0 0K 0 80211crypto 0 0K 0 80211vap 0 0K 0 iflib 0 0K 0 vlan 0 0K 0 gif 0 0K 0 ifdescr 0 0K 0 zlib 0 0K 0 fadvise 0 0K 0 VN POLL 0 0K 4 statfs 0 0K 232 namei_tracker 0 0K 0 export_host 0 0K 0 cl_savebuf 0 0K 10 lio 0 0K 0 acl 0 0K 0 mbuf_tag 0 0K 0 ktls 0 0K 0 accf 0 0K 0 pts 0 0K 0 timerfd 0 0K 0 ioctlops 0 0K 333 eventfd 0 0K 0 Witness 0 0K 0 stack 0 0K 0 sbuf 0 0K 288 firmware 0 0K 0 compressor 0 0K 0 SWAP 0 0K 0 sysctltmp 0 0K 772 sysctl 0 0K 3 ekcd 0 0K 0 dumper 0 0K 0 sendfile 0 0K 0 rctl 0 0K 0 cache 0 0K 0 prison_racct 0 0K 0 Fail Points 0 0K 0 sigio 0 0K 1 filedesc_to_leader 0 0K 0 pwd 0 0K 0 tty console 0 0K 0 boottrace 0 0K 0 isofs_node 0 0K 0 isofs_mount 0 0K 0 tr_raid5_data 0 0K 0 tr_raid1e_data 0 0K 0 tr_raid1_data 0 0K 0 tr_raid0_data 0 0K 0 tr_concat_data 0 0K 0 md_sii_data 0 0K 0 md_promise_data 0 0K 0 md_nvidia_data 0 0K 0 md_jmicron_data 0 0K 0 md_intel_data 0 0K 0 md_ddf_data 0 0K 0 raid_data 0 0K 72 geom_flashmap 0 0K 0 tmpfs dir 0 0K 0 tmpfs name 0 0K 0 tmpfs mount 0 0K 0 tmpfs extattr 0 0K 0 NFS FHA 0 0K 0 newnfsmnt 0 0K 0 newnfsclient_req 0 0K 0 NFSCL layrecall 0 0K 0 NFSCL session 0 0K 0 NFSCL sockreq 0 0K 0 NFSCL devinfo 0 0K 0 NFSCL flayout 0 0K 0 NFSCL layout 0 0K 0 NFSD rollback 0 0K 0 NFSCL diroff 0 0K 0 NEWdirectio 0 0K 0 NEWNFSnode 0 0K 0 NFSCL lck 0 0K 0 NFSCL lckown 0 0K 0 NFSCL client 0 0K 0 NFSCL deleg 0 0K 0 NFSCL open 0 0K 0 NFSCL owner 0 0K 0 NFS fh 0 0K 0 NFS req 0 0K 0 NFSD usrgroup 0 0K 0 NFSD string 0 0K 0 NFSD V4lock 0 0K 0 NFSD V4state 0 0K 0 msdosfs_fat 0 0K 0 msdosfs_mount 0 0K 0 msdosfs_node 0 0K 0 DEVFS4 0 0K 0 DEVFS2 0 0K 0 gntdev 0 0K 0 privcmd_dev 0 0K 0 evtchn_dev 0 0K 0 xenstore 0 0K 0 xnb 0 0K 0 xen_acpi 0 0K 0 xbbd 0 0K 0 xbd 0 0K 0 Balloon 0 0K 0 sysmouse 0 0K 0 vtfont 0 0K 0 pvscsi 0 0K 0 USBdev 0 0K 0 USB 0 0K 0 twsbuf 0 0K 0 tcp_log_dev 0 0K 60 midi buffers 0 0K 0 mixer 0 0K 0 ac97 0 0K 0 hdacc 0 0K 0 hdac 0 0K 0 hdaa 0 0K 0 SIIS driver 0 0K 0 PUC 0 0K 0 ppbusdev 0 0K 0 sr_iov 0 0K 0 OCS 0 0K 0 OCS 0 0K 0 nvme 0 0K 0 nvd 0 0K 0 netmap 0 0K 0 mwldev 0 0K 0 MVS driver 0 0K 0 mrsasbuf 0 0K 0 mpt_user 0 0K 0 mps_user 0 0K 0 MPSSAS 0 0K 0 mps 0 0K 0 mpr_user 0 0K 0 MPRSAS 0 0K 0 mpr 0 0K 0 mfibuf 0 0K 0 md_sectors 0 0K 0 md_disk 0 0K 0 malodev 0 0K 0 LED 0 0K 0 ix_sriov 0 0K 0 ix 0 0K 0 ipsbuf 0 0K 0 ciss_data 0 0K 0 BACKLIGHT 0 0K 0 ath_hal 0 0K 0 athdev 0 0K 0 ata_pci 0 0K 0 ata_dma 0 0K 0 ata_generic 0 0K 0 AHCI driver 0 0K 0 agp 0 0K 0 acpipwr 0 0K 0 acpi_perf 0 0K 0 acpicmbat 0 0K 0 aacraidcam 0 0K 0 aacraid_buf 0 0K 0 aaccam 0 0K 0 aacbuf 0 0K 0 zstd 0 0K 0 XZ_DEC 0 0K 0 nvlist 0 0K 0 SCSI ENC 0 0K 0 SCSI sa 0 0K 0 scsi_pass 0 0K 0 scsi_da 0 0K 69 ata_da 0 0K 0 scsi_ch 0 0K 0 scsi_cd 0 0K 0 nvme_da 0 0K 0 CAM CCB 0 0K 523 CAM ccb queue 0 0K 0 db> show uma Zone Size Used Free Requests Sleeps Bucket Total Mem XFree mbuf_jumbo_page 4096 8320 1078 966097 0 254 38494208 0 mbuf 256 8864 16804 3387743 0 254 6571008 0 RADIX NODE 144 12102 185 139843 0 62 1769328 0 BUF TRIE 144 206 11610 28746 0 62 1701504 0 malloc-384 384 4273 17 4276 0 30 1647360 0 mbuf_cluster 2048 762 0 762 0 254 1560576 0 malloc-128 128 11504 90 12118 0 126 1484032 0 malloc-4096 4096 326 2 488 0 2 1343488 0 UMA Slabs 0 112 11224 2 11224 0 126 1257312 0 sctp_asoc 2256 57 453 1075 0 254 1150560 0 vmem btag 56 18531 84 18531 0 254 1042440 0 malloc-16384 16384 47 5 2474 0 1 851968 0 ertt_txseginfo 40 956 17426 594582 0 254 735280 0 256 Bucket 2048 317 41 11047 0 9 733184 0 FFS inode 1168 559 29 3986 0 8 686784 0 sctp_chunk 152 45 4037 148995 0 254 620464 0 sctp_ep 1176 148 362 3324 0 254 599760 0 lkpicurr 176 2 3080 2 0 62 542432 0 tcp_log 416 0 1278 3544 0 254 531648 0 pbuf 2624 0 198 0 0 2 519552 0 malloc-4096 4096 115 5 3852 0 2 491520 0 VM OBJECT 264 1774 86 46467 0 30 491040 0 socket 960 185 323 6325 0 254 487680 0 THREAD 1824 200 24 4228 0 8 408576 0 malloc-65536 65536 4 2 199 0 1 393216 0 sctp_raddr 736 57 460 1235 0 254 380512 0 malloc-2048 2048 155 21 3716 0 8 360448 0 VNODE 448 600 93 4029 0 30 310464 0 MAP ENTRY 96 2690 334 138966 0 126 290304 0 sctp_readq 152 0 1794 15309 0 254 272688 0 malloc-256 256 378 687 32330 0 62 272640 0 malloc-64 64 3792 303 3793 0 254 262080 0 malloc-16 16 14379 371 14448 0 254 236000 0 malloc-32768 32768 0 7 13513 0 1 229376 0 DEVCTL 1024 8 212 151 0 0 225280 0 malloc-8192 8192 26 0 39 0 1 212992 0 FPU_save_area 832 202 41 9121 0 16 202176 0 malloc-65536 65536 0 3 118 0 1 196608 0 mbuf_packet 256 120 642 30965 0 254 195072 0 malloc-128 128 1345 174 29604 0 126 194432 0 UMA Zones 768 239 0 239 0 16 183552 0 malloc-32 32 5281 389 5291 0 254 181440 0 FFS2 dinode 256 559 131 3986 0 62 176640 0 lkpimm 56 1 3095 1 0 254 173376 0 PROC 1376 100 21 3350 0 8 166496 0 malloc-128 128 1082 189 7114 0 126 162688 0 malloc-1024 1024 120 24 785 0 16 147456 0 S VFS Cache 104 1017 387 4675 0 126 146016 0 filedesc0 1072 101 25 3364 0 8 135072 0 malloc-65536 65536 2 0 2 0 1 131072 0 malloc-65536 65536 2 0 2 0 1 131072 0 malloc-32768 32768 2 2 122 0 1 131072 0 malloc-4096 4096 25 7 957 0 2 131072 0 malloc-2048 2048 10 54 1042 0 8 131072 0 unpcb 256 40 470 1234 0 254 130560 0 malloc-256 256 245 265 22603 0 62 130560 0 ksiginfo 112 105 939 5093 0 126 116928 0 tcp_bbr_map 128 264 635 35951 0 126 115072 0 malloc-128 128 513 386 13030 0 126 115072 0 malloc-256 256 300 135 5281 0 62 111360 0 malloc-384 384 206 64 5138 0 30 103680 0 malloc-32768 32768 3 0 3 0 1 98304 0 malloc-1024 1024 65 31 1073 0 16 98304 0 malloc-4096 4096 22 0 22 0 2 90112 0 UMA Kegs 384 226 7 226 0 30 89472 0 tcp_inpcb 1312 30 36 1383 0 8 86592 0 64 Bucket 512 90 78 10084 0 30 86016 0 g_bio 408 0 210 204618 0 30 85680 0 128 Bucket 1024 59 24 433 0 16 84992 0 malloc-64 64 1000 323 5386 0 254 84672 0 malloc-16384 16384 5 0 5 0 1 81920 0 malloc-8192 8192 10 0 12 0 1 81920 0 malloc-8192 8192 7 2 108 0 1 73728 0 VMSPACE 616 80 34 3331 0 16 70224 0 malloc-64 64 793 278 60185 0 254 68544 0 malloc-65536 65536 0 1 8 0 1 65536 0 malloc-32768 32768 2 0 2 0 1 65536 0 malloc-8192 8192 2 6 60 0 1 65536 0 malloc-4096 4096 12 4 259 0 2 65536 0 malloc-2048 2048 12 20 136 0 8 65536 0 malloc-1024 1024 20 44 1698 0 16 65536 0 32 Bucket 256 85 170 3525 0 62 65280 0 sctp_stream_msg_out 112 13 527 303 0 254 60480 0 malloc-384 384 10 140 4639 0 30 57600 0 udp_inpcb 424 6 120 249 0 30 53424 0 malloc-64 64 185 634 23096 0 254 52416 0 Files 80 487 163 18954 0 126 52000 0 malloc-256 256 3 192 8999 0 62 49920 0 malloc-256 256 39 156 7339 0 62 49920 0 tcp_rack_pcb 1024 23 25 672 0 16 49152 0 DIRHASH 1024 35 13 35 0 16 49152 0 NAMEI 1024 0 48 29647 0 16 49152 0 malloc-16384 16384 1 2 161 0 1 49152 0 malloc-384 384 100 20 148 0 30 46080 0 da_ccb 544 0 84 51288 0 16 45696 0 tcp_bbr_pcb 832 2 52 475 0 16 44928 0 syncache 168 0 264 4 0 254 44352 0 pcpu-8 8 4947 173 6098 0 254 40960 0 pipe 728 30 25 396 0 16 40040 0 malloc-64 64 33 534 57 0 254 36288 0 malloc-64 64 138 429 3781 0 254 36288 0 16 Bucket 144 85 167 2360 0 62 36288 0 tcp_rack_map 128 51 228 1581 0 126 35712 0 malloc-128 128 25 254 292 0 126 35712 0 malloc-128 128 41 238 61 0 126 35712 0 malloc-128 128 12 267 12 0 126 35712 0 ripcb 392 12 78 83 0 30 35280 0 itimer 352 0 99 189 0 30 34848 0 routing nhops 256 27 108 34 0 62 34560 0 ttyoutq 256 72 63 160 0 62 34560 0 malloc-256 256 52 83 652 0 62 34560 0 malloc-256 256 7 128 22 0 62 34560 0 TURNSTILE 136 225 27 225 0 62 34272 0 malloc-32768 32768 1 0 1 0 1 32768 0 malloc-2048 2048 4 12 35 0 8 32768 0 malloc-2048 2048 2 14 27 0 8 32768 0 malloc-2048 2048 2 14 2 0 8 32768 0 malloc-2048 2048 12 4 12 0 8 32768 0 malloc-2048 2048 0 16 18 0 8 32768 0 malloc-1024 1024 5 27 45 0 16 32768 0 malloc-1024 1024 16 16 25 0 16 32768 0 malloc-1024 1024 4 28 4 0 16 32768 0 malloc-512 512 1 63 119 0 30 32768 0 malloc-512 512 8 56 89 0 30 32768 0 malloc-512 512 3 61 116 0 30 32768 0 malloc-512 512 8 56 514 0 30 32768 0 malloc-512 512 2 62 419 0 30 32768 0 malloc-512 512 10 54 66 0 30 32768 0 pcpu-64 64 487 25 487 0 254 32768 0 PWD 40 61 747 2490 0 254 32320 0 KNOTE 160 30 170 48484 0 62 32000 0 ttyinq 160 135 65 300 0 62 32000 0 PGRP 120 34 230 49 0 126 31680 0 clpbuf 2624 0 12 72 0 4 31488 0 sctp_laddr 48 72 516 1241 0 254 28224 0 tcp_inpcb ports 32 25 857 1087 0 254 28224 0 malloc-32 32 326 556 3070 0 254 28224 0 4 Bucket 48 5 583 9 0 254 28224 0 2 Bucket 32 60 822 3247 0 254 28224 0 AIO 208 0 133 50 0 62 27664 0 udplite_inpcb 424 0 63 13 0 30 26712 0 cpuset 200 7 121 235 0 62 25600 0 malloc-8192 8192 3 0 3 0 1 24576 0 malloc-4096 4096 3 3 6 0 2 24576 0 rl_entry 40 115 491 115 0 254 24240 0 rtentry 168 30 114 34 0 62 24192 0 8 Bucket 80 57 243 1617 0 126 24000 0 domainset 40 0 567 208 0 254 22680 0 SLEEPQUEUE 88 225 31 225 0 126 22528 0 hostcache 64 2 313 2 0 254 20160 0 udplite_inpcb ports 32 0 630 13 0 254 20160 0 udp_inpcb ports 32 3 627 37 0 254 20160 0 ertt 72 30 250 1383 0 126 20160 0 malloc-64 64 2 313 2 0 254 20160 0 malloc-64 64 1 314 1 0 254 20160 0 malloc-32 32 81 549 150 0 254 20160 0 malloc-32 32 259 371 4187 0 254 20160 0 malloc-32 32 111 519 2594 0 254 20160 0 malloc-32 32 261 369 7735 0 254 20160 0 malloc-32 32 5 625 5 0 254 20160 0 malloc-32 32 2 628 13 0 254 20160 0 malloc-128 128 2 153 2 0 126 19840 0 cryptop 280 0 70 17 0 30 19600 0 AIOCB 552 0 35 54 0 16 19320 0 malloc-256 256 11 64 11 0 62 19200 0 epoch_record pcpu 256 4 60 4 0 62 16384 0 malloc-16384 16384 1 0 1 0 1 16384 0 malloc-4096 4096 0 4 27 0 2 16384 0 malloc-512 512 1 31 1 0 30 16384 0 SMR CPU 32 7 504 7 0 254 16352 0 vtnet_tx_hdr 24 0 668 500456 0 254 16032 0 malloc-16 16 525 475 7874 0 254 16000 0 kenv 258 17 43 1081 0 30 15480 0 mqnode 416 3 33 3 0 30 14976 0 vmem 1856 1 7 1 0 8 14848 0 SMR SHARED 24 7 504 7 0 254 12264 0 malloc-16 16 96 654 5003 0 254 12000 0 malloc-16 16 10 740 11 0 254 12000 0 malloc-16 16 132 618 2181 0 254 12000 0 malloc-16 16 71 679 31200 0 254 12000 0 malloc-16 16 23 727 23 0 254 12000 0 malloc-16 16 218 532 4927 0 254 12000 0 malloc-384 384 11 19 11 0 30 11520 0 malloc-384 384 1 29 1 0 30 11520 0 malloc-384 384 13 17 13 0 30 11520 0 Mountpoints 2816 2 2 2 0 4 11264 0 malloc-8192 8192 1 0 1 0 1 8192 0 pcpu-16 16 4 252 4 0 254 4096 0 UMA Slabs 1 176 9 13 9 0 62 3872 0 KMAP ENTRY 96 12 27 14 0 0 3744 0 FFS1 dinode 128 0 0 0 0 126 0 0 ada_ccb 272 0 0 0 0 30 0 0 swblk 136 0 0 0 0 62 0 0 swpctrie 144 0 0 0 0 62 0 0 cdg_qdiffsample 16 0 0 0 0 254 0 0 pf state scrubs 40 0 0 0 0 254 0 0 pf frag entries 40 0 0 0 0 254 0 0 pf frags 248 0 0 0 0 62 0 0 pf table entries 160 0 0 0 0 254 0 0 pf table entry counters 64 0 0 0 0 254 0 0 pf source nodes 152 0 0 0 0 254 0 0 pf state keys 88 0 0 0 0 126 0 0 pf states 360 0 0 0 0 254 0 0 pf tags 104 0 0 0 0 126 0 0 pf mtags 184 0 0 0 0 62 0 0 tfo_ccache_entries 80 0 0 0 0 126 0 0 tfo 4 0 0 0 0 254 0 0 sackhole 32 0 0 0 0 254 0 0 ipq 56 0 0 0 0 254 0 0 sctp_asconf_ack 48 0 0 0 0 254 0 0 sctp_asconf 40 0 0 0 0 254 0 0 tcp_log_id_node 120 0 0 0 0 126 0 0 tcp_log_id_bucket 176 0 0 0 0 62 0 0 tcpreass 48 0 0 0 0 254 0 0 ripcb ports 32 0 0 0 0 254 0 0 IPsec SA lft_c 16 0 0 0 0 254 0 0 AIOLIO 272 0 0 0 0 30 0 0 mqnotifier 216 0 0 0 0 62 0 0 mvdata 64 0 0 0 0 254 0 0 mqueue 248 0 0 0 0 62 0 0 TMPFS node 232 0 0 0 0 62 0 0 NCLNODE 608 0 0 0 0 16 0 0 LTS VFS Cache 360 0 0 0 0 30 0 0 L VFS Cache 320 0 0 0 0 30 0 0 STS VFS Cache 144 0 0 0 0 62 0 0 linux_dma_object 32 0 0 0 0 254 0 0 linux_dma_pctrie 144 0 0 0 0 62 0 0 IOMMU_MAP_ENTRY 104 0 0 0 0 126 0 0 mbuf_jumbo_16k 16384 0 0 0 0 254 0 0 mbuf_jumbo_9k 9216 0 0 0 0 254 0 0 audit_record 1280 0 0 0 0 8 0 0 MAC labels 40 0 0 0 0 254 0 0 vnpbuf 2624 0 0 0 0 16 0 0 nfspbuf 2624 0 0 0 0 4 0 0 swwbuf 2624 0 0 0 0 2 0 0 swrbuf 2624 0 0 0 0 4 0 0 umtx_shm 88 0 0 0 0 126 0 0 umtx pi 96 0 0 0 0 126 0 0 rangeset pctrie nodes 144 0 0 0 0 62 0 0 malloc-65536 65536 0 0 0 0 1 0 0 malloc-65536 65536 0 0 0 0 1 0 0 malloc-65536 65536 0 0 0 0 1 0 0 malloc-32768 32768 0 0 0 0 1 0 0 malloc-32768 32768 0 0 0 0 1 0 0 malloc-32768 32768 0 0 0 0 1 0 0 malloc-16384 16384 0 0 0 0 1 0 0 malloc-16384 16384 0 0 0 0 1 0 0 malloc-16384 16384 0 0 0 0 1 0 0 malloc-16384 16384 0 0 0 0 1 0 0 malloc-8192 8192 0 0 0 0 1 0 0 malloc-8192 8192 0 0 0 0 1 0 0 malloc-4096 4096 0 0 0 0 2 0 0 malloc-1024 1024 0 0 0 0 16 0 0 malloc-1024 1024 0 0 0 0 16 0 0 malloc-512 512 0 0 0 0 30 0 0 malloc-384 384 0 0 0 0 30 0 0 pcpu-32 32 0 0 0 0 254 0 0 pcpu-4 4 0 0 0 0 254 0 0 fakepg 104 0 0 0 0 126 0 0 UMA Hash 256 0 0 0 0 62 0 0