panic: lock (sleep mutex) sctp-inp not locked @ /syzkaller/managers/main/kernel/sys/netinet/sctp_usrreq.c:958 cpuid = 0 time = 1705466795 KDB: stack backtrace: db_trace_self_wrapper() at db_trace_self_wrapper+0xc6/frame 0xfffffe00741f05b0 kdb_backtrace() at kdb_backtrace+0xd0/frame 0xfffffe00741f0710 vpanic() at vpanic+0x271/frame 0xfffffe00741f08b0 panic() at panic+0xb5/frame 0xfffffe00741f0970 witness_unlock() at witness_unlock+0x4fd/frame 0xfffffe00741f09f0 __mtx_unlock_flags() at __mtx_unlock_flags+0xec/frame 0xfffffe00741f0ab0 sctp_shutdown() at sctp_shutdown+0xf35/frame 0xfffffe00741f0c10 soshutdown() at soshutdown+0xb3/frame 0xfffffe00741f0c50 kern_shutdown() at kern_shutdown+0x154/frame 0xfffffe00741f0d10 amd64_syscall() at amd64_syscall+0x473/frame 0xfffffe00741f0f30 fast_syscall_common() at fast_syscall_common+0xf8/frame 0xfffffe00741f0f30 --- syscall (198, FreeBSD ELF64, __syscall), rip = 0x2aeb2a, rsp = 0x8257fcf08, rbp = 0x8257fcf80 --- KDB: enter: panic [ thread pid 2512 tid 102118 ] Stopped at kdb_enter+0x6e: movq $0,0x2194267(%rip) db> db> set $lines = 0 db> set $maxwidth = 0 db> show registers cs 0x20 ds 0x3b es 0x3b fs 0x13 gs 0x1b ss 0x28 rax 0x12 rcx 0xfffffe006d400000 rdx 0x3ffff rbx 0xffffffff826f86c0 .str.28 rsp 0xfffffe00741f06f0 rbp 0xfffffe00741f0710 rsi 0x40001 rdi 0xffffffff815de326 printf+0x156 r8 0 r9 0xffffffff r10 0 r11 0xfffffe00749c8c60 r12 0 r13 0xfffffe00749c8740 r14 0xffffffff826f86c0 .str.28 r15 0 rip 0xffffffff815cadae kdb_enter+0x6e rflags 0x46 kdb_enter+0x6e: movq $0,0x2194267(%rip) db> show proc Process 2512 (syz-executor.0) at 0xfffffe0057180020: state: NORMAL uid: 0 gids: 0, 0, 5 parent: pid 772 at 0xfffffe006cb1ab00 ABI: FreeBSD ELF64 flag: 0x10000080 flag2: 0 arguments: /root/syz-executor.0 exec reaper: 0xfffffe00541d1040 reapsubtree: 1 sigparent: 20 vmspace: 0xfffffe00742a24d0 (map 0xfffffe00742a24d0) (map.pmap 0xfffffe00742a2590) (pmap 0xfffffe00742a2600) threads: 3 100085 RunQ syz-executor.0 102116 S connec 0xfffffe0058ac9c1a syz-executor.0 102118 Run CPU 0 syz-executor.0 db> ps pid ppid pgrp uid state wmesg wchan cmd 2521 1 1 0 R init 2520 788 788 0 R syz-executor.3 2519 778 778 0 R (threaded) syz-executor.2 100096 RunQ syz-executor.2 102119 RunQ syz-executor.2 2518 1 1 0 S nanslp 0xffffffff83740401 init 2516 1 1 0 S nanslp 0xffffffff83740401 init 2512 772 772 0 R (threaded) syz-executor.0 100085 RunQ syz-executor.0 102116 S connec 0xfffffe0058ac9c1a syz-executor.0 102118 Run CPU 0 syz-executor.0 2506 1 1 0 S nanslp 0xffffffff83740401 init 2505 1 1 0 S nanslp 0xffffffff83740400 init 2504 1 1 0 S nanslp 0xffffffff83740400 init 2503 1 1 0 S nanslp 0xffffffff83740401 init 2459 773 773 0 R (threaded) syz-executor.1 100097 Run CPU 1 syz-executor.1 102095 S uwait 0xfffffe0072031e80 syz-executor.1 102098 S uwait 0xfffffe006cd86200 syz-executor.1 102101 S uwait 0xfffffe0074211380 syz-executor.1 102103 S uwait 0xfffffe006cd83000 syz-executor.1 1718 0 0 0 DL aiordy 0xfffffe0074278040 [aiod4] 1717 0 0 0 DL aiordy 0xfffffe0074496000 [aiod3] 1716 0 0 0 DL aiordy 0xfffffe006cceeac0 [aiod2] 1715 0 0 0 DL aiordy 0xfffffe0074496ac0 [aiod1] 1580 0 0 0 DL mdwait 0xfffffe0074510000 [md3] 1577 0 0 0 DL mdwait 0xfffffe006d1ae000 [md2] 1573 0 0 0 DL mdwait 0xfffffe006d1fc000 [md1] 1570 0 0 0 DL mdwait 0xfffffe006d1fe000 [md0] 1213 1 773 0 S uwait 0xfffffe0057a76500 syz-executor.1 1207 1 773 0 S uwait 0xfffffe0074212080 syz-executor.1 1204 1 773 0 S uwait 0xfffffe0074213200 syz-executor.1 1199 1 773 0 S uwait 0xfffffe0074213100 syz-executor.1 1196 1 773 0 S uwait 0xfffffe0074212d80 syz-executor.1 1194 1 773 0 S uwait 0xfffffe0074212c80 syz-executor.1 1190 1 778 0 S uwait 0xfffffe0074212b80 syz-executor.2 1188 1 773 0 S uwait 0xfffffe006cd84680 syz-executor.1 1187 1 788 0 S uwait 0xfffffe006cd83500 syz-executor.3 1184 1 778 0 S uwait 0xfffffe0057a79b00 syz-executor.2 1183 1 788 0 S uwait 0xfffffe0072031800 syz-executor.3 1182 1 773 0 S uwait 0xfffffe0072032700 syz-executor.1 1178 1 788 0 S uwait 0xfffffe005750a500 syz-executor.3 1177 1 773 0 S uwait 0xfffffe005750df00 syz-executor.1 1174 1 778 0 S uwait 0xfffffe006cd84380 syz-executor.2 1172 1 788 0 S uwait 0xfffffe006cd83180 syz-executor.3 1171 1 773 0 S uwait 0xfffffe0057a78480 syz-executor.1 1169 1 778 0 S uwait 0xfffffe0057a77080 syz-executor.2 1167 1 788 0 S uwait 0xfffffe006cd84b80 syz-executor.3 1164 1 773 0 S uwait 0xfffffe006cd83c00 syz-executor.1 1163 1 778 0 S uwait 0xfffffe0057a79f00 syz-executor.2 1162 1 788 0 S uwait 0xfffffe0057a79a00 syz-executor.3 1159 1 778 0 S uwait 0xfffffe006cd86080 syz-executor.2 1156 1 773 0 S uwait 0xfffffe0057a79c00 syz-executor.1 1155 1 788 0 S uwait 0xfffffe006cd85480 syz-executor.3 1151 1 773 0 S uwait 0xfffffe006cd85000 syz-executor.1 1149 1 778 0 S uwait 0xfffffe006cd86280 syz-executor.2 1147 1 788 0 S uwait 0xfffffe006cd85880 syz-executor.3 1146 1 778 0 S uwait 0xfffffe005750c300 syz-executor.2 1142 1 778 0 S uwait 0xfffffe005750de00 syz-executor.2 1141 1 788 0 S uwait 0xfffffe0057a79e00 syz-executor.3 1138 1 778 0 S uwait 0xfffffe0057a79d00 syz-executor.2 1113 1 778 0 S uwait 0xfffffe0057a76080 syz-executor.2 1084 1 778 0 S uwait 0xfffffe0057a76f00 syz-executor.2 1075 1 773 0 S uwait 0xfffffe006cd83700 syz-executor.1 1073 1 778 0 S uwait 0xfffffe005750af00 syz-executor.2 1071 1 772 0 S uwait 0xfffffe0007989880 syz-executor.0 1069 1 788 0 S uwait 0xfffffe006cd85380 syz-executor.3 1067 1 773 0 S uwait 0xfffffe006cd84180 syz-executor.1 1066 1 778 0 S uwait 0xfffffe006cd84980 syz-executor.2 1063 1 772 0 S uwait 0xfffffe006cd84080 syz-executor.0 1061 1 788 0 S uwait 0xfffffe006cd83800 syz-executor.3 1059 1 778 0 S uwait 0xfffffe0057a76180 syz-executor.2 1058 1 773 0 S uwait 0xfffffe006cd85100 syz-executor.1 1055 1 772 0 S uwait 0xfffffe006cd84480 syz-executor.0 1053 1 788 0 S uwait 0xfffffe006cd84a80 syz-executor.3 1051 1 778 0 S uwait 0xfffffe006cd83b00 syz-executor.2 1049 1 773 0 S uwait 0xfffffe006cd85300 syz-executor.1 1047 1 772 0 S uwait 0xfffffe005750da00 syz-executor.0 1045 1 778 0 S uwait 0xfffffe005750db00 syz-executor.2 1043 1 788 0 S uwait 0xfffffe006cd83600 syz-executor.3 1041 1 773 0 S uwait 0xfffffe005750dd00 syz-executor.1 1039 1 772 0 S uwait 0xfffffe006cd84580 syz-executor.0 1037 1 788 0 S uwait 0xfffffe006cd83d00 syz-executor.3 1035 1 778 0 S uwait 0xfffffe006cd84d80 syz-executor.2 1033 1 773 0 S uwait 0xfffffe006cd85200 syz-executor.1 1032 1 772 0 S uwait 0xfffffe006cd84780 syz-executor.0 1031 1 788 0 S uwait 0xfffffe006cd83a00 syz-executor.3 1028 1 778 0 S uwait 0xfffffe006cd83900 syz-executor.2 1023 1 778 0 S uwait 0xfffffe006cd84280 syz-executor.2 788 768 788 0 Ss nanslp 0xffffffff83740400 syz-executor.3 778 768 778 0 Rs syz-executor.2 773 768 773 0 Rs syz-executor.1 772 768 772 0 Ss nanslp 0xffffffff83740400 syz-executor.0 768 1 766 0 S (threaded) syz-fuzzer 100094 S uwait 0xfffffe005750b700 syz-fuzzer 100113 S uwait 0xfffffe0057a77980 syz-fuzzer 100114 S uwait 0xfffffe0057a77a80 syz-fuzzer 100115 S wait 0xfffffe0057b9c5a0 syz-fuzzer 100116 S uwait 0xfffffe006cd86700 syz-fuzzer 100117 S uwait 0xfffffe0057a76900 syz-fuzzer 100118 S wait 0xfffffe0057b9c5a0 syz-fuzzer 100120 S uwait 0xfffffe0057a76c00 syz-fuzzer 100122 S wait 0xfffffe0057b9c5a0 syz-fuzzer 100123 S wait 0xfffffe0057b9c5a0 syz-fuzzer 100124 S uwait 0xfffffe0007989600 syz-fuzzer 100142 S kqread 0xfffffe0057bbcc00 syz-fuzzer 100280 S uwait 0xfffffe006cd84880 syz-fuzzer 764 682 764 0 Ss select 0xfffffe00571b1d40 sshd 682 1 682 0 Ss select 0xfffffe00571b60c0 sshd 17 0 0 0 DL syncer 0xffffffff8385db20 [syncer] 16 0 0 0 DL vlruwt 0xfffffe0057181040 [vnlru] 15 0 0 0 DL (threaded) [bufdaemon] 100079 D psleep 0xffffffff8385c100 [bufdaemon] 100080 D - 0xffffffff82c0a140 [bufspacedaemon-0] 100095 D sdflush 0xfffffe00588a34e8 [/ worker] 9 0 0 0 DL psleep 0xffffffff838a55c0 [vmdaemon] 8 0 0 0 DL (threaded) [pagedaemon] 100077 D psleep 0xffffffff8388d438 [dom0] 100081 D launds 0xffffffff8388d444 [laundry: dom0] 100082 D umarcl 0xffffffff81d61000 [uma] 7 0 0 0 DL - 0xffffffff834bdc10 [rand_harvestq] 6 0 0 0 DL pftm 0xffffffff83f72ea0 [pf purge] 5 0 0 0 DL waiting 0xffffffff843905a0 [sctp_iterator] 4 0 0 0 DL (threaded) [cam] 100044 D - 0xffffffff83488340 [doneq0] 100045 D - 0xffffffff834882c0 [async] 100076 D - 0xffffffff83488140 [scanner] 3 0 0 0 DL (threaded) [crypto] 100041 D crypto_ 0xffffffff83888c60 [crypto] 100042 D crypto_ 0xfffffe00542d5730 [crypto returns 0] 100043 D crypto_ 0xfffffe00542d5780 [crypto returns 1] 14 0 0 0 DL seqstat 0xfffffe000796e888 [sequencer 00] 13 0 0 0 DL (threaded) [geom] 100035 D - 0xffffffff836e8d40 [g_event] 100036 D - 0xffffffff836e8d60 [g_up] 100037 D - 0xffffffff836e8d80 [g_down] 2 0 0 0 WL (threaded) [clock] 100029 I [clock (0)] 100030 I [clock (1)] 12 0 0 0 WL (threaded) [intr] 100012 I [swi6: task queue] 100013 I [swi6: Giant taskq] 100015 I [swi5: fast taskq] 100031 I [swi1: netisr 0] 100032 I [swi1: hpts] 100033 I [swi1: hpts] 100046 I [irq24: virtio_pci0] 100047 I [irq25: virtio_pci0] 100048 I [irq26: virtio_pci0] 100049 I [irq27: virtio_pci0] 100050 I [irq28: virtio_pci1] 100051 I [irq29: virtio_pci1] 100052 I [irq30: virtio_pci1] 100053 I [irq31: virtio_pci1] 100054 I [irq32: virtio_pci1] 100059 I [irq33: virtio_pci2] 100060 I [irq34: virtio_pci2] 100061 I [irq35: virtio_pci2] 100063 I [irq1: atkbd0] 100064 I [irq12: psm0] 100065 I [swi0: uart uart++] 100069 I [swi1: pf send] 11 0 0 0 RL (threaded) [idle] 100003 CanRun [idle: cpu0] 100004 CanRun [idle: cpu1] 1 0 1 0 RLs [init] 10 0 0 0 DL audit_w 0xffffffff838896c0 [audit] 0 0 0 0 DLs (threaded) [kernel] 100000 D swapin 0xffffffff836e9720 [swapper] 100005 D - 0xfffffe0054282100 [softirq_0] 100006 D - 0xfffffe0054282000 [softirq_1] 100007 D - 0xfffffe0054281e00 [if_io_tqg_0] 100008 D - 0xfffffe0054281d00 [if_io_tqg_1] 100009 D - 0xfffffe0054281c00 [if_config_tqg_0] 100010 D - 0xfffffe0007977900 [pci_hp taskq] 100011 D - 0xfffffe0007977800 [kqueue_ctx taskq] 100014 D - 0xfffffe0007977500 [thread taskq] 100016 D - 0xfffffe0007977300 [aiod_kick taskq] 100017 D - 0xfffffe0007977200 [deferred_unmount ta] 100018 D - 0xfffffe0007977100 [inm_free taskq] 100019 D - 0xfffffe0007977000 [in6m_free taskq] 100020 D - 0xfffffe0007976e00 [linuxkpi_irq_wq] 100021 D - 0xfffffe0007976d00 [linuxkpi_short_wq_0] 100022 D - 0xfffffe0007976d00 [linuxkpi_short_wq_1] 100023 D - 0xfffffe0007976d00 [linuxkpi_short_wq_2] 100024 D - 0xfffffe0007976d00 [linuxkpi_short_wq_3] 100025 D - 0xfffffe0007976c00 [linuxkpi_long_wq_0] 100026 D - 0xfffffe0007976c00 [linuxkpi_long_wq_1] 100027 D - 0xfffffe0007976c00 [linuxkpi_long_wq_2] 100028 D - 0xfffffe0007976c00 [linuxkpi_long_wq_3] 100034 D - 0xfffffe0007976a00 [firmware taskq] 100039 D - 0xfffffe0007976700 [crypto_0] 100040 D - 0xfffffe0007976700 [crypto_1] 100055 D - 0xfffffe00571c4b00 [vtnet0 rxq 0] 100056 D - 0xfffffe00571c4a00 [vtnet0 txq 0] 100057 D - 0xfffffe00571c4900 [vtnet0 rxq 1] 100058 D - 0xfffffe00571c4800 [vtnet0 txq 1] 100062 D vtbslp 0xfffffe00571b6580 [virtio_balloon] 100066 D - 0xffffffff826fd920 [deadlkres] 100070 D - 0xfffffe0057bbd100 [acpi_task_0] 100071 D - 0xfffffe0057bbd100 [acpi_task_1] 100072 D - 0xfffffe0057bbd100 [acpi_task_2] 100074 D - 0xfffffe0007977c00 [mca taskq] 100075 D - 0xfffffe0007976500 [CAM taskq] 2513 1 1 0 Z init db> show all locks db> show malloc Type InUse MemUse Requests pf_hash 5 11524K 5 tcp_hpts 7 4801K 7 devbuf 4192 4324K 4224 sysctloid 35285 2079K 35356 vtbuf 24 1968K 46 kobj 326 1304K 524 filedesc 137 1095K 2601 pcb 364 1053K 3833 newblk 11 1027K 4889 vfscache 3 1025K 3 inodedep 300 625K 1501 ufs_quota 1 512K 1 vfs_hash 1 512K 1 callout 2 512K 2 subproc 249 474K 2694 intr 4 472K 4 vmem 3 266K 6 filemon 28 224K 75 vnet_data 2 224K 2 acpitask 1 224K 1 acpica 1674 184K 60830 tidhash 3 141K 3 pagedep 10 131K 1251 linker 352 130K 386 tfo_ccache 1 128K 1 IP reass 1 128K 1 DEVFS1 113 113K 130 sem 4 106K 4 gtaskqueue 18 98K 18 sctp_atcl 258 97K 2559 sctp_stro 87 87K 606 bus 985 81K 5155 dirrem 288 72K 1367 mtx_pool 2 72K 2 syncache 1 68K 1 NFSD srvcache 3 68K 3 module 513 65K 513 ddb_capture 1 64K 1 kdtrace 324 62K 4642 umtx 480 60K 480 temp 35 37K 2885 freefile 287 36K 1304 DEVFS3 132 33K 142 hostcache 1 32K 1 shm 1 32K 1 msg 4 30K 4 kbdmux 6 28K 6 LRO 20 21K 20 DEVFS_RULE 56 20K 56 ifaddr 68 20K 70 ufs_mount 4 17K 5 proc 3 17K 3 md_disk 8 17K 8 tty 16 16K 16 md_sectors 4 16K 4 routetbl 129 16K 411 ithread 97 16K 97 GEOM 89 15K 601 lltable 46 15K 51 bus-sc 34 15K 1687 sctp_atky 345 14K 3282 eventhandler 161 14K 161 KTRACE 100 13K 128 ifnet 7 13K 7 ether_multi 152 13K 162 kenv 95 12K 95 rman 86 11K 431 CAM queue 5 11K 1528 sctp_timw 38 10K 38 kqueue 107 9K 2526 CC Mem 35 9K 757 in6_multi 65 9K 65 tcp_fsb 4 9K 380 bmsafemap 2 9K 1481 rpc 4 9K 4 devstat 4 9K 4 UART 12 9K 12 ksem 1 8K 34 shmfd 1 8K 1 pfs_vncache 1 8K 1 audit_evclass 238 8K 300 taskqueue 63 7K 258 pwddesc 104 7K 2573 sglist 6 7K 6 CAM DEV 3 6K 510 DEVFSP 86 6K 303 pfs_nodes 20 5K 20 hhook 15 5K 17 ufs_dirhash 24 5K 24 UMA 271 5K 271 pf_ifnet 10 5K 19 vt 11 5K 11 sctp_athm 258 5K 2659 memdesc 1 4K 1 MCA 32 4K 32 evdev 4 4K 4 acpisem 28 4K 28 kcovinfo 52 4K 52 terminal 11 3K 11 cred 11 3K 499 sctp_map 174 3K 1318 proc-args 103 3K 4062 clone 9 3K 9 uidinfo 2 3K 16 local_apic 1 2K 1 io_apic 1 2K 1 ipsec-saq 2 2K 2 ip6ndp 12 2K 13 plimit 7 2K 3679 sctp_ifa 13 2K 14 tun 7 2K 7 CAM XPT 22 2K 543 freework 7 2K 2444 Unitno 25 2K 19119 msi 12 2K 12 freeblks 6 2K 1329 in_multi 6 2K 8 toponodes 6 2K 6 lockf 12 2K 74 ipsecpolicy 2 2K 2 acpidev 20 2K 20 inpcbpolicy 39 2K 952 softdep 1 1K 1 mkdir 8 1K 2460 sahead 1 1K 1 secasvar 1 1K 1 nhops 6 1K 8 vnodemarker 2 1K 48 session 8 1K 166 NFSD session 1 1K 1 osd 40 1K 743 CAM periph 4 1K 271 sctp_ifn 6 1K 14 ipsec 3 1K 3 diradd 6 1K 1402 mld 6 1K 6 igmp 6 1K 6 pfil 6 1K 6 BPF 6 1K 65 isadev 6 1K 6 mount 16 1K 89 pci_link 10 1K 10 crypto 4 1K 118 netlink 3 1K 35 encap_export_host 12 1K 12 newdirblk 4 1K 1230 selfd 8 1K 30286 cdev 2 1K 2 lkpikmalloc 8 1K 9 chacha20random 1 1K 1 biobuf 1 1K 1 vnodes 1 1K 1 select 2 1K 37 NFSD lckfile 1 1K 1 NFSD V4client 1 1K 1 DEVFS 9 1K 10 CAM SIM 2 1K 2 feeder 7 1K 7 tcpfunc 3 1K 3 loginclass 3 1K 6 prison 6 1K 6 cryptodev 2 1K 267 nexusdev 8 1K 8 apmdev 1 1K 1 atkbddev 2 1K 2 aio 4 1K 4 pmchooks 1 1K 1 CAM path 4 1K 1034 CAM dev queue 2 1K 2 CAM I/O Scheduler 1 1K 1 sctp_vrf 1 1K 1 vnet 1 1K 1 pmc 1 1K 1 entropy 2 1K 39 acpiintr 1 1K 1 cpus 2 1K 2 vnet_data_free 1 1K 1 soname 1 1K 6195 Per-cpu 1 1K 1 p1003.1b 1 1K 1 tcp_do 0 0K 0 mqdata 0 0K 0 sctp_mcore 0 0K 0 sctp_socko 0 0K 884 sctp_iter 0 0K 19 sctp_mvrf 0 0K 0 sctp_cpal 0 0K 7 sctp_cmsg 0 0K 0 sctp_stre 0 0K 0 sctp_athi 0 0K 0 sctp_a_it 0 0K 12 sctp_aadr 0 0K 0 sctp_stri 0 0K 200 ipcomp 0 0K 0 esp 0 0K 0 ah 0 0K 0 pf_table 0 0K 0 pf_rule 0 0K 4 pf_altq 0 0K 0 pf_osfp 0 0K 0 pf_krule_item 0 0K 0 pf_temp 0 0K 0 madt_table 0 0K 2 smartpqi 0 0K 0 ixl 0 0K 0 ice-resmgr 0 0K 0 ice-osdep 0 0K 0 ice 0 0K 0 iavf 0 0K 0 axgbe 0 0K 0 fpukern_ctx 0 0K 0 xen_intr 0 0K 0 xen_hvm 0 0K 0 legacydrv 0 0K 0 bounce 0 0K 0 busdma 0 0K 0 qpidrv 0 0K 0 dmar_idpgtbl 0 0K 0 dmar_dom 0 0K 0 dmar_ctx 0 0K 0 isci 0 0K 0 iommu_dmamap 0 0K 0 hyperv_socket 0 0K 0 bxe_ilt 0 0K 0 aesni_data 0 0K 0 xenbus 0 0K 0 vm_fictitious 0 0K 0 UMAHash 0 0K 0 vm_pgdata 0 0K 0 jblocks 0 0K 0 savedino 0 0K 1251 sentinel 0 0K 0 jfsync 0 0K 0 jtrunc 0 0K 0 sbdep 0 0K 22 jsegdep 0 0K 0 jseg 0 0K 0 jfreefrag 0 0K 0 jfreeblk 0 0K 0 jnewblk 0 0K 0 jmvref 0 0K 0 jremref 0 0K 0 jaddref 0 0K 0 freedep 0 0K 0 freefrag 0 0K 2 allocindir 0 0K 0 indirdep 0 0K 1695 allocdirect 0 0K 0 ufs_trim 0 0K 0 mactemp 0 0K 0 audit_trigger 0 0K 0 audit_pipe_presel 0 0K 0 audit_pipeent 0 0K 0 audit_pipe 0 0K 0 audit_evname 0 0K 0 audit_bsm 0 0K 0 audit_gidset 0 0K 0 audit_text 0 0K 0 audit_path 0 0K 0 audit_data 0 0K 0 audit_cred 0 0K 0 ktls_ocf 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5E_TLS_RX 0 0K 0 MLX5EEPROM 0 0K 0 MLX5E_TLS 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EN 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5DUMP 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 simple_attr 0 0K 0 seq_file 0 0K 0 lkpiskb 0 0K 0 radix 0 0K 0 idr 0 0K 0 lkpindev 0 0K 0 lkpimhi 0 0K 0 lkpifw 0 0K 0 lkpi80211 0 0K 0 NLM 0 0K 0 ipsec-spdcache 0 0K 0 ipsec-reg 0 0K 0 ipsec-misc 0 0K 0 ipsecrequest 0 0K 0 ip6opt 0 0K 3 ip6_msource 0 0K 0 ip6_moptions 0 0K 0 in6_mfilter 0 0K 0 frag6 0 0K 0 tcplog 0 0K 0 tcp_hwpace 0 0K 0 ip_msource 0 0K 0 ip_moptions 0 0K 0 in_mfilter 0 0K 0 ipid 0 0K 0 80211scan 0 0K 0 80211ratectl 0 0K 0 80211power 0 0K 0 80211nodeie 0 0K 0 80211node 0 0K 0 80211mesh_gt 0 0K 0 80211mesh_rt 0 0K 0 80211perr 0 0K 0 80211prep 0 0K 0 80211preq 0 0K 0 80211dfs 0 0K 0 80211crypto 0 0K 0 80211vap 0 0K 0 iflib 0 0K 0 vlan 0 0K 0 gif 0 0K 0 ifdescr 0 0K 0 zlib 0 0K 0 fadvise 0 0K 0 VN POLL 0 0K 0 statfs 0 0K 192 namei_tracker 0 0K 0 export_host 0 0K 0 cl_savebuf 0 0K 2 lio 0 0K 4 acl 0 0K 0 mbuf_tag 0 0K 0 ktls 0 0K 0 accf 0 0K 0 pts 0 0K 0 timerfd 0 0K 0 procdesc 0 0K 10 iov 0 0K 18707 ioctlops 0 0K 158 eventfd 0 0K 0 Witness 0 0K 0 stack 0 0K 0 sbuf 0 0K 336 firmware 0 0K 0 compressor 0 0K 0 SWAP 0 0K 0 sysctltmp 0 0K 1072 sysctl 0 0K 3 ekcd 0 0K 0 dumper 0 0K 0 sendfile 0 0K 0 rctl 0 0K 0 cache 0 0K 0 prison_racct 0 0K 0 Fail Points 0 0K 0 filecaps 0 0K 79 sigio 0 0K 1 filedesc_to_leader 0 0K 0 pwd 0 0K 0 tty console 0 0K 0 boottrace 0 0K 0 isofs_node 0 0K 0 isofs_mount 0 0K 0 tr_raid5_data 0 0K 0 tr_raid1e_data 0 0K 0 tr_raid1_data 0 0K 0 tr_raid0_data 0 0K 0 tr_concat_data 0 0K 0 md_sii_data 0 0K 0 md_promise_data 0 0K 0 md_nvidia_data 0 0K 0 md_jmicron_data 0 0K 0 md_intel_data 0 0K 0 md_ddf_data 0 0K 0 raid_data 0 0K 96 geom_flashmap 0 0K 0 tmpfs dir 0 0K 0 tmpfs name 0 0K 0 tmpfs mount 0 0K 0 tmpfs extattr 0 0K 0 NFS FHA 0 0K 0 newnfsmnt 0 0K 0 newnfsclient_req 0 0K 0 NFSCL layrecall 0 0K 0 NFSCL session 0 0K 0 NFSCL sockreq 0 0K 0 NFSCL devinfo 0 0K 0 NFSCL flayout 0 0K 0 NFSCL layout 0 0K 0 NFSD rollback 0 0K 0 NFSCL diroff 0 0K 0 NEWdirectio 0 0K 0 NEWNFSnode 0 0K 0 NFSCL lck 0 0K 0 NFSCL lckown 0 0K 0 NFSCL client 0 0K 0 NFSCL deleg 0 0K 0 NFSCL open 0 0K 0 NFSCL owner 0 0K 0 NFS fh 0 0K 0 NFS req 0 0K 0 NFSD usrgroup 0 0K 0 NFSD string 0 0K 0 NFSD V4lock 0 0K 0 NFSD V4state 0 0K 0 msdosfs_fat 0 0K 0 msdosfs_mount 0 0K 0 msdosfs_node 0 0K 0 DEVFS4 0 0K 0 DEVFS2 0 0K 0 gntdev 0 0K 0 privcmd_dev 0 0K 0 evtchn_dev 0 0K 0 xenstore 0 0K 0 xnb 0 0K 0 xen_acpi 0 0K 0 xbbd 0 0K 0 xbd 0 0K 0 Balloon 0 0K 0 sysmouse 0 0K 0 vtfont 0 0K 0 pvscsi 0 0K 0 USBdev 0 0K 0 USB 0 0K 0 twsbuf 0 0K 0 tcp_log_dev 0 0K 10 midi buffers 0 0K 0 mixer 0 0K 0 ac97 0 0K 0 hdacc 0 0K 0 hdac 0 0K 0 hdaa 0 0K 0 SIIS driver 0 0K 0 PUC 0 0K 0 ppbusdev 0 0K 0 sr_iov 0 0K 0 OCS 0 0K 0 OCS 0 0K 0 nvme 0 0K 0 nvd 0 0K 0 netmap 0 0K 0 mwldev 0 0K 0 MVS driver 0 0K 0 mrsasbuf 0 0K 0 mpt_user 0 0K 0 mps_user 0 0K 0 MPSSAS 0 0K 0 mps 0 0K 0 mpr_user 0 0K 0 MPRSAS 0 0K 0 mpr 0 0K 0 mfibuf 0 0K 0 malodev 0 0K 0 LED 0 0K 0 ix_sriov 0 0K 0 ix 0 0K 0 ipsbuf 0 0K 0 ciss_data 0 0K 0 BACKLIGHT 0 0K 0 ath_hal 0 0K 0 athdev 0 0K 0 ata_pci 0 0K 0 ata_dma 0 0K 0 ata_generic 0 0K 0 AHCI driver 0 0K 0 agp 0 0K 0 acpipwr 0 0K 0 acpi_perf 0 0K 0 acpicmbat 0 0K 0 aacraidcam 0 0K 0 aacraid_buf 0 0K 0 aaccam 0 0K 0 aacbuf 0 0K 0 zstd 0 0K 0 XZ_DEC 0 0K 0 nvlist 0 0K 0 SCSI ENC 0 0K 0 SCSI sa 0 0K 0 scsi_pass 0 0K 0 scsi_da 0 0K 69 ata_da 0 0K 0 scsi_ch 0 0K 0 scsi_cd 0 0K 0 nvme_da 0 0K 0 CAM CCB 0 0K 523 CAM ccb queue 0 0K 0 db> show uma Zone Size Used Free Requests Sleeps Bucket Total Mem XFree mbuf_jumbo_page 4096 8334 1064 207462 0 254 38494208 0 mbuf 256 13854 11559 1265157 0 254 6505728 0 BUF TRIE 144 200 11616 3393 0 62 1701504 0 malloc-384 384 4295 25 4298 0 30 1658880 0 mbuf_cluster 2048 762 0 762 0 254 1560576 0 malloc-128 128 11659 214 12284 0 126 1519744 0 malloc-4096 4096 326 2 524 0 2 1343488 0 UMA Slabs 0 112 11248 14 11248 0 126 1261344 0 malloc-16384 16384 68 4 1298 0 1 1179648 0 RADIX NODE 144 7372 715 71162 0 62 1164528 0 sctp_asoc 2256 87 423 606 0 254 1150560 0 vmem btag 56 18907 140 18907 0 254 1066632 0 tcp_log 416 0 2547 14764 0 254 1059552 0 FFS inode 1168 571 31 1875 0 8 703136 0 sctp_chunk 152 1697 2645 74016 0 254 659984 0 256 Bucket 2048 279 27 5288 0 8 626688 0 ertt_txseginfo 40 0 15049 437666 0 254 601960 0 sctp_ep 1176 171 339 1853 0 254 599760 0 malloc-4096 4096 123 19 3024 0 2 581632 0 lkpicurr 176 2 3080 2 0 62 542432 0 pbuf 2624 0 198 0 0 2 519552 0 socket 960 125 383 4279 0 254 487680 0 VM OBJECT 264 1422 378 35646 0 30 475200 0 THREAD 1824 218 22 2119 0 8 437760 0 malloc-65536 65536 4 2 102 0 1 393216 0 malloc-2048 2048 178 14 2137 0 8 393216 0 tcp_bbr_map 128 0 3038 47817 0 126 388864 0 sctp_raddr 736 87 430 616 0 254 380512 0 malloc-256 256 348 897 7339 0 62 318720 0 MAP ENTRY 96 2289 987 105534 0 126 314496 0 VNODE 448 610 83 1917 0 30 310464 0 malloc-8192 8192 29 5 76 0 1 278528 0 malloc-64 64 3844 251 3845 0 254 262080 0 malloc-16 16 14579 171 14652 0 254 236000 0 malloc-32768 32768 0 7 1596 0 1 229376 0 DEVCTL 1024 23 197 157 0 0 225280 0 FPU_save_area 832 220 41 4426 0 16 217152 0 PROC 1376 104 39 2521 0 8 196768 0 mbuf_packet 256 139 623 18102 0 254 195072 0 malloc-128 128 1307 212 28817 0 126 194432 0 UMA Zones 768 243 1 243 0 16 187392 0 malloc-32 32 5341 329 5351 0 254 181440 0 malloc-2048 2048 10 78 1042 0 8 180224 0 FFS2 dinode 256 571 119 1875 0 62 176640 0 lkpimm 56 1 3095 1 0 254 173376 0 malloc-128 128 1080 191 3799 0 126 162688 0 filedesc0 1072 104 36 2573 0 8 150080 0 malloc-1024 1024 124 20 319 0 16 147456 0 S VFS Cache 104 1009 395 2464 0 126 146016 0 malloc-256 256 517 53 7846 0 62 145920 0 malloc-65536 65536 2 0 2 0 1 131072 0 malloc-65536 65536 0 2 135 0 1 131072 0 malloc-65536 65536 2 0 2 0 1 131072 0 unpcb 256 1 509 1376 0 254 130560 0 128 Bucket 1024 85 30 597 0 16 117760 0 ksiginfo 112 128 916 3151 0 126 116928 0 malloc-384 384 299 1 1501 0 30 115200 0 malloc-384 384 259 41 2939 0 30 115200 0 malloc-128 128 751 148 9428 0 126 115072 0 malloc-1024 1024 95 17 614 0 16 114688 0 malloc-256 256 327 108 2741 0 62 111360 0 malloc-32768 32768 3 0 3 0 1 98304 0 malloc-32768 32768 2 1 122 0 1 98304 0 g_bio 408 0 240 28851 0 30 97920 0 tcp_inpcb 1312 35 37 730 0 8 94464 0 UMA Kegs 384 230 3 230 0 30 89472 0 64 Bucket 512 106 62 3412 0 30 86016 0 malloc-64 64 960 363 4428 0 254 84672 0 malloc-16384 16384 5 0 5 0 1 81920 0 malloc-8192 8192 10 0 12 0 1 81920 0 VMSPACE 616 79 53 2498 0 16 81312 0 sctp_readq 152 0 520 82 0 254 79040 0 malloc-8192 8192 7 2 108 0 1 73728 0 malloc-64 64 728 343 33072 0 254 68544 0 malloc-65536 65536 0 1 8 0 1 65536 0 malloc-32768 32768 2 0 2 0 1 65536 0 malloc-4096 4096 12 4 219 0 2 65536 0 malloc-1024 1024 20 44 2020 0 16 65536 0 32 Bucket 256 95 160 3872 0 62 65280 0 sctp_stream_msg_out 112 47 493 175 0 254 60480 0 malloc-4096 4096 2 12 14 0 2 57344 0 Files 80 252 398 15837 0 126 52000 0 ttyoutq 256 8 187 4808 0 62 49920 0 malloc-256 256 3 192 1254 0 62 49920 0 malloc-256 256 23 172 6670 0 62 49920 0 tcp_rack_pcb 1024 2 46 190 0 16 49152 0 DIRHASH 1024 35 13 35 0 16 49152 0 NAMEI 1024 0 48 24537 0 16 49152 0 malloc-16384 16384 1 2 161 0 1 49152 0 malloc-4096 4096 4 8 273 0 2 49152 0 malloc-2048 2048 11 13 96 0 8 49152 0 malloc-384 384 95 25 102 0 30 46080 0 PWD 40 73 1038 1316 0 254 44440 0 syncache 168 0 264 4 0 254 44352 0 malloc-4096 4096 6 4 10 0 2 40960 0 pcpu-8 8 4766 354 5213 0 254 40960 0 pipe 728 17 38 439 0 16 40040 0 udp_inpcb 424 4 86 200 0 30 38160 0 da_ccb 544 0 70 7327 0 16 38080 0 hostcache 64 3 564 3 0 254 36288 0 malloc-64 64 165 402 20842 0 254 36288 0 malloc-64 64 32 535 54 0 254 36288 0 malloc-64 64 134 433 2705 0 254 36288 0 16 Bucket 144 76 176 585 0 62 36288 0 tcp_rack_map 128 5 274 1233 0 126 35712 0 malloc-128 128 26 253 428 0 126 35712 0 malloc-128 128 8 271 11 0 126 35712 0 malloc-128 128 7 272 10 0 126 35712 0 routing nhops 256 27 108 34 0 62 34560 0 malloc-256 256 56 79 549 0 62 34560 0 malloc-256 256 2 133 5 0 62 34560 0 TURNSTILE 136 241 11 241 0 62 34272 0 SLEEPQUEUE 88 241 143 241 0 126 33792 0 malloc-32768 32768 1 0 1 0 1 32768 0 malloc-8192 8192 2 2 60 0 1 32768 0 malloc-2048 2048 2 14 14 0 8 32768 0 malloc-2048 2048 2 14 6 0 8 32768 0 malloc-2048 2048 2 14 2 0 8 32768 0 malloc-2048 2048 10 6 10 0 8 32768 0 malloc-1024 1024 5 27 61 0 16 32768 0 malloc-1024 1024 16 16 26 0 16 32768 0 malloc-1024 1024 4 28 4 0 16 32768 0 malloc-512 512 1 63 147 0 30 32768 0 malloc-512 512 4 60 39 0 30 32768 0 malloc-512 512 0 64 28 0 30 32768 0 malloc-512 512 3 61 58 0 30 32768 0 malloc-512 512 8 56 208 0 30 32768 0 malloc-512 512 2 62 49 0 30 32768 0 malloc-512 512 0 64 41 0 30 32768 0 pcpu-64 64 487 25 487 0 254 32768 0 KNOTE 160 30 170 17382 0 62 32000 0 ttyinq 160 15 185 9015 0 62 32000 0 8 Bucket 80 64 336 968 0 126 32000 0 PGRP 120 8 256 166 0 126 31680 0 clpbuf 2624 0 12 64 0 4 31488 0 tcp_bbr_pcb 832 0 36 357 0 16 29952 0 sctp_laddr 48 66 522 589 0 254 28224 0 malloc-32 32 310 572 1519 0 254 28224 0 malloc-32 32 76 806 19180 0 254 28224 0 malloc-32 32 257 625 3854 0 254 28224 0 4 Bucket 48 7 581 12 0 254 28224 0 2 Bucket 32 70 812 888 0 254 28224 0 udplite_inpcb 424 0 63 8 0 30 26712 0 cpuset 200 7 121 119 0 62 25600 0 ripcb 392 0 63 14 0 30 24696 0 malloc-8192 8192 3 0 3 0 1 24576 0 rl_entry 40 76 530 76 0 254 24240 0 rtentry 168 30 114 34 0 62 24192 0 itimer 352 0 66 28 0 30 23232 0 domainset 40 0 567 84 0 254 22680 0 Mountpoints 2816 2 6 2 0 4 22528 0 udp_inpcb ports 32 2 628 37 0 254 20160 0 tcp_inpcb ports 32 5 625 468 0 254 20160 0 ertt 72 35 245 730 0 126 20160 0 malloc-64 64 2 313 2 0 254 20160 0 malloc-64 64 1 314 1 0 254 20160 0 malloc-32 32 136 494 1426 0 254 20160 0 malloc-32 32 294 336 5640 0 254 20160 0 malloc-32 32 5 625 5 0 254 20160 0 malloc-128 128 2 153 2 0 126 19840 0 cryptop 280 0 70 10 0 30 19600 0 L VFS Cache 320 0 60 4 0 30 19200 0 malloc-256 256 11 64 11 0 62 19200 0 AIOLIO 272 0 70 4 0 30 19040 0 epoch_record pcpu 256 4 60 4 0 62 16384 0 malloc-16384 16384 1 0 1 0 1 16384 0 malloc-512 512 1 31 1 0 30 16384 0 SMR CPU 32 7 504 7 0 254 16352 0 vtnet_tx_hdr 24 0 668 183036 0 254 16032 0 malloc-16 16 527 473 5027 0 254 16000 0 AIO 208 0 76 4 0 62 15808 0 kenv 258 17 43 1076 0 30 15480 0 mqnode 416 3 33 3 0 30 14976 0 vmem 1856 1 7 1 0 8 14848 0 SMR SHARED 24 7 504 7 0 254 12264 0 udplite_inpcb ports 32 0 378 8 0 254 12096 0 malloc-32 32 2 376 5 0 254 12096 0 malloc-16 16 103 647 3702 0 254 12000 0 malloc-16 16 10 740 11 0 254 12000 0 malloc-16 16 187 563 1237 0 254 12000 0 malloc-16 16 62 688 29251 0 254 12000 0 malloc-16 16 21 729 21 0 254 12000 0 malloc-16 16 271 479 2674 0 254 12000 0 malloc-384 384 11 19 11 0 30 11520 0 malloc-384 384 1 29 1 0 30 11520 0 malloc-384 384 13 17 13 0 30 11520 0 malloc-8192 8192 1 0 1 0 1 8192 0 pcpu-16 16 8 504 8 0 254 8192 0 UMA Slabs 1 176 9 13 9 0 62 3872 0 KMAP ENTRY 96 12 27 14 0 0 3744 0 md3 2 1 255 1 0 254 512 0 md2 2 1 255 1 0 254 512 0 md1 2 1 255 1 0 254 512 0 md0 2 1 255 1 0 254 512 0 FFS1 dinode 128 0 0 0 0 126 0 0 ada_ccb 272 0 0 0 0 30 0 0 swblk 136 0 0 0 0 62 0 0 swpctrie 144 0 0 0 0 62 0 0 cdg_qdiffsample 16 0 0 0 0 254 0 0 pf state scrubs 40 0 0 0 0 254 0 0 pf frag entries 40 0 0 0 0 254 0 0 pf frags 248 0 0 0 0 62 0 0 pf table entries 160 0 0 0 0 254 0 0 pf table entry counters 64 0 0 0 0 254 0 0 pf source nodes 152 0 0 0 0 254 0 0 pf state keys 88 0 0 0 0 126 0 0 pf states 360 0 0 0 0 254 0 0 pf tags 104 0 0 0 0 126 0 0 pf mtags 184 0 0 0 0 62 0 0 tfo_ccache_entries 80 0 0 0 0 126 0 0 tfo 4 0 0 0 0 254 0 0 sackhole 32 0 0 0 0 254 0 0 ipq 56 0 0 0 0 254 0 0 sctp_asconf_ack 48 0 0 0 0 254 0 0 sctp_asconf 40 0 0 0 0 254 0 0 tcp_log_id_node 120 0 0 0 0 126 0 0 tcp_log_id_bucket 176 0 0 0 0 62 0 0 tcpreass 48 0 0 0 0 254 0 0 ripcb ports 32 0 0 0 0 254 0 0 IPsec SA lft_c 16 0 0 0 0 254 0 0 AIOCB 552 0 0 0 0 16 0 0 mqnotifier 216 0 0 0 0 62 0 0 mvdata 64 0 0 0 0 254 0 0 mqueue 248 0 0 0 0 62 0 0 TMPFS node 232 0 0 0 0 62 0 0 NCLNODE 608 0 0 0 0 16 0 0 LTS VFS Cache 360 0 0 0 0 30 0 0 STS VFS Cache 144 0 0 0 0 62 0 0 linux_dma_object 32 0 0 0 0 254 0 0 linux_dma_pctrie 144 0 0 0 0 62 0 0 IOMMU_MAP_ENTRY 104 0 0 0 0 126 0 0 mbuf_jumbo_16k 16384 0 0 0 0 254 0 0 mbuf_jumbo_9k 9216 0 0 0 0 254 0 0 audit_record 1280 0 0 0 0 8 0 0 MAC labels 40 0 0 0 0 254 0 0 vnpbuf 2624 0 0 0 0 16 0 0 nfspbuf 2624 0 0 0 0 4 0 0 swwbuf 2624 0 0 0 0 2 0 0 swrbuf 2624 0 0 0 0 4 0 0 umtx_shm 88 0 0 0 0 126 0 0 umtx pi 96 0 0 0 0 126 0 0 rangeset pctrie nodes 144 0 0 0 0 62 0 0 malloc-65536 65536 0 0 0 0 1 0 0 malloc-65536 65536 0 0 0 0 1 0 0 malloc-65536 65536 0 0 0 0 1 0 0 malloc-32768 32768 0 0 0 0 1 0 0 malloc-32768 32768 0 0 0 0 1 0 0 malloc-32768 32768 0 0 0 0 1 0 0 malloc-16384 16384 0 0 0 0 1 0 0 malloc-16384 16384 0 0 0 0 1 0 0 malloc-16384 16384 0 0 0 0 1 0 0 malloc-16384 16384 0 0 0 0 1 0 0 malloc-8192 8192 0 0 0 0 1 0 0 malloc-8192 8192 0 0 0 0 1 0 0 malloc-4096 4096 0 0 0 0 2 0 0 malloc-4096 4096 0 0 0 0 2 0 0 malloc-2048 2048 0 0 0 0 8 0 0 malloc-1024 1024 0 0 0 0 16 0 0 malloc-1024 1024 0 0 0 0 16 0 0 malloc-384 384 0 0 0 0 30 0 0 pcpu-32 32 0 0 0 0 254 0 0 pcpu-4 4 0 0 0 0 254 0 0 fakepg 104 0 0 0 0 126 0 0 UMA Hash 256 0 0 0 0 62 0 0