panic: lock (sleep mutex) sctp-inp not locked @ /syzkaller/managers/main/kernel/sys/netinet/sctp_usrreq.c:958 cpuid = 0 time = 1705469386 KDB: stack backtrace: db_trace_self_wrapper() at db_trace_self_wrapper+0xc6/frame 0xfffffe00740615b0 kdb_backtrace() at kdb_backtrace+0xd0/frame 0xfffffe0074061710 vpanic() at vpanic+0x271/frame 0xfffffe00740618b0 panic() at panic+0xb5/frame 0xfffffe0074061970 witness_unlock() at witness_unlock+0x4fd/frame 0xfffffe00740619f0 __mtx_unlock_flags() at __mtx_unlock_flags+0xec/frame 0xfffffe0074061ab0 sctp_shutdown() at sctp_shutdown+0xf35/frame 0xfffffe0074061c10 soshutdown() at soshutdown+0xb3/frame 0xfffffe0074061c50 kern_shutdown() at kern_shutdown+0x154/frame 0xfffffe0074061d10 amd64_syscall() at amd64_syscall+0x473/frame 0xfffffe0074061f30 fast_syscall_common() at fast_syscall_common+0xf8/frame 0xfffffe0074061f30 --- syscall (198, FreeBSD ELF64, __syscall), rip = 0x2aeb2a, rsp = 0x826bc3f08, rbp = 0x826bc3f80 --- KDB: enter: panic [ thread pid 2203 tid 102182 ] Stopped at kdb_enter+0x6e: movq $0,0x2194267(%rip) db> db> set $lines = 0 db> set $maxwidth = 0 db> show registers cs 0x20 ds 0x3b es 0x3b fs 0x13 gs 0x1b ss 0x28 rax 0x12 rcx 0xfffffe006d200000 rdx 0x3ffff rbx 0xffffffff826f86c0 .str.28 rsp 0xfffffe00740616f0 rbp 0xfffffe0074061710 rsi 0x40001 rdi 0xffffffff815de326 printf+0x156 r8 0 r9 0xffffffff r10 0 r11 0 r12 0 r13 0xfffffe0073e2b000 r14 0xffffffff826f86c0 .str.28 r15 0 rip 0xffffffff815cadae kdb_enter+0x6e rflags 0x46 kdb_enter+0x6e: movq $0,0x2194267(%rip) db> show proc Process 2203 (syz-executor.1) at 0xfffffe006cd60ae0: state: NORMAL uid: 0 gids: 0, 0, 5 parent: pid 771 at 0xfffffe006d166000 ABI: FreeBSD ELF64 flag: 0x10000080 flag2: 0 arguments: /root/syz-executor.1 exec reaper: 0xfffffe00541d1040 reapsubtree: 1 sigparent: 20 vmspace: 0xfffffe006cb044d0 (map 0xfffffe006cb044d0) (map.pmap 0xfffffe006cb04590) (pmap 0xfffffe006cb04600) threads: 2 100150 RunQ syz-executor.1 102182 Run CPU 0 syz-executor.1 db> ps pid ppid pgrp uid state wmesg wchan cmd 2207 770 770 0 R syz-executor.0 2203 771 771 0 R (threaded) syz-executor.1 100150 RunQ syz-executor.1 102182 Run CPU 0 syz-executor.1 2200 802 802 0 S (threaded) syz-executor.3 101408 S nanslp 0xffffffff83740400 syz-executor.3 102174 S connec 0xfffffe00589a80da syz-executor.3 102180 S uwait 0xfffffe006cd30400 syz-executor.3 2017 2009 2017 0 Ss select 0xfffffe00571b6340 dhclient 2013 1 2013 0 Ss select 0xfffffe0057a937c0 dhclient 2009 2001 424 65 S select 0xfffffe0057a939c0 dhclient 2001 424 424 0 S wait 0xfffffe006cd61b00 sh 1991 1 1991 65 Ss select 0xfffffe00571d7dc0 dhclient 1916 1 802 0 S uwait 0xfffffe0071eb1380 syz-executor.3 1913 0 0 0 DL mdwait 0xfffffe0074e0c000 [md3] 1908 0 0 0 DL mdwait 0xfffffe0057bc2000 [md2] 1903 0 0 0 DL mdwait 0xfffffe006d1fa000 [md1] 1896 0 0 0 DL mdwait 0xfffffe006d1f9000 [md0] 1025 1 1025 0 Ss select 0xfffffe00571b1d40 dhclient 1021 1 1021 0 Ss select 0xfffffe0057a93a40 dhclient 802 768 802 0 Ss nanslp 0xffffffff83740401 syz-executor.3 792 768 792 0 Rs CPU 1 syz-executor.2 771 768 771 0 Ss nanslp 0xffffffff83740400 syz-executor.1 770 768 770 0 Rs syz-executor.0 768 766 766 0 S (threaded) syz-fuzzer 100091 S wait 0xfffffe0057b9c5a0 syz-fuzzer 100119 S uwait 0xfffffe0057a77300 syz-fuzzer 100120 S uwait 0xfffffe0057a77400 syz-fuzzer 100121 S uwait 0xfffffe0057a77500 syz-fuzzer 100122 S wait 0xfffffe0057b9c5a0 syz-fuzzer 100123 S uwait 0xfffffe0057a77700 syz-fuzzer 100124 S wait 0xfffffe0057b9c5a0 syz-fuzzer 100125 S uwait 0xfffffe0057a76880 syz-fuzzer 100127 S wait 0xfffffe0057b9c5a0 syz-fuzzer 100131 S uwait 0xfffffe0057a76d80 syz-fuzzer 100132 S kqread 0xfffffe0058ab9300 syz-fuzzer 100142 S uwait 0xfffffe0071eb1880 syz-fuzzer 100166 S uwait 0xfffffe005750d780 syz-fuzzer 766 764 766 0 Ss pause 0xfffffe006cd62670 csh 764 682 764 0 Ss select 0xfffffe00571b1dc0 sshd 748 1 748 0 Ss+ ttyin 0xfffffe005766fcb0 getty 747 1 747 0 Ss+ ttyin 0xfffffe00587670b0 getty 746 1 746 0 Ss+ ttyin 0xfffffe00587674b0 getty 745 1 745 0 Ss+ ttyin 0xfffffe00587678b0 getty 744 1 744 0 Ss+ ttyin 0xfffffe0058767cb0 getty 743 1 743 0 Ss+ ttyin 0xfffffe00587680b0 getty 742 1 742 0 Ss+ ttyin 0xfffffe00587684b0 getty 741 1 741 0 Ss+ ttyin 0xfffffe00587688b0 getty 740 1 740 0 Ss+ ttyin 0xfffffe0058768cb0 getty 686 1 686 0 Ss nanslp 0xffffffff83740400 cron 682 1 682 0 Ss select 0xfffffe00571b62c0 sshd 495 1 495 0 Ss select 0xfffffe0057a93b40 syslogd 424 1 424 0 Ss wait 0xfffffe0058ada560 devd 423 1 423 65 Ss select 0xfffffe00571b60c0 dhclient 338 1 338 0 Ss select 0xfffffe00571b6040 dhclient 335 1 335 0 Ss select 0xfffffe00571b61c0 dhclient 17 0 0 0 DL syncer 0xffffffff8385db20 [syncer] 16 0 0 0 DL vlruwt 0xfffffe0057182060 [vnlru] 15 0 0 0 DL (threaded) [bufdaemon] 100079 D psleep 0xffffffff8385c100 [bufdaemon] 100082 D - 0xffffffff82c0a140 [bufspacedaemon-0] 100095 D sdflush 0xfffffe0058b204e8 [/ worker] 9 0 0 0 DL psleep 0xffffffff838a55c0 [vmdaemon] 8 0 0 0 DL (threaded) [pagedaemon] 100077 D psleep 0xffffffff8388d438 [dom0] 100080 D launds 0xffffffff8388d444 [laundry: dom0] 100081 D umarcl 0xffffffff81d61000 [uma] 7 0 0 0 DL - 0xffffffff834bdc10 [rand_harvestq] 6 0 0 0 DL pftm 0xffffffff84050ea0 [pf purge] 5 0 0 0 DL waiting 0xffffffff844385a0 [sctp_iterator] 4 0 0 0 DL (threaded) [cam] 100044 D - 0xffffffff83488340 [doneq0] 100045 D - 0xffffffff834882c0 [async] 100076 D - 0xffffffff83488140 [scanner] 3 0 0 0 DL (threaded) [crypto] 100041 D crypto_ 0xffffffff83888c60 [crypto] 100042 D crypto_ 0xfffffe00542d5730 [crypto returns 0] 100043 D crypto_ 0xfffffe00542d5780 [crypto returns 1] 14 0 0 0 DL seqstat 0xfffffe000796e888 [sequencer 00] 13 0 0 0 DL (threaded) [geom] 100035 D - 0xffffffff836e8d40 [g_event] 100036 D - 0xffffffff836e8d60 [g_up] 100037 D - 0xffffffff836e8d80 [g_down] 2 0 0 0 WL (threaded) [clock] 100029 I [clock (0)] 100030 I [clock (1)] 12 0 0 0 WL (threaded) [intr] 100012 I [swi6: task queue] 100013 I [swi6: Giant taskq] 100015 I [swi5: fast taskq] 100031 I [swi1: netisr 0] 100032 I [swi1: hpts] 100033 I [swi1: hpts] 100046 I [irq24: virtio_pci0] 100047 I [irq25: virtio_pci0] 100048 I [irq26: virtio_pci0] 100049 I [irq27: virtio_pci0] 100050 I [irq28: virtio_pci1] 100051 I [irq29: virtio_pci1] 100052 I [irq30: virtio_pci1] 100053 I [irq31: virtio_pci1] 100054 I [irq32: virtio_pci1] 100059 I [irq33: virtio_pci2] 100060 I [irq34: virtio_pci2] 100061 I [irq35: virtio_pci2] 100063 I [irq1: atkbd0] 100064 I [irq12: psm0] 100065 I [swi0: uart uart++] 100069 I [swi1: pf send] 11 0 0 0 RL (threaded) [idle] 100003 CanRun [idle: cpu0] 100004 CanRun [idle: cpu1] 1 0 1 0 SLs wait 0xfffffe00541d1040 [init] 10 0 0 0 DL audit_w 0xffffffff838896c0 [audit] 0 0 0 0 DLs (threaded) [kernel] 100000 D swapin 0xffffffff836e9720 [swapper] 100005 D - 0xfffffe0054282100 [softirq_0] 100006 D - 0xfffffe0054282000 [softirq_1] 100007 D - 0xfffffe0054281e00 [if_io_tqg_0] 100008 D - 0xfffffe0054281d00 [if_io_tqg_1] 100009 D - 0xfffffe0054281c00 [if_config_tqg_0] 100010 D - 0xfffffe0007977900 [pci_hp taskq] 100011 D - 0xfffffe0007977800 [kqueue_ctx taskq] 100014 D - 0xfffffe0007977500 [thread taskq] 100016 D - 0xfffffe0007977300 [aiod_kick taskq] 100017 D - 0xfffffe0007977200 [deferred_unmount ta] 100018 D - 0xfffffe0007977100 [inm_free taskq] 100019 D - 0xfffffe0007977000 [in6m_free taskq] 100020 D - 0xfffffe0007976e00 [linuxkpi_irq_wq] 100021 D - 0xfffffe0007976d00 [linuxkpi_short_wq_0] 100022 D - 0xfffffe0007976d00 [linuxkpi_short_wq_1] 100023 D - 0xfffffe0007976d00 [linuxkpi_short_wq_2] 100024 D - 0xfffffe0007976d00 [linuxkpi_short_wq_3] 100025 D - 0xfffffe0007976c00 [linuxkpi_long_wq_0] 100026 D - 0xfffffe0007976c00 [linuxkpi_long_wq_1] 100027 D - 0xfffffe0007976c00 [linuxkpi_long_wq_2] 100028 D - 0xfffffe0007976c00 [linuxkpi_long_wq_3] 100034 D - 0xfffffe0007976a00 [firmware taskq] 100039 D - 0xfffffe0007976700 [crypto_0] 100040 D - 0xfffffe0007976700 [crypto_1] 100055 D - 0xfffffe00571c4b00 [vtnet0 rxq 0] 100056 D - 0xfffffe00571c4a00 [vtnet0 txq 0] 100057 D - 0xfffffe00571c4900 [vtnet0 rxq 1] 100058 D - 0xfffffe00571c4800 [vtnet0 txq 1] 100062 D vtbslp 0xfffffe00571b6580 [virtio_balloon] 100066 D - 0xffffffff826fd920 [deadlkres] 100070 D - 0xfffffe0057bbd100 [acpi_task_0] 100071 D - 0xfffffe0057bbd100 [acpi_task_1] 100072 D - 0xfffffe0057bbd100 [acpi_task_2] 100074 D - 0xfffffe0007977c00 [mca taskq] 100075 D - 0xfffffe0007976500 [CAM taskq] db> show all locks Process 792 (syz-executor.2) thread 0xfffffe006d197740 (100130) exclusive sleep mutex pmap (pmap) r = 0 (0xfffffe0074816398) locked @ /syzkaller/managers/main/kernel/sys/amd64/amd64/pmap.c:8088 exclusive sleep mutex pmap (pmap) r = 0 (0xfffffe0058907d38) locked @ /syzkaller/managers/main/kernel/sys/amd64/amd64/pmap.c:8087 exclusive sx vm map (user) (vm map (user)) r = 0 (0xfffffe00748162c8) locked @ /syzkaller/managers/main/kernel/sys/vm/vm_map.c:4343 exclusive sx vm map (user) (vm map (user)) r = 0 (0xfffffe0058907c68) locked @ /syzkaller/managers/main/kernel/sys/vm/vm_map.c:4339 shared sx killpg racer (killpg racer) r = 0 (0xfffffe00588dc580) locked @ /syzkaller/managers/main/kernel/sys/kern/kern_fork.c:975 db> show malloc Type InUse MemUse Requests pf_hash 5 11524K 5 tcp_hpts 7 4801K 7 devbuf 4192 4324K 4224 sysctloid 34817 2052K 34888 vtbuf 24 1968K 46 kobj 326 1304K 524 newblk 33 1032K 6860 vfscache 3 1025K 3 pcb 157 817K 4564 inodedep 232 599K 1572 ufs_quota 1 512K 1 vfs_hash 1 512K 1 callout 2 512K 2 intr 4 472K 4 vmem 3 266K 6 subproc 134 254K 2293 vnet_data 2 224K 2 acpitask 1 224K 1 acpica 1674 184K 60830 tidhash 3 141K 3 filedesc 18 137K 2607 pagedep 11 131K 1330 linker 352 130K 386 tfo_ccache 1 128K 1 IP reass 1 128K 1 DEVFS1 113 113K 130 sem 4 106K 4 gtaskqueue 18 98K 18 bus 985 81K 5155 mtx_pool 2 72K 2 syncache 1 68K 1 NFSD srvcache 3 68K 3 sctp_stro 66 66K 983 module 513 65K 513 ddb_capture 1 64K 1 dirrem 220 55K 1506 BPF 30 53K 103 sctp_atcl 132 50K 3407 umtx 368 46K 368 kdtrace 213 43K 4392 temp 35 37K 2158 DEVFS3 132 33K 142 hostcache 1 32K 1 shm 1 32K 11 msg 4 30K 4 kbdmux 6 28K 6 freefile 219 28K 1450 LRO 20 21K 20 DEVFS_RULE 56 20K 56 ifaddr 68 20K 70 ufs_mount 4 17K 5 proc 3 17K 3 sctp_timw 65 17K 65 tty 16 16K 16 md_disk 4 16K 4 routetbl 130 16K 410 ithread 97 16K 97 GEOM 89 15K 665 lltable 46 15K 58 bus-sc 34 15K 1687 eventhandler 161 14K 161 KTRACE 101 13K 101 ifnet 7 13K 7 ether_multi 152 13K 217 kenv 95 12K 95 rman 86 11K 431 CAM queue 5 11K 1528 in6_multi 65 9K 65 sctp_atky 198 9K 4614 bmsafemap 2 9K 1573 rpc 4 9K 4 devstat 4 9K 4 UART 12 9K 12 ksem 1 8K 37 shmfd 1 8K 5 pfs_vncache 1 8K 1 audit_evclass 238 8K 300 taskqueue 63 7K 63 sglist 6 7K 6 cred 24 6K 252 CAM DEV 3 6K 510 kqueue 58 6K 2218 plimit 22 6K 364 pfs_nodes 20 5K 20 hhook 15 5K 17 ufs_dirhash 24 5K 24 pf_ifnet 11 5K 23 UMA 267 5K 267 vt 11 5K 11 memdesc 1 4K 1 MCA 32 4K 32 DEVFSP 64 4K 245 evdev 4 4K 4 session 28 4K 41 pwddesc 56 4K 2221 acpisem 28 4K 28 kcovinfo 52 4K 52 proc-args 82 4K 3388 terminal 11 3K 11 lockf 23 3K 36 selfd 38 3K 36375 clone 9 3K 9 uidinfo 3 3K 7 sctp_athm 132 3K 3590 sctp_map 132 3K 2114 local_apic 1 2K 1 io_apic 1 2K 1 ipsec-saq 2 2K 2 ip6ndp 12 2K 13 sctp_ifa 13 2K 14 select 13 2K 43 Unitno 27 2K 49 tun 7 2K 7 CAM XPT 22 2K 543 msi 12 2K 12 in_multi 6 2K 19 toponodes 6 2K 6 freework 6 2K 3391 freeblks 5 2K 1558 ipsecpolicy 2 2K 2 CC Mem 5 2K 658 acpidev 20 2K 20 softdep 1 1K 1 mkdir 8 1K 2632 sahead 1 1K 1 secasvar 1 1K 1 nhops 6 1K 8 vnodemarker 2 1K 18 NFSD session 1 1K 1 CAM periph 4 1K 271 sctp_ifn 6 1K 14 ipsec 3 1K 3 mld 6 1K 6 igmp 6 1K 6 pfil 6 1K 6 isadev 6 1K 6 mount 16 1K 89 pci_link 10 1K 10 newdirblk 5 1K 1316 diradd 5 1K 1541 crypto 4 1K 46 encap_export_host 12 1K 12 cdev 2 1K 2 lkpikmalloc 8 1K 9 osd 10 1K 671 inpcbpolicy 14 1K 913 netlink 2 1K 40 chacha20random 1 1K 1 biobuf 1 1K 1 procdesc 3 1K 14 indirdep 1 1K 2717 vnodes 1 1K 1 NFSD lckfile 1 1K 1 NFSD V4client 1 1K 1 DEVFS 9 1K 10 CAM SIM 2 1K 2 feeder 7 1K 7 tcpfunc 3 1K 3 loginclass 3 1K 6 prison 6 1K 6 cryptodev 2 1K 181 nexusdev 8 1K 8 apmdev 1 1K 1 atkbddev 2 1K 2 soname 5 1K 6036 pmchooks 1 1K 1 CAM path 4 1K 1034 CAM dev queue 2 1K 2 CAM I/O Scheduler 1 1K 1 sctp_vrf 1 1K 1 vnet 1 1K 1 pmc 1 1K 1 filedesc_to_leader 1 1K 2 entropy 2 1K 40 acpiintr 1 1K 1 filecaps 3 1K 97 cpus 2 1K 2 vnet_data_free 1 1K 1 Per-cpu 1 1K 1 p1003.1b 1 1K 1 mqdata 0 0K 0 tcp_do 0 0K 0 tcp_fsb 0 0K 244 sctp_mcore 0 0K 0 sctp_socko 0 0K 1311 sctp_iter 0 0K 12 sctp_mvrf 0 0K 0 sctp_cpal 0 0K 0 sctp_cmsg 0 0K 0 sctp_stre 0 0K 0 sctp_athi 0 0K 0 sctp_a_it 0 0K 12 sctp_aadr 0 0K 1 sctp_stri 0 0K 352 filemon 0 0K 50 ipcomp 0 0K 0 esp 0 0K 0 ah 0 0K 0 pf_table 0 0K 0 pf_rule 0 0K 4 pf_altq 0 0K 0 pf_osfp 0 0K 0 pf_krule_item 0 0K 0 pf_temp 0 0K 0 madt_table 0 0K 2 smartpqi 0 0K 0 ixl 0 0K 0 ice-resmgr 0 0K 0 ice-osdep 0 0K 0 ice 0 0K 0 iavf 0 0K 0 axgbe 0 0K 0 fpukern_ctx 0 0K 0 xen_intr 0 0K 0 xen_hvm 0 0K 0 legacydrv 0 0K 0 bounce 0 0K 0 busdma 0 0K 0 qpidrv 0 0K 0 dmar_idpgtbl 0 0K 0 dmar_dom 0 0K 0 dmar_ctx 0 0K 0 isci 0 0K 0 iommu_dmamap 0 0K 0 hyperv_socket 0 0K 0 bxe_ilt 0 0K 0 aesni_data 0 0K 0 xenbus 0 0K 0 vm_fictitious 0 0K 0 UMAHash 0 0K 0 vm_pgdata 0 0K 0 jblocks 0 0K 0 savedino 0 0K 1658 sentinel 0 0K 0 jfsync 0 0K 0 jtrunc 0 0K 0 sbdep 0 0K 7 jsegdep 0 0K 0 jseg 0 0K 0 jfreefrag 0 0K 0 jfreeblk 0 0K 0 jnewblk 0 0K 0 jmvref 0 0K 0 jremref 0 0K 0 jaddref 0 0K 0 freedep 0 0K 0 freefrag 0 0K 16 allocindir 0 0K 0 allocdirect 0 0K 0 ufs_trim 0 0K 0 mactemp 0 0K 0 audit_trigger 0 0K 0 audit_pipe_presel 0 0K 0 audit_pipeent 0 0K 0 audit_pipe 0 0K 0 audit_evname 0 0K 0 audit_bsm 0 0K 0 audit_gidset 0 0K 0 audit_text 0 0K 0 audit_path 0 0K 0 audit_data 0 0K 0 audit_cred 0 0K 0 ktls_ocf 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5E_TLS_RX 0 0K 0 MLX5EEPROM 0 0K 0 MLX5E_TLS 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EN 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5DUMP 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 simple_attr 0 0K 0 seq_file 0 0K 0 lkpiskb 0 0K 0 radix 0 0K 0 idr 0 0K 0 lkpindev 0 0K 0 lkpimhi 0 0K 0 lkpifw 0 0K 0 lkpi80211 0 0K 0 NLM 0 0K 0 ipsec-spdcache 0 0K 0 ipsec-reg 0 0K 0 ipsec-misc 0 0K 0 ipsecrequest 0 0K 0 ip6opt 0 0K 6 ip6_msource 0 0K 0 ip6_moptions 0 0K 0 in6_mfilter 0 0K 0 frag6 0 0K 0 tcplog 0 0K 0 tcp_hwpace 0 0K 0 ip_msource 0 0K 60 ip_moptions 0 0K 32 in_mfilter 0 0K 96 ipid 0 0K 0 80211scan 0 0K 0 80211ratectl 0 0K 0 80211power 0 0K 0 80211nodeie 0 0K 0 80211node 0 0K 0 80211mesh_gt 0 0K 0 80211mesh_rt 0 0K 0 80211perr 0 0K 0 80211prep 0 0K 0 80211preq 0 0K 0 80211dfs 0 0K 0 80211crypto 0 0K 0 80211vap 0 0K 0 iflib 0 0K 0 vlan 0 0K 0 gif 0 0K 0 ifdescr 0 0K 0 zlib 0 0K 0 fadvise 0 0K 0 VN POLL 0 0K 0 statfs 0 0K 197 namei_tracker 0 0K 0 export_host 0 0K 0 cl_savebuf 0 0K 5 aio 0 0K 0 lio 0 0K 0 acl 0 0K 0 mbuf_tag 0 0K 0 ktls 0 0K 0 accf 0 0K 0 pts 0 0K 0 timerfd 0 0K 0 iov 0 0K 16095 ioctlops 0 0K 157 eventfd 0 0K 0 Witness 0 0K 0 stack 0 0K 0 sbuf 0 0K 336 firmware 0 0K 0 compressor 0 0K 0 SWAP 0 0K 0 sysctltmp 0 0K 700 sysctl 0 0K 3 ekcd 0 0K 0 dumper 0 0K 0 sendfile 0 0K 0 rctl 0 0K 0 cache 0 0K 0 prison_racct 0 0K 0 Fail Points 0 0K 0 sigio 0 0K 1 pwd 0 0K 0 tty console 0 0K 0 boottrace 0 0K 0 isofs_node 0 0K 0 isofs_mount 0 0K 0 tr_raid5_data 0 0K 0 tr_raid1e_data 0 0K 0 tr_raid1_data 0 0K 0 tr_raid0_data 0 0K 0 tr_concat_data 0 0K 0 md_sii_data 0 0K 0 md_promise_data 0 0K 0 md_nvidia_data 0 0K 0 md_jmicron_data 0 0K 0 md_intel_data 0 0K 0 md_ddf_data 0 0K 0 raid_data 0 0K 96 geom_flashmap 0 0K 0 tmpfs dir 0 0K 0 tmpfs name 0 0K 0 tmpfs mount 0 0K 0 tmpfs extattr 0 0K 0 NFS FHA 0 0K 0 newnfsmnt 0 0K 0 newnfsclient_req 0 0K 0 NFSCL layrecall 0 0K 0 NFSCL session 0 0K 0 NFSCL sockreq 0 0K 0 NFSCL devinfo 0 0K 0 NFSCL flayout 0 0K 0 NFSCL layout 0 0K 0 NFSD rollback 0 0K 0 NFSCL diroff 0 0K 0 NEWdirectio 0 0K 0 NEWNFSnode 0 0K 0 NFSCL lck 0 0K 0 NFSCL lckown 0 0K 0 NFSCL client 0 0K 0 NFSCL deleg 0 0K 0 NFSCL open 0 0K 0 NFSCL owner 0 0K 0 NFS fh 0 0K 0 NFS req 0 0K 0 NFSD usrgroup 0 0K 0 NFSD string 0 0K 0 NFSD V4lock 0 0K 0 NFSD V4state 0 0K 0 msdosfs_fat 0 0K 0 msdosfs_mount 0 0K 0 msdosfs_node 0 0K 0 DEVFS4 0 0K 0 DEVFS2 0 0K 0 gntdev 0 0K 0 privcmd_dev 0 0K 0 evtchn_dev 0 0K 0 xenstore 0 0K 0 xnb 0 0K 0 xen_acpi 0 0K 0 xbbd 0 0K 0 xbd 0 0K 0 Balloon 0 0K 0 sysmouse 0 0K 0 vtfont 0 0K 0 pvscsi 0 0K 0 USBdev 0 0K 0 USB 0 0K 0 twsbuf 0 0K 0 tcp_log_dev 0 0K 32 midi buffers 0 0K 0 mixer 0 0K 0 ac97 0 0K 0 hdacc 0 0K 0 hdac 0 0K 0 hdaa 0 0K 0 SIIS driver 0 0K 0 PUC 0 0K 0 ppbusdev 0 0K 0 sr_iov 0 0K 0 OCS 0 0K 0 OCS 0 0K 0 nvme 0 0K 0 nvd 0 0K 0 netmap 0 0K 0 mwldev 0 0K 0 MVS driver 0 0K 0 mrsasbuf 0 0K 0 mpt_user 0 0K 0 mps_user 0 0K 0 MPSSAS 0 0K 0 mps 0 0K 0 mpr_user 0 0K 0 MPRSAS 0 0K 0 mpr 0 0K 0 mfibuf 0 0K 0 md_sectors 0 0K 0 malodev 0 0K 0 LED 0 0K 0 ix_sriov 0 0K 0 ix 0 0K 0 ipsbuf 0 0K 0 ciss_data 0 0K 0 BACKLIGHT 0 0K 0 ath_hal 0 0K 0 athdev 0 0K 0 ata_pci 0 0K 0 ata_dma 0 0K 0 ata_generic 0 0K 0 AHCI driver 0 0K 0 agp 0 0K 0 acpipwr 0 0K 0 acpi_perf 0 0K 0 acpicmbat 0 0K 0 aacraidcam 0 0K 0 aacraid_buf 0 0K 0 aaccam 0 0K 0 aacbuf 0 0K 0 zstd 0 0K 0 XZ_DEC 0 0K 0 nvlist 0 0K 0 SCSI ENC 0 0K 0 SCSI sa 0 0K 0 scsi_pass 0 0K 0 scsi_da 0 0K 69 ata_da 0 0K 0 scsi_ch 0 0K 0 scsi_cd 0 0K 0 nvme_da 0 0K 0 CAM CCB 0 0K 523 CAM ccb queue 0 0K 0 db> show uma Zone Size Used Free Requests Sleeps Bucket Total Mem XFree mbuf_jumbo_page 4096 8321 1077 639871 0 254 38494208 0 mbuf 256 8931 20037 2017728 0 254 7415808 0 tcp_log 416 0 15246 44559 0 254 6342336 0 BUF TRIE 144 206 11610 6675 0 62 1701504 0 malloc-384 384 4229 31 4232 0 30 1635840 0 mbuf_cluster 2048 762 0 762 0 254 1560576 0 malloc-128 128 11504 214 12112 0 126 1499904 0 RADIX NODE 144 9551 272 83731 0 62 1414512 0 malloc-4096 4096 326 2 524 0 2 1343488 0 UMA Slabs 0 112 11072 10 11072 0 126 1241184 0 sctp_asoc 2256 66 444 983 0 254 1150560 0 vmem btag 56 20126 145 20126 0 254 1135176 0 ertt_txseginfo 40 0 26260 566429 0 254 1050400 0 256 Bucket 2048 405 19 9430 0 8 868352 0 FFS inode 1168 509 44 1960 0 8 645904 0 sctp_chunk 152 38 4044 69333 0 254 620464 0 sctp_ep 1176 66 444 2248 0 254 599760 0 lkpicurr 176 2 3080 2 0 62 542432 0 pbuf 2624 0 198 0 0 2 519552 0 socket 960 33 475 4428 0 254 487680 0 tcp_bbr_map 128 0 3410 39943 0 126 436480 0 malloc-65536 65536 4 2 124 0 1 393216 0 sctp_raddr 736 66 451 1069 0 254 380512 0 VM OBJECT 264 1197 93 31922 0 30 340560 0 THREAD 1824 155 29 2182 0 8 335616 0 malloc-256 256 308 997 9401 0 62 334080 0 malloc-4096 4096 70 6 2705 0 2 311296 0 VNODE 448 549 117 2002 0 30 298368 0 malloc-64 64 3792 303 3885 0 254 262080 0 malloc-16 16 14379 371 14448 0 254 236000 0 malloc-32768 32768 0 7 2563 0 1 229376 0 DEVCTL 1024 26 194 157 0 0 225280 0 malloc-16384 16384 8 5 1301 0 1 212992 0 malloc-256 256 456 354 8414 0 62 207360 0 malloc-2048 2048 73 23 2520 0 8 196608 0 malloc-384 384 231 279 1572 0 30 195840 0 mbuf_packet 256 166 596 12666 0 254 195072 0 malloc-128 128 1316 203 28744 0 126 194432 0 UMA Zones 768 239 0 239 0 16 183552 0 malloc-32 32 5277 393 5351 0 254 181440 0 FPU_save_area 832 157 59 4943 0 16 179712 0 lkpimm 56 1 3095 1 0 254 173376 0 MAP ENTRY 96 1330 308 97565 0 126 157248 0 malloc-1024 1024 124 20 327 0 16 147456 0 malloc-128 128 973 174 3875 0 126 146816 0 S VFS Cache 104 1007 397 2582 0 126 146016 0 FFS2 dinode 256 509 61 1959 0 62 145920 0 malloc-65536 65536 2 0 2 0 1 131072 0 malloc-65536 65536 0 2 110 0 1 131072 0 malloc-65536 65536 2 0 2 0 1 131072 0 malloc-2048 2048 10 54 1042 0 8 131072 0 malloc-128 128 703 320 8802 0 126 130944 0 unpcb 256 14 496 1237 0 254 130560 0 128 Bucket 1024 83 32 543 0 16 117760 0 ksiginfo 112 63 981 2845 0 126 116928 0 PROC 1376 56 21 2208 0 8 105952 0 malloc-32768 32768 3 0 3 0 1 98304 0 malloc-32768 32768 1 2 53 0 1 98304 0 malloc-32768 32768 2 1 122 0 1 98304 0 malloc-1024 1024 74 22 991 0 16 98304 0 malloc-256 256 222 153 3174 0 62 96000 0 UMA Kegs 384 226 7 226 0 30 89472 0 64 Bucket 512 100 68 3652 0 30 86016 0 malloc-64 64 912 411 4099 0 254 84672 0 filedesc0 1072 56 21 2221 0 8 82544 0 malloc-16384 16384 5 0 5 0 1 81920 0 malloc-8192 8192 10 0 12 0 1 81920 0 malloc-384 384 133 77 3782 0 30 80640 0 malloc-256 256 3 312 1661 0 62 80640 0 sctp_readq 152 0 520 217 0 254 79040 0 malloc-8192 8192 7 2 108 0 1 73728 0 g_bio 408 0 180 41466 0 30 73440 0 malloc-64 64 715 356 39633 0 254 68544 0 malloc-65536 65536 0 1 8 0 1 65536 0 malloc-32768 32768 2 0 2 0 1 65536 0 malloc-4096 4096 12 4 224 0 2 65536 0 malloc-1024 1024 20 44 1647 0 16 65536 0 32 Bucket 256 85 170 2074 0 62 65280 0 sctp_stream_msg_out 112 40 500 272 0 254 60480 0 malloc-4096 4096 2 12 209 0 2 57344 0 malloc-4096 4096 14 0 14 0 2 57344 0 tcp_inpcb 1312 5 37 658 0 8 55104 0 udp_inpcb 424 6 120 213 0 30 53424 0 tcp_rack_map 128 0 403 1506 0 126 51584 0 DIRHASH 1024 35 13 35 0 16 49152 0 NAMEI 1024 0 48 20058 0 16 49152 0 malloc-16384 16384 1 2 161 0 1 49152 0 malloc-8192 8192 1 5 51 0 1 49152 0 malloc-2048 2048 11 13 96 0 8 49152 0 malloc-384 384 95 25 109 0 30 46080 0 tcp_bbr_pcb 832 0 54 379 0 16 44928 0 syncache 168 0 264 4 0 254 44352 0 Files 80 194 356 12851 0 126 44000 0 malloc-4096 4096 7 3 10 0 2 40960 0 pcpu-8 8 4744 376 5463 0 254 40960 0 VMSPACE 616 36 30 2189 0 16 40656 0 pipe 728 21 34 389 0 16 40040 0 da_ccb 544 0 70 10466 0 16 38080 0 malloc-64 64 134 433 17679 0 254 36288 0 malloc-64 64 32 535 55 0 254 36288 0 malloc-64 64 92 475 2471 0 254 36288 0 16 Bucket 144 79 173 710 0 62 36288 0 malloc-128 128 29 250 212 0 126 35712 0 malloc-128 128 8 271 11 0 126 35712 0 malloc-128 128 10 269 10 0 126 35712 0 routing nhops 256 27 108 34 0 62 34560 0 ttyoutq 256 72 63 160 0 62 34560 0 malloc-256 256 58 77 620 0 62 34560 0 malloc-256 256 11 124 11 0 62 34560 0 malloc-256 256 39 96 3526 0 62 34560 0 malloc-256 256 5 130 5 0 62 34560 0 TURNSTILE 136 185 67 185 0 62 34272 0 tcp_rack_pcb 1024 0 32 122 0 16 32768 0 malloc-8192 8192 2 2 60 0 1 32768 0 malloc-2048 2048 2 14 18 0 8 32768 0 malloc-2048 2048 2 14 6 0 8 32768 0 malloc-2048 2048 2 14 2 0 8 32768 0 malloc-2048 2048 10 6 10 0 8 32768 0 malloc-2048 2048 0 16 10 0 8 32768 0 malloc-1024 1024 5 27 61 0 16 32768 0 malloc-1024 1024 16 16 16 0 16 32768 0 malloc-1024 1024 4 28 4 0 16 32768 0 malloc-512 512 1 63 179 0 30 32768 0 malloc-512 512 4 60 40 0 30 32768 0 malloc-512 512 3 61 58 0 30 32768 0 malloc-512 512 8 56 360 0 30 32768 0 malloc-512 512 6 58 79 0 30 32768 0 pcpu-64 64 487 25 487 0 254 32768 0 PWD 40 19 789 1390 0 254 32320 0 KNOTE 160 30 170 31954 0 62 32000 0 ttyinq 160 135 65 300 0 62 32000 0 PGRP 120 28 236 41 0 126 31680 0 clpbuf 2624 0 12 46 0 4 31488 0 sctp_laddr 48 85 503 1014 0 254 28224 0 malloc-32 32 290 592 1479 0 254 28224 0 4 Bucket 48 6 582 10 0 254 28224 0 2 Bucket 32 61 821 998 0 254 28224 0 udplite_inpcb 424 0 63 5 0 30 26712 0 cpuset 200 7 121 7 0 62 25600 0 ripcb 392 3 60 37 0 30 24696 0 malloc-8192 8192 3 0 3 0 1 24576 0 rl_entry 40 77 529 77 0 254 24240 0 rtentry 168 30 114 34 0 62 24192 0 8 Bucket 80 60 240 1384 0 126 24000 0 itimer 352 0 66 7 0 30 23232 0 malloc-384 384 11 49 11 0 30 23040 0 SLEEPQUEUE 88 185 71 185 0 126 22528 0 hostcache 64 4 311 4 0 254 20160 0 udp_inpcb ports 32 3 627 33 0 254 20160 0 tcp_inpcb ports 32 3 627 466 0 254 20160 0 ertt 72 5 275 658 0 126 20160 0 malloc-64 64 2 313 2 0 254 20160 0 malloc-64 64 1 314 1 0 254 20160 0 malloc-32 32 78 552 110 0 254 20160 0 malloc-32 32 260 370 3716 0 254 20160 0 malloc-32 32 72 558 1431 0 254 20160 0 malloc-32 32 172 458 6447 0 254 20160 0 malloc-32 32 5 625 5 0 254 20160 0 malloc-32 32 2 628 8 0 254 20160 0 malloc-128 128 2 153 2 0 126 19840 0 cryptop 280 0 70 21 0 30 19600 0 epoch_record pcpu 256 4 60 4 0 62 16384 0 malloc-16384 16384 1 0 1 0 1 16384 0 malloc-512 512 1 31 1 0 30 16384 0 malloc-512 512 2 30 19 0 30 16384 0 SMR CPU 32 7 504 7 0 254 16352 0 vtnet_tx_hdr 24 0 668 378762 0 254 16032 0 malloc-16 16 525 475 5239 0 254 16000 0 kenv 258 17 43 1079 0 30 15480 0 mqnode 416 3 33 3 0 30 14976 0 vmem 1856 1 7 1 0 8 14848 0 SMR SHARED 24 7 504 7 0 254 12264 0 malloc-16 16 59 691 3442 0 254 12000 0 malloc-16 16 10 740 11 0 254 12000 0 malloc-16 16 148 602 1991 0 254 12000 0 malloc-16 16 32 718 29509 0 254 12000 0 malloc-16 16 21 729 21 0 254 12000 0 malloc-16 16 145 605 3605 0 254 12000 0 malloc-384 384 1 29 1 0 30 11520 0 malloc-384 384 13 17 13 0 30 11520 0 Mountpoints 2816 2 2 2 0 4 11264 0 malloc-8192 8192 1 0 1 0 1 8192 0 pcpu-16 16 4 252 4 0 254 4096 0 UMA Slabs 1 176 9 13 9 0 62 3872 0 KMAP ENTRY 96 12 27 14 0 0 3744 0 FFS1 dinode 128 0 0 0 0 126 0 0 ada_ccb 272 0 0 0 0 30 0 0 swblk 136 0 0 0 0 62 0 0 swpctrie 144 0 0 0 0 62 0 0 cdg_qdiffsample 16 0 0 0 0 254 0 0 pf state scrubs 40 0 0 0 0 254 0 0 pf frag entries 40 0 0 0 0 254 0 0 pf frags 248 0 0 0 0 62 0 0 pf table entries 160 0 0 0 0 254 0 0 pf table entry counters 64 0 0 0 0 254 0 0 pf source nodes 152 0 0 0 0 254 0 0 pf state keys 88 0 0 0 0 126 0 0 pf states 360 0 0 0 0 254 0 0 pf tags 104 0 0 0 0 126 0 0 pf mtags 184 0 0 0 0 62 0 0 tfo_ccache_entries 80 0 0 0 0 126 0 0 tfo 4 0 0 0 0 254 0 0 sackhole 32 0 0 0 0 254 0 0 ipq 56 0 0 0 0 254 0 0 sctp_asconf_ack 48 0 0 0 0 254 0 0 sctp_asconf 40 0 0 0 0 254 0 0 tcp_log_id_node 120 0 0 0 0 126 0 0 tcp_log_id_bucket 176 0 0 0 0 62 0 0 tcpreass 48 0 0 0 0 254 0 0 udplite_inpcb ports 32 0 0 0 0 254 0 0 ripcb ports 32 0 0 0 0 254 0 0 IPsec SA lft_c 16 0 0 0 0 254 0 0 AIOLIO 272 0 0 0 0 30 0 0 AIOCB 552 0 0 0 0 16 0 0 AIO 208 0 0 0 0 62 0 0 mqnotifier 216 0 0 0 0 62 0 0 mvdata 64 0 0 0 0 254 0 0 mqueue 248 0 0 0 0 62 0 0 TMPFS node 232 0 0 0 0 62 0 0 NCLNODE 608 0 0 0 0 16 0 0 LTS VFS Cache 360 0 0 0 0 30 0 0 L VFS Cache 320 0 0 0 0 30 0 0 STS VFS Cache 144 0 0 0 0 62 0 0 linux_dma_object 32 0 0 0 0 254 0 0 linux_dma_pctrie 144 0 0 0 0 62 0 0 IOMMU_MAP_ENTRY 104 0 0 0 0 126 0 0 mbuf_jumbo_16k 16384 0 0 0 0 254 0 0 mbuf_jumbo_9k 9216 0 0 0 0 254 0 0 audit_record 1280 0 0 0 0 8 0 0 domainset 40 0 0 0 0 254 0 0 MAC labels 40 0 0 0 0 254 0 0 vnpbuf 2624 0 0 0 0 16 0 0 nfspbuf 2624 0 0 0 0 4 0 0 swwbuf 2624 0 0 0 0 2 0 0 swrbuf 2624 0 0 0 0 4 0 0 umtx_shm 88 0 0 0 0 126 0 0 umtx pi 96 0 0 0 0 126 0 0 rangeset pctrie nodes 144 0 0 0 0 62 0 0 malloc-65536 65536 0 0 0 0 1 0 0 malloc-65536 65536 0 0 0 0 1 0 0 malloc-65536 65536 0 0 0 0 1 0 0 malloc-32768 32768 0 0 0 0 1 0 0 malloc-32768 32768 0 0 0 0 1 0 0 malloc-32768 32768 0 0 0 0 1 0 0 malloc-16384 16384 0 0 0 0 1 0 0 malloc-16384 16384 0 0 0 0 1 0 0 malloc-16384 16384 0 0 0 0 1 0 0 malloc-16384 16384 0 0 0 0 1 0 0 malloc-8192 8192 0 0 0 0 1 0 0 malloc-8192 8192 0 0 0 0 1 0 0 malloc-4096 4096 0 0 0 0 2 0 0 malloc-4096 4096 0 0 0 0 2 0 0 malloc-1024 1024 0 0 0 0 16 0 0 malloc-1024 1024 0 0 0 0 16 0 0 malloc-512 512 0 0 0 0 30 0 0 malloc-384 384 0 0 0 0 30 0 0 pcpu-32 32 0 0 0 0 254 0 0 pcpu-4 4 0 0 0 0 254 0 0 fakepg 104 0 0 0 0 126 0 0 UMA Hash 256 0 0 0 0 62 0 0