panic: mtx_lock() of destroyed mutex @ /syzkaller/managers/main/kernel/sys/netinet/sctp_output.c:6825 cpuid = 0 time = 1703788897 KDB: stack backtrace: db_trace_self_wrapper() at db_trace_self_wrapper+0xc6/frame 0xfffffe0007bdc8b0 kdb_backtrace() at kdb_backtrace+0xd0/frame 0xfffffe0007bdca10 vpanic() at vpanic+0x271/frame 0xfffffe0007bdcbb0 panic() at panic+0xb5/frame 0xfffffe0007bdcc70 __mtx_lock_flags() at __mtx_lock_flags+0x2d5/frame 0xfffffe0007bdcd50 sctp_sendall_completes() at sctp_sendall_completes+0x41/frame 0xfffffe0007bdcd70 sctp_iterator_worker() at sctp_iterator_worker+0x1022/frame 0xfffffe0007bdced0 sctp_iterator_thread() at sctp_iterator_thread+0x5e/frame 0xfffffe0007bdcef0 fork_exit() at fork_exit+0xcc/frame 0xfffffe0007bdcf30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe0007bdcf30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- KDB: enter: panic [ thread pid 5 tid 100067 ] Stopped at kdb_enter+0x6e: movq $0,0x2195db7(%rip) db> db> set $lines = 0 db> set $maxwidth = 0 db> show registers cs 0x20 ds 0x3b es 0x3b fs 0x13 gs 0x1b ss 0x28 rax 0x12 rcx 0xffffffff815d8165 printf+0xf5 rdx 0x1 rbx 0xffffffff826f6260 .str.28 rsp 0xfffffe0007bdc9f0 rbp 0xfffffe0007bdca10 rsi 0 rdi 0xffffffff815d81c6 printf+0x156 r8 0 r9 0xffffffff r10 0x7 r11 0x6 r12 0 r13 0xfffffe0057911000 r14 0xffffffff826f6260 .str.28 r15 0 rip 0xffffffff815c725e kdb_enter+0x6e rflags 0x46 kdb_enter+0x6e: movq $0,0x2195db7(%rip) db> show proc Process 5 (sctp_iterator) at 0xfffffe0057a43560: state: NORMAL uid: 0 gids: 0 parent: pid 0 at 0xffffffff836e7720 ABI: null flag: 0x10000204 flag2: 0 reaper: 0xffffffff836e7720 reapsubtree: 5 sigparent: 20 vmspace: 0xffffffff836e86c0 (map 0xffffffff836e86c0) (map.pmap 0xffffffff836e8780) (pmap 0xffffffff836e87f0) threads: 1 100067 Run CPU 0 [sctp_iterator] db> ps pid ppid pgrp uid state wmesg wchan cmd 4195 771 771 0 RE CPU 1 syz-executor.0 4192 772 772 0 R (threaded) syz-executor.1 104619 RunQ syz-executor.1 104748 S connec 0xfffffe0058baa0da syz-executor.1 104749 S uwait 0xfffffe005424ee00 syz-executor.1 104750 S uwait 0xfffffe005790b000 syz-executor.1 4191 778 778 0 S (threaded) syz-executor.2 104744 S nanslp 0xffffffff8373e400 syz-executor.2 104745 S connec 0xfffffe0058bbec1a syz-executor.2 104746 S uwait 0xfffffe007747a000 syz-executor.2 4189 788 788 0 S (threaded) syz-executor.3 104560 S nanslp 0xffffffff8373e401 syz-executor.3 104740 S connec 0xfffffe0058b970da syz-executor.3 104747 S uwait 0xfffffe005790c180 syz-executor.3 4103 4095 4103 0 Ss select 0xfffffe0058a659c0 dhclient 4098 1 4098 0 Ss select 0xfffffe0058a65940 dhclient 4095 4085 424 65 S select 0xfffffe0058953140 dhclient 4085 424 424 0 S wait 0xfffffe006fa065a0 sh 4074 1 4074 65 Ss select 0xfffffe0058a65ac0 dhclient 3852 0 0 0 DL (threaded) [KTLS] 103626 D - 0xfffffe00770bf700 [thr_0] 103974 D - 0xfffffe00770bf780 [thr_1] 103975 D - 0xffffffff8384fa88 [reclaim_0] 3498 0 0 0 DL aiordy 0xfffffe0070278060 [aiod4] 3497 0 0 0 DL aiordy 0xfffffe0077187560 [aiod3] 3496 0 0 0 DL aiordy 0xfffffe0077188580 [aiod2] 3495 0 0 0 DL aiordy 0xfffffe0077187ac0 [aiod1] 3481 1 3481 0 Ss select 0xfffffe0058952f40 dhclient 3475 1 3475 0 Ss select 0xfffffe0058953040 dhclient 3449 1 3449 65 Ss select 0xfffffe00589532c0 dhclient 1507 1 1507 0 Ss select 0xfffffe0058953640 dhclient 1504 1 1504 0 Ss select 0xfffffe00589530c0 dhclient 1485 1 1485 65 Ss select 0xfffffe00589535c0 dhclient 800 1 800 0 Ss select 0xfffffe00589533c0 dhclient 797 1 797 0 Ss select 0xfffffe0058953440 dhclient 788 768 788 0 Ss nanslp 0xffffffff8373e401 syz-executor.3 778 768 778 0 Ss nanslp 0xffffffff8373e400 syz-executor.2 772 768 772 0 Ss nanslp 0xffffffff8373e400 syz-executor.1 771 768 771 0 Ss nanslp 0xffffffff8373e401 syz-executor.0 768 766 766 0 S (threaded) syz-fuzzer 100098 S uwait 0xfffffe0057270880 syz-fuzzer 100110 S uwait 0xfffffe005790c580 syz-fuzzer 100111 S uwait 0xfffffe005790c680 syz-fuzzer 100112 S uwait 0xfffffe005790c780 syz-fuzzer 100113 S wait 0xfffffe006fa06040 syz-fuzzer 100114 S uwait 0xfffffe0057270980 syz-fuzzer 100115 S wait 0xfffffe006fa06040 syz-fuzzer 100117 S wait 0xfffffe006fa06040 syz-fuzzer 100119 S wait 0xfffffe006fa06040 syz-fuzzer 100120 S uwait 0xfffffe0057270b80 syz-fuzzer 100129 S uwait 0xfffffe005790b200 syz-fuzzer 100140 S kqread 0xfffffe00085f4a00 syz-fuzzer 100447 S uwait 0xfffffe0057273e80 syz-fuzzer 766 764 766 0 Ss pause 0xfffffe006fbeb0d0 csh 764 682 764 0 Ss select 0xfffffe00589531c0 sshd 748 1 748 0 Ss+ ttyin 0xfffffe00574facb0 getty 747 1 747 0 Ss+ ttyin 0xfffffe00586f00b0 getty 746 1 746 0 Ss+ ttyin 0xfffffe00586f04b0 getty 745 1 745 0 Ss+ ttyin 0xfffffe00586f08b0 getty 744 1 744 0 Ss+ ttyin 0xfffffe00586f0cb0 getty 743 1 743 0 Ss+ ttyin 0xfffffe00586f20b0 getty 742 1 742 0 Ss+ ttyin 0xfffffe00586f24b0 getty 741 1 741 0 Ss+ ttyin 0xfffffe00586f28b0 getty 740 1 740 0 Ss+ ttyin 0xfffffe00586f2cb0 getty 686 1 686 0 Ss nanslp 0xffffffff8373e400 cron 682 1 682 0 Ss select 0xfffffe0058952740 sshd 495 1 495 0 Ss select 0xfffffe0058952ec0 syslogd 424 1 424 0 Ss wait 0xfffffe0057a44ae0 devd 423 1 423 65 Ss select 0xfffffe0058952dc0 dhclient 338 1 338 0 Ss select 0xfffffe0058952e40 dhclient 335 1 335 0 Ss select 0xfffffe0058ac4d40 dhclient 17 0 0 0 DL syncer 0xffffffff8385bb20 [syncer] 16 0 0 0 DL vlruwt 0xfffffe0057212060 [vnlru] 15 0 0 0 DL (threaded) [bufdaemon] 100079 D psleep 0xffffffff8385a100 [bufdaemon] 100082 D - 0xffffffff82c0a140 [bufspacedaemon-0] 100090 D sdflush 0xfffffe00540da8e8 [/ worker] 9 0 0 0 DL psleep 0xffffffff838a35c0 [vmdaemon] 8 0 0 0 DL (threaded) [pagedaemon] 100077 D psleep 0xffffffff8388b438 [dom0] 100080 D launds 0xffffffff8388b444 [laundry: dom0] 100081 D umarcl 0xffffffff81d5fff0 [uma] 7 0 0 0 DL - 0xffffffff834bbc10 [rand_harvestq] 6 0 0 0 DL pftm 0xffffffff84576d10 [pf purge] 5 0 0 0 RL CPU 0 [sctp_iterator] 4 0 0 0 DL (threaded) [cam] 100044 D - 0xffffffff83486340 [doneq0] 100045 D - 0xffffffff834862c0 [async] 100076 D - 0xffffffff83486140 [scanner] 3 0 0 0 DL (threaded) [crypto] 100041 D crypto_ 0xffffffff83886c60 [crypto] 100042 D crypto_ 0xfffffe00540fde30 [crypto returns 0] 100043 D crypto_ 0xfffffe00540fde80 [crypto returns 1] 14 0 0 0 DL seqstat 0xfffffe00570ef088 [sequencer 00] 13 0 0 0 DL (threaded) [geom] 100035 D - 0xffffffff836e6d40 [g_event] 100036 D - 0xffffffff836e6d60 [g_up] 100037 D - 0xffffffff836e6d80 [g_down] 2 0 0 0 WL (threaded) [clock] 100029 I [clock (0)] 100030 I [clock (1)] 12 0 0 0 WL (threaded) [intr] 100012 I [swi6: task queue] 100013 I [swi6: Giant taskq] 100015 I [swi5: fast taskq] 100031 I [swi1: netisr 0] 100032 I [swi1: hpts] 100033 I [swi1: hpts] 100046 I [irq24: virtio_pci0] 100047 I [irq25: virtio_pci0] 100048 I [irq26: virtio_pci0] 100049 I [irq27: virtio_pci0] 100050 I [irq28: virtio_pci1] 100051 I [irq29: virtio_pci1] 100052 I [irq30: virtio_pci1] 100053 I [irq31: virtio_pci1] 100054 I [irq32: virtio_pci1] 100059 I [irq33: virtio_pci2] 100060 I [irq34: virtio_pci2] 100061 I [irq35: virtio_pci2] 100063 I [irq1: atkbd0] 100064 I [irq12: psm0] 100065 I [swi0: uart uart++] 100069 I [swi1: pf send] 11 0 0 0 RL (threaded) [idle] 100003 CanRun [idle: cpu0] 100004 CanRun [idle: cpu1] 1 0 1 0 SLs wait 0xfffffe0054247040 [init] 10 0 0 0 DL audit_w 0xffffffff838876c0 [audit] 0 0 0 0 DLs (threaded) [kernel] 100000 D swapin 0xffffffff836e7720 [swapper] 100005 D - 0xfffffe00085f3a00 [softirq_0] 100006 D - 0xfffffe00085f3800 [softirq_1] 100007 D - 0xfffffe00085f3600 [if_io_tqg_0] 100008 D - 0xfffffe00085f3400 [if_io_tqg_1] 100009 D - 0xfffffe00085f3200 [if_config_tqg_0] 100010 D - 0xfffffe00085f3000 [pci_hp taskq] 100011 D - 0xfffffe00085f2d00 [kqueue_ctx taskq] 100014 D - 0xfffffe00085f2700 [thread taskq] 100016 D - 0xfffffe00085f2300 [aiod_kick taskq] 100017 D - 0xfffffe00085f2100 [deferred_unmount ta] 100018 D - 0xfffffe00085f1e00 [inm_free taskq] 100019 D - 0xfffffe00085f1c00 [in6m_free taskq] 100020 D - 0xfffffe00085f1a00 [linuxkpi_irq_wq] 100021 D - 0xfffffe00085f1800 [linuxkpi_short_wq_0] 100022 D - 0xfffffe00085f1800 [linuxkpi_short_wq_1] 100023 D - 0xfffffe00085f1800 [linuxkpi_short_wq_2] 100024 D - 0xfffffe00085f1800 [linuxkpi_short_wq_3] 100025 D - 0xfffffe00085f1300 [linuxkpi_long_wq_0] 100026 D - 0xfffffe00085f1300 [linuxkpi_long_wq_1] 100027 D - 0xfffffe00085f1300 [linuxkpi_long_wq_2] 100028 D - 0xfffffe00085f1300 [linuxkpi_long_wq_3] 100034 D - 0xfffffe00085f0800 [firmware taskq] 100039 D - 0xfffffe00085f0200 [crypto_0] 100040 D - 0xfffffe00085f0200 [crypto_1] 100055 D - 0xfffffe00571ba800 [vtnet0 rxq 0] 100056 D - 0xfffffe00571ba700 [vtnet0 txq 0] 100057 D - 0xfffffe00571ba600 [vtnet0 rxq 1] 100058 D - 0xfffffe00571ba500 [vtnet0 txq 1] 100062 D vtbslp 0xfffffe00571e3180 [virtio_balloon] 100066 D - 0xffffffff826fb4c0 [deadlkres] 100070 D - 0xfffffe0057914500 [acpi_task_0] 100071 D - 0xfffffe0057914500 [acpi_task_1] 100072 D - 0xfffffe0057914500 [acpi_task_2] 100074 D - 0xfffffe00085f4100 [mca taskq] 100075 D - 0xfffffe00571bb000 [CAM taskq] db> show all locks db> show malloc Type InUse MemUse Requests pf_hash 5 11524K 5 tcp_hpts 7 4801K 7 devbuf 4192 4324K 4220 sysctloid 35147 2071K 35218 vtbuf 24 1968K 46 kobj 326 1304K 488 newblk 60 1039K 5622 vfscache 3 1025K 3 pcb 271 943K 4011 inodedep 29 523K 3463 ufs_quota 1 512K 1 vfs_hash 1 512K 1 callout 2 512K 2 intr 4 472K 4 subproc 152 286K 4285 vmem 3 266K 6 vnet_data 2 224K 2 acpitask 1 224K 1 acpica 1674 184K 60830 tidhash 3 141K 3 filedesc 18 137K 6641 pagedep 10 131K 3329 linker 352 130K 386 tfo_ccache 1 128K 1 IP reass 1 128K 1 sctp_stro 118 118K 429 DEVFS1 109 109K 126 sem 4 106K 4 gtaskqueue 18 98K 18 sctp_atcl 240 90K 2525 BPF 46 88K 50 bus 985 81K 5155 mtx_pool 2 72K 2 syncache 1 68K 1 NFSD srvcache 3 68K 3 module 513 65K 513 ddb_capture 1 64K 1 umtx 384 48K 384 kdtrace 227 45K 8950 temp 35 37K 2180 hostcache 1 32K 1 shm 1 32K 1 DEVFS3 128 32K 138 msg 4 30K 4 kbdmux 6 28K 6 LRO 24 25K 24 ifaddr 70 20K 72 DEVFS_RULE 56 20K 56 ufs_mount 4 17K 5 proc 3 17K 3 tty 16 16K 16 routetbl 128 16K 411 ithread 97 16K 97 sctp_atky 358 15K 3113 bus-sc 34 15K 1687 eventhandler 161 14K 161 lltable 43 14K 51 KTRACE 100 13K 100 ifnet 7 13K 7 ether_multi 152 13K 162 kenv 95 12K 95 GEOM 61 11K 481 rman 86 11K 429 CAM queue 5 11K 1528 in6_multi 65 9K 65 bmsafemap 3 9K 3462 freework 34 9K 3961 tcp_fsb 4 9K 4734 rpc 4 9K 4 devstat 4 9K 4 UART 12 9K 12 sctp_timw 32 8K 32 ksem 1 8K 4 shmfd 1 8K 31 pfs_vncache 1 8K 1 audit_evclass 238 8K 300 taskqueue 63 7K 63 cred 26 7K 264 kqueue 65 7K 4202 sglist 6 7K 6 plimit 24 6K 409 CAM DEV 3 6K 510 pfs_nodes 20 5K 20 hhook 15 5K 17 ufs_dirhash 24 5K 24 UMA 270 5K 270 pf_ifnet 10 5K 19 session 34 5K 60 DEVFSP 68 5K 158 vt 11 5K 11 memdesc 1 4K 1 MCA 32 4K 32 evdev 4 4K 4 pwddesc 62 4K 4196 sctp_athm 240 4K 2684 sctp_map 236 4K 952 proc-args 94 4K 5426 dirrem 14 4K 3368 acpisem 28 4K 28 lockf 32 4K 54 selfd 52 4K 68046 kcovinfo 52 4K 52 CC Mem 12 3K 4200 terminal 11 3K 11 select 19 3K 49 freeblks 9 3K 3410 clone 9 3K 9 uidinfo 3 3K 8 local_apic 1 2K 1 io_apic 1 2K 1 freefile 16 2K 3366 ipsec-saq 2 2K 2 ip6ndp 12 2K 13 Unitno 31 2K 49 sctp_ifa 13 2K 14 tun 7 2K 7 CAM XPT 22 2K 543 msi 12 2K 12 in_multi 6 2K 8 toponodes 6 2K 6 ipsecpolicy 2 2K 2 acpidev 20 2K 20 softdep 1 1K 1 mkdir 8 1K 6620 sahead 1 1K 1 secasvar 1 1K 1 nhops 6 1K 8 vnodemarker 2 1K 28 NFSD session 1 1K 1 CAM periph 4 1K 271 sctp_ifn 6 1K 14 ipsec 3 1K 3 mld 6 1K 6 igmp 6 1K 6 pfil 6 1K 6 isadev 6 1K 6 inpcbpolicy 23 1K 4668 mount 16 1K 89 pci_link 10 1K 10 diradd 5 1K 3407 crypto 4 1K 40 encap_export_host 12 1K 12 procdesc 5 1K 14 osd 17 1K 4213 newdirblk 4 1K 3310 indirdep 2 1K 955 cdev 2 1K 2 lkpikmalloc 8 1K 9 netlink 2 1K 2 chacha20random 1 1K 1 biobuf 1 1K 1 sctp_iter 1 1K 83 vnodes 1 1K 1 ktls 1 1K 13 NFSD lckfile 1 1K 1 NFSD V4client 1 1K 1 DEVFS 9 1K 10 CAM SIM 2 1K 2 feeder 7 1K 7 soname 7 1K 16199 tcpfunc 3 1K 3 loginclass 3 1K 6 prison 6 1K 6 sctp_cpal 1 1K 74 cryptodev 2 1K 98 nexusdev 8 1K 8 apmdev 1 1K 1 atkbddev 2 1K 2 aio 4 1K 4 pmchooks 1 1K 1 CAM path 4 1K 1034 CAM dev queue 2 1K 2 CAM I/O Scheduler 1 1K 1 filecaps 5 1K 93 sctp_vrf 1 1K 1 vnet 1 1K 1 pmc 1 1K 1 entropy 2 1K 41 acpiintr 1 1K 1 cpus 2 1K 2 vnet_data_free 1 1K 1 Per-cpu 1 1K 1 p1003.1b 1 1K 1 pf_table 0 0K 0 pf_rule 0 0K 0 pf_altq 0 0K 0 pf_osfp 0 0K 0 pf_krule_item 0 0K 0 pf_temp 0 0K 0 sctp_mcore 0 0K 0 sctp_socko 0 0K 637 sctp_mvrf 0 0K 0 sctp_cmsg 0 0K 0 sctp_stre 0 0K 0 sctp_athi 0 0K 0 sctp_a_it 0 0K 12 sctp_aadr 0 0K 0 sctp_stri 0 0K 318 filemon 0 0K 38 tcp_do 0 0K 0 mqdata 0 0K 0 ipcomp 0 0K 0 esp 0 0K 0 ah 0 0K 0 madt_table 0 0K 2 smartpqi 0 0K 0 ixl 0 0K 0 ice-resmgr 0 0K 0 ice-osdep 0 0K 0 ice 0 0K 0 iavf 0 0K 0 axgbe 0 0K 0 fpukern_ctx 0 0K 0 xen_intr 0 0K 0 xen_hvm 0 0K 0 legacydrv 0 0K 0 bounce 0 0K 0 busdma 0 0K 0 qpidrv 0 0K 0 dmar_idpgtbl 0 0K 0 dmar_dom 0 0K 0 dmar_ctx 0 0K 0 isci 0 0K 0 iommu_dmamap 0 0K 0 hyperv_socket 0 0K 0 bxe_ilt 0 0K 0 aesni_data 0 0K 0 xenbus 0 0K 0 vm_fictitious 0 0K 0 UMAHash 0 0K 0 vm_pgdata 0 0K 0 jblocks 0 0K 0 savedino 0 0K 2121 sentinel 0 0K 0 jfsync 0 0K 0 jtrunc 0 0K 0 sbdep 0 0K 11 jsegdep 0 0K 0 jseg 0 0K 0 jfreefrag 0 0K 0 jfreeblk 0 0K 0 jnewblk 0 0K 0 jmvref 0 0K 0 jremref 0 0K 0 jaddref 0 0K 0 freedep 0 0K 0 freefrag 0 0K 23 allocindir 0 0K 0 allocdirect 0 0K 0 ufs_trim 0 0K 0 mactemp 0 0K 0 audit_trigger 0 0K 0 audit_pipe_presel 0 0K 0 audit_pipeent 0 0K 0 audit_pipe 0 0K 0 audit_evname 0 0K 0 audit_bsm 0 0K 0 audit_gidset 0 0K 0 audit_text 0 0K 0 audit_path 0 0K 0 audit_data 0 0K 0 audit_cred 0 0K 0 ktls_ocf 0 0K 3 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5E_TLS_RX 0 0K 0 MLX5EEPROM 0 0K 0 MLX5E_TLS 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EN 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5DUMP 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 simple_attr 0 0K 0 seq_file 0 0K 0 lkpiskb 0 0K 0 radix 0 0K 0 idr 0 0K 0 lkpindev 0 0K 0 lkpimhi 0 0K 0 lkpifw 0 0K 0 lkpi80211 0 0K 0 NLM 0 0K 0 ipsec-spdcache 0 0K 0 ipsec-reg 0 0K 0 ipsec-misc 0 0K 0 ipsecrequest 0 0K 0 ip6opt 0 0K 3 ip6_msource 0 0K 0 ip6_moptions 0 0K 0 in6_mfilter 0 0K 0 frag6 0 0K 0 tcplog 0 0K 0 tcp_hwpace 0 0K 0 ip_msource 0 0K 0 ip_moptions 0 0K 3 in_mfilter 0 0K 0 ipid 0 0K 0 80211scan 0 0K 0 80211ratectl 0 0K 0 80211power 0 0K 0 80211nodeie 0 0K 0 80211node 0 0K 0 80211mesh_gt 0 0K 0 80211mesh_rt 0 0K 0 80211perr 0 0K 0 80211prep 0 0K 0 80211preq 0 0K 0 80211dfs 0 0K 0 80211crypto 0 0K 0 80211vap 0 0K 0 iflib 0 0K 0 vlan 0 0K 0 gif 0 0K 0 ifdescr 0 0K 0 zlib 0 0K 0 fadvise 0 0K 0 VN POLL 0 0K 0 statfs 0 0K 198 namei_tracker 0 0K 0 export_host 0 0K 0 cl_savebuf 0 0K 20 lio 0 0K 0 acl 0 0K 0 mbuf_tag 0 0K 0 accf 0 0K 0 pts 0 0K 0 timerfd 0 0K 0 iov 0 0K 19350 ioctlops 0 0K 143 eventfd 0 0K 0 Witness 0 0K 0 stack 0 0K 0 sbuf 0 0K 288 firmware 0 0K 0 compressor 0 0K 0 SWAP 0 0K 0 sysctltmp 0 0K 767 sysctl 0 0K 3 ekcd 0 0K 0 dumper 0 0K 0 sendfile 0 0K 0 rctl 0 0K 0 cache 0 0K 0 prison_racct 0 0K 0 Fail Points 0 0K 0 sigio 0 0K 1 filedesc_to_leader 0 0K 0 pwd 0 0K 0 tty console 0 0K 0 boottrace 0 0K 0 isofs_node 0 0K 0 isofs_mount 0 0K 0 tr_raid5_data 0 0K 0 tr_raid1e_data 0 0K 0 tr_raid1_data 0 0K 0 tr_raid0_data 0 0K 0 tr_concat_data 0 0K 0 md_sii_data 0 0K 0 md_promise_data 0 0K 0 md_nvidia_data 0 0K 0 md_jmicron_data 0 0K 0 md_intel_data 0 0K 0 md_ddf_data 0 0K 0 raid_data 0 0K 72 geom_flashmap 0 0K 0 tmpfs dir 0 0K 0 tmpfs name 0 0K 0 tmpfs mount 0 0K 0 tmpfs extattr 0 0K 0 NFS FHA 0 0K 0 newnfsmnt 0 0K 0 newnfsclient_req 0 0K 0 NFSCL layrecall 0 0K 0 NFSCL session 0 0K 0 NFSCL sockreq 0 0K 0 NFSCL devinfo 0 0K 0 NFSCL flayout 0 0K 0 NFSCL layout 0 0K 0 NFSD rollback 0 0K 0 NFSCL diroff 0 0K 0 NEWdirectio 0 0K 0 NEWNFSnode 0 0K 0 NFSCL lck 0 0K 0 NFSCL lckown 0 0K 0 NFSCL client 0 0K 0 NFSCL deleg 0 0K 0 NFSCL open 0 0K 0 NFSCL owner 0 0K 0 NFS fh 0 0K 0 NFS req 0 0K 0 NFSD usrgroup 0 0K 0 NFSD string 0 0K 0 NFSD V4lock 0 0K 0 NFSD V4state 0 0K 0 msdosfs_fat 0 0K 0 msdosfs_mount 0 0K 0 msdosfs_node 0 0K 0 DEVFS4 0 0K 0 DEVFS2 0 0K 0 gntdev 0 0K 0 privcmd_dev 0 0K 0 evtchn_dev 0 0K 0 xenstore 0 0K 0 xnb 0 0K 0 xen_acpi 0 0K 0 xbbd 0 0K 0 xbd 0 0K 0 Balloon 0 0K 0 sysmouse 0 0K 0 vtfont 0 0K 0 pvscsi 0 0K 0 USBdev 0 0K 0 USB 0 0K 0 twsbuf 0 0K 0 tcp_log_dev 0 0K 0 midi buffers 0 0K 0 mixer 0 0K 0 ac97 0 0K 0 hdacc 0 0K 0 hdac 0 0K 0 hdaa 0 0K 0 SIIS driver 0 0K 0 PUC 0 0K 0 ppbusdev 0 0K 0 sr_iov 0 0K 0 OCS 0 0K 0 OCS 0 0K 0 nvme 0 0K 0 nvd 0 0K 0 netmap 0 0K 0 mwldev 0 0K 0 MVS driver 0 0K 0 mrsasbuf 0 0K 0 mpt_user 0 0K 0 mps_user 0 0K 0 MPSSAS 0 0K 0 mps 0 0K 0 mpr_user 0 0K 0 MPRSAS 0 0K 0 mpr 0 0K 0 mfibuf 0 0K 0 md_sectors 0 0K 0 md_disk 0 0K 0 malodev 0 0K 0 LED 0 0K 0 ix_sriov 0 0K 0 ix 0 0K 0 ipsbuf 0 0K 0 ciss_data 0 0K 0 BACKLIGHT 0 0K 0 ath_hal 0 0K 0 athdev 0 0K 0 ata_pci 0 0K 0 ata_dma 0 0K 0 ata_generic 0 0K 0 AHCI driver 0 0K 0 agp 0 0K 0 acpipwr 0 0K 0 acpi_perf 0 0K 0 acpicmbat 0 0K 0 aacraidcam 0 0K 0 aacraid_buf 0 0K 0 aaccam 0 0K 0 aacbuf 0 0K 0 zstd 0 0K 0 XZ_DEC 0 0K 0 nvlist 0 0K 0 SCSI ENC 0 0K 0 SCSI sa 0 0K 0 scsi_pass 0 0K 0 scsi_da 0 0K 69 ata_da 0 0K 0 scsi_ch 0 0K 0 scsi_cd 0 0K 0 nvme_da 0 0K 0 CAM CCB 0 0K 523 CAM ccb queue 0 0K 0 db> show uma Zone Size Used Free Requests Sleeps Bucket Total Mem XFree mbuf_jumbo_page 4096 8341 1057 1314269 0 254 38494208 0 mbuf 256 15739 7889 3161063 0 254 6048768 0 RADIX NODE 144 14063 212 152732 0 62 2055600 0 BUF TRIE 144 203 11613 2716 0 62 1701504 0 tcp_log 416 0 4068 114065 0 254 1692288 0 malloc-4096 4096 392 4 5243 0 2 1622016 0 malloc-384 384 4159 41 4161 0 30 1612800 0 malloc-128 128 12142 103 13092 0 126 1567360 0 mbuf_cluster 2048 762 0 762 0 254 1560576 0 UMA Slabs 0 112 11124 30 11124 0 126 1249248 0 sctp_asoc 2256 118 392 429 0 254 1150560 0 vmem btag 56 18779 124 18779 0 254 1058568 0 socket 960 53 711 7908 0 254 733440 0 sctp_chunk 152 2373 1969 67723 0 254 659984 0 pbuf 2624 0 246 0 0 2 645504 0 FFS inode 1168 514 25 3881 0 8 629552 0 sctp_ep 1176 122 388 1937 0 254 599760 0 lkpimm 176 1 3081 1 0 62 542432 0 lkpicurr 176 2 3080 2 0 62 542432 0 256 Bucket 2048 239 25 10718 0 8 540672 0 malloc-384 384 291 909 6057 0 30 460800 0 malloc-2048 2048 125 83 2487 0 8 425984 0 sctp_raddr 736 118 399 429 0 254 380512 0 VM OBJECT 264 1336 74 54421 0 30 372240 0 ertt_txseginfo 40 2589 6602 662960 0 254 367640 0 THREAD 1824 163 29 4753 0 8 350208 0 malloc-256 256 94 1151 2565 0 62 318720 0 malloc-2048 2048 10 142 520 0 8 311296 0 VNODE 448 553 113 3922 0 30 298368 0 malloc-256 256 140 985 5364 0 62 288000 0 malloc-64 64 3947 400 3972 0 254 278208 0 malloc-65536 65536 4 0 4 0 1 262144 0 malloc-16384 16384 11 4 3321 0 1 245760 0 malloc-16 16 14562 438 18862 0 254 240000 0 malloc-32768 32768 1 6 901 0 1 229376 0 DEVCTL 1024 10 210 151 0 0 225280 0 malloc-256 256 108 762 5787 0 62 222720 0 malloc-65536 65536 1 2 111 0 1 196608 0 mbuf_packet 256 148 614 6631 0 254 195072 0 UMA Zones 768 242 2 242 0 16 187392 0 malloc-32 32 5343 327 5385 0 254 181440 0 MAP ENTRY 96 1597 293 147525 0 126 181440 0 FPU_save_area 832 165 51 13315 0 16 179712 0 malloc-128 128 1146 249 28348 0 126 178560 0 malloc-128 128 85 1186 6816 0 126 162688 0 malloc-1024 1024 120 24 172 0 16 147456 0 S VFS Cache 104 1015 389 4428 0 126 146016 0 FFS2 dinode 256 514 56 3880 0 62 145920 0 malloc-4096 4096 29 5 36 0 2 139264 0 malloc-65536 65536 0 2 166 0 1 131072 0 malloc-65536 65536 2 0 2 0 1 131072 0 malloc-32768 32768 4 0 4 0 1 131072 0 malloc-1024 1024 123 5 589 0 16 131072 0 unpcb 256 20 490 1268 0 254 130560 0 PROC 1376 62 26 4195 0 8 121088 0 tcp_inpcb 1312 12 78 4200 0 8 118080 0 ksiginfo 112 64 980 8726 0 126 116928 0 128 Bucket 1024 72 27 478 0 16 101376 0 tcp_bbr_map 128 312 463 19419 0 126 99200 0 malloc-8192 8192 7 5 45 0 1 98304 0 malloc-256 256 213 162 7879 0 62 96000 0 malloc-256 256 251 124 8309 0 62 96000 0 filedesc0 1072 63 21 4196 0 8 90048 0 UMA Kegs 384 228 5 228 0 30 89472 0 pipe 728 23 98 468 0 16 88088 0 64 Bucket 512 126 42 4465 0 30 86016 0 malloc-128 128 417 234 434 0 126 83328 0 sctp_readq 152 0 520 50 0 254 79040 0 g_bio 408 0 180 19151 0 30 73440 0 malloc-64 64 592 479 5050 0 254 68544 0 malloc-64 64 522 549 1648 0 254 68544 0 malloc-128 128 334 193 650 0 126 67456 0 malloc-128 128 268 259 9806 0 126 67456 0 malloc-65536 65536 1 0 1 0 1 65536 0 malloc-32768 32768 2 0 2 0 1 65536 0 malloc-32768 32768 0 2 120 0 1 65536 0 malloc-4096 4096 14 2 25 0 2 65536 0 malloc-1024 1024 16 48 524 0 16 65536 0 32 Bucket 256 102 153 3667 0 62 65280 0 sctp_stream_msg_out 112 52 488 222 0 254 60480 0 udplite_inpcb 424 0 126 68 0 30 53424 0 udp_inpcb 424 6 120 273 0 30 53424 0 malloc-64 64 370 449 83692 0 254 52416 0 Files 80 234 416 18958 0 126 52000 0 VMSPACE 616 41 43 4175 0 16 51744 0 tcp_rack_map 128 0 403 15175 0 126 51584 0 malloc-256 256 94 101 1192 0 62 49920 0 malloc-256 256 49 146 703 0 62 49920 0 malloc-256 256 36 159 7809 0 62 49920 0 ripcb 392 5 121 127 0 30 49392 0 tcp_rack_pcb 1024 2 46 2367 0 16 49152 0 DIRHASH 1024 35 13 35 0 16 49152 0 NAMEI 1024 0 48 27879 0 16 49152 0 malloc-4096 4096 3 9 2370 0 2 49152 0 malloc-2048 2048 14 10 26 0 8 49152 0 malloc-384 384 89 31 432 0 30 46080 0 syncache 168 0 264 4 0 254 44352 0 malloc-8192 8192 3 2 106 0 1 40960 0 pcpu-8 8 4791 329 4975 0 254 40960 0 da_ccb 544 0 70 4910 0 16 38080 0 malloc-64 64 7 560 68 0 254 36288 0 malloc-64 64 126 441 9007 0 254 36288 0 malloc-64 64 62 505 123 0 254 36288 0 malloc-64 64 185 382 1147 0 254 36288 0 malloc-32 32 538 596 2944 0 254 36288 0 16 Bucket 144 80 172 461 0 62 36288 0 malloc-128 128 20 259 86 0 126 35712 0 malloc-128 128 73 206 6784 0 126 35712 0 routing nhops 256 27 108 34 0 62 34560 0 ttyoutq 256 72 63 160 0 62 34560 0 malloc-384 384 53 37 54 0 30 34560 0 TURNSTILE 136 193 59 193 0 62 34272 0 malloc-32768 32768 1 0 1 0 1 32768 0 malloc-16384 16384 2 0 2 0 1 32768 0 malloc-16384 16384 0 2 160 0 1 32768 0 malloc-8192 8192 3 1 29 0 1 32768 0 malloc-2048 2048 8 8 10 0 8 32768 0 malloc-2048 2048 1 15 51 0 8 32768 0 malloc-2048 2048 2 14 2 0 8 32768 0 malloc-2048 2048 4 12 195 0 8 32768 0 malloc-1024 1024 2 30 42 0 16 32768 0 malloc-1024 1024 1 31 1148 0 16 32768 0 malloc-1024 1024 10 22 14 0 16 32768 0 malloc-1024 1024 16 16 27 0 16 32768 0 malloc-1024 1024 3 29 3 0 16 32768 0 malloc-512 512 1 63 119 0 30 32768 0 malloc-512 512 13 51 83 0 30 32768 0 malloc-512 512 2 62 28 0 30 32768 0 malloc-512 512 8 56 23 0 30 32768 0 malloc-512 512 2 62 371 0 30 32768 0 malloc-512 512 2 62 23 0 30 32768 0 pcpu-64 64 487 25 487 0 254 32768 0 PWD 40 21 787 3399 0 254 32320 0 KNOTE 160 28 172 60677 0 62 32000 0 ttyinq 160 135 65 300 0 62 32000 0 PGRP 120 34 230 60 0 126 31680 0 clpbuf 2624 0 12 45 0 4 31488 0 tcp_bbr_pcb 832 2 34 568 0 16 29952 0 sctp_laddr 48 249 339 780 0 254 28224 0 tcp_inpcb ports 32 6 876 3054 0 254 28224 0 4 Bucket 48 5 583 9 0 254 28224 0 8 Bucket 80 82 268 2309 0 126 28000 0 AIO 208 0 133 6 0 62 27664 0 cpuset 200 8 120 8 0 62 25600 0 malloc-8192 8192 3 0 3 0 1 24576 0 malloc-8192 8192 3 0 3 0 1 24576 0 malloc-8192 8192 3 0 3 0 1 24576 0 malloc-4096 4096 4 2 5 0 2 24576 0 rl_entry 40 67 539 67 0 254 24240 0 rtentry 168 30 114 34 0 62 24192 0 malloc-384 384 14 46 14 0 30 23040 0 malloc-384 384 19 41 27 0 30 23040 0 malloc-384 384 2 58 3 0 30 23040 0 Mountpoints 2816 2 6 2 0 4 22528 0 SLEEPQUEUE 88 193 63 193 0 126 22528 0 hostcache 64 2 313 2 0 254 20160 0 udplite_inpcb ports 32 0 630 3 0 254 20160 0 udp_inpcb ports 32 3 627 48 0 254 20160 0 ertt 72 12 268 4200 0 126 20160 0 malloc-32 32 66 564 5881 0 254 20160 0 malloc-32 32 74 556 478 0 254 20160 0 malloc-32 32 110 520 1924 0 254 20160 0 malloc-32 32 159 471 4914 0 254 20160 0 malloc-32 32 25 605 2961 0 254 20160 0 2 Bucket 32 62 568 935 0 254 20160 0 malloc-16 16 544 706 7762 0 254 20000 0 AIOCB 552 0 35 6 0 16 19320 0 ktls_session 256 0 75 6 0 62 19200 0 epoch_record pcpu 256 4 60 4 0 62 16384 0 malloc-16384 16384 1 0 1 0 1 16384 0 malloc-16384 16384 1 0 1 0 1 16384 0 malloc-8192 8192 2 0 2 0 1 16384 0 malloc-4096 4096 1 3 2 0 2 16384 0 malloc-4096 4096 1 3 237 0 2 16384 0 malloc-2048 2048 4 4 62 0 8 16384 0 malloc-512 512 1 31 1 0 30 16384 0 SMR CPU 32 7 504 7 0 254 16352 0 vtnet_tx_hdr 24 0 668 653677 0 254 16032 0 malloc-16 16 436 564 4739 0 254 16000 0 kenv 258 17 43 1081 0 30 15480 0 mqnode 416 3 33 3 0 30 14976 0 vmem 1856 1 7 1 0 8 14848 0 SMR SHARED 24 7 504 7 0 254 12264 0 malloc-32 32 10 368 53 0 254 12096 0 malloc-16 16 23 727 66 0 254 12000 0 malloc-16 16 65 685 14354 0 254 12000 0 malloc-16 16 26 724 29 0 254 12000 0 malloc-16 16 10 740 13 0 254 12000 0 malloc-16 16 19 731 27741 0 254 12000 0 malloc-384 384 1 29 1 0 30 11520 0 pcpu-16 16 8 504 8 0 254 8192 0 UMA Slabs 1 176 9 13 9 0 62 3872 0 KMAP ENTRY 96 12 27 14 0 0 3744 0 FFS1 dinode 128 0 0 0 0 126 0 0 ada_ccb 272 0 0 0 0 30 0 0 swblk 136 0 0 0 0 62 0 0 swpctrie 144 0 0 0 0 62 0 0 cdg_qdiffsample 16 0 0 0 0 254 0 0 pf state scrubs 40 0 0 0 0 254 0 0 pf frag entries 40 0 0 0 0 254 0 0 pf frags 248 0 0 0 0 62 0 0 pf table entries 160 0 0 0 0 254 0 0 pf table entry counters 64 0 0 0 0 254 0 0 pf source nodes 152 0 0 0 0 254 0 0 pf state keys 88 0 0 0 0 126 0 0 pf states 352 0 0 0 0 254 0 0 pf tags 104 0 0 0 0 126 0 0 pf mtags 184 0 0 0 0 62 0 0 tfo_ccache_entries 80 0 0 0 0 126 0 0 tfo 4 0 0 0 0 254 0 0 sackhole 32 0 0 0 0 254 0 0 ipq 56 0 0 0 0 254 0 0 sctp_asconf_ack 48 0 0 0 0 254 0 0 sctp_asconf 40 0 0 0 0 254 0 0 tcp_log_id_node 120 0 0 0 0 126 0 0 tcp_log_id_bucket 176 0 0 0 0 62 0 0 tcpreass 48 0 0 0 0 254 0 0 ripcb ports 32 0 0 0 0 254 0 0 IPsec SA lft_c 16 0 0 0 0 254 0 0 netlink 2048 0 0 0 0 8 0 0 itimer 352 0 0 0 0 30 0 0 AIOLIO 272 0 0 0 0 30 0 0 mqnotifier 216 0 0 0 0 62 0 0 mvdata 64 0 0 0 0 254 0 0 mqueue 248 0 0 0 0 62 0 0 TMPFS node 232 0 0 0 0 62 0 0 NCLNODE 608 0 0 0 0 16 0 0 LTS VFS Cache 360 0 0 0 0 30 0 0 L VFS Cache 320 0 0 0 0 30 0 0 STS VFS Cache 144 0 0 0 0 62 0 0 cryptop 280 0 0 0 0 30 0 0 linux_dma_object 32 0 0 0 0 254 0 0 linux_dma_pctrie 144 0 0 0 0 62 0 0 IOMMU_MAP_ENTRY 104 0 0 0 0 126 0 0 mbuf_jumbo_16k 16384 0 0 0 0 254 0 0 mbuf_jumbo_9k 9216 0 0 0 0 254 0 0 audit_record 1280 0 0 0 0 8 0 0 domainset 40 0 0 0 0 254 0 0 MAC labels 40 0 0 0 0 254 0 0 vnpbuf 2624 0 0 0 0 16 0 0 nfspbuf 2624 0 0 0 0 4 0 0 swwbuf 2624 0 0 0 0 2 0 0 swrbuf 2624 0 0 0 0 4 0 0 umtx_shm 88 0 0 0 0 126 0 0 umtx pi 96 0 0 0 0 126 0 0 rangeset pctrie nodes 144 0 0 0 0 62 0 0 malloc-65536 65536 0 0 0 0 1 0 0 malloc-65536 65536 0 0 0 0 1 0 0 malloc-65536 65536 0 0 0 0 1 0 0 malloc-32768 32768 0 0 0 0 1 0 0 malloc-32768 32768 0 0 0 0 1 0 0 malloc-32768 32768 0 0 0 0 1 0 0 malloc-16384 16384 0 0 0 0 1 0 0 malloc-16384 16384 0 0 0 0 1 0 0 malloc-16384 16384 0 0 0 0 1 0 0 malloc-8192 8192 0 0 0 0 1 0 0 malloc-4096 4096 0 0 0 0 2 0 0 malloc-512 512 0 0 0 0 30 0 0 pcpu-32 32 0 0 0 0 254 0 0 pcpu-4 4 0 0 0 0 254 0 0 fakepg 104 0 0 0 0 126 0 0 UMA Hash 256 0 0 0 0 62 0 0