panic: _mtx_lock_sleep: recursed on non-recursive mutex sctp-create @ /syzkaller/managers/main/kernel/sys/netinet/sctp_pcb.c:3330 cpuid = 0 time = 1625187956 KDB: stack backtrace: db_trace_self_wrapper() at db_trace_self_wrapper+0x47/frame 0xfffffe005164fca0 vpanic() at vpanic+0x1cc/frame 0xfffffe005164fd00 panic() at panic+0x43/frame 0xfffffe005164fd60 __mtx_lock_sleep() at __mtx_lock_sleep+0x68c/frame 0xfffffe005164fdf0 __mtx_lock_flags() at __mtx_lock_flags+0x16d/frame 0xfffffe005164fe50 sctp_inpcb_free() at sctp_inpcb_free+0x2b6/frame 0xfffffe005164fed0 sctp_free_assoc() at sctp_free_assoc+0x288c/frame 0xfffffe005164ff60 sctp_process_cookie_new() at sctp_process_cookie_new+0x3f7/frame 0xfffffe0051650250 sctp_process_control() at sctp_process_control+0x51e2/frame 0xfffffe0051650740 sctp_common_input_processing() at sctp_common_input_processing+0x83a/frame 0xfffffe00516508c0 sctp_input_with_port() at sctp_input_with_port+0x30a/frame 0xfffffe00516509b0 sctp_input() at sctp_input+0x1f/frame 0xfffffe00516509d0 ip_input() at ip_input+0x38b/frame 0xfffffe0051650a90 swi_net() at swi_net+0x20e/frame 0xfffffe0051650b10 ithread_loop() at ithread_loop+0x33f/frame 0xfffffe0051650bb0 fork_exit() at fork_exit+0xb3/frame 0xfffffe0051650bf0 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe0051650bf0 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- KDB: enter: panic [ thread pid 12 tid 100033 ] Stopped at kdb_enter+0x67: movq $0,0x164029e(%rip) db> db> set $lines = 0 db> set $maxwidth = 0 db> show registers cs 0x20 ds 0x3b es 0x3b fs 0x13 gs 0x1b ss 0x28 rax 0x12 rcx 0xffffffff811325d0 vprintf+0x140 rdx 0x1 rbx 0 rsp 0xfffffe005164fc80 rbp 0xfffffe005164fca0 rsi 0 rdi 0xffffffff81132606 vprintf+0x176 r8 0 r9 0x8080808080808080 r10 0xfffffe005164fb70 r11 0x1ffaefff59c r12 0xffffffff82267d50 ddb_dbbe r13 0xffffffff81993800 r14 0xffffffff81a74f90 r15 0xffffffff81a74f90 rip 0xffffffff81129667 kdb_enter+0x67 rflags 0x82 kdb_enter+0x67: movq $0,0x164029e(%rip) db> show proc Process 12 (intr) at 0xfffff80004c74a70: state: NORMAL uid: 0 gids: 0 parent: pid 0 at 0xffffffff8271ce70 ABI: null flag: 0x10000284 flag2: 0 reaper: 0xffffffff8271ce70 reapsubtree: 12 sigparent: 20 vmspace: 0xffffffff8271daf0 (map 0xffffffff8271daf0) (map.pmap 0xffffffff8271dbb0) (pmap 0xffffffff8271dc10) threads: 23 100011 I [swi5: fast taskq] 100014 I [swi6: task queue] 100016 I [swi6: Giant taskq] 100030 I [swi3: vm] 100031 I [swi4: clock (0)] 100032 I [swi4: clock (1)] 100033 Run CPU 0 [swi1: netisr 0] 100046 I [irq24: virtio_pci0] 100047 I [irq25: virtio_pci0] 100048 I [irq26: virtio_pci0] 100049 I [irq27: virtio_pci0] 100050 I [irq28: virtio_pci1] 100051 I [irq29: virtio_pci1] 100052 I [irq30: virtio_pci1] 100053 I [irq31: virtio_pci1] 100054 I [irq32: virtio_pci1] 100059 I [irq10: virtio_pci2] 100061 I [irq1: atkbd0] 100062 I [irq12: psm0] 100063 I [swi0: uart uart++] 100071 I [swi1: pf send] 100084 I [swi1: hpts] 100085 I [swi1: hpts] db> ps pid ppid pgrp uid state wmesg wchan cmd 15018 814 814 0 LE *sctp-cr 0xfffff80027daa0c0 syz-executor.2 15013 867 867 0 T (threaded) syz-executor.3 112004 s syz-executor.3 115831 RunQ syz-executor.3 115836 RunQ syz-executor.3 11640 11639 784 0 SV uwait 0xfffff80027e22100 syz-executor.1 11639 1 784 0 DV ppwait 0xfffff80027f8aa20 syz-executor.1 10673 1 867 0 S uwait 0xfffff80027da8900 syz-executor.3 10672 1 867 0 S uwait 0xfffff80027e00500 syz-executor.3 6603 1 783 0 S uwait 0xfffff8002747d500 syz-executor.0 6602 1 783 0 S uwait 0xfffff80015ea4300 syz-executor.0 6107 1 6107 65 Ss select 0xfffff8002bd40440 dhclient 4946 1 4946 0 Ss select 0xfffff8002bb56a40 dhclient 4943 1 4943 0 Ss select 0xfffff80027f5cd40 dhclient 4924 1 4924 65 Ss select 0xfffff8002bb91bc0 dhclient 4240 1 4240 0 Ss select 0xfffff8002bb1f440 dhclient 4237 1 4237 0 Ss select 0xfffff8002b872bc0 dhclient 4218 1 4218 65 Ss select 0xfffff8002b0015c0 dhclient 3217 1 3217 0 Ss select 0xfffff80015e26440 dhclient 3214 1 3214 0 Ss select 0xfffff8002b8028c0 dhclient 3195 1 3195 65 Ss select 0xfffff80027cab5c0 dhclient 1387 0 0 0 DL aiordy 0xfffff8002b2ac538 [aiod4] 1386 0 0 0 DL aiordy 0xfffff80027d41a70 [aiod3] 1385 0 0 0 DL aiordy 0xfffff80027d41000 [aiod2] 1384 0 0 0 DL aiordy 0xfffff8002b2aca70 [aiod1] 1064 1 1064 0 Ss select 0xfffff80027f54740 dhclient 1061 1 1061 0 Ss select 0xfffff8002b001a40 dhclient 867 780 867 0 Ss wait 0xfffff8002726c538 syz-executor.3 814 780 814 0 Ss wait 0xfffff800273cea70 syz-executor.2 784 780 784 0 Ss piperd 0xfffff80027a4d2e8 syz-executor.1 783 780 783 0 Ss piperd 0xfffff80027c76ba0 syz-executor.0 780 778 778 0 S (threaded) syz-fuzzer 100096 S uwait 0xfffff80015ea4f00 syz-fuzzer 100122 S uwait 0xfffff80015d8a900 syz-fuzzer 100123 S uwait 0xfffff80015d8ab00 syz-fuzzer 100124 S uwait 0xfffff80015d8ad00 syz-fuzzer 100125 S uwait 0xfffff80015d8af00 syz-fuzzer 100126 S uwait 0xfffff80015ac1100 syz-fuzzer 100127 S kqread 0xfffff8002736c100 syz-fuzzer 100128 S uwait 0xfffff8002736c500 syz-fuzzer 100129 S uwait 0xfffff8002736c700 syz-fuzzer 778 776 778 0 Ss pause 0xfffff800273d1b20 csh 776 694 776 0 Ss select 0xfffff80015ea65c0 sshd 760 1 760 0 Ss+ ttyin 0xfffff800154a5cb0 getty 759 1 759 0 Ss+ ttyin 0xfffff80015b2b4b0 getty 758 1 758 0 Ss+ ttyin 0xfffff80015b2bcb0 getty 757 1 757 0 Ss+ ttyin 0xfffff80015b2f4b0 getty 756 1 756 0 Ss+ ttyin 0xfffff80015b2fcb0 getty 755 1 755 0 Ss+ ttyin 0xfffff80015ab64b0 getty 754 1 754 0 Ss+ ttyin 0xfffff80015ab6cb0 getty 753 1 753 0 Ss+ ttyin 0xfffff80015abc4b0 getty 752 1 752 0 Ss+ ttyin 0xfffff80015abccb0 getty 698 1 698 0 Ss nanslp 0xffffffff8273d0a1 cron 694 1 694 0 Ss select 0xfffff80015ea68c0 sshd 507 1 507 0 Ss select 0xfffff80015dfcbc0 syslogd 436 1 436 0 Ss select 0xfffff80015e795c0 devd 435 1 435 65 Ss select 0xfffff80015f6e140 dhclient 350 1 350 0 Ss select 0xfffff80015e4c140 dhclient 347 1 347 0 Ss select 0xfffff80015f6e8c0 dhclient 23 0 0 0 DL vlruwt 0xfffff80015b12000 [vnlru] 22 0 0 0 DL syncer 0xffffffff8282c450 [syncer] 21 0 0 0 DL (threaded) [bufdaemon] 100081 D qsleep 0xffffffff8282b520 [bufdaemon] 100086 D - 0xffffffff8220aec0 [bufspacedaemon-0] 100098 D sdflush 0xfffff80004ddc4e8 [/ worker] 20 0 0 0 DL psleep 0xffffffff82853388 [vmdaemon] 19 0 0 0 RL (threaded) [pagedaemon] 100079 Run CPU 1 [dom0] 100087 D launds 0xffffffff82847804 [laundry: dom0] 100088 D umarecl 0xffffffff82846c50 [uma] 18 0 0 0 DL - 0xffffffff82571458 [rand_harvestq] 17 0 0 0 DL waiting 0xffffffff830188b8 [sctp_iterator] 16 0 0 0 DL pftm 0xffffffff82e0f410 [pf purge] 15 0 0 0 DL - 0xffffffff82828bdc [soaiod4] 9 0 0 0 DL - 0xffffffff82828bdc [soaiod3] 8 0 0 0 DL - 0xffffffff82828bdc [soaiod2] 7 0 0 0 DL - 0xffffffff82828bdc [soaiod1] 6 0 0 0 DL (threaded) [cam] 100044 D - 0xffffffff82448b40 [doneq0] 100045 D - 0xffffffff82448ac0 [async] 100078 D - 0xffffffff82448990 [scanner] 14 0 0 0 DL seqstat 0xfffff80004d8a488 [sequencer 00] 5 0 0 0 DL crypto_ 0xfffff80004d53d80 [crypto returns 1] 4 0 0 0 DL crypto_ 0xfffff80004d53d30 [crypto returns 0] 3 0 0 0 DL crypto_ 0xffffffff82844ce0 [crypto] 13 0 0 0 DL (threaded) [geom] 100035 D - 0xffffffff8271c8e0 [g_event] 100036 D - 0xffffffff8271c8e8 [g_up] 100037 D - 0xffffffff8271c8f0 [g_down] 2 0 0 0 DL (threaded) [KTLS] 100028 D - 0xfffff80004c69100 [thr_0] 100029 D - 0xfffff80004c69180 [thr_1] 12 0 0 0 RL (threaded) [intr] 100011 I [swi5: fast taskq] 100014 I [swi6: task queue] 100016 I [swi6: Giant taskq] 100030 I [swi3: vm] 100031 I [swi4: clock (0)] 100032 I [swi4: clock (1)] 100033 Run CPU 0 [swi1: netisr 0] 100046 I [irq24: virtio_pci0] 100047 I [irq25: virtio_pci0] 100048 I [irq26: virtio_pci0] 100049 I [irq27: virtio_pci0] 100050 I [irq28: virtio_pci1] 100051 I [irq29: virtio_pci1] 100052 I [irq30: virtio_pci1] 100053 I [irq31: virtio_pci1] 100054 I [irq32: virtio_pci1] 100059 I [irq10: virtio_pci2] 100061 I [irq1: atkbd0] 100062 I [irq12: psm0] 100063 I [swi0: uart uart++] 100071 I [swi1: pf send] 100084 I [swi1: hpts] 100085 I [swi1: hpts] 11 0 0 0 RL (threaded) [idle] 100003 CanRun [idle: cpu0] 100004 CanRun [idle: cpu1] 1 0 1 0 SLs wait 0xfffff80004bda538 [init] 10 0 0 0 DL audit_w 0xffffffff828451f0 [audit] 0 0 0 0 DLs (threaded) [kernel] 100000 D swapin 0xffffffff8271ce70 [swapper] 100005 D - 0xfffff8000443b500 [if_io_tqg_0] 100006 D - 0xfffff8000443b100 [if_io_tqg_1] 100007 D - 0xfffff80004c78d00 [if_config_tqg_0] 100008 D - 0xfffff80004c78900 [softirq_0] 100009 D - 0xfffff80004c78500 [softirq_1] 100010 D - 0xfffff80004c78100 [aiod_kick taskq] 100012 D - 0xfffff80004c75900 [kqueue_ctx taskq] 100013 D - 0xfffff80004c75500 [pci_hp taskq] 100015 D - 0xfffff80004c6ed00 [inm_free taskq] 100017 D - 0xfffff80004c6e500 [linuxkpi_irq_wq] 100018 D - 0xfffff80004c6e100 [thread taskq] 100019 D - 0xfffff80004c69d00 [in6m_free taskq] 100020 D - 0xfffff80004c69900 [linuxkpi_short_wq_0] 100021 D - 0xfffff80004c69900 [linuxkpi_short_wq_1] 100022 D - 0xfffff80004c69900 [linuxkpi_short_wq_2] 100023 D - 0xfffff80004c69900 [linuxkpi_short_wq_3] 100024 D - 0xfffff80004c69500 [linuxkpi_long_wq_0] 100025 D - 0xfffff80004c69500 [linuxkpi_long_wq_1] 100026 D - 0xfffff80004c69500 [linuxkpi_long_wq_2] 100027 D - 0xfffff80004c69500 [linuxkpi_long_wq_3] 100034 D - 0xfffff80004cd3d00 [firmware taskq] 100038 D - 0xfffff80004cd3900 [crypto_0] 100039 D - 0xfffff80004cd3900 [crypto_1] 100055 D - 0xfffff800153ebd00 [vtnet0 rxq 0] 100056 D - 0xfffff800153eb900 [vtnet0 txq 0] 100057 D - 0xfffff800153eb500 [vtnet0 rxq 1] 100058 D - 0xfffff800153eb100 [vtnet0 txq 1] 100060 D vtbslp 0xfffff80015463700 [virtio_balloon] 100064 D - 0xfffff80015468900 [mca taskq] 100066 D - 0xffffffff81e218b0 [deadlkres] 100074 D - 0xfffff80015a47100 [acpi_task_0] 100075 D - 0xfffff80015a47100 [acpi_task_1] 100076 D - 0xfffff80015a47100 [acpi_task_2] 100077 D - 0xfffff80004cd3500 [CAM taskq] db> show all locks Process 19 (pagedaemon) thread 0xfffffe0055375c80 (100079) exclusive sleep mutex vm reserv (vm reserv) r = 0 (0xfffff8007c7f29a0) locked @ /syzkaller/managers/main/kernel/sys/vm/vm_reserv.c:1045 exclusive rw kernel vm object (kernel vm object) r = 0 (0xffffffff82847030) locked @ /syzkaller/managers/main/kernel/sys/vm/vm_kern.c:586 shared rw UMA lock (UMA lock) r = 0 (0xffffffff82244500) locked @ /syzkaller/managers/main/kernel/sys/vm/uma_core.c:3019 shared sx umareclaim (umareclaim) r = 0 (0xffffffff82846c50) locked @ /syzkaller/managers/main/kernel/sys/vm/uma_core.c:5132 Process 12 (intr) thread 0xfffffe00042be900 (100033) exclusive sleep mutex sctp-create (inp_create) r = 0 (0xfffff8002b01f488) locked @ /syzkaller/managers/main/kernel/sys/netinet/sctp_input.c:4922 db> show malloc Type InUse MemUse Requests sysctloid 34288 12858K 34355 pf_hash 5 11560K 5 devbuf 4216 6982K 4244 tcp_hpts 5 3219K 5 kobj 326 2608K 486 vtbuf 24 2064K 46 pcb 201 1389K 15898 newblk 11 1037K 15122 vfscache 3 1035K 3 inodedep 415 934K 14161 subproc 159 656K 15110 acpica 1674 649K 54806 callout 2 528K 2 ufs_quota 1 520K 1 vfs_hash 1 520K 1 intr 4 480K 4 filedesc 26 405K 28077 bus 986 377K 3469 BPF 46 337K 56 linker 348 270K 409 DEVFS1 107 214K 124 dirrem 404 202K 14076 kdtrace 237 193K 30866 module 510 192K 510 vnet_data 1 176K 1 sctp_atcl 174 174K 9424 sctp_stro 85 170K 2451 tidhash 3 164K 3 umtx 374 164K 374 freefile 404 152K 14069 pagedep 11 141K 14044 tfo_ccache 1 136K 1 sem 4 120K 4 UMA 270 102K 270 sctp_atky 259 98K 12991 audit_evclass 236 89K 294 temp 36 81K 5755 mtx_pool 2 80K 2 syncache 1 76K 1 routetbl 135 75K 8792 vmem 3 72K 5 msg 4 68K 4 ifaddr 70 66K 72 sctp_athm 174 66K 10504 acpitask 1 64K 1 ddb_capture 1 64K 1 sctp_map 170 64K 4902 DEVFS3 126 63K 136 ether_multi 152 57K 162 gtaskqueue 18 57K 18 pf_ifnet 102 55K 1856 DEVFS_RULE 56 54K 56 kenv 95 52K 95 eventhandler 133 50K 133 ithread 99 43K 99 rman 84 42K 425 lltable 46 40K 92 KTRACE 100 38K 100 taskqueue 60 36K 60 kqueue 73 36K 15034 sctp_timw 69 35K 69 proc 3 34K 3 ufs_mount 5 34K 6 bus-sc 33 34K 1684 devstat 4 33K 4 hostcache 1 32K 1 tty 16 32K 16 shm 1 32K 3 DEVFSP 84 32K 7507 GEOM 60 29K 489 in6_multi 65 29K 65 cred 28 28K 270 kbdmux 6 28K 6 pwddesc 70 27K 15022 ifnet 7 25K 7 plimit 24 24K 469 kcovinfo 64 24K 68 sctp_aadr 57 22K 473 CAM queue 5 21K 1528 selfd 55 21K 213815 pfs_nodes 20 20K 20 proc-args 52 20K 686 UART 12 18K 12 session 35 18K 89 bmsafemap 2 17K 14142 ksem 1 16K 458 rpc 2 16K 2 shmfd 1 16K 20 pfs_vncache 1 16K 1 sglist 5 13K 5 ufs_dirhash 20 13K 24 MCA 32 12K 32 CAM DEV 3 12K 510 lockf 32 12K 57 vt 11 11K 11 acpisem 28 11K 28 CAM XPT 22 11K 543 Unitno 27 11K 47 uidinfo 3 9K 10 pf_table 2 8K 2097 memdesc 1 8K 1 ipsec-saq 2 8K 2 evdev 4 8K 4 inpcbpolicy 20 8K 7278 acpidev 20 8K 20 hhook 15 8K 17 select 19 8K 70 mount 16 7K 90 fpukern_ctx 3 6K 3 terminal 11 6K 11 ip6ndp 12 6K 13 sctp_ifa 13 5K 14 osd 12 5K 6367 ipsecpolicy 2 5K 2 encap_export_host 12 5K 12 clone 9 5K 9 local_apic 1 4K 1 io_apic 1 4K 1 sahead 1 4K 1 secasvar 1 4K 1 nhops 6 4K 8 pci_link 10 4K 10 crypto 5 4K 3757 msi 9 4K 9 DEVFS 9 4K 10 tun 7 4K 7 sctp_stri 3 3K 2162 ipsec 3 3K 3 mkdir 8 3K 28044 xform 8 3K 11182 in_multi 6 3K 8 nexusdev 7 3K 7 feeder 7 3K 7 procdesc 5 3K 18 freework 5 3K 14370 sctp_ifn 6 3K 14 newdirblk 6 3K 14022 mld 6 3K 6 igmp 6 3K 6 toponodes 6 3K 6 prison 6 3K 6 isadev 6 3K 6 softdep 1 2K 1 freeblks 4 2K 14081 vnodemarker 2 2K 40 NFSD session 1 2K 1 lkpikmalloc 5 2K 6 CAM periph 4 2K 271 filecaps 5 2K 124 diradd 4 2K 14117 pfil 4 2K 4 CAM path 4 2K 1034 soname 4 2K 19819 tcpfunc 3 2K 3 loginclass 3 2K 6 chacha20random 1 1K 1 vnodes 1 1K 1 ktls 1 1K 1 cdev 2 1K 2 aesni_data 2 1K 2 pf_rule 2 1K 1996 pf_osfp 2 1K 2 cpus 2 1K 2 atkbddev 2 1K 2 CAM dev queue 2 1K 2 entropy 2 1K 45 CAM SIM 2 1K 2 NFSD lckfile 1 1K 1 NFSD V4client 1 1K 1 pmchooks 1 1K 1 sctp_vrf 1 1K 1 apmdev 1 1K 1 CAM I/O Scheduler 1 1K 1 vnet_data_free 1 1K 1 vnet 1 1K 1 Per-cpu 1 1K 1 p1003.1b 1 1K 1 acpiintr 1 1K 1 pmc 1 1K 1 cubic data 0 0K 0 chd data 0 0K 0 tcp_do 0 0K 0 tcp_fsb 0 0K 3977 cdg data 0 0K 0 sctp_mcore 0 0K 0 sctp_socko 0 0K 3860 sctp_iter 0 0K 107 sctp_mvrf 0 0K 0 sctp_cpal 0 0K 98 sctp_cmsg 0 0K 0 sctp_stre 0 0K 0 sctp_athi 0 0K 0 sctp_a_it 0 0K 9 dctcp data 0 0K 0 mqdata 0 0K 0 pf_altq 0 0K 0 pf_temp 0 0K 0 htcp data 0 0K 0 vegas data 0 0K 0 NFSCL lckown 0 0K 0 NFSCL client 0 0K 0 madt_table 0 0K 2 smartpqi 0 0K 0 NFSCL deleg 0 0K 0 NFSCL open 0 0K 0 NFSCL owner 0 0K 0 NFS fh 0 0K 0 NFS req 0 0K 0 NFSD usrgroup 0 0K 0 NFSD string 0 0K 0 iavf 0 0K 0 ixl 0 0K 0 NFSD V4lock 0 0K 0 NFSD V4state 0 0K 0 NFSD srvcache 0 0K 0 msdosfs_fat 0 0K 0 msdosfs_mount 0 0K 0 msdosfs_node 0 0K 0 ice-resmgr 0 0K 0 ice-osdep 0 0K 0 ice 0 0K 0 axgbe 0 0K 0 DEVFS4 0 0K 0 DEVFS2 0 0K 0 gntdev 0 0K 0 privcmd_dev 0 0K 0 evtchn_dev 0 0K 0 xen_intr 0 0K 0 xenstore 0 0K 0 ciss_data 0 0K 0 xen_hvm 0 0K 0 legacydrv 0 0K 0 qpidrv 0 0K 0 BACKLIGHT 0 0K 0 xnb 0 0K 0 dmar_idpgtbl 0 0K 0 dmar_dom 0 0K 0 dmar_ctx 0 0K 0 xbbd 0 0K 0 xbd 0 0K 0 Balloon 0 0K 0 isci 0 0K 0 iommu_dmamap 0 0K 0 sysmouse 0 0K 0 hyperv_socket 0 0K 0 bxe_ilt 0 0K 0 xenbus 0 0K 0 vtfont 0 0K 0 vm_fictitious 0 0K 0 ath_hal 0 0K 0 athdev 0 0K 0 ata_pci 0 0K 0 ata_dma 0 0K 0 ata_generic 0 0K 0 amr 0 0K 0 pvscsi 0 0K 0 scsi_da 0 0K 69 ata_da 0 0K 0 scsi_ch 0 0K 0 UMAHash 0 0K 0 scsi_cd 0 0K 0 AHCI driver 0 0K 0 vm_pgdata 0 0K 0 jblocks 0 0K 0 savedino 0 0K 11544 sentinel 0 0K 0 jfsync 0 0K 0 jtrunc 0 0K 0 sbdep 0 0K 17 jsegdep 0 0K 0 jseg 0 0K 0 jfreefrag 0 0K 0 jfreeblk 0 0K 0 jnewblk 0 0K 0 jmvref 0 0K 0 jremref 0 0K 0 jaddref 0 0K 0 freedep 0 0K 0 freefrag 0 0K 6 allocindir 0 0K 0 indirdep 0 0K 34 allocdirect 0 0K 0 ufs_trim 0 0K 0 mactemp 0 0K 0 audit_trigger 0 0K 0 audit_pipe_presel 0 0K 0 audit_pipeent 0 0K 0 audit_pipe 0 0K 0 audit_evname 0 0K 0 audit_bsm 0 0K 0 audit_gidset 0 0K 0 audit_text 0 0K 0 audit_path 0 0K 0 audit_data 0 0K 0 audit_cred 0 0K 0 USBdev 0 0K 0 USB 0 0K 0 agp 0 0K 0 nvme_da 0 0K 0 acpipwr 0 0K 0 acpi_perf 0 0K 0 ktls_ocf 0 0K 0 twsbuf 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5E_TLS 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EN 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5DUMP 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 seq_file 0 0K 0 radix 0 0K 0 idr 0 0K 0 lkpifw 0 0K 0 NLM 0 0K 0 ipsec-spdcache 0 0K 0 ipsec-reg 0 0K 0 ipsec-misc 0 0K 0 ipsecrequest 0 0K 0 ip6opt 0 0K 26 ip6_msource 0 0K 0 ip6_moptions 0 0K 0 in6_mfilter 0 0K 0 frag6 0 0K 0 tcplog 0 0K 0 tcp_hwpace 0 0K 0 twe_commands 0 0K 0 LRO 0 0K 0 newreno data 0 0K 2 ip_msource 0 0K 0 ip_moptions 0 0K 7 in_mfilter 0 0K 0 ipid 0 0K 0 80211scan 0 0K 0 80211ratectl 0 0K 0 80211power 0 0K 0 80211nodeie 0 0K 0 80211node 0 0K 0 80211mesh_gt 0 0K 0 80211mesh_rt 0 0K 0 80211perr 0 0K 0 80211prep 0 0K 0 80211preq 0 0K 0 80211dfs 0 0K 0 80211crypto 0 0K 0 80211vap 0 0K 0 iflib 0 0K 0 vlan 0 0K 0 gif 0 0K 0 ifdescr 0 0K 0 zlib 0 0K 0 fadvise 0 0K 2 VN POLL 0 0K 0 twa_commands 0 0K 0 statfs 0 0K 14215 namei_tracker 0 0K 0 export_host 0 0K 0 cl_savebuf 0 0K 28 tcp_log_dev 0 0K 1714 midi buffers 0 0K 0 mixer 0 0K 0 ac97 0 0K 0 hdacc 0 0K 0 hdac 0 0K 0 hdaa 0 0K 0 acpicmbat 0 0K 0 SIIS driver 0 0K 0 CAM CCB 0 0K 2428 PUC 0 0K 0 ppbusdev 0 0K 0 agtiapi_MemAlloc malloc 0 0K 0 osti_cacheable 0 0K 0 tempbuff 0 0K 0 biobuf 0 0K 0 aios 0 0K 4 lio 0 0K 2 acl 0 0K 0 tempbuff 0 0K 0 mbuf_tag 0 0K 120 ag_tgt_map_t malloc 0 0K 0 ag_slr_map_t malloc 0 0K 0 lDevFlags * malloc 0 0K 0 tiDeviceHandle_t * malloc 0 0K 0 ag_portal_data_t malloc 0 0K 0 ag_device_t malloc 0 0K 0 STLock malloc 0 0K 0 CCB List 0 0K 0 sr_iov 0 0K 0 OCS 0 0K 0 OCS 0 0K 0 nvme 0 0K 0 nvd 0 0K 0 netmap 0 0K 0 mwldev 0 0K 0 MVS driver 0 0K 0 CAM ccb queue 0 0K 0 mrsasbuf 0 0K 0 mpt_user 0 0K 0 mps_user 0 0K 0 accf 0 0K 0 pts 0 0K 0 iov 0 0K 20637 ioctlops 0 0K 8698 eventfd 0 0K 42 Witness 0 0K 0 stack 0 0K 0 MPSSAS 0 0K 0 mps 0 0K 0 mpr_user 0 0K 0 MPRSAS 0 0K 0 mpr 0 0K 0 mfibuf 0 0K 0 sbuf 0 0K 288 md_sectors 0 0K 0 firmware 0 0K 0 compressor 0 0K 0 md_disk 0 0K 0 SWAP 0 0K 0 malodev 0 0K 0 LED 0 0K 0 sysctltmp 0 0K 831 sysctl 0 0K 3 ekcd 0 0K 0 dumper 0 0K 0 sendfile 0 0K 0 rctl 0 0K 0 ix_sriov 0 0K 0 aacraidcam 0 0K 0 aacraid_buf 0 0K 0 ix 0 0K 0 ipsbuf 0 0K 0 cache 0 0K 0 iirbuf 0 0K 0 prison_racct 0 0K 0 Fail Points 0 0K 0 sigio 0 0K 9 filedesc_to_leader 0 0K 0 pwd 0 0K 0 tty console 0 0K 0 aaccam 0 0K 0 aacbuf 0 0K 0 zstd 0 0K 0 XZ_DEC 0 0K 0 nvlist 0 0K 0 SCSI ENC 0 0K 0 SCSI sa 0 0K 0 scsi_pass 0 0K 0 isofs_node 0 0K 0 isofs_mount 0 0K 0 tr_raid5_data 0 0K 0 tr_raid1e_data 0 0K 0 tr_raid1_data 0 0K 0 tr_raid0_data 0 0K 0 tr_concat_data 0 0K 0 md_sii_data 0 0K 0 md_promise_data 0 0K 0 md_nvidia_data 0 0K 0 md_jmicron_data 0 0K 0 md_intel_data 0 0K 0 md_ddf_data 0 0K 0 raid_data 0 0K 72 geom_flashmap 0 0K 0 tmpfs dir 0 0K 0 tmpfs name 0 0K 0 tmpfs mount 0 0K 0 NFS FHA 0 0K 0 newnfsmnt 0 0K 0 newnfsclient_req 0 0K 0 NFSCL layrecall 0 0K 0 NFSCL session 0 0K 0 NFSCL sockreq 0 0K 0 NFSCL devinfo 0 0K 0 NFSCL flayout 0 0K 0 NFSCL layout 0 0K 0 NFSD rollback 0 0K 0 NFSCL diroff 0 0K 0 NEWdirectio 0 0K 0 NEWNFSnode 0 0K 0 NFSCL lck 0 0K 0 db> show uma Zone Size Used Free Requests Sleeps Bucket Total Mem XFree mbuf_jumbo_page 4096 8324 720 1165378 0 253 37044224 0 malloc-384 384 35667 53 43968 0 30 13716480 0 RADIX NODE 144 32500 61 453249 0 62 4688784 0 malloc-1024 1024 4174 6 4235 0 16 4280320 0 malloc-1024 1024 495 2157 15586 0 16 2715648 0 malloc-8192 8192 328 3 1099 0 1 2711552 0 mbuf 256 8925 1650 1946835 0 254 2707200 0 tcp_log 416 950 5188 114139 0 253 2553408 0 pbuf 2624 0 973 0 0 2 2553152 0 malloc-1024 1024 135 2173 13130 0 16 2363392 0 BUF TRIE 144 194 13274 1146 0 62 1939392 0 UMA Slabs 0 112 13472 313 13760 0 126 1543920 0 malloc-512 512 546 2166 18848 0 30 1388544 0 mbuf_cluster 2048 585 1 585 0 253 1200128 0 malloc-384 384 949 2171 76229 0 30 1198080 0 malloc-8192 8192 88 10 5896 0 1 802816 0 malloc-32768 32768 15 6 14039 0 1 688128 0 malloc-384 384 1723 47 54702 0 30 679680 0 malloc-384 384 1579 111 261110 0 30 648960 0 malloc-8192 8192 73 6 15024 0 1 647168 0 FFS inode 1160 520 26 14590 0 8 633360 0 malloc-16384 16384 24 0 25 0 1 393216 0 VM OBJECT 264 1350 75 213974 0 30 376200 0 lkpimm 160 1 2324 1 0 62 372000 0 lkpicurr 160 2 2323 2 0 62 372000 0 malloc-512 512 33 639 16101 0 30 344064 0 THREAD 1808 165 22 15843 0 8 338096 0 256 Bucket 2048 110 52 17666 0 8 331776 0 malloc-1024 1024 265 39 26042 0 16 311296 0 VNODE 448 559 80 14631 0 30 286272 0 malloc-65536 65536 4 0 4 0 1 262144 0 malloc-2048 2048 125 3 147 0 8 262144 0 malloc-384 384 617 63 9151 0 30 261120 0 sctp_asoc 2288 85 25 2451 0 253 251680 0 malloc-512 512 113 359 2382 0 30 241664 0 malloc-1024 1024 182 42 11607 0 16 229376 0 tcp_inpcb 488 8 448 6355 0 253 222528 0 malloc-2048 2048 87 21 3681 0 8 221184 0 MAP ENTRY 96 1857 369 808805 0 126 213696 0 malloc-8192 8192 23 2 55 0 1 204800 0 malloc-65536 65536 3 0 3 0 1 196608 0 malloc-16384 16384 11 1 247 0 1 196608 0 socket 944 49 151 14539 0 253 188800 0 UMA Zones 768 242 2 242 0 16 187392 0 malloc-4096 4096 9 34 5486 0 2 176128 0 malloc-384 384 364 76 63604 0 30 168960 0 sctp_ep 1280 86 40 5891 0 253 161280 0 vmem btag 56 2790 78 2790 0 254 160608 0 tcp_bbr_map 128 0 1240 16920 0 126 158720 0 ertt_txseginfo 40 0 3838 70036 0 254 153520 0 S VFS Cache 104 1067 376 16860 0 126 150072 0 mbuf_packet 256 68 517 23713 0 254 149760 0 malloc-512 512 196 76 30880 0 30 139264 0 FFS2 dinode 256 520 20 14589 0 62 138240 0 malloc-384 384 295 65 27402 0 30 138240 0 VMSPACE 2544 43 11 14994 0 4 137376 0 malloc-65536 65536 2 0 2 0 1 131072 0 malloc-65536 65536 0 2 176 0 1 131072 0 tcpcb 1080 8 111 6355 0 253 128520 0 malloc-512 512 207 33 14306 0 30 122880 0 PROC 1336 70 17 15021 0 8 116232 0 malloc-16384 16384 2 5 34 0 1 114688 0 malloc-1024 1024 100 4 105 0 16 106496 0 DEVCTL 1024 0 100 153 0 0 102400 0 malloc-32768 32768 1 2 121 0 1 98304 0 malloc-16384 16384 4 2 6 0 1 98304 0 pcpu-8 8 6169 6119 49741 0 254 98304 0 filedesc0 1072 71 20 15022 0 8 97552 0 UMA Kegs 384 227 6 227 0 30 89472 0 sctp_raddr 736 100 21 2468 0 253 89056 0 32 Bucket 256 245 100 19631 0 62 88320 0 clpbuf 2624 0 32 45 0 15 83968 0 malloc-16384 16384 3 2 65 0 1 81920 0 128 Bucket 1024 39 40 1457 0 16 80896 0 malloc-512 512 119 33 14482 0 30 77824 0 malloc-4096 4096 13 5 3189 0 2 73728 0 malloc-2048 2048 17 17 602 0 8 69632 0 malloc-65536 65536 0 1 1 0 1 65536 0 malloc-32768 32768 1 1 16 0 1 65536 0 g_bio 408 0 160 7087 0 30 65280 0 malloc-2048 2048 10 20 518 0 8 61440 0 malloc-384 384 96 64 21852 0 30 61440 0 16 Bucket 144 332 60 4295 0 62 56448 0 malloc-4096 4096 7 6 92 0 2 53248 0 malloc-16384 16384 2 1 6 0 1 49152 0 malloc-16384 16384 3 0 3 0 1 49152 0 malloc-512 512 83 13 525 0 30 49152 0 malloc-8192 8192 2 3 14546 0 1 40960 0 64 Bucket 512 40 40 17307 0 30 40960 0 DIRHASH 1024 30 6 34 0 16 36864 0 NAMEI 1024 0 36 84894 0 16 36864 0 malloc-512 512 34 38 2046 0 30 36864 0 Files 80 255 195 49609 0 126 36000 0 pipe 744 25 20 725 0 16 33480 0 malloc-32768 32768 1 0 1 0 1 32768 0 malloc-32768 32768 1 0 1 0 1 32768 0 malloc-32768 32768 1 0 1 0 1 32768 0 malloc-8192 8192 3 1 7 0 1 32768 0 malloc-4096 4096 6 2 12 0 2 32768 0 malloc-2048 2048 2 14 151 0 8 32768 0 pcpu-64 64 487 25 487 0 254 32768 0 sctp_chunk 152 141 67 2413 0 253 31616 0 tcp_bbr_pcb 832 0 36 1391 0 16 29952 0 tcp_rack_pcb 832 0 36 3977 0 16 29952 0 TURNSTILE 136 188 22 188 0 62 28560 0 ksiginfo 112 72 180 1368 0 126 28224 0 malloc-8192 8192 0 3 13 0 1 24576 0 malloc-8192 8192 1 2 52 0 1 24576 0 malloc-4096 4096 1 5 10 0 2 24576 0 PWD 32 22 734 14118 0 254 24192 0 ttyinq 160 135 15 300 0 62 24000 0 8 Bucket 80 47 253 3168 0 126 24000 0 ttyoutq 256 72 18 160 0 62 23040 0 malloc-4096 4096 3 2 500 0 2 20480 0 malloc-4096 4096 2 3 7644 0 2 20480 0 2 Bucket 32 97 533 3763 0 254 20160 0 KNOTE 160 28 97 221704 0 62 20000 0 SLEEPQUEUE 88 188 36 188 0 126 19712 0 Mountpoints 2752 2 5 2 0 4 19264 0 malloc-16384 16384 1 0 1 0 1 16384 0 sctp_laddr 48 220 116 2714 0 254 16128 0 vtnet_tx_hdr 24 0 668 594143 0 254 16032 0 ripcb 488 5 27 178 0 253 15616 0 udplite_inpcb 488 1 31 423 0 253 15616 0 udp_inpcb 488 6 26 322 0 253 15616 0 malloc-2048 2048 3 3 163 0 8 12288 0 malloc-2048 2048 2 4 6 0 8 12288 0 tcp_rack_map 112 0 108 4894 0 126 12096 0 ertt 72 8 160 6355 0 126 12096 0 kenv 258 15 30 1042 0 30 11610 0 unpcb 256 20 25 1327 0 253 11520 0 malloc-2048 2048 4 0 4 0 8 8192 0 malloc-1024 1024 1 7 14 0 16 8192 0 malloc-1024 1024 0 8 33 0 16 8192 0 rtentry 176 30 16 34 0 62 8096 0 PGRP 88 35 57 89 0 126 8096 0 rl_entry 40 72 130 72 0 254 8080 0 sctp_stream_msg_out 112 3 69 262 0 253 8064 0 udpcb 32 7 245 745 0 254 8064 0 4 Bucket 48 5 163 341 0 254 8064 0 UMA Slabs 1 176 23 22 23 0 62 7920 0 sctp_readq 152 1 51 149 0 253 7904 0 AIO 208 0 38 20 0 62 7904 0 cryptop 280 0 28 65 0 30 7840 0 AIOCB 552 0 14 48 0 16 7728 0 routing nhops 256 27 3 34 0 62 7680 0 FPU_save_area 832 1 8 1 0 16 7488 0 cpuset 104 7 55 7 0 126 6448 0 epoch_record pcpu 256 4 12 4 0 62 4096 0 malloc-4096 4096 1 0 1 0 2 4096 0 pcpu-16 16 7 249 7 0 254 4096 0 hostcache 64 1 62 1 0 254 4032 0 syncache 168 0 24 4 0 253 4032 0 AIOP 32 4 122 4 0 254 4032 0 AIOLIO 272 0 14 2 0 30 3808 0 mqnode 416 3 6 3 0 30 3744 0 KMAP ENTRY 96 12 27 12 0 0 3744 0 vmem 1856 1 1 1 0 8 3712 0 SMR CPU 32 3 60 3 0 254 2016 0 SMR SHARED 24 3 60 3 0 254 1512 0 FFS1 dinode 128 0 0 0 0 126 0 0 da_ccb 544 0 0 0 0 16 0 0 ada_ccb 272 0 0 0 0 30 0 0 swblk 136 0 0 0 0 62 0 0 swpctrie 144 0 0 0 0 62 0 0 cdg_qdiffsample 16 0 0 0 0 254 0 0 sctp_asconf_ack 48 0 0 0 0 254 0 0 sctp_asconf 40 0 0 0 0 254 0 0 pf state scrubs 40 0 0 0 0 254 0 0 pf frag entries 40 0 0 0 0 254 0 0 pf frags 248 0 0 0 0 62 0 0 pf table entries 160 0 0 0 0 62 0 0 pf table entry counters 64 0 0 0 0 254 0 0 pf source nodes 136 0 0 0 0 253 0 0 pf state keys 88 0 0 0 0 126 0 0 pf states 304 0 0 0 0 253 0 0 pf tags 104 0 0 0 0 126 0 0 pf mtags 48 0 0 0 0 254 0 0 tcp_log_node 120 0 0 0 0 126 0 0 tcp_log_bucket 176 0 0 0 0 62 0 0 tcpreass 48 0 0 0 0 254 0 0 tfo_ccache_entries 80 0 0 0 0 126 0 0 tfo 4 0 0 0 0 254 0 0 sackhole 32 0 0 0 0 254 0 0 tcptw 88 0 0 0 0 253 0 0 ipq 56 0 0 0 0 254 0 0 IPsec SA lft_c 16 0 0 0 0 254 0 0 itimer 352 0 0 0 0 30 0 0 NCLNODE 584 0 0 0 0 16 0 0 TMPFS node 224 0 0 0 0 62 0 0 mqnotifier 216 0 0 0 0 62 0 0 mvdata 64 0 0 0 0 254 0 0 mqueue 248 0 0 0 0 62 0 0 LTS VFS Cache 360 0 0 0 0 30 0 0 L VFS Cache 320 0 0 0 0 30 0 0 STS VFS Cache 144 0 0 0 0 62 0 0 linux_dma_object 24 0 0 0 0 254 0 0 linux_dma_pctrie 144 0 0 0 0 62 0 0 IOMMU_MAP_ENTRY 120 0 0 0 0 126 0 0 ktls_session 128 0 0 0 0 126 0 0 mbuf_jumbo_16k 16384 0 0 0 0 253 0 0 mbuf_jumbo_9k 9216 0 0 0 0 253 0 0 audit_record 1280 0 0 0 0 8 0 0 domainset 40 0 0 0 0 254 0 0 MAC labels 40 0 0 0 0 254 0 0 vnpbuf 2624 0 0 0 0 63 0 0 mdpbuf 2624 0 0 0 0 3 0 0 nfspbuf 2624 0 0 0 0 15 0 0 swwbuf 2624 0 0 0 0 7 0 0 swrbuf 2624 0 0 0 0 15 0 0 umtx_shm 88 0 0 0 0 126 0 0 umtx pi 96 0 0 0 0 126 0 0 rangeset pctrie nodes 144 0 0 0 0 62 0 0 malloc-65536 65536 0 0 0 0 1 0 0 malloc-65536 65536 0 0 0 0 1 0 0 malloc-65536 65536 0 0 0 0 1 0 0 malloc-32768 32768 0 0 0 0 1 0 0 malloc-32768 32768 0 0 0 0 1 0 0 malloc-256 256 0 0 0 0 62 0 0 malloc-256 256 0 0 0 0 62 0 0 malloc-256 256 0 0 0 0 62 0 0 malloc-256 256 0 0 0 0 62 0 0 malloc-256 256 0 0 0 0 62 0 0 malloc-256 256 0 0 0 0 62 0 0 malloc-256 256 0 0 0 0 62 0 0 malloc-256 256 0 0 0 0 62 0 0 malloc-128 128 0 0 0 0 126 0 0 malloc-128 128 0 0 0 0 126 0 0 malloc-128 128 0 0 0 0 126 0 0 malloc-128 128 0 0 0 0 126 0 0 malloc-128 128 0 0 0 0 126 0 0 malloc-128 128 0 0 0 0 126 0 0 malloc-128 128 0 0 0 0 126 0 0 malloc-128 128 0 0 0 0 126 0 0 malloc-64 64 0 0 0 0 254 0 0 malloc-64 64 0 0 0 0 254 0 0 malloc-64 64 0 0 0 0 254 0 0 malloc-64 64 0 0 0 0 254 0 0 malloc-64 64 0 0 0 0 254 0 0 malloc-64 64 0 0 0 0 254 0 0 malloc-64 64 0 0 0 0 254 0 0 malloc-64 64 0 0 0 0 254 0 0 malloc-32 32 0 0 0 0 254 0 0 malloc-32 32 0 0 0 0 254 0 0 malloc-32 32 0 0 0 0 254 0 0 malloc-32 32 0 0 0 0 254 0 0 malloc-32 32 0 0 0 0 254 0 0 malloc-32 32 0 0 0 0 254 0 0 malloc-32 32 0 0 0 0 254 0 0 malloc-32 32 0 0 0 0 254 0 0 malloc-16 16 0 0 0 0 254 0 0 malloc-16 16 0 0 0 0 254 0 0 malloc-16 16 0 0 0 0 254 0 0 malloc-16 16 0 0 0 0 254 0 0 malloc-16 16 0 0 0 0 254 0 0 malloc-16 16 0 0 0 0 254 0 0 malloc-16 16 0 0 0 0 254 0 0 malloc-16 16 0 0 0 0 254 0 0 pcpu-32 32 0 0 0 0 254 0 0 pcpu-4 4 0 0 0 0 254 0 0 fakepg 104 0 0 0 0 126 0 0 UMA Hash 256 0 0 0 0 62 0 0