panic: _mtx_lock_sleep: recursed on non-recursive mutex sctp-create @ /syzkaller/managers/main/kernel/sys/netinet/sctp_pcb.c:3330
cpuid = 0
time = 1625187956
KDB: stack backtrace:
db_trace_self_wrapper() at db_trace_self_wrapper+0x47/frame 0xfffffe005164fca0
vpanic() at vpanic+0x1cc/frame 0xfffffe005164fd00
panic() at panic+0x43/frame 0xfffffe005164fd60
__mtx_lock_sleep() at __mtx_lock_sleep+0x68c/frame 0xfffffe005164fdf0
__mtx_lock_flags() at __mtx_lock_flags+0x16d/frame 0xfffffe005164fe50
sctp_inpcb_free() at sctp_inpcb_free+0x2b6/frame 0xfffffe005164fed0
sctp_free_assoc() at sctp_free_assoc+0x288c/frame 0xfffffe005164ff60
sctp_process_cookie_new() at sctp_process_cookie_new+0x3f7/frame 0xfffffe0051650250
sctp_process_control() at sctp_process_control+0x51e2/frame 0xfffffe0051650740
sctp_common_input_processing() at sctp_common_input_processing+0x83a/frame 0xfffffe00516508c0
sctp_input_with_port() at sctp_input_with_port+0x30a/frame 0xfffffe00516509b0
sctp_input() at sctp_input+0x1f/frame 0xfffffe00516509d0
ip_input() at ip_input+0x38b/frame 0xfffffe0051650a90
swi_net() at swi_net+0x20e/frame 0xfffffe0051650b10
ithread_loop() at ithread_loop+0x33f/frame 0xfffffe0051650bb0
fork_exit() at fork_exit+0xb3/frame 0xfffffe0051650bf0
fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe0051650bf0
--- trap 0, rip = 0, rsp = 0, rbp = 0 ---
KDB: enter: panic
[ thread pid 12 tid 100033 ]
Stopped at kdb_enter+0x67: movq $0,0x164029e(%rip)
db>
db> set $lines = 0
db> set $maxwidth = 0
db> show registers
cs 0x20
ds 0x3b
es 0x3b
fs 0x13
gs 0x1b
ss 0x28
rax 0x12
rcx 0xffffffff811325d0 vprintf+0x140
rdx 0x1
rbx 0
rsp 0xfffffe005164fc80
rbp 0xfffffe005164fca0
rsi 0
rdi 0xffffffff81132606 vprintf+0x176
r8 0
r9 0x8080808080808080
r10 0xfffffe005164fb70
r11 0x1ffaefff59c
r12 0xffffffff82267d50 ddb_dbbe
r13 0xffffffff81993800
r14 0xffffffff81a74f90
r15 0xffffffff81a74f90
rip 0xffffffff81129667 kdb_enter+0x67
rflags 0x82
kdb_enter+0x67: movq $0,0x164029e(%rip)
db> show proc
Process 12 (intr) at 0xfffff80004c74a70:
state: NORMAL
uid: 0 gids: 0
parent: pid 0 at 0xffffffff8271ce70
ABI: null
flag: 0x10000284 flag2: 0
reaper: 0xffffffff8271ce70 reapsubtree: 12
sigparent: 20
vmspace: 0xffffffff8271daf0
(map 0xffffffff8271daf0)
(map.pmap 0xffffffff8271dbb0)
(pmap 0xffffffff8271dc10)
threads: 23
100011 I [swi5: fast taskq]
100014 I [swi6: task queue]
100016 I [swi6: Giant taskq]
100030 I [swi3: vm]
100031 I [swi4: clock (0)]
100032 I [swi4: clock (1)]
100033 Run CPU 0 [swi1: netisr 0]
100046 I [irq24: virtio_pci0]
100047 I [irq25: virtio_pci0]
100048 I [irq26: virtio_pci0]
100049 I [irq27: virtio_pci0]
100050 I [irq28: virtio_pci1]
100051 I [irq29: virtio_pci1]
100052 I [irq30: virtio_pci1]
100053 I [irq31: virtio_pci1]
100054 I [irq32: virtio_pci1]
100059 I [irq10: virtio_pci2]
100061 I [irq1: atkbd0]
100062 I [irq12: psm0]
100063 I [swi0: uart uart++]
100071 I [swi1: pf send]
100084 I [swi1: hpts]
100085 I [swi1: hpts]
db> ps
pid ppid pgrp uid state wmesg wchan cmd
15018 814 814 0 LE *sctp-cr 0xfffff80027daa0c0 syz-executor.2
15013 867 867 0 T (threaded) syz-executor.3
112004 s syz-executor.3
115831 RunQ syz-executor.3
115836 RunQ syz-executor.3
11640 11639 784 0 SV uwait 0xfffff80027e22100 syz-executor.1
11639 1 784 0 DV ppwait 0xfffff80027f8aa20 syz-executor.1
10673 1 867 0 S uwait 0xfffff80027da8900 syz-executor.3
10672 1 867 0 S uwait 0xfffff80027e00500 syz-executor.3
6603 1 783 0 S uwait 0xfffff8002747d500 syz-executor.0
6602 1 783 0 S uwait 0xfffff80015ea4300 syz-executor.0
6107 1 6107 65 Ss select 0xfffff8002bd40440 dhclient
4946 1 4946 0 Ss select 0xfffff8002bb56a40 dhclient
4943 1 4943 0 Ss select 0xfffff80027f5cd40 dhclient
4924 1 4924 65 Ss select 0xfffff8002bb91bc0 dhclient
4240 1 4240 0 Ss select 0xfffff8002bb1f440 dhclient
4237 1 4237 0 Ss select 0xfffff8002b872bc0 dhclient
4218 1 4218 65 Ss select 0xfffff8002b0015c0 dhclient
3217 1 3217 0 Ss select 0xfffff80015e26440 dhclient
3214 1 3214 0 Ss select 0xfffff8002b8028c0 dhclient
3195 1 3195 65 Ss select 0xfffff80027cab5c0 dhclient
1387 0 0 0 DL aiordy 0xfffff8002b2ac538 [aiod4]
1386 0 0 0 DL aiordy 0xfffff80027d41a70 [aiod3]
1385 0 0 0 DL aiordy 0xfffff80027d41000 [aiod2]
1384 0 0 0 DL aiordy 0xfffff8002b2aca70 [aiod1]
1064 1 1064 0 Ss select 0xfffff80027f54740 dhclient
1061 1 1061 0 Ss select 0xfffff8002b001a40 dhclient
867 780 867 0 Ss wait 0xfffff8002726c538 syz-executor.3
814 780 814 0 Ss wait 0xfffff800273cea70 syz-executor.2
784 780 784 0 Ss piperd 0xfffff80027a4d2e8 syz-executor.1
783 780 783 0 Ss piperd 0xfffff80027c76ba0 syz-executor.0
780 778 778 0 S (threaded) syz-fuzzer
100096 S uwait 0xfffff80015ea4f00 syz-fuzzer
100122 S uwait 0xfffff80015d8a900 syz-fuzzer
100123 S uwait 0xfffff80015d8ab00 syz-fuzzer
100124 S uwait 0xfffff80015d8ad00 syz-fuzzer
100125 S uwait 0xfffff80015d8af00 syz-fuzzer
100126 S uwait 0xfffff80015ac1100 syz-fuzzer
100127 S kqread 0xfffff8002736c100 syz-fuzzer
100128 S uwait 0xfffff8002736c500 syz-fuzzer
100129 S uwait 0xfffff8002736c700 syz-fuzzer
778 776 778 0 Ss pause 0xfffff800273d1b20 csh
776 694 776 0 Ss select 0xfffff80015ea65c0 sshd
760 1 760 0 Ss+ ttyin 0xfffff800154a5cb0 getty
759 1 759 0 Ss+ ttyin 0xfffff80015b2b4b0 getty
758 1 758 0 Ss+ ttyin 0xfffff80015b2bcb0 getty
757 1 757 0 Ss+ ttyin 0xfffff80015b2f4b0 getty
756 1 756 0 Ss+ ttyin 0xfffff80015b2fcb0 getty
755 1 755 0 Ss+ ttyin 0xfffff80015ab64b0 getty
754 1 754 0 Ss+ ttyin 0xfffff80015ab6cb0 getty
753 1 753 0 Ss+ ttyin 0xfffff80015abc4b0 getty
752 1 752 0 Ss+ ttyin 0xfffff80015abccb0 getty
698 1 698 0 Ss nanslp 0xffffffff8273d0a1 cron
694 1 694 0 Ss select 0xfffff80015ea68c0 sshd
507 1 507 0 Ss select 0xfffff80015dfcbc0 syslogd
436 1 436 0 Ss select 0xfffff80015e795c0 devd
435 1 435 65 Ss select 0xfffff80015f6e140 dhclient
350 1 350 0 Ss select 0xfffff80015e4c140 dhclient
347 1 347 0 Ss select 0xfffff80015f6e8c0 dhclient
23 0 0 0 DL vlruwt 0xfffff80015b12000 [vnlru]
22 0 0 0 DL syncer 0xffffffff8282c450 [syncer]
21 0 0 0 DL (threaded) [bufdaemon]
100081 D qsleep 0xffffffff8282b520 [bufdaemon]
100086 D - 0xffffffff8220aec0 [bufspacedaemon-0]
100098 D sdflush 0xfffff80004ddc4e8 [/ worker]
20 0 0 0 DL psleep 0xffffffff82853388 [vmdaemon]
19 0 0 0 RL (threaded) [pagedaemon]
100079 Run CPU 1 [dom0]
100087 D launds 0xffffffff82847804 [laundry: dom0]
100088 D umarecl 0xffffffff82846c50 [uma]
18 0 0 0 DL - 0xffffffff82571458 [rand_harvestq]
17 0 0 0 DL waiting 0xffffffff830188b8 [sctp_iterator]
16 0 0 0 DL pftm 0xffffffff82e0f410 [pf purge]
15 0 0 0 DL - 0xffffffff82828bdc [soaiod4]
9 0 0 0 DL - 0xffffffff82828bdc [soaiod3]
8 0 0 0 DL - 0xffffffff82828bdc [soaiod2]
7 0 0 0 DL - 0xffffffff82828bdc [soaiod1]
6 0 0 0 DL (threaded) [cam]
100044 D - 0xffffffff82448b40 [doneq0]
100045 D - 0xffffffff82448ac0 [async]
100078 D - 0xffffffff82448990 [scanner]
14 0 0 0 DL seqstat 0xfffff80004d8a488 [sequencer 00]
5 0 0 0 DL crypto_ 0xfffff80004d53d80 [crypto returns 1]
4 0 0 0 DL crypto_ 0xfffff80004d53d30 [crypto returns 0]
3 0 0 0 DL crypto_ 0xffffffff82844ce0 [crypto]
13 0 0 0 DL (threaded) [geom]
100035 D - 0xffffffff8271c8e0 [g_event]
100036 D - 0xffffffff8271c8e8 [g_up]
100037 D - 0xffffffff8271c8f0 [g_down]
2 0 0 0 DL (threaded) [KTLS]
100028 D - 0xfffff80004c69100 [thr_0]
100029 D - 0xfffff80004c69180 [thr_1]
12 0 0 0 RL (threaded) [intr]
100011 I [swi5: fast taskq]
100014 I [swi6: task queue]
100016 I [swi6: Giant taskq]
100030 I [swi3: vm]
100031 I [swi4: clock (0)]
100032 I [swi4: clock (1)]
100033 Run CPU 0 [swi1: netisr 0]
100046 I [irq24: virtio_pci0]
100047 I [irq25: virtio_pci0]
100048 I [irq26: virtio_pci0]
100049 I [irq27: virtio_pci0]
100050 I [irq28: virtio_pci1]
100051 I [irq29: virtio_pci1]
100052 I [irq30: virtio_pci1]
100053 I [irq31: virtio_pci1]
100054 I [irq32: virtio_pci1]
100059 I [irq10: virtio_pci2]
100061 I [irq1: atkbd0]
100062 I [irq12: psm0]
100063 I [swi0: uart uart++]
100071 I [swi1: pf send]
100084 I [swi1: hpts]
100085 I [swi1: hpts]
11 0 0 0 RL (threaded) [idle]
100003 CanRun [idle: cpu0]
100004 CanRun [idle: cpu1]
1 0 1 0 SLs wait 0xfffff80004bda538 [init]
10 0 0 0 DL audit_w 0xffffffff828451f0 [audit]
0 0 0 0 DLs (threaded) [kernel]
100000 D swapin 0xffffffff8271ce70 [swapper]
100005 D - 0xfffff8000443b500 [if_io_tqg_0]
100006 D - 0xfffff8000443b100 [if_io_tqg_1]
100007 D - 0xfffff80004c78d00 [if_config_tqg_0]
100008 D - 0xfffff80004c78900 [softirq_0]
100009 D - 0xfffff80004c78500 [softirq_1]
100010 D - 0xfffff80004c78100 [aiod_kick taskq]
100012 D - 0xfffff80004c75900 [kqueue_ctx taskq]
100013 D - 0xfffff80004c75500 [pci_hp taskq]
100015 D - 0xfffff80004c6ed00 [inm_free taskq]
100017 D - 0xfffff80004c6e500 [linuxkpi_irq_wq]
100018 D - 0xfffff80004c6e100 [thread taskq]
100019 D - 0xfffff80004c69d00 [in6m_free taskq]
100020 D - 0xfffff80004c69900 [linuxkpi_short_wq_0]
100021 D - 0xfffff80004c69900 [linuxkpi_short_wq_1]
100022 D - 0xfffff80004c69900 [linuxkpi_short_wq_2]
100023 D - 0xfffff80004c69900 [linuxkpi_short_wq_3]
100024 D - 0xfffff80004c69500 [linuxkpi_long_wq_0]
100025 D - 0xfffff80004c69500 [linuxkpi_long_wq_1]
100026 D - 0xfffff80004c69500 [linuxkpi_long_wq_2]
100027 D - 0xfffff80004c69500 [linuxkpi_long_wq_3]
100034 D - 0xfffff80004cd3d00 [firmware taskq]
100038 D - 0xfffff80004cd3900 [crypto_0]
100039 D - 0xfffff80004cd3900 [crypto_1]
100055 D - 0xfffff800153ebd00 [vtnet0 rxq 0]
100056 D - 0xfffff800153eb900 [vtnet0 txq 0]
100057 D - 0xfffff800153eb500 [vtnet0 rxq 1]
100058 D - 0xfffff800153eb100 [vtnet0 txq 1]
100060 D vtbslp 0xfffff80015463700 [virtio_balloon]
100064 D - 0xfffff80015468900 [mca taskq]
100066 D - 0xffffffff81e218b0 [deadlkres]
100074 D - 0xfffff80015a47100 [acpi_task_0]
100075 D - 0xfffff80015a47100 [acpi_task_1]
100076 D - 0xfffff80015a47100 [acpi_task_2]
100077 D - 0xfffff80004cd3500 [CAM taskq]
db> show all locks
Process 19 (pagedaemon) thread 0xfffffe0055375c80 (100079)
exclusive sleep mutex vm reserv (vm reserv) r = 0 (0xfffff8007c7f29a0) locked @ /syzkaller/managers/main/kernel/sys/vm/vm_reserv.c:1045
exclusive rw kernel vm object (kernel vm object) r = 0 (0xffffffff82847030) locked @ /syzkaller/managers/main/kernel/sys/vm/vm_kern.c:586
shared rw UMA lock (UMA lock) r = 0 (0xffffffff82244500) locked @ /syzkaller/managers/main/kernel/sys/vm/uma_core.c:3019
shared sx umareclaim (umareclaim) r = 0 (0xffffffff82846c50) locked @ /syzkaller/managers/main/kernel/sys/vm/uma_core.c:5132
Process 12 (intr) thread 0xfffffe00042be900 (100033)
exclusive sleep mutex sctp-create (inp_create) r = 0 (0xfffff8002b01f488) locked @ /syzkaller/managers/main/kernel/sys/netinet/sctp_input.c:4922
db> show malloc
Type InUse MemUse Requests
sysctloid 34288 12858K 34355
pf_hash 5 11560K 5
devbuf 4216 6982K 4244
tcp_hpts 5 3219K 5
kobj 326 2608K 486
vtbuf 24 2064K 46
pcb 201 1389K 15898
newblk 11 1037K 15122
vfscache 3 1035K 3
inodedep 415 934K 14161
subproc 159 656K 15110
acpica 1674 649K 54806
callout 2 528K 2
ufs_quota 1 520K 1
vfs_hash 1 520K 1
intr 4 480K 4
filedesc 26 405K 28077
bus 986 377K 3469
BPF 46 337K 56
linker 348 270K 409
DEVFS1 107 214K 124
dirrem 404 202K 14076
kdtrace 237 193K 30866
module 510 192K 510
vnet_data 1 176K 1
sctp_atcl 174 174K 9424
sctp_stro 85 170K 2451
tidhash 3 164K 3
umtx 374 164K 374
freefile 404 152K 14069
pagedep 11 141K 14044
tfo_ccache 1 136K 1
sem 4 120K 4
UMA 270 102K 270
sctp_atky 259 98K 12991
audit_evclass 236 89K 294
temp 36 81K 5755
mtx_pool 2 80K 2
syncache 1 76K 1
routetbl 135 75K 8792
vmem 3 72K 5
msg 4 68K 4
ifaddr 70 66K 72
sctp_athm 174 66K 10504
acpitask 1 64K 1
ddb_capture 1 64K 1
sctp_map 170 64K 4902
DEVFS3 126 63K 136
ether_multi 152 57K 162
gtaskqueue 18 57K 18
pf_ifnet 102 55K 1856
DEVFS_RULE 56 54K 56
kenv 95 52K 95
eventhandler 133 50K 133
ithread 99 43K 99
rman 84 42K 425
lltable 46 40K 92
KTRACE 100 38K 100
taskqueue 60 36K 60
kqueue 73 36K 15034
sctp_timw 69 35K 69
proc 3 34K 3
ufs_mount 5 34K 6
bus-sc 33 34K 1684
devstat 4 33K 4
hostcache 1 32K 1
tty 16 32K 16
shm 1 32K 3
DEVFSP 84 32K 7507
GEOM 60 29K 489
in6_multi 65 29K 65
cred 28 28K 270
kbdmux 6 28K 6
pwddesc 70 27K 15022
ifnet 7 25K 7
plimit 24 24K 469
kcovinfo 64 24K 68
sctp_aadr 57 22K 473
CAM queue 5 21K 1528
selfd 55 21K 213815
pfs_nodes 20 20K 20
proc-args 52 20K 686
UART 12 18K 12
session 35 18K 89
bmsafemap 2 17K 14142
ksem 1 16K 458
rpc 2 16K 2
shmfd 1 16K 20
pfs_vncache 1 16K 1
sglist 5 13K 5
ufs_dirhash 20 13K 24
MCA 32 12K 32
CAM DEV 3 12K 510
lockf 32 12K 57
vt 11 11K 11
acpisem 28 11K 28
CAM XPT 22 11K 543
Unitno 27 11K 47
uidinfo 3 9K 10
pf_table 2 8K 2097
memdesc 1 8K 1
ipsec-saq 2 8K 2
evdev 4 8K 4
inpcbpolicy 20 8K 7278
acpidev 20 8K 20
hhook 15 8K 17
select 19 8K 70
mount 16 7K 90
fpukern_ctx 3 6K 3
terminal 11 6K 11
ip6ndp 12 6K 13
sctp_ifa 13 5K 14
osd 12 5K 6367
ipsecpolicy 2 5K 2
encap_export_host 12 5K 12
clone 9 5K 9
local_apic 1 4K 1
io_apic 1 4K 1
sahead 1 4K 1
secasvar 1 4K 1
nhops 6 4K 8
pci_link 10 4K 10
crypto 5 4K 3757
msi 9 4K 9
DEVFS 9 4K 10
tun 7 4K 7
sctp_stri 3 3K 2162
ipsec 3 3K 3
mkdir 8 3K 28044
xform 8 3K 11182
in_multi 6 3K 8
nexusdev 7 3K 7
feeder 7 3K 7
procdesc 5 3K 18
freework 5 3K 14370
sctp_ifn 6 3K 14
newdirblk 6 3K 14022
mld 6 3K 6
igmp 6 3K 6
toponodes 6 3K 6
prison 6 3K 6
isadev 6 3K 6
softdep 1 2K 1
freeblks 4 2K 14081
vnodemarker 2 2K 40
NFSD session 1 2K 1
lkpikmalloc 5 2K 6
CAM periph 4 2K 271
filecaps 5 2K 124
diradd 4 2K 14117
pfil 4 2K 4
CAM path 4 2K 1034
soname 4 2K 19819
tcpfunc 3 2K 3
loginclass 3 2K 6
chacha20random 1 1K 1
vnodes 1 1K 1
ktls 1 1K 1
cdev 2 1K 2
aesni_data 2 1K 2
pf_rule 2 1K 1996
pf_osfp 2 1K 2
cpus 2 1K 2
atkbddev 2 1K 2
CAM dev queue 2 1K 2
entropy 2 1K 45
CAM SIM 2 1K 2
NFSD lckfile 1 1K 1
NFSD V4client 1 1K 1
pmchooks 1 1K 1
sctp_vrf 1 1K 1
apmdev 1 1K 1
CAM I/O Scheduler 1 1K 1
vnet_data_free 1 1K 1
vnet 1 1K 1
Per-cpu 1 1K 1
p1003.1b 1 1K 1
acpiintr 1 1K 1
pmc 1 1K 1
cubic data 0 0K 0
chd data 0 0K 0
tcp_do 0 0K 0
tcp_fsb 0 0K 3977
cdg data 0 0K 0
sctp_mcore 0 0K 0
sctp_socko 0 0K 3860
sctp_iter 0 0K 107
sctp_mvrf 0 0K 0
sctp_cpal 0 0K 98
sctp_cmsg 0 0K 0
sctp_stre 0 0K 0
sctp_athi 0 0K 0
sctp_a_it 0 0K 9
dctcp data 0 0K 0
mqdata 0 0K 0
pf_altq 0 0K 0
pf_temp 0 0K 0
htcp data 0 0K 0
vegas data 0 0K 0
NFSCL lckown 0 0K 0
NFSCL client 0 0K 0
madt_table 0 0K 2
smartpqi 0 0K 0
NFSCL deleg 0 0K 0
NFSCL open 0 0K 0
NFSCL owner 0 0K 0
NFS fh 0 0K 0
NFS req 0 0K 0
NFSD usrgroup 0 0K 0
NFSD string 0 0K 0
iavf 0 0K 0
ixl 0 0K 0
NFSD V4lock 0 0K 0
NFSD V4state 0 0K 0
NFSD srvcache 0 0K 0
msdosfs_fat 0 0K 0
msdosfs_mount 0 0K 0
msdosfs_node 0 0K 0
ice-resmgr 0 0K 0
ice-osdep 0 0K 0
ice 0 0K 0
axgbe 0 0K 0
DEVFS4 0 0K 0
DEVFS2 0 0K 0
gntdev 0 0K 0
privcmd_dev 0 0K 0
evtchn_dev 0 0K 0
xen_intr 0 0K 0
xenstore 0 0K 0
ciss_data 0 0K 0
xen_hvm 0 0K 0
legacydrv 0 0K 0
qpidrv 0 0K 0
BACKLIGHT 0 0K 0
xnb 0 0K 0
dmar_idpgtbl 0 0K 0
dmar_dom 0 0K 0
dmar_ctx 0 0K 0
xbbd 0 0K 0
xbd 0 0K 0
Balloon 0 0K 0
isci 0 0K 0
iommu_dmamap 0 0K 0
sysmouse 0 0K 0
hyperv_socket 0 0K 0
bxe_ilt 0 0K 0
xenbus 0 0K 0
vtfont 0 0K 0
vm_fictitious 0 0K 0
ath_hal 0 0K 0
athdev 0 0K 0
ata_pci 0 0K 0
ata_dma 0 0K 0
ata_generic 0 0K 0
amr 0 0K 0
pvscsi 0 0K 0
scsi_da 0 0K 69
ata_da 0 0K 0
scsi_ch 0 0K 0
UMAHash 0 0K 0
scsi_cd 0 0K 0
AHCI driver 0 0K 0
vm_pgdata 0 0K 0
jblocks 0 0K 0
savedino 0 0K 11544
sentinel 0 0K 0
jfsync 0 0K 0
jtrunc 0 0K 0
sbdep 0 0K 17
jsegdep 0 0K 0
jseg 0 0K 0
jfreefrag 0 0K 0
jfreeblk 0 0K 0
jnewblk 0 0K 0
jmvref 0 0K 0
jremref 0 0K 0
jaddref 0 0K 0
freedep 0 0K 0
freefrag 0 0K 6
allocindir 0 0K 0
indirdep 0 0K 34
allocdirect 0 0K 0
ufs_trim 0 0K 0
mactemp 0 0K 0
audit_trigger 0 0K 0
audit_pipe_presel 0 0K 0
audit_pipeent 0 0K 0
audit_pipe 0 0K 0
audit_evname 0 0K 0
audit_bsm 0 0K 0
audit_gidset 0 0K 0
audit_text 0 0K 0
audit_path 0 0K 0
audit_data 0 0K 0
audit_cred 0 0K 0
USBdev 0 0K 0
USB 0 0K 0
agp 0 0K 0
nvme_da 0 0K 0
acpipwr 0 0K 0
acpi_perf 0 0K 0
ktls_ocf 0 0K 0
twsbuf 0 0K 0
MLX5EEPROM 0 0K 0
MLX5EEPROM 0 0K 0
MLX5EEPROM 0 0K 0
MLX5EEPROM 0 0K 0
MLX5E_TLS 0 0K 0
MLX5EEPROM 0 0K 0
MLX5EEPROM 0 0K 0
MLX5EEPROM 0 0K 0
MLX5EN 0 0K 0
MLX5EEPROM 0 0K 0
MLX5EEPROM 0 0K 0
MLX5EEPROM 0 0K 0
MLX5DUMP 0 0K 0
MLX5EEPROM 0 0K 0
MLX5EEPROM 0 0K 0
seq_file 0 0K 0
radix 0 0K 0
idr 0 0K 0
lkpifw 0 0K 0
NLM 0 0K 0
ipsec-spdcache 0 0K 0
ipsec-reg 0 0K 0
ipsec-misc 0 0K 0
ipsecrequest 0 0K 0
ip6opt 0 0K 26
ip6_msource 0 0K 0
ip6_moptions 0 0K 0
in6_mfilter 0 0K 0
frag6 0 0K 0
tcplog 0 0K 0
tcp_hwpace 0 0K 0
twe_commands 0 0K 0
LRO 0 0K 0
newreno data 0 0K 2
ip_msource 0 0K 0
ip_moptions 0 0K 7
in_mfilter 0 0K 0
ipid 0 0K 0
80211scan 0 0K 0
80211ratectl 0 0K 0
80211power 0 0K 0
80211nodeie 0 0K 0
80211node 0 0K 0
80211mesh_gt 0 0K 0
80211mesh_rt 0 0K 0
80211perr 0 0K 0
80211prep 0 0K 0
80211preq 0 0K 0
80211dfs 0 0K 0
80211crypto 0 0K 0
80211vap 0 0K 0
iflib 0 0K 0
vlan 0 0K 0
gif 0 0K 0
ifdescr 0 0K 0
zlib 0 0K 0
fadvise 0 0K 2
VN POLL 0 0K 0
twa_commands 0 0K 0
statfs 0 0K 14215
namei_tracker 0 0K 0
export_host 0 0K 0
cl_savebuf 0 0K 28
tcp_log_dev 0 0K 1714
midi buffers 0 0K 0
mixer 0 0K 0
ac97 0 0K 0
hdacc 0 0K 0
hdac 0 0K 0
hdaa 0 0K 0
acpicmbat 0 0K 0
SIIS driver 0 0K 0
CAM CCB 0 0K 2428
PUC 0 0K 0
ppbusdev 0 0K 0
agtiapi_MemAlloc malloc 0 0K 0
osti_cacheable 0 0K 0
tempbuff 0 0K 0
biobuf 0 0K 0
aios 0 0K 4
lio 0 0K 2
acl 0 0K 0
tempbuff 0 0K 0
mbuf_tag 0 0K 120
ag_tgt_map_t malloc 0 0K 0
ag_slr_map_t malloc 0 0K 0
lDevFlags * malloc 0 0K 0
tiDeviceHandle_t * malloc 0 0K 0
ag_portal_data_t malloc 0 0K 0
ag_device_t malloc 0 0K 0
STLock malloc 0 0K 0
CCB List 0 0K 0
sr_iov 0 0K 0
OCS 0 0K 0
OCS 0 0K 0
nvme 0 0K 0
nvd 0 0K 0
netmap 0 0K 0
mwldev 0 0K 0
MVS driver 0 0K 0
CAM ccb queue 0 0K 0
mrsasbuf 0 0K 0
mpt_user 0 0K 0
mps_user 0 0K 0
accf 0 0K 0
pts 0 0K 0
iov 0 0K 20637
ioctlops 0 0K 8698
eventfd 0 0K 42
Witness 0 0K 0
stack 0 0K 0
MPSSAS 0 0K 0
mps 0 0K 0
mpr_user 0 0K 0
MPRSAS 0 0K 0
mpr 0 0K 0
mfibuf 0 0K 0
sbuf 0 0K 288
md_sectors 0 0K 0
firmware 0 0K 0
compressor 0 0K 0
md_disk 0 0K 0
SWAP 0 0K 0
malodev 0 0K 0
LED 0 0K 0
sysctltmp 0 0K 831
sysctl 0 0K 3
ekcd 0 0K 0
dumper 0 0K 0
sendfile 0 0K 0
rctl 0 0K 0
ix_sriov 0 0K 0
aacraidcam 0 0K 0
aacraid_buf 0 0K 0
ix 0 0K 0
ipsbuf 0 0K 0
cache 0 0K 0
iirbuf 0 0K 0
prison_racct 0 0K 0
Fail Points 0 0K 0
sigio 0 0K 9
filedesc_to_leader 0 0K 0
pwd 0 0K 0
tty console 0 0K 0
aaccam 0 0K 0
aacbuf 0 0K 0
zstd 0 0K 0
XZ_DEC 0 0K 0
nvlist 0 0K 0
SCSI ENC 0 0K 0
SCSI sa 0 0K 0
scsi_pass 0 0K 0
isofs_node 0 0K 0
isofs_mount 0 0K 0
tr_raid5_data 0 0K 0
tr_raid1e_data 0 0K 0
tr_raid1_data 0 0K 0
tr_raid0_data 0 0K 0
tr_concat_data 0 0K 0
md_sii_data 0 0K 0
md_promise_data 0 0K 0
md_nvidia_data 0 0K 0
md_jmicron_data 0 0K 0
md_intel_data 0 0K 0
md_ddf_data 0 0K 0
raid_data 0 0K 72
geom_flashmap 0 0K 0
tmpfs dir 0 0K 0
tmpfs name 0 0K 0
tmpfs mount 0 0K 0
NFS FHA 0 0K 0
newnfsmnt 0 0K 0
newnfsclient_req 0 0K 0
NFSCL layrecall 0 0K 0
NFSCL session 0 0K 0
NFSCL sockreq 0 0K 0
NFSCL devinfo 0 0K 0
NFSCL flayout 0 0K 0
NFSCL layout 0 0K 0
NFSD rollback 0 0K 0
NFSCL diroff 0 0K 0
NEWdirectio 0 0K 0
NEWNFSnode 0 0K 0
NFSCL lck 0 0K 0
db> show uma
Zone Size Used Free Requests Sleeps Bucket Total Mem XFree
mbuf_jumbo_page 4096 8324 720 1165378 0 253 37044224 0
malloc-384 384 35667 53 43968 0 30 13716480 0
RADIX NODE 144 32500 61 453249 0 62 4688784 0
malloc-1024 1024 4174 6 4235 0 16 4280320 0
malloc-1024 1024 495 2157 15586 0 16 2715648 0
malloc-8192 8192 328 3 1099 0 1 2711552 0
mbuf 256 8925 1650 1946835 0 254 2707200 0
tcp_log 416 950 5188 114139 0 253 2553408 0
pbuf 2624 0 973 0 0 2 2553152 0
malloc-1024 1024 135 2173 13130 0 16 2363392 0
BUF TRIE 144 194 13274 1146 0 62 1939392 0
UMA Slabs 0 112 13472 313 13760 0 126 1543920 0
malloc-512 512 546 2166 18848 0 30 1388544 0
mbuf_cluster 2048 585 1 585 0 253 1200128 0
malloc-384 384 949 2171 76229 0 30 1198080 0
malloc-8192 8192 88 10 5896 0 1 802816 0
malloc-32768 32768 15 6 14039 0 1 688128 0
malloc-384 384 1723 47 54702 0 30 679680 0
malloc-384 384 1579 111 261110 0 30 648960 0
malloc-8192 8192 73 6 15024 0 1 647168 0
FFS inode 1160 520 26 14590 0 8 633360 0
malloc-16384 16384 24 0 25 0 1 393216 0
VM OBJECT 264 1350 75 213974 0 30 376200 0
lkpimm 160 1 2324 1 0 62 372000 0
lkpicurr 160 2 2323 2 0 62 372000 0
malloc-512 512 33 639 16101 0 30 344064 0
THREAD 1808 165 22 15843 0 8 338096 0
256 Bucket 2048 110 52 17666 0 8 331776 0
malloc-1024 1024 265 39 26042 0 16 311296 0
VNODE 448 559 80 14631 0 30 286272 0
malloc-65536 65536 4 0 4 0 1 262144 0
malloc-2048 2048 125 3 147 0 8 262144 0
malloc-384 384 617 63 9151 0 30 261120 0
sctp_asoc 2288 85 25 2451 0 253 251680 0
malloc-512 512 113 359 2382 0 30 241664 0
malloc-1024 1024 182 42 11607 0 16 229376 0
tcp_inpcb 488 8 448 6355 0 253 222528 0
malloc-2048 2048 87 21 3681 0 8 221184 0
MAP ENTRY 96 1857 369 808805 0 126 213696 0
malloc-8192 8192 23 2 55 0 1 204800 0
malloc-65536 65536 3 0 3 0 1 196608 0
malloc-16384 16384 11 1 247 0 1 196608 0
socket 944 49 151 14539 0 253 188800 0
UMA Zones 768 242 2 242 0 16 187392 0
malloc-4096 4096 9 34 5486 0 2 176128 0
malloc-384 384 364 76 63604 0 30 168960 0
sctp_ep 1280 86 40 5891 0 253 161280 0
vmem btag 56 2790 78 2790 0 254 160608 0
tcp_bbr_map 128 0 1240 16920 0 126 158720 0
ertt_txseginfo 40 0 3838 70036 0 254 153520 0
S VFS Cache 104 1067 376 16860 0 126 150072 0
mbuf_packet 256 68 517 23713 0 254 149760 0
malloc-512 512 196 76 30880 0 30 139264 0
FFS2 dinode 256 520 20 14589 0 62 138240 0
malloc-384 384 295 65 27402 0 30 138240 0
VMSPACE 2544 43 11 14994 0 4 137376 0
malloc-65536 65536 2 0 2 0 1 131072 0
malloc-65536 65536 0 2 176 0 1 131072 0
tcpcb 1080 8 111 6355 0 253 128520 0
malloc-512 512 207 33 14306 0 30 122880 0
PROC 1336 70 17 15021 0 8 116232 0
malloc-16384 16384 2 5 34 0 1 114688 0
malloc-1024 1024 100 4 105 0 16 106496 0
DEVCTL 1024 0 100 153 0 0 102400 0
malloc-32768 32768 1 2 121 0 1 98304 0
malloc-16384 16384 4 2 6 0 1 98304 0
pcpu-8 8 6169 6119 49741 0 254 98304 0
filedesc0 1072 71 20 15022 0 8 97552 0
UMA Kegs 384 227 6 227 0 30 89472 0
sctp_raddr 736 100 21 2468 0 253 89056 0
32 Bucket 256 245 100 19631 0 62 88320 0
clpbuf 2624 0 32 45 0 15 83968 0
malloc-16384 16384 3 2 65 0 1 81920 0
128 Bucket 1024 39 40 1457 0 16 80896 0
malloc-512 512 119 33 14482 0 30 77824 0
malloc-4096 4096 13 5 3189 0 2 73728 0
malloc-2048 2048 17 17 602 0 8 69632 0
malloc-65536 65536 0 1 1 0 1 65536 0
malloc-32768 32768 1 1 16 0 1 65536 0
g_bio 408 0 160 7087 0 30 65280 0
malloc-2048 2048 10 20 518 0 8 61440 0
malloc-384 384 96 64 21852 0 30 61440 0
16 Bucket 144 332 60 4295 0 62 56448 0
malloc-4096 4096 7 6 92 0 2 53248 0
malloc-16384 16384 2 1 6 0 1 49152 0
malloc-16384 16384 3 0 3 0 1 49152 0
malloc-512 512 83 13 525 0 30 49152 0
malloc-8192 8192 2 3 14546 0 1 40960 0
64 Bucket 512 40 40 17307 0 30 40960 0
DIRHASH 1024 30 6 34 0 16 36864 0
NAMEI 1024 0 36 84894 0 16 36864 0
malloc-512 512 34 38 2046 0 30 36864 0
Files 80 255 195 49609 0 126 36000 0
pipe 744 25 20 725 0 16 33480 0
malloc-32768 32768 1 0 1 0 1 32768 0
malloc-32768 32768 1 0 1 0 1 32768 0
malloc-32768 32768 1 0 1 0 1 32768 0
malloc-8192 8192 3 1 7 0 1 32768 0
malloc-4096 4096 6 2 12 0 2 32768 0
malloc-2048 2048 2 14 151 0 8 32768 0
pcpu-64 64 487 25 487 0 254 32768 0
sctp_chunk 152 141 67 2413 0 253 31616 0
tcp_bbr_pcb 832 0 36 1391 0 16 29952 0
tcp_rack_pcb 832 0 36 3977 0 16 29952 0
TURNSTILE 136 188 22 188 0 62 28560 0
ksiginfo 112 72 180 1368 0 126 28224 0
malloc-8192 8192 0 3 13 0 1 24576 0
malloc-8192 8192 1 2 52 0 1 24576 0
malloc-4096 4096 1 5 10 0 2 24576 0
PWD 32 22 734 14118 0 254 24192 0
ttyinq 160 135 15 300 0 62 24000 0
8 Bucket 80 47 253 3168 0 126 24000 0
ttyoutq 256 72 18 160 0 62 23040 0
malloc-4096 4096 3 2 500 0 2 20480 0
malloc-4096 4096 2 3 7644 0 2 20480 0
2 Bucket 32 97 533 3763 0 254 20160 0
KNOTE 160 28 97 221704 0 62 20000 0
SLEEPQUEUE 88 188 36 188 0 126 19712 0
Mountpoints 2752 2 5 2 0 4 19264 0
malloc-16384 16384 1 0 1 0 1 16384 0
sctp_laddr 48 220 116 2714 0 254 16128 0
vtnet_tx_hdr 24 0 668 594143 0 254 16032 0
ripcb 488 5 27 178 0 253 15616 0
udplite_inpcb 488 1 31 423 0 253 15616 0
udp_inpcb 488 6 26 322 0 253 15616 0
malloc-2048 2048 3 3 163 0 8 12288 0
malloc-2048 2048 2 4 6 0 8 12288 0
tcp_rack_map 112 0 108 4894 0 126 12096 0
ertt 72 8 160 6355 0 126 12096 0
kenv 258 15 30 1042 0 30 11610 0
unpcb 256 20 25 1327 0 253 11520 0
malloc-2048 2048 4 0 4 0 8 8192 0
malloc-1024 1024 1 7 14 0 16 8192 0
malloc-1024 1024 0 8 33 0 16 8192 0
rtentry 176 30 16 34 0 62 8096 0
PGRP 88 35 57 89 0 126 8096 0
rl_entry 40 72 130 72 0 254 8080 0
sctp_stream_msg_out 112 3 69 262 0 253 8064 0
udpcb 32 7 245 745 0 254 8064 0
4 Bucket 48 5 163 341 0 254 8064 0
UMA Slabs 1 176 23 22 23 0 62 7920 0
sctp_readq 152 1 51 149 0 253 7904 0
AIO 208 0 38 20 0 62 7904 0
cryptop 280 0 28 65 0 30 7840 0
AIOCB 552 0 14 48 0 16 7728 0
routing nhops 256 27 3 34 0 62 7680 0
FPU_save_area 832 1 8 1 0 16 7488 0
cpuset 104 7 55 7 0 126 6448 0
epoch_record pcpu 256 4 12 4 0 62 4096 0
malloc-4096 4096 1 0 1 0 2 4096 0
pcpu-16 16 7 249 7 0 254 4096 0
hostcache 64 1 62 1 0 254 4032 0
syncache 168 0 24 4 0 253 4032 0
AIOP 32 4 122 4 0 254 4032 0
AIOLIO 272 0 14 2 0 30 3808 0
mqnode 416 3 6 3 0 30 3744 0
KMAP ENTRY 96 12 27 12 0 0 3744 0
vmem 1856 1 1 1 0 8 3712 0
SMR CPU 32 3 60 3 0 254 2016 0
SMR SHARED 24 3 60 3 0 254 1512 0
FFS1 dinode 128 0 0 0 0 126 0 0
da_ccb 544 0 0 0 0 16 0 0
ada_ccb 272 0 0 0 0 30 0 0
swblk 136 0 0 0 0 62 0 0
swpctrie 144 0 0 0 0 62 0 0
cdg_qdiffsample 16 0 0 0 0 254 0 0
sctp_asconf_ack 48 0 0 0 0 254 0 0
sctp_asconf 40 0 0 0 0 254 0 0
pf state scrubs 40 0 0 0 0 254 0 0
pf frag entries 40 0 0 0 0 254 0 0
pf frags 248 0 0 0 0 62 0 0
pf table entries 160 0 0 0 0 62 0 0
pf table entry counters 64 0 0 0 0 254 0 0
pf source nodes 136 0 0 0 0 253 0 0
pf state keys 88 0 0 0 0 126 0 0
pf states 304 0 0 0 0 253 0 0
pf tags 104 0 0 0 0 126 0 0
pf mtags 48 0 0 0 0 254 0 0
tcp_log_node 120 0 0 0 0 126 0 0
tcp_log_bucket 176 0 0 0 0 62 0 0
tcpreass 48 0 0 0 0 254 0 0
tfo_ccache_entries 80 0 0 0 0 126 0 0
tfo 4 0 0 0 0 254 0 0
sackhole 32 0 0 0 0 254 0 0
tcptw 88 0 0 0 0 253 0 0
ipq 56 0 0 0 0 254 0 0
IPsec SA lft_c 16 0 0 0 0 254 0 0
itimer 352 0 0 0 0 30 0 0
NCLNODE 584 0 0 0 0 16 0 0
TMPFS node 224 0 0 0 0 62 0 0
mqnotifier 216 0 0 0 0 62 0 0
mvdata 64 0 0 0 0 254 0 0
mqueue 248 0 0 0 0 62 0 0
LTS VFS Cache 360 0 0 0 0 30 0 0
L VFS Cache 320 0 0 0 0 30 0 0
STS VFS Cache 144 0 0 0 0 62 0 0
linux_dma_object 24 0 0 0 0 254 0 0
linux_dma_pctrie 144 0 0 0 0 62 0 0
IOMMU_MAP_ENTRY 120 0 0 0 0 126 0 0
ktls_session 128 0 0 0 0 126 0 0
mbuf_jumbo_16k 16384 0 0 0 0 253 0 0
mbuf_jumbo_9k 9216 0 0 0 0 253 0 0
audit_record 1280 0 0 0 0 8 0 0
domainset 40 0 0 0 0 254 0 0
MAC labels 40 0 0 0 0 254 0 0
vnpbuf 2624 0 0 0 0 63 0 0
mdpbuf 2624 0 0 0 0 3 0 0
nfspbuf 2624 0 0 0 0 15 0 0
swwbuf 2624 0 0 0 0 7 0 0
swrbuf 2624 0 0 0 0 15 0 0
umtx_shm 88 0 0 0 0 126 0 0
umtx pi 96 0 0 0 0 126 0 0
rangeset pctrie nodes 144 0 0 0 0 62 0 0
malloc-65536 65536 0 0 0 0 1 0 0
malloc-65536 65536 0 0 0 0 1 0 0
malloc-65536 65536 0 0 0 0 1 0 0
malloc-32768 32768 0 0 0 0 1 0 0
malloc-32768 32768 0 0 0 0 1 0 0
malloc-256 256 0 0 0 0 62 0 0
malloc-256 256 0 0 0 0 62 0 0
malloc-256 256 0 0 0 0 62 0 0
malloc-256 256 0 0 0 0 62 0 0
malloc-256 256 0 0 0 0 62 0 0
malloc-256 256 0 0 0 0 62 0 0
malloc-256 256 0 0 0 0 62 0 0
malloc-256 256 0 0 0 0 62 0 0
malloc-128 128 0 0 0 0 126 0 0
malloc-128 128 0 0 0 0 126 0 0
malloc-128 128 0 0 0 0 126 0 0
malloc-128 128 0 0 0 0 126 0 0
malloc-128 128 0 0 0 0 126 0 0
malloc-128 128 0 0 0 0 126 0 0
malloc-128 128 0 0 0 0 126 0 0
malloc-128 128 0 0 0 0 126 0 0
malloc-64 64 0 0 0 0 254 0 0
malloc-64 64 0 0 0 0 254 0 0
malloc-64 64 0 0 0 0 254 0 0
malloc-64 64 0 0 0 0 254 0 0
malloc-64 64 0 0 0 0 254 0 0
malloc-64 64 0 0 0 0 254 0 0
malloc-64 64 0 0 0 0 254 0 0
malloc-64 64 0 0 0 0 254 0 0
malloc-32 32 0 0 0 0 254 0 0
malloc-32 32 0 0 0 0 254 0 0
malloc-32 32 0 0 0 0 254 0 0
malloc-32 32 0 0 0 0 254 0 0
malloc-32 32 0 0 0 0 254 0 0
malloc-32 32 0 0 0 0 254 0 0
malloc-32 32 0 0 0 0 254 0 0
malloc-32 32 0 0 0 0 254 0 0
malloc-16 16 0 0 0 0 254 0 0
malloc-16 16 0 0 0 0 254 0 0
malloc-16 16 0 0 0 0 254 0 0
malloc-16 16 0 0 0 0 254 0 0
malloc-16 16 0 0 0 0 254 0 0
malloc-16 16 0 0 0 0 254 0 0
malloc-16 16 0 0 0 0 254 0 0
malloc-16 16 0 0 0 0 254 0 0
pcpu-32 32 0 0 0 0 254 0 0
pcpu-4 4 0 0 0 0 254 0 0
fakepg 104 0 0 0 0 126 0 0
UMA Hash 256 0 0 0 0 62 0 0