panic: mtx_lock() of spin mutex (null) @ /syzkaller/managers/main/kernel/sys/kern/uipc_sockbuf.c:434 cpuid = 0 time = 207 KDB: stack backtrace: db_trace_self_wrapper() at db_trace_self_wrapper+0x47/frame 0xfffffe0051650100 vpanic() at vpanic+0x1cc/frame 0xfffffe0051650160 panic() at panic+0x43/frame 0xfffffe00516501c0 __mtx_lock_flags() at __mtx_lock_flags+0x1d7/frame 0xfffffe0051650220 socantrcvmore() at socantrcvmore+0x2c/frame 0xfffffe0051650240 sctp_notify_assoc_change() at sctp_notify_assoc_change+0x5b8/frame 0xfffffe00516502b0 sctp_process_control() at sctp_process_control+0x9034/frame 0xfffffe0051650740 sctp_common_input_processing() at sctp_common_input_processing+0x83a/frame 0xfffffe00516508c0 sctp_input_with_port() at sctp_input_with_port+0x30a/frame 0xfffffe00516509b0 sctp_input() at sctp_input+0x1f/frame 0xfffffe00516509d0 ip_input() at ip_input+0x38b/frame 0xfffffe0051650a90 swi_net() at swi_net+0x20e/frame 0xfffffe0051650b10 ithread_loop() at ithread_loop+0x33f/frame 0xfffffe0051650bb0 fork_exit() at fork_exit+0xb3/frame 0xfffffe0051650bf0 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe0051650bf0 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- KDB: enter: panic [ thread pid 12 tid 100033 ] Stopped at kdb_enter+0x67: movq $0,0x164017e(%rip) db> db> set $lines = 0 db> set $maxwidth = 0 db> show registers cs 0x20 ds 0x3b es 0x3b fs 0x13 gs 0x1b ss 0x28 rax 0x12 rcx 0xffffffff811326f0 vprintf+0x140 rdx 0x1 rbx 0 rsp 0xfffffe00516500e0 rbp 0xfffffe0051650100 rsi 0 rdi 0xffffffff81132726 vprintf+0x176 r8 0 r9 0x8080808080808080 r10 0xfffffe005164ffd0 r11 0x1ffaefff59c r12 0xffffffff82267d30 ddb_dbbe r13 0xffffffff81a6d500 r14 0xffffffff81a73fcf r15 0xffffffff81a73fcf rip 0xffffffff81129787 kdb_enter+0x67 rflags 0x82 kdb_enter+0x67: movq $0,0x164017e(%rip) db> show proc Process 12 (intr) at 0xfffff80004c6fa60: state: NORMAL uid: 0 gids: 0 parent: pid 0 at 0xffffffff8271ce70 ABI: null flag: 0x10000284 flag2: 0 reaper: 0xffffffff8271ce70 reapsubtree: 12 sigparent: 20 vmspace: 0xffffffff8271dad0 (map 0xffffffff8271dad0) (map.pmap 0xffffffff8271db90) (pmap 0xffffffff8271dbf0) threads: 23 100011 I [swi5: fast taskq] 100014 I [swi6: task queue] 100016 I [swi6: Giant taskq] 100030 I [swi3: vm] 100031 I [swi4: clock (0)] 100032 I [swi4: clock (1)] 100033 Run CPU 0 [swi1: netisr 0] 100046 I [irq24: virtio_pci0] 100047 I [irq25: virtio_pci0] 100048 I [irq26: virtio_pci0] 100049 I [irq27: virtio_pci0] 100050 I [irq28: virtio_pci1] 100051 I [irq29: virtio_pci1] 100052 I [irq30: virtio_pci1] 100053 I [irq31: virtio_pci1] 100054 I [irq32: virtio_pci1] 100059 I [irq10: virtio_pci2] 100061 I [irq1: atkbd0] 100062 I [irq12: psm0] 100063 I [swi0: uart uart++] 100071 I [swi1: pf send] 100087 I [swi1: hpts] 100088 I [swi1: hpts] db> ps pid ppid pgrp uid state wmesg wchan cmd 44470 861 861 0 R (threaded) syz-executor.3 141753 Run CPU 1 syz-executor.3 148402 S accept 0xfffff8006d2d0c68 syz-executor.3 148412 RunQ syz-executor.3 148415 S accept 0xfffff8006d268c68 syz-executor.3 43401 43400 783 0 SV uwait 0xfffff8002741f100 syz-executor.0 43400 1 783 0 DV ppwait 0xfffff8001eb16f40 syz-executor.0 32563 1 784 0 S uwait 0xfffff800273c0700 syz-executor.1 32562 1 784 0 S uwait 0xfffff80027dcc700 syz-executor.1 27111 0 0 0 DL - 0xffffffff8271e5c4 [accounting] 17846 1 783 0 S uwait 0xfffff80027e6f700 syz-executor.0 17845 1 783 0 S uwait 0xfffff80027dccb00 syz-executor.0 17385 1 783 0 S uwait 0xfffff800271d5300 syz-executor.0 17384 1 783 0 S uwait 0xfffff80027e6ff00 syz-executor.0 11734 1 11734 65 Ss select 0xfffff8001e29d8c0 dhclient 11365 1 11365 0 Ss select 0xfffff8001e4555c0 dhclient 11362 1 11362 0 Ss select 0xfffff8001e437440 dhclient 11343 1 11343 65 Ss select 0xfffff8001e455740 dhclient 10134 1 10134 0 Ss select 0xfffff8001e03e2c0 dhclient 10131 1 10131 0 Ss select 0xfffff8001e16d140 dhclient 10112 1 10112 65 Ss select 0xfffff8001e03eec0 dhclient 7209 1 7209 0 Ss select 0xfffff8001e4ccbc0 dhclient 7206 1 7206 0 Ss select 0xfffff8001e4cc2c0 dhclient 7187 1 7187 65 Ss select 0xfffff8001e3f5740 dhclient 1739 1 1739 0 Ss select 0xfffff80027cd78c0 dhclient 1732 1 1732 0 Ss select 0xfffff80027faa140 dhclient 1413 0 0 0 DL aiordy 0xfffff800272d7000 [aiod4] 1412 0 0 0 DL aiordy 0xfffff80015e35000 [aiod3] 1411 0 0 0 DL aiordy 0xfffff80027439a60 [aiod2] 1410 0 0 0 DL aiordy 0xfffff800273c9530 [aiod1] 883 1 861 0 SV uwait 0xfffff80027dd1d00 syz-executor.3 881 1 861 0 SV uwait 0xfffff800273d2500 syz-executor.3 861 780 861 0 Ss nanslp 0xffffffff8273d0a0 syz-executor.3 806 780 806 0 Ss piperd 0xfffff80015db12e8 syz-executor.2 784 780 784 0 Ss piperd 0xfffff80027468000 syz-executor.1 783 780 783 0 Ss piperd 0xfffff800274688b8 syz-executor.0 780 778 778 0 S (threaded) syz-fuzzer 100096 S uwait 0xfffff80015ab4300 syz-fuzzer 100122 S uwait 0xfffff80027a22900 syz-fuzzer 100123 S uwait 0xfffff80027a22b00 syz-fuzzer 100124 S uwait 0xfffff80027a22d00 syz-fuzzer 100125 S uwait 0xfffff80027a22f00 syz-fuzzer 100126 S uwait 0xfffff80027a22300 syz-fuzzer 100127 S uwait 0xfffff80027a22500 syz-fuzzer 100128 S kqread 0xfffff80015d08b00 syz-fuzzer 100142 S uwait 0xfffff800271d5700 syz-fuzzer 778 776 778 0 Ss pause 0xfffff80015e0c5e0 csh 776 694 776 0 Ss select 0xfffff80027394a40 sshd 760 1 760 0 Ss+ ttyin 0xfffff800154804b0 getty 759 1 759 0 Ss+ ttyin 0xfffff800159e74b0 getty 758 1 758 0 Ss+ ttyin 0xfffff800159e7cb0 getty 757 1 757 0 Ss+ ttyin 0xfffff800159ea4b0 getty 756 1 756 0 Ss+ ttyin 0xfffff800159eacb0 getty 755 1 755 0 Ss+ ttyin 0xfffff80015aaacb0 getty 754 1 754 0 Ss+ ttyin 0xfffff80015aad4b0 getty 753 1 753 0 Ss+ ttyin 0xfffff80015aadcb0 getty 752 1 752 0 Ss+ ttyin 0xfffff800154644b0 getty 698 1 698 0 Ss nanslp 0xffffffff8273d0a0 cron 694 1 694 0 Ss select 0xfffff80015d07740 sshd 507 1 507 0 Ss select 0xfffff80015d88740 syslogd 436 1 436 0 Ss select 0xfffff800159be2c0 devd 435 1 435 65 Ss select 0xfffff80015d078c0 dhclient 350 1 350 0 Ss select 0xfffff80015d88bc0 dhclient 347 1 347 0 Ss select 0xfffff80015d07ec0 dhclient 23 0 0 0 DL vlruwt 0xfffff80015b45000 [vnlru] 22 0 0 0 DL syncer 0xffffffff8282c450 [syncer] 21 0 0 0 DL (threaded) [bufdaemon] 100081 D qsleep 0xffffffff8282b520 [bufdaemon] 100084 D - 0xffffffff8220aec0 [bufspacedaemon-0] 100098 D sdflush 0xfffff80015e91ce8 [/ worker] 20 0 0 0 DL psleep 0xffffffff82853388 [vmdaemon] 19 0 0 0 DL (threaded) [pagedaemon] 100079 D psleep 0xffffffff828477f8 [dom0] 100082 D launds 0xffffffff82847804 [laundry: dom0] 100083 D umarcl 0xffffffff815c4160 [uma] 18 0 0 0 DL - 0xffffffff82571458 [rand_harvestq] 17 0 0 0 DL waiting 0xffffffff830ce8b8 [sctp_iterator] 16 0 0 0 DL pftm 0xffffffff82d79410 [pf purge] 15 0 0 0 DL - 0xffffffff82828bdc [soaiod4] 9 0 0 0 DL - 0xffffffff82828bdc [soaiod3] 8 0 0 0 DL - 0xffffffff82828bdc [soaiod2] 7 0 0 0 DL - 0xffffffff82828bdc [soaiod1] 6 0 0 0 DL (threaded) [cam] 100044 D - 0xffffffff82448b40 [doneq0] 100045 D - 0xffffffff82448ac0 [async] 100078 D - 0xffffffff82448990 [scanner] 14 0 0 0 DL seqstat 0xfffff80004d84c88 [sequencer 00] 5 0 0 0 DL crypto_ 0xfffff80004d4dd80 [crypto returns 1] 4 0 0 0 DL crypto_ 0xfffff80004d4dd30 [crypto returns 0] 3 0 0 0 DL crypto_ 0xffffffff82844ce0 [crypto] 13 0 0 0 DL (threaded) [geom] 100035 D - 0xffffffff8271c8e0 [g_event] 100036 D - 0xffffffff8271c8e8 [g_up] 100037 D - 0xffffffff8271c8f0 [g_down] 2 0 0 0 DL (threaded) [KTLS] 100028 D - 0xfffff80004c71500 [thr_0] 100029 D - 0xfffff80004c71580 [thr_1] 12 0 0 0 RL (threaded) [intr] 100011 I [swi5: fast taskq] 100014 I [swi6: task queue] 100016 I [swi6: Giant taskq] 100030 I [swi3: vm] 100031 I [swi4: clock (0)] 100032 I [swi4: clock (1)] 100033 Run CPU 0 [swi1: netisr 0] 100046 I [irq24: virtio_pci0] 100047 I [irq25: virtio_pci0] 100048 I [irq26: virtio_pci0] 100049 I [irq27: virtio_pci0] 100050 I [irq28: virtio_pci1] 100051 I [irq29: virtio_pci1] 100052 I [irq30: virtio_pci1] 100053 I [irq31: virtio_pci1] 100054 I [irq32: virtio_pci1] 100059 I [irq10: virtio_pci2] 100061 I [irq1: atkbd0] 100062 I [irq12: psm0] 100063 I [swi0: uart uart++] 100071 I [swi1: pf send] 100087 I [swi1: hpts] 100088 I [swi1: hpts] 11 0 0 0 RL (threaded) [idle] 100003 CanRun [idle: cpu0] 100004 CanRun [idle: cpu1] 1 0 1 0 SLs wait 0xfffff80004bd7530 [init] 10 0 0 0 DL audit_w 0xffffffff828451f0 [audit] 0 0 0 0 DLs (threaded) [kernel] 100000 D swapin 0xffffffff8271ce70 [swapper] 100005 D - 0xfffff80004c7a900 [if_io_tqg_0] 100006 D - 0xfffff80004c7a500 [if_io_tqg_1] 100007 D - 0xfffff80004c7a100 [if_config_tqg_0] 100008 D - 0xfffff80004c71d00 [softirq_0] 100009 D - 0xfffff80004c71900 [softirq_1] 100010 D - 0xfffff8000414f900 [aiod_kick taskq] 100012 D - 0xfffff8000414f100 [kqueue_ctx taskq] 100013 D - 0xfffff80004c69d00 [pci_hp taskq] 100015 D - 0xfffff80004c69500 [inm_free taskq] 100017 D - 0xfffff80004c66d00 [linuxkpi_irq_wq] 100018 D - 0xfffff80004c66900 [thread taskq] 100019 D - 0xfffff80004c66500 [in6m_free taskq] 100020 D - 0xfffff80004c66100 [linuxkpi_short_wq_0] 100021 D - 0xfffff80004c66100 [linuxkpi_short_wq_1] 100022 D - 0xfffff80004c66100 [linuxkpi_short_wq_2] 100023 D - 0xfffff80004c66100 [linuxkpi_short_wq_3] 100024 D - 0xfffff80004c5dd00 [linuxkpi_long_wq_0] 100025 D - 0xfffff80004c5dd00 [linuxkpi_long_wq_1] 100026 D - 0xfffff80004c5dd00 [linuxkpi_long_wq_2] 100027 D - 0xfffff80004c5dd00 [linuxkpi_long_wq_3] 100034 D - 0xfffff80004c5d900 [firmware taskq] 100038 D - 0xfffff80004c5d500 [crypto_0] 100039 D - 0xfffff80004c5d500 [crypto_1] 100055 D - 0xfffff800153e2d00 [vtnet0 rxq 0] 100056 D - 0xfffff800153e2900 [vtnet0 txq 0] 100057 D - 0xfffff800153e2500 [vtnet0 rxq 1] 100058 D - 0xfffff800153e2100 [vtnet0 txq 1] 100060 D vtbslp 0xfffff8001543c900 [virtio_balloon] 100064 D - 0xfffff800154bfd00 [mca taskq] 100066 D - 0xffffffff81e20771 [deadlkres] 100074 D - 0xfffff800154bf900 [acpi_task_0] 100075 D - 0xfffff800154bf900 [acpi_task_1] 100076 D - 0xfffff800154bf900 [acpi_task_2] 100077 D - 0xfffff80004c5d100 [CAM taskq] db> show all locks Process 44470 (syz-executor.3) thread 0xfffffe0088992e40 (141753) exclusive sleep mutex umtxql (umtxql) r = 0 (0xffffffff8275ea00) locked @ /syzkaller/managers/main/kernel/sys/kern/kern_umtx.c:946 Process 44470 (syz-executor.3) thread 0xfffffe0088c24900 (148412) exclusive sleep mutex sctp-create (inp_create) r = 0 (0xfffff8001e0f3488) locked @ /syzkaller/managers/main/kernel/sys/netinet/sctp_usrreq.c:7011 Process 12 (intr) thread 0xfffffe00042be900 (100033) exclusive sleep mutex sctp-tcb (tcb) r = 0 (0xfffffe00902e81a0) locked @ /syzkaller/managers/main/kernel/sys/netinet/sctp_pcb.c:2143 db> show malloc Type InUse MemUse Requests pcb 3037 14118K 148593 sysctloid 34288 12858K 34355 pf_hash 5 11560K 5 devbuf 4216 6982K 4244 tcp_hpts 5 3219K 5 sctp_atcl 2999 2999K 100781 sctp_stro 1497 2994K 25198 kobj 326 2608K 486 vtbuf 24 2064K 46 sctp_atky 4528 1698K 127012 sctp_athm 2999 1125K 100859 sctp_map 2994 1123K 50396 newblk 18 1041K 44314 vfscache 3 1035K 3 subproc 169 694K 44571 acpica 1674 649K 54806 inodedep 11 530K 43577 callout 2 528K 2 ufs_quota 1 520K 1 vfs_hash 1 520K 1 filedesc 32 502K 86883 intr 4 480K 4 bus 986 377K 3469 BPF 46 337K 127 linker 348 270K 497 DEVFS1 107 214K 124 kdtrace 246 200K 92893 module 510 192K 510 vnet_data 1 176K 1 umtx 396 174K 396 tidhash 3 164K 3 sctp_timw 288 144K 288 pagedep 10 141K 43450 tfo_ccache 1 136K 1 sem 4 120K 4 vmem 3 104K 7 UMA 270 102K 270 audit_evclass 236 89K 294 pf_rule 21 84K 24 mtx_pool 2 80K 2 temp 35 77K 2795 syncache 1 76K 1 routetbl 132 71K 421 msg 4 68K 4 ifaddr 70 66K 72 acpitask 1 64K 1 shm 5 64K 13 ddb_capture 1 64K 1 DEVFS3 126 63K 136 ether_multi 152 57K 162 gtaskqueue 18 57K 18 DEVFS_RULE 56 54K 56 kenv 95 52K 95 eventhandler 133 50K 133 ithread 99 43K 99 lltable 48 42K 170 rman 84 42K 425 KTRACE 101 38K 102 kqueue 77 37K 44489 taskqueue 60 36K 60 proc 3 34K 3 ufs_mount 5 34K 6 bus-sc 33 34K 1684 devstat 4 33K 4 hostcache 1 32K 1 tty 16 32K 16 DEVFSP 80 30K 806 GEOM 60 29K 489 in6_multi 65 29K 65 pwddesc 75 29K 44507 kbdmux 6 28K 6 cred 26 26K 301 ifnet 7 25K 7 plimit 24 24K 499 kcovinfo 64 24K 68 CAM queue 5 21K 1528 selfd 55 21K 586867 pfs_nodes 20 20K 20 proc-args 52 20K 719 UART 12 18K 12 session 35 18K 59 bmsafemap 2 17K 43551 ksem 1 16K 1604 rpc 2 16K 2 shmfd 1 16K 160 pfs_vncache 1 16K 1 ufs_dirhash 24 14K 24 sglist 5 13K 5 MCA 32 12K 32 CAM DEV 3 12K 510 lockf 32 12K 70 Unitno 30 12K 51 vt 11 11K 11 acpisem 28 11K 28 CAM XPT 22 11K 543 inpcbpolicy 26 10K 18355 uidinfo 3 9K 14 pf_ifnet 10 9K 19 memdesc 1 8K 1 ipsec-saq 2 8K 2 evdev 4 8K 4 acpidev 20 8K 20 hhook 15 8K 17 select 19 8K 121 osd 18 7K 13534 mount 16 7K 90 fpukern_ctx 3 6K 3 terminal 11 6K 11 ip6ndp 12 6K 14 sctp_ifa 13 5K 14 ipsecpolicy 2 5K 2 encap_export_host 12 5K 12 clone 9 5K 9 local_apic 1 4K 1 io_apic 1 4K 1 sahead 1 4K 1 secasvar 1 4K 1 nhops 6 4K 8 pci_link 10 4K 10 msi 9 4K 9 DEVFS 9 4K 10 tun 7 4K 7 ipsec 3 3K 3 in_multi 6 3K 8 nexusdev 7 3K 7 feeder 7 3K 7 procdesc 5 3K 18 sctp_ifn 6 3K 14 mld 6 3K 6 igmp 6 3K 6 toponodes 6 3K 6 prison 6 3K 6 isadev 6 3K 6 sctp_stri 2 2K 3560 softdep 1 2K 1 vnodemarker 2 2K 112 NFSD session 1 2K 1 newdirblk 5 2K 43430 mkdir 5 2K 86860 linux 5 2K 6 CAM periph 4 2K 271 soname 5 2K 65101 filecaps 5 2K 150 crypto 4 2K 75 diradd 4 2K 43506 pfil 4 2K 4 CAM path 4 2K 1034 tcpfunc 3 2K 3 loginclass 3 2K 6 chacha20random 1 1K 1 vnodes 1 1K 2 CAM SIM 2 1K 2 ktls 1 1K 1 cdev 2 1K 2 aesni_data 2 1K 2 freework 2 1K 43447 pf_osfp 2 1K 2 cpus 2 1K 2 atkbddev 2 1K 2 CAM dev queue 2 1K 2 xform 2 1K 822 entropy 2 1K 48 NFSD lckfile 1 1K 1 NFSD V4client 1 1K 1 dirrem 1 1K 43465 freeblks 1 1K 43436 indirdep 1 1K 19 pmchooks 1 1K 1 sctp_vrf 1 1K 1 apmdev 1 1K 1 freefile 1 1K 43456 CAM I/O Scheduler 1 1K 1 vnet_data_free 1 1K 1 vnet 1 1K 1 Per-cpu 1 1K 1 p1003.1b 1 1K 1 acpiintr 1 1K 1 pmc 1 1K 1 sctp_mcore 0 0K 0 sctp_socko 0 0K 29327 sctp_iter 0 0K 292 sctp_mvrf 0 0K 0 sctp_cpal 0 0K 283 sctp_cmsg 0 0K 0 sctp_stre 0 0K 0 sctp_athi 0 0K 0 sctp_a_it 0 0K 9 sctp_aadr 0 0K 0 cdg data 0 0K 0 mqdata 0 0K 0 tcp_do 0 0K 0 tcp_fsb 0 0K 2248 chd data 0 0K 0 cubic data 0 0K 0 dctcp data 0 0K 0 htcp data 0 0K 0 vegas data 0 0K 0 pf_table 0 0K 0 pf_altq 0 0K 0 pf_temp 0 0K 0 NFSCL lckown 0 0K 0 NFSCL client 0 0K 0 madt_table 0 0K 2 smartpqi 0 0K 0 NFSCL deleg 0 0K 0 NFSCL open 0 0K 0 NFSCL owner 0 0K 0 NFS fh 0 0K 0 NFS req 0 0K 0 NFSD usrgroup 0 0K 0 NFSD string 0 0K 0 iavf 0 0K 0 ixl 0 0K 0 NFSD V4lock 0 0K 0 NFSD V4state 0 0K 0 NFSD srvcache 0 0K 0 msdosfs_fat 0 0K 0 msdosfs_mount 0 0K 0 msdosfs_node 0 0K 0 ice-resmgr 0 0K 0 ice-osdep 0 0K 0 ice 0 0K 0 axgbe 0 0K 0 DEVFS4 0 0K 0 DEVFS2 0 0K 0 gntdev 0 0K 0 privcmd_dev 0 0K 0 evtchn_dev 0 0K 0 xen_intr 0 0K 0 xenstore 0 0K 0 ciss_data 0 0K 0 xen_hvm 0 0K 0 legacydrv 0 0K 0 qpidrv 0 0K 0 BACKLIGHT 0 0K 0 xnb 0 0K 0 dmar_idpgtbl 0 0K 0 dmar_dom 0 0K 0 dmar_ctx 0 0K 0 xbbd 0 0K 0 xbd 0 0K 0 Balloon 0 0K 0 isci 0 0K 0 iommu_dmamap 0 0K 0 sysmouse 0 0K 0 hyperv_socket 0 0K 0 bxe_ilt 0 0K 0 xenbus 0 0K 0 vtfont 0 0K 0 vm_fictitious 0 0K 0 ath_hal 0 0K 0 athdev 0 0K 0 ata_pci 0 0K 0 ata_dma 0 0K 0 ata_generic 0 0K 0 amr 0 0K 0 pvscsi 0 0K 0 scsi_da 0 0K 69 ata_da 0 0K 0 scsi_ch 0 0K 0 UMAHash 0 0K 0 scsi_cd 0 0K 0 AHCI driver 0 0K 0 vm_pgdata 0 0K 0 jblocks 0 0K 0 savedino 0 0K 35149 sentinel 0 0K 0 jfsync 0 0K 0 jtrunc 0 0K 0 sbdep 0 0K 55 jsegdep 0 0K 0 jseg 0 0K 0 jfreefrag 0 0K 0 jfreeblk 0 0K 0 jnewblk 0 0K 0 jmvref 0 0K 0 jremref 0 0K 0 jaddref 0 0K 0 freedep 0 0K 0 freefrag 0 0K 9 allocindir 0 0K 0 allocdirect 0 0K 0 ufs_trim 0 0K 0 mactemp 0 0K 0 audit_trigger 0 0K 0 audit_pipe_presel 0 0K 0 audit_pipeent 0 0K 0 audit_pipe 0 0K 0 audit_evname 0 0K 0 audit_bsm 0 0K 0 audit_gidset 0 0K 0 audit_text 0 0K 0 audit_path 0 0K 0 audit_data 0 0K 0 audit_cred 0 0K 0 USBdev 0 0K 0 USB 0 0K 0 agp 0 0K 0 nvme_da 0 0K 0 acpipwr 0 0K 0 acpi_perf 0 0K 0 ktls_ocf 0 0K 0 twsbuf 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5E_TLS 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EN 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5DUMP 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 seq_file 0 0K 0 radix 0 0K 0 idr 0 0K 0 lkpifw 0 0K 0 NLM 0 0K 0 ipsec-spdcache 0 0K 0 ipsec-reg 0 0K 0 ipsec-misc 0 0K 0 ipsecrequest 0 0K 0 ip6opt 0 0K 114 ip6_msource 0 0K 0 ip6_moptions 0 0K 0 in6_mfilter 0 0K 0 frag6 0 0K 0 tcplog 0 0K 0 tcp_hwpace 0 0K 0 twe_commands 0 0K 0 LRO 0 0K 0 newreno data 0 0K 1 ip_msource 0 0K 0 ip_moptions 0 0K 11 in_mfilter 0 0K 0 ipid 0 0K 0 80211scan 0 0K 0 80211ratectl 0 0K 0 80211power 0 0K 0 80211nodeie 0 0K 0 80211node 0 0K 0 80211mesh_gt 0 0K 0 80211mesh_rt 0 0K 0 80211perr 0 0K 0 80211prep 0 0K 0 80211preq 0 0K 0 80211dfs 0 0K 0 80211crypto 0 0K 0 80211vap 0 0K 0 iflib 0 0K 0 vlan 0 0K 0 gif 0 0K 0 ifdescr 0 0K 0 zlib 0 0K 0 fadvise 0 0K 3 VN POLL 0 0K 0 twa_commands 0 0K 0 statfs 0 0K 43676 namei_tracker 0 0K 0 export_host 0 0K 0 cl_savebuf 0 0K 5 tcp_log_dev 0 0K 5905 midi buffers 0 0K 0 mixer 0 0K 0 ac97 0 0K 0 hdacc 0 0K 0 hdac 0 0K 0 hdaa 0 0K 0 acpicmbat 0 0K 0 SIIS driver 0 0K 0 CAM CCB 0 0K 3308 PUC 0 0K 0 ppbusdev 0 0K 0 agtiapi_MemAlloc malloc 0 0K 0 osti_cacheable 0 0K 0 tempbuff 0 0K 0 biobuf 0 0K 0 aios 0 0K 28 lio 0 0K 8 acl 0 0K 0 tempbuff 0 0K 0 mbuf_tag 0 0K 139 ag_tgt_map_t malloc 0 0K 0 ag_slr_map_t malloc 0 0K 0 lDevFlags * malloc 0 0K 0 tiDeviceHandle_t * malloc 0 0K 0 ag_portal_data_t malloc 0 0K 0 ag_device_t malloc 0 0K 0 STLock malloc 0 0K 0 CCB List 0 0K 0 sr_iov 0 0K 0 OCS 0 0K 0 OCS 0 0K 0 nvme 0 0K 0 nvd 0 0K 0 netmap 0 0K 0 mwldev 0 0K 0 MVS driver 0 0K 0 CAM ccb queue 0 0K 0 mrsasbuf 0 0K 0 mpt_user 0 0K 0 mps_user 0 0K 0 accf 0 0K 0 pts 0 0K 0 iov 0 0K 40831 ioctlops 0 0K 738 eventfd 0 0K 0 Witness 0 0K 0 stack 0 0K 0 MPSSAS 0 0K 0 mps 0 0K 0 mpr_user 0 0K 0 MPRSAS 0 0K 0 mpr 0 0K 0 mfibuf 0 0K 0 sbuf 0 0K 288 md_sectors 0 0K 0 firmware 0 0K 0 compressor 0 0K 0 md_disk 0 0K 0 SWAP 0 0K 0 malodev 0 0K 0 LED 0 0K 0 sysctltmp 0 0K 857 sysctl 0 0K 3 ekcd 0 0K 0 dumper 0 0K 0 sendfile 0 0K 0 rctl 0 0K 0 ix_sriov 0 0K 0 aacraidcam 0 0K 0 aacraid_buf 0 0K 0 ix 0 0K 0 ipsbuf 0 0K 0 cache 0 0K 0 iirbuf 0 0K 0 prison_racct 0 0K 0 Fail Points 0 0K 0 sigio 0 0K 1 filedesc_to_leader 0 0K 19 pwd 0 0K 0 tty console 0 0K 0 aaccam 0 0K 0 aacbuf 0 0K 0 zstd 0 0K 0 XZ_DEC 0 0K 0 nvlist 0 0K 0 SCSI ENC 0 0K 0 SCSI sa 0 0K 0 scsi_pass 0 0K 0 isofs_node 0 0K 0 isofs_mount 0 0K 0 tr_raid5_data 0 0K 0 tr_raid1e_data 0 0K 0 tr_raid1_data 0 0K 0 tr_raid0_data 0 0K 0 tr_concat_data 0 0K 0 md_sii_data 0 0K 0 md_promise_data 0 0K 0 md_nvidia_data 0 0K 0 md_jmicron_data 0 0K 0 md_intel_data 0 0K 0 md_ddf_data 0 0K 0 raid_data 0 0K 72 geom_flashmap 0 0K 0 tmpfs dir 0 0K 0 tmpfs name 0 0K 0 tmpfs mount 0 0K 0 NFS FHA 0 0K 0 newnfsmnt 0 0K 0 newnfsclient_req 0 0K 0 NFSCL layrecall 0 0K 0 NFSCL session 0 0K 0 NFSCL sockreq 0 0K 0 NFSCL devinfo 0 0K 0 NFSCL flayout 0 0K 0 NFSCL layout 0 0K 0 NFSD rollback 0 0K 0 NFSCL diroff 0 0K 0 NEWdirectio 0 0K 0 NEWNFSnode 0 0K 0 NFSCL lck 0 0K 0 db> show uma Zone Size Used Free Requests Sleeps Bucket Total Mem XFree mbuf_jumbo_page 4096 8365 875 1241250 0 254 37847040 0 malloc-384 384 34714 96 210762 0 30 13367040 0 malloc-8192 8192 1503 9 70709 0 2 12386304 0 malloc-1024 1024 4519 49 175118 0 16 4677632 0 malloc-1024 1024 4126 10 4652 0 16 4235264 0 malloc-384 384 9402 88 364810 0 30 3644160 0 sctp_asoc 2288 1497 63 25198 0 254 3569280 0 malloc-8192 8192 401 7 44963 0 1 3342336 0 malloc-1024 1024 23 3053 43689 0 16 3149824 0 malloc-2048 2048 1499 13 25209 0 8 3096576 0 mbuf 256 10336 629 2283199 0 254 2807040 0 malloc-1024 1024 0 2632 35199 0 16 2695168 0 RADIX NODE 144 15473 400 1109678 0 63 2285712 0 tcp_log 416 0 5292 113294 0 254 2201472 0 pbuf 2624 0 778 0 0 2 2041472 0 sctp_ep 1280 1500 93 70465 0 254 2039040 0 UMA Slabs 0 112 17339 10 17339 0 126 1943088 0 BUF TRIE 144 207 13261 1610 0 62 1939392 0 malloc-384 384 1684 3056 91804 0 30 1820160 0 mbuf_cluster 2048 840 0 840 0 254 1720320 0 malloc-512 512 21 3059 43555 0 30 1576960 0 malloc-384 384 3357 33 79873 0 30 1301760 0 sctp_raddr 736 1507 176 42318 0 254 1238688 0 malloc-32768 32768 16 7 43440 0 1 753664 0 FFS inode 1160 525 42 43986 0 9 657720 0 socket 944 56 476 90366 0 254 502208 0 malloc-512 512 243 613 95788 0 30 438272 0 malloc-16384 16384 23 2 187 0 1 409600 0 VM OBJECT 264 1391 154 627869 0 31 407880 0 malloc-65536 65536 5 1 181 0 1 393216 0 malloc-4096 4096 12 83 4457 0 2 389120 0 lkpimm 160 1 2324 1 0 62 372000 0 lkpicurr 160 2 2323 2 0 62 372000 0 THREAD 1808 170 28 48415 0 8 357984 0 tcp_bbr_map 128 0 2604 50330 0 126 333312 0 malloc-1024 1024 275 33 49461 0 16 315392 0 256 Bucket 2048 140 14 18597 0 8 315392 0 VNODE 448 563 94 44026 0 30 294336 0 malloc-2048 2048 124 10 450 0 8 274432 0 tcp_inpcb 488 30 530 13522 0 254 273280 0 malloc-384 384 605 65 105390 0 30 257280 0 vmem btag 56 4399 125 4399 0 254 253344 0 tcpcb 1080 14 217 13522 0 254 249480 0 malloc-384 384 527 73 89510 0 30 230400 0 MAP ENTRY 96 1959 435 2402353 0 126 229824 0 malloc-16384 16384 10 4 14 0 1 229376 0 DEVCTL 1024 0 216 132 0 0 221184 0 mbuf_packet 256 238 602 85831 0 254 215040 0 sctp_chunk 152 1170 182 19613 0 254 205504 0 malloc-512 512 335 57 88942 0 30 200704 0 malloc-65536 65536 3 0 3 0 1 196608 0 malloc-65536 65536 0 3 7 0 1 196608 0 malloc-384 384 436 64 587630 0 30 192000 0 UMA Zones 768 242 2 242 0 16 187392 0 ertt_txseginfo 40 0 4545 135552 0 254 181800 0 malloc-8192 8192 18 2 652 0 1 163840 0 malloc-16384 16384 6 3 269 0 1 147456 0 malloc-512 512 262 26 4577 0 30 147456 0 S VFS Cache 104 1036 368 45922 0 126 146016 0 FFS2 dinode 256 525 45 43985 0 62 145920 0 VMSPACE 2544 45 12 44446 0 4 145008 0 malloc-16384 16384 7 1 12 0 1 131072 0 32 Bucket 256 306 204 36372 0 62 130560 0 PROC 1328 74 19 44476 0 8 123504 0 ksiginfo 112 78 966 1809 0 126 116928 0 malloc-4096 4096 25 2 36 0 2 110592 0 filedesc0 1072 75 23 44507 0 8 105056 0 malloc-1024 1024 98 2 104 0 16 102400 0 malloc-32768 32768 2 1 17 0 1 98304 0 malloc-1024 1024 95 1 95 0 16 98304 0 128 Bucket 1024 61 34 8968 0 16 97280 0 g_bio 408 0 230 10664 0 30 93840 0 malloc-8192 8192 6 5 85 0 1 90112 0 UMA Kegs 384 227 6 227 0 30 89472 0 16 Bucket 144 419 169 11041 0 62 84672 0 clpbuf 2624 0 32 35 0 16 83968 0 malloc-512 512 132 12 484 0 30 73728 0 malloc-2048 2048 29 5 33 0 8 69632 0 malloc-65536 65536 1 0 1 0 1 65536 0 malloc-65536 65536 1 0 1 0 1 65536 0 malloc-32768 32768 1 1 122 0 1 65536 0 malloc-8192 8192 2 6 43682 0 1 65536 0 malloc-8192 8192 7 1 9 0 1 65536 0 malloc-1024 1024 57 3 886 0 16 61440 0 64 Bucket 512 51 69 22653 0 30 61440 0 sctp_stream_msg_out 112 501 39 5349 0 254 60480 0 malloc-384 384 64 66 1511 0 30 49920 0 malloc-16384 16384 1 2 6 0 1 49152 0 malloc-4096 4096 9 3 344 0 2 49152 0 malloc-2048 2048 7 17 515 0 8 49152 0 sctp_laddr 48 868 140 39984 0 254 48384 0 pcpu-8 8 4869 763 5608 0 254 45056 0 tcp_bbr_pcb 832 0 54 9404 0 16 44928 0 malloc-4096 4096 3 7 28 0 2 40960 0 malloc-512 512 48 32 163 0 30 40960 0 Files 80 245 255 150075 0 126 40000 0 udp_inpcb 488 6 74 1114 0 254 39040 0 pipe 744 25 25 755 0 16 37200 0 DIRHASH 1024 34 2 34 0 16 36864 0 NAMEI 1024 0 36 194693 0 16 36864 0 udplite_inpcb 488 0 72 1790 0 254 35136 0 malloc-32768 32768 1 0 1 0 1 32768 0 malloc-32768 32768 1 0 1 0 1 32768 0 malloc-32768 32768 1 0 1 0 1 32768 0 malloc-16384 16384 0 2 2 0 1 32768 0 malloc-16384 16384 2 0 2 0 1 32768 0 malloc-4096 4096 6 2 23 0 2 32768 0 pcpu-64 64 487 25 487 0 254 32768 0 sctp_readq 152 0 208 1366 0 254 31616 0 tcp_rack_pcb 832 0 36 2248 0 16 29952 0 malloc-512 512 22 34 44696 0 30 28672 0 TURNSTILE 136 199 11 199 0 62 28560 0 PWD 32 24 858 43522 0 254 28224 0 KNOTE 160 28 147 511735 0 62 28000 0 malloc-8192 8192 2 1 4 0 1 24576 0 2 Bucket 32 116 640 19213 0 254 24192 0 ttyinq 160 135 15 300 0 62 24000 0 8 Bucket 80 38 262 7734 0 126 24000 0 ripcb 488 6 42 1929 0 254 23424 0 ttyoutq 256 72 18 160 0 62 23040 0 malloc-4096 4096 2 3 61 0 2 20480 0 ertt 72 14 266 13522 0 126 20160 0 SLEEPQUEUE 88 199 25 199 0 126 19712 0 Mountpoints 2752 2 5 2 0 4 19264 0 unpcb 256 20 55 1492 0 254 19200 0 malloc-16384 16384 1 0 1 0 1 16384 0 tcp_rack_map 112 0 144 2739 0 126 16128 0 vtnet_tx_hdr 24 0 668 653064 0 254 16032 0 malloc-4096 4096 3 0 3 0 2 12288 0 malloc-2048 2048 1 5 48 0 8 12288 0 malloc-512 512 4 20 68 0 30 12288 0 kenv 258 15 30 1042 0 30 11610 0 malloc-8192 8192 1 0 1 0 1 8192 0 malloc-4096 4096 0 2 162 0 2 8192 0 malloc-2048 2048 0 4 1564 0 8 8192 0 rtentry 176 30 16 34 0 62 8096 0 PGRP 88 35 57 61 0 126 8096 0 rl_entry 40 84 118 84 0 254 8080 0 udpcb 32 6 246 2904 0 254 8064 0 4 Bucket 48 6 162 1380 0 254 8064 0 AIO 208 0 38 73 0 62 7904 0 cryptop 280 0 28 9 0 30 7840 0 AIOCB 552 0 14 72 0 16 7728 0 routing nhops 256 27 3 34 0 62 7680 0 L VFS Cache 320 0 24 3 0 30 7680 0 AIOLIO 272 0 28 8 0 30 7616 0 FPU_save_area 832 1 8 1 0 16 7488 0 cpuset 104 7 55 15 0 126 6448 0 epoch_record pcpu 256 4 12 4 0 62 4096 0 malloc-2048 2048 1 1 1 0 8 4096 0 pcpu-16 16 7 249 7 0 254 4096 0 hostcache 64 1 62 1 0 254 4032 0 syncache 168 0 24 4 0 254 4032 0 ipq 56 0 72 1 0 254 4032 0 AIOP 32 4 122 4 0 254 4032 0 UMA Slabs 1 176 10 12 10 0 62 3872 0 mqnode 416 3 6 3 0 30 3744 0 KMAP ENTRY 96 12 27 12 0 0 3744 0 vmem 1856 1 1 1 0 8 3712 0 tfo_ccache_entries 80 3 29 3 0 126 2560 0 domainset 40 0 63 6 0 254 2520 0 SMR CPU 32 3 60 3 0 254 2016 0 SMR SHARED 24 3 60 3 0 254 1512 0 FFS1 dinode 128 0 0 0 0 126 0 0 da_ccb 544 0 0 0 0 16 0 0 ada_ccb 272 0 0 0 0 30 0 0 swblk 136 0 0 0 0 62 0 0 swpctrie 144 0 0 0 0 62 0 0 sctp_asconf_ack 48 0 0 0 0 254 0 0 sctp_asconf 40 0 0 0 0 254 0 0 cdg_qdiffsample 16 0 0 0 0 254 0 0 pf state scrubs 40 0 0 0 0 254 0 0 pf frag entries 40 0 0 0 0 254 0 0 pf frags 248 0 0 0 0 62 0 0 pf table entries 160 0 0 0 0 62 0 0 pf table entry counters 64 0 0 0 0 254 0 0 pf source nodes 136 0 0 0 0 254 0 0 pf state keys 88 0 0 0 0 126 0 0 pf states 304 0 0 0 0 254 0 0 pf tags 104 0 0 0 0 126 0 0 pf mtags 48 0 0 0 0 254 0 0 tcp_log_node 120 0 0 0 0 126 0 0 tcp_log_bucket 176 0 0 0 0 62 0 0 tcpreass 48 0 0 0 0 254 0 0 tfo 4 0 0 0 0 254 0 0 sackhole 32 0 0 0 0 254 0 0 tcptw 88 0 0 0 0 254 0 0 IPsec SA lft_c 16 0 0 0 0 254 0 0 itimer 352 0 0 0 0 30 0 0 TMPFS node 224 0 0 0 0 62 0 0 NCLNODE 584 0 0 0 0 16 0 0 mqnotifier 216 0 0 0 0 62 0 0 mvdata 64 0 0 0 0 254 0 0 mqueue 248 0 0 0 0 62 0 0 LTS VFS Cache 360 0 0 0 0 30 0 0 STS VFS Cache 144 0 0 0 0 62 0 0 linux_dma_object 24 0 0 0 0 254 0 0 linux_dma_pctrie 144 0 0 0 0 62 0 0 IOMMU_MAP_ENTRY 120 0 0 0 0 126 0 0 ktls_session 128 0 0 0 0 126 0 0 mbuf_jumbo_16k 16384 0 0 0 0 254 0 0 mbuf_jumbo_9k 9216 0 0 0 0 254 0 0 audit_record 1280 0 0 0 0 8 0 0 MAC labels 40 0 0 0 0 254 0 0 vnpbuf 2624 0 0 0 0 64 0 0 mdpbuf 2624 0 0 0 0 3 0 0 nfspbuf 2624 0 0 0 0 16 0 0 swwbuf 2624 0 0 0 0 8 0 0 swrbuf 2624 0 0 0 0 16 0 0 umtx_shm 88 0 0 0 0 126 0 0 umtx pi 96 0 0 0 0 126 0 0 rangeset pctrie nodes 144 0 0 0 0 62 0 0 malloc-65536 65536 0 0 0 0 1 0 0 malloc-65536 65536 0 0 0 0 1 0 0 malloc-65536 65536 0 0 0 0 1 0 0 malloc-32768 32768 0 0 0 0 1 0 0 malloc-32768 32768 0 0 0 0 1 0 0 malloc-2048 2048 0 0 0 0 8 0 0 malloc-256 256 0 0 0 0 62 0 0 malloc-256 256 0 0 0 0 62 0 0 malloc-256 256 0 0 0 0 62 0 0 malloc-256 256 0 0 0 0 62 0 0 malloc-256 256 0 0 0 0 62 0 0 malloc-256 256 0 0 0 0 62 0 0 malloc-256 256 0 0 0 0 62 0 0 malloc-256 256 0 0 0 0 62 0 0 malloc-128 128 0 0 0 0 126 0 0 malloc-128 128 0 0 0 0 126 0 0 malloc-128 128 0 0 0 0 126 0 0 malloc-128 128 0 0 0 0 126 0 0 malloc-128 128 0 0 0 0 126 0 0 malloc-128 128 0 0 0 0 126 0 0 malloc-128 128 0 0 0 0 126 0 0 malloc-128 128 0 0 0 0 126 0 0 malloc-64 64 0 0 0 0 254 0 0 malloc-64 64 0 0 0 0 254 0 0 malloc-64 64 0 0 0 0 254 0 0 malloc-64 64 0 0 0 0 254 0 0 malloc-64 64 0 0 0 0 254 0 0 malloc-64 64 0 0 0 0 254 0 0 malloc-64 64 0 0 0 0 254 0 0 malloc-64 64 0 0 0 0 254 0 0 malloc-32 32 0 0 0 0 254 0 0 malloc-32 32 0 0 0 0 254 0 0 malloc-32 32 0 0 0 0 254 0 0 malloc-32 32 0 0 0 0 254 0 0 malloc-32 32 0 0 0 0 254 0 0 malloc-32 32 0 0 0 0 254 0 0 malloc-32 32 0 0 0 0 254 0 0 malloc-32 32 0 0 0 0 254 0 0 malloc-16 16 0 0 0 0 254 0 0 malloc-16 16 0 0 0 0 254 0 0 malloc-16 16 0 0 0 0 254 0 0 malloc-16 16 0 0 0 0 254 0 0 malloc-16 16 0 0 0 0 254 0 0 malloc-16 16 0 0 0 0 254 0 0 malloc-16 16 0 0 0 0 254 0 0 malloc-16 16 0 0 0 0 254 0 0 pcpu-32 32 0 0 0 0 254 0 0 pcpu-4 4 0 0 0 0 254 0 0 fakepg 104 0 0 0 0 126 0 0 UMA Hash 256 0 0 0 0 62 0 0