panic: mtx_lock() of spin mutex (null) @ /syzkaller/managers/main/kernel/sys/kern/uipc_sockbuf.c:434 cpuid = 0 time = 158 KDB: stack backtrace: db_trace_self_wrapper() at db_trace_self_wrapper+0x47/frame 0xfffffe005164c100 vpanic() at vpanic+0x1c7/frame 0xfffffe005164c160 panic() at panic+0x43/frame 0xfffffe005164c1c0 __mtx_lock_flags() at __mtx_lock_flags+0x202/frame 0xfffffe005164c220 socantrcvmore() at socantrcvmore+0x2c/frame 0xfffffe005164c240 sctp_notify_assoc_change() at sctp_notify_assoc_change+0x5b8/frame 0xfffffe005164c2b0 sctp_process_control() at sctp_process_control+0x9040/frame 0xfffffe005164c740 sctp_common_input_processing() at sctp_common_input_processing+0x7d0/frame 0xfffffe005164c8c0 sctp_input_with_port() at sctp_input_with_port+0x30a/frame 0xfffffe005164c9b0 sctp_input() at sctp_input+0x1f/frame 0xfffffe005164c9d0 ip_input() at ip_input+0x38b/frame 0xfffffe005164ca90 swi_net() at swi_net+0x20d/frame 0xfffffe005164cb10 ithread_loop() at ithread_loop+0x33f/frame 0xfffffe005164cbb0 fork_exit() at fork_exit+0xb3/frame 0xfffffe005164cbf0 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe005164cbf0 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- KDB: enter: panic [ thread pid 12 tid 100033 ] Stopped at kdb_enter+0x67: movq $0,0x1635f0e(%rip) db> db> set $lines = 0 db> set $maxwidth = 0 db> show registers cs 0x20 ds 0x3b es 0x3b fs 0x13 gs 0x1b ss 0x28 rax 0x12 rcx 0xffffffff8113c250 vprintf+0x140 rdx 0x1 rbx 0 rsp 0xfffffe005164c0e0 rbp 0xfffffe005164c100 rsi 0 rdi 0xffffffff8113c286 vprintf+0x176 r8 0 r9 0x8080808080808080 r10 0xfffffe005164bfd0 r11 0x1ffaefff59c r12 0xffffffff82267ba0 ddb_dbbe r13 0 r14 0xffffffff81a7c9c8 r15 0xffffffff81a7c9c8 rip 0xffffffff81133277 kdb_enter+0x67 rflags 0x82 kdb_enter+0x67: movq $0,0x1635f0e(%rip) db> show proc Process 12 (intr) at 0xfffff80004c6fa60: state: NORMAL uid: 0 gids: 0 parent: pid 0 at 0xffffffff8271c6b0 ABI: null flag: 0x10000284 flag2: 0 reaper: 0xffffffff8271c6b0 reapsubtree: 12 sigparent: 20 vmspace: 0xffffffff8271d310 (map 0xffffffff8271d310) (map.pmap 0xffffffff8271d3d0) (pmap 0xffffffff8271d430) threads: 23 100011 I [swi5: fast taskq] 100014 I [swi6: task queue] 100016 I [swi6: Giant taskq] 100030 I [swi3: vm] 100031 I [swi4: clock (0)] 100032 I [swi4: clock (1)] 100033 Run CPU 0 [swi1: netisr 0] 100046 I [irq24: virtio_pci0] 100047 I [irq25: virtio_pci0] 100048 I [irq26: virtio_pci0] 100049 I [irq27: virtio_pci0] 100050 I [irq28: virtio_pci1] 100051 I [irq29: virtio_pci1] 100052 I [irq30: virtio_pci1] 100053 I [irq31: virtio_pci1] 100054 I [irq32: virtio_pci1] 100059 I [irq10: virtio_pci2] 100061 I [irq1: atkbd0] 100062 I [irq12: psm0] 100063 I [swi0: uart uart++] 100071 I [swi1: pf send] 100084 I [swi1: hpts] 100085 I [swi1: hpts] db> ps pid ppid pgrp uid state wmesg wchan cmd 56112 815 815 0 S (threaded) syz-executor.3 149759 S nanslp 0xffffffff8273c8e0 syz-executor.3 166166 S connec 0xfffff8004c25dbf0 syz-executor.3 166168 S uwait 0xfffff8006f4b3900 syz-executor.3 56111 800 800 0 S (threaded) syz-executor.2 161005 S nanslp 0xffffffff8273c8e1 syz-executor.2 166165 S connec 0xfffff8004c203840 syz-executor.2 166167 S uwait 0xfffff80015ebb500 syz-executor.2 56110 782 782 0 S (threaded) syz-executor.0 161117 S nanslp 0xffffffff8273c8e0 syz-executor.0 166163 S connec 0xfffff8004c1b7bf0 syz-executor.0 166164 S uwait 0xfffff8005bbf1700 syz-executor.0 56109 783 783 0 R (threaded) syz-executor.1 157572 Run CPU 1 syz-executor.1 166162 S accept 0xfffff8004c2658b8 syz-executor.1 166169 RunQ syz-executor.1 166170 S accept 0xfffff8004c1bd8b8 syz-executor.1 53628 1 800 0 SV uwait 0xfffff8005bd6f900 syz-executor.2 53619 1 800 0 SV uwait 0xfffff8006f526700 syz-executor.2 53610 1 800 0 SV uwait 0xfffff8006f4b3b00 syz-executor.2 53599 1 800 0 SV uwait 0xfffff8005bf9eb00 syz-executor.2 53591 1 800 0 SV uwait 0xfffff8006f4b3300 syz-executor.2 53582 1 800 0 SV uwait 0xfffff8006f001500 syz-executor.2 53570 1 800 0 SV uwait 0xfffff80042025b00 syz-executor.2 53554 1 800 0 SV uwait 0xfffff80015e73f00 syz-executor.2 51653 1 815 0 SV uwait 0xfffff8005bbf1100 syz-executor.3 51652 1 815 0 SV sbwait 0xfffff8005be60924 syz-executor.3 38997 1 38997 0 SVs uwait 0xfffff8005bfc1900 syz-executor.2 38990 1 38990 0 SVs sigwait 0xfffffe009d94b000 syz-executor.2 38909 1 800 0 SV sigwait 0xfffffe009d943000 syz-executor.2 38903 1 800 0 SV sigwait 0xfffffe009d936000 syz-executor.2 38248 1 782 0 SV sigwait 0xfffffe009d8f0000 syz-executor.0 38241 1 782 0 SV sigwait 0xfffffe009d8fc000 syz-executor.0 38196 1 815 0 SV sigwait 0xfffffe009d8d6000 syz-executor.3 38189 1 815 0 SV sigwait 0xfffffe009d8c7000 syz-executor.3 37390 1 800 0 SV sigwait 0xfffffe009d85c000 syz-executor.2 37382 1 800 0 SV sigwait 0xfffffe00955bf000 syz-executor.2 36980 1 800 0 SV sigwait 0xfffffe0095309000 syz-executor.2 36974 1 800 0 SV sigwait 0xfffffe00955f5000 syz-executor.2 36685 1 783 0 SV sigwait 0xfffffe00955bb000 syz-executor.1 36677 1 783 0 SV sigwait 0xfffffe009d814000 syz-executor.1 36159 1 800 0 S uwait 0xfffff8005b28d900 syz-executor.2 36158 1 800 0 S uwait 0xfffff8005bba6500 syz-executor.2 36150 1 800 0 S uwait 0xfffff8005bba6700 syz-executor.2 36149 1 800 0 S uwait 0xfffff80015d73900 syz-executor.2 15813 1 15813 65 Ss select 0xfffff80042f222c0 dhclient 13799 1 13799 0 Ss select 0xfffff80042ed6a40 dhclient 13794 1 13794 0 Ss select 0xfffff80042f28140 dhclient 13775 1 13775 65 Ss select 0xfffff80042f28ec0 dhclient 10656 1 10656 0 Ss select 0xfffff8005beec8c0 dhclient 10653 1 10653 0 Ss select 0xfffff80042e76440 dhclient 10634 1 10634 65 Ss select 0xfffff80042dab440 dhclient 7038 1 7038 0 Ss select 0xfffff800420d3740 dhclient 7035 1 7035 0 Ss select 0xfffff8004221dbc0 dhclient 7010 1 7010 65 Ss select 0xfffff80042078bc0 dhclient 1674 1 1674 0 Ss select 0xfffff80042078a40 dhclient 1671 1 1671 0 Ss select 0xfffff8005beecd40 dhclient 888 1 800 0 SV uwait 0xfffff8005bc5cf00 syz-executor.2 853 1 853 0 SVs uwait 0xfffff80015e73d00 syz-executor.2 852 0 0 0 DL aiordy 0xfffff8005be11000 [aiod4] 851 0 0 0 DL aiordy 0xfffff8005be5e530 [aiod3] 849 0 0 0 DL aiordy 0xfffff8005be11530 [aiod2] 848 0 0 0 DL aiordy 0xfffff8005be11a60 [aiod1] 815 780 815 0 Ss nanslp 0xffffffff8273c8e0 syz-executor.3 800 780 800 0 Ss nanslp 0xffffffff8273c8e0 syz-executor.2 783 780 783 0 Ss nanslp 0xffffffff8273c8e0 syz-executor.1 782 780 782 0 Ss nanslp 0xffffffff8273c8e1 syz-executor.0 780 778 778 0 S (threaded) syz-fuzzer 100092 S uwait 0xfffff800153e1100 syz-fuzzer 100116 S uwait 0xfffff80015da1500 syz-fuzzer 100117 S uwait 0xfffff80015da1700 syz-fuzzer 100118 S uwait 0xfffff8005b2d0700 syz-fuzzer 100119 S uwait 0xfffff8005b2d0900 syz-fuzzer 100120 S uwait 0xfffff8005b2d0b00 syz-fuzzer 100121 S uwait 0xfffff8005ba0bd00 syz-fuzzer 100122 S kqread 0xfffff80015ec7d00 syz-fuzzer 100127 S uwait 0xfffff80015e73300 syz-fuzzer 100128 S uwait 0xfffff8005ba0bb00 syz-fuzzer 778 776 778 0 Ss pause 0xfffff80015df35e0 csh 776 694 776 0 Ss select 0xfffff8005b1dad40 sshd 760 1 760 0 Ss+ ttyin 0xfffff8001541bcb0 getty 759 1 759 0 Ss+ ttyin 0xfffff80015aab4b0 getty 758 1 758 0 Ss+ ttyin 0xfffff80015aabcb0 getty 757 1 757 0 Ss+ ttyin 0xfffff80015a334b0 getty 756 1 756 0 Ss+ ttyin 0xfffff80015a33cb0 getty 755 1 755 0 Ss+ ttyin 0xfffff80015a364b0 getty 754 1 754 0 Ss+ ttyin 0xfffff80015a36cb0 getty 753 1 753 0 Ss+ ttyin 0xfffff80015a3d4b0 getty 752 1 752 0 Ss+ ttyin 0xfffff80015a3dcb0 getty 698 1 698 0 Ss nanslp 0xffffffff8273c8e1 cron 694 1 694 0 Ss select 0xfffff8005b1c2ec0 sshd 507 1 507 0 Ss select 0xfffff8005a23d2c0 syslogd 436 1 436 0 Ss select 0xfffff8005b277440 devd 435 1 435 65 Ss select 0xfffff8005b2772c0 dhclient 350 1 350 0 Ss select 0xfffff8005b277740 dhclient 347 1 347 0 Ss select 0xfffff8005b1dd740 dhclient 23 0 0 0 DL vlruwt 0xfffff80015c4c530 [vnlru] 22 0 0 0 DL syncer 0xffffffff8282bcd0 [syncer] 21 0 0 0 DL (threaded) [bufdaemon] 100081 D qsleep 0xffffffff8282ada0 [bufdaemon] 100086 D - 0xffffffff8220aec0 [bufspacedaemon-0] 100100 D sdflush 0xfffff80004d594e8 [/ worker] 20 0 0 0 DL psleep 0xffffffff82852c08 [vmdaemon] 19 0 0 0 DL (threaded) [pagedaemon] 100079 D psleep 0xffffffff82847078 [dom0] 100087 D launds 0xffffffff82847084 [laundry: dom0] 100088 D umarcl 0xffffffff815d0a60 [uma] 18 0 0 0 DL - 0xffffffff82570c88 [rand_harvestq] 17 0 0 0 DL waiting 0xffffffff82e52828 [sctp_iterator] 16 0 0 0 DL pftm 0xffffffff8303e3c0 [pf purge] 15 0 0 0 DL - 0xffffffff8282845c [soaiod4] 9 0 0 0 DL - 0xffffffff8282845c [soaiod3] 8 0 0 0 DL - 0xffffffff8282845c [soaiod2] 7 0 0 0 DL - 0xffffffff8282845c [soaiod1] 6 0 0 0 DL (threaded) [cam] 100044 D - 0xffffffff82448140 [doneq0] 100045 D - 0xffffffff824480c0 [async] 100078 D - 0xffffffff82447f90 [scanner] 14 0 0 0 DL seqstat 0xfffff80004debc88 [sequencer 00] 5 0 0 0 DL crypto_ 0xfffff80004d3dd80 [crypto returns 1] 4 0 0 0 DL crypto_ 0xfffff80004d3dd30 [crypto returns 0] 3 0 0 0 DL crypto_ 0xffffffff82844560 [crypto] 13 0 0 0 DL (threaded) [geom] 100035 D - 0xffffffff8271c120 [g_event] 100036 D - 0xffffffff8271c128 [g_up] 100037 D - 0xffffffff8271c130 [g_down] 2 0 0 0 DL (threaded) [KTLS] 100028 D - 0xfffff80004c4ad00 [thr_0] 100029 D - 0xfffff80004c4ad80 [thr_1] 12 0 0 0 RL (threaded) [intr] 100011 I [swi5: fast taskq] 100014 I [swi6: task queue] 100016 I [swi6: Giant taskq] 100030 I [swi3: vm] 100031 I [swi4: clock (0)] 100032 I [swi4: clock (1)] 100033 Run CPU 0 [swi1: netisr 0] 100046 I [irq24: virtio_pci0] 100047 I [irq25: virtio_pci0] 100048 I [irq26: virtio_pci0] 100049 I [irq27: virtio_pci0] 100050 I [irq28: virtio_pci1] 100051 I [irq29: virtio_pci1] 100052 I [irq30: virtio_pci1] 100053 I [irq31: virtio_pci1] 100054 I [irq32: virtio_pci1] 100059 I [irq10: virtio_pci2] 100061 I [irq1: atkbd0] 100062 I [irq12: psm0] 100063 I [swi0: uart uart++] 100071 I [swi1: pf send] 100084 I [swi1: hpts] 100085 I [swi1: hpts] 11 0 0 0 RL (threaded) [idle] 100003 CanRun [idle: cpu0] 100004 CanRun [idle: cpu1] 1 0 1 0 SLs wait 0xfffff80004bd7530 [init] 10 0 0 0 DL audit_w 0xffffffff82844a70 [audit] 0 0 0 0 DLs (threaded) [kernel] 100000 D swapin 0xffffffff8271c6b0 [swapper] 100005 D - 0xfffff8000443a500 [if_io_tqg_0] 100006 D - 0xfffff8000443a100 [if_io_tqg_1] 100007 D - 0xfffff80004c73d00 [if_config_tqg_0] 100008 D - 0xfffff80004c73900 [softirq_0] 100009 D - 0xfffff80004c73500 [softirq_1] 100010 D - 0xfffff80004c73100 [aiod_kick taskq] 100012 D - 0xfffff80004c70900 [kqueue_ctx taskq] 100013 D - 0xfffff80004c70500 [pci_hp taskq] 100015 D - 0xfffff80004c66d00 [inm_free taskq] 100017 D - 0xfffff80004c66500 [linuxkpi_irq_wq] 100018 D - 0xfffff80004c66100 [thread taskq] 100019 D - 0xfffff80004c63d00 [in6m_free taskq] 100020 D - 0xfffff80004c63900 [linuxkpi_short_wq_0] 100021 D - 0xfffff80004c63900 [linuxkpi_short_wq_1] 100022 D - 0xfffff80004c63900 [linuxkpi_short_wq_2] 100023 D - 0xfffff80004c63900 [linuxkpi_short_wq_3] 100024 D - 0xfffff80004c63500 [linuxkpi_long_wq_0] 100025 D - 0xfffff80004c63500 [linuxkpi_long_wq_1] 100026 D - 0xfffff80004c63500 [linuxkpi_long_wq_2] 100027 D - 0xfffff80004c63500 [linuxkpi_long_wq_3] 100034 D - 0xfffff80004c63100 [firmware taskq] 100038 D - 0xfffff80004db8d00 [crypto_0] 100039 D - 0xfffff80004db8d00 [crypto_1] 100055 D - 0xfffff80004db8100 [vtnet0 rxq 0] 100056 D - 0xfffff80015361d00 [vtnet0 txq 0] 100057 D - 0xfffff80015361900 [vtnet0 rxq 1] 100058 D - 0xfffff80015361500 [vtnet0 txq 1] 100060 D vtbslp 0xfffff80015372300 [virtio_balloon] 100064 D - 0xfffff80015361100 [mca taskq] 100066 D - 0xffffffff81e29990 [deadlkres] 100074 D - 0xfffff80015a50100 [acpi_task_0] 100075 D - 0xfffff80015a50100 [acpi_task_1] 100076 D - 0xfffff80015a50100 [acpi_task_2] 100077 D - 0xfffff80004db8500 [CAM taskq] db> show all locks Process 56109 (syz-executor.1) thread 0xfffffe009d9e1ac0 (166169) exclusive sleep mutex sctp-create (inp_create) r = 0 (0xfffff8006fe82488) locked @ /syzkaller/managers/main/kernel/sys/netinet/sctp_usrreq.c:7011 Process 51652 (syz-executor.3) thread 0xfffffe00950f3c80 (153084) exclusive sx so_rcv_sx (so_rcv_sx) r = 0 (0xfffff8005be608c8) locked @ /syzkaller/managers/main/kernel/sys/kern/uipc_sockbuf.c:467 Process 12 (intr) thread 0xfffffe00042bd900 (100033) exclusive sleep mutex sctp-tcb (tcb) r = 0 (0xfffffe009da45380) locked @ /syzkaller/managers/main/kernel/sys/netinet/sctp_pcb.c:2143 db> show malloc Type InUse MemUse Requests sysctloid 34288 12858K 34355 pf_hash 5 11560K 5 devbuf 4216 6982K 4244 pcb 631 3324K 159064 tcp_hpts 5 3219K 5 kobj 328 2624K 488 vtbuf 24 2064K 46 filedesc 78 1247K 110107 pf_rule 281 1106K 1109 newblk 9 1036K 55700 vfscache 3 1035K 3 subproc 212 891K 56229 acpica 1674 649K 54692 sctp_atcl 586 586K 113037 inodedep 54 573K 55176 sctp_stro 284 568K 36772 callout 2 528K 2 ufs_quota 1 520K 1 vfs_hash 1 520K 1 intr 4 480K 4 bus 990 378K 3485 BPF 46 337K 78 sctp_atky 870 327K 151888 linker 348 270K 537 kdtrace 296 236K 122284 sctp_athm 586 220K 113657 DEVFS1 107 214K 124 sctp_map 568 213K 73544 umtx 440 193K 440 module 512 192K 512 vnet_data 1 176K 1 tidhash 3 164K 3 sctp_timw 296 148K 296 pagedep 9 140K 55044 tfo_ccache 1 136K 1 sem 4 120K 4 shm 11 112K 121 UMA 270 102K 270 audit_evclass 236 89K 294 routetbl 150 84K 551 mtx_pool 2 80K 2 temp 35 77K 5010 syncache 1 76K 1 vmem 3 72K 5 msg 4 68K 4 ifaddr 71 67K 73 acpitask 1 64K 1 ddb_capture 1 64K 1 DEVFS3 126 63K 136 ether_multi 157 59K 167 gtaskqueue 18 57K 18 DEVFS_RULE 56 54K 56 kenv 95 52K 95 eventhandler 133 50K 133 kqueue 99 45K 56193 ithread 99 43K 99 rman 84 42K 425 KTRACE 100 38K 102 lltable 43 37K 260 pwddesc 97 37K 56111 taskqueue 60 36K 60 proc 3 34K 3 ufs_mount 5 34K 6 bus-sc 33 34K 1696 devstat 4 33K 4 hostcache 1 32K 1 tty 16 32K 16 in6_multi 71 31K 71 DEVFSP 80 30K 2338 GEOM 60 29K 489 kbdmux 6 28K 6 cred 27 27K 373 ifnet 7 25K 7 plimit 24 24K 592 kcovinfo 64 24K 68 dirrem 45 23K 55076 CAM queue 5 21K 1528 selfd 55 21K 731075 pfs_nodes 20 20K 20 proc-args 52 20K 767 session 38 19K 66 UART 12 18K 12 freefile 45 17K 55048 bmsafemap 2 17K 55181 ksem 1 16K 2820 pf_table 4 16K 32 rpc 2 16K 2 shmfd 1 16K 48 pfs_vncache 1 16K 1 ufs_dirhash 24 14K 24 sglist 5 13K 5 MCA 32 12K 32 CAM DEV 3 12K 510 lockf 32 12K 86 vt 11 11K 11 acpisem 28 11K 28 CAM XPT 22 11K 543 Unitno 27 11K 1127 inpcbpolicy 26 10K 18386 uidinfo 3 9K 20 pf_ifnet 10 9K 104 memdesc 1 8K 1 ipsec-saq 2 8K 2 evdev 4 8K 4 acpidev 20 8K 20 hhook 15 8K 17 select 19 8K 119 mount 16 7K 108 fpukern_ctx 3 6K 3 terminal 11 6K 11 sctp_ifa 14 6K 15 ip6ndp 12 6K 15 ipsecpolicy 2 5K 2 encap_export_host 12 5K 12 clone 9 5K 9 local_apic 1 4K 1 io_apic 1 4K 1 sahead 1 4K 1 secasvar 1 4K 1 nhops 6 4K 8 pci_link 10 4K 10 osd 9 4K 10704 msi 9 4K 9 DEVFS 9 4K 10 tun 7 4K 7 ipsec 3 3K 3 mkdir 8 3K 110040 in_multi 6 3K 8 soname 8 3K 87946 nexusdev 7 3K 7 feeder 7 3K 7 procdesc 5 3K 18 freework 5 3K 55031 sctp_ifn 6 3K 15 mld 6 3K 6 igmp 6 3K 6 toponodes 6 3K 6 prison 6 3K 6 isadev 6 3K 6 softdep 1 2K 1 freeblks 4 2K 55017 vnodemarker 2 2K 156 NFSD session 1 2K 1 linux 5 2K 6 CAM periph 4 2K 271 filecaps 5 2K 138 crypto 4 2K 1799 newdirblk 4 2K 55020 diradd 4 2K 55120 pfil 4 2K 4 CAM path 4 2K 1034 tcpfunc 3 2K 3 loginclass 3 2K 6 sctp_stri 1 1K 1197 chacha20random 1 1K 1 vnodes 1 1K 2 CAM SIM 2 1K 2 ktls 1 1K 1 cdev 2 1K 2 aesni_data 2 1K 2 pf_osfp 2 1K 2 cpus 2 1K 2 atkbddev 2 1K 2 CAM dev queue 2 1K 2 xform 2 1K 4472 entropy 2 1K 51 NFSD lckfile 1 1K 1 NFSD V4client 1 1K 1 pmchooks 1 1K 1 sctp_vrf 1 1K 1 apmdev 1 1K 1 CAM I/O Scheduler 1 1K 1 vnet_data_free 1 1K 1 vnet 1 1K 1 Per-cpu 1 1K 1 p1003.1b 1 1K 1 acpiintr 1 1K 1 pmc 1 1K 1 mqdata 0 0K 0 cdg data 0 0K 0 pf_altq 0 0K 0 pf_temp 0 0K 0 htcp data 0 0K 9 dctcp data 0 0K 0 tcp_do 0 0K 0 tcp_fsb 0 0K 913 cubic data 0 0K 0 chd data 0 0K 5 sctp_mcore 0 0K 0 sctp_socko 0 0K 50801 sctp_iter 0 0K 120 sctp_mvrf 0 0K 0 sctp_cpal 0 0K 109 sctp_cmsg 0 0K 0 sctp_stre 0 0K 0 sctp_athi 0 0K 0 sctp_a_it 0 0K 11 sctp_aadr 0 0K 264 vegas data 0 0K 0 NFSCL lckown 0 0K 0 NFSCL client 0 0K 0 madt_table 0 0K 2 smartpqi 0 0K 0 NFSCL deleg 0 0K 0 NFSCL open 0 0K 0 NFSCL owner 0 0K 0 NFS fh 0 0K 0 NFS req 0 0K 0 NFSD usrgroup 0 0K 0 NFSD string 0 0K 0 iavf 0 0K 0 ixl 0 0K 0 NFSD V4lock 0 0K 0 NFSD V4state 0 0K 0 NFSD srvcache 0 0K 0 msdosfs_fat 0 0K 0 msdosfs_mount 0 0K 0 msdosfs_node 0 0K 0 ice-resmgr 0 0K 0 ice-osdep 0 0K 0 ice 0 0K 0 axgbe 0 0K 0 DEVFS4 0 0K 0 DEVFS2 0 0K 0 gntdev 0 0K 0 privcmd_dev 0 0K 0 evtchn_dev 0 0K 0 xen_intr 0 0K 0 xenstore 0 0K 0 ciss_data 0 0K 0 xen_hvm 0 0K 0 legacydrv 0 0K 0 qpidrv 0 0K 0 BACKLIGHT 0 0K 0 xnb 0 0K 0 dmar_idpgtbl 0 0K 0 dmar_dom 0 0K 0 dmar_ctx 0 0K 0 xbbd 0 0K 0 xbd 0 0K 0 Balloon 0 0K 0 isci 0 0K 0 iommu_dmamap 0 0K 0 sysmouse 0 0K 0 hyperv_socket 0 0K 0 bxe_ilt 0 0K 0 xenbus 0 0K 0 vtfont 0 0K 0 vm_fictitious 0 0K 0 ath_hal 0 0K 0 athdev 0 0K 0 ata_pci 0 0K 0 ata_dma 0 0K 0 ata_generic 0 0K 0 amr 0 0K 0 pvscsi 0 0K 0 scsi_da 0 0K 69 ata_da 0 0K 0 scsi_ch 0 0K 0 UMAHash 0 0K 0 scsi_cd 0 0K 0 AHCI driver 0 0K 0 vm_pgdata 0 0K 0 jblocks 0 0K 0 savedino 0 0K 47778 sentinel 0 0K 0 jfsync 0 0K 0 jtrunc 0 0K 0 sbdep 0 0K 75 jsegdep 0 0K 0 jseg 0 0K 0 jfreefrag 0 0K 0 jfreeblk 0 0K 0 jnewblk 0 0K 0 jmvref 0 0K 0 jremref 0 0K 0 jaddref 0 0K 0 freedep 0 0K 0 freefrag 0 0K 7 allocindir 0 0K 0 indirdep 0 0K 30 allocdirect 0 0K 0 ufs_trim 0 0K 0 mactemp 0 0K 0 audit_trigger 0 0K 0 audit_pipe_presel 0 0K 0 audit_pipeent 0 0K 0 audit_pipe 0 0K 0 audit_evname 0 0K 0 audit_bsm 0 0K 0 audit_gidset 0 0K 0 audit_text 0 0K 0 audit_path 0 0K 0 audit_data 0 0K 0 audit_cred 0 0K 0 USBdev 0 0K 0 USB 0 0K 0 agp 0 0K 0 nvme_da 0 0K 0 acpipwr 0 0K 0 acpi_perf 0 0K 0 ktls_ocf 0 0K 0 twsbuf 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5E_TLS 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EN 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5DUMP 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 seq_file 0 0K 0 radix 0 0K 0 idr 0 0K 0 lkpifw 0 0K 0 NLM 0 0K 0 ipsec-spdcache 0 0K 0 ipsec-reg 0 0K 0 ipsec-misc 0 0K 0 ipsecrequest 0 0K 0 ip6opt 0 0K 171 ip6_msource 0 0K 0 ip6_moptions 0 0K 0 in6_mfilter 0 0K 0 frag6 0 0K 0 tcplog 0 0K 0 tcp_hwpace 0 0K 0 twe_commands 0 0K 0 LRO 0 0K 0 newreno data 0 0K 0 ip_msource 0 0K 0 ip_moptions 0 0K 4 in_mfilter 0 0K 0 ipid 0 0K 0 80211scan 0 0K 0 80211ratectl 0 0K 0 80211power 0 0K 0 80211nodeie 0 0K 0 80211node 0 0K 0 80211mesh_gt 0 0K 0 80211mesh_rt 0 0K 0 80211perr 0 0K 0 80211prep 0 0K 0 80211preq 0 0K 0 80211dfs 0 0K 0 80211crypto 0 0K 0 80211vap 0 0K 0 iflib 0 0K 0 vlan 0 0K 0 gif 0 0K 0 ifdescr 0 0K 0 zlib 0 0K 0 fadvise 0 0K 10 VN POLL 0 0K 0 twa_commands 0 0K 0 statfs 0 0K 55292 namei_tracker 0 0K 12 export_host 0 0K 0 cl_savebuf 0 0K 4 tcp_log_dev 0 0K 2512 midi buffers 0 0K 0 mixer 0 0K 0 ac97 0 0K 0 hdacc 0 0K 0 hdac 0 0K 0 hdaa 0 0K 0 acpicmbat 0 0K 0 SIIS driver 0 0K 0 CAM CCB 0 0K 3830 PUC 0 0K 0 ppbusdev 0 0K 0 agtiapi_MemAlloc malloc 0 0K 0 osti_cacheable 0 0K 0 tempbuff 0 0K 0 biobuf 0 0K 0 aios 0 0K 50 lio 0 0K 14 acl 0 0K 0 tempbuff 0 0K 0 mbuf_tag 0 0K 177 ag_tgt_map_t malloc 0 0K 0 ag_slr_map_t malloc 0 0K 0 lDevFlags * malloc 0 0K 0 tiDeviceHandle_t * malloc 0 0K 0 ag_portal_data_t malloc 0 0K 0 ag_device_t malloc 0 0K 0 STLock malloc 0 0K 0 CCB List 0 0K 0 sr_iov 0 0K 0 OCS 0 0K 0 OCS 0 0K 0 nvme 0 0K 0 nvd 0 0K 0 netmap 0 0K 0 mwldev 0 0K 0 MVS driver 0 0K 0 CAM ccb queue 0 0K 0 mrsasbuf 0 0K 0 mpt_user 0 0K 0 mps_user 0 0K 0 accf 0 0K 0 pts 0 0K 0 iov 0 0K 42629 ioctlops 0 0K 4085 eventfd 0 0K 0 Witness 0 0K 0 stack 0 0K 0 MPSSAS 0 0K 0 mps 0 0K 0 mpr_user 0 0K 0 MPRSAS 0 0K 0 mpr 0 0K 0 mfibuf 0 0K 0 sbuf 0 0K 288 md_sectors 0 0K 0 firmware 0 0K 0 compressor 0 0K 0 md_disk 0 0K 0 SWAP 0 0K 0 malodev 0 0K 0 LED 0 0K 0 sysctltmp 0 0K 985 sysctl 0 0K 3 ekcd 0 0K 0 dumper 0 0K 0 sendfile 0 0K 0 rctl 0 0K 0 ix_sriov 0 0K 0 aacraidcam 0 0K 0 aacraid_buf 0 0K 0 ix 0 0K 0 ipsbuf 0 0K 0 cache 0 0K 0 iirbuf 0 0K 0 prison_racct 0 0K 0 Fail Points 0 0K 0 sigio 0 0K 4 filedesc_to_leader 0 0K 1 pwd 0 0K 0 tty console 0 0K 0 aaccam 0 0K 0 aacbuf 0 0K 0 zstd 0 0K 0 XZ_DEC 0 0K 0 nvlist 0 0K 0 SCSI ENC 0 0K 0 SCSI sa 0 0K 0 scsi_pass 0 0K 0 isofs_node 0 0K 0 isofs_mount 0 0K 0 tr_raid5_data 0 0K 0 tr_raid1e_data 0 0K 0 tr_raid1_data 0 0K 0 tr_raid0_data 0 0K 0 tr_concat_data 0 0K 0 md_sii_data 0 0K 0 md_promise_data 0 0K 0 md_nvidia_data 0 0K 0 md_jmicron_data 0 0K 0 md_intel_data 0 0K 0 md_ddf_data 0 0K 0 raid_data 0 0K 72 geom_flashmap 0 0K 0 tmpfs dir 0 0K 0 tmpfs name 0 0K 0 tmpfs mount 0 0K 0 NFS FHA 0 0K 0 newnfsmnt 0 0K 0 newnfsclient_req 0 0K 0 NFSCL layrecall 0 0K 0 NFSCL session 0 0K 0 NFSCL sockreq 0 0K 0 NFSCL devinfo 0 0K 0 NFSCL flayout 0 0K 0 NFSCL layout 0 0K 0 NFSD rollback 0 0K 0 NFSCL diroff 0 0K 0 NEWdirectio 0 0K 0 NEWNFSnode 0 0K 0 NFSCL lck 0 0K 0 db> show uma Zone Size Used Free Requests Sleeps Bucket Total Mem XFree mbuf_jumbo_page 4096 8333 2913 1067321 0 254 46063616 0 malloc-384 384 35967 463 175046 0 30 13989120 0 malloc-1024 1024 4173 3511 51994 0 16 7868416 0 malloc-8192 8192 727 191 131787 0 1 7520256 0 malloc-1024 1024 75 3865 55205 0 16 4034560 0 mbuf 256 9124 3251 2134386 0 254 3168000 0 pbuf 2624 0 973 0 0 2 2553152 0 tcp_log 416 0 5247 119175 0 254 2182752 0 RADIX NODE 144 14029 360 1388418 0 64 2072016 0 malloc-512 512 69 3899 55100 0 30 2031616 0 UMA Slabs 0 112 17908 17 17995 0 126 2007600 0 BUF TRIE 144 187 13281 1866 0 62 1939392 0 malloc-384 384 704 3816 171446 0 30 1735680 0 mbuf_cluster 2048 825 1 825 0 254 1691648 0 sctp_asoc 2288 284 446 36772 0 254 1670240 0 malloc-32768 32768 38 4 55051 0 1 1376256 0 malloc-1024 1024 833 423 180179 0 16 1286144 0 malloc-4096 4096 286 6 3211 0 2 1196032 0 malloc-384 384 2017 1083 434445 0 30 1190400 0 malloc-2048 2048 288 226 36946 0 8 1052672 0 sctp_ep 1280 301 488 75185 0 254 1009920 0 malloc-1024 1024 447 213 75549 0 16 675840 0 malloc-384 384 1696 44 53644 0 30 668160 0 FFS inode 1160 540 27 55590 0 8 657720 0 sctp_raddr 736 301 513 40471 0 254 599104 0 VM OBJECT 264 1786 59 796768 0 30 487080 0 malloc-16384 16384 27 2 52 0 1 475136 0 malloc-65536 65536 2 5 21 0 1 458752 0 malloc-384 384 1025 15 47415 0 30 399360 0 THREAD 1808 198 22 66170 0 8 397760 0 socket 944 86 318 95195 0 254 381376 0 lkpimm 160 1 2324 1 0 62 372000 0 lkpicurr 160 2 2323 2 0 62 372000 0 malloc-512 512 62 594 56007 0 30 335872 0 256 Bucket 2048 149 11 20302 0 8 327680 0 tcp_bbr_map 128 3 2508 116952 0 126 321408 0 MAP ENTRY 96 2887 263 3039368 0 126 302400 0 VNODE 448 578 88 55630 0 30 298368 0 malloc-512 512 526 50 58145 0 30 294912 0 tcp_inpcb 488 5 571 10692 0 254 281088 0 malloc-8192 8192 28 5 242 0 1 270336 0 malloc-65536 65536 4 0 4 0 1 262144 0 malloc-384 384 620 60 863997 0 30 261120 0 malloc-2048 2048 120 2 139 0 8 249856 0 DEVCTL 1024 0 216 132 0 0 221184 0 mbuf_packet 256 150 675 116428 0 254 211200 0 vmem btag 56 3474 42 3474 0 254 196896 0 malloc-65536 65536 1 2 177 0 1 196608 0 ertt_txseginfo 40 0 4747 299457 0 254 189880 0 UMA Zones 768 242 2 242 0 16 187392 0 VMSPACE 2544 61 8 56061 0 4 175536 0 malloc-4096 4096 2 40 6319 0 2 172032 0 malloc-32768 32768 4 1 19 0 1 163840 0 malloc-16384 16384 6 4 44 0 1 163840 0 malloc-16384 16384 8 2 13 0 1 163840 0 PROC 1328 96 18 56112 0 8 151392 0 S VFS Cache 104 1084 359 58243 0 126 150072 0 malloc-384 384 312 78 141663 0 30 149760 0 FFS2 dinode 256 540 30 55590 0 62 145920 0 32 Bucket 256 383 172 40692 0 62 142080 0 malloc-65536 65536 0 2 288 0 1 131072 0 malloc-65536 65536 2 0 2 0 1 131072 0 filedesc0 1072 97 22 56111 0 8 127568 0 ksiginfo 112 99 945 2200 0 126 116928 0 malloc-16384 16384 1 6 151 0 1 114688 0 tcpcb 1064 5 93 10692 0 254 104272 0 16 Bucket 144 529 171 10923 0 62 100800 0 128 Bucket 1024 60 35 15980 0 16 97280 0 malloc-512 512 166 18 1994 0 30 94208 0 malloc-512 512 113 71 168227 0 30 94208 0 sctp_chunk 152 186 412 5030 0 254 90896 0 malloc-8192 8192 9 2 15 0 1 90112 0 UMA Kegs 384 227 6 227 0 30 89472 0 malloc-1024 1024 74 10 712 0 16 86016 0 clpbuf 2624 0 32 21 0 16 83968 0 g_bio 408 0 200 12767 0 30 81600 0 malloc-512 512 132 20 1525 0 30 77824 0 malloc-1024 1024 62 6 1019 0 16 69632 0 malloc-65536 65536 0 1 1 0 1 65536 0 malloc-32768 32768 0 2 130 0 1 65536 0 malloc-32768 32768 2 0 2 0 1 65536 0 malloc-8192 8192 3 5 55295 0 1 65536 0 malloc-2048 2048 11 21 1911 0 8 65536 0 malloc-4096 4096 8 7 532 0 2 61440 0 pcpu-8 8 6969 199 14229 0 254 57344 0 malloc-2048 2048 16 10 218 0 8 53248 0 64 Bucket 512 48 56 17340 0 30 53248 0 malloc-384 384 114 16 376 0 30 49920 0 malloc-16384 16384 3 0 3 0 1 49152 0 malloc-16384 16384 1 2 384 0 1 49152 0 malloc-8192 8192 2 4 16 0 1 49152 0 pipe 744 29 36 2407 0 16 48360 0 Files 80 292 258 175840 0 126 44000 0 malloc-8192 8192 2 3 1986 0 1 40960 0 malloc-4096 4096 6 4 73 0 2 40960 0 malloc-1024 1024 4 36 1887 0 16 40960 0 malloc-512 512 68 12 6623 0 30 40960 0 sctp_stream_msg_out 112 151 209 1839 0 254 40320 0 sctp_laddr 48 204 636 4907 0 254 40320 0 DIRHASH 1024 34 2 34 0 16 36864 0 NAMEI 1024 0 36 248760 0 16 36864 0 malloc-2048 2048 3 15 152 0 8 36864 0 TURNSTILE 136 221 31 221 0 62 34272 0 malloc-32768 32768 1 0 1 0 1 32768 0 malloc-32768 32768 1 0 1 0 1 32768 0 malloc-16384 16384 2 0 2 0 1 32768 0 malloc-16384 16384 2 0 2 0 1 32768 0 malloc-4096 4096 3 5 450 0 2 32768 0 pcpu-64 64 487 25 487 0 254 32768 0 udplite_inpcb 488 10 54 6564 0 254 31232 0 tcp_bbr_pcb 832 1 35 8191 0 16 29952 0 malloc-4096 4096 6 1 43 0 2 28672 0 PWD 32 40 842 55123 0 254 28224 0 KNOTE 160 28 147 631377 0 62 28000 0 8 Bucket 80 69 281 9432 0 126 28000 0 malloc-4096 4096 6 0 6 0 2 24576 0 malloc-512 512 12 36 1412 0 30 24576 0 2 Bucket 32 198 558 22687 0 254 24192 0 ttyinq 160 135 15 300 0 62 24000 0 ttyoutq 256 72 18 160 0 62 23040 0 SLEEPQUEUE 88 221 35 221 0 126 22528 0 tcp_rack_pcb 832 0 27 913 0 16 22464 0 Mountpoints 2752 2 5 2 0 4 19264 0 malloc-8192 8192 0 2 12 0 1 16384 0 malloc-2048 2048 8 0 8 0 8 16384 0 vtnet_tx_hdr 24 0 668 674896 0 254 16032 0 ripcb 488 5 27 740 0 254 15616 0 unpcb 256 26 34 1562 0 254 15360 0 malloc-1024 1024 9 3 14 0 16 12288 0 ertt 72 5 163 10692 0 126 12096 0 udp_inpcb 488 6 18 390 0 254 11712 0 kenv 258 15 30 1048 0 30 11610 0 routing nhops 256 27 18 35 0 62 11520 0 malloc-4096 4096 2 0 2 0 2 8192 0 pf tags 104 6 72 73 0 126 8112 0 rtentry 176 31 15 35 0 62 8096 0 PGRP 88 38 54 66 0 126 8096 0 rl_entry 40 79 123 79 0 254 8080 0 tcp_rack_map 112 0 72 777 0 126 8064 0 udpcb 32 16 236 6954 0 254 8064 0 AIOP 32 4 248 4 0 254 8064 0 4 Bucket 48 6 162 1190 0 254 8064 0 sctp_readq 152 0 52 39 0 254 7904 0 AIO 208 0 38 115 0 62 7904 0 cryptop 280 0 28 412 0 30 7840 0 AIOCB 552 0 14 154 0 16 7728 0 AIOLIO 272 0 28 14 0 30 7616 0 FPU_save_area 832 1 8 1 0 16 7488 0 cpuset 104 7 55 2157 0 126 6448 0 domainset 40 0 126 1610 0 254 5040 0 epoch_record pcpu 256 4 12 4 0 62 4096 0 malloc-2048 2048 2 0 2 0 8 4096 0 pcpu-16 16 7 249 7 0 254 4096 0 hostcache 64 1 62 1 0 254 4032 0 syncache 168 0 24 4 0 254 4032 0 itimer 352 0 11 2 0 30 3872 0 UMA Slabs 1 176 13 9 13 0 62 3872 0 mqnode 416 3 6 3 0 30 3744 0 KMAP ENTRY 96 12 27 12 0 0 3744 0 vmem 1856 1 1 1 0 8 3712 0 SMR CPU 32 3 60 3 0 254 2016 0 SMR SHARED 24 3 60 3 0 254 1512 0 FFS1 dinode 128 0 0 0 0 126 0 0 da_ccb 544 0 0 0 0 16 0 0 ada_ccb 272 0 0 0 0 30 0 0 swblk 136 0 0 0 0 62 0 0 swpctrie 144 0 0 0 0 62 0 0 cdg_qdiffsample 16 0 0 0 0 254 0 0 sctp_asconf_ack 48 0 0 0 0 254 0 0 sctp_asconf 40 0 0 0 0 254 0 0 pf state scrubs 40 0 0 0 0 254 0 0 pf frag entries 40 0 0 0 0 254 0 0 pf frags 248 0 0 0 0 62 0 0 pf table entries 160 0 0 0 0 62 0 0 pf table entry counters 64 0 0 0 0 254 0 0 pf source nodes 136 0 0 0 0 254 0 0 pf state keys 88 0 0 0 0 126 0 0 pf states 304 0 0 0 0 254 0 0 pf mtags 48 0 0 0 0 254 0 0 tcp_log_node 120 0 0 0 0 126 0 0 tcp_log_bucket 176 0 0 0 0 62 0 0 tcpreass 48 0 0 0 0 254 0 0 tfo_ccache_entries 80 0 0 0 0 126 0 0 tfo 4 0 0 0 0 254 0 0 sackhole 32 0 0 0 0 254 0 0 tcptw 88 0 0 0 0 254 0 0 ipq 56 0 0 0 0 254 0 0 IPsec SA lft_c 16 0 0 0 0 254 0 0 mqnotifier 216 0 0 0 0 62 0 0 mvdata 64 0 0 0 0 254 0 0 mqueue 248 0 0 0 0 62 0 0 TMPFS node 224 0 0 0 0 62 0 0 NCLNODE 584 0 0 0 0 16 0 0 LTS VFS Cache 360 0 0 0 0 30 0 0 L VFS Cache 320 0 0 0 0 30 0 0 STS VFS Cache 144 0 0 0 0 62 0 0 linux_dma_object 24 0 0 0 0 254 0 0 linux_dma_pctrie 144 0 0 0 0 62 0 0 IOMMU_MAP_ENTRY 120 0 0 0 0 126 0 0 ktls_session 128 0 0 0 0 126 0 0 mbuf_jumbo_16k 16384 0 0 0 0 254 0 0 mbuf_jumbo_9k 9216 0 0 0 0 254 0 0 audit_record 1280 0 0 0 0 8 0 0 MAC labels 40 0 0 0 0 254 0 0 vnpbuf 2624 0 0 0 0 64 0 0 mdpbuf 2624 0 0 0 0 3 0 0 nfspbuf 2624 0 0 0 0 16 0 0 swwbuf 2624 0 0 0 0 8 0 0 swrbuf 2624 0 0 0 0 16 0 0 umtx_shm 88 0 0 0 0 126 0 0 umtx pi 96 0 0 0 0 126 0 0 rangeset pctrie nodes 144 0 0 0 0 62 0 0 malloc-65536 65536 0 0 0 0 1 0 0 malloc-65536 65536 0 0 0 0 1 0 0 malloc-32768 32768 0 0 0 0 1 0 0 malloc-32768 32768 0 0 0 0 1 0 0 malloc-8192 8192 0 0 0 0 1 0 0 malloc-2048 2048 0 0 0 0 8 0 0 malloc-256 256 0 0 0 0 62 0 0 malloc-256 256 0 0 0 0 62 0 0 malloc-256 256 0 0 0 0 62 0 0 malloc-256 256 0 0 0 0 62 0 0 malloc-256 256 0 0 0 0 62 0 0 malloc-256 256 0 0 0 0 62 0 0 malloc-256 256 0 0 0 0 62 0 0 malloc-256 256 0 0 0 0 62 0 0 malloc-128 128 0 0 0 0 126 0 0 malloc-128 128 0 0 0 0 126 0 0 malloc-128 128 0 0 0 0 126 0 0 malloc-128 128 0 0 0 0 126 0 0 malloc-128 128 0 0 0 0 126 0 0 malloc-128 128 0 0 0 0 126 0 0 malloc-128 128 0 0 0 0 126 0 0 malloc-128 128 0 0 0 0 126 0 0 malloc-64 64 0 0 0 0 254 0 0 malloc-64 64 0 0 0 0 254 0 0 malloc-64 64 0 0 0 0 254 0 0 malloc-64 64 0 0 0 0 254 0 0 malloc-64 64 0 0 0 0 254 0 0 malloc-64 64 0 0 0 0 254 0 0 malloc-64 64 0 0 0 0 254 0 0 malloc-64 64 0 0 0 0 254 0 0 malloc-32 32 0 0 0 0 254 0 0 malloc-32 32 0 0 0 0 254 0 0 malloc-32 32 0 0 0 0 254 0 0 malloc-32 32 0 0 0 0 254 0 0 malloc-32 32 0 0 0 0 254 0 0 malloc-32 32 0 0 0 0 254 0 0 malloc-32 32 0 0 0 0 254 0 0 malloc-32 32 0 0 0 0 254 0 0 malloc-16 16 0 0 0 0 254 0 0 malloc-16 16 0 0 0 0 254 0 0 malloc-16 16 0 0 0 0 254 0 0 malloc-16 16 0 0 0 0 254 0 0 malloc-16 16 0 0 0 0 254 0 0 malloc-16 16 0 0 0 0 254 0 0 malloc-16 16 0 0 0 0 254 0 0 malloc-16 16 0 0 0 0 254 0 0 pcpu-32 32 0 0 0 0 254 0 0 pcpu-4 4 0 0 0 0 254 0 0 fakepg 104 0 0 0 0 126 0 0 UMA Hash 256 0 0 0 0 62 0 0