panic: mtx_lock() of spin mutex (null) @ /syzkaller/managers/main/kernel/sys/kern/uipc_sockbuf.c:434 cpuid = 0 time = 1622968906 KDB: stack backtrace: db_trace_self_wrapper() at db_trace_self_wrapper+0x47/frame 0xfffffe005164c100 vpanic() at vpanic+0x1c7/frame 0xfffffe005164c160 panic() at panic+0x43/frame 0xfffffe005164c1c0 __mtx_lock_flags() at __mtx_lock_flags+0x202/frame 0xfffffe005164c220 socantrcvmore() at socantrcvmore+0x2c/frame 0xfffffe005164c240 sctp_notify_assoc_change() at sctp_notify_assoc_change+0x5b8/frame 0xfffffe005164c2b0 sctp_process_control() at sctp_process_control+0x9040/frame 0xfffffe005164c740 sctp_common_input_processing() at sctp_common_input_processing+0x7d0/frame 0xfffffe005164c8c0 sctp_input_with_port() at sctp_input_with_port+0x30a/frame 0xfffffe005164c9b0 sctp_input() at sctp_input+0x1f/frame 0xfffffe005164c9d0 ip_input() at ip_input+0x38b/frame 0xfffffe005164ca90 swi_net() at swi_net+0x20d/frame 0xfffffe005164cb10 ithread_loop() at ithread_loop+0x33f/frame 0xfffffe005164cbb0 fork_exit() at fork_exit+0xb3/frame 0xfffffe005164cbf0 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe005164cbf0 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- KDB: enter: panic [ thread pid 12 tid 100033 ] Stopped at kdb_enter+0x67: movq $0,0x1635ede(%rip) db> db> set $lines = 0 db> set $maxwidth = 0 db> show registers cs 0x20 ds 0x3b es 0x3b fs 0x13 gs 0x1b ss 0x28 rax 0x12 rcx 0xffffffff8113c280 vprintf+0x140 rdx 0x1 rbx 0 rsp 0xfffffe005164c0e0 rbp 0xfffffe005164c100 rsi 0 rdi 0xffffffff8113c2b6 vprintf+0x176 r8 0 r9 0x8080808080808080 r10 0xfffffe005164bfd0 r11 0x1ffaefff59c r12 0xffffffff82267ba0 ddb_dbbe r13 0 r14 0xffffffff81a7c9c8 r15 0xffffffff81a7c9c8 rip 0xffffffff811332a7 kdb_enter+0x67 rflags 0x82 kdb_enter+0x67: movq $0,0x1635ede(%rip) db> show proc Process 12 (intr) at 0xfffff80004c6fa60: state: NORMAL uid: 0 gids: 0 parent: pid 0 at 0xffffffff8271c6b0 ABI: null flag: 0x10000284 flag2: 0 reaper: 0xffffffff8271c6b0 reapsubtree: 12 sigparent: 20 vmspace: 0xffffffff8271d310 (map 0xffffffff8271d310) (map.pmap 0xffffffff8271d3d0) (pmap 0xffffffff8271d430) threads: 23 100011 I [swi5: fast taskq] 100014 I [swi6: task queue] 100016 I [swi6: Giant taskq] 100030 I [swi3: vm] 100031 I [swi4: clock (0)] 100032 I [swi4: clock (1)] 100033 Run CPU 0 [swi1: netisr 0] 100046 I [irq24: virtio_pci0] 100047 I [irq25: virtio_pci0] 100048 I [irq26: virtio_pci0] 100049 I [irq27: virtio_pci0] 100050 I [irq28: virtio_pci1] 100051 I [irq29: virtio_pci1] 100052 I [irq30: virtio_pci1] 100053 I [irq31: virtio_pci1] 100054 I [irq32: virtio_pci1] 100059 I [irq10: virtio_pci2] 100061 I [irq1: atkbd0] 100062 I [irq12: psm0] 100063 I [swi0: uart uart++] 100071 I [swi1: pf send] 100084 I [swi1: hpts] 100085 I [swi1: hpts] db> ps pid ppid pgrp uid state wmesg wchan cmd 54522 785 785 0 R (threaded) syz-executor.0 103536 RunQ syz-executor.0 162859 S sbwait 0xfffff800659401c4 syz-executor.0 162868 RunQ syz-executor.0 162869 Run CPU 1 syz-executor.0 48563 1 786 0 S uwait 0xfffff80045162f00 syz-executor.1 48562 1 786 0 S uwait 0xfffff80050eee100 syz-executor.1 48411 1 788 0 S uwait 0xfffff80050eee700 syz-executor.3 48410 1 788 0 S uwait 0xfffff80045d75500 syz-executor.3 39013 1 787 0 S uwait 0xfffff80050d9db00 syz-executor.2 39009 1 787 0 S uwait 0xfffff80050dd2100 syz-executor.2 31843 1 786 0 S uwait 0xfffff80050dcc500 syz-executor.1 31839 1 786 0 S uwait 0xfffff80050eeed00 syz-executor.1 27872 1 788 0 S uwait 0xfffff80045299500 syz-executor.3 27866 1 788 0 S uwait 0xfffff80050eeeb00 syz-executor.3 20415 1 20415 65 Ss select 0xfffff80050e005c0 dhclient 16763 1 16763 0 Ss select 0xfffff80045a32740 dhclient 16760 1 16760 0 Ss select 0xfffff8004569d140 dhclient 16741 1 16741 65 Ss select 0xfffff800455e6140 dhclient 14956 1 786 0 S uwait 0xfffff80050d9d500 syz-executor.1 14953 1 786 0 S uwait 0xfffff80050eee900 syz-executor.1 8905 1 8905 0 Ss select 0xfffff8004551d5c0 dhclient 8896 1 8896 0 Ss select 0xfffff8004569dec0 dhclient 8862 1 8862 65 Ss select 0xfffff8004569d2c0 dhclient 4751 1 4751 0 Ss select 0xfffff80045527ec0 dhclient 4743 1 4743 0 Ss select 0xfffff800453bf5c0 dhclient 4704 1 4704 65 Ss select 0xfffff800450d38c0 dhclient 2638 1 2638 0 Ss select 0xfffff800450eca40 dhclient 2632 1 2632 0 Ss select 0xfffff800452e4ec0 dhclient 2399 0 0 0 DL aiordy 0xfffff80050fd0a60 [aiod4] 2398 0 0 0 DL aiordy 0xfffff80015c4b000 [aiod3] 2397 0 0 0 DL aiordy 0xfffff80045138a60 [aiod2] 2396 0 0 0 DL aiordy 0xfffff80050fd0000 [aiod1] 788 780 788 0 Ss piperd 0xfffff80050b8b000 syz-executor.3 787 780 787 0 Ss piperd 0xfffff80050b8b8b8 syz-executor.2 786 780 786 0 Ss piperd 0xfffff80015f152e8 syz-executor.1 785 780 785 0 Rs syz-executor.0 780 778 778 0 S (threaded) syz-fuzzer 100094 S uwait 0xfffff80015dfed00 syz-fuzzer 100121 S uwait 0xfffff800502eb900 syz-fuzzer 100122 S uwait 0xfffff800502ebb00 syz-fuzzer 100123 S uwait 0xfffff80050427700 syz-fuzzer 100124 S uwait 0xfffff80015d1c500 syz-fuzzer 100125 S uwait 0xfffff80050427900 syz-fuzzer 100126 S uwait 0xfffff80050427b00 syz-fuzzer 100127 S kqread 0xfffff80015e74d00 syz-fuzzer 100263 S uwait 0xfffff80015d1cf00 syz-fuzzer 778 776 778 0 Ss pause 0xfffff80015eb9b10 csh 776 694 776 0 Ss select 0xfffff800504bdbc0 sshd 760 1 760 0 Ss+ ttyin 0xfffff8001541bcb0 getty 759 1 759 0 Ss+ ttyin 0xfffff80015aaa4b0 getty 758 1 758 0 Ss+ ttyin 0xfffff80015aaacb0 getty 757 1 757 0 Ss+ ttyin 0xfffff80015aae4b0 getty 756 1 756 0 Ss+ ttyin 0xfffff80015aaecb0 getty 755 1 755 0 Ss+ ttyin 0xfffff80015a354b0 getty 754 1 754 0 Ss+ ttyin 0xfffff80015a35cb0 getty 753 1 753 0 Ss+ ttyin 0xfffff80015a3c4b0 getty 752 1 752 0 Ss+ ttyin 0xfffff80015a3ccb0 getty 698 1 698 0 Ss nanslp 0xffffffff8273c8e0 cron 694 1 694 0 Ss select 0xfffff8004fb49440 sshd 507 1 507 0 Ss select 0xfffff80050289d40 syslogd 436 1 436 0 Ss select 0xfffff800501e48c0 devd 435 1 435 65 Ss select 0xfffff8004fb495c0 dhclient 350 1 350 0 Ss select 0xfffff800501e45c0 dhclient 347 1 347 0 Ss select 0xfffff80015dba2c0 dhclient 23 0 0 0 DL vlruwt 0xfffff80015c4b530 [vnlru] 22 0 0 0 DL syncer 0xffffffff8282bcd0 [syncer] 21 0 0 0 DL (threaded) [bufdaemon] 100081 D qsleep 0xffffffff8282ada0 [bufdaemon] 100088 D - 0xffffffff8220aec0 [bufspacedaemon-0] 100098 D sdflush 0xfffff80004d594e8 [/ worker] 20 0 0 0 DL psleep 0xffffffff82852c08 [vmdaemon] 19 0 0 0 DL (threaded) [pagedaemon] 100079 D psleep 0xffffffff82847078 [dom0] 100086 D launds 0xffffffff82847084 [laundry: dom0] 100087 D umarcl 0xffffffff815d0a90 [uma] 18 0 0 0 DL - 0xffffffff82570c88 [rand_harvestq] 17 0 0 0 DL waiting 0xffffffff82efa828 [sctp_iterator] 16 0 0 0 DL pftm 0xffffffff830e73c0 [pf purge] 15 0 0 0 DL - 0xffffffff8282845c [soaiod4] 9 0 0 0 DL - 0xffffffff8282845c [soaiod3] 8 0 0 0 DL - 0xffffffff8282845c [soaiod2] 7 0 0 0 DL - 0xffffffff8282845c [soaiod1] 6 0 0 0 DL (threaded) [cam] 100044 D - 0xffffffff82448140 [doneq0] 100045 D - 0xffffffff824480c0 [async] 100078 D - 0xffffffff82447f90 [scanner] 14 0 0 0 DL seqstat 0xfffff80004debc88 [sequencer 00] 5 0 0 0 DL crypto_ 0xfffff80004d3dd80 [crypto returns 1] 4 0 0 0 DL crypto_ 0xfffff80004d3dd30 [crypto returns 0] 3 0 0 0 DL crypto_ 0xffffffff82844560 [crypto] 13 0 0 0 DL (threaded) [geom] 100035 D - 0xffffffff8271c120 [g_event] 100036 D - 0xffffffff8271c128 [g_up] 100037 D - 0xffffffff8271c130 [g_down] 2 0 0 0 DL (threaded) [KTLS] 100028 D - 0xfffff80004c4ad00 [thr_0] 100029 D - 0xfffff80004c4ad80 [thr_1] 12 0 0 0 RL (threaded) [intr] 100011 I [swi5: fast taskq] 100014 I [swi6: task queue] 100016 I [swi6: Giant taskq] 100030 I [swi3: vm] 100031 I [swi4: clock (0)] 100032 I [swi4: clock (1)] 100033 Run CPU 0 [swi1: netisr 0] 100046 I [irq24: virtio_pci0] 100047 I [irq25: virtio_pci0] 100048 I [irq26: virtio_pci0] 100049 I [irq27: virtio_pci0] 100050 I [irq28: virtio_pci1] 100051 I [irq29: virtio_pci1] 100052 I [irq30: virtio_pci1] 100053 I [irq31: virtio_pci1] 100054 I [irq32: virtio_pci1] 100059 I [irq10: virtio_pci2] 100061 I [irq1: atkbd0] 100062 I [irq12: psm0] 100063 I [swi0: uart uart++] 100071 I [swi1: pf send] 100084 I [swi1: hpts] 100085 I [swi1: hpts] 11 0 0 0 RL (threaded) [idle] 100003 CanRun [idle: cpu0] 100004 CanRun [idle: cpu1] 1 0 1 0 SLs wait 0xfffff80004bd7530 [init] 10 0 0 0 DL audit_w 0xffffffff82844a70 [audit] 0 0 0 0 DLs (threaded) [kernel] 100000 D swapin 0xffffffff8271c6b0 [swapper] 100005 D - 0xfffff8000443a500 [if_io_tqg_0] 100006 D - 0xfffff8000443a100 [if_io_tqg_1] 100007 D - 0xfffff80004c73d00 [if_config_tqg_0] 100008 D - 0xfffff80004c73900 [softirq_0] 100009 D - 0xfffff80004c73500 [softirq_1] 100010 D - 0xfffff80004c73100 [aiod_kick taskq] 100012 D - 0xfffff80004c70900 [kqueue_ctx taskq] 100013 D - 0xfffff80004c70500 [pci_hp taskq] 100015 D - 0xfffff80004c66d00 [inm_free taskq] 100017 D - 0xfffff80004c66500 [linuxkpi_irq_wq] 100018 D - 0xfffff80004c66100 [thread taskq] 100019 D - 0xfffff80004c63d00 [in6m_free taskq] 100020 D - 0xfffff80004c63900 [linuxkpi_short_wq_0] 100021 D - 0xfffff80004c63900 [linuxkpi_short_wq_1] 100022 D - 0xfffff80004c63900 [linuxkpi_short_wq_2] 100023 D - 0xfffff80004c63900 [linuxkpi_short_wq_3] 100024 D - 0xfffff80004c63500 [linuxkpi_long_wq_0] 100025 D - 0xfffff80004c63500 [linuxkpi_long_wq_1] 100026 D - 0xfffff80004c63500 [linuxkpi_long_wq_2] 100027 D - 0xfffff80004c63500 [linuxkpi_long_wq_3] 100034 D - 0xfffff80004c63100 [firmware taskq] 100038 D - 0xfffff80004db8d00 [crypto_0] 100039 D - 0xfffff80004db8d00 [crypto_1] 100055 D - 0xfffff80004db8100 [vtnet0 rxq 0] 100056 D - 0xfffff80015361d00 [vtnet0 txq 0] 100057 D - 0xfffff80015361900 [vtnet0 rxq 1] 100058 D - 0xfffff80015361500 [vtnet0 txq 1] 100060 D vtbslp 0xfffff80015372300 [virtio_balloon] 100064 D - 0xfffff80015361100 [mca taskq] 100066 D - 0xffffffff81e29960 [deadlkres] 100074 D - 0xfffff800159c2100 [acpi_task_0] 100075 D - 0xfffff800159c2100 [acpi_task_1] 100076 D - 0xfffff800159c2100 [acpi_task_2] 100077 D - 0xfffff80004db8500 [CAM taskq] db> show all locks Process 54522 (syz-executor.0) thread 0xfffffe00953ff740 (162859) exclusive sx so_rcv_sx (so_rcv_sx) r = 0 (0xfffff80065940168) locked @ /syzkaller/managers/main/kernel/sys/kern/uipc_sockbuf.c:467 Process 54522 (syz-executor.0) thread 0xfffffe0096989e40 (162868) exclusive sleep mutex sctp-create (inp_create) r = 0 (0xfffff80050ff4488) locked @ /syzkaller/managers/main/kernel/sys/netinet/sctp_usrreq.c:7011 Process 54522 (syz-executor.0) thread 0xfffffe009513f020 (162869) shared rw sctp-addr (sctp-addr) r = 0 (0xfffffe00041aff70) locked @ /syzkaller/managers/main/kernel/sys/netinet/sctp_output.c:3427 exclusive sleep mutex sctp-tcb (tcb) r = 0 (0xfffffe009dcd8380) locked @ /syzkaller/managers/main/kernel/sys/netinet/sctp_pcb.c:4339 exclusive sleep mutex sctp-inp (inp) r = 0 (0xfffff8004c85c968) locked @ /syzkaller/managers/main/kernel/sys/netinet/sctp_pcb.c:4326 exclusive sleep mutex sctp-create (inp_create) r = 0 (0xfffff8004c85c988) locked @ /syzkaller/managers/main/kernel/sys/netinet/sctp_usrreq.c:7011 Process 12 (intr) thread 0xfffffe00042bd900 (100033) exclusive sleep mutex sctp-tcb (tcb) r = 0 (0xfffffe009de0c8b0) locked @ /syzkaller/managers/main/kernel/sys/netinet/sctp_pcb.c:2143 db> show malloc Type InUse MemUse Requests sysctloid 34288 12858K 34355 pf_hash 5 11560K 5 devbuf 4216 6982K 4244 tcp_hpts 5 3219K 5 kobj 328 2624K 488 vtbuf 24 2064K 46 pcb 196 1370K 167953 inodedep 766 1285K 53591 newblk 9 1036K 54075 vfscache 3 1035K 3 subproc 170 702K 54625 acpica 1674 649K 54692 filedesc 36 567K 106895 callout 2 528K 2 ufs_quota 1 520K 1 vfs_hash 1 520K 1 intr 4 480K 4 dirrem 757 379K 53486 bus 990 378K 3485 BPF 46 337K 71 freefile 757 284K 53475 linker 348 270K 449 DEVFS1 107 214K 124 kdtrace 247 200K 117400 module 512 192K 512 vnet_data 1 176K 1 umtx 396 174K 396 sctp_timw 328 164K 328 tidhash 3 164K 3 sctp_atcl 159 159K 127281 sctp_stro 75 150K 38273 pagedep 9 140K 53455 tfo_ccache 1 136K 1 sem 4 120K 4 vmem 3 104K 6 UMA 270 102K 270 audit_evclass 236 89K 294 sctp_atky 233 88K 167271 mtx_pool 2 80K 2 temp 35 77K 4776 syncache 1 76K 1 routetbl 132 71K 416 msg 4 68K 4 ifaddr 71 67K 73 acpitask 1 64K 1 ddb_capture 1 64K 1 DEVFS3 126 63K 136 sctp_athm 159 60K 127535 ether_multi 157 59K 167 gtaskqueue 18 57K 18 sctp_map 150 57K 76546 DEVFS_RULE 56 54K 56 kenv 95 52K 95 eventhandler 133 50K 133 ithread 99 43K 99 lltable 48 42K 215 rman 84 42K 425 KTRACE 100 38K 100 kqueue 78 37K 54609 taskqueue 60 36K 60 proc 3 34K 3 ufs_mount 5 34K 6 bus-sc 33 34K 1696 devstat 4 33K 4 hostcache 1 32K 1 tty 16 32K 16 shm 1 32K 16 DEVFSP 82 31K 2118 in6_multi 71 31K 71 pwddesc 76 29K 54510 GEOM 60 29K 489 kbdmux 6 28K 6 cred 26 26K 358 ifnet 7 25K 7 plimit 24 24K 563 kcovinfo 64 24K 68 CAM queue 5 21K 1528 selfd 55 21K 714664 pfs_nodes 20 20K 20 proc-args 52 20K 771 UART 12 18K 12 session 35 18K 87 bmsafemap 2 17K 53595 ksem 1 16K 197 rpc 2 16K 2 shmfd 1 16K 58 pfs_vncache 1 16K 1 ufs_dirhash 24 14K 24 sglist 5 13K 5 MCA 32 12K 32 CAM DEV 3 12K 510 lockf 32 12K 82 Unitno 30 12K 85 vt 11 11K 11 acpisem 28 11K 28 CAM XPT 22 11K 543 uidinfo 3 9K 26 pf_ifnet 10 9K 19 memdesc 1 8K 1 ipsec-saq 2 8K 2 evdev 4 8K 4 acpidev 20 8K 20 hhook 15 8K 17 select 19 8K 142 mount 16 7K 312 fpukern_ctx 3 6K 3 inpcbpolicy 15 6K 11184 terminal 11 6K 11 sctp_ifa 14 6K 15 ip6ndp 12 6K 15 ipsecpolicy 2 5K 2 encap_export_host 12 5K 12 clone 9 5K 9 pf_rule 1 4K 63 local_apic 1 4K 1 io_apic 1 4K 1 sahead 1 4K 1 secasvar 1 4K 1 nhops 6 4K 8 pci_link 10 4K 10 msi 9 4K 9 DEVFS 9 4K 10 tun 7 4K 7 osd 8 4K 6561 ipsec 3 3K 3 mkdir 8 3K 106866 in_multi 6 3K 8 nexusdev 7 3K 7 feeder 7 3K 7 procdesc 5 3K 18 freework 5 3K 53437 sctp_ifn 6 3K 15 mld 6 3K 6 igmp 6 3K 6 soname 6 3K 76018 toponodes 6 3K 6 prison 6 3K 6 isadev 6 3K 6 softdep 1 2K 1 freeblks 4 2K 53436 vnodemarker 2 2K 160 NFSD session 1 2K 1 pf_osfp 5 2K 5 linux 5 2K 6 CAM periph 4 2K 271 filecaps 5 2K 130 crypto 4 2K 18 newdirblk 4 2K 53433 diradd 4 2K 53528 xform 4 2K 3436 pfil 4 2K 4 CAM path 4 2K 1034 tcpfunc 3 2K 3 loginclass 3 2K 6 sctp_stri 1 1K 6169 chacha20random 1 1K 1 vnodes 1 1K 1 CAM SIM 2 1K 2 ktls 1 1K 1 cdev 2 1K 2 aesni_data 2 1K 2 cpus 2 1K 2 atkbddev 2 1K 2 CAM dev queue 2 1K 2 entropy 2 1K 49 NFSD lckfile 1 1K 1 NFSD V4client 1 1K 1 pmchooks 1 1K 1 sctp_vrf 1 1K 1 apmdev 1 1K 1 CAM I/O Scheduler 1 1K 1 vnet_data_free 1 1K 1 vnet 1 1K 1 Per-cpu 1 1K 1 p1003.1b 1 1K 1 acpiintr 1 1K 1 pmc 1 1K 1 pf_table 0 0K 0 pf_altq 0 0K 0 pf_temp 0 0K 0 tcp_do 0 0K 0 tcp_fsb 0 0K 1343 mqdata 0 0K 0 cdg data 0 0K 15 chd data 0 0K 22 cubic data 0 0K 28 dctcp data 0 0K 22 htcp data 0 0K 0 vegas data 0 0K 0 sctp_mcore 0 0K 0 sctp_socko 0 0K 50722 sctp_iter 0 0K 315 sctp_mvrf 0 0K 0 sctp_cpal 0 0K 306 sctp_cmsg 0 0K 0 sctp_stre 0 0K 0 sctp_athi 0 0K 0 sctp_a_it 0 0K 9 sctp_aadr 0 0K 0 NFSCL lckown 0 0K 0 NFSCL client 0 0K 0 madt_table 0 0K 2 smartpqi 0 0K 0 NFSCL deleg 0 0K 0 NFSCL open 0 0K 0 NFSCL owner 0 0K 0 NFS fh 0 0K 0 NFS req 0 0K 0 NFSD usrgroup 0 0K 0 NFSD string 0 0K 0 iavf 0 0K 0 ixl 0 0K 0 NFSD V4lock 0 0K 0 NFSD V4state 0 0K 0 NFSD srvcache 0 0K 0 msdosfs_fat 0 0K 0 msdosfs_mount 0 0K 0 msdosfs_node 0 0K 0 ice-resmgr 0 0K 0 ice-osdep 0 0K 0 ice 0 0K 0 axgbe 0 0K 0 DEVFS4 0 0K 0 DEVFS2 0 0K 0 gntdev 0 0K 0 privcmd_dev 0 0K 0 evtchn_dev 0 0K 0 xen_intr 0 0K 0 xenstore 0 0K 0 ciss_data 0 0K 0 xen_hvm 0 0K 0 legacydrv 0 0K 0 qpidrv 0 0K 0 BACKLIGHT 0 0K 0 xnb 0 0K 0 dmar_idpgtbl 0 0K 0 dmar_dom 0 0K 0 dmar_ctx 0 0K 0 xbbd 0 0K 0 xbd 0 0K 0 Balloon 0 0K 0 isci 0 0K 0 iommu_dmamap 0 0K 0 sysmouse 0 0K 0 hyperv_socket 0 0K 0 bxe_ilt 0 0K 0 xenbus 0 0K 0 vtfont 0 0K 0 vm_fictitious 0 0K 0 ath_hal 0 0K 0 athdev 0 0K 0 ata_pci 0 0K 0 ata_dma 0 0K 0 ata_generic 0 0K 0 amr 0 0K 0 pvscsi 0 0K 0 scsi_da 0 0K 69 ata_da 0 0K 0 scsi_ch 0 0K 0 UMAHash 0 0K 0 scsi_cd 0 0K 0 AHCI driver 0 0K 0 vm_pgdata 0 0K 0 jblocks 0 0K 0 savedino 0 0K 42790 sentinel 0 0K 0 jfsync 0 0K 0 jtrunc 0 0K 0 sbdep 0 0K 77 jsegdep 0 0K 0 jseg 0 0K 0 jfreefrag 0 0K 0 jfreeblk 0 0K 0 jnewblk 0 0K 0 jmvref 0 0K 0 jremref 0 0K 0 jaddref 0 0K 0 freedep 0 0K 0 freefrag 0 0K 7 allocindir 0 0K 0 indirdep 0 0K 10 allocdirect 0 0K 0 ufs_trim 0 0K 0 mactemp 0 0K 0 audit_trigger 0 0K 0 audit_pipe_presel 0 0K 0 audit_pipeent 0 0K 0 audit_pipe 0 0K 0 audit_evname 0 0K 0 audit_bsm 0 0K 0 audit_gidset 0 0K 0 audit_text 0 0K 0 audit_path 0 0K 0 audit_data 0 0K 0 audit_cred 0 0K 0 USBdev 0 0K 0 USB 0 0K 0 agp 0 0K 0 nvme_da 0 0K 0 acpipwr 0 0K 0 acpi_perf 0 0K 0 ktls_ocf 0 0K 0 twsbuf 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5E_TLS 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EN 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5DUMP 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 seq_file 0 0K 0 radix 0 0K 0 idr 0 0K 0 lkpifw 0 0K 0 NLM 0 0K 0 ipsec-spdcache 0 0K 0 ipsec-reg 0 0K 0 ipsec-misc 0 0K 0 ipsecrequest 0 0K 0 ip6opt 0 0K 182 ip6_msource 0 0K 0 ip6_moptions 0 0K 0 in6_mfilter 0 0K 0 frag6 0 0K 0 tcplog 0 0K 0 tcp_hwpace 0 0K 0 twe_commands 0 0K 0 LRO 0 0K 0 newreno data 0 0K 7 ip_msource 0 0K 0 ip_moptions 0 0K 22 in_mfilter 0 0K 0 ipid 0 0K 0 80211scan 0 0K 0 80211ratectl 0 0K 0 80211power 0 0K 0 80211nodeie 0 0K 0 80211node 0 0K 0 80211mesh_gt 0 0K 0 80211mesh_rt 0 0K 0 80211perr 0 0K 0 80211prep 0 0K 0 80211preq 0 0K 0 80211dfs 0 0K 0 80211crypto 0 0K 0 80211vap 0 0K 0 iflib 0 0K 0 vlan 0 0K 0 gif 0 0K 0 ifdescr 0 0K 0 zlib 0 0K 0 fadvise 0 0K 4 VN POLL 0 0K 0 twa_commands 0 0K 0 statfs 0 0K 53696 namei_tracker 0 0K 2 export_host 0 0K 0 cl_savebuf 0 0K 4 tcp_log_dev 0 0K 480 midi buffers 0 0K 0 mixer 0 0K 0 ac97 0 0K 0 hdacc 0 0K 0 hdac 0 0K 0 hdaa 0 0K 0 acpicmbat 0 0K 0 SIIS driver 0 0K 0 CAM CCB 0 0K 3709 PUC 0 0K 0 ppbusdev 0 0K 0 agtiapi_MemAlloc malloc 0 0K 0 osti_cacheable 0 0K 0 tempbuff 0 0K 0 biobuf 0 0K 0 aios 0 0K 6 lio 0 0K 3 acl 0 0K 0 tempbuff 0 0K 0 mbuf_tag 0 0K 177 ag_tgt_map_t malloc 0 0K 0 ag_slr_map_t malloc 0 0K 0 lDevFlags * malloc 0 0K 0 tiDeviceHandle_t * malloc 0 0K 0 ag_portal_data_t malloc 0 0K 0 ag_device_t malloc 0 0K 0 STLock malloc 0 0K 0 CCB List 0 0K 0 sr_iov 0 0K 0 OCS 0 0K 0 OCS 0 0K 0 nvme 0 0K 0 nvd 0 0K 0 netmap 0 0K 0 mwldev 0 0K 0 MVS driver 0 0K 0 CAM ccb queue 0 0K 0 mrsasbuf 0 0K 0 mpt_user 0 0K 0 mps_user 0 0K 0 accf 0 0K 0 pts 0 0K 0 iov 0 0K 44691 ioctlops 0 0K 758 eventfd 0 0K 0 Witness 0 0K 0 stack 0 0K 0 MPSSAS 0 0K 0 mps 0 0K 0 mpr_user 0 0K 0 MPRSAS 0 0K 0 mpr 0 0K 0 mfibuf 0 0K 0 sbuf 0 0K 320 md_sectors 0 0K 0 firmware 0 0K 0 compressor 0 0K 0 md_disk 0 0K 0 SWAP 0 0K 0 malodev 0 0K 0 LED 0 0K 0 sysctltmp 0 0K 1050 sysctl 0 0K 3 ekcd 0 0K 0 dumper 0 0K 0 sendfile 0 0K 0 rctl 0 0K 0 ix_sriov 0 0K 0 aacraidcam 0 0K 0 aacraid_buf 0 0K 0 ix 0 0K 0 ipsbuf 0 0K 0 cache 0 0K 0 iirbuf 0 0K 0 prison_racct 0 0K 0 Fail Points 0 0K 0 sigio 0 0K 20 filedesc_to_leader 0 0K 19 pwd 0 0K 0 tty console 0 0K 0 aaccam 0 0K 0 aacbuf 0 0K 0 zstd 0 0K 0 XZ_DEC 0 0K 0 nvlist 0 0K 0 SCSI ENC 0 0K 0 SCSI sa 0 0K 0 scsi_pass 0 0K 0 isofs_node 0 0K 0 isofs_mount 0 0K 0 tr_raid5_data 0 0K 0 tr_raid1e_data 0 0K 0 tr_raid1_data 0 0K 0 tr_raid0_data 0 0K 0 tr_concat_data 0 0K 0 md_sii_data 0 0K 0 md_promise_data 0 0K 0 md_nvidia_data 0 0K 0 md_jmicron_data 0 0K 0 md_intel_data 0 0K 0 md_ddf_data 0 0K 0 raid_data 0 0K 72 geom_flashmap 0 0K 0 tmpfs dir 0 0K 0 tmpfs name 0 0K 0 tmpfs mount 0 0K 0 NFS FHA 0 0K 0 newnfsmnt 0 0K 0 newnfsclient_req 0 0K 0 NFSCL layrecall 0 0K 0 NFSCL session 0 0K 0 NFSCL sockreq 0 0K 0 NFSCL devinfo 0 0K 0 NFSCL flayout 0 0K 0 NFSCL layout 0 0K 0 NFSD rollback 0 0K 0 NFSCL diroff 0 0K 0 NEWdirectio 0 0K 0 NEWNFSnode 0 0K 0 NFSCL lck 0 0K 0 db> show uma Zone Size Used Free Requests Sleeps Bucket Total Mem XFree mbuf_jumbo_page 4096 8322 1646 1311385 0 254 40828928 0 malloc-8192 8192 489 2196 136144 0 2 21995520 0 malloc-384 384 35505 4425 172354 0 30 15333120 0 malloc-1024 1024 4173 3247 47027 0 16 7598080 0 sctp_asoc 2288 75 2210 38273 0 254 5228080 0 malloc-1024 1024 378 4406 190924 0 16 4898816 0 malloc-2048 2048 79 2201 38896 0 8 4669440 0 malloc-384 384 935 10985 456908 0 30 4577280 0 malloc-1024 1024 787 2913 53620 0 16 3788800 0 mbuf 256 8946 3189 2562493 0 254 3106560 0 sctp_ep 1280 84 2220 81125 0 254 2949120 0 pbuf 2624 0 973 0 0 2 2553152 0 malloc-1024 1024 214 2210 81423 0 16 2482176 0 RADIX NODE 144 16422 403 1382753 0 63 2422800 0 mbuf_cluster 2048 1140 2 1140 0 254 2338816 0 UMA Slabs 0 112 20305 32 20442 0 126 2277744 0 BUF TRIE 144 189 13279 1918 0 62 1939392 0 malloc-512 512 781 2971 53527 0 30 1921024 0 sctp_raddr 736 80 2208 62698 0 254 1683968 0 malloc-384 384 1367 2963 166465 0 30 1662720 0 tcp_log 416 0 2196 35515 0 254 913536 0 malloc-32768 32768 17 7 53445 0 1 786432 0 malloc-384 384 1696 44 53644 0 30 668160 0 FFS inode 1160 526 27 54002 0 8 641480 0 malloc-16384 16384 27 0 27 0 1 442368 0 VM OBJECT 264 1460 100 772394 0 30 411840 0 malloc-384 384 1028 22 46754 0 30 403200 0 256 Bucket 2048 173 17 20556 0 8 389120 0 socket 944 54 354 93855 0 254 385152 0 lkpimm 160 1 2324 1 0 62 372000 0 lkpicurr 160 2 2323 2 0 62 372000 0 THREAD 1808 170 28 62869 0 8 357984 0 malloc-512 512 62 578 54297 0 30 327680 0 vmem btag 56 5618 58 5618 0 254 317856 0 tcp_bbr_map 128 0 2294 67367 0 126 293632 0 mbuf_packet 256 218 922 106912 0 254 291840 0 VNODE 448 564 84 54042 0 30 290304 0 malloc-512 512 536 24 54533 0 30 286720 0 sctp_stream_msg_out 112 2 2446 6795 0 254 274176 0 malloc-384 384 620 70 837930 0 30 264960 0 malloc-65536 65536 4 0 4 0 1 262144 0 malloc-2048 2048 120 4 148 0 8 253952 0 tcp_inpcb 488 4 516 6549 0 254 253760 0 MAP ENTRY 96 2181 423 2945865 0 126 249984 0 32 Bucket 256 768 162 26359 0 62 238080 0 DEVCTL 1024 0 216 132 0 0 221184 0 malloc-65536 65536 1 2 177 0 1 196608 0 ertt_txseginfo 40 0 4747 208391 0 254 189880 0 UMA Zones 768 242 2 242 0 16 187392 0 malloc-4096 4096 2 43 4110 0 2 184320 0 malloc-16384 16384 6 5 43 0 1 180224 0 malloc-8192 8192 18 4 152 0 1 180224 0 malloc-32768 32768 3 2 19 0 1 163840 0 malloc-16384 16384 8 2 13 0 1 163840 0 malloc-384 384 302 108 127524 0 30 157440 0 FFS2 dinode 256 526 74 54001 0 62 153600 0 VMSPACE 2544 49 11 54504 0 4 152640 0 S VFS Cache 104 1044 399 55432 0 126 150072 0 16 Bucket 144 813 111 12215 0 62 133056 0 malloc-65536 65536 1 1 2 0 1 131072 0 malloc-65536 65536 2 0 2 0 1 131072 0 malloc-65536 65536 2 0 2 0 1 131072 0 PROC 1328 75 18 54529 0 8 123504 0 ksiginfo 112 78 966 3295 0 126 116928 0 filedesc0 1072 76 22 54510 0 8 105056 0 malloc-16384 16384 1 5 77 0 1 98304 0 sctp_chunk 152 77 547 27141 0 254 94848 0 malloc-512 512 166 18 2107 0 30 94208 0 128 Bucket 1024 56 35 11176 0 16 93184 0 malloc-8192 8192 9 2 22 0 1 90112 0 UMA Kegs 384 227 6 227 0 30 89472 0 malloc-512 512 105 63 161153 0 30 86016 0 g_bio 408 0 210 12283 0 30 85680 0 clpbuf 2624 0 32 21 0 16 83968 0 tcpcb 1064 4 73 6549 0 254 81928 0 malloc-16384 16384 1 4 313 0 1 81920 0 malloc-1024 1024 65 15 628 0 16 81920 0 malloc-512 512 129 31 1170 0 30 81920 0 malloc-8192 8192 3 6 53699 0 1 73728 0 sctp_laddr 48 164 1264 6770 0 254 68544 0 malloc-32768 32768 0 2 130 0 1 65536 0 malloc-32768 32768 2 0 2 0 1 65536 0 malloc-4096 4096 11 5 1701 0 2 65536 0 malloc-2048 2048 11 21 1910 0 8 65536 0 malloc-1024 1024 62 2 1084 0 16 65536 0 malloc-4096 4096 8 7 532 0 2 61440 0 sctp_readq 152 0 390 2412 0 254 59280 0 64 Bucket 512 49 55 24867 0 30 53248 0 malloc-384 384 114 16 359 0 30 49920 0 malloc-16384 16384 3 0 3 0 1 49152 0 2 Bucket 32 1199 187 26161 0 254 44352 0 Files 80 248 302 165060 0 126 44000 0 DIRHASH 1024 34 6 34 0 16 40960 0 malloc-1024 1024 4 36 6873 0 16 40960 0 malloc-512 512 68 12 7011 0 30 40960 0 malloc-512 512 12 68 1412 0 30 40960 0 pcpu-8 8 4713 407 5428 0 254 40960 0 udplite_inpcb 488 0 80 2832 0 254 39040 0 tcp_rack_pcb 832 0 45 1343 0 16 37440 0 NAMEI 1024 0 36 235582 0 16 36864 0 malloc-2048 2048 3 15 152 0 8 36864 0 8 Bucket 80 316 134 10516 0 126 36000 0 pipe 744 23 22 1078 0 16 33480 0 malloc-32768 32768 1 0 1 0 1 32768 0 malloc-32768 32768 1 0 1 0 1 32768 0 malloc-16384 16384 2 0 2 0 1 32768 0 malloc-16384 16384 2 0 2 0 1 32768 0 malloc-8192 8192 2 2 12 0 1 32768 0 malloc-8192 8192 2 2 876 0 1 32768 0 malloc-4096 4096 6 2 84 0 2 32768 0 malloc-4096 4096 3 5 450 0 2 32768 0 malloc-2048 2048 16 0 16 0 8 32768 0 pcpu-64 64 487 25 487 0 254 32768 0 TURNSTILE 136 199 32 199 0 62 31416 0 tcp_bbr_pcb 832 0 36 3819 0 16 29952 0 malloc-4096 4096 2 5 11 0 2 28672 0 KNOTE 160 28 147 849430 0 62 28000 0 malloc-4096 4096 6 0 6 0 2 24576 0 PWD 32 25 731 53535 0 254 24192 0 ttyinq 160 135 15 300 0 62 24000 0 ripcb 488 5 43 1377 0 254 23424 0 ttyoutq 256 72 18 160 0 62 23040 0 SLEEPQUEUE 88 199 57 199 0 126 22528 0 Mountpoints 2752 2 5 2 0 4 19264 0 malloc-8192 8192 0 2 12 0 1 16384 0 malloc-2048 2048 8 0 8 0 8 16384 0 tcp_rack_map 112 0 144 2021 0 126 16128 0 vtnet_tx_hdr 24 0 668 706161 0 254 16032 0 unpcb 256 20 40 1461 0 254 15360 0 malloc-1024 1024 9 3 14 0 16 12288 0 udp_inpcb 488 6 18 426 0 254 11712 0 kenv 258 15 30 1048 0 30 11610 0 malloc-4096 4096 2 0 2 0 2 8192 0 rtentry 176 31 15 35 0 62 8096 0 tcptw 88 0 92 8 0 254 8096 0 PGRP 88 35 57 89 0 126 8096 0 rl_entry 40 68 134 68 0 254 8080 0 udpcb 32 6 246 3258 0 254 8064 0 AIOP 32 4 248 4 0 254 8064 0 ertt 72 4 108 6549 0 126 8064 0 4 Bucket 48 7 161 1112 0 254 8064 0 AIOCB 552 0 14 79 0 16 7728 0 routing nhops 256 27 3 35 0 62 7680 0 AIOLIO 272 0 28 3 0 30 7616 0 FPU_save_area 832 1 8 1 0 16 7488 0 cpuset 104 7 55 83 0 126 6448 0 domainset 40 0 126 57 0 254 5040 0 epoch_record pcpu 256 4 12 4 0 62 4096 0 malloc-2048 2048 2 0 2 0 8 4096 0 pcpu-16 16 7 249 7 0 254 4096 0 tcp_log_bucket 176 0 23 1 0 62 4048 0 tcpreass 48 0 84 1 0 254 4032 0 hostcache 64 2 61 2 0 254 4032 0 syncache 168 0 24 4 0 254 4032 0 tcp_log_node 120 0 33 2 0 126 3960 0 AIO 208 0 19 57 0 62 3952 0 itimer 352 0 11 2 0 30 3872 0 UMA Slabs 1 176 9 13 9 0 62 3872 0 mqnode 416 3 6 3 0 30 3744 0 KMAP ENTRY 96 12 27 12 0 0 3744 0 vmem 1856 1 1 1 0 8 3712 0 SMR CPU 32 3 60 3 0 254 2016 0 SMR SHARED 24 3 60 3 0 254 1512 0 FFS1 dinode 128 0 0 0 0 126 0 0 da_ccb 544 0 0 0 0 16 0 0 ada_ccb 272 0 0 0 0 30 0 0 swblk 136 0 0 0 0 62 0 0 swpctrie 144 0 0 0 0 62 0 0 cdg_qdiffsample 16 0 0 0 0 254 0 0 sctp_asconf_ack 48 0 0 0 0 254 0 0 sctp_asconf 40 0 0 0 0 254 0 0 pf state scrubs 40 0 0 0 0 254 0 0 pf frag entries 40 0 0 0 0 254 0 0 pf frags 248 0 0 0 0 62 0 0 pf table entries 160 0 0 0 0 62 0 0 pf table entry counters 64 0 0 0 0 254 0 0 pf source nodes 136 0 0 0 0 254 0 0 pf state keys 88 0 0 0 0 126 0 0 pf states 304 0 0 0 0 0 0 0 pf tags 104 0 0 0 0 126 0 0 pf mtags 48 0 0 0 0 254 0 0 tfo_ccache_entries 80 0 0 0 0 126 0 0 tfo 4 0 0 0 0 254 0 0 sackhole 32 0 0 0 0 254 0 0 ipq 56 0 0 0 0 254 0 0 IPsec SA lft_c 16 0 0 0 0 254 0 0 TMPFS node 224 0 0 0 0 62 0 0 NCLNODE 584 0 0 0 0 16 0 0 mqnotifier 216 0 0 0 0 62 0 0 mvdata 64 0 0 0 0 254 0 0 mqueue 248 0 0 0 0 62 0 0 LTS VFS Cache 360 0 0 0 0 30 0 0 L VFS Cache 320 0 0 0 0 30 0 0 STS VFS Cache 144 0 0 0 0 62 0 0 cryptop 280 0 0 0 0 30 0 0 linux_dma_object 24 0 0 0 0 254 0 0 linux_dma_pctrie 144 0 0 0 0 62 0 0 IOMMU_MAP_ENTRY 120 0 0 0 0 126 0 0 ktls_session 128 0 0 0 0 126 0 0 mbuf_jumbo_16k 16384 0 0 0 0 254 0 0 mbuf_jumbo_9k 9216 0 0 0 0 254 0 0 audit_record 1280 0 0 0 0 8 0 0 MAC labels 40 0 0 0 0 254 0 0 vnpbuf 2624 0 0 0 0 64 0 0 mdpbuf 2624 0 0 0 0 3 0 0 nfspbuf 2624 0 0 0 0 16 0 0 swwbuf 2624 0 0 0 0 8 0 0 swrbuf 2624 0 0 0 0 16 0 0 umtx_shm 88 0 0 0 0 126 0 0 umtx pi 96 0 0 0 0 126 0 0 rangeset pctrie nodes 144 0 0 0 0 62 0 0 malloc-65536 65536 0 0 0 0 1 0 0 malloc-65536 65536 0 0 0 0 1 0 0 malloc-65536 65536 0 0 0 0 1 0 0 malloc-32768 32768 0 0 0 0 1 0 0 malloc-32768 32768 0 0 0 0 1 0 0 malloc-8192 8192 0 0 0 0 1 0 0 malloc-2048 2048 0 0 0 0 8 0 0 malloc-256 256 0 0 0 0 62 0 0 malloc-256 256 0 0 0 0 62 0 0 malloc-256 256 0 0 0 0 62 0 0 malloc-256 256 0 0 0 0 62 0 0 malloc-256 256 0 0 0 0 62 0 0 malloc-256 256 0 0 0 0 62 0 0 malloc-256 256 0 0 0 0 62 0 0 malloc-256 256 0 0 0 0 62 0 0 malloc-128 128 0 0 0 0 126 0 0 malloc-128 128 0 0 0 0 126 0 0 malloc-128 128 0 0 0 0 126 0 0 malloc-128 128 0 0 0 0 126 0 0 malloc-128 128 0 0 0 0 126 0 0 malloc-128 128 0 0 0 0 126 0 0 malloc-128 128 0 0 0 0 126 0 0 malloc-128 128 0 0 0 0 126 0 0 malloc-64 64 0 0 0 0 254 0 0 malloc-64 64 0 0 0 0 254 0 0 malloc-64 64 0 0 0 0 254 0 0 malloc-64 64 0 0 0 0 254 0 0 malloc-64 64 0 0 0 0 254 0 0 malloc-64 64 0 0 0 0 254 0 0 malloc-64 64 0 0 0 0 254 0 0 malloc-64 64 0 0 0 0 254 0 0 malloc-32 32 0 0 0 0 254 0 0 malloc-32 32 0 0 0 0 254 0 0 malloc-32 32 0 0 0 0 254 0 0 malloc-32 32 0 0 0 0 254 0 0 malloc-32 32 0 0 0 0 254 0 0 malloc-32 32 0 0 0 0 254 0 0 malloc-32 32 0 0 0 0 254 0 0 malloc-32 32 0 0 0 0 254 0 0 malloc-16 16 0 0 0 0 254 0 0 malloc-16 16 0 0 0 0 254 0 0 malloc-16 16 0 0 0 0 254 0 0 malloc-16 16 0 0 0 0 254 0 0 malloc-16 16 0 0 0 0 254 0 0 malloc-16 16 0 0 0 0 254 0 0 malloc-16 16 0 0 0 0 254 0 0 malloc-16 16 0 0 0 0 254 0 0 pcpu-32 32 0 0 0 0 254 0 0 pcpu-4 4 0 0 0 0 254 0 0 fakepg 104 0 0 0 0 126 0 0 UMA Hash 256 0 0 0 0 62 0 0