panic: mtx_lock() of spin mutex (null) @ /syzkaller/managers/main/kernel/sys/kern/uipc_sockbuf.c:434 cpuid = 1 time = 336 KDB: stack backtrace: db_trace_self_wrapper() at db_trace_self_wrapper+0x47/frame 0xfffffe005164c100 vpanic() at vpanic+0x1c7/frame 0xfffffe005164c160 panic() at panic+0x43/frame 0xfffffe005164c1c0 __mtx_lock_flags() at __mtx_lock_flags+0x202/frame 0xfffffe005164c220 socantrcvmore() at socantrcvmore+0x2c/frame 0xfffffe005164c240 sctp_notify_assoc_change() at sctp_notify_assoc_change+0x5b8/frame 0xfffffe005164c2b0 sctp_process_control() at sctp_process_control+0x9040/frame 0xfffffe005164c740 sctp_common_input_processing() at sctp_common_input_processing+0x7d0/frame 0xfffffe005164c8c0 sctp_input_with_port() at sctp_input_with_port+0x30a/frame 0xfffffe005164c9b0 sctp_input() at sctp_input+0x1f/frame 0xfffffe005164c9d0 ip_input() at ip_input+0x38b/frame 0xfffffe005164ca90 swi_net() at swi_net+0x20d/frame 0xfffffe005164cb10 ithread_loop() at ithread_loop+0x33f/frame 0xfffffe005164cbb0 fork_exit() at fork_exit+0xb3/frame 0xfffffe005164cbf0 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe005164cbf0 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- KDB: enter: panic [ thread pid 12 tid 100033 ] Stopped at kdb_enter+0x67: movq $0,0x1635d4e(%rip) db> db> set $lines = 0 db> set $maxwidth = 0 db> show registers cs 0x20 ds 0x3b es 0x3b fs 0x13 gs 0x1b ss 0x28 rax 0x12 rcx 0xffffffff8113c410 vprintf+0x140 rdx 0x1 rbx 0 rsp 0xfffffe005164c0e0 rbp 0xfffffe005164c100 rsi 0 rdi 0xffffffff8113c446 vprintf+0x176 r8 0 r9 0x8080808080808080 r10 0xfffffe005164bfd0 r11 0x1ffaefff59c r12 0xffffffff82267ba0 ddb_dbbe r13 0 r14 0xffffffff81a7ca38 r15 0xffffffff81a7ca38 rip 0xffffffff81133437 kdb_enter+0x67 rflags 0x82 kdb_enter+0x67: movq $0,0x1635d4e(%rip) db> show proc Process 12 (intr) at 0xfffff80004c72a60: state: NORMAL uid: 0 gids: 0 parent: pid 0 at 0xffffffff8271c6b0 ABI: null flag: 0x10000284 flag2: 0 reaper: 0xffffffff8271c6b0 reapsubtree: 12 sigparent: 20 vmspace: 0xffffffff8271d310 (map 0xffffffff8271d310) (map.pmap 0xffffffff8271d3d0) (pmap 0xffffffff8271d430) threads: 23 100011 I [swi5: fast taskq] 100014 I [swi6: task queue] 100016 I [swi6: Giant taskq] 100030 I [swi3: vm] 100031 I [swi4: clock (0)] 100032 I [swi4: clock (1)] 100033 Run CPU 1 [swi1: netisr 0] 100046 I [irq24: virtio_pci0] 100047 I [irq25: virtio_pci0] 100048 I [irq26: virtio_pci0] 100049 I [irq27: virtio_pci0] 100050 I [irq28: virtio_pci1] 100051 I [irq29: virtio_pci1] 100052 I [irq30: virtio_pci1] 100053 I [irq31: virtio_pci1] 100054 I [irq32: virtio_pci1] 100059 I [irq10: virtio_pci2] 100061 I [irq1: atkbd0] 100062 I [irq12: psm0] 100063 I [swi0: uart uart++] 100071 I [swi1: pf send] 100084 I [swi1: hpts] 100085 I [swi1: hpts] db> ps pid ppid pgrp uid state wmesg wchan cmd 9396 786 786 0 R (threaded) syz-executor.1 100680 Run CPU 0 syz-executor.1 110225 RunQ syz-executor.1 110226 S uwait 0xfffff80015cfd100 syz-executor.1 9394 785 785 0 S (threaded) syz-executor.0 103476 S nanslp 0xffffffff8273c8e0 syz-executor.0 110222 S connec 0xfffff80047c71bf0 syz-executor.0 110224 S uwait 0xfffff80015cfe100 syz-executor.0 9391 818 818 0 R (threaded) syz-executor.3 104730 RunQ syz-executor.3 110215 S accept 0xfffff80042b3d508 syz-executor.3 110219 S uwait 0xfffff80015cfdf00 syz-executor.3 9390 796 796 0 S (threaded) syz-executor.2 100260 S nanslp 0xffffffff8273c8e0 syz-executor.2 110214 S accept 0xfffff80042b8cc68 syz-executor.2 6487 1 786 0 S uwait 0xfffff80047cb6500 syz-executor.1 6485 1 786 0 S uwait 0xfffff80015cfb700 syz-executor.1 6070 1 796 0 SV uwait 0xfffff80015cfd900 syz-executor.2 6067 1 796 0 SV uwait 0xfffff800425e1f00 syz-executor.2 4095 0 0 0 DL aiordy 0xfffff8004223ea60 [aiod4] 4094 0 0 0 DL aiordy 0xfffff80047f1ca60 [aiod3] 4092 0 0 0 DL aiordy 0xfffff80047d63000 [aiod2] 4090 0 0 0 DL aiordy 0xfffff80047e47000 [aiod1] 1254 1 786 0 S uwait 0xfffff80015cfef00 syz-executor.1 1252 1 786 0 S uwait 0xfffff800157c1700 syz-executor.1 832 822 832 0 Ss select 0xfffff80047542bc0 dhclient 825 1 825 0 Ss select 0xfffff80047542a40 dhclient 822 808 436 65 S select 0xfffff80015d44bc0 dhclient 818 780 818 0 Ss nanslp 0xffffffff8273c8e0 syz-executor.3 808 436 436 0 S wait 0xfffff80015d84000 sh 796 780 796 0 Ss nanslp 0xffffffff8273c8e0 syz-executor.2 786 780 786 0 Ss nanslp 0xffffffff8273c8e0 syz-executor.1 785 780 785 0 Rs syz-executor.0 780 778 778 0 S (threaded) syz-fuzzer 100106 S uwait 0xfffff80015cf9d00 syz-fuzzer 100119 S uwait 0xfffff80015d01d00 syz-fuzzer 100120 S kqread 0xfffff80015d47900 syz-fuzzer 100121 S uwait 0xfffff80015d01f00 syz-fuzzer 100122 S uwait 0xfffff80015d06300 syz-fuzzer 100123 S uwait 0xfffff80015d06500 syz-fuzzer 100124 S uwait 0xfffff80015d06700 syz-fuzzer 100126 S uwait 0xfffff80015cfd700 syz-fuzzer 101430 S uwait 0xfffff80047cb6d00 syz-fuzzer 778 776 778 0 Ss pause 0xfffff800473bf5e0 csh 776 694 776 0 Ss select 0xfffff80015e13ec0 sshd 760 1 760 0 Ss+ ttyin 0xfffff80015423cb0 getty 759 1 759 0 Ss+ ttyin 0xfffff80015a454b0 getty 758 1 758 0 Ss+ ttyin 0xfffff80015a45cb0 getty 757 1 757 0 Ss+ ttyin 0xfffff80015a4b4b0 getty 756 1 756 0 Ss+ ttyin 0xfffff80015a4bcb0 getty 755 1 755 0 Ss+ ttyin 0xfffff80015a4d4b0 getty 754 1 754 0 Ss+ ttyin 0xfffff80015a4dcb0 getty 753 1 753 0 Ss+ ttyin 0xfffff80015a524b0 getty 752 1 752 0 Ss+ ttyin 0xfffff80015a52cb0 getty 698 1 698 0 Ss nanslp 0xffffffff8273c8e1 cron 694 1 694 0 Ss select 0xfffff80047263740 sshd 507 1 507 0 Ss select 0xfffff80015e13740 syslogd 436 1 436 0 Ss wait 0xfffff80015dfd530 devd 435 1 435 65 Ss select 0xfffff80047263d40 dhclient 350 1 350 0 Ss select 0xfffff80015c2f8c0 dhclient 347 1 347 0 Ss select 0xfffff80015c332c0 dhclient 23 0 0 0 DL vlruwt 0xfffff80015be7530 [vnlru] 22 0 0 0 DL syncer 0xffffffff8282bcd0 [syncer] 21 0 0 0 DL (threaded) [bufdaemon] 100081 D qsleep 0xffffffff8282ada0 [bufdaemon] 100086 D - 0xffffffff8220aec0 [bufspacedaemon-0] 100098 D sdflush 0xfffff80015d104e8 [/ worker] 20 0 0 0 DL psleep 0xffffffff82852c08 [vmdaemon] 19 0 0 0 DL (threaded) [pagedaemon] 100079 D psleep 0xffffffff82847078 [dom0] 100087 D launds 0xffffffff82847084 [laundry: dom0] 100088 D umarcl 0xffffffff815d0ee0 [uma] 18 0 0 0 DL - 0xffffffff82570c88 [rand_harvestq] 17 0 0 0 DL waiting 0xffffffff82e3e828 [sctp_iterator] 16 0 0 0 DL pftm 0xffffffff82f803c0 [pf purge] 15 0 0 0 DL - 0xffffffff8282845c [soaiod4] 9 0 0 0 DL - 0xffffffff8282845c [soaiod3] 8 0 0 0 DL - 0xffffffff8282845c [soaiod2] 7 0 0 0 DL - 0xffffffff8282845c [soaiod1] 6 0 0 0 DL (threaded) [cam] 100044 D - 0xffffffff82448140 [doneq0] 100045 D - 0xffffffff824480c0 [async] 100078 D - 0xffffffff82447f90 [scanner] 14 0 0 0 DL seqstat 0xfffff80004debc88 [sequencer 00] 5 0 0 0 DL crypto_ 0xfffff80004ccb380 [crypto returns 1] 4 0 0 0 DL crypto_ 0xfffff80004ccb330 [crypto returns 0] 3 0 0 0 DL crypto_ 0xffffffff82844560 [crypto] 13 0 0 0 DL (threaded) [geom] 100035 D - 0xffffffff8271c120 [g_event] 100036 D - 0xffffffff8271c128 [g_up] 100037 D - 0xffffffff8271c130 [g_down] 2 0 0 0 DL (threaded) [KTLS] 100028 D - 0xfffff80004c4fd00 [thr_0] 100029 D - 0xfffff80004c4fd80 [thr_1] 12 0 0 0 RL (threaded) [intr] 100011 I [swi5: fast taskq] 100014 I [swi6: task queue] 100016 I [swi6: Giant taskq] 100030 I [swi3: vm] 100031 I [swi4: clock (0)] 100032 I [swi4: clock (1)] 100033 Run CPU 1 [swi1: netisr 0] 100046 I [irq24: virtio_pci0] 100047 I [irq25: virtio_pci0] 100048 I [irq26: virtio_pci0] 100049 I [irq27: virtio_pci0] 100050 I [irq28: virtio_pci1] 100051 I [irq29: virtio_pci1] 100052 I [irq30: virtio_pci1] 100053 I [irq31: virtio_pci1] 100054 I [irq32: virtio_pci1] 100059 I [irq10: virtio_pci2] 100061 I [irq1: atkbd0] 100062 I [irq12: psm0] 100063 I [swi0: uart uart++] 100071 I [swi1: pf send] 100084 I [swi1: hpts] 100085 I [swi1: hpts] 11 0 0 0 RL (threaded) [idle] 100003 CanRun [idle: cpu0] 100004 CanRun [idle: cpu1] 1 0 1 0 SLs wait 0xfffff80004bd5530 [init] 10 0 0 0 DL audit_w 0xffffffff82844a70 [audit] 0 0 0 0 DLs (threaded) [kernel] 100000 D swapin 0xffffffff8271c6b0 [swapper] 100005 D - 0xfffff8000443e900 [if_io_tqg_0] 100006 D - 0xfffff8000443e500 [if_io_tqg_1] 100007 D - 0xfffff8000443e100 [if_config_tqg_0] 100008 D - 0xfffff80004c75d00 [softirq_0] 100009 D - 0xfffff80004c75900 [softirq_1] 100010 D - 0xfffff80004c75500 [aiod_kick taskq] 100012 D - 0xfffff80004c70d00 [kqueue_ctx taskq] 100013 D - 0xfffff80004c70900 [pci_hp taskq] 100015 D - 0xfffff80004c70100 [inm_free taskq] 100017 D - 0xfffff80004c69900 [linuxkpi_irq_wq] 100018 D - 0xfffff80004c69500 [thread taskq] 100019 D - 0xfffff80004c69100 [in6m_free taskq] 100020 D - 0xfffff80004c65d00 [linuxkpi_short_wq_0] 100021 D - 0xfffff80004c65d00 [linuxkpi_short_wq_1] 100022 D - 0xfffff80004c65d00 [linuxkpi_short_wq_2] 100023 D - 0xfffff80004c65d00 [linuxkpi_short_wq_3] 100024 D - 0xfffff80004c65900 [linuxkpi_long_wq_0] 100025 D - 0xfffff80004c65900 [linuxkpi_long_wq_1] 100026 D - 0xfffff80004c65900 [linuxkpi_long_wq_2] 100027 D - 0xfffff80004c65900 [linuxkpi_long_wq_3] 100034 D - 0xfffff80004c65500 [firmware taskq] 100038 D - 0xfffff80004db8d00 [crypto_0] 100039 D - 0xfffff80004db8d00 [crypto_1] 100055 D - 0xfffff80004db8100 [vtnet0 rxq 0] 100056 D - 0xfffff80015364d00 [vtnet0 txq 0] 100057 D - 0xfffff80015364900 [vtnet0 rxq 1] 100058 D - 0xfffff80015364500 [vtnet0 txq 1] 100060 D vtbslp 0xfffff80004ece300 [virtio_balloon] 100064 D - 0xfffff800153ecd00 [mca taskq] 100069 D - 0xffffffff81e29a01 [deadlkres] 100074 D - 0xfffff800153ec900 [acpi_task_0] 100075 D - 0xfffff800153ec900 [acpi_task_1] 100076 D - 0xfffff800153ec900 [acpi_task_2] 100077 D - 0xfffff80004db8900 [CAM taskq] db> show all locks Process 9396 (syz-executor.1) thread 0xfffffe00951faac0 (110225) exclusive sleep mutex sctp-create (inp_create) r = 0 (0xfffff80042b23488) locked @ /syzkaller/managers/main/kernel/sys/netinet/sctp_usrreq.c:7011 Process 12 (intr) thread 0xfffffe00042bd900 (100033) exclusive sleep mutex sctp-tcb (tcb) r = 0 (0xfffffe009549a8b0) locked @ /syzkaller/managers/main/kernel/sys/netinet/sctp_pcb.c:2143 db> show malloc Type InUse MemUse Requests sysctloid 34288 12858K 34355 pf_hash 5 11560K 5 devbuf 4216 6982K 4244 tcp_hpts 5 3219K 5 kobj 328 2624K 488 vtbuf 24 2064K 46 pcb 331 1948K 40992 newblk 12 1038K 9319 vfscache 3 1035K 3 acpica 1674 649K 54692 inodedep 96 615K 8684 subproc 147 602K 9479 callout 2 528K 2 ufs_quota 1 520K 1 vfs_hash 1 520K 1 intr 4 480K 4 filedesc 30 470K 17169 sctp_atcl 433 433K 38613 bus 990 378K 3485 sctp_stro 148 296K 10879 linker 348 270K 409 sctp_atky 586 220K 49671 DEVFS1 107 214K 124 module 512 192K 512 kdtrace 228 188K 19624 vnet_data 1 176K 1 tidhash 3 164K 3 umtx 374 164K 374 sctp_athm 433 163K 39841 pagedep 9 140K 8579 tfo_ccache 1 136K 1 BPF 22 136K 24 sctp_stri 126 126K 7132 sctp_timw 251 126K 251 sem 4 120K 4 sctp_map 296 111K 21026 UMA 270 102K 270 audit_evclass 236 89K 294 mtx_pool 2 80K 2 temp 35 77K 1923 syncache 1 76K 1 vmem 3 72K 5 routetbl 128 69K 419 msg 4 68K 4 acpitask 1 64K 1 ddb_capture 1 64K 1 DEVFS3 126 63K 136 ifaddr 67 63K 69 ether_multi 152 57K 162 gtaskqueue 18 57K 18 DEVFS_RULE 56 54K 56 kenv 95 52K 95 eventhandler 133 50K 133 dirrem 87 44K 8603 ithread 99 43K 99 rman 84 42K 425 KTRACE 102 39K 15102 lltable 44 38K 74 taskqueue 60 36K 60 proc 3 34K 3 ufs_mount 5 34K 6 bus-sc 33 34K 1696 kqueue 67 33K 9407 devstat 4 33K 4 freefile 87 33K 8596 hostcache 1 32K 1 tty 16 32K 16 shm 1 32K 1 GEOM 60 29K 489 in6_multi 65 29K 65 DEVFSP 74 28K 173 kbdmux 6 28K 6 ifnet 7 25K 7 pwddesc 65 25K 9397 kcovinfo 64 24K 68 cred 23 23K 233 plimit 21 21K 394 CAM queue 5 21K 1528 pfs_nodes 20 20K 20 UART 12 18K 12 proc-args 44 17K 566 bmsafemap 2 17K 8655 ksem 1 16K 1 rpc 2 16K 2 shmfd 1 16K 7 pfs_vncache 1 16K 1 ufs_dirhash 24 14K 24 sglist 5 13K 5 session 25 13K 40 MCA 32 12K 32 CAM DEV 3 12K 510 selfd 31 12K 146242 Unitno 31 12K 47 vt 11 11K 11 acpisem 28 11K 28 CAM XPT 22 11K 543 uidinfo 3 9K 9 pf_ifnet 10 9K 19 memdesc 1 8K 1 ipsec-saq 2 8K 2 evdev 4 8K 4 acpidev 20 8K 20 lockf 20 8K 36 hhook 15 8K 17 mount 16 7K 90 fpukern_ctx 3 6K 3 terminal 11 6K 11 inpcbpolicy 14 6K 2314 ip6ndp 12 6K 13 sctp_ifa 13 5K 14 ipsecpolicy 2 5K 2 encap_export_host 12 5K 12 clone 9 5K 9 local_apic 1 4K 1 io_apic 1 4K 1 sahead 1 4K 1 secasvar 1 4K 1 nhops 6 4K 8 osd 10 4K 1109 pci_link 10 4K 10 select 10 4K 34 msi 9 4K 9 DEVFS 9 4K 10 tun 7 4K 7 ipsec 3 3K 3 mkdir 8 3K 17134 in_multi 6 3K 8 nexusdev 7 3K 7 feeder 7 3K 7 sctp_ifn 6 3K 14 mld 6 3K 6 igmp 6 3K 6 soname 6 3K 23740 toponodes 6 3K 6 prison 6 3K 6 isadev 6 3K 6 softdep 1 2K 1 vnodemarker 2 2K 44 NFSD session 1 2K 1 freework 4 2K 8571 linux 5 2K 6 CAM periph 4 2K 271 crypto 4 2K 4 newdirblk 4 2K 8567 diradd 4 2K 8636 freeblks 3 2K 8570 pfil 4 2K 4 CAM path 4 2K 1034 tcpfunc 3 2K 3 loginclass 3 2K 7 chacha20random 1 1K 1 vnodes 1 1K 1 CAM SIM 2 1K 2 ktls 1 1K 1 procdesc 2 1K 8 cdev 2 1K 2 aesni_data 2 1K 2 cpus 2 1K 2 atkbddev 2 1K 2 CAM dev queue 2 1K 2 xform 2 1K 131 entropy 2 1K 38 filecaps 2 1K 72 NFSD lckfile 1 1K 1 NFSD V4client 1 1K 1 indirdep 1 1K 13 pmchooks 1 1K 1 sctp_vrf 1 1K 1 apmdev 1 1K 1 CAM I/O Scheduler 1 1K 1 vnet_data_free 1 1K 1 vnet 1 1K 1 Per-cpu 1 1K 1 p1003.1b 1 1K 1 acpiintr 1 1K 1 pmc 1 1K 1 mqdata 0 0K 0 cdg data 0 0K 0 chd data 0 0K 0 htcp data 0 0K 18 dctcp data 0 0K 0 cubic data 0 0K 0 tcp_do 0 0K 0 tcp_fsb 0 0K 919 pf_table 0 0K 0 pf_rule 0 0K 0 pf_altq 0 0K 0 pf_osfp 0 0K 0 pf_temp 0 0K 0 vegas data 0 0K 0 sctp_mcore 0 0K 0 sctp_socko 0 0K 6473 sctp_iter 0 0K 9 sctp_mvrf 0 0K 0 sctp_cpal 0 0K 0 sctp_cmsg 0 0K 0 sctp_stre 0 0K 0 sctp_athi 0 0K 0 sctp_a_it 0 0K 9 sctp_aadr 0 0K 0 NFSCL lckown 0 0K 0 NFSCL client 0 0K 0 madt_table 0 0K 2 smartpqi 0 0K 0 NFSCL deleg 0 0K 0 NFSCL open 0 0K 0 NFSCL owner 0 0K 0 NFS fh 0 0K 0 NFS req 0 0K 0 NFSD usrgroup 0 0K 0 NFSD string 0 0K 0 iavf 0 0K 0 ixl 0 0K 0 NFSD V4lock 0 0K 0 NFSD V4state 0 0K 0 NFSD srvcache 0 0K 0 msdosfs_fat 0 0K 0 msdosfs_mount 0 0K 0 msdosfs_node 0 0K 0 ice-resmgr 0 0K 0 ice-osdep 0 0K 0 ice 0 0K 0 axgbe 0 0K 0 DEVFS4 0 0K 0 DEVFS2 0 0K 0 gntdev 0 0K 0 privcmd_dev 0 0K 0 evtchn_dev 0 0K 0 xen_intr 0 0K 0 xenstore 0 0K 0 ciss_data 0 0K 0 xen_hvm 0 0K 0 legacydrv 0 0K 0 qpidrv 0 0K 0 BACKLIGHT 0 0K 0 xnb 0 0K 0 dmar_idpgtbl 0 0K 0 dmar_dom 0 0K 0 dmar_ctx 0 0K 0 xbbd 0 0K 0 xbd 0 0K 0 Balloon 0 0K 0 isci 0 0K 0 iommu_dmamap 0 0K 0 sysmouse 0 0K 0 hyperv_socket 0 0K 0 bxe_ilt 0 0K 0 xenbus 0 0K 0 vtfont 0 0K 0 vm_fictitious 0 0K 0 ath_hal 0 0K 0 athdev 0 0K 0 ata_pci 0 0K 0 ata_dma 0 0K 0 ata_generic 0 0K 0 amr 0 0K 0 pvscsi 0 0K 0 scsi_da 0 0K 69 ata_da 0 0K 0 scsi_ch 0 0K 0 UMAHash 0 0K 0 scsi_cd 0 0K 0 AHCI driver 0 0K 0 vm_pgdata 0 0K 0 jblocks 0 0K 0 savedino 0 0K 6750 sentinel 0 0K 0 jfsync 0 0K 0 jtrunc 0 0K 0 sbdep 0 0K 19 jsegdep 0 0K 0 jseg 0 0K 0 jfreefrag 0 0K 0 jfreeblk 0 0K 0 jnewblk 0 0K 0 jmvref 0 0K 0 jremref 0 0K 0 jaddref 0 0K 0 freedep 0 0K 0 freefrag 0 0K 28 allocindir 0 0K 0 allocdirect 0 0K 0 ufs_trim 0 0K 0 mactemp 0 0K 0 audit_trigger 0 0K 0 audit_pipe_presel 0 0K 0 audit_pipeent 0 0K 0 audit_pipe 0 0K 0 audit_evname 0 0K 0 audit_bsm 0 0K 0 audit_gidset 0 0K 0 audit_text 0 0K 0 audit_path 0 0K 0 audit_data 0 0K 0 audit_cred 0 0K 0 USBdev 0 0K 0 USB 0 0K 0 agp 0 0K 0 nvme_da 0 0K 0 acpipwr 0 0K 0 acpi_perf 0 0K 0 ktls_ocf 0 0K 0 twsbuf 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5E_TLS 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EN 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5DUMP 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 seq_file 0 0K 0 radix 0 0K 0 idr 0 0K 0 lkpifw 0 0K 0 NLM 0 0K 0 ipsec-spdcache 0 0K 0 ipsec-reg 0 0K 0 ipsec-misc 0 0K 0 ipsecrequest 0 0K 0 ip6opt 0 0K 3 ip6_msource 0 0K 0 ip6_moptions 0 0K 0 in6_mfilter 0 0K 0 frag6 0 0K 0 tcplog 0 0K 0 tcp_hwpace 0 0K 0 twe_commands 0 0K 0 LRO 0 0K 0 newreno data 0 0K 0 ip_msource 0 0K 0 ip_moptions 0 0K 0 in_mfilter 0 0K 0 ipid 0 0K 0 80211scan 0 0K 0 80211ratectl 0 0K 0 80211power 0 0K 0 80211nodeie 0 0K 0 80211node 0 0K 0 80211mesh_gt 0 0K 0 80211mesh_rt 0 0K 0 80211perr 0 0K 0 80211prep 0 0K 0 80211preq 0 0K 0 80211dfs 0 0K 0 80211crypto 0 0K 0 80211vap 0 0K 0 iflib 0 0K 0 vlan 0 0K 0 gif 0 0K 0 ifdescr 0 0K 0 zlib 0 0K 0 fadvise 0 0K 2 VN POLL 0 0K 0 twa_commands 0 0K 0 statfs 0 0K 8763 namei_tracker 0 0K 3 export_host 0 0K 0 cl_savebuf 0 0K 9 tcp_log_dev 0 0K 877 midi buffers 0 0K 0 mixer 0 0K 0 ac97 0 0K 0 hdacc 0 0K 0 hdac 0 0K 0 hdaa 0 0K 0 acpicmbat 0 0K 0 SIIS driver 0 0K 0 CAM CCB 0 0K 2240 PUC 0 0K 0 ppbusdev 0 0K 0 agtiapi_MemAlloc malloc 0 0K 0 osti_cacheable 0 0K 0 tempbuff 0 0K 0 biobuf 0 0K 0 aios 0 0K 0 lio 0 0K 2 acl 0 0K 0 tempbuff 0 0K 0 mbuf_tag 0 0K 101 ag_tgt_map_t malloc 0 0K 0 ag_slr_map_t malloc 0 0K 0 lDevFlags * malloc 0 0K 0 tiDeviceHandle_t * malloc 0 0K 0 ag_portal_data_t malloc 0 0K 0 ag_device_t malloc 0 0K 0 STLock malloc 0 0K 0 CCB List 0 0K 0 sr_iov 0 0K 0 OCS 0 0K 0 OCS 0 0K 0 nvme 0 0K 0 nvd 0 0K 0 netmap 0 0K 0 mwldev 0 0K 0 MVS driver 0 0K 0 CAM ccb queue 0 0K 0 mrsasbuf 0 0K 0 mpt_user 0 0K 0 mps_user 0 0K 0 accf 0 0K 0 pts 0 0K 0 iov 0 0K 22784 ioctlops 0 0K 100 eventfd 0 0K 0 Witness 0 0K 0 stack 0 0K 0 MPSSAS 0 0K 0 mps 0 0K 0 mpr_user 0 0K 0 MPRSAS 0 0K 0 mpr 0 0K 0 mfibuf 0 0K 0 sbuf 0 0K 288 md_sectors 0 0K 0 firmware 0 0K 0 compressor 0 0K 0 md_disk 0 0K 0 SWAP 0 0K 0 malodev 0 0K 0 LED 0 0K 0 sysctltmp 0 0K 633 sysctl 0 0K 3 ekcd 0 0K 0 dumper 0 0K 0 sendfile 0 0K 0 rctl 0 0K 0 ix_sriov 0 0K 0 aacraidcam 0 0K 0 aacraid_buf 0 0K 0 ix 0 0K 0 ipsbuf 0 0K 0 cache 0 0K 0 iirbuf 0 0K 0 prison_racct 0 0K 0 Fail Points 0 0K 0 sigio 0 0K 1 filedesc_to_leader 0 0K 0 pwd 0 0K 0 tty console 0 0K 0 aaccam 0 0K 0 aacbuf 0 0K 0 zstd 0 0K 0 XZ_DEC 0 0K 0 nvlist 0 0K 0 SCSI ENC 0 0K 0 SCSI sa 0 0K 0 scsi_pass 0 0K 0 isofs_node 0 0K 0 isofs_mount 0 0K 0 tr_raid5_data 0 0K 0 tr_raid1e_data 0 0K 0 tr_raid1_data 0 0K 0 tr_raid0_data 0 0K 0 tr_concat_data 0 0K 0 md_sii_data 0 0K 0 md_promise_data 0 0K 0 md_nvidia_data 0 0K 0 md_jmicron_data 0 0K 0 md_intel_data 0 0K 0 md_ddf_data 0 0K 0 raid_data 0 0K 72 geom_flashmap 0 0K 0 tmpfs dir 0 0K 0 tmpfs name 0 0K 0 tmpfs mount 0 0K 0 NFS FHA 0 0K 0 newnfsmnt 0 0K 0 newnfsclient_req 0 0K 0 NFSCL layrecall 0 0K 0 NFSCL session 0 0K 0 NFSCL sockreq 0 0K 0 NFSCL devinfo 0 0K 0 NFSCL flayout 0 0K 0 NFSCL layout 0 0K 0 NFSD rollback 0 0K 0 NFSCL diroff 0 0K 0 NEWdirectio 0 0K 0 NEWNFSnode 0 0K 0 NFSCL lck 0 0K 0 db> show uma Zone Size Used Free Requests Sleeps Bucket Total Mem XFree mbuf_jumbo_page 4096 8322 964 1419212 0 254 38035456 0 malloc-384 384 34460 50 41910 0 30 13251840 0 malloc-1024 1024 4120 4 4122 0 16 4222976 0 malloc-8192 8192 332 2 1022 0 1 2736128 0 pbuf 2624 0 989 0 0 2 2595136 0 mbuf 256 9451 584 2622552 0 254 2568960 0 RADIX NODE 144 16585 184 283159 0 62 2414736 0 tcp_log 416 0 5265 83103 0 254 2190240 0 mbuf_cluster 2048 975 1 975 0 254 1998848 0 malloc-1024 1024 769 1143 55488 0 16 1957888 0 BUF TRIE 144 196 13272 831 0 62 1939392 0 malloc-1024 1024 105 1527 7684 0 16 1671168 0 UMA Slabs 0 112 13315 2 13315 0 126 1491504 0 sctp_asoc 2288 148 417 10513 0 254 1292720 0 malloc-8192 8192 148 9 19997 0 1 1286144 0 malloc-384 384 1180 1450 29184 0 30 1009920 0 sctp_raddr 736 779 497 40058 0 254 939136 0 malloc-512 512 158 1466 18023 0 30 831488 0 malloc-384 384 2096 64 123631 0 30 829440 0 sctp_ep 1280 159 462 19965 0 254 794880 0 malloc-32768 32768 17 4 8715 0 1 688128 0 FFS inode 1160 514 25 9111 0 8 625240 0 malloc-8192 8192 65 4 9397 0 1 565248 0 malloc-384 384 1298 112 236211 0 30 541440 0 ertt_txseginfo 40 11129 587 28795 0 254 468640 0 malloc-384 384 1055 65 29216 0 30 430080 0 malloc-2048 2048 175 21 10549 0 8 401408 0 malloc-65536 65536 6 0 6 0 1 393216 0 lkpicurr 160 2 2348 2 0 62 376000 0 lkpimm 160 1 2324 1 0 62 372000 0 THREAD 1808 162 25 10226 0 8 338096 0 VM OBJECT 264 1201 59 134676 0 30 332640 0 malloc-16384 16384 12 6 275 0 1 294912 0 VNODE 448 551 88 9150 0 30 286272 0 256 Bucket 2048 111 19 17944 0 8 266240 0 malloc-65536 65536 2 2 4 0 1 262144 0 malloc-384 384 596 64 26362 0 30 253440 0 mbuf_packet 256 466 509 30944 0 254 249600 0 malloc-1024 1024 205 31 10269 0 16 241664 0 malloc-2048 2048 111 3 128 0 8 233472 0 DEVCTL 1024 12 204 132 0 0 221184 0 malloc-1024 1024 162 34 19973 0 16 200704 0 socket 944 42 170 23563 0 254 200128 0 malloc-384 384 435 65 20094 0 30 192000 0 malloc-4096 4096 9 37 2249 0 2 188416 0 UMA Zones 768 242 2 242 0 16 187392 0 MAP ENTRY 96 1634 256 504510 0 126 181440 0 malloc-16384 16384 11 0 11 0 1 180224 0 vmem btag 56 2947 65 2947 0 254 168672 0 malloc-384 384 415 15 923 0 30 165120 0 malloc-512 512 308 12 1906 0 30 163840 0 S VFS Cache 104 1014 351 10584 0 126 141960 0 FFS2 dinode 256 514 26 9110 0 62 138240 0 sctp_chunk 152 375 509 64148 0 254 134368 0 malloc-32768 32768 4 0 4 0 1 131072 0 ksiginfo 112 66 978 1110 0 126 116928 0 malloc-8192 8192 13 1 24 0 1 114688 0 malloc-512 512 211 13 574 0 30 114688 0 VMSPACE 2544 37 8 9369 0 4 114480 0 PROC 1328 64 17 9396 0 8 107568 0 malloc-512 512 201 7 839 0 30 106496 0 malloc-8192 8192 11 0 11 0 1 90112 0 malloc-1024 1024 84 4 84 0 16 90112 0 filedesc0 1072 65 19 9397 0 8 90048 0 UMA Kegs 384 227 6 227 0 30 89472 0 malloc-16384 16384 4 1 7 0 1 81920 0 malloc-512 512 74 70 34681 0 30 73728 0 32 Bucket 256 173 97 15112 0 62 69120 0 malloc-65536 65536 1 0 1 0 1 65536 0 malloc-65536 65536 0 1 176 0 1 65536 0 malloc-65536 65536 0 1 1 0 1 65536 0 malloc-32768 32768 1 1 16 0 1 65536 0 malloc-16384 16384 2 2 4 0 1 65536 0 malloc-8192 8192 2 6 9136 0 1 65536 0 g_bio 408 0 160 6404 0 30 65280 0 128 Bucket 1024 30 29 858 0 16 60416 0 malloc-4096 4096 7 7 564 0 2 57344 0 sctp_readq 152 7 357 2998 0 254 55328 0 malloc-4096 4096 7 6 59 0 2 53248 0 malloc-2048 2048 8 18 824 0 8 53248 0 malloc-512 512 76 28 296 0 30 53248 0 malloc-16384 16384 2 1 6 0 1 49152 0 malloc-16384 16384 3 0 3 0 1 49152 0 malloc-16384 16384 3 0 3 0 1 49152 0 malloc-8192 8192 4 2 104 0 1 49152 0 malloc-4096 4096 7 5 292 0 2 49152 0 64 Bucket 512 48 40 8727 0 30 45056 0 tcpcb 1064 6 36 1097 0 254 44688 0 16 Bucket 144 215 93 3343 0 62 44352 0 clpbuf 2624 0 16 27 0 16 41984 0 pcpu-8 8 4647 473 4817 0 254 40960 0 DIRHASH 1024 34 2 34 0 16 36864 0 NAMEI 1024 0 36 49519 0 16 36864 0 malloc-4096 4096 7 2 200 0 2 36864 0 malloc-16384 16384 1 1 3 0 1 32768 0 pcpu-64 64 487 25 487 0 254 32768 0 Files 80 206 194 37055 0 126 32000 0 tcp_inpcb 488 6 58 1097 0 254 31232 0 TURNSTILE 136 188 22 188 0 62 28560 0 KNOTE 160 28 147 263341 0 62 28000 0 malloc-2048 2048 11 1 11 0 8 24576 0 PWD 32 22 734 8658 0 254 24192 0 ttyinq 160 135 15 300 0 62 24000 0 8 Bucket 80 41 259 2471 0 126 24000 0 ttyoutq 256 72 18 160 0 62 23040 0 tcp_rack_pcb 832 0 27 919 0 16 22464 0 pipe 744 20 10 361 0 16 22320 0 malloc-512 512 32 8 40 0 30 20480 0 2 Bucket 32 90 540 4431 0 254 20160 0 SLEEPQUEUE 88 188 36 188 0 126 19712 0 Mountpoints 2752 2 5 2 0 4 19264 0 malloc-2048 2048 7 1 7 0 8 16384 0 malloc-1024 1024 11 5 20 0 16 16384 0 malloc-512 512 26 6 1128 0 30 16384 0 vtnet_tx_hdr 24 0 668 715436 0 254 16032 0 udplite_inpcb 488 0 32 950 0 254 15616 0 malloc-4096 4096 2 1 3 0 2 12288 0 sctp_laddr 48 68 184 1418 0 254 12096 0 tcp_rack_map 112 0 108 1800 0 126 12096 0 udp_inpcb 488 6 18 184 0 254 11712 0 kenv 258 15 30 1035 0 30 11610 0 routing nhops 256 27 18 34 0 62 11520 0 unpcb 256 11 34 1258 0 254 11520 0 malloc-2048 2048 0 4 1024 0 8 8192 0 rtentry 176 30 16 34 0 62 8096 0 PGRP 88 25 67 46 0 126 8096 0 rl_entry 40 52 150 52 0 254 8080 0 sctp_stream_msg_out 112 4 68 3271 0 254 8064 0 udpcb 32 6 246 1134 0 254 8064 0 AIOP 32 4 248 4 0 254 8064 0 ertt 72 6 106 1097 0 126 8064 0 4 Bucket 48 6 162 217 0 254 8064 0 tcp_bbr_map 128 16 46 16 0 126 7936 0 ripcb 488 2 14 83 0 254 7808 0 AIOCB 552 0 14 8 0 16 7728 0 AIOLIO 272 0 28 2 0 30 7616 0 tcp_bbr_pcb 832 1 8 1 0 16 7488 0 FPU_save_area 832 1 8 1 0 16 7488 0 cpuset 104 7 55 7 0 126 6448 0 epoch_record pcpu 256 4 12 4 0 62 4096 0 pcpu-16 16 7 249 7 0 254 4096 0 hostcache 64 1 62 1 0 254 4032 0 syncache 168 0 24 4 0 254 4032 0 AIO 208 0 19 1 0 62 3952 0 UMA Slabs 1 176 9 13 9 0 62 3872 0 mqnode 416 3 6 3 0 30 3744 0 KMAP ENTRY 96 12 27 12 0 0 3744 0 vmem 1856 1 1 1 0 8 3712 0 SMR CPU 32 3 60 3 0 254 2016 0 SMR SHARED 24 3 60 3 0 254 1512 0 FFS1 dinode 128 0 0 0 0 126 0 0 da_ccb 544 0 0 0 0 16 0 0 ada_ccb 272 0 0 0 0 30 0 0 swblk 136 0 0 0 0 62 0 0 swpctrie 144 0 0 0 0 62 0 0 cdg_qdiffsample 16 0 0 0 0 254 0 0 sctp_asconf_ack 48 0 0 0 0 254 0 0 sctp_asconf 40 0 0 0 0 254 0 0 pf state scrubs 40 0 0 0 0 254 0 0 pf frag entries 40 0 0 0 0 254 0 0 pf frags 248 0 0 0 0 62 0 0 pf table entries 160 0 0 0 0 62 0 0 pf table entry counters 64 0 0 0 0 254 0 0 pf source nodes 136 0 0 0 0 254 0 0 pf state keys 88 0 0 0 0 126 0 0 pf states 304 0 0 0 0 254 0 0 pf tags 104 0 0 0 0 126 0 0 pf mtags 48 0 0 0 0 254 0 0 tcp_log_node 120 0 0 0 0 126 0 0 tcp_log_bucket 176 0 0 0 0 62 0 0 tcpreass 48 0 0 0 0 254 0 0 tfo_ccache_entries 80 0 0 0 0 126 0 0 tfo 4 0 0 0 0 254 0 0 sackhole 32 0 0 0 0 254 0 0 tcptw 88 0 0 0 0 254 0 0 ipq 56 0 0 0 0 254 0 0 IPsec SA lft_c 16 0 0 0 0 254 0 0 itimer 352 0 0 0 0 30 0 0 mqnotifier 216 0 0 0 0 62 0 0 mvdata 64 0 0 0 0 254 0 0 mqueue 248 0 0 0 0 62 0 0 TMPFS node 224 0 0 0 0 62 0 0 NCLNODE 584 0 0 0 0 16 0 0 LTS VFS Cache 360 0 0 0 0 30 0 0 L VFS Cache 320 0 0 0 0 30 0 0 STS VFS Cache 144 0 0 0 0 62 0 0 cryptop 280 0 0 0 0 30 0 0 linux_dma_object 24 0 0 0 0 254 0 0 linux_dma_pctrie 144 0 0 0 0 62 0 0 IOMMU_MAP_ENTRY 120 0 0 0 0 126 0 0 ktls_session 128 0 0 0 0 126 0 0 mbuf_jumbo_16k 16384 0 0 0 0 254 0 0 mbuf_jumbo_9k 9216 0 0 0 0 254 0 0 audit_record 1280 0 0 0 0 8 0 0 domainset 40 0 0 0 0 254 0 0 MAC labels 40 0 0 0 0 254 0 0 vnpbuf 2624 0 0 0 0 64 0 0 mdpbuf 2624 0 0 0 0 3 0 0 nfspbuf 2624 0 0 0 0 16 0 0 swwbuf 2624 0 0 0 0 8 0 0 swrbuf 2624 0 0 0 0 16 0 0 umtx_shm 88 0 0 0 0 126 0 0 umtx pi 96 0 0 0 0 126 0 0 rangeset pctrie nodes 144 0 0 0 0 62 0 0 malloc-65536 65536 0 0 0 0 1 0 0 malloc-65536 65536 0 0 0 0 1 0 0 malloc-65536 65536 0 0 0 0 1 0 0 malloc-32768 32768 0 0 0 0 1 0 0 malloc-32768 32768 0 0 0 0 1 0 0 malloc-32768 32768 0 0 0 0 1 0 0 malloc-32768 32768 0 0 0 0 1 0 0 malloc-32768 32768 0 0 0 0 1 0 0 malloc-8192 8192 0 0 0 0 1 0 0 malloc-4096 4096 0 0 0 0 2 0 0 malloc-4096 4096 0 0 0 0 2 0 0 malloc-2048 2048 0 0 0 0 8 0 0 malloc-2048 2048 0 0 0 0 8 0 0 malloc-1024 1024 0 0 0 0 16 0 0 malloc-256 256 0 0 0 0 62 0 0 malloc-256 256 0 0 0 0 62 0 0 malloc-256 256 0 0 0 0 62 0 0 malloc-256 256 0 0 0 0 62 0 0 malloc-256 256 0 0 0 0 62 0 0 malloc-256 256 0 0 0 0 62 0 0 malloc-256 256 0 0 0 0 62 0 0 malloc-256 256 0 0 0 0 62 0 0 malloc-128 128 0 0 0 0 126 0 0 malloc-128 128 0 0 0 0 126 0 0 malloc-128 128 0 0 0 0 126 0 0 malloc-128 128 0 0 0 0 126 0 0 malloc-128 128 0 0 0 0 126 0 0 malloc-128 128 0 0 0 0 126 0 0 malloc-128 128 0 0 0 0 126 0 0 malloc-128 128 0 0 0 0 126 0 0 malloc-64 64 0 0 0 0 254 0 0 malloc-64 64 0 0 0 0 254 0 0 malloc-64 64 0 0 0 0 254 0 0 malloc-64 64 0 0 0 0 254 0 0 malloc-64 64 0 0 0 0 254 0 0 malloc-64 64 0 0 0 0 254 0 0 malloc-64 64 0 0 0 0 254 0 0 malloc-64 64 0 0 0 0 254 0 0 malloc-32 32 0 0 0 0 254 0 0 malloc-32 32 0 0 0 0 254 0 0 malloc-32 32 0 0 0 0 254 0 0 malloc-32 32 0 0 0 0 254 0 0 malloc-32 32 0 0 0 0 254 0 0 malloc-32 32 0 0 0 0 254 0 0 malloc-32 32 0 0 0 0 254 0 0 malloc-32 32 0 0 0 0 254 0 0 malloc-16 16 0 0 0 0 254 0 0 malloc-16 16 0 0 0 0 254 0 0 malloc-16 16 0 0 0 0 254 0 0 malloc-16 16 0 0 0 0 254 0 0 malloc-16 16 0 0 0 0 254 0 0 malloc-16 16 0 0 0 0 254 0 0 malloc-16 16 0 0 0 0 254 0 0 malloc-16 16 0 0 0 0 254 0 0 pcpu-32 32 0 0 0 0 254 0 0 pcpu-4 4 0 0 0 0 254 0 0 fakepg 104 0 0 0 0 126 0 0 UMA Hash 256 0 0 0 0 62 0 0