panic: mtx_lock() of spin mutex (null) @ /syzkaller/managers/main/kernel/sys/kern/uipc_sockbuf.c:434 cpuid = 1 time = 1624544028 KDB: stack backtrace: db_trace_self_wrapper() at db_trace_self_wrapper+0x47/frame 0xfffffe0051650100 vpanic() at vpanic+0x1cc/frame 0xfffffe0051650160 panic() at panic+0x43/frame 0xfffffe00516501c0 __mtx_lock_flags() at __mtx_lock_flags+0x1d7/frame 0xfffffe0051650220 socantrcvmore() at socantrcvmore+0x2c/frame 0xfffffe0051650240 sctp_notify_assoc_change() at sctp_notify_assoc_change+0x5b8/frame 0xfffffe00516502b0 sctp_process_control() at sctp_process_control+0x9034/frame 0xfffffe0051650740 sctp_common_input_processing() at sctp_common_input_processing+0x83a/frame 0xfffffe00516508c0 sctp_input_with_port() at sctp_input_with_port+0x30a/frame 0xfffffe00516509b0 sctp_input() at sctp_input+0x1f/frame 0xfffffe00516509d0 ip_input() at ip_input+0x38b/frame 0xfffffe0051650a90 swi_net() at swi_net+0x20e/frame 0xfffffe0051650b10 ithread_loop() at ithread_loop+0x33f/frame 0xfffffe0051650bb0 fork_exit() at fork_exit+0xb3/frame 0xfffffe0051650bf0 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe0051650bf0 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- KDB: enter: panic [ thread pid 12 tid 100033 ] Stopped at kdb_enter+0x67: movq $0,0x164097e(%rip) db> db> set $lines = 0 db> set $maxwidth = 0 db> show registers cs 0x20 ds 0x3b es 0x3b fs 0x13 gs 0x1b ss 0x28 rax 0x12 rcx 0xffffffff81131ef0 vprintf+0x140 rdx 0x1 rbx 0 rsp 0xfffffe00516500e0 rbp 0xfffffe0051650100 rsi 0 rdi 0xffffffff81131f26 vprintf+0x176 r8 0 r9 0x8080808080808080 r10 0xfffffe005164ffd0 r11 0x1ffaefff59c r12 0xffffffff82267d30 ddb_dbbe r13 0xffffffff81a6e500 r14 0xffffffff81a7500b r15 0xffffffff81a7500b rip 0xffffffff81128f87 kdb_enter+0x67 rflags 0x82 kdb_enter+0x67: movq $0,0x164097e(%rip) db> show proc Process 12 (intr) at 0xfffff80004c74a60: state: NORMAL uid: 0 gids: 0 parent: pid 0 at 0xffffffff8271ce70 ABI: null flag: 0x10000284 flag2: 0 reaper: 0xffffffff8271ce70 reapsubtree: 12 sigparent: 20 vmspace: 0xffffffff8271dad0 (map 0xffffffff8271dad0) (map.pmap 0xffffffff8271db90) (pmap 0xffffffff8271dbf0) threads: 23 100011 I [swi5: fast taskq] 100014 I [swi6: task queue] 100016 I [swi6: Giant taskq] 100030 I [swi3: vm] 100031 I [swi4: clock (0)] 100032 I [swi4: clock (1)] 100033 Run CPU 1 [swi1: netisr 0] 100046 I [irq24: virtio_pci0] 100047 I [irq25: virtio_pci0] 100048 I [irq26: virtio_pci0] 100049 I [irq27: virtio_pci0] 100050 I [irq28: virtio_pci1] 100051 I [irq29: virtio_pci1] 100052 I [irq30: virtio_pci1] 100053 I [irq31: virtio_pci1] 100054 I [irq32: virtio_pci1] 100059 I [irq10: virtio_pci2] 100061 I [irq1: atkbd0] 100062 I [irq12: psm0] 100063 I [swi0: uart uart++] 100071 I [swi1: pf send] 100084 I [swi1: hpts] 100085 I [swi1: hpts] db> ps pid ppid pgrp uid state wmesg wchan cmd 6988 810 810 0 S (threaded) syz-executor.3 106581 S nanslp 0xffffffff8273d0a1 syz-executor.3 106789 S sbwait 0xfffff8001e4a5574 syz-executor.3 6985 804 804 0 RE CPU 0 syz-executor.2 4695 4685 4695 0 Ss select 0xfffff8001e56ca40 dhclient 4691 1 4691 0 Ss select 0xfffff8001e117d40 dhclient 4685 4674 436 65 S select 0xfffff8001e6bdd40 dhclient 4674 436 436 0 S wait 0xfffff80027a43000 sh 4657 1 4657 65 Ss select 0xfffff8001e4e3440 dhclient 3449 1 3449 0 Ss select 0xfffff80027dc6a40 dhclient 3439 1 3439 0 Ss select 0xfffff8001e527140 dhclient 3409 1 3409 65 Ss select 0xfffff800154c2ec0 dhclient 1692 1 1692 0 Ss select 0xfffff80027f3aec0 dhclient 1689 1 1689 0 Ss select 0xfffff800279febc0 dhclient 1666 1 1666 65 Ss select 0xfffff80027ff4440 dhclient 847 0 0 0 DL aiordy 0xfffff80015e4ba60 [aiod4] 846 0 0 0 DL aiordy 0xfffff80027adc000 [aiod3] 845 0 0 0 DL aiordy 0xfffff80027adc530 [aiod2] 844 0 0 0 DL aiordy 0xfffff80027448000 [aiod1] 830 1 830 0 Ss select 0xfffff80015f018c0 dhclient 824 1 824 0 Ss select 0xfffff80027a26bc0 dhclient 810 780 810 0 Ss nanslp 0xffffffff8273d0a1 syz-executor.3 804 780 804 0 Ss nanslp 0xffffffff8273d0a0 syz-executor.2 784 780 784 0 Ss piperd 0xfffff80027a56ba0 syz-executor.1 783 780 783 0 Ss piperd 0xfffff80015e4c8b8 syz-executor.0 780 778 778 0 S (threaded) syz-fuzzer 100116 S uwait 0xfffff8002751d900 syz-fuzzer 100118 S uwait 0xfffff800273e1300 syz-fuzzer 100119 S uwait 0xfffff800273e1500 syz-fuzzer 100120 S uwait 0xfffff800273e1700 syz-fuzzer 100121 S uwait 0xfffff80027477100 syz-fuzzer 100122 S uwait 0xfffff80027477300 syz-fuzzer 100123 S uwait 0xfffff80027477500 syz-fuzzer 100125 S kqread 0xfffff8002751d100 syz-fuzzer 100187 S uwait 0xfffff80027b69300 syz-fuzzer 778 776 778 0 Ss pause 0xfffff80027a43b10 csh 776 694 776 0 Ss select 0xfffff800279fe8c0 sshd 760 1 760 0 Ss+ ttyin 0xfffff800154a5cb0 getty 759 1 759 0 Ss+ ttyin 0xfffff80015b2a4b0 getty 758 1 758 0 Ss+ ttyin 0xfffff80015b2acb0 getty 757 1 757 0 Ss+ ttyin 0xfffff80015b2e4b0 getty 756 1 756 0 Ss+ ttyin 0xfffff80015b2ecb0 getty 755 1 755 0 Ss+ ttyin 0xfffff80015b314b0 getty 754 1 754 0 Ss+ ttyin 0xfffff80015b31cb0 getty 753 1 753 0 Ss+ ttyin 0xfffff80015ab94b0 getty 752 1 752 0 Ss+ ttyin 0xfffff80015ab9cb0 getty 698 1 698 0 Ss nanslp 0xffffffff8273d0a0 cron 694 1 694 0 Ss select 0xfffff80015f01d40 sshd 507 1 507 0 Ss select 0xfffff800154c2440 syslogd 436 1 436 0 Ss wait 0xfffff80015ec6000 devd 435 1 435 65 Ss select 0xfffff80015f052c0 dhclient 350 1 350 0 Ss select 0xfffff80015f06bc0 dhclient 347 1 347 0 Ss select 0xfffff80015ed2bc0 dhclient 23 0 0 0 DL vlruwt 0xfffff80015ccd530 [vnlru] 22 0 0 0 DL syncer 0xffffffff8282c450 [syncer] 21 0 0 0 DL (threaded) [bufdaemon] 100081 D qsleep 0xffffffff8282b520 [bufdaemon] 100088 D - 0xffffffff8220aec0 [bufspacedaemon-0] 100097 D sdflush 0xfffff80004ddd4e8 [/ worker] 20 0 0 0 DL psleep 0xffffffff82853388 [vmdaemon] 19 0 0 0 DL (threaded) [pagedaemon] 100079 D psleep 0xffffffff828477f8 [dom0] 100086 D launds 0xffffffff82847804 [laundry: dom0] 100087 D umarcl 0xffffffff815c4f40 [uma] 18 0 0 0 DL - 0xffffffff82571458 [rand_harvestq] 17 0 0 0 DL waiting 0xffffffff82fd68b8 [sctp_iterator] 16 0 0 0 DL pftm 0xffffffff82d79410 [pf purge] 15 0 0 0 DL - 0xffffffff82828bdc [soaiod4] 9 0 0 0 DL - 0xffffffff82828bdc [soaiod3] 8 0 0 0 DL - 0xffffffff82828bdc [soaiod2] 7 0 0 0 DL - 0xffffffff82828bdc [soaiod1] 6 0 0 0 DL (threaded) [cam] 100044 D - 0xffffffff82448b40 [doneq0] 100045 D - 0xffffffff82448ac0 [async] 100078 D - 0xffffffff82448990 [scanner] 14 0 0 0 DL seqstat 0xfffff80004d8a488 [sequencer 00] 5 0 0 0 DL crypto_ 0xfffff80004d53d80 [crypto returns 1] 4 0 0 0 DL crypto_ 0xfffff80004d53d30 [crypto returns 0] 3 0 0 0 DL crypto_ 0xffffffff82844ce0 [crypto] 13 0 0 0 DL (threaded) [geom] 100035 D - 0xffffffff8271c8e0 [g_event] 100036 D - 0xffffffff8271c8e8 [g_up] 100037 D - 0xffffffff8271c8f0 [g_down] 2 0 0 0 DL (threaded) [KTLS] 100028 D - 0xfffff80004c69100 [thr_0] 100029 D - 0xfffff80004c69180 [thr_1] 12 0 0 0 RL (threaded) [intr] 100011 I [swi5: fast taskq] 100014 I [swi6: task queue] 100016 I [swi6: Giant taskq] 100030 I [swi3: vm] 100031 I [swi4: clock (0)] 100032 I [swi4: clock (1)] 100033 Run CPU 1 [swi1: netisr 0] 100046 I [irq24: virtio_pci0] 100047 I [irq25: virtio_pci0] 100048 I [irq26: virtio_pci0] 100049 I [irq27: virtio_pci0] 100050 I [irq28: virtio_pci1] 100051 I [irq29: virtio_pci1] 100052 I [irq30: virtio_pci1] 100053 I [irq31: virtio_pci1] 100054 I [irq32: virtio_pci1] 100059 I [irq10: virtio_pci2] 100061 I [irq1: atkbd0] 100062 I [irq12: psm0] 100063 I [swi0: uart uart++] 100071 I [swi1: pf send] 100084 I [swi1: hpts] 100085 I [swi1: hpts] 11 0 0 0 RL (threaded) [idle] 100003 CanRun [idle: cpu0] 100004 CanRun [idle: cpu1] 1 0 1 0 SLs wait 0xfffff80004bda530 [init] 10 0 0 0 DL audit_w 0xffffffff828451f0 [audit] 0 0 0 0 DLs (threaded) [kernel] 100000 D swapin 0xffffffff8271ce70 [swapper] 100005 D - 0xfffff8000443b500 [if_io_tqg_0] 100006 D - 0xfffff8000443b100 [if_io_tqg_1] 100007 D - 0xfffff80004c78d00 [if_config_tqg_0] 100008 D - 0xfffff80004c78900 [softirq_0] 100009 D - 0xfffff80004c78500 [softirq_1] 100010 D - 0xfffff80004c78100 [aiod_kick taskq] 100012 D - 0xfffff80004c75900 [kqueue_ctx taskq] 100013 D - 0xfffff80004c75500 [pci_hp taskq] 100015 D - 0xfffff80004c6ed00 [inm_free taskq] 100017 D - 0xfffff80004c6e500 [linuxkpi_irq_wq] 100018 D - 0xfffff80004c6e100 [thread taskq] 100019 D - 0xfffff80004c69d00 [in6m_free taskq] 100020 D - 0xfffff80004c69900 [linuxkpi_short_wq_0] 100021 D - 0xfffff80004c69900 [linuxkpi_short_wq_1] 100022 D - 0xfffff80004c69900 [linuxkpi_short_wq_2] 100023 D - 0xfffff80004c69900 [linuxkpi_short_wq_3] 100024 D - 0xfffff80004c69500 [linuxkpi_long_wq_0] 100025 D - 0xfffff80004c69500 [linuxkpi_long_wq_1] 100026 D - 0xfffff80004c69500 [linuxkpi_long_wq_2] 100027 D - 0xfffff80004c69500 [linuxkpi_long_wq_3] 100034 D - 0xfffff80004cd3d00 [firmware taskq] 100038 D - 0xfffff80004cd3900 [crypto_0] 100039 D - 0xfffff80004cd3900 [crypto_1] 100055 D - 0xfffff800153ecd00 [vtnet0 rxq 0] 100056 D - 0xfffff800153ec900 [vtnet0 txq 0] 100057 D - 0xfffff800153ec500 [vtnet0 rxq 1] 100058 D - 0xfffff800153ec100 [vtnet0 txq 1] 100060 D vtbslp 0xfffff80015463700 [virtio_balloon] 100064 D - 0xfffff80015469900 [mca taskq] 100069 D - 0xffffffff81e217a0 [deadlkres] 100074 D - 0xfffff80015abe100 [acpi_task_0] 100075 D - 0xfffff80015abe100 [acpi_task_1] 100076 D - 0xfffff80015abe100 [acpi_task_2] 100077 D - 0xfffff80004cd3500 [CAM taskq] db> show all locks Process 6988 (syz-executor.3) thread 0xfffffe009555cc80 (106789) exclusive sx so_rcv_sx (so_rcv_sx) r = 0 (0xfffff8001e4a5518) locked @ /syzkaller/managers/main/kernel/sys/kern/uipc_sockbuf.c:467 Process 6985 (syz-executor.2) thread 0xfffffe00950e8560 (106576) shared sx filedesc structure (filedesc structure) r = 0 (0xfffffe00950a0880) locked @ /syzkaller/managers/main/kernel/sys/kern/uipc_mqueue.c:2483 Process 12 (intr) thread 0xfffffe00042be900 (100033) exclusive sleep mutex sctp-tcb (tcb) r = 0 (0xfffffe00955528b0) locked @ /syzkaller/managers/main/kernel/sys/netinet/sctp_pcb.c:2143 db> show malloc Type InUse MemUse Requests sysctloid 34288 12858K 34355 pf_hash 5 11560K 5 devbuf 4216 6982K 4244 pcb 716 3710K 21431 tcp_hpts 5 3219K 5 kobj 326 2608K 486 vtbuf 24 2064K 46 newblk 8 1036K 6726 vfscache 3 1035K 3 inodedep 198 717K 6178 sctp_atcl 684 684K 17040 sctp_stro 340 680K 3800 acpica 1674 649K 54806 subproc 151 613K 7079 callout 2 528K 2 ufs_quota 1 520K 1 vfs_hash 1 520K 1 intr 4 480K 4 sctp_atky 1024 384K 19609 bus 986 377K 3469 BPF 46 337K 46 linker 348 270K 409 sctp_athm 684 257K 17040 sctp_map 680 255K 7600 DEVFS1 107 214K 124 filedesc 14 211K 12207 module 510 192K 510 kdtrace 230 189K 13792 vnet_data 1 176K 1 tidhash 3 164K 3 umtx 374 164K 374 pagedep 10 141K 6110 tfo_ccache 1 136K 1 sem 4 120K 4 UMA 270 102K 270 dirrem 188 94K 6099 sctp_timw 182 91K 182 audit_evclass 236 89K 294 mtx_pool 2 80K 2 temp 35 77K 2242 syncache 1 76K 1 vmem 3 72K 5 freefile 188 71K 6097 routetbl 131 70K 422 msg 4 68K 4 ifaddr 70 66K 72 acpitask 1 64K 1 ddb_capture 1 64K 1 DEVFS3 126 63K 136 ether_multi 152 57K 162 gtaskqueue 18 57K 18 DEVFS_RULE 56 54K 56 kenv 95 52K 95 eventhandler 133 50K 133 ithread 99 43K 99 rman 84 42K 425 KTRACE 100 38K 100 lltable 43 37K 54 taskqueue 60 36K 60 proc 3 34K 3 ufs_mount 5 34K 6 bus-sc 33 34K 1684 kqueue 68 34K 6998 devstat 4 33K 4 hostcache 1 32K 1 tty 16 32K 16 shm 1 32K 1 DEVFSP 80 30K 314 GEOM 60 29K 489 in6_multi 65 29K 65 kbdmux 6 28K 6 cred 26 26K 238 pwddesc 66 25K 6994 ifnet 7 25K 7 plimit 24 24K 424 kcovinfo 64 24K 68 CAM queue 5 21K 1528 proc-args 53 21K 643 pfs_nodes 20 20K 20 selfd 52 20K 113391 UART 12 18K 12 session 34 17K 46 bmsafemap 2 17K 6150 ksem 1 16K 1749 rpc 2 16K 2 shmfd 1 16K 3 pfs_vncache 1 16K 1 ufs_dirhash 24 14K 24 sglist 5 13K 5 MCA 32 12K 32 CAM DEV 3 12K 510 lockf 32 12K 54 vt 11 11K 11 acpisem 28 11K 28 CAM XPT 22 11K 543 Unitno 27 11K 47 uidinfo 3 9K 9 pf_ifnet 10 9K 19 memdesc 1 8K 1 ipsec-saq 2 8K 2 evdev 4 8K 4 acpidev 20 8K 20 hhook 15 8K 17 select 19 8K 50 mount 16 7K 90 fpukern_ctx 3 6K 3 inpcbpolicy 15 6K 4286 terminal 11 6K 11 ip6ndp 12 6K 13 sctp_ifa 13 5K 14 ipsecpolicy 2 5K 2 encap_export_host 12 5K 12 clone 9 5K 9 local_apic 1 4K 1 io_apic 1 4K 1 sahead 1 4K 1 secasvar 1 4K 1 nhops 6 4K 8 pci_link 10 4K 10 msi 9 4K 9 DEVFS 9 4K 10 tun 7 4K 7 osd 8 4K 3444 ipsec 3 3K 3 mkdir 8 3K 12182 in_multi 6 3K 8 nexusdev 7 3K 7 feeder 7 3K 7 procdesc 5 3K 14 sctp_ifn 6 3K 14 mld 6 3K 6 igmp 6 3K 6 toponodes 6 3K 6 prison 6 3K 6 isadev 6 3K 6 softdep 1 2K 1 vnodemarker 2 2K 30 NFSD session 1 2K 1 freework 4 2K 6100 diradd 5 2K 6139 linux 5 2K 6 CAM periph 4 2K 271 filecaps 5 2K 97 crypto 4 2K 4 newdirblk 4 2K 6091 freeblks 3 2K 6099 pfil 4 2K 4 CAM path 4 2K 1034 soname 4 2K 14156 tcpfunc 3 2K 3 loginclass 3 2K 6 chacha20random 1 1K 1 vnodes 1 1K 1 CAM SIM 2 1K 2 ktls 1 1K 1 cdev 2 1K 2 aesni_data 2 1K 2 cpus 2 1K 2 atkbddev 2 1K 2 CAM dev queue 2 1K 2 xform 2 1K 268 entropy 2 1K 42 NFSD lckfile 1 1K 1 NFSD V4client 1 1K 1 pmchooks 1 1K 1 sctp_vrf 1 1K 1 apmdev 1 1K 1 CAM I/O Scheduler 1 1K 1 vnet_data_free 1 1K 1 vnet 1 1K 1 iov 1 1K 18147 Per-cpu 1 1K 1 p1003.1b 1 1K 1 acpiintr 1 1K 1 pmc 1 1K 1 cdg data 0 0K 0 tcp_do 0 0K 0 tcp_fsb 0 0K 860 mqdata 0 0K 0 sctp_mcore 0 0K 0 sctp_socko 0 0K 3400 sctp_iter 0 0K 11 sctp_mvrf 0 0K 0 sctp_cpal 0 0K 0 sctp_cmsg 0 0K 0 sctp_stre 0 0K 0 sctp_athi 0 0K 0 sctp_a_it 0 0K 11 sctp_aadr 0 0K 0 sctp_stri 0 0K 2484 chd data 0 0K 0 cubic data 0 0K 0 dctcp data 0 0K 0 htcp data 0 0K 0 vegas data 0 0K 0 pf_table 0 0K 0 pf_rule 0 0K 0 pf_altq 0 0K 0 pf_osfp 0 0K 0 pf_temp 0 0K 0 NFSCL lckown 0 0K 0 NFSCL client 0 0K 0 madt_table 0 0K 2 smartpqi 0 0K 0 NFSCL deleg 0 0K 0 NFSCL open 0 0K 0 NFSCL owner 0 0K 0 NFS fh 0 0K 0 NFS req 0 0K 0 NFSD usrgroup 0 0K 0 NFSD string 0 0K 0 iavf 0 0K 0 ixl 0 0K 0 NFSD V4lock 0 0K 0 NFSD V4state 0 0K 0 NFSD srvcache 0 0K 0 msdosfs_fat 0 0K 0 msdosfs_mount 0 0K 0 msdosfs_node 0 0K 0 ice-resmgr 0 0K 0 ice-osdep 0 0K 0 ice 0 0K 0 axgbe 0 0K 0 DEVFS4 0 0K 0 DEVFS2 0 0K 0 gntdev 0 0K 0 privcmd_dev 0 0K 0 evtchn_dev 0 0K 0 xen_intr 0 0K 0 xenstore 0 0K 0 ciss_data 0 0K 0 xen_hvm 0 0K 0 legacydrv 0 0K 0 qpidrv 0 0K 0 BACKLIGHT 0 0K 0 xnb 0 0K 0 dmar_idpgtbl 0 0K 0 dmar_dom 0 0K 0 dmar_ctx 0 0K 0 xbbd 0 0K 0 xbd 0 0K 0 Balloon 0 0K 0 isci 0 0K 0 iommu_dmamap 0 0K 0 sysmouse 0 0K 0 hyperv_socket 0 0K 0 bxe_ilt 0 0K 0 xenbus 0 0K 0 vtfont 0 0K 0 vm_fictitious 0 0K 0 ath_hal 0 0K 0 athdev 0 0K 0 ata_pci 0 0K 0 ata_dma 0 0K 0 ata_generic 0 0K 0 amr 0 0K 0 pvscsi 0 0K 0 scsi_da 0 0K 69 ata_da 0 0K 0 scsi_ch 0 0K 0 UMAHash 0 0K 0 scsi_cd 0 0K 0 AHCI driver 0 0K 0 vm_pgdata 0 0K 0 jblocks 0 0K 0 savedino 0 0K 5910 sentinel 0 0K 0 jfsync 0 0K 0 jtrunc 0 0K 0 sbdep 0 0K 12 jsegdep 0 0K 0 jseg 0 0K 0 jfreefrag 0 0K 0 jfreeblk 0 0K 0 jnewblk 0 0K 0 jmvref 0 0K 0 jremref 0 0K 0 jaddref 0 0K 0 freedep 0 0K 0 freefrag 0 0K 6 allocindir 0 0K 0 indirdep 0 0K 10 allocdirect 0 0K 0 ufs_trim 0 0K 0 mactemp 0 0K 0 audit_trigger 0 0K 0 audit_pipe_presel 0 0K 0 audit_pipeent 0 0K 0 audit_pipe 0 0K 0 audit_evname 0 0K 0 audit_bsm 0 0K 0 audit_gidset 0 0K 0 audit_text 0 0K 0 audit_path 0 0K 0 audit_data 0 0K 0 audit_cred 0 0K 0 USBdev 0 0K 0 USB 0 0K 0 agp 0 0K 0 nvme_da 0 0K 0 acpipwr 0 0K 0 acpi_perf 0 0K 0 ktls_ocf 0 0K 0 twsbuf 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5E_TLS 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EN 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5DUMP 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 seq_file 0 0K 0 radix 0 0K 0 idr 0 0K 0 lkpifw 0 0K 0 NLM 0 0K 0 ipsec-spdcache 0 0K 0 ipsec-reg 0 0K 0 ipsec-misc 0 0K 0 ipsecrequest 0 0K 0 ip6opt 0 0K 7 ip6_msource 0 0K 0 ip6_moptions 0 0K 0 in6_mfilter 0 0K 0 frag6 0 0K 0 tcplog 0 0K 0 tcp_hwpace 0 0K 0 twe_commands 0 0K 0 LRO 0 0K 0 newreno data 0 0K 0 ip_msource 0 0K 0 ip_moptions 0 0K 0 in_mfilter 0 0K 0 ipid 0 0K 0 80211scan 0 0K 0 80211ratectl 0 0K 0 80211power 0 0K 0 80211nodeie 0 0K 0 80211node 0 0K 0 80211mesh_gt 0 0K 0 80211mesh_rt 0 0K 0 80211perr 0 0K 0 80211prep 0 0K 0 80211preq 0 0K 0 80211dfs 0 0K 0 80211crypto 0 0K 0 80211vap 0 0K 0 iflib 0 0K 0 vlan 0 0K 0 gif 0 0K 0 ifdescr 0 0K 0 zlib 0 0K 0 fadvise 0 0K 0 VN POLL 0 0K 0 twa_commands 0 0K 0 statfs 0 0K 6269 namei_tracker 0 0K 16 export_host 0 0K 0 cl_savebuf 0 0K 4 tcp_log_dev 0 0K 0 midi buffers 0 0K 0 mixer 0 0K 0 ac97 0 0K 0 hdacc 0 0K 0 hdac 0 0K 0 hdaa 0 0K 0 acpicmbat 0 0K 0 SIIS driver 0 0K 0 CAM CCB 0 0K 2145 PUC 0 0K 0 ppbusdev 0 0K 0 agtiapi_MemAlloc malloc 0 0K 0 osti_cacheable 0 0K 0 tempbuff 0 0K 0 biobuf 0 0K 0 aios 0 0K 0 lio 0 0K 0 acl 0 0K 0 tempbuff 0 0K 0 mbuf_tag 0 0K 111 ag_tgt_map_t malloc 0 0K 0 ag_slr_map_t malloc 0 0K 0 lDevFlags * malloc 0 0K 0 tiDeviceHandle_t * malloc 0 0K 0 ag_portal_data_t malloc 0 0K 0 ag_device_t malloc 0 0K 0 STLock malloc 0 0K 0 CCB List 0 0K 0 sr_iov 0 0K 0 OCS 0 0K 0 OCS 0 0K 0 nvme 0 0K 0 nvd 0 0K 0 netmap 0 0K 0 mwldev 0 0K 0 MVS driver 0 0K 0 CAM ccb queue 0 0K 0 mrsasbuf 0 0K 0 mpt_user 0 0K 0 mps_user 0 0K 0 accf 0 0K 0 pts 0 0K 0 ioctlops 0 0K 5249 eventfd 0 0K 0 Witness 0 0K 0 stack 0 0K 0 MPSSAS 0 0K 0 mps 0 0K 0 mpr_user 0 0K 0 MPRSAS 0 0K 0 mpr 0 0K 0 mfibuf 0 0K 0 sbuf 0 0K 288 md_sectors 0 0K 0 firmware 0 0K 0 compressor 0 0K 0 md_disk 0 0K 0 SWAP 0 0K 0 malodev 0 0K 0 LED 0 0K 0 sysctltmp 0 0K 789 sysctl 0 0K 3 ekcd 0 0K 0 dumper 0 0K 0 sendfile 0 0K 0 rctl 0 0K 0 ix_sriov 0 0K 0 aacraidcam 0 0K 0 aacraid_buf 0 0K 0 ix 0 0K 0 ipsbuf 0 0K 0 cache 0 0K 0 iirbuf 0 0K 0 prison_racct 0 0K 0 Fail Points 0 0K 0 sigio 0 0K 1 filedesc_to_leader 0 0K 0 pwd 0 0K 0 tty console 0 0K 0 aaccam 0 0K 0 aacbuf 0 0K 0 zstd 0 0K 0 XZ_DEC 0 0K 0 nvlist 0 0K 0 SCSI ENC 0 0K 0 SCSI sa 0 0K 0 scsi_pass 0 0K 0 isofs_node 0 0K 0 isofs_mount 0 0K 0 tr_raid5_data 0 0K 0 tr_raid1e_data 0 0K 0 tr_raid1_data 0 0K 0 tr_raid0_data 0 0K 0 tr_concat_data 0 0K 0 md_sii_data 0 0K 0 md_promise_data 0 0K 0 md_nvidia_data 0 0K 0 md_jmicron_data 0 0K 0 md_intel_data 0 0K 0 md_ddf_data 0 0K 0 raid_data 0 0K 72 geom_flashmap 0 0K 0 tmpfs dir 0 0K 0 tmpfs name 0 0K 0 tmpfs mount 0 0K 0 NFS FHA 0 0K 0 newnfsmnt 0 0K 0 newnfsclient_req 0 0K 0 NFSCL layrecall 0 0K 0 NFSCL session 0 0K 0 NFSCL sockreq 0 0K 0 NFSCL devinfo 0 0K 0 NFSCL flayout 0 0K 0 NFSCL layout 0 0K 0 NFSD rollback 0 0K 0 NFSCL diroff 0 0K 0 NEWdirectio 0 0K 0 NEWNFSnode 0 0K 0 NFSCL lck 0 0K 0 db> show uma Zone Size Used Free Requests Sleeps Bucket Total Mem XFree mbuf_jumbo_page 4096 8322 975 1277601 0 254 38080512 0 malloc-384 384 36172 58 46653 0 30 13912320 0 malloc-1024 1024 4174 2 4175 0 16 4276224 0 malloc-8192 8192 346 10 9522 0 1 2916352 0 malloc-8192 8192 328 2 1099 0 1 2703360 0 pbuf 2624 0 989 0 0 2 2595136 0 mbuf 256 9088 527 2218807 0 254 2461440 0 RADIX NODE 144 15662 379 238981 0 63 2309904 0 BUF TRIE 144 182 13286 758 0 62 1939392 0 malloc-1024 1024 133 1567 7423 0 16 1740800 0 malloc-1024 1024 275 1369 8403 0 16 1683456 0 UMA Slabs 0 112 13716 33 13716 0 126 1539888 0 mbuf_cluster 2048 615 1 615 0 254 1261568 0 malloc-384 384 2786 74 162253 0 30 1098240 0 malloc-512 512 326 1370 6685 0 30 868352 0 sctp_asoc 2288 340 30 3800 0 254 846560 0 malloc-384 384 724 1376 45049 0 30 806400 0 malloc-1024 1024 689 47 19538 0 16 753664 0 malloc-2048 2048 342 14 4958 0 8 729088 0 malloc-384 384 1728 32 53802 0 30 675840 0 FFS inode 1160 516 23 6614 0 8 625240 0 sctp_raddr 736 340 507 16195 0 254 623392 0 malloc-8192 8192 68 6 6996 0 1 606208 0 malloc-1024 1024 517 31 16421 0 16 561152 0 sctp_ep 1280 344 58 9517 0 254 514560 0 malloc-32768 32768 9 6 6104 0 1 491520 0 tcp_bbr_map 128 0 3379 64040 0 126 432512 0 malloc-16384 16384 24 0 25 0 1 393216 0 lkpimm 160 1 2324 1 0 62 372000 0 lkpicurr 160 2 2323 2 0 62 372000 0 VM OBJECT 264 1289 76 100659 0 30 360360 0 THREAD 1808 163 24 6797 0 8 338096 0 malloc-512 512 30 610 7214 0 30 327680 0 VNODE 448 554 76 6654 0 30 282240 0 256 Bucket 2048 113 17 17252 0 8 266240 0 ertt_txseginfo 40 0 6565 168316 0 254 262600 0 malloc-65536 65536 4 0 4 0 1 262144 0 malloc-2048 2048 125 3 142 0 8 262144 0 malloc-384 384 612 68 4969 0 30 261120 0 tcp_inpcb 488 4 516 3432 0 254 253760 0 DEVCTL 1024 6 214 153 0 0 225280 0 malloc-8192 8192 23 2 37 0 1 204800 0 malloc-65536 65536 3 0 3 0 1 196608 0 malloc-16384 16384 11 1 247 0 1 196608 0 MAP ENTRY 96 1613 361 375348 0 126 189504 0 UMA Zones 768 242 2 242 0 16 187392 0 malloc-4096 4096 7 38 3106 0 2 184320 0 vmem btag 56 3094 134 3094 0 254 180768 0 malloc-384 384 361 89 28085 0 30 172800 0 socket 944 49 123 15065 0 254 162368 0 mbuf_packet 256 125 490 11994 0 254 157440 0 FFS2 dinode 256 516 24 6613 0 62 138240 0 S VFS Cache 104 1025 301 7176 0 126 137904 0 malloc-65536 65536 2 0 2 0 1 131072 0 malloc-65536 65536 0 2 176 0 1 131072 0 malloc-384 384 284 56 8620 0 30 130560 0 VMSPACE 2544 39 12 6968 0 4 129744 0 sctp_chunk 152 335 497 12743 0 254 126464 0 malloc-512 512 194 38 12825 0 30 118784 0 ksiginfo 112 69 975 1318 0 126 116928 0 malloc-512 512 206 18 6307 0 30 114688 0 PROC 1328 65 19 6993 0 8 111552 0 malloc-1024 1024 97 3 102 0 16 102400 0 malloc-512 512 196 4 351 0 30 102400 0 malloc-32768 32768 1 2 121 0 1 98304 0 malloc-16384 16384 2 4 32 0 1 98304 0 128 Bucket 1024 62 29 1357 0 16 93184 0 filedesc0 1072 66 18 6994 0 8 90048 0 UMA Kegs 384 227 6 227 0 30 89472 0 g_bio 408 0 200 6027 0 30 81600 0 malloc-512 512 118 34 8178 0 30 77824 0 32 Bucket 256 188 82 15771 0 62 69120 0 tcpcb 1080 4 59 3432 0 254 68040 0 malloc-65536 65536 0 1 1 0 1 65536 0 malloc-32768 32768 1 1 16 0 1 65536 0 malloc-16384 16384 4 0 4 0 1 65536 0 malloc-16384 16384 2 2 6 0 1 65536 0 malloc-2048 2048 10 20 518 0 8 61440 0 malloc-4096 4096 12 2 45 0 2 57344 0 malloc-384 384 93 47 10354 0 30 53760 0 malloc-4096 4096 7 6 32 0 2 53248 0 malloc-16384 16384 3 0 3 0 1 49152 0 malloc-16384 16384 3 0 3 0 1 49152 0 malloc-8192 8192 2 4 6275 0 1 49152 0 64 Bucket 512 45 51 7159 0 30 49152 0 16 Bucket 144 238 98 3273 0 62 48384 0 clpbuf 2624 0 16 21 0 16 41984 0 DIRHASH 1024 34 6 34 0 16 40960 0 malloc-2048 2048 16 4 24 0 8 40960 0 malloc-512 512 25 55 1442 0 30 40960 0 pcpu-8 8 4701 419 4857 0 254 40960 0 NAMEI 1024 0 36 40911 0 16 36864 0 Files 80 246 204 32668 0 126 36000 0 malloc-32768 32768 1 0 1 0 1 32768 0 malloc-32768 32768 1 0 1 0 1 32768 0 malloc-32768 32768 1 0 1 0 1 32768 0 malloc-8192 8192 3 1 4 0 1 32768 0 pcpu-64 64 487 25 487 0 254 32768 0 tcp_bbr_pcb 832 0 36 2017 0 16 29952 0 pipe 744 25 15 406 0 16 29760 0 malloc-4096 4096 6 1 8 0 2 28672 0 malloc-2048 2048 2 12 582 0 8 28672 0 malloc-512 512 34 22 210 0 30 28672 0 TURNSTILE 136 188 22 188 0 62 28560 0 KNOTE 160 28 147 133661 0 62 28000 0 malloc-8192 8192 0 3 13 0 1 24576 0 malloc-8192 8192 1 2 43 0 1 24576 0 malloc-4096 4096 1 5 10 0 2 24576 0 ttyinq 160 135 15 300 0 62 24000 0 8 Bucket 80 45 255 2219 0 126 24000 0 ttyoutq 256 72 18 160 0 62 23040 0 tcp_rack_pcb 832 0 27 860 0 16 22464 0 malloc-4096 4096 3 2 59 0 2 20480 0 PWD 32 20 610 6185 0 254 20160 0 2 Bucket 32 98 532 3342 0 254 20160 0 sctp_readq 152 0 130 1007 0 254 19760 0 SLEEPQUEUE 88 188 36 188 0 126 19712 0 Mountpoints 2752 2 5 2 0 4 19264 0 malloc-16384 16384 1 0 1 0 1 16384 0 malloc-4096 4096 2 2 3435 0 2 16384 0 vtnet_tx_hdr 24 0 668 755543 0 254 16032 0 malloc-2048 2048 3 3 163 0 8 12288 0 malloc-2048 2048 2 4 6 0 8 12288 0 rtentry 176 30 39 34 0 62 12144 0 ripcb 488 5 19 621 0 254 11712 0 udp_inpcb 488 6 18 193 0 254 11712 0 kenv 258 15 30 1038 0 30 11610 0 routing nhops 256 27 18 34 0 62 11520 0 unpcb 256 20 25 1227 0 254 11520 0 malloc-2048 2048 4 0 4 0 8 8192 0 malloc-1024 1024 1 7 11 0 16 8192 0 PGRP 88 34 58 46 0 126 8096 0 rl_entry 40 49 153 49 0 254 8080 0 sctp_stream_msg_out 112 2 70 1310 0 254 8064 0 sctp_laddr 48 104 64 548 0 254 8064 0 tcp_rack_map 112 0 72 359 0 126 8064 0 udpcb 32 6 246 233 0 254 8064 0 AIOP 32 4 248 4 0 254 8064 0 ertt 72 4 108 3432 0 126 8064 0 4 Bucket 48 6 162 179 0 254 8064 0 udplite_inpcb 488 0 16 40 0 254 7808 0 AIOCB 552 0 14 2 0 16 7728 0 FPU_save_area 832 1 8 1 0 16 7488 0 cpuset 104 7 55 7 0 126 6448 0 epoch_record pcpu 256 4 12 4 0 62 4096 0 malloc-4096 4096 1 0 1 0 2 4096 0 pcpu-16 16 7 249 7 0 254 4096 0 hostcache 64 1 62 1 0 254 4032 0 syncache 168 0 24 4 0 254 4032 0 AIO 208 0 19 1 0 62 3952 0 UMA Slabs 1 176 9 13 9 0 62 3872 0 mqnode 416 3 6 3 0 30 3744 0 KMAP ENTRY 96 12 27 12 0 0 3744 0 vmem 1856 1 1 1 0 8 3712 0 SMR CPU 32 3 60 3 0 254 2016 0 SMR SHARED 24 3 60 3 0 254 1512 0 FFS1 dinode 128 0 0 0 0 126 0 0 da_ccb 544 0 0 0 0 16 0 0 ada_ccb 272 0 0 0 0 30 0 0 swblk 136 0 0 0 0 62 0 0 swpctrie 144 0 0 0 0 62 0 0 cdg_qdiffsample 16 0 0 0 0 254 0 0 sctp_asconf_ack 48 0 0 0 0 254 0 0 sctp_asconf 40 0 0 0 0 254 0 0 pf state scrubs 40 0 0 0 0 254 0 0 pf frag entries 40 0 0 0 0 254 0 0 pf frags 248 0 0 0 0 62 0 0 pf table entries 160 0 0 0 0 62 0 0 pf table entry counters 64 0 0 0 0 254 0 0 pf source nodes 136 0 0 0 0 254 0 0 pf state keys 88 0 0 0 0 126 0 0 pf states 304 0 0 0 0 254 0 0 pf tags 104 0 0 0 0 126 0 0 pf mtags 48 0 0 0 0 254 0 0 tcp_log_node 120 0 0 0 0 126 0 0 tcp_log_bucket 176 0 0 0 0 62 0 0 tcp_log 416 0 0 0 0 254 0 0 tcpreass 48 0 0 0 0 254 0 0 tfo_ccache_entries 80 0 0 0 0 126 0 0 tfo 4 0 0 0 0 254 0 0 sackhole 32 0 0 0 0 254 0 0 tcptw 88 0 0 0 0 254 0 0 ipq 56 0 0 0 0 254 0 0 IPsec SA lft_c 16 0 0 0 0 254 0 0 itimer 352 0 0 0 0 30 0 0 AIOLIO 272 0 0 0 0 30 0 0 TMPFS node 224 0 0 0 0 62 0 0 NCLNODE 584 0 0 0 0 16 0 0 mqnotifier 216 0 0 0 0 62 0 0 mvdata 64 0 0 0 0 254 0 0 mqueue 248 0 0 0 0 62 0 0 LTS VFS Cache 360 0 0 0 0 30 0 0 L VFS Cache 320 0 0 0 0 30 0 0 STS VFS Cache 144 0 0 0 0 62 0 0 cryptop 280 0 0 0 0 30 0 0 linux_dma_object 24 0 0 0 0 254 0 0 linux_dma_pctrie 144 0 0 0 0 62 0 0 IOMMU_MAP_ENTRY 120 0 0 0 0 126 0 0 ktls_session 128 0 0 0 0 126 0 0 mbuf_jumbo_16k 16384 0 0 0 0 254 0 0 mbuf_jumbo_9k 9216 0 0 0 0 254 0 0 audit_record 1280 0 0 0 0 8 0 0 domainset 40 0 0 0 0 254 0 0 MAC labels 40 0 0 0 0 254 0 0 vnpbuf 2624 0 0 0 0 64 0 0 mdpbuf 2624 0 0 0 0 3 0 0 nfspbuf 2624 0 0 0 0 16 0 0 swwbuf 2624 0 0 0 0 8 0 0 swrbuf 2624 0 0 0 0 16 0 0 umtx_shm 88 0 0 0 0 126 0 0 umtx pi 96 0 0 0 0 126 0 0 rangeset pctrie nodes 144 0 0 0 0 62 0 0 malloc-65536 65536 0 0 0 0 1 0 0 malloc-65536 65536 0 0 0 0 1 0 0 malloc-65536 65536 0 0 0 0 1 0 0 malloc-32768 32768 0 0 0 0 1 0 0 malloc-32768 32768 0 0 0 0 1 0 0 malloc-1024 1024 0 0 0 0 16 0 0 malloc-256 256 0 0 0 0 62 0 0 malloc-256 256 0 0 0 0 62 0 0 malloc-256 256 0 0 0 0 62 0 0 malloc-256 256 0 0 0 0 62 0 0 malloc-256 256 0 0 0 0 62 0 0 malloc-256 256 0 0 0 0 62 0 0 malloc-256 256 0 0 0 0 62 0 0 malloc-256 256 0 0 0 0 62 0 0 malloc-128 128 0 0 0 0 126 0 0 malloc-128 128 0 0 0 0 126 0 0 malloc-128 128 0 0 0 0 126 0 0 malloc-128 128 0 0 0 0 126 0 0 malloc-128 128 0 0 0 0 126 0 0 malloc-128 128 0 0 0 0 126 0 0 malloc-128 128 0 0 0 0 126 0 0 malloc-128 128 0 0 0 0 126 0 0 malloc-64 64 0 0 0 0 254 0 0 malloc-64 64 0 0 0 0 254 0 0 malloc-64 64 0 0 0 0 254 0 0 malloc-64 64 0 0 0 0 254 0 0 malloc-64 64 0 0 0 0 254 0 0 malloc-64 64 0 0 0 0 254 0 0 malloc-64 64 0 0 0 0 254 0 0 malloc-64 64 0 0 0 0 254 0 0 malloc-32 32 0 0 0 0 254 0 0 malloc-32 32 0 0 0 0 254 0 0 malloc-32 32 0 0 0 0 254 0 0 malloc-32 32 0 0 0 0 254 0 0 malloc-32 32 0 0 0 0 254 0 0 malloc-32 32 0 0 0 0 254 0 0 malloc-32 32 0 0 0 0 254 0 0 malloc-32 32 0 0 0 0 254 0 0 malloc-16 16 0 0 0 0 254 0 0 malloc-16 16 0 0 0 0 254 0 0 malloc-16 16 0 0 0 0 254 0 0 malloc-16 16 0 0 0 0 254 0 0 malloc-16 16 0 0 0 0 254 0 0 malloc-16 16 0 0 0 0 254 0 0 malloc-16 16 0 0 0 0 254 0 0 malloc-16 16 0 0 0 0 254 0 0 pcpu-32 32 0 0 0 0 254 0 0 pcpu-4 4 0 0 0 0 254 0 0 fakepg 104 0 0 0 0 126 0 0 UMA Hash 256 0 0 0 0 62 0 0