panic: Bad tailq NEXT(0xfffffe009daed430->tqh_last) != NULL cpuid = 0 time = 1625010204 KDB: stack backtrace: db_trace_self_wrapper() at db_trace_self_wrapper+0x47/frame 0xfffffe009c659470 vpanic() at vpanic+0x1cc/frame 0xfffffe009c6594d0 panic() at panic+0x43/frame 0xfffffe009c659530 sctp_ss_default_add() at sctp_ss_default_add+0x132/frame 0xfffffe009c659570 sctp_lower_sosend() at sctp_lower_sosend+0x5522/frame 0xfffffe009c659760 sctp_sosend() at sctp_sosend+0x49b/frame 0xfffffe009c6598a0 sosend() at sosend+0xc6/frame 0xfffffe009c659910 kern_sendit() at kern_sendit+0x33c/frame 0xfffffe009c6599c0 freebsd32_sendmsg() at freebsd32_sendmsg+0x270/frame 0xfffffe009c659ab0 ia32_syscall() at ia32_syscall+0x23a/frame 0xfffffe009c659bf0 int0x80_syscall_common() at int0x80_syscall_common+0x9c/frame 0xfbffcf78 KDB: enter: panic [ thread pid 29630 tid 130918 ] Stopped at kdb_enter+0x67: movq $0,0x164017e(%rip) db> db> set $lines = 0 db> set $maxwidth = 0 db> show registers cs 0x20 ds 0x3b es 0x3b fs 0x13 gs 0x1b ss 0 rax 0x12 rcx 0xfffffe0096600000 rdx 0x3ffff rbx 0 rsp 0xfffffe009c659450 rbp 0xfffffe009c659470 rsi 0x40001 rdi 0xffffffff81132726 vprintf+0x176 r8 0 r9 0x8080808080808080 r10 0xfffffe009c659340 r11 0x1ff63fff59c r12 0xffffffff82267d30 ddb_dbbe r13 0xffffffff82e49800 _refcount_update_saturated+0x9e0 r14 0xffffffff81a74f7b r15 0xffffffff81a74f7b rip 0xffffffff81129787 kdb_enter+0x67 rflags 0x200082 kernload+0x82 kdb_enter+0x67: movq $0,0x164017e(%rip) db> show proc Process 29630 (syz-executor.1) at 0xfffff800283f4000: state: NORMAL uid: 0 gids: 0, 0, 5 parent: pid 783 at 0xfffff8003744f530 ABI: FreeBSD ELF32 flag: 0x10080480 flag2: 0 arguments: /root/syz-executor.1 reaper: 0xfffff80004bda530 reapsubtree: 1 sigparent: 20 vmspace: 0xfffffe0095083000 (map 0xfffffe0095083000) (map.pmap 0xfffffe00950830c0) (pmap 0xfffffe0095083120) threads: 2 121862 s syz-executor.1 130918 Run CPU 0 syz-executor.1 db> ps pid ppid pgrp uid state wmesg wchan cmd 29630 783 783 0 T (threaded) syz-executor.1 121862 s syz-executor.1 130918 Run CPU 0 syz-executor.1 29626 792 792 0 S (threaded) syz-executor.2 116211 S nanslp 0xffffffff8273d0a0 syz-executor.2 130912 S accept 0xfffff800240e2c68 syz-executor.2 130917 S uwait 0xfffff8002823cb00 syz-executor.2 17189 1 783 0 S uwait 0xfffff80015ed7100 syz-executor.1 17181 1 783 0 S uwait 0xfffff8002823cd00 syz-executor.1 9965 1 9965 65 Ss select 0xfffff80028e84bc0 dhclient 7393 1 7393 0 Ss select 0xfffff8003759b2c0 dhclient 7390 1 7390 0 Ss select 0xfffff80028408ec0 dhclient 7371 1 7371 65 Ss select 0xfffff80028cd7d40 dhclient 4825 1 4825 0 Ss select 0xfffff8002854dd40 dhclient 4822 1 4822 0 Ss select 0xfffff80028208140 dhclient 4803 1 4803 65 Ss select 0xfffff8002829e8c0 dhclient 3246 1 783 0 S uwait 0xfffff8002804a500 syz-executor.1 3243 1 783 0 S uwait 0xfffff800153d0700 syz-executor.1 3204 1 792 0 S uwait 0xfffff80015cca700 syz-executor.2 3203 1 792 0 S uwait 0xfffff80037413300 syz-executor.2 3201 1 827 0 S uwait 0xfffff80037dc9b00 syz-executor.3 3195 1 827 0 S uwait 0xfffff80015ed7b00 syz-executor.3 3188 1 827 0 S uwait 0xfffff8002804ab00 syz-executor.3 3183 1 827 0 S uwait 0xfffff80037dc9900 syz-executor.3 3179 1 827 0 S uwait 0xfffff80037a56700 syz-executor.3 3177 1 792 0 S uwait 0xfffff80037d53f00 syz-executor.2 3175 1 792 0 S uwait 0xfffff80015ac2900 syz-executor.2 3172 1 792 0 S uwait 0xfffff80037a56100 syz-executor.2 3169 1 792 0 S uwait 0xfffff80037413700 syz-executor.2 3164 1 827 0 S uwait 0xfffff80037347100 syz-executor.3 3163 1 827 0 S uwait 0xfffff80037b68700 syz-executor.3 3046 1 3046 0 Ss select 0xfffff80015e2e140 dhclient 3038 1 3038 0 Ss select 0xfffff800281bb140 dhclient 3000 1 3000 65 Ss select 0xfffff8002829ea40 dhclient 1926 0 0 0 DL aiordy 0xfffff80037ebe530 [aiod4] 1925 0 0 0 DL aiordy 0xfffff80037d54a60 [aiod3] 1924 0 0 0 DL aiordy 0xfffff80037dd7000 [aiod2] 1923 0 0 0 DL aiordy 0xfffff80037e9ba60 [aiod1] 827 780 827 0 Rs syz-executor.3 823 1 823 0 Ss select 0xfffff80015e25a40 dhclient 817 1 817 0 Ss select 0xfffff80015e2e8c0 dhclient 792 780 792 0 Ss nanslp 0xffffffff8273d0a0 syz-executor.2 783 780 783 0 Ss nanslp 0xffffffff8273d0a0 syz-executor.1 782 780 782 0 Rs syz-executor.0 780 778 778 0 R (threaded) syz-fuzzer 100118 S uwait 0xfffff80037a7c700 syz-fuzzer 100119 RunQ syz-fuzzer 100120 S uwait 0xfffff80015ed8f00 syz-fuzzer 100121 S uwait 0xfffff800157c9100 syz-fuzzer 100122 RunQ syz-fuzzer 100123 S uwait 0xfffff80037a7c900 syz-fuzzer 100124 S uwait 0xfffff80037a7cb00 syz-fuzzer 100125 S uwait 0xfffff80037a7c100 syz-fuzzer 100127 S kqread 0xfffff80037a56f00 syz-fuzzer 778 776 778 0 Ss pause 0xfffff80037a39b10 csh 776 694 776 0 Ss select 0xfffff8003786d8c0 sshd 760 1 760 0 Ss+ ttyin 0xfffff800154a6cb0 getty 759 1 759 0 Ss+ ttyin 0xfffff80015b2a4b0 getty 758 1 758 0 Ss+ ttyin 0xfffff80015b2acb0 getty 757 1 757 0 Ss+ ttyin 0xfffff80015b2e4b0 getty 756 1 756 0 Ss+ ttyin 0xfffff80015b2ecb0 getty 755 1 755 0 Ss+ ttyin 0xfffff80015b314b0 getty 754 1 754 0 Ss+ ttyin 0xfffff80015b31cb0 getty 753 1 753 0 Ss+ ttyin 0xfffff80015ab94b0 getty 752 1 752 0 Ss+ ttyin 0xfffff80015ab9cb0 getty 698 1 698 0 Ss nanslp 0xffffffff8273d0a1 cron 694 1 694 0 Ss select 0xfffff80037319a40 sshd 507 1 507 0 Ss select 0xfffff80015ed4bc0 syslogd 436 1 436 0 Ss select 0xfffff80015f30140 devd 435 1 435 65 Ss select 0xfffff800157a2bc0 dhclient 350 1 350 0 Ss select 0xfffff80015e4fbc0 dhclient 347 1 347 0 Ss select 0xfffff80015dd2440 dhclient 23 0 0 0 DL vlruwt 0xfffff80015ccd530 [vnlru] 22 0 0 0 DL syncer 0xffffffff8282c450 [syncer] 21 0 0 0 DL (threaded) [bufdaemon] 100081 D qsleep 0xffffffff8282b520 [bufdaemon] 100086 D - 0xffffffff8220aec0 [bufspacedaemon-0] 100097 D sdflush 0xfffff80015e92ce8 [/ worker] 20 0 0 0 DL psleep 0xffffffff82853388 [vmdaemon] 19 0 0 0 DL (threaded) [pagedaemon] 100079 D psleep 0xffffffff828477f8 [dom0] 100087 D launds 0xffffffff82847804 [laundry: dom0] 100088 D umarcl 0xffffffff815c5a30 [uma] 18 0 0 0 DL - 0xffffffff82571458 [rand_harvestq] 17 0 0 0 DL waiting 0xffffffff82e5a8b8 [sctp_iterator] 16 0 0 0 RL [pf purge] 15 0 0 0 DL - 0xffffffff82828bdc [soaiod4] 9 0 0 0 DL - 0xffffffff82828bdc [soaiod3] 8 0 0 0 DL - 0xffffffff82828bdc [soaiod2] 7 0 0 0 DL - 0xffffffff82828bdc [soaiod1] 6 0 0 0 DL (threaded) [cam] 100044 D - 0xffffffff82448b40 [doneq0] 100045 D - 0xffffffff82448ac0 [async] 100078 D - 0xffffffff82448990 [scanner] 14 0 0 0 DL seqstat 0xfffff80004d8a488 [sequencer 00] 5 0 0 0 DL crypto_ 0xfffff80004d53d80 [crypto returns 1] 4 0 0 0 DL crypto_ 0xfffff80004d53d30 [crypto returns 0] 3 0 0 0 DL crypto_ 0xffffffff82844ce0 [crypto] 13 0 0 0 DL (threaded) [geom] 100035 D - 0xffffffff8271c8e0 [g_event] 100036 D - 0xffffffff8271c8e8 [g_up] 100037 D - 0xffffffff8271c8f0 [g_down] 2 0 0 0 DL (threaded) [KTLS] 100028 D - 0xfffff80004c69100 [thr_0] 100029 D - 0xfffff80004c69180 [thr_1] 12 0 0 0 WL (threaded) [intr] 100011 I [swi5: fast taskq] 100014 I [swi6: task queue] 100016 I [swi6: Giant taskq] 100030 I [swi3: vm] 100031 I [swi4: clock (0)] 100032 I [swi4: clock (1)] 100033 I [swi1: netisr 0] 100046 I [irq24: virtio_pci0] 100047 I [irq25: virtio_pci0] 100048 I [irq26: virtio_pci0] 100049 I [irq27: virtio_pci0] 100050 I [irq28: virtio_pci1] 100051 I [irq29: virtio_pci1] 100052 I [irq30: virtio_pci1] 100053 I [irq31: virtio_pci1] 100054 I [irq32: virtio_pci1] 100059 I [irq10: virtio_pci2] 100061 I [irq1: atkbd0] 100062 I [irq12: psm0] 100063 I [swi0: uart uart++] 100071 I [swi1: pf send] 100084 I [swi1: hpts] 100085 I [swi1: hpts] 11 0 0 0 RL (threaded) [idle] 100003 CanRun [idle: cpu0] 100004 CanRun [idle: cpu1] 1 0 1 0 SLs wait 0xfffff80004bda530 [init] 10 0 0 0 DL audit_w 0xffffffff828451f0 [audit] 0 0 0 0 RLs (threaded) [kernel] 100000 D swapin 0xffffffff8271ce70 [swapper] 100005 D - 0xfffff8000443b500 [if_io_tqg_0] 100006 D - 0xfffff8000443b100 [if_io_tqg_1] 100007 D - 0xfffff80004c78d00 [if_config_tqg_0] 100008 D - 0xfffff80004c78900 [softirq_0] 100009 D - 0xfffff80004c78500 [softirq_1] 100010 D - 0xfffff80004c78100 [aiod_kick taskq] 100012 D - 0xfffff80004c75900 [kqueue_ctx taskq] 100013 D - 0xfffff80004c75500 [pci_hp taskq] 100015 D - 0xfffff80004c6ed00 [inm_free taskq] 100017 D - 0xfffff80004c6e500 [linuxkpi_irq_wq] 100018 Run CPU 1 [thread taskq] 100019 D - 0xfffff80004c69d00 [in6m_free taskq] 100020 D - 0xfffff80004c69900 [linuxkpi_short_wq_0] 100021 D - 0xfffff80004c69900 [linuxkpi_short_wq_1] 100022 D - 0xfffff80004c69900 [linuxkpi_short_wq_2] 100023 D - 0xfffff80004c69900 [linuxkpi_short_wq_3] 100024 D - 0xfffff80004c69500 [linuxkpi_long_wq_0] 100025 D - 0xfffff80004c69500 [linuxkpi_long_wq_1] 100026 D - 0xfffff80004c69500 [linuxkpi_long_wq_2] 100027 D - 0xfffff80004c69500 [linuxkpi_long_wq_3] 100034 D - 0xfffff80004cd3d00 [firmware taskq] 100038 D - 0xfffff80004cd3900 [crypto_0] 100039 D - 0xfffff80004cd3900 [crypto_1] 100055 D - 0xfffff800153ecd00 [vtnet0 rxq 0] 100056 D - 0xfffff800153ec900 [vtnet0 txq 0] 100057 D - 0xfffff800153ec500 [vtnet0 rxq 1] 100058 D - 0xfffff800153ec100 [vtnet0 txq 1] 100060 D vtbslp 0xfffff80015463700 [virtio_balloon] 100064 D - 0xfffff80015469900 [mca taskq] 100069 D - 0xffffffff81e218a1 [deadlkres] 100074 D - 0xfffff80015ac0100 [acpi_task_0] 100075 D - 0xfffff80015ac0100 [acpi_task_1] 100076 D - 0xfffff80015ac0100 [acpi_task_2] 100077 D - 0xfffff80004cd3500 [CAM taskq] db> show all locks Process 29630 (syz-executor.1) thread 0xfffffe0095134e40 (130918) exclusive sleep mutex sctp-send-tcb (tcbs) r = 0 (0xfffffe009daed8d0) locked @ /syzkaller/managers/i386/kernel/sys/netinet/sctp_output.c:13125 Process 0 (kernel) thread 0xfffffe00042b6720 (100018) exclusive sleep mutex kernel arena (kernel arena) r = 0 (0xffffffff82786cc0) locked @ /syzkaller/managers/i386/kernel/sys/kern/subr_vmem.c:789 exclusive sleep mutex vmem list lock (vmem list lock) r = 0 (0xffffffff82202b40) locked @ /syzkaller/managers/i386/kernel/sys/kern/subr_vmem.c:784 db> show malloc Type InUse MemUse Requests pcb 3851 17786K 66911 sysctloid 34288 12858K 34355 pf_hash 5 11560K 5 devbuf 4216 6982K 4244 inodedep 3713 4232K 28810 sctp_stro 1913 3886K 12981 sctp_atcl 3847 3847K 43790 tcp_hpts 5 3219K 5 kobj 326 2608K 486 sctp_atky 5804 2177K 58746 vtbuf 24 2064K 46 dirrem 3704 1852K 28729 sctp_athm 3847 1443K 44663 sctp_map 3826 1435K 25958 freefile 3704 1389K 28716 newblk 9 1036K 29277 vfscache 3 1035K 3 filedesc 48 761K 57423 subproc 182 756K 29732 acpica 1674 649K 54806 linker 368 539K 435 callout 2 528K 2 ufs_quota 1 520K 1 vfs_hash 1 520K 1 intr 4 480K 4 bus 986 377K 3469 BPF 46 337K 50 DEVFS1 107 214K 124 kdtrace 260 209K 60551 module 511 192K 511 vnet_data 1 176K 1 umtx 396 174K 396 tidhash 3 164K 3 pagedep 9 140K 28704 tfo_ccache 1 136K 1 sem 4 120K 4 vmem 3 112K 7 UMA 270 102K 270 audit_evclass 236 89K 294 mtx_pool 2 80K 2 sctp_timw 157 79K 157 temp 35 77K 6601 syncache 1 76K 1 routetbl 131 70K 417 msg 4 68K 4 ifaddr 70 66K 72 acpitask 1 64K 1 ddb_capture 1 64K 1 DEVFS3 126 63K 136 ether_multi 152 57K 162 gtaskqueue 18 57K 18 DEVFS_RULE 56 54K 56 kenv 95 52K 95 eventhandler 133 50K 133 ithread 99 43K 99 rman 84 42K 425 kqueue 84 40K 29692 KTRACE 100 38K 100 taskqueue 60 36K 60 lltable 41 35K 134 proc 3 34K 3 ufs_mount 5 34K 6 bus-sc 33 34K 1684 devstat 4 33K 4 hostcache 1 32K 1 tty 16 32K 16 shm 1 32K 3 pwddesc 82 31K 29631 DEVFSP 80 30K 637 GEOM 60 29K 489 in6_multi 65 29K 65 kbdmux 6 28K 6 cred 26 26K 270 ifnet 7 25K 7 plimit 24 24K 462 kcovinfo 64 24K 68 CAM queue 5 21K 1528 selfd 55 21K 389641 pfs_nodes 20 20K 20 proc-args 52 20K 674 sctp_stri 18 18K 1866 UART 12 18K 12 session 35 18K 53 bmsafemap 2 17K 28802 ksem 1 16K 14 rpc 2 16K 2 shmfd 1 16K 25 pfs_vncache 1 16K 1 ufs_dirhash 24 14K 24 sglist 5 13K 5 MCA 32 12K 32 CAM DEV 3 12K 510 lockf 32 12K 60 vt 11 11K 11 acpisem 28 11K 28 CAM XPT 22 11K 543 Unitno 27 11K 47 uidinfo 3 9K 12 pf_ifnet 10 9K 21 memdesc 1 8K 1 ipsec-saq 2 8K 2 evdev 4 8K 4 acpidev 20 8K 20 hhook 15 8K 17 select 19 8K 66 mount 16 7K 90 fpukern_ctx 3 6K 3 inpcbpolicy 15 6K 19360 terminal 11 6K 11 ip6ndp 12 6K 13 sctp_ifa 13 5K 14 ipsecpolicy 2 5K 2 encap_export_host 12 5K 12 clone 9 5K 9 local_apic 1 4K 1 io_apic 1 4K 1 sahead 1 4K 1 secasvar 1 4K 1 nhops 6 4K 8 pci_link 10 4K 10 msi 9 4K 9 DEVFS 9 4K 10 tun 7 4K 7 osd 8 4K 12479 ipsec 3 3K 3 mkdir 8 3K 57372 in_multi 6 3K 8 nexusdev 7 3K 7 feeder 7 3K 7 procdesc 5 3K 14 freework 5 3K 28687 sctp_ifn 6 3K 14 mld 6 3K 6 igmp 6 3K 6 toponodes 6 3K 6 prison 6 3K 6 isadev 6 3K 6 softdep 1 2K 1 freeblks 4 2K 28686 vnodemarker 2 2K 74 NFSD session 1 2K 1 linux 5 2K 6 CAM periph 4 2K 271 soname 5 2K 38000 filecaps 5 2K 98 crypto 4 2K 4 newdirblk 4 2K 28686 diradd 4 2K 28769 pfil 4 2K 4 CAM path 4 2K 1034 tcpfunc 3 2K 3 loginclass 3 2K 6 chacha20random 1 1K 1 vnodes 1 1K 1 ktls 1 1K 1 cdev 2 1K 2 aesni_data 2 1K 2 cpus 2 1K 2 atkbddev 2 1K 2 CAM dev queue 2 1K 2 xform 2 1K 640 entropy 2 1K 46 CAM SIM 2 1K 2 NFSD lckfile 1 1K 1 NFSD V4client 1 1K 1 pmchooks 1 1K 1 sctp_vrf 1 1K 1 apmdev 1 1K 1 CAM I/O Scheduler 1 1K 1 vnet_data_free 1 1K 1 vnet 1 1K 1 iov 1 1K 41155 Per-cpu 1 1K 1 p1003.1b 1 1K 1 acpiintr 1 1K 1 pmc 1 1K 1 ext2_mount 0 0K 0 ext2_node 0 0K 0 ext2_extents 0 0K 0 pf_table 0 0K 0 pf_rule 0 0K 2 pf_altq 0 0K 0 pf_osfp 0 0K 0 pf_temp 0 0K 0 cdg data 0 0K 0 dctcp data 0 0K 0 cubic data 0 0K 10 chd data 0 0K 9 tcp_do 0 0K 0 tcp_fsb 0 0K 1019 sctp_mcore 0 0K 0 sctp_socko 0 0K 17565 sctp_iter 0 0K 316 sctp_mvrf 0 0K 0 sctp_cpal 0 0K 307 sctp_cmsg 0 0K 0 sctp_stre 0 0K 0 sctp_athi 0 0K 0 sctp_a_it 0 0K 9 sctp_aadr 0 0K 0 htcp data 0 0K 14 vegas data 0 0K 38 mqdata 0 0K 0 NFSCL lckown 0 0K 0 NFSCL client 0 0K 0 madt_table 0 0K 2 smartpqi 0 0K 0 NFSCL deleg 0 0K 0 NFSCL open 0 0K 0 NFSCL owner 0 0K 0 NFS fh 0 0K 0 NFS req 0 0K 0 NFSD usrgroup 0 0K 0 NFSD string 0 0K 0 iavf 0 0K 0 ixl 0 0K 0 NFSD V4lock 0 0K 0 NFSD V4state 0 0K 0 NFSD srvcache 0 0K 0 msdosfs_fat 0 0K 0 msdosfs_mount 0 0K 0 msdosfs_node 0 0K 0 ice-resmgr 0 0K 0 ice-osdep 0 0K 0 ice 0 0K 0 axgbe 0 0K 0 DEVFS4 0 0K 0 DEVFS2 0 0K 0 gntdev 0 0K 0 privcmd_dev 0 0K 0 evtchn_dev 0 0K 0 xen_intr 0 0K 0 xenstore 0 0K 0 ciss_data 0 0K 0 xen_hvm 0 0K 0 legacydrv 0 0K 0 qpidrv 0 0K 0 BACKLIGHT 0 0K 0 xnb 0 0K 0 dmar_idpgtbl 0 0K 0 dmar_dom 0 0K 0 dmar_ctx 0 0K 0 xbbd 0 0K 0 xbd 0 0K 0 Balloon 0 0K 0 isci 0 0K 0 iommu_dmamap 0 0K 0 sysmouse 0 0K 0 hyperv_socket 0 0K 0 bxe_ilt 0 0K 0 xenbus 0 0K 0 vtfont 0 0K 0 vm_fictitious 0 0K 0 ath_hal 0 0K 0 athdev 0 0K 0 ata_pci 0 0K 0 ata_dma 0 0K 0 ata_generic 0 0K 0 amr 0 0K 0 pvscsi 0 0K 0 scsi_da 0 0K 69 ata_da 0 0K 0 scsi_ch 0 0K 0 UMAHash 0 0K 0 scsi_cd 0 0K 0 AHCI driver 0 0K 0 vm_pgdata 0 0K 0 jblocks 0 0K 0 savedino 0 0K 22779 sentinel 0 0K 0 jfsync 0 0K 0 jtrunc 0 0K 0 sbdep 0 0K 35 jsegdep 0 0K 0 jseg 0 0K 0 jfreefrag 0 0K 0 jfreeblk 0 0K 0 jnewblk 0 0K 0 jmvref 0 0K 0 jremref 0 0K 0 jaddref 0 0K 0 freedep 0 0K 0 freefrag 0 0K 6 allocindir 0 0K 0 indirdep 0 0K 10 allocdirect 0 0K 0 ufs_trim 0 0K 0 mactemp 0 0K 0 audit_trigger 0 0K 0 audit_pipe_presel 0 0K 0 audit_pipeent 0 0K 0 audit_pipe 0 0K 0 audit_evname 0 0K 0 audit_bsm 0 0K 0 audit_gidset 0 0K 0 audit_text 0 0K 0 audit_path 0 0K 0 audit_data 0 0K 0 audit_cred 0 0K 0 USBdev 0 0K 0 USB 0 0K 0 agp 0 0K 0 nvme_da 0 0K 0 acpipwr 0 0K 0 acpi_perf 0 0K 0 ktls_ocf 0 0K 0 twsbuf 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5E_TLS 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EN 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5DUMP 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 seq_file 0 0K 0 radix 0 0K 0 idr 0 0K 0 lkpifw 0 0K 0 NLM 0 0K 0 ipsec-spdcache 0 0K 0 ipsec-reg 0 0K 0 ipsec-misc 0 0K 0 ipsecrequest 0 0K 0 ip6opt 0 0K 115 ip6_msource 0 0K 0 ip6_moptions 0 0K 0 in6_mfilter 0 0K 0 frag6 0 0K 0 tcplog 0 0K 0 tcp_hwpace 0 0K 0 twe_commands 0 0K 0 LRO 0 0K 0 newreno data 0 0K 0 ip_msource 0 0K 0 ip_moptions 0 0K 12 in_mfilter 0 0K 0 ipid 0 0K 0 80211scan 0 0K 0 80211ratectl 0 0K 0 80211power 0 0K 0 80211nodeie 0 0K 0 80211node 0 0K 0 80211mesh_gt 0 0K 0 80211mesh_rt 0 0K 0 80211perr 0 0K 0 80211prep 0 0K 0 80211preq 0 0K 0 80211dfs 0 0K 0 80211crypto 0 0K 0 80211vap 0 0K 0 iflib 0 0K 0 vlan 0 0K 0 gif 0 0K 0 ifdescr 0 0K 0 zlib 0 0K 0 fadvise 0 0K 0 VN POLL 0 0K 0 twa_commands 0 0K 0 statfs 0 0K 30113 namei_tracker 0 0K 0 export_host 0 0K 0 cl_savebuf 0 0K 4 tcp_log_dev 0 0K 1506 midi buffers 0 0K 0 mixer 0 0K 0 ac97 0 0K 0 hdacc 0 0K 0 hdac 0 0K 0 hdaa 0 0K 0 acpicmbat 0 0K 0 SIIS driver 0 0K 0 CAM CCB 0 0K 2725 PUC 0 0K 0 ppbusdev 0 0K 0 agtiapi_MemAlloc malloc 0 0K 0 osti_cacheable 0 0K 0 tempbuff 0 0K 0 biobuf 0 0K 0 aios 0 0K 4 lio 0 0K 22 acl 0 0K 0 tempbuff 0 0K 0 mbuf_tag 0 0K 132 ag_tgt_map_t malloc 0 0K 0 ag_slr_map_t malloc 0 0K 0 lDevFlags * malloc 0 0K 0 tiDeviceHandle_t * malloc 0 0K 0 ag_portal_data_t malloc 0 0K 0 ag_device_t malloc 0 0K 0 STLock malloc 0 0K 0 CCB List 0 0K 0 sr_iov 0 0K 0 OCS 0 0K 0 OCS 0 0K 0 nvme 0 0K 0 nvd 0 0K 0 netmap 0 0K 0 mwldev 0 0K 0 MVS driver 0 0K 0 CAM ccb queue 0 0K 0 mrsasbuf 0 0K 0 mpt_user 0 0K 0 mps_user 0 0K 0 accf 0 0K 0 pts 0 0K 0 ioctlops 0 0K 149 eventfd 0 0K 2 Witness 0 0K 0 stack 0 0K 0 MPSSAS 0 0K 0 mps 0 0K 0 mpr_user 0 0K 0 MPRSAS 0 0K 0 mpr 0 0K 0 mfibuf 0 0K 0 sbuf 0 0K 288 md_sectors 0 0K 0 firmware 0 0K 0 compressor 0 0K 0 md_disk 0 0K 0 SWAP 0 0K 0 malodev 0 0K 0 LED 0 0K 0 sysctltmp 0 0K 913 sysctl 0 0K 3 ekcd 0 0K 0 dumper 0 0K 0 sendfile 0 0K 0 rctl 0 0K 0 ix_sriov 0 0K 0 aacraidcam 0 0K 0 aacraid_buf 0 0K 0 ix 0 0K 0 ipsbuf 0 0K 0 cache 0 0K 0 iirbuf 0 0K 0 prison_racct 0 0K 0 Fail Points 0 0K 0 sigio 0 0K 1 filedesc_to_leader 0 0K 0 pwd 0 0K 0 tty console 0 0K 0 aaccam 0 0K 0 aacbuf 0 0K 0 zstd 0 0K 0 XZ_DEC 0 0K 0 nvlist 0 0K 0 SCSI ENC 0 0K 0 SCSI sa 0 0K 0 scsi_pass 0 0K 0 isofs_node 0 0K 0 isofs_mount 0 0K 0 tr_raid5_data 0 0K 0 tr_raid1e_data 0 0K 0 tr_raid1_data 0 0K 0 tr_raid0_data 0 0K 0 tr_concat_data 0 0K 0 md_sii_data 0 0K 0 md_promise_data 0 0K 0 md_nvidia_data 0 0K 0 md_jmicron_data 0 0K 0 md_intel_data 0 0K 0 md_ddf_data 0 0K 0 raid_data 0 0K 72 geom_flashmap 0 0K 0 tmpfs dir 0 0K 0 tmpfs name 0 0K 0 tmpfs mount 0 0K 0 NFS FHA 0 0K 0 newnfsmnt 0 0K 0 newnfsclient_req 0 0K 0 NFSCL layrecall 0 0K 0 NFSCL session 0 0K 0 NFSCL sockreq 0 0K 0 NFSCL devinfo 0 0K 0 NFSCL flayout 0 0K 0 NFSCL layout 0 0K 0 NFSD rollback 0 0K 0 NFSCL diroff 0 0K 0 NEWdirectio 0 0K 0 NEWNFSnode 0 0K 0 NFSCL lck 0 0K 0 db> show uma Zone Size Used Free Requests Sleeps Bucket Total Mem XFree mbuf_jumbo_page 4096 8322 949 750614 0 254 37974016 0 malloc-8192 8192 1909 493 29763 0 1 19677184 0 mbuf 256 71460 375 1494882 0 254 18389760 0 malloc-384 384 39318 1022 65094 0 30 15490560 0 sctp_asoc 2288 1913 537 12979 0 254 5605600 0 malloc-384 384 10749 2371 539966 0 30 5038080 0 sctp_chunk 152 32637 19 40621 0 254 4963712 0 malloc-2048 2048 1913 499 14193 0 8 4939776 0 malloc-1024 1024 3870 950 45762 0 16 4935680 0 malloc-1024 1024 4177 7 4184 0 16 4284416 0 tcp_log 416 0 9684 75545 0 254 4028544 0 malloc-1024 1024 3788 4 29453 0 16 3883008 0 mbuf_cluster 2048 1860 2 1860 0 254 3813376 0 sctp_ep 1280 1916 514 29758 0 254 3110400 0 malloc-8192 8192 328 2 1099 0 1 2703360 0 malloc-1024 1024 2103 509 60826 0 16 2674688 0 pbuf 2624 0 989 0 0 2 2595136 0 UMA Slabs 0 112 19868 1 19868 0 126 2225328 0 malloc-512 512 3842 14 29314 0 31 1974272 0 BUF TRIE 144 195 13273 1145 0 62 1939392 0 malloc-1024 1024 133 1659 24448 0 16 1835008 0 sctp_raddr 736 1913 540 13214 0 254 1805408 0 malloc-384 384 4264 16 137001 0 30 1643520 0 RADIX NODE 144 10702 411 675898 0 63 1600272 0 malloc-32768 32768 27 6 28714 0 1 1081344 0 malloc-8192 8192 84 6 29633 0 1 737280 0 malloc-384 384 1730 30 53827 0 30 675840 0 FFS inode 1160 527 26 29243 0 8 641480 0 mbuf_packet 256 1314 546 79946 0 254 476160 0 malloc-16384 16384 25 2 143 0 1 442368 0 256 Bucket 2048 162 48 24176 0 8 430080 0 VM OBJECT 264 1487 88 417498 0 30 415800 0 lkpimm 160 1 2324 1 0 62 372000 0 lkpicurr 160 2 2323 2 0 62 372000 0 THREAD 1808 177 21 30919 0 8 357984 0 socket 944 45 323 50519 0 254 347392 0 vmem btag 56 5833 59 5833 0 254 329952 0 malloc-65536 65536 4 1 5 0 1 327680 0 malloc-512 512 31 561 30606 0 30 303104 0 VNODE 448 566 91 29284 0 30 294336 0 tcp_bbr_map 128 0 2170 43551 0 126 277760 0 MAP ENTRY 96 2485 371 1595905 0 126 274176 0 malloc-2048 2048 126 2 145 0 8 262144 0 malloc-384 384 612 68 20050 0 30 261120 0 ertt_txseginfo 40 0 6363 168135 0 254 254520 0 tcp_inpcb 488 4 516 12467 0 254 253760 0 malloc-8192 8192 24 5 240 0 1 237568 0 DEVCTL 1024 0 220 153 0 0 225280 0 malloc-4096 4096 7 48 3686 0 2 225280 0 malloc-65536 65536 3 0 3 0 1 196608 0 malloc-16384 16384 11 1 247 0 1 196608 0 UMA Zones 768 242 2 242 0 16 187392 0 malloc-384 384 376 74 123543 0 30 172800 0 VMSPACE 2544 55 11 29610 0 4 167904 0 32 Bucket 256 211 434 23013 0 62 165120 0 S VFS Cache 104 1029 375 29866 0 126 146016 0 malloc-384 384 311 69 31645 0 30 145920 0 FFS2 dinode 256 527 28 29243 0 62 142080 0 tcpcb 1080 4 122 12467 0 254 136080 0 malloc-512 512 216 48 28962 0 30 135168 0 PROC 1328 81 18 29630 0 8 131472 0 malloc-65536 65536 2 0 2 0 1 131072 0 malloc-65536 65536 0 2 176 0 1 131072 0 malloc-512 512 199 57 59739 0 30 131072 0 malloc-1024 1024 112 4 118 0 16 118784 0 ksiginfo 112 84 960 959 0 126 116928 0 filedesc0 1072 82 23 29631 0 8 112560 0 16 Bucket 144 245 483 8054 0 62 104832 0 malloc-32768 32768 3 0 18 0 1 98304 0 malloc-32768 32768 1 2 121 0 1 98304 0 malloc-16384 16384 2 4 36 0 1 98304 0 malloc-16384 16384 4 2 6 0 1 98304 0 128 Bucket 1024 57 34 3913 0 16 93184 0 malloc-512 512 171 5 1945 0 30 90112 0 UMA Kegs 384 227 6 227 0 30 89472 0 sctp_stream_msg_out 112 706 86 8719 0 254 88704 0 g_bio 408 0 210 8365 0 30 85680 0 malloc-8192 8192 2 7 30124 0 1 73728 0 malloc-512 512 119 25 29097 0 30 73728 0 64 Bucket 512 57 79 17419 0 30 69632 0 malloc-65536 65536 0 1 1 0 1 65536 0 malloc-16384 16384 2 2 6 0 1 65536 0 malloc-2048 2048 10 20 518 0 8 61440 0 malloc-384 384 92 68 43194 0 30 61440 0 malloc-4096 4096 12 2 70 0 2 57344 0 ripcb 488 5 107 6496 0 254 54656 0 malloc-4096 4096 7 6 33 0 2 53248 0 malloc-16384 16384 3 0 3 0 1 49152 0 malloc-16384 16384 3 0 3 0 1 49152 0 malloc-8192 8192 3 3 13 0 1 49152 0 sctp_laddr 48 158 766 4467 0 254 44352 0 8 Bucket 80 108 442 6503 0 126 44000 0 clpbuf 2624 0 16 22 0 16 41984 0 malloc-4096 4096 8 2 35 0 2 40960 0 malloc-2048 2048 16 4 24 0 8 40960 0 malloc-512 512 23 57 1442 0 30 40960 0 pcpu-8 8 4701 419 4933 0 254 40960 0 Files 80 240 260 91924 0 126 40000 0 DIRHASH 1024 34 2 34 0 16 36864 0 NAMEI 1024 0 36 130893 0 16 36864 0 2 Bucket 32 346 788 9531 0 254 36288 0 malloc-32768 32768 1 0 1 0 1 32768 0 malloc-32768 32768 1 0 1 0 1 32768 0 malloc-32768 32768 1 0 1 0 1 32768 0 malloc-2048 2048 2 14 3715 0 8 32768 0 malloc-512 512 34 30 210 0 30 32768 0 pcpu-64 64 490 22 490 0 254 32768 0 TURNSTILE 136 199 32 199 0 62 31416 0 tcp_bbr_pcb 832 0 36 4652 0 16 29952 0 tcp_rack_pcb 832 0 36 1019 0 16 29952 0 KNOTE 160 28 147 428365 0 62 28000 0 pipe 744 24 11 491 0 16 26040 0 malloc-8192 8192 0 3 13 0 1 24576 0 malloc-8192 8192 1 2 61 0 1 24576 0 malloc-4096 4096 1 5 10 0 2 24576 0 PWD 32 29 727 28780 0 254 24192 0 ttyinq 160 135 15 300 0 62 24000 0 ttyoutq 256 72 18 160 0 62 23040 0 SLEEPQUEUE 88 199 57 199 0 126 22528 0 malloc-4096 4096 3 2 64 0 2 20480 0 Mountpoints 2752 2 5 2 0 4 19264 0 malloc-16384 16384 1 0 1 0 1 16384 0 malloc-4096 4096 2 2 30 0 2 16384 0 vtnet_tx_hdr 24 0 668 406564 0 254 16032 0 malloc-2048 2048 3 3 163 0 8 12288 0 malloc-2048 2048 2 4 6 0 8 12288 0 tcp_rack_map 112 0 108 1601 0 126 12096 0 ertt 72 4 164 12467 0 126 12096 0 4 Bucket 48 6 246 653 0 254 12096 0 udplite_inpcb 488 0 24 146 0 254 11712 0 udp_inpcb 488 6 18 251 0 254 11712 0 kenv 258 15 30 1038 0 30 11610 0 routing nhops 256 27 18 34 0 62 11520 0 unpcb 256 20 25 1352 0 254 11520 0 malloc-2048 2048 4 0 4 0 8 8192 0 malloc-1024 1024 1 7 9 0 16 8192 0 rtentry 176 30 16 34 0 62 8096 0 PGRP 88 35 57 55 0 126 8096 0 rl_entry 40 62 140 62 0 254 8080 0 udpcb 32 6 246 397 0 254 8064 0 sctp_readq 152 0 52 112 0 254 7904 0 AIO 208 0 38 9 0 62 7904 0 AIOCB 552 0 14 25 0 16 7728 0 AIOLIO 272 0 28 11 0 30 7616 0 FPU_save_area 832 1 8 1 0 16 7488 0 cpuset 104 7 55 7 0 126 6448 0 epoch_record pcpu 256 4 12 4 0 62 4096 0 malloc-4096 4096 1 0 1 0 2 4096 0 pcpu-16 16 7 249 7 0 254 4096 0 hostcache 64 1 62 1 0 254 4032 0 syncache 168 0 24 4 0 254 4032 0 AIOP 32 4 122 4 0 254 4032 0 itimer 352 0 11 2 0 30 3872 0 UMA Slabs 1 176 9 13 9 0 62 3872 0 mqnode 416 3 6 3 0 30 3744 0 KMAP ENTRY 96 15 24 16 0 0 3744 0 vmem 1856 1 1 1 0 8 3712 0 SMR CPU 32 3 60 3 0 254 2016 0 SMR SHARED 24 3 60 3 0 254 1512 0 FFS1 dinode 128 0 0 0 0 126 0 0 da_ccb 544 0 0 0 0 16 0 0 ada_ccb 272 0 0 0 0 30 0 0 swblk 136 0 0 0 0 62 0 0 swpctrie 144 0 0 0 0 62 0 0 sctp_asconf_ack 48 0 0 0 0 254 0 0 sctp_asconf 40 0 0 0 0 254 0 0 cdg_qdiffsample 16 0 0 0 0 254 0 0 pf state scrubs 40 0 0 0 0 254 0 0 pf frag entries 40 0 0 0 0 254 0 0 pf frags 248 0 0 0 0 62 0 0 pf table entries 160 0 0 0 0 62 0 0 pf table entry counters 64 0 0 0 0 254 0 0 pf source nodes 136 0 0 0 0 254 0 0 pf state keys 88 0 0 0 0 126 0 0 pf states 304 0 0 0 0 254 0 0 pf tags 104 0 0 0 0 126 0 0 pf mtags 48 0 0 0 0 254 0 0 tcp_log_node 120 0 0 0 0 126 0 0 tcp_log_bucket 176 0 0 0 0 62 0 0 tcpreass 48 0 0 0 0 254 0 0 tfo_ccache_entries 80 0 0 0 0 126 0 0 tfo 4 0 0 0 0 254 0 0 sackhole 32 0 0 0 0 254 0 0 tcptw 88 0 0 0 0 254 0 0 ipq 56 0 0 0 0 254 0 0 IPsec SA lft_c 16 0 0 0 0 254 0 0 TMPFS node 224 0 0 0 0 62 0 0 mqnotifier 216 0 0 0 0 62 0 0 mvdata 64 0 0 0 0 254 0 0 mqueue 248 0 0 0 0 62 0 0 NCLNODE 584 0 0 0 0 16 0 0 LTS VFS Cache 360 0 0 0 0 30 0 0 L VFS Cache 320 0 0 0 0 30 0 0 STS VFS Cache 144 0 0 0 0 62 0 0 cryptop 280 0 0 0 0 30 0 0 linux_dma_object 24 0 0 0 0 254 0 0 linux_dma_pctrie 144 0 0 0 0 62 0 0 IOMMU_MAP_ENTRY 120 0 0 0 0 126 0 0 ktls_session 128 0 0 0 0 126 0 0 mbuf_jumbo_16k 16384 0 0 0 0 254 0 0 mbuf_jumbo_9k 9216 0 0 0 0 254 0 0 audit_record 1280 0 0 0 0 8 0 0 domainset 40 0 0 0 0 254 0 0 MAC labels 40 0 0 0 0 254 0 0 vnpbuf 2624 0 0 0 0 64 0 0 mdpbuf 2624 0 0 0 0 3 0 0 nfspbuf 2624 0 0 0 0 16 0 0 swwbuf 2624 0 0 0 0 8 0 0 swrbuf 2624 0 0 0 0 16 0 0 umtx_shm 88 0 0 0 0 126 0 0 umtx pi 96 0 0 0 0 126 0 0 rangeset pctrie nodes 144 0 0 0 0 62 0 0 malloc-65536 65536 0 0 0 0 1 0 0 malloc-65536 65536 0 0 0 0 1 0 0 malloc-65536 65536 0 0 0 0 1 0 0 malloc-32768 32768 0 0 0 0 1 0 0 malloc-32768 32768 0 0 0 0 1 0 0 malloc-1024 1024 0 0 0 0 16 0 0 malloc-256 256 0 0 0 0 62 0 0 malloc-256 256 0 0 0 0 62 0 0 malloc-256 256 0 0 0 0 62 0 0 malloc-256 256 0 0 0 0 62 0 0 malloc-256 256 0 0 0 0 62 0 0 malloc-256 256 0 0 0 0 62 0 0 malloc-256 256 0 0 0 0 62 0 0 malloc-256 256 0 0 0 0 62 0 0 malloc-128 128 0 0 0 0 126 0 0 malloc-128 128 0 0 0 0 126 0 0 malloc-128 128 0 0 0 0 126 0 0 malloc-128 128 0 0 0 0 126 0 0 malloc-128 128 0 0 0 0 126 0 0 malloc-128 128 0 0 0 0 126 0 0 malloc-128 128 0 0 0 0 126 0 0 malloc-128 128 0 0 0 0 126 0 0 malloc-64 64 0 0 0 0 254 0 0 malloc-64 64 0 0 0 0 254 0 0 malloc-64 64 0 0 0 0 254 0 0 malloc-64 64 0 0 0 0 254 0 0 malloc-64 64 0 0 0 0 254 0 0 malloc-64 64 0 0 0 0 254 0 0 malloc-64 64 0 0 0 0 254 0 0 malloc-64 64 0 0 0 0 254 0 0 malloc-32 32 0 0 0 0 254 0 0 malloc-32 32 0 0 0 0 254 0 0 malloc-32 32 0 0 0 0 254 0 0 malloc-32 32 0 0 0 0 254 0 0 malloc-32 32 0 0 0 0 254 0 0 malloc-32 32 0 0 0 0 254 0 0 malloc-32 32 0 0 0 0 254 0 0 malloc-32 32 0 0 0 0 254 0 0 malloc-16 16 0 0 0 0 254 0 0 malloc-16 16 0 0 0 0 254 0 0 malloc-16 16 0 0 0 0 254 0 0 malloc-16 16 0 0 0 0 254 0 0 malloc-16 16 0 0 0 0 254 0 0 malloc-16 16 0 0 0 0 254 0 0 malloc-16 16 0 0 0 0 254 0 0 malloc-16 16 0 0 0 0 254 0 0 pcpu-32 32 0 0 0 0 254 0 0 pcpu-4 4 0 0 0 0 254 0 0 fakepg 104 0 0 0 0 126 0 0 UMA Hash 256 0 0 0 0 62 0 0