panic: mtx_lock() of spin mutex (null) @ /syzkaller/managers/i386/kernel/sys/modules/tcp/rack/../../../netinet/tcp_stacks/rack.c:16480 cpuid = 0 time = 1625081798 KDB: stack backtrace: db_trace_self_wrapper() at db_trace_self_wrapper+0x47/frame 0xfffffe00516fd550 vpanic() at vpanic+0x1cc/frame 0xfffffe00516fd5b0 panic() at panic+0x43/frame 0xfffffe00516fd610 __mtx_lock_flags() at __mtx_lock_flags+0x1d7/frame 0xfffffe00516fd670 rack_output() at rack_output+0x4b83/frame 0xfffffe00516fd9c0 tcp_hpts_thread() at tcp_hpts_thread+0xe50/frame 0xfffffe00516fdb10 ithread_loop() at ithread_loop+0x33f/frame 0xfffffe00516fdbb0 fork_exit() at fork_exit+0xb3/frame 0xfffffe00516fdbf0 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe00516fdbf0 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- KDB: enter: panic [ thread pid 12 tid 100084 ] Stopped at kdb_enter+0x67: movq $0,0x16400ee(%rip) db> db> set $lines = 0 db> set $maxwidth = 0 db> show registers cs 0x20 ds 0x3b es 0x3b fs 0x13 gs 0x1b ss 0 rax 0x12 rcx 0xffffffff81132780 vprintf+0x140 rdx 0x1 rbx 0 rsp 0xfffffe00516fd530 rbp 0xfffffe00516fd550 rsi 0 rdi 0xffffffff811327b6 vprintf+0x176 r8 0 r9 0x8080808080808080 r10 0xfffffe00516fd420 r11 0x1ffaefff59c r12 0xffffffff82267d30 ddb_dbbe r13 0xffffffff81a6e600 r14 0xffffffff81a74f75 r15 0xffffffff81a74f75 rip 0xffffffff81129817 kdb_enter+0x67 rflags 0x86 kdb_enter+0x67: movq $0,0x16400ee(%rip) db> show proc Process 12 (intr) at 0xfffff80004c74a70: state: NORMAL uid: 0 gids: 0 parent: pid 0 at 0xffffffff8271ce70 ABI: null flag: 0x10000284 flag2: 0 reaper: 0xffffffff8271ce70 reapsubtree: 12 sigparent: 20 vmspace: 0xffffffff8271daf0 (map 0xffffffff8271daf0) (map.pmap 0xffffffff8271dbb0) (pmap 0xffffffff8271dc10) threads: 23 100011 I [swi5: fast taskq] 100014 I [swi6: task queue] 100016 I [swi6: Giant taskq] 100030 I [swi3: vm] 100031 I [swi4: clock (0)] 100032 I [swi4: clock (1)] 100033 I [swi1: netisr 0] 100046 I [irq24: virtio_pci0] 100047 I [irq25: virtio_pci0] 100048 I [irq26: virtio_pci0] 100049 I [irq27: virtio_pci0] 100050 I [irq28: virtio_pci1] 100051 I [irq29: virtio_pci1] 100052 I [irq30: virtio_pci1] 100053 I [irq31: virtio_pci1] 100054 I [irq32: virtio_pci1] 100059 I [irq10: virtio_pci2] 100061 I [irq1: atkbd0] 100062 I [irq12: psm0] 100063 I [swi0: uart uart++] 100071 I [swi1: pf send] 100084 Run CPU 0 [swi1: hpts] 100085 I [swi1: hpts] db> ps pid ppid pgrp uid state wmesg wchan cmd 3620 808 808 0 R syz-executor.3 3619 788 788 0 R (threaded) syz-executor.1 100480 RunQ syz-executor.1 103740 RunQ syz-executor.1 3618 787 787 0 R (threaded) syz-executor.0 101225 Run CPU 1 syz-executor.0 103741 RunQ syz-executor.0 3616 799 799 0 R (threaded) syz-executor.2 101724 RunQ syz-executor.2 103736 S uwait 0xfffff8004edf0100 syz-executor.2 103739 S uwait 0xfffff80060096f00 syz-executor.2 3084 3078 3084 0 Ss select 0xfffff8004e3ad140 dhclient 3081 1 3081 0 Ss select 0xfffff8004efdb8c0 dhclient 3078 3071 436 65 S select 0xfffff8004ed362c0 dhclient 3071 436 436 0 S wait 0xfffff80015dff000 sh 3061 1 3061 65 Ss select 0xfffff8004eea8a40 dhclient 1986 1 1986 0 Ss select 0xfffff8004ed36740 dhclient 1983 1 1983 0 Ss select 0xfffff8004eeb5140 dhclient 1962 1 1962 65 Ss select 0xfffff8004efdbbc0 dhclient 1460 1 1460 0 Ss select 0xfffff8004ef0a5c0 dhclient 1457 1 1457 0 Ss select 0xfffff8004ef0abc0 dhclient 1438 1 1438 65 Ss select 0xfffff8004ed365c0 dhclient 1093 0 0 0 DL aiordy 0xfffff8004efbaa70 [aiod4] 1092 0 0 0 DL aiordy 0xfffff8004e42f000 [aiod3] 1091 0 0 0 DL aiordy 0xfffff8004e4b0538 [aiod2] 1090 0 0 0 DL aiordy 0xfffff8004e4b0000 [aiod1] 819 1 819 0 Ss select 0xfffff80015dcbbc0 dhclient 816 1 816 0 Ss select 0xfffff8004e362bc0 dhclient 808 780 808 0 Ss nanslp 0xffffffff8273d0a0 syz-executor.3 799 780 799 0 Ss nanslp 0xffffffff8273d0a0 syz-executor.2 788 780 788 0 Rs syz-executor.1 787 780 787 0 Ss nanslp 0xffffffff8273d0a0 syz-executor.0 780 778 778 0 S (threaded) syz-fuzzer 100103 S uwait 0xfffff80015edd100 syz-fuzzer 100122 S uwait 0xfffff8004e4f6f00 syz-fuzzer 100123 S uwait 0xfffff8004e45f100 syz-fuzzer 100124 S uwait 0xfffff8004e45f300 syz-fuzzer 100125 S uwait 0xfffff8004e45f500 syz-fuzzer 100126 S uwait 0xfffff8004e35ed00 syz-fuzzer 100127 S kqread 0xfffff8004e4f6100 syz-fuzzer 100128 S uwait 0xfffff8004e45f900 syz-fuzzer 778 776 778 0 Ss pause 0xfffff80015e55b20 csh 776 694 776 0 Ss select 0xfffff8004e2be2c0 sshd 760 1 760 0 Ss+ ttyin 0xfffff800154a5cb0 getty 759 1 759 0 Ss+ ttyin 0xfffff80015b2e4b0 getty 758 1 758 0 Ss+ ttyin 0xfffff80015b2ecb0 getty 757 1 757 0 Ss+ ttyin 0xfffff80015ab64b0 getty 756 1 756 0 Ss+ ttyin 0xfffff80015ab6cb0 getty 755 1 755 0 Ss+ ttyin 0xfffff80015ab84b0 getty 754 1 754 0 Ss+ ttyin 0xfffff80015ab8cb0 getty 753 1 753 0 Ss+ ttyin 0xfffff80015abf4b0 getty 752 1 752 0 Ss+ ttyin 0xfffff80015abfcb0 getty 698 1 698 0 Ss nanslp 0xffffffff8273d0a0 cron 694 1 694 0 Ss select 0xfffff80015f0c140 sshd 507 1 507 0 Ss select 0xfffff8004e214ec0 syslogd 436 1 436 0 Ss wait 0xfffff80015e4b000 devd 435 1 435 65 Ss select 0xfffff8004e214d40 dhclient 350 1 350 0 Ss select 0xfffff80015f90a40 dhclient 347 1 347 0 Ss select 0xfffff8004e2beec0 dhclient 23 0 0 0 DL vlruwt 0xfffff80015cd1538 [vnlru] 22 0 0 0 DL syncer 0xffffffff8282c450 [syncer] 21 0 0 0 DL (threaded) [bufdaemon] 100081 D qsleep 0xffffffff8282b520 [bufdaemon] 100086 D - 0xffffffff8220aec0 [bufspacedaemon-0] 100099 D sdflush 0xfffff80004ddc4e8 [/ worker] 20 0 0 0 DL psleep 0xffffffff82853388 [vmdaemon] 19 0 0 0 DL (threaded) [pagedaemon] 100079 D psleep 0xffffffff828477f8 [dom0] 100087 D launds 0xffffffff82847804 [laundry: dom0] 100088 D umarcl 0xffffffff815c5b10 [uma] 18 0 0 0 RL [rand_harvestq] 17 0 0 0 DL waiting 0xffffffff830da8b8 [sctp_iterator] 16 0 0 0 DL pftm 0xffffffff82dc1410 [pf purge] 15 0 0 0 DL - 0xffffffff82828bdc [soaiod4] 9 0 0 0 DL - 0xffffffff82828bdc [soaiod3] 8 0 0 0 DL - 0xffffffff82828bdc [soaiod2] 7 0 0 0 DL - 0xffffffff82828bdc [soaiod1] 6 0 0 0 DL (threaded) [cam] 100044 D - 0xffffffff82448b40 [doneq0] 100045 D - 0xffffffff82448ac0 [async] 100078 D - 0xffffffff82448990 [scanner] 14 0 0 0 DL seqstat 0xfffff80004d8a488 [sequencer 00] 5 0 0 0 DL crypto_ 0xfffff80004d53d80 [crypto returns 1] 4 0 0 0 DL crypto_ 0xfffff80004d53d30 [crypto returns 0] 3 0 0 0 DL crypto_ 0xffffffff82844ce0 [crypto] 13 0 0 0 DL (threaded) [geom] 100035 D - 0xffffffff8271c8e0 [g_event] 100036 D - 0xffffffff8271c8e8 [g_up] 100037 D - 0xffffffff8271c8f0 [g_down] 2 0 0 0 DL (threaded) [KTLS] 100028 D - 0xfffff80004c69100 [thr_0] 100029 D - 0xfffff80004c69180 [thr_1] 12 0 0 0 RL (threaded) [intr] 100011 I [swi5: fast taskq] 100014 I [swi6: task queue] 100016 I [swi6: Giant taskq] 100030 I [swi3: vm] 100031 I [swi4: clock (0)] 100032 I [swi4: clock (1)] 100033 I [swi1: netisr 0] 100046 I [irq24: virtio_pci0] 100047 I [irq25: virtio_pci0] 100048 I [irq26: virtio_pci0] 100049 I [irq27: virtio_pci0] 100050 I [irq28: virtio_pci1] 100051 I [irq29: virtio_pci1] 100052 I [irq30: virtio_pci1] 100053 I [irq31: virtio_pci1] 100054 I [irq32: virtio_pci1] 100059 I [irq10: virtio_pci2] 100061 I [irq1: atkbd0] 100062 I [irq12: psm0] 100063 I [swi0: uart uart++] 100071 I [swi1: pf send] 100084 Run CPU 0 [swi1: hpts] 100085 I [swi1: hpts] 11 0 0 0 RL (threaded) [idle] 100003 CanRun [idle: cpu0] 100004 CanRun [idle: cpu1] 1 0 1 0 SLs wait 0xfffff80004bda538 [init] 10 0 0 0 DL audit_w 0xffffffff828451f0 [audit] 0 0 0 0 DLs (threaded) [kernel] 100000 D swapin 0xffffffff8271ce70 [swapper] 100005 D - 0xfffff8000443b500 [if_io_tqg_0] 100006 D - 0xfffff8000443b100 [if_io_tqg_1] 100007 D - 0xfffff80004c78d00 [if_config_tqg_0] 100008 D - 0xfffff80004c78900 [softirq_0] 100009 D - 0xfffff80004c78500 [softirq_1] 100010 D - 0xfffff80004c78100 [aiod_kick taskq] 100012 D - 0xfffff80004c75900 [kqueue_ctx taskq] 100013 D - 0xfffff80004c75500 [pci_hp taskq] 100015 D - 0xfffff80004c6ed00 [inm_free taskq] 100017 D - 0xfffff80004c6e500 [linuxkpi_irq_wq] 100018 D - 0xfffff80004c6e100 [thread taskq] 100019 D - 0xfffff80004c69d00 [in6m_free taskq] 100020 D - 0xfffff80004c69900 [linuxkpi_short_wq_0] 100021 D - 0xfffff80004c69900 [linuxkpi_short_wq_1] 100022 D - 0xfffff80004c69900 [linuxkpi_short_wq_2] 100023 D - 0xfffff80004c69900 [linuxkpi_short_wq_3] 100024 D - 0xfffff80004c69500 [linuxkpi_long_wq_0] 100025 D - 0xfffff80004c69500 [linuxkpi_long_wq_1] 100026 D - 0xfffff80004c69500 [linuxkpi_long_wq_2] 100027 D - 0xfffff80004c69500 [linuxkpi_long_wq_3] 100034 D - 0xfffff80004cd3d00 [firmware taskq] 100038 D - 0xfffff80004cd3900 [crypto_0] 100039 D - 0xfffff80004cd3900 [crypto_1] 100055 D - 0xfffff800153ebd00 [vtnet0 rxq 0] 100056 D - 0xfffff800153eb900 [vtnet0 txq 0] 100057 D - 0xfffff800153eb500 [vtnet0 rxq 1] 100058 D - 0xfffff800153eb100 [vtnet0 txq 1] 100060 D vtbslp 0xfffff80015463700 [virtio_balloon] 100064 D - 0xfffff80015468900 [mca taskq] 100066 D - 0xffffffff81e218a0 [deadlkres] 100074 D - 0xfffff80015ac7100 [acpi_task_0] 100075 D - 0xfffff80015ac7100 [acpi_task_1] 100076 D - 0xfffff80015ac7100 [acpi_task_2] 100077 D - 0xfffff80004cd3500 [CAM taskq] db> show all locks Process 3620 (syz-executor.3) thread 0xfffffe00950df720 (101241) exclusive sx vm map (user) (vm map (user)) r = 0 (0xfffffe009517e440) locked @ /syzkaller/managers/i386/kernel/sys/vm/vm_map.c:4935 Process 12 (intr) thread 0xfffffe0055374000 (100084) exclusive rw tcpinp (tcpinp) r = 0 (0xfffff800604d5d78) locked @ /syzkaller/managers/i386/kernel/sys/netinet/tcp_hpts.c:1536 db> show malloc Type InUse MemUse Requests sysctloid 34288 12858K 34355 pf_hash 5 11560K 5 devbuf 4216 6982K 4244 tcp_hpts 5 3219K 5 kobj 326 2608K 486 vtbuf 24 2064K 46 newblk 9 1036K 3306 vfscache 3 1035K 3 pcb 67 761K 7860 inodedep 135 654K 2820 acpica 1674 649K 54806 subproc 150 626K 3703 callout 2 528K 2 ufs_quota 1 520K 1 vfs_hash 1 520K 1 intr 4 480K 4 bus 986 377K 3469 BPF 46 337K 50 filedesc 18 275K 5461 linker 348 270K 409 DEVFS1 107 214K 124 module 510 192K 510 kdtrace 227 185K 7363 vnet_data 1 176K 1 tidhash 3 164K 3 umtx 352 154K 352 pagedep 11 141K 2735 tfo_ccache 1 136K 1 sem 4 120K 4 UMA 270 102K 270 audit_evclass 236 89K 294 mtx_pool 2 80K 2 temp 35 77K 3142 syncache 1 76K 1 vmem 3 72K 5 routetbl 130 70K 418 msg 4 68K 4 ifaddr 70 66K 72 acpitask 1 64K 1 ddb_capture 1 64K 1 DEVFS3 126 63K 136 dirrem 126 63K 2734 ether_multi 152 57K 176 gtaskqueue 18 57K 18 DEVFS_RULE 56 54K 56 lltable 59 53K 72 kenv 95 52K 95 eventhandler 133 50K 133 freefile 126 48K 2732 ithread 99 43K 99 rman 84 42K 425 KTRACE 100 38K 100 taskqueue 60 36K 60 proc 3 34K 3 kqueue 70 34K 3629 ufs_mount 5 34K 6 bus-sc 33 34K 1684 devstat 4 33K 4 hostcache 1 32K 1 tty 16 32K 16 shm 1 32K 1 DEVFSP 80 30K 123 sctp_atcl 29 29K 3851 GEOM 60 29K 489 in6_multi 65 29K 65 kbdmux 6 28K 6 sctp_stro 13 26K 919 cred 26 26K 280 pwddesc 68 26K 3621 ifnet 7 25K 7 plimit 24 24K 424 kcovinfo 64 24K 68 CAM queue 5 21K 1528 proc-args 53 21K 643 pfs_nodes 20 20K 20 selfd 52 20K 62435 sctp_timw 38 19K 38 UART 12 18K 12 session 34 17K 46 bmsafemap 2 17K 2793 ksem 1 16K 12 rpc 2 16K 2 shmfd 1 16K 1 pfs_vncache 1 16K 1 sctp_atky 42 16K 4777 ufs_dirhash 24 14K 24 sglist 5 13K 5 MCA 32 12K 32 CAM DEV 3 12K 510 lockf 32 12K 54 Unitno 30 12K 47 vt 11 11K 11 sctp_athm 29 11K 3852 acpisem 28 11K 28 CAM XPT 22 11K 543 sctp_map 26 10K 1838 inpcbpolicy 24 9K 2753 uidinfo 3 9K 12 pf_ifnet 10 9K 19 memdesc 1 8K 1 ipsec-saq 2 8K 2 evdev 4 8K 4 acpidev 20 8K 20 hhook 15 8K 17 select 19 8K 50 mount 16 7K 90 osd 16 7K 2382 fpukern_ctx 3 6K 3 terminal 11 6K 11 ip6ndp 12 6K 13 sctp_ifa 13 5K 14 ipsecpolicy 2 5K 2 encap_export_host 12 5K 12 clone 9 5K 9 local_apic 1 4K 1 io_apic 1 4K 1 sahead 1 4K 1 secasvar 1 4K 1 nhops 6 4K 8 pci_link 10 4K 10 msi 9 4K 9 DEVFS 9 4K 10 tun 7 4K 7 ipsec 3 3K 3 mkdir 8 3K 5432 in_multi 6 3K 12 tcp_fsb 7 3K 1147 nexusdev 7 3K 7 feeder 7 3K 7 procdesc 5 3K 14 freework 5 3K 2725 sctp_ifn 6 3K 14 newdirblk 6 3K 2716 mld 6 3K 6 igmp 6 3K 6 toponodes 6 3K 6 prison 6 3K 6 isadev 6 3K 6 softdep 1 2K 1 freeblks 4 2K 2724 vnodemarker 2 2K 32 NFSD session 1 2K 1 lkpikmalloc 5 2K 6 CAM periph 4 2K 271 filecaps 5 2K 97 crypto 4 2K 8 diradd 4 2K 2774 pfil 4 2K 4 CAM path 4 2K 1034 soname 4 2K 8070 tcpfunc 3 2K 3 loginclass 3 2K 6 chacha20random 1 1K 1 vnodes 1 1K 1 ktls 1 1K 1 cdev 2 1K 2 aesni_data 2 1K 2 cpus 2 1K 2 atkbddev 2 1K 2 CAM dev queue 2 1K 2 xform 2 1K 75 entropy 2 1K 42 CAM SIM 2 1K 2 NFSD lckfile 1 1K 1 NFSD V4client 1 1K 1 pmchooks 1 1K 1 sctp_vrf 1 1K 1 apmdev 1 1K 1 CAM I/O Scheduler 1 1K 1 vnet_data_free 1 1K 1 vnet 1 1K 1 Per-cpu 1 1K 1 p1003.1b 1 1K 1 acpiintr 1 1K 1 pmc 1 1K 1 vegas data 0 0K 0 sctp_mcore 0 0K 0 sctp_socko 0 0K 1432 sctp_iter 0 0K 8 sctp_mvrf 0 0K 0 sctp_cpal 0 0K 0 sctp_cmsg 0 0K 0 sctp_stre 0 0K 0 sctp_athi 0 0K 0 sctp_a_it 0 0K 8 sctp_aadr 0 0K 0 sctp_stri 0 0K 2 tcp_do 0 0K 0 cdg data 0 0K 0 mqdata 0 0K 0 htcp data 0 0K 0 dctcp data 0 0K 0 chd data 0 0K 2 pf_table 0 0K 0 pf_rule 0 0K 0 pf_altq 0 0K 0 pf_osfp 0 0K 0 pf_temp 0 0K 0 cubic data 0 0K 0 NFSCL lckown 0 0K 0 NFSCL client 0 0K 0 madt_table 0 0K 2 smartpqi 0 0K 0 NFSCL deleg 0 0K 0 NFSCL open 0 0K 0 NFSCL owner 0 0K 0 NFS fh 0 0K 0 NFS req 0 0K 0 NFSD usrgroup 0 0K 0 NFSD string 0 0K 0 iavf 0 0K 0 ixl 0 0K 0 NFSD V4lock 0 0K 0 NFSD V4state 0 0K 0 NFSD srvcache 0 0K 0 msdosfs_fat 0 0K 0 msdosfs_mount 0 0K 0 msdosfs_node 0 0K 0 ice-resmgr 0 0K 0 ice-osdep 0 0K 0 ice 0 0K 0 axgbe 0 0K 0 DEVFS4 0 0K 0 DEVFS2 0 0K 0 gntdev 0 0K 0 privcmd_dev 0 0K 0 evtchn_dev 0 0K 0 xen_intr 0 0K 0 xenstore 0 0K 0 ciss_data 0 0K 0 xen_hvm 0 0K 0 legacydrv 0 0K 0 qpidrv 0 0K 0 BACKLIGHT 0 0K 0 xnb 0 0K 0 dmar_idpgtbl 0 0K 0 dmar_dom 0 0K 0 dmar_ctx 0 0K 0 xbbd 0 0K 0 xbd 0 0K 0 Balloon 0 0K 0 isci 0 0K 0 iommu_dmamap 0 0K 0 sysmouse 0 0K 0 hyperv_socket 0 0K 0 bxe_ilt 0 0K 0 xenbus 0 0K 0 vtfont 0 0K 0 vm_fictitious 0 0K 0 ath_hal 0 0K 0 athdev 0 0K 0 ata_pci 0 0K 0 ata_dma 0 0K 0 ata_generic 0 0K 0 amr 0 0K 0 pvscsi 0 0K 0 scsi_da 0 0K 69 ata_da 0 0K 0 scsi_ch 0 0K 0 UMAHash 0 0K 0 scsi_cd 0 0K 0 AHCI driver 0 0K 0 vm_pgdata 0 0K 0 jblocks 0 0K 0 savedino 0 0K 2283 sentinel 0 0K 0 jfsync 0 0K 0 jtrunc 0 0K 0 sbdep 0 0K 13 jsegdep 0 0K 0 jseg 0 0K 0 jfreefrag 0 0K 0 jfreeblk 0 0K 0 jnewblk 0 0K 0 jmvref 0 0K 0 jremref 0 0K 0 jaddref 0 0K 0 freedep 0 0K 0 freefrag 0 0K 6 allocindir 0 0K 0 indirdep 0 0K 10 allocdirect 0 0K 0 ufs_trim 0 0K 0 mactemp 0 0K 0 audit_trigger 0 0K 0 audit_pipe_presel 0 0K 0 audit_pipeent 0 0K 0 audit_pipe 0 0K 0 audit_evname 0 0K 0 audit_bsm 0 0K 0 audit_gidset 0 0K 0 audit_text 0 0K 0 audit_path 0 0K 0 audit_data 0 0K 0 audit_cred 0 0K 0 USBdev 0 0K 0 USB 0 0K 0 agp 0 0K 0 nvme_da 0 0K 0 acpipwr 0 0K 0 acpi_perf 0 0K 0 ktls_ocf 0 0K 0 twsbuf 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5E_TLS 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EN 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5DUMP 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 seq_file 0 0K 0 radix 0 0K 0 idr 0 0K 0 lkpifw 0 0K 0 NLM 0 0K 0 ipsec-spdcache 0 0K 0 ipsec-reg 0 0K 0 ipsec-misc 0 0K 0 ipsecrequest 0 0K 0 ip6opt 0 0K 997 ip6_msource 0 0K 0 ip6_moptions 0 0K 0 in6_mfilter 0 0K 0 frag6 0 0K 0 tcplog 0 0K 0 tcp_hwpace 0 0K 0 twe_commands 0 0K 0 LRO 0 0K 0 newreno data 0 0K 0 ip_msource 0 0K 0 ip_moptions 0 0K 11 in_mfilter 0 0K 8 ipid 0 0K 0 80211scan 0 0K 0 80211ratectl 0 0K 0 80211power 0 0K 0 80211nodeie 0 0K 0 80211node 0 0K 0 80211mesh_gt 0 0K 0 80211mesh_rt 0 0K 0 80211perr 0 0K 0 80211prep 0 0K 0 80211preq 0 0K 0 80211dfs 0 0K 0 80211crypto 0 0K 0 80211vap 0 0K 0 iflib 0 0K 0 vlan 0 0K 0 gif 0 0K 0 ifdescr 0 0K 0 zlib 0 0K 0 fadvise 0 0K 0 VN POLL 0 0K 0 twa_commands 0 0K 0 statfs 0 0K 2900 namei_tracker 0 0K 0 export_host 0 0K 0 cl_savebuf 0 0K 4 tcp_log_dev 0 0K 37 midi buffers 0 0K 0 mixer 0 0K 0 ac97 0 0K 0 hdacc 0 0K 0 hdac 0 0K 0 hdaa 0 0K 0 acpicmbat 0 0K 0 SIIS driver 0 0K 0 CAM CCB 0 0K 2127 PUC 0 0K 0 ppbusdev 0 0K 0 agtiapi_MemAlloc malloc 0 0K 0 osti_cacheable 0 0K 0 tempbuff 0 0K 0 biobuf 0 0K 0 aios 0 0K 2 lio 0 0K 4 acl 0 0K 0 tempbuff 0 0K 0 mbuf_tag 0 0K 121 ag_tgt_map_t malloc 0 0K 0 ag_slr_map_t malloc 0 0K 0 lDevFlags * malloc 0 0K 0 tiDeviceHandle_t * malloc 0 0K 0 ag_portal_data_t malloc 0 0K 0 ag_device_t malloc 0 0K 0 STLock malloc 0 0K 0 CCB List 0 0K 0 sr_iov 0 0K 0 OCS 0 0K 0 OCS 0 0K 0 nvme 0 0K 0 nvd 0 0K 0 netmap 0 0K 0 mwldev 0 0K 0 MVS driver 0 0K 0 CAM ccb queue 0 0K 0 mrsasbuf 0 0K 0 mpt_user 0 0K 0 mps_user 0 0K 0 accf 0 0K 0 pts 0 0K 0 iov 0 0K 16800 ioctlops 0 0K 114 eventfd 0 0K 0 Witness 0 0K 0 stack 0 0K 0 MPSSAS 0 0K 0 mps 0 0K 0 mpr_user 0 0K 0 MPRSAS 0 0K 0 mpr 0 0K 0 mfibuf 0 0K 0 sbuf 0 0K 288 md_sectors 0 0K 0 firmware 0 0K 0 compressor 0 0K 0 md_disk 0 0K 0 SWAP 0 0K 0 malodev 0 0K 0 LED 0 0K 0 sysctltmp 0 0K 782 sysctl 0 0K 3 ekcd 0 0K 0 dumper 0 0K 0 sendfile 0 0K 0 rctl 0 0K 0 ix_sriov 0 0K 0 aacraidcam 0 0K 0 aacraid_buf 0 0K 0 ix 0 0K 0 ipsbuf 0 0K 0 cache 0 0K 0 iirbuf 0 0K 0 prison_racct 0 0K 0 Fail Points 0 0K 0 sigio 0 0K 1 filedesc_to_leader 0 0K 0 pwd 0 0K 0 tty console 0 0K 0 aaccam 0 0K 0 aacbuf 0 0K 0 zstd 0 0K 0 XZ_DEC 0 0K 0 nvlist 0 0K 0 SCSI ENC 0 0K 0 SCSI sa 0 0K 0 scsi_pass 0 0K 0 isofs_node 0 0K 0 isofs_mount 0 0K 0 tr_raid5_data 0 0K 0 tr_raid1e_data 0 0K 0 tr_raid1_data 0 0K 0 tr_raid0_data 0 0K 0 tr_concat_data 0 0K 0 md_sii_data 0 0K 0 md_promise_data 0 0K 0 md_nvidia_data 0 0K 0 md_jmicron_data 0 0K 0 md_intel_data 0 0K 0 md_ddf_data 0 0K 0 raid_data 0 0K 72 geom_flashmap 0 0K 0 tmpfs dir 0 0K 0 tmpfs name 0 0K 0 tmpfs mount 0 0K 0 NFS FHA 0 0K 0 newnfsmnt 0 0K 0 newnfsclient_req 0 0K 0 NFSCL layrecall 0 0K 0 NFSCL session 0 0K 0 NFSCL sockreq 0 0K 0 NFSCL devinfo 0 0K 0 NFSCL flayout 0 0K 0 NFSCL layout 0 0K 0 NFSD rollback 0 0K 0 NFSCL diroff 0 0K 0 NEWdirectio 0 0K 0 NEWNFSnode 0 0K 0 NFSCL lck 0 0K 0 db> show uma Zone Size Used Free Requests Sleeps Bucket Total Mem XFree mbuf_jumbo_page 4096 8320 987 951449 0 254 38121472 0 malloc-384 384 35523 47 40910 0 30 13658880 0 malloc-1024 1024 4174 2 4179 0 16 4276224 0 malloc-8192 8192 328 3 1099 0 1 2711552 0 pbuf 2624 0 973 0 0 2 2553152 0 mbuf 256 8949 621 1575729 0 254 2449920 0 BUF TRIE 144 177 13291 723 0 62 1939392 0 RADIX NODE 144 11722 175 123118 0 62 1713168 0 UMA Slabs 0 112 12406 11 12406 0 126 1390704 0 mbuf_cluster 2048 525 1 525 0 254 1077248 0 malloc-1024 1024 133 791 3829 0 16 946176 0 malloc-1024 1024 228 644 3361 0 16 892928 0 malloc-384 384 1725 35 53799 0 30 675840 0 FFS inode 1160 516 23 3248 0 8 625240 0 malloc-8192 8192 70 4 3623 0 1 606208 0 malloc-384 384 663 667 31256 0 30 510720 0 malloc-32768 32768 11 4 2731 0 1 491520 0 malloc-512 512 264 664 3321 0 30 475136 0 malloc-384 384 1159 61 79283 0 30 468480 0 malloc-16384 16384 24 0 25 0 1 393216 0 lkpimm 160 1 2324 1 0 62 372000 0 lkpicurr 160 2 2323 2 0 62 372000 0 VM OBJECT 264 1302 63 53823 0 30 360360 0 THREAD 1808 158 18 3741 0 8 318208 0 malloc-512 512 31 545 3524 0 30 294912 0 VNODE 448 554 76 3288 0 30 282240 0 malloc-384 384 624 66 3457 0 30 264960 0 malloc-65536 65536 4 0 4 0 1 262144 0 malloc-2048 2048 125 3 142 0 8 262144 0 malloc-16384 16384 11 3 247 0 1 229376 0 DEVCTL 1024 8 212 153 0 0 225280 0 malloc-1024 1024 184 32 6779 0 16 221184 0 256 Bucket 2048 92 14 14600 0 8 217088 0 malloc-8192 8192 23 2 37 0 1 204800 0 malloc-8192 8192 18 7 2936 0 1 204800 0 malloc-65536 65536 3 0 3 0 1 196608 0 MAP ENTRY 96 1709 265 193462 0 126 189504 0 UMA Zones 768 242 2 242 0 16 187392 0 malloc-4096 4096 7 38 3088 0 2 184320 0 socket 944 54 134 6972 0 254 177472 0 malloc-384 384 368 62 14196 0 30 165120 0 vmem btag 56 2610 114 2610 0 254 152544 0 ertt_txseginfo 40 1 3534 15481 0 254 141400 0 FFS2 dinode 256 516 24 3248 0 62 138240 0 S VFS Cache 104 1026 300 4231 0 126 137904 0 mbuf_packet 256 25 500 3990 0 254 134400 0 malloc-65536 65536 2 0 2 0 1 131072 0 malloc-65536 65536 0 2 176 0 1 131072 0 VMSPACE 2544 41 10 3600 0 4 129744 0 tcp_inpcb 488 19 229 2370 0 254 121024 0 sctp_ep 1280 16 77 2931 0 254 119040 0 malloc-384 384 275 35 5037 0 30 119040 0 malloc-512 512 196 36 6541 0 30 118784 0 ksiginfo 112 66 978 784 0 126 116928 0 malloc-512 512 196 20 2925 0 30 110592 0 PROC 1336 67 14 3620 0 8 108216 0 malloc-1024 1024 94 6 99 0 16 102400 0 tcp_bbr_map 128 0 775 2757 0 126 99200 0 malloc-32768 32768 1 2 121 0 1 98304 0 malloc-16384 16384 2 4 32 0 1 98304 0 sctp_asoc 2288 13 27 919 0 254 91520 0 filedesc0 1072 68 16 3621 0 8 90048 0 UMA Kegs 384 227 6 227 0 30 89472 0 clpbuf 2624 0 32 20 0 16 83968 0 tcpcb 1080 12 65 2370 0 254 83160 0 g_bio 408 0 180 5973 0 30 73440 0 malloc-512 512 118 18 3079 0 30 69632 0 128 Bucket 1024 33 34 414 0 16 68608 0 malloc-65536 65536 0 1 1 0 1 65536 0 malloc-32768 32768 1 1 16 0 1 65536 0 malloc-16384 16384 4 0 4 0 1 65536 0 malloc-16384 16384 2 2 6 0 1 65536 0 malloc-2048 2048 15 17 2318 0 8 65536 0 malloc-1024 1024 34 30 4053 0 16 65536 0 malloc-2048 2048 10 20 518 0 8 61440 0 malloc-384 384 102 48 6003 0 30 57600 0 malloc-8192 8192 2 5 2902 0 1 57344 0 32 Bucket 256 133 77 12495 0 62 53760 0 malloc-4096 4096 7 6 31 0 2 53248 0 malloc-16384 16384 3 0 3 0 1 49152 0 malloc-16384 16384 3 0 3 0 1 49152 0 malloc-4096 4096 12 0 42 0 2 49152 0 malloc-2048 2048 2 18 153 0 8 40960 0 malloc-2048 2048 16 4 24 0 8 40960 0 malloc-512 512 23 57 1440 0 30 40960 0 pcpu-8 8 4701 419 4885 0 254 40960 0 16 Bucket 144 141 139 2091 0 62 40320 0 DIRHASH 1024 34 2 34 0 16 36864 0 NAMEI 1024 0 36 25984 0 16 36864 0 Files 80 245 205 16959 0 126 36000 0 malloc-32768 32768 1 0 1 0 1 32768 0 malloc-32768 32768 1 0 1 0 1 32768 0 malloc-32768 32768 1 0 1 0 1 32768 0 malloc-8192 8192 3 1 4 0 1 32768 0 malloc-512 512 52 12 236 0 30 32768 0 malloc-512 512 34 30 214 0 30 32768 0 pcpu-64 64 487 25 487 0 254 32768 0 64 Bucket 512 46 18 4509 0 30 32768 0 sctp_raddr 736 13 31 919 0 254 32384 0 tcp_rack_pcb 832 7 29 1147 0 16 29952 0 malloc-4096 4096 6 1 8 0 2 28672 0 TURNSTILE 136 177 33 177 0 62 28560 0 KNOTE 160 28 147 101023 0 62 28000 0 pipe 744 24 11 377 0 16 26040 0 malloc-8192 8192 1 2 45 0 1 24576 0 malloc-4096 4096 1 5 10 0 2 24576 0 PWD 32 21 735 2811 0 254 24192 0 ttyinq 160 135 15 300 0 62 24000 0 8 Bucket 80 43 257 1300 0 126 24000 0 ttyoutq 256 72 18 160 0 62 23040 0 tcp_bbr_pcb 832 0 27 803 0 16 22464 0 malloc-4096 4096 3 2 64 0 2 20480 0 2 Bucket 32 92 538 2157 0 254 20160 0 SLEEPQUEUE 88 177 47 177 0 126 19712 0 Mountpoints 2752 2 5 2 0 4 19264 0 malloc-16384 16384 1 0 1 0 1 16384 0 malloc-8192 8192 0 2 13 0 1 16384 0 malloc-4096 4096 2 2 8 0 2 16384 0 tcp_rack_map 112 14 130 2104 0 126 16128 0 vtnet_tx_hdr 24 0 668 475589 0 254 16032 0 malloc-2048 2048 3 3 165 0 8 12288 0 malloc-2048 2048 2 4 6 0 8 12288 0 ertt 72 12 156 2370 0 126 12096 0 ripcb 488 5 19 158 0 254 11712 0 udplite_inpcb 488 0 24 26 0 254 11712 0 udp_inpcb 488 6 18 199 0 254 11712 0 kenv 258 15 30 1038 0 30 11610 0 routing nhops 256 27 18 34 0 62 11520 0 unpcb 256 20 25 1253 0 254 11520 0 malloc-2048 2048 4 0 4 0 8 8192 0 malloc-1024 1024 1 7 9 0 16 8192 0 rtentry 176 30 16 34 0 62 8096 0 PGRP 88 34 58 48 0 126 8096 0 rl_entry 40 47 155 47 0 254 8080 0 sctp_stream_msg_out 112 0 72 18 0 254 8064 0 sctp_laddr 48 8 160 627 0 254 8064 0 udpcb 32 6 246 225 0 254 8064 0 AIOP 32 4 248 4 0 254 8064 0 4 Bucket 48 7 161 99 0 254 8064 0 sctp_chunk 152 16 36 17 0 254 7904 0 tcp_log 416 0 18 48 0 254 7488 0 FPU_save_area 832 1 8 1 0 16 7488 0 cpuset 104 7 55 7 0 126 6448 0 epoch_record pcpu 256 4 12 4 0 62 4096 0 malloc-4096 4096 1 0 1 0 2 4096 0 pcpu-16 16 7 249 7 0 254 4096 0 hostcache 64 1 62 1 0 254 4032 0 syncache 168 0 24 4 0 254 4032 0 AIO 208 0 19 3 0 62 3952 0 UMA Slabs 1 176 9 13 9 0 62 3872 0 AIOCB 552 0 7 6 0 16 3864 0 AIOLIO 272 0 14 2 0 30 3808 0 mqnode 416 3 6 3 0 30 3744 0 KMAP ENTRY 96 12 27 12 0 0 3744 0 vmem 1856 1 1 1 0 8 3712 0 SMR CPU 32 3 60 3 0 254 2016 0 SMR SHARED 24 3 60 3 0 254 1512 0 FFS1 dinode 128 0 0 0 0 126 0 0 da_ccb 544 0 0 0 0 16 0 0 ada_ccb 272 0 0 0 0 30 0 0 swblk 136 0 0 0 0 62 0 0 swpctrie 144 0 0 0 0 62 0 0 sctp_asconf_ack 48 0 0 0 0 254 0 0 sctp_asconf 40 0 0 0 0 254 0 0 sctp_readq 152 0 0 0 0 254 0 0 cdg_qdiffsample 16 0 0 0 0 254 0 0 pf state scrubs 40 0 0 0 0 254 0 0 pf frag entries 40 0 0 0 0 254 0 0 pf frags 248 0 0 0 0 62 0 0 pf table entries 160 0 0 0 0 62 0 0 pf table entry counters 64 0 0 0 0 254 0 0 pf source nodes 136 0 0 0 0 254 0 0 pf state keys 88 0 0 0 0 126 0 0 pf states 304 0 0 0 0 254 0 0 pf tags 104 0 0 0 0 126 0 0 pf mtags 48 0 0 0 0 254 0 0 tcp_log_node 120 0 0 0 0 126 0 0 tcp_log_bucket 176 0 0 0 0 62 0 0 tcpreass 48 0 0 0 0 254 0 0 tfo_ccache_entries 80 0 0 0 0 126 0 0 tfo 4 0 0 0 0 254 0 0 sackhole 32 0 0 0 0 254 0 0 tcptw 88 0 0 0 0 254 0 0 ipq 56 0 0 0 0 254 0 0 IPsec SA lft_c 16 0 0 0 0 254 0 0 itimer 352 0 0 0 0 30 0 0 mqnotifier 216 0 0 0 0 62 0 0 mvdata 64 0 0 0 0 254 0 0 mqueue 248 0 0 0 0 62 0 0 NCLNODE 584 0 0 0 0 16 0 0 TMPFS node 224 0 0 0 0 62 0 0 LTS VFS Cache 360 0 0 0 0 30 0 0 L VFS Cache 320 0 0 0 0 30 0 0 STS VFS Cache 144 0 0 0 0 62 0 0 cryptop 280 0 0 0 0 30 0 0 linux_dma_object 24 0 0 0 0 254 0 0 linux_dma_pctrie 144 0 0 0 0 62 0 0 IOMMU_MAP_ENTRY 120 0 0 0 0 126 0 0 ktls_session 128 0 0 0 0 126 0 0 mbuf_jumbo_16k 16384 0 0 0 0 254 0 0 mbuf_jumbo_9k 9216 0 0 0 0 254 0 0 audit_record 1280 0 0 0 0 8 0 0 domainset 40 0 0 0 0 254 0 0 MAC labels 40 0 0 0 0 254 0 0 vnpbuf 2624 0 0 0 0 64 0 0 mdpbuf 2624 0 0 0 0 3 0 0 nfspbuf 2624 0 0 0 0 16 0 0 swwbuf 2624 0 0 0 0 8 0 0 swrbuf 2624 0 0 0 0 16 0 0 umtx_shm 88 0 0 0 0 126 0 0 umtx pi 96 0 0 0 0 126 0 0 rangeset pctrie nodes 144 0 0 0 0 62 0 0 malloc-65536 65536 0 0 0 0 1 0 0 malloc-65536 65536 0 0 0 0 1 0 0 malloc-65536 65536 0 0 0 0 1 0 0 malloc-32768 32768 0 0 0 0 1 0 0 malloc-32768 32768 0 0 0 0 1 0 0 malloc-1024 1024 0 0 0 0 16 0 0 malloc-256 256 0 0 0 0 62 0 0 malloc-256 256 0 0 0 0 62 0 0 malloc-256 256 0 0 0 0 62 0 0 malloc-256 256 0 0 0 0 62 0 0 malloc-256 256 0 0 0 0 62 0 0 malloc-256 256 0 0 0 0 62 0 0 malloc-256 256 0 0 0 0 62 0 0 malloc-256 256 0 0 0 0 62 0 0 malloc-128 128 0 0 0 0 126 0 0 malloc-128 128 0 0 0 0 126 0 0 malloc-128 128 0 0 0 0 126 0 0 malloc-128 128 0 0 0 0 126 0 0 malloc-128 128 0 0 0 0 126 0 0 malloc-128 128 0 0 0 0 126 0 0 malloc-128 128 0 0 0 0 126 0 0 malloc-128 128 0 0 0 0 126 0 0 malloc-64 64 0 0 0 0 254 0 0 malloc-64 64 0 0 0 0 254 0 0 malloc-64 64 0 0 0 0 254 0 0 malloc-64 64 0 0 0 0 254 0 0 malloc-64 64 0 0 0 0 254 0 0 malloc-64 64 0 0 0 0 254 0 0 malloc-64 64 0 0 0 0 254 0 0 malloc-64 64 0 0 0 0 254 0 0 malloc-32 32 0 0 0 0 254 0 0 malloc-32 32 0 0 0 0 254 0 0 malloc-32 32 0 0 0 0 254 0 0 malloc-32 32 0 0 0 0 254 0 0 malloc-32 32 0 0 0 0 254 0 0 malloc-32 32 0 0 0 0 254 0 0 malloc-32 32 0 0 0 0 254 0 0 malloc-32 32 0 0 0 0 254 0 0 malloc-16 16 0 0 0 0 254 0 0 malloc-16 16 0 0 0 0 254 0 0 malloc-16 16 0 0 0 0 254 0 0 malloc-16 16 0 0 0 0 254 0 0 malloc-16 16 0 0 0 0 254 0 0 malloc-16 16 0 0 0 0 254 0 0 malloc-16 16 0 0 0 0 254 0 0 malloc-16 16 0 0 0 0 254 0 0 pcpu-32 32 0 0 0 0 254 0 0 pcpu-4 4 0 0 0 0 254 0 0 fakepg 104 0 0 0 0 126 0 0 UMA Hash 256 0 0 0 0 62 0 0