panic: mtx_lock() of spin mutex (null) @ /syzkaller/managers/i386/kernel/sys/netinet/tcp_output.c:352 cpuid = 1 time = 1610785270 KDB: stack backtrace: db_trace_self_wrapper() at db_trace_self_wrapper+0x47/frame 0xfffffe0051f55650 vpanic() at vpanic+0x1c7/frame 0xfffffe0051f556b0 panic() at panic+0x43/frame 0xfffffe0051f55710 __mtx_lock_flags() at __mtx_lock_flags+0x202/frame 0xfffffe0051f55770 tcp_output() at tcp_output+0x716/frame 0xfffffe0051f55940 tcp_usr_connect() at tcp_usr_connect+0x258/frame 0xfffffe0051f559b0 soconnectat() at soconnectat+0x183/frame 0xfffffe0051f55a10 kern_connectat() at kern_connectat+0x1e5/frame 0xfffffe0051f55a70 sys_connect() at sys_connect+0xd9/frame 0xfffffe0051f55ab0 ia32_syscall() at ia32_syscall+0x237/frame 0xfffffe0051f55bf0 int0x80_syscall_common() at int0x80_syscall_common+0x9c/frame 0xfbf99f78 KDB: enter: panic [ thread pid 2113 tid 102127 ] Stopped at kdb_enter+0x67: movq $0,0x1459b4e(%rip) db> db> set $lines = 0 db> set $maxwidth = 0 db> show registers cs 0x20 ds 0x3b es 0x3b fs 0x13 gs 0x1b ss 0 rax 0x12 rcx 0xfffffe0089600000 rdx 0x3ffff rbx 0 rsp 0xfffffe0051f55630 rbp 0xfffffe0051f55650 rsi 0x40001 rdi 0xffffffff81111976 vprintf+0x176 r8 0 r9 0xffffffff r10 0x88f8032d r11 0xeb7f61df r12 0xffffffff82067650 ddb_dbbe r13 0 r14 0xffffffff819b1ff1 r15 0xffffffff819b1ff1 rip 0xffffffff81105c37 kdb_enter+0x67 rflags 0x200086 kernload+0x86 kdb_enter+0x67: movq $0,0x1459b4e(%rip) db> show proc Process 2113 (syz-executor.0) at 0xfffff800162d1a50: state: NORMAL uid: 0 gids: 0, 0, 5 parent: pid 777 at 0xfffff800162d1528 ABI: FreeBSD ELF32 flag: 0x10000080 flag2: 0 arguments: /root/syz-executor.0 reaper: 0xfffff80004486528 reapsubtree: 1 sigparent: 20 vmspace: 0xfffffe0088a439f0 (map 0xfffffe0088a439f0) (map.pmap 0xfffffe0088a43ab0) (pmap 0xfffffe0088a43b10) threads: 5 101971 Run CPU 0 syz-executor.0 102116 S sbwait 0xfffff8001692b1c4 syz-executor.0 102121 S uwait 0xfffff80016b4ae00 syz-executor.0 102125 S sbwait 0xfffff8001692b574 syz-executor.0 102127 Run CPU 1 syz-executor.0 db> ps pid ppid pgrp uid state wmesg wchan cmd 2113 777 777 0 R (threaded) syz-executor.0 101971 Run CPU 0 syz-executor.0 102116 S sbwait 0xfffff8001692b1c4 syz-executor.0 102121 S uwait 0xfffff80016b4ae00 syz-executor.0 102125 S sbwait 0xfffff8001692b574 syz-executor.0 102127 Run CPU 1 syz-executor.0 1935 1927 1935 0 Ss select 0xfffff80016a62140 dhclient 1930 1 1930 0 Ss select 0xfffff80016a2f9c0 dhclient 1927 1920 436 65 S select 0xfffff800169db9c0 dhclient 1920 436 436 0 S wait 0xfffff80016c24000 sh 1911 1 1911 65 Ss select 0xfffff80004dace40 dhclient 1316 1 1316 0 Ss select 0xfffff80016912b40 dhclient 1313 1 1313 0 Ss select 0xfffff8001686b140 dhclient 1294 1 1294 65 Ss select 0xfffff8001686b040 dhclient 810 1 810 0 Ss select 0xfffff8001627f1c0 dhclient 808 774 808 0 Ss piperd 0xfffff80016b338b8 syz-executor.2 806 1 806 0 Ss select 0xfffff8001627f5c0 dhclient 797 774 797 0 Ss piperd 0xfffff800163c92e8 syz-executor.3 778 774 778 0 Ss piperd 0xfffff80004e6d5d0 syz-executor.1 777 774 777 0 Ss nanslp 0xffffffff82532f21 syz-executor.0 774 772 772 0 S (threaded) syz-fuzzer 100113 S uwait 0xfffff800163f8c00 syz-fuzzer 100114 S uwait 0xfffff80004eada80 syz-fuzzer 100115 S kqread 0xfffff800162d0b00 syz-fuzzer 100116 S uwait 0xfffff80004eadc80 syz-fuzzer 100117 S uwait 0xfffff80004eadd80 syz-fuzzer 100118 S uwait 0xfffff800163f8400 syz-fuzzer 100119 S uwait 0xfffff800163f8300 syz-fuzzer 100120 S uwait 0xfffff800163f8500 syz-fuzzer 102007 S uwait 0xfffff80016b94900 syz-fuzzer 772 770 772 0 Ss pause 0xfffff800163d20b0 csh 770 694 770 0 Ss select 0xfffff8001627f340 sshd 754 1 754 0 Ss+ ttyin 0xfffff800048d74b0 getty 753 1 753 0 Ss+ ttyin 0xfffff80004c0c0b0 getty 752 1 752 0 Ss+ ttyin 0xfffff80004c0c4b0 getty 751 1 751 0 Ss+ ttyin 0xfffff80004c0c8b0 getty 750 1 750 0 Ss+ ttyin 0xfffff80004c0ccb0 getty 749 1 749 0 Ss+ ttyin 0xfffff80004c160b0 getty 748 1 748 0 Ss+ ttyin 0xfffff80004c164b0 getty 747 1 747 0 Ss+ ttyin 0xfffff80004c168b0 getty 746 1 746 0 Ss+ ttyin 0xfffff80004c16cb0 getty 698 1 698 0 Ss nanslp 0xffffffff82532f20 cron 694 1 694 0 Ss select 0xfffff8001627f940 sshd 507 1 507 0 Ss select 0xfffff800048e31c0 syslogd 436 1 436 0 Ss wait 0xfffff80004f4d000 devd 435 1 435 65 Ss select 0xfffff8001627f9c0 dhclient 350 1 350 0 Ss select 0xfffff800048e3040 dhclient 347 1 347 0 Ss select 0xfffff800048e30c0 dhclient 23 0 0 0 DL vlruwt 0xfffff80004bba000 [vnlru] 22 0 0 0 DL syncer 0xffffffff826223b8 [syncer] 21 0 0 0 DL (threaded) [bufdaemon] 100070 D qsleep 0xffffffff826214a0 [bufdaemon] 100073 D - 0xffffffff8200ac80 [bufspacedaemon-0] 100087 D sdflush 0xfffff80004c064e8 [/ worker] 20 0 0 0 DL psleep 0xffffffff82648988 [vmdaemon] 19 0 0 0 DL (threaded) [pagedaemon] 100068 D psleep 0xffffffff8263cdf8 [dom0] 100071 D launds 0xffffffff8263ce04 [laundry: dom0] 100072 D umarcl 0xffffffff8151ee10 [uma] 18 0 0 0 DL - 0xffffffff823672f8 [rand_harvestq] 17 0 0 0 DL waiting 0xffffffff82e36818 [sctp_iterator] 16 0 0 0 DL pftm 0xffffffff82b5d430 [pf purge] 15 0 0 0 DL - 0xffffffff8261ea5c [soaiod4] 9 0 0 0 DL - 0xffffffff8261ea5c [soaiod3] 8 0 0 0 DL - 0xffffffff8261ea5c [soaiod2] 7 0 0 0 DL - 0xffffffff8261ea5c [soaiod1] 6 0 0 0 DL (threaded) [cam] 100034 D - 0xffffffff8223f040 [doneq0] 100067 D - 0xffffffff8223ef10 [scanner] 14 0 0 0 DL seqstat 0xfffff80004667c88 [sequencer 00] 5 0 0 0 DL crypto_ 0xfffff80004532d90 [crypto returns 1] 4 0 0 0 DL crypto_ 0xfffff80004532d30 [crypto returns 0] 3 0 0 0 DL crypto_ 0xffffffff8263a310 [crypto] 13 0 0 0 DL (threaded) [geom] 100025 D - 0xffffffff82512760 [g_event] 100026 D - 0xffffffff82512768 [g_up] 100027 D - 0xffffffff82512770 [g_down] 2 0 0 0 DL (threaded) [KTLS] 100018 D - 0xfffff80004470d00 [thr_0] 100019 D - 0xfffff80004470d80 [thr_1] 12 0 0 0 WL (threaded) [intr] 100010 I [swi6: Giant taskq] 100013 I [swi5: fast taskq] 100016 I [swi6: task queue] 100020 I [swi4: clock (0)] 100021 I [swi4: clock (1)] 100022 I [swi1: netisr 0] 100023 I [swi3: vm] 100035 I [irq24: virtio_pci0] 100036 I [irq25: virtio_pci0] 100037 I [irq26: virtio_pci0] 100038 I [irq27: virtio_pci0] 100039 I [irq28: virtio_pci1] 100040 I [irq29: virtio_pci1] 100041 I [irq30: virtio_pci1] 100042 I [irq31: virtio_pci1] 100043 I [irq32: virtio_pci1] 100048 I [irq10: virtio_pci2] 100050 I [irq1: atkbd0] 100051 I [irq12: psm0] 100052 I [swi0: uart uart++] 100060 I [swi1: pf send] 100076 I [swi1: hpts] 100077 I [swi1: hpts] 11 0 0 0 RL (threaded) [idle] 100003 CanRun [idle: cpu0] 100004 CanRun [idle: cpu1] 1 0 1 0 SLs wait 0xfffff80004486528 [init] 10 0 0 0 DL audit_w 0xffffffff8263a830 [audit] 0 0 0 0 DLs (threaded) [kernel] 100000 D swapin 0xffffffff82512cf0 [swapper] 100005 D - 0xfffff80004088400 [if_config_tqg_0] 100006 D - 0xfffff80004088200 [softirq_0] 100007 D - 0xfffff80004088000 [softirq_1] 100008 D - 0xfffff80004468d00 [if_io_tqg_0] 100009 D - 0xfffff80004468b00 [if_io_tqg_1] 100011 D - 0xfffff80004468700 [thread taskq] 100012 D - 0xfffff80004468500 [aiod_kick taskq] 100014 D - 0xfffff80004468100 [kqueue_ctx taskq] 100015 D - 0xfffff80004463e00 [in6m_free taskq] 100017 D - 0xfffff80004463a00 [inm_free taskq] 100024 D - 0xfffff80004463200 [firmware taskq] 100028 D - 0xfffff800044a4c00 [crypto_0] 100029 D - 0xfffff800044a4c00 [crypto_1] 100044 D - 0xfffff80004877e00 [vtnet0 rxq 0] 100045 D - 0xfffff80004877d00 [vtnet0 txq 0] 100046 D - 0xfffff80004877c00 [vtnet0 rxq 1] 100047 D - 0xfffff80004877b00 [vtnet0 txq 1] 100049 D vtbslp 0xfffff80004660100 [virtio_balloon] 100053 D - 0xfffff800048ddb00 [mca taskq] 100058 D - 0xffffffff81d598b0 [deadlkres] 100063 D - 0xfffff80004ae3e00 [acpi_task_0] 100064 D - 0xfffff80004ae3e00 [acpi_task_1] 100065 D - 0xfffff80004ae3e00 [acpi_task_2] 100066 D - 0xfffff800044a4500 [CAM taskq] db> show all locks Process 2113 (syz-executor.0) thread 0xfffffe00889abe00 (102116) exclusive sx so_rcv_sx (so_rcv_sx) r = 0 (0xfffff8001692b168) locked @ /syzkaller/managers/i386/kernel/sys/kern/uipc_sockbuf.c:467 Process 2113 (syz-executor.0) thread 0xfffffe00889dda00 (102125) exclusive sx so_rcv_sx (so_rcv_sx) r = 0 (0xfffff8001692b518) locked @ /syzkaller/managers/i386/kernel/sys/kern/uipc_sockbuf.c:467 Process 2113 (syz-executor.0) thread 0xfffffe00889dbe00 (102127) exclusive rw tcpinp (tcpinp) r = 0 (0xfffff800163753f0) locked @ /syzkaller/managers/i386/kernel/sys/netinet/tcp_usrreq.c:564 db> show malloc Type InUse MemUse Requests pf_hash 5 11524K 5 devbuf 4216 4339K 4244 tcp_hpts 5 3201K 5 vtbuf 24 1968K 46 sysctloid 30650 1798K 30716 kobj 328 1312K 486 newblk 7 1026K 2891 vfscache 3 1025K 3 pcb 29 539K 972 inodedep 10 516K 1867 ufs_quota 1 512K 1 vfs_hash 1 512K 1 callout 2 512K 2 intr 4 472K 4 subproc 134 261K 2195 acpica 1674 184K 58593 vnet_data 1 168K 1 tidhash 3 141K 3 pagedep 10 131K 2764 tfo_ccache 1 128K 1 sem 4 106K 4 DEVFS1 105 105K 122 linker 278 100K 311 filedesc 12 89K 2537 bus 982 80K 3359 mtx_pool 2 72K 2 BPF 38 71K 38 syncache 1 68K 1 acpitask 1 64K 1 ddb_capture 1 64K 1 module 502 63K 502 umtx 324 41K 324 kdtrace 201 40K 4247 temp 35 33K 2097 hostcache 1 32K 1 shm 1 32K 2 DEVFS3 124 31K 134 msg 4 30K 4 vmem 3 26K 6 gtaskqueue 18 26K 18 kbdmux 6 22K 6 ifaddr 71 21K 71 DEVFS_RULE 56 20K 56 cred 71 18K 828 ufs_mount 5 17K 6 proc 3 17K 3 tty 16 16K 16 ithread 99 16K 99 lltable 45 14K 48 bus-sc 32 14K 1594 KTRACE 100 13K 100 ifnet 7 13K 7 ether_multi 152 13K 162 kenv 92 12K 92 eventhandler 129 11K 129 GEOM 60 10K 486 rman 82 10K 423 in6_multi 65 9K 65 bmsafemap 2 9K 1462 UART 12 9K 12 devstat 4 9K 4 ksem 1 8K 1 rpc 2 8K 2 shmfd 1 8K 5 pfs_vncache 1 8K 1 pfs_nodes 20 8K 20 audit_evclass 236 8K 294 sctp_timw 27 7K 27 sglist 5 7K 5 CAM DEV 3 6K 510 kqueue 60 6K 2125 plimit 23 6K 394 CAM queue 5 6K 1528 taskqueue 48 6K 48 DEVFSP 76 5K 80 ufs_dirhash 24 5K 24 pf_ifnet 10 5K 19 UMA 261 5K 261 vt 11 5K 11 memdesc 1 4K 1 MCA 32 4K 32 kcovinfo 64 4K 68 evdev 4 4K 4 session 31 4K 42 routetbl 24 4K 24 pwddesc 58 4K 2119 hhook 13 4K 13 lockf 28 3K 47 selfd 45 3K 41129 terminal 11 3K 11 acpisem 22 3K 22 proc-args 50 3K 608 uidinfo 4 3K 9 local_apic 1 2K 1 io_apic 1 2K 1 fpukern_ctx 2 2K 2 ipsec-saq 2 2K 2 select 16 2K 43 ip6ndp 12 2K 13 sctp_ifa 13 2K 15 Unitno 28 2K 46 CAM XPT 22 2K 543 in_multi 6 2K 8 dirrem 5 2K 1359 ipsecpolicy 2 2K 2 acpidev 20 2K 20 msi 9 2K 9 clone 9 2K 9 tun 7 2K 7 sctp_stro 1 1K 63 softdep 1 1K 1 sahead 1 1K 1 secasvar 1 1K 1 nhops 6 1K 8 vnodemarker 2 1K 24 NFSD session 1 1K 1 CAM periph 4 1K 271 sctp_ifn 6 1K 15 sctp_atcl 2 1K 210 ipsec 3 1K 3 mld 6 1K 6 igmp 6 1K 6 toponodes 6 1K 6 isadev 6 1K 6 mount 16 1K 89 pci_link 10 1K 10 mkdir 5 1K 2514 diradd 5 1K 1396 freefile 5 1K 1357 crypto 4 1K 4 encap_export_host 12 1K 12 newdirblk 4 1K 1257 pfil 4 1K 4 procdesc 4 1K 12 CAM SIM 2 1K 2 cdev 2 1K 2 inpcbpolicy 13 1K 928 chacha20random 1 1K 1 osd 3 1K 10 vnodes 1 1K 1 ktls 1 1K 1 NFSD lckfile 1 1K 1 NFSD V4client 1 1K 1 DEVFS 9 1K 10 feeder 7 1K 7 tcpfunc 3 1K 3 loginclass 3 1K 6 prison 6 1K 6 aesni_data 2 1K 2 sctp_atky 3 1K 273 CAM dev queue 2 1K 2 CAM I/O Scheduler 1 1K 1 apmdev 1 1K 1 atkbddev 2 1K 2 CAM path 4 1K 1034 freework 1 1K 1699 soname 5 1K 4321 pmchooks 1 1K 1 nexusdev 6 1K 6 sctp_vrf 1 1K 1 entropy 2 1K 40 vnet 1 1K 1 pmc 1 1K 1 filecaps 4 1K 83 acpiintr 1 1K 1 sctp_athm 2 1K 210 sctp_map 2 1K 126 cpus 2 1K 2 vnet_data_free 1 1K 1 Per-cpu 1 1K 1 p1003.1b 1 1K 1 sctp_mcore 0 0K 0 sctp_socko 0 0K 18 sctp_iter 0 0K 10 sctp_mvrf 0 0K 0 sctp_cpal 0 0K 0 sctp_cmsg 0 0K 0 sctp_stre 0 0K 0 sctp_athi 0 0K 0 sctp_a_it 0 0K 10 sctp_aadr 0 0K 0 sctp_stri 0 0K 0 mqdata 0 0K 0 pf_table 0 0K 0 pf_rule 0 0K 0 pf_altq 0 0K 0 pf_osfp 0 0K 0 pf_temp 0 0K 0 nvlist 0 0K 0 SCSI ENC 0 0K 0 SCSI sa 0 0K 0 madt_table 0 0K 2 smartpqi 0 0K 0 scsi_pass 0 0K 0 ciss_data 0 0K 0 pvscsi 0 0K 0 ath_hal 0 0K 0 athdev 0 0K 0 ata_pci 0 0K 0 ata_dma 0 0K 0 ata_generic 0 0K 0 iavf 0 0K 0 ixl 0 0K 0 amr 0 0K 0 scsi_da 0 0K 69 ata_da 0 0K 0 scsi_ch 0 0K 0 scsi_cd 0 0K 0 USBdev 0 0K 0 ice-resmgr 0 0K 0 ice-osdep 0 0K 0 ice 0 0K 0 axgbe 0 0K 0 USB 0 0K 0 xen_intr 0 0K 0 AHCI driver 0 0K 0 agp 0 0K 0 xen_hvm 0 0K 0 legacydrv 0 0K 0 qpidrv 0 0K 0 dmar_idpgtbl 0 0K 0 dmar_dom 0 0K 0 dmar_ctx 0 0K 0 nvme_da 0 0K 0 acpipwr 0 0K 0 isci 0 0K 0 iommu_dmamap 0 0K 0 twsbuf 0 0K 0 hyperv_socket 0 0K 0 bxe_ilt 0 0K 0 xenbus 0 0K 0 twe_commands 0 0K 0 twa_commands 0 0K 0 tcp_log_dev 0 0K 0 vm_fictitious 0 0K 0 midi buffers 0 0K 0 mixer 0 0K 0 ac97 0 0K 0 hdacc 0 0K 0 hdac 0 0K 0 hdaa 0 0K 0 acpi_perf 0 0K 0 acpicmbat 0 0K 0 SIIS driver 0 0K 0 UMAHash 0 0K 0 CAM CCB 0 0K 8070 vm_pgdata 0 0K 0 jblocks 0 0K 0 savedino 0 0K 4085 sentinel 0 0K 0 jfsync 0 0K 0 jtrunc 0 0K 0 sbdep 0 0K 9 jsegdep 0 0K 0 jseg 0 0K 0 jfreefrag 0 0K 0 jfreeblk 0 0K 0 jnewblk 0 0K 0 jmvref 0 0K 0 jremref 0 0K 0 jaddref 0 0K 0 freedep 0 0K 0 freeblks 0 0K 1326 freefrag 0 0K 6 allocindir 0 0K 0 indirdep 0 0K 661 allocdirect 0 0K 0 ufs_trim 0 0K 0 mactemp 0 0K 0 audit_trigger 0 0K 0 audit_pipe_presel 0 0K 0 audit_pipeent 0 0K 0 audit_pipe 0 0K 0 audit_evname 0 0K 0 audit_bsm 0 0K 0 audit_gidset 0 0K 0 audit_text 0 0K 0 audit_path 0 0K 0 audit_data 0 0K 0 audit_cred 0 0K 0 xform 0 0K 0 NLM 0 0K 0 ipsec-spdcache 0 0K 0 ipsec-reg 0 0K 0 ipsec-misc 0 0K 0 ipsecrequest 0 0K 0 ip6opt 0 0K 3 ip6_msource 0 0K 0 ip6_moptions 0 0K 0 in6_mfilter 0 0K 0 frag6 0 0K 0 tcplog 0 0K 0 tcp_hwpace 0 0K 0 LRO 0 0K 0 newreno data 0 0K 0 ip_msource 0 0K 0 ip_moptions 0 0K 0 in_mfilter 0 0K 0 ipid 0 0K 0 80211scan 0 0K 0 80211ratectl 0 0K 0 80211power 0 0K 0 80211nodeie 0 0K 0 80211node 0 0K 0 80211mesh_gt 0 0K 0 80211mesh_rt 0 0K 0 80211perr 0 0K 0 80211prep 0 0K 0 80211preq 0 0K 0 80211dfs 0 0K 0 80211crypto 0 0K 0 80211vap 0 0K 0 iflib 0 0K 0 vlan 0 0K 0 gif 0 0K 0 ifdescr 0 0K 0 zlib 0 0K 0 fadvise 0 0K 0 VN POLL 0 0K 0 statfs 0 0K 1431 namei_tracker 0 0K 0 export_host 0 0K 0 cl_savebuf 0 0K 4 PUC 0 0K 0 ppbusdev 0 0K 0 agtiapi_MemAlloc malloc 0 0K 0 osti_cacheable 0 0K 0 tempbuff 0 0K 0 tempbuff 0 0K 0 ag_tgt_map_t malloc 0 0K 0 ag_slr_map_t malloc 0 0K 0 lDevFlags * malloc 0 0K 0 tiDeviceHandle_t * malloc 0 0K 0 ag_portal_data_t malloc 0 0K 0 ag_device_t malloc 0 0K 0 STLock malloc 0 0K 0 CCB List 0 0K 0 sr_iov 0 0K 0 OCS 0 0K 0 OCS 0 0K 0 nvme 0 0K 0 nvd 0 0K 0 netmap 0 0K 0 mwldev 0 0K 0 biobuf 0 0K 0 aios 0 0K 0 lio 0 0K 0 acl 0 0K 0 MVS driver 0 0K 0 mbuf_tag 0 0K 105 accf 0 0K 0 pts 0 0K 0 iov 0 0K 15199 ioctlops 0 0K 108 eventfd 0 0K 0 Witness 0 0K 0 stack 0 0K 0 CAM ccb queue 0 0K 0 mrsasbuf 0 0K 0 mpt_user 0 0K 0 sbuf 0 0K 288 mps_user 0 0K 0 firmware 0 0K 0 compressor 0 0K 0 MPSSAS 0 0K 0 SWAP 0 0K 0 mps 0 0K 0 mpr_user 0 0K 0 sysctltmp 0 0K 682 sysctl 0 0K 1 MPRSAS 0 0K 0 ekcd 0 0K 0 dumper 0 0K 0 sendfile 0 0K 0 rctl 0 0K 0 mpr 0 0K 0 mfibuf 0 0K 0 md_sectors 0 0K 0 md_disk 0 0K 0 malodev 0 0K 0 LED 0 0K 0 cache 0 0K 0 prison_racct 0 0K 0 Fail Points 0 0K 0 sigio 0 0K 1 filedesc_to_leader 0 0K 0 pwd 0 0K 0 tty console 0 0K 0 ix_sriov 0 0K 0 aacraidcam 0 0K 0 ix 0 0K 0 ipsbuf 0 0K 0 iirbuf 0 0K 0 aacraid_buf 0 0K 0 aaccam 0 0K 0 isofs_node 0 0K 0 isofs_mount 0 0K 0 tr_raid5_data 0 0K 0 tr_raid1e_data 0 0K 0 tr_raid1_data 0 0K 0 tr_raid0_data 0 0K 0 tr_concat_data 0 0K 0 md_sii_data 0 0K 0 md_promise_data 0 0K 0 md_nvidia_data 0 0K 0 md_jmicron_data 0 0K 0 md_intel_data 0 0K 0 md_ddf_data 0 0K 0 raid_data 0 0K 72 geom_flashmap 0 0K 0 tmpfs dir 0 0K 0 tmpfs name 0 0K 0 tmpfs mount 0 0K 0 NFS FHA 0 0K 0 newnfsmnt 0 0K 0 newnfsclient_req 0 0K 0 NFSCL layrecall 0 0K 0 NFSCL session 0 0K 0 NFSCL sockreq 0 0K 0 NFSCL devinfo 0 0K 0 NFSCL flayout 0 0K 0 NFSCL layout 0 0K 0 NFSD rollback 0 0K 0 NFSCL diroffdiroff 0 0K 0 NEWdirectio 0 0K 0 NEWNFSnode 0 0K 0 NFSCL lck 0 0K 0 NFSCL lckown 0 0K 0 NFSCL client 0 0K 0 NFSCL deleg 0 0K 0 NFSCL open 0 0K 0 NFSCL owner 0 0K 0 NFS fh 0 0K 0 NFS req 0 0K 0 NFSD usrgroup 0 0K 0 NFSD string 0 0K 0 NFSD V4lock 0 0K 0 NFSD V4state 0 0K 0 NFSD srvcache 0 0K 0 msdosfs_fat 0 0K 0 msdosfs_mount 0 0K 0 msdosfs_node 0 0K 0 DEVFS4 0 0K 0 DEVFS2 0 0K 0 gntdev 0 0K 0 privcmd_dev 0 0K 0 evtchn_dev 0 0K 0 xenstore 0 0K 0 aacbuf 0 0K 0 xnb 0 0K 0 xbbd 0 0K 0 xbd 0 0K 0 Balloon 0 0K 0 sysmouse 0 0K 0 vtfont 0 0K 0 zstd 0 0K 0 db> show uma Zone Size Used Free Requests Sleeps Bucket Total Mem XFree mbuf_cluster 2048 9698 2 9698 0 254 19865600 0 mbuf_packet 256 8223 1347 1318106 0 254 2449920 0 pbuf 2632 0 794 0 0 2 2089808 0 BUF TRIE 144 183 13285 5435 0 62 1939392 0 RADIX NODE 144 12662 243 92855 0 62 1858320 0 malloc-4096 4096 386 7 2605 0 2 1609728 0 malloc-384 384 4117 13 4431 0 30 1585920 0 malloc-128 128 10217 13 10240 0 126 1309440 0 UMA Slabs 0 112 5723 34 5723 0 126 644784 0 FFS inode 1128 512 27 1870 0 8 607992 0 VM OBJECT 264 1213 92 32709 0 30 344520 0 malloc-65536 65536 4 1 128 0 1 327680 0 VNODE 488 548 84 1908 0 30 308416 0 THREAD 1792 142 20 2127 0 8 290304 0 256 Bucket 2048 116 6 7807 0 8 249856 0 mbuf 256 480 495 663529 0 254 249600 0 malloc-32768 32768 1 6 621 0 1 229376 0 DEVCTL 1024 4 208 129 0 0 217088 0 malloc-16384 16384 6 7 1267 0 1 212992 0 malloc-64 64 3239 37 3255 0 254 209664 0 malloc-16 16 12711 289 12767 0 254 208000 0 malloc-256 256 294 501 3680 0 62 203520 0 malloc-65536 65536 0 3 134 0 1 196608 0 UMA Zones 768 233 1 233 0 16 179712 0 MAP ENTRY 96 1441 407 112384 0 126 177408 0 malloc-128 128 1215 56 27485 0 126 162688 0 FFS2 dinode 256 512 118 1869 0 62 161280 0 malloc-32 32 4830 84 4843 0 254 157248 0 mbuf_jumbo_page 4096 5 31 133 0 254 147456 0 malloc-1024 1024 120 20 335 0 16 143360 0 tcp_bbr_map 128 0 1116 1074 0 126 142848 0 S VFS Cache 104 1012 275 2483 0 126 133848 0 malloc-65536 65536 2 0 2 0 1 131072 0 malloc-65536 65536 2 0 2 0 1 131072 0 VMSPACE 2544 35 13 2102 0 4 122112 0 ksiginfo 112 60 984 1726 0 126 116928 0 vmem btag 56 1814 46 1814 0 254 104160 0 PROC 1320 57 18 2118 0 8 99000 0 malloc-256 256 193 167 3537 0 62 92160 0 malloc-2048 2048 4 40 8413 0 8 90112 0 UMA Kegs 384 219 4 219 0 30 85632 0 filedesc0 1072 58 19 2119 0 8 82544 0 malloc-128 128 364 256 4461 0 126 79360 0 128 Bucket 1024 52 23 755 0 16 76800 0 malloc-4096 4096 18 0 18 0 2 73728 0 malloc-128 128 503 55 3590 0 126 71424 0 64 Bucket 512 81 55 3833 0 30 69632 0 malloc-65536 65536 1 0 1 0 1 65536 0 malloc-32768 32768 0 2 117 0 1 65536 0 malloc-16384 16384 4 0 157 0 1 65536 0 g_bio 408 0 160 29747 0 30 65280 0 malloc-256 256 123 132 4669 0 62 65280 0 socket 944 38 30 2471 0 254 64192 0 malloc-4096 4096 13 2 26 0 2 61440 0 malloc-8192 8192 7 0 7 0 1 57344 0 malloc-64 64 565 317 43397 0 254 56448 0 malloc-384 384 9 131 1872 0 30 53760 0 malloc-8192 8192 5 1 125 0 1 49152 0 malloc-2048 2048 4 20 511 0 8 49152 0 malloc-64 64 654 102 3242 0 254 48384 0 malloc-128 128 347 25 1216 0 126 47616 0 tcpcb 1048 7 37 731 0 254 46112 0 malloc-256 256 27 153 4517 0 62 46080 0 32 Bucket 256 52 128 4187 0 62 46080 0 malloc-128 128 62 279 2719 0 126 43648 0 clpbuf 2632 0 16 19 0 16 42112 0 DIRHASH 1024 34 6 34 0 16 40960 0 malloc-8192 8192 3 2 5 0 1 40960 0 pcpu-8 8 4589 531 4737 0 254 40960 0 malloc-384 384 98 2 101 0 30 38400 0 NAMEI 1024 0 36 19890 0 16 36864 0 malloc-384 384 54 36 263 0 30 34560 0 sctp_asoc 2288 1 14 63 0 254 34320 0 malloc-32768 32768 1 0 1 0 1 32768 0 malloc-32768 32768 1 0 1 0 1 32768 0 malloc-16384 16384 1 1 7 0 1 32768 0 malloc-8192 8192 4 0 4 0 1 32768 0 malloc-1024 1024 27 5 43 0 16 32768 0 pcpu-64 64 456 56 456 0 254 32768 0 malloc-64 64 308 196 4436 0 254 32256 0 sctp_ep 1280 1 23 147 0 254 30720 0 malloc-4096 4096 4 3 601 0 2 28672 0 malloc-4096 4096 7 0 7 0 2 28672 0 malloc-2048 2048 13 1 14 0 8 28672 0 KNOTE 160 28 147 58292 0 62 28000 0 malloc-256 256 19 86 1483 0 62 26880 0 pipe 744 22 13 333 0 16 26040 0 TURNSTILE 136 163 26 163 0 62 25704 0 malloc-4096 4096 2 4 1434 0 2 24576 0 ttyinq 160 135 15 300 0 62 24000 0 Files 80 223 77 10183 0 126 24000 0 8 Bucket 80 43 257 1179 0 126 24000 0 malloc-128 128 154 32 650 0 126 23808 0 ttyoutq 256 72 18 160 0 62 23040 0 malloc-256 256 9 81 3248 0 62 23040 0 malloc-4096 4096 3 2 6 0 2 20480 0 PWD 32 18 612 1347 0 254 20160 0 Mountpoints 2752 2 5 2 0 4 19264 0 malloc-384 384 44 6 44 0 30 19200 0 SLEEPQUEUE 88 163 29 163 0 126 16896 0 malloc-16384 16384 1 0 1 0 1 16384 0 malloc-16384 16384 1 0 1 0 1 16384 0 malloc-8192 8192 2 0 2 0 1 16384 0 malloc-1024 1024 1 15 63 0 16 16384 0 malloc-1024 1024 9 7 20 0 16 16384 0 malloc-512 512 5 27 168 0 30 16384 0 sctp_raddr 736 1 21 63 0 254 16192 0 vtnet_tx_hdr 24 0 668 654860 0 254 16032 0 tcp_inpcb 488 7 25 731 0 254 15616 0 unpcb 256 21 39 1364 0 254 15360 0 malloc-2048 2048 4 2 52 0 8 12288 0 malloc-2048 2048 1 5 259 0 8 12288 0 malloc-1024 1024 7 5 7 0 16 12288 0 malloc-512 512 10 14 32 0 30 12288 0 rtentry 176 31 38 35 0 62 12144 0 malloc-64 64 77 112 14921 0 254 12096 0 malloc-64 64 132 57 151 0 254 12096 0 malloc-32 32 249 129 743 0 254 12096 0 malloc-32 32 151 227 1057 0 254 12096 0 2 Bucket 32 53 325 879 0 254 12096 0 malloc-16 16 469 281 1719 0 254 12000 0 udp_inpcb 488 2 22 172 0 254 11712 0 routing nhops 256 28 17 35 0 62 11520 0 malloc-384 384 13 17 13 0 30 11520 0 malloc-256 256 26 19 26 0 62 11520 0 malloc-256 256 27 18 139 0 62 11520 0 malloc-8192 8192 1 0 1 0 1 8192 0 malloc-2048 2048 4 0 4 0 8 8192 0 malloc-2048 2048 2 2 2 0 8 8192 0 malloc-1024 1024 2 6 2 0 16 8192 0 malloc-1024 1024 3 5 1132 0 16 8192 0 malloc-512 512 3 13 511 0 30 8192 0 malloc-512 512 8 8 9 0 30 8192 0 PGRP 88 31 61 50 0 126 8096 0 rl_entry 40 69 133 69 0 254 8080 0 sctp_laddr 48 0 168 260 0 254 8064 0 udpcb 32 2 250 181 0 254 8064 0 malloc-64 64 9 117 46 0 254 8064 0 malloc-32 32 106 146 1052 0 254 8064 0 malloc-32 32 33 219 4213 0 254 8064 0 malloc-32 32 40 212 806 0 254 8064 0 malloc-32 32 19 233 1073 0 254 8064 0 16 Bucket 144 41 15 286 0 62 8064 0 4 Bucket 48 6 162 79 0 254 8064 0 malloc-16 16 5 495 1536 0 254 8000 0 malloc-16 16 28 472 141 0 254 8000 0 malloc-16 16 29 471 26594 0 254 8000 0 malloc-16 16 36 464 313 0 254 8000 0 malloc-16 16 29 471 205 0 254 8000 0 ripcb 488 4 12 16 0 254 7808 0 udplite_inpcb 488 0 16 9 0 254 7808 0 kenv 258 3 27 1040 0 30 7740 0 malloc-384 384 1 19 33 0 30 7680 0 malloc-384 384 14 6 14 0 30 7680 0 tcp_bbr_pcb 832 0 9 1 0 16 7488 0 FPU_save_area 832 1 8 1 0 16 7488 0 cpuset 104 7 55 7 0 126 6448 0 epoch_record pcpu 256 4 12 4 0 62 4096 0 malloc-4096 4096 0 1 3 0 2 4096 0 malloc-2048 2048 2 0 2 0 8 4096 0 malloc-512 512 1 7 1 0 30 4096 0 pcpu-16 16 7 249 7 0 254 4096 0 sctp_stream_msg_out 112 0 36 2 0 254 4032 0 hostcache 96 1 41 1 0 254 4032 0 syncache 168 0 24 4 0 254 4032 0 malloc-64 64 3 60 3 0 254 4032 0 malloc-32 32 7 119 7 0 254 4032 0 malloc-16 16 2 248 2 0 254 4000 0 sctp_chunk 152 2 24 2 0 254 3952 0 UMA Slabs 1 176 9 13 9 0 62 3872 0 mqnode 416 3 6 3 0 30 3744 0 KMAP ENTRY 96 12 27 12 0 0 3744 0 vmem 1856 1 1 1 0 8 3712 0 SMR CPU 32 2 29 2 0 254 992 0 SMR SHARED 24 2 29 2 0 254 744 0 FFS1 dinode 128 0 0 0 0 126 0 0 swblk 136 0 0 0 0 62 0 0 swpctrie 144 0 0 0 0 62 0 0 sctp_asconf_ack 48 0 0 0 0 254 0 0 sctp_asconf 40 0 0 0 0 254 0 0 sctp_readq 152 0 0 0 0 254 0 0 pf state scrubs 40 0 0 0 0 254 0 0 pf frag entries 40 0 0 0 0 254 0 0 pf frags 248 0 0 0 0 62 0 0 pf table entries 160 0 0 0 0 62 0 0 pf table entry counters 64 0 0 0 0 254 0 0 pf source nodes 136 0 0 0 0 254 0 0 pf state keys 88 0 0 0 0 126 0 0 pf states 296 0 0 0 0 254 0 0 pf tags 104 0 0 0 0 126 0 0 pf mtags 48 0 0 0 0 254 0 0 tcp_rack_pcb 704 0 0 0 0 16 0 0 tcp_rack_map 120 0 0 0 0 126 0 0 IPsec SA lft_c 16 0 0 0 0 254 0 0 tcp_log_node 120 0 0 0 0 126 0 0 tcp_log_bucket 176 0 0 0 0 62 0 0 tcp_log 416 0 0 0 0 254 0 0 tcpreass 48 0 0 0 0 254 0 0 tfo_ccache_entries 80 0 0 0 0 126 0 0 tfo 4 0 0 0 0 254 0 0 sackhole 32 0 0 0 0 254 0 0 tcptw 88 0 0 0 0 254 0 0 ipq 56 0 0 0 0 254 0 0 itimer 352 0 0 0 0 30 0 0 AIOLIO 272 0 0 0 0 30 0 0 AIOCB 552 0 0 0 0 16 0 0 AIOP 32 0 0 0 0 254 0 0 AIO 208 0 0 0 0 62 0 0 NCLNODE 592 0 0 0 0 16 0 0 mqnotifier 216 0 0 0 0 62 0 0 mvdata 64 0 0 0 0 254 0 0 mqueue 248 0 0 0 0 62 0 0 TMPFS node 224 0 0 0 0 62 0 0 LTS VFS Cache 360 0 0 0 0 30 0 0 L VFS Cache 320 0 0 0 0 30 0 0 STS VFS Cache 144 0 0 0 0 62 0 0 crypto_session 72 0 0 0 0 126 0 0 cryptop 280 0 0 0 0 30 0 0 IOMMU_MAP_ENTRY 120 0 0 0 0 126 0 0 ktls_session 192 0 0 0 0 62 0 0 mbuf_jumbo_16k 16384 0 0 0 0 254 0 0 mbuf_jumbo_9k 9216 0 0 0 0 254 0 0 audit_record 1280 0 0 0 0 8 0 0 domainset 40 0 0 0 0 254 0 0 MAC labels 40 0 0 0 0 254 0 0 vnpbuf 2632 0 0 0 0 64 0 0 mdpbuf 2632 0 0 0 0 3 0 0 nfspbuf 2632 0 0 0 0 16 0 0 swwbuf 2632 0 0 0 0 8 0 0 swrbuf 2632 0 0 0 0 16 0 0 umtx_shm 88 0 0 0 0 126 0 0 umtx pi 96 0 0 0 0 126 0 0 rangeset pctrie nodes 144 0 0 0 0 62 0 0 malloc-65536 65536 0 0 0 0 1 0 0 malloc-65536 65536 0 0 0 0 1 0 0 malloc-65536 65536 0 0 0 0 1 0 0 malloc-32768 32768 0 0 0 0 1 0 0 malloc-32768 32768 0 0 0 0 1 0 0 malloc-32768 32768 0 0 0 0 1 0 0 malloc-32768 32768 0 0 0 0 1 0 0 malloc-16384 16384 0 0 0 0 1 0 0 malloc-16384 16384 0 0 0 0 1 0 0 malloc-16384 16384 0 0 0 0 1 0 0 malloc-8192 8192 0 0 0 0 1 0 0 malloc-8192 8192 0 0 0 0 1 0 0 malloc-1024 1024 0 0 0 0 16 0 0 malloc-512 512 0 0 0 0 30 0 0 malloc-512 512 0 0 0 0 30 0 0 malloc-512 512 0 0 0 0 30 0 0 malloc-128 128 0 0 0 0 126 0 0 pcpu-32 32 0 0 0 0 254 0 0 pcpu-4 4 0 0 0 0 254 0 0 fakepg 104 0 0 0 0 126 0 0 UMA Hash 256 0 0 0 0 62 0 0