panic: sleeping thread holds tun_mtx cpuid = 0 time = 1753880987 KDB: stack backtrace: db_trace_self_wrapper() at db_trace_self_wrapper+0xc6/frame 0xfffffe0056a7c9b0 kdb_backtrace() at kdb_backtrace+0xd0/frame 0xfffffe0056a7cb10 vpanic() at vpanic+0x257/frame 0xfffffe0056a7ccd0 panic() at panic+0xb5/frame 0xfffffe0056a7cd90 propagate_priority() at propagate_priority+0x521/frame 0xfffffe0056a7cde0 turnstile_wait() at turnstile_wait+0x663/frame 0xfffffe0056a7ce50 __mtx_lock_sleep() at __mtx_lock_sleep+0x452/frame 0xfffffe0056a7cf90 __mtx_lock_flags() at __mtx_lock_flags+0x1fe/frame 0xfffffe0056a7d080 tunstart_l2() at tunstart_l2+0x74/frame 0xfffffe0056a7d0e0 tap_transmit() at tap_transmit+0x1d0/frame 0xfffffe0056a7d130 ether_output_frame() at ether_output_frame+0x30c/frame 0xfffffe0056a7d1f0 ether_output() at ether_output+0x114b/frame 0xfffffe0056a7d360 ip6_output_send() at ip6_output_send+0x1b6/frame 0xfffffe0056a7d3e0 ip6_output() at ip6_output+0x484b/frame 0xfffffe0056a7d930 mld_dispatch_packet() at mld_dispatch_packet+0x8d1/frame 0xfffffe0056a7da90 mld_fasttimo() at mld_fasttimo+0x1900/frame 0xfffffe0056a7dcd0 softclock_call_cc() at softclock_call_cc+0x422/frame 0xfffffe0056a7de80 softclock_thread() at softclock_thread+0x200/frame 0xfffffe0056a7def0 fork_exit() at fork_exit+0xcc/frame 0xfffffe0056a7df30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe0056a7df30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- KDB: enter: panic [ thread pid 2 tid 100031 ] Stopped at kdb_enter+0x6e: movq $0,0x25c44a7(%rip) db> db> set $lines = 0 db> set $maxwidth = 0 db> show registers cs 0x20 ds 0x3b es 0x3b fs 0x13 gs 0x1b ss 0x28 rax 0x12 rcx 0xfffffe0002bf1850 rdx 0 rbx 0xffffffff827cb4e0 .str.27 rsp 0xfffffe0056a7caf0 rbp 0xfffffe0056a7cb10 rsi 0 rdi 0xffffffff81614bb9 printf+0x149 r8 0 r9 0xffffffff r10 0x6 r11 0x5 r12 0xfffffe000781a780 r13 0xfffffffffffffffe r14 0xffffffff827cb4e0 .str.27 r15 0 rip 0xffffffff815fe6ee kdb_enter+0x6e rflags 0x46 kdb_enter+0x6e: movq $0,0x25c44a7(%rip) db> show proc Process 2 (clock) at 0xfffffe0007808020: state: NORMAL uid: 0 gids: 0 parent: pid 0 at 0xffffffff83b4d060 ABI: null flag: 0x10000284 flag2: 0 reaper: 0xffffffff83b4d060 reapsubtree: 2 sigparent: 20 vmspace: 0xffffffff83b4e040 (map 0xffffffff83b4e040) (map.pmap 0xffffffff83b4e0e0) (pmap 0xffffffff83b4e150) threads: 2 100031 L *tun_mtx 0xfffffe00079f9600 [clock (0)] 100032 I [clock (1)] db> ps pid ppid pgrp uid state wmesg wchan cmd 899 764 764 0 R (threaded) syz-executor 100096 RunQ syz-executor 100203 S connec 0xfffffe0059b050da syz-executor 100204 S uwait 0xfffffe005969b280 syz-executor 890 765 765 0 R (threaded) syz-executor 100188 RunQ syz-executor 100193 S connec 0xfffffe0059b058da syz-executor 100195 S uwait 0xfffffe005825a480 syz-executor 889 1 765 0 S uwait 0xfffffe0059699380 syz-executor 883 0 0 0 DL - 0xffffffff83cb3e00 [soaiod4] 882 0 0 0 DL - 0xffffffff83cb3e00 [soaiod3] 881 0 0 0 DL - 0xffffffff83cb3e00 [soaiod2] 880 0 0 0 DL - 0xffffffff83cb3e00 [soaiod1] 870 1 765 0 S uwait 0xfffffe005969aa00 syz-executor 849 1 765 0 S uwait 0xfffffe0059698080 syz-executor 845 0 0 0 DL aiordy 0xfffffe00540f1060 [aiod4] 844 0 0 0 DL aiordy 0xfffffe00540f15c0 [aiod3] 843 0 0 0 DL aiordy 0xfffffe00540d1060 [aiod2] 842 0 0 0 DL aiordy 0xfffffe005410c560 [aiod1] 832 1 832 0 Ss+ ttyin 0xfffffe00585ce0b0 getty 831 1 831 0 Ss+ ttyin 0xfffffe00585cdcb0 getty 830 1 830 0 Ss+ ttyin 0xfffffe00585cd8b0 getty 829 1 829 0 Ss+ ttyin 0xfffffe00585cd4b0 getty 828 1 828 0 Ss+ ttyin 0xfffffe00585cd0b0 getty 827 1 827 0 Ss+ ttyin 0xfffffe00585cccb0 getty 826 1 826 0 Ss+ ttyin 0xfffffe00585cc8b0 getty 825 1 825 0 Ss+ ttyin 0xfffffe00585cc4b0 getty 824 1 824 0 Ss+ ttyin 0xfffffe00582904b0 getty 817 815 423 0 L *tun_mtx 0xfffffe00079f9600 ifconfig 815 809 423 0 S wait 0xfffffe005410dae0 sh 809 786 423 0 S wait 0xfffffe0007809b00 dhclient 805 763 763 0 T (threaded) syz-executor 100112 s syz-executor 100122 D tun_ioc 0xffffffff83cc3a40 syz-executor 786 423 423 0 S wait 0xfffffe000780a060 sh 766 762 766 0 R CPU 1 syz-executor 765 762 765 0 R syz-executor 764 762 764 0 S nanslp 0xffffffff83ba3c40 syz-executor 763 762 763 0 S nanslp 0xffffffff83ba3c40 syz-executor 762 760 760 0 R syz-executor 760 758 760 0 Ss sigsusp 0xfffffe00540020b0 csh 758 681 758 0 Ss select 0xfffffe006dfb64c0 sshd 737 1 17 0 S+ piperd 0xfffffe0059832700 logger 736 735 17 0 S+ nanslp 0xffffffff83ba3c40 sleep 735 1 17 0 S+ wait 0xfffffe00540cf580 sh 685 1 685 0 Ss nanslp 0xffffffff83ba3c41 cron 681 1 681 0 Ss select 0xfffffe0053ffb340 sshd 494 1 494 0 Ss select 0xfffffe0053ffb9c0 syslogd 423 1 423 0 Ss wait 0xfffffe00540d0040 devd 16 0 0 0 DL syncer 0xffffffff83cc1820 [syncer] 15 0 0 0 RL [vnlru] 14 0 0 0 RL (threaded) [bufdaemon] 100079 D psleep 0xffffffff83cbfd60 [bufdaemon] 100082 D - 0xffffffff83001ec0 [bufspacedaemon-0] 100095 RunQ [/ worker] 9 0 0 0 DL psleep 0xffffffff83d0acc0 [vmdaemon] 8 0 0 0 RL (threaded) [pagedaemon] 100077 RunQ [dom0] 100080 D launds 0xffffffff83cf0d94 [laundry: dom0] 100081 D umarcl 0xffffffff81de0fc0 [uma] 7 0 0 0 DL - 0xffffffff8391c5d8 [rand_harvestq] 6 0 0 0 DL pftm 0xffffffff847d99e0 [pf purge] 5 0 0 0 DL waiting 0xffffffff844af700 [sctp_iterator] 4 0 0 0 DL (threaded) [cam] 100045 D - 0xffffffff838e6340 [doneq0] 100046 D - 0xffffffff838e62c0 [async] 100075 D - 0xffffffff838e6140 [scanner] 3 0 0 0 DL (threaded) [crypto] 100042 D crypto_ 0xffffffff83cec640 [crypto] 100043 D crypto_ 0xfffffe0057d43030 [crypto returns 0] 100044 D crypto_ 0xfffffe0057d43080 [crypto returns 1] 13 0 0 0 DL (threaded) [geom] 100037 D - 0xffffffff83b4c620 [g_event] 100038 D - 0xffffffff83b4c640 [g_up] 100039 D - 0xffffffff83b4c660 [g_down] 2 0 0 0 LL (threaded) [clock] 100031 L *tun_mtx 0xfffffe00079f9600 [clock (0)] 100032 I [clock (1)] 12 0 0 0 RL (threaded) [intr] 100013 I [swi6: task queue] 100014 I [swi6: Giant taskq] 100016 I [swi5: fast taskq] 100033 I [swi1: netisr 0] 100034 I [swi1: hpts] 100035 RunQ [swi1: hpts] 100047 I [irq24: virtio_pci0] 100048 I [irq25: virtio_pci0] 100049 I [irq26: virtio_pci0] 100050 I [irq27: virtio_pci0] 100051 I [irq28: virtio_pci1] 100052 I [irq29: virtio_pci1] 100053 I [irq30: virtio_pci1] 100054 I [irq31: virtio_pci1] 100055 I [irq32: virtio_pci1] 100060 I [irq10: virtio_pci2] 100062 I [irq1: atkbd0] 100063 I [irq12: psm0] 100064 I [swi0: uart uart++] 100068 I [swi1: pf send] 11 0 0 0 RL (threaded) [idle] 100003 CanRun [idle: cpu0] 100004 CanRun [idle: cpu1] 1 0 1 0 SLs wait 0xfffffe0007809040 [init] 10 0 0 0 DL audit_w 0xffffffff83ced0e0 [audit] 0 0 0 0 RLs (threaded) [kernel] 100000 D parked 0xffffffff84c3dff0 [swapper] 100005 D - 0xfffffe0007768d00 [softirq_0] 100006 D - 0xfffffe0007768b00 [softirq_1] 100007 D - 0xfffffe0007768900 [if_io_tqg_0] 100008 D - 0xfffffe0007768700 [if_io_tqg_1] 100009 D - 0xfffffe0007768500 [if_config_tqg_0] 100010 D - 0xfffffe00083db100 [kqueue_ctx taskq] 100011 D - 0xfffffe00083db000 [jail_remove taskq] 100012 D - 0xfffffe00083dae00 [bus taskq] 100015 D - 0xfffffe00083da900 [thread taskq] 100017 D - 0xfffffe00083da600 [aiod_kick taskq] 100018 D - 0xfffffe00083da500 [deferred_unmount ta] 100019 D - 0xfffffe00083da400 [inm_free taskq] 100020 D - 0xfffffe00083da300 [in6m_free taskq] 100021 D - 0xfffffe00083da200 [linuxkpi_irq_wq] 100022 D - 0xfffffe00083da100 [linuxkpi_short_wq_0] 100023 D - 0xfffffe00083da100 [linuxkpi_short_wq_1] 100024 D - 0xfffffe00083da100 [linuxkpi_short_wq_2] 100025 D - 0xfffffe00083da100 [linuxkpi_short_wq_3] 100026 D - 0xfffffe00083da000 [linuxkpi_long_wq_0] 100027 D - 0xfffffe00083da000 [linuxkpi_long_wq_1] 100028 D - 0xfffffe00083da000 [linuxkpi_long_wq_2] 100029 D - 0xfffffe00083da000 [linuxkpi_long_wq_3] 100036 D - 0xfffffe00083d9900 [firmware taskq] 100040 D - 0xfffffe0057d47300 [crypto_0] 100041 D - 0xfffffe0057d47300 [crypto_1] 100056 D - 0xfffffe00083dd200 [vtnet0 rxq 0] 100057 D - 0xfffffe0058145500 [vtnet0 txq 0] 100058 D - 0xfffffe0058145400 [vtnet0 rxq 1] 100059 D - 0xfffffe0058145300 [vtnet0 txq 1] 100061 D vtbslp 0xfffffe0057d67400 [virtio_balloon] 100065 RunQ [deadlkres] 100069 D - 0xfffffe0057d46e00 [acpi_task_0] 100070 D - 0xfffffe0057d46e00 [acpi_task_1] 100071 D - 0xfffffe0057d46e00 [acpi_task_2] 100073 D - 0xfffffe00083dca00 [mca taskq] 100074 D - 0xfffffe0057d46b00 [CAM taskq] 100076 D - 0xfffffe0058142400 [ipsec_offload] db> show all locks Process 817 (ifconfig) thread 0xfffffe005411b000 (100130) exclusive sx tun_ioctl (tun_ioctl) r = 0 (0xffffffff83cc3a40) locked @ /syzkaller/managers/main/kernel/sys/net/if_tuntap.c:1340 exclusive sx in_control (in_control) r = 0 (0xffffffff83cc6660) locked @ /syzkaller/managers/main/kernel/sys/netinet/in.c:367 Process 805 (syz-executor) thread 0xfffffe005410a780 (100122) exclusive sleep mutex tun_mtx (tun_mtx) r = 0 (0xfffffe00595c0c88) locked @ /syzkaller/managers/main/kernel/sys/net/if_tuntap.c:1641 Process 766 (syz-executor) thread 0xfffffe00540df000 (100111) exclusive lockmgr bufwait (bufwait) r = 0 (0xfffffe0007cf4e38) locked @ /syzkaller/managers/main/kernel/sys/ufs/ffs/ffs_softdep.c:14492 exclusive lockmgr ufs (ufs) r = 0 (0xfffffe006e560750) locked @ /syzkaller/managers/main/kernel/sys/kern/vfs_subr.c:3384 exclusive lockmgr ufs (ufs) r = 0 (0xfffffe006df53e30) locked @ /syzkaller/managers/main/kernel/sys/kern/vfs_cache.c:4844 Process 2 (clock) thread 0xfffffe000781a780 (100031) exclusive sleep mutex mld_mtx (mld_mtx) r = 0 (0xffffffff83ce7d80) locked @ /syzkaller/managers/main/kernel/sys/netinet6/mld6.c:1349 exclusive sleep mutex in6_multi_list_mtx (in6_multi_list_mtx) r = 0 (0xffffffff83ce7ba0) locked @ /syzkaller/managers/main/kernel/sys/netinet6/mld6.c:1348 shared rw vnet_rwlock (vnet_rwlock) r = 0 (0xffffffff83cc5de0) locked @ /syzkaller/managers/main/kernel/sys/netinet6/mld6.c:1307 db> show malloc Type InUse MemUse Requests pf_hash 6 12804K 6 linker 376 5079K 486 tcp_hpts 7 4801K 7 devbuf 4187 4323K 4212 sysctloid 34891 2055K 34966 vtbuf 24 1968K 46 kobj 330 1320K 494 newblk 430 1132K 549 vfscache 3 1025K 3 pcb 25 671K 64 inodedep 86 544K 175 ufs_quota 1 512K 1 vfs_hash 1 512K 1 callout 2 512K 2 intr 4 472K 4 subproc 125 258K 975 vnet_data 2 224K 2 acpitask 1 224K 1 KTRACE 100 200K 100 acpica 1674 184K 54426 filedesc 22 169K 217 vmem 5 144K 7 tidhash 3 141K 3 pagedep 24 134K 98 tfo_ccache 1 128K 1 IP reass 1 128K 1 sem 4 106K 4 DEVFS1 106 106K 123 gtaskqueue 18 98K 18 bus 997 82K 5063 mtx_pool 3 74K 3 syncache 1 68K 1 NFSD srvcache 3 68K 3 module 521 66K 521 ddb_capture 1 64K 1 kdtrace 195 38K 1121 umtx 288 36K 288 hostcache 1 32K 1 shm 1 32K 1 DEVFS3 125 32K 135 msg 4 30K 4 kbdmux 6 28K 6 temp 31 21K 1863 DEVFS_RULE 56 20K 56 ifaddr 67 19K 69 ufs_mount 4 17K 5 proc 3 17K 3 LRO 16 17K 16 tty 16 16K 16 routetbl 124 16K 395 ithread 90 15K 90 bus-sc 34 15K 1647 lltable 44 14K 44 eventhandler 163 14K 163 ifnet 7 13K 7 ether_multi 152 13K 162 kenv 95 12K 95 GEOM 49 11K 431 CAM queue 5 11K 1528 rman 82 10K 437 rpc 8 9K 8 bmsafemap 4 9K 142 in6_multi 65 9K 65 dirrem 33 9K 121 plimit 22 9K 477 devstat 4 9K 4 UART 12 9K 12 ksem 1 8K 1 shmfd 1 8K 1 pfs_vncache 1 8K 1 audit_evclass 240 8K 303 taskqueue 69 8K 69 cred 25 7K 199 sglist 6 7K 6 CAM DEV 3 6K 510 pfs_nodes 22 6K 22 ufs_dirhash 24 5K 24 UMA 266 5K 266 pf_ifnet 10 5K 19 vt 11 5K 11 memdesc 1 4K 1 MCA 32 4K 32 evdev 4 4K 4 diradd 31 4K 135 kqueue 58 4K 922 pwddesc 58 4K 907 acpisem 28 4K 28 indirdep 13 4K 34 terminal 11 3K 11 DEVFSP 43 3K 53 acpidev 20 3K 20 proc-args 78 3K 1966 hhook 8 3K 10 clone 9 3K 9 selfd 36 3K 9800 kcovinfo 36 3K 36 freefile 17 3K 89 uidinfo 2 3K 10 session 17 3K 41 local_apic 1 2K 1 io_apic 1 2K 1 mkdir 16 2K 176 ipsec-saq 2 2K 2 ip6ndp 12 2K 13 Unitno 31 2K 50 newdirblk 14 2K 88 lockf 16 2K 26 sctp_ifa 13 2K 14 CAM XPT 22 2K 543 in_multi 6 2K 8 tun 4 2K 4 toponodes 6 2K 6 ipsecpolicy 2 2K 2 freework 6 2K 114 freeblks 5 2K 100 msi 9 2K 9 netlink 2 2K 70 sctp_stro 1 1K 2 softdep 1 1K 1 sahead 1 1K 1 secasvar 1 1K 1 nhops 6 1K 8 vnodemarker 2 1K 8 NFSD session 1 1K 1 CAM periph 4 1K 271 ipsec 3 1K 3 sctp_ifn 6 1K 14 sctp_atcl 2 1K 7 mld 6 1K 6 igmp 6 1K 6 pfil 6 1K 6 BPF 6 1K 18 select 6 1K 28 isadev 6 1K 6 mount 16 1K 89 pci_link 10 1K 10 crypto 4 1K 5 CC Mem 5 1K 17 encap_export_host 12 1K 12 osd 10 1K 34 cdev 2 1K 2 lkpikmalloc 8 1K 9 counter_rate 13 1K 13 inpcbpolicy 12 1K 182 chacha20random 1 1K 1 biobuf 1 1K 1 sctp_timw 1 1K 1 vnodes 1 1K 1 NFSD lckfile 1 1K 1 NFSD V4client 1 1K 1 DEVFS 9 1K 10 CAM SIM 2 1K 2 feeder 7 1K 7 cryptodev 3 1K 47 tcpfunc 3 1K 3 loginclass 3 1K 5 prison 6 1K 6 soname 6 1K 3336 sctp_atky 3 1K 9 nexusdev 8 1K 8 apmdev 1 1K 1 atkbddev 2 1K 2 aio 4 1K 4 pmchooks 1 1K 1 CAM path 4 1K 1034 CAM dev queue 2 1K 2 CAM I/O Scheduler 1 1K 1 filecaps 3 1K 70 sctp_vrf 1 1K 1 ip_msource 1 1K 1 vnet 1 1K 1 pmc 1 1K 1 entropy 2 1K 33 acpiintr 1 1K 1 sctp_athm 2 1K 7 sctp_map 2 1K 4 cpus 2 1K 2 vnet_data_free 1 1K 1 Per-cpu 1 1K 1 p1003.1b 1 1K 1 ipcomp 0 0K 0 esp 0 0K 0 ah 0 0K 0 pf_table 0 0K 0 pf_rule 0 0K 0 pf_altq 0 0K 0 pf_osfp 0 0K 0 pf_krule_item 0 0K 0 pf_temp 0 0K 0 mqdata 0 0K 0 filemon 0 0K 1 tcp_pcm_rack 0 0K 0 tcp_do_rack 0 0K 0 tcp_fsb_rack 0 0K 0 sctp_mcore 0 0K 0 sctp_socko 0 0K 0 sctp_iter 0 0K 12 sctp_mvrf 0 0K 0 sctp_cpal 0 0K 0 sctp_cmsg 0 0K 0 sctp_stre 0 0K 0 sctp_athi 0 0K 0 sctp_a_it 0 0K 12 sctp_aadr 0 0K 0 sctp_stri 0 0K 0 madt_table 0 0K 2 smartpqi 0 0K 0 ixl 0 0K 0 ice-resmgr 0 0K 0 ice-osdep 0 0K 0 ice 0 0K 0 iavf 0 0K 0 axgbe 0 0K 0 fpukern_ctx 0 0K 0 xen_intr 0 0K 0 xen_hvm 0 0K 0 legacydrv 0 0K 0 NMI handlers 0 0K 0 bounce 0 0K 0 busdma 0 0K 0 qpidrv 0 0K 0 dmar_idpgtbl 0 0K 0 dmar_dom 0 0K 0 dmar_ctx 0 0K 0 amdiommu_dom 0 0K 0 amdiommu_ctx 0 0K 0 isci 0 0K 0 iommu_dmamap 0 0K 0 hyperv_socket 0 0K 0 bxe_ilt 0 0K 0 aesni_data 0 0K 0 xenbus 0 0K 0 vm_fictitious 0 0K 0 UMAHash 0 0K 0 vm_pgdata 0 0K 0 jblocks 0 0K 0 savedino 0 0K 57 sentinel 0 0K 0 jfsync 0 0K 0 jtrunc 0 0K 0 sbdep 0 0K 2 jsegdep 0 0K 0 jseg 0 0K 0 jfreefrag 0 0K 0 jfreeblk 0 0K 0 jnewblk 0 0K 0 jmvref 0 0K 0 jremref 0 0K 0 jaddref 0 0K 0 freedep 0 0K 0 freefrag 0 0K 2 allocindir 0 0K 0 allocdirect 0 0K 0 ufs_trim 0 0K 0 mactemp 0 0K 0 audit_trigger 0 0K 0 audit_pipe_presel 0 0K 0 audit_pipeent 0 0K 0 audit_pipe 0 0K 0 audit_evname 0 0K 0 audit_bsm 0 0K 0 audit_gidset 0 0K 0 audit_text 0 0K 0 audit_path 0 0K 0 audit_data 0 0K 0 audit_cred 0 0K 0 ktls_ocf 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5E_TLS_RX 0 0K 0 MLX5EEPROM 0 0K 0 MLX5E_TLS 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EN 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5DUMP 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 simple_attr 0 0K 0 seq_file 0 0K 0 lkpiskb 0 0K 0 radix 0 0K 0 idr 0 0K 0 lkpindev 0 0K 0 lkpimhi 0 0K 0 lkpifw 0 0K 0 lkpi80211 0 0K 0 NLM 0 0K 0 ipsec-spdcache 0 0K 0 ipsec-reg 0 0K 0 ipsec-misc 0 0K 0 ipsecrequest 0 0K 0 ip6opt 0 0K 9 ip6_msource 0 0K 0 ip6_moptions 0 0K 0 in6_mfilter 0 0K 0 frag6 0 0K 0 tcplog 0 0K 0 tcp_hwpace 0 0K 0 ip_moptions 0 0K 1 in_mfilter 0 0K 2 ipid 0 0K 0 80211scan 0 0K 0 80211ratectl 0 0K 0 80211power 0 0K 0 80211nodeie 0 0K 0 80211node 0 0K 0 80211mesh_gt 0 0K 0 80211mesh_rt 0 0K 0 80211perr 0 0K 0 80211prep 0 0K 0 80211preq 0 0K 0 80211dfs 0 0K 0 80211crypto 0 0K 0 80211vap 0 0K 0 iflib 0 0K 0 vlan 0 0K 0 gif 0 0K 0 ifdescr 0 0K 0 zlib 0 0K 19 fadvise 0 0K 3 VN POLL 0 0K 0 statfs 0 0K 183 namei_tracker 0 0K 2 inotify 0 0K 0 export_host 0 0K 0 cl_savebuf 0 0K 5 lio 0 0K 1 acl 0 0K 0 mbuf_tag 0 0K 0 ktls 0 0K 0 accf 0 0K 0 pts 0 0K 0 timerfd 0 0K 0 procdesc 0 0K 8 iov 0 0K 13888 ioctlops 0 0K 96 eventfd 0 0K 1 Witness 0 0K 0 stack 0 0K 0 sbuf 0 0K 304 firmware 0 0K 0 compressor 0 0K 0 SWAP 0 0K 0 sysctltmp 0 0K 675 sysctl 0 0K 3 ekcd 0 0K 0 dumper 0 0K 0 sendfile 0 0K 0 rctl 0 0K 0 cache 0 0K 0 prison_racct 0 0K 0 Fail Points 0 0K 0 sigio 0 0K 1 filedesc_to_leader 0 0K 1 pwd 0 0K 0 tty console 0 0K 0 boottrace 0 0K 0 isofs_node 0 0K 0 isofs_mount 0 0K 0 tr_raid5_data 0 0K 0 tr_raid1e_data 0 0K 0 tr_raid1_data 0 0K 0 tr_raid0_data 0 0K 0 tr_concat_data 0 0K 0 md_sii_data 0 0K 0 md_promise_data 0 0K 0 md_nvidia_data 0 0K 0 md_jmicron_data 0 0K 0 md_intel_data 0 0K 0 md_ddf_data 0 0K 0 raid_data 0 0K 72 geom_flashmap 0 0K 0 tmpfs dir 0 0K 0 tmpfs name 0 0K 0 tmpfs mount 0 0K 0 tmpfs extattr 0 0K 0 NFS FHA 0 0K 0 newnfsmnt 0 0K 0 newnfsclient_req 0 0K 0 NFSCL layrecall 0 0K 0 NFSCL session 0 0K 0 NFSCL sockreq 0 0K 0 NFSCL devinfo 0 0K 0 NFSCL flayout 0 0K 0 NFSCL layout 0 0K 0 NFSD rollback 0 0K 0 NFSCL diroff 0 0K 0 NEWNFSnode 0 0K 0 NFSCL lck 0 0K 0 NFSCL lckown 0 0K 0 NFSCL client 0 0K 0 NFSCL deleg 0 0K 0 NFSCL open 0 0K 0 NFSCL owner 0 0K 0 NFS fh 0 0K 0 NFS req 0 0K 0 NFSD usrgroup 0 0K 0 NFSD string 0 0K 0 NFSD V4lock 0 0K 0 NFSD V4state 0 0K 0 msdosfs_fat 0 0K 0 msdosfs_mount 0 0K 0 msdosfs_node 0 0K 0 DEVFS4 0 0K 0 DEVFS2 0 0K 0 gntdev 0 0K 0 privcmd_dev 0 0K 0 evtchn_dev 0 0K 0 xenstore 0 0K 0 xnb 0 0K 0 xen_acpi 0 0K 0 xbbd 0 0K 0 xbd 0 0K 0 Balloon 0 0K 0 sysmouse 0 0K 0 vtfont 0 0K 0 pvscsi 0 0K 0 USBdev 0 0K 0 USB 0 0K 0 twsbuf 0 0K 0 tcp_log_dev 0 0K 0 midi buffers 0 0K 0 mixer 0 0K 0 ac97 0 0K 0 hdacc 0 0K 0 hdac 0 0K 0 hdaa 0 0K 0 SIIS driver 0 0K 0 PUC 0 0K 0 ppbusdev 0 0K 0 sr_iov 0 0K 0 OCS 0 0K 0 OCS 0 0K 0 nvme 0 0K 0 nvd 0 0K 0 netmap 0 0K 0 mwldev 0 0K 0 MVS driver 0 0K 0 mpi3mrbuf 0 0K 0 mrsasbuf 0 0K 0 mpt_user 0 0K 0 mps_user 0 0K 0 MPSSAS 0 0K 0 mps 0 0K 0 mpr_user 0 0K 0 MPRSAS 0 0K 0 mpr 0 0K 0 mfibuf 0 0K 0 md_sectors 0 0K 0 md_disk 0 0K 0 malodev 0 0K 0 LED 0 0K 0 ix_sriov 0 0K 0 ix 0 0K 0 ipsbuf 0 0K 0 ciss_data 0 0K 0 BACKLIGHT 0 0K 0 ath_hal 0 0K 0 athdev 0 0K 0 ata_pci 0 0K 0 ata_dma 0 0K 0 ata_generic 0 0K 0 AHCI driver 0 0K 0 agp 0 0K 0 acpipwr 0 0K 0 acpi_perf 0 0K 0 acpicmbat 0 0K 0 aacraidcam 0 0K 0 aacraid_buf 0 0K 0 aaccam 0 0K 0 aacbuf 0 0K 0 zstd 0 0K 0 XZ_DEC 0 0K 0 nvlist 0 0K 0 SCSI ENC 0 0K 0 SCSI sa 0 0K 0 scsi_pass 0 0K 0 scsi_da 0 0K 70 ata_da 0 0K 0 scsi_ch 0 0K 0 scsi_cd 0 0K 0 nvme_da 0 0K 0 CAM CCB 0 0K 523 CAM ccb queue 0 0K 0 db> show uma Zone Size Used Free Requests Sleeps Bucket Total Mem XFree mbuf_jumbo_page 4096 8322 1076 12367 0 254 38494208 0 mbuf 256 8631 1031 15486 0 254 2473472 0 BUF TRIE 152 211 11593 447 0 62 1794208 0 malloc-4096 4096 388 6 1402 0 2 1613824 0 malloc-384 384 4121 49 4464 0 30 1601280 0 malloc-128 128 11489 229 11628 0 126 1499904 0 tcp_log 416 1596 1968 5049 0 254 1482624 0 UMA Slabs 0 112 10902 0 10902 0 126 1221024 0 mbuf_cluster 2048 508 0 508 0 254 1040384 0 RADIX NODE 152 5972 237 28242 0 62 943768 0 vmem btag 56 16254 129 16254 0 254 917448 0 malloc-65536 65536 9 1 12 0 1 655360 0 FFS inode 1168 535 25 625 0 8 654080 0 sctp_ep 1152 1 510 5 0 254 588672 0 sctp_asoc 2256 1 254 2 0 254 575280 0 pbuf 2624 0 202 0 0 2 530048 0 socket 1024 20 488 1387 0 254 520192 0 lkpicurr 168 2 3094 2 0 62 520128 0 malloc-2048 2048 106 78 636 0 8 376832 0 malloc-65536 65536 4 1 114 0 1 327680 0 VM OBJECT 248 1029 219 14324 0 62 309504 0 malloc-32768 32768 4 5 24 0 1 294912 0 VNODE 440 572 94 664 0 30 293040 0 256 Bucket 2048 120 16 870 0 8 278528 0 malloc-64 64 4157 190 5888 0 254 278208 0 THREAD 1860 136 8 212 0 8 267840 0 malloc-16384 16384 11 5 107 0 1 262144 0 malloc-16 16 14398 102 14482 0 254 232000 0 DEVCTL 1024 20 200 147 0 0 225280 0 malloc-256 256 739 71 2004 0 62 207360 0 sctp_raddr 736 1 263 3 0 254 194304 0 UMA Zones 768 238 1 238 0 16 183552 0 malloc-32 32 5334 336 5359 0 254 181440 0 malloc-128 128 1214 181 25361 0 126 178560 0 lkpimm 56 1 3095 1 0 254 173376 0 unpcb 320 7 509 1175 0 254 165120 0 malloc-1024 1024 124 36 867 0 16 163840 0 FFS2 dinode 256 535 95 624 0 62 161280 0 MAP ENTRY 96 1182 330 44462 0 126 145152 0 FPU_save_area 832 138 24 297 0 16 134784 0 malloc-65536 65536 2 0 2 0 1 131072 0 malloc-65536 65536 2 0 2 0 1 131072 0 malloc-65536 65536 0 2 64 0 1 131072 0 malloc-32768 32768 3 1 14 0 1 131072 0 mbuf_packet 256 4 504 170 0 254 130048 0 S VFS Cache 104 993 177 1113 0 126 121680 0 ksiginfo 112 78 966 128 0 126 116928 0 malloc-128 128 522 253 3841 0 126 99200 0 PROC 1376 57 9 907 0 8 90816 0 UMA Kegs 384 225 8 225 0 30 89472 0 malloc-128 128 446 205 825 0 126 83328 0 filedesc0 1072 58 19 907 0 8 82544 0 g_bio 408 0 180 5119 0 30 73440 0 128 Bucket 1024 43 24 241 0 16 68608 0 malloc-64 64 554 517 11370 0 254 68544 0 malloc-128 128 308 219 1139 0 126 67456 0 malloc-32768 32768 0 2 120 0 1 65536 0 malloc-16384 16384 3 1 193 0 1 65536 0 malloc-4096 4096 13 3 27 0 2 65536 0 malloc-2048 2048 5 27 723 0 8 65536 0 64 Bucket 512 75 53 1737 0 30 65536 0 malloc-256 256 205 50 544 0 62 65280 0 malloc-384 384 91 59 181 0 30 57600 0 malloc-8192 8192 7 0 7 0 1 57344 0 malloc-8192 8192 7 0 8 0 1 57344 0 malloc-64 64 634 185 2376 0 254 52416 0 32 Bucket 256 66 129 701 0 62 49920 0 DIRHASH 1024 34 14 34 0 16 49152 0 NAMEI 1024 0 48 13274 0 16 49152 0 malloc-16384 16384 2 1 3 0 1 49152 0 malloc-8192 8192 4 2 113 0 1 49152 0 malloc-4096 4096 8 4 53 0 2 49152 0 malloc-384 384 88 32 88 0 30 46080 0 syncache 168 0 264 3 0 254 44352 0 pcpu-8 8 4693 427 4884 0 254 40960 0 VMSPACE 584 34 36 885 0 16 40880 0 pipe 736 19 36 291 0 16 40480 0 da_ccb 544 0 70 1378 0 16 38080 0 udp_inpcb 408 7 83 151 0 30 36720 0 malloc-64 64 64 503 13836 0 254 36288 0 malloc-64 64 160 407 185 0 254 36288 0 malloc-64 64 15 552 73 0 254 36288 0 malloc-64 64 14 553 34 0 254 36288 0 malloc-128 128 150 129 674 0 126 35712 0 malloc-128 128 73 206 280 0 126 35712 0 routing nhops 256 26 109 33 0 62 34560 0 ttyoutq 256 72 63 296 0 62 34560 0 malloc-384 384 52 38 53 0 30 34560 0 malloc-256 256 0 135 455 0 62 34560 0 malloc-256 256 18 117 21 0 62 34560 0 malloc-256 256 103 32 461 0 62 34560 0 malloc-256 256 25 110 302 0 62 34560 0 malloc-256 256 26 109 26 0 62 34560 0 malloc-256 256 24 111 332 0 62 34560 0 malloc-8192 8192 3 1 5 0 1 32768 0 malloc-4096 4096 5 3 563 0 2 32768 0 malloc-4096 4096 4 4 8 0 2 32768 0 malloc-2048 2048 2 14 2 0 8 32768 0 malloc-2048 2048 5 11 67 0 8 32768 0 malloc-2048 2048 9 7 9 0 8 32768 0 malloc-2048 2048 12 4 32 0 8 32768 0 malloc-2048 2048 1 15 49 0 8 32768 0 malloc-1024 1024 28 4 32 0 16 32768 0 malloc-1024 1024 9 23 14 0 16 32768 0 malloc-1024 1024 3 29 5 0 16 32768 0 malloc-1024 1024 8 24 12 0 16 32768 0 malloc-1024 1024 0 32 1003 0 16 32768 0 malloc-512 512 4 60 161 0 30 32768 0 malloc-512 512 2 62 22 0 30 32768 0 malloc-512 512 8 56 28 0 30 32768 0 pcpu-64 64 498 14 498 0 254 32768 0 ttyinq 160 135 65 555 0 62 32000 0 PGRP 120 21 243 45 0 126 31680 0 tcp_inpcb 1304 5 19 17 0 8 31296 0 malloc-32 32 277 605 3146 0 254 28224 0 16 Bucket 144 52 144 259 0 62 28224 0 4 Bucket 48 4 584 5 0 254 28224 0 AIO 208 0 133 3 0 62 27664 0 udplite_inpcb 408 0 63 3 0 30 25704 0 TURNSTILE 136 145 44 145 0 62 25704 0 cpuset 200 7 121 7 0 62 25600 0 malloc-4096 4096 3 3 189 0 2 24576 0 malloc-4096 4096 3 3 6 0 2 24576 0 ertt_txseginfo 40 2 604 363 0 254 24240 0 PWD 40 20 586 171 0 254 24240 0 rtentry 168 29 115 33 0 62 24192 0 Files 80 144 156 7134 0 126 24000 0 8 Bucket 80 55 245 603 0 126 24000 0 malloc-384 384 20 40 22 0 30 23040 0 malloc-384 384 36 24 492 0 30 23040 0 malloc-384 384 4 56 32 0 30 23040 0 malloc-384 384 26 34 37 0 30 23040 0 ripcb 376 0 60 11 0 30 22560 0 SLEEPQUEUE 88 145 111 145 0 126 22528 0 clpbuf 2624 0 8 17 0 4 20992 0 hostcache 64 1 314 1 0 254 20160 0 ertt 72 5 275 17 0 126 20160 0 malloc-64 64 3 312 3 0 254 20160 0 malloc-32 32 53 577 244 0 254 20160 0 malloc-32 32 93 537 125 0 254 20160 0 malloc-32 32 144 486 1326 0 254 20160 0 malloc-32 32 140 490 322 0 254 20160 0 malloc-32 32 26 604 1077 0 254 20160 0 2 Bucket 32 52 578 350 0 254 20160 0 KNOTE 160 0 125 14 0 62 20000 0 AIOCB 552 0 35 2 0 16 19320 0 vmem 1856 2 7 2 0 8 16704 0 epoch_record pcpu 256 4 60 4 0 62 16384 0 malloc-16384 16384 1 0 1 0 1 16384 0 malloc-16384 16384 1 0 1 0 1 16384 0 malloc-8192 8192 1 1 20 0 1 16384 0 malloc-2048 2048 4 4 4 0 8 16384 0 malloc-512 512 1 31 1 0 30 16384 0 malloc-512 512 1 31 2 0 30 16384 0 malloc-512 512 2 30 8 0 30 16384 0 malloc-512 512 2 30 9 0 30 16384 0 SMR CPU 32 8 503 8 0 254 16352 0 sctp_laddr 48 0 336 12 0 254 16128 0 malloc-16 16 480 520 3372 0 254 16000 0 kenv 258 17 43 1066 0 30 15480 0 mqnode 416 3 33 3 0 30 14976 0 SMR SHARED 24 8 503 8 0 254 12264 0 malloc-32 32 7 371 7 0 254 12096 0 malloc-16 16 37 713 1562 0 254 12000 0 malloc-16 16 23 727 136 0 254 12000 0 malloc-16 16 17 733 20 0 254 12000 0 malloc-16 16 35 715 24838 0 254 12000 0 malloc-16 16 10 740 10 0 254 12000 0 malloc-16 16 37 713 101 0 254 12000 0 cryptop 280 0 42 1 0 30 11760 0 AIOLIO 272 0 42 1 0 30 11424 0 Mountpoints 2816 2 2 2 0 4 11264 0 malloc-8192 8192 1 0 1 0 1 8192 0 malloc-4096 4096 0 2 3 0 2 8192 0 vtnet_tx_hdr 24 1 333 1592 0 254 8016 0 pcpu-16 16 4 252 4 0 254 4096 0 UMA Slabs 1 176 9 13 9 0 62 3872 0 KMAP ENTRY 96 12 27 14 0 0 3744 0 FFS1 dinode 128 0 0 0 0 126 0 0 ada_ccb 272 0 0 0 0 30 0 0 swblk 136 0 0 0 0 62 0 0 swpctrie 152 0 0 0 0 62 0 0 cdg_qdiffsample 16 0 0 0 0 254 0 0 pf state scrubs 40 0 0 0 0 254 0 0 pf frag entries 40 0 0 0 0 254 0 0 pf fragment node 72 0 0 0 0 126 0 0 pf frags 232 0 0 0 0 62 0 0 pf table entries 160 0 0 0 0 254 0 0 pf table entry counters 64 0 0 0 0 254 0 0 pf UDP mappings 104 0 0 0 0 126 0 0 pf source nodes 152 0 0 0 0 254 0 0 pf state keys 88 0 0 0 0 126 0 0 pf states 384 0 0 0 0 254 0 0 pf tags 104 0 0 0 0 126 0 0 pf mtags 184 0 0 0 0 62 0 0 tcp_bbr_pcb 896 0 0 0 0 16 0 0 tcp_bbr_map 128 0 0 0 0 126 0 0 tcp_rack_pcb 1088 0 0 0 0 8 0 0 tcp_rack_map 128 0 0 0 0 126 0 0 tfo_ccache_entries 80 0 0 0 0 126 0 0 tfo 4 0 0 0 0 254 0 0 sackhole 32 0 0 0 0 254 0 0 ipq 56 0 0 0 0 254 0 0 sctp_asconf_ack 48 0 0 0 0 254 0 0 sctp_asconf 40 0 0 0 0 254 0 0 sctp_stream_msg_out 112 0 0 0 0 254 0 0 sctp_readq 152 0 0 0 0 254 0 0 sctp_chunk 152 0 0 0 0 254 0 0 tcp_log_id_node 120 0 0 0 0 126 0 0 tcp_log_id_bucket 176 0 0 0 0 62 0 0 tcpreass 48 0 0 0 0 254 0 0 IPsec SA lft_c 16 0 0 0 0 254 0 0 itimer 352 0 0 0 0 30 0 0 mqnotifier 216 0 0 0 0 62 0 0 mvdata 64 0 0 0 0 254 0 0 mqueue 248 0 0 0 0 62 0 0 TMPFS node 240 0 0 0 0 62 0 0 NCLNODE 608 0 0 0 0 16 0 0 LTS VFS Cache 360 0 0 0 0 30 0 0 L VFS Cache 320 0 0 0 0 30 0 0 STS VFS Cache 144 0 0 0 0 62 0 0 linux_dma_object 32 0 0 0 0 254 0 0 linux_dma_pctrie 152 0 0 0 0 62 0 0 IOMMU_MAP_ENTRY 112 0 0 0 0 126 0 0 skbuff 1808 0 0 0 0 8 0 0 mbuf_jumbo_16k 16384 0 0 0 0 254 0 0 mbuf_jumbo_9k 9216 0 0 0 0 254 0 0 audit_record 1280 0 0 0 0 8 0 0 domainset 40 0 0 0 0 254 0 0 MAC labels 40 0 0 0 0 254 0 0 vnpbuf 2624 0 0 0 0 16 0 0 nfspbuf 2624 0 0 0 0 4 0 0 swwbuf 2624 0 0 0 0 2 0 0 swrbuf 2624 0 0 0 0 4 0 0 umtx_shm 88 0 0 0 0 126 0 0 umtx pi 96 0 0 0 0 126 0 0 rangeset pctrie nodes 152 0 0 0 0 62 0 0 rl_entry 48 0 0 0 0 254 0 0 malloc-65536 65536 0 0 0 0 1 0 0 malloc-65536 65536 0 0 0 0 1 0 0 malloc-65536 65536 0 0 0 0 1 0 0 malloc-32768 32768 0 0 0 0 1 0 0 malloc-32768 32768 0 0 0 0 1 0 0 malloc-32768 32768 0 0 0 0 1 0 0 malloc-32768 32768 0 0 0 0 1 0 0 malloc-32768 32768 0 0 0 0 1 0 0 malloc-16384 16384 0 0 0 0 1 0 0 malloc-16384 16384 0 0 0 0 1 0 0 malloc-16384 16384 0 0 0 0 1 0 0 malloc-8192 8192 0 0 0 0 1 0 0 malloc-8192 8192 0 0 0 0 1 0 0 malloc-1024 1024 0 0 0 0 16 0 0 malloc-1024 1024 0 0 0 0 16 0 0 malloc-512 512 0 0 0 0 30 0 0 malloc-128 128 0 0 0 0 126 0 0 pcpu-32 32 0 0 0 0 254 0 0 pcpu-4 4 0 0 0 0 254 0 0 fakepg 88 0 0 0 0 126 0 0 UMA Hash 256 0 0 0 0 62 0 0 db> show all tcpcbs/bl