panic: sleeping thread holds tun_mtx cpuid = 0 time = 9 KDB: stack backtrace: db_trace_self_wrapper() at db_trace_self_wrapper+0xc6/frame 0xfffffe0056fa7510 kdb_backtrace() at kdb_backtrace+0xd0/frame 0xfffffe0056fa7670 vpanic() at vpanic+0x257/frame 0xfffffe0056fa7830 panic() at panic+0xb5/frame 0xfffffe0056fa78f0 propagate_priority() at propagate_priority+0x521/frame 0xfffffe0056fa7940 turnstile_wait() at turnstile_wait+0x663/frame 0xfffffe0056fa79b0 __mtx_lock_sleep() at __mtx_lock_sleep+0x452/frame 0xfffffe0056fa7af0 __mtx_lock_flags() at __mtx_lock_flags+0x1fe/frame 0xfffffe0056fa7be0 tunstart_l2() at tunstart_l2+0x74/frame 0xfffffe0056fa7c40 tap_transmit() at tap_transmit+0x1d0/frame 0xfffffe0056fa7c90 ether_output_frame() at ether_output_frame+0x30c/frame 0xfffffe0056fa7d50 ether_output() at ether_output+0x114b/frame 0xfffffe0056fa7ec0 ip6_output_send() at ip6_output_send+0x2f9/frame 0xfffffe0056fa7f40 ip6_output() at ip6_output+0x44de/frame 0xfffffe0056fa8470 rip6_send() at rip6_send+0xe3d/frame 0xfffffe0056fa8770 sosend_generic_locked() at sosend_generic_locked+0xc62/frame 0xfffffe0056fa8940 sosend_generic() at sosend_generic+0x87/frame 0xfffffe0056fa89a0 sousrsend() at sousrsend+0x112/frame 0xfffffe0056fa8a30 kern_sendit() at kern_sendit+0x4fe/frame 0xfffffe0056fa8b90 sendit() at sendit+0x15f/frame 0xfffffe0056fa8bf0 sys_sendto() at sys_sendto+0x181/frame 0xfffffe0056fa8d10 amd64_syscall() at amd64_syscall+0x4e2/frame 0xfffffe0056fa8f30 fast_syscall_common() at fast_syscall_common+0xf8/frame 0xfffffe0056fa8f30 --- syscall (198, FreeBSD ELF64, __syscall), rip = 0x3a52ba, rsp = 0x826421f08, rbp = 0x826421f80 --- KDB: enter: panic [ thread pid 998 tid 100408 ] Stopped at kdb_enter+0x6e: movq $0,0x259f767(%rip) db> db> set $lines = 0 db> set $maxwidth = 0 db> show registers cs 0x20 ds 0x3b es 0x3b fs 0x13 gs 0x1b ss 0x28 rax 0x12 rcx 0xfffffe0075200000 rdx 0x7ffff rbx 0xffffffff82804a80 .str.27 rsp 0xfffffe0056fa7650 rbp 0xfffffe0056fa7670 rsi 0x80001 rdi 0xffffffff816418f9 printf+0x149 r8 0 r9 0xffffffff r10 0x6 r11 0x5 r12 0xfffffe0054121780 r13 0xfffffffffffffffe r14 0xffffffff82804a80 .str.27 r15 0 rip 0xffffffff8162b42e kdb_enter+0x6e rflags 0x46 kdb_enter+0x6e: movq $0,0x259f767(%rip) db> show proc Process 998 (syz-executor) at 0xfffffe0054130568: state: NORMAL uid: 0 gid: 0 supp gids: 0, 5 parent: pid 766 at 0xfffffe00540de558 ABI: FreeBSD ELF64 flag: 0x10000080 flag2: 0 arguments: ./syz-executor exec reaper: 0xfffffe0007809010 reapsubtree: 1 sigparent: 20 vmspace: 0xfffffe005410ddb0 (map 0xfffffe005410ddb0) (map.pmap 0xfffffe005410de50) (pmap 0xfffffe005410dec0) threads: 2 100407 RunQ syz-executor 100408 L *tun_mtx 0xfffffe000782de40 syz-executor db> ps pid ppid pgrp uid state wmesg wchan cmd 998 766 766 0 R (threaded) syz-executor 100407 RunQ syz-executor 100408 L *tun_mtx 0xfffffe000782de40 syz-executor 994 1 766 0 S uwait 0xfffffe006e5a7200 syz-executor 986 1 766 0 S uwait 0xfffffe00584e9500 syz-executor 983 1 766 0 S uwait 0xfffffe006e5a8180 syz-executor 978 1 766 0 S uwait 0xfffffe0057d94480 syz-executor 977 1 766 0 S uwait 0xfffffe006e5a4b00 syz-executor 972 764 764 0 T (threaded) syz-executor 100353 s syz-executor 100367 D in6_mul 0xffffffff83cefbe0 syz-executor 968 1 968 0 SVs uwait 0xfffffe0057d91200 syz-executor 964 1 764 0 S uwait 0xfffffe006e5a5680 syz-executor 958 762 958 0 D tun_ioc 0xffffffff83ccba40 syz-executor 955 1 945 0 S uwait 0xfffffe006e5a5380 syz-executor 953 1 764 0 S uwait 0xfffffe006dfc3a00 syz-executor 950 1 764 0 S uwait 0xfffffe006dfc4600 syz-executor 949 1 764 0 S uwait 0xfffffe006dfc4500 syz-executor 947 1 766 0 S uwait 0xfffffe006e5a7800 syz-executor 942 765 765 60928 T (threaded) syz-executor 100117 s syz-executor 100309 s syz-executor 100314 D tun_ioc 0xffffffff83ccba40 syz-executor 100318 D in6_mul 0xffffffff83cefbe0 syz-executor 100324 s syz-executor 935 1 764 0 S uwait 0xfffffe0057d01580 syz-executor 932 1 766 0 S uwait 0xfffffe006e5a7a00 syz-executor 927 778 423 0 L *tun_mtx 0xfffffe000782de40 ifconfig 926 921 763 0 S uwait 0xfffffe00584e9a80 syz-executor 921 1 763 0 T (threaded) syz-executor 100197 s syz-executor 100258 D tun_ioc 0xffffffff83ccba40 syz-executor 913 1 766 0 S uwait 0xfffffe006dfc4300 syz-executor 911 1 764 0 S uwait 0xfffffe00584e9b00 syz-executor 902 1 764 0 S uwait 0xfffffe006dfc5500 syz-executor 901 1 764 0 S uwait 0xfffffe0057d91300 syz-executor 894 1 765 0 S uwait 0xfffffe006e5a8580 syz-executor 881 1 765 0 SV uwait 0xfffffe0057d94780 syz-executor 856 1 766 0 SV select 0xfffffe00586131c0 syz-executor 850 0 0 0 DL (threaded) [KTLS] 100136 D - 0xfffffe0053e15600 [thr_0] 100160 D - 0xfffffe0053e15680 [thr_1] 100161 D - 0xffffffff83cbd628 [reclaim_0] 824 1 764 0 S uwait 0xfffffe00584e9200 syz-executor 820 1 766 0 S uwait 0xfffffe00584e9100 syz-executor 811 0 0 0 DL aiordy 0xfffffe00540c9000 [aiod4] 810 0 0 0 DL aiordy 0xfffffe00540c9558 [aiod3] 809 0 0 0 DL aiordy 0xfffffe00540c9ab0 [aiod2] 808 0 0 0 DL aiordy 0xfffffe00540e0010 [aiod1] 778 769 423 0 S wait 0xfffffe00540cc018 sh 769 423 423 0 S wait 0xfffffe00540e0ac0 sh 766 762 766 0 S nanslp 0xffffffff83babc40 syz-executor 765 762 765 0 S wait 0xfffffe00540e1570 syz-executor 764 762 764 0 S wait 0xfffffe00540cbac0 syz-executor 762 760 760 0 R syz-executor 760 758 760 0 Ss sigsusp 0xfffffe00540cb618 csh 758 681 758 0 Ss select 0xfffffe0057ddb9c0 sshd 747 1 747 0 Ss+ ttyin 0xfffffe0058279cb0 getty 746 1 746 0 Ss+ ttyin 0xfffffe005862f0b0 getty 745 1 745 0 Ss+ ttyin 0xfffffe005862f4b0 getty 744 1 744 0 Ss+ ttyin 0xfffffe005862f8b0 getty 743 1 743 0 Ss+ ttyin 0xfffffe005862fcb0 getty 742 1 742 0 Ss+ ttyin 0xfffffe00586300b0 getty 741 1 741 0 Ss+ ttyin 0xfffffe00586304b0 getty 740 1 740 0 Ss+ ttyin 0xfffffe00586308b0 getty 739 1 739 0 Ss+ ttyin 0xfffffe0058630cb0 getty 737 1 17 0 S+ piperd 0xfffffe00598358a0 logger 736 735 17 0 S+ nanslp 0xffffffff83babc41 sleep 735 1 17 0 S+ wait 0xfffffe00540de000 sh 685 1 685 0 Ss nanslp 0xffffffff83babc40 cron 681 1 681 0 Ss select 0xfffffe0057ddbec0 sshd 494 1 494 0 Ss select 0xfffffe0057de00c0 syslogd 423 1 423 0 Ss wait 0xfffffe005400c010 devd 422 1 422 65 Ss select 0xfffffe0057ddbcc0 dhclient 337 1 337 0 Ss select 0xfffffe0057ddbdc0 dhclient 334 1 334 0 Ss select 0xfffffe0058613740 dhclient 16 0 0 0 DL syncer 0xffffffff83cc9820 [syncer] 15 0 0 0 DL vlruwt 0xfffffe0054002558 [vnlru] 14 0 0 0 RL (threaded) [bufdaemon] 100079 D psleep 0xffffffff83cc7d60 [bufdaemon] 100080 RunQ [bufspacedaemon-0] 100094 D sdflush 0xfffffe005862dce8 [/ worker] 9 0 0 0 DL psleep 0xffffffff83d12cc0 [vmdaemon] 8 0 0 0 DL (threaded) [pagedaemon] 100077 D psleep 0xffffffff83cf8d88 [dom0] 100081 D launds 0xffffffff83cf8d94 [laundry: dom0] 100082 D umarcl 0xffffffff81e11220 [uma] 7 0 0 0 DL - 0xffffffff839245d8 [rand_harvestq] 6 0 0 0 DL pftm 0xffffffff844d8f20 [pf purge] 5 0 0 0 DL waiting 0xffffffff84907700 [sctp_iterator] 4 0 0 0 DL (threaded) [cam] 100045 D - 0xffffffff838ee340 [doneq0] 100046 D - 0xffffffff838ee2c0 [async] 100075 D - 0xffffffff838ee140 [scanner] 3 0 0 0 DL (threaded) [crypto] 100042 D crypto_ 0xffffffff83cf4660 [crypto] 100043 D crypto_ 0xfffffe0053ee4d30 [crypto returns 0] 100044 D crypto_ 0xfffffe0053ee4d80 [crypto returns 1] 13 0 0 0 DL (threaded) [geom] 100037 D - 0xffffffff83b54640 [g_event] 100038 D - 0xffffffff83b54660 [g_up] 100039 D - 0xffffffff83b54680 [g_down] 2 0 0 0 WL (threaded) [clock] 100031 I [clock (0)] 100032 I [clock (1)] 12 0 0 0 WL (threaded) [intr] 100013 I [swi6: task queue] 100014 I [swi6: Giant taskq] 100016 I [swi5: fast taskq] 100033 I [swi1: netisr 0] 100034 I [swi1: hpts] 100035 I [swi1: hpts] 100047 I [irq24: virtio_pci0] 100048 I [irq25: virtio_pci0] 100049 I [irq26: virtio_pci0] 100050 I [irq27: virtio_pci0] 100051 I [irq28: virtio_pci1] 100052 I [irq29: virtio_pci1] 100053 I [irq30: virtio_pci1] 100054 I [irq31: virtio_pci1] 100055 I [irq32: virtio_pci1] 100060 I [irq10: virtio_pci2] 100062 I [irq1: atkbd0] 100063 I [irq12: psm0] 100064 I [swi0: uart uart++] 100068 I [swi1: pf send] 11 0 0 0 RL (threaded) [idle] 100003 CanRun [idle: cpu0] 100004 CanRun [idle: cpu1] 1 0 1 0 SLs wait 0xfffffe0007809010 [init] 10 0 0 0 DL audit_w 0xffffffff83cf5100 [audit] 0 0 0 0 RLs (threaded) [kernel] 100000 D parked 0xffffffff84c48ff0 [swapper] 100005 D - 0xfffffe0053ea0100 [softirq_0] 100006 D - 0xfffffe0053ea0000 [softirq_1] 100007 D - 0xfffffe0053e9fe00 [if_io_tqg_0] 100008 D - 0xfffffe0053e9fd00 [if_io_tqg_1] 100009 D - 0xfffffe0053e9fc00 [if_config_tqg_0] 100010 D - 0xfffffe000776fb00 [kqueue_ctx taskq] 100011 D - 0xfffffe000776fa00 [jail_remove taskq] 100012 D - 0xfffffe000776f900 [bus taskq] 100015 D - 0xfffffe000776f600 [thread taskq] 100017 D - 0xfffffe000776f400 [aiod_kick taskq] 100018 D - 0xfffffe000776f300 [deferred_unmount ta] 100019 D - 0xfffffe000776f200 [inm_free taskq] 100020 D - 0xfffffe000776f100 [in6m_free taskq] 100021 D - 0xfffffe000776f000 [linuxkpi_irq_wq] 100022 D - 0xfffffe000776ee00 [linuxkpi_short_wq_0] 100023 D - 0xfffffe000776ee00 [linuxkpi_short_wq_1] 100024 D - 0xfffffe000776ee00 [linuxkpi_short_wq_2] 100025 D - 0xfffffe000776ee00 [linuxkpi_short_wq_3] 100026 D - 0xfffffe000776ed00 [linuxkpi_long_wq_0] 100027 D - 0xfffffe000776ed00 [linuxkpi_long_wq_1] 100028 D - 0xfffffe000776ed00 [linuxkpi_long_wq_2] 100029 D - 0xfffffe000776ed00 [linuxkpi_long_wq_3] 100036 D - 0xfffffe000776eb00 [firmware taskq] 100040 D - 0xfffffe000776ea00 [crypto_0] 100041 D - 0xfffffe000776ea00 [crypto_1] 100056 D - 0xfffffe0057de6e00 [vtnet0 rxq 0] 100057 D - 0xfffffe0057de6d00 [vtnet0 txq 0] 100058 D - 0xfffffe0057de6c00 [vtnet0 rxq 1] 100059 D - 0xfffffe0057de6b00 [vtnet0 txq 1] 100061 D vtbslp 0xfffffe0057de0380 [virtio_balloon] 100065 Run CPU 1 [deadlkres] 100069 D - 0xfffffe00593fd500 [acpi_task_0] 100070 D - 0xfffffe00593fd500 [acpi_task_1] 100071 D - 0xfffffe00593fd500 [acpi_task_2] 100073 D - 0xfffffe000776fc00 [mca taskq] 100074 D - 0xfffffe000776e800 [CAM taskq] 100076 D - 0xfffffe0007771600 [ipsec_offload] db> show all locks Process 998 (syz-executor) thread 0xfffffe0054121780 (100408) exclusive rw rawinp (rawinp) r = 0 (0xfffffe006dd9c4a0) locked @ /syzkaller/managers/main/kernel/sys/netinet6/raw_ip6.c:402 exclusive sx so_snd_sx (so_snd_sx) r = 0 (0xfffffe0059af2580) locked @ /syzkaller/managers/main/kernel/sys/kern/uipc_socket.c:4835 Process 958 (syz-executor) thread 0xfffffe0054121000 (100173) shared lockmgr devfs (devfs) r = 0 (0xfffffe0059654908) locked @ /syzkaller/managers/main/kernel/sys/kern/vfs_subr.c:6970 Process 942 (syz-executor) thread 0xfffffe005413c780 (100314) exclusive sx in6_multi_sx (in6_multi_sx) r = 0 (0xffffffff83cefbe0) locked @ /syzkaller/managers/main/kernel/sys/netinet6/in6_mcast.c:2015 Process 927 (ifconfig) thread 0xfffffe0054127000 (100163) exclusive sx tun_ioctl (tun_ioctl) r = 0 (0xffffffff83ccba40) locked @ /syzkaller/managers/main/kernel/sys/net/if_tuntap.c:1387 Process 921 (syz-executor) thread 0xfffffe0054119780 (100258) exclusive sleep mutex tun_mtx (tun_mtx) r = 0 (0xfffffe0059694988) locked @ /syzkaller/managers/main/kernel/sys/net/if_tuntap.c:1688 Process 0 (kernel) thread 0xfffffe00079fd780 (100065) exclusive sleep mutex process lock (process lock) r = 0 (0xfffffe0054130690) locked @ /syzkaller/managers/main/kernel/sys/kern/kern_clock.c:265 shared sx allproc (allproc) r = 0 (0xffffffff83001980) locked @ /syzkaller/managers/main/kernel/sys/kern/kern_clock.c:255 db> show malloc Type InUse MemUse Requests pf_hash 6 12804K 6 linker 377 5124K 495 tcp_hpts 7 4801K 7 devbuf 4187 4324K 4212 sysctloid 35338 2082K 35413 vtbuf 24 1968K 46 kobj 331 1324K 495 newblk 18 1029K 1440 vfscache 3 1025K 3 pcb 49 691K 146 inodedep 18 519K 356 ufs_quota 1 512K 1 vfs_hash 1 512K 1 callout 2 512K 2 filedesc 64 505K 325 intr 4 472K 4 subproc 179 350K 1101 vnet_data 2 224K 2 acpitask 1 224K 1 KTRACE 100 200K 10471 acpica 1674 184K 54444 vmem 5 144K 7 tidhash 3 141K 3 pagedep 10 131K 156 tfo_ccache 1 128K 1 IP reass 1 128K 1 DEVFS1 107 107K 126 sem 4 106K 4 gtaskqueue 18 98K 18 bus 1000 82K 5086 mtx_pool 3 74K 3 syncache 1 68K 1 NFSD srvcache 3 68K 3 module 523 66K 523 ddb_capture 1 64K 1 umtx 368 46K 368 kdtrace 243 46K 1410 hostcache 1 32K 1 shm 1 32K 2 DEVFS3 126 32K 136 msg 4 30K 4 kbdmux 6 28K 6 temp 33 21K 1938 DEVFS_RULE 56 20K 56 ifaddr 73 20K 75 BPF 14 19K 26 ufs_mount 4 17K 5 proc 3 17K 3 LRO 16 17K 16 tty 16 16K 16 routetbl 124 16K 395 ithread 90 15K 90 bus-sc 34 15K 1657 lltable 47 15K 48 ifnet 8 15K 8 eventhandler 163 14K 163 ether_multi 162 13K 172 kenv 95 12K 95 shmfd 7 11K 13 GEOM 49 11K 431 CAM queue 5 11K 1528 cred 26 10K 209 rman 82 10K 437 plimit 25 10K 330 rpc 8 9K 8 in6_multi 65 9K 65 bmsafemap 2 9K 286 devstat 4 9K 4 UART 12 9K 12 ksem 1 8K 2 kqueue 92 8K 1241 pfs_vncache 1 8K 1 audit_evclass 240 8K 306 taskqueue 69 8K 87 netlink 3 7K 77 sglist 6 7K 6 CAM DEV 3 6K 510 pfs_nodes 22 6K 22 pwddesc 79 5K 1039 sctp_atcl 13 5K 36 ufs_dirhash 24 5K 30 UMA 270 5K 270 pf_ifnet 10 5K 20 vt 11 5K 11 memdesc 1 4K 1 MCA 32 4K 32 evdev 4 4K 4 crypto 12 4K 34 freework 15 4K 321 acpisem 28 4K 28 DEVFSP 55 4K 117 proc-args 105 3K 2048 kcovinfo 45 3K 45 terminal 11 3K 11 session 21 3K 36 selfd 41 3K 35465 acpidev 20 3K 20 uidinfo 4 3K 14 hhook 8 3K 10 clone 9 3K 9 sctp_stro 2 2K 2 local_apic 1 2K 1 io_apic 1 2K 1 dirrem 8 2K 235 ipsec-saq 2 2K 2 ip6ndp 13 2K 15 tun 5 2K 5 in_multi 7 2K 9 Unitno 29 2K 63 sctp_ifa 13 2K 14 CC Mem 13 2K 130 CAM XPT 22 2K 543 lockf 15 2K 32 toponodes 6 2K 6 ipsecpolicy 2 2K 2 inpcbpolicy 39 2K 367 msi 9 2K 9 select 9 2K 43 softdep 1 1K 1 sahead 1 1K 1 secasvar 1 1K 1 nhops 6 1K 8 vnodemarker 2 1K 156 ioctlops 1 1K 110 NFSD session 1 1K 1 mount 19 1K 326 freefile 7 1K 184 mld 7 1K 7 igmp 7 1K 7 CAM periph 4 1K 271 sctp_ifn 6 1K 14 ipsec 3 1K 3 pfil 6 1K 6 isadev 6 1K 6 osd 18 1K 147 pci_link 10 1K 10 newdirblk 5 1K 126 encap_export_host 12 1K 12 sctp_atky 15 1K 39 cryptodev 9 1K 123 ip6opt 4 1K 20 sctp_stri 1 1K 2 diradd 4 1K 252 freeblks 2 1K 155 cdev 2 1K 2 lkpikmalloc 8 1K 9 counter_rate 13 1K 13 mkdir 3 1K 252 chacha20random 1 1K 1 biobuf 1 1K 1 ktls 3 1K 19 indirdep 1 1K 209 frag6 3 1K 6 inotify 1 1K 8 vnodes 1 1K 2 NFSD lckfile 1 1K 1 NFSD V4client 1 1K 1 DEVFS 9 1K 10 CAM SIM 2 1K 2 feeder 7 1K 7 sctp_athm 13 1K 37 tcpfunc 3 1K 3 loginclass 3 1K 5 prison 6 1K 6 soname 5 1K 3436 nexusdev 8 1K 8 apmdev 1 1K 1 atkbddev 2 1K 2 freefrag 1 1K 103 ktls_ocf 1 1K 3 ip6_msource 2 1K 2 in6_mfilter 2 1K 5 ip_msource 2 1K 2 VN POLL 1 1K 4 aio 4 1K 4 procdesc 1 1K 10 pmchooks 1 1K 1 CAM path 4 1K 1034 CAM dev queue 2 1K 2 CAM I/O Scheduler 1 1K 1 filecaps 4 1K 85 ip6_moptions 3 1K 4 in_mfilter 2 1K 5 sctp_vrf 1 1K 1 sctp_map 4 1K 4 ip_moptions 1 1K 3 vnet 1 1K 1 pmc 1 1K 1 filedesc_to_leader 1 1K 3 entropy 2 1K 34 acpiintr 1 1K 1 cpus 2 1K 2 vnet_data_free 1 1K 1 Per-cpu 1 1K 1 p1003.1b 1 1K 1 sctp_mcore 0 0K 0 sctp_socko 0 0K 4 sctp_iter 0 0K 12 sctp_mvrf 0 0K 0 sctp_timw 0 0K 0 sctp_cpal 0 0K 0 sctp_cmsg 0 0K 0 sctp_stre 0 0K 0 sctp_athi 0 0K 0 sctp_a_it 0 0K 12 sctp_aadr 0 0K 0 mqdata 0 0K 0 tcp_pcm_rack 0 0K 0 tcp_do_rack 0 0K 0 tcp_fsb_rack 0 0K 0 filemon 0 0K 2 pf_table 0 0K 0 pf 0 0K 0 pf_rule 0 0K 0 pf_altq 0 0K 0 pf_osfp 0 0K 0 pf_krule_item 0 0K 0 pf_temp 0 0K 0 ipcomp 0 0K 0 esp 0 0K 0 ah 0 0K 0 madt_table 0 0K 2 smartpqi 0 0K 0 ixl 0 0K 0 ice-resmgr 0 0K 0 ice-osdep 0 0K 0 ice 0 0K 0 iavf 0 0K 0 axgbe 0 0K 0 fpukern_ctx 0 0K 0 xen_intr 0 0K 0 xen_hvm 0 0K 0 legacydrv 0 0K 0 NMI handlers 0 0K 0 bounce 0 0K 0 busdma 0 0K 0 qpidrv 0 0K 0 dmar_idpgtbl 0 0K 0 dmar_dom 0 0K 0 dmar_ctx 0 0K 0 amdiommu_dom 0 0K 0 amdiommu_ctx 0 0K 0 isci 0 0K 0 iommu_dmamap 0 0K 0 hyperv_socket 0 0K 0 bxe_ilt 0 0K 0 aesni_data 0 0K 0 xenbus 0 0K 0 vm_fictitious 0 0K 0 UMAHash 0 0K 0 vm_pgdata 0 0K 0 jblocks 0 0K 0 savedino 0 0K 211 sentinel 0 0K 0 jfsync 0 0K 0 jtrunc 0 0K 0 sbdep 0 0K 57 jsegdep 0 0K 0 jseg 0 0K 0 jfreefrag 0 0K 0 jfreeblk 0 0K 0 jnewblk 0 0K 0 jmvref 0 0K 0 jremref 0 0K 0 jaddref 0 0K 0 freedep 0 0K 0 allocindir 0 0K 0 allocdirect 0 0K 0 ufs_trim 0 0K 0 mactemp 0 0K 0 audit_trigger 0 0K 0 audit_pipe_presel 0 0K 0 audit_pipeent 0 0K 0 audit_pipe 0 0K 0 audit_evname 0 0K 0 audit_bsm 0 0K 0 audit_gidset 0 0K 0 audit_text 0 0K 0 audit_path 0 0K 0 audit_data 0 0K 0 audit_cred 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5E_TLS_RX 0 0K 0 MLX5EEPROM 0 0K 0 MLX5E_TLS 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EN 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5DUMP 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 simple_attr 0 0K 0 seq_file 0 0K 0 lkpiskb 0 0K 0 radix 0 0K 0 idr 0 0K 0 lkpindev 0 0K 0 lkpimhi 0 0K 0 lkpifw 0 0K 0 lkpi80211 0 0K 0 NLM 0 0K 0 ipsec-spdcache 0 0K 0 ipsec-reg 0 0K 0 ipsec-misc 0 0K 0 ipsecrequest 0 0K 0 tcplog 0 0K 0 tcp_hwpace 0 0K 0 ipid 0 0K 0 80211scan 0 0K 0 80211ratectl 0 0K 0 80211power 0 0K 0 80211nodeie 0 0K 0 80211node 0 0K 0 80211mesh_gt 0 0K 0 80211mesh_rt 0 0K 0 80211perr 0 0K 0 80211prep 0 0K 0 80211preq 0 0K 0 80211dfs 0 0K 0 80211crypto 0 0K 0 80211vap 0 0K 0 iflib 0 0K 0 vlan 0 0K 0 gif 0 0K 0 ifdescr 0 0K 0 zlib 0 0K 19 fadvise 0 0K 2 statfs 0 0K 189 namei_tracker 0 0K 1 export_host 0 0K 0 cl_savebuf 0 0K 38 lio 0 0K 4 acl 0 0K 0 mbuf_tag 0 0K 0 accf 0 0K 0 pts 0 0K 0 timerfd 0 0K 0 iov 0 0K 14081 eventfd 0 0K 1 Witness 0 0K 0 stack 0 0K 0 sbuf 0 0K 218 firmware 0 0K 0 compressor 0 0K 0 SWAP 0 0K 0 sysctltmp 0 0K 635 sysctl 0 0K 3 ekcd 0 0K 0 dumper 0 0K 0 sendfile 0 0K 132 rctl 0 0K 0 cache 0 0K 0 jaildesc 0 0K 0 prison_racct 0 0K 0 Fail Points 0 0K 0 sigio 0 0K 2 pwd 0 0K 0 tty console 0 0K 0 boottrace 0 0K 0 isofs_node 0 0K 0 isofs_mount 0 0K 0 tr_raid5_data 0 0K 0 tr_raid1e_data 0 0K 0 tr_raid1_data 0 0K 0 tr_raid0_data 0 0K 0 tr_concat_data 0 0K 0 md_sii_data 0 0K 0 md_promise_data 0 0K 0 md_nvidia_data 0 0K 0 md_jmicron_data 0 0K 0 md_intel_data 0 0K 0 md_ddf_data 0 0K 0 raid_data 0 0K 72 geom_flashmap 0 0K 0 tmpfs dir 0 0K 0 tmpfs name 0 0K 0 tmpfs mount 0 0K 0 tmpfs extattr 0 0K 0 NFS FHA 0 0K 0 newnfsmnt 0 0K 0 newnfsclient_req 0 0K 0 NFSCL layrecall 0 0K 0 NFSCL session 0 0K 0 NFSCL sockreq 0 0K 0 NFSCL devinfo 0 0K 0 NFSCL flayout 0 0K 0 NFSCL layout 0 0K 0 NFSD rollback 0 0K 0 NFSCL diroff 0 0K 0 NEWNFSnode 0 0K 0 NFSCL lck 0 0K 0 NFSCL lckown 0 0K 0 NFSCL client 0 0K 0 NFSCL deleg 0 0K 0 NFSCL open 0 0K 0 NFSCL owner 0 0K 0 NFS fh 0 0K 0 NFS req 0 0K 0 NFSD usrgroup 0 0K 0 NFSD string 0 0K 0 NFSD V4lock 0 0K 0 NFSD V4state 0 0K 0 msdosfs_fat 0 0K 0 msdosfs_mount 0 0K 0 msdosfs_node 0 0K 0 DEVFS4 0 0K 0 DEVFS2 0 0K 0 gntdev 0 0K 0 privcmd_dev 0 0K 0 evtchn_dev 0 0K 0 xenstore 0 0K 0 xnb 0 0K 0 xen_acpi 0 0K 0 xbbd 0 0K 0 xbd 0 0K 0 Balloon 0 0K 0 sysmouse 0 0K 0 vtfont 0 0K 0 pvscsi 0 0K 0 USBdev 0 0K 0 USB 0 0K 0 ufshci 0 0K 0 twsbuf 0 0K 0 tcp_log_dev 0 0K 0 midi buffers 0 0K 0 mixer 0 0K 0 ac97 0 0K 0 hdacc 0 0K 0 hdac 0 0K 0 hdaa 0 0K 0 SIIS driver 0 0K 0 PUC 0 0K 0 ppbusdev 0 0K 0 sr_iov 0 0K 0 OCS 0 0K 0 OCS 0 0K 0 nvme 0 0K 0 nvd 0 0K 0 netmap 0 0K 0 mwldev 0 0K 0 MVS driver 0 0K 0 mpi3mrbuf 0 0K 0 mrsasbuf 0 0K 0 mpt_user 0 0K 0 mps_user 0 0K 0 MPSSAS 0 0K 0 mps 0 0K 0 mpr_user 0 0K 0 MPRSAS 0 0K 0 mpr 0 0K 0 mfibuf 0 0K 0 md_sectors 0 0K 0 md_disk 0 0K 1 malodev 0 0K 0 LED 0 0K 0 ix_sriov 0 0K 0 ix 0 0K 0 ipsbuf 0 0K 0 ciss_data 0 0K 0 BACKLIGHT 0 0K 0 ath_hal 0 0K 0 athdev 0 0K 0 ata_pci 0 0K 0 ata_dma 0 0K 0 ata_generic 0 0K 0 AHCI driver 0 0K 0 agp 0 0K 0 acpipwr 0 0K 0 acpi_perf 0 0K 0 acpicmbat 0 0K 0 aacraidcam 0 0K 0 aacraid_buf 0 0K 0 aaccam 0 0K 0 aacbuf 0 0K 0 zstd 0 0K 0 XZ_DEC 0 0K 0 nvlist 0 0K 0 SCSI ENC 0 0K 0 SCSI sa 0 0K 0 scsi_pass 0 0K 0 scsi_da 0 0K 70 ata_da 0 0K 0 scsi_ch 0 0K 0 scsi_cd 0 0K 0 nvme_da 0 0K 0 CAM CCB 0 0K 523 CAM ccb queue 0 0K 0 db> show uma Zone Size Used Free Requests Sleeps Bucket Total Mem XFree mbuf_jumbo_page 4096 8336 1062 13479 0 254 38494208 0 mbuf 256 8679 983 17992 0 254 2473472 0 BUF TRIE 152 231 11573 1241 0 62 1794208 0 malloc-384 384 4140 30 4155 0 30 1601280 0 malloc-128 128 11618 100 11635 0 126 1499904 0 malloc-4096 4096 334 4 605 0 2 1384448 0 tcp_log 416 2477 826 7783 0 254 1374048 0 RADIX NODE 152 8134 389 35684 0 62 1295496 0 UMA Slabs 0 112 11010 0 11010 0 126 1233120 0 sctp_asoc 2256 2 508 2 0 254 1150560 0 malloc-64 64 669 16908 39127 0 254 1124928 0 mbuf_cluster 2048 508 0 508 0 254 1040384 0 malloc-65536 65536 13 2 17 0 1 983040 0 vmem btag 56 16943 88 16943 0 254 953736 0 FFS inode 1168 545 29 734 0 8 670432 0 sctp_ep 1152 10 501 33 0 254 588672 0 malloc-16384 16384 31 2 158 0 1 540672 0 socket 1024 68 440 1651 0 254 520192 0 lkpicurr 168 2 3094 2 0 62 520128 0 pbuf 2624 0 182 0 0 2 477568 0 256 Bucket 2048 195 5 1074 0 8 409600 0 sctp_raddr 736 2 515 2 0 254 380512 0 malloc-4096 4096 84 4 1004 0 2 360448 0 VM OBJECT 248 1363 77 15423 0 62 357120 0 THREAD 1860 163 21 408 0 8 342240 0 VNODE 440 585 108 776 0 30 304920 0 malloc-2048 2048 117 11 398 0 8 262144 0 malloc-64 64 3884 211 3893 0 254 262080 0 malloc-16 16 14564 436 14637 0 254 240000 0 DEVCTL 1024 23 197 148 0 0 225280 0 MAP ENTRY 96 1961 181 49376 0 126 205632 0 UMA Zones 768 242 2 242 0 16 187392 0 malloc-32 32 5351 319 5397 0 254 181440 0 FPU_save_area 832 165 51 517 0 16 179712 0 malloc-128 128 1225 170 25700 0 126 178560 0 lkpimm 56 1 3095 1 0 254 173376 0 unpcb 320 18 498 1216 0 254 165120 0 malloc-32768 32768 0 5 14 0 1 163840 0 malloc-32768 32768 0 5 159 0 1 163840 0 FFS2 dinode 256 545 85 734 0 62 161280 0 malloc-1024 1024 120 24 326 0 16 147456 0 malloc-256 256 86 484 1652 0 62 145920 0 PROC 1368 78 21 1000 0 8 135432 0 S VFS Cache 104 999 288 1225 0 126 133848 0 malloc-65536 65536 0 2 67 0 1 131072 0 malloc-65536 65536 0 2 110 0 1 131072 0 malloc-65536 65536 2 0 2 0 1 131072 0 malloc-65536 65536 1 1 9 0 1 131072 0 malloc-32768 32768 2 2 14 0 1 131072 0 malloc-128 128 775 248 4413 0 126 130944 0 mbuf_packet 256 1 507 378 0 254 130048 0 ksiginfo 112 78 966 133 0 126 116928 0 malloc-128 128 615 160 1587 0 126 99200 0 malloc-32768 32768 3 0 3 0 1 98304 0 malloc-2048 2048 6 42 1039 0 8 98304 0 g_bio 408 0 240 9404 0 30 97920 0 filedesc0 1072 79 12 1039 0 8 97552 0 malloc-8192 8192 9 2 93 0 1 90112 0 UMA Kegs 384 228 5 228 0 30 89472 0 tcp_inpcb 1304 13 53 130 0 8 86064 0 malloc-16384 16384 5 0 6 0 1 81920 0 malloc-8192 8192 8 2 11 0 1 81920 0 malloc-4096 4096 15 5 221 0 2 81920 0 malloc-256 256 148 167 1331 0 62 80640 0 malloc-256 256 163 152 881 0 62 80640 0 malloc-256 256 209 106 507 0 62 80640 0 128 Bucket 1024 46 21 248 0 16 68608 0 malloc-64 64 831 240 2672 0 254 68544 0 malloc-32768 32768 0 2 120 0 1 65536 0 malloc-32768 32768 2 0 2 0 1 65536 0 malloc-16384 16384 1 3 191 0 1 65536 0 malloc-4096 4096 12 4 496 0 2 65536 0 malloc-1024 1024 26 38 557 0 16 65536 0 64 Bucket 512 95 33 2148 0 30 65536 0 32 Bucket 256 83 172 815 0 62 65280 0 malloc-384 384 61 89 732 0 30 57600 0 malloc-384 384 106 44 106 0 30 57600 0 udp_inpcb 408 15 111 191 0 30 51408 0 DIRHASH 1024 34 14 38 0 16 49152 0 NAMEI 1024 1 47 14148 0 16 49152 0 VMSPACE 584 58 26 978 0 16 49056 0 malloc-384 384 72 48 79 0 30 46080 0 da_ccb 544 0 84 2465 0 16 45696 0 syncache 168 0 264 3 0 254 44352 0 Files 80 243 307 8084 0 126 44000 0 vnpbuf 2624 0 16 1 0 16 41984 0 malloc-4096 4096 4 6 47 0 2 40960 0 pcpu-8 8 4793 327 5021 0 254 40960 0 pipe 736 23 32 340 0 16 40480 0 sctp_chunk 152 1 259 1 0 254 39520 0 malloc-64 64 23 544 13890 0 254 36288 0 malloc-64 64 32 535 62 0 254 36288 0 malloc-64 64 238 329 1230 0 254 36288 0 malloc-64 64 88 479 1086 0 254 36288 0 malloc-128 128 13 266 296 0 126 35712 0 malloc-128 128 121 158 588 0 126 35712 0 malloc-128 128 28 251 294 0 126 35712 0 ktls_session 256 1 134 5 0 62 34560 0 routing nhops 256 26 109 33 0 62 34560 0 ttyoutq 256 72 63 160 0 62 34560 0 malloc-256 256 27 108 549 0 62 34560 0 malloc-256 256 19 116 478 0 62 34560 0 malloc-256 256 41 94 476 0 62 34560 0 malloc-256 256 19 116 291 0 62 34560 0 TURNSTILE 136 185 67 185 0 62 34272 0 malloc-2048 2048 9 7 43 0 8 32768 0 malloc-2048 2048 3 13 49 0 8 32768 0 malloc-2048 2048 9 7 32 0 8 32768 0 malloc-2048 2048 9 7 12 0 8 32768 0 malloc-1024 1024 3 29 45 0 16 32768 0 malloc-1024 1024 19 13 1032 0 16 32768 0 malloc-1024 1024 4 28 8 0 16 32768 0 malloc-1024 1024 4 28 9 0 16 32768 0 malloc-512 512 15 49 55 0 30 32768 0 malloc-512 512 0 64 15 0 30 32768 0 malloc-512 512 2 62 59 0 30 32768 0 malloc-512 512 11 53 34 0 30 32768 0 malloc-512 512 2 62 161 0 30 32768 0 malloc-512 512 3 61 12 0 30 32768 0 pcpu-64 64 501 11 501 0 254 32768 0 sctp_stream_msg_out 112 1 287 1 0 254 32256 0 ttyinq 160 135 65 300 0 62 32000 0 PGRP 120 27 237 42 0 126 31680 0 clpbuf 2624 0 12 58 0 4 31488 0 sctp_laddr 48 4 584 16 0 254 28224 0 rl_entry 48 14 574 19 0 254 28224 0 malloc-32 32 162 720 1453 0 254 28224 0 malloc-32 32 369 513 8893 0 254 28224 0 16 Bucket 144 69 127 316 0 62 28224 0 4 Bucket 48 6 582 9 0 254 28224 0 AIO 208 2 131 23 0 62 27664 0 udplite_inpcb 408 0 63 6 0 30 25704 0 cpuset 200 8 120 11 0 62 25600 0 malloc-8192 8192 3 0 3 0 1 24576 0 ertt_txseginfo 40 1 605 749 0 254 24240 0 PWD 40 36 570 197 0 254 24240 0 rtentry 168 29 115 33 0 62 24192 0 8 Bucket 80 51 249 594 0 126 24000 0 malloc-384 384 1 59 5 0 30 23040 0 malloc-384 384 12 48 13 0 30 23040 0 malloc-384 384 41 19 604 0 30 23040 0 malloc-384 384 1 59 2 0 30 23040 0 ripcb 376 11 49 40 0 30 22560 0 SLEEPQUEUE 88 185 71 185 0 126 22528 0 hostcache 64 2 313 2 0 254 20160 0 ertt 72 13 267 130 0 126 20160 0 malloc-64 64 3 312 3 0 254 20160 0 malloc-32 32 83 547 557 0 254 20160 0 malloc-32 32 39 591 249 0 254 20160 0 malloc-32 32 61 569 97 0 254 20160 0 malloc-32 32 125 505 134 0 254 20160 0 malloc-32 32 17 613 28 0 254 20160 0 2 Bucket 32 66 564 429 0 254 20160 0 KNOTE 160 3 122 64 0 62 20000 0 malloc-128 128 18 137 167 0 126 19840 0 cryptop 280 0 70 7 0 30 19600 0 AIOCB 552 1 34 27 0 16 19320 0 L VFS Cache 320 0 60 21 0 30 19200 0 vmem 1856 2 7 2 0 8 16704 0 epoch_record pcpu 256 4 60 4 0 62 16384 0 malloc-16384 16384 1 0 1 0 1 16384 0 malloc-16384 16384 1 0 1 0 1 16384 0 malloc-8192 8192 0 2 19 0 1 16384 0 malloc-8192 8192 2 0 2 0 1 16384 0 malloc-4096 4096 0 4 7 0 2 16384 0 malloc-4096 4096 1 3 6 0 2 16384 0 malloc-4096 4096 0 4 2 0 2 16384 0 malloc-2048 2048 0 8 12 0 8 16384 0 malloc-2048 2048 2 6 2 0 8 16384 0 malloc-1024 1024 0 16 3 0 16 16384 0 malloc-1024 1024 2 14 2 0 16 16384 0 malloc-512 512 0 32 110 0 30 16384 0 malloc-512 512 2 30 2 0 30 16384 0 SMR CPU 32 8 503 8 0 254 16352 0 tcp_bbr_pcb 896 0 18 1 0 16 16128 0 malloc-16 16 511 489 3606 0 254 16000 0 tcp_log_id_node 120 1 131 1 0 126 15840 0 kenv 258 17 43 1065 0 30 15480 0 mqnode 416 3 33 3 0 30 14976 0 SMR SHARED 24 8 503 8 0 254 12264 0 tcp_log_id_bucket 176 1 68 1 0 62 12144 0 malloc-16 16 63 687 1584 0 254 12000 0 malloc-16 16 10 740 15 0 254 12000 0 malloc-16 16 44 706 129 0 254 12000 0 malloc-16 16 58 692 28906 0 254 12000 0 malloc-16 16 15 735 20 0 254 12000 0 malloc-16 16 21 729 52 0 254 12000 0 itimer 352 0 33 2 0 30 11616 0 AIOLIO 272 0 42 4 0 30 11424 0 Mountpoints 2816 2 2 4 0 4 11264 0 malloc-8192 8192 0 1 26 0 1 8192 0 malloc-8192 8192 1 0 1 0 1 8192 0 vtnet_tx_hdr 24 0 334 1662 0 254 8016 0 pcpu-16 16 4 252 4 0 254 4096 0 UMA Slabs 1 176 9 13 9 0 62 3872 0 KMAP ENTRY 96 12 27 14 0 0 3744 0 FFS1 dinode 128 0 0 0 0 126 0 0 ada_ccb 272 0 0 0 0 30 0 0 swblk 136 0 0 0 0 62 0 0 swpctrie 152 0 0 0 0 62 0 0 cdg_qdiffsample 16 0 0 0 0 254 0 0 pf state scrubs 40 0 0 0 0 254 0 0 pf frag entries 40 0 0 0 0 254 0 0 pf fragment node 72 0 0 0 0 126 0 0 pf frags 232 0 0 0 0 62 0 0 pf table entries 160 0 0 0 0 254 0 0 pf table entry counters 64 0 0 0 0 254 0 0 pf Ethernet anchors 1240 0 0 0 0 64 0 0 pf anchors 1664 0 0 0 0 64 0 0 pf UDP mappings 104 0 0 0 0 126 0 0 pf source nodes 152 0 0 0 0 254 0 0 pf state keys 88 0 0 0 0 126 0 0 pf states 384 0 0 0 0 254 0 0 pf tags 104 0 0 0 0 126 0 0 pf mtags 184 0 0 0 0 62 0 0 tcp_rack_pcb 1088 0 0 0 0 8 0 0 tcp_rack_map 128 0 0 0 0 126 0 0 tcp_bbr_map 128 0 0 0 0 126 0 0 tfo_ccache_entries 80 0 0 0 0 126 0 0 tfo 4 0 0 0 0 254 0 0 sackhole 32 0 0 0 0 254 0 0 ipq 56 0 0 0 0 254 0 0 sctp_asconf_ack 48 0 0 0 0 254 0 0 sctp_asconf 40 0 0 0 0 254 0 0 sctp_readq 152 0 0 0 0 254 0 0 tcpreass 48 0 0 0 0 254 0 0 IPsec SA lft_c 16 0 0 0 0 254 0 0 mqnotifier 216 0 0 0 0 62 0 0 mvdata 64 0 0 0 0 254 0 0 mqueue 248 0 0 0 0 62 0 0 TMPFS node 240 0 0 0 0 62 0 0 NCLNODE 608 0 0 0 0 16 0 0 LTS VFS Cache 360 0 0 0 0 30 0 0 STS VFS Cache 144 0 0 0 0 62 0 0 linux_dma_object 32 0 0 0 0 254 0 0 linux_dma_pctrie 152 0 0 0 0 62 0 0 IOMMU_MAP_ENTRY 112 0 0 0 0 126 0 0 skbuff 1808 0 0 0 0 8 0 0 mbuf_jumbo_16k 16384 0 0 0 0 254 0 0 mbuf_jumbo_9k 9216 0 0 0 0 254 0 0 audit_record 1280 0 0 0 0 8 0 0 domainset 40 0 0 0 0 254 0 0 MAC labels 40 0 0 0 0 254 0 0 nfspbuf 2624 0 0 0 0 4 0 0 swwbuf 2624 0 0 0 0 2 0 0 swrbuf 2624 0 0 0 0 4 0 0 umtx_shm 88 0 0 0 0 126 0 0 umtx pi 96 0 0 0 0 126 0 0 rangeset pctrie nodes 152 0 0 0 0 62 0 0 malloc-65536 65536 0 0 0 0 1 0 0 malloc-65536 65536 0 0 0 0 1 0 0 malloc-65536 65536 0 0 0 0 1 0 0 malloc-32768 32768 0 0 0 0 1 0 0 malloc-32768 32768 0 0 0 0 1 0 0 malloc-16384 16384 0 0 0 0 1 0 0 malloc-16384 16384 0 0 0 0 1 0 0 malloc-16384 16384 0 0 0 0 1 0 0 malloc-8192 8192 0 0 0 0 1 0 0 pcpu-32 32 0 0 0 0 254 0 0 pcpu-4 4 0 0 0 0 254 0 0 fakepg 88 0 0 0 0 126 0 0 UMA Hash 256 0 0 0 0 62 0 0 db> show all tcpcbs/bl