panic: Lock pf config not exclusively locked @ /syzkaller/managers/i386/kernel/sys/netpfil/pf/pf_ioctl.c:546 cpuid = 0 time = 1650732106 KDB: stack backtrace: db_trace_self_wrapper() at db_trace_self_wrapper+0xc7/frame 0xfffffe0094643ab0 kdb_backtrace() at kdb_backtrace+0xd3/frame 0xfffffe0094643c10 vpanic() at vpanic+0x2b8/frame 0xfffffe0094643cf0 panic() at panic+0xb5/frame 0xfffffe0094643db0 _sx_xunlock_hard() at _sx_xunlock_hard/frame 0xfffffe0094643e00 pf_free_rule() at pf_free_rule+0x4c/frame 0xfffffe0094643e30 pfioctl() at pfioctl+0xc535/frame 0xfffffe00946451d0 devfs_ioctl() at devfs_ioctl+0x22a/frame 0xfffffe00946452b0 VOP_IOCTL_APV() at VOP_IOCTL_APV+0xb0/frame 0xfffffe00946452f0 vn_ioctl() at vn_ioctl+0x215/frame 0xfffffe0094645640 devfs_ioctl_f() at devfs_ioctl_f+0x71/frame 0xfffffe0094645690 kern_ioctl() at kern_ioctl+0x631/frame 0xfffffe0094645790 sys_ioctl() at sys_ioctl+0x31f/frame 0xfffffe00946458f0 freebsd32_ioctl() at freebsd32_ioctl+0x629/frame 0xfffffe0094645d30 ia32_syscall() at ia32_syscall+0x419/frame 0xfffffe0094645f30 int0x80_syscall_common() at int0x80_syscall_common+0x9c/frame 0xfbffcf78 KDB: enter: panic [ thread pid 7994 tid 108757 ] Stopped at kdb_enter+0x6b: movq $0,0x270548a(%rip) db> db> set $lines = 0 db> set $maxwidth = 0 db> show registers cs 0x20 ds 0x3b es 0x3b fs 0x13 gs 0x1b ss 0 rax 0x12 rcx 0x3b75c571748c7c20 rdx 0x3ffff rbx 0 rsp 0xfffffe0094643bf0 rbp 0xfffffe0094643c10 rsi 0x40001 rdi 0xffffffff8178199a vprintf+0x35a r8 0 r9 0xffffffff r10 0 r11 0xfffffe0099f05510 r12 0xfffffe0099f05000 r13 0xfffffe0094643c01 r14 0xffffffff82bc8800 .str.26 r15 0xffffffff82bc8800 .str.26 rip 0xffffffff81774f0b kdb_enter+0x6b rflags 0x200046 kernload+0x46 kdb_enter+0x6b: movq $0,0x270548a(%rip) db> show proc Process 7994 (syz-executor.0) at 0xfffffe0099bd2548: state: NORMAL uid: 0 gids: 0, 0, 5 parent: pid 779 at 0xfffffe0053df0a90 ABI: FreeBSD ELF32 flag: 0x10000080 flag2: 0 arguments: /root/syz-executor.0 exec reaper: 0xfffffe0053df0000 reapsubtree: 1 sigparent: 20 vmspace: 0xfffffe0099ef29f8 (map 0xfffffe0099ef29f8) (map.pmap 0xfffffe0099ef2ab8) (pmap 0xfffffe0099ef2b20) threads: 2 107611 RunQ syz-executor.0 108757 Run CPU 0 syz-executor.0 db> ps pid ppid pgrp uid state wmesg wchan cmd 7994 779 779 0 R (threaded) syz-executor.0 107611 RunQ syz-executor.0 108757 Run CPU 0 syz-executor.0 7993 793 793 0 R (threaded) syz-executor.3 103256 RunQ syz-executor.3 108758 RunQ syz-executor.3 7992 789 789 0 R (threaded) syz-executor.2 103037 RunQ syz-executor.2 108755 D biowr 0xfffffe00079d6398 syz-executor.2 7991 780 780 0 R (threaded) syz-executor.1 108596 RunQ syz-executor.1 108756 S connec 0xfffffe0058c270da syz-executor.1 6475 1 780 0 S uwait 0xfffffe0057893680 syz-executor.1 6469 1 780 0 S uwait 0xfffffe0058bf5800 syz-executor.1 6467 1 780 0 S uwait 0xfffffe0058bf6080 syz-executor.1 6461 1 780 0 S uwait 0xfffffe0058bf5a00 syz-executor.1 6459 0 0 0 DL aiordy 0xfffffe0099bd1000 [aiod4] 6458 0 0 0 DL aiordy 0xfffffe0058c57548 [aiod3] 6457 0 0 0 DL aiordy 0xfffffe0094942a90 [aiod2] 6456 0 0 0 DL aiordy 0xfffffe0092bafa90 [aiod1] 5826 1 5826 65 Ss select 0xfffffe0058b1b340 dhclient 3537 1 3537 0 Ss select 0xfffffe0058b1c640 dhclient 3534 1 3534 0 Ss select 0xfffffe0092365340 dhclient 3515 1 3515 65 Ss select 0xfffffe0058b1c7c0 dhclient 3439 1 3439 0 SVs uwait 0xfffffe0058bf5700 syz-executor.1 2512 1 2512 0 Ss select 0xfffffe0092364c40 dhclient 2509 1 2509 0 Ss select 0xfffffe0058b1cd40 dhclient 2490 1 2490 65 Ss select 0xfffffe0092364e40 dhclient 1939 1 793 0 S uwait 0xfffffe0057893880 syz-executor.3 1938 1 1938 0 SVs uwait 0xfffffe0053df9880 syz-executor.3 1928 1925 1928 0 SVs uwait 0xfffffe0057893b80 syz-executor.3 1926 1925 1925 0 S uwait 0xfffffe0058bf5c00 syz-executor.3 1925 1 1925 0 DVs ppwait 0xfffffe00949424f8 syz-executor.3 1916 1 1916 0 Ss select 0xfffffe0092364740 dhclient 1913 1 1913 0 Ss select 0xfffffe0058b1dac0 dhclient 1899 1 1899 0 SVs uwait 0xfffffe0057892280 syz-executor.3 1891 1 1891 65 Ss select 0xfffffe0092365740 dhclient 1272 1 1272 0 Ss select 0xfffffe0058b1d740 dhclient 1269 1 1269 0 Ss select 0xfffffe0092365640 dhclient 793 774 793 0 Rs syz-executor.3 789 774 789 0 Rs syz-executor.2 780 774 780 0 Ss nanslp 0xffffffff83e47a40 syz-executor.1 779 774 779 0 Ss nanslp 0xffffffff83e47a40 syz-executor.0 774 772 772 0 S (threaded) syz-fuzzer 100099 S uwait 0xfffffe0058bf7d80 syz-fuzzer 100112 S uwait 0xfffffe0058bf6b00 syz-fuzzer 100113 S uwait 0xfffffe0058bf6c00 syz-fuzzer 100114 S uwait 0xfffffe0058bf6d00 syz-fuzzer 100115 S uwait 0xfffffe0058bf6e00 syz-fuzzer 100116 S uwait 0xfffffe0058bf7980 syz-fuzzer 100117 S uwait 0xfffffe0058bf7a80 syz-fuzzer 100119 S kqread 0xfffffe0058ae2400 syz-fuzzer 100121 S uwait 0xfffffe0057894700 syz-fuzzer 772 770 772 0 Ss pause 0xfffffe0058c565f8 csh 770 688 770 0 Ss select 0xfffffe00923652c0 sshd 754 1 754 0 Ss+ ttyin 0xfffffe00574774b0 getty 753 1 753 0 Ss+ ttyin 0xfffffe005868ecb0 getty 752 1 752 0 Ss+ ttyin 0xfffffe00586904b0 getty 751 1 751 0 Ss+ ttyin 0xfffffe0058690cb0 getty 750 1 750 0 Ss+ ttyin 0xfffffe00586914b0 getty 749 1 749 0 Ss+ ttyin 0xfffffe0058691cb0 getty 748 1 748 0 Ss+ ttyin 0xfffffe0053f414b0 getty 747 1 747 0 Ss+ ttyin 0xfffffe0053f41cb0 getty 746 1 746 0 Ss+ ttyin 0xfffffe0053f424b0 getty 692 1 692 0 Ss nanslp 0xffffffff83e47a40 cron 688 1 688 0 Ss select 0xfffffe00923656c0 sshd 501 1 501 0 Ss select 0xfffffe0092365e40 syslogd 430 1 430 0 Ss select 0xfffffe00923659c0 devd 429 1 429 65 Ss select 0xfffffe0058b1d9c0 dhclient 344 1 344 0 Ss select 0xfffffe00923658c0 dhclient 341 1 341 0 Ss select 0xfffffe0058b1da40 dhclient 17 0 0 0 DL vlruwt 0xfffffe0056f9e548 [vnlru] 16 0 0 0 DL syncer 0xffffffff83f6d260 [syncer] 15 0 0 0 DL (threaded) [bufdaemon] 100079 D psleep 0xffffffff83f6b860 [bufdaemon] 100082 D - 0xffffffff83211f80 [bufspacedaemon-0] 100093 D sdflush 0xfffffe0058c64ce8 [/ worker] 9 0 0 0 DL psleep 0xffffffff83f9f380 [vmdaemon] 8 0 0 0 DL (threaded) [pagedaemon] 100077 D psleep 0xffffffff83f93238 [dom0] 100080 D launds 0xffffffff83f93244 [laundry: dom0] 100081 D umarcl 0xffffffff81eb0e30 [uma] 7 0 0 0 DL - 0xffffffff83c03788 [rand_harvestq] 6 0 0 0 DL pftm 0xffffffff8459d530 [pf purge] 5 0 0 0 DL waiting 0xffffffff8496f4a0 [sctp_iterator] 4 0 0 0 RL (threaded) [cam] 100044 RunQ [doneq0] 100045 D - 0xffffffff83aa5640 [async] 100076 D - 0xffffffff83aa54c0 [scanner] 14 0 0 0 DL seqstat 0xfffffe0056f03c88 [sequencer 00] 3 0 0 0 DL (threaded) [crypto] 100040 D crypto_ 0xffffffff83f8ea40 [crypto] 100041 D crypto_ 0xfffffe0053f73030 [crypto returns 0] 100042 D crypto_ 0xfffffe0053f73080 [crypto returns 1] 13 0 0 0 DL (threaded) [geom] 100035 D - 0xffffffff83e1d000 [g_event] 100036 D - 0xffffffff83e1d020 [g_up] 100037 D - 0xffffffff83e1d040 [g_down] 2 0 0 0 WL (threaded) [clock] 100029 I [clock (0)] 100030 I [clock (1)] 12 0 0 0 RL (threaded) [intr] 100015 I [swi5: fast taskq] 100018 I [swi6: task queue] 100019 I [swi6: Giant taskq] 100031 I [swi1: netisr 0] 100032 I [swi1: hpts] 100033 Run CPU 1 [swi1: hpts] 100046 I [irq24: virtio_pci0] 100047 I [irq25: virtio_pci0] 100048 I [irq26: virtio_pci0] 100049 I [irq27: virtio_pci0] 100050 I [irq28: virtio_pci1] 100051 I [irq29: virtio_pci1] 100052 I [irq30: virtio_pci1] 100053 I [irq31: virtio_pci1] 100054 I [irq32: virtio_pci1] 100059 I [irq33: virtio_pci2] 100060 I [irq34: virtio_pci2] 100061 I [irq35: virtio_pci2] 100063 I [irq1: atkbd0] 100064 I [irq12: psm0] 100065 I [swi0: uart uart++] 100069 I [swi1: pf send] 11 0 0 0 RL (threaded) [idle] 100003 CanRun [idle: cpu0] 100004 CanRun [idle: cpu1] 1 0 1 0 SLs wait 0xfffffe0053df0000 [init] 10 0 0 0 DL audit_w 0xffffffff83f8f540 [audit] 0 0 0 0 DLs (threaded) [kernel] 100000 D swapin 0xffffffff83e1da40 [swapper] 100005 D - 0xfffffe0008191000 [if_config_tqg_0] 100006 D - 0xfffffe0008190e00 [softirq_0] 100007 D - 0xfffffe0008190d00 [softirq_1] 100008 D - 0xfffffe0008190c00 [if_io_tqg_0] 100009 D - 0xfffffe0008190b00 [if_io_tqg_1] 100010 D - 0xfffffe0008190a00 [inm_free taskq] 100011 D - 0xfffffe0008190900 [linuxkpi_irq_wq] 100012 D - 0xfffffe0008190800 [in6m_free taskq] 100013 D - 0xfffffe0008190700 [deferred_unmount ta] 100014 D - 0xfffffe0008190600 [thread taskq] 100016 D - 0xfffffe0008190400 [pci_hp taskq] 100017 D - 0xfffffe0008190300 [kqueue_ctx taskq] 100020 D - 0xfffffe0008190000 [aiod_kick taskq] 100021 D - 0xfffffe000818fe00 [linuxkpi_short_wq_0] 100022 D - 0xfffffe000818fe00 [linuxkpi_short_wq_1] 100023 D - 0xfffffe000818fe00 [linuxkpi_short_wq_2] 100024 D - 0xfffffe000818fe00 [linuxkpi_short_wq_3] 100025 D - 0xfffffe000818fd00 [linuxkpi_long_wq_0] 100026 D - 0xfffffe000818fd00 [linuxkpi_long_wq_1] 100027 D - 0xfffffe000818fd00 [linuxkpi_long_wq_2] 100028 D - 0xfffffe000818fd00 [linuxkpi_long_wq_3] 100034 D - 0xfffffe000818fc00 [firmware taskq] 100038 D - 0xfffffe000818f200 [crypto_0] 100039 D - 0xfffffe000818f200 [crypto_1] 100055 D - 0xfffffe000818e900 [vtnet0 rxq 0] 100056 D - 0xfffffe000818e800 [vtnet0 txq 0] 100057 D - 0xfffffe000818e700 [vtnet0 rxq 1] 100058 D - 0xfffffe000818e600 [vtnet0 txq 1] 100062 D vtbslp 0xfffffe0056f4be00 [virtio_balloon] 100066 D - 0xffffffff82bce680 [deadlkres] 100070 D - 0xfffffe0008191100 [mca taskq] 100071 D - 0xfffffe000818eb00 [acpi_task_0] 100072 D - 0xfffffe000818eb00 [acpi_task_1] 100073 D - 0xfffffe000818eb00 [acpi_task_2] 100075 D - 0xfffffe000818ea00 [CAM taskq] db> show all locks Process 7994 (syz-executor.0) thread 0xfffffe0099f05000 (108757) exclusive rm pf rulesets (pf rulesets) r = 0 (0xffffffff846596a0) locked @ /syzkaller/managers/i386/kernel/sys/netpfil/pf/pf_ioctl.c:3435 Process 7992 (syz-executor.2) thread 0xfffffe009482f3a0 (108755) exclusive lockmgr bufwait (bufwait) r = 0 (0xfffffe00079d6418) locked @ /syzkaller/managers/i386/kernel/sys/ufs/ffs/ffs_softdep.c:14720 exclusive lockmgr bufwait (bufwait) r = 0 (0xfffffe00079a1948) locked @ /syzkaller/managers/i386/kernel/sys/kern/vfs_bio.c:3988 exclusive lockmgr ufs (ufs) r = 0 (0xfffffe009251fcb0) locked @ /syzkaller/managers/i386/kernel/sys/kern/vfs_vnops.c:1166 db> show malloc Type InUse MemUse Requests pf_hash 5 11524K 5 tcp_hpts 7 4801K 7 devbuf 4217 4323K 4245 sysctloid 35322 2081K 35393 vtbuf 24 1968K 46 kobj 328 1312K 489 newblk 39 1034K 11281 vfscache 3 1025K 3 pcb 431 996K 13604 inodedep 14 517K 7854 ufs_quota 1 512K 1 vfs_hash 1 512K 1 callout 2 512K 2 intr 4 472K 4 subproc 163 323K 8086 filedesc 40 313K 14051 vmem 3 274K 6 sctp_stro 200 200K 1481 acpica 1674 184K 57552 vnet_data 1 168K 1 sctp_atcl 404 152K 8232 tidhash 3 141K 3 linker 358 134K 386 pagedep 9 130K 7005 tfo_ccache 1 128K 1 DEVFS1 109 109K 126 sem 4 106K 4 BPF 46 88K 61 bus 994 81K 5207 mtx_pool 2 72K 2 syncache 1 68K 1 module 513 65K 513 acpitask 1 64K 1 ddb_capture 1 64K 1 umtx 374 47K 374 kdtrace 241 47K 16754 temp 34 33K 4066 DEVFS3 128 32K 138 hostcache 1 32K 1 shm 1 32K 39 msg 4 30K 4 gtaskqueue 18 26K 18 sctp_atky 604 26K 9751 kbdmux 6 22K 6 ifaddr 70 20K 72 DEVFS_RULE 56 20K 56 ufs_mount 4 17K 5 proc 3 17K 3 tty 16 16K 16 routetbl 130 16K 410 ithread 97 16K 97 bus-sc 34 15K 1681 lltable 44 14K 56 KTRACE 100 13K 100 ifnet 7 13K 7 ether_multi 152 13K 162 sctp_timw 47 12K 47 kenv 95 12K 95 eventhandler 134 12K 134 rman 88 11K 431 GEOM 61 11K 490 CAM queue 5 11K 1528 in6_multi 65 9K 65 bmsafemap 2 9K 7617 UART 12 9K 12 devstat 4 9K 4 ksem 1 8K 15 rpc 2 8K 2 shmfd 1 8K 1 pfs_vncache 1 8K 1 pfs_nodes 20 8K 20 audit_evclass 237 8K 296 cred 27 7K 251 kqueue 74 7K 8008 taskqueue 63 7K 63 sctp_athm 404 7K 8232 sctp_map 400 7K 2962 sglist 5 7K 5 CAM DEV 3 6K 510 plimit 24 6K 425 session 40 5K 55 ufs_dirhash 24 5K 24 pwddesc 72 5K 8078 UMA 272 5K 272 pf_ifnet 10 5K 19 DEVFSP 68 5K 466 vt 11 5K 11 memdesc 1 4K 1 MCA 32 4K 32 ioctlops 1 4K 118 evdev 4 4K 4 freework 15 4K 8516 proc-args 103 4K 9318 acpisem 28 4K 28 selfd 55 4K 116088 lockf 32 4K 57 hhook 15 4K 17 kcovinfo 52 4K 52 terminal 11 3K 11 select 19 3K 72 uidinfo 4 3K 13 pf_rule 1 2K 1 local_apic 1 2K 1 io_apic 1 2K 1 fpukern_ctx 2 2K 2 ipsec-saq 2 2K 2 ip6ndp 12 2K 13 Unitno 29 2K 47 sctp_ifa 13 2K 14 CAM XPT 22 2K 543 msi 12 2K 12 in_multi 6 2K 8 ipsecpolicy 2 2K 2 acpidev 20 2K 20 clone 9 2K 9 tun 7 2K 7 softdep 1 1K 1 sahead 1 1K 1 secasvar 1 1K 1 nhops 6 1K 8 vnodemarker 2 1K 40 NFSD session 1 1K 1 mkdir 7 1K 13974 CAM periph 4 1K 271 sctp_ifn 6 1K 14 ipsec 3 1K 3 dirrem 3 1K 7465 mld 6 1K 6 igmp 6 1K 6 toponodes 6 1K 6 isadev 6 1K 6 mount 16 1K 89 pci_link 10 1K 10 inpcbpolicy 22 1K 3003 CC Mem 11 1K 2657 diradd 5 1K 7505 crypto 4 1K 4 encap_export_host 12 1K 12 procdesc 5 1K 18 newdirblk 4 1K 6987 freefile 4 1K 7455 freeblks 2 1K 7207 pfil 4 1K 4 osd 15 1K 2668 cdev 2 1K 2 chacha20random 1 1K 1 tcp_fsb 2 1K 763 NFSD lckfile 1 1K 1 NFSD V4client 1 1K 1 DEVFS 9 1K 10 indirdep 1 1K 2299 vnodes 1 1K 1 CAM SIM 2 1K 2 feeder 7 1K 7 pf_osfp 2 1K 2 tcpfunc 3 1K 3 loginclass 3 1K 6 prison 6 1K 6 lkpikmalloc 5 1K 6 aesni_data 2 1K 2 cryptodev 2 1K 62 nexusdev 8 1K 8 apmdev 1 1K 1 atkbddev 2 1K 2 CAM dev queue 2 1K 2 CAM I/O Scheduler 1 1K 1 CAM path 4 1K 1034 soname 5 1K 10625 iov 2 1K 18763 eventfd 1 1K 37 pmchooks 1 1K 1 filecaps 5 1K 112 sctp_vrf 1 1K 1 vnet 1 1K 1 entropy 2 1K 43 pmc 1 1K 1 acpiintr 1 1K 1 cpus 2 1K 2 vnet_data_free 1 1K 1 Per-cpu 1 1K 1 p1003.1b 1 1K 1 tcp_do 0 0K 0 sctp_mcore 0 0K 0 sctp_socko 0 0K 2335 sctp_iter 0 0K 11 sctp_mvrf 0 0K 0 sctp_cpal 0 0K 0 sctp_cmsg 0 0K 0 sctp_stre 0 0K 0 sctp_athi 0 0K 0 sctp_a_it 0 0K 11 sctp_aadr 0 0K 0 sctp_stri 0 0K 0 mqdata 0 0K 0 ipcomp 0 0K 0 esp 0 0K 0 ah 0 0K 0 pf_table 0 0K 0 pf_altq 0 0K 0 pf_temp 0 0K 0 filemon 0 0K 359 NFSCL layrecall 0 0K 0 NFSCL session 0 0K 0 madt_table 0 0K 2 smartpqi 0 0K 0 NFSCL sockreq 0 0K 0 NFSCL devinfo 0 0K 0 NFSCL flayout 0 0K 0 ixl 0 0K 0 NFSCL layout 0 0K 0 NFSD rollback 0 0K 0 NFSCL diroff 0 0K 0 NEWdirectio 0 0K 0 NEWNFSnode 0 0K 0 NFSCL lck 0 0K 0 NFSCL lckown 0 0K 0 ice-resmgr 0 0K 0 ice-osdep 0 0K 0 ice 0 0K 0 iavf 0 0K 0 axgbe 0 0K 0 NFSCL client 0 0K 0 NFSCL deleg 0 0K 0 NFSCL open 0 0K 0 NFSCL owner 0 0K 0 NFS fh 0 0K 0 NFS req 0 0K 0 NFSD usrgroup 0 0K 0 NFSD string 0 0K 0 NFSD V4lock 0 0K 0 xen_intr 0 0K 0 NFSD V4state 0 0K 0 xen_hvm 0 0K 0 legacydrv 0 0K 0 bounce 0 0K 0 busdma 0 0K 0 qpidrv 0 0K 0 NFSD srvcache 0 0K 0 msdosfs_fat 0 0K 0 dmar_idpgtbl 0 0K 0 dmar_dom 0 0K 0 dmar_ctx 0 0K 0 msdosfs_mount 0 0K 0 msdosfs_node 0 0K 0 isci 0 0K 0 iommu_dmamap 0 0K 0 hyperv_socket 0 0K 0 bxe_ilt 0 0K 0 xenbus 0 0K 0 DEVFS4 0 0K 0 vm_fictitious 0 0K 0 DEVFS2 0 0K 0 gntdev 0 0K 0 privcmd_dev 0 0K 0 evtchn_dev 0 0K 0 xenstore 0 0K 0 scsi_pass 0 0K 0 ciss_data 0 0K 0 xnb 0 0K 0 xen_acpi 0 0K 0 xbbd 0 0K 0 xbd 0 0K 0 UMAHash 0 0K 0 Balloon 0 0K 0 sysmouse 0 0K 0 vm_pgdata 0 0K 0 jblocks 0 0K 0 savedino 0 0K 9844 sentinel 0 0K 0 jfsync 0 0K 0 jtrunc 0 0K 0 sbdep 0 0K 17 jsegdep 0 0K 0 jseg 0 0K 0 jfreefrag 0 0K 0 jfreeblk 0 0K 0 jnewblk 0 0K 0 jmvref 0 0K 0 jremref 0 0K 0 jaddref 0 0K 0 freedep 0 0K 0 freefrag 0 0K 2 allocindir 0 0K 0 allocdirect 0 0K 0 ufs_trim 0 0K 0 mactemp 0 0K 0 audit_trigger 0 0K 0 audit_pipe_presel 0 0K 0 audit_pipeent 0 0K 0 audit_pipe 0 0K 0 audit_evname 0 0K 0 audit_bsm 0 0K 0 audit_gidset 0 0K 0 audit_text 0 0K 0 audit_path 0 0K 0 audit_data 0 0K 0 audit_cred 0 0K 0 vtfont 0 0K 0 BACKLIGHT 0 0K 0 ath_hal 0 0K 0 athdev 0 0K 0 ata_pci 0 0K 0 ata_dma 0 0K 0 ata_generic 0 0K 0 pvscsi 0 0K 0 scsi_da 0 0K 69 ata_da 0 0K 0 scsi_ch 0 0K 0 scsi_cd 0 0K 0 ktls_ocf 0 0K 0 AHCI driver 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5E_TLS_RX 0 0K 0 MLX5EEPROM 0 0K 0 MLX5E_TLS 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EN 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5DUMP 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 seq_file 0 0K 0 lkpiskb 0 0K 0 radix 0 0K 0 idr 0 0K 0 lkpindev 0 0K 0 lkpifw 0 0K 0 lkpi80211 0 0K 0 NLM 0 0K 0 ipsec-spdcache 0 0K 0 ipsec-reg 0 0K 0 ipsec-misc 0 0K 0 ipsecrequest 0 0K 0 ip6opt 0 0K 6 ip6_msource 0 0K 0 ip6_moptions 0 0K 0 in6_mfilter 0 0K 0 frag6 0 0K 0 tcplog 0 0K 0 tcp_hwpace 0 0K 0 USBdev 0 0K 0 USB 0 0K 0 LRO 0 0K 0 ip_msource 0 0K 0 ip_moptions 0 0K 0 in_mfilter 0 0K 0 ipid 0 0K 0 80211scan 0 0K 0 80211ratectl 0 0K 0 80211power 0 0K 0 80211nodeie 0 0K 0 80211node 0 0K 0 80211mesh_gt 0 0K 0 80211mesh_rt 0 0K 0 80211perr 0 0K 0 80211prep 0 0K 0 80211preq 0 0K 0 80211dfs 0 0K 0 80211crypto 0 0K 0 80211vap 0 0K 0 iflib 0 0K 0 vlan 0 0K 0 gif 0 0K 0 ifdescr 0 0K 0 zlib 0 0K 0 fadvise 0 0K 0 VN POLL 0 0K 0 agp 0 0K 0 statfs 0 0K 7238 namei_tracker 0 0K 0 export_host 0 0K 0 cl_savebuf 0 0K 3 nvme_da 0 0K 0 acpipwr 0 0K 0 acpi_perf 0 0K 0 twsbuf 0 0K 0 twe_commands 0 0K 0 tcp_log_dev 0 0K 301 midi buffers 0 0K 0 mixer 0 0K 0 ac97 0 0K 0 hdacc 0 0K 0 hdac 0 0K 0 hdaa 0 0K 0 acpicmbat 0 0K 0 SIIS driver 0 0K 0 CAM CCB 0 0K 523 biobuf 0 0K 0 aios 0 0K 35 lio 0 0K 0 acl 0 0K 0 mbuf_tag 0 0K 114 ktls 0 0K 0 PUC 0 0K 0 ppbusdev 0 0K 0 agtiapi_MemAlloc malloc 0 0K 0 osti_cacheable 0 0K 0 tempbuff 0 0K 0 tempbuff 0 0K 0 ag_tgt_map_t malloc 0 0K 0 ag_slr_map_t malloc 0 0K 0 lDevFlags * malloc 0 0K 0 tiDeviceHandle_t * malloc 0 0K 0 ag_portal_data_t malloc 0 0K 0 ag_device_t malloc 0 0K 0 STLock malloc 0 0K 0 CCB List 0 0K 0 sr_iov 0 0K 0 OCS 0 0K 0 OCS 0 0K 0 nvme 0 0K 0 nvd 0 0K 0 netmap 0 0K 0 mwldev 0 0K 0 MVS driver 0 0K 0 CAM ccb queue 0 0K 0 accf 0 0K 0 pts 0 0K 0 Witness 0 0K 0 stack 0 0K 0 mrsasbuf 0 0K 0 mpt_user 0 0K 0 mps_user 0 0K 0 MPSSAS 0 0K 0 mps 0 0K 0 sbuf 0 0K 288 mpr_user 0 0K 0 firmware 0 0K 0 compressor 0 0K 0 MPRSAS 0 0K 0 SWAP 0 0K 0 mpr 0 0K 0 mfibuf 0 0K 0 sysctltmp 0 0K 825 sysctl 0 0K 3 md_sectors 0 0K 0 ekcd 0 0K 0 dumper 0 0K 0 sendfile 0 0K 0 rctl 0 0K 0 md_disk 0 0K 0 malodev 0 0K 0 LED 0 0K 0 ix_sriov 0 0K 0 cache 0 0K 0 aacraidcam 0 0K 0 prison_racct 0 0K 0 Fail Points 0 0K 0 sigio 0 0K 1 filedesc_to_leader 0 0K 0 pwd 0 0K 0 tty console 0 0K 0 ix 0 0K 0 ipsbuf 0 0K 0 aacraid_buf 0 0K 0 aaccam 0 0K 0 boottrace 0 0K 0 aacbuf 0 0K 0 zstd 0 0K 0 XZ_DEC 0 0K 0 nvlist 0 0K 0 SCSI ENC 0 0K 0 SCSI sa 0 0K 0 isofs_node 0 0K 0 isofs_mount 0 0K 0 tr_raid5_data 0 0K 0 tr_raid1e_data 0 0K 0 tr_raid1_data 0 0K 0 tr_raid0_data 0 0K 0 tr_concat_data 0 0K 0 md_sii_data 0 0K 0 md_promise_data 0 0K 0 md_nvidia_data 0 0K 0 md_jmicron_data 0 0K 0 md_intel_data 0 0K 0 md_ddf_data 0 0K 0 raid_data 0 0K 72 geom_flashmap 0 0K 0 tmpfs dir 0 0K 0 tmpfs name 0 0K 0 tmpfs mount 0 0K 0 NFS FHA 0 0K 0 newnfsmnt 0 0K 0 newnfsclient_req 0 0K 0 db> show uma Zone Size Used Free Requests Sleeps Bucket Total Mem XFree mbuf_jumbo_page 4096 8392 1006 1269265 0 254 38494208 0 mbuf 256 9027 637 1989898 0 254 2473984 0 malloc-384 384 4130 1690 11971 0 30 2234880 0 tcp_log 416 6 5331 28250 0 254 2220192 0 mbuf_cluster 2048 1016 0 1016 0 254 2080768 0 pbuf 2624 0 778 0 0 2 2041472 0 RADIX NODE 144 13747 220 232814 0 62 2011248 0 BUF TRIE 144 183 11605 5003 0 62 1697472 0 malloc-128 128 11641 356 33185 0 126 1535616 0 malloc-4096 4096 331 1 494 0 2 1359872 0 UMA Slabs 0 112 11166 24 11166 0 126 1253280 0 sctp_asoc 2256 200 310 1481 0 254 1150560 0 vmem btag 56 18946 101 18946 0 254 1066632 0 sctp_ep 1208 204 566 6751 0 254 930160 0 socket 960 52 712 11138 0 254 733440 0 FFS inode 1160 523 30 7978 0 8 641480 0 VM OBJECT 264 2222 58 130693 0 30 601920 0 tcpcb 1104 11 500 2657 0 254 564144 0 malloc-2048 2048 212 20 6759 0 8 475136 0 malloc-256 256 66 1734 7890 0 62 460800 0 malloc-256 256 46 1754 17452 0 62 460800 0 256 Bucket 2048 196 20 5702 0 8 442368 0 malloc-65536 65536 6 0 6 0 1 393216 0 malloc-16384 16384 20 4 7024 0 1 393216 0 lkpimm 168 1 2327 1 0 62 391104 0 lkpicurr 168 2 2326 2 0 62 391104 0 sctp_raddr 736 200 317 1481 0 254 380512 0 THREAD 1808 168 19 8758 0 8 338096 0 malloc-4096 4096 77 5 8002 0 2 335872 0 MAP ENTRY 96 2948 328 313526 0 126 314496 0 VNODE 448 564 102 8021 0 30 298368 0 malloc-64 64 3970 629 127931 0 254 294336 0 tcp_bbr_map 128 0 2046 24184 0 126 261888 0 mbuf_packet 256 214 802 10831 0 254 260096 0 malloc-1024 1024 203 37 1497 0 16 245760 0 malloc-16 16 14620 380 17335 0 254 240000 0 malloc-128 128 37 1730 7643 0 126 226176 0 DEVCTL 1024 0 220 154 0 0 225280 0 malloc-256 256 237 573 12493 0 62 207360 0 malloc-65536 65536 1 2 177 0 1 196608 0 malloc-65536 65536 1 2 137 0 1 196608 0 malloc-384 384 435 75 8275 0 30 195840 0 malloc-32 32 5564 358 5640 0 254 189504 0 UMA Zones 768 244 0 244 0 16 187392 0 malloc-2048 2048 5 83 1228 0 8 180224 0 FPU_save_area 832 170 46 14136 0 16 179712 0 malloc-128 128 1241 154 27210 0 126 178560 0 malloc-32768 32768 0 5 2180 0 1 163840 0 malloc-1024 1024 137 23 154 0 16 163840 0 malloc-128 128 1132 139 5734 0 126 162688 0 S VFS Cache 104 1015 506 8624 0 126 158184 0 VMSPACE 2552 50 10 7975 0 4 153120 0 FFS2 dinode 256 523 47 7978 0 62 145920 0 128 Bucket 1024 92 39 992 0 16 134144 0 unpcb 256 20 490 1341 0 254 130560 0 ertt_txseginfo 40 0 3131 65887 0 254 125240 0 PROC 1352 71 19 7994 0 8 121680 0 64 Bucket 512 141 91 8105 0 30 118784 0 ksiginfo 112 80 964 5582 0 126 116928 0 malloc-256 256 265 170 9256 0 62 111360 0 malloc-256 256 279 156 22772 0 62 111360 0 malloc-32768 32768 0 3 120 0 1 98304 0 filedesc0 1072 72 19 8078 0 8 97552 0 malloc-8192 8192 9 2 38 0 1 90112 0 malloc-4096 4096 21 1 24 0 2 90112 0 malloc-4096 4096 19 3 33 0 2 90112 0 UMA Kegs 384 230 3 230 0 30 89472 0 g_bio 408 4 206 36914 0 30 85680 0 pipe 744 27 88 706 0 16 85560 0 clpbuf 2624 0 32 22 0 16 83968 0 32 Bucket 256 135 180 8674 0 62 80640 0 sctp_chunk 152 188 332 196 0 254 79040 0 malloc-8192 8192 8 1 10 0 1 73728 0 malloc-384 384 143 37 213 0 30 69120 0 malloc-64 64 566 505 1109 0 254 68544 0 malloc-64 64 770 301 3191 0 254 68544 0 malloc-128 128 374 153 8315 0 126 67456 0 malloc-65536 65536 1 0 1 0 1 65536 0 malloc-65536 65536 0 1 8 0 1 65536 0 malloc-32768 32768 2 0 2 0 1 65536 0 sctp_stream_msg_out 112 11 529 272 0 254 60480 0 malloc-8192 8192 1 6 360 0 1 57344 0 tcp_inpcb 424 11 115 2657 0 30 53424 0 udplite_inpcb 424 0 126 68 0 30 53424 0 malloc-64 64 114 705 8178 0 254 52416 0 Files 80 243 407 28566 0 126 52000 0 malloc-256 256 7 188 8420 0 62 49920 0 DIRHASH 1024 35 13 35 0 16 49152 0 NAMEI 1024 0 48 45296 0 16 49152 0 malloc-16384 16384 3 0 3 0 1 49152 0 malloc-1024 1024 5 43 1760 0 16 49152 0 tcp_rack_pcb 896 2 52 763 0 16 48384 0 tcp_bbr_pcb 832 0 54 1324 0 16 44928 0 syncache 168 0 264 4 0 254 44352 0 pcpu-8 8 4712 408 4973 0 254 40960 0 Mountpoints 2752 2 12 2 0 4 38528 0 udp_inpcb 424 6 84 226 0 30 38160 0 da_ccb 544 1 69 9340 0 16 38080 0 malloc-64 64 40 527 16386 0 254 36288 0 malloc-64 64 109 458 11392 0 254 36288 0 malloc-64 64 223 344 245 0 254 36288 0 malloc-64 64 165 402 568 0 254 36288 0 malloc-32 32 561 573 22235 0 254 36288 0 16 Bucket 144 74 178 568 0 62 36288 0 malloc-128 128 36 243 154 0 126 35712 0 malloc-128 128 31 248 539 0 126 35712 0 malloc-128 128 47 232 47 0 126 35712 0 routing nhops 256 27 108 34 0 62 34560 0 ttyoutq 256 72 63 160 0 62 34560 0 malloc-384 384 50 40 52 0 30 34560 0 malloc-256 256 68 67 194 0 62 34560 0 malloc-256 256 29 106 834 0 62 34560 0 TURNSTILE 136 188 64 188 0 62 34272 0 SLEEPQUEUE 88 188 196 188 0 126 33792 0 malloc-32768 32768 1 0 1 0 1 32768 0 malloc-16384 16384 2 0 17 0 1 32768 0 malloc-16384 16384 0 2 160 0 1 32768 0 malloc-8192 8192 2 2 103 0 1 32768 0 malloc-2048 2048 3 13 27 0 8 32768 0 malloc-2048 2048 11 5 11 0 8 32768 0 malloc-2048 2048 3 13 114 0 8 32768 0 malloc-2048 2048 0 16 50 0 8 32768 0 malloc-1024 1024 5 27 5 0 16 32768 0 malloc-1024 1024 3 29 168 0 16 32768 0 malloc-1024 1024 8 24 8 0 16 32768 0 malloc-1024 1024 3 29 20 0 16 32768 0 malloc-512 512 12 52 27 0 30 32768 0 malloc-512 512 0 64 356 0 30 32768 0 malloc-512 512 2 62 51 0 30 32768 0 malloc-512 512 3 61 41 0 30 32768 0 malloc-512 512 8 56 8 0 30 32768 0 pcpu-64 64 493 19 493 0 254 32768 0 KNOTE 160 28 172 122210 0 62 32000 0 ttyinq 160 135 65 300 0 62 32000 0 tcp_rack_map 120 2 262 505 0 126 31680 0 cpuset 104 7 272 7 0 126 29016 0 sctp_laddr 48 0 588 616 0 254 28224 0 tcp_inpcb ports 32 3 879 1603 0 254 28224 0 PWD 32 30 852 7076 0 254 28224 0 4 Bucket 48 7 581 193 0 254 28224 0 AIO 208 0 133 6 0 62 27664 0 ripcb 424 5 58 52 0 30 26712 0 malloc-8192 8192 3 0 3 0 1 24576 0 malloc-4096 4096 2 4 7863 0 2 24576 0 malloc-4096 4096 2 4 5 0 2 24576 0 rtentry 176 30 108 34 0 62 24288 0 PGRP 88 40 236 55 0 126 24288 0 rl_entry 40 73 533 73 0 254 24240 0 8 Bucket 80 49 251 2077 0 126 24000 0 malloc-384 384 3 57 61 0 30 23040 0 malloc-384 384 20 40 360 0 30 23040 0 hostcache 64 1 314 1 0 254 20160 0 udpcb 32 6 624 294 0 254 20160 0 udp_inpcb ports 32 3 627 33 0 254 20160 0 udplite_inpcb ports 32 0 630 3 0 254 20160 0 ertt 72 11 269 2657 0 126 20160 0 malloc-32 32 106 524 124 0 254 20160 0 malloc-32 32 26 604 164 0 254 20160 0 malloc-32 32 121 509 4292 0 254 20160 0 malloc-32 32 59 571 1125 0 254 20160 0 malloc-32 32 54 576 270 0 254 20160 0 malloc-32 32 5 625 9 0 254 20160 0 2 Bucket 32 61 569 1176 0 254 20160 0 malloc-16 16 649 601 16095 0 254 20000 0 epoch_record pcpu 256 4 60 4 0 62 16384 0 malloc-16384 16384 1 0 1 0 1 16384 0 malloc-16384 16384 1 0 1 0 1 16384 0 malloc-2048 2048 3 5 3 0 8 16384 0 malloc-2048 2048 3 5 3 0 8 16384 0 malloc-1024 1024 8 8 9 0 16 16384 0 malloc-512 512 0 32 1 0 30 16384 0 malloc-512 512 1 31 1 0 30 16384 0 malloc-512 512 0 32 1 0 30 16384 0 SMR CPU 32 7 504 7 0 254 16352 0 vtnet_tx_hdr 24 0 668 663497 0 254 16032 0 malloc-16 16 492 508 13042 0 254 16000 0 kenv 258 15 45 1053 0 30 15480 0 mqnode 416 3 33 3 0 30 14976 0 vmem 1856 1 7 1 0 8 14848 0 SMR SHARED 24 7 504 7 0 254 12264 0 AIOP 32 4 374 4 0 254 12096 0 malloc-16 16 14 736 1569 0 254 12000 0 malloc-16 16 38 712 164 0 254 12000 0 malloc-16 16 291 459 26523 0 254 12000 0 malloc-16 16 10 740 12 0 254 12000 0 AIOCB 552 0 21 6 0 16 11592 0 malloc-384 384 1 29 1 0 30 11520 0 malloc-384 384 2 28 2 0 30 11520 0 tfo_ccache_entries 80 1 127 1 0 126 10240 0 malloc-8192 8192 1 0 1 0 1 8192 0 malloc-8192 8192 1 0 1 0 1 8192 0 malloc-4096 4096 0 2 1 0 2 8192 0 pcpu-16 16 14 498 14 0 254 8192 0 malloc-16 16 0 500 2 0 254 8000 0 UMA Slabs 1 176 10 12 10 0 62 3872 0 KMAP ENTRY 96 12 27 14 0 0 3744 0 pcpu-4 4 1 511 1 0 254 2048 0 FFS1 dinode 128 0 0 0 0 126 0 0 ada_ccb 272 0 0 0 0 30 0 0 swblk 136 0 0 0 0 62 0 0 swpctrie 144 0 0 0 0 62 0 0 cdg_qdiffsample 16 0 0 0 0 254 0 0 pf state scrubs 40 0 0 0 0 254 0 0 pf frag entries 40 0 0 0 0 254 0 0 pf frags 248 0 0 0 0 62 0 0 pf table entries 160 0 0 0 0 254 0 0 pf table entry counters 64 0 0 0 0 254 0 0 pf source nodes 136 0 0 0 0 254 0 0 pf state keys 88 0 0 0 0 126 0 0 pf states 312 0 0 0 0 254 0 0 pf tags 104 0 0 0 0 126 0 0 pf mtags 56 0 0 0 0 254 0 0 tfo 4 0 0 0 0 254 0 0 sackhole 32 0 0 0 0 254 0 0 tcptw 72 0 0 0 0 254 0 0 ipq 56 0 0 0 0 254 0 0 sctp_asconf_ack 48 0 0 0 0 254 0 0 sctp_asconf 40 0 0 0 0 254 0 0 sctp_readq 152 0 0 0 0 254 0 0 tcp_log_id_node 120 0 0 0 0 126 0 0 tcp_log_id_bucket 176 0 0 0 0 62 0 0 tcpreass 48 0 0 0 0 254 0 0 ripcb ports 32 0 0 0 0 254 0 0 IPsec SA lft_c 16 0 0 0 0 254 0 0 itimer 352 0 0 0 0 30 0 0 AIOLIO 272 0 0 0 0 30 0 0 mqnotifier 216 0 0 0 0 62 0 0 mvdata 64 0 0 0 0 254 0 0 mqueue 248 0 0 0 0 62 0 0 TMPFS node 224 0 0 0 0 62 0 0 NCLNODE 608 0 0 0 0 16 0 0 LTS VFS Cache 360 0 0 0 0 30 0 0 L VFS Cache 320 0 0 0 0 30 0 0 STS VFS Cache 144 0 0 0 0 62 0 0 cryptop 280 0 0 0 0 30 0 0 linux_dma_object 32 0 0 0 0 254 0 0 linux_dma_pctrie 144 0 0 0 0 62 0 0 IOMMU_MAP_ENTRY 120 0 0 0 0 126 0 0 mbuf_jumbo_16k 16384 0 0 0 0 254 0 0 mbuf_jumbo_9k 9216 0 0 0 0 254 0 0 audit_record 1280 0 0 0 0 8 0 0 domainset 40 0 0 0 0 254 0 0 MAC labels 40 0 0 0 0 254 0 0 vnpbuf 2624 0 0 0 0 64 0 0 mdpbuf 2624 0 0 0 0 3 0 0 nfspbuf 2624 0 0 0 0 16 0 0 swwbuf 2624 0 0 0 0 8 0 0 swrbuf 2624 0 0 0 0 16 0 0 umtx_shm 88 0 0 0 0 126 0 0 umtx pi 96 0 0 0 0 126 0 0 rangeset pctrie nodes 144 0 0 0 0 62 0 0 malloc-65536 65536 0 0 0 0 1 0 0 malloc-65536 65536 0 0 0 0 1 0 0 malloc-65536 65536 0 0 0 0 1 0 0 malloc-32768 32768 0 0 0 0 1 0 0 malloc-32768 32768 0 0 0 0 1 0 0 malloc-32768 32768 0 0 0 0 1 0 0 malloc-32768 32768 0 0 0 0 1 0 0 malloc-16384 16384 0 0 0 0 1 0 0 malloc-16384 16384 0 0 0 0 1 0 0 malloc-8192 8192 0 0 0 0 1 0 0 malloc-4096 4096 0 0 0 0 2 0 0 pcpu-32 32 0 0 0 0 254 0 0 fakepg 104 0 0 0 0 126 0 0 UMA Hash 256 0 0 0 0 62 0 0