panic: mutex blocked lock not owned at /syzkaller/managers/main/kernel/sys/kern/sched_ule.c:2685 cpuid = 0 time = 1640953617 KDB: stack backtrace: db_trace_self_wrapper() at db_trace_self_wrapper+0xc7/frame 0xfffffe0054166190 kdb_backtrace() at kdb_backtrace+0xd3/frame 0xfffffe00541662f0 vpanic() at vpanic+0x2b8/frame 0xfffffe00541663d0 panic() at panic+0xb5/frame 0xfffffe00541664a0 __mtx_assert() at __mtx_assert+0x224/frame 0xfffffe00541664f0 sched_add() at sched_add+0x68/frame 0xfffffe0054166530 handleevents() at handleevents+0x3c7/frame 0xfffffe0054166590 hardclockintr() at hardclockintr+0x1fc/frame 0xfffffe0054166670 ipi_bitmap_handler() at ipi_bitmap_handler+0x1a8/frame 0xfffffe0054166810 Xipi_intr_bitmap_handler() at Xipi_intr_bitmap_handler+0xb3/frame 0xfffffe0054166810 --- interrupt, rip = 0xffffffff817b61b0, rsp = 0xfffffe00541668e0, rbp = 0xfffffe0054166ad0 --- pipe_write() at pipe_write+0xfb0/frame 0xfffffe0054166ad0 dofilewrite() at dofilewrite+0x13f/frame 0xfffffe0054166b50 kern_writev() at kern_writev+0xd5/frame 0xfffffe0054166c10 sys_write() at sys_write+0x1ed/frame 0xfffffe0054166d30 amd64_syscall() at amd64_syscall+0x410/frame 0xfffffe0054166f30 fast_syscall_common() at fast_syscall_common+0xf8/frame 0xfffffe0054166f30 --- syscall (4, FreeBSD ELF64, sys_write), rip = 0x464d95, rsp = 0x82804be08, rbp = 0x82804be38 --- KDB: enter: panic [ thread pid 774 tid 100123 ] Stopped at kdb_enter+0x6b: movq $0,0x250ff3a(%rip) db> db> set $lines = 0 db> set $maxwidth = 0 db> show registers cs 0x20 ds 0x3b es 0x3b fs 0x13 gs 0x1b ss 0x28 rax 0x12 rcx 0x127f9a1cf82f5d7d rdx 0x1 rbx 0 rsp 0xfffffe00541662d0 rbp 0xfffffe00541662f0 rsi 0 rdi 0xffffffff81761dfa vprintf+0x35a r8 0 r9 0xffffffff r10 0 r11 0xfffffe0092e5ca70 r12 0xfffffe0092e5c560 r13 0xfffffe0054166301 r14 0xffffffff82b547c0 .str.18 r15 0xffffffff82b547c0 .str.18 rip 0xffffffff8175535b kdb_enter+0x6b rflags 0x46 kdb_enter+0x6b: movq $0,0x250ff3a(%rip) db> show proc Process 774 (syz-fuzzer) at 0xfffffe0053df4000: state: NORMAL uid: 0 gids: 0, 0, 5 parent: pid 772 at 0xfffffe0092e2e000 ABI: FreeBSD ELF64 flag: 0x10004080 flag2: 0 arguments: ./syz-fuzzer -executor=./syz-executor -name=vm-1 -arch=amd64 -manager=10.128.0.248:47065 -sandbox=none -procs=4 -cover=true -debug=false -test=false -optional=slowdown=1 reaper: 0xfffffe0053df6000 reapsubtree: 1 sigparent: 20 vmspace: 0xfffffe00929f19f8 (map 0xfffffe00929f19f8) (map.pmap 0xfffffe00929f1ab8) (pmap 0xfffffe00929f1b20) threads: 10 100088 S uwait 0xfffffe0007975e80 syz-fuzzer 100118 S nanslp 0xffffffff83c32941 syz-fuzzer 100119 S uwait 0xfffffe0058d52880 syz-fuzzer 100120 S uwait 0xfffffe0058d52980 syz-fuzzer 100121 S uwait 0xfffffe0058d1db80 syz-fuzzer 100122 S uwait 0xfffffe0058d52a80 syz-fuzzer 100123 Run CPU 0 syz-fuzzer 100126 S uwait 0xfffffe0058d1dc80 syz-fuzzer 100127 S kqread 0xfffffe0053faca00 syz-fuzzer 103155 S uwait 0xfffffe0058d51f00 syz-fuzzer db> ps pid ppid pgrp uid state wmesg wchan cmd 8584 777 777 0 R CPU 1 syz-executor.1 8581 782 782 0 R (threaded) syz-executor.3 109668 RunQ syz-executor.3 109686 S uwait 0xfffffe0058d1ce00 syz-executor.3 8580 780 780 0 R (threaded) syz-executor.2 108772 RunQ syz-executor.2 109685 RunQ syz-executor.2 8489 1 782 0 S uwait 0xfffffe0058d1b300 syz-executor.3 8484 1 782 0 S uwait 0xfffffe0099a0fb00 syz-executor.3 8478 1 782 0 S uwait 0xfffffe0058d1c680 syz-executor.3 8473 1 782 0 S uwait 0xfffffe0058d53600 syz-executor.3 8077 1 782 0 S uwait 0xfffffe00999e1980 syz-executor.3 8073 1 782 0 S uwait 0xfffffe0058d51600 syz-executor.3 8069 1 782 0 S uwait 0xfffffe0099a0d380 syz-executor.3 8065 1 782 0 S uwait 0xfffffe0058d51900 syz-executor.3 7175 1 777 0 SV uwait 0xfffffe0099a0fc00 syz-executor.1 7166 1 777 0 SV uwait 0xfffffe0058d1d080 syz-executor.1 7157 1 777 0 SV uwait 0xfffffe0058d1b200 syz-executor.1 7148 1 777 0 SV uwait 0xfffffe0058d1be00 syz-executor.1 6751 1 6750 0 S uwait 0xfffffe0099a0dc00 syz-executor.0 6749 1 6745 0 S uwait 0xfffffe0053ef0700 syz-executor.0 6742 1 6738 0 S uwait 0xfffffe00999e1380 syz-executor.0 6737 1 6734 0 S uwait 0xfffffe00999e1180 syz-executor.0 5649 1 5648 0 S uwait 0xfffffe0007975080 syz-executor.1 5647 1 5645 0 S uwait 0xfffffe00999e1680 syz-executor.3 5643 1 5638 0 S uwait 0xfffffe0058d1c580 syz-executor.1 5642 1 5637 0 S uwait 0xfffffe0058d1c780 syz-executor.3 5641 1 5639 0 S uwait 0xfffffe0099a0f780 syz-executor.0 5636 1 5635 0 S uwait 0xfffffe0058d52e80 syz-executor.1 5634 1 5629 0 S uwait 0xfffffe0058d1c280 syz-executor.0 5633 1 5631 0 S uwait 0xfffffe0058d51b00 syz-executor.3 5627 1 5625 0 S uwait 0xfffffe0058d52480 syz-executor.0 5623 1 5621 0 S uwait 0xfffffe0058d51e00 syz-executor.0 5249 1 5248 0 S uwait 0xfffffe0058d1b400 syz-executor.2 5242 1 5236 0 S uwait 0xfffffe00999e1780 syz-executor.2 5235 1 5232 0 S uwait 0xfffffe00999e2e80 syz-executor.2 5230 1 5226 0 S uwait 0xfffffe0058d1c180 syz-executor.2 4326 1 4326 65 Ss select 0xfffffe0092930f40 dhclient 3898 0 0 0 DL mdwait 0xfffffe009b0a2000 [md3] 3892 0 0 0 DL mdwait 0xfffffe009b0a3000 [md2] 3888 0 0 0 DL mdwait 0xfffffe0053e16000 [md1] 3881 0 0 0 DL mdwait 0xfffffe0057b4e000 [md0] 3356 1 3356 0 Ss select 0xfffffe00081e51c0 dhclient 3353 1 3353 0 Ss select 0xfffffe0092931d40 dhclient 3352 0 0 0 DL aiordy 0xfffffe009a580548 [aiod4] 3351 0 0 0 DL aiordy 0xfffffe0092ff9000 [aiod3] 3350 0 0 0 DL aiordy 0xfffffe00999cc000 [aiod2] 3349 0 0 0 DL aiordy 0xfffffe009a57f548 [aiod1] 3306 1 3306 65 Ss select 0xfffffe0092931bc0 dhclient 2532 1 2532 0 Ss select 0xfffffe0092932840 dhclient 2529 1 2529 0 Ss select 0xfffffe0092932a40 dhclient 2502 1 2502 65 Ss select 0xfffffe00929323c0 dhclient 2421 1 782 0 S uwait 0xfffffe0058d1e280 syz-executor.3 2414 1 777 0 S uwait 0xfffffe0007976300 syz-executor.1 2413 1 782 0 S uwait 0xfffffe0058d52580 syz-executor.3 2408 1 777 0 S uwait 0xfffffe0007976400 syz-executor.1 2407 1 782 0 S uwait 0xfffffe0058d1bf00 syz-executor.3 2401 1 777 0 S uwait 0xfffffe00999e0f00 syz-executor.1 2398 1 777 0 S uwait 0xfffffe0058d51700 syz-executor.1 1605 1 1605 0 Ss select 0xfffffe00929322c0 dhclient 1602 1 1602 0 Ss select 0xfffffe00929326c0 dhclient 1588 1 1588 65 Ss select 0xfffffe0092932cc0 dhclient 887 1 887 0 Ss select 0xfffffe00929331c0 dhclient 874 1 874 0 Ss select 0xfffffe00929324c0 dhclient 782 774 782 0 Ss nanslp 0xffffffff83c32941 syz-executor.3 780 774 780 0 Ss nanslp 0xffffffff83c32941 syz-executor.2 777 774 777 0 Ss nanslp 0xffffffff83c32941 syz-executor.1 776 774 776 0 Rs syz-executor.0 774 772 772 0 R (threaded) syz-fuzzer 100088 S uwait 0xfffffe0007975e80 syz-fuzzer 100118 S nanslp 0xffffffff83c32941 syz-fuzzer 100119 S uwait 0xfffffe0058d52880 syz-fuzzer 100120 S uwait 0xfffffe0058d52980 syz-fuzzer 100121 S uwait 0xfffffe0058d1db80 syz-fuzzer 100122 S uwait 0xfffffe0058d52a80 syz-fuzzer 100123 Run CPU 0 syz-fuzzer 100126 S uwait 0xfffffe0058d1dc80 syz-fuzzer 100127 S kqread 0xfffffe0053faca00 syz-fuzzer 103155 S uwait 0xfffffe0058d51f00 syz-fuzzer 772 770 772 0 Ss pause 0xfffffe0092e2e0b0 csh 770 688 770 0 Ss select 0xfffffe0092932ec0 sshd 754 1 754 0 Ss+ ttyin 0xfffffe00572330b0 getty 753 1 753 0 Ss+ ttyin 0xfffffe00079664b0 getty 752 1 752 0 Ss+ ttyin 0xfffffe0007966cb0 getty 751 1 751 0 Ss+ ttyin 0xfffffe00543f74b0 getty 750 1 750 0 Ss+ ttyin 0xfffffe00079674b0 getty 749 1 749 0 Ss+ ttyin 0xfffffe00543f7cb0 getty 748 1 748 0 Ss+ ttyin 0xfffffe00543f84b0 getty 747 1 747 0 Ss+ ttyin 0xfffffe00543f8cb0 getty 746 1 746 0 Ss+ ttyin 0xfffffe00576da4b0 getty 692 1 692 0 Ss nanslp 0xffffffff83c32940 cron 688 1 688 0 Ss select 0xfffffe00571d7840 sshd 501 1 501 0 Ss select 0xfffffe00081e5440 syslogd 430 1 430 0 Ss select 0xfffffe00081e54c0 devd 429 1 429 65 Ss select 0xfffffe00081e5340 dhclient 344 1 344 0 Ss select 0xfffffe00081e5540 dhclient 341 1 341 0 Ss select 0xfffffe00571d77c0 dhclient 17 0 0 0 DL vlruwt 0xfffffe0058866548 [vnlru] 16 0 0 0 DL syncer 0xffffffff83d58140 [syncer] 15 0 0 0 DL (threaded) [bufdaemon] 100078 D qsleep 0xffffffff83d56760 [bufdaemon] 100081 D - 0xffffffff83011f80 [bufspacedaemon-0] 100094 D sdflush 0xfffffe0058ad34e8 [/ worker] 9 0 0 0 DL psleep 0xffffffff83d89c40 [vmdaemon] 8 0 0 0 DL (threaded) [pagedaemon] 100076 D psleep 0xffffffff83d7daf8 [dom0] 100079 D launds 0xffffffff83d7db04 [laundry: dom0] 100080 D umarcl 0xffffffff81e53a50 [uma] 7 0 0 0 DL - 0xffffffff839ee808 [rand_harvestq] 6 0 0 0 DL waiting 0xffffffff84493300 [sctp_iterator] 5 0 0 0 DL pftm 0xffffffff8465e530 [pf purge] 4 0 0 0 DL (threaded) [cam] 100045 D - 0xffffffff838907c0 [doneq0] 100046 D - 0xffffffff83890740 [async] 100075 D - 0xffffffff838905c0 [scanner] 14 0 0 0 DL seqstat 0xfffffe00570d2488 [sequencer 00] 3 0 0 0 DL (threaded) [crypto] 100041 D crypto_ 0xffffffff83d79340 [crypto] 100042 D crypto_ 0xfffffe0053fad030 [crypto returns 0] 100043 D crypto_ 0xfffffe0053fad080 [crypto returns 1] 13 0 0 0 DL (threaded) [geom] 100036 D - 0xffffffff83c08000 [g_event] 100037 D - 0xffffffff83c08020 [g_up] 100038 D - 0xffffffff83c08040 [g_down] 2 0 0 0 RL (threaded) [clock] 100029 CanRun [clock (0)] 100030 I [clock (1)] 12 0 0 0 WL (threaded) [intr] 100010 I [swi6: Giant taskq] 100017 I [swi5: fast taskq] 100020 I [swi6: task queue] 100031 I [swi1: netisr 0] 100032 I [swi1: hpts] 100033 I [swi1: hpts] 100034 I [swi3: busdma] 100047 I [irq24: virtio_pci0] 100048 I [irq25: virtio_pci0] 100049 I [irq26: virtio_pci0] 100050 I [irq27: virtio_pci0] 100051 I [irq28: virtio_pci1] 100052 I [irq29: virtio_pci1] 100053 I [irq30: virtio_pci1] 100054 I [irq31: virtio_pci1] 100055 I [irq32: virtio_pci1] 100060 I [irq10: virtio_pci2] 100062 I [irq1: atkbd0] 100063 I [irq12: psm0] 100064 I [swi0: uart uart++] 100067 I [swi1: pf send] 11 0 0 0 RL (threaded) [idle] 100003 CanRun [idle: cpu0] 100004 CanRun [idle: cpu1] 1 0 1 0 SLs wait 0xfffffe0053df6000 [init] 10 0 0 0 DL audit_w 0xffffffff83d79e20 [audit] 0 0 0 0 DLs (threaded) [kernel] 100000 D swapin 0xffffffff83c08aa0 [swapper] 100005 D - 0xfffffe00081dc500 [if_io_tqg_0] 100006 D - 0xfffffe00081dc400 [if_io_tqg_1] 100007 D - 0xfffffe00081dc300 [if_config_tqg_0] 100008 D - 0xfffffe00081dc200 [softirq_0] 100009 D - 0xfffffe00081dc100 [softirq_1] 100011 D - 0xfffffe00081dbd00 [aiod_kick taskq] 100012 D - 0xfffffe00081dbc00 [linuxkpi_irq_wq] 100013 D - 0xfffffe00081dbb00 [inm_free taskq] 100014 D - 0xfffffe00081dba00 [in6m_free taskq] 100015 D - 0xfffffe00081db900 [deferred_unmount ta] 100016 D - 0xfffffe00081db800 [thread taskq] 100018 D - 0xfffffe00081db500 [kqueue_ctx taskq] 100019 D - 0xfffffe00081db400 [pci_hp taskq] 100021 D - 0xfffffe00081db100 [linuxkpi_short_wq_0] 100022 D - 0xfffffe00081db100 [linuxkpi_short_wq_1] 100023 D - 0xfffffe00081db100 [linuxkpi_short_wq_2] 100024 D - 0xfffffe00081db100 [linuxkpi_short_wq_3] 100025 D - 0xfffffe00081db000 [linuxkpi_long_wq_0] 100026 D - 0xfffffe00081db000 [linuxkpi_long_wq_1] 100027 D - 0xfffffe00081db000 [linuxkpi_long_wq_2] 100028 D - 0xfffffe00081db000 [linuxkpi_long_wq_3] 100035 D - 0xfffffe0053f0ad00 [firmware taskq] 100039 D - 0xfffffe0053f0ac00 [crypto_0] 100040 D - 0xfffffe0053f0ac00 [crypto_1] 100056 D - 0xfffffe0053f0a400 [vtnet0 rxq 0] 100057 D - 0xfffffe0053f0a300 [vtnet0 txq 0] 100058 D - 0xfffffe0053f0a200 [vtnet0 rxq 1] 100059 D - 0xfffffe0053f0a100 [vtnet0 txq 1] 100061 D vtbslp 0xfffffe00571b3580 [virtio_balloon] 100065 D - 0xffffffff82b5a2a0 [deadlkres] 100070 D - 0xfffffe0053f09300 [acpi_task_0] 100071 D - 0xfffffe0053f09300 [acpi_task_1] 100072 D - 0xfffffe0053f09300 [acpi_task_2] 100073 D - 0xfffffe00081de100 [mca taskq] 100074 D - 0xfffffe0053f0a900 [CAM taskq] 8583 776 776 0 Z syz-executor.0 db> show all locks Process 774 (syz-fuzzer) thread 0xfffffe0092e5c560 (100123) exclusive sleep mutex pipe mutex (pipe mutex) r = 0 (0xfffffe0058b7e5a8) locked @ /syzkaller/managers/main/kernel/sys/kern/sys_pipe.c:1095 db> show malloc Type InUse MemUse Requests pf_hash 5 11524K 5 tcp_hpts 6 4801K 6 devbuf 4213 4323K 4245 sysctloid 35922 2116K 35993 vtbuf 24 1968K 46 kobj 322 1288K 517 pcb 481 1052K 15304 newblk 10 1027K 21237 vfscache 3 1025K 3 filedesc 90 714K 15331 inodedep 277 616K 8573 ufs_quota 1 512K 1 vfs_hash 1 512K 1 callout 2 512K 2 intr 4 472K 4 subproc 223 454K 8706 vmem 3 274K 6 sctp_stro 219 219K 3276 acpica 1674 184K 56282 filemon 22 176K 155 sctp_atcl 448 168K 11015 vnet_data 1 168K 1 tidhash 3 141K 3 linker 354 134K 382 pagedep 14 132K 7744 tfo_ccache 1 128K 1 DEVFS1 113 113K 130 sem 4 106K 4 BPF 46 88K 90 bus 973 80K 3426 mtx_pool 2 72K 2 syncache 1 68K 1 dirrem 268 67K 8257 acpitask 1 64K 1 ddb_capture 1 64K 1 module 505 64K 505 kdtrace 288 53K 18274 umtx 418 53K 418 temp 35 35K 3174 DEVFS3 132 33K 142 freefile 264 33K 8204 hostcache 1 32K 1 shm 1 32K 23 msg 4 30K 4 sctp_atky 667 28K 14799 gtaskqueue 18 26K 18 kbdmux 6 22K 6 routetbl 146 21K 946 ifaddr 70 20K 72 DEVFS_RULE 56 20K 56 lltable 55 18K 233 ufs_mount 4 17K 5 proc 3 17K 3 md_disk 8 17K 27 sctp_timw 64 16K 64 tty 16 16K 16 md_sectors 4 16K 4 cred 60 15K 315 ithread 93 15K 93 bus-sc 34 15K 1651 GEOM 89 14K 670 KTRACE 101 13K 11284 ether_multi 152 13K 188 ifnet 7 13K 7 kenv 95 12K 95 eventhandler 136 12K 136 CAM queue 5 11K 1528 rman 84 10K 425 kqueue 107 10K 8628 in6_multi 65 9K 65 bmsafemap 2 9K 8609 UART 12 9K 12 devstat 4 9K 4 ksem 1 8K 221 pf_table 4 8K 134 rpc 2 8K 2 shmfd 1 8K 33 pfs_vncache 1 8K 1 pf_ifnet 25 8K 220 pfs_nodes 20 8K 20 audit_evclass 237 8K 296 sctp_athm 448 7K 11236 sctp_map 438 7K 6510 session 53 7K 98 taskqueue 63 7K 63 pwddesc 101 7K 8651 sglist 5 7K 5 CAM DEV 3 6K 510 plimit 24 6K 477 DEVFSP 90 6K 971 ufs_dirhash 24 5K 24 UMA 276 5K 276 vt 11 5K 11 memdesc 1 4K 1 MCA 32 4K 32 evdev 4 4K 4 proc-args 128 4K 9857 acpisem 28 4K 28 selfd 55 4K 108126 lockf 32 4K 484 hhook 15 4K 17 kcovinfo 52 4K 52 ip6opt 20 3K 131 terminal 11 3K 11 pf_osfp 23 3K 23 select 19 3K 110 uidinfo 4 3K 38 local_apic 1 2K 1 io_apic 1 2K 1 fpukern_ctx 2 2K 2 ipsec-saq 2 2K 2 ip6ndp 12 2K 13 sctp_ifa 13 2K 14 Unitno 27 2K 191 CAM XPT 22 2K 543 in_multi 6 2K 21 ipsecpolicy 2 2K 2 acpidev 20 2K 20 tcp_fsb 27 2K 630 msi 9 2K 9 newdirblk 9 2K 7659 clone 9 2K 9 tun 7 2K 7 freework 5 2K 12889 softdep 1 1K 1 mkdir 8 1K 15318 freeblks 4 1K 8302 sahead 1 1K 1 secasvar 1 1K 1 nhops 6 1K 8 vnodemarker 2 1K 416 NFSD session 1 1K 1 CAM periph 4 1K 271 ipsec 3 1K 3 sctp_ifn 6 1K 14 mld 6 1K 6 ip_msource 12 1K 106 igmp 6 1K 6 toponodes 6 1K 6 isadev 6 1K 6 mount 16 1K 89 pci_link 10 1K 10 inpcbpolicy 22 1K 2900 CC Mem 11 1K 2322 crypto 4 1K 340 encap_export_host 12 1K 12 procdesc 5 1K 14 diradd 4 1K 8297 pfil 4 1K 4 osd 15 1K 2232 cdev 2 1K 2 sctp_aadr 7 1K 52 chacha20random 1 1K 1 pf_rule 2 1K 218 NFSD lckfile 1 1K 1 NFSD V4client 1 1K 1 DEVFS 9 1K 10 vnodes 1 1K 11 CAM SIM 2 1K 2 feeder 7 1K 7 tcpfunc 3 1K 3 loginclass 3 1K 6 prison 6 1K 6 lkpikmalloc 5 1K 6 aesni_data 2 1K 2 nexusdev 8 1K 8 apmdev 1 1K 1 atkbddev 2 1K 2 xform 2 1K 1209 CAM dev queue 2 1K 2 CAM I/O Scheduler 1 1K 1 CAM path 4 1K 1034 pmchooks 1 1K 1 soname 4 1K 12759 filecaps 5 1K 98 sctp_vrf 1 1K 1 vnet 1 1K 1 entropy 2 1K 45 acpiintr 1 1K 1 pmc 1 1K 1 cpus 2 1K 2 vnet_data_free 1 1K 1 Per-cpu 1 1K 1 p1003.1b 1 1K 1 mqdata 0 0K 0 pf_altq 0 0K 0 pf_temp 0 0K 0 sctp_mcore 0 0K 0 sctp_socko 0 0K 5749 sctp_iter 0 0K 50 sctp_mvrf 0 0K 0 sctp_cpal 0 0K 41 sctp_cmsg 0 0K 0 sctp_stre 0 0K 0 sctp_athi 0 0K 0 sctp_a_it 0 0K 9 sctp_stri 0 0K 408 NFSCL flayout 0 0K 0 NFSCL layout 0 0K 0 madt_table 0 0K 2 smartpqi 0 0K 0 NFSD rollback 0 0K 0 NFSCL diroff 0 0K 0 NEWdirectio 0 0K 0 ixl 0 0K 0 NEWNFSnode 0 0K 0 NFSCL lck 0 0K 0 NFSCL lckown 0 0K 0 NFSCL client 0 0K 0 NFSCL deleg 0 0K 0 NFSCL open 0 0K 0 NFSCL owner 0 0K 0 ice-resmgr 0 0K 0 ice-osdep 0 0K 0 ice 0 0K 0 iavf 0 0K 0 axgbe 0 0K 0 NFS fh 0 0K 0 NFS req 0 0K 0 NFSD usrgroup 0 0K 0 NFSD string 0 0K 0 NFSD V4lock 0 0K 0 NFSD V4state 0 0K 0 NFSD srvcache 0 0K 0 msdosfs_fat 0 0K 0 xen_intr 0 0K 0 msdosfs_mount 0 0K 0 msdosfs_node 0 0K 0 xen_hvm 0 0K 0 legacydrv 0 0K 0 qpidrv 0 0K 0 dmar_idpgtbl 0 0K 0 dmar_dom 0 0K 0 dmar_ctx 0 0K 0 DEVFS4 0 0K 0 isci 0 0K 0 iommu_dmamap 0 0K 0 hyperv_socket 0 0K 0 bxe_ilt 0 0K 0 DEVFS2 0 0K 0 xenbus 0 0K 0 gntdev 0 0K 0 privcmd_dev 0 0K 0 evtchn_dev 0 0K 0 vm_fictitious 0 0K 0 xenstore 0 0K 0 ciss_data 0 0K 0 BACKLIGHT 0 0K 0 xnb 0 0K 0 xbbd 0 0K 0 xbd 0 0K 0 Balloon 0 0K 0 sysmouse 0 0K 0 vtfont 0 0K 0 UMAHash 0 0K 0 ath_hal 0 0K 0 athdev 0 0K 0 vm_pgdata 0 0K 0 jblocks 0 0K 0 savedino 0 0K 7682 sentinel 0 0K 0 jfsync 0 0K 0 jtrunc 0 0K 0 sbdep 0 0K 205 jsegdep 0 0K 0 jseg 0 0K 0 jfreefrag 0 0K 0 jfreeblk 0 0K 0 jnewblk 0 0K 0 jmvref 0 0K 0 jremref 0 0K 0 jaddref 0 0K 0 freedep 0 0K 0 freefrag 0 0K 50 allocindir 0 0K 0 indirdep 0 0K 6965 allocdirect 0 0K 0 ufs_trim 0 0K 0 mactemp 0 0K 0 audit_trigger 0 0K 0 audit_pipe_presel 0 0K 0 audit_pipeent 0 0K 0 audit_pipe 0 0K 0 audit_evname 0 0K 0 audit_bsm 0 0K 0 audit_gidset 0 0K 0 audit_text 0 0K 0 audit_path 0 0K 0 audit_data 0 0K 0 audit_cred 0 0K 0 ata_pci 0 0K 0 ata_dma 0 0K 0 ata_generic 0 0K 0 scsi_da 0 0K 69 pvscsi 0 0K 0 ata_da 0 0K 0 scsi_ch 0 0K 0 scsi_cd 0 0K 0 AHCI driver 0 0K 0 agp 0 0K 0 USBdev 0 0K 0 USB 0 0K 0 ktls_ocf 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5E_TLS 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EN 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5DUMP 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 seq_file 0 0K 0 lkpiskb 0 0K 0 radix 0 0K 0 idr 0 0K 0 lkpindev 0 0K 0 lkpifw 0 0K 0 lkpi80211 0 0K 0 NLM 0 0K 0 ipsec-spdcache 0 0K 0 ipsec-reg 0 0K 0 ipsec-misc 0 0K 0 ipsecrequest 0 0K 0 ip6_msource 0 0K 0 ip6_moptions 0 0K 0 in6_mfilter 0 0K 0 frag6 0 0K 0 tcplog 0 0K 0 tcp_hwpace 0 0K 0 nvme_da 0 0K 0 LRO 0 0K 0 ip_moptions 0 0K 48 in_mfilter 0 0K 192 ipid 0 0K 0 80211scan 0 0K 0 80211ratectl 0 0K 0 80211power 0 0K 0 80211nodeie 0 0K 0 80211node 0 0K 0 80211mesh_gt 0 0K 0 80211mesh_rt 0 0K 0 80211perr 0 0K 0 80211prep 0 0K 0 80211preq 0 0K 0 80211dfs 0 0K 0 80211crypto 0 0K 0 80211vap 0 0K 0 iflib 0 0K 0 vlan 0 0K 0 gif 0 0K 0 ifdescr 0 0K 0 zlib 0 0K 0 fadvise 0 0K 0 VN POLL 0 0K 7 statfs 0 0K 7918 namei_tracker 0 0K 25 export_host 0 0K 0 cl_savebuf 0 0K 14 acpipwr 0 0K 0 acpi_perf 0 0K 0 acpicmbat 0 0K 0 twsbuf 0 0K 0 twe_commands 0 0K 0 tcp_log_dev 0 0K 256 midi buffers 0 0K 0 mixer 0 0K 0 ac97 0 0K 0 hdacc 0 0K 0 hdac 0 0K 0 hdaa 0 0K 0 SIIS driver 0 0K 0 CAM CCB 0 0K 523 PUC 0 0K 0 biobuf 0 0K 0 aios 0 0K 5 lio 0 0K 4 acl 0 0K 0 ppbusdev 0 0K 0 mbuf_tag 0 0K 177 ktls 0 0K 0 agtiapi_MemAlloc malloc 0 0K 0 osti_cacheable 0 0K 0 tempbuff 0 0K 0 tempbuff 0 0K 0 ag_tgt_map_t malloc 0 0K 0 ag_slr_map_t malloc 0 0K 0 lDevFlags * malloc 0 0K 0 tiDeviceHandle_t * malloc 0 0K 0 ag_portal_data_t malloc 0 0K 0 ag_device_t malloc 0 0K 0 STLock malloc 0 0K 0 CCB List 0 0K 0 sr_iov 0 0K 0 OCS 0 0K 0 OCS 0 0K 0 nvme 0 0K 0 nvd 0 0K 0 netmap 0 0K 0 mwldev 0 0K 0 MVS driver 0 0K 0 CAM ccb queue 0 0K 0 mrsasbuf 0 0K 0 mpt_user 0 0K 0 accf 0 0K 0 pts 0 0K 0 iov 0 0K 19903 ioctlops 0 0K 1284 eventfd 0 0K 0 Witness 0 0K 0 stack 0 0K 0 mps_user 0 0K 0 MPSSAS 0 0K 0 mps 0 0K 0 mpr_user 0 0K 0 MPRSAS 0 0K 0 mpr 0 0K 0 sbuf 0 0K 336 mfibuf 0 0K 0 firmware 0 0K 0 compressor 0 0K 0 SWAP 0 0K 0 malodev 0 0K 0 sysctltmp 0 0K 888 sysctl 0 0K 3 LED 0 0K 0 ekcd 0 0K 0 dumper 0 0K 0 sendfile 0 0K 0 rctl 0 0K 0 ix_sriov 0 0K 0 aacraidcam 0 0K 0 aacraid_buf 0 0K 0 ix 0 0K 0 cache 0 0K 0 ipsbuf 0 0K 0 prison_racct 0 0K 0 Fail Points 0 0K 0 sigio 0 0K 1 filedesc_to_leader 0 0K 0 pwd 0 0K 0 tty console 0 0K 0 aaccam 0 0K 0 aacbuf 0 0K 0 zstd 0 0K 0 XZ_DEC 0 0K 0 nvlist 0 0K 0 SCSI ENC 0 0K 0 SCSI sa 0 0K 0 scsi_pass 0 0K 0 isofs_node 0 0K 0 isofs_mount 0 0K 0 tr_raid5_data 0 0K 0 tr_raid1e_data 0 0K 0 tr_raid1_data 0 0K 0 tr_raid0_data 0 0K 0 tr_concat_data 0 0K 0 md_sii_data 0 0K 0 md_promise_data 0 0K 0 md_nvidia_data 0 0K 0 md_jmicron_data 0 0K 0 md_intel_data 0 0K 0 md_ddf_data 0 0K 0 raid_data 0 0K 96 geom_flashmap 0 0K 0 tmpfs dir 0 0K 0 tmpfs name 0 0K 0 tmpfs mount 0 0K 0 NFS FHA 0 0K 0 newnfsmnt 0 0K 0 newnfsclient_req 0 0K 0 NFSCL layrecall 0 0K 0 NFSCL session 0 0K 0 NFSCL sockreq 0 0K 0 NFSCL devinfo 0 0K 0 db> show uma Zone Size Used Free Requests Sleeps Bucket Total Mem XFree mbuf_jumbo_page 4096 8321 2347 51183 0 254 43696128 0 tcp_log 416 0 19305 363264 0 254 8030880 0 mbuf 256 9163 6590 1703200 0 254 4032768 0 pbuf 2624 0 762 0 0 2 1999488 0 malloc-384 384 4612 218 15179 0 30 1854720 0 malloc-4096 4096 447 3 9748 0 2 1843200 0 BUF TRIE 144 273 11515 14006 0 62 1697472 0 mbuf_cluster 2048 762 0 762 0 254 1560576 0 malloc-128 128 11801 196 12221 0 126 1535616 0 tcp_bbr_map 128 0 11873 386181 0 126 1519744 0 UMA Slabs 0 112 12555 3 12555 0 126 1406496 0 RADIX NODE 144 7845 1278 195817 0 62 1313712 0 vmem btag 56 22922 85 22922 0 254 1288392 0 sctp_asoc 2288 219 291 3255 0 254 1166880 0 malloc-16384 16384 45 5 7664 0 1 819200 0 256 Bucket 2048 365 19 7510 0 8 786432 0 VM OBJECT 264 2826 84 192728 0 30 768240 0 FFS inode 1160 554 34 8789 0 8 682080 0 malloc-2048 2048 231 97 8358 0 8 671744 0 ertt_txseginfo 40 3 15551 1183954 0 254 622160 0 sctp_ep 1208 229 281 7556 0 254 616080 0 tcpcb 1096 11 500 2221 0 254 560056 0 socket 960 58 450 11873 0 254 487680 0 malloc-1024 1024 359 105 3379 0 16 475136 0 MAP ENTRY 96 4416 372 511893 0 126 459648 0 malloc-65536 65536 6 0 6 0 1 393216 0 lkpimm 168 1 2327 1 0 62 391104 0 lkpicurr 168 2 2326 2 0 62 391104 0 sctp_raddr 736 226 291 3351 0 254 380512 0 THREAD 1808 186 23 9689 0 8 377872 0 malloc-65536 65536 0 5 136 0 1 327680 0 tcp_inpcb 424 11 754 2221 0 254 324360 0 sctp_chunk 152 201 1853 13882 0 254 312208 0 VNODE 448 595 98 8832 0 30 310464 0 malloc-8192 8192 34 2 174 0 1 294912 0 malloc-64 64 3950 397 4193 0 254 278208 0 malloc-16 16 14883 367 21405 0 254 244000 0 malloc-32768 32768 0 7 6557 0 1 229376 0 DEVCTL 1024 0 220 162 0 0 225280 0 malloc-384 384 276 294 8573 0 30 218880 0 ripcb 424 5 508 216 0 254 217512 0 udplite_inpcb 424 0 513 103 0 254 217512 0 udp_inpcb 424 6 507 360 0 254 217512 0 VMSPACE 2552 76 8 8538 0 4 214368 0 malloc-256 256 11 799 29176 0 62 207360 0 malloc-256 256 476 334 17177 0 62 207360 0 128 Bucket 1024 154 41 3845 0 16 199680 0 malloc-65536 65536 1 2 56 0 1 196608 0 malloc-65536 65536 1 2 307 0 1 196608 0 mbuf_packet 256 185 577 34258 0 254 195072 0 malloc-128 128 1380 139 27073 0 126 194432 0 UMA Zones 768 248 1 248 0 16 191232 0 FPU_save_area 832 187 38 9873 0 16 187200 0 malloc-32 32 5405 265 5686 0 254 181440 0 PROC 1352 101 19 8584 0 8 162240 0 FFS2 dinode 256 554 76 8789 0 62 161280 0 malloc-128 128 726 421 16992 0 126 146816 0 S VFS Cache 104 1023 381 9475 0 126 146016 0 unpcb 256 20 490 1349 0 254 130560 0 malloc-256 256 274 236 8563 0 62 130560 0 clpbuf 2624 0 48 613 0 16 125952 0 filedesc0 1072 101 11 8651 0 8 120064 0 ksiginfo 112 98 946 470 0 126 116928 0 malloc-16384 16384 3 4 7 0 1 114688 0 malloc-256 256 292 143 10476 0 62 111360 0 malloc-4096 4096 25 1 36 0 2 106496 0 malloc-384 384 215 55 1014 0 30 103680 0 64 Bucket 512 106 94 10611 0 30 102400 0 malloc-128 128 595 180 1756 0 126 99200 0 malloc-32768 32768 0 3 120 0 1 98304 0 g_bio 408 0 240 113246 0 30 97920 0 UMA Kegs 384 234 9 234 0 30 93312 0 malloc-256 256 155 160 9637 0 62 80640 0 malloc-256 256 113 202 13699 0 62 80640 0 sctp_readq 152 0 520 263 0 254 79040 0 malloc-64 64 785 286 27321 0 254 68544 0 malloc-128 128 317 210 8441 0 126 67456 0 malloc-65536 65536 1 0 1 0 1 65536 0 malloc-65536 65536 0 1 8 0 1 65536 0 malloc-32768 32768 2 0 2 0 1 65536 0 malloc-2048 2048 14 18 651 0 8 65536 0 malloc-2048 2048 7 25 562 0 8 65536 0 32 Bucket 256 91 164 8581 0 62 65280 0 sctp_stream_msg_out 112 101 439 867 0 254 60480 0 malloc-64 64 204 615 117906 0 254 52416 0 malloc-64 64 580 239 4987 0 254 52416 0 malloc-64 64 476 343 1051 0 254 52416 0 malloc-128 128 179 224 18994 0 126 51584 0 malloc-256 256 9 186 8767 0 62 49920 0 DIRHASH 1024 34 14 34 0 16 49152 0 NAMEI 1024 0 48 49947 0 16 49152 0 malloc-16384 16384 3 0 18 0 1 49152 0 malloc-8192 8192 3 3 63 0 1 49152 0 malloc-1024 1024 3 45 1769 0 16 49152 0 pcpu-8 8 5014 1130 10082 0 254 49152 0 da_ccb 544 0 84 28935 0 16 45696 0 tcp_bbr_pcb 832 0 54 1090 0 16 44928 0 syncache 168 0 264 4 0 254 44352 0 Files 80 258 292 29967 0 126 44000 0 malloc-4096 4096 4 6 7954 0 2 40960 0 malloc-4096 4096 5 5 25 0 2 40960 0 Mountpoints 2752 2 12 2 0 4 38528 0 pipe 744 23 27 515 0 16 37200 0 hostcache 64 2 565 2 0 254 36288 0 PWD 32 58 1076 7688 0 254 36288 0 malloc-64 64 62 505 876 0 254 36288 0 malloc-64 64 125 442 13009 0 254 36288 0 malloc-64 64 3 564 55 0 254 36288 0 malloc-32 32 543 591 12303 0 254 36288 0 16 Bucket 144 92 160 1447 0 62 36288 0 malloc-128 128 34 245 854 0 126 35712 0 malloc-128 128 12 267 317 0 126 35712 0 routing nhops 256 27 108 34 0 62 34560 0 ttyoutq 256 72 63 160 0 62 34560 0 malloc-256 256 36 99 1366 0 62 34560 0 TURNSTILE 136 210 42 210 0 62 34272 0 malloc-32768 32768 1 0 1 0 1 32768 0 malloc-16384 16384 0 2 160 0 1 32768 0 malloc-8192 8192 4 0 4 0 1 32768 0 malloc-2048 2048 0 16 16 0 8 32768 0 malloc-2048 2048 7 9 7 0 8 32768 0 malloc-2048 2048 4 12 333 0 8 32768 0 malloc-2048 2048 1 15 75 0 8 32768 0 malloc-2048 2048 5 11 196 0 8 32768 0 malloc-1024 1024 12 20 169 0 16 32768 0 malloc-1024 1024 3 29 3 0 16 32768 0 malloc-1024 1024 8 24 48 0 16 32768 0 malloc-512 512 1 63 175 0 30 32768 0 malloc-512 512 1 63 15 0 30 32768 0 malloc-512 512 1 63 409 0 30 32768 0 malloc-512 512 1 63 14 0 30 32768 0 malloc-512 512 14 50 506 0 30 32768 0 malloc-512 512 9 55 49 0 30 32768 0 pcpu-64 64 482 30 482 0 254 32768 0 tcp_rack_pcb 896 3 33 606 0 16 32256 0 KNOTE 160 29 171 69040 0 62 32000 0 ttyinq 160 135 65 300 0 62 32000 0 tcp_rack_map 120 6 258 1402 0 126 31680 0 cpuset 104 7 272 179 0 126 29016 0 sctp_asconf_ack 48 0 588 4 0 254 28224 0 sctp_laddr 48 76 512 3285 0 254 28224 0 tcp_inpcb 32 5 877 1520 0 254 28224 0 malloc-32 32 288 594 16174 0 254 28224 0 4 Bucket 48 6 582 216 0 254 28224 0 2 Bucket 32 72 810 2091 0 254 28224 0 AIO 208 0 133 36 0 62 27664 0 malloc-8192 8192 1 2 102 0 1 24576 0 malloc-8192 8192 1 2 8 0 1 24576 0 malloc-8192 8192 3 0 3 0 1 24576 0 malloc-4096 4096 2 4 5 0 2 24576 0 rtentry 176 30 108 34 0 62 24288 0 PGRP 88 53 223 120 0 126 24288 0 rl_entry 40 105 501 105 0 254 24240 0 8 Bucket 80 58 242 1909 0 126 24000 0 malloc-384 384 11 49 18 0 30 23040 0 malloc-384 384 1 59 78 0 30 23040 0 malloc-384 384 24 36 27 0 30 23040 0 malloc-384 384 1 59 34 0 30 23040 0 domainset 40 0 567 130 0 254 22680 0 SLEEPQUEUE 88 210 46 210 0 126 22528 0 udplite_inpcb 32 0 630 15 0 254 20160 0 udpcb 32 6 624 463 0 254 20160 0 udp_inpcb 32 3 627 63 0 254 20160 0 ertt 72 11 269 2221 0 126 20160 0 malloc-32 32 80 550 169 0 254 20160 0 malloc-32 32 66 564 799 0 254 20160 0 malloc-32 32 156 474 1047 0 254 20160 0 malloc-32 32 101 529 6148 0 254 20160 0 malloc-32 32 32 598 180 0 254 20160 0 malloc-16 16 486 764 11347 0 254 20000 0 malloc-16 16 463 787 34154 0 254 20000 0 cryptop 280 0 70 51 0 30 19600 0 AIOCB 552 0 35 40 0 16 19320 0 L VFS Cache 320 0 60 4 0 30 19200 0 AIOLIO 272 0 70 4 0 30 19040 0 epoch_record pcpu 256 4 60 4 0 62 16384 0 malloc-16384 16384 1 0 1 0 1 16384 0 malloc-4096 4096 0 4 6 0 2 16384 0 malloc-4096 4096 0 4 193 0 2 16384 0 malloc-1024 1024 2 14 171 0 16 16384 0 malloc-1024 1024 4 12 4 0 16 16384 0 malloc-1024 1024 4 12 37 0 16 16384 0 malloc-512 512 0 32 1 0 30 16384 0 SMR CPU 32 7 504 7 0 254 16352 0 tcpreass 48 0 336 1 0 254 16128 0 vtnet_tx_hdr 24 0 668 449152 0 254 16032 0 kenv 258 15 45 1041 0 30 15480 0 mqnode 416 3 33 3 0 30 14976 0 vmem 1856 1 7 1 0 8 14848 0 SMR SHARED 24 7 504 7 0 254 12264 0 AIOP 32 4 374 4 0 254 12096 0 malloc-16 16 15 735 172 0 254 12000 0 malloc-16 16 331 419 606 0 254 12000 0 malloc-16 16 5 745 10044 0 254 12000 0 malloc-16 16 266 484 10579 0 254 12000 0 malloc-16 16 17 733 33 0 254 12000 0 malloc-8192 8192 1 0 1 0 1 8192 0 pcpu-16 16 14 498 14 0 254 8192 0 UMA Slabs 1 176 11 11 11 0 62 3872 0 KMAP ENTRY 96 12 27 14 0 0 3744 0 md3 512 0 0 0 0 30 0 0 md2 512 0 0 0 0 30 0 0 md1 512 0 0 0 0 30 0 0 md0 512 0 0 0 0 30 0 0 FFS1 dinode 128 0 0 0 0 126 0 0 ada_ccb 272 0 0 0 0 30 0 0 swblk 136 0 0 0 0 62 0 0 swpctrie 144 0 0 0 0 62 0 0 cdg_qdiffsample 16 0 0 0 0 254 0 0 sctp_asconf 40 0 0 0 0 254 0 0 pf state scrubs 40 0 0 0 0 254 0 0 pf frag entries 40 0 0 0 0 254 0 0 pf frags 248 0 0 0 0 62 0 0 pf table entries 160 0 0 0 0 62 0 0 pf table entry counters 64 0 0 0 0 254 0 0 pf source nodes 136 0 0 0 0 254 0 0 pf state keys 88 0 0 0 0 126 0 0 pf states 312 0 0 0 0 0 0 0 pf tags 104 0 0 0 0 126 0 0 pf mtags 48 0 0 0 0 254 0 0 IPsec SA lft_c 16 0 0 0 0 254 0 0 ripcb 32 0 0 0 0 254 0 0 tcp_log_node 120 0 0 0 0 126 0 0 tcp_log_bucket 176 0 0 0 0 62 0 0 tfo_ccache_entries 80 0 0 0 0 126 0 0 tfo 4 0 0 0 0 254 0 0 sackhole 32 0 0 0 0 254 0 0 tcptw 72 0 0 0 0 254 0 0 ipq 56 0 0 0 0 254 0 0 itimer 352 0 0 0 0 30 0 0 NCLNODE 608 0 0 0 0 16 0 0 TMPFS node 224 0 0 0 0 62 0 0 mqnotifier 216 0 0 0 0 62 0 0 mvdata 64 0 0 0 0 254 0 0 mqueue 248 0 0 0 0 62 0 0 LTS VFS Cache 360 0 0 0 0 30 0 0 STS VFS Cache 144 0 0 0 0 62 0 0 linux_dma_object 32 0 0 0 0 254 0 0 linux_dma_pctrie 144 0 0 0 0 62 0 0 IOMMU_MAP_ENTRY 120 0 0 0 0 126 0 0 mbuf_jumbo_16k 16384 0 0 0 0 254 0 0 mbuf_jumbo_9k 9216 0 0 0 0 254 0 0 audit_record 1280 0 0 0 0 8 0 0 MAC labels 40 0 0 0 0 254 0 0 vnpbuf 2624 0 0 0 0 64 0 0 mdpbuf 2624 0 0 0 0 3 0 0 nfspbuf 2624 0 0 0 0 16 0 0 swwbuf 2624 0 0 0 0 8 0 0 swrbuf 2624 0 0 0 0 16 0 0 umtx_shm 88 0 0 0 0 126 0 0 umtx pi 96 0 0 0 0 126 0 0 rangeset pctrie nodes 144 0 0 0 0 62 0 0 malloc-65536 65536 0 0 0 0 1 0 0 malloc-65536 65536 0 0 0 0 1 0 0 malloc-32768 32768 0 0 0 0 1 0 0 malloc-32768 32768 0 0 0 0 1 0 0 malloc-32768 32768 0 0 0 0 1 0 0 malloc-32768 32768 0 0 0 0 1 0 0 malloc-16384 16384 0 0 0 0 1 0 0 malloc-16384 16384 0 0 0 0 1 0 0 malloc-16384 16384 0 0 0 0 1 0 0 malloc-8192 8192 0 0 0 0 1 0 0 malloc-4096 4096 0 0 0 0 2 0 0 malloc-512 512 0 0 0 0 30 0 0 malloc-384 384 0 0 0 0 30 0 0 pcpu-32 32 0 0 0 0 254 0 0 pcpu-4 4 0 0 0 0 254 0 0 fakepg 104 0 0 0 0 126 0 0 UMA Hash 256 0 0 0 0 62 0 0