panic: mutex blocked lock not owned at /syzkaller/managers/main/kernel/sys/kern/sched_ule.c:2685 cpuid = 0 time = 1640990927 KDB: stack backtrace: db_trace_self_wrapper() at db_trace_self_wrapper+0xc7/frame 0xfffffe00540f5650 kdb_backtrace() at kdb_backtrace+0xd3/frame 0xfffffe00540f57b0 vpanic() at vpanic+0x2b8/frame 0xfffffe00540f5890 panic() at panic+0xb5/frame 0xfffffe00540f5960 __mtx_assert() at __mtx_assert+0x224/frame 0xfffffe00540f59b0 sched_add() at sched_add+0x68/frame 0xfffffe00540f59f0 handleevents() at handleevents+0x3c7/frame 0xfffffe00540f5a50 timercb() at timercb+0x3c1/frame 0xfffffe00540f5b10 lapic_handle_timer() at lapic_handle_timer+0x143/frame 0xfffffe00540f5b50 Xtimerint() at Xtimerint+0xb1/frame 0xfffffe00540f5b50 --- interrupt, rip = 0xffffffff8125c005, rsp = 0xfffffe00540f5c20, rbp = 0xfffffe00540f5c70 --- virtqueue_poll() at virtqueue_poll+0x285/frame 0xfffffe00540f5c70 vtrnd_read() at vtrnd_read+0x1e4/frame 0xfffffe00540f5d90 random_kthread() at random_kthread+0x270/frame 0xfffffe00540f5ef0 fork_exit() at fork_exit+0xd0/frame 0xfffffe00540f5f30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe00540f5f30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- KDB: enter: panic [ thread pid 7 tid 100069 ] Stopped at kdb_enter+0x6b: movq $0,0x250ff3a(%rip) db> db> set $lines = 0 db> set $maxwidth = 0 db> show registers cs 0x20 ds 0x3b es 0x3b fs 0x13 gs 0x1b ss 0x28 rax 0x12 rcx 0xf59bcfa4faa78ac4 rdx 0xdffff7c000000000 rbx 0 rsp 0xfffffe00540f5790 rbp 0xfffffe00540f57b0 rsi 0x1 rdi 0 r8 0x3 r9 0xffffffff r10 0 r11 0 r12 0xfffffe00543dae40 r13 0xfffffe00540f5801 r14 0xffffffff82b547c0 .str.18 r15 0xffffffff82b547c0 .str.18 rip 0xffffffff8175535b kdb_enter+0x6b rflags 0x46 kdb_enter+0x6b: movq $0,0x250ff3a(%rip) db> show proc Process 7 (rand_harvestq) at 0xfffffe0053df6a90: state: NORMAL uid: 0 gids: 0 parent: pid 0 at 0xffffffff83c08aa0 ABI: null flag: 0x10000204 flag2: 0 reaper: 0xffffffff83c08aa0 reapsubtree: 7 sigparent: 20 vmspace: 0xffffffff83c09a40 (map 0xffffffff83c09a40) (map.pmap 0xffffffff83c09b00) (pmap 0xffffffff83c09b68) threads: 1 100069 Run CPU 0 [rand_harvestq] db> ps pid ppid pgrp uid state wmesg wchan cmd 44008 1 791 0 S uwait 0xfffffe0058d6e680 syz-executor.3 43988 1 791 0 S uwait 0xfffffe0058d6d880 syz-executor.3 43950 1 782 0 S uwait 0xfffffe0058d6d080 syz-executor.1 43838 1 791 0 S uwait 0xfffffe0058d6d180 syz-executor.3 43349 1 791 0 SV uwait 0xfffffe0099915580 syz-executor.3 43336 1 791 0 SV uwait 0xfffffe0058c8b780 syz-executor.3 36346 1 783 0 S uwait 0xfffffe009b3e7400 syz-executor.2 36345 1 783 0 SV uwait 0xfffffe0099914780 syz-executor.2 36339 1 783 0 SV uwait 0xfffffe0058c8b400 syz-executor.2 36334 36333 783 0 S uwait 0xfffffe0058d6e480 syz-executor.2 36333 1 783 0 SV wait 0xfffffe0058c84548 syz-executor.2 36321 36320 783 0 S uwait 0xfffffe0058d6df00 syz-executor.2 36320 1 783 0 SV wait 0xfffffe009a596a90 syz-executor.2 35634 1 783 0 SV uwait 0xfffffe0058d6db80 syz-executor.2 35625 1 783 0 SV uwait 0xfffffe009b4d7000 syz-executor.2 35616 1 783 0 SV uwait 0xfffffe009b3e7580 syz-executor.2 19547 1 783 0 S uwait 0xfffffe0058d6da80 syz-executor.2 19545 1 783 0 S uwait 0xfffffe0007976400 syz-executor.2 19123 1 782 0 S uwait 0xfffffe0058d6c180 syz-executor.1 4413 1 781 0 S uwait 0xfffffe0058c8b280 syz-executor.0 4396 1 781 0 S uwait 0xfffffe0058c8cd80 syz-executor.0 3221 0 0 0 DL aiordy 0xfffffe0092ea9000 [aiod4] 3220 0 0 0 DL aiordy 0xfffffe0058b7fa90 [aiod3] 3219 0 0 0 DL aiordy 0xfffffe0099c1b000 [aiod2] 3218 0 0 0 DL aiordy 0xfffffe0092eaaa90 [aiod1] 2822 1 2822 65 Ss select 0xfffffe0099c55a40 dhclient 2328 1 2328 0 Ss select 0xfffffe009293ad40 dhclient 2324 1 2324 0 Ss select 0xfffffe009293aec0 dhclient 2303 1 2303 65 Ss select 0xfffffe009293a840 dhclient 1902 1 1902 0 Ss select 0xfffffe009293ac40 dhclient 1899 1 1899 0 Ss select 0xfffffe009293c6c0 dhclient 1876 1 1876 65 Ss select 0xfffffe009293b040 dhclient 1487 1 1487 0 Ss select 0xfffffe009293adc0 dhclient 1484 1 1484 0 Ss select 0xfffffe009293aa40 dhclient 1465 1 1465 65 Ss select 0xfffffe009293b2c0 dhclient 1046 1 1046 0 Ss select 0xfffffe00081e52c0 dhclient 1041 1 1041 0 Ss select 0xfffffe009293b8c0 dhclient 890 1 791 0 S uwait 0xfffffe0058c8c900 syz-executor.3 885 1 791 0 S uwait 0xfffffe0058c8d180 syz-executor.3 882 1 791 0 S uwait 0xfffffe0058c8ca00 syz-executor.3 876 1 791 0 S uwait 0xfffffe0058d6ec80 syz-executor.3 791 778 791 0 Ss piperd 0xfffffe0092e922e8 syz-executor.3 782 778 782 0 Ss piperd 0xfffffe0058b9aba0 syz-executor.1 778 776 776 0 S (threaded) syz-fuzzer 100108 S uwait 0xfffffe0058c8d600 syz-fuzzer 100116 S uwait 0xfffffe0007979500 syz-fuzzer 100117 S uwait 0xfffffe0007979600 syz-fuzzer 100118 S uwait 0xfffffe0007979700 syz-fuzzer 100119 S uwait 0xfffffe0058c8c500 syz-fuzzer 100120 S uwait 0xfffffe0058c8c600 syz-fuzzer 100121 S uwait 0xfffffe0058c8c700 syz-fuzzer 100122 S uwait 0xfffffe0007979800 syz-fuzzer 100123 S uwait 0xfffffe0058c8c800 syz-fuzzer 109381 S kqread 0xfffffe0058d53a00 syz-fuzzer 109382 S uwait 0xfffffe0058d6e780 syz-fuzzer 776 774 776 0 Ss pause 0xfffffe0058b7e5f8 csh 774 688 774 0 Ss select 0xfffffe009293d040 sshd 754 1 754 0 Ss+ ttyin 0xfffffe00572330b0 getty 753 1 753 0 Ss+ ttyin 0xfffffe00543f68b0 getty 752 1 752 0 Ss+ ttyin 0xfffffe00543f70b0 getty 751 1 751 0 Ss+ ttyin 0xfffffe0007966cb0 getty 750 1 750 0 Ss+ ttyin 0xfffffe00079674b0 getty 749 1 749 0 Ss+ ttyin 0xfffffe00543f7cb0 getty 748 1 748 0 Ss+ ttyin 0xfffffe00543f84b0 getty 747 1 747 0 Ss+ ttyin 0xfffffe00543f8cb0 getty 746 1 746 0 Ss+ ttyin 0xfffffe00576da4b0 getty 692 1 692 0 Ss nanslp 0xffffffff83c32941 cron 688 1 688 0 Ss select 0xfffffe00081e5340 sshd 501 1 501 0 Ss select 0xfffffe00571d7840 syslogd 430 1 430 0 Ss select 0xfffffe00571d77c0 devd 429 1 429 65 Ss select 0xfffffe00081e5240 dhclient 344 1 344 0 Ss select 0xfffffe00081e53c0 dhclient 341 1 341 0 Ss select 0xfffffe00081e54c0 dhclient 17 0 0 0 DL vlruwt 0xfffffe0058ac9a90 [vnlru] 16 0 0 0 DL syncer 0xffffffff83d58140 [syncer] 15 0 0 0 DL (threaded) [bufdaemon] 100078 D qsleep 0xffffffff83d56760 [bufdaemon] 100081 D - 0xffffffff83011f80 [bufspacedaemon-0] 100093 D sdflush 0xfffffe0058b1fce8 [/ worker] 9 0 0 0 DL psleep 0xffffffff83d89c40 [vmdaemon] 8 0 0 0 DL (threaded) [pagedaemon] 100076 D psleep 0xffffffff83d7daf8 [dom0] 100079 D launds 0xffffffff83d7db04 [laundry: dom0] 100080 D umarcl 0xffffffff81e53a50 [uma] 7 0 0 0 RL CPU 0 [rand_harvestq] 6 0 0 0 DL waiting 0xffffffff848d2300 [sctp_iterator] 5 0 0 0 DL pftm 0xffffffff8437e530 [pf purge] 4 0 0 0 DL (threaded) [cam] 100045 D - 0xffffffff838907c0 [doneq0] 100046 D - 0xffffffff83890740 [async] 100075 D - 0xffffffff838905c0 [scanner] 14 0 0 0 DL seqstat 0xfffffe00570d2488 [sequencer 00] 3 0 0 0 DL (threaded) [crypto] 100041 D crypto_ 0xffffffff83d79340 [crypto] 100042 D crypto_ 0xfffffe0053fad030 [crypto returns 0] 100043 D crypto_ 0xfffffe0053fad080 [crypto returns 1] 13 0 0 0 DL (threaded) [geom] 100036 D - 0xffffffff83c08000 [g_event] 100037 D - 0xffffffff83c08020 [g_up] 100038 D - 0xffffffff83c08040 [g_down] 2 0 0 0 RL (threaded) [clock] 100029 CanRun [clock (0)] 100030 I [clock (1)] 12 0 0 0 WL (threaded) [intr] 100010 I [swi6: Giant taskq] 100017 I [swi5: fast taskq] 100020 I [swi6: task queue] 100031 I [swi1: netisr 0] 100032 I [swi1: hpts] 100033 I [swi1: hpts] 100034 I [swi3: busdma] 100047 I [irq24: virtio_pci0] 100048 I [irq25: virtio_pci0] 100049 I [irq26: virtio_pci0] 100050 I [irq27: virtio_pci0] 100051 I [irq28: virtio_pci1] 100052 I [irq29: virtio_pci1] 100053 I [irq30: virtio_pci1] 100054 I [irq31: virtio_pci1] 100055 I [irq32: virtio_pci1] 100060 I [irq10: virtio_pci2] 100062 I [irq1: atkbd0] 100063 I [irq12: psm0] 100064 I [swi0: uart uart++] 100067 I [swi1: pf send] 11 0 0 0 RL (threaded) [idle] 100003 CanRun [idle: cpu0] 100004 Run CPU 1 [idle: cpu1] 1 0 1 0 SLs wait 0xfffffe0053df6000 [init] 10 0 0 0 DL audit_w 0xffffffff83d79e20 [audit] 0 0 0 0 DLs (threaded) [kernel] 100000 D swapin 0xffffffff83c08aa0 [swapper] 100005 D - 0xfffffe00081dc500 [if_io_tqg_0] 100006 D - 0xfffffe00081dc400 [if_io_tqg_1] 100007 D - 0xfffffe00081dc300 [if_config_tqg_0] 100008 D - 0xfffffe00081dc200 [softirq_0] 100009 D - 0xfffffe00081dc100 [softirq_1] 100011 D - 0xfffffe00081dbd00 [aiod_kick taskq] 100012 D - 0xfffffe00081dbc00 [linuxkpi_irq_wq] 100013 D - 0xfffffe00081dbb00 [inm_free taskq] 100014 D - 0xfffffe00081dba00 [in6m_free taskq] 100015 D - 0xfffffe00081db900 [deferred_unmount ta] 100016 D - 0xfffffe00081db800 [thread taskq] 100018 D - 0xfffffe00081db500 [kqueue_ctx taskq] 100019 D - 0xfffffe00081db400 [pci_hp taskq] 100021 D - 0xfffffe00081db100 [linuxkpi_short_wq_0] 100022 D - 0xfffffe00081db100 [linuxkpi_short_wq_1] 100023 D - 0xfffffe00081db100 [linuxkpi_short_wq_2] 100024 D - 0xfffffe00081db100 [linuxkpi_short_wq_3] 100025 D - 0xfffffe00081db000 [linuxkpi_long_wq_0] 100026 D - 0xfffffe00081db000 [linuxkpi_long_wq_1] 100027 D - 0xfffffe00081db000 [linuxkpi_long_wq_2] 100028 D - 0xfffffe00081db000 [linuxkpi_long_wq_3] 100035 D - 0xfffffe0053f0ad00 [firmware taskq] 100039 D - 0xfffffe0053f0ac00 [crypto_0] 100040 D - 0xfffffe0053f0ac00 [crypto_1] 100056 D - 0xfffffe0053f0a400 [vtnet0 rxq 0] 100057 D - 0xfffffe0053f0a300 [vtnet0 txq 0] 100058 D - 0xfffffe0053f0a200 [vtnet0 rxq 1] 100059 D - 0xfffffe0053f0a100 [vtnet0 txq 1] 100061 D vtbslp 0xfffffe00571b3580 [virtio_balloon] 100065 D - 0xffffffff82b5a2a0 [deadlkres] 100070 D - 0xfffffe0053f09300 [acpi_task_0] 100071 D - 0xfffffe0053f09300 [acpi_task_1] 100072 D - 0xfffffe0053f09300 [acpi_task_2] 100073 D - 0xfffffe00081de100 [mca taskq] 100074 D - 0xfffffe0053f0a900 [CAM taskq] 781 778 781 0 Z syz-executor.0 783 778 783 0 Z syz-executor.2 db> show all locks db> show malloc Type InUse MemUse Requests pf_hash 5 11524K 5 tcp_hpts 6 4801K 6 devbuf 4213 4323K 4241 sysctloid 35454 2089K 35525 vtbuf 24 1968K 46 kobj 322 1288K 481 vfscache 3 1025K 3 newblk 1 1024K 70302 pcb 107 631K 127442 ufs_quota 1 512K 1 inodedep 1 512K 44556 vfs_hash 1 512K 1 callout 2 512K 2 intr 4 472K 4 filedesc 54 425K 86277 subproc 181 365K 44302 vmem 3 274K 6 acpica 1674 184K 56282 vnet_data 1 168K 1 tidhash 3 141K 3 linker 354 134K 454 pagedep 1 128K 43138 tfo_ccache 1 128K 1 DEVFS1 109 109K 126 sem 4 106K 4 BPF 46 88K 117 sctp_timw 317 80K 317 bus 973 80K 3426 mtx_pool 2 72K 2 syncache 1 68K 1 acpitask 1 64K 1 ddb_capture 1 64K 1 module 505 64K 505 umtx 418 53K 418 kdtrace 256 49K 97452 pf_osfp 328 40K 328 temp 35 35K 4355 shm 2 34K 145 DEVFS3 128 32K 138 hostcache 1 32K 1 msg 4 30K 4 gtaskqueue 18 26K 18 kbdmux 6 22K 6 ifaddr 71 21K 73 DEVFS_RULE 56 20K 56 routetbl 146 19K 3298 ufs_mount 4 17K 5 proc 3 17K 3 tty 16 16K 16 sctp_atcl 42 16K 93294 ithread 93 15K 93 bus-sc 34 15K 1651 KTRACE 102 13K 1768 ether_multi 157 13K 167 ifnet 7 13K 7 lltable 40 12K 121 kenv 95 12K 95 eventhandler 136 12K 136 CAM queue 5 11K 1528 GEOM 61 10K 490 rman 84 10K 425 in6_multi 71 9K 71 UART 12 9K 12 devstat 4 9K 4 ksem 1 8K 31 rpc 2 8K 2 bmsafemap 1 8K 44505 shmfd 1 8K 542 pfs_vncache 1 8K 1 pfs_nodes 20 8K 20 audit_evclass 237 8K 296 kqueue 84 8K 44287 taskqueue 63 7K 63 cred 26 7K 373 sglist 5 7K 5 CAM DEV 3 6K 510 plimit 24 6K 561 pwddesc 80 5K 44247 ufs_dirhash 24 5K 24 pf_ifnet 12 5K 952 UMA 272 5K 272 session 35 5K 59 DEVFSP 68 5K 1702 vt 11 5K 11 pf_table 2 4K 717 memdesc 1 4K 1 MCA 32 4K 32 evdev 4 4K 4 proc-args 111 4K 45741 lockf 34 4K 92 acpisem 28 4K 28 selfd 55 4K 578414 hhook 15 4K 17 kcovinfo 52 4K 52 terminal 11 3K 11 select 19 3K 185 uidinfo 3 3K 24 local_apic 1 2K 1 io_apic 1 2K 1 fpukern_ctx 2 2K 2 ipsec-saq 2 2K 2 ip6ndp 12 2K 15 sctp_ifa 14 2K 15 Unitno 29 2K 47 CAM XPT 22 2K 543 in_multi 6 2K 8 sctp_atky 42 2K 126568 ipsecpolicy 2 2K 2 acpidev 20 2K 20 msi 9 2K 9 clone 9 2K 9 tun 7 2K 7 softdep 1 1K 1 sahead 1 1K 1 secasvar 1 1K 1 nhops 6 1K 8 vnodemarker 2 1K 158 NFSD session 1 1K 1 CC Mem 15 1K 16993 CAM periph 4 1K 271 inpcbpolicy 26 1K 18709 ipsec 3 1K 3 sctp_ifn 6 1K 15 mld 6 1K 6 igmp 6 1K 6 toponodes 6 1K 6 isadev 6 1K 6 mount 16 1K 137 pci_link 10 1K 10 sctp_athm 42 1K 95142 crypto 4 1K 4 encap_export_host 12 1K 12 procdesc 5 1K 18 osd 19 1K 16969 pfil 4 1K 4 cdev 2 1K 2 chacha20random 1 1K 1 NFSD lckfile 1 1K 1 NFSD V4client 1 1K 1 DEVFS 9 1K 10 vnodes 1 1K 1 CAM SIM 2 1K 2 filedesc_to_leader 4 1K 11 feeder 7 1K 7 tcpfunc 3 1K 3 loginclass 3 1K 6 prison 6 1K 6 lkpikmalloc 5 1K 6 aesni_data 2 1K 2 pf_rule 1 1K 710 nexusdev 8 1K 8 apmdev 1 1K 1 atkbddev 2 1K 2 xform 2 1K 1568 CAM dev queue 2 1K 2 CAM I/O Scheduler 1 1K 1 CAM path 4 1K 1034 pmchooks 1 1K 1 soname 4 1K 94123 filecaps 5 1K 131 sctp_vrf 1 1K 1 vnet 1 1K 1 entropy 2 1K 50 acpiintr 1 1K 1 pmc 1 1K 1 cpus 2 1K 2 vnet_data_free 1 1K 1 Per-cpu 1 1K 1 freework 1 1K 54706 p1003.1b 1 1K 1 sctp_mcore 0 0K 0 sctp_socko 0 0K 46632 sctp_iter 0 0K 152 sctp_mvrf 0 0K 0 sctp_cpal 0 0K 156 sctp_cmsg 0 0K 0 sctp_stre 0 0K 0 sctp_athi 0 0K 0 sctp_a_it 0 0K 11 sctp_aadr 0 0K 0 sctp_stro 0 0K 28101 sctp_stri 0 0K 3198 sctp_map 0 0K 56202 mqdata 0 0K 0 tcp_do 0 0K 0 tcp_fsb 0 0K 3212 pf_altq 0 0K 0 pf_temp 0 0K 0 filemon 0 0K 128 NFSCL flayout 0 0K 0 NFSCL layout 0 0K 0 madt_table 0 0K 2 smartpqi 0 0K 0 NFSD rollback 0 0K 0 NFSCL diroff 0 0K 0 NEWdirectio 0 0K 0 ixl 0 0K 0 NEWNFSnode 0 0K 0 NFSCL lck 0 0K 0 NFSCL lckown 0 0K 0 NFSCL client 0 0K 0 NFSCL deleg 0 0K 0 NFSCL open 0 0K 0 NFSCL owner 0 0K 0 ice-resmgr 0 0K 0 ice-osdep 0 0K 0 ice 0 0K 0 iavf 0 0K 0 axgbe 0 0K 0 NFS fh 0 0K 0 NFS req 0 0K 0 NFSD usrgroup 0 0K 0 NFSD string 0 0K 0 NFSD V4lock 0 0K 0 NFSD V4state 0 0K 0 NFSD srvcache 0 0K 0 msdosfs_fat 0 0K 0 xen_intr 0 0K 0 msdosfs_mount 0 0K 0 msdosfs_node 0 0K 0 xen_hvm 0 0K 0 legacydrv 0 0K 0 qpidrv 0 0K 0 dmar_idpgtbl 0 0K 0 dmar_dom 0 0K 0 dmar_ctx 0 0K 0 DEVFS4 0 0K 0 isci 0 0K 0 iommu_dmamap 0 0K 0 hyperv_socket 0 0K 0 bxe_ilt 0 0K 0 DEVFS2 0 0K 0 xenbus 0 0K 0 gntdev 0 0K 0 privcmd_dev 0 0K 0 evtchn_dev 0 0K 0 vm_fictitious 0 0K 0 xenstore 0 0K 0 ciss_data 0 0K 0 BACKLIGHT 0 0K 0 xnb 0 0K 0 xbbd 0 0K 0 xbd 0 0K 0 Balloon 0 0K 0 sysmouse 0 0K 0 vtfont 0 0K 0 UMAHash 0 0K 0 ath_hal 0 0K 0 athdev 0 0K 0 vm_pgdata 0 0K 0 jblocks 0 0K 0 savedino 0 0K 47379 sentinel 0 0K 0 jfsync 0 0K 0 jtrunc 0 0K 0 sbdep 0 0K 73 jsegdep 0 0K 0 jseg 0 0K 0 jfreefrag 0 0K 0 jfreeblk 0 0K 0 jnewblk 0 0K 0 jmvref 0 0K 0 jremref 0 0K 0 jaddref 0 0K 0 freedep 0 0K 0 newdirblk 0 0K 43116 dirrem 0 0K 43958 mkdir 0 0K 86232 diradd 0 0K 44003 freefile 0 0K 43929 freeblks 0 0K 44399 freefrag 0 0K 14 allocindir 0 0K 0 indirdep 0 0K 15690 allocdirect 0 0K 0 ufs_trim 0 0K 0 mactemp 0 0K 0 audit_trigger 0 0K 0 audit_pipe_presel 0 0K 0 audit_pipeent 0 0K 0 audit_pipe 0 0K 0 audit_evname 0 0K 0 audit_bsm 0 0K 0 audit_gidset 0 0K 0 audit_text 0 0K 0 audit_path 0 0K 0 audit_data 0 0K 0 audit_cred 0 0K 0 ata_pci 0 0K 0 ata_dma 0 0K 0 ata_generic 0 0K 0 scsi_da 0 0K 69 pvscsi 0 0K 0 ata_da 0 0K 0 scsi_ch 0 0K 0 scsi_cd 0 0K 0 AHCI driver 0 0K 0 agp 0 0K 0 USBdev 0 0K 0 USB 0 0K 0 ktls_ocf 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5E_TLS 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EN 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5DUMP 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 seq_file 0 0K 0 lkpiskb 0 0K 0 radix 0 0K 0 idr 0 0K 0 lkpindev 0 0K 0 lkpifw 0 0K 0 lkpi80211 0 0K 0 NLM 0 0K 0 ipsec-spdcache 0 0K 0 ipsec-reg 0 0K 0 ipsec-misc 0 0K 0 ipsecrequest 0 0K 0 ip6opt 0 0K 112 ip6_msource 0 0K 0 ip6_moptions 0 0K 0 in6_mfilter 0 0K 0 frag6 0 0K 0 tcplog 0 0K 0 tcp_hwpace 0 0K 0 nvme_da 0 0K 0 LRO 0 0K 0 ip_msource 0 0K 0 ip_moptions 0 0K 0 in_mfilter 0 0K 0 ipid 0 0K 0 80211scan 0 0K 0 80211ratectl 0 0K 0 80211power 0 0K 0 80211nodeie 0 0K 0 80211node 0 0K 0 80211mesh_gt 0 0K 0 80211mesh_rt 0 0K 0 80211perr 0 0K 0 80211prep 0 0K 0 80211preq 0 0K 0 80211dfs 0 0K 0 80211crypto 0 0K 0 80211vap 0 0K 0 iflib 0 0K 0 vlan 0 0K 0 gif 0 0K 0 ifdescr 0 0K 0 zlib 0 0K 0 fadvise 0 0K 13 VN POLL 0 0K 0 statfs 0 0K 43742 namei_tracker 0 0K 0 export_host 0 0K 0 cl_savebuf 0 0K 6 acpipwr 0 0K 0 acpi_perf 0 0K 0 acpicmbat 0 0K 0 twsbuf 0 0K 0 twe_commands 0 0K 0 tcp_log_dev 0 0K 1740 midi buffers 0 0K 0 mixer 0 0K 0 ac97 0 0K 0 hdacc 0 0K 0 hdac 0 0K 0 hdaa 0 0K 0 SIIS driver 0 0K 0 CAM CCB 0 0K 523 PUC 0 0K 0 biobuf 0 0K 0 aios 0 0K 78 lio 0 0K 5 acl 0 0K 0 ppbusdev 0 0K 0 mbuf_tag 0 0K 144 ktls 0 0K 0 agtiapi_MemAlloc malloc 0 0K 0 osti_cacheable 0 0K 0 tempbuff 0 0K 0 tempbuff 0 0K 0 ag_tgt_map_t malloc 0 0K 0 ag_slr_map_t malloc 0 0K 0 lDevFlags * malloc 0 0K 0 tiDeviceHandle_t * malloc 0 0K 0 ag_portal_data_t malloc 0 0K 0 ag_device_t malloc 0 0K 0 STLock malloc 0 0K 0 CCB List 0 0K 0 sr_iov 0 0K 0 OCS 0 0K 0 OCS 0 0K 0 nvme 0 0K 0 nvd 0 0K 0 netmap 0 0K 0 mwldev 0 0K 0 MVS driver 0 0K 0 CAM ccb queue 0 0K 0 mrsasbuf 0 0K 0 mpt_user 0 0K 0 accf 0 0K 0 pts 0 0K 0 iov 0 0K 41820 ioctlops 0 0K 5231 eventfd 0 0K 8 Witness 0 0K 0 stack 0 0K 0 mps_user 0 0K 0 MPSSAS 0 0K 0 mps 0 0K 0 mpr_user 0 0K 0 MPRSAS 0 0K 0 mpr 0 0K 0 sbuf 0 0K 288 mfibuf 0 0K 0 firmware 0 0K 0 compressor 0 0K 0 md_sectors 0 0K 0 SWAP 0 0K 0 md_disk 0 0K 0 malodev 0 0K 0 sysctltmp 0 0K 1083 sysctl 0 0K 3 LED 0 0K 0 ekcd 0 0K 0 dumper 0 0K 0 sendfile 0 0K 0 rctl 0 0K 0 ix_sriov 0 0K 0 aacraidcam 0 0K 0 aacraid_buf 0 0K 0 ix 0 0K 0 cache 0 0K 0 ipsbuf 0 0K 0 prison_racct 0 0K 0 Fail Points 0 0K 0 sigio 0 0K 1 pwd 0 0K 0 tty console 0 0K 0 aaccam 0 0K 0 aacbuf 0 0K 0 zstd 0 0K 0 XZ_DEC 0 0K 0 nvlist 0 0K 0 SCSI ENC 0 0K 0 SCSI sa 0 0K 0 scsi_pass 0 0K 0 isofs_node 0 0K 0 isofs_mount 0 0K 0 tr_raid5_data 0 0K 0 tr_raid1e_data 0 0K 0 tr_raid1_data 0 0K 0 tr_raid0_data 0 0K 0 tr_concat_data 0 0K 0 md_sii_data 0 0K 0 md_promise_data 0 0K 0 md_nvidia_data 0 0K 0 md_jmicron_data 0 0K 0 md_intel_data 0 0K 0 md_ddf_data 0 0K 0 raid_data 0 0K 72 geom_flashmap 0 0K 0 tmpfs dir 0 0K 0 tmpfs name 0 0K 0 tmpfs mount 0 0K 0 NFS FHA 0 0K 0 newnfsmnt 0 0K 0 newnfsclient_req 0 0K 0 NFSCL layrecall 0 0K 0 NFSCL session 0 0K 0 NFSCL sockreq 0 0K 0 NFSCL devinfo 0 0K 0 db> show uma Zone Size Used Free Requests Sleeps Bucket Total Mem XFree mbuf_jumbo_page 4096 8320 1078 1025036 0 254 38494208 0 ertt_txseginfo 40 504401 2417 4655777 0 254 20272720 0 tcp_log 416 0 10161 150100 0 254 4226976 0 sctp_asoc 2288 0 1270 28101 0 254 2905760 0 mbuf 256 8716 2207 6593902 0 254 2796288 0 mbuf_cluster 2048 762 508 780 0 254 2600960 0 malloc-384 384 4206 1734 97458 0 30 2280960 0 pbuf 2624 0 778 0 0 2 2041472 0 RADIX NODE 144 13697 438 816311 0 64 2035440 0 malloc-4096 4096 427 7 45331 0 2 1777664 0 malloc-2048 2048 44 820 67662 0 8 1769472 0 BUF TRIE 144 197 11591 30460 0 62 1697472 0 sctp_ep 1208 42 1228 63594 0 254 1534160 0 malloc-128 128 11643 75 11897 0 126 1499904 0 vmem btag 56 26624 55 26624 0 254 1494024 0 UMA Slabs 0 112 11936 10 11936 0 126 1337952 0 malloc-1024 1024 136 904 28272 0 16 1064960 0 sctp_raddr 736 0 1276 36096 0 254 939136 0 malloc-384 384 0 2340 44556 0 30 898560 0 VM OBJECT 264 2778 162 950725 0 30 776160 0 socket 960 93 671 84688 0 254 733440 0 256 Bucket 2048 308 44 21647 0 8 720896 0 FFS inode 1160 544 37 44489 0 8 673960 0 malloc-8192 8192 12 69 143 0 1 663552 0 malloc-256 256 192 2358 88720 0 62 652800 0 malloc-16384 16384 27 10 43137 0 1 606208 0 malloc-256 256 2 2293 119422 0 62 587520 0 tcpcb 1096 15 496 16958 0 254 560056 0 tcp_bbr_map 128 4050 259 175223 0 126 551552 0 MAP ENTRY 96 3666 618 2249698 0 126 411264 0 lkpicurr 168 2 2350 2 0 62 395136 0 malloc-65536 65536 6 0 6 0 1 393216 0 lkpimm 168 1 2327 1 0 62 391104 0 THREAD 1808 173 36 53250 0 8 377872 0 malloc-128 128 458 2456 88486 0 126 372992 0 tcp_inpcb 424 15 750 16958 0 254 324360 0 udp_inpcb 424 6 759 1135 0 254 324360 0 VNODE 448 585 81 44532 0 30 298368 0 malloc-64 64 3894 453 4329 0 254 278208 0 malloc-65536 65536 0 4 136 0 1 262144 0 malloc-256 256 87 918 65535 0 62 257280 0 malloc-16 16 14679 571 71569 0 254 244000 0 malloc-32768 32768 0 7 14863 0 1 229376 0 DEVCTL 1024 0 220 155 0 0 225280 0 ripcb 424 5 508 367 0 254 217512 0 udplite_inpcb 424 0 513 249 0 254 217512 0 malloc-65536 65536 1 2 33 0 1 196608 0 malloc-65536 65536 1 2 513 0 1 196608 0 mbuf_packet 256 61 701 78734 0 254 195072 0 malloc-128 128 1364 155 31527 0 126 194432 0 sctp_chunk 152 0 1274 9529 0 254 193648 0 UMA Zones 768 244 0 244 0 16 187392 0 FPU_save_area 832 175 50 58386 0 16 187200 0 VMSPACE 2552 58 14 44172 0 4 183744 0 malloc-32 32 5345 325 6375 0 254 181440 0 malloc-256 256 522 168 54703 0 62 176640 0 FFS2 dinode 256 544 86 44488 0 62 161280 0 128 Bucket 1024 90 57 10185 0 16 150528 0 S VFS Cache 104 1045 359 45729 0 126 146016 0 malloc-32768 32768 0 4 120 0 1 131072 0 unpcb 256 20 490 2322 0 254 130560 0 malloc-256 256 108 402 46529 0 62 130560 0 PROC 1352 81 15 44200 0 8 129792 0 64 Bucket 512 170 62 24204 0 30 118784 0 ksiginfo 112 87 957 5606 0 126 116928 0 filedesc0 1072 80 25 44247 0 8 112560 0 malloc-64 64 339 1236 33408 0 254 100800 0 malloc-128 128 593 182 2553 0 126 99200 0 pipe 744 25 105 1013 0 16 96720 0 malloc-384 384 172 68 2045 0 30 92160 0 malloc-4096 4096 21 1 32 0 2 90112 0 UMA Kegs 384 230 3 230 0 30 89472 0 pcpu-8 8 4782 5970 25487 0 254 86016 0 malloc-64 64 760 563 66415 0 254 84672 0 clpbuf 2624 0 32 74 0 16 83968 0 malloc-128 128 465 186 90293 0 126 83328 0 malloc-128 128 308 343 43775 0 126 83328 0 malloc-256 256 107 208 56688 0 62 80640 0 32 Bucket 256 198 117 17437 0 62 80640 0 malloc-32 32 142 2252 95159 0 254 76608 0 g_bio 408 0 180 215035 0 30 73440 0 malloc-65536 65536 1 0 1 0 1 65536 0 malloc-65536 65536 0 1 8 0 1 65536 0 malloc-32768 32768 2 0 2 0 1 65536 0 malloc-8192 8192 4 4 84 0 1 65536 0 malloc-2048 2048 12 20 1234 0 8 65536 0 malloc-2048 2048 8 24 681 0 8 65536 0 sctp_stream_msg_out 112 0 540 2633 0 254 60480 0 malloc-8192 8192 3 4 128 0 1 57344 0 malloc-64 64 171 648 623237 0 254 52416 0 malloc-64 64 123 696 61266 0 254 52416 0 malloc-64 64 476 343 1151 0 254 52416 0 Files 80 273 377 149739 0 126 52000 0 malloc-256 256 4 191 44475 0 62 49920 0 DIRHASH 1024 34 14 34 0 16 49152 0 NAMEI 1024 0 48 199290 0 16 49152 0 malloc-16384 16384 3 0 18 0 1 49152 0 malloc-16384 16384 3 0 3 0 1 49152 0 malloc-16384 16384 0 3 160 0 1 49152 0 malloc-2048 2048 4 20 974 0 8 49152 0 malloc-1024 1024 3 45 2139 0 16 49152 0 malloc-512 512 1 95 3199 0 30 49152 0 malloc-512 512 14 82 281 0 30 49152 0 tcp_rack_pcb 896 0 54 3212 0 16 48384 0 tcp_bbr_pcb 832 7 47 9010 0 16 44928 0 syncache 168 0 264 4 0 254 44352 0 malloc-8192 8192 1 4 102 0 1 40960 0 malloc-4096 4096 4 6 43713 0 2 40960 0 sctp_readq 152 0 260 1408 0 254 39520 0 da_ccb 544 0 70 53874 0 16 38080 0 PWD 32 35 1099 43213 0 254 36288 0 malloc-64 64 62 505 1530 0 254 36288 0 malloc-64 64 3 564 47 0 254 36288 0 16 Bucket 144 143 109 2728 0 62 36288 0 malloc-16 16 80 2170 96048 0 254 36000 0 malloc-16 16 25 2225 96828 0 254 36000 0 8 Bucket 80 154 296 9483 0 126 36000 0 malloc-128 128 31 248 3274 0 126 35712 0 malloc-128 128 12 267 271 0 126 35712 0 routing nhops 256 27 108 35 0 62 34560 0 ttyoutq 256 72 63 160 0 62 34560 0 malloc-256 256 24 111 5527 0 62 34560 0 TURNSTILE 136 210 42 210 0 62 34272 0 malloc-32768 32768 1 0 1 0 1 32768 0 malloc-2048 2048 7 9 7 0 8 32768 0 malloc-2048 2048 1 15 75 0 8 32768 0 malloc-2048 2048 5 11 200 0 8 32768 0 malloc-1024 1024 2 30 46 0 16 32768 0 malloc-1024 1024 12 20 169 0 16 32768 0 malloc-1024 1024 3 29 57 0 16 32768 0 malloc-1024 1024 8 24 558 0 16 32768 0 malloc-512 512 1 63 119 0 30 32768 0 malloc-512 512 1 63 15 0 30 32768 0 malloc-512 512 0 64 4 0 30 32768 0 malloc-512 512 9 55 57 0 30 32768 0 pcpu-64 64 482 30 482 0 254 32768 0 KNOTE 160 24 176 458803 0 62 32000 0 ttyinq 160 135 65 300 0 62 32000 0 tcp_rack_map 120 0 264 5065 0 126 31680 0 cpuset 104 7 272 81 0 126 29016 0 sctp_laddr 48 0 588 1561 0 254 28224 0 tcp_inpcb 32 2 880 9907 0 254 28224 0 ertt 72 15 377 16958 0 126 28224 0 malloc-32 32 271 611 96483 0 254 28224 0 malloc-32 32 92 790 22240 0 254 28224 0 4 Bucket 48 6 582 982 0 254 28224 0 2 Bucket 32 106 776 6545 0 254 28224 0 AIO 208 0 133 63 0 62 27664 0 malloc-8192 8192 1 2 5 0 1 24576 0 malloc-8192 8192 3 0 3 0 1 24576 0 rtentry 176 31 107 35 0 62 24288 0 PGRP 88 35 241 63 0 126 24288 0 rl_entry 40 103 503 103 0 254 24240 0 itimer 352 2 64 4 0 30 23232 0 malloc-384 384 11 49 11 0 30 23040 0 malloc-384 384 25 35 29 0 30 23040 0 malloc-384 384 1 59 41 0 30 23040 0 domainset 40 0 567 74 0 254 22680 0 SLEEPQUEUE 88 210 46 210 0 126 22528 0 udplite_inpcb 32 0 630 21 0 254 20160 0 hostcache 64 1 314 1 0 254 20160 0 udpcb 32 6 624 1384 0 254 20160 0 udp_inpcb 32 3 627 291 0 254 20160 0 malloc-32 32 80 550 153 0 254 20160 0 malloc-32 32 68 562 622 0 254 20160 0 malloc-32 32 164 466 1065 0 254 20160 0 malloc-32 32 32 598 323 0 254 20160 0 AIOCB 552 0 35 129 0 16 19320 0 Mountpoints 2752 2 5 2 0 4 19264 0 AIOLIO 272 0 70 5 0 30 19040 0 epoch_record pcpu 256 4 60 4 0 62 16384 0 malloc-16384 16384 1 0 1 0 1 16384 0 malloc-8192 8192 0 2 36 0 1 16384 0 malloc-4096 4096 2 2 5 0 2 16384 0 malloc-4096 4096 1 3 41 0 2 16384 0 malloc-4096 4096 0 4 126 0 2 16384 0 malloc-2048 2048 0 8 25 0 8 16384 0 malloc-1024 1024 4 12 4 0 16 16384 0 malloc-1024 1024 4 12 4 0 16 16384 0 SMR CPU 32 7 504 7 0 254 16352 0 vtnet_tx_hdr 24 0 668 4908222 0 254 16032 0 malloc-16 16 249 751 46342 0 254 16000 0 kenv 258 15 45 1045 0 30 15480 0 mqnode 416 3 33 3 0 30 14976 0 vmem 1856 1 7 1 0 8 14848 0 SMR SHARED 24 7 504 7 0 254 12264 0 AIOP 32 4 374 4 0 254 12096 0 malloc-16 16 15 735 40 0 254 12000 0 malloc-16 16 327 423 704 0 254 12000 0 malloc-16 16 5 745 39296 0 254 12000 0 malloc-16 16 17 733 36 0 254 12000 0 malloc-384 384 0 30 1 0 30 11520 0 malloc-384 384 1 29 1 0 30 11520 0 malloc-8192 8192 1 0 1 0 1 8192 0 malloc-4096 4096 0 2 10 0 2 8192 0 pcpu-16 16 7 249 7 0 254 4096 0 UMA Slabs 1 176 20 2 20 0 62 3872 0 KMAP ENTRY 96 12 27 14 0 0 3744 0 FFS1 dinode 128 0 0 0 0 126 0 0 ada_ccb 272 0 0 0 0 30 0 0 swblk 136 0 0 0 0 62 0 0 swpctrie 144 0 0 0 0 62 0 0 cdg_qdiffsample 16 0 0 0 0 254 0 0 sctp_asconf_ack 48 0 0 0 0 254 0 0 sctp_asconf 40 0 0 0 0 254 0 0 pf state scrubs 40 0 0 0 0 254 0 0 pf frag entries 40 0 0 0 0 254 0 0 pf frags 248 0 0 0 0 62 0 0 pf table entries 160 0 0 0 0 62 0 0 pf table entry counters 64 0 0 0 0 254 0 0 pf source nodes 136 0 0 0 0 254 0 0 pf state keys 88 0 0 0 0 126 0 0 pf states 312 0 0 0 0 254 0 0 pf tags 104 0 0 0 0 126 0 0 pf mtags 48 0 0 0 0 254 0 0 IPsec SA lft_c 16 0 0 0 0 254 0 0 ripcb 32 0 0 0 0 254 0 0 tcp_log_node 120 0 0 0 0 126 0 0 tcp_log_bucket 176 0 0 0 0 62 0 0 tcpreass 48 0 0 0 0 254 0 0 tfo_ccache_entries 80 0 0 0 0 126 0 0 tfo 4 0 0 0 0 254 0 0 sackhole 32 0 0 0 0 254 0 0 tcptw 72 0 0 0 0 254 0 0 ipq 56 0 0 0 0 254 0 0 mqnotifier 216 0 0 0 0 62 0 0 mvdata 64 0 0 0 0 254 0 0 mqueue 248 0 0 0 0 62 0 0 TMPFS node 224 0 0 0 0 62 0 0 NCLNODE 608 0 0 0 0 16 0 0 LTS VFS Cache 360 0 0 0 0 30 0 0 L VFS Cache 320 0 0 0 0 30 0 0 STS VFS Cache 144 0 0 0 0 62 0 0 cryptop 280 0 0 0 0 30 0 0 linux_dma_object 32 0 0 0 0 254 0 0 linux_dma_pctrie 144 0 0 0 0 62 0 0 IOMMU_MAP_ENTRY 120 0 0 0 0 126 0 0 mbuf_jumbo_16k 16384 0 0 0 0 254 0 0 mbuf_jumbo_9k 9216 0 0 0 0 254 0 0 audit_record 1280 0 0 0 0 8 0 0 MAC labels 40 0 0 0 0 254 0 0 vnpbuf 2624 0 0 0 0 64 0 0 mdpbuf 2624 0 0 0 0 3 0 0 nfspbuf 2624 0 0 0 0 16 0 0 swwbuf 2624 0 0 0 0 8 0 0 swrbuf 2624 0 0 0 0 16 0 0 umtx_shm 88 0 0 0 0 126 0 0 umtx pi 96 0 0 0 0 126 0 0 rangeset pctrie nodes 144 0 0 0 0 62 0 0 malloc-65536 65536 0 0 0 0 1 0 0 malloc-65536 65536 0 0 0 0 1 0 0 malloc-32768 32768 0 0 0 0 1 0 0 malloc-32768 32768 0 0 0 0 1 0 0 malloc-32768 32768 0 0 0 0 1 0 0 malloc-32768 32768 0 0 0 0 1 0 0 malloc-16384 16384 0 0 0 0 1 0 0 malloc-16384 16384 0 0 0 0 1 0 0 malloc-16384 16384 0 0 0 0 1 0 0 malloc-4096 4096 0 0 0 0 2 0 0 malloc-512 512 0 0 0 0 30 0 0 malloc-512 512 0 0 0 0 30 0 0 pcpu-32 32 0 0 0 0 254 0 0 pcpu-4 4 0 0 0 0 254 0 0 fakepg 104 0 0 0 0 126 0 0 UMA Hash 256 0 0 0 0 62 0 0