panic: already suspended cpuid = 1 time = 1654031993 KDB: stack backtrace: db_trace_self_wrapper() at db_trace_self_wrapper+0xc7/frame 0xfffffe00a641b530 kdb_backtrace() at kdb_backtrace+0xd3/frame 0xfffffe00a641b690 vpanic() at vpanic+0x2b8/frame 0xfffffe00a641b770 panic() at panic+0xb5/frame 0xfffffe00a641b830 thread_suspend_one() at thread_suspend_one+0x107/frame 0xfffffe00a641b860 thread_single() at thread_single+0x79c/frame 0xfffffe00a641b910 reap_kill_proc() at reap_kill_proc+0x4b8/frame 0xfffffe00a641b990 reap_kill() at reap_kill+0x7fc/frame 0xfffffe00a641bba0 kern_procctl() at kern_procctl+0x51a/frame 0xfffffe00a641bc10 sys_procctl() at sys_procctl+0x247/frame 0xfffffe00a641bd30 amd64_syscall() at amd64_syscall+0x410/frame 0xfffffe00a641bf30 fast_syscall_common() at fast_syscall_common+0xf8/frame 0xfffffe00a641bf30 --- syscall (198, FreeBSD ELF64, nosys), rip = 0x28a47a, rsp = 0x830fa3f08, rbp = 0x830fa3f70 --- KDB: enter: panic [ thread pid 40461 tid 146121 ] Stopped at kdb_enter+0x6b: movq $0,0x275b69a(%rip) db> db> set $lines = 0 db> set $maxwidth = 0 db> show registers cs 0x20 ds 0x3b es 0x3b fs 0x13 gs 0x1b ss 0x28 rax 0x12 rcx 0xfffffe00a4200000 rdx 0x3ffff rbx 0 rsp 0xfffffe00a641b670 rbp 0xfffffe00a641b690 rsi 0x40001 rdi 0xffffffff8172b458 printf+0x158 r8 0 r9 0xffffffff r10 0 r11 0xfffffe0058d78c50 r12 0xfffffe00a660d560 r13 0xfffffe00a641b601 r14 0xffffffff82b598a0 .str.26 r15 0xffffffff82b598a0 .str.26 rip 0xffffffff8171af7b kdb_enter+0x6b rflags 0x46 kdb_enter+0x6b: movq $0,0x275b69a(%rip) db> show proc Process 40461 (syz-executor.3) at 0xfffffe00a5ed2a90: state: NORMAL uid: 0 gids: 0, 0, 5 parent: pid 797 at 0xfffffe009ebeb548 ABI: FreeBSD ELF64 flag: 0x10080480 flag2: 0 arguments: /root/syz-executor.3 exec reaper: 0xfffffe0053de1000 reapsubtree: 1 sigparent: 20 vmspace: 0xfffffe00a687c3f0 (map 0xfffffe00a687c3f0) (map.pmap 0xfffffe00a687c4b0) (pmap 0xfffffe00a687c518) threads: 2 146118 s syz-executor.3 146121 Run CPU 1 syz-executor.3 db> ps pid ppid pgrp uid state wmesg wchan cmd 40461 797 797 0 T (threaded) syz-executor.3 146118 s syz-executor.3 146121 Run CPU 1 syz-executor.3 40459 788 788 0 TL (threaded) syz-executor.2 145679 RunQ syz-executor.2 146119 Ss uwait 0xfffffe0058b30580 syz-executor.2 35544 1 786 0 S uwait 0xfffffe0057327780 syz-executor.1 35186 1 786 0 S uwait 0xfffffe0058b31200 syz-executor.1 33625 1 797 0 S uwait 0xfffffe005788a680 syz-executor.3 33616 1 797 0 S uwait 0xfffffe005732ab80 syz-executor.3 33607 1 797 0 S uwait 0xfffffe0057327680 syz-executor.3 30696 1 786 0 SV uwait 0xfffffe0058b31100 syz-executor.1 30691 1 786 0 SV uwait 0xfffffe0058b30980 syz-executor.1 30686 1 786 0 SV uwait 0xfffffe005732a400 syz-executor.1 23138 1 786 0 S uwait 0xfffffe005732a800 syz-executor.1 23134 1 786 0 S uwait 0xfffffe005788a380 syz-executor.1 23124 1 786 0 S uwait 0xfffffe0057889b00 syz-executor.1 11950 11947 785 0 SV uwait 0xfffffe005732a600 syz-executor.0 11947 1 785 0 DV ppwait 0xfffffe0058bc14f8 syz-executor.0 11941 11939 785 0 SV uwait 0xfffffe005788a780 syz-executor.0 11939 1 785 0 DV ppwait 0xfffffe00a5d7a4f8 syz-executor.0 11927 1 785 0 SV uwait 0xfffffe0058b2e880 syz-executor.0 10538 1 10538 65 Ss select 0xfffffe00a63300c0 dhclient 7342 1 7342 0 Ss select 0xfffffe00a5fbd840 dhclient 7332 1 7332 0 Ss select 0xfffffe00a5ff61c0 dhclient 7280 1 7280 65 Ss select 0xfffffe00a5f9a4c0 dhclient 3074 1 3074 0 Ss select 0xfffffe00a5eaadc0 dhclient 3064 1 3064 0 Ss select 0xfffffe00a5e9f1c0 dhclient 3006 1 3006 65 Ss select 0xfffffe00a5d99bc0 dhclient 1910 1 1910 0 Ss select 0xfffffe00578fdb40 dhclient 1907 1 1907 0 Ss select 0xfffffe00578fd640 dhclient 1880 1 1880 65 Ss select 0xfffffe00a5d99d40 dhclient 1000 1 1000 0 Ss select 0xfffffe00578fbec0 dhclient 996 1 996 0 Ss select 0xfffffe00578fbbc0 dhclient 831 0 0 0 DL aiordy 0xfffffe00a0c5d000 [aiod4] 830 0 0 0 DL aiordy 0xfffffe009efc1000 [aiod3] 829 0 0 0 DL aiordy 0xfffffe0058bc1548 [aiod2] 828 0 0 0 DL aiordy 0xfffffe009ebeba90 [aiod1] 797 782 797 0 Ss nanslp 0xffffffff83e43c81 syz-executor.3 788 782 788 0 Rs syz-executor.2 786 782 786 0 Rs syz-executor.1 785 782 785 0 Rs syz-executor.0 782 780 780 0 R (threaded) syz-fuzzer 100115 S kqread 0xfffffe0058d5b400 syz-fuzzer 100117 S nanslp 0xffffffff83e43c80 syz-fuzzer 100118 S uwait 0xfffffe005788cf00 syz-fuzzer 100119 Run CPU 0 syz-fuzzer 100120 S uwait 0xfffffe0058b2ee80 syz-fuzzer 100121 S uwait 0xfffffe0058b2e180 syz-fuzzer 100122 S uwait 0xfffffe0058b2e280 syz-fuzzer 100123 S uwait 0xfffffe0058b2f280 syz-fuzzer 100125 S uwait 0xfffffe0058b2ed80 syz-fuzzer 142212 S uwait 0xfffffe0057327d80 syz-fuzzer 780 778 780 0 Ss pause 0xfffffe009efc1b40 csh 778 688 778 0 Ss select 0xfffffe0056f6ab40 sshd 754 1 754 0 Ss+ ttyin 0xfffffe00574764b0 getty 753 1 753 0 Ss+ ttyin 0xfffffe00579c5cb0 getty 752 1 752 0 Ss+ ttyin 0xfffffe00579c60b0 getty 751 1 751 0 Ss+ ttyin 0xfffffe0057472cb0 getty 750 1 750 0 Ss+ ttyin 0xfffffe00579c64b0 getty 749 1 749 0 Ss+ ttyin 0xfffffe00579c68b0 getty 748 1 748 0 Ss+ ttyin 0xfffffe00579c6cb0 getty 747 1 747 0 Ss+ ttyin 0xfffffe00579c70b0 getty 746 1 746 0 Ss+ ttyin 0xfffffe00579c74b0 getty 692 1 692 0 Ss nanslp 0xffffffff83e43c81 cron 688 1 688 0 Ss select 0xfffffe0056f6b2c0 sshd 501 1 501 0 Ss select 0xfffffe0056f6bf40 syslogd 430 1 430 0 Ss select 0xfffffe0056f6af40 devd 429 1 429 65 Ss select 0xfffffe0056f6adc0 dhclient 344 1 344 0 Ss select 0xfffffe0056f6aec0 dhclient 341 1 341 0 Ss select 0xfffffe0056f6b440 dhclient 17 0 0 0 DL vlruwt 0xfffffe0056fa0548 [vnlru] 16 0 0 0 DL syncer 0xffffffff83f694e0 [syncer] 15 0 0 0 DL (threaded) [bufdaemon] 100079 D psleep 0xffffffff83f67ae0 [bufdaemon] 100082 D - 0xffffffff83211f80 [bufspacedaemon-0] 100090 D sdflush 0xfffffe0058b81ce8 [/ worker] 9 0 0 0 DL psleep 0xffffffff83f9b5c0 [vmdaemon] 8 0 0 0 DL (threaded) [pagedaemon] 100077 D psleep 0xffffffff83f8f478 [dom0] 100083 D launds 0xffffffff83f8f484 [laundry: dom0] 100084 D umarcl 0xffffffff81e4b980 [uma] 7 0 0 0 DL - 0xffffffff83c002a8 [rand_harvestq] 6 0 0 0 DL pftm 0xffffffff84a87550 [pf purge] 5 0 0 0 DL waiting 0xffffffff8485f460 [sctp_iterator] 4 0 0 0 DL (threaded) [cam] 100044 D - 0xffffffff83aa3240 [doneq0] 100045 D - 0xffffffff83aa31c0 [async] 100076 D - 0xffffffff83aa3040 [scanner] 14 0 0 0 DL seqstat 0xfffffe0053fd7488 [sequencer 00] 3 0 0 0 DL (threaded) [crypto] 100040 D crypto_ 0xffffffff83f8aca0 [crypto] 100041 D crypto_ 0xfffffe0053ecd830 [crypto returns 0] 100042 D crypto_ 0xfffffe0053ecd880 [crypto returns 1] 13 0 0 0 DL (threaded) [geom] 100035 D - 0xffffffff83e19280 [g_event] 100036 D - 0xffffffff83e192a0 [g_up] 100037 D - 0xffffffff83e192c0 [g_down] 2 0 0 0 WL (threaded) [clock] 100029 I [clock (0)] 100030 I [clock (1)] 12 0 0 0 WL (threaded) [intr] 100015 I [swi5: fast taskq] 100018 I [swi6: task queue] 100019 I [swi6: Giant taskq] 100031 I [swi1: netisr 0] 100032 I [swi1: hpts] 100033 I [swi1: hpts] 100046 I [irq24: virtio_pci0] 100047 I [irq25: virtio_pci0] 100048 I [irq26: virtio_pci0] 100049 I [irq27: virtio_pci0] 100050 I [irq28: virtio_pci1] 100051 I [irq29: virtio_pci1] 100052 I [irq30: virtio_pci1] 100053 I [irq31: virtio_pci1] 100054 I [irq32: virtio_pci1] 100059 I [irq33: virtio_pci2] 100060 I [irq34: virtio_pci2] 100061 I [irq35: virtio_pci2] 100063 I [irq1: atkbd0] 100064 I [irq12: psm0] 100065 I [swi0: uart uart++] 100069 I [swi1: pf send] 11 0 0 0 RL (threaded) [idle] 100003 CanRun [idle: cpu0] 100004 CanRun [idle: cpu1] 1 0 1 0 SLs wait 0xfffffe0053de1000 [init] 10 0 0 0 DL audit_w 0xffffffff83f8b780 [audit] 0 0 0 0 DLs (threaded) [kernel] 100000 D swapin 0xffffffff83e19c80 [swapper] 100005 D - 0xfffffe0053e84100 [if_config_tqg_0] 100006 D - 0xfffffe0053e84000 [softirq_0] 100007 D - 0xfffffe0053e83e00 [softirq_1] 100008 D - 0xfffffe0053e83d00 [if_io_tqg_0] 100009 D - 0xfffffe0053e83c00 [if_io_tqg_1] 100010 D - 0xfffffe000795a100 [inm_free taskq] 100011 D - 0xfffffe000795a000 [linuxkpi_irq_wq] 100012 D - 0xfffffe0007959e00 [in6m_free taskq] 100013 D - 0xfffffe0007959d00 [deferred_unmount ta] 100014 D - 0xfffffe0007959c00 [thread taskq] 100016 D - 0xfffffe0007959a00 [kqueue_ctx taskq] 100017 D - 0xfffffe0007959900 [pci_hp taskq] 100020 D - 0xfffffe0007959600 [aiod_kick taskq] 100021 D - 0xfffffe0007959500 [linuxkpi_short_wq_0] 100022 D - 0xfffffe0007959500 [linuxkpi_short_wq_1] 100023 D - 0xfffffe0007959500 [linuxkpi_short_wq_2] 100024 D - 0xfffffe0007959500 [linuxkpi_short_wq_3] 100025 D - 0xfffffe0007959400 [linuxkpi_long_wq_0] 100026 D - 0xfffffe0007959400 [linuxkpi_long_wq_1] 100027 D - 0xfffffe0007959400 [linuxkpi_long_wq_2] 100028 D - 0xfffffe0007959400 [linuxkpi_long_wq_3] 100034 D - 0xfffffe0007959300 [firmware taskq] 100038 D - 0xfffffe0007959200 [crypto_0] 100039 D - 0xfffffe0007959200 [crypto_1] 100055 D - 0xfffffe0007959000 [vtnet0 rxq 0] 100056 D - 0xfffffe0007958e00 [vtnet0 txq 0] 100057 D - 0xfffffe0007958d00 [vtnet0 rxq 1] 100058 D - 0xfffffe0007958c00 [vtnet0 txq 1] 100062 D vtbslp 0xfffffe0056f6c000 [virtio_balloon] 100066 D - 0xffffffff82b5f721 [deadlkres] 100070 D - 0xfffffe000795a200 [mca taskq] 100071 D - 0xfffffe00585ef600 [acpi_task_0] 100072 D - 0xfffffe00585ef600 [acpi_task_1] 100073 D - 0xfffffe00585ef600 [acpi_task_2] 100075 D - 0xfffffe0007959100 [CAM taskq] db> show all locks Process 40461 (syz-executor.3) thread 0xfffffe00a660d560 (146121) exclusive sleep mutex process lock (process lock) r = 0 (0xfffffe00a5ed3bb8) locked @ /syzkaller/managers/main/kernel/sys/kern/kern_thread.c:1523 exclusive sx sapblk (sapblk) r = 0 (0xffffffff83e32020) locked @ /syzkaller/managers/main/kernel/sys/kern/kern_proc.c:3405 db> show malloc Type InUse MemUse Requests pf_hash 5 11524K 5 pcb 4015 5028K 125752 tcp_hpts 7 4801K 7 devbuf 4217 4323K 4245 sysctloid 35322 2081K 35393 sctp_stro 1983 1983K 26265 vtbuf 24 1968K 46 sctp_atcl 3979 1493K 89361 inodedep 2229 1348K 42395 kobj 328 1312K 490 newblk 7 1026K 91213 vfscache 3 1025K 3 dirrem 2220 555K 41741 ufs_quota 1 512K 1 vfs_hash 1 512K 1 callout 2 512K 2 intr 4 472K 4 filedesc 46 361K 78907 subproc 172 337K 40565 freefile 2220 278K 41721 vmem 3 274K 5 sctp_atky 5962 249K 116685 acpica 1674 184K 57552 vnet_data 1 168K 1 tidhash 3 141K 3 linker 358 134K 418 pagedep 9 130K 39452 tfo_ccache 1 128K 1 DEVFS1 109 109K 126 sem 4 106K 4 sctp_timw 358 90K 358 BPF 46 88K 89 bus 995 81K 5208 mtx_pool 2 72K 2 syncache 1 68K 1 module 514 65K 514 acpitask 1 64K 1 ddb_capture 1 64K 1 sctp_athm 3979 63K 89769 sctp_map 3966 62K 52530 umtx 418 53K 418 kdtrace 251 49K 86584 temp 35 34K 2722 DEVFS3 128 32K 138 hostcache 1 32K 1 shm 1 32K 99 msg 4 30K 4 sctp_aadr 422 27K 517 gtaskqueue 18 26K 18 kbdmux 6 22K 6 ifaddr 70 20K 72 DEVFS_RULE 56 20K 56 ufs_mount 4 17K 5 proc 3 17K 3 tty 16 16K 16 routetbl 130 16K 410 ithread 97 16K 97 lltable 47 15K 128 bus-sc 34 15K 1681 KTRACE 101 13K 519 ifnet 7 13K 7 ether_multi 152 13K 162 eventhandler 136 12K 136 kenv 95 12K 95 rman 88 11K 431 GEOM 61 11K 490 CAM queue 5 11K 1528 in6_multi 65 9K 65 bmsafemap 2 9K 42643 UART 12 9K 12 devstat 4 9K 4 ksem 1 8K 12 rpc 2 8K 2 shmfd 1 8K 137 pfs_vncache 1 8K 1 pfs_nodes 20 8K 20 audit_evclass 237 8K 296 kqueue 77 7K 40518 cred 27 7K 351 taskqueue 63 7K 63 sglist 5 7K 5 CAM DEV 3 6K 510 plimit 24 6K 506 pwddesc 75 5K 40611 ufs_dirhash 24 5K 24 UMA 272 5K 272 session 35 5K 61 pf_ifnet 10 5K 19 DEVFSP 68 5K 2361 vt 11 5K 11 memdesc 1 4K 1 MCA 32 4K 32 evdev 4 4K 4 proc-args 106 4K 41878 acpisem 28 4K 28 selfd 55 4K 536794 lockf 32 4K 75 hhook 15 4K 17 kcovinfo 52 4K 52 terminal 11 3K 11 select 19 3K 111 uidinfo 3 3K 81 local_apic 1 2K 1 io_apic 1 2K 1 fpukern_ctx 2 2K 2 ipsec-saq 2 2K 2 ip6ndp 12 2K 13 Unitno 34 2K 2409 CC Mem 28 2K 10749 sctp_ifa 13 2K 14 tcp_fsb 13 2K 4416 CAM XPT 22 2K 543 msi 12 2K 12 in_multi 6 2K 8 ipsecpolicy 2 2K 2 inpcbpolicy 40 2K 12068 acpidev 20 2K 20 clone 9 2K 9 tun 7 2K 7 softdep 1 1K 1 mkdir 8 1K 78862 sahead 1 1K 1 secasvar 1 1K 1 nhops 6 1K 8 vnodemarker 2 1K 88 NFSD session 1 1K 1 CAM periph 4 1K 271 osd 32 1K 10756 sctp_ifn 6 1K 14 ipsec 3 1K 3 freework 3 1K 62632 mld 6 1K 6 igmp 6 1K 6 toponodes 6 1K 6 isadev 6 1K 6 mount 16 1K 105 pci_link 10 1K 10 crypto 4 1K 55 encap_export_host 12 1K 12 procdesc 5 1K 18 newdirblk 4 1K 39431 diradd 4 1K 41783 freeblks 2 1K 42413 pfil 4 1K 4 cdev 2 1K 2 chacha20random 1 1K 1 NFSD lckfile 1 1K 1 NFSD V4client 1 1K 1 DEVFS 9 1K 10 vnodes 1 1K 1 CAM SIM 2 1K 2 feeder 7 1K 7 pf_osfp 2 1K 2 tcpfunc 3 1K 3 loginclass 3 1K 7 prison 6 1K 6 lkpikmalloc 5 1K 6 aesni_data 2 1K 2 cryptodev 2 1K 2326 nexusdev 8 1K 8 apmdev 1 1K 1 atkbddev 2 1K 2 CAM dev queue 2 1K 2 CAM I/O Scheduler 1 1K 1 CAM path 4 1K 1034 pmchooks 1 1K 1 soname 4 1K 64195 filecaps 5 1K 193 sctp_vrf 1 1K 1 vnet 1 1K 1 entropy 2 1K 49 pmc 1 1K 1 acpiintr 1 1K 1 sigio 1 1K 3 cpus 2 1K 2 vnet_data_free 1 1K 1 Per-cpu 1 1K 1 p1003.1b 1 1K 1 pf_table 0 0K 0 pf_rule 0 0K 6 pf_altq 0 0K 0 pf_temp 0 0K 0 mqdata 0 0K 0 filemon 0 0K 154 sctp_mcore 0 0K 0 sctp_socko 0 0K 48597 sctp_iter 0 0K 11 sctp_mvrf 0 0K 0 sctp_cpal 0 0K 0 sctp_cmsg 0 0K 0 sctp_stre 0 0K 0 sctp_athi 0 0K 0 sctp_a_it 0 0K 11 sctp_stri 0 0K 623 ipcomp 0 0K 0 esp 0 0K 0 ah 0 0K 0 tcp_do 0 0K 0 NFSCL layrecall 0 0K 0 NFSCL session 0 0K 0 madt_table 0 0K 2 smartpqi 0 0K 0 NFSCL sockreq 0 0K 0 NFSCL devinfo 0 0K 0 NFSCL flayout 0 0K 0 ixl 0 0K 0 NFSCL layout 0 0K 0 NFSD rollback 0 0K 0 NFSCL diroff 0 0K 0 NEWdirectio 0 0K 0 NEWNFSnode 0 0K 0 NFSCL lck 0 0K 0 NFSCL lckown 0 0K 0 ice-resmgr 0 0K 0 ice-osdep 0 0K 0 ice 0 0K 0 iavf 0 0K 0 axgbe 0 0K 0 NFSCL client 0 0K 0 NFSCL deleg 0 0K 0 NFSCL open 0 0K 0 NFSCL owner 0 0K 0 NFS fh 0 0K 0 NFS req 0 0K 0 NFSD usrgroup 0 0K 0 NFSD string 0 0K 0 NFSD V4lock 0 0K 0 xen_intr 0 0K 0 NFSD V4state 0 0K 0 xen_hvm 0 0K 0 legacydrv 0 0K 0 bounce 0 0K 0 busdma 0 0K 0 qpidrv 0 0K 0 NFSD srvcache 0 0K 0 msdosfs_fat 0 0K 0 dmar_idpgtbl 0 0K 0 dmar_dom 0 0K 0 dmar_ctx 0 0K 0 msdosfs_mount 0 0K 0 msdosfs_node 0 0K 0 isci 0 0K 0 iommu_dmamap 0 0K 0 hyperv_socket 0 0K 0 bxe_ilt 0 0K 0 xenbus 0 0K 0 DEVFS4 0 0K 0 vm_fictitious 0 0K 0 DEVFS2 0 0K 0 gntdev 0 0K 0 privcmd_dev 0 0K 0 evtchn_dev 0 0K 0 xenstore 0 0K 0 scsi_pass 0 0K 0 ciss_data 0 0K 0 xnb 0 0K 0 xen_acpi 0 0K 0 xbbd 0 0K 0 xbd 0 0K 0 UMAHash 0 0K 0 Balloon 0 0K 0 sysmouse 0 0K 0 vm_pgdata 0 0K 0 jblocks 0 0K 0 savedino 0 0K 45951 sentinel 0 0K 0 jfsync 0 0K 0 jtrunc 0 0K 0 sbdep 0 0K 42 jsegdep 0 0K 0 jseg 0 0K 0 jfreefrag 0 0K 0 jfreeblk 0 0K 0 jnewblk 0 0K 0 jmvref 0 0K 0 jremref 0 0K 0 jaddref 0 0K 0 freedep 0 0K 0 freefrag 0 0K 28 allocindir 0 0K 0 indirdep 0 0K 29518 allocdirect 0 0K 0 ufs_trim 0 0K 0 mactemp 0 0K 0 audit_trigger 0 0K 0 audit_pipe_presel 0 0K 0 audit_pipeent 0 0K 0 audit_pipe 0 0K 0 audit_evname 0 0K 0 audit_bsm 0 0K 0 audit_gidset 0 0K 0 audit_text 0 0K 0 audit_path 0 0K 0 audit_data 0 0K 0 audit_cred 0 0K 0 vtfont 0 0K 0 BACKLIGHT 0 0K 0 ath_hal 0 0K 0 athdev 0 0K 0 ata_pci 0 0K 0 ata_dma 0 0K 0 ata_generic 0 0K 0 pvscsi 0 0K 0 scsi_da 0 0K 69 ata_da 0 0K 0 scsi_ch 0 0K 0 scsi_cd 0 0K 0 ktls_ocf 0 0K 0 AHCI driver 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5E_TLS_RX 0 0K 0 MLX5EEPROM 0 0K 0 MLX5E_TLS 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EN 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5DUMP 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 seq_file 0 0K 0 lkpiskb 0 0K 0 radix 0 0K 0 idr 0 0K 0 lkpindev 0 0K 0 lkpifw 0 0K 0 lkpi80211 0 0K 0 NLM 0 0K 0 ipsec-spdcache 0 0K 0 ipsec-reg 0 0K 0 ipsec-misc 0 0K 0 ipsecrequest 0 0K 0 ip6opt 0 0K 93 ip6_msource 0 0K 0 ip6_moptions 0 0K 0 in6_mfilter 0 0K 0 frag6 0 0K 5 tcplog 0 0K 0 tcp_hwpace 0 0K 0 USBdev 0 0K 0 USB 0 0K 0 LRO 0 0K 0 ip_msource 0 0K 0 ip_moptions 0 0K 3 in_mfilter 0 0K 0 ipid 0 0K 0 80211scan 0 0K 0 80211ratectl 0 0K 0 80211power 0 0K 0 80211nodeie 0 0K 0 80211node 0 0K 0 80211mesh_gt 0 0K 0 80211mesh_rt 0 0K 0 80211perr 0 0K 0 80211prep 0 0K 0 80211preq 0 0K 0 80211dfs 0 0K 0 80211crypto 0 0K 0 80211vap 0 0K 0 iflib 0 0K 0 vlan 0 0K 0 gif 0 0K 0 ifdescr 0 0K 0 zlib 0 0K 0 fadvise 0 0K 144 VN POLL 0 0K 0 agp 0 0K 0 statfs 0 0K 39685 namei_tracker 0 0K 2 export_host 0 0K 0 cl_savebuf 0 0K 2 nvme_da 0 0K 0 acpipwr 0 0K 0 acpi_perf 0 0K 0 twsbuf 0 0K 0 twe_commands 0 0K 0 tcp_log_dev 0 0K 464 midi buffers 0 0K 0 mixer 0 0K 0 ac97 0 0K 0 hdacc 0 0K 0 hdac 0 0K 0 hdaa 0 0K 0 acpicmbat 0 0K 0 SIIS driver 0 0K 0 CAM CCB 0 0K 523 biobuf 0 0K 0 aios 0 0K 11 lio 0 0K 21 acl 0 0K 0 mbuf_tag 0 0K 142 ktls 0 0K 0 PUC 0 0K 0 ppbusdev 0 0K 0 agtiapi_MemAlloc malloc 0 0K 0 osti_cacheable 0 0K 0 tempbuff 0 0K 0 tempbuff 0 0K 0 ag_tgt_map_t malloc 0 0K 0 ag_slr_map_t malloc 0 0K 0 lDevFlags * malloc 0 0K 0 tiDeviceHandle_t * malloc 0 0K 0 ag_portal_data_t malloc 0 0K 0 ag_device_t malloc 0 0K 0 STLock malloc 0 0K 0 CCB List 0 0K 0 sr_iov 0 0K 0 OCS 0 0K 0 OCS 0 0K 0 nvme 0 0K 0 nvd 0 0K 0 netmap 0 0K 0 mwldev 0 0K 0 MVS driver 0 0K 0 CAM ccb queue 0 0K 0 accf 0 0K 0 pts 0 0K 0 iov 0 0K 31105 ioctlops 0 0K 253 eventfd 0 0K 83 Witness 0 0K 0 stack 0 0K 0 mrsasbuf 0 0K 0 mpt_user 0 0K 0 mps_user 0 0K 0 MPSSAS 0 0K 0 mps 0 0K 0 sbuf 0 0K 288 mpr_user 0 0K 0 firmware 0 0K 0 compressor 0 0K 0 MPRSAS 0 0K 0 SWAP 0 0K 0 mpr 0 0K 0 mfibuf 0 0K 0 sysctltmp 0 0K 922 sysctl 0 0K 3 md_sectors 0 0K 0 ekcd 0 0K 0 dumper 0 0K 0 sendfile 0 0K 0 rctl 0 0K 0 md_disk 0 0K 0 malodev 0 0K 0 LED 0 0K 0 ix_sriov 0 0K 0 cache 0 0K 0 aacraidcam 0 0K 0 prison_racct 0 0K 0 Fail Points 0 0K 0 filedesc_to_leader 0 0K 0 pwd 0 0K 0 tty console 0 0K 0 ix 0 0K 0 ipsbuf 0 0K 0 aacraid_buf 0 0K 0 aaccam 0 0K 0 boottrace 0 0K 0 aacbuf 0 0K 0 zstd 0 0K 0 XZ_DEC 0 0K 0 nvlist 0 0K 0 SCSI ENC 0 0K 0 SCSI sa 0 0K 0 isofs_node 0 0K 0 isofs_mount 0 0K 0 tr_raid5_data 0 0K 0 tr_raid1e_data 0 0K 0 tr_raid1_data 0 0K 0 tr_raid0_data 0 0K 0 tr_concat_data 0 0K 0 md_sii_data 0 0K 0 md_promise_data 0 0K 0 md_nvidia_data 0 0K 0 md_jmicron_data 0 0K 0 md_intel_data 0 0K 0 md_ddf_data 0 0K 0 raid_data 0 0K 72 geom_flashmap 0 0K 0 tmpfs dir 0 0K 0 tmpfs name 0 0K 0 tmpfs mount 0 0K 0 NFS FHA 0 0K 0 newnfsmnt 0 0K 0 newnfsclient_req 0 0K 0 db> show uma Zone Size Used Free Requests Sleeps Bucket Total Mem XFree mbuf_jumbo_page 4096 8381 1017 1441275 0 254 38494208 0 sctp_asoc 2256 1983 307 26265 0 254 5166240 0 malloc-2048 2048 2000 160 62909 0 8 4423680 0 mbuf_cluster 2048 2032 0 2032 0 254 4161536 0 sctp_ep 1208 1996 544 62714 0 254 3068320 0 mbuf 256 10897 541 2500783 0 254 2928128 0 pbuf 2624 0 973 0 0 2 2553152 0 RADIX NODE 144 16369 258 836414 0 62 2394288 0 malloc-1024 1024 1990 42 26272 0 16 2080768 0 BUF TRIE 144 189 11599 56188 0 62 1697472 0 sctp_raddr 736 1983 305 26265 0 254 1683968 0 malloc-384 384 4169 31 4509 0 30 1612800 0 malloc-384 384 3981 219 89363 0 30 1612800 0 malloc-128 128 11626 371 53509 0 126 1535616 0 UMA Slabs 0 112 13088 10 13088 0 126 1466976 0 malloc-4096 4096 328 2 495 0 2 1351680 0 vmem btag 56 23155 68 23155 0 254 1300488 0 tcp_log 416 0 3051 16129 0 254 1269216 0 malloc-384 384 2242 818 42466 0 30 1175040 0 malloc-256 256 262 3038 324789 0 62 844800 0 tcpcb 1104 28 735 10745 0 254 842352 0 malloc-256 256 2295 870 44378 0 62 810240 0 socket 960 79 685 101245 0 254 733440 0 VM OBJECT 264 2493 162 869165 0 31 700920 0 malloc-256 256 2376 294 63653 0 62 683520 0 FFS inode 1160 534 47 42256 0 9 673960 0 256 Bucket 2048 232 26 7087 0 8 528384 0 malloc-16384 16384 26 6 39615 0 1 524288 0 mbuf_packet 256 1556 476 83314 0 254 520192 0 malloc-128 128 2562 848 81734 0 126 436480 0 malloc-4096 4096 97 7 40484 0 2 425984 0 lkpimm 168 1 2327 1 0 62 391104 0 lkpicurr 168 2 2326 2 0 62 391104 0 MAP ENTRY 96 3426 648 2049417 0 127 391104 0 THREAD 1808 175 34 46121 0 8 377872 0 sctp_chunk 152 2047 241 3220 0 254 347776 0 malloc-65536 65536 4 1 140 0 1 327680 0 VNODE 448 575 91 42299 0 30 298368 0 malloc-32768 32768 1 8 27978 0 1 294912 0 malloc-64 64 4148 451 45235 0 254 294336 0 tcp_bbr_map 128 12 2282 49236 0 126 293632 0 malloc-16 16 14655 345 14733 0 254 240000 0 ertt_txseginfo 40 1 5655 175695 0 254 226240 0 DEVCTL 1024 0 220 154 0 0 225280 0 malloc-64 64 2950 389 39821 0 254 213696 0 malloc-65536 65536 1 2 428 0 1 196608 0 unpcb 256 21 744 26407 0 254 195840 0 malloc-32 32 5361 561 5805 0 254 189504 0 UMA Zones 768 244 0 244 0 16 187392 0 FPU_save_area 832 177 48 55907 0 16 187200 0 malloc-128 128 1220 175 27014 0 126 178560 0 VMSPACE 2552 52 14 40434 0 4 168432 0 FFS2 dinode 256 534 96 42255 0 62 161280 0 S VFS Cache 104 1089 432 43503 0 126 158184 0 malloc-32 32 4248 666 92950 0 254 157248 0 128 Bucket 1024 90 57 3630 0 16 150528 0 malloc-1024 1024 121 23 338 0 16 147456 0 64 Bucket 512 148 116 41811 0 30 135168 0 malloc-65536 65536 2 0 2 0 1 131072 0 malloc-65536 65536 0 2 370 0 1 131072 0 PROC 1352 74 22 40461 0 8 129792 0 sctp_stream_msg_out 112 698 346 3650 0 254 116928 0 ksiginfo 112 87 957 10146 0 126 116928 0 filedesc0 1072 75 23 40611 0 8 105056 0 malloc-128 128 539 236 10085 0 126 99200 0 malloc-8192 8192 6 6 289 0 1 98304 0 malloc-256 256 224 151 46238 0 62 96000 0 UMA Kegs 384 230 3 230 0 30 89472 0 g_bio 408 0 210 405605 0 30 85680 0 pipe 744 24 91 843 0 16 85560 0 clpbuf 2624 0 32 53 0 16 83968 0 malloc-128 128 466 185 1504 0 126 83328 0 malloc-2048 2048 5 35 1222 0 8 81920 0 32 Bucket 256 114 201 26879 0 62 80640 0 sctp_readq 152 0 520 108 0 254 79040 0 tcp_inpcb 424 28 152 10745 0 30 76320 0 malloc-16 16 4038 712 126758 0 254 76000 0 malloc-16 16 3993 507 52622 0 254 72000 0 malloc-64 64 568 503 3120 0 254 68544 0 malloc-65536 65536 1 0 1 0 1 65536 0 malloc-65536 65536 1 0 1 0 1 65536 0 malloc-32768 32768 0 2 120 0 1 65536 0 malloc-4096 4096 13 3 27 0 2 65536 0 ripcb 424 6 147 635 0 30 64872 0 Files 80 261 539 162330 0 126 64000 0 udp_inpcb 424 6 120 490 0 30 53424 0 malloc-64 64 2 817 19629 0 254 52416 0 malloc-64 64 198 621 577560 0 254 52416 0 malloc-128 128 165 238 761 0 126 51584 0 malloc-128 128 188 215 311 0 126 51584 0 malloc-128 128 39 364 79049 0 126 51584 0 DIRHASH 1024 35 13 35 0 16 49152 0 NAMEI 1024 0 48 186128 0 16 49152 0 malloc-8192 8192 6 0 6 0 1 49152 0 malloc-2048 2048 10 14 13 0 8 49152 0 malloc-1024 1024 27 21 179 0 16 49152 0 malloc-1024 1024 6 42 531 0 16 49152 0 tcp_rack_pcb 896 13 41 4416 0 16 48384 0 malloc-384 384 97 23 98 0 30 46080 0 tcp_bbr_pcb 832 6 48 961 0 16 44928 0 syncache 168 0 264 4 0 254 44352 0 TURNSTILE 136 210 105 210 0 62 42840 0 malloc-8192 8192 4 1 6 0 1 40960 0 malloc-8192 8192 5 0 5 0 1 40960 0 malloc-4096 4096 5 5 634 0 2 40960 0 pcpu-8 8 4704 416 5161 0 254 40960 0 sctp_laddr 48 148 692 54523 0 254 40320 0 udplite_inpcb 424 0 90 198 0 30 38160 0 da_ccb 544 0 70 101506 0 16 38080 0 malloc-64 64 200 367 234 0 254 36288 0 malloc-64 64 105 462 42985 0 254 36288 0 malloc-64 64 18 549 43 0 254 36288 0 16 Bucket 144 92 160 6483 0 62 36288 0 8 Bucket 80 59 391 5087 0 126 36000 0 routing nhops 256 27 108 34 0 62 34560 0 ttyoutq 256 72 63 160 0 62 34560 0 malloc-384 384 43 47 45 0 30 34560 0 malloc-384 384 50 40 131 0 30 34560 0 malloc-256 256 2 133 6977 0 62 34560 0 malloc-256 256 46 89 50 0 62 34560 0 malloc-256 256 56 79 998 0 62 34560 0 malloc-256 256 18 117 686 0 62 34560 0 SLEEPQUEUE 88 210 174 210 0 126 33792 0 malloc-32768 32768 1 0 1 0 1 32768 0 malloc-32768 32768 1 0 1 0 1 32768 0 malloc-16384 16384 2 0 17 0 1 32768 0 malloc-2048 2048 3 13 8 0 8 32768 0 malloc-2048 2048 3 13 138 0 8 32768 0 malloc-2048 2048 7 9 117 0 8 32768 0 malloc-1024 1024 0 32 99 0 16 32768 0 malloc-1024 1024 6 26 1302 0 16 32768 0 malloc-512 512 11 53 97 0 30 32768 0 malloc-512 512 3 61 170 0 30 32768 0 malloc-512 512 2 62 625 0 30 32768 0 malloc-512 512 10 54 70 0 30 32768 0 pcpu-64 64 493 19 493 0 254 32768 0 KNOTE 160 28 172 301529 0 62 32000 0 ttyinq 160 135 65 300 0 62 32000 0 tcp_rack_map 120 0 264 5241 0 126 31680 0 cpuset 104 7 272 4737 0 126 29016 0 tcp_inpcb ports 32 2 880 4055 0 254 28224 0 ertt 72 28 364 10745 0 126 28224 0 PWD 32 34 848 39533 0 254 28224 0 malloc-32 32 177 705 14879 0 254 28224 0 4 Bucket 48 6 582 870 0 254 28224 0 2 Bucket 32 63 819 8412 0 254 28224 0 AIO 208 0 133 37 0 62 27664 0 malloc-8192 8192 1 2 10 0 1 24576 0 malloc-4096 4096 2 4 39730 0 2 24576 0 malloc-4096 4096 6 0 6 0 2 24576 0 rtentry 176 30 108 34 0 62 24288 0 PGRP 88 35 241 61 0 126 24288 0 rl_entry 40 91 515 91 0 254 24240 0 itimer 352 0 66 8 0 30 23232 0 domainset 40 0 567 3551 0 254 22680 0 hostcache 64 1 314 1 0 254 20160 0 udpcb 32 6 624 688 0 254 20160 0 udplite_inpcb ports 32 0 630 7 0 254 20160 0 udp_inpcb ports 32 3 627 44 0 254 20160 0 malloc-32 32 5 625 26043 0 254 20160 0 malloc-32 32 100 530 40358 0 254 20160 0 malloc-32 32 128 502 554 0 254 20160 0 malloc-32 32 56 574 270 0 254 20160 0 malloc-32 32 24 606 1094 0 254 20160 0 AIOCB 552 0 35 83 0 16 19320 0 Mountpoints 2752 2 5 2 0 4 19264 0 AIOLIO 272 0 70 21 0 30 19040 0 epoch_record pcpu 256 4 60 4 0 62 16384 0 malloc-16384 16384 0 1 1 0 1 16384 0 malloc-16384 16384 1 0 1 0 1 16384 0 malloc-16384 16384 1 0 1 0 1 16384 0 malloc-8192 8192 2 0 2 0 1 16384 0 malloc-4096 4096 3 1 3 0 2 16384 0 malloc-2048 2048 1 7 1 0 8 16384 0 malloc-2048 2048 2 6 2 0 8 16384 0 malloc-1024 1024 4 12 4 0 16 16384 0 malloc-1024 1024 1 15 1 0 16 16384 0 malloc-512 512 0 32 11 0 30 16384 0 malloc-512 512 0 32 7 0 30 16384 0 SMR CPU 32 7 504 7 0 254 16352 0 ipq 56 0 288 1 0 254 16128 0 vtnet_tx_hdr 24 0 668 764815 0 254 16032 0 malloc-16 16 482 518 3454 0 254 16000 0 malloc-16 16 55 945 41319 0 254 16000 0 kenv 258 15 45 1053 0 30 15480 0 mqnode 416 3 33 3 0 30 14976 0 vmem 1856 1 7 1 0 8 14848 0 SMR SHARED 24 7 504 7 0 254 12264 0 AIOP 32 4 374 4 0 254 12096 0 malloc-16 16 9 741 85578 0 254 12000 0 malloc-16 16 22 728 135 0 254 12000 0 malloc-16 16 20 730 60 0 254 12000 0 malloc-384 384 0 30 1 0 30 11520 0 malloc-384 384 1 29 1 0 30 11520 0 malloc-8192 8192 1 0 1 0 1 8192 0 malloc-4096 4096 0 2 1 0 2 8192 0 pcpu-16 16 7 249 7 0 254 4096 0 UMA Slabs 1 176 10 12 10 0 62 3872 0 KMAP ENTRY 96 12 27 14 0 0 3744 0 pcpu-4 4 1 511 1 0 254 2048 0 FFS1 dinode 128 0 0 0 0 126 0 0 ada_ccb 272 0 0 0 0 30 0 0 swblk 136 0 0 0 0 62 0 0 swpctrie 144 0 0 0 0 62 0 0 cdg_qdiffsample 16 0 0 0 0 254 0 0 pf state scrubs 40 0 0 0 0 254 0 0 pf frag entries 40 0 0 0 0 254 0 0 pf frags 248 0 0 0 0 62 0 0 pf table entries 160 0 0 0 0 254 0 0 pf table entry counters 64 0 0 0 0 254 0 0 pf source nodes 136 0 0 0 0 254 0 0 pf state keys 88 0 0 0 0 126 0 0 pf states 312 0 0 0 0 254 0 0 pf tags 104 0 0 0 0 126 0 0 pf mtags 184 0 0 0 0 62 0 0 tfo_ccache_entries 80 0 0 0 0 126 0 0 tfo 4 0 0 0 0 254 0 0 sackhole 32 0 0 0 0 254 0 0 tcptw 72 0 0 0 0 254 0 0 sctp_asconf_ack 48 0 0 0 0 254 0 0 sctp_asconf 40 0 0 0 0 254 0 0 tcp_log_id_node 120 0 0 0 0 126 0 0 tcp_log_id_bucket 176 0 0 0 0 62 0 0 tcpreass 48 0 0 0 0 254 0 0 ripcb ports 32 0 0 0 0 254 0 0 IPsec SA lft_c 16 0 0 0 0 254 0 0 TMPFS node 224 0 0 0 0 62 0 0 NCLNODE 608 0 0 0 0 16 0 0 mqnotifier 216 0 0 0 0 62 0 0 mvdata 64 0 0 0 0 254 0 0 mqueue 248 0 0 0 0 62 0 0 LTS VFS Cache 360 0 0 0 0 30 0 0 L VFS Cache 320 0 0 0 0 30 0 0 STS VFS Cache 144 0 0 0 0 62 0 0 cryptop 280 0 0 0 0 30 0 0 linux_dma_object 32 0 0 0 0 254 0 0 linux_dma_pctrie 144 0 0 0 0 62 0 0 IOMMU_MAP_ENTRY 120 0 0 0 0 126 0 0 mbuf_jumbo_16k 16384 0 0 0 0 254 0 0 mbuf_jumbo_9k 9216 0 0 0 0 254 0 0 audit_record 1280 0 0 0 0 8 0 0 MAC labels 40 0 0 0 0 254 0 0 vnpbuf 2624 0 0 0 0 64 0 0 mdpbuf 2624 0 0 0 0 3 0 0 nfspbuf 2624 0 0 0 0 16 0 0 swwbuf 2624 0 0 0 0 8 0 0 swrbuf 2624 0 0 0 0 16 0 0 umtx_shm 88 0 0 0 0 126 0 0 umtx pi 96 0 0 0 0 126 0 0 rangeset pctrie nodes 144 0 0 0 0 62 0 0 malloc-65536 65536 0 0 0 0 1 0 0 malloc-65536 65536 0 0 0 0 1 0 0 malloc-32768 32768 0 0 0 0 1 0 0 malloc-32768 32768 0 0 0 0 1 0 0 malloc-32768 32768 0 0 0 0 1 0 0 malloc-32768 32768 0 0 0 0 1 0 0 malloc-16384 16384 0 0 0 0 1 0 0 malloc-16384 16384 0 0 0 0 1 0 0 malloc-16384 16384 0 0 0 0 1 0 0 malloc-8192 8192 0 0 0 0 1 0 0 malloc-512 512 0 0 0 0 30 0 0 malloc-512 512 0 0 0 0 30 0 0 pcpu-32 32 0 0 0 0 254 0 0 fakepg 104 0 0 0 0 126 0 0 UMA Hash 256 0 0 0 0 62 0 0