panic: already suspended cpuid = 1 time = 1654582097 KDB: stack backtrace: db_trace_self_wrapper() at db_trace_self_wrapper+0xc7/frame 0xfffffe00a5bd4530 kdb_backtrace() at kdb_backtrace+0xd3/frame 0xfffffe00a5bd4690 vpanic() at vpanic+0x254/frame 0xfffffe00a5bd4770 panic() at panic+0xb5/frame 0xfffffe00a5bd4830 thread_suspend_one() at thread_suspend_one+0x107/frame 0xfffffe00a5bd4860 thread_single() at thread_single+0x79c/frame 0xfffffe00a5bd4910 reap_kill_proc() at reap_kill_proc+0x4b8/frame 0xfffffe00a5bd4990 reap_kill() at reap_kill+0x7fc/frame 0xfffffe00a5bd4ba0 kern_procctl() at kern_procctl+0x51a/frame 0xfffffe00a5bd4c10 sys_procctl() at sys_procctl+0x247/frame 0xfffffe00a5bd4d30 amd64_syscall() at amd64_syscall+0x410/frame 0xfffffe00a5bd4f30 fast_syscall_common() at fast_syscall_common+0xf8/frame 0xfffffe00a5bd4f30 --- syscall (198, FreeBSD ELF64, nosys), rip = 0x28a47a, rsp = 0x83452ef08, rbp = 0x83452ef70 --- KDB: enter: panic [ thread pid 30442 tid 135235 ] Stopped at kdb_enter+0x6b: movq $0,0x275b6aa(%rip) db> db> set $lines = 0 db> set $maxwidth = 0 db> show registers cs 0x20 ds 0x3b es 0x3b fs 0x13 gs 0x1b ss 0x28 rax 0x12 rcx 0xfffffe00a0c54000 rdx 0x3ffff rbx 0 rsp 0xfffffe00a5bd4670 rbp 0xfffffe00a5bd4690 rsi 0x40001 rdi 0xffffffff8172b448 printf+0x158 r8 0 r9 0xffffffff r10 0 r11 0xfffffe0058a9da70 r12 0 r13 0xfffffe009f135560 r14 0xffffffff82b598a0 .str.26 r15 0xffffffff82b598a0 .str.26 rip 0xffffffff8171af6b kdb_enter+0x6b rflags 0x46 kdb_enter+0x6b: movq $0,0x275b6aa(%rip) db> show proc Process 30442 (syz-executor.1) at 0xfffffe00a65b7000: state: NORMAL uid: 0 gids: 0, 0, 5 parent: pid 780 at 0xfffffe009eaf2548 ABI: FreeBSD ELF64 flag: 0x10080080 flag2: 0 arguments: /root/syz-executor.1 exec reaper: 0xfffffe0053ddd000 reapsubtree: 1 sigparent: 20 vmspace: 0xfffffe00a65953f0 (map 0xfffffe00a65953f0) (map.pmap 0xfffffe00a65954b0) (pmap 0xfffffe00a6595518) threads: 2 134943 s syz-executor.1 135235 Run CPU 1 syz-executor.1 db> ps pid ppid pgrp uid state wmesg wchan cmd 30444 779 779 0 RE CPU 0 syz-executor.0 30442 780 780 0 T (threaded) syz-executor.1 134943 s syz-executor.1 135235 Run CPU 1 syz-executor.1 30439 792 792 0 TL (threaded) syz-executor.3 135221 s syz-executor.3 135232 RunQ syz-executor.3 135236 Ss uwait 0xfffffe009f13be80 syz-executor.3 30208 1 780 0 S uwait 0xfffffe0057873580 syz-executor.1 30203 1 780 0 S uwait 0xfffffe00a6211400 syz-executor.1 30199 1 780 0 S uwait 0xfffffe009f0d7700 syz-executor.1 26123 1 790 0 S uwait 0xfffffe0053de7500 syz-executor.2 26120 1 790 0 S uwait 0xfffffe005730bc80 syz-executor.2 26119 1 790 0 S uwait 0xfffffe009f0d7180 syz-executor.2 26109 1 790 0 S uwait 0xfffffe009f13b300 syz-executor.2 8141 1 8141 65 Ss select 0xfffffe0007986ac0 dhclient 7087 1 7087 0 Ss select 0xfffffe0007986cc0 dhclient 7083 1 7083 0 Ss select 0xfffffe005788b3c0 dhclient 7055 1 7055 65 Ss select 0xfffffe005788b640 dhclient 6083 1 6083 0 Ss select 0xfffffe005788b6c0 dhclient 6079 1 6079 0 Ss select 0xfffffe005788b740 dhclient 6058 1 6058 65 Ss select 0xfffffe005788ba40 dhclient 4348 1 4348 0 Ss select 0xfffffe0007986d40 dhclient 4345 1 4345 0 Ss select 0xfffffe005788b940 dhclient 4323 1 4323 65 Ss select 0xfffffe0007986f40 dhclient 2626 1 792 0 SV uwait 0xfffffe0053de7a00 syz-executor.3 2614 1 792 0 SV uwait 0xfffffe005730ee00 syz-executor.3 2603 1 792 0 SV uwait 0xfffffe0057872e00 syz-executor.3 1043 1 1043 0 Ss select 0xfffffe0007986e40 dhclient 1040 1 1040 0 Ss select 0xfffffe0007986dc0 dhclient 901 1 780 0 SV uwait 0xfffffe0057873180 syz-executor.1 889 1 780 0 SV uwait 0xfffffe0057872f00 syz-executor.1 839 1 780 0 SV uwait 0xfffffe0057872880 syz-executor.1 836 0 0 0 DL aiordy 0xfffffe009c33f548 [aiod4] 834 0 0 0 DL aiordy 0xfffffe009f0c6000 [aiod3] 832 0 0 0 DL aiordy 0xfffffe009f03da90 [aiod2] 828 0 0 0 DL aiordy 0xfffffe009f03d000 [aiod1] 792 774 792 0 Ss nanslp 0xffffffff83e43c81 syz-executor.3 790 774 790 0 Ss piperd 0xfffffe009f0182e8 syz-executor.2 780 774 780 0 Ss nanslp 0xffffffff83e43c81 syz-executor.1 779 774 779 0 Rs syz-executor.0 774 772 772 0 S (threaded) syz-fuzzer 100106 S uwait 0xfffffe0057874300 syz-fuzzer 100111 S uwait 0xfffffe005730b080 syz-fuzzer 100112 S kqread 0xfffffe0053e8e200 syz-fuzzer 100113 S uwait 0xfffffe005730b280 syz-fuzzer 100114 S uwait 0xfffffe005730ba80 syz-fuzzer 100115 S uwait 0xfffffe005730b380 syz-fuzzer 100116 S uwait 0xfffffe005730b480 syz-fuzzer 100117 S uwait 0xfffffe005730b580 syz-fuzzer 100119 S uwait 0xfffffe005730bb80 syz-fuzzer 772 770 772 0 Ss pause 0xfffffe009c340b40 csh 770 688 770 0 Ss select 0xfffffe0007987340 sshd 754 1 754 0 Ss+ ttyin 0xfffffe00081ec8b0 getty 753 1 753 0 Ss+ ttyin 0xfffffe00586000b0 getty 752 1 752 0 Ss+ ttyin 0xfffffe00586004b0 getty 751 1 751 0 Ss+ ttyin 0xfffffe00586008b0 getty 750 1 750 0 Ss+ ttyin 0xfffffe0058600cb0 getty 749 1 749 0 Ss+ ttyin 0xfffffe00586010b0 getty 748 1 748 0 Ss+ ttyin 0xfffffe00586014b0 getty 747 1 747 0 Ss+ ttyin 0xfffffe00586018b0 getty 746 1 746 0 Ss+ ttyin 0xfffffe0058601cb0 getty 692 1 692 0 Ss nanslp 0xffffffff83e43c80 cron 688 1 688 0 Ss select 0xfffffe00079875c0 sshd 501 1 501 0 Ss select 0xfffffe0007987640 syslogd 430 1 430 0 Ss select 0xfffffe005788bec0 devd 429 1 429 65 Ss select 0xfffffe005788bc40 dhclient 344 1 344 0 Ss select 0xfffffe00079877c0 dhclient 341 1 341 0 Ss select 0xfffffe0007987a40 dhclient 17 0 0 0 DL vlruwt 0xfffffe0056f78548 [vnlru] 16 0 0 0 DL syncer 0xffffffff83f694e0 [syncer] 15 0 0 0 DL (threaded) [bufdaemon] 100079 D psleep 0xffffffff83f67ae0 [bufdaemon] 100082 D - 0xffffffff83211f80 [bufspacedaemon-0] 100095 D sdflush 0xfffffe00579c4ce8 [/ worker] 9 0 0 0 DL psleep 0xffffffff83f9b5c0 [vmdaemon] 8 0 0 0 DL (threaded) [pagedaemon] 100077 D psleep 0xffffffff83f8f478 [dom0] 100083 D launds 0xffffffff83f8f484 [laundry: dom0] 100084 D umarcl 0xffffffff81e4b8d0 [uma] 7 0 0 0 DL - 0xffffffff83c002a8 [rand_harvestq] 6 0 0 0 DL pftm 0xffffffff84571550 [pf purge] 5 0 0 0 DL waiting 0xffffffff848b3460 [sctp_iterator] 4 0 0 0 DL (threaded) [cam] 100044 D - 0xffffffff83aa3240 [doneq0] 100045 D - 0xffffffff83aa31c0 [async] 100076 D - 0xffffffff83aa3040 [scanner] 14 0 0 0 DL seqstat 0xfffffe0053f9f088 [sequencer 00] 3 0 0 0 DL (threaded) [crypto] 100040 D crypto_ 0xffffffff83f8aca0 [crypto] 100041 D crypto_ 0xfffffe0053c8b330 [crypto returns 0] 100042 D crypto_ 0xfffffe0053c8b380 [crypto returns 1] 13 0 0 0 DL (threaded) [geom] 100035 D - 0xffffffff83e19280 [g_event] 100036 D - 0xffffffff83e192a0 [g_up] 100037 D - 0xffffffff83e192c0 [g_down] 2 0 0 0 WL (threaded) [clock] 100029 I [clock (0)] 100030 I [clock (1)] 12 0 0 0 WL (threaded) [intr] 100015 I [swi5: fast taskq] 100018 I [swi6: task queue] 100019 I [swi6: Giant taskq] 100031 I [swi1: netisr 0] 100032 I [swi1: hpts] 100033 I [swi1: hpts] 100046 I [irq24: virtio_pci0] 100047 I [irq25: virtio_pci0] 100048 I [irq26: virtio_pci0] 100049 I [irq27: virtio_pci0] 100050 I [irq28: virtio_pci1] 100051 I [irq29: virtio_pci1] 100052 I [irq30: virtio_pci1] 100053 I [irq31: virtio_pci1] 100054 I [irq32: virtio_pci1] 100059 I [irq33: virtio_pci2] 100060 I [irq34: virtio_pci2] 100061 I [irq35: virtio_pci2] 100063 I [irq1: atkbd0] 100064 I [irq12: psm0] 100065 I [swi0: uart uart++] 100069 I [swi1: pf send] 11 0 0 0 RL (threaded) [idle] 100003 CanRun [idle: cpu0] 100004 CanRun [idle: cpu1] 1 0 1 0 SLs wait 0xfffffe0053ddd000 [init] 10 0 0 0 DL audit_w 0xffffffff83f8b780 [audit] 0 0 0 0 DLs (threaded) [kernel] 100000 D swapin 0xffffffff83e19c80 [swapper] 100005 D - 0xfffffe0053e98100 [if_config_tqg_0] 100006 D - 0xfffffe0053e98000 [softirq_0] 100007 D - 0xfffffe0053e97e00 [softirq_1] 100008 D - 0xfffffe0053e97d00 [if_io_tqg_0] 100009 D - 0xfffffe0053e97c00 [if_io_tqg_1] 100010 D - 0xfffffe0008183d00 [inm_free taskq] 100011 D - 0xfffffe0008183c00 [linuxkpi_irq_wq] 100012 D - 0xfffffe0008183b00 [in6m_free taskq] 100013 D - 0xfffffe0008183a00 [deferred_unmount ta] 100014 D - 0xfffffe0008183900 [thread taskq] 100016 D - 0xfffffe0008183700 [kqueue_ctx taskq] 100017 D - 0xfffffe0008183600 [pci_hp taskq] 100020 D - 0xfffffe0008183300 [aiod_kick taskq] 100021 D - 0xfffffe0008183200 [linuxkpi_short_wq_0] 100022 D - 0xfffffe0008183200 [linuxkpi_short_wq_1] 100023 D - 0xfffffe0008183200 [linuxkpi_short_wq_2] 100024 D - 0xfffffe0008183200 [linuxkpi_short_wq_3] 100025 D - 0xfffffe0008183100 [linuxkpi_long_wq_0] 100026 D - 0xfffffe0008183100 [linuxkpi_long_wq_1] 100027 D - 0xfffffe0008183100 [linuxkpi_long_wq_2] 100028 D - 0xfffffe0008183100 [linuxkpi_long_wq_3] 100034 D - 0xfffffe0008183000 [firmware taskq] 100038 D - 0xfffffe0008182d00 [crypto_0] 100039 D - 0xfffffe0008182d00 [crypto_1] 100055 D - 0xfffffe0056fe8b00 [vtnet0 rxq 0] 100056 D - 0xfffffe0056fe8a00 [vtnet0 txq 0] 100057 D - 0xfffffe0056fe8900 [vtnet0 rxq 1] 100058 D - 0xfffffe0056fe8800 [vtnet0 txq 1] 100062 D vtbslp 0xfffffe0057317c80 [virtio_balloon] 100066 D - 0xffffffff82b5f720 [deadlkres] 100070 D - 0xfffffe0008184100 [mca taskq] 100071 D - 0xfffffe0057926400 [acpi_task_0] 100072 D - 0xfffffe0057926400 [acpi_task_1] 100073 D - 0xfffffe0057926400 [acpi_task_2] 100075 D - 0xfffffe0008182a00 [CAM taskq] db> show all locks Process 30442 (syz-executor.1) thread 0xfffffe009f135560 (135235) exclusive sleep mutex process lock (process lock) r = 0 (0xfffffe00a65b9128) locked @ /syzkaller/managers/main/kernel/sys/kern/kern_thread.c:1523 exclusive sx sapblk (sapblk) r = 0 (0xffffffff83e32020) locked @ /syzkaller/managers/main/kernel/sys/kern/kern_proc.c:3405 Process 30439 (syz-executor.3) thread 0xfffffe009f01e740 (135232) exclusive lockmgr bufwait (bufwait) r = 0 (0xfffffe0007a85218) locked @ /syzkaller/managers/main/kernel/sys/kern/vfs_bio.c:1725 exclusive lockmgr bufwait (bufwait) r = 0 (0xfffffe0007aa9aa0) locked @ /syzkaller/managers/main/kernel/sys/kern/vfs_bio.c:3988 exclusive lockmgr ufs (ufs) r = 0 (0xfffffe00a5ca7cb0) locked @ /syzkaller/managers/main/kernel/sys/kern/vfs_vnops.c:1166 db> show malloc Type InUse MemUse Requests pf_hash 5 11524K 5 tcp_hpts 7 4801K 7 devbuf 4217 4323K 4245 sysctloid 35322 2081K 35393 vtbuf 24 1968K 46 kobj 328 1312K 490 newblk 54 1038K 99540 vfscache 3 1025K 3 pcb 417 980K 91598 inodedep 14 517K 36731 ufs_quota 1 512K 1 vfs_hash 1 512K 1 callout 2 512K 2 intr 4 472K 4 subproc 164 327K 30542 filedesc 40 313K 58873 vmem 3 274K 5 sctp_stro 189 189K 16390 acpica 1674 184K 57552 vnet_data 1 168K 1 sctp_atcl 386 145K 62450 tidhash 3 141K 3 linker 358 134K 386 pagedep 12 131K 29468 tfo_ccache 1 128K 1 DEVFS1 109 109K 126 sem 4 106K 4 BPF 46 88K 56 bus 995 81K 5208 mtx_pool 2 72K 2 sctp_timw 274 69K 274 syncache 1 68K 1 module 514 65K 514 acpitask 1 64K 1 ddb_capture 1 64K 1 umtx 418 53K 418 kdtrace 241 47K 65683 temp 35 34K 2773 DEVFS3 128 32K 138 hostcache 1 32K 1 shm 1 32K 152 msg 4 30K 4 gtaskqueue 18 26K 18 sctp_atky 575 24K 80284 kbdmux 6 22K 6 ifaddr 71 21K 73 DEVFS_RULE 56 20K 56 ufs_mount 4 17K 5 proc 3 17K 3 tty 16 16K 16 routetbl 136 16K 428 ithread 97 16K 97 bus-sc 34 15K 1681 lltable 43 14K 188 ether_multi 157 13K 167 KTRACE 100 13K 100 ifnet 7 13K 7 eventhandler 136 12K 136 kenv 95 12K 95 rman 88 11K 431 GEOM 61 11K 490 CAM queue 5 11K 1528 in6_multi 71 9K 71 bmsafemap 2 9K 33777 UART 12 9K 12 devstat 4 9K 4 ksem 1 8K 33 rpc 2 8K 2 shmfd 1 8K 20 pfs_vncache 1 8K 1 pfs_nodes 20 8K 20 audit_evclass 237 8K 296 kqueue 75 7K 30502 taskqueue 63 7K 63 cred 26 7K 303 sglist 5 7K 5 sctp_athm 386 7K 62878 CAM DEV 3 6K 510 plimit 24 6K 499 sctp_map 378 6K 32780 ufs_dirhash 24 5K 24 pwddesc 72 5K 30445 UMA 272 5K 272 session 35 5K 125 pf_ifnet 10 5K 19 DEVFSP 68 5K 275 vt 11 5K 11 memdesc 1 4K 1 MCA 32 4K 32 evdev 4 4K 4 freework 15 4K 58258 proc-args 104 4K 31861 acpisem 28 4K 28 selfd 55 4K 415585 lockf 32 4K 63 hhook 15 4K 17 kcovinfo 52 4K 52 terminal 11 3K 11 select 19 3K 65 uidinfo 3 3K 13 local_apic 1 2K 1 io_apic 1 2K 1 fpukern_ctx 2 2K 2 ipsec-saq 2 2K 2 ip6ndp 12 2K 15 Unitno 33 2K 8281 sctp_ifa 14 2K 15 CAM XPT 22 2K 543 msi 12 2K 12 in_multi 6 2K 8 ipsecpolicy 2 2K 2 acpidev 20 2K 20 clone 9 2K 9 tun 7 2K 7 softdep 1 1K 1 sahead 1 1K 1 secasvar 1 1K 1 nhops 6 1K 8 vnodemarker 2 1K 88 NFSD session 1 1K 1 newdirblk 7 1K 29446 CAM periph 4 1K 271 ipsec 3 1K 3 sctp_ifn 6 1K 15 dirrem 3 1K 31951 diradd 6 1K 31995 mld 6 1K 6 igmp 6 1K 6 toponodes 6 1K 6 isadev 6 1K 6 mount 16 1K 89 pci_link 10 1K 10 crypto 4 1K 24 encap_export_host 12 1K 12 procdesc 5 1K 18 inpcbpolicy 19 1K 14628 mkdir 4 1K 58892 freefile 4 1K 31934 freeblks 2 1K 33493 indirdep 2 1K 42813 pfil 4 1K 4 cdev 2 1K 2 osd 10 1K 12208 CC Mem 6 1K 12197 chacha20random 1 1K 1 pf_osfp 3 1K 3 NFSD lckfile 1 1K 1 NFSD V4client 1 1K 1 DEVFS 9 1K 10 vnodes 1 1K 1 CAM SIM 2 1K 2 feeder 7 1K 7 tcpfunc 3 1K 3 loginclass 3 1K 20 prison 6 1K 6 lkpikmalloc 5 1K 6 aesni_data 2 1K 2 cryptodev 2 1K 137 nexusdev 8 1K 8 apmdev 1 1K 1 atkbddev 2 1K 2 CAM dev queue 2 1K 2 CAM I/O Scheduler 1 1K 1 CAM path 4 1K 1034 iov 2 1K 41208 pmchooks 1 1K 1 soname 4 1K 62358 filecaps 5 1K 125 sctp_vrf 1 1K 1 vnet 1 1K 1 entropy 2 1K 49 pmc 1 1K 1 acpiintr 1 1K 1 cpus 2 1K 2 vnet_data_free 1 1K 1 Per-cpu 1 1K 1 p1003.1b 1 1K 1 tcp_do 0 0K 0 tcp_fsb 0 0K 838 ipcomp 0 0K 0 esp 0 0K 0 ah 0 0K 0 mqdata 0 0K 0 sctp_mcore 0 0K 0 sctp_socko 0 0K 35949 sctp_iter 0 0K 12 sctp_mvrf 0 0K 0 sctp_cpal 0 0K 0 sctp_cmsg 0 0K 0 sctp_stre 0 0K 0 sctp_athi 0 0K 0 sctp_a_it 0 0K 12 sctp_aadr 0 0K 0 sctp_stri 0 0K 690 filemon 0 0K 139 pf_table 0 0K 0 pf_rule 0 0K 0 pf_altq 0 0K 0 pf_temp 0 0K 0 NFSCL layrecall 0 0K 0 NFSCL session 0 0K 0 madt_table 0 0K 2 smartpqi 0 0K 0 NFSCL sockreq 0 0K 0 NFSCL devinfo 0 0K 0 NFSCL flayout 0 0K 0 ixl 0 0K 0 NFSCL layout 0 0K 0 NFSD rollback 0 0K 0 NFSCL diroff 0 0K 0 NEWdirectio 0 0K 0 NEWNFSnode 0 0K 0 NFSCL lck 0 0K 0 NFSCL lckown 0 0K 0 ice-resmgr 0 0K 0 ice-osdep 0 0K 0 ice 0 0K 0 iavf 0 0K 0 axgbe 0 0K 0 NFSCL client 0 0K 0 NFSCL deleg 0 0K 0 NFSCL open 0 0K 0 NFSCL owner 0 0K 0 NFS fh 0 0K 0 NFS req 0 0K 0 NFSD usrgroup 0 0K 0 NFSD string 0 0K 0 NFSD V4lock 0 0K 0 xen_intr 0 0K 0 NFSD V4state 0 0K 0 xen_hvm 0 0K 0 legacydrv 0 0K 0 bounce 0 0K 0 busdma 0 0K 0 qpidrv 0 0K 0 NFSD srvcache 0 0K 0 msdosfs_fat 0 0K 0 dmar_idpgtbl 0 0K 0 dmar_dom 0 0K 0 dmar_ctx 0 0K 0 msdosfs_mount 0 0K 0 msdosfs_node 0 0K 0 isci 0 0K 0 iommu_dmamap 0 0K 0 hyperv_socket 0 0K 0 bxe_ilt 0 0K 0 xenbus 0 0K 0 DEVFS4 0 0K 0 vm_fictitious 0 0K 0 DEVFS2 0 0K 0 gntdev 0 0K 0 privcmd_dev 0 0K 0 evtchn_dev 0 0K 0 xenstore 0 0K 0 scsi_pass 0 0K 0 ciss_data 0 0K 0 xnb 0 0K 0 xen_acpi 0 0K 0 xbbd 0 0K 0 xbd 0 0K 0 UMAHash 0 0K 0 Balloon 0 0K 0 sysmouse 0 0K 0 vm_pgdata 0 0K 0 jblocks 0 0K 0 savedino 0 0K 55574 sentinel 0 0K 0 jfsync 0 0K 0 jtrunc 0 0K 0 sbdep 0 0K 41 jsegdep 0 0K 0 jseg 0 0K 0 jfreefrag 0 0K 0 jfreeblk 0 0K 0 jnewblk 0 0K 0 jmvref 0 0K 0 jremref 0 0K 0 jaddref 0 0K 0 freedep 0 0K 0 freefrag 0 0K 2 allocindir 0 0K 0 allocdirect 0 0K 0 ufs_trim 0 0K 0 mactemp 0 0K 0 audit_trigger 0 0K 0 audit_pipe_presel 0 0K 0 audit_pipeent 0 0K 0 audit_pipe 0 0K 0 audit_evname 0 0K 0 audit_bsm 0 0K 0 audit_gidset 0 0K 0 audit_text 0 0K 0 audit_path 0 0K 0 audit_data 0 0K 0 audit_cred 0 0K 0 vtfont 0 0K 0 BACKLIGHT 0 0K 0 ath_hal 0 0K 0 athdev 0 0K 0 ata_pci 0 0K 0 ata_dma 0 0K 0 ata_generic 0 0K 0 pvscsi 0 0K 0 scsi_da 0 0K 69 ata_da 0 0K 0 scsi_ch 0 0K 0 scsi_cd 0 0K 0 ktls_ocf 0 0K 0 AHCI driver 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5E_TLS_RX 0 0K 0 MLX5EEPROM 0 0K 0 MLX5E_TLS 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EN 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5DUMP 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 seq_file 0 0K 0 lkpiskb 0 0K 0 radix 0 0K 0 idr 0 0K 0 lkpindev 0 0K 0 lkpifw 0 0K 0 lkpi80211 0 0K 0 NLM 0 0K 0 ipsec-spdcache 0 0K 0 ipsec-reg 0 0K 0 ipsec-misc 0 0K 0 ipsecrequest 0 0K 0 ip6opt 0 0K 66 ip6_msource 0 0K 0 ip6_moptions 0 0K 0 in6_mfilter 0 0K 0 frag6 0 0K 0 tcplog 0 0K 0 tcp_hwpace 0 0K 0 USBdev 0 0K 0 USB 0 0K 0 LRO 0 0K 0 ip_msource 0 0K 0 ip_moptions 0 0K 0 in_mfilter 0 0K 0 ipid 0 0K 0 80211scan 0 0K 0 80211ratectl 0 0K 0 80211power 0 0K 0 80211nodeie 0 0K 0 80211node 0 0K 0 80211mesh_gt 0 0K 0 80211mesh_rt 0 0K 0 80211perr 0 0K 0 80211prep 0 0K 0 80211preq 0 0K 0 80211dfs 0 0K 0 80211crypto 0 0K 0 80211vap 0 0K 0 iflib 0 0K 0 vlan 0 0K 0 gif 0 0K 0 ifdescr 0 0K 0 zlib 0 0K 0 fadvise 0 0K 0 VN POLL 0 0K 0 agp 0 0K 0 statfs 0 0K 30199 namei_tracker 0 0K 0 export_host 0 0K 0 cl_savebuf 0 0K 2 nvme_da 0 0K 0 acpipwr 0 0K 0 acpi_perf 0 0K 0 twsbuf 0 0K 0 twe_commands 0 0K 0 tcp_log_dev 0 0K 444 midi buffers 0 0K 0 mixer 0 0K 0 ac97 0 0K 0 hdacc 0 0K 0 hdac 0 0K 0 hdaa 0 0K 0 acpicmbat 0 0K 0 SIIS driver 0 0K 0 CAM CCB 0 0K 523 biobuf 0 0K 0 aios 0 0K 0 lio 0 0K 38 acl 0 0K 0 mbuf_tag 0 0K 166 ktls 0 0K 0 PUC 0 0K 0 ppbusdev 0 0K 0 agtiapi_MemAlloc malloc 0 0K 0 osti_cacheable 0 0K 0 tempbuff 0 0K 0 tempbuff 0 0K 0 ag_tgt_map_t malloc 0 0K 0 ag_slr_map_t malloc 0 0K 0 lDevFlags * malloc 0 0K 0 tiDeviceHandle_t * malloc 0 0K 0 ag_portal_data_t malloc 0 0K 0 ag_device_t malloc 0 0K 0 STLock malloc 0 0K 0 CCB List 0 0K 0 sr_iov 0 0K 0 OCS 0 0K 0 OCS 0 0K 0 nvme 0 0K 0 nvd 0 0K 0 netmap 0 0K 0 mwldev 0 0K 0 MVS driver 0 0K 0 CAM ccb queue 0 0K 0 accf 0 0K 0 pts 0 0K 0 ioctlops 0 0K 220 eventfd 0 0K 63 Witness 0 0K 0 stack 0 0K 0 mrsasbuf 0 0K 0 mpt_user 0 0K 0 mps_user 0 0K 0 MPSSAS 0 0K 0 mps 0 0K 0 sbuf 0 0K 288 mpr_user 0 0K 0 firmware 0 0K 0 compressor 0 0K 0 MPRSAS 0 0K 0 SWAP 0 0K 0 mpr 0 0K 0 mfibuf 0 0K 0 sysctltmp 0 0K 925 sysctl 0 0K 3 md_sectors 0 0K 0 ekcd 0 0K 0 dumper 0 0K 0 sendfile 0 0K 12 rctl 0 0K 0 md_disk 0 0K 0 malodev 0 0K 0 LED 0 0K 0 ix_sriov 0 0K 0 cache 0 0K 0 aacraidcam 0 0K 0 prison_racct 0 0K 0 Fail Points 0 0K 0 sigio 0 0K 10 filedesc_to_leader 0 0K 0 pwd 0 0K 0 tty console 0 0K 0 ix 0 0K 0 ipsbuf 0 0K 0 aacraid_buf 0 0K 0 aaccam 0 0K 0 boottrace 0 0K 0 aacbuf 0 0K 0 zstd 0 0K 0 XZ_DEC 0 0K 0 nvlist 0 0K 0 SCSI ENC 0 0K 0 SCSI sa 0 0K 0 isofs_node 0 0K 0 isofs_mount 0 0K 0 tr_raid5_data 0 0K 0 tr_raid1e_data 0 0K 0 tr_raid1_data 0 0K 0 tr_raid0_data 0 0K 0 tr_concat_data 0 0K 0 md_sii_data 0 0K 0 md_promise_data 0 0K 0 md_nvidia_data 0 0K 0 md_jmicron_data 0 0K 0 md_intel_data 0 0K 0 md_ddf_data 0 0K 0 raid_data 0 0K 72 geom_flashmap 0 0K 0 tmpfs dir 0 0K 0 tmpfs name 0 0K 0 tmpfs mount 0 0K 0 NFS FHA 0 0K 0 newnfsmnt 0 0K 0 newnfsclient_req 0 0K 0 db> show uma Zone Size Used Free Requests Sleeps Bucket Total Mem XFree mbuf_jumbo_page 4096 8335 1063 1780685 0 254 38494208 0 sctp_asoc 2256 189 2101 16390 0 254 5166240 0 mbuf_cluster 2048 2032 0 2032 0 254 4161536 0 malloc-2048 2048 198 1682 45716 0 8 3850240 0 RADIX NODE 144 19635 352 684756 0 64 2878128 0 mbuf 256 8894 2289 2954534 0 254 2862848 0 sctp_ep 1208 197 2093 45715 0 254 2766320 0 pbuf 2624 0 925 0 0 2 2427200 0 tcp_log 416 0 5337 36069 0 254 2220192 0 malloc-1024 1024 191 1697 17691 0 16 1933312 0 BUF TRIE 144 186 11602 83645 0 62 1697472 0 sctp_raddr 736 189 2099 16497 0 254 1683968 0 malloc-384 384 4155 45 4192 0 30 1612800 0 malloc-128 128 11909 88 12267 0 126 1535616 0 malloc-384 384 439 3371 62752 0 30 1463040 0 UMA Slabs 0 112 13023 3 13023 0 126 1458912 0 malloc-4096 4096 330 4 1119 0 2 1368064 0 vmem btag 56 22754 109 22754 0 254 1280328 0 malloc-384 384 16 2264 36755 0 30 875520 0 tcpcb 1104 6 757 12197 0 254 842352 0 socket 960 52 712 61737 0 254 733440 0 256 Bucket 2048 308 28 6369 0 8 688128 0 VM OBJECT 264 2478 102 658449 0 30 681120 0 malloc-256 256 228 2322 113255 0 62 652800 0 FFS inode 1160 533 27 32468 0 8 649600 0 malloc-8192 8192 1 66 140 0 1 548864 0 mbuf_packet 256 67 1965 69360 0 254 520192 0 malloc-16384 16384 21 6 29451 0 1 442368 0 lkpimm 168 1 2327 1 0 62 391104 0 lkpicurr 168 2 2326 2 0 62 391104 0 THREAD 1808 167 42 35237 0 8 377872 0 malloc-256 256 288 1152 85539 0 62 368640 0 MAP ENTRY 96 3421 359 1563060 0 126 362880 0 malloc-64 64 4361 742 420423 0 254 326592 0 malloc-4096 4096 73 5 30445 0 2 319488 0 sctp_chunk 152 189 1865 2770 0 254 312208 0 malloc-128 128 44 2374 40922 0 126 309504 0 VNODE 448 573 93 32510 0 30 298368 0 malloc-65536 65536 4 0 4 0 1 262144 0 malloc-16 16 14844 156 42319 0 254 240000 0 malloc-32768 32768 0 7 40747 0 1 229376 0 DEVCTL 1024 0 220 155 0 0 225280 0 malloc-65536 65536 1 2 417 0 1 196608 0 malloc-65536 65536 1 2 9 0 1 196608 0 malloc-128 128 1202 317 27110 0 126 194432 0 malloc-32 32 5675 247 15069 0 254 189504 0 UMA Zones 768 244 0 244 0 16 187392 0 FPU_save_area 832 169 47 47304 0 16 179712 0 sctp_stream_msg_out 112 0 1548 2504 0 254 173376 0 vnpbuf 2624 0 64 6 0 64 167936 0 ertt_txseginfo 40 0 4141 120131 0 254 165640 0 malloc-4096 4096 39 1 54 0 2 163840 0 tcp_bbr_map 128 4 1267 31159 0 126 162688 0 FFS2 dinode 256 533 97 32467 0 62 161280 0 S VFS Cache 104 1066 455 34271 0 126 158184 0 VMSPACE 2552 52 8 30419 0 4 153120 0 128 Bucket 1024 104 43 6448 0 16 150528 0 malloc-64 64 320 2011 17287 0 254 149184 0 malloc-256 256 98 472 157996 0 62 145920 0 malloc-32 32 413 3745 92234 0 254 133056 0 malloc-65536 65536 0 2 167 0 1 131072 0 malloc-65536 65536 2 0 2 0 1 131072 0 malloc-1024 1024 119 9 138 0 16 131072 0 unpcb 256 20 490 1344 0 254 130560 0 PROC 1352 72 18 30444 0 8 121680 0 ksiginfo 112 79 965 12321 0 126 116928 0 malloc-2048 2048 9 47 677 0 8 114688 0 32 Bucket 256 269 166 26582 0 62 111360 0 filedesc0 1072 72 26 30445 0 8 105056 0 64 Bucket 512 135 65 43122 0 30 102400 0 malloc-128 128 470 305 59481 0 126 99200 0 malloc-128 128 620 155 1657 0 126 99200 0 malloc-32768 32768 1 2 121 0 1 98304 0 pipe 744 23 107 1032 0 16 96720 0 malloc-256 256 221 154 35824 0 62 96000 0 UMA Kegs 384 230 3 230 0 30 89472 0 malloc-64 64 585 738 13865 0 254 84672 0 malloc-8192 8192 8 2 129 0 1 81920 0 sctp_readq 152 0 520 116 0 254 79040 0 g_bio 408 0 180 578961 0 30 73440 0 malloc-384 384 110 70 255 0 30 69120 0 malloc-64 64 322 749 22838 0 254 68544 0 malloc-16 16 446 3804 63975 0 254 68000 0 malloc-16 16 432 3818 75161 0 254 68000 0 malloc-65536 65536 1 0 1 0 1 65536 0 malloc-16384 16384 4 0 4 0 1 65536 0 malloc-1024 1024 18 46 533 0 16 65536 0 malloc-256 256 208 47 1035 0 62 65280 0 malloc-256 256 118 137 37831 0 62 65280 0 tcp_inpcb 424 6 147 12197 0 30 64872 0 16 Bucket 144 188 204 7237 0 62 56448 0 udplite_inpcb 424 0 126 1538 0 30 53424 0 ripcb 424 5 121 650 0 30 53424 0 malloc-64 64 162 657 61646 0 254 52416 0 malloc-64 64 167 652 30602 0 254 52416 0 Files 80 234 416 109243 0 126 52000 0 malloc-128 128 46 357 29643 0 126 51584 0 malloc-128 128 188 215 32519 0 126 51584 0 malloc-256 256 56 139 171 0 62 49920 0 DIRHASH 1024 35 13 35 0 16 49152 0 NAMEI 1024 0 48 144979 0 16 49152 0 malloc-8192 8192 4 2 32 0 1 49152 0 malloc-8192 8192 2 4 20 0 1 49152 0 malloc-2048 2048 3 21 180 0 8 49152 0 malloc-2048 2048 0 24 579 0 8 49152 0 tcp_rack_pcb 896 0 54 838 0 16 48384 0 tcp_bbr_pcb 832 2 52 3659 0 16 44928 0 syncache 168 0 264 4 0 254 44352 0 TURNSTILE 136 210 105 210 0 62 42840 0 clpbuf 2624 0 16 42 0 16 41984 0 pcpu-8 8 4709 411 4935 0 254 40960 0 sctp_laddr 48 12 828 46475 0 254 40320 0 udp_inpcb 424 8 82 243 0 30 38160 0 da_ccb 544 0 70 144845 0 16 38080 0 malloc-64 64 30 537 442 0 254 36288 0 8 Bucket 80 265 185 6924 0 126 36000 0 malloc-128 128 102 177 214 0 126 35712 0 routing nhops 256 27 108 35 0 62 34560 0 ttyoutq 256 72 63 160 0 62 34560 0 malloc-256 256 18 117 719 0 62 34560 0 SLEEPQUEUE 88 210 174 210 0 126 33792 0 malloc-32768 32768 1 0 1 0 1 32768 0 malloc-32768 32768 1 0 1 0 1 32768 0 malloc-16384 16384 0 2 160 0 1 32768 0 malloc-8192 8192 4 0 4 0 1 32768 0 malloc-8192 8192 4 0 4 0 1 32768 0 malloc-4096 4096 4 4 11 0 2 32768 0 malloc-4096 4096 1 7 30200 0 2 32768 0 malloc-2048 2048 1 15 13 0 8 32768 0 malloc-2048 2048 10 6 13 0 8 32768 0 malloc-2048 2048 9 7 9 0 8 32768 0 malloc-1024 1024 2 30 42 0 16 32768 0 malloc-1024 1024 7 25 219 0 16 32768 0 malloc-1024 1024 17 15 17 0 16 32768 0 malloc-512 512 2 62 206 0 30 32768 0 malloc-512 512 11 53 21 0 30 32768 0 malloc-512 512 0 64 3 0 30 32768 0 malloc-512 512 0 64 719 0 30 32768 0 malloc-512 512 11 53 60 0 30 32768 0 pcpu-64 64 493 19 493 0 254 32768 0 KNOTE 160 28 172 244986 0 62 32000 0 ttyinq 160 135 65 300 0 62 32000 0 tcp_rack_map 120 0 264 1535 0 126 31680 0 cpuset 104 7 272 19642 0 126 29016 0 tcp_inpcb ports 32 4 878 6346 0 254 28224 0 ertt 72 6 386 12197 0 126 28224 0 PWD 32 32 850 29514 0 254 28224 0 malloc-32 32 132 750 490 0 254 28224 0 malloc-32 32 95 787 18482 0 254 28224 0 4 Bucket 48 5 583 686 0 254 28224 0 2 Bucket 32 99 783 13697 0 254 28224 0 malloc-4096 4096 4 2 8 0 2 24576 0 rtentry 176 31 107 35 0 62 24288 0 PGRP 88 35 241 125 0 126 24288 0 rl_entry 40 96 510 96 0 254 24240 0 itimer 352 0 66 10 0 30 23232 0 malloc-384 384 20 40 645 0 30 23040 0 malloc-384 384 24 36 36 0 30 23040 0 domainset 40 0 567 15521 0 254 22680 0 hostcache 64 1 314 1 0 254 20160 0 udpcb 32 8 622 1781 0 254 20160 0 udplite_inpcb ports 32 0 630 108 0 254 20160 0 udp_inpcb ports 32 3 627 39 0 254 20160 0 AIOP 32 4 626 4 0 254 20160 0 malloc-32 32 37 593 482 0 254 20160 0 malloc-32 32 95 535 17011 0 254 20160 0 malloc-32 32 12 618 157 0 254 20160 0 malloc-32 32 33 597 8327 0 254 20160 0 cryptop 280 0 70 5 0 30 19600 0 AIOCB 552 0 35 101 0 16 19320 0 Mountpoints 2752 2 5 2 0 4 19264 0 AIOLIO 272 0 70 38 0 30 19040 0 epoch_record pcpu 256 4 60 4 0 62 16384 0 malloc-16384 16384 1 0 1 0 1 16384 0 malloc-16384 16384 1 0 1 0 1 16384 0 malloc-8192 8192 2 0 2 0 1 16384 0 malloc-4096 4096 1 3 8 0 2 16384 0 malloc-2048 2048 2 6 195 0 8 16384 0 malloc-1024 1024 6 10 6 0 16 16384 0 malloc-1024 1024 1 15 1 0 16 16384 0 malloc-512 512 1 31 1 0 30 16384 0 malloc-512 512 1 31 2 0 30 16384 0 SMR CPU 32 7 504 7 0 254 16352 0 vtnet_tx_hdr 24 0 668 917708 0 254 16032 0 AIO 208 0 76 24 0 62 15808 0 kenv 258 15 45 1053 0 30 15480 0 mqnode 416 3 33 3 0 30 14976 0 vmem 1856 1 7 1 0 8 14848 0 SMR SHARED 24 7 504 7 0 254 12264 0 malloc-16 16 36 714 55559 0 254 12000 0 malloc-16 16 15 735 57 0 254 12000 0 malloc-16 16 4 746 5 0 254 12000 0 malloc-16 16 266 484 289 0 254 12000 0 malloc-16 16 26 724 26234 0 254 12000 0 malloc-384 384 2 28 2 0 30 11520 0 pcpu-16 16 7 249 7 0 254 4096 0 UMA Slabs 1 176 10 12 10 0 62 3872 0 KMAP ENTRY 96 12 27 14 0 0 3744 0 pcpu-4 4 1 511 1 0 254 2048 0 FFS1 dinode 128 0 0 0 0 126 0 0 ada_ccb 272 0 0 0 0 30 0 0 swblk 136 0 0 0 0 62 0 0 swpctrie 144 0 0 0 0 62 0 0 cdg_qdiffsample 16 0 0 0 0 254 0 0 pf state scrubs 40 0 0 0 0 254 0 0 pf frag entries 40 0 0 0 0 254 0 0 pf frags 248 0 0 0 0 62 0 0 pf table entries 160 0 0 0 0 254 0 0 pf table entry counters 64 0 0 0 0 254 0 0 pf source nodes 136 0 0 0 0 254 0 0 pf state keys 88 0 0 0 0 126 0 0 pf states 312 0 0 0 0 254 0 0 pf tags 104 0 0 0 0 126 0 0 pf mtags 184 0 0 0 0 62 0 0 tfo_ccache_entries 80 0 0 0 0 126 0 0 tfo 4 0 0 0 0 254 0 0 sackhole 32 0 0 0 0 254 0 0 tcptw 72 0 0 0 0 254 0 0 ipq 56 0 0 0 0 254 0 0 sctp_asconf_ack 48 0 0 0 0 254 0 0 sctp_asconf 40 0 0 0 0 254 0 0 tcp_log_id_node 120 0 0 0 0 126 0 0 tcp_log_id_bucket 176 0 0 0 0 62 0 0 tcpreass 48 0 0 0 0 254 0 0 ripcb ports 32 0 0 0 0 254 0 0 IPsec SA lft_c 16 0 0 0 0 254 0 0 NCLNODE 608 0 0 0 0 16 0 0 mqnotifier 216 0 0 0 0 62 0 0 mvdata 64 0 0 0 0 254 0 0 mqueue 248 0 0 0 0 62 0 0 TMPFS node 224 0 0 0 0 62 0 0 LTS VFS Cache 360 0 0 0 0 30 0 0 L VFS Cache 320 0 0 0 0 30 0 0 STS VFS Cache 144 0 0 0 0 62 0 0 linux_dma_object 32 0 0 0 0 254 0 0 linux_dma_pctrie 144 0 0 0 0 62 0 0 IOMMU_MAP_ENTRY 120 0 0 0 0 126 0 0 mbuf_jumbo_16k 16384 0 0 0 0 254 0 0 mbuf_jumbo_9k 9216 0 0 0 0 254 0 0 audit_record 1280 0 0 0 0 8 0 0 MAC labels 40 0 0 0 0 254 0 0 mdpbuf 2624 0 0 0 0 3 0 0 nfspbuf 2624 0 0 0 0 16 0 0 swwbuf 2624 0 0 0 0 8 0 0 swrbuf 2624 0 0 0 0 16 0 0 umtx_shm 88 0 0 0 0 126 0 0 umtx pi 96 0 0 0 0 126 0 0 rangeset pctrie nodes 144 0 0 0 0 62 0 0 malloc-65536 65536 0 0 0 0 1 0 0 malloc-65536 65536 0 0 0 0 1 0 0 malloc-32768 32768 0 0 0 0 1 0 0 malloc-32768 32768 0 0 0 0 1 0 0 malloc-32768 32768 0 0 0 0 1 0 0 malloc-32768 32768 0 0 0 0 1 0 0 malloc-16384 16384 0 0 0 0 1 0 0 malloc-16384 16384 0 0 0 0 1 0 0 malloc-16384 16384 0 0 0 0 1 0 0 malloc-8192 8192 0 0 0 0 1 0 0 malloc-4096 4096 0 0 0 0 2 0 0 malloc-512 512 0 0 0 0 30 0 0 malloc-384 384 0 0 0 0 30 0 0 malloc-64 64 0 0 0 0 254 0 0 pcpu-32 32 0 0 0 0 254 0 0 fakepg 104 0 0 0 0 126 0 0 UMA Hash 256 0 0 0 0 62 0 0