panic: Thread not suspended cpuid = 1 time = 1652615896 KDB: stack backtrace: db_trace_self_wrapper() at db_trace_self_wrapper+0xc7/frame 0xfffffe005417bad0 kdb_backtrace() at kdb_backtrace+0xd3/frame 0xfffffe005417bc30 vpanic() at vpanic+0x2b8/frame 0xfffffe005417bd10 panic() at panic+0xb5/frame 0xfffffe005417bde0 thread_unsuspend_one() at thread_unsuspend_one+0x197/frame 0xfffffe005417be20 thread_suspend_check() at thread_suspend_check+0x38d/frame 0xfffffe005417beb0 ast() at ast+0xd73/frame 0xfffffe005417bf30 doreti_ast() at doreti_ast+0x1f/frame 0x820fae550 KDB: enter: panic [ thread pid 780 tid 100123 ] Stopped at kdb_enter+0x6b: movq $0,0x275a85a(%rip) db> db> set $lines = 0 db> set $maxwidth = 0 db> show registers cs 0x20 ds 0x3b es 0x3b fs 0x13 gs 0x1b ss 0x28 rax 0x12 rcx 0xffffffff81729cb6 printf+0xf6 rdx 0x1 rbx 0 rsp 0xfffffe005417bc10 rbp 0xfffffe005417bc30 rsi 0 rdi 0xffffffff81729d18 printf+0x158 r8 0 r9 0xffffffff r10 0 r11 0xfffffe0058b17c30 r12 0xfffffe009efd8000 r13 0xfffffe005417bc01 r14 0xffffffff82b4fd00 .str.26 r15 0xffffffff82b4fd00 .str.26 rip 0xffffffff8171983b kdb_enter+0x6b rflags 0x46 kdb_enter+0x6b: movq $0,0x275a85a(%rip) db> show proc Process 780 (syz-executor.2) at 0xfffffe0058dfaa90: state: NORMAL uid: 0 gids: 0, 0, 5 parent: pid 774 at 0xfffffe009eba0000 ABI: FreeBSD ELF64 flag: 0x12084000 flag2: 0 arguments: /root/syz-executor.2 exec reaper: 0xfffffe0053de1000 reapsubtree: 1 sigparent: 20 vmspace: 0xfffffe0053dee000 (map 0xfffffe0053dee000) (map.pmap 0xfffffe0053dee0c0) (pmap 0xfffffe0053dee128) threads: 1 100123 Run CPU 1 syz-executor.2 db> ps pid ppid pgrp uid state wmesg wchan cmd 6384 778 778 0 T (threaded) syz-executor.1 105003 s syz-executor.1 105731 RunQ syz-executor.1 5691 1 5691 0 Rs dhclient 5685 1 5685 0 Ds range 0xfffffe0058bb7438 dhclient 5655 430 430 0 RL sh 5380 1 777 0 RE syz-executor.0 5162 1 788 0 LE *pmap pv 0xfffffe0057887540 syz-executor.3 5123 1 788 0 LE *pmap pv 0xfffffe0057887540 syz-executor.3 1081 1 1081 0 Ds range 0xfffffe0058bcce38 dhclient 1077 1 1077 0 Ds range 0xfffffe0058bb7578 dhclient 780 774 780 0 TLs CPU 1 syz-executor.2 778 774 778 0 Ss nanslp 0xffffffff83e41701 syz-executor.1 777 774 777 0 Rs syz-executor.0 774 772 772 0 R (threaded) syz-fuzzer 100111 S uwait 0xfffffe0057328680 syz-fuzzer 100113 S nanslp 0xffffffff83e41701 syz-fuzzer 100114 S uwait 0xfffffe0057328580 syz-fuzzer 100115 Run CPU 0 syz-fuzzer 100116 S uwait 0xfffffe0058b25d00 syz-fuzzer 100117 S uwait 0xfffffe0058b24100 syz-fuzzer 100118 S uwait 0xfffffe0058b24200 syz-fuzzer 100119 RunQ syz-fuzzer 100122 S kqread 0xfffffe0058b5de00 syz-fuzzer 772 770 772 0 Ss pause 0xfffffe009eb9fb40 csh 770 688 770 0 Ss select 0xfffffe0056f6aec0 sshd 754 1 754 0 Ss+ ttyin 0xfffffe00574764b0 getty 753 1 753 0 Ss+ ttyin 0xfffffe00574724b0 getty 752 1 752 0 Ss+ ttyin 0xfffffe00574728b0 getty 751 1 751 0 Ss+ ttyin 0xfffffe0057472cb0 getty 750 1 750 0 Ss+ ttyin 0xfffffe00579c64b0 getty 749 1 749 0 Ss+ ttyin 0xfffffe00579c68b0 getty 748 1 748 0 Ss+ ttyin 0xfffffe00579c6cb0 getty 747 1 747 0 Ss+ ttyin 0xfffffe00579c70b0 getty 746 1 746 0 Ss+ ttyin 0xfffffe00579c74b0 getty 692 1 692 0 Ss nanslp 0xffffffff83e41701 cron 688 1 688 0 Ss select 0xfffffe0056f6b140 sshd 501 1 501 0 Ss select 0xfffffe0056f6ae40 syslogd 430 1 430 0 Ss wait 0xfffffe0058df5a90 devd 429 1 429 65 Ss select 0xfffffe0056f6bf40 dhclient 344 1 344 0 Ss select 0xfffffe0056f6af40 dhclient 341 1 341 0 Ss select 0xfffffe0056f6b7c0 dhclient 17 0 0 0 DL vlruwt 0xfffffe0056fa0548 [vnlru] 16 0 0 0 DL syncer 0xffffffff83f66f60 [syncer] 15 0 0 0 DL (threaded) [bufdaemon] 100079 D psleep 0xffffffff83f65560 [bufdaemon] 100082 D - 0xffffffff83211f80 [bufspacedaemon-0] 100095 D sdflush 0xfffffe0058b8cce8 [/ worker] 9 0 0 0 DL psleep 0xffffffff83f99000 [vmdaemon] 8 0 0 0 DL (threaded) [pagedaemon] 100077 D psleep 0xffffffff83f8ceb8 [dom0] 100083 D launds 0xffffffff83f8cec4 [laundry: dom0] 100084 D umarcl 0xffffffff81e46dc0 [uma] 7 0 0 0 DL - 0xffffffff83bfdd28 [rand_harvestq] 6 0 0 0 DL pftm 0xffffffff84b55550 [pf purge] 5 0 0 0 DL waiting 0xffffffff849904a0 [sctp_iterator] 4 0 0 0 DL (threaded) [cam] 100044 D - 0xffffffff83aa0cc0 [doneq0] 100045 D - 0xffffffff83aa0c40 [async] 100076 D - 0xffffffff83aa0ac0 [scanner] 14 0 0 0 DL seqstat 0xfffffe0053fd7488 [sequencer 00] 3 0 0 0 DL (threaded) [crypto] 100040 D crypto_ 0xffffffff83f886e0 [crypto] 100041 D crypto_ 0xfffffe0053ecd830 [crypto returns 0] 100042 D crypto_ 0xfffffe0053ecd880 [crypto returns 1] 13 0 0 0 DL (threaded) [geom] 100035 D - 0xffffffff83e16d00 [g_event] 100036 D - 0xffffffff83e16d20 [g_up] 100037 D - 0xffffffff83e16d40 [g_down] 2 0 0 0 WL (threaded) [clock] 100029 I [clock (0)] 100030 I [clock (1)] 12 0 0 0 WL (threaded) [intr] 100015 I [swi5: fast taskq] 100018 I [swi6: task queue] 100019 I [swi6: Giant taskq] 100031 I [swi1: netisr 0] 100032 I [swi1: hpts] 100033 I [swi1: hpts] 100046 I [irq24: virtio_pci0] 100047 I [irq25: virtio_pci0] 100048 I [irq26: virtio_pci0] 100049 I [irq27: virtio_pci0] 100050 I [irq28: virtio_pci1] 100051 I [irq29: virtio_pci1] 100052 I [irq30: virtio_pci1] 100053 I [irq31: virtio_pci1] 100054 I [irq32: virtio_pci1] 100059 I [irq33: virtio_pci2] 100060 I [irq34: virtio_pci2] 100061 I [irq35: virtio_pci2] 100063 I [irq1: atkbd0] 100064 I [irq12: psm0] 100065 I [swi0: uart uart++] 100069 I [swi1: pf send] 11 0 0 0 RL (threaded) [idle] 100003 CanRun [idle: cpu0] 100004 CanRun [idle: cpu1] 1 0 1 0 RLs [init] 10 0 0 0 DL audit_w 0xffffffff83f891c0 [audit] 0 0 0 0 DLs (threaded) [kernel] 100000 D swapin 0xffffffff83e17700 [swapper] 100005 D - 0xfffffe0053e84100 [if_config_tqg_0] 100006 D - 0xfffffe0053e84000 [softirq_0] 100007 D - 0xfffffe0053e83e00 [softirq_1] 100008 D - 0xfffffe0053e83d00 [if_io_tqg_0] 100009 D - 0xfffffe0053e83c00 [if_io_tqg_1] 100010 D - 0xfffffe000795a100 [inm_free taskq] 100011 D - 0xfffffe000795a000 [linuxkpi_irq_wq] 100012 D - 0xfffffe0007959e00 [in6m_free taskq] 100013 D - 0xfffffe0007959d00 [deferred_unmount ta] 100014 D - 0xfffffe0007959c00 [thread taskq] 100016 D - 0xfffffe0007959a00 [kqueue_ctx taskq] 100017 D - 0xfffffe0007959900 [pci_hp taskq] 100020 D - 0xfffffe0007959600 [aiod_kick taskq] 100021 D - 0xfffffe0007959500 [linuxkpi_short_wq_0] 100022 D - 0xfffffe0007959500 [linuxkpi_short_wq_1] 100023 D - 0xfffffe0007959500 [linuxkpi_short_wq_2] 100024 D - 0xfffffe0007959500 [linuxkpi_short_wq_3] 100025 D - 0xfffffe0007959400 [linuxkpi_long_wq_0] 100026 D - 0xfffffe0007959400 [linuxkpi_long_wq_1] 100027 D - 0xfffffe0007959400 [linuxkpi_long_wq_2] 100028 D - 0xfffffe0007959400 [linuxkpi_long_wq_3] 100034 D - 0xfffffe0007959300 [firmware taskq] 100038 D - 0xfffffe0007959200 [crypto_0] 100039 D - 0xfffffe0007959200 [crypto_1] 100055 D - 0xfffffe0007959000 [vtnet0 rxq 0] 100056 D - 0xfffffe0007958e00 [vtnet0 txq 0] 100057 D - 0xfffffe0007958d00 [vtnet0 rxq 1] 100058 D - 0xfffffe0007958c00 [vtnet0 txq 1] 100062 D vtbslp 0xfffffe0056f6c000 [virtio_balloon] 100066 D - 0xffffffff82b55b80 [deadlkres] 100070 D - 0xfffffe000795a200 [mca taskq] 100072 D - 0xfffffe00585ef600 [acpi_task_0] 100073 D - 0xfffffe00585ef600 [acpi_task_1] 100074 D - 0xfffffe00585ef600 [acpi_task_2] 100075 D - 0xfffffe0007959100 [CAM taskq] 6387 6384 778 0 Z syz-executor.1 5676 5655 430 65 Z dhclient 5118 1 788 0 Z syz-executor.3 db> show all locks Process 6384 (syz-executor.1) thread 0xfffffe00a68ff740 (105731) exclusive sx sapblk (sapblk) r = 0 (0xffffffff83e2faa0) locked @ /syzkaller/managers/main/kernel/sys/kern/kern_proc.c:3405 Process 5380 (syz-executor.0) thread 0xfffffe00a0b21000 (104890) exclusive rw pmap pv list (pmap pv list) r = 0 (0xfffffe0007786240) locked @ /syzkaller/managers/main/kernel/sys/amd64/amd64/pmap.c:8393 exclusive sleep mutex pmap (pmap) r = 0 (0xfffffe00a6883128) locked @ /syzkaller/managers/main/kernel/sys/amd64/amd64/pmap.c:8305 Process 5162 (syz-executor.3) thread 0xfffffe00a0b25020 (101105) exclusive sleep mutex pmap (pmap) r = 0 (0xfffffe0058d62128) locked @ /syzkaller/managers/main/kernel/sys/amd64/amd64/pmap.c:8305 Process 5123 (syz-executor.3) thread 0xfffffe009efdc020 (100919) exclusive sleep mutex pmap (pmap) r = 0 (0xfffffe0058cb3518) locked @ /syzkaller/managers/main/kernel/sys/amd64/amd64/pmap.c:8305 Process 780 (syz-executor.2) thread 0xfffffe009efd8000 (100123) exclusive sleep mutex process lock (process lock) r = 0 (0xfffffe0058dfabb8) locked @ /syzkaller/managers/main/kernel/sys/kern/subr_trap.c:368 db> show malloc Type InUse MemUse Requests pf_hash 5 11524K 5 tcp_hpts 7 4801K 7 devbuf 4217 4323K 4245 pcb 1905 2658K 13757 sysctloid 35322 2081K 35393 vtbuf 24 1968K 46 kobj 328 1312K 489 inodedep 1741 1165K 5584 newblk 22 1030K 9842 vfscache 3 1025K 3 sctp_stro 942 942K 3679 sctp_atcl 1885 707K 10528 ufs_quota 1 512K 1 vfs_hash 1 512K 1 callout 2 512K 2 intr 4 472K 4 dirrem 1832 458K 5467 vmem 3 274K 5 subproc 149 245K 6486 freefile 1830 229K 5463 acpica 1674 184K 57552 vnet_data 1 168K 1 tidhash 3 141K 3 linker 358 134K 386 pagedep 13 131K 5191 tfo_ccache 1 128K 1 sctp_atky 2827 118K 14207 DEVFS1 109 109K 126 sem 4 106K 4 bus 994 81K 5207 filedesc 10 73K 10967 mtx_pool 2 72K 2 syncache 1 68K 1 module 513 65K 513 acpitask 1 64K 1 ddb_capture 1 64K 1 temp 75 55K 2393 umtx 396 50K 396 sctp_timw 189 48K 189 kdtrace 226 47K 12123 shm 5 40K 715 BPF 22 36K 30 sctp_aadr 516 33K 516 DEVFS3 128 32K 138 hostcache 1 32K 1 msg 4 30K 4 sctp_athm 1885 30K 10528 sctp_map 1884 30K 7358 gtaskqueue 18 26K 18 kbdmux 6 22K 6 DEVFS_RULE 56 20K 56 ifaddr 67 19K 71 ufs_mount 4 17K 5 proc 3 17K 3 routetbl 157 17K 495 sbuf 2 17K 304 tty 16 16K 16 ithread 97 16K 97 bus-sc 34 15K 1681 KTRACE 100 13K 100 lltable 41 13K 50 ifnet 7 13K 7 ether_multi 144 12K 167 kenv 95 12K 95 eventhandler 134 12K 134 rman 88 11K 431 GEOM 61 11K 490 CAM queue 5 11K 1528 in6_multi 64 9K 71 bmsafemap 2 9K 5604 UART 12 9K 12 devstat 4 9K 4 ksem 1 8K 1 rpc 2 8K 2 shmfd 1 8K 16 pfs_vncache 1 8K 1 pfs_nodes 20 8K 20 audit_evclass 237 8K 296 taskqueue 63 7K 63 sglist 5 7K 5 CAM DEV 3 6K 510 cred 24 6K 210 plimit 22 6K 364 kqueue 54 6K 6395 ufs_dirhash 24 5K 24 Unitno 123 5K 2901 UMA 272 5K 272 pf_ifnet 10 5K 19 vt 11 5K 11 memdesc 1 4K 1 MCA 32 4K 32 evdev 4 4K 4 acpisem 28 4K 28 session 27 4K 41 hhook 15 4K 17 lockf 29 4K 42 proc-args 78 4K 7591 DEVFSP 48 3K 74 pwddesc 46 3K 6482 terminal 11 3K 11 kcovinfo 39 3K 52 uidinfo 3 3K 7 local_apic 1 2K 1 io_apic 1 2K 1 fpukern_ctx 2 2K 2 freework 8 2K 6894 ipsec-saq 2 2K 2 freeblks 7 2K 5429 sctp_ifa 13 2K 15 ip6ndp 11 2K 15 selfd 26 2K 102438 CAM XPT 22 2K 543 msi 12 2K 12 mkdir 12 2K 10352 diradd 12 2K 5501 select 11 2K 46 ipsecpolicy 2 2K 2 in_multi 5 2K 8 acpidev 20 2K 20 newdirblk 9 2K 5176 clone 9 2K 9 tun 7 2K 7 softdep 1 1K 1 sahead 1 1K 1 secasvar 1 1K 1 nhops 6 1K 8 vnodemarker 2 1K 22 NFSD session 1 1K 1 CAM periph 4 1K 271 ipsec 3 1K 3 mld 6 1K 6 igmp 6 1K 6 toponodes 6 1K 6 isadev 6 1K 6 mount 16 1K 89 pci_link 10 1K 10 sctp_ifn 5 1K 15 crypto 4 1K 4 encap_export_host 12 1K 12 pfil 4 1K 4 cdev 2 1K 2 inpcbpolicy 14 1K 1779 osd 9 1K 1437 chacha20random 1 1K 1 procdesc 3 1K 14 CC Mem 5 1K 1426 NFSD lckfile 1 1K 1 NFSD V4client 1 1K 1 DEVFS 9 1K 10 freefrag 2 1K 4 vnodes 1 1K 1 CAM SIM 2 1K 2 feeder 7 1K 7 tcpfunc 3 1K 3 loginclass 3 1K 6 prison 6 1K 6 lkpikmalloc 5 1K 6 aesni_data 2 1K 2 cryptodev 2 1K 48 tcp_fsb 1 1K 936 nexusdev 8 1K 8 apmdev 1 1K 1 atkbddev 2 1K 2 CAM dev queue 2 1K 2 CAM I/O Scheduler 1 1K 1 CAM path 4 1K 1034 pmchooks 1 1K 1 soname 4 1K 12342 sctp_vrf 1 1K 1 vnet 1 1K 1 entropy 2 1K 40 pmc 1 1K 1 acpiintr 1 1K 1 cpus 2 1K 2 vnet_data_free 1 1K 1 Per-cpu 1 1K 1 p1003.1b 1 1K 1 filecaps 1 1K 97 pf_table 0 0K 0 pf_rule 0 0K 0 pf_altq 0 0K 0 pf_osfp 0 0K 0 pf_temp 0 0K 0 sctp_mcore 0 0K 0 sctp_socko 0 0K 2473 sctp_iter 0 0K 13 sctp_mvrf 0 0K 0 sctp_cpal 0 0K 0 sctp_cmsg 0 0K 0 sctp_stre 0 0K 0 sctp_athi 0 0K 0 sctp_a_it 0 0K 13 sctp_stri 0 0K 0 tcp_do 0 0K 0 mqdata 0 0K 0 filemon 0 0K 0 ipcomp 0 0K 0 esp 0 0K 0 ah 0 0K 0 NFSCL layrecall 0 0K 0 NFSCL session 0 0K 0 madt_table 0 0K 2 smartpqi 0 0K 0 NFSCL sockreq 0 0K 0 NFSCL devinfo 0 0K 0 NFSCL flayout 0 0K 0 ixl 0 0K 0 NFSCL layout 0 0K 0 NFSD rollback 0 0K 0 NFSCL diroff 0 0K 0 NEWdirectio 0 0K 0 NEWNFSnode 0 0K 0 NFSCL lck 0 0K 0 NFSCL lckown 0 0K 0 ice-resmgr 0 0K 0 ice-osdep 0 0K 0 ice 0 0K 0 iavf 0 0K 0 axgbe 0 0K 0 NFSCL client 0 0K 0 NFSCL deleg 0 0K 0 NFSCL open 0 0K 0 NFSCL owner 0 0K 0 NFS fh 0 0K 0 NFS req 0 0K 0 NFSD usrgroup 0 0K 0 NFSD string 0 0K 0 NFSD V4lock 0 0K 0 xen_intr 0 0K 0 NFSD V4state 0 0K 0 xen_hvm 0 0K 0 legacydrv 0 0K 0 bounce 0 0K 0 busdma 0 0K 0 qpidrv 0 0K 0 NFSD srvcache 0 0K 0 msdosfs_fat 0 0K 0 dmar_idpgtbl 0 0K 0 dmar_dom 0 0K 0 dmar_ctx 0 0K 0 msdosfs_mount 0 0K 0 msdosfs_node 0 0K 0 isci 0 0K 0 iommu_dmamap 0 0K 0 hyperv_socket 0 0K 0 bxe_ilt 0 0K 0 xenbus 0 0K 0 DEVFS4 0 0K 0 vm_fictitious 0 0K 0 DEVFS2 0 0K 0 gntdev 0 0K 0 privcmd_dev 0 0K 0 evtchn_dev 0 0K 0 xenstore 0 0K 0 scsi_pass 0 0K 0 ciss_data 0 0K 0 xnb 0 0K 0 xen_acpi 0 0K 0 xbbd 0 0K 0 xbd 0 0K 0 UMAHash 0 0K 0 Balloon 0 0K 0 sysmouse 0 0K 0 vm_pgdata 0 0K 0 jblocks 0 0K 0 savedino 0 0K 3157 sentinel 0 0K 0 jfsync 0 0K 0 jtrunc 0 0K 0 sbdep 0 0K 8 jsegdep 0 0K 0 jseg 0 0K 0 jfreefrag 0 0K 0 jfreeblk 0 0K 0 jnewblk 0 0K 0 jmvref 0 0K 0 jremref 0 0K 0 jaddref 0 0K 0 freedep 0 0K 0 allocindir 0 0K 0 indirdep 0 0K 2572 allocdirect 0 0K 0 ufs_trim 0 0K 0 mactemp 0 0K 0 audit_trigger 0 0K 0 audit_pipe_presel 0 0K 0 audit_pipeent 0 0K 0 audit_pipe 0 0K 0 audit_evname 0 0K 0 audit_bsm 0 0K 0 audit_gidset 0 0K 0 audit_text 0 0K 0 audit_path 0 0K 0 audit_data 0 0K 0 audit_cred 0 0K 0 vtfont 0 0K 0 BACKLIGHT 0 0K 0 ath_hal 0 0K 0 athdev 0 0K 0 ata_pci 0 0K 0 ata_dma 0 0K 0 ata_generic 0 0K 0 pvscsi 0 0K 0 scsi_da 0 0K 69 ata_da 0 0K 0 scsi_ch 0 0K 0 scsi_cd 0 0K 0 ktls_ocf 0 0K 0 AHCI driver 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5E_TLS_RX 0 0K 0 MLX5EEPROM 0 0K 0 MLX5E_TLS 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EN 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5DUMP 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 seq_file 0 0K 0 lkpiskb 0 0K 0 radix 0 0K 0 idr 0 0K 0 lkpindev 0 0K 0 lkpifw 0 0K 0 lkpi80211 0 0K 0 NLM 0 0K 0 ipsec-spdcache 0 0K 0 ipsec-reg 0 0K 0 ipsec-misc 0 0K 0 ipsecrequest 0 0K 0 ip6opt 0 0K 12 ip6_msource 0 0K 0 ip6_moptions 0 0K 0 in6_mfilter 0 0K 0 frag6 0 0K 0 tcplog 0 0K 0 tcp_hwpace 0 0K 0 USBdev 0 0K 0 USB 0 0K 0 LRO 0 0K 0 ip_msource 0 0K 0 ip_moptions 0 0K 0 in_mfilter 0 0K 0 ipid 0 0K 0 80211scan 0 0K 0 80211ratectl 0 0K 0 80211power 0 0K 0 80211nodeie 0 0K 0 80211node 0 0K 0 80211mesh_gt 0 0K 0 80211mesh_rt 0 0K 0 80211perr 0 0K 0 80211prep 0 0K 0 80211preq 0 0K 0 80211dfs 0 0K 0 80211crypto 0 0K 0 80211vap 0 0K 0 iflib 0 0K 0 vlan 0 0K 0 gif 0 0K 0 ifdescr 0 0K 0 zlib 0 0K 0 fadvise 0 0K 0 VN POLL 0 0K 0 agp 0 0K 0 statfs 0 0K 5434 namei_tracker 0 0K 0 export_host 0 0K 0 cl_savebuf 0 0K 2 nvme_da 0 0K 0 acpipwr 0 0K 0 acpi_perf 0 0K 0 twsbuf 0 0K 0 twe_commands 0 0K 0 tcp_log_dev 0 0K 9 midi buffers 0 0K 0 mixer 0 0K 0 ac97 0 0K 0 hdacc 0 0K 0 hdac 0 0K 0 hdaa 0 0K 0 acpicmbat 0 0K 0 SIIS driver 0 0K 0 CAM CCB 0 0K 523 biobuf 0 0K 0 aios 0 0K 0 lio 0 0K 3 acl 0 0K 0 mbuf_tag 0 0K 120 ktls 0 0K 0 PUC 0 0K 0 ppbusdev 0 0K 0 agtiapi_MemAlloc malloc 0 0K 0 osti_cacheable 0 0K 0 tempbuff 0 0K 0 tempbuff 0 0K 0 ag_tgt_map_t malloc 0 0K 0 ag_slr_map_t malloc 0 0K 0 lDevFlags * malloc 0 0K 0 tiDeviceHandle_t * malloc 0 0K 0 ag_portal_data_t malloc 0 0K 0 ag_device_t malloc 0 0K 0 STLock malloc 0 0K 0 CCB List 0 0K 0 sr_iov 0 0K 0 OCS 0 0K 0 OCS 0 0K 0 nvme 0 0K 0 nvd 0 0K 0 netmap 0 0K 0 mwldev 0 0K 0 MVS driver 0 0K 0 CAM ccb queue 0 0K 0 accf 0 0K 0 pts 0 0K 0 iov 0 0K 19984 ioctlops 0 0K 108 eventfd 0 0K 0 Witness 0 0K 0 stack 0 0K 0 mrsasbuf 0 0K 0 mpt_user 0 0K 0 mps_user 0 0K 0 MPSSAS 0 0K 0 mps 0 0K 0 mpr_user 0 0K 0 firmware 0 0K 0 compressor 0 0K 0 MPRSAS 0 0K 0 SWAP 0 0K 0 mpr 0 0K 0 mfibuf 0 0K 0 sysctltmp 0 0K 703 sysctl 0 0K 3 md_sectors 0 0K 0 ekcd 0 0K 0 dumper 0 0K 0 sendfile 0 0K 0 rctl 0 0K 0 md_disk 0 0K 0 malodev 0 0K 0 LED 0 0K 0 ix_sriov 0 0K 0 cache 0 0K 0 aacraidcam 0 0K 0 prison_racct 0 0K 0 Fail Points 0 0K 0 sigio 0 0K 1 filedesc_to_leader 0 0K 0 pwd 0 0K 0 tty console 0 0K 0 ix 0 0K 0 ipsbuf 0 0K 0 aacraid_buf 0 0K 0 aaccam 0 0K 0 boottrace 0 0K 0 aacbuf 0 0K 0 zstd 0 0K 0 XZ_DEC 0 0K 0 nvlist 0 0K 0 SCSI ENC 0 0K 0 SCSI sa 0 0K 0 isofs_node 0 0K 0 isofs_mount 0 0K 0 tr_raid5_data 0 0K 0 tr_raid1e_data 0 0K 0 tr_raid1_data 0 0K 0 tr_raid0_data 0 0K 0 tr_concat_data 0 0K 0 md_sii_data 0 0K 0 md_promise_data 0 0K 0 md_nvidia_data 0 0K 0 md_jmicron_data 0 0K 0 md_intel_data 0 0K 0 md_ddf_data 0 0K 0 raid_data 0 0K 72 geom_flashmap 0 0K 0 tmpfs dir 0 0K 0 tmpfs name 0 0K 0 tmpfs mount 0 0K 0 NFS FHA 0 0K 0 newnfsmnt 0 0K 0 newnfsclient_req 0 0K 0 db> show uma Zone Size Used Free Requests Sleeps Bucket Total Mem XFree mbuf_jumbo_page 4096 8426 972 1492461 0 254 38494208 0 mbuf 256 9570 603 2413081 0 254 2604288 0 pbuf 2624 0 973 0 0 2 2553152 0 RADIX NODE 144 16389 1190 159237 0 63 2531376 0 sctp_asoc 2256 942 78 3679 0 254 2301120 0 malloc-2048 2048 947 45 7044 0 8 2031616 0 BUF TRIE 144 187 11601 4951 0 62 1697472 0 malloc-384 384 4169 31 4509 0 30 1612800 0 mbuf_cluster 2048 762 0 762 0 254 1560576 0 malloc-128 128 11634 84 17145 0 126 1499904 0 malloc-4096 4096 328 2 492 0 2 1351680 0 UMA Slabs 0 112 11552 34 11552 0 126 1297632 0 sctp_ep 1208 943 77 6849 0 254 1232160 0 vmem btag 56 19702 65 19705 0 254 1106952 0 malloc-1024 1024 949 11 3685 0 16 983040 0 malloc-384 384 1754 676 5657 0 30 933120 0 malloc-16384 16384 8 44 5645 0 1 851968 0 sctp_raddr 736 942 81 3679 0 254 752928 0 malloc-384 384 1887 63 10530 0 30 748800 0 VM OBJECT 264 1922 898 143643 0 30 744480 0 tcp_log 416 0 1782 6588 0 254 741312 0 FFS inode 1160 508 66 5971 0 8 665840 0 malloc-256 256 1907 643 6441 0 62 652800 0 tcpcb 1104 5 506 1426 0 254 564144 0 socket 960 30 478 9889 0 254 487680 0 malloc-4096 4096 63 45 6404 0 2 442368 0 MAP ENTRY 96 2293 2243 371362 0 126 435456 0 256 Bucket 2048 181 19 6389 0 8 409600 0 malloc-65536 65536 4 2 140 0 1 393216 0 lkpimm 168 1 2327 1 0 62 391104 0 lkpicurr 168 2 2326 2 0 62 391104 0 THREAD 1808 173 25 5733 0 8 357984 0 malloc-128 128 2171 619 11122 0 126 357120 0 malloc-256 256 1154 91 7084 0 62 318720 0 VNODE 448 547 119 6012 0 30 298368 0 malloc-64 64 4125 474 11140 0 254 294336 0 malloc-256 256 290 835 36602 0 62 288000 0 malloc-32768 32768 1 7 2441 0 1 262144 0 malloc-16 16 14655 345 14732 0 254 240000 0 DEVCTL 1024 23 197 157 0 0 225280 0 VMSPACE 2552 32 49 6373 0 4 206712 0 mbuf_packet 256 215 547 7700 0 254 195072 0 malloc-32 32 5361 561 5779 0 254 189504 0 UMA Zones 768 244 0 244 0 16 187392 0 FPU_save_area 832 175 41 11619 0 16 179712 0 tcp_bbr_map 128 0 1395 6427 0 126 178560 0 malloc-128 128 1210 185 26888 0 126 178560 0 FFS2 dinode 256 508 122 5971 0 62 161280 0 malloc-64 64 1980 351 7227 0 254 149184 0 malloc-1024 1024 121 23 337 0 16 147456 0 S VFS Cache 104 1007 397 6562 0 126 146016 0 sctp_stream_msg_out 112 1033 263 1045 0 254 145152 0 malloc-65536 65536 2 0 2 0 1 131072 0 malloc-65536 65536 0 2 116 0 1 131072 0 unpcb 256 14 496 1231 0 254 130560 0 PROC 1352 51 45 6388 0 8 129792 0 128 Bucket 1024 100 15 1492 0 16 117760 0 ksiginfo 112 88 956 6610 0 126 116928 0 malloc-2048 2048 10 46 1813 0 8 114688 0 sctp_laddr 48 2064 288 3519 0 254 112896 0 filedesc0 1072 46 59 6482 0 8 112560 0 malloc-128 128 525 250 4915 0 126 99200 0 UMA Kegs 384 230 3 230 0 30 89472 0 clpbuf 2624 0 32 25 0 16 83968 0 malloc-128 128 439 212 1351 0 126 83328 0 malloc-256 256 222 93 5799 0 62 80640 0 sctp_chunk 152 152 368 152 0 254 79040 0 malloc-32 32 2150 244 13641 0 254 76608 0 malloc-8192 8192 6 3 135 0 1 73728 0 ertt_txseginfo 40 0 1818 19722 0 254 72720 0 64 Bucket 512 98 38 6666 0 30 69632 0 malloc-64 64 547 524 824 0 254 68544 0 malloc-65536 65536 1 0 1 0 1 65536 0 malloc-65536 65536 1 0 1 0 1 65536 0 malloc-65536 65536 1 0 1 0 1 65536 0 malloc-32768 32768 0 2 120 0 1 65536 0 malloc-4096 4096 13 3 27 0 2 65536 0 32 Bucket 256 115 140 19802 0 62 65280 0 tcp_rack_pcb 896 1 71 936 0 16 64512 0 g_bio 408 0 150 39541 0 30 61200 0 tcp_inpcb 424 5 121 1426 0 30 53424 0 malloc-64 64 137 682 109085 0 254 52416 0 malloc-128 128 162 241 664 0 126 51584 0 malloc-128 128 185 218 337 0 126 51584 0 DIRHASH 1024 35 13 35 0 16 49152 0 NAMEI 1024 0 48 35649 0 16 49152 0 malloc-8192 8192 6 0 6 0 1 49152 0 malloc-2048 2048 10 14 12 0 8 49152 0 malloc-1024 1024 27 21 56 0 16 49152 0 malloc-1024 1024 6 42 531 0 16 49152 0 malloc-384 384 97 23 98 0 30 46080 0 syncache 168 0 264 4 0 254 44352 0 Files 80 165 385 22657 0 126 44000 0 TURNSTILE 136 199 116 199 0 62 42840 0 malloc-8192 8192 4 1 6 0 1 40960 0 malloc-8192 8192 5 0 5 0 1 40960 0 pcpu-8 8 4646 474 4828 0 254 40960 0 malloc-16 16 1921 579 38208 0 254 40000 0 Mountpoints 2752 2 12 2 0 4 38528 0 udp_inpcb 424 6 84 184 0 30 38160 0 PWD 32 16 1118 5266 0 254 36288 0 malloc-64 64 0 567 15438 0 254 36288 0 malloc-64 64 186 381 220 0 254 36288 0 malloc-64 64 79 488 6746 0 254 36288 0 malloc-64 64 50 517 72 0 254 36288 0 malloc-16 16 1911 339 7555 0 254 36000 0 malloc-128 128 43 236 10476 0 126 35712 0 routing nhops 256 23 112 35 0 62 34560 0 ttyoutq 256 72 63 160 0 62 34560 0 malloc-384 384 44 46 53 0 30 34560 0 malloc-256 256 2 133 748 0 62 34560 0 malloc-256 256 45 90 51 0 62 34560 0 malloc-256 256 51 84 716 0 62 34560 0 malloc-256 256 15 120 588 0 62 34560 0 SLEEPQUEUE 88 199 185 199 0 126 33792 0 malloc-32768 32768 1 0 1 0 1 32768 0 malloc-32768 32768 1 0 1 0 1 32768 0 malloc-16384 16384 1 1 2 0 1 32768 0 malloc-16384 16384 2 0 17 0 1 32768 0 malloc-4096 4096 5 3 634 0 2 32768 0 malloc-4096 4096 2 6 5441 0 2 32768 0 malloc-2048 2048 3 13 65 0 8 32768 0 malloc-2048 2048 7 9 33 0 8 32768 0 malloc-1024 1024 4 28 6 0 16 32768 0 malloc-1024 1024 11 21 1236 0 16 32768 0 malloc-512 512 11 53 31 0 30 32768 0 malloc-512 512 3 61 170 0 30 32768 0 malloc-512 512 4 60 26 0 30 32768 0 pcpu-64 64 493 19 493 0 254 32768 0 KNOTE 160 22 178 81507 0 62 32000 0 ttyinq 160 135 65 300 0 62 32000 0 tcp_rack_map 120 2 262 1578 0 126 31680 0 tcp_bbr_pcb 832 0 36 9 0 16 29952 0 cpuset 104 7 272 5280 0 126 29016 0 tcp_inpcb ports 32 3 879 851 0 254 28224 0 16 Bucket 144 68 128 681 0 62 28224 0 4 Bucket 48 14 574 179 0 254 28224 0 2 Bucket 32 94 788 1313 0 254 28224 0 ripcb 424 3 60 166 0 30 26712 0 udplite_inpcb 424 0 63 3 0 30 26712 0 da_ccb 544 0 49 9990 0 16 26656 0 pipe 744 18 17 401 0 16 26040 0 malloc-8192 8192 3 0 3 0 1 24576 0 malloc-4096 4096 6 0 6 0 2 24576 0 rtentry 176 27 111 35 0 62 24288 0 PGRP 88 27 249 41 0 126 24288 0 rl_entry 40 64 542 64 0 254 24240 0 8 Bucket 80 56 244 1980 0 126 24000 0 malloc-384 384 40 20 44 0 30 23040 0 domainset 40 0 567 3896 0 254 22680 0 hostcache 64 1 314 1 0 254 20160 0 udpcb 32 6 624 187 0 254 20160 0 udp_inpcb ports 32 3 627 33 0 254 20160 0 ertt 72 5 275 1426 0 126 20160 0 malloc-32 32 5 625 5221 0 254 20160 0 malloc-32 32 240 390 4921 0 254 20160 0 malloc-32 32 82 548 6380 0 254 20160 0 malloc-32 32 153 477 159 0 254 20160 0 malloc-32 32 53 577 244 0 254 20160 0 malloc-32 32 24 606 1080 0 254 20160 0 epoch_record pcpu 256 4 60 4 0 62 16384 0 malloc-16384 16384 1 0 1 0 1 16384 0 malloc-16384 16384 1 0 1 0 1 16384 0 malloc-4096 4096 3 1 3 0 2 16384 0 malloc-2048 2048 3 5 3 0 8 16384 0 malloc-2048 2048 1 7 1 0 8 16384 0 malloc-2048 2048 2 6 2 0 8 16384 0 malloc-1024 1024 1 15 1 0 16 16384 0 malloc-512 512 0 32 1 0 30 16384 0 malloc-512 512 2 30 2 0 30 16384 0 SMR CPU 32 7 504 7 0 254 16352 0 vtnet_tx_hdr 24 0 668 750090 0 254 16032 0 malloc-16 16 478 522 3445 0 254 16000 0 kenv 258 15 45 1047 0 30 15480 0 mqnode 416 3 33 3 0 30 14976 0 vmem 1856 1 7 1 0 8 14848 0 SMR SHARED 24 7 504 7 0 254 12264 0 malloc-16 16 9 741 9949 0 254 12000 0 malloc-16 16 22 728 135 0 254 12000 0 malloc-16 16 19 731 60 0 254 12000 0 malloc-16 16 33 717 6996 0 254 12000 0 malloc-384 384 0 30 1 0 30 11520 0 malloc-384 384 1 29 1 0 30 11520 0 malloc-8192 8192 1 0 1 0 1 8192 0 malloc-8192 8192 1 0 1 0 1 8192 0 pcpu-16 16 14 498 14 0 254 8192 0 UMA Slabs 1 176 10 12 10 0 62 3872 0 KMAP ENTRY 96 12 27 14 0 0 3744 0 pcpu-4 4 1 511 1 0 254 2048 0 FFS1 dinode 128 0 0 0 0 126 0 0 ada_ccb 272 0 0 0 0 30 0 0 swblk 136 0 0 0 0 62 0 0 swpctrie 144 0 0 0 0 62 0 0 cdg_qdiffsample 16 0 0 0 0 254 0 0 pf state scrubs 40 0 0 0 0 254 0 0 pf frag entries 40 0 0 0 0 254 0 0 pf frags 248 0 0 0 0 62 0 0 pf table entries 160 0 0 0 0 254 0 0 pf table entry counters 64 0 0 0 0 254 0 0 pf source nodes 136 0 0 0 0 254 0 0 pf state keys 88 0 0 0 0 126 0 0 pf states 312 0 0 0 0 254 0 0 pf tags 104 0 0 0 0 126 0 0 pf mtags 184 0 0 0 0 62 0 0 tfo_ccache_entries 80 0 0 0 0 126 0 0 tfo 4 0 0 0 0 254 0 0 sackhole 32 0 0 0 0 254 0 0 tcptw 72 0 0 0 0 254 0 0 ipq 56 0 0 0 0 254 0 0 tcp_log_id_node 120 0 0 0 0 126 0 0 tcp_log_id_bucket 176 0 0 0 0 62 0 0 tcpreass 48 0 0 0 0 254 0 0 sctp_asconf_ack 48 0 0 0 0 254 0 0 sctp_asconf 40 0 0 0 0 254 0 0 sctp_readq 152 0 0 0 0 254 0 0 ripcb ports 32 0 0 0 0 254 0 0 udplite_inpcb ports 32 0 0 0 0 254 0 0 IPsec SA lft_c 16 0 0 0 0 254 0 0 itimer 352 0 0 0 0 30 0 0 AIOLIO 272 0 0 0 0 30 0 0 AIOCB 552 0 0 0 0 16 0 0 AIOP 32 0 0 0 0 254 0 0 AIO 208 0 0 0 0 62 0 0 TMPFS node 224 0 0 0 0 62 0 0 NCLNODE 608 0 0 0 0 16 0 0 mqnotifier 216 0 0 0 0 62 0 0 mvdata 64 0 0 0 0 254 0 0 mqueue 248 0 0 0 0 62 0 0 LTS VFS Cache 360 0 0 0 0 30 0 0 L VFS Cache 320 0 0 0 0 30 0 0 STS VFS Cache 144 0 0 0 0 62 0 0 cryptop 280 0 0 0 0 30 0 0 linux_dma_object 32 0 0 0 0 254 0 0 linux_dma_pctrie 144 0 0 0 0 62 0 0 IOMMU_MAP_ENTRY 120 0 0 0 0 126 0 0 mbuf_jumbo_16k 16384 0 0 0 0 254 0 0 mbuf_jumbo_9k 9216 0 0 0 0 254 0 0 audit_record 1280 0 0 0 0 8 0 0 MAC labels 40 0 0 0 0 254 0 0 vnpbuf 2624 0 0 0 0 64 0 0 mdpbuf 2624 0 0 0 0 3 0 0 nfspbuf 2624 0 0 0 0 16 0 0 swwbuf 2624 0 0 0 0 8 0 0 swrbuf 2624 0 0 0 0 16 0 0 umtx_shm 88 0 0 0 0 126 0 0 umtx pi 96 0 0 0 0 126 0 0 rangeset pctrie nodes 144 0 0 0 0 62 0 0 malloc-65536 65536 0 0 0 0 1 0 0 malloc-65536 65536 0 0 0 0 1 0 0 malloc-32768 32768 0 0 0 0 1 0 0 malloc-32768 32768 0 0 0 0 1 0 0 malloc-32768 32768 0 0 0 0 1 0 0 malloc-32768 32768 0 0 0 0 1 0 0 malloc-16384 16384 0 0 0 0 1 0 0 malloc-16384 16384 0 0 0 0 1 0 0 malloc-16384 16384 0 0 0 0 1 0 0 malloc-8192 8192 0 0 0 0 1 0 0 malloc-4096 4096 0 0 0 0 2 0 0 malloc-1024 1024 0 0 0 0 16 0 0 malloc-512 512 0 0 0 0 30 0 0 malloc-512 512 0 0 0 0 30 0 0 malloc-512 512 0 0 0 0 30 0 0 pcpu-32 32 0 0 0 0 254 0 0 fakepg 104 0 0 0 0 126 0 0 UMA Hash 256 0 0 0 0 62 0 0