panic: ipreass_callout: stray callout on bucket 0xfffffe00579da100 cpuid = 1 time = 27 KDB: stack backtrace: db_trace_self_wrapper() at db_trace_self_wrapper+0xc7/frame 0xfffffe0053ab7950 kdb_backtrace() at kdb_backtrace+0xd3/frame 0xfffffe0053ab7ab0 vpanic() at vpanic+0x254/frame 0xfffffe0053ab7b90 panic() at panic+0xb5/frame 0xfffffe0053ab7c50 ipreass_callout() at ipreass_callout+0x3c9/frame 0xfffffe0053ab7cb0 softclock_call_cc() at softclock_call_cc+0x3d7/frame 0xfffffe0053ab7e80 softclock_thread() at softclock_thread+0x1ff/frame 0xfffffe0053ab7ef0 fork_exit() at fork_exit+0xd0/frame 0xfffffe0053ab7f30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe0053ab7f30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- KDB: enter: panic [ thread pid 2 tid 100029 ] Stopped at kdb_enter+0x6b: movq $0,0x275660a(%rip) db> db> set $lines = 0 db> set $maxwidth = 0 db> show registers cs 0x20 ds 0x3b es 0x3b fs 0x13 gs 0x1b ss 0 rax 0x12 rcx 0xffffffff8172d026 printf+0xf6 rdx 0x1 rbx 0 rsp 0xfffffe0053ab7a90 rbp 0xfffffe0053ab7ab0 rsi 0 rdi 0xffffffff8172d088 printf+0x158 r8 0 r9 0xffffffff r10 0 r11 0 r12 0 r13 0xfffffe0053e9c3a0 r14 0xffffffff82b6d440 .str.26 r15 0xffffffff82b6d440 .str.26 rip 0xffffffff8171cb8b kdb_enter+0x6b rflags 0x46 kdb_enter+0x6b: movq $0,0x275660a(%rip) db> show proc Process 2 (clock) at 0xfffffe0053ddb008: state: NORMAL uid: 0 gids: 0 parent: pid 0 at 0xffffffff83e167e0 ABI: null flag: 0x10000284 flag2: 0 reaper: 0xffffffff83e167e0 reapsubtree: 2 sigparent: 20 vmspace: 0xffffffff83e17780 (map 0xffffffff83e17780) (map.pmap 0xffffffff83e17840) (pmap 0xffffffff83e178b0) threads: 2 100029 Run CPU 1 [clock (0)] 100030 I [clock (1)] db> ps pid ppid pgrp uid state wmesg wchan cmd 19142 777 777 0 R (threaded) syz-executor.0 105950 RunQ syz-executor.0 121240 RunQ syz-executor.0 19138 785 785 0 R (threaded) syz-executor.2 118646 RunQ syz-executor.2 121234 RunQ syz-executor.2 121236 S uwait 0xfffffe0099abf900 syz-executor.2 19136 778 778 0 S (threaded) syz-executor.1 118844 S nanslp 0xffffffff83e40941 syz-executor.1 121231 S connec 0xfffffe009a1ff49a syz-executor.1 121235 S uwait 0xfffffe0058b48700 syz-executor.1 9177 1 790 0 SV uwait 0xfffffe009236b380 syz-executor.3 8531 1 785 0 SV uwait 0xfffffe0058a04a00 syz-executor.2 6099 1 785 0 SV uwait 0xfffffe00998f0380 syz-executor.2 3662 1 3662 65 Ss select 0xfffffe00998f0140 dhclient 3447 1 785 0 SV uwait 0xfffffe00998f0b80 syz-executor.2 3313 1 790 0 S umtxn 0xfffffe0058b48b00 syz-executor.3 3011 1 3011 0 Ss select 0xfffffe0058b4b5c0 dhclient 3008 1 3008 0 Ss select 0xfffffe0099ac0940 dhclient 2989 1 2989 65 Ss select 0xfffffe0099ac0ac0 dhclient 2680 1 790 0 S uwait 0xfffffe0058b48500 syz-executor.3 2640 1 778 0 S uwait 0xfffffe0058b4b500 syz-executor.1 2637 1 778 0 S uwait 0xfffffe005700a800 syz-executor.1 2632 1 778 0 S uwait 0xfffffe009236aa00 syz-executor.1 2626 1 778 0 S uwait 0xfffffe0058b48c00 syz-executor.1 2618 1 778 0 S uwait 0xfffffe0058a03a80 syz-executor.1 2612 1 778 0 S uwait 0xfffffe00998f0780 syz-executor.1 2608 1 778 0 S uwait 0xfffffe0058b49080 syz-executor.1 2490 0 0 0 DL - 0xffffffff83f60500 [soaiod4] 2489 0 0 0 DL - 0xffffffff83f60500 [soaiod3] 2488 0 0 0 DL - 0xffffffff83f60500 [soaiod2] 2487 0 0 0 DL - 0xffffffff83f60500 [soaiod1] 2432 1 2432 0 Ss select 0xfffffe00998f0240 dhclient 2429 1 2429 0 Ss select 0xfffffe00998f01c0 dhclient 2409 1 2409 65 Ss select 0xfffffe0099ac1740 dhclient 1867 1 1867 0 Ss select 0xfffffe00998f02c0 dhclient 1864 1 1864 0 Ss select 0xfffffe0099ac16c0 dhclient 1843 1 1843 65 Ss select 0xfffffe0099ac17c0 dhclient 1556 0 0 0 DL aiordy 0xfffffe0092bd7568 [aiod4] 1555 0 0 0 DL aiordy 0xfffffe0092666560 [aiod3] 1554 0 0 0 DL aiordy 0xfffffe0056f90008 [aiod2] 1553 0 0 0 DL aiordy 0xfffffe009249d558 [aiod1] 1228 1 1228 0 Ss select 0xfffffe0058b4b6c0 dhclient 1225 1 1225 0 Ss select 0xfffffe0058a05440 dhclient 790 774 790 0 Ss piperd 0xfffffe0092bb32d8 syz-executor.3 785 774 785 0 Rs syz-executor.2 778 774 778 0 Rs syz-executor.1 777 774 777 0 Rs syz-executor.0 774 772 772 0 S (threaded) syz-fuzzer 100098 S wait 0xfffffe0057998560 syz-fuzzer 100117 S uwait 0xfffffe0058b49280 syz-fuzzer 100118 S uwait 0xfffffe0058b49380 syz-fuzzer 100119 S uwait 0xfffffe0058b49480 syz-fuzzer 100120 S uwait 0xfffffe0058b49580 syz-fuzzer 100121 S uwait 0xfffffe009236bd00 syz-fuzzer 100122 S wait 0xfffffe0057998560 syz-fuzzer 100124 S wait 0xfffffe0057998560 syz-fuzzer 100125 S uwait 0xfffffe009236c080 syz-fuzzer 100126 S wait 0xfffffe0057998560 syz-fuzzer 100134 S uwait 0xfffffe009236ac00 syz-fuzzer 100145 S uwait 0xfffffe009236af00 syz-fuzzer 107317 S kqread 0xfffffe000817ec00 syz-fuzzer 772 770 772 0 Ss pause 0xfffffe0053ddd0c8 csh 770 688 770 0 Ss select 0xfffffe0058b49740 sshd 754 1 754 0 Ss+ ttyin 0xfffffe0057469cb0 getty 753 1 753 0 Ss+ ttyin 0xfffffe005794a8b0 getty 752 1 752 0 Ss+ ttyin 0xfffffe005794acb0 getty 751 1 751 0 Ss+ ttyin 0xfffffe005794b0b0 getty 750 1 750 0 Ss+ ttyin 0xfffffe0057468cb0 getty 749 1 749 0 Ss+ ttyin 0xfffffe005794b4b0 getty 748 1 748 0 Ss+ ttyin 0xfffffe005794b8b0 getty 747 1 747 0 Ss+ ttyin 0xfffffe005794bcb0 getty 746 1 746 0 Ss+ ttyin 0xfffffe005794c0b0 getty 692 1 692 0 Ss nanslp 0xffffffff83e40941 cron 688 1 688 0 Ss select 0xfffffe009236b740 sshd 501 1 501 0 Ss select 0xfffffe0058b49840 syslogd 430 1 430 0 Ss select 0xfffffe009236c440 devd 429 1 429 65 Ss select 0xfffffe009236c2c0 dhclient 344 1 344 0 Ss select 0xfffffe009236c340 dhclient 341 1 341 0 Ss select 0xfffffe0058b499c0 dhclient 17 0 0 0 DL syncer 0xffffffff83f65de0 [syncer] 16 0 0 0 DL vlruwt 0xfffffe0056f91010 [vnlru] 15 0 0 0 DL (threaded) [bufdaemon] 100079 D psleep 0xffffffff83f64400 [bufdaemon] 100082 D - 0xffffffff83212100 [bufspacedaemon-0] 100095 D sdflush 0xfffffe0056f588e8 [/ worker] 9 0 0 0 DL psleep 0xffffffff83f97e80 [vmdaemon] 8 0 0 0 DL (threaded) [pagedaemon] 100077 D psleep 0xffffffff83f8bd38 [dom0] 100080 D launds 0xffffffff83f8bd44 [laundry: dom0] 100081 D umarcl 0xffffffff81e5d8b0 [uma] 7 0 0 0 DL - 0xffffffff83bfcde8 [rand_harvestq] 6 0 0 0 DL pftm 0xffffffff845a3550 [pf purge] 5 0 0 0 DL waiting 0xffffffff848be640 [sctp_iterator] 4 0 0 0 DL (threaded) [cam] 100044 D - 0xffffffff83a9f340 [doneq0] 100045 D - 0xffffffff83a9f2c0 [async] 100076 D - 0xffffffff83a9f140 [scanner] 14 0 0 0 DL seqstat 0xfffffe0053fa7088 [sequencer 00] 3 0 0 0 DL (threaded) [crypto] 100040 D crypto_ 0xffffffff83f87460 [crypto] 100041 D crypto_ 0xfffffe0053c80e30 [crypto returns 0] 100042 D crypto_ 0xfffffe0053c80e80 [crypto returns 1] 13 0 0 0 DL (threaded) [geom] 100035 D - 0xffffffff83e15dc0 [g_event] 100036 D - 0xffffffff83e15de0 [g_up] 100037 D - 0xffffffff83e15e00 [g_down] 2 0 0 0 RL (threaded) [clock] 100029 Run CPU 1 [clock (0)] 100030 I [clock (1)] 12 0 0 0 RL (threaded) [intr] 100013 I [swi6: Giant taskq] 100015 I [swi5: fast taskq] 100018 I [swi6: task queue] 100031 I [swi1: netisr 0] 100032 Run CPU 0 [swi1: hpts] 100033 I [swi1: hpts] 100046 I [irq24: virtio_pci0] 100047 I [irq25: virtio_pci0] 100048 I [irq26: virtio_pci0] 100049 I [irq27: virtio_pci0] 100050 I [irq28: virtio_pci1] 100051 I [irq29: virtio_pci1] 100052 I [irq30: virtio_pci1] 100053 I [irq31: virtio_pci1] 100054 I [irq32: virtio_pci1] 100059 I [irq33: virtio_pci2] 100060 I [irq34: virtio_pci2] 100061 I [irq35: virtio_pci2] 100063 I [irq1: atkbd0] 100064 I [irq12: psm0] 100065 I [swi0: uart uart++] 100069 I [swi1: pf send] 11 0 0 0 RL (threaded) [idle] 100003 CanRun [idle: cpu0] 100004 CanRun [idle: cpu1] 1 0 1 0 SLs wait 0xfffffe0053ddc010 [init] 10 0 0 0 DL audit_w 0xffffffff83f88000 [audit] 0 0 0 0 DLs (threaded) [kernel] 100000 D swapin 0xffffffff83e167e0 [swapper] 100005 D - 0xfffffe0008181d00 [softirq_0] 100006 D - 0xfffffe0008181c00 [softirq_1] 100007 D - 0xfffffe0008181b00 [if_io_tqg_0] 100008 D - 0xfffffe0008181a00 [if_io_tqg_1] 100009 D - 0xfffffe0008181900 [if_config_tqg_0] 100010 D - 0xfffffe0008181800 [in6m_free taskq] 100011 D - 0xfffffe0008181700 [linuxkpi_irq_wq] 100012 D - 0xfffffe0008181600 [deferred_unmount ta] 100014 D - 0xfffffe0008181400 [thread taskq] 100016 D - 0xfffffe0008181200 [kqueue_ctx taskq] 100017 D - 0xfffffe0008181100 [pci_hp taskq] 100019 D - 0xfffffe0008180e00 [inm_free taskq] 100020 D - 0xfffffe0008180d00 [aiod_kick taskq] 100021 D - 0xfffffe0008180c00 [linuxkpi_short_wq_0] 100022 D - 0xfffffe0008180c00 [linuxkpi_short_wq_1] 100023 D - 0xfffffe0008180c00 [linuxkpi_short_wq_2] 100024 D - 0xfffffe0008180c00 [linuxkpi_short_wq_3] 100025 D - 0xfffffe0008180b00 [linuxkpi_long_wq_0] 100026 D - 0xfffffe0008180b00 [linuxkpi_long_wq_1] 100027 D - 0xfffffe0008180b00 [linuxkpi_long_wq_2] 100028 D - 0xfffffe0008180b00 [linuxkpi_long_wq_3] 100034 D - 0xfffffe0008180a00 [firmware taskq] 100038 D - 0xfffffe0008180900 [crypto_0] 100039 D - 0xfffffe0008180900 [crypto_1] 100055 D - 0xfffffe0008180500 [vtnet0 rxq 0] 100056 D - 0xfffffe0008180400 [vtnet0 txq 0] 100057 D - 0xfffffe0008180300 [vtnet0 rxq 1] 100058 D - 0xfffffe0008180200 [vtnet0 txq 1] 100062 D vtbslp 0xfffffe0056f47e80 [virtio_balloon] 100066 D - 0xffffffff82b734a1 [deadlkres] 100071 D - 0xfffffe00579c3000 [acpi_task_0] 100072 D - 0xfffffe00579c3000 [acpi_task_1] 100073 D - 0xfffffe00579c3000 [acpi_task_2] 100074 D - 0xfffffe0008182100 [mca taskq] 100075 D - 0xfffffe0008180600 [CAM taskq] db> show all locks Process 777 (syz-executor.0) thread 0xfffffe0058d623a0 (100112) exclusive sleep mutex process lock (process lock) r = 0 (0xfffffe0092665128) locked @ /syzkaller/managers/i386/kernel/sys/kern/subr_trap.c:105 Process 2 (clock) thread 0xfffffe0053e9c3a0 (100029) exclusive sleep mutex IP reassembly (IP reassembly) r = 0 (0xfffffe00579da110) locked @ /syzkaller/managers/i386/kernel/sys/kern/kern_mutex.c:211 Process 12 (intr) thread 0xfffffe0053e9ae40 (100032) exclusive sleep mutex tcp_hpts_lck (hpts) r = 0 (0xfffffe0053e0f000) locked @ /syzkaller/managers/i386/kernel/sys/netinet/tcp_hpts.c:1420 db> show malloc Type InUse MemUse Requests pf_hash 5 11524K 5 tcp_hpts 7 4801K 7 devbuf 4217 4323K 4245 sysctloid 35217 2075K 35288 vtbuf 24 1968K 46 kobj 329 1316K 493 pcb 559 1140K 41967 newblk 39 1034K 30099 vfscache 3 1025K 3 inodedep 19 519K 18939 ufs_quota 1 512K 1 vfs_hash 1 512K 1 callout 2 512K 2 intr 4 472K 4 subproc 177 346K 19243 filedesc 42 329K 36053 vmem 3 274K 6 sctp_stro 267 267K 6094 sctp_atcl 535 201K 27140 acpica 1674 184K 58126 tidhash 3 141K 3 linker 358 134K 386 pagedep 9 130K 18031 tfo_ccache 1 128K 1 IP reass 1 128K 1 vnet_data 1 112K 1 DEVFS1 109 109K 126 sem 4 106K 4 BPF 46 88K 91 bus 998 82K 5213 mtx_pool 2 72K 2 syncache 1 68K 1 module 516 65K 516 acpitask 1 64K 1 ddb_capture 1 64K 1 umtx 418 53K 418 kdtrace 259 51K 40384 sctp_atky 802 34K 33257 temp 34 33K 6217 DEVFS3 128 32K 138 hostcache 1 32K 1 shm 1 32K 8 msg 4 30K 4 gtaskqueue 18 26K 18 kbdmux 6 22K 6 ifaddr 70 20K 74 DEVFS_RULE 56 20K 56 sctp_timw 73 19K 73 ufs_mount 4 17K 5 proc 3 17K 3 routetbl 156 17K 497 tty 16 16K 16 ithread 97 16K 97 bus-sc 34 15K 1682 lltable 45 14K 123 KTRACE 100 13K 101 eventhandler 151 13K 151 ifnet 7 13K 7 ether_multi 152 13K 172 kenv 95 12K 95 rman 88 11K 431 GEOM 61 11K 483 CAM queue 5 11K 1528 in6_multi 65 9K 65 sctp_athm 535 9K 27217 sctp_map 534 9K 12188 bmsafemap 2 9K 18723 UART 12 9K 12 devstat 4 9K 4 ksem 1 8K 115 rpc 2 8K 2 shmfd 1 8K 9 pfs_vncache 1 8K 1 pfs_nodes 20 8K 20 audit_evclass 237 8K 296 kqueue 79 8K 19172 taskqueue 63 7K 63 cred 26 7K 279 sglist 5 7K 5 CAM DEV 3 6K 510 plimit 24 6K 455 pwddesc 77 5K 19143 ufs_dirhash 24 5K 24 UMA 271 5K 271 session 35 5K 89 pf_ifnet 10 5K 19 DEVFSP 68 5K 274 vt 11 5K 11 memdesc 1 4K 1 MCA 32 4K 32 evdev 4 4K 4 freework 16 4K 22863 proc-args 104 4K 20636 acpisem 28 4K 28 selfd 55 4K 255525 lockf 32 4K 60 hhook 15 4K 17 kcovinfo 52 4K 52 terminal 11 3K 11 select 19 3K 99 uidinfo 3 3K 20 local_apic 1 2K 1 io_apic 1 2K 1 fpukern_ctx 2 2K 2 ipsec-saq 2 2K 2 ip6ndp 12 2K 13 sctp_ifa 14 2K 15 dirrem 7 2K 18437 Unitno 31 2K 517 CAM XPT 22 2K 543 msi 12 2K 12 in_multi 6 2K 10 ipsecpolicy 2 2K 2 acpidev 20 2K 20 clone 9 2K 9 tun 7 2K 7 softdep 1 1K 1 mkdir 8 1K 36018 freefile 8 1K 18422 sahead 1 1K 1 secasvar 1 1K 1 nhops 6 1K 8 vnodemarker 2 1K 88 NFSD session 1 1K 1 CAM periph 4 1K 271 ipsec 3 1K 3 sctp_ifn 6 1K 15 freeblks 3 1K 18570 mld 6 1K 6 igmp 6 1K 6 toponodes 6 1K 6 isadev 6 1K 6 mount 16 1K 89 pci_link 10 1K 10 inpcbpolicy 21 1K 6627 diradd 5 1K 18478 crypto 4 1K 4 encap_export_host 12 1K 12 procdesc 5 1K 18 CC Mem 9 1K 4700 newdirblk 4 1K 18009 pfil 4 1K 4 cdev 2 1K 2 osd 13 1K 4698 chacha20random 1 1K 1 tcp_fsb 2 1K 2210 NFSD lckfile 1 1K 1 NFSD V4client 1 1K 1 DEVFS 9 1K 10 indirdep 1 1K 7119 vnodes 1 1K 1 CAM SIM 2 1K 2 feeder 7 1K 7 tcpfunc 3 1K 3 loginclass 3 1K 6 prison 6 1K 6 lkpikmalloc 5 1K 6 aesni_data 2 1K 2 soname 5 1K 21715 cryptodev 2 1K 131 nexusdev 8 1K 8 apmdev 1 1K 1 atkbddev 2 1K 2 CAM dev queue 2 1K 2 CAM I/O Scheduler 1 1K 1 CAM path 4 1K 1034 aio 4 1K 10 pmchooks 1 1K 1 filecaps 5 1K 117 sctp_vrf 1 1K 1 vnet 1 1K 1 entropy 2 1K 48 pmc 1 1K 1 acpiintr 1 1K 1 cpus 2 1K 2 vnet_data_free 1 1K 1 Per-cpu 1 1K 1 p1003.1b 1 1K 1 tcp_do 0 0K 0 ipcomp 0 0K 0 esp 0 0K 0 ah 0 0K 0 mqdata 0 0K 0 sctp_mcore 0 0K 0 sctp_socko 0 0K 10538 sctp_iter 0 0K 14 sctp_mvrf 0 0K 0 sctp_cpal 0 0K 0 sctp_cmsg 0 0K 0 sctp_stre 0 0K 0 sctp_athi 0 0K 0 sctp_a_it 0 0K 14 sctp_aadr 0 0K 103 sctp_stri 0 0K 254 pf_table 0 0K 0 pf_rule 0 0K 42 pf_altq 0 0K 0 pf_osfp 0 0K 0 pf_temp 0 0K 0 filemon 0 0K 105 NFSCL layrecall 0 0K 0 NFSCL session 0 0K 0 madt_table 0 0K 2 smartpqi 0 0K 0 NFSCL sockreq 0 0K 0 NFSCL devinfo 0 0K 0 NFSCL flayout 0 0K 0 ixl 0 0K 0 NFSCL layout 0 0K 0 NFSD rollback 0 0K 0 NFSCL diroff 0 0K 0 NEWdirectio 0 0K 0 NEWNFSnode 0 0K 0 NFSCL lck 0 0K 0 NFSCL lckown 0 0K 0 ice-resmgr 0 0K 0 ice-osdep 0 0K 0 ice 0 0K 0 iavf 0 0K 0 axgbe 0 0K 0 NFSCL client 0 0K 0 NFSCL deleg 0 0K 0 NFSCL open 0 0K 0 NFSCL owner 0 0K 0 NFS fh 0 0K 0 NFS req 0 0K 0 NFSD usrgroup 0 0K 0 NFSD string 0 0K 0 NFSD V4lock 0 0K 0 xen_intr 0 0K 0 NFSD V4state 0 0K 0 xen_hvm 0 0K 0 legacydrv 0 0K 0 bounce 0 0K 0 busdma 0 0K 0 qpidrv 0 0K 0 NFSD srvcache 0 0K 0 msdosfs_fat 0 0K 0 dmar_idpgtbl 0 0K 0 dmar_dom 0 0K 0 dmar_ctx 0 0K 0 msdosfs_mount 0 0K 0 msdosfs_node 0 0K 0 isci 0 0K 0 iommu_dmamap 0 0K 0 hyperv_socket 0 0K 0 bxe_ilt 0 0K 0 xenbus 0 0K 0 DEVFS4 0 0K 0 vm_fictitious 0 0K 0 DEVFS2 0 0K 0 gntdev 0 0K 0 privcmd_dev 0 0K 0 evtchn_dev 0 0K 0 xenstore 0 0K 0 scsi_pass 0 0K 0 ciss_data 0 0K 0 xnb 0 0K 0 xen_acpi 0 0K 0 xbbd 0 0K 0 xbd 0 0K 0 UMAHash 0 0K 0 Balloon 0 0K 0 sysmouse 0 0K 0 vm_pgdata 0 0K 0 jblocks 0 0K 0 savedino 0 0K 18160 sentinel 0 0K 0 jfsync 0 0K 0 jtrunc 0 0K 0 sbdep 0 0K 41 jsegdep 0 0K 0 jseg 0 0K 0 jfreefrag 0 0K 0 jfreeblk 0 0K 0 jnewblk 0 0K 0 jmvref 0 0K 0 jremref 0 0K 0 jaddref 0 0K 0 freedep 0 0K 0 freefrag 0 0K 2 allocindir 0 0K 0 allocdirect 0 0K 0 ufs_trim 0 0K 0 mactemp 0 0K 0 audit_trigger 0 0K 0 audit_pipe_presel 0 0K 0 audit_pipeent 0 0K 0 audit_pipe 0 0K 0 audit_evname 0 0K 0 audit_bsm 0 0K 0 audit_gidset 0 0K 0 audit_text 0 0K 0 audit_path 0 0K 0 audit_data 0 0K 0 audit_cred 0 0K 0 vtfont 0 0K 0 BACKLIGHT 0 0K 0 ath_hal 0 0K 0 athdev 0 0K 0 ata_pci 0 0K 0 ata_dma 0 0K 0 ata_generic 0 0K 0 pvscsi 0 0K 0 scsi_da 0 0K 69 ata_da 0 0K 0 scsi_ch 0 0K 0 scsi_cd 0 0K 0 ktls_ocf 0 0K 0 AHCI driver 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5E_TLS_RX 0 0K 0 MLX5EEPROM 0 0K 0 MLX5E_TLS 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EN 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5DUMP 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 seq_file 0 0K 0 lkpiskb 0 0K 0 radix 0 0K 0 idr 0 0K 0 lkpindev 0 0K 0 lkpifw 0 0K 0 lkpi80211 0 0K 0 NLM 0 0K 0 ipsec-spdcache 0 0K 0 ipsec-reg 0 0K 0 ipsec-misc 0 0K 0 ipsecrequest 0 0K 0 ip6opt 0 0K 451 ip6_msource 0 0K 0 ip6_moptions 0 0K 0 in6_mfilter 0 0K 0 frag6 0 0K 0 tcplog 0 0K 0 tcp_hwpace 0 0K 0 USBdev 0 0K 0 USB 0 0K 0 LRO 0 0K 0 ip_msource 0 0K 0 ip_moptions 0 0K 0 in_mfilter 0 0K 0 ipid 0 0K 0 80211scan 0 0K 0 80211ratectl 0 0K 0 80211power 0 0K 0 80211nodeie 0 0K 0 80211node 0 0K 0 80211mesh_gt 0 0K 0 80211mesh_rt 0 0K 0 80211perr 0 0K 0 80211prep 0 0K 0 80211preq 0 0K 0 80211dfs 0 0K 0 80211crypto 0 0K 0 80211vap 0 0K 0 iflib 0 0K 0 vlan 0 0K 0 gif 0 0K 0 ifdescr 0 0K 0 zlib 0 0K 0 fadvise 0 0K 0 VN POLL 0 0K 0 agp 0 0K 0 statfs 0 0K 18408 namei_tracker 0 0K 1 export_host 0 0K 0 cl_savebuf 0 0K 2 nvme_da 0 0K 0 acpipwr 0 0K 0 acpi_perf 0 0K 0 twsbuf 0 0K 0 twe_commands 0 0K 0 tcp_log_dev 0 0K 6 midi buffers 0 0K 0 mixer 0 0K 0 ac97 0 0K 0 hdacc 0 0K 0 hdac 0 0K 0 hdaa 0 0K 0 acpicmbat 0 0K 0 SIIS driver 0 0K 0 CAM CCB 0 0K 523 biobuf 0 0K 0 lio 0 0K 2 acl 0 0K 0 mbuf_tag 0 0K 0 ktls 0 0K 0 PUC 0 0K 0 ppbusdev 0 0K 0 agtiapi_MemAlloc malloc 0 0K 0 osti_cacheable 0 0K 0 tempbuff 0 0K 0 tempbuff 0 0K 0 ag_tgt_map_t malloc 0 0K 0 ag_slr_map_t malloc 0 0K 0 lDevFlags * malloc 0 0K 0 tiDeviceHandle_t * malloc 0 0K 0 ag_portal_data_t malloc 0 0K 0 ag_device_t malloc 0 0K 0 STLock malloc 0 0K 0 CCB List 0 0K 0 sr_iov 0 0K 0 OCS 0 0K 0 OCS 0 0K 0 nvme 0 0K 0 nvd 0 0K 0 netmap 0 0K 0 mwldev 0 0K 0 MVS driver 0 0K 0 CAM ccb queue 0 0K 0 accf 0 0K 0 pts 0 0K 0 iov 0 0K 21252 ioctlops 0 0K 504 eventfd 0 0K 2 Witness 0 0K 0 stack 0 0K 0 mrsasbuf 0 0K 0 mpt_user 0 0K 0 mps_user 0 0K 0 MPSSAS 0 0K 0 mps 0 0K 0 sbuf 0 0K 288 mpr_user 0 0K 0 firmware 0 0K 0 compressor 0 0K 0 MPRSAS 0 0K 0 SWAP 0 0K 0 mpr 0 0K 0 mfibuf 0 0K 0 sysctltmp 0 0K 976 sysctl 0 0K 3 md_sectors 0 0K 0 ekcd 0 0K 0 dumper 0 0K 0 sendfile 0 0K 0 rctl 0 0K 0 md_disk 0 0K 0 malodev 0 0K 0 LED 0 0K 0 ix_sriov 0 0K 0 cache 0 0K 0 aacraidcam 0 0K 0 prison_racct 0 0K 0 Fail Points 0 0K 0 sigio 0 0K 1 filedesc_to_leader 0 0K 0 pwd 0 0K 0 tty console 0 0K 0 ix 0 0K 0 ipsbuf 0 0K 0 aacraid_buf 0 0K 0 aaccam 0 0K 0 boottrace 0 0K 0 aacbuf 0 0K 0 zstd 0 0K 0 XZ_DEC 0 0K 0 nvlist 0 0K 0 SCSI ENC 0 0K 0 SCSI sa 0 0K 0 isofs_node 0 0K 0 isofs_mount 0 0K 0 tr_raid5_data 0 0K 0 tr_raid1e_data 0 0K 0 tr_raid1_data 0 0K 0 tr_raid0_data 0 0K 0 tr_concat_data 0 0K 0 md_sii_data 0 0K 0 md_promise_data 0 0K 0 md_nvidia_data 0 0K 0 md_jmicron_data 0 0K 0 md_intel_data 0 0K 0 md_ddf_data 0 0K 0 raid_data 0 0K 72 geom_flashmap 0 0K 0 tmpfs dir 0 0K 0 tmpfs name 0 0K 0 tmpfs mount 0 0K 0 NFS FHA 0 0K 0 newnfsmnt 0 0K 0 newnfsclient_req 0 0K 0 db> show uma Zone Size Used Free Requests Sleeps Bucket Total Mem XFree mbuf_jumbo_page 4096 8350 1048 1586237 0 254 38494208 0 mbuf 256 9113 1061 2613842 0 254 2604544 0 RADIX NODE 144 17390 553 476734 0 63 2583792 0 tcp_log 416 0 5337 10052 0 254 2220192 0 mbuf_cluster 2048 1016 0 1016 0 254 2080768 0 pbuf 2624 0 778 0 0 2 2041472 0 BUF TRIE 144 180 11580 13182 0 62 1693440 0 malloc-128 128 12589 152 12694 0 126 1630848 0 malloc-384 384 4156 44 4158 0 30 1612800 0 malloc-4096 4096 329 3 497 0 2 1359872 0 UMA Slabs 0 112 11322 12 11322 0 126 1269408 0 sctp_asoc 2256 267 243 6094 0 254 1150560 0 vmem btag 56 19047 144 19047 0 254 1074696 0 sctp_ep 1208 268 502 20782 0 254 930160 0 malloc-2048 2048 269 171 20825 0 8 901120 0 socket 960 49 715 28870 0 254 733440 0 VM OBJECT 264 2293 257 289223 0 30 673200 0 FFS inode 1160 530 30 18953 0 8 649600 0 tcpcb 1104 9 502 4687 0 254 564144 0 256 Bucket 2048 221 19 7126 0 8 491520 0 malloc-16384 16384 21 5 18025 0 1 425984 0 lkpimm 168 1 2327 1 0 62 391104 0 lkpicurr 168 2 2326 2 0 62 391104 0 THREAD 1824 181 28 21240 0 8 381216 0 sctp_raddr 736 267 250 6264 0 254 380512 0 MAP ENTRY 96 3076 830 647189 0 126 374976 0 malloc-384 384 18 942 18938 0 30 368640 0 malloc-256 256 515 925 60787 0 62 368640 0 malloc-4096 4096 77 7 19143 0 2 344064 0 malloc-1024 1024 269 67 6189 0 16 344064 0 malloc-256 256 166 1019 19135 0 62 303360 0 VNODE 448 571 95 18996 0 30 298368 0 malloc-64 64 3968 631 23092 0 254 294336 0 malloc-384 384 536 214 27141 0 30 288000 0 malloc-65536 65536 4 0 4 0 1 262144 0 malloc-65536 65536 1 3 383 0 1 262144 0 mbuf_packet 256 261 755 30166 0 254 260096 0 malloc-16 16 15118 382 31541 0 254 248000 0 malloc-32768 32768 0 7 6762 0 1 229376 0 malloc-2048 2048 6 106 1229 0 8 229376 0 DEVCTL 1024 0 220 156 0 0 225280 0 malloc-65536 65536 2 1 10 0 1 196608 0 malloc-32 32 5346 576 23414 0 254 189504 0 UMA Zones 768 243 1 243 0 16 187392 0 FPU_save_area 832 183 42 30420 0 16 187200 0 malloc-128 128 1135 260 27305 0 126 178560 0 VMSPACE 2560 52 14 19120 0 4 168960 0 tcp_bbr_map 128 0 1271 3690 0 126 162688 0 FFS2 dinode 256 530 100 18952 0 62 161280 0 S VFS Cache 104 1025 496 19578 0 126 158184 0 malloc-1024 1024 123 21 141 0 16 147456 0 malloc-128 128 81 1066 19750 0 126 146816 0 PROC 1368 76 23 19142 0 8 135432 0 malloc-65536 65536 2 0 2 0 1 131072 0 malloc-65536 65536 0 2 136 0 1 131072 0 malloc-4096 4096 27 5 112 0 2 131072 0 unpcb 256 21 489 1401 0 254 130560 0 malloc-256 256 150 360 31390 0 62 130560 0 128 Bucket 1024 94 33 1724 0 16 130048 0 ksiginfo 112 87 957 9355 0 126 116928 0 ertt_txseginfo 40 2 2826 40662 0 254 113120 0 filedesc0 1072 77 21 19143 0 8 105056 0 malloc-8192 8192 7 5 115 0 1 98304 0 pipe 728 23 109 845 0 16 96096 0 UMA Kegs 384 229 4 229 0 30 89472 0 64 Bucket 512 118 50 15958 0 30 86016 0 clpbuf 2624 0 32 16 0 16 83968 0 malloc-128 128 342 309 54581 0 126 83328 0 malloc-4096 4096 15 5 30 0 2 81920 0 malloc-256 256 87 228 60036 0 62 80640 0 sctp_readq 152 0 520 88 0 254 79040 0 sctp_chunk 152 267 253 1119 0 254 79040 0 udp_inpcb 424 6 174 1528 0 30 76320 0 g_bio 408 0 180 96540 0 30 73440 0 malloc-64 64 483 588 1077 0 254 68544 0 malloc-64 64 522 549 6303 0 254 68544 0 malloc-32768 32768 2 0 2 0 1 65536 0 malloc-32768 32768 0 2 120 0 1 65536 0 32 Bucket 256 126 129 9704 0 62 65280 0 sctp_stream_msg_out 112 1 539 718 0 254 60480 0 tcp_inpcb 424 9 117 4687 0 30 53424 0 sctp_laddr 48 0 1092 7553 0 254 52416 0 malloc-64 64 331 488 18670 0 254 52416 0 malloc-64 64 248 571 277881 0 254 52416 0 malloc-64 64 328 491 6412 0 254 52416 0 Files 80 229 421 59895 0 126 52000 0 malloc-128 128 138 265 18940 0 126 51584 0 malloc-128 128 181 222 554 0 126 51584 0 malloc-256 256 67 128 376 0 62 49920 0 malloc-256 256 93 102 605 0 62 49920 0 malloc-256 256 24 171 19948 0 62 49920 0 DIRHASH 1024 35 13 35 0 16 49152 0 NAMEI 1024 0 48 91475 0 16 49152 0 malloc-16384 16384 3 0 3 0 1 49152 0 malloc-16384 16384 2 1 17 0 1 49152 0 malloc-2048 2048 13 11 13 0 8 49152 0 tcp_rack_pcb 896 2 52 2210 0 16 48384 0 malloc-384 384 101 19 101 0 30 46080 0 syncache 168 0 264 4 0 254 44352 0 malloc-32 32 634 752 34142 0 254 44352 0 TURNSTILE 136 210 105 210 0 62 42840 0 malloc-8192 8192 5 0 5 0 1 40960 0 pcpu-8 8 4704 416 5396 0 254 40960 0 Mountpoints 2752 2 12 2 0 4 38528 0 ripcb 424 6 84 135 0 30 38160 0 udplite_inpcb 424 0 90 277 0 30 38160 0 da_ccb 544 0 70 24235 0 16 38080 0 hostcache 64 2 565 2 0 254 36288 0 malloc-64 64 79 488 127 0 254 36288 0 malloc-64 64 81 486 19284 0 254 36288 0 16 Bucket 144 86 166 1387 0 62 36288 0 8 Bucket 80 75 375 4182 0 126 36000 0 malloc-128 128 48 231 2362 0 126 35712 0 malloc-128 128 39 240 3510 0 126 35712 0 routing nhops 256 27 108 37 0 62 34560 0 ttyoutq 256 72 63 160 0 62 34560 0 malloc-256 256 2 133 364 0 62 34560 0 SLEEPQUEUE 88 210 174 210 0 126 33792 0 malloc-32768 32768 1 0 1 0 1 32768 0 malloc-16384 16384 0 2 160 0 1 32768 0 malloc-8192 8192 3 1 29 0 1 32768 0 malloc-8192 8192 2 2 103 0 1 32768 0 malloc-8192 8192 4 0 4 0 1 32768 0 malloc-4096 4096 4 4 712 0 2 32768 0 malloc-2048 2048 2 14 14 0 8 32768 0 malloc-2048 2048 4 12 143 0 8 32768 0 malloc-2048 2048 0 16 245 0 8 32768 0 malloc-2048 2048 4 12 4 0 8 32768 0 malloc-1024 1024 4 28 1873 0 16 32768 0 malloc-1024 1024 6 26 173 0 16 32768 0 malloc-1024 1024 14 18 14 0 16 32768 0 malloc-1024 1024 19 13 19 0 16 32768 0 malloc-1024 1024 2 30 29 0 16 32768 0 malloc-512 512 1 63 120 0 30 32768 0 malloc-512 512 11 53 40 0 30 32768 0 malloc-512 512 0 64 198 0 30 32768 0 malloc-512 512 5 59 144 0 30 32768 0 malloc-512 512 8 56 262 0 30 32768 0 pcpu-64 64 494 18 494 0 254 32768 0 KNOTE 160 28 172 173742 0 62 32000 0 ttyinq 160 135 65 300 0 62 32000 0 tcp_rack_map 120 4 260 4367 0 126 31680 0 tcp_bbr_pcb 832 0 36 108 0 16 29952 0 cpuset 104 8 271 961 0 126 29016 0 tcp_inpcb ports 32 5 877 3512 0 254 28224 0 ertt 72 9 383 4687 0 126 28224 0 PWD 32 34 848 18133 0 254 28224 0 4 Bucket 48 7 581 415 0 254 28224 0 2 Bucket 32 65 817 3382 0 254 28224 0 malloc-4096 4096 3 3 20 0 2 24576 0 malloc-4096 4096 1 5 18410 0 2 24576 0 PGRP 88 35 241 90 0 126 24288 0 rl_entry 40 85 521 85 0 254 24240 0 rtentry 168 30 114 37 0 62 24192 0 malloc-16 16 593 907 27506 0 254 24000 0 malloc-384 384 3 57 287 0 30 23040 0 malloc-384 384 20 40 396 0 30 23040 0 malloc-384 384 52 8 52 0 30 23040 0 malloc-384 384 44 16 126 0 30 23040 0 udpcb 32 6 624 1805 0 254 20160 0 udp_inpcb ports 32 3 627 55 0 254 20160 0 malloc-32 32 248 382 1167 0 254 20160 0 malloc-32 32 130 500 9216 0 254 20160 0 malloc-32 32 166 464 1935 0 254 20160 0 malloc-32 32 61 569 976 0 254 20160 0 malloc-32 32 29 601 140 0 254 20160 0 malloc-32 32 37 593 2837 0 254 20160 0 AIOCB 552 0 35 83 0 16 19320 0 L VFS Cache 320 0 60 2 0 30 19200 0 epoch_record pcpu 256 4 60 4 0 62 16384 0 malloc-16384 16384 1 0 1 0 1 16384 0 malloc-16384 16384 1 0 1 0 1 16384 0 malloc-8192 8192 2 0 2 0 1 16384 0 malloc-8192 8192 2 0 2 0 1 16384 0 malloc-2048 2048 5 3 5 0 8 16384 0 malloc-1024 1024 2 14 2 0 16 16384 0 malloc-512 512 1 31 1 0 30 16384 0 malloc-512 512 0 32 1 0 30 16384 0 SMR CPU 32 7 504 7 0 254 16352 0 ipq 56 1 287 1 0 254 16128 0 vtnet_tx_hdr 24 0 668 814026 0 254 16032 0 malloc-16 16 239 761 22997 0 254 16000 0 AIO 208 0 76 20 0 62 15808 0 kenv 258 15 45 1058 0 30 15480 0 mqnode 416 3 33 3 0 30 14976 0 vmem 1856 1 7 1 0 8 14848 0 domainset 40 0 315 718 0 254 12600 0 SMR SHARED 24 7 504 7 0 254 12264 0 udplite_inpcb ports 32 0 378 1 0 254 12096 0 malloc-16 16 21 729 8672 0 254 12000 0 malloc-16 16 46 704 12086 0 254 12000 0 malloc-16 16 8 742 10 0 254 12000 0 malloc-16 16 276 474 280 0 254 12000 0 malloc-16 16 26 724 26495 0 254 12000 0 AIOLIO 272 0 42 1 0 30 11424 0 malloc-4096 4096 1 1 2 0 2 8192 0 pcpu-16 16 14 498 14 0 254 8192 0 UMA Slabs 1 176 10 12 10 0 62 3872 0 KMAP ENTRY 96 12 27 14 0 0 3744 0 pcpu-4 4 1 511 43 0 254 2048 0 FFS1 dinode 128 0 0 0 0 126 0 0 ada_ccb 272 0 0 0 0 30 0 0 swblk 136 0 0 0 0 62 0 0 swpctrie 144 0 0 0 0 62 0 0 cdg_qdiffsample 16 0 0 0 0 254 0 0 pf state scrubs 40 0 0 0 0 254 0 0 pf frag entries 40 0 0 0 0 254 0 0 pf frags 248 0 0 0 0 62 0 0 pf table entries 160 0 0 0 0 254 0 0 pf table entry counters 64 0 0 0 0 254 0 0 pf source nodes 136 0 0 0 0 254 0 0 pf state keys 88 0 0 0 0 126 0 0 pf states 312 0 0 0 0 254 0 0 pf tags 104 0 0 0 0 126 0 0 pf mtags 184 0 0 0 0 62 0 0 tfo_ccache_entries 80 0 0 0 0 126 0 0 tfo 4 0 0 0 0 254 0 0 sackhole 32 0 0 0 0 254 0 0 tcptw 72 0 0 0 0 254 0 0 tcp_log_id_node 120 0 0 0 0 126 0 0 tcp_log_id_bucket 176 0 0 0 0 62 0 0 tcpreass 48 0 0 0 0 254 0 0 sctp_asconf_ack 48 0 0 0 0 254 0 0 sctp_asconf 40 0 0 0 0 254 0 0 ripcb ports 32 0 0 0 0 254 0 0 IPsec SA lft_c 16 0 0 0 0 254 0 0 itimer 352 0 0 0 0 30 0 0 NCLNODE 608 0 0 0 0 16 0 0 mqnotifier 216 0 0 0 0 62 0 0 mvdata 64 0 0 0 0 254 0 0 mqueue 248 0 0 0 0 62 0 0 TMPFS node 224 0 0 0 0 62 0 0 LTS VFS Cache 360 0 0 0 0 30 0 0 STS VFS Cache 144 0 0 0 0 62 0 0 cryptop 280 0 0 0 0 30 0 0 linux_dma_object 32 0 0 0 0 254 0 0 linux_dma_pctrie 144 0 0 0 0 62 0 0 IOMMU_MAP_ENTRY 104 0 0 0 0 126 0 0 mbuf_jumbo_16k 16384 0 0 0 0 254 0 0 mbuf_jumbo_9k 9216 0 0 0 0 254 0 0 audit_record 1280 0 0 0 0 8 0 0 MAC labels 40 0 0 0 0 254 0 0 vnpbuf 2624 0 0 0 0 64 0 0 mdpbuf 2624 0 0 0 0 3 0 0 nfspbuf 2624 0 0 0 0 16 0 0 swwbuf 2624 0 0 0 0 8 0 0 swrbuf 2624 0 0 0 0 16 0 0 umtx_shm 88 0 0 0 0 126 0 0 umtx pi 96 0 0 0 0 126 0 0 rangeset pctrie nodes 144 0 0 0 0 62 0 0 malloc-65536 65536 0 0 0 0 1 0 0 malloc-65536 65536 0 0 0 0 1 0 0 malloc-65536 65536 0 0 0 0 1 0 0 malloc-32768 32768 0 0 0 0 1 0 0 malloc-32768 32768 0 0 0 0 1 0 0 malloc-32768 32768 0 0 0 0 1 0 0 malloc-32768 32768 0 0 0 0 1 0 0 malloc-16384 16384 0 0 0 0 1 0 0 malloc-16384 16384 0 0 0 0 1 0 0 malloc-8192 8192 0 0 0 0 1 0 0 malloc-512 512 0 0 0 0 30 0 0 pcpu-32 32 0 0 0 0 254 0 0 fakepg 104 0 0 0 0 126 0 0 UMA Hash 256 0 0 0 0 62 0 0