panic: ipreass_callout: stray callout on bucket 0xfffffe00579c6b80, 884 < 885 cpuid = 0 time = 1662884024 KDB: stack backtrace: db_trace_self_wrapper() at db_trace_self_wrapper+0xc7/frame 0xfffffe0053ab7950 kdb_backtrace() at kdb_backtrace+0xd3/frame 0xfffffe0053ab7ab0 vpanic() at vpanic+0x254/frame 0xfffffe0053ab7b90 panic() at panic+0xb5/frame 0xfffffe0053ab7c50 ipreass_callout() at ipreass_callout+0x3e9/frame 0xfffffe0053ab7cb0 softclock_call_cc() at softclock_call_cc+0x3d7/frame 0xfffffe0053ab7e80 softclock_thread() at softclock_thread+0x1ff/frame 0xfffffe0053ab7ef0 fork_exit() at fork_exit+0xd0/frame 0xfffffe0053ab7f30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe0053ab7f30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- KDB: enter: panic [ thread pid 2 tid 100029 ] Stopped at kdb_enter+0x6b: movq $0,0x27566ba(%rip) db> db> set $lines = 0 db> set $maxwidth = 0 db> show registers cs 0x20 ds 0x3b es 0x3b fs 0x13 gs 0x1b ss 0 rax 0x12 rcx 0xffffffff8172cf76 printf+0xf6 rdx 0x1 rbx 0 rsp 0xfffffe0053ab7a90 rbp 0xfffffe0053ab7ab0 rsi 0 rdi 0xffffffff8172cfd8 printf+0x158 r8 0 r9 0xffffffff r10 0 r11 0xfffffe0057321540 r12 0 r13 0xfffffe0053e8c3a0 r14 0xffffffff82b6d3a0 .str.26 r15 0xffffffff82b6d3a0 .str.26 rip 0xffffffff8171cadb kdb_enter+0x6b rflags 0x46 kdb_enter+0x6b: movq $0,0x27566ba(%rip) db> show proc Process 2 (clock) at 0xfffffe0053dda008: state: NORMAL uid: 0 gids: 0 parent: pid 0 at 0xffffffff83e167e0 ABI: null flag: 0x10000284 flag2: 0 reaper: 0xffffffff83e167e0 reapsubtree: 2 sigparent: 20 vmspace: 0xffffffff83e17780 (map 0xffffffff83e17780) (map.pmap 0xffffffff83e17840) (pmap 0xffffffff83e178b0) threads: 2 100029 Run CPU 0 [clock (0)] 100030 I [clock (1)] db> ps pid ppid pgrp uid state wmesg wchan cmd 12508 786 786 0 R (threaded) syz-executor.1 111158 RunQ syz-executor.1 114031 RunQ syz-executor.1 12507 785 785 0 R (threaded) syz-executor.0 109212 RunQ syz-executor.0 114030 RunQ syz-executor.0 12506 789 789 0 RE CPU 1 syz-executor.2 10508 1 789 0 SV uwait 0xfffffe0057889380 syz-executor.2 9964 1 789 0 S uwait 0xfffffe0057328880 syz-executor.2 9954 1 789 0 S uwait 0xfffffe0099908380 syz-executor.2 8475 1 791 0 S uwait 0xfffffe00081cd980 syz-executor.3 8469 1 791 0 S umtxn 0xfffffe0092bbeb00 syz-executor.3 8460 1 791 0 S uwait 0xfffffe0057889180 syz-executor.3 8451 1 791 0 S uwait 0xfffffe00081cd280 syz-executor.3 8257 1 786 0 SV uwait 0xfffffe00081cd780 syz-executor.1 8223 0 0 0 DL - 0xffffffff83f60500 [soaiod4] 8222 0 0 0 DL - 0xffffffff83f60500 [soaiod3] 8221 0 0 0 DL - 0xffffffff83f60500 [soaiod2] 8220 0 0 0 DL - 0xffffffff83f60500 [soaiod1] 8205 0 0 0 DL aiordy 0xfffffe00998f6010 [aiod4] 8204 0 0 0 DL aiordy 0xfffffe00998f6ac0 [aiod3] 8203 0 0 0 DL aiordy 0xfffffe00998f6568 [aiod2] 8202 0 0 0 DL aiordy 0xfffffe0092745ab8 [aiod1] 8178 1 791 0 SV uwait 0xfffffe0092bbea00 syz-executor.3 8169 1 791 0 SV uwait 0xfffffe0057328980 syz-executor.3 8160 1 791 0 SV uwait 0xfffffe0057889d00 syz-executor.3 8151 1 791 0 SV uwait 0xfffffe0057888f00 syz-executor.3 8142 1 791 0 SV uwait 0xfffffe0057889f00 syz-executor.3 8133 1 791 0 SV uwait 0xfffffe0057325080 syz-executor.3 8124 1 791 0 SV uwait 0xfffffe0057889c00 syz-executor.3 5494 1 5494 65 Ss select 0xfffffe00998dcdc0 dhclient 5038 1 5038 0 Ss select 0xfffffe009a1dd640 dhclient 5029 1 5029 0 Ss select 0xfffffe009a1dd840 dhclient 4978 1 4978 65 Ss select 0xfffffe009a1c41c0 dhclient 2537 1 2537 0 Ss select 0xfffffe00998d2140 dhclient 2534 1 2534 0 Ss select 0xfffffe00998d21c0 dhclient 2515 1 2515 65 Ss select 0xfffffe00079848c0 dhclient 1910 1 1910 0 Ss select 0xfffffe00586151c0 dhclient 1907 1 1907 0 Ss select 0xfffffe00998cf1c0 dhclient 1884 1 1884 65 Ss select 0xfffffe0092787840 dhclient 1277 1 1277 0 Ss select 0xfffffe00998cf6c0 dhclient 1274 1 1274 0 Ss select 0xfffffe00998dd2c0 dhclient 791 774 791 0 Rs syz-executor.3 789 774 789 0 Rs syz-executor.2 786 774 786 0 Rs syz-executor.1 785 774 785 0 Ss nanslp 0xffffffff83e40941 syz-executor.0 774 772 772 0 S (threaded) syz-fuzzer 100085 S wait 0xfffffe0056fa3008 syz-fuzzer 100114 S uwait 0xfffffe0057325380 syz-fuzzer 100115 S uwait 0xfffffe0057325480 syz-fuzzer 100116 S kqread 0xfffffe0058ac7000 syz-fuzzer 100117 S uwait 0xfffffe0057325780 syz-fuzzer 100118 S uwait 0xfffffe0057325700 syz-fuzzer 100119 S wait 0xfffffe0056fa3008 syz-fuzzer 100120 S uwait 0xfffffe0057889a00 syz-fuzzer 100125 S wait 0xfffffe0056fa3008 syz-fuzzer 100126 S wait 0xfffffe0056fa3008 syz-fuzzer 100127 S uwait 0xfffffe00081cdc80 syz-fuzzer 100136 S uwait 0xfffffe00081cd480 syz-fuzzer 100169 S uwait 0xfffffe0057889900 syz-fuzzer 772 770 772 0 Ss pause 0xfffffe0092747620 csh 770 688 770 0 Ss select 0xfffffe0007981140 sshd 754 1 754 0 Ss+ ttyin 0xfffffe0056f56cb0 getty 753 1 753 0 Ss+ ttyin 0xfffffe0057a8b8b0 getty 752 1 752 0 Ss+ ttyin 0xfffffe0057a8bcb0 getty 751 1 751 0 Ss+ ttyin 0xfffffe0057a8c0b0 getty 750 1 750 0 Ss+ ttyin 0xfffffe0057a8c4b0 getty 749 1 749 0 Ss+ ttyin 0xfffffe0057a8c8b0 getty 748 1 748 0 Ss+ ttyin 0xfffffe0057a8ccb0 getty 747 1 747 0 Ss+ ttyin 0xfffffe0057a8d0b0 getty 746 1 746 0 Ss+ ttyin 0xfffffe0057a8d4b0 getty 692 1 692 0 Ss nanslp 0xffffffff83e40941 cron 688 1 688 0 Ss select 0xfffffe00079804c0 sshd 501 1 501 0 Ss select 0xfffffe00079805c0 syslogd 430 1 430 0 Ss select 0xfffffe0007980740 devd 429 1 429 65 Ss select 0xfffffe00079817c0 dhclient 344 1 344 0 Ss select 0xfffffe00079806c0 dhclient 341 1 341 0 Ss select 0xfffffe00079810c0 dhclient 17 0 0 0 DL syncer 0xffffffff83f65de0 [syncer] 16 0 0 0 DL vlruwt 0xfffffe0056fa4010 [vnlru] 15 0 0 0 DL (threaded) [bufdaemon] 100079 D psleep 0xffffffff83f64400 [bufdaemon] 100082 D - 0xffffffff83212100 [bufspacedaemon-0] 100094 D sdflush 0xfffffe00574900e8 [/ worker] 9 0 0 0 DL psleep 0xffffffff83f97e80 [vmdaemon] 8 0 0 0 DL (threaded) [pagedaemon] 100077 D psleep 0xffffffff83f8bd38 [dom0] 100080 D launds 0xffffffff83f8bd44 [laundry: dom0] 100081 D umarcl 0xffffffff81e5df00 [uma] 7 0 0 0 DL - 0xffffffff83bfcde8 [rand_harvestq] 6 0 0 0 DL pftm 0xffffffff84b1a330 [pf purge] 5 0 0 0 DL waiting 0xffffffff84695640 [sctp_iterator] 4 0 0 0 DL (threaded) [cam] 100044 D - 0xffffffff83a9f340 [doneq0] 100045 D - 0xffffffff83a9f2c0 [async] 100076 D - 0xffffffff83a9f140 [scanner] 14 0 0 0 DL seqstat 0xfffffe0056efcc88 [sequencer 00] 3 0 0 0 DL (threaded) [crypto] 100040 D crypto_ 0xffffffff83f87460 [crypto] 100041 D crypto_ 0xfffffe0053f6c030 [crypto returns 0] 100042 D crypto_ 0xfffffe0053f6c080 [crypto returns 1] 13 0 0 0 DL (threaded) [geom] 100035 D - 0xffffffff83e15dc0 [g_event] 100036 D - 0xffffffff83e15de0 [g_up] 100037 D - 0xffffffff83e15e00 [g_down] 2 0 0 0 RL (threaded) [clock] 100029 Run CPU 0 [clock (0)] 100030 I [clock (1)] 12 0 0 0 WL (threaded) [intr] 100013 I [swi6: Giant taskq] 100015 I [swi5: fast taskq] 100018 I [swi6: task queue] 100031 I [swi1: netisr 0] 100032 I [swi1: hpts] 100033 I [swi1: hpts] 100046 I [irq24: virtio_pci0] 100047 I [irq25: virtio_pci0] 100048 I [irq26: virtio_pci0] 100049 I [irq27: virtio_pci0] 100050 I [irq28: virtio_pci1] 100051 I [irq29: virtio_pci1] 100052 I [irq30: virtio_pci1] 100053 I [irq31: virtio_pci1] 100054 I [irq32: virtio_pci1] 100059 I [irq33: virtio_pci2] 100060 I [irq34: virtio_pci2] 100061 I [irq35: virtio_pci2] 100063 I [irq1: atkbd0] 100064 I [irq12: psm0] 100065 I [swi0: uart uart++] 100069 I [swi1: pf send] 11 0 0 0 RL (threaded) [idle] 100003 CanRun [idle: cpu0] 100004 CanRun [idle: cpu1] 1 0 1 0 SLs wait 0xfffffe0053ddb010 [init] 10 0 0 0 DL audit_w 0xffffffff83f88000 [audit] 0 0 0 0 DLs (threaded) [kernel] 100000 D swapin 0xffffffff83e167e0 [swapper] 100005 D - 0xfffffe0053c82700 [softirq_0] 100006 D - 0xfffffe0053c82600 [softirq_1] 100007 D - 0xfffffe0053c82500 [if_io_tqg_0] 100008 D - 0xfffffe0053c82400 [if_io_tqg_1] 100009 D - 0xfffffe0053c82300 [if_config_tqg_0] 100010 D - 0xfffffe000817bd00 [in6m_free taskq] 100011 D - 0xfffffe000817bc00 [linuxkpi_irq_wq] 100012 D - 0xfffffe000817bb00 [deferred_unmount ta] 100014 D - 0xfffffe000817b900 [thread taskq] 100016 D - 0xfffffe000817b700 [kqueue_ctx taskq] 100017 D - 0xfffffe000817b600 [pci_hp taskq] 100019 D - 0xfffffe000817b400 [inm_free taskq] 100020 D - 0xfffffe000817b300 [aiod_kick taskq] 100021 D - 0xfffffe000817b200 [linuxkpi_short_wq_0] 100022 D - 0xfffffe000817b200 [linuxkpi_short_wq_1] 100023 D - 0xfffffe000817b200 [linuxkpi_short_wq_2] 100024 D - 0xfffffe000817b200 [linuxkpi_short_wq_3] 100025 D - 0xfffffe000817b100 [linuxkpi_long_wq_0] 100026 D - 0xfffffe000817b100 [linuxkpi_long_wq_1] 100027 D - 0xfffffe000817b100 [linuxkpi_long_wq_2] 100028 D - 0xfffffe000817b100 [linuxkpi_long_wq_3] 100034 D - 0xfffffe000817b000 [firmware taskq] 100038 D - 0xfffffe000817ae00 [crypto_0] 100039 D - 0xfffffe000817ae00 [crypto_1] 100055 D - 0xfffffe000817ac00 [vtnet0 rxq 0] 100056 D - 0xfffffe000817ab00 [vtnet0 txq 0] 100057 D - 0xfffffe000817aa00 [vtnet0 rxq 1] 100058 D - 0xfffffe000817a900 [vtnet0 txq 1] 100062 D vtbslp 0xfffffe0007982100 [virtio_balloon] 100066 D - 0xffffffff82b73401 [deadlkres] 100071 D - 0xfffffe0057901c00 [acpi_task_0] 100072 D - 0xfffffe0057901c00 [acpi_task_1] 100073 D - 0xfffffe0057901c00 [acpi_task_2] 100074 D - 0xfffffe000817c100 [mca taskq] 100075 D - 0xfffffe000817ad00 [CAM taskq] db> show all locks Process 12507 (syz-executor.0) thread 0xfffffe009a2383a0 (114030) exclusive lockmgr bufwait (bufwait) r = 0 (0xfffffe0007a75900) locked @ /syzkaller/managers/i386/kernel/sys/kern/vfs_bio.c:1725 exclusive lockmgr ufs (ufs) r = 0 (0xfffffe0099766930) locked @ /syzkaller/managers/i386/kernel/sys/kern/vfs_vnops.c:1166 Process 2 (clock) thread 0xfffffe0053e8c3a0 (100029) exclusive sleep mutex IP reassembly (IP reassembly) r = 0 (0xfffffe00579c6b90) locked @ /syzkaller/managers/i386/kernel/sys/kern/kern_mutex.c:211 db> show malloc Type InUse MemUse Requests pf_hash 5 11524K 5 tcp_hpts 7 4801K 7 devbuf 4217 4323K 4245 sysctloid 35217 2075K 35288 vtbuf 24 1968K 46 kobj 329 1316K 493 newblk 41 1034K 21952 vfscache 3 1025K 3 pcb 59 577K 8787 inodedep 16 518K 13461 ufs_quota 1 512K 1 vfs_hash 1 512K 1 callout 2 512K 2 intr 4 472K 4 subproc 179 354K 12609 filedesc 44 345K 23021 vmem 3 274K 6 acpica 1674 184K 58126 tidhash 3 141K 3 linker 358 134K 386 pagedep 9 130K 11532 tfo_ccache 1 128K 1 IP reass 1 128K 1 vnet_data 1 112K 1 DEVFS1 109 109K 126 sem 4 106K 4 BPF 46 88K 92 bus 998 82K 5213 mtx_pool 2 72K 2 syncache 1 68K 1 module 516 65K 516 acpitask 1 64K 1 ddb_capture 1 64K 1 umtx 418 53K 418 kdtrace 258 50K 26541 temp 34 33K 6341 DEVFS3 128 32K 138 hostcache 1 32K 1 shm 1 32K 21 msg 4 30K 4 gtaskqueue 18 26K 18 kbdmux 6 22K 6 ifaddr 70 20K 72 DEVFS_RULE 56 20K 56 sctp_stro 18 18K 791 ufs_mount 4 17K 5 proc 3 17K 3 tty 16 16K 16 routetbl 130 16K 410 ithread 97 16K 97 bus-sc 34 15K 1682 sctp_atcl 36 14K 5222 KTRACE 100 13K 107 eventhandler 151 13K 151 lltable 41 13K 103 ifnet 7 13K 7 ether_multi 152 13K 162 kenv 95 12K 95 rman 88 11K 431 GEOM 61 11K 483 CAM queue 5 11K 1528 in6_multi 65 9K 65 bmsafemap 2 9K 12725 UART 12 9K 12 devstat 4 9K 4 ksem 1 8K 52 sctp_timw 32 8K 32 rpc 2 8K 2 shmfd 1 8K 1 pfs_vncache 1 8K 1 cred 30 8K 294 pfs_nodes 20 8K 20 pf_osfp 65 8K 65 audit_evclass 237 8K 296 kqueue 81 8K 12582 taskqueue 63 7K 63 sglist 5 7K 5 CAM DEV 3 6K 510 plimit 24 6K 486 pwddesc 78 5K 12509 ufs_dirhash 24 5K 24 pf_ifnet 11 5K 23 UMA 271 5K 271 session 35 5K 55 freework 18 5K 15615 DEVFSP 68 5K 2239 vt 11 5K 11 memdesc 1 4K 1 MCA 32 4K 32 evdev 4 4K 4 proc-args 106 4K 13888 lockf 33 4K 95 acpisem 28 4K 28 selfd 55 4K 174109 hhook 15 4K 17 kcovinfo 52 4K 52 terminal 11 3K 11 select 19 3K 310 uidinfo 4 3K 19 sctp_atky 54 3K 6106 local_apic 1 2K 1 io_apic 1 2K 1 fpukern_ctx 2 2K 2 ipsec-saq 2 2K 2 ip6ndp 12 2K 14 Unitno 31 2K 65 sctp_ifa 13 2K 14 CAM XPT 22 2K 543 msi 12 2K 12 in_multi 6 2K 8 dirrem 5 2K 12359 freeblks 5 2K 12098 ipsecpolicy 2 2K 2 acpidev 20 2K 20 clone 9 2K 9 tun 7 2K 7 softdep 1 1K 1 mkdir 8 1K 23020 sahead 1 1K 1 secasvar 1 1K 1 nhops 6 1K 8 vnodemarker 2 1K 64 NFSD session 1 1K 1 CAM periph 4 1K 271 ipsec 3 1K 3 sctp_ifn 6 1K 14 freefile 6 1K 12335 mld 6 1K 6 igmp 6 1K 6 toponodes 6 1K 6 isadev 6 1K 6 mount 16 1K 89 pci_link 10 1K 10 inpcbpolicy 21 1K 7723 diradd 5 1K 12402 crypto 4 1K 4 CC Mem 10 1K 7378 encap_export_host 12 1K 12 procdesc 5 1K 18 sctp_athm 36 1K 5302 sctp_map 36 1K 1566 pf_rule 4 1K 67 newdirblk 4 1K 11510 pfil 4 1K 4 cdev 2 1K 2 osd 14 1K 7389 chacha20random 1 1K 1 sigio 6 1K 8 NFSD lckfile 1 1K 1 NFSD V4client 1 1K 1 DEVFS 9 1K 10 indirdep 1 1K 6163 vnodes 1 1K 1 CAM SIM 2 1K 2 feeder 7 1K 7 tcpfunc 3 1K 3 loginclass 3 1K 6 prison 6 1K 6 lkpikmalloc 5 1K 6 aesni_data 2 1K 2 cryptodev 2 1K 2094 nexusdev 8 1K 8 apmdev 1 1K 1 atkbddev 2 1K 2 CAM dev queue 2 1K 2 CAM I/O Scheduler 1 1K 1 CAM path 4 1K 1034 aio 4 1K 10 iov 2 1K 24868 pmchooks 1 1K 1 soname 4 1K 23506 filecaps 5 1K 120 sctp_vrf 1 1K 1 vnet 1 1K 1 entropy 2 1K 49 pmc 1 1K 1 acpiintr 1 1K 1 cpus 2 1K 2 vnet_data_free 1 1K 1 Per-cpu 1 1K 1 p1003.1b 1 1K 1 filemon 0 0K 137 mqdata 0 0K 0 pf_table 0 0K 0 pf_altq 0 0K 0 pf_temp 0 0K 0 tcp_do 0 0K 0 tcp_fsb 0 0K 1785 ipcomp 0 0K 0 esp 0 0K 0 ah 0 0K 0 sctp_mcore 0 0K 0 sctp_socko 0 0K 2487 sctp_iter 0 0K 27 sctp_mvrf 0 0K 0 sctp_cpal 0 0K 16 sctp_cmsg 0 0K 0 sctp_stre 0 0K 0 sctp_athi 0 0K 0 sctp_a_it 0 0K 11 sctp_aadr 0 0K 2 sctp_stri 0 0K 162 NFSCL layrecall 0 0K 0 NFSCL session 0 0K 0 madt_table 0 0K 2 smartpqi 0 0K 0 NFSCL sockreq 0 0K 0 NFSCL devinfo 0 0K 0 NFSCL flayout 0 0K 0 ixl 0 0K 0 NFSCL layout 0 0K 0 NFSD rollback 0 0K 0 NFSCL diroff 0 0K 0 NEWdirectio 0 0K 0 NEWNFSnode 0 0K 0 NFSCL lck 0 0K 0 NFSCL lckown 0 0K 0 ice-resmgr 0 0K 0 ice-osdep 0 0K 0 ice 0 0K 0 iavf 0 0K 0 axgbe 0 0K 0 NFSCL client 0 0K 0 NFSCL deleg 0 0K 0 NFSCL open 0 0K 0 NFSCL owner 0 0K 0 NFS fh 0 0K 0 NFS req 0 0K 0 NFSD usrgroup 0 0K 0 NFSD string 0 0K 0 NFSD V4lock 0 0K 0 xen_intr 0 0K 0 NFSD V4state 0 0K 0 xen_hvm 0 0K 0 legacydrv 0 0K 0 bounce 0 0K 0 busdma 0 0K 0 qpidrv 0 0K 0 NFSD srvcache 0 0K 0 msdosfs_fat 0 0K 0 dmar_idpgtbl 0 0K 0 dmar_dom 0 0K 0 dmar_ctx 0 0K 0 msdosfs_mount 0 0K 0 msdosfs_node 0 0K 0 isci 0 0K 0 iommu_dmamap 0 0K 0 hyperv_socket 0 0K 0 bxe_ilt 0 0K 0 xenbus 0 0K 0 DEVFS4 0 0K 0 vm_fictitious 0 0K 0 DEVFS2 0 0K 0 gntdev 0 0K 0 privcmd_dev 0 0K 0 evtchn_dev 0 0K 0 xenstore 0 0K 0 scsi_pass 0 0K 0 ciss_data 0 0K 0 xnb 0 0K 0 xen_acpi 0 0K 0 xbbd 0 0K 0 xbd 0 0K 0 UMAHash 0 0K 0 Balloon 0 0K 0 sysmouse 0 0K 0 vm_pgdata 0 0K 0 jblocks 0 0K 0 savedino 0 0K 13382 sentinel 0 0K 0 jfsync 0 0K 0 jtrunc 0 0K 0 sbdep 0 0K 30 jsegdep 0 0K 0 jseg 0 0K 0 jfreefrag 0 0K 0 jfreeblk 0 0K 0 jnewblk 0 0K 0 jmvref 0 0K 0 jremref 0 0K 0 jaddref 0 0K 0 freedep 0 0K 0 freefrag 0 0K 2 allocindir 0 0K 0 allocdirect 0 0K 0 ufs_trim 0 0K 0 mactemp 0 0K 0 audit_trigger 0 0K 0 audit_pipe_presel 0 0K 0 audit_pipeent 0 0K 0 audit_pipe 0 0K 0 audit_evname 0 0K 0 audit_bsm 0 0K 0 audit_gidset 0 0K 0 audit_text 0 0K 0 audit_path 0 0K 0 audit_data 0 0K 0 audit_cred 0 0K 0 vtfont 0 0K 0 BACKLIGHT 0 0K 0 ath_hal 0 0K 0 athdev 0 0K 0 ata_pci 0 0K 0 ata_dma 0 0K 0 ata_generic 0 0K 0 pvscsi 0 0K 0 scsi_da 0 0K 69 ata_da 0 0K 0 scsi_ch 0 0K 0 scsi_cd 0 0K 0 ktls_ocf 0 0K 0 AHCI driver 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5E_TLS_RX 0 0K 0 MLX5EEPROM 0 0K 0 MLX5E_TLS 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EN 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5DUMP 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 seq_file 0 0K 0 lkpiskb 0 0K 0 radix 0 0K 0 idr 0 0K 0 lkpindev 0 0K 0 lkpifw 0 0K 0 lkpi80211 0 0K 0 NLM 0 0K 0 ipsec-spdcache 0 0K 0 ipsec-reg 0 0K 0 ipsec-misc 0 0K 0 ipsecrequest 0 0K 0 ip6opt 0 0K 540 ip6_msource 0 0K 0 ip6_moptions 0 0K 0 in6_mfilter 0 0K 0 frag6 0 0K 7 tcplog 0 0K 0 tcp_hwpace 0 0K 0 USBdev 0 0K 0 USB 0 0K 0 LRO 0 0K 0 ip_msource 0 0K 0 ip_moptions 0 0K 3 in_mfilter 0 0K 0 ipid 0 0K 0 80211scan 0 0K 0 80211ratectl 0 0K 0 80211power 0 0K 0 80211nodeie 0 0K 0 80211node 0 0K 0 80211mesh_gt 0 0K 0 80211mesh_rt 0 0K 0 80211perr 0 0K 0 80211prep 0 0K 0 80211preq 0 0K 0 80211dfs 0 0K 0 80211crypto 0 0K 0 80211vap 0 0K 0 iflib 0 0K 0 vlan 0 0K 0 gif 0 0K 0 ifdescr 0 0K 0 zlib 0 0K 0 fadvise 0 0K 0 VN POLL 0 0K 0 agp 0 0K 0 statfs 0 0K 11710 namei_tracker 0 0K 4 export_host 0 0K 0 cl_savebuf 0 0K 2 nvme_da 0 0K 0 acpipwr 0 0K 0 acpi_perf 0 0K 0 twsbuf 0 0K 0 twe_commands 0 0K 0 tcp_log_dev 0 0K 291 midi buffers 0 0K 0 mixer 0 0K 0 ac97 0 0K 0 hdacc 0 0K 0 hdac 0 0K 0 hdaa 0 0K 0 acpicmbat 0 0K 0 SIIS driver 0 0K 0 CAM CCB 0 0K 523 biobuf 0 0K 0 lio 0 0K 0 acl 0 0K 0 mbuf_tag 0 0K 0 ktls 0 0K 0 PUC 0 0K 0 ppbusdev 0 0K 0 agtiapi_MemAlloc malloc 0 0K 0 osti_cacheable 0 0K 0 tempbuff 0 0K 0 tempbuff 0 0K 0 ag_tgt_map_t malloc 0 0K 0 ag_slr_map_t malloc 0 0K 0 lDevFlags * malloc 0 0K 0 tiDeviceHandle_t * malloc 0 0K 0 ag_portal_data_t malloc 0 0K 0 ag_device_t malloc 0 0K 0 STLock malloc 0 0K 0 CCB List 0 0K 0 sr_iov 0 0K 0 OCS 0 0K 0 OCS 0 0K 0 nvme 0 0K 0 nvd 0 0K 0 netmap 0 0K 0 mwldev 0 0K 0 MVS driver 0 0K 0 CAM ccb queue 0 0K 0 accf 0 0K 0 pts 0 0K 0 ioctlops 0 0K 419 eventfd 0 0K 208 Witness 0 0K 0 stack 0 0K 0 mrsasbuf 0 0K 0 mpt_user 0 0K 0 mps_user 0 0K 0 MPSSAS 0 0K 0 mps 0 0K 0 sbuf 0 0K 288 mpr_user 0 0K 0 firmware 0 0K 0 compressor 0 0K 0 MPRSAS 0 0K 0 SWAP 0 0K 0 mpr 0 0K 0 mfibuf 0 0K 0 sysctltmp 0 0K 869 sysctl 0 0K 3 md_sectors 0 0K 0 ekcd 0 0K 0 dumper 0 0K 0 sendfile 0 0K 0 rctl 0 0K 0 md_disk 0 0K 0 malodev 0 0K 0 LED 0 0K 0 ix_sriov 0 0K 0 cache 0 0K 0 aacraidcam 0 0K 0 prison_racct 0 0K 0 Fail Points 0 0K 0 filedesc_to_leader 0 0K 0 pwd 0 0K 0 tty console 0 0K 0 ix 0 0K 0 ipsbuf 0 0K 0 aacraid_buf 0 0K 0 aaccam 0 0K 0 boottrace 0 0K 0 aacbuf 0 0K 0 zstd 0 0K 0 XZ_DEC 0 0K 0 nvlist 0 0K 0 SCSI ENC 0 0K 0 SCSI sa 0 0K 0 isofs_node 0 0K 0 isofs_mount 0 0K 0 tr_raid5_data 0 0K 0 tr_raid1e_data 0 0K 0 tr_raid1_data 0 0K 0 tr_raid0_data 0 0K 0 tr_concat_data 0 0K 0 md_sii_data 0 0K 0 md_promise_data 0 0K 0 md_nvidia_data 0 0K 0 md_jmicron_data 0 0K 0 md_intel_data 0 0K 0 md_ddf_data 0 0K 0 raid_data 0 0K 72 geom_flashmap 0 0K 0 tmpfs dir 0 0K 0 tmpfs name 0 0K 0 tmpfs mount 0 0K 0 NFS FHA 0 0K 0 newnfsmnt 0 0K 0 newnfsclient_req 0 0K 0 db> show uma Zone Size Used Free Requests Sleeps Bucket Total Mem XFree ertt_txseginfo 40 1065233 2337 1374932 0 254 42702800 0 mbuf_jumbo_page 4096 8322 1076 1514829 0 254 38494208 0 mbuf 256 8742 921 3803083 0 254 2473728 0 RADIX NODE 144 15634 265 358818 0 62 2289456 0 tcp_log 416 0 5337 47173 0 254 2220192 0 pbuf 2624 0 794 0 0 2 2083456 0 malloc-4096 4096 443 5 13595 0 2 1835008 0 BUF TRIE 144 175 11585 13176 0 62 1693440 0 vmem btag 56 29173 98 29173 0 254 1639176 0 malloc-384 384 4234 26 4296 0 30 1635840 0 mbuf_cluster 2048 762 0 762 0 254 1560576 0 malloc-128 128 11671 202 11805 0 126 1519744 0 UMA Slabs 0 112 10851 15 10851 0 126 1216992 0 sctp_asoc 2256 18 492 783 0 254 1150560 0 malloc-384 384 53 2017 13566 0 30 794880 0 FFS inode 1160 536 24 12871 0 8 649600 0 VM OBJECT 264 2294 106 195434 0 30 633600 0 sctp_ep 1208 18 492 4357 0 254 616080 0 tcpcb 1104 10 501 7378 0 254 564144 0 malloc-256 256 132 1983 47175 0 62 541440 0 256 Bucket 2048 206 38 14765 0 8 499712 0 malloc-16384 16384 24 6 11671 0 1 491520 0 socket 960 46 462 13727 0 254 487680 0 malloc-65536 65536 7 0 7 0 1 458752 0 tcp_bbr_map 128 505 2657 46310 0 126 404736 0 lkpimm 168 1 2327 1 0 62 391104 0 lkpicurr 168 2 2326 2 0 62 391104 0 THREAD 1824 178 31 14031 0 8 381216 0 sctp_raddr 736 18 499 1045 0 254 380512 0 malloc-128 128 649 2141 13441 0 126 357120 0 MAP ENTRY 96 3096 306 445340 0 126 326592 0 VNODE 448 576 90 12913 0 30 298368 0 malloc-64 64 4091 256 7553 0 254 278208 0 malloc-16 16 14633 117 20683 0 254 236000 0 DEVCTL 1024 0 220 152 0 0 225280 0 malloc-256 256 145 725 42250 0 62 222720 0 malloc-32768 32768 0 6 5860 0 1 196608 0 mbuf_packet 256 47 715 14315 0 254 195072 0 malloc-128 128 1266 253 42973 0 126 194432 0 malloc-32 32 5400 522 20380 0 254 189504 0 UMA Zones 768 243 1 243 0 16 187392 0 FPU_save_area 832 180 45 17665 0 16 187200 0 VMSPACE 2560 54 12 12481 0 4 168960 0 FFS2 dinode 256 536 94 12871 0 62 161280 0 S VFS Cache 104 1067 337 13932 0 126 146016 0 PROC 1368 78 21 12508 0 8 135432 0 128 Bucket 1024 97 34 1205 0 16 134144 0 malloc-65536 65536 0 2 136 0 1 131072 0 malloc-65536 65536 0 2 266 0 1 131072 0 malloc-1024 1024 109 19 126 0 16 131072 0 unpcb 256 20 490 1602 0 254 130560 0 malloc-8192 8192 9 6 274 0 1 122880 0 ksiginfo 112 87 957 3938 0 126 116928 0 filedesc0 1072 78 20 12509 0 8 105056 0 64 Bucket 512 126 74 12237 0 30 102400 0 malloc-256 256 224 151 15011 0 62 96000 0 UMA Kegs 384 229 4 229 0 30 89472 0 malloc-64 64 597 726 30622 0 254 84672 0 malloc-128 128 451 200 1528 0 126 83328 0 malloc-2048 2048 5 35 639 0 8 81920 0 malloc-2048 2048 21 19 4454 0 8 81920 0 32 Bucket 256 140 175 13108 0 62 80640 0 sctp_readq 152 0 520 13 0 254 79040 0 sctp_chunk 152 14 506 323 0 254 79040 0 g_bio 408 0 180 89668 0 30 73440 0 malloc-128 128 300 227 312 0 126 67456 0 DIRHASH 1024 35 29 35 0 16 65536 0 malloc-65536 65536 1 0 1 0 1 65536 0 malloc-65536 65536 0 1 1 0 1 65536 0 malloc-65536 65536 1 0 1 0 1 65536 0 malloc-32768 32768 0 2 120 0 1 65536 0 malloc-32768 32768 2 0 2 0 1 65536 0 malloc-8192 8192 7 1 8 0 1 65536 0 sctp_stream_msg_out 112 10 530 45 0 254 60480 0 tcp_inpcb 424 10 116 7378 0 30 53424 0 malloc-64 64 175 644 174290 0 254 52416 0 malloc-128 128 142 261 13930 0 126 51584 0 malloc-128 128 111 292 23481 0 126 51584 0 malloc-256 256 136 59 150 0 62 49920 0 malloc-256 256 83 112 12084 0 62 49920 0 NAMEI 1024 0 48 67860 0 16 49152 0 malloc-16384 16384 3 0 3 0 1 49152 0 malloc-16384 16384 1 2 6 0 1 49152 0 malloc-1024 1024 34 14 47 0 16 49152 0 malloc-1024 1024 23 25 1313 0 16 49152 0 tcp_rack_pcb 896 0 54 1785 0 16 48384 0 pipe 728 23 43 447 0 16 48048 0 tcp_rack_map 120 0 396 6847 0 126 47520 0 malloc-384 384 63 57 5249 0 30 46080 0 tcp_bbr_pcb 832 5 49 4444 0 16 44928 0 syncache 168 0 264 4 0 254 44352 0 TURNSTILE 136 210 105 210 0 62 42840 0 clpbuf 2624 0 16 26 0 16 41984 0 malloc-8192 8192 5 0 5 0 1 40960 0 malloc-4096 4096 4 6 11798 0 2 40960 0 pcpu-8 8 4720 400 5580 0 254 40960 0 Mountpoints 2752 2 12 2 0 4 38528 0 udp_inpcb 424 6 84 274 0 30 38160 0 hostcache 64 3 564 3 0 254 36288 0 PWD 32 35 1099 11584 0 254 36288 0 malloc-64 64 8 559 43 0 254 36288 0 malloc-64 64 238 329 2429 0 254 36288 0 malloc-64 64 185 382 32492 0 254 36288 0 malloc-64 64 491 76 696 0 254 36288 0 malloc-64 64 31 536 430 0 254 36288 0 16 Bucket 144 67 185 1213 0 62 36288 0 8 Bucket 80 46 404 2987 0 126 36000 0 malloc-128 128 18 261 22 0 126 35712 0 routing nhops 256 27 108 34 0 62 34560 0 ttyoutq 256 72 63 160 0 62 34560 0 malloc-384 384 53 37 393 0 30 34560 0 malloc-256 256 18 117 510 0 62 34560 0 malloc-256 256 56 79 1127 0 62 34560 0 malloc-256 256 25 110 4711 0 62 34560 0 SLEEPQUEUE 88 210 174 210 0 126 33792 0 malloc-32768 32768 1 0 1 0 1 32768 0 malloc-16384 16384 1 1 16 0 1 32768 0 malloc-4096 4096 8 0 8 0 2 32768 0 malloc-2048 2048 7 9 7 0 8 32768 0 malloc-2048 2048 5 11 146 0 8 32768 0 malloc-2048 2048 7 9 550 0 8 32768 0 malloc-2048 2048 7 9 199 0 8 32768 0 malloc-1024 1024 6 26 209 0 16 32768 0 malloc-1024 1024 4 28 5 0 16 32768 0 malloc-1024 1024 6 26 1264 0 16 32768 0 malloc-512 512 5 59 240 0 30 32768 0 malloc-512 512 10 54 172 0 30 32768 0 malloc-512 512 11 53 51 0 30 32768 0 pcpu-64 64 494 18 494 0 254 32768 0 KNOTE 160 28 172 133899 0 62 32000 0 ttyinq 160 135 65 300 0 62 32000 0 Files 80 227 173 37545 0 126 32000 0 cpuset 104 7 272 125 0 126 29016 0 sctp_laddr 48 0 588 113 0 254 28224 0 tcp_inpcb ports 32 2 880 6463 0 254 28224 0 malloc-32 32 308 574 11997 0 254 28224 0 4 Bucket 48 6 582 295 0 254 28224 0 2 Bucket 32 56 826 2963 0 254 28224 0 AIO 208 1 132 1588 0 62 27664 0 ripcb 424 5 58 50 0 30 26712 0 udplite_inpcb 424 0 63 21 0 30 26712 0 da_ccb 544 0 49 22517 0 16 26656 0 malloc-4096 4096 4 2 91 0 2 24576 0 PGRP 88 35 241 58 0 126 24288 0 rl_entry 40 89 517 89 0 254 24240 0 rtentry 168 30 114 34 0 62 24192 0 malloc-384 384 20 40 3158 0 30 23040 0 domainset 40 0 567 109 0 254 22680 0 udpcb 32 6 624 295 0 254 20160 0 udp_inpcb ports 32 3 627 36 0 254 20160 0 ertt 72 10 270 7378 0 126 20160 0 malloc-32 32 6 624 132 0 254 20160 0 malloc-32 32 176 454 214 0 254 20160 0 malloc-32 32 41 589 737 0 254 20160 0 malloc-32 32 65 565 8128 0 254 20160 0 malloc-32 32 50 580 1343 0 254 20160 0 AIOCB 552 0 35 1589 0 16 19320 0 epoch_record pcpu 256 4 60 4 0 62 16384 0 malloc-8192 8192 1 1 3 0 1 16384 0 malloc-8192 8192 1 1 2 0 1 16384 0 malloc-4096 4096 0 4 5 0 2 16384 0 malloc-2048 2048 1 7 1 0 8 16384 0 malloc-1024 1024 8 8 9 0 16 16384 0 malloc-512 512 0 32 22 0 30 16384 0 malloc-512 512 0 32 606 0 30 16384 0 SMR CPU 32 7 504 7 0 254 16352 0 ipq 56 1 287 4 0 254 16128 0 vtnet_tx_hdr 24 0 668 971968 0 254 16032 0 malloc-16 16 308 692 510 0 254 16000 0 kenv 258 15 45 1054 0 30 15480 0 mqnode 416 3 33 3 0 30 14976 0 vmem 1856 1 7 1 0 8 14848 0 SMR SHARED 24 7 504 7 0 254 12264 0 malloc-32 32 80 298 93 0 254 12096 0 malloc-16 16 22 728 7398 0 254 12000 0 malloc-16 16 18 732 132 0 254 12000 0 malloc-16 16 31 719 33 0 254 12000 0 malloc-16 16 185 565 2994 0 254 12000 0 malloc-16 16 15 735 49356 0 254 12000 0 malloc-16 16 121 629 14856 0 254 12000 0 L VFS Cache 320 0 36 1 0 30 11520 0 malloc-384 384 1 29 3 0 30 11520 0 malloc-8192 8192 1 0 1 0 1 8192 0 malloc-8192 8192 1 0 1 0 1 8192 0 malloc-4096 4096 0 2 2 0 2 8192 0 pcpu-16 16 14 498 14 0 254 8192 0 UMA Slabs 1 176 10 12 10 0 62 3872 0 KMAP ENTRY 96 12 27 14 0 0 3744 0 pcpu-4 4 1 511 64 0 254 2048 0 FFS1 dinode 128 0 0 0 0 126 0 0 ada_ccb 272 0 0 0 0 30 0 0 swblk 136 0 0 0 0 62 0 0 swpctrie 144 0 0 0 0 62 0 0 cdg_qdiffsample 16 0 0 0 0 254 0 0 pf state scrubs 40 0 0 0 0 254 0 0 pf frag entries 40 0 0 0 0 254 0 0 pf frags 248 0 0 0 0 62 0 0 pf table entries 160 0 0 0 0 254 0 0 pf table entry counters 64 0 0 0 0 254 0 0 pf source nodes 136 0 0 0 0 254 0 0 pf state keys 88 0 0 0 0 126 0 0 pf states 312 0 0 0 0 254 0 0 pf tags 104 0 0 0 0 126 0 0 pf mtags 184 0 0 0 0 62 0 0 tfo_ccache_entries 80 0 0 0 0 126 0 0 tfo 4 0 0 0 0 254 0 0 sackhole 32 0 0 0 0 254 0 0 tcptw 72 0 0 0 0 254 0 0 tcp_log_id_node 120 0 0 0 0 126 0 0 tcp_log_id_bucket 176 0 0 0 0 62 0 0 tcpreass 48 0 0 0 0 254 0 0 sctp_asconf_ack 48 0 0 0 0 254 0 0 sctp_asconf 40 0 0 0 0 254 0 0 ripcb ports 32 0 0 0 0 254 0 0 udplite_inpcb ports 32 0 0 0 0 254 0 0 IPsec SA lft_c 16 0 0 0 0 254 0 0 itimer 352 0 0 0 0 30 0 0 AIOLIO 272 0 0 0 0 30 0 0 TMPFS node 224 0 0 0 0 62 0 0 mqnotifier 216 0 0 0 0 62 0 0 mvdata 64 0 0 0 0 254 0 0 mqueue 248 0 0 0 0 62 0 0 NCLNODE 608 0 0 0 0 16 0 0 LTS VFS Cache 360 0 0 0 0 30 0 0 STS VFS Cache 144 0 0 0 0 62 0 0 cryptop 280 0 0 0 0 30 0 0 linux_dma_object 32 0 0 0 0 254 0 0 linux_dma_pctrie 144 0 0 0 0 62 0 0 IOMMU_MAP_ENTRY 104 0 0 0 0 126 0 0 mbuf_jumbo_16k 16384 0 0 0 0 254 0 0 mbuf_jumbo_9k 9216 0 0 0 0 254 0 0 audit_record 1280 0 0 0 0 8 0 0 MAC labels 40 0 0 0 0 254 0 0 vnpbuf 2624 0 0 0 0 64 0 0 mdpbuf 2624 0 0 0 0 3 0 0 nfspbuf 2624 0 0 0 0 16 0 0 swwbuf 2624 0 0 0 0 8 0 0 swrbuf 2624 0 0 0 0 16 0 0 umtx_shm 88 0 0 0 0 126 0 0 umtx pi 96 0 0 0 0 126 0 0 rangeset pctrie nodes 144 0 0 0 0 62 0 0 malloc-65536 65536 0 0 0 0 1 0 0 malloc-65536 65536 0 0 0 0 1 0 0 malloc-32768 32768 0 0 0 0 1 0 0 malloc-32768 32768 0 0 0 0 1 0 0 malloc-32768 32768 0 0 0 0 1 0 0 malloc-32768 32768 0 0 0 0 1 0 0 malloc-16384 16384 0 0 0 0 1 0 0 malloc-16384 16384 0 0 0 0 1 0 0 malloc-16384 16384 0 0 0 0 1 0 0 malloc-16384 16384 0 0 0 0 1 0 0 malloc-8192 8192 0 0 0 0 1 0 0 malloc-4096 4096 0 0 0 0 2 0 0 malloc-4096 4096 0 0 0 0 2 0 0 malloc-2048 2048 0 0 0 0 8 0 0 malloc-1024 1024 0 0 0 0 16 0 0 malloc-512 512 0 0 0 0 30 0 0 malloc-512 512 0 0 0 0 30 0 0 malloc-512 512 0 0 0 0 30 0 0 malloc-384 384 0 0 0 0 30 0 0 malloc-384 384 0 0 0 0 30 0 0 pcpu-32 32 0 0 0 0 254 0 0 fakepg 104 0 0 0 0 126 0 0 UMA Hash 256 0 0 0 0 62 0 0