panic: /syzkaller/managers/i386/kernel/sys/kern/kern_timeout.c:607: callout_cc_add: Bad list head 0xfffffe0007fbc388 first->prev != head cpuid = 0 time = 1747215974 KDB: stack backtrace: db_trace_self_wrapper() at db_trace_self_wrapper+0xc6/frame 0xfffffe00574cc710 kdb_backtrace() at kdb_backtrace+0xd0/frame 0xfffffe00574cc870 vpanic() at vpanic+0x257/frame 0xfffffe00574cca30 panic() at panic+0xb5/frame 0xfffffe00574ccaf0 callout_cc_add() at callout_cc_add+0x339/frame 0xfffffe00574ccb50 callout_reset_sbt_on() at callout_reset_sbt_on+0x74f/frame 0xfffffe00574ccc70 sleepq_set_timeout_sbt() at sleepq_set_timeout_sbt+0x20b/frame 0xfffffe00574ccd30 _sleep() at _sleep+0x468/frame 0xfffffe00574cce90 softdep_flush() at softdep_flush+0x308/frame 0xfffffe00574ccef0 fork_exit() at fork_exit+0xcc/frame 0xfffffe00574ccf30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe00574ccf30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- KDB: enter: panic [ thread pid 15 tid 100096 ] Stopped at kdb_enter+0x6e: movq $0,0x25bda37(%rip) db> db> set $lines = 0 db> set $maxwidth = 0 db> show registers cs 0x20 ds 0x3b es 0x3b fs 0x13 gs 0x1b ss 0 rax 0x12 rcx 0xfffffe00033eee30 rdx 0 rbx 0xffffffff827b0020 .str.27 rsp 0xfffffe00574cc850 rbp 0xfffffe00574cc870 rsi 0 rdi 0xffffffff816145e9 printf+0x149 r8 0 r9 0xffffffff r10 0xaf770e84c7e3034a r11 0x3f r12 0xfffffe00548e8740 r13 0xfffffffffffffffe r14 0xffffffff827b0020 .str.27 r15 0 rip 0xffffffff815fe75e kdb_enter+0x6e rflags 0x46 kdb_enter+0x6e: movq $0,0x25bda37(%rip) db> show proc Process 15 (bufdaemon) at 0xfffffe00080285a0: state: NORMAL uid: 0 gids: 0 parent: pid 0 at 0xffffffff83b468e0 ABI: null flag: 0x10000284 flag2: 0 reaper: 0xffffffff83b468e0 reapsubtree: 15 sigparent: 20 vmspace: 0xffffffff83b47880 (map 0xffffffff83b47880) (map.pmap 0xffffffff83b47920) (pmap 0xffffffff83b47990) threads: 3 100080 D psleep 0xffffffff83cb9360 [bufdaemon] 100083 D - 0xffffffff83002140 [bufspacedaemon-0] 100096 Run sdflush 0xfffffe0058ddfce8 [/ worker] db> ps pid ppid pgrp uid state wmesg wchan cmd 839 764 764 0 R (threaded) syz-executor 100141 RunQ syz-executor 100161 RunQ syz-executor 837 767 767 0 R (threaded) syz-executor 100116 Run CPU 1 syz-executor 100158 S uwait 0xfffffe0007f7e380 syz-executor 100159 S uwait 0xfffffe0071f4e080 syz-executor 835 834 765 0 SV uwait 0xfffffe0058950400 syz-executor 834 765 765 0 R (threaded) syz-executor 100117 RunQ syz-executor 100154 D ppwait 0xfffffe0054809ac0 syz-executor 100155 S uwait 0xfffffe0007f7e180 syz-executor 100156 S uwait 0xfffffe0071f4de00 syz-executor 829 766 766 0 R (threaded) syz-executor 100144 RunQ syz-executor 100146 S accept 0xfffffe005a327610 syz-executor 100149 S uwait 0xfffffe0058d07480 syz-executor 826 0 0 0 DL aiordy 0xfffffe00549115a0 [aiod4] 825 0 0 0 DL aiordy 0xfffffe0054911b00 [aiod3] 824 0 0 0 DL aiordy 0xfffffe0054912060 [aiod2] 823 0 0 0 DL aiordy 0xfffffe00548f25a0 [aiod1] 819 808 819 0 Ss select 0xfffffe005a0f7f40 dhclient 812 1 812 0 Ss select 0xfffffe005a0f7e40 dhclient 808 787 424 65 S select 0xfffffe0071b6aac0 dhclient 787 424 424 0 S wait 0xfffffe0054926000 sh 767 763 767 0 S nanslp 0xffffffff83b9d500 syz-executor 766 763 766 0 R syz-executor 765 763 765 0 R syz-executor 764 763 764 0 S nanslp 0xffffffff83b9d500 syz-executor 763 761 761 0 S select 0xfffffe0071b6abc0 syz-executor 761 759 761 0 Ss pause 0xfffffe0054808bb0 csh 759 682 759 0 Ss select 0xfffffe0071b6ac40 sshd 748 1 748 0 Ss+ ttyin 0xfffffe0058a9c8b0 getty 747 1 747 0 Ss+ ttyin 0xfffffe0058aa04b0 getty 746 1 746 0 Ss+ ttyin 0xfffffe0059cbc0b0 getty 745 1 745 0 Ss+ ttyin 0xfffffe0059cbc8b0 getty 744 1 744 0 Ss+ ttyin 0xfffffe005476d0b0 getty 743 1 743 0 Ss+ ttyin 0xfffffe005476d8b0 getty 742 1 742 0 Ss+ ttyin 0xfffffe005476e0b0 getty 741 1 741 0 Ss+ ttyin 0xfffffe005476e8b0 getty 740 1 740 0 Ss+ ttyin 0xfffffe005476f0b0 getty 738 1 18 0 S+ piperd 0xfffffe0059ff9000 logger 737 736 18 0 S+ nanslp 0xffffffff83b9d500 sleep 736 1 18 0 S+ wait 0xfffffe005490fac0 sh 686 1 686 0 Ss nanslp 0xffffffff83b9d500 cron 682 1 682 0 Ss select 0xfffffe005a0f7c40 sshd 495 1 495 0 Ss select 0xfffffe0059ec22c0 syslogd 424 1 424 0 Ss wait 0xfffffe0054807020 devd 423 1 423 65 Ss select 0xfffffe0059ec2140 dhclient 338 1 338 0 Ss select 0xfffffe0059e4d840 dhclient 335 1 335 0 Ss select 0xfffffe0059e4d5c0 dhclient 17 0 0 0 DL syncer 0xffffffff83cbada0 [syncer] 16 0 0 0 DL vlruwt 0xfffffe0008028040 [vnlru] 15 0 0 0 RL (threaded) [bufdaemon] 100080 D psleep 0xffffffff83cb9360 [bufdaemon] 100083 D - 0xffffffff83002140 [bufspacedaemon-0] 100096 Run sdflush 0xfffffe0058ddfce8 [/ worker] 9 0 0 0 DL psleep 0xffffffff83d04380 [vmdaemon] 8 0 0 0 DL (threaded) [pagedaemon] 100078 D psleep 0xffffffff83cea2f8 [dom0] 100084 D launds 0xffffffff83cea304 [laundry: dom0] 100085 D umarcl 0xffffffff81dcf0a0 [uma] 7 0 0 0 DL - 0xffffffff8391acd0 [rand_harvestq] 6 0 0 0 DL pftm 0xffffffff843c9850 [pf purge] 5 0 0 0 DL waiting 0xffffffff8473a700 [sctp_iterator] 4 0 0 0 DL (threaded) [cam] 100046 D - 0xffffffff838e5340 [doneq0] 100047 D - 0xffffffff838e52c0 [async] 100076 D - 0xffffffff838e5140 [scanner] 3 0 0 0 DL (threaded) [crypto] 100043 D crypto_ 0xffffffff83ce5b40 [crypto] 100044 D crypto_ 0xfffffe0058566330 [crypto returns 0] 100045 D crypto_ 0xfffffe0058566380 [crypto returns 1] 14 0 0 0 DL seqstat 0xfffffe00547edc88 [sequencer 00] 13 0 0 0 DL (threaded) [geom] 100037 D - 0xffffffff83b45f00 [g_event] 100038 D - 0xffffffff83b45f20 [g_up] 100039 D - 0xffffffff83b45f40 [g_down] 2 0 0 0 WL (threaded) [clock] 100031 I [clock (0)] 100032 I [clock (1)] 12 0 0 0 WL (threaded) [intr] 100013 I [swi6: task queue] 100014 I [swi6: Giant taskq] 100016 I [swi5: fast taskq] 100033 I [swi1: netisr 0] 100034 I [swi1: hpts] 100035 I [swi1: hpts] 100048 I [irq24: virtio_pci0] 100049 I [irq25: virtio_pci0] 100050 I [irq26: virtio_pci0] 100051 I [irq27: virtio_pci0] 100052 I [irq28: virtio_pci1] 100053 I [irq29: virtio_pci1] 100054 I [irq30: virtio_pci1] 100055 I [irq31: virtio_pci1] 100056 I [irq32: virtio_pci1] 100061 I [irq10: virtio_pci2] 100063 I [irq1: atkbd0] 100064 I [irq12: psm0] 100065 I [swi0: uart uart++] 100069 I [swi1: pf send] 11 0 0 0 RL (threaded) [idle] 100003 CanRun [idle: cpu0] 100004 CanRun [idle: cpu1] 1 0 1 0 SLs wait 0xfffffe0008009040 [init] 10 0 0 0 DL audit_w 0xffffffff83ce65e0 [audit] 0 0 0 0 DLs (threaded) [kernel] 100000 D parked 0xffffffff84c2cff0 [swapper] 100005 D - 0xfffffe005462d700 [softirq_0] 100006 D - 0xfffffe005462d600 [softirq_1] 100007 D - 0xfffffe005462d500 [if_io_tqg_0] 100008 D - 0xfffffe005462d400 [if_io_tqg_1] 100009 D - 0xfffffe005462d300 [if_config_tqg_0] 100010 D - 0xfffffe0008bf9500 [kqueue_ctx taskq] 100011 D - 0xfffffe0008bf9300 [jail_remove taskq] 100012 D - 0xfffffe0008bf9100 [bus taskq] 100015 D - 0xfffffe0008bf8a00 [thread taskq] 100017 D - 0xfffffe0008bf8600 [aiod_kick taskq] 100018 D - 0xfffffe0008bf8400 [deferred_unmount ta] 100019 D - 0xfffffe0008bf8200 [inm_free taskq] 100020 D - 0xfffffe0008bf8000 [in6m_free taskq] 100021 D - 0xfffffe0008bf7d00 [linuxkpi_irq_wq] 100022 D - 0xfffffe0008bf7b00 [linuxkpi_short_wq_0] 100023 D - 0xfffffe0008bf7b00 [linuxkpi_short_wq_1] 100024 D - 0xfffffe0008bf7b00 [linuxkpi_short_wq_2] 100025 D - 0xfffffe0008bf7b00 [linuxkpi_short_wq_3] 100026 D - 0xfffffe0008bf7600 [linuxkpi_long_wq_0] 100027 D - 0xfffffe0008bf7600 [linuxkpi_long_wq_1] 100028 D - 0xfffffe0008bf7600 [linuxkpi_long_wq_2] 100029 D - 0xfffffe0008bf7600 [linuxkpi_long_wq_3] 100036 D - 0xfffffe0008bf6a00 [firmware taskq] 100041 D - 0xfffffe0008bf6300 [crypto_0] 100042 D - 0xfffffe0008bf6300 [crypto_1] 100057 D - 0xfffffe0058565900 [vtnet0 rxq 0] 100058 D - 0xfffffe0058565800 [vtnet0 txq 0] 100059 D - 0xfffffe0058565700 [vtnet0 rxq 1] 100060 D - 0xfffffe0058565600 [vtnet0 txq 1] 100062 D vtbslp 0xfffffe005858d000 [virtio_balloon] 100066 D - 0xffffffff827b5360 [deadlkres] 100070 D - 0xfffffe0058d0dd00 [acpi_task_0] 100071 D - 0xfffffe0058d0dd00 [acpi_task_1] 100072 D - 0xfffffe0058d0dd00 [acpi_task_2] 100074 D - 0xfffffe0008bfa100 [mca taskq] 100075 D - 0xfffffe0058566000 [CAM taskq] 100077 D - 0xfffffe0058564400 [ipsec_offload] db> show all locks Process 837 (syz-executor) thread 0xfffffe0054931000 (100116) exclusive sleep mutex pmap (pmap) r = 0 (0xfffffe00548d9110) locked @ /syzkaller/managers/i386/kernel/sys/amd64/amd64/pmap.c:8871 shared rw vmobject (vmobject) r = 0 (0xfffffe005491c6c8) locked @ /syzkaller/managers/i386/kernel/sys/vm/vm_fault.c:364 shared sx vm map (user) (vm map (user)) r = 0 (0xfffffe00548d9060) locked @ /syzkaller/managers/i386/kernel/sys/vm/vm_map.c:4993 db> show malloc Type InUse MemUse Requests pf_hash 6 12804K 6 linker 376 5063K 486 tcp_hpts 7 4801K 7 devbuf 4188 4324K 4213 sysctloid 34774 2049K 34849 vtbuf 24 1968K 46 kobj 331 1324K 495 newblk 335 1108K 658 vfscache 3 1025K 3 pcb 28 673K 55 inodedep 68 538K 114 ufs_quota 1 512K 1 vfs_hash 1 512K 1 callout 2 512K 2 intr 4 472K 4 subproc 124 254K 908 vnet_data 2 224K 2 acpitask 1 224K 1 KTRACE 102 201K 135 acpica 1674 184K 54426 vmem 5 143K 5 tidhash 3 141K 3 filedesc 18 137K 105 pagedep 22 134K 42 tfo_ccache 1 128K 1 IP reass 1 128K 1 DEVFS1 109 109K 126 sem 4 106K 4 gtaskqueue 18 98K 18 bus 997 82K 5063 mtx_pool 3 74K 3 syncache 1 68K 1 NFSD srvcache 3 68K 3 module 521 66K 521 ddb_capture 1 64K 1 kdtrace 193 38K 1002 temp 35 37K 1833 umtx 288 36K 288 BPF 22 36K 24 hostcache 1 32K 1 shm 1 32K 1 DEVFS3 128 32K 138 msg 4 30K 4 kbdmux 6 28K 6 DEVFS_RULE 56 20K 56 ifaddr 67 19K 69 LRO 18 19K 18 ufs_mount 4 17K 5 proc 3 17K 3 tty 16 16K 16 routetbl 130 16K 410 ithread 90 15K 90 bus-sc 34 15K 1647 eventhandler 163 14K 163 lltable 43 14K 43 ifnet 7 13K 7 ether_multi 152 13K 162 kenv 95 12K 95 GEOM 61 11K 477 CAM queue 5 11K 1528 rman 82 10K 437 rpc 8 9K 8 plimit 23 9K 332 in6_multi 65 9K 65 bmsafemap 3 9K 87 devstat 4 9K 4 UART 12 9K 12 ksem 1 8K 1 shmfd 1 8K 1 pfs_vncache 1 8K 1 audit_evclass 239 8K 301 taskqueue 69 8K 69 sglist 6 7K 6 cred 24 6K 190 CAM DEV 3 6K 510 pfs_nodes 22 6K 22 ufs_dirhash 24 5K 24 UMA 265 5K 265 pf_ifnet 10 5K 19 dirrem 17 5K 60 diradd 33 5K 77 vt 11 5K 11 memdesc 1 4K 1 MCA 32 4K 32 evdev 4 4K 4 kqueue 58 4K 848 pwddesc 57 4K 844 acpisem 28 4K 28 selfd 48 3K 6291 DEVFSP 46 3K 50 indirdep 11 3K 95 terminal 11 3K 11 session 22 3K 33 proc-args 83 3K 1883 acpidev 20 3K 20 hhook 8 3K 10 clone 9 3K 9 uidinfo 3 3K 6 kcovinfo 36 3K 36 local_apic 1 2K 1 io_apic 1 2K 1 mkdir 16 2K 62 ipsec-saq 2 2K 2 lockf 19 2K 38 ip6ndp 12 2K 13 Unitno 30 2K 50 sctp_ifa 13 2K 14 CAM XPT 22 2K 543 sctp_atcl 4 2K 4 newdirblk 12 2K 31 in_multi 6 2K 8 tun 4 2K 4 toponodes 6 2K 6 ipsecpolicy 2 2K 2 select 11 2K 30 msi 9 2K 9 netlink 2 2K 71 sctp_stro 1 1K 1 softdep 1 1K 1 sahead 1 1K 1 secasvar 1 1K 1 nhops 6 1K 8 vnodemarker 2 1K 8 NFSD session 1 1K 1 CAM periph 4 1K 271 sctp_ifn 6 1K 14 ipsec 3 1K 3 mld 6 1K 6 igmp 6 1K 6 pfil 6 1K 6 isadev 6 1K 6 mount 16 1K 89 pci_link 10 1K 10 crypto 4 1K 4 encap_export_host 12 1K 12 osd 9 1K 25 sctp_stri 1 1K 2 CC Mem 4 1K 9 cdev 2 1K 2 lkpikmalloc 8 1K 9 inpcbpolicy 12 1K 161 chacha20random 1 1K 1 biobuf 1 1K 1 ip6opt 2 1K 5 vnodes 1 1K 1 procdesc 2 1K 8 NFSD lckfile 1 1K 1 NFSD V4client 1 1K 1 DEVFS 9 1K 10 CAM SIM 2 1K 2 sctp_atky 5 1K 6 feeder 7 1K 7 tcpfunc 3 1K 3 loginclass 3 1K 5 prison 6 1K 6 cryptodev 2 1K 35 nexusdev 8 1K 8 apmdev 1 1K 1 atkbddev 2 1K 2 aio 4 1K 4 pmchooks 1 1K 1 filecaps 5 1K 72 CAM path 4 1K 1034 CAM dev queue 2 1K 2 CAM I/O Scheduler 1 1K 1 soname 4 1K 3311 sctp_vrf 1 1K 1 sctp_athm 4 1K 5 freework 1 1K 102 vnet 1 1K 1 pmc 1 1K 1 entropy 2 1K 33 acpiintr 1 1K 1 sctp_map 2 1K 2 cpus 2 1K 2 vnet_data_free 1 1K 1 Per-cpu 1 1K 1 p1003.1b 1 1K 1 tcp_pcm_rack 0 0K 0 tcp_do_rack 0 0K 0 tcp_fsb_rack 0 0K 0 sctp_mcore 0 0K 0 sctp_socko 0 0K 1 sctp_iter 0 0K 11 sctp_mvrf 0 0K 0 sctp_timw 0 0K 0 sctp_cpal 0 0K 0 sctp_cmsg 0 0K 0 sctp_stre 0 0K 0 sctp_athi 0 0K 0 sctp_a_it 0 0K 11 sctp_aadr 0 0K 0 mqdata 0 0K 0 ipcomp 0 0K 0 esp 0 0K 0 ah 0 0K 0 filemon 0 0K 0 pf_table 0 0K 0 pf_rule 0 0K 0 pf_altq 0 0K 0 pf_osfp 0 0K 0 pf_krule_item 0 0K 0 pf_temp 0 0K 0 madt_table 0 0K 2 smartpqi 0 0K 0 ixl 0 0K 0 ice-resmgr 0 0K 0 ice-osdep 0 0K 0 ice 0 0K 0 iavf 0 0K 0 axgbe 0 0K 0 fpukern_ctx 0 0K 0 xen_intr 0 0K 0 xen_hvm 0 0K 0 legacydrv 0 0K 0 NMI handlers 0 0K 0 bounce 0 0K 0 busdma 0 0K 0 qpidrv 0 0K 0 dmar_idpgtbl 0 0K 0 dmar_dom 0 0K 0 dmar_ctx 0 0K 0 amdiommu_dom 0 0K 0 amdiommu_ctx 0 0K 0 isci 0 0K 0 iommu_dmamap 0 0K 0 hyperv_socket 0 0K 0 bxe_ilt 0 0K 0 aesni_data 0 0K 0 xenbus 0 0K 0 vm_fictitious 0 0K 0 UMAHash 0 0K 0 vm_pgdata 0 0K 0 jblocks 0 0K 0 savedino 0 0K 132 sentinel 0 0K 0 jfsync 0 0K 0 jtrunc 0 0K 0 sbdep 0 0K 2 jsegdep 0 0K 0 jseg 0 0K 0 jfreefrag 0 0K 0 jfreeblk 0 0K 0 jnewblk 0 0K 0 jmvref 0 0K 0 jremref 0 0K 0 jaddref 0 0K 0 freedep 0 0K 0 freefile 0 0K 30 freeblks 0 0K 49 freefrag 0 0K 48 allocindir 0 0K 0 allocdirect 0 0K 0 ufs_trim 0 0K 0 mactemp 0 0K 0 audit_trigger 0 0K 0 audit_pipe_presel 0 0K 0 audit_pipeent 0 0K 0 audit_pipe 0 0K 0 audit_evname 0 0K 0 audit_bsm 0 0K 0 audit_gidset 0 0K 0 audit_text 0 0K 0 audit_path 0 0K 0 audit_data 0 0K 0 audit_cred 0 0K 0 ktls_ocf 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5E_TLS_RX 0 0K 0 MLX5EEPROM 0 0K 0 MLX5E_TLS 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EN 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5DUMP 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 simple_attr 0 0K 0 seq_file 0 0K 0 lkpiskb 0 0K 0 radix 0 0K 0 idr 0 0K 0 lkpindev 0 0K 0 lkpimhi 0 0K 0 lkpifw 0 0K 0 lkpi80211 0 0K 0 NLM 0 0K 0 ipsec-spdcache 0 0K 0 ipsec-reg 0 0K 0 ipsec-misc 0 0K 0 ipsecrequest 0 0K 0 ip6_msource 0 0K 0 ip6_moptions 0 0K 0 in6_mfilter 0 0K 0 frag6 0 0K 0 tcplog 0 0K 0 tcp_hwpace 0 0K 0 ip_msource 0 0K 0 ip_moptions 0 0K 0 in_mfilter 0 0K 0 ipid 0 0K 0 80211scan 0 0K 0 80211ratectl 0 0K 0 80211power 0 0K 0 80211nodeie 0 0K 0 80211node 0 0K 0 80211mesh_gt 0 0K 0 80211mesh_rt 0 0K 0 80211perr 0 0K 0 80211prep 0 0K 0 80211preq 0 0K 0 80211dfs 0 0K 0 80211crypto 0 0K 0 80211vap 0 0K 0 iflib 0 0K 0 vlan 0 0K 0 gif 0 0K 0 ifdescr 0 0K 0 zlib 0 0K 19 fadvise 0 0K 0 VN POLL 0 0K 0 statfs 0 0K 181 namei_tracker 0 0K 0 export_host 0 0K 0 cl_savebuf 0 0K 13 lio 0 0K 6 acl 0 0K 0 mbuf_tag 0 0K 0 ktls 0 0K 0 accf 0 0K 0 pts 0 0K 0 timerfd 0 0K 0 iov 0 0K 13521 ioctlops 0 0K 96 eventfd 0 0K 0 Witness 0 0K 0 stack 0 0K 0 sbuf 0 0K 288 firmware 0 0K 0 compressor 0 0K 0 SWAP 0 0K 0 sysctltmp 0 0K 649 sysctl 0 0K 3 ekcd 0 0K 0 dumper 0 0K 0 sendfile 0 0K 0 rctl 0 0K 0 cache 0 0K 0 prison_racct 0 0K 0 Fail Points 0 0K 0 sigio 0 0K 1 filedesc_to_leader 0 0K 0 pwd 0 0K 0 tty console 0 0K 0 boottrace 0 0K 0 isofs_node 0 0K 0 isofs_mount 0 0K 0 tr_raid5_data 0 0K 0 tr_raid1e_data 0 0K 0 tr_raid1_data 0 0K 0 tr_raid0_data 0 0K 0 tr_concat_data 0 0K 0 md_sii_data 0 0K 0 md_promise_data 0 0K 0 md_nvidia_data 0 0K 0 md_jmicron_data 0 0K 0 md_intel_data 0 0K 0 md_ddf_data 0 0K 0 raid_data 0 0K 72 geom_flashmap 0 0K 0 tmpfs dir 0 0K 0 tmpfs name 0 0K 0 tmpfs mount 0 0K 0 tmpfs extattr 0 0K 0 NFS FHA 0 0K 0 newnfsmnt 0 0K 0 newnfsclient_req 0 0K 0 NFSCL layrecall 0 0K 0 NFSCL session 0 0K 0 NFSCL sockreq 0 0K 0 NFSCL devinfo 0 0K 0 NFSCL flayout 0 0K 0 NFSCL layout 0 0K 0 NFSD rollback 0 0K 0 NFSCL diroff 0 0K 0 NEWNFSnode 0 0K 0 NFSCL lck 0 0K 0 NFSCL lckown 0 0K 0 NFSCL client 0 0K 0 NFSCL deleg 0 0K 0 NFSCL open 0 0K 0 NFSCL owner 0 0K 0 NFS fh 0 0K 0 NFS req 0 0K 0 NFSD usrgroup 0 0K 0 NFSD string 0 0K 0 NFSD V4lock 0 0K 0 NFSD V4state 0 0K 0 msdosfs_fat 0 0K 0 msdosfs_mount 0 0K 0 msdosfs_node 0 0K 0 DEVFS4 0 0K 0 DEVFS2 0 0K 0 gntdev 0 0K 0 privcmd_dev 0 0K 0 evtchn_dev 0 0K 0 xenstore 0 0K 0 xnb 0 0K 0 xen_acpi 0 0K 0 xbbd 0 0K 0 xbd 0 0K 0 Balloon 0 0K 0 sysmouse 0 0K 0 vtfont 0 0K 0 pvscsi 0 0K 0 USBdev 0 0K 0 USB 0 0K 0 twsbuf 0 0K 0 tcp_log_dev 0 0K 0 midi buffers 0 0K 0 mixer 0 0K 0 ac97 0 0K 0 hdacc 0 0K 0 hdac 0 0K 0 hdaa 0 0K 0 SIIS driver 0 0K 0 PUC 0 0K 0 ppbusdev 0 0K 0 sr_iov 0 0K 0 OCS 0 0K 0 OCS 0 0K 0 nvme 0 0K 0 nvd 0 0K 0 netmap 0 0K 0 mwldev 0 0K 0 MVS driver 0 0K 0 mpi3mrbuf 0 0K 0 mrsasbuf 0 0K 0 mpt_user 0 0K 0 mps_user 0 0K 0 MPSSAS 0 0K 0 mps 0 0K 0 mpr_user 0 0K 0 MPRSAS 0 0K 0 mpr 0 0K 0 mfibuf 0 0K 0 md_sectors 0 0K 0 md_disk 0 0K 0 malodev 0 0K 0 LED 0 0K 0 ix_sriov 0 0K 0 ix 0 0K 0 ipsbuf 0 0K 0 ciss_data 0 0K 0 BACKLIGHT 0 0K 0 ath_hal 0 0K 0 athdev 0 0K 0 ata_pci 0 0K 0 ata_dma 0 0K 0 ata_generic 0 0K 0 AHCI driver 0 0K 0 agp 0 0K 0 acpipwr 0 0K 0 acpi_perf 0 0K 0 acpicmbat 0 0K 0 aacraidcam 0 0K 0 aacraid_buf 0 0K 0 aaccam 0 0K 0 aacbuf 0 0K 0 zstd 0 0K 0 XZ_DEC 0 0K 0 nvlist 0 0K 0 SCSI ENC 0 0K 0 SCSI sa 0 0K 0 scsi_pass 0 0K 0 scsi_da 0 0K 70 ata_da 0 0K 0 scsi_ch 0 0K 0 scsi_cd 0 0K 0 nvme_da 0 0K 0 CAM CCB 0 0K 523 CAM ccb queue 0 0K 0 db> show uma Zone Size Used Free Requests Sleeps Bucket Total Mem XFree mbuf_jumbo_page 4096 8336 1062 11324 0 254 38494208 0 mbuf 256 18419 1398 31425 0 254 5073152 0 BUF TRIE 152 210 11594 556 0 62 1794208 0 malloc-384 384 4165 35 4176 0 30 1612800 0 malloc-128 128 11399 195 14747 0 126 1484032 0 malloc-4096 4096 353 3 1082 0 2 1458176 0 UMA Slabs 0 112 10919 19 10919 0 126 1225056 0 malloc-65536 65536 15 1 18 0 1 1048576 0 mbuf_cluster 2048 508 0 508 0 254 1040384 0 RADIX NODE 152 6299 170 27003 0 62 983288 0 vmem btag 56 16730 85 16730 0 254 941640 0 pbuf 2624 0 257 0 0 2 674368 0 FFS inode 1168 536 24 566 0 8 654080 0 sctp_asoc 2256 1 254 1 0 254 575280 0 socket 1024 26 482 1354 0 254 520192 0 lkpicurr 168 2 3094 2 0 62 520128 0 sctp_chunk 152 3264 38 3264 0 254 501904 0 VM OBJECT 248 1142 106 13573 0 62 309504 0 sctp_ep 1152 2 257 2 0 254 298368 0 256 Bucket 2048 121 17 1071 0 8 282624 0 malloc-2048 2048 108 28 632 0 8 278528 0 malloc-64 64 4011 336 6640 0 254 278208 0 VNODE 440 571 59 603 0 30 277200 0 THREAD 1824 135 9 161 0 8 262656 0 malloc-16384 16384 12 3 55 0 1 245760 0 malloc-4096 4096 57 3 844 0 2 245760 0 malloc-16 16 14374 376 15846 0 254 236000 0 malloc-32768 32768 1 6 92 0 1 229376 0 DEVCTL 1024 21 199 148 0 0 225280 0 malloc-128 128 1310 209 25799 0 126 194432 0 sctp_raddr 736 1 263 1 0 254 194304 0 malloc-32 32 5577 345 5792 0 254 189504 0 UMA Zones 768 237 2 237 0 16 183552 0 lkpimm 56 1 3095 1 0 254 173376 0 unpcb 320 11 505 1165 0 254 165120 0 malloc-1024 1024 141 19 163 0 16 163840 0 FFS2 dinode 256 536 94 566 0 62 161280 0 malloc-256 256 336 234 819 0 62 145920 0 MAP ENTRY 96 1305 207 42067 0 126 145152 0 FPU_save_area 832 137 25 185 0 16 134784 0 S VFS Cache 104 995 292 1060 0 126 133848 0 malloc-65536 65536 0 2 60 0 1 131072 0 malloc-65536 65536 0 2 110 0 1 131072 0 malloc-65536 65536 1 1 9 0 1 131072 0 mbuf_packet 256 1 507 152 0 254 130048 0 ksiginfo 112 50 994 71 0 126 116928 0 malloc-128 128 641 258 1540 0 126 115072 0 malloc-2048 2048 6 50 582 0 8 114688 0 malloc-32768 32768 3 0 3 0 1 98304 0 PROC 1376 56 10 839 0 8 90816 0 UMA Kegs 384 224 9 224 0 30 89472 0 128 Bucket 1024 44 39 248 0 16 84992 0 filedesc0 1072 57 13 844 0 8 75040 0 g_bio 408 4 176 5903 0 30 73440 0 malloc-384 384 142 38 188 0 30 69120 0 malloc-64 64 644 427 14842 0 254 68544 0 malloc-128 128 391 136 524 0 126 67456 0 malloc-128 128 300 227 365 0 126 67456 0 malloc-65536 65536 1 0 1 0 1 65536 0 malloc-32768 32768 0 2 120 0 1 65536 0 malloc-32768 32768 2 0 2 0 1 65536 0 64 Bucket 512 75 53 1791 0 30 65536 0 malloc-256 256 200 55 339 0 62 65280 0 malloc-8192 8192 6 1 8 0 1 57344 0 malloc-4096 4096 11 3 20 0 2 57344 0 malloc-64 64 316 503 6614 0 254 52416 0 malloc-64 64 458 361 657 0 254 52416 0 malloc-128 128 6 397 260 0 126 51584 0 malloc-256 256 106 89 240 0 62 49920 0 malloc-256 256 57 138 755 0 62 49920 0 malloc-256 256 168 27 911 0 62 49920 0 malloc-256 256 78 117 180 0 62 49920 0 malloc-256 256 68 127 452 0 62 49920 0 32 Bucket 256 64 131 647 0 62 49920 0 DIRHASH 1024 34 14 34 0 16 49152 0 NAMEI 1024 0 48 12711 0 16 49152 0 malloc-2048 2048 15 9 31 0 8 49152 0 malloc-1024 1024 12 36 520 0 16 49152 0 syncache 168 0 264 3 0 254 44352 0 malloc-8192 8192 4 1 30 0 1 40960 0 malloc-8192 8192 5 0 6 0 1 40960 0 pcpu-8 8 4707 413 4870 0 254 40960 0 pipe 736 21 34 292 0 16 40480 0 da_ccb 544 1 69 1568 0 16 38080 0 udp_inpcb 408 6 84 147 0 30 36720 0 malloc-64 64 101 466 131 0 254 36288 0 malloc-64 64 14 553 14 0 254 36288 0 malloc-64 64 47 520 172 0 254 36288 0 malloc-64 64 14 553 677 0 254 36288 0 malloc-128 128 107 172 181 0 126 35712 0 malloc-128 128 9 270 12 0 126 35712 0 routing nhops 256 27 108 34 0 62 34560 0 ttyoutq 256 72 63 160 0 62 34560 0 malloc-384 384 63 27 63 0 30 34560 0 malloc-32768 32768 0 1 2 0 1 32768 0 malloc-32768 32768 1 0 1 0 1 32768 0 malloc-16384 16384 2 0 2 0 1 32768 0 malloc-16384 16384 2 0 2 0 1 32768 0 malloc-16384 16384 0 2 190 0 1 32768 0 malloc-8192 8192 4 0 4 0 1 32768 0 malloc-4096 4096 4 4 9 0 2 32768 0 malloc-2048 2048 1 15 13 0 8 32768 0 malloc-2048 2048 7 9 7 0 8 32768 0 malloc-2048 2048 0 16 50 0 8 32768 0 malloc-2048 2048 3 13 194 0 8 32768 0 malloc-2048 2048 5 11 7 0 8 32768 0 malloc-1024 1024 2 30 42 0 16 32768 0 malloc-1024 1024 7 25 991 0 16 32768 0 malloc-1024 1024 4 28 24 0 16 32768 0 malloc-1024 1024 10 22 168 0 16 32768 0 malloc-512 512 4 60 45 0 30 32768 0 malloc-512 512 1 63 2 0 30 32768 0 malloc-512 512 6 58 54 0 30 32768 0 malloc-512 512 8 56 8 0 30 32768 0 pcpu-64 64 496 16 496 0 254 32768 0 sctp_stream_msg_out 112 1 287 1 0 254 32256 0 ttyinq 160 135 65 300 0 62 32000 0 PGRP 120 26 238 37 0 126 31680 0 VMSPACE 584 36 13 820 0 16 28616 0 16 Bucket 144 54 142 275 0 62 28224 0 4 Bucket 48 4 584 4 0 254 28224 0 TURNSTILE 136 145 44 145 0 62 25704 0 cpuset 200 7 121 7 0 62 25600 0 malloc-8192 8192 1 2 82 0 1 24576 0 malloc-8192 8192 2 1 21 0 1 24576 0 malloc-4096 4096 3 3 184 0 2 24576 0 ertt_txseginfo 40 0 606 180 0 254 24240 0 PWD 40 19 587 115 0 254 24240 0 rtentry 168 30 114 34 0 62 24192 0 Files 80 168 132 6822 0 126 24000 0 8 Bucket 80 51 249 322 0 126 24000 0 tcp_inpcb 1304 4 14 9 0 8 23472 0 itimer 352 0 66 8 0 30 23232 0 malloc-384 384 1 59 8 0 30 23040 0 malloc-384 384 1 59 26 0 30 23040 0 malloc-384 384 1 59 344 0 30 23040 0 malloc-384 384 21 39 24 0 30 23040 0 malloc-384 384 28 32 337 0 30 23040 0 ripcb 376 2 58 5 0 30 22560 0 Mountpoints 2816 2 6 2 0 4 22528 0 SLEEPQUEUE 88 145 111 145 0 126 22528 0 hostcache 64 1 314 1 0 254 20160 0 ertt 72 4 276 9 0 126 20160 0 malloc-32 32 117 513 1197 0 254 20160 0 malloc-32 32 57 573 211 0 254 20160 0 malloc-32 32 33 597 763 0 254 20160 0 malloc-32 32 72 558 2706 0 254 20160 0 malloc-32 32 45 585 623 0 254 20160 0 2 Bucket 32 62 568 360 0 254 20160 0 malloc-256 256 6 69 272 0 62 19200 0 AIOLIO 272 0 70 3 0 30 19040 0 vmem 1856 2 7 2 0 8 16704 0 epoch_record pcpu 256 4 60 4 0 62 16384 0 malloc-8192 8192 2 0 2 0 1 16384 0 malloc-4096 4096 0 4 41 0 2 16384 0 malloc-4096 4096 3 1 3 0 2 16384 0 malloc-4096 4096 1 3 4 0 2 16384 0 malloc-1024 1024 1 15 1 0 16 16384 0 malloc-512 512 2 30 112 0 30 16384 0 malloc-512 512 3 29 8 0 30 16384 0 malloc-512 512 1 31 3 0 30 16384 0 SMR CPU 32 8 503 8 0 254 16352 0 sctp_laddr 48 1 335 13 0 254 16128 0 rl_entry 48 2 334 3 0 254 16128 0 AIO 208 0 76 3 0 62 15808 0 kenv 258 17 43 1066 0 30 15480 0 mqnode 416 3 33 3 0 30 14976 0 SMR SHARED 24 8 503 8 0 254 12264 0 malloc-32 32 71 307 93 0 254 12096 0 malloc-32 32 80 298 80 0 254 12096 0 KNOTE 160 0 75 10 0 62 12000 0 malloc-16 16 23 727 63 0 254 12000 0 malloc-16 16 296 454 500 0 254 12000 0 malloc-16 16 46 704 81 0 254 12000 0 malloc-16 16 188 562 3017 0 254 12000 0 malloc-16 16 35 715 24854 0 254 12000 0 malloc-16 16 22 728 47 0 254 12000 0 clpbuf 2624 0 4 12 0 4 10496 0 malloc-8192 8192 1 0 1 0 1 8192 0 pcpu-16 16 8 504 8 0 254 8192 0 vtnet_tx_hdr 24 0 334 904 0 254 8016 0 malloc-16 16 18 482 19 0 254 8000 0 UMA Slabs 1 176 9 13 9 0 62 3872 0 KMAP ENTRY 96 12 27 14 0 0 3744 0 FFS1 dinode 128 0 0 0 0 126 0 0 ada_ccb 272 0 0 0 0 30 0 0 swblk 136 0 0 0 0 62 0 0 swpctrie 152 0 0 0 0 62 0 0 cdg_qdiffsample 16 0 0 0 0 254 0 0 pf state scrubs 40 0 0 0 0 254 0 0 pf frag entries 40 0 0 0 0 254 0 0 pf frags 256 0 0 0 0 62 0 0 pf table entries 160 0 0 0 0 254 0 0 pf table entry counters 64 0 0 0 0 254 0 0 pf UDP mappings 104 0 0 0 0 126 0 0 pf source nodes 152 0 0 0 0 254 0 0 pf state keys 88 0 0 0 0 126 0 0 pf states 384 0 0 0 0 254 0 0 pf tags 104 0 0 0 0 126 0 0 pf mtags 184 0 0 0 0 62 0 0 tcp_rack_pcb 1088 0 0 0 0 8 0 0 tcp_rack_map 128 0 0 0 0 126 0 0 tcp_bbr_pcb 896 0 0 0 0 16 0 0 tcp_bbr_map 128 0 0 0 0 126 0 0 tfo_ccache_entries 80 0 0 0 0 126 0 0 tfo 4 0 0 0 0 254 0 0 sackhole 32 0 0 0 0 254 0 0 ipq 56 0 0 0 0 254 0 0 sctp_asconf_ack 48 0 0 0 0 254 0 0 sctp_asconf 40 0 0 0 0 254 0 0 sctp_readq 152 0 0 0 0 254 0 0 tcp_log_id_node 120 0 0 0 0 126 0 0 tcp_log_id_bucket 176 0 0 0 0 62 0 0 tcp_log 416 0 0 0 0 254 0 0 tcpreass 48 0 0 0 0 254 0 0 udplite_inpcb 408 0 0 0 0 30 0 0 IPsec SA lft_c 16 0 0 0 0 254 0 0 AIOCB 552 0 0 0 0 16 0 0 mqnotifier 216 0 0 0 0 62 0 0 mvdata 64 0 0 0 0 254 0 0 mqueue 248 0 0 0 0 62 0 0 TMPFS node 240 0 0 0 0 62 0 0 NCLNODE 608 0 0 0 0 16 0 0 LTS VFS Cache 360 0 0 0 0 30 0 0 L VFS Cache 320 0 0 0 0 30 0 0 STS VFS Cache 144 0 0 0 0 62 0 0 cryptop 280 0 0 0 0 30 0 0 linux_dma_object 32 0 0 0 0 254 0 0 linux_dma_pctrie 152 0 0 0 0 62 0 0 IOMMU_MAP_ENTRY 112 0 0 0 0 126 0 0 skbuff 1808 0 0 0 0 8 0 0 mbuf_jumbo_16k 16384 0 0 0 0 254 0 0 mbuf_jumbo_9k 9216 0 0 0 0 254 0 0 audit_record 1280 0 0 0 0 8 0 0 domainset 40 0 0 0 0 254 0 0 MAC labels 40 0 0 0 0 254 0 0 vnpbuf 2624 0 0 0 0 16 0 0 nfspbuf 2624 0 0 0 0 4 0 0 swwbuf 2624 0 0 0 0 2 0 0 swrbuf 2624 0 0 0 0 4 0 0 umtx_shm 88 0 0 0 0 126 0 0 umtx pi 96 0 0 0 0 126 0 0 rangeset pctrie nodes 152 0 0 0 0 62 0 0 malloc-65536 65536 0 0 0 0 1 0 0 malloc-65536 65536 0 0 0 0 1 0 0 malloc-65536 65536 0 0 0 0 1 0 0 malloc-32768 32768 0 0 0 0 1 0 0 malloc-32768 32768 0 0 0 0 1 0 0 malloc-16384 16384 0 0 0 0 1 0 0 malloc-16384 16384 0 0 0 0 1 0 0 malloc-16384 16384 0 0 0 0 1 0 0 malloc-16384 16384 0 0 0 0 1 0 0 malloc-1024 1024 0 0 0 0 16 0 0 malloc-512 512 0 0 0 0 30 0 0 pcpu-32 32 0 0 0 0 254 0 0 pcpu-4 4 0 0 0 0 254 0 0 fakepg 104 0 0 0 0 126 0 0 UMA Hash 256 0 0 0 0 62 0 0