panic: /syzkaller/managers/i386/kernel/sys/kern/kern_timeout.c:607: callout_cc_add: Bad list head 0xfffffe00077c1380 first->prev != head cpuid = 0 time = 1758918223 KDB: stack backtrace: db_trace_self_wrapper() at db_trace_self_wrapper+0xc6/frame 0xfffffe0056bfe5b0 kdb_backtrace() at kdb_backtrace+0xd0/frame 0xfffffe0056bfe710 vpanic() at vpanic+0x257/frame 0xfffffe0056bfe8d0 panic() at panic+0xb5/frame 0xfffffe0056bfe990 callout_cc_add() at callout_cc_add+0x339/frame 0xfffffe0056bfe9f0 callout_reset_sbt_on() at callout_reset_sbt_on+0x74f/frame 0xfffffe0056bfeb10 sleepq_set_timeout_sbt() at sleepq_set_timeout_sbt+0x20b/frame 0xfffffe0056bfebd0 _sleep() at _sleep+0x468/frame 0xfffffe0056bfed30 random_kthread() at random_kthread+0x737/frame 0xfffffe0056bfeef0 fork_exit() at fork_exit+0xcc/frame 0xfffffe0056bfef30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe0056bfef30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- KDB: enter: panic [ thread pid 7 tid 100072 ] Stopped at kdb_enter+0x6e: movq $0,0x259e887(%rip) db> db> set $lines = 0 db> set $maxwidth = 0 db> show registers cs 0x20 ds 0x3b es 0x3b fs 0x13 gs 0x1b ss 0 rax 0x12 rcx 0xfffffe0002bf1850 rdx 0 rbx 0xffffffff82809a20 .str.27 rsp 0xfffffe0056bfe6f0 rbp 0xfffffe0056bfe710 rsi 0 rdi 0xffffffff81644bb9 printf+0x149 r8 0 r9 0xffffffff r10 0x555ad4489f26c5fd r11 0xfffffe0054192550 r12 0xfffffe00079f7000 r13 0xfffffffffffffffe r14 0xffffffff82809a20 .str.27 r15 0 rip 0xffffffff8162e30e kdb_enter+0x6e rflags 0x46 kdb_enter+0x6e: movq $0,0x259e887(%rip) db> show proc Process 7 (rand_harvestq) at 0xfffffe0054002ab0: state: NORMAL uid: 0 gid: 0 supp gids: 0 parent: pid 0 at 0xffffffff83b57080 ABI: null flag: 0x10000204 flag2: 0 reaper: 0xffffffff83b57080 reapsubtree: 7 sigparent: 20 vmspace: 0xffffffff83b58060 (map 0xffffffff83b58060) (map.pmap 0xffffffff83b58100) (pmap 0xffffffff83b58170) threads: 1 100072 Run - 0xffffffff839265d8 [rand_harvestq] db> ps pid ppid pgrp uid state wmesg wchan cmd 1233 764 764 0 S (threaded) syz-executor 100638 S nanslp 0xffffffff83badc41 syz-executor 100822 S sbwait 0xfffffe006b73420c syz-executor 100823 S select 0xfffffe0058612840 syz-executor 100824 S uwait 0xfffffe006e504000 syz-executor 1229 766 766 0 T (threaded) syz-executor 100740 s syz-executor 100819 D fork 0xffffffff8280e100 syz-executor 1228 763 763 0 S (threaded) syz-executor 100792 S nanslp 0xffffffff83badc41 syz-executor 100816 S aiowc 0xfffffe005419a8f0 syz-executor 100817 S uwait 0xfffffe007799ae00 syz-executor 1226 765 765 0 S (threaded) syz-executor 100643 S nanslp 0xffffffff83badc41 syz-executor 100813 S aiowc 0xfffffe0054184e40 syz-executor 100814 S uwait 0xfffffe007799af00 syz-executor 1221 1 763 0 S uwait 0xfffffe00584eae00 syz-executor 1178 0 0 0 DL mdwait 0xfffffe006e5e3000 [md0] 1152 1 764 0 S uwait 0xfffffe006df7f180 syz-executor 1150 1 765 0 SV sigwait 0xfffffe0054129b70 syz-executor 1143 1 763 0 S uwait 0xfffffe00584ec800 syz-executor 1142 1 763 0 S uwait 0xfffffe00584ea480 syz-executor 1138 1 763 0 S uwait 0xfffffe006e505080 syz-executor 1137 1 765 0 S uwait 0xfffffe006df7e180 syz-executor 1134 1 766 0 S uwait 0xfffffe00584ec780 syz-executor 1132 1 764 0 S uwait 0xfffffe006df7f480 syz-executor 1128 1 765 0 S uwait 0xfffffe006e503800 syz-executor 1124 1 766 0 S uwait 0xfffffe006e505180 syz-executor 1115 1 763 0 S uwait 0xfffffe006e504280 syz-executor 1110 1 764 0 S uwait 0xfffffe006df7e380 syz-executor 1108 1 766 0 S uwait 0xfffffe006df7f600 syz-executor 1102 1 766 0 S uwait 0xfffffe006df7f680 syz-executor 1100 1 764 0 S uwait 0xfffffe006e505280 syz-executor 1099 1 765 0 S uwait 0xfffffe006e504780 syz-executor 1094 1 763 0 S uwait 0xfffffe006df7e500 syz-executor 1086 1 763 0 S uwait 0xfffffe006df7e080 syz-executor 1085 1 765 0 S uwait 0xfffffe006df7e880 syz-executor 1080 1 766 0 S uwait 0xfffffe006df7e900 syz-executor 1078 1 763 0 S uwait 0xfffffe006e504580 syz-executor 1075 1 764 0 S uwait 0xfffffe0057d94580 syz-executor 1064 1 763 0 S uwait 0xfffffe006df7ef00 syz-executor 1062 1 764 0 S uwait 0xfffffe006df7f580 syz-executor 1055 1 765 0 S uwait 0xfffffe006e505600 syz-executor 1054 1 764 0 S uwait 0xfffffe00584eaf00 syz-executor 1050 1 766 0 S uwait 0xfffffe006df7f380 syz-executor 1049 1 763 0 S uwait 0xfffffe006df80e80 syz-executor 1044 1 764 0 S uwait 0xfffffe006df7f080 syz-executor 1033 1 763 0 S uwait 0xfffffe006df7f980 syz-executor 1025 1 766 0 S uwait 0xfffffe00584eb080 syz-executor 1019 1 763 0 S uwait 0xfffffe006df80000 syz-executor 1016 1 766 0 S uwait 0xfffffe006df7e280 syz-executor 1013 1 765 0 S uwait 0xfffffe006df7e580 syz-executor 1010 1 763 0 S uwait 0xfffffe0057d01500 syz-executor 1007 1 766 0 S uwait 0xfffffe006e505380 syz-executor 999 1 763 0 S uwait 0xfffffe00584ece00 syz-executor 995 1 764 0 S uwait 0xfffffe006df7ea00 syz-executor 991 0 0 0 DL mdwait 0xfffffe006b781000 [md2147483519] 985 1 763 0 S uwait 0xfffffe006df80a80 syz-executor 979 973 979 0 Ss select 0xfffffe0057df7140 dhclient 976 1 976 0 Ss select 0xfffffe0057df72c0 dhclient 973 962 423 65 S select 0xfffffe0057df6ec0 dhclient 968 0 0 0 DL (threaded) [so_splice] 100122 D - 0xfffffe0007786f00 [thr_0] 100290 D - 0xfffffe0007786f40 [thr_1] 965 1 766 0 S uwait 0xfffffe0057d94800 syz-executor 962 423 423 0 S wait 0xfffffe00540f3ac0 sh 951 1 765 0 S uwait 0xfffffe006df7f800 syz-executor 942 1 764 0 S uwait 0xfffffe00584eb280 syz-executor 919 1 763 0 S uwait 0xfffffe00584eb380 syz-executor 910 1 766 0 S uwait 0xfffffe0057d01700 syz-executor 909 1 764 0 S uwait 0xfffffe0057d94d80 syz-executor 908 1 763 0 S uwait 0xfffffe006e505480 syz-executor 903 1 765 0 S uwait 0xfffffe006df80100 syz-executor 900 1 763 0 S uwait 0xfffffe006df80b80 syz-executor 895 1 765 0 S uwait 0xfffffe006df80200 syz-executor 894 1 766 0 S uwait 0xfffffe006df7fa80 syz-executor 880 1 765 0 S uwait 0xfffffe0057d92080 syz-executor 875 1 766 0 S uwait 0xfffffe006df81200 syz-executor 804 0 0 0 DL aiordy 0xfffffe005410e568 [aiod4] 803 0 0 0 DL aiordy 0xfffffe005410eac0 [aiod3] 802 0 0 0 DL aiordy 0xfffffe00540f2560 [aiod2] 801 0 0 0 DL aiordy 0xfffffe00540e2008 [aiod1] 766 762 766 0 S nanslp 0xffffffff83badc41 syz-executor 765 762 765 0 S nanslp 0xffffffff83badc41 syz-executor 764 762 764 0 S nanslp 0xffffffff83badc41 syz-executor 763 762 763 0 S nanslp 0xffffffff83badc41 syz-executor 762 760 760 0 S select 0xfffffe0057df78c0 syz-executor 760 758 760 0 Ss sigsusp 0xfffffe00540f1b60 csh 758 681 758 0 Ss select 0xfffffe0057df7940 sshd 747 1 747 0 Ss+ ttyin 0xfffffe0058279cb0 getty 746 1 746 0 Ss+ ttyin 0xfffffe005862f0b0 getty 745 1 745 0 Ss+ ttyin 0xfffffe005862f4b0 getty 744 1 744 0 Ss+ ttyin 0xfffffe005862f8b0 getty 743 1 743 0 Ss+ ttyin 0xfffffe005862fcb0 getty 742 1 742 0 Ss+ ttyin 0xfffffe00586300b0 getty 741 1 741 0 Ss+ ttyin 0xfffffe00586304b0 getty 740 1 740 0 Ss+ ttyin 0xfffffe00586308b0 getty 739 1 739 0 Ss+ ttyin 0xfffffe0058630cb0 getty 685 1 685 0 Ss nanslp 0xffffffff83badc40 cron 681 1 681 0 Ss select 0xfffffe0057ddbc40 sshd 494 1 494 0 Ss select 0xfffffe0057ddbcc0 syslogd 423 1 423 0 Ss wait 0xfffffe0054004010 devd 422 1 422 65 Ss select 0xfffffe00586137c0 dhclient 337 1 337 0 Ss select 0xfffffe0057de0040 dhclient 334 1 334 0 Ss select 0xfffffe00586138c0 dhclient 16 0 0 0 DL syncer 0xffffffff83ccb820 [syncer] 15 0 0 0 DL vlruwt 0xfffffe000780a018 [vnlru] 14 0 0 0 DL (threaded) [bufdaemon] 100079 D psleep 0xffffffff83cc9d60 [bufdaemon] 100082 D - 0xffffffff83001ec0 [bufspacedaemon-0] 100095 D sdflush 0xfffffe005862d8e8 [/ worker] 9 0 0 0 DL psleep 0xffffffff83d14cc0 [vmdaemon] 8 0 0 0 DL (threaded) [pagedaemon] 100077 D psleep 0xffffffff83cfad88 [dom0] 100080 D launds 0xffffffff83cfad94 [laundry: dom0] 100081 D umarcl 0xffffffff81e148e0 [uma] 7 0 0 0 RL - 0xffffffff839265d8 [rand_harvestq] 6 0 0 0 DL pftm 0xffffffff844d9f20 [pf purge] 5 0 0 0 DL waiting 0xffffffff848c5700 [sctp_iterator] 4 0 0 0 DL (threaded) [cam] 100045 D - 0xffffffff838f0340 [doneq0] 100046 D - 0xffffffff838f02c0 [async] 100075 D - 0xffffffff838f0140 [scanner] 3 0 0 0 DL (threaded) [crypto] 100042 D crypto_ 0xffffffff83cf6660 [crypto] 100043 D crypto_ 0xfffffe0053ee4d30 [crypto returns 0] 100044 D crypto_ 0xfffffe0053ee4d80 [crypto returns 1] 13 0 0 0 DL (threaded) [geom] 100037 D - 0xffffffff83b56640 [g_event] 100038 D - 0xffffffff83b56660 [g_up] 100039 D - 0xffffffff83b56680 [g_down] 2 0 0 0 WL (threaded) [clock] 100031 I [clock (0)] 100032 I [clock (1)] 12 0 0 0 WL (threaded) [intr] 100013 I [swi6: task queue] 100014 I [swi6: Giant taskq] 100016 I [swi5: fast taskq] 100033 I [swi1: netisr 0] 100034 I [swi1: hpts] 100035 I [swi1: hpts] 100047 I [irq24: virtio_pci0] 100048 I [irq25: virtio_pci0] 100049 I [irq26: virtio_pci0] 100050 I [irq27: virtio_pci0] 100051 I [irq28: virtio_pci1] 100052 I [irq29: virtio_pci1] 100053 I [irq30: virtio_pci1] 100054 I [irq31: virtio_pci1] 100055 I [irq32: virtio_pci1] 100060 I [irq10: virtio_pci2] 100062 I [irq1: atkbd0] 100063 I [irq12: psm0] 100064 I [swi0: uart uart++] 100068 I [swi1: pf send] 11 0 0 0 RL (threaded) [idle] 100003 CanRun [idle: cpu0] 100004 Run CPU 1 [idle: cpu1] 1 0 1 0 SLs wait 0xfffffe0007809010 [init] 10 0 0 0 DL audit_w 0xffffffff83cf7100 [audit] 0 0 0 0 DLs (threaded) [kernel] 100000 D parked 0xffffffff84c53ff0 [swapper] 100005 D - 0xfffffe0053ea0100 [softirq_0] 100006 D - 0xfffffe0053ea0000 [softirq_1] 100007 D - 0xfffffe0053e9fe00 [if_io_tqg_0] 100008 D - 0xfffffe0053e9fd00 [if_io_tqg_1] 100009 D - 0xfffffe0053e9fc00 [if_config_tqg_0] 100010 D - 0xfffffe000776aa00 [kqueue_ctx taskq] 100011 D - 0xfffffe000776a900 [jail_remove taskq] 100012 D - 0xfffffe000776a800 [bus taskq] 100015 D - 0xfffffe000776a500 [thread taskq] 100017 D - 0xfffffe000776a300 [aiod_kick taskq] 100018 D - 0xfffffe000776a200 [deferred_unmount ta] 100019 D - 0xfffffe000776a100 [inm_free taskq] 100020 D - 0xfffffe000776a000 [in6m_free taskq] 100021 D - 0xfffffe0007769e00 [linuxkpi_irq_wq] 100022 D - 0xfffffe0007769d00 [linuxkpi_short_wq_0] 100023 D - 0xfffffe0007769d00 [linuxkpi_short_wq_1] 100024 D - 0xfffffe0007769d00 [linuxkpi_short_wq_2] 100025 D - 0xfffffe0007769d00 [linuxkpi_short_wq_3] 100026 D - 0xfffffe0007769c00 [linuxkpi_long_wq_0] 100027 D - 0xfffffe0007769c00 [linuxkpi_long_wq_1] 100028 D - 0xfffffe0007769c00 [linuxkpi_long_wq_2] 100029 D - 0xfffffe0007769c00 [linuxkpi_long_wq_3] 100036 D - 0xfffffe0007769a00 [firmware taskq] 100040 D - 0xfffffe0007769900 [crypto_0] 100041 D - 0xfffffe0007769900 [crypto_1] 100056 D - 0xfffffe0057de6d00 [vtnet0 rxq 0] 100057 D - 0xfffffe0057de6c00 [vtnet0 txq 0] 100058 D - 0xfffffe0057de6b00 [vtnet0 rxq 1] 100059 D - 0xfffffe0057de6a00 [vtnet0 txq 1] 100061 D vtbslp 0xfffffe0057de0380 [virtio_balloon] 100065 D - 0xffffffff8280e101 [deadlkres] 100069 D - 0xfffffe00593fd500 [acpi_task_0] 100070 D - 0xfffffe00593fd500 [acpi_task_1] 100071 D - 0xfffffe00593fd500 [acpi_task_2] 100073 D - 0xfffffe000776ab00 [mca taskq] 100074 D - 0xfffffe0007769700 [CAM taskq] 100076 D - 0xfffffe00593fd400 [ipsec_offload] db> show all locks Process 1233 (syz-executor) thread 0xfffffe0054163780 (100822) exclusive sx so_rcv_sx (so_rcv_sx) r = 0 (0xfffffe006b7341c0) locked @ /syzkaller/managers/i386/kernel/sys/kern/uipc_socket.c:4835 Process 1229 (syz-executor) thread 0xfffffe0054119780 (100819) exclusive sx MD config lock (MD config lock) r = 0 (0xffffffff8391b400) locked @ /syzkaller/managers/i386/kernel/sys/dev/md/md.c:1805 db> show malloc Type InUse MemUse Requests pf_hash 6 12804K 6 linker 384 5188K 587 tcp_hpts 7 4801K 7 devbuf 4187 4324K 4217 sysctloid 35359 2083K 35434 vtbuf 24 1968K 46 kobj 331 1324K 513 newblk 26 1031K 4884 vfscache 3 1025K 3 filedesc 122 970K 693 pcb 137 747K 292 inodedep 18 519K 562 ufs_quota 1 512K 1 vfs_hash 1 512K 1 callout 2 512K 2 subproc 231 478K 1357 intr 4 472K 4 vnet_data 2 224K 2 acpitask 1 224K 1 KTRACE 102 201K 1155 acpica 1674 184K 54444 vmem 5 146K 8 tidhash 3 141K 3 pagedep 10 131K 298 tfo_ccache 1 128K 1 IP reass 1 128K 1 ifnet 61 121K 64 DEVFS1 108 108K 125 sem 4 106K 4 gtaskqueue 18 98K 18 bus 1000 82K 5086 ifaddr 391 75K 393 mtx_pool 3 74K 3 syncache 1 68K 1 NFSD srvcache 3 68K 3 lltable 259 68K 259 module 523 66K 523 ddb_capture 1 64K 1 kdtrace 299 55K 2060 umtx 416 52K 416 BPF 77 43K 240 hostcache 1 32K 1 shm 1 32K 1 DEVFS3 127 32K 137 msg 4 30K 4 kbdmux 6 28K 6 temp 249 25K 2402 plimit 64 24K 378 DEVFS_RULE 56 20K 56 LRO 18 19K 18 pf_ifnet 65 18K 182 ufs_mount 4 17K 5 proc 3 17K 3 tty 16 16K 16 routetbl 135 16K 425 ithread 90 15K 90 bus-sc 34 15K 1657 eventhandler 163 14K 163 sctp_atcl 34 13K 41 ether_multi 152 13K 172 GEOM 59 13K 495 md_disk 4 13K 10 kenv 95 12K 95 CAM queue 5 11K 1528 rman 82 10K 437 kqueue 113 10K 1358 shmfd 4 10K 8 rpc 8 9K 8 cred 23 9K 213 in6_multi 65 9K 65 bmsafemap 2 9K 519 devstat 4 9K 4 UART 12 9K 12 ksem 1 8K 1 pfs_vncache 1 8K 1 audit_evclass 240 8K 306 mld 60 8K 60 igmp 60 8K 60 taskqueue 69 8K 96 gif 54 7K 54 pwddesc 108 7K 1262 sglist 6 7K 6 CAM DEV 3 6K 510 pfs_nodes 22 6K 22 ip6ndp 66 6K 67 ufs_dirhash 24 5K 24 UMA 270 5K 270 vt 11 5K 11 memdesc 1 4K 1 MCA 32 4K 32 md_sectors 1 4K 1 evdev 4 4K 4 acpisem 28 4K 28 proc-args 129 4K 2357 DEVFSP 49 4K 316 mount 20 4K 277 selfd 48 3K 18526 terminal 11 3K 11 session 21 3K 37 lockf 24 3K 636 acpidev 20 3K 20 inpcbpolicy 75 3K 1028 hhook 8 3K 10 clone 9 3K 9 uidinfo 3 3K 13 kcovinfo 36 3K 36 netlink 2 3K 136 CC Mem 17 3K 578 local_apic 1 2K 1 io_apic 1 2K 1 ipsec-saq 2 2K 2 Unitno 33 2K 300 sctp_ifa 13 2K 14 crypto 5 2K 82 CAM XPT 22 2K 543 indirdep 6 2K 292 in_multi 6 2K 10 tun 4 2K 4 select 12 2K 106 toponodes 6 2K 6 ipsecpolicy 2 2K 2 msi 9 2K 9 sctp_atky 34 2K 41 freework 5 2K 529 softdep 1 1K 1 mkdir 8 1K 558 sahead 1 1K 1 secasvar 1 1K 1 nhops 6 1K 8 vnodemarker 2 1K 36 NFSD session 1 1K 1 CAM periph 4 1K 271 osd 22 1K 595 sctp_ifn 6 1K 14 ipsec 3 1K 3 diradd 6 1K 471 freeblks 3 1K 287 pfil 6 1K 6 isadev 6 1K 6 pci_link 10 1K 10 newdirblk 5 1K 279 encap_export_host 12 1K 12 sctp_athm 34 1K 41 dirrem 2 1K 453 freefile 4 1K 340 ioctlops 1 1K 151 cdev 2 1K 2 lkpikmalloc 8 1K 9 counter_rate 13 1K 13 cryptodev 6 1K 230 freefrag 3 1K 333 chacha20random 1 1K 1 biobuf 1 1K 1 vnodes 1 1K 3 procdesc 2 1K 12 NFSD lckfile 1 1K 1 NFSD V4client 1 1K 1 DEVFS 9 1K 10 CAM SIM 2 1K 2 feeder 7 1K 7 tcpfunc 3 1K 3 loginclass 3 1K 5 prison 6 1K 6 nexusdev 8 1K 8 apmdev 1 1K 1 atkbddev 2 1K 2 ip_msource 2 1K 2 VN POLL 1 1K 12 aio 4 1K 4 pmchooks 1 1K 1 CAM path 4 1K 1034 CAM dev queue 2 1K 2 CAM I/O Scheduler 1 1K 1 soname 4 1K 3546 sctp_vrf 1 1K 1 vnet 1 1K 1 pmc 1 1K 1 sigio 1 1K 2 entropy 2 1K 34 acpiintr 1 1K 1 cpus 2 1K 2 vnet_data_free 1 1K 1 Per-cpu 1 1K 1 filecaps 2 1K 97 p1003.1b 1 1K 1 mqdata 0 0K 0 sctp_mcore 0 0K 0 sctp_socko 0 0K 0 sctp_iter 0 0K 11 sctp_mvrf 0 0K 0 sctp_timw 0 0K 0 sctp_cpal 0 0K 0 sctp_cmsg 0 0K 0 sctp_stre 0 0K 0 sctp_athi 0 0K 0 sctp_a_it 0 0K 11 sctp_aadr 0 0K 0 sctp_stro 0 0K 0 sctp_stri 0 0K 0 sctp_map 0 0K 0 tcp_pcm_rack 0 0K 0 tcp_do_rack 0 0K 0 tcp_fsb_rack 0 0K 0 filemon 0 0K 2 pf_table 0 0K 0 pf 0 0K 0 pf_rule 0 0K 0 pf_altq 0 0K 0 pf_osfp 0 0K 0 pf_krule_item 0 0K 0 pf_temp 0 0K 0 ipcomp 0 0K 0 esp 0 0K 0 ah 0 0K 0 madt_table 0 0K 2 smartpqi 0 0K 0 ixl 0 0K 0 ice-resmgr 0 0K 0 ice-osdep 0 0K 0 ice 0 0K 0 iavf 0 0K 0 axgbe 0 0K 0 fpukern_ctx 0 0K 0 xen_intr 0 0K 0 xen_hvm 0 0K 0 legacydrv 0 0K 0 NMI handlers 0 0K 0 bounce 0 0K 0 busdma 0 0K 0 qpidrv 0 0K 0 dmar_idpgtbl 0 0K 0 dmar_dom 0 0K 0 dmar_ctx 0 0K 0 amdiommu_dom 0 0K 0 amdiommu_ctx 0 0K 0 isci 0 0K 0 iommu_dmamap 0 0K 0 hyperv_socket 0 0K 0 bxe_ilt 0 0K 0 aesni_data 0 0K 0 xenbus 0 0K 0 vm_fictitious 0 0K 0 UMAHash 0 0K 0 vm_pgdata 0 0K 0 jblocks 0 0K 0 savedino 0 0K 227 sentinel 0 0K 0 jfsync 0 0K 0 jtrunc 0 0K 0 sbdep 0 0K 17 jsegdep 0 0K 0 jseg 0 0K 0 jfreefrag 0 0K 0 jfreeblk 0 0K 0 jnewblk 0 0K 0 jmvref 0 0K 0 jremref 0 0K 0 jaddref 0 0K 0 freedep 0 0K 0 allocindir 0 0K 0 allocdirect 0 0K 0 ufs_trim 0 0K 0 mactemp 0 0K 0 audit_trigger 0 0K 0 audit_pipe_presel 0 0K 0 audit_pipeent 0 0K 0 audit_pipe 0 0K 0 audit_evname 0 0K 0 audit_bsm 0 0K 0 audit_gidset 0 0K 0 audit_text 0 0K 0 audit_path 0 0K 0 audit_data 0 0K 0 audit_cred 0 0K 0 ktls_ocf 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5E_TLS_RX 0 0K 0 MLX5EEPROM 0 0K 0 MLX5E_TLS 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EN 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5DUMP 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 MLX5EEPROM 0 0K 0 simple_attr 0 0K 0 seq_file 0 0K 0 lkpiskb 0 0K 0 radix 0 0K 0 idr 0 0K 0 lkpindev 0 0K 0 lkpimhi 0 0K 0 lkpifw 0 0K 0 lkpi80211 0 0K 0 NLM 0 0K 0 ipsec-spdcache 0 0K 0 ipsec-reg 0 0K 0 ipsec-misc 0 0K 0 ipsecrequest 0 0K 0 ip6opt 0 0K 91 ip6_msource 0 0K 0 ip6_moptions 0 0K 1 in6_mfilter 0 0K 0 frag6 0 0K 0 tcplog 0 0K 0 tcp_hwpace 0 0K 0 ip_moptions 0 0K 6 in_mfilter 0 0K 6 ipid 0 0K 0 80211scan 0 0K 0 80211ratectl 0 0K 0 80211power 0 0K 0 80211nodeie 0 0K 0 80211node 0 0K 0 80211mesh_gt 0 0K 0 80211mesh_rt 0 0K 0 80211perr 0 0K 0 80211prep 0 0K 0 80211preq 0 0K 0 80211dfs 0 0K 0 80211crypto 0 0K 0 80211vap 0 0K 0 iflib 0 0K 0 vlan 0 0K 0 ifdescr 0 0K 0 zlib 0 0K 20 fadvise 0 0K 0 statfs 0 0K 208 namei_tracker 0 0K 3 inotify 0 0K 14 export_host 0 0K 0 cl_savebuf 0 0K 159 lio 0 0K 66 acl 0 0K 0 mbuf_tag 0 0K 0 ktls 0 0K 0 accf 0 0K 0 pts 0 0K 0 timerfd 0 0K 0 iov 0 0K 14806 eventfd 0 0K 0 Witness 0 0K 0 stack 0 0K 0 sbuf 0 0K 212 firmware 0 0K 0 compressor 0 0K 0 SWAP 0 0K 0 sysctltmp 0 0K 675 sysctl 0 0K 3 ekcd 0 0K 0 dumper 0 0K 0 sendfile 0 0K 130 rctl 0 0K 0 cache 0 0K 0 jaildesc 0 0K 0 prison_racct 0 0K 0 Fail Points 0 0K 0 filedesc_to_leader 0 0K 0 pwd 0 0K 0 tty console 0 0K 0 boottrace 0 0K 0 isofs_node 0 0K 0 isofs_mount 0 0K 0 tr_raid5_data 0 0K 0 tr_raid1e_data 0 0K 0 tr_raid1_data 0 0K 0 tr_raid0_data 0 0K 0 tr_concat_data 0 0K 0 md_sii_data 0 0K 0 md_promise_data 0 0K 0 md_nvidia_data 0 0K 0 md_jmicron_data 0 0K 0 md_intel_data 0 0K 0 md_ddf_data 0 0K 0 raid_data 0 0K 84 geom_flashmap 0 0K 0 tmpfs dir 0 0K 0 tmpfs name 0 0K 0 tmpfs mount 0 0K 0 tmpfs extattr 0 0K 0 NFS FHA 0 0K 0 newnfsmnt 0 0K 0 newnfsclient_req 0 0K 0 NFSCL layrecall 0 0K 0 NFSCL session 0 0K 0 NFSCL sockreq 0 0K 0 NFSCL devinfo 0 0K 0 NFSCL flayout 0 0K 0 NFSCL layout 0 0K 0 NFSD rollback 0 0K 0 NFSCL diroff 0 0K 0 NEWNFSnode 0 0K 0 NFSCL lck 0 0K 0 NFSCL lckown 0 0K 0 NFSCL client 0 0K 0 NFSCL deleg 0 0K 0 NFSCL open 0 0K 0 NFSCL owner 0 0K 0 NFS fh 0 0K 0 NFS req 0 0K 0 NFSD usrgroup 0 0K 0 NFSD string 0 0K 0 NFSD V4lock 0 0K 0 NFSD V4state 0 0K 0 msdosfs_fat 0 0K 0 msdosfs_mount 0 0K 0 msdosfs_node 0 0K 0 DEVFS4 0 0K 0 DEVFS2 0 0K 0 gntdev 0 0K 0 privcmd_dev 0 0K 0 evtchn_dev 0 0K 0 xenstore 0 0K 0 xnb 0 0K 0 xen_acpi 0 0K 0 xbbd 0 0K 0 xbd 0 0K 0 Balloon 0 0K 0 sysmouse 0 0K 0 vtfont 0 0K 0 pvscsi 0 0K 0 USBdev 0 0K 0 USB 0 0K 0 ufshci 0 0K 0 twsbuf 0 0K 0 tcp_log_dev 0 0K 1 midi buffers 0 0K 0 mixer 0 0K 0 ac97 0 0K 0 hdacc 0 0K 0 hdac 0 0K 0 hdaa 0 0K 0 SIIS driver 0 0K 0 PUC 0 0K 0 ppbusdev 0 0K 0 sr_iov 0 0K 0 OCS 0 0K 0 OCS 0 0K 0 nvme 0 0K 0 nvd 0 0K 0 netmap 0 0K 0 mwldev 0 0K 0 MVS driver 0 0K 0 mpi3mrbuf 0 0K 0 mrsasbuf 0 0K 0 mpt_user 0 0K 0 mps_user 0 0K 0 MPSSAS 0 0K 0 mps 0 0K 0 mpr_user 0 0K 0 MPRSAS 0 0K 0 mpr 0 0K 0 mfibuf 0 0K 0 malodev 0 0K 0 LED 0 0K 0 ix_sriov 0 0K 0 ix 0 0K 0 ipsbuf 0 0K 0 ciss_data 0 0K 0 BACKLIGHT 0 0K 0 ath_hal 0 0K 0 athdev 0 0K 0 ata_pci 0 0K 0 ata_dma 0 0K 0 ata_generic 0 0K 0 AHCI driver 0 0K 0 agp 0 0K 0 acpipwr 0 0K 0 acpi_perf 0 0K 0 acpicmbat 0 0K 0 aacraidcam 0 0K 0 aacraid_buf 0 0K 0 aaccam 0 0K 0 aacbuf 0 0K 0 zstd 0 0K 0 XZ_DEC 0 0K 0 nvlist 0 0K 182 SCSI ENC 0 0K 0 SCSI sa 0 0K 0 scsi_pass 0 0K 0 scsi_da 0 0K 70 ata_da 0 0K 0 scsi_ch 0 0K 0 scsi_cd 0 0K 0 nvme_da 0 0K 0 CAM CCB 0 0K 523 CAM ccb queue 0 0K 0 db> show uma Zone Size Used Free Requests Sleeps Bucket Total Mem XFree mbuf_jumbo_page 4096 8321 1077 19250 0 254 38494208 0 RADIX NODE 152 21664 1601 78902 0 62 3536280 0 mbuf 256 9663 254 38635 0 254 2538752 0 tcp_log 416 5169 168 11347 0 254 2220192 0 BUF TRIE 152 518 11286 3033 0 62 1794208 0 malloc-384 384 4140 30 4154 0 30 1601280 0 malloc-128 128 11730 143 11748 0 126 1519744 0 malloc-4096 4096 334 4 596 0 2 1384448 0 UMA Slabs 0 112 11148 6 11148 0 126 1249248 0 malloc-65536 65536 14 2 19 0 1 1048576 0 malloc-16384 16384 60 4 344 0 1 1048576 0 mbuf_cluster 2048 508 0 508 0 254 1040384 0 vmem btag 56 17252 139 17252 0 254 973896 0 FFS inode 1168 615 29 959 0 8 752192 0 sctp_ep 1152 34 477 41 0 254 588672 0 VM OBJECT 248 2204 100 19559 0 62 571392 0 socket 1024 127 381 2610 0 254 520192 0 lkpicurr 168 2 3094 2 0 62 520128 0 malloc-4096 4096 118 4 1243 0 2 499712 0 pbuf 2624 0 178 0 0 2 467072 0 THREAD 1860 190 18 824 0 8 386880 0 MAP ENTRY 96 3218 310 63816 0 126 338688 0 VNODE 440 654 66 1000 0 30 316800 0 malloc-2048 2048 141 11 389 0 8 311296 0 256 Bucket 2048 139 13 1135 0 8 311296 0 malloc-256 256 197 988 5281 0 62 303360 0 malloc-64 64 3883 212 3892 0 254 262080 0 malloc-16 16 14571 179 14687 0 254 236000 0 DEVCTL 1024 74 146 206 0 0 225280 0 malloc-64 64 658 2429 21615 0 254 197568 0 malloc-32768 32768 0 6 164 0 1 196608 0 UMA Zones 768 242 2 242 0 16 187392 0 FPU_save_area 832 192 33 1110 0 16 187200 0 malloc-32 32 5355 315 5419 0 254 181440 0 malloc-128 128 1227 168 25763 0 126 178560 0 FFS2 dinode 256 615 75 959 0 62 176640 0 lkpimm 56 1 3095 1 0 254 173376 0 PROC 1368 107 14 1234 0 8 165528 0 unpcb 320 15 501 1446 0 254 165120 0 malloc-2048 2048 62 10 63 0 8 147456 0 malloc-2048 2048 6 66 1050 0 8 147456 0 malloc-1024 1024 120 24 308 0 16 147456 0 S VFS Cache 104 1023 381 1532 0 126 146016 0 filedesc0 1072 108 18 1262 0 8 135072 0 malloc-65536 65536 0 2 77 0 1 131072 0 malloc-65536 65536 0 2 110 0 1 131072 0 malloc-65536 65536 2 0 2 0 1 131072 0 malloc-65536 65536 1 1 9 0 1 131072 0 malloc-32768 32768 2 2 14 0 1 131072 0 malloc-128 128 823 200 5206 0 126 130944 0 mbuf_packet 256 7 501 691 0 254 130048 0 ksiginfo 112 105 939 210 0 126 116928 0 malloc-128 128 673 226 2162 0 126 115072 0 g_bio 408 0 270 10419 0 30 110160 0 malloc-32768 32768 3 0 3 0 1 98304 0 64 Bucket 512 121 71 12522 0 30 98304 0 malloc-256 256 257 118 1001 0 62 96000 0 malloc-8192 8192 9 2 92 0 1 90112 0 UMA Kegs 384 229 4 229 0 30 89472 0 malloc-64 64 855 468 3085 0 254 84672 0 malloc-16384 16384 5 0 7 0 1 81920 0 malloc-4096 4096 16 4 242 0 2 81920 0 malloc-384 384 100 110 989 0 30 80640 0 malloc-256 256 142 173 1637 0 62 80640 0 malloc-256 256 186 129 908 0 62 80640 0 malloc-256 256 236 79 927 0 62 80640 0 pcpu-8 8 9503 225 12380 0 254 77824 0 malloc-8192 8192 8 1 11 0 1 73728 0 128 Bucket 1024 49 18 277 0 16 68608 0 malloc-32768 32768 0 2 6 0 1 65536 0 malloc-32768 32768 0 2 120 0 1 65536 0 malloc-32768 32768 2 0 2 0 1 65536 0 malloc-1024 1024 26 38 558 0 16 65536 0 VMSPACE 584 85 27 1199 0 16 65408 0 32 Bucket 256 86 169 5805 0 62 65280 0 tcp_inpcb 1304 17 31 578 0 8 62592 0 malloc-384 384 126 24 139 0 30 57600 0 malloc-384 384 128 22 128 0 30 57600 0 malloc-4096 4096 13 1 496 0 2 57344 0 AIOCB 552 65 33 196 0 16 54096 0 malloc-64 64 429 390 2123 0 254 52416 0 malloc-128 128 84 319 359 0 126 51584 0 malloc-256 256 10 185 818 0 62 49920 0 DIRHASH 1024 34 14 34 0 16 49152 0 NAMEI 1024 0 48 15910 0 16 49152 0 malloc-16384 16384 1 2 191 0 1 49152 0 malloc-2048 2048 11 13 44 0 8 49152 0 malloc-1024 1024 18 30 1108 0 16 49152 0 da_ccb 544 0 84 2939 0 16 45696 0 ripcb 376 48 72 232 0 30 45120 0 syncache 168 0 264 4 0 254 44352 0 Files 80 418 132 9916 0 126 44000 0 TURNSTILE 136 209 106 209 0 62 42840 0 clpbuf 2624 0 16 231 0 4 41984 0 vnpbuf 2624 0 16 2 0 16 41984 0 malloc-4096 4096 6 4 107 0 2 40960 0 pipe 736 20 35 401 0 16 40480 0 udp_inpcb 408 10 80 201 0 30 36720 0 malloc-64 64 21 546 14512 0 254 36288 0 malloc-64 64 31 536 64 0 254 36288 0 malloc-64 64 117 450 1324 0 254 36288 0 malloc-128 128 22 257 182 0 126 35712 0 malloc-128 128 11 268 679 0 126 35712 0 malloc-128 128 122 157 596 0 126 35712 0 routing nhops 256 28 107 62 0 62 34560 0 ttyoutq 256 72 63 160 0 62 34560 0 malloc-384 384 59 31 599 0 30 34560 0 malloc-256 256 33 102 561 0 62 34560 0 malloc-256 256 73 62 502 0 62 34560 0 SLEEPQUEUE 88 209 175 209 0 126 33792 0 malloc-16384 16384 0 2 4 0 1 32768 0 malloc-8192 8192 3 1 4 0 1 32768 0 malloc-2048 2048 0 16 14 0 8 32768 0 malloc-2048 2048 3 13 134 0 8 32768 0 malloc-2048 2048 10 6 35 0 8 32768 0 malloc-1024 1024 3 29 53 0 16 32768 0 malloc-1024 1024 2 30 6 0 16 32768 0 malloc-1024 1024 4 28 11 0 16 32768 0 malloc-1024 1024 2 30 2 0 16 32768 0 malloc-512 512 0 64 121 0 30 32768 0 malloc-512 512 12 52 41 0 30 32768 0 malloc-512 512 0 64 16 0 30 32768 0 malloc-512 512 2 62 49 0 30 32768 0 malloc-512 512 10 54 32 0 30 32768 0 malloc-512 512 3 61 82 0 30 32768 0 malloc-512 512 6 58 169 0 30 32768 0 pcpu-64 64 501 11 501 0 254 32768 0 ttyinq 160 135 65 300 0 62 32000 0 PGRP 120 25 239 41 0 126 31680 0 sctp_laddr 48 0 588 12 0 254 28224 0 rl_entry 48 5 583 43 0 254 28224 0 malloc-32 32 157 725 1474 0 254 28224 0 malloc-32 32 433 449 4446 0 254 28224 0 16 Bucket 144 80 116 712 0 62 28224 0 4 Bucket 48 8 580 23 0 254 28224 0 AIO 208 3 130 224 0 62 27664 0 udplite_inpcb 408 0 63 17 0 30 25704 0 cpuset 200 7 121 1703 0 62 25600 0 malloc-4096 4096 0 6 22 0 2 24576 0 malloc-4096 4096 1 5 6 0 2 24576 0 ertt_txseginfo 40 0 606 2152 0 254 24240 0 PWD 40 71 535 349 0 254 24240 0 rtentry 168 31 113 62 0 62 24192 0 8 Bucket 80 53 247 470 0 126 24000 0 itimer 352 1 65 2 0 30 23232 0 malloc-384 384 12 48 14 0 30 23040 0 malloc-384 384 1 59 2 0 30 23040 0 domainset 40 0 567 1693 0 254 22680 0 Mountpoints 2816 2 6 5 0 4 22528 0 hostcache 64 2 313 2 0 254 20160 0 ertt 72 17 263 578 0 126 20160 0 malloc-64 64 3 312 11 0 254 20160 0 malloc-32 32 116 514 1233 0 254 20160 0 malloc-32 32 69 561 670 0 254 20160 0 malloc-32 32 115 515 153 0 254 20160 0 malloc-32 32 135 495 146 0 254 20160 0 malloc-32 32 16 614 32 0 254 20160 0 2 Bucket 32 66 564 488 0 254 20160 0 KNOTE 160 0 125 60 0 62 20000 0 cryptop 280 0 70 5 0 30 19600 0 L VFS Cache 320 0 60 41 0 30 19200 0 AIOLIO 272 0 70 33 0 30 19040 0 vmem 1856 2 7 2 0 8 16704 0 epoch_record pcpu 256 4 60 4 0 62 16384 0 malloc-16384 16384 1 0 1 0 1 16384 0 malloc-16384 16384 1 0 1 0 1 16384 0 malloc-8192 8192 0 2 42 0 1 16384 0 malloc-8192 8192 0 2 20 0 1 16384 0 malloc-8192 8192 2 0 2 0 1 16384 0 malloc-2048 2048 1 7 2 0 8 16384 0 malloc-1024 1024 0 16 2 0 16 16384 0 malloc-512 512 2 30 2 0 30 16384 0 SMR CPU 32 8 503 8 0 254 16352 0 ipq 56 0 288 1 0 254 16128 0 malloc-16 16 511 489 3671 0 254 16000 0 tcp_log_id_node 120 0 132 1 0 126 15840 0 kenv 258 17 43 1072 0 30 15480 0 mqnode 416 3 33 3 0 30 14976 0 SMR SHARED 24 8 503 8 0 254 12264 0 tcp_log_id_bucket 176 0 69 1 0 62 12144 0 malloc-16 16 90 660 1964 0 254 12000 0 malloc-16 16 10 740 16 0 254 12000 0 malloc-16 16 257 493 358 0 254 12000 0 malloc-16 16 56 694 25684 0 254 12000 0 malloc-16 16 16 734 17 0 254 12000 0 malloc-16 16 41 709 51 0 254 12000 0 malloc-384 384 1 29 2 0 30 11520 0 malloc-8192 8192 1 0 1 0 1 8192 0 pcpu-16 16 8 504 8 0 254 8192 0 vtnet_tx_hdr 24 0 334 3048 0 254 8016 0 UMA Slabs 1 176 18 4 18 0 62 3872 0 KMAP ENTRY 96 12 27 14 0 0 3744 0 md2147483519 4 0 0 0 0 254 0 0 splice 184 0 0 0 0 62 0 0 FFS1 dinode 128 0 0 0 0 126 0 0 ada_ccb 272 0 0 0 0 30 0 0 swblk 136 0 0 0 0 62 0 0 swpctrie 152 0 0 0 0 62 0 0 cdg_qdiffsample 16 0 0 0 0 254 0 0 pf state scrubs 40 0 0 0 0 254 0 0 pf frag entries 40 0 0 0 0 254 0 0 pf fragment node 72 0 0 0 0 126 0 0 pf frags 232 0 0 0 0 62 0 0 pf table entries 160 0 0 0 0 254 0 0 pf table entry counters 64 0 0 0 0 254 0 0 pf Ethernet anchors 1240 0 0 0 0 64 0 0 pf anchors 1664 0 0 0 0 64 0 0 pf UDP mappings 104 0 0 0 0 126 0 0 pf source nodes 152 0 0 0 0 254 0 0 pf state keys 88 0 0 0 0 126 0 0 pf states 384 0 0 0 0 254 0 0 pf tags 104 0 0 0 0 126 0 0 pf mtags 184 0 0 0 0 62 0 0 tcp_rack_pcb 1088 0 0 0 0 8 0 0 tcp_rack_map 128 0 0 0 0 126 0 0 tcp_bbr_pcb 896 0 0 0 0 16 0 0 tcp_bbr_map 128 0 0 0 0 126 0 0 tfo_ccache_entries 80 0 0 0 0 126 0 0 tfo 4 0 0 0 0 254 0 0 sackhole 32 0 0 0 0 254 0 0 sctp_asconf_ack 48 0 0 0 0 254 0 0 sctp_asconf 40 0 0 0 0 254 0 0 sctp_stream_msg_out 112 0 0 0 0 254 0 0 sctp_readq 152 0 0 0 0 254 0 0 sctp_chunk 152 0 0 0 0 254 0 0 sctp_raddr 736 0 0 0 0 254 0 0 sctp_asoc 2256 0 0 0 0 254 0 0 tcpreass 48 0 0 0 0 254 0 0 IPsec SA lft_c 16 0 0 0 0 254 0 0 mqnotifier 216 0 0 0 0 62 0 0 mvdata 64 0 0 0 0 254 0 0 mqueue 248 0 0 0 0 62 0 0 TMPFS node 240 0 0 0 0 62 0 0 NCLNODE 608 0 0 0 0 16 0 0 LTS VFS Cache 360 0 0 0 0 30 0 0 STS VFS Cache 144 0 0 0 0 62 0 0 linux_dma_object 32 0 0 0 0 254 0 0 linux_dma_pctrie 152 0 0 0 0 62 0 0 IOMMU_MAP_ENTRY 112 0 0 0 0 126 0 0 skbuff 1808 0 0 0 0 8 0 0 mbuf_jumbo_16k 16384 0 0 0 0 254 0 0 mbuf_jumbo_9k 9216 0 0 0 0 254 0 0 audit_record 1280 0 0 0 0 8 0 0 MAC labels 40 0 0 0 0 254 0 0 nfspbuf 2624 0 0 0 0 4 0 0 swwbuf 2624 0 0 0 0 2 0 0 swrbuf 2624 0 0 0 0 4 0 0 umtx_shm 88 0 0 0 0 126 0 0 umtx pi 96 0 0 0 0 126 0 0 rangeset pctrie nodes 152 0 0 0 0 62 0 0 malloc-65536 65536 0 0 0 0 1 0 0 malloc-65536 65536 0 0 0 0 1 0 0 malloc-65536 65536 0 0 0 0 1 0 0 malloc-32768 32768 0 0 0 0 1 0 0 malloc-32768 32768 0 0 0 0 1 0 0 malloc-16384 16384 0 0 0 0 1 0 0 malloc-16384 16384 0 0 0 0 1 0 0 malloc-8192 8192 0 0 0 0 1 0 0 malloc-4096 4096 0 0 0 0 2 0 0 pcpu-32 32 0 0 0 0 254 0 0 pcpu-4 4 0 0 0 0 254 0 0 fakepg 88 0 0 0 0 126 0 0 UMA Hash 256 0 0 0 0 62 0 0 db> show all tcpcbs/bl