syzbot


panic: /syzkaller/managers/main/kernel/sys/kern/kern_timeout.c:LINE: _callout_stop_safe: Bad link elm ADDR prev->next !=

Status: upstream: reported on 2025/05/12 11:48
Reported-by: syzbot+c0cc5646a14707f4470b@syzkaller.appspotmail.com
First crash: 19d, last: 19d

Sample crash report:
panic: /syzkaller/managers/main/kernel/sys/kern/kern_timeout.c:1323: _callout_stop_safe: Bad link elm 0xfffffe0008027b50 prev->next != elm
cpuid = 1
time = 14
KDB: stack backtrace:
db_trace_self_wrapper() at db_trace_self_wrapper+0xc6/frame 0xfffffe0057573b50
kdb_backtrace() at kdb_backtrace+0xd0/frame 0xfffffe0057573cb0
vpanic() at vpanic+0x257/frame 0xfffffe0057573e70
panic() at panic+0xb5/frame 0xfffffe0057573f30
_callout_stop_safe() at _callout_stop_safe+0xc62/frame 0xfffffe0057574050
exit1() at exit1+0x323/frame 0xfffffe00575740f0
sigexit() at sigexit+0x451/frame 0xfffffe0057574a70
postsig() at postsig+0x2c5/frame 0xfffffe0057574bf0
ast_sig() at ast_sig+0x85b/frame 0xfffffe0057574ea0
ast_handler() at ast_handler+0x2b0/frame 0xfffffe0057574f10
ast() at ast+0x25/frame 0xfffffe0057574f30
doreti_ast() at doreti_ast+0x1c/frame 0x8202baac0
KDB: enter: panic
[ thread pid 1252 tid 100107 ]
Stopped at      kdb_enter+0x6e: movq    $0,0x25bda97(%rip)
db> 
db> set $lines = 0
db> set $maxwidth = 0
db> show registers
cs                        0x20
ds                        0x3b
es                        0x3b
fs                        0x13
gs                        0x1b
ss                        0x28
rax                       0x12
rcx         0xfffffe00033eee30
rdx                          0
rbx         0xffffffff827aff60  .str.27
rsp         0xfffffe0057573c90
rbp         0xfffffe0057573cb0
rsi                          0
rdi         0xffffffff81614589  printf+0x149
r8                           0
r9                  0xffffffff
r10                        0x1
r11                        0x3
r12         0xfffffe00548dc740
r13         0xfffffffffffffffd
r14         0xffffffff827aff60  .str.27
r15                          0
rip         0xffffffff815fe6fe  kdb_enter+0x6e
rflags                    0x46
kdb_enter+0x6e: movq    $0,0x25bda97(%rip)
db> show proc
Process 1252 (getty) at 0xfffffe0008027ae0:
 state: NORMAL
 uid: 0  gids: 0
 parent: pid 1 at 0xfffffe0008009040
 ABI: FreeBSD ELF64
 flag: 0x10006002  flag2: 0xc0000
 arguments: /usr/libexec/getty Pc ttyv5
 reaper: 0xfffffe0008009040 reapsubtree: 1
 sigparent: 20
 vmspace: 0xfffffe00548ab6d8
   (map 0xfffffe00548ab6d8)
   (map.pmap 0xfffffe00548ab778)
   (pmap 0xfffffe00548ab7e8)
 threads: 1
100107                   Run     CPU 1                       getty
db> ps
  pid  ppid  pgrp   uid  state   wmesg   wchan               cmd
 1344     1     1     0  R       CPU 0                       init
 1343     1  1343     0  Ss+     ttyin   0xfffffe005476d8b0  getty
 1342     1  1342     0  Ss+     ttyin   0xfffffe005476e0b0  getty
 1341     1  1341     0  Ss+     ttyin   0xfffffe005476e8b0  getty
 1340     1  1340     0  Ss+     ttyin   0xfffffe005476f0b0  getty
 1339   765   765     0  R       (threaded)                  syz-executor
100094                   RunQ                                syz-executor
101028                   D       ufs     0xfffffe0078f78ac0  syz-executor
101030                   D       biowr   0xfffffe0008565af8  syz-executor
101031                   D       ufs     0xfffffe0078f78ac0  syz-executor
 1338  1337   767     0  SV      select  0xfffffe0078f7ae40  syz-executor
 1337   767   767     0  D       (threaded)                  syz-executor
100891                   S       nanslp  0xffffffff83b9d500  syz-executor
101027                   D       ppwait  0xfffffe0054809ac0  syz-executor
101029                   S       uwait   0xfffffe0079268c80  syz-executor
 1336   766   766     0  D       (threaded)                  syz-executor
100934                   S       nanslp  0xffffffff83b9d500  syz-executor
101023                   S       sapblk  0xffffffff83b8b800  syz-executor
101025                   D       reapst  0xfffffe0057972a08  syz-executor
 1334   868   868     0  D       (threaded)                  syz-executor
101005                   S       nanslp  0xffffffff83b9d500  syz-executor
101017                   S       uwait   0xfffffe0079269480  syz-executor
101021                   S       uwait   0xfffffe007926a700  syz-executor
101024                   S       uwait   0xfffffe0079268200  syz-executor
101026                   D       getblk  0xfffffe0008565b78  syz-executor
 1329     1   868     0  S       uwait   0xfffffe007926a300  syz-executor
 1323     1   765     0  S       uwait   0xfffffe006eb70000  syz-executor
 1321     1   765     0  S       uwait   0xfffffe0058950200  syz-executor
 1320     1   765     0  S       uwait   0xfffffe0058951600  syz-executor
 1314     1   766     0  SV      uwait   0xfffffe0058950000  syz-executor
 1300     1   766     0  S       uwait   0xfffffe0058950a80  syz-executor
 1294     1   766     0  S       uwait   0xfffffe0058950c80  syz-executor
 1290     1   765     0  SV      uwait   0xfffffe0058d0aa80  syz-executor
 1287     1   767     0  S       uwait   0xfffffe0058950880  syz-executor
 1281     1   765     0  S       uwait   0xfffffe0058952480  syz-executor
 1280     1   868     0  SV      uwait   0xfffffe0079269d80  syz-executor
 1277     1   766     0  S       uwait   0xfffffe0079269200  syz-executor
 1275     1   766     0  S       uwait   0xfffffe0079269600  syz-executor
 1264     1   765     0  S       uwait   0xfffffe0058950d80  syz-executor
 1252     1  1252     0  REs+    CPU 1                       getty
 1251     1  1251     0  TLs+                                getty
 1250     1  1250     0  Ss+     ttyin   0xfffffe0059cbf8b0  getty
 1249     1  1249     0  Ss+     ttyin   0xfffffe0058a9c8b0  getty
 1247     1   868     0  S       uwait   0xfffffe005894f680  syz-executor
 1237     1   868     0  S       uwait   0xfffffe007926b080  syz-executor
 1234     1   767     0  S       uwait   0xfffffe005894f100  syz-executor
 1207     1   765     0  S       uwait   0xfffffe0079268400  syz-executor
 1204     1   766     0  SV      uwait   0xfffffe0079269400  syz-executor
 1200     1   765     0  S       uwait   0xfffffe0079269c80  syz-executor
 1199     1   766     0  S       uwait   0xfffffe0079269100  syz-executor
 1198     1   765     0  S       uwait   0xfffffe0058d0a300  syz-executor
 1197     1   868     0  S       uwait   0xfffffe007926b100  syz-executor
 1180     1   765    -1  S       uwait   0xfffffe006eb6e800  syz-executor
 1175     1   765 60928  S       uwait   0xfffffe0079268900  syz-executor
 1172     1   766     0  S       uwait   0xfffffe0079269300  syz-executor
 1168     0     0     0  DL      -       0xffffffff83cad400  [soaiod4]
 1167     0     0     0  DL      -       0xffffffff83cad400  [soaiod3]
 1166     0     0     0  DL      -       0xffffffff83cad400  [soaiod2]
 1165     0     0     0  DL      -       0xffffffff83cad400  [soaiod1]
 1164     1   765     0  S       uwait   0xfffffe0079268d80  syz-executor
 1160     1   767     0  SV      uwait   0xfffffe006eb6e600  syz-executor
 1158     1   766     0  S       uwait   0xfffffe0079268e80  syz-executor
 1156     1   765     0  T                                   syz-executor
 1155     1   765     0  S       uwait   0xfffffe0058d0a900  syz-executor
 1153     1   868 60929  SV      sigwait 0xfffffe0054977670  syz-executor
 1150     1   766     0  S       uwait   0xfffffe0058d0a500  syz-executor
 1137     1   767     0  S       uwait   0xfffffe006eb6ec00  syz-executor
 1136     1   767     0  S       uwait   0xfffffe007926b000  syz-executor
 1134     1   767     0  S       uwait   0xfffffe006eb6e900  syz-executor
 1133     1   767     0  S       uwait   0xfffffe006eb6eb00  syz-executor
 1132     0     0     0  DL      (threaded)                  [so_splice]
100690                   D       -       0xfffffe0058e3bf00  [thr_0]
100691                   D       -       0xfffffe0058e3bf40  [thr_1]
 1128     1   868     0  S       uwait   0xfffffe0058952380  syz-executor
 1122     1   868     0  S       uwait   0xfffffe0058d0a400  syz-executor
 1120     1   766     0  S       uwait   0xfffffe0058d07780  syz-executor
 1115     1   765     0  S       uwait   0xfffffe0007f7e700  syz-executor
 1112     1   868     0  S       uwait   0xfffffe0058d07a80  syz-executor
 1103     1  1098     0  S       umtxn   0xfffffe0058d0a600  syz-executor
 1102     1  1098     0  S       uwait   0xfffffe006eb6db80  syz-executor
 1101     1  1098     0  S       uwait   0xfffffe005894f300  syz-executor
 1100     1  1098     0  S       uwait   0xfffffe0058950580  syz-executor
 1096     1   868     0  S       uwait   0xfffffe005894f280  syz-executor
 1092     1   765     0  SV      uwait   0xfffffe006eb6da80  syz-executor
 1081     1   868     0  T                                   syz-executor
 1058     1   868     0  S       uwait   0xfffffe006eb6e500  syz-executor
 1057     1   767     0  S       uwait   0xfffffe005894f500  syz-executor
 1044     1   868     0  S       uwait   0xfffffe0058d0a700  syz-executor
 1040     1   766     0  SV      uwait   0xfffffe006eb6ea00  syz-executor
 1026     1   868     0  T       uwait   0xfffffe005894fa80  syz-executor
 1020     1   765     0  SV      uwait   0xfffffe006eb6d880  syz-executor
  999     1   868     0  S       uwait   0xfffffe0058d0a800  syz-executor
  986     1   766     0  S       uwait   0xfffffe006eb6f280  syz-executor
  984     1   766     0  S       uwait   0xfffffe0007f7e600  syz-executor
  972     1   765     0  S       uwait   0xfffffe006eb6d980  syz-executor
  969     1   766     0  S       uwait   0xfffffe005894f700  syz-executor
  965     1   766     0  S       uwait   0xfffffe0007f7e180  syz-executor
  960     1   765     0  S       uwait   0xfffffe006eb6d780  syz-executor
  948     1   868     0  S       uwait   0xfffffe006eb6e700  syz-executor
  940     0     0     0  DL      mdwait  0xfffffe0078d5d000  [md1]
  939     1   766     0  S       uwait   0xfffffe006eb6f180  syz-executor
  926     1   766     0  S       uwait   0xfffffe0058952280  syz-executor
  909   900   909     0  Ss      select  0xfffffe0078b656c0  dhclient
  900   884   424    65  S       select  0xfffffe0059e7ec40  dhclient
  897     1   765     0  S       uwait   0xfffffe006eb6f380  syz-executor
  887     0     0     0  DL      (threaded)                  [KTLS]
100143                   D       -       0xfffffe0007f77700  [thr_0]
100244                   D       -       0xfffffe0007f77780  [thr_1]
100245                   D       -       0xffffffff83caec28  [reclaim_0]
  886     1   767     0  S       uwait   0xfffffe0058d08300  syz-executor
  884     1   424     0  S       wait    0xfffffe00548c9000  sh
  874     1   765     0  S       uwait   0xfffffe0058952800  syz-executor
  868   763   868     0  S       nanslp  0xffffffff83b9d500  syz-executor
  864     1   765     0  S       uwait   0xfffffe0007f7ea00  syz-executor
  854     1   765     0  S       uwait   0xfffffe0058952c00  syz-executor
  849     1   766     0  S       uwait   0xfffffe0058d08280  syz-executor
  844     1   767     0  S       uwait   0xfffffe0058d08180  syz-executor
  841     0     0     0  DL      -       0xffffffff83b47d40  [accounting]
  823     1   767     0  S       uwait   0xfffffe0058d07e00  syz-executor
  821     0     0     0  DL      aiordy  0xfffffe00548ec5a0  [aiod5]
  820     0     0     0  DL      aiordy  0xfffffe00548ecb00  [aiod4]
  819     0     0     0  DL      aiordy  0xfffffe005490eae0  [aiod3]
  817     0     0     0  DL      aiordy  0xfffffe005490f040  [aiod1]
  767   763   767     0  S       nanslp  0xffffffff83b9d500  syz-executor
  766   763   766     0  S       nanslp  0xffffffff83b9d500  syz-executor
  765   763   765     0  S       nanslp  0xffffffff83b9d500  syz-executor
  763   761   761     0  S       select  0xfffffe006e3d9c40  syz-executor
  761   759   761     0  Ss      pause   0xfffffe00548ea0b0  csh
  759     1   759     0  Ss      select  0xfffffe006e3d9cc0  sshd
  737     1    18     0  S+      nanslp  0xffffffff83b9d501  sleep
   17     0     0     0  DL      syncer  0xffffffff83cbada0  [syncer]
   16     0     0     0  DL      vlruwt  0xfffffe0008029060  [vnlru]
   15     0     0     0  DL      (threaded)                  [bufdaemon]
100080                   D       psleep  0xffffffff83cb9360  [bufdaemon]
100083                   D       -       0xffffffff83002140  [bufspacedaemon-0]
100092                   D       sdflush 0xfffffe0058ddf8e8  [/ worker]
    9     0     0     0  DL      psleep  0xffffffff83d04380  [vmdaemon]
    8     0     0     0  DL      (threaded)                  [pagedaemon]
100078                   D       psleep  0xffffffff83cea2f8  [dom0]
100081                   D       launds  0xffffffff83cea304  [laundry: dom0]
100082                   D       umarcl  0xffffffff81dceb60  [uma]
    7     0     0     0  DL      -       0xffffffff8391acd0  [rand_harvestq]
    6     0     0     0  TL      pftm    0xffffffff845d7850  [pf purge]
    5     0     0     0  DL      waiting 0xffffffff8449f700  [sctp_iterator]
    4     0     0     0  DL      (threaded)                  [cam]
100046                   D       -       0xffffffff838e5340  [doneq0]
100047                   D       -       0xffffffff838e52c0  [async]
100076                   D       -       0xffffffff838e5140  [scanner]
    3     0     0     0  DL      (threaded)                  [crypto]
100043                   D       crypto_ 0xffffffff83ce5b40  [crypto]
100044                   D       crypto_ 0xfffffe0058566330  [crypto returns 0]
100045                   D       crypto_ 0xfffffe0058566380  [crypto returns 1]
   14     0     0     0  DL      seqstat 0xfffffe00547edc88  [sequencer 00]
   13     0     0     0  DL      (threaded)                  [geom]
100037                   D       -       0xffffffff83b45f00  [g_event]
100038                   D       -       0xffffffff83b45f20  [g_up]
100039                   D       -       0xffffffff83b45f40  [g_down]
    2     0     0     0  WL      (threaded)                  [clock]
100031                   I                                   [clock (0)]
100032                   I                                   [clock (1)]
   12     0     0     0  WL      (threaded)                  [intr]
100013                   I                                   [swi6: task queue]
100014                   I                                   [swi6: Giant taskq]
100016                   I                                   [swi5: fast taskq]
100033                   I                                   [swi1: netisr 0]
100034                   I                                   [swi1: hpts]
100035                   I                                   [swi1: hpts]
100048                   I                                   [irq24: virtio_pci0]
100049                   I                                   [irq25: virtio_pci0]
100050                   I                                   [irq26: virtio_pci0]
100051                   I                                   [irq27: virtio_pci0]
100052                   I                                   [irq28: virtio_pci1]
100053                   I                                   [irq29: virtio_pci1]
100054                   I                                   [irq30: virtio_pci1]
100055                   I                                   [irq31: virtio_pci1]
100056                   I                                   [irq32: virtio_pci1]
100061                   I                                   [irq10: virtio_pci2]
100063                   I                                   [irq1: atkbd0]
100064                   I                                   [irq12: psm0]
100065                   I                                   [swi0: uart uart++]
100069                   I                                   [swi1: pf send]
   11     0     0     0  RL      (threaded)                  [idle]
100003                   CanRun                              [idle: cpu0]
100004                   CanRun                              [idle: cpu1]
    1     0     1     0  SLs     wait    0xfffffe0008009040  [init]
   10     0     0     0  DL      audit_w 0xffffffff83ce65e0  [audit]
    0     0     0     0  DLs     (threaded)                  [kernel]
100000                   D       parked  0xffffffff84c2cff0  [swapper]
100005                   D       -       0xfffffe005462d700  [softirq_0]
100006                   D       -       0xfffffe005462d600  [softirq_1]
100007                   D       -       0xfffffe005462d500  [if_io_tqg_0]
100008                   D       -       0xfffffe005462d400  [if_io_tqg_1]
100009                   D       -       0xfffffe005462d300  [if_config_tqg_0]
100010                   D       -       0xfffffe0008bf9500  [kqueue_ctx taskq]
100011                   D       -       0xfffffe0008bf9300  [jail_remove taskq]
100012                   D       -       0xfffffe0008bf9100  [bus taskq]
100015                   s                                   [thread taskq]
100017                   D       -       0xfffffe0008bf8600  [aiod_kick taskq]
100018                   D       -       0xfffffe0008bf8400  [deferred_unmount ta]
100019                   D       -       0xfffffe0008bf8200  [inm_free taskq]
100020                   D       -       0xfffffe0008bf8000  [in6m_free taskq]
100021                   D       -       0xfffffe0008bf7d00  [linuxkpi_irq_wq]
100022                   D       -       0xfffffe0008bf7b00  [linuxkpi_short_wq_0]
100023                   D       -       0xfffffe0008bf7b00  [linuxkpi_short_wq_1]
100024                   D       -       0xfffffe0008bf7b00  [linuxkpi_short_wq_2]
100025                   D       -       0xfffffe0008bf7b00  [linuxkpi_short_wq_3]
100026                   D       -       0xfffffe0008bf7600  [linuxkpi_long_wq_0]
100027                   D       -       0xfffffe0008bf7600  [linuxkpi_long_wq_1]
100028                   D       -       0xfffffe0008bf7600  [linuxkpi_long_wq_2]
100029                   D       -       0xfffffe0008bf7600  [linuxkpi_long_wq_3]
100036                   D       -       0xfffffe0008bf6a00  [firmware taskq]
100041                   D       -       0xfffffe0008bf6300  [crypto_0]
100042                   D       -       0xfffffe0008bf6300  [crypto_1]
100057                   D       -       0xfffffe0058565900  [vtnet0 rxq 0]
100058                   D       -       0xfffffe0058565800  [vtnet0 txq 0]
100059                   D       -       0xfffffe0058565700  [vtnet0 rxq 1]
100060                   D       -       0xfffffe0058565600  [vtnet0 txq 1]
100062                   D       vtbslp  0xfffffe005858d000  [virtio_balloon]
100066                   D       -       0xffffffff827b52a0  [deadlkres]
100070                   D       -       0xfffffe0058d0dd00  [acpi_task_0]
100071                   D       -       0xfffffe0058d0dd00  [acpi_task_1]
100072                   D       -       0xfffffe0058d0dd00  [acpi_task_2]
100074                   D       -       0xfffffe0008bfa100  [mca taskq]
100075                   D       -       0xfffffe0058566000  [CAM taskq]
100077                   D       -       0xfffffe0058564400  [ipsec_offload]
100149                   D       -       0xfffffe006ebdb200  [system_taskq_0]
100150                   D       -       0xfffffe006ebdb200  [system_taskq_1]
100151                   D       -       0xfffffe0008bfa300  [system_delay_taskq_]
100152                   D       -       0xfffffe0008bfa300  [system_delay_taskq_]
100153                   D       -       0xfffffe006ebda900  [arc_prune]
100154                   D       -       0xfffffe006ebda700  [arc_flush_0]
100155                   D       -       0xfffffe006ebda700  [arc_flush_1]
100169                   D       -       0xfffffe006ebda300  [dbu_evict]
100188                   D       -       0xfffffe0058d0bb00  [z_vdev_file_0]
100189                   D       -       0xfffffe0058d0bb00  [z_vdev_file_1]
100190                   D       -       0xfffffe0058d0bb00  [z_vdev_file_2]
100191                   D       -       0xfffffe0058d0bb00  [z_vdev_file_3]
100192                   D       -       0xfffffe0058d0bb00  [z_vdev_file_4]
100193                   D       -       0xfffffe0058d0bb00  [z_vdev_file_5]
100194                   D       -       0xfffffe0058d0bb00  [z_vdev_file_6]
100195                   D       -       0xfffffe0058d0bb00  [z_vdev_file_7]
100196                   D       -       0xfffffe0058d0bb00  [z_vdev_file_8]
100197                   D       -       0xfffffe0058d0bb00  [z_vdev_file_9]
100198                   D       -       0xfffffe0058d0bb00  [z_vdev_file_10]
100199                   D       -       0xfffffe0058d0bb00  [z_vdev_file_11]
100200                   D       -       0xfffffe0058d0bb00  [z_vdev_file_12]
100201                   D       -       0xfffffe0058d0bb00  [z_vdev_file_13]
100202                   D       -       0xfffffe0058d0bb00  [z_vdev_file_14]
100203                   D       -       0xfffffe0058d0bb00  [z_vdev_file_15]
100222                   D       -       0xfffffe006ebd9700  [zfsvfs]
100939                   D       -       0xfffffe0078f45400  [netlink_socket (PID]
db> show all locks
Process 1339 (syz-executor) thread 0xfffffe0054990740 (101030)
exclusive lockmgr bufwait (bufwait) r = 0 (0xfffffe0008565b78) locked @ /syzkaller/managers/main/kernel/sys/kern/vfs_bio.c:4023
exclusive lockmgr ufs (ufs) r = 0 (0xfffffe0078b54750) locked @ /syzkaller/managers/main/kernel/sys/kern/vfs_subr.c:3373
exclusive lockmgr ufs (ufs) r = 0 (0xfffffe0078f78ac0) locked @ /syzkaller/managers/main/kernel/sys/kern/vfs_subr.c:3373
Process 1336 (syz-executor) thread 0xfffffe00549a4000 (101025)
exclusive sx sapblk (sapblk) r = 0 (0xffffffff83b8b800) locked @ /syzkaller/managers/main/kernel/sys/kern/kern_proc.c:3464
Process 1334 (syz-executor) thread 0xfffffe0054960740 (101026)
exclusive lockmgr ufs (ufs) r = 0 (0xfffffe006ebd7070) locked @ /syzkaller/managers/main/kernel/sys/kern/vfs_subr.c:3373
db> show malloc
              Type        InUse        MemUse     Requests
           pf_hash            6        12804K            6
            linker          401        12258K          941
          tcp_hpts            7         4801K            7
         sctp_stro            2         4616K           16
            devbuf         4188         4324K         4216
           solaris         2244         3597K         4470
         sysctloid        44676         2624K        44788
             vtbuf           24         1968K           46
          filedesc          160         1335K          990
              kobj          331         1324K          521
            newblk           53         1037K         4147
          vfscache            3         1025K            3
               pcb           88          730K          730
           subproc          275          567K         1489
          inodedep           37          526K         1080
         ufs_quota            1          512K            1
          vfs_hash            1          512K            1
           callout            2          512K            2
              intr            4          472K            4
         vnet_data            2          224K            2
          acpitask            1          224K            1
            KTRACE          102          201K        50272
            acpica         1674          184K        54450
              vmem            5          148K            9
           tidhash            3          141K            3
           pagedep           24          134K          530
        tfo_ccache            1          128K            1
          IP reass            1          128K            1
            DEVFS1          112          112K          132
               sem            4          106K            4
        gtaskqueue           18           98K           18
               bus         1008           82K         5098
          mtx_pool            3           74K            3
          syncache            1           68K            1
     NFSD srvcache            3           68K            3
           kdtrace          366           68K         2380
            module          525           66K          535
              umtx          512           64K          512
       ddb_capture            1           64K            1
              temp           44           40K         2554
               shm            3           36K           15
            DEVFS3          131           33K          142
         hostcache            1           32K            1
               msg            4           30K            4
            kbdmux            6           28K            6
               LRO           22           23K           24
               BPF           20           21K           68
        DEVFS_RULE           56           20K           56
            ifaddr           73           20K           79
        kstat_data           19           19K           19
          routetbl          185           18K          571
         ufs_mount            4           17K            5
              proc            3           17K            3
           lltable           53           17K           65
           filemon            2           16K           48
               tty           16           16K           16
           ithread           90           15K           90
            bus-sc           34           15K         1659
             ifnet            8           15K            8
      eventhandler          166           14K          166
       ether_multi          154           13K          239
             shmfd           10           12K           22
              kenv           95           12K           95
              GEOM           68           12K          544
         sctp_atcl           29           11K          322
              cred           42           11K          396
             mount           87           11K         1451
         CAM queue            5           11K         1528
         taskqueue           93           10K          114
              rman           82           10K          477
              ksem            4           10K            7
               rpc            8            9K            8
            kqueue          134            9K         1934
         in6_multi           65            9K           78
            plimit           22            9K          592
           devstat            4            9K            4
              UART           12            9K           12
         bmsafemap            1            8K          895
       pfs_vncache            1            8K            1
           pwddesc          127            8K         1386
            crypto           17            8K           81
     audit_evclass          239            8K          301
            sglist            6            7K            6
               UMA          338            7K          338
           CAM DEV            3            6K          510
         pfs_nodes           22            6K           22
       ufs_dirhash           24            5K           39
            CC Mem           37            5K          274
             lockf           42            5K          538
          pf_ifnet           11            5K           22
                vt           11            5K           11
           memdesc            1            4K            1
               MCA           32            4K           32
           md_disk            1            4K            5
             evdev            4            4K            4
            DEVFSP           62            4K          326
          freework           16            4K         1305
            ip6opt            9            4K           46
           acpisem           28            4K           28
            Unitno           72            4K          270
         proc-args          135            4K         2489
         sctp_timw           11            3K           11
          terminal           11            3K           11
           uidinfo            6            3K           16
           acpidev           20            3K           20
         newdirblk           19            3K          474
             hhook            8            3K           10
       inpcbpolicy           72            3K          723
             clone            9            3K            9
          kcovinfo           36            3K           45
        local_apic            1            2K            1
           io_apic            1            2K            1
         ipsec-saq            2            2K            2
            ip6ndp           13            2K           15
               tun            5            2K            5
           session           15            2K           56
             selfd           29            2K       125051
          indirdep            7            2K          818
          in_multi            7            2K           18
               osd           67            2K          321
          sctp_ifa           13            2K           16
         cryptodev           23            2K          323
           CAM XPT           22            2K          543
       vnodemarker            3            2K           53
         toponodes            6            2K            6
       ipsecpolicy            2            2K            2
               msi            9            2K            9
           netlink            2            2K          109
         sctp_atky           31            2K          340
           softdep            1            1K            1
            dirrem            4            1K          831
             mkdir            8            1K          948
            sahead            1            1K            1
          secasvar            1            1K            1
             nhops            6            1K            8
      NFSD session            1            1K            1
            diradd            7            1K          853
               mld            7            1K            7
              igmp            7            1K            7
        CAM periph            4            1K          271
             ipsec            3            1K            3
          sctp_ifn            6            1K           16
          freeblks            3            1K          585
              pfil            6            1K            6
            isadev            6            1K           14
          pci_link           10            1K           10
 encap_export_host           12            1K           12
            select            5            1K           80
filedesc_to_leader            9            1K           17
              cdev            2            1K            2
         sctp_athm           29            1K          324
       lkpikmalloc            8            1K            9
              ktls            3            1K          156
          freefile            3            1K          658
       in6_mfilter            6            1K           20
        ip_msource            6            1K           35
    chacha20random            1            1K            1
            biobuf            1            1K            1
           eventfd            3            1K            8
            vnodes            2            1K           16
          freefrag            2            1K          196
           VN POLL            2            1K           18
      NFSD lckfile            1            1K            1
     NFSD V4client            1            1K            1
             DEVFS            9            1K           10
           CAM SIM            2            1K            2
            prison            8            1K            8
            feeder            7            1K            7
             taskq            2            1K            2
       ip6_msource            3            1K            7
             frag6            2            1K            2
           tcpfunc            3            1K            3
        loginclass            3            1K            5
           pf_rule            1            1K            2
          nexusdev            8            1K            8
            apmdev            1            1K            1
          atkbddev            2            1K            2
          ktls_ocf            1            1K            5
               aio            4            1K           11
               iov            2            1K        18010
          pmchooks            1            1K            1
             sigio            2            1K            6
          CAM path            4            1K         1034
     CAM dev queue            2            1K            2
 CAM I/O Scheduler            1            1K            1
          filecaps            4            1K           82
          sctp_vrf            1            1K            1
          sctp_map            4            1K           32
      ip6_moptions            2            1K            9
       ip_moptions            1            1K           31
        in_mfilter            1            1K           65
              vnet            1            1K            1
               pmc            1            1K            1
           entropy            2            1K           35
          acpiintr            1            1K            1
              cpus            2            1K            2
    vnet_data_free            1            1K            1
           Per-cpu            1            1K            1
            soname            1            1K         3762
          p1003.1b            1            1K            1
         sfs_nodes            0            0K            0
        zones_data            0            0K            0
            ipcomp            0            0K            0
               esp            0            0K            0
                ah            0            0K            0
            mqdata            0            0K            0
      tcp_pcm_rack            0            0K            7
       tcp_do_rack            0            0K            0
      tcp_fsb_rack            0            0K           14
          pf_table            0            0K            0
           pf_altq            0            0K            0
           pf_osfp            0            0K            0
     pf_krule_item            0            0K            0
           pf_temp            0            0K            0
        sctp_mcore            0            0K            0
        sctp_socko            0            0K           47
         sctp_iter            0            0K           16
         sctp_mvrf            0            0K            0
         sctp_cpal            0            0K            0
         sctp_cmsg            0            0K            0
         sctp_stre            0            0K            0
         sctp_athi            0            0K            0
         sctp_a_it            0            0K           16
         sctp_aadr            0            0K            0
         sctp_stri            0            0K            4
        madt_table            0            0K            2
          smartpqi            0            0K            0
               ixl            0            0K            0
        ice-resmgr            0            0K            0
         ice-osdep            0            0K            0
               ice            0            0K            0
              iavf            0            0K            0
             axgbe            0            0K            0
       fpukern_ctx            0            0K            0
          xen_intr            0            0K            0
           xen_hvm            0            0K            0
         legacydrv            0            0K            0
      NMI handlers            0            0K            0
            bounce            0            0K            0
            busdma            0            0K            0
            qpidrv            0            0K            0
      dmar_idpgtbl            0            0K            0
          dmar_dom            0            0K            0
          dmar_ctx            0            0K            0
      amdiommu_dom            0            0K            0
      amdiommu_ctx            0            0K            0
              isci            0            0K            0
      iommu_dmamap            0            0K            0
     hyperv_socket            0            0K            0
           bxe_ilt            0            0K            0
        aesni_data            0            0K            4
            xenbus            0            0K            0
     vm_fictitious            0            0K            0
           UMAHash            0            0K            0
         vm_pgdata            0            0K            0
           jblocks            0            0K            0
          savedino            0            0K          826
          sentinel            0            0K            0
            jfsync            0            0K            0
            jtrunc            0            0K            0
             sbdep            0            0K           24
           jsegdep            0            0K            0
              jseg            0            0K            0
         jfreefrag            0            0K            0
          jfreeblk            0            0K            0
           jnewblk            0            0K            0
            jmvref            0            0K            0
           jremref            0            0K            0
           jaddref            0            0K            0
           freedep            0            0K            0
        allocindir            0            0K            0
       allocdirect            0            0K            0
          ufs_trim            0            0K            0
           mactemp            0            0K            0
     audit_trigger            0            0K            0
 audit_pipe_presel            0            0K            0
     audit_pipeent            0            0K            0
        audit_pipe            0            0K            0
      audit_evname            0            0K            0
         audit_bsm            0            0K            0
      audit_gidset            0            0K            0
        audit_text            0            0K            0
        audit_path            0            0K            0
        audit_data            0            0K            0
        audit_cred            0            0K            0
        MLX5EEPROM            0            0K            0
        MLX5EEPROM            0            0K            0
        MLX5EEPROM            0            0K            0
        MLX5EEPROM            0            0K            0
        MLX5EEPROM            0            0K            0
      MLX5E_TLS_RX            0            0K            0
        MLX5EEPROM            0            0K            0
         MLX5E_TLS            0            0K            0
        MLX5EEPROM            0            0K            0
        MLX5EEPROM            0            0K            0
        MLX5EEPROM            0            0K            0
            MLX5EN            0            0K            0
        MLX5EEPROM            0            0K            0
        MLX5EEPROM            0            0K            0
        MLX5EEPROM            0            0K            0
        MLX5EEPROM            0            0K            0
        MLX5EEPROM            0            0K            0
        MLX5EEPROM            0            0K            0
        MLX5EEPROM            0            0K            0
          MLX5DUMP            0            0K            0
        MLX5EEPROM            0            0K            0
        MLX5EEPROM            0            0K            0
        MLX5EEPROM            0            0K            0
        MLX5EEPROM            0            0K            0
       simple_attr            0            0K            0
          seq_file            0            0K            0
           lkpiskb            0            0K            0
             radix            0            0K            0
               idr            0            0K            0
          lkpindev            0            0K            0
           lkpimhi            0            0K            0
            lkpifw            0            0K            0
         lkpi80211            0            0K            0
               NLM            0            0K            0
    ipsec-spdcache            0            0K            0
         ipsec-reg            0            0K            0
        ipsec-misc            0            0K            0
      ipsecrequest            0            0K            0
            tcplog            0            0K            0
        tcp_hwpace            0            0K            0
              ipid            0            0K            0
         80211scan            0            0K            0
      80211ratectl            0            0K            0
        80211power            0            0K            0
       80211nodeie            0            0K            0
         80211node            0            0K            0
      80211mesh_gt            0            0K            0
      80211mesh_rt            0            0K            0
         80211perr            0            0K            0
         80211prep            0            0K            0
         80211preq            0            0K            0
          80211dfs            0            0K            0
       80211crypto            0            0K            0
          80211vap            0            0K            0
             iflib            0            0K            0
              vlan            0            0K            0
               gif            0            0K            0
           ifdescr            0            0K            0
              zlib            0            0K           30
           fadvise            0            0K            5
            statfs            0            0K          211
     namei_tracker            0            0K            3
       export_host            0            0K            0
        cl_savebuf            0            0K           94
               lio            0            0K           20
               acl            0            0K            0
          mbuf_tag            0            0K            0
              accf            0            0K            0
               pts            0            0K            0
           timerfd            0            0K            0
          procdesc            0            0K           10
          ioctlops            0            0K          248
           Witness            0            0K            0
             stack            0            0K            0
              sbuf            0            0K          500
          firmware            0            0K            0
        compressor            0            0K            0
              SWAP            0            0K            0
         sysctltmp            0            0K          710
            sysctl            0            0K            3
              ekcd            0            0K            0
            dumper            0            0K            0
          sendfile            0            0K          220
              rctl            0            0K            0
             cache            0            0K            0
      prison_racct            0            0K            0
       Fail Points            0            0K            0
               pwd            0            0K            0
       tty console            0            0K            0
         boottrace            0            0K            0
        isofs_node            0            0K            0
       isofs_mount            0            0K            0
     tr_raid5_data            0            0K            0
    tr_raid1e_data            0            0K            0
     tr_raid1_data            0            0K            0
     tr_raid0_data            0            0K            0
    tr_concat_data            0            0K            0
       md_sii_data            0            0K            0
   md_promise_data            0            0K            0
    md_nvidia_data            0            0K            0
   md_jmicron_data            0            0K            0
     md_intel_data            0            0K            0
       md_ddf_data            0            0K            0
         raid_data            0            0K           84
     geom_flashmap            0            0K            0
         tmpfs dir            0            0K            0
        tmpfs name            0            0K            0
       tmpfs mount            0            0K            0
     tmpfs extattr            0            0K            0
           NFS FHA            0            0K            0
         newnfsmnt            0            0K            0
  newnfsclient_req            0            0K            0
   NFSCL layrecall            0            0K            0
     NFSCL session            0            0K            0
     NFSCL sockreq            0            0K            0
     NFSCL devinfo            0            0K            0
     NFSCL flayout            0            0K            0
      NFSCL layout            0            0K            0
     NFSD rollback            0            0K            0
      NFSCL diroff            0            0K            0
        NEWNFSnode            0            0K            0
         NFSCL lck            0            0K            0
      NFSCL lckown            0            0K            0
      NFSCL client            0            0K            0
       NFSCL deleg            0            0K            0
        NFSCL open            0            0K            0
       NFSCL owner            0            0K            0
            NFS fh            0            0K            0
           NFS req            0            0K            0
     NFSD usrgroup            0            0K            0
       NFSD string            0            0K            0
       NFSD V4lock            0            0K            0
      NFSD V4state            0            0K            0
       msdosfs_fat            0            0K            0
     msdosfs_mount            0            0K            0
      msdosfs_node            0            0K            0
            DEVFS4            0            0K            0
            DEVFS2            0            0K            0
            gntdev            0            0K            0
       privcmd_dev            0            0K            0
        evtchn_dev            0            0K            0
          xenstore            0            0K            0
               xnb            0            0K            0
          xen_acpi            0            0K            0
              xbbd            0            0K            0
               xbd            0            0K            0
           Balloon            0            0K            0
          sysmouse            0            0K            0
            vtfont            0            0K            0
            pvscsi            0            0K            0
            USBdev            0            0K            0
               USB            0            0K            0
            twsbuf            0            0K            0
       tcp_log_dev            0            0K            4
      midi buffers            0            0K            0
             mixer            0            0K            0
              ac97            0            0K            0
             hdacc            0            0K            0
              hdac            0            0K            0
              hdaa            0            0K            0
       SIIS driver            0            0K            0
               PUC            0            0K            0
          ppbusdev            0            0K            0
            sr_iov            0            0K            0
               OCS            0            0K            0
               OCS            0            0K            0
              nvme            0            0K            0
               nvd            0            0K            0
            netmap            0            0K            0
            mwldev            0            0K            0
        MVS driver            0            0K            0
         mpi3mrbuf            0            0K            0
          mrsasbuf            0            0K            0
          mpt_user            0            0K            0
          mps_user            0            0K            0
            MPSSAS            0            0K            0
               mps            0            0K            0
          mpr_user            0            0K            0
            MPRSAS            0            0K            0
               mpr            0            0K            0
            mfibuf            0            0K            0
        md_sectors            0            0K            0
           malodev            0            0K            0
               LED            0            0K            0
          ix_sriov            0            0K            0
                ix            0            0K            0
            ipsbuf            0            0K            0
         ciss_data            0            0K            0
         BACKLIGHT            0            0K            0
           ath_hal            0            0K            0
            athdev            0            0K            0
           ata_pci            0            0K            0
           ata_dma            0            0K            0
       ata_generic            0            0K            0
       AHCI driver            0            0K            0
               agp            0            0K            0
           acpipwr            0            0K            0
         acpi_perf            0            0K            0
         acpicmbat            0            0K            0
        aacraidcam            0            0K            0
       aacraid_buf            0            0K            0
            aaccam            0            0K            0
            aacbuf            0            0K            0
              zstd            0            0K            0
            XZ_DEC            0            0K            0
            nvlist            0            0K          525
          SCSI ENC            0            0K            0
           SCSI sa            0            0K            0
         scsi_pass            0            0K            0
           scsi_da            0            0K           70
            ata_da            0            0K            0
           scsi_ch            0            0K            0
           scsi_cd            0            0K            0
           nvme_da            0            0K            0
           CAM CCB            0            0K          523
     CAM ccb queue            0            0K            0
db> show uma
              Zone   Size    Used    Free    Requests  Sleeps  Bucket  Total Mem    XFree
   mbuf_jumbo_page   4096    8332    1066       18357       0     254   38494208        0
              mbuf    256    8882     780       35364       0     254    2473472        0
        malloc-128    128   14627     129       18743       0     126    1888768        0
          BUF TRIE    152     418   11386        3329       0      62    1794208        0
        RADIX NODE    152   11041     706       53003       0      63    1785544        0
        malloc-384    384    4175      25        4230       0      30    1612800        0
      mbuf_cluster   2048     508     254         511       0     254    1560576        0
       malloc-4096   4096     349      13        1131       0       2    1482752        0
      malloc-16384  16384      84       1         508       0       1    1392640        0
       UMA Slabs 0    112   11422      20       11422       0     126    1281504        0
      malloc-65536  65536      16       2          22       0       1    1179648        0
         sctp_asoc   2256       2     508          16       0     254    1150560        0
         malloc-64     64     323   17002      125705       0     254    1108800        0
zio_buf_comb_1048576 1048576       0       1          15       0       1    1048576        0
         vmem btag     56   17825      70       17825       0     254    1002120        0
         FFS inode   1168     629      29        1306       0       8     768544        0
           sctp_ep   1152      27     484         304       0     254     588672        0
        malloc-256    256    2210      85        3710       0      62     587520        0
       malloc-4096   4096     128       2        1339       0       2     532480        0
            socket   1024     105     403        2432       0     254     520192        0
          lkpicurr    168       2    3094           2       0      62     520128        0
        256 Bucket   2048     218      22        1573       0       8     491520        0
         VM OBJECT    248    1806     130       19993       0      62     480128        0
              pbuf   2624       0     182           0       0       2     477568        0
            THREAD   1824     237      19        1031       0       8     466944        0
        sctp_raddr    736       2     515          21       0     254     380512        0
         malloc-64     64    5180     427       31284       0     254     358848        0
             VNODE    440     669      87        1349       0      30     332640        0
       malloc-8192   8192       6      33          94       0       1     319488        0
         MAP ENTRY     96    3040     236       67793       0     126     314496        0
         malloc-16     16   18355     395       33236       0     254     300000        0
       malloc-2048   2048     110      26         837       0       8     278528        0
      malloc-32768  32768       1       7         679       0       1     262144        0
         malloc-32     32    7440     498       21810       0     254     254016        0
         UMA Zones    768     310       4         310       0      16     241152        0
     FPU_save_area    832     239      40        1418       0      16     232128        0
            DEVCTL   1024      36     184         168       0       0     225280        0
           tcp_log    416       0     513         111       0     254     213408        0
              PROC   1376     127      16        1347       0       8     196768        0
      malloc-65536  65536       1       2          16       0       1     196608        0
        malloc-128    128    1329     190       25898       0     126     194432        0
       malloc-2048   2048      30      58         309       0       8     180224        0
       malloc-1024   1024     165      11         196       0      16     180224        0
       FFS2 dinode    256     629      61        1305       0      62     176640        0
            lkpimm     56       1    3095           1       0     254     173376        0
             unpcb    320       5     511        1348       0     254     165120        0
        malloc-256    256      54     576        5048       0      62     161280        0
       S VFS Cache    104    1064     457        1953       0     126     158184        0
         filedesc0   1072     127      13        1386       0       8     150080        0
zio_buf_comb_131072 131072       0       1           1       0       1     131072        0
      malloc-65536  65536       0       2          95       0       1     131072        0
      malloc-65536  65536       0       2         110       0       1     131072        0
      malloc-65536  65536       2       0           2       0       1     131072        0
      malloc-32768  32768       0       4          57       0       1     131072        0
      malloc-32768  32768       2       2           4       0       1     131072        0
       malloc-2048   2048       8      56         681       0       8     131072        0
       mbuf_packet    256      12     496        1335       0     254     130048        0
          ksiginfo    112     122     922         311       0     126     116928        0
          UMA Kegs    384     296       7         296       0      30     116352        0
        malloc-128    128     655     244        2619       0     126     115072        0
       malloc-8192   8192       7       6          23       0       1     106496        0
        malloc-384    384     187      83        1231       0      30     103680        0
        malloc-128    128     595     180        2931       0     126      99200        0
      malloc-32768  32768       1       2           5       0       1      98304        0
      malloc-32768  32768       3       0           3       0       1      98304        0
        malloc-256    256     193     182        1290       0      62      96000        0
        malloc-256    256     283      92        2955       0      62      96000        0
       malloc-4096   4096      12      10          48       0       2      90112        0
          syncache    168       0     528           5       0     254      88704        0
        128 Bucket   1024      65      18         339       0      16      84992        0
         64 Bucket    512      96      64        4203       0      30      81920        0
        sctp_chunk    152       2     518           8       0     254      79040        0
         tcp_inpcb   1304      37      20         273       0       8      74328        0
             g_bio    408       4     176       18966       0      30      73440        0
    ertt_txseginfo     40     513    1305        3993       0     254      72720        0
         malloc-64     64     722     349       18976       0     254      68544        0
        malloc-128    128     302     225         378       0     126      67456        0
      malloc-32768  32768       0       2         120       0       1      65536        0
       malloc-2048   2048      18      14          60       0       8      65536        0
           VMSPACE    584      98      14        1301       0      16      65408        0
        malloc-256    256     102     153        3304       0      62      65280        0
         32 Bucket    256      91     164        2384       0      62      65280        0
             Files     80     351     449       11709       0     126      64000        0
       malloc-8192   8192       6       1          10       0       1      57344        0
       malloc-8192   8192       3       4          36       0       1      57344        0
         malloc-64     64     466     353         673       0     254      52416        0
         malloc-64     64      26     793         920       0     254      52416        0
        malloc-128    128      11     392         287       0     126      51584        0
            cpuset    200      70     186         204       0      62      51200        0
           ttyoutq    256      64     131         376       0      62      49920        0
        malloc-256    256     113      82         304       0      62      49920        0
        malloc-256    256      77     118         193       0      62      49920        0
        malloc-256    256      44     151         601       0      62      49920        0
zio_data_buf_16384  16384       0       3           9       0       1      49152        0
           DIRHASH   1024      34      14          44       0      16      49152        0
             NAMEI   1024       4      44       19581       0      16      49152        0
       malloc-1024   1024      11      37         532       0      16      49152        0
              pipe    736      25      41         372       0      16      48576        0
        malloc-384    384      52      68         915       0      30      46080        0
             ripcb    376      16     104         108       0      30      45120        0
            pcpu-8      8    5368     264        5766       0     254      45056        0
         TURNSTILE    136     257      58         257       0      62      42840        0
            vnpbuf   2624       0      16           2       0      16      41984        0
       malloc-8192   8192       4       1           5       0       1      40960        0
       malloc-4096   4096       6       4          25       0       2      40960        0
       malloc-4096   4096       4       6         406       0       2      40960        0
        sctp_readq    152       0     260           1       0     254      39520        0
         udp_inpcb    408      11      79         256       0      30      36720        0
         hostcache     64       3     564           3       0     254      36288        0
         malloc-64     64     110     457         284       0     254      36288        0
         malloc-64     64      20     547         282       0     254      36288        0
         malloc-64     64      71     496         572       0     254      36288        0
       tcp_bbr_map    128     101     178         246       0     126      35712        0
      tcp_rack_map    128       0     279          14       0     126      35712        0
        malloc-128    128     102     177         271       0     126      35712        0
        malloc-128    128      15     264         195       0     126      35712        0
            itimer    352       2      97         133       0      30      34848        0
      ktls_session    256       1     134           6       0      62      34560        0
     routing nhops    256      27     108          39       0      62      34560        0
        malloc-384    384      63      27          63       0      30      34560        0
        SLEEPQUEUE     88     257     127         257       0     126      33792        0
     zio_buf_16384  16384       0       2          10       0       1      32768        0
      malloc-16384  16384       2       0           2       0       1      32768        0
      malloc-16384  16384       2       0           2       0       1      32768        0
      malloc-16384  16384       0       2         190       0       1      32768        0
      malloc-16384  16384       0       2           2       0       1      32768        0
       malloc-4096   4096       3       5          82       0       2      32768        0
       malloc-2048   2048       1      15          13       0       8      32768        0
       malloc-2048   2048       8       8           8       0       8      32768        0
       malloc-2048   2048       0      16          50       0       8      32768        0
       malloc-2048   2048       7       9         204       0       8      32768        0
       malloc-1024   1024       2      30          50       0      16      32768        0
       malloc-1024   1024      12      20        1195       0      16      32768        0
       malloc-1024   1024      10      22         505       0      16      32768        0
       malloc-1024   1024      10      22         167       0      16      32768        0
       malloc-1024   1024       0      32        2139       0      16      32768        0
        malloc-512    512       3      61         113       0      30      32768        0
        malloc-512    512       6      58         157       0      30      32768        0
        malloc-512    512       4      60           9       0      30      32768        0
        malloc-512    512       1      63         217       0      30      32768        0
        malloc-512    512       9      55          86       0      30      32768        0
        malloc-512    512      10      54          16       0      30      32768        0
           pcpu-64     64     501      11         501       0     254      32768        0
       tcp_bbr_pcb    896       1      35          11       0      16      32256        0
            ttyinq    160     120      80         705       0      62      32000        0
              PGRP    120      19     245          61       0     126      31680        0
            clpbuf   2624       0      12         210       0       4      31488        0
       L VFS Cache    320       0      96          61       0      30      30720        0
        sctp_laddr     48       8     580          37       0     254      28224        0
          rl_entry     48       7     581         179       0     254      28224        0
         malloc-32     32     146     736        1612       0     254      28224        0
         16 Bucket    144      64     132         639       0      62      28224        0
          4 Bucket     48       7     581          22       0     254      28224        0
               AIO    208       4     129          96       0      62      27664        0
            da_ccb    544       1      48        4946       0      16      26656        0
     udplite_inpcb    408       8      55          86       0      30      25704        0
       malloc-8192   8192       1       2           5       0       1      24576        0
       malloc-8192   8192       1       2          82       0       1      24576        0
               PWD     40      75     531         478       0     254      24240        0
           rtentry    168      30     114          39       0      62      24192        0
          8 Bucket     80      65     235         569       0     126      24000        0
            splice    184       0     126           2       0      62      23184        0
        malloc-384    384       1      59          27       0      30      23040        0
        malloc-384    384       1      59         159       0      30      23040        0
        malloc-384    384       1      59         344       0      30      23040        0
        malloc-384    384      22      38          44       0      30      23040        0
      tcp_rack_pcb   1088       0      21           7       0       8      22848        0
         domainset     40       1     566         194       0     254      22680        0
       Mountpoints   2816       2       6          12       0       4      22528        0
              ertt     72      37     243         273       0     126      20160        0
         malloc-32     32      71     559         147       0     254      20160        0
         malloc-32     32      81     549         191       0     254      20160        0
         malloc-32     32     263     367        1688       0     254      20160        0
         malloc-32     32     128     502         865       0     254      20160        0
         malloc-32     32      33     597         771       0     254      20160        0
         malloc-32     32      73     557        2730       0     254      20160        0
          2 Bucket     32     109     521         706       0     254      20160        0
             KNOTE    160       0     125         268       0      62      20000        0
           cryptop    280       0      70          18       0      30      19600        0
             AIOCB    552       2      33         106       0      16      19320        0
            AIOLIO    272       0      70          20       0      30      19040        0
              vmem   1856       2       7           2       0       8      16704        0
 epoch_record pcpu    256       4      60           4       0      62      16384        0
       malloc-8192   8192       2       0           2       0       1      16384        0
       malloc-4096   4096       0       4          59       0       2      16384        0
       malloc-4096   4096       1       3           4       0       2      16384        0
       malloc-1024   1024       1      15           1       0      16      16384        0
        malloc-512    512       0      32           5       0      30      16384        0
           SMR CPU     32       8     503           8       0     254      16352        0
         malloc-16     16     368     632        1463       0     254      16000        0
            mqueue    248       1      63           1       0      62      15872        0
              kenv    258      17      43        1078       0      30      15480        0
            mqnode    416       4      32           4       0      30      14976        0
        SMR SHARED     24       8     503           8       0     254      12264        0
arc_buf_hdr_t_full    368       4      29           4       0      30      12144        0
         malloc-16     16      84     666         376       0     254      12000        0
         malloc-16     16      18     732         229       0     254      12000        0
         malloc-16     16      51     699         359       0     254      12000        0
         malloc-16     16     189     561        3018       0     254      12000        0
         malloc-16     16      45     705       24990       0     254      12000        0
         malloc-16     16      49     701         388       0     254      12000        0
           pcpu-16     16       8     504           8       0     254       8192        0
      vtnet_tx_hdr     24       0     334        2015       0     254       8016        0
       UMA Slabs 1    176      10      12          10       0      62       3872        0
        KMAP ENTRY     96      15      24          63       0       0       3744        0
   zfs_znode_cache    464       0       0           0       0      30          0        0
zap_attr_long_cache   1056       0       0           0       0       8          0        0
     zap_name_long   1096       0       0           0       0       8          0        0
    zap_attr_cache    288       0       0           0       0      30          0        0
          zap_name    328       0       0           0       0      30          0        0
       sio_cache_2    168       0       0           0       0      62          0        0
       sio_cache_1    152       0       0           0       0      62          0        0
       sio_cache_0    136       0       0           0       0      62          0        0
     zil_zcw_cache     80       0       0           0       0     126          0        0
     zil_lwb_cache    376       0       0           0       0      30          0        0
dbuf_dirty_record_t    368       0       0           0       0      30          0        0
    dmu_buf_impl_t    376       0       0           0       0      30          0        0
         arc_buf_t     32       0       0           0       0     254          0        0
arc_buf_hdr_t_l2only     96       0       0           0       0     126          0        0
           dnode_t    968       0       0           0       0      16          0        0
          sa_cache    272       0       0           0       0      30          0        0
         abd_chunk   4096       0       0           0       0       2          0        0
zio_buf_comb_16777216 16777216       0       0           0       0       1          0        0
zio_buf_comb_12582912 12582912       0       0           0       0       1          0        0
zio_buf_comb_8388608 8388608       0       0           0       0       1          0        0
zio_buf_comb_6291456 6291456       0       0           0       0       1          0        0
zio_buf_comb_4194304 4194304       0       0           0       0       1          0        0
zio_buf_comb_3145728 3145728       0       0           0       0       1          0        0
zio_buf_comb_2097152 2097152       0       0           0       0       1          0        0
zio_buf_comb_1572864 1572864       0       0           0       0       1          0        0
zio_buf_comb_786432 786432       0       0           0       0       1          0        0
zio_buf_comb_524288 524288       0       0           0       0       1          0        0
zio_buf_comb_393216 393216       0       0           0       0       1          0        0
zio_buf_comb_262144 262144       0       0           0       0       1          0        0
zio_buf_comb_196608 196608       0       0           0       0       1          0        0
zio_buf_comb_98304  98304       0       0           0       0       1          0        0
zio_buf_comb_65536  65536       0       0           0       0       1          0        0
zio_buf_comb_49152  49152       0       0           0       0       1          0        0
zio_buf_comb_32768  32768       0       0           0       0       1          0        0
zio_buf_comb_24576  24576       0       0           0       0       1          0        0
zio_data_buf_12288  12288       0       0           0       0       1          0        0
     zio_buf_12288  12288       0       0           0       0       1          0        0
 zio_data_buf_8192   8192       0       0           0       0       1          0        0
      zio_buf_8192   8192       0       0           0       0       1          0        0
 zio_data_buf_6144   6144       0       0           0       0       1          0        0
      zio_buf_6144   6144       0       0           0       0       1          0        0
 zio_data_buf_4096   4096       0       0           0       0       2          0        0
      zio_buf_4096   4096       0       0           0       0       2          0        0
 zio_data_buf_3072   3072       0       0           0       0       4          0        0
      zio_buf_3072   3072       0       0           0       0       4          0        0
 zio_data_buf_2048   2048       0       0           0       0       8          0        0
      zio_buf_2048   2048       0       0           0       0       8          0        0
 zio_data_buf_1536   1536       0       0           0       0       8          0        0
      zio_buf_1536   1536       0       0           0       0       8          0        0
 zio_data_buf_1024   1024       0       0           0       0      16          0        0
      zio_buf_1024   1024       0       0           0       0      16          0        0
  zio_data_buf_512    512       0       0           0       0      30          0        0
       zio_buf_512    512       0       0           0       0      30          0        0
    zio_link_cache     48       0       0           0       0     254          0        0
         zio_cache   1216       0       0           0       0       8          0        0
ddt_log_entry_trad_cache    328       0       0           0       0      30          0        0
ddt_log_entry_flat_cache    144       0       0           0       0      62          0        0
ddt_entry_trad_cache    368       0       0           0       0      30          0        0
ddt_entry_flat_cache    184       0       0           0       0      62          0        0
         ddt_cache  29240       0       0           0       0       1          0        0
   brt_entry_cache    168       0       0           0       0      62          0        0
metaslab_alloc_trace_cache     72       0       0           0       0     126          0        0
zfs_btree_leaf_cache   4096       0       0           0       0       2          0        0
   reference_cache     48       0       0           0       0     254          0        0
        taskq_zone    160       0       0           0       0      62          0        0
       FFS1 dinode    128       0       0           0       0     126          0        0
           ada_ccb    272       0       0           0       0      30          0        0
             swblk    136       0       0           0       0      62          0        0
          swpctrie    152       0       0           0       0      62          0        0
   cdg_qdiffsample     16       0       0           0       0     254          0        0
   pf state scrubs     40       0       0           0       0     254          0        0
   pf frag entries     40       0       0           0       0     254          0        0
          pf frags    256       0       0           0       0      62          0        0
  pf table entries    160       0       0           0       0     254          0        0
pf table entry counters     64       0       0           0       0     254          0        0
   pf UDP mappings    104       0       0           0       0     126          0        0
   pf source nodes    152       0       0           0       0     254          0        0
     pf state keys     88       0       0           0       0     126          0        0
         pf states    384       0       0           0       0     254          0        0
           pf tags    104       0       0           0       0     126          0        0
          pf mtags    184       0       0           0       0      62          0        0
tfo_ccache_entries     80       0       0           0       0     126          0        0
               tfo      4       0       0           0       0     254          0        0
          sackhole     32       0       0           0       0     254          0        0
               ipq     56       0       0           0       0     254          0        0
   sctp_asconf_ack     48       0       0           0       0     254          0        0
       sctp_asconf     40       0       0           0       0     254          0        0
sctp_stream_msg_out    112       0       0           0       0     254          0        0
   tcp_log_id_node    120       0       0           0       0     126          0        0
 tcp_log_id_bucket    176       0       0           0       0      62          0        0
          tcpreass     48       0       0           0       0     254          0        0
    IPsec SA lft_c     16       0       0           0       0     254          0        0
        mqnotifier    216       0       0           0       0      62          0        0
            mvdata     64       0       0           0       0     254          0        0
        TMPFS node    240       0       0           0       0      62          0        0
           NCLNODE    608       0       0           0       0      16          0        0
     LTS VFS Cache    360       0       0           0       0      30          0        0
     STS VFS Cache    144       0       0           0       0      62          0        0
  linux_dma_object     32       0       0           0       0     254          0        0
  linux_dma_pctrie    152       0       0           0       0      62          0        0
   IOMMU_MAP_ENTRY    112       0       0           0       0     126          0        0
            skbuff   1808       0       0           0       0       8          0        0
    mbuf_jumbo_16k  16384       0       0           0       0     254          0        0
     mbuf_jumbo_9k   9216       0       0           0       0     254          0        0
      audit_record   1280       0       0           0       0       8          0        0
        MAC labels     40       0       0           0       0     254          0        0
           nfspbuf   2624       0       0           0       0       4          0        0
            swwbuf   2624       0       0           0       0       2          0        0
            swrbuf   2624       0       0           0       0       4          0        0
          umtx_shm     88       0       0           0       0     126          0        0
           umtx pi     96       0       0           0       0     126          0        0
rangeset pctrie nodes    152       0       0           0       0      62          0        0
      malloc-65536  65536       0       0           0       0       1          0        0
      malloc-65536  65536       0       0           0       0       1          0        0
      malloc-65536  65536       0       0           0       0       1          0        0
      malloc-32768  32768       0       0           0       0       1          0        0
      malloc-32768  32768       0       0           0       0       1          0        0
      malloc-16384  16384       0       0           0       0       1          0        0
      malloc-16384  16384       0       0           0       0       1          0        0
      malloc-16384  16384       0       0           0       0       1          0        0
        malloc-512    512       0       0           0       0      30          0        0
           pcpu-32     32       0       0           0       0     254          0        0
            pcpu-4      4       0       0           0       0     254          0        0
            fakepg    104       0       0           0       0     126          0        0
          UMA Hash    256       0       0           0       0      62          0        0

Crashes (1):
Time Kernel Commit Syzkaller Config Log Report Syz repro C repro VM info Assets (help?) Manager Title
2025/05/12 11:47 freebsd-src 79e70e12f5c9 77908e5f console log report ci-freebsd-main panic: /syzkaller/managers/main/kernel/sys/kern/kern_timeout.c:LINE: _callout_stop_safe: Bad link elm ADDR prev->next !=
* Struck through repros no longer work on HEAD.