syzbot


witness: reversal: &sched_lock &cq->cq_mtx

Status: upstream: reported on 2025/05/15 15:16
Reported-by: syzbot+9ec54ee1623e4ac93f78@syzkaller.appspotmail.com
First crash: 37d, last: 1d07h

Sample crash report:
witness: lock order reversal:
 1st 0xffffffff839f8580 &sched_lock (&sched_lock)
 2nd 0xffff8000299dee00 &cq->cq_mtx (&cq->cq_mtx)
lock order [1] &sched_lock (&sched_lock) -> [2] &cq->cq_mtx (&cq->cq_mtx)
#0  mtx_enter+0x4a sys/kern/kern_lock.c:257
#1  clockintr_cancel+0x2f sys/kern/kern_clockintr.c:323
#2  mi_switch+0x172 x86_atomic_clearbits_u32 machine/atomic.h:-1 [inline]
#2  mi_switch+0x172 sys/kern/sched_bsd.c:374
#3  ast+0x165 mi_ast sys/sys/syscall_mi.h:252 [inline]
#3  ast+0x165 sys/arch/amd64/amd64/trap.c:543
#4  intr_user_exit+0x3c
lock order [2] &cq->cq_mtx (&cq->cq_mtx) -> [1] &sched_lock (&sched_lock)
#0  mtx_enter+0x4a sys/kern/kern_lock.c:257
#1  sleep_setup+0xc6 sys/kern/kern_synch.c:302
#2  msleep_nsec+0x119 sys/kern/kern_synch.c:217
#3  dt_ioctl_record_stop+0xbc sys/dev/dt/dt_dev.c:576
#4  dtclose+0x105 dt_pcb_purge sys/dev/dt/dt_dev.c:-1 [inline]
#4  dtclose+0x105 sys/dev/dt/dt_dev.c:232
#5  spec_close+0x45f sys/kern/spec_vnops.c:-1
#6  VOP_CLOSE+0x133 sys/kern/vfs_vops.c:156
#7  vn_closefile+0x12b vn_close sys/kern/vfs_vnops.c:292 [inline]
#7  vn_closefile+0x12b sys/kern/vfs_vnops.c:615
#8  fdrop+0x126 sys/kern/kern_descrip.c:1267
#9  closef+0x192 sys/kern/kern_descrip.c:1251
#10 fdfree+0x116 sys/kern/kern_descrip.c:1182
#11 exit1+0x59c sys/kern/kern_exit.c:215
#12 sys_exit+0x1a sys/kern/kern_exit.c:-1
#13 syscall+0xb08 mi_syscall sys/sys/syscall_mi.h:176 [inline]
#13 syscall+0xb08 sys/arch/amd64/amd64/trap.c:579
#14 Xsyscall+0x128
Stopped at      db_enter+0x25:  addq    $0x8,%rsp
ddb{1}> 
ddb{1}> set $lines = 0
ddb{1}> set $maxwidth = 0
ddb{1}> show panic
the kernel did not panic
ddb{1}> trace
db_enter() at db_enter+0x25 sys/arch/amd64/amd64/db_interface.c:438
witness_checkorder(ffff8000299dee00,9,0) at witness_checkorder+0x10fb sys/kern/subr_witness.c:-1
mtx_enter(ffff8000299dedf0) at mtx_enter+0x4a sys/kern/kern_lock.c:257
clockintr_cancel(ffff8000299de2f8) at clockintr_cancel+0x2f sys/kern/kern_clockintr.c:323
mi_switch() at mi_switch+0x172 x86_atomic_clearbits_u32 machine/atomic.h:-1 [inline]
mi_switch() at mi_switch+0x172 sys/kern/sched_bsd.c:374
ast(ffff80003c427db0) at ast+0x165 mi_ast sys/sys/syscall_mi.h:252 [inline]
ast(ffff80003c427db0) at ast+0x165 sys/arch/amd64/amd64/trap.c:543
intr_user_exit() at intr_user_exit+0x3c
end of kernel
end trace frame: 0x70ba05b8db90, count: -7
ddb{1}> show registers
rdi                                0
rsi                                0
rbp               0xffff80003c427bd0
rbx               0xfffffd800473ec48
rdx                                0
rcx               0xffff800031c7f9d8
rax               0xffff8000299ddff0
r8                0xffff80003c427ab0
r9                0x8080808080808080
r10               0xaa7685a789caa619
r11               0x8b0406ddcdb4a540
r12                                0
r13               0xfffffd8003f50dc0
r14                              0x3
r15                       0xffffffff
rip               0xffffffff81bd4c95    db_enter+0x25
cs                               0x8
rflags                         0x246
rsp               0xffff80003c427bc0
ss                                 0
db_enter+0x25:  addq    $0x8,%rsp
ddb{1}> show proc
PROC (syz-executor) tid=442605 pid=6253 tcnt=3 stat=run
    flags process=4000000<ITIMER> proc=0
    runpri=86, usrpri=86, slppri=17, nice=20
    wchan=0x0, wmesg=, ps_single=0x0 scnt=0 ecnt=0
    forw=0x0, list=0xffff800031c7ef98,0xffff80003c416d08
    process=0xffff800031c82bc0 user=0xffff80003c422000, vmspace=0xfffffd806d368990
    estcpu=36, cpticks=2, pctcpu=0.0, user=1, sys=1, intr=0
ddb{1}> ps
   PID     TID   PPID    UID  S       FLAGS  WAIT          COMMAND
 75068  270414  73749      0  2           0                syz-executor
 75068  388897  73749      0  3   0x4000080  fsleep        syz-executor
* 6253  442605  60089      0  2   0x4000000                syz-executor
  6253  226358  60089      0  3   0x4000080  kqsel         syz-executor
  6253  493143  60089      0  2   0x4000000                syz-executor
 97384  421531  98844      0  2        0x10                syz-executor
 97384  403792  98844      0  3   0x4000010  smrbar        syz-executor
 97384  491115  98844      0  7   0x4000010                syz-executor
 50698  241995  49844  60929  3        0x90  nanoslp       syz-executor
 50698  422343  49844  60929  3   0x4000090  fsleep        syz-executor
 50698  370340  49844  60929  3   0x4000090  fsleep        syz-executor
 53394   91049  42062      0  3        0x80  nanoslp       syz-executor
 53394  244803  42062      0  3   0x4000080  kqread        syz-executor
 53394   55661  42062      0  3   0x4000080  fsleep        syz-executor
 29295  435313  56478      0  3        0x80  nanoslp       syz-executor
 29295  136008  56478      0  3   0x4000080  kqsel         syz-executor
 29295  296462  56478      0  3   0x4000080  fsleep        syz-executor
 60089   90336  12212      0  3        0x82  nanoslp       syz-executor
 77410  408490      1      0  3    0x100083  ttyin         getty
 98844    7485  12212      0  3        0x82  nanoslp       syz-executor
 70801   84779      0      0  3     0x14280  nfsidl        nfsio
 70306  350818      0      0  3     0x14280  nfsidl        nfsio
 69471  459627      0      0  3     0x14280  nfsidl        nfsio
 63413  152677      0      0  3     0x14280  nfsidl        nfsio
 58292   27529      0      0  3     0x14280  nfsidl        nfsio
  8794  129468      0      0  3     0x14280  nfsidl        nfsio
 28662  357643      0      0  3     0x14280  nfsidl        nfsio
 32004  502832      0      0  3     0x14280  nfsidl        nfsio
 30090  445926      0      0  3     0x14280  nfsidl        nfsio
 43399  499675      0      0  3     0x14280  nfsidl        nfsio
 96001   93020      0      0  3     0x14280  nfsidl        nfsio
  6789  173711      0      0  3     0x14280  nfsidl        nfsio
 25398  432090      0      0  3     0x14280  nfsidl        nfsio
 34023  335175      0      0  3     0x14280  nfsidl        nfsio
 15290  106702      0      0  3     0x14280  nfsidl        nfsio
  6875  331112      0      0  3     0x14280  nfsidl        nfsio
 52841  295359      0      0  3     0x14280  nfsidl        nfsio
 34121  309808      0      0  3     0x14280  nfsidl        nfsio
  5593  171240      0      0  3     0x14280  nfsidl        nfsio
 21367  149562      0      0  3     0x14280  nfsidl        nfsio
 49844  189441  12212      0  3        0x82  nanoslp       syz-executor
 78720  223978      0      0  3     0x14200  bored         sosplice
 73749   52431  12212      0  3        0x82  nanoslp       syz-executor
 56478  142225  12212      0  3        0x82  nanoslp       syz-executor
 64230  141652  12212      0  3        0x82  wait          syz-executor
 42062  146848  12212      0  3        0x82  nanoslp       syz-executor
 20356  142887  12212      0  2         0x2                syz-executor
 12212  206101   4779      0  3        0x82  kqread        syz-executor
  4779   33565  89064      0  3    0x10008a  sigsusp       ksh
 89064  479842  83250      0  3        0x98  kqread        sshd-session
 83250  495534  71570      0  3        0x92  kqread        sshd-session
 71570  210236      1      0  3        0x88  kqread        sshd
 76684   45266  34048     74  3   0x1100092  bpf           pflogd
 34048  390215      1      0  3        0x80  sbwait        pflogd
 73805  109613  33205     73  3   0x1100090  kqread        syslogd
 33205  224357      1      0  3    0x100082  sbwait        syslogd
  3761  293589      1      0  3    0x100080  kqread        resolvd
 11506  293249  38577     77  3    0x100092  kqread        dhcpleased
 57335  360166  38577     77  3    0x100092  kqread        dhcpleased
 38577   91246      1      0  3        0x80  kqread        dhcpleased
 68996    4702      0      0  3     0x14200  bored         smr
 24610  513435      0      0  2     0x14200                zerothread
 73076  395900      0      0  3     0x14200  aiodoned      aiodoned
  6555  124793      0      0  3     0x14200  syncer        update
 78236  362212      0      0  3     0x14200  cleaner       cleaner
 89673   85462      0      0  3     0x14200  reaper        reaper
 79274  259866      0      0  3     0x14200  pgdaemon      pagedaemon
 84450   91847      0      0  3     0x14200  bored         viomb
  3434  224289      0      0  3  0x40014200  acpi0         acpi0
 12800  268734      0      0  3  0x40014200                idle1
 76511   90133      0      0  3     0x14200  bored         softnet3
 39620  230713      0      0  3     0x14200  bored         softnet2
 38970  275537      0      0  3     0x14200  bored         softnet1
 43566  136732      0      0  3     0x14200  bored         softnet0
 16106  328342      0      0  3     0x14200  bored         systqmp
 62313  446563      0      0  3     0x14200  bored         systq
 91983  390627      0      0  3     0x14200  tmoslp        softclockmp
 36221  319477      0      0  3  0x40014200  tmoslp        softclock
 33225  308306      0      0  3  0x40014200                idle0
     1  372039      0      0  3        0x82  wait          init
     0       0     -1      0  3  0x10010200  scheduler     swapper
ddb{1}> show all locks
CPU 0:
exclusive mutex &vioq->viq_rxmtx r = 0 (0xffff80000002c4b0)
#0  witness_lock+0x5bb stacktrace_save sys/sys/stacktrace.h:37 [inline]
#0  witness_lock+0x5bb sys/kern/subr_witness.c:1160
#1  mtx_enter_try+0x1ad sys/kern/kern_lock.c:301
#2  mtx_enter+0x62 sys/kern/kern_lock.c:258
#3  vio_rx_intr+0x82 sys/dev/pv/if_vio.c:1567
#4  vio_queue_intr+0x59 sys/dev/pv/if_vio.c:909
#5  intr_handler+0x11d sys/arch/amd64/amd64/intr.c:563
#6  Xintr_ioapic_edge26_untramp+0x18f
#7  __sanitizer_cov_trace_const_cmp4+0x1e kd_curproc sys/dev/kcov.c:583 [inline]
#7  __sanitizer_cov_trace_const_cmp4+0x1e sys/dev/kcov.c:230
#8  mtx_enter+0xaf sys/kern/kern_lock.c:261
#9  endtsleep+0x39 sys/kern/kern_synch.c:538
#10 timeout_run+0x13d sys/kern/kern_timeout.c:707
#11 softclock_process_tick_timeout+0x22b sys/kern/kern_timeout.c:765
#12 softclock+0x16b sys/kern/kern_timeout.c:797
#13 softintr_dispatch+0x140 sys/kern/kern_softintr.c:84
#14 dosoftint+0x54 sys/arch/amd64/amd64/intr.c:847
#15 Xsoftclock+0x27
#16 __sanitizer_cov_trace_pc+0x95 kd_claim sys/dev/kcov.c:632 [inline]
#16 __sanitizer_cov_trace_pc+0x95 sys/dev/kcov.c:157
#17 mtx_enter+0xdd sys/kern/kern_lock.c:261
#18 sleep_setup+0xc6 sys/kern/kern_synch.c:302
shared mutex timeout r = 0 (0xffffffff83861ff0)
#0  witness_lock+0x5bb stacktrace_save sys/sys/stacktrace.h:37 [inline]
#0  witness_lock+0x5bb sys/kern/subr_witness.c:1160
#1  timeout_run+0x115 sys/kern/kern_timeout.c:703
#2  softclock_process_tick_timeout+0x22b sys/kern/kern_timeout.c:765
#3  softclock+0x16b sys/kern/kern_timeout.c:797
#4  softintr_dispatch+0x140 sys/kern/kern_softintr.c:84
#5  dosoftint+0x54 sys/arch/amd64/amd64/intr.c:847
#6  Xsoftclock+0x27
#7  __sanitizer_cov_trace_pc+0x95 kd_claim sys/dev/kcov.c:632 [inline]
#7  __sanitizer_cov_trace_pc+0x95 sys/dev/kcov.c:157
#8  mtx_enter+0xdd sys/kern/kern_lock.c:261
#9  sleep_setup+0xc6 sys/kern/kern_synch.c:302
#10 sys_futex+0xa04 futex_wait sys/kern/sys_futex.c:301 [inline]
#10 sys_futex+0xa04 sys/kern/sys_futex.c:151
#11 syscall+0xbc6 mi_syscall sys/sys/syscall_mi.h:176 [inline]
#11 syscall+0xbc6 sys/arch/amd64/amd64/trap.c:579
#12 Xsyscall+0x128
CPU 1:
exclusive mutex &sched_lock r = 0 (0xffffffff839f8580)
#0  witness_lock+0x5bb stacktrace_save sys/sys/stacktrace.h:37 [inline]
#0  witness_lock+0x5bb sys/kern/subr_witness.c:1160
#1  mtx_enter_try+0x1ad sys/kern/kern_lock.c:301
#2  mtx_enter+0x13b sys/kern/kern_lock.c:258
#3  preempt+0x35 sys/kern/sched_bsd.c:339
#4  ast+0x165 mi_ast sys/sys/syscall_mi.h:252 [inline]
#4  ast+0x165 sys/arch/amd64/amd64/trap.c:543
#5  intr_user_exit+0x3c
Process 97384 (syz-executor) thread 0xffff800031c7ef98 (403792)
exclusive rwlock clonelk r = 0 (0xffffffff8385ac08)
#0  witness_lock+0x5bb stacktrace_save sys/sys/stacktrace.h:37 [inline]
#0  witness_lock+0x5bb sys/kern/subr_witness.c:1160
#1  rw_do_enter_write+0x377 sys/kern/kern_rwlock.c:310
#2  if_clone_destroy+0x67 sys/net/if.c:-1
#3  tun_dev_close+0x1dc sys/net/if_tun.c:529
#4  spec_close+0x412 sys/kern/spec_vnops.c:-1
#5  VOP_CLOSE+0x133 sys/kern/vfs_vops.c:156
#6  vn_closefile+0x12b vn_close sys/kern/vfs_vnops.c:292 [inline]
#6  vn_closefile+0x12b sys/kern/vfs_vnops.c:615
#7  fdrop+0x126 sys/kern/kern_descrip.c:1267
#8  closef+0x192 sys/kern/kern_descrip.c:1251
#9  syscall+0xbc6 mi_syscall sys/sys/syscall_mi.h:176 [inline]
#9  syscall+0xbc6 sys/arch/amd64/amd64/trap.c:579
#10 Xsyscall+0x128
Process 97384 (syz-executor) thread 0xffff800031c7f228 (491115)
exclusive kernel_lock &kernel_lock r = 0 (0xffffffff839140c0)
#0  witness_lock+0x5bb stacktrace_save sys/sys/stacktrace.h:37 [inline]
#0  witness_lock+0x5bb sys/kern/subr_witness.c:1160
#1  softintr_dispatch+0x12a sys/kern/kern_softintr.c:83
#2  dosoftint+0x54 sys/arch/amd64/amd64/intr.c:847
#3  Xsoftclock+0x27
#4  __sanitizer_cov_trace_pc+0x95 kd_claim sys/dev/kcov.c:632 [inline]
#4  __sanitizer_cov_trace_pc+0x95 sys/dev/kcov.c:157
#5  mtx_enter+0xdd sys/kern/kern_lock.c:261
#6  sleep_setup+0xc6 sys/kern/kern_synch.c:302
#7  sys_futex+0xa04 futex_wait sys/kern/sys_futex.c:301 [inline]
#7  sys_futex+0xa04 sys/kern/sys_futex.c:151
#8  syscall+0xbc6 mi_syscall sys/sys/syscall_mi.h:176 [inline]
#8  syscall+0xbc6 sys/arch/amd64/amd64/trap.c:579
#9  Xsyscall+0x128
Process 20356 (syz-executor) thread 0xffff80002a2baf78 (142887)
exclusive rrwlock inode r = 0 (0xfffffd806e059b00)
#0  witness_lock+0x5bb stacktrace_save sys/sys/stacktrace.h:37 [inline]
#0  witness_lock+0x5bb sys/kern/subr_witness.c:1160
#1  rw_do_enter_write+0x377 sys/kern/kern_rwlock.c:310
#2  rrw_enter+0xc6 sys/kern/kern_rwlock.c:605
#3  VOP_LOCK+0xa6 sys/kern/vfs_vops.c:527
#4  ufs_ihashins+0x4f ufs_ihash sys/ufs/ufs/ufs_ihash.c:-1 [inline]
#4  ufs_ihashins+0x4f sys/ufs/ufs/ufs_ihash.c:159
#5  ffs_vget+0x187 sys/ufs/ffs/ffs_vfsops.c:1232
#6  ffs_inode_alloc+0x283 sys/ufs/ffs/ffs_alloc.c:393
#7  ufs_mkdir+0x113 sys/ufs/ufs/ufs_vnops.c:1112
#8  VOP_MKDIR+0x102 sys/kern/vfs_vops.c:394
#9  domkdirat+0x179 sys/kern/vfs_syscalls.c:3109
#10 syscall+0xb08 mi_syscall sys/sys/syscall_mi.h:176 [inline]
#10 syscall+0xb08 sys/arch/amd64/amd64/trap.c:579
#11 Xsyscall+0x128
exclusive rrwlock inode r = 0 (0xfffffd806d05f310)
#0  witness_lock+0x5bb stacktrace_save sys/sys/stacktrace.h:37 [inline]
#0  witness_lock+0x5bb sys/kern/subr_witness.c:1160
#1  rw_do_enter_write+0x377 sys/kern/kern_rwlock.c:310
#2  rrw_enter+0xc6 sys/kern/kern_rwlock.c:605
#3  VOP_LOCK+0xa6 sys/kern/vfs_vops.c:527
#4  vn_lock+0xa4 sys/kern/vfs_vnops.c:570
#5  vfs_lookup+0x109 sys/kern/vfs_lookup.c:418
#6  namei+0x7aa sys/kern/vfs_lookup.c:250
#7  domkdirat+0x8b sys/kern/vfs_syscalls.c:3094
#8  syscall+0xb08 mi_syscall sys/sys/syscall_mi.h:176 [inline]
#8  syscall+0xb08 sys/arch/amd64/amd64/trap.c:579
#9  Xsyscall+0x128
ddb{1}> show malloc
           Type InUse  MemUse  HighUse   Limit  Requests Type Lim
         devbuf 10311  11463K   12573K 166960K     13595        0
            pcb    17     16K      18K 166960K       423        0
         rtable   126     10K      11K 166960K       594        0
             pf    45     19K   67486K 166960K       305        0
         ifaddr    28      3K       7K 166960K       108        0
        ifgroup    64      2K       2K 166960K       187        0
         sysctl     4      1K       9K 166960K        14        0
       counters    72     37K      38K 166960K       192        0
       ioctlops     0      0K       4K 166960K      1752        0
            iov     0      0K      24K 166960K        62        0
          mount     1      1K       1K 166960K         1        0
            log     0      0K       0K 166960K         4        0
         vnodes  1526     96K      96K 166960K      2779        0
      UFS quota     1     32K      32K 166960K         1        0
      UFS mount     5     36K      36K 166960K         5        0
            shm     2      1K       9K 166960K        33        0
         VM map     2      1K       1K 166960K         2        0
            sem    12      0K       0K 166960K       112        0
        dirhash    15      2K       2K 166960K        27        0
           ACPI  1692    195K     286K 166960K     12470        0
      file desc    17     61K     110K 166960K      1264        0
          sigio     1      0K       0K 166960K        25        0
           proc    72     91K     116K 166960K       702        0
        subproc    72      4K       4K 166960K        99        0
    NFS srvsock     1      0K       0K 166960K         1        0
     NFS daemon     1     16K      16K 166960K         1        0
    ip_moptions     0      0K       0K 166960K       240        0
       in_multi    37      2K       6K 166960K       191        0
    ether_multi     1      0K       0K 166960K        14        0
            mrt     0      0K       0K 166960K        11        0
    ISOFS mount     1     32K      32K 166960K         1        0
  MSDOSFS mount     1     16K      16K 166960K         1        0
           ttys    73    334K     334K 166960K        73        0
           exec     0      0K       1K 166960K       656        0
   fusefs mount     1     32K      32K 166960K         1        0
     pfkey data     0      0K       0K 166960K         1        0
            tdb     3      0K       0K 166960K         3        0
        VM swap     8     62K      64K 166960K        10        0
       UVM amap   246    175K     191K 166960K     12749        0
       UVM aobj   111      3K       3K 166960K       114        0
     pinsyscall    42     84K     104K 166960K      2445        0
        memdesc     1      4K       4K 166960K         1        0
    crypto data     1      1K       1K 166960K         1        0
    ip6_options     0      0K       1K 166960K        76        0
            NDP    16      0K       1K 166960K        75        0
           temp    70   8693K    8947K 166960K     46920        0
         kqueue    16     26K      28K 166960K       187        0
      SYN cache     2     16K      16K 166960K         2        0
ddb{1}> show all pools
Name      Size Requests Fail Releases Pgreq Pgrel Npage Hiwat Minpg Maxpg Idle
plcache    128       26    0        0     1     0     1     1     0     8    0
rtpcb      120      112    0      109     1     0     1     1     0     8    0
rtentry    176      170    0      129     5     0     5     5     0     8    0
unpcb      144      863    0      842     9     7     2     6     0     8    1
syncache   336        4    0        4     2     2     0     1     0     8    0
tcpqe       32        1    0        1     1     1     0     1     0     8    0
tcpcb      736      660    0      652    14    13     1    13     0     8    0
arp        128       25    0       18     1     0     1     1     0     8    0
inpcb      328     1677    0     1665    20    13     7    12     0     8    5
nd6        144       36    0       29     1     0     1     1     0     8    0
pkpcb       40        8    0        8     3     2     1     1     0     8    1
kcovpl      48       11    0        3     1     0     1     1     0     8    0
mppekey    1024       1    0        1     1     1     0     1     0     8    0
ppxss      1192      43    0       43     2     1     1     1     0     8    1
pppxif     1504       8    0        8     2     1     1     1     0     8    1
pffrag     232       28    0       18     1     0     1     1     0   482    0
pffrnode    88       26    0       18     1     0     1     1     0     8    0
pffrent     40       52    0       41     1     0     1     1     0     8    0
pfosfp      40     1428    0     1005     5     0     5     5     0     8    0
pfosfpen   112     1428    0      714    21     0    21    21     0     8    0
pfrktable  1344       6    0        2     1     0     1     1     0     8    0
pfanchor   1288       5    0        0     1     0     1     1     0     8    0
pftag       88        2    0        1     1     0     1     1     0     8    0
pfstitem    24       96    0       23     1     0     1     1     0     8    0
pfstkey    128       98    0       25     3     0     3     3     0     8    0
pfstate    384       96    0       24     8     0     8     8     0     8    0
pfrule     1344      95    0       22     9     2     7     7     0     8    0
rttmr      136        3    0        3     3     2     1     1     0     8    1
art_heap8  4096       5    0        0     5     0     5     5     0     8    0
art_heap4  256      747    0      541    31     9    22    28     0     8    8
art_table   32      752    0      541     4     0     4     4     0     8    0
art_node    16      166    0      129     1     0     1     1     0     8    0
sysvmsgpl   40       13    0        7     1     0     1     1     0     8    0
semupl     112        3    0        3     2     2     0     1     0     8    0
semapl     112      107    0       97     1     0     1     1     0     8    0
shmpl      112      111    0        3     4     0     4     4     0     8    0
dirhash    1024      27    0        8     3     0     3     3     0     8    0
dino2pl    256     3677    0     2164    95     0    95    95     0     8    0
ffsino     288     3677    0     2164   109     0   109   109     0     8    0
nchpl      144     5350    0     3650    64     0    64    64     0     8    0
rtmask      32       10    0       10     3     2     1     1     0     8    1
uvmvnodes   80     4755    0        0    98     0    98    98     0     8    0
vnodes     216     4755    0        0   265     0   265   265     0     8    0
namei      1024   19014    0    19013     2     1     1     1     0     8    0
percpumem   16      111    0       60     1     0     1     1     0     8    0
pfiaddrpl  120       65    0        0     2     0     2     2     0     8    0
kstatmem   264      110    0       76     3     0     3     3     0     8    0
scsiplug    72        4    0        4     2     2     0     1     0     8    0
scxspl     216    23651    0    23651    12    10     2     8     1     8    2
plimitpl   152      412    0      393     1     0     1     1     0     8    0
sigapl     424     1582    0     1512     9     0     9     9     0     8    0
knotepl    120      585    0        0    18     0    18    18     0     8    0
kqueuepl   224      385    0      369     5     4     1     5     0     8    0
pipepl     336      257    0      228     3     0     3     3     0     8    0
fdescpl    520     1535    0     1504     3     0     3     3     0     8    0
filepl     160    10308    0    10063    29    13    16    23     0     8    0
lockfpl    104      796    0      788     2     1     1     2     0     8    0
lockfspl    48      291    0      283     1     0     1     1     0     8    0
sessionpl  144       27    0       18     1     0     1     1     0     8    0
pgrppl      48       52    0       35     1     0     1     1     0     8    0
ucredpl    104     1424    0     1407     1     0     1     1     0     8    0
zombiepl   144     1513    0     1512     1     0     1     1     0     8    0
processpl  1240    1582    0     1512     6     0     6     6     0     8    0
procpl     656     3467    0     3386     8     0     8     8     0     8    0
srpgc       96       18    0       18     3     2     1     1     0     8    1
sosppl     168        6    0        6     3     2     1     1     0     8    1
sockpl     728     2684    0     2648    38    28    10    22     0     8    6
mcl64k     65536      5    0        0     1     0     1     1     0     8    0
mcl16k     16384      4    0        0     1     0     1     1     0     8    0
mcl12k     12288      1    0        0     1     0     1     1     0     8    0
mcl9k      9216       1    0        0     1     0     1     1     0     8    0
mcl8k      8192       3    0        0     1     0     1     1     0     8    0
mcl4k      4096     118    0        0    15     0    15    15     0     8    0
mcl2k      2048      41    0        0     5     0     5     5     0     8    0
mtagpl      96       21    0        0     1     0     1     1     0     8    0
mbufpl     256     1190    0        0    74     0    74    74     0     8    0
bufpl      280     8104    0     1962   439     0   439   439     0     8    0
anonpl      32    11524    0        0    93     0    93    93     0   246    0
amapchunkpl 152   44143    0    43632    47    16    31    38     0   158   10
amappl16   200     4628    0     4594    31    16    15    18     0     8    8
amappl15   192        2    0        2     1     1     0     1     0     8    0
amappl14   184      126    0      114     1     0     1     1     0     8    0
amappl13   176        2    0        2     1     1     0     1     0     8    0
amappl12   168     2235    0     2204     3     1     2     2     0     8    0
amappl11   160       51    0       36     1     0     1     1     0     8    0
amappl10   152       11    0       11     1     1     0     1     0     8    0
amappl9    144      250    0      250     1     1     0     1     0     8    0
amappl8    136       29    0       27     1     0     1     1     0     8    0
amappl7    128      121    0      109     1     0     1     1     0     8    0
amappl6    120      221    0      218     1     0     1     1     0     8    0
amappl5    112      156    0      146     1     0     1     1     0     8    0
amappl4    104      333    0      312     1     0     1     1     0     8    0
amappl3     96     8776    0     8658     4     0     4     4     0     8    0
amappl2     88      698    0      636     2     0     2     2     0     8    0
amappl1     80    13576    0    12969    18     3    15    15     0     8    0
amappl      88    11793    0    11623     5     0     5     5     0    92    0
dma4096    4096       2    0        2     2     1     1     1     0     8    1
dma1024    1024       1    0        0     1     0     1     1     0     8    0
dma256     256        6    0        6     1     1     0     1     0     8    0
dma128     128      254    0      254     2     2     0     1     0     8    0
dma64       64        6    0        6     1     1     0     1     0     8    0
dma32       32        7    0        7     1     1     0     1     0     8    0
dma16       16       19    0       18     1     0     1     1     0     8    0
aobjpl      72      113    0        3     2     0     2     2     0     8    0
uaddrrnd    24     1535    0     1504     1     0     1     1     0     8    0
uaddrbest   32        2    0        0     1     0     1     1     0     8    0
uaddr       24     1535    0     1504     1     0     1     1     0     8    0
vmmpekpl   168    13887    0    13833     3     0     3     3     0     8    0
vmmpepl    168   101372    0    99358   112    10   102   104     0   357    4
vmsppl     480     1534    0     1504     7     2     5     5     0     8    0
rwobjpl     72    32154    0    26338   106     0   106   106     0     8    0
pdppl      4096    3077    0     3008   109    36    73    85     0     8    4
pvpl        32    17719    0        0   143     0   143   143     0   265    0
pmappl     256     1534    0     1504     3     0     3     3     0     8    0
extentpl    40       45    0       27     1     0     1     1     0     8    0
phpool     112      478    0       64    13     0    13    13     0     8    0
ddb{1}> machine ddbcpu 0
Stopped at      x86_ipi_db+0x27:        addq    $0x8,%rsp
ddb{0}> trace
x86_ipi_db(ffffffff83879ff0) at x86_ipi_db+0x27 sys/arch/amd64/amd64/db_interface.c:394
x86_ipi_handler() at x86_ipi_handler+0xd9 sys/arch/amd64/amd64/ipi.c:106
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x27
mtx_enter(ffffffff839f8570) at mtx_enter+0xd2 sys/kern/kern_lock.c:261
wakeup_n(ffff80000002c000,1) at wakeup_n+0x54 sys/kern/kern_synch.c:579
task_add(ffff80000002c000,ffff800000039530) at task_add+0x167 sys/kern/kern_task.c:372
ifiq_input(ffff8000000394c0,ffff80003c4308b8) at ifiq_input+0x4b0 sys/net/ifq.c:794
vio_rxeof(ffff80000002c400) at vio_rxeof+0x40b sys/dev/pv/if_vio.c:1553
vio_rx_intr(ffff80000019f800) at vio_rx_intr+0x98 sys/dev/pv/if_vio.c:1569
vio_queue_intr(ffff80000002c400) at vio_queue_intr+0x59 sys/dev/pv/if_vio.c:909
intr_handler(ffff80003c430a60,ffff800000079d80) at intr_handler+0x11d sys/arch/amd64/amd64/intr.c:563
Xintr_ioapic_edge26_untramp() at Xintr_ioapic_edge26_untramp+0x18f
__sanitizer_cov_trace_const_cmp4(0,2) at __sanitizer_cov_trace_const_cmp4+0x1e kd_curproc sys/dev/kcov.c:583 [inline]
__sanitizer_cov_trace_const_cmp4(0,2) at __sanitizer_cov_trace_const_cmp4+0x1e sys/dev/kcov.c:230
mtx_enter(ffffffff839f8570) at mtx_enter+0xaf sys/kern/kern_lock.c:261
endtsleep(ffff80002a2ba2a8) at endtsleep+0x39 sys/kern/kern_synch.c:538
timeout_run(ffffffff83862048,ffff80002a2ba338) at timeout_run+0x13d sys/kern/kern_timeout.c:707
softclock_process_tick_timeout(ffff80002a2ba338,0) at softclock_process_tick_timeout+0x22b sys/kern/kern_timeout.c:765
softclock(0) at softclock+0x16b sys/kern/kern_timeout.c:797
softintr_dispatch(0) at softintr_dispatch+0x140 sys/kern/kern_softintr.c:84
dosoftint(0) at dosoftint+0x54 sys/arch/amd64/amd64/intr.c:847
Xsoftclock() at Xsoftclock+0x27
__sanitizer_cov_trace_pc() at __sanitizer_cov_trace_pc+0x95 kd_claim sys/dev/kcov.c:632 [inline]
__sanitizer_cov_trace_pc() at __sanitizer_cov_trace_pc+0x95 sys/dev/kcov.c:157
mtx_enter(ffffffff839f8570) at mtx_enter+0xdd sys/kern/kern_lock.c:261
sleep_setup(ffff80003c430f10,120,ffffffff833d7766) at sleep_setup+0xc6 sys/kern/kern_synch.c:302
sys_futex(ffff800031c7f228,ffff80003c4310b0,ffff80003c431000) at sys_futex+0xa04 futex_wait sys/kern/sys_futex.c:301 [inline]
sys_futex(ffff800031c7f228,ffff80003c4310b0,ffff80003c431000) at sys_futex+0xa04 sys/kern/sys_futex.c:151
syscall(ffff80003c4310b0) at syscall+0xbc6 mi_syscall sys/sys/syscall_mi.h:176 [inline]
syscall(ffff80003c4310b0) at syscall+0xbc6 sys/arch/amd64/amd64/trap.c:579
Xsyscall() at Xsyscall+0x128
end of kernel
end trace frame: 0xd8ef6a57620, count: -27
ddb{0}> machine ddbcpu 1
Stopped at      db_enter+0x25:  addq    $0x8,%rsp
ddb{1}> trace
db_enter() at db_enter+0x25 sys/arch/amd64/amd64/db_interface.c:438
witness_checkorder(ffff8000299dee00,9,0) at witness_checkorder+0x10fb sys/kern/subr_witness.c:-1
mtx_enter(ffff8000299dedf0) at mtx_enter+0x4a sys/kern/kern_lock.c:257
clockintr_cancel(ffff8000299de2f8) at clockintr_cancel+0x2f sys/kern/kern_clockintr.c:323
mi_switch() at mi_switch+0x172 x86_atomic_clearbits_u32 machine/atomic.h:-1 [inline]
mi_switch() at mi_switch+0x172 sys/kern/sched_bsd.c:374
ast(ffff80003c427db0) at ast+0x165 mi_ast sys/sys/syscall_mi.h:252 [inline]
ast(ffff80003c427db0) at ast+0x165 sys/arch/amd64/amd64/trap.c:543
intr_user_exit() at intr_user_exit+0x3c
end of kernel
end trace frame: 0x70ba05b8db90, count: -7

Crashes (3):
Time Kernel Commit Syzkaller Config Log Report Syz repro C repro VM info Assets (help?) Manager Title
2025/06/20 16:17 openbsd 58d1a60550ab e3003213 .config console log report [disk image] [bsd.gdb] [kernel image] ci-openbsd-multicore witness: reversal: &sched_lock &cq->cq_mtx
2025/06/07 08:03 openbsd 16c80f155e04 4826c28e .config console log report [disk image] [bsd.gdb] [kernel image] ci-openbsd-multicore witness: reversal: &sched_lock &cq->cq_mtx
2025/05/15 15:16 openbsd 8b901a85ba65 cfde8269 .config console log report [disk image] [bsd.gdb] [kernel image] ci-openbsd-multicore witness: reversal: &sched_lock &cq->cq_mtx
* Struck through repros no longer work on HEAD.