panic: softclock: invalid to_clock: -558907731 Stopped at db_enter+0x1c: addq $0x8,%rsp TID PID UID PRFLAGS PFLAGS CPU COMMAND db_enter() at db_enter+0x1c sys/arch/amd64/amd64/db_interface.c:437 panic(ffffffff8291a984) at panic+0x17b sys/kern/subr_prf.c:198 softclock(0) at softclock+0x22d sys/kern/kern_timeout.c:759 softintr_dispatch(0) at softintr_dispatch+0xfb sys/arch/amd64/amd64/softintr.c:90 Xsoftclock() at Xsoftclock+0x27 acpicpu_idle() at acpicpu_idle+0x312 sys/dev/acpi/acpicpu.c:1206 sched_idle(ffffffff82d0aff0) at sched_idle+0x41e sys/kern/kern_sched.c:183 end trace frame: 0x0, count: 8 https://www.openbsd.org/ddb.html describes the minimum info required in bug reports. Insufficient info makes it difficult to find and fix bugs. ddb{0}> ddb{0}> set $lines = 0 ddb{0}> set $maxwidth = 0 ddb{0}> show panic *cpu0: softclock: invalid to_clock: -558907731 ddb{0}> trace db_enter() at db_enter+0x1c sys/arch/amd64/amd64/db_interface.c:437 panic(ffffffff8291a984) at panic+0x17b sys/kern/subr_prf.c:198 softclock(0) at softclock+0x22d sys/kern/kern_timeout.c:759 softintr_dispatch(0) at softintr_dispatch+0xfb sys/arch/amd64/amd64/softintr.c:90 Xsoftclock() at Xsoftclock+0x27 acpicpu_idle() at acpicpu_idle+0x312 sys/dev/acpi/acpicpu.c:1206 sched_idle(ffffffff82d0aff0) at sched_idle+0x41e sys/kern/kern_sched.c:183 end trace frame: 0x0, count: -7 ddb{0}> show registers rdi 0 rsi 0x1 rbp 0xffff80002a101f00 rbx 0xffffffff82d0bcb7 cpu_info_full_primary+0x2cb7 rdx 0 rcx 0xffff80002a0fbaa0 rax 0xffffffff82d0aff0 cpu_info_full_primary+0x1ff0 r8 0x101010101010101 r9 0x8080808080808080 r10 0x38791a12891f66fb r11 0x38f10be164fcbca r12 0xffffffff82d0bab8 cpu_info_full_primary+0x2ab8 r13 0 r14 0 r15 0x1 rip 0xffffffff81ec64bc db_enter+0x1c cs 0x8 rflags 0x246 rsp 0xffff80002a101ef0 ss 0x10 db_enter+0x1c: addq $0x8,%rsp ddb{0}> show proc PROC (idle0) tid=365690 pid=4204 tcnt=1 stat=onproc flags process=14000 proc=40000200 runpri=0, usrpri=50, slppri=0, nice=20 wchan=0x0, wmesg=, ps_single=0x0 forw=0x11a648560653aa72, list=0xffff80002a0fc298,0xffff80002a0fb808 process=0xffff8000fffff0d0 user=0xffff80002a0fd000, vmspace=0xffffffff82e83f80 estcpu=0, cpticks=60133, pctcpu=0.0, user=0, sys=0, intr=0 ddb{0}> ps PID TID PPID UID S FLAGS WAIT COMMAND 37813 455167 97131 0 3 0x82 piperd syz-executor.3 1039 450945 97131 0 3 0x82 piperd syz-executor.1 84255 226358 97131 0 3 0x82 piperd syz-executor.4 69676 50738 1 0 3 0x100083 ttyin getty 97090 281008 97131 0 3 0x82 piperd syz-executor.7 84155 77648 97131 0 3 0x82 piperd syz-executor.6 38181 30586 97131 0 3 0x82 piperd syz-executor.2 7187 197976 0 0 3 0x14200 acct acct 84310 9716 97131 0 3 0x82 piperd syz-executor.0 57293 56669 97131 0 3 0x82 wait syz-executor.5 51087 512994 0 0 3 0x14280 nfsidl nfsio 77950 476895 0 0 3 0x14280 nfsidl nfsio 18057 445638 0 0 3 0x14280 nfsidl nfsio 51553 494003 0 0 3 0x14280 nfsidl nfsio 2688 33637 0 0 3 0x14280 nfsidl nfsio 89735 457644 0 0 3 0x14200 bored sosplice 42221 498626 13337 0 3 0x100082 netio arp 13337 110471 1 0 3 0x10008a sigsusp sh 97131 441063 94452 0 3 0x2000082 thrsleep syz-fuzzer 97131 462008 94452 0 3 0x6000082 thrsleep syz-fuzzer 97131 299140 94452 0 3 0x6000082 wait syz-fuzzer 97131 65159 94452 0 3 0x6000082 wait syz-fuzzer 97131 252045 94452 0 3 0x6000082 wait syz-fuzzer 97131 32081 94452 0 3 0x6000082 thrsleep syz-fuzzer 97131 294657 94452 0 3 0x6000082 wait syz-fuzzer 97131 432269 94452 0 3 0x6000082 thrsleep syz-fuzzer 97131 333125 94452 0 3 0x6000082 kqread syz-fuzzer 97131 296140 94452 0 3 0x6000082 thrsleep syz-fuzzer 97131 285248 94452 0 3 0x6000082 thrsleep syz-fuzzer 97131 389618 94452 0 3 0x6000082 wait syz-fuzzer 97131 145873 94452 0 3 0x6000082 wait syz-fuzzer 97131 225659 94452 0 3 0x6000082 wait syz-fuzzer 97131 467237 94452 0 3 0x6000082 thrsleep syz-fuzzer 97131 386341 94452 0 3 0x6000082 wait syz-fuzzer 94452 305768 23519 0 3 0x10008a sigsusp ksh 23519 33484 37255 0 3 0x9a kqread sshd 37255 269298 1 0 3 0x88 kqread sshd 13663 112172 9673 74 3 0x1100092 bpf pflogd 9673 335835 1 0 3 0x80 netio pflogd 697 419810 966 73 3 0x1100090 kqread syslogd 966 87069 1 0 3 0x100082 netio syslogd 77104 308842 1 0 3 0x100080 kqread resolvd 57506 398815 45592 77 3 0x100092 kqread dhcpleased 86429 439692 45592 77 3 0x100092 kqread dhcpleased 45592 93345 1 0 3 0x80 kqread dhcpleased 23998 416933 0 0 3 0x14200 bored smr 70959 415555 0 0 3 0x14200 pgzero zerothread 73026 192530 0 0 3 0x14200 aiodoned aiodoned 15512 421723 0 0 3 0x14200 syncer update 72465 44637 0 0 3 0x14200 cleaner cleaner 38287 182858 0 0 3 0x14200 reaper reaper 31588 261811 0 0 3 0x14200 pgdaemon pagedaemon 40918 79838 0 0 3 0x14200 bored viomb 81261 406147 0 0 3 0x40014200 acpi0 acpi0 39610 349401 0 0 7 0x40014200 idle1 34703 331901 0 0 3 0x14200 bored softnet3 27722 506195 0 0 3 0x14200 bored softnet2 35406 420599 0 0 3 0x14200 bored softnet1 81875 284234 0 0 3 0x14200 bored softnet0 53988 496520 0 0 3 0x14200 bored systqmp 32691 389035 0 0 3 0x14200 bored systq 75347 506186 0 0 3 0x14200 tmoslp softclockmp 35463 263920 0 0 3 0x40014200 tmoslp softclock * 4204 365690 0 0 7 0x40014200 idle0 1 370890 0 0 3 0x80082 wait init 0 0 -1 0 3 0x10200 scheduler swapper ddb{0}> show all locks CPU 0: exclusive mutex /syzkaller/managers/multicore/kernel/sys/kern/kern_timeout.c:57 r = 0 (0xffffffff82cfed60) #0 witness_lock+0x447 #1 mtx_enter_try+0x104 #2 mtx_enter+0x4f sys/kern/kern_lock.c:266 #3 softclock+0x31 sys/kern/kern_timeout.c:746 #4 softintr_dispatch+0xfb sys/arch/amd64/amd64/softintr.c:90 #5 Xsoftclock+0x27 #6 acpicpu_idle+0x312 sys/dev/acpi/acpicpu.c:1206 #7 sched_idle+0x41e sys/kern/kern_sched.c:183 #8 proc_trampoline+0x10 Process 4204 (idle0) thread 0xffff80002a0fbaa0 (365690) ddb{0}> show malloc Type InUse MemUse HighUse Limit Requests Type Lim devbuf 10231 6520K 14837K 166960K 79944 0 pcb 15 24K 29K 166960K 2178 0 rtable 245 15K 16K 166960K 5340 0 pf 38 10K 11K 166960K 604 0 ifaddr 48 19K 20K 166960K 700 0 ifgroup 67 2K 3K 166960K 934 0 sysctl 4 1K 2K 166960K 24 0 counters 70 37K 37K 166960K 574 0 ioctlops 0 0K 4K 166960K 2897 0 iov 0 0K 32K 166960K 6424 0 mount 1 1K 1K 166960K 1 0 log 0 0K 0K 166960K 4 0 vnodes 1614 101K 101K 166960K 20576 0 UFS quota 1 32K 32K 166960K 1 0 UFS mount 5 36K 36K 166960K 5 0 shm 2 1K 13K 166960K 327 0 VM map 2 1K 1K 166960K 2 0 sem 12 0K 0K 166960K 6394 0 dirhash 12 2K 2K 166960K 126 0 ACPI 1697 195K 286K 166960K 12548 0 file desc 13 45K 89K 166960K 32681 0 sigio 0 0K 0K 166960K 1303 0 proc 72 91K 128K 166960K 4136 0 subproc 117 7K 9K 166960K 1221 0 NFS srvsock 1 0K 0K 166960K 1 0 NFS daemon 1 16K 16K 166960K 1 0 ip_moptions 0 0K 0K 166960K 1355 0 in_multi 99 7K 7K 166960K 2123 0 ether_multi 1 0K 0K 166960K 32 0 mrt 1 0K 0K 166960K 14 0 ISOFS mount 1 32K 32K 166960K 1 0 MSDOSFS mount 1 16K 16K 166960K 1 0 ttys 229 1023K 1023K 166960K 229 0 exec 0 0K 1K 166960K 14800 0 pfkey data 0 0K 4K 166960K 56 0 tdb 3 0K 0K 166960K 3 0 VM swap 8 62K 64K 166960K 10 0 UVM amap 646 1357K 1362K 166960K 299932 0 UVM aobj 131 5K 5K 166960K 134 0 memdesc 1 4K 4K 166960K 1 0 crypto data 1 1K 1K 166960K 1 0 ip6_options 0 0K 0K 166960K 1268 0 NDP 15 0K 1K 166960K 591 0 temp 74 6772K 7412K 166960K 239940 0 kqueue 12 18K 30K 166960K 2174 0 SYN cache 2 36K 44K 166960K 3 0 ddb{0}> show all pools Name Size Requests Fail Releases Pgreq Pgrel Npage Hiwat Minpg Maxpg Idle plcache 128 24 0 0 1 0 1 1 0 8 0 rtpcb 120 1529 0 1525 16 15 1 3 0 8 0 rtentry 112 1851 0 1740 5 1 4 4 0 8 0 unpcb 144 23597 0 23582 198 194 4 11 0 8 3 syncache 336 257 0 257 41 41 0 1 0 8 0 sackhl 24 3 50 3 1 1 0 1 0 8 0 tcpqe 32 485 0 485 29 29 0 1 0 8 0 tcpcb 808 8096 0 8091 209 205 4 19 0 8 3 arp 120 239 0 221 1 0 1 1 0 8 0 inpcb 392 25542 0 25534 308 303 5 24 0 8 4 nd6 136 422 0 397 1 0 1 1 0 8 0 pkpcb 40 218 0 218 15 15 0 1 0 8 0 kcovpl 48 87 0 78 1 0 1 1 0 8 0 ppxss 1168 96 0 96 24 23 1 1 0 8 1 pffrag 232 136 0 135 8 7 1 1 0 482 0 pffrnode 88 136 0 135 8 7 1 1 0 8 0 pffrent 40 609 0 608 8 7 1 1 0 8 0 pfosfp 40 1428 0 1005 5 0 5 5 0 8 0 pfosfpen 112 1428 0 714 21 0 21 21 0 8 0 pfstitem 24 875 0 844 1 0 1 1 0 8 0 pfstkey 128 875 0 844 4 2 2 2 0 8 0 pfstate 376 875 0 844 17 13 4 6 0 8 0 pfrule 1344 21 0 16 2 1 1 2 0 8 0 rttmr 136 2 0 2 1 1 0 1 0 8 0 art_heap8 4096 6 0 5 4 3 1 3 0 8 0 art_heap4 256 8098 0 7627 59 29 30 31 0 8 0 art_table 32 8104 0 7632 4 0 4 4 0 8 0 art_node 16 1772 0 1671 1 0 1 1 0 8 0 sysvmsgpl 40 6 0 0 1 0 1 1 0 8 0 semupl 112 3 0 3 1 1 0 1 0 8 0 semapl 112 6392 0 6382 1 0 1 1 0 8 0 shmpl 112 131 0 3 4 0 4 4 0 8 0 dirhash 1024 93 0 76 3 0 3 3 0 8 0 dino2pl 256 45517 0 43967 98 0 98 98 0 8 0 ffsino 272 45517 0 43967 104 0 104 104 0 8 0 nchpl 144 90168 0 88521 63 0 63 63 0 8 0 uvmvnodes 80 6462 0 0 132 0 132 132 0 8 0 vnodes 216 6462 0 0 359 0 359 359 0 8 0 namei 1024 307972 0 307972 13 12 1 2 0 8 1 percpumem 16 301 0 252 1 0 1 1 0 8 0 vcpupl 2048 315 0 1 40 0 40 40 0 8 0 vmpool 696 686 0 372 29 0 29 29 0 8 0 kstatmem 264 534 0 504 9 6 3 3 0 8 0 pool(kstatmem): free list modified: page 0xfffffd806b679000; item ordinal 2; addr 0xfffffd806b679e70 (p 0xfffffd806b679000); offset 0x0=0xffffffff kstatmem: pool(0xffffffff82e5ce08:kstatmem): page inconsistency: page 0xfffffd806b679000; item ordinal 3; addr 0xbd0278465193e197 scxspl 216 262419 0 262419 51 50 1 8 1 8 1 plimitpl 152 3682 0 3665 1 0 1 1 0 8 0 sigapl 424 32910 0 32859 12 4 8 9 0 8 0 futexpl 64 323691 0 323691 4 3 1 1 0 8 1 knotepl 120 1077 0 0 20 1 19 19 0 8 0 kqueuepl 216 6368 0 6360 116 111 5 8 0 8 4 pipepl 320 4746 0 4715 147 144 3 11 0 8 0 fdescpl 496 32838 0 32812 5 0 5 5 0 8 0 filepl 152 197564 0 197305 268 254 14 28 0 8 1 lockfpl 104 8887 0 8885 15 14 1 3 0 8 0 lockfspl 48 2924 0 2922 1 0 1 1 0 8 0 sessionpl 144 117 0 99 1 0 1 1 0 8 0 pgrppl 48 415 0 397 1 0 1 1 0 8 0 ucredpl 104 24760 0 24746 1 0 1 1 0 8 0 zombiepl 144 32860 0 32859 4 3 1 1 0 8 0 processpl 1136 32910 0 32859 7 2 5 6 0 8 0 procpl 680 79755 0 79689 21 12 9 10 0 8 0 srpgc 96 84 0 84 34 34 0 1 0 8 0 sosppl 168 409 0 409 28 27 1 1 0 8 1 sockpl 584 50991 0 50964 390 380 10 25 0 8 8 mcl64k 65536 17 0 0 3 0 3 3 0 8 0 mcl16k 16384 25 0 0 3 0 3 3 0 8 0 mcl12k 12288 17 0 0 2 0 2 2 0 8 0 mcl9k 9216 15 0 0 2 0 2 2 0 8 0 mcl8k 8192 18 0 0 3 0 3 3 0 8 0 mcl4k 4096 21 0 0 3 0 3 3 0 8 0 mcl2k2 2112 4 0 0 1 0 1 1 0 8 0 mcl2k 2048 674 0 0 39 14 25 39 0 8 0 mtagpl 96 1015 0 0 13 0 13 13 0 8 0 mbufpl 256 7660 0 0 474 0 474 474 0 8 0 bufpl 280 58581 0 52119 462 0 462 462 0 8 0 anonpl 24 3005097 0 2990057 259 123 136 149 0 186 0 amapchunkpl 152 948620 0 947760 163 117 46 53 0 158 1 amappl16 200 54890 0 54123 212 158 54 55 0 8 6 amappl15 192 134 0 131 1 0 1 1 0 8 0 amappl14 184 457 0 432 4 2 2 2 0 8 0 amappl13 176 24 0 23 1 0 1 1 0 8 0 amappl12 168 34730 0 34695 4 1 3 3 0 8 0 amappl11 160 60 0 46 1 0 1 1 0 8 0 amappl10 152 154 0 134 1 0 1 1 0 8 0 amappl9 144 2196 0 2195 1 0 1 1 0 8 0 amappl8 136 1231 0 1041 9 0 9 9 0 8 0 amappl7 128 485 0 456 2 0 2 2 0 8 0 amappl6 120 1687 0 1664 1 0 1 1 0 8 0 amappl5 112 673 0 661 1 0 1 1 0 8 0 amappl4 104 1514 0 1468 4 2 2 2 0 8 0 amappl3 96 186643 0 186572 8 5 3 4 0 8 0 amappl2 88 34631 0 34541 6 3 3 3 0 8 0 amappl1 80 130847 0 130271 23 10 13 23 0 8 0 amappl 88 297835 0 297515 12 3 9 9 0 92 0 dma4096 4096 1 0 1 1 1 0 1 0 8 0 dma1024 1024 1 0 0 1 0 1 1 0 8 0 dma256 256 6 0 6 1 1 0 1 0 8 0 dma128 128 253 0 253 1 1 0 1 0 8 0 dma64 64 6 0 6 1 1 0 1 0 8 0 dma32 32 7 0 7 1 1 0 1 0 8 0 dma16 16 18 0 17 1 0 1 1 0 8 0 aobjpl 72 133 0 3 3 0 3 3 0 8 0 uaddrrnd 24 33524 0 33184 3 0 3 3 0 8 0 uaddrbest 32 2 0 0 1 0 1 1 0 8 0 uaddr 24 33524 0 33184 3 0 3 3 0 8 0 vmmpekpl 168 196229 0 196113 8 2 6 6 0 8 0 vmmpepl 168 1913307 0 1910260 863 660 203 203 0 357 36 vmsppl 448 33523 0 33184 42 3 39 39 0 8 0 rwobjpl 56 427379 0 418627 149 20 129 129 0 8 0 pdppl 4096 67055 0 66682 1077 692 385 389 0 8 12 pvpl 32 44650 0 0 361 0 361 361 0 265 0 pmappl 248 33523 0 33184 22 0 22 22 0 8 0 extentpl 40 56 0 38 1 0 1 1 0 8 0 phpool 112 2653 0 1416 37 1 36 36 0 8 0 ddb{0}> machine ddbcpu 0 Invalid cpu 0 ddb{0}> trace db_enter() at db_enter+0x1c sys/arch/amd64/amd64/db_interface.c:437 panic(ffffffff8291a984) at panic+0x17b sys/kern/subr_prf.c:198 softclock(0) at softclock+0x22d sys/kern/kern_timeout.c:759 softintr_dispatch(0) at softintr_dispatch+0xfb sys/arch/amd64/amd64/softintr.c:90 Xsoftclock() at Xsoftclock+0x27 acpicpu_idle() at acpicpu_idle+0x312 sys/dev/acpi/acpicpu.c:1206 sched_idle(ffffffff82d0aff0) at sched_idle+0x41e sys/kern/kern_sched.c:183 end trace frame: 0x0, count: -7 ddb{0}> machine ddbcpu 1 Stopped at x86_ipi_db+0x1e: addq $0x8,%rsp x86_ipi_db(ffff800029cebff0) at x86_ipi_db+0x1e sys/arch/amd64/amd64/db_interface.c:393 x86_ipi_handler() at x86_ipi_handler+0xb7 sys/arch/amd64/amd64/ipi.c:106 Xresume_lapic_ipi() at Xresume_lapic_ipi+0x27 acpicpu_idle() at acpicpu_idle+0x312 sys/dev/acpi/acpicpu.c:1206 sched_idle(ffff800029cebff0) at sched_idle+0x41e sys/kern/kern_sched.c:183 end trace frame: 0x0, count: 10 ddb{1}> trace x86_ipi_db(ffff800029cebff0) at x86_ipi_db+0x1e sys/arch/amd64/amd64/db_interface.c:393 x86_ipi_handler() at x86_ipi_handler+0xb7 sys/arch/amd64/amd64/ipi.c:106 Xresume_lapic_ipi() at Xresume_lapic_ipi+0x27 acpicpu_idle() at acpicpu_idle+0x312 sys/dev/acpi/acpicpu.c:1206 sched_idle(ffff800029cebff0) at sched_idle+0x41e sys/kern/kern_sched.c:183 end trace frame: 0x0, count: -5