panic: softclock: invalid to_clock: 249 Stopped at db_enter+0x1c: addq $0x8,%rsp TID PID UID PRFLAGS PFLAGS CPU COMMAND db_enter() at db_enter+0x1c sys/arch/amd64/amd64/db_interface.c:437 panic(ffffffff8285200a) at panic+0x17b sys/kern/subr_prf.c:198 softclock(0) at softclock+0x22d sys/kern/kern_timeout.c:759 softintr_dispatch(0) at softintr_dispatch+0xcd sys/arch/amd64/amd64/softintr.c:90 Xsoftclock() at Xsoftclock+0x27 acpicpu_idle() at acpicpu_idle+0x2f1 sys/dev/acpi/acpicpu.c:1206 sched_idle(ffffffff82cc1ff0) at sched_idle+0x41d sys/kern/kern_sched.c:183 end trace frame: 0x0, count: 8 https://www.openbsd.org/ddb.html describes the minimum info required in bug reports. Insufficient info makes it difficult to find and fix bugs. ddb{0}> ddb{0}> set $lines = 0 ddb{0}> set $maxwidth = 0 ddb{0}> show panic *cpu0: softclock: invalid to_clock: 249 ddb{0}> trace db_enter() at db_enter+0x1c sys/arch/amd64/amd64/db_interface.c:437 panic(ffffffff8285200a) at panic+0x17b sys/kern/subr_prf.c:198 softclock(0) at softclock+0x22d sys/kern/kern_timeout.c:759 softintr_dispatch(0) at softintr_dispatch+0xcd sys/arch/amd64/amd64/softintr.c:90 Xsoftclock() at Xsoftclock+0x27 acpicpu_idle() at acpicpu_idle+0x2f1 sys/dev/acpi/acpicpu.c:1206 sched_idle(ffffffff82cc1ff0) at sched_idle+0x41d sys/kern/kern_sched.c:183 end trace frame: 0x0, count: -7 ddb{0}> show registers rdi 0 rsi 0x1 rbp 0xffff80002a101df0 rbx 0xffffffff82cc2cb7 cpu_info_full_primary+0x2cb7 rdx 0 rcx 0xffff80002a0fca90 rax 0xffffffff82cc1ff0 cpu_info_full_primary+0x1ff0 r8 0x101010101010101 r9 0x8080808080808080 r10 0x43eb102368590a28 r11 0xaefb82423af01ee4 r12 0xffffffff82cc2ab8 cpu_info_full_primary+0x2ab8 r13 0 r14 0 r15 0x1 rip 0xffffffff826c18ec db_enter+0x1c cs 0x8 rflags 0x246 rsp 0xffff80002a101de0 ss 0x10 db_enter+0x1c: addq $0x8,%rsp ddb{0}> show proc PROC (idle0) tid=317630 pid=80769 tcnt=1 stat=onproc flags process=14000 proc=40000200 runpri=0, usrpri=50, slppri=0, nice=20 wchan=0x0, wmesg=, ps_single=0x0 forw=0xeb138b991ca6586b, list=0xffff80002a0fcd38,0xffff80002a0fc550 process=0xffff8000fffff540 user=0xffff80002a0fd000, vmspace=0xffffffff82d63498 estcpu=0, cpticks=80370, pctcpu=0.0, user=0, sys=0, intr=0 ddb{0}> ps PID TID PPID UID S FLAGS WAIT COMMAND 63931 341069 30239 0 3 0x82 piperd syz-executor.2 92232 204101 30239 0 3 0x82 piperd syz-executor.1 61956 152772 30239 0 3 0x82 piperd syz-executor.0 87960 445605 30239 0 3 0x82 piperd syz-executor.4 84826 299530 30239 0 3 0x82 piperd syz-executor.5 44161 94500 30239 0 3 0x82 piperd syz-executor.3 31952 143662 30239 0 3 0x82 piperd syz-executor.7 83056 416415 1 0 3 0x80 fsleep syz-executor.5 83056 415130 1 0 3 0x4000080 netcon syz-executor.5 15907 498508 30239 0 3 0x82 wait syz-executor.6 52952 484806 1 0 3 0x80 fsleep syz-executor.4 52952 514621 1 0 3 0x4000080 netcon syz-executor.4 6678 143895 1 0 3 0x18100083 ttyopn getty 23664 120231 1 0 3 0x80 fsleep syz-executor.0 23664 283922 1 0 3 0x4000080 fsleep syz-executor.0 23664 326237 1 0 3 0x4000080 netcon syz-executor.0 64267 120905 0 0 3 0x14280 nfsidl nfsio 56576 402637 0 0 3 0x14280 nfsidl nfsio 2849 201110 0 0 3 0x14280 nfsidl nfsio 2445 463099 0 0 3 0x14280 nfsidl nfsio 33745 325181 0 0 3 0x14280 nfsidl nfsio 99674 522861 0 0 3 0x14280 nfsidl nfsio 10558 432157 0 0 3 0x14280 nfsidl nfsio 96468 262344 0 0 3 0x14280 nfsidl nfsio 24519 421074 0 0 3 0x14280 nfsidl nfsio 71048 4630 0 0 3 0x14280 nfsidl nfsio 99533 402635 0 0 3 0x14280 nfsidl nfsio 91713 316626 0 0 3 0x14280 nfsidl nfsio 19394 469829 0 0 3 0x14280 nfsidl nfsio 90471 268004 0 0 3 0x14280 nfsidl nfsio 88228 68489 0 0 3 0x14280 nfsidl nfsio 76573 458605 0 0 3 0x14280 nfsidl nfsio 49395 49790 0 0 3 0x14280 nfsidl nfsio 64414 366850 0 0 3 0x14280 nfsidl nfsio 69807 167516 0 0 3 0x14280 nfsidl nfsio 6648 204136 0 0 3 0x14280 nfsidl nfsio 12903 371366 1 0 3 0x80 fsleep syz-executor.5 12903 329260 1 0 3 0x4000080 fsleep syz-executor.5 12903 191630 1 0 3 0x4000080 ttyin syz-executor.5 35538 224653 0 0 3 0x14200 bored sosplice 30239 362135 51407 0 3 0x1a000082 thrsleep syz-fuzzer 30239 122063 51407 0 3 0x1e000082 thrsleep syz-fuzzer 30239 396743 51407 0 3 0x1e000082 wait syz-fuzzer 30239 251802 51407 0 3 0x1e000082 wait syz-fuzzer 30239 61342 51407 0 3 0x1e000082 wait syz-fuzzer 30239 366808 51407 0 3 0x1e000082 kqread syz-fuzzer 30239 34293 51407 0 3 0x1e000082 wait syz-fuzzer 30239 67969 51407 0 3 0x1e000082 thrsleep syz-fuzzer 30239 409067 51407 0 3 0x1e000082 thrsleep syz-fuzzer 30239 207522 51407 0 3 0x1e000082 wait syz-fuzzer 30239 107334 51407 0 3 0x1e000082 thrsleep syz-fuzzer 30239 15840 51407 0 3 0x1e000082 wait syz-fuzzer 30239 386778 51407 0 3 0x1e000082 wait syz-fuzzer 30239 22857 51407 0 3 0x1e000082 thrsleep syz-fuzzer 30239 418633 51407 0 3 0x1e000082 thrsleep syz-fuzzer 30239 126343 51407 0 3 0x1e000082 wait syz-fuzzer 51407 48229 5740 0 3 0x810008a sigsusp ksh 5740 104898 44468 0 3 0x1800009a kqread sshd 44468 9795 1 0 3 0x18000088 kqread sshd 1338 12037 84075 74 3 0x19100092 bpf pflogd 84075 341928 1 0 3 0x18000080 netio pflogd 63551 447457 35186 73 3 0x19100090 kqread syslogd 35186 198442 1 0 3 0x18100082 netio syslogd 74283 54895 1 0 3 0x18100080 kqread resolvd 7445 78802 30715 77 3 0x18100092 kqread dhcpleased 87950 113731 30715 77 3 0x18100092 kqread dhcpleased 30715 501281 1 0 3 0x18000080 kqread dhcpleased 26941 509662 0 0 3 0x14200 bored smr 31969 329210 0 0 3 0x14200 pgzero zerothread 99896 91834 0 0 3 0x14200 aiodoned aiodoned 15018 478211 0 0 3 0x14200 syncer update 32818 137124 0 0 3 0x14200 cleaner cleaner 47516 152254 0 0 3 0x14200 reaper reaper 48515 199415 0 0 3 0x14200 pgdaemon pagedaemon 61576 205221 0 0 3 0x14200 bored viomb 60558 130535 0 0 3 0x40014200 acpi0 acpi0 67464 1874 0 0 7 0x40014200 idle1 90692 271265 0 0 3 0x14200 bored softnet3 24989 490553 0 0 3 0x14200 bored softnet2 8161 330249 0 0 3 0x14200 bored softnet1 89391 329394 0 0 3 0x14200 bored softnet0 53966 490536 0 0 3 0x14200 bored systqmp 83401 505724 0 0 3 0x14200 bored systq 64886 283885 0 0 3 0x14200 tmoslp softclockmp 84782 24741 0 0 3 0x40014200 tmoslp softclock *80769 317630 0 0 7 0x40014200 idle0 1 117549 0 0 3 0x8080082 wait init 0 0 -1 0 3 0x10200 scheduler swapper ddb{0}> show all locks CPU 0: exclusive mutex /syzkaller/managers/multicore/kernel/sys/kern/kern_timeout.c:57 r = 0 (0xffffffff82cbd260) #0 witness_lock+0x446 stacktrace_save sys/sys/stacktrace.h:37 [inline] #0 witness_lock+0x446 sys/kern/subr_witness.c:1187 #1 mtx_enter_try+0x103 #2 mtx_enter+0x4e sys/kern/kern_lock.c:266 #3 softclock+0x31 sys/kern/kern_timeout.c:746 #4 softintr_dispatch+0xcd sys/arch/amd64/amd64/softintr.c:90 #5 Xsoftclock+0x27 #6 acpicpu_idle+0x2f1 sys/dev/acpi/acpicpu.c:1206 #7 sched_idle+0x41d sys/kern/kern_sched.c:183 #8 proc_trampoline+0x10 Process 80769 (idle0) thread 0xffff80002a0fca90 (317630) exclusive kernel_lock &kernel_lock r = 0 (0xffffffff82d60cf8) #0 witness_lock+0x446 stacktrace_save sys/sys/stacktrace.h:37 [inline] #0 witness_lock+0x446 sys/kern/subr_witness.c:1187 #1 softintr_dispatch+0x52 sys/arch/amd64/amd64/softintr.c:88 #2 Xsoftclock+0x27 #3 acpicpu_idle+0x2f1 sys/dev/acpi/acpicpu.c:1206 #4 sched_idle+0x41d sys/kern/kern_sched.c:183 #5 proc_trampoline+0x10 exclusive mutex /syzkaller/managers/multicore/kernel/sys/kern/kern_timeout.c:57 r = 0 (0xffffffff82cbd260) #0 witness_lock+0x446 stacktrace_save sys/sys/stacktrace.h:37 [inline] #0 witness_lock+0x446 sys/kern/subr_witness.c:1187 #1 mtx_enter_try+0x103 #2 mtx_enter+0x4e sys/kern/kern_lock.c:266 #3 softclock+0x31 sys/kern/kern_timeout.c:746 #4 softintr_dispatch+0xcd sys/arch/amd64/amd64/softintr.c:90 #5 Xsoftclock+0x27 #6 acpicpu_idle+0x2f1 sys/dev/acpi/acpicpu.c:1206 #7 sched_idle+0x41d sys/kern/kern_sched.c:183 #8 proc_trampoline+0x10 ddb{0}> show malloc Type InUse MemUse HighUse Limit Requests Type Lim devbuf 10230 6512K 14688K 166960K 23356 0 pcb 15 20K 22K 166960K 1486 0 rtable 158 5K 7K 166960K 3848 0 pf 31 9K 10K 166960K 354 0 ifaddr 33 14K 18K 166960K 366 0 ifgroup 52 2K 2K 166960K 578 0 sysctl 3 0K 4K 166960K 16 0 counters 62 36K 36K 166960K 364 0 ioctlops 0 0K 4K 166960K 2196 0 iov 0 0K 24K 166960K 1729 0 mount 1 1K 1K 166960K 1 0 log 0 0K 0K 166960K 4 0 vnodes 1547 97K 98K 166960K 7210 0 UFS quota 1 32K 32K 166960K 1 0 UFS mount 5 36K 36K 166960K 5 0 shm 2 1K 5K 166960K 145 0 VM map 2 1K 1K 166960K 2 0 sem 12 0K 0K 166960K 2307 0 dirhash 12 2K 2K 166960K 108 0 ACPI 1697 195K 286K 166960K 12548 0 file desc 15 53K 93K 166960K 16463 0 sigio 0 0K 0K 166960K 520 0 proc 74 91K 116K 166960K 2809 0 subproc 156 9K 9K 166960K 849 0 NFS srvsock 1 0K 0K 166960K 1 0 NFS daemon 1 16K 16K 166960K 1 0 ip_moptions 0 0K 0K 166960K 1116 0 in_multi 56 4K 7K 166960K 1013 0 ether_multi 1 0K 0K 166960K 8 0 mrt 1 0K 0K 166960K 9 0 ISOFS mount 1 32K 32K 166960K 1 0 MSDOSFS mount 1 16K 16K 166960K 1 0 ttys 301 1341K 1341K 166960K 301 0 exec 0 0K 1K 166960K 2985 0 pfkey data 0 0K 0K 166960K 82 0 tdb 3 0K 0K 166960K 3 0 VM swap 8 62K 64K 166960K 10 0 UVM amap 547 954K 970K 166960K 162734 0 UVM aobj 131 6K 6K 166960K 140 0 pinsyscall 26 52K 108K 166960K 2925 0 memdesc 1 4K 4K 166960K 1 0 crypto data 1 1K 1K 166960K 1 0 ip6_options 0 0K 1K 166960K 613 0 NDP 11 0K 1K 166960K 281 0 temp 90 6813K 6941K 166960K 131405 0 kqueue 12 18K 27K 166960K 1298 0 SYN cache 2 16K 16K 166960K 2 0 ddb{0}> show all pools Name Size Requests Fail Releases Pgreq Pgrel Npage Hiwat Minpg Maxpg Idle plcache 128 24 0 0 1 0 1 1 0 8 0 rtpcb 120 1377 0 1374 20 19 1 4 0 8 0 rtentry 112 1317 0 1248 5 1 4 4 0 8 0 unpcb 144 14855 0 14840 144 143 1 8 0 8 0 syncache 336 124 0 124 27 27 0 1 0 8 0 tcpqe 32 331 0 331 20 20 0 1 0 8 0 tcpcb 808 5044 0 5019 126 119 7 18 0 8 0 arp 120 282 0 271 1 0 1 1 0 8 0 inpcb 392 13000 0 12970 197 191 6 20 0 8 0 nd6 136 186 0 171 1 0 1 1 0 8 0 pkpcb 40 99 0 99 13 12 1 1 0 8 1 kcovpl 48 64 0 52 1 0 1 1 0 8 0 ppxss 1168 45 0 45 10 10 0 1 0 8 0 pffrag 232 118 0 113 6 5 1 1 0 482 0 pffrnode 88 118 0 113 6 5 1 1 0 8 0 pffrent 40 364 0 359 6 5 1 1 0 8 0 pfosfp 40 1428 0 1005 5 0 5 5 0 8 0 pfosfpen 112 1428 0 714 21 0 21 21 0 8 0 pfstitem 24 491 0 469 1 0 1 1 0 8 0 pfstkey 128 491 0 469 4 2 2 2 0 8 0 pfstate 376 491 0 469 12 8 4 4 0 8 0 pfrule 1344 21 0 16 2 1 1 2 0 8 0 rttmr 136 2 0 2 1 1 0 1 0 8 0 art_heap8 4096 1 0 0 1 0 1 1 0 8 0 art_heap4 256 3520 0 3224 51 23 28 31 0 8 0 art_table 32 3521 0 3224 5 1 4 5 0 8 0 art_node 16 1224 0 1163 1 0 1 1 0 8 0 sysvmsgpl 40 70 0 67 1 0 1 1 0 8 0 semapl 112 2305 0 2295 1 0 1 1 0 8 0 shmpl 112 137 0 9 4 0 4 4 0 8 0 dirhash 1024 81 0 64 3 0 3 3 0 8 0 dino2pl 256 22622 0 21014 102 0 102 102 0 8 0 ffsino 272 22622 0 21014 108 0 108 108 0 8 0 nchpl 144 44862 0 43128 67 0 67 67 0 8 0 uvmvnodes 80 6615 0 0 135 0 135 135 0 8 0 vnodes 216 6615 0 0 368 0 368 368 0 8 0 namei 1024 146036 0 146036 4 3 1 2 0 8 1 percpumem 16 196 0 151 1 0 1 1 0 8 0 vcpupl 2048 178 0 1 23 0 23 23 0 8 0 vmpool 696 211 0 34 19 2 17 17 0 8 0 kstatmem 264 324 0 302 2 0 2 2 0 8 0 scxspl 216 141691 0 141691 54 53 1 8 1 8 1 plimitpl 152 1457 0 1437 1 0 1 1 0 8 0 sigapl 424 16699 0 16632 9 0 9 9 0 8 0 futexpl 64 154417 0 154411 1 0 1 1 0 8 0 knotepl 120 1144 0 0 14 3 11 12 0 8 0 kqueuepl 216 3233 0 3225 72 71 1 6 0 8 0 pipepl 320 2870 0 2838 85 82 3 9 0 8 0 fdescpl 496 16659 0 16631 8 3 5 5 0 8 0 filepl 152 97704 0 97391 189 175 14 27 0 8 0 lockfpl 104 3995 0 3993 8 7 1 2 0 8 0 lockfspl 48 1366 0 1364 1 0 1 1 0 8 0 sessionpl 144 75 0 54 1 0 1 1 0 8 0 pgrppl 48 174 0 153 1 0 1 1 0 8 0 ucredpl 104 12857 0 12840 1 0 1 1 0 8 0 zombiepl 144 16633 0 16632 2 1 1 1 0 8 0 processpl 1136 16699 0 16632 6 0 6 6 0 8 0 procpl 680 45757 0 45669 26 16 10 10 0 8 0 srpgc 96 79 0 79 27 27 0 1 0 8 0 sosppl 168 181 0 176 9 8 1 1 0 8 0 sockpl 584 29343 0 29294 251 245 6 19 0 8 0 mcl64k 65536 32 0 0 3 0 3 3 0 8 0 mcl16k 16384 25 0 0 3 0 3 3 0 8 0 mcl12k 12288 33 0 0 2 0 2 2 0 8 0 mcl9k 9216 16 0 0 2 0 2 2 0 8 0 mcl8k 8192 58 0 0 5 2 3 3 0 8 0 mcl4k 4096 103 0 0 9 4 5 8 0 8 0 mcl2k2 2112 71 0 0 5 2 3 5 0 8 0 mcl2k 2048 432 0 0 35 18 17 35 0 8 0 mtagpl 96 253 0 0 5 0 5 5 0 8 0 mbufpl 256 1792 0 0 66 0 66 66 0 8 0 bufpl 280 32748 0 26132 473 0 473 473 0 8 0 anonpl 24 1695099 0 1680564 253 146 107 120 0 186 0 amapchunkpl 152 521024 0 520144 128 83 45 55 0 158 0 amappl16 200 32779 0 32314 139 109 30 39 0 8 0 amappl15 192 89 0 87 1 0 1 1 0 8 0 amappl14 184 344 0 326 4 3 1 2 0 8 0 amappl13 176 12 0 12 2 2 0 1 0 8 0 amappl12 168 18058 0 18026 4 1 3 3 0 8 0 amappl11 160 61 0 47 1 0 1 1 0 8 0 amappl10 152 153 0 134 1 0 1 1 0 8 0 amappl9 144 287 0 285 1 0 1 1 0 8 0 amappl8 136 878 0 726 6 0 6 6 0 8 0 amappl7 128 136 0 120 1 0 1 1 0 8 0 amappl6 120 1077 0 1053 2 1 1 2 0 8 0 amappl5 112 680 0 663 1 0 1 1 0 8 0 amappl4 104 1283 0 1235 3 1 2 3 0 8 0 amappl3 96 100009 0 99929 7 3 4 4 0 8 0 amappl2 88 17759 0 17680 4 2 2 4 0 8 0 amappl1 80 70169 0 69624 23 10 13 22 0 8 0 amappl 88 161336 0 161043 15 6 9 9 0 92 0 dma4096 4096 1 0 1 1 1 0 1 0 8 0 dma1024 1024 1 0 0 1 0 1 1 0 8 0 dma256 256 6 0 6 1 1 0 1 0 8 0 dma128 128 253 0 253 1 1 0 1 0 8 0 dma64 64 6 0 6 1 1 0 1 0 8 0 dma32 32 7 0 7 1 1 0 1 0 8 0 dma16 16 18 0 17 1 0 1 1 0 8 0 aobjpl 72 139 0 9 3 0 3 3 0 8 0 uaddrrnd 24 16871 0 16666 2 0 2 2 0 8 0 uaddrbest 32 2 0 0 1 0 1 1 0 8 0 uaddr 24 16871 0 16666 2 0 2 2 0 8 0 vmmpekpl 168 127005 0 126904 5 0 5 5 0 8 0 vmmpepl 168 1009606 0 1006996 555 410 145 162 0 357 0 vmsppl 448 16870 0 16666 26 2 24 24 0 8 0 rwobjpl 56 245767 0 237256 138 15 123 125 0 8 0 pdppl 4096 33749 0 33509 1132 878 254 256 0 8 14 pvpl 32 44758 0 0 363 2 361 361 0 265 0 pmappl 248 16870 0 16666 16 2 14 14 0 8 0 extentpl 40 56 0 38 1 0 1 1 0 8 0 phpool 112 2014 0 1349 20 0 20 20 0 8 0 ddb{0}> machine ddbcpu 0 Invalid cpu 0 ddb{0}> trace db_enter() at db_enter+0x1c sys/arch/amd64/amd64/db_interface.c:437 panic(ffffffff8285200a) at panic+0x17b sys/kern/subr_prf.c:198 softclock(0) at softclock+0x22d sys/kern/kern_timeout.c:759 softintr_dispatch(0) at softintr_dispatch+0xcd sys/arch/amd64/amd64/softintr.c:90 Xsoftclock() at Xsoftclock+0x27 acpicpu_idle() at acpicpu_idle+0x2f1 sys/dev/acpi/acpicpu.c:1206 sched_idle(ffffffff82cc1ff0) at sched_idle+0x41d sys/kern/kern_sched.c:183 end trace frame: 0x0, count: -7 ddb{0}> machine ddbcpu 1 Stopped at x86_ipi_db+0x1e: addq $0x8,%rsp x86_ipi_db(ffff800029cebff0) at x86_ipi_db+0x1e sys/arch/amd64/amd64/db_interface.c:393 x86_ipi_handler() at x86_ipi_handler+0xb7 sys/arch/amd64/amd64/ipi.c:106 Xresume_lapic_ipi() at Xresume_lapic_ipi+0x27 acpicpu_idle() at acpicpu_idle+0x2f1 sys/dev/acpi/acpicpu.c:1206 sched_idle(ffff800029cebff0) at sched_idle+0x41d sys/kern/kern_sched.c:183 end trace frame: 0x0, count: 10 ddb{1}> trace x86_ipi_db(ffff800029cebff0) at x86_ipi_db+0x1e sys/arch/amd64/amd64/db_interface.c:393 x86_ipi_handler() at x86_ipi_handler+0xb7 sys/arch/amd64/amd64/ipi.c:106 Xresume_lapic_ipi() at Xresume_lapic_ipi+0x27 acpicpu_idle() at acpicpu_idle+0x2f1 sys/dev/acpi/acpicpu.c:1206 sched_idle(ffff800029cebff0) at sched_idle+0x41d sys/kern/kern_sched.c:183 end trace frame: 0x0, count: -5