panic: wakeup: p_stat is 0 Stopped at db_enter+0x18: addq $0x8,%rsp TID PID UID PRFLAGS PFLAGS CPU COMMAND db_enter() at db_enter+0x18 sys/arch/amd64/amd64/db_interface.c:437 panic(ffffffff826347ad) at panic+0x161 sys/kern/subr_prf.c:198 wakeup_n(fffffd806e3d1c80,ffffffff) at wakeup_n+0x131 sys/kern/kern_synch.c:549 sd_buf_done(fffffd806761a480) at sd_buf_done+0x1f9 sys/scsi/sd.c:772 vioscsi_vq_done(ffff8000000a2228) at vioscsi_vq_done+0xa1 sys/dev/pv/vioscsi.c:349 virtio_check_vqs(ffff80000002f400) at virtio_check_vqs+0x160 sys/dev/pv/virtio.c:228 intr_handler(ffff800029a75480,ffff80000004f980) at intr_handler+0x4b sys/arch/amd64/amd64/intr.c:537 Xintr_ioapic_edge21_untramp() at Xintr_ioapic_edge21_untramp+0x18f Xspllower() at Xspllower+0x19 softintr_dispatch(0) at softintr_dispatch+0xc4 sys/arch/amd64/amd64/softintr.c:103 Xsoftclock() at Xsoftclock+0x1f buf_get(0,0,10000000) at buf_get+0x750 sys/kern/vfs_bio.c:1186 geteblk(10000000) at geteblk+0x2c sys/kern/vfs_bio.c:1061 readdisklabel(2902,ffffffff817d8de0,ffff800000bc2200,0) at readdisklabel+0x145 sys/arch/amd64/amd64/disksubr.c:96 end trace frame: 0xffff800029a75930, count: 0 https://www.openbsd.org/ddb.html describes the minimum info required in bug reports. Insufficient info makes it difficult to find and fix bugs. ddb> ddb> set $lines = 0 ddb> set $maxwidth = 0 ddb> show panic *cpu0: wakeup: p_stat is 0 ddb> trace db_enter() at db_enter+0x18 sys/arch/amd64/amd64/db_interface.c:437 panic(ffffffff826347ad) at panic+0x161 sys/kern/subr_prf.c:198 wakeup_n(fffffd806e3d1c80,ffffffff) at wakeup_n+0x131 sys/kern/kern_synch.c:549 sd_buf_done(fffffd806761a480) at sd_buf_done+0x1f9 sys/scsi/sd.c:772 vioscsi_vq_done(ffff8000000a2228) at vioscsi_vq_done+0xa1 sys/dev/pv/vioscsi.c:349 virtio_check_vqs(ffff80000002f400) at virtio_check_vqs+0x160 sys/dev/pv/virtio.c:228 intr_handler(ffff800029a75480,ffff80000004f980) at intr_handler+0x4b sys/arch/amd64/amd64/intr.c:537 Xintr_ioapic_edge21_untramp() at Xintr_ioapic_edge21_untramp+0x18f Xspllower() at Xspllower+0x19 softintr_dispatch(0) at softintr_dispatch+0xc4 sys/arch/amd64/amd64/softintr.c:103 Xsoftclock() at Xsoftclock+0x1f buf_get(0,0,10000000) at buf_get+0x750 sys/kern/vfs_bio.c:1186 geteblk(10000000) at geteblk+0x2c sys/kern/vfs_bio.c:1061 readdisklabel(2902,ffffffff817d8de0,ffff800000bc2200,0) at readdisklabel+0x145 sys/arch/amd64/amd64/disksubr.c:96 vndopen(2902,1,2000,ffff800027e1dcf0) at vndopen+0x17a sys/dev/vnd.c:203 spec_open(ffff800029a759b8) at spec_open+0x3df sys/kern/spec_vnops.c:150 VOP_OPEN(fffffd806f02cdd8,1,fffffd807f7d7680,ffff800027e1dcf0) at VOP_OPEN+0x6c sys/kern/vfs_vops.c:138 vn_open(ffff800029a75c08,1,0) at vn_open+0x467 sys/kern/vfs_vnops.c:177 doopenat(ffff800027e1dcf0,ffffff9c,20000040,0,0,ffff800029a75df0) at doopenat+0x26a sys/kern/vfs_syscalls.c:1127 syscall(ffff800029a75e60) at syscall+0x447 sys/arch/amd64/amd64/trap.c:585 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0x9e83e1b9e40, count: -21 ddb> show registers rdi 0 rsi 0x1 rbp 0xffff800029a75240 rbx 0 rdx 0 rcx 0 rax 0xffff800027e1dcf0 r8 0x101010101010101 r9 0x8080808080808080 r10 0x9fee00f88777e673 r11 0x4d01e3c773007859 r12 0 r13 0xffff800021732540 r14 0 r15 0x1 rip 0xffffffff810fcaa8 db_enter+0x18 cs 0x8 rflags 0x246 rsp 0xffff800029a75230 ss 0x10 db_enter+0x18: addq $0x8,%rsp ddb> show proc PROC () pid=117103 stat=onproc flags process=0 proc=4000000 pri=32, usrpri=16, nice=0 forw=0xffffffffffffffff, list=0xffff800027e1c7f0,0xffffffff82a15f48 process=0xffff800021713b80 user=0xffff800029a70000, vmspace=0xfffffd807b486450 estcpu=6, cpticks=16, pctcpu=0.0 user=0, sys=15, intr=1 ddb> ps PID TID PPID UID S FLAGS WAIT COMMAND ddb> show all locks No such command ddb> show malloc Type InUse MemUse HighUse Limit Requests Type Lim devbuf 10177 6407K 6993K 78643K 11432 0 pcb 13 12K 17K 78643K 156 0 rtable 223 9K 10K 78643K 384 0 ifaddr 86 17K 18K 78643K 110 0 counters 26 17K 17K 78643K 34 0 ioctlops 0 0K 4K 78643K 185 0 iov 0 0K 16K 78643K 8 0 mount 1 1K 1K 78643K 1 0 log 0 0K 0K 78643K 4 0 vnodes 1275 80K 80K 78643K 1309 0 UFS quota 1 32K 32K 78643K 1 0 UFS mount 5 36K 36K 78643K 5 0 shm 2 1K 1K 78643K 2 0 VM map 2 0K 0K 78643K 2 0 sem 9 0K 0K 78643K 9 0 dirhash 12 2K 2K 78643K 12 0 ACPI 1697 195K 286K 78643K 12548 0 file desc 16 57K 65K 78643K 275 0 proc 56 58K 75K 78643K 451 0 subproc 104 6K 6K 78643K 104 0 NFS srvsock 1 0K 0K 78643K 1 0 NFS daemon 1 16K 16K 78643K 1 0 ip_moptions 0 0K 0K 78643K 3 0 in_multi 92 6K 6K 78643K 122 0 ether_multi 1 0K 0K 78643K 1 0 ISOFS mount 1 32K 32K 78643K 1 0 MSDOSFS mount 1 16K 16K 78643K 1 0 ttys 49 228K 228K 78643K 49 0 exec 0 0K 2K 78643K 632 0 tdb 3 0K 0K 78643K 3 0 pagedep 1 8K 8K 78643K 1 0 inodedep 1 32K 32K 78643K 1 0 newblk 1 0K 0K 78643K 1 0 VM swap 8 62K 64K 78643K 10 0 UVM amap 249 129K 129K 78643K 2936 0 UVM aobj 3 2K 2K 78643K 3 0 memdesc 1 4K 4K 78643K 1 0 crypto data 1 1K 1K 78643K 1 0 NDP 11 0K 2K 78643K 32 0 temp 83 4712K 5296K 78643K 4054 0 kqueue 14 22K 22K 78643K 36 0 SYN cache 2 16K 16K 78643K 2 0 ddb> show all pools Name Size Requests Fail Releases Pgreq Pgrel Npage Hiwat Minpg Maxpg Idle rtpcb 120 41 0 38 1 0 1 1 0 8 0 rtentry 112 118 0 17 4 0 4 4 0 8 0 unpcb 144 81 0 68 1 0 1 1 0 8 0 syncache 296 10 0 10 3 3 0 1 0 8 0 tcpqe 32 528 0 528 2 2 0 2 0 8 0 tcpcb 768 145 0 138 12 10 2 12 0 8 0 arp 88 18 0 2 1 0 1 1 0 8 0 ipq 40 1 0 1 1 1 0 1 0 8 0 ipqe 40 1 0 1 1 1 0 1 0 8 0 inpcb 336 373 0 360 15 13 2 10 0 8 0 nd6 48 27 0 4 1 0 1 1 0 8 0 kcovpl 48 8 0 0 1 0 1 1 0 8 0 ppxss 1160 7 0 7 1 1 0 1 0 8 0 pfrule 1360 4 0 4 1 1 0 1 0 8 0 art_heap8 4096 2 0 1 2 1 1 2 0 8 0 art_heap4 256 515 0 92 30 2 28 30 0 8 0 art_table 32 517 0 93 4 0 4 4 0 8 0 art_node 16 117 0 25 1 0 1 1 0 8 0 semupl 112 2 0 2 1 1 0 1 0 8 0 semapl 112 7 0 0 1 0 1 1 0 8 0 dirhash 1024 17 0 0 3 0 3 3 0 8 0 dirhash: pool(0xffffffff829c04c8:dirhash): free list modified: page 0xffff8000216f8000; item ordinal 0; addr 0xffff8000216f8400 (p 0xfffffd806e918000); offset 0x0=0x0 pool(dirhash): free list modified: page 0xffff8000216f8000; item ordinal 0; addr 0xffff8000216f8400 (p 0xfffffd806e918000); offset 0x0=0x0 dirhash: pool(0xffffffff829c04c8:dirhash): page inconsistency: page 0xffff8000216f8000; item ordinal 1; addr 0x6b3af71e702d172b dino2pl 256 1660 0 229 90 0 90 90 0 8 0 ffsino 240 1660 0 229 85 0 85 85 0 8 0 nchpl 144 2073 0 403 63 0 63 63 0 8 0 rtmask 32 3 0 3 1 1 0 1 0 8 0 uvmvnodes 80 1795 0 0 37 0 37 37 0 8 0 vnodes 216 1795 0 0 100 0 100 100 0 8 0 namei 1024 6887 0 6884 4 3 1 2 0 8 0 namei: pool(0xffffffff82a139e8:namei): free list modified: page 0xffff8000216df000; item ordinal 0; addr 0xffff8000216dfc00 (p 0xfffffd8069608000); offset 0x0=0x0 pool(namei): free list modified: page 0xffff8000216df000; item ordinal 0; addr 0xffff8000216dfc00 (p 0xfffffd8069608000); offset 0x0=0x0 namei: pool(0xffffffff82a139e8:namei): page inconsistency: page 0xffff8000216df000; item ordinal 1; addr 0x11862ad599d3ac62 vcpupl 2048 4 0 0 1 0 1 1 0 8 0 vcpupl: pool(0xffffffff829b2728:vcpupl): free list modified: page 0xffff800024c0c000; item ordinal 0; addr 0xffff800024c0d000 (p 0xfffffd8069608000); offset 0x0=0x0 pool(vcpupl): free list modified: page 0xffff800024c0c000; item ordinal 0; addr 0xffff800024c0d000 (p 0xfffffd8069608000); offset 0x0=0x0 vcpupl: pool(0xffffffff829b2728:vcpupl): page inconsistency: page 0xffff800024c0c000; item ordinal 1; addr 0x96891d3b3a99c842 vmpool 536 4 0 0 1 0 1 1 0 8 0 kstatmem 264 30 0 8 2 0 2 2 0 8 0 scxspl 216 6519 0 6516 11 10 1 8 0 8 0 plimitpl 152 23 0 8 1 0 1 1 0 8 0 sigapl 424 567 0 523 6 0 6 6 0 8 0 futexpl 64 1480 0 1477 1 0 1 1 0 8 0 knotepl 120 8933 0 8718 8 1 7 7 0 8 0 kqueuepl 184 35 0 24 1 0 1 1 0 8 0 pipepl 288 137 0 109 4 2 2 3 0 8 0 fdescpl 432 550 0 523 5 1 4 4 0 8 0 filepl 120 2331 0 2085 13 5 8 11 0 8 0 lockfpl 104 6 0 4 1 0 1 1 0 8 0 lockfspl 48 4 0 2 1 0 1 1 0 8 0 sessionpl 144 23 0 7 1 0 1 1 0 8 0 pgrppl 48 26 0 10 1 0 1 1 0 8 0 ucredpl 104 167 0 156 1 0 1 1 0 8 0 zombiepl 144 524 0 523 2 1 1 1 0 8 0 processpl 1000 567 0 523 7 1 6 6 0 8 0 processpl: pool(0xffffffff82a16148:processpl): page inconsistency: page 0x0; at page head addr 0xffff80002170df90 (p 0xffff80002170c000) procpl 672 864 0 797 8 2 6 7 0 8 0 procpl: pool(0xffffffff82a15fa0:procpl): page inconsistency: page 0x0; at page head addr 0xffff800021705f90 (p 0xffff800021704000) sosppl 168 3 0 3 1 1 0 1 0 8 0 sockpl 456 495 0 466 26 20 6 17 0 8 0 mcl12k 12288 3 0 2 2 1 1 1 0 8 0 mcl9k 9216 3 0 3 1 1 0 1 0 8 0 mcl8k 8192 14 0 14 2 2 0 1 0 8 0 mcl4k 4096 19 0 19 3 2 1 1 0 8 1 mcl2k2 2112 2 0 2 1 1 0 1 0 8 0 mcl2k 2048 70004 0 69915 33 21 12 32 0 8 0 mtagpl 96 50 0 4 3 1 2 2 0 8 0 mbufpl 256 112105 0 111779 24 3 21 21 0 8 0 bufpl 288 3739 0 124 259 0 259 259 0 8 0 anonpl 24 91445 0 75208 106 6 100 101 0 188 0 amapchunkpl 152 7421 0 6749 29 1 28 28 0 158 0 amappl16 200 1156 0 612 29 0 29 29 0 8 0 amappl15 192 2 0 0 1 0 1 1 0 8 0 amappl14 184 15 0 14 1 0 1 1 0 8 0 amappl13 176 103 0 99 1 0 1 1 0 8 0 amappl12 168 31 0 26 1 0 1 1 0 8 0 amappl11 160 126 0 105 1 0 1 1 0 8 0 amappl10 152 58 0 51 1 0 1 1 0 8 0 amappl9 144 553 0 548 1 0 1 1 0 8 0 amappl8 136 562 0 506 2 0 2 2 0 8 0 amappl7 128 111 0 99 1 0 1 1 0 8 0 amappl6 120 214 0 198 2 1 1 2 0 8 0 amappl5 112 238 0 224 1 0 1 1 0 8 0 amappl4 104 873 0 843 2 1 1 2 0 8 0 amappl3 96 1032 0 978 3 1 2 2 0 8 0 amappl2 88 490 0 437 3 1 2 2 0 8 0 amappl1 80 15313 0 14569 22 5 17 21 0 8 0 amappl 88 2473 0 2313 4 0 4 4 0 92 0 dma4096 4096 1 0 1 1 1 0 1 0 8 0 dma1024 1024 1 0 0 1 0 1 1 0 8 0 dma256 256 6 0 6 1 1 0 1 0 8 0 dma128 128 253 0 253 1 1 0 1 0 8 0 dma64 64 6 0 6 1 1 0 1 0 8 0 dma32 32 7 0 7 1 1 0 1 0 8 0 dma16 16 18 0 17 1 0 1 1 0 8 0 aobjpl 72 2 0 0 1 0 1 1 0 8 0 uaddrrnd 24 554 0 523 1 0 1 1 0 8 0 uaddrbest 32 2 0 0 1 0 1 1 0 8 0 uaddr 24 554 0 523 1 0 1 1 0 8 0 vmmpekpl 168 9600 0 9557 3 0 3 3 0 8 0 vmmpepl 168 57385 0 54707 142 18 124 130 0 357 2 vmsppl 272 553 0 523 3 0 3 3 0 8 0 rwobjpl 24 16786 0 13361 21 0 21 21 0 8 0 pdppl 4096 1114 0 1050 137 70 67 67 0 8 3 pvpl 32 292508 0 271477 263 91 172 260 0 265 0 pmappl 216 553 0 523 3 1 2 2 0 8 0 extentpl 40 56 0 38 1 0 1 1 0 8 0 phpool 112 699 0 122 17 0 17 17 0 8 0 ddb> machine ddbcpu 0 No such command ddb> trace db_enter() at db_enter+0x18 sys/arch/amd64/amd64/db_interface.c:437 panic(ffffffff826347ad) at panic+0x161 sys/kern/subr_prf.c:198 wakeup_n(fffffd806e3d1c80,ffffffff) at wakeup_n+0x131 sys/kern/kern_synch.c:549 sd_buf_done(fffffd806761a480) at sd_buf_done+0x1f9 sys/scsi/sd.c:772 vioscsi_vq_done(ffff8000000a2228) at vioscsi_vq_done+0xa1 sys/dev/pv/vioscsi.c:349 virtio_check_vqs(ffff80000002f400) at virtio_check_vqs+0x160 sys/dev/pv/virtio.c:228 intr_handler(ffff800029a75480,ffff80000004f980) at intr_handler+0x4b sys/arch/amd64/amd64/intr.c:537 Xintr_ioapic_edge21_untramp() at Xintr_ioapic_edge21_untramp+0x18f Xspllower() at Xspllower+0x19 softintr_dispatch(0) at softintr_dispatch+0xc4 sys/arch/amd64/amd64/softintr.c:103 Xsoftclock() at Xsoftclock+0x1f buf_get(0,0,10000000) at buf_get+0x750 sys/kern/vfs_bio.c:1186 geteblk(10000000) at geteblk+0x2c sys/kern/vfs_bio.c:1061 readdisklabel(2902,ffffffff817d8de0,ffff800000bc2200,0) at readdisklabel+0x145 sys/arch/amd64/amd64/disksubr.c:96 vndopen(2902,1,2000,ffff800027e1dcf0) at vndopen+0x17a sys/dev/vnd.c:203 spec_open(ffff800029a759b8) at spec_open+0x3df sys/kern/spec_vnops.c:150 VOP_OPEN(fffffd806f02cdd8,1,fffffd807f7d7680,ffff800027e1dcf0) at VOP_OPEN+0x6c sys/kern/vfs_vops.c:138 vn_open(ffff800029a75c08,1,0) at vn_open+0x467 sys/kern/vfs_vnops.c:177 doopenat(ffff800027e1dcf0,ffffff9c,20000040,0,0,ffff800029a75df0) at doopenat+0x26a sys/kern/vfs_syscalls.c:1127 syscall(ffff800029a75e60) at syscall+0x447 sys/arch/amd64/amd64/trap.c:585 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0x9e83e1b9e40, count: -21 ddb> machine ddbcpu 1 No such command ddb> trace db_enter() at db_enter+0x18 sys/arch/amd64/amd64/db_interface.c:437 panic(ffffffff826347ad) at panic+0x161 sys/kern/subr_prf.c:198 wakeup_n(fffffd806e3d1c80,ffffffff) at wakeup_n+0x131 sys/kern/kern_synch.c:549 sd_buf_done(fffffd806761a480) at sd_buf_done+0x1f9 sys/scsi/sd.c:772 vioscsi_vq_done(ffff8000000a2228) at vioscsi_vq_done+0xa1 sys/dev/pv/vioscsi.c:349 virtio_check_vqs(ffff80000002f400) at virtio_check_vqs+0x160 sys/dev/pv/virtio.c:228 intr_handler(ffff800029a75480,ffff80000004f980) at intr_handler+0x4b sys/arch/amd64/amd64/intr.c:537 Xintr_ioapic_edge21_untramp() at Xintr_ioapic_edge21_untramp+0x18f Xspllower() at Xspllower+0x19 softintr_dispatch(0) at softintr_dispatch+0xc4 sys/arch/amd64/amd64/softintr.c:103 Xsoftclock() at Xsoftclock+0x1f buf_get(0,0,10000000) at buf_get+0x750 sys/kern/vfs_bio.c:1186 geteblk(10000000) at geteblk+0x2c sys/kern/vfs_bio.c:1061 readdisklabel(2902,ffffffff817d8de0,ffff800000bc2200,0) at readdisklabel+0x145 sys/arch/amd64/amd64/disksubr.c:96 vndopen(2902,1,2000,ffff800027e1dcf0) at vndopen+0x17a sys/dev/vnd.c:203 spec_open(ffff800029a759b8) at spec_open+0x3df sys/kern/spec_vnops.c:150 VOP_OPEN(fffffd806f02cdd8,1,fffffd807f7d7680,ffff800027e1dcf0) at VOP_OPEN+0x6c sys/kern/vfs_vops.c:138 vn_open(ffff800029a75c08,1,0) at vn_open+0x467 sys/kern/vfs_vnops.c:177 doopenat(ffff800027e1dcf0,ffffff9c,20000040,0,0,ffff800029a75df0) at doopenat+0x26a sys/kern/vfs_syscalls.c:1127 syscall(ffff800029a75e60) at syscall+0x447 sys/arch/amd64/amd64/trap.c:585 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0x9e83e1b9e40, count: -21