panic: acquiring blockable sleep lock with spinlock or critical section held (rwlock) vmmaplk Stopped at db_enter+0x18: addq $0x8,%rsp TID PID UID PRFLAGS PFLAGS CPU COMMAND * 60436 3135 0 0x14000 0x200 1K reaper db_enter() at db_enter+0x18 sys/arch/amd64/amd64/db_interface.c:398 panic(ffffffff8220b555) at panic+0x15c sys/kern/subr_prf.c:207 witness_checkorder(ffffffff8263e458,1,0) at witness_checkorder+0x10e0 sys/kern/subr_witness.c:821 rw_enter_read(ffffffff8263e448) at rw_enter_read+0x66 sys/kern/kern_rwlock.c:111 uvmfault_lookup(ffff800020e5f978,0) at uvmfault_lookup+0xd9 sys/uvm/uvm_fault.c:1448 uvm_fault(ffffffff8263e440,6b19aa5a3000,0,1) at uvm_fault+0x7f sys/uvm/uvm_fault.c:524 pageflttrap(ffff800020e5fb10,0) at pageflttrap+0x1b8 sys/arch/amd64/amd64/trap.c:221 kerntrap(ffff800020e5fb10) at kerntrap+0xec sys/arch/amd64/amd64/trap.c:302 alltraps_kern_meltdown() at alltraps_kern_meltdown+0x7b pool_do_put(ffffffff826553e0,ffff800020efe000) at pool_do_put+0x12e sys/kern/subr_pool.c:836 pool_put(ffffffff826553e0,ffff800020efe000) at pool_put+0x6f sys/kern/subr_pool.c:794 pmap_destroy(fffffd807efff9f8) at pmap_destroy+0x248 sys/arch/amd64/amd64/pmap.c:1399 uvm_map_teardown(fffffd806e9488a8) at uvm_map_teardown+0x269 sys/uvm/uvm_map.c:2760 uvmspace_free(fffffd806e9488a8) at uvmspace_free+0x86 sys/uvm/uvm_map.c:3646 end trace frame: 0xffff800020e5fdd0, count: 0 https://www.openbsd.org/ddb.html describes the minimum info required in bug reports. Insufficient info makes it difficult to find and fix bugs. ddb{1}> ddb{1}> set $lines = 0 ddb{1}> set $maxwidth = 0 ddb{1}> show panic acquiring blockable sleep lock with spinlock or critical section held (rwlock) vmmaplk ddb{1}> trace db_enter() at db_enter+0x18 sys/arch/amd64/amd64/db_interface.c:398 panic(ffffffff8220b555) at panic+0x15c sys/kern/subr_prf.c:207 witness_checkorder(ffffffff8263e458,1,0) at witness_checkorder+0x10e0 sys/kern/subr_witness.c:821 rw_enter_read(ffffffff8263e448) at rw_enter_read+0x66 sys/kern/kern_rwlock.c:111 uvmfault_lookup(ffff800020e5f978,0) at uvmfault_lookup+0xd9 sys/uvm/uvm_fault.c:1448 uvm_fault(ffffffff8263e440,6b19aa5a3000,0,1) at uvm_fault+0x7f sys/uvm/uvm_fault.c:524 pageflttrap(ffff800020e5fb10,0) at pageflttrap+0x1b8 sys/arch/amd64/amd64/trap.c:221 kerntrap(ffff800020e5fb10) at kerntrap+0xec sys/arch/amd64/amd64/trap.c:302 alltraps_kern_meltdown() at alltraps_kern_meltdown+0x7b pool_do_put(ffffffff826553e0,ffff800020efe000) at pool_do_put+0x12e sys/kern/subr_pool.c:836 pool_put(ffffffff826553e0,ffff800020efe000) at pool_put+0x6f sys/kern/subr_pool.c:794 pmap_destroy(fffffd807efff9f8) at pmap_destroy+0x248 sys/arch/amd64/amd64/pmap.c:1399 uvm_map_teardown(fffffd806e9488a8) at uvm_map_teardown+0x269 sys/uvm/uvm_map.c:2760 uvmspace_free(fffffd806e9488a8) at uvmspace_free+0x86 sys/uvm/uvm_map.c:3646 uvm_exit(ffff800020ec87c8) at uvm_exit+0x29 sys/uvm/uvm_glue.c:297 reaper(ffff800020e19110) at reaper+0x189 sys/kern/kern_exit.c:456 end trace frame: 0x0, count: -16 ddb{1}> show registers rdi 0 rsi 0x1 rbp 0xffff800020e5f5f0 rbx 0xffff800020e5f6a0 rdx 0xffff800020e19110 rcx 0 rax 0 r8 0xffffffff8199412f kprintf+0x16f r9 0x1 r10 0x25 r11 0xe26ec0448a5e1f3a r12 0x3000000008 r13 0xffff800020e5f600 r14 0x100 r15 0x1 rip 0xffffffff810c3ce8 db_enter+0x18 cs 0x8 rflags 0x246 rsp 0xffff800020e5f5e0 ss 0x10 db_enter+0x18: addq $0x8,%rsp ddb{1}> show proc PROC (reaper) pid=60436 stat=onproc flags process=14000 proc=200 pri=4, usrpri=53, nice=20 forw=0xffffffffffffffff, list=0xffff800020e18ea0,0xffff800020e19600 process=0xffff800020e4c7b8 user=0xffff800020e5a000, vmspace=0xffffffff8263e440 estcpu=3, cpticks=11, pctcpu=0.2 user=0, sys=12, intr=0 ddb{1}> ps PID TID PPID UID S FLAGS WAIT COMMAND 43984 479628 0 0 3 0x14200 bored sosplice 15572 361989 87737 0 3 0x10008a pause ksh 87737 126117 87818 0 3 0x92 select sshd 57558 234646 1 0 3 0x100083 ttyin getty 87818 332619 1 0 3 0x80 select sshd 90891 377711 69305 74 3 0x100092 bpf pflogd 69305 73377 1 0 3 0x80 netio pflogd 73098 311520 36230 73 3 0x100090 kqread syslogd 36230 167117 1 0 3 0x100082 netio syslogd 25268 219509 1 77 3 0x100090 poll dhclient 67692 315408 1 0 3 0x80 poll dhclient 36021 387889 0 0 2 0x40014200 smr 61237 247074 0 0 3 0x14200 pgzero zerothread 57758 472440 0 0 3 0x14200 aiodoned aiodoned 28232 96609 0 0 3 0x14200 syncer update 34268 512122 0 0 3 0x14200 cleaner cleaner * 3135 60436 0 0 7 0x14200 reaper 57018 269434 0 0 3 0x14200 pgdaemon pagedaemon 48528 240327 0 0 3 0x14200 bored crynlk 86793 359106 0 0 3 0x14200 bored crypto 58001 103997 0 0 3 0x40014200 acpi0 acpi0 10240 47925 0 0 3 0x40014200 idle1 71977 157013 0 0 3 0x14200 bored softnet 11814 311247 0 0 3 0x14200 bored systqmp 50838 20370 0 0 3 0x14200 bored systq 56963 372736 0 0 3 0x40014200 bored softclock 59373 305523 0 0 7 0x40014200 idle0 1 444187 0 0 3 0x82 wait init 0 0 -1 0 3 0x10200 scheduler swapper ddb{1}> show all locks CPU 1: exclusive mutex pdppl r = 0 (0xffffffff826553f0) #0 witness_lock+0x4c7 stacktrace_save sys/sys/stacktrace.h:36 [inline] #0 witness_lock+0x4c7 sys/kern/subr_witness.c:1164 #1 mtx_enter_try+0x102 #2 mtx_enter+0x4b sys/kern/kern_lock.c:266 #3 pool_put+0x64 sys/kern/subr_pool.c:792 #4 pmap_destroy+0x248 sys/arch/amd64/amd64/pmap.c:1399 #5 uvm_map_teardown+0x269 sys/uvm/uvm_map.c:2760 #6 uvmspace_free+0x86 sys/uvm/uvm_map.c:3646 #7 uvm_exit+0x29 sys/uvm/uvm_glue.c:297 #8 reaper+0x189 sys/kern/kern_exit.c:456 #9 proc_trampoline+0x1c Process 3135 (reaper) thread 0xffff800020e19110 (60436) exclusive kernel_lock &kernel_lock r = 1 (0xffffffff8265c968) #0 witness_lock+0x4c7 stacktrace_save sys/sys/stacktrace.h:36 [inline] #0 witness_lock+0x4c7 sys/kern/subr_witness.c:1164 #1 uvm_map_teardown+0x261 sys/uvm/uvm_map.c:2759 #2 uvmspace_free+0x86 sys/uvm/uvm_map.c:3646 #3 uvm_exit+0x29 sys/uvm/uvm_glue.c:297 #4 reaper+0x189 sys/kern/kern_exit.c:456 #5 proc_trampoline+0x1c exclusive mutex pdppl r = 0 (0xffffffff826553f0) #0 witness_lock+0x4c7 stacktrace_save sys/sys/stacktrace.h:36 [inline] #0 witness_lock+0x4c7 sys/kern/subr_witness.c:1164 #1 mtx_enter_try+0x102 #2 mtx_enter+0x4b sys/kern/kern_lock.c:266 #3 pool_put+0x64 sys/kern/subr_pool.c:792 #4 pmap_destroy+0x248 sys/arch/amd64/amd64/pmap.c:1399 #5 uvm_map_teardown+0x269 sys/uvm/uvm_map.c:2760 #6 uvmspace_free+0x86 sys/uvm/uvm_map.c:3646 #7 uvm_exit+0x29 sys/uvm/uvm_glue.c:297 #8 reaper+0x189 sys/kern/kern_exit.c:456 #9 proc_trampoline+0x1c ddb{1}> show malloc Type InUse MemUse HighUse Limit Requests Type Lim devbuf 9501 6417K 6924K 78643K 11271 0 pcb 13 8K 8K 78643K 82 0 rtable 87 3K 4K 78643K 321 0 ifaddr 58 11K 12K 78643K 96 0 counters 43 33K 34K 78643K 53 0 ioctlops 0 0K 4K 78643K 1476 0 iov 0 0K 12K 78643K 28 0 mount 1 1K 1K 78643K 1 0 vnodes 1239 78K 78K 78643K 1621 0 UFS quota 1 32K 32K 78643K 1 0 UFS mount 5 36K 36K 78643K 5 0 shm 2 1K 5K 78643K 3 0 VM map 2 1K 1K 78643K 2 0 sem 12 0K 1K 78643K 33 0 dirhash 12 2K 2K 78643K 12 0 ACPI 1809 196K 290K 78643K 12766 0 file desc 3 8K 25K 78643K 286 0 sigio 0 0K 0K 78643K 7 0 proc 59 63K 95K 78643K 598 0 subproc 22 1K 3K 78643K 238 0 NFS srvsock 1 0K 0K 78643K 1 0 NFS daemon 1 16K 16K 78643K 1 0 ip_moptions 0 0K 0K 78643K 23 0 in_multi 37 2K 2K 78643K 93 0 ether_multi 1 0K 0K 78643K 17 0 ISOFS mount 1 32K 32K 78643K 1 0 MSDOSFS mount 1 16K 16K 78643K 1 0 ttys 43 201K 201K 78643K 43 0 exec 0 0K 1K 78643K 266 0 pagedep 1 8K 8K 78643K 1 0 inodedep 1 32K 32K 78643K 1 0 newblk 1 0K 0K 78643K 1 0 VM swap 7 26K 26K 78643K 7 0 UVM amap 71 35K 54K 78643K 1829 0 UVM aobj 4 2K 2K 78643K 6 0 memdesc 1 4K 4K 78643K 1 0 crypto data 1 1K 1K 78643K 1 0 ip6_options 0 0K 0K 78643K 58 0 NDP 10 0K 0K 78643K 21 0 temp 94 3038K 3102K 78643K 3467 0 kqueue 2 2K 8K 78643K 13 0 SYN cache 2 16K 16K 78643K 2 0 ddb{1}> show all pools Name Size Requests Fail Releases Pgreq Pgrel Npage Hiwat Minpg Maxpg Idle arp 64 8 0 5 1 0 1 1 0 8 0 plcache 128 20 0 0 1 0 1 1 0 8 0 rtpcb 80 37 0 35 1 0 1 1 0 8 0 rtentry 112 60 0 28 2 1 1 2 0 8 0 unpcb 120 148 0 138 1 0 1 1 0 8 0 syncache 264 8 0 8 2 2 0 1 0 8 0 tcpqe 32 223 0 223 3 2 1 1 0 8 1 tcpcb 544 154 0 151 2 1 1 2 0 8 0 inpcb 280 385 0 379 2 1 1 2 0 8 0 nd6 48 9 0 6 1 0 1 1 0 8 0 ppxss 1128 2 0 2 1 1 0 1 0 8 0 pffrag 232 1 0 1 1 1 0 1 0 482 0 pffrnode 88 1 0 1 1 1 0 1 0 8 0 pffrent 40 45 0 45 1 1 0 1 0 8 0 pfosfp 40 846 0 423 5 0 5 5 0 8 0 pfosfpen 112 1428 0 714 21 0 21 21 0 8 0 pfstitem 24 34 0 29 1 0 1 1 0 8 0 pfstkey 112 34 0 29 1 0 1 1 0 8 0 pfstate 328 34 0 29 3 0 3 3 0 8 0 pfrule 1360 21 0 16 2 1 1 2 0 8 0 art_heap8 4096 1 0 0 1 0 1 1 0 8 0 art_heap4 256 248 0 100 12 2 10 12 0 8 0 art_table 32 249 0 100 2 0 2 2 0 8 0 art_node 16 59 0 30 1 0 1 1 0 8 0 sysvmsgpl 40 4 0 4 1 1 0 1 0 8 0 semupl 112 1 0 1 1 1 0 1 0 8 0 semapl 112 27 0 17 1 0 1 1 0 8 0 shmpl 112 4 0 2 1 0 1 1 0 8 0 dirhash 1024 17 0 0 3 0 3 3 0 8 0 dino2pl 256 1686 0 284 89 0 89 89 0 8 0 ffsino 272 1686 0 284 95 0 95 95 0 8 0 nchpl 144 2303 0 708 61 0 61 61 0 8 0 uvmvnodes 72 2095 0 0 39 0 39 39 0 8 0 vnodes 208 2095 0 0 111 0 111 111 0 8 0 namei 1024 6536 0 6536 2 1 1 1 0 8 1 percpumem 16 37 0 5 1 0 1 1 0 8 0 vcpupl 1984 2 0 0 1 0 1 1 0 8 0 vmpool 560 2 0 0 1 0 1 1 0 8 0 scxspl 192 6458 0 6458 12 11 1 7 0 8 1 plimitpl 152 45 0 37 1 0 1 1 0 8 0 sigapl 424 466 0 436 4 0 4 4 0 8 0 futexpl 56 3134 0 3134 2 2 0 1 0 8 0 knotepl 112 208 0 203 1 0 1 1 0 8 0 kqueuepl 144 28 0 27 1 0 1 1 0 8 0 pipelkpl 48 142 0 136 1 0 1 1 0 8 0 pipepl 120 284 0 275 1 0 1 1 0 8 0 fdescpl 496 450 0 436 3 0 3 3 0 8 0 filepl 152 2874 0 2796 5 0 5 5 0 8 0 lockfpl 104 43 0 42 1 0 1 1 0 8 0 lockfspl 48 19 0 18 1 0 1 1 0 8 0 sessionpl 112 30 0 19 1 0 1 1 0 8 0 pgrppl 48 36 0 25 1 0 1 1 0 8 0 ucredpl 96 177 0 168 1 0 1 1 0 8 0 zombiepl 144 438 0 435 1 0 1 1 0 8 0 processpl 984 466 0 435 5 0 5 5 0 8 0 procpl 624 883 0 852 4 0 4 4 0 8 0 srpgc 64 2 0 2 1 1 0 1 0 8 0 sosppl 128 5 0 5 1 1 0 1 0 8 0 sockpl 400 570 0 552 6 3 3 5 0 8 1 mcl64k 65536 7 0 0 1 0 1 1 0 8 0 mcl16k 16384 1 0 0 1 0 1 1 0 8 0 mcl12k 12288 3 0 0 1 0 1 1 0 8 0 mcl9k 9216 2 0 0 1 0 1 1 0 8 0 mcl8k 8192 3 0 0 1 0 1 1 0 8 0 mcl4k 4096 6 0 0 1 0 1 1 0 8 0 mcl2k 2048 190 0 0 23 0 23 23 0 8 0 mtagpl 80 43 0 0 1 0 1 1 0 8 0 mbufpl 256 488 0 0 30 0 30 30 0 8 0 bufpl 280 4409 0 191 302 0 302 302 0 8 0 anonpl 16 57810 0 56469 66 1 65 65 0 124 51 amapchunkpl 152 3182 0 3137 20 13 7 20 0 158 3 amappl16 192 1834 0 1818 44 0 44 44 0 8 35 amappl15 184 115 0 113 2 1 1 1 0 8 0 amappl14 176 26 0 23 1 0 1 1 0 8 0 amappl13 168 47 0 46 1 0 1 1 0 8 0 amappl12 160 142 0 138 1 0 1 1 0 8 0 amappl11 152 57 0 42 1 0 1 1 0 8 0 amappl10 144 21 0 19 1 0 1 1 0 8 0 amappl9 136 391 0 390 1 0 1 1 0 8 0 amappl8 128 442 0 440 2 0 2 2 0 8 1 amappl7 120 153 0 145 1 0 1 1 0 8 0 amappl6 112 33 0 33 2 1 1 1 0 8 1 amappl5 104 332 0 315 1 0 1 1 0 8 0 amappl4 96 542 0 517 1 0 1 1 0 8 0 amappl3 88 117 0 112 1 0 1 1 0 8 0 amappl2 80 2464 0 2397 3 1 2 3 0 8 0 amappl1 72 20016 0 19587 26 15 11 20 0 8 0 amappl 80 1225 0 1197 2 0 2 2 0 84 0 dma4096 4096 1 0 1 1 1 0 1 0 8 0 dma256 256 6 0 6 1 1 0 1 0 8 0 dma128 128 253 0 253 1 1 0 1 0 8 0 dma64 64 6 0 6 1 1 0 1 0 8 0 dma32 32 7 0 7 1 1 0 1 0 8 0 dma16 16 18 0 17 1 0 1 1 0 8 0 aobjpl 64 5 0 2 1 0 1 1 0 8 0 uaddrrnd 24 452 0 436 1 0 1 1 0 8 0 uaddrbest 32 2 0 0 1 0 1 1 0 8 0 uaddr 24 452 0 436 1 0 1 1 0 8 0 vmmpekpl 168 7834 0 7801 2 0 2 2 0 8 0 vmmpepl 168 71394 0 70409 162 36 126 126 0 357 70 vmsppl 368 451 0 435 2 0 2 2 0 8 0 pdppl 4096 912 0 872 6 0 6 6 0 8 0 pdppl: pool(0xffffffff826553e0:pdppl): page inconsistency: page 0xffff800020efa000; item ordinal 0; addr 0x6b19aa5a3888 pvpl 32 195136 0 191946 158 1 157 158 0 265 92 pmappl 232 451 0 435 2 0 2 2 0 8 0 extentpl 40 46 0 29 1 0 1 1 0 8 0 phpool 112 283 0 10 8 0 8 8 0 8 0