panic: acquiring blockable sleep lock with spinlock or critical section held (kernel_lock) &kernel_lock Stopped at db_enter+0x18: addq $0x8,%rsp TID PID UID PRFLAGS PFLAGS CPU COMMAND *225384 50058 0 0x14000 0x200 1 systqmp db_enter() at db_enter+0x18 sys/arch/amd64/amd64/db_interface.c:398 panic(ffffffff823fea24) at panic+0x15c sys/kern/subr_prf.c:207 witness_checkorder(ffffffff828b9310,9,0) at witness_checkorder+0x10e0 sys/kern/subr_witness.c:821 __mp_lock(ffffffff828b9108) at __mp_lock+0xa1 read_rflags machine/cpufunc.h:195 [inline] __mp_lock(ffffffff828b9108) at __mp_lock+0xa1 intr_disable machine/cpufunc.h:216 [inline] __mp_lock(ffffffff828b9108) at __mp_lock+0xa1 sys/kern/kern_lock.c:142 pageflttrap(ffff800020da35a0,0) at pageflttrap+0x7f sys/arch/amd64/amd64/trap.c:180 kerntrap(ffff800020da35a0) at kerntrap+0xec sys/arch/amd64/amd64/trap.c:302 alltraps_kern_meltdown() at alltraps_kern_meltdown+0x7b pool_cache_gc(ffffffff828ab988) at pool_cache_gc+0x138 pool_cache_list_put sys/kern/subr_pool.c:1970 [inline] pool_cache_gc(ffffffff828ab988) at pool_cache_gc+0x138 sys/kern/subr_pool.c:2025 pool_gc_pages(0) at pool_gc_pages+0x6c sys/kern/subr_pool.c:1559 taskq_thread(ffffffff8277c5f0) at taskq_thread+0xec sys/kern/kern_task.c:437 end trace frame: 0x0, count: 5 https://www.openbsd.org/ddb.html describes the minimum info required in bug reports. Insufficient info makes it difficult to find and fix bugs. ddb{1}> ddb{1}> set $lines = 0 ddb{1}> set $maxwidth = 0 ddb{1}> show panic acquiring blockable sleep lock with spinlock or critical section held (kernel_lock) &kernel_lock ddb{1}> trace db_enter() at db_enter+0x18 sys/arch/amd64/amd64/db_interface.c:398 panic(ffffffff823fea24) at panic+0x15c sys/kern/subr_prf.c:207 witness_checkorder(ffffffff828b9310,9,0) at witness_checkorder+0x10e0 sys/kern/subr_witness.c:821 __mp_lock(ffffffff828b9108) at __mp_lock+0xa1 read_rflags machine/cpufunc.h:195 [inline] __mp_lock(ffffffff828b9108) at __mp_lock+0xa1 intr_disable machine/cpufunc.h:216 [inline] __mp_lock(ffffffff828b9108) at __mp_lock+0xa1 sys/kern/kern_lock.c:142 pageflttrap(ffff800020da35a0,0) at pageflttrap+0x7f sys/arch/amd64/amd64/trap.c:180 kerntrap(ffff800020da35a0) at kerntrap+0xec sys/arch/amd64/amd64/trap.c:302 alltraps_kern_meltdown() at alltraps_kern_meltdown+0x7b pool_cache_gc(ffffffff828ab988) at pool_cache_gc+0x138 pool_cache_list_put sys/kern/subr_pool.c:1970 [inline] pool_cache_gc(ffffffff828ab988) at pool_cache_gc+0x138 sys/kern/subr_pool.c:2025 pool_gc_pages(0) at pool_gc_pages+0x6c sys/kern/subr_pool.c:1559 taskq_thread(ffffffff8277c5f0) at taskq_thread+0xec sys/kern/kern_task.c:437 end trace frame: 0x0, count: -10 ddb{1}> show registers rdi 0 rsi 0x1 rbp 0xffff800020da3310 rbx 0xffff800020da33c0 rdx 0x8b rcx 0x2 rax 0x1 r8 0xffffffff813837bf kprintf+0x16f r9 0x1 r10 0x2 r11 0x333159c36fbc35ca r12 0x3000000008 r13 0xffff800020da3320 r14 0x100 r15 0x1 rip 0xffffffff8196be18 db_enter+0x18 cs 0x8 rflags 0x246 rsp 0xffff800020da3300 ss 0x10 db_enter+0x18: addq $0x8,%rsp ddb{1}> show proc PROC (systqmp) pid=225384 stat=onproc flags process=14000 proc=200 pri=32, usrpri=51, nice=20 forw=0xffffffffffffffff, list=0xffff800020d89110,0xffff800020d889d0 process=0xffff800020d8a7b0 user=0xffff800020d9e000, vmspace=0xffffffff828b0500 estcpu=1, cpticks=1, pctcpu=0.0 user=0, sys=1, intr=0 ddb{1}> ps PID TID PPID UID S FLAGS WAIT COMMAND 50592 292699 0 0 3 0x14200 bored sosplice 52449 372922 1 0 3 0x100083 ttyin getty 74434 305089 1 0 3 0x80 select sshd 93180 189019 81211 74 3 0x100092 bpf pflogd 81211 259358 1 0 3 0x80 netio pflogd 55780 518319 55669 73 3 0x100090 kqread syslogd 55669 331888 1 0 3 0x100082 netio syslogd 45584 520740 1 77 3 0x100090 poll dhclient 85076 436870 1 0 3 0x80 poll dhclient 43169 180903 0 0 3 0x14200 bored smr 7013 91596 0 0 3 0x14200 pgzero zerothread 65166 321749 0 0 3 0x14200 aiodoned aiodoned 83199 353561 0 0 3 0x14200 syncer update 93104 247447 0 0 3 0x14200 cleaner cleaner 88403 175530 0 0 3 0x14200 reaper reaper 18035 172618 0 0 3 0x14200 pgdaemon pagedaemon 3342 469986 0 0 3 0x14200 bored crynlk 20386 164775 0 0 3 0x14200 bored crypto 11176 420452 0 0 3 0x40014200 acpi0 acpi0 92909 346706 0 0 3 0x40014200 idle1 76585 520830 0 0 3 0x14200 bored softnet *50058 225384 0 0 7 0x14200 systqmp 19790 252238 0 0 3 0x14200 bored systq 46572 495552 0 0 3 0x40014200 bored softclock 8896 152916 0 0 7 0x40014200 idle0 1 113541 0 0 3 0x82 wait init 0 0 -1 0 3 0x10200 scheduler swapper ddb{1}> show all locks CPU 1: exclusive mutex mbufpl r = 0 (0xffffffff828ab998) #0 witness_lock+0x4c7 stacktrace_save sys/sys/stacktrace.h:36 [inline] #0 witness_lock+0x4c7 sys/kern/subr_witness.c:1164 #1 mtx_enter_try+0x102 #2 mtx_enter+0x4b sys/kern/kern_lock.c:266 #3 pool_cache_gc+0x12d pl_enter sys/kern/subr_pool.c:104 [inline] #3 pool_cache_gc+0x12d pool_cache_list_put sys/kern/subr_pool.c:1968 [inline] #3 pool_cache_gc+0x12d sys/kern/subr_pool.c:2025 #4 pool_gc_pages+0x6c sys/kern/subr_pool.c:1559 #5 taskq_thread+0xec sys/kern/kern_task.c:437 #6 proc_trampoline+0x1c Process 50058 (systqmp) thread 0xffff800020d88c30 (225384) shared rwlock pools r = 0 (0xffffffff827575d0) #0 witness_lock+0x4c7 stacktrace_save sys/sys/stacktrace.h:36 [inline] #0 witness_lock+0x4c7 sys/kern/subr_witness.c:1164 #1 pool_gc_pages+0x21 sys/kern/subr_pool.c:1555 #2 taskq_thread+0xec sys/kern/kern_task.c:437 #3 proc_trampoline+0x1c shared rwlock systqmp r = 0 (0xffffffff8277c660) #0 witness_lock+0x4c7 stacktrace_save sys/sys/stacktrace.h:36 [inline] #0 witness_lock+0x4c7 sys/kern/subr_witness.c:1164 #1 taskq_thread+0xdf sys/kern/kern_task.c:436 #2 proc_trampoline+0x1c exclusive mutex mbufpl r = 0 (0xffffffff828ab998) #0 witness_lock+0x4c7 stacktrace_save sys/sys/stacktrace.h:36 [inline] #0 witness_lock+0x4c7 sys/kern/subr_witness.c:1164 #1 mtx_enter_try+0x102 #2 mtx_enter+0x4b sys/kern/kern_lock.c:266 #3 pool_cache_gc+0x12d pl_enter sys/kern/subr_pool.c:104 [inline] #3 pool_cache_gc+0x12d pool_cache_list_put sys/kern/subr_pool.c:1968 [inline] #3 pool_cache_gc+0x12d sys/kern/subr_pool.c:2025 #4 pool_gc_pages+0x6c sys/kern/subr_pool.c:1559 #5 taskq_thread+0xec sys/kern/kern_task.c:437 #6 proc_trampoline+0x1c ddb{1}> show malloc Type InUse MemUse HighUse Limit Requests Type Lim devbuf 9500 6416K 6984K 78643K 11485 0 pcb 13 8K 8K 78643K 65 0 rtable 64 2K 5K 78643K 294 0 ifaddr 46 10K 13K 78643K 101 0 sysctl 2 0K 0K 78643K 2 0 counters 39 33K 34K 78643K 47 0 ioctlops 0 0K 4K 78643K 1517 0 iov 0 0K 16K 78643K 47 0 mount 1 1K 1K 78643K 1 0 vnodes 1221 77K 77K 78643K 1524 0 UFS quota 1 32K 32K 78643K 1 0 UFS mount 5 36K 36K 78643K 5 0 shm 2 1K 5K 78643K 5 0 VM map 2 1K 1K 78643K 2 0 sem 11 1K 2K 78643K 28 0 dirhash 12 2K 2K 78643K 12 0 ACPI 1824 197K 290K 78643K 13058 0 file desc 1 0K 25K 78643K 600 0 sigio 0 0K 0K 78643K 7 0 proc 63 63K 95K 78643K 498 0 subproc 0 0K 2K 78643K 34 0 NFS srvsock 1 0K 0K 78643K 1 0 NFS daemon 1 16K 16K 78643K 1 0 ip_moptions 0 0K 0K 78643K 57 0 in_multi 32 1K 2K 78643K 74 0 ether_multi 1 0K 0K 78643K 5 0 mrt 0 0K 0K 78643K 2 0 ISOFS mount 1 32K 32K 78643K 1 0 MSDOSFS mount 1 16K 16K 78643K 1 0 ttys 49 228K 228K 78643K 49 0 exec 0 0K 1K 78643K 235 0 pagedep 1 8K 8K 78643K 1 0 inodedep 1 32K 32K 78643K 1 0 newblk 1 0K 0K 78643K 1 0 VM swap 7 26K 26K 78643K 7 0 UVM amap 51 67K 88K 78643K 2614 0 UVM aobj 8 4K 4K 78643K 8 0 memdesc 1 4K 4K 78643K 1 0 crypto data 1 1K 1K 78643K 1 0 ip6_options 0 0K 0K 78643K 69 0 NDP 8 0K 0K 78643K 17 0 temp 62 3850K 3917K 78643K 3980 0 kqueue 2 2K 8K 78643K 28 0 SYN cache 2 16K 16K 78643K 2 0 ddb{1}> show all pools Name Size Requests Fail Releases Pgreq Pgrel Npage Hiwat Minpg Maxpg Idle arp 64 7 0 5 1 0 1 1 0 8 0 plcache 128 20 0 0 1 0 1 1 0 8 0 rtpcb 80 58 0 56 1 0 1 1 0 8 0 rtentry 112 51 0 29 2 1 1 2 0 8 0 unpcb 120 205 0 194 1 0 1 1 0 8 0 syncache 264 14 0 14 4 4 0 1 0 8 0 tcpqe 32 489 0 489 2 2 0 1 0 8 0 tcpcb 544 174 0 172 1 0 1 1 0 8 0 inpcb 296 477 0 472 3 2 1 2 0 8 0 rttmr 72 1 0 1 1 1 0 1 0 8 0 nd6 48 7 0 7 1 1 0 1 0 8 0 pkpcb 40 11 0 11 2 2 0 1 0 8 0 ppxss 1128 1 0 1 1 1 0 1 0 8 0 pfstscr 40 4 0 4 1 1 0 1 0 8 0 pffrag 232 3 0 3 3 3 0 1 0 482 0 pffrnode 88 3 0 3 3 3 0 1 0 8 0 pffrent 40 7 0 7 3 3 0 1 0 8 0 pfosfp 40 846 0 423 5 0 5 5 0 8 0 pfosfpen 112 1428 0 714 21 0 21 21 0 8 0 pfrktable 1344 14 0 14 2 2 0 1 0 8 0 pfstitem 24 34 0 34 1 1 0 1 0 8 0 pfstkey 112 36 0 36 1 1 0 1 0 8 0 pfstate 328 34 0 34 3 3 0 3 0 8 0 pfrule 1360 27 0 20 2 1 1 2 0 8 0 art_heap8 4096 1 0 0 1 0 1 1 0 8 0 art_heap4 256 212 0 116 13 7 6 13 0 8 0 art_table 32 213 0 116 2 1 1 2 0 8 0 art_node 16 50 0 30 1 0 1 1 0 8 0 sysvmsgpl 40 26 0 7 1 0 1 1 0 8 0 semupl 112 6 0 6 1 1 0 1 0 8 0 semapl 112 9 0 0 1 0 1 1 0 8 0 shmpl 112 6 0 0 1 0 1 1 0 8 0 dirhash 1024 17 0 0 3 0 3 3 0 8 0 dino2pl 256 2209 0 808 89 1 88 89 0 8 0 ffsino 272 2209 0 808 95 0 95 95 0 8 0 nchpl 144 3143 0 1546 60 0 60 60 0 8 0 uvmvnodes 72 2477 0 0 46 0 46 46 0 8 0 vnodes 208 2477 0 0 131 0 131 131 0 8 0 namei 1024 8372 0 8372 1 1 0 1 0 8 0 percpumem 16 34 0 4 1 0 1 1 0 8 0 vcpupl 1984 5 0 0 1 0 1 1 0 8 0 vmpool 560 5 0 0 1 0 1 1 0 8 0 pfiaddrpl 120 2 0 2 1 1 0 1 0 8 0 scsiplug 64 1 0 1 1 1 0 1 0 8 0 scxspl 192 10169 0 10169 14 13 1 7 0 8 1 plimitpl 152 58 0 53 1 0 1 1 0 8 0 sigapl 424 816 0 790 4 0 4 4 0 8 0 futexpl 56 4773 0 4773 1 1 0 1 0 8 0 knotepl 112 90 0 85 1 0 1 1 0 8 0 kqueuepl 144 55 0 51 1 0 1 1 0 8 0 pipelkpl 48 97 0 97 1 1 0 1 0 8 0 pipepl 120 194 0 194 1 1 0 1 0 8 0 fdescpl 496 800 0 790 3 0 3 3 0 8 0 filepl 152 3469 0 3431 6 4 2 5 0 8 0 lockfpl 104 121 0 120 1 0 1 1 0 8 0 lockfspl 48 49 0 48 1 0 1 1 0 8 0 sessionpl 112 18 0 11 1 0 1 1 0 8 0 pgrppl 48 19 0 12 1 0 1 1 0 8 0 ucredpl 96 467 0 459 1 0 1 1 0 8 0 zombiepl 144 790 0 790 1 1 0 1 0 8 0 processpl 984 816 0 790 5 0 5 5 0 8 0 procpl 624 1797 0 1771 4 0 4 4 0 8 0 sosppl 128 7 0 7 2 2 0 1 0 8 0 sockpl 400 753 0 735 5 2 3 4 0 8 0 mcl64k 65536 3 0 0 1 0 1 1 0 8 0 mcl12k 12288 1 0 0 1 0 1 1 0 8 0 mcl9k 9216 4 0 0 1 0 1 1 0 8 0 mcl8k 8192 3 0 0 1 0 1 1 0 8 0 mcl4k 4096 6 0 0 1 0 1 1 0 8 0 mcl2k 2048 197 0 0 24 1 23 24 0 8 0 mtagpl 96 47 0 0 2 0 2 2 0 8 0 mbufpl 256 337 0 0 21 0 21 21 0 8 0 bufpl 280 4872 0 138 339 0 339 339 0 8 0 anonpl 16 94914 0 93926 77 69 8 70 0 124 0 amapchunkpl 152 5470 0 5427 21 17 4 20 0 158 0 amappl16 192 4299 0 4292 54 51 3 47 0 8 0 amappl15 184 1 0 1 1 1 0 1 0 8 0 amappl14 176 325 0 323 1 0 1 1 0 8 0 amappl13 168 304 0 304 1 1 0 1 0 8 0 amappl12 160 8 0 8 2 2 0 1 0 8 0 amappl11 152 55 0 40 1 0 1 1 0 8 0 amappl10 144 30 0 28 1 0 1 1 0 8 0 amappl9 136 391 0 390 1 0 1 1 0 8 0 amappl8 128 351 0 350 2 1 1 2 0 8 0 amappl7 120 117 0 110 1 0 1 1 0 8 0 amappl6 112 29 0 28 1 0 1 1 0 8 0 amappl5 104 705 0 693 1 0 1 1 0 8 0 amappl4 96 758 0 740 1 0 1 1 0 8 0 amappl3 88 396 0 392 1 0 1 1 0 8 0 amappl2 80 5465 0 5423 2 1 1 2 0 8 0 amappl1 72 25453 0 25157 24 14 10 18 0 8 0 amappl 80 2068 0 2048 2 0 2 2 0 84 0 dma4096 4096 1 0 1 1 1 0 1 0 8 0 dma256 256 6 0 6 1 1 0 1 0 8 0 dma128 128 253 0 253 1 1 0 1 0 8 0 dma64 64 6 0 6 1 1 0 1 0 8 0 dma32 32 7 0 7 1 1 0 1 0 8 0 dma16 16 18 0 17 1 0 1 1 0 8 0 aobjpl 64 7 0 0 1 0 1 1 0 8 0 uaddrrnd 24 805 0 790 1 0 1 1 0 8 0 uaddrbest 32 2 0 0 1 0 1 1 0 8 0 uaddr 24 805 0 790 1 0 1 1 0 8 0 vmmpekpl 168 8858 0 8833 2 0 2 2 0 8 0 vmmpepl 168 101827 0 101206 119 79 40 111 0 357 0 vmsppl 368 804 0 790 2 0 2 2 0 8 0 pdppl 4096 1617 0 1585 6 0 6 6 0 8 0 pvpl 32 284512 0 282522 176 100 76 170 0 265 50 pmappl 232 804 0 790 3 1 2 2 0 8 0 extentpl 40 53 0 36 1 0 1 1 0 8 0 phpool 112 272 0 15 8 0 8 8 0 8 0 ddb{1}> machine ddbcpu 0 Stopped at x86_ipi_db+0x1a: addq $0x8,%rsp x86_ipi_db(ffffffff826fcff0) at x86_ipi_db+0x1a sys/arch/amd64/amd64/db_interface.c:352 x86_ipi_handler() at x86_ipi_handler+0xc6 sys/arch/amd64/amd64/ipi.c:106 Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23 acpicpu_idle() at acpicpu_idle+0x331 sys/dev/acpi/acpicpu.c:1187 sched_idle(ffffffff826fcff0) at sched_idle+0x3f7 sys/kern/kern_sched.c:178 end trace frame: 0x0, count: 10 ddb{0}> trace x86_ipi_db(ffffffff826fcff0) at x86_ipi_db+0x1a sys/arch/amd64/amd64/db_interface.c:352 x86_ipi_handler() at x86_ipi_handler+0xc6 sys/arch/amd64/amd64/ipi.c:106 Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23 acpicpu_idle() at acpicpu_idle+0x331 sys/dev/acpi/acpicpu.c:1187 sched_idle(ffffffff826fcff0) at sched_idle+0x3f7 sys/kern/kern_sched.c:178 end trace frame: 0x0, count: -5 ddb{0}> machine ddbcpu 1 Stopped at db_enter+0x18: addq $0x8,%rsp db_enter() at db_enter+0x18 sys/arch/amd64/amd64/db_interface.c:398 panic(ffffffff823fea24) at panic+0x15c sys/kern/subr_prf.c:207 witness_checkorder(ffffffff828b9310,9,0) at witness_checkorder+0x10e0 sys/kern/subr_witness.c:821 __mp_lock(ffffffff828b9108) at __mp_lock+0xa1 read_rflags machine/cpufunc.h:195 [inline] __mp_lock(ffffffff828b9108) at __mp_lock+0xa1 intr_disable machine/cpufunc.h:216 [inline] __mp_lock(ffffffff828b9108) at __mp_lock+0xa1 sys/kern/kern_lock.c:142 pageflttrap(ffff800020da35a0,0) at pageflttrap+0x7f sys/arch/amd64/amd64/trap.c:180 kerntrap(ffff800020da35a0) at kerntrap+0xec sys/arch/amd64/amd64/trap.c:302 alltraps_kern_meltdown() at alltraps_kern_meltdown+0x7b pool_cache_gc(ffffffff828ab988) at pool_cache_gc+0x138 pool_cache_list_put sys/kern/subr_pool.c:1970 [inline] pool_cache_gc(ffffffff828ab988) at pool_cache_gc+0x138 sys/kern/subr_pool.c:2025 pool_gc_pages(0) at pool_gc_pages+0x6c sys/kern/subr_pool.c:1559 taskq_thread(ffffffff8277c5f0) at taskq_thread+0xec sys/kern/kern_task.c:437 end trace frame: 0x0, count: 5 ddb{1}> trace db_enter() at db_enter+0x18 sys/arch/amd64/amd64/db_interface.c:398 panic(ffffffff823fea24) at panic+0x15c sys/kern/subr_prf.c:207 witness_checkorder(ffffffff828b9310,9,0) at witness_checkorder+0x10e0 sys/kern/subr_witness.c:821 __mp_lock(ffffffff828b9108) at __mp_lock+0xa1 read_rflags machine/cpufunc.h:195 [inline] __mp_lock(ffffffff828b9108) at __mp_lock+0xa1 intr_disable machine/cpufunc.h:216 [inline] __mp_lock(ffffffff828b9108) at __mp_lock+0xa1 sys/kern/kern_lock.c:142 pageflttrap(ffff800020da35a0,0) at pageflttrap+0x7f sys/arch/amd64/amd64/trap.c:180 kerntrap(ffff800020da35a0) at kerntrap+0xec sys/arch/amd64/amd64/trap.c:302 alltraps_kern_meltdown() at alltraps_kern_meltdown+0x7b pool_cache_gc(ffffffff828ab988) at pool_cache_gc+0x138 pool_cache_list_put sys/kern/subr_pool.c:1970 [inline] pool_cache_gc(ffffffff828ab988) at pool_cache_gc+0x138 sys/kern/subr_pool.c:2025 pool_gc_pages(0) at pool_gc_pages+0x6c sys/kern/subr_pool.c:1559 taskq_thread(ffffffff8277c5f0) at taskq_thread+0xec sys/kern/kern_task.c:437 end trace frame: 0x0, count: -10