panic: acquiring blockable sleep lock with spinlock or critical section held (kernel_lock) &kernel_lock Stopped at db_enter+0x18: addq $0x8,%rsp TID PID UID PRFLAGS PFLAGS CPU COMMAND 496045 39540 0 0x2 0x4000000 1 syz-fuzzer * 33166 74809 0 0x14000 0x200 0 systqmp db_enter() at db_enter+0x18 sys/arch/amd64/amd64/db_interface.c:398 panic(ffffffff823ff160) at panic+0x15c sys/kern/subr_prf.c:207 witness_checkorder(ffffffff82899198,9,0) at witness_checkorder+0x10e0 sys/kern/subr_witness.c:821 __mp_lock(ffffffff82898f90) at __mp_lock+0xa1 read_rflags machine/cpufunc.h:195 [inline] __mp_lock(ffffffff82898f90) at __mp_lock+0xa1 intr_disable machine/cpufunc.h:216 [inline] __mp_lock(ffffffff82898f90) at __mp_lock+0xa1 sys/kern/kern_lock.c:142 pageflttrap(ffff800020db3180,0) at pageflttrap+0x7f sys/arch/amd64/amd64/trap.c:180 kerntrap(ffff800020db3180) at kerntrap+0xec sys/arch/amd64/amd64/trap.c:302 alltraps_kern_meltdown() at alltraps_kern_meltdown+0x7b pool_cache_gc(ffffffff828c78a0) at pool_cache_gc+0x138 pool_cache_list_put sys/kern/subr_pool.c:1970 [inline] pool_cache_gc(ffffffff828c78a0) at pool_cache_gc+0x138 sys/kern/subr_pool.c:2025 pool_gc_pages(0) at pool_gc_pages+0x6c sys/kern/subr_pool.c:1559 taskq_thread(ffffffff826d45b0) at taskq_thread+0xec sys/kern/kern_task.c:437 end trace frame: 0x0, count: 5 https://www.openbsd.org/ddb.html describes the minimum info required in bug reports. Insufficient info makes it difficult to find and fix bugs. ddb{0}> ddb{0}> set $lines = 0 ddb{0}> set $maxwidth = 0 ddb{0}> show panic acquiring blockable sleep lock with spinlock or critical section held (kernel_lock) &kernel_lock ddb{0}> trace db_enter() at db_enter+0x18 sys/arch/amd64/amd64/db_interface.c:398 panic(ffffffff823ff160) at panic+0x15c sys/kern/subr_prf.c:207 witness_checkorder(ffffffff82899198,9,0) at witness_checkorder+0x10e0 sys/kern/subr_witness.c:821 __mp_lock(ffffffff82898f90) at __mp_lock+0xa1 read_rflags machine/cpufunc.h:195 [inline] __mp_lock(ffffffff82898f90) at __mp_lock+0xa1 intr_disable machine/cpufunc.h:216 [inline] __mp_lock(ffffffff82898f90) at __mp_lock+0xa1 sys/kern/kern_lock.c:142 pageflttrap(ffff800020db3180,0) at pageflttrap+0x7f sys/arch/amd64/amd64/trap.c:180 kerntrap(ffff800020db3180) at kerntrap+0xec sys/arch/amd64/amd64/trap.c:302 alltraps_kern_meltdown() at alltraps_kern_meltdown+0x7b pool_cache_gc(ffffffff828c78a0) at pool_cache_gc+0x138 pool_cache_list_put sys/kern/subr_pool.c:1970 [inline] pool_cache_gc(ffffffff828c78a0) at pool_cache_gc+0x138 sys/kern/subr_pool.c:2025 pool_gc_pages(0) at pool_gc_pages+0x6c sys/kern/subr_pool.c:1559 taskq_thread(ffffffff826d45b0) at taskq_thread+0xec sys/kern/kern_task.c:437 end trace frame: 0x0, count: -10 ddb{0}> show registers rdi 0 rsi 0x1 rbp 0xffff800020db2ef0 rbx 0xffff800020db2fa0 rdx 0x8b rcx 0x2 rax 0x1 r8 0xffffffff821c2b4f kprintf+0x16f r9 0x1 r10 0x2 r11 0x15893b54d2a9fb50 r12 0x3000000008 r13 0xffff800020db2f00 r14 0x100 r15 0x1 rip 0xffffffff811a3778 db_enter+0x18 cs 0x8 rflags 0x246 rsp 0xffff800020db2ee0 ss 0x10 db_enter+0x18: addq $0x8,%rsp ddb{0}> show proc PROC (systqmp) pid=33166 stat=onproc flags process=14000 proc=200 pri=32, usrpri=50, nice=20 forw=0xffffffffffffffff, list=0xffff800020d98ea0,0xffff800020d989d0 process=0xffff800020d9a3d8 user=0xffff800020dae000, vmspace=0xffffffff82910998 estcpu=0, cpticks=0, pctcpu=0.0 user=0, sys=0, intr=0 ddb{0}> ps PID TID PPID UID S FLAGS WAIT COMMAND 92364 74044 39458 0 2 0 syz-executor.1 92364 400085 39458 0 3 0x4000080 netio syz-executor.1 92364 35969 39458 0 2 0x4000000 syz-executor.1 50278 144294 0 0 3 0x14200 bored sosplice 39458 259156 39540 0 2 0x482 syz-executor.1 45905 418934 0 0 3 0x14280 nfsidl nfsio 9010 71611 0 0 3 0x14280 nfsidl nfsio 24243 64442 0 0 3 0x14280 nfsidl nfsio 35797 389649 0 0 3 0x14280 nfsidl nfsio 50856 513315 0 0 3 0x14280 nfsidl nfsio 88321 403466 0 0 3 0x14280 nfsidl nfsio 46026 445758 0 0 3 0x14280 nfsidl nfsio 69220 372153 0 0 3 0x14280 nfsidl nfsio 73248 434686 0 0 3 0x14280 nfsidl nfsio 6832 279377 0 0 3 0x14280 nfsidl nfsio 39519 390183 0 0 3 0x14280 nfsidl nfsio 65753 368006 0 0 3 0x14280 nfsidl nfsio 59510 474221 0 0 3 0x14280 nfsidl nfsio 57816 51241 0 0 3 0x14280 nfsidl nfsio 56882 325585 0 0 3 0x14280 nfsidl nfsio 21337 147592 0 0 3 0x14280 nfsidl nfsio 53763 336283 0 0 3 0x14280 nfsidl nfsio 59520 393747 0 0 3 0x14280 nfsidl nfsio 77371 330543 0 0 3 0x14280 nfsidl nfsio 42456 322568 0 0 3 0x14280 nfsidl nfsio 15561 253069 0 0 3 0x14200 acct acct 94315 76683 39540 0 3 0x82 piperd syz-executor.0 39540 95743 49354 0 3 0x82 thrsleep syz-fuzzer 39540 55178 49354 0 3 0x4000082 nanosleep syz-fuzzer 39540 25659 49354 0 3 0x4000082 thrsleep syz-fuzzer 39540 185308 49354 0 3 0x4000082 kqread syz-fuzzer 39540 24343 49354 0 3 0x4000082 thrsleep syz-fuzzer 39540 244062 49354 0 3 0x4000082 thrsleep syz-fuzzer 39540 39215 49354 0 3 0x4000082 thrsleep syz-fuzzer 39540 129138 49354 0 3 0x4000082 thrsleep syz-fuzzer 39540 496045 49354 0 7 0x4000002 syz-fuzzer 39540 199921 49354 0 3 0x4000082 thrsleep syz-fuzzer 49354 247622 86569 0 3 0x10008a pause ksh 86569 4076 41908 0 3 0x92 select sshd 9760 269786 1 0 3 0x100083 ttyin getty 41908 207375 1 0 3 0x80 select sshd 80618 30762 88539 74 3 0x100092 bpf pflogd 88539 471733 1 0 3 0x80 netio pflogd 64898 400594 95390 73 3 0x100090 kqread syslogd 95390 354065 1 0 3 0x100082 netio syslogd 22074 517019 1 77 3 0x100090 poll dhclient 4285 394632 1 0 3 0x80 poll dhclient 73060 517922 0 0 3 0x14200 bored smr 77723 501949 0 0 2 0x14200 zerothread 54602 47187 0 0 3 0x14200 aiodoned aiodoned 67229 16902 0 0 3 0x14200 syncer update 99112 218183 0 0 3 0x14200 cleaner cleaner 82419 479884 0 0 3 0x14200 reaper reaper 95502 187591 0 0 3 0x14200 pgdaemon pagedaemon 72517 141642 0 0 3 0x14200 bored crynlk 35620 11700 0 0 3 0x14200 bored crypto 47646 268212 0 0 3 0x40014200 acpi0 acpi0 66985 459454 0 0 3 0x40014200 idle1 36003 77806 0 0 3 0x14200 bored softnet *74809 33166 0 0 7 0x14200 systqmp 18190 379074 0 0 3 0x14200 bored systq 77386 317522 0 0 3 0x40014200 bored softclock 23351 81414 0 0 3 0x40014200 idle0 1 246134 0 0 3 0x82 wait init 0 0 -1 0 3 0x10200 scheduler swapper ddb{0}> show all locks CPU 0: exclusive mutex mbufpl r = 0 (0xffffffff828c78b0) #0 witness_lock+0x4c7 stacktrace_save sys/sys/stacktrace.h:36 [inline] #0 witness_lock+0x4c7 sys/kern/subr_witness.c:1164 #1 mtx_enter_try+0x102 #2 mtx_enter+0x4b sys/kern/kern_lock.c:266 #3 pool_cache_gc+0x12d pl_enter sys/kern/subr_pool.c:104 [inline] #3 pool_cache_gc+0x12d pool_cache_list_put sys/kern/subr_pool.c:1968 [inline] #3 pool_cache_gc+0x12d sys/kern/subr_pool.c:2025 #4 pool_gc_pages+0x6c sys/kern/subr_pool.c:1559 #5 taskq_thread+0xec sys/kern/kern_task.c:437 #6 proc_trampoline+0x1c Process 74809 (systqmp) thread 0xffff800020d98c30 (33166) shared rwlock pools r = 0 (0xffffffff82742140) #0 witness_lock+0x4c7 stacktrace_save sys/sys/stacktrace.h:36 [inline] #0 witness_lock+0x4c7 sys/kern/subr_witness.c:1164 #1 pool_gc_pages+0x21 sys/kern/subr_pool.c:1555 #2 taskq_thread+0xec sys/kern/kern_task.c:437 #3 proc_trampoline+0x1c shared rwlock systqmp r = 0 (0xffffffff826d4620) #0 witness_lock+0x4c7 stacktrace_save sys/sys/stacktrace.h:36 [inline] #0 witness_lock+0x4c7 sys/kern/subr_witness.c:1164 #1 taskq_thread+0xdf sys/kern/kern_task.c:436 #2 proc_trampoline+0x1c exclusive mutex mbufpl r = 0 (0xffffffff828c78b0) #0 witness_lock+0x4c7 stacktrace_save sys/sys/stacktrace.h:36 [inline] #0 witness_lock+0x4c7 sys/kern/subr_witness.c:1164 #1 mtx_enter_try+0x102 #2 mtx_enter+0x4b sys/kern/kern_lock.c:266 #3 pool_cache_gc+0x12d pl_enter sys/kern/subr_pool.c:104 [inline] #3 pool_cache_gc+0x12d pool_cache_list_put sys/kern/subr_pool.c:1968 [inline] #3 pool_cache_gc+0x12d sys/kern/subr_pool.c:2025 #4 pool_gc_pages+0x6c sys/kern/subr_pool.c:1559 #5 taskq_thread+0xec sys/kern/kern_task.c:437 #6 proc_trampoline+0x1c ddb{0}> show malloc Type InUse MemUse HighUse Limit Requests Type Lim devbuf 9518 6416K 7124K 78643K 13748 0 pcb 13 8K 8K 78643K 63 0 rtable 123 6K 9K 78643K 444 0 ifaddr 79 16K 17K 78643K 170 0 counters 43 33K 34K 78643K 55 0 ioctlops 0 0K 4K 78643K 1519 0 iov 0 0K 16K 78643K 109 0 mount 1 1K 1K 78643K 1 0 vnodes 1224 77K 77K 78643K 1983 0 UFS quota 1 32K 32K 78643K 1 0 UFS mount 5 36K 36K 78643K 5 0 shm 2 1K 5K 78643K 3 0 VM map 2 1K 1K 78643K 2 0 sem 12 0K 0K 78643K 78 0 dirhash 12 2K 2K 78643K 12 0 ACPI 1824 197K 290K 78643K 13058 0 file desc 5 13K 25K 78643K 549 0 sigio 0 0K 0K 78643K 22 0 proc 62 63K 95K 78643K 499 0 subproc 32 2K 2K 78643K 51 0 NFS srvsock 1 0K 0K 78643K 1 0 NFS daemon 1 16K 16K 78643K 1 0 ip_moptions 0 0K 0K 78643K 14 0 in_multi 44 2K 2K 78643K 86 0 ether_multi 1 0K 0K 78643K 7 0 mrt 0 0K 0K 78643K 5 0 ISOFS mount 1 32K 32K 78643K 1 0 MSDOSFS mount 1 16K 16K 78643K 1 0 ttys 43 201K 201K 78643K 43 0 exec 0 0K 1K 78643K 238 0 pagedep 1 8K 8K 78643K 1 0 inodedep 1 32K 32K 78643K 1 0 newblk 1 0K 0K 78643K 1 0 VM swap 7 26K 26K 78643K 7 0 UVM amap 122 23K 24K 78643K 2463 0 UVM aobj 8 4K 4K 78643K 8 0 memdesc 1 4K 4K 78643K 1 0 crypto data 1 1K 1K 78643K 1 0 ip6_options 0 0K 0K 78643K 39 0 NDP 12 0K 0K 78643K 26 0 temp 99 3848K 3915K 78643K 7433 0 kqueue 3 4K 16K 78643K 19 0 SYN cache 2 16K 16K 78643K 2 0 ddb{0}> show all pools Name Size Requests Fail Releases Pgreq Pgrel Npage Hiwat Minpg Maxpg Idle arp 64 9 0 2 1 0 1 1 0 8 0 plcache 128 20 0 0 1 0 1 1 0 8 0 rtpcb 80 37 0 35 1 0 1 1 0 8 0 rtentry 112 70 0 24 2 0 2 2 0 8 0 unpcb 120 159 0 147 1 0 1 1 0 8 0 syncache 264 8 0 8 3 3 0 1 0 8 0 tcpqe 32 28 0 28 1 1 0 1 0 8 0 tcpcb 544 82 0 78 1 0 1 1 0 8 0 inpcb 280 269 0 262 3 1 2 2 0 8 1 rttmr 72 2 0 2 2 2 0 1 0 8 0 nd6 48 13 0 7 1 0 1 1 0 8 0 pkpcb 40 4 0 4 1 1 0 1 0 8 0 ppxss 1128 3 0 3 2 2 0 1 0 8 0 pfstscr 40 1 0 0 1 0 1 1 0 8 0 pfosfp 40 846 0 423 5 0 5 5 0 8 0 pfosfpen 112 1428 0 714 21 0 21 21 0 8 0 pfrktable 1344 63 0 58 1 0 1 1 0 8 0 pfstitem 24 14 0 5 1 0 1 1 0 8 0 pfstkey 112 14 0 5 1 0 1 1 0 8 0 pfstate 328 13 0 5 1 0 1 1 0 8 0 pfsrctr 152 51 0 46 1 0 1 1 0 8 0 pfrule 1360 38 0 27 3 1 2 2 0 8 1 art_heap8 4096 1 0 0 1 0 1 1 0 8 0 art_heap4 256 295 0 92 14 1 13 13 0 8 0 art_table 32 296 0 92 2 0 2 2 0 8 0 art_node 16 69 0 27 1 0 1 1 0 8 0 sysvmsgpl 40 10 0 8 1 0 1 1 0 8 0 semupl 112 1 0 1 1 1 0 1 0 8 0 semapl 112 74 0 64 1 0 1 1 0 8 0 shmpl 112 6 0 0 1 0 1 1 0 8 0 dirhash 1024 17 0 0 3 0 3 3 0 8 0 dino2pl 256 2036 0 632 89 0 89 89 0 8 0 ffsino 272 2036 0 632 94 0 94 94 0 8 0 nchpl 144 2974 0 1379 60 0 60 60 0 8 0 uvmvnodes 72 2796 0 0 51 0 51 51 0 8 0 vnodes 208 2796 0 0 148 0 148 148 0 8 0 namei 1024 8687 0 8687 1 0 1 1 0 8 1 percpumem 16 38 0 6 1 0 1 1 0 8 0 pfiaddrpl 120 20 0 16 1 0 1 1 0 8 0 scxspl 192 8866 0 8866 9 8 1 6 0 8 1 plimitpl 152 24 0 16 1 0 1 1 0 8 0 sigapl 424 783 0 730 6 0 6 6 0 8 0 futexpl 56 8464 0 8464 1 0 1 1 0 8 1 knotepl 112 92 0 73 1 0 1 1 0 8 0 kqueuepl 144 48 0 44 1 0 1 1 0 8 0 pipelkpl 48 257 0 247 1 0 1 1 0 8 0 pipepl 120 514 0 495 1 0 1 1 0 8 0 fdescpl 496 746 0 730 3 0 3 3 0 8 0 filepl 152 4272 0 4169 8 3 5 6 0 8 1 lockfpl 104 67 0 66 1 0 1 1 0 8 0 lockfspl 48 21 0 20 1 0 1 1 0 8 0 sessionpl 112 19 0 8 1 0 1 1 0 8 0 pgrppl 48 21 0 10 1 0 1 1 0 8 0 ucredpl 96 222 0 213 1 0 1 1 0 8 0 zombiepl 144 730 0 730 1 0 1 1 0 8 1 processpl 984 783 0 730 8 1 7 7 0 8 0 procpl 624 1680 0 1616 6 0 6 6 0 8 1 sockpl 400 469 0 448 7 4 3 5 0 8 0 mcl64k 65536 6 0 0 1 0 1 1 0 8 0 mcl16k 16384 1 0 0 1 0 1 1 0 8 0 mcl12k 12288 8 0 0 1 0 1 1 0 8 0 mcl9k 9216 2 0 0 1 0 1 1 0 8 0 mcl8k 8192 4 0 0 1 0 1 1 0 8 0 mcl4k 4096 13 0 0 2 0 2 2 0 8 0 mcl2k2 2112 3 0 0 1 0 1 1 0 8 0 mcl2k 2048 243 0 0 25 0 25 25 0 8 0 mtagpl 80 295 0 0 7 0 7 7 0 8 0 mbufpl 256 717 0 0 44 0 44 44 0 8 0 bufpl 280 4386 0 135 304 0 304 304 0 8 0 anonpl 16 76541 0 60134 68 2 66 66 0 124 0 amapchunkpl 152 3743 0 3581 12 3 9 10 0 158 2 amappl16 192 3159 0 2293 46 2 44 44 0 8 0 amappl15 184 1 0 0 1 0 1 1 0 8 0 amappl14 176 25 0 22 1 0 1 1 0 8 0 amappl13 168 51 0 46 1 0 1 1 0 8 0 amappl12 160 18 0 17 2 1 1 1 0 8 0 amappl11 152 175 0 160 1 0 1 1 0 8 0 amappl10 144 403 0 395 1 0 1 1 0 8 0 amappl9 136 385 0 384 1 0 1 1 0 8 0 amappl8 128 362 0 331 1 0 1 1 0 8 0 amappl7 120 507 0 493 1 0 1 1 0 8 0 amappl6 112 144 0 139 1 0 1 1 0 8 0 amappl5 104 639 0 623 1 0 1 1 0 8 0 amappl4 96 504 0 476 1 0 1 1 0 8 0 amappl3 88 133 0 128 1 0 1 1 0 8 0 amappl2 80 4955 0 4884 2 0 2 2 0 8 0 amappl1 72 24935 0 24482 22 11 11 18 0 8 0 amappl 80 1914 0 1868 2 0 2 2 0 84 0 dma4096 4096 1 0 1 1 1 0 1 0 8 0 dma256 256 6 0 6 1 1 0 1 0 8 0 dma128 128 253 0 253 1 1 0 1 0 8 0 dma64 64 6 0 6 1 1 0 1 0 8 0 dma32 32 7 0 7 1 1 0 1 0 8 0 dma16 16 18 0 17 1 0 1 1 0 8 0 aobjpl 64 7 0 0 1 0 1 1 0 8 0 uaddrrnd 24 746 0 730 1 0 1 1 0 8 0 uaddrbest 32 2 0 0 1 0 1 1 0 8 0 uaddr 24 746 0 730 1 0 1 1 0 8 0 vmmpekpl 168 9911 0 9873 2 0 2 2 0 8 0 vmmpepl 168 94435 0 92404 142 49 93 121 0 357 4 vmsppl 368 745 0 730 2 0 2 2 0 8 0 pdppl 4096 1499 0 1460 6 0 6 6 0 8 0 pvpl 32 241760 0 222074 173 10 163 163 0 265 4 pmappl 232 745 0 730 2 1 1 2 0 8 0 extentpl 40 53 0 36 1 0 1 1 0 8 0 phpool 112 303 0 8 9 0 9 9 0 8 0 ddb{0}> machine ddbcpu 0 Invalid cpu 0 ddb{0}> trace db_enter() at db_enter+0x18 sys/arch/amd64/amd64/db_interface.c:398 panic(ffffffff823ff160) at panic+0x15c sys/kern/subr_prf.c:207 witness_checkorder(ffffffff82899198,9,0) at witness_checkorder+0x10e0 sys/kern/subr_witness.c:821 __mp_lock(ffffffff82898f90) at __mp_lock+0xa1 read_rflags machine/cpufunc.h:195 [inline] __mp_lock(ffffffff82898f90) at __mp_lock+0xa1 intr_disable machine/cpufunc.h:216 [inline] __mp_lock(ffffffff82898f90) at __mp_lock+0xa1 sys/kern/kern_lock.c:142 pageflttrap(ffff800020db3180,0) at pageflttrap+0x7f sys/arch/amd64/amd64/trap.c:180 kerntrap(ffff800020db3180) at kerntrap+0xec sys/arch/amd64/amd64/trap.c:302 alltraps_kern_meltdown() at alltraps_kern_meltdown+0x7b pool_cache_gc(ffffffff828c78a0) at pool_cache_gc+0x138 pool_cache_list_put sys/kern/subr_pool.c:1970 [inline] pool_cache_gc(ffffffff828c78a0) at pool_cache_gc+0x138 sys/kern/subr_pool.c:2025 pool_gc_pages(0) at pool_gc_pages+0x6c sys/kern/subr_pool.c:1559 taskq_thread(ffffffff826d45b0) at taskq_thread+0xec sys/kern/kern_task.c:437 end trace frame: 0x0, count: -10 ddb{0}> machine ddbcpu 1 Stopped at x86_ipi_db+0x1a: addq $0x8,%rsp x86_ipi_db(ffff800020d80ff0) at x86_ipi_db+0x1a sys/arch/amd64/amd64/db_interface.c:352 x86_ipi_handler() at x86_ipi_handler+0xc6 sys/arch/amd64/amd64/ipi.c:106 Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23 end of kernel end trace frame: 0xc00050d5d0, count: 12 ddb{1}> trace x86_ipi_db(ffff800020d80ff0) at x86_ipi_db+0x1a sys/arch/amd64/amd64/db_interface.c:352 x86_ipi_handler() at x86_ipi_handler+0xc6 sys/arch/amd64/amd64/ipi.c:106 Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23 end of kernel end trace frame: 0xc00050d5d0, count: -3