panic: mutex 0xfffffd806a82ea00 not held in mtx_leave Stopped at db_enter+0x25: addq $0x8,%rsp TID PID UID PRFLAGS PFLAGS CPU COMMAND 226131 38848 32767 0x10 0 1 syz-executor db_enter() at db_enter+0x25 sys/arch/amd64/amd64/db_interface.c:437 panic(ffffffff83078e14) at panic+0x1e5 sys/kern/subr_prf.c:198 mtx_leave(fffffd806a82ea00) at mtx_leave+0x17c sys/kern/kern_lock.c:335 msleep(fffffd806a82e918,fffffd806a82ea00,4,ffffffff832a21d0,0) at msleep+0x125 sys/kern/kern_synch.c:249 vm_map_lock_ln(fffffd806a82e8e8,fffffd805fb423c0,0) at vm_map_lock_ln+0xd4 sys/uvm/uvm_map.c:5250 uvm_mapanon(fffffd806a82e8e8,ffff80002a10ff00,21000,1000,2080713) at uvm_mapanon+0x292 uvm_mmapanon(fffffd806a82e8e8,ffff80002a10ff00,21000,3,7,5002,cbbd73caab65f10b,5002) at uvm_mmapanon+0x1d0 sys/uvm/uvm_mmap.c:1019 sys_mmap(ffff80002e37ccd8,ffff80002a110070,ffff80002a10ffc0) at sys_mmap+0xa96 sys/uvm/uvm_mmap.c:421 syscall(ffff80002a110070) at syscall+0xbb6 mi_syscall sys/sys/syscall_mi.h:179 [inline] syscall(ffff80002a110070) at syscall+0xbb6 sys/arch/amd64/amd64/trap.c:577 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0x77daa4887e80, count: 5 https://www.openbsd.org/ddb.html describes the minimum info required in bug reports. Insufficient info makes it difficult to find and fix bugs. ddb{0}> ddb{0}> set $lines = 0 ddb{0}> set $maxwidth = 0 ddb{0}> show panic *cpu0: mutex 0xfffffd806a82ea00 not held in mtx_leave ddb{0}> trace db_enter() at db_enter+0x25 sys/arch/amd64/amd64/db_interface.c:437 panic(ffffffff83078e14) at panic+0x1e5 sys/kern/subr_prf.c:198 mtx_leave(fffffd806a82ea00) at mtx_leave+0x17c sys/kern/kern_lock.c:335 msleep(fffffd806a82e918,fffffd806a82ea00,4,ffffffff832a21d0,0) at msleep+0x125 sys/kern/kern_synch.c:249 vm_map_lock_ln(fffffd806a82e8e8,fffffd805fb423c0,0) at vm_map_lock_ln+0xd4 sys/uvm/uvm_map.c:5250 uvm_mapanon(fffffd806a82e8e8,ffff80002a10ff00,21000,1000,2080713) at uvm_mapanon+0x292 uvm_mmapanon(fffffd806a82e8e8,ffff80002a10ff00,21000,3,7,5002,cbbd73caab65f10b,5002) at uvm_mmapanon+0x1d0 sys/uvm/uvm_mmap.c:1019 sys_mmap(ffff80002e37ccd8,ffff80002a110070,ffff80002a10ffc0) at sys_mmap+0xa96 sys/uvm/uvm_mmap.c:421 syscall(ffff80002a110070) at syscall+0xbb6 mi_syscall sys/sys/syscall_mi.h:179 [inline] syscall(ffff80002a110070) at syscall+0xbb6 sys/arch/amd64/amd64/trap.c:577 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0x77daa4887e80, count: -10 ddb{0}> show registers rdi 0 rsi 0x1 rbp 0xffff80002a10fae0 rbx 0xffffffff834aedbf cpu_info_full_primary+0x2dbf rdx 0 rcx 0xffff80002e37ccd8 rax 0xffffffff834adff0 cpu_info_full_primary+0x1ff0 r8 0x101010101010101 r9 0x8080808080808080 r10 0xe083e7e00fbe7efa r11 0xaf077144303edfd9 r12 0xffffffff834aebc0 cpu_info_full_primary+0x2bc0 r13 0 r14 0 r15 0x1 rip 0xffffffff816a99e5 db_enter+0x25 cs 0x8 rflags 0x246 rsp 0xffff80002a10fad0 ss 0x10 db_enter+0x25: addq $0x8,%rsp ddb{0}> show proc PROC (syz-executor) tid=209228 pid=91672 tcnt=2 stat=sleep flags process=10 proc=20 runpri=50, usrpri=50, slppri=4, nice=20 wchan=0x0, wmesg=, ps_single=0x0 scnt=0 ecnt=0 forw=0xffffffffffffffff, list=0xffff80002e37d980,0xffff80002e37c550 process=0xffff80002df756c0 user=0xffff80002a10b000, vmspace=0xfffffd806a82e8e8 estcpu=36, cpticks=1, pctcpu=0.0, user=0, sys=1, intr=0 ddb{0}> ps PID TID PPID UID S FLAGS WAIT COMMAND *91672 209228 3604 32767 3 0x30 syz-executor 91672 23 3604 32767 3 0x4000090 fsleep syz-executor 38848 226131 35774 32767 7 0x10 syz-executor 38848 365481 35774 32767 3 0x4000090 lockf syz-executor 38848 331776 35774 32767 2 0x4000010 syz-executor 79216 211853 22128 32767 2 0x10 syz-executor 79216 333448 22128 32767 3 0x4000090 fsleep syz-executor 14724 98553 6665 32767 2 0x10 syz-executor 14724 35996 6665 32767 3 0x4000090 sbwait syz-executor 65321 444102 97744 32767 2 0x10 syz-executor 65321 113035 97744 32767 3 0x4000090 fsleep syz-executor 41444 137111 33010 32767 3 0x90 nanoslp syz-executor 41444 196571 33010 32767 3 0x4000090 netacc syz-executor 41444 489390 33010 32767 3 0x4000090 fsleep syz-executor 458 337578 6131 32767 2 0x10 syz-executor 458 105857 6131 32767 3 0x4000090 sbwait syz-executor 458 364772 6131 32767 3 0x4000090 fsleep syz-executor 35774 69549 75564 32767 3 0x90 nanoslp syz-executor 6131 189656 47651 32767 3 0x90 nanoslp syz-executor 75564 387578 43610 0 3 0x82 wait syz-executor 3604 193978 49514 32767 3 0x90 nanoslp syz-executor 58696 158459 3667 32767 2 0x10 syz-executor 6665 253270 76539 32767 3 0x90 nanoslp syz-executor 22128 220433 16666 32767 3 0x90 nanoslp syz-executor 97744 451856 2195 32767 3 0x90 nanoslp syz-executor 33010 452597 61661 32767 3 0x90 nanoslp syz-executor 49514 253219 43610 0 3 0x82 wait syz-executor 76539 271635 43610 0 3 0x82 wait syz-executor 16666 142235 43610 0 3 0x82 wait syz-executor 47651 411163 43610 0 3 0x82 wait syz-executor 2195 518127 43610 0 3 0x82 wait syz-executor 3667 230016 43610 0 3 0x82 wait syz-executor 61661 193131 43610 0 3 0x82 wait syz-executor 61441 80535 0 0 3 0x14200 bored sosplice 43610 964 57071 0 3 0x82 kqread syz-executor 57071 111741 48875 0 3 0x10008a sigsusp ksh 48875 513731 25000 0 3 0x98 kqread sshd-session 25000 95163 60767 0 3 0x92 kqread sshd-session 77995 57797 1 0 3 0x100083 ttyin getty 60767 157874 1 0 3 0x88 kqread sshd 59594 128206 82009 73 3 0x1100010 biowait syslogd 82009 493770 1 0 3 0x100082 sbwait syslogd 20570 431086 1 0 3 0x100080 kqread resolvd 29579 9313 1841 77 3 0x100092 kqread dhcpleased 83396 519280 1841 77 3 0x100092 kqread dhcpleased 1841 245798 1 0 3 0x80 kqread dhcpleased 77868 90912 0 0 3 0x14200 bored smr 952 451139 0 0 2 0x14200 zerothread 16379 246175 0 0 3 0x14200 aiodoned aiodoned 42054 34811 0 0 3 0x14200 syncer update 23362 366636 0 0 3 0x14200 cleaner cleaner 75420 482306 0 0 3 0x14200 reaper reaper 59009 27882 0 0 3 0x14200 pgdaemon pagedaemon 50516 458466 0 0 3 0x14200 bored viomb 7401 7662 0 0 3 0x40014200 acpi0 acpi0 88141 116382 0 0 3 0x40014200 idle1 25563 244674 0 0 3 0x14200 bored softnet3 36861 465070 0 0 3 0x14200 bored softnet2 88792 235236 0 0 3 0x14200 bored softnet1 91677 36966 0 0 3 0x14200 bored softnet0 54191 330301 0 0 3 0x14200 bored systqmp 34559 466199 0 0 3 0x14200 bored systq 16910 304834 0 0 3 0x14200 tmoslp softclockmp 74260 222602 0 0 3 0x40014200 tmoslp softclock 57686 176696 0 0 3 0x40014200 idle0 1 295596 0 0 3 0x82 wait init 0 0 -1 0 3 0x10200 scheduler swapper ddb{0}> show all locks CPU 0: exclusive mutex &map->flags_lock r = 0 (0xfffffd806a82ea40) #0 witness_lock+0x5bb stacktrace_save sys/sys/stacktrace.h:37 [inline] #0 witness_lock+0x5bb sys/kern/subr_witness.c:1155 #1 mtx_enter_try+0x178 #2 mtx_enter+0x60 sys/kern/kern_lock.c:239 #3 vm_map_lock_ln+0x153 sys/uvm/uvm_map.c:5258 #4 uvm_mapanon+0x292 #5 uvm_mmapanon+0x1d0 sys/uvm/uvm_mmap.c:1019 #6 sys_mmap+0xa96 sys/uvm/uvm_mmap.c:421 #7 syscall+0xbb6 mi_syscall sys/sys/syscall_mi.h:179 [inline] #7 syscall+0xbb6 sys/arch/amd64/amd64/trap.c:577 #8 Xsyscall+0x128 Process 91672 (syz-executor) thread 0xffff80002e37c540 (23) shared rwlock vmmaplk r = 0 (0xfffffd806a82e9e0) #0 witness_lock+0x5bb stacktrace_save sys/sys/stacktrace.h:37 [inline] #0 witness_lock+0x5bb sys/kern/subr_witness.c:1155 #1 uvmfault_lookup+0x122 sys/uvm/uvm_fault.c:1785 #2 uvm_fault_check+0x47 sys/uvm/uvm_fault.c:672 #3 uvm_fault+0x112 sys/uvm/uvm_fault.c:600 #4 uvm_fault_wire+0x73 sys/uvm/uvm_fault.c:1602 #5 uvm_map_pageable_wire+0x40f sys/uvm/uvm_map.c:2127 #6 sys_mlock+0x23d sys/uvm/uvm_mmap.c:837 #7 syscall+0xaf8 mi_syscall sys/sys/syscall_mi.h:179 [inline] #7 syscall+0xaf8 sys/arch/amd64/amd64/trap.c:577 #8 Xsyscall+0x128 exclusive kernel_lock &kernel_lock r = 0 (0xffffffff835a2200) #0 witness_lock+0x5bb stacktrace_save sys/sys/stacktrace.h:37 [inline] #0 witness_lock+0x5bb sys/kern/subr_witness.c:1155 #1 __mp_acquire_count+0x58 #2 mi_switch+0x4b7 sys/kern/sched_bsd.c:441 #3 sleep_finish+0x21e sys/kern/kern_synch.c:416 #4 rw_enter+0x348 sys/kern/kern_rwlock.c:285 #5 vm_map_lock_ln+0x142 sys/uvm/uvm_map.c:5257 #6 uvm_map_pageable+0x187 sys/uvm/uvm_map.c:2216 #7 sys_mlock+0x23d sys/uvm/uvm_mmap.c:837 #8 syscall+0xaf8 mi_syscall sys/sys/syscall_mi.h:179 [inline] #8 syscall+0xaf8 sys/arch/amd64/amd64/trap.c:577 #9 Xsyscall+0x128 Process 59594 (syslogd) thread 0xffff8000ffffdbe8 (128206) exclusive rrwlock inode r = 0 (0xfffffd806e2abb40) #0 witness_lock+0x5bb stacktrace_save sys/sys/stacktrace.h:37 [inline] #0 witness_lock+0x5bb sys/kern/subr_witness.c:1155 #1 rw_enter+0x41b sys/kern/kern_rwlock.c:309 #2 rrw_enter+0xbe sys/kern/kern_rwlock.c:464 #3 VOP_LOCK+0xa6 sys/kern/vfs_vops.c:524 #4 vn_lock+0xa4 sys/kern/vfs_vnops.c:564 #5 sys_fsync+0x152 sys/kern/vfs_syscalls.c:2926 #6 syscall+0xaf8 mi_syscall sys/sys/syscall_mi.h:179 [inline] #6 syscall+0xaf8 sys/arch/amd64/amd64/trap.c:577 #7 Xsyscall+0x128 ddb{0}> show malloc Type InUse MemUse HighUse Limit Requests Type Lim devbuf 10181 11047K 11059K 166960K 11289 0 pcb 17 12K 12K 166960K 17 0 rtable 226 6K 6K 166960K 616 0 pf 31 16K 16K 166960K 52 0 ifaddr 41 7K 7K 166960K 83 0 ifgroup 50 2K 2K 166960K 91 0 sysctl 3 1K 1K 166960K 3 0 counters 64 36K 36K 166960K 84 0 ioctlops 0 0K 2K 166960K 38 0 iov 0 0K 4K 166960K 4 0 mount 1 1K 1K 166960K 1 0 log 0 0K 0K 166960K 4 0 vnodes 1458 92K 92K 166960K 1518 0 UFS quota 1 32K 32K 166960K 1 0 UFS mount 5 36K 36K 166960K 5 0 shm 2 1K 5K 166960K 4 0 VM map 2 1K 1K 166960K 2 0 sem 5 0K 0K 166960K 5 0 dirhash 12 2K 2K 166960K 12 0 ACPI 1690 195K 286K 166960K 12468 0 file desc 26 97K 157K 166960K 368 0 sigio 0 0K 0K 166960K 2 0 proc 58 79K 176K 166960K 744 0 subproc 104 6K 12K 166960K 234 0 NFS srvsock 1 0K 0K 166960K 1 0 NFS daemon 1 16K 16K 166960K 1 0 ip_moptions 0 0K 0K 166960K 12 0 in_multi 98 7K 7K 166960K 199 0 ether_multi 1 0K 0K 166960K 1 0 ISOFS mount 1 32K 32K 166960K 1 0 MSDOSFS mount 1 16K 16K 166960K 1 0 ttys 55 254K 254K 166960K 55 0 exec 0 0K 1K 166960K 470 0 tdb 3 0K 0K 166960K 3 0 VM swap 8 62K 64K 166960K 10 0 UVM amap 255 74K 130K 166960K 4089 0 UVM aobj 10 4K 6K 166960K 11 0 pinsyscall 47 94K 126K 166960K 1600 0 memdesc 1 4K 4K 166960K 1 0 crypto data 1 1K 1K 166960K 1 0 ip6_options 2 0K 0K 166960K 6 0 NDP 26 2K 2K 166960K 54 0 temp 35 6810K 6874K 166960K 5157 0 kqueue 16 22K 26K 166960K 38 0 SYN cache 2 16K 16K 166960K 2 0 ddb{0}> show all pools Name Size Requests Fail Releases Pgreq Pgrel Npage Hiwat Minpg Maxpg Idle plcache 128 24 0 0 1 0 1 1 0 8 0 rtpcb 120 63 0 60 1 0 1 1 0 8 0 rtentry 112 209 0 103 4 0 4 4 0 8 0 unpcb 144 100 0 77 3 1 2 2 0 8 1 syncache 336 7 0 7 2 1 1 1 0 8 1 tcpqe 32 1 0 1 1 1 0 1 0 8 0 tcpcb 808 57 0 51 3 1 2 2 0 8 0 arp 120 35 0 19 1 0 1 1 0 8 0 ipq 40 1 0 0 1 0 1 1 0 8 0 ipqe 40 1 0 0 1 0 1 1 0 8 0 inpcb 336 190 0 180 2 0 2 2 0 8 0 nd6 136 51 0 27 1 0 1 1 0 8 0 kcovpl 48 18 0 10 1 0 1 1 0 8 0 art_heap8 4096 1 0 0 1 0 1 1 0 8 0 art_heap4 256 854 0 403 35 6 29 29 0 8 0 art_table 32 855 0 403 4 0 4 4 0 8 0 art_node 16 208 0 112 1 0 1 1 0 8 0 semupl 112 1 0 1 1 0 1 1 0 8 1 semapl 112 3 0 0 1 0 1 1 0 8 0 shmpl 112 8 0 1 1 0 1 1 0 8 0 dirhash 1024 17 0 0 3 0 3 3 0 8 0 dino2pl 256 1678 0 158 96 0 96 96 0 8 0 ffsino 272 1678 0 158 102 0 102 102 0 8 0 nchpl 144 1976 0 288 63 0 63 63 0 8 0 uvmvnodes 80 1921 0 0 40 0 40 40 0 8 0 vnodes 216 1921 0 0 107 0 107 107 0 8 0 namei 1024 6761 0 6761 4 2 2 2 0 8 2 percpumem 16 56 0 10 1 0 1 1 0 8 0 kstatmem 264 42 0 20 2 0 2 2 0 8 0 scxspl 216 8190 0 8189 4 2 2 3 1 8 1 plimitpl 152 67 0 43 2 0 2 2 0 8 0 sigapl 424 622 0 565 9 0 9 9 0 8 0 futexpl 64 1276 0 1271 2 1 1 1 0 8 0 knotepl 120 64 0 0 2 0 2 2 0 8 0 kqueuepl 216 39 0 28 1 0 1 1 0 8 0 pipepl 320 137 0 110 5 0 5 5 0 8 0 fdescpl 496 603 0 565 8 1 7 7 0 8 0 filepl 152 2145 0 1894 14 0 14 14 0 8 1 lockfpl 104 27 0 23 1 0 1 1 0 8 0 lockfspl 48 13 0 10 1 0 1 1 0 8 0 sessionpl 144 33 0 17 1 0 1 1 0 8 0 pgrppl 48 52 0 28 1 0 1 1 0 8 0 ucredpl 104 213 0 194 1 0 1 1 0 8 0 zombiepl 144 566 0 565 1 0 1 1 0 8 0 processpl 1160 622 0 565 6 0 6 6 0 8 1 procpl 648 750 0 683 7 0 7 7 0 8 0 sosppl 168 2 0 2 2 1 1 1 0 8 1 sockpl 664 354 0 318 5 1 4 5 0 8 1 mcl64k 65536 3 0 0 1 0 1 1 0 8 0 mcl16k 16384 1 0 0 1 0 1 1 0 8 0 mcl12k 12288 1 0 0 1 0 1 1 0 8 0 mcl8k 8192 3 0 0 1 0 1 1 0 8 0 mcl4k 4096 145 0 0 19 0 19 19 0 8 0 mcl2k 2048 19 0 0 3 0 3 3 0 8 0 mtagpl 96 1 0 0 1 0 1 1 0 8 0 mbufpl 256 196 0 0 13 0 13 13 0 8 0 bufpl 280 4628 0 94 324 0 324 324 0 8 0 anonpl 24 160302 0 156921 51 0 51 51 0 185 26 amapchunkpl 152 13362 0 12787 34 0 34 34 0 158 11 amappl16 200 3527 0 3517 5 3 2 5 0 8 1 amappl15 192 9 0 8 1 0 1 1 0 8 0 amappl14 184 130 0 119 1 0 1 1 0 8 0 amappl13 176 11 0 11 1 1 0 1 0 8 0 amappl12 168 1353 0 1315 3 0 3 3 0 8 1 amappl11 160 48 0 38 1 0 1 1 0 8 0 amappl10 152 12 0 12 1 1 0 1 0 8 0 amappl9 144 137 0 137 1 1 0 1 0 8 0 amappl8 136 19 0 18 1 0 1 1 0 8 0 amappl7 128 111 0 101 1 0 1 1 0 8 0 amappl6 120 256 0 255 1 0 1 1 0 8 0 amappl5 112 169 0 161 1 0 1 1 0 8 0 amappl4 104 313 0 298 1 0 1 1 0 8 0 amappl3 96 2460 0 2326 4 0 4 4 0 8 0 amappl2 88 783 0 723 2 0 2 2 0 8 0 amappl1 80 9041 0 8488 16 1 15 15 0 8 0 amappl 88 3679 0 3478 5 0 5 5 0 92 0 dma4096 4096 1 0 1 1 1 0 1 0 8 0 dma1024 1024 1 0 0 1 0 1 1 0 8 0 dma256 256 6 0 6 1 1 0 1 0 8 0 dma128 128 253 0 253 1 1 0 1 0 8 0 dma64 64 6 0 6 1 1 0 1 0 8 0 dma32 32 7 0 7 1 1 0 1 0 8 0 dma16 16 18 0 17 1 0 1 1 0 8 0 aobjpl 72 10 0 1 1 0 1 1 0 8 0 uaddrrnd 24 603 0 565 1 0 1 1 0 8 0 uaddrbest 32 2 0 0 1 0 1 1 0 8 0 uaddr 24 603 0 565 1 0 1 1 0 8 0 vmmpekpl 168 6352 0 6289 4 0 4 4 0 8 0 vmmpepl 168 45228 0 43202 100 1 99 99 0 357 4 vmsppl 448 602 0 565 6 0 6 6 0 8 0 rwobjpl 56 18803 0 16032 43 1 42 43 0 8 0 pdppl 4096 1213 0 1130 141 40 101 113 0 8 18 pvpl 32 27109 0 0 220 1 219 219 0 265 0 pmappl 248 602 0 565 5 1 4 4 0 8 0 extentpl 40 55 0 38 1 0 1 1 0 8 0 phpool 112 436 0 52 12 0 12 12 0 8 0 ddb{0}> machine ddbcpu 0 Invalid cpu 0 ddb{0}> trace db_enter() at db_enter+0x25 sys/arch/amd64/amd64/db_interface.c:437 panic(ffffffff83078e14) at panic+0x1e5 sys/kern/subr_prf.c:198 mtx_leave(fffffd806a82ea00) at mtx_leave+0x17c sys/kern/kern_lock.c:335 msleep(fffffd806a82e918,fffffd806a82ea00,4,ffffffff832a21d0,0) at msleep+0x125 sys/kern/kern_synch.c:249 vm_map_lock_ln(fffffd806a82e8e8,fffffd805fb423c0,0) at vm_map_lock_ln+0xd4 sys/uvm/uvm_map.c:5250 uvm_mapanon(fffffd806a82e8e8,ffff80002a10ff00,21000,1000,2080713) at uvm_mapanon+0x292 uvm_mmapanon(fffffd806a82e8e8,ffff80002a10ff00,21000,3,7,5002,cbbd73caab65f10b,5002) at uvm_mmapanon+0x1d0 sys/uvm/uvm_mmap.c:1019 sys_mmap(ffff80002e37ccd8,ffff80002a110070,ffff80002a10ffc0) at sys_mmap+0xa96 sys/uvm/uvm_mmap.c:421 syscall(ffff80002a110070) at syscall+0xbb6 mi_syscall sys/sys/syscall_mi.h:179 [inline] syscall(ffff80002a110070) at syscall+0xbb6 sys/arch/amd64/amd64/trap.c:577 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0x77daa4887e80, count: -10 ddb{0}> machine ddbcpu 1 Stopped at x86_ipi_db+0x27: addq $0x8,%rsp x86_ipi_db(ffff800029b7bff0) at x86_ipi_db+0x27 sys/arch/amd64/amd64/db_interface.c:393 x86_ipi_handler() at x86_ipi_handler+0xd9 sys/arch/amd64/amd64/ipi.c:106 Xresume_lapic_ipi() at Xresume_lapic_ipi+0x27 end of kernel end trace frame: 0x7b363e072df0, count: 12 ddb{1}> trace x86_ipi_db(ffff800029b7bff0) at x86_ipi_db+0x27 sys/arch/amd64/amd64/db_interface.c:393 x86_ipi_handler() at x86_ipi_handler+0xd9 sys/arch/amd64/amd64/ipi.c:106 Xresume_lapic_ipi() at Xresume_lapic_ipi+0x27 end of kernel end trace frame: 0x7b363e072df0, count: -3