witness: userret: returning with the following locks held: exclusive rwlock solock r = 0 (0xfffffd8067207220) #0 witness_lock+0x44d #1 unp_solock_peer+0x64 sys/kern/uipc_usrreq.c:168 #2 uipc_usrreq+0x7c6 sys/kern/uipc_usrreq.c:350 #3 sosend+0x61b sys/kern/uipc_socket.c:657 #4 sendit+0x65d sys/kern/uipc_syscalls.c:682 #5 sys_sendmsg+0x198 sys/kern/uipc_syscalls.c:589 #6 syscall+0x4c3 mi_syscall sys/sys/syscall_mi.h:101 [inline] #6 syscall+0x4c3 sys/arch/amd64/amd64/trap.c:585 #7 Xsyscall+0x128 panic: witness_warn Stopped at db_enter+0x18: addq $0x8,%rsp TID PID UID PRFLAGS PFLAGS CPU COMMAND *141881 95028 32767 0x10 0x4000000 0 syz-executor.1 db_enter() at db_enter+0x18 sys/arch/amd64/amd64/db_interface.c:437 panic(ffffffff82595df3) at panic+0x177 sys/kern/subr_prf.c:202 witness_warn(2,0,ffffffff8262507e) at witness_warn+0x65e witness_debugger sys/kern/subr_witness.c:2505 [inline] witness_warn(2,0,ffffffff8262507e) at witness_warn+0x65e sys/kern/subr_witness.c:1473 userret(ffff800029a3c2a8) at userret+0x265 sys/kern/kern_sig.c:2012 syscall(ffff80002e49e670) at syscall+0x57e mi_syscall_return sys/sys/syscall_mi.h:128 [inline] syscall(ffff80002e49e670) at syscall+0x57e sys/arch/amd64/amd64/trap.c:607 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0x86e6da3d160, count: 9 https://www.openbsd.org/ddb.html describes the minimum info required in bug reports. Insufficient info makes it difficult to find and fix bugs. ddb{0}> ddb{0}> set $lines = 0 ddb{0}> set $maxwidth = 0 ddb{0}> show panic *cpu0: witness_warn ddb{0}> trace db_enter() at db_enter+0x18 sys/arch/amd64/amd64/db_interface.c:437 panic(ffffffff82595df3) at panic+0x177 sys/kern/subr_prf.c:202 witness_warn(2,0,ffffffff8262507e) at witness_warn+0x65e witness_debugger sys/kern/subr_witness.c:2505 [inline] witness_warn(2,0,ffffffff8262507e) at witness_warn+0x65e sys/kern/subr_witness.c:1473 userret(ffff800029a3c2a8) at userret+0x265 sys/kern/kern_sig.c:2012 syscall(ffff80002e49e670) at syscall+0x57e mi_syscall_return sys/sys/syscall_mi.h:128 [inline] syscall(ffff80002e49e670) at syscall+0x57e sys/arch/amd64/amd64/trap.c:607 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0x86e6da3d160, count: -6 ddb{0}> show registers rdi 0 rsi 0x1 rbp 0xffff80002e49e3a0 rbx 0xffffffff82926bb7 cpu_info_full_primary+0x2bb7 rdx 0xffff800000bce200 rcx 0 rax 0xffff800029a3c2a8 r8 0x101010101010101 r9 0x8080808080808080 r10 0xd71f9966c6efa715 r11 0x793d1dc03338e780 r12 0xffffffff829269b8 cpu_info_full_primary+0x29b8 r13 0 r14 0 r15 0x1 rip 0xffffffff81844108 db_enter+0x18 cs 0x8 rflags 0x246 rsp 0xffff80002e49e390 ss 0x10 db_enter+0x18: addq $0x8,%rsp ddb{0}> show proc PROC (syz-executor.1) pid=141881 stat=onproc flags process=10 proc=4000000 pri=32, usrpri=86, nice=20 forw=0xffffffffffffffff, list=0xffff800029a3dce8,0xffff800029a3cd38 process=0xffff800022d5dd30 user=0xffff80002e499000, vmspace=0xfffffd806c42d188 estcpu=36, cpticks=4, pctcpu=0.0 user=0, sys=4, intr=0 ddb{0}> ps PID TID PPID UID S FLAGS WAIT COMMAND 458 103129 74602 32767 3 0x90 nanoslp syz-executor.6 458 335193 74602 32767 3 0x4000090 fsleep syz-executor.6 16831 488529 11481 32767 2 0x10 syz-executor.3 95028 142088 95784 32767 2 0x10 syz-executor.1 *95028 141881 95784 32767 7 0x4000010 syz-executor.1 51700 456618 49376 32767 3 0x90 nanoslp syz-executor.5 51700 233651 49376 32767 3 0x4000090 netio syz-executor.5 51700 503373 49376 32767 3 0x4000090 fsleep syz-executor.5 51700 295812 49376 32767 3 0x4000090 fsleep syz-executor.5 86631 302503 51160 32767 3 0x3010 suspend syz-executor.0 86631 421318 51160 32767 2 0x4081090 syz-executor.0 70149 373118 79282 32767 3 0x90 nanoslp syz-executor.7 79282 113392 11515 0 3 0x82 wait syz-executor.7 95784 381601 78663 32767 2 0x490 syz-executor.1 78663 137595 11515 0 3 0x82 wait syz-executor.1 70064 360291 36158 32767 2 0x490 syz-executor.4 36158 210495 11515 0 3 0x82 wait syz-executor.4 49376 373731 78184 32767 3 0x90 nanoslp syz-executor.5 78184 266543 11515 0 3 0x82 wait syz-executor.5 74602 481264 9878 32767 3 0x90 nanoslp syz-executor.6 9878 175665 11515 0 3 0x82 wait syz-executor.6 61636 122538 0 0 3 0x14200 bored sosplice 11859 172203 63453 32767 2 0x490 syz-executor.2 11481 207757 11907 32767 3 0x90 nanoslp syz-executor.3 11907 250202 11515 0 3 0x82 wait syz-executor.3 51160 195289 46673 32767 3 0x90 nanoslp syz-executor.0 63453 252092 11515 0 3 0x82 wait syz-executor.2 46673 385381 11515 0 3 0x82 wait syz-executor.0 11515 321124 26671 0 3 0x82 thrsleep syz-fuzzer 11515 63080 26671 0 3 0x4000082 nanoslp syz-fuzzer 11515 174622 26671 0 3 0x4000082 thrsleep syz-fuzzer 11515 372328 26671 0 3 0x4000082 thrsleep syz-fuzzer 11515 245453 26671 0 2 0x4000082 syz-fuzzer 11515 439475 26671 0 3 0x4000082 thrsleep syz-fuzzer 11515 472375 26671 0 3 0x4000082 thrsleep syz-fuzzer 11515 306223 26671 0 3 0x4000082 thrsleep syz-fuzzer 26671 259038 95191 0 3 0x10008a sigsusp ksh 95191 166784 28753 0 3 0x9a kqread sshd 2699 200287 1 0 3 0x100083 ttyin getty 28753 483432 1 0 3 0x88 kqread sshd 21257 451905 7088 73 3 0x1100090 kqread syslogd 7088 201545 1 0 3 0x100082 netio syslogd 24933 476955 1 0 3 0x100080 kqread resolvd 9149 480798 63350 77 3 0x100092 kqread dhcpleased 41191 91643 63350 77 3 0x100092 kqread dhcpleased 63350 279899 1 0 3 0x80 kqread dhcpleased 51602 187454 0 0 3 0x14200 bored smr 9698 269788 0 0 3 0x14200 pgzero zerothread 55226 369281 0 0 3 0x14200 aiodoned aiodoned 99882 181166 0 0 3 0x14200 syncer update 23635 295041 0 0 3 0x14200 cleaner cleaner 15126 126946 0 0 3 0x14200 reaper reaper 7869 173596 0 0 3 0x14200 pgdaemon pagedaemon 31352 137860 0 0 3 0x14200 bored viomb 47450 356242 0 0 3 0x40014200 acpi0 acpi0 15719 129421 0 0 7 0x40014200 idle1 99681 131835 0 0 3 0x14200 bored softnet 76369 169207 0 0 3 0x14200 bored softnet 20886 296918 0 0 3 0x14200 bored softnet 35508 185252 0 0 3 0x14200 bored softnet 21769 147771 0 0 3 0x14200 bored systqmp 34089 13557 0 0 3 0x14200 bored systq 64289 24446 0 0 2 0x40014200 softclock 10472 250443 0 0 3 0x40014200 idle0 1 13365 0 0 3 0x82 wait init 0 0 -1 0 3 0x10200 scheduler swapper ddb{0}> show all locks Process 95028 (syz-executor.1) thread 0xffff800029a3c2a8 (141881) exclusive rwlock solock r = 0 (0xfffffd8067207220) #0 witness_lock+0x44d #1 unp_solock_peer+0x64 sys/kern/uipc_usrreq.c:168 #2 uipc_usrreq+0x7c6 sys/kern/uipc_usrreq.c:350 #3 sosend+0x61b sys/kern/uipc_socket.c:657 #4 sendit+0x65d sys/kern/uipc_syscalls.c:682 #5 sys_sendmsg+0x198 sys/kern/uipc_syscalls.c:589 #6 syscall+0x4c3 mi_syscall sys/sys/syscall_mi.h:101 [inline] #6 syscall+0x4c3 sys/arch/amd64/amd64/trap.c:585 #7 Xsyscall+0x128 ddb{0}> show malloc Type InUse MemUse HighUse Limit Requests Type Lim devbuf 10206 6412K 6420K 78643K 11358 0 pcb 13 10K 12K 78643K 15 0 rtable 236 6K 7K 78643K 1355 0 ifaddr 81 17K 17K 78643K 223 0 sysctl 3 1K 1K 78643K 3 0 counters 56 35K 35K 78643K 96 0 ioctlops 0 0K 2K 78643K 78 0 iov 0 0K 28K 78643K 1211 0 mount 1 1K 1K 78643K 1 0 log 0 0K 0K 78643K 4 0 vnodes 1271 79K 79K 78643K 5115 0 UFS quota 1 32K 32K 78643K 1 0 UFS mount 5 36K 36K 78643K 5 0 shm 2 1K 5K 78643K 90 0 VM map 2 1K 1K 78643K 2 0 sem 12 0K 1K 78643K 2078 0 dirhash 12 2K 2K 78643K 12 0 ACPI 1697 195K 286K 78643K 12548 0 file desc 24 89K 117K 78643K 8457 0 sigio 0 0K 0K 78643K 200 0 proc 56 78K 115K 78643K 1725 0 subproc 104 6K 6K 78643K 364 0 NFS srvsock 1 0K 0K 78643K 1 0 NFS daemon 1 16K 16K 78643K 1 0 ip_moptions 0 0K 0K 78643K 1526 0 in_multi 99 6K 7K 78643K 475 0 ether_multi 1 0K 0K 78643K 64 0 mrt 2 0K 0K 78643K 2 0 ISOFS mount 1 32K 32K 78643K 1 0 MSDOSFS mount 1 16K 16K 78643K 1 0 ttys 235 1049K 1049K 78643K 235 0 exec 0 0K 2K 78643K 2640 0 tdb 3 0K 0K 78643K 3 0 pagedep 1 8K 8K 78643K 1 0 inodedep 1 32K 32K 78643K 1 0 newblk 1 0K 0K 78643K 1 0 VM swap 7 26K 26K 78643K 7 0 UVM amap 271 82K 99K 78643K 52772 0 UVM aobj 131 4K 4K 78643K 134 0 memdesc 1 4K 4K 78643K 1 0 crypto data 1 1K 1K 78643K 1 0 ip6_options 0 0K 0K 78643K 347 0 NDP 11 0K 2K 78643K 87 0 temp 124 4726K 4790K 78643K 23678 0 kqueue 12 18K 26K 78643K 729 0 SYN cache 2 16K 16K 78643K 2 0 ddb{0}> show all pools Name Size Requests Fail Releases Pgreq Pgrel Npage Hiwat Minpg Maxpg Idle plcache 128 22 0 0 1 0 1 1 0 8 0 rtpcb 120 892 0 888 13 12 1 3 0 8 0 rtentry 112 338 0 227 4 0 4 4 0 8 0 unpcb 144 6264 0 6247 69 63 6 8 0 8 5 syncache 296 71 0 71 18 17 1 1 0 8 1 sackhl 24 1 0 1 1 1 0 1 0 8 0 tcpqe 32 195 0 195 9 8 1 1 0 8 1 tcpcb 736 4363 0 4354 133 128 5 16 0 8 1 arp 120 60 0 41 1 0 1 1 0 8 0 ipq 40 1 0 1 1 0 1 1 0 8 1 ipqe 40 4 0 4 1 0 1 1 0 8 1 inpcb 320 7673 0 7655 108 104 4 12 0 8 1 ip6q 72 6 0 6 2 2 0 1 0 8 0 ip6af 40 18 0 18 2 2 0 1 0 8 0 nd6 48 89 0 65 1 0 1 1 0 8 0 kcovpl 48 28 0 20 1 0 1 1 0 8 0 art_heap8 4096 1 0 0 1 0 1 1 0 8 0 art_heap4 256 1364 0 909 30 1 29 30 0 8 0 art_table 32 1365 0 909 4 0 4 4 0 8 0 art_node 16 337 0 236 1 0 1 1 0 8 0 sysvmsgpl 40 11 0 8 1 0 1 1 0 8 0 semapl 112 2076 0 2066 1 0 1 1 0 8 0 shmpl 112 131 0 3 4 0 4 4 0 8 0 dirhash 1024 17 0 0 3 0 3 3 0 8 0 dino2pl 256 14581 0 13129 92 0 92 92 0 8 0 ffsino 272 14581 0 13129 98 0 98 98 0 8 0 nchpl 144 26037 0 24395 63 0 63 63 0 8 0 uvmvnodes 80 5926 0 0 121 0 121 121 0 8 0 vnodes 224 5926 0 0 349 0 349 349 0 8 0 namei 1024 102393 0 102393 6 5 1 2 0 8 1 percpumem 16 60 0 20 1 0 1 1 0 8 0 kstatmem 264 62 0 40 2 0 2 2 0 8 0 scxspl 216 82439 0 82439 33 32 1 8 0 8 1 plimitpl 152 1553 0 1531 5 4 1 2 0 8 0 sigapl 424 8701 0 8646 7 0 7 7 0 8 0 futexpl 64 78200 0 78196 4 3 1 1 0 8 0 knotepl 120 746 0 0 18 1 17 17 0 8 0 kqueuepl 224 2804 0 2796 45 41 4 5 0 8 3 pipepl 336 2002 0 1972 56 53 3 8 0 8 0 fdescpl 496 8683 0 8648 7 2 5 6 0 8 0 filepl 152 69987 0 69732 125 107 18 20 0 8 6 lockfpl 104 1453 0 1451 3 2 1 2 0 8 0 lockfspl 48 359 0 357 1 0 1 1 0 8 0 sessionpl 144 43 0 27 1 0 1 1 0 8 0 pgrppl 48 121 0 105 1 0 1 1 0 8 0 ucredpl 104 23272 0 23254 1 0 1 1 0 8 0 zombiepl 144 8650 0 8646 1 0 1 1 0 8 0 processpl 1064 8701 0 8646 4 0 4 4 0 8 0 procpl 672 25393 0 25325 23 16 7 8 0 8 0 sosppl 168 111 0 111 16 16 0 1 0 8 0 sockpl 504 15068 0 15027 337 323 14 37 0 8 8 mcl64k 65536 34 0 0 3 0 3 3 0 8 0 mcl16k 16384 17 0 0 3 0 3 3 0 8 0 mcl12k 12288 25 0 0 2 0 2 2 0 8 0 mcl9k 9216 25 0 0 2 0 2 2 0 8 0 mcl8k 8192 21 0 0 3 0 3 3 0 8 0 mcl4k 4096 17 0 0 3 0 3 3 0 8 0 mcl2k2 2112 12 0 0 1 0 1 1 0 8 0 mcl2k 2048 299 0 0 20 2 18 20 0 8 0 mtagpl 96 3 0 0 1 0 1 1 0 8 0 mbufpl 256 864 0 0 42 0 42 42 0 8 0 bufpl 288 17743 0 11416 453 0 453 453 0 8 0 anonpl 24 1852662 0 1839463 248 147 101 126 0 186 1 amapchunkpl 152 178036 0 177434 106 80 26 37 0 158 0 amappl16 200 30458 0 30066 168 142 26 36 0 8 0 amappl15 192 1983 0 1979 1 0 1 1 0 8 0 amappl14 184 2922 0 2909 1 0 1 1 0 8 0 amappl13 176 202 0 201 1 0 1 1 0 8 0 amappl12 168 1402 0 1396 1 0 1 1 0 8 0 amappl11 160 1428 0 1408 1 0 1 1 0 8 0 amappl10 152 824 0 819 1 0 1 1 0 8 0 amappl9 144 2306 0 2299 1 0 1 1 0 8 0 amappl8 136 1244 0 1158 4 0 4 4 0 8 0 amappl7 128 415 0 401 1 0 1 1 0 8 0 amappl6 120 2159 0 2128 3 1 2 2 0 8 0 amappl5 112 5022 0 5006 1 0 1 1 0 8 0 amappl4 104 6082 0 6044 3 1 2 2 0 8 0 amappl3 96 27566 0 27506 2 0 2 2 0 8 0 amappl2 88 10310 0 10239 3 1 2 3 0 8 0 amappl1 80 211260 0 210526 30 13 17 19 0 8 1 amappl 88 51725 0 51547 6 1 5 5 0 92 0 dma4096 4096 1 0 1 1 1 0 1 0 8 0 dma1024 1024 1 0 0 1 0 1 1 0 8 0 dma256 256 6 0 6 1 1 0 1 0 8 0 dma128 128 253 0 253 1 1 0 1 0 8 0 dma64 64 6 0 6 1 1 0 1 0 8 0 dma32 32 7 0 7 1 1 0 1 0 8 0 dma16 16 18 0 17 1 0 1 1 0 8 0 aobjpl 72 133 0 3 3 0 3 3 0 8 0 uaddrrnd 24 8683 0 8648 1 0 1 1 0 8 0 uaddrbest 32 2 0 0 1 0 1 1 0 8 0 uaddr 24 8683 0 8648 1 0 1 1 0 8 0 vmmpekpl 168 72323 0 72270 3 0 3 3 0 8 0 vmmpepl 168 868603 0 865816 279 149 130 145 0 357 0 vmsppl 368 8682 0 8648 4 0 4 4 0 8 0 rwobjpl 56 225493 0 218002 121 12 109 110 0 8 0 pdppl 4096 17373 0 17296 375 294 81 91 0 8 4 pvpl 32 3751914 0 3733077 534 356 178 239 0 265 14 pmappl 248 8682 0 8648 4 1 3 3 0 8 0 extentpl 40 58 0 38 1 0 1 1 0 8 0 phpool 112 1665 0 808 26 0 26 26 0 8 0 ddb{0}> machine ddbcpu 0 Invalid cpu 0 ddb{0}> trace db_enter() at db_enter+0x18 sys/arch/amd64/amd64/db_interface.c:437 panic(ffffffff82595df3) at panic+0x177 sys/kern/subr_prf.c:202 witness_warn(2,0,ffffffff8262507e) at witness_warn+0x65e witness_debugger sys/kern/subr_witness.c:2505 [inline] witness_warn(2,0,ffffffff8262507e) at witness_warn+0x65e sys/kern/subr_witness.c:1473 userret(ffff800029a3c2a8) at userret+0x265 sys/kern/kern_sig.c:2012 syscall(ffff80002e49e670) at syscall+0x57e mi_syscall_return sys/sys/syscall_mi.h:128 [inline] syscall(ffff80002e49e670) at syscall+0x57e sys/arch/amd64/amd64/trap.c:607 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0x86e6da3d160, count: -6 ddb{0}> machine ddbcpu 1 Stopped at x86_ipi_db+0x1a: addq $0x8,%rsp x86_ipi_db(ffff800020dd8ff0) at x86_ipi_db+0x1a sys/arch/amd64/amd64/db_interface.c:393 x86_ipi_handler() at x86_ipi_handler+0xb7 sys/arch/amd64/amd64/ipi.c:106 Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23 acpicpu_idle() at acpicpu_idle+0x312 sys/dev/acpi/acpicpu.c:1206 sched_idle(ffff800020dd8ff0) at sched_idle+0x417 sys/kern/kern_sched.c:178 end trace frame: 0x0, count: 10 ddb{1}> trace x86_ipi_db(ffff800020dd8ff0) at x86_ipi_db+0x1a sys/arch/amd64/amd64/db_interface.c:393 x86_ipi_handler() at x86_ipi_handler+0xb7 sys/arch/amd64/amd64/ipi.c:106 Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23 acpicpu_idle() at acpicpu_idle+0x312 sys/dev/acpi/acpicpu.c:1206 sched_idle(ffff800020dd8ff0) at sched_idle+0x417 sys/kern/kern_sched.c:178 end trace frame: 0x0, count: -5