witness: lock order reversal: 1st 0xfffffd806aee1438 fdlock (&newfdp->fd_fd.fd_lock) 2nd 0xfffffd80710ee0a8 inode (&ip->i_lock) lock order [1] fdlock (&newfdp->fd_fd.fd_lock) -> [2] inode (&ip->i_lock) lock order data 0xffffffff8294f5d0 -> 0xffffffff828d64a9 is missing lock order [2] inode (&ip->i_lock) -> [3] sbufrcv (&so->so_rcv.sb_lock) #0 rw_enter+0xd1 #1 sblock+0x8a sys/kern/uipc_socket2.c:548 #2 soreceive+0x201 sys/kern/uipc_socket.c:876 #3 fifo_read+0xd2 sys/miscfs/fifofs/fifo_vnops.c:256 #4 VOP_READ+0xc3 sys/kern/vfs_vops.c:227 #5 vn_rdwr+0x10a #6 vndsetcred+0x86 sys/dev/vnd.c:684 #7 vndioctl+0xac1 sys/dev/vnd.c:485 #8 VOP_IOCTL+0x9a sys/kern/vfs_vops.c:264 #9 vn_ioctl+0xc0 sys/kern/vfs_vnops.c:525 #10 sys_ioctl+0x4a9 #11 syscall+0x8cf mi_syscall sys/sys/syscall_mi.h:180 [inline] #11 syscall+0x8cf sys/arch/amd64/amd64/trap.c:577 #12 Xsyscall+0x128 lock order [3] sbufrcv (&so->so_rcv.sb_lock) -> [1] fdlock (&newfdp->fd_fd.fd_lock) #0 rw_enter_write+0x5f sys/kern/kern_rwlock.c:128 #1 unp_externalize+0x2df sys/kern/uipc_usrreq.c:1074 #2 soreceive+0x9ff sys/kern/uipc_socket.c:1021 #3 recvit+0x321 sys/kern/uipc_syscalls.c:1082 #4 sys_recvmsg+0x138 sys/kern/uipc_syscalls.c:882 #5 syscall+0x8cf mi_syscall sys/sys/syscall_mi.h:180 [inline] #5 syscall+0x8cf sys/arch/amd64/amd64/trap.c:577 #6 Xsyscall+0x128 Stopped at db_enter+0x1c: addq $0x8,%rsp ddb{1}> ddb{1}> set $lines = 0 ddb{1}> set $maxwidth = 0 ddb{1}> show panic the kernel did not panic ddb{1}> trace db_enter() at db_enter+0x1c sys/arch/amd64/amd64/db_interface.c:437 witness_checkorder(fffffd80710ee0a8,9,0) at witness_checkorder+0xc05 rw_enter(fffffd80710ee098,1) at rw_enter+0xd1 rrw_enter(fffffd80710ee098,1) at rrw_enter+0x91 sys/kern/kern_rwlock.c:464 VOP_LOCK(fffffd80710ef550,2001) at VOP_LOCK+0x8b sys/kern/vfs_vops.c:524 vn_lock(fffffd80710ef550,2001) at vn_lock+0x85 sys/kern/vfs_vnops.c:564 vn_closefile(fffffd80687a2148,ffff800037399c60) at vn_closefile+0xc9 vn_close sys/kern/vfs_vnops.c:292 [inline] vn_closefile(fffffd80687a2148,ffff800037399c60) at vn_closefile+0xc9 sys/kern/vfs_vnops.c:609 fdrop(fffffd80687a2148,ffff800037399c60) at fdrop+0xd5 sys/kern/kern_descrip.c:1274 knote_remove(ffff800037399c60,fffffd806ee7a1b0,fffffd806ee7a238,3,0) at knote_remove+0x329 sys/kern/kern_event.c:1899 knote_fdclose(ffff800037399c60,3) at knote_fdclose+0xae sys/kern/kern_event.c:1934 fdrelease(ffff800037399c60,3) at fdrelease+0xa6 sys/kern/kern_descrip.c:758 syscall(ffff80002a210300) at syscall+0x8cf mi_syscall sys/sys/syscall_mi.h:180 [inline] syscall(ffff80002a210300) at syscall+0x8cf sys/arch/amd64/amd64/trap.c:577 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0xa5d62c994b0, count: -13 ddb{1}> show registers rdi 0 rsi 0x40000 acpi_pdirpa+0x2be71 rbp 0xffff80002a20fdc0 rbx 0xffffffff828e9404 pp_r600_decoded_lanes+0x2e88d rdx 0xffff800000de1dc0 rcx 0xffff800037399c60 rax 0x3ffff acpi_pdirpa+0x2be70 r8 0xffff80002a20fca0 r9 0x8080808080808080 r10 0x4e93435b108bfb5a r11 0x327a42100cbc839f r12 0xfffffd8003b25690 r13 0xfffffd800333ddc0 r14 0x3 r15 0xffffffff rip 0xffffffff81cae62c db_enter+0x1c cs 0x8 rflags 0x246 rsp 0xffff80002a20fdb0 ss 0x10 db_enter+0x1c: addq $0x8,%rsp ddb{1}> show proc PROC (syz-executor.6) tid=26168 pid=59164 tcnt=3 stat=onproc flags process=8000000 proc=4000000 runpri=32, usrpri=85, slppri=32, nice=20 wchan=0x0, wmesg=, ps_single=0x0 scnt=0 ecnt=0 forw=0xffffffffffffffff, list=0xffff800037398f90,0xffffffff82ea1428 process=0xffff80002a290d80 user=0xffff80002a20b000, vmspace=0xfffffd806adf6380 estcpu=35, cpticks=0, pctcpu=0.0, user=0, sys=0, intr=0 ddb{1}> ps PID TID PPID UID S FLAGS WAIT COMMAND 10177 421954 57642 0 2 0x8000000 syz-executor.0 10177 22344 57642 0 7 0xc000000 syz-executor.0 59164 31862 82373 0 2 0x8000000 syz-executor.6 59164 455686 82373 0 3 0xc000080 piperd syz-executor.6 *59164 26168 82373 0 7 0xc000000 syz-executor.6 68857 417085 82850 0 2 0x8000000 syz-executor.4 68857 63431 82850 0 3 0xc000080 fsleep syz-executor.4 66096 510160 26900 0 3 0x8000082 nanoslp syz-executor.7 44250 252869 26900 0 3 0x8000002 biowait syz-executor.3 82850 435511 26900 0 3 0x8000082 nanoslp syz-executor.4 88523 474413 26900 0 3 0x8000082 nanoslp syz-executor.5 78658 327292 26900 0 3 0x8000082 nanoslp syz-executor.2 81841 380696 0 0 3 0x14200 bored sosplice 82373 471906 26900 0 3 0x8000082 nanoslp syz-executor.6 96415 424418 26900 0 3 0x8000082 nanoslp syz-executor.1 57642 318094 26900 0 3 0x8000082 nanoslp syz-executor.0 26900 287217 47506 0 3 0x1a000082 wait syz-fuzzer 26900 357045 47506 0 3 0x1e000082 nanoslp syz-fuzzer 26900 173921 47506 0 3 0x1e000082 kqread syz-fuzzer 26900 297790 47506 0 3 0x1e000082 wait syz-fuzzer 26900 251164 47506 0 3 0x1e000082 thrsleep syz-fuzzer 26900 54070 47506 0 3 0x1e000082 wait syz-fuzzer 26900 44295 47506 0 3 0x1e000082 thrsleep syz-fuzzer 26900 284385 47506 0 3 0x1e000082 thrsleep syz-fuzzer 26900 2717 47506 0 3 0x1e000082 thrsleep syz-fuzzer 26900 514446 47506 0 3 0x1e000082 wait syz-fuzzer 26900 171568 47506 0 3 0x1e000082 thrsleep syz-fuzzer 26900 309695 47506 0 3 0x1e000082 wait syz-fuzzer 26900 403844 47506 0 3 0x1e000082 thrsleep syz-fuzzer 26900 520072 47506 0 3 0x1e000082 wait syz-fuzzer 26900 468591 47506 0 3 0x1e000082 wait syz-fuzzer 26900 298103 47506 0 3 0x1e000082 wait syz-fuzzer 47506 315262 28713 0 3 0x810008a sigsusp ksh 28713 500351 46060 0 3 0x1800009a kqread sshd 52752 92555 1 0 3 0x18100083 ttyin getty 46060 198046 1 0 3 0x18000088 kqread sshd 40476 367665 67117 73 3 0x19100010 ffs_fsync syslogd 67117 124650 1 0 3 0x18100082 sbwait syslogd 14567 298932 1 0 3 0x18100080 kqread resolvd 38871 434705 45357 77 3 0x18100092 kqread dhcpleased 75016 466258 45357 77 3 0x18100092 kqread dhcpleased 45357 239588 1 0 3 0x18000080 kqread dhcpleased 24860 31675 0 0 3 0x14200 bored smr 16174 191697 0 0 2 0x14200 zerothread 52157 511692 0 0 3 0x14200 aiodoned aiodoned 12851 170560 0 0 3 0x14200 syncer update 12185 173957 0 0 3 0x14200 cleaner cleaner 38252 78982 0 0 3 0x14200 reaper reaper 2825 348161 0 0 3 0x14200 pgdaemon pagedaemon 44693 66820 0 0 3 0x14200 bored viomb 61854 28690 0 0 3 0x40014200 acpi0 acpi0 60692 262945 0 0 3 0x40014200 idle1 66061 483298 0 0 3 0x14200 bored softnet3 44865 327406 0 0 3 0x14200 bored softnet2 59524 399254 0 0 3 0x14200 bored softnet1 32851 446189 0 0 3 0x14200 bored softnet0 99525 19922 0 0 3 0x14200 bored systqmp 18596 501918 0 0 3 0x14200 bored systq 79154 313697 0 0 3 0x14200 tmoslp softclockmp 39217 146931 0 0 3 0x40014200 tmoslp softclock 49970 42431 0 0 3 0x40014200 idle0 1 62636 0 0 3 0x8000082 wait init 0 0 -1 0 3 0x10200 scheduler swapper ddb{1}> show all locks Process 59164 (syz-executor.6) thread 0xffff800037399c60 (26168) exclusive kernel_lock &kernel_lock r = 0 (0xffffffff82dea520) #0 witness_lock+0x446 stacktrace_save sys/sys/stacktrace.h:37 [inline] #0 witness_lock+0x446 sys/kern/subr_witness.c:1157 #1 vn_closefile+0x38 sys/kern/vfs_vnops.c:602 #2 fdrop+0xd5 sys/kern/kern_descrip.c:1274 #3 knote_remove+0x329 sys/kern/kern_event.c:1899 #4 knote_fdclose+0xae sys/kern/kern_event.c:1934 #5 fdrelease+0xa6 sys/kern/kern_descrip.c:758 #6 syscall+0x8cf mi_syscall sys/sys/syscall_mi.h:180 [inline] #6 syscall+0x8cf sys/arch/amd64/amd64/trap.c:577 #7 Xsyscall+0x128 exclusive rwlock fdlock r = 0 (0xfffffd806aee1438) #0 witness_lock+0x446 stacktrace_save sys/sys/stacktrace.h:37 [inline] #0 witness_lock+0x446 sys/kern/subr_witness.c:1157 #1 sys_close+0x8e sys/kern/kern_descrip.c:776 #2 syscall+0x8cf mi_syscall sys/sys/syscall_mi.h:180 [inline] #2 syscall+0x8cf sys/arch/amd64/amd64/trap.c:577 #3 Xsyscall+0x128 Process 44250 (syz-executor.3) thread 0xffff8000373999d0 (252869) exclusive rrwlock inode r = 0 (0xfffffd80650e3928) #0 witness_lock+0x446 stacktrace_save sys/sys/stacktrace.h:37 [inline] #0 witness_lock+0x446 sys/kern/subr_witness.c:1157 #1 rw_enter+0x32d sys/kern/kern_rwlock.c:309 #2 rrw_enter+0x91 sys/kern/kern_rwlock.c:464 #3 VOP_LOCK+0x8b sys/kern/vfs_vops.c:524 #4 ufs_ihashins+0x46 #5 ffs_vget+0x141 sys/ufs/ffs/ffs_vfsops.c:1230 #6 ffs_inode_alloc+0x1e4 sys/ufs/ffs/ffs_alloc.c:393 #7 ufs_mkdir+0xe6 sys/ufs/ufs/ufs_vnops.c:1112 #8 VOP_MKDIR+0xc3 sys/kern/vfs_vops.c:394 #9 domkdirat+0x125 sys/kern/vfs_syscalls.c:3104 #10 syscall+0x854 mi_syscall sys/sys/syscall_mi.h:180 [inline] #10 syscall+0x854 sys/arch/amd64/amd64/trap.c:577 #11 Xsyscall+0x128 exclusive rrwlock inode r = 0 (0xfffffd806f4b41b0) #0 witness_lock+0x446 stacktrace_save sys/sys/stacktrace.h:37 [inline] #0 witness_lock+0x446 sys/kern/subr_witness.c:1157 #1 rw_enter+0x32d sys/kern/kern_rwlock.c:309 #2 rrw_enter+0x91 sys/kern/kern_rwlock.c:464 #3 VOP_LOCK+0x8b sys/kern/vfs_vops.c:524 #4 vn_lock+0x85 sys/kern/vfs_vnops.c:564 #5 vfs_lookup+0xd3 sys/kern/vfs_lookup.c:418 #6 namei+0x56a sys/kern/vfs_lookup.c:250 #7 domkdirat+0x79 sys/kern/vfs_syscalls.c:3089 #8 syscall+0x854 mi_syscall sys/sys/syscall_mi.h:180 [inline] #8 syscall+0x854 sys/arch/amd64/amd64/trap.c:577 #9 Xsyscall+0x128 Process 40476 (syslogd) thread 0xffff8000ffffdc40 (367665) exclusive rrwlock inode r = 0 (0xfffffd806e75e0a0) #0 witness_lock+0x446 stacktrace_save sys/sys/stacktrace.h:37 [inline] #0 witness_lock+0x446 sys/kern/subr_witness.c:1157 #1 rw_enter+0x32d sys/kern/kern_rwlock.c:309 #2 rrw_enter+0x91 sys/kern/kern_rwlock.c:464 #3 VOP_LOCK+0x8b sys/kern/vfs_vops.c:524 #4 vn_lock+0x85 sys/kern/vfs_vnops.c:564 #5 sys_fsync+0xf1 sys/kern/vfs_syscalls.c:2931 #6 syscall+0x854 mi_syscall sys/sys/syscall_mi.h:180 [inline] #6 syscall+0x854 sys/arch/amd64/amd64/trap.c:577 #7 Xsyscall+0x128 ddb{1}> show malloc Type InUse MemUse HighUse Limit Requests Type Lim devbuf 10202 6626K 7338K 166960K 13298 0 pcb 17 14K 16K 166960K 276 0 rtable 234 7K 8K 166960K 717 0 pf 30 9K 10K 166960K 84 0 ifaddr 45 15K 16K 166960K 100 0 ifgroup 53 2K 2K 166960K 122 0 sysctl 4 1K 1K 166960K 10 0 counters 64 36K 36K 166960K 104 0 ioctlops 0 0K 4K 166960K 1543 0 iov 0 0K 18K 166960K 60 0 mount 1 1K 1K 166960K 1 0 log 0 0K 0K 166960K 4 0 vnodes 1385 87K 88K 166960K 2339 0 UFS quota 1 32K 32K 166960K 1 0 UFS mount 5 36K 36K 166960K 5 0 shm 2 1K 13K 166960K 26 0 VM map 2 1K 1K 166960K 2 0 sem 12 0K 0K 166960K 81 0 dirhash 12 2K 3K 166960K 45 0 ACPI 1697 195K 286K 166960K 12548 0 file desc 13 45K 93K 166960K 1061 0 sigio 0 0K 0K 166960K 16 0 proc 58 79K 103K 166960K 771 0 subproc 104 6K 7K 166960K 208 0 NFS srvsock 1 0K 0K 166960K 1 0 NFS daemon 1 16K 16K 166960K 1 0 ip_moptions 0 0K 0K 166960K 156 0 in_multi 99 7K 7K 166960K 236 0 ether_multi 1 0K 0K 166960K 7 0 mrt 3 0K 0K 166960K 7 0 ISOFS mount 1 32K 32K 166960K 1 0 MSDOSFS mount 1 16K 16K 166960K 1 0 ttys 97 440K 440K 166960K 97 0 exec 0 0K 1K 166960K 583 0 pfkey data 0 0K 0K 166960K 2 0 tdb 3 0K 0K 166960K 3 0 VM swap 8 62K 64K 166960K 10 0 UVM amap 253 79K 114K 166960K 11438 0 UVM aobj 37 6K 6K 166960K 39 0 pinsyscall 33 66K 100K 166960K 2414 0 memdesc 1 4K 4K 166960K 1 0 crypto data 1 1K 1K 166960K 1 0 ip6_options 0 0K 0K 166960K 74 0 NDP 13 0K 2K 166960K 66 0 temp 74 6808K 6879K 166960K 51182 0 kqueue 13 20K 28K 166960K 167 0 SYN cache 2 16K 16K 166960K 2 0 ddb{1}> show all pools Name Size Requests Fail Releases Pgreq Pgrel Npage Hiwat Minpg Maxpg Idle plcache 128 24 0 0 1 0 1 1 0 8 0 rtpcb 120 133 0 130 1 0 1 1 0 8 0 rtentry 112 245 0 137 4 0 4 4 0 8 0 unpcb 144 793 0 779 3 2 1 2 0 8 0 syncache 336 8 0 8 4 3 1 1 0 8 1 tcpqe 32 26 1 26 1 1 0 1 0 8 0 tcpcb 808 272 0 265 5 4 1 2 0 8 0 arp 120 43 0 23 1 0 1 1 0 8 0 inpcb 384 1162 0 1149 18 16 2 12 0 8 0 nd6 136 67 0 38 2 0 2 2 0 8 0 pkpcb 40 7 0 7 5 5 0 1 0 8 0 kcovpl 48 16 0 8 1 0 1 1 0 8 0 ppxss 1168 7 0 7 6 5 1 1 0 8 1 pffrag 232 5 0 4 1 0 1 1 0 482 0 pffrnode 88 5 0 4 1 0 1 1 0 8 0 pffrent 40 7 0 6 2 1 1 1 0 8 0 pfosfp 40 1428 0 1005 5 0 5 5 0 8 0 pfosfpen 112 1428 0 714 21 0 21 21 0 8 0 pfstitem 24 105 0 84 1 0 1 1 0 8 0 pfstkey 128 105 0 84 2 0 2 2 0 8 0 pfstate 376 105 0 84 5 1 4 5 0 8 0 pfrule 1344 21 0 16 2 1 1 2 0 8 0 art_heap8 4096 1 0 0 1 0 1 1 0 8 0 art_heap4 256 927 0 459 39 9 30 30 0 8 0 art_table 32 928 0 459 4 0 4 4 0 8 0 art_node 16 240 0 144 1 0 1 1 0 8 0 sysvmsgpl 40 22 0 14 1 0 1 1 0 8 0 semapl 112 78 0 68 1 0 1 1 0 8 0 shmpl 112 36 0 2 1 0 1 1 0 8 0 dirhash 1024 39 0 22 3 0 3 3 0 8 0 dino2pl 256 3162 0 1645 96 0 96 96 0 8 0 ffsino 272 3162 0 1645 102 0 102 102 0 8 0 nchpl 144 4817 0 3076 67 0 67 67 0 8 0 uvmvnodes 80 3977 0 0 82 0 82 82 0 8 0 vnodes 216 3977 0 0 221 0 221 221 0 8 0 namei 1024 16970 0 16969 6 5 1 2 0 8 0 percpumem 16 66 0 20 1 0 1 1 0 8 0 vcpupl 3904 3 0 0 1 0 1 1 0 8 0 vmpool 696 7 0 4 2 1 1 1 0 8 0 kstatmem 264 62 0 40 2 0 2 2 0 8 0 scsiplug 72 1 0 1 1 1 0 1 0 8 0 scxspl 216 31018 0 31016 16 15 1 8 1 8 0 plimitpl 152 195 0 178 1 0 1 1 0 8 0 sigapl 424 1376 0 1329 9 2 7 8 0 8 0 futexpl 64 17010 0 17009 5 4 1 1 0 8 0 knotepl 120 645 0 0 20 1 19 19 0 8 0 kqueuepl 216 353 0 343 2 1 1 2 0 8 0 pipepl 320 280 0 251 3 0 3 3 0 8 0 fdescpl 496 1336 0 1312 6 2 4 5 0 8 0 filepl 152 8682 0 8435 16 5 11 15 0 8 0 lockfpl 104 425 0 423 1 0 1 1 0 8 0 lockfspl 48 186 0 184 1 0 1 1 0 8 0 sessionpl 144 31 0 15 1 0 1 1 0 8 0 pgrppl 48 36 0 20 1 0 1 1 0 8 0 ucredpl 104 1264 0 1252 1 0 1 1 0 8 0 zombiepl 144 1333 0 1329 1 0 1 1 0 8 0 processpl 1144 1376 0 1329 5 1 4 5 0 8 0 procpl 656 2445 0 2379 9 2 7 9 0 8 0 srpgc 96 7 0 7 3 3 0 1 0 8 0 sosppl 168 7 0 7 3 2 1 1 0 8 1 sockpl 664 2099 0 2070 19 15 4 11 0 8 0 mcl64k 65536 19 0 0 3 0 3 3 0 8 0 mcl16k 16384 5 0 0 1 0 1 1 0 8 0 mcl12k 12288 4 0 0 1 0 1 1 0 8 0 mcl9k 9216 4 0 0 1 0 1 1 0 8 0 mcl8k 8192 7 0 0 1 0 1 1 0 8 0 mcl4k 4096 6 0 0 1 0 1 1 0 8 0 mcl2k2 2112 1 0 0 1 0 1 1 0 8 0 mcl2k 2048 370 0 0 47 0 47 47 0 8 0 mtagpl 96 24 0 0 1 0 1 1 0 8 0 mbufpl 256 379 0 0 24 0 24 24 0 8 0 bufpl 280 11520 0 4176 525 0 525 525 0 8 0 anonpl 24 330548 0 324542 100 33 67 94 0 186 1 amapchunkpl 152 37894 0 37305 46 7 39 42 0 158 6 amappl16 200 7811 0 7677 36 21 15 23 0 8 0 amappl15 192 8 0 8 1 1 0 1 0 8 0 amappl14 184 252 0 240 2 1 1 2 0 8 0 amappl13 176 16 0 16 1 1 0 1 0 8 0 amappl12 168 2102 0 2077 2 0 2 2 0 8 0 amappl11 160 67 0 48 1 0 1 1 0 8 0 amappl10 152 79 0 75 1 0 1 1 0 8 0 amappl9 144 147 0 147 3 3 0 1 0 8 0 amappl8 136 157 0 124 2 0 2 2 0 8 0 amappl7 128 56 0 42 1 0 1 1 0 8 0 amappl6 120 447 0 434 2 1 1 2 0 8 0 amappl5 112 192 0 179 1 0 1 1 0 8 0 amappl4 104 609 0 572 3 1 2 3 0 8 0 amappl3 96 7791 0 7728 3 0 3 3 0 8 0 amappl2 88 1764 0 1695 4 2 2 4 0 8 0 amappl1 80 12942 0 12458 22 11 11 22 0 8 0 amappl 88 10750 0 10583 7 1 6 6 0 92 0 dma4096 4096 1 0 1 1 1 0 1 0 8 0 dma1024 1024 1 0 0 1 0 1 1 0 8 0 dma256 256 6 0 6 1 1 0 1 0 8 0 dma128 128 253 0 253 1 1 0 1 0 8 0 dma64 64 6 0 6 1 1 0 1 0 8 0 dma32 32 7 0 7 1 1 0 1 0 8 0 dma16 16 18 0 17 1 0 1 1 0 8 0 aobjpl 72 38 0 2 1 0 1 1 0 8 0 uaddrrnd 24 1343 0 1316 1 0 1 1 0 8 0 uaddrbest 32 2 0 0 1 0 1 1 0 8 0 uaddr 24 1343 0 1316 1 0 1 1 0 8 0 vmmpekpl 168 13406 0 13335 4 0 4 4 0 8 0 vmmpepl 168 103065 0 101368 134 40 94 117 0 357 0 vmsppl 440 1342 0 1316 4 0 4 4 0 8 0 rwobjpl 56 35748 0 30680 76 3 73 73 0 8 0 pdppl 4096 2693 0 2635 124 56 68 79 0 8 10 pvpl 32 49172 0 0 399 2 397 397 0 265 0 pmappl 248 1342 0 1316 8 5 3 3 0 8 1 extentpl 40 56 0 38 1 0 1 1 0 8 0 phpool 112 466 0 84 12 0 12 12 0 8 0 ddb{1}> machine ddbcpu 0 Stopped at x86_ipi_db+0x1e: addq $0x8,%rsp ddb{0}> trace x86_ipi_db(ffffffff82c95ff0) at x86_ipi_db+0x1e sys/arch/amd64/amd64/db_interface.c:393 x86_ipi_handler() at x86_ipi_handler+0xb7 sys/arch/amd64/amd64/ipi.c:106 Xresume_lapic_ipi() at Xresume_lapic_ipi+0x27 __sanitizer_cov_trace_pc() at __sanitizer_cov_trace_pc intr_handler(ffff8000373aad40,ffff800000069c00) at intr_handler+0xa4 sys/arch/amd64/amd64/intr.c:549 Xintr_ioapic_edge16_untramp() at Xintr_ioapic_edge16_untramp+0x18f __mp_lock(ffffffff82dea318) at __mp_lock+0x122 __mp_lock_spin sys/kern/kern_lock.c:116 [inline] __mp_lock(ffffffff82dea318) at __mp_lock+0x122 sys/kern/kern_lock.c:147 intr_handler(ffff8000373aaea0,ffff80000067b200) at intr_handler+0xa4 sys/arch/amd64/amd64/intr.c:549 Xintr_ioapic_edge25_untramp() at Xintr_ioapic_edge25_untramp+0x18f __sanitizer_cov_trace_pc() at __sanitizer_cov_trace_pc+0x67 kd_curproc sys/dev/kcov.c:597 [inline] __sanitizer_cov_trace_pc() at __sanitizer_cov_trace_pc+0x67 sys/dev/kcov.c:158 __mp_lock(ffffffff82dea318) at __mp_lock+0x133 __mp_lock_spin sys/kern/kern_lock.c:116 [inline] __mp_lock(ffffffff82dea318) at __mp_lock+0x133 sys/kern/kern_lock.c:147 intr_handler(ffff8000373ab010,ffff80000006bc00) at intr_handler+0xa4 sys/arch/amd64/amd64/intr.c:549 Xintr_ioapic_edge23_untramp() at Xintr_ioapic_edge23_untramp+0x18f __mp_lock(ffffffff82dea318) at __mp_lock+0x12e __mp_lock_spin sys/kern/kern_lock.c:116 [inline] __mp_lock(ffffffff82dea318) at __mp_lock+0x12e sys/kern/kern_lock.c:147 vn_ioctl(fffffd806b00a870,80104277,ffff8000373ab220,ffff800037398f90) at vn_ioctl+0x44 sys/kern/vfs_vnops.c:508 sys_ioctl(ffff800037398f90,ffff8000373ab400,ffff8000373ab350) at sys_ioctl+0x4a9 syscall(ffff8000373ab400) at syscall+0x8cf mi_syscall sys/sys/syscall_mi.h:180 [inline] syscall(ffff8000373ab400) at syscall+0x8cf sys/arch/amd64/amd64/trap.c:577 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0x58f6914b8d0, count: -18 ddb{0}> machine ddbcpu 1 Stopped at db_enter+0x1c: addq $0x8,%rsp ddb{1}> trace db_enter() at db_enter+0x1c sys/arch/amd64/amd64/db_interface.c:437 witness_checkorder(fffffd80710ee0a8,9,0) at witness_checkorder+0xc05 rw_enter(fffffd80710ee098,1) at rw_enter+0xd1 rrw_enter(fffffd80710ee098,1) at rrw_enter+0x91 sys/kern/kern_rwlock.c:464 VOP_LOCK(fffffd80710ef550,2001) at VOP_LOCK+0x8b sys/kern/vfs_vops.c:524 vn_lock(fffffd80710ef550,2001) at vn_lock+0x85 sys/kern/vfs_vnops.c:564 vn_closefile(fffffd80687a2148,ffff800037399c60) at vn_closefile+0xc9 vn_close sys/kern/vfs_vnops.c:292 [inline] vn_closefile(fffffd80687a2148,ffff800037399c60) at vn_closefile+0xc9 sys/kern/vfs_vnops.c:609 fdrop(fffffd80687a2148,ffff800037399c60) at fdrop+0xd5 sys/kern/kern_descrip.c:1274 knote_remove(ffff800037399c60,fffffd806ee7a1b0,fffffd806ee7a238,3,0) at knote_remove+0x329 sys/kern/kern_event.c:1899 knote_fdclose(ffff800037399c60,3) at knote_fdclose+0xae sys/kern/kern_event.c:1934 fdrelease(ffff800037399c60,3) at fdrelease+0xa6 sys/kern/kern_descrip.c:758 syscall(ffff80002a210300) at syscall+0x8cf mi_syscall sys/sys/syscall_mi.h:180 [inline] syscall(ffff80002a210300) at syscall+0x8cf sys/arch/amd64/amd64/trap.c:577 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0xa5d62c994b0, count: -13