witness: lock order reversal: 1st 0xffffffff83829df8 &cq->cq_mtx (&cq->cq_mtx) 2nd 0xffffffff838eec28 &sched_lock (&sched_lock) lock order [1] &cq->cq_mtx (&cq->cq_mtx) -> [2] &sched_lock (&sched_lock) #0 mtx_enter+0x47 sys/kern/kern_lock.c:238 #1 sleep_setup+0xc6 sys/kern/kern_synch.c:348 #2 msleep+0x114 sys/kern/kern_synch.c:245 #3 dt_ioctl_record_stop+0xbc sys/dev/dt/dt_dev.c:575 #4 dtclose+0x105 dt_pcb_purge sys/dev/dt/dt_dev.c:-1 [inline] #4 dtclose+0x105 sys/dev/dt/dt_dev.c:232 #5 spec_close+0x45f sys/kern/spec_vnops.c:-1 #6 VOP_CLOSE+0x133 sys/kern/vfs_vops.c:156 #7 vn_closefile+0x12b vn_close sys/kern/vfs_vnops.c:292 [inline] #7 vn_closefile+0x12b sys/kern/vfs_vnops.c:615 #8 fdrop+0x126 sys/kern/kern_descrip.c:1265 #9 closef+0x192 sys/kern/kern_descrip.c:1249 #10 fdfree+0x116 sys/kern/kern_descrip.c:1181 #11 exit1+0x58f sys/kern/kern_exit.c:214 #12 sys_exit+0x1a sys/kern/kern_exit.c:-1 #13 syscall+0xb08 mi_syscall sys/sys/syscall_mi.h:176 [inline] #13 syscall+0xb08 sys/arch/amd64/amd64/trap.c:577 #14 Xsyscall+0x128 lock order [2] &sched_lock (&sched_lock) -> [1] &cq->cq_mtx (&cq->cq_mtx) #0 mtx_enter+0x47 sys/kern/kern_lock.c:238 #1 clockintr_cancel+0x2f sys/kern/kern_clockintr.c:323 #2 mi_switch+0x165 x86_atomic_clearbits_u32 machine/atomic.h:-1 [inline] #2 mi_switch+0x165 sys/kern/sched_bsd.c:371 #3 sleep_finish+0x24f sys/kern/kern_synch.c:412 #4 rwsleep+0xf2 sys/kern/kern_synch.c:298 #5 futex_wait+0x372 sys/kern/sys_futex.c:250 #6 sys_futex+0x149 sys/kern/sys_futex.c:101 #7 syscall+0xbc6 mi_syscall sys/sys/syscall_mi.h:176 [inline] #7 syscall+0xbc6 sys/arch/amd64/amd64/trap.c:577 #8 Xsyscall+0x128 Stopped at db_enter+0x25: addq $0x8,%rsp ddb{1}> ddb{1}> set $lines = 0 ddb{1}> set $maxwidth = 0 ddb{1}> show panic the kernel did not panic ddb{1}> trace db_enter() at db_enter+0x25 sys/arch/amd64/amd64/db_interface.c:438 witness_checkorder(ffffffff838eec28,9,0) at witness_checkorder+0x10fb sys/kern/subr_witness.c:-1 mtx_enter(ffffffff838eec18) at mtx_enter+0x47 sys/kern/kern_lock.c:238 sleep_setup(ffffffff83829e48,220,ffffffff83399e6d) at sleep_setup+0xc6 sys/kern/kern_synch.c:348 msleep(ffffffff83829e48,ffffffff83829de8,220,ffffffff83399e6d,0) at msleep+0x114 sys/kern/kern_synch.c:245 dt_ioctl_record_stop(ffff80000146d000) at dt_ioctl_record_stop+0xbc sys/dev/dt/dt_dev.c:575 dtclose(11e5f,81,2000,ffff80003c468d08) at dtclose+0x105 dt_pcb_purge sys/dev/dt/dt_dev.c:-1 [inline] dtclose(11e5f,81,2000,ffff80003c468d08) at dtclose+0x105 sys/dev/dt/dt_dev.c:232 spec_close(ffff80002f7d6790) at spec_close+0x45f sys/kern/spec_vnops.c:-1 VOP_CLOSE(fffffd805d402878,81,fffffd807f7d3138,ffff80003c468d08) at VOP_CLOSE+0x133 sys/kern/vfs_vops.c:156 vn_closefile(fffffd806be189b0,ffff80003c468d08) at vn_closefile+0x12b vn_close sys/kern/vfs_vnops.c:292 [inline] vn_closefile(fffffd806be189b0,ffff80003c468d08) at vn_closefile+0x12b sys/kern/vfs_vnops.c:615 fdrop(fffffd806be189b0,ffff80003c468d08) at fdrop+0x126 sys/kern/kern_descrip.c:1265 closef(fffffd806be189b0,ffff80003c468d08) at closef+0x192 sys/kern/kern_descrip.c:1249 fdfree(ffff80003c468d08) at fdfree+0x116 sys/kern/kern_descrip.c:1181 exit1(ffff80003c468d08,0,0,1) at exit1+0x58f sys/kern/kern_exit.c:214 sys_exit(ffff80003c468d08,ffff80002f7d6b00,ffff80002f7d6a50) at sys_exit+0x1a sys/kern/kern_exit.c:-1 syscall(ffff80002f7d6b00) at syscall+0xb08 mi_syscall sys/sys/syscall_mi.h:176 [inline] syscall(ffff80002f7d6b00) at syscall+0xb08 sys/arch/amd64/amd64/trap.c:577 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0x6fab7c8a6560, count: -17 ddb{1}> show registers rdi 0 rsi 0 rbp 0xffff80002f7d64a0 rbx 0xfffffd8004738c48 rdx 0 rcx 0xffff80003c468d08 rax 0xffff8000299ddff0 r8 0xffff80002f7d6380 r9 0x8080808080808080 r10 0x17317a6c5f8b6cb0 r11 0x2587842f6385fd26 r12 0 r13 0xfffffd8003f48340 r14 0x3 r15 0xffffffff rip 0xffffffff8224c1f5 db_enter+0x25 cs 0x8 rflags 0x246 rsp 0xffff80002f7d6490 ss 0 db_enter+0x25: addq $0x8,%rsp ddb{1}> show proc PROC (syz-executor) tid=112427 pid=11540 tcnt=0 stat=onproc flags process=1008 proc=2000 runpri=32, usrpri=50, slppri=32, nice=20 wchan=0x0, wmesg=, ps_single=0xffff80003c468d08 scnt=-1 ecnt=1 forw=0xffffffffffffffff, list=0xffff80003c43b740,0xffff80003c469c78 process=0xffff80003c4a12c8 user=0xffff80002f7d1000, vmspace=0xfffffd806bece018 estcpu=36, cpticks=1, pctcpu=0.0, user=0, sys=0, intr=0 ddb{1}> ps PID TID PPID UID S FLAGS WAIT COMMAND 93029 231135 33289 0 7 0 syz-executor 93029 128391 33289 0 2 0x4000000 syz-executor 85774 104267 82866 -1 3 0x90 nanoslp syz-executor 85774 461876 82866 -1 3 0x4000090 lockf syz-executor 85774 17008 82866 -1 3 0x4000090 ttyretype syz-executor 85774 76914 82866 -1 3 0x4000090 fsleep syz-executor 43105 296196 95900 0 3 0x80 nanoslp syz-executor 43105 283006 95900 0 3 0x4000080 sbwait syz-executor 43105 468604 95900 0 3 0x4000080 fsleep syz-executor 37939 409355 8291 0 3 0x80 nanoslp syz-executor 37939 3005 8291 0 3 0x4000080 lockf syz-executor 37939 451246 8291 0 3 0x4000080 fsleep syz-executor 95975 264455 11624 0 3 0x80 nanoslp syz-executor 95975 346282 11624 0 3 0x4000080 ttyout syz-executor 95975 141223 11624 0 3 0x4000080 fsleep syz-executor 73328 263465 34245 60929 3 0x90 nanoslp syz-executor 73328 456597 34245 60929 3 0x4000090 ttyin syz-executor 73328 97957 34245 60929 3 0x4000090 fsleep syz-executor 8291 289568 47418 0 3 0x82 nanoslp syz-executor 11624 279218 47418 0 3 0x82 nanoslp syz-executor 15355 208020 1 0 3 0x100083 ttyin getty 12753 230276 0 0 3 0x14280 nfsidl nfsio 45780 390283 0 0 3 0x14280 nfsidl nfsio 54151 368768 0 0 3 0x14280 nfsidl nfsio 34864 75353 0 0 3 0x14280 nfsidl nfsio 78768 118328 0 0 3 0x14280 nfsidl nfsio 84069 518516 0 0 3 0x14280 nfsidl nfsio 97149 111121 0 0 3 0x14280 nfsidl nfsio 38074 438790 0 0 3 0x14280 nfsidl nfsio 93903 52968 0 0 3 0x14280 nfsidl nfsio 96958 355150 0 0 3 0x14280 nfsidl nfsio 47831 412502 0 0 3 0x14280 nfsidl nfsio 42676 258424 0 0 3 0x14280 nfsidl nfsio 68647 506769 0 0 3 0x14280 nfsidl nfsio 2465 313745 0 0 3 0x14280 nfsidl nfsio 57125 199308 0 0 3 0x14280 nfsidl nfsio 78428 294914 0 0 3 0x14280 nfsidl nfsio 61913 409268 0 0 3 0x14280 nfsidl nfsio 40962 54827 0 0 3 0x14280 nfsidl nfsio 92247 321513 0 0 3 0x14280 nfsidl nfsio 65434 249928 0 0 3 0x14280 nfsidl nfsio 40347 276163 47418 0 3 0x82 nanoslp syz-executor 49790 310151 0 0 3 0x14200 bored sosplice 34245 457191 47418 0 3 0x82 nanoslp syz-executor 82866 23169 47418 0 3 0x82 nanoslp syz-executor 74804 222465 47418 0 3 0x82 wait syz-executor 95900 462073 47418 0 3 0x82 nanoslp syz-executor 33289 72637 47418 0 3 0x82 nanoslp syz-executor 47418 98022 38446 0 3 0x82 kqread syz-executor 38446 271984 75097 0 3 0x10008a sigsusp ksh 75097 424587 96488 0 3 0x98 kqread sshd-session 96488 522818 3091 0 3 0x92 kqread sshd-session 3091 422368 1 0 3 0x88 kqread sshd 77513 233509 96052 74 3 0x1100092 bpf pflogd 96052 484283 1 0 3 0x80 sbwait pflogd 74737 176012 91272 73 3 0x1100090 kqread syslogd 91272 439502 1 0 3 0x100082 sbwait syslogd 89852 523226 1 0 3 0x100080 kqread resolvd 72797 96231 87001 77 3 0x100092 kqread dhcpleased 90150 408546 87001 77 3 0x100092 kqread dhcpleased 87001 450389 1 0 3 0x80 kqread dhcpleased 4468 1501 0 0 3 0x14200 bored smr 57868 152479 0 0 3 0x14200 pgzero zerothread 56185 39639 0 0 3 0x14200 aiodoned aiodoned 87728 494331 0 0 3 0x14200 syncer update 28772 101145 0 0 3 0x14200 cleaner cleaner 79624 215608 0 0 3 0x14200 reaper reaper 54594 445620 0 0 3 0x14200 pgdaemon pagedaemon 79435 313387 0 0 3 0x14200 bored viomb 12856 119324 0 0 3 0x40014200 acpi0 acpi0 88850 32513 0 0 3 0x40014200 idle1 99373 121305 0 0 3 0x14200 bored softnet3 87498 69956 0 0 3 0x14200 bored softnet2 74295 290753 0 0 3 0x14200 bored softnet1 76364 501558 0 0 3 0x14200 bored softnet0 83537 237041 0 0 3 0x14200 bored systqmp 9965 408687 0 0 3 0x14200 bored systq 71509 185238 0 0 3 0x14200 tmoslp softclockmp 82917 502259 0 0 3 0x40014200 tmoslp softclock 13325 5982 0 0 3 0x40014200 idle0 1 207150 0 0 3 0x82 wait init 0 0 -1 0 3 0x10200 scheduler swapper ddb{1}> show all locks CPU 1: exclusive mutex &cq->cq_mtx r = 0 (0xffffffff83829df8) #0 witness_lock+0x5bb stacktrace_save sys/sys/stacktrace.h:37 [inline] #0 witness_lock+0x5bb sys/kern/subr_witness.c:1160 #1 mtx_enter_try+0x178 sys/kern/kern_lock.c:-1 #2 mtx_enter+0x60 sys/kern/kern_lock.c:239 #3 clockintr_unbind+0x56 sys/kern/kern_clockintr.c:375 #4 dt_ioctl_record_stop+0xbc sys/dev/dt/dt_dev.c:575 #5 dtclose+0x105 dt_pcb_purge sys/dev/dt/dt_dev.c:-1 [inline] #5 dtclose+0x105 sys/dev/dt/dt_dev.c:232 #6 spec_close+0x45f sys/kern/spec_vnops.c:-1 #7 VOP_CLOSE+0x133 sys/kern/vfs_vops.c:156 #8 vn_closefile+0x12b vn_close sys/kern/vfs_vnops.c:292 [inline] #8 vn_closefile+0x12b sys/kern/vfs_vnops.c:615 #9 fdrop+0x126 sys/kern/kern_descrip.c:1265 #10 closef+0x192 sys/kern/kern_descrip.c:1249 #11 fdfree+0x116 sys/kern/kern_descrip.c:1181 #12 exit1+0x58f sys/kern/kern_exit.c:214 #13 sys_exit+0x1a sys/kern/kern_exit.c:-1 #14 syscall+0xb08 mi_syscall sys/sys/syscall_mi.h:176 [inline] #14 syscall+0xb08 sys/arch/amd64/amd64/trap.c:577 #15 Xsyscall+0x128 ddb{1}> show malloc Type InUse MemUse HighUse Limit Requests Type Lim devbuf 10224 11137K 11503K 166960K 12301 0 pcb 18 14K 16K 166960K 198 0 rtable 218 10K 11K 166960K 569 0 pf 39 18K 82K 166960K 116 0 ifaddr 38 6K 8K 166960K 88 0 ifgroup 59 2K 2K 166960K 139 0 sysctl 4 1K 1K 166960K 4 0 counters 66 36K 37K 166960K 128 0 ioctlops 0 0K 4K 166960K 1557 0 iov 0 0K 24K 166960K 35 0 mount 1 1K 1K 166960K 1 0 log 0 0K 0K 166960K 4 0 vnodes 1397 88K 88K 166960K 1942 0 UFS quota 1 32K 32K 166960K 1 0 UFS mount 5 36K 36K 166960K 5 0 shm 2 1K 5K 166960K 16 0 VM map 2 1K 1K 166960K 2 0 sem 12 0K 1K 166960K 38 0 dirhash 12 2K 2K 166960K 15 0 ACPI 1692 195K 286K 166960K 12470 0 file desc 18 65K 93K 166960K 765 0 sigio 0 0K 0K 166960K 20 0 proc 72 91K 152K 166960K 743 0 subproc 72 4K 4K 166960K 108 0 NFS srvsock 1 0K 0K 166960K 1 0 NFS daemon 1 16K 16K 166960K 1 0 ip_moptions 0 0K 0K 166960K 161 0 in_multi 77 5K 7K 166960K 163 0 ether_multi 1 0K 0K 166960K 4 0 mrt 0 0K 0K 166960K 4 0 ISOFS mount 1 32K 32K 166960K 1 0 MSDOSFS mount 1 16K 16K 166960K 1 0 ttys 97 440K 440K 166960K 97 0 exec 0 0K 1K 166960K 473 0 fusefs mount 1 32K 32K 166960K 1 0 tdb 3 0K 0K 166960K 3 0 VM swap 8 62K 64K 166960K 10 0 UVM amap 246 169K 184K 166960K 8491 0 UVM aobj 22 6K 8K 166960K 27 0 pinsyscall 43 86K 105K 166960K 1976 0 memdesc 1 4K 4K 166960K 1 0 crypto data 1 1K 1K 166960K 1 0 ip6_options 0 0K 0K 166960K 45 0 NDP 13 0K 2K 166960K 57 0 temp 55 8686K 8808K 166960K 15882 0 kqueue 15 22K 28K 166960K 118 0 SYN cache 2 32K 40K 166960K 3 0 ddb{1}> show all pools Name Size Requests Fail Releases Pgreq Pgrel Npage Hiwat Minpg Maxpg Idle plcache 128 26 0 0 1 0 1 1 0 8 0 rtpcb 120 75 0 72 1 0 1 1 0 8 0 rtentry 176 176 0 85 6 0 6 6 0 8 0 unpcb 144 426 0 398 5 3 2 4 0 8 0 syncache 336 6 0 6 2 1 1 1 0 8 1 tcpcb 808 184 0 180 6 2 4 4 0 8 3 arp 128 28 0 14 1 0 1 1 0 8 0 inpcb 384 660 0 650 9 2 7 8 0 8 5 nd6 144 41 0 22 1 0 1 1 0 8 0 pkpcb 40 5 0 5 1 1 0 1 0 8 0 kcovpl 48 12 0 4 1 0 1 1 0 8 0 mppekey 1024 1 0 1 1 1 0 1 0 8 0 ppxss 1192 16 0 16 1 0 1 1 0 8 1 pppxif 1504 6 0 6 2 1 1 1 0 8 1 pffrag 232 1 0 0 1 0 1 1 0 482 0 pffrnode 88 1 0 0 1 0 1 1 0 8 0 pffrent 40 3 0 2 1 0 1 1 0 8 0 pfosfp 40 1428 0 1005 5 0 5 5 0 8 0 pfosfpen 112 1428 0 714 21 0 21 21 0 8 0 pfrktable 1344 4 0 1 1 0 1 1 0 8 0 pfanchor 1288 2 0 0 1 0 1 1 0 8 0 pftag 88 2 0 0 1 0 1 1 0 8 0 pfstitem 24 63 0 17 1 0 1 1 0 8 0 pfstkey 128 63 0 17 2 0 2 2 0 8 0 pfstate 384 62 0 16 6 0 6 6 0 8 0 pfrule 1344 29 0 19 2 1 1 2 0 8 0 art_heap8 4096 3 0 0 3 0 3 3 0 8 0 art_heap4 256 682 0 308 29 0 29 29 0 8 4 art_table 32 685 0 308 4 0 4 4 0 8 0 art_node 16 172 0 89 1 0 1 1 0 8 0 sysvmsgpl 40 13 0 7 1 0 1 1 0 8 0 semupl 112 1 0 1 1 1 0 1 0 8 0 semapl 112 34 0 24 1 0 1 1 0 8 0 shmpl 112 24 0 5 1 0 1 1 0 8 0 dirhash 1024 19 0 2 3 0 3 3 0 8 0 dino2pl 256 2629 0 1112 96 0 96 96 0 8 0 ffsino 288 2629 0 1112 109 0 109 109 0 8 0 nchpl 144 3663 0 1957 64 0 64 64 0 8 0 rtmask 32 4 0 4 2 1 1 1 0 8 1 uvmvnodes 80 3077 0 0 63 0 63 63 0 8 0 vnodes 216 3077 0 0 171 0 171 171 0 8 0 namei 1024 12191 0 12191 2 1 1 2 0 8 1 percpumem 16 79 0 31 1 0 1 1 0 8 0 pfiaddrpl 120 2 0 0 1 0 1 1 0 8 0 kstatmem 264 66 0 40 3 0 3 3 0 8 1 scsiplug 72 2 0 2 2 1 1 1 0 8 1 scxspl 216 13310 0 13310 10 9 1 8 1 8 1 plimitpl 152 103 0 84 1 0 1 1 0 8 0 sigapl 424 1083 0 1012 9 0 9 9 0 8 0 futexpl 64 9324 0 9319 1 0 1 1 0 8 0 knotepl 120 552 0 0 17 0 17 17 0 8 0 kqueuepl 224 329 0 319 7 3 4 5 0 8 3 pipepl 336 178 0 150 3 0 3 3 0 8 0 fdescpl 520 1044 0 1012 3 0 3 3 0 8 0 filepl 160 5566 0 5318 13 1 12 13 0 8 0 lockfpl 104 285 0 276 1 0 1 1 0 8 0 lockfspl 48 78 0 74 1 0 1 1 0 8 0 sessionpl 144 30 0 21 1 0 1 1 0 8 0 pgrppl 48 52 0 35 1 0 1 1 0 8 0 ucredpl 104 879 0 863 1 0 1 1 0 8 0 zombiepl 144 1014 0 1012 1 0 1 1 0 8 0 processpl 1192 1083 0 1012 6 0 6 6 0 8 0 procpl 656 2100 0 2017 8 0 8 8 0 8 0 srpgc 96 12 0 12 1 0 1 1 0 8 1 sosppl 168 6 0 6 1 0 1 1 0 8 1 sockpl 728 1176 0 1135 9 1 8 9 0 8 4 mcl64k 65536 2 0 0 1 0 1 1 0 8 0 mcl16k 16384 1 0 0 1 0 1 1 0 8 0 mcl8k 8192 3 0 0 1 0 1 1 0 8 0 mcl4k 4096 114 0 0 15 0 15 15 0 8 0 mcl2k2 2112 1 0 0 1 0 1 1 0 8 0 mcl2k 2048 33 0 0 5 0 5 5 0 8 0 mtagpl 96 29 0 0 1 0 1 1 0 8 0 mbufpl 256 252 0 0 16 0 16 16 0 8 0 bufpl 280 4306 0 124 299 0 299 299 0 8 0 anonpl 32 11685 0 0 95 0 95 95 0 246 0 amapchunkpl 152 27448 0 26944 30 0 30 30 0 158 7 amappl16 200 3652 0 3615 30 15 15 23 0 8 10 amappl15 192 6 0 6 2 2 0 1 0 8 0 amappl14 184 124 0 112 1 0 1 1 0 8 0 amappl13 176 3 0 3 1 1 0 1 0 8 0 amappl12 168 1755 0 1723 4 2 2 3 0 8 0 amappl11 160 72 0 57 1 0 1 1 0 8 0 amappl10 152 6 0 6 2 2 0 1 0 8 0 amappl9 144 255 0 255 1 1 0 1 0 8 0 amappl8 136 23 0 21 1 0 1 1 0 8 0 amappl7 128 148 0 134 1 0 1 1 0 8 0 amappl6 120 223 0 220 1 0 1 1 0 8 0 amappl5 112 136 0 126 1 0 1 1 0 8 0 amappl4 104 341 0 321 1 0 1 1 0 8 0 amappl3 96 5270 0 5155 4 0 4 4 0 8 0 amappl2 88 741 0 677 2 0 2 2 0 8 0 amappl1 80 11788 0 11172 17 1 16 16 0 8 0 amappl 88 7604 0 7434 5 0 5 5 0 92 0 dma4096 4096 1 0 1 1 1 0 1 0 8 0 dma1024 1024 1 0 0 1 0 1 1 0 8 0 dma256 256 7 0 7 2 2 0 1 0 8 0 dma128 128 254 0 254 2 2 0 1 0 8 0 dma64 64 6 0 6 1 1 0 1 0 8 0 dma32 32 7 0 7 1 1 0 1 0 8 0 dma16 16 18 0 17 1 0 1 1 0 8 0 aobjpl 72 26 0 5 1 0 1 1 0 8 0 uaddrrnd 24 1044 0 1012 1 0 1 1 0 8 0 uaddrbest 32 2 0 0 1 0 1 1 0 8 0 uaddr 24 1044 0 1012 1 0 1 1 0 8 0 vmmpekpl 168 9880 0 9829 3 0 3 3 0 8 0 vmmpepl 168 72855 0 70781 114 2 112 112 0 357 8 vmsppl 480 1043 0 1012 5 0 5 5 0 8 0 rwobjpl 72 24664 0 20612 80 0 80 80 0 8 0 pdppl 4096 2095 0 2024 115 42 73 85 0 8 2 pvpl 32 18602 0 0 153 2 151 151 0 265 0 pmappl 256 1043 0 1012 3 0 3 3 0 8 0 extentpl 40 45 0 27 1 0 1 1 0 8 0 phpool 112 415 0 50 11 0 11 11 0 8 0 ddb{1}> machine ddbcpu 0 Stopped at x86_ipi_db+0x27: addq $0x8,%rsp ddb{0}> trace x86_ipi_db(ffffffff83828ff0) at x86_ipi_db+0x27 sys/arch/amd64/amd64/db_interface.c:394 x86_ipi_handler() at x86_ipi_handler+0xd9 sys/arch/amd64/amd64/ipi.c:106 Xresume_lapic_ipi() at Xresume_lapic_ipi+0x27 mtx_enter(ffffffff83829de8) at mtx_enter+0xa5 sys/kern/kern_lock.c:241 clockintr_dispatch(ffff80003c4d3e60) at clockintr_dispatch+0x3b4 sys/kern/kern_clockintr.c:215 lapic_clockintr(0,0) at lapic_clockintr+0x51 sys/arch/amd64/amd64/lapic.c:489 Xresume_lapic_ltimer() at Xresume_lapic_ltimer+0x2a end of kernel end trace frame: 0x7d3726045730, count: -7 ddb{0}> machine ddbcpu 1 Stopped at db_enter+0x25: addq $0x8,%rsp ddb{1}> trace db_enter() at db_enter+0x25 sys/arch/amd64/amd64/db_interface.c:438 witness_checkorder(ffffffff838eec28,9,0) at witness_checkorder+0x10fb sys/kern/subr_witness.c:-1 mtx_enter(ffffffff838eec18) at mtx_enter+0x47 sys/kern/kern_lock.c:238 sleep_setup(ffffffff83829e48,220,ffffffff83399e6d) at sleep_setup+0xc6 sys/kern/kern_synch.c:348 msleep(ffffffff83829e48,ffffffff83829de8,220,ffffffff83399e6d,0) at msleep+0x114 sys/kern/kern_synch.c:245 dt_ioctl_record_stop(ffff80000146d000) at dt_ioctl_record_stop+0xbc sys/dev/dt/dt_dev.c:575 dtclose(11e5f,81,2000,ffff80003c468d08) at dtclose+0x105 dt_pcb_purge sys/dev/dt/dt_dev.c:-1 [inline] dtclose(11e5f,81,2000,ffff80003c468d08) at dtclose+0x105 sys/dev/dt/dt_dev.c:232 spec_close(ffff80002f7d6790) at spec_close+0x45f sys/kern/spec_vnops.c:-1 VOP_CLOSE(fffffd805d402878,81,fffffd807f7d3138,ffff80003c468d08) at VOP_CLOSE+0x133 sys/kern/vfs_vops.c:156 vn_closefile(fffffd806be189b0,ffff80003c468d08) at vn_closefile+0x12b vn_close sys/kern/vfs_vnops.c:292 [inline] vn_closefile(fffffd806be189b0,ffff80003c468d08) at vn_closefile+0x12b sys/kern/vfs_vnops.c:615 fdrop(fffffd806be189b0,ffff80003c468d08) at fdrop+0x126 sys/kern/kern_descrip.c:1265 closef(fffffd806be189b0,ffff80003c468d08) at closef+0x192 sys/kern/kern_descrip.c:1249 fdfree(ffff80003c468d08) at fdfree+0x116 sys/kern/kern_descrip.c:1181 exit1(ffff80003c468d08,0,0,1) at exit1+0x58f sys/kern/kern_exit.c:214 sys_exit(ffff80003c468d08,ffff80002f7d6b00,ffff80002f7d6a50) at sys_exit+0x1a sys/kern/kern_exit.c:-1 syscall(ffff80002f7d6b00) at syscall+0xb08 mi_syscall sys/sys/syscall_mi.h:176 [inline] syscall(ffff80002f7d6b00) at syscall+0xb08 sys/arch/amd64/amd64/trap.c:577 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0x6fab7c8a6560, count: -17