login: witness: lock order reversal:
1st 0xffffffff83861e30 &cq->cq_mtx (&cq->cq_mtx)
2nd 0xffffffff8399fb88 &sched_lock (&sched_lock)
lock order [1] &cq->cq_mtx (&cq->cq_mtx) -> [2] &sched_lock (&sched_lock)
#0 mtx_enter+0x95 sys/kern/kern_lock.c:406
#1 sleep_setup+0xed sys/kern/kern_synch.c:304
#2 msleep_nsec+0x10b sys/kern/kern_synch.c:217
#3 dt_ioctl_record_stop+0xbc sys/dev/dt/dt_dev.c:589
#4 dtclose+0x109 dt_pcb_purge sys/dev/dt/dt_dev.c:-1 [inline]
#4 dtclose+0x109 sys/dev/dt/dt_dev.c:239
#5 spec_close+0x466 sys/kern/spec_vnops.c:-1
#6 VOP_CLOSE+0x132 sys/kern/vfs_vops.c:156
#7 vn_closefile+0x12b vn_close sys/kern/vfs_vnops.c:292 [inline]
#7 vn_closefile+0x12b sys/kern/vfs_vnops.c:615
#8 fdrop+0x121 sys/kern/kern_descrip.c:1280
#9 closef+0x192 sys/kern/kern_descrip.c:1264
#10 fdfree+0x116 sys/kern/kern_descrip.c:1195
#11 exit1+0x576 sys/kern/kern_exit.c:215
#12 sys_exit+0x1a sys/kern/kern_exit.c:-1
#13 syscall+0xb17 mi_syscall sys/sys/syscall_mi.h:176 [inline]
#13 syscall+0xb17 sys/arch/amd64/amd64/trap.c:775
#14 Xsyscall+0x128
lock order [2] &sched_lock (&sched_lock) -> [1] &cq->cq_mtx (&cq->cq_mtx)
#0 mtx_enter+0x95 sys/kern/kern_lock.c:406
#1 clockintr_cancel+0x2f sys/kern/kern_clockintr.c:323
#2 mi_switch+0x187 sys/kern/sched_bsd.c:376
#3 sleep_finish+0x2d8 sys/kern/kern_synch.c:369
#4 sys_futex+0xb90 futex_wait sys/kern/sys_futex.c:306 [inline]
#4 sys_futex+0xb90 sys/kern/sys_futex.c:155
#5 syscall+0xbd4 mi_syscall sys/sys/syscall_mi.h:176 [inline]
#5 syscall+0xbd4 sys/arch/amd64/amd64/trap.c:775
#6 Xsyscall+0x128
Stopped at db_enter+0x25: addq $0x8,%rsp
ddb{1}>
ddb{1}> set $lines = 0
ddb{1}> set $maxwidth = 0
ddb{1}> show panic
the kernel did not panic
ddb{1}> trace
db_enter() at db_enter+0x25 sys/arch/amd64/amd64/db_interface.c:438
witness_checkorder(ffffffff8399fb88,9,0) at witness_checkorder+0x10d1 sys/kern/subr_witness.c:-1
mtx_enter(ffffffff8399fb78) at mtx_enter+0x95 sys/kern/kern_lock.c:406
sleep_setup(ffffffff83861e80,220,ffffffff83434290) at sleep_setup+0xed sys/kern/kern_synch.c:304
msleep_nsec(ffffffff83861e80,ffffffff83861e20,220,ffffffff83434290,ffffffffffffffff) at msleep_nsec+0x10b sys/kern/kern_synch.c:217
dt_ioctl_record_stop(ffff800001689000) at dt_ioctl_record_stop+0xbc sys/dev/dt/dt_dev.c:589
dtclose(11e5f,81,2000,ffff80002f3cfcc8) at dtclose+0x109 dt_pcb_purge sys/dev/dt/dt_dev.c:-1 [inline]
dtclose(11e5f,81,2000,ffff80002f3cfcc8) at dtclose+0x109 sys/dev/dt/dt_dev.c:239
spec_close(ffff80003c4e1a50) at spec_close+0x466 sys/kern/spec_vnops.c:-1
VOP_CLOSE(fffffd805b26fcc8,81,fffffd80097fd3a8,ffff80002f3cfcc8) at VOP_CLOSE+0x132 sys/kern/vfs_vops.c:156
vn_closefile(fffffd8063dad068,ffff80002f3cfcc8) at vn_closefile+0x12b vn_close sys/kern/vfs_vnops.c:292 [inline]
vn_closefile(fffffd8063dad068,ffff80002f3cfcc8) at vn_closefile+0x12b sys/kern/vfs_vnops.c:615
fdrop(fffffd8063dad068,ffff80002f3cfcc8) at fdrop+0x121 sys/kern/kern_descrip.c:1280
closef(fffffd8063dad068,ffff80002f3cfcc8) at closef+0x192 sys/kern/kern_descrip.c:1264
fdfree(ffff80002f3cfcc8) at fdfree+0x116 sys/kern/kern_descrip.c:1195
exit1(ffff80002f3cfcc8,0,0,1) at exit1+0x576 sys/kern/kern_exit.c:215
sys_exit(ffff80002f3cfcc8,ffff80003c4e1dc0,ffff80003c4e1d10) at sys_exit+0x1a sys/kern/kern_exit.c:-1
syscall(ffff80003c4e1dc0) at syscall+0xb17 mi_syscall sys/sys/syscall_mi.h:176 [inline]
syscall(ffff80003c4e1dc0) at syscall+0xb17 sys/arch/amd64/amd64/trap.c:775
Xsyscall() at Xsyscall+0x128
end of kernel
end trace frame: 0x78b9f1790ba0, count: -17
ddb{1}> show registers
rdi 0
rsi 0
rbp 0xffff80003c4e1720
rbx 0
rdx 0
rcx 0xffff80002f3cfcc8
rax 0xffff8000299ddff0
r8 0xffff80003c4e1600
r9 0x8080808080808080
r10 0x24d576e5b34ab1a8
r11 0x9c678a37278e5881
r12 0xfffffd8003f4de40
r13 0xfffffd800473bd58
r14 0x3
r15 0xffffffff8341707e substchar+0x63e6
rip 0xffffffff81677ba5 db_enter+0x25
cs 0x8
rflags 0x246
rsp 0xffff80003c4e1710
ss 0x10
db_enter+0x25: addq $0x8,%rsp
ddb{1}> show proc
PROC (syz-executor) tid=71060 pid=3945 tcnt=0 stat=onproc
flags process=1008<EXITING,SINGLEEXIT> proc=2000<WEXIT>
runpri=32, usrpri=86, slppri=32, nice=20
wchan=0x0, wmesg=, ps_single=0xffff80002f3cfcc8 scnt=-1 ecnt=1
forw=0xffffffffffffffff, list=0xffff800032fdbcd0,0xffffffff839524c0
process=0xffff80003c4cd840 user=0xffff80003c4dc000, vmspace=0xfffffd806eac71f8
estcpu=36, cpticks=1, pctcpu=0.0, user=0, sys=0, intr=0
ddb{1}> ps
PID TID PPID UID S FLAGS WAIT COMMAND
30597 377555 25149 60929 3 0x90 nanoslp syz-executor
30597 177207 25149 60929 3 0x4000090 ttyout syz-executor
30597 510753 25149 60929 3 0x4000090 fsleep syz-executor
25553 89649 70884 0 3 0x80 nanoslp syz-executor
25553 402322 70884 0 3 0x4000080 wsevent_read syz-executor
25553 328975 70884 0 3 0x4000080 fsleep syz-executor
8721 25700 88861 0 3 0x80 nanoslp syz-executor
8721 310512 88861 0 3 0x4000080 kqsel syz-executor
8721 240285 88861 0 3 0x4000080 fsleep syz-executor
81442 153930 54915 0 3 0x3000 suspend syz-executor
81442 270066 54915 0 3 0x4081000 biowait syz-executor
81442 440577 54915 0 3 0x4081000 inode syz-executor
81442 447304 54915 0 3 0x4081000 inode syz-executor
47252 238913 1 0 3 0x100083 ttyopn getty
59050 321690 0 0 3 0x14200 acct acct
70884 17800 6507 0 3 0x82 nanoslp syz-executor
25149 124743 6507 0 3 0x82 nanoslp syz-executor
3218 376357 6507 0 3 0x82 wait syz-executor
55493 467570 6507 0 3 0x82 wait syz-executor
54915 17916 6507 0 3 0x82 nanoslp syz-executor
5946 40520 6507 0 3 0x82 wait syz-executor
88861 422913 6507 0 3 0x82 nanoslp syz-executor
57442 38216 47261 0 3 0x100082 sbwait arp
47261 262882 1 0 3 0x10008a sigsusp sh
82028 282325 6507 0 3 0x82 nanoslp syz-executor
6507 508631 20534 0 3 0x82 kqread syz-executor
20534 165033 95447 0 3 0x10008a sigsusp ksh
95447 378254 77175 0 3 0x98 kqread sshd-session
77175 318817 41021 0 3 0x92 kqread sshd-session
41021 2641 1 0 3 0x88 kqread sshd
30576 384195 18805 74 3 0x1100092 bpf pflogd
18805 444907 1 0 3 0x80 sbwait pflogd
69856 493512 98791 73 3 0x1100090 kqread syslogd
98791 409546 1 0 3 0x100082 sbwait syslogd
42960 166838 1 0 3 0x100080 kqread resolvd
92439 405784 70314 77 3 0x100092 kqread dhcpleased
74820 170971 70314 77 3 0x100092 kqread dhcpleased
70314 169374 1 0 3 0x80 kqread dhcpleased
63912 224183 0 0 3 0x14200 bored smr
37574 402764 0 0 3 0x14200 pgzero zerothread
79546 41002 0 0 3 0x14200 aiodoned aiodoned
43600 248176 0 0 3 0x14200 syncer update
9635 302717 0 0 3 0x14200 cleaner cleaner
51138 7711 0 0 2 0x14200 reaper
68122 129354 0 0 3 0x14200 pgdaemon pagedaemon
96143 503999 0 0 3 0x14200 bored viomb
31183 312471 0 0 3 0x40014200 acpi0 acpi0
37332 465747 0 0 3 0x40014200 idle1
71957 102964 0 0 3 0x14200 bored softnet1
94354 407216 0 0 3 0x14200 bored softnet0
18001 331256 0 0 3 0x14200 smrbar systqmp
11082 244052 0 0 3 0x14200 bored systq
7435 11933 0 0 3 0x14200 tmoslp softclockmp
53410 406110 0 0 3 0x40014200 tmoslp softclock
52262 274426 0 0 7 0x40014200 idle0
1 462260 0 0 3 0x80082 wait init
0 0 -1 0 3 0x10010200 scheduler swapper
ddb{1}> show all locks
CPU 1:
exclusive mutex &cq->cq_mtx r = 0 (0xffffffff83861e30)
#0 witness_lock+0x5f1 stacktrace_save sys/sys/stacktrace.h:37 [inline]
#0 witness_lock+0x5f1 sys/kern/subr_witness.c:1160
#1 mtx_enter+0x4b4 sys/kern/kern_lock.c:487
#2 clockintr_unbind+0x56 sys/kern/kern_clockintr.c:375
#3 dt_ioctl_record_stop+0xbc sys/dev/dt/dt_dev.c:589
#4 dtclose+0x109 dt_pcb_purge sys/dev/dt/dt_dev.c:-1 [inline]
#4 dtclose+0x109 sys/dev/dt/dt_dev.c:239
#5 spec_close+0x466 sys/kern/spec_vnops.c:-1
#6 VOP_CLOSE+0x132 sys/kern/vfs_vops.c:156
#7 vn_closefile+0x12b vn_close sys/kern/vfs_vnops.c:292 [inline]
#7 vn_closefile+0x12b sys/kern/vfs_vnops.c:615
#8 fdrop+0x121 sys/kern/kern_descrip.c:1280
#9 closef+0x192 sys/kern/kern_descrip.c:1264
#10 fdfree+0x116 sys/kern/kern_descrip.c:1195
#11 exit1+0x576 sys/kern/kern_exit.c:215
#12 sys_exit+0x1a sys/kern/kern_exit.c:-1
#13 syscall+0xb17 mi_syscall sys/sys/syscall_mi.h:176 [inline]
#13 syscall+0xb17 sys/arch/amd64/amd64/trap.c:775
#14 Xsyscall+0x128
Process 81442 (syz-executor) thread 0xffff80002f3ce2d8 (270066)
exclusive rrwlock inode r = 0 (0xfffffd8058dd8588)
#0 witness_lock+0x5f1 stacktrace_save sys/sys/stacktrace.h:37 [inline]
#0 witness_lock+0x5f1 sys/kern/subr_witness.c:1160
#1 rw_do_enter_write+0x419 sys/kern/kern_rwlock.c:320
#2 rrw_enter+0xc6 sys/kern/kern_rwlock.c:621
#3 VOP_LOCK+0xbd sys/kern/vfs_vops.c:527
#4 vn_lock+0xa4 sys/kern/vfs_vnops.c:570
#5 vn_write+0x18f sys/kern/vfs_vnops.c:405
#6 dofilewritev+0x242 sys/kern/sys_generic.c:380
#7 sys_write+0xa2 sys/kern/sys_generic.c:300
#8 syscall+0xbd4 mi_syscall sys/sys/syscall_mi.h:176 [inline]
#8 syscall+0xbd4 sys/arch/amd64/amd64/trap.c:775
#9 Xsyscall+0x128
Process 81442 (syz-executor) thread 0xffff80002f3ce040 (440577)
exclusive rrwlock inode r = 0 (0xfffffd806e7bb340)
#0 witness_lock+0x5f1 stacktrace_save sys/sys/stacktrace.h:37 [inline]
#0 witness_lock+0x5f1 sys/kern/subr_witness.c:1160
#1 rw_do_enter_write+0x419 sys/kern/kern_rwlock.c:320
#2 rrw_enter+0xc6 sys/kern/kern_rwlock.c:621
#3 VOP_LOCK+0xbd sys/kern/vfs_vops.c:527
#4 vn_lock+0xa4 sys/kern/vfs_vnops.c:570
#5 vfs_lookup+0x11c sys/kern/vfs_lookup.c:-1
#6 namei+0x7ca sys/kern/vfs_lookup.c:250
#7 domknodat+0xb4 sys/kern/vfs_syscalls.c:1592
#8 syscall+0xb17 mi_syscall sys/sys/syscall_mi.h:176 [inline]
#8 syscall+0xb17 sys/arch/amd64/amd64/trap.c:775
#9 Xsyscall+0x128
Process 18001 (systqmp) thread 0xffff8000ffffe298 (331256)
shared rwlock systqmp r = 0 (0xffffffff838a7d18)
#0 witness_lock+0x5f1 stacktrace_save sys/sys/stacktrace.h:37 [inline]
#0 witness_lock+0x5f1 sys/kern/subr_witness.c:1160
#1 taskq_thread+0x12a sys/kern/kern_task.c:442
#2 proc_trampoline+0x10
ddb{1}> show malloc
Type InUse MemUse HighUse Limit Requests Type Lim
devbuf 11103 12148K 14040K 166960K 17041 0
pcb 17 18K 26K 166960K 1398 0
rtable 241 11K 12K 166960K 1193 0
pf 40 19K 82K 166960K 377 0
ifaddr 40 8K 11K 166960K 246 0
ifgroup 58 2K 3K 166960K 473 0
sysctl 4 1K 9K 166960K 26 0
counters 70 37K 38K 166960K 524 0
ioctlops 0 0K 4K 166960K 2670 0
iov 0 0K 28K 166960K 392 0
mount 1 1K 1K 166960K 1 0
log 0 0K 0K 166960K 4 0
vnodes 1525 96K 97K 166960K 5334 0
UFS quota 1 32K 32K 166960K 1 0
UFS mount 5 36K 36K 166960K 5 0
shm 2 2K 10K 166960K 43 0
VM map 2 1K 1K 166960K 2 0
sem 12 0K 1K 166960K 256 0
dirhash 12 2K 2K 166960K 99 0
ACPI 1692 195K 286K 166960K 12470 0
file desc 20 73K 244K 166960K 4099 0
sigio 0 0K 0K 166960K 127 0
proc 73 115K 212K 166960K 1251 0
subproc 81 5K 5K 166960K 181 0
NFS srvsock 1 0K 0K 166960K 1 0
NFS daemon 1 16K 16K 166960K 1 0
ip_moptions 0 0K 0K 166960K 586 0
in_multi 85 6K 7K 166960K 332 0
ether_multi 1 0K 0K 166960K 40 0
mrt 1 0K 0K 166960K 36 0
ISOFS mount 1 32K 32K 166960K 1 0
MSDOSFS mount 1 16K 16K 166960K 1 0
ttys 247 1102K 1102K 166960K 247 0
exec 0 0K 1K 166960K 1960 0
fusefs mount 1 32K 32K 166960K 1 0
pfkey data 0 0K 0K 166960K 11 0
tdb 3 0K 0K 166960K 3 0
VM swap 8 62K 64K 166960K 10 0
UVM amap 280 164K 187K 166960K 38769 0
UVM aobj 65 51K 51K 166960K 75 0
pinsyscall 46 92K 104K 166960K 5574 0
memdesc 1 4K 4K 166960K 1 0
crypto data 1 1K 1K 166960K 1 0
ip6_options 0 0K 1K 166960K 497 0
NDP 12 0K 1K 166960K 180 0
temp 85 8680K 8935K 166960K 236245 0
kqueue 15 24K 36K 166960K 831 0
SYN cache 2 16K 16K 166960K 2 0
ddb{1}> show all pools
Name Size Requests Fail Releases Pgreq Pgrel Npage Hiwat Minpg Maxpg Idle
plcache 128 26 0 0 1 0 1 1 0 8 0
rtpcb 120 404 0 400 3 2 1 3 0 8 0
rtentry 176 320 0 223 6 0 6 6 0 8 0
unpcb 144 3513 0 3494 27 25 2 9 0 8 1
syncache 336 24 0 24 5 4 1 1 0 8 1
tcpqe 32 50 0 50 5 5 0 1 0 8 0
tcpcb 736 1705 0 1701 38 31 7 13 0 8 6
arp 136 51 0 33 1 0 1 1 0 8 0
inpcb 328 5384 0 5374 57 50 7 15 0 8 5
nd6 152 70 0 47 2 0 2 2 0 8 0
pkpcb 40 25 0 25 9 9 0 1 0 8 0
kcovpl 48 20 0 11 1 0 1 1 0 8 0
mppekey 1024 5 0 5 5 4 1 1 0 8 1
ppxss 1192 168 0 167 1 0 1 1 0 8 0
pppxif 1504 23 0 23 9 8 1 1 0 8 1
pffrag 232 35 0 25 1 0 1 1 0 482 0
pffrnode 88 28 0 20 1 0 1 1 0 8 0
pffrent 40 64 0 53 1 0 1 1 0 8 0
pfosfp 40 1428 0 1005 5 0 5 5 0 8 0
pfosfpen 112 1428 0 714 21 0 21 21 0 8 0
pfrktable 1344 1 0 1 1 1 0 1 0 8 0
pfstitem 24 7 0 1 1 0 1 1 0 8 0
pfstkey 128 139 0 134 1 0 1 1 0 8 0
pfstate 448 70 0 67 1 0 1 1 0 8 0
pfrule 1360 3 0 3 2 2 0 1 0 8 0
rttmr 136 3 0 3 3 3 0 1 0 8 0
art_heap8 4096 5 0 0 5 0 5 5 0 8 0
art_heap4 256 1353 0 926 46 15 31 33 0 8 0
art_table 40 1358 0 926 6 0 6 6 0 8 0
art_node 32 316 0 226 1 0 1 1 0 8 0
sysvmsgpl 40 120 0 110 2 1 1 1 0 8 0
semupl 112 3 0 3 3 2 1 1 0 8 1
semapl 112 243 0 233 1 0 1 1 0 8 0
shmpl 112 41 0 3 2 0 2 2 0 8 0
dirhash 1024 76 0 59 3 0 3 3 0 8 0
dino2pl 256 9612 0 8088 96 0 96 96 0 8 0
ffsino 296 9612 0 8088 118 0 118 118 0 8 0
nchpl 144 15141 0 13404 65 0 65 65 0 8 0
rtmask 32 26 0 26 8 7 1 1 0 8 1
vnodes 216 5926 0 0 330 0 330 330 0 8 0
namei 1024 57793 0 57792 4 3 1 2 0 8 0
percpumem 16 277 0 227 1 0 1 1 0 8 0
vcpupl 3968 25 0 1 3 0 3 3 0 8 0
vmpool 848 31 0 7 3 0 3 3 0 8 0
kstatmem 264 284 0 258 7 4 3 3 0 8 0
scsiplug 72 14 0 14 6 5 1 1 0 8 1
scxspl 216 123071 0 123070 19 16 3 8 1 8 2
plimitpl 152 1114 0 1093 1 0 1 1 0 8 0
sigapl 424 4423 0 4372 13 5 8 9 0 8 0
knotepl 120 1122 0 0 31 0 31 31 0 8 0
kqueuepl 224 1497 0 1485 16 14 2 4 0 8 1
pipepl 344 662 0 632 9 5 4 6 0 8 0
fdescpl 528 4351 0 4317 3 0 3 3 0 8 0
filepl 160 33094 0 32852 51 34 17 23 0 8 3
lockfpl 104 2093 0 2091 6 5 1 3 0 8 0
lockfspl 48 789 0 787 1 0 1 1 0 8 0
sessionpl 144 50 0 41 1 0 1 1 0 8 0
pgrppl 48 286 0 268 1 0 1 1 0 8 0
ucredpl 104 5575 0 5560 1 0 1 1 0 8 0
zombiepl 144 4864 0 4859 1 0 1 1 0 8 0
processpl 1232 4423 0 4372 7 1 6 6 0 8 0
procpl 664 11111 0 11050 10 2 8 8 0 8 0
sosppl 176 32 0 31 6 5 1 1 0 8 0
sockpl 752 9569 0 9536 86 75 11 25 0 8 7
mcl64k 65536 11 0 0 2 0 2 2 0 8 0
mcl16k 16384 4 0 0 1 0 1 1 0 8 0
mcl12k 12288 1 0 0 1 0 1 1 0 8 0
mcl9k 9216 2 0 0 1 0 1 1 0 8 0
mcl8k 8192 6 0 0 1 0 1 1 0 8 0
mcl4k 4096 146 0 0 18 0 18 18 0 8 0
mcl2k 2048 106 0 0 8 1 7 8 0 8 0
mtagpl 96 10 0 0 1 0 1 1 0 8 0
mbufpl 256 925 0 0 50 0 50 50 0 8 0
bufpl 280 51471 0 45335 439 0 439 439 0 8 0
anonpl 32 18895 0 0 152 0 152 152 0 246 0
amapchunkpl 152 138278 0 137634 97 60 37 48 0 158 11
amappl16 200 16980 0 16656 120 82 38 46 0 8 8
amappl15 192 4 0 4 2 2 0 1 0 8 0
amappl14 184 16 0 16 2 1 1 1 0 8 1
amappl13 176 568 0 566 1 0 1 1 0 8 0
amappl12 168 4833 0 4787 3 0 3 3 0 8 0
amappl11 160 10 0 10 3 3 0 1 0 8 0
amappl10 152 49 0 35 1 0 1 1 0 8 0
amappl9 144 275 0 275 1 1 0 1 0 8 0
amappl8 136 32 0 29 1 0 1 1 0 8 0
amappl7 128 133 0 130 1 0 1 1 0 8 0
amappl6 120 454 0 437 1 0 1 1 0 8 0
amappl5 112 90 0 79 1 0 1 1 0 8 0
amappl4 104 572 0 539 1 0 1 1 0 8 0
amappl3 96 24033 0 23922 5 1 4 4 0 8 0
amappl2 88 4539 0 4459 2 0 2 2 0 8 0
amappl1 80 29671 0 29038 16 0 16 16 0 8 0
amappl 88 37126 0 36930 7 1 6 6 0 92 0
uvmvnodes 80 217 0 0 5 0 5 5 0 8 0
dma65536 65536 3 0 3 2 2 0 1 0 8 0
dma4096 4096 1 0 1 1 1 0 1 0 8 0
dma1024 1024 2 0 1 1 0 1 1 0 8 0
dma512 512 3 0 3 3 2 1 1 0 8 1
dma256 256 9 0 9 3 3 0 1 0 8 0
dma128 128 256 0 256 3 3 0 1 0 8 0
dma64 64 8 0 8 2 2 0 1 0 8 0
dma32 32 7 0 7 1 1 0 1 0 8 0
dma16 16 20 0 19 1 0 1 1 0 8 0
aobjpl 72 74 0 10 2 0 2 2 0 8 0
uaddrrnd 24 4352 0 4318 1 0 1 1 0 8 0
uaddrbest 32 2 0 0 1 0 1 1 0 8 0
uaddr 24 4352 0 4318 1 0 1 1 0 8 0
vmmpekpl 168 33652 0 33594 4 1 3 4 0 8 0
vmmpepl 168 284085 0 281680 180 57 123 130 0 357 3
vmsppl 488 4351 0 4318 5 0 5 5 0 8 0
rwobjpl 80 75067 0 73473 53 11 42 44 0 8 0
pdppl 4096 8773 0 8674 173 74 99 100 0 8 0
pvpl 32 26399 0 0 213 0 213 213 0 265 0
pmappl 256 4382 0 4325 4 0 4 4 0 8 0
extentpl 40 45 0 27 1 0 1 1 0 8 0
phpool 112 472 0 142 10 0 10 10 0 8 0
ddb{1}> machine ddbcpu 0
Stopped at x86_ipi_db+0x27: addq $0x8,%rsp
ddb{0}> trace
x86_ipi_db(ffffffff83860ff0) at x86_ipi_db+0x27 sys/arch/amd64/amd64/db_interface.c:394
x86_ipi_handler() at x86_ipi_handler+0xd9 sys/arch/amd64/amd64/ipi.c:106
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x27
__sanitizer_cov_trace_pc() at __sanitizer_cov_trace_pc+0x33 kd_curproc sys/dev/kcov.c:585 [inline]
__sanitizer_cov_trace_pc() at __sanitizer_cov_trace_pc+0x33 sys/dev/kcov.c:153
mtx_enter(ffffffff83861e20) at mtx_enter+0x375 sys/kern/kern_lock.c:455
clockintr_dispatch(ffff80002a1f24c0) at clockintr_dispatch+0x401 sys/kern/kern_clockintr.c:215
lapic_clockintr(0,0) at lapic_clockintr+0x51 sys/arch/amd64/amd64/lapic.c:482
Xresume_lapic_ltimer() at Xresume_lapic_ltimer+0x2a
acpicpu_idle() at acpicpu_idle+0x457 sys/dev/acpi/acpicpu_x86.c:1224
sched_idle(ffffffff83860ff0) at sched_idle+0x391 sys/kern/kern_sched.c:191
end trace frame: 0x0, count: -10
ddb{0}> machine ddbcpu 1
Stopped at db_enter+0x25: addq $0x8,%rsp
ddb{1}> trace
db_enter() at db_enter+0x25 sys/arch/amd64/amd64/db_interface.c:438
witness_checkorder(ffffffff8399fb88,9,0) at witness_checkorder+0x10d1 sys/kern/subr_witness.c:-1
mtx_enter(ffffffff8399fb78) at mtx_enter+0x95 sys/kern/kern_lock.c:406
sleep_setup(ffffffff83861e80,220,ffffffff83434290) at sleep_setup+0xed sys/kern/kern_synch.c:304
msleep_nsec(ffffffff83861e80,ffffffff83861e20,220,ffffffff83434290,ffffffffffffffff) at msleep_nsec+0x10b sys/kern/kern_synch.c:217
dt_ioctl_record_stop(ffff800001689000) at dt_ioctl_record_stop+0xbc sys/dev/dt/dt_dev.c:589
dtclose(11e5f,81,2000,ffff80002f3cfcc8) at dtclose+0x109 dt_pcb_purge sys/dev/dt/dt_dev.c:-1 [inline]
dtclose(11e5f,81,2000,ffff80002f3cfcc8) at dtclose+0x109 sys/dev/dt/dt_dev.c:239
spec_close(ffff80003c4e1a50) at spec_close+0x466 sys/kern/spec_vnops.c:-1
VOP_CLOSE(fffffd805b26fcc8,81,fffffd80097fd3a8,ffff80002f3cfcc8) at VOP_CLOSE+0x132 sys/kern/vfs_vops.c:156
vn_closefile(fffffd8063dad068,ffff80002f3cfcc8) at vn_closefile+0x12b vn_close sys/kern/vfs_vnops.c:292 [inline]
vn_closefile(fffffd8063dad068,ffff80002f3cfcc8) at vn_closefile+0x12b sys/kern/vfs_vnops.c:615
fdrop(fffffd8063dad068,ffff80002f3cfcc8) at fdrop+0x121 sys/kern/kern_descrip.c:1280
closef(fffffd8063dad068,ffff80002f3cfcc8) at closef+0x192 sys/kern/kern_descrip.c:1264
fdfree(ffff80002f3cfcc8) at fdfree+0x116 sys/kern/kern_descrip.c:1195
exit1(ffff80002f3cfcc8,0,0,1) at exit1+0x576 sys/kern/kern_exit.c:215
sys_exit(ffff80002f3cfcc8,ffff80003c4e1dc0,ffff80003c4e1d10) at sys_exit+0x1a sys/kern/kern_exit.c:-1
syscall(ffff80003c4e1dc0) at syscall+0xb17 mi_syscall sys/sys/syscall_mi.h:176 [inline]
syscall(ffff80003c4e1dc0) at syscall+0xb17 sys/arch/amd64/amd64/trap.c:775
Xsyscall() at Xsyscall+0x128
end of kernel
end trace frame: 0x78b9f1790ba0, count: -17