witness: lock order reversal:
1st 0xffffffff837f9e08 &cq->cq_mtx (&cq->cq_mtx)
2nd 0xffffffff83902fe8 &sched_lock (&sched_lock)
lock order [1] &cq->cq_mtx (&cq->cq_mtx) -> [2] &sched_lock (&sched_lock)
#0 mtx_enter+0x4a sys/kern/kern_lock.c:260
#1 sleep_setup+0xc6 sys/kern/kern_synch.c:302
#2 msleep_nsec+0x10b sys/kern/kern_synch.c:217
#3 dt_ioctl_record_stop+0xbc sys/dev/dt/dt_dev.c:582
#4 dtclose+0x109 dt_pcb_purge sys/dev/dt/dt_dev.c:-1 [inline]
#4 dtclose+0x109 sys/dev/dt/dt_dev.c:232
#5 spec_close+0x466 sys/kern/spec_vnops.c:-1
#6 VOP_CLOSE+0x132 sys/kern/vfs_vops.c:156
#7 vn_closefile+0x12b vn_close sys/kern/vfs_vnops.c:292 [inline]
#7 vn_closefile+0x12b sys/kern/vfs_vnops.c:615
#8 fdrop+0x121 sys/kern/kern_descrip.c:1280
#9 closef+0x192 sys/kern/kern_descrip.c:1264
#10 syscall+0xbd4 mi_syscall sys/sys/syscall_mi.h:176 [inline]
#10 syscall+0xbd4 sys/arch/amd64/amd64/trap.c:746
#11 Xsyscall+0x128
lock order [2] &sched_lock (&sched_lock) -> [1] &cq->cq_mtx (&cq->cq_mtx)
#0 mtx_enter+0x4a sys/kern/kern_lock.c:260
#1 clockintr_cancel+0x2f sys/kern/kern_clockintr.c:323
#2 mi_switch+0x16f sys/kern/sched_bsd.c:378
#3 sleep_finish+0x2d8 sys/kern/kern_synch.c:367
#4 sys_futex+0xb90 futex_wait sys/kern/sys_futex.c:306 [inline]
#4 sys_futex+0xb90 sys/kern/sys_futex.c:155
#5 syscall+0xbd4 mi_syscall sys/sys/syscall_mi.h:176 [inline]
#5 syscall+0xbd4 sys/arch/amd64/amd64/trap.c:746
#6 Xsyscall+0x128
Stopped at db_enter+0x25: addq $0x8,%rsp
ddb{1}>
ddb{1}> set $lines = 0
ddb{1}> set $maxwidth = 0
ddb{1}> show panic
the kernel did not panic
ddb{1}> trace
db_enter() at db_enter+0x25 sys/arch/amd64/amd64/db_interface.c:438
witness_checkorder(ffffffff83902fe8,9,0) at witness_checkorder+0x10b9 sys/kern/subr_witness.c:-1
mtx_enter(ffffffff83902fd8) at mtx_enter+0x4a sys/kern/kern_lock.c:260
sleep_setup(ffffffff837f9e58,220,ffffffff833f2a81) at sleep_setup+0xc6 sys/kern/kern_synch.c:302
msleep_nsec(ffffffff837f9e58,ffffffff837f9df8,220,ffffffff833f2a81,ffffffffffffffff) at msleep_nsec+0x10b sys/kern/kern_synch.c:217
dt_ioctl_record_stop(ffff800001604000) at dt_ioctl_record_stop+0xbc sys/dev/dt/dt_dev.c:582
dtclose(11e5f,81,2000,ffff800037009ca8) at dtclose+0x109 dt_pcb_purge sys/dev/dt/dt_dev.c:-1 [inline]
dtclose(11e5f,81,2000,ffff800037009ca8) at dtclose+0x109 sys/dev/dt/dt_dev.c:232
spec_close(ffff80003c477770) at spec_close+0x466 sys/kern/spec_vnops.c:-1
VOP_CLOSE(fffffd80629ca710,81,fffffd80097fb618,ffff800037009ca8) at VOP_CLOSE+0x132 sys/kern/vfs_vops.c:156
vn_closefile(fffffd80654dbb00,ffff800037009ca8) at vn_closefile+0x12b vn_close sys/kern/vfs_vnops.c:292 [inline]
vn_closefile(fffffd80654dbb00,ffff800037009ca8) at vn_closefile+0x12b sys/kern/vfs_vnops.c:615
fdrop(fffffd80654dbb00,ffff800037009ca8) at fdrop+0x121 sys/kern/kern_descrip.c:1280
closef(fffffd80654dbb00,ffff800037009ca8) at closef+0x192 sys/kern/kern_descrip.c:1264
syscall(ffff80003c4779d0) at syscall+0xbd4 mi_syscall sys/sys/syscall_mi.h:176 [inline]
syscall(ffff80003c4779d0) at syscall+0xbd4 sys/arch/amd64/amd64/trap.c:746
Xsyscall() at Xsyscall+0x128
end of kernel
end trace frame: 0x9b041331680, count: -14
ddb{1}> show registers
rdi 0
rsi 0x80000 acpi_pdirpa+0x6be71
rbp 0xffff80003c477480
rbx 0
rdx 0xffff80000147c600
rcx 0xffff800037009ca8
rax 0x7ffff acpi_pdirpa+0x6be70
r8 0xffff80003c477360
r9 0x8080808080808080
r10 0xff106ce3393712fb
r11 0x5d8abbb150cac7a6
r12 0xfffffd8003efcdc0
r13 0xfffffd80046eaa28
r14 0x3
r15 0xffffffff833e3067 substchar+0x1221e
rip 0xffffffff8324dbb5 db_enter+0x25
cs 0x8
rflags 0x246
rsp 0xffff80003c477470
ss 0x10
db_enter+0x25: addq $0x8,%rsp
ddb{1}> show proc
PROC (syz-executor) tid=16747 pid=31190 tcnt=2 stat=onproc
flags process=10<SUGID> proc=4000000<THREAD>
runpri=32, usrpri=80, slppri=32, nice=20
wchan=0x0, wmesg=, ps_single=0x0 scnt=0 ecnt=0
forw=0xffffffffffffffff, list=0xffff80003c459258,0xffffffff83927c28
process=0xffff80003bc26b70 user=0xffff80003c472000, vmspace=0xfffffd80661a2208
estcpu=36, cpticks=0, pctcpu=0.0, user=0, sys=0, intr=0
ddb{1}> ps
PID TID PPID UID S FLAGS WAIT COMMAND
31190 210396 81378 60928 7 0x10 syz-executor
*31190 16747 81378 60928 7 0x4000010 syz-executor
59153 161244 13164 0 2 0 syz-executor
59153 32509 13164 0 3 0x4000080 sbwait syz-executor
17750 120125 61267 0 3 0x80 nanoslp syz-executor
17750 19694 61267 0 3 0x4000080 pipewr syz-executor
17750 196707 61267 0 3 0x4000080 pipewr syz-executor
68935 488528 34826 0 3 0x80 nanoslp syz-executor
68935 154992 34826 0 3 0x4000080 fsleep syz-executor
68935 288796 34826 0 3 0x4000080 sbwait syz-executor
1962 267087 42299 0 3 0x80 nanoslp syz-executor
1962 322573 42299 0 3 0x4000080 ttyout syz-executor
1962 75991 42299 0 3 0x4000080 fsleep syz-executor
21067 509117 0 0 3 0x14200 acct acct
81378 269894 8240 0 3 0x82 nanoslp syz-executor
72048 513939 1 0 3 0x100083 ttyin getty
42299 429626 8240 0 3 0x82 nanoslp syz-executor
60921 343325 0 0 3 0x14200 bored sosplice
66473 139271 8240 0 3 0x82 wait syz-executor
70652 349383 8240 0 3 0x82 nanoslp syz-executor
61267 92539 8240 0 3 0x82 nanoslp syz-executor
13164 295268 8240 0 3 0x82 nanoslp syz-executor
34826 353959 8240 0 3 0x82 nanoslp syz-executor
2834 282618 8240 0 3 0x82 nanoslp syz-executor
8240 310991 24030 0 3 0x82 kqread syz-executor
24030 208725 96719 0 3 0x10008a sigsusp ksh
96719 209978 58427 0 3 0x98 kqread sshd-session
58427 407741 56009 0 3 0x92 kqread sshd-session
56009 267181 1 0 3 0x88 kqread sshd
78568 326869 84173 74 3 0x1100092 bpf pflogd
84173 150110 1 0 3 0x80 sbwait pflogd
48646 304976 29322 73 3 0x1100090 kqread syslogd
29322 70887 1 0 3 0x100082 sbwait syslogd
59750 501671 1 0 3 0x100080 kqread resolvd
57274 269888 50814 77 3 0x100092 kqread dhcpleased
33365 290685 50814 77 3 0x100092 kqread dhcpleased
50814 284793 1 0 3 0x80 kqread dhcpleased
4447 201733 0 0 3 0x14200 bored smr
96718 349361 0 0 3 0x14200 pgzero zerothread
61173 385369 0 0 3 0x14200 aiodoned aiodoned
51016 481664 0 0 3 0x14200 syncer update
79306 306818 0 0 3 0x14200 cleaner cleaner
58493 301995 0 0 3 0x14200 reaper reaper
47344 165472 0 0 3 0x14200 pgdaemon pagedaemon
12447 51558 0 0 3 0x14200 bored viomb
86306 152187 0 0 3 0x40014200 acpi0 acpi0
77185 162750 0 0 3 0x40014200 idle1
50752 57537 0 0 3 0x14200 bored softnet7
87265 28780 0 0 3 0x14200 bored softnet6
14354 87491 0 0 3 0x14200 bored softnet5
41437 120780 0 0 3 0x14200 bored softnet4
58896 92512 0 0 3 0x14200 bored softnet3
85063 99612 0 0 3 0x14200 bored softnet2
96452 372981 0 0 3 0x14200 bored softnet1
67577 25282 0 0 3 0x14200 bored softnet0
1416 319777 0 0 3 0x14200 bored systqmp
24656 221749 0 0 3 0x14200 bored systq
53762 401840 0 0 3 0x14200 tmoslp softclockmp
19665 290774 0 0 3 0x40014200 tmoslp softclock
70322 324436 0 0 3 0x40014200 idle0
1 389789 0 0 3 0x82 wait init
0 0 -1 0 3 0x10010200 scheduler swapper
ddb{1}> show all locks
CPU 1:
exclusive mutex &cq->cq_mtx r = 0 (0xffffffff837f9e08)
#0 witness_lock+0x5f1 stacktrace_save sys/sys/stacktrace.h:37 [inline]
#0 witness_lock+0x5f1 sys/kern/subr_witness.c:1160
#1 mtx_enter_try+0x1ad sys/kern/kern_lock.c:311
#2 mtx_enter+0x13b sys/kern/kern_lock.c:261
#3 clockintr_unbind+0x56 sys/kern/kern_clockintr.c:375
#4 dt_ioctl_record_stop+0xbc sys/dev/dt/dt_dev.c:582
#5 dtclose+0x109 dt_pcb_purge sys/dev/dt/dt_dev.c:-1 [inline]
#5 dtclose+0x109 sys/dev/dt/dt_dev.c:232
#6 spec_close+0x466 sys/kern/spec_vnops.c:-1
#7 VOP_CLOSE+0x132 sys/kern/vfs_vops.c:156
#8 vn_closefile+0x12b vn_close sys/kern/vfs_vnops.c:292 [inline]
#8 vn_closefile+0x12b sys/kern/vfs_vnops.c:615
#9 fdrop+0x121 sys/kern/kern_descrip.c:1280
#10 closef+0x192 sys/kern/kern_descrip.c:1264
#11 syscall+0xbd4 mi_syscall sys/sys/syscall_mi.h:176 [inline]
#11 syscall+0xbd4 sys/arch/amd64/amd64/trap.c:746
#12 Xsyscall+0x128
Process 31190 (syz-executor) thread 0xffff800037009ca8 (16747)
exclusive rwlock dtlk r = 0 (0xffffffff837709c8)
#0 witness_lock+0x5f1 stacktrace_save sys/sys/stacktrace.h:37 [inline]
#0 witness_lock+0x5f1 sys/kern/subr_witness.c:1160
#1 rw_do_enter_write+0x419 sys/kern/kern_rwlock.c:320
#2 dt_ioctl_record_stop+0x2e sys/dev/dt/dt_dev.c:565
#3 dtclose+0x109 dt_pcb_purge sys/dev/dt/dt_dev.c:-1 [inline]
#3 dtclose+0x109 sys/dev/dt/dt_dev.c:232
#4 spec_close+0x466 sys/kern/spec_vnops.c:-1
#5 VOP_CLOSE+0x132 sys/kern/vfs_vops.c:156
#6 vn_closefile+0x12b vn_close sys/kern/vfs_vnops.c:292 [inline]
#6 vn_closefile+0x12b sys/kern/vfs_vnops.c:615
#7 fdrop+0x121 sys/kern/kern_descrip.c:1280
#8 closef+0x192 sys/kern/kern_descrip.c:1264
#9 syscall+0xbd4 mi_syscall sys/sys/syscall_mi.h:176 [inline]
#9 syscall+0xbd4 sys/arch/amd64/amd64/trap.c:746
#10 Xsyscall+0x128
exclusive kernel_lock &kernel_lock r = 0 (0xffffffff83873798)
#0 witness_lock+0x5f1 stacktrace_save sys/sys/stacktrace.h:37 [inline]
#0 witness_lock+0x5f1 sys/kern/subr_witness.c:1160
#1 vn_closefile+0x41 sys/kern/vfs_vnops.c:608
#2 fdrop+0x121 sys/kern/kern_descrip.c:1280
#3 closef+0x192 sys/kern/kern_descrip.c:1264
#4 syscall+0xbd4 mi_syscall sys/sys/syscall_mi.h:176 [inline]
#4 syscall+0xbd4 sys/arch/amd64/amd64/trap.c:746
#5 Xsyscall+0x128
exclusive mutex &cq->cq_mtx r = 0 (0xffffffff837f9e08)
#0 witness_lock+0x5f1 stacktrace_save sys/sys/stacktrace.h:37 [inline]
#0 witness_lock+0x5f1 sys/kern/subr_witness.c:1160
#1 mtx_enter_try+0x1ad sys/kern/kern_lock.c:311
#2 mtx_enter+0x13b sys/kern/kern_lock.c:261
#3 clockintr_unbind+0x56 sys/kern/kern_clockintr.c:375
#4 dt_ioctl_record_stop+0xbc sys/dev/dt/dt_dev.c:582
#5 dtclose+0x109 dt_pcb_purge sys/dev/dt/dt_dev.c:-1 [inline]
#5 dtclose+0x109 sys/dev/dt/dt_dev.c:232
#6 spec_close+0x466 sys/kern/spec_vnops.c:-1
#7 VOP_CLOSE+0x132 sys/kern/vfs_vops.c:156
#8 vn_closefile+0x12b vn_close sys/kern/vfs_vnops.c:292 [inline]
#8 vn_closefile+0x12b sys/kern/vfs_vnops.c:615
#9 fdrop+0x121 sys/kern/kern_descrip.c:1280
#10 closef+0x192 sys/kern/kern_descrip.c:1264
#11 syscall+0xbd4 mi_syscall sys/sys/syscall_mi.h:176 [inline]
#11 syscall+0xbd4 sys/arch/amd64/amd64/trap.c:746
#12 Xsyscall+0x128
Process 59153 (syz-executor) thread 0xffff80003c4594f0 (32509)
exclusive rwlock sbufrcv r = 0 (0xffff8000014246e0)
#0 witness_lock+0x5f1 stacktrace_save sys/sys/stacktrace.h:37 [inline]
#0 witness_lock+0x5f1 sys/kern/subr_witness.c:1160
#1 rw_do_enter_write+0x419 sys/kern/kern_rwlock.c:320
#2 sblock+0xb6 sys/kern/uipc_socket2.c:536
#3 soreceive+0x27d sys/kern/uipc_socket.c:870
#4 recvit+0x40b sys/kern/uipc_syscalls.c:1078
#5 sys_recvmmsg+0x410 sys/kern/uipc_syscalls.c:969
#6 syscall+0xbd4 mi_syscall sys/sys/syscall_mi.h:176 [inline]
#6 syscall+0xbd4 sys/arch/amd64/amd64/trap.c:746
#7 Xsyscall+0x128
ddb{1}> show malloc
Type InUse MemUse HighUse Limit Requests Type Lim
devbuf 10251 11079K 11869K 166960K 12127 0
pcb 17 14K 16K 166960K 297 0
rtable 189 11K 11K 166960K 432 0
pf 42 19K 26K 166960K 174 0
ifaddr 40 6K 7K 166960K 100 0
ifgroup 67 2K 3K 166960K 164 0
sysctl 3 1K 9K 166960K 18 0
counters 74 37K 38K 166960K 316 0
ioctlops 0 0K 4K 166960K 1801 0
iov 1 4K 18K 166960K 47 0
mount 1 1K 1K 166960K 1 0
log 0 0K 0K 166960K 4 0
vnodes 1410 89K 89K 166960K 2154 0
UFS quota 1 32K 32K 166960K 1 0
UFS mount 5 36K 36K 166960K 5 0
shm 2 1K 9K 166960K 17 0
VM map 2 1K 1K 166960K 2 0
sem 20 2K 2K 166960K 94 0
dirhash 15 2K 2K 166960K 21 0
ACPI 1692 195K 286K 166960K 12470 0
file desc 18 65K 240K 166960K 1014 0
sigio 0 0K 0K 166960K 163 0
proc 72 115K 180K 166960K 678 0
subproc 72 4K 4K 166960K 90 0
NFS srvsock 1 0K 0K 166960K 1 0
NFS daemon 1 16K 16K 166960K 1 0
ip_moptions 0 0K 0K 166960K 176 0
in_multi 66 5K 7K 166960K 162 0
ether_multi 1 0K 0K 166960K 7 0
mrt 0 0K 0K 166960K 5 0
ISOFS mount 1 32K 32K 166960K 1 0
MSDOSFS mount 1 16K 16K 166960K 1 0
ttys 217 970K 970K 166960K 217 0
exec 0 0K 1K 166960K 660 0
fusefs mount 1 32K 32K 166960K 1 0
tdb 3 0K 0K 166960K 3 0
VM swap 8 62K 64K 166960K 10 0
UVM amap 251 160K 177K 166960K 10467 0
UVM aobj 19 2K 2K 166960K 21 0
pinsyscall 43 86K 102K 166960K 2134 0
memdesc 1 4K 4K 166960K 1 0
crypto data 1 1K 1K 166960K 1 0
ip6_options 0 0K 0K 166960K 89 0
NDP 14 0K 2K 166960K 71 0
temp 63 8647K 8712K 166960K 47878 0
kqueue 13 20K 32K 166960K 221 0
SYN cache 2 16K 16K 166960K 2 0
ddb{1}> show all pools
Name Size Requests Fail Releases Pgreq Pgrel Npage Hiwat Minpg Maxpg Idle
plcache 128 26 0 0 1 0 1 1 0 8 0
rtpcb 120 90 0 87 1 0 1 1 0 8 0
rtentry 176 149 0 80 5 1 4 5 0 8 0
unpcb 144 745 0 724 6 2 4 4 0 8 3
syncache 336 9 0 9 2 1 1 1 0 8 1
tcpqe 32 4 0 4 1 0 1 1 0 8 1
tcpcb 736 295 0 286 4 2 2 4 0 8 0
arp 128 19 0 10 1 0 1 1 0 8 0
inpcb 328 1166 0 1151 8 0 8 8 0 8 6
nd6 144 23 0 10 1 0 1 1 0 8 0
pkpcb 40 3 0 3 1 0 1 1 0 8 1
kcovpl 48 10 0 2 1 0 1 1 0 8 0
ppxss 1192 113 0 112 1 0 1 1 0 8 0
pppxif 1504 4 0 3 2 1 1 1 0 8 0
pfstscr 40 2 0 2 1 1 0 1 0 8 0
pffrag 232 9 0 4 1 0 1 1 0 482 0
pffrnode 88 8 0 4 1 0 1 1 0 8 0
pffrent 40 112 0 105 1 0 1 1 0 8 0
pfosfp 40 1428 0 1005 5 0 5 5 0 8 0
pfosfpen 112 1428 0 714 21 0 21 21 0 8 0
pfrktable 1344 1 0 1 1 1 0 1 0 8 0
pfanchor 1288 3 0 0 1 0 1 1 0 8 0
pfstitem 24 21 0 9 1 0 1 1 0 8 0
pfstkey 128 24 0 12 1 0 1 1 0 8 0
pfstate 384 22 0 11 2 0 2 2 0 8 0
pfrule 1344 27 0 21 2 1 1 2 0 8 0
rttmr 136 1 0 1 1 1 0 1 0 8 0
art_heap8 4096 5 0 0 5 0 5 5 0 8 0
art_heap4 256 633 0 321 30 7 23 28 0 8 3
art_table 40 638 0 321 5 1 4 5 0 8 0
art_node 32 146 0 88 1 0 1 1 0 8 0
sysvmsgpl 40 13 0 9 1 0 1 1 0 8 0
semapl 112 87 0 69 1 0 1 1 0 8 0
shmpl 112 18 0 2 1 0 1 1 0 8 0
dirhash 1024 23 0 4 3 0 3 3 0 8 0
dino2pl 256 3258 0 1746 96 1 95 96 0 8 0
ffsino 296 3258 0 1746 118 0 118 118 0 8 0
nchpl 144 4593 0 2894 64 0 64 64 0 8 0
rtmask 32 15 0 15 1 0 1 1 0 8 1
uvmvnodes 80 3698 0 0 76 0 76 76 0 8 0
vnodes 216 3698 0 0 206 0 206 206 0 8 0
namei 1024 18651 0 18651 1 0 1 1 0 8 1
percpumem 16 173 0 121 1 0 1 1 0 8 0
kstatmem 264 104 0 72 3 0 3 3 0 8 0
scsiplug 72 5 0 5 2 1 1 1 0 8 1
scxspl 216 28730 0 28730 11 8 3 8 1 8 3
plimitpl 152 306 0 288 1 0 1 1 0 8 0
sigapl 424 1302 0 1246 9 2 7 9 0 8 0
knotepl 120 553 0 0 17 0 17 17 0 8 0
kqueuepl 224 400 0 390 3 1 2 3 0 8 1
pipepl 344 238 0 209 6 0 6 6 0 8 3
fdescpl 528 1257 0 1225 3 0 3 3 0 8 0
filepl 160 7922 0 7690 16 1 15 15 0 8 3
lockfpl 104 280 0 276 1 0 1 1 0 8 0
lockfspl 48 122 0 118 1 0 1 1 0 8 0
sessionpl 144 30 0 21 1 0 1 1 0 8 0
pgrppl 48 51 0 34 1 0 1 1 0 8 0
ucredpl 104 1372 0 1358 1 0 1 1 0 8 0
zombiepl 144 1249 0 1246 1 0 1 1 0 8 0
processpl 1232 1302 0 1246 6 1 5 6 0 8 0
procpl 664 2799 0 2735 8 1 7 8 0 8 0
sosppl 168 9 0 9 1 0 1 1 0 8 1
sockpl 752 2024 0 1985 19 8 11 17 0 8 6
mcl64k 65536 17 0 0 3 0 3 3 0 8 0
mcl16k 16384 2 0 0 1 0 1 1 0 8 0
mcl12k 12288 1 0 0 1 0 1 1 0 8 0
mcl9k 9216 1 0 0 1 0 1 1 0 8 0
mcl8k 8192 3 0 0 1 0 1 1 0 8 0
mcl4k 4096 122 0 0 16 0 16 16 0 8 0
mcl2k 2048 25 0 0 4 0 4 4 0 8 0
mtagpl 96 3 0 0 1 0 1 1 0 8 0
mbufpl 256 194 0 0 13 0 13 13 0 8 0
bufpl 280 11895 0 5752 440 0 440 440 0 8 0
anonpl 32 12044 0 0 98 0 98 98 0 246 0
amapchunkpl 152 35094 0 34558 31 5 26 29 0 158 2
amappl16 200 4663 0 4612 34 22 12 27 0 8 5
amappl15 192 8 0 7 1 0 1 1 0 8 0
amappl14 184 126 0 114 1 0 1 1 0 8 0
amappl13 176 4 0 4 1 1 0 1 0 8 0
amappl12 168 1946 0 1913 3 1 2 2 0 8 0
amappl11 160 52 0 38 1 0 1 1 0 8 0
amappl10 152 7 0 7 1 1 0 1 0 8 0
amappl9 144 257 0 257 1 1 0 1 0 8 0
amappl8 136 46 0 43 1 0 1 1 0 8 0
amappl7 128 115 0 103 1 0 1 1 0 8 0
amappl6 120 222 0 218 1 0 1 1 0 8 0
amappl5 112 147 0 137 1 0 1 1 0 8 0
amappl4 104 317 0 294 1 0 1 1 0 8 0
amappl3 96 6141 0 6039 4 1 3 4 0 8 0
amappl2 88 1526 0 1446 2 0 2 2 0 8 0
amappl1 80 13062 0 12453 17 2 15 16 0 8 0
amappl 88 9586 0 9408 5 0 5 5 0 92 0
dma32768 32768 1 0 1 1 1 0 1 0 8 0
dma4096 4096 1 0 1 1 1 0 1 0 8 0
dma1024 1024 1 0 0 1 0 1 1 0 8 0
dma256 256 8 0 8 2 2 0 1 0 8 0
dma128 128 254 0 254 2 1 1 1 0 8 1
dma64 64 6 0 6 1 1 0 1 0 8 0
dma32 32 7 0 7 1 1 0 1 0 8 0
dma16 16 18 0 17 1 0 1 1 0 8 0
aobjpl 72 20 0 2 1 0 1 1 0 8 0
uaddrrnd 24 1257 0 1225 1 0 1 1 0 8 0
uaddrbest 32 2 0 0 1 0 1 1 0 8 0
uaddr 24 1257 0 1225 1 0 1 1 0 8 0
vmmpekpl 168 12016 0 11967 3 0 3 3 0 8 0
vmmpepl 168 85980 0 83899 112 11 101 112 0 357 2
vmsppl 488 1256 0 1225 5 0 5 5 0 8 0
rwobjpl 80 29056 0 24364 99 0 99 99 0 8 0
pdppl 4096 2522 0 2450 104 32 72 84 0 8 0
pvpl 32 22903 0 0 185 0 185 185 0 265 0
pmappl 256 1256 0 1225 3 0 3 3 0 8 0
extentpl 40 45 0 27 1 0 1 1 0 8 0
phpool 112 291 0 47 8 0 8 8 0 8 0
ddb{1}> machine ddbcpu 0
Stopped at x86_ipi_db+0x27: addq $0x8,%rsp
ddb{0}> trace
x86_ipi_db(ffffffff837f8ff0) at x86_ipi_db+0x27 sys/arch/amd64/amd64/db_interface.c:394
x86_ipi_handler() at x86_ipi_handler+0xd9 sys/arch/amd64/amd64/ipi.c:106
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x27
mtx_enter(ffffffff837f9df8) at mtx_enter+0xf0 sys/kern/kern_lock.c:264
clockintr_dispatch(ffff80003c44b230) at clockintr_dispatch+0x3b1 sys/kern/kern_clockintr.c:215
lapic_clockintr(0,0) at lapic_clockintr+0x51 sys/arch/amd64/amd64/lapic.c:484
Xresume_lapic_ltimer() at Xresume_lapic_ltimer+0x2a
end of kernel
end trace frame: 0x7382f7219a50, count: -7
ddb{0}> machine ddbcpu 1
Stopped at db_enter+0x25: addq $0x8,%rsp
ddb{1}> trace
db_enter() at db_enter+0x25 sys/arch/amd64/amd64/db_interface.c:438
witness_checkorder(ffffffff83902fe8,9,0) at witness_checkorder+0x10b9 sys/kern/subr_witness.c:-1
mtx_enter(ffffffff83902fd8) at mtx_enter+0x4a sys/kern/kern_lock.c:260
sleep_setup(ffffffff837f9e58,220,ffffffff833f2a81) at sleep_setup+0xc6 sys/kern/kern_synch.c:302
msleep_nsec(ffffffff837f9e58,ffffffff837f9df8,220,ffffffff833f2a81,ffffffffffffffff) at msleep_nsec+0x10b sys/kern/kern_synch.c:217
dt_ioctl_record_stop(ffff800001604000) at dt_ioctl_record_stop+0xbc sys/dev/dt/dt_dev.c:582
dtclose(11e5f,81,2000,ffff800037009ca8) at dtclose+0x109 dt_pcb_purge sys/dev/dt/dt_dev.c:-1 [inline]
dtclose(11e5f,81,2000,ffff800037009ca8) at dtclose+0x109 sys/dev/dt/dt_dev.c:232
spec_close(ffff80003c477770) at spec_close+0x466 sys/kern/spec_vnops.c:-1
VOP_CLOSE(fffffd80629ca710,81,fffffd80097fb618,ffff800037009ca8) at VOP_CLOSE+0x132 sys/kern/vfs_vops.c:156
vn_closefile(fffffd80654dbb00,ffff800037009ca8) at vn_closefile+0x12b vn_close sys/kern/vfs_vnops.c:292 [inline]
vn_closefile(fffffd80654dbb00,ffff800037009ca8) at vn_closefile+0x12b sys/kern/vfs_vnops.c:615
fdrop(fffffd80654dbb00,ffff800037009ca8) at fdrop+0x121 sys/kern/kern_descrip.c:1280
closef(fffffd80654dbb00,ffff800037009ca8) at closef+0x192 sys/kern/kern_descrip.c:1264
syscall(ffff80003c4779d0) at syscall+0xbd4 mi_syscall sys/sys/syscall_mi.h:176 [inline]
syscall(ffff80003c4779d0) at syscall+0xbd4 sys/arch/amd64/amd64/trap.c:746
Xsyscall() at Xsyscall+0x128
end of kernel
end trace frame: 0x9b041331680, count: -14