witness: lock order reversal:
1st 0xffff8000299bee30 &cq->cq_mtx (&cq->cq_mtx)
2nd 0xffffffff83a4a1c8 &sched_lock (&sched_lock)
lock order [1] &cq->cq_mtx (&cq->cq_mtx) -> [2] &sched_lock (&sched_lock)
#0 mtx_enter+0x95 sys/kern/kern_lock.c:406
#1 sleep_setup+0xed sys/kern/kern_synch.c:304
#2 msleep_nsec+0x10b sys/kern/kern_synch.c:217
#3 dt_ioctl_record_stop+0xbc sys/dev/dt/dt_dev.c:589
#4 dtclose+0x109 dt_pcb_purge sys/dev/dt/dt_dev.c:-1 [inline]
#4 dtclose+0x109 sys/dev/dt/dt_dev.c:239
#5 spec_close+0x466 sys/kern/spec_vnops.c:-1
#6 VOP_CLOSE+0x132 sys/kern/vfs_vops.c:156
#7 vn_closefile+0x12b vn_close sys/kern/vfs_vnops.c:292 [inline]
#7 vn_closefile+0x12b sys/kern/vfs_vnops.c:615
#8 fdrop+0x121 sys/kern/kern_descrip.c:1281
#9 closef+0x192 sys/kern/kern_descrip.c:1265
#10 syscall+0xbd4 mi_syscall sys/sys/syscall_mi.h:176 [inline]
#10 syscall+0xbd4 sys/arch/amd64/amd64/trap.c:783
#11 Xsyscall+0x128
lock order [2] &sched_lock (&sched_lock) -> [1] &cq->cq_mtx (&cq->cq_mtx)
#0 mtx_enter+0x95 sys/kern/kern_lock.c:406
#1 clockintr_cancel+0x2f sys/kern/kern_clockintr.c:323
#2 mi_switch+0x187 sys/kern/sched_bsd.c:376
#3 ast+0x15c mi_ast sys/sys/syscall_mi.h:252 [inline]
#3 ast+0x15c sys/arch/amd64/amd64/trap.c:747
#4 intr_user_exit+0x3c
Stopped at db_enter+0x25: addq $0x8,%rsp
ddb{0}>
ddb{0}> set $lines = 0
ddb{0}> set $maxwidth = 0
ddb{0}> show panic
the kernel did not panic
ddb{0}> trace
db_enter() at db_enter+0x25 sys/arch/amd64/amd64/db_interface.c:438
witness_checkorder(ffffffff83a4a1c8,9,0) at witness_checkorder+0x10d1 sys/kern/subr_witness.c:-1
mtx_enter(ffffffff83a4a1b8) at mtx_enter+0x95 sys/kern/kern_lock.c:406
sleep_setup(ffff8000299bee80,220,ffffffff834cb92c) at sleep_setup+0xed sys/kern/kern_synch.c:304
msleep_nsec(ffff8000299bee80,ffff8000299bee20,220,ffffffff834cb92c,ffffffffffffffff) at msleep_nsec+0x10b sys/kern/kern_synch.c:217
dt_ioctl_record_stop(ffff800001669000) at dt_ioctl_record_stop+0xbc sys/dev/dt/dt_dev.c:589
dtclose(21e5f,81,2000,ffff80003c3f6558) at dtclose+0x109 dt_pcb_purge sys/dev/dt/dt_dev.c:-1 [inline]
dtclose(21e5f,81,2000,ffff80003c3f6558) at dtclose+0x109 sys/dev/dt/dt_dev.c:239
spec_close(ffff80003c4017c0) at spec_close+0x466 sys/kern/spec_vnops.c:-1
VOP_CLOSE(fffffd805f7a62d8,81,fffffd80097fd2d8,ffff80003c3f6558) at VOP_CLOSE+0x132 sys/kern/vfs_vops.c:156
vn_closefile(fffffd806c19eea0,ffff80003c3f6558) at vn_closefile+0x12b vn_close sys/kern/vfs_vnops.c:292 [inline]
vn_closefile(fffffd806c19eea0,ffff80003c3f6558) at vn_closefile+0x12b sys/kern/vfs_vnops.c:615
fdrop(fffffd806c19eea0,ffff80003c3f6558) at fdrop+0x121 sys/kern/kern_descrip.c:1281
closef(fffffd806c19eea0,ffff80003c3f6558) at closef+0x192 sys/kern/kern_descrip.c:1265
syscall(ffff80003c401a20) at syscall+0xbd4 mi_syscall sys/sys/syscall_mi.h:176 [inline]
syscall(ffff80003c401a20) at syscall+0xbd4 sys/arch/amd64/amd64/trap.c:783
Xsyscall() at Xsyscall+0x128
end of kernel
end trace frame: 0xefd3ba36cb0, count: -14
ddb{0}> show registers
rdi 0
rsi 0x80000 acpi_pdirpa+0x6be71
rbp 0xffff80003c401490
rbx 0
rdx 0xffff8000015f3780
rcx 0xffff80003c3f6558
rax 0x7ffff acpi_pdirpa+0x6be70
r8 0xffff80003c401370
r9 0x8080808080808080
r10 0x6d9c6ee3acdd3690
r11 0xbe6b39d38d36cf0c
r12 0xfffffd800403ce40
r13 0xfffffd80047e7330
r14 0x3
r15 0xffffffff834b72b3 substchar+0x1f5f
rip 0xffffffff82669d45 db_enter+0x25
cs 0x8
rflags 0x246
rsp 0xffff80003c401480
ss 0x10
db_enter+0x25: addq $0x8,%rsp
ddb{0}> show proc
PROC (syz-executor) tid=39841 pid=21809 tcnt=2 stat=onproc
flags process=0 proc=4000000<THREAD>
runpri=32, usrpri=50, slppri=32, nice=20
wchan=0x0, wmesg=, ps_single=0x0 scnt=0 ecnt=0
forw=0xffffffffffffffff, list=0xffff80003c3f67f0,0xffff80003c3f6a98
process=0xffff80003c3f9d00 user=0xffff80003c3fc000, vmspace=0xfffffd800f263b90
estcpu=36, cpticks=0, pctcpu=0.0, user=0, sys=0, intr=0
ddb{0}> ps
PID TID PPID UID S FLAGS WAIT COMMAND
14793 404676 21374 0 7 0 syz-executor
14793 310800 21374 0 2 0x4000000 syz-executor
2357 310695 94124 0 2 0 syz-executor
2357 460262 94124 0 3 0x4000080 fsleep syz-executor
21809 299932 45938 0 2 0 syz-executor
*21809 39841 45938 0 7 0x4000000 syz-executor
28135 279617 34420 0 2 0 syz-executor
28135 421725 34420 0 3 0x4000080 fsleep syz-executor
11993 387011 4788 0 2 0 syz-executor
11993 437860 4788 0 3 0x4000080 fsleep syz-executor
11993 126139 4788 0 3 0x4000080 fsleep syz-executor
36959 117080 72570 0 2 0 syz-executor
36959 118987 72570 0 3 0x4000080 fsleep syz-executor
40855 151952 63964 0 2 0 syz-executor
40855 113540 63964 0 3 0x4000080 fsleep syz-executor
45938 132711 95568 0 3 0x82 nanoslp syz-executor
61971 56641 95568 0 3 0x2 biowait syz-executor
94124 89618 95568 0 3 0x82 nanoslp syz-executor
72570 268984 95568 0 3 0x82 nanoslp syz-executor
21374 505804 95568 0 3 0x82 nanoslp syz-executor
34420 391513 95568 0 3 0x82 nanoslp syz-executor
63964 143070 95568 0 3 0x82 nanoslp syz-executor
4788 19530 95568 0 3 0x82 nanoslp syz-executor
95568 243334 45468 0 3 0x82 kqread syz-executor
45468 367638 61800 0 3 0x10008a sigsusp ksh
61800 307194 96473 0 3 0x98 kqread sshd-session
96473 99133 7515 0 3 0x92 kqread sshd-session
20277 454032 1 0 3 0x100083 ttyin getty
7515 194856 1 0 3 0x88 kqread sshd
48961 234457 79622 74 3 0x1100092 bpf pflogd
79622 199961 1 0 3 0x80 sbwait pflogd
53604 153694 71334 73 3 0x1100090 kqread syslogd
71334 93242 1 0 3 0x100082 sbwait syslogd
47449 187759 1 0 3 0x100080 kqread resolvd
98473 115573 71865 77 3 0x100092 kqread dhcpleased
17492 152196 71865 77 3 0x100092 kqread dhcpleased
71865 233369 1 0 3 0x80 kqread dhcpleased
43538 146707 0 0 3 0x14200 bored smr
42173 499177 0 0 2 0x14200 zerothread
30960 202486 0 0 3 0x14200 aiodoned aiodoned
58013 45443 0 0 3 0x14200 syncer update
90859 243273 0 0 3 0x14200 cleaner cleaner
61062 488410 0 0 3 0x14200 reaper reaper
20369 289171 0 0 3 0x14200 pgdaemon pagedaemon
68998 230390 0 0 3 0x14200 bored viomb
99469 275876 0 0 3 0x40014200 acpi0 acpi0
24696 357891 0 0 3 0x40014200 idle1
15117 71973 0 0 3 0x14200 bored softnet1
42042 444334 0 0 3 0x14200 bored softnet0
44442 55654 0 0 3 0x14200 bored systqmp
66664 408761 0 0 3 0x14200 bored systq
85289 35222 0 0 3 0x14200 tmoslp softclockmp
77972 172335 0 0 3 0x40014200 tmoslp softclock
84884 6438 0 0 3 0x40014200 idle0
1 82064 0 0 3 0x82 wait init
0 0 -1 0 3 0x10010200 scheduler swapper
ddb{0}> show all locks
CPU 0:
exclusive mutex &cq->cq_mtx r = 0 (0xffff8000299bee30)
#0 witness_lock+0x5f1 stacktrace_save sys/sys/stacktrace.h:37 [inline]
#0 witness_lock+0x5f1 sys/kern/subr_witness.c:1160
#1 mtx_enter+0x4b4 sys/kern/kern_lock.c:487
#2 clockintr_unbind+0x56 sys/kern/kern_clockintr.c:375
#3 dt_ioctl_record_stop+0xbc sys/dev/dt/dt_dev.c:589
#4 dtclose+0x109 dt_pcb_purge sys/dev/dt/dt_dev.c:-1 [inline]
#4 dtclose+0x109 sys/dev/dt/dt_dev.c:239
#5 spec_close+0x466 sys/kern/spec_vnops.c:-1
#6 VOP_CLOSE+0x132 sys/kern/vfs_vops.c:156
#7 vn_closefile+0x12b vn_close sys/kern/vfs_vnops.c:292 [inline]
#7 vn_closefile+0x12b sys/kern/vfs_vnops.c:615
#8 fdrop+0x121 sys/kern/kern_descrip.c:1281
#9 closef+0x192 sys/kern/kern_descrip.c:1265
#10 syscall+0xbd4 mi_syscall sys/sys/syscall_mi.h:176 [inline]
#10 syscall+0xbd4 sys/arch/amd64/amd64/trap.c:783
#11 Xsyscall+0x128
Process 21809 (syz-executor) thread 0xffff80003c3f6558 (39841)
exclusive rwlock dtlk r = 0 (0xffffffff838c6f38)
#0 witness_lock+0x5f1 stacktrace_save sys/sys/stacktrace.h:37 [inline]
#0 witness_lock+0x5f1 sys/kern/subr_witness.c:1160
#1 rw_do_enter_write+0x419 sys/kern/kern_rwlock.c:320
#2 dt_ioctl_record_stop+0x2e sys/dev/dt/dt_dev.c:572
#3 dtclose+0x109 dt_pcb_purge sys/dev/dt/dt_dev.c:-1 [inline]
#3 dtclose+0x109 sys/dev/dt/dt_dev.c:239
#4 spec_close+0x466 sys/kern/spec_vnops.c:-1
#5 VOP_CLOSE+0x132 sys/kern/vfs_vops.c:156
#6 vn_closefile+0x12b vn_close sys/kern/vfs_vnops.c:292 [inline]
#6 vn_closefile+0x12b sys/kern/vfs_vnops.c:615
#7 fdrop+0x121 sys/kern/kern_descrip.c:1281
#8 closef+0x192 sys/kern/kern_descrip.c:1265
#9 syscall+0xbd4 mi_syscall sys/sys/syscall_mi.h:176 [inline]
#9 syscall+0xbd4 sys/arch/amd64/amd64/trap.c:783
#10 Xsyscall+0x128
exclusive kernel_lock &kernel_lock r = 0 (0xffffffff83aba240)
#0 witness_lock+0x5f1 stacktrace_save sys/sys/stacktrace.h:37 [inline]
#0 witness_lock+0x5f1 sys/kern/subr_witness.c:1160
#1 vn_closefile+0x41 sys/kern/vfs_vnops.c:608
#2 fdrop+0x121 sys/kern/kern_descrip.c:1281
#3 closef+0x192 sys/kern/kern_descrip.c:1265
#4 syscall+0xbd4 mi_syscall sys/sys/syscall_mi.h:176 [inline]
#4 syscall+0xbd4 sys/arch/amd64/amd64/trap.c:783
#5 Xsyscall+0x128
exclusive mutex &cq->cq_mtx r = 0 (0xffff8000299bee30)
#0 witness_lock+0x5f1 stacktrace_save sys/sys/stacktrace.h:37 [inline]
#0 witness_lock+0x5f1 sys/kern/subr_witness.c:1160
#1 mtx_enter+0x4b4 sys/kern/kern_lock.c:487
#2 clockintr_unbind+0x56 sys/kern/kern_clockintr.c:375
#3 dt_ioctl_record_stop+0xbc sys/dev/dt/dt_dev.c:589
#4 dtclose+0x109 dt_pcb_purge sys/dev/dt/dt_dev.c:-1 [inline]
#4 dtclose+0x109 sys/dev/dt/dt_dev.c:239
#5 spec_close+0x466 sys/kern/spec_vnops.c:-1
#6 VOP_CLOSE+0x132 sys/kern/vfs_vops.c:156
#7 vn_closefile+0x12b vn_close sys/kern/vfs_vnops.c:292 [inline]
#7 vn_closefile+0x12b sys/kern/vfs_vnops.c:615
#8 fdrop+0x121 sys/kern/kern_descrip.c:1281
#9 closef+0x192 sys/kern/kern_descrip.c:1265
#10 syscall+0xbd4 mi_syscall sys/sys/syscall_mi.h:176 [inline]
#10 syscall+0xbd4 sys/arch/amd64/amd64/trap.c:783
#11 Xsyscall+0x128
Process 61971 (syz-executor) thread 0xffff8000fffef770 (56641)
exclusive rrwlock inode r = 0 (0xfffffd806b6141f0)
#0 witness_lock+0x5f1 stacktrace_save sys/sys/stacktrace.h:37 [inline]
#0 witness_lock+0x5f1 sys/kern/subr_witness.c:1160
#1 rw_do_enter_write+0x419 sys/kern/kern_rwlock.c:320
#2 rrw_enter+0xc6 sys/kern/kern_rwlock.c:621
#3 VOP_LOCK+0xbd sys/kern/vfs_vops.c:527
#4 ufs_ihashins+0x4f ufs_ihash sys/ufs/ufs/ufs_ihash.c:-1 [inline]
#4 ufs_ihashins+0x4f sys/ufs/ufs/ufs_ihash.c:159
#5 ffs_vget+0x187 sys/ufs/ffs/ffs_vfsops.c:1232
#6 ffs_inode_alloc+0x279 sys/ufs/ffs/ffs_alloc.c:393
#7 ufs_mkdir+0xfc sys/ufs/ufs/ufs_vnops.c:1112
#8 VOP_MKDIR+0x101 sys/kern/vfs_vops.c:394
#9 domkdirat+0x179 sys/kern/vfs_syscalls.c:3143
#10 syscall+0xb17 mi_syscall sys/sys/syscall_mi.h:176 [inline]
#10 syscall+0xb17 sys/arch/amd64/amd64/trap.c:783
#11 Xsyscall+0x128
exclusive rrwlock inode r = 0 (0xfffffd806ce22d70)
#0 witness_lock+0x5f1 stacktrace_save sys/sys/stacktrace.h:37 [inline]
#0 witness_lock+0x5f1 sys/kern/subr_witness.c:1160
#1 rw_do_enter_write+0x419 sys/kern/kern_rwlock.c:320
#2 rrw_enter+0xc6 sys/kern/kern_rwlock.c:621
#3 VOP_LOCK+0xbd sys/kern/vfs_vops.c:527
#4 vn_lock+0xa4 sys/kern/vfs_vnops.c:570
#5 vfs_lookup+0x11c sys/kern/vfs_lookup.c:-1
#6 namei+0x7ca sys/kern/vfs_lookup.c:250
#7 domkdirat+0x8b sys/kern/vfs_syscalls.c:3128
#8 syscall+0xb17 mi_syscall sys/sys/syscall_mi.h:176 [inline]
#8 syscall+0xb17 sys/arch/amd64/amd64/trap.c:783
#9 Xsyscall+0x128
ddb{0}> show malloc
Type InUse MemUse HighUse Limit Requests Type Lim
devbuf 11074 12209K 12344K 166960K 12350 0
pcb 17 12K 12K 166960K 58 0
rtable 231 7K 7K 166960K 376 0
pf 35 17K 25K 166960K 106 0
ifaddr 41 7K 7K 166960K 48 0
ifgroup 55 2K 2K 166960K 70 0
sysctl 4 1K 9K 166960K 9 0
counters 70 37K 37K 166960K 76 0
ioctlops 0 0K 4K 166960K 1628 0
iov 0 0K 12K 166960K 9 0
mount 1 1K 1K 166960K 1 0
log 0 0K 0K 166960K 4 0
vnodes 1363 86K 86K 166960K 1626 0
UFS quota 1 32K 32K 166960K 1 0
UFS mount 5 36K 36K 166960K 5 0
shm 2 1K 5K 166960K 5 0
VM map 2 1K 1K 166960K 2 0
sem 8 0K 0K 166960K 10 0
dirhash 12 2K 2K 166960K 12 0
ACPI 1692 195K 286K 166960K 12470 0
file desc 17 61K 89K 166960K 399 0
proc 72 115K 180K 166960K 565 0
subproc 72 4K 4K 166960K 72 0
NFS srvsock 1 0K 0K 166960K 1 0
NFS daemon 1 16K 16K 166960K 1 0
ip_moptions 0 0K 0K 166960K 8 0
in_multi 89 6K 7K 166960K 101 0
ether_multi 1 0K 0K 166960K 1 0
mrt 0 0K 0K 166960K 12 0
ISOFS mount 1 32K 32K 166960K 1 0
MSDOSFS mount 1 16K 16K 166960K 1 0
ttys 67 307K 307K 166960K 67 0
exec 0 0K 1K 166960K 392 0
fusefs mount 1 32K 32K 166960K 1 0
pfkey data 0 0K 0K 166960K 1 0
tdb 3 0K 0K 166960K 3 0
VM swap 8 62K 64K 166960K 10 0
UVM amap 231 167K 185K 166960K 5354 0
UVM aobj 9 2K 2K 166960K 10 0
pinsyscall 42 84K 104K 166960K 1556 0
memdesc 1 4K 4K 166960K 1 0
crypto data 1 1K 1K 166960K 1 0
ip6_options 0 0K 0K 166960K 11 0
NDP 12 0K 2K 166960K 31 0
temp 40 9071K 9139K 166960K 23760 0
kqueue 13 20K 28K 166960K 56 0
SYN cache 2 16K 16K 166960K 2 0
ddb{0}> show all pools
Name Size Requests Fail Releases Pgreq Pgrel Npage Hiwat Minpg Maxpg Idle
plcache 128 26 0 0 1 0 1 1 0 8 0
rtpcb 120 76 0 73 2 0 2 2 0 8 1
rtentry 176 113 0 9 6 0 6 6 0 8 0
unpcb 144 139 0 122 2 0 2 2 0 8 1
syncache 336 5 0 5 1 0 1 1 0 8 1
tcpqe 32 1 0 1 1 0 1 1 0 8 1
tcpcb 736 36 0 31 1 0 1 1 0 8 0
arp 136 18 0 0 1 0 1 1 0 8 0
inpcb 328 183 0 175 2 0 2 2 0 8 0
nd6 152 24 0 3 1 0 1 1 0 8 0
pkpcb 40 2 0 2 1 0 1 1 0 8 1
kcovpl 48 8 0 0 1 0 1 1 0 8 0
ppxss 1192 1 0 1 1 0 1 1 0 8 1
pfstscr 40 10 0 9 1 0 1 1 0 8 0
pffrag 232 2 0 0 1 0 1 1 0 482 0
pffrnode 88 2 0 0 1 0 1 1 0 8 0
pffrent 40 3 0 1 1 0 1 1 0 8 0
pfosfp 40 1428 0 1005 5 0 5 5 0 8 0
pfosfpen 112 1428 0 714 21 0 21 21 0 8 0
pfanchor 1288 3 0 0 1 0 1 1 0 8 0
pfstitem 24 30 0 0 1 0 1 1 0 8 0
pfstkey 128 44 0 14 2 0 2 2 0 8 1
pfstate 448 36 0 7 4 0 4 4 0 8 0
pfrule 1360 43 0 38 2 1 1 2 0 8 0
rttmr 136 1 0 1 1 0 1 1 0 8 1
art_heap8 4096 2 0 0 2 0 2 2 0 8 0
art_heap4 256 453 0 40 29 0 29 29 0 8 0
art_table 40 455 0 40 5 0 5 5 0 8 0
art_node 32 113 0 18 1 0 1 1 0 8 0
sysvmsgpl 40 3 0 0 1 0 1 1 0 8 0
semapl 112 7 0 1 1 0 1 1 0 8 0
shmpl 112 7 0 1 1 0 1 1 0 8 0
dirhash 1024 17 0 0 3 0 3 3 0 8 0
dino2pl 256 2002 0 531 93 0 93 93 0 8 0
ffsino 296 2002 0 531 114 0 114 114 0 8 0
nchpl 144 2571 0 873 64 0 64 64 0 8 0
vnodes 216 2297 0 0 128 0 128 128 0 8 0
namei 1024 7745 0 7744 1 0 1 1 0 8 0
percpumem 16 53 0 3 1 0 1 1 0 8 0
kstatmem 264 32 0 6 2 0 2 2 0 8 0
acpiwqpl 32 1 0 1 1 0 1 1 1 8 1
scsiplug 72 1 0 1 1 0 1 1 0 8 1
scxspl 216 8977 0 8976 11 3 8 8 1 8 7
plimitpl 152 45 0 27 1 0 1 1 0 8 0
sigapl 424 716 0 669 7 0 7 7 0 8 1
knotepl 120 295 0 0 9 0 9 9 0 8 0
kqueuepl 224 72 0 63 1 0 1 1 0 8 0
pipepl 344 136 0 109 3 0 3 3 0 8 0
fdescpl 528 699 0 668 3 0 3 3 0 8 0
filepl 160 2787 0 2573 13 0 13 13 0 8 4
lockfpl 104 99 0 96 1 0 1 1 0 8 0
lockfspl 48 42 0 39 1 0 1 1 0 8 0
sessionpl 144 27 0 18 1 0 1 1 0 8 0
pgrppl 48 35 0 18 1 0 1 1 0 8 0
ucredpl 104 235 0 222 1 0 1 1 0 8 0
zombiepl 144 669 0 669 1 0 1 1 0 8 1
processpl 1232 716 0 669 5 0 5 5 0 8 0
procpl 664 1058 0 1003 6 0 6 6 0 8 1
sosppl 176 1 0 1 1 0 1 1 0 8 1
sockpl 752 409 0 381 7 0 7 7 0 8 3
mcl64k 65536 18 0 0 3 0 3 3 0 8 0
mcl16k 16384 14 0 0 2 0 2 2 0 8 0
mcl12k 12288 2 0 0 1 0 1 1 0 8 0
mcl9k 9216 2 0 0 1 0 1 1 0 8 0
mcl8k 8192 4 0 0 1 0 1 1 0 8 0
mcl4k 4096 122 0 0 16 0 16 16 0 8 0
mcl2k2 2112 2 0 0 1 0 1 1 0 8 0
mcl2k 2048 20 0 0 3 0 3 3 0 8 0
mtagpl 96 3 0 0 1 0 1 1 0 8 0
mbufpl 256 237 0 0 15 0 15 15 0 8 0
bufpl 280 3238 0 106 224 0 224 224 0 8 0
anonpl 32 5795 0 0 47 0 47 47 0 246 0
amapchunkpl 152 16773 0 16304 28 0 28 28 0 158 9
amappl16 200 2334 0 2306 5 3 2 5 0 8 0
amappl15 192 6 0 6 1 1 0 1 0 8 0
amappl14 184 533 0 532 1 0 1 1 0 8 0
amappl13 176 130 0 118 1 0 1 1 0 8 0
amappl12 168 950 0 919 2 0 2 2 0 8 0
amappl11 160 8 0 8 1 1 0 1 0 8 0
amappl10 152 99 0 85 1 0 1 1 0 8 0
amappl9 144 265 0 265 1 1 0 1 0 8 0
amappl8 136 112 0 110 1 0 1 1 0 8 0
amappl7 128 147 0 134 1 0 1 1 0 8 0
amappl6 120 166 0 164 1 0 1 1 0 8 0
amappl5 112 100 0 89 1 0 1 1 0 8 0
amappl4 104 297 0 277 1 0 1 1 0 8 0
amappl3 96 3215 0 3103 4 0 4 4 0 8 0
amappl2 88 544 0 483 2 0 2 2 0 8 0
amappl1 80 11366 0 10772 16 0 16 16 0 8 1
amappl 88 4560 0 4402 5 0 5 5 0 92 1
uvmvnodes 80 121 0 0 3 0 3 3 0 8 0
dma4096 4096 1 0 1 1 1 0 1 0 8 0
dma1024 1024 1 0 0 1 0 1 1 0 8 0
dma256 256 6 0 6 1 1 0 1 0 8 0
dma128 128 254 0 254 2 1 1 1 0 8 1
dma64 64 6 0 6 1 1 0 1 0 8 0
dma32 32 7 0 7 1 1 0 1 0 8 0
dma16 16 18 0 17 1 0 1 1 0 8 0
aobjpl 72 9 0 1 1 0 1 1 0 8 0
uaddrrnd 24 699 0 668 1 0 1 1 0 8 0
uaddrbest 32 2 0 0 1 0 1 1 0 8 0
uaddr 24 699 0 668 1 0 1 1 0 8 0
vmmpekpl 168 7375 0 7341 2 0 2 2 0 8 0
vmmpepl 168 52967 0 51067 97 0 97 97 0 357 11
vmsppl 488 698 0 668 5 0 5 5 0 8 1
rwobjpl 80 17431 0 16391 27 1 26 26 0 8 1
pdppl 4096 1405 0 1336 99 24 75 83 0 8 6
pvpl 32 14313 0 0 116 0 116 116 0 265 0
pmappl 256 698 0 668 3 0 3 3 0 8 0
extentpl 40 45 0 27 1 0 1 1 0 8 0
phpool 112 281 0 27 8 0 8 8 0 8 0
ddb{0}> machine ddbcpu 0
Invalid cpu 0
ddb{0}> trace
db_enter() at db_enter+0x25 sys/arch/amd64/amd64/db_interface.c:438
witness_checkorder(ffffffff83a4a1c8,9,0) at witness_checkorder+0x10d1 sys/kern/subr_witness.c:-1
mtx_enter(ffffffff83a4a1b8) at mtx_enter+0x95 sys/kern/kern_lock.c:406
sleep_setup(ffff8000299bee80,220,ffffffff834cb92c) at sleep_setup+0xed sys/kern/kern_synch.c:304
msleep_nsec(ffff8000299bee80,ffff8000299bee20,220,ffffffff834cb92c,ffffffffffffffff) at msleep_nsec+0x10b sys/kern/kern_synch.c:217
dt_ioctl_record_stop(ffff800001669000) at dt_ioctl_record_stop+0xbc sys/dev/dt/dt_dev.c:589
dtclose(21e5f,81,2000,ffff80003c3f6558) at dtclose+0x109 dt_pcb_purge sys/dev/dt/dt_dev.c:-1 [inline]
dtclose(21e5f,81,2000,ffff80003c3f6558) at dtclose+0x109 sys/dev/dt/dt_dev.c:239
spec_close(ffff80003c4017c0) at spec_close+0x466 sys/kern/spec_vnops.c:-1
VOP_CLOSE(fffffd805f7a62d8,81,fffffd80097fd2d8,ffff80003c3f6558) at VOP_CLOSE+0x132 sys/kern/vfs_vops.c:156
vn_closefile(fffffd806c19eea0,ffff80003c3f6558) at vn_closefile+0x12b vn_close sys/kern/vfs_vnops.c:292 [inline]
vn_closefile(fffffd806c19eea0,ffff80003c3f6558) at vn_closefile+0x12b sys/kern/vfs_vnops.c:615
fdrop(fffffd806c19eea0,ffff80003c3f6558) at fdrop+0x121 sys/kern/kern_descrip.c:1281
closef(fffffd806c19eea0,ffff80003c3f6558) at closef+0x192 sys/kern/kern_descrip.c:1265
syscall(ffff80003c401a20) at syscall+0xbd4 mi_syscall sys/sys/syscall_mi.h:176 [inline]
syscall(ffff80003c401a20) at syscall+0xbd4 sys/arch/amd64/amd64/trap.c:783
Xsyscall() at Xsyscall+0x128
end of kernel
end trace frame: 0xefd3ba36cb0, count: -14
ddb{0}> machine ddbcpu 1
Stopped at x86_ipi_db+0x27: addq $0x8,%rsp
ddb{1}> trace
x86_ipi_db(ffff8000299bdff0) at x86_ipi_db+0x27 sys/arch/amd64/amd64/db_interface.c:394
x86_ipi_handler() at x86_ipi_handler+0xd9 sys/arch/amd64/amd64/ipi.c:106
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x27
mtx_enter(ffff8000299bee20) at mtx_enter+0x377 sys/kern/kern_lock.c:454
clockintr_dispatch(ffff80002eba5180) at clockintr_dispatch+0x401 sys/kern/kern_clockintr.c:215
lapic_clockintr(0,0) at lapic_clockintr+0x51 sys/arch/amd64/amd64/lapic.c:482
Xresume_lapic_ltimer() at Xresume_lapic_ltimer+0x2a
end of kernel
end trace frame: 0x78a86ea8bdd0, count: -7