witness: lock order reversal: 1st 0xffffffff837f9e08 &cq->cq_mtx (&cq->cq_mtx) 2nd 0xffffffff83902fe8 &sched_lock (&sched_lock) lock order [1] &cq->cq_mtx (&cq->cq_mtx) -> [2] &sched_lock (&sched_lock) #0 mtx_enter+0x4a sys/kern/kern_lock.c:260 #1 sleep_setup+0xc6 sys/kern/kern_synch.c:302 #2 msleep_nsec+0x10b sys/kern/kern_synch.c:217 #3 dt_ioctl_record_stop+0xbc sys/dev/dt/dt_dev.c:582 #4 dtclose+0x109 dt_pcb_purge sys/dev/dt/dt_dev.c:-1 [inline] #4 dtclose+0x109 sys/dev/dt/dt_dev.c:232 #5 spec_close+0x466 sys/kern/spec_vnops.c:-1 #6 VOP_CLOSE+0x132 sys/kern/vfs_vops.c:156 #7 vn_closefile+0x12b vn_close sys/kern/vfs_vnops.c:292 [inline] #7 vn_closefile+0x12b sys/kern/vfs_vnops.c:615 #8 fdrop+0x121 sys/kern/kern_descrip.c:1280 #9 closef+0x192 sys/kern/kern_descrip.c:1264 #10 syscall+0xbd4 mi_syscall sys/sys/syscall_mi.h:176 [inline] #10 syscall+0xbd4 sys/arch/amd64/amd64/trap.c:746 #11 Xsyscall+0x128 lock order [2] &sched_lock (&sched_lock) -> [1] &cq->cq_mtx (&cq->cq_mtx) #0 mtx_enter+0x4a sys/kern/kern_lock.c:260 #1 clockintr_cancel+0x2f sys/kern/kern_clockintr.c:323 #2 mi_switch+0x16f sys/kern/sched_bsd.c:378 #3 sleep_finish+0x2d8 sys/kern/kern_synch.c:367 #4 sys_futex+0xb90 futex_wait sys/kern/sys_futex.c:306 [inline] #4 sys_futex+0xb90 sys/kern/sys_futex.c:155 #5 syscall+0xbd4 mi_syscall sys/sys/syscall_mi.h:176 [inline] #5 syscall+0xbd4 sys/arch/amd64/amd64/trap.c:746 #6 Xsyscall+0x128 Stopped at db_enter+0x25: addq $0x8,%rsp ddb{1}> ddb{1}> set $lines = 0 ddb{1}> set $maxwidth = 0 ddb{1}> show panic the kernel did not panic ddb{1}> trace db_enter() at db_enter+0x25 sys/arch/amd64/amd64/db_interface.c:438 witness_checkorder(ffffffff83902fe8,9,0) at witness_checkorder+0x10b9 sys/kern/subr_witness.c:-1 mtx_enter(ffffffff83902fd8) at mtx_enter+0x4a sys/kern/kern_lock.c:260 sleep_setup(ffffffff837f9e58,220,ffffffff833f2a81) at sleep_setup+0xc6 sys/kern/kern_synch.c:302 msleep_nsec(ffffffff837f9e58,ffffffff837f9df8,220,ffffffff833f2a81,ffffffffffffffff) at msleep_nsec+0x10b sys/kern/kern_synch.c:217 dt_ioctl_record_stop(ffff800001604000) at dt_ioctl_record_stop+0xbc sys/dev/dt/dt_dev.c:582 dtclose(11e5f,81,2000,ffff800037009ca8) at dtclose+0x109 dt_pcb_purge sys/dev/dt/dt_dev.c:-1 [inline] dtclose(11e5f,81,2000,ffff800037009ca8) at dtclose+0x109 sys/dev/dt/dt_dev.c:232 spec_close(ffff80003c477770) at spec_close+0x466 sys/kern/spec_vnops.c:-1 VOP_CLOSE(fffffd80629ca710,81,fffffd80097fb618,ffff800037009ca8) at VOP_CLOSE+0x132 sys/kern/vfs_vops.c:156 vn_closefile(fffffd80654dbb00,ffff800037009ca8) at vn_closefile+0x12b vn_close sys/kern/vfs_vnops.c:292 [inline] vn_closefile(fffffd80654dbb00,ffff800037009ca8) at vn_closefile+0x12b sys/kern/vfs_vnops.c:615 fdrop(fffffd80654dbb00,ffff800037009ca8) at fdrop+0x121 sys/kern/kern_descrip.c:1280 closef(fffffd80654dbb00,ffff800037009ca8) at closef+0x192 sys/kern/kern_descrip.c:1264 syscall(ffff80003c4779d0) at syscall+0xbd4 mi_syscall sys/sys/syscall_mi.h:176 [inline] syscall(ffff80003c4779d0) at syscall+0xbd4 sys/arch/amd64/amd64/trap.c:746 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0x9b041331680, count: -14 ddb{1}> show registers rdi 0 rsi 0x80000 acpi_pdirpa+0x6be71 rbp 0xffff80003c477480 rbx 0 rdx 0xffff80000147c600 rcx 0xffff800037009ca8 rax 0x7ffff acpi_pdirpa+0x6be70 r8 0xffff80003c477360 r9 0x8080808080808080 r10 0xff106ce3393712fb r11 0x5d8abbb150cac7a6 r12 0xfffffd8003efcdc0 r13 0xfffffd80046eaa28 r14 0x3 r15 0xffffffff833e3067 substchar+0x1221e rip 0xffffffff8324dbb5 db_enter+0x25 cs 0x8 rflags 0x246 rsp 0xffff80003c477470 ss 0x10 db_enter+0x25: addq $0x8,%rsp ddb{1}> show proc PROC (syz-executor) tid=16747 pid=31190 tcnt=2 stat=onproc flags process=10 proc=4000000 runpri=32, usrpri=80, slppri=32, nice=20 wchan=0x0, wmesg=, ps_single=0x0 scnt=0 ecnt=0 forw=0xffffffffffffffff, list=0xffff80003c459258,0xffffffff83927c28 process=0xffff80003bc26b70 user=0xffff80003c472000, vmspace=0xfffffd80661a2208 estcpu=36, cpticks=0, pctcpu=0.0, user=0, sys=0, intr=0 ddb{1}> ps PID TID PPID UID S FLAGS WAIT COMMAND 31190 210396 81378 60928 7 0x10 syz-executor *31190 16747 81378 60928 7 0x4000010 syz-executor 59153 161244 13164 0 2 0 syz-executor 59153 32509 13164 0 3 0x4000080 sbwait syz-executor 17750 120125 61267 0 3 0x80 nanoslp syz-executor 17750 19694 61267 0 3 0x4000080 pipewr syz-executor 17750 196707 61267 0 3 0x4000080 pipewr syz-executor 68935 488528 34826 0 3 0x80 nanoslp syz-executor 68935 154992 34826 0 3 0x4000080 fsleep syz-executor 68935 288796 34826 0 3 0x4000080 sbwait syz-executor 1962 267087 42299 0 3 0x80 nanoslp syz-executor 1962 322573 42299 0 3 0x4000080 ttyout syz-executor 1962 75991 42299 0 3 0x4000080 fsleep syz-executor 21067 509117 0 0 3 0x14200 acct acct 81378 269894 8240 0 3 0x82 nanoslp syz-executor 72048 513939 1 0 3 0x100083 ttyin getty 42299 429626 8240 0 3 0x82 nanoslp syz-executor 60921 343325 0 0 3 0x14200 bored sosplice 66473 139271 8240 0 3 0x82 wait syz-executor 70652 349383 8240 0 3 0x82 nanoslp syz-executor 61267 92539 8240 0 3 0x82 nanoslp syz-executor 13164 295268 8240 0 3 0x82 nanoslp syz-executor 34826 353959 8240 0 3 0x82 nanoslp syz-executor 2834 282618 8240 0 3 0x82 nanoslp syz-executor 8240 310991 24030 0 3 0x82 kqread syz-executor 24030 208725 96719 0 3 0x10008a sigsusp ksh 96719 209978 58427 0 3 0x98 kqread sshd-session 58427 407741 56009 0 3 0x92 kqread sshd-session 56009 267181 1 0 3 0x88 kqread sshd 78568 326869 84173 74 3 0x1100092 bpf pflogd 84173 150110 1 0 3 0x80 sbwait pflogd 48646 304976 29322 73 3 0x1100090 kqread syslogd 29322 70887 1 0 3 0x100082 sbwait syslogd 59750 501671 1 0 3 0x100080 kqread resolvd 57274 269888 50814 77 3 0x100092 kqread dhcpleased 33365 290685 50814 77 3 0x100092 kqread dhcpleased 50814 284793 1 0 3 0x80 kqread dhcpleased 4447 201733 0 0 3 0x14200 bored smr 96718 349361 0 0 3 0x14200 pgzero zerothread 61173 385369 0 0 3 0x14200 aiodoned aiodoned 51016 481664 0 0 3 0x14200 syncer update 79306 306818 0 0 3 0x14200 cleaner cleaner 58493 301995 0 0 3 0x14200 reaper reaper 47344 165472 0 0 3 0x14200 pgdaemon pagedaemon 12447 51558 0 0 3 0x14200 bored viomb 86306 152187 0 0 3 0x40014200 acpi0 acpi0 77185 162750 0 0 3 0x40014200 idle1 50752 57537 0 0 3 0x14200 bored softnet7 87265 28780 0 0 3 0x14200 bored softnet6 14354 87491 0 0 3 0x14200 bored softnet5 41437 120780 0 0 3 0x14200 bored softnet4 58896 92512 0 0 3 0x14200 bored softnet3 85063 99612 0 0 3 0x14200 bored softnet2 96452 372981 0 0 3 0x14200 bored softnet1 67577 25282 0 0 3 0x14200 bored softnet0 1416 319777 0 0 3 0x14200 bored systqmp 24656 221749 0 0 3 0x14200 bored systq 53762 401840 0 0 3 0x14200 tmoslp softclockmp 19665 290774 0 0 3 0x40014200 tmoslp softclock 70322 324436 0 0 3 0x40014200 idle0 1 389789 0 0 3 0x82 wait init 0 0 -1 0 3 0x10010200 scheduler swapper ddb{1}> show all locks CPU 1: exclusive mutex &cq->cq_mtx r = 0 (0xffffffff837f9e08) #0 witness_lock+0x5f1 stacktrace_save sys/sys/stacktrace.h:37 [inline] #0 witness_lock+0x5f1 sys/kern/subr_witness.c:1160 #1 mtx_enter_try+0x1ad sys/kern/kern_lock.c:311 #2 mtx_enter+0x13b sys/kern/kern_lock.c:261 #3 clockintr_unbind+0x56 sys/kern/kern_clockintr.c:375 #4 dt_ioctl_record_stop+0xbc sys/dev/dt/dt_dev.c:582 #5 dtclose+0x109 dt_pcb_purge sys/dev/dt/dt_dev.c:-1 [inline] #5 dtclose+0x109 sys/dev/dt/dt_dev.c:232 #6 spec_close+0x466 sys/kern/spec_vnops.c:-1 #7 VOP_CLOSE+0x132 sys/kern/vfs_vops.c:156 #8 vn_closefile+0x12b vn_close sys/kern/vfs_vnops.c:292 [inline] #8 vn_closefile+0x12b sys/kern/vfs_vnops.c:615 #9 fdrop+0x121 sys/kern/kern_descrip.c:1280 #10 closef+0x192 sys/kern/kern_descrip.c:1264 #11 syscall+0xbd4 mi_syscall sys/sys/syscall_mi.h:176 [inline] #11 syscall+0xbd4 sys/arch/amd64/amd64/trap.c:746 #12 Xsyscall+0x128 Process 31190 (syz-executor) thread 0xffff800037009ca8 (16747) exclusive rwlock dtlk r = 0 (0xffffffff837709c8) #0 witness_lock+0x5f1 stacktrace_save sys/sys/stacktrace.h:37 [inline] #0 witness_lock+0x5f1 sys/kern/subr_witness.c:1160 #1 rw_do_enter_write+0x419 sys/kern/kern_rwlock.c:320 #2 dt_ioctl_record_stop+0x2e sys/dev/dt/dt_dev.c:565 #3 dtclose+0x109 dt_pcb_purge sys/dev/dt/dt_dev.c:-1 [inline] #3 dtclose+0x109 sys/dev/dt/dt_dev.c:232 #4 spec_close+0x466 sys/kern/spec_vnops.c:-1 #5 VOP_CLOSE+0x132 sys/kern/vfs_vops.c:156 #6 vn_closefile+0x12b vn_close sys/kern/vfs_vnops.c:292 [inline] #6 vn_closefile+0x12b sys/kern/vfs_vnops.c:615 #7 fdrop+0x121 sys/kern/kern_descrip.c:1280 #8 closef+0x192 sys/kern/kern_descrip.c:1264 #9 syscall+0xbd4 mi_syscall sys/sys/syscall_mi.h:176 [inline] #9 syscall+0xbd4 sys/arch/amd64/amd64/trap.c:746 #10 Xsyscall+0x128 exclusive kernel_lock &kernel_lock r = 0 (0xffffffff83873798) #0 witness_lock+0x5f1 stacktrace_save sys/sys/stacktrace.h:37 [inline] #0 witness_lock+0x5f1 sys/kern/subr_witness.c:1160 #1 vn_closefile+0x41 sys/kern/vfs_vnops.c:608 #2 fdrop+0x121 sys/kern/kern_descrip.c:1280 #3 closef+0x192 sys/kern/kern_descrip.c:1264 #4 syscall+0xbd4 mi_syscall sys/sys/syscall_mi.h:176 [inline] #4 syscall+0xbd4 sys/arch/amd64/amd64/trap.c:746 #5 Xsyscall+0x128 exclusive mutex &cq->cq_mtx r = 0 (0xffffffff837f9e08) #0 witness_lock+0x5f1 stacktrace_save sys/sys/stacktrace.h:37 [inline] #0 witness_lock+0x5f1 sys/kern/subr_witness.c:1160 #1 mtx_enter_try+0x1ad sys/kern/kern_lock.c:311 #2 mtx_enter+0x13b sys/kern/kern_lock.c:261 #3 clockintr_unbind+0x56 sys/kern/kern_clockintr.c:375 #4 dt_ioctl_record_stop+0xbc sys/dev/dt/dt_dev.c:582 #5 dtclose+0x109 dt_pcb_purge sys/dev/dt/dt_dev.c:-1 [inline] #5 dtclose+0x109 sys/dev/dt/dt_dev.c:232 #6 spec_close+0x466 sys/kern/spec_vnops.c:-1 #7 VOP_CLOSE+0x132 sys/kern/vfs_vops.c:156 #8 vn_closefile+0x12b vn_close sys/kern/vfs_vnops.c:292 [inline] #8 vn_closefile+0x12b sys/kern/vfs_vnops.c:615 #9 fdrop+0x121 sys/kern/kern_descrip.c:1280 #10 closef+0x192 sys/kern/kern_descrip.c:1264 #11 syscall+0xbd4 mi_syscall sys/sys/syscall_mi.h:176 [inline] #11 syscall+0xbd4 sys/arch/amd64/amd64/trap.c:746 #12 Xsyscall+0x128 Process 59153 (syz-executor) thread 0xffff80003c4594f0 (32509) exclusive rwlock sbufrcv r = 0 (0xffff8000014246e0) #0 witness_lock+0x5f1 stacktrace_save sys/sys/stacktrace.h:37 [inline] #0 witness_lock+0x5f1 sys/kern/subr_witness.c:1160 #1 rw_do_enter_write+0x419 sys/kern/kern_rwlock.c:320 #2 sblock+0xb6 sys/kern/uipc_socket2.c:536 #3 soreceive+0x27d sys/kern/uipc_socket.c:870 #4 recvit+0x40b sys/kern/uipc_syscalls.c:1078 #5 sys_recvmmsg+0x410 sys/kern/uipc_syscalls.c:969 #6 syscall+0xbd4 mi_syscall sys/sys/syscall_mi.h:176 [inline] #6 syscall+0xbd4 sys/arch/amd64/amd64/trap.c:746 #7 Xsyscall+0x128 ddb{1}> show malloc Type InUse MemUse HighUse Limit Requests Type Lim devbuf 10251 11079K 11869K 166960K 12127 0 pcb 17 14K 16K 166960K 297 0 rtable 189 11K 11K 166960K 432 0 pf 42 19K 26K 166960K 174 0 ifaddr 40 6K 7K 166960K 100 0 ifgroup 67 2K 3K 166960K 164 0 sysctl 3 1K 9K 166960K 18 0 counters 74 37K 38K 166960K 316 0 ioctlops 0 0K 4K 166960K 1801 0 iov 1 4K 18K 166960K 47 0 mount 1 1K 1K 166960K 1 0 log 0 0K 0K 166960K 4 0 vnodes 1410 89K 89K 166960K 2154 0 UFS quota 1 32K 32K 166960K 1 0 UFS mount 5 36K 36K 166960K 5 0 shm 2 1K 9K 166960K 17 0 VM map 2 1K 1K 166960K 2 0 sem 20 2K 2K 166960K 94 0 dirhash 15 2K 2K 166960K 21 0 ACPI 1692 195K 286K 166960K 12470 0 file desc 18 65K 240K 166960K 1014 0 sigio 0 0K 0K 166960K 163 0 proc 72 115K 180K 166960K 678 0 subproc 72 4K 4K 166960K 90 0 NFS srvsock 1 0K 0K 166960K 1 0 NFS daemon 1 16K 16K 166960K 1 0 ip_moptions 0 0K 0K 166960K 176 0 in_multi 66 5K 7K 166960K 162 0 ether_multi 1 0K 0K 166960K 7 0 mrt 0 0K 0K 166960K 5 0 ISOFS mount 1 32K 32K 166960K 1 0 MSDOSFS mount 1 16K 16K 166960K 1 0 ttys 217 970K 970K 166960K 217 0 exec 0 0K 1K 166960K 660 0 fusefs mount 1 32K 32K 166960K 1 0 tdb 3 0K 0K 166960K 3 0 VM swap 8 62K 64K 166960K 10 0 UVM amap 251 160K 177K 166960K 10467 0 UVM aobj 19 2K 2K 166960K 21 0 pinsyscall 43 86K 102K 166960K 2134 0 memdesc 1 4K 4K 166960K 1 0 crypto data 1 1K 1K 166960K 1 0 ip6_options 0 0K 0K 166960K 89 0 NDP 14 0K 2K 166960K 71 0 temp 63 8647K 8712K 166960K 47878 0 kqueue 13 20K 32K 166960K 221 0 SYN cache 2 16K 16K 166960K 2 0 ddb{1}> show all pools Name Size Requests Fail Releases Pgreq Pgrel Npage Hiwat Minpg Maxpg Idle plcache 128 26 0 0 1 0 1 1 0 8 0 rtpcb 120 90 0 87 1 0 1 1 0 8 0 rtentry 176 149 0 80 5 1 4 5 0 8 0 unpcb 144 745 0 724 6 2 4 4 0 8 3 syncache 336 9 0 9 2 1 1 1 0 8 1 tcpqe 32 4 0 4 1 0 1 1 0 8 1 tcpcb 736 295 0 286 4 2 2 4 0 8 0 arp 128 19 0 10 1 0 1 1 0 8 0 inpcb 328 1166 0 1151 8 0 8 8 0 8 6 nd6 144 23 0 10 1 0 1 1 0 8 0 pkpcb 40 3 0 3 1 0 1 1 0 8 1 kcovpl 48 10 0 2 1 0 1 1 0 8 0 ppxss 1192 113 0 112 1 0 1 1 0 8 0 pppxif 1504 4 0 3 2 1 1 1 0 8 0 pfstscr 40 2 0 2 1 1 0 1 0 8 0 pffrag 232 9 0 4 1 0 1 1 0 482 0 pffrnode 88 8 0 4 1 0 1 1 0 8 0 pffrent 40 112 0 105 1 0 1 1 0 8 0 pfosfp 40 1428 0 1005 5 0 5 5 0 8 0 pfosfpen 112 1428 0 714 21 0 21 21 0 8 0 pfrktable 1344 1 0 1 1 1 0 1 0 8 0 pfanchor 1288 3 0 0 1 0 1 1 0 8 0 pfstitem 24 21 0 9 1 0 1 1 0 8 0 pfstkey 128 24 0 12 1 0 1 1 0 8 0 pfstate 384 22 0 11 2 0 2 2 0 8 0 pfrule 1344 27 0 21 2 1 1 2 0 8 0 rttmr 136 1 0 1 1 1 0 1 0 8 0 art_heap8 4096 5 0 0 5 0 5 5 0 8 0 art_heap4 256 633 0 321 30 7 23 28 0 8 3 art_table 40 638 0 321 5 1 4 5 0 8 0 art_node 32 146 0 88 1 0 1 1 0 8 0 sysvmsgpl 40 13 0 9 1 0 1 1 0 8 0 semapl 112 87 0 69 1 0 1 1 0 8 0 shmpl 112 18 0 2 1 0 1 1 0 8 0 dirhash 1024 23 0 4 3 0 3 3 0 8 0 dino2pl 256 3258 0 1746 96 1 95 96 0 8 0 ffsino 296 3258 0 1746 118 0 118 118 0 8 0 nchpl 144 4593 0 2894 64 0 64 64 0 8 0 rtmask 32 15 0 15 1 0 1 1 0 8 1 uvmvnodes 80 3698 0 0 76 0 76 76 0 8 0 vnodes 216 3698 0 0 206 0 206 206 0 8 0 namei 1024 18651 0 18651 1 0 1 1 0 8 1 percpumem 16 173 0 121 1 0 1 1 0 8 0 kstatmem 264 104 0 72 3 0 3 3 0 8 0 scsiplug 72 5 0 5 2 1 1 1 0 8 1 scxspl 216 28730 0 28730 11 8 3 8 1 8 3 plimitpl 152 306 0 288 1 0 1 1 0 8 0 sigapl 424 1302 0 1246 9 2 7 9 0 8 0 knotepl 120 553 0 0 17 0 17 17 0 8 0 kqueuepl 224 400 0 390 3 1 2 3 0 8 1 pipepl 344 238 0 209 6 0 6 6 0 8 3 fdescpl 528 1257 0 1225 3 0 3 3 0 8 0 filepl 160 7922 0 7690 16 1 15 15 0 8 3 lockfpl 104 280 0 276 1 0 1 1 0 8 0 lockfspl 48 122 0 118 1 0 1 1 0 8 0 sessionpl 144 30 0 21 1 0 1 1 0 8 0 pgrppl 48 51 0 34 1 0 1 1 0 8 0 ucredpl 104 1372 0 1358 1 0 1 1 0 8 0 zombiepl 144 1249 0 1246 1 0 1 1 0 8 0 processpl 1232 1302 0 1246 6 1 5 6 0 8 0 procpl 664 2799 0 2735 8 1 7 8 0 8 0 sosppl 168 9 0 9 1 0 1 1 0 8 1 sockpl 752 2024 0 1985 19 8 11 17 0 8 6 mcl64k 65536 17 0 0 3 0 3 3 0 8 0 mcl16k 16384 2 0 0 1 0 1 1 0 8 0 mcl12k 12288 1 0 0 1 0 1 1 0 8 0 mcl9k 9216 1 0 0 1 0 1 1 0 8 0 mcl8k 8192 3 0 0 1 0 1 1 0 8 0 mcl4k 4096 122 0 0 16 0 16 16 0 8 0 mcl2k 2048 25 0 0 4 0 4 4 0 8 0 mtagpl 96 3 0 0 1 0 1 1 0 8 0 mbufpl 256 194 0 0 13 0 13 13 0 8 0 bufpl 280 11895 0 5752 440 0 440 440 0 8 0 anonpl 32 12044 0 0 98 0 98 98 0 246 0 amapchunkpl 152 35094 0 34558 31 5 26 29 0 158 2 amappl16 200 4663 0 4612 34 22 12 27 0 8 5 amappl15 192 8 0 7 1 0 1 1 0 8 0 amappl14 184 126 0 114 1 0 1 1 0 8 0 amappl13 176 4 0 4 1 1 0 1 0 8 0 amappl12 168 1946 0 1913 3 1 2 2 0 8 0 amappl11 160 52 0 38 1 0 1 1 0 8 0 amappl10 152 7 0 7 1 1 0 1 0 8 0 amappl9 144 257 0 257 1 1 0 1 0 8 0 amappl8 136 46 0 43 1 0 1 1 0 8 0 amappl7 128 115 0 103 1 0 1 1 0 8 0 amappl6 120 222 0 218 1 0 1 1 0 8 0 amappl5 112 147 0 137 1 0 1 1 0 8 0 amappl4 104 317 0 294 1 0 1 1 0 8 0 amappl3 96 6141 0 6039 4 1 3 4 0 8 0 amappl2 88 1526 0 1446 2 0 2 2 0 8 0 amappl1 80 13062 0 12453 17 2 15 16 0 8 0 amappl 88 9586 0 9408 5 0 5 5 0 92 0 dma32768 32768 1 0 1 1 1 0 1 0 8 0 dma4096 4096 1 0 1 1 1 0 1 0 8 0 dma1024 1024 1 0 0 1 0 1 1 0 8 0 dma256 256 8 0 8 2 2 0 1 0 8 0 dma128 128 254 0 254 2 1 1 1 0 8 1 dma64 64 6 0 6 1 1 0 1 0 8 0 dma32 32 7 0 7 1 1 0 1 0 8 0 dma16 16 18 0 17 1 0 1 1 0 8 0 aobjpl 72 20 0 2 1 0 1 1 0 8 0 uaddrrnd 24 1257 0 1225 1 0 1 1 0 8 0 uaddrbest 32 2 0 0 1 0 1 1 0 8 0 uaddr 24 1257 0 1225 1 0 1 1 0 8 0 vmmpekpl 168 12016 0 11967 3 0 3 3 0 8 0 vmmpepl 168 85980 0 83899 112 11 101 112 0 357 2 vmsppl 488 1256 0 1225 5 0 5 5 0 8 0 rwobjpl 80 29056 0 24364 99 0 99 99 0 8 0 pdppl 4096 2522 0 2450 104 32 72 84 0 8 0 pvpl 32 22903 0 0 185 0 185 185 0 265 0 pmappl 256 1256 0 1225 3 0 3 3 0 8 0 extentpl 40 45 0 27 1 0 1 1 0 8 0 phpool 112 291 0 47 8 0 8 8 0 8 0 ddb{1}> machine ddbcpu 0 Stopped at x86_ipi_db+0x27: addq $0x8,%rsp ddb{0}> trace x86_ipi_db(ffffffff837f8ff0) at x86_ipi_db+0x27 sys/arch/amd64/amd64/db_interface.c:394 x86_ipi_handler() at x86_ipi_handler+0xd9 sys/arch/amd64/amd64/ipi.c:106 Xresume_lapic_ipi() at Xresume_lapic_ipi+0x27 mtx_enter(ffffffff837f9df8) at mtx_enter+0xf0 sys/kern/kern_lock.c:264 clockintr_dispatch(ffff80003c44b230) at clockintr_dispatch+0x3b1 sys/kern/kern_clockintr.c:215 lapic_clockintr(0,0) at lapic_clockintr+0x51 sys/arch/amd64/amd64/lapic.c:484 Xresume_lapic_ltimer() at Xresume_lapic_ltimer+0x2a end of kernel end trace frame: 0x7382f7219a50, count: -7 ddb{0}> machine ddbcpu 1 Stopped at db_enter+0x25: addq $0x8,%rsp ddb{1}> trace db_enter() at db_enter+0x25 sys/arch/amd64/amd64/db_interface.c:438 witness_checkorder(ffffffff83902fe8,9,0) at witness_checkorder+0x10b9 sys/kern/subr_witness.c:-1 mtx_enter(ffffffff83902fd8) at mtx_enter+0x4a sys/kern/kern_lock.c:260 sleep_setup(ffffffff837f9e58,220,ffffffff833f2a81) at sleep_setup+0xc6 sys/kern/kern_synch.c:302 msleep_nsec(ffffffff837f9e58,ffffffff837f9df8,220,ffffffff833f2a81,ffffffffffffffff) at msleep_nsec+0x10b sys/kern/kern_synch.c:217 dt_ioctl_record_stop(ffff800001604000) at dt_ioctl_record_stop+0xbc sys/dev/dt/dt_dev.c:582 dtclose(11e5f,81,2000,ffff800037009ca8) at dtclose+0x109 dt_pcb_purge sys/dev/dt/dt_dev.c:-1 [inline] dtclose(11e5f,81,2000,ffff800037009ca8) at dtclose+0x109 sys/dev/dt/dt_dev.c:232 spec_close(ffff80003c477770) at spec_close+0x466 sys/kern/spec_vnops.c:-1 VOP_CLOSE(fffffd80629ca710,81,fffffd80097fb618,ffff800037009ca8) at VOP_CLOSE+0x132 sys/kern/vfs_vops.c:156 vn_closefile(fffffd80654dbb00,ffff800037009ca8) at vn_closefile+0x12b vn_close sys/kern/vfs_vnops.c:292 [inline] vn_closefile(fffffd80654dbb00,ffff800037009ca8) at vn_closefile+0x12b sys/kern/vfs_vnops.c:615 fdrop(fffffd80654dbb00,ffff800037009ca8) at fdrop+0x121 sys/kern/kern_descrip.c:1280 closef(fffffd80654dbb00,ffff800037009ca8) at closef+0x192 sys/kern/kern_descrip.c:1264 syscall(ffff80003c4779d0) at syscall+0xbd4 mi_syscall sys/sys/syscall_mi.h:176 [inline] syscall(ffff80003c4779d0) at syscall+0xbd4 sys/arch/amd64/amd64/trap.c:746 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0x9b041331680, count: -14