witness: lock order reversal: 1st 0xffff8000299dee08 &cq->cq_mtx (&cq->cq_mtx) 2nd 0xffffffff83952458 &sched_lock (&sched_lock) lock order [1] &cq->cq_mtx (&cq->cq_mtx) -> [2] &sched_lock (&sched_lock) #0 mtx_enter+0x4a sys/kern/kern_lock.c:260 #1 sleep_setup+0xc6 sys/kern/kern_synch.c:302 #2 msleep_nsec+0x119 sys/kern/kern_synch.c:217 #3 dt_ioctl_record_stop+0xbc sys/dev/dt/dt_dev.c:576 #4 dtclose+0x105 dt_pcb_purge sys/dev/dt/dt_dev.c:-1 [inline] #4 dtclose+0x105 sys/dev/dt/dt_dev.c:232 #5 spec_close+0x45f sys/kern/spec_vnops.c:-1 #6 VOP_CLOSE+0x133 sys/kern/vfs_vops.c:156 #7 vn_closefile+0x12b vn_close sys/kern/vfs_vnops.c:292 [inline] #7 vn_closefile+0x12b sys/kern/vfs_vnops.c:615 #8 fdrop+0x126 sys/kern/kern_descrip.c:1267 #9 closef+0x192 sys/kern/kern_descrip.c:1251 #10 fdfree+0x116 sys/kern/kern_descrip.c:1182 #11 exit1+0x59c sys/kern/kern_exit.c:215 #12 sys_exit+0x1a sys/kern/kern_exit.c:-1 #13 syscall+0xb08 mi_syscall sys/sys/syscall_mi.h:176 [inline] #13 syscall+0xb08 sys/arch/amd64/amd64/trap.c:748 #14 Xsyscall+0x128 lock order [2] &sched_lock (&sched_lock) -> [1] &cq->cq_mtx (&cq->cq_mtx) #0 mtx_enter+0x4a sys/kern/kern_lock.c:260 #1 clockintr_cancel+0x2f sys/kern/kern_clockintr.c:323 #2 mi_switch+0x172 x86_atomic_clearbits_u32 machine/atomic.h:-1 [inline] #2 mi_switch+0x172 sys/kern/sched_bsd.c:374 #3 sleep_finish+0x2da sys/kern/kern_synch.c:366 #4 sys_futex+0xa24 futex_wait sys/kern/sys_futex.c:301 [inline] #4 sys_futex+0xa24 sys/kern/sys_futex.c:151 #5 syscall+0xbc6 mi_syscall sys/sys/syscall_mi.h:176 [inline] #5 syscall+0xbc6 sys/arch/amd64/amd64/trap.c:748 #6 Xsyscall+0x128 Stopped at db_enter+0x25: addq $0x8,%rsp ddb{0}> ddb{0}> set $lines = 0 ddb{0}> set $maxwidth = 0 ddb{0}> show panic the kernel did not panic ddb{0}> trace db_enter() at db_enter+0x25 sys/arch/amd64/amd64/db_interface.c:438 witness_checkorder(ffffffff83952458,9,0) at witness_checkorder+0x10fb sys/kern/subr_witness.c:-1 mtx_enter(ffffffff83952448) at mtx_enter+0x4a sys/kern/kern_lock.c:260 sleep_setup(ffff8000299dee58,220,ffffffff833adfc6) at sleep_setup+0xc6 sys/kern/kern_synch.c:302 msleep_nsec(ffff8000299dee58,ffff8000299dedf8,220,ffffffff833adfc6,ffffffffffffffff) at msleep_nsec+0x119 sys/kern/kern_synch.c:217 dt_ioctl_record_stop(ffff800001478000) at dt_ioctl_record_stop+0xbc sys/dev/dt/dt_dev.c:576 dtclose(21e5f,81,2000,ffff800032ff8f80) at dtclose+0x105 dt_pcb_purge sys/dev/dt/dt_dev.c:-1 [inline] dtclose(21e5f,81,2000,ffff800032ff8f80) at dtclose+0x105 sys/dev/dt/dt_dev.c:232 spec_close(ffff80002ebeae20) at spec_close+0x45f sys/kern/spec_vnops.c:-1 VOP_CLOSE(fffffd8079029cd8,81,fffffd80097fb4e0,ffff800032ff8f80) at VOP_CLOSE+0x133 sys/kern/vfs_vops.c:156 vn_closefile(fffffd805dcbb910,ffff800032ff8f80) at vn_closefile+0x12b vn_close sys/kern/vfs_vnops.c:292 [inline] vn_closefile(fffffd805dcbb910,ffff800032ff8f80) at vn_closefile+0x12b sys/kern/vfs_vnops.c:615 fdrop(fffffd805dcbb910,ffff800032ff8f80) at fdrop+0x126 sys/kern/kern_descrip.c:1267 closef(fffffd805dcbb910,ffff800032ff8f80) at closef+0x192 sys/kern/kern_descrip.c:1251 fdfree(ffff800032ff8f80) at fdfree+0x116 sys/kern/kern_descrip.c:1182 exit1(ffff800032ff8f80,0,0,1) at exit1+0x59c sys/kern/kern_exit.c:215 sys_exit(ffff800032ff8f80,ffff80002ebeb190,ffff80002ebeb0e0) at sys_exit+0x1a sys/kern/kern_exit.c:-1 syscall(ffff80002ebeb190) at syscall+0xb08 mi_syscall sys/sys/syscall_mi.h:176 [inline] syscall(ffff80002ebeb190) at syscall+0xb08 sys/arch/amd64/amd64/trap.c:748 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0x70697113bd70, count: -17 ddb{0}> show registers rdi 0 rsi 0 rbp 0xffff80002ebeab20 rbx 0xfffffd800470b330 rdx 0 rcx 0xffff800032ff8f80 rax 0xffffffff837f0ff0 cpu_info_full_primary+0x1ff0 r8 0xffff80002ebeaa00 r9 0x8080808080808080 r10 0x38347a9147ff9e35 r11 0xa30708ff06f322e2 r12 0 r13 0xfffffd8003f5e300 r14 0x3 r15 0xffffffff rip 0xffffffff81c232f5 db_enter+0x25 cs 0x8 rflags 0x246 rsp 0xffff80002ebeab10 ss 0x10 db_enter+0x25: addq $0x8,%rsp ddb{0}> show proc PROC (syz-executor) tid=112134 pid=19961 tcnt=0 stat=onproc flags process=1008 proc=2000 runpri=32, usrpri=50, slppri=32, nice=20 wchan=0x0, wmesg=, ps_single=0xffff800032ff8f80 scnt=-1 ecnt=1 forw=0xffffffffffffffff, list=0xffff800032ff94a0,0xffff80002a2a99c8 process=0xffff8000334170e0 user=0xffff80002ebe6000, vmspace=0xfffffd800bb9d5d8 estcpu=36, cpticks=2, pctcpu=0.0, user=0, sys=0, intr=0 ddb{0}> ps PID TID PPID UID S FLAGS WAIT COMMAND 22424 14246 70387 0 2 0 syz-executor 76253 507626 87866 0 2 0 syz-executor 76253 447530 87866 0 2 0x4000000 syz-executor 76253 423871 87866 0 3 0x4000080 fsleep syz-executor 28069 69012 24682 0 2 0 syz-executor 28069 240363 24682 0 3 0x4000080 fifor syz-executor 13454 303922 59092 0 3 0x80 nanoslp syz-executor 13454 336600 59092 0 3 0x4000000 biowait syz-executor 13454 277127 59092 0 3 0x4000080 fsleep syz-executor 7175 86136 10042 0 3 0x3000 suspend syz-executor 7175 292650 10042 0 3 0x4081000 biowait syz-executor 7175 474381 10042 0 3 0x4081000 inode syz-executor 10042 301753 66216 0 3 0x82 nanoslp syz-executor 6540 48535 0 0 3 0x14200 acct acct 24682 89703 66216 0 3 0x82 nanoslp syz-executor 98486 438883 1 0 3 0x100083 ttyopn getty 91291 336492 0 0 3 0x14200 bored sosplice 65343 322369 66216 0 3 0x82 nanoslp syz-executor 70387 389773 66216 0 2 0x2 syz-executor 59092 167482 66216 0 2 0x2 syz-executor 60541 492009 66216 0 2 0x2 syz-executor 87866 176601 66216 0 3 0x82 nanoslp syz-executor 58445 470797 66216 0 3 0x82 nanoslp syz-executor 66216 316257 10856 0 2 0x2 syz-executor 10856 387724 78748 0 3 0x10008a sigsusp ksh 78748 124124 19275 0 3 0x98 kqread sshd-session 19275 16395 11420 0 3 0x92 kqread sshd-session 11420 158808 1 0 3 0x88 kqread sshd 32675 112602 72512 74 3 0x1100092 bpf pflogd 72512 331966 1 0 3 0x80 sbwait pflogd 79199 28264 87003 73 3 0x1100090 kqread syslogd 87003 65806 1 0 3 0x100082 sbwait syslogd 70202 18461 1 0 3 0x100080 kqread resolvd 99420 36966 9032 77 3 0x100092 kqread dhcpleased 52521 220513 9032 77 3 0x100092 kqread dhcpleased 9032 259477 1 0 3 0x80 kqread dhcpleased 500 158748 0 0 3 0x14200 bored smr 53230 294870 0 0 2 0x14200 zerothread 89597 454049 0 0 3 0x14200 aiodoned aiodoned 10989 468660 0 0 3 0x14200 syncer update 12576 250856 0 0 3 0x14200 cleaner cleaner 93336 476122 0 0 3 0x14200 reaper reaper 81669 368302 0 0 3 0x14200 pgdaemon pagedaemon 33213 253876 0 0 3 0x14200 bored viomb 6538 397840 0 0 3 0x40014200 acpi0 acpi0 16121 49826 0 0 7 0x40014200 idle1 80062 102428 0 0 3 0x14200 bored softnet7 380 118890 0 0 3 0x14200 bored softnet6 46461 439113 0 0 3 0x14200 bored softnet5 49071 129804 0 0 3 0x14200 bored softnet4 32925 477540 0 0 3 0x14200 bored softnet3 4320 465601 0 0 3 0x14200 bored softnet2 62583 195176 0 0 3 0x14200 bored softnet1 94839 433743 0 0 3 0x14200 bored softnet0 69929 135596 0 0 3 0x14200 bored systqmp 70878 196044 0 0 3 0x14200 bored systq 51494 154480 0 0 3 0x14200 tmoslp softclockmp 40071 426130 0 0 3 0x40014200 tmoslp softclock 30744 11785 0 0 3 0x40014200 idle0 1 254352 0 0 3 0x82 wait init 0 0 -1 0 3 0x10010200 scheduler swapper ddb{0}> show all locks CPU 0: exclusive mutex &cq->cq_mtx r = 0 (0xffff8000299dee08) #0 witness_lock+0x5bb stacktrace_save sys/sys/stacktrace.h:37 [inline] #0 witness_lock+0x5bb sys/kern/subr_witness.c:1160 #1 mtx_enter_try+0x1ad sys/kern/kern_lock.c:311 #2 mtx_enter+0x62 sys/kern/kern_lock.c:261 #3 clockintr_unbind+0x56 sys/kern/kern_clockintr.c:375 #4 dt_ioctl_record_stop+0xbc sys/dev/dt/dt_dev.c:576 #5 dtclose+0x105 dt_pcb_purge sys/dev/dt/dt_dev.c:-1 [inline] #5 dtclose+0x105 sys/dev/dt/dt_dev.c:232 #6 spec_close+0x45f sys/kern/spec_vnops.c:-1 #7 VOP_CLOSE+0x133 sys/kern/vfs_vops.c:156 #8 vn_closefile+0x12b vn_close sys/kern/vfs_vnops.c:292 [inline] #8 vn_closefile+0x12b sys/kern/vfs_vnops.c:615 #9 fdrop+0x126 sys/kern/kern_descrip.c:1267 #10 closef+0x192 sys/kern/kern_descrip.c:1251 #11 fdfree+0x116 sys/kern/kern_descrip.c:1182 #12 exit1+0x59c sys/kern/kern_exit.c:215 #13 sys_exit+0x1a sys/kern/kern_exit.c:-1 #14 syscall+0xb08 mi_syscall sys/sys/syscall_mi.h:176 [inline] #14 syscall+0xb08 sys/arch/amd64/amd64/trap.c:748 #15 Xsyscall+0x128 Process 13454 (syz-executor) thread 0xffff80003c448a68 (336600) exclusive rrwlock inode r = 0 (0xfffffd806b8558f8) #0 witness_lock+0x5bb stacktrace_save sys/sys/stacktrace.h:37 [inline] #0 witness_lock+0x5bb sys/kern/subr_witness.c:1160 #1 rw_do_enter_write+0x42b sys/kern/kern_rwlock.c:319 #2 rrw_enter+0xc6 sys/kern/kern_rwlock.c:621 #3 VOP_LOCK+0xa6 sys/kern/vfs_vops.c:527 #4 vn_lock+0xa4 sys/kern/vfs_vnops.c:570 #5 vn_write+0x190 sys/kern/vfs_vnops.c:405 #6 dofilewritev+0x23c sys/kern/sys_generic.c:380 #7 sys_write+0xa2 sys/kern/sys_generic.c:300 #8 syscall+0xbc6 mi_syscall sys/sys/syscall_mi.h:176 [inline] #8 syscall+0xbc6 sys/arch/amd64/amd64/trap.c:748 #9 Xsyscall+0x128 Process 7175 (syz-executor) thread 0xffff800032ff8a60 (292650) exclusive rrwlock inode r = 0 (0xfffffd806b855a20) #0 witness_lock+0x5bb stacktrace_save sys/sys/stacktrace.h:37 [inline] #0 witness_lock+0x5bb sys/kern/subr_witness.c:1160 #1 rw_do_enter_write+0x42b sys/kern/kern_rwlock.c:319 #2 rrw_enter+0xc6 sys/kern/kern_rwlock.c:621 #3 VOP_LOCK+0xa6 sys/kern/vfs_vops.c:527 #4 vn_lock+0xa4 sys/kern/vfs_vnops.c:570 #5 vn_write+0x190 sys/kern/vfs_vnops.c:405 #6 dofilewritev+0x23c sys/kern/sys_generic.c:380 #7 sys_write+0xa2 sys/kern/sys_generic.c:300 #8 syscall+0xbc6 mi_syscall sys/sys/syscall_mi.h:176 [inline] #8 syscall+0xbc6 sys/arch/amd64/amd64/trap.c:748 #9 Xsyscall+0x128 Process 7175 (syz-executor) thread 0xffff800032ff8540 (474381) exclusive rrwlock inode r = 0 (0xfffffd806e042488) #0 witness_lock+0x5bb stacktrace_save sys/sys/stacktrace.h:37 [inline] #0 witness_lock+0x5bb sys/kern/subr_witness.c:1160 #1 rw_do_enter_write+0x42b sys/kern/kern_rwlock.c:319 #2 rrw_enter+0xc6 sys/kern/kern_rwlock.c:621 #3 VOP_LOCK+0xa6 sys/kern/vfs_vops.c:527 #4 vn_lock+0xa4 sys/kern/vfs_vnops.c:570 #5 vfs_lookup+0x109 sys/kern/vfs_lookup.c:418 #6 namei+0x7aa sys/kern/vfs_lookup.c:250 #7 domknodat+0xb4 sys/kern/vfs_syscalls.c:1588 #8 syscall+0xb08 mi_syscall sys/sys/syscall_mi.h:176 [inline] #8 syscall+0xb08 sys/arch/amd64/amd64/trap.c:748 #9 Xsyscall+0x128 ddb{0}> show malloc Type InUse MemUse HighUse Limit Requests Type Lim devbuf 10222 11172K 11344K 166960K 12160 0 pcb 17 16K 18K 166960K 207 0 rtable 173 6K 7K 166960K 359 0 pf 33 17K 81K 166960K 100 0 ifaddr 37 6K 8K 166960K 75 0 ifgroup 51 2K 2K 166960K 117 0 sysctl 4 1K 9K 166960K 17 0 counters 66 36K 37K 166960K 120 0 ioctlops 0 0K 4K 166960K 1612 0 iov 0 0K 28K 166960K 45 0 mount 1 1K 1K 166960K 1 0 log 0 0K 0K 166960K 4 0 vnodes 1339 84K 85K 166960K 1959 0 UFS quota 1 32K 32K 166960K 1 0 UFS mount 5 36K 36K 166960K 5 0 shm 2 1K 5K 166960K 12 0 VM map 2 1K 1K 166960K 2 0 sem 12 0K 0K 166960K 38 0 dirhash 12 2K 2K 166960K 30 0 ACPI 1692 195K 286K 166960K 12470 0 file desc 17 61K 232K 166960K 798 0 sigio 0 0K 0K 166960K 9 0 proc 72 115K 164K 166960K 667 0 subproc 72 4K 4K 166960K 90 0 NFS srvsock 1 0K 0K 166960K 1 0 NFS daemon 1 16K 16K 166960K 1 0 ip_moptions 2 0K 0K 166960K 68 0 in_multi 87 6K 7K 166960K 142 0 ether_multi 1 0K 0K 166960K 3 0 mrt 0 0K 0K 166960K 4 0 ISOFS mount 1 32K 32K 166960K 1 0 MSDOSFS mount 1 16K 16K 166960K 1 0 ttys 97 440K 440K 166960K 97 0 exec 0 0K 1K 166960K 475 0 fusefs mount 1 32K 32K 166960K 1 0 tdb 3 0K 0K 166960K 3 0 VM swap 8 62K 64K 166960K 10 0 UVM amap 238 151K 165K 166960K 8449 0 UVM aobj 59 7K 7K 166960K 61 0 pinsyscall 42 84K 98K 166960K 1921 0 memdesc 1 4K 4K 166960K 1 0 crypto data 1 1K 1K 166960K 1 0 ip6_options 0 0K 0K 166960K 25 0 NDP 11 0K 1K 166960K 48 0 temp 51 8677K 8743K 166960K 27755 0 kqueue 14 22K 28K 166960K 134 0 SYN cache 2 16K 16K 166960K 2 0 ddb{0}> show all pools Name Size Requests Fail Releases Pgreq Pgrel Npage Hiwat Minpg Maxpg Idle plcache 128 26 0 0 1 0 1 1 0 8 0 rtpcb 120 170 0 167 3 2 1 3 0 8 0 rtentry 176 126 0 54 5 0 5 5 0 8 0 unpcb 144 563 0 542 7 5 2 6 0 8 1 syncache 336 4 0 4 2 2 0 1 0 8 0 tcpcb 736 307 0 302 7 3 4 7 0 8 3 arp 128 12 0 5 1 0 1 1 0 8 0 inpcb 328 1071 0 1059 26 17 9 15 0 8 7 nd6 144 21 0 7 1 0 1 1 0 8 0 pkpcb 40 3 0 3 2 2 0 1 0 8 0 kcovpl 48 10 0 2 1 0 1 1 0 8 0 ppxss 1192 16 0 16 3 2 1 1 0 8 1 pppxif 1504 4 0 4 3 3 0 1 0 8 0 pffrag 232 4 0 2 1 0 1 1 0 482 0 pffrnode 88 4 0 2 1 0 1 1 0 8 0 pffrent 40 7 0 5 1 0 1 1 0 8 0 pfosfp 40 1428 0 1005 5 0 5 5 0 8 0 pfosfpen 112 1428 0 714 21 0 21 21 0 8 0 pfstitem 24 51 0 21 1 0 1 1 0 8 0 pfstkey 128 52 0 22 2 0 2 2 0 8 0 pfstate 384 52 0 22 5 0 5 5 0 8 0 pfrule 1344 23 0 16 2 1 1 2 0 8 0 art_heap8 4096 3 0 0 3 0 3 3 0 8 0 art_heap4 256 624 0 236 32 3 29 30 0 8 4 art_table 40 627 0 236 5 0 5 5 0 8 0 art_node 32 126 0 63 1 0 1 1 0 8 0 sysvmsgpl 40 4 0 4 3 3 0 1 0 8 0 semupl 112 2 0 2 2 1 1 1 0 8 1 semapl 112 34 0 24 1 0 1 1 0 8 0 shmpl 112 58 0 2 2 0 2 2 0 8 0 dirhash 1024 29 0 12 3 0 3 3 0 8 0 dino2pl 256 2777 0 1271 95 0 95 95 0 8 0 ffsino 296 2777 0 1271 117 0 117 117 0 8 0 nchpl 144 3815 0 2114 64 0 64 64 0 8 0 rtmask 32 4 0 4 1 0 1 1 0 8 1 uvmvnodes 80 3175 0 0 65 0 65 65 0 8 0 vnodes 216 3175 0 0 177 0 177 177 0 8 0 namei 1024 13099 0 13098 3 2 1 1 0 8 0 percpumem 16 75 0 27 1 0 1 1 0 8 0 kstatmem 264 58 0 34 4 2 2 3 0 8 0 scxspl 216 22452 0 22450 13 12 1 8 1 8 0 plimitpl 152 144 0 125 1 0 1 1 0 8 0 sigapl 424 1079 0 1024 7 0 7 7 0 8 0 knotepl 120 427 0 0 13 0 13 13 0 8 0 kqueuepl 224 257 0 245 3 2 1 3 0 8 0 pipepl 344 186 0 157 3 0 3 3 0 8 0 fdescpl 528 1053 0 1022 3 0 3 3 0 8 0 filepl 160 6342 0 6109 30 14 16 18 0 8 5 lockfpl 104 309 0 306 1 0 1 1 0 8 0 lockfspl 48 137 0 134 1 0 1 1 0 8 0 sessionpl 144 26 0 17 1 0 1 1 0 8 0 pgrppl 48 45 0 28 1 0 1 1 0 8 0 ucredpl 104 934 0 921 1 0 1 1 0 8 0 zombiepl 144 1027 0 1024 1 0 1 1 0 8 0 processpl 1248 1079 0 1024 5 0 5 5 0 8 0 procpl 656 2094 0 2032 9 2 7 7 0 8 1 sosppl 168 3 0 3 2 1 1 1 0 8 1 sockpl 752 1824 0 1787 42 30 12 21 0 8 7 mcl64k 65536 4 0 0 1 0 1 1 0 8 0 mcl16k 16384 2 0 0 1 0 1 1 0 8 0 mcl8k 8192 2 0 0 1 0 1 1 0 8 0 mcl4k 4096 127 0 0 16 0 16 16 0 8 0 mcl2k2 2112 1 0 0 1 0 1 1 0 8 0 mcl2k 2048 21 0 0 3 0 3 3 0 8 0 mtagpl 96 10 0 0 1 0 1 1 0 8 0 mbufpl 256 161 0 0 11 0 11 11 0 8 0 bufpl 280 9182 0 3040 439 0 439 439 0 8 0 anonpl 32 10142 0 0 82 0 82 82 0 246 0 amapchunkpl 152 27990 0 27479 36 8 28 31 0 158 4 amappl16 200 3720 0 3469 48 26 22 22 0 8 1 amappl15 192 3 0 1 1 0 1 1 0 8 0 amappl14 184 118 0 106 1 0 1 1 0 8 0 amappl13 176 7 0 6 2 1 1 1 0 8 0 amappl12 168 1731 0 1701 3 1 2 2 0 8 0 amappl11 160 48 0 34 1 0 1 1 0 8 0 amappl10 152 10 0 10 1 1 0 1 0 8 0 amappl9 144 247 0 247 1 1 0 1 0 8 0 amappl8 136 25 0 22 1 0 1 1 0 8 0 amappl7 128 117 0 105 1 0 1 1 0 8 0 amappl6 120 200 0 197 1 0 1 1 0 8 0 amappl5 112 134 0 125 1 0 1 1 0 8 0 amappl4 104 302 0 281 1 0 1 1 0 8 0 amappl3 96 5411 0 5296 5 1 4 4 0 8 0 amappl2 88 675 0 613 2 0 2 2 0 8 0 amappl1 80 11174 0 10580 16 2 14 15 0 8 0 amappl 88 7627 0 7461 5 0 5 5 0 92 0 dma4096 4096 1 0 1 1 1 0 1 0 8 0 dma1024 1024 1 0 0 1 0 1 1 0 8 0 dma256 256 9 0 9 2 2 0 1 0 8 0 dma128 128 253 0 253 1 1 0 1 0 8 0 dma64 64 6 0 6 1 1 0 1 0 8 0 dma32 32 7 0 7 1 1 0 1 0 8 0 dma16 16 19 0 18 1 0 1 1 0 8 0 aobjpl 72 60 0 2 2 0 2 2 0 8 0 uaddrrnd 24 1053 0 1022 1 0 1 1 0 8 0 uaddrbest 32 2 0 0 1 0 1 1 0 8 0 uaddr 24 1053 0 1022 1 0 1 1 0 8 0 vmmpekpl 168 9953 0 9909 3 0 3 3 0 8 0 vmmpepl 168 71421 0 69206 123 15 108 108 0 357 0 vmsppl 488 1052 0 1022 6 1 5 5 0 8 0 rwobjpl 80 24303 0 19906 96 2 94 94 0 8 0 pdppl 4096 2114 0 2044 108 36 72 82 0 8 2 pvpl 32 17226 0 0 139 0 139 139 0 265 0 pmappl 256 1052 0 1022 3 0 3 3 0 8 0 extentpl 40 45 0 27 1 0 1 1 0 8 0 phpool 112 293 0 49 8 0 8 8 0 8 0 ddb{0}> machine ddbcpu 0 Invalid cpu 0 ddb{0}> trace db_enter() at db_enter+0x25 sys/arch/amd64/amd64/db_interface.c:438 witness_checkorder(ffffffff83952458,9,0) at witness_checkorder+0x10fb sys/kern/subr_witness.c:-1 mtx_enter(ffffffff83952448) at mtx_enter+0x4a sys/kern/kern_lock.c:260 sleep_setup(ffff8000299dee58,220,ffffffff833adfc6) at sleep_setup+0xc6 sys/kern/kern_synch.c:302 msleep_nsec(ffff8000299dee58,ffff8000299dedf8,220,ffffffff833adfc6,ffffffffffffffff) at msleep_nsec+0x119 sys/kern/kern_synch.c:217 dt_ioctl_record_stop(ffff800001478000) at dt_ioctl_record_stop+0xbc sys/dev/dt/dt_dev.c:576 dtclose(21e5f,81,2000,ffff800032ff8f80) at dtclose+0x105 dt_pcb_purge sys/dev/dt/dt_dev.c:-1 [inline] dtclose(21e5f,81,2000,ffff800032ff8f80) at dtclose+0x105 sys/dev/dt/dt_dev.c:232 spec_close(ffff80002ebeae20) at spec_close+0x45f sys/kern/spec_vnops.c:-1 VOP_CLOSE(fffffd8079029cd8,81,fffffd80097fb4e0,ffff800032ff8f80) at VOP_CLOSE+0x133 sys/kern/vfs_vops.c:156 vn_closefile(fffffd805dcbb910,ffff800032ff8f80) at vn_closefile+0x12b vn_close sys/kern/vfs_vnops.c:292 [inline] vn_closefile(fffffd805dcbb910,ffff800032ff8f80) at vn_closefile+0x12b sys/kern/vfs_vnops.c:615 fdrop(fffffd805dcbb910,ffff800032ff8f80) at fdrop+0x126 sys/kern/kern_descrip.c:1267 closef(fffffd805dcbb910,ffff800032ff8f80) at closef+0x192 sys/kern/kern_descrip.c:1251 fdfree(ffff800032ff8f80) at fdfree+0x116 sys/kern/kern_descrip.c:1182 exit1(ffff800032ff8f80,0,0,1) at exit1+0x59c sys/kern/kern_exit.c:215 sys_exit(ffff800032ff8f80,ffff80002ebeb190,ffff80002ebeb0e0) at sys_exit+0x1a sys/kern/kern_exit.c:-1 syscall(ffff80002ebeb190) at syscall+0xb08 mi_syscall sys/sys/syscall_mi.h:176 [inline] syscall(ffff80002ebeb190) at syscall+0xb08 sys/arch/amd64/amd64/trap.c:748 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0x70697113bd70, count: -17 ddb{0}> machine ddbcpu 1 Stopped at x86_ipi_db+0x27: addq $0x8,%rsp ddb{1}> trace x86_ipi_db(ffff8000299ddff0) at x86_ipi_db+0x27 sys/arch/amd64/amd64/db_interface.c:394 x86_ipi_handler() at x86_ipi_handler+0xd9 sys/arch/amd64/amd64/ipi.c:106 Xresume_lapic_ipi() at Xresume_lapic_ipi+0x27 mtx_enter(ffff8000299dedf8) at mtx_enter+0xd2 sys/kern/kern_lock.c:264 clockintr_dispatch(ffff80002a240020) at clockintr_dispatch+0x3b4 sys/kern/kern_clockintr.c:215 lapic_clockintr(0,0) at lapic_clockintr+0x51 sys/arch/amd64/amd64/lapic.c:489 Xresume_lapic_ltimer() at Xresume_lapic_ltimer+0x2a acpicpu_idle() at acpicpu_idle+0x41e sys/dev/acpi/acpicpu_x86.c:1218 sched_idle(ffff8000299ddff0) at sched_idle+0x4d8 sys/kern/kern_sched.c:191 end trace frame: 0x0, count: -9