witness: lock order reversal: 1st 0xffff8000299dee30 &cq->cq_mtx (&cq->cq_mtx) 2nd 0xffffffff839c1760 &sched_lock (&sched_lock) lock order [1] &cq->cq_mtx (&cq->cq_mtx) -> [2] &sched_lock (&sched_lock) #0 mtx_enter+0x95 sys/kern/kern_lock.c:406 #1 sleep_setup+0xed sys/kern/kern_synch.c:304 #2 msleep_nsec+0x10b sys/kern/kern_synch.c:217 #3 dt_ioctl_record_stop+0xbc sys/dev/dt/dt_dev.c:589 #4 dtclose+0x109 dt_pcb_purge sys/dev/dt/dt_dev.c:-1 [inline] #4 dtclose+0x109 sys/dev/dt/dt_dev.c:239 #5 spec_close+0x466 sys/kern/spec_vnops.c:-1 #6 VOP_CLOSE+0x132 sys/kern/vfs_vops.c:156 #7 vn_closefile+0x12b vn_close sys/kern/vfs_vnops.c:292 [inline] #7 vn_closefile+0x12b sys/kern/vfs_vnops.c:615 #8 fdrop+0x121 sys/kern/kern_descrip.c:1280 #9 closef+0x192 sys/kern/kern_descrip.c:1264 #10 syscall+0xbd4 mi_syscall sys/sys/syscall_mi.h:176 [inline] #10 syscall+0xbd4 sys/arch/amd64/amd64/trap.c:775 #11 Xsyscall+0x128 lock order [2] &sched_lock (&sched_lock) -> [1] &cq->cq_mtx (&cq->cq_mtx) #0 mtx_enter+0x95 sys/kern/kern_lock.c:406 #1 clockintr_cancel+0x2f sys/kern/kern_clockintr.c:323 #2 mi_switch+0x187 sys/kern/sched_bsd.c:376 #3 sleep_finish+0x2d8 sys/kern/kern_synch.c:369 #4 sys_futex+0xb90 futex_wait sys/kern/sys_futex.c:306 [inline] #4 sys_futex+0xb90 sys/kern/sys_futex.c:155 #5 syscall+0xbd4 mi_syscall sys/sys/syscall_mi.h:176 [inline] #5 syscall+0xbd4 sys/arch/amd64/amd64/trap.c:775 #6 Xsyscall+0x128 Stopped at db_enter+0x25: addq $0x8,%rsp ddb{0}> ddb{0}> set $lines = 0 ddb{0}> set $maxwidth = 0 ddb{0}> show panic the kernel did not panic ddb{0}> trace db_enter() at db_enter+0x25 sys/arch/amd64/amd64/db_interface.c:438 witness_checkorder(ffffffff839c1760,9,0) at witness_checkorder+0x10d1 sys/kern/subr_witness.c:-1 mtx_enter(ffffffff839c1750) at mtx_enter+0x95 sys/kern/kern_lock.c:406 sleep_setup(ffff8000299dee80,220,ffffffff8343582c) at sleep_setup+0xed sys/kern/kern_synch.c:304 msleep_nsec(ffff8000299dee80,ffff8000299dee20,220,ffffffff8343582c,ffffffffffffffff) at msleep_nsec+0x10b sys/kern/kern_synch.c:217 dt_ioctl_record_stop(ffff80000164c000) at dt_ioctl_record_stop+0xbc sys/dev/dt/dt_dev.c:589 dtclose(21e5f,81,2000,ffff8000fffeed10) at dtclose+0x109 dt_pcb_purge sys/dev/dt/dt_dev.c:-1 [inline] dtclose(21e5f,81,2000,ffff8000fffeed10) at dtclose+0x109 sys/dev/dt/dt_dev.c:239 spec_close(ffff80002a30e750) at spec_close+0x466 sys/kern/spec_vnops.c:-1 VOP_CLOSE(fffffd805bf60950,81,fffffd80097fd750,ffff8000fffeed10) at VOP_CLOSE+0x132 sys/kern/vfs_vops.c:156 vn_closefile(fffffd805e388998,ffff8000fffeed10) at vn_closefile+0x12b vn_close sys/kern/vfs_vnops.c:292 [inline] vn_closefile(fffffd805e388998,ffff8000fffeed10) at vn_closefile+0x12b sys/kern/vfs_vnops.c:615 fdrop(fffffd805e388998,ffff8000fffeed10) at fdrop+0x121 sys/kern/kern_descrip.c:1280 closef(fffffd805e388998,ffff8000fffeed10) at closef+0x192 sys/kern/kern_descrip.c:1264 syscall(ffff80002a30e9b0) at syscall+0xbd4 mi_syscall sys/sys/syscall_mi.h:176 [inline] syscall(ffff80002a30e9b0) at syscall+0xbd4 sys/arch/amd64/amd64/trap.c:775 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0x513df6e28f0, count: -14 ddb{0}> show registers rdi 0 rsi 0x80000 acpi_pdirpa+0x6be71 rbp 0xffff80002a30e420 rbx 0 rdx 0xffff8000015dd7c0 rcx 0xffff8000fffeed10 rax 0x7ffff acpi_pdirpa+0x6be70 r8 0xffff80002a30e300 r9 0x8080808080808080 r10 0xd481c7aa9d3603fa r11 0xf6af044dffcb5d1a r12 0xfffffd8003f52e40 r13 0xfffffd80046fd330 r14 0x3 r15 0xffffffff8341c01d substchar+0x4423 rip 0xffffffff8278f895 db_enter+0x25 cs 0x8 rflags 0x246 rsp 0xffff80002a30e410 ss 0x10 db_enter+0x25: addq $0x8,%rsp ddb{0}> show proc PROC (syz-executor) tid=504921 pid=54402 tcnt=2 stat=onproc flags process=0 proc=4000000 runpri=32, usrpri=50, slppri=32, nice=20 wchan=0x0, wmesg=, ps_single=0x0 scnt=0 ecnt=0 forw=0xffffffffffffffff, list=0xffff8000fffee2b0,0xffffffff8394b178 process=0xffff8000383e8020 user=0xffff80002a309000, vmspace=0xfffffd8073ebb008 estcpu=36, cpticks=0, pctcpu=0.0, user=0, sys=0, intr=0 ddb{0}> ps PID TID PPID UID S FLAGS WAIT COMMAND 54402 459495 62424 0 2 0 syz-executor *54402 504921 62424 0 7 0x4000000 syz-executor 34183 283295 5765 0 2 0 syz-executor 50817 232310 87985 0 2 0 syz-executor 50817 213834 87985 0 3 0x4000080 fsleep syz-executor 50817 212962 87985 0 3 0x4000080 fsleep syz-executor 64719 40231 50337 0 2 0 syz-executor 64719 255979 50337 0 3 0x4000080 fsleep syz-executor 64719 339181 50337 0 3 0x4000080 fsleep syz-executor 83423 123623 25588 0 2 0 syz-executor 83423 402602 25588 0 3 0x4000080 kqsel syz-executor 83423 230307 25588 0 2 0x4000000 syz-executor 83423 227291 25588 0 3 0x4000080 fsleep syz-executor 87985 94188 15151 0 3 0x82 nanoslp syz-executor 99605 165453 15151 0 2 0x2 syz-executor 62424 311214 15151 0 3 0x82 nanoslp syz-executor 85355 259551 15151 0 2 0x2 syz-executor 71610 89421 0 0 3 0x14200 acct acct 25588 323377 15151 0 3 0x82 nanoslp syz-executor 50337 143001 15151 0 3 0x82 nanoslp syz-executor 5765 439958 15151 0 3 0x82 nanoslp syz-executor 263 355931 15151 0 2 0x2 syz-executor 15151 25659 85936 0 3 0x82 kqread syz-executor 85936 202 41756 0 3 0x10008a sigsusp ksh 41756 302258 29951 0 3 0x98 kqread sshd-session 29951 498355 66214 0 3 0x92 kqread sshd-session 47109 298450 1 0 3 0x100083 ttyopn getty 66214 348036 1 0 3 0x88 kqread sshd 34977 364611 72285 74 3 0x1100092 bpf pflogd 72285 252277 1 0 3 0x80 sbwait pflogd 86437 30468 53403 73 3 0x1100090 kqread syslogd 53403 334723 1 0 3 0x100082 sbwait syslogd 19149 327162 1 0 3 0x100080 kqread resolvd 90354 478911 0 0 3 0x14200 bored smr 74122 385260 0 0 2 0x14200 zerothread 97201 7899 0 0 3 0x14200 aiodoned aiodoned 97889 28902 0 0 3 0x14200 syncer update 64611 265954 0 0 3 0x14200 cleaner cleaner 30919 292852 0 0 3 0x14200 reaper reaper 54301 390493 0 0 3 0x14200 pgdaemon pagedaemon 60104 206258 0 0 3 0x14200 bored viomb 24558 83231 0 0 3 0x40014200 acpi0 acpi0 4315 240328 0 0 7 0x40014200 idle1 52867 481278 0 0 3 0x14200 bored softnet1 7111 405124 0 0 3 0x14200 bored softnet0 12835 56063 0 0 3 0x14200 bored systqmp 20448 330319 0 0 3 0x14200 bored systq 71838 49402 0 0 3 0x14200 tmoslp softclockmp 24557 469003 0 0 3 0x40014200 tmoslp softclock 85776 83640 0 0 3 0x40014200 idle0 1 106400 0 0 3 0x80082 wait init 0 0 -1 0 3 0x10010200 scheduler swapper ddb{0}> show all locks CPU 0: exclusive mutex &cq->cq_mtx r = 0 (0xffff8000299dee30) #0 witness_lock+0x5f1 stacktrace_save sys/sys/stacktrace.h:37 [inline] #0 witness_lock+0x5f1 sys/kern/subr_witness.c:1160 #1 mtx_enter+0x4b4 sys/kern/kern_lock.c:487 #2 clockintr_unbind+0x56 sys/kern/kern_clockintr.c:375 #3 dt_ioctl_record_stop+0xbc sys/dev/dt/dt_dev.c:589 #4 dtclose+0x109 dt_pcb_purge sys/dev/dt/dt_dev.c:-1 [inline] #4 dtclose+0x109 sys/dev/dt/dt_dev.c:239 #5 spec_close+0x466 sys/kern/spec_vnops.c:-1 #6 VOP_CLOSE+0x132 sys/kern/vfs_vops.c:156 #7 vn_closefile+0x12b vn_close sys/kern/vfs_vnops.c:292 [inline] #7 vn_closefile+0x12b sys/kern/vfs_vnops.c:615 #8 fdrop+0x121 sys/kern/kern_descrip.c:1280 #9 closef+0x192 sys/kern/kern_descrip.c:1264 #10 syscall+0xbd4 mi_syscall sys/sys/syscall_mi.h:176 [inline] #10 syscall+0xbd4 sys/arch/amd64/amd64/trap.c:775 #11 Xsyscall+0x128 Process 54402 (syz-executor) thread 0xffff8000fffeed10 (504921) exclusive rwlock dtlk r = 0 (0xffffffff83892248) #0 witness_lock+0x5f1 stacktrace_save sys/sys/stacktrace.h:37 [inline] #0 witness_lock+0x5f1 sys/kern/subr_witness.c:1160 #1 rw_do_enter_write+0x419 sys/kern/kern_rwlock.c:320 #2 dt_ioctl_record_stop+0x2e sys/dev/dt/dt_dev.c:572 #3 dtclose+0x109 dt_pcb_purge sys/dev/dt/dt_dev.c:-1 [inline] #3 dtclose+0x109 sys/dev/dt/dt_dev.c:239 #4 spec_close+0x466 sys/kern/spec_vnops.c:-1 #5 VOP_CLOSE+0x132 sys/kern/vfs_vops.c:156 #6 vn_closefile+0x12b vn_close sys/kern/vfs_vnops.c:292 [inline] #6 vn_closefile+0x12b sys/kern/vfs_vnops.c:615 #7 fdrop+0x121 sys/kern/kern_descrip.c:1280 #8 closef+0x192 sys/kern/kern_descrip.c:1264 #9 syscall+0xbd4 mi_syscall sys/sys/syscall_mi.h:176 [inline] #9 syscall+0xbd4 sys/arch/amd64/amd64/trap.c:775 #10 Xsyscall+0x128 exclusive kernel_lock &kernel_lock r = 0 (0xffffffff83948c00) #0 witness_lock+0x5f1 stacktrace_save sys/sys/stacktrace.h:37 [inline] #0 witness_lock+0x5f1 sys/kern/subr_witness.c:1160 #1 vn_closefile+0x41 sys/kern/vfs_vnops.c:608 #2 fdrop+0x121 sys/kern/kern_descrip.c:1280 #3 closef+0x192 sys/kern/kern_descrip.c:1264 #4 syscall+0xbd4 mi_syscall sys/sys/syscall_mi.h:176 [inline] #4 syscall+0xbd4 sys/arch/amd64/amd64/trap.c:775 #5 Xsyscall+0x128 exclusive mutex &cq->cq_mtx r = 0 (0xffff8000299dee30) #0 witness_lock+0x5f1 stacktrace_save sys/sys/stacktrace.h:37 [inline] #0 witness_lock+0x5f1 sys/kern/subr_witness.c:1160 #1 mtx_enter+0x4b4 sys/kern/kern_lock.c:487 #2 clockintr_unbind+0x56 sys/kern/kern_clockintr.c:375 #3 dt_ioctl_record_stop+0xbc sys/dev/dt/dt_dev.c:589 #4 dtclose+0x109 dt_pcb_purge sys/dev/dt/dt_dev.c:-1 [inline] #4 dtclose+0x109 sys/dev/dt/dt_dev.c:239 #5 spec_close+0x466 sys/kern/spec_vnops.c:-1 #6 VOP_CLOSE+0x132 sys/kern/vfs_vops.c:156 #7 vn_closefile+0x12b vn_close sys/kern/vfs_vnops.c:292 [inline] #7 vn_closefile+0x12b sys/kern/vfs_vnops.c:615 #8 fdrop+0x121 sys/kern/kern_descrip.c:1280 #9 closef+0x192 sys/kern/kern_descrip.c:1264 #10 syscall+0xbd4 mi_syscall sys/sys/syscall_mi.h:176 [inline] #10 syscall+0xbd4 sys/arch/amd64/amd64/trap.c:775 #11 Xsyscall+0x128 Process 83423 (syz-executor) thread 0xffff80002a262fa0 (230307) exclusive rwlock fdlock r = 0 (0xffff8000fffed720) #0 witness_lock+0x5f1 stacktrace_save sys/sys/stacktrace.h:37 [inline] #0 witness_lock+0x5f1 sys/kern/subr_witness.c:1160 #1 rw_do_enter_write+0x419 sys/kern/kern_rwlock.c:320 #2 doopenat+0x18e sys/kern/vfs_syscalls.c:1105 #3 syscall+0xbd4 mi_syscall sys/sys/syscall_mi.h:176 [inline] #3 syscall+0xbd4 sys/arch/amd64/amd64/trap.c:775 #4 Xsyscall+0x128 Process 99605 (syz-executor) thread 0xffff8000383f0030 (165453) exclusive rwlock vmmaplk r = 0 (0xfffffd806c976aa0) #0 witness_lock+0x5f1 stacktrace_save sys/sys/stacktrace.h:37 [inline] #0 witness_lock+0x5f1 sys/kern/subr_witness.c:1160 #1 rw_do_enter_write+0x419 sys/kern/kern_rwlock.c:320 #2 vm_map_lock_ln+0x12e sys/uvm/uvm_map.c:5165 #3 uvmspace_fork+0x12b sys/uvm/uvm_map.c:3746 #4 process_new+0x577 sys/kern/kern_fork.c:281 #5 fork1+0x3f6 sys/kern/kern_fork.c:-1 #6 syscall+0xb17 mi_syscall sys/sys/syscall_mi.h:176 [inline] #6 syscall+0xb17 sys/arch/amd64/amd64/trap.c:775 #7 Xsyscall+0x128 exclusive rwlock vmmaplk r = 0 (0xfffffd806c9764e8) #0 witness_lock+0x5f1 stacktrace_save sys/sys/stacktrace.h:37 [inline] #0 witness_lock+0x5f1 sys/kern/subr_witness.c:1160 #1 rw_do_enter_write+0x419 sys/kern/kern_rwlock.c:320 #2 vm_map_lock_ln+0x12e sys/uvm/uvm_map.c:5165 #3 uvmspace_fork+0x44 sys/uvm/uvm_map.c:3737 #4 process_new+0x577 sys/kern/kern_fork.c:281 #5 fork1+0x3f6 sys/kern/kern_fork.c:-1 #6 syscall+0xb17 mi_syscall sys/sys/syscall_mi.h:176 [inline] #6 syscall+0xb17 sys/arch/amd64/amd64/trap.c:775 #7 Xsyscall+0x128 Process 85355 (syz-executor) thread 0xffff8000fffee548 (259551) exclusive rwlock vmmaplk r = 0 (0xfffffd8073ebb6c0) #0 witness_lock+0x5f1 stacktrace_save sys/sys/stacktrace.h:37 [inline] #0 witness_lock+0x5f1 sys/kern/subr_witness.c:1160 #1 rw_do_enter_write+0x419 sys/kern/kern_rwlock.c:320 #2 vm_map_lock_ln+0x12e sys/uvm/uvm_map.c:5165 #3 uvmspace_fork+0x12b sys/uvm/uvm_map.c:3746 #4 process_new+0x577 sys/kern/kern_fork.c:281 #5 fork1+0x3f6 sys/kern/kern_fork.c:-1 #6 syscall+0xb17 mi_syscall sys/sys/syscall_mi.h:176 [inline] #6 syscall+0xb17 sys/arch/amd64/amd64/trap.c:775 #7 Xsyscall+0x128 exclusive rwlock vmmaplk r = 0 (0xfffffd8073ebb8a8) #0 witness_lock+0x5f1 stacktrace_save sys/sys/stacktrace.h:37 [inline] #0 witness_lock+0x5f1 sys/kern/subr_witness.c:1160 #1 rw_do_enter_write+0x419 sys/kern/kern_rwlock.c:320 #2 vm_map_lock_ln+0x12e sys/uvm/uvm_map.c:5165 #3 uvmspace_fork+0x44 sys/uvm/uvm_map.c:3737 #4 process_new+0x577 sys/kern/kern_fork.c:281 #5 fork1+0x3f6 sys/kern/kern_fork.c:-1 #6 syscall+0xb17 mi_syscall sys/sys/syscall_mi.h:176 [inline] #6 syscall+0xb17 sys/arch/amd64/amd64/trap.c:775 #7 Xsyscall+0x128 Process 263 (syz-executor) thread 0xffff80002a2627d8 (355931) exclusive rwlock vmmaplk r = 0 (0xfffffd8073ebbe60) #0 witness_lock+0x5f1 stacktrace_save sys/sys/stacktrace.h:37 [inline] #0 witness_lock+0x5f1 sys/kern/subr_witness.c:1160 #1 rw_do_enter_write+0x419 sys/kern/kern_rwlock.c:320 #2 vm_map_lock_ln+0x12e sys/uvm/uvm_map.c:5165 #3 uvmspace_fork+0x12b sys/uvm/uvm_map.c:3746 #4 process_new+0x577 sys/kern/kern_fork.c:281 #5 fork1+0x3f6 sys/kern/kern_fork.c:-1 #6 syscall+0xb17 mi_syscall sys/sys/syscall_mi.h:176 [inline] #6 syscall+0xb17 sys/arch/amd64/amd64/trap.c:775 #7 Xsyscall+0x128 exclusive rwlock vmmaplk r = 0 (0xfffffd8073ebb4d8) #0 witness_lock+0x5f1 stacktrace_save sys/sys/stacktrace.h:37 [inline] #0 witness_lock+0x5f1 sys/kern/subr_witness.c:1160 #1 rw_do_enter_write+0x419 sys/kern/kern_rwlock.c:320 #2 vm_map_lock_ln+0x12e sys/uvm/uvm_map.c:5165 #3 uvmspace_fork+0x44 sys/uvm/uvm_map.c:3737 #4 process_new+0x577 sys/kern/kern_fork.c:281 #5 fork1+0x3f6 sys/kern/kern_fork.c:-1 #6 syscall+0xb17 mi_syscall sys/sys/syscall_mi.h:176 [inline] #6 syscall+0xb17 sys/arch/amd64/amd64/trap.c:775 #7 Xsyscall+0x128 ddb{0}> show malloc Type InUse MemUse HighUse Limit Requests Type Lim devbuf 11111 12174K 13950K 166960K 15607 0 pcb 17 16K 17K 166960K 1102 0 rtable 229 16K 16K 166960K 1281 0 pf 42 19K 82K 166960K 474 0 ifaddr 38 7K 8K 166960K 324 0 ifgroup 64 2K 3K 166960K 575 0 sysctl 4 1K 9K 166960K 32 0 counters 74 37K 38K 166960K 620 0 ioctlops 0 0K 4K 166960K 2609 0 iov 0 0K 32K 166960K 485 0 mount 1 1K 1K 166960K 1 0 log 0 0K 0K 166960K 4 0 vnodes 1574 99K 99K 166960K 4817 0 UFS quota 1 32K 32K 166960K 1 0 UFS mount 5 36K 36K 166960K 5 0 shm 2 2K 5K 166960K 7 0 VM map 2 1K 1K 166960K 2 0 sem 12 1K 2K 166960K 298 0 dirhash 12 2K 2K 166960K 60 0 ACPI 1692 195K 286K 166960K 12470 0 file desc 18 65K 236K 166960K 3828 0 sigio 0 0K 0K 166960K 65 0 proc 66 83K 180K 166960K 1332 0 subproc 72 4K 4K 166960K 217 0 NFS srvsock 1 0K 0K 166960K 1 0 NFS daemon 1 16K 16K 166960K 1 0 ip_moptions 0 0K 0K 166960K 672 0 in_multi 69 4K 7K 166960K 465 0 ether_multi 1 0K 0K 166960K 49 0 mrt 2 0K 0K 166960K 34 0 ISOFS mount 1 32K 32K 166960K 1 0 MSDOSFS mount 1 16K 16K 166960K 1 0 ttys 241 1076K 1076K 166960K 241 0 exec 0 0K 1K 166960K 1080 0 fusefs mount 1 32K 32K 166960K 1 0 pfkey data 0 0K 0K 166960K 5 0 tdb 3 0K 0K 166960K 3 0 VM swap 8 62K 64K 166960K 10 0 UVM amap 202 138K 181K 166960K 36162 0 UVM aobj 162 141K 141K 166960K 166 0 pinsyscall 37 74K 106K 166960K 5346 0 memdesc 1 4K 4K 166960K 1 0 crypto data 1 1K 1K 166960K 1 0 ip6_options 0 0K 1K 166960K 261 0 NDP 16 0K 2K 166960K 251 0 temp 81 8688K 8769K 166960K 187322 0 kqueue 7 12K 30K 166960K 797 0 SYN cache 2 16K 16K 166960K 2 0 ddb{0}> show all pools Name Size Requests Fail Releases Pgreq Pgrel Npage Hiwat Minpg Maxpg Idle plcache 128 26 0 0 1 0 1 1 0 8 0 rtpcb 120 503 0 502 3 2 1 2 0 8 0 rtentry 176 398 0 321 6 2 4 6 0 8 0 unpcb 144 2774 0 2764 17 16 1 6 0 8 0 syncache 336 8 0 8 5 5 0 1 0 8 0 tcpqe 32 1 0 1 1 1 0 1 0 8 0 tcpcb 736 1339 0 1335 24 23 1 7 0 8 0 arp 136 66 0 47 1 0 1 1 0 8 0 inpcb 328 4395 0 4390 44 39 5 10 0 8 3 ip6q 72 1 0 1 1 1 0 1 0 8 0 ip6af 40 2 0 2 1 1 0 1 0 8 0 nd6 152 80 0 64 1 0 1 1 0 8 0 pkpcb 40 29 0 29 10 9 1 1 0 8 1 kcovpl 48 24 0 16 1 0 1 1 0 8 0 mppekey 1024 2 0 2 2 2 0 1 0 8 0 ppxss 1192 212 0 212 3 2 1 1 0 8 1 pppxif 1504 21 0 21 7 7 0 1 0 8 0 pffrag 232 31 0 27 1 0 1 1 0 482 0 pffrnode 88 29 0 25 1 0 1 1 0 8 0 pffrent 40 52 0 48 1 0 1 1 0 8 0 pfosfp 40 1428 0 1005 5 0 5 5 0 8 0 pfosfpen 112 1428 0 714 21 0 21 21 0 8 0 pfrktable 1344 9 0 6 1 0 1 1 0 8 0 pfanchor 1288 1 0 0 1 0 1 1 0 8 0 pftag 88 1 0 0 1 0 1 1 0 8 0 pfrule 1360 3 0 2 1 0 1 1 0 8 0 rttmr 136 8 0 8 5 5 0 1 0 8 0 art_heap8 4096 5 0 0 5 0 5 5 0 8 0 art_heap4 256 1797 0 1486 40 20 20 31 0 8 0 art_table 40 1802 0 1486 5 1 4 5 0 8 0 art_node 32 392 0 327 1 0 1 1 0 8 0 sysvmsgpl 40 22 0 11 1 0 1 1 0 8 0 semupl 112 2 0 2 2 2 0 1 0 8 0 semapl 112 292 0 282 1 0 1 1 0 8 0 shmpl 112 162 0 4 5 0 5 5 0 8 0 dirhash 1024 49 0 32 3 0 3 3 0 8 0 dino2pl 256 8976 0 7437 97 0 97 97 0 8 0 ffsino 296 8976 0 7437 120 1 119 119 0 8 0 nchpl 144 13983 0 12251 65 0 65 65 0 8 0 rtmask 32 40 0 40 10 10 0 1 0 8 0 vnodes 216 5926 0 0 330 0 330 330 0 8 0 namei 1024 50984 0 50984 12 10 2 3 0 8 2 percpumem 16 325 0 273 1 0 1 1 0 8 0 vcpupl 3968 1 0 0 1 0 1 1 0 8 0 vmpool 848 1 0 0 1 0 1 1 0 8 0 pfiaddrpl 120 1 0 0 1 0 1 1 0 8 0 kstatmem 264 388 0 356 5 2 3 3 0 8 0 acpiwqpl 32 3 0 3 1 0 1 1 1 8 1 scsiplug 72 15 0 15 5 4 1 1 0 8 1 scxspl 216 110684 0 110684 19 18 1 8 1 8 1 plimitpl 152 906 0 890 1 0 1 1 0 8 0 sigapl 424 4102 0 4056 9 2 7 8 0 8 0 knotepl 120 818 0 0 24 0 24 24 0 8 0 kqueuepl 224 1490 0 1482 14 13 1 5 0 8 0 pipepl 344 699 0 672 18 9 9 9 0 8 6 fdescpl 528 4052 0 4023 3 0 3 3 0 8 0 filepl 160 29232 0 29035 52 32 20 20 0 8 7 lockfpl 104 1499 0 1498 2 1 1 2 0 8 0 lockfspl 48 599 0 598 1 0 1 1 0 8 0 sessionpl 144 41 0 33 1 0 1 1 0 8 0 pgrppl 48 115 0 99 1 0 1 1 0 8 0 ucredpl 104 5871 0 5860 1 0 1 1 0 8 0 zombiepl 144 5312 0 5312 1 0 1 1 0 8 1 processpl 1232 4102 0 4056 6 2 4 6 0 8 0 procpl 664 10226 0 10172 8 2 6 8 0 8 0 sosppl 176 34 0 34 6 5 1 1 0 8 1 sockpl 752 7884 0 7868 83 76 7 18 0 8 4 mcl64k 65536 10 0 0 2 0 2 2 0 8 0 mcl16k 16384 3 0 0 1 0 1 1 0 8 0 mcl12k 12288 1 0 0 1 0 1 1 0 8 0 mcl9k 9216 1 0 0 1 0 1 1 0 8 0 mcl8k 8192 2 0 0 1 0 1 1 0 8 0 mcl4k 4096 115 0 0 14 0 14 14 0 8 0 mcl2k2 2112 1 0 0 1 0 1 1 0 8 0 mcl2k 2048 31 0 0 4 0 4 4 0 8 0 mtagpl 96 9 0 0 1 0 1 1 0 8 0 mbufpl 256 1213 0 0 75 0 75 75 0 8 0 bufpl 280 46513 0 40377 439 0 439 439 0 8 0 anonpl 32 13198 0 0 108 1 107 107 0 246 0 amapchunkpl 152 124447 0 123983 75 47 28 35 0 158 5 amappl16 200 15886 0 15861 88 73 15 30 0 8 3 amappl15 192 4 0 4 1 1 0 1 0 8 0 amappl14 184 4 0 3 1 0 1 1 0 8 0 amappl13 176 628 0 627 1 0 1 1 0 8 0 amappl12 168 4525 0 4491 3 0 3 3 0 8 0 amappl11 160 23 0 22 2 1 1 1 0 8 0 amappl10 152 66 0 57 1 0 1 1 0 8 0 amappl9 144 293 0 292 2 1 1 1 0 8 0 amappl8 136 40 0 37 1 0 1 1 0 8 0 amappl7 128 142 0 140 1 0 1 1 0 8 0 amappl6 120 474 0 463 1 0 1 1 0 8 0 amappl5 112 79 0 70 1 0 1 1 0 8 0 amappl4 104 528 0 503 1 0 1 1 0 8 0 amappl3 96 22273 0 22196 4 1 3 3 0 8 0 amappl2 88 4141 0 4083 2 0 2 2 0 8 0 amappl1 80 26034 0 25536 16 2 14 15 0 8 0 amappl 88 34523 0 34379 5 0 5 5 0 92 0 uvmvnodes 80 216 0 0 5 0 5 5 0 8 0 dma65536 65536 1 0 1 1 1 0 1 0 8 0 dma8192 8192 2 0 2 2 2 0 1 0 8 0 dma4096 4096 1 0 1 1 1 0 1 0 8 0 dma2048 2048 1 0 1 1 1 0 1 0 8 0 dma1024 1024 1 0 0 1 0 1 1 0 8 0 dma256 256 7 0 7 2 2 0 1 0 8 0 dma128 128 260 0 260 4 3 1 1 0 8 1 dma64 64 12 0 12 6 6 0 1 0 8 0 dma32 32 7 0 7 1 1 0 1 0 8 0 dma16 16 19 0 18 1 0 1 1 0 8 0 aobjpl 72 165 0 4 3 0 3 3 0 8 0 uaddrrnd 24 4052 0 4023 1 0 1 1 0 8 0 uaddrbest 32 2 0 0 1 0 1 1 0 8 0 uaddr 24 4052 0 4023 1 0 1 1 0 8 0 vmmpekpl 168 29875 0 29809 4 0 4 4 0 8 0 vmmpepl 168 260114 0 258472 138 48 90 110 0 357 1 vmsppl 488 4051 0 4023 6 1 5 5 0 8 1 rwobjpl 80 68731 0 67583 36 3 33 34 0 8 0 pdppl 4096 8113 0 8047 137 71 66 85 0 8 0 pvpl 32 21487 0 0 173 0 173 173 0 265 0 pmappl 256 4052 0 4023 3 0 3 3 0 8 0 extentpl 40 45 0 27 1 0 1 1 0 8 0 phpool 112 434 0 138 10 0 10 10 0 8 0 ddb{0}> machine ddbcpu 0 Invalid cpu 0 ddb{0}> trace db_enter() at db_enter+0x25 sys/arch/amd64/amd64/db_interface.c:438 witness_checkorder(ffffffff839c1760,9,0) at witness_checkorder+0x10d1 sys/kern/subr_witness.c:-1 mtx_enter(ffffffff839c1750) at mtx_enter+0x95 sys/kern/kern_lock.c:406 sleep_setup(ffff8000299dee80,220,ffffffff8343582c) at sleep_setup+0xed sys/kern/kern_synch.c:304 msleep_nsec(ffff8000299dee80,ffff8000299dee20,220,ffffffff8343582c,ffffffffffffffff) at msleep_nsec+0x10b sys/kern/kern_synch.c:217 dt_ioctl_record_stop(ffff80000164c000) at dt_ioctl_record_stop+0xbc sys/dev/dt/dt_dev.c:589 dtclose(21e5f,81,2000,ffff8000fffeed10) at dtclose+0x109 dt_pcb_purge sys/dev/dt/dt_dev.c:-1 [inline] dtclose(21e5f,81,2000,ffff8000fffeed10) at dtclose+0x109 sys/dev/dt/dt_dev.c:239 spec_close(ffff80002a30e750) at spec_close+0x466 sys/kern/spec_vnops.c:-1 VOP_CLOSE(fffffd805bf60950,81,fffffd80097fd750,ffff8000fffeed10) at VOP_CLOSE+0x132 sys/kern/vfs_vops.c:156 vn_closefile(fffffd805e388998,ffff8000fffeed10) at vn_closefile+0x12b vn_close sys/kern/vfs_vnops.c:292 [inline] vn_closefile(fffffd805e388998,ffff8000fffeed10) at vn_closefile+0x12b sys/kern/vfs_vnops.c:615 fdrop(fffffd805e388998,ffff8000fffeed10) at fdrop+0x121 sys/kern/kern_descrip.c:1280 closef(fffffd805e388998,ffff8000fffeed10) at closef+0x192 sys/kern/kern_descrip.c:1264 syscall(ffff80002a30e9b0) at syscall+0xbd4 mi_syscall sys/sys/syscall_mi.h:176 [inline] syscall(ffff80002a30e9b0) at syscall+0xbd4 sys/arch/amd64/amd64/trap.c:775 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0x513df6e28f0, count: -14 ddb{0}> machine ddbcpu 1 Stopped at x86_ipi_db+0x27: addq $0x8,%rsp ddb{1}> trace x86_ipi_db(ffff8000299ddff0) at x86_ipi_db+0x27 sys/arch/amd64/amd64/db_interface.c:394 x86_ipi_handler() at x86_ipi_handler+0xd9 sys/arch/amd64/amd64/ipi.c:106 Xresume_lapic_ipi() at Xresume_lapic_ipi+0x27 __sanitizer_cov_trace_pc() at __sanitizer_cov_trace_pc+0xf kd_curproc sys/dev/kcov.c:580 [inline] __sanitizer_cov_trace_pc() at __sanitizer_cov_trace_pc+0xf sys/dev/kcov.c:153 clockintr_dispatch(ffff80002a21cbe0) at clockintr_dispatch+0x401 sys/kern/kern_clockintr.c:215 lapic_clockintr(0,0) at lapic_clockintr+0x51 sys/arch/amd64/amd64/lapic.c:482 Xresume_lapic_ltimer() at Xresume_lapic_ltimer+0x2a acpicpu_idle() at acpicpu_idle+0x457 sys/dev/acpi/acpicpu_x86.c:1224 sched_idle(ffff8000299ddff0) at sched_idle+0x391 sys/kern/kern_sched.c:191 end trace frame: 0x0, count: -9