witness: lock order reversal: 1st 0xffff8000299dee00 &cq->cq_mtx (&cq->cq_mtx) 2nd 0xffffffff83915080 &sched_lock (&sched_lock) lock order [1] &cq->cq_mtx (&cq->cq_mtx) -> [2] &sched_lock (&sched_lock) #0 mtx_enter+0x47 sys/kern/kern_lock.c:238 #1 sleep_setup+0xc6 sys/kern/kern_synch.c:348 #2 msleep+0x114 sys/kern/kern_synch.c:245 #3 dt_ioctl_record_stop+0xbc sys/dev/dt/dt_dev.c:575 #4 dtclose+0x105 dt_pcb_purge sys/dev/dt/dt_dev.c:-1 [inline] #4 dtclose+0x105 sys/dev/dt/dt_dev.c:232 #5 spec_close+0x45f sys/kern/spec_vnops.c:-1 #6 VOP_CLOSE+0x133 sys/kern/vfs_vops.c:156 #7 vn_closefile+0x12b vn_close sys/kern/vfs_vnops.c:292 [inline] #7 vn_closefile+0x12b sys/kern/vfs_vnops.c:615 #8 fdrop+0x126 sys/kern/kern_descrip.c:1267 #9 closef+0x192 sys/kern/kern_descrip.c:1251 #10 syscall+0xbc6 mi_syscall sys/sys/syscall_mi.h:176 [inline] #10 syscall+0xbc6 sys/arch/amd64/amd64/trap.c:579 #11 Xsyscall+0x128 lock order [2] &sched_lock (&sched_lock) -> [1] &cq->cq_mtx (&cq->cq_mtx) #0 mtx_enter+0x47 sys/kern/kern_lock.c:238 #1 clockintr_cancel+0x2f sys/kern/kern_clockintr.c:323 #2 mi_switch+0x165 x86_atomic_clearbits_u32 machine/atomic.h:-1 [inline] #2 mi_switch+0x165 sys/kern/sched_bsd.c:371 #3 sleep_finish+0x24f sys/kern/kern_synch.c:412 #4 sys_futex+0xa58 futex_wait sys/kern/sys_futex.c:304 [inline] #4 sys_futex+0xa58 sys/kern/sys_futex.c:152 #5 syscall+0xbc6 mi_syscall sys/sys/syscall_mi.h:176 [inline] #5 syscall+0xbc6 sys/arch/amd64/amd64/trap.c:579 #6 Xsyscall+0x128 Stopped at db_enter+0x25: addq $0x8,%rsp ddb{0}> ddb{0}> set $lines = 0 ddb{0}> set $maxwidth = 0 ddb{0}> show panic the kernel did not panic ddb{0}> trace db_enter() at db_enter+0x25 sys/arch/amd64/amd64/db_interface.c:438 witness_checkorder(ffffffff83915080,9,0) at witness_checkorder+0x10fb sys/kern/subr_witness.c:-1 mtx_enter(ffffffff83915070) at mtx_enter+0x47 sys/kern/kern_lock.c:238 sleep_setup(ffff8000299dee50,220,ffffffff8339f469) at sleep_setup+0xc6 sys/kern/kern_synch.c:348 msleep(ffff8000299dee50,ffff8000299dedf0,220,ffffffff8339f469,0) at msleep+0x114 sys/kern/kern_synch.c:245 dt_ioctl_record_stop(ffff80000160c000) at dt_ioctl_record_stop+0xbc sys/dev/dt/dt_dev.c:575 dtclose(41e5f,81,2000,ffff8000fffeb4a0) at dtclose+0x105 dt_pcb_purge sys/dev/dt/dt_dev.c:-1 [inline] dtclose(41e5f,81,2000,ffff8000fffeb4a0) at dtclose+0x105 sys/dev/dt/dt_dev.c:232 spec_close(ffff80003c445630) at spec_close+0x45f sys/kern/spec_vnops.c:-1 VOP_CLOSE(fffffd806dd5a708,81,fffffd807f7d3410,ffff8000fffeb4a0) at VOP_CLOSE+0x133 sys/kern/vfs_vops.c:156 vn_closefile(fffffd800beda498,ffff8000fffeb4a0) at vn_closefile+0x12b vn_close sys/kern/vfs_vnops.c:292 [inline] vn_closefile(fffffd800beda498,ffff8000fffeb4a0) at vn_closefile+0x12b sys/kern/vfs_vnops.c:615 fdrop(fffffd800beda498,ffff8000fffeb4a0) at fdrop+0x126 sys/kern/kern_descrip.c:1267 closef(fffffd800beda498,ffff8000fffeb4a0) at closef+0x192 sys/kern/kern_descrip.c:1251 syscall(ffff80003c445890) at syscall+0xbc6 mi_syscall sys/sys/syscall_mi.h:176 [inline] syscall(ffff80003c445890) at syscall+0xbc6 sys/arch/amd64/amd64/trap.c:579 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0x49826bae450, count: -14 ddb{0}> show registers rdi 0 rsi 0x80000 acpi_pdirpa+0x6be71 rbp 0xffff80003c445340 rbx 0xfffffd80046f8330 rdx 0xffff80000147e180 rcx 0xffff8000fffeb4a0 rax 0x7ffff acpi_pdirpa+0x6be70 r8 0xffff80003c445220 r9 0x8080808080808080 r10 0x23a0228273372967 r11 0x307cf89875d7a4a5 r12 0 r13 0xfffffd8003f4b300 r14 0x3 r15 0xffffffff rip 0xffffffff831228b5 db_enter+0x25 cs 0x8 rflags 0x246 rsp 0xffff80003c445330 ss 0x10 db_enter+0x25: addq $0x8,%rsp ddb{0}> show proc PROC (syz-executor) tid=358449 pid=15666 tcnt=2 stat=onproc flags process=0 proc=4000000 runpri=32, usrpri=50, slppri=32, nice=20 wchan=0x0, wmesg=, ps_single=0x0 scnt=0 ecnt=0 forw=0xffffffffffffffff, list=0xffff8000fffe22b8,0xffffffff838de550 process=0xffff8000fffdd308 user=0xffff80003c440000, vmspace=0xfffffd80659895c0 estcpu=36, cpticks=0, pctcpu=0.0, user=0, sys=0, intr=0 ddb{0}> ps PID TID PPID UID S FLAGS WAIT COMMAND 15666 322720 86008 0 2 0 syz-executor *15666 358449 86008 0 7 0x4000000 syz-executor 5879 224313 67906 0 2 0 syz-executor 5879 204103 67906 0 3 0x4000080 fsleep syz-executor 60984 20554 40816 0 2 0 syz-executor 24906 78321 13675 0 2 0 syz-executor 24906 413543 13675 0 3 0x4000080 fsleep syz-executor 24906 437305 13675 0 3 0x4000080 fsleep syz-executor 24906 177161 13675 0 3 0x4000080 fsleep syz-executor 68677 379424 87193 0 3 0x2 biowait syz-executor 52610 23706 1 0 3 0x100083 ttyin getty 40816 204739 87193 0 3 0x82 nanoslp syz-executor 37843 492850 87193 0 3 0x2 biowait syz-executor 56485 104422 87193 0 3 0x82 nanoslp syz-executor 67906 323212 87193 0 3 0x82 nanoslp syz-executor 3743 403007 0 0 3 0x14280 nfsidl nfsio 80570 280505 0 0 3 0x14280 nfsidl nfsio 58972 196003 0 0 3 0x14280 nfsidl nfsio 69892 403249 0 0 3 0x14280 nfsidl nfsio 41675 321754 0 0 3 0x14280 nfsidl nfsio 10559 157705 0 0 3 0x14280 nfsidl nfsio 81447 489812 0 0 3 0x14280 nfsidl nfsio 37443 341752 0 0 3 0x14280 nfsidl nfsio 68760 49544 0 0 3 0x14280 nfsidl nfsio 96854 37449 0 0 3 0x14280 nfsidl nfsio 98117 136973 0 0 3 0x14280 nfsidl nfsio 36344 227806 0 0 3 0x14280 nfsidl nfsio 10539 260918 0 0 3 0x14280 nfsidl nfsio 55230 149953 0 0 3 0x14280 nfsidl nfsio 55156 355773 0 0 3 0x14280 nfsidl nfsio 72677 269216 0 0 3 0x14280 nfsidl nfsio 63074 159485 0 0 3 0x14280 nfsidl nfsio 29787 307914 0 0 3 0x14280 nfsidl nfsio 62914 295168 0 0 3 0x14280 nfsidl nfsio 84381 227292 0 0 3 0x14280 nfsidl nfsio 35468 331470 0 0 3 0x14200 bored sosplice 70480 239091 90112 0 3 0x100082 sbwait arp 90112 70932 1 0 3 0x10008a sigsusp sh 13675 271584 87193 0 3 0x82 nanoslp syz-executor 86008 334846 87193 0 3 0x82 nanoslp syz-executor 87193 122964 14476 0 3 0x82 wait syz-executor 14476 241935 21103 0 3 0x10008a sigsusp ksh 21103 192920 950 0 3 0x98 kqread sshd-session 950 469591 44258 0 3 0x92 kqread sshd-session 44258 359545 1 0 3 0x88 kqread sshd 45300 319673 93760 74 3 0x1100092 bpf pflogd 93760 4740 1 0 3 0x80 sbwait pflogd 28274 103753 54497 73 3 0x1100090 kqread syslogd 54497 3028 1 0 3 0x100082 sbwait syslogd 15562 481576 1 0 3 0x100080 kqread resolvd 40983 188324 83113 77 3 0x100092 kqread dhcpleased 30774 444176 83113 77 3 0x100092 kqread dhcpleased 83113 174474 1 0 3 0x80 kqread dhcpleased 31509 231651 0 0 3 0x14200 bored smr 35586 116419 0 0 2 0x14200 zerothread 86754 83915 0 0 3 0x14200 aiodoned aiodoned 15603 64044 0 0 3 0x14200 syncer update 49709 381422 0 0 3 0x14200 cleaner cleaner 98074 182330 0 0 3 0x14200 reaper reaper 20402 321150 0 0 3 0x14200 pgdaemon pagedaemon 41104 282727 0 0 3 0x14200 bored viomb 70121 217253 0 0 3 0x40014200 acpi0 acpi0 96511 190370 0 0 7 0x40014200 idle1 20047 42865 0 0 3 0x14200 bored softnet3 61833 8001 0 0 3 0x14200 bored softnet2 29031 176284 0 0 3 0x14200 bored softnet1 59408 368241 0 0 3 0x14200 bored softnet0 72084 399548 0 0 3 0x14200 bored systqmp 56985 396340 0 0 3 0x14200 bored systq 47706 247805 0 0 3 0x14200 tmoslp softclockmp 10542 410164 0 0 3 0x40014200 tmoslp softclock 84222 15620 0 0 3 0x40014200 idle0 1 352743 0 0 3 0x82 wait init 0 0 -1 0 3 0x10010200 scheduler swapper ddb{0}> show all locks CPU 0: exclusive mutex &cq->cq_mtx r = 0 (0xffff8000299dee00) #0 witness_lock+0x5bb stacktrace_save sys/sys/stacktrace.h:37 [inline] #0 witness_lock+0x5bb sys/kern/subr_witness.c:1160 #1 mtx_enter_try+0x178 sys/kern/kern_lock.c:-1 #2 mtx_enter+0x60 sys/kern/kern_lock.c:239 #3 clockintr_unbind+0x56 sys/kern/kern_clockintr.c:375 #4 dt_ioctl_record_stop+0xbc sys/dev/dt/dt_dev.c:575 #5 dtclose+0x105 dt_pcb_purge sys/dev/dt/dt_dev.c:-1 [inline] #5 dtclose+0x105 sys/dev/dt/dt_dev.c:232 #6 spec_close+0x45f sys/kern/spec_vnops.c:-1 #7 VOP_CLOSE+0x133 sys/kern/vfs_vops.c:156 #8 vn_closefile+0x12b vn_close sys/kern/vfs_vnops.c:292 [inline] #8 vn_closefile+0x12b sys/kern/vfs_vnops.c:615 #9 fdrop+0x126 sys/kern/kern_descrip.c:1267 #10 closef+0x192 sys/kern/kern_descrip.c:1251 #11 syscall+0xbc6 mi_syscall sys/sys/syscall_mi.h:176 [inline] #11 syscall+0xbc6 sys/arch/amd64/amd64/trap.c:579 #12 Xsyscall+0x128 Process 15666 (syz-executor) thread 0xffff8000fffeb4a0 (358449) exclusive rwlock dtlk r = 0 (0xffffffff838a0c08) #0 witness_lock+0x5bb stacktrace_save sys/sys/stacktrace.h:37 [inline] #0 witness_lock+0x5bb sys/kern/subr_witness.c:1160 #1 rw_do_enter_write+0x367 sys/kern/kern_rwlock.c:311 #2 dt_ioctl_record_stop+0x68 sys/dev/dt/dt_dev.c:-1 #3 dtclose+0x105 dt_pcb_purge sys/dev/dt/dt_dev.c:-1 [inline] #3 dtclose+0x105 sys/dev/dt/dt_dev.c:232 #4 spec_close+0x45f sys/kern/spec_vnops.c:-1 #5 VOP_CLOSE+0x133 sys/kern/vfs_vops.c:156 #6 vn_closefile+0x12b vn_close sys/kern/vfs_vnops.c:292 [inline] #6 vn_closefile+0x12b sys/kern/vfs_vnops.c:615 #7 fdrop+0x126 sys/kern/kern_descrip.c:1267 #8 closef+0x192 sys/kern/kern_descrip.c:1251 #9 syscall+0xbc6 mi_syscall sys/sys/syscall_mi.h:176 [inline] #9 syscall+0xbc6 sys/arch/amd64/amd64/trap.c:579 #10 Xsyscall+0x128 exclusive kernel_lock &kernel_lock r = 0 (0xffffffff83904be0) #0 witness_lock+0x5bb stacktrace_save sys/sys/stacktrace.h:37 [inline] #0 witness_lock+0x5bb sys/kern/subr_witness.c:1160 #1 vn_closefile+0x41 sys/kern/vfs_vnops.c:608 #2 fdrop+0x126 sys/kern/kern_descrip.c:1267 #3 closef+0x192 sys/kern/kern_descrip.c:1251 #4 syscall+0xbc6 mi_syscall sys/sys/syscall_mi.h:176 [inline] #4 syscall+0xbc6 sys/arch/amd64/amd64/trap.c:579 #5 Xsyscall+0x128 exclusive mutex &cq->cq_mtx r = 0 (0xffff8000299dee00) #0 witness_lock+0x5bb stacktrace_save sys/sys/stacktrace.h:37 [inline] #0 witness_lock+0x5bb sys/kern/subr_witness.c:1160 #1 mtx_enter_try+0x178 sys/kern/kern_lock.c:-1 #2 mtx_enter+0x60 sys/kern/kern_lock.c:239 #3 clockintr_unbind+0x56 sys/kern/kern_clockintr.c:375 #4 dt_ioctl_record_stop+0xbc sys/dev/dt/dt_dev.c:575 #5 dtclose+0x105 dt_pcb_purge sys/dev/dt/dt_dev.c:-1 [inline] #5 dtclose+0x105 sys/dev/dt/dt_dev.c:232 #6 spec_close+0x45f sys/kern/spec_vnops.c:-1 #7 VOP_CLOSE+0x133 sys/kern/vfs_vops.c:156 #8 vn_closefile+0x12b vn_close sys/kern/vfs_vnops.c:292 [inline] #8 vn_closefile+0x12b sys/kern/vfs_vnops.c:615 #9 fdrop+0x126 sys/kern/kern_descrip.c:1267 #10 closef+0x192 sys/kern/kern_descrip.c:1251 #11 syscall+0xbc6 mi_syscall sys/sys/syscall_mi.h:176 [inline] #11 syscall+0xbc6 sys/arch/amd64/amd64/trap.c:579 #12 Xsyscall+0x128 Process 68677 (syz-executor) thread 0xffff8000fffe0038 (379424) exclusive rrwlock inode r = 0 (0xfffffd805f434888) #0 witness_lock+0x5bb stacktrace_save sys/sys/stacktrace.h:37 [inline] #0 witness_lock+0x5bb sys/kern/subr_witness.c:1160 #1 rw_do_enter_write+0x367 sys/kern/kern_rwlock.c:311 #2 rrw_enter+0xc6 sys/kern/kern_rwlock.c:606 #3 VOP_LOCK+0xa6 sys/kern/vfs_vops.c:527 #4 ufs_ihashins+0x4f ufs_ihash sys/ufs/ufs/ufs_ihash.c:-1 [inline] #4 ufs_ihashins+0x4f sys/ufs/ufs/ufs_ihash.c:159 #5 ffs_vget+0x187 sys/ufs/ffs/ffs_vfsops.c:1230 #6 ffs_inode_alloc+0x283 sys/ufs/ffs/ffs_alloc.c:393 #7 ufs_mkdir+0x113 sys/ufs/ufs/ufs_vnops.c:1112 #8 VOP_MKDIR+0x102 sys/kern/vfs_vops.c:394 #9 domkdirat+0x179 sys/kern/vfs_syscalls.c:3109 #10 syscall+0xb08 mi_syscall sys/sys/syscall_mi.h:176 [inline] #10 syscall+0xb08 sys/arch/amd64/amd64/trap.c:579 #11 Xsyscall+0x128 exclusive rrwlock inode r = 0 (0xfffffd806f1de780) #0 witness_lock+0x5bb stacktrace_save sys/sys/stacktrace.h:37 [inline] #0 witness_lock+0x5bb sys/kern/subr_witness.c:1160 #1 rw_do_enter_write+0x367 sys/kern/kern_rwlock.c:311 #2 rrw_enter+0xc6 sys/kern/kern_rwlock.c:606 #3 VOP_LOCK+0xa6 sys/kern/vfs_vops.c:527 #4 vn_lock+0xa4 sys/kern/vfs_vnops.c:570 #5 vfs_lookup+0x109 sys/kern/vfs_lookup.c:418 #6 namei+0x7aa sys/kern/vfs_lookup.c:250 #7 domkdirat+0x8b sys/kern/vfs_syscalls.c:3094 #8 syscall+0xb08 mi_syscall sys/sys/syscall_mi.h:176 [inline] #8 syscall+0xb08 sys/arch/amd64/amd64/trap.c:579 #9 Xsyscall+0x128 Process 37843 (syz-executor) thread 0xffff8000fffea540 (492850) exclusive rrwlock inode r = 0 (0xfffffd805f434648) #0 witness_lock+0x5bb stacktrace_save sys/sys/stacktrace.h:37 [inline] #0 witness_lock+0x5bb sys/kern/subr_witness.c:1160 #1 rw_do_enter_write+0x367 sys/kern/kern_rwlock.c:311 #2 rrw_enter+0xc6 sys/kern/kern_rwlock.c:606 #3 VOP_LOCK+0xa6 sys/kern/vfs_vops.c:527 #4 vn_lock+0xa4 sys/kern/vfs_vnops.c:570 #5 vget+0x2bd sys/kern/vfs_subr.c:693 #6 ufs_ihashget+0x185 sys/ufs/ufs/ufs_ihash.c:98 #7 ffs_vget+0x8c sys/ufs/ffs/ffs_vfsops.c:1201 #8 ufs_lookup+0x19f8 sys/ufs/ufs/ufs_lookup.c:478 #9 VOP_LOOKUP+0x6e sys/kern/vfs_vops.c:85 #10 vfs_lookup+0x8fa sys/kern/vfs_lookup.c:566 #11 namei+0x7aa sys/kern/vfs_lookup.c:250 #12 dounlinkat+0xc1 sys/kern/vfs_syscalls.c:1864 #13 syscall+0xb08 mi_syscall sys/sys/syscall_mi.h:176 [inline] #13 syscall+0xb08 sys/arch/amd64/amd64/trap.c:579 #14 Xsyscall+0x128 exclusive rrwlock inode r = 0 (0xfffffd806f1de8a0) #0 witness_lock+0x5bb stacktrace_save sys/sys/stacktrace.h:37 [inline] #0 witness_lock+0x5bb sys/kern/subr_witness.c:1160 #1 rw_do_enter_write+0x367 sys/kern/kern_rwlock.c:311 #2 rrw_enter+0xc6 sys/kern/kern_rwlock.c:606 #3 VOP_LOCK+0xa6 sys/kern/vfs_vops.c:527 #4 vn_lock+0xa4 sys/kern/vfs_vnops.c:570 #5 vfs_lookup+0x109 sys/kern/vfs_lookup.c:418 #6 namei+0x7aa sys/kern/vfs_lookup.c:250 #7 dounlinkat+0xc1 sys/kern/vfs_syscalls.c:1864 #8 syscall+0xb08 mi_syscall sys/sys/syscall_mi.h:176 [inline] #8 syscall+0xb08 sys/arch/amd64/amd64/trap.c:579 #9 Xsyscall+0x128 ddb{0}> show malloc Type InUse MemUse HighUse Limit Requests Type Lim devbuf 10274 11273K 12459K 166960K 17605 0 pcb 18 15K 17K 166960K 651 0 rtable 226 13K 13K 166960K 1002 0 pf 43 19K 23K 166960K 296 0 ifaddr 43 8K 9K 166960K 193 0 ifgroup 61 2K 3K 166960K 324 0 sysctl 4 1K 9K 166960K 23 0 counters 72 37K 38K 166960K 392 0 ioctlops 0 0K 4K 166960K 1887 0 iov 0 0K 26K 166960K 224 0 mount 1 1K 1K 166960K 1 0 log 0 0K 0K 166960K 4 0 vnodes 1567 99K 99K 166960K 4893 0 UFS quota 1 32K 32K 166960K 1 0 UFS mount 5 36K 36K 166960K 5 0 shm 3 5K 13K 166960K 35 0 VM map 2 1K 1K 166960K 2 0 sem 25 21K 21K 166960K 85 0 dirhash 12 2K 2K 166960K 51 0 ACPI 1692 195K 286K 166960K 12470 0 file desc 17 61K 248K 166960K 2974 0 sigio 0 0K 0K 166960K 59 0 proc 72 91K 140K 166960K 1196 0 subproc 81 5K 5K 166960K 189 0 NFS srvsock 1 0K 0K 166960K 1 0 NFS daemon 1 16K 16K 166960K 1 0 ip_moptions 0 0K 0K 166960K 390 0 in_multi 80 5K 7K 166960K 303 0 ether_multi 1 0K 0K 166960K 15 0 mrt 2 0K 0K 166960K 22 0 ISOFS mount 1 32K 32K 166960K 1 0 MSDOSFS mount 1 16K 16K 166960K 1 0 ttys 205 917K 917K 166960K 205 0 exec 0 0K 1K 166960K 944 0 fusefs mount 1 32K 32K 166960K 1 0 pfkey data 0 0K 0K 166960K 4 0 tdb 3 0K 0K 166960K 3 0 VM swap 8 62K 64K 166960K 10 0 UVM amap 232 166K 179K 166960K 28738 0 UVM aobj 40 4K 4K 166960K 46 0 pinsyscall 43 86K 105K 166960K 4391 0 memdesc 1 4K 4K 166960K 1 0 crypto data 1 1K 1K 166960K 1 0 ip6_options 0 0K 0K 166960K 176 0 NDP 14 0K 1K 166960K 142 0 temp 79 8684K 8940K 166960K 115251 0 kqueue 13 20K 30K 166960K 506 0 SYN cache 2 16K 16K 166960K 2 0 ddb{0}> show all pools Name Size Requests Fail Releases Pgreq Pgrel Npage Hiwat Minpg Maxpg Idle plcache 128 26 0 0 1 0 1 1 0 8 0 rtpcb 120 403 0 399 6 5 1 3 0 8 0 rtentry 176 309 0 219 6 0 6 6 0 8 0 unpcb 144 2248 0 2231 17 16 1 6 0 8 0 syncache 336 9 0 9 4 4 0 1 0 8 0 tcpqe 32 3 0 3 2 2 0 1 0 8 0 tcpcb 808 1377 0 1373 34 32 2 11 0 8 1 arp 128 56 0 35 1 0 1 1 0 8 0 inpcb 384 3639 0 3630 47 45 2 15 0 8 0 nd6 144 65 0 47 2 1 1 2 0 8 0 pkpcb 40 23 0 23 8 8 0 1 0 8 0 kcovpl 48 21 0 12 1 0 1 1 0 8 0 mppekey 1024 2 0 2 2 2 0 1 0 8 0 ppxss 1192 127 0 126 3 2 1 1 0 8 0 pppxif 1504 38 0 38 3 2 1 1 0 8 1 pfstscr 40 5 0 5 4 4 0 1 0 8 0 pffrag 232 19 0 18 1 0 1 1 0 482 0 pffrnode 88 17 0 16 1 0 1 1 0 8 0 pffrent 40 67 0 66 1 0 1 1 0 8 0 pfosfp 40 1428 0 1005 5 0 5 5 0 8 0 pfosfpen 112 1428 0 714 21 0 21 21 0 8 0 pfrktable 1344 2 0 1 2 1 1 1 0 8 0 pfanchor 1288 2 0 0 1 0 1 1 0 8 0 pftag 88 2 0 0 1 0 1 1 0 8 0 pfstitem 24 186 0 136 1 0 1 1 0 8 0 pfstkey 128 195 0 145 3 0 3 3 0 8 0 pfstate 384 191 0 141 7 0 7 7 0 8 0 pfrule 1344 46 0 39 2 1 1 2 0 8 0 rttmr 136 5 0 5 3 3 0 1 0 8 0 art_heap8 4096 3 0 0 3 0 3 3 0 8 0 art_heap4 256 1292 0 920 38 13 25 31 0 8 0 art_table 32 1295 0 920 4 0 4 4 0 8 0 art_node 16 300 0 225 1 0 1 1 0 8 0 sysvmsgpl 40 14 0 13 2 1 1 1 0 8 0 semupl 112 2 0 2 1 1 0 1 0 8 0 semapl 112 75 0 52 1 0 1 1 0 8 0 shmpl 112 43 0 6 2 0 2 2 0 8 0 dirhash 1024 43 0 26 3 0 3 3 0 8 0 dino2pl 256 6704 0 5179 96 0 96 96 0 8 0 ffsino 288 6704 0 5179 110 0 110 110 0 8 0 nchpl 144 10620 0 8900 65 0 65 65 0 8 0 rtmask 32 17 0 17 6 6 0 1 0 8 0 uvmvnodes 80 5926 0 0 121 0 121 121 0 8 0 vnodes 216 5926 0 0 330 0 330 330 0 8 0 namei 1024 38604 0 38603 4 2 2 2 0 8 1 percpumem 16 211 0 160 1 0 1 1 0 8 0 kstatmem 264 206 0 174 4 1 3 3 0 8 0 acpiwqpl 32 1 0 1 1 0 1 1 1 8 1 scsiplug 72 14 0 14 7 7 0 1 0 8 0 scxspl 216 34593 0 34591 16 15 1 8 1 8 0 plimitpl 152 431 0 413 1 0 1 1 0 8 0 sigapl 424 3240 0 3170 9 0 9 9 0 8 0 knotepl 120 616 0 0 17 0 17 17 0 8 0 kqueuepl 224 1285 0 1276 16 15 1 5 0 8 0 pipepl 336 452 0 421 12 9 3 6 0 8 0 fdescpl 520 3194 0 3163 3 0 3 3 0 8 0 filepl 160 23381 0 23148 37 21 16 22 0 8 0 lockfpl 104 1035 0 1033 2 1 1 2 0 8 0 lockfspl 48 312 0 310 1 0 1 1 0 8 0 sessionpl 144 46 0 37 1 0 1 1 0 8 0 pgrppl 48 253 0 235 1 0 1 1 0 8 0 ucredpl 104 3412 0 3399 1 0 1 1 0 8 0 zombiepl 144 5017 0 5015 2 1 1 1 0 8 0 processpl 1208 3240 0 3170 6 0 6 6 0 8 0 procpl 656 8001 0 7926 9 1 8 9 0 8 0 srpgc 96 25 0 25 6 6 0 1 0 8 0 sosppl 168 18 0 18 6 6 0 1 0 8 0 sockpl 728 6380 0 6351 70 66 4 22 0 8 1 mcl64k 65536 6 0 0 1 0 1 1 0 8 0 mcl16k 16384 3 0 0 1 0 1 1 0 8 0 mcl9k 9216 1 0 0 1 0 1 1 0 8 0 mcl8k 8192 3 0 0 1 0 1 1 0 8 0 mcl4k 4096 114 0 0 15 0 15 15 0 8 0 mcl2k2 2112 1 0 0 1 0 1 1 0 8 0 mcl2k 2048 51 0 0 5 1 4 5 0 8 0 mtagpl 96 92 0 0 3 0 3 3 0 8 0 mbufpl 256 396 0 0 21 0 21 21 0 8 0 bufpl 280 10542 0 4400 439 0 439 439 0 8 0 anonpl 32 8048 0 0 65 0 65 65 0 246 0 amapchunkpl 152 95909 0 95469 48 26 22 29 0 158 0 amappl16 200 6970 0 6940 55 50 5 15 0 8 0 amappl15 192 4 0 4 1 1 0 1 0 8 0 amappl14 184 159 0 144 1 0 1 1 0 8 0 amappl13 176 5 0 5 2 2 0 1 0 8 0 amappl12 168 4052 0 4020 4 2 2 3 0 8 0 amappl11 160 73 0 59 1 0 1 1 0 8 0 amappl10 152 5 0 4 1 0 1 1 0 8 0 amappl9 144 279 0 279 1 1 0 1 0 8 0 amappl8 136 28 0 25 1 0 1 1 0 8 0 amappl7 128 163 0 149 1 0 1 1 0 8 0 amappl6 120 394 0 389 1 0 1 1 0 8 0 amappl5 112 178 0 168 1 0 1 1 0 8 0 amappl4 104 397 0 374 1 0 1 1 0 8 0 amappl3 96 20101 0 19998 4 0 4 4 0 8 0 amappl2 88 881 0 813 2 0 2 2 0 8 0 amappl1 80 22239 0 21619 16 1 15 15 0 8 0 amappl 88 27286 0 27135 5 0 5 5 0 92 0 dma8192 8192 1 0 1 1 1 0 1 0 8 0 dma4096 4096 1 0 1 1 1 0 1 0 8 0 dma1024 1024 2 0 1 1 0 1 1 0 8 0 dma256 256 6 0 6 1 1 0 1 0 8 0 dma128 128 257 0 257 4 4 0 1 0 8 0 dma64 64 7 0 7 2 2 0 1 0 8 0 dma32 32 7 0 7 1 1 0 1 0 8 0 dma16 16 18 0 17 1 0 1 1 0 8 0 aobjpl 72 45 0 6 1 0 1 1 0 8 0 uaddrrnd 24 3194 0 3163 1 0 1 1 0 8 0 uaddrbest 32 2 0 0 1 0 1 1 0 8 0 uaddr 24 3194 0 3163 1 0 1 1 0 8 0 vmmpekpl 168 25589 0 25533 4 0 4 4 0 8 0 vmmpepl 168 201847 0 199858 135 38 97 106 0 357 0 vmsppl 480 3193 0 3163 5 0 5 5 0 8 0 rwobjpl 72 56281 0 49350 137 6 131 132 0 8 0 pdppl 4096 6396 0 6326 138 60 78 88 0 8 8 pvpl 32 16142 0 0 130 0 130 130 0 265 0 pmappl 256 3193 0 3163 3 0 3 3 0 8 0 extentpl 40 45 0 27 1 0 1 1 0 8 0 phpool 112 495 0 126 11 0 11 11 0 8 0 ddb{0}> machine ddbcpu 0 Invalid cpu 0 ddb{0}> trace db_enter() at db_enter+0x25 sys/arch/amd64/amd64/db_interface.c:438 witness_checkorder(ffffffff83915080,9,0) at witness_checkorder+0x10fb sys/kern/subr_witness.c:-1 mtx_enter(ffffffff83915070) at mtx_enter+0x47 sys/kern/kern_lock.c:238 sleep_setup(ffff8000299dee50,220,ffffffff8339f469) at sleep_setup+0xc6 sys/kern/kern_synch.c:348 msleep(ffff8000299dee50,ffff8000299dedf0,220,ffffffff8339f469,0) at msleep+0x114 sys/kern/kern_synch.c:245 dt_ioctl_record_stop(ffff80000160c000) at dt_ioctl_record_stop+0xbc sys/dev/dt/dt_dev.c:575 dtclose(41e5f,81,2000,ffff8000fffeb4a0) at dtclose+0x105 dt_pcb_purge sys/dev/dt/dt_dev.c:-1 [inline] dtclose(41e5f,81,2000,ffff8000fffeb4a0) at dtclose+0x105 sys/dev/dt/dt_dev.c:232 spec_close(ffff80003c445630) at spec_close+0x45f sys/kern/spec_vnops.c:-1 VOP_CLOSE(fffffd806dd5a708,81,fffffd807f7d3410,ffff8000fffeb4a0) at VOP_CLOSE+0x133 sys/kern/vfs_vops.c:156 vn_closefile(fffffd800beda498,ffff8000fffeb4a0) at vn_closefile+0x12b vn_close sys/kern/vfs_vnops.c:292 [inline] vn_closefile(fffffd800beda498,ffff8000fffeb4a0) at vn_closefile+0x12b sys/kern/vfs_vnops.c:615 fdrop(fffffd800beda498,ffff8000fffeb4a0) at fdrop+0x126 sys/kern/kern_descrip.c:1267 closef(fffffd800beda498,ffff8000fffeb4a0) at closef+0x192 sys/kern/kern_descrip.c:1251 syscall(ffff80003c445890) at syscall+0xbc6 mi_syscall sys/sys/syscall_mi.h:176 [inline] syscall(ffff80003c445890) at syscall+0xbc6 sys/arch/amd64/amd64/trap.c:579 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0x49826bae450, count: -14 ddb{0}> machine ddbcpu 1 Stopped at x86_ipi_db+0x27: addq $0x8,%rsp ddb{1}> trace x86_ipi_db(ffff8000299ddff0) at x86_ipi_db+0x27 sys/arch/amd64/amd64/db_interface.c:394 x86_ipi_handler() at x86_ipi_handler+0xd9 sys/arch/amd64/amd64/ipi.c:106 Xresume_lapic_ipi() at Xresume_lapic_ipi+0x27 mtx_enter(ffff8000299dedf0) at mtx_enter+0xa7 sys/kern/kern_lock.c:250 clockintr_dispatch(ffff80002a228020) at clockintr_dispatch+0x3b4 sys/kern/kern_clockintr.c:215 lapic_clockintr(0,0) at lapic_clockintr+0x51 sys/arch/amd64/amd64/lapic.c:489 Xresume_lapic_ltimer() at Xresume_lapic_ltimer+0x2a acpicpu_idle() at acpicpu_idle+0x41e sys/dev/acpi/acpicpu.c:1218 sched_idle(ffff8000299ddff0) at sched_idle+0x4f8 sys/kern/kern_sched.c:182 end trace frame: 0x0, count: -9