witness: lock order reversal: 1st 0xffffffff838e3468 &sched_lock (&sched_lock) 2nd 0xffff8000299eee08 &cq->cq_mtx (&cq->cq_mtx) lock order [1] &sched_lock (&sched_lock) -> [2] &cq->cq_mtx (&cq->cq_mtx) #0 mtx_enter+0x4a sys/kern/kern_lock.c:260 #1 clockintr_cancel+0x2f sys/kern/kern_clockintr.c:323 #2 mi_switch+0x16f sys/kern/sched_bsd.c:376 #3 sleep_finish+0x2d8 sys/kern/kern_synch.c:367 #4 sys_futex+0xb90 futex_wait sys/kern/sys_futex.c:306 [inline] #4 sys_futex+0xb90 sys/kern/sys_futex.c:155 #5 syscall+0xbd4 mi_syscall sys/sys/syscall_mi.h:176 [inline] #5 syscall+0xbd4 sys/arch/amd64/amd64/trap.c:765 #6 Xsyscall+0x128 lock order [2] &cq->cq_mtx (&cq->cq_mtx) -> [1] &sched_lock (&sched_lock) #0 mtx_enter+0x4a sys/kern/kern_lock.c:260 #1 sleep_setup+0xc6 sys/kern/kern_synch.c:302 #2 msleep_nsec+0x10b sys/kern/kern_synch.c:217 #3 dt_ioctl_record_stop+0xbc sys/dev/dt/dt_dev.c:589 #4 dtclose+0x109 dt_pcb_purge sys/dev/dt/dt_dev.c:-1 [inline] #4 dtclose+0x109 sys/dev/dt/dt_dev.c:239 #5 spec_close+0x466 sys/kern/spec_vnops.c:-1 #6 VOP_CLOSE+0x132 sys/kern/vfs_vops.c:156 #7 vn_closefile+0x12b vn_close sys/kern/vfs_vnops.c:292 [inline] #7 vn_closefile+0x12b sys/kern/vfs_vnops.c:615 #8 fdrop+0x121 sys/kern/kern_descrip.c:1280 #9 closef+0x192 sys/kern/kern_descrip.c:1264 #10 finishdup+0x478 sys/kern/kern_descrip.c:688 #11 syscall+0xbd4 mi_syscall sys/sys/syscall_mi.h:176 [inline] #11 syscall+0xbd4 sys/arch/amd64/amd64/trap.c:765 #12 Xsyscall+0x128 Stopped at db_enter+0x25: addq $0x8,%rsp ddb{1}> ddb{1}> set $lines = 0 ddb{1}> set $maxwidth = 0 ddb{1}> show panic the kernel did not panic ddb{1}> trace db_enter() at db_enter+0x25 sys/arch/amd64/amd64/db_interface.c:438 witness_checkorder(ffff8000299eee08,9,0) at witness_checkorder+0x10b9 sys/kern/subr_witness.c:-1 mtx_enter(ffff8000299eedf8) at mtx_enter+0x4a sys/kern/kern_lock.c:260 clockintr_cancel(ffff8000299ee2f8) at clockintr_cancel+0x2f sys/kern/kern_clockintr.c:323 mi_switch() at mi_switch+0x16f sys/kern/sched_bsd.c:376 sleep_finish(ffffffffffffffff,1) at sleep_finish+0x2d8 sys/kern/kern_synch.c:367 sys_futex(ffff8000fffeefa8,ffff80003c4572a0,ffff80003c4571f0) at sys_futex+0xb90 futex_wait sys/kern/sys_futex.c:306 [inline] sys_futex(ffff8000fffeefa8,ffff80003c4572a0,ffff80003c4571f0) at sys_futex+0xb90 sys/kern/sys_futex.c:155 syscall(ffff80003c4572a0) at syscall+0xbd4 mi_syscall sys/sys/syscall_mi.h:176 [inline] syscall(ffff80003c4572a0) at syscall+0xbd4 sys/arch/amd64/amd64/trap.c:765 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0x6d0df1f29a0, count: -9 ddb{1}> show registers rdi 0 rsi 0x80000 acpi_pdirpa+0x6be71 rbp 0xffff80003c456eb0 rbx 0 rdx 0xffff800001440ac0 rcx 0xffff8000fffeefa8 rax 0x7ffff acpi_pdirpa+0x6be70 r8 0xffff80003c456d90 r9 0x8080808080808080 r10 0x84a361da7a5a361f r11 0xf643c9408893fcad r12 0xfffffd8003f19300 r13 0xfffffd8004709d58 r14 0x3 r15 0xffffffff833f8b17 substchar+0x10351 rip 0xffffffff81320f65 db_enter+0x25 cs 0x8 rflags 0x246 rsp 0xffff80003c456ea0 ss 0x10 db_enter+0x25: addq $0x8,%rsp ddb{1}> show proc PROC (syz-executor) tid=263766 pid=90836 tcnt=2 stat=sleep flags process=4000000 proc=4000080 runpri=86, usrpri=86, slppri=32, nice=20 wchan=0xffff80003c457110, wmesg=fsleep, ps_single=0x0 scnt=0 ecnt=0 forw=0x0, list=0xffff8000fffee018,0xffffffff838e47f0 process=0xffff8000fffe84e8 user=0xffff80003c452000, vmspace=0xfffffd806c0189a8 estcpu=36, cpticks=0, pctcpu=0.0, user=0, sys=0, intr=0 ddb{1}> ps PID TID PPID UID S FLAGS WAIT COMMAND 90836 433587 64077 0 2 0x4000000 syz-executor *90836 263766 64077 0 3 0x4000080 fsleep syz-executor 30968 286456 30337 0 2 0 syz-executor 30968 236719 30337 0 7 0x4000000 syz-executor 50143 16378 20951 0 3 0x80 nanoslp syz-executor 50143 334351 20951 0 3 0x4000000 smrbar syz-executor 79652 374048 8081 0 3 0x80 nanoslp syz-executor 79652 450578 8081 0 3 0x4000080 sbwait syz-executor 79652 14652 8081 0 3 0x4000080 fsleep syz-executor 13005 50714 11668 0 3 0x80 nanoslp syz-executor 13005 76187 11668 0 3 0x4000000 biowait syz-executor 13005 230098 11668 0 3 0x4000000 inode syz-executor 13005 113813 11668 0 3 0x4000080 fsleep syz-executor 10491 255848 3606 0 3 0x80 nanoslp syz-executor 10491 346263 3606 0 3 0x4000080 piperd syz-executor 10491 221491 3606 0 3 0x4000080 fsleep syz-executor 56751 188755 39499 0 3 0x90 nanoslp syz-executor 56751 501892 39499 0 3 0x4000090 fsleep syz-executor 56751 18287 39499 0 3 0x4000090 lockf syz-executor 64345 10733 0 0 3 0x14280 nfsidl nfsio 49403 229002 0 0 3 0x14280 nfsidl nfsio 24938 305502 0 0 3 0x14200 acct acct 4263 281481 1 0 3 0x100083 ttyin getty 39499 43676 66758 0 3 0x82 nanoslp syz-executor 12882 448778 20237 0 3 0x100082 sbwait arp 20237 55435 38225 0 3 0x10008a sigsusp sh 20951 313028 66758 0 3 0x82 nanoslp syz-executor 3606 273933 66758 0 3 0x82 nanoslp syz-executor 38225 355764 66758 0 3 0x82 wait syz-executor 11668 161203 66758 0 3 0x82 nanoslp syz-executor 30337 439568 66758 0 3 0x82 nanoslp syz-executor 64077 73796 66758 0 3 0x82 nanoslp syz-executor 8081 58879 66758 0 3 0x82 nanoslp syz-executor 66758 234378 86913 0 3 0x82 kqread syz-executor 86913 489631 33698 0 3 0x10008a sigsusp ksh 33698 19442 99441 0 3 0x98 kqread sshd-session 99441 251369 37138 0 3 0x92 kqread sshd-session 37138 417333 1 0 3 0x88 kqread sshd 23209 19467 80326 74 3 0x1100092 bpf pflogd 80326 23706 1 0 3 0x80 sbwait pflogd 6426 393740 88758 73 3 0x1100090 kqread syslogd 88758 504433 1 0 3 0x100082 sbwait syslogd 7058 94487 1 0 3 0x100080 kqread resolvd 52105 92435 2830 77 3 0x100092 kqread dhcpleased 62251 436456 2830 77 3 0x100092 kqread dhcpleased 2830 116068 1 0 3 0x80 kqread dhcpleased 78898 433214 0 0 3 0x14200 bored smr 34705 358053 0 0 3 0x14200 pgzero zerothread 59702 420871 0 0 3 0x14200 aiodoned aiodoned 13962 389632 0 0 3 0x14200 syncer update 67050 477722 0 0 3 0x14200 cleaner cleaner 71925 118069 0 0 3 0x14200 reaper reaper 93297 491695 0 0 3 0x14200 pgdaemon pagedaemon 56672 338596 0 0 3 0x14200 bored viomb 65050 220052 0 0 3 0x40014200 acpi0 acpi0 91603 71534 0 0 3 0x40014200 idle1 19183 265735 0 0 3 0x14200 bored softnet1 31952 135787 0 0 3 0x14200 bored softnet0 93382 175488 0 0 3 0x14200 smrbar systqmp 11737 471814 0 0 3 0x14200 bored systq 93914 127389 0 0 3 0x14200 tmoslp softclockmp 50407 74040 0 0 3 0x40014200 tmoslp softclock 27848 125396 0 0 3 0x40014200 idle0 1 411709 0 0 3 0x82 wait init 0 0 -1 0 3 0x10010200 scheduler swapper ddb{1}> show all locks CPU 0: exclusive mutex &vioq->viq_rxmtx r = 0 (0xffff8000001a0db0) #0 witness_lock+0x5f1 stacktrace_save sys/sys/stacktrace.h:37 [inline] #0 witness_lock+0x5f1 sys/kern/subr_witness.c:1160 #1 mtx_enter_try+0x1ad sys/kern/kern_lock.c:311 #2 mtx_enter+0x62 sys/kern/kern_lock.c:261 #3 vio_rx_intr+0x82 sys/dev/pv/if_vio.c:1569 #4 vio_queue_intr+0x59 sys/dev/pv/if_vio.c:911 #5 intr_handler+0x125 sys/arch/amd64/amd64/intr.c:563 #6 Xintr_ioapic_edge26_untramp+0x18f #7 __sanitizer_cov_trace_pc #8 sleep_setup+0xc6 sys/kern/kern_synch.c:302 #9 tsleep_nsec+0x1a1 sys/kern/kern_synch.c:153 #10 biowait+0xc6 sys/kern/vfs_bio.c:1242 #11 bwrite+0x2e7 sys/kern/vfs_bio.c:754 #12 ffs_write+0x810 sys/ufs/ffs/ffs_vnops.c:377 #13 VOP_WRITE+0x101 sys/kern/vfs_vops.c:245 #14 vn_write+0x1d3 sys/kern/vfs_vnops.c:408 #15 dofilewritev+0x242 sys/kern/sys_generic.c:380 #16 sys_write+0xa2 sys/kern/sys_generic.c:300 #17 syscall+0xbd4 mi_syscall sys/sys/syscall_mi.h:176 [inline] #17 syscall+0xbd4 sys/arch/amd64/amd64/trap.c:765 #18 Xsyscall+0x128 CPU 1: exclusive mutex &sched_lock r = 0 (0xffffffff838e3468) #0 witness_lock+0x5f1 stacktrace_save sys/sys/stacktrace.h:37 [inline] #0 witness_lock+0x5f1 sys/kern/subr_witness.c:1160 #1 mtx_enter_try+0x1ad sys/kern/kern_lock.c:311 #2 mtx_enter+0x62 sys/kern/kern_lock.c:261 #3 sleep_finish+0x1d6 sys/kern/kern_synch.c:355 #4 sys_futex+0xb90 futex_wait sys/kern/sys_futex.c:306 [inline] #4 sys_futex+0xb90 sys/kern/sys_futex.c:155 #5 syscall+0xbd4 mi_syscall sys/sys/syscall_mi.h:176 [inline] #5 syscall+0xbd4 sys/arch/amd64/amd64/trap.c:765 #6 Xsyscall+0x128 Process 30968 (syz-executor) thread 0xffff8000fffee7e0 (236719) exclusive kernel_lock &kernel_lock r = 0 (0xffffffff838e3758) #0 witness_lock+0x5f1 stacktrace_save sys/sys/stacktrace.h:37 [inline] #0 witness_lock+0x5f1 sys/kern/subr_witness.c:1160 #1 __mp_acquire_count+0x58 sys/kern/kern_lock.c:-1 #2 sleep_finish+0x2d8 sys/kern/kern_synch.c:367 #3 biowait+0xc6 sys/kern/vfs_bio.c:1242 #4 bwrite+0x2e7 sys/kern/vfs_bio.c:754 #5 ffs2_balloc+0x18b4 sys/ufs/ffs/ffs_balloc.c:701 #6 ffs_write+0x4f9 sys/ufs/ffs/ffs_vnops.c:345 #7 VOP_WRITE+0x101 sys/kern/vfs_vops.c:245 #8 vn_write+0x1d3 sys/kern/vfs_vnops.c:408 #9 dofilewritev+0x242 sys/kern/sys_generic.c:380 #10 sys_write+0xa2 sys/kern/sys_generic.c:300 #11 syscall+0xbd4 mi_syscall sys/sys/syscall_mi.h:176 [inline] #11 syscall+0xbd4 sys/arch/amd64/amd64/trap.c:765 #12 Xsyscall+0x128 exclusive rrwlock inode r = 0 (0xfffffd806e20fdb0) #0 witness_lock+0x5f1 stacktrace_save sys/sys/stacktrace.h:37 [inline] #0 witness_lock+0x5f1 sys/kern/subr_witness.c:1160 #1 rw_do_enter_write+0x419 sys/kern/kern_rwlock.c:320 #2 rrw_enter+0xc6 sys/kern/kern_rwlock.c:621 #3 VOP_LOCK+0xa3 sys/kern/vfs_vops.c:527 #4 vn_lock+0xa4 sys/kern/vfs_vnops.c:570 #5 vn_write+0x18f sys/kern/vfs_vnops.c:405 #6 dofilewritev+0x242 sys/kern/sys_generic.c:380 #7 sys_write+0xa2 sys/kern/sys_generic.c:300 #8 syscall+0xbd4 mi_syscall sys/sys/syscall_mi.h:176 [inline] #8 syscall+0xbd4 sys/arch/amd64/amd64/trap.c:765 #9 Xsyscall+0x128 Process 50143 (syz-executor) thread 0xffff80003c438a88 (334351) exclusive rwlock clonelk r = 0 (0xffffffff8387d8f8) #0 witness_lock+0x5f1 stacktrace_save sys/sys/stacktrace.h:37 [inline] #0 witness_lock+0x5f1 sys/kern/subr_witness.c:1160 #1 rw_do_enter_write+0x419 sys/kern/kern_rwlock.c:320 #2 if_clone_destroy+0x67 sys/net/if.c:-1 #3 ifioctl+0x59d sys/net/if.c:2092 #4 sys_ioctl+0x674 sys/kern/sys_generic.c:-1 #5 syscall+0xbd4 mi_syscall sys/sys/syscall_mi.h:176 [inline] #5 syscall+0xbd4 sys/arch/amd64/amd64/trap.c:765 #6 Xsyscall+0x128 Process 13005 (syz-executor) thread 0xffff8000fffef770 (76187) exclusive rrwlock inode r = 0 (0xfffffd806e184590) #0 witness_lock+0x5f1 stacktrace_save sys/sys/stacktrace.h:37 [inline] #0 witness_lock+0x5f1 sys/kern/subr_witness.c:1160 #1 rw_do_enter_write+0x419 sys/kern/kern_rwlock.c:320 #2 rrw_enter+0xc6 sys/kern/kern_rwlock.c:621 #3 VOP_LOCK+0xa3 sys/kern/vfs_vops.c:527 #4 vn_lock+0xa4 sys/kern/vfs_vnops.c:570 #5 vn_write+0x18f sys/kern/vfs_vnops.c:405 #6 dofilewritev+0x242 sys/kern/sys_generic.c:380 #7 sys_write+0xa2 sys/kern/sys_generic.c:300 #8 syscall+0xbd4 mi_syscall sys/sys/syscall_mi.h:176 [inline] #8 syscall+0xbd4 sys/arch/amd64/amd64/trap.c:765 #9 Xsyscall+0x128 Process 13005 (syz-executor) thread 0xffff8000fffe6fb0 (230098) exclusive rrwlock inode r = 0 (0xfffffd8069ec1a28) #0 witness_lock+0x5f1 stacktrace_save sys/sys/stacktrace.h:37 [inline] #0 witness_lock+0x5f1 sys/kern/subr_witness.c:1160 #1 rw_do_enter_write+0x419 sys/kern/kern_rwlock.c:320 #2 rrw_enter+0xc6 sys/kern/kern_rwlock.c:621 #3 VOP_LOCK+0xa3 sys/kern/vfs_vops.c:527 #4 vn_lock+0xa4 sys/kern/vfs_vnops.c:570 #5 vfs_lookup+0x11c sys/kern/vfs_lookup.c:-1 #6 namei+0x7ca sys/kern/vfs_lookup.c:250 #7 uipc_bind+0x328 sys/kern/uipc_usrreq.c:371 #8 sys_bind+0x2f6 sys/kern/uipc_syscalls.c:190 #9 syscall+0xbd4 mi_syscall sys/sys/syscall_mi.h:176 [inline] #9 syscall+0xbd4 sys/arch/amd64/amd64/trap.c:765 #10 Xsyscall+0x128 Process 93382 (systqmp) thread 0xffff8000ffffe000 (175488) shared rwlock systqmp r = 0 (0xffffffff8384ea68) #0 witness_lock+0x5f1 stacktrace_save sys/sys/stacktrace.h:37 [inline] #0 witness_lock+0x5f1 sys/kern/subr_witness.c:1160 #1 taskq_thread+0x12a sys/kern/kern_task.c:442 #2 proc_trampoline+0x10 ddb{1}> show malloc Type InUse MemUse HighUse Limit Requests Type Lim devbuf 10216 11109K 11385K 166960K 11957 0 pcb 19 12K 13K 166960K 123 0 rtable 198 7K 8K 166960K 476 0 pf 35 17K 24K 166960K 100 0 ifaddr 39 6K 7K 166960K 73 0 ifgroup 56 2K 2K 166960K 110 0 sysctl 2 1K 9K 166960K 15 0 counters 70 37K 38K 166960K 138 0 ioctlops 0 0K 4K 166960K 1647 0 iov 0 0K 18K 166960K 36 0 mount 1 1K 1K 166960K 1 0 log 0 0K 0K 166960K 4 0 vnodes 1463 92K 92K 166960K 1997 0 UFS quota 1 32K 32K 166960K 1 0 UFS mount 5 36K 36K 166960K 5 0 shm 2 1K 5K 166960K 13 0 VM map 2 1K 1K 166960K 2 0 sem 12 0K 0K 166960K 44 0 dirhash 12 2K 2K 166960K 45 0 ACPI 1692 195K 286K 166960K 12470 0 file desc 19 69K 85K 166960K 825 0 sigio 0 0K 0K 166960K 17 0 proc 72 115K 164K 166960K 664 0 subproc 72 4K 4K 166960K 81 0 NFS srvsock 1 0K 0K 166960K 1 0 NFS daemon 1 16K 16K 166960K 1 0 ip_moptions 0 0K 0K 166960K 409 0 in_multi 73 5K 6K 166960K 130 0 ether_multi 2 0K 0K 166960K 5 0 mrt 1 0K 0K 166960K 3 0 ISOFS mount 1 32K 32K 166960K 1 0 MSDOSFS mount 1 16K 16K 166960K 1 0 ttys 85 387K 387K 166960K 85 0 exec 0 0K 1K 166960K 501 0 fusefs mount 1 32K 32K 166960K 1 0 pfkey data 0 0K 0K 166960K 2 0 tdb 3 0K 0K 166960K 3 0 VM swap 8 62K 64K 166960K 10 0 UVM amap 274 178K 185K 166960K 9573 0 UVM aobj 9 2K 2K 166960K 10 0 pinsyscall 45 90K 102K 166960K 1982 0 memdesc 1 4K 4K 166960K 1 0 crypto data 1 1K 1K 166960K 1 0 ip6_options 1 0K 1K 166960K 53 0 NDP 12 0K 1K 166960K 48 0 temp 49 8653K 8721K 166960K 32118 0 kqueue 16 26K 28K 166960K 176 0 SYN cache 2 16K 16K 166960K 2 0 ddb{1}> show all pools Name Size Requests Fail Releases Pgreq Pgrel Npage Hiwat Minpg Maxpg Idle plcache 128 26 0 0 1 0 1 1 0 8 0 rtpcb 120 98 0 94 1 0 1 1 0 8 0 rtentry 176 143 0 60 5 0 5 5 0 8 0 unpcb 144 527 0 506 7 3 4 4 0 8 3 syncache 336 4 0 4 2 2 0 1 0 8 0 tcpqe 32 8 0 8 1 1 0 1 0 8 0 tcpcb 736 385 0 379 12 9 3 7 0 8 2 arp 136 26 0 9 1 0 1 1 0 8 0 inpcb 328 893 0 877 9 6 3 7 0 8 1 nd6 152 29 0 12 1 0 1 1 0 8 0 pkpcb 40 4 0 4 2 1 1 1 0 8 1 kcovpl 48 9 0 1 1 0 1 1 0 8 0 mppekey 1024 1 0 1 1 1 0 1 0 8 0 ppxss 1192 29 0 28 1 0 1 1 0 8 0 pppxif 1504 4 0 3 1 0 1 1 0 8 0 pffrag 232 13 0 4 1 0 1 1 0 482 0 pffrnode 88 13 0 4 1 0 1 1 0 8 0 pffrent 40 17 0 8 1 0 1 1 0 8 0 pfosfp 40 1430 0 1005 5 0 5 5 0 8 0 pfosfpen 112 1430 0 714 21 0 21 21 0 8 0 pfanchor 1288 1 0 0 1 0 1 1 0 8 0 pfstitem 24 76 0 19 1 0 1 1 0 8 0 pfstkey 128 76 0 19 2 0 2 2 0 8 0 pfstate 384 76 0 19 6 0 6 6 0 8 0 pfrule 1344 24 0 19 2 1 1 2 0 8 0 art_heap8 4096 3 0 0 3 0 3 3 0 8 0 art_heap4 256 530 0 181 27 2 25 27 0 8 1 art_table 40 533 0 181 5 0 5 5 0 8 0 art_node 32 143 0 68 1 0 1 1 0 8 0 sysvmsgpl 40 9 0 7 2 1 1 1 0 8 0 semupl 112 2 0 2 1 1 0 1 0 8 0 semapl 112 41 0 31 1 0 1 1 0 8 0 shmpl 112 7 0 1 1 0 1 1 0 8 0 dirhash 1024 40 0 23 3 0 3 3 0 8 0 dino2pl 256 2939 0 1422 96 0 96 96 0 8 0 ffsino 296 2939 0 1422 118 0 118 118 0 8 0 nchpl 144 4057 0 2350 64 0 64 64 0 8 0 rtmask 32 7 0 7 2 1 1 1 0 8 1 vnodes 216 3376 0 0 188 0 188 188 0 8 0 namei 1024 14312 0 14311 3 1 2 2 0 8 1 percpumem 16 84 0 34 1 0 1 1 0 8 0 kstatmem 264 64 0 38 3 0 3 3 0 8 1 scsiplug 72 3 0 3 2 1 1 1 0 8 1 scxspl 216 20549 0 20547 13 10 3 8 1 8 2 plimitpl 152 191 0 172 1 0 1 1 0 8 0 sigapl 424 1142 0 1090 7 0 7 7 0 8 0 knotepl 120 812 0 0 25 0 25 25 0 8 0 kqueuepl 224 346 0 334 3 0 3 3 0 8 2 pipepl 344 184 0 156 3 0 3 3 0 8 0 fdescpl 528 1119 0 1086 3 0 3 3 0 8 0 filepl 160 6980 0 6732 18 3 15 18 0 8 4 lockfpl 104 214 0 210 1 0 1 1 0 8 0 lockfspl 48 97 0 94 1 0 1 1 0 8 0 sessionpl 144 26 0 17 1 0 1 1 0 8 0 pgrppl 48 48 0 31 1 0 1 1 0 8 0 ucredpl 104 903 0 889 1 0 1 1 0 8 0 zombiepl 144 1090 0 1090 1 0 1 1 0 8 1 processpl 1232 1142 0 1090 5 0 5 5 0 8 0 procpl 664 2350 0 2286 6 0 6 6 0 8 0 sosppl 176 6 0 4 3 2 1 1 0 8 0 sockpl 752 1538 0 1497 15 4 11 12 0 8 6 mcl64k 65536 6 0 0 1 0 1 1 0 8 0 mcl16k 16384 2 0 0 1 0 1 1 0 8 0 mcl12k 12288 1 0 0 1 0 1 1 0 8 0 mcl9k 9216 1 0 0 1 0 1 1 0 8 0 mcl8k 8192 2 0 0 1 0 1 1 0 8 0 mcl4k 4096 130 0 0 17 0 17 17 0 8 1 mcl2k2 2112 1 0 0 1 0 1 1 0 8 0 mcl2k 2048 41 0 0 5 0 5 5 0 8 0 mtagpl 96 2 0 0 1 0 1 1 0 8 0 mbufpl 256 1171 0 0 73 0 73 73 0 8 0 bufpl 280 7675 0 1538 439 0 439 439 0 8 0 anonpl 32 10218 0 0 83 0 83 83 0 246 0 amapchunkpl 152 31009 0 30454 35 7 28 34 0 158 6 amappl16 200 4127 0 4012 30 15 15 20 0 8 8 amappl15 192 4 0 4 1 1 0 1 0 8 0 amappl14 184 4 0 4 1 1 0 1 0 8 0 amappl13 176 434 0 432 1 0 1 1 0 8 0 amappl12 168 1530 0 1485 3 0 3 3 0 8 0 amappl11 160 3 0 3 1 1 0 1 0 8 0 amappl10 152 49 0 35 1 0 1 1 0 8 0 amappl9 144 252 0 252 1 1 0 1 0 8 0 amappl8 136 30 0 27 1 0 1 1 0 8 0 amappl7 128 85 0 84 1 0 1 1 0 8 0 amappl6 120 289 0 273 1 0 1 1 0 8 0 amappl5 112 79 0 68 1 0 1 1 0 8 0 amappl4 104 430 0 397 1 0 1 1 0 8 0 amappl3 96 5331 0 5227 4 1 3 3 0 8 0 amappl2 88 1237 0 1158 2 0 2 2 0 8 0 amappl1 80 12170 0 11549 15 1 14 15 0 8 0 amappl 88 8718 0 8530 5 0 5 5 0 92 0 uvmvnodes 80 3376 0 0 69 0 69 69 0 8 0 dma65536 65536 1 0 1 1 1 0 1 0 8 0 dma16384 16384 1 0 1 1 1 0 1 0 8 0 dma8192 8192 1 0 1 1 1 0 1 0 8 0 dma4096 4096 1 0 1 1 1 0 1 0 8 0 dma1024 1024 1 0 0 1 0 1 1 0 8 0 dma256 256 7 0 7 2 2 0 1 0 8 0 dma128 128 253 0 253 1 1 0 1 0 8 0 dma64 64 7 0 7 2 2 0 1 0 8 0 dma32 32 8 0 8 2 2 0 1 0 8 0 dma16 16 18 0 17 1 0 1 1 0 8 0 aobjpl 72 9 0 1 1 0 1 1 0 8 0 uaddrrnd 24 1119 0 1086 1 0 1 1 0 8 0 uaddrbest 32 2 0 0 1 0 1 1 0 8 0 uaddr 24 1119 0 1086 1 0 1 1 0 8 0 vmmpekpl 168 10831 0 10794 3 0 3 3 0 8 0 vmmpepl 168 77281 0 75166 104 7 97 104 0 357 5 vmsppl 488 1118 0 1086 5 0 5 5 0 8 0 rwobjpl 80 26507 0 22042 92 0 92 92 0 8 0 pdppl 4096 2246 0 2172 100 26 74 82 0 8 0 pvpl 32 17981 0 0 146 0 146 146 0 265 0 pmappl 256 1118 0 1086 3 0 3 3 0 8 0 extentpl 40 45 0 27 1 0 1 1 0 8 0 phpool 112 355 0 47 10 0 10 10 0 8 0 ddb{1}> machine ddbcpu 0 Stopped at x86_ipi_db+0x27: addq $0x8,%rsp ddb{0}> trace x86_ipi_db(ffffffff837dbff0) at x86_ipi_db+0x27 sys/arch/amd64/amd64/db_interface.c:394 x86_ipi_handler() at x86_ipi_handler+0xd9 sys/arch/amd64/amd64/ipi.c:106 Xresume_lapic_ipi() at Xresume_lapic_ipi+0x27 mtx_enter(ffffffff838e3458) at mtx_enter+0xd2 sys/kern/kern_lock.c:264 wakeup_n(ffff80000002c000,1) at wakeup_n+0x54 sys/kern/kern_synch.c:579 task_add(ffff80000002c000,ffff800000031d30) at task_add+0x163 sys/kern/kern_task.c:372 ifiq_input(ffff800000031cc0,ffff80003bc139e8) at ifiq_input+0x4af sys/net/ifq.c:784 vio_rxeof(ffff8000001a0d00) at vio_rxeof+0x41d sys/dev/pv/if_vio.c:1555 vio_rx_intr(ffff8000001a1400) at vio_rx_intr+0x98 sys/dev/pv/if_vio.c:1571 vio_queue_intr(ffff8000001a0d00) at vio_queue_intr+0x59 sys/dev/pv/if_vio.c:911 intr_handler(ffff80003bc13b90,ffff80000007ad00) at intr_handler+0x125 sys/arch/amd64/amd64/intr.c:563 Xintr_ioapic_edge26_untramp() at Xintr_ioapic_edge26_untramp+0x18f __sanitizer_cov_trace_pc() at __sanitizer_cov_trace_pc sleep_setup(fffffd805fc248e8,11,ffffffff83431d8e) at sleep_setup+0xc6 sys/kern/kern_synch.c:302 tsleep_nsec(fffffd805fc248e8,11,ffffffff83431d8e,ffffffffffffffff) at tsleep_nsec+0x1a1 sys/kern/kern_synch.c:153 biowait(fffffd805fc248e8) at biowait+0xc6 sys/kern/vfs_bio.c:1242 bwrite(fffffd805fc248e8) at bwrite+0x2e7 sys/kern/vfs_bio.c:754 ffs_write(ffff80003bc13ec0) at ffs_write+0x810 sys/ufs/ffs/ffs_vnops.c:377 VOP_WRITE(fffffd805e03ccd0,ffff80003bc14078,7,fffffd80097fb7b8) at VOP_WRITE+0x101 sys/kern/vfs_vops.c:245 vn_write(fffffd806b71b7c8,ffff80003bc14078,0) at vn_write+0x1d3 sys/kern/vfs_vnops.c:408 dofilewritev(ffff8000fffee7e0,4,ffff80003bc14078,0,ffff80003bc14130) at dofilewritev+0x242 sys/kern/sys_generic.c:380 sys_write(ffff8000fffee7e0,ffff80003bc141e0,ffff80003bc14130) at sys_write+0xa2 sys/kern/sys_generic.c:300 syscall(ffff80003bc141e0) at syscall+0xbd4 mi_syscall sys/sys/syscall_mi.h:176 [inline] syscall(ffff80003bc141e0) at syscall+0xbd4 sys/arch/amd64/amd64/trap.c:765 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0xf653f7979c0, count: -24 ddb{0}> machine ddbcpu 1 Stopped at db_enter+0x25: addq $0x8,%rsp ddb{1}> trace db_enter() at db_enter+0x25 sys/arch/amd64/amd64/db_interface.c:438 witness_checkorder(ffff8000299eee08,9,0) at witness_checkorder+0x10b9 sys/kern/subr_witness.c:-1 mtx_enter(ffff8000299eedf8) at mtx_enter+0x4a sys/kern/kern_lock.c:260 clockintr_cancel(ffff8000299ee2f8) at clockintr_cancel+0x2f sys/kern/kern_clockintr.c:323 mi_switch() at mi_switch+0x16f sys/kern/sched_bsd.c:376 sleep_finish(ffffffffffffffff,1) at sleep_finish+0x2d8 sys/kern/kern_synch.c:367 sys_futex(ffff8000fffeefa8,ffff80003c4572a0,ffff80003c4571f0) at sys_futex+0xb90 futex_wait sys/kern/sys_futex.c:306 [inline] sys_futex(ffff8000fffeefa8,ffff80003c4572a0,ffff80003c4571f0) at sys_futex+0xb90 sys/kern/sys_futex.c:155 syscall(ffff80003c4572a0) at syscall+0xbd4 mi_syscall sys/sys/syscall_mi.h:176 [inline] syscall(ffff80003c4572a0) at syscall+0xbd4 sys/arch/amd64/amd64/trap.c:765 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0x6d0df1f29a0, count: -9