witness: lock order reversal:
1st 0xffffffff838e3468 &sched_lock (&sched_lock)
2nd 0xffff8000299eee08 &cq->cq_mtx (&cq->cq_mtx)
lock order [1] &sched_lock (&sched_lock) -> [2] &cq->cq_mtx (&cq->cq_mtx)
#0 mtx_enter+0x4a sys/kern/kern_lock.c:260
#1 clockintr_cancel+0x2f sys/kern/kern_clockintr.c:323
#2 mi_switch+0x16f sys/kern/sched_bsd.c:376
#3 sleep_finish+0x2d8 sys/kern/kern_synch.c:367
#4 sys_futex+0xb90 futex_wait sys/kern/sys_futex.c:306 [inline]
#4 sys_futex+0xb90 sys/kern/sys_futex.c:155
#5 syscall+0xbd4 mi_syscall sys/sys/syscall_mi.h:176 [inline]
#5 syscall+0xbd4 sys/arch/amd64/amd64/trap.c:765
#6 Xsyscall+0x128
lock order [2] &cq->cq_mtx (&cq->cq_mtx) -> [1] &sched_lock (&sched_lock)
#0 mtx_enter+0x4a sys/kern/kern_lock.c:260
#1 sleep_setup+0xc6 sys/kern/kern_synch.c:302
#2 msleep_nsec+0x10b sys/kern/kern_synch.c:217
#3 dt_ioctl_record_stop+0xbc sys/dev/dt/dt_dev.c:589
#4 dtclose+0x109 dt_pcb_purge sys/dev/dt/dt_dev.c:-1 [inline]
#4 dtclose+0x109 sys/dev/dt/dt_dev.c:239
#5 spec_close+0x466 sys/kern/spec_vnops.c:-1
#6 VOP_CLOSE+0x132 sys/kern/vfs_vops.c:156
#7 vn_closefile+0x12b vn_close sys/kern/vfs_vnops.c:292 [inline]
#7 vn_closefile+0x12b sys/kern/vfs_vnops.c:615
#8 fdrop+0x121 sys/kern/kern_descrip.c:1280
#9 closef+0x192 sys/kern/kern_descrip.c:1264
#10 finishdup+0x478 sys/kern/kern_descrip.c:688
#11 syscall+0xbd4 mi_syscall sys/sys/syscall_mi.h:176 [inline]
#11 syscall+0xbd4 sys/arch/amd64/amd64/trap.c:765
#12 Xsyscall+0x128
Stopped at db_enter+0x25: addq $0x8,%rsp
ddb{1}>
ddb{1}> set $lines = 0
ddb{1}> set $maxwidth = 0
ddb{1}> show panic
the kernel did not panic
ddb{1}> trace
db_enter() at db_enter+0x25 sys/arch/amd64/amd64/db_interface.c:438
witness_checkorder(ffff8000299eee08,9,0) at witness_checkorder+0x10b9 sys/kern/subr_witness.c:-1
mtx_enter(ffff8000299eedf8) at mtx_enter+0x4a sys/kern/kern_lock.c:260
clockintr_cancel(ffff8000299ee2f8) at clockintr_cancel+0x2f sys/kern/kern_clockintr.c:323
mi_switch() at mi_switch+0x16f sys/kern/sched_bsd.c:376
sleep_finish(ffffffffffffffff,1) at sleep_finish+0x2d8 sys/kern/kern_synch.c:367
sys_futex(ffff8000fffeefa8,ffff80003c4572a0,ffff80003c4571f0) at sys_futex+0xb90 futex_wait sys/kern/sys_futex.c:306 [inline]
sys_futex(ffff8000fffeefa8,ffff80003c4572a0,ffff80003c4571f0) at sys_futex+0xb90 sys/kern/sys_futex.c:155
syscall(ffff80003c4572a0) at syscall+0xbd4 mi_syscall sys/sys/syscall_mi.h:176 [inline]
syscall(ffff80003c4572a0) at syscall+0xbd4 sys/arch/amd64/amd64/trap.c:765
Xsyscall() at Xsyscall+0x128
end of kernel
end trace frame: 0x6d0df1f29a0, count: -9
ddb{1}> show registers
rdi 0
rsi 0x80000 acpi_pdirpa+0x6be71
rbp 0xffff80003c456eb0
rbx 0
rdx 0xffff800001440ac0
rcx 0xffff8000fffeefa8
rax 0x7ffff acpi_pdirpa+0x6be70
r8 0xffff80003c456d90
r9 0x8080808080808080
r10 0x84a361da7a5a361f
r11 0xf643c9408893fcad
r12 0xfffffd8003f19300
r13 0xfffffd8004709d58
r14 0x3
r15 0xffffffff833f8b17 substchar+0x10351
rip 0xffffffff81320f65 db_enter+0x25
cs 0x8
rflags 0x246
rsp 0xffff80003c456ea0
ss 0x10
db_enter+0x25: addq $0x8,%rsp
ddb{1}> show proc
PROC (syz-executor) tid=263766 pid=90836 tcnt=2 stat=sleep
flags process=4000000<ITIMER> proc=4000080<SINTR,THREAD>
runpri=86, usrpri=86, slppri=32, nice=20
wchan=0xffff80003c457110, wmesg=fsleep, ps_single=0x0 scnt=0 ecnt=0
forw=0x0, list=0xffff8000fffee018,0xffffffff838e47f0
process=0xffff8000fffe84e8 user=0xffff80003c452000, vmspace=0xfffffd806c0189a8
estcpu=36, cpticks=0, pctcpu=0.0, user=0, sys=0, intr=0
ddb{1}> ps
PID TID PPID UID S FLAGS WAIT COMMAND
90836 433587 64077 0 2 0x4000000 syz-executor
*90836 263766 64077 0 3 0x4000080 fsleep syz-executor
30968 286456 30337 0 2 0 syz-executor
30968 236719 30337 0 7 0x4000000 syz-executor
50143 16378 20951 0 3 0x80 nanoslp syz-executor
50143 334351 20951 0 3 0x4000000 smrbar syz-executor
79652 374048 8081 0 3 0x80 nanoslp syz-executor
79652 450578 8081 0 3 0x4000080 sbwait syz-executor
79652 14652 8081 0 3 0x4000080 fsleep syz-executor
13005 50714 11668 0 3 0x80 nanoslp syz-executor
13005 76187 11668 0 3 0x4000000 biowait syz-executor
13005 230098 11668 0 3 0x4000000 inode syz-executor
13005 113813 11668 0 3 0x4000080 fsleep syz-executor
10491 255848 3606 0 3 0x80 nanoslp syz-executor
10491 346263 3606 0 3 0x4000080 piperd syz-executor
10491 221491 3606 0 3 0x4000080 fsleep syz-executor
56751 188755 39499 0 3 0x90 nanoslp syz-executor
56751 501892 39499 0 3 0x4000090 fsleep syz-executor
56751 18287 39499 0 3 0x4000090 lockf syz-executor
64345 10733 0 0 3 0x14280 nfsidl nfsio
49403 229002 0 0 3 0x14280 nfsidl nfsio
24938 305502 0 0 3 0x14200 acct acct
4263 281481 1 0 3 0x100083 ttyin getty
39499 43676 66758 0 3 0x82 nanoslp syz-executor
12882 448778 20237 0 3 0x100082 sbwait arp
20237 55435 38225 0 3 0x10008a sigsusp sh
20951 313028 66758 0 3 0x82 nanoslp syz-executor
3606 273933 66758 0 3 0x82 nanoslp syz-executor
38225 355764 66758 0 3 0x82 wait syz-executor
11668 161203 66758 0 3 0x82 nanoslp syz-executor
30337 439568 66758 0 3 0x82 nanoslp syz-executor
64077 73796 66758 0 3 0x82 nanoslp syz-executor
8081 58879 66758 0 3 0x82 nanoslp syz-executor
66758 234378 86913 0 3 0x82 kqread syz-executor
86913 489631 33698 0 3 0x10008a sigsusp ksh
33698 19442 99441 0 3 0x98 kqread sshd-session
99441 251369 37138 0 3 0x92 kqread sshd-session
37138 417333 1 0 3 0x88 kqread sshd
23209 19467 80326 74 3 0x1100092 bpf pflogd
80326 23706 1 0 3 0x80 sbwait pflogd
6426 393740 88758 73 3 0x1100090 kqread syslogd
88758 504433 1 0 3 0x100082 sbwait syslogd
7058 94487 1 0 3 0x100080 kqread resolvd
52105 92435 2830 77 3 0x100092 kqread dhcpleased
62251 436456 2830 77 3 0x100092 kqread dhcpleased
2830 116068 1 0 3 0x80 kqread dhcpleased
78898 433214 0 0 3 0x14200 bored smr
34705 358053 0 0 3 0x14200 pgzero zerothread
59702 420871 0 0 3 0x14200 aiodoned aiodoned
13962 389632 0 0 3 0x14200 syncer update
67050 477722 0 0 3 0x14200 cleaner cleaner
71925 118069 0 0 3 0x14200 reaper reaper
93297 491695 0 0 3 0x14200 pgdaemon pagedaemon
56672 338596 0 0 3 0x14200 bored viomb
65050 220052 0 0 3 0x40014200 acpi0 acpi0
91603 71534 0 0 3 0x40014200 idle1
19183 265735 0 0 3 0x14200 bored softnet1
31952 135787 0 0 3 0x14200 bored softnet0
93382 175488 0 0 3 0x14200 smrbar systqmp
11737 471814 0 0 3 0x14200 bored systq
93914 127389 0 0 3 0x14200 tmoslp softclockmp
50407 74040 0 0 3 0x40014200 tmoslp softclock
27848 125396 0 0 3 0x40014200 idle0
1 411709 0 0 3 0x82 wait init
0 0 -1 0 3 0x10010200 scheduler swapper
ddb{1}> show all locks
CPU 0:
exclusive mutex &vioq->viq_rxmtx r = 0 (0xffff8000001a0db0)
#0 witness_lock+0x5f1 stacktrace_save sys/sys/stacktrace.h:37 [inline]
#0 witness_lock+0x5f1 sys/kern/subr_witness.c:1160
#1 mtx_enter_try+0x1ad sys/kern/kern_lock.c:311
#2 mtx_enter+0x62 sys/kern/kern_lock.c:261
#3 vio_rx_intr+0x82 sys/dev/pv/if_vio.c:1569
#4 vio_queue_intr+0x59 sys/dev/pv/if_vio.c:911
#5 intr_handler+0x125 sys/arch/amd64/amd64/intr.c:563
#6 Xintr_ioapic_edge26_untramp+0x18f
#7 __sanitizer_cov_trace_pc
#8 sleep_setup+0xc6 sys/kern/kern_synch.c:302
#9 tsleep_nsec+0x1a1 sys/kern/kern_synch.c:153
#10 biowait+0xc6 sys/kern/vfs_bio.c:1242
#11 bwrite+0x2e7 sys/kern/vfs_bio.c:754
#12 ffs_write+0x810 sys/ufs/ffs/ffs_vnops.c:377
#13 VOP_WRITE+0x101 sys/kern/vfs_vops.c:245
#14 vn_write+0x1d3 sys/kern/vfs_vnops.c:408
#15 dofilewritev+0x242 sys/kern/sys_generic.c:380
#16 sys_write+0xa2 sys/kern/sys_generic.c:300
#17 syscall+0xbd4 mi_syscall sys/sys/syscall_mi.h:176 [inline]
#17 syscall+0xbd4 sys/arch/amd64/amd64/trap.c:765
#18 Xsyscall+0x128
CPU 1:
exclusive mutex &sched_lock r = 0 (0xffffffff838e3468)
#0 witness_lock+0x5f1 stacktrace_save sys/sys/stacktrace.h:37 [inline]
#0 witness_lock+0x5f1 sys/kern/subr_witness.c:1160
#1 mtx_enter_try+0x1ad sys/kern/kern_lock.c:311
#2 mtx_enter+0x62 sys/kern/kern_lock.c:261
#3 sleep_finish+0x1d6 sys/kern/kern_synch.c:355
#4 sys_futex+0xb90 futex_wait sys/kern/sys_futex.c:306 [inline]
#4 sys_futex+0xb90 sys/kern/sys_futex.c:155
#5 syscall+0xbd4 mi_syscall sys/sys/syscall_mi.h:176 [inline]
#5 syscall+0xbd4 sys/arch/amd64/amd64/trap.c:765
#6 Xsyscall+0x128
Process 30968 (syz-executor) thread 0xffff8000fffee7e0 (236719)
exclusive kernel_lock &kernel_lock r = 0 (0xffffffff838e3758)
#0 witness_lock+0x5f1 stacktrace_save sys/sys/stacktrace.h:37 [inline]
#0 witness_lock+0x5f1 sys/kern/subr_witness.c:1160
#1 __mp_acquire_count+0x58 sys/kern/kern_lock.c:-1
#2 sleep_finish+0x2d8 sys/kern/kern_synch.c:367
#3 biowait+0xc6 sys/kern/vfs_bio.c:1242
#4 bwrite+0x2e7 sys/kern/vfs_bio.c:754
#5 ffs2_balloc+0x18b4 sys/ufs/ffs/ffs_balloc.c:701
#6 ffs_write+0x4f9 sys/ufs/ffs/ffs_vnops.c:345
#7 VOP_WRITE+0x101 sys/kern/vfs_vops.c:245
#8 vn_write+0x1d3 sys/kern/vfs_vnops.c:408
#9 dofilewritev+0x242 sys/kern/sys_generic.c:380
#10 sys_write+0xa2 sys/kern/sys_generic.c:300
#11 syscall+0xbd4 mi_syscall sys/sys/syscall_mi.h:176 [inline]
#11 syscall+0xbd4 sys/arch/amd64/amd64/trap.c:765
#12 Xsyscall+0x128
exclusive rrwlock inode r = 0 (0xfffffd806e20fdb0)
#0 witness_lock+0x5f1 stacktrace_save sys/sys/stacktrace.h:37 [inline]
#0 witness_lock+0x5f1 sys/kern/subr_witness.c:1160
#1 rw_do_enter_write+0x419 sys/kern/kern_rwlock.c:320
#2 rrw_enter+0xc6 sys/kern/kern_rwlock.c:621
#3 VOP_LOCK+0xa3 sys/kern/vfs_vops.c:527
#4 vn_lock+0xa4 sys/kern/vfs_vnops.c:570
#5 vn_write+0x18f sys/kern/vfs_vnops.c:405
#6 dofilewritev+0x242 sys/kern/sys_generic.c:380
#7 sys_write+0xa2 sys/kern/sys_generic.c:300
#8 syscall+0xbd4 mi_syscall sys/sys/syscall_mi.h:176 [inline]
#8 syscall+0xbd4 sys/arch/amd64/amd64/trap.c:765
#9 Xsyscall+0x128
Process 50143 (syz-executor) thread 0xffff80003c438a88 (334351)
exclusive rwlock clonelk r = 0 (0xffffffff8387d8f8)
#0 witness_lock+0x5f1 stacktrace_save sys/sys/stacktrace.h:37 [inline]
#0 witness_lock+0x5f1 sys/kern/subr_witness.c:1160
#1 rw_do_enter_write+0x419 sys/kern/kern_rwlock.c:320
#2 if_clone_destroy+0x67 sys/net/if.c:-1
#3 ifioctl+0x59d sys/net/if.c:2092
#4 sys_ioctl+0x674 sys/kern/sys_generic.c:-1
#5 syscall+0xbd4 mi_syscall sys/sys/syscall_mi.h:176 [inline]
#5 syscall+0xbd4 sys/arch/amd64/amd64/trap.c:765
#6 Xsyscall+0x128
Process 13005 (syz-executor) thread 0xffff8000fffef770 (76187)
exclusive rrwlock inode r = 0 (0xfffffd806e184590)
#0 witness_lock+0x5f1 stacktrace_save sys/sys/stacktrace.h:37 [inline]
#0 witness_lock+0x5f1 sys/kern/subr_witness.c:1160
#1 rw_do_enter_write+0x419 sys/kern/kern_rwlock.c:320
#2 rrw_enter+0xc6 sys/kern/kern_rwlock.c:621
#3 VOP_LOCK+0xa3 sys/kern/vfs_vops.c:527
#4 vn_lock+0xa4 sys/kern/vfs_vnops.c:570
#5 vn_write+0x18f sys/kern/vfs_vnops.c:405
#6 dofilewritev+0x242 sys/kern/sys_generic.c:380
#7 sys_write+0xa2 sys/kern/sys_generic.c:300
#8 syscall+0xbd4 mi_syscall sys/sys/syscall_mi.h:176 [inline]
#8 syscall+0xbd4 sys/arch/amd64/amd64/trap.c:765
#9 Xsyscall+0x128
Process 13005 (syz-executor) thread 0xffff8000fffe6fb0 (230098)
exclusive rrwlock inode r = 0 (0xfffffd8069ec1a28)
#0 witness_lock+0x5f1 stacktrace_save sys/sys/stacktrace.h:37 [inline]
#0 witness_lock+0x5f1 sys/kern/subr_witness.c:1160
#1 rw_do_enter_write+0x419 sys/kern/kern_rwlock.c:320
#2 rrw_enter+0xc6 sys/kern/kern_rwlock.c:621
#3 VOP_LOCK+0xa3 sys/kern/vfs_vops.c:527
#4 vn_lock+0xa4 sys/kern/vfs_vnops.c:570
#5 vfs_lookup+0x11c sys/kern/vfs_lookup.c:-1
#6 namei+0x7ca sys/kern/vfs_lookup.c:250
#7 uipc_bind+0x328 sys/kern/uipc_usrreq.c:371
#8 sys_bind+0x2f6 sys/kern/uipc_syscalls.c:190
#9 syscall+0xbd4 mi_syscall sys/sys/syscall_mi.h:176 [inline]
#9 syscall+0xbd4 sys/arch/amd64/amd64/trap.c:765
#10 Xsyscall+0x128
Process 93382 (systqmp) thread 0xffff8000ffffe000 (175488)
shared rwlock systqmp r = 0 (0xffffffff8384ea68)
#0 witness_lock+0x5f1 stacktrace_save sys/sys/stacktrace.h:37 [inline]
#0 witness_lock+0x5f1 sys/kern/subr_witness.c:1160
#1 taskq_thread+0x12a sys/kern/kern_task.c:442
#2 proc_trampoline+0x10
ddb{1}> show malloc
Type InUse MemUse HighUse Limit Requests Type Lim
devbuf 10216 11109K 11385K 166960K 11957 0
pcb 19 12K 13K 166960K 123 0
rtable 198 7K 8K 166960K 476 0
pf 35 17K 24K 166960K 100 0
ifaddr 39 6K 7K 166960K 73 0
ifgroup 56 2K 2K 166960K 110 0
sysctl 2 1K 9K 166960K 15 0
counters 70 37K 38K 166960K 138 0
ioctlops 0 0K 4K 166960K 1647 0
iov 0 0K 18K 166960K 36 0
mount 1 1K 1K 166960K 1 0
log 0 0K 0K 166960K 4 0
vnodes 1463 92K 92K 166960K 1997 0
UFS quota 1 32K 32K 166960K 1 0
UFS mount 5 36K 36K 166960K 5 0
shm 2 1K 5K 166960K 13 0
VM map 2 1K 1K 166960K 2 0
sem 12 0K 0K 166960K 44 0
dirhash 12 2K 2K 166960K 45 0
ACPI 1692 195K 286K 166960K 12470 0
file desc 19 69K 85K 166960K 825 0
sigio 0 0K 0K 166960K 17 0
proc 72 115K 164K 166960K 664 0
subproc 72 4K 4K 166960K 81 0
NFS srvsock 1 0K 0K 166960K 1 0
NFS daemon 1 16K 16K 166960K 1 0
ip_moptions 0 0K 0K 166960K 409 0
in_multi 73 5K 6K 166960K 130 0
ether_multi 2 0K 0K 166960K 5 0
mrt 1 0K 0K 166960K 3 0
ISOFS mount 1 32K 32K 166960K 1 0
MSDOSFS mount 1 16K 16K 166960K 1 0
ttys 85 387K 387K 166960K 85 0
exec 0 0K 1K 166960K 501 0
fusefs mount 1 32K 32K 166960K 1 0
pfkey data 0 0K 0K 166960K 2 0
tdb 3 0K 0K 166960K 3 0
VM swap 8 62K 64K 166960K 10 0
UVM amap 274 178K 185K 166960K 9573 0
UVM aobj 9 2K 2K 166960K 10 0
pinsyscall 45 90K 102K 166960K 1982 0
memdesc 1 4K 4K 166960K 1 0
crypto data 1 1K 1K 166960K 1 0
ip6_options 1 0K 1K 166960K 53 0
NDP 12 0K 1K 166960K 48 0
temp 49 8653K 8721K 166960K 32118 0
kqueue 16 26K 28K 166960K 176 0
SYN cache 2 16K 16K 166960K 2 0
ddb{1}> show all pools
Name Size Requests Fail Releases Pgreq Pgrel Npage Hiwat Minpg Maxpg Idle
plcache 128 26 0 0 1 0 1 1 0 8 0
rtpcb 120 98 0 94 1 0 1 1 0 8 0
rtentry 176 143 0 60 5 0 5 5 0 8 0
unpcb 144 527 0 506 7 3 4 4 0 8 3
syncache 336 4 0 4 2 2 0 1 0 8 0
tcpqe 32 8 0 8 1 1 0 1 0 8 0
tcpcb 736 385 0 379 12 9 3 7 0 8 2
arp 136 26 0 9 1 0 1 1 0 8 0
inpcb 328 893 0 877 9 6 3 7 0 8 1
nd6 152 29 0 12 1 0 1 1 0 8 0
pkpcb 40 4 0 4 2 1 1 1 0 8 1
kcovpl 48 9 0 1 1 0 1 1 0 8 0
mppekey 1024 1 0 1 1 1 0 1 0 8 0
ppxss 1192 29 0 28 1 0 1 1 0 8 0
pppxif 1504 4 0 3 1 0 1 1 0 8 0
pffrag 232 13 0 4 1 0 1 1 0 482 0
pffrnode 88 13 0 4 1 0 1 1 0 8 0
pffrent 40 17 0 8 1 0 1 1 0 8 0
pfosfp 40 1430 0 1005 5 0 5 5 0 8 0
pfosfpen 112 1430 0 714 21 0 21 21 0 8 0
pfanchor 1288 1 0 0 1 0 1 1 0 8 0
pfstitem 24 76 0 19 1 0 1 1 0 8 0
pfstkey 128 76 0 19 2 0 2 2 0 8 0
pfstate 384 76 0 19 6 0 6 6 0 8 0
pfrule 1344 24 0 19 2 1 1 2 0 8 0
art_heap8 4096 3 0 0 3 0 3 3 0 8 0
art_heap4 256 530 0 181 27 2 25 27 0 8 1
art_table 40 533 0 181 5 0 5 5 0 8 0
art_node 32 143 0 68 1 0 1 1 0 8 0
sysvmsgpl 40 9 0 7 2 1 1 1 0 8 0
semupl 112 2 0 2 1 1 0 1 0 8 0
semapl 112 41 0 31 1 0 1 1 0 8 0
shmpl 112 7 0 1 1 0 1 1 0 8 0
dirhash 1024 40 0 23 3 0 3 3 0 8 0
dino2pl 256 2939 0 1422 96 0 96 96 0 8 0
ffsino 296 2939 0 1422 118 0 118 118 0 8 0
nchpl 144 4057 0 2350 64 0 64 64 0 8 0
rtmask 32 7 0 7 2 1 1 1 0 8 1
vnodes 216 3376 0 0 188 0 188 188 0 8 0
namei 1024 14312 0 14311 3 1 2 2 0 8 1
percpumem 16 84 0 34 1 0 1 1 0 8 0
kstatmem 264 64 0 38 3 0 3 3 0 8 1
scsiplug 72 3 0 3 2 1 1 1 0 8 1
scxspl 216 20549 0 20547 13 10 3 8 1 8 2
plimitpl 152 191 0 172 1 0 1 1 0 8 0
sigapl 424 1142 0 1090 7 0 7 7 0 8 0
knotepl 120 812 0 0 25 0 25 25 0 8 0
kqueuepl 224 346 0 334 3 0 3 3 0 8 2
pipepl 344 184 0 156 3 0 3 3 0 8 0
fdescpl 528 1119 0 1086 3 0 3 3 0 8 0
filepl 160 6980 0 6732 18 3 15 18 0 8 4
lockfpl 104 214 0 210 1 0 1 1 0 8 0
lockfspl 48 97 0 94 1 0 1 1 0 8 0
sessionpl 144 26 0 17 1 0 1 1 0 8 0
pgrppl 48 48 0 31 1 0 1 1 0 8 0
ucredpl 104 903 0 889 1 0 1 1 0 8 0
zombiepl 144 1090 0 1090 1 0 1 1 0 8 1
processpl 1232 1142 0 1090 5 0 5 5 0 8 0
procpl 664 2350 0 2286 6 0 6 6 0 8 0
sosppl 176 6 0 4 3 2 1 1 0 8 0
sockpl 752 1538 0 1497 15 4 11 12 0 8 6
mcl64k 65536 6 0 0 1 0 1 1 0 8 0
mcl16k 16384 2 0 0 1 0 1 1 0 8 0
mcl12k 12288 1 0 0 1 0 1 1 0 8 0
mcl9k 9216 1 0 0 1 0 1 1 0 8 0
mcl8k 8192 2 0 0 1 0 1 1 0 8 0
mcl4k 4096 130 0 0 17 0 17 17 0 8 1
mcl2k2 2112 1 0 0 1 0 1 1 0 8 0
mcl2k 2048 41 0 0 5 0 5 5 0 8 0
mtagpl 96 2 0 0 1 0 1 1 0 8 0
mbufpl 256 1171 0 0 73 0 73 73 0 8 0
bufpl 280 7675 0 1538 439 0 439 439 0 8 0
anonpl 32 10218 0 0 83 0 83 83 0 246 0
amapchunkpl 152 31009 0 30454 35 7 28 34 0 158 6
amappl16 200 4127 0 4012 30 15 15 20 0 8 8
amappl15 192 4 0 4 1 1 0 1 0 8 0
amappl14 184 4 0 4 1 1 0 1 0 8 0
amappl13 176 434 0 432 1 0 1 1 0 8 0
amappl12 168 1530 0 1485 3 0 3 3 0 8 0
amappl11 160 3 0 3 1 1 0 1 0 8 0
amappl10 152 49 0 35 1 0 1 1 0 8 0
amappl9 144 252 0 252 1 1 0 1 0 8 0
amappl8 136 30 0 27 1 0 1 1 0 8 0
amappl7 128 85 0 84 1 0 1 1 0 8 0
amappl6 120 289 0 273 1 0 1 1 0 8 0
amappl5 112 79 0 68 1 0 1 1 0 8 0
amappl4 104 430 0 397 1 0 1 1 0 8 0
amappl3 96 5331 0 5227 4 1 3 3 0 8 0
amappl2 88 1237 0 1158 2 0 2 2 0 8 0
amappl1 80 12170 0 11549 15 1 14 15 0 8 0
amappl 88 8718 0 8530 5 0 5 5 0 92 0
uvmvnodes 80 3376 0 0 69 0 69 69 0 8 0
dma65536 65536 1 0 1 1 1 0 1 0 8 0
dma16384 16384 1 0 1 1 1 0 1 0 8 0
dma8192 8192 1 0 1 1 1 0 1 0 8 0
dma4096 4096 1 0 1 1 1 0 1 0 8 0
dma1024 1024 1 0 0 1 0 1 1 0 8 0
dma256 256 7 0 7 2 2 0 1 0 8 0
dma128 128 253 0 253 1 1 0 1 0 8 0
dma64 64 7 0 7 2 2 0 1 0 8 0
dma32 32 8 0 8 2 2 0 1 0 8 0
dma16 16 18 0 17 1 0 1 1 0 8 0
aobjpl 72 9 0 1 1 0 1 1 0 8 0
uaddrrnd 24 1119 0 1086 1 0 1 1 0 8 0
uaddrbest 32 2 0 0 1 0 1 1 0 8 0
uaddr 24 1119 0 1086 1 0 1 1 0 8 0
vmmpekpl 168 10831 0 10794 3 0 3 3 0 8 0
vmmpepl 168 77281 0 75166 104 7 97 104 0 357 5
vmsppl 488 1118 0 1086 5 0 5 5 0 8 0
rwobjpl 80 26507 0 22042 92 0 92 92 0 8 0
pdppl 4096 2246 0 2172 100 26 74 82 0 8 0
pvpl 32 17981 0 0 146 0 146 146 0 265 0
pmappl 256 1118 0 1086 3 0 3 3 0 8 0
extentpl 40 45 0 27 1 0 1 1 0 8 0
phpool 112 355 0 47 10 0 10 10 0 8 0
ddb{1}> machine ddbcpu 0
Stopped at x86_ipi_db+0x27: addq $0x8,%rsp
ddb{0}> trace
x86_ipi_db(ffffffff837dbff0) at x86_ipi_db+0x27 sys/arch/amd64/amd64/db_interface.c:394
x86_ipi_handler() at x86_ipi_handler+0xd9 sys/arch/amd64/amd64/ipi.c:106
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x27
mtx_enter(ffffffff838e3458) at mtx_enter+0xd2 sys/kern/kern_lock.c:264
wakeup_n(ffff80000002c000,1) at wakeup_n+0x54 sys/kern/kern_synch.c:579
task_add(ffff80000002c000,ffff800000031d30) at task_add+0x163 sys/kern/kern_task.c:372
ifiq_input(ffff800000031cc0,ffff80003bc139e8) at ifiq_input+0x4af sys/net/ifq.c:784
vio_rxeof(ffff8000001a0d00) at vio_rxeof+0x41d sys/dev/pv/if_vio.c:1555
vio_rx_intr(ffff8000001a1400) at vio_rx_intr+0x98 sys/dev/pv/if_vio.c:1571
vio_queue_intr(ffff8000001a0d00) at vio_queue_intr+0x59 sys/dev/pv/if_vio.c:911
intr_handler(ffff80003bc13b90,ffff80000007ad00) at intr_handler+0x125 sys/arch/amd64/amd64/intr.c:563
Xintr_ioapic_edge26_untramp() at Xintr_ioapic_edge26_untramp+0x18f
__sanitizer_cov_trace_pc() at __sanitizer_cov_trace_pc
sleep_setup(fffffd805fc248e8,11,ffffffff83431d8e) at sleep_setup+0xc6 sys/kern/kern_synch.c:302
tsleep_nsec(fffffd805fc248e8,11,ffffffff83431d8e,ffffffffffffffff) at tsleep_nsec+0x1a1 sys/kern/kern_synch.c:153
biowait(fffffd805fc248e8) at biowait+0xc6 sys/kern/vfs_bio.c:1242
bwrite(fffffd805fc248e8) at bwrite+0x2e7 sys/kern/vfs_bio.c:754
ffs_write(ffff80003bc13ec0) at ffs_write+0x810 sys/ufs/ffs/ffs_vnops.c:377
VOP_WRITE(fffffd805e03ccd0,ffff80003bc14078,7,fffffd80097fb7b8) at VOP_WRITE+0x101 sys/kern/vfs_vops.c:245
vn_write(fffffd806b71b7c8,ffff80003bc14078,0) at vn_write+0x1d3 sys/kern/vfs_vnops.c:408
dofilewritev(ffff8000fffee7e0,4,ffff80003bc14078,0,ffff80003bc14130) at dofilewritev+0x242 sys/kern/sys_generic.c:380
sys_write(ffff8000fffee7e0,ffff80003bc141e0,ffff80003bc14130) at sys_write+0xa2 sys/kern/sys_generic.c:300
syscall(ffff80003bc141e0) at syscall+0xbd4 mi_syscall sys/sys/syscall_mi.h:176 [inline]
syscall(ffff80003bc141e0) at syscall+0xbd4 sys/arch/amd64/amd64/trap.c:765
Xsyscall() at Xsyscall+0x128
end of kernel
end trace frame: 0xf653f7979c0, count: -24
ddb{0}> machine ddbcpu 1
Stopped at db_enter+0x25: addq $0x8,%rsp
ddb{1}> trace
db_enter() at db_enter+0x25 sys/arch/amd64/amd64/db_interface.c:438
witness_checkorder(ffff8000299eee08,9,0) at witness_checkorder+0x10b9 sys/kern/subr_witness.c:-1
mtx_enter(ffff8000299eedf8) at mtx_enter+0x4a sys/kern/kern_lock.c:260
clockintr_cancel(ffff8000299ee2f8) at clockintr_cancel+0x2f sys/kern/kern_clockintr.c:323
mi_switch() at mi_switch+0x16f sys/kern/sched_bsd.c:376
sleep_finish(ffffffffffffffff,1) at sleep_finish+0x2d8 sys/kern/kern_synch.c:367
sys_futex(ffff8000fffeefa8,ffff80003c4572a0,ffff80003c4571f0) at sys_futex+0xb90 futex_wait sys/kern/sys_futex.c:306 [inline]
sys_futex(ffff8000fffeefa8,ffff80003c4572a0,ffff80003c4571f0) at sys_futex+0xb90 sys/kern/sys_futex.c:155
syscall(ffff80003c4572a0) at syscall+0xbd4 mi_syscall sys/sys/syscall_mi.h:176 [inline]
syscall(ffff80003c4572a0) at syscall+0xbd4 sys/arch/amd64/amd64/trap.c:765
Xsyscall() at Xsyscall+0x128
end of kernel
end trace frame: 0x6d0df1f29a0, count: -9