witness: lock order reversal:
1st 0xffffffff839c4440 &sched_lock (&sched_lock)
2nd 0xffff8000299dee30 &cq->cq_mtx (&cq->cq_mtx)
lock order [1] &sched_lock (&sched_lock) -> [2] &cq->cq_mtx (&cq->cq_mtx)
#0 mtx_enter+0x95 sys/kern/kern_lock.c:406
#1 clockintr_cancel+0x2f sys/kern/kern_clockintr.c:323
#2 mi_switch+0x187 sys/kern/sched_bsd.c:376
#3 ast+0x15a mi_ast sys/sys/syscall_mi.h:252 [inline]
#3 ast+0x15a sys/arch/amd64/amd64/trap.c:739
#4 intr_user_exit+0x3c
lock order [2] &cq->cq_mtx (&cq->cq_mtx) -> [1] &sched_lock (&sched_lock)
#0 mtx_enter+0x95 sys/kern/kern_lock.c:406
#1 sleep_setup+0xed sys/kern/kern_synch.c:304
#2 msleep_nsec+0x10b sys/kern/kern_synch.c:217
#3 dt_ioctl_record_stop+0xbc sys/dev/dt/dt_dev.c:589
#4 dtclose+0x109 dt_pcb_purge sys/dev/dt/dt_dev.c:-1 [inline]
#4 dtclose+0x109 sys/dev/dt/dt_dev.c:239
#5 spec_close+0x466 sys/kern/spec_vnops.c:-1
#6 VOP_CLOSE+0x132 sys/kern/vfs_vops.c:156
#7 vn_closefile+0x12b vn_close sys/kern/vfs_vnops.c:292 [inline]
#7 vn_closefile+0x12b sys/kern/vfs_vnops.c:615
#8 fdrop+0x121 sys/kern/kern_descrip.c:1280
#9 closef+0x192 sys/kern/kern_descrip.c:1264
#10 syscall+0xbd4 mi_syscall sys/sys/syscall_mi.h:176 [inline]
#10 syscall+0xbd4 sys/arch/amd64/amd64/trap.c:775
#11 Xsyscall+0x128
Stopped at db_enter+0x25: addq $0x8,%rsp
ddb{1}>
ddb{1}> set $lines = 0
ddb{1}> set $maxwidth = 0
ddb{1}> show panic
the kernel did not panic
ddb{1}> trace
db_enter() at db_enter+0x25 sys/arch/amd64/amd64/db_interface.c:438
witness_checkorder(ffff8000299dee30,9,0) at witness_checkorder+0x10d1 sys/kern/subr_witness.c:-1
mtx_enter(ffff8000299dee20) at mtx_enter+0x95 sys/kern/kern_lock.c:406
clockintr_cancel(ffff8000299de2f8) at clockintr_cancel+0x2f sys/kern/kern_clockintr.c:323
mi_switch() at mi_switch+0x187 sys/kern/sched_bsd.c:376
ast(ffff80003c41f540) at ast+0x15a mi_ast sys/sys/syscall_mi.h:252 [inline]
ast(ffff80003c41f540) at ast+0x15a sys/arch/amd64/amd64/trap.c:739
intr_user_exit() at intr_user_exit+0x3c
end of kernel
end trace frame: 0x78ea0fe55b50, count: -7
ddb{1}> show registers
rdi 0
rsi 0
rbp 0xffff80003c41f340
rbx 0
rdx 0
rcx 0xffff80003c400fb8
rax 0xffff8000299ddff0
r8 0xffff80003c41f220
r9 0x8080808080808080
r10 0x154809d71db2a30
r11 0x26a396e4b6c96989
r12 0xfffffd8003f54300
r13 0xfffffd8004744d58
r14 0x3
r15 0xffffffff8341be15 substchar+0x52f5
rip 0xffffffff814c93d5 db_enter+0x25
cs 0x8
rflags 0x246
rsp 0xffff80003c41f330
ss 0
db_enter+0x25: addq $0x8,%rsp
ddb{1}> show proc
PROC (syz-executor) tid=38742 pid=62541 tcnt=2 stat=run
flags process=4000000<ITIMER> proc=0
runpri=50, usrpri=50, slppri=16, nice=20
wchan=0x0, wmesg=, ps_single=0x0 scnt=0 ecnt=0
forw=0x0, list=0xffff80003c401cb0,0xffff80003c400038
process=0xffff8000fffece88 user=0xffff80003c41a000, vmspace=0xfffffd800f7e9020
estcpu=36, cpticks=0, pctcpu=0.0, user=0, sys=0, intr=0
ddb{1}> ps
PID TID PPID UID S FLAGS WAIT COMMAND
*62541 38742 69154 0 2 0x4000000 syz-executor
62541 93217 69154 0 2 0x4000000 syz-executor
80550 134181 9697 0 2 0 syz-executor
80550 467973 9697 0 3 0x4000080 fsleep syz-executor
2731 450202 50768 0 2 0 syz-executor
22900 173589 45105 0 2 0 syz-executor
22900 307962 45105 0 2 0x4000000 syz-executor
63036 473707 24780 0 2 0 syz-executor
63036 219327 24780 0 3 0x4000080 kqpoll syz-executor
63036 55908 24780 0 2 0x4000000 syz-executor
63036 194534 24780 0 3 0x4000080 fsleep syz-executor
37298 458766 76659 0 3 0x80 nanoslp syz-executor
37298 69426 76659 0 3 0x4000080 fsleep syz-executor
37298 426373 76659 0 3 0x4000080 kqread syz-executor
37298 483562 76659 0 3 0x4000080 fsleep syz-executor
45105 235011 80617 0 3 0x82 nanoslp syz-executor
57086 319623 80617 0 3 0x82 piperd syz-executor
99943 347083 80617 0 3 0x82 nanoslp syz-executor
24780 478058 80617 0 3 0x82 nanoslp syz-executor
76659 190410 80617 0 3 0x82 nanoslp syz-executor
69154 226192 80617 0 3 0x82 nanoslp syz-executor
9697 77143 80617 0 3 0x82 nanoslp syz-executor
50768 454753 80617 0 3 0x82 nanoslp syz-executor
80617 194125 94339 0 2 0x2 syz-executor
94339 46454 8749 0 3 0x10008a sigsusp ksh
8749 447920 50452 0 3 0x98 kqread sshd-session
50452 504390 16356 0 3 0x92 kqread sshd-session
70185 351813 1 0 3 0x100083 ttyin getty
16356 261548 1 0 3 0x88 kqread sshd
920 6359 33379 74 3 0x1100092 bpf pflogd
33379 517626 1 0 3 0x80 sbwait pflogd
82517 483481 26581 73 3 0x1100090 kqread syslogd
26581 358606 1 0 3 0x100082 sbwait syslogd
495 244452 1 0 3 0x100080 kqread resolvd
33689 143493 46945 77 3 0x100092 kqread dhcpleased
78149 155215 46945 77 3 0x100092 kqread dhcpleased
46945 484222 1 0 3 0x80 kqread dhcpleased
25603 313171 0 0 3 0x14200 bored smr
32866 436073 0 0 2 0x14200 zerothread
45479 353379 0 0 3 0x14200 aiodoned aiodoned
57171 447812 0 0 3 0x14200 syncer update
9905 191418 0 0 3 0x14200 cleaner cleaner
33841 277466 0 0 3 0x14200 reaper reaper
13023 97333 0 0 3 0x14200 pgdaemon pagedaemon
7941 124929 0 0 3 0x14200 bored viomb
30109 520669 0 0 3 0x40014200 acpi0 acpi0
81581 395066 0 0 3 0x40014200 idle1
91595 399934 0 0 3 0x14200 bored softnet1
23888 368207 0 0 3 0x14200 bored softnet0
30647 492126 0 0 3 0x14200 bored systqmp
84644 363129 0 0 3 0x14200 bored systq
20163 176626 0 0 3 0x14200 tmoslp softclockmp
19185 359036 0 0 3 0x40014200 tmoslp softclock
5474 515203 0 0 3 0x40014200 idle0
1 299072 0 0 3 0x82 wait init
0 0 -1 0 3 0x10200 scheduler swapper
ddb{1}> show all locks
CPU 0:
exclusive mutex &uvm.fpageqlock r = 0 (0xffffffff838eeea8)
#0 witness_lock+0x5f1 stacktrace_save sys/sys/stacktrace.h:37 [inline]
#0 witness_lock+0x5f1 sys/kern/subr_witness.c:1160
#1 mtx_enter+0x4b4 sys/kern/kern_lock.c:487
#2 uvm_pmr_freepages+0x1a8 sys/uvm/uvm_pmemrange.c:-1
#3 pmap_do_remove+0x8af sys/arch/amd64/amd64/pmap.c:1939
#4 uvm_unmap_kill_entry_withlock+0x269 sys/uvm/uvm_map.c:1863
#5 uvm_map_teardown+0x117 uvm_map_addr_RBT_LEFT sys/uvm/uvm_map.h:-1 [inline]
#5 uvm_map_teardown+0x117 sys/uvm/uvm_map.c:2491
#6 exit1+0x6fc sys/kern/kern_exit.c:260
#7 sys_exit+0x1a sys/kern/kern_exit.c:-1
#8 syscall+0xb17 mi_syscall sys/sys/syscall_mi.h:176 [inline]
#8 syscall+0xb17 sys/arch/amd64/amd64/trap.c:775
#9 Xsyscall+0x128
CPU 1:
exclusive mutex &sched_lock r = 0 (0xffffffff839c4440)
#0 witness_lock+0x5f1 stacktrace_save sys/sys/stacktrace.h:37 [inline]
#0 witness_lock+0x5f1 sys/kern/subr_witness.c:1160
#1 mtx_enter+0x4b4 sys/kern/kern_lock.c:487
#2 preempt+0x35 sys/kern/sched_bsd.c:338
#3 ast+0x15a mi_ast sys/sys/syscall_mi.h:252 [inline]
#3 ast+0x15a sys/arch/amd64/amd64/trap.c:739
#4 intr_user_exit+0x3c
Process 22900 (syz-executor) thread 0xffff80002a261ca0 (307962)
exclusive rrwlock inode r = 0 (0xfffffd806c761458)
#0 witness_lock+0x5f1 stacktrace_save sys/sys/stacktrace.h:37 [inline]
#0 witness_lock+0x5f1 sys/kern/subr_witness.c:1160
#1 rw_do_enter_write+0x419 sys/kern/kern_rwlock.c:320
#2 rrw_enter+0xc6 sys/kern/kern_rwlock.c:621
#3 VOP_LOCK+0xbd sys/kern/vfs_vops.c:527
#4 vn_lock+0xa4 sys/kern/vfs_vnops.c:570
#5 vn_write+0x18f sys/kern/vfs_vnops.c:405
#6 dofilewritev+0x242 sys/kern/sys_generic.c:380
#7 sys_write+0xa2 sys/kern/sys_generic.c:300
#8 syscall+0xbd4 mi_syscall sys/sys/syscall_mi.h:176 [inline]
#8 syscall+0xbd4 sys/arch/amd64/amd64/trap.c:775
#9 Xsyscall+0x128
ddb{1}> show malloc
Type InUse MemUse HighUse Limit Requests Type Lim
devbuf 11058 12090K 12462K 166960K 12538 0
pcb 17 13K 14K 166960K 115 0
rtable 235 8K 8K 166960K 402 0
pf 35 17K 22K 166960K 73 0
ifaddr 40 7K 7K 166960K 58 0
ifgroup 54 2K 2K 166960K 83 0
sysctl 1 1K 9K 166960K 5 0
counters 68 36K 38K 166960K 98 0
ioctlops 0 0K 4K 166960K 1509 0
iov 0 0K 16K 166960K 80 0
mount 1 1K 1K 166960K 1 0
log 0 0K 0K 166960K 4 0
vnodes 1398 88K 89K 166960K 1653 0
UFS quota 1 32K 32K 166960K 1 0
UFS mount 5 36K 36K 166960K 5 0
shm 2 1K 5K 166960K 6 0
VM map 2 1K 1K 166960K 2 0
sem 12 0K 0K 166960K 20 0
dirhash 12 2K 2K 166960K 21 0
ACPI 1692 195K 286K 166960K 12470 0
file desc 16 57K 228K 166960K 442 0
sigio 0 0K 0K 166960K 8 0
proc 73 115K 180K 166960K 565 0
subproc 72 4K 4K 166960K 72 0
NFS srvsock 1 0K 0K 166960K 1 0
NFS daemon 1 16K 16K 166960K 1 0
ip_moptions 0 0K 0K 166960K 32 0
in_multi 92 6K 7K 166960K 115 0
ether_multi 1 0K 0K 166960K 9 0
mrt 0 0K 0K 166960K 6 0
ISOFS mount 1 32K 32K 166960K 1 0
MSDOSFS mount 1 16K 16K 166960K 1 0
ttys 73 334K 334K 166960K 73 0
exec 0 0K 1K 166960K 416 0
fusefs mount 1 32K 32K 166960K 1 0
tdb 3 0K 0K 166960K 3 0
VM swap 8 62K 64K 166960K 10 0
UVM amap 234 167K 182K 166960K 5851 0
UVM aobj 11 4K 4K 166960K 12 0
pinsyscall 41 82K 106K 166960K 1553 0
memdesc 1 4K 4K 166960K 1 0
crypto data 1 1K 1K 166960K 1 0
ip6_options 0 0K 0K 166960K 29 0
NDP 11 0K 2K 166960K 39 0
temp 49 8677K 8742K 166960K 14692 0
kqueue 14 22K 34K 166960K 97 0
SYN cache 2 16K 16K 166960K 2 0
ddb{1}> show all pools
Name Size Requests Fail Releases Pgreq Pgrel Npage Hiwat Minpg Maxpg Idle
plcache 128 26 0 0 1 0 1 1 0 8 0
rtpcb 120 50 0 46 1 0 1 1 0 8 0
rtentry 176 120 0 16 6 0 6 6 0 8 0
unpcb 144 211 0 192 4 0 4 4 0 8 3
syncache 336 3 0 3 1 1 0 1 0 8 0
tcpcb 736 83 0 78 1 0 1 1 0 8 0
arp 136 18 0 2 1 0 1 1 0 8 0
inpcb 328 324 0 316 4 0 4 4 0 8 2
nd6 152 28 0 5 2 0 2 2 0 8 0
kcovpl 48 8 0 0 1 0 1 1 0 8 0
ppxss 1192 12 0 12 1 0 1 1 0 8 1
pffrag 232 2 0 1 1 0 1 1 0 482 0
pffrnode 88 2 0 1 1 0 1 1 0 8 0
pffrent 40 4 0 3 1 0 1 1 0 8 0
pfosfp 40 1428 0 1005 5 0 5 5 0 8 0
pfosfpen 112 1428 0 714 21 0 21 21 0 8 0
pfrktable 1344 1 0 1 1 0 1 1 0 8 1
pfanchor 1288 1 0 0 1 0 1 1 0 8 0
pftag 88 4 0 4 1 0 1 1 0 8 1
pfstkey 128 2 0 2 1 1 0 1 0 8 0
pfstate 448 1 0 1 1 1 0 1 0 8 0
pfrule 1360 4 0 3 1 0 1 1 0 8 0
rttmr 136 2 0 2 1 0 1 1 0 8 1
art_heap8 4096 2 0 0 2 0 2 2 0 8 0
art_heap4 256 479 0 61 30 0 30 30 0 8 0
art_table 40 481 0 61 5 0 5 5 0 8 0
art_node 32 120 0 25 1 0 1 1 0 8 0
sysvmsgpl 40 2 0 1 1 0 1 1 0 8 0
semupl 112 1 0 1 1 0 1 1 0 8 1
semapl 112 16 0 6 1 0 1 1 0 8 0
shmpl 112 8 0 1 1 0 1 1 0 8 0
dirhash 1024 23 0 6 3 0 3 3 0 8 0
dino2pl 256 2146 0 641 95 0 95 95 0 8 0
ffsino 296 2146 0 641 117 0 117 117 0 8 0
nchpl 144 2766 0 1066 64 0 64 64 0 8 0
rtmask 32 2 0 2 1 0 1 1 0 8 1
vnodes 216 2409 0 0 134 0 134 134 0 8 0
namei 1024 9235 0 9235 3 1 2 2 0 8 2
percpumem 16 64 0 15 1 0 1 1 0 8 0
vcpupl 3968 1 0 0 1 0 1 1 0 8 0
vmpool 848 1 0 0 1 0 1 1 0 8 0
kstatmem 264 46 0 22 3 0 3 3 0 8 1
scxspl 216 15753 0 15753 6 4 2 5 1 8 2
plimitpl 152 152 0 133 1 0 1 1 0 8 0
sigapl 424 737 0 690 7 1 6 7 0 8 0
knotepl 120 538 0 0 17 0 17 17 0 8 0
kqueuepl 224 195 0 184 5 0 5 5 0 8 4
pipepl 344 171 0 144 6 1 5 6 0 8 2
fdescpl 528 721 0 691 3 0 3 3 0 8 0
filepl 160 3569 0 3317 18 1 17 17 0 8 5
lockfpl 104 236 0 233 2 1 1 2 0 8 0
lockfspl 48 109 0 106 1 0 1 1 0 8 0
sessionpl 144 22 0 13 1 0 1 1 0 8 0
pgrppl 48 46 0 29 1 0 1 1 0 8 0
ucredpl 104 366 0 353 1 0 1 1 0 8 0
zombiepl 144 691 0 690 1 0 1 1 0 8 0
processpl 1232 737 0 690 5 1 4 5 0 8 0
procpl 664 1263 0 1207 7 0 7 7 0 8 2
sosppl 176 1 0 1 1 0 1 1 0 8 1
sockpl 752 659 0 628 12 1 11 11 0 8 7
mcl64k 65536 4 0 0 1 0 1 1 0 8 0
mcl8k 8192 2 0 0 1 0 1 1 0 8 0
mcl4k 4096 123 0 0 16 0 16 16 0 8 0
mcl2k 2048 24 0 0 3 0 3 3 0 8 0
mtagpl 96 5 0 0 1 0 1 1 0 8 0
mbufpl 256 239 0 0 15 0 15 15 0 8 0
bufpl 280 6384 0 248 439 0 439 439 0 8 0
anonpl 32 10963 0 0 89 0 89 89 0 246 0
amapchunkpl 152 18252 0 17791 34 2 32 32 0 158 10
amappl16 200 2283 0 2233 29 6 23 23 0 8 16
amappl15 192 5 0 5 1 1 0 1 0 8 0
amappl14 184 6 0 6 1 1 0 1 0 8 0
amappl13 176 421 0 420 1 0 1 1 0 8 0
amappl12 168 1094 0 1053 3 1 2 3 0 8 0
amappl11 160 7 0 7 1 1 0 1 0 8 0
amappl10 152 49 0 35 1 0 1 1 0 8 0
amappl9 144 261 0 260 2 1 1 1 0 8 0
amappl8 136 24 0 21 1 0 1 1 0 8 0
amappl7 128 109 0 108 1 0 1 1 0 8 0
amappl6 120 268 0 255 1 0 1 1 0 8 0
amappl5 112 68 0 58 1 0 1 1 0 8 0
amappl4 104 412 0 383 1 0 1 1 0 8 0
amappl3 96 3017 0 2929 4 1 3 3 0 8 0
amappl2 88 829 0 754 2 0 2 2 0 8 0
amappl1 80 10296 0 9708 14 1 13 14 0 8 0
amappl 88 5074 0 4918 5 0 5 5 0 92 0
uvmvnodes 80 111 0 0 3 0 3 3 0 8 0
dma65536 65536 1 0 1 1 0 1 1 0 8 1
dma4096 4096 2 0 2 2 1 1 1 0 8 1
dma1024 1024 1 0 0 1 0 1 1 0 8 0
dma256 256 7 0 7 2 1 1 1 0 8 1
dma128 128 253 0 253 1 1 0 1 0 8 0
dma64 64 6 0 6 1 1 0 1 0 8 0
dma32 32 7 0 7 1 1 0 1 0 8 0
dma16 16 18 0 17 1 0 1 1 0 8 0
aobjpl 72 11 0 1 1 0 1 1 0 8 0
uaddrrnd 24 721 0 690 1 0 1 1 0 8 0
uaddrbest 32 2 0 0 1 0 1 1 0 8 0
uaddr 24 721 0 690 1 0 1 1 0 8 0
vmmpekpl 168 7626 0 7589 3 0 3 3 0 8 0
vmmpepl 168 52380 0 50456 104 0 104 104 0 357 9
vmsppl 488 720 0 690 5 1 4 5 0 8 0
rwobjpl 80 16887 0 15831 30 0 30 30 0 8 2
pdppl 4096 1452 0 1381 102 30 72 86 0 8 1
pvpl 32 18230 0 0 149 1 148 148 0 265 0
pmappl 256 721 0 690 3 1 2 3 0 8 0
extentpl 40 45 0 27 1 0 1 1 0 8 0
phpool 112 284 0 36 8 0 8 8 0 8 0
ddb{1}> machine ddbcpu 0
Stopped at x86_ipi_db+0x27: addq $0x8,%rsp
ddb{0}> trace
x86_ipi_db(ffffffff837d3ff0) at x86_ipi_db+0x27 sys/arch/amd64/amd64/db_interface.c:394
x86_ipi_handler() at x86_ipi_handler+0xd9 sys/arch/amd64/amd64/ipi.c:106
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x27
mtx_enter(ffffffff839c4430) at mtx_enter+0x377 sys/kern/kern_lock.c:454
wakeup_n(ffffffff838eecf8,ffffffff) at wakeup_n+0x54 sys/kern/kern_synch.c:581
uvm_pmr_freepages(fffffd80082bf180,1) at uvm_pmr_freepages+0x3e8 sys/uvm/uvm_pmemrange.c:1341
pmap_do_remove(fffffd800b062400,69f2a48e000,69f2a4ce000,0) at pmap_do_remove+0x8af sys/arch/amd64/amd64/pmap.c:1939
uvm_unmap_kill_entry_withlock(fffffd800f7e95d8,fffffd806c5f3d88,0) at uvm_unmap_kill_entry_withlock+0x269 sys/uvm/uvm_map.c:1863
uvm_map_teardown(fffffd800f7e95d8) at uvm_map_teardown+0x117 uvm_map_addr_RBT_LEFT sys/uvm/uvm_map.h:-1 [inline]
uvm_map_teardown(fffffd800f7e95d8) at uvm_map_teardown+0x117 sys/uvm/uvm_map.c:2491
exit1(ffff80003c400028,b,0,1) at exit1+0x6fc sys/kern/kern_exit.c:260
sys_exit(ffff80003c400028,ffff80003c43d270,ffff80003c43d1c0) at sys_exit+0x1a sys/kern/kern_exit.c:-1
syscall(ffff80003c43d270) at syscall+0xb17 mi_syscall sys/sys/syscall_mi.h:176 [inline]
syscall(ffff80003c43d270) at syscall+0xb17 sys/arch/amd64/amd64/trap.c:775
Xsyscall() at Xsyscall+0x128
end of kernel
end trace frame: 0x77696f4069b0, count: -13
ddb{0}> machine ddbcpu 1
Stopped at db_enter+0x25: addq $0x8,%rsp
ddb{1}> trace
db_enter() at db_enter+0x25 sys/arch/amd64/amd64/db_interface.c:438
witness_checkorder(ffff8000299dee30,9,0) at witness_checkorder+0x10d1 sys/kern/subr_witness.c:-1
mtx_enter(ffff8000299dee20) at mtx_enter+0x95 sys/kern/kern_lock.c:406
clockintr_cancel(ffff8000299de2f8) at clockintr_cancel+0x2f sys/kern/kern_clockintr.c:323
mi_switch() at mi_switch+0x187 sys/kern/sched_bsd.c:376
ast(ffff80003c41f540) at ast+0x15a mi_ast sys/sys/syscall_mi.h:252 [inline]
ast(ffff80003c41f540) at ast+0x15a sys/arch/amd64/amd64/trap.c:739
intr_user_exit() at intr_user_exit+0x3c
end of kernel
end trace frame: 0x78ea0fe55b50, count: -7