kernel: protection fault trap, code=0
Stopped at witness_checkorder+0xb5: movl 0x20(%r14),%r15d
ddb{1}>
ddb{1}> set $lines = 0
ddb{1}> set $maxwidth = 0
ddb{1}> show panic
the kernel did not panic
ddb{1}> trace
witness_checkorder(deadbeefdeadc017,9,0) at witness_checkorder+0xb5 sys/kern/subr_witness.c:779
mtx_enter(deadbeefdeadc007) at mtx_enter+0x4a sys/kern/kern_lock.c:260
prsignal(deadbeefdeadbeef,14) at prsignal+0x36 sys/kern/kern_sig.c:904
reaper(ffff8000ffffc2a0) at reaper+0x2cc sys/kern/kern_exit.c:516
end trace frame: 0x0, count: -4
ddb{1}> show registers
rdi 0
rsi 0
rbp 0xffff80002a278020
rbx 0
rdx 0
rcx 0xffff8000ffffc2a0
rax 0xffff8000299fdffe
r8 0x7f7fffffc000
r9 0x1
r10 0x33651f4e9fe7f6a
r11 0x6d881aa3132f7c80
r12 0xdeadbeefdeadbefe
r13 0x9
r14 0xdeadbeefdeadc017
r15 0
rip 0xffffffff82079745 witness_checkorder+0xb5
cs 0x8
rflags 0x10246 __ALIGN_SIZE+0xf246
rsp 0xffff80002a277f80
ss 0x10
witness_checkorder+0xb5: movl 0x20(%r14),%r15d
ddb{1}> show proc
PROC (reaper) tid=183144 pid=51211 tcnt=1 stat=onproc
flags process=14000<NOZOMBIE,SYSTEM> proc=200<SYSTEM>
runpri=32, usrpri=50, slppri=32, nice=20
wchan=0x0, wmesg=, ps_single=0x0 scnt=0 ecnt=0
forw=0xffffffffffffffff, list=0xffff8000ffffc538,0xffff8000ffffc018
process=0xffff8000ffff49c8 user=0xffff80002a273000, vmspace=0xffffffff8386bc08
estcpu=0, cpticks=3, pctcpu=0.0, user=0, sys=0, intr=0
ddb{1}> ps
PID TID PPID UID S FLAGS WAIT COMMAND
90786 27067 70240 32767 2 0x10 syz-executor
28360 264248 44085 32767 2 0x10 syz-executor
28360 26783 44085 32767 7 0x4000010 syz-executor
88124 218079 9770 32767 2 0x10 syz-executor
88124 30352 9770 32767 3 0x4000090 fsleep syz-executor
14197 80942 96158 32767 2 0x10 syz-executor
14197 256221 96158 32767 3 0x4000090 fsleep syz-executor
44447 237384 76406 0 2 0x100000 sh
54066 241328 25545 0 2 0x100000 sh
25545 481702 63252 0 3 0x10008a sigsusp sh
76406 515186 11002 0 3 0x10008a sigsusp sh
9770 327597 6865 32767 2 0x10 syz-executor
96158 63908 61217 32767 3 0x90 nanoslp syz-executor
63252 179761 86169 0 3 0x80 wait syz-executor
93019 439396 79620 32767 2 0x10000010 syz-executor
71042 193756 1603 32767 2 0x10 syz-executor
70240 102203 61061 32767 3 0x90 nanoslp syz-executor
11002 100334 13420 0 3 0x80 wait syz-executor
44085 482351 32274 32767 3 0x90 nanoslp syz-executor
61061 126426 88470 0 3 0x82 wait syz-executor
1603 318526 88470 0 3 0x82 wait syz-executor
61217 57132 88470 0 3 0x82 wait syz-executor
32274 359841 88470 0 3 0x82 wait syz-executor
13420 137376 88470 0 3 0x82 wait syz-executor
6865 90499 88470 0 3 0x82 wait syz-executor
86169 387001 88470 0 3 0x82 wait syz-executor
79620 264665 88470 0 3 0x82 wait syz-executor
88470 109058 43154 0 2 0x2 syz-executor
43154 390045 176 0 3 0x10008a sigsusp ksh
176 263494 81965 0 3 0x98 kqread sshd-session
81965 475269 6326 0 3 0x92 kqread sshd-session
12374 45405 1 0 3 0x100083 ttyin getty
6326 459438 1 0 3 0x88 kqread sshd
39327 438197 86299 73 3 0x1100090 kqread syslogd
86299 145237 1 0 3 0x100082 sbwait syslogd
69586 423375 1 0 3 0x100080 kqread resolvd
21899 475965 14266 77 3 0x100092 kqread dhcpleased
32952 272637 14266 77 3 0x100092 kqread dhcpleased
14266 230408 1 0 3 0x80 kqread dhcpleased
55753 60224 0 0 2 0x40014200 smr
92897 519465 0 0 2 0x14200 zerothread
85953 219608 0 0 3 0x14200 aiodoned aiodoned
56299 429795 0 0 3 0x14200 syncer update
48902 132788 0 0 3 0x14200 cleaner cleaner
*51211 183144 0 0 7 0x14200 reaper
75545 166977 0 0 3 0x14200 pgdaemon pagedaemon
32843 507809 0 0 3 0x14200 bored viomb
80893 499293 0 0 3 0x40014200 acpi0 acpi0
74420 138027 0 0 3 0x40014200 idle1
46900 365132 0 0 3 0x14200 bored softnet7
69308 57091 0 0 3 0x14200 bored softnet6
4254 22334 0 0 3 0x14200 bored softnet5
22312 505180 0 0 3 0x14200 bored softnet4
51101 178772 0 0 3 0x14200 bored softnet3
11254 138711 0 0 3 0x14200 bored softnet2
20329 207374 0 0 3 0x14200 bored softnet1
58663 143253 0 0 2 0x14200 softnet0
80829 104343 0 0 3 0x14200 smrbar systqmp
37002 386224 0 0 3 0x14200 bored systq
46738 344544 0 0 3 0x14200 tmoslp softclockmp
7758 302582 0 0 3 0x40014200 tmoslp softclock
86789 491163 0 0 3 0x40014200 idle0
1 349333 0 0 3 0x82 wait init
0 0 -1 0 3 0x10200 scheduler swapper
ddb{1}> show all locks
Process 51211 (reaper) thread 0xffff8000ffffc2a0 (183144)
exclusive kernel_lock &kernel_lock r = 0 (0xffffffff839227f8)
#0 witness_lock+0x5f1 stacktrace_save sys/sys/stacktrace.h:37 [inline]
#0 witness_lock+0x5f1 sys/kern/subr_witness.c:1160
#1 __mp_acquire_count+0x58 sys/kern/kern_lock.c:-1
#2 sleep_finish+0x2d8 sys/kern/kern_synch.c:367
#3 rw_do_enter_write+0x1dc sys/kern/kern_rwlock.c:298
#4 knote_processexit+0x2b sys/kern/kern_event.c:2217
#5 reaper+0x24d sys/kern/kern_exit.c:513
#6 proc_trampoline+0x10
Process 80829 (systqmp) thread 0xffff8000ffffe7c8 (104343)
shared rwlock systqmp r = 0 (0xffffffff8381d4e8)
#0 witness_lock+0x5f1 stacktrace_save sys/sys/stacktrace.h:37 [inline]
#0 witness_lock+0x5f1 sys/kern/subr_witness.c:1160
#1 taskq_thread+0x12a sys/kern/kern_task.c:442
#2 proc_trampoline+0x10
ddb{1}> show malloc
Type InUse MemUse HighUse Limit Requests Type Lim
devbuf 10186 10955K 10968K 166960K 11264 0
pcb 17 12K 12K 166960K 17 0
rtable 187 5K 5K 166960K 233 0
pf 31 16K 16K 166960K 31 0
ifaddr 39 6K 6K 166960K 41 0
ifgroup 50 2K 2K 166960K 50 0
sysctl 1 1K 9K 166960K 5 0
counters 68 36K 36K 166960K 68 0
ioctlops 0 0K 2K 166960K 28 0
iov 0 0K 0K 166960K 1 0
mount 1 1K 1K 166960K 1 0
log 0 0K 0K 166960K 4 0
vnodes 1334 84K 84K 166960K 1350 0
UFS quota 1 32K 32K 166960K 1 0
UFS mount 5 36K 36K 166960K 5 0
shm 2 1K 1K 166960K 2 0
VM map 2 1K 1K 166960K 2 0
sem 2 0K 0K 166960K 2 0
dirhash 12 2K 2K 166960K 12 0
ACPI 1692 195K 286K 166960K 12470 0
file desc 26 97K 121K 166960K 142 0
proc 58 99K 163K 166960K 459 0
subproc 72 4K 4K 166960K 72 0
NFS srvsock 1 0K 0K 166960K 1 0
NFS daemon 1 16K 16K 166960K 1 0
in_multi 88 6K 6K 166960K 88 0
ether_multi 1 0K 0K 166960K 1 0
ISOFS mount 1 32K 32K 166960K 1 0
MSDOSFS mount 1 16K 16K 166960K 1 0
ttys 25 122K 122K 166960K 25 0
exec 0 0K 1K 166960K 328 0
fusefs mount 1 32K 32K 166960K 1 0
tdb 3 0K 0K 166960K 3 0
VM swap 8 62K 64K 166960K 10 0
UVM amap 221 161K 167K 166960K 2725 0
UVM aobj 3 2K 2K 166960K 3 0
pinsyscall 47 94K 110K 166960K 1118 0
memdesc 1 4K 4K 166960K 1 0
crypto data 1 1K 1K 166960K 1 0
NDP 25 1K 1K 166960K 25 0
temp 33 8634K 8698K 166960K 3559 0
kqueue 14 20K 20K 166960K 22 0
SYN cache 2 16K 16K 166960K 2 0
ddb{1}> show all pools
Name Size Requests Fail Releases Pgreq Pgrel Npage Hiwat Minpg Maxpg Idle
plcache 128 26 0 0 1 0 1 1 0 8 0
rtpcb 120 30 0 27 1 0 1 1 0 8 0
rtentry 176 86 0 1 4 0 4 4 0 8 0
unpcb 144 34 0 19 1 0 1 1 0 8 0
syncache 336 3 0 3 1 0 1 1 0 8 1
tcpcb 736 7 0 3 1 0 1 1 0 8 0
arp 128 9 0 0 1 0 1 1 0 8 0
inpcb 328 53 0 46 1 0 1 1 0 8 0
nd6 144 14 0 0 1 0 1 1 0 8 0
kcovpl 48 8 0 0 1 0 1 1 0 8 0
art_heap8 4096 1 0 0 1 0 1 1 0 8 0
art_heap4 256 395 0 0 25 0 25 25 0 8 0
art_table 40 396 0 0 4 0 4 4 0 8 0
art_node 32 86 0 4 1 0 1 1 0 8 0
dirhash 1024 17 0 0 3 0 3 3 0 8 0
dino2pl 256 1546 0 45 94 0 94 94 0 8 0
ffsino 296 1546 0 45 116 0 116 116 0 8 0
nchpl 144 1731 0 58 63 0 63 63 0 8 0
uvmvnodes 80 1627 0 0 34 0 34 34 0 8 0
vnodes 216 1627 0 0 91 0 91 91 0 8 0
namei 1024 4991 0 4991 3 1 2 2 0 8 2
percpumem 16 49 0 0 1 0 1 1 0 8 0
kstatmem 264 24 0 0 2 0 2 2 0 8 0
scxspl 216 6275 0 6275 4 2 2 2 1 8 2
plimitpl 152 34 0 10 1 0 1 1 0 8 0
sigapl 424 417 0 356 8 0 8 8 0 8 0
knotepl 120 53 0 0 2 0 2 2 0 8 0
kqueuepl 224 18 0 8 1 0 1 1 0 8 0
pipepl 344 95 0 68 4 1 3 3 0 8 0
fdescpl 528 395 0 357 3 0 3 3 0 8 0
filepl 160 1248 0 1044 9 0 9 9 0 8 0
lockfpl 104 6 0 4 1 0 1 1 0 8 0
lockfspl 48 4 0 2 1 0 1 1 0 8 0
sessionpl 144 21 0 5 1 0 1 1 0 8 0
pgrppl 48 29 0 5 1 0 1 1 0 8 0
ucredpl 104 82 0 65 1 0 1 1 0 8 0
zombiepl 144 357 0 356 2 1 1 1 0 8 0
processpl 1248 417 0 356 7 1 6 6 0 8 0
procpl 664 426 0 362 6 0 6 6 0 8 0
sockpl 752 117 0 92 3 0 3 3 0 8 0
mcl8k 8192 2 0 0 1 0 1 1 0 8 0
mcl4k 4096 119 0 0 15 0 15 15 0 8 0
mcl2k 2048 17 0 0 3 0 3 3 0 8 0
mtagpl 96 3 0 0 1 0 1 1 0 8 0
mbufpl 256 148 0 0 9 0 9 9 0 8 0
bufpl 280 2803 0 116 192 0 192 192 0 8 0
anonpl 32 3640 0 0 30 0 30 30 0 246 0
amapchunkpl 152 7762 0 7315 23 4 19 19 0 158 1
amappl16 200 1534 0 1522 5 0 5 5 0 8 4
amappl15 192 11 0 11 2 1 1 1 0 8 1
amappl14 184 96 0 86 1 0 1 1 0 8 0
amappl13 176 3 0 3 1 0 1 1 0 8 1
amappl12 168 975 0 937 2 0 2 2 0 8 0
amappl11 160 43 0 33 1 0 1 1 0 8 0
amappl10 152 28 0 28 2 1 1 1 0 8 1
amappl9 144 237 0 236 1 0 1 1 0 8 0
amappl8 136 34 0 33 1 0 1 1 0 8 0
amappl7 128 92 0 82 1 0 1 1 0 8 0
amappl6 120 156 0 151 2 1 1 1 0 8 0
amappl5 112 171 0 165 1 0 1 1 0 8 0
amappl4 104 259 0 244 1 0 1 1 0 8 0
amappl3 96 1139 0 1036 3 0 3 3 0 8 0
amappl2 88 578 0 522 2 0 2 2 0 8 0
amappl1 80 7598 0 7022 14 0 14 14 0 8 1
amappl 88 2140 0 1985 4 0 4 4 0 92 0
dma4096 4096 1 0 1 1 1 0 1 0 8 0
dma1024 1024 1 0 0 1 0 1 1 0 8 0
dma256 256 6 0 6 1 1 0 1 0 8 0
dma128 128 253 0 253 1 1 0 1 0 8 0
dma64 64 6 0 6 1 1 0 1 0 8 0
dma32 32 7 0 7 1 1 0 1 0 8 0
dma16 16 18 0 17 1 0 1 1 0 8 0
aobjpl 72 2 0 0 1 0 1 1 0 8 0
uaddrrnd 24 395 0 357 1 0 1 1 0 8 0
uaddrbest 32 2 0 0 1 0 1 1 0 8 0
uaddr 24 395 0 357 1 0 1 1 0 8 0
vmmpekpl 168 4587 0 4555 2 0 2 2 0 8 0
vmmpepl 168 31155 0 29156 94 2 92 92 0 357 1
vmsppl 488 394 0 357 7 1 6 6 0 8 0
rwobjpl 80 12789 0 10344 53 0 53 53 0 8 0
pdppl 4096 798 0 714 108 12 96 96 0 8 12
pvpl 32 9432 0 0 77 1 76 76 0 265 0
pmappl 256 394 0 357 4 1 3 3 0 8 0
extentpl 40 45 0 27 1 0 1 1 0 8 0
phpool 112 271 0 16 8 0 8 8 0 8 0
ddb{1}> machine ddbcpu 0
Stopped at x86_ipi_db+0x27: addq $0x8,%rsp
ddb{0}> trace
x86_ipi_db(ffffffff8379aff0) at x86_ipi_db+0x27 sys/arch/amd64/amd64/db_interface.c:394
x86_ipi_handler() at x86_ipi_handler+0xd9 sys/arch/amd64/amd64/ipi.c:106
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x27
__mp_lock(ffffffff839225f0) at __mp_lock+0x192 __mp_lock_spin sys/kern/kern_lock.c:134 [inline]
__mp_lock(ffffffff839225f0) at __mp_lock+0x192 sys/kern/kern_lock.c:165
vn_ioctl(fffffd806c503a28,80044b02,ffff800038441950,ffff800038037cb0) at vn_ioctl+0x4d sys/kern/vfs_vnops.c:514
sys_ioctl(ffff800038037cb0,ffff800038441b30,ffff800038441a80) at sys_ioctl+0x674 sys/kern/sys_generic.c:-1
syscall(ffff800038441b30) at syscall+0xbd4 mi_syscall sys/sys/syscall_mi.h:176 [inline]
syscall(ffff800038441b30) at syscall+0xbd4 sys/arch/amd64/amd64/trap.c:748
Xsyscall() at Xsyscall+0x128
end of kernel
end trace frame: 0x34fceff8aa0, count: -8
ddb{0}> machine ddbcpu 1
Stopped at witness_checkorder+0xb5: movl 0x20(%r14),%r15d
ddb{1}> trace
witness_checkorder(deadbeefdeadc017,9,0) at witness_checkorder+0xb5 sys/kern/subr_witness.c:779
mtx_enter(deadbeefdeadc007) at mtx_enter+0x4a sys/kern/kern_lock.c:260
prsignal(deadbeefdeadbeef,14) at prsignal+0x36 sys/kern/kern_sig.c:904
reaper(ffff8000ffffc2a0) at reaper+0x2cc sys/kern/kern_exit.c:516
end trace frame: 0x0, count: -4