kernel: protection fault trap, code=0 Stopped at witness_checkorder+0x4f5: movl 0x10(%r14),%ecx ddb{0}> ddb{0}> set $lines = 0 ddb{0}> set $maxwidth = 0 ddb{0}> show panic the kernel did not panic ddb{0}> trace witness_checkorder(fffffd806d5f3030,9,0) at witness_checkorder+0x4f5 witness_lock_order_check sys/kern/subr_witness.c:2442 [inline] witness_checkorder(fffffd806d5f3030,9,0) at witness_checkorder+0x4f5 sys/kern/subr_witness.c:890 rw_enter(fffffd806d5f3020,1) at rw_enter+0xd5 sys/kern/kern_rwlock.c:249 rwsleep(fffffd806d5f3160,fffffd806d5f3020,118,ffffffff82823f2b,0) at rwsleep+0xd6 sys/kern/kern_synch.c:303 sosend(fffffd806d5f3018,0,ffff8000275632f8,0,0,0) at sosend+0x76b sys/kern/uipc_socket.c:623 fifo_write(ffff800027563240) at fifo_write+0x80 sys/miscfs/fifofs/fifo_vnops.c:279 VOP_WRITE(fffffd806664d570,ffff8000275632f8,3,fffffd807f7d75b0) at VOP_WRITE+0xc3 sys/kern/vfs_vops.c:245 ktrwriteraw(ffff8000211ce000,fffffd806664d570,fffffd807f7d75b0,ffff8000275633c0,ffff8000275633a0) at ktrwriteraw+0x16c sys/kern/kern_ktrace.c:664 ktrstruct(ffff8000211ce000,ffffffff8285703b,ffff800027563570,20) at ktrstruct+0x15c ktrwrite2 sys/kern/kern_ktrace.c:627 [inline] ktrstruct(ffff8000211ce000,ffffffff8285703b,ffff800027563570,20) at ktrstruct+0x15c sys/kern/kern_ktrace.c:308 sys_kevent(ffff8000211ce000,ffff800027563730,ffff800027563780) at sys_kevent+0x560 sys/kern/kern_event.c:1069 syscall(ffff800027563800) at syscall+0x606 mi_syscall sys/sys/syscall_mi.h:110 [inline] syscall(ffff800027563800) at syscall+0x606 sys/arch/amd64/amd64/trap.c:623 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0x2954e5a30, count: -11 ddb{0}> show registers rdi 0xffffffff rsi 0xffffffff rbp 0xffff800027562fc0 rbx 0xffffffff rdx 0 rcx 0xc rax 0xfffffd80031f8000 r8 0 r9 0 r10 0x6ed67853935f5daa r11 0x342cd926b92b10cc r12 0xfffffd80031d0300 r13 0xfffffd806d5f3030 r14 0xdeaf4152deaf4152 r15 0xfffffd80039bf9b8 rip 0xffffffff81e42ee5 witness_checkorder+0x4f5 cs 0x8 rflags 0x10212 __ALIGN_SIZE+0xf212 rsp 0xffff800027562f10 ss 0x10 witness_checkorder+0x4f5: movl 0x10(%r14),%ecx ddb{0}> show proc PROC (syz-fuzzer) tid=512377 pid=15380 tcnt=16 stat=onproc flags process=2000002 proc=4000001 runpri=24, usrpri=76, slppri=24, nice=20 wchan=0x0, wmesg=, ps_single=0x0 forw=0xffffffffffffffff, list=0xffff8000211e9800,0xffff80002122cab8 process=0xffff800021220440 user=0xffff80002755e000, vmspace=0xfffffd806e3e1580 estcpu=26, cpticks=0, pctcpu=1.64, user=0, sys=0, intr=0 ddb{0}> ps PID TID PPID UID S FLAGS WAIT COMMAND 52969 308781 0 0 3 0x14280 nfsidl nfsio 27520 133795 0 0 3 0x14280 nfsidl nfsio 97783 46267 0 0 3 0x14280 nfsidl nfsio 65111 338864 0 0 3 0x14280 nfsidl nfsio 87121 462932 0 0 3 0x14280 nfsidl nfsio 38493 38953 0 0 3 0x14280 nfsidl nfsio 83133 371032 0 0 3 0x14280 nfsidl nfsio 97591 200454 0 0 3 0x14280 nfsidl nfsio 72870 424513 0 0 3 0x14280 nfsidl nfsio 93377 301281 0 0 3 0x14280 nfsidl nfsio 56420 391691 0 0 3 0x14280 nfsidl nfsio 80361 482787 0 0 3 0x14280 nfsidl nfsio 32660 506246 0 0 3 0x14280 nfsidl nfsio 38102 442273 0 0 3 0x14280 nfsidl nfsio 34867 190314 0 0 3 0x14280 nfsidl nfsio 74138 498903 0 0 3 0x14280 nfsidl nfsio 36143 85314 0 0 3 0x14280 nfsidl nfsio 41944 228561 0 0 3 0x14280 nfsidl nfsio 50058 416987 0 0 3 0x14280 nfsidl nfsio 1697 377179 0 0 3 0x14280 nfsidl nfsio 86812 246188 15380 0 3 0x82 piperd syz-executor.2 75422 400684 0 0 3 0x14200 acct acct 44297 403282 15380 0 2 0x83 syz-executor.6 19754 191996 15380 0 2 0x83 syz-executor.0 32921 254099 15380 0 3 0x82 piperd syz-executor.3 38110 353886 15380 0 2 0x83 syz-executor.4 51373 301864 15380 0 3 0x82 piperd syz-executor.7 55756 23379 15380 0 3 0x82 nanoslp syz-executor.5 45570 398383 1 0 3 0x100083 ttyopn getty 74079 183420 0 0 3 0x14200 bored sosplice 15380 250307 64885 0 3 0x2000082 wait syz-fuzzer 15380 339788 64885 0 2 0x6000083 syz-fuzzer 15380 324940 64885 0 3 0x6000082 thrsleep syz-fuzzer 15380 408977 64885 0 3 0x6000082 thrsleep syz-fuzzer 15380 281023 64885 0 3 0x6000082 wait syz-fuzzer 15380 358419 64885 0 3 0x6000082 wait syz-fuzzer 15380 520223 64885 0 3 0x6000082 wait syz-fuzzer 15380 179365 64885 0 3 0x6000082 wait syz-fuzzer 15380 515486 64885 0 3 0x6000082 thrsleep syz-fuzzer 15380 179979 64885 0 3 0x6000082 wait syz-fuzzer 15380 3610 64885 0 3 0x6000082 thrsleep syz-fuzzer 15380 167493 64885 0 3 0x6000082 thrsleep syz-fuzzer *15380 512377 64885 0 7 0x6000003 syz-fuzzer 15380 76613 64885 0 3 0x6000082 thrsleep syz-fuzzer 15380 339247 64885 0 3 0x6000082 thrsleep syz-fuzzer 15380 500881 64885 0 3 0x6000082 wait syz-fuzzer 64885 16147 77058 0 3 0x10008a sigsusp ksh 77058 288522 52197 0 3 0x9a kqread sshd 52197 9697 1 0 3 0x88 kqread sshd 59042 293936 83632 74 3 0x1100092 bpf pflogd 83632 132043 1 0 3 0x80 netio pflogd 8028 360787 71152 73 3 0x1100090 kqread syslogd 71152 300464 1 0 3 0x100082 netio syslogd 94538 498132 1 0 3 0x100080 kqread resolvd 11359 423954 49867 77 3 0x100092 kqread dhcpleased 78998 269970 49867 77 3 0x100092 kqread dhcpleased 49867 488164 1 0 3 0x80 kqread dhcpleased 31401 34408 0 0 3 0x14200 bored smr 88731 428543 0 0 2 0x14200 zerothread 56382 268763 0 0 3 0x14200 aiodoned aiodoned 87931 63467 0 0 3 0x14200 syncer update 53896 50588 0 0 3 0x14200 cleaner cleaner 86657 288431 0 0 3 0x14200 reaper reaper 84269 76466 0 0 3 0x14200 pgdaemon pagedaemon 25084 290194 0 0 3 0x14200 bored viomb 66988 331542 0 0 3 0x40014200 acpi0 acpi0 7392 139644 0 0 7 0x40014200 idle1 6625 129675 0 0 3 0x14200 bored softnet3 74722 455643 0 0 3 0x14200 bored softnet2 69237 109492 0 0 3 0x14200 bored softnet1 51689 33589 0 0 3 0x14200 bored softnet0 82194 119102 0 0 3 0x14200 bored systqmp 94851 234132 0 0 3 0x14200 bored systq 87238 483504 0 0 3 0x14200 tmoslp softclockmp 10835 177042 0 0 3 0x40014200 tmoslp softclock 51773 376508 0 0 3 0x40014200 idle0 1 297110 0 0 3 0x82 wait init 0 0 -1 0 3 0x10200 scheduler swapper ddb{0}> show all locks Process 15380 (syz-fuzzer) thread 0xffff8000211ce000 (512377) exclusive kernel_lock &kernel_lock r = 0 (0xffffffff82d8ce00) #0 witness_lock+0x447 #1 __mp_acquire_count+0x48 sys/kern/kern_lock.c:227 #2 mi_switch+0x46d sys/kern/sched_bsd.c:470 #3 sleep_finish+0x19b sys/kern/kern_synch.c:414 #4 rwsleep+0xab sys/kern/kern_synch.c:300 #5 sosend+0x76b sys/kern/uipc_socket.c:623 #6 fifo_write+0x80 sys/miscfs/fifofs/fifo_vnops.c:279 #7 VOP_WRITE+0xc3 sys/kern/vfs_vops.c:245 #8 ktrwriteraw+0x16c sys/kern/kern_ktrace.c:664 #9 ktrstruct+0x15c ktrwrite2 sys/kern/kern_ktrace.c:627 [inline] #9 ktrstruct+0x15c sys/kern/kern_ktrace.c:308 #10 sys_kevent+0x560 sys/kern/kern_event.c:1069 #11 syscall+0x606 mi_syscall sys/sys/syscall_mi.h:110 [inline] #11 syscall+0x606 sys/arch/amd64/amd64/trap.c:623 #12 Xsyscall+0x128 ddb{0}> show malloc Type InUse MemUse HighUse Limit Requests Type Lim devbuf 10224 6822K 7203K 78643K 37909 0 pcb 13 18K 20K 78643K 1297 0 rtable 223 6K 7K 78643K 1594 0 pf 33 9K 11K 78643K 503 0 ifaddr 42 16K 18K 78643K 366 0 ifgroup 56 2K 2K 78643K 781 0 sysctl 3 0K 0K 78643K 11 0 counters 60 35K 36K 78643K 502 0 ioctlops 0 0K 4K 78643K 2146 0 iov 0 0K 20K 78643K 1246 0 mount 1 1K 1K 78643K 1 0 log 0 0K 0K 78643K 4 0 vnodes 1714 107K 107K 78643K 10305 0 UFS quota 1 32K 32K 78643K 1 0 UFS mount 5 36K 36K 78643K 5 0 shm 2 1K 9K 78643K 151 0 VM map 2 1K 1K 78643K 2 0 sem 12 0K 0K 78643K 1514 0 dirhash 12 2K 2K 78643K 75 0 ACPI 1697 195K 286K 78643K 12548 0 file desc 9 29K 89K 78643K 15498 0 sigio 0 0K 0K 78643K 305 0 proc 147 96K 118K 78643K 2102 0 subproc 91 5K 7K 78643K 501 0 NFS srvsock 1 0K 0K 78643K 1 0 NFS daemon 1 16K 16K 78643K 1 0 ip_moptions 0 0K 0K 78643K 3072 0 in_multi 88 6K 7K 78643K 606 0 ether_multi 1 0K 0K 78643K 19 0 mrt 1 0K 0K 78643K 13 0 ISOFS mount 1 32K 32K 78643K 1 0 MSDOSFS mount 1 16K 16K 78643K 1 0 ttys 67 307K 307K 78643K 67 0 exec 0 0K 1K 78643K 2976 0 pfkey data 0 0K 0K 78643K 1 0 tdb 3 0K 0K 78643K 3 0 pagedep 1 8K 8K 78643K 1 0 inodedep 1 32K 32K 78643K 1 0 newblk 1 0K 0K 78643K 1 0 VM swap 8 62K 64K 78643K 10 0 UVM amap 483 210K 220K 78643K 150381 0 UVM aobj 131 4K 4K 78643K 134 0 memdesc 1 4K 4K 78643K 1 0 crypto data 1 1K 1K 78643K 1 0 ip6_options 0 0K 0K 78643K 355 0 NDP 12 0K 1K 78643K 309 0 temp 75 5919K 6048K 78643K 179378 0 kqueue 12 18K 42K 78643K 993 0 SYN cache 2 16K 16K 78643K 2 0 ddb{0}> show all pools Name Size Requests Fail Releases Pgreq Pgrel Npage Hiwat Minpg Maxpg Idle plcache 128 22 0 0 1 0 1 1 0 8 0 rtpcb 120 847 0 844 11 10 1 3 0 8 0 rtentry 112 478 0 375 4 0 4 4 0 8 0 unpcb 144 39883 0 39868 148 142 6 13 0 8 5 syncache 304 135 0 135 25 25 0 1 0 8 0 tcpqe 32 443 0 443 21 21 0 1 0 8 0 tcpcb 808 4284 0 4234 167 155 12 18 0 8 6 arp 120 83 0 66 1 0 1 1 0 8 0 inpcb 368 13683 0 13627 196 184 12 17 0 8 5 nd6 136 136 0 112 2 0 2 2 0 8 0 pkpcb 40 55 0 55 4 4 0 1 0 8 0 kcovpl 48 38 0 31 1 0 1 1 0 8 0 ppxss 1256 88 0 88 16 16 0 1 0 8 0 pffrag 232 154 0 154 1 1 0 1 0 482 0 pffrnode 88 150 0 150 1 1 0 1 0 8 0 pffrent 40 434 0 434 1 1 0 1 0 8 0 pfosfp 40 1428 0 1005 5 0 5 5 0 8 0 pfosfpen 112 1428 0 714 21 0 21 21 0 8 0 pfstitem 24 401 0 374 1 0 1 1 0 8 0 pfstkey 128 401 0 374 2 0 2 2 0 8 0 pfstate 376 401 0 374 9 5 4 6 0 8 0 pfrule 1344 21 0 16 2 1 1 2 0 8 0 art_heap8 4096 1 0 0 1 0 1 1 0 8 0 art_heap4 256 1890 0 1462 32 1 31 31 0 8 0 art_table 32 1891 0 1462 4 0 4 4 0 8 0 art_node 16 467 0 373 1 0 1 1 0 8 0 sysvmsgpl 40 34 0 24 1 0 1 1 0 8 0 semupl 112 1 0 1 1 1 0 1 0 8 0 semapl 112 1510 0 1500 1 0 1 1 0 8 0 shmpl 112 131 0 3 4 0 4 4 0 8 0 dirhash 1024 59 0 42 3 0 3 3 0 8 0 dino2pl 256 20093 0 18524 99 0 99 99 0 8 0 ffsino 272 20093 0 18524 105 0 105 105 0 8 0 nchpl 144 40407 0 39921 63 40 23 63 0 8 0 uvmvnodes 80 7704 0 0 158 0 158 158 0 8 0 vnodes 216 7704 0 0 428 0 428 428 0 8 0 namei 1024 142718 0 142718 6 5 1 2 0 8 1 percpumem 16 264 0 221 1 0 1 1 0 8 0 vcpupl 2048 118 0 0 15 0 15 15 0 8 0 vmpool 696 131 0 13 11 0 11 11 0 8 0 kstatmem 264 466 0 442 5 3 2 3 0 8 0 scxspl 216 128111 0 128111 33 32 1 8 1 8 1 plimitpl 152 1504 0 1489 1 0 1 1 0 8 0 sigapl 424 15971 0 15908 10 2 8 9 0 8 0 futexpl 64 130755 0 130755 2 1 1 1 0 8 1 knotepl 120 950 0 0 18 0 18 18 0 8 0 kqueuepl 216 2231 0 2223 44 43 1 8 0 8 0 pipepl 320 3173 0 3148 82 76 6 9 0 8 3 fdescpl 496 15741 0 15719 7 3 4 5 0 8 0 filepl 152 118631 0 118411 184 168 16 29 0 8 4 lockfpl 104 5888 0 5886 12 11 1 4 0 8 0 lockfspl 48 2468 0 2466 3 2 1 2 0 8 0 sessionpl 144 57 0 41 1 0 1 1 0 8 0 pgrppl 48 378 0 362 1 0 1 1 0 8 0 ucredpl 104 13090 0 13074 1 0 1 1 0 8 0 zombiepl 144 15909 0 15908 1 0 1 1 0 8 0 processpl 1072 15971 0 15908 6 1 5 5 0 8 0 procpl 680 41564 0 41486 15 6 9 10 0 8 0 srpgc 96 2 0 2 1 1 0 1 0 8 0 sosppl 168 158 0 155 18 17 1 1 0 8 0 sockpl 488 54697 0 54626 1064 1045 19 46 0 8 8 mcl64k 65536 17 0 0 3 0 3 3 0 8 0 mcl16k 16384 28 0 0 4 2 2 3 0 8 0 mcl12k 12288 17 0 0 2 0 2 2 0 8 0 mcl9k 9216 17 0 0 2 0 2 2 0 8 0 mcl8k 8192 25 0 0 3 0 3 3 0 8 0 mcl4k 4096 28 0 0 3 0 3 3 0 8 0 mcl2k2 2112 17 0 0 2 0 2 2 0 8 0 mcl2k 2048 368 0 0 33 6 27 33 0 8 0 mtagpl 96 934 0 0 19 0 19 19 0 8 0 mbufpl 256 1793 0 0 97 0 97 97 0 8 0 bufpl 288 30524 0 22818 551 0 551 551 0 8 0 anonpl 24 1442130 0 1430100 164 48 116 120 0 186 0 amapchunkpl 152 474169 0 473432 108 68 40 50 0 158 1 amappl16 200 24475 0 23873 91 57 34 45 0 8 1 amappl15 192 20 0 19 1 0 1 1 0 8 0 amappl14 184 280 0 262 2 1 1 2 0 8 0 amappl13 176 9 0 9 1 1 0 1 0 8 0 amappl12 168 16864 0 16838 5 3 2 3 0 8 0 amappl11 160 72 0 56 1 0 1 1 0 8 0 amappl10 152 76 0 64 1 0 1 1 0 8 0 amappl9 144 405 0 404 1 0 1 1 0 8 0 amappl8 136 859 0 674 7 0 7 7 0 8 0 amappl7 128 171 0 152 2 0 2 2 0 8 0 amappl6 120 598 0 565 7 6 1 2 0 8 0 amappl5 112 583 0 572 1 0 1 1 0 8 0 amappl4 104 1070 0 1019 3 1 2 3 0 8 0 amappl3 96 93736 0 93677 4 1 3 3 0 8 0 amappl2 88 16517 0 16452 3 1 2 3 0 8 0 amappl1 80 65782 0 65248 23 11 12 23 0 8 0 amappl 88 149134 0 148898 11 3 8 8 0 92 0 dma4096 4096 1 0 1 1 1 0 1 0 8 0 dma1024 1024 1 0 0 1 0 1 1 0 8 0 dma256 256 6 0 6 1 1 0 1 0 8 0 dma128 128 253 0 253 1 1 0 1 0 8 0 dma64 64 6 0 6 1 1 0 1 0 8 0 dma32 32 7 0 7 1 1 0 1 0 8 0 dma16 16 18 0 17 1 0 1 1 0 8 0 aobjpl 72 133 0 3 3 0 3 3 0 8 0 uaddrrnd 24 15872 0 15732 1 0 1 1 0 8 0 uaddrbest 32 2 0 0 1 0 1 1 0 8 0 uaddr 24 15872 0 15732 1 0 1 1 0 8 0 vmmpekpl 168 114109 0 114017 5 0 5 5 0 8 0 vmmpepl 168 928273 0 925792 481 327 154 155 0 357 21 vmsppl 464 15871 0 15732 21 2 19 19 0 8 0 rwobjpl 56 221162 0 211521 148 11 137 137 0 8 0 pdppl 4096 31752 0 31582 660 476 184 188 0 8 14 pvpl 32 4279094 0 4261584 535 337 198 364 0 265 0 pmappl 248 15871 0 15732 11 1 10 10 0 8 0 extentpl 40 56 0 38 1 0 1 1 0 8 0 phpool 112 2012 0 853 34 0 34 34 0 8 0 ddb{0}> machine ddbcpu 0 Invalid cpu 0 ddb{0}> trace witness_checkorder(fffffd806d5f3030,9,0) at witness_checkorder+0x4f5 witness_lock_order_check sys/kern/subr_witness.c:2442 [inline] witness_checkorder(fffffd806d5f3030,9,0) at witness_checkorder+0x4f5 sys/kern/subr_witness.c:890 rw_enter(fffffd806d5f3020,1) at rw_enter+0xd5 sys/kern/kern_rwlock.c:249 rwsleep(fffffd806d5f3160,fffffd806d5f3020,118,ffffffff82823f2b,0) at rwsleep+0xd6 sys/kern/kern_synch.c:303 sosend(fffffd806d5f3018,0,ffff8000275632f8,0,0,0) at sosend+0x76b sys/kern/uipc_socket.c:623 fifo_write(ffff800027563240) at fifo_write+0x80 sys/miscfs/fifofs/fifo_vnops.c:279 VOP_WRITE(fffffd806664d570,ffff8000275632f8,3,fffffd807f7d75b0) at VOP_WRITE+0xc3 sys/kern/vfs_vops.c:245 ktrwriteraw(ffff8000211ce000,fffffd806664d570,fffffd807f7d75b0,ffff8000275633c0,ffff8000275633a0) at ktrwriteraw+0x16c sys/kern/kern_ktrace.c:664 ktrstruct(ffff8000211ce000,ffffffff8285703b,ffff800027563570,20) at ktrstruct+0x15c ktrwrite2 sys/kern/kern_ktrace.c:627 [inline] ktrstruct(ffff8000211ce000,ffffffff8285703b,ffff800027563570,20) at ktrstruct+0x15c sys/kern/kern_ktrace.c:308 sys_kevent(ffff8000211ce000,ffff800027563730,ffff800027563780) at sys_kevent+0x560 sys/kern/kern_event.c:1069 syscall(ffff800027563800) at syscall+0x606 mi_syscall sys/sys/syscall_mi.h:110 [inline] syscall(ffff800027563800) at syscall+0x606 sys/arch/amd64/amd64/trap.c:623 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0x2954e5a30, count: -11 ddb{0}> machine ddbcpu 1 Stopped at x86_ipi_db+0x1e: addq $0x8,%rsp ddb{1}> trace x86_ipi_db(ffff800020d48ff0) at x86_ipi_db+0x1e sys/arch/amd64/amd64/db_interface.c:393 x86_ipi_handler() at x86_ipi_handler+0xb7 sys/arch/amd64/amd64/ipi.c:106 Xresume_lapic_ipi() at Xresume_lapic_ipi+0x27 acpicpu_idle() at acpicpu_idle+0x312 sys/dev/acpi/acpicpu.c:1206 sched_idle(ffff800020d48ff0) at sched_idle+0x41e sys/kern/kern_sched.c:191 end trace frame: 0x0, count: -5