login: kernel: protection fault trap, code=0 Stopped at witness_checkorder+0xb5: movl 0x20(%r14),%r15d ddb{1}> ddb{1}> set $lines = 0 ddb{1}> set $maxwidth = 0 ddb{1}> show panic the kernel did not panic ddb{1}> trace witness_checkorder(deafbeaddeafbfc5,9,0) at witness_checkorder+0xb5 sys/kern/subr_witness.c:779 mtx_enter(deafbeaddeafbfb5) at mtx_enter+0x95 sys/kern/kern_lock.c:406 prsignal(deafbeaddeafbead,14) at prsignal+0x36 sys/kern/kern_sig.c:904 reaper(ffff8000ffffd9f8) at reaper+0x2ec sys/kern/kern_exit.c:516 end trace frame: 0x0, count: -4 ddb{1}> show registers rdi 0 rsi 0 rbp 0xffff80002a1f47a0 rbx 0 rdx 0 rcx 0xffff8000ffffd9f8 rax 0xffff80002999dffe r8 0xffffffffffffffff r9 0x1 r10 0x6df5843e2f54860c r11 0x354e540f66a9bf5c r12 0xdeafbeaddeafbffe r13 0x9 r14 0xdeafbeaddeafbfc5 r15 0 rip 0xffffffff8151ba85 witness_checkorder+0xb5 cs 0x8 rflags 0x10246 __ALIGN_SIZE+0xf246 rsp 0xffff80002a1f4700 ss 0x10 witness_checkorder+0xb5: movl 0x20(%r14),%r15d ddb{1}> show proc PROC (reaper) tid=81978 pid=64760 tcnt=1 stat=onproc flags process=14000 proc=200 runpri=32, usrpri=50, slppri=32, nice=20 wchan=0x0, wmesg=, ps_single=0x0 scnt=0 ecnt=0 forw=0xffffffffffffffff, list=0xffff8000fffffc88,0xffff8000ffffd770 process=0xffff8000ffffb9c0 user=0xffff80002a1ef000, vmspace=0xffffffff83affb10 estcpu=0, cpticks=6, pctcpu=0.1, user=0, sys=0, intr=0 ddb{1}> ps PID TID PPID UID S FLAGS WAIT COMMAND 17983 174271 0 0 3 0x14280 nfsidl nfsio 98675 192790 0 0 3 0x14280 nfsidl nfsio 21343 426319 0 0 3 0x14280 nfsidl nfsio 82006 295788 0 0 3 0x14280 nfsidl nfsio 3516 69093 0 0 3 0x14280 nfsidl nfsio 9575 318865 36277 -1 7 0x10 syz-executor 9575 510243 36277 -1 3 0x4000090 sysctllk syz-executor 71115 21805 0 0 3 0x14280 nfsidl nfsio 58680 374163 0 0 3 0x14280 nfsidl nfsio 27897 227269 0 0 3 0x14280 nfsidl nfsio 43490 317964 0 0 3 0x14280 nfsidl nfsio 71074 180939 0 0 3 0x14280 nfsidl nfsio 59615 88559 20048 0 2 0 syz-executor 59615 498718 20048 0 3 0x4000080 fsleep syz-executor 61029 426044 0 0 3 0x14280 nfsidl nfsio 821 298466 0 0 3 0x14280 nfsidl nfsio 64199 60709 0 0 3 0x14280 nfsidl nfsio 28517 495915 0 0 3 0x14280 nfsidl nfsio 91799 62944 0 0 3 0x14280 nfsidl nfsio 69589 447876 0 0 3 0x14280 nfsidl nfsio 13383 166773 0 0 3 0x14280 nfsidl nfsio 20042 146453 0 0 3 0x14280 nfsidl nfsio 70026 387476 0 0 3 0x14280 nfsidl nfsio 84519 383139 58456 0 2 0 syz-executor 84519 254153 58456 0 2 0x4000000 syz-executor 8466 120309 67856 0 3 0x80 nanoslp syz-executor 8466 451425 67856 0 3 0x4000080 wsevent_read syz-executor 8466 352147 67856 0 3 0x4000080 fsleep syz-executor 8466 132326 67856 0 3 0x4000080 fsleep syz-executor 897 433497 839 0 4 0x48000000 syz-executor 897 156294 839 0 4 0x4c000000 syz-executor 897 388116 839 0 4 0x4c000080 fsleep syz-executor 58796 518639 1 0 3 0x100083 ttyin getty 24108 491970 58626 0 3 0x2 biowait syz-executor 20048 100314 58626 0 3 0x82 nanoslp syz-executor 65522 184421 58626 0 3 0x82 nanoslp syz-executor 839 230915 58626 0 3 0x82 nanoslp syz-executor 67856 34214 58626 0 2 0x3 syz-executor 36277 501728 58626 0 3 0x82 nanoslp syz-executor 58456 360768 58626 0 3 0x82 nanoslp syz-executor 29771 1726 58626 0 3 0x82 nanoslp syz-executor 58626 254428 1 0 3 0x82 kqread syz-executor 3335 152340 1 74 3 0x1100092 bpf pflogd 52635 330819 1 73 3 0x1100090 kqread syslogd 23845 405435 0 0 3 0x14200 bored smr 44374 361652 0 0 3 0x14200 pgzero zerothread 12231 240284 0 0 3 0x14200 aiodoned aiodoned 42987 136588 0 0 3 0x14200 syncer update 2533 90199 0 0 3 0x14200 cleaner cleaner *64760 81978 0 0 7 0x14200 reaper 77516 266106 0 0 3 0x14200 pgdaemon pagedaemon 47215 458388 0 0 3 0x14200 bored viomb 90147 320080 0 0 3 0x40014200 acpi0 acpi0 46691 324224 0 0 3 0x40014200 idle1 75440 309586 0 0 3 0x14200 bored softnet1 84485 201602 0 0 3 0x14200 bored softnet0 77988 414971 0 0 3 0x14200 bored systqmp 68577 249251 0 0 3 0x14200 bored systq 18382 464802 0 0 3 0x14200 tmoslp softclockmp 24704 232901 0 0 3 0x40014200 tmoslp softclock 39737 63126 0 0 3 0x40014200 idle0 1 160527 0 0 3 0x82 wait init 0 0 -1 0 3 0x10200 scheduler swapper ddb{1}> show all locks Process 84519 (syz-executor) thread 0xffff800039fb9a10 (254153) exclusive rwlock sysctllk r = 0 (0xffffffff8397f7e8) #0 witness_lock+0x5f1 stacktrace_save sys/sys/stacktrace.h:37 [inline] #0 witness_lock+0x5f1 sys/kern/subr_witness.c:1160 #1 rw_do_enter_write+0x419 sys/kern/kern_rwlock.c:320 #2 sysctl_vslock+0x45 sys/kern/kern_sysctl.c:192 #3 sys_sysctl+0x371 sys/kern/kern_sysctl.c:304 #4 syscall+0xbd4 mi_syscall sys/sys/syscall_mi.h:176 [inline] #4 syscall+0xbd4 sys/arch/amd64/amd64/trap.c:783 #5 Xsyscall+0x128 Process 8466 (syz-executor) thread 0xffff8000ffffd230 (451425) exclusive rrwlock inode r = 0 (0xffffee806ee40118) #0 witness_lock+0x5f1 stacktrace_save sys/sys/stacktrace.h:37 [inline] #0 witness_lock+0x5f1 sys/kern/subr_witness.c:1160 #1 rw_do_enter_write+0x419 sys/kern/kern_rwlock.c:320 #2 rrw_enter+0xc6 sys/kern/kern_rwlock.c:621 #3 VOP_LOCK+0xbd sys/kern/vfs_vops.c:527 #4 vn_lock+0xa4 sys/kern/vfs_vnops.c:576 #5 spec_open+0x2f2 sys/kern/spec_vnops.c:151 #6 VOP_OPEN+0x8b sys/kern/vfs_vops.c:138 #7 vn_open+0x7a5 sys/kern/vfs_vnops.c:183 #8 vndioctl+0xc43 sys/dev/vnd.c:458 #9 VOP_IOCTL+0xac sys/kern/vfs_vops.c:264 #10 vn_ioctl+0xf8 sys/kern/vfs_vnops.c:537 #11 sys_ioctl+0x674 sys/kern/sys_generic.c:-1 #12 syscall+0xbd4 mi_syscall sys/sys/syscall_mi.h:176 [inline] #12 syscall+0xbd4 sys/arch/amd64/amd64/trap.c:783 #13 Xsyscall+0x128 Process 24108 (syz-executor) thread 0xffff8000fffee548 (491970) exclusive rrwlock inode r = 0 (0xffffee806c3de1f0) #0 witness_lock+0x5f1 stacktrace_save sys/sys/stacktrace.h:37 [inline] #0 witness_lock+0x5f1 sys/kern/subr_witness.c:1160 #1 rw_do_enter_write+0x419 sys/kern/kern_rwlock.c:320 #2 rrw_enter+0xc6 sys/kern/kern_rwlock.c:621 #3 VOP_LOCK+0xbd sys/kern/vfs_vops.c:527 #4 vn_lock+0xa4 sys/kern/vfs_vnops.c:576 #5 vget+0x2a2 sys/kern/vfs_subr.c:686 #6 ufs_ihashget+0x185 sys/ufs/ufs/ufs_ihash.c:98 #7 ffs_vget+0x8c sys/ufs/ffs/ffs_vfsops.c:1203 #8 ufs_lookup+0x1a36 sys/ufs/ufs/ufs_lookup.c:478 #9 VOP_LOOKUP+0x6e sys/kern/vfs_vops.c:85 #10 vfs_lookup+0x963 sys/kern/vfs_lookup.c:580 #11 namei+0x7c5 sys/kern/vfs_lookup.c:250 #12 dounlinkat+0xc1 sys/kern/vfs_syscalls.c:1806 #13 syscall+0xb17 mi_syscall sys/sys/syscall_mi.h:176 [inline] #13 syscall+0xb17 sys/arch/amd64/amd64/trap.c:783 #14 Xsyscall+0x128 exclusive rrwlock inode r = 0 (0xffffee806f991a00) #0 witness_lock+0x5f1 stacktrace_save sys/sys/stacktrace.h:37 [inline] #0 witness_lock+0x5f1 sys/kern/subr_witness.c:1160 #1 rw_do_enter_write+0x419 sys/kern/kern_rwlock.c:320 #2 rrw_enter+0xc6 sys/kern/kern_rwlock.c:621 #3 VOP_LOCK+0xbd sys/kern/vfs_vops.c:527 #4 vn_lock+0xa4 sys/kern/vfs_vnops.c:576 #5 vget+0x2a2 sys/kern/vfs_subr.c:686 #6 cache_lookup+0x351 sys/kern/vfs_cache.c:222 #7 ufs_lookup+0x1e3 sys/ufs/ufs/ufs_lookup.c:160 #8 VOP_LOOKUP+0x6e sys/kern/vfs_vops.c:85 #9 vfs_lookup+0x963 sys/kern/vfs_lookup.c:580 #10 namei+0x7c5 sys/kern/vfs_lookup.c:250 #11 dounlinkat+0xc1 sys/kern/vfs_syscalls.c:1806 #12 syscall+0xb17 mi_syscall sys/sys/syscall_mi.h:176 [inline] #12 syscall+0xb17 sys/arch/amd64/amd64/trap.c:783 #13 Xsyscall+0x128 Process 64760 (reaper) thread 0xffff8000ffffd9f8 (81978) exclusive kernel_lock &kernel_lock r = 0 (0xffffffff83a102c0) #0 witness_lock+0x5f1 stacktrace_save sys/sys/stacktrace.h:37 [inline] #0 witness_lock+0x5f1 sys/kern/subr_witness.c:1160 #1 __mp_acquire_count+0x58 sys/kern/kern_lock.c:-1 #2 sleep_finish+0x2d8 sys/kern/kern_synch.c:369 #3 rw_do_enter_write+0x1dc sys/kern/kern_rwlock.c:298 #4 knote_processexit+0x2b sys/kern/kern_event.c:2235 #5 reaper+0x26d sys/kern/kern_exit.c:513 #6 proc_trampoline+0x10 ddb{1}> show malloc Type InUse MemUse HighUse Limit Requests Type Lim devbuf 11049 12088K 12423K 166960K 12244 0 pcb 18 14K 15K 166960K 50 0 rtable 247 8K 8K 166960K 372 0 pf 35 17K 18K 166960K 50 0 ifaddr 43 7K 7K 166960K 46 0 ifgroup 58 2K 2K 166960K 60 0 sysctl 1 1K 9K 166960K 5 0 counters 70 37K 37K 166960K 76 0 ioctlops 0 0K 4K 166960K 1539 0 iov 0 0K 1K 166960K 1 0 mount 1 1K 1K 166960K 1 0 log 0 0K 0K 166960K 4 0 vnodes 1329 84K 84K 166960K 1500 0 UFS quota 1 32K 32K 166960K 1 0 UFS mount 5 36K 36K 166960K 5 0 shm 2 1K 5K 166960K 3 0 VM map 2 1K 1K 166960K 2 0 sem 6 0K 0K 166960K 9 0 dirhash 12 2K 2K 166960K 12 0 ACPI 1692 195K 286K 166960K 12470 0 file desc 16 57K 97K 166960K 221 0 sigio 0 0K 0K 166960K 4 0 proc 21 33K 180K 166960K 568 0 subproc 72 4K 4K 166960K 72 0 NFS srvsock 1 0K 0K 166960K 1 0 NFS daemon 1 16K 16K 166960K 1 0 ip_moptions 0 0K 0K 166960K 11 0 in_multi 99 7K 7K 166960K 105 0 ether_multi 1 0K 0K 166960K 1 0 mrt 1 0K 0K 166960K 12 0 ISOFS mount 1 32K 32K 166960K 1 0 MSDOSFS mount 1 16K 16K 166960K 1 0 ttys 55 254K 254K 166960K 55 0 exec 0 0K 1K 166960K 390 0 fusefs mount 1 32K 32K 166960K 1 0 tdb 3 0K 0K 166960K 3 0 VM swap 8 62K 64K 166960K 10 0 UVM amap 156 91K 168K 166960K 3806 0 UVM aobj 37 2K 2K 166960K 38 0 pinsyscall 22 44K 106K 166960K 1381 0 memdesc 1 4K 4K 166960K 1 0 crypto data 1 1K 1K 166960K 1 0 ip6_options 0 0K 0K 166960K 4 0 NDP 12 0K 2K 166960K 29 0 temp 42 9115K 9146K 166960K 7054 0 kqueue 5 7K 24K 166960K 44 0 SYN cache 2 16K 16K 166960K 2 0 ddb{1}> show all pools Name Size Requests Fail Releases Pgreq Pgrel Npage Hiwat Minpg Maxpg Idle plcache 128 26 0 0 1 0 1 1 0 8 0 rtpcb 120 37 0 37 1 0 1 1 0 8 1 rtentry 176 112 0 2 6 1 5 6 0 8 0 unpcb 144 120 0 111 2 0 2 2 0 8 1 syncache 336 4 0 4 1 1 0 1 0 8 0 tcpcb 736 30 0 29 1 0 1 1 0 8 0 arp 136 18 0 0 1 0 1 1 0 8 0 inpcb 328 221 0 216 8 1 7 7 0 8 6 nd6 152 24 0 0 1 0 1 1 0 8 0 kcovpl 48 8 0 0 1 0 1 1 0 8 0 ppxss 1192 3 0 3 1 1 0 1 0 8 0 pffrag 232 1 0 0 1 0 1 1 0 482 0 pffrnode 88 1 0 0 1 0 1 1 0 8 0 pffrent 40 1 0 0 1 0 1 1 0 8 0 pfosfp 40 1428 0 1005 5 0 5 5 0 8 0 pfosfpen 112 1428 0 714 21 0 21 21 0 8 0 pfstitem 24 30 0 0 1 0 1 1 0 8 0 pfstkey 128 30 0 0 1 0 1 1 0 8 0 pfstate 448 30 0 0 4 0 4 4 0 8 0 pfrule 1360 23 0 17 2 1 1 2 0 8 0 art_heap8 4096 1 0 0 1 0 1 1 0 8 0 art_heap4 256 451 0 0 29 0 29 29 0 8 0 art_table 40 452 0 0 5 0 5 5 0 8 0 art_node 32 111 0 11 1 0 1 1 0 8 0 sysvmsgpl 40 2 0 1 1 0 1 1 0 8 0 semapl 72 6 0 2 1 0 1 1 0 8 0 shmpl 112 35 0 1 1 0 1 1 0 8 0 dirhash 1024 17 0 0 3 0 3 3 0 8 0 dino2pl 256 1757 0 285 93 0 93 93 0 8 0 ffsino 296 1757 0 285 114 0 114 114 0 8 0 nchpl 144 2065 0 354 64 0 64 64 0 8 0 rtmask 32 1 0 1 1 1 0 1 0 8 0 vnodes 216 1923 0 0 107 0 107 107 0 8 0 namei 1024 6515 0 6515 3 2 1 2 0 8 1 percpumem 16 53 0 3 1 0 1 1 0 8 0 kstatmem 264 29 0 2 2 0 2 2 0 8 0 scsiplug 72 1 0 1 1 0 1 1 0 8 1 scxspl 216 6862 0 6861 5 3 2 3 1 8 1 plimitpl 152 37 0 25 1 0 1 1 0 8 0 sigapl 424 558 0 502 8 1 7 7 0 8 0 knotepl 120 429 0 0 13 0 13 13 0 8 0 kqueuepl 224 51 0 48 1 0 1 1 0 8 0 pipepl 344 132 0 105 3 0 3 3 0 8 0 fdescpl 528 523 0 503 3 0 3 3 0 8 0 filepl 160 2276 0 2087 14 0 14 14 0 8 5 lockfpl 104 50 0 50 1 0 1 1 0 8 1 lockfspl 48 23 0 23 1 0 1 1 0 8 1 sessionpl 144 30 0 25 1 0 1 1 0 8 0 pgrppl 48 39 0 25 1 0 1 1 0 8 0 ucredpl 104 288 0 277 1 0 1 1 0 8 0 zombiepl 144 553 0 551 1 0 1 1 0 8 0 processpl 1232 558 0 502 6 1 5 5 0 8 0 procpl 664 717 0 648 8 1 7 8 0 8 0 sosppl 176 2 0 2 1 0 1 1 0 8 1 sockpl 752 380 0 366 14 4 10 12 0 8 8 mcl64k 65536 4 0 0 1 0 1 1 0 8 0 mcl16k 16384 1 0 0 1 0 1 1 0 8 0 mcl8k 8192 2 0 0 1 0 1 1 0 8 0 mcl4k 4096 116 0 0 15 0 15 15 0 8 0 mcl2k2 2112 1 0 0 1 0 1 1 0 8 0 mcl2k 2048 19 0 0 3 0 3 3 0 8 0 mtagpl 96 3 0 0 1 0 1 1 0 8 0 mbufpl 256 148 0 0 10 0 10 10 0 8 0 bufpl 272 2503 0 105 160 0 160 160 0 8 0 anonpl 32 4161 0 0 34 0 34 34 0 246 0 amapchunkpl 152 11093 0 10705 28 6 22 25 0 158 2 amappl16 200 1625 0 1609 5 3 2 5 0 8 0 amappl15 192 6 0 6 1 1 0 1 0 8 0 amappl14 184 424 0 424 1 1 0 1 0 8 0 amappl13 176 122 0 119 1 0 1 1 0 8 0 amappl12 168 774 0 755 2 0 2 2 0 8 0 amappl11 160 6 0 6 1 1 0 1 0 8 0 amappl10 152 63 0 59 1 0 1 1 0 8 0 amappl9 144 275 0 275 1 1 0 1 0 8 0 amappl8 136 98 0 97 1 0 1 1 0 8 0 amappl7 128 149 0 144 1 0 1 1 0 8 0 amappl6 120 158 0 157 1 0 1 1 0 8 0 amappl5 112 98 0 96 1 0 1 1 0 8 0 amappl4 104 301 0 292 1 0 1 1 0 8 0 amappl3 96 2059 0 1975 3 0 3 3 0 8 0 amappl2 88 547 0 528 2 0 2 2 0 8 0 amappl1 80 10815 0 10639 15 3 12 15 0 8 1 amappl 88 3067 0 2937 4 0 4 4 0 92 0 uvmvnodes 80 104 0 0 3 0 3 3 0 8 0 dma4096 4096 1 0 1 1 1 0 1 0 8 0 dma1024 1024 1 0 0 1 0 1 1 0 8 0 dma256 256 6 0 6 1 1 0 1 0 8 0 dma128 128 253 0 253 1 1 0 1 0 8 0 dma64 64 6 0 6 1 1 0 1 0 8 0 dma32 32 7 0 7 1 1 0 1 0 8 0 dma16 16 18 0 17 1 0 1 1 0 8 0 aobjpl 72 37 0 1 1 0 1 1 0 8 0 uaddrrnd 24 523 0 503 1 0 1 1 0 8 0 uaddrbest 32 2 0 0 1 0 1 1 0 8 0 uaddr 24 523 0 503 1 0 1 1 0 8 0 vmmpekpl 168 6401 0 6361 2 0 2 2 0 8 0 vmmpepl 168 42197 0 41224 87 26 61 87 0 357 2 vmsppl 488 522 0 502 5 0 5 5 0 8 1 rwobjpl 80 15403 0 14848 24 0 24 24 0 8 0 pdppl 4096 1053 0 1004 101 50 51 87 0 8 2 pvpl 32 9287 0 0 76 1 75 75 0 265 0 pmappl 256 522 0 502 3 0 3 3 0 8 0 extentpl 40 45 0 27 1 0 1 1 0 8 0 phpool 112 435 0 56 12 0 12 12 0 8 0 ddb{1}> machine ddbcpu 0 Stopped at x86_ipi_db+0x27: addq $0x8,%rsp ddb{0}> trace x86_ipi_db(ffffffff83910ff0) at x86_ipi_db+0x27 sys/arch/amd64/amd64/db_interface.c:394 x86_ipi_handler() at x86_ipi_handler+0xd9 sys/arch/amd64/amd64/ipi.c:106 Xresume_lapic_ipi() at Xresume_lapic_ipi+0x27 __sanitizer_cov_trace_pc() at __sanitizer_cov_trace_pc+0x25 kd_curproc sys/dev/kcov.c:584 [inline] __sanitizer_cov_trace_pc() at __sanitizer_cov_trace_pc+0x25 sys/dev/kcov.c:153 __mp_lock(ffffffff83a0fac0) at __mp_lock+0x1a3 __mp_lock_spin sys/kern/kern_lock.c:142 [inline] __mp_lock(ffffffff83a0fac0) at __mp_lock+0x1a3 sys/kern/kern_lock.c:173 intr_handler(ffff80003c435100,ffff8000002a3480) at intr_handler+0xe9 sys/arch/amd64/amd64/intr.c:560 Xintr_ioapic_edge23_untramp() at Xintr_ioapic_edge23_untramp+0x18f end of kernel end trace frame: 0x72013d063c40, count: -7 ddb{0}> machine ddbcpu 1 Stopped at witness_checkorder+0xb5: movl 0x20(%r14),%r15d ddb{1}> trace witness_checkorder(deafbeaddeafbfc5,9,0) at witness_checkorder+0xb5 sys/kern/subr_witness.c:779 mtx_enter(deafbeaddeafbfb5) at mtx_enter+0x95 sys/kern/kern_lock.c:406 prsignal(deafbeaddeafbead,14) at prsignal+0x36 sys/kern/kern_sig.c:904 reaper(ffff8000ffffd9f8) at reaper+0x2ec sys/kern/kern_exit.c:516 end trace frame: 0x0, count: -4 ddb{1}>