login: witness: lock order reversal: 1st 0xfffffd806e85d890 vmmaplk (&map->lock) 2nd 0xfffffd806da442c0 inode (&ip->i_lock) lock order "&ip->i_lock"(rrwlock) -> "&map->lock"(rwlock) first seen at: #0 witness_checkorder+0x6a7 sys/kern/subr_witness.c:877 #1 rw_enter_read+0x66 sys/kern/kern_rwlock.c:101 #2 uvmfault_lookup+0xd9 sys/uvm/uvm_fault.c:1448 #3 uvm_fault+0x7c sys/uvm/uvm_fault.c:524 #4 pageflttrap+0x20b sys/arch/amd64/amd64/trap.c:200 #5 kerntrap+0xec sys/arch/amd64/amd64/trap.c:288 #6 alltraps_kern+0x7b #7 copyout+0x53 #8 ffs_read+0x362 sys/ufs/ffs/ffs_vnops.c:254 #9 VOP_READ+0x63 sys/kern/vfs_vops.c:247 #10 vn_read+0x1c3 sys/kern/vfs_vnops.c:365 #11 dofilereadv+0x19f sys/kern/sys_generic.c:236 #12 sys_read+0x83 sys/kern/sys_generic.c:156 #13 syscall+0x552 mi_syscall sys/sys/syscall_mi.h:99 [inline] #13 syscall+0x552 sys/arch/amd64/amd64/trap.c:574 #14 Xsyscall+0x128 lock order "&map->lock"(rwlock) -> "&ip->i_lock"(rrwlock) first seen at: #0 witness_checkorder+0x6a7 sys/kern/subr_witness.c:877 #1 rw_enter+0xa7 #2 rrw_enter+0x4f sys/kern/kern_rwlock.c:407 #3 VOP_LOCK+0x4b sys/kern/vfs_vops.c:602 #4 vn_lock+0x6e sys/kern/vfs_vnops.c:549 #5 uvn_io+0x2c3 sys/uvm/uvm_vnode.c:1188 #6 uvn_get+0x226 sys/uvm/uvm_vnode.c:1048 #7 uvm_fault+0x11a5 sys/uvm/uvm_fault.c:1023 #8 uvm_fault_wire+0x70 sys/uvm/uvm_fault.c:1295 #9 uvm_map_pageable_wire+0x2dd sys/uvm/uvm_map.c:2266 #10 sys_mlockall+0x6d sys/uvm/uvm_mmap.c:800 #11 syscall+0x552 mi_syscall sys/sys/syscall_mi.h:99 [inline] #11 syscall+0x552 sys/arch/amd64/amd64/trap.c:574 #12 Xsyscall+0x128 Stopped at db_enter+0x18: addq $0x8,%rsp ddb{1}> ddb{1}> set $lines = 0 ddb{1}> set $maxwidth = 0 ddb{1}> show panic the kernel did not panic ddb{1}> trace db_enter() at db_enter+0x18 sys/arch/amd64/amd64/db_interface.c:398 witness_checkorder(fffffd806da442c0,9,0) at witness_checkorder+0xffc witness_debugger sys/kern/subr_witness.c:2504 [inline] witness_checkorder(fffffd806da442c0,9,0) at witness_checkorder+0xffc sys/kern/subr_witness.c:1084 rw_enter(fffffd806da442b0,81) at rw_enter+0xa7 rrw_enter(fffffd806da442b0,81) at rrw_enter+0x4f sys/kern/kern_rwlock.c:407 VOP_LOCK(fffffd8071a70328,81) at VOP_LOCK+0x4b sys/kern/vfs_vops.c:602 vn_lock(fffffd8071a70328,81) at vn_lock+0x6e sys/kern/vfs_vnops.c:549 uvn_io(fffffd806dbba2d8,ffff800020bc70b8,1,2,0) at uvn_io+0x2c3 sys/uvm/uvm_vnode.c:1188 uvn_get(fffffd806dbba2d8,4000,ffff800020bc7310,ffff800020bc72ac,0,1) at uvn_get+0x226 sys/uvm/uvm_vnode.c:1048 uvm_fault(fffffd806e85d878,48a66a51000,2,1) at uvm_fault+0x11a5 sys/uvm/uvm_fault.c:1023 uvm_fault_wire(fffffd806e85d878,48a66a4d000,48a66a54000,1) at uvm_fault_wire+0x70 sys/uvm/uvm_fault.c:1295 uvm_map_pageable_wire(fffffd806e85d878,fffffd806f10ebe0,0,fffffd8002c6bea0,ffffffff8155a568,0) at uvm_map_pageable_wire+0x2dd sys/uvm/uvm_map.c:2266 sys_mlockall(ffff800020b84010,ffff800020bc7508,ffff800020bc7570) at sys_mlockall+0x6d sys/uvm/uvm_mmap.c:800 syscall(ffff800020bc75e0) at syscall+0x552 mi_syscall sys/sys/syscall_mi.h:99 [inline] syscall(ffff800020bc75e0) at syscall+0x552 sys/arch/amd64/amd64/trap.c:574 Xsyscall(6,0,48a66a750f0,0,48a66a750d0,48a66a750c8) at Xsyscall+0x128 end of kernel end trace frame: 0x30656c69662f2e, count: -14 ddb{1}> show registers rdi 0x3 rsi 0xffffffff821d2cb0 __sancov_gen_cov_switch_values.122 rbp 0xffff800020bc6d30 rbx 0x3 rdx 0x8b rcx 0x3 rax 0x3 r8 0xffffffff81646be5 witness_checkorder+0xfd5 r9 0x5 r10 0xa57349b5ef739668 r11 0xaf19c71296956390 r12 0xfffffd8002653ec0 r13 0 r14 0xffffffff822e22d0 w_lodata+0x4f940 r15 0xffffffff822e69a0 w_lodata+0x54010 rip 0xffffffff811b7a58 db_enter+0x18 cs 0x8 rflags 0x246 rsp 0xffff800020bc6d20 ss 0x10 db_enter+0x18: addq $0x8,%rsp ddb{1}> show proc PROC (syz-executor8217) pid=209331 stat=onproc flags process=2 proc=4000000 pri=53, usrpri=53, nice=20 forw=0xffffffffffffffff, list=0xffff800020b85528,0xffffffff823503e0 process=0xffff800020b3b708 user=0xffff800020bc2000, vmspace=0xfffffd806e85d878 estcpu=3, cpticks=3, pctcpu=0.0 user=0, sys=3, intr=0 ddb{1}> ps PID TID PPID UID S FLAGS WAIT COMMAND 32418 316164 53060 0 7 0x2 syz-executor8217 32418 362432 53060 0 3 0x4000002 bqwait syz-executor8217 *32418 209331 53060 0 7 0x4000002 syz-executor8217 53060 99226 16948 0 3 0x10008a pause ksh 16948 502814 14723 0 3 0x92 select sshd 23773 217138 1 0 3 0x100083 ttyin getty 14723 504456 1 0 3 0x80 select sshd 48846 186060 7982 74 3 0x100092 bpf pflogd 7982 448831 1 0 3 0x80 netio pflogd 52703 121025 40929 73 2 0x100090 syslogd 40929 352526 1 0 3 0x100082 netio syslogd 2100 439563 1 77 3 0x100090 poll dhclient 21460 170063 1 0 3 0x80 poll dhclient 25314 119435 0 0 2 0x14200 zerothread 58272 95456 0 0 3 0x14200 aiodoned aiodoned 14078 53992 0 0 3 0x14200 syncer update 69499 506273 0 0 3 0x14200 cleaner cleaner 80757 374369 0 0 3 0x14200 reaper reaper 94539 140773 0 0 3 0x14200 pgdaemon pagedaemon 68565 187627 0 0 3 0x14200 bored crynlk 76225 155975 0 0 3 0x14200 bored crypto 31857 335019 0 0 3 0x40014200 acpi0 acpi0 31908 354967 0 0 3 0x40014200 idle1 49597 435819 0 0 3 0x14200 bored softnet 85779 273331 0 0 3 0x14200 bored systqmp 57558 192334 0 0 3 0x14200 bored systq 44508 274261 0 0 3 0x40014200 bored softclock 91293 182457 0 0 3 0x40014200 idle0 58838 345896 0 0 3 0x14200 bored smr 1 261577 0 0 3 0x82 wait init 0 0 -1 0 3 0x10200 scheduler swapper ddb{1}> show all locks Process 32418 (syz-executor8217) thread 0xffff800020b85528 (362432) exclusive rrwlock inode r = 0 (0xfffffd806da44810) #0 witness_lock+0x52e sys/kern/subr_witness.c:1161 #1 rw_enter+0x414 sys/kern/kern_rwlock.c:278 #2 rrw_enter+0x4f sys/kern/kern_rwlock.c:407 #3 VOP_LOCK+0x4b sys/kern/vfs_vops.c:602 #4 vn_write+0x169 vn_lock sys/kern/vfs_vnops.c:549 [inline] #4 vn_write+0x169 sys/kern/vfs_vnops.c:393 #5 dofilewritev+0x1a9 sys/kern/sys_generic.c:364 #6 sys_pwritev+0xb8 sys/kern/vfs_syscalls.c:3161 #7 syscall+0x552 mi_syscall sys/sys/syscall_mi.h:99 [inline] #7 syscall+0x552 sys/arch/amd64/amd64/trap.c:574 #8 Xsyscall+0x128 Process 32418 (syz-executor8217) thread 0xffff800020b84010 (209331) shared rwlock vmmaplk r = 0 (0xfffffd806e85d890) #0 witness_lock+0x52e sys/kern/subr_witness.c:1161 #1 rw_enter+0x414 sys/kern/kern_rwlock.c:278 #2 vm_map_lock_ln+0x10d sys/uvm/uvm_map.c:5260 #3 uvm_map_pageable_all+0x56 sys/uvm/uvm_map.c:2475 #4 sys_mlockall+0x6d sys/uvm/uvm_mmap.c:800 #5 syscall+0x552 mi_syscall sys/sys/syscall_mi.h:99 [inline] #5 syscall+0x552 sys/arch/amd64/amd64/trap.c:574 #6 Xsyscall+0x128 exclusive kernel_lock &kernel_lock r = 0 (0xffffffff82351510) #0 witness_lock+0x52e sys/kern/subr_witness.c:1161 #1 syscall+0x43a mi_syscall sys/sys/syscall_mi.h:91 [inline] #1 syscall+0x43a sys/arch/amd64/amd64/trap.c:574 #2 Xsyscall+0x128 ddb{1}> show malloc Type InUse MemUse HighUse Limit Requests Type Lim Kern Lim devbuf 9450 6382K 6383K 78643K 10537 0 0 pcb 25 9K 9K 78643K 57 0 0 rtable 61 2K 2K 78643K 125 0 0 ifaddr 25 7K 7K 78643K 26 0 0 counters 39 33K 33K 78643K 39 0 0 ioctlops 0 0K 4K 78643K 1467 0 0 iov 1 12K 12K 78643K 1 0 0 mount 1 1K 1K 78643K 1 0 0 vnodes 1167 73K 73K 78643K 1172 0 0 UFS quota 1 32K 32K 78643K 1 0 0 UFS mount 5 36K 36K 78643K 5 0 0 shm 2 1K 1K 78643K 2 0 0 VM map 2 1K 1K 78643K 2 0 0 sem 2 0K 0K 78643K 2 0 0 dirhash 12 2K 2K 78643K 12 0 0 ACPI 1808 196K 290K 78643K 12628 0 0 file desc 1 0K 0K 78643K 1 0 0 proc 52 50K 58K 78643K 279 0 0 NFS srvsock 1 0K 0K 78643K 1 0 0 NFS daemon 1 16K 16K 78643K 1 0 0 in_multi 11 0K 0K 78643K 11 0 0 ether_multi 1 0K 0K 78643K 1 0 0 ISOFS mount 1 32K 32K 78643K 1 0 0 MSDOSFS mount 1 16K 16K 78643K 1 0 0 ttys 18 79K 79K 78643K 18 0 0 exec 0 0K 1K 78643K 179 0 0 pagedep 1 8K 8K 78643K 1 0 0 inodedep 1 32K 32K 78643K 1 0 0 newblk 1 0K 0K 78643K 1 0 0 VM swap 7 26K 26K 78643K 7 0 0 UVM amap 58 4K 4K 78643K 772 0 0 UVM aobj 2 2K 2K 78643K 2 0 0 memdesc 1 4K 4K 78643K 1 0 0 crypto data 1 1K 1K 78643K 1 0 0 NDP 4 0K 0K 78643K 4 0 0 temp 39 2700K 2764K 78643K 1993 0 0 SYN cache 2 16K 16K 78643K 2 0 0 ddb{1}> show all pools Name Size Requests Fail Releases Pgreq Pgrel Npage Hiwat Minpg Maxpg Idle arp 64 2 0 0 1 0 1 1 0 8 0 inpcbpl 280 29 0 23 1 0 1 1 0 8 0 plimitpl 152 14 0 8 1 0 1 1 0 8 0 plcache 128 20 0 0 1 0 1 1 0 8 0 rtentry 112 23 0 1 1 0 1 1 0 8 0 syncache 264 5 0 5 1 0 1 1 0 8 1 tcpcb 544 8 0 5 1 0 1 1 0 8 0 pfosfp 40 846 0 423 5 0 5 5 0 8 0 pfosfpen 112 1428 0 714 21 0 21 21 0 8 0 pfstitem 24 8 0 0 1 0 1 1 0 8 0 pfstkey 112 8 0 0 1 0 1 1 0 8 0 pfstate 328 8 0 0 1 0 1 1 0 8 0 pfrule 1360 21 0 16 2 1 1 2 0 8 0 art_heap8 4096 1 0 0 1 0 1 1 0 8 0 art_heap4 256 96 0 0 6 0 6 6 0 8 0 art_table 32 97 0 0 1 0 1 1 0 8 0 art_node 16 22 0 2 1 0 1 1 0 8 0 dirhash 1024 17 0 0 3 0 3 3 0 8 0 dino1pl 128 1391 0 17 45 0 45 45 0 8 0 ffsino 272 1391 0 17 92 0 92 92 0 8 0 nchpl 144 1564 0 33 57 0 57 57 0 8 0 uvmvnodes 72 1401 0 0 26 0 26 26 0 8 0 vnodes 200 1401 0 0 74 0 74 74 0 8 0 namei 1024 3549 0 3549 2 1 1 1 0 8 1 percpumem 16 30 0 0 1 0 1 1 0 8 0 scxspl 192 3015 0 2946 8 1 7 7 0 8 1 sigapl 432 205 0 193 2 0 2 2 0 8 0 futexpl 56 1 0 1 1 0 1 1 0 8 1 knotepl 112 5 0 0 1 0 1 1 0 8 0 kqueuepl 104 1 0 0 1 0 1 1 0 8 0 pipepl 112 134 0 127 2 1 1 1 0 8 0 fdescpl 488 206 0 193 3 0 3 3 0 8 0 filepl 152 961 0 913 2 0 2 2 0 8 0 lockfpl 104 7 0 6 2 1 1 1 0 8 0 lockfspl 32 4 0 3 2 1 1 1 0 8 0 sessionpl 112 18 0 9 1 0 1 1 0 8 0 pgrppl 48 18 0 9 1 0 1 1 0 8 0 ucredpl 96 52 0 43 1 0 1 1 0 8 0 zombiepl 144 193 0 193 2 1 1 1 0 8 1 processpl 840 221 0 193 4 0 4 4 0 8 0 procpl 600 223 0 193 3 0 3 3 0 8 0 sockpl 384 73 0 55 2 0 2 2 0 8 0 mcl4k 4096 1 0 0 1 0 1 1 0 8 0 mcl2k 2048 71 0 0 9 0 9 9 0 8 0 mtagpl 80 1 0 0 1 0 1 1 0 8 0 mbufpl 256 98 0 0 6 0 6 6 0 8 0 bufpl 256 2718 0 271 153 0 153 153 0 8 0 anonpl 16 23499 0 18163 24 2 22 22 0 125 0 amapchunkpl 152 678 0 525 6 0 6 6 0 158 0 amappl16 192 226 0 77 8 0 8 8 0 8 0 amappl14 176 16 0 15 2 1 1 1 0 8 0 amappl12 160 9 0 9 1 0 1 1 0 8 1 amappl11 152 41 0 23 1 0 1 1 0 8 0 amappl10 144 57 0 55 1 0 1 1 0 8 0 amappl9 136 429 0 428 1 0 1 1 0 8 0 amappl8 128 98 0 93 1 0 1 1 0 8 0 amappl7 120 19 0 18 1 0 1 1 0 8 0 amappl6 112 45 0 41 1 0 1 1 0 8 0 amappl5 104 113 0 99 1 0 1 1 0 8 0 amappl4 96 454 0 428 1 0 1 1 0 8 0 amappl3 88 105 0 99 1 0 1 1 0 8 0 amappl2 80 650 0 600 2 0 2 2 0 8 0 amappl1 72 13467 0 13031 15 5 10 15 0 8 0 amappl 72 433 0 409 1 0 1 1 0 75 0 dma4096 4096 1 0 1 1 1 0 1 0 8 0 dma256 256 6 0 6 1 1 0 1 0 8 0 dma64 64 259 0 259 1 1 0 1 0 8 0 dma32 32 7 0 7 1 1 0 1 0 8 0 dma16 16 17 0 17 1 1 0 1 0 8 0 aobjpl 64 1 0 0 1 0 1 1 0 8 0 uaddrrnd 24 206 0 193 1 0 1 1 0 8 0 uaddrbest 32 2 0 0 1 0 1 1 0 8 0 uaddr 24 206 0 193 1 0 1 1 0 8 0 vmmpekpl 168 5601 0 5579 2 0 2 2 0 8 0 vmmpepl 168 27506 0 26549 53 11 42 47 0 357 0 vmsppl 360 205 0 193 2 0 2 2 0 8 0 pdppl 4096 420 0 386 6 0 6 6 0 8 0 pvpl 32 87099 0 80011 61 3 58 58 0 265 0 pmappl 232 205 0 193 1 0 1 1 0 8 0 extentpl 40 39 0 25 1 0 1 1 0 8 0 phpool 112 284 0 4 8 0 8 8 0 8 0 ddb{1}>