login: witness: lock order reversal: 1st 0xfffffd806e8df750 vmmaplk (&map->lock) 2nd 0xfffffd8077e7f700 inode (&ip->i_lock) lock order "&ip->i_lock"(rrwlock) -> "&map->lock"(rwlock) first seen at: #0 witness_checkorder+0x6a7 sys/kern/subr_witness.c:877 #1 rw_enter_read+0x66 sys/kern/kern_rwlock.c:109 #2 uvmfault_lookup+0xd9 sys/uvm/uvm_fault.c:1448 #3 uvm_fault+0x7c sys/uvm/uvm_fault.c:524 #4 pageflttrap+0x20b sys/arch/amd64/amd64/trap.c:199 #5 kerntrap+0xec sys/arch/amd64/amd64/trap.c:287 #6 alltraps_kern+0x7b #7 copyout+0x53 #8 ffs_read+0x362 sys/ufs/ffs/ffs_vnops.c:254 #9 VOP_READ+0x63 sys/kern/vfs_vops.c:247 #10 vn_read+0x1c3 sys/kern/vfs_vnops.c:365 #11 dofilereadv+0x1a2 sys/kern/sys_generic.c:236 #12 sys_read+0x83 sys/kern/sys_generic.c:156 #13 syscall+0x552 mi_syscall sys/sys/syscall_mi.h:90 [inline] #13 syscall+0x552 sys/arch/amd64/amd64/trap.c:554 #14 Xsyscall+0x128 lock order "&map->lock"(rwlock) -> "&ip->i_lock"(rrwlock) first seen at: #0 witness_checkorder+0x6a7 sys/kern/subr_witness.c:877 #1 rw_enter+0xd1 sys/kern/kern_rwlock.c:247 #2 rrw_enter+0x4f sys/kern/kern_rwlock.c:435 #3 VOP_LOCK+0x4b sys/kern/vfs_vops.c:602 #4 vn_lock+0x6e sys/kern/vfs_vnops.c:549 #5 uvn_io+0x2c3 sys/uvm/uvm_vnode.c:1188 #6 uvn_get+0x226 sys/uvm/uvm_vnode.c:1048 #7 uvm_fault+0x11a5 sys/uvm/uvm_fault.c:1023 #8 uvm_fault_wire+0x70 sys/uvm/uvm_fault.c:1295 #9 uvm_map_pageable_wire+0x2dd sys/uvm/uvm_map.c:2340 #10 uvm_mmaplock+0x181 sys/uvm/uvm_mmap.c:859 #11 sys_mmap+0xc77 sys/uvm/uvm_mmap.c:394 #12 syscall+0x552 mi_syscall sys/sys/syscall_mi.h:90 [inline] #12 syscall+0x552 sys/arch/amd64/amd64/trap.c:554 #13 Xsyscall+0x128 Stopped at db_enter+0x18: addq $0x8,%rsp ddb{1}> ddb{1}> set $lines = 0 ddb{1}> set $maxwidth = 0 ddb{1}> show panic the kernel did not panic ddb{1}> trace db_enter() at db_enter+0x18 sys/arch/amd64/amd64/db_interface.c:398 witness_checkorder(fffffd8077e7f700,9,0) at witness_checkorder+0xffc witness_debugger sys/kern/subr_witness.c:2504 [inline] witness_checkorder(fffffd8077e7f700,9,0) at witness_checkorder+0xffc sys/kern/subr_witness.c:1084 rw_enter(fffffd8077e7f6f0,81) at rw_enter+0xd1 sys/kern/kern_rwlock.c:247 rrw_enter(fffffd8077e7f6f0,81) at rrw_enter+0x4f sys/kern/kern_rwlock.c:435 VOP_LOCK(fffffd806dc0d198,81) at VOP_LOCK+0x4b sys/kern/vfs_vops.c:602 vn_lock(fffffd806dc0d198,81) at vn_lock+0x6e sys/kern/vfs_vnops.c:549 uvn_io(fffffd806e014560,ffff800020bd1818,1,2,0) at uvn_io+0x2c3 sys/uvm/uvm_vnode.c:1188 uvn_get(fffffd806e014560,0,ffff800020bd1a70,ffff800020bd1a0c,0,5) at uvn_get+0x226 sys/uvm/uvm_vnode.c:1048 uvm_fault(fffffd806e8df738,20000000,2,5) at uvm_fault+0x11a5 sys/uvm/uvm_fault.c:1023 uvm_fault_wire(fffffd806e8df738,20000000,20013000,5) at uvm_fault_wire+0x70 sys/uvm/uvm_fault.c:1295 uvm_map_pageable_wire(fffffd806e8df738,fffffd806dc9d0c0,fffffd806dc9d7f8,0,1,1) at uvm_map_pageable_wire+0x2dd sys/uvm/uvm_map.c:2340 uvm_mmaplock(fffffd806e8df738,ffff800020bd1d60,13000,5,28445aaa) at uvm_mmaplock+0x181 sys/uvm/uvm_mmap.c:859 sys_mmap(ffff800020b14008,ffff800020bd1e08,ffff800020bd1e70) at sys_mmap+0xc77 sys/uvm/uvm_mmap.c:394 syscall(ffff800020bd1ee0) at syscall+0x552 mi_syscall sys/sys/syscall_mi.h:90 [inline] syscall(ffff800020bd1ee0) at syscall+0x552 sys/arch/amd64/amd64/trap.c:554 Xsyscall(6,0,a3d03511110,0,a3d035110f0,a3d035110e8) at Xsyscall+0x128 end of kernel end trace frame: 0xa3f80e32150, count: -15 ddb{1}> show registers rdi 0x3 rsi 0xffffffff821cbb60 __sancov_gen_cov_switch_values.122 rbp 0xffff800020bd14a0 rbx 0x3 rdx 0x8b rcx 0x3 rax 0x3 r8 0xffffffff81242635 witness_checkorder+0xfd5 r9 0x5 r10 0xe623f7868459c1cf r11 0xd7be421f2d9d5063 r12 0xfffffd8002667f80 r13 0 r14 0xffffffff822ebb70 w_lodata+0x4ece0 r15 0xffffffff822f0ea0 w_lodata+0x54010 rip 0xffffffff8172c2d8 db_enter+0x18 cs 0x8 rflags 0x246 rsp 0xffff800020bd1490 ss 0x10 db_enter+0x18: addq $0x8,%rsp ddb{1}> show proc PROC (syz-executor5314) pid=220168 stat=onproc flags process=0 proc=4000000 pri=50, usrpri=50, nice=20 forw=0xffffffffffffffff, list=0xffff800020b155f8,0xffffffff82347f00 process=0xffff800020b3a9e8 user=0xffff800020bcc000, vmspace=0xfffffd806e8df738 estcpu=0, cpticks=0, pctcpu=0.0 user=0, sys=0, intr=0 ddb{1}> ps PID TID PPID UID S FLAGS WAIT COMMAND 41809 457430 18646 0 7 0 syz-executor5314 41809 33751 18646 0 2 0x4000080 syz-executor5314 *41809 220168 18646 0 7 0x4000000 syz-executor5314 99812 217086 96160 0 4 0x80800 syz-executor5314 99812 65707 96160 0 2 0x4002800 syz-executor5314 18646 147080 58562 0 3 0x80 nanosleep syz-executor5314 96160 383932 58562 0 3 0x80 nanosleep syz-executor5314 58562 10063 44483 0 3 0x82 nanosleep syz-executor5314 44483 95603 37593 0 3 0x10008a pause ksh 37593 101869 81568 0 3 0x92 select sshd 14298 295896 1 0 3 0x100083 ttyin getty 81568 170431 1 0 3 0x80 select sshd 94689 501011 57204 73 2 0x100090 syslogd 57204 372836 1 0 3 0x100082 netio syslogd 11087 4358 1 77 3 0x100090 poll dhclient 77657 135330 1 0 3 0x80 poll dhclient 77344 467589 0 0 2 0x14200 zerothread 44797 245340 0 0 3 0x14200 aiodoned aiodoned 27327 209022 0 0 3 0x14200 syncer update 5055 15431 0 0 3 0x14200 cleaner cleaner 65638 413069 0 0 3 0x14200 reaper reaper 94865 27698 0 0 3 0x14200 pgdaemon pagedaemon 37136 392146 0 0 3 0x14200 bored crynlk 11303 500570 0 0 3 0x14200 bored crypto 72215 346823 0 0 3 0x40014200 acpi0 acpi0 87255 143417 0 0 3 0x40014200 idle1 39989 109155 0 0 3 0x14200 bored softnet 30923 460222 0 0 3 0x14200 bored systqmp 79027 406030 0 0 3 0x14200 bored systq 95508 519117 0 0 3 0x40014200 bored softclock 93574 227433 0 0 3 0x40014200 idle0 12479 231713 0 0 3 0x14200 bored smr 1 61243 0 0 3 0x82 wait init 0 0 -1 0 3 0x10200 scheduler swapper ddb{1}> show all locks Process 41809 (syz-executor5314) thread 0xffff800020b14008 (220168) shared rwlock vmmaplk r = 0 (0xfffffd806e8df750) #0 witness_lock+0x52e sys/kern/subr_witness.c:1161 #1 rw_enter+0x46d sys/kern/kern_rwlock.c:306 #2 vm_map_lock_ln+0x10d sys/uvm/uvm_map.c:5338 #3 uvm_mmaplock+0x64 sys/uvm/uvm_mmap.c:839 #4 sys_mmap+0xc77 sys/uvm/uvm_mmap.c:394 #5 syscall+0x552 mi_syscall sys/sys/syscall_mi.h:90 [inline] #5 syscall+0x552 sys/arch/amd64/amd64/trap.c:554 #6 Xsyscall+0x128 exclusive kernel_lock &kernel_lock r = 2 (0xffffffff8239dcf0) #0 witness_lock+0x52e sys/kern/subr_witness.c:1161 #1 syscall+0x412 mi_syscall sys/sys/syscall_mi.h:81 [inline] #1 syscall+0x412 sys/arch/amd64/amd64/trap.c:554 #2 Xsyscall+0x128 ddb{1}> show malloc Type InUse MemUse HighUse Limit Requests Type Lim Kern Lim devbuf 9442 6316K 6317K 78643K 10529 0 0 pcb 23 9K 9K 78643K 55 0 0 rtable 61 2K 2K 78643K 115 0 0 ifaddr 21 7K 7K 78643K 21 0 0 counters 39 33K 33K 78643K 39 0 0 ioctlops 0 0K 2K 78643K 13 0 0 mount 1 1K 1K 78643K 1 0 0 vnodes 1174 74K 74K 78643K 1179 0 0 UFS quota 1 32K 32K 78643K 1 0 0 UFS mount 5 36K 36K 78643K 5 0 0 shm 2 1K 1K 78643K 2 0 0 VM map 2 1K 1K 78643K 2 0 0 sem 2 0K 0K 78643K 2 0 0 dirhash 12 2K 2K 78643K 12 0 0 ACPI 1808 196K 290K 78643K 12628 0 0 file desc 1 0K 0K 78643K 1 0 0 proc 40 38K 46K 78643K 207 0 0 NFS srvsock 1 0K 0K 78643K 1 0 0 NFS daemon 1 16K 16K 78643K 1 0 0 in_multi 11 0K 0K 78643K 11 0 0 ether_multi 1 0K 0K 78643K 1 0 0 ISOFS mount 1 32K 32K 78643K 1 0 0 MSDOSFS mount 1 16K 16K 78643K 1 0 0 ttys 18 79K 79K 78643K 18 0 0 exec 0 0K 1K 78643K 152 0 0 pagedep 1 8K 8K 78643K 1 0 0 inodedep 1 32K 32K 78643K 1 0 0 newblk 1 0K 0K 78643K 1 0 0 VM swap 7 26K 26K 78643K 7 0 0 UVM amap 62 3K 11K 78643K 764 0 0 UVM aobj 2 2K 2K 78643K 2 0 0 memdesc 1 4K 4K 78643K 1 0 0 crypto data 1 1K 1K 78643K 1 0 0 NDP 3 0K 0K 78643K 3 0 0 temp 31 2719K 2779K 78643K 1729 0 0 SYN cache 2 16K 16K 78643K 2 0 0 ddb{1}> show all pools Name Size Requests Fail Releases Pgreq Pgrel Npage Hiwat Minpg Maxpg Idle arp 64 2 0 0 1 0 1 1 0 8 0 inpcbpl 280 22 0 16 1 0 1 1 0 8 0 plcache 128 20 0 0 1 0 1 1 0 8 0 rtentry 112 23 0 1 1 0 1 1 0 8 0 syncache 264 5 0 5 1 0 1 1 0 8 1 tcpcb 544 8 0 5 1 0 1 1 0 8 0 art_heap8 4096 1 0 0 1 0 1 1 0 8 0 art_heap4 256 96 0 0 6 0 6 6 0 8 0 art_table 32 97 0 0 1 0 1 1 0 8 0 art_node 16 22 0 2 1 0 1 1 0 8 0 dirhash 1024 17 0 0 3 0 3 3 0 8 0 dino1pl 128 1387 0 16 45 0 45 45 0 8 0 ffsino 272 1387 0 16 92 0 92 92 0 8 0 nchpl 144 1560 0 30 57 0 57 57 0 8 0 uvmvnodes 72 1396 0 0 26 0 26 26 0 8 0 vnodes 200 1396 0 0 74 0 74 74 0 8 0 namei 1024 3299 0 3299 2 1 1 1 0 8 1 percpumem 16 30 0 0 1 0 1 1 0 8 0 scxspl 192 11386 0 11381 8 2 6 6 0 8 5 plimitpl 152 13 0 8 1 0 1 1 0 8 0 sigapl 432 188 0 174 2 0 2 2 0 8 0 futexpl 56 23 0 23 1 0 1 1 0 8 1 knotepl 112 5 0 0 1 0 1 1 0 8 0 kqueuepl 104 1 0 0 1 0 1 1 0 8 0 pipepl 112 118 0 111 2 1 1 1 0 8 0 fdescpl 488 189 0 174 2 0 2 2 0 8 0 filepl 152 826 0 782 2 0 2 2 0 8 0 lockfpl 104 6 0 6 1 1 0 1 0 8 0 lockfspl 48 3 0 3 1 1 0 1 0 8 0 sessionpl 112 17 0 9 1 0 1 1 0 8 0 pgrppl 48 17 0 9 1 0 1 1 0 8 0 ucredpl 96 56 0 48 1 0 1 1 0 8 0 zombiepl 144 174 0 174 2 1 1 1 0 8 1 processpl 840 204 0 174 4 0 4 4 0 8 0 procpl 624 216 0 183 3 0 3 3 0 8 0 sockpl 384 64 0 48 2 0 2 2 0 8 0 mcl4k 4096 4 0 0 1 0 1 1 0 8 0 mcl2k 2048 80 0 0 10 0 10 10 0 8 0 mtagpl 80 1 0 0 1 0 1 1 0 8 0 mbufpl 256 104 0 0 7 0 7 7 0 8 0 bufpl 256 7544 0 1230 395 0 395 395 0 8 0 anonpl 16 53431 0 49453 38 1 37 37 0 125 21 amapchunkpl 152 649 0 604 3 0 3 3 0 158 1 amappl16 192 2165 0 1987 25 4 21 25 0 8 12 amappl14 176 14 0 13 2 1 1 1 0 8 0 amappl13 168 10 0 9 1 0 1 1 0 8 0 amappl12 160 9 0 9 1 1 0 1 0 8 0 amappl11 152 43 0 29 1 0 1 1 0 8 0 amappl10 144 46 0 46 2 1 1 1 0 8 1 amappl9 136 394 0 393 1 0 1 1 0 8 0 amappl8 128 99 0 89 1 0 1 1 0 8 0 amappl7 120 14 0 13 1 0 1 1 0 8 0 amappl6 112 40 0 36 1 0 1 1 0 8 0 amappl5 104 112 0 102 1 0 1 1 0 8 0 amappl4 96 312 0 295 1 0 1 1 0 8 0 amappl3 88 155 0 144 1 0 1 1 0 8 0 amappl2 80 801 0 735 3 1 2 2 0 8 0 amappl1 72 12023 0 11604 14 4 10 14 0 8 0 amappl 80 402 0 376 1 0 1 1 0 84 0 dma4096 4096 1 0 1 1 1 0 1 0 8 0 dma256 256 6 0 6 1 1 0 1 0 8 0 dma64 64 259 0 259 1 1 0 1 0 8 0 dma32 32 7 0 7 1 1 0 1 0 8 0 dma16 16 17 0 17 1 1 0 1 0 8 0 aobjpl 64 1 0 0 1 0 1 1 0 8 0 uaddrrnd 24 189 0 174 1 0 1 1 0 8 0 uaddrbest 32 2 0 0 1 0 1 1 0 8 0 uaddr 24 189 0 174 1 0 1 1 0 8 0 vmmpekpl 168 5314 0 5295 1 0 1 1 0 8 0 vmmpepl 168 27867 0 26812 74 15 59 59 0 357 13 vmsppl 368 188 0 174 2 0 2 2 0 8 0 pdppl 4096 386 0 348 5 0 5 5 0 8 0 pvpl 32 108095 0 102367 91 2 89 89 0 265 42 pmappl 232 188 0 174 1 0 1 1 0 8 0 extentpl 40 41 0 26 1 0 1 1 0 8 0 phpool 112 530 0 3 16 0 16 16 0 8 0 ddb{1}>