witness: lock order reversal: 1st 0xfffffd806b5a4758 vmmaplk (&map->lock) 2nd 0xfffffd806af0dc48 inode (&ip->i_lock) lock order "&ip->i_lock"(rrwlock) -> "&map->lock"(rwlock) first seen at: #0 witness_checkorder+0x65e witness_lock_order_add sys/kern/subr_witness.c:2463 [inline] #0 witness_checkorder+0x65e sys/kern/subr_witness.c:880 #1 rw_enter_read+0x66 sys/kern/kern_rwlock.c:111 #2 uvmfault_lookup+0xd9 sys/uvm/uvm_fault.c:1495 #3 uvm_fault_check+0x3d sys/uvm/uvm_fault.c:513 #4 uvm_fault+0xdb sys/uvm/uvm_fault.c:694 #5 kpageflttrap+0x202 sys/arch/amd64/amd64/trap.c:265 #6 kerntrap+0xef sys/arch/amd64/amd64/trap.c:321 #7 alltraps_kern_meltdown+0x7b #8 copyout+0x53 #9 ffs_read+0x3e2 sys/ufs/ffs/ffs_vnops.c:258 #10 VOP_READ+0xbf sys/kern/vfs_vops.c:253 #11 vn_rdwr+0x10b #12 vmcmd_map_readvn+0xda sys/kern/exec_subr.c:246 #13 exec_process_vmcmds+0xb2 sys/kern/exec_subr.c:143 #14 sys_execve+0x9ff sys/kern/kern_exec.c:461 #15 syscall+0x4a1 mi_syscall sys/sys/syscall_mi.h:102 [inline] #15 syscall+0x4a1 sys/arch/amd64/amd64/trap.c:590 #16 Xsyscall+0x128 lock order "&map->lock"(rwlock) -> "&ip->i_lock"(rrwlock) first seen at: #0 witness_checkorder+0x65e witness_lock_order_add sys/kern/subr_witness.c:2463 [inline] #0 witness_checkorder+0x65e sys/kern/subr_witness.c:880 #1 rw_enter+0xd4 #2 rrw_enter+0x88 sys/kern/kern_rwlock.c:462 #3 VOP_LOCK+0x4b sys/kern/vfs_vops.c:614 #4 vn_lock+0x6c sys/kern/vfs_vnops.c:575 #5 uvn_get+0x276 uvm_vnode_lock sys/uvm/uvm_vnode.c:1499 [inline] #5 uvn_get+0x276 sys/uvm/uvm_vnode.c:993 #6 uvm_fault+0xa41 sys/uvm/uvm_fault.c:1073 #7 uvm_fault_wire+0x63 sys/uvm/uvm_fault.c:1342 #8 uvm_map_pageable_wire+0x30d sys/uvm/uvm_map.c:2367 #9 sys_mlock+0x180 sys/uvm/uvm_mmap.c:772 #10 syscall+0x4a1 mi_syscall sys/sys/syscall_mi.h:102 [inline] #10 syscall+0x4a1 sys/arch/amd64/amd64/trap.c:590 #11 Xsyscall+0x128 Stopped at db_enter+0x18: addq $0x8,%rsp ddb{0}> ddb{0}> set $lines = 0 ddb{0}> set $maxwidth = 0 ddb{0}> show panic the kernel did not panic ddb{0}> trace db_enter() at db_enter+0x18 sys/arch/amd64/amd64/db_interface.c:398 witness_checkorder(fffffd806af0dc48,9,0) at witness_checkorder+0xf5a witness_debugger sys/kern/subr_witness.c:2490 [inline] witness_checkorder(fffffd806af0dc48,9,0) at witness_checkorder+0xf5a sys/kern/subr_witness.c:1087 rw_enter(fffffd806af0dc38,81) at rw_enter+0xd4 rrw_enter(fffffd806af0dc38,81) at rrw_enter+0x88 sys/kern/kern_rwlock.c:462 VOP_LOCK(fffffd8065df6eb0,81) at VOP_LOCK+0x4b sys/kern/vfs_vops.c:614 vn_lock(fffffd8065df6eb0,81) at vn_lock+0x6c sys/kern/vfs_vnops.c:575 uvn_get(fffffd8068bc8050,0,ffff8000222e12f8,ffff8000222e1294,0,1) at uvn_get+0x276 uvm_vnode_lock sys/uvm/uvm_vnode.c:1499 [inline] uvn_get(fffffd8068bc8050,0,ffff8000222e12f8,ffff8000222e1294,0,1) at uvn_get+0x276 sys/uvm/uvm_vnode.c:993 uvm_fault(fffffd806b5a4740,20002000,2,1) at uvm_fault+0xa41 sys/uvm/uvm_fault.c:1073 uvm_fault_wire(fffffd806b5a4740,20002000,20005000,1) at uvm_fault_wire+0x63 sys/uvm/uvm_fault.c:1342 uvm_map_pageable_wire(fffffd806b5a4740,fffffd806c67e198,fffffd806ad24b40,11b,0,0) at uvm_map_pageable_wire+0x30d sys/uvm/uvm_map.c:2367 sys_mlock(ffff8000212819b8,ffff8000222e14d8,ffff8000222e1520) at sys_mlock+0x180 sys/uvm/uvm_mmap.c:772 syscall(ffff8000222e15a0) at syscall+0x4a1 mi_syscall sys/sys/syscall_mi.h:102 [inline] syscall(ffff8000222e15a0) at syscall+0x4a1 sys/arch/amd64/amd64/trap.c:590 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0xa825a0594a0, count: -13 ddb{0}> show registers rdi 0x3 rsi 0x40000 acpi_pdirpa+0x2be68 rbp 0xffff8000222e0dd0 rbx 0x3 rdx 0xffff800000b08340 rcx 0x3 rax 0x3ffff acpi_pdirpa+0x2be67 r8 0xffffffff823582d3 witness_checkorder+0xf33 r9 0x5 r10 0xbd0ab69ee5fbb2e6 r11 0x92fe87af798181b3 r12 0xffffffff828c54e0 w_lodata+0x542d0 r13 0 r14 0xffffffff828bc7f0 w_lodata+0x4b5e0 r15 0xfffffd8002ce5500 rip 0xffffffff81f33568 db_enter+0x18 cs 0x8 rflags 0x246 rsp 0xffff8000222e0dc0 ss 0x10 db_enter+0x18: addq $0x8,%rsp ddb{0}> show proc PROC (syz-executor.0) pid=443895 stat=onproc flags process=0 proc=4000000 pri=32, usrpri=75, nice=20 forw=0xffffffffffffffff, list=0xffff800021280ce8,0xffffffff828de248 process=0xffff800021234c70 user=0xffff8000222dc000, vmspace=0xfffffd806b5a4740 estcpu=25, cpticks=0, pctcpu=2.18 user=0, sys=0, intr=0 ddb{0}> ps PID TID PPID UID S FLAGS WAIT COMMAND 63611 66396 63567 0 2 0 syz-executor.0 *63611 443895 63567 0 7 0x4000000 syz-executor.0 95110 436095 0 0 3 0x14200 bored sosplice 65827 302292 15189 0 3 0x82 nanosleep syz-executor.1 63567 222544 15189 0 3 0x82 nanosleep syz-executor.0 15189 232053 30963 0 3 0x82 nanosleep syz-fuzzer 15189 36950 30963 0 3 0x4000082 nanosleep syz-fuzzer 15189 157026 30963 0 3 0x4000082 thrsleep syz-fuzzer 15189 409926 30963 0 3 0x4000082 thrsleep syz-fuzzer 15189 436365 30963 0 7 0x4000002 syz-fuzzer 15189 458730 30963 0 3 0x4000082 thrsleep syz-fuzzer 15189 271004 30963 0 3 0x4000082 thrsleep syz-fuzzer 15189 218907 30963 0 3 0x4000082 thrsleep syz-fuzzer 30963 172804 85192 0 3 0x10008a pause ksh 85192 102385 7032 0 3 0x92 select sshd 83840 382564 1 0 3 0x100083 ttyin getty 7032 295673 1 0 3 0x80 select sshd 33417 376141 46174 74 3 0x100092 bpf pflogd 46174 472792 1 0 3 0x80 netio pflogd 97681 214152 92472 73 3 0x100090 kqread syslogd 92472 173119 1 0 3 0x100082 netio syslogd 85591 315539 1 77 3 0x100090 poll dhclient 7490 43485 1 0 3 0x80 poll dhclient 72412 13931 0 0 3 0x14200 bored smr 56433 136012 0 0 2 0x14200 zerothread 36408 499037 0 0 3 0x14200 aiodoned aiodoned 77286 160959 0 0 3 0x14200 syncer update 6565 456672 0 0 3 0x14200 cleaner cleaner 51569 41287 0 0 3 0x14200 reaper reaper 20533 521357 0 0 3 0x14200 pgdaemon pagedaemon 84165 313476 0 0 3 0x14200 bored crynlk 38231 99989 0 0 3 0x14200 bored crypto 92094 404636 0 0 3 0x14200 bored viomb 44908 311521 0 0 3 0x40014200 acpi0 acpi0 79382 11301 0 0 3 0x40014200 idle1 37710 73453 0 0 3 0x14200 bored softnet 76741 213843 0 0 3 0x14200 bored systqmp 8491 481626 0 0 3 0x14200 bored systq 16700 344456 0 0 3 0x40014200 bored softclock 5421 457454 0 0 3 0x40014200 idle0 1 172530 0 0 3 0x82 wait init 0 0 -1 0 3 0x10200 scheduler swapper ddb{0}> show all locks Process 63611 (syz-executor.0) thread 0xffff8000212819b8 (443895) shared rwlock vmmaplk r = 0 (0xfffffd806b5a4758) #0 witness_lock+0x4b0 stacktrace_save sys/sys/stacktrace.h:36 [inline] #0 witness_lock+0x4b0 sys/kern/subr_witness.c:1164 #1 rw_enter+0x446 sys/kern/kern_rwlock.c:311 #2 vm_map_lock_ln+0xda sys/uvm/uvm_map.c:5463 #3 uvm_map_pageable+0x120 sys/uvm/uvm_map.c:2463 #4 sys_mlock+0x180 sys/uvm/uvm_mmap.c:772 #5 syscall+0x4a1 mi_syscall sys/sys/syscall_mi.h:102 [inline] #5 syscall+0x4a1 sys/arch/amd64/amd64/trap.c:590 #6 Xsyscall+0x128 exclusive kernel_lock &kernel_lock r = 0 (0xffffffff828db810) #0 witness_lock+0x4b0 stacktrace_save sys/sys/stacktrace.h:36 [inline] #0 witness_lock+0x4b0 sys/kern/subr_witness.c:1164 #1 syscall+0x3fd mi_syscall sys/sys/syscall_mi.h:93 [inline] #1 syscall+0x3fd sys/arch/amd64/amd64/trap.c:590 #2 Xsyscall+0x128 ddb{0}> show malloc Type InUse MemUse HighUse Limit Requests Type Lim devbuf 9488 6424K 7780K 78643K 11648 0 pcb 13 8K 8K 78643K 64 0 rtable 112 4K 5K 78643K 223 0 ifaddr 46 11K 11K 78643K 47 0 sysctl 0 0K 0K 78643K 11 0 counters 43 33K 33K 78643K 43 0 ioctlops 0 0K 4K 78643K 1514 0 iov 0 0K 2K 78643K 14 0 mount 1 1K 1K 78643K 1 0 vnodes 1224 77K 77K 78643K 1481 0 UFS quota 1 32K 32K 78643K 1 0 UFS mount 5 36K 36K 78643K 5 0 shm 2 1K 5K 78643K 16 0 VM map 2 1K 1K 78643K 2 0 sem 12 0K 1K 78643K 25 0 dirhash 12 2K 2K 78643K 12 0 ACPI 1825 197K 290K 78643K 13109 0 file desc 5 13K 25K 78643K 718 0 sigio 0 0K 0K 78643K 13 0 proc 60 63K 95K 78643K 576 0 subproc 32 2K 2K 78643K 34 0 NFS srvsock 1 0K 0K 78643K 1 0 NFS daemon 1 16K 16K 78643K 1 0 ip_moptions 0 0K 0K 78643K 19 0 in_multi 33 2K 2K 78643K 41 0 ether_multi 1 0K 0K 78643K 15 0 mrt 0 0K 0K 78643K 20 0 ISOFS mount 1 32K 32K 78643K 1 0 MSDOSFS mount 1 16K 16K 78643K 1 0 ttys 55 254K 254K 78643K 55 0 exec 0 0K 2K 78643K 401 0 pagedep 1 8K 8K 78643K 1 0 inodedep 1 32K 32K 78643K 1 0 newblk 1 0K 0K 78643K 1 0 VM swap 7 26K 26K 78643K 7 0 UVM amap 150 235K 235K 78643K 2325 0 UVM aobj 14 2K 2K 78643K 14 0 memdesc 1 4K 4K 78643K 1 0 crypto data 1 1K 1K 78643K 1 0 ip6_options 0 0K 0K 78643K 29 0 NDP 6 0K 0K 78643K 10 0 temp 72 3959K 4023K 78643K 7415 0 kqueue 5 5K 8K 78643K 38 0 SYN cache 2 16K 16K 78643K 2 0 ddb{0}> show all pools Name Size Requests Fail Releases Pgreq Pgrel Npage Hiwat Minpg Maxpg Idle arp 64 6 0 0 1 0 1 1 0 8 0 plcache 128 20 0 0 1 0 1 1 0 8 0 rtpcb 120 24 0 22 1 0 1 1 0 8 0 rtentry 112 46 0 1 2 0 2 2 0 8 0 unpcb 120 183 0 173 1 0 1 1 0 8 0 syncache 296 4 0 4 1 1 0 1 0 8 0 tcpqe 32 82 0 82 1 1 0 1 0 8 0 tcpcb 736 70 0 65 3 0 3 3 0 8 2 inpcb 296 338 0 330 1 0 1 1 0 8 0 nd6 48 7 0 0 1 0 1 1 0 8 0 kcovpl 48 2 0 0 1 0 1 1 0 8 0 pfosfp 40 1428 0 1005 5 0 5 5 0 8 0 pfosfpen 112 1428 0 714 21 0 21 21 0 8 0 pfrktable 1344 3 0 1 1 0 1 1 0 8 0 pfstitem 24 12 0 3 1 0 1 1 0 8 0 pfstkey 112 12 0 3 1 0 1 1 0 8 0 pfstate 328 12 0 3 1 0 1 1 0 8 0 pfrule 1360 31 0 21 2 1 1 2 0 8 0 art_heap8 4096 1 0 0 1 0 1 1 0 8 0 art_heap4 256 190 0 0 12 0 12 12 0 8 0 art_table 32 191 0 0 2 0 2 2 0 8 0 art_node 16 45 0 4 1 0 1 1 0 8 0 semupl 112 1 0 1 1 1 0 1 0 8 0 semapl 112 17 0 7 1 0 1 1 0 8 0 shmpl 112 11 0 0 1 0 1 1 0 8 0 dirhash 1024 17 0 0 3 0 3 3 0 8 0 dino2pl 256 2245 0 840 88 0 88 88 0 8 0 ffsino 272 2245 0 840 94 0 94 94 0 8 0 nchpl 144 3348 0 1743 60 0 60 60 0 8 0 uvmvnodes 72 2509 0 0 46 0 46 46 0 8 0 vnodes 208 2509 0 0 133 0 133 133 0 8 0 namei 1024 8329 0 8329 1 0 1 1 0 8 1 percpumem 16 32 0 0 1 0 1 1 0 8 0 vcpupl 1984 13 0 0 2 0 2 2 0 8 0 vmpool 560 13 0 0 1 0 1 1 0 8 0 pfiaddrpl 120 5 0 0 1 0 1 1 0 8 0 scxspl 216 9504 0 9504 9 5 4 8 0 8 4 plimitpl 152 31 0 23 1 0 1 1 0 8 0 sigapl 424 935 0 902 4 0 4 4 0 8 0 futexpl 56 5508 0 5508 1 0 1 1 0 8 1 knotepl 112 93 0 72 1 0 1 1 0 8 0 kqueuepl 152 257 0 254 1 0 1 1 0 8 0 pipepl 304 105 0 93 3 1 2 2 0 8 1 fdescpl 496 918 0 902 3 0 3 3 0 8 0 filepl 152 3374 0 3267 5 0 5 5 0 8 0 lockfpl 104 34 0 33 1 0 1 1 0 8 0 lockfspl 48 10 0 9 1 0 1 1 0 8 0 sessionpl 144 18 0 7 1 0 1 1 0 8 0 pgrppl 48 18 0 7 1 0 1 1 0 8 0 ucredpl 96 289 0 280 1 0 1 1 0 8 0 zombiepl 144 902 0 901 1 0 1 1 0 8 0 processpl 1056 935 0 901 3 0 3 3 0 8 0 procpl 656 1732 0 1690 4 0 4 4 0 8 0 sockpl 400 545 0 525 3 0 3 3 0 8 0 mcl64k 65536 1 0 0 1 0 1 1 0 8 0 mcl12k 12288 12 0 0 2 0 2 2 0 8 0 mcl8k 8192 2 0 0 1 0 1 1 0 8 0 mcl4k 4096 3 0 0 1 0 1 1 0 8 0 mcl2k 2048 160 0 0 19 0 19 19 0 8 0 mtagpl 96 83 0 0 3 0 3 3 0 8 0 mbufpl 256 473 0 0 29 0 29 29 0 8 0 bufpl 280 4242 0 181 291 0 291 291 0 8 0 anonpl 16 73318 0 64623 37 1 36 36 0 124 0 amapchunkpl 152 3428 0 3254 9 2 7 8 0 158 0 amappl16 192 2490 0 2104 20 0 20 20 0 8 0 amappl15 184 1 0 0 1 0 1 1 0 8 0 amappl14 176 24 0 18 1 0 1 1 0 8 0 amappl13 168 40 0 39 1 0 1 1 0 8 0 amappl12 160 9 0 5 2 1 1 1 0 8 0 amappl11 152 51 0 36 1 0 1 1 0 8 0 amappl10 144 708 0 702 1 0 1 1 0 8 0 amappl9 136 184 0 184 1 1 0 1 0 8 0 amappl8 128 143 0 105 2 0 2 2 0 8 0 amappl7 120 244 0 236 1 0 1 1 0 8 0 amappl6 112 754 0 739 1 0 1 1 0 8 0 amappl5 104 1423 0 1405 1 0 1 1 0 8 0 amappl4 96 274 0 250 1 0 1 1 0 8 0 amappl3 88 147 0 139 1 0 1 1 0 8 0 amappl2 80 5771 0 5704 3 0 3 3 0 8 0 amappl1 72 34634 0 34151 25 14 11 19 0 8 0 amappl 80 2002 0 1935 2 0 2 2 0 84 0 dma4096 4096 1 0 1 1 1 0 1 0 8 0 dma1024 1024 1 0 0 1 0 1 1 0 8 0 dma256 256 6 0 6 1 1 0 1 0 8 0 dma128 128 253 0 253 1 1 0 1 0 8 0 dma64 64 6 0 6 1 1 0 1 0 8 0 dma32 32 7 0 7 1 1 0 1 0 8 0 dma16 16 18 0 17 1 0 1 1 0 8 0 aobjpl 64 13 0 0 1 0 1 1 0 8 0 uaddrrnd 24 931 0 902 1 0 1 1 0 8 0 uaddrbest 32 2 0 0 1 0 1 1 0 8 0 uaddr 24 931 0 902 1 0 1 1 0 8 0 vmmpekpl 168 8335 0 8303 2 0 2 2 0 8 0 vmmpepl 168 118352 0 116748 93 18 75 75 0 357 1 vmsppl 368 930 0 902 3 0 3 3 0 8 0 pdppl 4096 1869 0 1817 70 16 54 54 0 8 2 pvpl 32 303728 0 291414 114 12 102 114 0 265 2 pmappl 232 930 0 902 3 1 2 2 0 8 0 extentpl 40 57 0 39 1 0 1 1 0 8 0 phpool 112 334 0 19 9 0 9 9 0 8 0 ddb{0}> machine ddbcpu 0 Invalid cpu 0 ddb{0}> trace db_enter() at db_enter+0x18 sys/arch/amd64/amd64/db_interface.c:398 witness_checkorder(fffffd806af0dc48,9,0) at witness_checkorder+0xf5a witness_debugger sys/kern/subr_witness.c:2490 [inline] witness_checkorder(fffffd806af0dc48,9,0) at witness_checkorder+0xf5a sys/kern/subr_witness.c:1087 rw_enter(fffffd806af0dc38,81) at rw_enter+0xd4 rrw_enter(fffffd806af0dc38,81) at rrw_enter+0x88 sys/kern/kern_rwlock.c:462 VOP_LOCK(fffffd8065df6eb0,81) at VOP_LOCK+0x4b sys/kern/vfs_vops.c:614 vn_lock(fffffd8065df6eb0,81) at vn_lock+0x6c sys/kern/vfs_vnops.c:575 uvn_get(fffffd8068bc8050,0,ffff8000222e12f8,ffff8000222e1294,0,1) at uvn_get+0x276 uvm_vnode_lock sys/uvm/uvm_vnode.c:1499 [inline] uvn_get(fffffd8068bc8050,0,ffff8000222e12f8,ffff8000222e1294,0,1) at uvn_get+0x276 sys/uvm/uvm_vnode.c:993 uvm_fault(fffffd806b5a4740,20002000,2,1) at uvm_fault+0xa41 sys/uvm/uvm_fault.c:1073 uvm_fault_wire(fffffd806b5a4740,20002000,20005000,1) at uvm_fault_wire+0x63 sys/uvm/uvm_fault.c:1342 uvm_map_pageable_wire(fffffd806b5a4740,fffffd806c67e198,fffffd806ad24b40,11b,0,0) at uvm_map_pageable_wire+0x30d sys/uvm/uvm_map.c:2367 sys_mlock(ffff8000212819b8,ffff8000222e14d8,ffff8000222e1520) at sys_mlock+0x180 sys/uvm/uvm_mmap.c:772 syscall(ffff8000222e15a0) at syscall+0x4a1 mi_syscall sys/sys/syscall_mi.h:102 [inline] syscall(ffff8000222e15a0) at syscall+0x4a1 sys/arch/amd64/amd64/trap.c:590 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0xa825a0594a0, count: -13 ddb{0}> machine ddbcpu 1 Stopped at x86_ipi_db+0x1a: addq $0x8,%rsp ddb{1}> trace x86_ipi_db(ffff800020d68ff0) at x86_ipi_db+0x1a sys/arch/amd64/amd64/db_interface.c:352 x86_ipi_handler() at x86_ipi_handler+0xb7 sys/arch/amd64/amd64/ipi.c:106 Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23 __sanitizer_cov_trace_pc() at __sanitizer_cov_trace_pc+0x37 kd_curproc sys/dev/kcov.c:570 [inline] __sanitizer_cov_trace_pc() at __sanitizer_cov_trace_pc+0x37 sys/dev/kcov.c:143 __mp_lock(ffffffff828db608) at __mp_lock+0x133 __mp_lock_spin sys/kern/kern_lock.c:116 [inline] __mp_lock(ffffffff828db608) at __mp_lock+0x133 sys/kern/kern_lock.c:147 syscall(ffff800021271ab0) at syscall+0x3fd mi_syscall sys/sys/syscall_mi.h:93 [inline] syscall(ffff800021271ab0) at syscall+0x3fd sys/arch/amd64/amd64/trap.c:590 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0xc00004bb48, count: -7