witness: lock order reversal: 1st 0xfffffd806bf88f18 vmmaplk (&map->lock) 2nd 0xfffffd8066567810 inode (&ip->i_lock) lock order [1] vmmaplk (&map->lock) -> [2] inode (&ip->i_lock) #0 rw_enter+0x122 #1 rrw_enter+0xbe sys/kern/kern_rwlock.c:464 #2 VOP_LOCK+0xa6 sys/kern/vfs_vops.c:524 #3 vn_lock+0xa4 sys/kern/vfs_vnops.c:564 #4 vn_rdwr+0xd1 sys/kern/vfs_vnops.c:320 #5 vndstrategy+0x4ff sys/dev/vnd.c:342 #6 physio+0x2f6 sys/kern/kern_physio.c:162 #7 spec_read+0x155 sys/kern/spec_vnops.c:215 #8 VOP_READ+0x102 sys/kern/vfs_vops.c:227 #9 vn_read+0x17b sys/kern/vfs_vnops.c:369 #10 dofilereadv+0x230 sys/kern/sys_generic.c:252 #11 sys_read+0xa2 sys/kern/sys_generic.c:172 #12 syscall+0xbb6 mi_syscall sys/sys/syscall_mi.h:178 [inline] #12 syscall+0xbb6 sys/arch/amd64/amd64/trap.c:577 #13 Xsyscall+0x128 lock order [2] inode (&ip->i_lock) -> [1] vmmaplk (&map->lock) #0 rw_enter_read+0xab sys/kern/kern_rwlock.c:112 #1 uvmfault_lookup+0x122 sys/uvm/uvm_fault.c:1819 #2 uvm_fault_check+0x49 sys/uvm/uvm_fault.c:672 #3 uvm_fault+0xf5 sys/uvm/uvm_fault.c:600 #4 kpageflttrap+0x2d0 sys/arch/amd64/amd64/trap.c:279 #5 kerntrap+0x14a sys/arch/amd64/amd64/trap.c:332 #6 alltraps_kern_meltdown+0x7b #7 copyout+0x57 #8 ffs_read+0x422 sys/ufs/ffs/ffs_vnops.c:254 #9 VOP_READ+0x102 sys/kern/vfs_vops.c:227 #10 vn_rdwr+0x15b #11 vmcmd_map_readvn+0x142 sys/kern/exec_subr.c:249 #12 exec_process_vmcmds+0xfb sys/kern/exec_subr.c:139 #13 sys_execve+0xbe4 sys/kern/kern_exec.c:468 #14 start_init+0x3c6 sys/kern/init_main.c:714 #15 proc_trampoline+0x10 Stopped at db_enter+0x25: addq $0x8,%rsp ddb{0}> ddb{0}> set $lines = 0 ddb{0}> set $maxwidth = 0 ddb{0}> show panic the kernel did not panic ddb{0}> trace db_enter() at db_enter+0x25 sys/arch/amd64/amd64/db_interface.c:437 witness_checkorder(fffffd8066567810,9,0) at witness_checkorder+0x1047 rw_enter(fffffd8066567800,1) at rw_enter+0x122 rrw_enter(fffffd8066567800,1) at rrw_enter+0xbe sys/kern/kern_rwlock.c:464 VOP_LOCK(fffffd8067dcf980,2001) at VOP_LOCK+0xa6 sys/kern/vfs_vops.c:524 vn_lock(fffffd8067dcf980,2001) at vn_lock+0xa4 sys/kern/vfs_vnops.c:564 vn_rdwr(0,fffffd8067dcf980,ffff800011a26140,df,0,1,c9d3c0cb3c87d63f,0,ffff800000b3e000,0) at vn_rdwr+0xd1 sys/kern/vfs_vnops.c:320 vndstrategy(fffffd805ff456d0) at vndstrategy+0x4ff sys/dev/vnd.c:342 physio(ffffffff82b2d360,2902,8000,ffffffff82abc400,ffff800037215898) at physio+0x2f6 sys/kern/kern_physio.c:162 spec_read(ffff800037215700) at spec_read+0x155 sys/kern/spec_vnops.c:215 VOP_READ(fffffd806dedbd08,ffff800037215898,0,fffffd807f7d34e0) at VOP_READ+0x102 sys/kern/vfs_vops.c:227 vn_read(fffffd8067bdd300,ffff800037215898,0) at vn_read+0x17b sys/kern/vfs_vnops.c:369 dofilereadv(ffff800035f9cf58,4,ffff800037215898,0,ffff800037215950) at dofilereadv+0x230 sys/kern/sys_generic.c:252 sys_read(ffff800035f9cf58,ffff800037215a00,ffff800037215950) at sys_read+0xa2 sys/kern/sys_generic.c:172 syscall(ffff800037215a00) at syscall+0xbb6 mi_syscall sys/sys/syscall_mi.h:178 [inline] syscall(ffff800037215a00) at syscall+0xbb6 sys/arch/amd64/amd64/trap.c:577 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0x857cac862f0, count: -16 ddb{0}> show registers rdi 0 rsi 0 rbp 0xffff8000372151c0 rbx 0xfffffd80042cae88 rdx 0 rcx 0xffff800035f9cf58 rax 0xffffffff834d4ff0 cpu_info_full_primary+0x1ff0 r8 0xffff8000372150a0 r9 0x8080808080808080 r10 0x9f92bac745ac3cc9 r11 0x3171b3e476408835 r12 0 r13 0xfffffd8003ae2e00 r14 0x3 r15 0xffffffff rip 0xffffffff81e356c5 db_enter+0x25 cs 0x8 rflags 0x246 rsp 0xffff8000372151b0 ss 0x10 db_enter+0x25: addq $0x8,%rsp ddb{0}> show proc PROC (syz-executor) tid=474694 pid=6814 tcnt=2 stat=onproc flags process=0 proc=4000000 runpri=32, usrpri=50, slppri=32, nice=20 wchan=0x0, wmesg=, ps_single=0x0 scnt=0 ecnt=0 forw=0xffffffffffffffff, list=0xffff800035f9d6f0,0xffffffff835851b8 process=0xffff8000371c4930 user=0xffff800037210000, vmspace=0xfffffd806bf88e20 estcpu=36, cpticks=1, pctcpu=0.0, user=0, sys=1, intr=0 ddb{0}> ps PID TID PPID UID S FLAGS WAIT COMMAND 6814 14242 34164 0 2 0 syz-executor * 6814 474694 34164 0 7 0x4000000 syz-executor 28110 298428 15548 0 2 0 syz-executor 28110 445044 15548 0 3 0x4000080 rest syz-executor 28110 451893 15548 0 3 0x4000080 bell syz-executor 81305 18827 62455 0 2 0 syz-executor 81305 91607 62455 0 3 0x4000080 fsleep syz-executor 81305 279532 62455 0 3 0x4000080 fsleep syz-executor 57466 453916 69468 0 2 0 syz-executor 57466 169589 69468 0 3 0x4000000 fltagain2 syz-executor 57466 398044 69468 0 2 0x4000000 syz-executor 55370 341057 36479 0 7 0 syz-executor 55370 16968 36479 0 3 0x4000080 sbwait syz-executor 55370 65721 36479 0 3 0x4000080 fsleep syz-executor 31579 46546 52411 0 2 0 syz-executor 31579 288674 52411 0 3 0x4000080 fsleep syz-executor 31579 454273 52411 0 3 0x4000080 fsleep syz-executor 31579 1727 52411 0 3 0x4000080 fsleep syz-executor 14148 414538 0 0 3 0x14200 bored sosplice 52411 444950 25287 0 3 0x82 nanoslp syz-executor 64505 341926 25287 0 3 0x82 wait syz-executor 36479 300067 25287 0 3 0x82 nanoslp syz-executor 5107 393254 25287 0 2 0x2 syz-executor 34164 444078 25287 0 2 0x482 syz-executor 15548 228447 25287 0 3 0x82 nanoslp syz-executor 62455 196248 25287 0 3 0x82 nanoslp syz-executor 69468 97559 25287 0 2 0x482 syz-executor 25287 353900 18452 0 3 0x82 pipeiolk syz-executor 18452 347582 51764 0 3 0x10008a sigsusp ksh 51764 254907 84135 0 3 0x98 kqread sshd-session 84135 215186 69525 0 3 0x92 kqread sshd-session 96990 364472 1 0 3 0x100083 ttyin getty 69525 190007 1 0 3 0x88 kqread sshd 97517 43608 53624 74 3 0x1100092 bpf pflogd 53624 209818 1 0 3 0x80 sbwait pflogd 98587 163097 12399 73 3 0x1100090 kqread syslogd 12399 494700 1 0 3 0x100082 sbwait syslogd 10966 96781 1 0 3 0x100080 kqread resolvd 6484 5539 54512 77 3 0x100092 kqread dhcpleased 15300 427744 54512 77 3 0x100092 kqread dhcpleased 54512 472134 1 0 3 0x80 kqread dhcpleased 29489 296904 0 0 3 0x14200 bored smr 24543 404153 0 0 2 0x14200 zerothread 92761 276676 0 0 3 0x14200 aiodoned aiodoned 28340 392767 0 0 3 0x14200 syncer update 85365 204145 0 0 3 0x14200 cleaner cleaner 75736 323870 0 0 3 0x14200 reaper reaper 35295 483075 0 0 3 0x14200 pgdaemon pagedaemon 30140 35298 0 0 3 0x14200 bored viomb 32302 238768 0 0 3 0x40014200 acpi0 acpi0 38001 301959 0 0 3 0x40014200 idle1 70073 486452 0 0 3 0x14200 bored softnet3 13122 426165 0 0 3 0x14200 bored softnet2 25827 334155 0 0 3 0x14200 bored softnet1 74608 407831 0 0 3 0x14200 bored softnet0 54714 401920 0 0 3 0x14200 bored systqmp 35949 376820 0 0 3 0x14200 bored systq 14437 142399 0 0 3 0x14200 tmoslp softclockmp 70670 456064 0 0 2 0x40014200 softclock 36405 38738 0 0 3 0x40014200 idle0 1 475036 0 0 3 0x82 wait init 0 0 -1 0 3 0x10200 scheduler swapper ddb{0}> show all locks Process 6814 (syz-executor) thread 0xffff800035f9cf58 (474694) shared rwlock vmmaplk r = 0 (0xfffffd806bf88f18) #0 witness_lock+0x5bb stacktrace_save sys/sys/stacktrace.h:37 [inline] #0 witness_lock+0x5bb sys/kern/subr_witness.c:1155 #1 uvm_vslock_device+0x149 sys/uvm/uvm_glue.c:172 #2 physio+0x277 sys/kern/kern_physio.c:139 #3 spec_read+0x155 sys/kern/spec_vnops.c:215 #4 VOP_READ+0x102 sys/kern/vfs_vops.c:227 #5 vn_read+0x17b sys/kern/vfs_vnops.c:369 #6 dofilereadv+0x230 sys/kern/sys_generic.c:252 #7 sys_read+0xa2 sys/kern/sys_generic.c:172 #8 syscall+0xbb6 mi_syscall sys/sys/syscall_mi.h:178 [inline] #8 syscall+0xbb6 sys/arch/amd64/amd64/trap.c:577 #9 Xsyscall+0x128 exclusive kernel_lock &kernel_lock r = 0 (0xffffffff8363bb40) #0 witness_lock+0x5bb stacktrace_save sys/sys/stacktrace.h:37 [inline] #0 witness_lock+0x5bb sys/kern/subr_witness.c:1155 #1 vn_read+0x56 sys/kern/vfs_vnops.c:351 #2 dofilereadv+0x230 sys/kern/sys_generic.c:252 #3 sys_read+0xa2 sys/kern/sys_generic.c:172 #4 syscall+0xbb6 mi_syscall sys/sys/syscall_mi.h:178 [inline] #4 syscall+0xbb6 sys/arch/amd64/amd64/trap.c:577 #5 Xsyscall+0x128 ddb{0}> show malloc Type InUse MemUse HighUse Limit Requests Type Lim devbuf 10192 11115K 11437K 166960K 11637 0 pcb 17 13K 14K 166960K 103 0 rtable 201 6K 7K 166960K 384 0 pf 35 17K 25K 166960K 53 0 ifaddr 40 6K 7K 166960K 50 0 ifgroup 55 2K 2K 166960K 65 0 counters 64 36K 36K 166960K 68 0 ioctlops 0 0K 4K 166960K 1497 0 iov 0 0K 12K 166960K 10 0 mount 1 1K 1K 166960K 1 0 log 0 0K 0K 166960K 4 0 vnodes 1432 90K 90K 166960K 1682 0 UFS quota 1 32K 32K 166960K 1 0 UFS mount 5 36K 36K 166960K 5 0 shm 2 1K 5K 166960K 7 0 VM map 2 1K 1K 166960K 2 0 sem 12 0K 0K 166960K 81 0 dirhash 12 2K 2K 166960K 21 0 ACPI 1690 195K 286K 166960K 12468 0 file desc 18 65K 97K 166960K 353 0 sigio 0 0K 0K 166960K 4 0 proc 72 91K 164K 166960K 570 0 subproc 104 6K 6K 166960K 105 0 NFS srvsock 1 0K 0K 166960K 1 0 NFS daemon 1 16K 16K 166960K 1 0 ip_moptions 0 0K 0K 166960K 37 0 in_multi 93 6K 7K 166960K 121 0 ether_multi 1 0K 0K 166960K 3 0 mrt 1 0K 0K 166960K 3 0 ISOFS mount 1 32K 32K 166960K 1 0 MSDOSFS mount 1 16K 16K 166960K 1 0 ttys 67 307K 307K 166960K 67 0 exec 0 0K 1K 166960K 386 0 fusefs mount 1 32K 32K 166960K 1 0 pfkey data 0 0K 0K 166960K 1 0 tdb 3 0K 0K 166960K 3 0 VM swap 8 62K 64K 166960K 10 0 UVM amap 260 73K 87K 166960K 4955 0 UVM aobj 11 2K 2K 166960K 13 0 pinsyscall 43 86K 104K 166960K 1427 0 memdesc 1 4K 4K 166960K 1 0 crypto data 1 1K 1K 166960K 1 0 ip6_options 1 0K 0K 166960K 39 0 NDP 12 0K 1K 166960K 31 0 temp 79 6828K 6892K 166960K 7038 0 kqueue 13 20K 28K 166960K 54 0 SYN cache 2 16K 16K 166960K 2 0 ddb{0}> show all pools Name Size Requests Fail Releases Pgreq Pgrel Npage Hiwat Minpg Maxpg Idle plcache 128 24 0 0 1 0 1 1 0 8 0 rtpcb 120 47 0 44 1 0 1 1 0 8 0 rtentry 112 120 0 28 4 0 4 4 0 8 0 unpcb 144 371 0 352 6 0 6 6 0 8 5 syncache 336 4 0 4 1 0 1 1 0 8 1 tcpcb 808 135 0 62 8 0 8 8 0 8 0 arp 120 21 0 6 1 0 1 1 0 8 0 inpcb 336 456 0 315 12 0 12 12 0 8 0 nd6 136 25 0 4 1 0 1 1 0 8 0 pkpcb 40 1 0 1 1 0 1 1 0 8 1 kcovpl 48 8 0 0 1 0 1 1 0 8 0 pfstscr 40 2 0 2 1 0 1 1 0 8 1 pfosfp 40 1428 0 1005 5 0 5 5 0 8 0 pfosfpen 112 1428 0 714 21 0 21 21 0 8 0 pfstitem 24 40 0 0 1 0 1 1 0 8 0 pfstkey 128 42 0 2 2 0 2 2 0 8 0 pfstate 376 41 0 1 4 0 4 4 0 8 0 pfrule 1344 22 0 16 2 0 2 2 0 8 0 art_heap8 4096 1 0 0 1 0 1 1 0 8 0 art_heap4 256 463 0 49 29 0 29 29 0 8 0 art_table 32 464 0 49 4 0 4 4 0 8 0 art_node 16 119 0 36 1 0 1 1 0 8 0 semupl 112 1 0 1 1 0 1 1 0 8 1 semapl 112 77 0 67 1 0 1 1 0 8 0 shmpl 112 10 0 2 1 0 1 1 0 8 0 dirhash 1024 23 0 6 3 0 3 3 0 8 0 dino2pl 256 1980 0 476 95 0 95 95 0 8 0 ffsino 272 1980 0 476 101 0 101 101 0 8 0 nchpl 144 2489 0 800 63 0 63 63 0 8 0 uvmvnodes 80 2306 0 0 48 0 48 48 0 8 0 vnodes 216 2306 0 0 129 0 129 129 0 8 0 namei 1024 8166 0 8166 2 0 2 2 0 8 2 percpumem 16 48 0 2 1 0 1 1 0 8 0 kstatmem 264 28 0 4 2 0 2 2 0 8 0 scsiplug 72 3 0 3 1 0 1 1 0 8 1 scxspl 216 7311 0 7311 10 2 8 8 1 8 8 plimitpl 152 126 0 106 1 0 1 1 0 8 0 sigapl 424 653 0 602 8 1 7 7 0 8 1 futexpl 64 3507 0 3501 1 0 1 1 0 8 0 knotepl 120 308 0 0 10 0 10 10 0 8 0 kqueuepl 216 208 0 199 5 0 5 5 0 8 4 pipepl 320 185 0 157 8 0 8 8 0 8 5 fdescpl 496 634 0 602 6 0 6 6 0 8 1 filepl 152 3691 0 3266 17 0 17 17 0 8 0 lockfpl 104 97 0 92 1 0 1 1 0 8 0 lockfspl 48 42 0 37 1 0 1 1 0 8 0 sessionpl 144 23 0 14 1 0 1 1 0 8 0 pgrppl 48 34 0 17 1 0 1 1 0 8 0 ucredpl 104 437 0 421 1 0 1 1 0 8 0 zombiepl 144 604 0 602 1 0 1 1 0 8 0 processpl 1160 653 0 602 5 0 5 5 0 8 1 procpl 648 1047 0 984 7 1 6 6 0 8 0 sockpl 664 878 0 715 16 2 14 14 0 8 0 mcl64k 65536 2 0 0 1 0 1 1 0 8 0 mcl9k 9216 1 0 0 1 0 1 1 0 8 0 mcl8k 8192 5 0 0 1 0 1 1 0 8 0 mcl4k 4096 148 0 0 19 0 19 19 0 8 0 mcl2k2 2112 1 0 0 1 0 1 1 0 8 0 mcl2k 2048 20 0 0 3 0 3 3 0 8 0 mtagpl 96 10 0 0 1 0 1 1 0 8 0 mbufpl 256 175 0 0 11 0 11 11 0 8 0 bufpl 280 3626 0 109 252 0 252 252 0 8 0 anonpl 24 166684 0 163031 48 1 47 47 0 185 13 amapchunkpl 152 16318 0 15739 29 0 29 29 0 158 6 amappl16 200 4419 0 4393 19 4 15 15 0 8 12 amappl15 192 3 0 3 1 1 0 1 0 8 0 amappl14 184 116 0 104 1 0 1 1 0 8 0 amappl13 176 12 0 12 1 1 0 1 0 8 0 amappl12 168 1279 0 1247 4 1 3 3 0 8 1 amappl11 160 56 0 42 1 0 1 1 0 8 0 amappl10 152 12 0 12 1 1 0 1 0 8 0 amappl9 144 144 0 144 1 1 0 1 0 8 0 amappl8 136 30 0 26 1 0 1 1 0 8 0 amappl7 128 103 0 91 1 0 1 1 0 8 0 amappl6 120 162 0 161 1 0 1 1 0 8 0 amappl5 112 135 0 124 1 0 1 1 0 8 0 amappl4 104 313 0 292 1 0 1 1 0 8 0 amappl3 96 3001 0 2891 3 0 3 3 0 8 0 amappl2 88 905 0 819 3 0 3 3 0 8 0 amappl1 80 8377 0 7809 14 0 14 14 0 8 0 amappl 88 4568 0 4370 5 0 5 5 0 92 0 dma4096 4096 1 0 1 1 1 0 1 0 8 0 dma1024 1024 1 0 0 1 0 1 1 0 8 0 dma256 256 6 0 6 1 1 0 1 0 8 0 dma128 128 253 0 253 1 1 0 1 0 8 0 dma64 64 6 0 6 1 1 0 1 0 8 0 dma32 32 7 0 7 1 1 0 1 0 8 0 dma16 16 18 0 17 1 0 1 1 0 8 0 aobjpl 72 12 0 2 1 0 1 1 0 8 0 uaddrrnd 24 634 0 602 1 0 1 1 0 8 0 uaddrbest 32 2 0 0 1 0 1 1 0 8 0 uaddr 24 634 0 602 1 0 1 1 0 8 0 vmmpekpl 168 6865 0 6833 2 0 2 2 0 8 0 vmmpepl 168 47249 0 45312 95 0 95 95 0 357 10 vmsppl 448 633 0 602 6 1 5 5 0 8 1 rwobjpl 56 19846 0 16569 49 0 49 49 0 8 2 pdppl 4096 1275 0 1204 101 30 71 87 0 8 0 pvpl 32 14817 0 0 120 0 120 120 0 265 0 pmappl 248 633 0 602 3 0 3 3 0 8 0 extentpl 40 55 0 38 1 0 1 1 0 8 0 phpool 112 397 0 33 11 0 11 11 0 8 0 ddb{0}> machine ddbcpu 0 Invalid cpu 0 ddb{0}> trace db_enter() at db_enter+0x25 sys/arch/amd64/amd64/db_interface.c:437 witness_checkorder(fffffd8066567810,9,0) at witness_checkorder+0x1047 rw_enter(fffffd8066567800,1) at rw_enter+0x122 rrw_enter(fffffd8066567800,1) at rrw_enter+0xbe sys/kern/kern_rwlock.c:464 VOP_LOCK(fffffd8067dcf980,2001) at VOP_LOCK+0xa6 sys/kern/vfs_vops.c:524 vn_lock(fffffd8067dcf980,2001) at vn_lock+0xa4 sys/kern/vfs_vnops.c:564 vn_rdwr(0,fffffd8067dcf980,ffff800011a26140,df,0,1,c9d3c0cb3c87d63f,0,ffff800000b3e000,0) at vn_rdwr+0xd1 sys/kern/vfs_vnops.c:320 vndstrategy(fffffd805ff456d0) at vndstrategy+0x4ff sys/dev/vnd.c:342 physio(ffffffff82b2d360,2902,8000,ffffffff82abc400,ffff800037215898) at physio+0x2f6 sys/kern/kern_physio.c:162 spec_read(ffff800037215700) at spec_read+0x155 sys/kern/spec_vnops.c:215 VOP_READ(fffffd806dedbd08,ffff800037215898,0,fffffd807f7d34e0) at VOP_READ+0x102 sys/kern/vfs_vops.c:227 vn_read(fffffd8067bdd300,ffff800037215898,0) at vn_read+0x17b sys/kern/vfs_vnops.c:369 dofilereadv(ffff800035f9cf58,4,ffff800037215898,0,ffff800037215950) at dofilereadv+0x230 sys/kern/sys_generic.c:252 sys_read(ffff800035f9cf58,ffff800037215a00,ffff800037215950) at sys_read+0xa2 sys/kern/sys_generic.c:172 syscall(ffff800037215a00) at syscall+0xbb6 mi_syscall sys/sys/syscall_mi.h:178 [inline] syscall(ffff800037215a00) at syscall+0xbb6 sys/arch/amd64/amd64/trap.c:577 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0x857cac862f0, count: -16 ddb{0}> machine ddbcpu 1 Stopped at x86_ipi_db+0x27: addq $0x8,%rsp ddb{1}> trace x86_ipi_db(ffff800029b7bff0) at x86_ipi_db+0x27 sys/arch/amd64/amd64/db_interface.c:393 x86_ipi_handler() at x86_ipi_handler+0xd9 sys/arch/amd64/amd64/ipi.c:106 Xresume_lapic_ipi() at Xresume_lapic_ipi+0x27 __sanitizer_cov_trace_pc() at __sanitizer_cov_trace_pc+0xf sys/dev/kcov.c:149 syscall(ffff8000371af050) at syscall+0x2cc mi_syscall sys/sys/syscall_mi.h:155 [inline] syscall(ffff8000371af050) at syscall+0x2cc sys/arch/amd64/amd64/trap.c:577 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0x74744c5f9ed0, count: -6