witness: lock order reversal: 1st 0xfffffd807f007b98 vmmaplk (&map->lock) 2nd 0xfffffd80643790a0 inode (&ip->i_lock) lock order "&ip->i_lock"(rrwlock) -> "&map->lock"(rwlock) first seen at: #0 witness_checkorder+0x69c witness_lock_order_check sys/kern/subr_witness.c:2441 [inline] #0 witness_checkorder+0x69c sys/kern/subr_witness.c:872 #1 rw_enter_read+0x66 sys/kern/kern_rwlock.c:109 #2 uvmfault_lookup+0xd9 sys/uvm/uvm_fault.c:1448 #3 uvm_fault+0x7f sys/uvm/uvm_fault.c:524 #4 pageflttrap+0x20b sys/arch/amd64/amd64/trap.c:199 #5 kerntrap+0xec sys/arch/amd64/amd64/trap.c:287 #6 alltraps_kern_meltdown+0x7b #7 copyout+0x53 #8 ffs_read+0x362 sys/ufs/ffs/ffs_vnops.c:254 #9 VOP_READ+0xbf sys/kern/vfs_vops.c:248 #10 vn_read+0x124 sys/kern/vfs_vnops.c:375 #11 dofilereadv+0x1a2 sys/kern/sys_generic.c:236 #12 sys_read+0x83 sys/kern/sys_generic.c:156 #13 syscall+0x4a4 mi_syscall sys/sys/syscall_mi.h:92 [inline] #13 syscall+0x4a4 sys/arch/amd64/amd64/trap.c:555 #14 Xsyscall+0x128 lock order "&map->lock"(rwlock) -> "&ip->i_lock"(rrwlock) first seen at: #0 witness_checkorder+0x69c witness_lock_order_check sys/kern/subr_witness.c:2441 [inline] #0 witness_checkorder+0x69c sys/kern/subr_witness.c:872 #1 rw_enter+0xd4 #2 rrw_enter+0x88 sys/kern/kern_rwlock.c:453 #3 VOP_LOCK+0xf9 sys/kern/vfs_vops.c:615 #4 vn_lock+0x81 sys/kern/vfs_vnops.c:571 #5 uvn_io+0x2c8 sys/uvm/uvm_vnode.c:1189 #6 uvn_get+0x226 sys/uvm/uvm_vnode.c:1049 #7 uvm_fault+0x11cc sys/uvm/uvm_fault.c:1023 #8 uvm_fault_wire+0x70 sys/uvm/uvm_fault.c:1295 #9 uvm_map_pageable_wire+0x2dd sys/uvm/uvm_map.c:2358 #10 sys_mlock+0x185 sys/uvm/uvm_mmap.c:765 #11 syscall+0x4a4 mi_syscall sys/sys/syscall_mi.h:92 [inline] #11 syscall+0x4a4 sys/arch/amd64/amd64/trap.c:555 #12 Xsyscall+0x128 Stopped at db_enter+0x18: addq $0x8,%rsp ddb{0}> ddb{0}> set $lines = 0 ddb{0}> set $maxwidth = 0 ddb{0}> show panic the kernel did not panic ddb{0}> trace db_enter() at db_enter+0x18 sys/arch/amd64/amd64/db_interface.c:398 witness_checkorder(fffffd80643790a0,9,0) at witness_checkorder+0x1009 witness_debugger sys/kern/subr_witness.c:2504 [inline] witness_checkorder(fffffd80643790a0,9,0) at witness_checkorder+0x1009 sys/kern/subr_witness.c:1086 rw_enter(fffffd8064379090,81) at rw_enter+0xd4 rrw_enter(fffffd8064379090,81) at rrw_enter+0x88 sys/kern/kern_rwlock.c:453 VOP_LOCK(fffffd8077650000,81) at VOP_LOCK+0xf9 sys/kern/vfs_vops.c:615 vn_lock(fffffd8077650000,81) at vn_lock+0x81 sys/kern/vfs_vnops.c:571 uvn_io(fffffd80741c8918,ffff800024fbb5b8,1,2,0) at uvn_io+0x2c8 sys/uvm/uvm_vnode.c:1189 uvn_get(fffffd80741c8918,0,ffff800024fbb810,ffff800024fbb7a4,0,1) at uvn_get+0x226 sys/uvm/uvm_vnode.c:1049 uvm_fault(fffffd807f007b80,20002000,2,1) at uvm_fault+0x11cc sys/uvm/uvm_fault.c:1023 uvm_fault_wire(fffffd807f007b80,20002000,20005000,1) at uvm_fault_wire+0x70 sys/uvm/uvm_fault.c:1295 uvm_map_pageable_wire(fffffd807f007b80,fffffd806f32c268,fffffd806f32caf0,ffff800022d4c000,0,0) at uvm_map_pageable_wire+0x2dd sys/uvm/uvm_map.c:2358 sys_mlock(ffff800020a998c8,ffff800024fbba08,ffff800024fbba50) at sys_mlock+0x185 sys/uvm/uvm_mmap.c:765 syscall(ffff800024fbbad0) at syscall+0x4a4 mi_syscall sys/sys/syscall_mi.h:92 [inline] syscall(ffff800024fbbad0) at syscall+0x4a4 sys/arch/amd64/amd64/trap.c:555 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0xaaedec55b00, count: -14 ddb{0}> show registers rdi 0x3 rsi 0x3ffff acpi_pdirpa+0x2be67 rbp 0xffff800024fbb230 rbx 0x3 rdx 0x40000 acpi_pdirpa+0x2be68 rcx 0xffff800022d4c000 rax 0xffff8000009f8700 r8 0xffffffff81ad1322 witness_checkorder+0xfe2 r9 0x5 r10 0x3 r11 0xd09eac2c775e322 r12 0xfffffd800297f240 r13 0 r14 0xffffffff825c8260 w_lodata+0x41500 r15 0xffffffff825dae20 w_lodata+0x540c0 rip 0xffffffff8129eb48 db_enter+0x18 cs 0x8 rflags 0x246 rsp 0xffff800024fbb220 ss 0x10 db_enter+0x18: addq $0x8,%rsp ddb{0}> show proc PROC (syz-executor.1) pid=481562 stat=onproc flags process=10 proc=4000000 pri=81, usrpri=81, nice=20 forw=0xffffffffffffffff, list=0xffff800020a989f8,0xffffffff8260dd10 process=0xffff800020a9b540 user=0xffff800024fb6000, vmspace=0xfffffd807f007b80 estcpu=36, cpticks=2, pctcpu=0.0 user=0, sys=0, intr=0 ddb{0}> ps PID TID PPID UID S FLAGS WAIT COMMAND 58858 131660 99586 32767 7 0x10 syz-executor.1 *58858 481562 99586 32767 7 0x4000010 syz-executor.1 3578 66325 32927 32767 3 0x90 nanosleep syz-executor.0 3578 413737 32927 32767 3 0x4000090 kqread syz-executor.0 3578 165578 32927 32767 3 0x4000090 kqread syz-executor.0 3578 517332 32927 32767 3 0x4000090 kqread syz-executor.0 3578 505720 32927 32767 3 0x4000090 kqread syz-executor.0 3578 79952 32927 32767 3 0x4000090 fsleep syz-executor.0 32927 291635 25440 32767 3 0x90 nanosleep syz-executor.0 25440 131573 16122 0 3 0x82 wait syz-executor.0 99586 267634 88023 32767 3 0x90 nanosleep syz-executor.1 88023 378174 16122 0 3 0x82 wait syz-executor.1 53187 87683 0 0 3 0x14200 bored sosplice 16122 93603 18044 0 3 0x82 thrsleep syz-fuzzer 16122 457826 18044 0 3 0x4000082 nanosleep syz-fuzzer 16122 176884 18044 0 3 0x4000082 thrsleep syz-fuzzer 16122 4109 18044 0 3 0x4000082 thrsleep syz-fuzzer 16122 9988 18044 0 3 0x4000082 kqread syz-fuzzer 16122 477980 18044 0 3 0x4000082 thrsleep syz-fuzzer 16122 40147 18044 0 3 0x4000082 thrsleep syz-fuzzer 16122 372948 18044 0 3 0x4000082 thrsleep syz-fuzzer 16122 42540 18044 0 3 0x4000082 thrsleep syz-fuzzer 16122 332017 18044 0 3 0x4000082 thrsleep syz-fuzzer 18044 475068 93072 0 3 0x10008a pause ksh 93072 513315 66549 0 3 0x92 select sshd 50628 371305 1 0 3 0x100083 ttyin getty 66549 447163 1 0 3 0x80 select sshd 84733 420537 7527 73 3 0x100090 kqread syslogd 7527 30805 1 0 3 0x100082 netio syslogd 43051 72262 1 77 3 0x100090 poll dhclient 14989 90129 1 0 3 0x80 poll dhclient 95195 318712 0 0 3 0x14200 pgzero zerothread 54852 436385 0 0 3 0x14200 aiodoned aiodoned 19329 430843 0 0 3 0x14200 syncer update 46476 249232 0 0 3 0x14200 cleaner cleaner 2835 419987 0 0 3 0x14200 reaper reaper 2211 41937 0 0 3 0x14200 pgdaemon pagedaemon 50843 127999 0 0 3 0x14200 bored crynlk 23867 38875 0 0 3 0x14200 bored crypto 31296 81435 0 0 3 0x40014200 acpi0 acpi0 94260 435881 0 0 3 0x40014200 idle1 14036 177724 0 0 3 0x14200 bored softnet 14502 292228 0 0 3 0x14200 bored systqmp 71523 187067 0 0 3 0x14200 bored systq 2474 430697 0 0 3 0x40014200 bored softclock 69490 123487 0 0 3 0x40014200 idle0 30443 23668 0 0 3 0x14200 bored smr 1 6888 0 0 3 0x82 wait init 0 0 -1 0 3 0x10200 scheduler swapper ddb{0}> show all locks Process 58858 (syz-executor.1) thread 0xffff800020a998c8 (481562) shared rwlock vmmaplk r = 0 (0xfffffd807f007b98) #0 witness_lock+0x52e sys/kern/subr_witness.c:1163 #1 rw_enter+0x453 sys/kern/kern_rwlock.c:309 #2 vm_map_lock_ln+0x10d sys/uvm/uvm_map.c:5409 #3 uvm_map_pageable+0x11d sys/uvm/uvm_map.c:2454 #4 sys_mlock+0x185 sys/uvm/uvm_mmap.c:765 #5 syscall+0x4a4 mi_syscall sys/sys/syscall_mi.h:92 [inline] #5 syscall+0x4a4 sys/arch/amd64/amd64/trap.c:555 #6 Xsyscall+0x128 exclusive kernel_lock &kernel_lock r = 0 (0xffffffff82650348) #0 witness_lock+0x52e sys/kern/subr_witness.c:1163 #1 syscall+0x400 mi_syscall sys/sys/syscall_mi.h:83 [inline] #1 syscall+0x400 sys/arch/amd64/amd64/trap.c:555 #2 Xsyscall+0x128 ddb{0}> show malloc Type InUse MemUse HighUse Limit Requests Type Lim devbuf 9483 6334K 6334K 78643K 10979 0 pcb 13 14K 18K 78643K 17 0 rtable 105 3K 3K 78643K 4818 0 ifaddr 39 11K 12K 78643K 778 0 counters 39 33K 33K 78643K 39 0 ioctlops 0 0K 2K 78643K 252 0 iov 0 0K 48K 78643K 1513 0 mount 1 1K 1K 78643K 1 0 vnodes 1214 76K 76K 78643K 3529 0 UFS quota 1 32K 32K 78643K 1 0 UFS mount 5 36K 36K 78643K 5 0 shm 2 1K 9K 78643K 56 0 VM map 2 1K 1K 78643K 2 0 sem 12 1K 1K 78643K 143 0 dirhash 12 2K 2K 78643K 12 0 ACPI 1809 196K 290K 78643K 12766 0 file desc 8 25K 33K 78643K 6633 0 sigio 0 0K 0K 78643K 409 0 proc 48 50K 83K 78643K 6010 0 subproc 34 2K 2K 78643K 1751 0 NFS srvsock 1 0K 0K 78643K 1 0 NFS daemon 1 16K 16K 78643K 1 0 ip_moptions 0 0K 0K 78643K 787 0 in_multi 33 2K 2K 78643K 1229 0 ether_multi 1 0K 0K 78643K 29 0 ISOFS mount 1 32K 32K 78643K 1 0 MSDOSFS mount 1 16K 16K 78643K 1 0 ttys 138 609K 609K 78643K 138 0 exec 0 0K 1K 78643K 2149 0 pagedep 1 8K 8K 78643K 1 0 inodedep 1 32K 32K 78643K 1 0 newblk 1 0K 0K 78643K 1 0 VM swap 7 26K 26K 78643K 7 0 UVM amap 155 33K 38K 78643K 21868 0 UVM aobj 130 4K 4K 78643K 130 0 memdesc 1 4K 4K 78643K 1 0 crypto data 1 1K 1K 78643K 1 0 ip6_options 0 0K 1K 78643K 756 0 NDP 5 0K 0K 78643K 312 0 temp 106 3038K 3124K 78643K 29042 0 kqueue 0 0K 0K 78643K 253 0 SYN cache 2 16K 16K 78643K 2 0 ddb{0}> show all pools Name Size Requests Fail Releases Pgreq Pgrel Npage Hiwat Minpg Maxpg Idle arp 64 213 0 207 1 0 1 1 0 8 0 plcache 128 20 0 0 1 0 1 1 0 8 0 rtpcb 80 958 0 956 1 0 1 1 0 8 0 rtentry 112 1060 0 1016 2 0 2 2 0 8 0 unpcb 120 4864 0 4835 11 9 2 2 0 8 1 syncache 264 75 0 75 14 14 0 1 0 8 0 tcpqe 32 42 0 42 12 12 0 1 0 8 0 tcpcb 544 3547 0 3542 24 21 3 14 0 8 2 ipq 40 30 0 30 9 8 1 1 0 8 1 ipqe 40 1189 0 1189 9 8 1 1 0 8 1 inpcb 280 7433 0 7424 35 31 4 15 0 8 3 ip6q 72 2 0 2 1 1 0 1 0 8 0 ip6af 40 6 0 6 1 1 0 1 0 8 0 nd6 48 206 0 202 1 0 1 1 0 8 0 art_heap8 4096 1 0 0 1 0 1 1 0 8 0 art_heap4 256 4756 0 4541 14 0 14 14 0 8 0 art_table 32 4757 0 4541 2 0 2 2 0 8 0 art_node 16 1059 0 1019 1 0 1 1 0 8 0 sysvmsgpl 40 101 0 101 1 1 0 1 0 8 0 semupl 112 6 0 6 1 1 0 1 0 8 0 semapl 112 141 0 131 1 0 1 1 0 8 0 shmpl 112 128 0 0 4 0 4 4 0 8 0 dirhash 1024 17 0 0 3 0 3 3 0 8 0 dino1pl 128 8704 0 7284 46 0 46 46 0 8 0 ffsino 272 8704 0 7284 96 1 95 96 0 8 0 nchpl 144 18753 0 17120 61 0 61 61 0 8 0 uvmvnodes 72 5926 0 0 108 0 108 108 0 8 0 vnodes 208 5926 0 0 312 0 312 312 0 8 0 namei 1024 83468 0 83468 2 1 1 2 0 8 1 percpumem 16 30 0 0 1 0 1 1 0 8 0 scxspl 192 52581 0 52581 39 38 1 7 0 8 1 plimitpl 152 905 0 896 1 0 1 1 0 8 0 sigapl 432 6501 0 6485 7 5 2 3 0 8 0 futexpl 56 119987 0 119986 1 0 1 1 0 8 0 knotepl 112 2421 0 2402 8 6 2 2 0 8 1 kqueuepl 104 4225 0 4218 14 13 1 6 0 8 0 pipepl 160 4120 0 4101 20 18 2 2 0 8 1 fdescpl 488 6502 0 6485 3 0 3 3 0 8 0 filepl 152 60982 0 60873 67 61 6 20 0 8 1 lockfpl 104 1918 0 1917 1 0 1 1 0 8 0 lockfspl 48 535 0 534 1 0 1 1 0 8 0 sessionpl 112 118 0 108 1 0 1 1 0 8 0 pgrppl 48 183 0 173 1 0 1 1 0 8 0 ucredpl 96 13402 0 13393 1 0 1 1 0 8 0 zombiepl 144 6485 0 6485 2 1 1 1 0 8 1 processpl 904 6518 0 6485 4 0 4 4 0 8 0 procpl 632 16228 0 16180 17 12 5 5 0 8 1 srpgc 64 202 0 202 19 18 1 1 0 8 1 sosppl 128 245 0 245 16 15 1 1 0 8 1 sockpl 384 13402 0 13362 57 48 9 24 0 8 4 mcl64k 65536 35 0 0 3 0 3 3 0 8 0 mcl16k 16384 35 0 0 4 1 3 3 0 8 0 mcl12k 12288 17 0 0 2 0 2 2 0 8 0 mcl9k 9216 17 0 0 2 0 2 2 0 8 0 mcl8k 8192 17 0 0 3 0 3 3 0 8 0 mcl4k 4096 41 0 0 5 2 3 3 0 8 0 mcl2k2 2112 8 0 0 1 0 1 1 0 8 0 mcl2k 2048 453 0 0 17 3 14 17 0 8 0 mtagpl 80 1 0 0 1 0 1 1 0 8 0 mbufpl 256 803 0 0 46 1 45 45 0 8 0 bufpl 280 175675 0 168609 506 0 506 506 0 8 0 anonpl 16 1790685 0 1780960 120 74 46 61 0 125 0 amapchunkpl 152 53763 0 53534 59 47 12 12 0 158 3 amappl16 192 95335 0 94956 164 143 21 41 0 8 1 amappl15 184 647 0 644 1 0 1 1 0 8 0 amappl14 176 2157 0 2151 2 1 1 1 0 8 0 amappl13 168 244 0 242 7 6 1 1 0 8 0 amappl12 160 555 0 551 1 0 1 1 0 8 0 amappl11 152 1544 0 1532 1 0 1 1 0 8 0 amappl10 144 929 0 926 1 0 1 1 0 8 0 amappl9 136 2381 0 2375 1 0 1 1 0 8 0 amappl8 128 1607 0 1532 4 1 3 3 0 8 0 amappl7 120 1622 0 1607 1 0 1 1 0 8 0 amappl6 112 1341 0 1326 1 0 1 1 0 8 0 amappl5 104 1518 0 1507 1 0 1 1 0 8 0 amappl4 96 6434 0 6405 1 0 1 1 0 8 0 amappl3 88 2074 0 2059 1 0 1 1 0 8 0 amappl2 80 44027 0 43943 4 2 2 3 0 8 0 amappl1 72 171312 0 170829 26 16 10 20 0 8 0 amappl 80 18414 0 18370 2 0 2 2 0 84 0 dma4096 4096 1 0 1 1 1 0 1 0 8 0 dma256 256 6 0 6 1 1 0 1 0 8 0 dma128 128 253 0 253 1 1 0 1 0 8 0 dma64 64 6 0 6 1 1 0 1 0 8 0 dma32 32 7 0 7 1 1 0 1 0 8 0 dma16 16 18 0 17 1 0 1 1 0 8 0 aobjpl 64 129 0 0 3 0 3 3 0 8 0 uaddrrnd 24 6502 0 6485 1 0 1 1 0 8 0 uaddrbest 32 2 0 0 1 0 1 1 0 8 0 uaddr 24 6502 0 6485 1 0 1 1 0 8 0 vmmpekpl 168 52765 0 52733 2 0 2 2 0 8 0 vmmpepl 168 876381 0 874573 174 90 84 96 0 357 0 vmsppl 368 6501 0 6485 2 0 2 2 0 8 0 pdppl 4096 13011 0 12970 6 0 6 6 0 8 0 pvpl 32 3454199 0 3440494 321 202 119 155 0 265 0 pmappl 232 6501 0 6485 10 9 1 2 0 8 0 extentpl 40 46 0 29 1 0 1 1 0 8 0 phpool 112 232 0 32 7 0 7 7 0 8 0