witness: lock order reversal: 1st 0xffff800001bff310 vmmaplk (&map->lock) 2nd 0xffff800001786108 inode (&ip->i_lock) lock order [1] vmmaplk (&map->lock) -> [2] inode (&ip->i_lock) #0 rw_do_enter_write+0xb8 sys/kern/kern_rwlock.c:234 #1 rrw_enter+0xf5 sys/kern/kern_rwlock.c:621 #2 VOP_LOCK+0x186 sys/kern/vfs_vops.c:527 #3 vn_lock+0xba sys/kern/vfs_vnops.c:576 #4 vn_rdwr+0x160 sys/kern/vfs_vnops.c:326 #5 vndstrategy+0x71c sys/dev/vnd.c:342 #6 physio+0x549 sys/kern/kern_physio.c:162 #7 spec_read+0x2bf sys/kern/spec_vnops.c:215 #8 VOP_READ+0x1f1 sys/kern/vfs_vops.c:227 #9 vn_read+0x20e sys/kern/vfs_vnops.c:375 #10 dofilereadv+0x39e sys/kern/sys_generic.c:252 #11 sys_preadv+0x1f8 sys/kern/vfs_syscalls.c:3283 #12 syscall+0x118e mi_syscall sys/sys/syscall_mi.h:176 [inline] #12 syscall+0x118e sys/arch/amd64/amd64/trap.c:804 #13 Xsyscall+0x128 lock order [2] inode (&ip->i_lock) -> [1] vmmaplk (&map->lock) #0 rw_do_enter_read+0x83 sys/kern/kern_rwlock.c:355 #1 uvmfault_lookup+0x1a2 sys/uvm/uvm_fault.c:-1 #2 uvm_fault_check+0x4b sys/uvm/uvm_fault.c:693 #3 uvm_fault+0x2d4 sys/uvm/uvm_fault.c:-1 #4 kpageflttrap+0x427 sys/arch/amd64/amd64/trap.c:285 #5 kerntrap+0x1bb sys/arch/amd64/amd64/trap.c:545 #6 alltraps_kern_meltdown+0x7b #7 copyout+0x59 #8 ffs_read+0x772 sys/ufs/ffs/ffs_vnops.c:254 #9 VOP_READ+0x1f1 sys/kern/vfs_vops.c:227 #10 vn_rdwr+0x1db sys/kern/vfs_vnops.c:-1 #11 vmcmd_map_readvn+0x1f6 sys/kern/exec_subr.c:250 #12 exec_process_vmcmds+0x163 sys/kern/exec_subr.c:139 #13 sys_execve+0x1358 sys/kern/kern_exec.c:540 #14 start_init+0x6b5 sys/kern/init_main.c:728 #15 proc_trampoline+0x10 Stopped at db_enter+0x17: popq %rbp ddb{0}> ddb{0}> set $lines = 0 ddb{0}> set $maxwidth = 0 ddb{0}> show panic the kernel did not panic ddb{0}> show kasan kasan: no report recorded this boot ddb{0}> trace db_enter() at db_enter+0x17 sys/arch/amd64/amd64/db_interface.c:457 witness_checkorder(ffff800001786108,9,0) at witness_checkorder+0x17f2 sys/kern/subr_witness.c:-1 rw_do_enter_write(ffff8000017860f0,1) at rw_do_enter_write+0xb8 sys/kern/kern_rwlock.c:234 rrw_enter(ffff8000017860f0,1) at rrw_enter+0xf5 sys/kern/kern_rwlock.c:621 VOP_LOCK(ffff8000017f8378,2001) at VOP_LOCK+0x186 sys/kern/vfs_vops.c:527 vn_lock(ffff8000017f8378,2001) at vn_lock+0xba sys/kern/vfs_vnops.c:576 vn_rdwr(0,ffff8000017f8378,ffff800011b061c0,47,0,1,0,ffff800001cd1670,0,ffffffff85117ff0) at vn_rdwr+0x160 sys/kern/vfs_vnops.c:326 vndstrategy(ffff800001cd1670) at vndstrategy+0x71c sys/dev/vnd.c:342 physio(ffffffff84136100,2902,8000,ffffffff8380fe70,ffff8000298ea0a0) at physio+0x549 sys/kern/kern_physio.c:162 spec_read(ffff8000298e9da0) at spec_read+0x2bf sys/kern/spec_vnops.c:215 VOP_READ(ffff800001968518,ffff8000298ea0a0,0,ffff8000001194e0) at VOP_READ+0x1f1 sys/kern/vfs_vops.c:227 vn_read(ffff800001919290,ffff8000298ea0a0,1) at vn_read+0x20e sys/kern/vfs_vnops.c:375 dofilereadv(ffff80003baecd18,3,ffff8000298ea0a0,1,ffff8000298ea280) at dofilereadv+0x39e sys/kern/sys_generic.c:252 sys_preadv(ffff80003baecd18,ffff8000298ea320,ffff8000298ea280) at sys_preadv+0x1f8 sys/kern/vfs_syscalls.c:3283 syscall(ffff8000298ea320) at syscall+0x118e mi_syscall sys/sys/syscall_mi.h:176 [inline] syscall(ffff8000298ea320) at syscall+0x118e sys/arch/amd64/amd64/trap.c:804 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0x59c79ded990, count: -16 ddb{0}> show registers rdi 0x80000 acpi_pdirpa+0x6be71 rsi 0 rbp 0xffff8000298e95a0 rbx 0xffffffff84d13ee0 .str.41 rdx 0xffff80000014ea80 rcx 0x7ffff acpi_pdirpa+0x6be70 rax 0xffff80002b280000 r8 0xffff907ff0a23001 r9 0x298e9180 r10 0xffff8000298e97b8 r11 0xffffffff8180f6b0 x86_bus_space_io_read_1 r12 0xffffffff84e7fea0 ffs_vget.__lock_type r13 0xffff800001bff310 r14 0x3 r15 0xffffffff84e7fee0 .str.13 rip 0xffffffff82d3c837 db_enter+0x17 cs 0x8 rflags 0x246 rsp 0xffff8000298e95a0 ss 0x10 db_enter+0x17: popq %rbp ddb{0}> show proc PROC (syz-executor) tid=249903 pid=6602 tcnt=2 stat=onproc flags process=0 proc=4000000 runpri=32, usrpri=81, slppri=32, nice=20 wchan=0x0, wmesg=, ps_single=0x0 scnt=0 ecnt=0 forw=0xffffffffffffffff, list=0xffff80003baed4e0,0xffff80003baec2c8 process=0xffff800029a7ebb0 user=0xffff8000298df000, vmspace=0xffff800001bff210 estcpu=31, cpticks=1, pctcpu=0.0, user=0, sys=1, intr=0 ddb{0}> ps PID TID PPID UID S PRLAGS PFLAGS WAIT COMMAND 84817 146100 31239 0 2 0x2 0 sh 85960 213506 44824 0 7 0x2 0 ndp 54452 371020 69053 0 2 0 0 syz-executor 54452 81538 69053 0 2 0 0x4000000 syz-executor 44824 282728 67032 0 3 0x100002 0x88 sigsusp sh 6602 177506 45415 0 2 0 0 syz-executor * 6602 249903 45415 0 7 0 0x4000000 syz-executor 31441 277233 20369 0 2 0 0 syz-executor 31441 431236 20369 0 3 0 0x4000080 fsleep syz-executor 31239 166903 47040 0 3 0x2 0x80 wait syz-executor 67032 249713 47040 0 3 0x2 0x80 wait syz-executor 11673 69669 1 0 3 0 0x80 nanoslp init 40211 429747 47040 0 2 0x10000002 0 syz-executor 45415 326199 47040 0 3 0x2 0x80 nanoslp syz-executor 75333 389826 47040 0 3 0x2 0x80 nanoslp syz-executor 69053 16612 47040 0 2 0x2 0xc80 syz-executor 20369 463963 47040 0 3 0x2 0x80 nanoslp syz-executor 47040 371946 1 0 2 0x2 0 syz-executor 58924 100192 10290 74 3 0x1100012 0x80 bpf pflogd 10290 484823 1 0 3 0 0x80 sbwait pflogd 85246 4501 0 0 3 0x14000 0x200 bored smr 85219 339151 0 0 2 0x14000 0x200 zerothread 15512 77850 0 0 3 0x14000 0x200 aiodoned aiodoned 90420 188342 0 0 3 0x14000 0x200 syncer update 71952 304110 0 0 3 0x14000 0x200 cleaner cleaner 48912 242601 0 0 3 0x14000 0x200 reaper reaper 63833 196568 0 0 3 0x14000 0x200 pgdaemon pagedaemon 31111 521608 0 0 3 0x14000 0x200 bored viomb 53304 262919 0 0 3 0x14000 0x40000200 acpi0 acpi0 18836 305977 0 0 3 0x14000 0x40000200 idle1 87739 285697 0 0 3 0x14000 0x200 bored softnet1 40807 77772 0 0 3 0x14000 0x200 bored softnet0 48699 365225 0 0 3 0x14000 0x200 smrbar systqmp 7923 248448 0 0 3 0x14000 0x200 bored systq 82414 90158 0 0 3 0x14000 0x200 tmoslp softclockmp 48707 121259 0 0 3 0x14000 0x40000200 tmoslp softclock 82532 168933 0 0 3 0x14000 0x40000200 idle0 1 212974 0 0 3 0x2 0x80 wait init 0 0 -1 0 3 0x10000 0x200 scheduler swapper ddb{0}> show all locks Process 6602 (syz-executor) thread 0xffff80003baecd18 (249903) shared rwlock vmmaplk r = 0 (0xffff800001bff310) #0 witness_lock+0x534 stacktrace_save sys/sys/stacktrace.h:37 [inline] #0 witness_lock+0x534 sys/kern/subr_witness.c:1160 #1 rw_do_enter_read+0x475 sys/kern/kern_rwlock.c:413 #2 uvm_vslock_device+0x1ce sys/uvm/uvm_glue.c:173 #3 physio+0x470 sys/kern/kern_physio.c:139 #4 spec_read+0x2bf sys/kern/spec_vnops.c:215 #5 VOP_READ+0x1f1 sys/kern/vfs_vops.c:227 #6 vn_read+0x20e sys/kern/vfs_vnops.c:375 #7 dofilereadv+0x39e sys/kern/sys_generic.c:252 #8 sys_preadv+0x1f8 sys/kern/vfs_syscalls.c:3283 #9 syscall+0x118e mi_syscall sys/sys/syscall_mi.h:176 [inline] #9 syscall+0x118e sys/arch/amd64/amd64/trap.c:804 #10 Xsyscall+0x128 exclusive kernel_lock &kernel_lock r = 0 (0xffffffff85e956c0) #0 witness_lock+0x534 stacktrace_save sys/sys/stacktrace.h:37 [inline] #0 witness_lock+0x534 sys/kern/subr_witness.c:1160 #1 vn_read+0x82 sys/kern/vfs_vnops.c:357 #2 dofilereadv+0x39e sys/kern/sys_generic.c:252 #3 sys_preadv+0x1f8 sys/kern/vfs_syscalls.c:3283 #4 syscall+0x118e mi_syscall sys/sys/syscall_mi.h:176 [inline] #4 syscall+0x118e sys/arch/amd64/amd64/trap.c:804 #5 Xsyscall+0x128 Process 48699 (systqmp) thread 0xffff800045ffe000 (365225) shared rwlock systqmp r = 0 (0xffffffff85437738) #0 witness_lock+0x534 stacktrace_save sys/sys/stacktrace.h:37 [inline] #0 witness_lock+0x534 sys/kern/subr_witness.c:1160 #1 taskq_thread+0x1da sys/kern/kern_task.c:442 #2 proc_trampoline+0x10 ddb{0}> show malloc Type InUse MemUse HighUse Limit Requests Type Lim devbuf 11052 12234K 12399K 166960K 12388 0 pcb 18 28K 32K 166960K 56 0 rtable 219 12K 29K 166960K 748 0 pf 34 22K 26K 166960K 92 0 ifaddr 42 12K 15K 166960K 107 0 ifgroup 55 2K 3K 166960K 129 0 sysctl 3 1K 13K 166960K 10 0 counters 70 42K 44K 166960K 108 0 ioctlops 1 0K 195K 166960K 1531 0 iov 0 0K 16K 166960K 9 0 mount 1 1K 1K 166960K 1 0 log 9 24K 24K 166960K 13 0 vnodes 1344 86K 89K 166960K 1704 0 UFS quota 1 36K 36K 166960K 1 0 UFS mount 5 44K 44K 166960K 5 0 shm 2 2K 6K 166960K 9 0 VM map 2 1K 1K 166960K 2 0 sem 10 0K 0K 166960K 12 0 dirhash 12 2K 2K 166960K 12 0 ACPI 1692 195K 405K 166960K 12472 0 file desc 16 61K 113K 166960K 519 0 sigio 0 0K 0K 166960K 2 0 proc 33 47K 182K 166960K 979 0 subproc 72 9K 12K 166960K 198 0 NFS srvsock 1 0K 0K 166960K 1 0 NFS daemon 1 20K 20K 166960K 1 0 ip_moptions 0 0K 0K 166960K 18 0 in_multi 89 6K 9K 166960K 256 0 ether_multi 1 0K 0K 166960K 1 0 mrt 0 0K 0K 166960K 15 0 ISOFS mount 1 36K 36K 166960K 1 0 MSDOSFS mount 1 20K 20K 166960K 1 0 ttys 61 425K 425K 166960K 61 0 exec 5 1K 18K 166960K 586 0 fusefs mount 1 36K 36K 166960K 1 0 tdb 3 1K 1K 166960K 3 0 VM swap 14 124K 124K 166960K 15 0 UVM amap 202 144K 384K 166960K 5906 0 UVM aobj 8 4K 4K 166960K 8 0 pinsyscall 40 73K 383K 166960K 2019 0 memdesc 1 4K 4K 166960K 1 0 crypto data 1 1K 1K 166960K 1 0 ip6_options 0 0K 0K 166960K 15 0 NDP 14 0K 2K 166960K 73 0 temp 945 9257K 9470K 166960K 15420 0 kqueue 2 8K 84K 166960K 64 0 SYN cache 2 16K 16K 166960K 2 0 ddb{0}> show all pools Name Size Requests Fail Releases Pgreq Pgrel Npage Hiwat Minpg Maxpg Idle plcache 128 28 0 0 1 0 1 1 0 8 0 rtpcb 120 67 0 67 1 0 1 1 0 8 1 rtentry 176 260 0 156 6 0 6 6 0 8 0 unpcb 144 170 0 165 1 0 1 1 0 8 0 syncache 336 5 0 5 3 3 0 1 0 8 0 tcpcb 736 31 0 30 1 0 1 1 0 8 0 arp 136 46 0 28 1 0 1 1 0 8 0 inpcb 328 318 0 316 6 0 6 6 0 8 5 nd6 152 63 0 43 1 0 1 1 0 8 0 kcovpl 48 22 0 14 1 0 1 1 0 8 0 ppxss 1192 2 0 2 1 0 1 1 0 8 1 pffrag 232 1 0 1 1 0 1 1 0 482 1 pffrnode 88 1 0 1 1 0 1 1 0 8 1 pffrent 40 2 0 2 1 0 1 1 0 8 1 pfosfp 40 1428 0 1428 5 5 0 5 0 8 0 pfosfpen 112 1428 0 1428 21 7 14 21 0 8 14 pftag 88 1 0 0 1 0 1 1 0 8 0 pfstitem 24 43 0 4 1 0 1 1 0 8 0 pfstkey 128 43 0 4 2 0 2 2 0 8 0 pfstate 448 43 0 4 5 0 5 5 0 8 0 pfrule 1360 24 0 19 2 1 1 2 0 8 0 art_heap8 4096 1 0 0 1 0 1 1 0 8 0 art_heap4 256 1038 0 618 29 0 29 29 0 8 2 art_table 40 1039 0 618 5 0 5 5 0 8 0 art_node 32 260 0 164 1 0 1 1 0 8 0 sysvmsgpl 40 2 0 0 1 0 1 1 0 8 0 semapl 72 10 0 2 1 0 1 1 0 8 0 shmpl 120 5 0