witness: lock order reversal: 1st 0xffff800001bee8c8 vmmaplk (&map->lock) 2nd 0xffff800001b98590 inode (&ip->i_lock) lock order [1] vmmaplk (&map->lock) -> [2] inode (&ip->i_lock) #0 rw_do_enter_write+0xb8 sys/kern/kern_rwlock.c:234 #1 rrw_enter+0xf5 sys/kern/kern_rwlock.c:621 #2 VOP_LOCK+0x186 sys/kern/vfs_vops.c:527 #3 vn_lock+0xba sys/kern/vfs_vnops.c:576 #4 vn_rdwr+0x160 sys/kern/vfs_vnops.c:326 #5 vndstrategy+0x71c sys/dev/vnd.c:342 #6 physio+0x549 sys/kern/kern_physio.c:162 #7 spec_read+0x2bf sys/kern/spec_vnops.c:215 #8 VOP_READ+0x1f1 sys/kern/vfs_vops.c:227 #9 vn_read+0x20e sys/kern/vfs_vnops.c:375 #10 dofilereadv+0x39e sys/kern/sys_generic.c:252 #11 sys_read+0x157 sys/kern/sys_generic.c:172 #12 syscall+0x118e mi_syscall sys/sys/syscall_mi.h:176 [inline] #12 syscall+0x118e sys/arch/amd64/amd64/trap.c:804 #13 Xsyscall+0x128 lock order [2] inode (&ip->i_lock) -> [1] vmmaplk (&map->lock) #0 rw_do_enter_read+0x83 sys/kern/kern_rwlock.c:355 #1 uvmfault_lookup+0x1a2 sys/uvm/uvm_fault.c:-1 #2 uvm_fault_check+0x4b sys/uvm/uvm_fault.c:693 #3 uvm_fault+0x2d4 sys/uvm/uvm_fault.c:-1 #4 kpageflttrap+0x427 sys/arch/amd64/amd64/trap.c:285 #5 kerntrap+0x1bb sys/arch/amd64/amd64/trap.c:545 #6 alltraps_kern_meltdown+0x7b #7 copyout+0x59 #8 ffs_read+0x772 sys/ufs/ffs/ffs_vnops.c:254 #9 VOP_READ+0x1f1 sys/kern/vfs_vops.c:227 #10 vn_rdwr+0x1db sys/kern/vfs_vnops.c:-1 #11 vmcmd_map_readvn+0x1f6 sys/kern/exec_subr.c:250 #12 exec_process_vmcmds+0x163 sys/kern/exec_subr.c:139 #13 sys_execve+0x1358 sys/kern/kern_exec.c:540 #14 start_init+0x6b5 sys/kern/init_main.c:728 #15 proc_trampoline+0x10 Stopped at db_enter+0x17: popq %rbp ddb{0}> ddb{0}> set $lines = 0 ddb{0}> set $maxwidth = 0 ddb{0}> show panic the kernel did not panic ddb{0}> show kasan kasan: no report recorded this boot ddb{0}> trace db_enter() at db_enter+0x17 sys/arch/amd64/amd64/db_interface.c:457 witness_checkorder(ffff800001b98590,9,0) at witness_checkorder+0x17f2 sys/kern/subr_witness.c:-1 rw_do_enter_write(ffff800001b98578,1) at rw_do_enter_write+0xb8 sys/kern/kern_rwlock.c:234 rrw_enter(ffff800001b98578,1) at rrw_enter+0xf5 sys/kern/kern_rwlock.c:621 VOP_LOCK(ffff800001bd3620,2001) at VOP_LOCK+0x186 sys/kern/vfs_vops.c:527 vn_lock(ffff800001bd3620,2001) at vn_lock+0xba sys/kern/vfs_vnops.c:576 vn_rdwr(0,ffff800001bd3620,ffff800011b060c0,16,0,1,0,ffff800001bd0450,0,ffffffff85117ff0) at vn_rdwr+0x160 sys/kern/vfs_vnops.c:326 vndstrategy(ffff800001bd0450) at vndstrategy+0x71c sys/dev/vnd.c:342 physio(ffffffff84136100,2902,8000,ffffffff8380fe70,ffff8000396b7320) at physio+0x549 sys/kern/kern_physio.c:162 spec_read(ffff8000396b70c0) at spec_read+0x2bf sys/kern/spec_vnops.c:215 VOP_READ(ffff8000019646c8,ffff8000396b7320,0,ffff8000001193a8) at VOP_READ+0x1f1 sys/kern/vfs_vops.c:227 vn_read(ffff800001c622c8,ffff8000396b7320,0) at vn_read+0x20e sys/kern/vfs_vnops.c:375 dofilereadv(ffff80003bb82028,3,ffff8000396b7320,0,ffff8000396b74e0) at dofilereadv+0x39e sys/kern/sys_generic.c:252 sys_read(ffff80003bb82028,ffff8000396b7580,ffff8000396b74e0) at sys_read+0x157 sys/kern/sys_generic.c:172 syscall(ffff8000396b7580) at syscall+0x118e mi_syscall sys/sys/syscall_mi.h:176 [inline] syscall(ffff8000396b7580) at syscall+0x118e sys/arch/amd64/amd64/trap.c:804 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0x476f654f4e0, count: -16 ddb{0}> show registers rdi 0 rsi 0 rbp 0xffff8000396b68c0 rbx 0xffffffff84d13ee0 .str.41 rdx 0 rcx 0xffffffff85117ff0 cpu_info_full_primary+0x1ff0 rax 0 r8 0xffff907ff0a23001 r9 0x396b64a0 r10 0xffff8000396b6ad8 r11 0xffffffff8180f6b0 x86_bus_space_io_read_1 r12 0xffffffff84e7fea0 ffs_vget.__lock_type r13 0xffff800001bee8c8 r14 0x3 r15 0xffffffff84e7fee0 .str.13 rip 0xffffffff82d3c837 db_enter+0x17 cs 0x8 rflags 0x246 rsp 0xffff8000396b68c0 ss 0x10 db_enter+0x17: popq %rbp ddb{0}> show proc PROC (syz-executor) tid=495230 pid=57536 tcnt=2 stat=onproc flags process=0 proc=4000000 runpri=32, usrpri=86, slppri=32, nice=20 wchan=0x0, wmesg=, ps_single=0x0 scnt=0 ecnt=0 forw=0xffffffffffffffff, list=0xffff80003bb83250,0xffffffff85e9ac60 process=0xffff80003ab1c020 user=0xffff8000396ac000, vmspace=0xffff800001bee7c8 estcpu=36, cpticks=0, pctcpu=0.0, user=0, sys=0, intr=0 ddb{0}> ps PID TID PPID UID S PRLAGS PFLAGS WAIT COMMAND 57536 489778 20524 0 2 0 0 syz-executor *57536 495230 20524 0 7 0 0x4000000 syz-executor 44699 176325 49885 0 2 0 0 syz-executor 44699 192415 49885 0 3 0 0x4000080 fsleep syz-executor 97009 88901 75203 0 2 0 0 syz-executor 97009 437693 75203 0 3 0 0x4000080 fsleep syz-executor 72025 506727 89165 0 2 0 0 syz-executor 72025 250033 89165 0 3 0 0x4000080 fsleep syz-executor 84881 162599 44212 0 2 0 0 syz-executor 84881 493528 44212 0 3 0 0x4000080 sbwait syz-executor 8184 392993 49324 0 3 0x2 0x80 piperd syz-executor 20524 272557 49324 0 3 0x2 0x80 nanoslp syz-executor 44212 177394 49324 0 7 0x2 0x1 syz-executor 49885 401247 49324 0 3 0x2 0x80 nanoslp syz-executor 89165 431909 49324 0 3 0x2 0x80 nanoslp syz-executor 75203 469820 49324 0 2 0x2 0xc80 syz-executor 83874 282956 49324 0 3 0x2 0x80 piperd syz-executor 49324 507118 72294 0 3 0x2 0x80 wait syz-executor 72294 197213 64568 0 3 0x100002 0x88 sigsusp ksh 64568 396747 20469 0 3 0x10 0x88 kqread sshd-session 20469 213582 19165 0 3 0x12 0x80 kqread sshd-session 50279 350814 1 0 3 0x100003 0x80 ttyin getty 19165 110119 1 0 3 0 0x88 kqread sshd 37929 373065 41844 74 3 0x1100012 0x80 bpf pflogd 41844 297645 1 0 3 0 0x80 sbwait pflogd 84324 493736 18611 73 3 0x1100010 0x80 kqread syslogd 18611 505006 1 0 3 0x100002 0x80 sbwait syslogd-parent 53099 279269 1 0 3 0x100000 0x80 kqread resolvd 33016 151206 82053 77 3 0x100012 0x80 kqread dhcpleased 46169 8292 82053 77 3 0x100012 0x80 kqread dhcpleased 82053 452350 1 0 3 0 0x80 kqread dhcpleased 6213 6477 0 0 3 0x14000 0x200 bored smr 3917 152478 0 0 2 0x14000 0x200 zerothread 53631 194965 0 0 3 0x14000 0x200 aiodoned aiodoned 83681 196543 0 0 3 0x14000 0x200 syncer update 20190 445562 0 0 3 0x14000 0x200 cleaner cleaner 53127 231112 0 0 3 0x14000 0x200 reaper reaper 89685 492236 0 0 3 0x14000 0x200 pgdaemon pagedaemon 23329 399789 0 0 3 0x14000 0x200 bored viomb 87401 341169 0 0 3 0x14000 0x40000200 acpi0 acpi0 47397 86094 0 0 3 0x14000 0x40000200 idle1 94835 452464 0 0 3 0x14000 0x200 bored softnet1 85061 134030 0 0 3 0x14000 0x200 bored softnet0 26959 87315 0 0 3 0x14000 0x200 bored systqmp 34960 330554 0 0 3 0x14000 0x200 bored systq 19542 260105 0 0 3 0x14000 0x200 tmoslp softclockmp 34870 10633 0 0 3 0x14000 0x40000200 tmoslp softclock 51435 108776 0 0 3 0x14000 0x40000200 idle0 1 393717 0 0 3 0x2 0x80 wait init 0 0 -1 0 3 0x10000 0x200 scheduler swapper ddb{0}> show all locks Process 57536 (syz-executor) thread 0xffff80003bb82028 (495230) shared rwlock vmmaplk r = 0 (0xffff800001bee8c8) #0 witness_lock+0x534 stacktrace_save sys/sys/stacktrace.h:37 [inline] #0 witness_lock+0x534 sys/kern/subr_witness.c:1160 #1 rw_do_enter_read+0x475 sys/kern/kern_rwlock.c:413 #2 uvm_vslock_device+0x1ce sys/uvm/uvm_glue.c:173 #3 physio+0x470 sys/kern/kern_physio.c:139 #4 spec_read+0x2bf sys/kern/spec_vnops.c:215 #5 VOP_READ+0x1f1 sys/kern/vfs_vops.c:227 #6 vn_read+0x20e sys/kern/vfs_vnops.c:375 #7 dofilereadv+0x39e sys/kern/sys_generic.c:252 #8 sys_read+0x157 sys/kern/sys_generic.c:172 #9 syscall+0x118e mi_syscall sys/sys/syscall_mi.h:176 [inline] #9 syscall+0x118e sys/arch/amd64/amd64/trap.c:804 #10 Xsyscall+0x128 exclusive kernel_lock &kernel_lock r = 0 (0xffffffff85e956c0) #0 witness_lock+0x534 stacktrace_save sys/sys/stacktrace.h:37 [inline] #0 witness_lock+0x534 sys/kern/subr_witness.c:1160 #1 vn_read+0x82 sys/kern/vfs_vnops.c:357 #2 dofilereadv+0x39e sys/kern/sys_generic.c:252 #3 sys_read+0x157 sys/kern/sys_generic.c:172 #4 syscall+0x118e mi_syscall sys/sys/syscall_mi.h:176 [inline] #4 syscall+0x118e sys/arch/amd64/amd64/trap.c:804 #5 Xsyscall+0x128 ddb{0}> show malloc Type InUse MemUse HighUse Limit Requests Type Lim devbuf 11058 12249K 12394K 166960K 12157 0 pcb 22 24K 24K 166960K 23 0 rtable 231 14K 24K 166960K 341 0 pf 57 68K 68K 166960K 68 0 ifaddr 43 12K 12K 166960K 45 0 ifgroup 55 2K 2K 166960K 55 0 sysctl 1 1K 13K 166960K 5 0 counters 70 42K 42K 166960K 70 0 ioctlops 0 0K 197K 166960K 1487 0 iov 1 0K 0K 166960K 1 0 mount 1 1K 1K 166960K 1 0 log 0 0K 0K 166960K 4 0 vnodes 1297 83K 84K 166960K 1376 0 UFS quota 1 36K 36K 166960K 1 0 UFS mount 5 44K 44K 166960K 5 0 shm 2 2K 2K 166960K 2 0 VM map 2 1K 1K 166960K 2 0 sem 2 0K 0K 166960K 2 0 dirhash 12 2K 2K 166960K 12 0 ACPI 1692 195K 405K 166960K 12472 0 file desc 15 53K 113K 166960K 161 0 sigio 1 0K 0K 166960K 1 0 proc 70 128K 196K 166960K 553 0 subproc 72 9K 9K 166960K 72 0 NFS srvsock 1 0K 0K 166960K 1 0 NFS daemon 1 20K 20K 166960K 1 0 ip_moptions 2 0K 0K 166960K 2 0 in_multi 99 7K 7K 166960K 99 0 ether_multi 1 0K 0K 166960K 1 0 mrt 1 0K 0K 166960K 2 0 ISOFS mount 1 36K 36K 166960K 1 0 MSDOSFS mount 1 20K 20K 166960K 1 0 ttys 55 384K 384K 166960K 55 0 exec 3 0K 18K 166960K 391 0 fusefs mount 1 36K 36K 166960K 1 0 tdb 3 1K 1K 166960K 3 0 VM swap 14 124K 124K 166960K 15 0 UVM amap 351 243K 347K 166960K 3258 0 UVM aobj 4 4K 4K 166960K 4 0 pinsyscall 59 116K 382K 166960K 1297 0 memdesc 1 4K 4K 166960K 1 0 crypto data 1 1K 1K 166960K 1 0 ip6_options 1 0K 0K 166960K 3 0 NDP 12 0K 2K 166960K 28 0 temp 853 9249K 9472K 166960K 6024 0 kqueue 16 52K 84K 166960K 30 0 SYN cache 2 16K 16K 166960K 2 0 ddb{0}> show all pools Name Size Requests Fail Releases Pgreq Pgrel Npage Hiwat Minpg Maxpg Idle plcache 128 28 0 0 1 0 1 1 0 8 0 rtpcb 120 33 0 30 1 0 1 1 0 8 0 rtentry 176 111 0 1 5 0 5 5 0 8 0 unpcb 144 56 0 36 1 0 1 1 0 8 0 syncache 336 4 0 4 2 1 1 1 0 8 1 tcpcb 736 9 0 5 1 0 1 1 0 8 0 arp 136 18 0 0 1 0 1 1 0 8 0 inpcb 328 86 0 76 2 0 2 2 0 8 1 nd6 152 24 0 0 1 0 1 1 0 8 0 kcovpl 48 8 0 0 1 0 1 1 0 8 0 pfosfp 40 1428 0 1005 5 0 5 5 0 8 0 pfosfpen 112 1428 0 714 21 0 21 21 0 8 0 pfanchor 1288 10 0 0 1 0 1 1 0 8 0 pfstitem 24 27 0 0 1 0 1 1 0 8 0 pfstkey 128 27 0 0 1 0 1 1 0 8 0 pfstate 448 27 0 0 3 0 3 3 0 8 0 pfrule 1360 21 0 16 2 1 1 2 0 8 0 art_heap8 4096 1 0 0 1 0 1 1 0 8 0 art_heap4 256 452 0 0 29 0 29 29 0 8 0 art_table 40 453 0 0 5 0 5 5 0 8 0 art_node 32 111 0 11 1 0 1 1 0 8 0 shmpl 120 1 0 0 1 0 1 1 0 8 0 dirhash 1024 17 0 0 3 0 3 3 0 8 0 dino2pl 256 1609 0 134 93 0 93 93 0 8 0 ffsino 296 1609 0 134 114 0 114 114 0 8 0 nchpl 144 1847 0 148 64 0 64 64 0 8 0 vnodes 216 1705 0 0 95 0 95 95 0 8 0 namei 1024 6280 0 6280 3 1 2 2 0 8 2 percpumem 16 51 0 0 1 0 1 1 0 8 0 kstatmem 264 27 0 0 2 0 2 2 0 8 0 scxspl 216 6908 0 6908 3 2 1 2 1 8 1 plimitpl 152 31 0 14 1 0 1 1 0 8 0 sigapl 424 465 0 420 7 0 7 7 0 8 1 knotepl 120 143 0 0 5 0 5 5 0 8 0 kqueuepl 224 28 0 18 1 0 1 1 0 8 0 pipepl 344 131 0 103 3 0 3 3 0 8 0 fdescpl 528 449 0 420 3 0 3 3 0 8 0 filepl 160 1651 0 1432 10 0 10 10 0 8 0 lockfpl 104 17 0 15 1 0 1 1 0 8 0 lockfspl 48 8 0 6 1 0 1 1 0 8 0 sessionpl 144 26 0 17 1 0 1 1 0 8 0 pgrppl 48 34 0 17 1 0 1 1 0 8 0 ucredpl 104 112 0 98 1 0 1 1 0 8 0 zombiepl 144 421 0 420 1 0 1 1 0 8 0 processpl 1240 465 0 420 5 0 5 5 0 8 0 procpl 664 507 0 457 6 0 6 6 0 8 0 sockpl 752 175 0 142 4 0 4 4 0 8 0 mcl8k 8192 2 0 0 1 0 1 1 0 8 0 mcl4k 4096 122 0 0 16 0 16 16 0 8 0 mcl2k 2048 21 0 0 3 0 3 3 0 8 0 mextrefs 64 18 0 0 1 0 1 1 0 8 0 mtagpl 96 2 0 0 1 0 1 1 0 8 0 mbufpl 256 253 0 0 16 0 16 16 0 8 0 bufpl 272 2421 0 88 156 0 156 156 0 8 0 anonpl 32 3759 0 0 31 0 31 31 0 222 0 amapchunkpl 152 8987 0 8583 19 0 19 19 0 158 0 amappl16 200 1836 0 1817 5 1 4 5 0 8 2 amappl15 192 20 0 20 2 2 0 1 0 8 0 amappl14 184 413 0 412 1 0 1 1 0 8 0 amappl13 176 118 0 105 1 0 1 1 0 8 0 amappl12 168 698 0 669 2 0 2 2 0 8 0 amappl11 160 11 0 10 1 0 1 1 0 8 0 amappl10 152 67 0 55 1 0 1 1 0 8 0 amappl9 144 282 0 282 1 1 0 1 0 8 0 amappl8 136 91 0 90 1 0 1 1 0 8 0 amappl7 128 145 0 133 1 0 1 1 0 8 0 amappl6 120 113 0 112 1 0 1 1 0 8 0 amappl5 112 241 0 232 1 0 1 1 0 8 0 amappl4 104 302 0 284 1 0 1 1 0 8 0 amappl3 96 1630 0 1528 3 0 3 3 0 8 0 amappl2 88 445 0 390 2 0 2 2 0 8 0 amappl1 80 10339 0 9748 15 0 15 15 0 8 1 amappl 88 2574 0 2434 4 0 4 4 0 92 0 uvmvnodes 80 102 0 0 3 0 3 3 0 8 0 dma4096 4096 1 0 1 1 1 0 1 0 8 0 dma1024 1024 1 0 0 1 0 1 1 0 8 0 dma256 256 6 0 6 1 1 0 1 0 8 0 dma128 128 253 0 253 1 1 0 1 0 8 0 dma64 64 6 0 6 1 1 0 1 0 8 0 dma32 32 7 0 7 1 1 0 1 0 8 0 dma16 16 18 0 17 1 0 1 1 0 8 0 aobjpl 72 3 0 0 1 0 1 1 0 8 0 uaddrrnd 24 449 0 420 1 0 1 1 0 8 0 uaddrbest 32 2 0 0 1 0 1 1 0 8 0 uaddr 24 449 0 420 1 0 1 1 0 8 0 vmmpekpl 168 5360 0 5324 2 0 2 2 0 8 0 vmmpepl 168 38568 0 36794 89 0 89 89 0 357 4 vmsppl 488 448 0 420 6 1 5 5 0 8 1 rwobjpl 80 14580 0 13595 26 1 25 25 0 8 0 pdppl 4096 906 0 840 92 18 74 86 0 8 8 pvpl 32 10189 0 0 85 1 84 84 0 265 1 pmappl 256 448 0 420 4 1 3 3 0 8 0 extentpl 40 46 0 28 1 0 1 1 0 8 0 phpool 120 428 0 24 13 0 13 13 0 8 0 ddb{0}> machine ddbcpu 0 Invalid cpu 0 ddb{0}> trace db_enter() at db_enter+0x17 sys/arch/amd64/amd64/db_interface.c:457 witness_checkorder(ffff800001b98590,9,0) at witness_checkorder+0x17f2 sys/kern/subr_witness.c:-1 rw_do_enter_write(ffff800001b98578,1) at rw_do_enter_write+0xb8 sys/kern/kern_rwlock.c:234 rrw_enter(ffff800001b98578,1) at rrw_enter+0xf5 sys/kern/kern_rwlock.c:621 VOP_LOCK(ffff800001bd3620,2001) at VOP_LOCK+0x186 sys/kern/vfs_vops.c:527 vn_lock(ffff800001bd3620,2001) at vn_lock+0xba sys/kern/vfs_vnops.c:576 vn_rdwr(0,ffff800001bd3620,ffff800011b060c0,16,0,1,0,ffff800001bd0450,0,ffffffff85117ff0) at vn_rdwr+0x160 sys/kern/vfs_vnops.c:326 vndstrategy(ffff800001bd0450) at vndstrategy+0x71c sys/dev/vnd.c:342 physio(ffffffff84136100,2902,8000,ffffffff8380fe70,ffff8000396b7320) at physio+0x549 sys/kern/kern_physio.c:162 spec_read(ffff8000396b70c0) at spec_read+0x2bf sys/kern/spec_vnops.c:215 VOP_READ(ffff8000019646c8,ffff8000396b7320,0,ffff8000001193a8) at VOP_READ+0x1f1 sys/kern/vfs_vops.c:227 vn_read(ffff800001c622c8,ffff8000396b7320,0) at vn_read+0x20e sys/kern/vfs_vnops.c:375 dofilereadv(ffff80003bb82028,3,ffff8000396b7320,0,ffff8000396b74e0) at dofilereadv+0x39e sys/kern/sys_generic.c:252 sys_read(ffff80003bb82028,ffff8000396b7580,ffff8000396b74e0) at sys_read+0x157 sys/kern/sys_generic.c:172 syscall(ffff8000396b7580) at syscall+0x118e mi_syscall sys/sys/syscall_mi.h:176 [inline] syscall(ffff8000396b7580) at syscall+0x118e sys/arch/amd64/amd64/trap.c:804 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0x476f654f4e0, count: -16 ddb{0}> machine ddbcpu 1 Stopped at x86_ipi_db+0x1f: addq $0x8,%rsp ddb{1}> trace x86_ipi_db(ffff800028f93ff0) at x86_ipi_db+0x1f sys/arch/amd64/amd64/db_interface.c:412 x86_ipi_handler() at x86_ipi_handler+0xfd sys/arch/amd64/amd64/ipi.c:106 Xresume_lapic_ipi() at Xresume_lapic_ipi+0x27 __asan_load4_noabort(ffffffff85e956b8) at __asan_load4_noabort+0x9 sys/kern/subr_kasan.c:1098 __mp_lock(ffffffff85e94ec0) at __mp_lock+0x27a __mp_lock_spin sys/kern/kern_lock.c:142 [inline] __mp_lock(ffffffff85e94ec0) at __mp_lock+0x27a sys/kern/kern_lock.c:173 ktrsysret(ffff8000298d1a08,5b,0,ffff800029a2ed40) at ktrsysret+0x19b ktrwrite2 sys/kern/kern_ktrace.c:-1 [inline] ktrsysret(ffff8000298d1a08,5b,0,ffff800029a2ed40) at ktrsysret+0x19b sys/kern/kern_ktrace.c:209 syscall(ffff800029a2ede0) at syscall+0x4b0 mi_syscall_return sys/sys/syscall_mi.h:204 [inline] syscall(ffff800029a2ede0) at syscall+0x4b0 sys/arch/amd64/amd64/trap.c:825 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0x7be1c7f9b4d0, count: -8