witness: lock order reversal: 1st 0xfffffd806f1ad060 fdlock (&newfdp->fd_fd.fd_lock) 2nd 0xfffffd80660303d8 inode (&ip->i_lock) lock order data w2 -> w1 missing lock order data w1 -> w2 missing Stopped at db_enter+0x18: addq $0x8,%rsp ddb{0}> ddb{0}> set $lines = 0 ddb{0}> set $maxwidth = 0 ddb{0}> show panic the kernel did not panic ddb{0}> trace db_enter() at db_enter+0x18 sys/arch/amd64/amd64/db_interface.c:440 witness_checkorder(fffffd80660303d8,9,0) at witness_checkorder+0x10f3 witness_debugger sys/kern/subr_witness.c:2502 [inline] witness_checkorder(fffffd80660303d8,9,0) at witness_checkorder+0x10f3 sys/kern/subr_witness.c:1105 rw_enter(fffffd80660303c8,81) at rw_enter+0xd1 sys/kern/kern_rwlock.c:250 rrw_enter(fffffd80660303c8,81) at rrw_enter+0x8b sys/kern/kern_rwlock.c:461 VOP_LOCK(fffffd8070fc3a98,81) at VOP_LOCK+0x87 sys/kern/vfs_vops.c:614 vn_lock(fffffd8070fc3a98,81) at vn_lock+0x84 sys/kern/vfs_vnops.c:579 uvn_get(fffffd8068b9c2a0,0,ffff800023467398,ffff800023467368,0,0,2645db62f56b32e2,ffffffff81cebdd0) at uvn_get+0x256 uvm_vnode_lock sys/uvm/uvm_vnode.c:1499 [inline] uvn_get(fffffd8068b9c2a0,0,ffff800023467398,ffff800023467368,0,0,2645db62f56b32e2,ffffffff81cebdd0) at uvn_get+0x256 sys/uvm/uvm_vnode.c:993 uvm_fault_lower(ffff800023467500,ffff800023467538,ffff800023467480,0) at uvm_fault_lower+0x302 sys/uvm/uvm_fault.c:1251 uvm_fault(fffffd8008592450,20000000,0,2) at uvm_fault+0x240 sys/uvm/uvm_fault.c:638 kpageflttrap(ffff800023467690,20000040) at kpageflttrap+0x1fd sys/arch/amd64/amd64/trap.c:264 kerntrap(ffff800023467690) at kerntrap+0xef sys/arch/amd64/amd64/trap.c:318 alltraps_kern_meltdown() at alltraps_kern_meltdown+0x7b copyout() at copyout+0x53 syscall(ffff8000234678c0) at syscall+0x5a9 mi_syscall sys/sys/syscall_mi.h:102 [inline] syscall(ffff8000234678c0) at syscall+0x5a9 sys/arch/amd64/amd64/trap.c:587 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0xdff2ad7f430, count: -15 ddb{0}> show registers rdi 0xffff80002125c000 rsi 0x6c2c __ALIGN_SIZE+0x5c2c rbp 0xffff800023467010 rbx 0x3 rdx 0xffff80002125c000 rcx 0x6c2b __ALIGN_SIZE+0x5c2b rax 0xffffffff816c33e7 db_enter+0x17 r8 0xffffffff81b25ebc witness_checkorder+0x10cc r9 0x5 r10 0xc1e7e5a2d5f2ceb3 r11 0xd0f9c94af3af7784 r12 0xfffffd8002dbb880 r13 0 r14 0 r15 0 rip 0xffffffff816c33e8 db_enter+0x18 cs 0x8 rflags 0x246 rsp 0xffff800023467000 ss 0x10 db_enter+0x18: addq $0x8,%rsp ddb{0}> show proc PROC (syz-executor.0) pid=130759 stat=onproc flags process=10 proc=4000000 pri=32, usrpri=86, nice=20 forw=0xffffffffffffffff, list=0xffff800021210d20,0xffff800021210550 process=0xffff8000ffff8438 user=0xffff800023462000, vmspace=0xfffffd8008592450 estcpu=36, cpticks=1, pctcpu=0.0 user=0, sys=0, intr=0 ddb{0}> ps PID TID PPID UID S FLAGS WAIT COMMAND 69828 107243 31880 32767 2 0x10 syz-executor.1 49379 197388 20761 32767 7 0x10 syz-executor.0 *49379 130759 20761 32767 7 0x4000010 syz-executor.0 20761 35576 67418 32767 3 0x90 nanoslp syz-executor.0 67418 442491 69277 0 3 0x82 wait syz-executor.0 31880 391530 5628 32767 3 0x90 nanoslp syz-executor.1 5628 283345 69277 0 3 0x82 wait syz-executor.1 87789 345367 0 0 3 0x14200 bored sosplice 69277 19599 25357 0 3 0x82 thrsleep syz-fuzzer 69277 414113 25357 0 3 0x4000082 nanoslp syz-fuzzer 69277 324468 25357 0 3 0x4000082 thrsleep syz-fuzzer 69277 447407 25357 0 3 0x4000082 thrsleep syz-fuzzer 69277 243940 25357 0 3 0x4000082 kqread syz-fuzzer 69277 248374 25357 0 3 0x4000082 thrsleep syz-fuzzer 69277 281499 25357 0 3 0x4000082 thrsleep syz-fuzzer 69277 113368 25357 0 3 0x4000082 nanoslp syz-fuzzer 25357 521518 68026 0 3 0x10008a sigsusp ksh 68026 106829 8209 0 3 0x9a select sshd 55988 200723 1 0 3 0x100083 ttyin getty 8209 182318 1 0 3 0x88 select sshd 48949 407384 77305 73 3 0x100090 kqread syslogd 77305 299915 1 0 3 0x100082 netio syslogd 36878 384476 1 0 3 0x100080 kqread resolvd 60194 370491 24648 77 3 0x100092 kqread dhcpleased 1566 71774 24648 77 3 0x100092 kqread dhcpleased 24648 293881 1 0 3 0x80 kqread dhcpleased 10014 393688 0 0 3 0x14200 bored smr 10205 430650 0 0 2 0x14200 zerothread 28074 181159 0 0 3 0x14200 aiodoned aiodoned 46832 140923 0 0 3 0x14200 syncer update 3210 192112 0 0 3 0x14200 cleaner cleaner 50644 30618 0 0 3 0x14200 reaper reaper 94244 421420 0 0 3 0x14200 pgdaemon pagedaemon 9969 175574 0 0 3 0x14200 bored crynlk 47174 191030 0 0 3 0x14200 bored crypto 98653 468108 0 0 3 0x14200 bored viomb 96227 182855 0 0 3 0x40014200 acpi0 acpi0 4510 403468 0 0 3 0x40014200 idle1 22462 418284 0 0 3 0x14200 bored softnet 5090 17284 0 0 3 0x14200 bored systqmp 36913 335385 0 0 3 0x14200 bored systq 36609 303852 0 0 3 0x40014200 bored softclock 55843 298170 0 0 3 0x40014200 idle0 1 376247 0 0 3 0x82 wait init 0 0 -1 0 3 0x10200 scheduler swapper ddb{0}> show all locks Process 49379 (syz-executor.0) thread 0xffff800021210a80 (130759) exclusive kernel_lock &kernel_lock r = 0 (0xffffffff82993e48) #0 witness_lock+0x4b0 stacktrace_save sys/sys/stacktrace.h:36 [inline] #0 witness_lock+0x4b0 sys/kern/subr_witness.c:1182 #1 uvm_fault+0x224 sys/uvm/uvm_fault.c:637 #2 kpageflttrap+0x1fd sys/arch/amd64/amd64/trap.c:264 #3 kerntrap+0xef sys/arch/amd64/amd64/trap.c:318 #4 alltraps_kern_meltdown+0x7b #5 copyout+0x53 #6 syscall+0x5a9 mi_syscall sys/sys/syscall_mi.h:102 [inline] #6 syscall+0x5a9 sys/arch/amd64/amd64/trap.c:587 #7 Xsyscall+0x128 exclusive rwlock fdlock r = 0 (0xfffffd806f1ad060) #0 witness_lock+0x4b0 stacktrace_save sys/sys/stacktrace.h:36 [inline] #0 witness_lock+0x4b0 sys/kern/subr_witness.c:1182 #1 sys_socketpair+0x20f #2 syscall+0x5a9 mi_syscall sys/sys/syscall_mi.h:102 [inline] #2 syscall+0x5a9 sys/arch/amd64/amd64/trap.c:587 #3 Xsyscall+0x128 ddb{0}> show malloc Type InUse MemUse HighUse Limit Requests Type Lim devbuf 10117 6407K 6433K 78643K 11234 0 pcb 13 8K 8K 78643K 13 0 rtable 110 3K 3K 78643K 560 0 ifaddr 39 10K 10K 78643K 105 0 counters 44 34K 34K 78643K 62 0 ioctlops 0 0K 2K 78643K 1533 0 iov 0 0K 24K 78643K 217 0 mount 1 1K 1K 78643K 1 0 log 0 0K 0K 78643K 6 0 vnodes 1216 76K 77K 78643K 1543 0 UFS quota 1 32K 32K 78643K 1 0 UFS mount 5 36K 36K 78643K 5 0 shm 2 1K 9K 78643K 53 0 VM map 2 1K 1K 78643K 2 0 sem 12 0K 0K 78643K 4946 0 dirhash 12 2K 2K 78643K 12 0 ACPI 1697 195K 286K 78643K 12598 0 file desc 8 25K 33K 78643K 10698 0 sigio 0 0K 0K 78643K 41 0 proc 56 74K 98K 78643K 839 0 subproc 34 2K 2K 78643K 187 0 NFS srvsock 1 0K 0K 78643K 1 0 NFS daemon 1 16K 16K 78643K 1 0 ip_moptions 0 0K 0K 78643K 532 0 in_multi 33 2K 2K 78643K 259 0 ether_multi 1 0K 0K 78643K 30 0 ISOFS mount 1 32K 32K 78643K 1 0 MSDOSFS mount 1 16K 16K 78643K 1 0 ttys 49 228K 228K 78643K 49 0 exec 0 0K 2K 78643K 984 0 pagedep 1 8K 8K 78643K 1 0 inodedep 1 32K 32K 78643K 1 0 newblk 1 0K 0K 78643K 1 0 VM swap 7 26K 26K 78643K 7 0 UVM amap 335 41K 41K 78643K 130426 0 UVM aobj 131 6K 6K 78643K 143 0 memdesc 1 4K 4K 78643K 1 0 crypto data 1 1K 1K 78643K 1 0 ip6_options 1 0K 0K 78643K 247 0 NDP 5 0K 0K 78643K 36 0 temp 102 4213K 4325K 78643K 24794 0 kqueue 10 14K 18K 78643K 968 0 SYN cache 2 16K 16K 78643K 2 0 ddb{0}> show all pools Name Size Requests Fail Releases Pgreq Pgrel Npage Hiwat Minpg Maxpg Idle plcache 128 22 0 0 1 0 1 1 0 8 0 rtpcb 120 186 0 183 1 0 1 1 0 8 0 rtentry 112 151 0 105 2 0 2 2 0 8 0 unpcb 120 1796 0 1777 1 0 1 1 0 8 0 syncache 296 92 0 92 9 9 0 1 0 8 0 tcpqe 32 394 0 394 10 10 0 1 0 8 0 tcpcb 736 1279 0 1271 23 22 1 6 0 8 0 arp 120 26 0 20 1 0 1 1 0 8 0 ipq 40 3 0 3 3 3 0 1 0 8 0 ipqe 40 9 0 9 3 3 0 1 0 8 0 inpcb 304 2480 0 2471 7 6 1 2 0 8 0 ip6q 72 6 0 6 1 1 0 1 0 8 0 ip6af 40 10 0 10 1 1 0 1 0 8 0 nd6 48 38 0 30 1 0 1 1 0 8 0 kcovpl 48 11 0 9 1 0 1 1 0 8 0 art_heap8 4096 1 0 0 1 0 1 1 0 8 0 art_heap4 256 643 0 423 15 1 14 14 0 8 0 art_table 32 644 0 423 2 0 2 2 0 8 0 art_node 16 150 0 108 1 0 1 1 0 8 0 sysvmsgpl 40 26 0 26 2 2 0 1 0 8 0 semapl 112 4944 0 4934 1 0 1 1 0 8 0 shmpl 112 140 0 12 4 0 4 4 0 8 0 dirhash 1024 17 0 0 3 0 3 3 0 8 0 dino2pl 256 12861 0 11438 93 3 90 90 0 8 0 ffsino 272 12861 0 11438 96 0 96 96 0 8 0 nchpl 144 26055 0 24454 61 1 60 61 0 8 0 uvmvnodes 72 5926 0 0 108 0 108 108 0 8 0 vnodes 224 5926 0 0 349 0 349 349 0 8 0 namei 1024 64376 0 64376 2 1 1 1 0 8 1 percpumem 16 43 0 9 1 0 1 1 0 8 0 scxspl 216 74678 0 74678 14 13 1 8 0 8 1 plimitpl 152 160 0 150 1 0 1 1 0 8 0 sigapl 424 10894 0 10858 7 2 5 5 0 8 1 futexpl 56 116398 0 116398 2 1 1 1 0 8 1 knotepl 112 68 0 0 2 0 2 2 0 8 0 kqueuepl 216 16654 0 16641 14 12 2 2 0 8 1 pipepl 336 545 0 535 10 8 2 2 0 8 1 fdescpl 496 10877 0 10858 3 0 3 3 0 8 0 filepl 152 44337 0 44216 7 2 5 6 0 8 0 lockfpl 104 832 0 829 1 0 1 1 0 8 0 lockfspl 48 315 0 312 1 0 1 1 0 8 0 sessionpl 144 26 0 16 1 0 1 1 0 8 0 pgrppl 48 45 0 35 1 0 1 1 0 8 0 ucredpl 96 7204 0 7192 1 0 1 1 0 8 0 zombiepl 144 10858 0 10858 1 0 1 1 0 8 1 processpl 1072 10894 0 10858 3 0 3 3 0 8 0 procpl 672 23302 0 23258 15 10 5 5 0 8 1 sosppl 168 134 0 134 6 6 0 1 0 8 0 sockpl 480 4475 0 4444 10 5 5 5 0 8 1 mcl64k 65536 15 0 0 2 0 2 2 0 8 0 mcl16k 16384 25 0 0 3 0 3 3 0 8 0 mcl12k 12288 25 0 0 2 0 2 2 0 8 0 mcl9k 9216 16 0 0 2 0 2 2 0 8 0 mcl8k 8192 17 0 0 3 0 3 3 0 8 0 mcl4k 4096 26 0 0 4 1 3 3 0 8 0 mcl2k2 2112 13 0 0 1 0 1 1 0 8 0 mcl2k 2048 472 0 0 24 0 24 24 0 8 0 mtagpl 96 2 0 0 1 0 1 1 0 8 0 mbufpl 256 997 0 0 60 0 60 60 0 8 0 bufpl 280 15917 0 9585 453 0 453 453 0 8 0 anonpl 24 2797034 0 2775662 207 67 140 141 0 186 0 amapchunkpl 152 303004 0 302391 35 10 25 26 0 158 1 amappl16 200 21574 0 20629 102 52 50 50 0 8 0 amappl15 192 907 0 907 2 2 0 1 0 8 0 amappl14 184 2847 0 2844 2 1 1 1 0 8 0 amappl13 176 2461 0 2459 1 0 1 1 0 8 0 amappl12 168 10 0 9 1 0 1 1 0 8 0 amappl11 160 54 0 43 1 0 1 1 0 8 0 amappl10 152 63 0 55 1 0 1 1 0 8 0 amappl9 144 246 0 244 1 0 1 1 0 8 0 amappl8 136 5271 0 5141 5 0 5 5 0 8 0 amappl7 128 4638 0 4621 1 0 1 1 0 8 0 amappl6 120 198 0 173 1 0 1 1 0 8 0 amappl5 112 10138 0 10120 1 0 1 1 0 8 0 amappl4 104 920 0 887 1 0 1 1 0 8 0 amappl3 96 1285 0 1284 1 0 1 1 0 8 0 amappl2 88 3930 0 3876 2 0 2 2 0 8 0 amappl1 80 187581 0 187132 13 3 10 12 0 8 0 amappl 88 127114 0 126941 6 1 5 5 0 92 0 dma4096 4096 1 0 1 1 1 0 1 0 8 0 dma1024 1024 1 0 0 1 0 1 1 0 8 0 dma256 256 6 0 6 1 1 0 1 0 8 0 dma128 128 253 0 253 1 1 0 1 0 8 0 dma64 64 6 0 6 1 1 0 1 0 8 0 dma32 32 7 0 7 1 1 0 1 0 8 0 dma16 16 18 0 17 1 0 1 1 0 8 0 aobjpl 64 142 0 12 3 0 3 3 0 8 0 uaddrrnd 24 10877 0 10858 1 0 1 1 0 8 0 uaddrbest 32 2 0 0 1 0 1 1 0 8 0 uaddr 24 10877 0 10858 1 0 1 1 0 8 0 vmmpekpl 168 48884 0 48858 2 0 2 2 0 8 0 vmmpepl 168 1256785 0 1254228 193 79 114 116 0 357 2 vmsppl 368 10876 0 10858 2 0 2 2 0 8 0 rwobjpl 56 211813 0 210052 41 15 26 26 0 8 1 pdppl 4096 21762 0 21716 74 28 46 50 0 8 0 pvpl 32 6617899 0 6592758 349 131 218 222 0 265 0 pmappl 224 10876 0 10858 2 0 2 2 0 8 0 extentpl 40 58 0 40 1 0 1 1 0 8 0 phpool 112 418 0 66 11 0 11 11 0 8 0 ddb{0}> machine ddbcpu 0 Invalid cpu 0 ddb{0}> trace db_enter() at db_enter+0x18 sys/arch/amd64/amd64/db_interface.c:440 witness_checkorder(fffffd80660303d8,9,0) at witness_checkorder+0x10f3 witness_debugger sys/kern/subr_witness.c:2502 [inline] witness_checkorder(fffffd80660303d8,9,0) at witness_checkorder+0x10f3 sys/kern/subr_witness.c:1105 rw_enter(fffffd80660303c8,81) at rw_enter+0xd1 sys/kern/kern_rwlock.c:250 rrw_enter(fffffd80660303c8,81) at rrw_enter+0x8b sys/kern/kern_rwlock.c:461 VOP_LOCK(fffffd8070fc3a98,81) at VOP_LOCK+0x87 sys/kern/vfs_vops.c:614 vn_lock(fffffd8070fc3a98,81) at vn_lock+0x84 sys/kern/vfs_vnops.c:579 uvn_get(fffffd8068b9c2a0,0,ffff800023467398,ffff800023467368,0,0,2645db62f56b32e2,ffffffff81cebdd0) at uvn_get+0x256 uvm_vnode_lock sys/uvm/uvm_vnode.c:1499 [inline] uvn_get(fffffd8068b9c2a0,0,ffff800023467398,ffff800023467368,0,0,2645db62f56b32e2,ffffffff81cebdd0) at uvn_get+0x256 sys/uvm/uvm_vnode.c:993 uvm_fault_lower(ffff800023467500,ffff800023467538,ffff800023467480,0) at uvm_fault_lower+0x302 sys/uvm/uvm_fault.c:1251 uvm_fault(fffffd8008592450,20000000,0,2) at uvm_fault+0x240 sys/uvm/uvm_fault.c:638 kpageflttrap(ffff800023467690,20000040) at kpageflttrap+0x1fd sys/arch/amd64/amd64/trap.c:264 kerntrap(ffff800023467690) at kerntrap+0xef sys/arch/amd64/amd64/trap.c:318 alltraps_kern_meltdown() at alltraps_kern_meltdown+0x7b copyout() at copyout+0x53 syscall(ffff8000234678c0) at syscall+0x5a9 mi_syscall sys/sys/syscall_mi.h:102 [inline] syscall(ffff8000234678c0) at syscall+0x5a9 sys/arch/amd64/amd64/trap.c:587 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0xdff2ad7f430, count: -15 ddb{0}> machine ddbcpu 1 Stopped at x86_ipi_db+0x1a: addq $0x8,%rsp ddb{1}> trace x86_ipi_db(ffff800020d38ff0) at x86_ipi_db+0x1a sys/arch/amd64/amd64/db_interface.c:393 x86_ipi_handler() at x86_ipi_handler+0xb7 sys/arch/amd64/amd64/ipi.c:106 Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23 __mp_lock(ffffffff82993c40) at __mp_lock+0x122 __mp_lock_spin sys/kern/kern_lock.c:116 [inline] __mp_lock(ffffffff82993c40) at __mp_lock+0x122 sys/kern/kern_lock.c:147 __mp_acquire_count(ffffffff82993c40,1) at __mp_acquire_count+0x4c sys/kern/kern_lock.c:227 mi_switch() at mi_switch+0x3d3 sys/kern/sched_bsd.c:433 sleep_finish(ffff8000234613e8,1) at sleep_finish+0x1b2 sys/kern/kern_synch.c:433 rw_enter(fffffd8008592458,21) at rw_enter+0x35b sys/kern/kern_rwlock.c:286 vm_map_lock_ln(fffffd8008592450,fffffd80712474f0,0) at vm_map_lock_ln+0xda sys/uvm/uvm_map.c:5425 uvm_mapanon(fffffd8008592450,ffff800023461690,21000,1000,2080713) at uvm_mapanon+0x22f sys/uvm/uvm_map.c:1007 sys_mmap(ffff800021210d20,ffff800023461738,ffff800023461780) at sys_mmap+0x991 uvm_mmapanon sys/uvm/uvm_mmap.c:954 [inline] sys_mmap(ffff800021210d20,ffff800023461738,ffff800023461780) at sys_mmap+0x991 sys/uvm/uvm_mmap.c:425 syscall(ffff800023461800) at syscall+0x5a9 mi_syscall sys/sys/syscall_mi.h:102 [inline] syscall(ffff800023461800) at syscall+0x5a9 sys/arch/amd64/amd64/trap.c:587 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0x7f7fffff7520, count: -13