witness: lock order reversal: 1st 0xfffffd806fd4d440 fdlock (&newfdp->fd_fd.fd_lock) 2nd 0xfffffd806abf53c8 inode (&ip->i_lock) lock order data w2 -> w1 missing lock order data w1 -> w2 missing Stopped at db_enter+0x18: addq $0x8,%rsp ddb{1}> ddb{1}> set $lines = 0 ddb{1}> set $maxwidth = 0 ddb{1}> show panic the kernel did not panic ddb{1}> trace db_enter() at db_enter+0x18 sys/arch/amd64/amd64/db_interface.c:440 witness_checkorder(fffffd806abf53c8,9,0) at witness_checkorder+0x10f3 witness_debugger sys/kern/subr_witness.c:2502 [inline] witness_checkorder(fffffd806abf53c8,9,0) at witness_checkorder+0x10f3 sys/kern/subr_witness.c:1105 rw_enter(fffffd806abf53b8,81) at rw_enter+0xd1 sys/kern/kern_rwlock.c:250 rrw_enter(fffffd806abf53b8,81) at rrw_enter+0x8b sys/kern/kern_rwlock.c:461 VOP_LOCK(fffffd80790c2268,81) at VOP_LOCK+0x87 sys/kern/vfs_vops.c:614 vn_lock(fffffd80790c2268,81) at vn_lock+0x84 sys/kern/vfs_vnops.c:579 uvn_get(fffffd80636ceca8,0,ffff8000256c37e8,ffff8000256c37b8,0,0,f80a701f8b557bbc,ffffffff8121fe60) at uvn_get+0x256 uvm_vnode_lock sys/uvm/uvm_vnode.c:1499 [inline] uvn_get(fffffd80636ceca8,0,ffff8000256c37e8,ffff8000256c37b8,0,0,f80a701f8b557bbc,ffffffff8121fe60) at uvn_get+0x256 sys/uvm/uvm_vnode.c:993 uvm_fault_lower(ffff8000256c3950,ffff8000256c3988,ffff8000256c38d0,0) at uvm_fault_lower+0x302 sys/uvm/uvm_fault.c:1251 uvm_fault(fffffd800858e2e0,20000000,0,2) at uvm_fault+0x240 sys/uvm/uvm_fault.c:638 kpageflttrap(ffff8000256c3ae0,20000180) at kpageflttrap+0x1fd sys/arch/amd64/amd64/trap.c:264 kerntrap(ffff8000256c3ae0) at kerntrap+0xef sys/arch/amd64/amd64/trap.c:318 alltraps_kern_meltdown() at alltraps_kern_meltdown+0x7b copyout() at copyout+0x53 syscall(ffff8000256c3d20) at syscall+0x5a9 mi_syscall sys/sys/syscall_mi.h:102 [inline] syscall(ffff8000256c3d20) at syscall+0x5a9 sys/arch/amd64/amd64/trap.c:587 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0x9bf589b5cd0, count: -15 ddb{1}> show registers rdi 0xffff8000238b2000 rsi 0x744d __ALIGN_SIZE+0x644d rbp 0xffff8000256c3460 rbx 0x3 rdx 0xffff8000238b2000 rcx 0x744c __ALIGN_SIZE+0x644c rax 0xffffffff81cc95e7 db_enter+0x17 r8 0xffffffff8129221c witness_checkorder+0x10cc r9 0x5 r10 0x5217ad979bed7909 r11 0xfe1a7f91dcdbe8ca r12 0xfffffd8002db7840 r13 0 r14 0 r15 0 rip 0xffffffff81cc95e8 db_enter+0x18 cs 0x8 rflags 0x246 rsp 0xffff8000256c3450 ss 0x10 db_enter+0x18: addq $0x8,%rsp ddb{1}> show proc PROC (syz-executor.1) pid=402849 stat=onproc flags process=10 proc=4000000 pri=32, usrpri=75, nice=20 forw=0xffffffffffffffff, list=0xffff800021211ce0,0xffff800021211510 process=0xffff8000ffff8008 user=0xffff8000256be000, vmspace=0xfffffd800858e2e0 estcpu=25, cpticks=0, pctcpu=0.0 user=0, sys=0, intr=0 ddb{1}> ps PID TID PPID UID S FLAGS WAIT COMMAND 67409 205531 93677 32767 7 0x10 syz-executor.1 67409 234428 93677 32767 3 0x4000090 poll syz-executor.1 67409 217384 93677 32767 2 0x4000010 syz-executor.1 *67409 402849 93677 32767 7 0x4000010 syz-executor.1 67409 98692 93677 32767 3 0x4000090 fsleep syz-executor.1 41387 227221 90295 32767 3 0x90 piperd syz-executor.0 90295 446430 26001 0 3 0x82 wait syz-executor.0 93677 487598 50680 32767 3 0x90 nanoslp syz-executor.1 50680 360944 26001 0 3 0x82 wait syz-executor.1 16506 14970 0 0 3 0x14200 bored sosplice 26001 492623 65103 0 3 0x82 thrsleep syz-fuzzer 26001 428265 65103 0 3 0x4000082 thrsleep syz-fuzzer 26001 228211 65103 0 3 0x4000082 thrsleep syz-fuzzer 26001 186270 65103 0 3 0x4000082 thrsleep syz-fuzzer 26001 51747 65103 0 3 0x4000082 thrsleep syz-fuzzer 26001 236016 65103 0 3 0x4000082 thrsleep syz-fuzzer 26001 105790 65103 0 3 0x4000082 thrsleep syz-fuzzer 26001 95773 65103 0 3 0x4000082 kqread syz-fuzzer 65103 436218 756 0 3 0x10008a sigsusp ksh 756 455393 12248 0 3 0x9a select sshd 53562 404512 1 0 3 0x100083 ttyin getty 12248 435626 1 0 3 0x88 select sshd 92063 375477 62228 73 3 0x100090 kqread syslogd 62228 196584 1 0 3 0x100082 netio syslogd 16203 5370 1 0 3 0x100080 kqread resolvd 26892 53274 87469 77 3 0x100092 kqread dhcpleased 9976 150159 87469 77 3 0x100092 kqread dhcpleased 87469 487026 1 0 3 0x80 kqread dhcpleased 22822 307197 0 0 3 0x14200 bored smr 8884 437274 0 0 3 0x14200 pgzero zerothread 98354 81476 0 0 3 0x14200 aiodoned aiodoned 23809 458402 0 0 3 0x14200 syncer update 42272 299587 0 0 3 0x14200 cleaner cleaner 6424 162502 0 0 3 0x14200 reaper reaper 23362 234833 0 0 3 0x14200 pgdaemon pagedaemon 50475 200170 0 0 3 0x14200 bored crynlk 30782 100989 0 0 3 0x14200 bored crypto 48040 170416 0 0 3 0x14200 bored viomb 19839 194368 0 0 3 0x40014200 acpi0 acpi0 35157 230146 0 0 3 0x40014200 idle1 37468 446271 0 0 3 0x14200 bored softnet 10661 112335 0 0 3 0x14200 bored systqmp 1247 64373 0 0 3 0x14200 bored systq 85872 374715 0 0 3 0x40014200 bored softclock 16228 423369 0 0 3 0x40014200 idle0 1 143101 0 0 3 0x82 wait init 0 0 -1 0 3 0x10200 scheduler swapper ddb{1}> show all locks Process 67409 (syz-executor.1) thread 0xffff800021210d20 (402849) exclusive kernel_lock &kernel_lock r = 0 (0xffffffff82875630) #0 witness_lock+0x4b0 stacktrace_save sys/sys/stacktrace.h:36 [inline] #0 witness_lock+0x4b0 sys/kern/subr_witness.c:1182 #1 uvm_fault+0x224 sys/uvm/uvm_fault.c:637 #2 kpageflttrap+0x1fd sys/arch/amd64/amd64/trap.c:264 #3 kerntrap+0xef sys/arch/amd64/amd64/trap.c:318 #4 alltraps_kern_meltdown+0x7b #5 copyout+0x53 #6 syscall+0x5a9 mi_syscall sys/sys/syscall_mi.h:102 [inline] #6 syscall+0x5a9 sys/arch/amd64/amd64/trap.c:587 #7 Xsyscall+0x128 exclusive rwlock fdlock r = 0 (0xfffffd806fd4d440) #0 witness_lock+0x4b0 stacktrace_save sys/sys/stacktrace.h:36 [inline] #0 witness_lock+0x4b0 sys/kern/subr_witness.c:1182 #1 dopipe+0xd6 #2 syscall+0x5a9 mi_syscall sys/sys/syscall_mi.h:102 [inline] #2 syscall+0x5a9 sys/arch/amd64/amd64/trap.c:587 #3 Xsyscall+0x128 ddb{1}> show malloc Type InUse MemUse HighUse Limit Requests Type Lim devbuf 10117 6407K 6433K 78643K 11237 0 pcb 13 8K 8K 78643K 13 0 rtable 108 3K 3K 78643K 598 0 ifaddr 39 10K 10K 78643K 110 0 counters 44 34K 34K 78643K 64 0 ioctlops 0 0K 2K 78643K 123 0 iov 0 0K 16K 78643K 237 0 mount 1 1K 1K 78643K 1 0 log 0 0K 0K 78643K 6 0 vnodes 1216 76K 77K 78643K 1463 0 UFS quota 1 32K 32K 78643K 1 0 UFS mount 5 36K 36K 78643K 5 0 shm 2 1K 5K 78643K 25 0 VM map 2 1K 1K 78643K 2 0 sem 12 0K 0K 78643K 5384 0 dirhash 12 2K 2K 78643K 12 0 ACPI 1697 195K 286K 78643K 12598 0 file desc 7 21K 33K 78643K 7285 0 sigio 0 0K 0K 78643K 53 0 proc 56 74K 99K 78643K 695 0 subproc 34 2K 2K 78643K 204 0 NFS srvsock 1 0K 0K 78643K 1 0 NFS daemon 1 16K 16K 78643K 1 0 ip_moptions 0 0K 0K 78643K 479 0 in_multi 33 2K 2K 78643K 346 0 ether_multi 1 0K 0K 78643K 59 0 mrt 1 0K 0K 78643K 1 0 ISOFS mount 1 32K 32K 78643K 1 0 MSDOSFS mount 1 16K 16K 78643K 1 0 ttys 49 228K 228K 78643K 49 0 exec 0 0K 2K 78643K 1059 0 pagedep 1 8K 8K 78643K 1 0 inodedep 1 32K 32K 78643K 1 0 newblk 1 0K 0K 78643K 1 0 VM swap 7 26K 26K 78643K 7 0 UVM amap 384 50K 66K 78643K 91427 0 UVM aobj 56 6K 6K 78643K 89 0 memdesc 1 4K 4K 78643K 1 0 crypto data 1 1K 1K 78643K 1 0 ip6_options 0 0K 0K 78643K 200 0 NDP 5 0K 0K 78643K 39 0 temp 102 4213K 4321K 78643K 18084 0 kqueue 10 14K 19K 78643K 219 0 SYN cache 2 16K 16K 78643K 2 0 ddb{1}> show all pools Name Size Requests Fail Releases Pgreq Pgrel Npage Hiwat Minpg Maxpg Idle plcache 128 22 0 0 1 0 1 1 0 8 0 rtpcb 120 241 0 238 1 0 1 1 0 8 0 rtentry 112 159 0 114 2 0 2 2 0 8 0 unpcb 120 2111 0 2089 3 2 1 2 0 8 0 syncache 296 76 0 76 7 7 0 1 0 8 0 tcpqe 32 562 0 562 9 9 0 2 0 8 0 tcpcb 736 1360 0 1356 15 12 3 5 0 8 2 arp 120 26 0 20 1 0 1 1 0 8 0 ipq 40 10 0 10 3 3 0 1 0 8 0 ipqe 40 356 0 356 3 3 0 1 0 8 0 inpcb 304 2517 0 2510 6 5 1 3 0 8 0 ip6q 72 1 0 1 1 1 0 1 0 8 0 ip6af 40 2 0 2 1 1 0 1 0 8 0 nd6 48 40 0 33 1 0 1 1 0 8 0 kcovpl 48 12 0 10 1 0 1 1 0 8 0 art_heap8 4096 1 0 0 1 0 1 1 0 8 0 art_heap4 256 693 0 503 16 4 12 14 0 8 0 art_table 32 694 0 503 2 0 2 2 0 8 0 art_node 16 158 0 117 1 0 1 1 0 8 0 sysvmsgpl 40 10 0 0 1 0 1 1 0 8 0 semupl 112 9 0 9 1 1 0 1 0 8 0 semapl 112 5382 0 5372 1 0 1 1 0 8 0 shmpl 112 86 0 33 2 0 2 2 0 8 0 dirhash 1024 17 0 0 3 0 3 3 0 8 0 dino2pl 256 9405 0 7988 90 1 89 89 0 8 0 ffsino 272 9405 0 7988 95 0 95 95 0 8 0 nchpl 144 19355 0 17754 61 1 60 61 0 8 0 uvmvnodes 72 5926 0 0 108 0 108 108 0 8 0 vnodes 224 5926 0 0 349 0 349 349 0 8 0 namei 1024 44774 0 44774 1 0 1 1 0 8 1 percpumem 16 44 0 10 1 0 1 1 0 8 0 scxspl 216 53751 0 53751 19 18 1 6 0 8 1 plimitpl 152 197 0 187 1 0 1 1 0 8 0 sigapl 424 7478 0 7443 5 0 5 5 0 8 0 futexpl 56 102882 0 102881 1 0 1 1 0 8 0 knotepl 112 68 0 0 2 0 2 2 0 8 0 kqueuepl 216 10446 0 10431 8 6 2 2 0 8 1 pipepl 336 716 0 704 9 7 2 2 0 8 0 fdescpl 496 7461 0 7443 3 0 3 3 0 8 0 filepl 152 31050 0 30935 8 3 5 6 0 8 0 lockfpl 104 854 0 851 1 0 1 1 0 8 0 lockfspl 48 340 0 337 1 0 1 1 0 8 0 sessionpl 144 27 0 17 1 0 1 1 0 8 0 pgrppl 48 64 0 54 1 0 1 1 0 8 0 ucredpl 96 2104 0 2092 1 0 1 1 0 8 0 zombiepl 144 7443 0 7443 1 0 1 1 0 8 1 processpl 1072 7478 0 7443 3 0 3 3 0 8 0 procpl 672 17006 0 16960 10 6 4 5 0 8 0 sosppl 168 84 0 84 8 8 0 1 0 8 0 sockpl 480 4897 0 4865 13 8 5 10 0 8 0 mcl64k 65536 21 0 0 3 0 3 3 0 8 0 mcl16k 16384 17 0 0 3 0 3 3 0 8 0 mcl12k 12288 17 0 0 2 0 2 2 0 8 0 mcl9k 9216 10 0 0 1 0 1 1 0 8 0 mcl8k 8192 25 0 0 4 1 3 3 0 8 0 mcl4k 4096 33 0 0 4 1 3 3 0 8 0 mcl2k2 2112 5 0 0 1 0 1 1 0 8 0 mcl2k 2048 519 0 0 42 0 42 42 0 8 0 mtagpl 96 2 0 0 1 0 1 1 0 8 0 mbufpl 256 1084 0 0 56 0 56 56 0 8 0 bufpl 280 14648 0 8316 453 0 453 453 0 8 0 anonpl 24 1974185 0 1948661 209 36 173 185 0 186 0 amapchunkpl 152 214654 0 213809 54 20 34 41 0 158 0 amappl16 200 14694 0 13613 89 31 58 70 0 8 0 amappl15 192 803 0 803 3 3 0 1 0 8 0 amappl14 184 14 0 10 1 0 1 1 0 8 0 amappl13 176 348 0 346 1 0 1 1 0 8 0 amappl12 168 1933 0 1928 1 0 1 1 0 8 0 amappl11 160 47 0 37 1 0 1 1 0 8 0 amappl10 152 56 0 48 1 0 1 1 0 8 0 amappl9 144 4159 0 4153 1 0 1 1 0 8 0 amappl8 136 1056 0 926 5 0 5 5 0 8 0 amappl7 128 468 0 454 1 0 1 1 0 8 0 amappl6 120 4112 0 4095 1 0 1 1 0 8 0 amappl5 112 5978 0 5961 1 0 1 1 0 8 0 amappl4 104 1002 0 961 2 0 2 2 0 8 0 amappl3 96 2530 0 2527 1 0 1 1 0 8 0 amappl2 88 2366 0 2310 2 0 2 2 0 8 0 amappl1 80 130451 0 130016 12 2 10 12 0 8 0 amappl 88 88542 0 88313 6 0 6 6 0 92 0 dma4096 4096 1 0 1 1 1 0 1 0 8 0 dma1024 1024 1 0 0 1 0 1 1 0 8 0 dma256 256 6 0 6 1 1 0 1 0 8 0 dma128 128 253 0 253 1 1 0 1 0 8 0 dma64 64 6 0 6 1 1 0 1 0 8 0 dma32 32 7 0 7 1 1 0 1 0 8 0 dma16 16 18 0 17 1 0 1 1 0 8 0 aobjpl 64 88 0 33 1 0 1 1 0 8 0 uaddrrnd 24 7461 0 7443 1 0 1 1 0 8 0 uaddrbest 32 2 0 0 1 0 1 1 0 8 0 uaddr 24 7461 0 7443 1 0 1 1 0 8 0 vmmpekpl 168 42324 0 42302 2 0 2 2 0 8 0 vmmpepl 168 866554 0 863883 256 135 121 170 0 357 0 vmsppl 368 7460 0 7443 2 0 2 2 0 8 0 rwobjpl 56 150250 0 148306 35 7 28 31 0 8 0 pdppl 4096 14930 0 14886 64 18 46 50 0 8 2 pvpl 32 4844285 0 4815071 333 73 260 285 0 265 0 pmappl 224 7460 0 7443 2 0 2 2 0 8 0 extentpl 40 58 0 40 1 0 1 1 0 8 0 phpool 112 411 0 46 11 0 11 11 0 8 0 ddb{1}> machine ddbcpu 0 Stopped at x86_ipi_db+0x1a: addq $0x8,%rsp ddb{0}> trace x86_ipi_db(ffffffff8281eff0) at x86_ipi_db+0x1a sys/arch/amd64/amd64/db_interface.c:393 x86_ipi_handler() at x86_ipi_handler+0xb7 sys/arch/amd64/amd64/ipi.c:106 Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23 __mp_lock(ffffffff82875428) at __mp_lock+0x122 __mp_lock_spin sys/kern/kern_lock.c:116 [inline] __mp_lock(ffffffff82875428) at __mp_lock+0x122 sys/kern/kern_lock.c:147 softintr_dispatch(0) at softintr_dispatch+0x4e sys/arch/amd64/amd64/softintr.c:90 Xsoftclock() at Xsoftclock+0x1f end of kernel end trace frame: 0x7f7ffffc16c0, count: -6 ddb{0}> machine ddbcpu 1 Stopped at db_enter+0x18: addq $0x8,%rsp ddb{1}> trace db_enter() at db_enter+0x18 sys/arch/amd64/amd64/db_interface.c:440 witness_checkorder(fffffd806abf53c8,9,0) at witness_checkorder+0x10f3 witness_debugger sys/kern/subr_witness.c:2502 [inline] witness_checkorder(fffffd806abf53c8,9,0) at witness_checkorder+0x10f3 sys/kern/subr_witness.c:1105 rw_enter(fffffd806abf53b8,81) at rw_enter+0xd1 sys/kern/kern_rwlock.c:250 rrw_enter(fffffd806abf53b8,81) at rrw_enter+0x8b sys/kern/kern_rwlock.c:461 VOP_LOCK(fffffd80790c2268,81) at VOP_LOCK+0x87 sys/kern/vfs_vops.c:614 vn_lock(fffffd80790c2268,81) at vn_lock+0x84 sys/kern/vfs_vnops.c:579 uvn_get(fffffd80636ceca8,0,ffff8000256c37e8,ffff8000256c37b8,0,0,f80a701f8b557bbc,ffffffff8121fe60) at uvn_get+0x256 uvm_vnode_lock sys/uvm/uvm_vnode.c:1499 [inline] uvn_get(fffffd80636ceca8,0,ffff8000256c37e8,ffff8000256c37b8,0,0,f80a701f8b557bbc,ffffffff8121fe60) at uvn_get+0x256 sys/uvm/uvm_vnode.c:993 uvm_fault_lower(ffff8000256c3950,ffff8000256c3988,ffff8000256c38d0,0) at uvm_fault_lower+0x302 sys/uvm/uvm_fault.c:1251 uvm_fault(fffffd800858e2e0,20000000,0,2) at uvm_fault+0x240 sys/uvm/uvm_fault.c:638 kpageflttrap(ffff8000256c3ae0,20000180) at kpageflttrap+0x1fd sys/arch/amd64/amd64/trap.c:264 kerntrap(ffff8000256c3ae0) at kerntrap+0xef sys/arch/amd64/amd64/trap.c:318 alltraps_kern_meltdown() at alltraps_kern_meltdown+0x7b copyout() at copyout+0x53 syscall(ffff8000256c3d20) at syscall+0x5a9 mi_syscall sys/sys/syscall_mi.h:102 [inline] syscall(ffff8000256c3d20) at syscall+0x5a9 sys/arch/amd64/amd64/trap.c:587 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0x9bf589b5cd0, count: -15