witness: lock order reversal: 1st 0xfffffd806f1fa820 fdlock (&newfdp->fd_fd.fd_lock) 2nd 0xfffffd806a0b6b38 inode (&ip->i_lock) lock order data w2 -> w1 missing lock order data w1 -> w2 missing Stopped at db_enter+0x18: addq $0x8,%rsp ddb{0}> ddb{0}> set $lines = 0 ddb{0}> set $maxwidth = 0 ddb{0}> show panic the kernel did not panic ddb{0}> trace db_enter() at db_enter+0x18 sys/arch/amd64/amd64/db_interface.c:440 witness_checkorder(fffffd806a0b6b38,9,0) at witness_checkorder+0x10f3 witness_debugger sys/kern/subr_witness.c:2502 [inline] witness_checkorder(fffffd806a0b6b38,9,0) at witness_checkorder+0x10f3 sys/kern/subr_witness.c:1105 rw_enter(fffffd806a0b6b28,81) at rw_enter+0xd1 sys/kern/kern_rwlock.c:250 rrw_enter(fffffd806a0b6b28,81) at rrw_enter+0x8b sys/kern/kern_rwlock.c:461 VOP_LOCK(fffffd807190b788,81) at VOP_LOCK+0x87 sys/kern/vfs_vops.c:614 vn_lock(fffffd807190b788,81) at vn_lock+0x84 sys/kern/vfs_vnops.c:579 uvn_get(fffffd806f162120,8000000000000000,ffff800023b54ec8,ffff800023b54e98,0,0) at uvn_get+0x256 uvm_vnode_lock sys/uvm/uvm_vnode.c:1499 [inline] uvn_get(fffffd806f162120,8000000000000000,ffff800023b54ec8,ffff800023b54e98,0,0) at uvn_get+0x256 sys/uvm/uvm_vnode.c:993 uvm_fault_lower(ffff800023b55030,ffff800023b55068,ffff800023b54fb0,0) at uvm_fault_lower+0x302 sys/uvm/uvm_fault.c:1251 uvm_fault(fffffd80694118b0,20000000,0,2) at uvm_fault+0x240 sys/uvm/uvm_fault.c:638 kpageflttrap(ffff800023b551c0,20000000) at kpageflttrap+0x1fd sys/arch/amd64/amd64/trap.c:264 kerntrap(ffff800023b551c0) at kerntrap+0xef sys/arch/amd64/amd64/trap.c:318 alltraps_kern_meltdown() at alltraps_kern_meltdown+0x7b copyout() at copyout+0x53 syscall(ffff800023b55400) at syscall+0x5a9 mi_syscall sys/sys/syscall_mi.h:102 [inline] syscall(ffff800023b55400) at syscall+0x5a9 sys/arch/amd64/amd64/trap.c:587 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0x47e5a9216c0, count: -15 ddb{0}> show registers rdi 0xffff800023344000 rsi 0x8b7f __ALIGN_SIZE+0x7b7f rbp 0xffff800023b54b40 rbx 0x3 rdx 0xffff800023344000 rcx 0x8b7e __ALIGN_SIZE+0x7b7e rax 0xffffffff8230ccb7 db_enter+0x17 r8 0xffffffff81afeecc witness_checkorder+0x10cc r9 0x5 r10 0xb141049b72d62b1a r11 0xa0ad6e5fea4c2fd r12 0xfffffd8002daf800 r13 0 r14 0 r15 0 rip 0xffffffff8230ccb8 db_enter+0x18 cs 0x8 rflags 0x246 rsp 0xffff800023b54b30 ss 0x10 db_enter+0x18: addq $0x8,%rsp ddb{0}> show proc PROC (syz-executor.1) pid=18873 stat=onproc flags process=0 proc=4000000 pri=32, usrpri=79, nice=20 forw=0xffffffffffffffff, list=0xffff8000212987e8,0xffffffff82960ef0 process=0xffff800021208870 user=0xffff800023b50000, vmspace=0xfffffd80694118b0 estcpu=29, cpticks=0, pctcpu=0.0 user=0, sys=0, intr=0 ddb{0}> ps PID TID PPID UID S FLAGS WAIT COMMAND 82046 479678 3216 0 2 0 syz-executor.1 82046 255791 3216 0 3 0x4000080 poll syz-executor.1 *82046 18873 3216 0 7 0x4000000 syz-executor.1 3216 514088 63958 0 3 0x82 nanoslp syz-executor.1 37372 76779 63958 0 3 0x82 nanoslp syz-executor.0 53522 392755 1 0 3 0x100083 ttyin getty 64939 87743 0 0 3 0x14280 nfsidl nfsio 84183 53235 0 0 3 0x14280 nfsidl nfsio 1836 452555 0 0 3 0x14280 nfsidl nfsio 98921 59870 0 0 3 0x14280 nfsidl nfsio 17320 147713 0 0 3 0x14280 nfsidl nfsio 75002 436934 0 0 3 0x14280 nfsidl nfsio 49489 522816 0 0 3 0x14280 nfsidl nfsio 78377 271196 0 0 3 0x14280 nfsidl nfsio 75476 8578 0 0 3 0x14280 nfsidl nfsio 38786 332453 0 0 3 0x14280 nfsidl nfsio 6395 229423 0 0 3 0x14280 nfsidl nfsio 41485 440133 0 0 3 0x14280 nfsidl nfsio 66590 400647 0 0 3 0x14280 nfsidl nfsio 34059 104758 0 0 3 0x14280 nfsidl nfsio 98760 229727 0 0 3 0x14280 nfsidl nfsio 74596 130835 0 0 3 0x14280 nfsidl nfsio 21811 80956 0 0 3 0x14280 nfsidl nfsio 5886 417997 0 0 3 0x14280 nfsidl nfsio 10438 237777 0 0 3 0x14280 nfsidl nfsio 47699 188310 0 0 3 0x14280 nfsidl nfsio 39055 342240 0 0 3 0x14200 bored sosplice 63958 324378 224 0 3 0x82 thrsleep syz-fuzzer 63958 44789 224 0 3 0x4000082 thrsleep syz-fuzzer 63958 190659 224 0 3 0x4000082 thrsleep syz-fuzzer 63958 130372 224 0 3 0x4000082 thrsleep syz-fuzzer 63958 302270 224 0 3 0x4000082 thrsleep syz-fuzzer 63958 187240 224 0 3 0x4000082 kqread syz-fuzzer 63958 292573 224 0 3 0x4000082 thrsleep syz-fuzzer 63958 19212 224 0 3 0x4000082 thrsleep syz-fuzzer 224 289519 53602 0 3 0x10008a sigsusp ksh 53602 220670 20879 0 3 0x9a select sshd 20879 491874 1 0 3 0x88 select sshd 1847 401687 77042 74 3 0x100092 bpf pflogd 77042 331235 1 0 3 0x80 netio pflogd 3145 362628 10257 73 3 0x100090 kqread syslogd 10257 124838 1 0 3 0x100082 netio syslogd 55242 435953 1 0 3 0x100080 kqread resolvd 32816 408681 25470 77 3 0x100092 kqread dhcpleased 10398 424430 25470 77 3 0x100092 kqread dhcpleased 25470 26542 1 0 3 0x80 kqread dhcpleased 62406 313476 0 0 3 0x14200 bored smr 63984 214664 0 0 2 0x14200 zerothread 94409 327753 0 0 3 0x14200 aiodoned aiodoned 64766 234497 0 0 3 0x14200 syncer update 67065 482298 0 0 3 0x14200 cleaner cleaner 72977 339889 0 0 3 0x14200 reaper reaper 34940 252020 0 0 3 0x14200 pgdaemon pagedaemon 43211 146743 0 0 3 0x14200 bored crynlk 93155 395645 0 0 3 0x14200 bored crypto 37574 278260 0 0 3 0x14200 bored viomb 90354 121692 0 0 3 0x40014200 acpi0 acpi0 1549 134278 0 0 7 0x40014200 idle1 79755 455915 0 0 3 0x14200 bored softnet 27133 340989 0 0 3 0x14200 bored systqmp 67555 333161 0 0 3 0x14200 bored systq 6469 294305 0 0 3 0x40014200 bored softclock 10218 433748 0 0 3 0x40014200 idle0 1 110596 0 0 3 0x82 wait init 0 0 -1 0 3 0x10200 scheduler swapper ddb{0}> show all locks Process 82046 (syz-executor.1) thread 0xffff800021298a88 (18873) exclusive kernel_lock &kernel_lock r = 0 (0xffffffff8283ff70) #0 witness_lock+0x4b0 stacktrace_save sys/sys/stacktrace.h:36 [inline] #0 witness_lock+0x4b0 sys/kern/subr_witness.c:1182 #1 uvm_fault+0x224 sys/uvm/uvm_fault.c:637 #2 kpageflttrap+0x1fd sys/arch/amd64/amd64/trap.c:264 #3 kerntrap+0xef sys/arch/amd64/amd64/trap.c:318 #4 alltraps_kern_meltdown+0x7b #5 copyout+0x53 #6 syscall+0x5a9 mi_syscall sys/sys/syscall_mi.h:102 [inline] #6 syscall+0x5a9 sys/arch/amd64/amd64/trap.c:587 #7 Xsyscall+0x128 exclusive rwlock fdlock r = 0 (0xfffffd806f1fa820) #0 witness_lock+0x4b0 stacktrace_save sys/sys/stacktrace.h:36 [inline] #0 witness_lock+0x4b0 sys/kern/subr_witness.c:1182 #1 dopipe+0xd6 #2 syscall+0x5a9 mi_syscall sys/sys/syscall_mi.h:102 [inline] #2 syscall+0x5a9 sys/arch/amd64/amd64/trap.c:587 #3 Xsyscall+0x128 ddb{0}> show malloc Type InUse MemUse HighUse Limit Requests Type Lim devbuf 10196 6575K 7075K 78643K 17800 0 pcb 13 8K 8K 78643K 443 0 rtable 104 3K 5K 78643K 779 0 ifaddr 73 14K 15K 78643K 316 0 sysctl 2 0K 0K 78643K 2 0 counters 44 34K 34K 78643K 122 0 ioctlops 0 0K 4K 78643K 1808 0 iov 0 0K 32K 78643K 249 0 mount 1 1K 1K 78643K 1 0 log 0 0K 0K 78643K 6 0 vnodes 1223 77K 77K 78643K 3393 0 UFS quota 1 32K 32K 78643K 1 0 UFS mount 5 36K 36K 78643K 5 0 shm 2 1K 9K 78643K 63 0 VM map 2 1K 1K 78643K 2 0 sem 12 0K 0K 78643K 1049 0 dirhash 12 2K 2K 78643K 12 0 ACPI 1697 195K 286K 78643K 12598 0 file desc 5 13K 25K 78643K 8473 0 sigio 0 0K 0K 78643K 104 0 proc 70 87K 111K 78643K 850 0 subproc 32 2K 2K 78643K 170 0 NFS srvsock 1 0K 0K 78643K 1 0 NFS daemon 1 16K 16K 78643K 1 0 ip_moptions 0 0K 0K 78643K 426 0 in_multi 29 1K 2K 78643K 338 0 ether_multi 1 0K 0K 78643K 63 0 mrt 0 0K 0K 78643K 43 0 ISOFS mount 1 32K 32K 78643K 1 0 MSDOSFS mount 1 16K 16K 78643K 1 0 ttys 73 334K 334K 78643K 73 0 exec 0 0K 2K 78643K 834 0 pfkey data 0 0K 0K 78643K 5 0 pagedep 1 8K 8K 78643K 1 0 inodedep 1 32K 32K 78643K 1 0 newblk 1 0K 0K 78643K 1 0 VM swap 7 26K 26K 78643K 7 0 UVM amap 299 98K 106K 78643K 102293 0 UVM aobj 50 7K 7K 78643K 58 0 memdesc 1 4K 4K 78643K 1 0 crypto data 1 1K 1K 78643K 1 0 ip6_options 0 0K 0K 78643K 235 0 NDP 13 0K 0K 78643K 87 0 temp 134 4206K 4318K 78643K 59101 0 kqueue 10 14K 34K 78643K 430 0 SYN cache 2 16K 16K 78643K 2 0 ddb{0}> show all pools Name Size Requests Fail Releases Pgreq Pgrel Npage Hiwat Minpg Maxpg Idle plcache 128 22 0 0 1 0 1 1 0 8 0 rtpcb 120 380 0 377 1 0 1 1 0 8 0 rtentry 112 181 0 140 2 0 2 2 0 8 0 unpcb 120 1491 0 1469 2 1 1 2 0 8 0 syncache 296 37 0 37 5 5 0 1 0 8 0 tcpqe 32 29 0 29 4 4 0 1 0 8 0 tcpcb 736 1020 0 1016 23 22 1 5 0 8 0 arp 120 24 0 18 1 0 1 1 0 8 0 inpcb 304 3104 0 3097 16 15 1 2 0 8 0 rttmr 72 12 0 12 3 3 0 1 0 8 0 nd6 48 47 0 43 1 0 1 1 0 8 0 pkpcb 40 32 0 32 5 5 0 1 0 8 0 kcovpl 48 10 0 8 1 0 1 1 0 8 0 ppxss 1248 22 0 22 7 7 0 1 0 8 0 pffrag 232 2 0 2 1 1 0 1 0 482 0 pffrnode 88 2 0 2 1 1 0 1 0 8 0 pffrent 40 14 0 14 3 3 0 1 0 8 0 pfosfp 40 1428 0 1005 5 0 5 5 0 8 0 pfosfpen 112 1428 0 714 21 0 21 21 0 8 0 pfrktable 1344 14 0 14 2 2 0 1 0 8 0 pftag 88 5 0 0 1 0 1 1 0 8 0 pfqueue 264 5 0 0 1 0 1 1 0 8 0 pfstitem 24 68 0 66 1 0 1 1 0 8 0 pfstkey 112 68 0 66 2 1 1 2 0 8 0 pfstate 320 68 0 66 5 4 1 5 0 8 0 pfrule 1360 151 0 140 3 1 2 2 0 8 0 art_heap8 4096 1 0 0 1 0 1 1 0 8 0 art_heap4 256 679 0 504 14 2 12 13 0 8 1 art_table 32 680 0 504 2 0 2 2 0 8 0 art_node 16 180 0 142 1 0 1 1 0 8 0 sysvmsgpl 40 20 0 1 1 0 1 1 0 8 0 semapl 112 1047 0 1037 1 0 1 1 0 8 0 shmpl 112 55 0 8 2 0 2 2 0 8 0 dirhash 1024 17 0 0 3 0 3 3 0 8 0 dino2pl 256 11752 0 10317 91 0 91 91 0 8 0 ffsino 272 11752 0 10317 97 0 97 97 0 8 0 nchpl 144 21680 0 20071 61 1 60 61 0 8 0 uvmvnodes 72 5926 0 0 108 0 108 108 0 8 0 vnodes 224 5926 0 0 349 0 349 349 0 8 0 namei 1024 52019 0 52019 2 1 1 1 0 8 1 percpumem 16 73 0 39 1 0 1 1 0 8 0 vcpupl 2048 5 0 0 1 0 1 1 0 8 0 vmpool 560 5 0 0 1 0 1 1 0 8 0 pfiaddrpl 120 16 0 16 1 1 0 1 0 8 0 scsiplug 72 8 0 8 2 2 0 1 0 8 0 scxspl 216 67701 0 67701 10 9 1 8 0 8 1 plimitpl 152 182 0 173 1 0 1 1 0 8 0 sigapl 424 8721 0 8666 7 0 7 7 0 8 0 futexpl 56 61773 0 61773 2 1 1 1 0 8 1 knotepl 112 116 0 0 3 0 3 3 0 8 0 kqueuepl 216 5029 0 5022 3 2 1 2 0 8 0 pipepl 336 502 0 490 14 12 2 2 0 8 0 fdescpl 496 8684 0 8666 3 0 3 3 0 8 0 filepl 152 27050 0 26932 12 6 6 6 0 8 1 lockfpl 104 1006 0 1003 1 0 1 1 0 8 0 lockfspl 48 366 0 363 1 0 1 1 0 8 0 sessionpl 144 27 0 16 1 0 1 1 0 8 0 pgrppl 48 31 0 20 1 0 1 1 0 8 0 ucredpl 96 2385 0 2369 1 0 1 1 0 8 0 zombiepl 144 8666 0 8665 2 1 1 1 0 8 0 processpl 1072 8721 0 8665 4 0 4 4 0 8 0 procpl 672 18645 0 18580 8 2 6 6 0 8 0 sosppl 168 122 0 122 11 11 0 1 0 8 0 sockpl 480 5050 0 5018 21 16 5 7 0 8 0 mcl64k 65536 19 0 0 3 0 3 3 0 8 0 mcl16k 16384 20 0 0 3 1 2 3 0 8 0 mcl12k 12288 17 0 0 2 0 2 2 0 8 0 mcl9k 9216 6 0 0 1 0 1 1 0 8 0 mcl8k 8192 17 0 0 3 0 3 3 0 8 0 mcl4k 4096 33 0 0 4 1 3 3 0 8 0 mcl2k2 2112 6 0 0 1 0 1 1 0 8 0 mcl2k 2048 362 0 0 20 0 20 20 0 8 0 mtagpl 96 248 0 0 5 1 4 5 0 8 0 mbufpl 256 917 0 0 46 0 46 46 0 8 0 bufpl 280 16240 0 9904 453 0 453 453 0 8 0 anonpl 24 2261453 0 2251278 170 98 72 98 0 186 1 amapchunkpl 152 245616 0 245170 49 30 19 31 0 158 0 amappl16 200 19861 0 19459 76 53 23 37 0 8 1 amappl15 192 5 0 3 1 0 1 1 0 8 0 amappl14 184 1686 0 1685 1 0 1 1 0 8 0 amappl13 176 784 0 783 1 0 1 1 0 8 0 amappl12 168 206 0 204 1 0 1 1 0 8 0 amappl11 160 1993 0 1977 1 0 1 1 0 8 0 amappl10 152 855 0 847 1 0 1 1 0 8 0 amappl9 144 246 0 244 1 0 1 1 0 8 0 amappl8 136 3758 0 3640 5 0 5 5 0 8 0 amappl7 128 3142 0 3127 1 0 1 1 0 8 0 amappl6 120 199 0 182 1 0 1 1 0 8 0 amappl5 112 7704 0 7684 1 0 1 1 0 8 0 amappl4 104 4243 0 4205 2 0 2 2 0 8 0 amappl3 96 477 0 473 1 0 1 1 0 8 0 amappl2 88 1622 0 1563 2 0 2 2 0 8 0 amappl1 80 149213 0 148764 13 3 10 13 0 8 0 amappl 88 101369 0 101211 4 0 4 4 0 92 0 dma4096 4096 1 0 1 1 1 0 1 0 8 0 dma1024 1024 1 0 0 1 0 1 1 0 8 0 dma256 256 6 0 6 1 1 0 1 0 8 0 dma128 128 253 0 253 1 1 0 1 0 8 0 dma64 64 6 0 6 1 1 0 1 0 8 0 dma32 32 7 0 7 1 1 0 1 0 8 0 dma16 16 18 0 17 1 0 1 1 0 8 0 aobjpl 64 57 0 8 1 0 1 1 0 8 0 uaddrrnd 24 8689 0 8666 1 0 1 1 0 8 0 uaddrbest 32 2 0 0 1 0 1 1 0 8 0 uaddr 24 8689 0 8666 1 0 1 1 0 8 0 vmmpekpl 168 44955 0 44920 3 0 3 3 0 8 0 vmmpepl 168 971839 0 970007 189 86 103 103 0 357 17 vmsppl 368 8688 0 8666 3 0 3 3 0 8 0 rwobjpl 56 167975 0 166734 33 14 19 21 0 8 1 pdppl 4096 17386 0 17337 73 22 51 53 0 8 2 pvpl 32 5176591 0 5162780 288 162 126 166 0 265 0 pmappl 224 8688 0 8666 2 0 2 2 0 8 0 extentpl 40 58 0 40 1 0 1 1 0 8 0 phpool 112 407 0 64 10 0 10 10 0 8 0 ddb{0}> machine ddbcpu 0 Invalid cpu 0 ddb{0}> trace db_enter() at db_enter+0x18 sys/arch/amd64/amd64/db_interface.c:440 witness_checkorder(fffffd806a0b6b38,9,0) at witness_checkorder+0x10f3 witness_debugger sys/kern/subr_witness.c:2502 [inline] witness_checkorder(fffffd806a0b6b38,9,0) at witness_checkorder+0x10f3 sys/kern/subr_witness.c:1105 rw_enter(fffffd806a0b6b28,81) at rw_enter+0xd1 sys/kern/kern_rwlock.c:250 rrw_enter(fffffd806a0b6b28,81) at rrw_enter+0x8b sys/kern/kern_rwlock.c:461 VOP_LOCK(fffffd807190b788,81) at VOP_LOCK+0x87 sys/kern/vfs_vops.c:614 vn_lock(fffffd807190b788,81) at vn_lock+0x84 sys/kern/vfs_vnops.c:579 uvn_get(fffffd806f162120,8000000000000000,ffff800023b54ec8,ffff800023b54e98,0,0) at uvn_get+0x256 uvm_vnode_lock sys/uvm/uvm_vnode.c:1499 [inline] uvn_get(fffffd806f162120,8000000000000000,ffff800023b54ec8,ffff800023b54e98,0,0) at uvn_get+0x256 sys/uvm/uvm_vnode.c:993 uvm_fault_lower(ffff800023b55030,ffff800023b55068,ffff800023b54fb0,0) at uvm_fault_lower+0x302 sys/uvm/uvm_fault.c:1251 uvm_fault(fffffd80694118b0,20000000,0,2) at uvm_fault+0x240 sys/uvm/uvm_fault.c:638 kpageflttrap(ffff800023b551c0,20000000) at kpageflttrap+0x1fd sys/arch/amd64/amd64/trap.c:264 kerntrap(ffff800023b551c0) at kerntrap+0xef sys/arch/amd64/amd64/trap.c:318 alltraps_kern_meltdown() at alltraps_kern_meltdown+0x7b copyout() at copyout+0x53 syscall(ffff800023b55400) at syscall+0x5a9 mi_syscall sys/sys/syscall_mi.h:102 [inline] syscall(ffff800023b55400) at syscall+0x5a9 sys/arch/amd64/amd64/trap.c:587 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0x47e5a9216c0, count: -15 ddb{0}> machine ddbcpu 1 Stopped at x86_ipi_db+0x1a: addq $0x8,%rsp ddb{1}> trace x86_ipi_db(ffff800020d38ff0) at x86_ipi_db+0x1a sys/arch/amd64/amd64/db_interface.c:393 x86_ipi_handler() at x86_ipi_handler+0xb7 sys/arch/amd64/amd64/ipi.c:106 Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23 acpicpu_idle() at acpicpu_idle+0x2eb sys/dev/acpi/acpicpu.c:1206 sched_idle(ffff800020d38ff0) at sched_idle+0x417 sys/kern/kern_sched.c:178 end trace frame: 0x0, count: -5