witness: lock order reversal: 1st 0xfffffd80643d5808 inode (&ip->i_lock) 2nd 0xfffffd80767af620 fdlock (&newfdp->fd_fd.fd_lock) lock order "&newfdp->fd_fd.fd_lock"(rwlock) -> "&ip->i_lock"(rrwlock) first seen at: #0 witness_checkorder+0x6a7 sys/kern/subr_witness.c:879 #1 rw_enter+0xd1 sys/kern/kern_rwlock.c:247 #2 rrw_enter+0x4f sys/kern/kern_rwlock.c:435 #3 VOP_LOCK+0x4b sys/kern/vfs_vops.c:602 #4 vn_closefile+0x119 vn_lock sys/kern/vfs_vnops.c:561 [inline] #4 vn_closefile+0x119 vn_close sys/kern/vfs_vnops.c:288 [inline] #4 vn_closefile+0x119 sys/kern/vfs_vnops.c:587 #5 fdrop+0xc9 sys/kern/kern_descrip.c:1273 #6 closef+0x11d sys/kern/kern_descrip.c:1257 #7 finishdup+0x2b6 sys/kern/kern_descrip.c:686 #8 dodup3+0x5c5 sys/kern/kern_descrip.c:377 #9 syscall+0x552 mi_syscall sys/sys/syscall_mi.h:92 [inline] #9 syscall+0x552 sys/arch/amd64/amd64/trap.c:555 #10 Xsyscall+0x128 lock order "&ip->i_lock"(rrwlock) -> "&newfdp->fd_fd.fd_lock"(rwlock) first seen at: #0 witness_checkorder+0x6a7 sys/kern/subr_witness.c:879 #1 rw_enter_write+0x5b sys/kern/kern_rwlock.c:125 #2 diskmapioctl+0x237 sys/dev/diskmap.c:106 #3 VOP_IOCTL+0x88 sys/kern/vfs_vops.c:290 #4 vn_ioctl+0xb7 sys/kern/vfs_vnops.c:524 #5 sys_ioctl+0x5b8 #6 syscall+0x552 mi_syscall sys/sys/syscall_mi.h:92 [inline] #6 syscall+0x552 sys/arch/amd64/amd64/trap.c:555 #7 Xsyscall+0x128 Stopped at db_enter+0x18: addq $0x8,%rsp ddb{1}> ddb{1}> set $lines = 0 ddb{1}> set $maxwidth = 0 ddb{1}> show panic the kernel did not panic ddb{1}> trace db_enter() at db_enter+0x18 sys/arch/amd64/amd64/db_interface.c:398 witness_checkorder(fffffd80767af620,9,0) at witness_checkorder+0xffc witness_debugger sys/kern/subr_witness.c:2506 [inline] witness_checkorder(fffffd80767af620,9,0) at witness_checkorder+0xffc sys/kern/subr_witness.c:1086 rw_enter_write(fffffd80767af610) at rw_enter_write+0x5b sys/kern/kern_rwlock.c:125 diskmapioctl(5a00,c0106477,ffff800023b8adf0,1,ffff800020ab0018) at diskmapioctl+0x237 sys/dev/diskmap.c:106 VOP_IOCTL(fffffd806fa6e580,c0106477,ffff800023b8adf0,1,fffffd807f7c64e0,ffff800020ab0018) at VOP_IOCTL+0x88 sys/kern/vfs_vops.c:290 vn_ioctl(fffffd8067c76738,c0106477,ffff800023b8adf0,ffff800020ab0018) at vn_ioctl+0xb7 sys/kern/vfs_vnops.c:524 sys_ioctl(ffff800020ab0018,ffff800023b8af08,ffff800023b8af70) at sys_ioctl+0x5b8 syscall(ffff800023b8afe0) at syscall+0x552 mi_syscall sys/sys/syscall_mi.h:92 [inline] syscall(ffff800023b8afe0) at syscall+0x552 sys/arch/amd64/amd64/trap.c:555 Xsyscall(6,0,ffffffffffffff1b,0,3,b639cb3a1b0) at Xsyscall+0x128 end of kernel end trace frame: 0xb65bcc1e830, count: -9 ddb{1}> show registers rdi 0x3 rsi 0x3ffff acpi_pdirpa+0x2be67 rbp 0xffff800023b8a9b0 rbx 0x3 rdx 0x40000 acpi_pdirpa+0x2be68 rcx 0xffff800020d4e000 rax 0xffff800000a46ac0 r8 0xffffffff81756bc5 witness_checkorder+0xfd5 r9 0x5 r10 0x3 r11 0xe318634a6d4399b6 r12 0xfffffd80029b8c00 r13 0 r14 0xffffffff825ce220 w_lodata+0x37fd0 r15 0xffffffff825ea730 w_lodata+0x544e0 rip 0xffffffff8209fa08 db_enter+0x18 cs 0x8 rflags 0x246 rsp 0xffff800023b8a9a0 ss 0x10 db_enter+0x18: addq $0x8,%rsp ddb{1}> show proc PROC (syz-executor.1) pid=273998 stat=onproc flags process=0 proc=4000000 pri=32, usrpri=80, nice=20 forw=0xffffffffffffffff, list=0xffff800020ab0508,0xffff800020ab0790 process=0xffff800020a8b190 user=0xffff800023b86000, vmspace=0xfffffd807f00c170 estcpu=30, cpticks=0, pctcpu=0.0 user=0, sys=0, intr=0 ddb{1}> ps PID TID PPID UID S FLAGS WAIT COMMAND 45564 499631 75901 0 7 0 syz-executor.1 45564 422732 75901 0 2 0x4000000 syz-executor.1 45564 155296 75901 0 2 0x4000000 syz-executor.1 *45564 273998 75901 0 7 0x4000000 syz-executor.1 45564 482263 75901 0 2 0x4000000 syz-executor.1 98806 286641 50669 0 3 0x82 piperd syz-executor.0 75901 362565 50669 0 3 0x82 nanosleep syz-executor.1 41767 1419 0 0 3 0x14200 acct acct 68390 502756 1 0 3 0x100083 ttyin getty 96186 318420 0 0 3 0x14200 bored sosplice 50669 74296 74478 0 3 0x82 thrsleep syz-fuzzer 50669 432492 74478 0 3 0x4000082 thrsleep syz-fuzzer 50669 459127 74478 0 3 0x4000082 thrsleep syz-fuzzer 50669 498350 74478 0 3 0x4000082 thrsleep syz-fuzzer 50669 348692 74478 0 3 0x4000082 thrsleep syz-fuzzer 50669 336997 74478 0 3 0x4000082 thrsleep syz-fuzzer 50669 261973 74478 0 3 0x4000082 thrsleep syz-fuzzer 50669 336882 74478 0 3 0x4000082 kqread syz-fuzzer 50669 465767 74478 0 3 0x4000082 thrsleep syz-fuzzer 50669 386780 74478 0 3 0x4000082 thrsleep syz-fuzzer 74478 133669 55107 0 3 0x10008a pause ksh 55107 477298 20831 0 3 0x92 select sshd 20831 426915 1 0 3 0x80 select sshd 49510 78956 88223 74 3 0x100092 bpf pflogd 88223 489212 1 0 3 0x80 netio pflogd 23922 296370 49975 73 3 0x100090 kqread syslogd 49975 29339 1 0 3 0x100082 netio syslogd 34878 177961 0 0 3 0x14200 pgzero zerothread 31695 319065 0 0 3 0x14200 aiodoned aiodoned 55709 59254 0 0 3 0x14200 syncer update 58399 265548 0 0 3 0x14200 cleaner cleaner 40135 115022 0 0 3 0x14200 reaper reaper 77168 504109 0 0 3 0x14200 pgdaemon pagedaemon 76571 465355 0 0 3 0x14200 bored crynlk 52398 22169 0 0 3 0x14200 bored crypto 47423 46675 0 0 3 0x40014200 acpi0 acpi0 22640 509357 0 0 3 0x40014200 idle1 14685 376223 0 0 3 0x14200 bored softnet 69261 50551 0 0 3 0x14200 bored systqmp 42371 443244 0 0 3 0x14200 bored systq 63342 342455 0 0 3 0x40014200 bored softclock 10550 192705 0 0 3 0x40014200 idle0 62759 165981 0 0 3 0x14200 bored smr 1 523416 0 0 3 0x82 wait init 0 0 -1 0 3 0x10200 scheduler swapper ddb{1}> show all locks Process 45564 (syz-executor.1) thread 0xffff800020ab0018 (273998) exclusive rrwlock inode r = 0 (0xfffffd80643d5808) #0 witness_lock+0x52e sys/kern/subr_witness.c:1163 #1 rw_enter+0x46d sys/kern/kern_rwlock.c:306 #2 rrw_enter+0x4f sys/kern/kern_rwlock.c:435 #3 VOP_LOCK+0x4b sys/kern/vfs_vops.c:602 #4 vn_lock+0x6e sys/kern/vfs_vnops.c:561 #5 spec_open+0x431 sys/kern/spec_vnops.c:159 #6 VOP_OPEN+0x6a sys/kern/vfs_vops.c:153 #7 vn_open+0x495 sys/kern/vfs_vnops.c:174 #8 diskmapioctl+0x18e sys/dev/diskmap.c:91 #9 VOP_IOCTL+0x88 sys/kern/vfs_vops.c:290 #10 vn_ioctl+0xb7 sys/kern/vfs_vnops.c:524 #11 sys_ioctl+0x5b8 #12 syscall+0x552 mi_syscall sys/sys/syscall_mi.h:92 [inline] #12 syscall+0x552 sys/arch/amd64/amd64/trap.c:555 #13 Xsyscall+0x128 exclusive kernel_lock &kernel_lock r = 0 (0xffffffff826491e0) #0 witness_lock+0x52e sys/kern/subr_witness.c:1163 #1 __mp_acquire_count+0x51 sys/kern/kern_lock.c:227 #2 mi_switch+0x38f sys/kern/sched_bsd.c:441 #3 sleep_finish+0x113 sys/kern/kern_synch.c:373 #4 rw_enter+0x366 sys/kern/kern_rwlock.c:282 #5 rrw_enter+0x4f sys/kern/kern_rwlock.c:435 #6 VOP_LOCK+0x4b sys/kern/vfs_vops.c:602 #7 vn_lock+0x6e sys/kern/vfs_vnops.c:561 #8 spec_open+0x431 sys/kern/spec_vnops.c:159 #9 VOP_OPEN+0x6a sys/kern/vfs_vops.c:153 #10 vn_open+0x495 sys/kern/vfs_vnops.c:174 #11 diskmapioctl+0x18e sys/dev/diskmap.c:91 #12 VOP_IOCTL+0x88 sys/kern/vfs_vops.c:290 #13 vn_ioctl+0xb7 sys/kern/vfs_vnops.c:524 #14 sys_ioctl+0x5b8 #15 syscall+0x552 mi_syscall sys/sys/syscall_mi.h:92 [inline] #15 syscall+0x552 sys/arch/amd64/amd64/trap.c:555 #16 Xsyscall+0x128 ddb{1}> show malloc Type InUse MemUse HighUse Limit Requests Type Lim Kern Lim devbuf 9630 6418K 14917K 78643K 62288 0 0 pcb 13 8K 8K 78643K 1509 0 0 rtable 117 12K 13K 78643K 4945 0 0 ifaddr 93 28K 35K 78643K 2343 0 0 counters 39 33K 33K 78643K 39 0 0 ioctlops 0 0K 4K 78643K 2448 0 0 iov 0 0K 32K 78643K 2118 0 0 mount 1 1K 1K 78643K 1 0 0 vnodes 1191 75K 77K 78643K 20089 0 0 UFS quota 1 32K 32K 78643K 1 0 0 UFS mount 5 36K 36K 78643K 5 0 0 shm 2 1K 5K 78643K 286 0 0 VM map 2 1K 1K 78643K 2 0 0 sem 12 0K 1K 78643K 2435 0 0 dirhash 12 2K 2K 78643K 12 0 0 ACPI 1808 196K 290K 78643K 12765 0 0 file desc 5 13K 25K 78643K 20829 0 0 sigio 0 0K 0K 78643K 275 0 0 proc 54 51K 83K 78643K 4996 0 0 subproc 32 2K 2K 78643K 1187 0 0 NFS srvsock 1 0K 0K 78643K 1 0 0 NFS daemon 1 16K 16K 78643K 1 0 0 ip_moptions 0 0K 0K 78643K 1913 0 0 in_multi 33 2K 2K 78643K 1440 0 0 ether_multi 1 0K 0K 78643K 92 0 0 mrt 0 0K 0K 78643K 78 0 0 ISOFS mount 1 32K 32K 78643K 1 0 0 MSDOSFS mount 1 16K 16K 78643K 1 0 0 ttys 138 609K 609K 78643K 138 0 0 exec 0 0K 1K 78643K 2660 0 0 pfkey data 0 0K 4K 78643K 1 0 0 pagedep 1 8K 8K 78643K 1 0 0 inodedep 1 32K 32K 78643K 1 0 0 newblk 1 0K 0K 78643K 1 0 0 VM swap 7 26K 26K 78643K 7 0 0 UVM amap 116 22K 32K 78643K 67141 0 0 UVM aobj 130 4K 4K 78643K 147 0 0 memdesc 1 4K 4K 78643K 1 0 0 crypto data 1 1K 1K 78643K 1 0 0 ip6_options 0 0K 1K 78643K 1408 0 0 NDP 23 0K 0K 78643K 728 0 0 temp 249 3557K 4197K 78643K 347240 0 0 kqueue 0 0K 0K 78643K 181 0 0 SYN cache 2 16K 16K 78643K 2 0 0 ddb{1}> show all pools Name Size Requests Fail Releases Pgreq Pgrel Npage Hiwat Minpg Maxpg Idle arp 64 146 0 139 1 0 1 1 0 8 0 plcache 128 20 0 0 1 0 1 1 0 8 0 rtpcb 80 966 0 966 11 10 1 1 0 8 1 rtentry 112 808 0 763 4 2 2 2 0 8 0 unpcb 120 6662 0 6654 1 0 1 1 0 8 0 syncache 264 4 0 4 1 1 0 1 0 8 0 sackhl 24 2 0 2 2 2 0 1 0 8 0 tcpqe 32 9290 0 9290 2 2 0 2 0 8 0 tcpcb 544 3434 0 3430 1 0 1 1 0 8 0 inpcb 280 9939 0 9932 17 16 1 2 0 8 0 rttmr 72 14 0 14 12 12 0 1 0 8 0 ip6q 72 2 0 2 1 1 0 1 0 8 0 ip6af 48 4 0 4 1 1 0 1 0 8 0 nd6 48 207 0 201 1 0 1 1 0 8 0 pkpcb 40 119 0 119 39 39 0 1 0 8 0 swfcl 56 9 0 0 1 0 1 1 0 8 0 ppxss 1128 393 0 393 44 43 1 1 0 8 1 pffrag 232 51 0 51 27 27 0 1 0 482 0 pffrnode 88 51 0 51 27 27 0 1 0 8 0 pffrent 40 126 0 126 27 27 0 1 0 8 0 pfosfp 40 846 0 846 5 5 0 5 0 8 0 pfosfpen 112 1428 0 1428 21 21 0 21 0 8 0 pfstitem 24 508 0 491 1 0 1 1 0 8 0 pfstkey 112 509 0 492 2 1 1 2 0 8 0 pfstate 328 509 0 492 7 5 2 4 0 8 0 pfrule 1360 21 0 16 2 1 1 2 0 8 0 art_heap8 4096 1 0 0 1 0 1 1 0 8 0 art_heap4 256 3338 0 3140 26 13 13 14 0 8 0 art_table 32 3339 0 3140 2 0 2 2 0 8 0 art_node 16 801 0 760 1 0 1 1 0 8 0 sysvmsgpl 40 18 0 16 2 1 1 1 0 8 0 semapl 112 2431 0 2421 1 0 1 1 0 8 0 shmpl 112 145 0 17 4 0 4 4 0 8 0 dirhash 1024 17 0 0 3 0 3 3 0 8 0 dino1pl 128 33548 0 32072 48 0 48 48 0 8 0 ffsino 272 33548 0 32072 100 1 99 99 0 8 0 nchpl 144 62479 0 60843 62 0 62 62 0 8 0 uvmvnodes 72 7262 0 0 133 0 133 133 0 8 0 vnodes 200 7262 0 0 383 0 383 383 0 8 0 namei 1024 204689 0 204689 5 4 1 1 0 8 1 percpumem 16 30 0 0 1 0 1 1 0 8 0 scsiplug 64 56 0 56 31 31 0 1 0 8 0 scxspl 192 184649 0 184649 101 98 3 7 0 8 3 plimitpl 152 1882 0 1875 1 0 1 1 0 8 0 sigapl 432 20829 0 20816 3 1 2 3 0 8 0 futexpl 56 263342 0 263342 4 3 1 1 0 8 1 knotepl 112 4654 0 4635 1 0 1 1 0 8 0 kqueuepl 104 4364 0 4362 1 0 1 1 0 8 0 pipepl 112 11312 0 11293 15 14 1 2 0 8 0 fdescpl 488 20830 0 20816 3 0 3 3 0 8 0 filepl 152 121545 0 121449 37 32 5 7 0 8 1 lockfpl 104 6735 0 6735 7 6 1 1 0 8 1 lockfspl 48 2292 0 2292 7 6 1 1 0 8 1 sessionpl 112 86 0 77 1 0 1 1 0 8 0 pgrppl 48 297 0 288 1 0 1 1 0 8 0 ucredpl 96 22511 0 22502 1 0 1 1 0 8 0 zombiepl 144 20821 0 20821 4 3 1 1 0 8 1 processpl 896 20852 0 20821 4 0 4 4 0 8 0 procpl 632 62847 0 62803 29 25 4 5 0 8 0 srpgc 64 148 0 148 57 57 0 1 0 8 0 sosppl 128 205 0 205 55 54 1 1 0 8 1 sockpl 384 17900 0 17888 32 29 3 4 0 8 1 mcl64k 65536 1341 0 0 161 112 49 128 0 8 1 mcl16k 16384 42 0 0 6 4 2 3 0 8 0 mcl12k 12288 73 0 0 2 0 2 2 0 8 0 mcl9k 9216 57 0 0 3 1 2 2 0 8 0 mcl8k 8192 41 0 0 5 2 3 3 0 8 0 mcl4k 4096 41 0 0 5 2 3 3 0 8 0 mcl2k2 2112 24 0 0 2 0 2 2 0 8 0 mcl2k 2048 272 0 0 24 14 10 24 0 8 0 mtagpl 80 157 0 0 2 0 2 2 0 8 0 mbufpl 256 1551 0 0 69 1 68 68 0 8 0 bufpl 256 62003 0 54741 456 1 455 455 0 8 0 anonpl 16 1872355 0 1860148 370 312 58 83 0 124 0 amapchunkpl 152 123201 0 123069 87 80 7 20 0 158 0 amappl16 192 103405 0 102731 450 414 36 55 0 8 0 amappl15 184 5769 0 5768 10 9 1 1 0 8 0 amappl14 176 5015 0 5009 1 0 1 1 0 8 0 amappl13 168 4353 0 4353 10 10 0 1 0 8 0 amappl12 160 350 0 348 2 1 1 1 0 8 0 amappl11 152 3564 0 3554 1 0 1 1 0 8 0 amappl10 144 2082 0 2080 1 0 1 1 0 8 0 amappl9 136 2426 0 2423 1 0 1 1 0 8 0 amappl8 128 1771 0 1736 2 0 2 2 0 8 0 amappl7 120 2224 0 2213 1 0 1 1 0 8 0 amappl6 112 3449 0 3434 1 0 1 1 0 8 0 amappl5 104 1062 0 1051 1 0 1 1 0 8 0 amappl4 96 22444 0 22415 2 1 1 2 0 8 0 amappl3 88 4774 0 4764 1 0 1 1 0 8 0 amappl2 80 167034 0 166971 3 1 2 3 0 8 0 amappl1 72 453324 0 452915 25 15 10 19 0 8 0 amappl 80 64369 0 64327 2 0 2 2 0 84 0 dma4096 4096 1 0 1 1 1 0 1 0 8 0 dma256 256 6 0 6 1 1 0 1 0 8 0 dma64 64 259 0 259 1 1 0 1 0 8 0 dma32 32 7 0 7 1 1 0 1 0 8 0 dma16 16 17 0 17 1 1 0 1 0 8 0 aobjpl 64 146 0 17 3 0 3 3 0 8 0 uaddrrnd 24 20830 0 20816 1 0 1 1 0 8 0 uaddrbest 32 2 0 0 1 0 1 1 0 8 0 uaddr 24 20830 0 20816 1 0 1 1 0 8 0 vmmpekpl 168 151174 0 151138 2 0 2 2 0 8 0 vmmpepl 168 2521882 0 2520137 723 639 84 108 0 357 0 vmsppl 368 20829 0 20816 2 0 2 2 0 8 0 pdppl 4096 41667 0 41632 8 2 6 6 0 8 0 pvpl 32 5006607 0 4991230 822 686 136 196 0 265 0 pmappl 232 20829 0 20816 4 3 1 2 0 8 0 extentpl 40 41 0 26 1 0 1 1 0 8 0 phpool 112 876 0 159 23 0 23 23 0 8 0