witness: lock order reversal: 1st 0xfffffd807b015c08 fdlock (&newfdp->fd_fd.fd_lock) 2nd 0xfffffd8067faf5f0 inode (&ip->i_lock) lock order data w2 -> w1 missing lock order data w1 -> w2 missing Stopped at db_enter+0x18: addq $0x8,%rsp ddb{0}> ddb{0}> set $lines = 0 ddb{0}> set $maxwidth = 0 ddb{0}> show panic the kernel did not panic ddb{0}> trace db_enter() at db_enter+0x18 sys/arch/amd64/amd64/db_interface.c:398 witness_checkorder(fffffd8067faf5f0,9,0) at witness_checkorder+0x108b rw_enter(fffffd8067faf5e0,1) at rw_enter+0xd4 rrw_enter(fffffd8067faf5e0,1) at rrw_enter+0x88 sys/kern/kern_rwlock.c:461 VOP_LOCK(fffffd806a1b98d0,2001) at VOP_LOCK+0x87 sys/kern/vfs_vops.c:614 vn_lock(fffffd806a1b98d0,2001) at vn_lock+0x84 sys/kern/vfs_vnops.c:579 vget(fffffd806a1b98d0,2001) at vget+0x1f7 sys/kern/vfs_subr.c:676 ktrwriteraw(ffff8000234bb7b0,fffffd806a1b98d0,fffffd807f7d89c0,ffff800021287700,ffff8000212876e0) at ktrwriteraw+0x138 sys/kern/kern_ktrace.c:659 ktrstruct(ffff8000234bb7b0,ffffffff823b92f4,ffff8000212877e8,8) at ktrstruct+0x169 ktrwrite2 sys/kern/kern_ktrace.c:627 [inline] ktrstruct(ffff8000234bb7b0,ffffffff823b92f4,ffff8000212877e8,8) at ktrstruct+0x169 sys/kern/kern_ktrace.c:311 sys_socketpair(ffff8000234bb7b0,ffff800021287858,ffff8000212878a0) at sys_socketpair+0x3ed sys/kern/uipc_syscalls.c:470 syscall(ffff800021287920) at syscall+0x5bf mi_syscall sys/sys/syscall_mi.h:102 [inline] syscall(ffff800021287920) at syscall+0x5bf sys/arch/amd64/amd64/trap.c:590 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0x49b1cc58970, count: -12 ddb{0}> show registers rdi 0xffff8000234c2000 rsi 0x132c4 mp_pdirpa+0x22be rbp 0xffff800021287310 rbx 0x3 rdx 0xffff8000234c2000 rcx 0x132c3 mp_pdirpa+0x22bd rax 0xffffffff81d2dab7 db_enter+0x17 r8 0xffffffff81fdeae1 witness_checkorder+0x1061 r9 0x5 r10 0x2a9529db45515225 r11 0xf6154ca4550a1c2f r12 0 r13 0xfffffd8067faf5f0 r14 0 r15 0xfffffd8002cf56c0 rip 0xffffffff81d2dab8 db_enter+0x18 cs 0x8 rflags 0x246 rsp 0xffff800021287300 ss 0x10 db_enter+0x18: addq $0x8,%rsp ddb{0}> show proc PROC (syz-executor.1) pid=165711 stat=onproc flags process=0 proc=4000001 pri=32, usrpri=86, nice=20 forw=0xffffffffffffffff, list=0xffff8000234bb510,0xffffffff827e0ab0 process=0xffff800022499530 user=0xffff800021282000, vmspace=0xfffffd80084d25c0 estcpu=36, cpticks=1, pctcpu=0.0 user=0, sys=0, intr=0 ddb{0}> ps PID TID PPID UID S FLAGS WAIT COMMAND 18933 275292 64721 0 7 0 syz-executor.1 *18933 165711 64721 0 7 0x4000001 syz-executor.1 88570 318465 40000 0 3 0x82 nanoslp syz-executor.0 78575 453014 0 0 3 0x14280 nfsidl nfsio 78833 361637 0 0 3 0x14280 nfsidl nfsio 62478 216470 0 0 3 0x14280 nfsidl nfsio 76405 487113 0 0 3 0x14280 nfsidl nfsio 54352 362744 0 0 3 0x14280 nfsidl nfsio 47709 212246 0 0 3 0x14280 nfsidl nfsio 49285 93966 0 0 3 0x14280 nfsidl nfsio 4368 490870 0 0 3 0x14280 nfsidl nfsio 98415 124538 0 0 3 0x14280 nfsidl nfsio 68887 9884 0 0 3 0x14280 nfsidl nfsio 2804 148046 0 0 3 0x14280 nfsidl nfsio 59582 124797 0 0 3 0x14280 nfsidl nfsio 44230 83264 0 0 3 0x14280 nfsidl nfsio 14825 441717 0 0 3 0x14280 nfsidl nfsio 54407 506148 0 0 3 0x14280 nfsidl nfsio 22911 293981 0 0 3 0x14280 nfsidl nfsio 50919 298106 0 0 3 0x14280 nfsidl nfsio 39603 316973 0 0 3 0x14280 nfsidl nfsio 88245 72348 0 0 3 0x14280 nfsidl nfsio 46935 172115 0 0 3 0x14280 nfsidl nfsio 64721 136924 40000 0 3 0x82 nanoslp syz-executor.1 40000 39480 57828 0 3 0x82 thrsleep syz-fuzzer 40000 310089 57828 0 3 0x4000082 thrsleep syz-fuzzer 40000 306195 57828 0 3 0x4000082 thrsleep syz-fuzzer 40000 23910 57828 0 3 0x4000082 thrsleep syz-fuzzer 40000 114190 57828 0 3 0x4000082 thrsleep syz-fuzzer 40000 91043 57828 0 3 0x4000082 thrsleep syz-fuzzer 40000 448982 57828 0 3 0x4000082 thrsleep syz-fuzzer 40000 297138 57828 0 3 0x4000082 kqread syz-fuzzer 57828 427858 83037 0 3 0x10008a sigsusp ksh 83037 8954 54378 0 3 0x92 select sshd 64223 338287 1 0 3 0x100083 ttyin getty 54378 61221 1 0 3 0x80 select sshd 12635 373212 69408 74 3 0x100092 bpf pflogd 69408 246989 1 0 3 0x80 netio pflogd 223 313642 13282 73 3 0x100090 kqread syslogd 13282 301378 1 0 3 0x100082 netio syslogd 2844 9593 1 77 3 0x100090 poll dhclient 44984 291884 1 0 3 0x80 poll dhclient 67406 121578 0 0 3 0x14200 bored smr 26070 362515 0 0 3 0x14200 pgzero zerothread 16521 22450 0 0 3 0x14200 aiodoned aiodoned 13006 375015 0 0 3 0x14200 syncer update 10106 155389 0 0 3 0x14200 cleaner cleaner 80777 5345 0 0 3 0x14200 reaper reaper 67060 424487 0 0 3 0x14200 pgdaemon pagedaemon 39067 63617 0 0 3 0x14200 bored crynlk 84558 351970 0 0 3 0x14200 bored crypto 8889 118714 0 0 3 0x14200 bored viomb 3136 64626 0 0 3 0x40014200 acpi0 acpi0 60408 390976 0 0 3 0x40014200 idle1 9481 247996 0 0 3 0x14200 bored softnet 89540 506280 0 0 3 0x14200 bored systqmp 24843 504571 0 0 3 0x14200 bored systq 5845 305786 0 0 3 0x40014200 bored softclock 53765 509068 0 0 3 0x40014200 idle0 1 21724 0 0 3 0x82 wait init 0 0 -1 0 3 0x10200 scheduler swapper ddb{0}> show all locks Process 18933 (syz-executor.1) thread 0xffff8000234bb7b0 (165711) exclusive kernel_lock &kernel_lock r = 0 (0xffffffff827f3e18) #0 witness_lock+0x4b0 stacktrace_save sys/sys/stacktrace.h:36 [inline] #0 witness_lock+0x4b0 sys/kern/subr_witness.c:1182 #1 ktrstruct+0xee #2 sys_socketpair+0x3ed sys/kern/uipc_syscalls.c:470 #3 syscall+0x5bf mi_syscall sys/sys/syscall_mi.h:102 [inline] #3 syscall+0x5bf sys/arch/amd64/amd64/trap.c:590 #4 Xsyscall+0x128 exclusive rwlock fdlock r = 0 (0xfffffd807b015c08) #0 witness_lock+0x4b0 stacktrace_save sys/sys/stacktrace.h:36 [inline] #0 witness_lock+0x4b0 sys/kern/subr_witness.c:1182 #1 sys_socketpair+0x219 #2 syscall+0x5bf mi_syscall sys/sys/syscall_mi.h:102 [inline] #2 syscall+0x5bf sys/arch/amd64/amd64/trap.c:590 #3 Xsyscall+0x128 ddb{0}> show malloc Type InUse MemUse HighUse Limit Requests Type Lim devbuf 10120 6496K 6691K 78643K 19252 0 pcb 13 8K 8K 78643K 868 0 rtable 105 3K 3K 78643K 279 0 ifaddr 44 10K 10K 78643K 60 0 sysctl 2 0K 0K 78643K 2 0 counters 44 34K 34K 78643K 48 0 ioctlops 0 0K 4K 78643K 1974 0 iov 0 0K 4K 78643K 439 0 mount 1 1K 1K 78643K 1 0 log 0 0K 0K 78643K 1 0 vnodes 1216 76K 77K 78643K 5891 0 UFS quota 1 32K 32K 78643K 1 0 UFS mount 5 36K 36K 78643K 5 0 shm 2 1K 9K 78643K 262 0 VM map 2 1K 1K 78643K 2 0 sem 12 0K 0K 78643K 388 0 dirhash 12 2K 2K 78643K 12 0 ACPI 1697 195K 286K 78643K 12598 0 file desc 5 13K 25K 78643K 44513 0 sigio 0 0K 0K 78643K 54 0 proc 60 63K 95K 78643K 518 0 subproc 32 2K 2K 78643K 68 0 NFS srvsock 1 0K 0K 78643K 1 0 NFS daemon 1 16K 16K 78643K 1 0 in_multi 33 2K 2K 78643K 55 0 ether_multi 1 0K 0K 78643K 1 0 ISOFS mount 1 32K 32K 78643K 1 0 MSDOSFS mount 1 16K 16K 78643K 1 0 ttys 19 95K 95K 78643K 19 0 exec 0 0K 2K 78643K 404 0 pagedep 1 8K 8K 78643K 1 0 inodedep 1 32K 32K 78643K 1 0 newblk 1 0K 0K 78643K 1 0 VM swap 7 26K 26K 78643K 7 0 UVM amap 271 28K 28K 78643K 533207 0 UVM aobj 131 6K 6K 78643K 140 0 memdesc 1 4K 4K 78643K 1 0 crypto data 1 1K 1K 78643K 1 0 NDP 6 0K 0K 78643K 16 0 temp 108 3985K 4050K 78643K 95503 0 kqueue 7 12K 18K 78643K 1955 0 SYN cache 2 16K 16K 78643K 2 0 ddb{0}> show all pools Name Size Requests Fail Releases Pgreq Pgrel Npage Hiwat Minpg Maxpg Idle plcache 128 20 0 0 1 0 1 1 0 8 0 rtpcb 120 23 0 21 1 0 1 1 0 8 0 rtentry 112 67 0 23 2 0 2 2 0 8 0 unpcb 120 9427 0 9415 1 0 1 1 0 8 0 syncache 296 4 0 4 1 1 0 1 0 8 0 sackhl 24 1 0 1 1 1 0 1 0 8 0 tcpcb 736 219 0 215 7 6 1 3 0 8 0 arp 120 10 0 4 1 0 1 1 0 8 0 inpcb 304 2042 0 2036 1 0 1 1 0 8 0 nd6 48 12 0 6 1 0 1 1 0 8 0 kcovpl 48 4 0 2 1 0 1 1 0 8 0 pffrag 232 26 0 24 2 1 1 1 0 482 0 pffrnode 88 20 0 18 2 1 1 1 0 8 0 pffrent 40 416 0 414 2 1 1 1 0 8 0 pfosfp 40 1428 0 1005 5 0 5 5 0 8 0 pfosfpen 112 1428 0 714 21 0 21 21 0 8 0 pfstitem 24 44 0 41 1 0 1 1 0 8 0 pfstkey 112 44 0 41 1 0 1 1 0 8 0 pfstate 320 44 0 41 2 1 1 2 0 8 0 pfrule 1360 21 0 16 2 1 1 2 0 8 0 art_heap8 4096 1 0 0 1 0 1 1 0 8 0 art_heap4 256 276 0 88 12 0 12 12 0 8 0 art_table 32 277 0 88 2 0 2 2 0 8 0 art_node 16 66 0 26 1 0 1 1 0 8 0 semapl 112 386 0 376 1 0 1 1 0 8 0 shmpl 112 137 0 9 4 0 4 4 0 8 0 dirhash 1024 17 0 0 3 0 3 3 0 8 0 dino2pl 256 49515 0 48109 89 0 89 89 0 8 0 ffsino 272 49515 0 48109 95 0 95 95 0 8 0 nchpl 144 103856 0 102260 60 0 60 60 0 8 0 uvmvnodes 72 5926 0 0 108 0 108 108 0 8 0 vnodes 224 5926 0 0 349 0 349 349 0 8 0 namei 1024 249658 0 249658 1 0 1 1 0 8 1 percpumem 16 35 0 2 1 0 1 1 0 8 0 scxspl 216 280563 0 280563 9 8 1 8 0 8 1 plimitpl 152 17 0 9 1 0 1 1 0 8 0 sigapl 424 44742 0 44690 6 0 6 6 0 8 0 futexpl 56 187753 0 187753 1 0 1 1 0 8 1 knotepl 112 2325 0 2305 1 0 1 1 0 8 0 kqueuepl 168 6210 0 6103 6 1 5 6 0 8 0 pipepl 336 3217 0 3204 20 18 2 2 0 8 0 fdescpl 496 44706 0 44690 3 0 3 3 0 8 0 filepl 152 121165 0 121051 31 25 6 6 0 8 1 lockfpl 104 3035 0 3033 1 0 1 1 0 8 0 lockfspl 48 1281 0 1279 1 0 1 1 0 8 0 sessionpl 144 20 0 9 1 0 1 1 0 8 0 pgrppl 48 224 0 213 1 0 1 1 0 8 0 ucredpl 96 778 0 769 1 0 1 1 0 8 0 zombiepl 144 44690 0 44689 1 0 1 1 0 8 0 processpl 1080 44742 0 44689 4 0 4 4 0 8 0 procpl 672 99167 0 99106 6 0 6 6 0 8 0 sockpl 480 11532 0 11512 21 18 3 4 0 8 0 mcl64k 65536 18 0 0 3 0 3 3 0 8 0 mcl16k 16384 33 0 0 4 1 3 3 0 8 0 mcl12k 12288 35 0 0 2 0 2 2 0 8 0 mcl9k 9216 49 0 0 3 1 2 2 0 8 0 mcl8k 8192 25 0 0 3 0 3 3 0 8 0 mcl4k 4096 33 0 0 4 1 3 3 0 8 0 mcl2k2 2112 17 0 0 2 0 2 2 0 8 0 mcl2k 2048 176 0 0 11 0 11 11 0 8 0 mtagpl 96 1 0 0 1 0 1 1 0 8 0 mbufpl 256 1736 0 0 18 3 15 16 0 8 0 bufpl 280 48148 0 41882 448 0 448 448 0 8 0 anonpl 24 11337995 0 11331469 97 53 44 62 0 186 1 amapchunkpl 152 1265276 0 1264912 53 37 16 20 0 158 0 amappl16 200 86733 0 86575 24 15 9 20 0 8 0 amappl15 192 65 0 59 1 0 1 1 0 8 0 amappl14 184 5 0 3 1 0 1 1 0 8 0 amappl13 176 37551 0 37546 1 0 1 1 0 8 0 amappl12 168 41 0 35 1 0 1 1 0 8 0 amappl11 160 46 0 35 1 0 1 1 0 8 0 amappl10 152 25 0 18 1 0 1 1 0 8 0 amappl9 144 290 0 288 1 0 1 1 0 8 0 amappl8 136 14618 0 14465 6 0 6 6 0 8 0 amappl7 128 64 0 55 1 0 1 1 0 8 0 amappl6 120 122 0 107 1 0 1 1 0 8 0 amappl5 112 23823 0 23812 1 0 1 1 0 8 0 amappl4 104 1295 0 1264 1 0 1 1 0 8 0 amappl3 96 61797 0 61787 1 0 1 1 0 8 0 amappl2 88 1243 0 1197 2 0 2 2 0 8 0 amappl1 80 730485 0 730067 12 2 10 12 0 8 0 amappl 88 532727 0 532611 4 0 4 4 0 92 0 dma4096 4096 1 0 1 1 1 0 1 0 8 0 dma1024 1024 1 0 0 1 0 1 1 0 8 0 dma256 256 6 0 6 1 1 0 1 0 8 0 dma128 128 253 0 253 1 1 0 1 0 8 0 dma64 64 6 0 6 1 1 0 1 0 8 0 dma32 32 7 0 7 1 1 0 1 0 8 0 dma16 16 18 0 17 1 0 1 1 0 8 0 aobjpl 64 139 0 9 3 0 3 3 0 8 0 uaddrrnd 24 44706 0 44690 1 0 1 1 0 8 0 uaddrbest 32 2 0 0 1 0 1 1 0 8 0 uaddr 24 44706 0 44690 1 0 1 1 0 8 0 vmmpekpl 168 234703 0 234676 2 0 2 2 0 8 0 vmmpepl 168 4858606 0 4857104 111 40 71 77 0 357 1 vmsppl 368 44705 0 44690 2 0 2 2 0 8 0 rwobjpl 56 826303 0 825358 22 8 14 16 0 8 0 pdppl 4096 89419 0 89380 57 16 41 45 0 8 2 pvpl 32 18925708 0 18915669 408 317 91 129 0 265 6 pmappl 232 44705 0 44690 2 1 1 2 0 8 0 extentpl 40 58 0 40 1 0 1 1 0 8 0 phpool 112 335 0 48 9 0 9 9 0 8 0 ddb{0}> machine ddbcpu 0 Invalid cpu 0 ddb{0}> trace db_enter() at db_enter+0x18 sys/arch/amd64/amd64/db_interface.c:398 witness_checkorder(fffffd8067faf5f0,9,0) at witness_checkorder+0x108b rw_enter(fffffd8067faf5e0,1) at rw_enter+0xd4 rrw_enter(fffffd8067faf5e0,1) at rrw_enter+0x88 sys/kern/kern_rwlock.c:461 VOP_LOCK(fffffd806a1b98d0,2001) at VOP_LOCK+0x87 sys/kern/vfs_vops.c:614 vn_lock(fffffd806a1b98d0,2001) at vn_lock+0x84 sys/kern/vfs_vnops.c:579 vget(fffffd806a1b98d0,2001) at vget+0x1f7 sys/kern/vfs_subr.c:676 ktrwriteraw(ffff8000234bb7b0,fffffd806a1b98d0,fffffd807f7d89c0,ffff800021287700,ffff8000212876e0) at ktrwriteraw+0x138 sys/kern/kern_ktrace.c:659 ktrstruct(ffff8000234bb7b0,ffffffff823b92f4,ffff8000212877e8,8) at ktrstruct+0x169 ktrwrite2 sys/kern/kern_ktrace.c:627 [inline] ktrstruct(ffff8000234bb7b0,ffffffff823b92f4,ffff8000212877e8,8) at ktrstruct+0x169 sys/kern/kern_ktrace.c:311 sys_socketpair(ffff8000234bb7b0,ffff800021287858,ffff8000212878a0) at sys_socketpair+0x3ed sys/kern/uipc_syscalls.c:470 syscall(ffff800021287920) at syscall+0x5bf mi_syscall sys/sys/syscall_mi.h:102 [inline] syscall(ffff800021287920) at syscall+0x5bf sys/arch/amd64/amd64/trap.c:590 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0x49b1cc58970, count: -12 ddb{0}> machine ddbcpu 1 Stopped at x86_ipi_db+0x1a: addq $0x8,%rsp ddb{1}> trace x86_ipi_db(ffff800020d58ff0) at x86_ipi_db+0x1a sys/arch/amd64/amd64/db_interface.c:352 x86_ipi_handler() at x86_ipi_handler+0xb7 sys/arch/amd64/amd64/ipi.c:106 Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23 end of kernel end trace frame: 0x7f7fffff7b10, count: -3