witness: lock order reversal: 1st 0xfffffd80776edc08 fdlock (&newfdp->fd_fd.fd_lock) 2nd 0xfffffd80668cac48 inode (&ip->i_lock) lock order data w2 -> w1 missing lock order data w1 -> w2 missing Stopped at db_enter+0x18: addq $0x8,%rsp ddb{1}> ddb{1}> set $lines = 0 ddb{1}> set $maxwidth = 0 ddb{1}> show panic the kernel did not panic ddb{1}> trace db_enter() at db_enter+0x18 sys/arch/amd64/amd64/db_interface.c:398 witness_checkorder(fffffd80668cac48,9,0) at witness_checkorder+0x108b rw_enter(fffffd80668cac38,1) at rw_enter+0xd4 rrw_enter(fffffd80668cac38,1) at rrw_enter+0x88 sys/kern/kern_rwlock.c:461 VOP_LOCK(fffffd807acd40e0,2001) at VOP_LOCK+0x87 sys/kern/vfs_vops.c:614 vn_lock(fffffd807acd40e0,2001) at vn_lock+0x84 sys/kern/vfs_vnops.c:579 vget(fffffd807acd40e0,2001) at vget+0x1f7 sys/kern/vfs_subr.c:676 ktrwriteraw(ffff8000212787e0,fffffd807acd40e0,fffffd807f7d8780,ffff8000244e0ef0,ffff8000244e0ed0) at ktrwriteraw+0x138 sys/kern/kern_ktrace.c:659 ktrstruct(ffff8000212787e0,ffffffff823b9dc3,ffff8000244e0fd8,8) at ktrstruct+0x169 ktrwrite2 sys/kern/kern_ktrace.c:627 [inline] ktrstruct(ffff8000212787e0,ffffffff823b9dc3,ffff8000244e0fd8,8) at ktrstruct+0x169 sys/kern/kern_ktrace.c:311 sys_socketpair(ffff8000212787e0,ffff8000244e1048,ffff8000244e1090) at sys_socketpair+0x3ed sys/kern/uipc_syscalls.c:470 syscall(ffff8000244e1110) at syscall+0x5bf mi_syscall sys/sys/syscall_mi.h:102 [inline] syscall(ffff8000244e1110) at syscall+0x5bf sys/arch/amd64/amd64/trap.c:590 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0x2ed3050000, count: -12 ddb{1}> show registers rdi 0xffff8000230ba000 rsi 0xf805 __ALIGN_SIZE+0xe805 rbp 0xffff8000244e0b00 rbx 0x3 rdx 0xffff8000230ba000 rcx 0xf804 __ALIGN_SIZE+0xe804 rax 0xffffffff8183eaa7 db_enter+0x17 r8 0xffffffff81567231 witness_checkorder+0x1061 r9 0x5 r10 0x27e39f8066caa10f r11 0x5889acdaeb005564 r12 0 r13 0xfffffd80668cac48 r14 0 r15 0xfffffd8002cf4740 rip 0xffffffff8183eaa8 db_enter+0x18 cs 0x8 rflags 0x246 rsp 0xffff8000244e0af0 ss 0x10 db_enter+0x18: addq $0x8,%rsp ddb{1}> show proc PROC (syz-executor.1) pid=115927 stat=onproc flags process=0 proc=4000001 pri=32, usrpri=86, nice=20 forw=0xffffffffffffffff, list=0xffff800021278540,0xffffffff828e0d20 process=0xffff800021224cb8 user=0xffff8000244dc000, vmspace=0xfffffd80084d2b80 estcpu=36, cpticks=0, pctcpu=0.0 user=0, sys=0, intr=0 ddb{1}> ps PID TID PPID UID S FLAGS WAIT COMMAND 34207 449629 75413 0 7 0 syz-executor.1 *34207 115927 75413 0 7 0x4000001 syz-executor.1 7820 485363 0 0 3 0x14200 acct acct 5580 459077 36474 0 3 0x82 nanoslp syz-executor.0 75413 81625 36474 0 3 0x82 nanoslp syz-executor.1 36474 307868 39887 0 3 0x82 thrsleep syz-fuzzer 36474 363765 39887 0 3 0x4000082 nanoslp syz-fuzzer 36474 47856 39887 0 3 0x4000082 kqread syz-fuzzer 36474 450418 39887 0 3 0x4000082 thrsleep syz-fuzzer 36474 369551 39887 0 3 0x4000082 thrsleep syz-fuzzer 36474 349882 39887 0 3 0x4000082 thrsleep syz-fuzzer 36474 512687 39887 0 3 0x4000082 thrsleep syz-fuzzer 36474 488685 39887 0 3 0x4000082 thrsleep syz-fuzzer 39887 222100 41100 0 3 0x10008a sigsusp ksh 41100 233174 15877 0 3 0x92 select sshd 68030 42083 1 0 3 0x100083 ttyin getty 15877 257737 1 0 3 0x80 select sshd 64303 499637 37119 74 3 0x100092 bpf pflogd 37119 219179 1 0 3 0x80 netio pflogd 87583 35243 20282 73 3 0x100090 kqread syslogd 20282 513554 1 0 3 0x100082 netio syslogd 47064 475209 1 77 3 0x100090 poll dhclient 92462 296461 1 0 3 0x80 poll dhclient 18378 412164 0 0 3 0x14200 bored smr 5666 278451 0 0 3 0x14200 pgzero zerothread 70430 54232 0 0 3 0x14200 aiodoned aiodoned 82081 43209 0 0 3 0x14200 syncer update 39344 128397 0 0 3 0x14200 cleaner cleaner 16078 131141 0 0 3 0x14200 reaper reaper 29219 304175 0 0 3 0x14200 pgdaemon pagedaemon 50373 507336 0 0 3 0x14200 bored crynlk 40051 395063 0 0 3 0x14200 bored crypto 3535 416592 0 0 3 0x14200 bored viomb 40732 402409 0 0 3 0x40014200 acpi0 acpi0 71522 405775 0 0 3 0x40014200 idle1 62794 203997 0 0 3 0x14200 bored softnet 49265 53481 0 0 3 0x14200 bored systqmp 8896 395582 0 0 3 0x14200 bored systq 8930 251751 0 0 3 0x40014200 bored softclock 28877 184348 0 0 3 0x40014200 idle0 1 438244 0 0 3 0x82 wait init 0 0 -1 0 3 0x10200 scheduler swapper ddb{1}> show all locks Process 34207 (syz-executor.1) thread 0xffff8000212787e0 (115927) exclusive kernel_lock &kernel_lock r = 0 (0xffffffff82795c40) #0 witness_lock+0x4b0 stacktrace_save sys/sys/stacktrace.h:36 [inline] #0 witness_lock+0x4b0 sys/kern/subr_witness.c:1182 #1 ktrstruct+0xee #2 sys_socketpair+0x3ed sys/kern/uipc_syscalls.c:470 #3 syscall+0x5bf mi_syscall sys/sys/syscall_mi.h:102 [inline] #3 syscall+0x5bf sys/arch/amd64/amd64/trap.c:590 #4 Xsyscall+0x128 exclusive rwlock fdlock r = 0 (0xfffffd80776edc08) #0 witness_lock+0x4b0 stacktrace_save sys/sys/stacktrace.h:36 [inline] #0 witness_lock+0x4b0 sys/kern/subr_witness.c:1182 #1 sys_socketpair+0x219 #2 syscall+0x5bf mi_syscall sys/sys/syscall_mi.h:102 [inline] #2 syscall+0x5bf sys/arch/amd64/amd64/trap.c:590 #3 Xsyscall+0x128 ddb{1}> show malloc Type InUse MemUse HighUse Limit Requests Type Lim devbuf 10125 6501K 6514K 78643K 16885 0 pcb 13 8K 8K 78643K 985 0 rtable 109 3K 3K 78643K 287 0 ifaddr 44 10K 10K 78643K 60 0 sysctl 2 0K 0K 78643K 2 0 counters 44 34K 34K 78643K 48 0 ioctlops 0 0K 4K 78643K 2928 0 iov 0 0K 12K 78643K 1444 0 mount 1 1K 1K 78643K 1 0 log 0 0K 0K 78643K 1 0 vnodes 1218 77K 77K 78643K 6867 0 UFS quota 1 32K 32K 78643K 1 0 UFS mount 5 36K 36K 78643K 5 0 shm 2 1K 9K 78643K 443 0 VM map 2 1K 1K 78643K 2 0 sem 12 0K 0K 78643K 625 0 dirhash 12 2K 2K 78643K 12 0 ACPI 1697 195K 286K 78643K 12598 0 file desc 5 13K 25K 78643K 48852 0 sigio 1 0K 0K 78643K 2450 0 proc 59 63K 95K 78643K 517 0 subproc 32 2K 2K 78643K 68 0 NFS srvsock 1 0K 0K 78643K 1 0 NFS daemon 1 16K 16K 78643K 1 0 in_multi 33 2K 2K 78643K 55 0 ether_multi 1 0K 0K 78643K 1 0 ISOFS mount 1 32K 32K 78643K 1 0 MSDOSFS mount 1 16K 16K 78643K 1 0 ttys 43 201K 201K 78643K 43 0 exec 0 0K 2K 78643K 2835 0 pagedep 1 8K 8K 78643K 1 0 inodedep 1 32K 32K 78643K 1 0 newblk 1 0K 0K 78643K 1 0 VM swap 7 26K 26K 78643K 7 0 UVM amap 254 28K 28K 78643K 585160 0 UVM aobj 131 4K 4K 78643K 132 0 memdesc 1 4K 4K 78643K 1 0 crypto data 1 1K 1K 78643K 1 0 NDP 6 0K 0K 78643K 16 0 temp 108 3985K 4053K 78643K 115208 0 kqueue 15 19K 24K 78643K 649 0 SYN cache 2 16K 16K 78643K 2 0 ddb{1}> show all pools Name Size Requests Fail Releases Pgreq Pgrel Npage Hiwat Minpg Maxpg Idle plcache 128 20 0 0 1 0 1 1 0 8 0 rtpcb 120 23 0 21 1 0 1 1 0 8 0 rtentry 112 70 0 24 2 0 2 2 0 8 0 unpcb 120 30437 0 30425 1 0 1 1 0 8 0 syncache 296 4 0 4 1 1 0 1 0 8 0 tcpcb 736 120 0 116 4 3 1 2 0 8 0 arp 120 10 0 4 1 0 1 1 0 8 0 inpcb 304 3849 0 3843 1 0 1 1 0 8 0 nd6 48 15 0 7 1 0 1 1 0 8 0 kcovpl 48 4 0 2 1 0 1 1 0 8 0 pffrag 232 24 0 24 2 2 0 2 0 482 0 pffrnode 88 22 0 22 1 1 0 1 0 8 0 pffrent 40 251 0 251 1 1 0 1 0 8 0 pfosfp 40 1428 0 1005 5 0 5 5 0 8 0 pfosfpen 112 1428 0 714 21 0 21 21 0 8 0 pfstitem 24 44 0 42 1 0 1 1 0 8 0 pfstkey 112 44 0 42 1 0 1 1 0 8 0 pfstate 320 44 0 42 2 1 1 2 0 8 0 pfrule 1360 21 0 16 2 1 1 2 0 8 0 art_heap8 4096 1 0 0 1 0 1 1 0 8 0 art_heap4 256 281 0 89 13 1 12 13 0 8 0 art_table 32 282 0 89 2 0 2 2 0 8 0 art_node 16 69 0 27 1 0 1 1 0 8 0 semupl 112 18 0 18 1 1 0 1 0 8 0 semapl 112 622 0 612 1 0 1 1 0 8 0 shmpl 112 129 0 1 4 0 4 4 0 8 0 dirhash 1024 17 0 0 3 0 3 3 0 8 0 dino2pl 256 60854 0 59404 92 1 91 91 0 8 0 ffsino 272 60854 0 59404 97 0 97 97 0 8 0 nchpl 144 126229 0 124630 60 0 60 60 0 8 0 uvmvnodes 72 5926 0 0 108 0 108 108 0 8 0 vnodes 224 5926 0 0 349 0 349 349 0 8 0 namei 1024 309966 0 309966 1 0 1 1 0 8 1 percpumem 16 35 0 2 1 0 1 1 0 8 0 scxspl 216 332152 0 332152 9 8 1 8 0 8 1 plimitpl 152 17 0 9 1 0 1 1 0 8 0 sigapl 424 49065 0 49032 4 0 4 4 0 8 0 futexpl 56 215980 0 215980 1 0 1 1 0 8 1 knotepl 112 899 0 879 1 0 1 1 0 8 0 kqueuepl 168 2081 0 1817 12 0 12 12 0 8 0 pipepl 336 2129 0 2118 45 43 2 2 0 8 1 fdescpl 496 49045 0 49029 3 0 3 3 0 8 0 filepl 152 152197 0 152094 16 11 5 6 0 8 1 lockfpl 104 4281 0 4280 1 0 1 1 0 8 0 lockfspl 48 1737 0 1736 1 0 1 1 0 8 0 sessionpl 144 20 0 9 1 0 1 1 0 8 0 pgrppl 48 229 0 218 1 0 1 1 0 8 0 ucredpl 96 2296 0 2287 1 0 1 1 0 8 0 zombiepl 144 49032 0 49031 1 0 1 1 0 8 0 processpl 1080 49065 0 49031 3 0 3 3 0 8 0 procpl 672 108734 0 108692 4 0 4 4 0 8 0 sockpl 480 34309 0 34289 59 55 4 5 0 8 1 mcl64k 65536 26 0 0 3 0 3 3 0 8 0 mcl16k 16384 25 0 0 4 1 3 3 0 8 0 mcl12k 12288 49 0 0 2 0 2 2 0 8 0 mcl9k 9216 36 0 0 3 1 2 2 0 8 0 mcl8k 8192 26 0 0 4 1 3 3 0 8 0 mcl4k 4096 33 0 0 4 1 3 3 0 8 0 mcl2k2 2112 14 0 0 1 0 1 1 0 8 0 mcl2k 2048 124 0 0 11 1 10 11 0 8 0 mtagpl 96 1 0 0 1 0 1 1 0 8 0 mbufpl 256 1547 0 0 23 5 18 19 0 8 0 bufpl 280 55852 0 49586 448 0 448 448 0 8 0 anonpl 24 12498837 0 12492225 78 35 43 44 0 186 0 amapchunkpl 152 1388383 0 1388023 49 33 16 16 0 158 1 amappl16 200 120394 0 120232 13 4 9 10 0 8 0 amappl15 192 64 0 59 1 0 1 1 0 8 0 amappl14 184 597 0 593 1 0 1 1 0 8 0 amappl13 176 38 0 36 1 0 1 1 0 8 0 amappl12 168 44 0 33 1 0 1 1 0 8 0 amappl11 160 23923 0 23912 1 0 1 1 0 8 0 amappl10 152 24 0 17 1 0 1 1 0 8 0 amappl9 144 285 0 285 2 2 0 1 0 8 0 amappl8 136 1473 0 1339 5 0 5 5 0 8 0 amappl7 128 64 0 53 1 0 1 1 0 8 0 amappl6 120 121 0 103 1 0 1 1 0 8 0 amappl5 112 49806 0 49793 1 0 1 1 0 8 0 amappl4 104 25242 0 25210 1 0 1 1 0 8 0 amappl3 96 3564 0 3560 1 0 1 1 0 8 0 amappl2 88 2600 0 2551 2 0 2 2 0 8 0 amappl1 80 834361 0 833941 12 2 10 12 0 8 0 amappl 88 584685 0 584569 3 0 3 3 0 92 0 dma4096 4096 1 0 1 1 1 0 1 0 8 0 dma1024 1024 1 0 0 1 0 1 1 0 8 0 dma256 256 6 0 6 1 1 0 1 0 8 0 dma128 128 253 0 253 1 1 0 1 0 8 0 dma64 64 6 0 6 1 1 0 1 0 8 0 dma32 32 7 0 7 1 1 0 1 0 8 0 dma16 16 18 0 17 1 0 1 1 0 8 0 aobjpl 64 131 0 1 3 0 3 3 0 8 0 uaddrrnd 24 49045 0 49029 1 0 1 1 0 8 0 uaddrbest 32 2 0 0 1 0 1 1 0 8 0 uaddr 24 49045 0 49029 1 0 1 1 0 8 0 vmmpekpl 168 198744 0 198713 2 0 2 2 0 8 0 vmmpepl 168 5417170 0 5415680 121 50 71 78 0 357 0 vmsppl 368 49044 0 49029 2 0 2 2 0 8 0 rwobjpl 56 921891 0 920960 28 14 14 14 0 8 0 pdppl 4096 98097 0 98058 55 14 41 45 0 8 2 pvpl 32 20930828 0 20920890 345 257 88 128 0 265 3 pmappl 232 49044 0 49029 2 1 1 2 0 8 0 extentpl 40 58 0 40 1 0 1 1 0 8 0 phpool 112 367 0 74 10 1 9 9 0 8 0 ddb{1}> machine ddbcpu 0 Stopped at x86_ipi_db+0x1a: addq $0x8,%rsp ddb{0}> trace x86_ipi_db(ffffffff826fbff0) at x86_ipi_db+0x1a sys/arch/amd64/amd64/db_interface.c:352 x86_ipi_handler() at x86_ipi_handler+0xb7 sys/arch/amd64/amd64/ipi.c:106 Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23 __mp_lock(ffffffff82795a38) at __mp_lock+0x122 __mp_lock_spin sys/kern/kern_lock.c:116 [inline] __mp_lock(ffffffff82795a38) at __mp_lock+0x122 sys/kern/kern_lock.c:147 softintr_dispatch(0) at softintr_dispatch+0x4e sys/arch/amd64/amd64/softintr.c:90 Xsoftclock() at Xsoftclock+0x1f end of kernel end trace frame: 0x7f7fffff7340, count: -6 ddb{0}> machine ddbcpu 1 Stopped at db_enter+0x18: addq $0x8,%rsp ddb{1}> trace db_enter() at db_enter+0x18 sys/arch/amd64/amd64/db_interface.c:398 witness_checkorder(fffffd80668cac48,9,0) at witness_checkorder+0x108b rw_enter(fffffd80668cac38,1) at rw_enter+0xd4 rrw_enter(fffffd80668cac38,1) at rrw_enter+0x88 sys/kern/kern_rwlock.c:461 VOP_LOCK(fffffd807acd40e0,2001) at VOP_LOCK+0x87 sys/kern/vfs_vops.c:614 vn_lock(fffffd807acd40e0,2001) at vn_lock+0x84 sys/kern/vfs_vnops.c:579 vget(fffffd807acd40e0,2001) at vget+0x1f7 sys/kern/vfs_subr.c:676 ktrwriteraw(ffff8000212787e0,fffffd807acd40e0,fffffd807f7d8780,ffff8000244e0ef0,ffff8000244e0ed0) at ktrwriteraw+0x138 sys/kern/kern_ktrace.c:659 ktrstruct(ffff8000212787e0,ffffffff823b9dc3,ffff8000244e0fd8,8) at ktrstruct+0x169 ktrwrite2 sys/kern/kern_ktrace.c:627 [inline] ktrstruct(ffff8000212787e0,ffffffff823b9dc3,ffff8000244e0fd8,8) at ktrstruct+0x169 sys/kern/kern_ktrace.c:311 sys_socketpair(ffff8000212787e0,ffff8000244e1048,ffff8000244e1090) at sys_socketpair+0x3ed sys/kern/uipc_syscalls.c:470 syscall(ffff8000244e1110) at syscall+0x5bf mi_syscall sys/sys/syscall_mi.h:102 [inline] syscall(ffff8000244e1110) at syscall+0x5bf sys/arch/amd64/amd64/trap.c:590 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0x2ed3050000, count: -12