witness: lock order reversal: 1st 0xfffffd807d3ba258 fdlock (&newfdp->fd_fd.fd_lock) 2nd 0xfffffd806abdfa28 inode (&ip->i_lock) lock order data w2 -> w1 missing lock order data w1 -> w2 missing Stopped at db_enter+0x18: addq $0x8,%rsp ddb{1}> ddb{1}> set $lines = 0 ddb{1}> set $maxwidth = 0 ddb{1}> show panic the kernel did not panic ddb{1}> trace db_enter() at db_enter+0x18 sys/arch/amd64/amd64/db_interface.c:398 witness_checkorder(fffffd806abdfa28,9,0) at witness_checkorder+0x108b rw_enter(fffffd806abdfa18,1) at rw_enter+0xd4 rrw_enter(fffffd806abdfa18,1) at rrw_enter+0x88 sys/kern/kern_rwlock.c:461 VOP_LOCK(fffffd8069b33110,2001) at VOP_LOCK+0x87 sys/kern/vfs_vops.c:614 vn_lock(fffffd8069b33110,2001) at vn_lock+0x84 sys/kern/vfs_vnops.c:579 vget(fffffd8069b33110,2001) at vget+0x1f7 sys/kern/vfs_subr.c:676 ktrwriteraw(ffff8000212727e0,fffffd8069b33110,fffffd807f7d8780,ffff8000222cb390,ffff8000222cb370) at ktrwriteraw+0x138 sys/kern/kern_ktrace.c:659 ktrstruct(ffff8000212727e0,ffffffff823b7d71,ffff8000222cb478,8) at ktrstruct+0x169 ktrwrite2 sys/kern/kern_ktrace.c:627 [inline] ktrstruct(ffff8000212727e0,ffffffff823b7d71,ffff8000222cb478,8) at ktrstruct+0x169 sys/kern/kern_ktrace.c:311 sys_socketpair(ffff8000212727e0,ffff8000222cb4e8,ffff8000222cb530) at sys_socketpair+0x3ed sys/kern/uipc_syscalls.c:470 syscall(ffff8000222cb5b0) at syscall+0x5bf mi_syscall sys/sys/syscall_mi.h:102 [inline] syscall(ffff8000222cb5b0) at syscall+0x5bf sys/arch/amd64/amd64/trap.c:590 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0xcb71326bb00, count: -12 ddb{1}> show registers rdi 0xffff8000234b4000 rsi 0xdfe0 __ALIGN_SIZE+0xcfe0 rbp 0xffff8000222cafa0 rbx 0x3 rdx 0xffff8000234b4000 rcx 0xdfdf __ALIGN_SIZE+0xcfdf rax 0xffffffff81ab6387 db_enter+0x17 r8 0xffffffff8225fa91 witness_checkorder+0x1061 r9 0x5 r10 0xc29c7cc0917934dc r11 0x222dfbfb5062bdc7 r12 0 r13 0xfffffd806abdfa28 r14 0 r15 0xfffffd8002cf46c0 rip 0xffffffff81ab6388 db_enter+0x18 cs 0x8 rflags 0x246 rsp 0xffff8000222caf90 ss 0x10 db_enter+0x18: addq $0x8,%rsp ddb{1}> show proc PROC (syz-executor.1) pid=365650 stat=onproc flags process=0 proc=4000001 pri=76, usrpri=76, nice=20 forw=0xffffffffffffffff, list=0xffff800021273260,0xffffffff828b2590 process=0xffff800021225528 user=0xffff8000222c6000, vmspace=0xfffffd80084d28a0 estcpu=36, cpticks=0, pctcpu=0.0 user=0, sys=0, intr=0 ddb{1}> ps PID TID PPID UID S FLAGS WAIT COMMAND 25451 278486 61081 0 7 0 syz-executor.1 *25451 365650 61081 0 7 0x4000001 syz-executor.1 43128 517687 0 0 3 0x14200 bored sosplice 61081 242336 20000 0 3 0x82 nanoslp syz-executor.1 62088 192251 20000 0 2 0x2 syz-executor.0 20000 193542 88027 0 3 0x82 kqread syz-fuzzer 20000 103250 88027 0 3 0x4000082 nanoslp syz-fuzzer 20000 44750 88027 0 3 0x4000082 thrsleep syz-fuzzer 20000 273873 88027 0 3 0x4000082 thrsleep syz-fuzzer 20000 118149 88027 0 3 0x4000082 thrsleep syz-fuzzer 20000 290781 88027 0 3 0x4000082 thrsleep syz-fuzzer 20000 29066 88027 0 3 0x4000082 thrsleep syz-fuzzer 20000 439924 88027 0 3 0x4000082 thrsleep syz-fuzzer 88027 117820 99564 0 3 0x10008a sigsusp ksh 99564 27018 37727 0 3 0x92 select sshd 58336 119370 1 0 3 0x100083 ttyin getty 37727 172718 1 0 3 0x80 select sshd 62978 512571 57889 74 3 0x100092 bpf pflogd 57889 477733 1 0 3 0x80 netio pflogd 6079 137254 13889 73 3 0x100090 kqread syslogd 13889 501865 1 0 3 0x100082 netio syslogd 19845 351116 1 77 3 0x100090 poll dhclient 41933 438281 1 0 3 0x80 poll dhclient 91888 479310 0 0 3 0x14200 bored smr 83600 404958 0 0 3 0x14200 pgzero zerothread 91641 403333 0 0 3 0x14200 aiodoned aiodoned 5175 148496 0 0 3 0x14200 syncer update 70392 496615 0 0 3 0x14200 cleaner cleaner 35411 206162 0 0 3 0x14200 reaper reaper 71529 52536 0 0 3 0x14200 pgdaemon pagedaemon 62688 489956 0 0 3 0x14200 bored crynlk 74512 284052 0 0 3 0x14200 bored crypto 63033 20961 0 0 3 0x14200 bored viomb 20139 208757 0 0 3 0x40014200 acpi0 acpi0 83801 188248 0 0 3 0x40014200 idle1 51416 85824 0 0 3 0x14200 bored softnet 28268 446510 0 0 3 0x14200 bored systqmp 44420 158561 0 0 3 0x14200 bored systq 86847 434226 0 0 3 0x40014200 bored softclock 19051 352052 0 0 3 0x40014200 idle0 1 261354 0 0 3 0x82 wait init 0 0 -1 0 3 0x10200 scheduler swapper ddb{1}> show all locks Process 25451 (syz-executor.1) thread 0xffff8000212727e0 (365650) exclusive kernel_lock &kernel_lock r = 0 (0xffffffff827f1098) #0 witness_lock+0x4b0 stacktrace_save sys/sys/stacktrace.h:36 [inline] #0 witness_lock+0x4b0 sys/kern/subr_witness.c:1182 #1 ktrstruct+0xee #2 sys_socketpair+0x3ed sys/kern/uipc_syscalls.c:470 #3 syscall+0x5bf mi_syscall sys/sys/syscall_mi.h:102 [inline] #3 syscall+0x5bf sys/arch/amd64/amd64/trap.c:590 #4 Xsyscall+0x128 exclusive rwlock fdlock r = 0 (0xfffffd807d3ba258) #0 witness_lock+0x4b0 stacktrace_save sys/sys/stacktrace.h:36 [inline] #0 witness_lock+0x4b0 sys/kern/subr_witness.c:1182 #1 sys_socketpair+0x219 #2 syscall+0x5bf mi_syscall sys/sys/syscall_mi.h:102 [inline] #2 syscall+0x5bf sys/arch/amd64/amd64/trap.c:590 #3 Xsyscall+0x128 ddb{1}> show malloc Type InUse MemUse HighUse Limit Requests Type Lim devbuf 10132 6482K 7265K 78643K 12846 0 pcb 13 8K 8K 78643K 39 0 rtable 97 3K 3K 78643K 399 0 ifaddr 47 10K 10K 78643K 65 0 counters 44 34K 34K 78643K 44 0 ioctlops 0 0K 4K 78643K 1547 0 iov 0 0K 16K 78643K 8 0 mount 1 1K 1K 78643K 1 0 log 0 0K 0K 78643K 1 0 vnodes 1223 77K 77K 78643K 1668 0 UFS quota 1 32K 32K 78643K 1 0 UFS mount 5 36K 36K 78643K 5 0 shm 2 1K 1K 78643K 2 0 VM map 2 1K 1K 78643K 2 0 sem 12 0K 1K 78643K 54 0 dirhash 12 2K 2K 78643K 12 0 ACPI 1697 195K 286K 78643K 12598 0 file desc 5 13K 25K 78643K 1213 0 proc 62 63K 95K 78643K 494 0 subproc 32 2K 2K 78643K 34 0 NFS srvsock 1 0K 0K 78643K 1 0 NFS daemon 1 16K 16K 78643K 1 0 ip_moptions 0 0K 0K 78643K 256 0 in_multi 25 1K 2K 78643K 150 0 ether_multi 1 0K 0K 78643K 11 0 mrt 0 0K 0K 78643K 8 0 ISOFS mount 1 32K 32K 78643K 1 0 MSDOSFS mount 1 16K 16K 78643K 1 0 ttys 55 254K 254K 78643K 55 0 exec 0 0K 2K 78643K 394 0 pagedep 1 8K 8K 78643K 1 0 inodedep 1 32K 32K 78643K 1 0 newblk 1 0K 0K 78643K 1 0 VM swap 7 26K 26K 78643K 7 0 UVM amap 176 26K 27K 78643K 15805 0 UVM aobj 11 2K 2K 78643K 11 0 memdesc 1 4K 4K 78643K 1 0 crypto data 1 1K 1K 78643K 1 0 ip6_options 0 0K 0K 78643K 49 0 NDP 7 0K 0K 78643K 26 0 temp 111 3982K 4046K 78643K 4906 0 kqueue 3 4K 8K 78643K 33 0 SYN cache 2 16K 16K 78643K 2 0 ddb{1}> show all pools Name Size Requests Fail Releases Pgreq Pgrel Npage Hiwat Minpg Maxpg Idle plcache 128 20 0 0 1 0 1 1 0 8 0 rtpcb 120 49 0 47 1 0 1 1 0 8 0 rtentry 112 105 0 67 2 0 2 2 0 8 0 unpcb 120 218 0 206 1 0 1 1 0 8 0 syncache 296 35 0 35 2 1 1 1 0 8 1 tcpqe 32 323 0 323 3 3 0 1 0 8 0 tcpcb 736 230 0 226 8 4 4 4 0 8 3 arp 120 6 0 0 1 0 1 1 0 8 0 inpcb 304 543 0 537 4 3 1 2 0 8 0 nd6 48 21 0 19 1 0 1 1 0 8 0 pkpcb 40 5 0 5 1 1 0 1 0 8 0 kcovpl 48 2 0 0 1 0 1 1 0 8 0 pffrag 232 1 0 0 1 0 1 1 0 482 0 pffrnode 88 1 0 0 1 0 1 1 0 8 0 pffrent 40 5 0 4 1 0 1 1 0 8 0 pfosfp 40 1428 0 1005 5 0 5 5 0 8 0 pfosfpen 112 1428 0 714 21 0 21 21 0 8 0 pfrktable 1344 1 0 0 1 0 1 1 0 8 0 pftag 88 1 0 0 1 0 1 1 0 8 0 pfstitem 24 23 0 6 1 0 1 1 0 8 0 pfstkey 112 23 0 6 1 0 1 1 0 8 0 pfstate 320 23 0 6 2 0 2 2 0 8 0 pfrule 1360 52 0 40 3 1 2 2 0 8 0 art_heap8 4096 1 0 0 1 0 1 1 0 8 0 art_heap4 256 549 0 392 13 1 12 13 0 8 0 art_table 32 550 0 392 2 0 2 2 0 8 0 art_node 16 104 0 68 1 0 1 1 0 8 0 sysvmsgpl 40 17 0 12 1 0 1 1 0 8 0 semupl 112 9 0 9 1 1 0 1 0 8 0 semapl 112 52 0 42 1 0 1 1 0 8 0 shmpl 112 8 0 0 1 0 1 1 0 8 0 dirhash 1024 17 0 0 3 0 3 3 0 8 0 dino2pl 256 2843 0 1438 89 1 88 89 0 8 0 ffsino 272 2843 0 1438 94 0 94 94 0 8 0 nchpl 144 4444 0 2834 60 0 60 60 0 8 0 uvmvnodes 72 3240 0 0 59 0 59 59 0 8 0 vnodes 224 3240 0 0 191 0 191 191 0 8 0 namei 1024 10778 0 10778 1 0 1 1 0 8 1 percpumem 16 33 0 0 1 0 1 1 0 8 0 scxspl 216 12668 0 12668 9 8 1 8 0 8 1 plimitpl 152 41 0 33 1 0 1 1 0 8 0 sigapl 424 1429 0 1396 4 0 4 4 0 8 0 futexpl 56 8781 0 8781 1 0 1 1 0 8 1 knotepl 112 86 0 66 1 0 1 1 0 8 0 kqueuepl 168 606 0 604 1 0 1 1 0 8 0 pipepl 336 131 0 120 4 2 2 2 0 8 1 fdescpl 496 1412 0 1396 3 0 3 3 0 8 0 filepl 152 4817 0 4714 5 0 5 5 0 8 1 lockfpl 104 112 0 111 1 0 1 1 0 8 0 lockfspl 48 46 0 45 1 0 1 1 0 8 0 sessionpl 144 18 0 7 1 0 1 1 0 8 0 pgrppl 48 18 0 7 1 0 1 1 0 8 0 ucredpl 96 492 0 483 1 0 1 1 0 8 0 zombiepl 144 1396 0 1396 1 0 1 1 0 8 1 processpl 1080 1429 0 1396 3 0 3 3 0 8 0 procpl 672 2854 0 2813 4 0 4 4 0 8 0 sosppl 168 22 0 22 1 0 1 1 0 8 1 sockpl 432 815 0 795 5 2 3 4 0 8 0 mcl16k 16384 17 0 0 3 0 3 3 0 8 0 mcl9k 9216 6 0 0 1 0 1 1 0 8 0 mcl8k 8192 3 0 0 1 0 1 1 0 8 0 mcl4k 4096 6 0 0 1 0 1 1 0 8 0 mcl2k 2048 160 0 0 18 0 18 18 0 8 0 mtagpl 96 154 0 0 4 0 4 4 0 8 0 mbufpl 256 635 0 0 38 0 38 38 0 8 0 bufpl 280 4637 0 155 321 0 321 321 0 8 0 anonpl 24 335089 0 328123 46 0 46 46 0 186 0 amapchunkpl 152 36970 0 36586 19 2 17 17 0 158 1 amappl16 200 2272 0 2090 11 1 10 10 0 8 0 amappl15 192 61 0 54 1 0 1 1 0 8 0 amappl14 184 2 0 2 1 1 0 1 0 8 0 amappl13 176 630 0 627 1 0 1 1 0 8 0 amappl12 168 33 0 26 1 0 1 1 0 8 0 amappl11 160 46 0 34 1 0 1 1 0 8 0 amappl10 152 617 0 609 1 0 1 1 0 8 0 amappl9 144 292 0 290 1 0 1 1 0 8 0 amappl8 136 338 0 297 2 0 2 2 0 8 0 amappl7 128 53 0 45 1 0 1 1 0 8 0 amappl6 120 700 0 683 1 0 1 1 0 8 0 amappl5 112 718 0 707 1 0 1 1 0 8 0 amappl4 104 543 0 513 1 0 1 1 0 8 0 amappl3 96 1233 0 1227 1 0 1 1 0 8 0 amappl2 88 435 0 389 2 0 2 2 0 8 0 amappl1 80 26172 0 25755 12 2 10 12 0 8 0 amappl 88 15470 0 15351 4 0 4 4 0 92 0 dma4096 4096 1 0 1 1 1 0 1 0 8 0 dma1024 1024 1 0 0 1 0 1 1 0 8 0 dma256 256 6 0 6 1 1 0 1 0 8 0 dma128 128 253 0 253 1 1 0 1 0 8 0 dma64 64 6 0 6 1 1 0 1 0 8 0 dma32 32 7 0 7 1 1 0 1 0 8 0 dma16 16 18 0 17 1 0 1 1 0 8 0 aobjpl 64 10 0 0 1 0 1 1 0 8 0 uaddrrnd 24 1412 0 1396 1 0 1 1 0 8 0 uaddrbest 32 2 0 0 1 0 1 1 0 8 0 uaddr 24 1412 0 1396 1 0 1 1 0 8 0 vmmpekpl 168 10226 0 10203 2 0 2 2 0 8 0 vmmpepl 168 155100 0 153683 68 0 68 68 0 357 1 vmsppl 368 1411 0 1396 2 0 2 2 0 8 0 rwobjpl 56 28477 0 27620 14 1 13 13 0 8 0 pdppl 4096 2831 0 2792 57 16 41 45 0 8 2 pvpl 32 763986 0 753781 133 42 91 129 0 265 2 pmappl 232 1411 0 1396 2 1 1 2 0 8 0 extentpl 40 58 0 40 1 0 1 1 0 8 0 phpool 112 339 0 27 9 0 9 9 0 8 0 ddb{1}> machine ddbcpu 0 Stopped at x86_ipi_db+0x1a: addq $0x8,%rsp ddb{0}> trace x86_ipi_db(ffffffff8273cff0) at x86_ipi_db+0x1a sys/arch/amd64/amd64/db_interface.c:352 x86_ipi_handler() at x86_ipi_handler+0xb7 sys/arch/amd64/amd64/ipi.c:106 Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23 __mp_lock(ffffffff827f0e90) at __mp_lock+0x122 __mp_lock_spin sys/kern/kern_lock.c:116 [inline] __mp_lock(ffffffff827f0e90) at __mp_lock+0x122 sys/kern/kern_lock.c:147 softintr_dispatch(0) at softintr_dispatch+0x4e sys/arch/amd64/amd64/softintr.c:90 Xsoftclock() at Xsoftclock+0x1f end of kernel end trace frame: 0x7f7ffffd8710, count: -6 ddb{0}> machine ddbcpu 1 Stopped at db_enter+0x18: addq $0x8,%rsp ddb{1}> trace db_enter() at db_enter+0x18 sys/arch/amd64/amd64/db_interface.c:398 witness_checkorder(fffffd806abdfa28,9,0) at witness_checkorder+0x108b rw_enter(fffffd806abdfa18,1) at rw_enter+0xd4 rrw_enter(fffffd806abdfa18,1) at rrw_enter+0x88 sys/kern/kern_rwlock.c:461 VOP_LOCK(fffffd8069b33110,2001) at VOP_LOCK+0x87 sys/kern/vfs_vops.c:614 vn_lock(fffffd8069b33110,2001) at vn_lock+0x84 sys/kern/vfs_vnops.c:579 vget(fffffd8069b33110,2001) at vget+0x1f7 sys/kern/vfs_subr.c:676 ktrwriteraw(ffff8000212727e0,fffffd8069b33110,fffffd807f7d8780,ffff8000222cb390,ffff8000222cb370) at ktrwriteraw+0x138 sys/kern/kern_ktrace.c:659 ktrstruct(ffff8000212727e0,ffffffff823b7d71,ffff8000222cb478,8) at ktrstruct+0x169 ktrwrite2 sys/kern/kern_ktrace.c:627 [inline] ktrstruct(ffff8000212727e0,ffffffff823b7d71,ffff8000222cb478,8) at ktrstruct+0x169 sys/kern/kern_ktrace.c:311 sys_socketpair(ffff8000212727e0,ffff8000222cb4e8,ffff8000222cb530) at sys_socketpair+0x3ed sys/kern/uipc_syscalls.c:470 syscall(ffff8000222cb5b0) at syscall+0x5bf mi_syscall sys/sys/syscall_mi.h:102 [inline] syscall(ffff8000222cb5b0) at syscall+0x5bf sys/arch/amd64/amd64/trap.c:590 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0xcb71326bb00, count: -12