witness: lock order reversal: 1st 0xffff800010fdf970 sbufsnd (&so->so_snd.sb_lock) 2nd 0xfffffd807431e118 inode (&ip->i_lock) lock order [1] sbufsnd (&so->so_snd.sb_lock) -> [2] inode (&ip->i_lock) lock order data 0xffffffff8350b9a9 -> 0xffffffff8347e106 is missing lock order [2] inode (&ip->i_lock) -> [3] sbufrcv (&so->so_rcv.sb_lock) #0 rw_do_enter_write+0xba sys/kern/kern_rwlock.c:234 #1 sblock+0xb6 sys/kern/uipc_socket2.c:536 #2 soreceive+0x27d sys/kern/uipc_socket.c:890 #3 fifo_read+0x117 sys/miscfs/fifofs/fifo_vnops.c:264 #4 VOP_READ+0x101 sys/kern/vfs_vops.c:227 #5 vn_rdwr+0x15b sys/kern/vfs_vnops.c:-1 #6 vndsetcred+0xa1 sys/dev/vnd.c:685 #7 vndioctl+0xdfc sys/dev/vnd.c:486 #8 VOP_IOCTL+0xac sys/kern/vfs_vops.c:264 #9 vn_ioctl+0xf8 sys/kern/vfs_vnops.c:537 #10 sys_ioctl+0x674 sys/kern/sys_generic.c:-1 #11 syscall+0xbd4 mi_syscall sys/sys/syscall_mi.h:176 [inline] #11 syscall+0xbd4 sys/arch/amd64/amd64/trap.c:783 #12 Xsyscall+0x128 lock order [3] sbufrcv (&so->so_rcv.sb_lock) -> [1] sbufsnd (&so->so_snd.sb_lock) #0 rw_do_enter_write+0xba sys/kern/kern_rwlock.c:234 #1 sblock+0xb6 sys/kern/uipc_socket2.c:536 #2 sosplice+0x312 sys/kern/uipc_socket.c:1347 #3 sys_setsockopt+0x2ba sys/kern/uipc_syscalls.c:1226 #4 syscall+0xbd4 mi_syscall sys/sys/syscall_mi.h:176 [inline] #4 syscall+0xbd4 sys/arch/amd64/amd64/trap.c:783 #5 Xsyscall+0x128 Stopped at db_enter+0x25: addq $0x8,%rsp ddb{0}> ddb{0}> set $lines = 0 ddb{0}> set $maxwidth = 0 ddb{0}> show panic the kernel did not panic ddb{0}> trace db_enter() at db_enter+0x25 sys/arch/amd64/amd64/db_interface.c:438 witness_checkorder(fffffd807431e118,9,0) at witness_checkorder+0x10d1 sys/kern/subr_witness.c:-1 rw_do_enter_write(fffffd807431e100,1) at rw_do_enter_write+0xba sys/kern/kern_rwlock.c:234 rrw_enter(fffffd807431e100,1) at rrw_enter+0xc6 sys/kern/kern_rwlock.c:621 VOP_LOCK(fffffd805ac91998,2001) at VOP_LOCK+0xbd sys/kern/vfs_vops.c:527 vn_lock(fffffd805ac91998,2001) at vn_lock+0xa4 sys/kern/vfs_vnops.c:576 vfs_lookup(ffff80002a254590) at vfs_lookup+0x10f sys/kern/vfs_lookup.c:431 namei(ffff80002a254590) at namei+0x7c5 sys/kern/vfs_lookup.c:250 unp_connect(ffff800010fdf788,fffffd800bb21a00,ffff800034b99268) at unp_connect+0x29d sys/kern/uipc_usrreq.c:872 uipc_dgram_send(ffff800010fdf788,fffffd800bb21100,fffffd800bb21a00,0) at uipc_dgram_send+0x163 sys/kern/uipc_usrreq.c:609 sosend(ffff800010fdf788,fffffd800bb21a00,ffff80002a254818,0,0,402) at sosend+0x804 sys/kern/uipc_socket.c:-1 sendit(ffff800034b99268,8,ffff80002a254990,402,ffff80002a2549c8) at sendit+0x5a5 sys/kern/uipc_syscalls.c:785 sys_sendmmsg(ffff800034b99268,ffff80002a254b40,ffff80002a254a90) at sys_sendmmsg+0x3f3 sys/kern/uipc_syscalls.c:676 syscall(ffff80002a254b40) at syscall+0xbd4 mi_syscall sys/sys/syscall_mi.h:176 [inline] syscall(ffff80002a254b40) at syscall+0xbd4 sys/arch/amd64/amd64/trap.c:783 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0xbe7f6e23e70, count: -15 ddb{0}> show registers rdi 0 rsi 0x80000 acpi_pdirpa+0x6be71 rbp 0xffff80002a254200 rbx 0 rdx 0xffff8000015c7b40 rcx 0xffff800034b99268 rax 0x7ffff acpi_pdirpa+0x6be70 r8 0xffff80002a2540e0 r9 0x8080808080808080 r10 0x4cc328917c407639 r11 0x8e986f43f7dc67cd r12 0xfffffd80040aa8c0 r13 0xfffffd8004892350 r14 0x3 r15 0xffffffff83512289 substchar+0xfc85 rip 0xffffffff831a0d75 db_enter+0x25 cs 0x8 rflags 0x246 rsp 0xffff80002a2541f0 ss 0x10 db_enter+0x25: addq $0x8,%rsp ddb{0}> show proc PROC (syz-executor) tid=348438 pid=95883 tcnt=2 stat=onproc flags process=0 proc=4000000 runpri=32, usrpri=86, slppri=32, nice=20 wchan=0x0, wmesg=, ps_single=0x0 scnt=0 ecnt=0 forw=0xffffffffffffffff, list=0xffff800034b99500,0xffff800034b99a40 process=0xffff8000ffff0e80 user=0xffff80002a24f000, vmspace=0xfffffd800b063988 estcpu=36, cpticks=1, pctcpu=0.0, user=0, sys=1, intr=0 ddb{0}> ps PID TID PPID UID S FLAGS WAIT COMMAND 15257 437309 26792 0 7 0 syz-executor 15257 402416 26792 0 2 0x4000080 syz-executor 95883 261065 45874 0 2 0 syz-executor *95883 348438 45874 0 7 0x4000000 syz-executor 20531 23619 87093 0 2 0 syz-executor 20531 155965 87093 0 3 0x4000080 sbwait syz-executor 20531 149125 87093 0 3 0x4000080 fsleep syz-executor 56949 31819 36449 0 2 0 syz-executor 75447 438335 80743 0 2 0 syz-executor 75447 368590 80743 0 3 0x4000080 fsleep syz-executor 19376 426520 1 0 3 0x80 nanoslp init 62566 133802 1969 0 3 0x80 nanoslp syz-executor 62566 515607 1969 0 3 0x4000080 ttyin syz-executor 62566 239283 1969 0 3 0x4000080 fsleep syz-executor 80743 30763 47698 0 2 0x2 syz-executor 36230 483847 98256 0 3 0x3000 suspend syz-executor 36230 18399 98256 0 3 0x4081000 biowait syz-executor 36230 7047 98256 0 3 0x4081000 fltagain2 syz-executor 36449 104105 47698 0 2 0x2 syz-executor 1969 277943 47698 0 3 0x82 nanoslp syz-executor 15280 316000 0 0 3 0x14280 nfsidl nfsio 95118 348924 0 0 3 0x14280 nfsidl nfsio 46805 70118 0 0 3 0x14280 nfsidl nfsio 86608 289059 0 0 3 0x14280 nfsidl nfsio 17740 71978 0 0 3 0x14280 nfsidl nfsio 59497 504727 0 0 3 0x14280 nfsidl nfsio 62567 24789 0 0 3 0x14280 nfsidl nfsio 99468 28637 0 0 3 0x14280 nfsidl nfsio 415 484576 0 0 3 0x14280 nfsidl nfsio 3709 437617 0 0 3 0x14280 nfsidl nfsio 68755 492138 0 0 3 0x14280 nfsidl nfsio 28741 521033 0 0 3 0x14280 nfsidl nfsio 65930 207226 0 0 3 0x14280 nfsidl nfsio 80053 334330 0 0 3 0x14280 nfsidl nfsio 70809 523680 0 0 3 0x14280 nfsidl nfsio 8100 126331 0 0 3 0x14280 nfsidl nfsio 45424 317989 0 0 3 0x14280 nfsidl nfsio 17753 224778 0 0 3 0x14280 nfsidl nfsio 65281 120857 0 0 3 0x14280 nfsidl nfsio 33622 173357 0 0 3 0x14280 nfsidl nfsio 87093 506530 47698 0 3 0x82 nanoslp syz-executor 22449 34302 47698 0 3 0x2 biowait syz-executor 98256 42853 47698 0 3 0x82 wait syz-executor 26792 197606 47698 0 3 0x82 nanoslp syz-executor 45874 99764 47698 0 3 0x82 nanoslp syz-executor 47698 258937 1 0 3 0x82 kqread syz-executor 93179 408373 1 74 3 0x1100092 bpf pflogd 62287 504876 0 0 3 0x14200 bored smr 878 343419 0 0 2 0x14200 zerothread 99209 173303 0 0 3 0x14200 aiodoned aiodoned 16041 262243 0 0 3 0x14200 syncer update 74470 217080 0 0 3 0x14200 cleaner cleaner 27777 22221 0 0 3 0x14200 reaper reaper 1885 242166 0 0 3 0x14200 pgdaemon pagedaemon 81664 307766 0 0 3 0x14200 bored viomb 42315 19314 0 0 3 0x40014200 acpi0 acpi0 9969 184911 0 0 3 0x40014200 idle1 36865 214453 0 0 3 0x14200 bored softnet1 5473 498812 0 0 3 0x14200 bored softnet0 20161 176468 0 0 3 0x14200 bored systqmp 89993 33007 0 0 3 0x14200 bored systq 59196 441024 0 0 3 0x14200 tmoslp softclockmp 82735 189124 0 0 3 0x40014200 tmoslp softclock 7499 72407 0 0 3 0x40014200 idle0 1 312240 0 0 3 0x82 wait init 0 0 -1 0 3 0x10200 scheduler swapper ddb{0}> show all locks Process 95883 (syz-executor) thread 0xffff800034b99268 (348438) exclusive kernel_lock &kernel_lock r = 0 (0xffffffff83a9fc40) #0 witness_lock+0x5f1 stacktrace_save sys/sys/stacktrace.h:37 [inline] #0 witness_lock+0x5f1 sys/kern/subr_witness.c:1160 #1 unp_connect+0x28c sys/kern/uipc_usrreq.c:872 #2 uipc_dgram_send+0x163 sys/kern/uipc_usrreq.c:609 #3 sosend+0x804 sys/kern/uipc_socket.c:-1 #4 sendit+0x5a5 sys/kern/uipc_syscalls.c:785 #5 sys_sendmmsg+0x3f3 sys/kern/uipc_syscalls.c:676 #6 syscall+0xbd4 mi_syscall sys/sys/syscall_mi.h:176 [inline] #6 syscall+0xbd4 sys/arch/amd64/amd64/trap.c:783 #7 Xsyscall+0x128 exclusive rwlock sbufsnd r = 0 (0xffff800010fdf970) #0 witness_lock+0x5f1 stacktrace_save sys/sys/stacktrace.h:37 [inline] #0 witness_lock+0x5f1 sys/kern/subr_witness.c:1160 #1 rw_do_enter_write+0x419 sys/kern/kern_rwlock.c:320 #2 sblock+0xb6 sys/kern/uipc_socket2.c:536 #3 sosend+0x2e9 sys/kern/uipc_socket.c:639 #4 sendit+0x5a5 sys/kern/uipc_syscalls.c:785 #5 sys_sendmmsg+0x3f3 sys/kern/uipc_syscalls.c:676 #6 syscall+0xbd4 mi_syscall sys/sys/syscall_mi.h:176 [inline] #6 syscall+0xbd4 sys/arch/amd64/amd64/trap.c:783 #7 Xsyscall+0x128 Process 36230 (syz-executor) thread 0xffff80003c456030 (18399) exclusive rrwlock inode r = 0 (0xfffffd806f7c2c58) #0 witness_lock+0x5f1 stacktrace_save sys/sys/stacktrace.h:37 [inline] #0 witness_lock+0x5f1 sys/kern/subr_witness.c:1160 #1 rw_do_enter_write+0x419 sys/kern/kern_rwlock.c:320 #2 rrw_enter+0xc6 sys/kern/kern_rwlock.c:621 #3 VOP_LOCK+0xbd sys/kern/vfs_vops.c:527 #4 vn_lock+0xa4 sys/kern/vfs_vnops.c:576 #5 vn_write+0x18f sys/kern/vfs_vnops.c:411 #6 dofilewritev+0x2bd sys/kern/sys_generic.c:384 #7 sys_write+0xa2 sys/kern/sys_generic.c:300 #8 syscall+0xbd4 mi_syscall sys/sys/syscall_mi.h:176 [inline] #8 syscall+0xbd4 sys/arch/amd64/amd64/trap.c:783 #9 Xsyscall+0x128 Process 22449 (syz-executor) thread 0xffff8000fffef240 (34302) exclusive rrwlock inode r = 0 (0xfffffd807431e490) #0 witness_lock+0x5f1 stacktrace_save sys/sys/stacktrace.h:37 [inline] #0 witness_lock+0x5f1 sys/kern/subr_witness.c:1160 #1 rw_do_enter_write+0x419 sys/kern/kern_rwlock.c:320 #2 rrw_enter+0xc6 sys/kern/kern_rwlock.c:621 #3 VOP_LOCK+0xbd sys/kern/vfs_vops.c:527 #4 ufs_ihashins+0x4f ufs_ihash sys/ufs/ufs/ufs_ihash.c:-1 [inline] #4 ufs_ihashins+0x4f sys/ufs/ufs/ufs_ihash.c:159 #5 ffs_vget+0x187 sys/ufs/ffs/ffs_vfsops.c:1232 #6 ffs_inode_alloc+0x279 sys/ufs/ffs/ffs_alloc.c:393 #7 ufs_mkdir+0xfc sys/ufs/ufs/ufs_vnops.c:1112 #8 VOP_MKDIR+0x101 sys/kern/vfs_vops.c:394 #9 domkdirat+0x179 sys/kern/vfs_syscalls.c:3143 #10 syscall+0xb17 mi_syscall sys/sys/syscall_mi.h:176 [inline] #10 syscall+0xb17 sys/arch/amd64/amd64/trap.c:783 #11 Xsyscall+0x128 exclusive rrwlock inode r = 0 (0xfffffd806ce17c48) #0 witness_lock+0x5f1 stacktrace_save sys/sys/stacktrace.h:37 [inline] #0 witness_lock+0x5f1 sys/kern/subr_witness.c:1160 #1 rw_do_enter_write+0x419 sys/kern/kern_rwlock.c:320 #2 rrw_enter+0xc6 sys/kern/kern_rwlock.c:621 #3 VOP_LOCK+0xbd sys/kern/vfs_vops.c:527 #4 vn_lock+0xa4 sys/kern/vfs_vnops.c:576 #5 vfs_lookup+0x10f sys/kern/vfs_lookup.c:431 #6 namei+0x7c5 sys/kern/vfs_lookup.c:250 #7 domkdirat+0x8b sys/kern/vfs_syscalls.c:3128 #8 syscall+0xb17 mi_syscall sys/sys/syscall_mi.h:176 [inline] #8 syscall+0xb17 sys/arch/amd64/amd64/trap.c:783 #9 Xsyscall+0x128 ddb{0}> show malloc Type InUse MemUse HighUse Limit Requests Type Lim devbuf 11063 12161K 12459K 166960K 12891 0 pcb 20 12K 12K 166960K 88 0 rtable 261 8K 9K 166960K 555 0 pf 39 18K 25K 166960K 121 0 ifaddr 43 7K 7K 166960K 77 0 ifgroup 55 2K 2K 166960K 112 0 sysctl 4 1K 9K 166960K 13 0 counters 70 37K 37K 166960K 104 0 ioctlops 0 0K 4K 166960K 1642 0 iov 2 12K 16K 166960K 152 0 mount 1 1K 1K 166960K 1 0 log 1 0K 0K 166960K 5 0 vnodes 1478 93K 93K 166960K 2055 0 UFS quota 1 32K 32K 166960K 1 0 UFS mount 5 36K 36K 166960K 5 0 shm 2 1K 9K 166960K 11 0 VM map 2 1K 1K 166960K 2 0 sem 12 0K 0K 166960K 29 0 dirhash 12 2K 2K 166960K 15 0 ACPI 1692 195K 286K 166960K 12470 0 file desc 16 61K 110K 166960K 636 0 sigio 0 0K 0K 166960K 3 0 proc 12 17K 180K 166960K 735 0 subproc 72 4K 4K 166960K 108 0 NFS srvsock 1 0K 0K 166960K 1 0 NFS daemon 1 16K 16K 166960K 1 0 ip_moptions 0 0K 0K 166960K 51 0 in_multi 99 7K 7K 166960K 153 0 ether_multi 1 0K 0K 166960K 3 0 mrt 1 0K 0K 166960K 28 0 ISOFS mount 1 32K 32K 166960K 1 0 MSDOSFS mount 1 16K 16K 166960K 1 0 ttys 73 334K 334K 166960K 73 0 exec 0 0K 1K 166960K 474 0 fusefs mount 1 32K 32K 166960K 1 0 pfkey data 0 0K 0K 166960K 1 0 tdb 3 0K 0K 166960K 3 0 VM swap 8 62K 64K 166960K 10 0 UVM amap 136 88K 186K 166960K 7488 0 UVM aobj 15 4K 4K 166960K 16 0 pinsyscall 20 40K 103K 166960K 1889 0 memdesc 1 4K 4K 166960K 1 0 crypto data 1 1K 1K 166960K 1 0 ip6_options 0 0K 0K 166960K 21 0 NDP 14 0K 1K 166960K 54 0 temp 76 9112K 9180K 166960K 22500 0 kqueue 1 2K 28K 166960K 107 0 SYN cache 2 16K 16K 166960K 2 0 ddb{0}>