witness: lock order reversal: 1st 0xffff800010fdd0a8 sbufsnd (&so->so_snd.sb_lock) 2nd 0xfffffd806c375318 inode (&ip->i_lock) lock order [1] sbufsnd (&so->so_snd.sb_lock) -> [2] inode (&ip->i_lock) lock order data 0xffffffff8350b9a9 -> 0xffffffff8347e106 is missing lock order [2] inode (&ip->i_lock) -> [3] sbufrcv (&so->so_rcv.sb_lock) #0 rw_do_enter_write+0xba sys/kern/kern_rwlock.c:234 #1 sblock+0xb6 sys/kern/uipc_socket2.c:536 #2 soreceive+0x27d sys/kern/uipc_socket.c:890 #3 fifo_read+0x117 sys/miscfs/fifofs/fifo_vnops.c:264 #4 VOP_READ+0x101 sys/kern/vfs_vops.c:227 #5 vn_rdwr+0x15b sys/kern/vfs_vnops.c:-1 #6 vndsetcred+0xa1 sys/dev/vnd.c:685 #7 vndioctl+0xdfc sys/dev/vnd.c:486 #8 VOP_IOCTL+0xac sys/kern/vfs_vops.c:264 #9 vn_ioctl+0xf8 sys/kern/vfs_vnops.c:537 #10 sys_ioctl+0x674 sys/kern/sys_generic.c:-1 #11 syscall+0xbd4 mi_syscall sys/sys/syscall_mi.h:176 [inline] #11 syscall+0xbd4 sys/arch/amd64/amd64/trap.c:783 #12 Xsyscall+0x128 lock order [3] sbufrcv (&so->so_rcv.sb_lock) -> [1] sbufsnd (&so->so_snd.sb_lock) #0 rw_do_enter_write+0xba sys/kern/kern_rwlock.c:234 #1 sblock+0xb6 sys/kern/uipc_socket2.c:536 #2 sosplice+0x312 sys/kern/uipc_socket.c:1347 #3 sys_setsockopt+0x2ba sys/kern/uipc_syscalls.c:1226 #4 syscall+0xbd4 mi_syscall sys/sys/syscall_mi.h:176 [inline] #4 syscall+0xbd4 sys/arch/amd64/amd64/trap.c:783 #5 Xsyscall+0x128 Stopped at db_enter+0x25: addq $0x8,%rsp ddb{0}> ddb{0}> set $lines = 0 ddb{0}> set $maxwidth = 0 ddb{0}> show panic the kernel did not panic ddb{0}> trace db_enter() at db_enter+0x25 sys/arch/amd64/amd64/db_interface.c:438 witness_checkorder(fffffd806c375318,9,0) at witness_checkorder+0x10d1 sys/kern/subr_witness.c:-1 rw_do_enter_write(fffffd806c375300,1) at rw_do_enter_write+0xba sys/kern/kern_rwlock.c:234 rrw_enter(fffffd806c375300,1) at rrw_enter+0xc6 sys/kern/kern_rwlock.c:621 VOP_LOCK(fffffd806c57cb30,2001) at VOP_LOCK+0xbd sys/kern/vfs_vops.c:527 vn_lock(fffffd806c57cb30,2001) at vn_lock+0xa4 sys/kern/vfs_vnops.c:576 vfs_lookup(ffff800034394060) at vfs_lookup+0x10f sys/kern/vfs_lookup.c:431 namei(ffff800034394060) at namei+0x7c5 sys/kern/vfs_lookup.c:250 unp_connect(ffff800010fdcec0,fffffd807b0f7d00,ffff8000fffe4d20) at unp_connect+0x29d sys/kern/uipc_usrreq.c:872 uipc_dgram_send(ffff800010fdcec0,fffffd806d0f9200,fffffd807b0f7d00,0) at uipc_dgram_send+0x163 sys/kern/uipc_usrreq.c:609 sosend(ffff800010fdcec0,fffffd807b0f7d00,ffff8000343942e8,0,0,402) at sosend+0x804 sys/kern/uipc_socket.c:-1 sendit(ffff8000fffe4d20,5,ffff800034394460,402,ffff800034394498) at sendit+0x5a5 sys/kern/uipc_syscalls.c:785 sys_sendmmsg(ffff8000fffe4d20,ffff800034394610,ffff800034394560) at sys_sendmmsg+0x3f3 sys/kern/uipc_syscalls.c:676 syscall(ffff800034394610) at syscall+0xbd4 mi_syscall sys/sys/syscall_mi.h:176 [inline] syscall(ffff800034394610) at syscall+0xbd4 sys/arch/amd64/amd64/trap.c:783 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0x64661315ab0, count: -15 ddb{0}> show registers rdi 0 rsi 0 rbp 0xffff800034393cd0 rbx 0 rdx 0 rcx 0xffff8000fffe4d20 rax 0xffffffff838d4ff0 cpu_info_full_primary+0x1ff0 r8 0xffff800034393bb0 r9 0x8080808080808080 r10 0x11ed486ea312e5e8 r11 0x714e1c1404da2d32 r12 0xfffffd80040aa8c0 r13 0xfffffd8004893450 r14 0x3 r15 0xffffffff83512289 substchar+0xfc85 rip 0xffffffff831a0d75 db_enter+0x25 cs 0x8 rflags 0x246 rsp 0xffff800034393cc0 ss 0x10 db_enter+0x25: addq $0x8,%rsp ddb{0}> show proc PROC (syz-executor) tid=343076 pid=51885 tcnt=4 stat=onproc flags process=10 proc=4000000 runpri=32, usrpri=50, slppri=32, nice=20 wchan=0x0, wmesg=, ps_single=0x0 scnt=0 ecnt=0 forw=0xffffffffffffffff, list=0xffff8000fffe4558,0xffff8000fffe5a28 process=0xffff8000fffe1830 user=0xffff80003438f000, vmspace=0xfffffd800b0633d0 estcpu=36, cpticks=0, pctcpu=0.0, user=0, sys=0, intr=0 ddb{0}> ps PID TID PPID UID S FLAGS WAIT COMMAND 51885 126838 30791 60929 2 0x10 syz-executor *51885 343076 30791 60929 7 0x4000010 syz-executor 51885 439595 30791 60929 3 0x4000090 fsleep syz-executor 51885 183685 30791 60929 3 0x4000090 fsleep syz-executor 22952 156946 55898 0 2 0 syz-executor 22952 421405 55898 0 3 0x4000080 fsleep syz-executor 69569 87963 45024 0 2 0 syz-executor 69569 20277 45024 0 3 0x4000080 fsleep syz-executor 10367 37179 9394 0 2 0 syz-executor 10367 286772 9394 0 3 0x4000080 fsleep syz-executor 95025 449403 82251 0 3 0x80 nanoslp syz-executor 95025 356867 82251 0 3 0x4000080 ttyretype syz-executor 95025 57649 82251 0 3 0x4000080 fsleep syz-executor 33996 451980 99120 60929 2 0x10 syz-executor 33996 397061 99120 60929 3 0x4000090 msgwait syz-executor 33996 411510 99120 60929 3 0x4000090 fsleep syz-executor 34327 362664 86407 0 2 0 syz-executor 34327 115449 86407 0 3 0x4000080 fsleep syz-executor 10724 136643 84721 0 3 0x80 nanoslp syz-executor 10724 285741 84721 0 3 0x4000080 kqpoll syz-executor 10724 68540 84721 0 3 0x4000080 fsleep syz-executor 45024 507158 35764 0 3 0x82 nanoslp syz-executor 9394 271116 35764 0 3 0x82 nanoslp syz-executor 99120 307466 35764 0 3 0x82 nanoslp syz-executor 86407 326683 35764 0 3 0x82 nanoslp syz-executor 55898 285872 35764 0 3 0x82 nanoslp syz-executor 84721 159697 35764 0 3 0x82 nanoslp syz-executor 30791 428987 35764 0 3 0x82 nanoslp syz-executor 82251 225234 35764 0 3 0x82 nanoslp syz-executor 35764 150710 3755 0 3 0x82 kqread syz-executor 3755 404473 69019 0 3 0x10008a sigsusp ksh 69019 295624 15770 0 3 0x98 kqread sshd-session 15770 144881 94043 0 3 0x92 kqread sshd-session 3115 370420 1 0 3 0x100083 ttyin getty 94043 419462 1 0 3 0x88 kqread sshd 18913 97282 90051 74 3 0x1100092 bpf pflogd 90051 460360 1 0 3 0x80 sbwait pflogd 29687 131281 5462 73 3 0x1100090 kqread syslogd 5462 269858 1 0 3 0x100082 sbwait syslogd 89549 416822 1 0 3 0x100080 kqread resolvd 48291 250323 14549 77 3 0x100092 kqread dhcpleased 17442 279106 14549 77 3 0x100092 kqread dhcpleased 14549 112090 1 0 3 0x80 kqread dhcpleased 88786 151299 0 0 3 0x14200 bored smr 58233 233303 0 0 2 0x14200 zerothread 10733 194073 0 0 3 0x14200 aiodoned aiodoned 19567 45889 0 0 3 0x14200 syncer update 9852 60115 0 0 3 0x14200 cleaner cleaner 34159 93420 0 0 3 0x14200 reaper reaper 43363 382855 0 0 3 0x14200 pgdaemon pagedaemon 91226 12318 0 0 3 0x14200 bored viomb 65103 13715 0 0 3 0x40014200 acpi0 acpi0 72520 66047 0 0 7 0x40014200 idle1 69047 113737 0 0 3 0x14200 bored softnet1 19412 231649 0 0 3 0x14200 bored softnet0 95318 461620 0 0 3 0x14200 bored systqmp 62745 326758 0 0 3 0x14200 bored systq 78261 351279 0 0 3 0x14200 tmoslp softclockmp 78069 409324 0 0 3 0x40014200 tmoslp softclock 4068 150333 0 0 3 0x40014200 idle0 1 504548 0 0 3 0x82 wait init 0 0 -1 0 3 0x10200 scheduler swapper ddb{0}> show all locks Process 51885 (syz-executor) thread 0xffff8000fffe4d20 (343076) exclusive kernel_lock &kernel_lock r = 0 (0xffffffff83a9fc40) #0 witness_lock+0x5f1 stacktrace_save sys/sys/stacktrace.h:37 [inline] #0 witness_lock+0x5f1 sys/kern/subr_witness.c:1160 #1 unp_connect+0x28c sys/kern/uipc_usrreq.c:872 #2 uipc_dgram_send+0x163 sys/kern/uipc_usrreq.c:609 #3 sosend+0x804 sys/kern/uipc_socket.c:-1 #4 sendit+0x5a5 sys/kern/uipc_syscalls.c:785 #5 sys_sendmmsg+0x3f3 sys/kern/uipc_syscalls.c:676 #6 syscall+0xbd4 mi_syscall sys/sys/syscall_mi.h:176 [inline] #6 syscall+0xbd4 sys/arch/amd64/amd64/trap.c:783 #7 Xsyscall+0x128 exclusive rwlock sbufsnd r = 0 (0xffff800010fdd0a8) #0 witness_lock+0x5f1 stacktrace_save sys/sys/stacktrace.h:37 [inline] #0 witness_lock+0x5f1 sys/kern/subr_witness.c:1160 #1 rw_do_enter_write+0x419 sys/kern/kern_rwlock.c:320 #2 sblock+0xb6 sys/kern/uipc_socket2.c:536 #3 sosend+0x2e9 sys/kern/uipc_socket.c:639 #4 sendit+0x5a5 sys/kern/uipc_syscalls.c:785 #5 sys_sendmmsg+0x3f3 sys/kern/uipc_syscalls.c:676 #6 syscall+0xbd4 mi_syscall sys/sys/syscall_mi.h:176 [inline] #6 syscall+0xbd4 sys/arch/amd64/amd64/trap.c:783 #7 Xsyscall+0x128 ddb{0}> show malloc Type InUse MemUse HighUse Limit Requests Type Lim devbuf 11077 12293K 12321K 166960K 12559 0 pcb 17 14K 15K 166960K 137 0 rtable 221 7K 8K 166960K 390 0 pf 32 17K 24K 166960K 71 0 ifaddr 40 7K 7K 166960K 54 0 ifgroup 51 2K 2K 166960K 65 0 sysctl 4 1K 9K 166960K 10 0 counters 68 36K 37K 166960K 84 0 ioctlops 0 0K 4K 166960K 1512 0 iov 0 0K 12K 166960K 8 0 mount 1 1K 1K 166960K 1 0 log 0 0K 0K 166960K 4 0 vnodes 1299 82K 82K 166960K 1473 0 UFS quota 1 32K 32K 166960K 1 0 UFS mount 5 36K 36K 166960K 5 0 shm 2 1K 5K 166960K 7 0 VM map 2 1K 1K 166960K 2 0 sem 9 0K 0K 166960K 11 0 dirhash 12 2K 2K 166960K 12 0 ACPI 1692 195K 286K 166960K 12470 0 file desc 18 65K 81K 166960K 300 0 sigio 0 0K 0K 166960K 5 0 proc 73 131K 164K 166960K 568 0 subproc 72 4K 4K 166960K 72 0 NFS srvsock 1 0K 0K 166960K 1 0 NFS daemon 1 16K 16K 166960K 1 0 ip_moptions 0 0K 0K 166960K 34 0 in_multi 88 6K 7K 166960K 108 0 ether_multi 1 0K 0K 166960K 2 0 mrt 2 0K 0K 166960K 9 0 ISOFS mount 1 32K 32K 166960K 1 0 MSDOSFS mount 1 16K 16K 166960K 1 0 ttys 73 334K 334K 166960K 73 0 exec 0 0K 1K 166960K 396 0 fusefs mount 1 32K 32K 166960K 1 0 pfkey data 0 0K 0K 166960K 1 0 tdb 3 0K 0K 166960K 3 0 VM swap 8 62K 64K 166960K 10 0 UVM amap 251 167K 177K 166960K 4723 0 UVM aobj 7 2K 2K 166960K 11 0 pinsyscall 43 86K 100K 166960K 1470 0 memdesc 1 4K 4K 166960K 1 0 crypto data 1 1K 1K 166960K 1 0 ip6_options 0 0K 0K 166960K 13 0 NDP 11 0K 1K 166960K 33 0 temp 42 9103K 9167K 166960K 6295 0 kqueue 14 22K 26K 166960K 54 0 SYN cache 2 16K 16K 166960K 2 0 ddb{0}> show all pools Name Size Requests Fail Releases Pgreq Pgrel Npage Hiwat Minpg Maxpg Idle plcache 128 26 0 0 1 0 1 1 0 8 0 rtpcb 120 46 0 43 1 0 1 1 0 8 0 rtentry 176 115 0 18 6 0 6 6 0 8 0 unpcb 144 109 0 86 1 0 1 1 0 8 0 syncache 336 4 0 4 1 0 1 1 0 8 1 tcpcb 736 42 0 37 1 0 1 1 0 8 0 arp 136 18 0 2 1 0 1 1 0 8 0 inpcb 328 335 0 327 7 0 7 7 0 8 6 nd6 152 26 0 4 1 0 1 1 0 8 0 pkpcb 40 1 0 1 1 0 1 1 0 8 1 kcovpl 48 8 0 0 1 0 1 1 0 8 0 ppxss 1192 7 0 7 1 0 1 1 0 8 1 pffrag 232 33 0 32 1 0 1 1 0 482 0 pffrnode 88 33 0 32 1 0 1 1 0 8 0 pffrent 40 66 0 65 1 0 1 1 0 8 0 pfosfp 40 1428 0 1005 5 0 5 5 0 8 0 pfosfpen 112 1428 0 714 21 0 21 21 0 8 0 pfrktable 1344 1 0 1 1 0 1 1 0 8 1 pfsrclim 320 1 0 1 1 0 1 1 0 8 1 pfanchor 1288 4 0 0 1 0 1 1 0 8 0 pfstitem 24 39 0 0 1 0 1 1 0 8 0 pfstkey 128 39 0 0 2 0 2 2 0 8 0 pfstate 448 39 0 0 5 0 5 5 0 8 0 pfrule 1360 23 0 17 2 1 1 2 0 8 0 art_heap8 4096 1 0 0 1 0 1 1 0 8 0 art_heap4 256 477 0 63 30 0 30 30 0 8 3 art_table 40 478 0 63 5 0 5 5 0 8 0 art_node 32 113 0 25 1 0 1 1 0 8 0 sysvmsgpl 40 8 0 4 1 0 1 1 0 8 0 semapl 64 9 0 2 1 0 1 1 0 8 0 shmpl 112 8 0 4 1 0 1 1 0 8 0 dirhash 1024 17 0 0 3 0 3 3 0 8 0 dino2pl 256 1841 0 372 93 0 93 93 0 8 0 ffsino 296 1841 0 372 114 0 114 114 0 8 0 nchpl 144 2266 0 567 64 0 64 64 0 8 0 rtmask 32 2 0 2 1 0 1 1 0 8 1 vnodes 216 2002 0 0 112 0 112 112 0 8 0 namei 1024 7054 0 7053 1 0 1 1 0 8 0 percpumem 16 57 0 8 1 0 1 1 0 8 0 kstatmem 264 37 0 12 3 0 3 3 0 8 1 scxspl 216 8254 0 8254 10 2 8 8 1 8 8 plimitpl 152 43 0 25 1 0 1 1 0 8 0 sigapl 424 622 0 574 6 0 6 6 0 8 0 knotepl 120 293 0 0 9 0 9 9 0 8 0 kqueuepl 224 99 0 87 3 0 3 3 0 8 2 pipepl 344 135 0 107 3 0 3 3 0 8 0 fdescpl 528 606 0 574 3 0 3 3 0 8 0 filepl 160 2631 0 2406 13 0 13 13 0 8 2 lockfpl 104 116 0 114 1 0 1 1 0 8 0 lockfspl 48 23 0 21 1 0 1 1 0 8 0 sessionpl 144 33 0 23 1 0 1 1 0 8 0 pgrppl 48 43 0 25 1 0 1 1 0 8 0 ucredpl 104 237 0 221 1 0 1 1 0 8 0 zombiepl 144 574 0 574 1 0 1 1 0 8 1 processpl 1232 622 0 574 5 0 5 5 0 8 0 procpl 664 886 0 825 6 0 6 6 0 8 0 sosppl 176 2 0 2 1 0 1 1 0 8 1 sockpl 752 503 0 469 10 0 10 10 0 8 6 mcl64k 65536 6 0 0 1 0 1 1 0 8 0 mcl16k 16384 6 0 0 1 0 1 1 0 8 0 mcl8k 8192 2 0 0 1 0 1 1 0 8 0 mcl4k 4096 123 0 0 16 0 16 16 0 8 0 mcl2k 2048 29 0 0 4 0 4 4 0 8 0 mtagpl 96 4 0 0 1 0 1 1 0 8 0 mbufpl 256 166 0 0 11 0 11 11 0 8 0 bufpl 280 2852 0 101 197 0 197 197 0 8 0 anonpl 32 5405 0 0 44 0 44 44 0 246 0 amapchunkpl 152 13820 0 13287 25 0 25 25 0 158 4 amappl16 200 1850 0 1818 5 2 3 5 0 8 0 amappl15 192 5 0 5 1 1 0 1 0 8 0 amappl14 184 468 0 466 1 0 1 1 0 8 0 amappl13 176 121 0 108 1 0 1 1 0 8 0 amappl12 168 865 0 834 2 0 2 2 0 8 0 amappl11 160 18 0 18 2 2 0 1 0 8 0 amappl10 152 66 0 51 1 0 1 1 0 8 0 amappl9 144 269 0 269 1 1 0 1 0 8 0 amappl8 136 104 0 102 1 0 1 1 0 8 0 amappl7 128 151 0 138 1 0 1 1 0 8 0 amappl6 120 250 0 249 1 0 1 1 0 8 0 amappl5 112 93 0 82 1 0 1 1 0 8 0 amappl4 104 293 0 273 1 0 1 1 0 8 0 amappl3 96 2564 0 2440 4 0 4 4 0 8 0 amappl2 88 549 0 490 2 0 2 2 0 8 0 amappl1 80 10088 0 9482 14 0 14 14 0 8 1 amappl 88 3965 0 3787 5 0 5 5 0 92 0 uvmvnodes 80 105 0 0 3 0 3 3 0 8 0 dma4096 4096 1 0 1 1 1 0 1 0 8 0 dma1024 1024 1 0 0 1 0 1 1 0 8 0 dma256 256 6 0 6 1 1 0 1 0 8 0 dma128 128 253 0 253 1 1 0 1 0 8 0 dma64 64 6 0 6 1 1 0 1 0 8 0 dma32 32 7 0 7 1 1 0 1 0 8 0 dma16 16 18 0 17 1 0 1 1 0 8 0 aobjpl 72 10 0 4 1 0 1 1 0 8 0 uaddrrnd 24 606 0 574 1 0 1 1 0 8 0 uaddrbest 32 2 0 0 1 0 1 1 0 8 0 uaddr 24 606 0 574 1 0 1 1 0 8 0 vmmpekpl 168 6483 0 6447 2 0 2 2 0 8 0 vmmpepl 168 45734 0 43755 87 0 87 87 0 357 0 vmsppl 488 605 0 574 5 0 5 5 0 8 1 rwobjpl 80 15531 0 14469 23 0 23 23 0 8 0 pdppl 4096 1219 0 1148 99 26 73 79 0 8 2 pvpl 32 11724 0 0 95 0 95 95 0 265 0 pmappl 256 605 0 574 3 0 3 3 0 8 0 extentpl 40 45 0 27 1 0 1 1 0 8 0 phpool 112 276 0 29 8 0 8 8 0 8 0 ddb{0}> machine ddbcpu 0 Invalid cpu 0 ddb{0}> trace db_enter() at db_enter+0x25 sys/arch/amd64/amd64/db_interface.c:438 witness_checkorder(fffffd806c375318,9,0) at witness_checkorder+0x10d1 sys/kern/subr_witness.c:-1 rw_do_enter_write(fffffd806c375300,1) at rw_do_enter_write+0xba sys/kern/kern_rwlock.c:234 rrw_enter(fffffd806c375300,1) at rrw_enter+0xc6 sys/kern/kern_rwlock.c:621 VOP_LOCK(fffffd806c57cb30,2001) at VOP_LOCK+0xbd sys/kern/vfs_vops.c:527 vn_lock(fffffd806c57cb30,2001) at vn_lock+0xa4 sys/kern/vfs_vnops.c:576 vfs_lookup(ffff800034394060) at vfs_lookup+0x10f sys/kern/vfs_lookup.c:431 namei(ffff800034394060) at namei+0x7c5 sys/kern/vfs_lookup.c:250 unp_connect(ffff800010fdcec0,fffffd807b0f7d00,ffff8000fffe4d20) at unp_connect+0x29d sys/kern/uipc_usrreq.c:872 uipc_dgram_send(ffff800010fdcec0,fffffd806d0f9200,fffffd807b0f7d00,0) at uipc_dgram_send+0x163 sys/kern/uipc_usrreq.c:609 sosend(ffff800010fdcec0,fffffd807b0f7d00,ffff8000343942e8,0,0,402) at sosend+0x804 sys/kern/uipc_socket.c:-1 sendit(ffff8000fffe4d20,5,ffff800034394460,402,ffff800034394498) at sendit+0x5a5 sys/kern/uipc_syscalls.c:785 sys_sendmmsg(ffff8000fffe4d20,ffff800034394610,ffff800034394560) at sys_sendmmsg+0x3f3 sys/kern/uipc_syscalls.c:676 syscall(ffff800034394610) at syscall+0xbd4 mi_syscall sys/sys/syscall_mi.h:176 [inline] syscall(ffff800034394610) at syscall+0xbd4 sys/arch/amd64/amd64/trap.c:783 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0x64661315ab0, count: -15 ddb{0}> machine ddbcpu 1 Stopped at x86_ipi_db+0x27: addq $0x8,%rsp ddb{1}> trace x86_ipi_db(ffff80002999dff0) at x86_ipi_db+0x27 sys/arch/amd64/amd64/db_interface.c:394 x86_ipi_handler() at x86_ipi_handler+0xd9 sys/arch/amd64/amd64/ipi.c:106 Xresume_lapic_ipi() at Xresume_lapic_ipi+0x27 acpicpu_idle() at acpicpu_idle+0x457 sys/dev/acpi/acpicpu_x86.c:1224 sched_idle(ffff80002999dff0) at sched_idle+0x371 sys/kern/kern_sched.c:192 end trace frame: 0x0, count: -5