witness: lock order reversal: 1st 0xffff800010fd3398 sbufsnd (&so->so_snd.sb_lock) 2nd 0xfffffd806e07c808 inode (&ip->i_lock) lock order [1] sbufsnd (&so->so_snd.sb_lock) -> [2] inode (&ip->i_lock) lock order data 0xffffffff833fbf4e -> 0xffffffff8336b151 is missing lock order [2] inode (&ip->i_lock) -> [3] sbufrcv (&so->so_rcv.sb_lock) #0 rw_do_enter_write+186 #1 sblock+182 #2 soreceive+637 #3 fifo_read+279 #4 VOP_READ+257 #5 vn_rdwr+347 #6 vndsetcred+161 #7 vndioctl+3579 #8 VOP_IOCTL+172 #9 vn_ioctl+248 #10 sys_ioctl+1652 #11 syscall+3028 #12 Xsyscall+296 lock order [3] sbufrcv (&so->so_rcv.sb_lock) -> [1] sbufsnd (&so->so_snd.sb_lock) #0 rw_do_enter_write+186 #1 sblock+182 #2 sosplice+786 #3 sys_setsockopt+698 #4 syscall+3028 #5 Xsyscall+296 Stopped at db_enter+37: addq $8,%rsp ddb{1}> ddb{1}> set $lines = 0 ddb{1}> set $maxwidth = 0 ddb{1}> show panic the kernel did not panic ddb{1}> trace db_enter() at db_enter+37 witness_checkorder(fffffd806e07c808,9,0) at witness_checkorder+4281 rw_do_enter_write(fffffd806e07c7f0,1) at rw_do_enter_write+186 rrw_enter(fffffd806e07c7f0,1) at rrw_enter+198 VOP_LOCK(fffffd805fc340f0,2001) at VOP_LOCK+163 vn_lock(fffffd805fc340f0,2001) at vn_lock+164 vfs_lookup(ffff80003c429220) at vfs_lookup+284 namei(ffff80003c429220) at namei+1994 unp_connect(ffff800010fd31b0,fffffd805e529d00,ffff80003b7fc2d0) at unp_connect+669 uipc_dgram_send(ffff800010fd31b0,fffffd806051a500,fffffd805e529d00,0) at uipc_dgram_send+355 sosend(ffff800010fd31b0,fffffd805e529d00,ffff80003c4294a8,0,0,e) at sosend+2052 sendit(ffff80003b7fc2d0,6,ffff80003c429628,e,ffff80003c4296e0) at sendit+1445 sys_sendmsg(ffff80003b7fc2d0,ffff80003c429790,ffff80003c4296e0) at sys_sendmsg+582 syscall(ffff80003c429790) at syscall+3028 Xsyscall() at Xsyscall+296 end of kernel end trace frame: 0xdd1e4415840, count: -15 ddb{1}> show registers rdi 0 rsi 0 rbp 18446603337232191136 rbx 0 rdx 0 rcx 18446603337219424976 rax 18446603336919474160 r8 18446603337232190848 r9 9259542123273814144 r10 3407017622381670547 r11 12609429161091206786 r12 18446741324996716160 r13 18446741325005003608 r14 3 r15 18446744071616606847 substchar+60174 rip 18446744071603069125 db_enter+37 cs 8 rflags 582 rsp 18446603337232191120 ss 16 db_enter+37: addq $8,%rsp ddb{1}> show proc PROC (syz-executor) tid=299450 pid=55808 tcnt=2 stat=onproc flags process=0 proc=4000000 runpri=32, usrpri=50, slppri=32, nice=20 wchan=0x0, wmesg=, ps_single=0x0 scnt=0 ecnt=0 forw=0xffffffffffffffff, list=0xffff80003b7fd260,0xffffffff839e0230 process=0xffff80003c4eeb80 user=0xffff80003c424000, vmspace=0xfffffd806c042d78 estcpu=36, cpticks=0, pctcpu=0.0, user=0, sys=0, intr=0 ddb{1}> ps PID TID PPID UID S FLAGS WAIT COMMAND 55808 427637 96636 0 7 0 syz-executor *55808 299450 96636 0 7 0x4000000 syz-executor 93635 464447 93738 0 3 0x80 nanoslp syz-executor 93635 435672 93738 0 3 0x4000080 ttyin syz-executor 93635 464961 93738 0 3 0x4000080 fsleep syz-executor 90739 373549 93344 0 3 0x80 nanoslp syz-executor 90739 14626 93344 0 3 0x4000000 biowait syz-executor 90739 298378 93344 0 3 0x4000080 fsleep syz-executor 14447 239032 69644 0 3 0x80 nanoslp syz-executor 14447 486275 69644 0 3 0x4000080 lockf syz-executor 14447 210441 69644 0 3 0x4000080 fsleep syz-executor 78614 448178 20579 0 3 0x80 nanoslp syz-executor 78614 363765 20579 0 3 0x4000080 kqsel syz-executor 78614 88306 20579 0 3 0x4000080 fsleep syz-executor 61355 524279 1 0 3 0x100083 ttyopn getty 88554 396298 12264 0 3 0x82 nanoslp syz-executor 69644 412046 12264 0 3 0x82 nanoslp syz-executor 93738 481913 12264 0 3 0x82 nanoslp syz-executor 98058 192953 12264 0 3 0x82 nanoslp syz-executor 93344 296864 12264 0 3 0x82 nanoslp syz-executor 20579 155747 12264 0 3 0x82 nanoslp syz-executor 70633 15888 12264 0 3 0x82 nanoslp syz-executor 96636 354709 12264 0 3 0x82 nanoslp syz-executor 12264 303149 94033 0 3 0x82 kqread syz-executor 94033 492327 29878 0 3 0x10008a sigsusp ksh 29878 228116 7133 0 3 0x98 kqread sshd-session 7133 192613 81582 0 3 0x92 kqread sshd-session 81582 473717 1 0 3 0x88 kqread sshd 30830 436309 44494 74 3 0x1100092 bpf pflogd 44494 451550 1 0 3 0x80 sbwait pflogd 41851 484095 43326 73 3 0x1100090 kqread syslogd 43326 430241 1 0 3 0x100082 sbwait syslogd 38292 129457 1 0 3 0x100080 kqread resolvd 6357 103006 32013 77 3 0x100092 kqread dhcpleased 18788 129943 32013 77 3 0x100092 kqread dhcpleased 32013 371383 1 0 3 0x80 kqread dhcpleased 15070 98226 0 0 3 0x14200 bored smr 83646 51009 0 0 3 0x14200 pgzero zerothread 54123 353624 0 0 3 0x14200 aiodoned aiodoned 6666 447009 0 0 3 0x14200 syncer update 28208 181850 0 0 3 0x14200 cleaner cleaner 14590 162544 0 0 3 0x14200 reaper reaper 42543 211329 0 0 3 0x14200 pgdaemon pagedaemon 67305 240427 0 0 3 0x14200 bored viomb 47975 281024 0 0 3 0x40014200 acpi0 acpi0 84532 24415 0 0 3 0x40014200 idle1 95208 29911 0 0 3 0x14200 bored softnet1 65654 293516 0 0 2 0x14200 softnet0 56455 360728 0 0 3 0x14200 bored systqmp 11495 493514 0 0 3 0x14200 bored systq 53531 28990 0 0 3 0x14200 tmoslp softclockmp 9164 349494 0 0 3 0x40014200 tmoslp softclock 71191 507942 0 0 3 0x40014200 idle0 1 294177 0 0 3 0x80082 wait init 0 0 -1 0 3 0x10010200 scheduler swapper ddb{1}> show all locks Process 55808 (syz-executor) thread 0xffff80003b7fc2d0 (299450) exclusive kernel_lock &kernel_lock r = 0 (0xffffffff8389da60) #0 witness_lock+1521 #1 unp_connect+652 #2 uipc_dgram_send+355 #3 sosend+2052 #4 sendit+1445 #5 sys_sendmsg+582 #6 syscall+3028 #7 Xsyscall+296 exclusive rwlock sbufsnd r = 0 (0xffff800010fd3398) #0 witness_lock+1521 #1 rw_do_enter_write+1049 #2 sblock+182 #3 sosend+745 #4 sendit+1445 #5 sys_sendmsg+582 #6 syscall+3028 #7 Xsyscall+296 ddb{1}> show malloc Type InUse MemUse HighUse Limit Requests Type Lim devbuf 10261 11099K 11330K 166960K 14385 0 pcb 18 16K 17K 166960K 592 0 rtable 198 14K 14K 166960K 689 0 pf 35 17K 20K 166960K 319 0 ifaddr 34 6K 8K 166960K 168 0 ifgroup 58 2K 3K 166960K 297 0 sysctl 4 1K 9K 166960K 22 0 counters 68 36K 38K 166960K 400 0 ioctlops 0 0K 4K 166960K 2124 0 iov 0 0K 28K 166960K 119 0 mount 1 1K 1K 166960K 1 0 log 0 0K 0K 166960K 4 0 vnodes 1448 91K 91K 166960K 3302 0 UFS quota 1 32K 32K 166960K 1 0 UFS mount 5 36K 36K 166960K 5 0 shm 2 1K 9K 166960K 27 0 VM map 2 1K 1K 166960K 2 0 sem 37 6K 6K 166960K 54 0 dirhash 12 2K 2K 166960K 51 0 ACPI 1692 195K 286K 166960K 12470 0 file desc 18 65K 240K 166960K 2249 0 sigio 1 0K 0K 166960K 45 0 proc 72 115K 180K 166960K 778 0 subproc 72 4K 4K 166960K 81 0 NFS srvsock 1 0K 0K 166960K 1 0 NFS daemon 1 16K 16K 166960K 1 0 ip_moptions 0 0K 0K 166960K 400 0 in_multi 56 4K 7K 166960K 269 0 ether_multi 1 0K 0K 166960K 39 0 mrt 1 0K 0K 166960K 19 0 ISOFS mount 1 32K 32K 166960K 1 0 MSDOSFS mount 1 16K 16K 166960K 1 0 ttys 253 1129K 1129K 166960K 253 0 exec 0 0K 1K 166960K 601 0 fusefs mount 1 32K 32K 166960K 1 0 pfkey data 0 0K 0K 166960K 4 0 tdb 3 0K 0K 166960K 3 0 VM swap 8 62K 64K 166960K 10 0 UVM amap 279 154K 170K 166960K 21920 0 UVM aobj 5 2K 2K 166960K 5 0 pinsyscall 43 86K 103K 166960K 3334 0 memdesc 1 4K 4K 166960K 1 0 crypto data 1 1K 1K 166960K 1 0 ip6_options 2 0K 0K 166960K 153 0 NDP 12 0K 2K 166960K 115 0 temp 84 8660K 8740K 166960K 131039 0 kqueue 15 24K 32K 166960K 469 0 SYN cache 2 16K 16K 166960K 2 0 ddb{1}> show all pools Name Size Requests Fail Releases Pgreq Pgrel Npage Hiwat Minpg Maxpg Idle plcache 128 26 0 0 1 0 1 1 0 8 0 rtpcb 120 284 0 281 4 1 3 3 0 8 2 rtentry 176 199 0 135 6 1 5 6 0 8 0 unpcb 144 1690 0 1672 16 14 2 8 0 8 1 syncache 336 6 0 6 3 3 0 1 0 8 0 tcpqe 32 1 0 1 1 1 0 1 0 8 0 tcpcb 736 1008 0 1002 24 17 7 7 0 8 6 arp 136 28 0 18 1 0 1 1 0 8 0 inpcb 328 2904 0 2890 19 12 7 7 0 8 5 nd6 152 39 0 26 2 0 2 2 0 8 0 pkpcb 40 18 0 18 6 5 1 1 0 8 1 kcovpl 48 9 0 1 1 0 1 1 0 8 0 mppekey 1024 1 0 1 1 1 0 1 0 8 0 ppxss 1192 140 0 140 1 0 1 1 0 8 1 pppxif 1504 33 0 33 5 4 1 1 0 8 1 pffrag 232 24 0 15 1 0 1 1 0 482 0 pffrnode 88 20 0 12 1 0 1 1 0 8 0 pffrent 40 36 0 26 1 0 1 1 0 8 0 pfosfp 40 1428 0 1005 5 0 5 5 0 8 0 pfosfpen 112 1428 0 714 21 0 21 21 0 8 0 pfstitem 24 145 0 82 1 0 1 1 0 8 0 pfstkey 128 145 0 82 4 0 4 4 0 8 0 pfstate 384 145 0 82 10 0 10 10 0 8 0 pfrule 1344 21 0 16 2 1 1 2 0 8 0 rttmr 136 2 0 2 2 1 1 1 0 8 1 art_heap8 4096 6 0 0 6 0 6 6 0 8 0 art_heap4 256 1065 0 749 34 11 23 31 0 8 1 art_table 40 1071 0 749 5 0 5 5 0 8 0 art_node 32 197 0 140 1 0 1 1 0 8 0 sysvmsgpl 40 15 0 9 1 0 1 1 0 8 0 semupl 112 3 0 3 3 3 0 1 0 8 0 semapl 112 46 0 11 2 1 1 2 0 8 0 shmpl 112 2 0 0 1 0 1 1 0 8 0 dirhash 1024 43 0 26 3 0 3 3 0 8 0 dino2pl 256 5606 0 4094 96 0 96 96 0 8 0 ffsino 296 5606 0 4094 118 0 118 118 0 8 0 nchpl 144 8732 0 7018 64 0 64 64 0 8 0 rtmask 32 28 0 28 5 4 1 1 0 8 1 vnodes 216 5926 0 0 330 0 330 330 0 8 0 namei 1024 29270 0 29269 2 1 1 2 0 8 0 percpumem 16 215 0 166 1 0 1 1 0 8 0 kstatmem 264 196 0 166 4 1 3 3 0 8 0 scsiplug 72 6 0 6 4 3 1 1 0 8 1 scxspl 216 53542 0 53541 15 13 2 8 1 8 1 plimitpl 152 575 0 558 1 0 1 1 0 8 0 sigapl 424 2507 0 2459 9 2 7 8 0 8 0 knotepl 120 841 0 0 24 0 24 24 0 8 0 kqueuepl 224 1041 0 1028 13 8 5 6 0 8 3 pipepl 344 493 0 465 9 6 3 9 0 8 0 fdescpl 528 2463 0 2431 3 0 3 3 0 8 0 filepl 160 17575 0 17325 34 16 18 20 0 8 7 lockfpl 104 1027 0 1023 2 1 1 2 0 8 0 lockfspl 48 452 0 449 1 0 1 1 0 8 0 sessionpl 144 26 0 17 1 0 1 1 0 8 0 pgrppl 48 87 0 70 1 0 1 1 0 8 0 ucredpl 104 2721 0 2708 1 0 1 1 0 8 0 zombiepl 144 2462 0 2459 1 0 1 1 0 8 0 processpl 1232 2507 0 2459 7 2 5 6 0 8 0 procpl 664 6152 0 6095 8 2 6 8 0 8 0 sosppl 176 30 0 30 4 3 1 1 0 8 1 sockpl 752 4977 0 4942 50 39 11 23 0 8 6 mcl64k 65536 31 0 0 4 0 4 4 0 8 0 mcl16k 16384 2 0 0 1 0 1 1 0 8 0 mcl12k 12288 3 0 0 1 0 1 1 0 8 0 mcl9k 9216 1 0 0 1 0 1 1 0 8 0 mcl8k 8192 5 0 0 1 0 1 1 0 8 0 mcl4k 4096 134 0 0 17 0 17 17 0 8 0 mcl2k2 2112 1 0 0 1 0 1 1 0 8 0 mcl2k 2048 48 0 0 6 0 6 6 0 8 0 mtagpl 96 73 0 0 2 0 2 2 0 8 0 mbufpl 256 368 0 0 20 0 20 20 0 8 0 bufpl 280 20766 0 14629 439 0 439 439 0 8 0 anonpl 32 14822 0 0 121 1 120 120 0 246 0 amapchunkpl 152 75201 0 74566 48 20 28 39 0 158 3 amappl16 200 7870 0 7827 64 51 13 31 0 8 8 amappl15 192 7 0 7 1 1 0 1 0 8 0 amappl14 184 6 0 6 3 3 0 1 0 8 0 amappl13 176 437 0 436 1 0 1 1 0 8 0 amappl12 168 2835 0 2791 3 0 3 3 0 8 0 amappl11 160 5 0 4 1 0 1 1 0 8 0 amappl10 152 51 0 37 1 0 1 1 0 8 0 amappl9 144 250 0 250 1 1 0 1 0 8 0 amappl8 136 22 0 19 1 0 1 1 0 8 0 amappl7 128 104 0 102 1 0 1 1 0 8 0 amappl6 120 293 0 279 1 0 1 1 0 8 0 amappl5 112 83 0 71 1 0 1 1 0 8 0 amappl4 104 438 0 407 1 0 1 1 0 8 0 amappl3 96 13322 0 13210 4 1 3 3 0 8 0 amappl2 88 2596 0 2517 2 0 2 2 0 8 0 amappl1 80 18158 0 17569 15 1 14 14 0 8 0 amappl 88 20773 0 20573 5 0 5 5 0 92 0 uvmvnodes 80 5926 0 0 121 0 121 121 0 8 0 dma65536 65536 2 0 2 2 2 0 1 0 8 0 dma8192 8192 1 0 1 1 1 0 1 0 8 0 dma4096 4096 1 0 1 1 1 0 1 0 8 0 dma1024 1024 1 0 0 1 0 1 1 0 8 0 dma256 256 6 0 6 1 1 0 1 0 8 0 dma128 128 254 0 254 2 2 0 1 0 8 0 dma64 64 7 0 7 2 1 1 1 0 8 1 dma32 32 7 0 7 1 1 0 1 0 8 0 dma16 16 19 0 18 1 0 1 1 0 8 0 aobjpl 72 4 0 0 1 0 1 1 0 8 0 uaddrrnd 24 2463 0 2431 1 0 1 1 0 8 0 uaddrbest 32 2 0 0 1 0 1 1 0 8 0 uaddr 24 2463 0 2431 1 0 1 1 0 8 0 vmmpekpl 168 22286 0 22240 3 0 3 3 0 8 0 vmmpepl 168 159125 0 157066 121 21 100 114 0 357 4 vmsppl 488 2462 0 2431 5 0 5 5 0 8 0 rwobjpl 80 47411 0 40496 146 0 146 146 0 8 3 pdppl 4096 4934 0 4862 112 40 72 88 0 8 0 pvpl 32 23666 0 0 193 2 191 192 0 265 0 pmappl 256 2462 0 2431 3 0 3 3 0 8 0 extentpl 40 45 0 27 1 0 1 1 0 8 0 phpool 112 339 0 78 8 0 8 8 0 8 0 ddb{1}> machine ddbcpu 0 Stopped at x86_ipi_db+39: addq $8,%rsp ddb{0}> trace x86_ipi_db(ffffffff8388cff0) at x86_ipi_db+39 x86_ipi_handler() at x86_ipi_handler+217 Xresume_lapic_ipi() at Xresume_lapic_ipi+39 __mp_lock(ffffffff8389d858) at __mp_lock+409 intr_handler(ffff80003c475440,ffff80000006ac00) at intr_handler+233 Xintr_ioapic_edge16_untramp() at Xintr_ioapic_edge16_untramp+399 __sanitizer_cov_trace_pc() at __sanitizer_cov_trace_pc+15 intr_handler(ffff80003c4755b0,ffff80000007aa80) at intr_handler+233 Xintr_ioapic_edge23_untramp() at Xintr_ioapic_edge23_untramp+399 end of kernel end trace frame: 0x7479d30f8170, count: -9 ddb{0}> machine ddbcpu 1 Stopped at db_enter+37: addq $8,%rsp ddb{1}> trace db_enter() at db_enter+37 witness_checkorder(fffffd806e07c808,9,0) at witness_checkorder+4281 rw_do_enter_write(fffffd806e07c7f0,1) at rw_do_enter_write+186 rrw_enter(fffffd806e07c7f0,1) at rrw_enter+198 VOP_LOCK(fffffd805fc340f0,2001) at VOP_LOCK+163 vn_lock(fffffd805fc340f0,2001) at vn_lock+164 vfs_lookup(ffff80003c429220) at vfs_lookup+284 namei(ffff80003c429220) at namei+1994 unp_connect(ffff800010fd31b0,fffffd805e529d00,ffff80003b7fc2d0) at unp_connect+669 uipc_dgram_send(ffff800010fd31b0,fffffd806051a500,fffffd805e529d00,0) at uipc_dgram_send+355 sosend(ffff800010fd31b0,fffffd805e529d00,ffff80003c4294a8,0,0,e) at sosend+2052 sendit(ffff80003b7fc2d0,6,ffff80003c429628,e,ffff80003c4296e0) at sendit+1445 sys_sendmsg(ffff80003b7fc2d0,ffff80003c429790,ffff80003c4296e0) at sys_sendmsg+582 syscall(ffff80003c429790) at syscall+3028 Xsyscall() at Xsyscall+296 end of kernel end trace frame: 0xdd1e4415840, count: -15