witness: lock order reversal: 1st 0xffff800001500258 sbufsnd (&so->so_snd.sb_lock) 2nd 0xfffffd806bd5dc78 inode (&ip->i_lock) lock order [1] sbufsnd (&so->so_snd.sb_lock) -> [2] inode (&ip->i_lock) lock order data 0xffffffff833f0cbb -> 0xffffffff833676f0 is missing lock order [2] inode (&ip->i_lock) -> [3] sbufrcv (&so->so_rcv.sb_lock) #0 rw_do_enter_write+186 #1 sblock+182 #2 soreceive+637 #3 fifo_read+279 #4 VOP_READ+257 #5 vn_rdwr+347 #6 vndsetcred+161 #7 vndioctl+3579 #8 VOP_IOCTL+172 #9 vn_ioctl+248 #10 sys_ioctl+1652 #11 syscall+3028 #12 Xsyscall+296 lock order [3] sbufrcv (&so->so_rcv.sb_lock) -> [1] sbufsnd (&so->so_snd.sb_lock) #0 rw_do_enter_write+186 #1 sblock+182 #2 sosplice+786 #3 sys_setsockopt+698 #4 syscall+3028 #5 Xsyscall+296 Stopped at db_enter+37: addq $8,%rsp ddb{0}> ddb{0}> set $lines = 0 ddb{0}> set $maxwidth = 0 ddb{0}> show panic the kernel did not panic ddb{0}> trace db_enter() at db_enter+37 witness_checkorder(fffffd806bd5dc78,9,0) at witness_checkorder+4281 rw_do_enter_write(fffffd806bd5dc60,1) at rw_do_enter_write+186 rrw_enter(fffffd806bd5dc60,1) at rrw_enter+198 VOP_LOCK(fffffd8062f9bc20,2001) at VOP_LOCK+163 vn_lock(fffffd8062f9bc20,2001) at vn_lock+164 vfs_lookup(ffff80003c4db3a0) at vfs_lookup+284 namei(ffff80003c4db3a0) at namei+1994 unp_connect(ffff800001500070,fffffd806bf32c00,ffff800032fe4a80) at unp_connect+669 uipc_dgram_send(ffff800001500070,fffffd806bf50100,fffffd806bf32c00,0) at uipc_dgram_send+355 sosend(ffff800001500070,fffffd806bf32c00,ffff80003c4db628,0,0,e) at sosend+2052 sendit(ffff800032fe4a80,3,ffff80003c4db7a8,e,ffff80003c4db860) at sendit+1445 sys_sendmsg(ffff800032fe4a80,ffff80003c4db910,ffff80003c4db860) at sys_sendmsg+582 syscall(ffff80003c4db910) at syscall+3028 Xsyscall() at Xsyscall+296 end of kernel end trace frame: 0x73567157b00, count: -15 ddb{0}> show registers rdi 0 rsi 524288 acpi_pdirpa+441969 rbp 18446603337232920608 rbx 0 rdx 18446603336242467072 rcx 18446603337076722304 rax 524287 acpi_pdirpa+441968 r8 18446603337232920320 r9 9259542123273814144 r10 5501977492370387713 r11 9582357806786761310 r12 18446741324996724352 r13 18446741325005013704 r14 3 r15 18446744071616612849 substchar+41884 rip 18446744071579008149 db_enter+37 cs 8 rflags 582 rsp 18446603337232920592 ss 16 db_enter+37: addq $8,%rsp ddb{0}> show proc PROC (syz-executor) tid=98714 pid=2092 tcnt=2 stat=onproc flags process=0 proc=4000000 runpri=32, usrpri=50, slppri=32, nice=20 wchan=0x0, wmesg=, ps_single=0x0 scnt=0 ecnt=0 forw=0xffffffffffffffff, list=0xffff800032fe4fb0,0xffffffff839ef390 process=0xffff80003c4b26a8 user=0xffff80003c4d6000, vmspace=0xfffffd800b027000 estcpu=36, cpticks=0, pctcpu=0.0, user=0, sys=0, intr=0 ddb{0}> ps PID TID PPID UID S FLAGS WAIT COMMAND 2092 473634 39824 0 2 0 syz-executor * 2092 98714 39824 0 7 0x4000000 syz-executor 84771 103224 24601 0 2 0 syz-executor 73664 126595 25599 0 2 0 syz-executor 73664 352512 25599 0 3 0x4000080 fsleep syz-executor 40785 16483 41231 0 2 0 syz-executor 40785 239073 41231 0 3 0x4000080 fsleep syz-executor 99734 305190 59324 0 2 0 syz-executor 99734 180232 59324 0 3 0x4000080 fsleep syz-executor 74949 447902 44520 0 2 0 syz-executor 74949 300281 44520 0 3 0x4000080 fsleep syz-executor 77203 10680 11881 0 2 0 syz-executor 77203 462836 11881 0 3 0x4000080 fsleep syz-executor 77203 156028 11881 0 3 0x4000080 fsleep syz-executor 64555 40866 48808 0 3 0x3000 suspend syz-executor 64555 378695 48808 0 3 0x4081000 biowait syz-executor 64555 218434 48808 0 3 0x4081000 inode syz-executor 48568 367286 0 0 3 0x14200 acct acct 24601 254702 72029 0 2 0x2 syz-executor 41231 282869 72029 0 3 0x82 nanoslp syz-executor 25599 77375 72029 0 3 0x82 nanoslp syz-executor 11881 316127 72029 0 3 0x82 nanoslp syz-executor 44520 102531 72029 0 3 0x82 nanoslp syz-executor 59324 265245 72029 0 3 0x82 nanoslp syz-executor 39824 247615 72029 0 3 0x82 nanoslp syz-executor 48808 348987 72029 0 3 0x82 wait syz-executor 72029 71019 70331 0 3 0x82 kqread syz-executor 70331 236632 3062 0 3 0x10008a sigsusp ksh 3062 67816 60843 0 3 0x98 kqread sshd-session 60843 380634 62888 0 3 0x92 kqread sshd-session 81593 52009 1 0 3 0x100083 ttyin getty 62888 232497 1 0 3 0x88 kqread sshd 92989 23052 58229 74 3 0x1100092 bpf pflogd 58229 362508 1 0 3 0x80 sbwait pflogd 53948 314210 4942 73 3 0x1100090 kqread syslogd 4942 465625 1 0 3 0x100082 sbwait syslogd 50070 184508 1 0 3 0x100080 kqread resolvd 77950 99283 12253 77 3 0x100092 kqread dhcpleased 10708 281291 12253 77 3 0x100092 kqread dhcpleased 12253 367452 1 0 3 0x80 kqread dhcpleased 69008 36734 0 0 3 0x14200 bored smr 64278 111238 0 0 2 0x14200 zerothread 99787 49123 0 0 3 0x14200 aiodoned aiodoned 66031 87528 0 0 3 0x14200 syncer update 80258 414013 0 0 3 0x14200 cleaner cleaner 15122 33276 0 0 3 0x14200 reaper reaper 49725 140789 0 0 3 0x14200 pgdaemon pagedaemon 16599 204652 0 0 3 0x14200 bored viomb 79055 16704 0 0 3 0x40014200 acpi0 acpi0 49915 45554 0 0 7 0x40014200 idle1 38290 338591 0 0 3 0x14200 bored softnet1 53522 35197 0 0 3 0x14200 bored softnet0 94331 264269 0 0 3 0x14200 bored systqmp 45600 273901 0 0 3 0x14200 bored systq 77908 436553 0 0 3 0x14200 tmoslp softclockmp 53238 376419 0 0 3 0x40014200 tmoslp softclock 26443 446888 0 0 3 0x40014200 idle0 1 31469 0 0 3 0x82 wait init 0 0 -1 0 3 0x10010200 scheduler swapper ddb{0}> show all locks Process 2092 (syz-executor) thread 0xffff800032fe4a80 (98714) exclusive kernel_lock &kernel_lock r = 0 (0xffffffff8389d228) #0 witness_lock+1521 #1 unp_connect+652 #2 uipc_dgram_send+355 #3 sosend+2052 #4 sendit+1445 #5 sys_sendmsg+582 #6 syscall+3028 #7 Xsyscall+296 exclusive rwlock sbufsnd r = 0 (0xffff800001500258) #0 witness_lock+1521 #1 rw_do_enter_write+1049 #2 sblock+182 #3 sosend+745 #4 sendit+1445 #5 sys_sendmsg+582 #6 syscall+3028 #7 Xsyscall+296 Process 64555 (syz-executor) thread 0xffff8000fffe6a88 (378695) exclusive rrwlock inode r = 0 (0xfffffd807ba7e550) #0 witness_lock+1521 #1 rw_do_enter_write+1049 #2 rrw_enter+198 #3 VOP_LOCK+163 #4 vn_lock+164 #5 vn_write+399 #6 dofilewritev+578 #7 sys_write+162 #8 syscall+3028 #9 Xsyscall+296 ddb{0}> show malloc Type InUse MemUse HighUse Limit Requests Type Lim devbuf 10224 11058K 12339K 166960K 12124 0 pcb 17 13K 13K 166960K 218 0 rtable 206 9K 9K 166960K 480 0 pf 41 18K 20K 166960K 165 0 ifaddr 40 6K 8K 166960K 100 0 ifgroup 63 2K 2K 166960K 187 0 sysctl 4 1K 9K 166960K 17 0 counters 72 37K 38K 166960K 180 0 ioctlops 0 0K 4K 166960K 1632 0 iov 0 0K 16K 166960K 57 0 mount 1 1K 1K 166960K 1 0 log 0 0K 0K 166960K 4 0 vnodes 1460 92K 92K 166960K 2044 0 UFS quota 1 32K 32K 166960K 1 0 UFS mount 5 36K 36K 166960K 5 0 shm 2 1K 9K 166960K 22 0 VM map 2 1K 1K 166960K 2 0 sem 12 0K 0K 166960K 193 0 dirhash 12 2K 2K 166960K 42 0 ACPI 1692 195K 286K 166960K 12470 0 file desc 18 65K 93K 166960K 1151 0 sigio 0 0K 0K 166960K 161 0 proc 73 115K 164K 166960K 630 0 subproc 72 4K 4K 166960K 72 0 NFS srvsock 1 0K 0K 166960K 1 0 NFS daemon 1 16K 16K 166960K 1 0 ip_moptions 0 0K 0K 166960K 275 0 in_multi 83 6K 7K 166960K 142 0 ether_multi 1 0K 0K 166960K 12 0 mrt 2 0K 0K 166960K 10 0 ISOFS mount 1 32K 32K 166960K 1 0 MSDOSFS mount 1 16K 16K 166960K 1 0 ttys 97 440K 440K 166960K 97 0 exec 0 0K 1K 166960K 482 0 fusefs mount 1 32K 32K 166960K 1 0 tdb 3 0K 0K 166960K 3 0 VM swap 8 62K 64K 166960K 10 0 UVM amap 247 160K 187K 166960K 12687 0 UVM aobj 94 4K 4K 166960K 95 0 pinsyscall 43 86K 102K 166960K 2308 0 memdesc 1 4K 4K 166960K 1 0 crypto data 1 1K 1K 166960K 1 0 ip6_options 0 0K 0K 166960K 77 0 NDP 14 0K 1K 166960K 69 0 temp 60 8655K 8719K 166960K 57275 0 kqueue 15 20K 30K 166960K 237 0 SYN cache 2 16K 16K 166960K 2 0 ddb{0}> show all pools Name Size Requests Fail Releases Pgreq Pgrel Npage Hiwat Minpg Maxpg Idle plcache 128 26 0 0 1 0 1 1 0 8 0 rtpcb 120 105 0 102 1 0 1 1 0 8 0 rtentry 176 139 0 58 6 1 5 6 0 8 0 unpcb 144 636 0 617 4 2 2 2 0 8 1 syncache 336 10 0 10 4 3 1 1 0 8 1 tcpqe 32 4 0 4 2 1 1 1 0 8 1 tcpcb 736 569 0 561 7 6 1 7 0 8 0 arp 136 22 0 7 1 0 1 1 0 8 0 inpcb 328 1540 0 1528 17 10 7 8 0 8 5 nd6 152 25 0 5 1 0 1 1 0 8 0 pkpcb 40 15 0 15 3 2 1 1 0 8 1 kcovpl 48 8 0 0 1 0 1 1 0 8 0 ppxss 1192 36 0 36 1 0 1 1 0 8 1 pppxif 1504 7 0 7 4 3 1 1 0 8 1 pfstscr 40 3 0 2 1 0 1 1 0 8 0 pffrag 232 71 0 66 1 0 1 1 0 482 0 pffrnode 88 70 0 66 1 0 1 1 0 8 0 pffrent 40 202 0 197 1 0 1 1 0 8 0 pfosfp 40 1429 0 1005 5 0 5 5 0 8 0 pfosfpen 112 1429 0 714 21 0 21 21 0 8 0 pfrktable 1344 1 0 1 1 1 0 1 0 8 0 pfstitem 24 74 0 21 1 0 1 1 0 8 0 pfstkey 128 79 0 27 2 0 2 2 0 8 0 pfstate 384 76 0 24 6 0 6 6 0 8 0 pfrule 1344 28 0 22 2 1 1 2 0 8 0 rttmr 136 2 0 2 2 1 1 1 0 8 1 art_heap8 4096 2 0 0 2 0 2 2 0 8 0 art_heap4 256 604 0 245 30 3 27 29 0 8 2 art_table 40 606 0 245 5 0 5 5 0 8 0 art_node 32 137 0 64 1 0 1 1 0 8 0 sysvmsgpl 40 11 0 8 1 0 1 1 0 8 0 semupl 112 3 0 3 2 1 1 1 0 8 1 semapl 112 184 0 174 1 0 1 1 0 8 0 shmpl 112 92 0 1 3 0 3 3 0 8 0 dirhash 1024 38 0 21 3 0 3 3 0 8 0 dino2pl 256 3545 0 2037 96 0 96 96 0 8 0 ffsino 296 3545 0 2037 118 0 118 118 0 8 0 nchpl 144 5237 0 3537 64 0 64 64 0 8 0 rtmask 32 6 0 6 2 1 1 1 0 8 1 vnodes 216 3944 0 0 220 0 220 220 0 8 0 namei 1024 17579 0 17578 2 1 1 2 0 8 0 percpumem 16 105 0 54 1 0 1 1 0 8 0 kstatmem 264 112 0 78 5 2 3 3 0 8 0 scsiplug 72 3 0 3 2 2 0 1 0 8 0 scxspl 216 35394 0 35393 11 10 1 8 1 8 0 plimitpl 152 378 0 359 1 0 1 1 0 8 0 sigapl 424 1489 0 1440 8 2 6 8 0 8 0 knotepl 120 592 0 0 18 0 18 18 0 8 0 kqueuepl 224 539 0 529 7 6 1 5 0 8 0 pipepl 344 334 0 306 15 6 9 9 0 8 6 fdescpl 528 1450 0 1418 3 0 3 3 0 8 0 filepl 160 9448 0 9223 24 8 16 17 0 8 5 lockfpl 104 481 0 478 1 0 1 1 0 8 0 lockfspl 48 175 0 172 1 0 1 1 0 8 0 sessionpl 144 28 0 19 1 0 1 1 0 8 0 pgrppl 48 56 0 39 1 0 1 1 0 8 0 ucredpl 104 1380 0 1367 1 0 1 1 0 8 0 zombiepl 144 1441 0 1440 1 0 1 1 0 8 0 processpl 1232 1489 0 1440 6 1 5 6 0 8 0 procpl 664 3349 0 3291 9 3 6 8 0 8 0 sosppl 176 10 0 10 3 2 1 1 0 8 1 sockpl 752 2321 0 2287 25 11 14 14 0 8 10 mcl64k 65536 5 0 0 1 0 1 1 0 8 0 mcl16k 16384 3 0 0 1 0 1 1 0 8 0 mcl12k 12288 2 0 0 1 0 1 1 0 8 0 mcl9k 9216 1 0 0 1 0 1 1 0 8 0 mcl8k 8192 2 0 0 1 0 1 1 0 8 0 mcl4k 4096 131 0 0 17 0 17 17 0 8 0 mcl2k 2048 29 0 0 4 0 4 4 0 8 0 mtagpl 96 6 0 0 1 0 1 1 0 8 0 mbufpl 256 1759 0 0 110 0 110 110 0 8 0 bufpl 280 14006 0 7869 439 0 439 439 0 8 0 anonpl 32 13650 0 0 111 0 111 111 0 246 0 amapchunkpl 152 42789 0 42159 42 11 31 35 0 158 4 amappl16 200 4869 0 4591 51 31 20 34 0 8 3 amappl15 192 7 0 7 1 1 0 1 0 8 0 amappl14 184 7 0 7 1 1 0 1 0 8 0 amappl13 176 430 0 429 1 0 1 1 0 8 0 amappl12 168 1827 0 1785 3 0 3 3 0 8 0 amappl11 160 4 0 4 1 1 0 1 0 8 0 amappl10 152 49 0 35 1 0 1 1 0 8 0 amappl9 144 253 0 253 1 1 0 1 0 8 0 amappl8 136 30 0 27 1 0 1 1 0 8 0 amappl7 128 79 0 77 1 0 1 1 0 8 0 amappl6 120 456 0 442 1 0 1 1 0 8 0 amappl5 112 84 0 72 1 0 1 1 0 8 0 amappl4 104 444 0 414 1 0 1 1 0 8 0 amappl3 96 7379 0 7288 4 1 3 3 0 8 0 amappl2 88 1612 0 1535 2 0 2 2 0 8 0 amappl1 80 14966 0 14384 15 1 14 14 0 8 0 amappl 88 11735 0 11566 5 0 5 5 0 92 0 uvmvnodes 80 3944 0 0 81 0 81 81 0 8 0 dma65536 65536 1 0 1 1 1 0 1 0 8 0 dma8192 8192 1 0 1 1 1 0 1 0 8 0 dma4096 4096 1 0 1 1 1 0 1 0 8 0 dma1024 1024 1 0 0 1 0 1 1 0 8 0 dma256 256 6 0 6 1 1 0 1 0 8 0 dma128 128 254 0 254 2 2 0 1 0 8 0 dma64 64 6 0 6 1 1 0 1 0 8 0 dma32 32 7 0 7 1 1 0 1 0 8 0 dma16 16 18 0 17 1 0 1 1 0 8 0 aobjpl 72 94 0 1 2 0 2 2 0 8 0 uaddrrnd 24 1450 0 1418 1 0 1 1 0 8 0 uaddrbest 32 2 0 0 1 0 1 1 0 8 0 uaddr 24 1450 0 1418 1 0 1 1 0 8 0 vmmpekpl 168 14273 0 14235 3 0 3 3 0 8 0 vmmpepl 168 99970 0 97763 125 21 104 114 0 357 0 vmsppl 488 1449 0 1418 8 3 5 5 0 8 1 rwobjpl 80 32849 0 27642 109 0 109 109 0 8 0 pdppl 4096 2908 0 2836 108 34 74 86 0 8 2 pvpl 32 20784 0 0 168 0 168 168 0 265 0 pmappl 256 1449 0 1418 3 0 3 3 0 8 0 extentpl 40 45 0 27 1 0 1 1 0 8 0 phpool 112 394 0 51 11 0 11 11 0 8 0 ddb{0}> machine ddbcpu 0 Invalid cpu 0 ddb{0}> trace db_enter() at db_enter+37 witness_checkorder(fffffd806bd5dc78,9,0) at witness_checkorder+4281 rw_do_enter_write(fffffd806bd5dc60,1) at rw_do_enter_write+186 rrw_enter(fffffd806bd5dc60,1) at rrw_enter+198 VOP_LOCK(fffffd8062f9bc20,2001) at VOP_LOCK+163 vn_lock(fffffd8062f9bc20,2001) at vn_lock+164 vfs_lookup(ffff80003c4db3a0) at vfs_lookup+284 namei(ffff80003c4db3a0) at namei+1994 unp_connect(ffff800001500070,fffffd806bf32c00,ffff800032fe4a80) at unp_connect+669 uipc_dgram_send(ffff800001500070,fffffd806bf50100,fffffd806bf32c00,0) at uipc_dgram_send+355 sosend(ffff800001500070,fffffd806bf32c00,ffff80003c4db628,0,0,e) at sosend+2052 sendit(ffff800032fe4a80,3,ffff80003c4db7a8,e,ffff80003c4db860) at sendit+1445 sys_sendmsg(ffff800032fe4a80,ffff80003c4db910,ffff80003c4db860) at sys_sendmsg+582 syscall(ffff80003c4db910) at syscall+3028 Xsyscall() at Xsyscall+296 end of kernel end trace frame: 0x73567157b00, count: -15 ddb{0}> machine ddbcpu 1 Stopped at x86_ipi_db+39: addq $8,%rsp ddb{1}> trace x86_ipi_db(ffff8000299edff0) at x86_ipi_db+39 x86_ipi_handler() at x86_ipi_handler+217 Xresume_lapic_ipi() at Xresume_lapic_ipi+39 acpicpu_idle() at acpicpu_idle+1111 sched_idle(ffff8000299edff0) at sched_idle+913 end trace frame: 0x0, count: -5