witness: lock order reversal: 1st 0xffff8000015f7d10 sbufrcv (&so->so_rcv.sb_lock) 2nd 0xfffffd806f1051c8 inode (&ip->i_lock) lock order [1] sbufrcv (&so->so_rcv.sb_lock) -> [2] inode (&ip->i_lock) lock order data 0xffffffff83334578 -> 0xffffffff8338072b is missing lock order [2] inode (&ip->i_lock) -> [1] sbufrcv (&so->so_rcv.sb_lock) #0 rw_do_enter_write+186 #1 sblock+182 #2 soreceive+637 #3 fifo_read+279 #4 VOP_READ+257 #5 vn_rdwr+347 #6 vndsetcred+161 #7 vndioctl+3580 #8 VOP_IOCTL+172 #9 vn_ioctl+248 #10 sys_ioctl+1652 #11 syscall+3028 #12 Xsyscall+296 Stopped at db_enter+37: addq $8,%rsp ddb{1}> ddb{1}> set $lines = 0 ddb{1}> set $maxwidth = 0 ddb{1}> show panic the kernel did not panic ddb{1}> trace db_enter() at db_enter+37 witness_checkorder(fffffd806f1051c8,9,0) at witness_checkorder+4305 rw_do_enter_write(fffffd806f1051b0,1) at rw_do_enter_write+186 rrw_enter(fffffd806f1051b0,1) at rrw_enter+198 VOP_LOCK(fffffd805c45e6c0,2001) at VOP_LOCK+189 vn_lock(fffffd805c45e6c0,2001) at vn_lock+164 vfs_getcwd_common(fffffd805c45e6c0,fffffd805b6cc7b8,0,0,200,0,3f9108398b3bceeb) at vfs_getcwd_common+209 vn_isunder(fffffd805c45e6c0,fffffd805b6cc7b8,ffff8000333defc0) at vn_isunder+86 unp_externalize(fffffd8059dc7100,33,0) at unp_externalize+623 soreceive(ffff8000015f7c20,ffff80003c439568,ffff80003c439518,0,ffff80003c439550,ffff80003c4396dc,f2eb1a85b894eb00) at soreceive+3108 recvit(ffff8000333defc0,9,ffff80003c4396b0,0,ffff80003c439760) at recvit+1035 sys_recvmsg(ffff8000333defc0,ffff80003c439810,ffff80003c439760) at sys_recvmsg+447 syscall(ffff80003c439810) at syscall+3028 Xsyscall() at Xsyscall+296 end of kernel end trace frame: 0x2bb4d1cb190, count: -14 ddb{1}> show registers rdi 0 rsi 524288 acpi_pdirpa+441969 rbp 18446603337232257008 rbx 0 rdx 18446603336243850048 rcx 18446603337080893376 rax 524287 acpi_pdirpa+441968 r8 18446603337232256720 r9 9259542123273814144 r10 1879028908741542892 r11 12217407682369821277 r12 18446741324996876416 r13 18446741325005163352 r14 3 r15 18446744071616710248 substchar+45326 rip 18446744071582932741 db_enter+37 cs 8 rflags 582 rsp 18446603337232256992 ss 16 db_enter+37: addq $8,%rsp ddb{1}> show proc PROC (syz-executor) tid=18918 pid=81233 tcnt=2 stat=onproc flags process=1000000 proc=4000000 runpri=32, usrpri=84, slppri=32, nice=20 wchan=0x0, wmesg=, ps_single=0x0 scnt=0 ecnt=0 forw=0xffffffffffffffff, list=0xffff8000333de2c8,0xffff80003c413790 process=0xffff80003c4179e0 user=0xffff80003c434000, vmspace=0xfffffd806c4729a0 estcpu=36, cpticks=0, pctcpu=0.0, user=0, sys=0, intr=0 ddb{1}> ps PID TID PPID UID S FLAGS WAIT COMMAND 64440 304135 90223 0 2 0 syz-executor 64440 198404 90223 0 2 0x4000000 syz-executor 82148 44762 69729 0 2 0 syz-executor 82148 408172 69729 0 3 0x4000000 biowait syz-executor 82148 443128 69729 0 3 0x4000080 fsleep syz-executor 81233 358589 64733 0 2 0x1000000 syz-executor *81233 18918 64733 0 7 0x5000000 syz-executor 48756 462236 19394 0 2 0 syz-executor 48756 370221 19394 0 3 0x4000080 fsleep syz-executor 48756 51475 19394 0 3 0x4000000 biowait syz-executor 68670 424889 77008 0 2 0 syz-executor 68670 160120 77008 0 3 0x4000080 fsleep syz-executor 68670 87533 77008 0 3 0x4000080 fsleep syz-executor 34519 60604 27958 0 7 0 syz-executor 34519 512511 27958 0 3 0x4000000 biowait syz-executor 34519 278727 27958 0 3 0x4000080 fsleep syz-executor 74544 251981 52440 0 3 0x82 nanoslp syz-executor 69729 349846 52440 0 3 0x82 nanoslp syz-executor 50641 115019 1 0 3 0x100083 ttyopn getty 90223 6979 52440 0 3 0x82 nanoslp syz-executor 64733 125104 52440 0 3 0x82 nanoslp syz-executor 96093 26039 25892 0 3 0x82 sbwait sshd-session 1520 138313 90574 0 3 0x100082 sbwait arp 90574 298144 1 0 3 0x10008a sigsusp sh 27958 410029 52440 0 3 0x82 nanoslp syz-executor 77008 207300 52440 0 3 0x82 nanoslp syz-executor 19394 152267 52440 0 3 0x82 nanoslp syz-executor 99789 456894 52440 0 3 0x82 wait syz-executor 52440 147377 73807 0 3 0x82 kqread syz-executor 73807 10472 7165 0 3 0x10008a sigsusp ksh 7165 286455 75345 0 3 0x98 kqread sshd-session 75345 493896 25892 0 3 0x92 kqread sshd-session 25892 509081 1 0 3 0x88 kqread sshd 62338 225939 62038 74 3 0x1100092 bpf pflogd 62038 165845 1 0 3 0x80 sbwait pflogd 4319 411763 71053 73 3 0x1100090 kqread syslogd 71053 44755 1 0 3 0x100082 sbwait syslogd 93033 117408 1 0 3 0x100080 kqread resolvd 45256 493116 38831 77 3 0x100092 kqread dhcpleased 54265 296713 38831 77 3 0x100092 kqread dhcpleased 38831 462316 1 0 3 0x80 kqread dhcpleased 13897 434844 0 0 3 0x14200 bored smr 31114 125491 0 0 2 0x14200 zerothread 49093 498824 0 0 3 0x14200 aiodoned aiodoned 6888 354034 0 0 3 0x14200 syncer update 3364 421445 0 0 3 0x14200 cleaner cleaner 94108 4868 0 0 3 0x14200 reaper reaper 93616 386394 0 0 3 0x14200 pgdaemon pagedaemon 485 103851 0 0 3 0x14200 bored viomb 81814 211109 0 0 3 0x40014200 acpi0 acpi0 93381 237280 0 0 3 0x40014200 idle1 44437 182712 0 0 3 0x14200 bored softnet1 89483 216924 0 0 2 0x14200 softnet0 86227 365700 0 0 3 0x14200 bored systqmp 62478 191601 0 0 3 0x14200 bored systq 19423 208833 0 0 3 0x14200 tmoslp softclockmp 62348 401504 0 0 3 0x40014200 tmoslp softclock 45899 160561 0 0 3 0x40014200 idle0 1 193960 0 0 3 0x82 wait init 0 0 -1 0 3 0x10010200 scheduler swapper ddb{1}> show all locks Process 82148 (syz-executor) thread 0xffff80003c4127f0 (408172) exclusive rrwlock inode r = 0 (0xfffffd806e996ee0) #0 witness_lock+1521 #1 rw_do_enter_write+1049 #2 rrw_enter+198 #3 VOP_LOCK+189 #4 vn_lock+164 #5 vn_write+399 #6 dofilewritev+578 #7 sys_write+162 #8 syscall+3028 #9 Xsyscall+296 Process 81233 (syz-executor) thread 0xffff8000333defc0 (18918) exclusive kernel_lock &kernel_lock r = 0 (0xffffffff839345c0) #0 witness_lock+1521 #1 unp_externalize+341 #2 soreceive+3108 #3 recvit+1035 #4 sys_recvmsg+447 #5 syscall+3028 #6 Xsyscall+296 exclusive rwlock sbufrcv r = 0 (0xffff8000015f7d10) #0 witness_lock+1521 #1 rw_do_enter_write+1049 #2 sblock+182 #3 soreceive+637 #4 recvit+1035 #5 sys_recvmsg+447 #6 syscall+3028 #7 Xsyscall+296 Process 48756 (syz-executor) thread 0xffff80003c413cb0 (51475) exclusive rrwlock inode r = 0 (0xfffffd806e79ab30) #0 witness_lock+1521 #1 rw_do_enter_write+1049 #2 rrw_enter+198 #3 VOP_LOCK+189 #4 vn_lock+164 #5 sys_fsync+324 #6 syscall+2839 #7 Xsyscall+296 Process 34519 (syz-executor) thread 0xffff8000333df788 (512511) exclusive rrwlock inode r = 0 (0xfffffd806f1052f0) #0 witness_lock+1521 #1 rw_do_enter_write+1049 #2 rrw_enter+198 #3 VOP_LOCK+189 #4 vn_lock+164 #5 vget+674 #6 ufs_ihashget+389 #7 ffs_vget+140 #8 ufs_lookup+6710 #9 VOP_LOOKUP+110 #10 vfs_relookup+212 #11 ufs_rename+5290 #12 VOP_RENAME+311 #13 dorenameat+1015 #14 syscall+2839 #15 Xsyscall+296 exclusive rrwlock inode r = 0 (0xfffffd806e983920) #0 witness_lock+1521 #1 rw_do_enter_write+1049 #2 rrw_enter+198 #3 VOP_LOCK+189 #4 vn_lock+164 #5 vfs_relookup+92 #6 ufs_rename+5290 #7 VOP_RENAME+311 #8 dorenameat+1015 #9 syscall+2839 #10 Xsyscall+296 ddb{1}> show malloc Type InUse MemUse HighUse Limit Requests Type Lim devbuf 11074 12173K 14104K 166960K 15416 0 pcb 18 20K 22K 166960K 693 0 rtable 204 12K 13K 166960K 908 0 pf 34 17K 21K 166960K 360 0 ifaddr 33 7K 9K 166960K 244 0 ifgroup 51 2K 2K 166960K 463 0 sysctl 4 1K 9K 166960K 30 0 counters 68 36K 38K 166960K 512 0 ioctlops 0 0K 4K 166960K 1996 0 iov 0 0K 28K 166960K 302 0 mount 1 1K 1K 166960K 1 0 log 0 0K 0K 166960K 4 0 vnodes 1653 104K 105K 166960K 3921 0 UFS quota 1 32K 32K 166960K 1 0 UFS mount 5 36K 36K 166960K 5 0 shm 2 2K 6K 166960K 33 0 VM map 2 1K 1K 166960K 2 0 sem 12 0K 0K 166960K 264 0 dirhash 12 2K 2K 166960K 57 0 ACPI 1692 195K 286K 166960K 12470 0 file desc 20 73K 244K 166960K 2789 0 sigio 0 0K 0K 166960K 125 0 proc 74 115K 196K 166960K 924 0 subproc 81 5K 5K 166960K 108 0 NFS srvsock 1 0K 0K 166960K 1 0 NFS daemon 1 16K 16K 166960K 1 0 ip_moptions 0 0K 0K 166960K 396 0 in_multi 59 4K 6K 166960K 290 0 ether_multi 1 0K 0K 166960K 39 0 mrt 1 0K 0K 166960K 44 0 ISOFS mount 1 32K 32K 166960K 1 0 MSDOSFS mount 1 16K 16K 166960K 1 0 ttys 121 546K 546K 166960K 121 0 exec 0 0K 1K 166960K 980 0 fusefs mount 1 32K 32K 166960K 1 0 pfkey data 0 0K 0K 166960K 3 0 tdb 3 0K 0K 166960K 3 0 VM swap 8 62K 64K 166960K 10 0 UVM amap 290 188K 207K 166960K 27240 0 UVM aobj 104 17K 17K 166960K 110 0 pinsyscall 48 96K 106K 166960K 4057 0 memdesc 1 4K 4K 166960K 1 0 crypto data 1 1K 1K 166960K 1 0 ip6_options 2 0K 0K 166960K 278 0 NDP 11 0K 1K 166960K 175 0 temp 115 8680K 8808K 166960K 154071 0 kqueue 13 20K 33K 166960K 650 0 SYN cache 2 16K 16K 166960K 2 0 ddb{1}> show all pools Name Size Requests Fail Releases Pgreq Pgrel Npage Hiwat Minpg Maxpg Idle plcache 128 26 0 0 1 0 1 1 0 8 0 rtpcb 120 390 0 386 6 5 1 3 0 8 0 rtentry 176 280 0 207 5 1 4 5 0 8 0 unpcb 144 2452 0 2431 25 21 4 6 0 8 3 syncache 336 13 0 13 6 6 0 1 0 8 0 tcpqe 32 7 0 7 4 4 0 1 0 8 0 tcpcb 736 1078 0 1068 34 32 2 13 0 8 0 arp 136 31 0 19 1 0 1 1 0 8 0 inpcb 328 3772 0 3690 49 42 7 13 0 8 0 nd6 152 44 0 27 1 0 1 1 0 8 0 pkpcb 40 27 0 27 4 4 0 1 0 8 0 kcovpl 48 12 0 3 1 0 1 1 0 8 0 mppekey 1024 2 0 2 2 2 0 1 0 8 0 ppxss 1192 172 0 172 5 4 1 1 0 8 1 pppxif 1504 32 0 32 5 5 0 1 0 8 0 pffrag 232 25 0 15 1 0 1 1 0 482 0 pffrnode 88 20 0 10 1 0 1 1 0 8 0 pffrent 40 44 0 33 1 0 1 1 0 8 0 pfosfp 40 1428 0 1005 5 0 5 5 0 8 0 pfosfpen 112 1428 0 714 21 0 21 21 0 8 0 pfrktable 1344 4 0 4 3 3 0 1 0 8 0 pfanchor 1288 1 0 0 1 0 1 1 0 8 0 pftag 88 2 0 0 1 0 1 1 0 8 0 pfstitem 24 2 0 0 1 0 1 1 0 8 0 pfstkey 128 2 0 0 1 0 1 1 0 8 0 pfstate 448 1 0 0 1 0 1 1 0 8 0 pfrule 1360 3 0 2 1 0 1 1 0 8 0 rttmr 136 11 0 11 6 5 1 1 0 8 1 art_heap8 4096 5 0 0 5 0 5 5 0 8 0 art_heap4 256 1236 0 936 38 16 22 30 0 8 0 art_table 40 1241 0 936 5 0 5 5 0 8 0 art_node 32 276 0 212 1 0 1 1 0 8 0 sysvmsgpl 40 6 0 3 1 0 1 1 0 8 0 semupl 112 1 0 1 1 1 0 1 0 8 0 semapl 112 260 0 250 1 0 1 1 0 8 0 shmpl 112 100 0 4 3 0 3 3 0 8 0 dirhash 1024 47 0 30 3 0 3 3 0 8 0 dino2pl 256 6822 0 5313 96 1 95 96 0 8 0 ffsino 296 6822 0 5313 117 0 117 117 0 8 0 nchpl 144 10670 0 8958 64 0 64 64 0 8 0 rtmask 32 38 0 38 6 5 1 1 0 8 1 vnodes 216 5926 0 0 330 0 330 330 0 8 0 namei 1024 38219 0 38217 6 5 1 2 0 8 0 percpumem 16 271 0 222 1 0 1 1 0 8 0 vcpupl 3968 5 0 0 1 0 1 1 0 8 0 vmpool 848 7 0 2 2 1 1 1 0 8 0 kstatmem 264 298 0 274 5 2 3 3 0 8 1 acpiwqpl 32 3 0 3 1 0 1 1 1 8 1 scsiplug 72 10 0 10 6 6 0 1 0 8 0 scxspl 216 90091 0 90088 12 11 1 5 1 8 0 plimitpl 152 1015 0 997 2 1 1 2 0 8 0 sigapl 424 3084 0 3033 8 1 7 7 0 8 0 knotepl 120 614 0 0 17 0 17 17 0 8 0 kqueuepl 224 1321 0 1312 14 13 1 5 0 8 0 pipepl 344 437 0 406 10 7 3 6 0 8 0 fdescpl 528 3056 0 3021 3 0 3 3 0 8 0 filepl 160 22215 0 21903 46 30 16 21 0 8 3 lockfpl 104 1466 0 1464 4 3 1 2 0 8 0 lockfspl 48 421 0 419 1 0 1 1 0 8 0 sessionpl 144 36 0 26 1 0 1 1 0 8 0 pgrppl 48 90 0 71 1 0 1 1 0 8 0 ucredpl 104 3163 0 3150 1 0 1 1 0 8 0 zombiepl 144 3471 0 3469 1 0 1 1 0 8 0 processpl 1232 3084 0 3033 5 0 5 5 0 8 0 procpl 664 7621 0 7560 7 0 7 7 0 8 0 sosppl 176 17 0 17 5 5 0 1 0 8 0 sockpl 752 6740 0 6632 84 71 13 21 0 8 2 mcl64k 65536 7 0 0 1 0 1 1 0 8 0 mcl16k 16384 2 0 0 1 0 1 1 0 8 0 mcl12k 12288 2 0 0 1 0 1 1 0 8 0 mcl9k 9216 2 0 0 1 0 1 1 0 8 0 mcl8k 8192 34 0 0 5 1 4 5 0 8 0 mcl4k 4096 127 0 0 15 0 15 15 0 8 0 mcl2k2 2112 1 0 0 1 0 1 1 0 8 0 mcl2k 2048 156 0 0 17 0 17 17 0 8 0 mtagpl 96 10 0 0 1 0 1 1 0 8 0 mbufpl 256 1176 0 0 73 0 73 73 0 8 0 bufpl 280 37666 0 31530 439 0 439 439 0 8 0 anonpl 32 25859 0 0 209 0 209 209 0 246 0 amapchunkpl 152 95653 0 94965 76 39 37 40 0 158 7 amappl16 200 14355 0 13969 140 100 40 66 0 8 8 amappl15 192 31 0 31 2 2 0 1 0 8 0 amappl14 184 4 0 4 1 1 0 1 0 8 0 amappl13 176 486 0 484 1 0 1 1 0 8 0 amappl12 168 3478 0 3429 3 0 3 3 0 8 0 amappl11 160 7 0 7 1 1 0 1 0 8 0 amappl10 152 48 0 34 1 0 1 1 0 8 0 amappl9 144 252 0 251 2 1 1 1 0 8 0 amappl8 136 30 0 27 1 0 1 1 0 8 0 amappl7 128 102 0 100 1 0 1 1 0 8 0 amappl6 120 340 0 323 1 0 1 1 0 8 0 amappl5 112 82 0 70 1 0 1 1 0 8 0 amappl4 104 500 0 466 1 0 1 1 0 8 0 amappl3 96 16762 0 16652 4 1 3 3 0 8 0 amappl2 88 3242 0 3153 4 1 3 3 0 8 0 amappl1 80 22652 0 21944 19 3 16 17 0 8 0 amappl 88 25895 0 25697 7 2 5 6 0 92 0 uvmvnodes 80 191 0 0 4 0 4 4 0 8 0 dma16384 16384 1 0 1 1 1 0 1 0 8 0 dma8192 8192 1 0 1 1 1 0 1 0 8 0 dma4096 4096 3 0 3 2 2 0 1 0 8 0 dma1024 1024 5 0 4 1 0 1 1 0 8 0 dma512 512 3 0 3 3 3 0 1 0 8 0 dma256 256 8 0 8 3 3 0 1 0 8 0 dma128 128 255 0 255 3 3 0 1 0 8 0 dma64 64 6 0 6 1 1 0 1 0 8 0 dma32 32 7 0 7 1 1 0 1 0 8 0 dma16 16 19 0 18 1 0 1 1 0 8 0 aobjpl 72 109 0 6 2 0 2 2 0 8 0 uaddrrnd 24 3056 0 3021 1 0 1 1 0 8 0 uaddrbest 32 2 0 0 1 0 1 1 0 8 0 uaddr 24 3056 0 3021 1 0 1 1 0 8 0 vmmpekpl 168 25516 0 25464 4 1 3 3 0 8 0 vmmpepl 168 203856 0 201280 181 50 131 142 0 357 2 vmsppl 488 3055 0 3021 5 0 5 5 0 8 0 rwobjpl 80 56796 0 55034 60 15 45 51 0 8 0 pdppl 4096 6134 0 6051 143 60 83 86 0 8 0 pvpl 32 33626 0 0 271 0 271 271 0 265 0 pmappl 256 3062 0 3023 3 0 3 3 0 8 0 extentpl 40 45 0 27 1 0 1 1 0 8 0 phpool 112 461 0 133 10 0 10 10 0 8 0 ddb{1}> machine ddbcpu 0 Stopped at x86_ipi_db+39: addq $8,%rsp ddb{0}> trace x86_ipi_db(ffffffff837c4ff0) at x86_ipi_db+39 x86_ipi_handler() at x86_ipi_handler+217 Xresume_lapic_ipi() at Xresume_lapic_ipi+39 __mp_lock(ffffffff83933dc0) at __mp_lock+402 intr_handler(ffff80003c433d80,ffff800000079600) at intr_handler+233 Xintr_ioapic_edge16_untramp() at Xintr_ioapic_edge16_untramp+399 __mp_lock(ffffffff83933dc0) at __mp_lock+402 intr_handler(ffff80003c433ee0,ffff8000002a3480) at intr_handler+233 Xintr_ioapic_edge23_untramp() at Xintr_ioapic_edge23_untramp+399 end of kernel end trace frame: 0x72e5d1757360, count: -9 ddb{0}> machine ddbcpu 1 Stopped at db_enter+37: addq $8,%rsp ddb{1}> trace db_enter() at db_enter+37 witness_checkorder(fffffd806f1051c8,9,0) at witness_checkorder+4305 rw_do_enter_write(fffffd806f1051b0,1) at rw_do_enter_write+186 rrw_enter(fffffd806f1051b0,1) at rrw_enter+198 VOP_LOCK(fffffd805c45e6c0,2001) at VOP_LOCK+189 vn_lock(fffffd805c45e6c0,2001) at vn_lock+164 vfs_getcwd_common(fffffd805c45e6c0,fffffd805b6cc7b8,0,0,200,0,3f9108398b3bceeb) at vfs_getcwd_common+209 vn_isunder(fffffd805c45e6c0,fffffd805b6cc7b8,ffff8000333defc0) at vn_isunder+86 unp_externalize(fffffd8059dc7100,33,0) at unp_externalize+623 soreceive(ffff8000015f7c20,ffff80003c439568,ffff80003c439518,0,ffff80003c439550,ffff80003c4396dc,f2eb1a85b894eb00) at soreceive+3108 recvit(ffff8000333defc0,9,ffff80003c4396b0,0,ffff80003c439760) at recvit+1035 sys_recvmsg(ffff8000333defc0,ffff80003c439810,ffff80003c439760) at sys_recvmsg+447 syscall(ffff80003c439810) at syscall+3028 Xsyscall() at Xsyscall+296 end of kernel end trace frame: 0x2bb4d1cb190, count: -14