witness: lock order reversal: 1st 0xffff800001292610 sbufrcv (&so->so_rcv.sb_lock) 2nd 0xfffffd806bda7f78 inode (&ip->i_lock) lock order [1] sbufrcv (&so->so_rcv.sb_lock) -> [2] inode (&ip->i_lock) lock order data 0xffffffff830f26fc -> 0xffffffff8306e212 is missing lock order [2] inode (&ip->i_lock) -> [1] sbufrcv (&so->so_rcv.sb_lock) #0 rw_enter+0x122 #1 sblock+0xb7 sys/kern/uipc_socket2.c:549 #2 soreceive+0x295 sys/kern/uipc_socket.c:945 #3 fifo_read+0x11a sys/miscfs/fifofs/fifo_vnops.c:264 #4 VOP_READ+0x102 sys/kern/vfs_vops.c:227 #5 vn_rdwr+0x15b #6 vndsetcred+0xa1 sys/dev/vnd.c:684 #7 vndioctl+0xe6c sys/dev/vnd.c:485 #8 VOP_IOCTL+0xac sys/kern/vfs_vops.c:264 #9 vn_ioctl+0xf6 sys/kern/vfs_vnops.c:525 #10 sys_ioctl+0x67c #11 syscall+0xbb6 mi_syscall sys/sys/syscall_mi.h:178 [inline] #11 syscall+0xbb6 sys/arch/amd64/amd64/trap.c:577 #12 Xsyscall+0x128 Stopped at db_enter+0x25: addq $0x8,%rsp ddb{0}> ddb{0}> set $lines = 0 ddb{0}> set $maxwidth = 0 ddb{0}> show panic the kernel did not panic ddb{0}> trace db_enter() at db_enter+0x25 sys/arch/amd64/amd64/db_interface.c:437 witness_checkorder(fffffd806bda7f78,9,0) at witness_checkorder+0x1047 rw_enter(fffffd806bda7f68,1) at rw_enter+0x122 rrw_enter(fffffd806bda7f68,1) at rrw_enter+0xbe sys/kern/kern_rwlock.c:464 VOP_LOCK(fffffd806a279620,2001) at VOP_LOCK+0xa6 sys/kern/vfs_vops.c:524 vn_lock(fffffd806a279620,2001) at vn_lock+0xa4 sys/kern/vfs_vnops.c:564 vfs_getcwd_common(fffffd806a279620,fffffd806d17c8a0,0,0,200,0,ab51c7dac9f89e7d) at vfs_getcwd_common+0xd1 sys/kern/vfs_getcwd.c:287 vn_isunder(fffffd806a279620,fffffd806d17c8a0,ffff80002a04e7b8) at vn_isunder+0x56 sys/kern/vfs_vnops.c:688 unp_externalize(fffffd806cc04700,ec,0) at unp_externalize+0x286 sys/kern/uipc_usrreq.c:1094 soreceive(ffff800001292540,ffff80002a14cac8,ffff80002a14ca78,0,ffff80002a14cab8,ffff80002a14cc3c,febb15cfe5d4fe9c) at soreceive+0xd3e sys/kern/uipc_socket.c:1090 recvit(ffff80002a04e7b8,5,ffff80002a14cc10,0,ffff80002a14ccc0) at recvit+0x40a sys/kern/uipc_syscalls.c:1079 sys_recvmsg(ffff80002a04e7b8,ffff80002a14cd70,ffff80002a14ccc0) at sys_recvmsg+0x1bf sys/kern/uipc_syscalls.c:879 syscall(ffff80002a14cd70) at syscall+0xbb6 mi_syscall sys/sys/syscall_mi.h:178 [inline] syscall(ffff80002a14cd70) at syscall+0xbb6 sys/arch/amd64/amd64/trap.c:577 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0x45c11b446d0, count: -14 ddb{0}> show registers rdi 0 rsi 0x40000 acpi_pdirpa+0x2be71 rbp 0xffff80002a14c520 rbx 0xfffffd80042c72c8 rdx 0xffff80000128e800 rcx 0xffff80002a04e7b8 rax 0x3ffff acpi_pdirpa+0x2be70 r8 0xffff80002a14c400 r9 0x8080808080808080 r10 0xfcfcbd1a77d9a1ef r11 0x2683c782623c0ba r12 0 r13 0xfffffd8003adee00 r14 0x3 r15 0xffffffff rip 0xffffffff81ac00e5 db_enter+0x25 cs 0x8 rflags 0x246 rsp 0xffff80002a14c510 ss 0x10 db_enter+0x25: addq $0x8,%rsp ddb{0}> show proc PROC (syz-executor) tid=134097 pid=1497 tcnt=3 stat=onproc flags process=1000000 proc=4000000 runpri=24, usrpri=50, slppri=24, nice=20 wchan=0x0, wmesg=, ps_single=0x0 scnt=0 ecnt=0 forw=0xffffffffffffffff, list=0xffff80002a04ea40,0xffff80002a04f1e8 process=0xffff8000ffff4018 user=0xffff80002a147000, vmspace=0xfffffd806ceb2018 estcpu=36, cpticks=0, pctcpu=0.0, user=0, sys=0, intr=0 ddb{0}> ps PID TID PPID UID S FLAGS WAIT COMMAND 1497 444619 31388 0 7 0x1000000 syz-executor * 1497 134097 31388 0 7 0x5000000 syz-executor 1497 341776 31388 0 2 0x5000000 syz-executor 88084 486797 40197 0 3 0x82 piperd syz-executor 70517 322304 0 0 3 0x14200 bored sosplice 3344 433827 40197 0 3 0x82 piperd syz-executor 29179 232519 40197 0 3 0x82 piperd syz-executor 75604 187981 40197 0 3 0x82 piperd syz-executor 31388 174227 40197 0 3 0x82 nanoslp syz-executor 93003 442249 40197 0 3 0x82 nanoslp syz-executor 40197 441664 37321 0 3 0x82 nanoslp syz-executor 37321 225849 62447 0 3 0x10008a sigsusp ksh 62447 104767 45441 0 3 0x98 kqread sshd-session 45441 253039 62548 0 3 0x92 kqread sshd-session 53235 54242 1 0 3 0x100083 ttyin getty 62548 470219 1 0 3 0x88 kqread sshd 57776 344798 63132 74 3 0x1100092 bpf pflogd 63132 450929 1 0 3 0x80 sbwait pflogd 10038 467612 35465 73 3 0x1100090 kqread syslogd 35465 502145 1 0 3 0x100082 sbwait syslogd 4712 328545 1 0 3 0x100080 kqread resolvd 85723 371779 34537 77 3 0x100092 kqread dhcpleased 6625 103993 34537 77 3 0x100092 kqread dhcpleased 34537 349810 1 0 3 0x80 kqread dhcpleased 52795 401769 0 0 3 0x14200 bored smr 24308 183971 0 0 3 0x14200 pgzero zerothread 26942 12375 0 0 3 0x14200 aiodoned aiodoned 52386 446400 0 0 3 0x14200 syncer update 60196 286797 0 0 3 0x14200 cleaner cleaner 30474 421281 0 0 3 0x14200 reaper reaper 85653 174376 0 0 3 0x14200 pgdaemon pagedaemon 1026 339063 0 0 3 0x14200 bored viomb 34713 54795 0 0 3 0x40014200 acpi0 acpi0 39784 408929 0 0 3 0x40014200 idle1 83673 32607 0 0 3 0x14200 bored softnet3 95463 292711 0 0 3 0x14200 bored softnet2 77824 148528 0 0 3 0x14200 bored softnet1 80799 422375 0 0 3 0x14200 bored softnet0 73056 273599 0 0 3 0x14200 bored systqmp 71007 405022 0 0 3 0x14200 bored systq 33522 273259 0 0 3 0x14200 tmoslp softclockmp 57186 358183 0 0 3 0x40014200 tmoslp softclock 32443 509412 0 0 3 0x40014200 idle0 1 295127 0 0 3 0x80082 wait init 0 0 -1 0 3 0x10200 scheduler swapper ddb{0}> show all locks Process 1497 (syz-executor) thread 0xffff80002a04e7b8 (134097) exclusive kernel_lock &kernel_lock r = 0 (0xffffffff835a0b68) #0 witness_lock+0x5bb stacktrace_save sys/sys/stacktrace.h:37 [inline] #0 witness_lock+0x5bb sys/kern/subr_witness.c:1155 #1 unp_externalize+0x14e #2 soreceive+0xd3e sys/kern/uipc_socket.c:1090 #3 recvit+0x40a sys/kern/uipc_syscalls.c:1079 #4 sys_recvmsg+0x1bf sys/kern/uipc_syscalls.c:879 #5 syscall+0xbb6 mi_syscall sys/sys/syscall_mi.h:178 [inline] #5 syscall+0xbb6 sys/arch/amd64/amd64/trap.c:577 #6 Xsyscall+0x128 exclusive rwlock sbufrcv r = 0 (0xffff800001292610) #0 witness_lock+0x5bb stacktrace_save sys/sys/stacktrace.h:37 [inline] #0 witness_lock+0x5bb sys/kern/subr_witness.c:1155 #1 rw_enter+0x41b sys/kern/kern_rwlock.c:309 #2 sblock+0xb7 sys/kern/uipc_socket2.c:549 #3 soreceive+0x295 sys/kern/uipc_socket.c:945 #4 recvit+0x40a sys/kern/uipc_syscalls.c:1079 #5 sys_recvmsg+0x1bf sys/kern/uipc_syscalls.c:879 #6 syscall+0xbb6 mi_syscall sys/sys/syscall_mi.h:178 [inline] #6 syscall+0xbb6 sys/arch/amd64/amd64/trap.c:577 #7 Xsyscall+0x128 ddb{0}> show malloc Type InUse MemUse HighUse Limit Requests Type Lim devbuf 10220 11177K 11508K 166960K 12329 0 pcb 17 12K 13K 166960K 117 0 rtable 165 6K 7K 166960K 474 0 pf 35 18K 20K 166960K 95 0 ifaddr 33 5K 7K 166960K 68 0 ifgroup 47 2K 2K 166960K 100 0 sysctl 2 0K 0K 166960K 2 0 counters 60 35K 36K 166960K 88 0 ioctlops 0 0K 4K 166960K 1579 0 iov 0 0K 16K 166960K 32 0 mount 1 1K 1K 166960K 1 0 log 0 0K 0K 166960K 4 0 vnodes 1431 90K 91K 166960K 2425 0 UFS quota 1 32K 32K 166960K 1 0 UFS mount 5 36K 36K 166960K 5 0 shm 2 1K 9K 166960K 14 0 VM map 2 1K 1K 166960K 2 0 sem 12 0K 0K 166960K 30 0 dirhash 12 2K 2K 166960K 24 0 ACPI 1690 195K 286K 166960K 12468 0 file desc 10 33K 85K 166960K 858 0 sigio 0 0K 0K 166960K 11 0 proc 72 91K 115K 166960K 645 0 subproc 78 4K 6K 166960K 494 0 NFS srvsock 1 0K 0K 166960K 1 0 NFS daemon 1 16K 16K 166960K 1 0 ip_moptions 0 0K 0K 166960K 150 0 in_multi 72 5K 7K 166960K 147 0 ether_multi 1 0K 0K 166960K 4 0 ISOFS mount 1 32K 32K 166960K 1 0 MSDOSFS mount 1 16K 16K 166960K 1 0 ttys 241 1076K 1076K 166960K 241 0 exec 0 0K 1K 166960K 553 0 fusefs mount 1 32K 32K 166960K 1 0 pfkey data 0 0K 0K 166960K 33 0 tdb 3 0K 0K 166960K 3 0 VM swap 8 62K 64K 166960K 10 0 UVM amap 180 55K 77K 166960K 9194 0 UVM aobj 89 3K 3K 166960K 91 0 pinsyscall 35 70K 100K 166960K 2042 0 memdesc 1 4K 4K 166960K 1 0 crypto data 1 1K 1K 166960K 1 0 ip6_options 0 0K 0K 166960K 108 0 NDP 10 0K 2K 166960K 46 0 temp 53 6824K 6889K 166960K 33462 0 kqueue 13 20K 24K 166960K 105 0 SYN cache 2 16K 16K 166960K 2 0 ddb{0}> show all pools Name Size Requests Fail Releases Pgreq Pgrel Npage Hiwat Minpg Maxpg Idle plcache 128 24 0 0 1 0 1 1 0 8 0 rtpcb 120 73 0 70 1 0 1 1 0 8 0 rtentry 112 143 0 72 4 0 4 4 0 8 0 unpcb 144 458 0 437 5 4 1 4 0 8 0 syncache 336 7 0 7 2 2 0 1 0 8 0 tcpqe 32 1 0 1 1 1 0 1 0 8 0 tcpcb 808 387 0 381 9 8 1 8 0 8 0 arp 120 25 0 12 1 0 1 1 0 8 0 inpcb 336 1097 0 1087 9 7 2 7 0 8 1 nd6 136 32 0 15 1 0 1 1 0 8 0 pkpcb 40 39 0 39 2 1 1 1 0 8 1 kcovpl 48 38 0 32 1 0 1 1 0 8 0 ppxss 1168 4 0 4 2 2 0 1 0 8 0 pfstscr 40 3 0 2 1 0 1 1 0 8 0 pffrag 232 22 0 18 1 0 1 1 0 482 0 pffrnode 88 22 0 18 1 0 1 1 0 8 0 pffrent 40 41 0 37 1 0 1 1 0 8 0 pfosfp 40 1428 0 1005 5 0 5 5 0 8 0 pfosfpen 112 1428 0 714 21 0 21 21 0 8 0 pfrktable 1344 2 0 1 1 0 1 1 0 8 0 pftag 88 4 0 2 1 0 1 1 0 8 0 pfstitem 24 48 0 27 1 0 1 1 0 8 0 pfstkey 128 49 0 28 2 0 2 2 0 8 0 pfstate 376 48 0 28 4 0 4 4 0 8 0 pfrule 1344 27 0 19 2 0 2 2 0 8 0 art_heap8 4096 1 0 0 1 0 1 1 0 8 0 art_heap4 256 631 0 290 30 3 27 29 0 8 5 art_table 32 632 0 290 4 0 4 4 0 8 0 art_node 16 142 0 78 1 0 1 1 0 8 0 sysvmsgpl 40 12 0 7 1 0 1 1 0 8 0 semapl 112 26 0 16 1 0 1 1 0 8 0 shmpl 112 88 0 2 3 0 3 3 0 8 0 dirhash 1024 25 0 8 3 0 3 3 0 8 0 dino2pl 256 2729 0 1196 96 0 96 96 0 8 0 ffsino 272 2729 0 1196 103 0 103 103 0 8 0 nchpl 144 3813 0 2101 64 0 64 64 0 8 0 uvmvnodes 80 3650 0 0 75 0 75 75 0 8 0 vnodes 216 3650 0 0 203 0 203 203 0 8 0 namei 1024 13391 0 13391 3 2 1 1 0 8 1 percpumem 16 58 0 14 1 0 1 1 0 8 0 kstatmem 264 48 0 28 2 0 2 2 0 8 0 scsiplug 72 3 0 3 1 1 0 1 0 8 0 scxspl 216 20167 0 20167 6 4 2 3 1 8 2 plimitpl 152 145 0 129 1 0 1 1 0 8 0 sigapl 424 1114 0 1070 7 1 6 7 0 8 0 futexpl 64 9676 0 9676 2 1 1 1 0 8 1 knotepl 120 529 0 0 16 0 16 16 0 8 0 kqueuepl 216 225 0 215 5 4 1 5 0 8 0 pipepl 320 252 0 225 3 0 3 3 0 8 0 fdescpl 496 1095 0 1071 5 1 4 5 0 8 0 filepl 152 6975 0 6760 19 8 11 18 0 8 1 lockfpl 104 346 0 344 2 1 1 2 0 8 0 lockfspl 48 156 0 154 1 0 1 1 0 8 0 sessionpl 144 56 0 47 1 0 1 1 0 8 0 pgrppl 48 104 0 88 1 0 1 1 0 8 0 ucredpl 104 1154 0 1141 1 0 1 1 0 8 0 zombiepl 144 1072 0 1070 2 1 1 1 0 8 0 processpl 1160 1114 0 1070 4 0 4 4 0 8 0 procpl 648 2210 0 2164 7 1 6 7 0 8 1 srpgc 96 4 0 4 1 1 0 1 0 8 0 sosppl 168 1 0 1 1 1 0 1 0 8 0 sockpl 664 1674 0 1641 11 7 4 9 0 8 1 mcl64k 65536 3 0 0 1 0 1 1 0 8 0 mcl16k 16384 1 0 0 1 0 1 1 0 8 0 mcl8k 8192 7 0 0 1 0 1 1 0 8 0 mcl4k 4096 169 0 0 22 0 22 22 0 8 0 mcl2k2 2112 1 0 0 1 0 1 1 0 8 0 mcl2k 2048 21 0 0 3 0 3 3 0 8 0 mtagpl 96 87 0 0 3 0 3 3 0 8 0 mbufpl 256 273 0 0 17 0 17 17 0 8 0 bufpl 280 6024 0 119 422 0 422 422 0 8 0 anonpl 24 231987 0 228971 54 3 51 51 0 185 24 amapchunkpl 152 30798 0 30462 38 5 33 33 0 158 14 amappl16 200 5775 0 5756 23 17 6 15 0 8 4 amappl15 192 6 0 6 1 1 0 1 0 8 0 amappl14 184 189 0 176 1 0 1 1 0 8 0 amappl13 176 11 0 11 1 1 0 1 0 8 0 amappl12 168 1846 0 1822 3 1 2 2 0 8 0 amappl11 160 57 0 43 1 0 1 1 0 8 0 amappl10 152 34 0 34 1 1 0 1 0 8 0 amappl9 144 145 0 145 1 1 0 1 0 8 0 amappl8 136 28 0 25 1 0 1 1 0 8 0 amappl7 128 118 0 105 1 0 1 1 0 8 0 amappl6 120 196 0 195 1 0 1 1 0 8 0 amappl5 112 150 0 138 1 0 1 1 0 8 0 amappl4 104 340 0 320 1 0 1 1 0 8 0 amappl3 96 5535 0 5466 3 0 3 3 0 8 0 amappl2 88 1419 0 1341 3 0 3 3 0 8 0 amappl1 80 10893 0 10353 14 0 14 14 0 8 0 amappl 88 8707 0 8587 7 2 5 5 0 92 0 dma65536 65536 2 0 2 1 1 0 1 0 8 0 dma4096 4096 1 0 1 1 1 0 1 0 8 0 dma1024 1024 2 0 1 1 0 1 1 0 8 0 dma512 512 1 0 1 1 1 0 1 0 8 0 dma256 256 6 0 6 1 1 0 1 0 8 0 dma128 128 253 0 253 1 1 0 1 0 8 0 dma64 64 6 0 6 1 1 0 1 0 8 0 dma32 32 7 0 7 1 1 0 1 0 8 0 dma16 16 18 0 17 1 0 1 1 0 8 0 aobjpl 72 90 0 2 2 0 2 2 0 8 0 uaddrrnd 24 1095 0 1071 1 0 1 1 0 8 0 uaddrbest 32 2 0 0 1 0 1 1 0 8 0 uaddr 24 1095 0 1071 1 0 1 1 0 8 0 vmmpekpl 168 11350 0 11308 3 0 3 3 0 8 0 vmmpepl 168 77774 0 76247 95 6 89 95 0 357 6 vmsppl 448 1094 0 1071 6 2 4 5 0 8 1 rwobjpl 56 28678 0 24089 66 0 66 66 0 8 0 pdppl 4096 2197 0 2142 105 40 65 81 0 8 10 pvpl 32 27164 0 0 221 1 220 220 0 265 0 pmappl 248 1094 0 1071 3 0 3 3 0 8 1 extentpl 40 55 0 38 1 0 1 1 0 8 0 phpool 112 419 0 70 11 0 11 11 0 8 0 ddb{0}> machine ddbcpu 0 Invalid cpu 0 ddb{0}> trace db_enter() at db_enter+0x25 sys/arch/amd64/amd64/db_interface.c:437 witness_checkorder(fffffd806bda7f78,9,0) at witness_checkorder+0x1047 rw_enter(fffffd806bda7f68,1) at rw_enter+0x122 rrw_enter(fffffd806bda7f68,1) at rrw_enter+0xbe sys/kern/kern_rwlock.c:464 VOP_LOCK(fffffd806a279620,2001) at VOP_LOCK+0xa6 sys/kern/vfs_vops.c:524 vn_lock(fffffd806a279620,2001) at vn_lock+0xa4 sys/kern/vfs_vnops.c:564 vfs_getcwd_common(fffffd806a279620,fffffd806d17c8a0,0,0,200,0,ab51c7dac9f89e7d) at vfs_getcwd_common+0xd1 sys/kern/vfs_getcwd.c:287 vn_isunder(fffffd806a279620,fffffd806d17c8a0,ffff80002a04e7b8) at vn_isunder+0x56 sys/kern/vfs_vnops.c:688 unp_externalize(fffffd806cc04700,ec,0) at unp_externalize+0x286 sys/kern/uipc_usrreq.c:1094 soreceive(ffff800001292540,ffff80002a14cac8,ffff80002a14ca78,0,ffff80002a14cab8,ffff80002a14cc3c,febb15cfe5d4fe9c) at soreceive+0xd3e sys/kern/uipc_socket.c:1090 recvit(ffff80002a04e7b8,5,ffff80002a14cc10,0,ffff80002a14ccc0) at recvit+0x40a sys/kern/uipc_syscalls.c:1079 sys_recvmsg(ffff80002a04e7b8,ffff80002a14cd70,ffff80002a14ccc0) at sys_recvmsg+0x1bf sys/kern/uipc_syscalls.c:879 syscall(ffff80002a14cd70) at syscall+0xbb6 mi_syscall sys/sys/syscall_mi.h:178 [inline] syscall(ffff80002a14cd70) at syscall+0xbb6 sys/arch/amd64/amd64/trap.c:577 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0x45c11b446d0, count: -14 ddb{0}> machine ddbcpu 1 Stopped at x86_ipi_db+0x27: addq $0x8,%rsp ddb{1}> trace x86_ipi_db(ffff800029b7bff0) at x86_ipi_db+0x27 sys/arch/amd64/amd64/db_interface.c:393 x86_ipi_handler() at x86_ipi_handler+0xd9 sys/arch/amd64/amd64/ipi.c:106 Xresume_lapic_ipi() at Xresume_lapic_ipi+0x27 __mp_lock(ffffffff835a0960) at __mp_lock+0x199 __mp_lock_spin sys/kern/kern_lock.c:113 [inline] __mp_lock(ffffffff835a0960) at __mp_lock+0x199 sys/kern/kern_lock.c:144 syscall(ffff80002a110970) at syscall+0xad6 mi_syscall sys/sys/syscall_mi.h:178 [inline] syscall(ffff80002a110970) at syscall+0xad6 sys/arch/amd64/amd64/trap.c:577 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0x777b2d2601c0, count: -6