witness: lock order reversal: 1st 0xffff8000014d32d0 sbufrcv (&so->so_rcv.sb_lock) 2nd 0xfffffd806c7db0e0 inode (&ip->i_lock) lock order [1] sbufrcv (&so->so_rcv.sb_lock) -> [2] inode (&ip->i_lock) lock order data 0xffffffff83331184 -> 0xffffffff8337ffd2 is missing lock order [2] inode (&ip->i_lock) -> [1] sbufrcv (&so->so_rcv.sb_lock) #0 rw_do_enter_write+0xba sys/kern/kern_rwlock.c:234 #1 sblock+0xb6 sys/kern/uipc_socket2.c:536 #2 soreceive+0x27d sys/kern/uipc_socket.c:878 #3 fifo_read+0x117 sys/miscfs/fifofs/fifo_vnops.c:264 #4 VOP_READ+0x101 sys/kern/vfs_vops.c:227 #5 vn_rdwr+0x15b sys/kern/vfs_vnops.c:-1 #6 vndsetcred+0xa1 sys/dev/vnd.c:685 #7 vndioctl+0xdfc sys/dev/vnd.c:486 #8 VOP_IOCTL+0xac sys/kern/vfs_vops.c:264 #9 vn_ioctl+0xf8 sys/kern/vfs_vnops.c:531 #10 sys_ioctl+0x674 sys/kern/sys_generic.c:-1 #11 syscall+0xbd4 mi_syscall sys/sys/syscall_mi.h:176 [inline] #11 syscall+0xbd4 sys/arch/amd64/amd64/trap.c:765 #12 Xsyscall+0x128 Stopped at db_enter+0x25: addq $0x8,%rsp ddb{1}> ddb{1}> set $lines = 0 ddb{1}> set $maxwidth = 0 ddb{1}> show panic the kernel did not panic ddb{1}> trace db_enter() at db_enter+0x25 sys/arch/amd64/amd64/db_interface.c:438 witness_checkorder(fffffd806c7db0e0,9,0) at witness_checkorder+0x10b9 sys/kern/subr_witness.c:-1 rw_do_enter_write(fffffd806c7db0c8,1) at rw_do_enter_write+0xba sys/kern/kern_rwlock.c:234 rrw_enter(fffffd806c7db0c8,1) at rrw_enter+0xc6 sys/kern/kern_rwlock.c:621 VOP_LOCK(fffffd805e238128,2001) at VOP_LOCK+0xa3 sys/kern/vfs_vops.c:527 vn_lock(fffffd805e238128,2001) at vn_lock+0xa4 sys/kern/vfs_vnops.c:570 vfs_getcwd_common(fffffd805e238128,fffffd805e238488,0,0,200,0,7259501d6837ffd3) at vfs_getcwd_common+0xd1 sys/kern/vfs_getcwd.c:287 vn_isunder(fffffd805e238128,fffffd805e238488,ffff800033fdbca8) at vn_isunder+0x56 sys/kern/vfs_vnops.c:694 unp_externalize(fffffd8069f15d00,33,0) at unp_externalize+0x26f sys/kern/uipc_usrreq.c:1090 soreceive(ffff8000014d31e0,ffff80003bc08448,ffff80003bc083f8,0,ffff80003bc08430,ffff80003bc085bc,f1fb712e45169fda) at soreceive+0xc24 sys/kern/uipc_socket.c:1017 recvit(ffff800033fdbca8,4,ffff80003bc08590,0,ffff80003bc08640) at recvit+0x40b sys/kern/uipc_syscalls.c:1078 sys_recvmsg(ffff800033fdbca8,ffff80003bc086f0,ffff80003bc08640) at sys_recvmsg+0x1bf sys/kern/uipc_syscalls.c:878 syscall(ffff80003bc086f0) at syscall+0xbd4 mi_syscall sys/sys/syscall_mi.h:176 [inline] syscall(ffff80003bc086f0) at syscall+0xbd4 sys/arch/amd64/amd64/trap.c:765 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0x45935dd6b50, count: -14 ddb{1}> show registers rdi 0 rsi 0x80000 acpi_pdirpa+0x6be71 rbp 0xffff80003bc07ee0 rbx 0 rdx 0xffff800001477180 rcx 0xffff800033fdbca8 rax 0x7ffff acpi_pdirpa+0x6be70 r8 0xffff80003bc07dc0 r9 0x8080808080808080 r10 0x1a3deb0944cdbf42 r11 0x96c10b91798d13f0 r12 0xfffffd8003f45880 r13 0xfffffd800472d818 r14 0x3 r15 0xffffffff834145c3 substchar+0x7bc8 rip 0xffffffff82bb41e5 db_enter+0x25 cs 0x8 rflags 0x246 rsp 0xffff80003bc07ed0 ss 0x10 db_enter+0x25: addq $0x8,%rsp ddb{1}> show proc PROC (syz-executor) tid=369006 pid=21626 tcnt=4 stat=onproc flags process=1000000 proc=4000000 runpri=32, usrpri=72, slppri=32, nice=20 wchan=0x0, wmesg=, ps_single=0x0 scnt=0 ecnt=0 forw=0xffffffffffffffff, list=0xffff800033fdb4e0,0xffff800033fda560 process=0xffff80003bc1f040 user=0xffff80003bc03000, vmspace=0xfffffd806c8d37c0 estcpu=22, cpticks=1, pctcpu=0.0, user=0, sys=1, intr=0 ddb{1}> ps PID TID PPID UID S FLAGS WAIT COMMAND 54088 374714 98677 0 7 0 syz-executor 54088 88912 98677 0 2 0x4000000 syz-executor 21626 5543 72303 0 2 0x1000000 syz-executor 21626 150923 72303 0 2 0x5000000 syz-executor *21626 369006 72303 0 7 0x5000000 syz-executor 21626 23858 72303 0 3 0x5000080 fsleep syz-executor 22054 183345 90333 0 3 0x80 nanoslp syz-executor 22054 241772 90333 0 3 0x4000080 kqsel syz-executor 22054 445494 90333 0 3 0x4000080 fsleep syz-executor 78746 78623 15126 60928 2 0x10 syz-executor 78746 427038 15126 60928 3 0x4000090 fsleep syz-executor 78746 249029 15126 60928 3 0x4000090 fsleep syz-executor 72141 291371 68460 0 3 0x80 nanoslp syz-executor 72141 344004 68460 0 3 0x4000080 fsleep syz-executor 72141 492447 68460 0 3 0x4000080 fsleep syz-executor 2068 133987 45815 0 3 0x80 nanoslp syz-executor 2068 163186 45815 0 3 0x4000080 kqsel syz-executor 2068 81987 45815 0 3 0x4000080 fsleep syz-executor 2068 310761 45815 0 3 0x4000080 fsleep syz-executor 16709 129282 0 0 3 0x14280 nfsidl nfsio 15126 435526 16056 0 2 0x3 syz-executor 72303 145146 16056 0 2 0x3 syz-executor 45815 87388 16056 0 2 0x3 syz-executor 98677 272729 16056 0 2 0x3 syz-executor 90333 132905 16056 0 3 0x82 nanoslp syz-executor 39660 7952 16056 0 2 0x2 syz-executor 59650 317725 16056 0 2 0x3 syz-executor 68460 314614 16056 0 3 0x82 nanoslp syz-executor 16056 233227 40472 0 3 0x82 kqread syz-executor 40472 468306 26278 0 3 0x10008a sigsusp ksh 26278 489133 24660 0 3 0x98 kqread sshd-session 24660 465061 68565 0 3 0x92 kqread sshd-session 60983 166397 1 0 3 0x100083 ttyopn getty 68565 130527 1 0 3 0x88 kqread sshd 4157 350483 23259 74 3 0x1100092 bpf pflogd 23259 509644 1 0 3 0x80 sbwait pflogd 481 72504 60237 73 3 0x1100090 kqread syslogd 60237 139980 1 0 3 0x100082 sbwait syslogd 84608 278967 1 0 3 0x100080 kqread resolvd 39851 388772 83393 77 3 0x100092 kqread dhcpleased 51973 146041 83393 77 3 0x100092 kqread dhcpleased 83393 27990 1 0 3 0x80 kqread dhcpleased 29636 132412 0 0 3 0x14200 bored smr 88343 399756 0 0 3 0x14200 pgzero zerothread 49597 97553 0 0 3 0x14200 aiodoned aiodoned 41304 465657 0 0 3 0x14200 syncer update 91628 100310 0 0 3 0x14200 cleaner cleaner 75412 446571 0 0 3 0x14200 reaper reaper 67187 123025 0 0 3 0x14200 pgdaemon pagedaemon 49218 476223 0 0 3 0x14200 bored viomb 96140 48377 0 0 3 0x40014200 acpi0 acpi0 98609 215774 0 0 3 0x40014200 idle1 66849 374420 0 0 3 0x14200 bored softnet1 87225 145773 0 0 2 0x14200 softnet0 25198 370963 0 0 3 0x14200 bored systqmp 28225 405962 0 0 3 0x14200 bored systq 38823 44322 0 0 3 0x14200 tmoslp softclockmp 17601 507233 0 0 3 0x40014200 tmoslp softclock 12767 246542 0 0 3 0x40014200 idle0 1 183383 0 0 3 0x82 wait init 0 0 -1 0 3 0x10010200 scheduler swapper ddb{1}> show all locks Process 21626 (syz-executor) thread 0xffff800033fdbca8 (369006) exclusive kernel_lock &kernel_lock r = 0 (0xffffffff83a0a408) #0 witness_lock+0x5f1 stacktrace_save sys/sys/stacktrace.h:37 [inline] #0 witness_lock+0x5f1 sys/kern/subr_witness.c:1160 #1 unp_externalize+0x155 sys/kern/uipc_usrreq.c:-1 #2 soreceive+0xc24 sys/kern/uipc_socket.c:1017 #3 recvit+0x40b sys/kern/uipc_syscalls.c:1078 #4 sys_recvmsg+0x1bf sys/kern/uipc_syscalls.c:878 #5 syscall+0xbd4 mi_syscall sys/sys/syscall_mi.h:176 [inline] #5 syscall+0xbd4 sys/arch/amd64/amd64/trap.c:765 #6 Xsyscall+0x128 exclusive rwlock sbufrcv r = 0 (0xffff8000014d32d0) #0 witness_lock+0x5f1 stacktrace_save sys/sys/stacktrace.h:37 [inline] #0 witness_lock+0x5f1 sys/kern/subr_witness.c:1160 #1 rw_do_enter_write+0x419 sys/kern/kern_rwlock.c:320 #2 sblock+0xb6 sys/kern/uipc_socket2.c:536 #3 soreceive+0x27d sys/kern/uipc_socket.c:878 #4 recvit+0x40b sys/kern/uipc_syscalls.c:1078 #5 sys_recvmsg+0x1bf sys/kern/uipc_syscalls.c:878 #6 syscall+0xbd4 mi_syscall sys/sys/syscall_mi.h:176 [inline] #6 syscall+0xbd4 sys/arch/amd64/amd64/trap.c:765 #7 Xsyscall+0x128 ddb{1}> show malloc Type InUse MemUse HighUse Limit Requests Type Lim devbuf 10207 11098K 11315K 166960K 11546 0 pcb 18 13K 13K 166960K 66 0 rtable 268 11K 11K 166960K 397 0 pf 38 18K 22K 166960K 71 0 ifaddr 46 8K 8K 166960K 60 0 ifgroup 59 2K 2K 166960K 79 0 sysctl 3 1K 9K 166960K 7 0 counters 74 37K 37K 166960K 116 0 ioctlops 0 0K 4K 166960K 1549 0 iov 0 0K 16K 166960K 13 0 mount 1 1K 1K 166960K 1 0 log 0 0K 0K 166960K 4 0 vnodes 1344 85K 85K 166960K 1499 0 UFS quota 1 32K 32K 166960K 1 0 UFS mount 5 36K 36K 166960K 5 0 shm 2 1K 5K 166960K 4 0 VM map 2 1K 1K 166960K 2 0 sem 10 0K 0K 166960K 11 0 dirhash 12 2K 2K 166960K 12 0 ACPI 1692 195K 286K 166960K 12470 0 file desc 17 61K 114K 166960K 330 0 sigio 0 0K 0K 166960K 10 0 proc 72 115K 164K 166960K 560 0 subproc 72 4K 4K 166960K 72 0 NFS srvsock 1 0K 0K 166960K 1 0 NFS daemon 1 16K 16K 166960K 1 0 ip_moptions 0 0K 0K 166960K 15 0 in_multi 99 7K 7K 166960K 104 0 ether_multi 1 0K 0K 166960K 2 0 ISOFS mount 1 32K 32K 166960K 1 0 MSDOSFS mount 1 16K 16K 166960K 1 0 ttys 73 334K 334K 166960K 73 0 exec 0 0K 1K 166960K 386 0 fusefs mount 1 32K 32K 166960K 1 0 tdb 3 0K 0K 166960K 3 0 VM swap 8 62K 64K 166960K 10 0 UVM amap 261 168K 176K 166960K 4805 0 UVM aobj 6 2K 2K 166960K 7 0 pinsyscall 43 86K 102K 166960K 1448 0 memdesc 1 4K 4K 166960K 1 0 crypto data 1 1K 1K 166960K 1 0 ip6_options 0 0K 1K 166960K 16 0 NDP 14 0K 2K 166960K 40 0 temp 42 8663K 8918K 166960K 13752 0 kqueue 13 20K 26K 166960K 63 0 SYN cache 2 16K 16K 166960K 2 0 ddb{1}> show all pools Name Size Requests Fail Releases Pgreq Pgrel Npage Hiwat Minpg Maxpg Idle plcache 128 26 0 0 1 0 1 1 0 8 0 rtpcb 120 45 0 42 1 0 1 1 0 8 0 rtentry 176 118 0 1 6 0 6 6 0 8 0 unpcb 144 132 0 109 1 0 1 1 0 8 0 syncache 336 3 0 3 1 1 0 1 0 8 0 tcpcb 736 65 0 60 1 0 1 1 0 8 0 arp 136 20 0 0 1 0 1 1 0 8 0 inpcb 328 264 0 217 5 1 4 4 0 8 0 nd6 152 27 0 0 2 0 2 2 0 8 0 kcovpl 48 8 0 0 1 0 1 1 0 8 0 ppxss 1192 24 0 21 1 0 1 1 0 8 0 pffrag 232 1 0 0 1 0 1 1 0 482 0 pffrnode 88 1 0 0 1 0 1 1 0 8 0 pffrent 40 1 0 0 1 0 1 1 0 8 0 pfosfp 40 1428 0 1005 5 0 5 5 0 8 0 pfosfpen 112 1428 0 714 21 0 21 21 0 8 0 pfstitem 24 29 0 0 1 0 1 1 0 8 0 pfstkey 128 29 0 0 1 0 1 1 0 8 0 pfstate 448 29 0 0 4 0 4 4 0 8 0 pfrule 1344 21 0 16 2 1 1 2 0 8 0 art_heap8 4096 2 0 0 2 0 2 2 0 8 0 art_heap4 256 462 0 0 29 0 29 29 0 8 0 art_table 40 464 0 0 5 0 5 5 0 8 0 art_node 32 118 0 12 1 0 1 1 0 8 0 sysvmsgpl 40 1 0 1 1 1 0 1 0 8 0 semupl 112 1 0 1 1 0 1 1 0 8 1 semapl 112 9 0 1 1 0 1 1 0 8 0 shmpl 112 4 0 1 1 0 1 1 0 8 0 dirhash 1024 17 0 0 3 0 3 3 0 8 0 dino2pl 256 1907 0 393 96 0 96 96 0 8 0 ffsino 296 1907 0 393 117 0 117 117 0 8 0 nchpl 144 2365 0 661 64 0 64 64 0 8 0 rtmask 32 7 0 6 1 0 1 1 0 8 0 vnodes 216 2064 0 0 115 0 115 115 0 8 0 namei 1024 7229 0 7229 3 1 2 2 0 8 2 percpumem 16 73 0 21 1 0 1 1 0 8 0 kstatmem 264 50 0 20 3 0 3 3 0 8 0 scsiplug 72 1 0 1 1 0 1 1 0 8 1 scxspl 216 11625 0 11625 3 2 1 2 1 8 1 plimitpl 152 50 0 33 1 0 1 1 0 8 0 sigapl 424 635 0 587 6 0 6 6 0 8 0 knotepl 120 548 0 0 17 0 17 17 0 8 0 kqueuepl 224 85 0 74 2 1 1 2 0 8 0 pipepl 344 141 0 114 3 0 3 3 0 8 0 fdescpl 528 616 0 584 3 0 3 3 0 8 0 filepl 160 2576 0 2314 12 1 11 11 0 8 0 lockfpl 104 75 0 71 1 0 1 1 0 8 0 lockfspl 48 36 0 32 1 0 1 1 0 8 0 sessionpl 144 22 0 13 1 0 1 1 0 8 0 pgrppl 48 35 0 18 1 0 1 1 0 8 0 ucredpl 104 250 0 235 1 0 1 1 0 8 0 zombiepl 144 615 0 614 1 0 1 1 0 8 0 processpl 1232 636 0 587 5 0 5 5 0 8 0 procpl 664 977 0 915 6 0 6 6 0 8 0 sockpl 752 442 0 369 9 1 8 8 0 8 0 mcl64k 65536 17 0 0 3 0 3 3 0 8 0 mcl16k 16384 1 0 0 1 0 1 1 0 8 0 mcl8k 8192 3 0 0 1 0 1 1 0 8 0 mcl4k 4096 130 0 0 17 0 17 17 0 8 0 mcl2k2 2112 1 0 0 1 0 1 1 0 8 0 mcl2k 2048 25 0 0 4 0 4 4 0 8 0 mtagpl 96 13 0 0 1 0 1 1 0 8 0 mbufpl 256 1153 0 0 73 0 73 73 0 8 0 bufpl 280 4810 0 130 335 0 335 335 0 8 0 anonpl 32 11991 0 0 97 0 97 97 0 246 0 amapchunkpl 152 14850 0 14314 39 6 33 35 0 158 8 amappl16 200 2592 0 2565 20 5 15 17 0 8 6 amappl15 192 4 0 4 1 1 0 1 0 8 0 amappl14 184 36 0 36 1 1 0 1 0 8 0 amappl13 176 423 0 422 1 0 1 1 0 8 0 amappl12 168 987 0 945 3 1 2 3 0 8 0 amappl11 160 3 0 3 1 1 0 1 0 8 0 amappl10 152 126 0 112 1 0 1 1 0 8 0 amappl9 144 257 0 257 1 1 0 1 0 8 0 amappl8 136 27 0 25 1 0 1 1 0 8 0 amappl7 128 83 0 82 1 0 1 1 0 8 0 amappl6 120 281 0 268 1 0 1 1 0 8 0 amappl5 112 81 0 71 1 0 1 1 0 8 0 amappl4 104 420 0 389 1 0 1 1 0 8 0 amappl3 96 2384 0 2283 4 1 3 3 0 8 0 amappl2 88 734 0 658 2 0 2 2 0 8 0 amappl1 80 9807 0 9225 14 2 12 14 0 8 0 amappl 88 4055 0 3871 5 0 5 5 0 92 0 uvmvnodes 80 107 0 0 3 0 3 3 0 8 0 dma4096 4096 1 0 1 1 1 0 1 0 8 0 dma1024 1024 1 0 0 1 0 1 1 0 8 0 dma256 256 6 0 6 1 1 0 1 0 8 0 dma128 128 253 0 253 1 1 0 1 0 8 0 dma64 64 6 0 6 1 1 0 1 0 8 0 dma32 32 7 0 7 1 1 0 1 0 8 0 dma16 16 18 0 17 1 0 1 1 0 8 0 aobjpl 72 6 0 1 1 0 1 1 0 8 0 uaddrrnd 24 615 0 584 1 0 1 1 0 8 0 uaddrbest 32 2 0 0 1 0 1 1 0 8 0 uaddr 24 615 0 584 1 0 1 1 0 8 0 vmmpekpl 168 6763 0 6725 3 0 3 3 0 8 0 vmmpepl 168 46794 0 44862 101 3 98 99 0 357 2 vmsppl 488 614 0 584 5 1 4 5 0 8 0 rwobjpl 80 15984 0 14939 28 0 28 28 0 8 0 pdppl 4096 1237 0 1168 95 24 71 81 0 8 2 pvpl 32 20534 0 0 166 0 166 166 0 265 0 pmappl 256 614 0 584 3 0 3 3 0 8 0 extentpl 40 45 0 27 1 0 1 1 0 8 0 phpool 112 339 0 28 9 0 9 9 0 8 0 ddb{1}> machine ddbcpu 0 Stopped at x86_ipi_db+0x27: addq $0x8,%rsp ddb{0}> trace x86_ipi_db(ffffffff8380aff0) at x86_ipi_db+0x27 sys/arch/amd64/amd64/db_interface.c:394 x86_ipi_handler() at x86_ipi_handler+0xd9 sys/arch/amd64/amd64/ipi.c:106 Xresume_lapic_ipi() at Xresume_lapic_ipi+0x27 __mp_lock(ffffffff83a0a200) at __mp_lock+0x1a3 __mp_lock_spin sys/kern/kern_lock.c:134 [inline] __mp_lock(ffffffff83a0a200) at __mp_lock+0x1a3 sys/kern/kern_lock.c:165 softintr_dispatch(0) at softintr_dispatch+0x125 sys/kern/kern_softintr.c:83 dosoftint(0) at dosoftint+0x54 sys/arch/amd64/amd64/intr.c:862 Xsoftclock() at Xsoftclock+0x27 end of kernel end trace frame: 0x7d5bb8e19ba0, count: -7 ddb{0}> machine ddbcpu 1 Stopped at db_enter+0x25: addq $0x8,%rsp ddb{1}> trace db_enter() at db_enter+0x25 sys/arch/amd64/amd64/db_interface.c:438 witness_checkorder(fffffd806c7db0e0,9,0) at witness_checkorder+0x10b9 sys/kern/subr_witness.c:-1 rw_do_enter_write(fffffd806c7db0c8,1) at rw_do_enter_write+0xba sys/kern/kern_rwlock.c:234 rrw_enter(fffffd806c7db0c8,1) at rrw_enter+0xc6 sys/kern/kern_rwlock.c:621 VOP_LOCK(fffffd805e238128,2001) at VOP_LOCK+0xa3 sys/kern/vfs_vops.c:527 vn_lock(fffffd805e238128,2001) at vn_lock+0xa4 sys/kern/vfs_vnops.c:570 vfs_getcwd_common(fffffd805e238128,fffffd805e238488,0,0,200,0,7259501d6837ffd3) at vfs_getcwd_common+0xd1 sys/kern/vfs_getcwd.c:287 vn_isunder(fffffd805e238128,fffffd805e238488,ffff800033fdbca8) at vn_isunder+0x56 sys/kern/vfs_vnops.c:694 unp_externalize(fffffd8069f15d00,33,0) at unp_externalize+0x26f sys/kern/uipc_usrreq.c:1090 soreceive(ffff8000014d31e0,ffff80003bc08448,ffff80003bc083f8,0,ffff80003bc08430,ffff80003bc085bc,f1fb712e45169fda) at soreceive+0xc24 sys/kern/uipc_socket.c:1017 recvit(ffff800033fdbca8,4,ffff80003bc08590,0,ffff80003bc08640) at recvit+0x40b sys/kern/uipc_syscalls.c:1078 sys_recvmsg(ffff800033fdbca8,ffff80003bc086f0,ffff80003bc08640) at sys_recvmsg+0x1bf sys/kern/uipc_syscalls.c:878 syscall(ffff80003bc086f0) at syscall+0xbd4 mi_syscall sys/sys/syscall_mi.h:176 [inline] syscall(ffff80003bc086f0) at syscall+0xbd4 sys/arch/amd64/amd64/trap.c:765 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0x45935dd6b50, count: -14