witness: lock order reversal: 1st 0xffff80000128cbb0 sbufrcv (&so->so_rcv.sb_lock) 2nd 0xfffffd8065607848 inode (&ip->i_lock) lock order [1] sbufrcv (&so->so_rcv.sb_lock) -> [2] inode (&ip->i_lock) lock order data 0xffffffff831566ce -> 0xffffffff830cf67f is missing lock order [2] inode (&ip->i_lock) -> [1] sbufrcv (&so->so_rcv.sb_lock) #0 rw_do_enter_write+0xb7 sys/kern/kern_rwlock.c:233 #1 sblock+0xb7 sys/kern/uipc_socket2.c:537 #2 soreceive+0x28e sys/kern/uipc_socket.c:876 #3 fifo_read+0x11a sys/miscfs/fifofs/fifo_vnops.c:264 #4 VOP_READ+0x102 sys/kern/vfs_vops.c:227 #5 vn_rdwr+0x15b #6 vndsetcred+0xa1 sys/dev/vnd.c:684 #7 vndioctl+0xe6c sys/dev/vnd.c:485 #8 VOP_IOCTL+0xac sys/kern/vfs_vops.c:264 #9 vn_ioctl+0xf8 sys/kern/vfs_vnops.c:531 #10 sys_ioctl+0x5c3 #11 syscall+0xbc6 mi_syscall sys/sys/syscall_mi.h:176 [inline] #11 syscall+0xbc6 sys/arch/amd64/amd64/trap.c:577 #12 Xsyscall+0x128 Stopped at db_enter+0x25: addq $0x8,%rsp ddb{1}> ddb{1}> set $lines = 0 ddb{1}> set $maxwidth = 0 ddb{1}> show panic the kernel did not panic ddb{1}> trace db_enter() at db_enter+0x25 sys/arch/amd64/amd64/db_interface.c:437 witness_checkorder(fffffd8065607848,9,0) at witness_checkorder+0x1047 rw_do_enter_write(fffffd8065607830,1) at rw_do_enter_write+0xb7 sys/kern/kern_rwlock.c:233 rrw_enter(fffffd8065607830,1) at rrw_enter+0xc6 sys/kern/kern_rwlock.c:616 VOP_LOCK(fffffd806df5d460,2001) at VOP_LOCK+0xa6 sys/kern/vfs_vops.c:524 vn_lock(fffffd806df5d460,2001) at vn_lock+0xa4 sys/kern/vfs_vnops.c:570 vfs_getcwd_common(fffffd806df5d460,fffffd806df5d460,0,0,200,0,f2d54a3fcb02ee77) at vfs_getcwd_common+0xd1 sys/kern/vfs_getcwd.c:287 vn_isunder(fffffd806df5d460,fffffd806df5d460,ffff80002a43c020) at vn_isunder+0x56 sys/kern/vfs_vnops.c:694 unp_externalize(fffffd8060f10c00,ec,0) at unp_externalize+0x286 sys/kern/uipc_usrreq.c:1086 soreceive(ffff80000128cad0,ffff80003b99e628,ffff80003b99e5d8,0,ffff80003b99e618,ffff80003b99e79c,9449d065c30a975a) at soreceive+0xc73 sys/kern/uipc_socket.c:1015 recvit(ffff80002a43c020,6,ffff80003b99e770,0,ffff80003b99e820) at recvit+0x40a sys/kern/uipc_syscalls.c:1072 sys_recvmsg(ffff80002a43c020,ffff80003b99e8d0,ffff80003b99e820) at sys_recvmsg+0x1bf sys/kern/uipc_syscalls.c:872 syscall(ffff80003b99e8d0) at syscall+0xbc6 mi_syscall sys/sys/syscall_mi.h:176 [inline] syscall(ffff80003b99e8d0) at syscall+0xbc6 sys/arch/amd64/amd64/trap.c:577 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0x3a152be7170, count: -14 ddb{1}> show registers rdi 0 rsi 0x80000 acpi_pdirpa+0x6be71 rbp 0xffff80003b99e0b0 rbx 0xfffffd800433ef98 rdx 0xffff80000128b640 rcx 0xffff80002a43c020 rax 0x7ffff acpi_pdirpa+0x6be70 r8 0xffff80003b99df90 r9 0x8080808080808080 r10 0x4e269ccd819dd322 r11 0xc702ab28d1ec7d41 r12 0 r13 0xfffffd8003b56f00 r14 0x3 r15 0xffffffff rip 0xffffffff82a6fbd5 db_enter+0x25 cs 0x8 rflags 0x246 rsp 0xffff80003b99e0a0 ss 0x10 db_enter+0x25: addq $0x8,%rsp ddb{1}> show proc PROC (syz-executor) tid=302942 pid=56495 tcnt=3 stat=onproc flags process=1000000 proc=4000000 runpri=24, usrpri=50, slppri=24, nice=20 wchan=0x0, wmesg=, ps_single=0x0 scnt=0 ecnt=0 forw=0xffffffffffffffff, list=0xffff80002a43d460,0xffff8000ffff9c10 process=0xffff80003c5b1dd0 user=0xffff80003b999000, vmspace=0xfffffd80655ab3c0 estcpu=36, cpticks=0, pctcpu=0.0, user=0, sys=0, intr=0 ddb{1}> ps PID TID PPID UID S FLAGS WAIT COMMAND 14829 302624 26013 0 2 0 syz-executor 14829 279362 26013 0 3 0x4000080 fsleep syz-executor 43511 56297 73918 0 2 0 syz-executor 43511 93203 73918 0 2 0x4000000 syz-executor 56495 394681 81461 0 7 0x1000000 syz-executor *56495 302942 81461 0 7 0x5000000 syz-executor 56495 244100 81461 0 2 0x5000000 syz-executor 44423 141673 51690 0 2 0 syz-executor 44423 508964 51690 0 3 0x4000080 fsleep syz-executor 81176 515514 28557 0 2 0 syz-executor 81176 296748 28557 0 3 0x4000080 fsleep syz-executor 17535 46509 1 0 3 0x82 nanoslp getty 43172 227415 33597 0 3 0x80 nanoslp syz-executor 43172 455606 33597 0 3 0x4000080 fsleep syz-executor 43172 238283 33597 0 3 0x4000080 fsleep syz-executor 43172 72120 33597 0 3 0x4000080 kqread syz-executor 33597 189671 94248 0 3 0x82 nanoslp syz-executor 97596 294011 0 0 3 0x14200 bored sosplice 28557 267051 94248 0 3 0x82 nanoslp syz-executor 26013 393658 94248 0 3 0x82 nanoslp syz-executor 58262 324368 94248 0 3 0x82 wait syz-executor 51690 191150 94248 0 3 0x82 nanoslp syz-executor 81461 464345 94248 0 3 0x82 nanoslp syz-executor 86455 36260 94248 0 3 0x82 piperd syz-executor 73918 260912 94248 0 3 0x82 nanoslp syz-executor 94248 11533 1850 0 2 0x2 syz-executor 1850 333581 41993 0 3 0x10008a sigsusp ksh 41993 383817 88727 0 3 0x98 kqread sshd-session 88727 319004 76655 0 3 0x92 kqread sshd-session 76655 239728 1 0 3 0x88 kqread sshd 57925 100241 98971 74 3 0x1100092 bpf pflogd 98971 170999 1 0 3 0x80 sbwait pflogd 68952 299972 13728 73 3 0x1100090 kqread syslogd 13728 131166 1 0 3 0x100082 sbwait syslogd 45402 357848 1 0 3 0x100080 kqread resolvd 35819 60514 89710 77 3 0x100092 kqread dhcpleased 79070 459487 89710 77 3 0x100092 kqread dhcpleased 89710 33321 1 0 3 0x80 kqread dhcpleased 75320 1578 0 0 3 0x14200 bored smr 63358 30035 0 0 2 0x14200 zerothread 33633 57340 0 0 3 0x14200 aiodoned aiodoned 80546 191634 0 0 3 0x14200 syncer update 32271 243638 0 0 3 0x14200 cleaner cleaner 59150 6092 0 0 3 0x14200 reaper reaper 62186 497157 0 0 3 0x14200 pgdaemon pagedaemon 75711 44608 0 0 3 0x14200 bored viomb 4668 277754 0 0 3 0x40014200 acpi0 acpi0 46261 103559 0 0 3 0x40014200 idle1 32884 426709 0 0 3 0x14200 bored softnet3 60429 341107 0 0 3 0x14200 bored softnet2 60235 119605 0 0 3 0x14200 bored softnet1 52040 160867 0 0 2 0x14200 softnet0 73186 229888 0 0 3 0x14200 bored systqmp 48503 450012 0 0 3 0x14200 bored systq 59949 293265 0 0 3 0x14200 tmoslp softclockmp 23830 309929 0 0 3 0x40014200 tmoslp softclock 63737 491981 0 0 3 0x40014200 idle0 1 514122 0 0 3 0x82 wait init 0 0 -1 0 3 0x10200 scheduler swapper ddb{1}> show all locks Process 56495 (syz-executor) thread 0xffff80002a43c020 (302942) exclusive kernel_lock &kernel_lock r = 0 (0xffffffff8360db10) #0 witness_lock+0x5bb stacktrace_save sys/sys/stacktrace.h:37 [inline] #0 witness_lock+0x5bb sys/kern/subr_witness.c:1155 #1 unp_externalize+0x14e #2 soreceive+0xc73 sys/kern/uipc_socket.c:1015 #3 recvit+0x40a sys/kern/uipc_syscalls.c:1072 #4 sys_recvmsg+0x1bf sys/kern/uipc_syscalls.c:872 #5 syscall+0xbc6 mi_syscall sys/sys/syscall_mi.h:176 [inline] #5 syscall+0xbc6 sys/arch/amd64/amd64/trap.c:577 #6 Xsyscall+0x128 exclusive rwlock sbufrcv r = 0 (0xffff80000128cbb0) #0 witness_lock+0x5bb stacktrace_save sys/sys/stacktrace.h:37 [inline] #0 witness_lock+0x5bb sys/kern/subr_witness.c:1155 #1 rw_do_enter_write+0x3ea sys/kern/kern_rwlock.c:316 #2 sblock+0xb7 sys/kern/uipc_socket2.c:537 #3 soreceive+0x28e sys/kern/uipc_socket.c:876 #4 recvit+0x40a sys/kern/uipc_syscalls.c:1072 #5 sys_recvmsg+0x1bf sys/kern/uipc_syscalls.c:872 #6 syscall+0xbc6 mi_syscall sys/sys/syscall_mi.h:176 [inline] #6 syscall+0xbc6 sys/arch/amd64/amd64/trap.c:577 #7 Xsyscall+0x128 ddb{1}> show malloc Type InUse MemUse HighUse Limit Requests Type Lim devbuf 10209 11157K 11410K 166960K 11975 0 pcb 17 13K 14K 166960K 151 0 rtable 199 6K 6K 166960K 438 0 pf 32 17K 21K 166960K 66 0 ifaddr 39 6K 7K 166960K 56 0 ifgroup 51 2K 2K 166960K 68 0 sysctl 3 1K 1K 166960K 3 0 counters 62 36K 36K 166960K 72 0 ioctlops 0 0K 4K 166960K 1525 0 iov 0 0K 16K 166960K 31 0 mount 1 1K 1K 166960K 1 0 log 0 0K 0K 166960K 4 0 vnodes 1388 87K 88K 166960K 1794 0 UFS quota 1 32K 32K 166960K 1 0 UFS mount 5 36K 36K 166960K 5 0 shm 2 1K 9K 166960K 9 0 VM map 2 1K 1K 166960K 2 0 sem 11 0K 0K 166960K 95 0 dirhash 12 2K 2K 166960K 27 0 ACPI 1690 195K 286K 166960K 12468 0 file desc 17 61K 93K 166960K 675 0 sigio 0 0K 0K 166960K 7 0 proc 72 91K 128K 166960K 628 0 subproc 72 4K 4K 166960K 81 0 NFS srvsock 1 0K 0K 166960K 1 0 NFS daemon 1 16K 16K 166960K 1 0 ip_moptions 0 0K 0K 166960K 93 0 in_multi 88 6K 7K 166960K 132 0 ether_multi 1 0K 0K 166960K 4 0 mrt 0 0K 0K 166960K 2 0 ISOFS mount 1 32K 32K 166960K 1 0 MSDOSFS mount 1 16K 16K 166960K 1 0 ttys 73 334K 334K 166960K 73 0 exec 0 0K 1K 166960K 441 0 fusefs mount 1 32K 32K 166960K 1 0 tdb 3 0K 0K 166960K 3 0 VM swap 8 62K 64K 166960K 10 0 UVM amap 222 72K 81K 166960K 7902 0 UVM aobj 18 4K 4K 166960K 22 0 pinsyscall 42 84K 104K 166960K 1835 0 memdesc 1 4K 4K 166960K 1 0 crypto data 1 1K 1K 166960K 1 0 ip6_options 0 0K 0K 166960K 36 0 NDP 11 0K 2K 166960K 35 0 temp 53 6894K 6962K 166960K 24930 0 kqueue 14 22K 28K 166960K 103 0 SYN cache 2 16K 16K 166960K 2 0 ddb{1}> show all pools Name Size Requests Fail Releases Pgreq Pgrel Npage Hiwat Minpg Maxpg Idle plcache 128 24 0 0 1 0 1 1 0 8 0 rtpcb 120 75 0 72 1 0 1 1 0 8 0 rtentry 112 133 0 42 4 0 4 4 0 8 0 unpcb 144 363 0 343 4 3 1 2 0 8 0 syncache 336 6 0 6 3 2 1 1 0 8 1 tcpqe 32 2 0 2 1 0 1 1 0 8 1 tcpcb 808 133 0 123 4 2 2 2 0 8 0 arp 120 22 0 5 1 0 1 1 0 8 0 inpcb 376 559 0 546 8 6 2 5 0 8 0 nd6 136 30 0 8 1 0 1 1 0 8 0 pkpcb 40 3 0 3 2 2 0 1 0 8 0 kcovpl 48 9 0 1 1 0 1 1 0 8 0 ppxss 1168 2 0 2 1 1 0 1 0 8 0 pffrag 232 2 0 0 1 0 1 1 0 482 0 pffrnode 88 2 0 0 1 0 1 1 0 8 0 pffrent 40 3 0 1 1 0 1 1 0 8 0 pfosfp 40 1428 0 1005 5 0 5 5 0 8 0 pfosfpen 112 1428 0 714 21 0 21 21 0 8 0 pfanchor 1288 2 0 0 1 0 1 1 0 8 0 pfstitem 24 56 0 12 1 0 1 1 0 8 0 pfstkey 128 56 0 12 2 0 2 2 0 8 0 pfstate 376 56 0 12 6 0 6 6 0 8 0 pfrule 1344 23 0 17 2 1 1 2 0 8 0 art_heap8 4096 1 0 0 1 0 1 1 0 8 0 art_heap4 256 556 0 156 30 1 29 29 0 8 2 art_table 32 557 0 156 4 0 4 4 0 8 0 art_node 16 130 0 49 1 0 1 1 0 8 0 sysvmsgpl 40 11 0 8 1 0 1 1 0 8 0 semupl 112 2 0 2 1 1 0 1 0 8 0 semapl 112 89 0 80 1 0 1 1 0 8 0 shmpl 112 19 0 4 1 0 1 1 0 8 0 dirhash 1024 28 0 11 3 0 3 3 0 8 0 dino2pl 256 2552 0 1054 95 0 95 95 0 8 0 ffsino 280 2552 0 1054 109 0 109 109 0 8 0 nchpl 144 3509 0 2968 63 40 23 63 0 8 0 uvmvnodes 80 2913 0 0 60 0 60 60 0 8 0 vnodes 216 2913 0 0 162 0 162 162 0 8 0 namei 1024 12012 0 12012 3 2 1 2 0 8 1 percpumem 16 50 0 5 1 0 1 1 0 8 0 kstatmem 264 32 0 10 2 0 2 2 0 8 0 scsiplug 72 2 0 2 1 1 0 1 0 8 0 scxspl 216 10437 0 10437 10 9 1 8 1 8 1 plimitpl 152 86 0 68 1 0 1 1 0 8 0 sigapl 424 988 0 938 7 1 6 7 0 8 0 futexpl 64 8376 0 8371 1 0 1 1 0 8 0 knotepl 120 558 0 0 17 0 17 17 0 8 0 kqueuepl 216 178 0 133 3 0 3 3 0 8 0 pipepl 328 162 0 135 3 0 3 3 0 8 0 fdescpl 504 968 0 937 5 0 5 5 0 8 0 filepl 152 5393 0 5062 19 5 14 17 0 8 0 lockfpl 104 451 0 448 2 1 1 2 0 8 0 lockfspl 48 90 0 87 1 0 1 1 0 8 0 sessionpl 144 27 0 19 1 0 1 1 0 8 0 pgrppl 48 39 0 23 1 0 1 1 0 8 0 ucredpl 104 679 0 665 1 0 1 1 0 8 0 zombiepl 144 939 0 938 1 0 1 1 0 8 0 processpl 1168 988 0 938 5 1 4 5 0 8 0 procpl 648 1871 0 1812 6 0 6 6 0 8 0 srpgc 96 4 0 4 2 2 0 1 0 8 0 sosppl 168 1 0 1 1 1 0 1 0 8 0 sockpl 688 1071 0 1035 9 5 4 7 0 8 0 mcl64k 65536 2 0 0 1 0 1 1 0 8 0 mcl16k 16384 3 0 0 1 0 1 1 0 8 0 mcl12k 12288 1 0 0 1 0 1 1 0 8 0 mcl8k 8192 4 0 0 1 0 1 1 0 8 0 mcl4k 4096 115 0 0 15 0 15 15 0 8 0 mcl2k 2048 28 0 0 4 0 4 4 0 8 0 mtagpl 96 80 0 0 2 0 2 2 0 8 0 mbufpl 256 234 0 0 15 0 15 15 0 8 0 bufpl 280 3392 0 134 233 0 233 233 0 8 0 anonpl 24 148136 0 144775 51 16 35 46 0 185 6 amapchunkpl 152 24762 0 24276 28 5 23 25 0 158 2 amappl16 200 2784 0 2747 16 9 7 12 0 8 3 amappl15 192 4 0 4 1 1 0 1 0 8 0 amappl14 184 125 0 112 1 0 1 1 0 8 0 amappl13 176 3 0 3 1 1 0 1 0 8 0 amappl12 168 1653 0 1622 4 1 3 3 0 8 0 amappl11 160 53 0 38 1 0 1 1 0 8 0 amappl10 152 33 0 33 1 1 0 1 0 8 0 amappl9 144 246 0 246 1 1 0 1 0 8 0 amappl8 136 21 0 19 1 0 1 1 0 8 0 amappl7 128 116 0 104 1 0 1 1 0 8 0 amappl6 120 196 0 193 1 0 1 1 0 8 0 amappl5 112 135 0 126 1 0 1 1 0 8 0 amappl4 104 345 0 324 1 0 1 1 0 8 0 amappl3 96 4912 0 4799 4 0 4 4 0 8 0 amappl2 88 771 0 708 2 0 2 2 0 8 0 amappl1 80 10301 0 9736 18 2 16 16 0 8 0 amappl 88 7454 0 7288 5 0 5 5 0 92 0 dma4096 4096 1 0 1 1 1 0 1 0 8 0 dma1024 1024 1 0 0 1 0 1 1 0 8 0 dma256 256 6 0 6 1 1 0 1 0 8 0 dma128 128 255 0 255 2 2 0 1 0 8 0 dma64 64 6 0 6 1 1 0 1 0 8 0 dma32 32 7 0 7 1 1 0 1 0 8 0 dma16 16 18 0 17 1 0 1 1 0 8 0 aobjpl 72 21 0 4 1 0 1 1 0 8 0 uaddrrnd 24 968 0 937 1 0 1 1 0 8 0 uaddrbest 32 2 0 0 1 0 1 1 0 8 0 uaddr 24 968 0 937 1 0 1 1 0 8 0 vmmpekpl 168 9565 0 9530 3 0 3 3 0 8 0 vmmpepl 168 65085 0 63219 101 6 95 95 0 357 1 vmsppl 456 967 0 937 7 2 5 5 0 8 0 rwobjpl 64 22824 0 18966 67 1 66 66 0 8 0 pdppl 4096 1943 0 1874 121 46 75 85 0 8 6 pvpl 32 14667 0 0 120 1 119 120 0 265 0 pmappl 248 967 0 937 3 0 3 3 0 8 0 extentpl 40 55 0 38 1 0 1 1 0 8 0 phpool 112 311 0 57 8 0 8 8 0 8 0 ddb{1}> machine ddbcpu 0 Stopped at x86_ipi_db+0x27: addq $0x8,%rsp ddb{0}> trace x86_ipi_db(ffffffff834ccff0) at x86_ipi_db+0x27 sys/arch/amd64/amd64/db_interface.c:393 x86_ipi_handler() at x86_ipi_handler+0xd9 sys/arch/amd64/amd64/ipi.c:106 Xresume_lapic_ipi() at Xresume_lapic_ipi+0x27 __mp_lock(ffffffff8360d908) at __mp_lock+0x192 __mp_lock_spin sys/kern/kern_lock.c:113 [inline] __mp_lock(ffffffff8360d908) at __mp_lock+0x192 sys/kern/kern_lock.c:144 softintr_dispatch(0) at softintr_dispatch+0x5b sys/arch/amd64/amd64/softintr.c:88 Xsoftclock() at Xsoftclock+0x27 end of kernel end trace frame: 0x733a9d644070, count: -6 ddb{0}> machine ddbcpu 1 Stopped at db_enter+0x25: addq $0x8,%rsp ddb{1}> trace db_enter() at db_enter+0x25 sys/arch/amd64/amd64/db_interface.c:437 witness_checkorder(fffffd8065607848,9,0) at witness_checkorder+0x1047 rw_do_enter_write(fffffd8065607830,1) at rw_do_enter_write+0xb7 sys/kern/kern_rwlock.c:233 rrw_enter(fffffd8065607830,1) at rrw_enter+0xc6 sys/kern/kern_rwlock.c:616 VOP_LOCK(fffffd806df5d460,2001) at VOP_LOCK+0xa6 sys/kern/vfs_vops.c:524 vn_lock(fffffd806df5d460,2001) at vn_lock+0xa4 sys/kern/vfs_vnops.c:570 vfs_getcwd_common(fffffd806df5d460,fffffd806df5d460,0,0,200,0,f2d54a3fcb02ee77) at vfs_getcwd_common+0xd1 sys/kern/vfs_getcwd.c:287 vn_isunder(fffffd806df5d460,fffffd806df5d460,ffff80002a43c020) at vn_isunder+0x56 sys/kern/vfs_vnops.c:694 unp_externalize(fffffd8060f10c00,ec,0) at unp_externalize+0x286 sys/kern/uipc_usrreq.c:1086 soreceive(ffff80000128cad0,ffff80003b99e628,ffff80003b99e5d8,0,ffff80003b99e618,ffff80003b99e79c,9449d065c30a975a) at soreceive+0xc73 sys/kern/uipc_socket.c:1015 recvit(ffff80002a43c020,6,ffff80003b99e770,0,ffff80003b99e820) at recvit+0x40a sys/kern/uipc_syscalls.c:1072 sys_recvmsg(ffff80002a43c020,ffff80003b99e8d0,ffff80003b99e820) at sys_recvmsg+0x1bf sys/kern/uipc_syscalls.c:872 syscall(ffff80003b99e8d0) at syscall+0xbc6 mi_syscall sys/sys/syscall_mi.h:176 [inline] syscall(ffff80003b99e8d0) at syscall+0xbc6 sys/arch/amd64/amd64/trap.c:577 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0x3a152be7170, count: -14