pf: key search, out on vio0: witness: TCPlock order reversal: wire: (0) 1st 0xffff800039baa6a0 fdlock (&newfdp->fd_fd.fd_lock) 10.128.15.235 2nd 0xfffffc00701e4e98 inode (&ip->i_lock) :30002lock order [1] fdlock (&newfdp->fd_fd.fd_lock) -> [2] inode (&ip->i_lock) lock order data 0xffffffff834a11a8 -> 0xffffffff83493481 is missing 10.128.1.42lock order [2] inode (&ip->i_lock) -> [3] sbufrcv (&so->so_rcv.sb_lock) :31598#0 rw_do_enter_write+0xba #1 sblock+0xb6 sys/kern/uipc_socket2.c:536 #2 soreceive+0x27d sys/kern/uipc_socket.c:890 #3 fifo_read+0x117 sys/miscfs/fifofs/fifo_vnops.c:264 #4 VOP_READ+0x101 sys/kern/vfs_vops.c:227 #5 vn_rdwr+0x15b sys/kern/vfs_vnops.c:-1 #6 vndsetcred+0xa1 sys/dev/vnd.c:685 #7 vndioctl+0xdfc sys/dev/vnd.c:486 #8 VOP_IOCTL+0xac sys/kern/vfs_vops.c:264 #9 vn_ioctl+0xf8 sys/kern/vfs_vnops.c:537 #10 sys_ioctl+0x674 sys/kern/sys_generic.c:-1 #11 syscall+0xbd4 mi_syscall sys/sys/syscall_mi.h:176 [inline] #11 syscall+0xbd4 sys/arch/amd64/amd64/trap.c:783 #12 Xsyscall+0x128 lock order [3] sbufrcv (&so->so_rcv.sb_lock) -> [1] fdlock (&newfdp->fd_fd.fd_lock) #0 rw_do_enter_write+0xba sys/kern/kern_rwlock.c:234 #1 unp_externalize+0x3cf sys/kern/uipc_usrreq.c:1074 #2 soreceive+0xc24 sys/kern/uipc_socket.c:1029 #3 recvit+0x40b sys/kern/uipc_syscalls.c:1078 #4 sys_recvmsg+0x1bf sys/kern/uipc_syscalls.c:878 #5 syscall+0xbd4 mi_syscall sys/sys/syscall_mi.h:176 [inline] #5 syscall+0xbd4 sys/arch/amd64/amd64/trap.c:783 #6 Xsyscall+0x128 Stopped at db_enter+0x25: addq $0x8,%rsp ddb{0}> ddb{0}> set $lines = 0 ddb{0}> set $maxwidth = 0 ddb{0}> show panic the kernel did not panic ddb{0}> trace db_enter() at db_enter+0x25 sys/arch/amd64/amd64/db_interface.c:438 witness_checkorder(fffffc00701e4e98,9,0) at witness_checkorder+0x10d1 sys/kern/subr_witness.c:-1 rw_do_enter_write(fffffc00701e4e80,1) at rw_do_enter_write+0xba sys/kern/kern_rwlock.c:234 rrw_enter(fffffc00701e4e80,1) at rrw_enter+0xc6 sys/kern/kern_rwlock.c:621 VOP_LOCK(fffffc00702e3298,2001) at VOP_LOCK+0xbd sys/kern/vfs_vops.c:527 vn_lock(fffffc00702e3298,2001) at vn_lock+0xa4 sys/kern/vfs_vnops.c:576 vn_closefile(fffffc006f98d828,ffff800045ffd9f8) at vn_closefile+0x111 vn_close sys/kern/vfs_vnops.c:298 [inline] vn_closefile(fffffc006f98d828,ffff800045ffd9f8) at vn_closefile+0x111 sys/kern/vfs_vnops.c:621 fdrop(fffffc006f98d828,ffff800045ffd9f8) at fdrop+0x121 sys/kern/kern_descrip.c:1281 knote_drop(fffffc006c9c7b50,ffff800045ffd9f8) at knote_drop+0x1a1 sys/kern/kern_event.c:2314 knote_remove(ffff800045ffd9f8,fffffc006fad87e0,fffffc006fad8870,5,0) at knote_remove+0x215 sys/kern/kern_event.c:-1 knote_fdclose(ffff800045ffd9f8,5) at knote_fdclose+0xf9 sys/kern/kern_event.c:2221 fdrelease(ffff800045ffd9f8,5) at fdrelease+0xf9 sys/kern/kern_descrip.c:761 sys_closefrom(ffff800045ffd9f8,ffff80002a28e3e0,ffff80002a28e330) at sys_closefrom+0x13c sys/kern/kern_descrip.c:1471 syscall(ffff80002a28e3e0) at syscall+0xbd4 mi_syscall sys/sys/syscall_mi.h:176 [inline] syscall(ffff80002a28e3e0) at syscall+0xbd4 sys/arch/amd64/amd64/trap.c:783 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0x12d6d127c40, count: -15 ddb{0}> show registers rdi 0 rsi 0 rbp 0xffff80002a28de30 rbx 0 rdx 0 rcx 0xffff800045ffd9f8 rax 0xffffffff8399fff0 cpu_info_full_primary+0x1ff0 r8 0xffff80002a28dd10 r9 0x8080808080808080 r10 0x6a5aabfc57bbcec1 r11 0x6523ade52a6cec82 r12 0xfffffc00040c3d00 r13 0xfffffc00048b0b48 r14 0x3 r15 0xffffffff835239a2 substchar+0x3b32 rip 0xffffffff816aff35 db_enter+0x25 cs 0x8 rflags 0x246 rsp 0xffff80002a28de20 ss 0x10 db_enter+0x25: addq $0x8,%rsp ddb{0}> show proc PROC (syz-executor) tid=326169 pid=63237 tcnt=3 stat=onproc flags process=0 proc=4000000 runpri=32, usrpri=50, slppri=32, nice=20 wchan=0x0, wmesg=, ps_single=0x0 scnt=0 ecnt=0 forw=0xffffffffffffffff, list=0xffff800045ffc2a0,0xffff80002a223a10 process=0xffff80003478d830 user=0xffff80002a289000, vmspace=0xfffffc000b1f71e8 estcpu=36, cpticks=0, pctcpu=0.0, user=0, sys=0, intr=0 ddb{0}> ps PID TID PPID UID S FLAGS WAIT COMMAND 48350 192720 47541 0 2 0 syz-executor 48350 72041 47541 0 3 0x4000080 fsleep syz-executor 95583 395014 22506 0 2 0 syz-executor 95583 185043 22506 0 2 0x4000000 syz-executor 95583 455634 22506 0 3 0x4000080 fsleep syz-executor 63237 403300 15041 0 2 0 syz-executor *63237 326169 15041 0 7 0x4000000 syz-executor 63237 212174 15041 0 3 0x4000080 fsleep syz-executor 72780 277110 32852 0 2 0 syz-executor 72780 200948 32852 0 3 0x4000080 fsleep syz-executor 72780 276422 32852 0 3 0x4000080 fsleep syz-executor 67845 470349 5501 0 2 0 syz-executor 67845 381850 5501 0 3 0x4000080 fsleep syz-executor 89464 415916 26607 0 2 0xc80 syz-executor 89464 108872 26607 0 3 0x4000080 fsleep syz-executor 89464 22607 26607 0 2 0x4000000 syz-executor 85832 193459 52452 -1 2 0xc90 syz-executor 85832 485064 52452 -1 3 0x4000090 msgwait syz-executor 85832 137140 52452 -1 3 0x4000090 fsleep syz-executor 85832 395748 52452 -1 3 0x4000090 fsleep syz-executor 2973 481330 1 0 3 0x80 nanoslp init 22506 239445 96729 0 3 0x82 nanoslp syz-executor 32852 466853 96729 0 2 0xc82 syz-executor 52452 279129 96729 0 2 0xc82 syz-executor 5501 434838 96729 0 2 0xc82 syz-executor 26607 320220 96729 0 2 0xc82 syz-executor 15041 351393 96729 0 7 0x3 syz-executor 47541 358536 96729 0 2 0xc82 syz-executor 72406 264254 96729 0 2 0x10000c82 syz-executor 96729 118061 1 0 2 0x2 syz-executor 25897 257838 0 0 3 0x14200 bored smr 23184 120948 0 0 2 0x14200 zerothread 36138 409396 0 0 3 0x14200 aiodoned aiodoned 92899 149414 0 0 3 0x14200 syncer update 39840 122009 0 0 3 0x14200 cleaner cleaner 13873 340549 0 0 3 0x14200 reaper reaper 81872 144044 0 0 3 0x14200 pgdaemon pagedaemon 24078 255401 0 0 3 0x14200 bored viomb 62855 321543 0 0 3 0x40014200 acpi0 acpi0 47240 303795 0 0 3 0x40014200 idle1 17876 25619 0 0 3 0x14200 bored softnet1 28606 203655 0 0 2 0x14200 softnet0 39743 136494 0 0 3 0x14200 bored systqmp 21716 255291 0 0 3 0x14200 bored systq 31593 270626 0 0 3 0x14200 tmoslp softclockmp 65400 474034 0 0 3 0x40014200 tmoslp softclock 64664 173475 0 0 3 0x40014200 idle0 1 266828 0 0 3 0x82 wait init 0 0 -1 0 3 0x10200 scheduler swapper ddb{0}> show all locks Process 63237 (syz-executor) thread 0xffff800045ffd9f8 (326169) exclusive kernel_lock &kernel_lock r = 0 (0xffffffff83aaa180) #0 witness_lock+0x5f1 stacktrace_save sys/sys/stacktrace.h:37 [inline] #0 witness_lock+0x5f1 sys/kern/subr_witness.c:1160 #1 vn_closefile+0x41 sys/kern/vfs_vnops.c:614 #2 fdrop+0x121 sys/kern/kern_descrip.c:1281 #3 knote_drop+0x1a1 sys/kern/kern_event.c:2314 #4 knote_remove+0x215 sys/kern/kern_event.c:-1 #5 knote_fdclose+0xf9 sys/kern/kern_event.c:2221 #6 fdrelease+0xf9 sys/kern/kern_descrip.c:761 #7 sys_closefrom+0x13c sys/kern/kern_descrip.c:1471 #8 syscall+0xbd4 mi_syscall sys/sys/syscall_mi.h:176 [inline] #8 syscall+0xbd4 sys/arch/amd64/amd64/trap.c:783 #9 Xsyscall+0x128 exclusive rwlock fdlock r = 0 (0xffff800039baa6a0) #0 witness_lock+0x5f1 stacktrace_save sys/sys/stacktrace.h:37 [inline] #0 witness_lock+0x5f1 sys/kern/subr_witness.c:1160 #1 rw_do_enter_write+0x419 sys/kern/kern_rwlock.c:320 #2 sys_closefrom+0x1ed sys/kern/kern_descrip.c:1468 #3 syscall+0xbd4 mi_syscall sys/sys/syscall_mi.h:176 [inline] #3 syscall+0xbd4 sys/arch/amd64/amd64/trap.c:783 #4 Xsyscall+0x128 ddb{0}> show malloc Type InUse MemUse HighUse Limit Requests Type Lim devbuf 11049 12020K 12279K 166960K 12516 0 pcb 18 14K 15K 166960K 87 0 rtable 213 8K 9K 166960K 401 0 pf 35 17K 24K 166960K 108 0 ifaddr 40 7K 7K 166960K 58 0 ifgroup 51 2K 2K 166960K 79 0 sysctl 1 1K 9K 166960K 6 0 counters 68 36K 37K 166960K 106 0 ioctlops 0 0K 4K 166960K 1722 0 iov 0 0K 16K 166960K 19 0 mount 1 1K 1K 166960K 1 0 log 1 0K 0K 166960K 5 0 vnodes 1356 85K 86K 166960K 1779 0 UFS quota 1 32K 32K 166960K 1 0 UFS mount 5 36K 36K 166960K 5 0 shm 3 5K 5K 166960K 9 0 VM map 2 1K 1K 166960K 2 0 sem 12 0K 0K 166960K 19 0 dirhash 12 2K 2K 166960K 12 0 ACPI 1692 195K 286K 166960K 12470 0 file desc 16 61K 93K 166960K 487 0 sigio 0 0K 0K 166960K 1 0 proc 12 17K 164K 166960K 594 0 subproc 72 4K 4K 166960K 72 0 NFS srvsock 1 0K 0K 166960K 1 0 NFS daemon 1 16K 16K 166960K 1 0 ip_moptions 0 0K 0K 166960K 38 0 in_multi 92 6K 7K 166960K 113 0 ether_multi 1 0K 0K 166960K 4 0 mrt 0 0K 0K 166960K 15 0 ISOFS mount 1 32K 32K 166960K 1 0 MSDOSFS mount 1 16K 16K 166960K 1 0 ttys 79 360K 360K 166960K 79 0 exec 0 0K 1K 166960K 516 0 fusefs mount 1 32K 32K 166960K 1 0 tdb 3 0K 0K 166960K 3 0 VM swap 8 62K 64K 166960K 10 0 UVM amap 132 78K 174K 166960K 6218 0 UVM aobj 6 2K 2K 166960K 6 0 pinsyscall 18 36K 104K 166960K 1678 0 memdesc 1 4K 4K 166960K 1 0 crypto data 1 1K 1K 166960K 1 0 ip6_options 0 0K 0K 166960K 75 0 NDP 11 0K 1K 166960K 37 0 temp 51 9122K 9141K 166960K 23606 0 kqueue 3 6K 26K 166960K 87 0 SYN cache 2 16K 16K 166960K 2 0 ddb{0}> show all pools Name Size Requests Fail Releases Pgreq Pgrel Npage Hiwat Minpg Maxpg Idle plcache 128 26 0 0 1 0 1 1 0 8 0 rtpcb 120 54 0 54 1 0 1 1 0 8 1 rtentry 176 113 0 23 6 0 6 6 0 8 0 unpcb 144 312 0 308 4 2 2 4 0 8 1 syncache 336 6 0 6 2 1 1 1 0 8 1 tcpcb 736 70 0 68 1 0 1 1 0 8 0 arp 136 18 0 2 1 0 1 1 0 8 0 inpcb 328 427 0 423 7 3 4 7 0 8 3 nd6 152 24 0 3 1 0 1 1 0 8 0 pkpcb 40 1 0 1 1 1 0 1 0 8 0 kcovpl 48 8 0 0 1 0 1 1 0 8 0 ppxss 1192 16 0 16 1 0 1 1 0 8 1 pfstscr 40 6 0 4 1 0 1 1 0 8 0 pffrag 232 8 0 3 1 0 1 1 0 482 0 pffrnode 88 8 0 3 1 0 1 1 0 8 0 pffrent 40 12 0 7 1 0 1 1 0 8 0 pfosfp 40 1428 0 1005 5 0 5 5 0 8 0 pfosfpen 112 1428 0 714 21 0 21 21 0 8 0 pfrktable 1344 2 0 2 1 0 1 1 0 8 1 pfsrclim 320 1 0 1 1 0 1 1 0 8 1 pfstlim 224 1 0 1 1 1 0 1 0 8 0 pfanchor 1288 3 0 3 1 0 1 1 0 8 1 pfqueue 320 65 0 65 1 0 1 1 0 8 1 pfstitem 24 37 0 1 1 0 1 1 0 8 0 pfstkey 128 43 0 7 2 0 2 2 0 8 0 pfstate 448 39 0 4 4 0 4 4 0 8 0 pfrule 1360 27 0 20 2 0 2 2 0 8 0 art_heap8 4096 2 0 0 2 0 2 2 0 8 0 art_heap4 256 484 0 85 29 0 29 29 0 8 4 art_table 40 486 0 85 5 0 5 5 0 8 0 art_node 32 113 0 32 1 0 1 1 0 8 0 sysvmsgpl 40 14 0 3 1 0 1 1 0 8 0 semapl 72 16 0 6 1 0 1 1 0 8 0 shmpl 112 3 0 0 1 0 1 1 0 8 0 dirhash 1024 17 0 0 3 0 3 3 0 8 0 dino2pl 256 2215 0 750 93 0 93 93 0 8 0 ffsino 296 2215 0 750 114 0 114 114 0 8 0 nchpl 144 2908 0 1201 64 0 64 64 0 8 0 rtmask 32 5 0 5 1 0 1 1 0 8 1 vnodes 216 2609 0 0 145 0 145 145 0 8 0 namei 1024 9898 0 9898 2 1 1 2 0 8 1 percpumem 16 68 0 19 1 0 1 1 0 8 0 kstatmem 264 45 0 20 3 0 3 3 0 8 1 scsiplug 72 2 0 2 1 0 1 1 0 8 1 scxspl 216 12873 0 12873 11 3 8 8 1 8 8 plimitpl 152 61 0 52 1 0 1 1 0 8 0 sigapl 424 815 0 779 7 1 6 7 0 8 0 knotepl 120 525 0 0 16 0 16 16 0 8 0 kqueuepl 224 218 0 213 5 0 5 5 0 8 4 pipepl 344 150 0 123 3 0 3 3 0 8 0 fdescpl 528 799 0 780 3 0 3 3 0 8 0 filepl 160 4192 0 4026 14 1 13 13 0 8 4 lockfpl 104 191 0 189 1 0 1 1 0 8 0 lockfspl 48 84 0 82 1 0 1 1 0 8 0 sessionpl 144 36 0 34 1 0 1 1 0 8 0 pgrppl 48 45 0 35 1 0 1 1 0 8 0 ucredpl 104 463 0 459 1 0 1 1 0 8 0 zombiepl 144 780 0 779 1 0 1 1 0 8 0 processpl 1232 815 0 779 5 0 5 5 0 8 0 procpl 664 1417 0 1368 6 0 6 6 0 8 1 sosppl 176 2 0 2 1 1 0 1 0 8 0 sockpl 752 811 0 803 10 3 7 10 0 8 6 mcl64k 65536 3 0 0 1 0 1 1 0 8 0 mcl8k 8192 5 0 0 1 0 1 1 0 8 0 mcl4k 4096 120 0 0 15 0 15 15 0 8 0 mcl2k 2048 24 0 0 3 0 3 3 0 8 0 mtagpl 96 4 0 0 1 0 1 1 0 8 0 mbufpl 256 207 0 0 13 0 13 13 0 8 0 bufpl 272 4482 0 108 292 0 292 292 0 8 0 anonpl 32 8322 0 0 68 0 68 68 0 247 0 amapchunkpl 152 19872 0 19511 33 1 32 32 0 158 17 amappl16 200 2619 0 2594 19 8 11 15 0 8 8 amappl15 192 19 0 19 1 1 0 1 0 8 0 amappl14 184 483 0 483 1 0 1 1 0 8 1 amappl13 176 239 0 239 1 0 1 1 0 8 1 amappl12 168 1057 0 1040 2 0 2 2 0 8 0 amappl11 160 17 0 17 1 1 0 1 0 8 0 amappl10 152 70 0 70 1 0 1 1 0 8 1 amappl9 144 299 0 299 1 1 0 1 0 8 0 amappl8 136 101 0 101 1 0 1 1 0 8 1 amappl7 128 150 0 148 1 0 1 1 0 8 0 amappl6 120 174 0 174 1 0 1 1 0 8 1 amappl5 112 101 0 100 1 0 1 1 0 8 0 amappl4 104 305 0 303 1 0 1 1 0 8 0 amappl3 96 3914 0 3835 4 0 4 4 0 8 0 amappl2 88 561 0 552 2 0 2 2 0 8 0 amappl1 80 12051 0 11957 15 2 13 15 0 8 7 amappl 88 5395 0 5274 5 0 5 5 0 92 1 uvmvnodes 80 116 0 0 3 0 3 3 0 8 0 dma65536 65536 1 0 1 1 0 1 1 0 8 1 dma4096 4096 1 0 1 1 1 0 1 0 8 0 dma1024 1024 1 0 0 1 0 1 1 0 8 0 dma256 256 6 0 6 1 1 0 1 0 8 0 dma128 128 253 0 253 1 1 0 1 0 8 0 dma64 64 6 0 6 1 1 0 1 0 8 0 dma32 32 8 0 8 2 1 1 1 0 8 1 dma16 16 18 0 17 1 0 1 1 0 8 0 aobjpl 72 5 0 0 1 0 1 1 0 8 0 uaddrrnd 24 799 0 780 1 0 1 1 0 8 0 uaddrbest 32 2 0 0 1 0 1 1 0 8 0 uaddr 24 799 0 780 1 0 1 1 0 8 0 vmmpekpl 168 8293 0 8261 2 0 2 2 0 8 0 vmmpepl 168 59291 0 58467 99 0 99 99 0 357 60 vmsppl 488 798 0 780 5 1 4 5 0 8 0 rwobjpl 80 19208 0 18802 28 0 28 28 0 8 2 pdppl 4096 1605 0 1560 103 56 47 85 0 8 2 pvpl 32 15807 0 0 128 0 128 128 0 265 0 pmappl 256 798 0 780 3 0 3 3 0 8 0 extentpl 40 45 0 27 1 0 1 1 0 8 0 phpool 112 573 0 60 15 0 15 15 0 8 0 ddb{0}> machine ddbcpu 0 Invalid cpu 0 ddb{0}> trace db_enter() at db_enter+0x25 sys/arch/amd64/amd64/db_interface.c:438 witness_checkorder(fffffc00701e4e98,9,0) at witness_checkorder+0x10d1 sys/kern/subr_witness.c:-1 rw_do_enter_write(fffffc00701e4e80,1) at rw_do_enter_write+0xba sys/kern/kern_rwlock.c:234 rrw_enter(fffffc00701e4e80,1) at rrw_enter+0xc6 sys/kern/kern_rwlock.c:621 VOP_LOCK(fffffc00702e3298,2001) at VOP_LOCK+0xbd sys/kern/vfs_vops.c:527 vn_lock(fffffc00702e3298,2001) at vn_lock+0xa4 sys/kern/vfs_vnops.c:576 vn_closefile(fffffc006f98d828,ffff800045ffd9f8) at vn_closefile+0x111 vn_close sys/kern/vfs_vnops.c:298 [inline] vn_closefile(fffffc006f98d828,ffff800045ffd9f8) at vn_closefile+0x111 sys/kern/vfs_vnops.c:621 fdrop(fffffc006f98d828,ffff800045ffd9f8) at fdrop+0x121 sys/kern/kern_descrip.c:1281 knote_drop(fffffc006c9c7b50,ffff800045ffd9f8) at knote_drop+0x1a1 sys/kern/kern_event.c:2314 knote_remove(ffff800045ffd9f8,fffffc006fad87e0,fffffc006fad8870,5,0) at knote_remove+0x215 sys/kern/kern_event.c:-1 knote_fdclose(ffff800045ffd9f8,5) at knote_fdclose+0xf9 sys/kern/kern_event.c:2221 fdrelease(ffff800045ffd9f8,5) at fdrelease+0xf9 sys/kern/kern_descrip.c:761 sys_closefrom(ffff800045ffd9f8,ffff80002a28e3e0,ffff80002a28e330) at sys_closefrom+0x13c sys/kern/kern_descrip.c:1471 syscall(ffff80002a28e3e0) at syscall+0xbd4 mi_syscall sys/sys/syscall_mi.h:176 [inline] syscall(ffff80002a28e3e0) at syscall+0xbd4 sys/arch/amd64/amd64/trap.c:783 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0x12d6d127c40, count: -15 ddb{0}> machine ddbcpu 1 Stopped at x86_ipi_db+0x27: addq $0x8,%rsp ddb{1}> trace x86_ipi_db(ffff80002999dff0) at x86_ipi_db+0x27 sys/arch/amd64/amd64/db_interface.c:394 x86_ipi_handler() at x86_ipi_handler+0xd9 sys/arch/amd64/amd64/ipi.c:106 Xresume_lapic_ipi() at Xresume_lapic_ipi+0x27 __sanitizer_cov_trace_pc() at __sanitizer_cov_trace_pc ktrsysret(ffff80002a222540,5b,0,ffff80002a2e4ce0) at ktrsysret+0xde ktrwrite2 sys/kern/kern_ktrace.c:-1 [inline] ktrsysret(ffff80002a222540,5b,0,ffff80002a2e4ce0) at ktrsysret+0xde sys/kern/kern_ktrace.c:209 syscall(ffff80002a2e4d90) at syscall+0xa51 mi_syscall_return sys/sys/syscall_mi.h:204 [inline] syscall(ffff80002a2e4d90) at syscall+0xa51 sys/arch/amd64/amd64/trap.c:804 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0x721286f5dcd0, count: -7 ddb{1}>