diff --git a/net/mptcp/pm_netlink.c b/net/mptcp/pm_netlink.c
index 4ff8d55cbe82..3186d33b5208 100644
--- a/net/mptcp/pm_netlink.c
+++ b/net/mptcp/pm_netlink.c
@@ -700,6 +700,9 @@ static void mptcp_pm_nl_rm_addr_or_subflow(struct mptcp_sock *msk,
 
        msk_owned_by_me(msk);
 
+       if (sk->sk_state == TCP_LISTEN)
+               return;
+
        if (!rm_list->nr)
                return;