diff --git a/include/net/sock.h b/include/net/sock.h
index 8036b3b79cd8be64550dcfd6ce213039460acb1f..7ef728324e4e7e6e9cfa7aad56d4f46b29de2e06 100644
--- a/include/net/sock.h
+++ b/include/net/sock.h
@@ -1751,6 +1751,7 @@ static inline bool sock_allow_reclassification(const struct sock *csk)
 struct sock *sk_alloc(struct net *net, int family, gfp_t priority,
                      struct proto *prot, int kern);
 void sk_free(struct sock *sk);
+void sk_net_refcnt_upgrade(struct sock *sk);
 void sk_destruct(struct sock *sk);
 struct sock *sk_clone_lock(const struct sock *sk, const gfp_t priority);
 void sk_free_unlock_clone(struct sock *sk);
diff --git a/net/core/sock.c b/net/core/sock.c
index eae2ae70a2e03df370d8ef7750a7bb13cc3b8d8f..6c0e87f97fa4a7220950ff576b10291e3e55731d 100644
--- a/net/core/sock.c
+++ b/net/core/sock.c
@@ -2246,6 +2246,7 @@ struct sock *sk_alloc(struct net *net, int family, gfp_t priority,
                        get_net_track(net, &sk->ns_tracker, priority);
                        sock_inuse_add(net, 1);
                } else {
+                       net_passive_inc(net);
                        __netns_tracker_alloc(net, &sk->ns_tracker,
                                              false, priority);
                }
@@ -2270,6 +2271,7 @@ EXPORT_SYMBOL(sk_alloc);
 static void __sk_destruct(struct rcu_head *head)
 {
        struct sock *sk = container_of(head, struct sock, sk_rcu);
+       struct net *net = sock_net(sk);
        struct sk_filter *filter;
 
        if (sk->sk_destruct)
@@ -2301,14 +2303,28 @@ static void __sk_destruct(struct rcu_head *head)
        put_cred(sk->sk_peer_cred);
        put_pid(sk->sk_peer_pid);
 
-       if (likely(sk->sk_net_refcnt))
-               put_net_track(sock_net(sk), &sk->ns_tracker);
-       else
-               __netns_tracker_free(sock_net(sk), &sk->ns_tracker, false);
-
+       if (likely(sk->sk_net_refcnt)) {
+               put_net_track(net, &sk->ns_tracker);
+       } else {
+               __netns_tracker_free(net, &sk->ns_tracker, false);
+               net_passive_dec(net);
+       }
        sk_prot_free(sk->sk_prot_creator, sk);
 }
 
+void sk_net_refcnt_upgrade(struct sock *sk)
+{
+       struct net *net = sock_net(sk);
+
+       WARN_ON_ONCE(sk->sk_net_refcnt);
+       __netns_tracker_free(net, &sk->ns_tracker, false);
+       net_passive_dec(net);
+       sk->sk_net_refcnt = 1;
+       get_net_track(net, &sk->ns_tracker, GFP_KERNEL);
+       sock_inuse_add(net, 1);
+}
+EXPORT_SYMBOL_GPL(sk_net_refcnt_upgrade);
+
 void sk_destruct(struct sock *sk)
 {
        bool use_call_rcu = sock_flag(sk, SOCK_RCU_FREE);
@@ -2405,6 +2421,7 @@ struct sock *sk_clone_lock(const struct sock *sk, const gfp_t priority)
                 * is not properly dismantling its kernel sockets at netns
                 * destroy time.
                 */
+               net_passive_inc(sock_net(newsk));
                __netns_tracker_alloc(sock_net(newsk), &newsk->ns_tracker,
                                      false, priority);
        }
diff --git a/net/mptcp/subflow.c b/net/mptcp/subflow.c
index fd021cf8286eff9234b950a4d4c083ea7756eba3..dfcbef9c46246983d21c827d9551d4eb2c95430e 100644
--- a/net/mptcp/subflow.c
+++ b/net/mptcp/subflow.c
@@ -1772,10 +1772,7 @@ int mptcp_subflow_create_socket(struct sock *sk, unsigned short family,
         * needs it.
         * Update ns_tracker to current stack trace and refcounted tracker.
         */
-       __netns_tracker_free(net, &sf->sk->ns_tracker, false);
-       sf->sk->sk_net_refcnt = 1;
-       get_net_track(net, &sf->sk->ns_tracker, GFP_KERNEL);
-       sock_inuse_add(net, 1);
+       sk_net_refcnt_upgrade(sf->sk);
        err = tcp_set_ulp(sf->sk, "mptcp");
        if (err)
                goto err_free;
diff --git a/net/netlink/af_netlink.c b/net/netlink/af_netlink.c
index 85311226183a259574c64643eae9e906ccc9c1da..a53ea60d0a78d1b739ab392b16d383686c613d7f 100644
--- a/net/netlink/af_netlink.c
+++ b/net/netlink/af_netlink.c
@@ -795,16 +795,6 @@ static int netlink_release(struct socket *sock)
 
        sock_prot_inuse_add(sock_net(sk), &netlink_proto, -1);
 
-       /* Because struct net might disappear soon, do not keep a pointer. */
-       if (!sk->sk_net_refcnt && sock_net(sk) != &init_net) {
-               __netns_tracker_free(sock_net(sk), &sk->ns_tracker, false);
-               /* Because of deferred_put_nlk_sk and use of work queue,
-                * it is possible  netns will be freed before this socket.
-                */
-               sock_net_set(sk, &init_net);
-               __netns_tracker_alloc(&init_net, &sk->ns_tracker,
-                                     false, GFP_KERNEL);
-       }
        call_rcu(&nlk->rcu, deferred_put_nlk_sk);
        return 0;
 }
diff --git a/net/rds/tcp.c b/net/rds/tcp.c
index 0581c53e6517043ad6c2ad4207b26ab169989ed8..3cc2f303bf786579a77b8b67ddc77468e085a3d3 100644
--- a/net/rds/tcp.c
+++ b/net/rds/tcp.c
@@ -504,12 +504,8 @@ bool rds_tcp_tune(struct socket *sock)
                        release_sock(sk);
                        return false;
                }
-               /* Update ns_tracker to current stack trace and refcounted tracker */
-               __netns_tracker_free(net, &sk->ns_tracker, false);
-
-               sk->sk_net_refcnt = 1;
-               netns_tracker_alloc(net, &sk->ns_tracker, GFP_KERNEL);
-               sock_inuse_add(net, 1);
+               sk_net_refcnt_upgrade(sk);
+               put_net(net);
        }
        rtn = net_generic(net, rds_tcp_netid);
        if (rtn->sndbuf_size > 0) {
diff --git a/net/smc/af_smc.c b/net/smc/af_smc.c
index ca6984541edbda33a3539a48ed7c2aefecfa690e..3e6cb35baf25aff51518fe98a0bcbf1c3cec1629 100644
--- a/net/smc/af_smc.c
+++ b/net/smc/af_smc.c
@@ -3337,10 +3337,7 @@ int smc_create_clcsk(struct net *net, struct sock *sk, int family)
         * which need net ref.
         */
        sk = smc->clcsock->sk;
-       __netns_tracker_free(net, &sk->ns_tracker, false);
-       sk->sk_net_refcnt = 1;
-       get_net_track(net, &sk->ns_tracker, GFP_KERNEL);
-       sock_inuse_add(net, 1);
+       sk_net_refcnt_upgrade(sk);
        return 0;
 }
 
diff --git a/net/sunrpc/svcsock.c b/net/sunrpc/svcsock.c
index cb3bd12f5818bad13226d9b91be2881c55f56bc2..72e5a01df3d352582a5c25e0b8081a041e3792ee 100644
--- a/net/sunrpc/svcsock.c
+++ b/net/sunrpc/svcsock.c
@@ -1541,10 +1541,7 @@ static struct svc_xprt *svc_create_socket(struct svc_serv *serv,
        newlen = error;
 
        if (protocol == IPPROTO_TCP) {
-               __netns_tracker_free(net, &sock->sk->ns_tracker, false);
-               sock->sk->sk_net_refcnt = 1;
-               get_net_track(net, &sock->sk->ns_tracker, GFP_KERNEL);
-               sock_inuse_add(net, 1);
+               sk_net_refcnt_upgrade(sock->sk);
                if ((error = kernel_listen(sock, 64)) < 0)
                        goto bummer;
        }
diff --git a/net/sunrpc/xprtsock.c b/net/sunrpc/xprtsock.c
index c60936d8cef71b3eb7937a3ec8dd8a4edbaaff10..940fe65b2a3511933a3f87465cdee1095ec096c6 100644
--- a/net/sunrpc/xprtsock.c
+++ b/net/sunrpc/xprtsock.c
@@ -1941,12 +1941,8 @@ static struct socket *xs_create_sock(struct rpc_xprt *xprt,
                goto out;
        }
 
-       if (protocol == IPPROTO_TCP) {
-               __netns_tracker_free(xprt->xprt_net, &sock->sk->ns_tracker, false);
-               sock->sk->sk_net_refcnt = 1;
-               get_net_track(xprt->xprt_net, &sock->sk->ns_tracker, GFP_KERNEL);
-               sock_inuse_add(xprt->xprt_net, 1);
-       }
+       if (protocol == IPPROTO_TCP)
+               sk_net_refcnt_upgrade(sock->sk);
 
        filp = sock_alloc_file(sock, O_NONBLOCK, NULL);
        if (IS_ERR(filp))