diff --git a/fs/nfs/callback.c b/fs/nfs/callback.c
index bc282b744f34..29134e2c56cd 100644
--- a/fs/nfs/callback.c
+++ b/fs/nfs/callback.c
@@ -61,9 +61,11 @@ static int nfs4_callback_up_net(struct svc_serv *serv, struct net *net)
                dprintk("NFS: Callback listener port = %u (af %u, net %x)\n",
                        nn->nfs_callback_tcpport6, PF_INET6, net->ns.inum);
        } else if (ret != -EAFNOSUPPORT)
-               goto out_err;
+               goto out_destroy_xprts;
        return 0;
 
+out_destroy_xprts:
+       svc_xprt_destroy_all(serv, net, false);
 out_err:
        return (ret) ? ret : -ENOMEM;
 }