diff --git a/net/dsa/user.c b/net/dsa/user.c
index b738a466e2dc..e6bb2c527067 100644
--- a/net/dsa/user.c
+++ b/net/dsa/user.c
@@ -2865,7 +2865,8 @@ static int dsa_user_changeupper(struct net_device *dev,
 static int dsa_user_prechangeupper(struct net_device *dev,
 				   struct netdev_notifier_changeupper_info *info)
 {
-	struct dsa_port *dp = dsa_user_to_port(dev);
+	const struct rtnl_link_ops *ops = dev->rtnl_link_ops;
+	struct dsa_port *dp = ops->priv_size ? dsa_user_to_port(dev) : NULL;
 
 	if (!dsa_user_dev_check(dev))
 		return NOTIFY_DONE;