diff --git a/net/dsa/user.c b/net/dsa/user.c
index b738a466e2dc..2034587b3a68 100644
--- a/net/dsa/user.c
+++ b/net/dsa/user.c
@@ -2865,14 +2865,16 @@ static int dsa_user_changeupper(struct net_device *dev,
 static int dsa_user_prechangeupper(struct net_device *dev,
 				   struct netdev_notifier_changeupper_info *info)
 {
-	struct dsa_port *dp = dsa_user_to_port(dev);
+	const struct rtnl_link_ops *ops = dev->rtnl_link_ops;
+	struct dsa_port *dp = ops->priv_size >= sizeof(*dp) ? 
+		dsa_user_to_port(dev) : NULL;
 
 	if (!dsa_user_dev_check(dev))
 		return NOTIFY_DONE;
 
-	if (netif_is_bridge_master(info->upper_dev) && !info->linking)
+	if (netif_is_bridge_master(info->upper_dev) && !info->linking && dp)
 		dsa_port_pre_bridge_leave(dp, info->upper_dev);
-	else if (netif_is_lag_master(info->upper_dev) && !info->linking)
+	else if (netif_is_lag_master(info->upper_dev) && !info->linking && dp)
 		dsa_port_pre_lag_leave(dp, info->upper_dev);
 	/* dsa_port_pre_hsr_leave is not yet necessary since hsr devices cannot
 	 * meaningfully placed under a bridge yet