diff --git a/drivers/usb/gadget/legacy/raw_gadget.c b/drivers/usb/gadget/legacy/raw_gadget.c
index 112fd18d8c99..3da022aae830 100644
--- a/drivers/usb/gadget/legacy/raw_gadget.c
+++ b/drivers/usb/gadget/legacy/raw_gadget.c
@@ -445,7 +445,6 @@ static int raw_release(struct inode *inode, struct file *fd)
 	int ret = 0;
 	struct raw_dev *dev = fd->private_data;
 	unsigned long flags;
-	bool unregister = false;
 
 	spin_lock_irqsave(&dev->lock, flags);
 	dev->state = STATE_DEV_CLOSED;
@@ -453,20 +452,24 @@ static int raw_release(struct inode *inode, struct file *fd)
 		spin_unlock_irqrestore(&dev->lock, flags);
 		goto out_put;
 	}
-	if (dev->gadget_registered)
-		unregister = true;
+	if (!dev->gadget_registered) {
+		if (kref_read(&dev->count)) {
+			/* Matches dev_new() in raw_open(). */
+			kref_put(&dev->count, dev_free);
+		}
+		spin_unlock_irqrestore(&dev->lock, flags);
+		return ret;
+	}
 	dev->gadget_registered = false;
 	spin_unlock_irqrestore(&dev->lock, flags);
 
-	if (unregister) {
-		ret = usb_gadget_unregister_driver(&dev->driver);
-		if (ret != 0)
-			dev_err(dev->dev,
-				"usb_gadget_unregister_driver() failed with %d\n",
-				ret);
-		/* Matches kref_get() in raw_ioctl_run(). */
-		kref_put(&dev->count, dev_free);
-	}
+	ret = usb_gadget_unregister_driver(&dev->driver);
+	if (ret != 0)
+		dev_err(dev->dev,
+			"usb_gadget_unregister_driver() failed with %d\n",
+			ret);
+	/* Matches kref_get() in raw_ioctl_run(). */
+	kref_put(&dev->count, dev_free);
 
 out_put:
 	/* Matches dev_new() in raw_open(). */