diff --git a/drivers/net/caif/caif_serial.c b/drivers/net/caif/caif_serial.c
index c398ac42eae9..8da3522ee36a 100644
--- a/drivers/net/caif/caif_serial.c
+++ b/drivers/net/caif/caif_serial.c
@@ -294,9 +294,16 @@ static void ser_release(struct work_struct *work)
 		list_for_each_entry_safe(ser, tmp, &list, node) {
 			dev_close(ser->dev);
 			unregister_netdevice(ser->dev);
-			debugfs_deinit(ser);
 		}
 		rtnl_unlock();
+
+		while (test_bit(CAIF_SENDING, &ser->state))
+			cpu_relax();
+
+		list_for_each_entry_safe(ser, tmp, &list, node) {
+			tty_kref_put(ser->tty);
+			debugfs_deinit(ser);
+		}
 	}
 }
 
@@ -355,8 +362,6 @@ static void ldisc_close(struct tty_struct *tty)
 {
 	struct ser_device *ser = tty->disc_data;
 
-	tty_kref_put(ser->tty);
-
 	spin_lock(&ser_lock);
 	list_move(&ser->node, &ser_release_list);
 	spin_unlock(&ser_lock);