diff --git a/drivers/vhost/vhost.c b/drivers/vhost/vhost.c
index dc9301d31f12..ee32b1fae5cf 100644
--- a/drivers/vhost/vhost.c
+++ b/drivers/vhost/vhost.c
@@ -308,6 +308,14 @@ static void vhost_map_unprefetch(struct vhost_map *map)
 	map->addr = NULL;
 }
 
+static void vhost_free_map(struct rcu_head *rcu)
+{
+	struct vhost_map *map = container_of(rcu, struct vhost_map, rcu);
+
+	kfree(map->pages);
+	kfree(map);
+}
+
 static void vhost_uninit_vq_maps(struct vhost_virtqueue *vq)
 {
 	struct vhost_map *map[VHOST_NUM_ADDRS];
@@ -322,11 +330,9 @@ static void vhost_uninit_vq_maps(struct vhost_virtqueue *vq)
 	}
 	spin_unlock(&vq->mmu_lock);
 
-	synchronize_rcu();
-
 	for (i = 0; i < VHOST_NUM_ADDRS; i++)
 		if (map[i])
-			vhost_map_unprefetch(map[i]);
+			call_rcu(&map->rcu, vhost_free_map);
 
 }
 
diff --git a/drivers/vhost/vhost.h b/drivers/vhost/vhost.h
index 819296332913..6b1704b9f44a 100644
--- a/drivers/vhost/vhost.h
+++ b/drivers/vhost/vhost.h
@@ -87,6 +87,7 @@ struct vhost_map {
 	int npages;
 	void *addr;
 	struct page **pages;
+	struct rcu_head rcu;
 };
 
 struct vhost_uaddr {