diff --git a/mm/huge_memory.c b/mm/huge_memory.c
index 054a144cb569..3d8237d9ccd1 100644
--- a/mm/huge_memory.c
+++ b/mm/huge_memory.c
@@ -2450,35 +2450,41 @@ int zap_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma,
 			folio_remove_rmap_pmd(folio, page, vma);
 			WARN_ON_ONCE(folio_mapcount(folio) < 0);
 			VM_BUG_ON_PAGE(!PageHead(page), page);
-		} else if (pmd_is_valid_softleaf(orig_pmd)) {
-			const softleaf_t entry = softleaf_from_pmd(orig_pmd);
-
-			folio = softleaf_to_folio(entry);
+		} else {
 			flush_needed = 0;
 
-			if (!thp_migration_supported())
-				WARN_ONCE(1, "Non present huge pmd without pmd migration enabled!");
+			if (pmd_is_valid_softleaf(orig_pmd)) {
+				const softleaf_t entry = softleaf_from_pmd(orig_pmd);
+
+				folio = softleaf_to_folio(entry);
+
+				if (!thp_migration_supported())
+					WARN_ONCE(1, "Non present huge pmd without pmd migration enabled!");
+			}
 		}
 
-		if (folio_test_anon(folio)) {
+		if (folio && folio_test_anon(folio)) {
 			zap_deposited_table(tlb->mm, pmd);
 			add_mm_counter(tlb->mm, MM_ANONPAGES, -HPAGE_PMD_NR);
 		} else {
-			if (arch_needs_pgtable_deposit())
+			if (vma_is_anonymous(vma) || arch_needs_pgtable_deposit())
 				zap_deposited_table(tlb->mm, pmd);
-			add_mm_counter(tlb->mm, mm_counter_file(folio),
-				       -HPAGE_PMD_NR);
 
-			/*
-			 * Use flush_needed to indicate whether the PMD entry
-			 * is present, instead of checking pmd_present() again.
-			 */
-			if (flush_needed && pmd_young(orig_pmd) &&
-			    likely(vma_has_recency(vma)))
-				folio_mark_accessed(folio);
+			if (folio) {
+				add_mm_counter(tlb->mm, mm_counter_file(folio),
+					       -HPAGE_PMD_NR);
+
+				/*
+				 * Use flush_needed to indicate whether the PMD entry
+				 * is present, instead of checking pmd_present() again.
+				 */
+				if (flush_needed && pmd_young(orig_pmd) &&
+				    likely(vma_has_recency(vma)))
+					folio_mark_accessed(folio);
+			}
 		}
 
-		if (folio_is_device_private(folio)) {
+		if (folio && folio_is_device_private(folio)) {
 			folio_remove_rmap_pmd(folio, &folio->page, vma);
 			WARN_ON_ONCE(folio_mapcount(folio) < 0);
 			folio_put(folio);