diff --git a/mm/swap.c b/mm/swap.c
index 320b959b74c6..062c8565b899 100644
--- a/mm/swap.c
+++ b/mm/swap.c
@@ -384,7 +384,7 @@ static void lru_gen_inc_refs(struct folio *folio)
 {
 	unsigned long new_flags, old_flags = READ_ONCE(folio->flags);
 
-	if (folio_test_unevictable(folio))
+	if (!folio_test_lru(folio) || folio_test_unevictable(folio))
 		return;
 
 	/* see the comment on LRU_REFS_FLAGS */
@@ -405,14 +405,17 @@ static void lru_gen_inc_refs(struct folio *folio)
 
 static bool lru_gen_clear_refs(struct folio *folio)
 {
-	int gen = folio_lru_gen(folio);
+	struct lru_gen_folio *lrugen;
 	int type = folio_is_file_lru(folio);
-	struct lru_gen_folio *lrugen = &folio_lruvec(folio)->lrugen;
+
+	if (!folio_test_lru(folio) || folio_test_unevictable(folio))
+		return true;
 
 	set_mask_bits(&folio->flags, LRU_REFS_FLAGS | BIT(PG_workingset), 0);
 
+	lrugen = &folio_lruvec(folio)->lrugen;
 	/* whether can do without shuffling under the LRU lock */
-	return gen == lru_gen_from_seq(READ_ONCE(lrugen->min_seq[type]));
+	return folio_lru_gen(folio) == lru_gen_from_seq(READ_ONCE(lrugen->min_seq[type]));
 }
 
 #else /* !CONFIG_LRU_GEN */