diff --git a/mm/filemap.c b/mm/filemap.c
index c943d1b90cc2..9ff9a885e9bf 100644
--- a/mm/filemap.c
+++ b/mm/filemap.c
@@ -2039,7 +2039,6 @@ static inline struct folio *find_get_entry(struct xa_state *xas, pgoff_t max,
* @start: The starting page cache index
* @end: The final page index (inclusive).
* @fbatch: Where the resulting entries are placed.
- * @indices: The cache indices corresponding to the entries in @entries
*
* find_get_entries() will search for and return a batch of entries in
* the mapping. The entries are placed in @fbatch. find_get_entries()
@@ -2052,21 +2051,31 @@ static inline struct folio *find_get_entry(struct xa_state *xas, pgoff_t max,
* shmem/tmpfs, are included in the returned array.
*
* Return: The number of entries which were found.
+ * Also updates @start to be positioned after the last found entry
*/
-unsigned find_get_entries(struct address_space *mapping, pgoff_t start,
- pgoff_t end, struct folio_batch *fbatch, pgoff_t *indices)
+unsigned find_get_entries(struct address_space *mapping, pgoff_t *start,
+ pgoff_t end, struct folio_batch *fbatch)
{
- XA_STATE(xas, &mapping->i_pages, start);
+ XA_STATE(xas, &mapping->i_pages, *start);
+ unsigned long nr;
struct folio *folio;
rcu_read_lock();
while ((folio = find_get_entry(&xas, end, XA_PRESENT)) != NULL) {
- indices[fbatch->nr] = xas.xa_index;
if (!folio_batch_add(fbatch, folio))
break;
}
rcu_read_unlock();
+ nr = folio_batch_count(fbatch);
+ if (nr) {
+ folio = fbatch->folios[nr - 1];
+ nr = folio_nr_pages(folio);
+
+ if (folio_test_hugetlb(folio))
+ nr = 1;
+ *start = folio->index + nr;
+ }
return folio_batch_count(fbatch);
}
@@ -2076,7 +2085,6 @@ unsigned find_get_entries(struct address_space *mapping, pgoff_t start,
* @start: The starting page cache index.
* @end: The final page index (inclusive).
* @fbatch: Where the resulting entries are placed.
- * @indices: The cache indices of the entries in @fbatch.
*
* find_lock_entries() will return a batch of entries from @mapping.
* Swap, shadow and DAX entries are included. Folios are returned
@@ -2089,17 +2097,19 @@ unsigned find_get_entries(struct address_space *mapping, pgoff_t start,
* locked or folios under writeback.
*
* Return: The number of entries which were found.
+ * Also updates @start to be positioned after the last found entry
*/
-unsigned find_lock_entries(struct address_space *mapping, pgoff_t start,
- pgoff_t end, struct folio_batch *fbatch, pgoff_t *indices)
+unsigned find_lock_entries(struct address_space *mapping, pgoff_t *start,
+ pgoff_t end, struct folio_batch *fbatch)
{
- XA_STATE(xas, &mapping->i_pages, start);
+ XA_STATE(xas, &mapping->i_pages, *start);
+ unsigned long nr;
struct folio *folio;
rcu_read_lock();
while ((folio = find_get_entry(&xas, end, XA_PRESENT))) {
if (!xa_is_value(folio)) {
- if (folio->index < start)
+ if (folio->index < *start)
goto put;
if (folio->index + folio_nr_pages(folio) - 1 > end)
goto put;
@@ -2111,7 +2121,6 @@ unsigned find_lock_entries(struct address_space *mapping, pgoff_t start,
VM_BUG_ON_FOLIO(!folio_contains(folio, xas.xa_index),
folio);
}
- indices[fbatch->nr] = xas.xa_index;
if (!folio_batch_add(fbatch, folio))
break;
continue;
@@ -2121,7 +2130,16 @@ unsigned find_lock_entries(struct address_space *mapping, pgoff_t start,
folio_put(folio);
}
rcu_read_unlock();
+ nr = folio_batch_count(fbatch);
+ if (nr) {
+ folio = fbatch->folios[nr - 1];
+ nr = folio_nr_pages(folio);
+
+ if (folio_test_hugetlb(folio))
+ nr = 1;
+ *start = folio->index + nr;
+ }
return folio_batch_count(fbatch);
}
diff --git a/mm/internal.h b/mm/internal.h
index 785409805ed7..ccce1369702c 100644
--- a/mm/internal.h
+++ b/mm/internal.h
@@ -104,10 +104,10 @@ static inline void force_page_cache_readahead(struct address_space *mapping,
force_page_cache_ra(&ractl, nr_to_read);
}
-unsigned find_lock_entries(struct address_space *mapping, pgoff_t start,
- pgoff_t end, struct folio_batch *fbatch, pgoff_t *indices);
-unsigned find_get_entries(struct address_space *mapping, pgoff_t start,
- pgoff_t end, struct folio_batch *fbatch, pgoff_t *indices);
+unsigned find_lock_entries(struct address_space *mapping, pgoff_t *start,
+ pgoff_t end, struct folio_batch *fbatch);
+unsigned find_get_entries(struct address_space *mapping, pgoff_t *start,
+ pgoff_t end, struct folio_batch *fbatch);
void filemap_free_folio(struct address_space *mapping, struct folio *folio);
int truncate_inode_folio(struct address_space *mapping, struct folio *folio);
bool truncate_inode_partial_folio(struct folio *folio, loff_t start,
diff --git a/mm/shmem.c b/mm/shmem.c
index 42e5888bf84d..3341b47593ba 100644
--- a/mm/shmem.c
+++ b/mm/shmem.c
@@ -917,7 +917,6 @@ static void shmem_undo_range(struct inode *inode, loff_t lstart, loff_t lend,
pgoff_t start = (lstart + PAGE_SIZE - 1) >> PAGE_SHIFT;
pgoff_t end = (lend + 1) >> PAGE_SHIFT;
struct folio_batch fbatch;
- pgoff_t indices[PAGEVEC_SIZE];
struct folio *folio;
bool same_folio;
long nr_swaps_freed = 0;
@@ -932,21 +931,18 @@ static void shmem_undo_range(struct inode *inode, loff_t lstart, loff_t lend,
folio_batch_init(&fbatch);
index = start;
- while (index < end && find_lock_entries(mapping, index, end - 1,
- &fbatch, indices)) {
+ while (index < end && find_lock_entries(mapping, &index, end - 1,
+ &fbatch)) {
for (i = 0; i < folio_batch_count(&fbatch); i++) {
folio = fbatch.folios[i];
- index = indices[i];
-
if (xa_is_value(folio)) {
if (unfalloc)
continue;
nr_swaps_freed += !shmem_free_swap(mapping,
- index, folio);
+ folio->index, folio);
continue;
}
- index += folio_nr_pages(folio) - 1;
if (!unfalloc || !folio_test_uptodate(folio))
truncate_inode_folio(mapping, folio);
@@ -955,7 +951,6 @@ static void shmem_undo_range(struct inode *inode, loff_t lstart, loff_t lend,
folio_batch_remove_exceptionals(&fbatch);
folio_batch_release(&fbatch);
cond_resched();
- index++;
}
same_folio = (lstart >> PAGE_SHIFT) == (lend >> PAGE_SHIFT);
@@ -987,8 +982,7 @@ static void shmem_undo_range(struct inode *inode, loff_t lstart, loff_t lend,
while (index < end) {
cond_resched();
- if (!find_get_entries(mapping, index, end - 1, &fbatch,
- indices)) {
+ if (!find_get_entries(mapping, &index, end - 1, &fbatch)) {
/* If all gone or hole-punch or unfalloc, we're done */
if (index == start || end != -1)
break;
@@ -999,13 +993,12 @@ static void shmem_undo_range(struct inode *inode, loff_t lstart, loff_t lend,
for (i = 0; i < folio_batch_count(&fbatch); i++) {
folio = fbatch.folios[i];
- index = indices[i];
if (xa_is_value(folio)) {
if (unfalloc)
continue;
- if (shmem_free_swap(mapping, index, folio)) {
+ if (shmem_free_swap(mapping, folio->index, folio)) {
/* Swap was replaced by page: retry */
- index--;
+ index = folio->index;
break;
}
nr_swaps_freed++;
@@ -1018,19 +1011,17 @@ static void shmem_undo_range(struct inode *inode, loff_t lstart, loff_t lend,
if (folio_mapping(folio) != mapping) {
/* Page was replaced by swap: retry */
folio_unlock(folio);
- index--;
+ index = folio->index;
break;
}
VM_BUG_ON_FOLIO(folio_test_writeback(folio),
folio);
truncate_inode_folio(mapping, folio);
}
- index = folio->index + folio_nr_pages(folio) - 1;
folio_unlock(folio);
}
folio_batch_remove_exceptionals(&fbatch);
folio_batch_release(&fbatch);
- index++;
}
spin_lock_irq(&info->lock);
diff --git a/mm/truncate.c b/mm/truncate.c
index 0b0708bf935f..8652abfe9c84 100644
--- a/mm/truncate.c
+++ b/mm/truncate.c
@@ -58,7 +58,7 @@ static void clear_shadow_entry(struct address_space *mapping, pgoff_t index,
* exceptional entries similar to what folio_batch_remove_exceptionals() does.
*/
static void truncate_folio_batch_exceptionals(struct address_space *mapping,
- struct folio_batch *fbatch, pgoff_t *indices)
+ struct folio_batch *fbatch)
{
int i, j;
bool dax;
@@ -82,7 +82,6 @@ static void truncate_folio_batch_exceptionals(struct address_space *mapping,
for (i = j; i < folio_batch_count(fbatch); i++) {
struct folio *folio = fbatch->folios[i];
- pgoff_t index = indices[i];
if (!xa_is_value(folio)) {
fbatch->folios[j++] = folio;
@@ -90,11 +89,11 @@ static void truncate_folio_batch_exceptionals(struct address_space *mapping,
}
if (unlikely(dax)) {
- dax_delete_mapping_entry(mapping, index);
+ dax_delete_mapping_entry(mapping, folio->index);
continue;
}
- __clear_shadow_entry(mapping, index, folio);
+ __clear_shadow_entry(mapping, folio->index, folio);
}
if (!dax) {
@@ -333,7 +332,6 @@ void truncate_inode_pages_range(struct address_space *mapping,
pgoff_t start; /* inclusive */
pgoff_t end; /* exclusive */
struct folio_batch fbatch;
- pgoff_t indices[PAGEVEC_SIZE];
pgoff_t index;
int i;
struct folio *folio;
@@ -361,10 +359,9 @@ void truncate_inode_pages_range(struct address_space *mapping,
folio_batch_init(&fbatch);
index = start;
- while (index < end && find_lock_entries(mapping, index, end - 1,
- &fbatch, indices)) {
- index = indices[folio_batch_count(&fbatch) - 1] + 1;
- truncate_folio_batch_exceptionals(mapping, &fbatch, indices);
+ while (index < end && find_lock_entries(mapping, &index, end - 1,
+ &fbatch)) {
+ truncate_folio_batch_exceptionals(mapping, &fbatch);
for (i = 0; i < folio_batch_count(&fbatch); i++)
truncate_cleanup_folio(fbatch.folios[i]);
delete_from_page_cache_batch(mapping, &fbatch);
@@ -401,8 +398,7 @@ void truncate_inode_pages_range(struct address_space *mapping,
index = start;
while (index < end) {
cond_resched();
- if (!find_get_entries(mapping, index, end - 1, &fbatch,
- indices)) {
+ if (!find_get_entries(mapping, &index, end - 1, &fbatch)) {
/* If all gone from start onwards, we're done */
if (index == start)
break;
@@ -415,21 +411,18 @@ void truncate_inode_pages_range(struct address_space *mapping,
struct folio *folio = fbatch.folios[i];
/* We rely upon deletion not changing page->index */
- index = indices[i];
-
if (xa_is_value(folio))
continue;
folio_lock(folio);
- VM_BUG_ON_FOLIO(!folio_contains(folio, index), folio);
+ VM_BUG_ON_FOLIO(!folio_contains(folio, folio->index),
+ folio);
folio_wait_writeback(folio);
truncate_inode_folio(mapping, folio);
folio_unlock(folio);
- index = folio_index(folio) + folio_nr_pages(folio) - 1;
}
- truncate_folio_batch_exceptionals(mapping, &fbatch, indices);
+ truncate_folio_batch_exceptionals(mapping, &fbatch);
folio_batch_release(&fbatch);
- index++;
}
}
EXPORT_SYMBOL(truncate_inode_pages_range);
@@ -502,7 +495,6 @@ EXPORT_SYMBOL(truncate_inode_pages_final);
unsigned long invalidate_mapping_pagevec(struct address_space *mapping,
pgoff_t start, pgoff_t end, unsigned long *nr_pagevec)
{
- pgoff_t indices[PAGEVEC_SIZE];
struct folio_batch fbatch;
pgoff_t index = start;
unsigned long ret;
@@ -510,20 +502,18 @@ unsigned long invalidate_mapping_pagevec(struct address_space *mapping,
int i;
folio_batch_init(&fbatch);
- while (find_lock_entries(mapping, index, end, &fbatch, indices)) {
+ while (find_lock_entries(mapping, &index, end, &fbatch)) {
for (i = 0; i < folio_batch_count(&fbatch); i++) {
struct folio *folio = fbatch.folios[i];
/* We rely upon deletion not changing folio->index */
- index = indices[i];
if (xa_is_value(folio)) {
count += invalidate_exceptional_entry(mapping,
- index,
- folio);
+ folio->index,
+ folio);
continue;
}
- index += folio_nr_pages(folio) - 1;
ret = mapping_evict_folio(mapping, folio);
folio_unlock(folio);
@@ -542,7 +532,6 @@ unsigned long invalidate_mapping_pagevec(struct address_space *mapping,
folio_batch_remove_exceptionals(&fbatch);
folio_batch_release(&fbatch);
cond_resched();
- index++;
}
return count;
}
@@ -628,7 +617,6 @@ static int folio_launder(struct address_space *mapping, struct folio *folio)
int invalidate_inode_pages2_range(struct address_space *mapping,
pgoff_t start, pgoff_t end)
{
- pgoff_t indices[PAGEVEC_SIZE];
struct folio_batch fbatch;
pgoff_t index;
int i;
@@ -641,16 +629,14 @@ int invalidate_inode_pages2_range(struct address_space *mapping,
folio_batch_init(&fbatch);
index = start;
- while (find_get_entries(mapping, index, end, &fbatch, indices)) {
+ while (find_get_entries(mapping, &index, end, &fbatch)) {
for (i = 0; i < folio_batch_count(&fbatch); i++) {
struct folio *folio = fbatch.folios[i];
/* We rely upon deletion not changing folio->index */
- index = indices[i];
-
if (xa_is_value(folio)) {
if (!invalidate_exceptional_entry2(mapping,
- index, folio))
+ folio->index, folio))
ret = -EBUSY;
continue;
}
@@ -660,13 +646,14 @@ int invalidate_inode_pages2_range(struct address_space *mapping,
* If folio is mapped, before taking its lock,
* zap the rest of the file in one hit.
*/
- unmap_mapping_pages(mapping, index,
- (1 + end - index), false);
+ unmap_mapping_pages(mapping, folio->index,
+ (1 + end - folio->index), false);
did_range_unmap = 1;
}
folio_lock(folio);
- VM_BUG_ON_FOLIO(!folio_contains(folio, index), folio);
+ VM_BUG_ON_FOLIO(!folio_contains(folio, folio->index),
+ folio);
if (folio->mapping != mapping) {
folio_unlock(folio);
continue;
@@ -689,7 +676,6 @@ int invalidate_inode_pages2_range(struct address_space *mapping,
folio_batch_remove_exceptionals(&fbatch);
folio_batch_release(&fbatch);
cond_resched();
- index++;
}
/*
* For DAX we invalidate page tables after invalidating page cache. We