diff --git a/mm/zswap.c b/mm/zswap.c index 5a27af8d86ea9..4b0eb370823e8 100644 --- a/mm/zswap.c +++ b/mm/zswap.c @@ -864,12 +864,22 @@ static int zswap_cpu_comp_prepare(unsigned int cpu, struct hlist_node *node) return ret; } +DEFINE_STATIC_SRCU(acomp_srcu); + static int zswap_cpu_comp_dead(unsigned int cpu, struct hlist_node *node) { struct zswap_pool *pool = hlist_entry(node, struct zswap_pool, node); struct crypto_acomp_ctx *acomp_ctx = per_cpu_ptr(pool->acomp_ctx, cpu); if (!IS_ERR_OR_NULL(acomp_ctx)) { + /* + * Even though the acomp_ctx should not be currently in use on + * @cpu, it may still be used by compress/decompress operations + * that started on @cpu and migrated to a different CPU. Wait + * for such usages to complete, any news usages would be a bug. + */ + synchronize_srcu(&acomp_srcu); + if (!IS_ERR_OR_NULL(acomp_ctx->req)) acomp_request_free(acomp_ctx->req); if (!IS_ERR_OR_NULL(acomp_ctx->acomp)) @@ -880,16 +890,16 @@ static int zswap_cpu_comp_dead(unsigned int cpu, struct hlist_node *node) return 0; } -/* Prevent CPU hotplug from freeing up the per-CPU acomp_ctx resources */ -static struct crypto_acomp_ctx *acomp_ctx_get_cpu(struct crypto_acomp_ctx __percpu *acomp_ctx) +static struct crypto_acomp_ctx *acomp_ctx_get_cpu(struct crypto_acomp_ctx __percpu *acomp_ctx, + int *idx) { - cpus_read_lock(); + *idx = srcu_read_lock(&acomp_srcu); return raw_cpu_ptr(acomp_ctx); } -static void acomp_ctx_put_cpu(void) +static void acomp_ctx_put_cpu(int idx) { - cpus_read_unlock(); + srcu_read_unlock(&acomp_srcu, idx); } static bool zswap_compress(struct page *page, struct zswap_entry *entry, @@ -904,8 +914,9 @@ static bool zswap_compress(struct page *page, struct zswap_entry *entry, char *buf; gfp_t gfp; u8 *dst; + int idx; - acomp_ctx = acomp_ctx_get_cpu(pool->acomp_ctx); + acomp_ctx = acomp_ctx_get_cpu(pool->acomp_ctx, &idx); mutex_lock(&acomp_ctx->mutex); dst = acomp_ctx->buffer; @@ -961,7 +972,7 @@ static bool zswap_compress(struct page *page, struct zswap_entry *entry, zswap_reject_alloc_fail++; mutex_unlock(&acomp_ctx->mutex); - acomp_ctx_put_cpu(); + acomp_ctx_put_cpu(idx); return comp_ret == 0 && alloc_ret == 0; } @@ -971,8 +982,9 @@ static void zswap_decompress(struct zswap_entry *entry, struct folio *folio) struct scatterlist input, output; struct crypto_acomp_ctx *acomp_ctx; u8 *src; + int idx; - acomp_ctx = acomp_ctx_get_cpu(entry->pool->acomp_ctx); + acomp_ctx = acomp_ctx_get_cpu(entry->pool->acomp_ctx, &idx); mutex_lock(&acomp_ctx->mutex); src = zpool_map_handle(zpool, entry->handle, ZPOOL_MM_RO); @@ -1002,7 +1014,7 @@ static void zswap_decompress(struct zswap_entry *entry, struct folio *folio) if (src != acomp_ctx->buffer) zpool_unmap_handle(zpool, entry->handle); - acomp_ctx_put_cpu(); + acomp_ctx_put_cpu(idx); } /*********************************