diff options
| -rw-r--r-- | mm/huge_memory.c | 43 |
1 files changed, 29 insertions, 14 deletions
diff --git a/mm/huge_memory.c b/mm/huge_memory.c index 58cabe6af33d..f43852ef9944 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -41,6 +41,7 @@ #include <linux/pgalloc.h> #include <linux/pgalloc_tag.h> #include <linux/pagewalk.h> +#include <linux/cleanup.h> #include <asm/tlb.h> #include "internal.h" @@ -78,6 +79,7 @@ static unsigned long deferred_split_scan(struct shrinker *shrink, static bool split_underused_thp = true; static atomic_t huge_zero_refcount; +static DEFINE_SPINLOCK(huge_zero_lock); struct folio *huge_zero_folio __read_mostly; unsigned long huge_zero_pfn __read_mostly = ~0UL; unsigned long huge_anon_orders_always __read_mostly; @@ -224,7 +226,8 @@ unsigned long __thp_vma_allowable_orders(struct vm_area_struct *vma, static bool get_huge_zero_folio(void) { struct folio *zero_folio; -retry: + + /* Paired with atomic_set_release(). */ if (likely(atomic_inc_not_zero(&huge_zero_refcount))) return true; @@ -237,17 +240,22 @@ retry: } /* Ensure zero folio won't have large_rmappable flag set. */ folio_clear_large_rmappable(zero_folio); - preempt_disable(); - if (cmpxchg(&huge_zero_folio, NULL, zero_folio)) { - preempt_enable(); + + /* Paired with critical section in shrink_huge_zero_folio_scan(). */ + spin_lock(&huge_zero_lock); + if (huge_zero_folio) { + /* Somebody else already installed it. */ + atomic_inc(&huge_zero_refcount); + spin_unlock(&huge_zero_lock); folio_put(zero_folio); - goto retry; + return true; } + WRITE_ONCE(huge_zero_folio, zero_folio); WRITE_ONCE(huge_zero_pfn, folio_pfn(zero_folio)); + /* Paired with atomic_inc_not_zero(). +1 for shrinker pin. */ + atomic_set_release(&huge_zero_refcount, 2); + spin_unlock(&huge_zero_lock); - /* We take additional reference here. It will be put back by shrinker */ - atomic_set(&huge_zero_refcount, 2); - preempt_enable(); count_vm_event(THP_ZERO_PAGE_ALLOC); return true; } @@ -297,15 +305,22 @@ static unsigned long shrink_huge_zero_folio_count(struct shrinker *shrink, static unsigned long shrink_huge_zero_folio_scan(struct shrinker *shrink, struct shrink_control *sc) { - if (atomic_cmpxchg(&huge_zero_refcount, 1, 0) == 1) { - struct folio *zero_folio = xchg(&huge_zero_folio, NULL); - BUG_ON(zero_folio == NULL); + struct folio *zero_folio; + + /* Paired with critical section in get_huge_zero_folio(). */ + scoped_guard(spinlock, &huge_zero_lock) { + /* Paired with atomic_inc_not_zero() in get_huge_zero_folio(). */ + if (atomic_cmpxchg(&huge_zero_refcount, 1, 0) != 1) + return 0; + + zero_folio = huge_zero_folio; + VM_WARN_ON_ONCE(!zero_folio); + WRITE_ONCE(huge_zero_folio, NULL); WRITE_ONCE(huge_zero_pfn, ~0UL); - folio_put(zero_folio); - return HPAGE_PMD_NR; } - return 0; + folio_put(zero_folio); + return HPAGE_PMD_NR; } static struct shrinker *huge_zero_folio_shrinker; |