samsung_sm8750/android_kernel_samsung_sm8750
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

BACKPORT: mm: page_alloc: close migratetype race between freeing and stealing

Browse Source 
There are three freeing paths that read the page's migratetype
optimistically before grabbing the zone lock.  When this races with block
stealing, those pages go on the wrong freelist.

The paths in question are:
- when freeing >costly orders that aren't THP
- when freeing pages to the buddy upon pcp lock contention
- when freeing pages that are isolated
- when freeing pages initially during boot
- when freeing the remainder in alloc_pages_exact()
- when "accepting" unaccepted VM host memory before first use
- when freeing pages during unpoisoning

None of these are so hot that they would need this optimization at the
cost of hampering defrag efforts.  Especially when contrasted with the
fact that the most common buddy freeing path - free_pcppages_bulk - is
checking the migratetype under the zone->lock just fine.

In addition, isolated pages need to look up the migratetype under the lock
anyway, which adds branches to the locked section, and results in a double
lookup when the pages are in fact isolated.

Move the lookups into the lock.

Link: https://lkml.kernel.org/r/20240320180429.678181-8-hannes@cmpxchg.org
Signed-off-by: Johannes Weiner <hannes@cmpxchg.org>
Reported-by: Vlastimil Babka <vbabka@suse.cz>
Reviewed-by: Vlastimil Babka <vbabka@suse.cz>
Tested-by: Baolin Wang <baolin.wang@linux.alibaba.com>
Cc: David Hildenbrand <david@redhat.com>
Cc: "Huang, Ying" <ying.huang@intel.com>
Cc: Mel Gorman <mgorman@techsingularity.net>
Cc: Zi Yan <ziy@nvidia.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>

Bug: 406708503
(cherry picked from commit 55612e80e722ac554cc5e80df05555b4f8d40c37)
[
1. Calling get_pfnblock_migratetype() under the zone->lock in __free_pages_ok to preserve the vendor hook.
2. In the patch, the function free_unref_folios has been modified, and the corresponding function
in android15-6.6 is free_unref_page_list.
]

Change-Id: I785fdb603b226845e1b1152116244ecb97502375
Signed-off-by: yipeng xiang <yipengxiang@honor.corp-partner.google.com>
This commit is contained in:
Johannes Weiner
2024-03-20 14:02:12 -04:00
committed by Suren Baghdasaryan
parent 4e814d99e0
commit 7bd0ba0831
1 changed files with 33 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files

60
mm/page_alloc.c
View File

@@ -1377,18 +1377,15 @@ static void free_pcppages_bulk(struct zone *zone, int count,
spin_unlock_irqrestore(&zone->lock, flags);
}
static void free_one_page(struct zone *zone,
struct page *page, unsigned long pfn,
unsigned int order,
int migratetype, fpi_t fpi_flags)
static void free_one_page(struct zone *zone, struct page *page,
unsigned long pfn, unsigned int order,
fpi_t fpi_flags)
{
unsigned long flags;
int migratetype;
spin_lock_irqsave(&zone->lock, flags);
if (unlikely(has_isolate_pageblock(zone) ||
is_migrate_isolate(migratetype))) {
migratetype = get_pfnblock_migratetype(page, pfn);
}
migratetype = get_pfnblock_migratetype(page, pfn);
__free_one_page(page, pfn, zone, order, migratetype, fpi_flags);
spin_unlock_irqrestore(&zone->lock, flags);
}
@@ -1416,17 +1413,15 @@ skip_prepare:
fpi_flags, &skip_free_pages_ok);
if (skip_free_pages_ok)
return;
/*
* Calling get_pfnblock_migratetype() without spin_lock_irqsave() here
* is used to avoid calling get_pfnblock_migratetype() under the lock.
* This will reduce the lock holding time.
*/
migratetype = get_pfnblock_migratetype(page, pfn);
trace_android_vh_free_unref_page_bypass(page, order, migratetype, &skip_free_unref_page);
if (skip_free_unref_page)
return;
spin_lock_irqsave(&zone->lock, flags);
migratetype = get_pfnblock_migratetype(page, pfn);
trace_android_vh_free_unref_page_bypass(page, order, migratetype, &skip_free_unref_page);
if (skip_free_unref_page) {
spin_unlock_irqrestore(&zone->lock, flags);
return;
}
if (unlikely(has_isolate_pageblock(zone) ||
is_migrate_isolate(migratetype))) {
migratetype = get_pfnblock_migratetype(page, pfn);
@@ -2642,7 +2637,7 @@ void free_unref_page(struct page *page, unsigned int order)
struct per_cpu_pages *pcp;
struct zone *zone;
unsigned long pfn = page_to_pfn(page);
int migratetype, pcpmigratetype;
int migratetype;
bool skip_free_unref_page = false;
if (!free_pages_prepare(page, order, FPI_NONE))
@@ -2656,29 +2651,29 @@ void free_unref_page(struct page *page, unsigned int order)
* get those areas back if necessary. Otherwise, we may have to free
* excessively into the page allocator
*/
migratetype = pcpmigratetype = get_pfnblock_migratetype(page, pfn);
migratetype = get_pfnblock_migratetype(page, pfn);
trace_android_vh_free_unref_page_bypass(page, order, migratetype, &skip_free_unref_page);
if (skip_free_unref_page)
return;
if (unlikely(migratetype > MIGRATE_RECLAIMABLE)) {
if (unlikely(is_migrate_isolate(migratetype))) {
free_one_page(page_zone(page), page, pfn, order, migratetype, FPI_NONE);
free_one_page(page_zone(page), page, pfn, order, FPI_NONE);
return;
}
#ifdef CONFIG_CMA
if (!cma_has_pcplist() || migratetype != MIGRATE_CMA)
#endif
pcpmigratetype = MIGRATE_MOVABLE;
migratetype = MIGRATE_MOVABLE;
}
zone = page_zone(page);
pcp_trylock_prepare(UP_flags);
pcp = pcp_spin_trylock(zone->per_cpu_pageset);
if (pcp) {
free_unref_page_commit(zone, pcp, page, pcpmigratetype, order);
free_unref_page_commit(zone, pcp, page, migratetype, order);
pcp_spin_unlock(pcp);
} else {
free_one_page(zone, page, pfn, order, migratetype, FPI_NONE);
free_one_page(zone, page, pfn, order, FPI_NONE);
}
pcp_trylock_finish(UP_flags);
}
@@ -2711,7 +2706,7 @@ void free_unref_page_list(struct list_head *list)
migratetype = get_pfnblock_migratetype(page, pfn);
if (unlikely(is_migrate_isolate(migratetype))) {
list_del(&page->lru);
free_one_page(page_zone(page), page, pfn, 0, migratetype, FPI_NONE);
free_one_page(page_zone(page), page, pfn, 0, FPI_NONE);
continue;
}
}
@@ -2738,6 +2733,16 @@ void free_unref_page_list(struct list_head *list)
pcp_trylock_finish(UP_flags);
}
/*
* Free isolated pages directly to the
* allocator, see comment in free_unref_page.
*/
if (is_migrate_isolate(migratetype)) {
free_one_page(zone, page, page_to_pfn(page),
0, FPI_NONE);
continue;
}
batch_count = 0;
/*
@@ -2749,7 +2754,7 @@ void free_unref_page_list(struct list_head *list)
if (unlikely(!pcp)) {
pcp_trylock_finish(UP_flags);
free_one_page(zone, page, pfn,
0, migratetype, FPI_NONE);
0, FPI_NONE);
locked_zone = NULL;
continue;
}
@@ -6999,13 +7004,14 @@ bool take_page_off_buddy(struct page *page)
bool put_page_back_buddy(struct page *page)
{
struct zone *zone = page_zone(page);
unsigned long pfn = page_to_pfn(page);
unsigned long flags;
int migratetype = get_pfnblock_migratetype(page, pfn);
bool ret = false;
spin_lock_irqsave(&zone->lock, flags);
if (put_page_testzero(page)) {
unsigned long pfn = page_to_pfn(page);
int migratetype = get_pfnblock_migratetype(page, pfn);
ClearPageHWPoisonTakenOff(page);
__free_one_page(page, pfn, zone, 0, migratetype, FPI_NONE);
if (TestClearPageHWPoison(page)) {