samsung_sm8750/android_kernel_samsung_sm8750
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Revert "BACKPORT: mm: page_alloc: consolidate free page accounting"

Browse Source 
This reverts commit 3dc7946030.

Change-Id: I2dce18da522526ea7b48c446dcfaf1f86d853552
Bug: 420771453
Signed-off-by: Kalesh Singh <kaleshsingh@google.com>
This commit is contained in:
Kalesh Singh
2025-05-27 21:09:52 -07:00
parent 8a91cd1d26
commit 7d424e0f80
5 changed files with 119 additions and 121 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
include/linux/mm.h
View File

@@ -3757,22 +3757,24 @@ static inline bool page_is_guard(struct page *page)
return PageGuard(page); return PageGuard(page);
} }
bool __set_page_guard(struct zone *zone, struct page *page, unsigned int order); bool __set_page_guard(struct zone *zone, struct page *page, unsigned int order,
int migratetype);
static inline bool set_page_guard(struct zone *zone, struct page *page, static inline bool set_page_guard(struct zone *zone, struct page *page,
unsigned int order) unsigned int order, int migratetype)
{ {
if (!debug_guardpage_enabled()) if (!debug_guardpage_enabled())
return false; return false;
return __set_page_guard(zone, page, order); return __set_page_guard(zone, page, order, migratetype);
} }
void __clear_page_guard(struct zone *zone, struct page *page, unsigned int order); void __clear_page_guard(struct zone *zone, struct page *page, unsigned int order,
int migratetype);
static inline void clear_page_guard(struct zone *zone, struct page *page, static inline void clear_page_guard(struct zone *zone, struct page *page,
unsigned int order) unsigned int order, int migratetype)
{ {
if (!debug_guardpage_enabled()) if (!debug_guardpage_enabled())
return; return;
__clear_page_guard(zone, page, order); __clear_page_guard(zone, page, order, migratetype);
} }
#else /* CONFIG_DEBUG_PAGEALLOC */ #else /* CONFIG_DEBUG_PAGEALLOC */
@@ -3782,9 +3784,9 @@ static inline unsigned int debug_guardpage_minorder(void) { return 0; }
static inline bool debug_guardpage_enabled(void) { return false; } static inline bool debug_guardpage_enabled(void) { return false; }
static inline bool page_is_guard(struct page *page) { return false; } static inline bool page_is_guard(struct page *page) { return false; }
static inline bool set_page_guard(struct zone *zone, struct page *page, static inline bool set_page_guard(struct zone *zone, struct page *page,
unsigned int order) { return false; } unsigned int order, int migratetype) { return false; }
static inline void clear_page_guard(struct zone *zone, struct page *page, static inline void clear_page_guard(struct zone *zone, struct page *page,
unsigned int order) {} unsigned int order, int migratetype) {}
#endif /* CONFIG_DEBUG_PAGEALLOC */ #endif /* CONFIG_DEBUG_PAGEALLOC */
#ifdef __HAVE_ARCH_GATE_AREA #ifdef __HAVE_ARCH_GATE_AREA

8
include/linux/vmstat.h
View File

@@ -487,6 +487,14 @@ static inline void node_stat_sub_folio(struct folio *folio,
mod_node_page_state(folio_pgdat(folio), item, -folio_nr_pages(folio)); mod_node_page_state(folio_pgdat(folio), item, -folio_nr_pages(folio));
} }
static inline void __mod_zone_freepage_state(struct zone *zone, int nr_pages,
int migratetype)
{
__mod_zone_page_state(zone, NR_FREE_PAGES, nr_pages);
if (is_migrate_cma(migratetype))
__mod_zone_page_state(zone, NR_FREE_CMA_PAGES, nr_pages);
}
extern const char * const vmstat_text[]; extern const char * const vmstat_text[];
static inline const char *zone_stat_name(enum zone_stat_item item) static inline const char *zone_stat_name(enum zone_stat_item item)

12
mm/debug_page_alloc.c
View File

@@ -32,7 +32,8 @@ static int __init debug_guardpage_minorder_setup(char *buf)
} }
early_param("debug_guardpage_minorder", debug_guardpage_minorder_setup); early_param("debug_guardpage_minorder", debug_guardpage_minorder_setup);
bool __set_page_guard(struct zone *zone, struct page *page, unsigned int order) bool __set_page_guard(struct zone *zone, struct page *page, unsigned int order,
int migratetype)
{ {
if (order >= debug_guardpage_minorder()) if (order >= debug_guardpage_minorder())
return false; return false;
@@ -40,12 +41,19 @@ bool __set_page_guard(struct zone *zone, struct page *page, unsigned int order)
__SetPageGuard(page); __SetPageGuard(page);
INIT_LIST_HEAD(&page->buddy_list); INIT_LIST_HEAD(&page->buddy_list);
set_page_private(page, order); set_page_private(page, order);
/* Guard pages are not available for any usage */
if (!is_migrate_isolate(migratetype))
__mod_zone_freepage_state(zone, -(1 << order), migratetype);
return true; return true;
} }
void __clear_page_guard(struct zone *zone, struct page *page, unsigned int order) void __clear_page_guard(struct zone *zone, struct page *page, unsigned int order,
int migratetype)
{ {
__ClearPageGuard(page); __ClearPageGuard(page);
set_page_private(page, 0); set_page_private(page, 0);
if (!is_migrate_isolate(migratetype))
__mod_zone_freepage_state(zone, (1 << order), migratetype);
} }

5
mm/internal.h
View File

@@ -1213,6 +1213,11 @@ static inline bool is_migrate_highatomic(enum migratetype migratetype)
return migratetype == MIGRATE_HIGHATOMIC; return migratetype == MIGRATE_HIGHATOMIC;
} }
static inline bool is_migrate_highatomic_page(struct page *page)
{
return get_pageblock_migratetype(page) == MIGRATE_HIGHATOMIC;
}
void setup_zone_pageset(struct zone *zone); void setup_zone_pageset(struct zone *zone);
struct migration_target_control { struct migration_target_control {

197
mm/page_alloc.c
View File

@@ -756,33 +756,23 @@ compaction_capture(struct capture_control *capc, struct page *page,
} }
#endif /* CONFIG_COMPACTION */ #endif /* CONFIG_COMPACTION */
static inline void account_freepages(struct zone *zone, int nr_pages,
int migratetype)
{
if (is_migrate_isolate(migratetype))
return;
__mod_zone_page_state(zone, NR_FREE_PAGES, nr_pages);
if (is_migrate_cma(migratetype))
__mod_zone_page_state(zone, NR_FREE_CMA_PAGES, nr_pages);
}
/* Used for pages not on another list */ /* Used for pages not on another list */
static inline void __add_to_free_list(struct page *page, struct zone *zone, static inline void add_to_free_list(struct page *page, struct zone *zone,
unsigned int order, int migratetype, unsigned int order, int migratetype)
bool tail)
{ {
struct free_area *area = &zone->free_area[order]; struct free_area *area = &zone->free_area[order];
VM_WARN_ONCE(get_pageblock_migratetype(page) != migratetype, list_add(&page->buddy_list, &area->free_list[migratetype]);
"page type is %lu, passed migratetype is %d (nr=%d)\n", area->nr_free++;
get_pageblock_migratetype(page), migratetype, 1 << order); }
if (tail) /* Used for pages not on another list */
list_add_tail(&page->buddy_list, &area->free_list[migratetype]); static inline void add_to_free_list_tail(struct page *page, struct zone *zone,
else unsigned int order, int migratetype)
list_add(&page->buddy_list, &area->free_list[migratetype]); {
struct free_area *area = &zone->free_area[order];
list_add_tail(&page->buddy_list, &area->free_list[migratetype]);
area->nr_free++; area->nr_free++;
} }
@@ -792,28 +782,16 @@ static inline void __add_to_free_list(struct page *page, struct zone *zone,
* allocation again (e.g., optimization for memory onlining). * allocation again (e.g., optimization for memory onlining).
*/ */
static inline void move_to_free_list(struct page *page, struct zone *zone, static inline void move_to_free_list(struct page *page, struct zone *zone,
unsigned int order, int old_mt, int new_mt) unsigned int order, int migratetype)
{ {
struct free_area *area = &zone->free_area[order]; struct free_area *area = &zone->free_area[order];
/* Free page moving can fail, so it happens before the type update */ list_move_tail(&page->buddy_list, &area->free_list[migratetype]);
VM_WARN_ONCE(get_pageblock_migratetype(page) != old_mt,
"page type is %lu, passed migratetype is %d (nr=%d)\n",
get_pageblock_migratetype(page), old_mt, 1 << order);
list_move_tail(&page->buddy_list, &area->free_list[new_mt]);
account_freepages(zone, -(1 << order), old_mt);
account_freepages(zone, 1 << order, new_mt);
} }
static inline void __del_page_from_free_list(struct page *page, struct zone *zone, static inline void del_page_from_free_list(struct page *page, struct zone *zone,
unsigned int order, int migratetype) unsigned int order)
{ {
VM_WARN_ONCE(get_pageblock_migratetype(page) != migratetype,
"page type is %lu, passed migratetype is %d (nr=%d)\n",
get_pageblock_migratetype(page), migratetype, 1 << order);
/* clear reported state and update reported page count */ /* clear reported state and update reported page count */
if (page_reported(page)) if (page_reported(page))
__ClearPageReported(page); __ClearPageReported(page);
@@ -824,13 +802,6 @@ static inline void __del_page_from_free_list(struct page *page, struct zone *zon
zone->free_area[order].nr_free--; zone->free_area[order].nr_free--;
} }
static inline void del_page_from_free_list(struct page *page, struct zone *zone,
unsigned int order, int migratetype)
{
__del_page_from_free_list(page, zone, order, migratetype);
account_freepages(zone, -(1 << order), migratetype);
}
static inline struct page *get_page_from_free_area(struct free_area *area, static inline struct page *get_page_from_free_area(struct free_area *area,
int migratetype) int migratetype)
{ {
@@ -915,17 +886,16 @@ static inline void __free_one_page(struct page *page,
VM_BUG_ON_PAGE(page->flags & check_flags, page); VM_BUG_ON_PAGE(page->flags & check_flags, page);
VM_BUG_ON(migratetype == -1); VM_BUG_ON(migratetype == -1);
if (likely(!is_migrate_isolate(migratetype)))
__mod_zone_freepage_state(zone, 1 << order, migratetype);
VM_BUG_ON_PAGE(pfn & ((1 << order) - 1), page); VM_BUG_ON_PAGE(pfn & ((1 << order) - 1), page);
VM_BUG_ON_PAGE(bad_range(zone, page), page); VM_BUG_ON_PAGE(bad_range(zone, page), page);
account_freepages(zone, 1 << order, migratetype);
while (order < max_order) { while (order < max_order) {
int buddy_mt = migratetype;
if (compaction_capture(capc, page, order, migratetype)) { if (compaction_capture(capc, page, order, migratetype)) {
account_freepages(zone, -(1 << order), migratetype); __mod_zone_freepage_state(zone, -(1 << order),
migratetype);
return; return;
} }
@@ -940,12 +910,19 @@ static inline void __free_one_page(struct page *page,
* pageblock isolation could cause incorrect freepage or CMA * pageblock isolation could cause incorrect freepage or CMA
* accounting or HIGHATOMIC accounting. * accounting or HIGHATOMIC accounting.
*/ */
buddy_mt = get_pfnblock_migratetype(buddy, buddy_pfn); int buddy_mt = get_pfnblock_migratetype(buddy, buddy_pfn);
if (migratetype != buddy_mt && if (migratetype != buddy_mt) {
(!migratetype_is_mergeable(migratetype) || if (!migratetype_is_mergeable(migratetype) ||
!migratetype_is_mergeable(buddy_mt))) !migratetype_is_mergeable(buddy_mt))
goto done_merging; goto done_merging;
/*
* Match buddy type. This ensures that
* an expand() down the line puts the
* sub-blocks on the right freelists.
*/
set_pageblock_migratetype(buddy, migratetype);
}
} }
/* /*
@@ -953,19 +930,9 @@ static inline void __free_one_page(struct page *page,
* merge with it and move up one order. * merge with it and move up one order.
*/ */
if (page_is_guard(buddy)) if (page_is_guard(buddy))
clear_page_guard(zone, buddy, order); clear_page_guard(zone, buddy, order, migratetype);
else else
__del_page_from_free_list(buddy, zone, order, buddy_mt); del_page_from_free_list(buddy, zone, order);
if (unlikely(buddy_mt != migratetype)) {
/*
* Match buddy type. This ensures that an
* expand() down the line puts the sub-blocks
* on the right freelists.
*/
set_pageblock_migratetype(buddy, migratetype);
}
combined_pfn = buddy_pfn & pfn; combined_pfn = buddy_pfn & pfn;
page = page + (combined_pfn - pfn); page = page + (combined_pfn - pfn);
pfn = combined_pfn; pfn = combined_pfn;
@@ -984,7 +951,10 @@ done_merging:
else else
to_tail = buddy_merge_likely(pfn, buddy_pfn, page, order); to_tail = buddy_merge_likely(pfn, buddy_pfn, page, order);
__add_to_free_list(page, zone, order, migratetype, to_tail); if (to_tail)
add_to_free_list_tail(page, zone, order, migratetype);
else
add_to_free_list(page, zone, order, migratetype);
/* Notify page reporting subsystem of freed page */ /* Notify page reporting subsystem of freed page */
if (!(fpi_flags & FPI_SKIP_REPORT_NOTIFY)) if (!(fpi_flags & FPI_SKIP_REPORT_NOTIFY))
@@ -1531,10 +1501,10 @@ static inline void expand(struct zone *zone, struct page *page,
* Corresponding page table entries will not be touched, * Corresponding page table entries will not be touched,
* pages will stay not present in virtual address space * pages will stay not present in virtual address space
*/ */
if (set_page_guard(zone, &page[size], high)) if (set_page_guard(zone, &page[size], high, migratetype))
continue; continue;
__add_to_free_list(&page[size], zone, high, migratetype, false); add_to_free_list(&page[size], zone, high, migratetype);
set_buddy_order(&page[size], high); set_buddy_order(&page[size], high);
} }
} }
@@ -1721,7 +1691,7 @@ struct page *__rmqueue_smallest(struct zone *zone, unsigned int order,
page = get_page_from_free_area(area, migratetype); page = get_page_from_free_area(area, migratetype);
if (!page) if (!page)
continue; continue;
del_page_from_free_list(page, zone, current_order, migratetype); del_page_from_free_list(page, zone, current_order);
expand(zone, page, order, current_order, migratetype); expand(zone, page, order, current_order, migratetype);
trace_mm_page_alloc_zone_locked(page, order, migratetype, trace_mm_page_alloc_zone_locked(page, order, migratetype,
pcp_allowed_order(order) && pcp_allowed_order(order) &&
@@ -1761,7 +1731,7 @@ static inline struct page *__rmqueue_cma_fallback(struct zone *zone,
* type's freelist. * type's freelist.
*/ */
static int move_freepages(struct zone *zone, unsigned long start_pfn, static int move_freepages(struct zone *zone, unsigned long start_pfn,
unsigned long end_pfn, int old_mt, int new_mt) unsigned long end_pfn, int migratetype)
{ {
struct page *page; struct page *page;
unsigned long pfn; unsigned long pfn;
@@ -1783,14 +1753,12 @@ static int move_freepages(struct zone *zone, unsigned long start_pfn,
VM_BUG_ON_PAGE(page_zone(page) != zone, page); VM_BUG_ON_PAGE(page_zone(page) != zone, page);
order = buddy_order(page); order = buddy_order(page);
move_to_free_list(page, zone, order, migratetype);
move_to_free_list(page, zone, order, old_mt, new_mt);
pfn += 1 << order; pfn += 1 << order;
pages_moved += 1 << order; pages_moved += 1 << order;
} }
set_pageblock_migratetype(pfn_to_page(start_pfn), new_mt); set_pageblock_migratetype(pfn_to_page(start_pfn), migratetype);
return pages_moved; return pages_moved;
} }
@@ -1848,7 +1816,7 @@ static bool prep_move_freepages_block(struct zone *zone, struct page *page,
} }
static int move_freepages_block(struct zone *zone, struct page *page, static int move_freepages_block(struct zone *zone, struct page *page,
int old_mt, int new_mt) int migratetype)
{ {
unsigned long start_pfn, end_pfn; unsigned long start_pfn, end_pfn;
@@ -1856,7 +1824,7 @@ static int move_freepages_block(struct zone *zone, struct page *page,
NULL, NULL)) NULL, NULL))
return -1; return -1;
return move_freepages(zone, start_pfn, end_pfn, old_mt, new_mt); return move_freepages(zone, start_pfn, end_pfn, migratetype);
} }
#ifdef CONFIG_MEMORY_ISOLATION #ifdef CONFIG_MEMORY_ISOLATION
@@ -1928,6 +1896,7 @@ bool move_freepages_block_isolate(struct zone *zone, struct page *page,
int migratetype) int migratetype)
{ {
unsigned long start_pfn, end_pfn, pfn; unsigned long start_pfn, end_pfn, pfn;
int nr_moved, mt;
if (!prep_move_freepages_block(zone, page, &start_pfn, &end_pfn, if (!prep_move_freepages_block(zone, page, &start_pfn, &end_pfn,
NULL, NULL)) NULL, NULL))
@@ -1942,9 +1911,11 @@ bool move_freepages_block_isolate(struct zone *zone, struct page *page,
if (pfn != start_pfn) { if (pfn != start_pfn) {
struct page *buddy = pfn_to_page(pfn); struct page *buddy = pfn_to_page(pfn);
int order = buddy_order(buddy); int order = buddy_order(buddy);
int mt = get_pfnblock_migratetype(buddy, pfn);
del_page_from_free_list(buddy, zone, order, if (!is_migrate_isolate(mt))
get_pfnblock_migratetype(buddy, pfn)); __mod_zone_freepage_state(zone, -(1UL << order), mt);
del_page_from_free_list(buddy, zone, order);
set_pageblock_migratetype(page, migratetype); set_pageblock_migratetype(page, migratetype);
split_large_buddy(zone, buddy, pfn, order); split_large_buddy(zone, buddy, pfn, order);
return true; return true;
@@ -1952,17 +1923,23 @@ bool move_freepages_block_isolate(struct zone *zone, struct page *page,
/* We're the starting block of a larger buddy */ /* We're the starting block of a larger buddy */
if (PageBuddy(page) && buddy_order(page) > pageblock_order) { if (PageBuddy(page) && buddy_order(page) > pageblock_order) {
int mt = get_pfnblock_migratetype(page, pfn);
int order = buddy_order(page); int order = buddy_order(page);
del_page_from_free_list(page, zone, order, if (!is_migrate_isolate(mt))
get_pfnblock_migratetype(page, pfn)); __mod_zone_freepage_state(zone, -(1UL << order), mt);
del_page_from_free_list(page, zone, order);
set_pageblock_migratetype(page, migratetype); set_pageblock_migratetype(page, migratetype);
split_large_buddy(zone, page, pfn, order); split_large_buddy(zone, page, pfn, order);
return true; return true;
} }
move: move:
move_freepages(zone, start_pfn, end_pfn, mt = get_pfnblock_migratetype(page, start_pfn);
get_pfnblock_migratetype(page, start_pfn), migratetype); nr_moved = move_freepages(zone, start_pfn, end_pfn, migratetype);
if (!is_migrate_isolate(mt))
__mod_zone_freepage_state(zone, -nr_moved, mt);
else if (!is_migrate_isolate(migratetype))
__mod_zone_freepage_state(zone, nr_moved, migratetype);
return true; return true;
} }
#endif /* CONFIG_MEMORY_ISOLATION */ #endif /* CONFIG_MEMORY_ISOLATION */
@@ -2076,7 +2053,7 @@ steal_suitable_fallback(struct zone *zone, struct page *page,
/* Take ownership for orders >= pageblock_order */ /* Take ownership for orders >= pageblock_order */
if (current_order >= pageblock_order) { if (current_order >= pageblock_order) {
del_page_from_free_list(page, zone, current_order, block_type); del_page_from_free_list(page, zone, current_order);
change_pageblock_range(page, current_order, start_type); change_pageblock_range(page, current_order, start_type);
expand(zone, page, order, current_order, start_type); expand(zone, page, order, current_order, start_type);
return page; return page;
@@ -2126,12 +2103,12 @@ steal_suitable_fallback(struct zone *zone, struct page *page,
*/ */
if (free_pages + alike_pages >= (1 << (pageblock_order-1)) || if (free_pages + alike_pages >= (1 << (pageblock_order-1)) ||
page_group_by_mobility_disabled) { page_group_by_mobility_disabled) {
move_freepages(zone, start_pfn, end_pfn, block_type, start_type); move_freepages(zone, start_pfn, end_pfn, start_type);
return __rmqueue_smallest(zone, order, start_type); return __rmqueue_smallest(zone, order, start_type);
} }
single_page: single_page:
del_page_from_free_list(page, zone, current_order, block_type); del_page_from_free_list(page, zone, current_order);
expand(zone, page, order, current_order, block_type); expand(zone, page, order, current_order, block_type);
return page; return page;
} }
@@ -2205,7 +2182,7 @@ static void reserve_highatomic_pageblock(struct page *page, struct zone *zone)
mt = get_pageblock_migratetype(page); mt = get_pageblock_migratetype(page);
/* Only reserve normal pageblocks (i.e., they can merge with others) */ /* Only reserve normal pageblocks (i.e., they can merge with others) */
if (migratetype_is_mergeable(mt)) if (migratetype_is_mergeable(mt))
if (move_freepages_block(zone, page, mt, if (move_freepages_block(zone, page,
MIGRATE_HIGHATOMIC) != -1) MIGRATE_HIGHATOMIC) != -1)
zone->nr_reserved_highatomic += pageblock_nr_pages; zone->nr_reserved_highatomic += pageblock_nr_pages;
@@ -2252,13 +2229,11 @@ static bool unreserve_highatomic_pageblock(const struct alloc_context *ac,
spin_lock_irqsave(&zone->lock, flags); spin_lock_irqsave(&zone->lock, flags);
for (order = 0; order < NR_PAGE_ORDERS; order++) { for (order = 0; order < NR_PAGE_ORDERS; order++) {
struct free_area *area = &(zone->free_area[order]); struct free_area *area = &(zone->free_area[order]);
int mt;
page = get_page_from_free_area(area, MIGRATE_HIGHATOMIC); page = get_page_from_free_area(area, MIGRATE_HIGHATOMIC);
if (!page) if (!page)
continue; continue;
mt = get_pageblock_migratetype(page);
/* /*
* In page freeing path, migratetype change is racy so * In page freeing path, migratetype change is racy so
* we can counter several free pages in a pageblock * we can counter several free pages in a pageblock
@@ -2266,7 +2241,7 @@ static bool unreserve_highatomic_pageblock(const struct alloc_context *ac,
* from highatomic to ac->migratetype. So we should * from highatomic to ac->migratetype. So we should
* adjust the count once. * adjust the count once.
*/ */
if (is_migrate_highatomic(mt)) { if (is_migrate_highatomic_page(page)) {
/* /*
* It should never happen but changes to * It should never happen but changes to
* locking could inadvertently allow a per-cpu * locking could inadvertently allow a per-cpu
@@ -2288,8 +2263,7 @@ static bool unreserve_highatomic_pageblock(const struct alloc_context *ac,
* of pageblocks that cannot be completely freed * of pageblocks that cannot be completely freed
* may increase. * may increase.
*/ */
ret = move_freepages_block(zone, page, mt, ret = move_freepages_block(zone, page, ac->migratetype);
ac->migratetype);
/* /*
* Reserving this block already succeeded, so this should * Reserving this block already succeeded, so this should
* not fail on zone boundaries. * not fail on zone boundaries.
@@ -2474,7 +2448,12 @@ static int rmqueue_bulk(struct zone *zone, unsigned int order,
* pages are ordered properly. * pages are ordered properly.
*/ */
list_add_tail(&page->pcp_list, list); list_add_tail(&page->pcp_list, list);
if (is_migrate_cma(get_pageblock_migratetype(page)))
__mod_zone_page_state(zone, NR_FREE_CMA_PAGES,
-(1 << order));
} }
__mod_zone_page_state(zone, NR_FREE_PAGES, -(i << order));
spin_unlock_irqrestore(&zone->lock, flags); spin_unlock_irqrestore(&zone->lock, flags);
return i; return i;
@@ -2902,9 +2881,11 @@ int __isolate_free_page(struct page *page, unsigned int order)
watermark = zone->_watermark[WMARK_MIN] + (1UL << order); watermark = zone->_watermark[WMARK_MIN] + (1UL << order);
if (!zone_watermark_ok(zone, 0, watermark, 0, ALLOC_CMA)) if (!zone_watermark_ok(zone, 0, watermark, 0, ALLOC_CMA))
return 0; return 0;
__mod_zone_freepage_state(zone, -(1UL << order), mt);
} }
del_page_from_free_list(page, zone, order, mt); del_page_from_free_list(page, zone, order);
/* /*
* Set the pageblock if the isolated page is at least half of a * Set the pageblock if the isolated page is at least half of a
@@ -2919,7 +2900,7 @@ int __isolate_free_page(struct page *page, unsigned int order)
* with others) * with others)
*/ */
if (migratetype_is_mergeable(mt)) if (migratetype_is_mergeable(mt))
move_freepages_block(zone, page, mt, move_freepages_block(zone, page,
MIGRATE_MOVABLE); MIGRATE_MOVABLE);
} }
} }
@@ -3009,6 +2990,8 @@ struct page *rmqueue_buddy(struct zone *preferred_zone, struct zone *zone,
return NULL; return NULL;
} }
} }
__mod_zone_freepage_state(zone, -(1 << order),
get_pageblock_migratetype(page));
spin_unlock_irqrestore(&zone->lock, flags); spin_unlock_irqrestore(&zone->lock, flags);
} while (check_new_pages(page, order)); } while (check_new_pages(page, order));
@@ -6953,9 +6936,8 @@ void __offline_isolated_pages(unsigned long start_pfn, unsigned long end_pfn)
BUG_ON(page_count(page)); BUG_ON(page_count(page));
BUG_ON(!PageBuddy(page)); BUG_ON(!PageBuddy(page));
VM_WARN_ON(get_pageblock_migratetype(page) != MIGRATE_ISOLATE);
order = buddy_order(page); order = buddy_order(page);
del_page_from_free_list(page, zone, order, MIGRATE_ISOLATE); del_page_from_free_list(page, zone, order);
pfn += (1 << order); pfn += (1 << order);
} }
spin_unlock_irqrestore(&zone->lock, flags); spin_unlock_irqrestore(&zone->lock, flags);
@@ -6983,14 +6965,6 @@ bool is_free_buddy_page(struct page *page)
EXPORT_SYMBOL(is_free_buddy_page); EXPORT_SYMBOL(is_free_buddy_page);
#ifdef CONFIG_MEMORY_FAILURE #ifdef CONFIG_MEMORY_FAILURE
static inline void add_to_free_list(struct page *page, struct zone *zone,
unsigned int order, int migratetype,
bool tail)
{
__add_to_free_list(page, zone, order, migratetype, tail);
account_freepages(zone, 1 << order, migratetype);
}
/* /*
* Break down a higher-order page in sub-pages, and keep our target out of * Break down a higher-order page in sub-pages, and keep our target out of
* buddy allocator. * buddy allocator.
@@ -7015,11 +6989,11 @@ static void break_down_buddy_pages(struct zone *zone, struct page *page,
} }
page = next_page; page = next_page;
if (set_page_guard(zone, current_buddy, high)) if (set_page_guard(zone, current_buddy, high, migratetype))
continue; continue;
if (current_buddy != target) { if (current_buddy != target) {
add_to_free_list(current_buddy, zone, high, migratetype, false); add_to_free_list(current_buddy, zone, high, migratetype);
set_buddy_order(current_buddy, high); set_buddy_order(current_buddy, high);
} }
} }
@@ -7046,11 +7020,12 @@ bool take_page_off_buddy(struct page *page)
int migratetype = get_pfnblock_migratetype(page_head, int migratetype = get_pfnblock_migratetype(page_head,
pfn_head); pfn_head);
del_page_from_free_list(page_head, zone, page_order, del_page_from_free_list(page_head, zone, page_order);
migratetype);
break_down_buddy_pages(zone, page_head, page, 0, break_down_buddy_pages(zone, page_head, page, 0,
page_order, migratetype); page_order, migratetype);
SetPageHWPoisonTakenOff(page); SetPageHWPoisonTakenOff(page);
if (!is_migrate_isolate(migratetype))
__mod_zone_freepage_state(zone, -1, migratetype);
ret = true; ret = true;
break; break;
} }
@@ -7155,7 +7130,7 @@ static bool try_to_accept_memory_one(struct zone *zone)
list_del(&page->lru); list_del(&page->lru);
last = list_empty(&zone->unaccepted_pages); last = list_empty(&zone->unaccepted_pages);
account_freepages(zone, -MAX_ORDER_NR_PAGES, MIGRATE_MOVABLE); __mod_zone_freepage_state(zone, -MAX_ORDER_NR_PAGES, MIGRATE_MOVABLE);
__mod_zone_page_state(zone, NR_UNACCEPTED, -MAX_ORDER_NR_PAGES); __mod_zone_page_state(zone, NR_UNACCEPTED, -MAX_ORDER_NR_PAGES);
spin_unlock_irqrestore(&zone->lock, flags); spin_unlock_irqrestore(&zone->lock, flags);
@@ -7223,7 +7198,7 @@ static bool __free_unaccepted(struct page *page)
spin_lock_irqsave(&zone->lock, flags); spin_lock_irqsave(&zone->lock, flags);
first = list_empty(&zone->unaccepted_pages); first = list_empty(&zone->unaccepted_pages);
list_add_tail(&page->lru, &zone->unaccepted_pages); list_add_tail(&page->lru, &zone->unaccepted_pages);
account_freepages(zone, MAX_ORDER_NR_PAGES, MIGRATE_MOVABLE); __mod_zone_freepage_state(zone, MAX_ORDER_NR_PAGES, MIGRATE_MOVABLE);
__mod_zone_page_state(zone, NR_UNACCEPTED, MAX_ORDER_NR_PAGES); __mod_zone_page_state(zone, NR_UNACCEPTED, MAX_ORDER_NR_PAGES);
spin_unlock_irqrestore(&zone->lock, flags); spin_unlock_irqrestore(&zone->lock, flags);