brintos / linux-shallow public Read only

1// SPDX-License-Identifier: GPL-2.02/*3 * linux/mm/compaction.c4 *5 * Memory compaction for the reduction of external fragmentation. Note that6 * this heavily depends upon page migration to do all the real heavy7 * lifting8 *9 * Copyright IBM Corp. 2007-2010 Mel Gorman <mel@csn.ul.ie>10 */11#include <linux/cpu.h>12#include <linux/swap.h>13#include <linux/migrate.h>14#include <linux/compaction.h>15#include <linux/mm_inline.h>16#include <linux/sched/signal.h>17#include <linux/backing-dev.h>18#include <linux/sysctl.h>19#include <linux/sysfs.h>20#include <linux/page-isolation.h>21#include <linux/kasan.h>22#include <linux/kthread.h>23#include <linux/freezer.h>24#include <linux/page_owner.h>25#include <linux/psi.h>26#include <linux/cpuset.h>27#include "internal.h"28 29#ifdef CONFIG_COMPACTION30/*31 * Fragmentation score check interval for proactive compaction purposes.32 */33#define HPAGE_FRAG_CHECK_INTERVAL_MSEC	(500)34 35static inline void count_compact_event(enum vm_event_item item)36{37	count_vm_event(item);38}39 40static inline void count_compact_events(enum vm_event_item item, long delta)41{42	count_vm_events(item, delta);43}44 45/*46 * order == -1 is expected when compacting proactively via47 * 1. /proc/sys/vm/compact_memory48 * 2. /sys/devices/system/node/nodex/compact49 * 3. /proc/sys/vm/compaction_proactiveness50 */51static inline bool is_via_compact_memory(int order)52{53	return order == -1;54}55 56#else57#define count_compact_event(item) do { } while (0)58#define count_compact_events(item, delta) do { } while (0)59static inline bool is_via_compact_memory(int order) { return false; }60#endif61 62#if defined CONFIG_COMPACTION || defined CONFIG_CMA63 64#define CREATE_TRACE_POINTS65#include <trace/events/compaction.h>66 67#define block_start_pfn(pfn, order)	round_down(pfn, 1UL << (order))68#define block_end_pfn(pfn, order)	ALIGN((pfn) + 1, 1UL << (order))69 70/*71 * Page order with-respect-to which proactive compaction72 * calculates external fragmentation, which is used as73 * the "fragmentation score" of a node/zone.74 */75#if defined CONFIG_TRANSPARENT_HUGEPAGE76#define COMPACTION_HPAGE_ORDER	HPAGE_PMD_ORDER77#elif defined CONFIG_HUGETLBFS78#define COMPACTION_HPAGE_ORDER	HUGETLB_PAGE_ORDER79#else80#define COMPACTION_HPAGE_ORDER	(PMD_SHIFT - PAGE_SHIFT)81#endif82 83static struct page *mark_allocated_noprof(struct page *page, unsigned int order, gfp_t gfp_flags)84{85	post_alloc_hook(page, order, __GFP_MOVABLE);86	return page;87}88#define mark_allocated(...)	alloc_hooks(mark_allocated_noprof(__VA_ARGS__))89 90static unsigned long release_free_list(struct list_head *freepages)91{92	int order;93	unsigned long high_pfn = 0;94 95	for (order = 0; order < NR_PAGE_ORDERS; order++) {96		struct page *page, *next;97 98		list_for_each_entry_safe(page, next, &freepages[order], lru) {99			unsigned long pfn = page_to_pfn(page);100 101			list_del(&page->lru);102			/*103			 * Convert free pages into post allocation pages, so104			 * that we can free them via __free_page.105			 */106			mark_allocated(page, order, __GFP_MOVABLE);107			__free_pages(page, order);108			if (pfn > high_pfn)109				high_pfn = pfn;110		}111	}112	return high_pfn;113}114 115#ifdef CONFIG_COMPACTION116bool PageMovable(struct page *page)117{118	const struct movable_operations *mops;119 120	VM_BUG_ON_PAGE(!PageLocked(page), page);121	if (!__PageMovable(page))122		return false;123 124	mops = page_movable_ops(page);125	if (mops)126		return true;127 128	return false;129}130 131void __SetPageMovable(struct page *page, const struct movable_operations *mops)132{133	VM_BUG_ON_PAGE(!PageLocked(page), page);134	VM_BUG_ON_PAGE((unsigned long)mops & PAGE_MAPPING_MOVABLE, page);135	page->mapping = (void *)((unsigned long)mops | PAGE_MAPPING_MOVABLE);136}137EXPORT_SYMBOL(__SetPageMovable);138 139void __ClearPageMovable(struct page *page)140{141	VM_BUG_ON_PAGE(!PageMovable(page), page);142	/*143	 * This page still has the type of a movable page, but it's144	 * actually not movable any more.145	 */146	page->mapping = (void *)PAGE_MAPPING_MOVABLE;147}148EXPORT_SYMBOL(__ClearPageMovable);149 150/* Do not skip compaction more than 64 times */151#define COMPACT_MAX_DEFER_SHIFT 6152 153/*154 * Compaction is deferred when compaction fails to result in a page155 * allocation success. 1 << compact_defer_shift, compactions are skipped up156 * to a limit of 1 << COMPACT_MAX_DEFER_SHIFT157 */158static void defer_compaction(struct zone *zone, int order)159{160	zone->compact_considered = 0;161	zone->compact_defer_shift++;162 163	if (order < zone->compact_order_failed)164		zone->compact_order_failed = order;165 166	if (zone->compact_defer_shift > COMPACT_MAX_DEFER_SHIFT)167		zone->compact_defer_shift = COMPACT_MAX_DEFER_SHIFT;168 169	trace_mm_compaction_defer_compaction(zone, order);170}171 172/* Returns true if compaction should be skipped this time */173static bool compaction_deferred(struct zone *zone, int order)174{175	unsigned long defer_limit = 1UL << zone->compact_defer_shift;176 177	if (order < zone->compact_order_failed)178		return false;179 180	/* Avoid possible overflow */181	if (++zone->compact_considered >= defer_limit) {182		zone->compact_considered = defer_limit;183		return false;184	}185 186	trace_mm_compaction_deferred(zone, order);187 188	return true;189}190 191/*192 * Update defer tracking counters after successful compaction of given order,193 * which means an allocation either succeeded (alloc_success == true) or is194 * expected to succeed.195 */196void compaction_defer_reset(struct zone *zone, int order,197		bool alloc_success)198{199	if (alloc_success) {200		zone->compact_considered = 0;201		zone->compact_defer_shift = 0;202	}203	if (order >= zone->compact_order_failed)204		zone->compact_order_failed = order + 1;205 206	trace_mm_compaction_defer_reset(zone, order);207}208 209/* Returns true if restarting compaction after many failures */210static bool compaction_restarting(struct zone *zone, int order)211{212	if (order < zone->compact_order_failed)213		return false;214 215	return zone->compact_defer_shift == COMPACT_MAX_DEFER_SHIFT &&216		zone->compact_considered >= 1UL << zone->compact_defer_shift;217}218 219/* Returns true if the pageblock should be scanned for pages to isolate. */220static inline bool isolation_suitable(struct compact_control *cc,221					struct page *page)222{223	if (cc->ignore_skip_hint)224		return true;225 226	return !get_pageblock_skip(page);227}228 229static void reset_cached_positions(struct zone *zone)230{231	zone->compact_cached_migrate_pfn[0] = zone->zone_start_pfn;232	zone->compact_cached_migrate_pfn[1] = zone->zone_start_pfn;233	zone->compact_cached_free_pfn =234				pageblock_start_pfn(zone_end_pfn(zone) - 1);235}236 237#ifdef CONFIG_SPARSEMEM238/*239 * If the PFN falls into an offline section, return the start PFN of the240 * next online section. If the PFN falls into an online section or if241 * there is no next online section, return 0.242 */243static unsigned long skip_offline_sections(unsigned long start_pfn)244{245	unsigned long start_nr = pfn_to_section_nr(start_pfn);246 247	if (online_section_nr(start_nr))248		return 0;249 250	while (++start_nr <= __highest_present_section_nr) {251		if (online_section_nr(start_nr))252			return section_nr_to_pfn(start_nr);253	}254 255	return 0;256}257 258/*259 * If the PFN falls into an offline section, return the end PFN of the260 * next online section in reverse. If the PFN falls into an online section261 * or if there is no next online section in reverse, return 0.262 */263static unsigned long skip_offline_sections_reverse(unsigned long start_pfn)264{265	unsigned long start_nr = pfn_to_section_nr(start_pfn);266 267	if (!start_nr || online_section_nr(start_nr))268		return 0;269 270	while (start_nr-- > 0) {271		if (online_section_nr(start_nr))272			return section_nr_to_pfn(start_nr) + PAGES_PER_SECTION;273	}274 275	return 0;276}277#else278static unsigned long skip_offline_sections(unsigned long start_pfn)279{280	return 0;281}282 283static unsigned long skip_offline_sections_reverse(unsigned long start_pfn)284{285	return 0;286}287#endif288 289/*290 * Compound pages of >= pageblock_order should consistently be skipped until291 * released. It is always pointless to compact pages of such order (if they are292 * migratable), and the pageblocks they occupy cannot contain any free pages.293 */294static bool pageblock_skip_persistent(struct page *page)295{296	if (!PageCompound(page))297		return false;298 299	page = compound_head(page);300 301	if (compound_order(page) >= pageblock_order)302		return true;303 304	return false;305}306 307static bool308__reset_isolation_pfn(struct zone *zone, unsigned long pfn, bool check_source,309							bool check_target)310{311	struct page *page = pfn_to_online_page(pfn);312	struct page *block_page;313	struct page *end_page;314	unsigned long block_pfn;315 316	if (!page)317		return false;318	if (zone != page_zone(page))319		return false;320	if (pageblock_skip_persistent(page))321		return false;322 323	/*324	 * If skip is already cleared do no further checking once the325	 * restart points have been set.326	 */327	if (check_source && check_target && !get_pageblock_skip(page))328		return true;329 330	/*331	 * If clearing skip for the target scanner, do not select a332	 * non-movable pageblock as the starting point.333	 */334	if (!check_source && check_target &&335	    get_pageblock_migratetype(page) != MIGRATE_MOVABLE)336		return false;337 338	/* Ensure the start of the pageblock or zone is online and valid */339	block_pfn = pageblock_start_pfn(pfn);340	block_pfn = max(block_pfn, zone->zone_start_pfn);341	block_page = pfn_to_online_page(block_pfn);342	if (block_page) {343		page = block_page;344		pfn = block_pfn;345	}346 347	/* Ensure the end of the pageblock or zone is online and valid */348	block_pfn = pageblock_end_pfn(pfn) - 1;349	block_pfn = min(block_pfn, zone_end_pfn(zone) - 1);350	end_page = pfn_to_online_page(block_pfn);351	if (!end_page)352		return false;353 354	/*355	 * Only clear the hint if a sample indicates there is either a356	 * free page or an LRU page in the block. One or other condition357	 * is necessary for the block to be a migration source/target.358	 */359	do {360		if (check_source && PageLRU(page)) {361			clear_pageblock_skip(page);362			return true;363		}364 365		if (check_target && PageBuddy(page)) {366			clear_pageblock_skip(page);367			return true;368		}369 370		page += (1 << PAGE_ALLOC_COSTLY_ORDER);371	} while (page <= end_page);372 373	return false;374}375 376/*377 * This function is called to clear all cached information on pageblocks that378 * should be skipped for page isolation when the migrate and free page scanner379 * meet.380 */381static HWJS_SUSPENDS void __reset_isolation_suitable(struct zone *zone)382{383	unsigned long migrate_pfn = zone->zone_start_pfn;384	unsigned long free_pfn = zone_end_pfn(zone) - 1;385	unsigned long reset_migrate = free_pfn;386	unsigned long reset_free = migrate_pfn;387	bool source_set = false;388	bool free_set = false;389 390	/* Only flush if a full compaction finished recently */391	if (!zone->compact_blockskip_flush)392		return;393 394	zone->compact_blockskip_flush = false;395 396	/*397	 * Walk the zone and update pageblock skip information. Source looks398	 * for PageLRU while target looks for PageBuddy. When the scanner399	 * is found, both PageBuddy and PageLRU are checked as the pageblock400	 * is suitable as both source and target.401	 */402	for (; migrate_pfn < free_pfn; migrate_pfn += pageblock_nr_pages,403					free_pfn -= pageblock_nr_pages) {404		cond_resched();405 406		/* Update the migrate PFN */407		if (__reset_isolation_pfn(zone, migrate_pfn, true, source_set) &&408		    migrate_pfn < reset_migrate) {409			source_set = true;410			reset_migrate = migrate_pfn;411			zone->compact_init_migrate_pfn = reset_migrate;412			zone->compact_cached_migrate_pfn[0] = reset_migrate;413			zone->compact_cached_migrate_pfn[1] = reset_migrate;414		}415 416		/* Update the free PFN */417		if (__reset_isolation_pfn(zone, free_pfn, free_set, true) &&418		    free_pfn > reset_free) {419			free_set = true;420			reset_free = free_pfn;421			zone->compact_init_free_pfn = reset_free;422			zone->compact_cached_free_pfn = reset_free;423		}424	}425 426	/* Leave no distance if no suitable block was reset */427	if (reset_migrate >= reset_free) {428		zone->compact_cached_migrate_pfn[0] = migrate_pfn;429		zone->compact_cached_migrate_pfn[1] = migrate_pfn;430		zone->compact_cached_free_pfn = free_pfn;431	}432}433 434void reset_isolation_suitable(pg_data_t *pgdat)435{436	int zoneid;437 438	for (zoneid = 0; zoneid < MAX_NR_ZONES; zoneid++) {439		struct zone *zone = &pgdat->node_zones[zoneid];440		if (!populated_zone(zone))441			continue;442 443		__reset_isolation_suitable(zone);444	}445}446 447/*448 * Sets the pageblock skip bit if it was clear. Note that this is a hint as449 * locks are not required for read/writers. Returns true if it was already set.450 */451static bool test_and_set_skip(struct compact_control *cc, struct page *page)452{453	bool skip;454 455	/* Do not update if skip hint is being ignored */456	if (cc->ignore_skip_hint)457		return false;458 459	skip = get_pageblock_skip(page);460	if (!skip && !cc->no_set_skip_hint)461		set_pageblock_skip(page);462 463	return skip;464}465 466static void update_cached_migrate(struct compact_control *cc, unsigned long pfn)467{468	struct zone *zone = cc->zone;469 470	/* Set for isolation rather than compaction */471	if (cc->no_set_skip_hint)472		return;473 474	pfn = pageblock_end_pfn(pfn);475 476	/* Update where async and sync compaction should restart */477	if (pfn > zone->compact_cached_migrate_pfn[0])478		zone->compact_cached_migrate_pfn[0] = pfn;479	if (cc->mode != MIGRATE_ASYNC &&480	    pfn > zone->compact_cached_migrate_pfn[1])481		zone->compact_cached_migrate_pfn[1] = pfn;482}483 484/*485 * If no pages were isolated then mark this pageblock to be skipped in the486 * future. The information is later cleared by __reset_isolation_suitable().487 */488static void update_pageblock_skip(struct compact_control *cc,489			struct page *page, unsigned long pfn)490{491	struct zone *zone = cc->zone;492 493	if (cc->no_set_skip_hint)494		return;495 496	set_pageblock_skip(page);497 498	if (pfn < zone->compact_cached_free_pfn)499		zone->compact_cached_free_pfn = pfn;500}501#else502static inline bool isolation_suitable(struct compact_control *cc,503					struct page *page)504{505	return true;506}507 508static inline bool pageblock_skip_persistent(struct page *page)509{510	return false;511}512 513static inline void update_pageblock_skip(struct compact_control *cc,514			struct page *page, unsigned long pfn)515{516}517 518static void update_cached_migrate(struct compact_control *cc, unsigned long pfn)519{520}521 522static bool test_and_set_skip(struct compact_control *cc, struct page *page)523{524	return false;525}526#endif /* CONFIG_COMPACTION */527 528/*529 * Compaction requires the taking of some coarse locks that are potentially530 * very heavily contended. For async compaction, trylock and record if the531 * lock is contended. The lock will still be acquired but compaction will532 * abort when the current block is finished regardless of success rate.533 * Sync compaction acquires the lock.534 *535 * Always returns true which makes it easier to track lock state in callers.536 */537static bool compact_lock_irqsave(spinlock_t *lock, unsigned long *flags,538						struct compact_control *cc)539	__acquires(lock)540{541	/* Track if the lock is contended in async mode */542	if (cc->mode == MIGRATE_ASYNC && !cc->contended) {543		if (spin_trylock_irqsave(lock, *flags))544			return true;545 546		cc->contended = true;547	}548 549	spin_lock_irqsave(lock, *flags);550	return true;551}552 553/*554 * Compaction requires the taking of some coarse locks that are potentially555 * very heavily contended. The lock should be periodically unlocked to avoid556 * having disabled IRQs for a long time, even when there is nobody waiting on557 * the lock. It might also be that allowing the IRQs will result in558 * need_resched() becoming true. If scheduling is needed, compaction schedules.559 * Either compaction type will also abort if a fatal signal is pending.560 * In either case if the lock was locked, it is dropped and not regained.561 *562 * Returns true if compaction should abort due to fatal signal pending.563 * Returns false when compaction can continue.564 */565static HWJS_SUSPENDS bool compact_unlock_should_abort(spinlock_t *lock,566		unsigned long flags, bool *locked, struct compact_control *cc)567{568	if (*locked) {569		spin_unlock_irqrestore(lock, flags);570		*locked = false;571	}572 573	if (fatal_signal_pending(current)) {574		cc->contended = true;575		return true;576	}577 578	cond_resched();579 580	return false;581}582 583/*584 * Isolate free pages onto a private freelist. If @strict is true, will abort585 * returning 0 on any invalid PFNs or non-free pages inside of the pageblock586 * (even though it may still end up isolating some pages).587 */588static HWJS_SUSPENDS unsigned long isolate_freepages_block(struct compact_control *cc,589				unsigned long *start_pfn,590				unsigned long end_pfn,591				struct list_head *freelist,592				unsigned int stride,593				bool strict)594{595	int nr_scanned = 0, total_isolated = 0;596	struct page *page;597	unsigned long flags = 0;598	bool locked = false;599	unsigned long blockpfn = *start_pfn;600	unsigned int order;601 602	/* Strict mode is for isolation, speed is secondary */603	if (strict)604		stride = 1;605 606	page = pfn_to_page(blockpfn);607 608	/* Isolate free pages. */609	for (; blockpfn < end_pfn; blockpfn += stride, page += stride) {610		int isolated;611 612		/*613		 * Periodically drop the lock (if held) regardless of its614		 * contention, to give chance to IRQs. Abort if fatal signal615		 * pending.616		 */617		if (!(blockpfn % COMPACT_CLUSTER_MAX)618		    && compact_unlock_should_abort(&cc->zone->lock, flags,619								&locked, cc))620			break;621 622		nr_scanned++;623 624		/*625		 * For compound pages such as THP and hugetlbfs, we can save626		 * potentially a lot of iterations if we skip them at once.627		 * The check is racy, but we can consider only valid values628		 * and the only danger is skipping too much.629		 */630		if (PageCompound(page)) {631			const unsigned int order = compound_order(page);632 633			if (blockpfn + (1UL << order) <= end_pfn) {634				blockpfn += (1UL << order) - 1;635				page += (1UL << order) - 1;636				nr_scanned += (1UL << order) - 1;637			}638 639			goto isolate_fail;640		}641 642		if (!PageBuddy(page))643			goto isolate_fail;644 645		/* If we already hold the lock, we can skip some rechecking. */646		if (!locked) {647			locked = compact_lock_irqsave(&cc->zone->lock,648								&flags, cc);649 650			/* Recheck this is a buddy page under lock */651			if (!PageBuddy(page))652				goto isolate_fail;653		}654 655		/* Found a free page, will break it into order-0 pages */656		order = buddy_order(page);657		isolated = __isolate_free_page(page, order);658		if (!isolated)659			break;660		set_page_private(page, order);661 662		nr_scanned += isolated - 1;663		total_isolated += isolated;664		cc->nr_freepages += isolated;665		list_add_tail(&page->lru, &freelist[order]);666 667		if (!strict && cc->nr_migratepages <= cc->nr_freepages) {668			blockpfn += isolated;669			break;670		}671		/* Advance to the end of split page */672		blockpfn += isolated - 1;673		page += isolated - 1;674		continue;675 676isolate_fail:677		if (strict)678			break;679 680	}681 682	if (locked)683		spin_unlock_irqrestore(&cc->zone->lock, flags);684 685	/*686	 * Be careful to not go outside of the pageblock.687	 */688	if (unlikely(blockpfn > end_pfn))689		blockpfn = end_pfn;690 691	trace_mm_compaction_isolate_freepages(*start_pfn, blockpfn,692					nr_scanned, total_isolated);693 694	/* Record how far we have got within the block */695	*start_pfn = blockpfn;696 697	/*698	 * If strict isolation is requested by CMA then check that all the699	 * pages requested were isolated. If there were any failures, 0 is700	 * returned and CMA will fail.701	 */702	if (strict && blockpfn < end_pfn)703		total_isolated = 0;704 705	cc->total_free_scanned += nr_scanned;706	if (total_isolated)707		count_compact_events(COMPACTISOLATED, total_isolated);708	return total_isolated;709}710 711/**712 * isolate_freepages_range() - isolate free pages.713 * @cc:        Compaction control structure.714 * @start_pfn: The first PFN to start isolating.715 * @end_pfn:   The one-past-last PFN.716 *717 * Non-free pages, invalid PFNs, or zone boundaries within the718 * [start_pfn, end_pfn) range are considered errors, cause function to719 * undo its actions and return zero. cc->freepages[] are empty.720 *721 * Otherwise, function returns one-past-the-last PFN of isolated page722 * (which may be greater then end_pfn if end fell in a middle of723 * a free page). cc->freepages[] contain free pages isolated.724 */725unsigned long726isolate_freepages_range(struct compact_control *cc,727			unsigned long start_pfn, unsigned long end_pfn)728{729	unsigned long isolated, pfn, block_start_pfn, block_end_pfn;730	int order;731 732	for (order = 0; order < NR_PAGE_ORDERS; order++)733		INIT_LIST_HEAD(&cc->freepages[order]);734 735	pfn = start_pfn;736	block_start_pfn = pageblock_start_pfn(pfn);737	if (block_start_pfn < cc->zone->zone_start_pfn)738		block_start_pfn = cc->zone->zone_start_pfn;739	block_end_pfn = pageblock_end_pfn(pfn);740 741	for (; pfn < end_pfn; pfn += isolated,742				block_start_pfn = block_end_pfn,743				block_end_pfn += pageblock_nr_pages) {744		/* Protect pfn from changing by isolate_freepages_block */745		unsigned long isolate_start_pfn = pfn;746 747		/*748		 * pfn could pass the block_end_pfn if isolated freepage749		 * is more than pageblock order. In this case, we adjust750		 * scanning range to right one.751		 */752		if (pfn >= block_end_pfn) {753			block_start_pfn = pageblock_start_pfn(pfn);754			block_end_pfn = pageblock_end_pfn(pfn);755		}756 757		block_end_pfn = min(block_end_pfn, end_pfn);758 759		if (!pageblock_pfn_to_page(block_start_pfn,760					block_end_pfn, cc->zone))761			break;762 763		isolated = isolate_freepages_block(cc, &isolate_start_pfn,764					block_end_pfn, cc->freepages, 0, true);765 766		/*767		 * In strict mode, isolate_freepages_block() returns 0 if768		 * there are any holes in the block (ie. invalid PFNs or769		 * non-free pages).770		 */771		if (!isolated)772			break;773 774		/*775		 * If we managed to isolate pages, it is always (1 << n) *776		 * pageblock_nr_pages for some non-negative n.  (Max order777		 * page may span two pageblocks).778		 */779	}780 781	if (pfn < end_pfn) {782		/* Loop terminated early, cleanup. */783		release_free_list(cc->freepages);784		return 0;785	}786 787	/* We don't use freelists for anything. */788	return pfn;789}790 791/* Similar to reclaim, but different enough that they don't share logic */792static HWJS_SUSPENDS bool too_many_isolated(struct compact_control *cc)793{794	pg_data_t *pgdat = cc->zone->zone_pgdat;795	bool too_many;796 797	unsigned long active, inactive, isolated;798 799	inactive = node_page_state(pgdat, NR_INACTIVE_FILE) +800			node_page_state(pgdat, NR_INACTIVE_ANON);801	active = node_page_state(pgdat, NR_ACTIVE_FILE) +802			node_page_state(pgdat, NR_ACTIVE_ANON);803	isolated = node_page_state(pgdat, NR_ISOLATED_FILE) +804			node_page_state(pgdat, NR_ISOLATED_ANON);805 806	/*807	 * Allow GFP_NOFS to isolate past the limit set for regular808	 * compaction runs. This prevents an ABBA deadlock when other809	 * compactors have already isolated to the limit, but are810	 * blocked on filesystem locks held by the GFP_NOFS thread.811	 */812	if (cc->gfp_mask & __GFP_FS) {813		inactive >>= 3;814		active >>= 3;815	}816 817	too_many = isolated > (inactive + active) / 2;818	if (!too_many)819		wake_throttle_isolated(pgdat);820 821	return too_many;822}823 824/**825 * skip_isolation_on_order() - determine when to skip folio isolation based on826 *			       folio order and compaction target order827 * @order:		to-be-isolated folio order828 * @target_order:	compaction target order829 *830 * This avoids unnecessary folio isolations during compaction.831 */832static bool skip_isolation_on_order(int order, int target_order)833{834	/*835	 * Unless we are performing global compaction (i.e.,836	 * is_via_compact_memory), skip any folios that are larger than the837	 * target order: we wouldn't be here if we'd have a free folio with838	 * the desired target_order, so migrating this folio would likely fail839	 * later.840	 */841	if (!is_via_compact_memory(target_order) && order >= target_order)842		return true;843	/*844	 * We limit memory compaction to pageblocks and won't try845	 * creating free blocks of memory that are larger than that.846	 */847	return order >= pageblock_order;848}849 850/**851 * isolate_migratepages_block() - isolate all migrate-able pages within852 *				  a single pageblock853 * @cc:		Compaction control structure.854 * @low_pfn:	The first PFN to isolate855 * @end_pfn:	The one-past-the-last PFN to isolate, within same pageblock856 * @mode:	Isolation mode to be used.857 *858 * Isolate all pages that can be migrated from the range specified by859 * [low_pfn, end_pfn). The range is expected to be within same pageblock.860 * Returns errno, like -EAGAIN or -EINTR in case e.g signal pending or congestion,861 * -ENOMEM in case we could not allocate a page, or 0.862 * cc->migrate_pfn will contain the next pfn to scan.863 *864 * The pages are isolated on cc->migratepages list (not required to be empty),865 * and cc->nr_migratepages is updated accordingly.866 */867static HWJS_SUSPENDS int868isolate_migratepages_block(struct compact_control *cc, unsigned long low_pfn,869			unsigned long end_pfn, isolate_mode_t mode)870{871	pg_data_t *pgdat = cc->zone->zone_pgdat;872	unsigned long nr_scanned = 0, nr_isolated = 0;873	struct lruvec *lruvec;874	unsigned long flags = 0;875	struct lruvec *locked = NULL;876	struct folio *folio = NULL;877	struct page *page = NULL, *valid_page = NULL;878	struct address_space *mapping;879	unsigned long start_pfn = low_pfn;880	bool skip_on_failure = false;881	unsigned long next_skip_pfn = 0;882	bool skip_updated = false;883	int ret = 0;884 885	cc->migrate_pfn = low_pfn;886 887	/*888	 * Ensure that there are not too many pages isolated from the LRU889	 * list by either parallel reclaimers or compaction. If there are,890	 * delay for some time until fewer pages are isolated891	 */892	while (unlikely(too_many_isolated(cc))) {893		/* stop isolation if there are still pages not migrated */894		if (cc->nr_migratepages)895			return -EAGAIN;896 897		/* async migration should just abort */898		if (cc->mode == MIGRATE_ASYNC)899			return -EAGAIN;900 901		reclaim_throttle(pgdat, VMSCAN_THROTTLE_ISOLATED);902 903		if (fatal_signal_pending(current))904			return -EINTR;905	}906 907	cond_resched();908 909	if (cc->direct_compaction && (cc->mode == MIGRATE_ASYNC)) {910		skip_on_failure = true;911		next_skip_pfn = block_end_pfn(low_pfn, cc->order);912	}913 914	/* Time to isolate some pages for migration */915	for (; low_pfn < end_pfn; low_pfn++) {916		bool is_dirty, is_unevictable;917 918		if (skip_on_failure && low_pfn >= next_skip_pfn) {919			/*920			 * We have isolated all migration candidates in the921			 * previous order-aligned block, and did not skip it due922			 * to failure. We should migrate the pages now and923			 * hopefully succeed compaction.924			 */925			if (nr_isolated)926				break;927 928			/*929			 * We failed to isolate in the previous order-aligned930			 * block. Set the new boundary to the end of the931			 * current block. Note we can't simply increase932			 * next_skip_pfn by 1 << order, as low_pfn might have933			 * been incremented by a higher number due to skipping934			 * a compound or a high-order buddy page in the935			 * previous loop iteration.936			 */937			next_skip_pfn = block_end_pfn(low_pfn, cc->order);938		}939 940		/*941		 * Periodically drop the lock (if held) regardless of its942		 * contention, to give chance to IRQs. Abort completely if943		 * a fatal signal is pending.944		 */945		if (!(low_pfn % COMPACT_CLUSTER_MAX)) {946			if (locked) {947				unlock_page_lruvec_irqrestore(locked, flags);948				locked = NULL;949			}950 951			if (fatal_signal_pending(current)) {952				cc->contended = true;953				ret = -EINTR;954 955				goto fatal_pending;956			}957 958			cond_resched();959		}960 961		nr_scanned++;962 963		page = pfn_to_page(low_pfn);964 965		/*966		 * Check if the pageblock has already been marked skipped.967		 * Only the first PFN is checked as the caller isolates968		 * COMPACT_CLUSTER_MAX at a time so the second call must969		 * not falsely conclude that the block should be skipped.970		 */971		if (!valid_page && (pageblock_aligned(low_pfn) ||972				    low_pfn == cc->zone->zone_start_pfn)) {973			if (!isolation_suitable(cc, page)) {974				low_pfn = end_pfn;975				folio = NULL;976				goto isolate_abort;977			}978			valid_page = page;979		}980 981		if (PageHuge(page)) {982			/*983			 * skip hugetlbfs if we are not compacting for pages984			 * bigger than its order. THPs and other compound pages985			 * are handled below.986			 */987			if (!cc->alloc_contig) {988				const unsigned int order = compound_order(page);989 990				if (order <= MAX_PAGE_ORDER) {991					low_pfn += (1UL << order) - 1;992					nr_scanned += (1UL << order) - 1;993				}994				goto isolate_fail;995			}996			/* for alloc_contig case */997			if (locked) {998				unlock_page_lruvec_irqrestore(locked, flags);999				locked = NULL;1000			}1001 1002			ret = isolate_or_dissolve_huge_page(page, &cc->migratepages);1003 1004			/*1005			 * Fail isolation in case isolate_or_dissolve_huge_page()1006			 * reports an error. In case of -ENOMEM, abort right away.1007			 */1008			if (ret < 0) {1009				 /* Do not report -EBUSY down the chain */1010				if (ret == -EBUSY)1011					ret = 0;1012				low_pfn += compound_nr(page) - 1;1013				nr_scanned += compound_nr(page) - 1;1014				goto isolate_fail;1015			}1016 1017			if (PageHuge(page)) {1018				/*1019				 * Hugepage was successfully isolated and placed1020				 * on the cc->migratepages list.1021				 */1022				folio = page_folio(page);1023				low_pfn += folio_nr_pages(folio) - 1;1024				goto isolate_success_no_list;1025			}1026 1027			/*1028			 * Ok, the hugepage was dissolved. Now these pages are1029			 * Buddy and cannot be re-allocated because they are1030			 * isolated. Fall-through as the check below handles1031			 * Buddy pages.1032			 */1033		}1034 1035		/*1036		 * Skip if free. We read page order here without zone lock1037		 * which is generally unsafe, but the race window is small and1038		 * the worst thing that can happen is that we skip some1039		 * potential isolation targets.1040		 */1041		if (PageBuddy(page)) {1042			unsigned long freepage_order = buddy_order_unsafe(page);1043 1044			/*1045			 * Without lock, we cannot be sure that what we got is1046			 * a valid page order. Consider only values in the1047			 * valid order range to prevent low_pfn overflow.1048			 */1049			if (freepage_order > 0 && freepage_order <= MAX_PAGE_ORDER) {1050				low_pfn += (1UL << freepage_order) - 1;1051				nr_scanned += (1UL << freepage_order) - 1;1052			}1053			continue;1054		}1055 1056		/*1057		 * Regardless of being on LRU, compound pages such as THP1058		 * (hugetlbfs is handled above) are not to be compacted unless1059		 * we are attempting an allocation larger than the compound1060		 * page size. We can potentially save a lot of iterations if we1061		 * skip them at once. The check is racy, but we can consider1062		 * only valid values and the only danger is skipping too much.1063		 */1064		if (PageCompound(page) && !cc->alloc_contig) {1065			const unsigned int order = compound_order(page);1066 1067			/* Skip based on page order and compaction target order. */1068			if (skip_isolation_on_order(order, cc->order)) {1069				if (order <= MAX_PAGE_ORDER) {1070					low_pfn += (1UL << order) - 1;1071					nr_scanned += (1UL << order) - 1;1072				}1073				goto isolate_fail;1074			}1075		}1076 1077		/*1078		 * Check may be lockless but that's ok as we recheck later.1079		 * It's possible to migrate LRU and non-lru movable pages.1080		 * Skip any other type of page1081		 */1082		if (!PageLRU(page)) {1083			/*1084			 * __PageMovable can return false positive so we need1085			 * to verify it under page_lock.1086			 */1087			if (unlikely(__PageMovable(page)) &&1088					!PageIsolated(page)) {1089				if (locked) {1090					unlock_page_lruvec_irqrestore(locked, flags);1091					locked = NULL;1092				}1093 1094				if (isolate_movable_page(page, mode)) {1095					folio = page_folio(page);1096					goto isolate_success;1097				}1098			}1099 1100			goto isolate_fail;1101		}1102 1103		/*1104		 * Be careful not to clear PageLRU until after we're1105		 * sure the page is not being freed elsewhere -- the1106		 * page release code relies on it.1107		 */1108		folio = folio_get_nontail_page(page);1109		if (unlikely(!folio))1110			goto isolate_fail;1111 1112		/*1113		 * Migration will fail if an anonymous page is pinned in memory,1114		 * so avoid taking lru_lock and isolating it unnecessarily in an1115		 * admittedly racy check.1116		 */1117		mapping = folio_mapping(folio);1118		if (!mapping && (folio_ref_count(folio) - 1) > folio_mapcount(folio))1119			goto isolate_fail_put;1120 1121		/*1122		 * Only allow to migrate anonymous pages in GFP_NOFS context1123		 * because those do not depend on fs locks.1124		 */1125		if (!(cc->gfp_mask & __GFP_FS) && mapping)1126			goto isolate_fail_put;1127 1128		/* Only take pages on LRU: a check now makes later tests safe */1129		if (!folio_test_lru(folio))1130			goto isolate_fail_put;1131 1132		is_unevictable = folio_test_unevictable(folio);1133 1134		/* Compaction might skip unevictable pages but CMA takes them */1135		if (!(mode & ISOLATE_UNEVICTABLE) && is_unevictable)1136			goto isolate_fail_put;1137 1138		/*1139		 * To minimise LRU disruption, the caller can indicate with1140		 * ISOLATE_ASYNC_MIGRATE that it only wants to isolate pages1141		 * it will be able to migrate without blocking - clean pages1142		 * for the most part.  PageWriteback would require blocking.1143		 */1144		if ((mode & ISOLATE_ASYNC_MIGRATE) && folio_test_writeback(folio))1145			goto isolate_fail_put;1146 1147		is_dirty = folio_test_dirty(folio);1148 1149		if (((mode & ISOLATE_ASYNC_MIGRATE) && is_dirty) ||1150		    (mapping && is_unevictable)) {1151			bool migrate_dirty = true;1152			bool is_inaccessible;1153 1154			/*1155			 * Only folios without mappings or that have1156			 * a ->migrate_folio callback are possible to migrate1157			 * without blocking.1158			 *1159			 * Folios from inaccessible mappings are not migratable.1160			 *1161			 * However, we can be racing with truncation, which can1162			 * free the mapping that we need to check. Truncation1163			 * holds the folio lock until after the folio is removed1164			 * from the page so holding it ourselves is sufficient.1165			 *1166			 * To avoid locking the folio just to check inaccessible,1167			 * assume every inaccessible folio is also unevictable,1168			 * which is a cheaper test.  If our assumption goes1169			 * wrong, it's not a correctness bug, just potentially1170			 * wasted cycles.1171			 */1172			if (!folio_trylock(folio))1173				goto isolate_fail_put;1174 1175			mapping = folio_mapping(folio);1176			if ((mode & ISOLATE_ASYNC_MIGRATE) && is_dirty) {1177				migrate_dirty = !mapping ||1178						mapping->a_ops->migrate_folio;1179			}1180			is_inaccessible = mapping && mapping_inaccessible(mapping);1181			folio_unlock(folio);1182			if (!migrate_dirty || is_inaccessible)1183				goto isolate_fail_put;1184		}1185 1186		/* Try isolate the folio */1187		if (!folio_test_clear_lru(folio))1188			goto isolate_fail_put;1189 1190		lruvec = folio_lruvec(folio);1191 1192		/* If we already hold the lock, we can skip some rechecking */1193		if (lruvec != locked) {1194			if (locked)1195				unlock_page_lruvec_irqrestore(locked, flags);1196 1197			compact_lock_irqsave(&lruvec->lru_lock, &flags, cc);1198			locked = lruvec;1199 1200			lruvec_memcg_debug(lruvec, folio);1201 1202			/*1203			 * Try get exclusive access under lock. If marked for1204			 * skip, the scan is aborted unless the current context1205			 * is a rescan to reach the end of the pageblock.1206			 */1207			if (!skip_updated && valid_page) {1208				skip_updated = true;1209				if (test_and_set_skip(cc, valid_page) &&1210				    !cc->finish_pageblock) {1211					low_pfn = end_pfn;1212					goto isolate_abort;1213				}1214			}1215 1216			/*1217			 * Check LRU folio order under the lock1218			 */1219			if (unlikely(skip_isolation_on_order(folio_order(folio),1220							     cc->order) &&1221				     !cc->alloc_contig)) {1222				low_pfn += folio_nr_pages(folio) - 1;1223				nr_scanned += folio_nr_pages(folio) - 1;1224				folio_set_lru(folio);1225				goto isolate_fail_put;1226			}1227		}1228 1229		/* The folio is taken off the LRU */1230		if (folio_test_large(folio))1231			low_pfn += folio_nr_pages(folio) - 1;1232 1233		/* Successfully isolated */1234		lruvec_del_folio(lruvec, folio);1235		node_stat_mod_folio(folio,1236				NR_ISOLATED_ANON + folio_is_file_lru(folio),1237				folio_nr_pages(folio));1238 1239isolate_success:1240		list_add(&folio->lru, &cc->migratepages);1241isolate_success_no_list:1242		cc->nr_migratepages += folio_nr_pages(folio);1243		nr_isolated += folio_nr_pages(folio);1244		nr_scanned += folio_nr_pages(folio) - 1;1245 1246		/*1247		 * Avoid isolating too much unless this block is being1248		 * fully scanned (e.g. dirty/writeback pages, parallel allocation)1249		 * or a lock is contended. For contention, isolate quickly to1250		 * potentially remove one source of contention.1251		 */1252		if (cc->nr_migratepages >= COMPACT_CLUSTER_MAX &&1253		    !cc->finish_pageblock && !cc->contended) {1254			++low_pfn;1255			break;1256		}1257 1258		continue;1259 1260isolate_fail_put:1261		/* Avoid potential deadlock in freeing page under lru_lock */1262		if (locked) {1263			unlock_page_lruvec_irqrestore(locked, flags);1264			locked = NULL;1265		}1266		folio_put(folio);1267 1268isolate_fail:1269		if (!skip_on_failure && ret != -ENOMEM)1270			continue;1271 1272		/*1273		 * We have isolated some pages, but then failed. Release them1274		 * instead of migrating, as we cannot form the cc->order buddy1275		 * page anyway.1276		 */1277		if (nr_isolated) {1278			if (locked) {1279				unlock_page_lruvec_irqrestore(locked, flags);1280				locked = NULL;1281			}1282			putback_movable_pages(&cc->migratepages);1283			cc->nr_migratepages = 0;1284			nr_isolated = 0;1285		}1286 1287		if (low_pfn < next_skip_pfn) {1288			low_pfn = next_skip_pfn - 1;1289			/*1290			 * The check near the loop beginning would have updated1291			 * next_skip_pfn too, but this is a bit simpler.1292			 */1293			next_skip_pfn += 1UL << cc->order;1294		}1295 1296		if (ret == -ENOMEM)1297			break;1298	}1299 1300	/*1301	 * The PageBuddy() check could have potentially brought us outside1302	 * the range to be scanned.1303	 */1304	if (unlikely(low_pfn > end_pfn))1305		low_pfn = end_pfn;1306 1307	folio = NULL;1308 1309isolate_abort:1310	if (locked)1311		unlock_page_lruvec_irqrestore(locked, flags);1312	if (folio) {1313		folio_set_lru(folio);1314		folio_put(folio);1315	}1316 1317	/*1318	 * Update the cached scanner pfn once the pageblock has been scanned.1319	 * Pages will either be migrated in which case there is no point1320	 * scanning in the near future or migration failed in which case the1321	 * failure reason may persist. The block is marked for skipping if1322	 * there were no pages isolated in the block or if the block is1323	 * rescanned twice in a row.1324	 */1325	if (low_pfn == end_pfn && (!nr_isolated || cc->finish_pageblock)) {1326		if (!cc->no_set_skip_hint && valid_page && !skip_updated)1327			set_pageblock_skip(valid_page);1328		update_cached_migrate(cc, low_pfn);1329	}1330 1331	trace_mm_compaction_isolate_migratepages(start_pfn, low_pfn,1332						nr_scanned, nr_isolated);1333 1334fatal_pending:1335	cc->total_migrate_scanned += nr_scanned;1336	if (nr_isolated)1337		count_compact_events(COMPACTISOLATED, nr_isolated);1338 1339	cc->migrate_pfn = low_pfn;1340 1341	return ret;1342}1343 1344/**1345 * isolate_migratepages_range() - isolate migrate-able pages in a PFN range1346 * @cc:        Compaction control structure.1347 * @start_pfn: The first PFN to start isolating.1348 * @end_pfn:   The one-past-last PFN.1349 *1350 * Returns -EAGAIN when contented, -EINTR in case of a signal pending, -ENOMEM1351 * in case we could not allocate a page, or 0.1352 */1353int1354isolate_migratepages_range(struct compact_control *cc, unsigned long start_pfn,1355							unsigned long end_pfn)1356{1357	unsigned long pfn, block_start_pfn, block_end_pfn;1358	int ret = 0;1359 1360	/* Scan block by block. First and last block may be incomplete */1361	pfn = start_pfn;1362	block_start_pfn = pageblock_start_pfn(pfn);1363	if (block_start_pfn < cc->zone->zone_start_pfn)1364		block_start_pfn = cc->zone->zone_start_pfn;1365	block_end_pfn = pageblock_end_pfn(pfn);1366 1367	for (; pfn < end_pfn; pfn = block_end_pfn,1368				block_start_pfn = block_end_pfn,1369				block_end_pfn += pageblock_nr_pages) {1370 1371		block_end_pfn = min(block_end_pfn, end_pfn);1372 1373		if (!pageblock_pfn_to_page(block_start_pfn,1374					block_end_pfn, cc->zone))1375			continue;1376 1377		ret = isolate_migratepages_block(cc, pfn, block_end_pfn,1378						 ISOLATE_UNEVICTABLE);1379 1380		if (ret)1381			break;1382 1383		if (cc->nr_migratepages >= COMPACT_CLUSTER_MAX)1384			break;1385	}1386 1387	return ret;1388}1389 1390#endif /* CONFIG_COMPACTION || CONFIG_CMA */1391#ifdef CONFIG_COMPACTION1392 1393static bool suitable_migration_source(struct compact_control *cc,1394							struct page *page)1395{1396	int block_mt;1397 1398	if (pageblock_skip_persistent(page))1399		return false;1400 1401	if ((cc->mode != MIGRATE_ASYNC) || !cc->direct_compaction)1402		return true;1403 1404	block_mt = get_pageblock_migratetype(page);1405 1406	if (cc->migratetype == MIGRATE_MOVABLE)1407		return is_migrate_movable(block_mt);1408	else1409		return block_mt == cc->migratetype;1410}1411 1412/* Returns true if the page is within a block suitable for migration to */1413static bool suitable_migration_target(struct compact_control *cc,1414							struct page *page)1415{1416	/* If the page is a large free page, then disallow migration */1417	if (PageBuddy(page)) {1418		int order = cc->order > 0 ? cc->order : pageblock_order;1419 1420		/*1421		 * We are checking page_order without zone->lock taken. But1422		 * the only small danger is that we skip a potentially suitable1423		 * pageblock, so it's not worth to check order for valid range.1424		 */1425		if (buddy_order_unsafe(page) >= order)1426			return false;1427	}1428 1429	if (cc->ignore_block_suitable)1430		return true;1431 1432	/* If the block is MIGRATE_MOVABLE or MIGRATE_CMA, allow migration */1433	if (is_migrate_movable(get_pageblock_migratetype(page)))1434		return true;1435 1436	/* Otherwise skip the block */1437	return false;1438}1439 1440static inline unsigned int1441freelist_scan_limit(struct compact_control *cc)1442{1443	unsigned short shift = BITS_PER_LONG - 1;1444 1445	return (COMPACT_CLUSTER_MAX >> min(shift, cc->fast_search_fail)) + 1;1446}1447 1448/*1449 * Test whether the free scanner has reached the same or lower pageblock than1450 * the migration scanner, and compaction should thus terminate.1451 */1452static inline bool compact_scanners_met(struct compact_control *cc)1453{1454	return (cc->free_pfn >> pageblock_order)1455		<= (cc->migrate_pfn >> pageblock_order);1456}1457 1458/*1459 * Used when scanning for a suitable migration target which scans freelists1460 * in reverse. Reorders the list such as the unscanned pages are scanned1461 * first on the next iteration of the free scanner1462 */1463static void1464move_freelist_head(struct list_head *freelist, struct page *freepage)1465{1466	LIST_HEAD(sublist);1467 1468	if (!list_is_first(&freepage->buddy_list, freelist)) {1469		list_cut_before(&sublist, freelist, &freepage->buddy_list);1470		list_splice_tail(&sublist, freelist);1471	}1472}1473 1474/*1475 * Similar to move_freelist_head except used by the migration scanner1476 * when scanning forward. It's possible for these list operations to1477 * move against each other if they search the free list exactly in1478 * lockstep.1479 */1480static void1481move_freelist_tail(struct list_head *freelist, struct page *freepage)1482{1483	LIST_HEAD(sublist);1484 1485	if (!list_is_last(&freepage->buddy_list, freelist)) {1486		list_cut_position(&sublist, freelist, &freepage->buddy_list);1487		list_splice_tail(&sublist, freelist);1488	}1489}1490 1491static HWJS_SUSPENDS void1492fast_isolate_around(struct compact_control *cc, unsigned long pfn)1493{1494	unsigned long start_pfn, end_pfn;1495	struct page *page;1496 1497	/* Do not search around if there are enough pages already */1498	if (cc->nr_freepages >= cc->nr_migratepages)1499		return;1500 1501	/* Minimise scanning during async compaction */1502	if (cc->direct_compaction && cc->mode == MIGRATE_ASYNC)1503		return;1504 1505	/* Pageblock boundaries */1506	start_pfn = max(pageblock_start_pfn(pfn), cc->zone->zone_start_pfn);1507	end_pfn = min(pageblock_end_pfn(pfn), zone_end_pfn(cc->zone));1508 1509	page = pageblock_pfn_to_page(start_pfn, end_pfn, cc->zone);1510	if (!page)1511		return;1512 1513	isolate_freepages_block(cc, &start_pfn, end_pfn, cc->freepages, 1, false);1514 1515	/* Skip this pageblock in the future as it's full or nearly full */1516	if (start_pfn == end_pfn && !cc->no_set_skip_hint)1517		set_pageblock_skip(page);1518}1519 1520/* Search orders in round-robin fashion */1521static int next_search_order(struct compact_control *cc, int order)1522{1523	order--;1524	if (order < 0)1525		order = cc->order - 1;1526 1527	/* Search wrapped around? */1528	if (order == cc->search_order) {1529		cc->search_order--;1530		if (cc->search_order < 0)1531			cc->search_order = cc->order - 1;1532		return -1;1533	}1534 1535	return order;1536}1537 1538static HWJS_SUSPENDS void fast_isolate_freepages(struct compact_control *cc)1539{1540	unsigned int limit = max(1U, freelist_scan_limit(cc) >> 1);1541	unsigned int nr_scanned = 0, total_isolated = 0;1542	unsigned long low_pfn, min_pfn, highest = 0;1543	unsigned long nr_isolated = 0;1544	unsigned long distance;1545	struct page *page = NULL;1546	bool scan_start = false;1547	int order;1548 1549	/* Full compaction passes in a negative order */1550	if (cc->order <= 0)1551		return;1552 1553	/*1554	 * If starting the scan, use a deeper search and use the highest1555	 * PFN found if a suitable one is not found.1556	 */1557	if (cc->free_pfn >= cc->zone->compact_init_free_pfn) {1558		limit = pageblock_nr_pages >> 1;1559		scan_start = true;1560	}1561 1562	/*1563	 * Preferred point is in the top quarter of the scan space but take1564	 * a pfn from the top half if the search is problematic.1565	 */1566	distance = (cc->free_pfn - cc->migrate_pfn);1567	low_pfn = pageblock_start_pfn(cc->free_pfn - (distance >> 2));1568	min_pfn = pageblock_start_pfn(cc->free_pfn - (distance >> 1));1569 1570	if (WARN_ON_ONCE(min_pfn > low_pfn))1571		low_pfn = min_pfn;1572 1573	/*1574	 * Search starts from the last successful isolation order or the next1575	 * order to search after a previous failure1576	 */1577	cc->search_order = min_t(unsigned int, cc->order - 1, cc->search_order);1578 1579	for (order = cc->search_order;1580	     !page && order >= 0;1581	     order = next_search_order(cc, order)) {1582		struct free_area *area = &cc->zone->free_area[order];1583		struct list_head *freelist;1584		struct page *freepage;1585		unsigned long flags;1586		unsigned int order_scanned = 0;1587		unsigned long high_pfn = 0;1588 1589		if (!area->nr_free)1590			continue;1591 1592		spin_lock_irqsave(&cc->zone->lock, flags);1593		freelist = &area->free_list[MIGRATE_MOVABLE];1594		list_for_each_entry_reverse(freepage, freelist, buddy_list) {1595			unsigned long pfn;1596 1597			order_scanned++;1598			nr_scanned++;1599			pfn = page_to_pfn(freepage);1600 1601			if (pfn >= highest)1602				highest = max(pageblock_start_pfn(pfn),1603					      cc->zone->zone_start_pfn);1604 1605			if (pfn >= low_pfn) {1606				cc->fast_search_fail = 0;1607				cc->search_order = order;1608				page = freepage;1609				break;1610			}1611 1612			if (pfn >= min_pfn && pfn > high_pfn) {1613				high_pfn = pfn;1614 1615				/* Shorten the scan if a candidate is found */1616				limit >>= 1;1617			}1618 1619			if (order_scanned >= limit)1620				break;1621		}1622 1623		/* Use a maximum candidate pfn if a preferred one was not found */1624		if (!page && high_pfn) {1625			page = pfn_to_page(high_pfn);1626 1627			/* Update freepage for the list reorder below */1628			freepage = page;1629		}1630 1631		/* Reorder to so a future search skips recent pages */1632		move_freelist_head(freelist, freepage);1633 1634		/* Isolate the page if available */1635		if (page) {1636			if (__isolate_free_page(page, order)) {1637				set_page_private(page, order);1638				nr_isolated = 1 << order;1639				nr_scanned += nr_isolated - 1;1640				total_isolated += nr_isolated;1641				cc->nr_freepages += nr_isolated;1642				list_add_tail(&page->lru, &cc->freepages[order]);1643				count_compact_events(COMPACTISOLATED, nr_isolated);1644			} else {1645				/* If isolation fails, abort the search */1646				order = cc->search_order + 1;1647				page = NULL;1648			}1649		}1650 1651		spin_unlock_irqrestore(&cc->zone->lock, flags);1652 1653		/* Skip fast search if enough freepages isolated */1654		if (cc->nr_freepages >= cc->nr_migratepages)1655			break;1656 1657		/*1658		 * Smaller scan on next order so the total scan is related1659		 * to freelist_scan_limit.1660		 */1661		if (order_scanned >= limit)1662			limit = max(1U, limit >> 1);1663	}1664 1665	trace_mm_compaction_fast_isolate_freepages(min_pfn, cc->free_pfn,1666						   nr_scanned, total_isolated);1667 1668	if (!page) {1669		cc->fast_search_fail++;1670		if (scan_start) {1671			/*1672			 * Use the highest PFN found above min. If one was1673			 * not found, be pessimistic for direct compaction1674			 * and use the min mark.1675			 */1676			if (highest >= min_pfn) {1677				page = pfn_to_page(highest);1678				cc->free_pfn = highest;1679			} else {1680				if (cc->direct_compaction && pfn_valid(min_pfn)) {1681					page = pageblock_pfn_to_page(min_pfn,1682						min(pageblock_end_pfn(min_pfn),1683						    zone_end_pfn(cc->zone)),1684						cc->zone);1685					if (page && !suitable_migration_target(cc, page))1686						page = NULL;1687 1688					cc->free_pfn = min_pfn;1689				}1690			}1691		}1692	}1693 1694	if (highest && highest >= cc->zone->compact_cached_free_pfn) {1695		highest -= pageblock_nr_pages;1696		cc->zone->compact_cached_free_pfn = highest;1697	}1698 1699	cc->total_free_scanned += nr_scanned;1700	if (!page)1701		return;1702 1703	low_pfn = page_to_pfn(page);1704	fast_isolate_around(cc, low_pfn);1705}1706 1707/*1708 * Based on information in the current compact_control, find blocks1709 * suitable for isolating free pages from and then isolate them.1710 */1711static HWJS_SUSPENDS void isolate_freepages(struct compact_control *cc)1712{1713	struct zone *zone = cc->zone;1714	struct page *page;1715	unsigned long block_start_pfn;	/* start of current pageblock */1716	unsigned long isolate_start_pfn; /* exact pfn we start at */1717	unsigned long block_end_pfn;	/* end of current pageblock */1718	unsigned long low_pfn;	     /* lowest pfn scanner is able to scan */1719	unsigned int stride;1720 1721	/* Try a small search of the free lists for a candidate */1722	fast_isolate_freepages(cc);1723	if (cc->nr_freepages)1724		return;1725 1726	/*1727	 * Initialise the free scanner. The starting point is where we last1728	 * successfully isolated from, zone-cached value, or the end of the1729	 * zone when isolating for the first time. For looping we also need1730	 * this pfn aligned down to the pageblock boundary, because we do1731	 * block_start_pfn -= pageblock_nr_pages in the for loop.1732	 * For ending point, take care when isolating in last pageblock of a1733	 * zone which ends in the middle of a pageblock.1734	 * The low boundary is the end of the pageblock the migration scanner1735	 * is using.1736	 */1737	isolate_start_pfn = cc->free_pfn;1738	block_start_pfn = pageblock_start_pfn(isolate_start_pfn);1739	block_end_pfn = min(block_start_pfn + pageblock_nr_pages,1740						zone_end_pfn(zone));1741	low_pfn = pageblock_end_pfn(cc->migrate_pfn);1742	stride = cc->mode == MIGRATE_ASYNC ? COMPACT_CLUSTER_MAX : 1;1743 1744	/*1745	 * Isolate free pages until enough are available to migrate the1746	 * pages on cc->migratepages. We stop searching if the migrate1747	 * and free page scanners meet or enough free pages are isolated.1748	 */1749	for (; block_start_pfn >= low_pfn;1750				block_end_pfn = block_start_pfn,1751				block_start_pfn -= pageblock_nr_pages,1752				isolate_start_pfn = block_start_pfn) {1753		unsigned long nr_isolated;1754 1755		/*1756		 * This can iterate a massively long zone without finding any1757		 * suitable migration targets, so periodically check resched.1758		 */1759		if (!(block_start_pfn % (COMPACT_CLUSTER_MAX * pageblock_nr_pages)))1760			cond_resched();1761 1762		page = pageblock_pfn_to_page(block_start_pfn, block_end_pfn,1763									zone);1764		if (!page) {1765			unsigned long next_pfn;1766 1767			next_pfn = skip_offline_sections_reverse(block_start_pfn);1768			if (next_pfn)1769				block_start_pfn = max(next_pfn, low_pfn);1770 1771			continue;1772		}1773 1774		/* Check the block is suitable for migration */1775		if (!suitable_migration_target(cc, page))1776			continue;1777 1778		/* If isolation recently failed, do not retry */1779		if (!isolation_suitable(cc, page))1780			continue;1781 1782		/* Found a block suitable for isolating free pages from. */1783		nr_isolated = isolate_freepages_block(cc, &isolate_start_pfn,1784					block_end_pfn, cc->freepages, stride, false);1785 1786		/* Update the skip hint if the full pageblock was scanned */1787		if (isolate_start_pfn == block_end_pfn)1788			update_pageblock_skip(cc, page, block_start_pfn -1789					      pageblock_nr_pages);1790 1791		/* Are enough freepages isolated? */1792		if (cc->nr_freepages >= cc->nr_migratepages) {1793			if (isolate_start_pfn >= block_end_pfn) {1794				/*1795				 * Restart at previous pageblock if more1796				 * freepages can be isolated next time.1797				 */1798				isolate_start_pfn =1799					block_start_pfn - pageblock_nr_pages;1800			}1801			break;1802		} else if (isolate_start_pfn < block_end_pfn) {1803			/*1804			 * If isolation failed early, do not continue1805			 * needlessly.1806			 */1807			break;1808		}1809 1810		/* Adjust stride depending on isolation */1811		if (nr_isolated) {1812			stride = 1;1813			continue;1814		}1815		stride = min_t(unsigned int, COMPACT_CLUSTER_MAX, stride << 1);1816	}1817 1818	/*1819	 * Record where the free scanner will restart next time. Either we1820	 * broke from the loop and set isolate_start_pfn based on the last1821	 * call to isolate_freepages_block(), or we met the migration scanner1822	 * and the loop terminated due to isolate_start_pfn < low_pfn1823	 */1824	cc->free_pfn = isolate_start_pfn;1825}1826 1827/*1828 * This is a migrate-callback that "allocates" freepages by taking pages1829 * from the isolated freelists in the block we are migrating to.1830 */1831static HWJS_SUSPENDS struct folio *compaction_alloc_noprof(struct folio *src, unsigned long data)1832{1833	struct compact_control *cc = (struct compact_control *)data;1834	struct folio *dst;1835	int order = folio_order(src);1836	bool has_isolated_pages = false;1837	int start_order;1838	struct page *freepage;1839	unsigned long size;1840 1841again:1842	for (start_order = order; start_order < NR_PAGE_ORDERS; start_order++)1843		if (!list_empty(&cc->freepages[start_order]))1844			break;1845 1846	/* no free pages in the list */1847	if (start_order == NR_PAGE_ORDERS) {1848		if (has_isolated_pages)1849			return NULL;1850		isolate_freepages(cc);1851		has_isolated_pages = true;1852		goto again;1853	}1854 1855	freepage = list_first_entry(&cc->freepages[start_order], struct page,1856				lru);1857	size = 1 << start_order;1858 1859	list_del(&freepage->lru);1860 1861	while (start_order > order) {1862		start_order--;1863		size >>= 1;1864 1865		list_add(&freepage[size].lru, &cc->freepages[start_order]);1866		set_page_private(&freepage[size], start_order);1867	}1868	dst = (struct folio *)freepage;1869 1870	post_alloc_hook(&dst->page, order, __GFP_MOVABLE);1871	if (order)1872		prep_compound_page(&dst->page, order);1873	cc->nr_freepages -= 1 << order;1874	cc->nr_migratepages -= 1 << order;1875	return page_rmappable_folio(&dst->page);1876}1877 1878static HWJS_SUSPENDS struct folio *compaction_alloc(struct folio *src, unsigned long data)1879{1880	return alloc_hooks(compaction_alloc_noprof(src, data));1881}1882 1883/*1884 * This is a migrate-callback that "frees" freepages back to the isolated1885 * freelist.  All pages on the freelist are from the same zone, so there is no1886 * special handling needed for NUMA.1887 */1888static void compaction_free(struct folio *dst, unsigned long data)1889{1890	struct compact_control *cc = (struct compact_control *)data;1891	int order = folio_order(dst);1892	struct page *page = &dst->page;1893 1894	if (folio_put_testzero(dst)) {1895		free_pages_prepare(page, order);1896		list_add(&dst->lru, &cc->freepages[order]);1897		cc->nr_freepages += 1 << order;1898	}1899	cc->nr_migratepages += 1 << order;1900	/*1901	 * someone else has referenced the page, we cannot take it back to our1902	 * free list.1903	 */1904}1905 1906/* possible outcome of isolate_migratepages */1907typedef enum {1908	ISOLATE_ABORT,		/* Abort compaction now */1909	ISOLATE_NONE,		/* No pages isolated, continue scanning */1910	ISOLATE_SUCCESS,	/* Pages isolated, migrate */1911} isolate_migrate_t;1912 1913/*1914 * Allow userspace to control policy on scanning the unevictable LRU for1915 * compactable pages.1916 */1917static int sysctl_compact_unevictable_allowed __read_mostly = CONFIG_COMPACT_UNEVICTABLE_DEFAULT;1918/*1919 * Tunable for proactive compaction. It determines how1920 * aggressively the kernel should compact memory in the1921 * background. It takes values in the range [0, 100].1922 */1923static unsigned int __read_mostly sysctl_compaction_proactiveness = 20;1924static int sysctl_extfrag_threshold = 500;1925static int __read_mostly sysctl_compact_memory;1926 1927static inline void1928update_fast_start_pfn(struct compact_control *cc, unsigned long pfn)1929{1930	if (cc->fast_start_pfn == ULONG_MAX)1931		return;1932 1933	if (!cc->fast_start_pfn)1934		cc->fast_start_pfn = pfn;1935 1936	cc->fast_start_pfn = min(cc->fast_start_pfn, pfn);1937}1938 1939static inline unsigned long1940reinit_migrate_pfn(struct compact_control *cc)1941{1942	if (!cc->fast_start_pfn || cc->fast_start_pfn == ULONG_MAX)1943		return cc->migrate_pfn;1944 1945	cc->migrate_pfn = cc->fast_start_pfn;1946	cc->fast_start_pfn = ULONG_MAX;1947 1948	return cc->migrate_pfn;1949}1950 1951/*1952 * Briefly search the free lists for a migration source that already has1953 * some free pages to reduce the number of pages that need migration1954 * before a pageblock is free.1955 */1956static unsigned long fast_find_migrateblock(struct compact_control *cc)1957{1958	unsigned int limit = freelist_scan_limit(cc);1959	unsigned int nr_scanned = 0;1960	unsigned long distance;1961	unsigned long pfn = cc->migrate_pfn;1962	unsigned long high_pfn;1963	int order;1964	bool found_block = false;1965 1966	/* Skip hints are relied on to avoid repeats on the fast search */1967	if (cc->ignore_skip_hint)1968		return pfn;1969 1970	/*1971	 * If the pageblock should be finished then do not select a different1972	 * pageblock.1973	 */1974	if (cc->finish_pageblock)1975		return pfn;1976 1977	/*1978	 * If the migrate_pfn is not at the start of a zone or the start1979	 * of a pageblock then assume this is a continuation of a previous1980	 * scan restarted due to COMPACT_CLUSTER_MAX.1981	 */1982	if (pfn != cc->zone->zone_start_pfn && pfn != pageblock_start_pfn(pfn))1983		return pfn;1984 1985	/*1986	 * For smaller orders, just linearly scan as the number of pages1987	 * to migrate should be relatively small and does not necessarily1988	 * justify freeing up a large block for a small allocation.1989	 */1990	if (cc->order <= PAGE_ALLOC_COSTLY_ORDER)1991		return pfn;1992 1993	/*1994	 * Only allow kcompactd and direct requests for movable pages to1995	 * quickly clear out a MOVABLE pageblock for allocation. This1996	 * reduces the risk that a large movable pageblock is freed for1997	 * an unmovable/reclaimable small allocation.1998	 */1999	if (cc->direct_compaction && cc->migratetype != MIGRATE_MOVABLE)2000		return pfn;2001 2002	/*2003	 * When starting the migration scanner, pick any pageblock within the2004	 * first half of the search space. Otherwise try and pick a pageblock2005	 * within the first eighth to reduce the chances that a migration2006	 * target later becomes a source.2007	 */2008	distance = (cc->free_pfn - cc->migrate_pfn) >> 1;2009	if (cc->migrate_pfn != cc->zone->zone_start_pfn)2010		distance >>= 2;2011	high_pfn = pageblock_start_pfn(cc->migrate_pfn + distance);2012 2013	for (order = cc->order - 1;2014	     order >= PAGE_ALLOC_COSTLY_ORDER && !found_block && nr_scanned < limit;2015	     order--) {2016		struct free_area *area = &cc->zone->free_area[order];2017		struct list_head *freelist;2018		unsigned long flags;2019		struct page *freepage;2020 2021		if (!area->nr_free)2022			continue;2023 2024		spin_lock_irqsave(&cc->zone->lock, flags);2025		freelist = &area->free_list[MIGRATE_MOVABLE];2026		list_for_each_entry(freepage, freelist, buddy_list) {2027			unsigned long free_pfn;2028 2029			if (nr_scanned++ >= limit) {2030				move_freelist_tail(freelist, freepage);2031				break;2032			}2033 2034			free_pfn = page_to_pfn(freepage);2035			if (free_pfn < high_pfn) {2036				/*2037				 * Avoid if skipped recently. Ideally it would2038				 * move to the tail but even safe iteration of2039				 * the list assumes an entry is deleted, not2040				 * reordered.2041				 */2042				if (get_pageblock_skip(freepage))2043					continue;2044 2045				/* Reorder to so a future search skips recent pages */2046				move_freelist_tail(freelist, freepage);2047 2048				update_fast_start_pfn(cc, free_pfn);2049				pfn = pageblock_start_pfn(free_pfn);2050				if (pfn < cc->zone->zone_start_pfn)2051					pfn = cc->zone->zone_start_pfn;2052				cc->fast_search_fail = 0;2053				found_block = true;2054				break;2055			}2056		}2057		spin_unlock_irqrestore(&cc->zone->lock, flags);2058	}2059 2060	cc->total_migrate_scanned += nr_scanned;2061 2062	/*2063	 * If fast scanning failed then use a cached entry for a page block2064	 * that had free pages as the basis for starting a linear scan.2065	 */2066	if (!found_block) {2067		cc->fast_search_fail++;2068		pfn = reinit_migrate_pfn(cc);2069	}2070	return pfn;2071}2072 2073/*2074 * Isolate all pages that can be migrated from the first suitable block,2075 * starting at the block pointed to by the migrate scanner pfn within2076 * compact_control.2077 */2078static HWJS_SUSPENDS isolate_migrate_t isolate_migratepages(struct compact_control *cc)2079{2080	unsigned long block_start_pfn;2081	unsigned long block_end_pfn;2082	unsigned long low_pfn;2083	struct page *page;2084	const isolate_mode_t isolate_mode =2085		(sysctl_compact_unevictable_allowed ? ISOLATE_UNEVICTABLE : 0) |2086		(cc->mode != MIGRATE_SYNC ? ISOLATE_ASYNC_MIGRATE : 0);2087	bool fast_find_block;2088 2089	/*2090	 * Start at where we last stopped, or beginning of the zone as2091	 * initialized by compact_zone(). The first failure will use2092	 * the lowest PFN as the starting point for linear scanning.2093	 */2094	low_pfn = fast_find_migrateblock(cc);2095	block_start_pfn = pageblock_start_pfn(low_pfn);2096	if (block_start_pfn < cc->zone->zone_start_pfn)2097		block_start_pfn = cc->zone->zone_start_pfn;2098 2099	/*2100	 * fast_find_migrateblock() has already ensured the pageblock is not2101	 * set with a skipped flag, so to avoid the isolation_suitable check2102	 * below again, check whether the fast search was successful.2103	 */2104	fast_find_block = low_pfn != cc->migrate_pfn && !cc->fast_search_fail;2105 2106	/* Only scan within a pageblock boundary */2107	block_end_pfn = pageblock_end_pfn(low_pfn);2108 2109	/*2110	 * Iterate over whole pageblocks until we find the first suitable.2111	 * Do not cross the free scanner.2112	 */2113	for (; block_end_pfn <= cc->free_pfn;2114			fast_find_block = false,2115			cc->migrate_pfn = low_pfn = block_end_pfn,2116			block_start_pfn = block_end_pfn,2117			block_end_pfn += pageblock_nr_pages) {2118 2119		/*2120		 * This can potentially iterate a massively long zone with2121		 * many pageblocks unsuitable, so periodically check if we2122		 * need to schedule.2123		 */2124		if (!(low_pfn % (COMPACT_CLUSTER_MAX * pageblock_nr_pages)))2125			cond_resched();2126 2127		page = pageblock_pfn_to_page(block_start_pfn,2128						block_end_pfn, cc->zone);2129		if (!page) {2130			unsigned long next_pfn;2131 2132			next_pfn = skip_offline_sections(block_start_pfn);2133			if (next_pfn)2134				block_end_pfn = min(next_pfn, cc->free_pfn);2135			continue;2136		}2137 2138		/*2139		 * If isolation recently failed, do not retry. Only check the2140		 * pageblock once. COMPACT_CLUSTER_MAX causes a pageblock2141		 * to be visited multiple times. Assume skip was checked2142		 * before making it "skip" so other compaction instances do2143		 * not scan the same block.2144		 */2145		if ((pageblock_aligned(low_pfn) ||2146		     low_pfn == cc->zone->zone_start_pfn) &&2147		    !fast_find_block && !isolation_suitable(cc, page))2148			continue;2149 2150		/*2151		 * For async direct compaction, only scan the pageblocks of the2152		 * same migratetype without huge pages. Async direct compaction2153		 * is optimistic to see if the minimum amount of work satisfies2154		 * the allocation. The cached PFN is updated as it's possible2155		 * that all remaining blocks between source and target are2156		 * unsuitable and the compaction scanners fail to meet.2157		 */2158		if (!suitable_migration_source(cc, page)) {2159			update_cached_migrate(cc, block_end_pfn);2160			continue;2161		}2162 2163		/* Perform the isolation */2164		if (isolate_migratepages_block(cc, low_pfn, block_end_pfn,2165						isolate_mode))2166			return ISOLATE_ABORT;2167 2168		/*2169		 * Either we isolated something and proceed with migration. Or2170		 * we failed and compact_zone should decide if we should2171		 * continue or not.2172		 */2173		break;2174	}2175 2176	return cc->nr_migratepages ? ISOLATE_SUCCESS : ISOLATE_NONE;2177}2178 2179/*2180 * Determine whether kswapd is (or recently was!) running on this node.2181 *2182 * pgdat_kswapd_lock() pins pgdat->kswapd, so a concurrent kswapd_stop() can't2183 * zero it.2184 */2185static bool kswapd_is_running(pg_data_t *pgdat)2186{2187	bool running;2188 2189	pgdat_kswapd_lock(pgdat);2190	running = pgdat->kswapd && task_is_running(pgdat->kswapd);2191	pgdat_kswapd_unlock(pgdat);2192 2193	return running;2194}2195 2196/*2197 * A zone's fragmentation score is the external fragmentation wrt to the2198 * COMPACTION_HPAGE_ORDER. It returns a value in the range [0, 100].2199 */2200static unsigned int fragmentation_score_zone(struct zone *zone)2201{2202	return extfrag_for_order(zone, COMPACTION_HPAGE_ORDER);2203}2204 2205/*2206 * A weighted zone's fragmentation score is the external fragmentation2207 * wrt to the COMPACTION_HPAGE_ORDER scaled by the zone's size. It2208 * returns a value in the range [0, 100].2209 *2210 * The scaling factor ensures that proactive compaction focuses on larger2211 * zones like ZONE_NORMAL, rather than smaller, specialized zones like2212 * ZONE_DMA32. For smaller zones, the score value remains close to zero,2213 * and thus never exceeds the high threshold for proactive compaction.2214 */2215static unsigned int fragmentation_score_zone_weighted(struct zone *zone)2216{2217	unsigned long score;2218 2219	score = zone->present_pages * fragmentation_score_zone(zone);2220	return div64_ul(score, zone->zone_pgdat->node_present_pages + 1);2221}2222 2223/*2224 * The per-node proactive (background) compaction process is started by its2225 * corresponding kcompactd thread when the node's fragmentation score2226 * exceeds the high threshold. The compaction process remains active till2227 * the node's score falls below the low threshold, or one of the back-off2228 * conditions is met.2229 */2230static unsigned int fragmentation_score_node(pg_data_t *pgdat)2231{2232	unsigned int score = 0;2233	int zoneid;2234 2235	for (zoneid = 0; zoneid < MAX_NR_ZONES; zoneid++) {2236		struct zone *zone;2237 2238		zone = &pgdat->node_zones[zoneid];2239		if (!populated_zone(zone))2240			continue;2241		score += fragmentation_score_zone_weighted(zone);2242	}2243 2244	return score;2245}2246 2247static unsigned int fragmentation_score_wmark(bool low)2248{2249	unsigned int wmark_low;2250 2251	/*2252	 * Cap the low watermark to avoid excessive compaction2253	 * activity in case a user sets the proactiveness tunable2254	 * close to 100 (maximum).2255	 */2256	wmark_low = max(100U - sysctl_compaction_proactiveness, 5U);2257	return low ? wmark_low : min(wmark_low + 10, 100U);2258}2259 2260static bool should_proactive_compact_node(pg_data_t *pgdat)2261{2262	int wmark_high;2263 2264	if (!sysctl_compaction_proactiveness || kswapd_is_running(pgdat))2265		return false;2266 2267	wmark_high = fragmentation_score_wmark(false);2268	return fragmentation_score_node(pgdat) > wmark_high;2269}2270 2271static enum compact_result __compact_finished(struct compact_control *cc)2272{2273	unsigned int order;2274	const int migratetype = cc->migratetype;2275	int ret;2276 2277	/* Compaction run completes if the migrate and free scanner meet */2278	if (compact_scanners_met(cc)) {2279		/* Let the next compaction start anew. */2280		reset_cached_positions(cc->zone);2281 2282		/*2283		 * Mark that the PG_migrate_skip information should be cleared2284		 * by kswapd when it goes to sleep. kcompactd does not set the2285		 * flag itself as the decision to be clear should be directly2286		 * based on an allocation request.2287		 */2288		if (cc->direct_compaction)2289			cc->zone->compact_blockskip_flush = true;2290 2291		if (cc->whole_zone)2292			return COMPACT_COMPLETE;2293		else2294			return COMPACT_PARTIAL_SKIPPED;2295	}2296 2297	if (cc->proactive_compaction) {2298		int score, wmark_low;2299		pg_data_t *pgdat;2300 2301		pgdat = cc->zone->zone_pgdat;2302		if (kswapd_is_running(pgdat))2303			return COMPACT_PARTIAL_SKIPPED;2304 2305		score = fragmentation_score_zone(cc->zone);2306		wmark_low = fragmentation_score_wmark(true);2307 2308		if (score > wmark_low)2309			ret = COMPACT_CONTINUE;2310		else2311			ret = COMPACT_SUCCESS;2312 2313		goto out;2314	}2315 2316	if (is_via_compact_memory(cc->order))2317		return COMPACT_CONTINUE;2318 2319	/*2320	 * Always finish scanning a pageblock to reduce the possibility of2321	 * fallbacks in the future. This is particularly important when2322	 * migration source is unmovable/reclaimable but it's not worth2323	 * special casing.2324	 */2325	if (!pageblock_aligned(cc->migrate_pfn))2326		return COMPACT_CONTINUE;2327 2328	/* Direct compactor: Is a suitable page free? */2329	ret = COMPACT_NO_SUITABLE_PAGE;2330	for (order = cc->order; order < NR_PAGE_ORDERS; order++) {2331		struct free_area *area = &cc->zone->free_area[order];2332		bool can_steal;2333 2334		/* Job done if page is free of the right migratetype */2335		if (!free_area_empty(area, migratetype))2336			return COMPACT_SUCCESS;2337 2338#ifdef CONFIG_CMA2339		/* MIGRATE_MOVABLE can fallback on MIGRATE_CMA */2340		if (migratetype == MIGRATE_MOVABLE &&2341			!free_area_empty(area, MIGRATE_CMA))2342			return COMPACT_SUCCESS;2343#endif2344		/*2345		 * Job done if allocation would steal freepages from2346		 * other migratetype buddy lists.2347		 */2348		if (find_suitable_fallback(area, order, migratetype,2349						true, &can_steal) != -1)2350			/*2351			 * Movable pages are OK in any pageblock. If we are2352			 * stealing for a non-movable allocation, make sure2353			 * we finish compacting the current pageblock first2354			 * (which is assured by the above migrate_pfn align2355			 * check) so it is as free as possible and we won't2356			 * have to steal another one soon.2357			 */2358			return COMPACT_SUCCESS;2359	}2360 2361out:2362	if (cc->contended || fatal_signal_pending(current))2363		ret = COMPACT_CONTENDED;2364 2365	return ret;2366}2367 2368static enum compact_result compact_finished(struct compact_control *cc)2369{2370	int ret;2371 2372	ret = __compact_finished(cc);2373	trace_mm_compaction_finished(cc->zone, cc->order, ret);2374	if (ret == COMPACT_NO_SUITABLE_PAGE)2375		ret = COMPACT_CONTINUE;2376 2377	return ret;2378}2379 2380static bool __compaction_suitable(struct zone *zone, int order,2381				  int highest_zoneidx,2382				  unsigned long wmark_target)2383{2384	unsigned long watermark;2385	/*2386	 * Watermarks for order-0 must be met for compaction to be able to2387	 * isolate free pages for migration targets. This means that the2388	 * watermark and alloc_flags have to match, or be more pessimistic than2389	 * the check in __isolate_free_page(). We don't use the direct2390	 * compactor's alloc_flags, as they are not relevant for freepage2391	 * isolation. We however do use the direct compactor's highest_zoneidx2392	 * to skip over zones where lowmem reserves would prevent allocation2393	 * even if compaction succeeds.2394	 * For costly orders, we require low watermark instead of min for2395	 * compaction to proceed to increase its chances.2396	 * ALLOC_CMA is used, as pages in CMA pageblocks are considered2397	 * suitable migration targets2398	 */2399	watermark = (order > PAGE_ALLOC_COSTLY_ORDER) ?2400				low_wmark_pages(zone) : min_wmark_pages(zone);2401	watermark += compact_gap(order);2402	return __zone_watermark_ok(zone, 0, watermark, highest_zoneidx,2403				   ALLOC_CMA, wmark_target);2404}2405 2406/*2407 * compaction_suitable: Is this suitable to run compaction on this zone now?2408 */2409bool compaction_suitable(struct zone *zone, int order, int highest_zoneidx)2410{2411	enum compact_result compact_result;2412	bool suitable;2413 2414	suitable = __compaction_suitable(zone, order, highest_zoneidx,2415					 zone_page_state(zone, NR_FREE_PAGES));2416	/*2417	 * fragmentation index determines if allocation failures are due to2418	 * low memory or external fragmentation2419	 *2420	 * index of -1000 would imply allocations might succeed depending on2421	 * watermarks, but we already failed the high-order watermark check2422	 * index towards 0 implies failure is due to lack of memory2423	 * index towards 1000 implies failure is due to fragmentation2424	 *2425	 * Only compact if a failure would be due to fragmentation. Also2426	 * ignore fragindex for non-costly orders where the alternative to2427	 * a successful reclaim/compaction is OOM. Fragindex and the2428	 * vm.extfrag_threshold sysctl is meant as a heuristic to prevent2429	 * excessive compaction for costly orders, but it should not be at the2430	 * expense of system stability.2431	 */2432	if (suitable) {2433		compact_result = COMPACT_CONTINUE;2434		if (order > PAGE_ALLOC_COSTLY_ORDER) {2435			int fragindex = fragmentation_index(zone, order);2436 2437			if (fragindex >= 0 &&2438			    fragindex <= sysctl_extfrag_threshold) {2439				suitable = false;2440				compact_result = COMPACT_NOT_SUITABLE_ZONE;2441			}2442		}2443	} else {2444		compact_result = COMPACT_SKIPPED;2445	}2446 2447	trace_mm_compaction_suitable(zone, order, compact_result);2448 2449	return suitable;2450}2451 2452bool compaction_zonelist_suitable(struct alloc_context *ac, int order,2453		int alloc_flags)2454{2455	struct zone *zone;2456	struct zoneref *z;2457 2458	/*2459	 * Make sure at least one zone would pass __compaction_suitable if we continue2460	 * retrying the reclaim.2461	 */2462	for_each_zone_zonelist_nodemask(zone, z, ac->zonelist,2463				ac->highest_zoneidx, ac->nodemask) {2464		unsigned long available;2465 2466		/*2467		 * Do not consider all the reclaimable memory because we do not2468		 * want to trash just for a single high order allocation which2469		 * is even not guaranteed to appear even if __compaction_suitable2470		 * is happy about the watermark check.2471		 */2472		available = zone_reclaimable_pages(zone) / order;2473		available += zone_page_state_snapshot(zone, NR_FREE_PAGES);2474		if (__compaction_suitable(zone, order, ac->highest_zoneidx,2475					  available))2476			return true;2477	}2478 2479	return false;2480}2481 2482/*2483 * Should we do compaction for target allocation order.2484 * Return COMPACT_SUCCESS if allocation for target order can be already2485 * satisfied2486 * Return COMPACT_SKIPPED if compaction for target order is likely to fail2487 * Return COMPACT_CONTINUE if compaction for target order should be ran2488 */2489static enum compact_result2490compaction_suit_allocation_order(struct zone *zone, unsigned int order,2491				 int highest_zoneidx, unsigned int alloc_flags)2492{2493	unsigned long watermark;2494 2495	watermark = wmark_pages(zone, alloc_flags & ALLOC_WMARK_MASK);2496	if (zone_watermark_ok(zone, order, watermark, highest_zoneidx,2497			      alloc_flags))2498		return COMPACT_SUCCESS;2499 2500	if (!compaction_suitable(zone, order, highest_zoneidx))2501		return COMPACT_SKIPPED;2502 2503	return COMPACT_CONTINUE;2504}2505 2506static HWJS_SUSPENDS enum compact_result2507compact_zone(struct compact_control *cc, struct capture_control *capc)2508{2509	enum compact_result ret;2510	unsigned long start_pfn = cc->zone->zone_start_pfn;2511	unsigned long end_pfn = zone_end_pfn(cc->zone);2512	unsigned long last_migrated_pfn;2513	const bool sync = cc->mode != MIGRATE_ASYNC;2514	bool update_cached;2515	unsigned int nr_succeeded = 0, nr_migratepages;2516	int order;2517 2518	/*2519	 * These counters track activities during zone compaction.  Initialize2520	 * them before compacting a new zone.2521	 */2522	cc->total_migrate_scanned = 0;2523	cc->total_free_scanned = 0;2524	cc->nr_migratepages = 0;2525	cc->nr_freepages = 0;2526	for (order = 0; order < NR_PAGE_ORDERS; order++)2527		INIT_LIST_HEAD(&cc->freepages[order]);2528	INIT_LIST_HEAD(&cc->migratepages);2529 2530	cc->migratetype = gfp_migratetype(cc->gfp_mask);2531 2532	if (!is_via_compact_memory(cc->order)) {2533		ret = compaction_suit_allocation_order(cc->zone, cc->order,2534						       cc->highest_zoneidx,2535						       cc->alloc_flags);2536		if (ret != COMPACT_CONTINUE)2537			return ret;2538	}2539 2540	/*2541	 * Clear pageblock skip if there were failures recently and compaction2542	 * is about to be retried after being deferred.2543	 */2544	if (compaction_restarting(cc->zone, cc->order))2545		__reset_isolation_suitable(cc->zone);2546 2547	/*2548	 * Setup to move all movable pages to the end of the zone. Used cached2549	 * information on where the scanners should start (unless we explicitly2550	 * want to compact the whole zone), but check that it is initialised2551	 * by ensuring the values are within zone boundaries.2552	 */2553	cc->fast_start_pfn = 0;2554	if (cc->whole_zone) {2555		cc->migrate_pfn = start_pfn;2556		cc->free_pfn = pageblock_start_pfn(end_pfn - 1);2557	} else {2558		cc->migrate_pfn = cc->zone->compact_cached_migrate_pfn[sync];2559		cc->free_pfn = cc->zone->compact_cached_free_pfn;2560		if (cc->free_pfn < start_pfn || cc->free_pfn >= end_pfn) {2561			cc->free_pfn = pageblock_start_pfn(end_pfn - 1);2562			cc->zone->compact_cached_free_pfn = cc->free_pfn;2563		}2564		if (cc->migrate_pfn < start_pfn || cc->migrate_pfn >= end_pfn) {2565			cc->migrate_pfn = start_pfn;2566			cc->zone->compact_cached_migrate_pfn[0] = cc->migrate_pfn;2567			cc->zone->compact_cached_migrate_pfn[1] = cc->migrate_pfn;2568		}2569 2570		if (cc->migrate_pfn <= cc->zone->compact_init_migrate_pfn)2571			cc->whole_zone = true;2572	}2573 2574	last_migrated_pfn = 0;2575 2576	/*2577	 * Migrate has separate cached PFNs for ASYNC and SYNC* migration on2578	 * the basis that some migrations will fail in ASYNC mode. However,2579	 * if the cached PFNs match and pageblocks are skipped due to having2580	 * no isolation candidates, then the sync state does not matter.2581	 * Until a pageblock with isolation candidates is found, keep the2582	 * cached PFNs in sync to avoid revisiting the same blocks.2583	 */2584	update_cached = !sync &&2585		cc->zone->compact_cached_migrate_pfn[0] == cc->zone->compact_cached_migrate_pfn[1];2586 2587	trace_mm_compaction_begin(cc, start_pfn, end_pfn, sync);2588 2589	/* lru_add_drain_all could be expensive with involving other CPUs */2590	lru_add_drain();2591 2592	while ((ret = compact_finished(cc)) == COMPACT_CONTINUE) {2593		int err;2594		unsigned long iteration_start_pfn = cc->migrate_pfn;2595 2596		/*2597		 * Avoid multiple rescans of the same pageblock which can2598		 * happen if a page cannot be isolated (dirty/writeback in2599		 * async mode) or if the migrated pages are being allocated2600		 * before the pageblock is cleared.  The first rescan will2601		 * capture the entire pageblock for migration. If it fails,2602		 * it'll be marked skip and scanning will proceed as normal.2603		 */2604		cc->finish_pageblock = false;2605		if (pageblock_start_pfn(last_migrated_pfn) ==2606		    pageblock_start_pfn(iteration_start_pfn)) {2607			cc->finish_pageblock = true;2608		}2609 2610rescan:2611		switch (isolate_migratepages(cc)) {2612		case ISOLATE_ABORT:2613			ret = COMPACT_CONTENDED;2614			putback_movable_pages(&cc->migratepages);2615			cc->nr_migratepages = 0;2616			goto out;2617		case ISOLATE_NONE:2618			if (update_cached) {2619				cc->zone->compact_cached_migrate_pfn[1] =2620					cc->zone->compact_cached_migrate_pfn[0];2621			}2622 2623			/*2624			 * We haven't isolated and migrated anything, but2625			 * there might still be unflushed migrations from2626			 * previous cc->order aligned block.2627			 */2628			goto check_drain;2629		case ISOLATE_SUCCESS:2630			update_cached = false;2631			last_migrated_pfn = max(cc->zone->zone_start_pfn,2632				pageblock_start_pfn(cc->migrate_pfn - 1));2633		}2634 2635		/*2636		 * Record the number of pages to migrate since the2637		 * compaction_alloc/free() will update cc->nr_migratepages2638		 * properly.2639		 */2640		nr_migratepages = cc->nr_migratepages;2641		err = migrate_pages(&cc->migratepages, compaction_alloc,2642				compaction_free, (unsigned long)cc, cc->mode,2643				MR_COMPACTION, &nr_succeeded);2644 2645		trace_mm_compaction_migratepages(nr_migratepages, nr_succeeded);2646 2647		/* All pages were either migrated or will be released */2648		cc->nr_migratepages = 0;2649		if (err) {2650			putback_movable_pages(&cc->migratepages);2651			/*2652			 * migrate_pages() may return -ENOMEM when scanners meet2653			 * and we want compact_finished() to detect it2654			 */2655			if (err == -ENOMEM && !compact_scanners_met(cc)) {2656				ret = COMPACT_CONTENDED;2657				goto out;2658			}2659			/*2660			 * If an ASYNC or SYNC_LIGHT fails to migrate a page2661			 * within the pageblock_order-aligned block and2662			 * fast_find_migrateblock may be used then scan the2663			 * remainder of the pageblock. This will mark the2664			 * pageblock "skip" to avoid rescanning in the near2665			 * future. This will isolate more pages than necessary2666			 * for the request but avoid loops due to2667			 * fast_find_migrateblock revisiting blocks that were2668			 * recently partially scanned.2669			 */2670			if (!pageblock_aligned(cc->migrate_pfn) &&2671			    !cc->ignore_skip_hint && !cc->finish_pageblock &&2672			    (cc->mode < MIGRATE_SYNC)) {2673				cc->finish_pageblock = true;2674 2675				/*2676				 * Draining pcplists does not help THP if2677				 * any page failed to migrate. Even after2678				 * drain, the pageblock will not be free.2679				 */2680				if (cc->order == COMPACTION_HPAGE_ORDER)2681					last_migrated_pfn = 0;2682 2683				goto rescan;2684			}2685		}2686 2687		/* Stop if a page has been captured */2688		if (capc && capc->page) {2689			ret = COMPACT_SUCCESS;2690			break;2691		}2692 2693check_drain:2694		/*2695		 * Has the migration scanner moved away from the previous2696		 * cc->order aligned block where we migrated from? If yes,2697		 * flush the pages that were freed, so that they can merge and2698		 * compact_finished() can detect immediately if allocation2699		 * would succeed.2700		 */2701		if (cc->order > 0 && last_migrated_pfn) {2702			unsigned long current_block_start =2703				block_start_pfn(cc->migrate_pfn, cc->order);2704 2705			if (last_migrated_pfn < current_block_start) {2706				lru_add_drain_cpu_zone(cc->zone);2707				/* No more flushing until we migrate again */2708				last_migrated_pfn = 0;2709			}2710		}2711	}2712 2713out:2714	/*2715	 * Release free pages and update where the free scanner should restart,2716	 * so we don't leave any returned pages behind in the next attempt.2717	 */2718	if (cc->nr_freepages > 0) {2719		unsigned long free_pfn = release_free_list(cc->freepages);2720 2721		cc->nr_freepages = 0;2722		VM_BUG_ON(free_pfn == 0);2723		/* The cached pfn is always the first in a pageblock */2724		free_pfn = pageblock_start_pfn(free_pfn);2725		/*2726		 * Only go back, not forward. The cached pfn might have been2727		 * already reset to zone end in compact_finished()2728		 */2729		if (free_pfn > cc->zone->compact_cached_free_pfn)2730			cc->zone->compact_cached_free_pfn = free_pfn;2731	}2732 2733	count_compact_events(COMPACTMIGRATE_SCANNED, cc->total_migrate_scanned);2734	count_compact_events(COMPACTFREE_SCANNED, cc->total_free_scanned);2735 2736	trace_mm_compaction_end(cc, start_pfn, end_pfn, sync, ret);2737 2738	VM_BUG_ON(!list_empty(&cc->migratepages));2739 2740	return ret;2741}2742 2743static HWJS_SUSPENDS enum compact_result compact_zone_order(struct zone *zone, int order,2744		gfp_t gfp_mask, enum compact_priority prio,2745		unsigned int alloc_flags, int highest_zoneidx,2746		struct page **capture)2747{2748	enum compact_result ret;2749	struct compact_control cc = {2750		.order = order,2751		.search_order = order,2752		.gfp_mask = gfp_mask,2753		.zone = zone,2754		.mode = (prio == COMPACT_PRIO_ASYNC) ?2755					MIGRATE_ASYNC :	MIGRATE_SYNC_LIGHT,2756		.alloc_flags = alloc_flags,2757		.highest_zoneidx = highest_zoneidx,2758		.direct_compaction = true,2759		.whole_zone = (prio == MIN_COMPACT_PRIORITY),2760		.ignore_skip_hint = (prio == MIN_COMPACT_PRIORITY),2761		.ignore_block_suitable = (prio == MIN_COMPACT_PRIORITY)2762	};2763	struct capture_control capc = {2764		.cc = &cc,2765		.page = NULL,2766	};2767 2768	/*2769	 * Make sure the structs are really initialized before we expose the2770	 * capture control, in case we are interrupted and the interrupt handler2771	 * frees a page.2772	 */2773	barrier();2774	WRITE_ONCE(current->capture_control, &capc);2775 2776	ret = compact_zone(&cc, &capc);2777 2778	/*2779	 * Make sure we hide capture control first before we read the captured2780	 * page pointer, otherwise an interrupt could free and capture a page2781	 * and we would leak it.2782	 */2783	WRITE_ONCE(current->capture_control, NULL);2784	*capture = READ_ONCE(capc.page);2785	/*2786	 * Technically, it is also possible that compaction is skipped but2787	 * the page is still captured out of luck(IRQ came and freed the page).2788	 * Returning COMPACT_SUCCESS in such cases helps in properly accounting2789	 * the COMPACT[STALL|FAIL] when compaction is skipped.2790	 */2791	if (*capture)2792		ret = COMPACT_SUCCESS;2793 2794	return ret;2795}2796 2797/**2798 * try_to_compact_pages - Direct compact to satisfy a high-order allocation2799 * @gfp_mask: The GFP mask of the current allocation2800 * @order: The order of the current allocation2801 * @alloc_flags: The allocation flags of the current allocation2802 * @ac: The context of current allocation2803 * @prio: Determines how hard direct compaction should try to succeed2804 * @capture: Pointer to free page created by compaction will be stored here2805 *2806 * This is the main entry point for direct page compaction.2807 */2808enum compact_result try_to_compact_pages(gfp_t gfp_mask, unsigned int order,2809		unsigned int alloc_flags, const struct alloc_context *ac,2810		enum compact_priority prio, struct page **capture)2811{2812	struct zoneref *z;2813	struct zone *zone;2814	enum compact_result rc = COMPACT_SKIPPED;2815 2816	if (!gfp_compaction_allowed(gfp_mask))2817		return COMPACT_SKIPPED;2818 2819	trace_mm_compaction_try_to_compact_pages(order, gfp_mask, prio);2820 2821	/* Compact each zone in the list */2822	for_each_zone_zonelist_nodemask(zone, z, ac->zonelist,2823					ac->highest_zoneidx, ac->nodemask) {2824		enum compact_result status;2825 2826		if (cpusets_enabled() &&2827			(alloc_flags & ALLOC_CPUSET) &&2828			!__cpuset_zone_allowed(zone, gfp_mask))2829				continue;2830 2831		if (prio > MIN_COMPACT_PRIORITY2832					&& compaction_deferred(zone, order)) {2833			rc = max_t(enum compact_result, COMPACT_DEFERRED, rc);2834			continue;2835		}2836 2837		status = compact_zone_order(zone, order, gfp_mask, prio,2838				alloc_flags, ac->highest_zoneidx, capture);2839		rc = max(status, rc);2840 2841		/* The allocation should succeed, stop compacting */2842		if (status == COMPACT_SUCCESS) {2843			/*2844			 * We think the allocation will succeed in this zone,2845			 * but it is not certain, hence the false. The caller2846			 * will repeat this with true if allocation indeed2847			 * succeeds in this zone.2848			 */2849			compaction_defer_reset(zone, order, false);2850 2851			break;2852		}2853 2854		if (prio != COMPACT_PRIO_ASYNC && (status == COMPACT_COMPLETE ||2855					status == COMPACT_PARTIAL_SKIPPED))2856			/*2857			 * We think that allocation won't succeed in this zone2858			 * so we defer compaction there. If it ends up2859			 * succeeding after all, it will be reset.2860			 */2861			defer_compaction(zone, order);2862 2863		/*2864		 * We might have stopped compacting due to need_resched() in2865		 * async compaction, or due to a fatal signal detected. In that2866		 * case do not try further zones2867		 */2868		if ((prio == COMPACT_PRIO_ASYNC && need_resched())2869					|| fatal_signal_pending(current))2870			break;2871	}2872 2873	return rc;2874}2875 2876/*2877 * compact_node() - compact all zones within a node2878 * @pgdat: The node page data2879 * @proactive: Whether the compaction is proactive2880 *2881 * For proactive compaction, compact till each zone's fragmentation score2882 * reaches within proactive compaction thresholds (as determined by the2883 * proactiveness tunable), it is possible that the function returns before2884 * reaching score targets due to various back-off conditions, such as,2885 * contention on per-node or per-zone locks.2886 */2887static HWJS_SUSPENDS int compact_node(pg_data_t *pgdat, bool proactive)2888{2889	int zoneid;2890	struct zone *zone;2891	struct compact_control cc = {2892		.order = -1,2893		.mode = proactive ? MIGRATE_SYNC_LIGHT : MIGRATE_SYNC,2894		.ignore_skip_hint = true,2895		.whole_zone = true,2896		.gfp_mask = GFP_KERNEL,2897		.proactive_compaction = proactive,2898	};2899 2900	for (zoneid = 0; zoneid < MAX_NR_ZONES; zoneid++) {2901		zone = &pgdat->node_zones[zoneid];2902		if (!populated_zone(zone))2903			continue;2904 2905		if (fatal_signal_pending(current))2906			return -EINTR;2907 2908		cc.zone = zone;2909 2910		compact_zone(&cc, NULL);2911 2912		if (proactive) {2913			count_compact_events(KCOMPACTD_MIGRATE_SCANNED,2914					     cc.total_migrate_scanned);2915			count_compact_events(KCOMPACTD_FREE_SCANNED,2916					     cc.total_free_scanned);2917		}2918	}2919 2920	return 0;2921}2922 2923/* Compact all zones of all nodes in the system */2924static HWJS_SUSPENDS int compact_nodes(void)2925{2926	int ret, nid;2927 2928	/* Flush pending updates to the LRU lists */2929	lru_add_drain_all();2930 2931	for_each_online_node(nid) {2932		ret = compact_node(NODE_DATA(nid), false);2933		if (ret)2934			return ret;2935	}2936 2937	return 0;2938}2939 2940static HWJS_SUSPENDS int compaction_proactiveness_sysctl_handler(const struct ctl_table *table, int write,2941		void *buffer, size_t *length, loff_t *ppos)2942{2943	int rc, nid;2944 2945	rc = proc_dointvec_minmax(table, write, buffer, length, ppos);2946	if (rc)2947		return rc;2948 2949	if (write && sysctl_compaction_proactiveness) {2950		for_each_online_node(nid) {2951			pg_data_t *pgdat = NODE_DATA(nid);2952 2953			if (pgdat->proactive_compact_trigger)2954				continue;2955 2956			pgdat->proactive_compact_trigger = true;2957			trace_mm_compaction_wakeup_kcompactd(pgdat->node_id, -1,2958							     pgdat->nr_zones - 1);2959			wake_up_interruptible(&pgdat->kcompactd_wait);2960		}2961	}2962 2963	return 0;2964}2965 2966/*2967 * This is the entry point for compacting all nodes via2968 * /proc/sys/vm/compact_memory2969 */2970static HWJS_SUSPENDS int sysctl_compaction_handler(const struct ctl_table *table, int write,2971			void *buffer, size_t *length, loff_t *ppos)2972{2973	int ret;2974 2975	ret = proc_dointvec(table, write, buffer, length, ppos);2976	if (ret)2977		return ret;2978 2979	if (sysctl_compact_memory != 1)2980		return -EINVAL;2981 2982	if (write)2983		ret = compact_nodes();2984 2985	return ret;2986}2987 2988#if defined(CONFIG_SYSFS) && defined(CONFIG_NUMA)2989static ssize_t compact_store(struct device *dev,2990			     struct device_attribute *attr,2991			     const char *buf, size_t count)2992{2993	int nid = dev->id;2994 2995	if (nid >= 0 && nid < nr_node_ids && node_online(nid)) {2996		/* Flush pending updates to the LRU lists */2997		lru_add_drain_all();2998 2999		compact_node(NODE_DATA(nid), false);3000	}3001 3002	return count;3003}3004static DEVICE_ATTR_WO(compact);3005 3006int compaction_register_node(struct node *node)3007{3008	return device_create_file(&node->dev, &dev_attr_compact);3009}3010 3011void compaction_unregister_node(struct node *node)3012{3013	device_remove_file(&node->dev, &dev_attr_compact);3014}3015#endif /* CONFIG_SYSFS && CONFIG_NUMA */3016 3017static inline bool kcompactd_work_requested(pg_data_t *pgdat)3018{3019	return pgdat->kcompactd_max_order > 0 || kthread_should_stop() ||3020		pgdat->proactive_compact_trigger;3021}3022 3023static bool kcompactd_node_suitable(pg_data_t *pgdat)3024{3025	int zoneid;3026	struct zone *zone;3027	enum zone_type highest_zoneidx = pgdat->kcompactd_highest_zoneidx;3028	enum compact_result ret;3029 3030	for (zoneid = 0; zoneid <= highest_zoneidx; zoneid++) {3031		zone = &pgdat->node_zones[zoneid];3032 3033		if (!populated_zone(zone))3034			continue;3035 3036		ret = compaction_suit_allocation_order(zone,3037				pgdat->kcompactd_max_order,3038				highest_zoneidx, ALLOC_WMARK_MIN);3039		if (ret == COMPACT_CONTINUE)3040			return true;3041	}3042 3043	return false;3044}3045 3046static HWJS_SUSPENDS void kcompactd_do_work(pg_data_t *pgdat)3047{3048	/*3049	 * With no special task, compact all zones so that a page of requested3050	 * order is allocatable.3051	 */3052	int zoneid;3053	struct zone *zone;3054	struct compact_control cc = {3055		.order = pgdat->kcompactd_max_order,3056		.search_order = pgdat->kcompactd_max_order,3057		.highest_zoneidx = pgdat->kcompactd_highest_zoneidx,3058		.mode = MIGRATE_SYNC_LIGHT,3059		.ignore_skip_hint = false,3060		.gfp_mask = GFP_KERNEL,3061	};3062	enum compact_result ret;3063 3064	trace_mm_compaction_kcompactd_wake(pgdat->node_id, cc.order,3065							cc.highest_zoneidx);3066	count_compact_event(KCOMPACTD_WAKE);3067 3068	for (zoneid = 0; zoneid <= cc.highest_zoneidx; zoneid++) {3069		int status;3070 3071		zone = &pgdat->node_zones[zoneid];3072		if (!populated_zone(zone))3073			continue;3074 3075		if (compaction_deferred(zone, cc.order))3076			continue;3077 3078		ret = compaction_suit_allocation_order(zone,3079				cc.order, zoneid, ALLOC_WMARK_MIN);3080		if (ret != COMPACT_CONTINUE)3081			continue;3082 3083		if (kthread_should_stop())3084			return;3085 3086		cc.zone = zone;3087		status = compact_zone(&cc, NULL);3088 3089		if (status == COMPACT_SUCCESS) {3090			compaction_defer_reset(zone, cc.order, false);3091		} else if (status == COMPACT_PARTIAL_SKIPPED || status == COMPACT_COMPLETE) {3092			/*3093			 * Buddy pages may become stranded on pcps that could3094			 * otherwise coalesce on the zone's free area for3095			 * order >= cc.order.  This is ratelimited by the3096			 * upcoming deferral.3097			 */3098			drain_all_pages(zone);3099 3100			/*3101			 * We use sync migration mode here, so we defer like3102			 * sync direct compaction does.3103			 */3104			defer_compaction(zone, cc.order);3105		}3106 3107		count_compact_events(KCOMPACTD_MIGRATE_SCANNED,3108				     cc.total_migrate_scanned);3109		count_compact_events(KCOMPACTD_FREE_SCANNED,3110				     cc.total_free_scanned);3111	}3112 3113	/*3114	 * Regardless of success, we are done until woken up next. But remember3115	 * the requested order/highest_zoneidx in case it was higher/tighter3116	 * than our current ones3117	 */3118	if (pgdat->kcompactd_max_order <= cc.order)3119		pgdat->kcompactd_max_order = 0;3120	if (pgdat->kcompactd_highest_zoneidx >= cc.highest_zoneidx)3121		pgdat->kcompactd_highest_zoneidx = pgdat->nr_zones - 1;3122}3123 3124void wakeup_kcompactd(pg_data_t *pgdat, int order, int highest_zoneidx)3125{3126	if (!order)3127		return;3128 3129	if (pgdat->kcompactd_max_order < order)3130		pgdat->kcompactd_max_order = order;3131 3132	if (pgdat->kcompactd_highest_zoneidx > highest_zoneidx)3133		pgdat->kcompactd_highest_zoneidx = highest_zoneidx;3134 3135	/*3136	 * Pairs with implicit barrier in wait_event_freezable()3137	 * such that wakeups are not missed.3138	 */3139	if (!wq_has_sleeper(&pgdat->kcompactd_wait))3140		return;3141 3142	if (!kcompactd_node_suitable(pgdat))3143		return;3144 3145	trace_mm_compaction_wakeup_kcompactd(pgdat->node_id, order,3146							highest_zoneidx);3147	wake_up_interruptible(&pgdat->kcompactd_wait);3148}3149 3150/*3151 * The background compaction daemon, started as a kernel thread3152 * from the init process.3153 */3154static HWJS_SUSPENDS int kcompactd(void *p)3155{3156	pg_data_t *pgdat = (pg_data_t *)p;3157	struct task_struct *tsk = current;3158	long default_timeout = msecs_to_jiffies(HPAGE_FRAG_CHECK_INTERVAL_MSEC);3159	long timeout = default_timeout;3160 3161	const struct cpumask *cpumask = cpumask_of_node(pgdat->node_id);3162 3163	if (!cpumask_empty(cpumask))3164		set_cpus_allowed_ptr(tsk, cpumask);3165 3166	set_freezable();3167 3168	pgdat->kcompactd_max_order = 0;3169	pgdat->kcompactd_highest_zoneidx = pgdat->nr_zones - 1;3170 3171	while (!kthread_should_stop()) {3172		unsigned long pflags;3173 3174		/*3175		 * Avoid the unnecessary wakeup for proactive compaction3176		 * when it is disabled.3177		 */3178		if (!sysctl_compaction_proactiveness)3179			timeout = MAX_SCHEDULE_TIMEOUT;3180		trace_mm_compaction_kcompactd_sleep(pgdat->node_id);3181		if (wait_event_freezable_timeout(pgdat->kcompactd_wait,3182			kcompactd_work_requested(pgdat), timeout) &&3183			!pgdat->proactive_compact_trigger) {3184 3185			psi_memstall_enter(&pflags);3186			kcompactd_do_work(pgdat);3187			psi_memstall_leave(&pflags);3188			/*3189			 * Reset the timeout value. The defer timeout from3190			 * proactive compaction is lost here but that is fine3191			 * as the condition of the zone changing substantionally3192			 * then carrying on with the previous defer interval is3193			 * not useful.3194			 */3195			timeout = default_timeout;3196			continue;3197		}3198 3199		/*3200		 * Start the proactive work with default timeout. Based3201		 * on the fragmentation score, this timeout is updated.3202		 */3203		timeout = default_timeout;3204		if (should_proactive_compact_node(pgdat)) {3205			unsigned int prev_score, score;3206 3207			prev_score = fragmentation_score_node(pgdat);3208			compact_node(pgdat, true);3209			score = fragmentation_score_node(pgdat);3210			/*3211			 * Defer proactive compaction if the fragmentation3212			 * score did not go down i.e. no progress made.3213			 */3214			if (unlikely(score >= prev_score))3215				timeout =3216				   default_timeout << COMPACT_MAX_DEFER_SHIFT;3217		}3218		if (unlikely(pgdat->proactive_compact_trigger))3219			pgdat->proactive_compact_trigger = false;3220	}3221 3222	return 0;3223}3224 3225/*3226 * This kcompactd start function will be called by init and node-hot-add.3227 * On node-hot-add, kcompactd will moved to proper cpus if cpus are hot-added.3228 */3229void __meminit kcompactd_run(int nid)3230{3231	pg_data_t *pgdat = NODE_DATA(nid);3232 3233	if (pgdat->kcompactd)3234		return;3235 3236	pgdat->kcompactd = kthread_run(kcompactd, pgdat, "kcompactd%d", nid);3237	if (IS_ERR(pgdat->kcompactd)) {3238		pr_err("Failed to start kcompactd on node %d\n", nid);3239		pgdat->kcompactd = NULL;3240	}3241}3242 3243/*3244 * Called by memory hotplug when all memory in a node is offlined. Caller must3245 * be holding mem_hotplug_begin/done().3246 */3247void __meminit kcompactd_stop(int nid)3248{3249	struct task_struct *kcompactd = NODE_DATA(nid)->kcompactd;3250 3251	if (kcompactd) {3252		kthread_stop(kcompactd);3253		NODE_DATA(nid)->kcompactd = NULL;3254	}3255}3256 3257/*3258 * It's optimal to keep kcompactd on the same CPUs as their memory, but3259 * not required for correctness. So if the last cpu in a node goes3260 * away, we get changed to run anywhere: as the first one comes back,3261 * restore their cpu bindings.3262 */3263static HWJS_SUSPENDS int kcompactd_cpu_online(unsigned int cpu)3264{3265	int nid;3266 3267	for_each_node_state(nid, N_MEMORY) {3268		pg_data_t *pgdat = NODE_DATA(nid);3269		const struct cpumask *mask;3270 3271		mask = cpumask_of_node(pgdat->node_id);3272 3273		if (cpumask_any_and(cpu_online_mask, mask) < nr_cpu_ids)3274			/* One of our CPUs online: restore mask */3275			if (pgdat->kcompactd)3276				set_cpus_allowed_ptr(pgdat->kcompactd, mask);3277	}3278	return 0;3279}3280 3281static int proc_dointvec_minmax_warn_RT_change(const struct ctl_table *table,3282		int write, void *buffer, size_t *lenp, loff_t *ppos)3283{3284	int ret, old;3285 3286	if (!IS_ENABLED(CONFIG_PREEMPT_RT) || !write)3287		return proc_dointvec_minmax(table, write, buffer, lenp, ppos);3288 3289	old = *(int *)table->data;3290	ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);3291	if (ret)3292		return ret;3293	if (old != *(int *)table->data)3294		pr_warn_once("sysctl attribute %s changed by %s[%d]\n",3295			     table->procname, current->comm,3296			     task_pid_nr(current));3297	return ret;3298}3299 3300static struct ctl_table vm_compaction[] = {3301	{3302		.procname	= "compact_memory",3303		.data		= &sysctl_compact_memory,3304		.maxlen		= sizeof(int),3305		.mode		= 0200,3306		.proc_handler	= sysctl_compaction_handler,3307	},3308	{3309		.procname	= "compaction_proactiveness",3310		.data		= &sysctl_compaction_proactiveness,3311		.maxlen		= sizeof(sysctl_compaction_proactiveness),3312		.mode		= 0644,3313		.proc_handler	= compaction_proactiveness_sysctl_handler,3314		.extra1		= SYSCTL_ZERO,3315		.extra2		= SYSCTL_ONE_HUNDRED,3316	},3317	{3318		.procname	= "extfrag_threshold",3319		.data		= &sysctl_extfrag_threshold,3320		.maxlen		= sizeof(int),3321		.mode		= 0644,3322		.proc_handler	= proc_dointvec_minmax,3323		.extra1		= SYSCTL_ZERO,3324		.extra2		= SYSCTL_ONE_THOUSAND,3325	},3326	{3327		.procname	= "compact_unevictable_allowed",3328		.data		= &sysctl_compact_unevictable_allowed,3329		.maxlen		= sizeof(int),3330		.mode		= 0644,3331		.proc_handler	= proc_dointvec_minmax_warn_RT_change,3332		.extra1		= SYSCTL_ZERO,3333		.extra2		= SYSCTL_ONE,3334	},3335};3336 3337static HWJS_SUSPENDS int __init kcompactd_init(void)3338{3339	int nid;3340	int ret;3341 3342	ret = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN,3343					"mm/compaction:online",3344					kcompactd_cpu_online, NULL);3345	if (ret < 0) {3346		pr_err("kcompactd: failed to register hotplug callbacks.\n");3347		return ret;3348	}3349 3350	for_each_node_state(nid, N_MEMORY)3351		kcompactd_run(nid);3352	register_sysctl_init("vm", vm_compaction);3353	return 0;3354}3355subsys_initcall(kcompactd_init)3356 3357#endif /* CONFIG_COMPACTION */3358