brintos / linux-shallow public Read only

1// SPDX-License-Identifier: GPL-2.0-only2/*3 * mm_init.c - Memory initialisation verification and debugging4 *5 * Copyright 2008 IBM Corporation, 20086 * Author Mel Gorman <mel@csn.ul.ie>7 *8 */9#include <linux/kernel.h>10#include <linux/init.h>11#include <linux/kobject.h>12#include <linux/export.h>13#include <linux/memory.h>14#include <linux/notifier.h>15#include <linux/sched.h>16#include <linux/mman.h>17#include <linux/memblock.h>18#include <linux/page-isolation.h>19#include <linux/padata.h>20#include <linux/nmi.h>21#include <linux/buffer_head.h>22#include <linux/kmemleak.h>23#include <linux/kfence.h>24#include <linux/page_ext.h>25#include <linux/pti.h>26#include <linux/pgtable.h>27#include <linux/stackdepot.h>28#include <linux/swap.h>29#include <linux/cma.h>30#include <linux/crash_dump.h>31#include <linux/execmem.h>32#include <linux/vmstat.h>33#include "internal.h"34#include "slab.h"35#include "shuffle.h"36 37#include <asm/setup.h>38 39#ifdef CONFIG_DEBUG_MEMORY_INIT40int __meminitdata mminit_loglevel;41 42/* The zonelists are simply reported, validation is manual. */43void __init mminit_verify_zonelist(void)44{45	int nid;46 47	if (mminit_loglevel < MMINIT_VERIFY)48		return;49 50	for_each_online_node(nid) {51		pg_data_t *pgdat = NODE_DATA(nid);52		struct zone *zone;53		struct zoneref *z;54		struct zonelist *zonelist;55		int i, listid, zoneid;56 57		for (i = 0; i < MAX_ZONELISTS * MAX_NR_ZONES; i++) {58 59			/* Identify the zone and nodelist */60			zoneid = i % MAX_NR_ZONES;61			listid = i / MAX_NR_ZONES;62			zonelist = &pgdat->node_zonelists[listid];63			zone = &pgdat->node_zones[zoneid];64			if (!populated_zone(zone))65				continue;66 67			/* Print information about the zonelist */68			printk(KERN_DEBUG "mminit::zonelist %s %d:%s = ",69				listid > 0 ? "thisnode" : "general", nid,70				zone->name);71 72			/* Iterate the zonelist */73			for_each_zone_zonelist(zone, z, zonelist, zoneid)74				pr_cont("%d:%s ", zone_to_nid(zone), zone->name);75			pr_cont("\n");76		}77	}78}79 80void __init mminit_verify_pageflags_layout(void)81{82	int shift, width;83	unsigned long or_mask, add_mask;84 85	shift = BITS_PER_LONG;86	width = shift - SECTIONS_WIDTH - NODES_WIDTH - ZONES_WIDTH87		- LAST_CPUPID_SHIFT - KASAN_TAG_WIDTH - LRU_GEN_WIDTH - LRU_REFS_WIDTH;88	mminit_dprintk(MMINIT_TRACE, "pageflags_layout_widths",89		"Section %d Node %d Zone %d Lastcpupid %d Kasantag %d Gen %d Tier %d Flags %d\n",90		SECTIONS_WIDTH,91		NODES_WIDTH,92		ZONES_WIDTH,93		LAST_CPUPID_WIDTH,94		KASAN_TAG_WIDTH,95		LRU_GEN_WIDTH,96		LRU_REFS_WIDTH,97		NR_PAGEFLAGS);98	mminit_dprintk(MMINIT_TRACE, "pageflags_layout_shifts",99		"Section %d Node %d Zone %d Lastcpupid %d Kasantag %d\n",100		SECTIONS_SHIFT,101		NODES_SHIFT,102		ZONES_SHIFT,103		LAST_CPUPID_SHIFT,104		KASAN_TAG_WIDTH);105	mminit_dprintk(MMINIT_TRACE, "pageflags_layout_pgshifts",106		"Section %lu Node %lu Zone %lu Lastcpupid %lu Kasantag %lu\n",107		(unsigned long)SECTIONS_PGSHIFT,108		(unsigned long)NODES_PGSHIFT,109		(unsigned long)ZONES_PGSHIFT,110		(unsigned long)LAST_CPUPID_PGSHIFT,111		(unsigned long)KASAN_TAG_PGSHIFT);112	mminit_dprintk(MMINIT_TRACE, "pageflags_layout_nodezoneid",113		"Node/Zone ID: %lu -> %lu\n",114		(unsigned long)(ZONEID_PGOFF + ZONEID_SHIFT),115		(unsigned long)ZONEID_PGOFF);116	mminit_dprintk(MMINIT_TRACE, "pageflags_layout_usage",117		"location: %d -> %d layout %d -> %d unused %d -> %d page-flags\n",118		shift, width, width, NR_PAGEFLAGS, NR_PAGEFLAGS, 0);119#ifdef NODE_NOT_IN_PAGE_FLAGS120	mminit_dprintk(MMINIT_TRACE, "pageflags_layout_nodeflags",121		"Node not in page flags");122#endif123#ifdef LAST_CPUPID_NOT_IN_PAGE_FLAGS124	mminit_dprintk(MMINIT_TRACE, "pageflags_layout_nodeflags",125		"Last cpupid not in page flags");126#endif127 128	if (SECTIONS_WIDTH) {129		shift -= SECTIONS_WIDTH;130		BUG_ON(shift != SECTIONS_PGSHIFT);131	}132	if (NODES_WIDTH) {133		shift -= NODES_WIDTH;134		BUG_ON(shift != NODES_PGSHIFT);135	}136	if (ZONES_WIDTH) {137		shift -= ZONES_WIDTH;138		BUG_ON(shift != ZONES_PGSHIFT);139	}140 141	/* Check for bitmask overlaps */142	or_mask = (ZONES_MASK << ZONES_PGSHIFT) |143			(NODES_MASK << NODES_PGSHIFT) |144			(SECTIONS_MASK << SECTIONS_PGSHIFT);145	add_mask = (ZONES_MASK << ZONES_PGSHIFT) +146			(NODES_MASK << NODES_PGSHIFT) +147			(SECTIONS_MASK << SECTIONS_PGSHIFT);148	BUG_ON(or_mask != add_mask);149}150 151static __init int set_mminit_loglevel(char *str)152{153	get_option(&str, &mminit_loglevel);154	return 0;155}156early_param("mminit_loglevel", set_mminit_loglevel);157#endif /* CONFIG_DEBUG_MEMORY_INIT */158 159struct kobject *mm_kobj;160 161#ifdef CONFIG_SMP162s32 vm_committed_as_batch = 32;163 164void mm_compute_batch(int overcommit_policy)165{166	u64 memsized_batch;167	s32 nr = num_present_cpus();168	s32 batch = max_t(s32, nr*2, 32);169	unsigned long ram_pages = totalram_pages();170 171	/*172	 * For policy OVERCOMMIT_NEVER, set batch size to 0.4% of173	 * (total memory/#cpus), and lift it to 25% for other policies174	 * to easy the possible lock contention for percpu_counter175	 * vm_committed_as, while the max limit is INT_MAX176	 */177	if (overcommit_policy == OVERCOMMIT_NEVER)178		memsized_batch = min_t(u64, ram_pages/nr/256, INT_MAX);179	else180		memsized_batch = min_t(u64, ram_pages/nr/4, INT_MAX);181 182	vm_committed_as_batch = max_t(s32, memsized_batch, batch);183}184 185static int __meminit mm_compute_batch_notifier(struct notifier_block *self,186					unsigned long action, void *arg)187{188	switch (action) {189	case MEM_ONLINE:190	case MEM_OFFLINE:191		mm_compute_batch(sysctl_overcommit_memory);192		break;193	default:194		break;195	}196	return NOTIFY_OK;197}198 199static int __init mm_compute_batch_init(void)200{201	mm_compute_batch(sysctl_overcommit_memory);202	hotplug_memory_notifier(mm_compute_batch_notifier, MM_COMPUTE_BATCH_PRI);203	return 0;204}205 206__initcall(mm_compute_batch_init);207 208#endif209 210static int __init mm_sysfs_init(void)211{212	mm_kobj = kobject_create_and_add("mm", kernel_kobj);213	if (!mm_kobj)214		return -ENOMEM;215 216	return 0;217}218postcore_initcall(mm_sysfs_init);219 220static unsigned long arch_zone_lowest_possible_pfn[MAX_NR_ZONES] __initdata;221static unsigned long arch_zone_highest_possible_pfn[MAX_NR_ZONES] __initdata;222static unsigned long zone_movable_pfn[MAX_NUMNODES] __initdata;223 224static unsigned long required_kernelcore __initdata;225static unsigned long required_kernelcore_percent __initdata;226static unsigned long required_movablecore __initdata;227static unsigned long required_movablecore_percent __initdata;228 229static unsigned long nr_kernel_pages __initdata;230static unsigned long nr_all_pages __initdata;231 232static bool deferred_struct_pages __meminitdata;233 234static DEFINE_PER_CPU(struct per_cpu_nodestat, boot_nodestats);235 236static int __init cmdline_parse_core(char *p, unsigned long *core,237				     unsigned long *percent)238{239	unsigned long long coremem;240	char *endptr;241 242	if (!p)243		return -EINVAL;244 245	/* Value may be a percentage of total memory, otherwise bytes */246	coremem = simple_strtoull(p, &endptr, 0);247	if (*endptr == '%') {248		/* Paranoid check for percent values greater than 100 */249		WARN_ON(coremem > 100);250 251		*percent = coremem;252	} else {253		coremem = memparse(p, &p);254		/* Paranoid check that UL is enough for the coremem value */255		WARN_ON((coremem >> PAGE_SHIFT) > ULONG_MAX);256 257		*core = coremem >> PAGE_SHIFT;258		*percent = 0UL;259	}260	return 0;261}262 263bool mirrored_kernelcore __initdata_memblock;264 265/*266 * kernelcore=size sets the amount of memory for use for allocations that267 * cannot be reclaimed or migrated.268 */269static int __init cmdline_parse_kernelcore(char *p)270{271	/* parse kernelcore=mirror */272	if (parse_option_str(p, "mirror")) {273		mirrored_kernelcore = true;274		return 0;275	}276 277	return cmdline_parse_core(p, &required_kernelcore,278				  &required_kernelcore_percent);279}280early_param("kernelcore", cmdline_parse_kernelcore);281 282/*283 * movablecore=size sets the amount of memory for use for allocations that284 * can be reclaimed or migrated.285 */286static int __init cmdline_parse_movablecore(char *p)287{288	return cmdline_parse_core(p, &required_movablecore,289				  &required_movablecore_percent);290}291early_param("movablecore", cmdline_parse_movablecore);292 293/*294 * early_calculate_totalpages()295 * Sum pages in active regions for movable zone.296 * Populate N_MEMORY for calculating usable_nodes.297 */298static unsigned long __init early_calculate_totalpages(void)299{300	unsigned long totalpages = 0;301	unsigned long start_pfn, end_pfn;302	int i, nid;303 304	for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid) {305		unsigned long pages = end_pfn - start_pfn;306 307		totalpages += pages;308		if (pages)309			node_set_state(nid, N_MEMORY);310	}311	return totalpages;312}313 314/*315 * This finds a zone that can be used for ZONE_MOVABLE pages. The316 * assumption is made that zones within a node are ordered in monotonic317 * increasing memory addresses so that the "highest" populated zone is used318 */319static void __init find_usable_zone_for_movable(void)320{321	int zone_index;322	for (zone_index = MAX_NR_ZONES - 1; zone_index >= 0; zone_index--) {323		if (zone_index == ZONE_MOVABLE)324			continue;325 326		if (arch_zone_highest_possible_pfn[zone_index] >327				arch_zone_lowest_possible_pfn[zone_index])328			break;329	}330 331	VM_BUG_ON(zone_index == -1);332	movable_zone = zone_index;333}334 335/*336 * Find the PFN the Movable zone begins in each node. Kernel memory337 * is spread evenly between nodes as long as the nodes have enough338 * memory. When they don't, some nodes will have more kernelcore than339 * others340 */341static void __init find_zone_movable_pfns_for_nodes(void)342{343	int i, nid;344	unsigned long usable_startpfn;345	unsigned long kernelcore_node, kernelcore_remaining;346	/* save the state before borrow the nodemask */347	nodemask_t saved_node_state = node_states[N_MEMORY];348	unsigned long totalpages = early_calculate_totalpages();349	int usable_nodes = nodes_weight(node_states[N_MEMORY]);350	struct memblock_region *r;351 352	/* Need to find movable_zone earlier when movable_node is specified. */353	find_usable_zone_for_movable();354 355	/*356	 * If movable_node is specified, ignore kernelcore and movablecore357	 * options.358	 */359	if (movable_node_is_enabled()) {360		for_each_mem_region(r) {361			if (!memblock_is_hotpluggable(r))362				continue;363 364			nid = memblock_get_region_node(r);365 366			usable_startpfn = memblock_region_memory_base_pfn(r);367			zone_movable_pfn[nid] = zone_movable_pfn[nid] ?368				min(usable_startpfn, zone_movable_pfn[nid]) :369				usable_startpfn;370		}371 372		goto out2;373	}374 375	/*376	 * If kernelcore=mirror is specified, ignore movablecore option377	 */378	if (mirrored_kernelcore) {379		bool mem_below_4gb_not_mirrored = false;380 381		if (!memblock_has_mirror()) {382			pr_warn("The system has no mirror memory, ignore kernelcore=mirror.\n");383			goto out;384		}385 386		if (is_kdump_kernel()) {387			pr_warn("The system is under kdump, ignore kernelcore=mirror.\n");388			goto out;389		}390 391		for_each_mem_region(r) {392			if (memblock_is_mirror(r))393				continue;394 395			nid = memblock_get_region_node(r);396 397			usable_startpfn = memblock_region_memory_base_pfn(r);398 399			if (usable_startpfn < PHYS_PFN(SZ_4G)) {400				mem_below_4gb_not_mirrored = true;401				continue;402			}403 404			zone_movable_pfn[nid] = zone_movable_pfn[nid] ?405				min(usable_startpfn, zone_movable_pfn[nid]) :406				usable_startpfn;407		}408 409		if (mem_below_4gb_not_mirrored)410			pr_warn("This configuration results in unmirrored kernel memory.\n");411 412		goto out2;413	}414 415	/*416	 * If kernelcore=nn% or movablecore=nn% was specified, calculate the417	 * amount of necessary memory.418	 */419	if (required_kernelcore_percent)420		required_kernelcore = (totalpages * 100 * required_kernelcore_percent) /421				       10000UL;422	if (required_movablecore_percent)423		required_movablecore = (totalpages * 100 * required_movablecore_percent) /424					10000UL;425 426	/*427	 * If movablecore= was specified, calculate what size of428	 * kernelcore that corresponds so that memory usable for429	 * any allocation type is evenly spread. If both kernelcore430	 * and movablecore are specified, then the value of kernelcore431	 * will be used for required_kernelcore if it's greater than432	 * what movablecore would have allowed.433	 */434	if (required_movablecore) {435		unsigned long corepages;436 437		/*438		 * Round-up so that ZONE_MOVABLE is at least as large as what439		 * was requested by the user440		 */441		required_movablecore =442			roundup(required_movablecore, MAX_ORDER_NR_PAGES);443		required_movablecore = min(totalpages, required_movablecore);444		corepages = totalpages - required_movablecore;445 446		required_kernelcore = max(required_kernelcore, corepages);447	}448 449	/*450	 * If kernelcore was not specified or kernelcore size is larger451	 * than totalpages, there is no ZONE_MOVABLE.452	 */453	if (!required_kernelcore || required_kernelcore >= totalpages)454		goto out;455 456	/* usable_startpfn is the lowest possible pfn ZONE_MOVABLE can be at */457	usable_startpfn = arch_zone_lowest_possible_pfn[movable_zone];458 459restart:460	/* Spread kernelcore memory as evenly as possible throughout nodes */461	kernelcore_node = required_kernelcore / usable_nodes;462	for_each_node_state(nid, N_MEMORY) {463		unsigned long start_pfn, end_pfn;464 465		/*466		 * Recalculate kernelcore_node if the division per node467		 * now exceeds what is necessary to satisfy the requested468		 * amount of memory for the kernel469		 */470		if (required_kernelcore < kernelcore_node)471			kernelcore_node = required_kernelcore / usable_nodes;472 473		/*474		 * As the map is walked, we track how much memory is usable475		 * by the kernel using kernelcore_remaining. When it is476		 * 0, the rest of the node is usable by ZONE_MOVABLE477		 */478		kernelcore_remaining = kernelcore_node;479 480		/* Go through each range of PFNs within this node */481		for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) {482			unsigned long size_pages;483 484			start_pfn = max(start_pfn, zone_movable_pfn[nid]);485			if (start_pfn >= end_pfn)486				continue;487 488			/* Account for what is only usable for kernelcore */489			if (start_pfn < usable_startpfn) {490				unsigned long kernel_pages;491				kernel_pages = min(end_pfn, usable_startpfn)492								- start_pfn;493 494				kernelcore_remaining -= min(kernel_pages,495							kernelcore_remaining);496				required_kernelcore -= min(kernel_pages,497							required_kernelcore);498 499				/* Continue if range is now fully accounted */500				if (end_pfn <= usable_startpfn) {501 502					/*503					 * Push zone_movable_pfn to the end so504					 * that if we have to rebalance505					 * kernelcore across nodes, we will506					 * not double account here507					 */508					zone_movable_pfn[nid] = end_pfn;509					continue;510				}511				start_pfn = usable_startpfn;512			}513 514			/*515			 * The usable PFN range for ZONE_MOVABLE is from516			 * start_pfn->end_pfn. Calculate size_pages as the517			 * number of pages used as kernelcore518			 */519			size_pages = end_pfn - start_pfn;520			if (size_pages > kernelcore_remaining)521				size_pages = kernelcore_remaining;522			zone_movable_pfn[nid] = start_pfn + size_pages;523 524			/*525			 * Some kernelcore has been met, update counts and526			 * break if the kernelcore for this node has been527			 * satisfied528			 */529			required_kernelcore -= min(required_kernelcore,530								size_pages);531			kernelcore_remaining -= size_pages;532			if (!kernelcore_remaining)533				break;534		}535	}536 537	/*538	 * If there is still required_kernelcore, we do another pass with one539	 * less node in the count. This will push zone_movable_pfn[nid] further540	 * along on the nodes that still have memory until kernelcore is541	 * satisfied542	 */543	usable_nodes--;544	if (usable_nodes && required_kernelcore > usable_nodes)545		goto restart;546 547out2:548	/* Align start of ZONE_MOVABLE on all nids to MAX_ORDER_NR_PAGES */549	for (nid = 0; nid < MAX_NUMNODES; nid++) {550		unsigned long start_pfn, end_pfn;551 552		zone_movable_pfn[nid] =553			roundup(zone_movable_pfn[nid], MAX_ORDER_NR_PAGES);554 555		get_pfn_range_for_nid(nid, &start_pfn, &end_pfn);556		if (zone_movable_pfn[nid] >= end_pfn)557			zone_movable_pfn[nid] = 0;558	}559 560out:561	/* restore the node_state */562	node_states[N_MEMORY] = saved_node_state;563}564 565void __meminit __init_single_page(struct page *page, unsigned long pfn,566				unsigned long zone, int nid)567{568	mm_zero_struct_page(page);569	set_page_links(page, zone, nid, pfn);570	init_page_count(page);571	atomic_set(&page->_mapcount, -1);572	page_cpupid_reset_last(page);573	page_kasan_tag_reset(page);574 575	INIT_LIST_HEAD(&page->lru);576#ifdef WANT_PAGE_VIRTUAL577	/* The shift won't overflow because ZONE_NORMAL is below 4G. */578	if (!is_highmem_idx(zone))579		set_page_address(page, __va(pfn << PAGE_SHIFT));580#endif581}582 583#ifdef CONFIG_NUMA584/*585 * During memory init memblocks map pfns to nids. The search is expensive and586 * this caches recent lookups. The implementation of __early_pfn_to_nid587 * treats start/end as pfns.588 */589struct mminit_pfnnid_cache {590	unsigned long last_start;591	unsigned long last_end;592	int last_nid;593};594 595static struct mminit_pfnnid_cache early_pfnnid_cache __meminitdata;596 597/*598 * Required by SPARSEMEM. Given a PFN, return what node the PFN is on.599 */600static int __meminit __early_pfn_to_nid(unsigned long pfn,601					struct mminit_pfnnid_cache *state)602{603	unsigned long start_pfn, end_pfn;604	int nid;605 606	if (state->last_start <= pfn && pfn < state->last_end)607		return state->last_nid;608 609	nid = memblock_search_pfn_nid(pfn, &start_pfn, &end_pfn);610	if (nid != NUMA_NO_NODE) {611		state->last_start = start_pfn;612		state->last_end = end_pfn;613		state->last_nid = nid;614	}615 616	return nid;617}618 619int __meminit early_pfn_to_nid(unsigned long pfn)620{621	static DEFINE_SPINLOCK(early_pfn_lock);622	int nid;623 624	spin_lock(&early_pfn_lock);625	nid = __early_pfn_to_nid(pfn, &early_pfnnid_cache);626	if (nid < 0)627		nid = first_online_node;628	spin_unlock(&early_pfn_lock);629 630	return nid;631}632 633int hashdist = HASHDIST_DEFAULT;634 635static int __init set_hashdist(char *str)636{637	if (!str)638		return 0;639	hashdist = simple_strtoul(str, &str, 0);640	return 1;641}642__setup("hashdist=", set_hashdist);643 644static inline void fixup_hashdist(void)645{646	if (num_node_state(N_MEMORY) == 1)647		hashdist = 0;648}649#else650static inline void fixup_hashdist(void) {}651#endif /* CONFIG_NUMA */652 653#ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT654static inline void pgdat_set_deferred_range(pg_data_t *pgdat)655{656	pgdat->first_deferred_pfn = ULONG_MAX;657}658 659/* Returns true if the struct page for the pfn is initialised */660static inline bool __meminit early_page_initialised(unsigned long pfn, int nid)661{662	if (node_online(nid) && pfn >= NODE_DATA(nid)->first_deferred_pfn)663		return false;664 665	return true;666}667 668/*669 * Returns true when the remaining initialisation should be deferred until670 * later in the boot cycle when it can be parallelised.671 */672static bool __meminit673defer_init(int nid, unsigned long pfn, unsigned long end_pfn)674{675	static unsigned long prev_end_pfn, nr_initialised;676 677	if (early_page_ext_enabled())678		return false;679 680	/* Always populate low zones for address-constrained allocations */681	if (end_pfn < pgdat_end_pfn(NODE_DATA(nid)))682		return false;683 684	if (NODE_DATA(nid)->first_deferred_pfn != ULONG_MAX)685		return true;686 687	/*688	 * prev_end_pfn static that contains the end of previous zone689	 * No need to protect because called very early in boot before smp_init.690	 */691	if (prev_end_pfn != end_pfn) {692		prev_end_pfn = end_pfn;693		nr_initialised = 0;694	}695 696	/*697	 * We start only with one section of pages, more pages are added as698	 * needed until the rest of deferred pages are initialized.699	 */700	nr_initialised++;701	if ((nr_initialised > PAGES_PER_SECTION) &&702	    (pfn & (PAGES_PER_SECTION - 1)) == 0) {703		NODE_DATA(nid)->first_deferred_pfn = pfn;704		return true;705	}706	return false;707}708 709static void __meminit init_reserved_page(unsigned long pfn, int nid)710{711	pg_data_t *pgdat;712	int zid;713 714	if (early_page_initialised(pfn, nid))715		return;716 717	pgdat = NODE_DATA(nid);718 719	for (zid = 0; zid < MAX_NR_ZONES; zid++) {720		struct zone *zone = &pgdat->node_zones[zid];721 722		if (zone_spans_pfn(zone, pfn))723			break;724	}725	__init_single_page(pfn_to_page(pfn), pfn, zid, nid);726}727#else728static inline void pgdat_set_deferred_range(pg_data_t *pgdat) {}729 730static inline bool early_page_initialised(unsigned long pfn, int nid)731{732	return true;733}734 735static inline bool defer_init(int nid, unsigned long pfn, unsigned long end_pfn)736{737	return false;738}739 740static inline void init_reserved_page(unsigned long pfn, int nid)741{742}743#endif /* CONFIG_DEFERRED_STRUCT_PAGE_INIT */744 745/*746 * Initialised pages do not have PageReserved set. This function is747 * called for each range allocated by the bootmem allocator and748 * marks the pages PageReserved. The remaining valid pages are later749 * sent to the buddy page allocator.750 */751void __meminit reserve_bootmem_region(phys_addr_t start,752				      phys_addr_t end, int nid)753{754	unsigned long start_pfn = PFN_DOWN(start);755	unsigned long end_pfn = PFN_UP(end);756 757	for (; start_pfn < end_pfn; start_pfn++) {758		if (pfn_valid(start_pfn)) {759			struct page *page = pfn_to_page(start_pfn);760 761			init_reserved_page(start_pfn, nid);762 763			/*764			 * no need for atomic set_bit because the struct765			 * page is not visible yet so nobody should766			 * access it yet.767			 */768			__SetPageReserved(page);769		}770	}771}772 773/* If zone is ZONE_MOVABLE but memory is mirrored, it is an overlapped init */774static bool __meminit775overlap_memmap_init(unsigned long zone, unsigned long *pfn)776{777	static struct memblock_region *r;778 779	if (mirrored_kernelcore && zone == ZONE_MOVABLE) {780		if (!r || *pfn >= memblock_region_memory_end_pfn(r)) {781			for_each_mem_region(r) {782				if (*pfn < memblock_region_memory_end_pfn(r))783					break;784			}785		}786		if (*pfn >= memblock_region_memory_base_pfn(r) &&787		    memblock_is_mirror(r)) {788			*pfn = memblock_region_memory_end_pfn(r);789			return true;790		}791	}792	return false;793}794 795/*796 * Only struct pages that correspond to ranges defined by memblock.memory797 * are zeroed and initialized by going through __init_single_page() during798 * memmap_init_zone_range().799 *800 * But, there could be struct pages that correspond to holes in801 * memblock.memory. This can happen because of the following reasons:802 * - physical memory bank size is not necessarily the exact multiple of the803 *   arbitrary section size804 * - early reserved memory may not be listed in memblock.memory805 * - non-memory regions covered by the contigious flatmem mapping806 * - memory layouts defined with memmap= kernel parameter may not align807 *   nicely with memmap sections808 *809 * Explicitly initialize those struct pages so that:810 * - PG_Reserved is set811 * - zone and node links point to zone and node that span the page if the812 *   hole is in the middle of a zone813 * - zone and node links point to adjacent zone/node if the hole falls on814 *   the zone boundary; the pages in such holes will be prepended to the815 *   zone/node above the hole except for the trailing pages in the last816 *   section that will be appended to the zone/node below.817 */818static void __init init_unavailable_range(unsigned long spfn,819					  unsigned long epfn,820					  int zone, int node)821{822	unsigned long pfn;823	u64 pgcnt = 0;824 825	for (pfn = spfn; pfn < epfn; pfn++) {826		if (!pfn_valid(pageblock_start_pfn(pfn))) {827			pfn = pageblock_end_pfn(pfn) - 1;828			continue;829		}830		__init_single_page(pfn_to_page(pfn), pfn, zone, node);831		__SetPageReserved(pfn_to_page(pfn));832		pgcnt++;833	}834 835	if (pgcnt)836		pr_info("On node %d, zone %s: %lld pages in unavailable ranges\n",837			node, zone_names[zone], pgcnt);838}839 840/*841 * Initially all pages are reserved - free ones are freed842 * up by memblock_free_all() once the early boot process is843 * done. Non-atomic initialization, single-pass.844 *845 * All aligned pageblocks are initialized to the specified migratetype846 * (usually MIGRATE_MOVABLE). Besides setting the migratetype, no related847 * zone stats (e.g., nr_isolate_pageblock) are touched.848 */849void __meminit memmap_init_range(unsigned long size, int nid, unsigned long zone,850		unsigned long start_pfn, unsigned long zone_end_pfn,851		enum meminit_context context,852		struct vmem_altmap *altmap, int migratetype)853{854	unsigned long pfn, end_pfn = start_pfn + size;855	struct page *page;856 857	if (highest_memmap_pfn < end_pfn - 1)858		highest_memmap_pfn = end_pfn - 1;859 860#ifdef CONFIG_ZONE_DEVICE861	/*862	 * Honor reservation requested by the driver for this ZONE_DEVICE863	 * memory. We limit the total number of pages to initialize to just864	 * those that might contain the memory mapping. We will defer the865	 * ZONE_DEVICE page initialization until after we have released866	 * the hotplug lock.867	 */868	if (zone == ZONE_DEVICE) {869		if (!altmap)870			return;871 872		if (start_pfn == altmap->base_pfn)873			start_pfn += altmap->reserve;874		end_pfn = altmap->base_pfn + vmem_altmap_offset(altmap);875	}876#endif877 878	for (pfn = start_pfn; pfn < end_pfn; ) {879		/*880		 * There can be holes in boot-time mem_map[]s handed to this881		 * function.  They do not exist on hotplugged memory.882		 */883		if (context == MEMINIT_EARLY) {884			if (overlap_memmap_init(zone, &pfn))885				continue;886			if (defer_init(nid, pfn, zone_end_pfn)) {887				deferred_struct_pages = true;888				break;889			}890		}891 892		page = pfn_to_page(pfn);893		__init_single_page(page, pfn, zone, nid);894		if (context == MEMINIT_HOTPLUG) {895#ifdef CONFIG_ZONE_DEVICE896			if (zone == ZONE_DEVICE)897				__SetPageReserved(page);898			else899#endif900				__SetPageOffline(page);901		}902 903		/*904		 * Usually, we want to mark the pageblock MIGRATE_MOVABLE,905		 * such that unmovable allocations won't be scattered all906		 * over the place during system boot.907		 */908		if (pageblock_aligned(pfn)) {909			set_pageblock_migratetype(page, migratetype);910			cond_resched();911		}912		pfn++;913	}914}915 916static HWJS_SUSPENDS void __init memmap_init_zone_range(struct zone *zone,917					  unsigned long start_pfn,918					  unsigned long end_pfn,919					  unsigned long *hole_pfn)920{921	unsigned long zone_start_pfn = zone->zone_start_pfn;922	unsigned long zone_end_pfn = zone_start_pfn + zone->spanned_pages;923	int nid = zone_to_nid(zone), zone_id = zone_idx(zone);924 925	start_pfn = clamp(start_pfn, zone_start_pfn, zone_end_pfn);926	end_pfn = clamp(end_pfn, zone_start_pfn, zone_end_pfn);927 928	if (start_pfn >= end_pfn)929		return;930 931	memmap_init_range(end_pfn - start_pfn, nid, zone_id, start_pfn,932			  zone_end_pfn, MEMINIT_EARLY, NULL, MIGRATE_MOVABLE);933 934	if (*hole_pfn < start_pfn)935		init_unavailable_range(*hole_pfn, start_pfn, zone_id, nid);936 937	*hole_pfn = end_pfn;938}939 940static HWJS_SUSPENDS void __init memmap_init(void)941{942	unsigned long start_pfn, end_pfn;943	unsigned long hole_pfn = 0;944	int i, j, zone_id = 0, nid;945 946	for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid) {947		struct pglist_data *node = NODE_DATA(nid);948 949		for (j = 0; j < MAX_NR_ZONES; j++) {950			struct zone *zone = node->node_zones + j;951 952			if (!populated_zone(zone))953				continue;954 955			memmap_init_zone_range(zone, start_pfn, end_pfn,956					       &hole_pfn);957			zone_id = j;958		}959	}960 961#ifdef CONFIG_SPARSEMEM962	/*963	 * Initialize the memory map for hole in the range [memory_end,964	 * section_end].965	 * Append the pages in this hole to the highest zone in the last966	 * node.967	 * The call to init_unavailable_range() is outside the ifdef to968	 * silence the compiler warining about zone_id set but not used;969	 * for FLATMEM it is a nop anyway970	 */971	end_pfn = round_up(end_pfn, PAGES_PER_SECTION);972	if (hole_pfn < end_pfn)973#endif974		init_unavailable_range(hole_pfn, end_pfn, zone_id, nid);975}976 977#ifdef CONFIG_ZONE_DEVICE978static void __ref __init_zone_device_page(struct page *page, unsigned long pfn,979					  unsigned long zone_idx, int nid,980					  struct dev_pagemap *pgmap)981{982 983	__init_single_page(page, pfn, zone_idx, nid);984 985	/*986	 * Mark page reserved as it will need to wait for onlining987	 * phase for it to be fully associated with a zone.988	 *989	 * We can use the non-atomic __set_bit operation for setting990	 * the flag as we are still initializing the pages.991	 */992	__SetPageReserved(page);993 994	/*995	 * ZONE_DEVICE pages union ->lru with a ->pgmap back pointer996	 * and zone_device_data.  It is a bug if a ZONE_DEVICE page is997	 * ever freed or placed on a driver-private list.998	 */999	page->pgmap = pgmap;1000	page->zone_device_data = NULL;1001 1002	/*1003	 * Mark the block movable so that blocks are reserved for1004	 * movable at startup. This will force kernel allocations1005	 * to reserve their blocks rather than leaking throughout1006	 * the address space during boot when many long-lived1007	 * kernel allocations are made.1008	 *1009	 * Please note that MEMINIT_HOTPLUG path doesn't clear memmap1010	 * because this is done early in section_activate()1011	 */1012	if (pageblock_aligned(pfn)) {1013		set_pageblock_migratetype(page, MIGRATE_MOVABLE);1014		cond_resched();1015	}1016 1017	/*1018	 * ZONE_DEVICE pages are released directly to the driver page allocator1019	 * which will set the page count to 1 when allocating the page.1020	 */1021	if (pgmap->type == MEMORY_DEVICE_PRIVATE ||1022	    pgmap->type == MEMORY_DEVICE_COHERENT)1023		set_page_count(page, 0);1024}1025 1026/*1027 * With compound page geometry and when struct pages are stored in ram most1028 * tail pages are reused. Consequently, the amount of unique struct pages to1029 * initialize is a lot smaller that the total amount of struct pages being1030 * mapped. This is a paired / mild layering violation with explicit knowledge1031 * of how the sparse_vmemmap internals handle compound pages in the lack1032 * of an altmap. See vmemmap_populate_compound_pages().1033 */1034static inline unsigned long compound_nr_pages(struct vmem_altmap *altmap,1035					      struct dev_pagemap *pgmap)1036{1037	if (!vmemmap_can_optimize(altmap, pgmap))1038		return pgmap_vmemmap_nr(pgmap);1039 1040	return VMEMMAP_RESERVE_NR * (PAGE_SIZE / sizeof(struct page));1041}1042 1043static void __ref memmap_init_compound(struct page *head,1044				       unsigned long head_pfn,1045				       unsigned long zone_idx, int nid,1046				       struct dev_pagemap *pgmap,1047				       unsigned long nr_pages)1048{1049	unsigned long pfn, end_pfn = head_pfn + nr_pages;1050	unsigned int order = pgmap->vmemmap_shift;1051 1052	__SetPageHead(head);1053	for (pfn = head_pfn + 1; pfn < end_pfn; pfn++) {1054		struct page *page = pfn_to_page(pfn);1055 1056		__init_zone_device_page(page, pfn, zone_idx, nid, pgmap);1057		prep_compound_tail(head, pfn - head_pfn);1058		set_page_count(page, 0);1059 1060		/*1061		 * The first tail page stores important compound page info.1062		 * Call prep_compound_head() after the first tail page has1063		 * been initialized, to not have the data overwritten.1064		 */1065		if (pfn == head_pfn + 1)1066			prep_compound_head(head, order);1067	}1068}1069 1070void __ref memmap_init_zone_device(struct zone *zone,1071				   unsigned long start_pfn,1072				   unsigned long nr_pages,1073				   struct dev_pagemap *pgmap)1074{1075	unsigned long pfn, end_pfn = start_pfn + nr_pages;1076	struct pglist_data *pgdat = zone->zone_pgdat;1077	struct vmem_altmap *altmap = pgmap_altmap(pgmap);1078	unsigned int pfns_per_compound = pgmap_vmemmap_nr(pgmap);1079	unsigned long zone_idx = zone_idx(zone);1080	unsigned long start = jiffies;1081	int nid = pgdat->node_id;1082 1083	if (WARN_ON_ONCE(!pgmap || zone_idx != ZONE_DEVICE))1084		return;1085 1086	/*1087	 * The call to memmap_init should have already taken care1088	 * of the pages reserved for the memmap, so we can just jump to1089	 * the end of that region and start processing the device pages.1090	 */1091	if (altmap) {1092		start_pfn = altmap->base_pfn + vmem_altmap_offset(altmap);1093		nr_pages = end_pfn - start_pfn;1094	}1095 1096	for (pfn = start_pfn; pfn < end_pfn; pfn += pfns_per_compound) {1097		struct page *page = pfn_to_page(pfn);1098 1099		__init_zone_device_page(page, pfn, zone_idx, nid, pgmap);1100 1101		if (pfns_per_compound == 1)1102			continue;1103 1104		memmap_init_compound(page, pfn, zone_idx, nid, pgmap,1105				     compound_nr_pages(altmap, pgmap));1106	}1107 1108	pr_debug("%s initialised %lu pages in %ums\n", __func__,1109		nr_pages, jiffies_to_msecs(jiffies - start));1110}1111#endif1112 1113/*1114 * The zone ranges provided by the architecture do not include ZONE_MOVABLE1115 * because it is sized independent of architecture. Unlike the other zones,1116 * the starting point for ZONE_MOVABLE is not fixed. It may be different1117 * in each node depending on the size of each node and how evenly kernelcore1118 * is distributed. This helper function adjusts the zone ranges1119 * provided by the architecture for a given node by using the end of the1120 * highest usable zone for ZONE_MOVABLE. This preserves the assumption that1121 * zones within a node are in order of monotonic increases memory addresses1122 */1123static void __init adjust_zone_range_for_zone_movable(int nid,1124					unsigned long zone_type,1125					unsigned long node_end_pfn,1126					unsigned long *zone_start_pfn,1127					unsigned long *zone_end_pfn)1128{1129	/* Only adjust if ZONE_MOVABLE is on this node */1130	if (zone_movable_pfn[nid]) {1131		/* Size ZONE_MOVABLE */1132		if (zone_type == ZONE_MOVABLE) {1133			*zone_start_pfn = zone_movable_pfn[nid];1134			*zone_end_pfn = min(node_end_pfn,1135				arch_zone_highest_possible_pfn[movable_zone]);1136 1137		/* Adjust for ZONE_MOVABLE starting within this range */1138		} else if (!mirrored_kernelcore &&1139			*zone_start_pfn < zone_movable_pfn[nid] &&1140			*zone_end_pfn > zone_movable_pfn[nid]) {1141			*zone_end_pfn = zone_movable_pfn[nid];1142 1143		/* Check if this whole range is within ZONE_MOVABLE */1144		} else if (*zone_start_pfn >= zone_movable_pfn[nid])1145			*zone_start_pfn = *zone_end_pfn;1146	}1147}1148 1149/*1150 * Return the number of holes in a range on a node. If nid is MAX_NUMNODES,1151 * then all holes in the requested range will be accounted for.1152 */1153static unsigned long __init __absent_pages_in_range(int nid,1154				unsigned long range_start_pfn,1155				unsigned long range_end_pfn)1156{1157	unsigned long nr_absent = range_end_pfn - range_start_pfn;1158	unsigned long start_pfn, end_pfn;1159	int i;1160 1161	for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) {1162		start_pfn = clamp(start_pfn, range_start_pfn, range_end_pfn);1163		end_pfn = clamp(end_pfn, range_start_pfn, range_end_pfn);1164		nr_absent -= end_pfn - start_pfn;1165	}1166	return nr_absent;1167}1168 1169/**1170 * absent_pages_in_range - Return number of page frames in holes within a range1171 * @start_pfn: The start PFN to start searching for holes1172 * @end_pfn: The end PFN to stop searching for holes1173 *1174 * Return: the number of pages frames in memory holes within a range.1175 */1176unsigned long __init absent_pages_in_range(unsigned long start_pfn,1177							unsigned long end_pfn)1178{1179	return __absent_pages_in_range(MAX_NUMNODES, start_pfn, end_pfn);1180}1181 1182/* Return the number of page frames in holes in a zone on a node */1183static unsigned long __init zone_absent_pages_in_node(int nid,1184					unsigned long zone_type,1185					unsigned long zone_start_pfn,1186					unsigned long zone_end_pfn)1187{1188	unsigned long nr_absent;1189 1190	/* zone is empty, we don't have any absent pages */1191	if (zone_start_pfn == zone_end_pfn)1192		return 0;1193 1194	nr_absent = __absent_pages_in_range(nid, zone_start_pfn, zone_end_pfn);1195 1196	/*1197	 * ZONE_MOVABLE handling.1198	 * Treat pages to be ZONE_MOVABLE in ZONE_NORMAL as absent pages1199	 * and vice versa.1200	 */1201	if (mirrored_kernelcore && zone_movable_pfn[nid]) {1202		unsigned long start_pfn, end_pfn;1203		struct memblock_region *r;1204 1205		for_each_mem_region(r) {1206			start_pfn = clamp(memblock_region_memory_base_pfn(r),1207					  zone_start_pfn, zone_end_pfn);1208			end_pfn = clamp(memblock_region_memory_end_pfn(r),1209					zone_start_pfn, zone_end_pfn);1210 1211			if (zone_type == ZONE_MOVABLE &&1212			    memblock_is_mirror(r))1213				nr_absent += end_pfn - start_pfn;1214 1215			if (zone_type == ZONE_NORMAL &&1216			    !memblock_is_mirror(r))1217				nr_absent += end_pfn - start_pfn;1218		}1219	}1220 1221	return nr_absent;1222}1223 1224/*1225 * Return the number of pages a zone spans in a node, including holes1226 * present_pages = zone_spanned_pages_in_node() - zone_absent_pages_in_node()1227 */1228static unsigned long __init zone_spanned_pages_in_node(int nid,1229					unsigned long zone_type,1230					unsigned long node_start_pfn,1231					unsigned long node_end_pfn,1232					unsigned long *zone_start_pfn,1233					unsigned long *zone_end_pfn)1234{1235	unsigned long zone_low = arch_zone_lowest_possible_pfn[zone_type];1236	unsigned long zone_high = arch_zone_highest_possible_pfn[zone_type];1237 1238	/* Get the start and end of the zone */1239	*zone_start_pfn = clamp(node_start_pfn, zone_low, zone_high);1240	*zone_end_pfn = clamp(node_end_pfn, zone_low, zone_high);1241	adjust_zone_range_for_zone_movable(nid, zone_type, node_end_pfn,1242					   zone_start_pfn, zone_end_pfn);1243 1244	/* Check that this node has pages within the zone's required range */1245	if (*zone_end_pfn < node_start_pfn || *zone_start_pfn > node_end_pfn)1246		return 0;1247 1248	/* Move the zone boundaries inside the node if necessary */1249	*zone_end_pfn = min(*zone_end_pfn, node_end_pfn);1250	*zone_start_pfn = max(*zone_start_pfn, node_start_pfn);1251 1252	/* Return the spanned pages */1253	return *zone_end_pfn - *zone_start_pfn;1254}1255 1256static void __init reset_memoryless_node_totalpages(struct pglist_data *pgdat)1257{1258	struct zone *z;1259 1260	for (z = pgdat->node_zones; z < pgdat->node_zones + MAX_NR_ZONES; z++) {1261		z->zone_start_pfn = 0;1262		z->spanned_pages = 0;1263		z->present_pages = 0;1264#if defined(CONFIG_MEMORY_HOTPLUG)1265		z->present_early_pages = 0;1266#endif1267	}1268 1269	pgdat->node_spanned_pages = 0;1270	pgdat->node_present_pages = 0;1271	pr_debug("On node %d totalpages: 0\n", pgdat->node_id);1272}1273 1274static void __init calc_nr_kernel_pages(void)1275{1276	unsigned long start_pfn, end_pfn;1277	phys_addr_t start_addr, end_addr;1278	u64 u;1279#ifdef CONFIG_HIGHMEM1280	unsigned long high_zone_low = arch_zone_lowest_possible_pfn[ZONE_HIGHMEM];1281#endif1282 1283	for_each_free_mem_range(u, NUMA_NO_NODE, MEMBLOCK_NONE, &start_addr, &end_addr, NULL) {1284		start_pfn = PFN_UP(start_addr);1285		end_pfn   = PFN_DOWN(end_addr);1286 1287		if (start_pfn < end_pfn) {1288			nr_all_pages += end_pfn - start_pfn;1289#ifdef CONFIG_HIGHMEM1290			start_pfn = clamp(start_pfn, 0, high_zone_low);1291			end_pfn = clamp(end_pfn, 0, high_zone_low);1292#endif1293			nr_kernel_pages += end_pfn - start_pfn;1294		}1295	}1296}1297 1298static void __init calculate_node_totalpages(struct pglist_data *pgdat,1299						unsigned long node_start_pfn,1300						unsigned long node_end_pfn)1301{1302	unsigned long realtotalpages = 0, totalpages = 0;1303	enum zone_type i;1304 1305	for (i = 0; i < MAX_NR_ZONES; i++) {1306		struct zone *zone = pgdat->node_zones + i;1307		unsigned long zone_start_pfn, zone_end_pfn;1308		unsigned long spanned, absent;1309		unsigned long real_size;1310 1311		spanned = zone_spanned_pages_in_node(pgdat->node_id, i,1312						     node_start_pfn,1313						     node_end_pfn,1314						     &zone_start_pfn,1315						     &zone_end_pfn);1316		absent = zone_absent_pages_in_node(pgdat->node_id, i,1317						   zone_start_pfn,1318						   zone_end_pfn);1319 1320		real_size = spanned - absent;1321 1322		if (spanned)1323			zone->zone_start_pfn = zone_start_pfn;1324		else1325			zone->zone_start_pfn = 0;1326		zone->spanned_pages = spanned;1327		zone->present_pages = real_size;1328#if defined(CONFIG_MEMORY_HOTPLUG)1329		zone->present_early_pages = real_size;1330#endif1331 1332		totalpages += spanned;1333		realtotalpages += real_size;1334	}1335 1336	pgdat->node_spanned_pages = totalpages;1337	pgdat->node_present_pages = realtotalpages;1338	pr_debug("On node %d totalpages: %lu\n", pgdat->node_id, realtotalpages);1339}1340 1341#ifdef CONFIG_TRANSPARENT_HUGEPAGE1342static void pgdat_init_split_queue(struct pglist_data *pgdat)1343{1344	struct deferred_split *ds_queue = &pgdat->deferred_split_queue;1345 1346	spin_lock_init(&ds_queue->split_queue_lock);1347	INIT_LIST_HEAD(&ds_queue->split_queue);1348	ds_queue->split_queue_len = 0;1349}1350#else1351static void pgdat_init_split_queue(struct pglist_data *pgdat) {}1352#endif1353 1354#ifdef CONFIG_COMPACTION1355static void pgdat_init_kcompactd(struct pglist_data *pgdat)1356{1357	init_waitqueue_head(&pgdat->kcompactd_wait);1358}1359#else1360static void pgdat_init_kcompactd(struct pglist_data *pgdat) {}1361#endif1362 1363static void __meminit pgdat_init_internals(struct pglist_data *pgdat)1364{1365	int i;1366 1367	pgdat_resize_init(pgdat);1368	pgdat_kswapd_lock_init(pgdat);1369 1370	pgdat_init_split_queue(pgdat);1371	pgdat_init_kcompactd(pgdat);1372 1373	init_waitqueue_head(&pgdat->kswapd_wait);1374	init_waitqueue_head(&pgdat->pfmemalloc_wait);1375 1376	for (i = 0; i < NR_VMSCAN_THROTTLE; i++)1377		init_waitqueue_head(&pgdat->reclaim_wait[i]);1378 1379	pgdat_page_ext_init(pgdat);1380	lruvec_init(&pgdat->__lruvec);1381}1382 1383static void __meminit zone_init_internals(struct zone *zone, enum zone_type idx, int nid,1384							unsigned long remaining_pages)1385{1386	atomic_long_set(&zone->managed_pages, remaining_pages);1387	zone_set_nid(zone, nid);1388	zone->name = zone_names[idx];1389	zone->zone_pgdat = NODE_DATA(nid);1390	spin_lock_init(&zone->lock);1391	zone_seqlock_init(zone);1392	zone_pcp_init(zone);1393}1394 1395static void __meminit zone_init_free_lists(struct zone *zone)1396{1397	unsigned int order, t;1398	for_each_migratetype_order(order, t) {1399		INIT_LIST_HEAD(&zone->free_area[order].free_list[t]);1400		zone->free_area[order].nr_free = 0;1401	}1402 1403#ifdef CONFIG_UNACCEPTED_MEMORY1404	INIT_LIST_HEAD(&zone->unaccepted_pages);1405#endif1406}1407 1408void __meminit init_currently_empty_zone(struct zone *zone,1409					unsigned long zone_start_pfn,1410					unsigned long size)1411{1412	struct pglist_data *pgdat = zone->zone_pgdat;1413	int zone_idx = zone_idx(zone) + 1;1414 1415	if (zone_idx > pgdat->nr_zones)1416		pgdat->nr_zones = zone_idx;1417 1418	zone->zone_start_pfn = zone_start_pfn;1419 1420	mminit_dprintk(MMINIT_TRACE, "memmap_init",1421			"Initialising map node %d zone %lu pfns %lu -> %lu\n",1422			pgdat->node_id,1423			(unsigned long)zone_idx(zone),1424			zone_start_pfn, (zone_start_pfn + size));1425 1426	zone_init_free_lists(zone);1427	zone->initialized = 1;1428}1429 1430#ifndef CONFIG_SPARSEMEM1431/*1432 * Calculate the size of the zone->blockflags rounded to an unsigned long1433 * Start by making sure zonesize is a multiple of pageblock_order by rounding1434 * up. Then use 1 NR_PAGEBLOCK_BITS worth of bits per pageblock, finally1435 * round what is now in bits to nearest long in bits, then return it in1436 * bytes.1437 */1438static unsigned long __init usemap_size(unsigned long zone_start_pfn, unsigned long zonesize)1439{1440	unsigned long usemapsize;1441 1442	zonesize += zone_start_pfn & (pageblock_nr_pages-1);1443	usemapsize = roundup(zonesize, pageblock_nr_pages);1444	usemapsize = usemapsize >> pageblock_order;1445	usemapsize *= NR_PAGEBLOCK_BITS;1446	usemapsize = roundup(usemapsize, BITS_PER_LONG);1447 1448	return usemapsize / BITS_PER_BYTE;1449}1450 1451static void __ref setup_usemap(struct zone *zone)1452{1453	unsigned long usemapsize = usemap_size(zone->zone_start_pfn,1454					       zone->spanned_pages);1455	zone->pageblock_flags = NULL;1456	if (usemapsize) {1457		zone->pageblock_flags =1458			memblock_alloc_node(usemapsize, SMP_CACHE_BYTES,1459					    zone_to_nid(zone));1460		if (!zone->pageblock_flags)1461			panic("Failed to allocate %ld bytes for zone %s pageblock flags on node %d\n",1462			      usemapsize, zone->name, zone_to_nid(zone));1463	}1464}1465#else1466static inline void setup_usemap(struct zone *zone) {}1467#endif /* CONFIG_SPARSEMEM */1468 1469#ifdef CONFIG_HUGETLB_PAGE_SIZE_VARIABLE1470 1471/* Initialise the number of pages represented by NR_PAGEBLOCK_BITS */1472void __init set_pageblock_order(void)1473{1474	unsigned int order = MAX_PAGE_ORDER;1475 1476	/* Check that pageblock_nr_pages has not already been setup */1477	if (pageblock_order)1478		return;1479 1480	/* Don't let pageblocks exceed the maximum allocation granularity. */1481	if (HPAGE_SHIFT > PAGE_SHIFT && HUGETLB_PAGE_ORDER < order)1482		order = HUGETLB_PAGE_ORDER;1483 1484	/*1485	 * Assume the largest contiguous order of interest is a huge page.1486	 * This value may be variable depending on boot parameters on powerpc.1487	 */1488	pageblock_order = order;1489}1490#else /* CONFIG_HUGETLB_PAGE_SIZE_VARIABLE */1491 1492/*1493 * When CONFIG_HUGETLB_PAGE_SIZE_VARIABLE is not set, set_pageblock_order()1494 * is unused as pageblock_order is set at compile-time. See1495 * include/linux/pageblock-flags.h for the values of pageblock_order based on1496 * the kernel config1497 */1498void __init set_pageblock_order(void)1499{1500}1501 1502#endif /* CONFIG_HUGETLB_PAGE_SIZE_VARIABLE */1503 1504/*1505 * Set up the zone data structures1506 * - init pgdat internals1507 * - init all zones belonging to this node1508 *1509 * NOTE: this function is only called during memory hotplug1510 */1511#ifdef CONFIG_MEMORY_HOTPLUG1512void __ref free_area_init_core_hotplug(struct pglist_data *pgdat)1513{1514	int nid = pgdat->node_id;1515	enum zone_type z;1516	int cpu;1517 1518	pgdat_init_internals(pgdat);1519 1520	if (pgdat->per_cpu_nodestats == &boot_nodestats)1521		pgdat->per_cpu_nodestats = alloc_percpu(struct per_cpu_nodestat);1522 1523	/*1524	 * Reset the nr_zones, order and highest_zoneidx before reuse.1525	 * Note that kswapd will init kswapd_highest_zoneidx properly1526	 * when it starts in the near future.1527	 */1528	pgdat->nr_zones = 0;1529	pgdat->kswapd_order = 0;1530	pgdat->kswapd_highest_zoneidx = 0;1531	pgdat->node_start_pfn = 0;1532	pgdat->node_present_pages = 0;1533 1534	for_each_online_cpu(cpu) {1535		struct per_cpu_nodestat *p;1536 1537		p = per_cpu_ptr(pgdat->per_cpu_nodestats, cpu);1538		memset(p, 0, sizeof(*p));1539	}1540 1541	/*1542	 * When memory is hot-added, all the memory is in offline state. So1543	 * clear all zones' present_pages and managed_pages because they will1544	 * be updated in online_pages() and offline_pages().1545	 */1546	for (z = 0; z < MAX_NR_ZONES; z++) {1547		struct zone *zone = pgdat->node_zones + z;1548 1549		zone->present_pages = 0;1550		zone_init_internals(zone, z, nid, 0);1551	}1552}1553#endif1554 1555static void __init free_area_init_core(struct pglist_data *pgdat)1556{1557	enum zone_type j;1558	int nid = pgdat->node_id;1559 1560	pgdat_init_internals(pgdat);1561	pgdat->per_cpu_nodestats = &boot_nodestats;1562 1563	for (j = 0; j < MAX_NR_ZONES; j++) {1564		struct zone *zone = pgdat->node_zones + j;1565		unsigned long size = zone->spanned_pages;1566 1567		/*1568		 * Initialize zone->managed_pages as 0 , it will be reset1569		 * when memblock allocator frees pages into buddy system.1570		 */1571		zone_init_internals(zone, j, nid, zone->present_pages);1572 1573		if (!size)1574			continue;1575 1576		setup_usemap(zone);1577		init_currently_empty_zone(zone, zone->zone_start_pfn, size);1578	}1579}1580 1581void __init *memmap_alloc(phys_addr_t size, phys_addr_t align,1582			  phys_addr_t min_addr, int nid, bool exact_nid)1583{1584	void *ptr;1585 1586	if (exact_nid)1587		ptr = memblock_alloc_exact_nid_raw(size, align, min_addr,1588						   MEMBLOCK_ALLOC_ACCESSIBLE,1589						   nid);1590	else1591		ptr = memblock_alloc_try_nid_raw(size, align, min_addr,1592						 MEMBLOCK_ALLOC_ACCESSIBLE,1593						 nid);1594 1595	if (ptr && size > 0)1596		page_init_poison(ptr, size);1597 1598	return ptr;1599}1600 1601#ifdef CONFIG_FLATMEM1602static void __init alloc_node_mem_map(struct pglist_data *pgdat)1603{1604	unsigned long start, offset, size, end;1605	struct page *map;1606 1607	/* Skip empty nodes */1608	if (!pgdat->node_spanned_pages)1609		return;1610 1611	start = pgdat->node_start_pfn & ~(MAX_ORDER_NR_PAGES - 1);1612	offset = pgdat->node_start_pfn - start;1613	/*1614		 * The zone's endpoints aren't required to be MAX_PAGE_ORDER1615	 * aligned but the node_mem_map endpoints must be in order1616	 * for the buddy allocator to function correctly.1617	 */1618	end = ALIGN(pgdat_end_pfn(pgdat), MAX_ORDER_NR_PAGES);1619	size =  (end - start) * sizeof(struct page);1620	map = memmap_alloc(size, SMP_CACHE_BYTES, MEMBLOCK_LOW_LIMIT,1621			   pgdat->node_id, false);1622	if (!map)1623		panic("Failed to allocate %ld bytes for node %d memory map\n",1624		      size, pgdat->node_id);1625	pgdat->node_mem_map = map + offset;1626	memmap_boot_pages_add(DIV_ROUND_UP(size, PAGE_SIZE));1627	pr_debug("%s: node %d, pgdat %08lx, node_mem_map %08lx\n",1628		 __func__, pgdat->node_id, (unsigned long)pgdat,1629		 (unsigned long)pgdat->node_mem_map);1630#ifndef CONFIG_NUMA1631	/* the global mem_map is just set as node 0's */1632	if (pgdat == NODE_DATA(0)) {1633		mem_map = NODE_DATA(0)->node_mem_map;1634		if (page_to_pfn(mem_map) != pgdat->node_start_pfn)1635			mem_map -= offset;1636	}1637#endif1638}1639#else1640static inline void alloc_node_mem_map(struct pglist_data *pgdat) { }1641#endif /* CONFIG_FLATMEM */1642 1643/**1644 * get_pfn_range_for_nid - Return the start and end page frames for a node1645 * @nid: The nid to return the range for. If MAX_NUMNODES, the min and max PFN are returned.1646 * @start_pfn: Passed by reference. On return, it will have the node start_pfn.1647 * @end_pfn: Passed by reference. On return, it will have the node end_pfn.1648 *1649 * It returns the start and end page frame of a node based on information1650 * provided by memblock_set_node(). If called for a node1651 * with no available memory, the start and end PFNs will be 0.1652 */1653void __init get_pfn_range_for_nid(unsigned int nid,1654			unsigned long *start_pfn, unsigned long *end_pfn)1655{1656	unsigned long this_start_pfn, this_end_pfn;1657	int i;1658 1659	*start_pfn = -1UL;1660	*end_pfn = 0;1661 1662	for_each_mem_pfn_range(i, nid, &this_start_pfn, &this_end_pfn, NULL) {1663		*start_pfn = min(*start_pfn, this_start_pfn);1664		*end_pfn = max(*end_pfn, this_end_pfn);1665	}1666 1667	if (*start_pfn == -1UL)1668		*start_pfn = 0;1669}1670 1671static void __init free_area_init_node(int nid)1672{1673	pg_data_t *pgdat = NODE_DATA(nid);1674	unsigned long start_pfn = 0;1675	unsigned long end_pfn = 0;1676 1677	/* pg_data_t should be reset to zero when it's allocated */1678	WARN_ON(pgdat->nr_zones || pgdat->kswapd_highest_zoneidx);1679 1680	get_pfn_range_for_nid(nid, &start_pfn, &end_pfn);1681 1682	pgdat->node_id = nid;1683	pgdat->node_start_pfn = start_pfn;1684	pgdat->per_cpu_nodestats = NULL;1685 1686	if (start_pfn != end_pfn) {1687		pr_info("Initmem setup node %d [mem %#018Lx-%#018Lx]\n", nid,1688			(u64)start_pfn << PAGE_SHIFT,1689			end_pfn ? ((u64)end_pfn << PAGE_SHIFT) - 1 : 0);1690 1691		calculate_node_totalpages(pgdat, start_pfn, end_pfn);1692	} else {1693		pr_info("Initmem setup node %d as memoryless\n", nid);1694 1695		reset_memoryless_node_totalpages(pgdat);1696	}1697 1698	alloc_node_mem_map(pgdat);1699	pgdat_set_deferred_range(pgdat);1700 1701	free_area_init_core(pgdat);1702	lru_gen_init_pgdat(pgdat);1703}1704 1705/* Any regular or high memory on that node ? */1706static void __init check_for_memory(pg_data_t *pgdat)1707{1708	enum zone_type zone_type;1709 1710	for (zone_type = 0; zone_type <= ZONE_MOVABLE - 1; zone_type++) {1711		struct zone *zone = &pgdat->node_zones[zone_type];1712		if (populated_zone(zone)) {1713			if (IS_ENABLED(CONFIG_HIGHMEM))1714				node_set_state(pgdat->node_id, N_HIGH_MEMORY);1715			if (zone_type <= ZONE_NORMAL)1716				node_set_state(pgdat->node_id, N_NORMAL_MEMORY);1717			break;1718		}1719	}1720}1721 1722#if MAX_NUMNODES > 11723/*1724 * Figure out the number of possible node ids.1725 */1726void __init setup_nr_node_ids(void)1727{1728	unsigned int highest;1729 1730	highest = find_last_bit(node_possible_map.bits, MAX_NUMNODES);1731	nr_node_ids = highest + 1;1732}1733#endif1734 1735/*1736 * Some architectures, e.g. ARC may have ZONE_HIGHMEM below ZONE_NORMAL. For1737 * such cases we allow max_zone_pfn sorted in the descending order1738 */1739static bool arch_has_descending_max_zone_pfns(void)1740{1741	return IS_ENABLED(CONFIG_ARC) && !IS_ENABLED(CONFIG_ARC_HAS_PAE40);1742}1743 1744/**1745 * free_area_init - Initialise all pg_data_t and zone data1746 * @max_zone_pfn: an array of max PFNs for each zone1747 *1748 * This will call free_area_init_node() for each active node in the system.1749 * Using the page ranges provided by memblock_set_node(), the size of each1750 * zone in each node and their holes is calculated. If the maximum PFN1751 * between two adjacent zones match, it is assumed that the zone is empty.1752 * For example, if arch_max_dma_pfn == arch_max_dma32_pfn, it is assumed1753 * that arch_max_dma32_pfn has no pages. It is also assumed that a zone1754 * starts where the previous one ended. For example, ZONE_DMA32 starts1755 * at arch_max_dma_pfn.1756 */1757void __init free_area_init(unsigned long *max_zone_pfn)1758{1759	unsigned long start_pfn, end_pfn;1760	int i, nid, zone;1761	bool descending;1762 1763	/* Record where the zone boundaries are */1764	memset(arch_zone_lowest_possible_pfn, 0,1765				sizeof(arch_zone_lowest_possible_pfn));1766	memset(arch_zone_highest_possible_pfn, 0,1767				sizeof(arch_zone_highest_possible_pfn));1768 1769	start_pfn = PHYS_PFN(memblock_start_of_DRAM());1770	descending = arch_has_descending_max_zone_pfns();1771 1772	for (i = 0; i < MAX_NR_ZONES; i++) {1773		if (descending)1774			zone = MAX_NR_ZONES - i - 1;1775		else1776			zone = i;1777 1778		if (zone == ZONE_MOVABLE)1779			continue;1780 1781		end_pfn = max(max_zone_pfn[zone], start_pfn);1782		arch_zone_lowest_possible_pfn[zone] = start_pfn;1783		arch_zone_highest_possible_pfn[zone] = end_pfn;1784 1785		start_pfn = end_pfn;1786	}1787 1788	/* Find the PFNs that ZONE_MOVABLE begins at in each node */1789	memset(zone_movable_pfn, 0, sizeof(zone_movable_pfn));1790	find_zone_movable_pfns_for_nodes();1791 1792	/* Print out the zone ranges */1793	pr_info("Zone ranges:\n");1794	for (i = 0; i < MAX_NR_ZONES; i++) {1795		if (i == ZONE_MOVABLE)1796			continue;1797		pr_info("  %-8s ", zone_names[i]);1798		if (arch_zone_lowest_possible_pfn[i] ==1799				arch_zone_highest_possible_pfn[i])1800			pr_cont("empty\n");1801		else1802			pr_cont("[mem %#018Lx-%#018Lx]\n",1803				(u64)arch_zone_lowest_possible_pfn[i]1804					<< PAGE_SHIFT,1805				((u64)arch_zone_highest_possible_pfn[i]1806					<< PAGE_SHIFT) - 1);1807	}1808 1809	/* Print out the PFNs ZONE_MOVABLE begins at in each node */1810	pr_info("Movable zone start for each node\n");1811	for (i = 0; i < MAX_NUMNODES; i++) {1812		if (zone_movable_pfn[i])1813			pr_info("  Node %d: %#018Lx\n", i,1814			       (u64)zone_movable_pfn[i] << PAGE_SHIFT);1815	}1816 1817	/*1818	 * Print out the early node map, and initialize the1819	 * subsection-map relative to active online memory ranges to1820	 * enable future "sub-section" extensions of the memory map.1821	 */1822	pr_info("Early memory node ranges\n");1823	for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid) {1824		pr_info("  node %3d: [mem %#018Lx-%#018Lx]\n", nid,1825			(u64)start_pfn << PAGE_SHIFT,1826			((u64)end_pfn << PAGE_SHIFT) - 1);1827		subsection_map_init(start_pfn, end_pfn - start_pfn);1828	}1829 1830	/* Initialise every node */1831	mminit_verify_pageflags_layout();1832	setup_nr_node_ids();1833	set_pageblock_order();1834 1835	for_each_node(nid) {1836		pg_data_t *pgdat;1837 1838		if (!node_online(nid))1839			alloc_offline_node_data(nid);1840 1841		pgdat = NODE_DATA(nid);1842		free_area_init_node(nid);1843 1844		/*1845		 * No sysfs hierarcy will be created via register_one_node()1846		 *for memory-less node because here it's not marked as N_MEMORY1847		 *and won't be set online later. The benefit is userspace1848		 *program won't be confused by sysfs files/directories of1849		 *memory-less node. The pgdat will get fully initialized by1850		 *hotadd_init_pgdat() when memory is hotplugged into this node.1851		 */1852		if (pgdat->node_present_pages) {1853			node_set_state(nid, N_MEMORY);1854			check_for_memory(pgdat);1855		}1856	}1857 1858	calc_nr_kernel_pages();1859	memmap_init();1860 1861	/* disable hash distribution for systems with a single node */1862	fixup_hashdist();1863}1864 1865/**1866 * node_map_pfn_alignment - determine the maximum internode alignment1867 *1868 * This function should be called after node map is populated and sorted.1869 * It calculates the maximum power of two alignment which can distinguish1870 * all the nodes.1871 *1872 * For example, if all nodes are 1GiB and aligned to 1GiB, the return value1873 * would indicate 1GiB alignment with (1 << (30 - PAGE_SHIFT)).  If the1874 * nodes are shifted by 256MiB, 256MiB.  Note that if only the last node is1875 * shifted, 1GiB is enough and this function will indicate so.1876 *1877 * This is used to test whether pfn -> nid mapping of the chosen memory1878 * model has fine enough granularity to avoid incorrect mapping for the1879 * populated node map.1880 *1881 * Return: the determined alignment in pfn's.  0 if there is no alignment1882 * requirement (single node).1883 */1884unsigned long __init node_map_pfn_alignment(void)1885{1886	unsigned long accl_mask = 0, last_end = 0;1887	unsigned long start, end, mask;1888	int last_nid = NUMA_NO_NODE;1889	int i, nid;1890 1891	for_each_mem_pfn_range(i, MAX_NUMNODES, &start, &end, &nid) {1892		if (!start || last_nid < 0 || last_nid == nid) {1893			last_nid = nid;1894			last_end = end;1895			continue;1896		}1897 1898		/*1899		 * Start with a mask granular enough to pin-point to the1900		 * start pfn and tick off bits one-by-one until it becomes1901		 * too coarse to separate the current node from the last.1902		 */1903		mask = ~((1 << __ffs(start)) - 1);1904		while (mask && last_end <= (start & (mask << 1)))1905			mask <<= 1;1906 1907		/* accumulate all internode masks */1908		accl_mask |= mask;1909	}1910 1911	/* convert mask to number of pages */1912	return ~accl_mask + 1;1913}1914 1915#ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT1916static void __init deferred_free_pages(unsigned long pfn,1917		unsigned long nr_pages)1918{1919	struct page *page;1920	unsigned long i;1921 1922	if (!nr_pages)1923		return;1924 1925	page = pfn_to_page(pfn);1926 1927	/* Free a large naturally-aligned chunk if possible */1928	if (nr_pages == MAX_ORDER_NR_PAGES && IS_MAX_ORDER_ALIGNED(pfn)) {1929		for (i = 0; i < nr_pages; i += pageblock_nr_pages)1930			set_pageblock_migratetype(page + i, MIGRATE_MOVABLE);1931		__free_pages_core(page, MAX_PAGE_ORDER, MEMINIT_EARLY);1932		return;1933	}1934 1935	/* Accept chunks smaller than MAX_PAGE_ORDER upfront */1936	accept_memory(PFN_PHYS(pfn), nr_pages * PAGE_SIZE);1937 1938	for (i = 0; i < nr_pages; i++, page++, pfn++) {1939		if (pageblock_aligned(pfn))1940			set_pageblock_migratetype(page, MIGRATE_MOVABLE);1941		__free_pages_core(page, 0, MEMINIT_EARLY);1942	}1943}1944 1945/* Completion tracking for deferred_init_memmap() threads */1946static atomic_t pgdat_init_n_undone __initdata;1947static __initdata DECLARE_COMPLETION(pgdat_init_all_done_comp);1948 1949static inline void __init pgdat_init_report_one_done(void)1950{1951	if (atomic_dec_and_test(&pgdat_init_n_undone))1952		complete(&pgdat_init_all_done_comp);1953}1954 1955/*1956 * Initialize struct pages.  We minimize pfn page lookups and scheduler checks1957 * by performing it only once every MAX_ORDER_NR_PAGES.1958 * Return number of pages initialized.1959 */1960static unsigned long __init deferred_init_pages(struct zone *zone,1961		unsigned long pfn, unsigned long end_pfn)1962{1963	int nid = zone_to_nid(zone);1964	unsigned long nr_pages = end_pfn - pfn;1965	int zid = zone_idx(zone);1966	struct page *page = pfn_to_page(pfn);1967 1968	for (; pfn < end_pfn; pfn++, page++)1969		__init_single_page(page, pfn, zid, nid);1970	return nr_pages;1971}1972 1973/*1974 * This function is meant to pre-load the iterator for the zone init from1975 * a given point.1976 * Specifically it walks through the ranges starting with initial index1977 * passed to it until we are caught up to the first_init_pfn value and1978 * exits there. If we never encounter the value we return false indicating1979 * there are no valid ranges left.1980 */1981static bool __init1982deferred_init_mem_pfn_range_in_zone(u64 *i, struct zone *zone,1983				    unsigned long *spfn, unsigned long *epfn,1984				    unsigned long first_init_pfn)1985{1986	u64 j = *i;1987 1988	if (j == 0)1989		__next_mem_pfn_range_in_zone(&j, zone, spfn, epfn);1990 1991	/*1992	 * Start out by walking through the ranges in this zone that have1993	 * already been initialized. We don't need to do anything with them1994	 * so we just need to flush them out of the system.1995	 */1996	for_each_free_mem_pfn_range_in_zone_from(j, zone, spfn, epfn) {1997		if (*epfn <= first_init_pfn)1998			continue;1999		if (*spfn < first_init_pfn)2000			*spfn = first_init_pfn;2001		*i = j;2002		return true;2003	}2004 2005	return false;2006}2007 2008/*2009 * Initialize and free pages. We do it in two loops: first we initialize2010 * struct page, then free to buddy allocator, because while we are2011 * freeing pages we can access pages that are ahead (computing buddy2012 * page in __free_one_page()).2013 *2014 * In order to try and keep some memory in the cache we have the loop2015 * broken along max page order boundaries. This way we will not cause2016 * any issues with the buddy page computation.2017 */2018static unsigned long __init2019deferred_init_maxorder(u64 *i, struct zone *zone, unsigned long *start_pfn,2020		       unsigned long *end_pfn)2021{2022	unsigned long mo_pfn = ALIGN(*start_pfn + 1, MAX_ORDER_NR_PAGES);2023	unsigned long spfn = *start_pfn, epfn = *end_pfn;2024	unsigned long nr_pages = 0;2025	u64 j = *i;2026 2027	/* First we loop through and initialize the page values */2028	for_each_free_mem_pfn_range_in_zone_from(j, zone, start_pfn, end_pfn) {2029		unsigned long t;2030 2031		if (mo_pfn <= *start_pfn)2032			break;2033 2034		t = min(mo_pfn, *end_pfn);2035		nr_pages += deferred_init_pages(zone, *start_pfn, t);2036 2037		if (mo_pfn < *end_pfn) {2038			*start_pfn = mo_pfn;2039			break;2040		}2041	}2042 2043	/* Reset values and now loop through freeing pages as needed */2044	swap(j, *i);2045 2046	for_each_free_mem_pfn_range_in_zone_from(j, zone, &spfn, &epfn) {2047		unsigned long t;2048 2049		if (mo_pfn <= spfn)2050			break;2051 2052		t = min(mo_pfn, epfn);2053		deferred_free_pages(spfn, t - spfn);2054 2055		if (mo_pfn <= epfn)2056			break;2057	}2058 2059	return nr_pages;2060}2061 2062static void __init2063deferred_init_memmap_chunk(unsigned long start_pfn, unsigned long end_pfn,2064			   void *arg)2065{2066	unsigned long spfn, epfn;2067	struct zone *zone = arg;2068	u64 i = 0;2069 2070	deferred_init_mem_pfn_range_in_zone(&i, zone, &spfn, &epfn, start_pfn);2071 2072	/*2073	 * Initialize and free pages in MAX_PAGE_ORDER sized increments so that2074	 * we can avoid introducing any issues with the buddy allocator.2075	 */2076	while (spfn < end_pfn) {2077		deferred_init_maxorder(&i, zone, &spfn, &epfn);2078		cond_resched();2079	}2080}2081 2082static unsigned int __init2083deferred_page_init_max_threads(const struct cpumask *node_cpumask)2084{2085	return max(cpumask_weight(node_cpumask), 1U);2086}2087 2088/* Initialise remaining memory on a node */2089static int __init deferred_init_memmap(void *data)2090{2091	pg_data_t *pgdat = data;2092	const struct cpumask *cpumask = cpumask_of_node(pgdat->node_id);2093	unsigned long spfn = 0, epfn = 0;2094	unsigned long first_init_pfn, flags;2095	unsigned long start = jiffies;2096	struct zone *zone;2097	int max_threads;2098	u64 i = 0;2099 2100	/* Bind memory initialisation thread to a local node if possible */2101	if (!cpumask_empty(cpumask))2102		set_cpus_allowed_ptr(current, cpumask);2103 2104	pgdat_resize_lock(pgdat, &flags);2105	first_init_pfn = pgdat->first_deferred_pfn;2106	if (first_init_pfn == ULONG_MAX) {2107		pgdat_resize_unlock(pgdat, &flags);2108		pgdat_init_report_one_done();2109		return 0;2110	}2111 2112	/* Sanity check boundaries */2113	BUG_ON(pgdat->first_deferred_pfn < pgdat->node_start_pfn);2114	BUG_ON(pgdat->first_deferred_pfn > pgdat_end_pfn(pgdat));2115	pgdat->first_deferred_pfn = ULONG_MAX;2116 2117	/*2118	 * Once we unlock here, the zone cannot be grown anymore, thus if an2119	 * interrupt thread must allocate this early in boot, zone must be2120	 * pre-grown prior to start of deferred page initialization.2121	 */2122	pgdat_resize_unlock(pgdat, &flags);2123 2124	/* Only the highest zone is deferred */2125	zone = pgdat->node_zones + pgdat->nr_zones - 1;2126 2127	max_threads = deferred_page_init_max_threads(cpumask);2128 2129	while (deferred_init_mem_pfn_range_in_zone(&i, zone, &spfn, &epfn, first_init_pfn)) {2130		first_init_pfn = ALIGN(epfn, PAGES_PER_SECTION);2131		struct padata_mt_job job = {2132			.thread_fn   = deferred_init_memmap_chunk,2133			.fn_arg      = zone,2134			.start       = spfn,2135			.size        = first_init_pfn - spfn,2136			.align       = PAGES_PER_SECTION,2137			.min_chunk   = PAGES_PER_SECTION,2138			.max_threads = max_threads,2139			.numa_aware  = false,2140		};2141 2142		padata_do_multithreaded(&job);2143	}2144 2145	/* Sanity check that the next zone really is unpopulated */2146	WARN_ON(pgdat->nr_zones < MAX_NR_ZONES && populated_zone(++zone));2147 2148	pr_info("node %d deferred pages initialised in %ums\n",2149		pgdat->node_id, jiffies_to_msecs(jiffies - start));2150 2151	pgdat_init_report_one_done();2152	return 0;2153}2154 2155/*2156 * If this zone has deferred pages, try to grow it by initializing enough2157 * deferred pages to satisfy the allocation specified by order, rounded up to2158 * the nearest PAGES_PER_SECTION boundary.  So we're adding memory in increments2159 * of SECTION_SIZE bytes by initializing struct pages in increments of2160 * PAGES_PER_SECTION * sizeof(struct page) bytes.2161 *2162 * Return true when zone was grown, otherwise return false. We return true even2163 * when we grow less than requested, to let the caller decide if there are2164 * enough pages to satisfy the allocation.2165 */2166bool __init deferred_grow_zone(struct zone *zone, unsigned int order)2167{2168	unsigned long nr_pages_needed = ALIGN(1 << order, PAGES_PER_SECTION);2169	pg_data_t *pgdat = zone->zone_pgdat;2170	unsigned long first_deferred_pfn = pgdat->first_deferred_pfn;2171	unsigned long spfn, epfn, flags;2172	unsigned long nr_pages = 0;2173	u64 i = 0;2174 2175	/* Only the last zone may have deferred pages */2176	if (zone_end_pfn(zone) != pgdat_end_pfn(pgdat))2177		return false;2178 2179	pgdat_resize_lock(pgdat, &flags);2180 2181	/*2182	 * If someone grew this zone while we were waiting for spinlock, return2183	 * true, as there might be enough pages already.2184	 */2185	if (first_deferred_pfn != pgdat->first_deferred_pfn) {2186		pgdat_resize_unlock(pgdat, &flags);2187		return true;2188	}2189 2190	/* If the zone is empty somebody else may have cleared out the zone */2191	if (!deferred_init_mem_pfn_range_in_zone(&i, zone, &spfn, &epfn,2192						 first_deferred_pfn)) {2193		pgdat->first_deferred_pfn = ULONG_MAX;2194		pgdat_resize_unlock(pgdat, &flags);2195		/* Retry only once. */2196		return first_deferred_pfn != ULONG_MAX;2197	}2198 2199	/*2200	 * Initialize and free pages in MAX_PAGE_ORDER sized increments so2201	 * that we can avoid introducing any issues with the buddy2202	 * allocator.2203	 */2204	while (spfn < epfn) {2205		/* update our first deferred PFN for this section */2206		first_deferred_pfn = spfn;2207 2208		nr_pages += deferred_init_maxorder(&i, zone, &spfn, &epfn);2209		touch_nmi_watchdog();2210 2211		/* We should only stop along section boundaries */2212		if ((first_deferred_pfn ^ spfn) < PAGES_PER_SECTION)2213			continue;2214 2215		/* If our quota has been met we can stop here */2216		if (nr_pages >= nr_pages_needed)2217			break;2218	}2219 2220	pgdat->first_deferred_pfn = spfn;2221	pgdat_resize_unlock(pgdat, &flags);2222 2223	return nr_pages > 0;2224}2225 2226#endif /* CONFIG_DEFERRED_STRUCT_PAGE_INIT */2227 2228#ifdef CONFIG_CMA2229void __init init_cma_reserved_pageblock(struct page *page)2230{2231	unsigned i = pageblock_nr_pages;2232	struct page *p = page;2233 2234	do {2235		__ClearPageReserved(p);2236		set_page_count(p, 0);2237	} while (++p, --i);2238 2239	set_pageblock_migratetype(page, MIGRATE_CMA);2240	set_page_refcounted(page);2241	/* pages were reserved and not allocated */2242	clear_page_tag_ref(page);2243	__free_pages(page, pageblock_order);2244 2245	adjust_managed_page_count(page, pageblock_nr_pages);2246	page_zone(page)->cma_pages += pageblock_nr_pages;2247}2248#endif2249 2250void set_zone_contiguous(struct zone *zone)2251{2252	unsigned long block_start_pfn = zone->zone_start_pfn;2253	unsigned long block_end_pfn;2254 2255	block_end_pfn = pageblock_end_pfn(block_start_pfn);2256	for (; block_start_pfn < zone_end_pfn(zone);2257			block_start_pfn = block_end_pfn,2258			 block_end_pfn += pageblock_nr_pages) {2259 2260		block_end_pfn = min(block_end_pfn, zone_end_pfn(zone));2261 2262		if (!__pageblock_pfn_to_page(block_start_pfn,2263					     block_end_pfn, zone))2264			return;2265		cond_resched();2266	}2267 2268	/* We confirm that there is no hole */2269	zone->contiguous = true;2270}2271 2272static void __init mem_init_print_info(void);2273void __init page_alloc_init_late(void)2274{2275	struct zone *zone;2276	int nid;2277 2278#ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT2279 2280	/* There will be num_node_state(N_MEMORY) threads */2281	atomic_set(&pgdat_init_n_undone, num_node_state(N_MEMORY));2282	for_each_node_state(nid, N_MEMORY) {2283		kthread_run(deferred_init_memmap, NODE_DATA(nid), "pgdatinit%d", nid);2284	}2285 2286	/* Block until all are initialised */2287	wait_for_completion(&pgdat_init_all_done_comp);2288 2289	/*2290	 * We initialized the rest of the deferred pages.  Permanently disable2291	 * on-demand struct page initialization.2292	 */2293	static_branch_disable(&deferred_pages);2294 2295	/* Reinit limits that are based on free pages after the kernel is up */2296	files_maxfiles_init();2297#endif2298 2299	/* Accounting of total+free memory is stable at this point. */2300	mem_init_print_info();2301	buffer_init();2302 2303	/* Discard memblock private memory */2304	memblock_discard();2305 2306	for_each_node_state(nid, N_MEMORY)2307		shuffle_free_memory(NODE_DATA(nid));2308 2309	for_each_populated_zone(zone)2310		set_zone_contiguous(zone);2311 2312	/* Initialize page ext after all struct pages are initialized. */2313	if (deferred_struct_pages)2314		page_ext_init();2315 2316	page_alloc_sysctl_init();2317}2318 2319/*2320 * Adaptive scale is meant to reduce sizes of hash tables on large memory2321 * machines. As memory size is increased the scale is also increased but at2322 * slower pace.  Starting from ADAPT_SCALE_BASE (64G), every time memory2323 * quadruples the scale is increased by one, which means the size of hash table2324 * only doubles, instead of quadrupling as well.2325 * Because 32-bit systems cannot have large physical memory, where this scaling2326 * makes sense, it is disabled on such platforms.2327 */2328#if __BITS_PER_LONG > 322329#define ADAPT_SCALE_BASE	(64ul << 30)2330#define ADAPT_SCALE_SHIFT	22331#define ADAPT_SCALE_NPAGES	(ADAPT_SCALE_BASE >> PAGE_SHIFT)2332#endif2333 2334/*2335 * allocate a large system hash table from bootmem2336 * - it is assumed that the hash table must contain an exact power-of-22337 *   quantity of entries2338 * - limit is the number of hash buckets, not the total allocation size2339 */2340void *__init alloc_large_system_hash(const char *tablename,2341				     unsigned long bucketsize,2342				     unsigned long numentries,2343				     int scale,2344				     int flags,2345				     unsigned int *_hash_shift,2346				     unsigned int *_hash_mask,2347				     unsigned long low_limit,2348				     unsigned long high_limit)2349{2350	unsigned long long max = high_limit;2351	unsigned long log2qty, size;2352	void *table;2353	gfp_t gfp_flags;2354	bool virt;2355	bool huge;2356 2357	/* allow the kernel cmdline to have a say */2358	if (!numentries) {2359		/* round applicable memory size up to nearest megabyte */2360		numentries = nr_kernel_pages;2361 2362		/* It isn't necessary when PAGE_SIZE >= 1MB */2363		if (PAGE_SIZE < SZ_1M)2364			numentries = round_up(numentries, SZ_1M / PAGE_SIZE);2365 2366#if __BITS_PER_LONG > 322367		if (!high_limit) {2368			unsigned long adapt;2369 2370			for (adapt = ADAPT_SCALE_NPAGES; adapt < numentries;2371			     adapt <<= ADAPT_SCALE_SHIFT)2372				scale++;2373		}2374#endif2375 2376		/* limit to 1 bucket per 2^scale bytes of low memory */2377		if (scale > PAGE_SHIFT)2378			numentries >>= (scale - PAGE_SHIFT);2379		else2380			numentries <<= (PAGE_SHIFT - scale);2381 2382		if (unlikely((numentries * bucketsize) < PAGE_SIZE))2383			numentries = PAGE_SIZE / bucketsize;2384	}2385	numentries = roundup_pow_of_two(numentries);2386 2387	/* limit allocation size to 1/16 total memory by default */2388	if (max == 0) {2389		max = ((unsigned long long)nr_all_pages << PAGE_SHIFT) >> 4;2390		do_div(max, bucketsize);2391	}2392	max = min(max, 0x80000000ULL);2393 2394	if (numentries < low_limit)2395		numentries = low_limit;2396	if (numentries > max)2397		numentries = max;2398 2399	log2qty = ilog2(numentries);2400 2401	gfp_flags = (flags & HASH_ZERO) ? GFP_ATOMIC | __GFP_ZERO : GFP_ATOMIC;2402	do {2403		virt = false;2404		size = bucketsize << log2qty;2405		if (flags & HASH_EARLY) {2406			if (flags & HASH_ZERO)2407				table = memblock_alloc(size, SMP_CACHE_BYTES);2408			else2409				table = memblock_alloc_raw(size,2410							   SMP_CACHE_BYTES);2411		} else if (get_order(size) > MAX_PAGE_ORDER || hashdist) {2412			table = vmalloc_huge(size, gfp_flags);2413			virt = true;2414			if (table)2415				huge = is_vm_area_hugepages(table);2416		} else {2417			/*2418			 * If bucketsize is not a power-of-two, we may free2419			 * some pages at the end of hash table which2420			 * alloc_pages_exact() automatically does2421			 */2422			table = alloc_pages_exact(size, gfp_flags);2423			kmemleak_alloc(table, size, 1, gfp_flags);2424		}2425	} while (!table && size > PAGE_SIZE && --log2qty);2426 2427	if (!table)2428		panic("Failed to allocate %s hash table\n", tablename);2429 2430	pr_info("%s hash table entries: %ld (order: %d, %lu bytes, %s)\n",2431		tablename, 1UL << log2qty, ilog2(size) - PAGE_SHIFT, size,2432		virt ? (huge ? "vmalloc hugepage" : "vmalloc") : "linear");2433 2434	if (_hash_shift)2435		*_hash_shift = log2qty;2436	if (_hash_mask)2437		*_hash_mask = (1 << log2qty) - 1;2438 2439	return table;2440}2441 2442void __init memblock_free_pages(struct page *page, unsigned long pfn,2443							unsigned int order)2444{2445	if (IS_ENABLED(CONFIG_DEFERRED_STRUCT_PAGE_INIT)) {2446		int nid = early_pfn_to_nid(pfn);2447 2448		if (!early_page_initialised(pfn, nid))2449			return;2450	}2451 2452	if (!kmsan_memblock_free_pages(page, order)) {2453		/* KMSAN will take care of these pages. */2454		return;2455	}2456 2457	/* pages were reserved and not allocated */2458	clear_page_tag_ref(page);2459	__free_pages_core(page, order, MEMINIT_EARLY);2460}2461 2462DEFINE_STATIC_KEY_MAYBE(CONFIG_INIT_ON_ALLOC_DEFAULT_ON, init_on_alloc);2463EXPORT_SYMBOL(init_on_alloc);2464 2465DEFINE_STATIC_KEY_MAYBE(CONFIG_INIT_ON_FREE_DEFAULT_ON, init_on_free);2466EXPORT_SYMBOL(init_on_free);2467 2468static bool _init_on_alloc_enabled_early __read_mostly2469				= IS_ENABLED(CONFIG_INIT_ON_ALLOC_DEFAULT_ON);2470static int __init early_init_on_alloc(char *buf)2471{2472 2473	return kstrtobool(buf, &_init_on_alloc_enabled_early);2474}2475early_param("init_on_alloc", early_init_on_alloc);2476 2477static bool _init_on_free_enabled_early __read_mostly2478				= IS_ENABLED(CONFIG_INIT_ON_FREE_DEFAULT_ON);2479static int __init early_init_on_free(char *buf)2480{2481	return kstrtobool(buf, &_init_on_free_enabled_early);2482}2483early_param("init_on_free", early_init_on_free);2484 2485DEFINE_STATIC_KEY_MAYBE(CONFIG_DEBUG_VM, check_pages_enabled);2486 2487/*2488 * Enable static keys related to various memory debugging and hardening options.2489 * Some override others, and depend on early params that are evaluated in the2490 * order of appearance. So we need to first gather the full picture of what was2491 * enabled, and then make decisions.2492 */2493static void __init mem_debugging_and_hardening_init(void)2494{2495	bool page_poisoning_requested = false;2496	bool want_check_pages = false;2497 2498#ifdef CONFIG_PAGE_POISONING2499	/*2500	 * Page poisoning is debug page alloc for some arches. If2501	 * either of those options are enabled, enable poisoning.2502	 */2503	if (page_poisoning_enabled() ||2504	     (!IS_ENABLED(CONFIG_ARCH_SUPPORTS_DEBUG_PAGEALLOC) &&2505	      debug_pagealloc_enabled())) {2506		static_branch_enable(&_page_poisoning_enabled);2507		page_poisoning_requested = true;2508		want_check_pages = true;2509	}2510#endif2511 2512	if ((_init_on_alloc_enabled_early || _init_on_free_enabled_early) &&2513	    page_poisoning_requested) {2514		pr_info("mem auto-init: CONFIG_PAGE_POISONING is on, "2515			"will take precedence over init_on_alloc and init_on_free\n");2516		_init_on_alloc_enabled_early = false;2517		_init_on_free_enabled_early = false;2518	}2519 2520	if (_init_on_alloc_enabled_early) {2521		want_check_pages = true;2522		static_branch_enable(&init_on_alloc);2523	} else {2524		static_branch_disable(&init_on_alloc);2525	}2526 2527	if (_init_on_free_enabled_early) {2528		want_check_pages = true;2529		static_branch_enable(&init_on_free);2530	} else {2531		static_branch_disable(&init_on_free);2532	}2533 2534	if (IS_ENABLED(CONFIG_KMSAN) &&2535	    (_init_on_alloc_enabled_early || _init_on_free_enabled_early))2536		pr_info("mem auto-init: please make sure init_on_alloc and init_on_free are disabled when running KMSAN\n");2537 2538#ifdef CONFIG_DEBUG_PAGEALLOC2539	if (debug_pagealloc_enabled()) {2540		want_check_pages = true;2541		static_branch_enable(&_debug_pagealloc_enabled);2542 2543		if (debug_guardpage_minorder())2544			static_branch_enable(&_debug_guardpage_enabled);2545	}2546#endif2547 2548	/*2549	 * Any page debugging or hardening option also enables sanity checking2550	 * of struct pages being allocated or freed. With CONFIG_DEBUG_VM it's2551	 * enabled already.2552	 */2553	if (!IS_ENABLED(CONFIG_DEBUG_VM) && want_check_pages)2554		static_branch_enable(&check_pages_enabled);2555}2556 2557/* Report memory auto-initialization states for this boot. */2558static void __init report_meminit(void)2559{2560	const char *stack;2561 2562	if (IS_ENABLED(CONFIG_INIT_STACK_ALL_PATTERN))2563		stack = "all(pattern)";2564	else if (IS_ENABLED(CONFIG_INIT_STACK_ALL_ZERO))2565		stack = "all(zero)";2566	else if (IS_ENABLED(CONFIG_GCC_PLUGIN_STRUCTLEAK_BYREF_ALL))2567		stack = "byref_all(zero)";2568	else if (IS_ENABLED(CONFIG_GCC_PLUGIN_STRUCTLEAK_BYREF))2569		stack = "byref(zero)";2570	else if (IS_ENABLED(CONFIG_GCC_PLUGIN_STRUCTLEAK_USER))2571		stack = "__user(zero)";2572	else2573		stack = "off";2574 2575	pr_info("mem auto-init: stack:%s, heap alloc:%s, heap free:%s\n",2576		stack, want_init_on_alloc(GFP_KERNEL) ? "on" : "off",2577		want_init_on_free() ? "on" : "off");2578	if (want_init_on_free())2579		pr_info("mem auto-init: clearing system memory may take some time...\n");2580}2581 2582static void __init mem_init_print_info(void)2583{2584	unsigned long physpages, codesize, datasize, rosize, bss_size;2585	unsigned long init_code_size, init_data_size;2586 2587	physpages = get_num_physpages();2588	codesize = _etext - _stext;2589	datasize = _edata - _sdata;2590	rosize = __end_rodata - __start_rodata;2591	bss_size = __bss_stop - __bss_start;2592	init_data_size = __init_end - __init_begin;2593	init_code_size = _einittext - _sinittext;2594 2595	/*2596	 * Detect special cases and adjust section sizes accordingly:2597	 * 1) .init.* may be embedded into .data sections2598	 * 2) .init.text.* may be out of [__init_begin, __init_end],2599	 *    please refer to arch/tile/kernel/vmlinux.lds.S.2600	 * 3) .rodata.* may be embedded into .text or .data sections.2601	 */2602#define adj_init_size(start, end, size, pos, adj) \2603	do { \2604		if (&start[0] <= &pos[0] && &pos[0] < &end[0] && size > adj) \2605			size -= adj; \2606	} while (0)2607 2608	adj_init_size(__init_begin, __init_end, init_data_size,2609		     _sinittext, init_code_size);2610	adj_init_size(_stext, _etext, codesize, _sinittext, init_code_size);2611	adj_init_size(_sdata, _edata, datasize, __init_begin, init_data_size);2612	adj_init_size(_stext, _etext, codesize, __start_rodata, rosize);2613	adj_init_size(_sdata, _edata, datasize, __start_rodata, rosize);2614 2615#undef	adj_init_size2616 2617	pr_info("Memory: %luK/%luK available (%luK kernel code, %luK rwdata, %luK rodata, %luK init, %luK bss, %luK reserved, %luK cma-reserved"2618#ifdef	CONFIG_HIGHMEM2619		", %luK highmem"2620#endif2621		")\n",2622		K(nr_free_pages()), K(physpages),2623		codesize / SZ_1K, datasize / SZ_1K, rosize / SZ_1K,2624		(init_data_size + init_code_size) / SZ_1K, bss_size / SZ_1K,2625		K(physpages - totalram_pages() - totalcma_pages),2626		K(totalcma_pages)2627#ifdef	CONFIG_HIGHMEM2628		, K(totalhigh_pages())2629#endif2630		);2631}2632 2633/*2634 * Set up kernel memory allocators2635 */2636void __init mm_core_init(void)2637{2638	/* Initializations relying on SMP setup */2639	BUILD_BUG_ON(MAX_ZONELISTS > 2);2640	build_all_zonelists(NULL);2641	page_alloc_init_cpuhp();2642 2643	/*2644	 * page_ext requires contiguous pages,2645	 * bigger than MAX_PAGE_ORDER unless SPARSEMEM.2646	 */2647	page_ext_init_flatmem();2648	mem_debugging_and_hardening_init();2649	kfence_alloc_pool_and_metadata();2650	report_meminit();2651	kmsan_init_shadow();2652	stack_depot_early_init();2653	mem_init();2654	kmem_cache_init();2655	/*2656	 * page_owner must be initialized after buddy is ready, and also after2657	 * slab is ready so that stack_depot_init() works properly2658	 */2659	page_ext_init_flatmem_late();2660	kmemleak_init();2661	ptlock_cache_init();2662	pgtable_cache_init();2663	debug_objects_mem_init();2664	vmalloc_init();2665	/* If no deferred init page_ext now, as vmap is fully initialized */2666	if (!deferred_struct_pages)2667		page_ext_init();2668	/* Should be run before the first non-init thread is created */2669	init_espfix_bsp();2670	/* Should be run after espfix64 is set up. */2671	pti_init();2672	kmsan_init_runtime();2673	mm_cache_init();2674	execmem_init();2675}2676