2435 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2/*3 * linux/mm/memory_hotplug.c4 *5 * Copyright (C)6 */7 8#include <linux/stddef.h>9#include <linux/mm.h>10#include <linux/sched/signal.h>11#include <linux/swap.h>12#include <linux/interrupt.h>13#include <linux/pagemap.h>14#include <linux/compiler.h>15#include <linux/export.h>16#include <linux/writeback.h>17#include <linux/slab.h>18#include <linux/sysctl.h>19#include <linux/cpu.h>20#include <linux/memory.h>21#include <linux/memremap.h>22#include <linux/memory_hotplug.h>23#include <linux/vmalloc.h>24#include <linux/ioport.h>25#include <linux/delay.h>26#include <linux/migrate.h>27#include <linux/page-isolation.h>28#include <linux/pfn.h>29#include <linux/suspend.h>30#include <linux/mm_inline.h>31#include <linux/firmware-map.h>32#include <linux/stop_machine.h>33#include <linux/hugetlb.h>34#include <linux/memblock.h>35#include <linux/compaction.h>36#include <linux/rmap.h>37#include <linux/module.h>38 39#include <asm/tlbflush.h>40 41#include "internal.h"42#include "shuffle.h"43 44enum {45 MEMMAP_ON_MEMORY_DISABLE = 0,46 MEMMAP_ON_MEMORY_ENABLE,47 MEMMAP_ON_MEMORY_FORCE,48};49 50static int memmap_mode __read_mostly = MEMMAP_ON_MEMORY_DISABLE;51 52static inline unsigned long memory_block_memmap_size(void)53{54 return PHYS_PFN(memory_block_size_bytes()) * sizeof(struct page);55}56 57static inline unsigned long memory_block_memmap_on_memory_pages(void)58{59 unsigned long nr_pages = PFN_UP(memory_block_memmap_size());60 61 /*62 * In "forced" memmap_on_memory mode, we add extra pages to align the63 * vmemmap size to cover full pageblocks. That way, we can add memory64 * even if the vmemmap size is not properly aligned, however, we might waste65 * memory.66 */67 if (memmap_mode == MEMMAP_ON_MEMORY_FORCE)68 return pageblock_align(nr_pages);69 return nr_pages;70}71 72#ifdef CONFIG_MHP_MEMMAP_ON_MEMORY73/*74 * memory_hotplug.memmap_on_memory parameter75 */76static int set_memmap_mode(const char *val, const struct kernel_param *kp)77{78 int ret, mode;79 bool enabled;80 81 if (sysfs_streq(val, "force") || sysfs_streq(val, "FORCE")) {82 mode = MEMMAP_ON_MEMORY_FORCE;83 } else {84 ret = kstrtobool(val, &enabled);85 if (ret < 0)86 return ret;87 if (enabled)88 mode = MEMMAP_ON_MEMORY_ENABLE;89 else90 mode = MEMMAP_ON_MEMORY_DISABLE;91 }92 *((int *)kp->arg) = mode;93 if (mode == MEMMAP_ON_MEMORY_FORCE) {94 unsigned long memmap_pages = memory_block_memmap_on_memory_pages();95 96 pr_info_once("Memory hotplug will waste %ld pages in each memory block\n",97 memmap_pages - PFN_UP(memory_block_memmap_size()));98 }99 return 0;100}101 102static int get_memmap_mode(char *buffer, const struct kernel_param *kp)103{104 int mode = *((int *)kp->arg);105 106 if (mode == MEMMAP_ON_MEMORY_FORCE)107 return sprintf(buffer, "force\n");108 return sprintf(buffer, "%c\n", mode ? 'Y' : 'N');109}110 111static const struct kernel_param_ops memmap_mode_ops = {112 .set = set_memmap_mode,113 .get = get_memmap_mode,114};115module_param_cb(memmap_on_memory, &memmap_mode_ops, &memmap_mode, 0444);116MODULE_PARM_DESC(memmap_on_memory, "Enable memmap on memory for memory hotplug\n"117 "With value \"force\" it could result in memory wastage due "118 "to memmap size limitations (Y/N/force)");119 120static inline bool mhp_memmap_on_memory(void)121{122 return memmap_mode != MEMMAP_ON_MEMORY_DISABLE;123}124#else125static inline bool mhp_memmap_on_memory(void)126{127 return false;128}129#endif130 131enum {132 ONLINE_POLICY_CONTIG_ZONES = 0,133 ONLINE_POLICY_AUTO_MOVABLE,134};135 136static const char * const online_policy_to_str[] = {137 [ONLINE_POLICY_CONTIG_ZONES] = "contig-zones",138 [ONLINE_POLICY_AUTO_MOVABLE] = "auto-movable",139};140 141static int set_online_policy(const char *val, const struct kernel_param *kp)142{143 int ret = sysfs_match_string(online_policy_to_str, val);144 145 if (ret < 0)146 return ret;147 *((int *)kp->arg) = ret;148 return 0;149}150 151static int get_online_policy(char *buffer, const struct kernel_param *kp)152{153 return sprintf(buffer, "%s\n", online_policy_to_str[*((int *)kp->arg)]);154}155 156/*157 * memory_hotplug.online_policy: configure online behavior when onlining without158 * specifying a zone (MMOP_ONLINE)159 *160 * "contig-zones": keep zone contiguous161 * "auto-movable": online memory to ZONE_MOVABLE if the configuration162 * (auto_movable_ratio, auto_movable_numa_aware) allows for it163 */164static int online_policy __read_mostly = ONLINE_POLICY_CONTIG_ZONES;165static const struct kernel_param_ops online_policy_ops = {166 .set = set_online_policy,167 .get = get_online_policy,168};169module_param_cb(online_policy, &online_policy_ops, &online_policy, 0644);170MODULE_PARM_DESC(online_policy,171 "Set the online policy (\"contig-zones\", \"auto-movable\") "172 "Default: \"contig-zones\"");173 174/*175 * memory_hotplug.auto_movable_ratio: specify maximum MOVABLE:KERNEL ratio176 *177 * The ratio represent an upper limit and the kernel might decide to not178 * online some memory to ZONE_MOVABLE -- e.g., because hotplugged KERNEL memory179 * doesn't allow for more MOVABLE memory.180 */181static unsigned int auto_movable_ratio __read_mostly = 301;182module_param(auto_movable_ratio, uint, 0644);183MODULE_PARM_DESC(auto_movable_ratio,184 "Set the maximum ratio of MOVABLE:KERNEL memory in the system "185 "in percent for \"auto-movable\" online policy. Default: 301");186 187/*188 * memory_hotplug.auto_movable_numa_aware: consider numa node stats189 */190#ifdef CONFIG_NUMA191static bool auto_movable_numa_aware __read_mostly = true;192module_param(auto_movable_numa_aware, bool, 0644);193MODULE_PARM_DESC(auto_movable_numa_aware,194 "Consider numa node stats in addition to global stats in "195 "\"auto-movable\" online policy. Default: true");196#endif /* CONFIG_NUMA */197 198/*199 * online_page_callback contains pointer to current page onlining function.200 * Initially it is generic_online_page(). If it is required it could be201 * changed by calling set_online_page_callback() for callback registration202 * and restore_online_page_callback() for generic callback restore.203 */204 205static online_page_callback_t online_page_callback = generic_online_page;206static DEFINE_MUTEX(online_page_callback_lock);207 208DEFINE_STATIC_PERCPU_RWSEM(mem_hotplug_lock);209 210void get_online_mems(void)211{212 percpu_down_read(&mem_hotplug_lock);213}214 215void put_online_mems(void)216{217 percpu_up_read(&mem_hotplug_lock);218}219 220bool movable_node_enabled = false;221 222#ifndef CONFIG_MEMORY_HOTPLUG_DEFAULT_ONLINE223int mhp_default_online_type = MMOP_OFFLINE;224#else225int mhp_default_online_type = MMOP_ONLINE;226#endif227 228static int __init setup_memhp_default_state(char *str)229{230 const int online_type = mhp_online_type_from_str(str);231 232 if (online_type >= 0)233 mhp_default_online_type = online_type;234 235 return 1;236}237__setup("memhp_default_state=", setup_memhp_default_state);238 239void mem_hotplug_begin(void)240{241 cpus_read_lock();242 percpu_down_write(&mem_hotplug_lock);243}244 245void mem_hotplug_done(void)246{247 percpu_up_write(&mem_hotplug_lock);248 cpus_read_unlock();249}250 251u64 max_mem_size = U64_MAX;252 253/* add this memory to iomem resource */254static struct resource *register_memory_resource(u64 start, u64 size,255 const char *resource_name)256{257 struct resource *res;258 unsigned long flags = IORESOURCE_SYSTEM_RAM | IORESOURCE_BUSY;259 260 if (strcmp(resource_name, "System RAM"))261 flags |= IORESOURCE_SYSRAM_DRIVER_MANAGED;262 263 if (!mhp_range_allowed(start, size, true))264 return ERR_PTR(-E2BIG);265 266 /*267 * Make sure value parsed from 'mem=' only restricts memory adding268 * while booting, so that memory hotplug won't be impacted. Please269 * refer to document of 'mem=' in kernel-parameters.txt for more270 * details.271 */272 if (start + size > max_mem_size && system_state < SYSTEM_RUNNING)273 return ERR_PTR(-E2BIG);274 275 /*276 * Request ownership of the new memory range. This might be277 * a child of an existing resource that was present but278 * not marked as busy.279 */280 res = __request_region(&iomem_resource, start, size,281 resource_name, flags);282 283 if (!res) {284 pr_debug("Unable to reserve System RAM region: %016llx->%016llx\n",285 start, start + size);286 return ERR_PTR(-EEXIST);287 }288 return res;289}290 291static void release_memory_resource(struct resource *res)292{293 if (!res)294 return;295 release_resource(res);296 kfree(res);297}298 299static int check_pfn_span(unsigned long pfn, unsigned long nr_pages)300{301 /*302 * Disallow all operations smaller than a sub-section and only303 * allow operations smaller than a section for304 * SPARSEMEM_VMEMMAP. Note that check_hotplug_memory_range()305 * enforces a larger memory_block_size_bytes() granularity for306 * memory that will be marked online, so this check should only307 * fire for direct arch_{add,remove}_memory() users outside of308 * add_memory_resource().309 */310 unsigned long min_align;311 312 if (IS_ENABLED(CONFIG_SPARSEMEM_VMEMMAP))313 min_align = PAGES_PER_SUBSECTION;314 else315 min_align = PAGES_PER_SECTION;316 if (!IS_ALIGNED(pfn | nr_pages, min_align))317 return -EINVAL;318 return 0;319}320 321/*322 * Return page for the valid pfn only if the page is online. All pfn323 * walkers which rely on the fully initialized page->flags and others324 * should use this rather than pfn_valid && pfn_to_page325 */326struct page *pfn_to_online_page(unsigned long pfn)327{328 unsigned long nr = pfn_to_section_nr(pfn);329 struct dev_pagemap *pgmap;330 struct mem_section *ms;331 332 if (nr >= NR_MEM_SECTIONS)333 return NULL;334 335 ms = __nr_to_section(nr);336 if (!online_section(ms))337 return NULL;338 339 /*340 * Save some code text when online_section() +341 * pfn_section_valid() are sufficient.342 */343 if (IS_ENABLED(CONFIG_HAVE_ARCH_PFN_VALID) && !pfn_valid(pfn))344 return NULL;345 346 if (!pfn_section_valid(ms, pfn))347 return NULL;348 349 if (!online_device_section(ms))350 return pfn_to_page(pfn);351 352 /*353 * Slowpath: when ZONE_DEVICE collides with354 * ZONE_{NORMAL,MOVABLE} within the same section some pfns in355 * the section may be 'offline' but 'valid'. Only356 * get_dev_pagemap() can determine sub-section online status.357 */358 pgmap = get_dev_pagemap(pfn, NULL);359 put_dev_pagemap(pgmap);360 361 /* The presence of a pgmap indicates ZONE_DEVICE offline pfn */362 if (pgmap)363 return NULL;364 365 return pfn_to_page(pfn);366}367EXPORT_SYMBOL_GPL(pfn_to_online_page);368 369int __add_pages(int nid, unsigned long pfn, unsigned long nr_pages,370 struct mhp_params *params)371{372 const unsigned long end_pfn = pfn + nr_pages;373 unsigned long cur_nr_pages;374 int err;375 struct vmem_altmap *altmap = params->altmap;376 377 if (WARN_ON_ONCE(!pgprot_val(params->pgprot)))378 return -EINVAL;379 380 VM_BUG_ON(!mhp_range_allowed(PFN_PHYS(pfn), nr_pages * PAGE_SIZE, false));381 382 if (altmap) {383 /*384 * Validate altmap is within bounds of the total request385 */386 if (altmap->base_pfn != pfn387 || vmem_altmap_offset(altmap) > nr_pages) {388 pr_warn_once("memory add fail, invalid altmap\n");389 return -EINVAL;390 }391 altmap->alloc = 0;392 }393 394 if (check_pfn_span(pfn, nr_pages)) {395 WARN(1, "Misaligned %s start: %#lx end: %#lx\n", __func__, pfn, pfn + nr_pages - 1);396 return -EINVAL;397 }398 399 for (; pfn < end_pfn; pfn += cur_nr_pages) {400 /* Select all remaining pages up to the next section boundary */401 cur_nr_pages = min(end_pfn - pfn,402 SECTION_ALIGN_UP(pfn + 1) - pfn);403 err = sparse_add_section(nid, pfn, cur_nr_pages, altmap,404 params->pgmap);405 if (err)406 break;407 cond_resched();408 }409 vmemmap_populate_print_last();410 return err;411}412 413/* find the smallest valid pfn in the range [start_pfn, end_pfn) */414static unsigned long find_smallest_section_pfn(int nid, struct zone *zone,415 unsigned long start_pfn,416 unsigned long end_pfn)417{418 for (; start_pfn < end_pfn; start_pfn += PAGES_PER_SUBSECTION) {419 if (unlikely(!pfn_to_online_page(start_pfn)))420 continue;421 422 if (unlikely(pfn_to_nid(start_pfn) != nid))423 continue;424 425 if (zone != page_zone(pfn_to_page(start_pfn)))426 continue;427 428 return start_pfn;429 }430 431 return 0;432}433 434/* find the biggest valid pfn in the range [start_pfn, end_pfn). */435static unsigned long find_biggest_section_pfn(int nid, struct zone *zone,436 unsigned long start_pfn,437 unsigned long end_pfn)438{439 unsigned long pfn;440 441 /* pfn is the end pfn of a memory section. */442 pfn = end_pfn - 1;443 for (; pfn >= start_pfn; pfn -= PAGES_PER_SUBSECTION) {444 if (unlikely(!pfn_to_online_page(pfn)))445 continue;446 447 if (unlikely(pfn_to_nid(pfn) != nid))448 continue;449 450 if (zone != page_zone(pfn_to_page(pfn)))451 continue;452 453 return pfn;454 }455 456 return 0;457}458 459static void shrink_zone_span(struct zone *zone, unsigned long start_pfn,460 unsigned long end_pfn)461{462 unsigned long pfn;463 int nid = zone_to_nid(zone);464 465 if (zone->zone_start_pfn == start_pfn) {466 /*467 * If the section is smallest section in the zone, it need468 * shrink zone->zone_start_pfn and zone->zone_spanned_pages.469 * In this case, we find second smallest valid mem_section470 * for shrinking zone.471 */472 pfn = find_smallest_section_pfn(nid, zone, end_pfn,473 zone_end_pfn(zone));474 if (pfn) {475 zone->spanned_pages = zone_end_pfn(zone) - pfn;476 zone->zone_start_pfn = pfn;477 } else {478 zone->zone_start_pfn = 0;479 zone->spanned_pages = 0;480 }481 } else if (zone_end_pfn(zone) == end_pfn) {482 /*483 * If the section is biggest section in the zone, it need484 * shrink zone->spanned_pages.485 * In this case, we find second biggest valid mem_section for486 * shrinking zone.487 */488 pfn = find_biggest_section_pfn(nid, zone, zone->zone_start_pfn,489 start_pfn);490 if (pfn)491 zone->spanned_pages = pfn - zone->zone_start_pfn + 1;492 else {493 zone->zone_start_pfn = 0;494 zone->spanned_pages = 0;495 }496 }497}498 499static void update_pgdat_span(struct pglist_data *pgdat)500{501 unsigned long node_start_pfn = 0, node_end_pfn = 0;502 struct zone *zone;503 504 for (zone = pgdat->node_zones;505 zone < pgdat->node_zones + MAX_NR_ZONES; zone++) {506 unsigned long end_pfn = zone_end_pfn(zone);507 508 /* No need to lock the zones, they can't change. */509 if (!zone->spanned_pages)510 continue;511 if (!node_end_pfn) {512 node_start_pfn = zone->zone_start_pfn;513 node_end_pfn = end_pfn;514 continue;515 }516 517 if (end_pfn > node_end_pfn)518 node_end_pfn = end_pfn;519 if (zone->zone_start_pfn < node_start_pfn)520 node_start_pfn = zone->zone_start_pfn;521 }522 523 pgdat->node_start_pfn = node_start_pfn;524 pgdat->node_spanned_pages = node_end_pfn - node_start_pfn;525}526 527void remove_pfn_range_from_zone(struct zone *zone,528 unsigned long start_pfn,529 unsigned long nr_pages)530{531 const unsigned long end_pfn = start_pfn + nr_pages;532 struct pglist_data *pgdat = zone->zone_pgdat;533 unsigned long pfn, cur_nr_pages;534 535 /* Poison struct pages because they are now uninitialized again. */536 for (pfn = start_pfn; pfn < end_pfn; pfn += cur_nr_pages) {537 cond_resched();538 539 /* Select all remaining pages up to the next section boundary */540 cur_nr_pages =541 min(end_pfn - pfn, SECTION_ALIGN_UP(pfn + 1) - pfn);542 page_init_poison(pfn_to_page(pfn),543 sizeof(struct page) * cur_nr_pages);544 }545 546 /*547 * Zone shrinking code cannot properly deal with ZONE_DEVICE. So548 * we will not try to shrink the zones - which is okay as549 * set_zone_contiguous() cannot deal with ZONE_DEVICE either way.550 */551 if (zone_is_zone_device(zone))552 return;553 554 clear_zone_contiguous(zone);555 556 shrink_zone_span(zone, start_pfn, start_pfn + nr_pages);557 update_pgdat_span(pgdat);558 559 set_zone_contiguous(zone);560}561 562/**563 * __remove_pages() - remove sections of pages564 * @pfn: starting pageframe (must be aligned to start of a section)565 * @nr_pages: number of pages to remove (must be multiple of section size)566 * @altmap: alternative device page map or %NULL if default memmap is used567 *568 * Generic helper function to remove section mappings and sysfs entries569 * for the section of the memory we are removing. Caller needs to make570 * sure that pages are marked reserved and zones are adjust properly by571 * calling offline_pages().572 */573void __remove_pages(unsigned long pfn, unsigned long nr_pages,574 struct vmem_altmap *altmap)575{576 const unsigned long end_pfn = pfn + nr_pages;577 unsigned long cur_nr_pages;578 579 if (check_pfn_span(pfn, nr_pages)) {580 WARN(1, "Misaligned %s start: %#lx end: %#lx\n", __func__, pfn, pfn + nr_pages - 1);581 return;582 }583 584 for (; pfn < end_pfn; pfn += cur_nr_pages) {585 cond_resched();586 /* Select all remaining pages up to the next section boundary */587 cur_nr_pages = min(end_pfn - pfn,588 SECTION_ALIGN_UP(pfn + 1) - pfn);589 sparse_remove_section(pfn, cur_nr_pages, altmap);590 }591}592 593int set_online_page_callback(online_page_callback_t callback)594{595 int rc = -EINVAL;596 597 get_online_mems();598 mutex_lock(&online_page_callback_lock);599 600 if (online_page_callback == generic_online_page) {601 online_page_callback = callback;602 rc = 0;603 }604 605 mutex_unlock(&online_page_callback_lock);606 put_online_mems();607 608 return rc;609}610EXPORT_SYMBOL_GPL(set_online_page_callback);611 612int restore_online_page_callback(online_page_callback_t callback)613{614 int rc = -EINVAL;615 616 get_online_mems();617 mutex_lock(&online_page_callback_lock);618 619 if (online_page_callback == callback) {620 online_page_callback = generic_online_page;621 rc = 0;622 }623 624 mutex_unlock(&online_page_callback_lock);625 put_online_mems();626 627 return rc;628}629EXPORT_SYMBOL_GPL(restore_online_page_callback);630 631/* we are OK calling __meminit stuff here - we have CONFIG_MEMORY_HOTPLUG */632void generic_online_page(struct page *page, unsigned int order)633{634 __free_pages_core(page, order, MEMINIT_HOTPLUG);635}636EXPORT_SYMBOL_GPL(generic_online_page);637 638static void online_pages_range(unsigned long start_pfn, unsigned long nr_pages)639{640 const unsigned long end_pfn = start_pfn + nr_pages;641 unsigned long pfn;642 643 /*644 * Online the pages in MAX_PAGE_ORDER aligned chunks. The callback might645 * decide to not expose all pages to the buddy (e.g., expose them646 * later). We account all pages as being online and belonging to this647 * zone ("present").648 * When using memmap_on_memory, the range might not be aligned to649 * MAX_ORDER_NR_PAGES - 1, but pageblock aligned. __ffs() will detect650 * this and the first chunk to online will be pageblock_nr_pages.651 */652 for (pfn = start_pfn; pfn < end_pfn;) {653 int order;654 655 /*656 * Free to online pages in the largest chunks alignment allows.657 *658 * __ffs() behaviour is undefined for 0. start == 0 is659 * MAX_PAGE_ORDER-aligned, Set order to MAX_PAGE_ORDER for660 * the case.661 */662 if (pfn)663 order = min_t(int, MAX_PAGE_ORDER, __ffs(pfn));664 else665 order = MAX_PAGE_ORDER;666 667 (*online_page_callback)(pfn_to_page(pfn), order);668 pfn += (1UL << order);669 }670 671 /* mark all involved sections as online */672 online_mem_sections(start_pfn, end_pfn);673}674 675/* check which state of node_states will be changed when online memory */676static void node_states_check_changes_online(unsigned long nr_pages,677 struct zone *zone, struct memory_notify *arg)678{679 int nid = zone_to_nid(zone);680 681 arg->status_change_nid = NUMA_NO_NODE;682 arg->status_change_nid_normal = NUMA_NO_NODE;683 684 if (!node_state(nid, N_MEMORY))685 arg->status_change_nid = nid;686 if (zone_idx(zone) <= ZONE_NORMAL && !node_state(nid, N_NORMAL_MEMORY))687 arg->status_change_nid_normal = nid;688}689 690static void node_states_set_node(int node, struct memory_notify *arg)691{692 if (arg->status_change_nid_normal >= 0)693 node_set_state(node, N_NORMAL_MEMORY);694 695 if (arg->status_change_nid >= 0)696 node_set_state(node, N_MEMORY);697}698 699static void __meminit resize_zone_range(struct zone *zone, unsigned long start_pfn,700 unsigned long nr_pages)701{702 unsigned long old_end_pfn = zone_end_pfn(zone);703 704 if (zone_is_empty(zone) || start_pfn < zone->zone_start_pfn)705 zone->zone_start_pfn = start_pfn;706 707 zone->spanned_pages = max(start_pfn + nr_pages, old_end_pfn) - zone->zone_start_pfn;708}709 710static void __meminit resize_pgdat_range(struct pglist_data *pgdat, unsigned long start_pfn,711 unsigned long nr_pages)712{713 unsigned long old_end_pfn = pgdat_end_pfn(pgdat);714 715 if (!pgdat->node_spanned_pages || start_pfn < pgdat->node_start_pfn)716 pgdat->node_start_pfn = start_pfn;717 718 pgdat->node_spanned_pages = max(start_pfn + nr_pages, old_end_pfn) - pgdat->node_start_pfn;719 720}721 722#ifdef CONFIG_ZONE_DEVICE723static void section_taint_zone_device(unsigned long pfn)724{725 struct mem_section *ms = __pfn_to_section(pfn);726 727 ms->section_mem_map |= SECTION_TAINT_ZONE_DEVICE;728}729#else730static inline void section_taint_zone_device(unsigned long pfn)731{732}733#endif734 735/*736 * Associate the pfn range with the given zone, initializing the memmaps737 * and resizing the pgdat/zone data to span the added pages. After this738 * call, all affected pages are PageOffline().739 *740 * All aligned pageblocks are initialized to the specified migratetype741 * (usually MIGRATE_MOVABLE). Besides setting the migratetype, no related742 * zone stats (e.g., nr_isolate_pageblock) are touched.743 */744void move_pfn_range_to_zone(struct zone *zone, unsigned long start_pfn,745 unsigned long nr_pages,746 struct vmem_altmap *altmap, int migratetype)747{748 struct pglist_data *pgdat = zone->zone_pgdat;749 int nid = pgdat->node_id;750 751 clear_zone_contiguous(zone);752 753 if (zone_is_empty(zone))754 init_currently_empty_zone(zone, start_pfn, nr_pages);755 resize_zone_range(zone, start_pfn, nr_pages);756 resize_pgdat_range(pgdat, start_pfn, nr_pages);757 758 /*759 * Subsection population requires care in pfn_to_online_page().760 * Set the taint to enable the slow path detection of761 * ZONE_DEVICE pages in an otherwise ZONE_{NORMAL,MOVABLE}762 * section.763 */764 if (zone_is_zone_device(zone)) {765 if (!IS_ALIGNED(start_pfn, PAGES_PER_SECTION))766 section_taint_zone_device(start_pfn);767 if (!IS_ALIGNED(start_pfn + nr_pages, PAGES_PER_SECTION))768 section_taint_zone_device(start_pfn + nr_pages);769 }770 771 /*772 * TODO now we have a visible range of pages which are not associated773 * with their zone properly. Not nice but set_pfnblock_flags_mask774 * expects the zone spans the pfn range. All the pages in the range775 * are reserved so nobody should be touching them so we should be safe776 */777 memmap_init_range(nr_pages, nid, zone_idx(zone), start_pfn, 0,778 MEMINIT_HOTPLUG, altmap, migratetype);779 780 set_zone_contiguous(zone);781}782 783struct auto_movable_stats {784 unsigned long kernel_early_pages;785 unsigned long movable_pages;786};787 788static void auto_movable_stats_account_zone(struct auto_movable_stats *stats,789 struct zone *zone)790{791 if (zone_idx(zone) == ZONE_MOVABLE) {792 stats->movable_pages += zone->present_pages;793 } else {794 stats->kernel_early_pages += zone->present_early_pages;795#ifdef CONFIG_CMA796 /*797 * CMA pages (never on hotplugged memory) behave like798 * ZONE_MOVABLE.799 */800 stats->movable_pages += zone->cma_pages;801 stats->kernel_early_pages -= zone->cma_pages;802#endif /* CONFIG_CMA */803 }804}805struct auto_movable_group_stats {806 unsigned long movable_pages;807 unsigned long req_kernel_early_pages;808};809 810static int auto_movable_stats_account_group(struct memory_group *group,811 void *arg)812{813 const int ratio = READ_ONCE(auto_movable_ratio);814 struct auto_movable_group_stats *stats = arg;815 long pages;816 817 /*818 * We don't support modifying the config while the auto-movable online819 * policy is already enabled. Just avoid the division by zero below.820 */821 if (!ratio)822 return 0;823 824 /*825 * Calculate how many early kernel pages this group requires to826 * satisfy the configured zone ratio.827 */828 pages = group->present_movable_pages * 100 / ratio;829 pages -= group->present_kernel_pages;830 831 if (pages > 0)832 stats->req_kernel_early_pages += pages;833 stats->movable_pages += group->present_movable_pages;834 return 0;835}836 837static bool auto_movable_can_online_movable(int nid, struct memory_group *group,838 unsigned long nr_pages)839{840 unsigned long kernel_early_pages, movable_pages;841 struct auto_movable_group_stats group_stats = {};842 struct auto_movable_stats stats = {};843 struct zone *zone;844 int i;845 846 /* Walk all relevant zones and collect MOVABLE vs. KERNEL stats. */847 if (nid == NUMA_NO_NODE) {848 /* TODO: cache values */849 for_each_populated_zone(zone)850 auto_movable_stats_account_zone(&stats, zone);851 } else {852 for (i = 0; i < MAX_NR_ZONES; i++) {853 pg_data_t *pgdat = NODE_DATA(nid);854 855 zone = pgdat->node_zones + i;856 if (populated_zone(zone))857 auto_movable_stats_account_zone(&stats, zone);858 }859 }860 861 kernel_early_pages = stats.kernel_early_pages;862 movable_pages = stats.movable_pages;863 864 /*865 * Kernel memory inside dynamic memory group allows for more MOVABLE866 * memory within the same group. Remove the effect of all but the867 * current group from the stats.868 */869 walk_dynamic_memory_groups(nid, auto_movable_stats_account_group,870 group, &group_stats);871 if (kernel_early_pages <= group_stats.req_kernel_early_pages)872 return false;873 kernel_early_pages -= group_stats.req_kernel_early_pages;874 movable_pages -= group_stats.movable_pages;875 876 if (group && group->is_dynamic)877 kernel_early_pages += group->present_kernel_pages;878 879 /*880 * Test if we could online the given number of pages to ZONE_MOVABLE881 * and still stay in the configured ratio.882 */883 movable_pages += nr_pages;884 return movable_pages <= (auto_movable_ratio * kernel_early_pages) / 100;885}886 887/*888 * Returns a default kernel memory zone for the given pfn range.889 * If no kernel zone covers this pfn range it will automatically go890 * to the ZONE_NORMAL.891 */892static struct zone *default_kernel_zone_for_pfn(int nid, unsigned long start_pfn,893 unsigned long nr_pages)894{895 struct pglist_data *pgdat = NODE_DATA(nid);896 int zid;897 898 for (zid = 0; zid < ZONE_NORMAL; zid++) {899 struct zone *zone = &pgdat->node_zones[zid];900 901 if (zone_intersects(zone, start_pfn, nr_pages))902 return zone;903 }904 905 return &pgdat->node_zones[ZONE_NORMAL];906}907 908/*909 * Determine to which zone to online memory dynamically based on user910 * configuration and system stats. We care about the following ratio:911 *912 * MOVABLE : KERNEL913 *914 * Whereby MOVABLE is memory in ZONE_MOVABLE and KERNEL is memory in915 * one of the kernel zones. CMA pages inside one of the kernel zones really916 * behaves like ZONE_MOVABLE, so we treat them accordingly.917 *918 * We don't allow for hotplugged memory in a KERNEL zone to increase the919 * amount of MOVABLE memory we can have, so we end up with:920 *921 * MOVABLE : KERNEL_EARLY922 *923 * Whereby KERNEL_EARLY is memory in one of the kernel zones, available sinze924 * boot. We base our calculation on KERNEL_EARLY internally, because:925 *926 * a) Hotplugged memory in one of the kernel zones can sometimes still get927 * hotunplugged, especially when hot(un)plugging individual memory blocks.928 * There is no coordination across memory devices, therefore "automatic"929 * hotunplugging, as implemented in hypervisors, could result in zone930 * imbalances.931 * b) Early/boot memory in one of the kernel zones can usually not get932 * hotunplugged again (e.g., no firmware interface to unplug, fragmented933 * with unmovable allocations). While there are corner cases where it might934 * still work, it is barely relevant in practice.935 *936 * Exceptions are dynamic memory groups, which allow for more MOVABLE937 * memory within the same memory group -- because in that case, there is938 * coordination within the single memory device managed by a single driver.939 *940 * We rely on "present pages" instead of "managed pages", as the latter is941 * highly unreliable and dynamic in virtualized environments, and does not942 * consider boot time allocations. For example, memory ballooning adjusts the943 * managed pages when inflating/deflating the balloon, and balloon compaction944 * can even migrate inflated pages between zones.945 *946 * Using "present pages" is better but some things to keep in mind are:947 *948 * a) Some memblock allocations, such as for the crashkernel area, are949 * effectively unused by the kernel, yet they account to "present pages".950 * Fortunately, these allocations are comparatively small in relevant setups951 * (e.g., fraction of system memory).952 * b) Some hotplugged memory blocks in virtualized environments, esecially953 * hotplugged by virtio-mem, look like they are completely present, however,954 * only parts of the memory block are actually currently usable.955 * "present pages" is an upper limit that can get reached at runtime. As956 * we base our calculations on KERNEL_EARLY, this is not an issue.957 */958static struct zone *auto_movable_zone_for_pfn(int nid,959 struct memory_group *group,960 unsigned long pfn,961 unsigned long nr_pages)962{963 unsigned long online_pages = 0, max_pages, end_pfn;964 struct page *page;965 966 if (!auto_movable_ratio)967 goto kernel_zone;968 969 if (group && !group->is_dynamic) {970 max_pages = group->s.max_pages;971 online_pages = group->present_movable_pages;972 973 /* If anything is !MOVABLE online the rest !MOVABLE. */974 if (group->present_kernel_pages)975 goto kernel_zone;976 } else if (!group || group->d.unit_pages == nr_pages) {977 max_pages = nr_pages;978 } else {979 max_pages = group->d.unit_pages;980 /*981 * Take a look at all online sections in the current unit.982 * We can safely assume that all pages within a section belong983 * to the same zone, because dynamic memory groups only deal984 * with hotplugged memory.985 */986 pfn = ALIGN_DOWN(pfn, group->d.unit_pages);987 end_pfn = pfn + group->d.unit_pages;988 for (; pfn < end_pfn; pfn += PAGES_PER_SECTION) {989 page = pfn_to_online_page(pfn);990 if (!page)991 continue;992 /* If anything is !MOVABLE online the rest !MOVABLE. */993 if (!is_zone_movable_page(page))994 goto kernel_zone;995 online_pages += PAGES_PER_SECTION;996 }997 }998 999 /*1000 * Online MOVABLE if we could *currently* online all remaining parts1001 * MOVABLE. We expect to (add+) online them immediately next, so if1002 * nobody interferes, all will be MOVABLE if possible.1003 */1004 nr_pages = max_pages - online_pages;1005 if (!auto_movable_can_online_movable(NUMA_NO_NODE, group, nr_pages))1006 goto kernel_zone;1007 1008#ifdef CONFIG_NUMA1009 if (auto_movable_numa_aware &&1010 !auto_movable_can_online_movable(nid, group, nr_pages))1011 goto kernel_zone;1012#endif /* CONFIG_NUMA */1013 1014 return &NODE_DATA(nid)->node_zones[ZONE_MOVABLE];1015kernel_zone:1016 return default_kernel_zone_for_pfn(nid, pfn, nr_pages);1017}1018 1019static inline struct zone *default_zone_for_pfn(int nid, unsigned long start_pfn,1020 unsigned long nr_pages)1021{1022 struct zone *kernel_zone = default_kernel_zone_for_pfn(nid, start_pfn,1023 nr_pages);1024 struct zone *movable_zone = &NODE_DATA(nid)->node_zones[ZONE_MOVABLE];1025 bool in_kernel = zone_intersects(kernel_zone, start_pfn, nr_pages);1026 bool in_movable = zone_intersects(movable_zone, start_pfn, nr_pages);1027 1028 /*1029 * We inherit the existing zone in a simple case where zones do not1030 * overlap in the given range1031 */1032 if (in_kernel ^ in_movable)1033 return (in_kernel) ? kernel_zone : movable_zone;1034 1035 /*1036 * If the range doesn't belong to any zone or two zones overlap in the1037 * given range then we use movable zone only if movable_node is1038 * enabled because we always online to a kernel zone by default.1039 */1040 return movable_node_enabled ? movable_zone : kernel_zone;1041}1042 1043struct zone *zone_for_pfn_range(int online_type, int nid,1044 struct memory_group *group, unsigned long start_pfn,1045 unsigned long nr_pages)1046{1047 if (online_type == MMOP_ONLINE_KERNEL)1048 return default_kernel_zone_for_pfn(nid, start_pfn, nr_pages);1049 1050 if (online_type == MMOP_ONLINE_MOVABLE)1051 return &NODE_DATA(nid)->node_zones[ZONE_MOVABLE];1052 1053 if (online_policy == ONLINE_POLICY_AUTO_MOVABLE)1054 return auto_movable_zone_for_pfn(nid, group, start_pfn, nr_pages);1055 1056 return default_zone_for_pfn(nid, start_pfn, nr_pages);1057}1058 1059/*1060 * This function should only be called by memory_block_{online,offline},1061 * and {online,offline}_pages.1062 */1063void adjust_present_page_count(struct page *page, struct memory_group *group,1064 long nr_pages)1065{1066 struct zone *zone = page_zone(page);1067 const bool movable = zone_idx(zone) == ZONE_MOVABLE;1068 1069 /*1070 * We only support onlining/offlining/adding/removing of complete1071 * memory blocks; therefore, either all is either early or hotplugged.1072 */1073 if (early_section(__pfn_to_section(page_to_pfn(page))))1074 zone->present_early_pages += nr_pages;1075 zone->present_pages += nr_pages;1076 zone->zone_pgdat->node_present_pages += nr_pages;1077 1078 if (group && movable)1079 group->present_movable_pages += nr_pages;1080 else if (group && !movable)1081 group->present_kernel_pages += nr_pages;1082}1083 1084int mhp_init_memmap_on_memory(unsigned long pfn, unsigned long nr_pages,1085 struct zone *zone, bool mhp_off_inaccessible)1086{1087 unsigned long end_pfn = pfn + nr_pages;1088 int ret, i;1089 1090 ret = kasan_add_zero_shadow(__va(PFN_PHYS(pfn)), PFN_PHYS(nr_pages));1091 if (ret)1092 return ret;1093 1094 /*1095 * Memory block is accessible at this stage and hence poison the struct1096 * pages now. If the memory block is accessible during memory hotplug1097 * addition phase, then page poisining is already performed in1098 * sparse_add_section().1099 */1100 if (mhp_off_inaccessible)1101 page_init_poison(pfn_to_page(pfn), sizeof(struct page) * nr_pages);1102 1103 move_pfn_range_to_zone(zone, pfn, nr_pages, NULL, MIGRATE_UNMOVABLE);1104 1105 for (i = 0; i < nr_pages; i++) {1106 struct page *page = pfn_to_page(pfn + i);1107 1108 __ClearPageOffline(page);1109 SetPageVmemmapSelfHosted(page);1110 }1111 1112 /*1113 * It might be that the vmemmap_pages fully span sections. If that is1114 * the case, mark those sections online here as otherwise they will be1115 * left offline.1116 */1117 if (nr_pages >= PAGES_PER_SECTION)1118 online_mem_sections(pfn, ALIGN_DOWN(end_pfn, PAGES_PER_SECTION));1119 1120 return ret;1121}1122 1123void mhp_deinit_memmap_on_memory(unsigned long pfn, unsigned long nr_pages)1124{1125 unsigned long end_pfn = pfn + nr_pages;1126 1127 /*1128 * It might be that the vmemmap_pages fully span sections. If that is1129 * the case, mark those sections offline here as otherwise they will be1130 * left online.1131 */1132 if (nr_pages >= PAGES_PER_SECTION)1133 offline_mem_sections(pfn, ALIGN_DOWN(end_pfn, PAGES_PER_SECTION));1134 1135 /*1136 * The pages associated with this vmemmap have been offlined, so1137 * we can reset its state here.1138 */1139 remove_pfn_range_from_zone(page_zone(pfn_to_page(pfn)), pfn, nr_pages);1140 kasan_remove_zero_shadow(__va(PFN_PHYS(pfn)), PFN_PHYS(nr_pages));1141}1142 1143/*1144 * Must be called with mem_hotplug_lock in write mode.1145 */1146int online_pages(unsigned long pfn, unsigned long nr_pages,1147 struct zone *zone, struct memory_group *group)1148{1149 unsigned long flags;1150 int need_zonelists_rebuild = 0;1151 const int nid = zone_to_nid(zone);1152 int ret;1153 struct memory_notify arg;1154 1155 /*1156 * {on,off}lining is constrained to full memory sections (or more1157 * precisely to memory blocks from the user space POV).1158 * memmap_on_memory is an exception because it reserves initial part1159 * of the physical memory space for vmemmaps. That space is pageblock1160 * aligned.1161 */1162 if (WARN_ON_ONCE(!nr_pages || !pageblock_aligned(pfn) ||1163 !IS_ALIGNED(pfn + nr_pages, PAGES_PER_SECTION)))1164 return -EINVAL;1165 1166 1167 /* associate pfn range with the zone */1168 move_pfn_range_to_zone(zone, pfn, nr_pages, NULL, MIGRATE_ISOLATE);1169 1170 arg.start_pfn = pfn;1171 arg.nr_pages = nr_pages;1172 node_states_check_changes_online(nr_pages, zone, &arg);1173 1174 ret = memory_notify(MEM_GOING_ONLINE, &arg);1175 ret = notifier_to_errno(ret);1176 if (ret)1177 goto failed_addition;1178 1179 /*1180 * Fixup the number of isolated pageblocks before marking the sections1181 * onlining, such that undo_isolate_page_range() works correctly.1182 */1183 spin_lock_irqsave(&zone->lock, flags);1184 zone->nr_isolate_pageblock += nr_pages / pageblock_nr_pages;1185 spin_unlock_irqrestore(&zone->lock, flags);1186 1187 /*1188 * If this zone is not populated, then it is not in zonelist.1189 * This means the page allocator ignores this zone.1190 * So, zonelist must be updated after online.1191 */1192 if (!populated_zone(zone)) {1193 need_zonelists_rebuild = 1;1194 setup_zone_pageset(zone);1195 }1196 1197 online_pages_range(pfn, nr_pages);1198 adjust_present_page_count(pfn_to_page(pfn), group, nr_pages);1199 1200 node_states_set_node(nid, &arg);1201 if (need_zonelists_rebuild)1202 build_all_zonelists(NULL);1203 1204 /* Basic onlining is complete, allow allocation of onlined pages. */1205 undo_isolate_page_range(pfn, pfn + nr_pages, MIGRATE_MOVABLE);1206 1207 /*1208 * Freshly onlined pages aren't shuffled (e.g., all pages are placed to1209 * the tail of the freelist when undoing isolation). Shuffle the whole1210 * zone to make sure the just onlined pages are properly distributed1211 * across the whole freelist - to create an initial shuffle.1212 */1213 shuffle_zone(zone);1214 1215 /* reinitialise watermarks and update pcp limits */1216 init_per_zone_wmark_min();1217 1218 kswapd_run(nid);1219 kcompactd_run(nid);1220 1221 writeback_set_ratelimit();1222 1223 memory_notify(MEM_ONLINE, &arg);1224 return 0;1225 1226failed_addition:1227 pr_debug("online_pages [mem %#010llx-%#010llx] failed\n",1228 (unsigned long long) pfn << PAGE_SHIFT,1229 (((unsigned long long) pfn + nr_pages) << PAGE_SHIFT) - 1);1230 memory_notify(MEM_CANCEL_ONLINE, &arg);1231 remove_pfn_range_from_zone(zone, pfn, nr_pages);1232 return ret;1233}1234 1235/* we are OK calling __meminit stuff here - we have CONFIG_MEMORY_HOTPLUG */1236static pg_data_t *hotadd_init_pgdat(int nid)1237{1238 struct pglist_data *pgdat;1239 1240 /*1241 * NODE_DATA is preallocated (free_area_init) but its internal1242 * state is not allocated completely. Add missing pieces.1243 * Completely offline nodes stay around and they just need1244 * reintialization.1245 */1246 pgdat = NODE_DATA(nid);1247 1248 /* init node's zones as empty zones, we don't have any present pages.*/1249 free_area_init_core_hotplug(pgdat);1250 1251 /*1252 * The node we allocated has no zone fallback lists. For avoiding1253 * to access not-initialized zonelist, build here.1254 */1255 build_all_zonelists(pgdat);1256 1257 return pgdat;1258}1259 1260/*1261 * __try_online_node - online a node if offlined1262 * @nid: the node ID1263 * @set_node_online: Whether we want to online the node1264 * called by cpu_up() to online a node without onlined memory.1265 *1266 * Returns:1267 * 1 -> a new node has been allocated1268 * 0 -> the node is already online1269 * -ENOMEM -> the node could not be allocated1270 */1271static int __try_online_node(int nid, bool set_node_online)1272{1273 pg_data_t *pgdat;1274 int ret = 1;1275 1276 if (node_online(nid))1277 return 0;1278 1279 pgdat = hotadd_init_pgdat(nid);1280 if (!pgdat) {1281 pr_err("Cannot online node %d due to NULL pgdat\n", nid);1282 ret = -ENOMEM;1283 goto out;1284 }1285 1286 if (set_node_online) {1287 node_set_online(nid);1288 ret = register_one_node(nid);1289 BUG_ON(ret);1290 }1291out:1292 return ret;1293}1294 1295/*1296 * Users of this function always want to online/register the node1297 */1298int try_online_node(int nid)1299{1300 int ret;1301 1302 mem_hotplug_begin();1303 ret = __try_online_node(nid, true);1304 mem_hotplug_done();1305 return ret;1306}1307 1308static int check_hotplug_memory_range(u64 start, u64 size)1309{1310 /* memory range must be block size aligned */1311 if (!size || !IS_ALIGNED(start, memory_block_size_bytes()) ||1312 !IS_ALIGNED(size, memory_block_size_bytes())) {1313 pr_err("Block size [%#lx] unaligned hotplug range: start %#llx, size %#llx",1314 memory_block_size_bytes(), start, size);1315 return -EINVAL;1316 }1317 1318 return 0;1319}1320 1321static int online_memory_block(struct memory_block *mem, void *arg)1322{1323 mem->online_type = mhp_default_online_type;1324 return device_online(&mem->dev);1325}1326 1327#ifndef arch_supports_memmap_on_memory1328static inline bool arch_supports_memmap_on_memory(unsigned long vmemmap_size)1329{1330 /*1331 * As default, we want the vmemmap to span a complete PMD such that we1332 * can map the vmemmap using a single PMD if supported by the1333 * architecture.1334 */1335 return IS_ALIGNED(vmemmap_size, PMD_SIZE);1336}1337#endif1338 1339bool mhp_supports_memmap_on_memory(void)1340{1341 unsigned long vmemmap_size = memory_block_memmap_size();1342 unsigned long memmap_pages = memory_block_memmap_on_memory_pages();1343 1344 /*1345 * Besides having arch support and the feature enabled at runtime, we1346 * need a few more assumptions to hold true:1347 *1348 * a) The vmemmap pages span complete PMDs: We don't want vmemmap code1349 * to populate memory from the altmap for unrelated parts (i.e.,1350 * other memory blocks)1351 *1352 * b) The vmemmap pages (and thereby the pages that will be exposed to1353 * the buddy) have to cover full pageblocks: memory onlining/offlining1354 * code requires applicable ranges to be page-aligned, for example, to1355 * set the migratetypes properly.1356 *1357 * TODO: Although we have a check here to make sure that vmemmap pages1358 * fully populate a PMD, it is not the right place to check for1359 * this. A much better solution involves improving vmemmap code1360 * to fallback to base pages when trying to populate vmemmap using1361 * altmap as an alternative source of memory, and we do not exactly1362 * populate a single PMD.1363 */1364 if (!mhp_memmap_on_memory())1365 return false;1366 1367 /*1368 * Make sure the vmemmap allocation is fully contained1369 * so that we always allocate vmemmap memory from altmap area.1370 */1371 if (!IS_ALIGNED(vmemmap_size, PAGE_SIZE))1372 return false;1373 1374 /*1375 * start pfn should be pageblock_nr_pages aligned for correctly1376 * setting migrate types1377 */1378 if (!pageblock_aligned(memmap_pages))1379 return false;1380 1381 if (memmap_pages == PHYS_PFN(memory_block_size_bytes()))1382 /* No effective hotplugged memory doesn't make sense. */1383 return false;1384 1385 return arch_supports_memmap_on_memory(vmemmap_size);1386}1387EXPORT_SYMBOL_GPL(mhp_supports_memmap_on_memory);1388 1389static void remove_memory_blocks_and_altmaps(u64 start, u64 size)1390{1391 unsigned long memblock_size = memory_block_size_bytes();1392 u64 cur_start;1393 1394 /*1395 * For memmap_on_memory, the altmaps were added on a per-memblock1396 * basis; we have to process each individual memory block.1397 */1398 for (cur_start = start; cur_start < start + size;1399 cur_start += memblock_size) {1400 struct vmem_altmap *altmap = NULL;1401 struct memory_block *mem;1402 1403 mem = find_memory_block(pfn_to_section_nr(PFN_DOWN(cur_start)));1404 if (WARN_ON_ONCE(!mem))1405 continue;1406 1407 altmap = mem->altmap;1408 mem->altmap = NULL;1409 1410 remove_memory_block_devices(cur_start, memblock_size);1411 1412 arch_remove_memory(cur_start, memblock_size, altmap);1413 1414 /* Verify that all vmemmap pages have actually been freed. */1415 WARN(altmap->alloc, "Altmap not fully unmapped");1416 kfree(altmap);1417 }1418}1419 1420static int create_altmaps_and_memory_blocks(int nid, struct memory_group *group,1421 u64 start, u64 size, mhp_t mhp_flags)1422{1423 unsigned long memblock_size = memory_block_size_bytes();1424 u64 cur_start;1425 int ret;1426 1427 for (cur_start = start; cur_start < start + size;1428 cur_start += memblock_size) {1429 struct mhp_params params = { .pgprot =1430 pgprot_mhp(PAGE_KERNEL) };1431 struct vmem_altmap mhp_altmap = {1432 .base_pfn = PHYS_PFN(cur_start),1433 .end_pfn = PHYS_PFN(cur_start + memblock_size - 1),1434 };1435 1436 mhp_altmap.free = memory_block_memmap_on_memory_pages();1437 if (mhp_flags & MHP_OFFLINE_INACCESSIBLE)1438 mhp_altmap.inaccessible = true;1439 params.altmap = kmemdup(&mhp_altmap, sizeof(struct vmem_altmap),1440 GFP_KERNEL);1441 if (!params.altmap) {1442 ret = -ENOMEM;1443 goto out;1444 }1445 1446 /* call arch's memory hotadd */1447 ret = arch_add_memory(nid, cur_start, memblock_size, ¶ms);1448 if (ret < 0) {1449 kfree(params.altmap);1450 goto out;1451 }1452 1453 /* create memory block devices after memory was added */1454 ret = create_memory_block_devices(cur_start, memblock_size,1455 params.altmap, group);1456 if (ret) {1457 arch_remove_memory(cur_start, memblock_size, NULL);1458 kfree(params.altmap);1459 goto out;1460 }1461 }1462 1463 return 0;1464out:1465 if (ret && cur_start != start)1466 remove_memory_blocks_and_altmaps(start, cur_start - start);1467 return ret;1468}1469 1470/*1471 * NOTE: The caller must call lock_device_hotplug() to serialize hotplug1472 * and online/offline operations (triggered e.g. by sysfs).1473 *1474 * we are OK calling __meminit stuff here - we have CONFIG_MEMORY_HOTPLUG1475 */1476int add_memory_resource(int nid, struct resource *res, mhp_t mhp_flags)1477{1478 struct mhp_params params = { .pgprot = pgprot_mhp(PAGE_KERNEL) };1479 enum memblock_flags memblock_flags = MEMBLOCK_NONE;1480 struct memory_group *group = NULL;1481 u64 start, size;1482 bool new_node = false;1483 int ret;1484 1485 start = res->start;1486 size = resource_size(res);1487 1488 ret = check_hotplug_memory_range(start, size);1489 if (ret)1490 return ret;1491 1492 if (mhp_flags & MHP_NID_IS_MGID) {1493 group = memory_group_find_by_id(nid);1494 if (!group)1495 return -EINVAL;1496 nid = group->nid;1497 }1498 1499 if (!node_possible(nid)) {1500 WARN(1, "node %d was absent from the node_possible_map\n", nid);1501 return -EINVAL;1502 }1503 1504 mem_hotplug_begin();1505 1506 if (IS_ENABLED(CONFIG_ARCH_KEEP_MEMBLOCK)) {1507 if (res->flags & IORESOURCE_SYSRAM_DRIVER_MANAGED)1508 memblock_flags = MEMBLOCK_DRIVER_MANAGED;1509 ret = memblock_add_node(start, size, nid, memblock_flags);1510 if (ret)1511 goto error_mem_hotplug_end;1512 }1513 1514 ret = __try_online_node(nid, false);1515 if (ret < 0)1516 goto error;1517 new_node = ret;1518 1519 /*1520 * Self hosted memmap array1521 */1522 if ((mhp_flags & MHP_MEMMAP_ON_MEMORY) &&1523 mhp_supports_memmap_on_memory()) {1524 ret = create_altmaps_and_memory_blocks(nid, group, start, size, mhp_flags);1525 if (ret)1526 goto error;1527 } else {1528 ret = arch_add_memory(nid, start, size, ¶ms);1529 if (ret < 0)1530 goto error;1531 1532 /* create memory block devices after memory was added */1533 ret = create_memory_block_devices(start, size, NULL, group);1534 if (ret) {1535 arch_remove_memory(start, size, params.altmap);1536 goto error;1537 }1538 }1539 1540 if (new_node) {1541 /* If sysfs file of new node can't be created, cpu on the node1542 * can't be hot-added. There is no rollback way now.1543 * So, check by BUG_ON() to catch it reluctantly..1544 * We online node here. We can't roll back from here.1545 */1546 node_set_online(nid);1547 ret = __register_one_node(nid);1548 BUG_ON(ret);1549 }1550 1551 register_memory_blocks_under_node(nid, PFN_DOWN(start),1552 PFN_UP(start + size - 1),1553 MEMINIT_HOTPLUG);1554 1555 /* create new memmap entry */1556 if (!strcmp(res->name, "System RAM"))1557 firmware_map_add_hotplug(start, start + size, "System RAM");1558 1559 /* device_online() will take the lock when calling online_pages() */1560 mem_hotplug_done();1561 1562 /*1563 * In case we're allowed to merge the resource, flag it and trigger1564 * merging now that adding succeeded.1565 */1566 if (mhp_flags & MHP_MERGE_RESOURCE)1567 merge_system_ram_resource(res);1568 1569 /* online pages if requested */1570 if (mhp_default_online_type != MMOP_OFFLINE)1571 walk_memory_blocks(start, size, NULL, online_memory_block);1572 1573 return ret;1574error:1575 if (IS_ENABLED(CONFIG_ARCH_KEEP_MEMBLOCK))1576 memblock_remove(start, size);1577error_mem_hotplug_end:1578 mem_hotplug_done();1579 return ret;1580}1581 1582/* requires device_hotplug_lock, see add_memory_resource() */1583int __add_memory(int nid, u64 start, u64 size, mhp_t mhp_flags)1584{1585 struct resource *res;1586 int ret;1587 1588 res = register_memory_resource(start, size, "System RAM");1589 if (IS_ERR(res))1590 return PTR_ERR(res);1591 1592 ret = add_memory_resource(nid, res, mhp_flags);1593 if (ret < 0)1594 release_memory_resource(res);1595 return ret;1596}1597 1598int add_memory(int nid, u64 start, u64 size, mhp_t mhp_flags)1599{1600 int rc;1601 1602 lock_device_hotplug();1603 rc = __add_memory(nid, start, size, mhp_flags);1604 unlock_device_hotplug();1605 1606 return rc;1607}1608EXPORT_SYMBOL_GPL(add_memory);1609 1610/*1611 * Add special, driver-managed memory to the system as system RAM. Such1612 * memory is not exposed via the raw firmware-provided memmap as system1613 * RAM, instead, it is detected and added by a driver - during cold boot,1614 * after a reboot, and after kexec.1615 *1616 * Reasons why this memory should not be used for the initial memmap of a1617 * kexec kernel or for placing kexec images:1618 * - The booting kernel is in charge of determining how this memory will be1619 * used (e.g., use persistent memory as system RAM)1620 * - Coordination with a hypervisor is required before this memory1621 * can be used (e.g., inaccessible parts).1622 *1623 * For this memory, no entries in /sys/firmware/memmap ("raw firmware-provided1624 * memory map") are created. Also, the created memory resource is flagged1625 * with IORESOURCE_SYSRAM_DRIVER_MANAGED, so in-kernel users can special-case1626 * this memory as well (esp., not place kexec images onto it).1627 *1628 * The resource_name (visible via /proc/iomem) has to have the format1629 * "System RAM ($DRIVER)".1630 */1631int add_memory_driver_managed(int nid, u64 start, u64 size,1632 const char *resource_name, mhp_t mhp_flags)1633{1634 struct resource *res;1635 int rc;1636 1637 if (!resource_name ||1638 strstr(resource_name, "System RAM (") != resource_name ||1639 resource_name[strlen(resource_name) - 1] != ')')1640 return -EINVAL;1641 1642 lock_device_hotplug();1643 1644 res = register_memory_resource(start, size, resource_name);1645 if (IS_ERR(res)) {1646 rc = PTR_ERR(res);1647 goto out_unlock;1648 }1649 1650 rc = add_memory_resource(nid, res, mhp_flags);1651 if (rc < 0)1652 release_memory_resource(res);1653 1654out_unlock:1655 unlock_device_hotplug();1656 return rc;1657}1658EXPORT_SYMBOL_GPL(add_memory_driver_managed);1659 1660/*1661 * Platforms should define arch_get_mappable_range() that provides1662 * maximum possible addressable physical memory range for which the1663 * linear mapping could be created. The platform returned address1664 * range must adhere to these following semantics.1665 *1666 * - range.start <= range.end1667 * - Range includes both end points [range.start..range.end]1668 *1669 * There is also a fallback definition provided here, allowing the1670 * entire possible physical address range in case any platform does1671 * not define arch_get_mappable_range().1672 */1673struct range __weak arch_get_mappable_range(void)1674{1675 struct range mhp_range = {1676 .start = 0UL,1677 .end = -1ULL,1678 };1679 return mhp_range;1680}1681 1682struct range mhp_get_pluggable_range(bool need_mapping)1683{1684 const u64 max_phys = PHYSMEM_END;1685 struct range mhp_range;1686 1687 if (need_mapping) {1688 mhp_range = arch_get_mappable_range();1689 if (mhp_range.start > max_phys) {1690 mhp_range.start = 0;1691 mhp_range.end = 0;1692 }1693 mhp_range.end = min_t(u64, mhp_range.end, max_phys);1694 } else {1695 mhp_range.start = 0;1696 mhp_range.end = max_phys;1697 }1698 return mhp_range;1699}1700EXPORT_SYMBOL_GPL(mhp_get_pluggable_range);1701 1702bool mhp_range_allowed(u64 start, u64 size, bool need_mapping)1703{1704 struct range mhp_range = mhp_get_pluggable_range(need_mapping);1705 u64 end = start + size;1706 1707 if (start < end && start >= mhp_range.start && (end - 1) <= mhp_range.end)1708 return true;1709 1710 pr_warn("Hotplug memory [%#llx-%#llx] exceeds maximum addressable range [%#llx-%#llx]\n",1711 start, end, mhp_range.start, mhp_range.end);1712 return false;1713}1714 1715#ifdef CONFIG_MEMORY_HOTREMOVE1716/*1717 * Scan pfn range [start,end) to find movable/migratable pages (LRU pages,1718 * non-lru movable pages and hugepages). Will skip over most unmovable1719 * pages (esp., pages that can be skipped when offlining), but bail out on1720 * definitely unmovable pages.1721 *1722 * Returns:1723 * 0 in case a movable page is found and movable_pfn was updated.1724 * -ENOENT in case no movable page was found.1725 * -EBUSY in case a definitely unmovable page was found.1726 */1727static int scan_movable_pages(unsigned long start, unsigned long end,1728 unsigned long *movable_pfn)1729{1730 unsigned long pfn;1731 1732 for (pfn = start; pfn < end; pfn++) {1733 struct page *page;1734 struct folio *folio;1735 1736 if (!pfn_valid(pfn))1737 continue;1738 page = pfn_to_page(pfn);1739 if (PageLRU(page))1740 goto found;1741 if (__PageMovable(page))1742 goto found;1743 1744 /*1745 * PageOffline() pages that are not marked __PageMovable() and1746 * have a reference count > 0 (after MEM_GOING_OFFLINE) are1747 * definitely unmovable. If their reference count would be 0,1748 * they could at least be skipped when offlining memory.1749 */1750 if (PageOffline(page) && page_count(page))1751 return -EBUSY;1752 1753 if (!PageHuge(page))1754 continue;1755 folio = page_folio(page);1756 /*1757 * This test is racy as we hold no reference or lock. The1758 * hugetlb page could have been free'ed and head is no longer1759 * a hugetlb page before the following check. In such unlikely1760 * cases false positives and negatives are possible. Calling1761 * code must deal with these scenarios.1762 */1763 if (folio_test_hugetlb_migratable(folio))1764 goto found;1765 pfn |= folio_nr_pages(folio) - 1;1766 }1767 return -ENOENT;1768found:1769 *movable_pfn = pfn;1770 return 0;1771}1772 1773static void do_migrate_range(unsigned long start_pfn, unsigned long end_pfn)1774{1775 struct folio *folio;1776 unsigned long pfn;1777 LIST_HEAD(source);1778 static DEFINE_RATELIMIT_STATE(migrate_rs, DEFAULT_RATELIMIT_INTERVAL,1779 DEFAULT_RATELIMIT_BURST);1780 1781 for (pfn = start_pfn; pfn < end_pfn; pfn++) {1782 struct page *page;1783 1784 if (!pfn_valid(pfn))1785 continue;1786 page = pfn_to_page(pfn);1787 folio = page_folio(page);1788 1789 /*1790 * No reference or lock is held on the folio, so it might1791 * be modified concurrently (e.g. split). As such,1792 * folio_nr_pages() may read garbage. This is fine as the outer1793 * loop will revisit the split folio later.1794 */1795 if (folio_test_large(folio))1796 pfn = folio_pfn(folio) + folio_nr_pages(folio) - 1;1797 1798 /*1799 * HWPoison pages have elevated reference counts so the migration would1800 * fail on them. It also doesn't make any sense to migrate them in the1801 * first place. Still try to unmap such a page in case it is still mapped1802 * (keep the unmap as the catch all safety net).1803 */1804 if (folio_test_hwpoison(folio) ||1805 (folio_test_large(folio) && folio_test_has_hwpoisoned(folio))) {1806 if (WARN_ON(folio_test_lru(folio)))1807 folio_isolate_lru(folio);1808 if (folio_mapped(folio))1809 unmap_poisoned_folio(folio, TTU_IGNORE_MLOCK);1810 continue;1811 }1812 1813 if (!folio_try_get(folio))1814 continue;1815 1816 if (unlikely(page_folio(page) != folio))1817 goto put_folio;1818 1819 if (!isolate_folio_to_list(folio, &source)) {1820 if (__ratelimit(&migrate_rs)) {1821 pr_warn("failed to isolate pfn %lx\n",1822 page_to_pfn(page));1823 dump_page(page, "isolation failed");1824 }1825 }1826put_folio:1827 folio_put(folio);1828 }1829 if (!list_empty(&source)) {1830 nodemask_t nmask = node_states[N_MEMORY];1831 struct migration_target_control mtc = {1832 .nmask = &nmask,1833 .gfp_mask = GFP_USER | __GFP_MOVABLE | __GFP_RETRY_MAYFAIL,1834 .reason = MR_MEMORY_HOTPLUG,1835 };1836 int ret;1837 1838 /*1839 * We have checked that migration range is on a single zone so1840 * we can use the nid of the first page to all the others.1841 */1842 mtc.nid = folio_nid(list_first_entry(&source, struct folio, lru));1843 1844 /*1845 * try to allocate from a different node but reuse this node1846 * if there are no other online nodes to be used (e.g. we are1847 * offlining a part of the only existing node)1848 */1849 node_clear(mtc.nid, nmask);1850 if (nodes_empty(nmask))1851 node_set(mtc.nid, nmask);1852 ret = migrate_pages(&source, alloc_migration_target, NULL,1853 (unsigned long)&mtc, MIGRATE_SYNC, MR_MEMORY_HOTPLUG, NULL);1854 if (ret) {1855 list_for_each_entry(folio, &source, lru) {1856 if (__ratelimit(&migrate_rs)) {1857 pr_warn("migrating pfn %lx failed ret:%d\n",1858 folio_pfn(folio), ret);1859 dump_page(&folio->page,1860 "migration failure");1861 }1862 }1863 putback_movable_pages(&source);1864 }1865 }1866}1867 1868static int __init cmdline_parse_movable_node(char *p)1869{1870 movable_node_enabled = true;1871 return 0;1872}1873early_param("movable_node", cmdline_parse_movable_node);1874 1875/* check which state of node_states will be changed when offline memory */1876static void node_states_check_changes_offline(unsigned long nr_pages,1877 struct zone *zone, struct memory_notify *arg)1878{1879 struct pglist_data *pgdat = zone->zone_pgdat;1880 unsigned long present_pages = 0;1881 enum zone_type zt;1882 1883 arg->status_change_nid = NUMA_NO_NODE;1884 arg->status_change_nid_normal = NUMA_NO_NODE;1885 1886 /*1887 * Check whether node_states[N_NORMAL_MEMORY] will be changed.1888 * If the memory to be offline is within the range1889 * [0..ZONE_NORMAL], and it is the last present memory there,1890 * the zones in that range will become empty after the offlining,1891 * thus we can determine that we need to clear the node from1892 * node_states[N_NORMAL_MEMORY].1893 */1894 for (zt = 0; zt <= ZONE_NORMAL; zt++)1895 present_pages += pgdat->node_zones[zt].present_pages;1896 if (zone_idx(zone) <= ZONE_NORMAL && nr_pages >= present_pages)1897 arg->status_change_nid_normal = zone_to_nid(zone);1898 1899 /*1900 * We have accounted the pages from [0..ZONE_NORMAL); ZONE_HIGHMEM1901 * does not apply as we don't support 32bit.1902 * Here we count the possible pages from ZONE_MOVABLE.1903 * If after having accounted all the pages, we see that the nr_pages1904 * to be offlined is over or equal to the accounted pages,1905 * we know that the node will become empty, and so, we can clear1906 * it for N_MEMORY as well.1907 */1908 present_pages += pgdat->node_zones[ZONE_MOVABLE].present_pages;1909 1910 if (nr_pages >= present_pages)1911 arg->status_change_nid = zone_to_nid(zone);1912}1913 1914static void node_states_clear_node(int node, struct memory_notify *arg)1915{1916 if (arg->status_change_nid_normal >= 0)1917 node_clear_state(node, N_NORMAL_MEMORY);1918 1919 if (arg->status_change_nid >= 0)1920 node_clear_state(node, N_MEMORY);1921}1922 1923static int count_system_ram_pages_cb(unsigned long start_pfn,1924 unsigned long nr_pages, void *data)1925{1926 unsigned long *nr_system_ram_pages = data;1927 1928 *nr_system_ram_pages += nr_pages;1929 return 0;1930}1931 1932/*1933 * Must be called with mem_hotplug_lock in write mode.1934 */1935int offline_pages(unsigned long start_pfn, unsigned long nr_pages,1936 struct zone *zone, struct memory_group *group)1937{1938 const unsigned long end_pfn = start_pfn + nr_pages;1939 unsigned long pfn, managed_pages, system_ram_pages = 0;1940 const int node = zone_to_nid(zone);1941 unsigned long flags;1942 struct memory_notify arg;1943 char *reason;1944 int ret;1945 1946 /*1947 * {on,off}lining is constrained to full memory sections (or more1948 * precisely to memory blocks from the user space POV).1949 * memmap_on_memory is an exception because it reserves initial part1950 * of the physical memory space for vmemmaps. That space is pageblock1951 * aligned.1952 */1953 if (WARN_ON_ONCE(!nr_pages || !pageblock_aligned(start_pfn) ||1954 !IS_ALIGNED(start_pfn + nr_pages, PAGES_PER_SECTION)))1955 return -EINVAL;1956 1957 /*1958 * Don't allow to offline memory blocks that contain holes.1959 * Consequently, memory blocks with holes can never get onlined1960 * via the hotplug path - online_pages() - as hotplugged memory has1961 * no holes. This way, we don't have to worry about memory holes,1962 * don't need pfn_valid() checks, and can avoid using1963 * walk_system_ram_range() later.1964 */1965 walk_system_ram_range(start_pfn, nr_pages, &system_ram_pages,1966 count_system_ram_pages_cb);1967 if (system_ram_pages != nr_pages) {1968 ret = -EINVAL;1969 reason = "memory holes";1970 goto failed_removal;1971 }1972 1973 /*1974 * We only support offlining of memory blocks managed by a single zone,1975 * checked by calling code. This is just a sanity check that we might1976 * want to remove in the future.1977 */1978 if (WARN_ON_ONCE(page_zone(pfn_to_page(start_pfn)) != zone ||1979 page_zone(pfn_to_page(end_pfn - 1)) != zone)) {1980 ret = -EINVAL;1981 reason = "multizone range";1982 goto failed_removal;1983 }1984 1985 /*1986 * Disable pcplists so that page isolation cannot race with freeing1987 * in a way that pages from isolated pageblock are left on pcplists.1988 */1989 zone_pcp_disable(zone);1990 lru_cache_disable();1991 1992 /* set above range as isolated */1993 ret = start_isolate_page_range(start_pfn, end_pfn,1994 MIGRATE_MOVABLE,1995 MEMORY_OFFLINE | REPORT_FAILURE,1996 GFP_USER | __GFP_MOVABLE | __GFP_RETRY_MAYFAIL);1997 if (ret) {1998 reason = "failure to isolate range";1999 goto failed_removal_pcplists_disabled;2000 }2001 2002 arg.start_pfn = start_pfn;2003 arg.nr_pages = nr_pages;2004 node_states_check_changes_offline(nr_pages, zone, &arg);2005 2006 ret = memory_notify(MEM_GOING_OFFLINE, &arg);2007 ret = notifier_to_errno(ret);2008 if (ret) {2009 reason = "notifier failure";2010 goto failed_removal_isolated;2011 }2012 2013 do {2014 pfn = start_pfn;2015 do {2016 /*2017 * Historically we always checked for any signal and2018 * can't limit it to fatal signals without eventually2019 * breaking user space.2020 */2021 if (signal_pending(current)) {2022 ret = -EINTR;2023 reason = "signal backoff";2024 goto failed_removal_isolated;2025 }2026 2027 cond_resched();2028 2029 ret = scan_movable_pages(pfn, end_pfn, &pfn);2030 if (!ret) {2031 /*2032 * TODO: fatal migration failures should bail2033 * out2034 */2035 do_migrate_range(pfn, end_pfn);2036 }2037 } while (!ret);2038 2039 if (ret != -ENOENT) {2040 reason = "unmovable page";2041 goto failed_removal_isolated;2042 }2043 2044 /*2045 * Dissolve free hugetlb folios in the memory block before doing2046 * offlining actually in order to make hugetlbfs's object2047 * counting consistent.2048 */2049 ret = dissolve_free_hugetlb_folios(start_pfn, end_pfn);2050 if (ret) {2051 reason = "failure to dissolve huge pages";2052 goto failed_removal_isolated;2053 }2054 2055 ret = test_pages_isolated(start_pfn, end_pfn, MEMORY_OFFLINE);2056 2057 } while (ret);2058 2059 /* Mark all sections offline and remove free pages from the buddy. */2060 managed_pages = __offline_isolated_pages(start_pfn, end_pfn);2061 pr_debug("Offlined Pages %ld\n", nr_pages);2062 2063 /*2064 * The memory sections are marked offline, and the pageblock flags2065 * effectively stale; nobody should be touching them. Fixup the number2066 * of isolated pageblocks, memory onlining will properly revert this.2067 */2068 spin_lock_irqsave(&zone->lock, flags);2069 zone->nr_isolate_pageblock -= nr_pages / pageblock_nr_pages;2070 spin_unlock_irqrestore(&zone->lock, flags);2071 2072 lru_cache_enable();2073 zone_pcp_enable(zone);2074 2075 /* removal success */2076 adjust_managed_page_count(pfn_to_page(start_pfn), -managed_pages);2077 adjust_present_page_count(pfn_to_page(start_pfn), group, -nr_pages);2078 2079 /* reinitialise watermarks and update pcp limits */2080 init_per_zone_wmark_min();2081 2082 /*2083 * Make sure to mark the node as memory-less before rebuilding the zone2084 * list. Otherwise this node would still appear in the fallback lists.2085 */2086 node_states_clear_node(node, &arg);2087 if (!populated_zone(zone)) {2088 zone_pcp_reset(zone);2089 build_all_zonelists(NULL);2090 }2091 2092 if (arg.status_change_nid >= 0) {2093 kcompactd_stop(node);2094 kswapd_stop(node);2095 }2096 2097 writeback_set_ratelimit();2098 2099 memory_notify(MEM_OFFLINE, &arg);2100 remove_pfn_range_from_zone(zone, start_pfn, nr_pages);2101 return 0;2102 2103failed_removal_isolated:2104 /* pushback to free area */2105 undo_isolate_page_range(start_pfn, end_pfn, MIGRATE_MOVABLE);2106 memory_notify(MEM_CANCEL_OFFLINE, &arg);2107failed_removal_pcplists_disabled:2108 lru_cache_enable();2109 zone_pcp_enable(zone);2110failed_removal:2111 pr_debug("memory offlining [mem %#010llx-%#010llx] failed due to %s\n",2112 (unsigned long long) start_pfn << PAGE_SHIFT,2113 ((unsigned long long) end_pfn << PAGE_SHIFT) - 1,2114 reason);2115 return ret;2116}2117 2118static int check_memblock_offlined_cb(struct memory_block *mem, void *arg)2119{2120 int *nid = arg;2121 2122 *nid = mem->nid;2123 if (unlikely(mem->state != MEM_OFFLINE)) {2124 phys_addr_t beginpa, endpa;2125 2126 beginpa = PFN_PHYS(section_nr_to_pfn(mem->start_section_nr));2127 endpa = beginpa + memory_block_size_bytes() - 1;2128 pr_warn("removing memory fails, because memory [%pa-%pa] is onlined\n",2129 &beginpa, &endpa);2130 2131 return -EBUSY;2132 }2133 return 0;2134}2135 2136static int count_memory_range_altmaps_cb(struct memory_block *mem, void *arg)2137{2138 u64 *num_altmaps = (u64 *)arg;2139 2140 if (mem->altmap)2141 *num_altmaps += 1;2142 2143 return 0;2144}2145 2146static int check_cpu_on_node(int nid)2147{2148 int cpu;2149 2150 for_each_present_cpu(cpu) {2151 if (cpu_to_node(cpu) == nid)2152 /*2153 * the cpu on this node isn't removed, and we can't2154 * offline this node.2155 */2156 return -EBUSY;2157 }2158 2159 return 0;2160}2161 2162static int check_no_memblock_for_node_cb(struct memory_block *mem, void *arg)2163{2164 int nid = *(int *)arg;2165 2166 /*2167 * If a memory block belongs to multiple nodes, the stored nid is not2168 * reliable. However, such blocks are always online (e.g., cannot get2169 * offlined) and, therefore, are still spanned by the node.2170 */2171 return mem->nid == nid ? -EEXIST : 0;2172}2173 2174/**2175 * try_offline_node2176 * @nid: the node ID2177 *2178 * Offline a node if all memory sections and cpus of the node are removed.2179 *2180 * NOTE: The caller must call lock_device_hotplug() to serialize hotplug2181 * and online/offline operations before this call.2182 */2183void try_offline_node(int nid)2184{2185 int rc;2186 2187 /*2188 * If the node still spans pages (especially ZONE_DEVICE), don't2189 * offline it. A node spans memory after move_pfn_range_to_zone(),2190 * e.g., after the memory block was onlined.2191 */2192 if (node_spanned_pages(nid))2193 return;2194 2195 /*2196 * Especially offline memory blocks might not be spanned by the2197 * node. They will get spanned by the node once they get onlined.2198 * However, they link to the node in sysfs and can get onlined later.2199 */2200 rc = for_each_memory_block(&nid, check_no_memblock_for_node_cb);2201 if (rc)2202 return;2203 2204 if (check_cpu_on_node(nid))2205 return;2206 2207 /*2208 * all memory/cpu of this node are removed, we can offline this2209 * node now.2210 */2211 node_set_offline(nid);2212 unregister_one_node(nid);2213}2214EXPORT_SYMBOL(try_offline_node);2215 2216static int memory_blocks_have_altmaps(u64 start, u64 size)2217{2218 u64 num_memblocks = size / memory_block_size_bytes();2219 u64 num_altmaps = 0;2220 2221 if (!mhp_memmap_on_memory())2222 return 0;2223 2224 walk_memory_blocks(start, size, &num_altmaps,2225 count_memory_range_altmaps_cb);2226 2227 if (num_altmaps == 0)2228 return 0;2229 2230 if (WARN_ON_ONCE(num_memblocks != num_altmaps))2231 return -EINVAL;2232 2233 return 1;2234}2235 2236static int try_remove_memory(u64 start, u64 size)2237{2238 int rc, nid = NUMA_NO_NODE;2239 2240 BUG_ON(check_hotplug_memory_range(start, size));2241 2242 /*2243 * All memory blocks must be offlined before removing memory. Check2244 * whether all memory blocks in question are offline and return error2245 * if this is not the case.2246 *2247 * While at it, determine the nid. Note that if we'd have mixed nodes,2248 * we'd only try to offline the last determined one -- which is good2249 * enough for the cases we care about.2250 */2251 rc = walk_memory_blocks(start, size, &nid, check_memblock_offlined_cb);2252 if (rc)2253 return rc;2254 2255 /* remove memmap entry */2256 firmware_map_remove(start, start + size, "System RAM");2257 2258 mem_hotplug_begin();2259 2260 rc = memory_blocks_have_altmaps(start, size);2261 if (rc < 0) {2262 mem_hotplug_done();2263 return rc;2264 } else if (!rc) {2265 /*2266 * Memory block device removal under the device_hotplug_lock is2267 * a barrier against racing online attempts.2268 * No altmaps present, do the removal directly2269 */2270 remove_memory_block_devices(start, size);2271 arch_remove_memory(start, size, NULL);2272 } else {2273 /* all memblocks in the range have altmaps */2274 remove_memory_blocks_and_altmaps(start, size);2275 }2276 2277 if (IS_ENABLED(CONFIG_ARCH_KEEP_MEMBLOCK))2278 memblock_remove(start, size);2279 2280 release_mem_region_adjustable(start, size);2281 2282 if (nid != NUMA_NO_NODE)2283 try_offline_node(nid);2284 2285 mem_hotplug_done();2286 return 0;2287}2288 2289/**2290 * __remove_memory - Remove memory if every memory block is offline2291 * @start: physical address of the region to remove2292 * @size: size of the region to remove2293 *2294 * NOTE: The caller must call lock_device_hotplug() to serialize hotplug2295 * and online/offline operations before this call, as required by2296 * try_offline_node().2297 */2298void __remove_memory(u64 start, u64 size)2299{2300 2301 /*2302 * trigger BUG() if some memory is not offlined prior to calling this2303 * function2304 */2305 if (try_remove_memory(start, size))2306 BUG();2307}2308 2309/*2310 * Remove memory if every memory block is offline, otherwise return -EBUSY is2311 * some memory is not offline2312 */2313int remove_memory(u64 start, u64 size)2314{2315 int rc;2316 2317 lock_device_hotplug();2318 rc = try_remove_memory(start, size);2319 unlock_device_hotplug();2320 2321 return rc;2322}2323EXPORT_SYMBOL_GPL(remove_memory);2324 2325static int try_offline_memory_block(struct memory_block *mem, void *arg)2326{2327 uint8_t online_type = MMOP_ONLINE_KERNEL;2328 uint8_t **online_types = arg;2329 struct page *page;2330 int rc;2331 2332 /*2333 * Sense the online_type via the zone of the memory block. Offlining2334 * with multiple zones within one memory block will be rejected2335 * by offlining code ... so we don't care about that.2336 */2337 page = pfn_to_online_page(section_nr_to_pfn(mem->start_section_nr));2338 if (page && zone_idx(page_zone(page)) == ZONE_MOVABLE)2339 online_type = MMOP_ONLINE_MOVABLE;2340 2341 rc = device_offline(&mem->dev);2342 /*2343 * Default is MMOP_OFFLINE - change it only if offlining succeeded,2344 * so try_reonline_memory_block() can do the right thing.2345 */2346 if (!rc)2347 **online_types = online_type;2348 2349 (*online_types)++;2350 /* Ignore if already offline. */2351 return rc < 0 ? rc : 0;2352}2353 2354static int try_reonline_memory_block(struct memory_block *mem, void *arg)2355{2356 uint8_t **online_types = arg;2357 int rc;2358 2359 if (**online_types != MMOP_OFFLINE) {2360 mem->online_type = **online_types;2361 rc = device_online(&mem->dev);2362 if (rc < 0)2363 pr_warn("%s: Failed to re-online memory: %d",2364 __func__, rc);2365 }2366 2367 /* Continue processing all remaining memory blocks. */2368 (*online_types)++;2369 return 0;2370}2371 2372/*2373 * Try to offline and remove memory. Might take a long time to finish in case2374 * memory is still in use. Primarily useful for memory devices that logically2375 * unplugged all memory (so it's no longer in use) and want to offline + remove2376 * that memory.2377 */2378int offline_and_remove_memory(u64 start, u64 size)2379{2380 const unsigned long mb_count = size / memory_block_size_bytes();2381 uint8_t *online_types, *tmp;2382 int rc;2383 2384 if (!IS_ALIGNED(start, memory_block_size_bytes()) ||2385 !IS_ALIGNED(size, memory_block_size_bytes()) || !size)2386 return -EINVAL;2387 2388 /*2389 * We'll remember the old online type of each memory block, so we can2390 * try to revert whatever we did when offlining one memory block fails2391 * after offlining some others succeeded.2392 */2393 online_types = kmalloc_array(mb_count, sizeof(*online_types),2394 GFP_KERNEL);2395 if (!online_types)2396 return -ENOMEM;2397 /*2398 * Initialize all states to MMOP_OFFLINE, so when we abort processing in2399 * try_offline_memory_block(), we'll skip all unprocessed blocks in2400 * try_reonline_memory_block().2401 */2402 memset(online_types, MMOP_OFFLINE, mb_count);2403 2404 lock_device_hotplug();2405 2406 tmp = online_types;2407 rc = walk_memory_blocks(start, size, &tmp, try_offline_memory_block);2408 2409 /*2410 * In case we succeeded to offline all memory, remove it.2411 * This cannot fail as it cannot get onlined in the meantime.2412 */2413 if (!rc) {2414 rc = try_remove_memory(start, size);2415 if (rc)2416 pr_err("%s: Failed to remove memory: %d", __func__, rc);2417 }2418 2419 /*2420 * Rollback what we did. While memory onlining might theoretically fail2421 * (nacked by a notifier), it barely ever happens.2422 */2423 if (rc) {2424 tmp = online_types;2425 walk_memory_blocks(start, size, &tmp,2426 try_reonline_memory_block);2427 }2428 unlock_device_hotplug();2429 2430 kfree(online_types);2431 return rc;2432}2433EXPORT_SYMBOL_GPL(offline_and_remove_memory);2434#endif /* CONFIG_MEMORY_HOTREMOVE */2435