129 lines · c
1// SPDX-License-Identifier: GPL-2.02/*3 * Bootmem core functions.4 *5 * Copyright (c) 2020, Bytedance.6 *7 * Author: Muchun Song <songmuchun@bytedance.com>8 *9 */10#include <linux/mm.h>11#include <linux/compiler.h>12#include <linux/memblock.h>13#include <linux/bootmem_info.h>14#include <linux/memory_hotplug.h>15#include <linux/kmemleak.h>16 17void get_page_bootmem(unsigned long info, struct page *page, unsigned long type)18{19 page->index = type;20 SetPagePrivate(page);21 set_page_private(page, info);22 page_ref_inc(page);23}24 25void put_page_bootmem(struct page *page)26{27 unsigned long type = page->index;28 29 BUG_ON(type < MEMORY_HOTPLUG_MIN_BOOTMEM_TYPE ||30 type > MEMORY_HOTPLUG_MAX_BOOTMEM_TYPE);31 32 if (page_ref_dec_return(page) == 1) {33 page->index = 0;34 ClearPagePrivate(page);35 set_page_private(page, 0);36 INIT_LIST_HEAD(&page->lru);37 kmemleak_free_part_phys(PFN_PHYS(page_to_pfn(page)), PAGE_SIZE);38 free_reserved_page(page);39 }40}41 42#ifndef CONFIG_SPARSEMEM_VMEMMAP43static void __init register_page_bootmem_info_section(unsigned long start_pfn)44{45 unsigned long mapsize, section_nr, i;46 struct mem_section *ms;47 struct page *page, *memmap;48 struct mem_section_usage *usage;49 50 section_nr = pfn_to_section_nr(start_pfn);51 ms = __nr_to_section(section_nr);52 53 /* Get section's memmap address */54 memmap = sparse_decode_mem_map(ms->section_mem_map, section_nr);55 56 /*57 * Get page for the memmap's phys address58 * XXX: need more consideration for sparse_vmemmap...59 */60 page = virt_to_page(memmap);61 mapsize = sizeof(struct page) * PAGES_PER_SECTION;62 mapsize = PAGE_ALIGN(mapsize) >> PAGE_SHIFT;63 64 /* remember memmap's page */65 for (i = 0; i < mapsize; i++, page++)66 get_page_bootmem(section_nr, page, SECTION_INFO);67 68 usage = ms->usage;69 page = virt_to_page(usage);70 71 mapsize = PAGE_ALIGN(mem_section_usage_size()) >> PAGE_SHIFT;72 73 for (i = 0; i < mapsize; i++, page++)74 get_page_bootmem(section_nr, page, MIX_SECTION_INFO);75 76}77#else /* CONFIG_SPARSEMEM_VMEMMAP */78static void __init register_page_bootmem_info_section(unsigned long start_pfn)79{80 unsigned long mapsize, section_nr, i;81 struct mem_section *ms;82 struct page *page, *memmap;83 struct mem_section_usage *usage;84 85 section_nr = pfn_to_section_nr(start_pfn);86 ms = __nr_to_section(section_nr);87 88 memmap = sparse_decode_mem_map(ms->section_mem_map, section_nr);89 90 register_page_bootmem_memmap(section_nr, memmap, PAGES_PER_SECTION);91 92 usage = ms->usage;93 page = virt_to_page(usage);94 95 mapsize = PAGE_ALIGN(mem_section_usage_size()) >> PAGE_SHIFT;96 97 for (i = 0; i < mapsize; i++, page++)98 get_page_bootmem(section_nr, page, MIX_SECTION_INFO);99}100#endif /* !CONFIG_SPARSEMEM_VMEMMAP */101 102void __init register_page_bootmem_info_node(struct pglist_data *pgdat)103{104 unsigned long i, pfn, end_pfn, nr_pages;105 int node = pgdat->node_id;106 struct page *page;107 108 nr_pages = PAGE_ALIGN(sizeof(struct pglist_data)) >> PAGE_SHIFT;109 page = virt_to_page(pgdat);110 111 for (i = 0; i < nr_pages; i++, page++)112 get_page_bootmem(node, page, NODE_INFO);113 114 pfn = pgdat->node_start_pfn;115 end_pfn = pgdat_end_pfn(pgdat);116 117 /* register section info */118 for (; pfn < end_pfn; pfn += PAGES_PER_SECTION) {119 /*120 * Some platforms can assign the same pfn to multiple nodes - on121 * node0 as well as nodeN. To avoid registering a pfn against122 * multiple nodes we check that this pfn does not already123 * reside in some other nodes.124 */125 if (pfn_valid(pfn) && (early_pfn_to_nid(pfn) == node))126 register_page_bootmem_info_section(pfn);127 }128}129