2815 lines · c
1// SPDX-License-Identifier: GPL-2.02#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt3 4#include <linux/mm.h>5#include <linux/sched.h>6#include <linux/sched/mm.h>7#include <linux/sched/coredump.h>8#include <linux/mmu_notifier.h>9#include <linux/rmap.h>10#include <linux/swap.h>11#include <linux/mm_inline.h>12#include <linux/kthread.h>13#include <linux/khugepaged.h>14#include <linux/freezer.h>15#include <linux/mman.h>16#include <linux/hashtable.h>17#include <linux/userfaultfd_k.h>18#include <linux/page_idle.h>19#include <linux/page_table_check.h>20#include <linux/rcupdate_wait.h>21#include <linux/swapops.h>22#include <linux/shmem_fs.h>23#include <linux/ksm.h>24 25#include <asm/tlb.h>26#include <asm/pgalloc.h>27#include "internal.h"28#include "mm_slot.h"29 30enum scan_result {31 SCAN_FAIL,32 SCAN_SUCCEED,33 SCAN_PMD_NULL,34 SCAN_PMD_NONE,35 SCAN_PMD_MAPPED,36 SCAN_EXCEED_NONE_PTE,37 SCAN_EXCEED_SWAP_PTE,38 SCAN_EXCEED_SHARED_PTE,39 SCAN_PTE_NON_PRESENT,40 SCAN_PTE_UFFD_WP,41 SCAN_PTE_MAPPED_HUGEPAGE,42 SCAN_PAGE_RO,43 SCAN_LACK_REFERENCED_PAGE,44 SCAN_PAGE_NULL,45 SCAN_SCAN_ABORT,46 SCAN_PAGE_COUNT,47 SCAN_PAGE_LRU,48 SCAN_PAGE_LOCK,49 SCAN_PAGE_ANON,50 SCAN_PAGE_COMPOUND,51 SCAN_ANY_PROCESS,52 SCAN_VMA_NULL,53 SCAN_VMA_CHECK,54 SCAN_ADDRESS_RANGE,55 SCAN_DEL_PAGE_LRU,56 SCAN_ALLOC_HUGE_PAGE_FAIL,57 SCAN_CGROUP_CHARGE_FAIL,58 SCAN_TRUNCATED,59 SCAN_PAGE_HAS_PRIVATE,60 SCAN_STORE_FAILED,61 SCAN_COPY_MC,62 SCAN_PAGE_FILLED,63};64 65#define CREATE_TRACE_POINTS66#include <trace/events/huge_memory.h>67 68static struct task_struct *khugepaged_thread __read_mostly;69static DEFINE_MUTEX(khugepaged_mutex);70 71/* default scan 8*512 pte (or vmas) every 30 second */72static unsigned int khugepaged_pages_to_scan __read_mostly;73static unsigned int khugepaged_pages_collapsed;74static unsigned int khugepaged_full_scans;75static unsigned int khugepaged_scan_sleep_millisecs __read_mostly = 10000;76/* during fragmentation poll the hugepage allocator once every minute */77static unsigned int khugepaged_alloc_sleep_millisecs __read_mostly = 60000;78static unsigned long khugepaged_sleep_expire;79static DEFINE_SPINLOCK(khugepaged_mm_lock);80static DECLARE_WAIT_QUEUE_HEAD(khugepaged_wait);81/*82 * default collapse hugepages if there is at least one pte mapped like83 * it would have happened if the vma was large enough during page84 * fault.85 *86 * Note that these are only respected if collapse was initiated by khugepaged.87 */88unsigned int khugepaged_max_ptes_none __read_mostly;89static unsigned int khugepaged_max_ptes_swap __read_mostly;90static unsigned int khugepaged_max_ptes_shared __read_mostly;91 92#define MM_SLOTS_HASH_BITS 1093static DEFINE_READ_MOSTLY_HASHTABLE(mm_slots_hash, MM_SLOTS_HASH_BITS);94 95static struct kmem_cache *mm_slot_cache __ro_after_init;96 97struct collapse_control {98 bool is_khugepaged;99 100 /* Num pages scanned per node */101 u32 node_load[MAX_NUMNODES];102 103 /* nodemask for allocation fallback */104 nodemask_t alloc_nmask;105};106 107/**108 * struct khugepaged_mm_slot - khugepaged information per mm that is being scanned109 * @slot: hash lookup from mm to mm_slot110 */111struct khugepaged_mm_slot {112 struct mm_slot slot;113};114 115/**116 * struct khugepaged_scan - cursor for scanning117 * @mm_head: the head of the mm list to scan118 * @mm_slot: the current mm_slot we are scanning119 * @address: the next address inside that to be scanned120 *121 * There is only the one khugepaged_scan instance of this cursor structure.122 */123struct khugepaged_scan {124 struct list_head mm_head;125 struct khugepaged_mm_slot *mm_slot;126 unsigned long address;127};128 129static struct khugepaged_scan khugepaged_scan = {130 .mm_head = LIST_HEAD_INIT(khugepaged_scan.mm_head),131};132 133#ifdef CONFIG_SYSFS134static ssize_t scan_sleep_millisecs_show(struct kobject *kobj,135 struct kobj_attribute *attr,136 char *buf)137{138 return sysfs_emit(buf, "%u\n", khugepaged_scan_sleep_millisecs);139}140 141static ssize_t scan_sleep_millisecs_store(struct kobject *kobj,142 struct kobj_attribute *attr,143 const char *buf, size_t count)144{145 unsigned int msecs;146 int err;147 148 err = kstrtouint(buf, 10, &msecs);149 if (err)150 return -EINVAL;151 152 khugepaged_scan_sleep_millisecs = msecs;153 khugepaged_sleep_expire = 0;154 wake_up_interruptible(&khugepaged_wait);155 156 return count;157}158static struct kobj_attribute scan_sleep_millisecs_attr =159 __ATTR_RW(scan_sleep_millisecs);160 161static ssize_t alloc_sleep_millisecs_show(struct kobject *kobj,162 struct kobj_attribute *attr,163 char *buf)164{165 return sysfs_emit(buf, "%u\n", khugepaged_alloc_sleep_millisecs);166}167 168static ssize_t alloc_sleep_millisecs_store(struct kobject *kobj,169 struct kobj_attribute *attr,170 const char *buf, size_t count)171{172 unsigned int msecs;173 int err;174 175 err = kstrtouint(buf, 10, &msecs);176 if (err)177 return -EINVAL;178 179 khugepaged_alloc_sleep_millisecs = msecs;180 khugepaged_sleep_expire = 0;181 wake_up_interruptible(&khugepaged_wait);182 183 return count;184}185static struct kobj_attribute alloc_sleep_millisecs_attr =186 __ATTR_RW(alloc_sleep_millisecs);187 188static ssize_t pages_to_scan_show(struct kobject *kobj,189 struct kobj_attribute *attr,190 char *buf)191{192 return sysfs_emit(buf, "%u\n", khugepaged_pages_to_scan);193}194static ssize_t pages_to_scan_store(struct kobject *kobj,195 struct kobj_attribute *attr,196 const char *buf, size_t count)197{198 unsigned int pages;199 int err;200 201 err = kstrtouint(buf, 10, &pages);202 if (err || !pages)203 return -EINVAL;204 205 khugepaged_pages_to_scan = pages;206 207 return count;208}209static struct kobj_attribute pages_to_scan_attr =210 __ATTR_RW(pages_to_scan);211 212static ssize_t pages_collapsed_show(struct kobject *kobj,213 struct kobj_attribute *attr,214 char *buf)215{216 return sysfs_emit(buf, "%u\n", khugepaged_pages_collapsed);217}218static struct kobj_attribute pages_collapsed_attr =219 __ATTR_RO(pages_collapsed);220 221static ssize_t full_scans_show(struct kobject *kobj,222 struct kobj_attribute *attr,223 char *buf)224{225 return sysfs_emit(buf, "%u\n", khugepaged_full_scans);226}227static struct kobj_attribute full_scans_attr =228 __ATTR_RO(full_scans);229 230static ssize_t defrag_show(struct kobject *kobj,231 struct kobj_attribute *attr, char *buf)232{233 return single_hugepage_flag_show(kobj, attr, buf,234 TRANSPARENT_HUGEPAGE_DEFRAG_KHUGEPAGED_FLAG);235}236static ssize_t defrag_store(struct kobject *kobj,237 struct kobj_attribute *attr,238 const char *buf, size_t count)239{240 return single_hugepage_flag_store(kobj, attr, buf, count,241 TRANSPARENT_HUGEPAGE_DEFRAG_KHUGEPAGED_FLAG);242}243static struct kobj_attribute khugepaged_defrag_attr =244 __ATTR_RW(defrag);245 246/*247 * max_ptes_none controls if khugepaged should collapse hugepages over248 * any unmapped ptes in turn potentially increasing the memory249 * footprint of the vmas. When max_ptes_none is 0 khugepaged will not250 * reduce the available free memory in the system as it251 * runs. Increasing max_ptes_none will instead potentially reduce the252 * free memory in the system during the khugepaged scan.253 */254static ssize_t max_ptes_none_show(struct kobject *kobj,255 struct kobj_attribute *attr,256 char *buf)257{258 return sysfs_emit(buf, "%u\n", khugepaged_max_ptes_none);259}260static ssize_t max_ptes_none_store(struct kobject *kobj,261 struct kobj_attribute *attr,262 const char *buf, size_t count)263{264 int err;265 unsigned long max_ptes_none;266 267 err = kstrtoul(buf, 10, &max_ptes_none);268 if (err || max_ptes_none > HPAGE_PMD_NR - 1)269 return -EINVAL;270 271 khugepaged_max_ptes_none = max_ptes_none;272 273 return count;274}275static struct kobj_attribute khugepaged_max_ptes_none_attr =276 __ATTR_RW(max_ptes_none);277 278static ssize_t max_ptes_swap_show(struct kobject *kobj,279 struct kobj_attribute *attr,280 char *buf)281{282 return sysfs_emit(buf, "%u\n", khugepaged_max_ptes_swap);283}284 285static ssize_t max_ptes_swap_store(struct kobject *kobj,286 struct kobj_attribute *attr,287 const char *buf, size_t count)288{289 int err;290 unsigned long max_ptes_swap;291 292 err = kstrtoul(buf, 10, &max_ptes_swap);293 if (err || max_ptes_swap > HPAGE_PMD_NR - 1)294 return -EINVAL;295 296 khugepaged_max_ptes_swap = max_ptes_swap;297 298 return count;299}300 301static struct kobj_attribute khugepaged_max_ptes_swap_attr =302 __ATTR_RW(max_ptes_swap);303 304static ssize_t max_ptes_shared_show(struct kobject *kobj,305 struct kobj_attribute *attr,306 char *buf)307{308 return sysfs_emit(buf, "%u\n", khugepaged_max_ptes_shared);309}310 311static ssize_t max_ptes_shared_store(struct kobject *kobj,312 struct kobj_attribute *attr,313 const char *buf, size_t count)314{315 int err;316 unsigned long max_ptes_shared;317 318 err = kstrtoul(buf, 10, &max_ptes_shared);319 if (err || max_ptes_shared > HPAGE_PMD_NR - 1)320 return -EINVAL;321 322 khugepaged_max_ptes_shared = max_ptes_shared;323 324 return count;325}326 327static struct kobj_attribute khugepaged_max_ptes_shared_attr =328 __ATTR_RW(max_ptes_shared);329 330static struct attribute *khugepaged_attr[] = {331 &khugepaged_defrag_attr.attr,332 &khugepaged_max_ptes_none_attr.attr,333 &khugepaged_max_ptes_swap_attr.attr,334 &khugepaged_max_ptes_shared_attr.attr,335 &pages_to_scan_attr.attr,336 &pages_collapsed_attr.attr,337 &full_scans_attr.attr,338 &scan_sleep_millisecs_attr.attr,339 &alloc_sleep_millisecs_attr.attr,340 NULL,341};342 343struct attribute_group khugepaged_attr_group = {344 .attrs = khugepaged_attr,345 .name = "khugepaged",346};347#endif /* CONFIG_SYSFS */348 349int hugepage_madvise(struct vm_area_struct *vma,350 unsigned long *vm_flags, int advice)351{352 switch (advice) {353 case MADV_HUGEPAGE:354#ifdef CONFIG_S390355 /*356 * qemu blindly sets MADV_HUGEPAGE on all allocations, but s390357 * can't handle this properly after s390_enable_sie, so we simply358 * ignore the madvise to prevent qemu from causing a SIGSEGV.359 */360 if (mm_has_pgste(vma->vm_mm))361 return 0;362#endif363 *vm_flags &= ~VM_NOHUGEPAGE;364 *vm_flags |= VM_HUGEPAGE;365 /*366 * If the vma become good for khugepaged to scan,367 * register it here without waiting a page fault that368 * may not happen any time soon.369 */370 khugepaged_enter_vma(vma, *vm_flags);371 break;372 case MADV_NOHUGEPAGE:373 *vm_flags &= ~VM_HUGEPAGE;374 *vm_flags |= VM_NOHUGEPAGE;375 /*376 * Setting VM_NOHUGEPAGE will prevent khugepaged from scanning377 * this vma even if we leave the mm registered in khugepaged if378 * it got registered before VM_NOHUGEPAGE was set.379 */380 break;381 }382 383 return 0;384}385 386int __init khugepaged_init(void)387{388 mm_slot_cache = KMEM_CACHE(khugepaged_mm_slot, 0);389 if (!mm_slot_cache)390 return -ENOMEM;391 392 khugepaged_pages_to_scan = HPAGE_PMD_NR * 8;393 khugepaged_max_ptes_none = HPAGE_PMD_NR - 1;394 khugepaged_max_ptes_swap = HPAGE_PMD_NR / 8;395 khugepaged_max_ptes_shared = HPAGE_PMD_NR / 2;396 397 return 0;398}399 400void __init khugepaged_destroy(void)401{402 kmem_cache_destroy(mm_slot_cache);403}404 405static inline int hpage_collapse_test_exit(struct mm_struct *mm)406{407 return atomic_read(&mm->mm_users) == 0;408}409 410static inline int hpage_collapse_test_exit_or_disable(struct mm_struct *mm)411{412 return hpage_collapse_test_exit(mm) ||413 test_bit(MMF_DISABLE_THP, &mm->flags);414}415 416static bool hugepage_pmd_enabled(void)417{418 /*419 * We cover both the anon and the file-backed case here; file-backed420 * hugepages, when configured in, are determined by the global control.421 * Anon pmd-sized hugepages are determined by the pmd-size control.422 */423 if (IS_ENABLED(CONFIG_READ_ONLY_THP_FOR_FS) &&424 hugepage_global_enabled())425 return true;426 if (test_bit(PMD_ORDER, &huge_anon_orders_always))427 return true;428 if (test_bit(PMD_ORDER, &huge_anon_orders_madvise))429 return true;430 if (test_bit(PMD_ORDER, &huge_anon_orders_inherit) &&431 hugepage_global_enabled())432 return true;433 return false;434}435 436void __khugepaged_enter(struct mm_struct *mm)437{438 struct khugepaged_mm_slot *mm_slot;439 struct mm_slot *slot;440 int wakeup;441 442 /* __khugepaged_exit() must not run from under us */443 VM_BUG_ON_MM(hpage_collapse_test_exit(mm), mm);444 if (unlikely(test_and_set_bit(MMF_VM_HUGEPAGE, &mm->flags)))445 return;446 447 mm_slot = mm_slot_alloc(mm_slot_cache);448 if (!mm_slot)449 return;450 451 slot = &mm_slot->slot;452 453 spin_lock(&khugepaged_mm_lock);454 mm_slot_insert(mm_slots_hash, mm, slot);455 /*456 * Insert just behind the scanning cursor, to let the area settle457 * down a little.458 */459 wakeup = list_empty(&khugepaged_scan.mm_head);460 list_add_tail(&slot->mm_node, &khugepaged_scan.mm_head);461 spin_unlock(&khugepaged_mm_lock);462 463 mmgrab(mm);464 if (wakeup)465 wake_up_interruptible(&khugepaged_wait);466}467 468void khugepaged_enter_vma(struct vm_area_struct *vma,469 unsigned long vm_flags)470{471 if (!test_bit(MMF_VM_HUGEPAGE, &vma->vm_mm->flags) &&472 hugepage_pmd_enabled()) {473 if (thp_vma_allowable_order(vma, vm_flags, TVA_ENFORCE_SYSFS,474 PMD_ORDER))475 __khugepaged_enter(vma->vm_mm);476 }477}478 479void __khugepaged_exit(struct mm_struct *mm)480{481 struct khugepaged_mm_slot *mm_slot;482 struct mm_slot *slot;483 int free = 0;484 485 spin_lock(&khugepaged_mm_lock);486 slot = mm_slot_lookup(mm_slots_hash, mm);487 mm_slot = mm_slot_entry(slot, struct khugepaged_mm_slot, slot);488 if (mm_slot && khugepaged_scan.mm_slot != mm_slot) {489 hash_del(&slot->hash);490 list_del(&slot->mm_node);491 free = 1;492 }493 spin_unlock(&khugepaged_mm_lock);494 495 if (free) {496 clear_bit(MMF_VM_HUGEPAGE, &mm->flags);497 mm_slot_free(mm_slot_cache, mm_slot);498 mmdrop(mm);499 } else if (mm_slot) {500 /*501 * This is required to serialize against502 * hpage_collapse_test_exit() (which is guaranteed to run503 * under mmap sem read mode). Stop here (after we return all504 * pagetables will be destroyed) until khugepaged has finished505 * working on the pagetables under the mmap_lock.506 */507 mmap_write_lock(mm);508 mmap_write_unlock(mm);509 }510}511 512static void release_pte_folio(struct folio *folio)513{514 node_stat_mod_folio(folio,515 NR_ISOLATED_ANON + folio_is_file_lru(folio),516 -folio_nr_pages(folio));517 folio_unlock(folio);518 folio_putback_lru(folio);519}520 521static void release_pte_pages(pte_t *pte, pte_t *_pte,522 struct list_head *compound_pagelist)523{524 struct folio *folio, *tmp;525 526 while (--_pte >= pte) {527 pte_t pteval = ptep_get(_pte);528 unsigned long pfn;529 530 if (pte_none(pteval))531 continue;532 pfn = pte_pfn(pteval);533 if (is_zero_pfn(pfn))534 continue;535 folio = pfn_folio(pfn);536 if (folio_test_large(folio))537 continue;538 release_pte_folio(folio);539 }540 541 list_for_each_entry_safe(folio, tmp, compound_pagelist, lru) {542 list_del(&folio->lru);543 release_pte_folio(folio);544 }545}546 547static bool is_refcount_suitable(struct folio *folio)548{549 int expected_refcount = folio_mapcount(folio);550 551 if (!folio_test_anon(folio) || folio_test_swapcache(folio))552 expected_refcount += folio_nr_pages(folio);553 554 if (folio_test_private(folio))555 expected_refcount++;556 557 return folio_ref_count(folio) == expected_refcount;558}559 560static int __collapse_huge_page_isolate(struct vm_area_struct *vma,561 unsigned long address,562 pte_t *pte,563 struct collapse_control *cc,564 struct list_head *compound_pagelist)565{566 struct page *page = NULL;567 struct folio *folio = NULL;568 pte_t *_pte;569 int none_or_zero = 0, shared = 0, result = SCAN_FAIL, referenced = 0;570 bool writable = false;571 572 for (_pte = pte; _pte < pte + HPAGE_PMD_NR;573 _pte++, address += PAGE_SIZE) {574 pte_t pteval = ptep_get(_pte);575 if (pte_none(pteval) || (pte_present(pteval) &&576 is_zero_pfn(pte_pfn(pteval)))) {577 ++none_or_zero;578 if (!userfaultfd_armed(vma) &&579 (!cc->is_khugepaged ||580 none_or_zero <= khugepaged_max_ptes_none)) {581 continue;582 } else {583 result = SCAN_EXCEED_NONE_PTE;584 count_vm_event(THP_SCAN_EXCEED_NONE_PTE);585 goto out;586 }587 }588 if (!pte_present(pteval)) {589 result = SCAN_PTE_NON_PRESENT;590 goto out;591 }592 if (pte_uffd_wp(pteval)) {593 result = SCAN_PTE_UFFD_WP;594 goto out;595 }596 page = vm_normal_page(vma, address, pteval);597 if (unlikely(!page) || unlikely(is_zone_device_page(page))) {598 result = SCAN_PAGE_NULL;599 goto out;600 }601 602 folio = page_folio(page);603 VM_BUG_ON_FOLIO(!folio_test_anon(folio), folio);604 605 /* See hpage_collapse_scan_pmd(). */606 if (folio_likely_mapped_shared(folio)) {607 ++shared;608 if (cc->is_khugepaged &&609 shared > khugepaged_max_ptes_shared) {610 result = SCAN_EXCEED_SHARED_PTE;611 count_vm_event(THP_SCAN_EXCEED_SHARED_PTE);612 goto out;613 }614 }615 616 if (folio_test_large(folio)) {617 struct folio *f;618 619 /*620 * Check if we have dealt with the compound page621 * already622 */623 list_for_each_entry(f, compound_pagelist, lru) {624 if (folio == f)625 goto next;626 }627 }628 629 /*630 * We can do it before folio_isolate_lru because the631 * folio can't be freed from under us. NOTE: PG_lock632 * is needed to serialize against split_huge_page633 * when invoked from the VM.634 */635 if (!folio_trylock(folio)) {636 result = SCAN_PAGE_LOCK;637 goto out;638 }639 640 /*641 * Check if the page has any GUP (or other external) pins.642 *643 * The page table that maps the page has been already unlinked644 * from the page table tree and this process cannot get645 * an additional pin on the page.646 *647 * New pins can come later if the page is shared across fork,648 * but not from this process. The other process cannot write to649 * the page, only trigger CoW.650 */651 if (!is_refcount_suitable(folio)) {652 folio_unlock(folio);653 result = SCAN_PAGE_COUNT;654 goto out;655 }656 657 /*658 * Isolate the page to avoid collapsing an hugepage659 * currently in use by the VM.660 */661 if (!folio_isolate_lru(folio)) {662 folio_unlock(folio);663 result = SCAN_DEL_PAGE_LRU;664 goto out;665 }666 node_stat_mod_folio(folio,667 NR_ISOLATED_ANON + folio_is_file_lru(folio),668 folio_nr_pages(folio));669 VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio);670 VM_BUG_ON_FOLIO(folio_test_lru(folio), folio);671 672 if (folio_test_large(folio))673 list_add_tail(&folio->lru, compound_pagelist);674next:675 /*676 * If collapse was initiated by khugepaged, check that there is677 * enough young pte to justify collapsing the page678 */679 if (cc->is_khugepaged &&680 (pte_young(pteval) || folio_test_young(folio) ||681 folio_test_referenced(folio) || mmu_notifier_test_young(vma->vm_mm,682 address)))683 referenced++;684 685 if (pte_write(pteval))686 writable = true;687 }688 689 if (unlikely(!writable)) {690 result = SCAN_PAGE_RO;691 } else if (unlikely(cc->is_khugepaged && !referenced)) {692 result = SCAN_LACK_REFERENCED_PAGE;693 } else {694 result = SCAN_SUCCEED;695 trace_mm_collapse_huge_page_isolate(&folio->page, none_or_zero,696 referenced, writable, result);697 return result;698 }699out:700 release_pte_pages(pte, _pte, compound_pagelist);701 trace_mm_collapse_huge_page_isolate(&folio->page, none_or_zero,702 referenced, writable, result);703 return result;704}705 706static void __collapse_huge_page_copy_succeeded(pte_t *pte,707 struct vm_area_struct *vma,708 unsigned long address,709 spinlock_t *ptl,710 struct list_head *compound_pagelist)711{712 struct folio *src, *tmp;713 pte_t *_pte;714 pte_t pteval;715 716 for (_pte = pte; _pte < pte + HPAGE_PMD_NR;717 _pte++, address += PAGE_SIZE) {718 pteval = ptep_get(_pte);719 if (pte_none(pteval) || is_zero_pfn(pte_pfn(pteval))) {720 add_mm_counter(vma->vm_mm, MM_ANONPAGES, 1);721 if (is_zero_pfn(pte_pfn(pteval))) {722 /*723 * ptl mostly unnecessary.724 */725 spin_lock(ptl);726 ptep_clear(vma->vm_mm, address, _pte);727 spin_unlock(ptl);728 ksm_might_unmap_zero_page(vma->vm_mm, pteval);729 }730 } else {731 struct page *src_page = pte_page(pteval);732 733 src = page_folio(src_page);734 if (!folio_test_large(src))735 release_pte_folio(src);736 /*737 * ptl mostly unnecessary, but preempt has to738 * be disabled to update the per-cpu stats739 * inside folio_remove_rmap_pte().740 */741 spin_lock(ptl);742 ptep_clear(vma->vm_mm, address, _pte);743 folio_remove_rmap_pte(src, src_page, vma);744 spin_unlock(ptl);745 free_page_and_swap_cache(src_page);746 }747 }748 749 list_for_each_entry_safe(src, tmp, compound_pagelist, lru) {750 list_del(&src->lru);751 node_stat_sub_folio(src, NR_ISOLATED_ANON +752 folio_is_file_lru(src));753 folio_unlock(src);754 free_swap_cache(src);755 folio_putback_lru(src);756 }757}758 759static void __collapse_huge_page_copy_failed(pte_t *pte,760 pmd_t *pmd,761 pmd_t orig_pmd,762 struct vm_area_struct *vma,763 struct list_head *compound_pagelist)764{765 spinlock_t *pmd_ptl;766 767 /*768 * Re-establish the PMD to point to the original page table769 * entry. Restoring PMD needs to be done prior to releasing770 * pages. Since pages are still isolated and locked here,771 * acquiring anon_vma_lock_write is unnecessary.772 */773 pmd_ptl = pmd_lock(vma->vm_mm, pmd);774 pmd_populate(vma->vm_mm, pmd, pmd_pgtable(orig_pmd));775 spin_unlock(pmd_ptl);776 /*777 * Release both raw and compound pages isolated778 * in __collapse_huge_page_isolate.779 */780 release_pte_pages(pte, pte + HPAGE_PMD_NR, compound_pagelist);781}782 783/*784 * __collapse_huge_page_copy - attempts to copy memory contents from raw785 * pages to a hugepage. Cleans up the raw pages if copying succeeds;786 * otherwise restores the original page table and releases isolated raw pages.787 * Returns SCAN_SUCCEED if copying succeeds, otherwise returns SCAN_COPY_MC.788 *789 * @pte: starting of the PTEs to copy from790 * @folio: the new hugepage to copy contents to791 * @pmd: pointer to the new hugepage's PMD792 * @orig_pmd: the original raw pages' PMD793 * @vma: the original raw pages' virtual memory area794 * @address: starting address to copy795 * @ptl: lock on raw pages' PTEs796 * @compound_pagelist: list that stores compound pages797 */798static int __collapse_huge_page_copy(pte_t *pte, struct folio *folio,799 pmd_t *pmd, pmd_t orig_pmd, struct vm_area_struct *vma,800 unsigned long address, spinlock_t *ptl,801 struct list_head *compound_pagelist)802{803 unsigned int i;804 int result = SCAN_SUCCEED;805 806 /*807 * Copying pages' contents is subject to memory poison at any iteration.808 */809 for (i = 0; i < HPAGE_PMD_NR; i++) {810 pte_t pteval = ptep_get(pte + i);811 struct page *page = folio_page(folio, i);812 unsigned long src_addr = address + i * PAGE_SIZE;813 struct page *src_page;814 815 if (pte_none(pteval) || is_zero_pfn(pte_pfn(pteval))) {816 clear_user_highpage(page, src_addr);817 continue;818 }819 src_page = pte_page(pteval);820 if (copy_mc_user_highpage(page, src_page, src_addr, vma) > 0) {821 result = SCAN_COPY_MC;822 break;823 }824 }825 826 if (likely(result == SCAN_SUCCEED))827 __collapse_huge_page_copy_succeeded(pte, vma, address, ptl,828 compound_pagelist);829 else830 __collapse_huge_page_copy_failed(pte, pmd, orig_pmd, vma,831 compound_pagelist);832 833 return result;834}835 836static void khugepaged_alloc_sleep(void)837{838 DEFINE_WAIT(wait);839 840 add_wait_queue(&khugepaged_wait, &wait);841 __set_current_state(TASK_INTERRUPTIBLE|TASK_FREEZABLE);842 schedule_timeout(msecs_to_jiffies(khugepaged_alloc_sleep_millisecs));843 remove_wait_queue(&khugepaged_wait, &wait);844}845 846struct collapse_control khugepaged_collapse_control = {847 .is_khugepaged = true,848};849 850static bool hpage_collapse_scan_abort(int nid, struct collapse_control *cc)851{852 int i;853 854 /*855 * If node_reclaim_mode is disabled, then no extra effort is made to856 * allocate memory locally.857 */858 if (!node_reclaim_enabled())859 return false;860 861 /* If there is a count for this node already, it must be acceptable */862 if (cc->node_load[nid])863 return false;864 865 for (i = 0; i < MAX_NUMNODES; i++) {866 if (!cc->node_load[i])867 continue;868 if (node_distance(nid, i) > node_reclaim_distance)869 return true;870 }871 return false;872}873 874#define khugepaged_defrag() \875 (transparent_hugepage_flags & \876 (1<<TRANSPARENT_HUGEPAGE_DEFRAG_KHUGEPAGED_FLAG))877 878/* Defrag for khugepaged will enter direct reclaim/compaction if necessary */879static inline gfp_t alloc_hugepage_khugepaged_gfpmask(void)880{881 return khugepaged_defrag() ? GFP_TRANSHUGE : GFP_TRANSHUGE_LIGHT;882}883 884#ifdef CONFIG_NUMA885static int hpage_collapse_find_target_node(struct collapse_control *cc)886{887 int nid, target_node = 0, max_value = 0;888 889 /* find first node with max normal pages hit */890 for (nid = 0; nid < MAX_NUMNODES; nid++)891 if (cc->node_load[nid] > max_value) {892 max_value = cc->node_load[nid];893 target_node = nid;894 }895 896 for_each_online_node(nid) {897 if (max_value == cc->node_load[nid])898 node_set(nid, cc->alloc_nmask);899 }900 901 return target_node;902}903#else904static int hpage_collapse_find_target_node(struct collapse_control *cc)905{906 return 0;907}908#endif909 910/*911 * If mmap_lock temporarily dropped, revalidate vma912 * before taking mmap_lock.913 * Returns enum scan_result value.914 */915 916static int hugepage_vma_revalidate(struct mm_struct *mm, unsigned long address,917 bool expect_anon,918 struct vm_area_struct **vmap,919 struct collapse_control *cc)920{921 struct vm_area_struct *vma;922 unsigned long tva_flags = cc->is_khugepaged ? TVA_ENFORCE_SYSFS : 0;923 924 if (unlikely(hpage_collapse_test_exit_or_disable(mm)))925 return SCAN_ANY_PROCESS;926 927 *vmap = vma = find_vma(mm, address);928 if (!vma)929 return SCAN_VMA_NULL;930 931 if (!thp_vma_suitable_order(vma, address, PMD_ORDER))932 return SCAN_ADDRESS_RANGE;933 if (!thp_vma_allowable_order(vma, vma->vm_flags, tva_flags, PMD_ORDER))934 return SCAN_VMA_CHECK;935 /*936 * Anon VMA expected, the address may be unmapped then937 * remapped to file after khugepaged reaquired the mmap_lock.938 *939 * thp_vma_allowable_order may return true for qualified file940 * vmas.941 */942 if (expect_anon && (!(*vmap)->anon_vma || !vma_is_anonymous(*vmap)))943 return SCAN_PAGE_ANON;944 return SCAN_SUCCEED;945}946 947static int find_pmd_or_thp_or_none(struct mm_struct *mm,948 unsigned long address,949 pmd_t **pmd)950{951 pmd_t pmde;952 953 *pmd = mm_find_pmd(mm, address);954 if (!*pmd)955 return SCAN_PMD_NULL;956 957 pmde = pmdp_get_lockless(*pmd);958 if (pmd_none(pmde))959 return SCAN_PMD_NONE;960 if (!pmd_present(pmde))961 return SCAN_PMD_NULL;962 if (pmd_trans_huge(pmde))963 return SCAN_PMD_MAPPED;964 if (pmd_devmap(pmde))965 return SCAN_PMD_NULL;966 if (pmd_bad(pmde))967 return SCAN_PMD_NULL;968 return SCAN_SUCCEED;969}970 971static int check_pmd_still_valid(struct mm_struct *mm,972 unsigned long address,973 pmd_t *pmd)974{975 pmd_t *new_pmd;976 int result = find_pmd_or_thp_or_none(mm, address, &new_pmd);977 978 if (result != SCAN_SUCCEED)979 return result;980 if (new_pmd != pmd)981 return SCAN_FAIL;982 return SCAN_SUCCEED;983}984 985/*986 * Bring missing pages in from swap, to complete THP collapse.987 * Only done if hpage_collapse_scan_pmd believes it is worthwhile.988 *989 * Called and returns without pte mapped or spinlocks held.990 * Returns result: if not SCAN_SUCCEED, mmap_lock has been released.991 */992static int __collapse_huge_page_swapin(struct mm_struct *mm,993 struct vm_area_struct *vma,994 unsigned long haddr, pmd_t *pmd,995 int referenced)996{997 int swapped_in = 0;998 vm_fault_t ret = 0;999 unsigned long address, end = haddr + (HPAGE_PMD_NR * PAGE_SIZE);1000 int result;1001 pte_t *pte = NULL;1002 spinlock_t *ptl;1003 1004 for (address = haddr; address < end; address += PAGE_SIZE) {1005 struct vm_fault vmf = {1006 .vma = vma,1007 .address = address,1008 .pgoff = linear_page_index(vma, address),1009 .flags = FAULT_FLAG_ALLOW_RETRY,1010 .pmd = pmd,1011 };1012 1013 if (!pte++) {1014 pte = pte_offset_map_nolock(mm, pmd, address, &ptl);1015 if (!pte) {1016 mmap_read_unlock(mm);1017 result = SCAN_PMD_NULL;1018 goto out;1019 }1020 }1021 1022 vmf.orig_pte = ptep_get_lockless(pte);1023 if (!is_swap_pte(vmf.orig_pte))1024 continue;1025 1026 vmf.pte = pte;1027 vmf.ptl = ptl;1028 ret = do_swap_page(&vmf);1029 /* Which unmaps pte (after perhaps re-checking the entry) */1030 pte = NULL;1031 1032 /*1033 * do_swap_page returns VM_FAULT_RETRY with released mmap_lock.1034 * Note we treat VM_FAULT_RETRY as VM_FAULT_ERROR here because1035 * we do not retry here and swap entry will remain in pagetable1036 * resulting in later failure.1037 */1038 if (ret & VM_FAULT_RETRY) {1039 /* Likely, but not guaranteed, that page lock failed */1040 result = SCAN_PAGE_LOCK;1041 goto out;1042 }1043 if (ret & VM_FAULT_ERROR) {1044 mmap_read_unlock(mm);1045 result = SCAN_FAIL;1046 goto out;1047 }1048 swapped_in++;1049 }1050 1051 if (pte)1052 pte_unmap(pte);1053 1054 /* Drain LRU cache to remove extra pin on the swapped in pages */1055 if (swapped_in)1056 lru_add_drain();1057 1058 result = SCAN_SUCCEED;1059out:1060 trace_mm_collapse_huge_page_swapin(mm, swapped_in, referenced, result);1061 return result;1062}1063 1064static int alloc_charge_folio(struct folio **foliop, struct mm_struct *mm,1065 struct collapse_control *cc)1066{1067 gfp_t gfp = (cc->is_khugepaged ? alloc_hugepage_khugepaged_gfpmask() :1068 GFP_TRANSHUGE);1069 int node = hpage_collapse_find_target_node(cc);1070 struct folio *folio;1071 1072 folio = __folio_alloc(gfp, HPAGE_PMD_ORDER, node, &cc->alloc_nmask);1073 if (!folio) {1074 *foliop = NULL;1075 count_vm_event(THP_COLLAPSE_ALLOC_FAILED);1076 return SCAN_ALLOC_HUGE_PAGE_FAIL;1077 }1078 1079 count_vm_event(THP_COLLAPSE_ALLOC);1080 if (unlikely(mem_cgroup_charge(folio, mm, gfp))) {1081 folio_put(folio);1082 *foliop = NULL;1083 return SCAN_CGROUP_CHARGE_FAIL;1084 }1085 1086 count_memcg_folio_events(folio, THP_COLLAPSE_ALLOC, 1);1087 1088 *foliop = folio;1089 return SCAN_SUCCEED;1090}1091 1092static int collapse_huge_page(struct mm_struct *mm, unsigned long address,1093 int referenced, int unmapped,1094 struct collapse_control *cc)1095{1096 LIST_HEAD(compound_pagelist);1097 pmd_t *pmd, _pmd;1098 pte_t *pte;1099 pgtable_t pgtable;1100 struct folio *folio;1101 spinlock_t *pmd_ptl, *pte_ptl;1102 int result = SCAN_FAIL;1103 struct vm_area_struct *vma;1104 struct mmu_notifier_range range;1105 1106 VM_BUG_ON(address & ~HPAGE_PMD_MASK);1107 1108 /*1109 * Before allocating the hugepage, release the mmap_lock read lock.1110 * The allocation can take potentially a long time if it involves1111 * sync compaction, and we do not need to hold the mmap_lock during1112 * that. We will recheck the vma after taking it again in write mode.1113 */1114 mmap_read_unlock(mm);1115 1116 result = alloc_charge_folio(&folio, mm, cc);1117 if (result != SCAN_SUCCEED)1118 goto out_nolock;1119 1120 mmap_read_lock(mm);1121 result = hugepage_vma_revalidate(mm, address, true, &vma, cc);1122 if (result != SCAN_SUCCEED) {1123 mmap_read_unlock(mm);1124 goto out_nolock;1125 }1126 1127 result = find_pmd_or_thp_or_none(mm, address, &pmd);1128 if (result != SCAN_SUCCEED) {1129 mmap_read_unlock(mm);1130 goto out_nolock;1131 }1132 1133 if (unmapped) {1134 /*1135 * __collapse_huge_page_swapin will return with mmap_lock1136 * released when it fails. So we jump out_nolock directly in1137 * that case. Continuing to collapse causes inconsistency.1138 */1139 result = __collapse_huge_page_swapin(mm, vma, address, pmd,1140 referenced);1141 if (result != SCAN_SUCCEED)1142 goto out_nolock;1143 }1144 1145 mmap_read_unlock(mm);1146 /*1147 * Prevent all access to pagetables with the exception of1148 * gup_fast later handled by the ptep_clear_flush and the VM1149 * handled by the anon_vma lock + PG_lock.1150 *1151 * UFFDIO_MOVE is prevented to race as well thanks to the1152 * mmap_lock.1153 */1154 mmap_write_lock(mm);1155 result = hugepage_vma_revalidate(mm, address, true, &vma, cc);1156 if (result != SCAN_SUCCEED)1157 goto out_up_write;1158 /* check if the pmd is still valid */1159 result = check_pmd_still_valid(mm, address, pmd);1160 if (result != SCAN_SUCCEED)1161 goto out_up_write;1162 1163 vma_start_write(vma);1164 anon_vma_lock_write(vma->anon_vma);1165 1166 mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, mm, address,1167 address + HPAGE_PMD_SIZE);1168 mmu_notifier_invalidate_range_start(&range);1169 1170 pmd_ptl = pmd_lock(mm, pmd); /* probably unnecessary */1171 /*1172 * This removes any huge TLB entry from the CPU so we won't allow1173 * huge and small TLB entries for the same virtual address to1174 * avoid the risk of CPU bugs in that area.1175 *1176 * Parallel GUP-fast is fine since GUP-fast will back off when1177 * it detects PMD is changed.1178 */1179 _pmd = pmdp_collapse_flush(vma, address, pmd);1180 spin_unlock(pmd_ptl);1181 mmu_notifier_invalidate_range_end(&range);1182 tlb_remove_table_sync_one();1183 1184 pte = pte_offset_map_lock(mm, &_pmd, address, &pte_ptl);1185 if (pte) {1186 result = __collapse_huge_page_isolate(vma, address, pte, cc,1187 &compound_pagelist);1188 spin_unlock(pte_ptl);1189 } else {1190 result = SCAN_PMD_NULL;1191 }1192 1193 if (unlikely(result != SCAN_SUCCEED)) {1194 if (pte)1195 pte_unmap(pte);1196 spin_lock(pmd_ptl);1197 BUG_ON(!pmd_none(*pmd));1198 /*1199 * We can only use set_pmd_at when establishing1200 * hugepmds and never for establishing regular pmds that1201 * points to regular pagetables. Use pmd_populate for that1202 */1203 pmd_populate(mm, pmd, pmd_pgtable(_pmd));1204 spin_unlock(pmd_ptl);1205 anon_vma_unlock_write(vma->anon_vma);1206 goto out_up_write;1207 }1208 1209 /*1210 * All pages are isolated and locked so anon_vma rmap1211 * can't run anymore.1212 */1213 anon_vma_unlock_write(vma->anon_vma);1214 1215 result = __collapse_huge_page_copy(pte, folio, pmd, _pmd,1216 vma, address, pte_ptl,1217 &compound_pagelist);1218 pte_unmap(pte);1219 if (unlikely(result != SCAN_SUCCEED))1220 goto out_up_write;1221 1222 /*1223 * The smp_wmb() inside __folio_mark_uptodate() ensures the1224 * copy_huge_page writes become visible before the set_pmd_at()1225 * write.1226 */1227 __folio_mark_uptodate(folio);1228 pgtable = pmd_pgtable(_pmd);1229 1230 _pmd = mk_huge_pmd(&folio->page, vma->vm_page_prot);1231 _pmd = maybe_pmd_mkwrite(pmd_mkdirty(_pmd), vma);1232 1233 spin_lock(pmd_ptl);1234 BUG_ON(!pmd_none(*pmd));1235 folio_add_new_anon_rmap(folio, vma, address, RMAP_EXCLUSIVE);1236 folio_add_lru_vma(folio, vma);1237 pgtable_trans_huge_deposit(mm, pmd, pgtable);1238 set_pmd_at(mm, address, pmd, _pmd);1239 update_mmu_cache_pmd(vma, address, pmd);1240 deferred_split_folio(folio, false);1241 spin_unlock(pmd_ptl);1242 1243 folio = NULL;1244 1245 result = SCAN_SUCCEED;1246out_up_write:1247 mmap_write_unlock(mm);1248out_nolock:1249 if (folio)1250 folio_put(folio);1251 trace_mm_collapse_huge_page(mm, result == SCAN_SUCCEED, result);1252 return result;1253}1254 1255static int hpage_collapse_scan_pmd(struct mm_struct *mm,1256 struct vm_area_struct *vma,1257 unsigned long address, bool *mmap_locked,1258 struct collapse_control *cc)1259{1260 pmd_t *pmd;1261 pte_t *pte, *_pte;1262 int result = SCAN_FAIL, referenced = 0;1263 int none_or_zero = 0, shared = 0;1264 struct page *page = NULL;1265 struct folio *folio = NULL;1266 unsigned long _address;1267 spinlock_t *ptl;1268 int node = NUMA_NO_NODE, unmapped = 0;1269 bool writable = false;1270 1271 VM_BUG_ON(address & ~HPAGE_PMD_MASK);1272 1273 result = find_pmd_or_thp_or_none(mm, address, &pmd);1274 if (result != SCAN_SUCCEED)1275 goto out;1276 1277 memset(cc->node_load, 0, sizeof(cc->node_load));1278 nodes_clear(cc->alloc_nmask);1279 pte = pte_offset_map_lock(mm, pmd, address, &ptl);1280 if (!pte) {1281 result = SCAN_PMD_NULL;1282 goto out;1283 }1284 1285 for (_address = address, _pte = pte; _pte < pte + HPAGE_PMD_NR;1286 _pte++, _address += PAGE_SIZE) {1287 pte_t pteval = ptep_get(_pte);1288 if (is_swap_pte(pteval)) {1289 ++unmapped;1290 if (!cc->is_khugepaged ||1291 unmapped <= khugepaged_max_ptes_swap) {1292 /*1293 * Always be strict with uffd-wp1294 * enabled swap entries. Please see1295 * comment below for pte_uffd_wp().1296 */1297 if (pte_swp_uffd_wp_any(pteval)) {1298 result = SCAN_PTE_UFFD_WP;1299 goto out_unmap;1300 }1301 continue;1302 } else {1303 result = SCAN_EXCEED_SWAP_PTE;1304 count_vm_event(THP_SCAN_EXCEED_SWAP_PTE);1305 goto out_unmap;1306 }1307 }1308 if (pte_none(pteval) || is_zero_pfn(pte_pfn(pteval))) {1309 ++none_or_zero;1310 if (!userfaultfd_armed(vma) &&1311 (!cc->is_khugepaged ||1312 none_or_zero <= khugepaged_max_ptes_none)) {1313 continue;1314 } else {1315 result = SCAN_EXCEED_NONE_PTE;1316 count_vm_event(THP_SCAN_EXCEED_NONE_PTE);1317 goto out_unmap;1318 }1319 }1320 if (pte_uffd_wp(pteval)) {1321 /*1322 * Don't collapse the page if any of the small1323 * PTEs are armed with uffd write protection.1324 * Here we can also mark the new huge pmd as1325 * write protected if any of the small ones is1326 * marked but that could bring unknown1327 * userfault messages that falls outside of1328 * the registered range. So, just be simple.1329 */1330 result = SCAN_PTE_UFFD_WP;1331 goto out_unmap;1332 }1333 if (pte_write(pteval))1334 writable = true;1335 1336 page = vm_normal_page(vma, _address, pteval);1337 if (unlikely(!page) || unlikely(is_zone_device_page(page))) {1338 result = SCAN_PAGE_NULL;1339 goto out_unmap;1340 }1341 folio = page_folio(page);1342 1343 if (!folio_test_anon(folio)) {1344 result = SCAN_PAGE_ANON;1345 goto out_unmap;1346 }1347 1348 /*1349 * We treat a single page as shared if any part of the THP1350 * is shared. "False negatives" from1351 * folio_likely_mapped_shared() are not expected to matter1352 * much in practice.1353 */1354 if (folio_likely_mapped_shared(folio)) {1355 ++shared;1356 if (cc->is_khugepaged &&1357 shared > khugepaged_max_ptes_shared) {1358 result = SCAN_EXCEED_SHARED_PTE;1359 count_vm_event(THP_SCAN_EXCEED_SHARED_PTE);1360 goto out_unmap;1361 }1362 }1363 1364 /*1365 * Record which node the original page is from and save this1366 * information to cc->node_load[].1367 * Khugepaged will allocate hugepage from the node has the max1368 * hit record.1369 */1370 node = folio_nid(folio);1371 if (hpage_collapse_scan_abort(node, cc)) {1372 result = SCAN_SCAN_ABORT;1373 goto out_unmap;1374 }1375 cc->node_load[node]++;1376 if (!folio_test_lru(folio)) {1377 result = SCAN_PAGE_LRU;1378 goto out_unmap;1379 }1380 if (folio_test_locked(folio)) {1381 result = SCAN_PAGE_LOCK;1382 goto out_unmap;1383 }1384 1385 /*1386 * Check if the page has any GUP (or other external) pins.1387 *1388 * Here the check may be racy:1389 * it may see folio_mapcount() > folio_ref_count().1390 * But such case is ephemeral we could always retry collapse1391 * later. However it may report false positive if the page1392 * has excessive GUP pins (i.e. 512). Anyway the same check1393 * will be done again later the risk seems low.1394 */1395 if (!is_refcount_suitable(folio)) {1396 result = SCAN_PAGE_COUNT;1397 goto out_unmap;1398 }1399 1400 /*1401 * If collapse was initiated by khugepaged, check that there is1402 * enough young pte to justify collapsing the page1403 */1404 if (cc->is_khugepaged &&1405 (pte_young(pteval) || folio_test_young(folio) ||1406 folio_test_referenced(folio) || mmu_notifier_test_young(vma->vm_mm,1407 address)))1408 referenced++;1409 }1410 if (!writable) {1411 result = SCAN_PAGE_RO;1412 } else if (cc->is_khugepaged &&1413 (!referenced ||1414 (unmapped && referenced < HPAGE_PMD_NR / 2))) {1415 result = SCAN_LACK_REFERENCED_PAGE;1416 } else {1417 result = SCAN_SUCCEED;1418 }1419out_unmap:1420 pte_unmap_unlock(pte, ptl);1421 if (result == SCAN_SUCCEED) {1422 result = collapse_huge_page(mm, address, referenced,1423 unmapped, cc);1424 /* collapse_huge_page will return with the mmap_lock released */1425 *mmap_locked = false;1426 }1427out:1428 trace_mm_khugepaged_scan_pmd(mm, &folio->page, writable, referenced,1429 none_or_zero, result, unmapped);1430 return result;1431}1432 1433static void collect_mm_slot(struct khugepaged_mm_slot *mm_slot)1434{1435 struct mm_slot *slot = &mm_slot->slot;1436 struct mm_struct *mm = slot->mm;1437 1438 lockdep_assert_held(&khugepaged_mm_lock);1439 1440 if (hpage_collapse_test_exit(mm)) {1441 /* free mm_slot */1442 hash_del(&slot->hash);1443 list_del(&slot->mm_node);1444 1445 /*1446 * Not strictly needed because the mm exited already.1447 *1448 * clear_bit(MMF_VM_HUGEPAGE, &mm->flags);1449 */1450 1451 /* khugepaged_mm_lock actually not necessary for the below */1452 mm_slot_free(mm_slot_cache, mm_slot);1453 mmdrop(mm);1454 }1455}1456 1457#ifdef CONFIG_SHMEM1458/* hpage must be locked, and mmap_lock must be held */1459static int set_huge_pmd(struct vm_area_struct *vma, unsigned long addr,1460 pmd_t *pmdp, struct page *hpage)1461{1462 struct vm_fault vmf = {1463 .vma = vma,1464 .address = addr,1465 .flags = 0,1466 .pmd = pmdp,1467 };1468 1469 VM_BUG_ON(!PageTransHuge(hpage));1470 mmap_assert_locked(vma->vm_mm);1471 1472 if (do_set_pmd(&vmf, hpage))1473 return SCAN_FAIL;1474 1475 get_page(hpage);1476 return SCAN_SUCCEED;1477}1478 1479/**1480 * collapse_pte_mapped_thp - Try to collapse a pte-mapped THP for mm at1481 * address haddr.1482 *1483 * @mm: process address space where collapse happens1484 * @addr: THP collapse address1485 * @install_pmd: If a huge PMD should be installed1486 *1487 * This function checks whether all the PTEs in the PMD are pointing to the1488 * right THP. If so, retract the page table so the THP can refault in with1489 * as pmd-mapped. Possibly install a huge PMD mapping the THP.1490 */1491int collapse_pte_mapped_thp(struct mm_struct *mm, unsigned long addr,1492 bool install_pmd)1493{1494 struct mmu_notifier_range range;1495 bool notified = false;1496 unsigned long haddr = addr & HPAGE_PMD_MASK;1497 struct vm_area_struct *vma = vma_lookup(mm, haddr);1498 struct folio *folio;1499 pte_t *start_pte, *pte;1500 pmd_t *pmd, pgt_pmd;1501 spinlock_t *pml = NULL, *ptl;1502 int nr_ptes = 0, result = SCAN_FAIL;1503 int i;1504 1505 mmap_assert_locked(mm);1506 1507 /* First check VMA found, in case page tables are being torn down */1508 if (!vma || !vma->vm_file ||1509 !range_in_vma(vma, haddr, haddr + HPAGE_PMD_SIZE))1510 return SCAN_VMA_CHECK;1511 1512 /* Fast check before locking page if already PMD-mapped */1513 result = find_pmd_or_thp_or_none(mm, haddr, &pmd);1514 if (result == SCAN_PMD_MAPPED)1515 return result;1516 1517 /*1518 * If we are here, we've succeeded in replacing all the native pages1519 * in the page cache with a single hugepage. If a mm were to fault-in1520 * this memory (mapped by a suitably aligned VMA), we'd get the hugepage1521 * and map it by a PMD, regardless of sysfs THP settings. As such, let's1522 * analogously elide sysfs THP settings here.1523 */1524 if (!thp_vma_allowable_order(vma, vma->vm_flags, 0, PMD_ORDER))1525 return SCAN_VMA_CHECK;1526 1527 /* Keep pmd pgtable for uffd-wp; see comment in retract_page_tables() */1528 if (userfaultfd_wp(vma))1529 return SCAN_PTE_UFFD_WP;1530 1531 folio = filemap_lock_folio(vma->vm_file->f_mapping,1532 linear_page_index(vma, haddr));1533 if (IS_ERR(folio))1534 return SCAN_PAGE_NULL;1535 1536 if (folio_order(folio) != HPAGE_PMD_ORDER) {1537 result = SCAN_PAGE_COMPOUND;1538 goto drop_folio;1539 }1540 1541 result = find_pmd_or_thp_or_none(mm, haddr, &pmd);1542 switch (result) {1543 case SCAN_SUCCEED:1544 break;1545 case SCAN_PMD_NONE:1546 /*1547 * All pte entries have been removed and pmd cleared.1548 * Skip all the pte checks and just update the pmd mapping.1549 */1550 goto maybe_install_pmd;1551 default:1552 goto drop_folio;1553 }1554 1555 result = SCAN_FAIL;1556 start_pte = pte_offset_map_lock(mm, pmd, haddr, &ptl);1557 if (!start_pte) /* mmap_lock + page lock should prevent this */1558 goto drop_folio;1559 1560 /* step 1: check all mapped PTEs are to the right huge page */1561 for (i = 0, addr = haddr, pte = start_pte;1562 i < HPAGE_PMD_NR; i++, addr += PAGE_SIZE, pte++) {1563 struct page *page;1564 pte_t ptent = ptep_get(pte);1565 1566 /* empty pte, skip */1567 if (pte_none(ptent))1568 continue;1569 1570 /* page swapped out, abort */1571 if (!pte_present(ptent)) {1572 result = SCAN_PTE_NON_PRESENT;1573 goto abort;1574 }1575 1576 page = vm_normal_page(vma, addr, ptent);1577 if (WARN_ON_ONCE(page && is_zone_device_page(page)))1578 page = NULL;1579 /*1580 * Note that uprobe, debugger, or MAP_PRIVATE may change the1581 * page table, but the new page will not be a subpage of hpage.1582 */1583 if (folio_page(folio, i) != page)1584 goto abort;1585 }1586 1587 pte_unmap_unlock(start_pte, ptl);1588 mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, mm,1589 haddr, haddr + HPAGE_PMD_SIZE);1590 mmu_notifier_invalidate_range_start(&range);1591 notified = true;1592 1593 /*1594 * pmd_lock covers a wider range than ptl, and (if split from mm's1595 * page_table_lock) ptl nests inside pml. The less time we hold pml,1596 * the better; but userfaultfd's mfill_atomic_pte() on a private VMA1597 * inserts a valid as-if-COWed PTE without even looking up page cache.1598 * So page lock of folio does not protect from it, so we must not drop1599 * ptl before pgt_pmd is removed, so uffd private needs pml taken now.1600 */1601 if (userfaultfd_armed(vma) && !(vma->vm_flags & VM_SHARED))1602 pml = pmd_lock(mm, pmd);1603 1604 start_pte = pte_offset_map_nolock(mm, pmd, haddr, &ptl);1605 if (!start_pte) /* mmap_lock + page lock should prevent this */1606 goto abort;1607 if (!pml)1608 spin_lock(ptl);1609 else if (ptl != pml)1610 spin_lock_nested(ptl, SINGLE_DEPTH_NESTING);1611 1612 /* step 2: clear page table and adjust rmap */1613 for (i = 0, addr = haddr, pte = start_pte;1614 i < HPAGE_PMD_NR; i++, addr += PAGE_SIZE, pte++) {1615 struct page *page;1616 pte_t ptent = ptep_get(pte);1617 1618 if (pte_none(ptent))1619 continue;1620 /*1621 * We dropped ptl after the first scan, to do the mmu_notifier:1622 * page lock stops more PTEs of the folio being faulted in, but1623 * does not stop write faults COWing anon copies from existing1624 * PTEs; and does not stop those being swapped out or migrated.1625 */1626 if (!pte_present(ptent)) {1627 result = SCAN_PTE_NON_PRESENT;1628 goto abort;1629 }1630 page = vm_normal_page(vma, addr, ptent);1631 if (folio_page(folio, i) != page)1632 goto abort;1633 1634 /*1635 * Must clear entry, or a racing truncate may re-remove it.1636 * TLB flush can be left until pmdp_collapse_flush() does it.1637 * PTE dirty? Shmem page is already dirty; file is read-only.1638 */1639 ptep_clear(mm, addr, pte);1640 folio_remove_rmap_pte(folio, page, vma);1641 nr_ptes++;1642 }1643 1644 pte_unmap(start_pte);1645 if (!pml)1646 spin_unlock(ptl);1647 1648 /* step 3: set proper refcount and mm_counters. */1649 if (nr_ptes) {1650 folio_ref_sub(folio, nr_ptes);1651 add_mm_counter(mm, mm_counter_file(folio), -nr_ptes);1652 }1653 1654 /* step 4: remove empty page table */1655 if (!pml) {1656 pml = pmd_lock(mm, pmd);1657 if (ptl != pml)1658 spin_lock_nested(ptl, SINGLE_DEPTH_NESTING);1659 }1660 pgt_pmd = pmdp_collapse_flush(vma, haddr, pmd);1661 pmdp_get_lockless_sync();1662 if (ptl != pml)1663 spin_unlock(ptl);1664 spin_unlock(pml);1665 1666 mmu_notifier_invalidate_range_end(&range);1667 1668 mm_dec_nr_ptes(mm);1669 page_table_check_pte_clear_range(mm, haddr, pgt_pmd);1670 pte_free_defer(mm, pmd_pgtable(pgt_pmd));1671 1672maybe_install_pmd:1673 /* step 5: install pmd entry */1674 result = install_pmd1675 ? set_huge_pmd(vma, haddr, pmd, &folio->page)1676 : SCAN_SUCCEED;1677 goto drop_folio;1678abort:1679 if (nr_ptes) {1680 flush_tlb_mm(mm);1681 folio_ref_sub(folio, nr_ptes);1682 add_mm_counter(mm, mm_counter_file(folio), -nr_ptes);1683 }1684 if (start_pte)1685 pte_unmap_unlock(start_pte, ptl);1686 if (pml && pml != ptl)1687 spin_unlock(pml);1688 if (notified)1689 mmu_notifier_invalidate_range_end(&range);1690drop_folio:1691 folio_unlock(folio);1692 folio_put(folio);1693 return result;1694}1695 1696static void retract_page_tables(struct address_space *mapping, pgoff_t pgoff)1697{1698 struct vm_area_struct *vma;1699 1700 i_mmap_lock_read(mapping);1701 vma_interval_tree_foreach(vma, &mapping->i_mmap, pgoff, pgoff) {1702 struct mmu_notifier_range range;1703 struct mm_struct *mm;1704 unsigned long addr;1705 pmd_t *pmd, pgt_pmd;1706 spinlock_t *pml;1707 spinlock_t *ptl;1708 bool skipped_uffd = false;1709 1710 /*1711 * Check vma->anon_vma to exclude MAP_PRIVATE mappings that1712 * got written to. These VMAs are likely not worth removing1713 * page tables from, as PMD-mapping is likely to be split later.1714 */1715 if (READ_ONCE(vma->anon_vma))1716 continue;1717 1718 addr = vma->vm_start + ((pgoff - vma->vm_pgoff) << PAGE_SHIFT);1719 if (addr & ~HPAGE_PMD_MASK ||1720 vma->vm_end < addr + HPAGE_PMD_SIZE)1721 continue;1722 1723 mm = vma->vm_mm;1724 if (find_pmd_or_thp_or_none(mm, addr, &pmd) != SCAN_SUCCEED)1725 continue;1726 1727 if (hpage_collapse_test_exit(mm))1728 continue;1729 /*1730 * When a vma is registered with uffd-wp, we cannot recycle1731 * the page table because there may be pte markers installed.1732 * Other vmas can still have the same file mapped hugely, but1733 * skip this one: it will always be mapped in small page size1734 * for uffd-wp registered ranges.1735 */1736 if (userfaultfd_wp(vma))1737 continue;1738 1739 /* PTEs were notified when unmapped; but now for the PMD? */1740 mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, mm,1741 addr, addr + HPAGE_PMD_SIZE);1742 mmu_notifier_invalidate_range_start(&range);1743 1744 pml = pmd_lock(mm, pmd);1745 ptl = pte_lockptr(mm, pmd);1746 if (ptl != pml)1747 spin_lock_nested(ptl, SINGLE_DEPTH_NESTING);1748 1749 /*1750 * Huge page lock is still held, so normally the page table1751 * must remain empty; and we have already skipped anon_vma1752 * and userfaultfd_wp() vmas. But since the mmap_lock is not1753 * held, it is still possible for a racing userfaultfd_ioctl()1754 * to have inserted ptes or markers. Now that we hold ptlock,1755 * repeating the anon_vma check protects from one category,1756 * and repeating the userfaultfd_wp() check from another.1757 */1758 if (unlikely(vma->anon_vma || userfaultfd_wp(vma))) {1759 skipped_uffd = true;1760 } else {1761 pgt_pmd = pmdp_collapse_flush(vma, addr, pmd);1762 pmdp_get_lockless_sync();1763 }1764 1765 if (ptl != pml)1766 spin_unlock(ptl);1767 spin_unlock(pml);1768 1769 mmu_notifier_invalidate_range_end(&range);1770 1771 if (!skipped_uffd) {1772 mm_dec_nr_ptes(mm);1773 page_table_check_pte_clear_range(mm, addr, pgt_pmd);1774 pte_free_defer(mm, pmd_pgtable(pgt_pmd));1775 }1776 }1777 i_mmap_unlock_read(mapping);1778}1779 1780/**1781 * collapse_file - collapse filemap/tmpfs/shmem pages into huge one.1782 *1783 * @mm: process address space where collapse happens1784 * @addr: virtual collapse start address1785 * @file: file that collapse on1786 * @start: collapse start address1787 * @cc: collapse context and scratchpad1788 *1789 * Basic scheme is simple, details are more complex:1790 * - allocate and lock a new huge page;1791 * - scan page cache, locking old pages1792 * + swap/gup in pages if necessary;1793 * - copy data to new page1794 * - handle shmem holes1795 * + re-validate that holes weren't filled by someone else1796 * + check for userfaultfd1797 * - finalize updates to the page cache;1798 * - if replacing succeeds:1799 * + unlock huge page;1800 * + free old pages;1801 * - if replacing failed;1802 * + unlock old pages1803 * + unlock and free huge page;1804 */1805static int collapse_file(struct mm_struct *mm, unsigned long addr,1806 struct file *file, pgoff_t start,1807 struct collapse_control *cc)1808{1809 struct address_space *mapping = file->f_mapping;1810 struct page *dst;1811 struct folio *folio, *tmp, *new_folio;1812 pgoff_t index = 0, end = start + HPAGE_PMD_NR;1813 LIST_HEAD(pagelist);1814 XA_STATE_ORDER(xas, &mapping->i_pages, start, HPAGE_PMD_ORDER);1815 int nr_none = 0, result = SCAN_SUCCEED;1816 bool is_shmem = shmem_file(file);1817 1818 VM_BUG_ON(!IS_ENABLED(CONFIG_READ_ONLY_THP_FOR_FS) && !is_shmem);1819 VM_BUG_ON(start & (HPAGE_PMD_NR - 1));1820 1821 result = alloc_charge_folio(&new_folio, mm, cc);1822 if (result != SCAN_SUCCEED)1823 goto out;1824 1825 __folio_set_locked(new_folio);1826 if (is_shmem)1827 __folio_set_swapbacked(new_folio);1828 new_folio->index = start;1829 new_folio->mapping = mapping;1830 1831 /*1832 * Ensure we have slots for all the pages in the range. This is1833 * almost certainly a no-op because most of the pages must be present1834 */1835 do {1836 xas_lock_irq(&xas);1837 xas_create_range(&xas);1838 if (!xas_error(&xas))1839 break;1840 xas_unlock_irq(&xas);1841 if (!xas_nomem(&xas, GFP_KERNEL)) {1842 result = SCAN_FAIL;1843 goto rollback;1844 }1845 } while (1);1846 1847 for (index = start; index < end;) {1848 xas_set(&xas, index);1849 folio = xas_load(&xas);1850 1851 VM_BUG_ON(index != xas.xa_index);1852 if (is_shmem) {1853 if (!folio) {1854 /*1855 * Stop if extent has been truncated or1856 * hole-punched, and is now completely1857 * empty.1858 */1859 if (index == start) {1860 if (!xas_next_entry(&xas, end - 1)) {1861 result = SCAN_TRUNCATED;1862 goto xa_locked;1863 }1864 }1865 nr_none++;1866 index++;1867 continue;1868 }1869 1870 if (xa_is_value(folio) || !folio_test_uptodate(folio)) {1871 xas_unlock_irq(&xas);1872 /* swap in or instantiate fallocated page */1873 if (shmem_get_folio(mapping->host, index, 0,1874 &folio, SGP_NOALLOC)) {1875 result = SCAN_FAIL;1876 goto xa_unlocked;1877 }1878 /* drain lru cache to help folio_isolate_lru() */1879 lru_add_drain();1880 } else if (folio_trylock(folio)) {1881 folio_get(folio);1882 xas_unlock_irq(&xas);1883 } else {1884 result = SCAN_PAGE_LOCK;1885 goto xa_locked;1886 }1887 } else { /* !is_shmem */1888 if (!folio || xa_is_value(folio)) {1889 xas_unlock_irq(&xas);1890 page_cache_sync_readahead(mapping, &file->f_ra,1891 file, index,1892 end - index);1893 /* drain lru cache to help folio_isolate_lru() */1894 lru_add_drain();1895 folio = filemap_lock_folio(mapping, index);1896 if (IS_ERR(folio)) {1897 result = SCAN_FAIL;1898 goto xa_unlocked;1899 }1900 } else if (folio_test_dirty(folio)) {1901 /*1902 * khugepaged only works on read-only fd,1903 * so this page is dirty because it hasn't1904 * been flushed since first write. There1905 * won't be new dirty pages.1906 *1907 * Trigger async flush here and hope the1908 * writeback is done when khugepaged1909 * revisits this page.1910 *1911 * This is a one-off situation. We are not1912 * forcing writeback in loop.1913 */1914 xas_unlock_irq(&xas);1915 filemap_flush(mapping);1916 result = SCAN_FAIL;1917 goto xa_unlocked;1918 } else if (folio_test_writeback(folio)) {1919 xas_unlock_irq(&xas);1920 result = SCAN_FAIL;1921 goto xa_unlocked;1922 } else if (folio_trylock(folio)) {1923 folio_get(folio);1924 xas_unlock_irq(&xas);1925 } else {1926 result = SCAN_PAGE_LOCK;1927 goto xa_locked;1928 }1929 }1930 1931 /*1932 * The folio must be locked, so we can drop the i_pages lock1933 * without racing with truncate.1934 */1935 VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio);1936 1937 /* make sure the folio is up to date */1938 if (unlikely(!folio_test_uptodate(folio))) {1939 result = SCAN_FAIL;1940 goto out_unlock;1941 }1942 1943 /*1944 * If file was truncated then extended, or hole-punched, before1945 * we locked the first folio, then a THP might be there already.1946 * This will be discovered on the first iteration.1947 */1948 if (folio_order(folio) == HPAGE_PMD_ORDER &&1949 folio->index == start) {1950 /* Maybe PMD-mapped */1951 result = SCAN_PTE_MAPPED_HUGEPAGE;1952 goto out_unlock;1953 }1954 1955 if (folio_mapping(folio) != mapping) {1956 result = SCAN_TRUNCATED;1957 goto out_unlock;1958 }1959 1960 if (!is_shmem && (folio_test_dirty(folio) ||1961 folio_test_writeback(folio))) {1962 /*1963 * khugepaged only works on read-only fd, so this1964 * folio is dirty because it hasn't been flushed1965 * since first write.1966 */1967 result = SCAN_FAIL;1968 goto out_unlock;1969 }1970 1971 if (!folio_isolate_lru(folio)) {1972 result = SCAN_DEL_PAGE_LRU;1973 goto out_unlock;1974 }1975 1976 if (!filemap_release_folio(folio, GFP_KERNEL)) {1977 result = SCAN_PAGE_HAS_PRIVATE;1978 folio_putback_lru(folio);1979 goto out_unlock;1980 }1981 1982 if (folio_mapped(folio))1983 try_to_unmap(folio,1984 TTU_IGNORE_MLOCK | TTU_BATCH_FLUSH);1985 1986 xas_lock_irq(&xas);1987 1988 VM_BUG_ON_FOLIO(folio != xa_load(xas.xa, index), folio);1989 1990 /*1991 * We control 2 + nr_pages references to the folio:1992 * - we hold a pin on it;1993 * - nr_pages reference from page cache;1994 * - one from lru_isolate_folio;1995 * If those are the only references, then any new usage1996 * of the folio will have to fetch it from the page1997 * cache. That requires locking the folio to handle1998 * truncate, so any new usage will be blocked until we1999 * unlock folio after collapse/during rollback.2000 */2001 if (folio_ref_count(folio) != 2 + folio_nr_pages(folio)) {2002 result = SCAN_PAGE_COUNT;2003 xas_unlock_irq(&xas);2004 folio_putback_lru(folio);2005 goto out_unlock;2006 }2007 2008 /*2009 * Accumulate the folios that are being collapsed.2010 */2011 list_add_tail(&folio->lru, &pagelist);2012 index += folio_nr_pages(folio);2013 continue;2014out_unlock:2015 folio_unlock(folio);2016 folio_put(folio);2017 goto xa_unlocked;2018 }2019 2020 if (!is_shmem) {2021 filemap_nr_thps_inc(mapping);2022 /*2023 * Paired with the fence in do_dentry_open() -> get_write_access()2024 * to ensure i_writecount is up to date and the update to nr_thps2025 * is visible. Ensures the page cache will be truncated if the2026 * file is opened writable.2027 */2028 smp_mb();2029 if (inode_is_open_for_write(mapping->host)) {2030 result = SCAN_FAIL;2031 filemap_nr_thps_dec(mapping);2032 }2033 }2034 2035xa_locked:2036 xas_unlock_irq(&xas);2037xa_unlocked:2038 2039 /*2040 * If collapse is successful, flush must be done now before copying.2041 * If collapse is unsuccessful, does flush actually need to be done?2042 * Do it anyway, to clear the state.2043 */2044 try_to_unmap_flush();2045 2046 if (result == SCAN_SUCCEED && nr_none &&2047 !shmem_charge(mapping->host, nr_none))2048 result = SCAN_FAIL;2049 if (result != SCAN_SUCCEED) {2050 nr_none = 0;2051 goto rollback;2052 }2053 2054 /*2055 * The old folios are locked, so they won't change anymore.2056 */2057 index = start;2058 dst = folio_page(new_folio, 0);2059 list_for_each_entry(folio, &pagelist, lru) {2060 int i, nr_pages = folio_nr_pages(folio);2061 2062 while (index < folio->index) {2063 clear_highpage(dst);2064 index++;2065 dst++;2066 }2067 2068 for (i = 0; i < nr_pages; i++) {2069 if (copy_mc_highpage(dst, folio_page(folio, i)) > 0) {2070 result = SCAN_COPY_MC;2071 goto rollback;2072 }2073 index++;2074 dst++;2075 }2076 }2077 while (index < end) {2078 clear_highpage(dst);2079 index++;2080 dst++;2081 }2082 2083 if (nr_none) {2084 struct vm_area_struct *vma;2085 int nr_none_check = 0;2086 2087 i_mmap_lock_read(mapping);2088 xas_lock_irq(&xas);2089 2090 xas_set(&xas, start);2091 for (index = start; index < end; index++) {2092 if (!xas_next(&xas)) {2093 xas_store(&xas, XA_RETRY_ENTRY);2094 if (xas_error(&xas)) {2095 result = SCAN_STORE_FAILED;2096 goto immap_locked;2097 }2098 nr_none_check++;2099 }2100 }2101 2102 if (nr_none != nr_none_check) {2103 result = SCAN_PAGE_FILLED;2104 goto immap_locked;2105 }2106 2107 /*2108 * If userspace observed a missing page in a VMA with2109 * a MODE_MISSING userfaultfd, then it might expect a2110 * UFFD_EVENT_PAGEFAULT for that page. If so, we need to2111 * roll back to avoid suppressing such an event. Since2112 * wp/minor userfaultfds don't give userspace any2113 * guarantees that the kernel doesn't fill a missing2114 * page with a zero page, so they don't matter here.2115 *2116 * Any userfaultfds registered after this point will2117 * not be able to observe any missing pages due to the2118 * previously inserted retry entries.2119 */2120 vma_interval_tree_foreach(vma, &mapping->i_mmap, start, end) {2121 if (userfaultfd_missing(vma)) {2122 result = SCAN_EXCEED_NONE_PTE;2123 goto immap_locked;2124 }2125 }2126 2127immap_locked:2128 i_mmap_unlock_read(mapping);2129 if (result != SCAN_SUCCEED) {2130 xas_set(&xas, start);2131 for (index = start; index < end; index++) {2132 if (xas_next(&xas) == XA_RETRY_ENTRY)2133 xas_store(&xas, NULL);2134 }2135 2136 xas_unlock_irq(&xas);2137 goto rollback;2138 }2139 } else {2140 xas_lock_irq(&xas);2141 }2142 2143 if (is_shmem)2144 __lruvec_stat_mod_folio(new_folio, NR_SHMEM_THPS, HPAGE_PMD_NR);2145 else2146 __lruvec_stat_mod_folio(new_folio, NR_FILE_THPS, HPAGE_PMD_NR);2147 2148 if (nr_none) {2149 __lruvec_stat_mod_folio(new_folio, NR_FILE_PAGES, nr_none);2150 /* nr_none is always 0 for non-shmem. */2151 __lruvec_stat_mod_folio(new_folio, NR_SHMEM, nr_none);2152 }2153 2154 /*2155 * Mark new_folio as uptodate before inserting it into the2156 * page cache so that it isn't mistaken for an fallocated but2157 * unwritten page.2158 */2159 folio_mark_uptodate(new_folio);2160 folio_ref_add(new_folio, HPAGE_PMD_NR - 1);2161 2162 if (is_shmem)2163 folio_mark_dirty(new_folio);2164 folio_add_lru(new_folio);2165 2166 /* Join all the small entries into a single multi-index entry. */2167 xas_set_order(&xas, start, HPAGE_PMD_ORDER);2168 xas_store(&xas, new_folio);2169 WARN_ON_ONCE(xas_error(&xas));2170 xas_unlock_irq(&xas);2171 2172 /*2173 * Remove pte page tables, so we can re-fault the page as huge.2174 * If MADV_COLLAPSE, adjust result to call collapse_pte_mapped_thp().2175 */2176 retract_page_tables(mapping, start);2177 if (cc && !cc->is_khugepaged)2178 result = SCAN_PTE_MAPPED_HUGEPAGE;2179 folio_unlock(new_folio);2180 2181 /*2182 * The collapse has succeeded, so free the old folios.2183 */2184 list_for_each_entry_safe(folio, tmp, &pagelist, lru) {2185 list_del(&folio->lru);2186 folio->mapping = NULL;2187 folio_clear_active(folio);2188 folio_clear_unevictable(folio);2189 folio_unlock(folio);2190 folio_put_refs(folio, 2 + folio_nr_pages(folio));2191 }2192 2193 goto out;2194 2195rollback:2196 /* Something went wrong: roll back page cache changes */2197 if (nr_none) {2198 xas_lock_irq(&xas);2199 mapping->nrpages -= nr_none;2200 xas_unlock_irq(&xas);2201 shmem_uncharge(mapping->host, nr_none);2202 }2203 2204 list_for_each_entry_safe(folio, tmp, &pagelist, lru) {2205 list_del(&folio->lru);2206 folio_unlock(folio);2207 folio_putback_lru(folio);2208 folio_put(folio);2209 }2210 /*2211 * Undo the updates of filemap_nr_thps_inc for non-SHMEM2212 * file only. This undo is not needed unless failure is2213 * due to SCAN_COPY_MC.2214 */2215 if (!is_shmem && result == SCAN_COPY_MC) {2216 filemap_nr_thps_dec(mapping);2217 /*2218 * Paired with the fence in do_dentry_open() -> get_write_access()2219 * to ensure the update to nr_thps is visible.2220 */2221 smp_mb();2222 }2223 2224 new_folio->mapping = NULL;2225 2226 folio_unlock(new_folio);2227 folio_put(new_folio);2228out:2229 VM_BUG_ON(!list_empty(&pagelist));2230 trace_mm_khugepaged_collapse_file(mm, new_folio, index, addr, is_shmem, file, HPAGE_PMD_NR, result);2231 return result;2232}2233 2234static int hpage_collapse_scan_file(struct mm_struct *mm, unsigned long addr,2235 struct file *file, pgoff_t start,2236 struct collapse_control *cc)2237{2238 struct folio *folio = NULL;2239 struct address_space *mapping = file->f_mapping;2240 XA_STATE(xas, &mapping->i_pages, start);2241 int present, swap;2242 int node = NUMA_NO_NODE;2243 int result = SCAN_SUCCEED;2244 2245 present = 0;2246 swap = 0;2247 memset(cc->node_load, 0, sizeof(cc->node_load));2248 nodes_clear(cc->alloc_nmask);2249 rcu_read_lock();2250 xas_for_each(&xas, folio, start + HPAGE_PMD_NR - 1) {2251 if (xas_retry(&xas, folio))2252 continue;2253 2254 if (xa_is_value(folio)) {2255 swap += 1 << xas_get_order(&xas);2256 if (cc->is_khugepaged &&2257 swap > khugepaged_max_ptes_swap) {2258 result = SCAN_EXCEED_SWAP_PTE;2259 count_vm_event(THP_SCAN_EXCEED_SWAP_PTE);2260 break;2261 }2262 continue;2263 }2264 2265 if (folio_order(folio) == HPAGE_PMD_ORDER &&2266 folio->index == start) {2267 /* Maybe PMD-mapped */2268 result = SCAN_PTE_MAPPED_HUGEPAGE;2269 /*2270 * For SCAN_PTE_MAPPED_HUGEPAGE, further processing2271 * by the caller won't touch the page cache, and so2272 * it's safe to skip LRU and refcount checks before2273 * returning.2274 */2275 break;2276 }2277 2278 node = folio_nid(folio);2279 if (hpage_collapse_scan_abort(node, cc)) {2280 result = SCAN_SCAN_ABORT;2281 break;2282 }2283 cc->node_load[node]++;2284 2285 if (!folio_test_lru(folio)) {2286 result = SCAN_PAGE_LRU;2287 break;2288 }2289 2290 if (!is_refcount_suitable(folio)) {2291 result = SCAN_PAGE_COUNT;2292 break;2293 }2294 2295 /*2296 * We probably should check if the folio is referenced2297 * here, but nobody would transfer pte_young() to2298 * folio_test_referenced() for us. And rmap walk here2299 * is just too costly...2300 */2301 2302 present += folio_nr_pages(folio);2303 2304 if (need_resched()) {2305 xas_pause(&xas);2306 cond_resched_rcu();2307 }2308 }2309 rcu_read_unlock();2310 2311 if (result == SCAN_SUCCEED) {2312 if (cc->is_khugepaged &&2313 present < HPAGE_PMD_NR - khugepaged_max_ptes_none) {2314 result = SCAN_EXCEED_NONE_PTE;2315 count_vm_event(THP_SCAN_EXCEED_NONE_PTE);2316 } else {2317 result = collapse_file(mm, addr, file, start, cc);2318 }2319 }2320 2321 trace_mm_khugepaged_scan_file(mm, folio, file, present, swap, result);2322 return result;2323}2324#else2325static int hpage_collapse_scan_file(struct mm_struct *mm, unsigned long addr,2326 struct file *file, pgoff_t start,2327 struct collapse_control *cc)2328{2329 BUILD_BUG();2330}2331#endif2332 2333static unsigned int khugepaged_scan_mm_slot(unsigned int pages, int *result,2334 struct collapse_control *cc)2335 __releases(&khugepaged_mm_lock)2336 __acquires(&khugepaged_mm_lock)2337{2338 struct vma_iterator vmi;2339 struct khugepaged_mm_slot *mm_slot;2340 struct mm_slot *slot;2341 struct mm_struct *mm;2342 struct vm_area_struct *vma;2343 int progress = 0;2344 2345 VM_BUG_ON(!pages);2346 lockdep_assert_held(&khugepaged_mm_lock);2347 *result = SCAN_FAIL;2348 2349 if (khugepaged_scan.mm_slot) {2350 mm_slot = khugepaged_scan.mm_slot;2351 slot = &mm_slot->slot;2352 } else {2353 slot = list_entry(khugepaged_scan.mm_head.next,2354 struct mm_slot, mm_node);2355 mm_slot = mm_slot_entry(slot, struct khugepaged_mm_slot, slot);2356 khugepaged_scan.address = 0;2357 khugepaged_scan.mm_slot = mm_slot;2358 }2359 spin_unlock(&khugepaged_mm_lock);2360 2361 mm = slot->mm;2362 /*2363 * Don't wait for semaphore (to avoid long wait times). Just move to2364 * the next mm on the list.2365 */2366 vma = NULL;2367 if (unlikely(!mmap_read_trylock(mm)))2368 goto breakouterloop_mmap_lock;2369 2370 progress++;2371 if (unlikely(hpage_collapse_test_exit_or_disable(mm)))2372 goto breakouterloop;2373 2374 vma_iter_init(&vmi, mm, khugepaged_scan.address);2375 for_each_vma(vmi, vma) {2376 unsigned long hstart, hend;2377 2378 cond_resched();2379 if (unlikely(hpage_collapse_test_exit_or_disable(mm))) {2380 progress++;2381 break;2382 }2383 if (!thp_vma_allowable_order(vma, vma->vm_flags,2384 TVA_ENFORCE_SYSFS, PMD_ORDER)) {2385skip:2386 progress++;2387 continue;2388 }2389 hstart = round_up(vma->vm_start, HPAGE_PMD_SIZE);2390 hend = round_down(vma->vm_end, HPAGE_PMD_SIZE);2391 if (khugepaged_scan.address > hend)2392 goto skip;2393 if (khugepaged_scan.address < hstart)2394 khugepaged_scan.address = hstart;2395 VM_BUG_ON(khugepaged_scan.address & ~HPAGE_PMD_MASK);2396 2397 while (khugepaged_scan.address < hend) {2398 bool mmap_locked = true;2399 2400 cond_resched();2401 if (unlikely(hpage_collapse_test_exit_or_disable(mm)))2402 goto breakouterloop;2403 2404 VM_BUG_ON(khugepaged_scan.address < hstart ||2405 khugepaged_scan.address + HPAGE_PMD_SIZE >2406 hend);2407 if (IS_ENABLED(CONFIG_SHMEM) && vma->vm_file) {2408 struct file *file = get_file(vma->vm_file);2409 pgoff_t pgoff = linear_page_index(vma,2410 khugepaged_scan.address);2411 2412 mmap_read_unlock(mm);2413 mmap_locked = false;2414 *result = hpage_collapse_scan_file(mm,2415 khugepaged_scan.address, file, pgoff, cc);2416 fput(file);2417 if (*result == SCAN_PTE_MAPPED_HUGEPAGE) {2418 mmap_read_lock(mm);2419 if (hpage_collapse_test_exit_or_disable(mm))2420 goto breakouterloop;2421 *result = collapse_pte_mapped_thp(mm,2422 khugepaged_scan.address, false);2423 if (*result == SCAN_PMD_MAPPED)2424 *result = SCAN_SUCCEED;2425 mmap_read_unlock(mm);2426 }2427 } else {2428 *result = hpage_collapse_scan_pmd(mm, vma,2429 khugepaged_scan.address, &mmap_locked, cc);2430 }2431 2432 if (*result == SCAN_SUCCEED)2433 ++khugepaged_pages_collapsed;2434 2435 /* move to next address */2436 khugepaged_scan.address += HPAGE_PMD_SIZE;2437 progress += HPAGE_PMD_NR;2438 if (!mmap_locked)2439 /*2440 * We released mmap_lock so break loop. Note2441 * that we drop mmap_lock before all hugepage2442 * allocations, so if allocation fails, we are2443 * guaranteed to break here and report the2444 * correct result back to caller.2445 */2446 goto breakouterloop_mmap_lock;2447 if (progress >= pages)2448 goto breakouterloop;2449 }2450 }2451breakouterloop:2452 mmap_read_unlock(mm); /* exit_mmap will destroy ptes after this */2453breakouterloop_mmap_lock:2454 2455 spin_lock(&khugepaged_mm_lock);2456 VM_BUG_ON(khugepaged_scan.mm_slot != mm_slot);2457 /*2458 * Release the current mm_slot if this mm is about to die, or2459 * if we scanned all vmas of this mm.2460 */2461 if (hpage_collapse_test_exit(mm) || !vma) {2462 /*2463 * Make sure that if mm_users is reaching zero while2464 * khugepaged runs here, khugepaged_exit will find2465 * mm_slot not pointing to the exiting mm.2466 */2467 if (slot->mm_node.next != &khugepaged_scan.mm_head) {2468 slot = list_entry(slot->mm_node.next,2469 struct mm_slot, mm_node);2470 khugepaged_scan.mm_slot =2471 mm_slot_entry(slot, struct khugepaged_mm_slot, slot);2472 khugepaged_scan.address = 0;2473 } else {2474 khugepaged_scan.mm_slot = NULL;2475 khugepaged_full_scans++;2476 }2477 2478 collect_mm_slot(mm_slot);2479 }2480 2481 return progress;2482}2483 2484static int khugepaged_has_work(void)2485{2486 return !list_empty(&khugepaged_scan.mm_head) && hugepage_pmd_enabled();2487}2488 2489static int khugepaged_wait_event(void)2490{2491 return !list_empty(&khugepaged_scan.mm_head) ||2492 kthread_should_stop();2493}2494 2495static void khugepaged_do_scan(struct collapse_control *cc)2496{2497 unsigned int progress = 0, pass_through_head = 0;2498 unsigned int pages = READ_ONCE(khugepaged_pages_to_scan);2499 bool wait = true;2500 int result = SCAN_SUCCEED;2501 2502 lru_add_drain_all();2503 2504 while (true) {2505 cond_resched();2506 2507 if (unlikely(kthread_should_stop()))2508 break;2509 2510 spin_lock(&khugepaged_mm_lock);2511 if (!khugepaged_scan.mm_slot)2512 pass_through_head++;2513 if (khugepaged_has_work() &&2514 pass_through_head < 2)2515 progress += khugepaged_scan_mm_slot(pages - progress,2516 &result, cc);2517 else2518 progress = pages;2519 spin_unlock(&khugepaged_mm_lock);2520 2521 if (progress >= pages)2522 break;2523 2524 if (result == SCAN_ALLOC_HUGE_PAGE_FAIL) {2525 /*2526 * If fail to allocate the first time, try to sleep for2527 * a while. When hit again, cancel the scan.2528 */2529 if (!wait)2530 break;2531 wait = false;2532 khugepaged_alloc_sleep();2533 }2534 }2535}2536 2537static bool khugepaged_should_wakeup(void)2538{2539 return kthread_should_stop() ||2540 time_after_eq(jiffies, khugepaged_sleep_expire);2541}2542 2543static void khugepaged_wait_work(void)2544{2545 if (khugepaged_has_work()) {2546 const unsigned long scan_sleep_jiffies =2547 msecs_to_jiffies(khugepaged_scan_sleep_millisecs);2548 2549 if (!scan_sleep_jiffies)2550 return;2551 2552 khugepaged_sleep_expire = jiffies + scan_sleep_jiffies;2553 wait_event_freezable_timeout(khugepaged_wait,2554 khugepaged_should_wakeup(),2555 scan_sleep_jiffies);2556 return;2557 }2558 2559 if (hugepage_pmd_enabled())2560 wait_event_freezable(khugepaged_wait, khugepaged_wait_event());2561}2562 2563static int khugepaged(void *none)2564{2565 struct khugepaged_mm_slot *mm_slot;2566 2567 set_freezable();2568 set_user_nice(current, MAX_NICE);2569 2570 while (!kthread_should_stop()) {2571 khugepaged_do_scan(&khugepaged_collapse_control);2572 khugepaged_wait_work();2573 }2574 2575 spin_lock(&khugepaged_mm_lock);2576 mm_slot = khugepaged_scan.mm_slot;2577 khugepaged_scan.mm_slot = NULL;2578 if (mm_slot)2579 collect_mm_slot(mm_slot);2580 spin_unlock(&khugepaged_mm_lock);2581 return 0;2582}2583 2584static void set_recommended_min_free_kbytes(void)2585{2586 struct zone *zone;2587 int nr_zones = 0;2588 unsigned long recommended_min;2589 2590 if (!hugepage_pmd_enabled()) {2591 calculate_min_free_kbytes();2592 goto update_wmarks;2593 }2594 2595 for_each_populated_zone(zone) {2596 /*2597 * We don't need to worry about fragmentation of2598 * ZONE_MOVABLE since it only has movable pages.2599 */2600 if (zone_idx(zone) > gfp_zone(GFP_USER))2601 continue;2602 2603 nr_zones++;2604 }2605 2606 /* Ensure 2 pageblocks are free to assist fragmentation avoidance */2607 recommended_min = pageblock_nr_pages * nr_zones * 2;2608 2609 /*2610 * Make sure that on average at least two pageblocks are almost free2611 * of another type, one for a migratetype to fall back to and a2612 * second to avoid subsequent fallbacks of other types There are 32613 * MIGRATE_TYPES we care about.2614 */2615 recommended_min += pageblock_nr_pages * nr_zones *2616 MIGRATE_PCPTYPES * MIGRATE_PCPTYPES;2617 2618 /* don't ever allow to reserve more than 5% of the lowmem */2619 recommended_min = min(recommended_min,2620 (unsigned long) nr_free_buffer_pages() / 20);2621 recommended_min <<= (PAGE_SHIFT-10);2622 2623 if (recommended_min > min_free_kbytes) {2624 if (user_min_free_kbytes >= 0)2625 pr_info("raising min_free_kbytes from %d to %lu to help transparent hugepage allocations\n",2626 min_free_kbytes, recommended_min);2627 2628 min_free_kbytes = recommended_min;2629 }2630 2631update_wmarks:2632 setup_per_zone_wmarks();2633}2634 2635int start_stop_khugepaged(void)2636{2637 int err = 0;2638 2639 mutex_lock(&khugepaged_mutex);2640 if (hugepage_pmd_enabled()) {2641 if (!khugepaged_thread)2642 khugepaged_thread = kthread_run(khugepaged, NULL,2643 "khugepaged");2644 if (IS_ERR(khugepaged_thread)) {2645 pr_err("khugepaged: kthread_run(khugepaged) failed\n");2646 err = PTR_ERR(khugepaged_thread);2647 khugepaged_thread = NULL;2648 goto fail;2649 }2650 2651 if (!list_empty(&khugepaged_scan.mm_head))2652 wake_up_interruptible(&khugepaged_wait);2653 } else if (khugepaged_thread) {2654 kthread_stop(khugepaged_thread);2655 khugepaged_thread = NULL;2656 }2657 set_recommended_min_free_kbytes();2658fail:2659 mutex_unlock(&khugepaged_mutex);2660 return err;2661}2662 2663void khugepaged_min_free_kbytes_update(void)2664{2665 mutex_lock(&khugepaged_mutex);2666 if (hugepage_pmd_enabled() && khugepaged_thread)2667 set_recommended_min_free_kbytes();2668 mutex_unlock(&khugepaged_mutex);2669}2670 2671bool current_is_khugepaged(void)2672{2673 return kthread_func(current) == khugepaged;2674}2675 2676static int madvise_collapse_errno(enum scan_result r)2677{2678 /*2679 * MADV_COLLAPSE breaks from existing madvise(2) conventions to provide2680 * actionable feedback to caller, so they may take an appropriate2681 * fallback measure depending on the nature of the failure.2682 */2683 switch (r) {2684 case SCAN_ALLOC_HUGE_PAGE_FAIL:2685 return -ENOMEM;2686 case SCAN_CGROUP_CHARGE_FAIL:2687 case SCAN_EXCEED_NONE_PTE:2688 return -EBUSY;2689 /* Resource temporary unavailable - trying again might succeed */2690 case SCAN_PAGE_COUNT:2691 case SCAN_PAGE_LOCK:2692 case SCAN_PAGE_LRU:2693 case SCAN_DEL_PAGE_LRU:2694 case SCAN_PAGE_FILLED:2695 return -EAGAIN;2696 /*2697 * Other: Trying again likely not to succeed / error intrinsic to2698 * specified memory range. khugepaged likely won't be able to collapse2699 * either.2700 */2701 default:2702 return -EINVAL;2703 }2704}2705 2706int madvise_collapse(struct vm_area_struct *vma, struct vm_area_struct **prev,2707 unsigned long start, unsigned long end)2708{2709 struct collapse_control *cc;2710 struct mm_struct *mm = vma->vm_mm;2711 unsigned long hstart, hend, addr;2712 int thps = 0, last_fail = SCAN_FAIL;2713 bool mmap_locked = true;2714 2715 BUG_ON(vma->vm_start > start);2716 BUG_ON(vma->vm_end < end);2717 2718 *prev = vma;2719 2720 if (!thp_vma_allowable_order(vma, vma->vm_flags, 0, PMD_ORDER))2721 return -EINVAL;2722 2723 cc = kmalloc(sizeof(*cc), GFP_KERNEL);2724 if (!cc)2725 return -ENOMEM;2726 cc->is_khugepaged = false;2727 2728 mmgrab(mm);2729 lru_add_drain_all();2730 2731 hstart = (start + ~HPAGE_PMD_MASK) & HPAGE_PMD_MASK;2732 hend = end & HPAGE_PMD_MASK;2733 2734 for (addr = hstart; addr < hend; addr += HPAGE_PMD_SIZE) {2735 int result = SCAN_FAIL;2736 2737 if (!mmap_locked) {2738 cond_resched();2739 mmap_read_lock(mm);2740 mmap_locked = true;2741 result = hugepage_vma_revalidate(mm, addr, false, &vma,2742 cc);2743 if (result != SCAN_SUCCEED) {2744 last_fail = result;2745 goto out_nolock;2746 }2747 2748 hend = min(hend, vma->vm_end & HPAGE_PMD_MASK);2749 }2750 mmap_assert_locked(mm);2751 memset(cc->node_load, 0, sizeof(cc->node_load));2752 nodes_clear(cc->alloc_nmask);2753 if (IS_ENABLED(CONFIG_SHMEM) && vma->vm_file) {2754 struct file *file = get_file(vma->vm_file);2755 pgoff_t pgoff = linear_page_index(vma, addr);2756 2757 mmap_read_unlock(mm);2758 mmap_locked = false;2759 result = hpage_collapse_scan_file(mm, addr, file, pgoff,2760 cc);2761 fput(file);2762 } else {2763 result = hpage_collapse_scan_pmd(mm, vma, addr,2764 &mmap_locked, cc);2765 }2766 if (!mmap_locked)2767 *prev = NULL; /* Tell caller we dropped mmap_lock */2768 2769handle_result:2770 switch (result) {2771 case SCAN_SUCCEED:2772 case SCAN_PMD_MAPPED:2773 ++thps;2774 break;2775 case SCAN_PTE_MAPPED_HUGEPAGE:2776 BUG_ON(mmap_locked);2777 BUG_ON(*prev);2778 mmap_read_lock(mm);2779 result = collapse_pte_mapped_thp(mm, addr, true);2780 mmap_read_unlock(mm);2781 goto handle_result;2782 /* Whitelisted set of results where continuing OK */2783 case SCAN_PMD_NULL:2784 case SCAN_PTE_NON_PRESENT:2785 case SCAN_PTE_UFFD_WP:2786 case SCAN_PAGE_RO:2787 case SCAN_LACK_REFERENCED_PAGE:2788 case SCAN_PAGE_NULL:2789 case SCAN_PAGE_COUNT:2790 case SCAN_PAGE_LOCK:2791 case SCAN_PAGE_COMPOUND:2792 case SCAN_PAGE_LRU:2793 case SCAN_DEL_PAGE_LRU:2794 last_fail = result;2795 break;2796 default:2797 last_fail = result;2798 /* Other error, exit */2799 goto out_maybelock;2800 }2801 }2802 2803out_maybelock:2804 /* Caller expects us to hold mmap_lock on return */2805 if (!mmap_locked)2806 mmap_read_lock(mm);2807out_nolock:2808 mmap_assert_locked(mm);2809 mmdrop(mm);2810 kfree(cc);2811 2812 return thps == ((hend - hstart) >> HPAGE_PMD_SHIFT) ? 02813 : madvise_collapse_errno(last_fail);2814}2815