610 lines · c
1// SPDX-License-Identifier: GPL-2.0-or-later2/*3 * Copyright 2013 Red Hat Inc.4 *5 * Authors: Jérôme Glisse <jglisse@redhat.com>6 */7/*8 * Refer to include/linux/hmm.h for information about heterogeneous memory9 * management or HMM for short.10 */11#include <linux/pagewalk.h>12#include <linux/hmm.h>13#include <linux/init.h>14#include <linux/rmap.h>15#include <linux/swap.h>16#include <linux/slab.h>17#include <linux/sched.h>18#include <linux/mmzone.h>19#include <linux/pagemap.h>20#include <linux/swapops.h>21#include <linux/hugetlb.h>22#include <linux/memremap.h>23#include <linux/sched/mm.h>24#include <linux/jump_label.h>25#include <linux/dma-mapping.h>26#include <linux/mmu_notifier.h>27#include <linux/memory_hotplug.h>28 29#include "internal.h"30 31struct hmm_vma_walk {32 struct hmm_range *range;33 unsigned long last;34};35 36enum {37 HMM_NEED_FAULT = 1 << 0,38 HMM_NEED_WRITE_FAULT = 1 << 1,39 HMM_NEED_ALL_BITS = HMM_NEED_FAULT | HMM_NEED_WRITE_FAULT,40};41 42static int hmm_pfns_fill(unsigned long addr, unsigned long end,43 struct hmm_range *range, unsigned long cpu_flags)44{45 unsigned long i = (addr - range->start) >> PAGE_SHIFT;46 47 for (; addr < end; addr += PAGE_SIZE, i++)48 range->hmm_pfns[i] = cpu_flags;49 return 0;50}51 52/*53 * hmm_vma_fault() - fault in a range lacking valid pmd or pte(s)54 * @addr: range virtual start address (inclusive)55 * @end: range virtual end address (exclusive)56 * @required_fault: HMM_NEED_* flags57 * @walk: mm_walk structure58 * Return: -EBUSY after page fault, or page fault error59 *60 * This function will be called whenever pmd_none() or pte_none() returns true,61 * or whenever there is no page directory covering the virtual address range.62 */63static int hmm_vma_fault(unsigned long addr, unsigned long end,64 unsigned int required_fault, struct mm_walk *walk)65{66 struct hmm_vma_walk *hmm_vma_walk = walk->private;67 struct vm_area_struct *vma = walk->vma;68 unsigned int fault_flags = FAULT_FLAG_REMOTE;69 70 WARN_ON_ONCE(!required_fault);71 hmm_vma_walk->last = addr;72 73 if (required_fault & HMM_NEED_WRITE_FAULT) {74 if (!(vma->vm_flags & VM_WRITE))75 return -EPERM;76 fault_flags |= FAULT_FLAG_WRITE;77 }78 79 for (; addr < end; addr += PAGE_SIZE)80 if (handle_mm_fault(vma, addr, fault_flags, NULL) &81 VM_FAULT_ERROR)82 return -EFAULT;83 return -EBUSY;84}85 86static unsigned int hmm_pte_need_fault(const struct hmm_vma_walk *hmm_vma_walk,87 unsigned long pfn_req_flags,88 unsigned long cpu_flags)89{90 struct hmm_range *range = hmm_vma_walk->range;91 92 /*93 * So we not only consider the individual per page request we also94 * consider the default flags requested for the range. The API can95 * be used 2 ways. The first one where the HMM user coalesces96 * multiple page faults into one request and sets flags per pfn for97 * those faults. The second one where the HMM user wants to pre-98 * fault a range with specific flags. For the latter one it is a99 * waste to have the user pre-fill the pfn arrays with a default100 * flags value.101 */102 pfn_req_flags &= range->pfn_flags_mask;103 pfn_req_flags |= range->default_flags;104 105 /* We aren't ask to do anything ... */106 if (!(pfn_req_flags & HMM_PFN_REQ_FAULT))107 return 0;108 109 /* Need to write fault ? */110 if ((pfn_req_flags & HMM_PFN_REQ_WRITE) &&111 !(cpu_flags & HMM_PFN_WRITE))112 return HMM_NEED_FAULT | HMM_NEED_WRITE_FAULT;113 114 /* If CPU page table is not valid then we need to fault */115 if (!(cpu_flags & HMM_PFN_VALID))116 return HMM_NEED_FAULT;117 return 0;118}119 120static unsigned int121hmm_range_need_fault(const struct hmm_vma_walk *hmm_vma_walk,122 const unsigned long hmm_pfns[], unsigned long npages,123 unsigned long cpu_flags)124{125 struct hmm_range *range = hmm_vma_walk->range;126 unsigned int required_fault = 0;127 unsigned long i;128 129 /*130 * If the default flags do not request to fault pages, and the mask does131 * not allow for individual pages to be faulted, then132 * hmm_pte_need_fault() will always return 0.133 */134 if (!((range->default_flags | range->pfn_flags_mask) &135 HMM_PFN_REQ_FAULT))136 return 0;137 138 for (i = 0; i < npages; ++i) {139 required_fault |= hmm_pte_need_fault(hmm_vma_walk, hmm_pfns[i],140 cpu_flags);141 if (required_fault == HMM_NEED_ALL_BITS)142 return required_fault;143 }144 return required_fault;145}146 147static int hmm_vma_walk_hole(unsigned long addr, unsigned long end,148 __always_unused int depth, struct mm_walk *walk)149{150 struct hmm_vma_walk *hmm_vma_walk = walk->private;151 struct hmm_range *range = hmm_vma_walk->range;152 unsigned int required_fault;153 unsigned long i, npages;154 unsigned long *hmm_pfns;155 156 i = (addr - range->start) >> PAGE_SHIFT;157 npages = (end - addr) >> PAGE_SHIFT;158 hmm_pfns = &range->hmm_pfns[i];159 required_fault =160 hmm_range_need_fault(hmm_vma_walk, hmm_pfns, npages, 0);161 if (!walk->vma) {162 if (required_fault)163 return -EFAULT;164 return hmm_pfns_fill(addr, end, range, HMM_PFN_ERROR);165 }166 if (required_fault)167 return hmm_vma_fault(addr, end, required_fault, walk);168 return hmm_pfns_fill(addr, end, range, 0);169}170 171static inline unsigned long hmm_pfn_flags_order(unsigned long order)172{173 return order << HMM_PFN_ORDER_SHIFT;174}175 176static inline unsigned long pmd_to_hmm_pfn_flags(struct hmm_range *range,177 pmd_t pmd)178{179 if (pmd_protnone(pmd))180 return 0;181 return (pmd_write(pmd) ? (HMM_PFN_VALID | HMM_PFN_WRITE) :182 HMM_PFN_VALID) |183 hmm_pfn_flags_order(PMD_SHIFT - PAGE_SHIFT);184}185 186#ifdef CONFIG_TRANSPARENT_HUGEPAGE187static int hmm_vma_handle_pmd(struct mm_walk *walk, unsigned long addr,188 unsigned long end, unsigned long hmm_pfns[],189 pmd_t pmd)190{191 struct hmm_vma_walk *hmm_vma_walk = walk->private;192 struct hmm_range *range = hmm_vma_walk->range;193 unsigned long pfn, npages, i;194 unsigned int required_fault;195 unsigned long cpu_flags;196 197 npages = (end - addr) >> PAGE_SHIFT;198 cpu_flags = pmd_to_hmm_pfn_flags(range, pmd);199 required_fault =200 hmm_range_need_fault(hmm_vma_walk, hmm_pfns, npages, cpu_flags);201 if (required_fault)202 return hmm_vma_fault(addr, end, required_fault, walk);203 204 pfn = pmd_pfn(pmd) + ((addr & ~PMD_MASK) >> PAGE_SHIFT);205 for (i = 0; addr < end; addr += PAGE_SIZE, i++, pfn++)206 hmm_pfns[i] = pfn | cpu_flags;207 return 0;208}209#else /* CONFIG_TRANSPARENT_HUGEPAGE */210/* stub to allow the code below to compile */211int hmm_vma_handle_pmd(struct mm_walk *walk, unsigned long addr,212 unsigned long end, unsigned long hmm_pfns[], pmd_t pmd);213#endif /* CONFIG_TRANSPARENT_HUGEPAGE */214 215static inline unsigned long pte_to_hmm_pfn_flags(struct hmm_range *range,216 pte_t pte)217{218 if (pte_none(pte) || !pte_present(pte) || pte_protnone(pte))219 return 0;220 return pte_write(pte) ? (HMM_PFN_VALID | HMM_PFN_WRITE) : HMM_PFN_VALID;221}222 223static int hmm_vma_handle_pte(struct mm_walk *walk, unsigned long addr,224 unsigned long end, pmd_t *pmdp, pte_t *ptep,225 unsigned long *hmm_pfn)226{227 struct hmm_vma_walk *hmm_vma_walk = walk->private;228 struct hmm_range *range = hmm_vma_walk->range;229 unsigned int required_fault;230 unsigned long cpu_flags;231 pte_t pte = ptep_get(ptep);232 uint64_t pfn_req_flags = *hmm_pfn;233 234 if (pte_none_mostly(pte)) {235 required_fault =236 hmm_pte_need_fault(hmm_vma_walk, pfn_req_flags, 0);237 if (required_fault)238 goto fault;239 *hmm_pfn = 0;240 return 0;241 }242 243 if (!pte_present(pte)) {244 swp_entry_t entry = pte_to_swp_entry(pte);245 246 /*247 * Don't fault in device private pages owned by the caller,248 * just report the PFN.249 */250 if (is_device_private_entry(entry) &&251 pfn_swap_entry_to_page(entry)->pgmap->owner ==252 range->dev_private_owner) {253 cpu_flags = HMM_PFN_VALID;254 if (is_writable_device_private_entry(entry))255 cpu_flags |= HMM_PFN_WRITE;256 *hmm_pfn = swp_offset_pfn(entry) | cpu_flags;257 return 0;258 }259 260 required_fault =261 hmm_pte_need_fault(hmm_vma_walk, pfn_req_flags, 0);262 if (!required_fault) {263 *hmm_pfn = 0;264 return 0;265 }266 267 if (!non_swap_entry(entry))268 goto fault;269 270 if (is_device_private_entry(entry))271 goto fault;272 273 if (is_device_exclusive_entry(entry))274 goto fault;275 276 if (is_migration_entry(entry)) {277 pte_unmap(ptep);278 hmm_vma_walk->last = addr;279 migration_entry_wait(walk->mm, pmdp, addr);280 return -EBUSY;281 }282 283 /* Report error for everything else */284 pte_unmap(ptep);285 return -EFAULT;286 }287 288 cpu_flags = pte_to_hmm_pfn_flags(range, pte);289 required_fault =290 hmm_pte_need_fault(hmm_vma_walk, pfn_req_flags, cpu_flags);291 if (required_fault)292 goto fault;293 294 /*295 * Bypass devmap pte such as DAX page when all pfn requested296 * flags(pfn_req_flags) are fulfilled.297 * Since each architecture defines a struct page for the zero page, just298 * fall through and treat it like a normal page.299 */300 if (!vm_normal_page(walk->vma, addr, pte) &&301 !pte_devmap(pte) &&302 !is_zero_pfn(pte_pfn(pte))) {303 if (hmm_pte_need_fault(hmm_vma_walk, pfn_req_flags, 0)) {304 pte_unmap(ptep);305 return -EFAULT;306 }307 *hmm_pfn = HMM_PFN_ERROR;308 return 0;309 }310 311 *hmm_pfn = pte_pfn(pte) | cpu_flags;312 return 0;313 314fault:315 pte_unmap(ptep);316 /* Fault any virtual address we were asked to fault */317 return hmm_vma_fault(addr, end, required_fault, walk);318}319 320static int hmm_vma_walk_pmd(pmd_t *pmdp,321 unsigned long start,322 unsigned long end,323 struct mm_walk *walk)324{325 struct hmm_vma_walk *hmm_vma_walk = walk->private;326 struct hmm_range *range = hmm_vma_walk->range;327 unsigned long *hmm_pfns =328 &range->hmm_pfns[(start - range->start) >> PAGE_SHIFT];329 unsigned long npages = (end - start) >> PAGE_SHIFT;330 unsigned long addr = start;331 pte_t *ptep;332 pmd_t pmd;333 334again:335 pmd = pmdp_get_lockless(pmdp);336 if (pmd_none(pmd))337 return hmm_vma_walk_hole(start, end, -1, walk);338 339 if (thp_migration_supported() && is_pmd_migration_entry(pmd)) {340 if (hmm_range_need_fault(hmm_vma_walk, hmm_pfns, npages, 0)) {341 hmm_vma_walk->last = addr;342 pmd_migration_entry_wait(walk->mm, pmdp);343 return -EBUSY;344 }345 return hmm_pfns_fill(start, end, range, 0);346 }347 348 if (!pmd_present(pmd)) {349 if (hmm_range_need_fault(hmm_vma_walk, hmm_pfns, npages, 0))350 return -EFAULT;351 return hmm_pfns_fill(start, end, range, HMM_PFN_ERROR);352 }353 354 if (pmd_devmap(pmd) || pmd_trans_huge(pmd)) {355 /*356 * No need to take pmd_lock here, even if some other thread357 * is splitting the huge pmd we will get that event through358 * mmu_notifier callback.359 *360 * So just read pmd value and check again it's a transparent361 * huge or device mapping one and compute corresponding pfn362 * values.363 */364 pmd = pmdp_get_lockless(pmdp);365 if (!pmd_devmap(pmd) && !pmd_trans_huge(pmd))366 goto again;367 368 return hmm_vma_handle_pmd(walk, addr, end, hmm_pfns, pmd);369 }370 371 /*372 * We have handled all the valid cases above ie either none, migration,373 * huge or transparent huge. At this point either it is a valid pmd374 * entry pointing to pte directory or it is a bad pmd that will not375 * recover.376 */377 if (pmd_bad(pmd)) {378 if (hmm_range_need_fault(hmm_vma_walk, hmm_pfns, npages, 0))379 return -EFAULT;380 return hmm_pfns_fill(start, end, range, HMM_PFN_ERROR);381 }382 383 ptep = pte_offset_map(pmdp, addr);384 if (!ptep)385 goto again;386 for (; addr < end; addr += PAGE_SIZE, ptep++, hmm_pfns++) {387 int r;388 389 r = hmm_vma_handle_pte(walk, addr, end, pmdp, ptep, hmm_pfns);390 if (r) {391 /* hmm_vma_handle_pte() did pte_unmap() */392 return r;393 }394 }395 pte_unmap(ptep - 1);396 return 0;397}398 399#if defined(CONFIG_ARCH_HAS_PTE_DEVMAP) && \400 defined(CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD)401static inline unsigned long pud_to_hmm_pfn_flags(struct hmm_range *range,402 pud_t pud)403{404 if (!pud_present(pud))405 return 0;406 return (pud_write(pud) ? (HMM_PFN_VALID | HMM_PFN_WRITE) :407 HMM_PFN_VALID) |408 hmm_pfn_flags_order(PUD_SHIFT - PAGE_SHIFT);409}410 411static int hmm_vma_walk_pud(pud_t *pudp, unsigned long start, unsigned long end,412 struct mm_walk *walk)413{414 struct hmm_vma_walk *hmm_vma_walk = walk->private;415 struct hmm_range *range = hmm_vma_walk->range;416 unsigned long addr = start;417 pud_t pud;418 spinlock_t *ptl = pud_trans_huge_lock(pudp, walk->vma);419 420 if (!ptl)421 return 0;422 423 /* Normally we don't want to split the huge page */424 walk->action = ACTION_CONTINUE;425 426 pud = READ_ONCE(*pudp);427 if (!pud_present(pud)) {428 spin_unlock(ptl);429 return hmm_vma_walk_hole(start, end, -1, walk);430 }431 432 if (pud_leaf(pud) && pud_devmap(pud)) {433 unsigned long i, npages, pfn;434 unsigned int required_fault;435 unsigned long *hmm_pfns;436 unsigned long cpu_flags;437 438 i = (addr - range->start) >> PAGE_SHIFT;439 npages = (end - addr) >> PAGE_SHIFT;440 hmm_pfns = &range->hmm_pfns[i];441 442 cpu_flags = pud_to_hmm_pfn_flags(range, pud);443 required_fault = hmm_range_need_fault(hmm_vma_walk, hmm_pfns,444 npages, cpu_flags);445 if (required_fault) {446 spin_unlock(ptl);447 return hmm_vma_fault(addr, end, required_fault, walk);448 }449 450 pfn = pud_pfn(pud) + ((addr & ~PUD_MASK) >> PAGE_SHIFT);451 for (i = 0; i < npages; ++i, ++pfn)452 hmm_pfns[i] = pfn | cpu_flags;453 goto out_unlock;454 }455 456 /* Ask for the PUD to be split */457 walk->action = ACTION_SUBTREE;458 459out_unlock:460 spin_unlock(ptl);461 return 0;462}463#else464#define hmm_vma_walk_pud NULL465#endif466 467#ifdef CONFIG_HUGETLB_PAGE468static int hmm_vma_walk_hugetlb_entry(pte_t *pte, unsigned long hmask,469 unsigned long start, unsigned long end,470 struct mm_walk *walk)471{472 unsigned long addr = start, i, pfn;473 struct hmm_vma_walk *hmm_vma_walk = walk->private;474 struct hmm_range *range = hmm_vma_walk->range;475 struct vm_area_struct *vma = walk->vma;476 unsigned int required_fault;477 unsigned long pfn_req_flags;478 unsigned long cpu_flags;479 spinlock_t *ptl;480 pte_t entry;481 482 ptl = huge_pte_lock(hstate_vma(vma), walk->mm, pte);483 entry = huge_ptep_get(walk->mm, addr, pte);484 485 i = (start - range->start) >> PAGE_SHIFT;486 pfn_req_flags = range->hmm_pfns[i];487 cpu_flags = pte_to_hmm_pfn_flags(range, entry) |488 hmm_pfn_flags_order(huge_page_order(hstate_vma(vma)));489 required_fault =490 hmm_pte_need_fault(hmm_vma_walk, pfn_req_flags, cpu_flags);491 if (required_fault) {492 int ret;493 494 spin_unlock(ptl);495 hugetlb_vma_unlock_read(vma);496 /*497 * Avoid deadlock: drop the vma lock before calling498 * hmm_vma_fault(), which will itself potentially take and499 * drop the vma lock. This is also correct from a500 * protection point of view, because there is no further501 * use here of either pte or ptl after dropping the vma502 * lock.503 */504 ret = hmm_vma_fault(addr, end, required_fault, walk);505 hugetlb_vma_lock_read(vma);506 return ret;507 }508 509 pfn = pte_pfn(entry) + ((start & ~hmask) >> PAGE_SHIFT);510 for (; addr < end; addr += PAGE_SIZE, i++, pfn++)511 range->hmm_pfns[i] = pfn | cpu_flags;512 513 spin_unlock(ptl);514 return 0;515}516#else517#define hmm_vma_walk_hugetlb_entry NULL518#endif /* CONFIG_HUGETLB_PAGE */519 520static int hmm_vma_walk_test(unsigned long start, unsigned long end,521 struct mm_walk *walk)522{523 struct hmm_vma_walk *hmm_vma_walk = walk->private;524 struct hmm_range *range = hmm_vma_walk->range;525 struct vm_area_struct *vma = walk->vma;526 527 if (!(vma->vm_flags & (VM_IO | VM_PFNMAP)) &&528 vma->vm_flags & VM_READ)529 return 0;530 531 /*532 * vma ranges that don't have struct page backing them or map I/O533 * devices directly cannot be handled by hmm_range_fault().534 *535 * If the vma does not allow read access, then assume that it does not536 * allow write access either. HMM does not support architectures that537 * allow write without read.538 *539 * If a fault is requested for an unsupported range then it is a hard540 * failure.541 */542 if (hmm_range_need_fault(hmm_vma_walk,543 range->hmm_pfns +544 ((start - range->start) >> PAGE_SHIFT),545 (end - start) >> PAGE_SHIFT, 0))546 return -EFAULT;547 548 hmm_pfns_fill(start, end, range, HMM_PFN_ERROR);549 550 /* Skip this vma and continue processing the next vma. */551 return 1;552}553 554static const struct mm_walk_ops hmm_walk_ops = {555 .pud_entry = hmm_vma_walk_pud,556 .pmd_entry = hmm_vma_walk_pmd,557 .pte_hole = hmm_vma_walk_hole,558 .hugetlb_entry = hmm_vma_walk_hugetlb_entry,559 .test_walk = hmm_vma_walk_test,560 .walk_lock = PGWALK_RDLOCK,561};562 563/**564 * hmm_range_fault - try to fault some address in a virtual address range565 * @range: argument structure566 *567 * Returns 0 on success or one of the following error codes:568 *569 * -EINVAL: Invalid arguments or mm or virtual address is in an invalid vma570 * (e.g., device file vma).571 * -ENOMEM: Out of memory.572 * -EPERM: Invalid permission (e.g., asking for write and range is read573 * only).574 * -EBUSY: The range has been invalidated and the caller needs to wait for575 * the invalidation to finish.576 * -EFAULT: A page was requested to be valid and could not be made valid577 * ie it has no backing VMA or it is illegal to access578 *579 * This is similar to get_user_pages(), except that it can read the page tables580 * without mutating them (ie causing faults).581 */582int hmm_range_fault(struct hmm_range *range)583{584 struct hmm_vma_walk hmm_vma_walk = {585 .range = range,586 .last = range->start,587 };588 struct mm_struct *mm = range->notifier->mm;589 int ret;590 591 mmap_assert_locked(mm);592 593 do {594 /* If range is no longer valid force retry. */595 if (mmu_interval_check_retry(range->notifier,596 range->notifier_seq))597 return -EBUSY;598 ret = walk_page_range(mm, hmm_vma_walk.last, range->end,599 &hmm_walk_ops, &hmm_vma_walk);600 /*601 * When -EBUSY is returned the loop restarts with602 * hmm_vma_walk.last set to an address that has not been stored603 * in pfns. All entries < last in the pfn array are set to their604 * output, and all >= are still at their input values.605 */606 } while (ret == -EBUSY);607 return ret;608}609EXPORT_SYMBOL(hmm_range_fault);610