192 lines · c
1/* SPDX-License-Identifier: GPL-2.0 */2#ifndef _LINUX_PAGEWALK_H3#define _LINUX_PAGEWALK_H4 5#include <linux/mm.h>6 7struct mm_walk;8 9/* Locking requirement during a page walk. */10enum page_walk_lock {11 /* mmap_lock should be locked for read to stabilize the vma tree */12 PGWALK_RDLOCK = 0,13 /* vma will be write-locked during the walk */14 PGWALK_WRLOCK = 1,15 /* vma is expected to be already write-locked during the walk */16 PGWALK_WRLOCK_VERIFY = 2,17};18 19/**20 * struct mm_walk_ops - callbacks for walk_page_range21 * @pgd_entry: if set, called for each non-empty PGD (top-level) entry22 * @p4d_entry: if set, called for each non-empty P4D entry23 * @pud_entry: if set, called for each non-empty PUD entry24 * @pmd_entry: if set, called for each non-empty PMD entry25 * this handler is required to be able to handle26 * pmd_trans_huge() pmds. They may simply choose to27 * split_huge_page() instead of handling it explicitly.28 * @pte_entry: if set, called for each PTE (lowest-level) entry,29 * including empty ones30 * @pte_hole: if set, called for each hole at all levels,31 * depth is -1 if not known, 0:PGD, 1:P4D, 2:PUD, 3:PMD.32 * Any folded depths (where PTRS_PER_P?D is equal to 1)33 * are skipped.34 * @hugetlb_entry: if set, called for each hugetlb entry. This hook35 * function is called with the vma lock held, in order to36 * protect against a concurrent freeing of the pte_t* or37 * the ptl. In some cases, the hook function needs to drop38 * and retake the vma lock in order to avoid deadlocks39 * while calling other functions. In such cases the hook40 * function must either refrain from accessing the pte or41 * ptl after dropping the vma lock, or else revalidate42 * those items after re-acquiring the vma lock and before43 * accessing them.44 * @test_walk: caller specific callback function to determine whether45 * we walk over the current vma or not. Returning 0 means46 * "do page table walk over the current vma", returning47 * a negative value means "abort current page table walk48 * right now" and returning 1 means "skip the current vma"49 * Note that this callback is not called when the caller50 * passes in a single VMA as for walk_page_vma().51 * @pre_vma: if set, called before starting walk on a non-null vma.52 * @post_vma: if set, called after a walk on a non-null vma, provided53 * that @pre_vma and the vma walk succeeded.54 *55 * p?d_entry callbacks are called even if those levels are folded on a56 * particular architecture/configuration.57 */58struct mm_walk_ops {59 int (*pgd_entry)(pgd_t *pgd, unsigned long addr,60 unsigned long next, struct mm_walk *walk);61 int (*p4d_entry)(p4d_t *p4d, unsigned long addr,62 unsigned long next, struct mm_walk *walk);63 int (*pud_entry)(pud_t *pud, unsigned long addr,64 unsigned long next, struct mm_walk *walk);65 int (*pmd_entry)(pmd_t *pmd, unsigned long addr,66 unsigned long next, struct mm_walk *walk);67 int (*pte_entry)(pte_t *pte, unsigned long addr,68 unsigned long next, struct mm_walk *walk);69 int (*pte_hole)(unsigned long addr, unsigned long next,70 int depth, struct mm_walk *walk);71 int (*hugetlb_entry)(pte_t *pte, unsigned long hmask,72 unsigned long addr, unsigned long next,73 struct mm_walk *walk);74 int (*test_walk)(unsigned long addr, unsigned long next,75 struct mm_walk *walk);76 int (*pre_vma)(unsigned long start, unsigned long end,77 struct mm_walk *walk);78 void (*post_vma)(struct mm_walk *walk);79 enum page_walk_lock walk_lock;80};81 82/*83 * Action for pud_entry / pmd_entry callbacks.84 * ACTION_SUBTREE is the default85 */86enum page_walk_action {87 /* Descend to next level, splitting huge pages if needed and possible */88 ACTION_SUBTREE = 0,89 /* Continue to next entry at this level (ignoring any subtree) */90 ACTION_CONTINUE = 1,91 /* Call again for this entry */92 ACTION_AGAIN = 293};94 95/**96 * struct mm_walk - walk_page_range data97 * @ops: operation to call during the walk98 * @mm: mm_struct representing the target process of page table walk99 * @pgd: pointer to PGD; only valid with no_vma (otherwise set to NULL)100 * @vma: vma currently walked (NULL if walking outside vmas)101 * @action: next action to perform (see enum page_walk_action)102 * @no_vma: walk ignoring vmas (vma will always be NULL)103 * @private: private data for callbacks' usage104 *105 * (see the comment on walk_page_range() for more details)106 */107struct mm_walk {108 const struct mm_walk_ops *ops;109 struct mm_struct *mm;110 pgd_t *pgd;111 struct vm_area_struct *vma;112 enum page_walk_action action;113 bool no_vma;114 void *private;115};116 117int walk_page_range(struct mm_struct *mm, unsigned long start,118 unsigned long end, const struct mm_walk_ops *ops,119 void *private);120int walk_page_range_novma(struct mm_struct *mm, unsigned long start,121 unsigned long end, const struct mm_walk_ops *ops,122 pgd_t *pgd,123 void *private);124int walk_page_range_vma(struct vm_area_struct *vma, unsigned long start,125 unsigned long end, const struct mm_walk_ops *ops,126 void *private);127int walk_page_vma(struct vm_area_struct *vma, const struct mm_walk_ops *ops,128 void *private);129int walk_page_mapping(struct address_space *mapping, pgoff_t first_index,130 pgoff_t nr, const struct mm_walk_ops *ops,131 void *private);132 133typedef int __bitwise folio_walk_flags_t;134 135/*136 * Walk migration entries as well. Careful: a large folio might get split137 * concurrently.138 */139#define FW_MIGRATION ((__force folio_walk_flags_t)BIT(0))140 141/* Walk shared zeropages (small + huge) as well. */142#define FW_ZEROPAGE ((__force folio_walk_flags_t)BIT(1))143 144enum folio_walk_level {145 FW_LEVEL_PTE,146 FW_LEVEL_PMD,147 FW_LEVEL_PUD,148};149 150/**151 * struct folio_walk - folio_walk_start() / folio_walk_end() data152 * @page: exact folio page referenced (if applicable)153 * @level: page table level identifying the entry type154 * @pte: pointer to the page table entry (FW_LEVEL_PTE).155 * @pmd: pointer to the page table entry (FW_LEVEL_PMD).156 * @pud: pointer to the page table entry (FW_LEVEL_PUD).157 * @ptl: pointer to the page table lock.158 *159 * (see folio_walk_start() documentation for more details)160 */161struct folio_walk {162 /* public */163 struct page *page;164 enum folio_walk_level level;165 union {166 pte_t *ptep;167 pud_t *pudp;168 pmd_t *pmdp;169 };170 union {171 pte_t pte;172 pud_t pud;173 pmd_t pmd;174 };175 /* private */176 struct vm_area_struct *vma;177 spinlock_t *ptl;178};179 180struct folio *folio_walk_start(struct folio_walk *fw,181 struct vm_area_struct *vma, unsigned long addr,182 folio_walk_flags_t flags);183 184#define folio_walk_end(__fw, __vma) do { \185 spin_unlock((__fw)->ptl); \186 if (likely((__fw)->level == FW_LEVEL_PTE)) \187 pte_unmap((__fw)->ptep); \188 vma_pgtable_walk_end(__vma); \189} while (0)190 191#endif /* _LINUX_PAGEWALK_H */192