754 lines · c
1/* SPDX-License-Identifier: GPL-2.0-or-later */2/* include/asm-generic/tlb.h3 *4 * Generic TLB shootdown code5 *6 * Copyright 2001 Red Hat, Inc.7 * Based on code from mm/memory.c Copyright Linus Torvalds and others.8 *9 * Copyright 2011 Red Hat, Inc., Peter Zijlstra10 */11#ifndef _ASM_GENERIC__TLB_H12#define _ASM_GENERIC__TLB_H13 14#include <linux/mmu_notifier.h>15#include <linux/swap.h>16#include <linux/hugetlb_inline.h>17#include <asm/tlbflush.h>18#include <asm/cacheflush.h>19 20/*21 * Blindly accessing user memory from NMI context can be dangerous22 * if we're in the middle of switching the current user task or switching23 * the loaded mm.24 */25#ifndef nmi_uaccess_okay26# define nmi_uaccess_okay() true27#endif28 29#ifdef CONFIG_MMU30 31/*32 * Generic MMU-gather implementation.33 *34 * The mmu_gather data structure is used by the mm code to implement the35 * correct and efficient ordering of freeing pages and TLB invalidations.36 *37 * This correct ordering is:38 *39 * 1) unhook page40 * 2) TLB invalidate page41 * 3) free page42 *43 * That is, we must never free a page before we have ensured there are no live44 * translations left to it. Otherwise it might be possible to observe (or45 * worse, change) the page content after it has been reused.46 *47 * The mmu_gather API consists of:48 *49 * - tlb_gather_mmu() / tlb_gather_mmu_fullmm() / tlb_finish_mmu()50 *51 * start and finish a mmu_gather52 *53 * Finish in particular will issue a (final) TLB invalidate and free54 * all (remaining) queued pages.55 *56 * - tlb_start_vma() / tlb_end_vma(); marks the start / end of a VMA57 *58 * Defaults to flushing at tlb_end_vma() to reset the range; helps when59 * there's large holes between the VMAs.60 *61 * - tlb_remove_table()62 *63 * tlb_remove_table() is the basic primitive to free page-table directories64 * (__p*_free_tlb()). In it's most primitive form it is an alias for65 * tlb_remove_page() below, for when page directories are pages and have no66 * additional constraints.67 *68 * See also MMU_GATHER_TABLE_FREE and MMU_GATHER_RCU_TABLE_FREE.69 *70 * - tlb_remove_page() / __tlb_remove_page()71 * - tlb_remove_page_size() / __tlb_remove_page_size()72 * - __tlb_remove_folio_pages()73 *74 * __tlb_remove_page_size() is the basic primitive that queues a page for75 * freeing. __tlb_remove_page() assumes PAGE_SIZE. Both will return a76 * boolean indicating if the queue is (now) full and a call to77 * tlb_flush_mmu() is required.78 *79 * tlb_remove_page() and tlb_remove_page_size() imply the call to80 * tlb_flush_mmu() when required and has no return value.81 *82 * __tlb_remove_folio_pages() is similar to __tlb_remove_page(), however,83 * instead of removing a single page, remove the given number of consecutive84 * pages that are all part of the same (large) folio: just like calling85 * __tlb_remove_page() on each page individually.86 *87 * - tlb_change_page_size()88 *89 * call before __tlb_remove_page*() to set the current page-size; implies a90 * possible tlb_flush_mmu() call.91 *92 * - tlb_flush_mmu() / tlb_flush_mmu_tlbonly()93 *94 * tlb_flush_mmu_tlbonly() - does the TLB invalidate (and resets95 * related state, like the range)96 *97 * tlb_flush_mmu() - in addition to the above TLB invalidate, also frees98 * whatever pages are still batched.99 *100 * - mmu_gather::fullmm101 *102 * A flag set by tlb_gather_mmu_fullmm() to indicate we're going to free103 * the entire mm; this allows a number of optimizations.104 *105 * - We can ignore tlb_{start,end}_vma(); because we don't106 * care about ranges. Everything will be shot down.107 *108 * - (RISC) architectures that use ASIDs can cycle to a new ASID109 * and delay the invalidation until ASID space runs out.110 *111 * - mmu_gather::need_flush_all112 *113 * A flag that can be set by the arch code if it wants to force114 * flush the entire TLB irrespective of the range. For instance115 * x86-PAE needs this when changing top-level entries.116 *117 * And allows the architecture to provide and implement tlb_flush():118 *119 * tlb_flush() may, in addition to the above mentioned mmu_gather fields, make120 * use of:121 *122 * - mmu_gather::start / mmu_gather::end123 *124 * which provides the range that needs to be flushed to cover the pages to125 * be freed.126 *127 * - mmu_gather::freed_tables128 *129 * set when we freed page table pages130 *131 * - tlb_get_unmap_shift() / tlb_get_unmap_size()132 *133 * returns the smallest TLB entry size unmapped in this range.134 *135 * If an architecture does not provide tlb_flush() a default implementation136 * based on flush_tlb_range() will be used, unless MMU_GATHER_NO_RANGE is137 * specified, in which case we'll default to flush_tlb_mm().138 *139 * Additionally there are a few opt-in features:140 *141 * MMU_GATHER_PAGE_SIZE142 *143 * This ensures we call tlb_flush() every time tlb_change_page_size() actually144 * changes the size and provides mmu_gather::page_size to tlb_flush().145 *146 * This might be useful if your architecture has size specific TLB147 * invalidation instructions.148 *149 * MMU_GATHER_TABLE_FREE150 *151 * This provides tlb_remove_table(), to be used instead of tlb_remove_page()152 * for page directores (__p*_free_tlb()).153 *154 * Useful if your architecture has non-page page directories.155 *156 * When used, an architecture is expected to provide __tlb_remove_table()157 * which does the actual freeing of these pages.158 *159 * MMU_GATHER_RCU_TABLE_FREE160 *161 * Like MMU_GATHER_TABLE_FREE, and adds semi-RCU semantics to the free (see162 * comment below).163 *164 * Useful if your architecture doesn't use IPIs for remote TLB invalidates165 * and therefore doesn't naturally serialize with software page-table walkers.166 *167 * MMU_GATHER_NO_FLUSH_CACHE168 *169 * Indicates the architecture has flush_cache_range() but it needs *NOT* be called170 * before unmapping a VMA.171 *172 * NOTE: strictly speaking we shouldn't have this knob and instead rely on173 * flush_cache_range() being a NOP, except Sparc64 seems to be174 * different here.175 *176 * MMU_GATHER_MERGE_VMAS177 *178 * Indicates the architecture wants to merge ranges over VMAs; typical when179 * multiple range invalidates are more expensive than a full invalidate.180 *181 * MMU_GATHER_NO_RANGE182 *183 * Use this if your architecture lacks an efficient flush_tlb_range(). This184 * option implies MMU_GATHER_MERGE_VMAS above.185 *186 * MMU_GATHER_NO_GATHER187 *188 * If the option is set the mmu_gather will not track individual pages for189 * delayed page free anymore. A platform that enables the option needs to190 * provide its own implementation of the __tlb_remove_page_size() function to191 * free pages.192 *193 * This is useful if your architecture already flushes TLB entries in the194 * various ptep_get_and_clear() functions.195 */196 197#ifdef CONFIG_MMU_GATHER_TABLE_FREE198 199struct mmu_table_batch {200#ifdef CONFIG_MMU_GATHER_RCU_TABLE_FREE201 struct rcu_head rcu;202#endif203 unsigned int nr;204 void *tables[];205};206 207#define MAX_TABLE_BATCH \208 ((PAGE_SIZE - sizeof(struct mmu_table_batch)) / sizeof(void *))209 210extern void tlb_remove_table(struct mmu_gather *tlb, void *table);211 212#else /* !CONFIG_MMU_GATHER_HAVE_TABLE_FREE */213 214/*215 * Without MMU_GATHER_TABLE_FREE the architecture is assumed to have page based216 * page directories and we can use the normal page batching to free them.217 */218#define tlb_remove_table(tlb, page) tlb_remove_page((tlb), (page))219 220#endif /* CONFIG_MMU_GATHER_TABLE_FREE */221 222#ifdef CONFIG_MMU_GATHER_RCU_TABLE_FREE223/*224 * This allows an architecture that does not use the linux page-tables for225 * hardware to skip the TLBI when freeing page tables.226 */227#ifndef tlb_needs_table_invalidate228#define tlb_needs_table_invalidate() (true)229#endif230 231void tlb_remove_table_sync_one(void);232 233#else234 235#ifdef tlb_needs_table_invalidate236#error tlb_needs_table_invalidate() requires MMU_GATHER_RCU_TABLE_FREE237#endif238 239static inline void tlb_remove_table_sync_one(void) { }240 241#endif /* CONFIG_MMU_GATHER_RCU_TABLE_FREE */242 243 244#ifndef CONFIG_MMU_GATHER_NO_GATHER245/*246 * If we can't allocate a page to make a big batch of page pointers247 * to work on, then just handle a few from the on-stack structure.248 */249#define MMU_GATHER_BUNDLE 8250 251struct mmu_gather_batch {252 struct mmu_gather_batch *next;253 unsigned int nr;254 unsigned int max;255 struct encoded_page *encoded_pages[];256};257 258#define MAX_GATHER_BATCH \259 ((PAGE_SIZE - sizeof(struct mmu_gather_batch)) / sizeof(void *))260 261/*262 * Limit the maximum number of mmu_gather batches to reduce a risk of soft263 * lockups for non-preemptible kernels on huge machines when a lot of memory264 * is zapped during unmapping.265 * 10K pages freed at once should be safe even without a preemption point.266 */267#define MAX_GATHER_BATCH_COUNT (10000UL/MAX_GATHER_BATCH)268 269extern bool __tlb_remove_page_size(struct mmu_gather *tlb, struct page *page,270 bool delay_rmap, int page_size);271bool __tlb_remove_folio_pages(struct mmu_gather *tlb, struct page *page,272 unsigned int nr_pages, bool delay_rmap);273 274#ifdef CONFIG_SMP275/*276 * This both sets 'delayed_rmap', and returns true. It would be an inline277 * function, except we define it before the 'struct mmu_gather'.278 */279#define tlb_delay_rmap(tlb) (((tlb)->delayed_rmap = 1), true)280extern void tlb_flush_rmaps(struct mmu_gather *tlb, struct vm_area_struct *vma);281#endif282 283#endif284 285/*286 * We have a no-op version of the rmap removal that doesn't287 * delay anything. That is used on S390, which flushes remote288 * TLBs synchronously, and on UP, which doesn't have any289 * remote TLBs to flush and is not preemptible due to this290 * all happening under the page table lock.291 */292#ifndef tlb_delay_rmap293#define tlb_delay_rmap(tlb) (false)294static inline void tlb_flush_rmaps(struct mmu_gather *tlb, struct vm_area_struct *vma) { }295#endif296 297/*298 * struct mmu_gather is an opaque type used by the mm code for passing around299 * any data needed by arch specific code for tlb_remove_page.300 */301struct mmu_gather {302 struct mm_struct *mm;303 304#ifdef CONFIG_MMU_GATHER_TABLE_FREE305 struct mmu_table_batch *batch;306#endif307 308 unsigned long start;309 unsigned long end;310 /*311 * we are in the middle of an operation to clear312 * a full mm and can make some optimizations313 */314 unsigned int fullmm : 1;315 316 /*317 * we have performed an operation which318 * requires a complete flush of the tlb319 */320 unsigned int need_flush_all : 1;321 322 /*323 * we have removed page directories324 */325 unsigned int freed_tables : 1;326 327 /*328 * Do we have pending delayed rmap removals?329 */330 unsigned int delayed_rmap : 1;331 332 /*333 * at which levels have we cleared entries?334 */335 unsigned int cleared_ptes : 1;336 unsigned int cleared_pmds : 1;337 unsigned int cleared_puds : 1;338 unsigned int cleared_p4ds : 1;339 340 /*341 * tracks VM_EXEC | VM_HUGETLB in tlb_start_vma342 */343 unsigned int vma_exec : 1;344 unsigned int vma_huge : 1;345 unsigned int vma_pfn : 1;346 347 unsigned int batch_count;348 349#ifndef CONFIG_MMU_GATHER_NO_GATHER350 struct mmu_gather_batch *active;351 struct mmu_gather_batch local;352 struct page *__pages[MMU_GATHER_BUNDLE];353 354#ifdef CONFIG_MMU_GATHER_PAGE_SIZE355 unsigned int page_size;356#endif357#endif358};359 360void tlb_flush_mmu(struct mmu_gather *tlb) HWJS_SUSPENDS;361 362static inline void __tlb_adjust_range(struct mmu_gather *tlb,363 unsigned long address,364 unsigned int range_size)365{366 tlb->start = min(tlb->start, address);367 tlb->end = max(tlb->end, address + range_size);368}369 370static inline void __tlb_reset_range(struct mmu_gather *tlb)371{372 if (tlb->fullmm) {373 tlb->start = tlb->end = ~0;374 } else {375 tlb->start = TASK_SIZE;376 tlb->end = 0;377 }378 tlb->freed_tables = 0;379 tlb->cleared_ptes = 0;380 tlb->cleared_pmds = 0;381 tlb->cleared_puds = 0;382 tlb->cleared_p4ds = 0;383 /*384 * Do not reset mmu_gather::vma_* fields here, we do not385 * call into tlb_start_vma() again to set them if there is an386 * intermediate flush.387 */388}389 390#ifdef CONFIG_MMU_GATHER_NO_RANGE391 392#if defined(tlb_flush)393#error MMU_GATHER_NO_RANGE relies on default tlb_flush()394#endif395 396/*397 * When an architecture does not have efficient means of range flushing TLBs398 * there is no point in doing intermediate flushes on tlb_end_vma() to keep the399 * range small. We equally don't have to worry about page granularity or other400 * things.401 *402 * All we need to do is issue a full flush for any !0 range.403 */404static inline void tlb_flush(struct mmu_gather *tlb)405{406 if (tlb->end)407 flush_tlb_mm(tlb->mm);408}409 410#else /* CONFIG_MMU_GATHER_NO_RANGE */411 412#ifndef tlb_flush413/*414 * When an architecture does not provide its own tlb_flush() implementation415 * but does have a reasonably efficient flush_vma_range() implementation416 * use that.417 */418static inline void tlb_flush(struct mmu_gather *tlb)419{420 if (tlb->fullmm || tlb->need_flush_all) {421 flush_tlb_mm(tlb->mm);422 } else if (tlb->end) {423 struct vm_area_struct vma = {424 .vm_mm = tlb->mm,425 .vm_flags = (tlb->vma_exec ? VM_EXEC : 0) |426 (tlb->vma_huge ? VM_HUGETLB : 0),427 };428 429 flush_tlb_range(&vma, tlb->start, tlb->end);430 }431}432#endif433 434#endif /* CONFIG_MMU_GATHER_NO_RANGE */435 436static inline void437tlb_update_vma_flags(struct mmu_gather *tlb, struct vm_area_struct *vma)438{439 /*440 * flush_tlb_range() implementations that look at VM_HUGETLB (tile,441 * mips-4k) flush only large pages.442 *443 * flush_tlb_range() implementations that flush I-TLB also flush D-TLB444 * (tile, xtensa, arm), so it's ok to just add VM_EXEC to an existing445 * range.446 *447 * We rely on tlb_end_vma() to issue a flush, such that when we reset448 * these values the batch is empty.449 */450 tlb->vma_huge = is_vm_hugetlb_page(vma);451 tlb->vma_exec = !!(vma->vm_flags & VM_EXEC);452 tlb->vma_pfn = !!(vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP));453}454 455static inline void tlb_flush_mmu_tlbonly(struct mmu_gather *tlb)456{457 /*458 * Anything calling __tlb_adjust_range() also sets at least one of459 * these bits.460 */461 if (!(tlb->freed_tables || tlb->cleared_ptes || tlb->cleared_pmds ||462 tlb->cleared_puds || tlb->cleared_p4ds))463 return;464 465 tlb_flush(tlb);466 __tlb_reset_range(tlb);467}468 469static inline void tlb_remove_page_size(struct mmu_gather *tlb,470 struct page *page, int page_size)471{472 if (__tlb_remove_page_size(tlb, page, false, page_size))473 tlb_flush_mmu(tlb);474}475 476static __always_inline bool __tlb_remove_page(struct mmu_gather *tlb,477 struct page *page, bool delay_rmap)478{479 return __tlb_remove_page_size(tlb, page, delay_rmap, PAGE_SIZE);480}481 482/* tlb_remove_page483 * Similar to __tlb_remove_page but will call tlb_flush_mmu() itself when484 * required.485 */486static inline void tlb_remove_page(struct mmu_gather *tlb, struct page *page)487{488 return tlb_remove_page_size(tlb, page, PAGE_SIZE);489}490 491static inline void tlb_remove_ptdesc(struct mmu_gather *tlb, void *pt)492{493 tlb_remove_table(tlb, pt);494}495 496/* Like tlb_remove_ptdesc, but for page-like page directories. */497static inline void tlb_remove_page_ptdesc(struct mmu_gather *tlb, struct ptdesc *pt)498{499 tlb_remove_page(tlb, ptdesc_page(pt));500}501 502static inline void tlb_change_page_size(struct mmu_gather *tlb,503 unsigned int page_size)504{505#ifdef CONFIG_MMU_GATHER_PAGE_SIZE506 if (tlb->page_size && tlb->page_size != page_size) {507 if (!tlb->fullmm && !tlb->need_flush_all)508 tlb_flush_mmu(tlb);509 }510 511 tlb->page_size = page_size;512#endif513}514 515static inline unsigned long tlb_get_unmap_shift(struct mmu_gather *tlb)516{517 if (tlb->cleared_ptes)518 return PAGE_SHIFT;519 if (tlb->cleared_pmds)520 return PMD_SHIFT;521 if (tlb->cleared_puds)522 return PUD_SHIFT;523 if (tlb->cleared_p4ds)524 return P4D_SHIFT;525 526 return PAGE_SHIFT;527}528 529static inline unsigned long tlb_get_unmap_size(struct mmu_gather *tlb)530{531 return 1UL << tlb_get_unmap_shift(tlb);532}533 534/*535 * In the case of tlb vma handling, we can optimise these away in the536 * case where we're doing a full MM flush. When we're doing a munmap,537 * the vmas are adjusted to only cover the region to be torn down.538 */539static inline void tlb_start_vma(struct mmu_gather *tlb, struct vm_area_struct *vma)540{541 if (tlb->fullmm)542 return;543 544 tlb_update_vma_flags(tlb, vma);545#ifndef CONFIG_MMU_GATHER_NO_FLUSH_CACHE546 flush_cache_range(vma, vma->vm_start, vma->vm_end);547#endif548}549 550static inline void tlb_end_vma(struct mmu_gather *tlb, struct vm_area_struct *vma)551{552 if (tlb->fullmm)553 return;554 555 /*556 * VM_PFNMAP is more fragile because the core mm will not track the557 * page mapcount -- there might not be page-frames for these PFNs after558 * all. Force flush TLBs for such ranges to avoid munmap() vs559 * unmap_mapping_range() races.560 */561 if (tlb->vma_pfn || !IS_ENABLED(CONFIG_MMU_GATHER_MERGE_VMAS)) {562 /*563 * Do a TLB flush and reset the range at VMA boundaries; this avoids564 * the ranges growing with the unused space between consecutive VMAs.565 */566 tlb_flush_mmu_tlbonly(tlb);567 }568}569 570/*571 * tlb_flush_{pte|pmd|pud|p4d}_range() adjust the tlb->start and tlb->end,572 * and set corresponding cleared_*.573 */574static inline void tlb_flush_pte_range(struct mmu_gather *tlb,575 unsigned long address, unsigned long size)576{577 __tlb_adjust_range(tlb, address, size);578 tlb->cleared_ptes = 1;579}580 581static inline void tlb_flush_pmd_range(struct mmu_gather *tlb,582 unsigned long address, unsigned long size)583{584 __tlb_adjust_range(tlb, address, size);585 tlb->cleared_pmds = 1;586}587 588static inline void tlb_flush_pud_range(struct mmu_gather *tlb,589 unsigned long address, unsigned long size)590{591 __tlb_adjust_range(tlb, address, size);592 tlb->cleared_puds = 1;593}594 595static inline void tlb_flush_p4d_range(struct mmu_gather *tlb,596 unsigned long address, unsigned long size)597{598 __tlb_adjust_range(tlb, address, size);599 tlb->cleared_p4ds = 1;600}601 602#ifndef __tlb_remove_tlb_entry603static inline void __tlb_remove_tlb_entry(struct mmu_gather *tlb, pte_t *ptep, unsigned long address)604{605}606#endif607 608/**609 * tlb_remove_tlb_entry - remember a pte unmapping for later tlb invalidation.610 *611 * Record the fact that pte's were really unmapped by updating the range,612 * so we can later optimise away the tlb invalidate. This helps when613 * userspace is unmapping already-unmapped pages, which happens quite a lot.614 */615#define tlb_remove_tlb_entry(tlb, ptep, address) \616 do { \617 tlb_flush_pte_range(tlb, address, PAGE_SIZE); \618 __tlb_remove_tlb_entry(tlb, ptep, address); \619 } while (0)620 621/**622 * tlb_remove_tlb_entries - remember unmapping of multiple consecutive ptes for623 * later tlb invalidation.624 *625 * Similar to tlb_remove_tlb_entry(), but remember unmapping of multiple626 * consecutive ptes instead of only a single one.627 */628static inline void tlb_remove_tlb_entries(struct mmu_gather *tlb,629 pte_t *ptep, unsigned int nr, unsigned long address)630{631 tlb_flush_pte_range(tlb, address, PAGE_SIZE * nr);632 for (;;) {633 __tlb_remove_tlb_entry(tlb, ptep, address);634 if (--nr == 0)635 break;636 ptep++;637 address += PAGE_SIZE;638 }639}640 641#define tlb_remove_huge_tlb_entry(h, tlb, ptep, address) \642 do { \643 unsigned long _sz = huge_page_size(h); \644 if (_sz >= P4D_SIZE) \645 tlb_flush_p4d_range(tlb, address, _sz); \646 else if (_sz >= PUD_SIZE) \647 tlb_flush_pud_range(tlb, address, _sz); \648 else if (_sz >= PMD_SIZE) \649 tlb_flush_pmd_range(tlb, address, _sz); \650 else \651 tlb_flush_pte_range(tlb, address, _sz); \652 __tlb_remove_tlb_entry(tlb, ptep, address); \653 } while (0)654 655/**656 * tlb_remove_pmd_tlb_entry - remember a pmd mapping for later tlb invalidation657 * This is a nop so far, because only x86 needs it.658 */659#ifndef __tlb_remove_pmd_tlb_entry660#define __tlb_remove_pmd_tlb_entry(tlb, pmdp, address) do {} while (0)661#endif662 663#define tlb_remove_pmd_tlb_entry(tlb, pmdp, address) \664 do { \665 tlb_flush_pmd_range(tlb, address, HPAGE_PMD_SIZE); \666 __tlb_remove_pmd_tlb_entry(tlb, pmdp, address); \667 } while (0)668 669/**670 * tlb_remove_pud_tlb_entry - remember a pud mapping for later tlb671 * invalidation. This is a nop so far, because only x86 needs it.672 */673#ifndef __tlb_remove_pud_tlb_entry674#define __tlb_remove_pud_tlb_entry(tlb, pudp, address) do {} while (0)675#endif676 677#define tlb_remove_pud_tlb_entry(tlb, pudp, address) \678 do { \679 tlb_flush_pud_range(tlb, address, HPAGE_PUD_SIZE); \680 __tlb_remove_pud_tlb_entry(tlb, pudp, address); \681 } while (0)682 683/*684 * For things like page tables caches (ie caching addresses "inside" the685 * page tables, like x86 does), for legacy reasons, flushing an686 * individual page had better flush the page table caches behind it. This687 * is definitely how x86 works, for example. And if you have an688 * architected non-legacy page table cache (which I'm not aware of689 * anybody actually doing), you're going to have some architecturally690 * explicit flushing for that, likely *separate* from a regular TLB entry691 * flush, and thus you'd need more than just some range expansion..692 *693 * So if we ever find an architecture694 * that would want something that odd, I think it is up to that695 * architecture to do its own odd thing, not cause pain for others696 * http://lkml.kernel.org/r/CA+55aFzBggoXtNXQeng5d_mRoDnaMBE5Y+URs+PHR67nUpMtaw@mail.gmail.com697 *698 * For now w.r.t page table cache, mark the range_size as PAGE_SIZE699 */700 701#ifndef pte_free_tlb702#define pte_free_tlb(tlb, ptep, address) \703 do { \704 tlb_flush_pmd_range(tlb, address, PAGE_SIZE); \705 tlb->freed_tables = 1; \706 __pte_free_tlb(tlb, ptep, address); \707 } while (0)708#endif709 710#ifndef pmd_free_tlb711#define pmd_free_tlb(tlb, pmdp, address) \712 do { \713 tlb_flush_pud_range(tlb, address, PAGE_SIZE); \714 tlb->freed_tables = 1; \715 __pmd_free_tlb(tlb, pmdp, address); \716 } while (0)717#endif718 719#ifndef pud_free_tlb720#define pud_free_tlb(tlb, pudp, address) \721 do { \722 tlb_flush_p4d_range(tlb, address, PAGE_SIZE); \723 tlb->freed_tables = 1; \724 __pud_free_tlb(tlb, pudp, address); \725 } while (0)726#endif727 728#ifndef p4d_free_tlb729#define p4d_free_tlb(tlb, pudp, address) \730 do { \731 __tlb_adjust_range(tlb, address, PAGE_SIZE); \732 tlb->freed_tables = 1; \733 __p4d_free_tlb(tlb, pudp, address); \734 } while (0)735#endif736 737#ifndef pte_needs_flush738static inline bool pte_needs_flush(pte_t oldpte, pte_t newpte)739{740 return true;741}742#endif743 744#ifndef huge_pmd_needs_flush745static inline bool huge_pmd_needs_flush(pmd_t oldpmd, pmd_t newpmd)746{747 return true;748}749#endif750 751#endif /* CONFIG_MMU */752 753#endif /* _ASM_GENERIC__TLB_H */754