brintos / linux-shallow public Read only

1// SPDX-License-Identifier: GPL-2.0-only OR MIT2/* Copyright (c) 2023 Imagination Technologies Ltd. */3 4#include "pvr_mmu.h"5 6#include "pvr_ccb.h"7#include "pvr_device.h"8#include "pvr_fw.h"9#include "pvr_gem.h"10#include "pvr_power.h"11#include "pvr_rogue_fwif.h"12#include "pvr_rogue_mmu_defs.h"13 14#include <drm/drm_drv.h>15#include <linux/atomic.h>16#include <linux/bitops.h>17#include <linux/dma-mapping.h>18#include <linux/kmemleak.h>19#include <linux/minmax.h>20#include <linux/sizes.h>21 22#define PVR_SHIFT_FROM_SIZE(size_) (__builtin_ctzll(size_))23#define PVR_MASK_FROM_SIZE(size_) (~((size_) - U64_C(1)))24 25/*26 * The value of the device page size (%PVR_DEVICE_PAGE_SIZE) is currently27 * pegged to the host page size (%PAGE_SIZE). This chunk of macro goodness both28 * ensures that the selected host page size corresponds to a valid device page29 * size and sets up values needed by the MMU code below.30 */31#if (PVR_DEVICE_PAGE_SIZE == SZ_4K)32# define ROGUE_MMUCTRL_PAGE_SIZE_X ROGUE_MMUCTRL_PAGE_SIZE_4KB33# define ROGUE_MMUCTRL_PAGE_X_RANGE_SHIFT ROGUE_MMUCTRL_PAGE_4KB_RANGE_SHIFT34# define ROGUE_MMUCTRL_PAGE_X_RANGE_CLRMSK ROGUE_MMUCTRL_PAGE_4KB_RANGE_CLRMSK35#elif (PVR_DEVICE_PAGE_SIZE == SZ_16K)36# define ROGUE_MMUCTRL_PAGE_SIZE_X ROGUE_MMUCTRL_PAGE_SIZE_16KB37# define ROGUE_MMUCTRL_PAGE_X_RANGE_SHIFT ROGUE_MMUCTRL_PAGE_16KB_RANGE_SHIFT38# define ROGUE_MMUCTRL_PAGE_X_RANGE_CLRMSK ROGUE_MMUCTRL_PAGE_16KB_RANGE_CLRMSK39#elif (PVR_DEVICE_PAGE_SIZE == SZ_64K)40# define ROGUE_MMUCTRL_PAGE_SIZE_X ROGUE_MMUCTRL_PAGE_SIZE_64KB41# define ROGUE_MMUCTRL_PAGE_X_RANGE_SHIFT ROGUE_MMUCTRL_PAGE_64KB_RANGE_SHIFT42# define ROGUE_MMUCTRL_PAGE_X_RANGE_CLRMSK ROGUE_MMUCTRL_PAGE_64KB_RANGE_CLRMSK43#elif (PVR_DEVICE_PAGE_SIZE == SZ_256K)44# define ROGUE_MMUCTRL_PAGE_SIZE_X ROGUE_MMUCTRL_PAGE_SIZE_256KB45# define ROGUE_MMUCTRL_PAGE_X_RANGE_SHIFT ROGUE_MMUCTRL_PAGE_256KB_RANGE_SHIFT46# define ROGUE_MMUCTRL_PAGE_X_RANGE_CLRMSK ROGUE_MMUCTRL_PAGE_256KB_RANGE_CLRMSK47#elif (PVR_DEVICE_PAGE_SIZE == SZ_1M)48# define ROGUE_MMUCTRL_PAGE_SIZE_X ROGUE_MMUCTRL_PAGE_SIZE_1MB49# define ROGUE_MMUCTRL_PAGE_X_RANGE_SHIFT ROGUE_MMUCTRL_PAGE_1MB_RANGE_SHIFT50# define ROGUE_MMUCTRL_PAGE_X_RANGE_CLRMSK ROGUE_MMUCTRL_PAGE_1MB_RANGE_CLRMSK51#elif (PVR_DEVICE_PAGE_SIZE == SZ_2M)52# define ROGUE_MMUCTRL_PAGE_SIZE_X ROGUE_MMUCTRL_PAGE_SIZE_2MB53# define ROGUE_MMUCTRL_PAGE_X_RANGE_SHIFT ROGUE_MMUCTRL_PAGE_2MB_RANGE_SHIFT54# define ROGUE_MMUCTRL_PAGE_X_RANGE_CLRMSK ROGUE_MMUCTRL_PAGE_2MB_RANGE_CLRMSK55#else56# error Unsupported device page size PVR_DEVICE_PAGE_SIZE57#endif58 59#define ROGUE_MMUCTRL_ENTRIES_PT_VALUE_X   \60	(ROGUE_MMUCTRL_ENTRIES_PT_VALUE >> \61	 (PVR_DEVICE_PAGE_SHIFT - PVR_SHIFT_FROM_SIZE(SZ_4K)))62 63enum pvr_mmu_sync_level {64	PVR_MMU_SYNC_LEVEL_NONE = -1,65	PVR_MMU_SYNC_LEVEL_0 = 0,66	PVR_MMU_SYNC_LEVEL_1 = 1,67	PVR_MMU_SYNC_LEVEL_2 = 2,68};69 70#define PVR_MMU_SYNC_LEVEL_0_FLAGS (ROGUE_FWIF_MMUCACHEDATA_FLAGS_PT | \71				    ROGUE_FWIF_MMUCACHEDATA_FLAGS_INTERRUPT | \72				    ROGUE_FWIF_MMUCACHEDATA_FLAGS_TLB)73#define PVR_MMU_SYNC_LEVEL_1_FLAGS (PVR_MMU_SYNC_LEVEL_0_FLAGS | ROGUE_FWIF_MMUCACHEDATA_FLAGS_PD)74#define PVR_MMU_SYNC_LEVEL_2_FLAGS (PVR_MMU_SYNC_LEVEL_1_FLAGS | ROGUE_FWIF_MMUCACHEDATA_FLAGS_PC)75 76/**77 * pvr_mmu_set_flush_flags() - Set MMU cache flush flags for next call to78 *                             pvr_mmu_flush_exec().79 * @pvr_dev: Target PowerVR device.80 * @flags: MMU flush flags. Must be one of %PVR_MMU_SYNC_LEVEL_*_FLAGS.81 *82 * This function must be called following any possible change to the MMU page83 * tables.84 */85static void pvr_mmu_set_flush_flags(struct pvr_device *pvr_dev, u32 flags)86{87	atomic_fetch_or(flags, &pvr_dev->mmu_flush_cache_flags);88}89 90/**91 * pvr_mmu_flush_request_all() - Request flush of all MMU caches when92 * subsequently calling pvr_mmu_flush_exec().93 * @pvr_dev: Target PowerVR device.94 *95 * This function must be called following any possible change to the MMU page96 * tables.97 */98void pvr_mmu_flush_request_all(struct pvr_device *pvr_dev)99{100	pvr_mmu_set_flush_flags(pvr_dev, PVR_MMU_SYNC_LEVEL_2_FLAGS);101}102 103/**104 * pvr_mmu_flush_exec() - Execute a flush of all MMU caches previously105 * requested.106 * @pvr_dev: Target PowerVR device.107 * @wait: Do not return until the flush is completed.108 *109 * This function must be called prior to submitting any new GPU job. The flush110 * will complete before the jobs are scheduled, so this can be called once after111 * a series of maps. However, a single unmap should always be immediately112 * followed by a flush and it should be explicitly waited by setting @wait.113 *114 * As a failure to flush the MMU caches could risk memory corruption, if the115 * flush fails (implying the firmware is not responding) then the GPU device is116 * marked as lost.117 *118 * Returns:119 *  * 0 on success when @wait is true, or120 *  * -%EIO if the device is unavailable, or121 *  * Any error encountered while submitting the flush command via the KCCB.122 */123int pvr_mmu_flush_exec(struct pvr_device *pvr_dev, bool wait)124{125	struct rogue_fwif_kccb_cmd cmd_mmu_cache = {};126	struct rogue_fwif_mmucachedata *cmd_mmu_cache_data =127		&cmd_mmu_cache.cmd_data.mmu_cache_data;128	int err = 0;129	u32 slot;130	int idx;131 132	if (!drm_dev_enter(from_pvr_device(pvr_dev), &idx))133		return -EIO;134 135	/* Can't flush MMU if the firmware hasn't booted yet. */136	if (!pvr_dev->fw_dev.booted)137		goto err_drm_dev_exit;138 139	cmd_mmu_cache_data->cache_flags =140		atomic_xchg(&pvr_dev->mmu_flush_cache_flags, 0);141 142	if (!cmd_mmu_cache_data->cache_flags)143		goto err_drm_dev_exit;144 145	cmd_mmu_cache.cmd_type = ROGUE_FWIF_KCCB_CMD_MMUCACHE;146 147	pvr_fw_object_get_fw_addr(pvr_dev->fw_dev.mem.mmucache_sync_obj,148				  &cmd_mmu_cache_data->mmu_cache_sync_fw_addr);149	cmd_mmu_cache_data->mmu_cache_sync_update_value = 0;150 151	err = pvr_kccb_send_cmd(pvr_dev, &cmd_mmu_cache, &slot);152	if (err)153		goto err_reset_and_retry;154 155	err = pvr_kccb_wait_for_completion(pvr_dev, slot, HZ, NULL);156	if (err)157		goto err_reset_and_retry;158 159	drm_dev_exit(idx);160 161	return 0;162 163err_reset_and_retry:164	/*165	 * Flush command failure is most likely the result of a firmware lockup. Hard166	 * reset the GPU and retry.167	 */168	err = pvr_power_reset(pvr_dev, true);169	if (err)170		goto err_drm_dev_exit; /* Device is lost. */171 172	/* Retry sending flush request. */173	err = pvr_kccb_send_cmd(pvr_dev, &cmd_mmu_cache, &slot);174	if (err) {175		pvr_device_lost(pvr_dev);176		goto err_drm_dev_exit;177	}178 179	if (wait) {180		err = pvr_kccb_wait_for_completion(pvr_dev, slot, HZ, NULL);181		if (err)182			pvr_device_lost(pvr_dev);183	}184 185err_drm_dev_exit:186	drm_dev_exit(idx);187 188	return err;189}190 191/**192 * DOC: PowerVR Virtual Memory Handling193 */194/**195 * DOC: PowerVR Virtual Memory Handling (constants)196 *197 * .. c:macro:: PVR_IDX_INVALID198 *199 *    Default value for a u16-based index.200 *201 *    This value cannot be zero, since zero is a valid index value.202 */203#define PVR_IDX_INVALID ((u16)(-1))204 205/**206 * DOC: MMU backing pages207 */208/**209 * DOC: MMU backing pages (constants)210 *211 * .. c:macro:: PVR_MMU_BACKING_PAGE_SIZE212 *213 *    Page size of a PowerVR device's integrated MMU. The CPU page size must be214 *    at least as large as this value for the current implementation; this is215 *    checked at compile-time.216 */217#define PVR_MMU_BACKING_PAGE_SIZE SZ_4K218static_assert(PAGE_SIZE >= PVR_MMU_BACKING_PAGE_SIZE);219 220/**221 * struct pvr_mmu_backing_page - Represents a single page used to back a page222 *                              table of any level.223 * @dma_addr: DMA address of this page.224 * @host_ptr: CPU address of this page.225 * @pvr_dev: The PowerVR device to which this page is associated. **For226 *           internal use only.**227 */228struct pvr_mmu_backing_page {229	dma_addr_t dma_addr;230	void *host_ptr;231/* private: internal use only */232	struct page *raw_page;233	struct pvr_device *pvr_dev;234};235 236/**237 * pvr_mmu_backing_page_init() - Initialize a MMU backing page.238 * @page: Target backing page.239 * @pvr_dev: Target PowerVR device.240 *241 * This function performs three distinct operations:242 *243 * 1. Allocate a single page,244 * 2. Map the page to the CPU, and245 * 3. Map the page to DMA-space.246 *247 * It is expected that @page be zeroed (e.g. from kzalloc()) before calling248 * this function.249 *250 * Return:251 *  * 0 on success, or252 *  * -%ENOMEM if allocation of the backing page or mapping of the backing253 *    page to DMA fails.254 */255static int256pvr_mmu_backing_page_init(struct pvr_mmu_backing_page *page,257			  struct pvr_device *pvr_dev)258{259	struct device *dev = from_pvr_device(pvr_dev)->dev;260 261	struct page *raw_page;262	int err;263 264	dma_addr_t dma_addr;265	void *host_ptr;266 267	raw_page = alloc_page(__GFP_ZERO | GFP_KERNEL);268	if (!raw_page)269		return -ENOMEM;270 271	host_ptr = vmap(&raw_page, 1, VM_MAP, pgprot_writecombine(PAGE_KERNEL));272	if (!host_ptr) {273		err = -ENOMEM;274		goto err_free_page;275	}276 277	dma_addr = dma_map_page(dev, raw_page, 0, PVR_MMU_BACKING_PAGE_SIZE,278				DMA_TO_DEVICE);279	if (dma_mapping_error(dev, dma_addr)) {280		err = -ENOMEM;281		goto err_unmap_page;282	}283 284	page->dma_addr = dma_addr;285	page->host_ptr = host_ptr;286	page->pvr_dev = pvr_dev;287	page->raw_page = raw_page;288	kmemleak_alloc(page->host_ptr, PAGE_SIZE, 1, GFP_KERNEL);289 290	return 0;291 292err_unmap_page:293	vunmap(host_ptr);294 295err_free_page:296	__free_page(raw_page);297 298	return err;299}300 301/**302 * pvr_mmu_backing_page_fini() - Teardown a MMU backing page.303 * @page: Target backing page.304 *305 * This function performs the mirror operations to pvr_mmu_backing_page_init(),306 * in reverse order:307 *308 * 1. Unmap the page from DMA-space,309 * 2. Unmap the page from the CPU, and310 * 3. Free the page.311 *312 * It also zeros @page.313 *314 * It is a no-op to call this function a second (or further) time on any @page.315 */316static void317pvr_mmu_backing_page_fini(struct pvr_mmu_backing_page *page)318{319	struct device *dev;320 321	/* Do nothing if no allocation is present. */322	if (!page->pvr_dev)323		return;324 325	dev = from_pvr_device(page->pvr_dev)->dev;326 327	dma_unmap_page(dev, page->dma_addr, PVR_MMU_BACKING_PAGE_SIZE,328		       DMA_TO_DEVICE);329 330	kmemleak_free(page->host_ptr);331	vunmap(page->host_ptr);332 333	__free_page(page->raw_page);334 335	memset(page, 0, sizeof(*page));336}337 338/**339 * pvr_mmu_backing_page_sync() - Flush a MMU backing page from the CPU to the340 * device.341 * @page: Target backing page.342 * @flags: MMU flush flags. Must be one of %PVR_MMU_SYNC_LEVEL_*_FLAGS.343 *344 * .. caution::345 *346 *    **This is potentially an expensive function call.** Only call347 *    pvr_mmu_backing_page_sync() once you're sure you have no more changes to348 *    make to the backing page in the immediate future.349 */350static void351pvr_mmu_backing_page_sync(struct pvr_mmu_backing_page *page, u32 flags)352{353	struct pvr_device *pvr_dev = page->pvr_dev;354	struct device *dev;355 356	/*357	 * Do nothing if no allocation is present. This may be the case if358	 * we are unmapping pages.359	 */360	if (!pvr_dev)361		return;362 363	dev = from_pvr_device(pvr_dev)->dev;364 365	dma_sync_single_for_device(dev, page->dma_addr,366				   PVR_MMU_BACKING_PAGE_SIZE, DMA_TO_DEVICE);367 368	pvr_mmu_set_flush_flags(pvr_dev, flags);369}370 371/**372 * DOC: Raw page tables373 */374 375#define PVR_PAGE_TABLE_TYPEOF_ENTRY(level_) \376	typeof_member(struct pvr_page_table_l##level_##_entry_raw, val)377 378#define PVR_PAGE_TABLE_FIELD_GET(level_, name_, field_, entry_)           \379	(((entry_).val &                                           \380	  ~ROGUE_MMUCTRL_##name_##_DATA_##field_##_CLRMSK) >> \381	 ROGUE_MMUCTRL_##name_##_DATA_##field_##_SHIFT)382 383#define PVR_PAGE_TABLE_FIELD_PREP(level_, name_, field_, val_)            \384	((((PVR_PAGE_TABLE_TYPEOF_ENTRY(level_))(val_))            \385	  << ROGUE_MMUCTRL_##name_##_DATA_##field_##_SHIFT) & \386	 ~ROGUE_MMUCTRL_##name_##_DATA_##field_##_CLRMSK)387 388/**389 * struct pvr_page_table_l2_entry_raw - A single entry in a level 2 page table.390 * @val: The raw value of this entry.391 *392 * This type is a structure for type-checking purposes. At compile-time, its393 * size is checked against %ROGUE_MMUCTRL_ENTRY_SIZE_PC_VALUE.394 *395 * The value stored in this structure can be decoded using the following bitmap:396 *397 * .. flat-table::398 *    :widths: 1 5399 *    :stub-columns: 1400 *401 *    * - 31..4402 *      - **Level 1 Page Table Base Address:** Bits 39..12 of the L1403 *        page table base address, which is 4KiB aligned.404 *405 *    * - 3..2406 *      - *(reserved)*407 *408 *    * - 1409 *      - **Pending:** When valid bit is not set, indicates that a valid410 *        entry is pending and the MMU should wait for the driver to map411 *        the entry. This is used to support page demand mapping of412 *        memory.413 *414 *    * - 0415 *      - **Valid:** Indicates that the entry contains a valid L1 page416 *        table. If the valid bit is not set, then an attempted use of417 *        the page would result in a page fault.418 */419struct pvr_page_table_l2_entry_raw {420	u32 val;421} __packed;422static_assert(sizeof(struct pvr_page_table_l2_entry_raw) * 8 ==423	      ROGUE_MMUCTRL_ENTRY_SIZE_PC_VALUE);424 425static bool426pvr_page_table_l2_entry_raw_is_valid(struct pvr_page_table_l2_entry_raw entry)427{428	return PVR_PAGE_TABLE_FIELD_GET(2, PC, VALID, entry);429}430 431/**432 * pvr_page_table_l2_entry_raw_set() - Write a valid entry into a raw level 2433 *                                     page table.434 * @entry: Target raw level 2 page table entry.435 * @child_table_dma_addr: DMA address of the level 1 page table to be436 *                        associated with @entry.437 *438 * When calling this function, @child_table_dma_addr must be a valid DMA439 * address and a multiple of %ROGUE_MMUCTRL_PC_DATA_PD_BASE_ALIGNSIZE.440 */441static void442pvr_page_table_l2_entry_raw_set(struct pvr_page_table_l2_entry_raw *entry,443				dma_addr_t child_table_dma_addr)444{445	child_table_dma_addr >>= ROGUE_MMUCTRL_PC_DATA_PD_BASE_ALIGNSHIFT;446 447	WRITE_ONCE(entry->val,448		   PVR_PAGE_TABLE_FIELD_PREP(2, PC, VALID, true) |449		   PVR_PAGE_TABLE_FIELD_PREP(2, PC, ENTRY_PENDING, false) |450		   PVR_PAGE_TABLE_FIELD_PREP(2, PC, PD_BASE, child_table_dma_addr));451}452 453static void454pvr_page_table_l2_entry_raw_clear(struct pvr_page_table_l2_entry_raw *entry)455{456	WRITE_ONCE(entry->val, 0);457}458 459/**460 * struct pvr_page_table_l1_entry_raw - A single entry in a level 1 page table.461 * @val: The raw value of this entry.462 *463 * This type is a structure for type-checking purposes. At compile-time, its464 * size is checked against %ROGUE_MMUCTRL_ENTRY_SIZE_PD_VALUE.465 *466 * The value stored in this structure can be decoded using the following bitmap:467 *468 * .. flat-table::469 *    :widths: 1 5470 *    :stub-columns: 1471 *472 *    * - 63..41473 *      - *(reserved)*474 *475 *    * - 40476 *      - **Pending:** When valid bit is not set, indicates that a valid entry477 *        is pending and the MMU should wait for the driver to map the entry.478 *        This is used to support page demand mapping of memory.479 *480 *    * - 39..5481 *      - **Level 0 Page Table Base Address:** The way this value is482 *        interpreted depends on the page size. Bits not specified in the483 *        table below (e.g. bits 11..5 for page size 4KiB) should be484 *        considered reserved.485 *486 *        This table shows the bits used in an L1 page table entry to487 *        represent the Physical Table Base Address for a given Page Size.488 *        Since each L1 page table entry covers 2MiB of address space, the489 *        maximum page size is 2MiB.490 *491 *        .. flat-table::492 *           :widths: 1 1 1 1493 *           :header-rows: 1494 *           :stub-columns: 1495 *496 *           * - Page size497 *             - L0 page table base address bits498 *             - Number of L0 page table entries499 *             - Size of L0 page table500 *501 *           * - 4KiB502 *             - 39..12503 *             - 512504 *             - 4KiB505 *506 *           * - 16KiB507 *             - 39..10508 *             - 128509 *             - 1KiB510 *511 *           * - 64KiB512 *             - 39..8513 *             - 32514 *             - 256B515 *516 *           * - 256KiB517 *             - 39..6518 *             - 8519 *             - 64B520 *521 *           * - 1MiB522 *             - 39..5 (4 = '0')523 *             - 2524 *             - 16B525 *526 *           * - 2MiB527 *             - 39..5 (4..3 = '00')528 *             - 1529 *             - 8B530 *531 *    * - 4532 *      - *(reserved)*533 *534 *    * - 3..1535 *      - **Page Size:** Sets the page size, from 4KiB to 2MiB.536 *537 *    * - 0538 *      - **Valid:** Indicates that the entry contains a valid L0 page table.539 *        If the valid bit is not set, then an attempted use of the page would540 *        result in a page fault.541 */542struct pvr_page_table_l1_entry_raw {543	u64 val;544} __packed;545static_assert(sizeof(struct pvr_page_table_l1_entry_raw) * 8 ==546	      ROGUE_MMUCTRL_ENTRY_SIZE_PD_VALUE);547 548static bool549pvr_page_table_l1_entry_raw_is_valid(struct pvr_page_table_l1_entry_raw entry)550{551	return PVR_PAGE_TABLE_FIELD_GET(1, PD, VALID, entry);552}553 554/**555 * pvr_page_table_l1_entry_raw_set() - Write a valid entry into a raw level 1556 *                                     page table.557 * @entry: Target raw level 1 page table entry.558 * @child_table_dma_addr: DMA address of the level 0 page table to be559 *                        associated with @entry.560 *561 * When calling this function, @child_table_dma_addr must be a valid DMA562 * address and a multiple of 4 KiB.563 */564static void565pvr_page_table_l1_entry_raw_set(struct pvr_page_table_l1_entry_raw *entry,566				dma_addr_t child_table_dma_addr)567{568	WRITE_ONCE(entry->val,569		   PVR_PAGE_TABLE_FIELD_PREP(1, PD, VALID, true) |570		   PVR_PAGE_TABLE_FIELD_PREP(1, PD, ENTRY_PENDING, false) |571		   PVR_PAGE_TABLE_FIELD_PREP(1, PD, PAGE_SIZE, ROGUE_MMUCTRL_PAGE_SIZE_X) |572		   /*573		    * The use of a 4K-specific macro here is correct. It is574		    * a future optimization to allocate sub-host-page-sized575		    * blocks for individual tables, so the condition that any576		    * page table address is aligned to the size of the577		    * largest (a 4KB) table currently holds.578		    */579		   (child_table_dma_addr & ~ROGUE_MMUCTRL_PT_BASE_4KB_RANGE_CLRMSK));580}581 582static void583pvr_page_table_l1_entry_raw_clear(struct pvr_page_table_l1_entry_raw *entry)584{585	WRITE_ONCE(entry->val, 0);586}587 588/**589 * struct pvr_page_table_l0_entry_raw - A single entry in a level 0 page table.590 * @val: The raw value of this entry.591 *592 * This type is a structure for type-checking purposes. At compile-time, its593 * size is checked against %ROGUE_MMUCTRL_ENTRY_SIZE_PT_VALUE.594 *595 * The value stored in this structure can be decoded using the following bitmap:596 *597 * .. flat-table::598 *    :widths: 1 5599 *    :stub-columns: 1600 *601 *    * - 63602 *      - *(reserved)*603 *604 *    * - 62605 *      - **PM/FW Protect:** Indicates a protected region which only the606 *        Parameter Manager (PM) or firmware processor can write to.607 *608 *    * - 61..40609 *      - **VP Page (High):** Virtual-physical page used for Parameter Manager610 *        (PM) memory. This field is only used if the additional level of PB611 *        virtualization is enabled. The VP Page field is needed by the PM in612 *        order to correctly reconstitute the free lists after render613 *        completion. This (High) field holds bits 39..18 of the value; the614 *        Low field holds bits 17..12. Bits 11..0 are always zero because the615 *        value is always aligned to the 4KiB page size.616 *617 *    * - 39..12618 *      - **Physical Page Address:** The way this value is interpreted depends619 *        on the page size. Bits not specified in the table below (e.g. bits620 *        20..12 for page size 2MiB) should be considered reserved.621 *622 *        This table shows the bits used in an L0 page table entry to represent623 *        the Physical Page Address for a given page size (as defined in the624 *        associated L1 page table entry).625 *626 *        .. flat-table::627 *           :widths: 1 1628 *           :header-rows: 1629 *           :stub-columns: 1630 *631 *           * - Page size632 *             - Physical address bits633 *634 *           * - 4KiB635 *             - 39..12636 *637 *           * - 16KiB638 *             - 39..14639 *640 *           * - 64KiB641 *             - 39..16642 *643 *           * - 256KiB644 *             - 39..18645 *646 *           * - 1MiB647 *             - 39..20648 *649 *           * - 2MiB650 *             - 39..21651 *652 *    * - 11..6653 *      - **VP Page (Low):** Continuation of VP Page (High).654 *655 *    * - 5656 *      - **Pending:** When valid bit is not set, indicates that a valid entry657 *        is pending and the MMU should wait for the driver to map the entry.658 *        This is used to support page demand mapping of memory.659 *660 *    * - 4661 *      - **PM Src:** Set on Parameter Manager (PM) allocated page table662 *        entries when indicated by the PM. Note that this bit will only be set663 *        by the PM, not by the device driver.664 *665 *    * - 3666 *      - **SLC Bypass Control:** Specifies requests to this page should bypass667 *        the System Level Cache (SLC), if enabled in SLC configuration.668 *669 *    * - 2670 *      - **Cache Coherency:** Indicates that the page is coherent (i.e. it671 *        does not require a cache flush between operations on the CPU and the672 *        device).673 *674 *    * - 1675 *      - **Read Only:** If set, this bit indicates that the page is read only.676 *        An attempted write to this page would result in a write-protection677 *        fault.678 *679 *    * - 0680 *      - **Valid:** Indicates that the entry contains a valid page. If the681 *        valid bit is not set, then an attempted use of the page would result682 *        in a page fault.683 */684struct pvr_page_table_l0_entry_raw {685	u64 val;686} __packed;687static_assert(sizeof(struct pvr_page_table_l0_entry_raw) * 8 ==688	      ROGUE_MMUCTRL_ENTRY_SIZE_PT_VALUE);689 690/**691 * struct pvr_page_flags_raw - The configurable flags from a single entry in a692 *                             level 0 page table.693 * @val: The raw value of these flags. Since these are a strict subset of694 *       &struct pvr_page_table_l0_entry_raw; use that type for our member here.695 *696 * The flags stored in this type are: PM/FW Protect; SLC Bypass Control; Cache697 * Coherency, and Read Only (bits 62, 3, 2 and 1 respectively).698 *699 * This type should never be instantiated directly; instead use700 * pvr_page_flags_raw_create() to ensure only valid bits of @val are set.701 */702struct pvr_page_flags_raw {703	struct pvr_page_table_l0_entry_raw val;704} __packed;705static_assert(sizeof(struct pvr_page_flags_raw) ==706	      sizeof(struct pvr_page_table_l0_entry_raw));707 708static bool709pvr_page_table_l0_entry_raw_is_valid(struct pvr_page_table_l0_entry_raw entry)710{711	return PVR_PAGE_TABLE_FIELD_GET(0, PT, VALID, entry);712}713 714/**715 * pvr_page_table_l0_entry_raw_set() - Write a valid entry into a raw level 0716 *                                     page table.717 * @entry: Target raw level 0 page table entry.718 * @dma_addr: DMA address of the physical page to be associated with @entry.719 * @flags: Options to be set on @entry.720 *721 * When calling this function, @child_table_dma_addr must be a valid DMA722 * address and a multiple of %PVR_DEVICE_PAGE_SIZE.723 *724 * The @flags parameter is directly assigned into @entry. It is the callers725 * responsibility to ensure that only bits specified in726 * &struct pvr_page_flags_raw are set in @flags.727 */728static void729pvr_page_table_l0_entry_raw_set(struct pvr_page_table_l0_entry_raw *entry,730				dma_addr_t dma_addr,731				struct pvr_page_flags_raw flags)732{733	WRITE_ONCE(entry->val, PVR_PAGE_TABLE_FIELD_PREP(0, PT, VALID, true) |734			       PVR_PAGE_TABLE_FIELD_PREP(0, PT, ENTRY_PENDING, false) |735			       (dma_addr & ~ROGUE_MMUCTRL_PAGE_X_RANGE_CLRMSK) |736			       flags.val.val);737}738 739static void740pvr_page_table_l0_entry_raw_clear(struct pvr_page_table_l0_entry_raw *entry)741{742	WRITE_ONCE(entry->val, 0);743}744 745/**746 * pvr_page_flags_raw_create() - Initialize the flag bits of a raw level 0 page747 *                               table entry.748 * @read_only: This page is read-only (see: Read Only).749 * @cache_coherent: This page does not require cache flushes (see: Cache750 *                  Coherency).751 * @slc_bypass: This page bypasses the device cache (see: SLC Bypass Control).752 * @pm_fw_protect: This page is only for use by the firmware or Parameter753 *                 Manager (see PM/FW Protect).754 *755 * For more details on the use of these four options, see their respective756 * entries in the table under &struct pvr_page_table_l0_entry_raw.757 *758 * Return:759 * A new &struct pvr_page_flags_raw instance which can be passed directly to760 * pvr_page_table_l0_entry_raw_set() or pvr_page_table_l0_insert().761 */762static struct pvr_page_flags_raw763pvr_page_flags_raw_create(bool read_only, bool cache_coherent, bool slc_bypass,764			  bool pm_fw_protect)765{766	struct pvr_page_flags_raw flags;767 768	flags.val.val =769		PVR_PAGE_TABLE_FIELD_PREP(0, PT, READ_ONLY, read_only) |770		PVR_PAGE_TABLE_FIELD_PREP(0, PT, CC, cache_coherent) |771		PVR_PAGE_TABLE_FIELD_PREP(0, PT, SLC_BYPASS_CTRL, slc_bypass) |772		PVR_PAGE_TABLE_FIELD_PREP(0, PT, PM_META_PROTECT, pm_fw_protect);773 774	return flags;775}776 777/**778 * struct pvr_page_table_l2_raw - The raw data of a level 2 page table.779 *780 * This type is a structure for type-checking purposes. At compile-time, its781 * size is checked against %PVR_MMU_BACKING_PAGE_SIZE.782 */783struct pvr_page_table_l2_raw {784	/** @entries: The raw values of this table. */785	struct pvr_page_table_l2_entry_raw786		entries[ROGUE_MMUCTRL_ENTRIES_PC_VALUE];787} __packed;788static_assert(sizeof(struct pvr_page_table_l2_raw) == PVR_MMU_BACKING_PAGE_SIZE);789 790/**791 * struct pvr_page_table_l1_raw - The raw data of a level 1 page table.792 *793 * This type is a structure for type-checking purposes. At compile-time, its794 * size is checked against %PVR_MMU_BACKING_PAGE_SIZE.795 */796struct pvr_page_table_l1_raw {797	/** @entries: The raw values of this table. */798	struct pvr_page_table_l1_entry_raw799		entries[ROGUE_MMUCTRL_ENTRIES_PD_VALUE];800} __packed;801static_assert(sizeof(struct pvr_page_table_l1_raw) == PVR_MMU_BACKING_PAGE_SIZE);802 803/**804 * struct pvr_page_table_l0_raw - The raw data of a level 0 page table.805 *806 * This type is a structure for type-checking purposes. At compile-time, its807 * size is checked against %PVR_MMU_BACKING_PAGE_SIZE.808 *809 * .. caution::810 *811 *    The size of level 0 page tables is variable depending on the page size812 *    specified in the associated level 1 page table entry. Since the device813 *    page size in use is pegged to the host page size, it cannot vary at814 *    runtime. This structure is therefore only defined to contain the required815 *    number of entries for the current device page size. **You should never816 *    read or write beyond the last supported entry.**817 */818struct pvr_page_table_l0_raw {819	/** @entries: The raw values of this table. */820	struct pvr_page_table_l0_entry_raw821		entries[ROGUE_MMUCTRL_ENTRIES_PT_VALUE_X];822} __packed;823static_assert(sizeof(struct pvr_page_table_l0_raw) <= PVR_MMU_BACKING_PAGE_SIZE);824 825/**826 * DOC: Mirror page tables827 */828 829/*830 * We pre-declare these types because they cross-depend on pointers to each831 * other.832 */833struct pvr_page_table_l1;834struct pvr_page_table_l0;835 836/**837 * struct pvr_page_table_l2 - A wrapped level 2 page table.838 *839 * To access the raw part of this table, use pvr_page_table_l2_get_raw().840 * Alternatively to access a raw entry directly, use841 * pvr_page_table_l2_get_entry_raw().842 *843 * A level 2 page table forms the root of the page table tree structure, so844 * this type has no &parent or &parent_idx members.845 */846struct pvr_page_table_l2 {847	/**848	 * @entries: The children of this node in the page table tree849	 * structure. These are also mirror tables. The indexing of this array850	 * is identical to that of the raw equivalent851	 * (&pvr_page_table_l1_raw.entries).852	 */853	struct pvr_page_table_l1 *entries[ROGUE_MMUCTRL_ENTRIES_PC_VALUE];854 855	/**856	 * @backing_page: A handle to the memory which holds the raw857	 * equivalent of this table. **For internal use only.**858	 */859	struct pvr_mmu_backing_page backing_page;860 861	/**862	 * @entry_count: The current number of valid entries (that we know of)863	 * in this table. This value is essentially a refcount - the table is864	 * destroyed when this value is decremented to zero by865	 * pvr_page_table_l2_remove().866	 */867	u16 entry_count;868};869 870/**871 * pvr_page_table_l2_init() - Initialize a level 2 page table.872 * @table: Target level 2 page table.873 * @pvr_dev: Target PowerVR device874 *875 * It is expected that @table be zeroed (e.g. from kzalloc()) before calling876 * this function.877 *878 * Return:879 *  * 0 on success, or880 *  * Any error encountered while intializing &table->backing_page using881 *    pvr_mmu_backing_page_init().882 */883static int884pvr_page_table_l2_init(struct pvr_page_table_l2 *table,885		       struct pvr_device *pvr_dev)886{887	return pvr_mmu_backing_page_init(&table->backing_page, pvr_dev);888}889 890/**891 * pvr_page_table_l2_fini() - Teardown a level 2 page table.892 * @table: Target level 2 page table.893 *894 * It is an error to attempt to use @table after calling this function.895 */896static void897pvr_page_table_l2_fini(struct pvr_page_table_l2 *table)898{899	pvr_mmu_backing_page_fini(&table->backing_page);900}901 902/**903 * pvr_page_table_l2_sync() - Flush a level 2 page table from the CPU to the904 *                            device.905 * @table: Target level 2 page table.906 *907 * This is just a thin wrapper around pvr_mmu_backing_page_sync(), so the908 * warning there applies here too: **Only call pvr_page_table_l2_sync() once909 * you're sure you have no more changes to make to** @table **in the immediate910 * future.**911 *912 * If child level 1 page tables of @table also need to be flushed, this should913 * be done first using pvr_page_table_l1_sync() *before* calling this function.914 */915static void916pvr_page_table_l2_sync(struct pvr_page_table_l2 *table)917{918	pvr_mmu_backing_page_sync(&table->backing_page, PVR_MMU_SYNC_LEVEL_2_FLAGS);919}920 921/**922 * pvr_page_table_l2_get_raw() - Access the raw equivalent of a mirror level 2923 *                               page table.924 * @table: Target level 2 page table.925 *926 * Essentially returns the CPU address of the raw equivalent of @table, cast to927 * a &struct pvr_page_table_l2_raw pointer.928 *929 * You probably want to call pvr_page_table_l2_get_entry_raw() instead.930 *931 * Return:932 * The raw equivalent of @table.933 */934static struct pvr_page_table_l2_raw *935pvr_page_table_l2_get_raw(struct pvr_page_table_l2 *table)936{937	return table->backing_page.host_ptr;938}939 940/**941 * pvr_page_table_l2_get_entry_raw() - Access an entry from the raw equivalent942 *                                     of a mirror level 2 page table.943 * @table: Target level 2 page table.944 * @idx: Index of the entry to access.945 *946 * Technically this function returns a pointer to a slot in a raw level 2 page947 * table, since the returned "entry" is not guaranteed to be valid. The caller948 * must verify the validity of the entry at the returned address (perhaps using949 * pvr_page_table_l2_entry_raw_is_valid()) before reading or overwriting it.950 *951 * The value of @idx is not checked here; it is the callers responsibility to952 * ensure @idx refers to a valid index within @table before dereferencing the953 * returned pointer.954 *955 * Return:956 * A pointer to the requested raw level 2 page table entry.957 */958static struct pvr_page_table_l2_entry_raw *959pvr_page_table_l2_get_entry_raw(struct pvr_page_table_l2 *table, u16 idx)960{961	return &pvr_page_table_l2_get_raw(table)->entries[idx];962}963 964/**965 * pvr_page_table_l2_entry_is_valid() - Check if a level 2 page table entry is966 *                                      marked as valid.967 * @table: Target level 2 page table.968 * @idx: Index of the entry to check.969 *970 * The value of @idx is not checked here; it is the callers responsibility to971 * ensure @idx refers to a valid index within @table before calling this972 * function.973 */974static bool975pvr_page_table_l2_entry_is_valid(struct pvr_page_table_l2 *table, u16 idx)976{977	struct pvr_page_table_l2_entry_raw entry_raw =978		*pvr_page_table_l2_get_entry_raw(table, idx);979 980	return pvr_page_table_l2_entry_raw_is_valid(entry_raw);981}982 983/**984 * struct pvr_page_table_l1 - A wrapped level 1 page table.985 *986 * To access the raw part of this table, use pvr_page_table_l1_get_raw().987 * Alternatively to access a raw entry directly, use988 * pvr_page_table_l1_get_entry_raw().989 */990struct pvr_page_table_l1 {991	/**992	 * @entries: The children of this node in the page table tree993	 * structure. These are also mirror tables. The indexing of this array994	 * is identical to that of the raw equivalent995	 * (&pvr_page_table_l0_raw.entries).996	 */997	struct pvr_page_table_l0 *entries[ROGUE_MMUCTRL_ENTRIES_PD_VALUE];998 999	/**1000	 * @backing_page: A handle to the memory which holds the raw1001	 * equivalent of this table. **For internal use only.**1002	 */1003	struct pvr_mmu_backing_page backing_page;1004 1005	union {1006		/**1007		 * @parent: The parent of this node in the page table tree structure.1008		 *1009		 * This is also a mirror table.1010		 *1011		 * Only valid when the L1 page table is active. When the L1 page table1012		 * has been removed and queued for destruction, the next_free field1013		 * should be used instead.1014		 */1015		struct pvr_page_table_l2 *parent;1016 1017		/**1018		 * @next_free: Pointer to the next L1 page table to take/free.1019		 *1020		 * Used to form a linked list of L1 page tables. This is used1021		 * when preallocating tables and when the page table has been1022		 * removed and queued for destruction.1023		 */1024		struct pvr_page_table_l1 *next_free;1025	};1026 1027	/**1028	 * @parent_idx: The index of the entry in the parent table (see1029	 * @parent) which corresponds to this table.1030	 */1031	u16 parent_idx;1032 1033	/**1034	 * @entry_count: The current number of valid entries (that we know of)1035	 * in this table. This value is essentially a refcount - the table is1036	 * destroyed when this value is decremented to zero by1037	 * pvr_page_table_l1_remove().1038	 */1039	u16 entry_count;1040};1041 1042/**1043 * pvr_page_table_l1_init() - Initialize a level 1 page table.1044 * @table: Target level 1 page table.1045 * @pvr_dev: Target PowerVR device1046 *1047 * When this function returns successfully, @table is still not considered1048 * valid. It must be inserted into the page table tree structure with1049 * pvr_page_table_l2_insert() before it is ready for use.1050 *1051 * It is expected that @table be zeroed (e.g. from kzalloc()) before calling1052 * this function.1053 *1054 * Return:1055 *  * 0 on success, or1056 *  * Any error encountered while intializing &table->backing_page using1057 *    pvr_mmu_backing_page_init().1058 */1059static int1060pvr_page_table_l1_init(struct pvr_page_table_l1 *table,1061		       struct pvr_device *pvr_dev)1062{1063	table->parent_idx = PVR_IDX_INVALID;1064 1065	return pvr_mmu_backing_page_init(&table->backing_page, pvr_dev);1066}1067 1068/**1069 * pvr_page_table_l1_free() - Teardown a level 1 page table.1070 * @table: Target level 1 page table.1071 *1072 * It is an error to attempt to use @table after calling this function, even1073 * indirectly. This includes calling pvr_page_table_l2_remove(), which must1074 * be called *before* pvr_page_table_l1_free().1075 */1076static void1077pvr_page_table_l1_free(struct pvr_page_table_l1 *table)1078{1079	pvr_mmu_backing_page_fini(&table->backing_page);1080	kfree(table);1081}1082 1083/**1084 * pvr_page_table_l1_sync() - Flush a level 1 page table from the CPU to the1085 *                            device.1086 * @table: Target level 1 page table.1087 *1088 * This is just a thin wrapper around pvr_mmu_backing_page_sync(), so the1089 * warning there applies here too: **Only call pvr_page_table_l1_sync() once1090 * you're sure you have no more changes to make to** @table **in the immediate1091 * future.**1092 *1093 * If child level 0 page tables of @table also need to be flushed, this should1094 * be done first using pvr_page_table_l0_sync() *before* calling this function.1095 */1096static void1097pvr_page_table_l1_sync(struct pvr_page_table_l1 *table)1098{1099	pvr_mmu_backing_page_sync(&table->backing_page, PVR_MMU_SYNC_LEVEL_1_FLAGS);1100}1101 1102/**1103 * pvr_page_table_l1_get_raw() - Access the raw equivalent of a mirror level 11104 *                               page table.1105 * @table: Target level 1 page table.1106 *1107 * Essentially returns the CPU address of the raw equivalent of @table, cast to1108 * a &struct pvr_page_table_l1_raw pointer.1109 *1110 * You probably want to call pvr_page_table_l1_get_entry_raw() instead.1111 *1112 * Return:1113 * The raw equivalent of @table.1114 */1115static struct pvr_page_table_l1_raw *1116pvr_page_table_l1_get_raw(struct pvr_page_table_l1 *table)1117{1118	return table->backing_page.host_ptr;1119}1120 1121/**1122 * pvr_page_table_l1_get_entry_raw() - Access an entry from the raw equivalent1123 *                                     of a mirror level 1 page table.1124 * @table: Target level 1 page table.1125 * @idx: Index of the entry to access.1126 *1127 * Technically this function returns a pointer to a slot in a raw level 1 page1128 * table, since the returned "entry" is not guaranteed to be valid. The caller1129 * must verify the validity of the entry at the returned address (perhaps using1130 * pvr_page_table_l1_entry_raw_is_valid()) before reading or overwriting it.1131 *1132 * The value of @idx is not checked here; it is the callers responsibility to1133 * ensure @idx refers to a valid index within @table before dereferencing the1134 * returned pointer.1135 *1136 * Return:1137 * A pointer to the requested raw level 1 page table entry.1138 */1139static struct pvr_page_table_l1_entry_raw *1140pvr_page_table_l1_get_entry_raw(struct pvr_page_table_l1 *table, u16 idx)1141{1142	return &pvr_page_table_l1_get_raw(table)->entries[idx];1143}1144 1145/**1146 * pvr_page_table_l1_entry_is_valid() - Check if a level 1 page table entry is1147 *                                      marked as valid.1148 * @table: Target level 1 page table.1149 * @idx: Index of the entry to check.1150 *1151 * The value of @idx is not checked here; it is the callers responsibility to1152 * ensure @idx refers to a valid index within @table before calling this1153 * function.1154 */1155static bool1156pvr_page_table_l1_entry_is_valid(struct pvr_page_table_l1 *table, u16 idx)1157{1158	struct pvr_page_table_l1_entry_raw entry_raw =1159		*pvr_page_table_l1_get_entry_raw(table, idx);1160 1161	return pvr_page_table_l1_entry_raw_is_valid(entry_raw);1162}1163 1164/**1165 * struct pvr_page_table_l0 - A wrapped level 0 page table.1166 *1167 * To access the raw part of this table, use pvr_page_table_l0_get_raw().1168 * Alternatively to access a raw entry directly, use1169 * pvr_page_table_l0_get_entry_raw().1170 *1171 * There is no mirror representation of an individual page, so this type has no1172 * &entries member.1173 */1174struct pvr_page_table_l0 {1175	/**1176	 * @backing_page: A handle to the memory which holds the raw1177	 * equivalent of this table. **For internal use only.**1178	 */1179	struct pvr_mmu_backing_page backing_page;1180 1181	union {1182		/**1183		 * @parent: The parent of this node in the page table tree structure.1184		 *1185		 * This is also a mirror table.1186		 *1187		 * Only valid when the L0 page table is active. When the L0 page table1188		 * has been removed and queued for destruction, the next_free field1189		 * should be used instead.1190		 */1191		struct pvr_page_table_l1 *parent;1192 1193		/**1194		 * @next_free: Pointer to the next L0 page table to take/free.1195		 *1196		 * Used to form a linked list of L0 page tables. This is used1197		 * when preallocating tables and when the page table has been1198		 * removed and queued for destruction.1199		 */1200		struct pvr_page_table_l0 *next_free;1201	};1202 1203	/**1204	 * @parent_idx: The index of the entry in the parent table (see1205	 * @parent) which corresponds to this table.1206	 */1207	u16 parent_idx;1208 1209	/**1210	 * @entry_count: The current number of valid entries (that we know of)1211	 * in this table. This value is essentially a refcount - the table is1212	 * destroyed when this value is decremented to zero by1213	 * pvr_page_table_l0_remove().1214	 */1215	u16 entry_count;1216};1217 1218/**1219 * pvr_page_table_l0_init() - Initialize a level 0 page table.1220 * @table: Target level 0 page table.1221 * @pvr_dev: Target PowerVR device1222 *1223 * When this function returns successfully, @table is still not considered1224 * valid. It must be inserted into the page table tree structure with1225 * pvr_page_table_l1_insert() before it is ready for use.1226 *1227 * It is expected that @table be zeroed (e.g. from kzalloc()) before calling1228 * this function.1229 *1230 * Return:1231 *  * 0 on success, or1232 *  * Any error encountered while intializing &table->backing_page using1233 *    pvr_mmu_backing_page_init().1234 */1235static int1236pvr_page_table_l0_init(struct pvr_page_table_l0 *table,1237		       struct pvr_device *pvr_dev)1238{1239	table->parent_idx = PVR_IDX_INVALID;1240 1241	return pvr_mmu_backing_page_init(&table->backing_page, pvr_dev);1242}1243 1244/**1245 * pvr_page_table_l0_free() - Teardown a level 0 page table.1246 * @table: Target level 0 page table.1247 *1248 * It is an error to attempt to use @table after calling this function, even1249 * indirectly. This includes calling pvr_page_table_l1_remove(), which must1250 * be called *before* pvr_page_table_l0_free().1251 */1252static void1253pvr_page_table_l0_free(struct pvr_page_table_l0 *table)1254{1255	pvr_mmu_backing_page_fini(&table->backing_page);1256	kfree(table);1257}1258 1259/**1260 * pvr_page_table_l0_sync() - Flush a level 0 page table from the CPU to the1261 *                            device.1262 * @table: Target level 0 page table.1263 *1264 * This is just a thin wrapper around pvr_mmu_backing_page_sync(), so the1265 * warning there applies here too: **Only call pvr_page_table_l0_sync() once1266 * you're sure you have no more changes to make to** @table **in the immediate1267 * future.**1268 *1269 * If child pages of @table also need to be flushed, this should be done first1270 * using a DMA sync function (e.g. dma_sync_sg_for_device()) *before* calling1271 * this function.1272 */1273static void1274pvr_page_table_l0_sync(struct pvr_page_table_l0 *table)1275{1276	pvr_mmu_backing_page_sync(&table->backing_page, PVR_MMU_SYNC_LEVEL_0_FLAGS);1277}1278 1279/**1280 * pvr_page_table_l0_get_raw() - Access the raw equivalent of a mirror level 01281 *                               page table.1282 * @table: Target level 0 page table.1283 *1284 * Essentially returns the CPU address of the raw equivalent of @table, cast to1285 * a &struct pvr_page_table_l0_raw pointer.1286 *1287 * You probably want to call pvr_page_table_l0_get_entry_raw() instead.1288 *1289 * Return:1290 * The raw equivalent of @table.1291 */1292static struct pvr_page_table_l0_raw *1293pvr_page_table_l0_get_raw(struct pvr_page_table_l0 *table)1294{1295	return table->backing_page.host_ptr;1296}1297 1298/**1299 * pvr_page_table_l0_get_entry_raw() - Access an entry from the raw equivalent1300 *                                     of a mirror level 0 page table.1301 * @table: Target level 0 page table.1302 * @idx: Index of the entry to access.1303 *1304 * Technically this function returns a pointer to a slot in a raw level 0 page1305 * table, since the returned "entry" is not guaranteed to be valid. The caller1306 * must verify the validity of the entry at the returned address (perhaps using1307 * pvr_page_table_l0_entry_raw_is_valid()) before reading or overwriting it.1308 *1309 * The value of @idx is not checked here; it is the callers responsibility to1310 * ensure @idx refers to a valid index within @table before dereferencing the1311 * returned pointer. This is espcially important for level 0 page tables, which1312 * can have a variable number of entries.1313 *1314 * Return:1315 * A pointer to the requested raw level 0 page table entry.1316 */1317static struct pvr_page_table_l0_entry_raw *1318pvr_page_table_l0_get_entry_raw(struct pvr_page_table_l0 *table, u16 idx)1319{1320	return &pvr_page_table_l0_get_raw(table)->entries[idx];1321}1322 1323/**1324 * pvr_page_table_l0_entry_is_valid() - Check if a level 0 page table entry is1325 *                                      marked as valid.1326 * @table: Target level 0 page table.1327 * @idx: Index of the entry to check.1328 *1329 * The value of @idx is not checked here; it is the callers responsibility to1330 * ensure @idx refers to a valid index within @table before calling this1331 * function.1332 */1333static bool1334pvr_page_table_l0_entry_is_valid(struct pvr_page_table_l0 *table, u16 idx)1335{1336	struct pvr_page_table_l0_entry_raw entry_raw =1337		*pvr_page_table_l0_get_entry_raw(table, idx);1338 1339	return pvr_page_table_l0_entry_raw_is_valid(entry_raw);1340}1341 1342/**1343 * struct pvr_mmu_context - context holding data for operations at page1344 * catalogue level, intended for use with a VM context.1345 */1346struct pvr_mmu_context {1347	/** @pvr_dev: The PVR device associated with the owning VM context. */1348	struct pvr_device *pvr_dev;1349 1350	/** @page_table_l2: The MMU table root. */1351	struct pvr_page_table_l2 page_table_l2;1352};1353 1354/**1355 * struct pvr_page_table_ptr - A reference to a single physical page as indexed1356 * by the page table structure.1357 *1358 * Intended for embedding in a &struct pvr_mmu_op_context.1359 */1360struct pvr_page_table_ptr {1361	/**1362	 * @l1_table: A cached handle to the level 1 page table the1363	 * context is currently traversing.1364	 */1365	struct pvr_page_table_l1 *l1_table;1366 1367	/**1368	 * @l0_table: A cached handle to the level 0 page table the1369	 * context is currently traversing.1370	 */1371	struct pvr_page_table_l0 *l0_table;1372 1373	/**1374	 * @l2_idx: Index into the level 2 page table the context is1375	 * currently referencing.1376	 */1377	u16 l2_idx;1378 1379	/**1380	 * @l1_idx: Index into the level 1 page table the context is1381	 * currently referencing.1382	 */1383	u16 l1_idx;1384 1385	/**1386	 * @l0_idx: Index into the level 0 page table the context is1387	 * currently referencing.1388	 */1389	u16 l0_idx;1390};1391 1392/**1393 * struct pvr_mmu_op_context - context holding data for individual1394 * device-virtual mapping operations. Intended for use with a VM bind operation.1395 */1396struct pvr_mmu_op_context {1397	/** @mmu_ctx: The MMU context associated with the owning VM context. */1398	struct pvr_mmu_context *mmu_ctx;1399 1400	/** @map: Data specifically for map operations. */1401	struct {1402		/**1403		 * @sgt: Scatter gather table containing pages pinned for use by1404		 * this context - these are currently pinned when initialising1405		 * the VM bind operation.1406		 */1407		struct sg_table *sgt;1408 1409		/** @sgt_offset: Start address of the device-virtual mapping. */1410		u64 sgt_offset;1411 1412		/**1413		 * @l1_prealloc_tables: Preallocated l1 page table objects1414		 * use by this context when creating a page mapping. Linked list1415		 * fully created during initialisation.1416		 */1417		struct pvr_page_table_l1 *l1_prealloc_tables;1418 1419		/**1420		 * @l0_prealloc_tables: Preallocated l0 page table objects1421		 * use by this context when creating a page mapping. Linked list1422		 * fully created during initialisation.1423		 */1424		struct pvr_page_table_l0 *l0_prealloc_tables;1425	} map;1426 1427	/** @unmap: Data specifically for unmap operations. */1428	struct {1429		/**1430		 * @l1_free_tables: Collects page table objects freed by unmap1431		 * ops. Linked list empty at creation.1432		 */1433		struct pvr_page_table_l1 *l1_free_tables;1434 1435		/**1436		 * @l0_free_tables: Collects page table objects freed by unmap1437		 * ops. Linked list empty at creation.1438		 */1439		struct pvr_page_table_l0 *l0_free_tables;1440	} unmap;1441 1442	/**1443	 * @curr_page: A reference to a single physical page as indexed by the1444	 * page table structure.1445	 */1446	struct pvr_page_table_ptr curr_page;1447 1448	/**1449	 * @sync_level_required: The maximum level of the page table tree1450	 * structure which has (possibly) been modified since it was last1451	 * flushed to the device.1452	 *1453	 * This field should only be set with pvr_mmu_op_context_require_sync()1454	 * or indirectly by pvr_mmu_op_context_sync_partial().1455	 */1456	enum pvr_mmu_sync_level sync_level_required;1457};1458 1459/**1460 * pvr_page_table_l2_insert() - Insert an entry referring to a level 1 page1461 * table into a level 2 page table.1462 * @op_ctx: Target MMU op context pointing at the entry to insert the L1 page1463 * table into.1464 * @child_table: Target level 1 page table to be referenced by the new entry.1465 *1466 * It is the caller's responsibility to ensure @op_ctx.curr_page points to a1467 * valid L2 entry.1468 *1469 * It is the caller's responsibility to execute any memory barries to ensure1470 * that the creation of @child_table is ordered before the L2 entry is inserted.1471 */1472static void1473pvr_page_table_l2_insert(struct pvr_mmu_op_context *op_ctx,1474			 struct pvr_page_table_l1 *child_table)1475{1476	struct pvr_page_table_l2 *l2_table =1477		&op_ctx->mmu_ctx->page_table_l2;1478	struct pvr_page_table_l2_entry_raw *entry_raw =1479		pvr_page_table_l2_get_entry_raw(l2_table,1480						op_ctx->curr_page.l2_idx);1481 1482	pvr_page_table_l2_entry_raw_set(entry_raw,1483					child_table->backing_page.dma_addr);1484 1485	child_table->parent = l2_table;1486	child_table->parent_idx = op_ctx->curr_page.l2_idx;1487	l2_table->entries[op_ctx->curr_page.l2_idx] = child_table;1488	++l2_table->entry_count;1489	op_ctx->curr_page.l1_table = child_table;1490}1491 1492/**1493 * pvr_page_table_l2_remove() - Remove a level 1 page table from a level 2 page1494 * table.1495 * @op_ctx: Target MMU op context pointing at the L2 entry to remove.1496 *1497 * It is the caller's responsibility to ensure @op_ctx.curr_page points to a1498 * valid L2 entry.1499 */1500static void1501pvr_page_table_l2_remove(struct pvr_mmu_op_context *op_ctx)1502{1503	struct pvr_page_table_l2 *l2_table =1504		&op_ctx->mmu_ctx->page_table_l2;1505	struct pvr_page_table_l2_entry_raw *entry_raw =1506		pvr_page_table_l2_get_entry_raw(l2_table,1507						op_ctx->curr_page.l1_table->parent_idx);1508 1509	WARN_ON(op_ctx->curr_page.l1_table->parent != l2_table);1510 1511	pvr_page_table_l2_entry_raw_clear(entry_raw);1512 1513	l2_table->entries[op_ctx->curr_page.l1_table->parent_idx] = NULL;1514	op_ctx->curr_page.l1_table->parent_idx = PVR_IDX_INVALID;1515	op_ctx->curr_page.l1_table->next_free = op_ctx->unmap.l1_free_tables;1516	op_ctx->unmap.l1_free_tables = op_ctx->curr_page.l1_table;1517	op_ctx->curr_page.l1_table = NULL;1518 1519	--l2_table->entry_count;1520}1521 1522/**1523 * pvr_page_table_l1_insert() - Insert an entry referring to a level 0 page1524 * table into a level 1 page table.1525 * @op_ctx: Target MMU op context pointing at the entry to insert the L0 page1526 * table into.1527 * @child_table: L0 page table to insert.1528 *1529 * It is the caller's responsibility to ensure @op_ctx.curr_page points to a1530 * valid L1 entry.1531 *1532 * It is the caller's responsibility to execute any memory barries to ensure1533 * that the creation of @child_table is ordered before the L1 entry is inserted.1534 */1535static void1536pvr_page_table_l1_insert(struct pvr_mmu_op_context *op_ctx,1537			 struct pvr_page_table_l0 *child_table)1538{1539	struct pvr_page_table_l1_entry_raw *entry_raw =1540		pvr_page_table_l1_get_entry_raw(op_ctx->curr_page.l1_table,1541						op_ctx->curr_page.l1_idx);1542 1543	pvr_page_table_l1_entry_raw_set(entry_raw,1544					child_table->backing_page.dma_addr);1545 1546	child_table->parent = op_ctx->curr_page.l1_table;1547	child_table->parent_idx = op_ctx->curr_page.l1_idx;1548	op_ctx->curr_page.l1_table->entries[op_ctx->curr_page.l1_idx] = child_table;1549	++op_ctx->curr_page.l1_table->entry_count;1550	op_ctx->curr_page.l0_table = child_table;1551}1552 1553/**1554 * pvr_page_table_l1_remove() - Remove a level 0 page table from a level 1 page1555 *                              table.1556 * @op_ctx: Target MMU op context pointing at the L1 entry to remove.1557 *1558 * If this function results in the L1 table becoming empty, it will be removed1559 * from its parent level 2 page table and destroyed.1560 *1561 * It is the caller's responsibility to ensure @op_ctx.curr_page points to a1562 * valid L1 entry.1563 */1564static void1565pvr_page_table_l1_remove(struct pvr_mmu_op_context *op_ctx)1566{1567	struct pvr_page_table_l1_entry_raw *entry_raw =1568		pvr_page_table_l1_get_entry_raw(op_ctx->curr_page.l0_table->parent,1569						op_ctx->curr_page.l0_table->parent_idx);1570 1571	WARN_ON(op_ctx->curr_page.l0_table->parent !=1572		op_ctx->curr_page.l1_table);1573 1574	pvr_page_table_l1_entry_raw_clear(entry_raw);1575 1576	op_ctx->curr_page.l1_table->entries[op_ctx->curr_page.l0_table->parent_idx] = NULL;1577	op_ctx->curr_page.l0_table->parent_idx = PVR_IDX_INVALID;1578	op_ctx->curr_page.l0_table->next_free = op_ctx->unmap.l0_free_tables;1579	op_ctx->unmap.l0_free_tables = op_ctx->curr_page.l0_table;1580	op_ctx->curr_page.l0_table = NULL;1581 1582	if (--op_ctx->curr_page.l1_table->entry_count == 0) {1583		/* Clear the parent L2 page table entry. */1584		if (op_ctx->curr_page.l1_table->parent_idx != PVR_IDX_INVALID)1585			pvr_page_table_l2_remove(op_ctx);1586	}1587}1588 1589/**1590 * pvr_page_table_l0_insert() - Insert an entry referring to a physical page1591 * into a level 0 page table.1592 * @op_ctx: Target MMU op context pointing at the L0 entry to insert.1593 * @dma_addr: Target DMA address to be referenced by the new entry.1594 * @flags: Page options to be stored in the new entry.1595 *1596 * It is the caller's responsibility to ensure @op_ctx.curr_page points to a1597 * valid L0 entry.1598 */1599static void1600pvr_page_table_l0_insert(struct pvr_mmu_op_context *op_ctx,1601			 dma_addr_t dma_addr, struct pvr_page_flags_raw flags)1602{1603	struct pvr_page_table_l0_entry_raw *entry_raw =1604		pvr_page_table_l0_get_entry_raw(op_ctx->curr_page.l0_table,1605						op_ctx->curr_page.l0_idx);1606 1607	pvr_page_table_l0_entry_raw_set(entry_raw, dma_addr, flags);1608 1609	/*1610	 * There is no entry to set here - we don't keep a mirror of1611	 * individual pages.1612	 */1613 1614	++op_ctx->curr_page.l0_table->entry_count;1615}1616 1617/**1618 * pvr_page_table_l0_remove() - Remove a physical page from a level 0 page1619 * table.1620 * @op_ctx: Target MMU op context pointing at the L0 entry to remove.1621 *1622 * If this function results in the L0 table becoming empty, it will be removed1623 * from its parent L1 page table and destroyed.1624 *1625 * It is the caller's responsibility to ensure @op_ctx.curr_page points to a1626 * valid L0 entry.1627 */1628static void1629pvr_page_table_l0_remove(struct pvr_mmu_op_context *op_ctx)1630{1631	struct pvr_page_table_l0_entry_raw *entry_raw =1632		pvr_page_table_l0_get_entry_raw(op_ctx->curr_page.l0_table,1633						op_ctx->curr_page.l0_idx);1634 1635	pvr_page_table_l0_entry_raw_clear(entry_raw);1636 1637	/*1638	 * There is no entry to clear here - we don't keep a mirror of1639	 * individual pages.1640	 */1641 1642	if (--op_ctx->curr_page.l0_table->entry_count == 0) {1643		/* Clear the parent L1 page table entry. */1644		if (op_ctx->curr_page.l0_table->parent_idx != PVR_IDX_INVALID)1645			pvr_page_table_l1_remove(op_ctx);1646	}1647}1648 1649/**1650 * DOC: Page table index utilities1651 */1652 1653/**1654 * pvr_page_table_l2_idx() - Calculate the level 2 page table index for a1655 *                           device-virtual address.1656 * @device_addr: Target device-virtual address.1657 *1658 * This function does not perform any bounds checking - it is the caller's1659 * responsibility to ensure that @device_addr is valid before interpreting1660 * the result.1661 *1662 * Return:1663 * The index into a level 2 page table corresponding to @device_addr.1664 */1665static u161666pvr_page_table_l2_idx(u64 device_addr)1667{1668	return (device_addr & ~ROGUE_MMUCTRL_VADDR_PC_INDEX_CLRMSK) >>1669	       ROGUE_MMUCTRL_VADDR_PC_INDEX_SHIFT;1670}1671 1672/**1673 * pvr_page_table_l1_idx() - Calculate the level 1 page table index for a1674 *                           device-virtual address.1675 * @device_addr: Target device-virtual address.1676 *1677 * This function does not perform any bounds checking - it is the caller's1678 * responsibility to ensure that @device_addr is valid before interpreting1679 * the result.1680 *1681 * Return:1682 * The index into a level 1 page table corresponding to @device_addr.1683 */1684static u161685pvr_page_table_l1_idx(u64 device_addr)1686{1687	return (device_addr & ~ROGUE_MMUCTRL_VADDR_PD_INDEX_CLRMSK) >>1688	       ROGUE_MMUCTRL_VADDR_PD_INDEX_SHIFT;1689}1690 1691/**1692 * pvr_page_table_l0_idx() - Calculate the level 0 page table index for a1693 *                           device-virtual address.1694 * @device_addr: Target device-virtual address.1695 *1696 * This function does not perform any bounds checking - it is the caller's1697 * responsibility to ensure that @device_addr is valid before interpreting1698 * the result.1699 *1700 * Return:1701 * The index into a level 0 page table corresponding to @device_addr.1702 */1703static u161704pvr_page_table_l0_idx(u64 device_addr)1705{1706	return (device_addr & ~ROGUE_MMUCTRL_VADDR_PT_INDEX_CLRMSK) >>1707	       ROGUE_MMUCTRL_PAGE_X_RANGE_SHIFT;1708}1709 1710/**1711 * DOC: High-level page table operations1712 */1713 1714/**1715 * pvr_page_table_l1_get_or_insert() - Retrieves (optionally inserting if1716 * necessary) a level 1 page table from the specified level 2 page table entry.1717 * @op_ctx: Target MMU op context.1718 * @should_insert: [IN] Specifies whether new page tables should be inserted1719 * when empty page table entries are encountered during traversal.1720 *1721 * Return:1722 *  * 0 on success, or1723 *1724 *    If @should_insert is %false:1725 *     * -%ENXIO if a level 1 page table would have been inserted.1726 *1727 *    If @should_insert is %true:1728 *     * Any error encountered while inserting the level 1 page table.1729 */1730static int1731pvr_page_table_l1_get_or_insert(struct pvr_mmu_op_context *op_ctx,1732				bool should_insert)1733{1734	struct pvr_page_table_l2 *l2_table =1735		&op_ctx->mmu_ctx->page_table_l2;1736	struct pvr_page_table_l1 *table;1737 1738	if (pvr_page_table_l2_entry_is_valid(l2_table,1739					     op_ctx->curr_page.l2_idx)) {1740		op_ctx->curr_page.l1_table =1741			l2_table->entries[op_ctx->curr_page.l2_idx];1742		return 0;1743	}1744 1745	if (!should_insert)1746		return -ENXIO;1747 1748	/* Take a prealloced table. */1749	table = op_ctx->map.l1_prealloc_tables;1750	if (!table)1751		return -ENOMEM;1752 1753	/* Pop */1754	op_ctx->map.l1_prealloc_tables = table->next_free;1755	table->next_free = NULL;1756 1757	/* Ensure new table is fully written out before adding to L2 page table. */1758	wmb();1759 1760	pvr_page_table_l2_insert(op_ctx, table);1761 1762	return 0;1763}1764 1765/**1766 * pvr_page_table_l0_get_or_insert() - Retrieves (optionally inserting if1767 * necessary) a level 0 page table from the specified level 1 page table entry.1768 * @op_ctx: Target MMU op context.1769 * @should_insert: [IN] Specifies whether new page tables should be inserted1770 * when empty page table entries are encountered during traversal.1771 *1772 * Return:1773 *  * 0 on success,1774 *1775 *    If @should_insert is %false:1776 *     * -%ENXIO if a level 0 page table would have been inserted.1777 *1778 *    If @should_insert is %true:1779 *     * Any error encountered while inserting the level 0 page table.1780 */1781static int1782pvr_page_table_l0_get_or_insert(struct pvr_mmu_op_context *op_ctx,1783				bool should_insert)1784{1785	struct pvr_page_table_l0 *table;1786 1787	if (pvr_page_table_l1_entry_is_valid(op_ctx->curr_page.l1_table,1788					     op_ctx->curr_page.l1_idx)) {1789		op_ctx->curr_page.l0_table =1790			op_ctx->curr_page.l1_table->entries[op_ctx->curr_page.l1_idx];1791		return 0;1792	}1793 1794	if (!should_insert)1795		return -ENXIO;1796 1797	/* Take a prealloced table. */1798	table = op_ctx->map.l0_prealloc_tables;1799	if (!table)1800		return -ENOMEM;1801 1802	/* Pop */1803	op_ctx->map.l0_prealloc_tables = table->next_free;1804	table->next_free = NULL;1805 1806	/* Ensure new table is fully written out before adding to L1 page table. */1807	wmb();1808 1809	pvr_page_table_l1_insert(op_ctx, table);1810 1811	return 0;1812}1813 1814/**1815 * pvr_mmu_context_create() - Create an MMU context.1816 * @pvr_dev: PVR device associated with owning VM context.1817 *1818 * Returns:1819 *  * Newly created MMU context object on success, or1820 *  * -%ENOMEM if no memory is available,1821 *  * Any error code returned by pvr_page_table_l2_init().1822 */1823struct pvr_mmu_context *pvr_mmu_context_create(struct pvr_device *pvr_dev)1824{1825	struct pvr_mmu_context *ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);1826	int err;1827 1828	if (!ctx)1829		return ERR_PTR(-ENOMEM);1830 1831	err = pvr_page_table_l2_init(&ctx->page_table_l2, pvr_dev);1832	if (err)1833		return ERR_PTR(err);1834 1835	ctx->pvr_dev = pvr_dev;1836 1837	return ctx;1838}1839 1840/**1841 * pvr_mmu_context_destroy() - Destroy an MMU context.1842 * @ctx: Target MMU context.1843 */1844void pvr_mmu_context_destroy(struct pvr_mmu_context *ctx)1845{1846	pvr_page_table_l2_fini(&ctx->page_table_l2);1847	kfree(ctx);1848}1849 1850/**1851 * pvr_mmu_get_root_table_dma_addr() - Get the DMA address of the root of the1852 * page table structure behind a VM context.1853 * @ctx: Target MMU context.1854 */1855dma_addr_t pvr_mmu_get_root_table_dma_addr(struct pvr_mmu_context *ctx)1856{1857	return ctx->page_table_l2.backing_page.dma_addr;1858}1859 1860/**1861 * pvr_page_table_l1_alloc() - Allocate a l1 page_table object.1862 * @ctx: MMU context of owning VM context.1863 *1864 * Returns:1865 *  * Newly created page table object on success, or1866 *  * -%ENOMEM if no memory is available,1867 *  * Any error code returned by pvr_page_table_l1_init().1868 */1869static struct pvr_page_table_l1 *1870pvr_page_table_l1_alloc(struct pvr_mmu_context *ctx)1871{1872	int err;1873 1874	struct pvr_page_table_l1 *table =1875		kzalloc(sizeof(*table), GFP_KERNEL);1876 1877	if (!table)1878		return ERR_PTR(-ENOMEM);1879 1880	err = pvr_page_table_l1_init(table, ctx->pvr_dev);1881	if (err) {1882		kfree(table);1883		return ERR_PTR(err);1884	}1885 1886	return table;1887}1888 1889/**1890 * pvr_page_table_l0_alloc() - Allocate a l0 page_table object.1891 * @ctx: MMU context of owning VM context.1892 *1893 * Returns:1894 *  * Newly created page table object on success, or1895 *  * -%ENOMEM if no memory is available,1896 *  * Any error code returned by pvr_page_table_l0_init().1897 */1898static struct pvr_page_table_l0 *1899pvr_page_table_l0_alloc(struct pvr_mmu_context *ctx)1900{1901	int err;1902 1903	struct pvr_page_table_l0 *table =1904		kzalloc(sizeof(*table), GFP_KERNEL);1905 1906	if (!table)1907		return ERR_PTR(-ENOMEM);1908 1909	err = pvr_page_table_l0_init(table, ctx->pvr_dev);1910	if (err) {1911		kfree(table);1912		return ERR_PTR(err);1913	}1914 1915	return table;1916}1917 1918/**1919 * pvr_mmu_op_context_require_sync() - Mark an MMU op context as requiring a1920 * sync operation for the referenced page tables up to a specified level.1921 * @op_ctx: Target MMU op context.1922 * @level: Maximum page table level for which a sync is required.1923 */1924static void1925pvr_mmu_op_context_require_sync(struct pvr_mmu_op_context *op_ctx,1926				enum pvr_mmu_sync_level level)1927{1928	if (op_ctx->sync_level_required < level)1929		op_ctx->sync_level_required = level;1930}1931 1932/**1933 * pvr_mmu_op_context_sync_manual() - Trigger a sync of some or all of the1934 * page tables referenced by a MMU op context.1935 * @op_ctx: Target MMU op context.1936 * @level: Maximum page table level to sync.1937 *1938 * Do not call this function directly. Instead use1939 * pvr_mmu_op_context_sync_partial() which is checked against the current1940 * value of &op_ctx->sync_level_required as set by1941 * pvr_mmu_op_context_require_sync().1942 */1943static void1944pvr_mmu_op_context_sync_manual(struct pvr_mmu_op_context *op_ctx,1945			       enum pvr_mmu_sync_level level)1946{1947	/*1948	 * We sync the page table levels in ascending order (starting from the1949	 * leaf node) to ensure consistency.1950	 */1951 1952	WARN_ON(level < PVR_MMU_SYNC_LEVEL_NONE);1953 1954	if (level <= PVR_MMU_SYNC_LEVEL_NONE)1955		return;1956 1957	if (op_ctx->curr_page.l0_table)1958		pvr_page_table_l0_sync(op_ctx->curr_page.l0_table);1959 1960	if (level < PVR_MMU_SYNC_LEVEL_1)1961		return;1962 1963	if (op_ctx->curr_page.l1_table)1964		pvr_page_table_l1_sync(op_ctx->curr_page.l1_table);1965 1966	if (level < PVR_MMU_SYNC_LEVEL_2)1967		return;1968 1969	pvr_page_table_l2_sync(&op_ctx->mmu_ctx->page_table_l2);1970}1971 1972/**1973 * pvr_mmu_op_context_sync_partial() - Trigger a sync of some or all of the1974 * page tables referenced by a MMU op context.1975 * @op_ctx: Target MMU op context.1976 * @level: Requested page table level to sync up to (inclusive).1977 *1978 * If @level is greater than the maximum level recorded by @op_ctx as requiring1979 * a sync operation, only the previously recorded maximum will be used.1980 *1981 * Additionally, if @level is greater than or equal to the maximum level1982 * recorded by @op_ctx as requiring a sync operation, that maximum level will be1983 * reset as a full sync will be performed. This is equivalent to calling1984 * pvr_mmu_op_context_sync().1985 */1986static void1987pvr_mmu_op_context_sync_partial(struct pvr_mmu_op_context *op_ctx,1988				enum pvr_mmu_sync_level level)1989{1990	/*1991	 * If the requested sync level is greater than or equal to the1992	 * currently required sync level, we do two things:1993	 *  * Don't waste time syncing levels we haven't previously marked as1994	 *    requiring a sync, and1995	 *  * Reset the required sync level since we are about to sync1996	 *    everything that was previously marked as requiring a sync.1997	 */1998	if (level >= op_ctx->sync_level_required) {1999		level = op_ctx->sync_level_required;2000		op_ctx->sync_level_required = PVR_MMU_SYNC_LEVEL_NONE;2001	}2002 2003	pvr_mmu_op_context_sync_manual(op_ctx, level);2004}2005 2006/**2007 * pvr_mmu_op_context_sync() - Trigger a sync of every page table referenced by2008 * a MMU op context.2009 * @op_ctx: Target MMU op context.2010 *2011 * The maximum level marked internally as requiring a sync will be reset so2012 * that subsequent calls to this function will be no-ops unless @op_ctx is2013 * otherwise updated.2014 */2015static void2016pvr_mmu_op_context_sync(struct pvr_mmu_op_context *op_ctx)2017{2018	pvr_mmu_op_context_sync_manual(op_ctx, op_ctx->sync_level_required);2019 2020	op_ctx->sync_level_required = PVR_MMU_SYNC_LEVEL_NONE;2021}2022 2023/**2024 * pvr_mmu_op_context_load_tables() - Load pointers to tables in each level of2025 * the page table tree structure needed to reference the physical page2026 * referenced by a MMU op context.2027 * @op_ctx: Target MMU op context.2028 * @should_create: Specifies whether new page tables should be created when2029 * empty page table entries are encountered during traversal.2030 * @load_level_required: Maximum page table level to load.2031 *2032 * If @should_create is %true, this function may modify the stored required2033 * sync level of @op_ctx as new page tables are created and inserted into their2034 * respective parents.2035 *2036 * Since there is only one root page table, it is technically incorrect to call2037 * this function with a value of @load_level_required greater than or equal to2038 * the root level number. However, this is not explicitly disallowed here.2039 *2040 * Return:2041 *  * 0 on success,2042 *  * Any error returned by pvr_page_table_l1_get_or_create() if2043 *    @load_level_required >= 1 except -%ENXIO, or2044 *  * Any error returned by pvr_page_table_l0_get_or_create() if2045 *    @load_level_required >= 0 except -%ENXIO.2046 */2047static int2048pvr_mmu_op_context_load_tables(struct pvr_mmu_op_context *op_ctx,2049			       bool should_create,2050			       enum pvr_mmu_sync_level load_level_required)2051{2052	const struct pvr_page_table_l1 *l1_head_before =2053		op_ctx->map.l1_prealloc_tables;2054	const struct pvr_page_table_l0 *l0_head_before =2055		op_ctx->map.l0_prealloc_tables;2056	int err;2057 2058	/* Clear tables we're about to fetch in case of error states. */2059	if (load_level_required >= PVR_MMU_SYNC_LEVEL_1)2060		op_ctx->curr_page.l1_table = NULL;2061 2062	if (load_level_required >= PVR_MMU_SYNC_LEVEL_0)2063		op_ctx->curr_page.l0_table = NULL;2064 2065	/* Get or create L1 page table. */2066	if (load_level_required >= PVR_MMU_SYNC_LEVEL_1) {2067		err = pvr_page_table_l1_get_or_insert(op_ctx, should_create);2068		if (err) {2069			/*2070			 * If @should_create is %false and no L1 page table was2071			 * found, return early but without an error. Since2072			 * pvr_page_table_l1_get_or_create() can only return2073			 * -%ENXIO if @should_create is %false, there is no2074			 * need to check it here.2075			 */2076			if (err == -ENXIO)2077				err = 0;2078 2079			return err;2080		}2081	}2082 2083	/* Get or create L0 page table. */2084	if (load_level_required >= PVR_MMU_SYNC_LEVEL_0) {2085		err = pvr_page_table_l0_get_or_insert(op_ctx, should_create);2086		if (err) {2087			/*2088			 * If @should_create is %false and no L0 page table was2089			 * found, return early but without an error. Since2090			 * pvr_page_table_l0_get_or_insert() can only return2091			 * -%ENXIO if @should_create is %false, there is no2092			 * need to check it here.2093			 */2094			if (err == -ENXIO)2095				err = 0;2096 2097			/*2098			 * At this point, an L1 page table could have been2099			 * inserted but is now empty due to the failed attempt2100			 * at inserting an L0 page table. In this instance, we2101			 * must remove the empty L1 page table ourselves as2102			 * pvr_page_table_l1_remove() is never called as part2103			 * of the error path in2104			 * pvr_page_table_l0_get_or_insert().2105			 */2106			if (l1_head_before != op_ctx->map.l1_prealloc_tables) {2107				pvr_page_table_l2_remove(op_ctx);2108				pvr_mmu_op_context_require_sync(op_ctx, PVR_MMU_SYNC_LEVEL_2);2109			}2110 2111			return err;2112		}2113	}2114 2115	/*2116	 * A sync is only needed if table objects were inserted. This can be2117	 * inferred by checking if the pointer at the head of the linked list2118	 * has changed.2119	 */2120	if (l1_head_before != op_ctx->map.l1_prealloc_tables)2121		pvr_mmu_op_context_require_sync(op_ctx, PVR_MMU_SYNC_LEVEL_2);2122	else if (l0_head_before != op_ctx->map.l0_prealloc_tables)2123		pvr_mmu_op_context_require_sync(op_ctx, PVR_MMU_SYNC_LEVEL_1);2124 2125	return 0;2126}2127 2128/**2129 * pvr_mmu_op_context_set_curr_page() - Reassign the current page of an MMU op2130 * context, syncing any page tables previously assigned to it which are no2131 * longer relevant.2132 * @op_ctx: Target MMU op context.2133 * @device_addr: New pointer target.2134 * @should_create: Specify whether new page tables should be created when2135 * empty page table entries are encountered during traversal.2136 *2137 * This function performs a full sync on the pointer, regardless of which2138 * levels are modified.2139 *2140 * Return:2141 *  * 0 on success, or2142 *  * Any error returned by pvr_mmu_op_context_load_tables().2143 */2144static int2145pvr_mmu_op_context_set_curr_page(struct pvr_mmu_op_context *op_ctx,2146				 u64 device_addr, bool should_create)2147{2148	pvr_mmu_op_context_sync(op_ctx);2149 2150	op_ctx->curr_page.l2_idx = pvr_page_table_l2_idx(device_addr);2151	op_ctx->curr_page.l1_idx = pvr_page_table_l1_idx(device_addr);2152	op_ctx->curr_page.l0_idx = pvr_page_table_l0_idx(device_addr);2153	op_ctx->curr_page.l1_table = NULL;2154	op_ctx->curr_page.l0_table = NULL;2155 2156	return pvr_mmu_op_context_load_tables(op_ctx, should_create,2157					      PVR_MMU_SYNC_LEVEL_1);2158}2159 2160/**2161 * pvr_mmu_op_context_next_page() - Advance the current page of an MMU op2162 * context.2163 * @op_ctx: Target MMU op context.2164 * @should_create: Specify whether new page tables should be created when2165 * empty page table entries are encountered during traversal.2166 *2167 * If @should_create is %false, it is the caller's responsibility to verify that2168 * the state of the table references in @op_ctx is valid on return. If -%ENXIO2169 * is returned, at least one of the table references is invalid. It should be2170 * noted that @op_ctx as a whole will be left in a valid state if -%ENXIO is2171 * returned, unlike other error codes. The caller should check which references2172 * are invalid by comparing them to %NULL. Only &@ptr->l2_table is guaranteed2173 * to be valid, since it represents the root of the page table tree structure.2174 *2175 * Return:2176 *  * 0 on success,2177 *  * -%EPERM if the operation would wrap at the top of the page table2178 *    hierarchy,2179 *  * -%ENXIO if @should_create is %false and a page table of any level would2180 *    have otherwise been created, or2181 *  * Any error returned while attempting to create missing page tables if2182 *    @should_create is %true.2183 */2184static int2185pvr_mmu_op_context_next_page(struct pvr_mmu_op_context *op_ctx,2186			     bool should_create)2187{2188	s8 load_level_required = PVR_MMU_SYNC_LEVEL_NONE;2189 2190	if (++op_ctx->curr_page.l0_idx != ROGUE_MMUCTRL_ENTRIES_PT_VALUE_X)2191		goto load_tables;2192 2193	op_ctx->curr_page.l0_idx = 0;2194	load_level_required = PVR_MMU_SYNC_LEVEL_0;2195 2196	if (++op_ctx->curr_page.l1_idx != ROGUE_MMUCTRL_ENTRIES_PD_VALUE)2197		goto load_tables;2198 2199	op_ctx->curr_page.l1_idx = 0;2200	load_level_required = PVR_MMU_SYNC_LEVEL_1;2201 2202	if (++op_ctx->curr_page.l2_idx != ROGUE_MMUCTRL_ENTRIES_PC_VALUE)2203		goto load_tables;2204 2205	/*2206	 * If the pattern continued, we would set &op_ctx->curr_page.l2_idx to2207	 * zero here. However, that would wrap the top layer of the page table2208	 * hierarchy which is not a valid operation. Instead, we warn and return2209	 * an error.2210	 */2211	WARN(true,2212	     "%s(%p) attempted to loop the top of the page table hierarchy",2213	     __func__, op_ctx);2214	return -EPERM;2215 2216	/* If indices have wrapped, we need to load new tables. */2217load_tables:2218	/* First, flush tables which will be unloaded. */2219	pvr_mmu_op_context_sync_partial(op_ctx, load_level_required);2220 2221	/* Then load tables from the required level down. */2222	return pvr_mmu_op_context_load_tables(op_ctx, should_create,2223					      load_level_required);2224}2225 2226/**2227 * DOC: Single page operations2228 */2229 2230/**2231 * pvr_page_create() - Create a device-virtual memory page and insert it into2232 * a level 0 page table.2233 * @op_ctx: Target MMU op context pointing at the device-virtual address of the2234 * target page.2235 * @dma_addr: DMA address of the physical page backing the created page.2236 * @flags: Page options saved on the level 0 page table entry for reading by2237 *         the device.2238 *2239 * Return:2240 *  * 0 on success, or2241 *  * -%EEXIST if the requested page already exists.2242 */2243static int2244pvr_page_create(struct pvr_mmu_op_context *op_ctx, dma_addr_t dma_addr,2245		struct pvr_page_flags_raw flags)2246{2247	/* Do not create a new page if one already exists. */2248	if (pvr_page_table_l0_entry_is_valid(op_ctx->curr_page.l0_table,2249					     op_ctx->curr_page.l0_idx)) {2250		return -EEXIST;2251	}2252 2253	pvr_page_table_l0_insert(op_ctx, dma_addr, flags);2254 2255	pvr_mmu_op_context_require_sync(op_ctx, PVR_MMU_SYNC_LEVEL_0);2256 2257	return 0;2258}2259 2260/**2261 * pvr_page_destroy() - Destroy a device page after removing it from its2262 * parent level 0 page table.2263 * @op_ctx: Target MMU op context.2264 */2265static void2266pvr_page_destroy(struct pvr_mmu_op_context *op_ctx)2267{2268	/* Do nothing if the page does not exist. */2269	if (!pvr_page_table_l0_entry_is_valid(op_ctx->curr_page.l0_table,2270					      op_ctx->curr_page.l0_idx)) {2271		return;2272	}2273 2274	/* Clear the parent L0 page table entry. */2275	pvr_page_table_l0_remove(op_ctx);2276 2277	pvr_mmu_op_context_require_sync(op_ctx, PVR_MMU_SYNC_LEVEL_0);2278}2279 2280/**2281 * pvr_mmu_op_context_destroy() - Destroy an MMU op context.2282 * @op_ctx: Target MMU op context.2283 */2284void pvr_mmu_op_context_destroy(struct pvr_mmu_op_context *op_ctx)2285{2286	const bool flush_caches =2287		op_ctx->sync_level_required != PVR_MMU_SYNC_LEVEL_NONE;2288 2289	pvr_mmu_op_context_sync(op_ctx);2290 2291	/* Unmaps should be flushed immediately. Map flushes can be deferred. */2292	if (flush_caches && !op_ctx->map.sgt)2293		pvr_mmu_flush_exec(op_ctx->mmu_ctx->pvr_dev, true);2294 2295	while (op_ctx->map.l0_prealloc_tables) {2296		struct pvr_page_table_l0 *tmp = op_ctx->map.l0_prealloc_tables;2297 2298		op_ctx->map.l0_prealloc_tables =2299			op_ctx->map.l0_prealloc_tables->next_free;2300		pvr_page_table_l0_free(tmp);2301	}2302 2303	while (op_ctx->map.l1_prealloc_tables) {2304		struct pvr_page_table_l1 *tmp = op_ctx->map.l1_prealloc_tables;2305 2306		op_ctx->map.l1_prealloc_tables =2307			op_ctx->map.l1_prealloc_tables->next_free;2308		pvr_page_table_l1_free(tmp);2309	}2310 2311	while (op_ctx->unmap.l0_free_tables) {2312		struct pvr_page_table_l0 *tmp = op_ctx->unmap.l0_free_tables;2313 2314		op_ctx->unmap.l0_free_tables =2315			op_ctx->unmap.l0_free_tables->next_free;2316		pvr_page_table_l0_free(tmp);2317	}2318 2319	while (op_ctx->unmap.l1_free_tables) {2320		struct pvr_page_table_l1 *tmp = op_ctx->unmap.l1_free_tables;2321 2322		op_ctx->unmap.l1_free_tables =2323			op_ctx->unmap.l1_free_tables->next_free;2324		pvr_page_table_l1_free(tmp);2325	}2326 2327	kfree(op_ctx);2328}2329 2330/**2331 * pvr_mmu_op_context_create() - Create an MMU op context.2332 * @ctx: MMU context associated with owning VM context.2333 * @sgt: Scatter gather table containing pages pinned for use by this context.2334 * @sgt_offset: Start offset of the requested device-virtual memory mapping.2335 * @size: Size in bytes of the requested device-virtual memory mapping. For an2336 * unmapping, this should be zero so that no page tables are allocated.2337 *2338 * Returns:2339 *  * Newly created MMU op context object on success, or2340 *  * -%ENOMEM if no memory is available,2341 *  * Any error code returned by pvr_page_table_l2_init().2342 */2343struct pvr_mmu_op_context *2344pvr_mmu_op_context_create(struct pvr_mmu_context *ctx, struct sg_table *sgt,2345			  u64 sgt_offset, u64 size)2346{2347	int err;2348 2349	struct pvr_mmu_op_context *op_ctx =2350		kzalloc(sizeof(*op_ctx), GFP_KERNEL);2351 2352	if (!op_ctx)2353		return ERR_PTR(-ENOMEM);2354 2355	op_ctx->mmu_ctx = ctx;2356	op_ctx->map.sgt = sgt;2357	op_ctx->map.sgt_offset = sgt_offset;2358	op_ctx->sync_level_required = PVR_MMU_SYNC_LEVEL_NONE;2359 2360	if (size) {2361		/*2362		 * The number of page table objects we need to prealloc is2363		 * indicated by the mapping size, start offset and the sizes2364		 * of the areas mapped per PT or PD. The range calculation is2365		 * identical to that for the index into a table for a device2366		 * address, so we reuse those functions here.2367		 */2368		const u32 l1_start_idx = pvr_page_table_l2_idx(sgt_offset);2369		const u32 l1_end_idx = pvr_page_table_l2_idx(sgt_offset + size);2370		const u32 l1_count = l1_end_idx - l1_start_idx + 1;2371		const u32 l0_start_idx = pvr_page_table_l1_idx(sgt_offset);2372		const u32 l0_end_idx = pvr_page_table_l1_idx(sgt_offset + size);2373		const u32 l0_count = l0_end_idx - l0_start_idx + 1;2374 2375		/*2376		 * Alloc and push page table entries until we have enough of2377		 * each type, ending with linked lists of l0 and l1 entries in2378		 * reverse order.2379		 */2380		for (int i = 0; i < l1_count; i++) {2381			struct pvr_page_table_l1 *l1_tmp =2382				pvr_page_table_l1_alloc(ctx);2383 2384			err = PTR_ERR_OR_ZERO(l1_tmp);2385			if (err)2386				goto err_cleanup;2387 2388			l1_tmp->next_free = op_ctx->map.l1_prealloc_tables;2389			op_ctx->map.l1_prealloc_tables = l1_tmp;2390		}2391 2392		for (int i = 0; i < l0_count; i++) {2393			struct pvr_page_table_l0 *l0_tmp =2394				pvr_page_table_l0_alloc(ctx);2395 2396			err = PTR_ERR_OR_ZERO(l0_tmp);2397			if (err)2398				goto err_cleanup;2399 2400			l0_tmp->next_free = op_ctx->map.l0_prealloc_tables;2401			op_ctx->map.l0_prealloc_tables = l0_tmp;2402		}2403	}2404 2405	return op_ctx;2406 2407err_cleanup:2408	pvr_mmu_op_context_destroy(op_ctx);2409 2410	return ERR_PTR(err);2411}2412 2413/**2414 * pvr_mmu_op_context_unmap_curr_page() - Unmap pages from a memory context2415 * starting from the current page of an MMU op context.2416 * @op_ctx: Target MMU op context pointing at the first page to unmap.2417 * @nr_pages: Number of pages to unmap.2418 *2419 * Return:2420 *  * 0 on success, or2421 *  * Any error encountered while advancing @op_ctx.curr_page with2422 *    pvr_mmu_op_context_next_page() (except -%ENXIO).2423 */2424static int2425pvr_mmu_op_context_unmap_curr_page(struct pvr_mmu_op_context *op_ctx,2426				   u64 nr_pages)2427{2428	int err;2429 2430	if (nr_pages == 0)2431		return 0;2432 2433	/*2434	 * Destroy first page outside loop, as it doesn't require a page2435	 * advance beforehand. If the L0 page table reference in2436	 * @op_ctx.curr_page is %NULL, there cannot be a mapped page at2437	 * @op_ctx.curr_page (so skip ahead).2438	 */2439	if (op_ctx->curr_page.l0_table)2440		pvr_page_destroy(op_ctx);2441 2442	for (u64 page = 1; page < nr_pages; ++page) {2443		err = pvr_mmu_op_context_next_page(op_ctx, false);2444		/*2445		 * If the page table tree structure at @op_ctx.curr_page is2446		 * incomplete, skip ahead. We don't care about unmapping pages2447		 * that cannot exist.2448		 *2449		 * FIXME: This could be made more efficient by jumping ahead2450		 * using pvr_mmu_op_context_set_curr_page().2451		 */2452		if (err == -ENXIO)2453			continue;2454		else if (err)2455			return err;2456 2457		pvr_page_destroy(op_ctx);2458	}2459 2460	return 0;2461}2462 2463/**2464 * pvr_mmu_unmap() - Unmap pages from a memory context.2465 * @op_ctx: Target MMU op context.2466 * @device_addr: First device-virtual address to unmap.2467 * @size: Size in bytes to unmap.2468 *2469 * The total amount of device-virtual memory unmapped is2470 * @nr_pages * %PVR_DEVICE_PAGE_SIZE.2471 *2472 * Returns:2473 *  * 0 on success, or2474 *  * Any error code returned by pvr_page_table_ptr_init(), or2475 *  * Any error code returned by pvr_page_table_ptr_unmap().2476 */2477int pvr_mmu_unmap(struct pvr_mmu_op_context *op_ctx, u64 device_addr, u64 size)2478{2479	int err = pvr_mmu_op_context_set_curr_page(op_ctx, device_addr, false);2480 2481	if (err)2482		return err;2483 2484	return pvr_mmu_op_context_unmap_curr_page(op_ctx,2485						  size >> PVR_DEVICE_PAGE_SHIFT);2486}2487 2488/**2489 * pvr_mmu_map_sgl() - Map part of a scatter-gather table entry to2490 * device-virtual memory.2491 * @op_ctx: Target MMU op context pointing to the first page that should be2492 * mapped.2493 * @sgl: Target scatter-gather table entry.2494 * @offset: Offset into @sgl to map from. Must result in a starting address2495 * from @sgl which is CPU page-aligned.2496 * @size: Size of the memory to be mapped in bytes. Must be a non-zero multiple2497 * of the device page size.2498 * @page_flags: Page options to be applied to every device-virtual memory page2499 * in the created mapping.2500 *2501 * Return:2502 *  * 0 on success,2503 *  * -%EINVAL if the range specified by @offset and @size is not completely2504 *    within @sgl, or2505 *  * Any error encountered while creating a page with pvr_page_create(), or2506 *  * Any error encountered while advancing @op_ctx.curr_page with2507 *    pvr_mmu_op_context_next_page().2508 */2509static int2510pvr_mmu_map_sgl(struct pvr_mmu_op_context *op_ctx, struct scatterlist *sgl,2511		u64 offset, u64 size, struct pvr_page_flags_raw page_flags)2512{2513	const unsigned int pages = size >> PVR_DEVICE_PAGE_SHIFT;2514	dma_addr_t dma_addr = sg_dma_address(sgl) + offset;2515	const unsigned int dma_len = sg_dma_len(sgl);2516	struct pvr_page_table_ptr ptr_copy;2517	unsigned int page;2518	int err;2519 2520	if (size > dma_len || offset > dma_len - size)2521		return -EINVAL;2522 2523	/*2524	 * Before progressing, save a copy of the start pointer so we can use2525	 * it again if we enter an error state and have to destroy pages.2526	 */2527	memcpy(&ptr_copy, &op_ctx->curr_page, sizeof(ptr_copy));2528 2529	/*2530	 * Create first page outside loop, as it doesn't require a page advance2531	 * beforehand.2532	 */2533	err = pvr_page_create(op_ctx, dma_addr, page_flags);2534	if (err)2535		return err;2536 2537	for (page = 1; page < pages; ++page) {2538		err = pvr_mmu_op_context_next_page(op_ctx, true);2539		if (err)2540			goto err_destroy_pages;2541 2542		dma_addr += PVR_DEVICE_PAGE_SIZE;2543 2544		err = pvr_page_create(op_ctx, dma_addr, page_flags);2545		if (err)2546			goto err_destroy_pages;2547	}2548 2549	return 0;2550 2551err_destroy_pages:2552	memcpy(&op_ctx->curr_page, &ptr_copy, sizeof(op_ctx->curr_page));2553	err = pvr_mmu_op_context_unmap_curr_page(op_ctx, page);2554 2555	return err;2556}2557 2558/**2559 * pvr_mmu_map() - Map an object's virtual memory to physical memory.2560 * @op_ctx: Target MMU op context.2561 * @size: Size of memory to be mapped in bytes. Must be a non-zero multiple2562 * of the device page size.2563 * @flags: Flags from pvr_gem_object associated with the mapping.2564 * @device_addr: Virtual device address to map to. Must be device page-aligned.2565 *2566 * Returns:2567 *  * 0 on success, or2568 *  * Any error code returned by pvr_page_table_ptr_init(), or2569 *  * Any error code returned by pvr_mmu_map_sgl(), or2570 *  * Any error code returned by pvr_page_table_ptr_next_page().2571 */2572int pvr_mmu_map(struct pvr_mmu_op_context *op_ctx, u64 size, u64 flags,2573		u64 device_addr)2574{2575	struct pvr_page_table_ptr ptr_copy;2576	struct pvr_page_flags_raw flags_raw;2577	struct scatterlist *sgl;2578	u64 mapped_size = 0;2579	unsigned int count;2580	int err;2581 2582	if (!size)2583		return 0;2584 2585	if ((op_ctx->map.sgt_offset | size) & ~PVR_DEVICE_PAGE_MASK)2586		return -EINVAL;2587 2588	err = pvr_mmu_op_context_set_curr_page(op_ctx, device_addr, true);2589	if (err)2590		return -EINVAL;2591 2592	memcpy(&ptr_copy, &op_ctx->curr_page, sizeof(ptr_copy));2593 2594	flags_raw = pvr_page_flags_raw_create(false, false,2595					      flags & DRM_PVR_BO_BYPASS_DEVICE_CACHE,2596					      flags & DRM_PVR_BO_PM_FW_PROTECT);2597 2598	/* Map scatter gather table */2599	for_each_sgtable_dma_sg(op_ctx->map.sgt, sgl, count) {2600		const size_t sgl_len = sg_dma_len(sgl);2601		u64 sgl_offset, map_sgl_len;2602 2603		if (sgl_len <= op_ctx->map.sgt_offset) {2604			op_ctx->map.sgt_offset -= sgl_len;2605			continue;2606		}2607 2608		sgl_offset = op_ctx->map.sgt_offset;2609		map_sgl_len = min_t(u64, sgl_len - sgl_offset, size - mapped_size);2610 2611		err = pvr_mmu_map_sgl(op_ctx, sgl, sgl_offset, map_sgl_len,2612				      flags_raw);2613		if (err)2614			break;2615 2616		/*2617		 * Flag the L0 page table as requiring a flush when the MMU op2618		 * context is destroyed.2619		 */2620		pvr_mmu_op_context_require_sync(op_ctx, PVR_MMU_SYNC_LEVEL_0);2621 2622		op_ctx->map.sgt_offset = 0;2623		mapped_size += map_sgl_len;2624 2625		if (mapped_size >= size)2626			break;2627 2628		err = pvr_mmu_op_context_next_page(op_ctx, true);2629		if (err)2630			break;2631	}2632 2633	if (err && mapped_size) {2634		memcpy(&op_ctx->curr_page, &ptr_copy, sizeof(op_ctx->curr_page));2635		pvr_mmu_op_context_unmap_curr_page(op_ctx,2636						   mapped_size >> PVR_DEVICE_PAGE_SHIFT);2637	}2638 2639	return err;2640}2641