639 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2/*3 * Copyright (C) 2020-2023 Intel Corporation4 */5 6#include <linux/bitfield.h>7#include <linux/highmem.h>8#include <linux/set_memory.h>9#include <linux/vmalloc.h>10 11#include <drm/drm_cache.h>12 13#include "ivpu_drv.h"14#include "ivpu_hw.h"15#include "ivpu_mmu.h"16#include "ivpu_mmu_context.h"17 18#define IVPU_MMU_VPU_ADDRESS_MASK GENMASK(47, 12)19#define IVPU_MMU_PGD_INDEX_MASK GENMASK(47, 39)20#define IVPU_MMU_PUD_INDEX_MASK GENMASK(38, 30)21#define IVPU_MMU_PMD_INDEX_MASK GENMASK(29, 21)22#define IVPU_MMU_PTE_INDEX_MASK GENMASK(20, 12)23#define IVPU_MMU_ENTRY_FLAGS_MASK (BIT(52) | GENMASK(11, 0))24#define IVPU_MMU_ENTRY_FLAG_CONT BIT(52)25#define IVPU_MMU_ENTRY_FLAG_NG BIT(11)26#define IVPU_MMU_ENTRY_FLAG_AF BIT(10)27#define IVPU_MMU_ENTRY_FLAG_RO BIT(7)28#define IVPU_MMU_ENTRY_FLAG_USER BIT(6)29#define IVPU_MMU_ENTRY_FLAG_LLC_COHERENT BIT(2)30#define IVPU_MMU_ENTRY_FLAG_TYPE_PAGE BIT(1)31#define IVPU_MMU_ENTRY_FLAG_VALID BIT(0)32 33#define IVPU_MMU_PAGE_SIZE SZ_4K34#define IVPU_MMU_CONT_PAGES_SIZE (IVPU_MMU_PAGE_SIZE * 16)35#define IVPU_MMU_PTE_MAP_SIZE (IVPU_MMU_PGTABLE_ENTRIES * IVPU_MMU_PAGE_SIZE)36#define IVPU_MMU_PMD_MAP_SIZE (IVPU_MMU_PGTABLE_ENTRIES * IVPU_MMU_PTE_MAP_SIZE)37#define IVPU_MMU_PUD_MAP_SIZE (IVPU_MMU_PGTABLE_ENTRIES * IVPU_MMU_PMD_MAP_SIZE)38#define IVPU_MMU_PGD_MAP_SIZE (IVPU_MMU_PGTABLE_ENTRIES * IVPU_MMU_PUD_MAP_SIZE)39#define IVPU_MMU_PGTABLE_SIZE (IVPU_MMU_PGTABLE_ENTRIES * sizeof(u64))40 41#define IVPU_MMU_DUMMY_ADDRESS 0xdeadb00042#define IVPU_MMU_ENTRY_VALID (IVPU_MMU_ENTRY_FLAG_TYPE_PAGE | IVPU_MMU_ENTRY_FLAG_VALID)43#define IVPU_MMU_ENTRY_INVALID (IVPU_MMU_DUMMY_ADDRESS & ~IVPU_MMU_ENTRY_FLAGS_MASK)44#define IVPU_MMU_ENTRY_MAPPED (IVPU_MMU_ENTRY_FLAG_AF | IVPU_MMU_ENTRY_FLAG_USER | \45 IVPU_MMU_ENTRY_FLAG_NG | IVPU_MMU_ENTRY_VALID)46 47static void *ivpu_pgtable_alloc_page(struct ivpu_device *vdev, dma_addr_t *dma)48{49 dma_addr_t dma_addr;50 struct page *page;51 void *cpu;52 53 page = alloc_page(GFP_KERNEL | __GFP_HIGHMEM | __GFP_ZERO);54 if (!page)55 return NULL;56 57 set_pages_array_wc(&page, 1);58 59 dma_addr = dma_map_page(vdev->drm.dev, page, 0, PAGE_SIZE, DMA_BIDIRECTIONAL);60 if (dma_mapping_error(vdev->drm.dev, dma_addr))61 goto err_free_page;62 63 cpu = vmap(&page, 1, VM_MAP, pgprot_writecombine(PAGE_KERNEL));64 if (!cpu)65 goto err_dma_unmap_page;66 67 68 *dma = dma_addr;69 return cpu;70 71err_dma_unmap_page:72 dma_unmap_page(vdev->drm.dev, dma_addr, PAGE_SIZE, DMA_BIDIRECTIONAL);73 74err_free_page:75 put_page(page);76 return NULL;77}78 79static void ivpu_pgtable_free_page(struct ivpu_device *vdev, u64 *cpu_addr, dma_addr_t dma_addr)80{81 struct page *page;82 83 if (cpu_addr) {84 page = vmalloc_to_page(cpu_addr);85 vunmap(cpu_addr);86 dma_unmap_page(vdev->drm.dev, dma_addr & ~IVPU_MMU_ENTRY_FLAGS_MASK, PAGE_SIZE,87 DMA_BIDIRECTIONAL);88 set_pages_array_wb(&page, 1);89 put_page(page);90 }91}92 93static int ivpu_mmu_pgtable_init(struct ivpu_device *vdev, struct ivpu_mmu_pgtable *pgtable)94{95 dma_addr_t pgd_dma;96 97 pgtable->pgd_dma_ptr = ivpu_pgtable_alloc_page(vdev, &pgd_dma);98 if (!pgtable->pgd_dma_ptr)99 return -ENOMEM;100 101 pgtable->pgd_dma = pgd_dma;102 103 return 0;104}105 106static void ivpu_mmu_pgtables_free(struct ivpu_device *vdev, struct ivpu_mmu_pgtable *pgtable)107{108 int pgd_idx, pud_idx, pmd_idx;109 dma_addr_t pud_dma, pmd_dma, pte_dma;110 u64 *pud_dma_ptr, *pmd_dma_ptr, *pte_dma_ptr;111 112 for (pgd_idx = 0; pgd_idx < IVPU_MMU_PGTABLE_ENTRIES; ++pgd_idx) {113 pud_dma_ptr = pgtable->pud_ptrs[pgd_idx];114 pud_dma = pgtable->pgd_dma_ptr[pgd_idx];115 116 if (!pud_dma_ptr)117 continue;118 119 for (pud_idx = 0; pud_idx < IVPU_MMU_PGTABLE_ENTRIES; ++pud_idx) {120 pmd_dma_ptr = pgtable->pmd_ptrs[pgd_idx][pud_idx];121 pmd_dma = pgtable->pud_ptrs[pgd_idx][pud_idx];122 123 if (!pmd_dma_ptr)124 continue;125 126 for (pmd_idx = 0; pmd_idx < IVPU_MMU_PGTABLE_ENTRIES; ++pmd_idx) {127 pte_dma_ptr = pgtable->pte_ptrs[pgd_idx][pud_idx][pmd_idx];128 pte_dma = pgtable->pmd_ptrs[pgd_idx][pud_idx][pmd_idx];129 130 ivpu_pgtable_free_page(vdev, pte_dma_ptr, pte_dma);131 }132 133 kfree(pgtable->pte_ptrs[pgd_idx][pud_idx]);134 ivpu_pgtable_free_page(vdev, pmd_dma_ptr, pmd_dma);135 }136 137 kfree(pgtable->pmd_ptrs[pgd_idx]);138 kfree(pgtable->pte_ptrs[pgd_idx]);139 ivpu_pgtable_free_page(vdev, pud_dma_ptr, pud_dma);140 }141 142 ivpu_pgtable_free_page(vdev, pgtable->pgd_dma_ptr, pgtable->pgd_dma);143}144 145static u64*146ivpu_mmu_ensure_pud(struct ivpu_device *vdev, struct ivpu_mmu_pgtable *pgtable, int pgd_idx)147{148 u64 *pud_dma_ptr = pgtable->pud_ptrs[pgd_idx];149 dma_addr_t pud_dma;150 151 if (pud_dma_ptr)152 return pud_dma_ptr;153 154 pud_dma_ptr = ivpu_pgtable_alloc_page(vdev, &pud_dma);155 if (!pud_dma_ptr)156 return NULL;157 158 drm_WARN_ON(&vdev->drm, pgtable->pmd_ptrs[pgd_idx]);159 pgtable->pmd_ptrs[pgd_idx] = kzalloc(IVPU_MMU_PGTABLE_SIZE, GFP_KERNEL);160 if (!pgtable->pmd_ptrs[pgd_idx])161 goto err_free_pud_dma_ptr;162 163 drm_WARN_ON(&vdev->drm, pgtable->pte_ptrs[pgd_idx]);164 pgtable->pte_ptrs[pgd_idx] = kzalloc(IVPU_MMU_PGTABLE_SIZE, GFP_KERNEL);165 if (!pgtable->pte_ptrs[pgd_idx])166 goto err_free_pmd_ptrs;167 168 pgtable->pud_ptrs[pgd_idx] = pud_dma_ptr;169 pgtable->pgd_dma_ptr[pgd_idx] = pud_dma | IVPU_MMU_ENTRY_VALID;170 171 return pud_dma_ptr;172 173err_free_pmd_ptrs:174 kfree(pgtable->pmd_ptrs[pgd_idx]);175 176err_free_pud_dma_ptr:177 ivpu_pgtable_free_page(vdev, pud_dma_ptr, pud_dma);178 return NULL;179}180 181static u64*182ivpu_mmu_ensure_pmd(struct ivpu_device *vdev, struct ivpu_mmu_pgtable *pgtable, int pgd_idx,183 int pud_idx)184{185 u64 *pmd_dma_ptr = pgtable->pmd_ptrs[pgd_idx][pud_idx];186 dma_addr_t pmd_dma;187 188 if (pmd_dma_ptr)189 return pmd_dma_ptr;190 191 pmd_dma_ptr = ivpu_pgtable_alloc_page(vdev, &pmd_dma);192 if (!pmd_dma_ptr)193 return NULL;194 195 drm_WARN_ON(&vdev->drm, pgtable->pte_ptrs[pgd_idx][pud_idx]);196 pgtable->pte_ptrs[pgd_idx][pud_idx] = kzalloc(IVPU_MMU_PGTABLE_SIZE, GFP_KERNEL);197 if (!pgtable->pte_ptrs[pgd_idx][pud_idx])198 goto err_free_pmd_dma_ptr;199 200 pgtable->pmd_ptrs[pgd_idx][pud_idx] = pmd_dma_ptr;201 pgtable->pud_ptrs[pgd_idx][pud_idx] = pmd_dma | IVPU_MMU_ENTRY_VALID;202 203 return pmd_dma_ptr;204 205err_free_pmd_dma_ptr:206 ivpu_pgtable_free_page(vdev, pmd_dma_ptr, pmd_dma);207 return NULL;208}209 210static u64*211ivpu_mmu_ensure_pte(struct ivpu_device *vdev, struct ivpu_mmu_pgtable *pgtable,212 int pgd_idx, int pud_idx, int pmd_idx)213{214 u64 *pte_dma_ptr = pgtable->pte_ptrs[pgd_idx][pud_idx][pmd_idx];215 dma_addr_t pte_dma;216 217 if (pte_dma_ptr)218 return pte_dma_ptr;219 220 pte_dma_ptr = ivpu_pgtable_alloc_page(vdev, &pte_dma);221 if (!pte_dma_ptr)222 return NULL;223 224 pgtable->pte_ptrs[pgd_idx][pud_idx][pmd_idx] = pte_dma_ptr;225 pgtable->pmd_ptrs[pgd_idx][pud_idx][pmd_idx] = pte_dma | IVPU_MMU_ENTRY_VALID;226 227 return pte_dma_ptr;228}229 230static int231ivpu_mmu_context_map_page(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx,232 u64 vpu_addr, dma_addr_t dma_addr, u64 prot)233{234 u64 *pte;235 int pgd_idx = FIELD_GET(IVPU_MMU_PGD_INDEX_MASK, vpu_addr);236 int pud_idx = FIELD_GET(IVPU_MMU_PUD_INDEX_MASK, vpu_addr);237 int pmd_idx = FIELD_GET(IVPU_MMU_PMD_INDEX_MASK, vpu_addr);238 int pte_idx = FIELD_GET(IVPU_MMU_PTE_INDEX_MASK, vpu_addr);239 240 /* Allocate PUD - second level page table if needed */241 if (!ivpu_mmu_ensure_pud(vdev, &ctx->pgtable, pgd_idx))242 return -ENOMEM;243 244 /* Allocate PMD - third level page table if needed */245 if (!ivpu_mmu_ensure_pmd(vdev, &ctx->pgtable, pgd_idx, pud_idx))246 return -ENOMEM;247 248 /* Allocate PTE - fourth level page table if needed */249 pte = ivpu_mmu_ensure_pte(vdev, &ctx->pgtable, pgd_idx, pud_idx, pmd_idx);250 if (!pte)251 return -ENOMEM;252 253 /* Update PTE */254 pte[pte_idx] = dma_addr | prot;255 256 return 0;257}258 259static int260ivpu_mmu_context_map_cont_64k(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx, u64 vpu_addr,261 dma_addr_t dma_addr, u64 prot)262{263 size_t size = IVPU_MMU_CONT_PAGES_SIZE;264 265 drm_WARN_ON(&vdev->drm, !IS_ALIGNED(vpu_addr, size));266 drm_WARN_ON(&vdev->drm, !IS_ALIGNED(dma_addr, size));267 268 prot |= IVPU_MMU_ENTRY_FLAG_CONT;269 270 while (size) {271 int ret = ivpu_mmu_context_map_page(vdev, ctx, vpu_addr, dma_addr, prot);272 273 if (ret)274 return ret;275 276 size -= IVPU_MMU_PAGE_SIZE;277 vpu_addr += IVPU_MMU_PAGE_SIZE;278 dma_addr += IVPU_MMU_PAGE_SIZE;279 }280 281 return 0;282}283 284static void ivpu_mmu_context_unmap_page(struct ivpu_mmu_context *ctx, u64 vpu_addr)285{286 int pgd_idx = FIELD_GET(IVPU_MMU_PGD_INDEX_MASK, vpu_addr);287 int pud_idx = FIELD_GET(IVPU_MMU_PUD_INDEX_MASK, vpu_addr);288 int pmd_idx = FIELD_GET(IVPU_MMU_PMD_INDEX_MASK, vpu_addr);289 int pte_idx = FIELD_GET(IVPU_MMU_PTE_INDEX_MASK, vpu_addr);290 291 /* Update PTE with dummy physical address and clear flags */292 ctx->pgtable.pte_ptrs[pgd_idx][pud_idx][pmd_idx][pte_idx] = IVPU_MMU_ENTRY_INVALID;293}294 295static int296ivpu_mmu_context_map_pages(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx,297 u64 vpu_addr, dma_addr_t dma_addr, size_t size, u64 prot)298{299 int map_size;300 int ret;301 302 while (size) {303 if (!ivpu_disable_mmu_cont_pages && size >= IVPU_MMU_CONT_PAGES_SIZE &&304 IS_ALIGNED(vpu_addr | dma_addr, IVPU_MMU_CONT_PAGES_SIZE)) {305 ret = ivpu_mmu_context_map_cont_64k(vdev, ctx, vpu_addr, dma_addr, prot);306 map_size = IVPU_MMU_CONT_PAGES_SIZE;307 } else {308 ret = ivpu_mmu_context_map_page(vdev, ctx, vpu_addr, dma_addr, prot);309 map_size = IVPU_MMU_PAGE_SIZE;310 }311 312 if (ret)313 return ret;314 315 vpu_addr += map_size;316 dma_addr += map_size;317 size -= map_size;318 }319 320 return 0;321}322 323static void ivpu_mmu_context_set_page_ro(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx,324 u64 vpu_addr)325{326 int pgd_idx = FIELD_GET(IVPU_MMU_PGD_INDEX_MASK, vpu_addr);327 int pud_idx = FIELD_GET(IVPU_MMU_PUD_INDEX_MASK, vpu_addr);328 int pmd_idx = FIELD_GET(IVPU_MMU_PMD_INDEX_MASK, vpu_addr);329 int pte_idx = FIELD_GET(IVPU_MMU_PTE_INDEX_MASK, vpu_addr);330 331 ctx->pgtable.pte_ptrs[pgd_idx][pud_idx][pmd_idx][pte_idx] |= IVPU_MMU_ENTRY_FLAG_RO;332}333 334static void ivpu_mmu_context_split_page(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx,335 u64 vpu_addr)336{337 int pgd_idx = FIELD_GET(IVPU_MMU_PGD_INDEX_MASK, vpu_addr);338 int pud_idx = FIELD_GET(IVPU_MMU_PUD_INDEX_MASK, vpu_addr);339 int pmd_idx = FIELD_GET(IVPU_MMU_PMD_INDEX_MASK, vpu_addr);340 int pte_idx = FIELD_GET(IVPU_MMU_PTE_INDEX_MASK, vpu_addr);341 342 ctx->pgtable.pte_ptrs[pgd_idx][pud_idx][pmd_idx][pte_idx] &= ~IVPU_MMU_ENTRY_FLAG_CONT;343}344 345static void ivpu_mmu_context_split_64k_page(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx,346 u64 vpu_addr)347{348 u64 start = ALIGN_DOWN(vpu_addr, IVPU_MMU_CONT_PAGES_SIZE);349 u64 end = ALIGN(vpu_addr, IVPU_MMU_CONT_PAGES_SIZE);350 u64 offset = 0;351 352 ivpu_dbg(vdev, MMU_MAP, "Split 64K page ctx: %u vpu_addr: 0x%llx\n", ctx->id, vpu_addr);353 354 while (start + offset < end) {355 ivpu_mmu_context_split_page(vdev, ctx, start + offset);356 offset += IVPU_MMU_PAGE_SIZE;357 }358}359 360int361ivpu_mmu_context_set_pages_ro(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx, u64 vpu_addr,362 size_t size)363{364 u64 end = vpu_addr + size;365 size_t size_left = size;366 int ret;367 368 if (size == 0)369 return 0;370 371 if (drm_WARN_ON(&vdev->drm, !IS_ALIGNED(vpu_addr | size, IVPU_MMU_PAGE_SIZE)))372 return -EINVAL;373 374 mutex_lock(&ctx->lock);375 376 ivpu_dbg(vdev, MMU_MAP, "Set read-only pages ctx: %u vpu_addr: 0x%llx size: %lu\n",377 ctx->id, vpu_addr, size);378 379 if (!ivpu_disable_mmu_cont_pages) {380 /* Split 64K contiguous page at the beginning if needed */381 if (!IS_ALIGNED(vpu_addr, IVPU_MMU_CONT_PAGES_SIZE))382 ivpu_mmu_context_split_64k_page(vdev, ctx, vpu_addr);383 384 /* Split 64K contiguous page at the end if needed */385 if (!IS_ALIGNED(vpu_addr + size, IVPU_MMU_CONT_PAGES_SIZE))386 ivpu_mmu_context_split_64k_page(vdev, ctx, vpu_addr + size);387 }388 389 while (size_left) {390 if (vpu_addr < end)391 ivpu_mmu_context_set_page_ro(vdev, ctx, vpu_addr);392 393 vpu_addr += IVPU_MMU_PAGE_SIZE;394 size_left -= IVPU_MMU_PAGE_SIZE;395 }396 397 /* Ensure page table modifications are flushed from wc buffers to memory */398 wmb();399 400 mutex_unlock(&ctx->lock);401 ret = ivpu_mmu_invalidate_tlb(vdev, ctx->id);402 if (ret)403 ivpu_err(vdev, "Failed to invalidate TLB for ctx %u: %d\n", ctx->id, ret);404 405 return 0;406}407 408static void ivpu_mmu_context_unmap_pages(struct ivpu_mmu_context *ctx, u64 vpu_addr, size_t size)409{410 while (size) {411 ivpu_mmu_context_unmap_page(ctx, vpu_addr);412 vpu_addr += IVPU_MMU_PAGE_SIZE;413 size -= IVPU_MMU_PAGE_SIZE;414 }415}416 417int418ivpu_mmu_context_map_sgt(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx,419 u64 vpu_addr, struct sg_table *sgt, bool llc_coherent)420{421 struct scatterlist *sg;422 int ret;423 u64 prot;424 u64 i;425 426 if (drm_WARN_ON(&vdev->drm, !ctx))427 return -EINVAL;428 429 if (!IS_ALIGNED(vpu_addr, IVPU_MMU_PAGE_SIZE))430 return -EINVAL;431 432 if (vpu_addr & ~IVPU_MMU_VPU_ADDRESS_MASK)433 return -EINVAL;434 435 prot = IVPU_MMU_ENTRY_MAPPED;436 if (llc_coherent)437 prot |= IVPU_MMU_ENTRY_FLAG_LLC_COHERENT;438 439 mutex_lock(&ctx->lock);440 441 for_each_sgtable_dma_sg(sgt, sg, i) {442 dma_addr_t dma_addr = sg_dma_address(sg) - sg->offset;443 size_t size = sg_dma_len(sg) + sg->offset;444 445 ivpu_dbg(vdev, MMU_MAP, "Map ctx: %u dma_addr: 0x%llx vpu_addr: 0x%llx size: %lu\n",446 ctx->id, dma_addr, vpu_addr, size);447 448 ret = ivpu_mmu_context_map_pages(vdev, ctx, vpu_addr, dma_addr, size, prot);449 if (ret) {450 ivpu_err(vdev, "Failed to map context pages\n");451 mutex_unlock(&ctx->lock);452 return ret;453 }454 vpu_addr += size;455 }456 457 /* Ensure page table modifications are flushed from wc buffers to memory */458 wmb();459 460 mutex_unlock(&ctx->lock);461 462 ret = ivpu_mmu_invalidate_tlb(vdev, ctx->id);463 if (ret)464 ivpu_err(vdev, "Failed to invalidate TLB for ctx %u: %d\n", ctx->id, ret);465 return ret;466}467 468void469ivpu_mmu_context_unmap_sgt(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx,470 u64 vpu_addr, struct sg_table *sgt)471{472 struct scatterlist *sg;473 int ret;474 u64 i;475 476 if (drm_WARN_ON(&vdev->drm, !ctx))477 return;478 479 mutex_lock(&ctx->lock);480 481 for_each_sgtable_dma_sg(sgt, sg, i) {482 dma_addr_t dma_addr = sg_dma_address(sg) - sg->offset;483 size_t size = sg_dma_len(sg) + sg->offset;484 485 ivpu_dbg(vdev, MMU_MAP, "Unmap ctx: %u dma_addr: 0x%llx vpu_addr: 0x%llx size: %lu\n",486 ctx->id, dma_addr, vpu_addr, size);487 488 ivpu_mmu_context_unmap_pages(ctx, vpu_addr, size);489 vpu_addr += size;490 }491 492 /* Ensure page table modifications are flushed from wc buffers to memory */493 wmb();494 495 mutex_unlock(&ctx->lock);496 497 ret = ivpu_mmu_invalidate_tlb(vdev, ctx->id);498 if (ret)499 ivpu_warn(vdev, "Failed to invalidate TLB for ctx %u: %d\n", ctx->id, ret);500}501 502int503ivpu_mmu_context_insert_node(struct ivpu_mmu_context *ctx, const struct ivpu_addr_range *range,504 u64 size, struct drm_mm_node *node)505{506 int ret;507 508 WARN_ON(!range);509 510 mutex_lock(&ctx->lock);511 if (!ivpu_disable_mmu_cont_pages && size >= IVPU_MMU_CONT_PAGES_SIZE) {512 ret = drm_mm_insert_node_in_range(&ctx->mm, node, size, IVPU_MMU_CONT_PAGES_SIZE, 0,513 range->start, range->end, DRM_MM_INSERT_BEST);514 if (!ret)515 goto unlock;516 }517 518 ret = drm_mm_insert_node_in_range(&ctx->mm, node, size, IVPU_MMU_PAGE_SIZE, 0,519 range->start, range->end, DRM_MM_INSERT_BEST);520unlock:521 mutex_unlock(&ctx->lock);522 return ret;523}524 525void526ivpu_mmu_context_remove_node(struct ivpu_mmu_context *ctx, struct drm_mm_node *node)527{528 mutex_lock(&ctx->lock);529 drm_mm_remove_node(node);530 mutex_unlock(&ctx->lock);531}532 533static int534ivpu_mmu_context_init(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx, u32 context_id)535{536 u64 start, end;537 int ret;538 539 mutex_init(&ctx->lock);540 541 ret = ivpu_mmu_pgtable_init(vdev, &ctx->pgtable);542 if (ret) {543 ivpu_err(vdev, "Failed to initialize pgtable for ctx %u: %d\n", context_id, ret);544 return ret;545 }546 547 if (!context_id) {548 start = vdev->hw->ranges.global.start;549 end = vdev->hw->ranges.shave.end;550 } else {551 start = vdev->hw->ranges.user.start;552 end = vdev->hw->ranges.dma.end;553 }554 555 drm_mm_init(&ctx->mm, start, end - start);556 ctx->id = context_id;557 558 return 0;559}560 561static void ivpu_mmu_context_fini(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx)562{563 if (drm_WARN_ON(&vdev->drm, !ctx->pgtable.pgd_dma_ptr))564 return;565 566 mutex_destroy(&ctx->lock);567 ivpu_mmu_pgtables_free(vdev, &ctx->pgtable);568 drm_mm_takedown(&ctx->mm);569 570 ctx->pgtable.pgd_dma_ptr = NULL;571 ctx->pgtable.pgd_dma = 0;572}573 574int ivpu_mmu_global_context_init(struct ivpu_device *vdev)575{576 return ivpu_mmu_context_init(vdev, &vdev->gctx, IVPU_GLOBAL_CONTEXT_MMU_SSID);577}578 579void ivpu_mmu_global_context_fini(struct ivpu_device *vdev)580{581 return ivpu_mmu_context_fini(vdev, &vdev->gctx);582}583 584int ivpu_mmu_reserved_context_init(struct ivpu_device *vdev)585{586 return ivpu_mmu_user_context_init(vdev, &vdev->rctx, IVPU_RESERVED_CONTEXT_MMU_SSID);587}588 589void ivpu_mmu_reserved_context_fini(struct ivpu_device *vdev)590{591 return ivpu_mmu_user_context_fini(vdev, &vdev->rctx);592}593 594void ivpu_mmu_user_context_mark_invalid(struct ivpu_device *vdev, u32 ssid)595{596 struct ivpu_file_priv *file_priv;597 598 xa_lock(&vdev->context_xa);599 600 file_priv = xa_load(&vdev->context_xa, ssid);601 if (file_priv)602 file_priv->has_mmu_faults = true;603 604 xa_unlock(&vdev->context_xa);605}606 607int ivpu_mmu_user_context_init(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx, u32 ctx_id)608{609 int ret;610 611 drm_WARN_ON(&vdev->drm, !ctx_id);612 613 ret = ivpu_mmu_context_init(vdev, ctx, ctx_id);614 if (ret) {615 ivpu_err(vdev, "Failed to initialize context %u: %d\n", ctx_id, ret);616 return ret;617 }618 619 ret = ivpu_mmu_set_pgtable(vdev, ctx_id, &ctx->pgtable);620 if (ret) {621 ivpu_err(vdev, "Failed to set page table for context %u: %d\n", ctx_id, ret);622 goto err_context_fini;623 }624 625 return 0;626 627err_context_fini:628 ivpu_mmu_context_fini(vdev, ctx);629 return ret;630}631 632void ivpu_mmu_user_context_fini(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx)633{634 drm_WARN_ON(&vdev->drm, !ctx->id);635 636 ivpu_mmu_clear_pgtable(vdev, ctx->id);637 ivpu_mmu_context_fini(vdev, ctx);638}639