315 lines · c
1// SPDX-License-Identifier: GPL-2.02/*3 * Copyright (C) 2016-2018 Etnaviv Project4 */5 6#include <linux/bitops.h>7#include <linux/dma-mapping.h>8#include <linux/platform_device.h>9#include <linux/sizes.h>10#include <linux/slab.h>11#include <linux/vmalloc.h>12 13#include "etnaviv_cmdbuf.h"14#include "etnaviv_gpu.h"15#include "etnaviv_mmu.h"16#include "state.xml.h"17#include "state_hi.xml.h"18 19#define MMUv2_PTE_PRESENT BIT(0)20#define MMUv2_PTE_EXCEPTION BIT(1)21#define MMUv2_PTE_WRITEABLE BIT(2)22 23#define MMUv2_MTLB_MASK 0xffc0000024#define MMUv2_MTLB_SHIFT 2225#define MMUv2_STLB_MASK 0x003ff00026#define MMUv2_STLB_SHIFT 1227 28#define MMUv2_MAX_STLB_ENTRIES 102429 30struct etnaviv_iommuv2_context {31 struct etnaviv_iommu_context base;32 unsigned short id;33 /* M(aster) TLB aka first level pagetable */34 u32 *mtlb_cpu;35 dma_addr_t mtlb_dma;36 /* S(lave) TLB aka second level pagetable */37 u32 *stlb_cpu[MMUv2_MAX_STLB_ENTRIES];38 dma_addr_t stlb_dma[MMUv2_MAX_STLB_ENTRIES];39};40 41static struct etnaviv_iommuv2_context *42to_v2_context(struct etnaviv_iommu_context *context)43{44 return container_of(context, struct etnaviv_iommuv2_context, base);45}46 47static void etnaviv_iommuv2_free(struct etnaviv_iommu_context *context)48{49 struct etnaviv_iommuv2_context *v2_context = to_v2_context(context);50 int i;51 52 drm_mm_takedown(&context->mm);53 54 for (i = 0; i < MMUv2_MAX_STLB_ENTRIES; i++) {55 if (v2_context->stlb_cpu[i])56 dma_free_wc(context->global->dev, SZ_4K,57 v2_context->stlb_cpu[i],58 v2_context->stlb_dma[i]);59 }60 61 dma_free_wc(context->global->dev, SZ_4K, v2_context->mtlb_cpu,62 v2_context->mtlb_dma);63 64 clear_bit(v2_context->id, context->global->v2.pta_alloc);65 66 vfree(v2_context);67}68static int69etnaviv_iommuv2_ensure_stlb(struct etnaviv_iommuv2_context *v2_context,70 int stlb)71{72 if (v2_context->stlb_cpu[stlb])73 return 0;74 75 v2_context->stlb_cpu[stlb] =76 dma_alloc_wc(v2_context->base.global->dev, SZ_4K,77 &v2_context->stlb_dma[stlb],78 GFP_KERNEL);79 80 if (!v2_context->stlb_cpu[stlb])81 return -ENOMEM;82 83 memset32(v2_context->stlb_cpu[stlb], MMUv2_PTE_EXCEPTION,84 SZ_4K / sizeof(u32));85 86 v2_context->mtlb_cpu[stlb] =87 v2_context->stlb_dma[stlb] | MMUv2_PTE_PRESENT;88 89 return 0;90}91 92static int etnaviv_iommuv2_map(struct etnaviv_iommu_context *context,93 unsigned long iova, phys_addr_t paddr,94 size_t size, int prot)95{96 struct etnaviv_iommuv2_context *v2_context = to_v2_context(context);97 int mtlb_entry, stlb_entry, ret;98 u32 entry = lower_32_bits(paddr) | MMUv2_PTE_PRESENT;99 100 if (size != SZ_4K)101 return -EINVAL;102 103 if (IS_ENABLED(CONFIG_PHYS_ADDR_T_64BIT))104 entry |= (upper_32_bits(paddr) & 0xff) << 4;105 106 if (prot & ETNAVIV_PROT_WRITE)107 entry |= MMUv2_PTE_WRITEABLE;108 109 mtlb_entry = (iova & MMUv2_MTLB_MASK) >> MMUv2_MTLB_SHIFT;110 stlb_entry = (iova & MMUv2_STLB_MASK) >> MMUv2_STLB_SHIFT;111 112 ret = etnaviv_iommuv2_ensure_stlb(v2_context, mtlb_entry);113 if (ret)114 return ret;115 116 v2_context->stlb_cpu[mtlb_entry][stlb_entry] = entry;117 118 return 0;119}120 121static size_t etnaviv_iommuv2_unmap(struct etnaviv_iommu_context *context,122 unsigned long iova, size_t size)123{124 struct etnaviv_iommuv2_context *etnaviv_domain = to_v2_context(context);125 int mtlb_entry, stlb_entry;126 127 if (size != SZ_4K)128 return -EINVAL;129 130 mtlb_entry = (iova & MMUv2_MTLB_MASK) >> MMUv2_MTLB_SHIFT;131 stlb_entry = (iova & MMUv2_STLB_MASK) >> MMUv2_STLB_SHIFT;132 133 etnaviv_domain->stlb_cpu[mtlb_entry][stlb_entry] = MMUv2_PTE_EXCEPTION;134 135 return SZ_4K;136}137 138static size_t etnaviv_iommuv2_dump_size(struct etnaviv_iommu_context *context)139{140 struct etnaviv_iommuv2_context *v2_context = to_v2_context(context);141 size_t dump_size = SZ_4K;142 int i;143 144 for (i = 0; i < MMUv2_MAX_STLB_ENTRIES; i++)145 if (v2_context->mtlb_cpu[i] & MMUv2_PTE_PRESENT)146 dump_size += SZ_4K;147 148 return dump_size;149}150 151static void etnaviv_iommuv2_dump(struct etnaviv_iommu_context *context, void *buf)152{153 struct etnaviv_iommuv2_context *v2_context = to_v2_context(context);154 int i;155 156 memcpy(buf, v2_context->mtlb_cpu, SZ_4K);157 buf += SZ_4K;158 for (i = 0; i < MMUv2_MAX_STLB_ENTRIES; i++)159 if (v2_context->mtlb_cpu[i] & MMUv2_PTE_PRESENT) {160 memcpy(buf, v2_context->stlb_cpu[i], SZ_4K);161 buf += SZ_4K;162 }163}164 165static void etnaviv_iommuv2_restore_nonsec(struct etnaviv_gpu *gpu,166 struct etnaviv_iommu_context *context)167{168 struct etnaviv_iommuv2_context *v2_context = to_v2_context(context);169 u16 prefetch;170 171 /* If the MMU is already enabled the state is still there. */172 if (gpu_read(gpu, VIVS_MMUv2_CONTROL) & VIVS_MMUv2_CONTROL_ENABLE)173 return;174 175 if (gpu->mmu_context)176 etnaviv_iommu_context_put(gpu->mmu_context);177 gpu->mmu_context = etnaviv_iommu_context_get(context);178 179 prefetch = etnaviv_buffer_config_mmuv2(gpu,180 (u32)v2_context->mtlb_dma,181 (u32)context->global->bad_page_dma);182 etnaviv_gpu_start_fe(gpu, (u32)etnaviv_cmdbuf_get_pa(&gpu->buffer),183 prefetch);184 etnaviv_gpu_wait_idle(gpu, 100);185 186 gpu_write(gpu, VIVS_MMUv2_CONTROL, VIVS_MMUv2_CONTROL_ENABLE);187}188 189static void etnaviv_iommuv2_restore_sec(struct etnaviv_gpu *gpu,190 struct etnaviv_iommu_context *context)191{192 struct etnaviv_iommuv2_context *v2_context = to_v2_context(context);193 u16 prefetch;194 195 /* If the MMU is already enabled the state is still there. */196 if (gpu_read(gpu, VIVS_MMUv2_SEC_CONTROL) & VIVS_MMUv2_SEC_CONTROL_ENABLE)197 return;198 199 if (gpu->mmu_context)200 etnaviv_iommu_context_put(gpu->mmu_context);201 gpu->mmu_context = etnaviv_iommu_context_get(context);202 203 gpu_write(gpu, VIVS_MMUv2_PTA_ADDRESS_LOW,204 lower_32_bits(context->global->v2.pta_dma));205 gpu_write(gpu, VIVS_MMUv2_PTA_ADDRESS_HIGH,206 upper_32_bits(context->global->v2.pta_dma));207 gpu_write(gpu, VIVS_MMUv2_PTA_CONTROL, VIVS_MMUv2_PTA_CONTROL_ENABLE);208 209 gpu_write(gpu, VIVS_MMUv2_NONSEC_SAFE_ADDR_LOW,210 lower_32_bits(context->global->bad_page_dma));211 gpu_write(gpu, VIVS_MMUv2_SEC_SAFE_ADDR_LOW,212 lower_32_bits(context->global->bad_page_dma));213 gpu_write(gpu, VIVS_MMUv2_SAFE_ADDRESS_CONFIG,214 VIVS_MMUv2_SAFE_ADDRESS_CONFIG_NON_SEC_SAFE_ADDR_HIGH(215 upper_32_bits(context->global->bad_page_dma)) |216 VIVS_MMUv2_SAFE_ADDRESS_CONFIG_SEC_SAFE_ADDR_HIGH(217 upper_32_bits(context->global->bad_page_dma)));218 219 context->global->v2.pta_cpu[v2_context->id] = v2_context->mtlb_dma |220 VIVS_MMUv2_CONFIGURATION_MODE_MODE4_K;221 222 /* trigger a PTA load through the FE */223 prefetch = etnaviv_buffer_config_pta(gpu, v2_context->id);224 etnaviv_gpu_start_fe(gpu, (u32)etnaviv_cmdbuf_get_pa(&gpu->buffer),225 prefetch);226 etnaviv_gpu_wait_idle(gpu, 100);227 228 gpu_write(gpu, VIVS_MMUv2_SEC_CONTROL, VIVS_MMUv2_SEC_CONTROL_ENABLE);229}230 231u32 etnaviv_iommuv2_get_mtlb_addr(struct etnaviv_iommu_context *context)232{233 struct etnaviv_iommuv2_context *v2_context = to_v2_context(context);234 235 return v2_context->mtlb_dma;236}237 238unsigned short etnaviv_iommuv2_get_pta_id(struct etnaviv_iommu_context *context)239{240 struct etnaviv_iommuv2_context *v2_context = to_v2_context(context);241 242 return v2_context->id;243}244static void etnaviv_iommuv2_restore(struct etnaviv_gpu *gpu,245 struct etnaviv_iommu_context *context)246{247 switch (gpu->sec_mode) {248 case ETNA_SEC_NONE:249 etnaviv_iommuv2_restore_nonsec(gpu, context);250 break;251 case ETNA_SEC_KERNEL:252 etnaviv_iommuv2_restore_sec(gpu, context);253 break;254 default:255 WARN(1, "unhandled GPU security mode\n");256 break;257 }258}259 260const struct etnaviv_iommu_ops etnaviv_iommuv2_ops = {261 .free = etnaviv_iommuv2_free,262 .map = etnaviv_iommuv2_map,263 .unmap = etnaviv_iommuv2_unmap,264 .dump_size = etnaviv_iommuv2_dump_size,265 .dump = etnaviv_iommuv2_dump,266 .restore = etnaviv_iommuv2_restore,267};268 269struct etnaviv_iommu_context *270etnaviv_iommuv2_context_alloc(struct etnaviv_iommu_global *global)271{272 struct etnaviv_iommuv2_context *v2_context;273 struct etnaviv_iommu_context *context;274 275 v2_context = vzalloc(sizeof(*v2_context));276 if (!v2_context)277 return NULL;278 279 mutex_lock(&global->lock);280 v2_context->id = find_first_zero_bit(global->v2.pta_alloc,281 ETNAVIV_PTA_ENTRIES);282 if (v2_context->id < ETNAVIV_PTA_ENTRIES) {283 set_bit(v2_context->id, global->v2.pta_alloc);284 } else {285 mutex_unlock(&global->lock);286 goto out_free;287 }288 mutex_unlock(&global->lock);289 290 v2_context->mtlb_cpu = dma_alloc_wc(global->dev, SZ_4K,291 &v2_context->mtlb_dma, GFP_KERNEL);292 if (!v2_context->mtlb_cpu)293 goto out_free_id;294 295 memset32(v2_context->mtlb_cpu, MMUv2_PTE_EXCEPTION,296 MMUv2_MAX_STLB_ENTRIES);297 298 global->v2.pta_cpu[v2_context->id] = v2_context->mtlb_dma;299 300 context = &v2_context->base;301 context->global = global;302 kref_init(&context->refcount);303 mutex_init(&context->lock);304 INIT_LIST_HEAD(&context->mappings);305 drm_mm_init(&context->mm, SZ_4K, (u64)SZ_1G * 4 - SZ_4K);306 307 return context;308 309out_free_id:310 clear_bit(v2_context->id, global->v2.pta_alloc);311out_free:312 vfree(v2_context);313 return NULL;314}315