323 lines · c
1// SPDX-License-Identifier: GPL-2.0 OR MIT2/* Copyright 2017-2019 Qiang Yu <yuq825@gmail.com> */3 4#include <linux/slab.h>5#include <linux/dma-mapping.h>6 7#include "lima_device.h"8#include "lima_vm.h"9#include "lima_gem.h"10#include "lima_regs.h"11 12struct lima_bo_va {13 struct list_head list;14 unsigned int ref_count;15 16 struct drm_mm_node node;17 18 struct lima_vm *vm;19};20 21#define LIMA_VM_PD_SHIFT 2222#define LIMA_VM_PT_SHIFT 1223#define LIMA_VM_PB_SHIFT (LIMA_VM_PD_SHIFT + LIMA_VM_NUM_PT_PER_BT_SHIFT)24#define LIMA_VM_BT_SHIFT LIMA_VM_PT_SHIFT25 26#define LIMA_VM_PT_MASK ((1 << LIMA_VM_PD_SHIFT) - 1)27#define LIMA_VM_BT_MASK ((1 << LIMA_VM_PB_SHIFT) - 1)28 29#define LIMA_PDE(va) (va >> LIMA_VM_PD_SHIFT)30#define LIMA_PTE(va) ((va & LIMA_VM_PT_MASK) >> LIMA_VM_PT_SHIFT)31#define LIMA_PBE(va) (va >> LIMA_VM_PB_SHIFT)32#define LIMA_BTE(va) ((va & LIMA_VM_BT_MASK) >> LIMA_VM_BT_SHIFT)33 34 35static void lima_vm_unmap_range(struct lima_vm *vm, u32 start, u32 end)36{37 u32 addr;38 39 for (addr = start; addr <= end; addr += LIMA_PAGE_SIZE) {40 u32 pbe = LIMA_PBE(addr);41 u32 bte = LIMA_BTE(addr);42 43 vm->bts[pbe].cpu[bte] = 0;44 }45}46 47static int lima_vm_map_page(struct lima_vm *vm, dma_addr_t pa, u32 va)48{49 u32 pbe = LIMA_PBE(va);50 u32 bte = LIMA_BTE(va);51 52 if (!vm->bts[pbe].cpu) {53 dma_addr_t pts;54 u32 *pd;55 int j;56 57 vm->bts[pbe].cpu = dma_alloc_wc(58 vm->dev->dev, LIMA_PAGE_SIZE << LIMA_VM_NUM_PT_PER_BT_SHIFT,59 &vm->bts[pbe].dma, GFP_KERNEL | __GFP_NOWARN | __GFP_ZERO);60 if (!vm->bts[pbe].cpu)61 return -ENOMEM;62 63 pts = vm->bts[pbe].dma;64 pd = vm->pd.cpu + (pbe << LIMA_VM_NUM_PT_PER_BT_SHIFT);65 for (j = 0; j < LIMA_VM_NUM_PT_PER_BT; j++) {66 pd[j] = pts | LIMA_VM_FLAG_PRESENT;67 pts += LIMA_PAGE_SIZE;68 }69 }70 71 vm->bts[pbe].cpu[bte] = pa | LIMA_VM_FLAGS_CACHE;72 73 return 0;74}75 76static struct lima_bo_va *77lima_vm_bo_find(struct lima_vm *vm, struct lima_bo *bo)78{79 struct lima_bo_va *bo_va, *ret = NULL;80 81 list_for_each_entry(bo_va, &bo->va, list) {82 if (bo_va->vm == vm) {83 ret = bo_va;84 break;85 }86 }87 88 return ret;89}90 91int lima_vm_bo_add(struct lima_vm *vm, struct lima_bo *bo, bool create)92{93 struct lima_bo_va *bo_va;94 struct sg_dma_page_iter sg_iter;95 int offset = 0, err;96 97 mutex_lock(&bo->lock);98 99 bo_va = lima_vm_bo_find(vm, bo);100 if (bo_va) {101 bo_va->ref_count++;102 mutex_unlock(&bo->lock);103 return 0;104 }105 106 /* should not create new bo_va if not asked by caller */107 if (!create) {108 mutex_unlock(&bo->lock);109 return -ENOENT;110 }111 112 bo_va = kzalloc(sizeof(*bo_va), GFP_KERNEL);113 if (!bo_va) {114 err = -ENOMEM;115 goto err_out0;116 }117 118 bo_va->vm = vm;119 bo_va->ref_count = 1;120 121 mutex_lock(&vm->lock);122 123 err = drm_mm_insert_node(&vm->mm, &bo_va->node, lima_bo_size(bo));124 if (err)125 goto err_out1;126 127 for_each_sgtable_dma_page(bo->base.sgt, &sg_iter, 0) {128 err = lima_vm_map_page(vm, sg_page_iter_dma_address(&sg_iter),129 bo_va->node.start + offset);130 if (err)131 goto err_out2;132 133 offset += PAGE_SIZE;134 }135 136 mutex_unlock(&vm->lock);137 138 list_add_tail(&bo_va->list, &bo->va);139 140 mutex_unlock(&bo->lock);141 return 0;142 143err_out2:144 if (offset)145 lima_vm_unmap_range(vm, bo_va->node.start, bo_va->node.start + offset - 1);146 drm_mm_remove_node(&bo_va->node);147err_out1:148 mutex_unlock(&vm->lock);149 kfree(bo_va);150err_out0:151 mutex_unlock(&bo->lock);152 return err;153}154 155void lima_vm_bo_del(struct lima_vm *vm, struct lima_bo *bo)156{157 struct lima_bo_va *bo_va;158 u32 size;159 160 mutex_lock(&bo->lock);161 162 bo_va = lima_vm_bo_find(vm, bo);163 if (--bo_va->ref_count > 0) {164 mutex_unlock(&bo->lock);165 return;166 }167 168 mutex_lock(&vm->lock);169 170 size = bo->heap_size ? bo->heap_size : bo_va->node.size;171 lima_vm_unmap_range(vm, bo_va->node.start,172 bo_va->node.start + size - 1);173 174 drm_mm_remove_node(&bo_va->node);175 176 mutex_unlock(&vm->lock);177 178 list_del(&bo_va->list);179 180 mutex_unlock(&bo->lock);181 182 kfree(bo_va);183}184 185u32 lima_vm_get_va(struct lima_vm *vm, struct lima_bo *bo)186{187 struct lima_bo_va *bo_va;188 u32 ret;189 190 mutex_lock(&bo->lock);191 192 bo_va = lima_vm_bo_find(vm, bo);193 ret = bo_va->node.start;194 195 mutex_unlock(&bo->lock);196 197 return ret;198}199 200struct lima_vm *lima_vm_create(struct lima_device *dev)201{202 struct lima_vm *vm;203 204 vm = kzalloc(sizeof(*vm), GFP_KERNEL);205 if (!vm)206 return NULL;207 208 vm->dev = dev;209 mutex_init(&vm->lock);210 kref_init(&vm->refcount);211 212 vm->pd.cpu = dma_alloc_wc(dev->dev, LIMA_PAGE_SIZE, &vm->pd.dma,213 GFP_KERNEL | __GFP_NOWARN | __GFP_ZERO);214 if (!vm->pd.cpu)215 goto err_out0;216 217 if (dev->dlbu_cpu) {218 int err = lima_vm_map_page(219 vm, dev->dlbu_dma, LIMA_VA_RESERVE_DLBU);220 if (err)221 goto err_out1;222 }223 224 drm_mm_init(&vm->mm, dev->va_start, dev->va_end - dev->va_start);225 226 return vm;227 228err_out1:229 dma_free_wc(dev->dev, LIMA_PAGE_SIZE, vm->pd.cpu, vm->pd.dma);230err_out0:231 kfree(vm);232 return NULL;233}234 235void lima_vm_release(struct kref *kref)236{237 struct lima_vm *vm = container_of(kref, struct lima_vm, refcount);238 int i;239 240 drm_mm_takedown(&vm->mm);241 242 for (i = 0; i < LIMA_VM_NUM_BT; i++) {243 if (vm->bts[i].cpu)244 dma_free_wc(vm->dev->dev, LIMA_PAGE_SIZE << LIMA_VM_NUM_PT_PER_BT_SHIFT,245 vm->bts[i].cpu, vm->bts[i].dma);246 }247 248 if (vm->pd.cpu)249 dma_free_wc(vm->dev->dev, LIMA_PAGE_SIZE, vm->pd.cpu, vm->pd.dma);250 251 kfree(vm);252}253 254void lima_vm_print(struct lima_vm *vm)255{256 int i, j, k;257 u32 *pd, *pt;258 259 if (!vm->pd.cpu)260 return;261 262 pd = vm->pd.cpu;263 for (i = 0; i < LIMA_VM_NUM_BT; i++) {264 if (!vm->bts[i].cpu)265 continue;266 267 pt = vm->bts[i].cpu;268 for (j = 0; j < LIMA_VM_NUM_PT_PER_BT; j++) {269 int idx = (i << LIMA_VM_NUM_PT_PER_BT_SHIFT) + j;270 271 printk(KERN_INFO "lima vm pd %03x:%08x\n", idx, pd[idx]);272 273 for (k = 0; k < LIMA_PAGE_ENT_NUM; k++) {274 u32 pte = *pt++;275 276 if (pte)277 printk(KERN_INFO " pt %03x:%08x\n", k, pte);278 }279 }280 }281}282 283int lima_vm_map_bo(struct lima_vm *vm, struct lima_bo *bo, int pageoff)284{285 struct lima_bo_va *bo_va;286 struct sg_dma_page_iter sg_iter;287 int offset = 0, err;288 u32 base;289 290 mutex_lock(&bo->lock);291 292 bo_va = lima_vm_bo_find(vm, bo);293 if (!bo_va) {294 err = -ENOENT;295 goto err_out0;296 }297 298 mutex_lock(&vm->lock);299 300 base = bo_va->node.start + (pageoff << PAGE_SHIFT);301 for_each_sgtable_dma_page(bo->base.sgt, &sg_iter, pageoff) {302 err = lima_vm_map_page(vm, sg_page_iter_dma_address(&sg_iter),303 base + offset);304 if (err)305 goto err_out1;306 307 offset += PAGE_SIZE;308 }309 310 mutex_unlock(&vm->lock);311 312 mutex_unlock(&bo->lock);313 return 0;314 315err_out1:316 if (offset)317 lima_vm_unmap_range(vm, base, base + offset - 1);318 mutex_unlock(&vm->lock);319err_out0:320 mutex_unlock(&bo->lock);321 return err;322}323