444 lines · c
1/*2 * Copyright 2012 Red Hat Inc.3 *4 * Permission is hereby granted, free of charge, to any person obtaining a5 * copy of this software and associated documentation files (the "Software"),6 * to deal in the Software without restriction, including without limitation7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,8 * and/or sell copies of the Software, and to permit persons to whom the9 * Software is furnished to do so, subject to the following conditions:10 *11 * The above copyright notice and this permission notice shall be included in12 * all copies or substantial portions of the Software.13 *14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR20 * OTHER DEALINGS IN THE SOFTWARE.21 *22 * Authors: Ben Skeggs23 */24#define nv50_instmem(p) container_of((p), struct nv50_instmem, base)25#include "priv.h"26 27#include <core/memory.h>28#include <subdev/bar.h>29#include <subdev/fb.h>30#include <subdev/gsp.h>31#include <subdev/mmu.h>32 33struct nv50_instmem {34 struct nvkm_instmem base;35 u64 addr;36 37 /* Mappings that can be evicted when BAR2 space has been exhausted. */38 struct list_head lru;39};40 41/******************************************************************************42 * instmem object implementation43 *****************************************************************************/44#define nv50_instobj(p) container_of((p), struct nv50_instobj, base.memory)45 46struct nv50_instobj {47 struct nvkm_instobj base;48 struct nv50_instmem *imem;49 struct nvkm_memory *ram;50 struct nvkm_vma *bar;51 refcount_t maps;52 void *map;53 struct list_head lru;54};55 56static void57nv50_instobj_wr32_slow(struct nvkm_memory *memory, u64 offset, u32 data)58{59 struct nv50_instobj *iobj = nv50_instobj(memory);60 struct nv50_instmem *imem = iobj->imem;61 struct nvkm_device *device = imem->base.subdev.device;62 u64 base = (nvkm_memory_addr(iobj->ram) + offset) & 0xffffff00000ULL;63 u64 addr = (nvkm_memory_addr(iobj->ram) + offset) & 0x000000fffffULL;64 unsigned long flags;65 66 spin_lock_irqsave(&imem->base.lock, flags);67 if (unlikely(imem->addr != base)) {68 nvkm_wr32(device, 0x001700, base >> 16);69 imem->addr = base;70 }71 nvkm_wr32(device, 0x700000 + addr, data);72 spin_unlock_irqrestore(&imem->base.lock, flags);73}74 75static u3276nv50_instobj_rd32_slow(struct nvkm_memory *memory, u64 offset)77{78 struct nv50_instobj *iobj = nv50_instobj(memory);79 struct nv50_instmem *imem = iobj->imem;80 struct nvkm_device *device = imem->base.subdev.device;81 u64 base = (nvkm_memory_addr(iobj->ram) + offset) & 0xffffff00000ULL;82 u64 addr = (nvkm_memory_addr(iobj->ram) + offset) & 0x000000fffffULL;83 u32 data;84 unsigned long flags;85 86 spin_lock_irqsave(&imem->base.lock, flags);87 if (unlikely(imem->addr != base)) {88 nvkm_wr32(device, 0x001700, base >> 16);89 imem->addr = base;90 }91 data = nvkm_rd32(device, 0x700000 + addr);92 spin_unlock_irqrestore(&imem->base.lock, flags);93 return data;94}95 96static const struct nvkm_memory_ptrs97nv50_instobj_slow = {98 .rd32 = nv50_instobj_rd32_slow,99 .wr32 = nv50_instobj_wr32_slow,100};101 102static void103nv50_instobj_wr32(struct nvkm_memory *memory, u64 offset, u32 data)104{105 iowrite32_native(data, nv50_instobj(memory)->map + offset);106}107 108static u32109nv50_instobj_rd32(struct nvkm_memory *memory, u64 offset)110{111 return ioread32_native(nv50_instobj(memory)->map + offset);112}113 114static const struct nvkm_memory_ptrs115nv50_instobj_fast = {116 .rd32 = nv50_instobj_rd32,117 .wr32 = nv50_instobj_wr32,118};119 120static void121nv50_instobj_kmap(struct nv50_instobj *iobj, struct nvkm_vmm *vmm)122{123 struct nv50_instmem *imem = iobj->imem;124 struct nv50_instobj *eobj;125 struct nvkm_memory *memory = &iobj->base.memory;126 struct nvkm_subdev *subdev = &imem->base.subdev;127 struct nvkm_device *device = subdev->device;128 struct nvkm_vma *bar = NULL, *ebar;129 u64 size = nvkm_memory_size(memory);130 void *emap;131 int ret;132 133 /* Attempt to allocate BAR2 address-space and map the object134 * into it. The lock has to be dropped while doing this due135 * to the possibility of recursion for page table allocation.136 */137 mutex_unlock(&imem->base.mutex);138 while ((ret = nvkm_vmm_get(vmm, 12, size, &bar))) {139 /* Evict unused mappings, and keep retrying until we either140 * succeed,or there's no more objects left on the LRU.141 */142 mutex_lock(&imem->base.mutex);143 eobj = list_first_entry_or_null(&imem->lru, typeof(*eobj), lru);144 if (eobj) {145 nvkm_debug(subdev, "evict %016llx %016llx @ %016llx\n",146 nvkm_memory_addr(&eobj->base.memory),147 nvkm_memory_size(&eobj->base.memory),148 eobj->bar->addr);149 list_del_init(&eobj->lru);150 ebar = eobj->bar;151 eobj->bar = NULL;152 emap = eobj->map;153 eobj->map = NULL;154 }155 mutex_unlock(&imem->base.mutex);156 if (!eobj)157 break;158 iounmap(emap);159 nvkm_vmm_put(vmm, &ebar);160 }161 162 if (ret == 0)163 ret = nvkm_memory_map(memory, 0, vmm, bar, NULL, 0);164 mutex_lock(&imem->base.mutex);165 if (ret || iobj->bar) {166 /* We either failed, or another thread beat us. */167 mutex_unlock(&imem->base.mutex);168 nvkm_vmm_put(vmm, &bar);169 mutex_lock(&imem->base.mutex);170 return;171 }172 173 /* Make the mapping visible to the host. */174 iobj->bar = bar;175 iobj->map = ioremap_wc(device->func->resource_addr(device, 3) +176 (u32)iobj->bar->addr, size);177 if (!iobj->map) {178 nvkm_warn(subdev, "PRAMIN ioremap failed\n");179 nvkm_vmm_put(vmm, &iobj->bar);180 }181}182 183static int184nv50_instobj_map(struct nvkm_memory *memory, u64 offset, struct nvkm_vmm *vmm,185 struct nvkm_vma *vma, void *argv, u32 argc)186{187 memory = nv50_instobj(memory)->ram;188 return nvkm_memory_map(memory, offset, vmm, vma, argv, argc);189}190 191static void192nv50_instobj_release(struct nvkm_memory *memory)193{194 struct nv50_instobj *iobj = nv50_instobj(memory);195 struct nv50_instmem *imem = iobj->imem;196 struct nvkm_subdev *subdev = &imem->base.subdev;197 198 wmb();199 nvkm_bar_flush(subdev->device->bar);200 201 if (refcount_dec_and_mutex_lock(&iobj->maps, &imem->base.mutex)) {202 /* Add the now-unused mapping to the LRU instead of directly203 * unmapping it here, in case we need to map it again later.204 */205 if (likely(iobj->lru.next) && iobj->map) {206 BUG_ON(!list_empty(&iobj->lru));207 list_add_tail(&iobj->lru, &imem->lru);208 }209 210 /* Switch back to NULL accessors when last map is gone. */211 iobj->base.memory.ptrs = NULL;212 mutex_unlock(&imem->base.mutex);213 }214}215 216static void __iomem *217nv50_instobj_acquire(struct nvkm_memory *memory)218{219 struct nv50_instobj *iobj = nv50_instobj(memory);220 struct nvkm_instmem *imem = &iobj->imem->base;221 struct nvkm_vmm *vmm;222 void __iomem *map = NULL;223 224 /* Already mapped? */225 if (refcount_inc_not_zero(&iobj->maps)) {226 /* read barrier match the wmb on refcount set */227 smp_rmb();228 return iobj->map;229 }230 231 /* Take the lock, and re-check that another thread hasn't232 * already mapped the object in the meantime.233 */234 mutex_lock(&imem->mutex);235 if (refcount_inc_not_zero(&iobj->maps)) {236 mutex_unlock(&imem->mutex);237 return iobj->map;238 }239 240 /* Attempt to get a direct CPU mapping of the object. */241 if ((vmm = nvkm_bar_bar2_vmm(imem->subdev.device))) {242 if (!iobj->map)243 nv50_instobj_kmap(iobj, vmm);244 map = iobj->map;245 }246 247 if (!refcount_inc_not_zero(&iobj->maps)) {248 /* Exclude object from eviction while it's being accessed. */249 if (likely(iobj->lru.next))250 list_del_init(&iobj->lru);251 252 if (map)253 iobj->base.memory.ptrs = &nv50_instobj_fast;254 else255 iobj->base.memory.ptrs = &nv50_instobj_slow;256 /* barrier to ensure the ptrs are written before refcount is set */257 smp_wmb();258 refcount_set(&iobj->maps, 1);259 }260 261 mutex_unlock(&imem->mutex);262 return map;263}264 265static void266nv50_instobj_boot(struct nvkm_memory *memory, struct nvkm_vmm *vmm)267{268 struct nv50_instobj *iobj = nv50_instobj(memory);269 struct nvkm_instmem *imem = &iobj->imem->base;270 271 /* Exclude bootstrapped objects (ie. the page tables for the272 * instmem BAR itself) from eviction.273 */274 mutex_lock(&imem->mutex);275 if (likely(iobj->lru.next)) {276 list_del_init(&iobj->lru);277 iobj->lru.next = NULL;278 }279 280 nv50_instobj_kmap(iobj, vmm);281 nvkm_instmem_boot(imem);282 mutex_unlock(&imem->mutex);283}284 285static u64286nv50_instobj_size(struct nvkm_memory *memory)287{288 return nvkm_memory_size(nv50_instobj(memory)->ram);289}290 291static u64292nv50_instobj_addr(struct nvkm_memory *memory)293{294 return nvkm_memory_addr(nv50_instobj(memory)->ram);295}296 297static u64298nv50_instobj_bar2(struct nvkm_memory *memory)299{300 struct nv50_instobj *iobj = nv50_instobj(memory);301 u64 addr = ~0ULL;302 if (nv50_instobj_acquire(&iobj->base.memory)) {303 iobj->lru.next = NULL; /* Exclude from eviction. */304 addr = iobj->bar->addr;305 }306 nv50_instobj_release(&iobj->base.memory);307 return addr;308}309 310static enum nvkm_memory_target311nv50_instobj_target(struct nvkm_memory *memory)312{313 return nvkm_memory_target(nv50_instobj(memory)->ram);314}315 316static void *317nv50_instobj_dtor(struct nvkm_memory *memory)318{319 struct nv50_instobj *iobj = nv50_instobj(memory);320 struct nvkm_instmem *imem = &iobj->imem->base;321 struct nvkm_vma *bar;322 void *map;323 324 mutex_lock(&imem->mutex);325 if (likely(iobj->lru.next))326 list_del(&iobj->lru);327 map = iobj->map;328 bar = iobj->bar;329 mutex_unlock(&imem->mutex);330 331 if (map) {332 struct nvkm_vmm *vmm = nvkm_bar_bar2_vmm(imem->subdev.device);333 iounmap(map);334 if (likely(vmm)) /* Can be NULL during BAR destructor. */335 nvkm_vmm_put(vmm, &bar);336 }337 338 nvkm_memory_unref(&iobj->ram);339 nvkm_instobj_dtor(imem, &iobj->base);340 return iobj;341}342 343static const struct nvkm_memory_func344nv50_instobj_func = {345 .dtor = nv50_instobj_dtor,346 .target = nv50_instobj_target,347 .bar2 = nv50_instobj_bar2,348 .addr = nv50_instobj_addr,349 .size = nv50_instobj_size,350 .boot = nv50_instobj_boot,351 .acquire = nv50_instobj_acquire,352 .release = nv50_instobj_release,353 .map = nv50_instobj_map,354};355 356static int357nv50_instobj_wrap(struct nvkm_instmem *base,358 struct nvkm_memory *memory, struct nvkm_memory **pmemory)359{360 struct nv50_instmem *imem = nv50_instmem(base);361 struct nv50_instobj *iobj;362 363 if (!(iobj = kzalloc(sizeof(*iobj), GFP_KERNEL)))364 return -ENOMEM;365 *pmemory = &iobj->base.memory;366 367 nvkm_instobj_ctor(&nv50_instobj_func, &imem->base, &iobj->base);368 iobj->imem = imem;369 refcount_set(&iobj->maps, 0);370 INIT_LIST_HEAD(&iobj->lru);371 372 iobj->ram = nvkm_memory_ref(memory);373 return 0;374}375 376static int377nv50_instobj_new(struct nvkm_instmem *imem, u32 size, u32 align, bool zero,378 struct nvkm_memory **pmemory)379{380 u8 page = max(order_base_2(align), 12);381 struct nvkm_memory *ram;382 int ret;383 384 ret = nvkm_ram_get(imem->subdev.device, 0, 1, page, size, true, true, &ram);385 if (ret)386 return ret;387 388 ret = nv50_instobj_wrap(imem, ram, pmemory);389 nvkm_memory_unref(&ram);390 return ret;391}392 393/******************************************************************************394 * instmem subdev implementation395 *****************************************************************************/396 397static void398nv50_instmem_fini(struct nvkm_instmem *base)399{400 nv50_instmem(base)->addr = ~0ULL;401}402 403static void *404nv50_instmem_dtor(struct nvkm_instmem *base)405{406 return nv50_instmem(base);407}408 409static const struct nvkm_instmem_func410nv50_instmem = {411 .dtor = nv50_instmem_dtor,412 .fini = nv50_instmem_fini,413 .suspend = nv04_instmem_suspend,414 .resume = nv04_instmem_resume,415 .memory_new = nv50_instobj_new,416 .memory_wrap = nv50_instobj_wrap,417 .zero = false,418};419 420int421nv50_instmem_new_(const struct nvkm_instmem_func *func,422 struct nvkm_device *device, enum nvkm_subdev_type type, int inst,423 struct nvkm_instmem **pimem)424{425 struct nv50_instmem *imem;426 427 if (!(imem = kzalloc(sizeof(*imem), GFP_KERNEL)))428 return -ENOMEM;429 nvkm_instmem_ctor(func, device, type, inst, &imem->base);430 INIT_LIST_HEAD(&imem->lru);431 *pimem = &imem->base;432 return 0;433}434 435int436nv50_instmem_new(struct nvkm_device *device, enum nvkm_subdev_type type, int inst,437 struct nvkm_instmem **pimem)438{439 if (nvkm_gsp_rm(device->gsp))440 return r535_instmem_new(&nv50_instmem, device, type, inst, pimem);441 442 return nv50_instmem_new_(&nv50_instmem, device, type, inst, pimem);443}444