427 lines · c
1/*2 * Copyright 2017 Red Hat Inc.3 *4 * Permission is hereby granted, free of charge, to any person obtaining a5 * copy of this software and associated documentation files (the "Software"),6 * to deal in the Software without restriction, including without limitation7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,8 * and/or sell copies of the Software, and to permit persons to whom the9 * Software is furnished to do so, subject to the following conditions:10 *11 * The above copyright notice and this permission notice shall be included in12 * all copies or substantial portions of the Software.13 *14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR20 * OTHER DEALINGS IN THE SOFTWARE.21 */22#include "vmm.h"23 24#include <subdev/fb.h>25#include <subdev/ltc.h>26#include <subdev/timer.h>27 28#include <nvif/if900d.h>29#include <nvif/unpack.h>30 31static inline void32gf100_vmm_pgt_pte(struct nvkm_vmm *vmm, struct nvkm_mmu_pt *pt,33 u32 ptei, u32 ptes, struct nvkm_vmm_map *map, u64 addr)34{35 u64 base = (addr >> 8) | map->type;36 u64 data = base;37 38 if (map->ctag && !(map->next & (1ULL << 44))) {39 while (ptes--) {40 data = base | ((map->ctag >> 1) << 44);41 if (!(map->ctag++ & 1))42 data |= BIT_ULL(60);43 44 VMM_WO064(pt, vmm, ptei++ * 8, data);45 base += map->next;46 }47 } else {48 map->type += ptes * map->ctag;49 50 while (ptes--) {51 VMM_WO064(pt, vmm, ptei++ * 8, data);52 data += map->next;53 }54 }55}56 57void58gf100_vmm_pgt_sgl(struct nvkm_vmm *vmm, struct nvkm_mmu_pt *pt,59 u32 ptei, u32 ptes, struct nvkm_vmm_map *map)60{61 VMM_MAP_ITER_SGL(vmm, pt, ptei, ptes, map, gf100_vmm_pgt_pte);62}63 64void65gf100_vmm_pgt_dma(struct nvkm_vmm *vmm, struct nvkm_mmu_pt *pt,66 u32 ptei, u32 ptes, struct nvkm_vmm_map *map)67{68 if (map->page->shift == PAGE_SHIFT) {69 VMM_SPAM(vmm, "DMAA %08x %08x PTE(s)", ptei, ptes);70 nvkm_kmap(pt->memory);71 while (ptes--) {72 const u64 data = (*map->dma++ >> 8) | map->type;73 VMM_WO064(pt, vmm, ptei++ * 8, data);74 map->type += map->ctag;75 }76 nvkm_done(pt->memory);77 return;78 }79 80 VMM_MAP_ITER_DMA(vmm, pt, ptei, ptes, map, gf100_vmm_pgt_pte);81}82 83void84gf100_vmm_pgt_mem(struct nvkm_vmm *vmm, struct nvkm_mmu_pt *pt,85 u32 ptei, u32 ptes, struct nvkm_vmm_map *map)86{87 VMM_MAP_ITER_MEM(vmm, pt, ptei, ptes, map, gf100_vmm_pgt_pte);88}89 90void91gf100_vmm_pgt_unmap(struct nvkm_vmm *vmm,92 struct nvkm_mmu_pt *pt, u32 ptei, u32 ptes)93{94 VMM_FO064(pt, vmm, ptei * 8, 0ULL, ptes);95}96 97const struct nvkm_vmm_desc_func98gf100_vmm_pgt = {99 .unmap = gf100_vmm_pgt_unmap,100 .mem = gf100_vmm_pgt_mem,101 .dma = gf100_vmm_pgt_dma,102 .sgl = gf100_vmm_pgt_sgl,103};104 105void106gf100_vmm_pgd_pde(struct nvkm_vmm *vmm, struct nvkm_vmm_pt *pgd, u32 pdei)107{108 struct nvkm_vmm_pt *pgt = pgd->pde[pdei];109 struct nvkm_mmu_pt *pd = pgd->pt[0];110 struct nvkm_mmu_pt *pt;111 u64 data = 0;112 113 if ((pt = pgt->pt[0])) {114 switch (nvkm_memory_target(pt->memory)) {115 case NVKM_MEM_TARGET_VRAM: data |= 1ULL << 0; break;116 case NVKM_MEM_TARGET_HOST: data |= 2ULL << 0;117 data |= BIT_ULL(35); /* VOL */118 break;119 case NVKM_MEM_TARGET_NCOH: data |= 3ULL << 0; break;120 default:121 WARN_ON(1);122 return;123 }124 data |= pt->addr >> 8;125 }126 127 if ((pt = pgt->pt[1])) {128 switch (nvkm_memory_target(pt->memory)) {129 case NVKM_MEM_TARGET_VRAM: data |= 1ULL << 32; break;130 case NVKM_MEM_TARGET_HOST: data |= 2ULL << 32;131 data |= BIT_ULL(34); /* VOL */132 break;133 case NVKM_MEM_TARGET_NCOH: data |= 3ULL << 32; break;134 default:135 WARN_ON(1);136 return;137 }138 data |= pt->addr << 24;139 }140 141 nvkm_kmap(pd->memory);142 VMM_WO064(pd, vmm, pdei * 8, data);143 nvkm_done(pd->memory);144}145 146const struct nvkm_vmm_desc_func147gf100_vmm_pgd = {148 .unmap = gf100_vmm_pgt_unmap,149 .pde = gf100_vmm_pgd_pde,150};151 152static const struct nvkm_vmm_desc153gf100_vmm_desc_17_12[] = {154 { SPT, 15, 8, 0x1000, &gf100_vmm_pgt },155 { PGD, 13, 8, 0x1000, &gf100_vmm_pgd },156 {}157};158 159static const struct nvkm_vmm_desc160gf100_vmm_desc_17_17[] = {161 { LPT, 10, 8, 0x1000, &gf100_vmm_pgt },162 { PGD, 13, 8, 0x1000, &gf100_vmm_pgd },163 {}164};165 166static const struct nvkm_vmm_desc167gf100_vmm_desc_16_12[] = {168 { SPT, 14, 8, 0x1000, &gf100_vmm_pgt },169 { PGD, 14, 8, 0x1000, &gf100_vmm_pgd },170 {}171};172 173static const struct nvkm_vmm_desc174gf100_vmm_desc_16_16[] = {175 { LPT, 10, 8, 0x1000, &gf100_vmm_pgt },176 { PGD, 14, 8, 0x1000, &gf100_vmm_pgd },177 {}178};179 180void181gf100_vmm_invalidate_pdb(struct nvkm_vmm *vmm, u64 addr)182{183 struct nvkm_device *device = vmm->mmu->subdev.device;184 nvkm_wr32(device, 0x100cb8, addr);185}186 187void188gf100_vmm_invalidate(struct nvkm_vmm *vmm, u32 type)189{190 struct nvkm_device *device = vmm->mmu->subdev.device;191 struct nvkm_mmu_pt *pd = vmm->pd->pt[0];192 u64 addr = 0;193 194 mutex_lock(&vmm->mmu->mutex);195 /* Looks like maybe a "free flush slots" counter, the196 * faster you write to 0x100cbc to more it decreases.197 */198 nvkm_msec(device, 2000,199 if (nvkm_rd32(device, 0x100c80) & 0x00ff0000)200 break;201 );202 203 if (!(type & 0x00000002) /* ALL_PDB. */) {204 switch (nvkm_memory_target(pd->memory)) {205 case NVKM_MEM_TARGET_VRAM: addr |= 0x00000000; break;206 case NVKM_MEM_TARGET_HOST: addr |= 0x00000002; break;207 case NVKM_MEM_TARGET_NCOH: addr |= 0x00000003; break;208 default:209 WARN_ON(1);210 break;211 }212 addr |= (vmm->pd->pt[0]->addr >> 12) << 4;213 214 vmm->func->invalidate_pdb(vmm, addr);215 }216 217 nvkm_wr32(device, 0x100cbc, 0x80000000 | type);218 219 /* Wait for flush to be queued? */220 nvkm_msec(device, 2000,221 if (nvkm_rd32(device, 0x100c80) & 0x00008000)222 break;223 );224 mutex_unlock(&vmm->mmu->mutex);225}226 227void228gf100_vmm_flush(struct nvkm_vmm *vmm, int depth)229{230 u32 type = 0x00000001; /* PAGE_ALL */231 if (atomic_read(&vmm->engref[NVKM_SUBDEV_BAR]))232 type |= 0x00000004; /* HUB_ONLY */233 gf100_vmm_invalidate(vmm, type);234}235 236int237gf100_vmm_valid(struct nvkm_vmm *vmm, void *argv, u32 argc,238 struct nvkm_vmm_map *map)239{240 const enum nvkm_memory_target target = nvkm_memory_target(map->memory);241 const struct nvkm_vmm_page *page = map->page;242 const bool gm20x = page->desc->func->sparse != NULL;243 union {244 struct gf100_vmm_map_vn vn;245 struct gf100_vmm_map_v0 v0;246 } *args = argv;247 struct nvkm_device *device = vmm->mmu->subdev.device;248 struct nvkm_memory *memory = map->memory;249 u8 kind, kind_inv, priv, ro, vol;250 int kindn, aper, ret = -ENOSYS;251 const u8 *kindm;252 253 map->next = (1 << page->shift) >> 8;254 map->type = map->ctag = 0;255 256 if (!(ret = nvif_unpack(ret, &argv, &argc, args->v0, 0, 0, false))) {257 vol = !!args->v0.vol;258 ro = !!args->v0.ro;259 priv = !!args->v0.priv;260 kind = args->v0.kind;261 } else262 if (!(ret = nvif_unvers(ret, &argv, &argc, args->vn))) {263 vol = target == NVKM_MEM_TARGET_HOST;264 ro = 0;265 priv = 0;266 kind = 0x00;267 } else {268 VMM_DEBUG(vmm, "args");269 return ret;270 }271 272 aper = vmm->func->aper(target);273 if (WARN_ON(aper < 0))274 return aper;275 276 kindm = vmm->mmu->func->kind(vmm->mmu, &kindn, &kind_inv);277 if (kind >= kindn || kindm[kind] == kind_inv) {278 VMM_DEBUG(vmm, "kind %02x", kind);279 return -EINVAL;280 }281 282 if (kindm[kind] != kind) {283 u32 comp = (page->shift == 16 && !gm20x) ? 16 : 17;284 u32 tags = ALIGN(nvkm_memory_size(memory), 1 << 17) >> comp;285 if (aper != 0 || !(page->type & NVKM_VMM_PAGE_COMP)) {286 VMM_DEBUG(vmm, "comp %d %02x", aper, page->type);287 return -EINVAL;288 }289 290 if (!map->no_comp) {291 ret = nvkm_memory_tags_get(memory, device, tags,292 nvkm_ltc_tags_clear,293 &map->tags);294 if (ret) {295 VMM_DEBUG(vmm, "comp %d", ret);296 return ret;297 }298 }299 300 if (!map->no_comp && map->tags->mn) {301 u64 tags = map->tags->mn->offset + (map->offset >> 17);302 if (page->shift == 17 || !gm20x) {303 map->type |= tags << 44;304 map->ctag |= 1ULL << 44;305 map->next |= 1ULL << 44;306 } else {307 map->ctag |= tags << 1 | 1;308 }309 } else {310 kind = kindm[kind];311 }312 }313 314 map->type |= BIT(0);315 map->type |= (u64)priv << 1;316 map->type |= (u64) ro << 2;317 map->type |= (u64) vol << 32;318 map->type |= (u64)aper << 33;319 map->type |= (u64)kind << 36;320 return 0;321}322 323int324gf100_vmm_aper(enum nvkm_memory_target target)325{326 switch (target) {327 case NVKM_MEM_TARGET_VRAM: return 0;328 case NVKM_MEM_TARGET_HOST: return 2;329 case NVKM_MEM_TARGET_NCOH: return 3;330 default:331 return -EINVAL;332 }333}334 335void336gf100_vmm_part(struct nvkm_vmm *vmm, struct nvkm_memory *inst)337{338 nvkm_fo64(inst, 0x0200, 0x00000000, 2);339}340 341int342gf100_vmm_join_(struct nvkm_vmm *vmm, struct nvkm_memory *inst, u64 base)343{344 struct nvkm_mmu_pt *pd = vmm->pd->pt[0];345 346 switch (nvkm_memory_target(pd->memory)) {347 case NVKM_MEM_TARGET_VRAM: base |= 0ULL << 0; break;348 case NVKM_MEM_TARGET_HOST: base |= 2ULL << 0;349 base |= BIT_ULL(2) /* VOL. */;350 break;351 case NVKM_MEM_TARGET_NCOH: base |= 3ULL << 0; break;352 default:353 WARN_ON(1);354 return -EINVAL;355 }356 base |= pd->addr;357 358 nvkm_kmap(inst);359 nvkm_wo64(inst, 0x0200, base);360 nvkm_wo64(inst, 0x0208, vmm->limit - 1);361 nvkm_done(inst);362 return 0;363}364 365int366gf100_vmm_join(struct nvkm_vmm *vmm, struct nvkm_memory *inst)367{368 return gf100_vmm_join_(vmm, inst, 0);369}370 371static const struct nvkm_vmm_func372gf100_vmm_17 = {373 .join = gf100_vmm_join,374 .part = gf100_vmm_part,375 .aper = gf100_vmm_aper,376 .valid = gf100_vmm_valid,377 .flush = gf100_vmm_flush,378 .invalidate_pdb = gf100_vmm_invalidate_pdb,379 .page = {380 { 17, &gf100_vmm_desc_17_17[0], NVKM_VMM_PAGE_xVxC },381 { 12, &gf100_vmm_desc_17_12[0], NVKM_VMM_PAGE_xVHx },382 {}383 }384};385 386static const struct nvkm_vmm_func387gf100_vmm_16 = {388 .join = gf100_vmm_join,389 .part = gf100_vmm_part,390 .aper = gf100_vmm_aper,391 .valid = gf100_vmm_valid,392 .flush = gf100_vmm_flush,393 .invalidate_pdb = gf100_vmm_invalidate_pdb,394 .page = {395 { 16, &gf100_vmm_desc_16_16[0], NVKM_VMM_PAGE_xVxC },396 { 12, &gf100_vmm_desc_16_12[0], NVKM_VMM_PAGE_xVHx },397 {}398 }399};400 401int402gf100_vmm_new_(const struct nvkm_vmm_func *func_16,403 const struct nvkm_vmm_func *func_17,404 struct nvkm_mmu *mmu, bool managed, u64 addr, u64 size,405 void *argv, u32 argc, struct lock_class_key *key,406 const char *name, struct nvkm_vmm **pvmm)407{408 switch (mmu->subdev.device->fb->page) {409 case 16: return nv04_vmm_new_(func_16, mmu, 0, managed, addr, size,410 argv, argc, key, name, pvmm);411 case 17: return nv04_vmm_new_(func_17, mmu, 0, managed, addr, size,412 argv, argc, key, name, pvmm);413 default:414 WARN_ON(1);415 return -EINVAL;416 }417}418 419int420gf100_vmm_new(struct nvkm_mmu *mmu, bool managed, u64 addr, u64 size,421 void *argv, u32 argc, struct lock_class_key *key,422 const char *name, struct nvkm_vmm **pvmm)423{424 return gf100_vmm_new_(&gf100_vmm_16, &gf100_vmm_17, mmu, managed, addr,425 size, argv, argc, key, name, pvmm);426}427