589 lines · c
1/*2 * Copyright 2017 Red Hat Inc.3 *4 * Permission is hereby granted, free of charge, to any person obtaining a5 * copy of this software and associated documentation files (the "Software"),6 * to deal in the Software without restriction, including without limitation7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,8 * and/or sell copies of the Software, and to permit persons to whom the9 * Software is furnished to do so, subject to the following conditions:10 *11 * The above copyright notice and this permission notice shall be included in12 * all copies or substantial portions of the Software.13 *14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR20 * OTHER DEALINGS IN THE SOFTWARE.21 */22#include "uvmm.h"23#include "umem.h"24#include "ummu.h"25 26#include <core/client.h>27#include <core/memory.h>28 29#include <nvif/if000c.h>30#include <nvif/unpack.h>31 32static const struct nvkm_object_func nvkm_uvmm;33struct nvkm_vmm *34nvkm_uvmm_search(struct nvkm_client *client, u64 handle)35{36 struct nvkm_object *object;37 38 object = nvkm_object_search(client, handle, &nvkm_uvmm);39 if (IS_ERR(object))40 return (void *)object;41 42 return nvkm_vmm_ref(nvkm_uvmm(object)->vmm);43}44 45static int46nvkm_uvmm_mthd_pfnclr(struct nvkm_uvmm *uvmm, void *argv, u32 argc)47{48 union {49 struct nvif_vmm_pfnclr_v0 v0;50 } *args = argv;51 struct nvkm_vmm *vmm = uvmm->vmm;52 int ret = -ENOSYS;53 u64 addr, size;54 55 if (!(ret = nvif_unpack(ret, &argv, &argc, args->v0, 0, 0, false))) {56 addr = args->v0.addr;57 size = args->v0.size;58 } else59 return ret;60 61 if (nvkm_vmm_in_managed_range(vmm, addr, size) && vmm->managed.raw)62 return -EINVAL;63 64 if (size) {65 mutex_lock(&vmm->mutex.vmm);66 ret = nvkm_vmm_pfn_unmap(vmm, addr, size);67 mutex_unlock(&vmm->mutex.vmm);68 }69 70 return ret;71}72 73static int74nvkm_uvmm_mthd_pfnmap(struct nvkm_uvmm *uvmm, void *argv, u32 argc)75{76 union {77 struct nvif_vmm_pfnmap_v0 v0;78 } *args = argv;79 struct nvkm_vmm *vmm = uvmm->vmm;80 int ret = -ENOSYS;81 u64 addr, size, *phys;82 u8 page;83 84 if (!(ret = nvif_unpack(ret, &argv, &argc, args->v0, 0, 0, true))) {85 page = args->v0.page;86 addr = args->v0.addr;87 size = args->v0.size;88 phys = args->v0.phys;89 if (argc != (size >> page) * sizeof(args->v0.phys[0]))90 return -EINVAL;91 } else92 return ret;93 94 if (nvkm_vmm_in_managed_range(vmm, addr, size) && vmm->managed.raw)95 return -EINVAL;96 97 if (size) {98 mutex_lock(&vmm->mutex.vmm);99 ret = nvkm_vmm_pfn_map(vmm, page, addr, size, phys);100 mutex_unlock(&vmm->mutex.vmm);101 }102 103 return ret;104}105 106static int107nvkm_uvmm_mthd_unmap(struct nvkm_uvmm *uvmm, void *argv, u32 argc)108{109 union {110 struct nvif_vmm_unmap_v0 v0;111 } *args = argv;112 struct nvkm_vmm *vmm = uvmm->vmm;113 struct nvkm_vma *vma;114 int ret = -ENOSYS;115 u64 addr;116 117 if (!(ret = nvif_unpack(ret, &argv, &argc, args->v0, 0, 0, false))) {118 addr = args->v0.addr;119 } else120 return ret;121 122 if (nvkm_vmm_in_managed_range(vmm, addr, 0) && vmm->managed.raw)123 return -EINVAL;124 125 mutex_lock(&vmm->mutex.vmm);126 vma = nvkm_vmm_node_search(vmm, addr);127 if (ret = -ENOENT, !vma || vma->addr != addr) {128 VMM_DEBUG(vmm, "lookup %016llx: %016llx",129 addr, vma ? vma->addr : ~0ULL);130 goto done;131 }132 133 if (ret = -ENOENT, vma->busy) {134 VMM_DEBUG(vmm, "denied %016llx: %d", addr, vma->busy);135 goto done;136 }137 138 if (ret = -EINVAL, !vma->memory) {139 VMM_DEBUG(vmm, "unmapped");140 goto done;141 }142 143 nvkm_vmm_unmap_locked(vmm, vma, false);144 ret = 0;145done:146 mutex_unlock(&vmm->mutex.vmm);147 return ret;148}149 150static int151nvkm_uvmm_mthd_map(struct nvkm_uvmm *uvmm, void *argv, u32 argc)152{153 struct nvkm_client *client = uvmm->object.client;154 union {155 struct nvif_vmm_map_v0 v0;156 } *args = argv;157 u64 addr, size, handle, offset;158 struct nvkm_vmm *vmm = uvmm->vmm;159 struct nvkm_vma *vma;160 struct nvkm_memory *memory;161 int ret = -ENOSYS;162 163 if (!(ret = nvif_unpack(ret, &argv, &argc, args->v0, 0, 0, true))) {164 addr = args->v0.addr;165 size = args->v0.size;166 handle = args->v0.memory;167 offset = args->v0.offset;168 } else169 return ret;170 171 if (nvkm_vmm_in_managed_range(vmm, addr, size) && vmm->managed.raw)172 return -EINVAL;173 174 memory = nvkm_umem_search(client, handle);175 if (IS_ERR(memory)) {176 VMM_DEBUG(vmm, "memory %016llx %ld\n", handle, PTR_ERR(memory));177 return PTR_ERR(memory);178 }179 180 mutex_lock(&vmm->mutex.vmm);181 if (ret = -ENOENT, !(vma = nvkm_vmm_node_search(vmm, addr))) {182 VMM_DEBUG(vmm, "lookup %016llx", addr);183 goto fail;184 }185 186 if (ret = -ENOENT, vma->busy) {187 VMM_DEBUG(vmm, "denied %016llx: %d", addr, vma->busy);188 goto fail;189 }190 191 if (ret = -EINVAL, vma->mapped && !vma->memory) {192 VMM_DEBUG(vmm, "pfnmap %016llx", addr);193 goto fail;194 }195 196 if (ret = -EINVAL, vma->addr != addr || vma->size != size) {197 if (addr + size > vma->addr + vma->size || vma->memory ||198 (vma->refd == NVKM_VMA_PAGE_NONE && !vma->mapref)) {199 VMM_DEBUG(vmm, "split %d %d %d "200 "%016llx %016llx %016llx %016llx",201 !!vma->memory, vma->refd, vma->mapref,202 addr, size, vma->addr, (u64)vma->size);203 goto fail;204 }205 206 vma = nvkm_vmm_node_split(vmm, vma, addr, size);207 if (!vma) {208 ret = -ENOMEM;209 goto fail;210 }211 }212 vma->busy = true;213 mutex_unlock(&vmm->mutex.vmm);214 215 ret = nvkm_memory_map(memory, offset, vmm, vma, argv, argc);216 if (ret == 0) {217 /* Successful map will clear vma->busy. */218 nvkm_memory_unref(&memory);219 return 0;220 }221 222 mutex_lock(&vmm->mutex.vmm);223 vma->busy = false;224 nvkm_vmm_unmap_region(vmm, vma);225fail:226 mutex_unlock(&vmm->mutex.vmm);227 nvkm_memory_unref(&memory);228 return ret;229}230 231static int232nvkm_uvmm_mthd_put(struct nvkm_uvmm *uvmm, void *argv, u32 argc)233{234 union {235 struct nvif_vmm_put_v0 v0;236 } *args = argv;237 struct nvkm_vmm *vmm = uvmm->vmm;238 struct nvkm_vma *vma;239 int ret = -ENOSYS;240 u64 addr;241 242 if (!(ret = nvif_unpack(ret, &argv, &argc, args->v0, 0, 0, false))) {243 addr = args->v0.addr;244 } else245 return ret;246 247 mutex_lock(&vmm->mutex.vmm);248 vma = nvkm_vmm_node_search(vmm, args->v0.addr);249 if (ret = -ENOENT, !vma || vma->addr != addr || vma->part) {250 VMM_DEBUG(vmm, "lookup %016llx: %016llx %d", addr,251 vma ? vma->addr : ~0ULL, vma ? vma->part : 0);252 goto done;253 }254 255 if (ret = -ENOENT, vma->busy) {256 VMM_DEBUG(vmm, "denied %016llx: %d", addr, vma->busy);257 goto done;258 }259 260 nvkm_vmm_put_locked(vmm, vma);261 ret = 0;262done:263 mutex_unlock(&vmm->mutex.vmm);264 return ret;265}266 267static int268nvkm_uvmm_mthd_get(struct nvkm_uvmm *uvmm, void *argv, u32 argc)269{270 union {271 struct nvif_vmm_get_v0 v0;272 } *args = argv;273 struct nvkm_vmm *vmm = uvmm->vmm;274 struct nvkm_vma *vma;275 int ret = -ENOSYS;276 bool getref, mapref, sparse;277 u8 page, align;278 u64 size;279 280 if (!(ret = nvif_unpack(ret, &argv, &argc, args->v0, 0, 0, false))) {281 getref = args->v0.type == NVIF_VMM_GET_V0_PTES;282 mapref = args->v0.type == NVIF_VMM_GET_V0_ADDR;283 sparse = args->v0.sparse;284 page = args->v0.page;285 align = args->v0.align;286 size = args->v0.size;287 } else288 return ret;289 290 mutex_lock(&vmm->mutex.vmm);291 ret = nvkm_vmm_get_locked(vmm, getref, mapref, sparse,292 page, align, size, &vma);293 mutex_unlock(&vmm->mutex.vmm);294 if (ret)295 return ret;296 297 args->v0.addr = vma->addr;298 return ret;299}300 301static int302nvkm_uvmm_mthd_page(struct nvkm_uvmm *uvmm, void *argv, u32 argc)303{304 union {305 struct nvif_vmm_page_v0 v0;306 } *args = argv;307 const struct nvkm_vmm_page *page;308 int ret = -ENOSYS;309 u8 type, index, nr;310 311 page = uvmm->vmm->func->page;312 for (nr = 0; page[nr].shift; nr++);313 314 if (!(nvif_unpack(ret, &argv, &argc, args->v0, 0, 0, false))) {315 if ((index = args->v0.index) >= nr)316 return -EINVAL;317 type = page[index].type;318 args->v0.shift = page[index].shift;319 args->v0.sparse = !!(type & NVKM_VMM_PAGE_SPARSE);320 args->v0.vram = !!(type & NVKM_VMM_PAGE_VRAM);321 args->v0.host = !!(type & NVKM_VMM_PAGE_HOST);322 args->v0.comp = !!(type & NVKM_VMM_PAGE_COMP);323 } else324 return -ENOSYS;325 326 return 0;327}328 329static inline int330nvkm_uvmm_page_index(struct nvkm_uvmm *uvmm, u64 size, u8 shift, u8 *refd)331{332 struct nvkm_vmm *vmm = uvmm->vmm;333 const struct nvkm_vmm_page *page;334 335 if (likely(shift)) {336 for (page = vmm->func->page; page->shift; page++) {337 if (shift == page->shift)338 break;339 }340 341 if (!page->shift || !IS_ALIGNED(size, 1ULL << page->shift)) {342 VMM_DEBUG(vmm, "page %d %016llx", shift, size);343 return -EINVAL;344 }345 } else {346 return -EINVAL;347 }348 *refd = page - vmm->func->page;349 350 return 0;351}352 353static int354nvkm_uvmm_mthd_raw_get(struct nvkm_uvmm *uvmm, struct nvif_vmm_raw_v0 *args)355{356 struct nvkm_vmm *vmm = uvmm->vmm;357 u8 refd;358 int ret;359 360 if (!nvkm_vmm_in_managed_range(vmm, args->addr, args->size))361 return -EINVAL;362 363 ret = nvkm_uvmm_page_index(uvmm, args->size, args->shift, &refd);364 if (ret)365 return ret;366 367 return nvkm_vmm_raw_get(vmm, args->addr, args->size, refd);368}369 370static int371nvkm_uvmm_mthd_raw_put(struct nvkm_uvmm *uvmm, struct nvif_vmm_raw_v0 *args)372{373 struct nvkm_vmm *vmm = uvmm->vmm;374 u8 refd;375 int ret;376 377 if (!nvkm_vmm_in_managed_range(vmm, args->addr, args->size))378 return -EINVAL;379 380 ret = nvkm_uvmm_page_index(uvmm, args->size, args->shift, &refd);381 if (ret)382 return ret;383 384 nvkm_vmm_raw_put(vmm, args->addr, args->size, refd);385 386 return 0;387}388 389static int390nvkm_uvmm_mthd_raw_map(struct nvkm_uvmm *uvmm, struct nvif_vmm_raw_v0 *args)391{392 struct nvkm_client *client = uvmm->object.client;393 struct nvkm_vmm *vmm = uvmm->vmm;394 struct nvkm_vma vma = {395 .addr = args->addr,396 .size = args->size,397 .used = true,398 .mapref = false,399 .no_comp = true,400 };401 struct nvkm_memory *memory;402 void *argv = (void *)(uintptr_t)args->argv;403 unsigned int argc = args->argc;404 u64 handle = args->memory;405 u8 refd;406 int ret;407 408 if (!nvkm_vmm_in_managed_range(vmm, args->addr, args->size))409 return -EINVAL;410 411 ret = nvkm_uvmm_page_index(uvmm, args->size, args->shift, &refd);412 if (ret)413 return ret;414 415 vma.page = vma.refd = refd;416 417 memory = nvkm_umem_search(client, args->memory);418 if (IS_ERR(memory)) {419 VMM_DEBUG(vmm, "memory %016llx %ld\n", handle, PTR_ERR(memory));420 return PTR_ERR(memory);421 }422 423 ret = nvkm_memory_map(memory, args->offset, vmm, &vma, argv, argc);424 425 nvkm_memory_unref(&vma.memory);426 nvkm_memory_unref(&memory);427 return ret;428}429 430static int431nvkm_uvmm_mthd_raw_unmap(struct nvkm_uvmm *uvmm, struct nvif_vmm_raw_v0 *args)432{433 struct nvkm_vmm *vmm = uvmm->vmm;434 u8 refd;435 int ret;436 437 if (!nvkm_vmm_in_managed_range(vmm, args->addr, args->size))438 return -EINVAL;439 440 ret = nvkm_uvmm_page_index(uvmm, args->size, args->shift, &refd);441 if (ret)442 return ret;443 444 nvkm_vmm_raw_unmap(vmm, args->addr, args->size,445 args->sparse, refd);446 447 return 0;448}449 450static int451nvkm_uvmm_mthd_raw_sparse(struct nvkm_uvmm *uvmm, struct nvif_vmm_raw_v0 *args)452{453 struct nvkm_vmm *vmm = uvmm->vmm;454 455 if (!nvkm_vmm_in_managed_range(vmm, args->addr, args->size))456 return -EINVAL;457 458 return nvkm_vmm_raw_sparse(vmm, args->addr, args->size, args->ref);459}460 461static int462nvkm_uvmm_mthd_raw(struct nvkm_uvmm *uvmm, void *argv, u32 argc)463{464 union {465 struct nvif_vmm_raw_v0 v0;466 } *args = argv;467 int ret = -ENOSYS;468 469 if (!uvmm->vmm->managed.raw)470 return -EINVAL;471 472 if ((ret = nvif_unpack(ret, &argv, &argc, args->v0, 0, 0, true)))473 return ret;474 475 switch (args->v0.op) {476 case NVIF_VMM_RAW_V0_GET:477 return nvkm_uvmm_mthd_raw_get(uvmm, &args->v0);478 case NVIF_VMM_RAW_V0_PUT:479 return nvkm_uvmm_mthd_raw_put(uvmm, &args->v0);480 case NVIF_VMM_RAW_V0_MAP:481 return nvkm_uvmm_mthd_raw_map(uvmm, &args->v0);482 case NVIF_VMM_RAW_V0_UNMAP:483 return nvkm_uvmm_mthd_raw_unmap(uvmm, &args->v0);484 case NVIF_VMM_RAW_V0_SPARSE:485 return nvkm_uvmm_mthd_raw_sparse(uvmm, &args->v0);486 default:487 return -EINVAL;488 };489}490 491static int492nvkm_uvmm_mthd(struct nvkm_object *object, u32 mthd, void *argv, u32 argc)493{494 struct nvkm_uvmm *uvmm = nvkm_uvmm(object);495 switch (mthd) {496 case NVIF_VMM_V0_PAGE : return nvkm_uvmm_mthd_page (uvmm, argv, argc);497 case NVIF_VMM_V0_GET : return nvkm_uvmm_mthd_get (uvmm, argv, argc);498 case NVIF_VMM_V0_PUT : return nvkm_uvmm_mthd_put (uvmm, argv, argc);499 case NVIF_VMM_V0_MAP : return nvkm_uvmm_mthd_map (uvmm, argv, argc);500 case NVIF_VMM_V0_UNMAP : return nvkm_uvmm_mthd_unmap (uvmm, argv, argc);501 case NVIF_VMM_V0_PFNMAP: return nvkm_uvmm_mthd_pfnmap(uvmm, argv, argc);502 case NVIF_VMM_V0_PFNCLR: return nvkm_uvmm_mthd_pfnclr(uvmm, argv, argc);503 case NVIF_VMM_V0_RAW : return nvkm_uvmm_mthd_raw (uvmm, argv, argc);504 case NVIF_VMM_V0_MTHD(0x00) ... NVIF_VMM_V0_MTHD(0x7f):505 if (uvmm->vmm->func->mthd) {506 return uvmm->vmm->func->mthd(uvmm->vmm,507 uvmm->object.client,508 mthd, argv, argc);509 }510 break;511 default:512 break;513 }514 return -EINVAL;515}516 517static void *518nvkm_uvmm_dtor(struct nvkm_object *object)519{520 struct nvkm_uvmm *uvmm = nvkm_uvmm(object);521 nvkm_vmm_unref(&uvmm->vmm);522 return uvmm;523}524 525static const struct nvkm_object_func526nvkm_uvmm = {527 .dtor = nvkm_uvmm_dtor,528 .mthd = nvkm_uvmm_mthd,529};530 531int532nvkm_uvmm_new(const struct nvkm_oclass *oclass, void *argv, u32 argc,533 struct nvkm_object **pobject)534{535 struct nvkm_mmu *mmu = nvkm_ummu(oclass->parent)->mmu;536 const bool more = oclass->base.maxver >= 0;537 union {538 struct nvif_vmm_v0 v0;539 } *args = argv;540 const struct nvkm_vmm_page *page;541 struct nvkm_uvmm *uvmm;542 int ret = -ENOSYS;543 u64 addr, size;544 bool managed, raw;545 546 if (!(ret = nvif_unpack(ret, &argv, &argc, args->v0, 0, 0, more))) {547 managed = args->v0.type == NVIF_VMM_V0_TYPE_MANAGED;548 raw = args->v0.type == NVIF_VMM_V0_TYPE_RAW;549 addr = args->v0.addr;550 size = args->v0.size;551 } else552 return ret;553 554 if (!(uvmm = kzalloc(sizeof(*uvmm), GFP_KERNEL)))555 return -ENOMEM;556 557 nvkm_object_ctor(&nvkm_uvmm, oclass, &uvmm->object);558 *pobject = &uvmm->object;559 560 if (!mmu->vmm) {561 ret = mmu->func->vmm.ctor(mmu, managed || raw, addr, size,562 argv, argc, NULL, "user", &uvmm->vmm);563 if (ret)564 return ret;565 566 uvmm->vmm->debug = max(uvmm->vmm->debug, oclass->client->debug);567 } else {568 if (size)569 return -EINVAL;570 571 uvmm->vmm = nvkm_vmm_ref(mmu->vmm);572 }573 uvmm->vmm->managed.raw = raw;574 575 if (mmu->func->promote_vmm) {576 ret = mmu->func->promote_vmm(uvmm->vmm);577 if (ret)578 return ret;579 }580 581 page = uvmm->vmm->func->page;582 args->v0.page_nr = 0;583 while (page && (page++)->shift)584 args->v0.page_nr++;585 args->v0.addr = uvmm->vmm->start;586 args->v0.size = uvmm->vmm->limit;587 return 0;588}589