1935 lines · c
1// SPDX-License-Identifier: MIT2 3/*4 * Locking:5 *6 * The uvmm mutex protects any operations on the GPU VA space provided by the7 * DRM GPU VA manager.8 *9 * The GEMs dma_resv lock protects the GEMs GPUVA list, hence link/unlink of a10 * mapping to it's backing GEM must be performed under this lock.11 *12 * Actual map/unmap operations within the fence signalling critical path are13 * protected by installing DMA fences to the corresponding GEMs DMA14 * reservations, such that concurrent BO moves, which itself walk the GEMs GPUVA15 * list in order to map/unmap it's entries, can't occur concurrently.16 *17 * Accessing the DRM_GPUVA_INVALIDATED flag doesn't need any separate18 * protection, since there are no accesses other than from BO move callbacks19 * and from the fence signalling critical path, which are already protected by20 * the corresponding GEMs DMA reservation fence.21 */22 23#include "nouveau_drv.h"24#include "nouveau_gem.h"25#include "nouveau_mem.h"26#include "nouveau_uvmm.h"27 28#include <nvif/vmm.h>29#include <nvif/mem.h>30 31#include <nvif/class.h>32#include <nvif/if000c.h>33#include <nvif/if900d.h>34 35#define NOUVEAU_VA_SPACE_BITS 47 /* FIXME */36#define NOUVEAU_VA_SPACE_START 0x037#define NOUVEAU_VA_SPACE_END (1ULL << NOUVEAU_VA_SPACE_BITS)38 39#define list_last_op(_ops) list_last_entry(_ops, struct bind_job_op, entry)40#define list_prev_op(_op) list_prev_entry(_op, entry)41#define list_for_each_op(_op, _ops) list_for_each_entry(_op, _ops, entry)42#define list_for_each_op_from_reverse(_op, _ops) \43 list_for_each_entry_from_reverse(_op, _ops, entry)44#define list_for_each_op_safe(_op, _n, _ops) list_for_each_entry_safe(_op, _n, _ops, entry)45 46enum vm_bind_op {47 OP_MAP = DRM_NOUVEAU_VM_BIND_OP_MAP,48 OP_UNMAP = DRM_NOUVEAU_VM_BIND_OP_UNMAP,49 OP_MAP_SPARSE,50 OP_UNMAP_SPARSE,51};52 53struct nouveau_uvma_prealloc {54 struct nouveau_uvma *map;55 struct nouveau_uvma *prev;56 struct nouveau_uvma *next;57};58 59struct bind_job_op {60 struct list_head entry;61 62 enum vm_bind_op op;63 u32 flags;64 65 struct drm_gpuvm_bo *vm_bo;66 67 struct {68 u64 addr;69 u64 range;70 } va;71 72 struct {73 u32 handle;74 u64 offset;75 struct drm_gem_object *obj;76 } gem;77 78 struct nouveau_uvma_region *reg;79 struct nouveau_uvma_prealloc new;80 struct drm_gpuva_ops *ops;81};82 83struct uvmm_map_args {84 struct nouveau_uvma_region *region;85 u64 addr;86 u64 range;87 u8 kind;88};89 90static int91nouveau_uvmm_vmm_sparse_ref(struct nouveau_uvmm *uvmm,92 u64 addr, u64 range)93{94 struct nvif_vmm *vmm = &uvmm->vmm.vmm;95 96 return nvif_vmm_raw_sparse(vmm, addr, range, true);97}98 99static int100nouveau_uvmm_vmm_sparse_unref(struct nouveau_uvmm *uvmm,101 u64 addr, u64 range)102{103 struct nvif_vmm *vmm = &uvmm->vmm.vmm;104 105 return nvif_vmm_raw_sparse(vmm, addr, range, false);106}107 108static int109nouveau_uvmm_vmm_get(struct nouveau_uvmm *uvmm,110 u64 addr, u64 range)111{112 struct nvif_vmm *vmm = &uvmm->vmm.vmm;113 114 return nvif_vmm_raw_get(vmm, addr, range, PAGE_SHIFT);115}116 117static int118nouveau_uvmm_vmm_put(struct nouveau_uvmm *uvmm,119 u64 addr, u64 range)120{121 struct nvif_vmm *vmm = &uvmm->vmm.vmm;122 123 return nvif_vmm_raw_put(vmm, addr, range, PAGE_SHIFT);124}125 126static int127nouveau_uvmm_vmm_unmap(struct nouveau_uvmm *uvmm,128 u64 addr, u64 range, bool sparse)129{130 struct nvif_vmm *vmm = &uvmm->vmm.vmm;131 132 return nvif_vmm_raw_unmap(vmm, addr, range, PAGE_SHIFT, sparse);133}134 135static int136nouveau_uvmm_vmm_map(struct nouveau_uvmm *uvmm,137 u64 addr, u64 range,138 u64 bo_offset, u8 kind,139 struct nouveau_mem *mem)140{141 struct nvif_vmm *vmm = &uvmm->vmm.vmm;142 union {143 struct gf100_vmm_map_v0 gf100;144 } args;145 u32 argc = 0;146 147 switch (vmm->object.oclass) {148 case NVIF_CLASS_VMM_GF100:149 case NVIF_CLASS_VMM_GM200:150 case NVIF_CLASS_VMM_GP100:151 args.gf100.version = 0;152 if (mem->mem.type & NVIF_MEM_VRAM)153 args.gf100.vol = 0;154 else155 args.gf100.vol = 1;156 args.gf100.ro = 0;157 args.gf100.priv = 0;158 args.gf100.kind = kind;159 argc = sizeof(args.gf100);160 break;161 default:162 WARN_ON(1);163 return -ENOSYS;164 }165 166 return nvif_vmm_raw_map(vmm, addr, range, PAGE_SHIFT,167 &args, argc,168 &mem->mem, bo_offset);169}170 171static int172nouveau_uvma_region_sparse_unref(struct nouveau_uvma_region *reg)173{174 u64 addr = reg->va.addr;175 u64 range = reg->va.range;176 177 return nouveau_uvmm_vmm_sparse_unref(reg->uvmm, addr, range);178}179 180static int181nouveau_uvma_vmm_put(struct nouveau_uvma *uvma)182{183 u64 addr = uvma->va.va.addr;184 u64 range = uvma->va.va.range;185 186 return nouveau_uvmm_vmm_put(to_uvmm(uvma), addr, range);187}188 189static int190nouveau_uvma_map(struct nouveau_uvma *uvma,191 struct nouveau_mem *mem)192{193 u64 addr = uvma->va.va.addr;194 u64 offset = uvma->va.gem.offset;195 u64 range = uvma->va.va.range;196 197 return nouveau_uvmm_vmm_map(to_uvmm(uvma), addr, range,198 offset, uvma->kind, mem);199}200 201static int202nouveau_uvma_unmap(struct nouveau_uvma *uvma)203{204 u64 addr = uvma->va.va.addr;205 u64 range = uvma->va.va.range;206 bool sparse = !!uvma->region;207 208 if (drm_gpuva_invalidated(&uvma->va))209 return 0;210 211 return nouveau_uvmm_vmm_unmap(to_uvmm(uvma), addr, range, sparse);212}213 214static int215nouveau_uvma_alloc(struct nouveau_uvma **puvma)216{217 *puvma = kzalloc(sizeof(**puvma), GFP_KERNEL);218 if (!*puvma)219 return -ENOMEM;220 221 return 0;222}223 224static void225nouveau_uvma_free(struct nouveau_uvma *uvma)226{227 kfree(uvma);228}229 230static void231nouveau_uvma_gem_get(struct nouveau_uvma *uvma)232{233 drm_gem_object_get(uvma->va.gem.obj);234}235 236static void237nouveau_uvma_gem_put(struct nouveau_uvma *uvma)238{239 drm_gem_object_put(uvma->va.gem.obj);240}241 242static int243nouveau_uvma_region_alloc(struct nouveau_uvma_region **preg)244{245 *preg = kzalloc(sizeof(**preg), GFP_KERNEL);246 if (!*preg)247 return -ENOMEM;248 249 kref_init(&(*preg)->kref);250 251 return 0;252}253 254static void255nouveau_uvma_region_free(struct kref *kref)256{257 struct nouveau_uvma_region *reg =258 container_of(kref, struct nouveau_uvma_region, kref);259 260 kfree(reg);261}262 263static void264nouveau_uvma_region_get(struct nouveau_uvma_region *reg)265{266 kref_get(®->kref);267}268 269static void270nouveau_uvma_region_put(struct nouveau_uvma_region *reg)271{272 kref_put(®->kref, nouveau_uvma_region_free);273}274 275static int276__nouveau_uvma_region_insert(struct nouveau_uvmm *uvmm,277 struct nouveau_uvma_region *reg)278{279 u64 addr = reg->va.addr;280 u64 range = reg->va.range;281 u64 last = addr + range - 1;282 MA_STATE(mas, &uvmm->region_mt, addr, addr);283 284 if (unlikely(mas_walk(&mas)))285 return -EEXIST;286 287 if (unlikely(mas.last < last))288 return -EEXIST;289 290 mas.index = addr;291 mas.last = last;292 293 mas_store_gfp(&mas, reg, GFP_KERNEL);294 295 reg->uvmm = uvmm;296 297 return 0;298}299 300static int301nouveau_uvma_region_insert(struct nouveau_uvmm *uvmm,302 struct nouveau_uvma_region *reg,303 u64 addr, u64 range)304{305 int ret;306 307 reg->uvmm = uvmm;308 reg->va.addr = addr;309 reg->va.range = range;310 311 ret = __nouveau_uvma_region_insert(uvmm, reg);312 if (ret)313 return ret;314 315 return 0;316}317 318static void319nouveau_uvma_region_remove(struct nouveau_uvma_region *reg)320{321 struct nouveau_uvmm *uvmm = reg->uvmm;322 MA_STATE(mas, &uvmm->region_mt, reg->va.addr, 0);323 324 mas_erase(&mas);325}326 327static int328nouveau_uvma_region_create(struct nouveau_uvmm *uvmm,329 u64 addr, u64 range)330{331 struct nouveau_uvma_region *reg;332 int ret;333 334 if (!drm_gpuvm_interval_empty(&uvmm->base, addr, range))335 return -ENOSPC;336 337 ret = nouveau_uvma_region_alloc(®);338 if (ret)339 return ret;340 341 ret = nouveau_uvma_region_insert(uvmm, reg, addr, range);342 if (ret)343 goto err_free_region;344 345 ret = nouveau_uvmm_vmm_sparse_ref(uvmm, addr, range);346 if (ret)347 goto err_region_remove;348 349 return 0;350 351err_region_remove:352 nouveau_uvma_region_remove(reg);353err_free_region:354 nouveau_uvma_region_put(reg);355 return ret;356}357 358static struct nouveau_uvma_region *359nouveau_uvma_region_find_first(struct nouveau_uvmm *uvmm,360 u64 addr, u64 range)361{362 MA_STATE(mas, &uvmm->region_mt, addr, 0);363 364 return mas_find(&mas, addr + range - 1);365}366 367static struct nouveau_uvma_region *368nouveau_uvma_region_find(struct nouveau_uvmm *uvmm,369 u64 addr, u64 range)370{371 struct nouveau_uvma_region *reg;372 373 reg = nouveau_uvma_region_find_first(uvmm, addr, range);374 if (!reg)375 return NULL;376 377 if (reg->va.addr != addr ||378 reg->va.range != range)379 return NULL;380 381 return reg;382}383 384static bool385nouveau_uvma_region_empty(struct nouveau_uvma_region *reg)386{387 struct nouveau_uvmm *uvmm = reg->uvmm;388 389 return drm_gpuvm_interval_empty(&uvmm->base,390 reg->va.addr,391 reg->va.range);392}393 394static int395__nouveau_uvma_region_destroy(struct nouveau_uvma_region *reg)396{397 struct nouveau_uvmm *uvmm = reg->uvmm;398 u64 addr = reg->va.addr;399 u64 range = reg->va.range;400 401 if (!nouveau_uvma_region_empty(reg))402 return -EBUSY;403 404 nouveau_uvma_region_remove(reg);405 nouveau_uvmm_vmm_sparse_unref(uvmm, addr, range);406 nouveau_uvma_region_put(reg);407 408 return 0;409}410 411static int412nouveau_uvma_region_destroy(struct nouveau_uvmm *uvmm,413 u64 addr, u64 range)414{415 struct nouveau_uvma_region *reg;416 417 reg = nouveau_uvma_region_find(uvmm, addr, range);418 if (!reg)419 return -ENOENT;420 421 return __nouveau_uvma_region_destroy(reg);422}423 424static void425nouveau_uvma_region_dirty(struct nouveau_uvma_region *reg)426{427 428 init_completion(®->complete);429 reg->dirty = true;430}431 432static void433nouveau_uvma_region_complete(struct nouveau_uvma_region *reg)434{435 complete_all(®->complete);436}437 438static void439op_map_prepare_unwind(struct nouveau_uvma *uvma)440{441 struct drm_gpuva *va = &uvma->va;442 nouveau_uvma_gem_put(uvma);443 drm_gpuva_remove(va);444 nouveau_uvma_free(uvma);445}446 447static void448op_unmap_prepare_unwind(struct drm_gpuva *va)449{450 drm_gpuva_insert(va->vm, va);451}452 453static void454nouveau_uvmm_sm_prepare_unwind(struct nouveau_uvmm *uvmm,455 struct nouveau_uvma_prealloc *new,456 struct drm_gpuva_ops *ops,457 struct drm_gpuva_op *last,458 struct uvmm_map_args *args)459{460 struct drm_gpuva_op *op = last;461 u64 vmm_get_start = args ? args->addr : 0;462 u64 vmm_get_end = args ? args->addr + args->range : 0;463 464 /* Unwind GPUVA space. */465 drm_gpuva_for_each_op_from_reverse(op, ops) {466 switch (op->op) {467 case DRM_GPUVA_OP_MAP:468 op_map_prepare_unwind(new->map);469 break;470 case DRM_GPUVA_OP_REMAP: {471 struct drm_gpuva_op_remap *r = &op->remap;472 struct drm_gpuva *va = r->unmap->va;473 474 if (r->next)475 op_map_prepare_unwind(new->next);476 477 if (r->prev)478 op_map_prepare_unwind(new->prev);479 480 op_unmap_prepare_unwind(va);481 break;482 }483 case DRM_GPUVA_OP_UNMAP:484 op_unmap_prepare_unwind(op->unmap.va);485 break;486 default:487 break;488 }489 }490 491 /* Unmap operation don't allocate page tables, hence skip the following492 * page table unwind.493 */494 if (!args)495 return;496 497 drm_gpuva_for_each_op(op, ops) {498 switch (op->op) {499 case DRM_GPUVA_OP_MAP: {500 u64 vmm_get_range = vmm_get_end - vmm_get_start;501 502 if (vmm_get_range)503 nouveau_uvmm_vmm_put(uvmm, vmm_get_start,504 vmm_get_range);505 break;506 }507 case DRM_GPUVA_OP_REMAP: {508 struct drm_gpuva_op_remap *r = &op->remap;509 struct drm_gpuva *va = r->unmap->va;510 u64 ustart = va->va.addr;511 u64 urange = va->va.range;512 u64 uend = ustart + urange;513 514 if (r->prev)515 vmm_get_start = uend;516 517 if (r->next)518 vmm_get_end = ustart;519 520 if (r->prev && r->next)521 vmm_get_start = vmm_get_end = 0;522 523 break;524 }525 case DRM_GPUVA_OP_UNMAP: {526 struct drm_gpuva_op_unmap *u = &op->unmap;527 struct drm_gpuva *va = u->va;528 u64 ustart = va->va.addr;529 u64 urange = va->va.range;530 u64 uend = ustart + urange;531 532 /* Nothing to do for mappings we merge with. */533 if (uend == vmm_get_start ||534 ustart == vmm_get_end)535 break;536 537 if (ustart > vmm_get_start) {538 u64 vmm_get_range = ustart - vmm_get_start;539 540 nouveau_uvmm_vmm_put(uvmm, vmm_get_start,541 vmm_get_range);542 }543 vmm_get_start = uend;544 break;545 }546 default:547 break;548 }549 550 if (op == last)551 break;552 }553}554 555static void556nouveau_uvmm_sm_map_prepare_unwind(struct nouveau_uvmm *uvmm,557 struct nouveau_uvma_prealloc *new,558 struct drm_gpuva_ops *ops,559 u64 addr, u64 range)560{561 struct drm_gpuva_op *last = drm_gpuva_last_op(ops);562 struct uvmm_map_args args = {563 .addr = addr,564 .range = range,565 };566 567 nouveau_uvmm_sm_prepare_unwind(uvmm, new, ops, last, &args);568}569 570static void571nouveau_uvmm_sm_unmap_prepare_unwind(struct nouveau_uvmm *uvmm,572 struct nouveau_uvma_prealloc *new,573 struct drm_gpuva_ops *ops)574{575 struct drm_gpuva_op *last = drm_gpuva_last_op(ops);576 577 nouveau_uvmm_sm_prepare_unwind(uvmm, new, ops, last, NULL);578}579 580static int581op_map_prepare(struct nouveau_uvmm *uvmm,582 struct nouveau_uvma **puvma,583 struct drm_gpuva_op_map *op,584 struct uvmm_map_args *args)585{586 struct nouveau_uvma *uvma;587 int ret;588 589 ret = nouveau_uvma_alloc(&uvma);590 if (ret)591 return ret;592 593 uvma->region = args->region;594 uvma->kind = args->kind;595 596 drm_gpuva_map(&uvmm->base, &uvma->va, op);597 598 /* Keep a reference until this uvma is destroyed. */599 nouveau_uvma_gem_get(uvma);600 601 *puvma = uvma;602 return 0;603}604 605static void606op_unmap_prepare(struct drm_gpuva_op_unmap *u)607{608 drm_gpuva_unmap(u);609}610 611/*612 * Note: @args should not be NULL when calling for a map operation.613 */614static int615nouveau_uvmm_sm_prepare(struct nouveau_uvmm *uvmm,616 struct nouveau_uvma_prealloc *new,617 struct drm_gpuva_ops *ops,618 struct uvmm_map_args *args)619{620 struct drm_gpuva_op *op;621 u64 vmm_get_start = args ? args->addr : 0;622 u64 vmm_get_end = args ? args->addr + args->range : 0;623 int ret;624 625 drm_gpuva_for_each_op(op, ops) {626 switch (op->op) {627 case DRM_GPUVA_OP_MAP: {628 u64 vmm_get_range = vmm_get_end - vmm_get_start;629 630 ret = op_map_prepare(uvmm, &new->map, &op->map, args);631 if (ret)632 goto unwind;633 634 if (vmm_get_range) {635 ret = nouveau_uvmm_vmm_get(uvmm, vmm_get_start,636 vmm_get_range);637 if (ret) {638 op_map_prepare_unwind(new->map);639 goto unwind;640 }641 }642 643 break;644 }645 case DRM_GPUVA_OP_REMAP: {646 struct drm_gpuva_op_remap *r = &op->remap;647 struct drm_gpuva *va = r->unmap->va;648 struct uvmm_map_args remap_args = {649 .kind = uvma_from_va(va)->kind,650 .region = uvma_from_va(va)->region,651 };652 u64 ustart = va->va.addr;653 u64 urange = va->va.range;654 u64 uend = ustart + urange;655 656 op_unmap_prepare(r->unmap);657 658 if (r->prev) {659 ret = op_map_prepare(uvmm, &new->prev, r->prev,660 &remap_args);661 if (ret)662 goto unwind;663 664 if (args)665 vmm_get_start = uend;666 }667 668 if (r->next) {669 ret = op_map_prepare(uvmm, &new->next, r->next,670 &remap_args);671 if (ret) {672 if (r->prev)673 op_map_prepare_unwind(new->prev);674 goto unwind;675 }676 677 if (args)678 vmm_get_end = ustart;679 }680 681 if (args && (r->prev && r->next))682 vmm_get_start = vmm_get_end = 0;683 684 break;685 }686 case DRM_GPUVA_OP_UNMAP: {687 struct drm_gpuva_op_unmap *u = &op->unmap;688 struct drm_gpuva *va = u->va;689 u64 ustart = va->va.addr;690 u64 urange = va->va.range;691 u64 uend = ustart + urange;692 693 op_unmap_prepare(u);694 695 if (!args)696 break;697 698 /* Nothing to do for mappings we merge with. */699 if (uend == vmm_get_start ||700 ustart == vmm_get_end)701 break;702 703 if (ustart > vmm_get_start) {704 u64 vmm_get_range = ustart - vmm_get_start;705 706 ret = nouveau_uvmm_vmm_get(uvmm, vmm_get_start,707 vmm_get_range);708 if (ret) {709 op_unmap_prepare_unwind(va);710 goto unwind;711 }712 }713 vmm_get_start = uend;714 715 break;716 }717 default:718 ret = -EINVAL;719 goto unwind;720 }721 }722 723 return 0;724 725unwind:726 if (op != drm_gpuva_first_op(ops))727 nouveau_uvmm_sm_prepare_unwind(uvmm, new, ops,728 drm_gpuva_prev_op(op),729 args);730 return ret;731}732 733static int734nouveau_uvmm_sm_map_prepare(struct nouveau_uvmm *uvmm,735 struct nouveau_uvma_prealloc *new,736 struct nouveau_uvma_region *region,737 struct drm_gpuva_ops *ops,738 u64 addr, u64 range, u8 kind)739{740 struct uvmm_map_args args = {741 .region = region,742 .addr = addr,743 .range = range,744 .kind = kind,745 };746 747 return nouveau_uvmm_sm_prepare(uvmm, new, ops, &args);748}749 750static int751nouveau_uvmm_sm_unmap_prepare(struct nouveau_uvmm *uvmm,752 struct nouveau_uvma_prealloc *new,753 struct drm_gpuva_ops *ops)754{755 return nouveau_uvmm_sm_prepare(uvmm, new, ops, NULL);756}757 758static struct drm_gem_object *759op_gem_obj(struct drm_gpuva_op *op)760{761 switch (op->op) {762 case DRM_GPUVA_OP_MAP:763 return op->map.gem.obj;764 case DRM_GPUVA_OP_REMAP:765 /* Actually, we're looking for the GEMs backing remap.prev and766 * remap.next, but since this is a remap they're identical to767 * the GEM backing the unmapped GPUVA.768 */769 return op->remap.unmap->va->gem.obj;770 case DRM_GPUVA_OP_UNMAP:771 return op->unmap.va->gem.obj;772 default:773 WARN(1, "Unknown operation.\n");774 return NULL;775 }776}777 778static void779op_map(struct nouveau_uvma *uvma)780{781 struct nouveau_bo *nvbo = nouveau_gem_object(uvma->va.gem.obj);782 783 nouveau_uvma_map(uvma, nouveau_mem(nvbo->bo.resource));784}785 786static void787op_unmap(struct drm_gpuva_op_unmap *u)788{789 struct drm_gpuva *va = u->va;790 struct nouveau_uvma *uvma = uvma_from_va(va);791 792 /* nouveau_uvma_unmap() does not unmap if backing BO is evicted. */793 if (!u->keep)794 nouveau_uvma_unmap(uvma);795}796 797static void798op_unmap_range(struct drm_gpuva_op_unmap *u,799 u64 addr, u64 range)800{801 struct nouveau_uvma *uvma = uvma_from_va(u->va);802 bool sparse = !!uvma->region;803 804 if (!drm_gpuva_invalidated(u->va))805 nouveau_uvmm_vmm_unmap(to_uvmm(uvma), addr, range, sparse);806}807 808static void809op_remap(struct drm_gpuva_op_remap *r,810 struct nouveau_uvma_prealloc *new)811{812 struct drm_gpuva_op_unmap *u = r->unmap;813 struct nouveau_uvma *uvma = uvma_from_va(u->va);814 u64 addr = uvma->va.va.addr;815 u64 end = uvma->va.va.addr + uvma->va.va.range;816 817 if (r->prev)818 addr = r->prev->va.addr + r->prev->va.range;819 820 if (r->next)821 end = r->next->va.addr;822 823 op_unmap_range(u, addr, end - addr);824}825 826static int827nouveau_uvmm_sm(struct nouveau_uvmm *uvmm,828 struct nouveau_uvma_prealloc *new,829 struct drm_gpuva_ops *ops)830{831 struct drm_gpuva_op *op;832 833 drm_gpuva_for_each_op(op, ops) {834 switch (op->op) {835 case DRM_GPUVA_OP_MAP:836 op_map(new->map);837 break;838 case DRM_GPUVA_OP_REMAP:839 op_remap(&op->remap, new);840 break;841 case DRM_GPUVA_OP_UNMAP:842 op_unmap(&op->unmap);843 break;844 default:845 break;846 }847 }848 849 return 0;850}851 852static int853nouveau_uvmm_sm_map(struct nouveau_uvmm *uvmm,854 struct nouveau_uvma_prealloc *new,855 struct drm_gpuva_ops *ops)856{857 return nouveau_uvmm_sm(uvmm, new, ops);858}859 860static int861nouveau_uvmm_sm_unmap(struct nouveau_uvmm *uvmm,862 struct nouveau_uvma_prealloc *new,863 struct drm_gpuva_ops *ops)864{865 return nouveau_uvmm_sm(uvmm, new, ops);866}867 868static void869nouveau_uvmm_sm_cleanup(struct nouveau_uvmm *uvmm,870 struct nouveau_uvma_prealloc *new,871 struct drm_gpuva_ops *ops, bool unmap)872{873 struct drm_gpuva_op *op;874 875 drm_gpuva_for_each_op(op, ops) {876 switch (op->op) {877 case DRM_GPUVA_OP_MAP:878 break;879 case DRM_GPUVA_OP_REMAP: {880 struct drm_gpuva_op_remap *r = &op->remap;881 struct drm_gpuva_op_map *p = r->prev;882 struct drm_gpuva_op_map *n = r->next;883 struct drm_gpuva *va = r->unmap->va;884 struct nouveau_uvma *uvma = uvma_from_va(va);885 886 if (unmap) {887 u64 addr = va->va.addr;888 u64 end = addr + va->va.range;889 890 if (p)891 addr = p->va.addr + p->va.range;892 893 if (n)894 end = n->va.addr;895 896 nouveau_uvmm_vmm_put(uvmm, addr, end - addr);897 }898 899 nouveau_uvma_gem_put(uvma);900 nouveau_uvma_free(uvma);901 break;902 }903 case DRM_GPUVA_OP_UNMAP: {904 struct drm_gpuva_op_unmap *u = &op->unmap;905 struct drm_gpuva *va = u->va;906 struct nouveau_uvma *uvma = uvma_from_va(va);907 908 if (unmap)909 nouveau_uvma_vmm_put(uvma);910 911 nouveau_uvma_gem_put(uvma);912 nouveau_uvma_free(uvma);913 break;914 }915 default:916 break;917 }918 }919}920 921static void922nouveau_uvmm_sm_map_cleanup(struct nouveau_uvmm *uvmm,923 struct nouveau_uvma_prealloc *new,924 struct drm_gpuva_ops *ops)925{926 nouveau_uvmm_sm_cleanup(uvmm, new, ops, false);927}928 929static void930nouveau_uvmm_sm_unmap_cleanup(struct nouveau_uvmm *uvmm,931 struct nouveau_uvma_prealloc *new,932 struct drm_gpuva_ops *ops)933{934 nouveau_uvmm_sm_cleanup(uvmm, new, ops, true);935}936 937static int938nouveau_uvmm_validate_range(struct nouveau_uvmm *uvmm, u64 addr, u64 range)939{940 if (addr & ~PAGE_MASK)941 return -EINVAL;942 943 if (range & ~PAGE_MASK)944 return -EINVAL;945 946 if (!drm_gpuvm_range_valid(&uvmm->base, addr, range))947 return -EINVAL;948 949 return 0;950}951 952static int953nouveau_uvmm_bind_job_alloc(struct nouveau_uvmm_bind_job **pjob)954{955 *pjob = kzalloc(sizeof(**pjob), GFP_KERNEL);956 if (!*pjob)957 return -ENOMEM;958 959 kref_init(&(*pjob)->kref);960 961 return 0;962}963 964static void965nouveau_uvmm_bind_job_free(struct kref *kref)966{967 struct nouveau_uvmm_bind_job *job =968 container_of(kref, struct nouveau_uvmm_bind_job, kref);969 struct bind_job_op *op, *next;970 971 list_for_each_op_safe(op, next, &job->ops) {972 list_del(&op->entry);973 kfree(op);974 }975 976 nouveau_job_free(&job->base);977 kfree(job);978}979 980static void981nouveau_uvmm_bind_job_get(struct nouveau_uvmm_bind_job *job)982{983 kref_get(&job->kref);984}985 986static void987nouveau_uvmm_bind_job_put(struct nouveau_uvmm_bind_job *job)988{989 kref_put(&job->kref, nouveau_uvmm_bind_job_free);990}991 992static int993bind_validate_op(struct nouveau_job *job,994 struct bind_job_op *op)995{996 struct nouveau_uvmm *uvmm = nouveau_cli_uvmm(job->cli);997 struct drm_gem_object *obj = op->gem.obj;998 999 if (op->op == OP_MAP) {1000 if (op->gem.offset & ~PAGE_MASK)1001 return -EINVAL;1002 1003 if (obj->size <= op->gem.offset)1004 return -EINVAL;1005 1006 if (op->va.range > (obj->size - op->gem.offset))1007 return -EINVAL;1008 }1009 1010 return nouveau_uvmm_validate_range(uvmm, op->va.addr, op->va.range);1011}1012 1013static void1014bind_validate_map_sparse(struct nouveau_job *job, u64 addr, u64 range)1015{1016 struct nouveau_sched *sched = job->sched;1017 struct nouveau_job *__job;1018 struct bind_job_op *op;1019 u64 end = addr + range;1020 1021again:1022 spin_lock(&sched->job.list.lock);1023 list_for_each_entry(__job, &sched->job.list.head, entry) {1024 struct nouveau_uvmm_bind_job *bind_job = to_uvmm_bind_job(__job);1025 1026 list_for_each_op(op, &bind_job->ops) {1027 if (op->op == OP_UNMAP) {1028 u64 op_addr = op->va.addr;1029 u64 op_end = op_addr + op->va.range;1030 1031 if (!(end <= op_addr || addr >= op_end)) {1032 nouveau_uvmm_bind_job_get(bind_job);1033 spin_unlock(&sched->job.list.lock);1034 wait_for_completion(&bind_job->complete);1035 nouveau_uvmm_bind_job_put(bind_job);1036 goto again;1037 }1038 }1039 }1040 }1041 spin_unlock(&sched->job.list.lock);1042}1043 1044static int1045bind_validate_map_common(struct nouveau_job *job, u64 addr, u64 range,1046 bool sparse)1047{1048 struct nouveau_uvmm *uvmm = nouveau_cli_uvmm(job->cli);1049 struct nouveau_uvma_region *reg;1050 u64 reg_addr, reg_end;1051 u64 end = addr + range;1052 1053again:1054 nouveau_uvmm_lock(uvmm);1055 reg = nouveau_uvma_region_find_first(uvmm, addr, range);1056 if (!reg) {1057 nouveau_uvmm_unlock(uvmm);1058 return 0;1059 }1060 1061 /* Generally, job submits are serialized, hence only1062 * dirty regions can be modified concurrently.1063 */1064 if (reg->dirty) {1065 nouveau_uvma_region_get(reg);1066 nouveau_uvmm_unlock(uvmm);1067 wait_for_completion(®->complete);1068 nouveau_uvma_region_put(reg);1069 goto again;1070 }1071 nouveau_uvmm_unlock(uvmm);1072 1073 if (sparse)1074 return -ENOSPC;1075 1076 reg_addr = reg->va.addr;1077 reg_end = reg_addr + reg->va.range;1078 1079 /* Make sure the mapping is either outside of a1080 * region or fully enclosed by a region.1081 */1082 if (reg_addr > addr || reg_end < end)1083 return -ENOSPC;1084 1085 return 0;1086}1087 1088static int1089bind_validate_region(struct nouveau_job *job)1090{1091 struct nouveau_uvmm_bind_job *bind_job = to_uvmm_bind_job(job);1092 struct bind_job_op *op;1093 int ret;1094 1095 list_for_each_op(op, &bind_job->ops) {1096 u64 op_addr = op->va.addr;1097 u64 op_range = op->va.range;1098 bool sparse = false;1099 1100 switch (op->op) {1101 case OP_MAP_SPARSE:1102 sparse = true;1103 bind_validate_map_sparse(job, op_addr, op_range);1104 fallthrough;1105 case OP_MAP:1106 ret = bind_validate_map_common(job, op_addr, op_range,1107 sparse);1108 if (ret)1109 return ret;1110 break;1111 default:1112 break;1113 }1114 }1115 1116 return 0;1117}1118 1119static void1120bind_link_gpuvas(struct bind_job_op *bop)1121{1122 struct nouveau_uvma_prealloc *new = &bop->new;1123 struct drm_gpuvm_bo *vm_bo = bop->vm_bo;1124 struct drm_gpuva_ops *ops = bop->ops;1125 struct drm_gpuva_op *op;1126 1127 drm_gpuva_for_each_op(op, ops) {1128 switch (op->op) {1129 case DRM_GPUVA_OP_MAP:1130 drm_gpuva_link(&new->map->va, vm_bo);1131 break;1132 case DRM_GPUVA_OP_REMAP: {1133 struct drm_gpuva *va = op->remap.unmap->va;1134 1135 if (op->remap.prev)1136 drm_gpuva_link(&new->prev->va, va->vm_bo);1137 if (op->remap.next)1138 drm_gpuva_link(&new->next->va, va->vm_bo);1139 drm_gpuva_unlink(va);1140 break;1141 }1142 case DRM_GPUVA_OP_UNMAP:1143 drm_gpuva_unlink(op->unmap.va);1144 break;1145 default:1146 break;1147 }1148 }1149}1150 1151static int1152bind_lock_validate(struct nouveau_job *job, struct drm_exec *exec,1153 unsigned int num_fences)1154{1155 struct nouveau_uvmm_bind_job *bind_job = to_uvmm_bind_job(job);1156 struct bind_job_op *op;1157 int ret;1158 1159 list_for_each_op(op, &bind_job->ops) {1160 struct drm_gpuva_op *va_op;1161 1162 if (!op->ops)1163 continue;1164 1165 drm_gpuva_for_each_op(va_op, op->ops) {1166 struct drm_gem_object *obj = op_gem_obj(va_op);1167 1168 if (unlikely(!obj))1169 continue;1170 1171 ret = drm_exec_prepare_obj(exec, obj, num_fences);1172 if (ret)1173 return ret;1174 1175 /* Don't validate GEMs backing mappings we're about to1176 * unmap, it's not worth the effort.1177 */1178 if (va_op->op == DRM_GPUVA_OP_UNMAP)1179 continue;1180 1181 ret = nouveau_bo_validate(nouveau_gem_object(obj),1182 true, false);1183 if (ret)1184 return ret;1185 }1186 }1187 1188 return 0;1189}1190 1191static int1192nouveau_uvmm_bind_job_submit(struct nouveau_job *job,1193 struct drm_gpuvm_exec *vme)1194{1195 struct nouveau_uvmm *uvmm = nouveau_cli_uvmm(job->cli);1196 struct nouveau_uvmm_bind_job *bind_job = to_uvmm_bind_job(job);1197 struct drm_exec *exec = &vme->exec;1198 struct bind_job_op *op;1199 int ret;1200 1201 list_for_each_op(op, &bind_job->ops) {1202 if (op->op == OP_MAP) {1203 struct drm_gem_object *obj = op->gem.obj =1204 drm_gem_object_lookup(job->file_priv,1205 op->gem.handle);1206 if (!obj)1207 return -ENOENT;1208 1209 dma_resv_lock(obj->resv, NULL);1210 op->vm_bo = drm_gpuvm_bo_obtain(&uvmm->base, obj);1211 dma_resv_unlock(obj->resv);1212 if (IS_ERR(op->vm_bo))1213 return PTR_ERR(op->vm_bo);1214 1215 drm_gpuvm_bo_extobj_add(op->vm_bo);1216 }1217 1218 ret = bind_validate_op(job, op);1219 if (ret)1220 return ret;1221 }1222 1223 /* If a sparse region or mapping overlaps a dirty region, we need to1224 * wait for the region to complete the unbind process. This is due to1225 * how page table management is currently implemented. A future1226 * implementation might change this.1227 */1228 ret = bind_validate_region(job);1229 if (ret)1230 return ret;1231 1232 /* Once we start modifying the GPU VA space we need to keep holding the1233 * uvmm lock until we can't fail anymore. This is due to the set of GPU1234 * VA space changes must appear atomically and we need to be able to1235 * unwind all GPU VA space changes on failure.1236 */1237 nouveau_uvmm_lock(uvmm);1238 1239 list_for_each_op(op, &bind_job->ops) {1240 switch (op->op) {1241 case OP_MAP_SPARSE:1242 ret = nouveau_uvma_region_create(uvmm,1243 op->va.addr,1244 op->va.range);1245 if (ret)1246 goto unwind_continue;1247 1248 break;1249 case OP_UNMAP_SPARSE:1250 op->reg = nouveau_uvma_region_find(uvmm, op->va.addr,1251 op->va.range);1252 if (!op->reg || op->reg->dirty) {1253 ret = -ENOENT;1254 goto unwind_continue;1255 }1256 1257 op->ops = drm_gpuvm_sm_unmap_ops_create(&uvmm->base,1258 op->va.addr,1259 op->va.range);1260 if (IS_ERR(op->ops)) {1261 ret = PTR_ERR(op->ops);1262 goto unwind_continue;1263 }1264 1265 ret = nouveau_uvmm_sm_unmap_prepare(uvmm, &op->new,1266 op->ops);1267 if (ret) {1268 drm_gpuva_ops_free(&uvmm->base, op->ops);1269 op->ops = NULL;1270 op->reg = NULL;1271 goto unwind_continue;1272 }1273 1274 nouveau_uvma_region_dirty(op->reg);1275 1276 break;1277 case OP_MAP: {1278 struct nouveau_uvma_region *reg;1279 1280 reg = nouveau_uvma_region_find_first(uvmm,1281 op->va.addr,1282 op->va.range);1283 if (reg) {1284 u64 reg_addr = reg->va.addr;1285 u64 reg_end = reg_addr + reg->va.range;1286 u64 op_addr = op->va.addr;1287 u64 op_end = op_addr + op->va.range;1288 1289 if (unlikely(reg->dirty)) {1290 ret = -EINVAL;1291 goto unwind_continue;1292 }1293 1294 /* Make sure the mapping is either outside of a1295 * region or fully enclosed by a region.1296 */1297 if (reg_addr > op_addr || reg_end < op_end) {1298 ret = -ENOSPC;1299 goto unwind_continue;1300 }1301 }1302 1303 op->ops = drm_gpuvm_sm_map_ops_create(&uvmm->base,1304 op->va.addr,1305 op->va.range,1306 op->gem.obj,1307 op->gem.offset);1308 if (IS_ERR(op->ops)) {1309 ret = PTR_ERR(op->ops);1310 goto unwind_continue;1311 }1312 1313 ret = nouveau_uvmm_sm_map_prepare(uvmm, &op->new,1314 reg, op->ops,1315 op->va.addr,1316 op->va.range,1317 op->flags & 0xff);1318 if (ret) {1319 drm_gpuva_ops_free(&uvmm->base, op->ops);1320 op->ops = NULL;1321 goto unwind_continue;1322 }1323 1324 break;1325 }1326 case OP_UNMAP:1327 op->ops = drm_gpuvm_sm_unmap_ops_create(&uvmm->base,1328 op->va.addr,1329 op->va.range);1330 if (IS_ERR(op->ops)) {1331 ret = PTR_ERR(op->ops);1332 goto unwind_continue;1333 }1334 1335 ret = nouveau_uvmm_sm_unmap_prepare(uvmm, &op->new,1336 op->ops);1337 if (ret) {1338 drm_gpuva_ops_free(&uvmm->base, op->ops);1339 op->ops = NULL;1340 goto unwind_continue;1341 }1342 1343 break;1344 default:1345 ret = -EINVAL;1346 goto unwind_continue;1347 }1348 }1349 1350 drm_exec_init(exec, vme->flags, 0);1351 drm_exec_until_all_locked(exec) {1352 ret = bind_lock_validate(job, exec, vme->num_fences);1353 drm_exec_retry_on_contention(exec);1354 if (ret) {1355 op = list_last_op(&bind_job->ops);1356 goto unwind;1357 }1358 }1359 1360 /* Link and unlink GPUVAs while holding the dma_resv lock.1361 *1362 * As long as we validate() all GEMs and add fences to all GEMs DMA1363 * reservations backing map and remap operations we can be sure there1364 * won't be any concurrent (in)validations during job execution, hence1365 * we're safe to check drm_gpuva_invalidated() within the fence1366 * signalling critical path without holding a separate lock.1367 *1368 * GPUVAs about to be unmapped are safe as well, since they're unlinked1369 * already.1370 *1371 * GEMs from map and remap operations must be validated before linking1372 * their corresponding mappings to prevent the actual PT update to1373 * happen right away in validate() rather than asynchronously as1374 * intended.1375 *1376 * Note that after linking and unlinking the GPUVAs in this loop this1377 * function cannot fail anymore, hence there is no need for an unwind1378 * path.1379 */1380 list_for_each_op(op, &bind_job->ops) {1381 switch (op->op) {1382 case OP_UNMAP_SPARSE:1383 case OP_MAP:1384 case OP_UNMAP:1385 bind_link_gpuvas(op);1386 break;1387 default:1388 break;1389 }1390 }1391 nouveau_uvmm_unlock(uvmm);1392 1393 return 0;1394 1395unwind_continue:1396 op = list_prev_op(op);1397unwind:1398 list_for_each_op_from_reverse(op, &bind_job->ops) {1399 switch (op->op) {1400 case OP_MAP_SPARSE:1401 nouveau_uvma_region_destroy(uvmm, op->va.addr,1402 op->va.range);1403 break;1404 case OP_UNMAP_SPARSE:1405 __nouveau_uvma_region_insert(uvmm, op->reg);1406 nouveau_uvmm_sm_unmap_prepare_unwind(uvmm, &op->new,1407 op->ops);1408 break;1409 case OP_MAP:1410 nouveau_uvmm_sm_map_prepare_unwind(uvmm, &op->new,1411 op->ops,1412 op->va.addr,1413 op->va.range);1414 break;1415 case OP_UNMAP:1416 nouveau_uvmm_sm_unmap_prepare_unwind(uvmm, &op->new,1417 op->ops);1418 break;1419 }1420 1421 drm_gpuva_ops_free(&uvmm->base, op->ops);1422 op->ops = NULL;1423 op->reg = NULL;1424 }1425 1426 nouveau_uvmm_unlock(uvmm);1427 drm_gpuvm_exec_unlock(vme);1428 return ret;1429}1430 1431static void1432nouveau_uvmm_bind_job_armed_submit(struct nouveau_job *job,1433 struct drm_gpuvm_exec *vme)1434{1435 drm_gpuvm_exec_resv_add_fence(vme, job->done_fence,1436 job->resv_usage, job->resv_usage);1437 drm_gpuvm_exec_unlock(vme);1438}1439 1440static struct dma_fence *1441nouveau_uvmm_bind_job_run(struct nouveau_job *job)1442{1443 struct nouveau_uvmm_bind_job *bind_job = to_uvmm_bind_job(job);1444 struct nouveau_uvmm *uvmm = nouveau_cli_uvmm(job->cli);1445 struct bind_job_op *op;1446 int ret = 0;1447 1448 list_for_each_op(op, &bind_job->ops) {1449 switch (op->op) {1450 case OP_MAP_SPARSE:1451 /* noop */1452 break;1453 case OP_MAP:1454 ret = nouveau_uvmm_sm_map(uvmm, &op->new, op->ops);1455 if (ret)1456 goto out;1457 break;1458 case OP_UNMAP_SPARSE:1459 fallthrough;1460 case OP_UNMAP:1461 ret = nouveau_uvmm_sm_unmap(uvmm, &op->new, op->ops);1462 if (ret)1463 goto out;1464 break;1465 }1466 }1467 1468out:1469 if (ret)1470 NV_PRINTK(err, job->cli, "bind job failed: %d\n", ret);1471 return ERR_PTR(ret);1472}1473 1474static void1475nouveau_uvmm_bind_job_cleanup(struct nouveau_job *job)1476{1477 struct nouveau_uvmm_bind_job *bind_job = to_uvmm_bind_job(job);1478 struct nouveau_uvmm *uvmm = nouveau_cli_uvmm(job->cli);1479 struct bind_job_op *op;1480 1481 list_for_each_op(op, &bind_job->ops) {1482 struct drm_gem_object *obj = op->gem.obj;1483 1484 /* When nouveau_uvmm_bind_job_submit() fails op->ops and op->reg1485 * will be NULL, hence skip the cleanup.1486 */1487 switch (op->op) {1488 case OP_MAP_SPARSE:1489 /* noop */1490 break;1491 case OP_UNMAP_SPARSE:1492 if (!IS_ERR_OR_NULL(op->ops))1493 nouveau_uvmm_sm_unmap_cleanup(uvmm, &op->new,1494 op->ops);1495 1496 if (op->reg) {1497 nouveau_uvma_region_sparse_unref(op->reg);1498 nouveau_uvmm_lock(uvmm);1499 nouveau_uvma_region_remove(op->reg);1500 nouveau_uvmm_unlock(uvmm);1501 nouveau_uvma_region_complete(op->reg);1502 nouveau_uvma_region_put(op->reg);1503 }1504 1505 break;1506 case OP_MAP:1507 if (!IS_ERR_OR_NULL(op->ops))1508 nouveau_uvmm_sm_map_cleanup(uvmm, &op->new,1509 op->ops);1510 break;1511 case OP_UNMAP:1512 if (!IS_ERR_OR_NULL(op->ops))1513 nouveau_uvmm_sm_unmap_cleanup(uvmm, &op->new,1514 op->ops);1515 break;1516 }1517 1518 if (!IS_ERR_OR_NULL(op->ops))1519 drm_gpuva_ops_free(&uvmm->base, op->ops);1520 1521 if (!IS_ERR_OR_NULL(op->vm_bo)) {1522 dma_resv_lock(obj->resv, NULL);1523 drm_gpuvm_bo_put(op->vm_bo);1524 dma_resv_unlock(obj->resv);1525 }1526 1527 if (obj)1528 drm_gem_object_put(obj);1529 }1530 1531 nouveau_job_done(job);1532 complete_all(&bind_job->complete);1533 1534 nouveau_uvmm_bind_job_put(bind_job);1535}1536 1537static const struct nouveau_job_ops nouveau_bind_job_ops = {1538 .submit = nouveau_uvmm_bind_job_submit,1539 .armed_submit = nouveau_uvmm_bind_job_armed_submit,1540 .run = nouveau_uvmm_bind_job_run,1541 .free = nouveau_uvmm_bind_job_cleanup,1542};1543 1544static int1545bind_job_op_from_uop(struct bind_job_op **pop,1546 struct drm_nouveau_vm_bind_op *uop)1547{1548 struct bind_job_op *op;1549 1550 op = *pop = kzalloc(sizeof(*op), GFP_KERNEL);1551 if (!op)1552 return -ENOMEM;1553 1554 switch (uop->op) {1555 case OP_MAP:1556 op->op = uop->flags & DRM_NOUVEAU_VM_BIND_SPARSE ?1557 OP_MAP_SPARSE : OP_MAP;1558 break;1559 case OP_UNMAP:1560 op->op = uop->flags & DRM_NOUVEAU_VM_BIND_SPARSE ?1561 OP_UNMAP_SPARSE : OP_UNMAP;1562 break;1563 default:1564 op->op = uop->op;1565 break;1566 }1567 1568 op->flags = uop->flags;1569 op->va.addr = uop->addr;1570 op->va.range = uop->range;1571 op->gem.handle = uop->handle;1572 op->gem.offset = uop->bo_offset;1573 1574 return 0;1575}1576 1577static void1578bind_job_ops_free(struct list_head *ops)1579{1580 struct bind_job_op *op, *next;1581 1582 list_for_each_op_safe(op, next, ops) {1583 list_del(&op->entry);1584 kfree(op);1585 }1586}1587 1588static int1589nouveau_uvmm_bind_job_init(struct nouveau_uvmm_bind_job **pjob,1590 struct nouveau_uvmm_bind_job_args *__args)1591{1592 struct nouveau_uvmm_bind_job *job;1593 struct nouveau_job_args args = {};1594 struct bind_job_op *op;1595 int i, ret;1596 1597 ret = nouveau_uvmm_bind_job_alloc(&job);1598 if (ret)1599 return ret;1600 1601 INIT_LIST_HEAD(&job->ops);1602 1603 for (i = 0; i < __args->op.count; i++) {1604 ret = bind_job_op_from_uop(&op, &__args->op.s[i]);1605 if (ret)1606 goto err_free;1607 1608 list_add_tail(&op->entry, &job->ops);1609 }1610 1611 init_completion(&job->complete);1612 1613 args.file_priv = __args->file_priv;1614 1615 args.sched = __args->sched;1616 args.credits = 1;1617 1618 args.in_sync.count = __args->in_sync.count;1619 args.in_sync.s = __args->in_sync.s;1620 1621 args.out_sync.count = __args->out_sync.count;1622 args.out_sync.s = __args->out_sync.s;1623 1624 args.sync = !(__args->flags & DRM_NOUVEAU_VM_BIND_RUN_ASYNC);1625 args.ops = &nouveau_bind_job_ops;1626 args.resv_usage = DMA_RESV_USAGE_BOOKKEEP;1627 1628 ret = nouveau_job_init(&job->base, &args);1629 if (ret)1630 goto err_free;1631 1632 *pjob = job;1633 return 0;1634 1635err_free:1636 bind_job_ops_free(&job->ops);1637 kfree(job);1638 *pjob = NULL;1639 1640 return ret;1641}1642 1643static int1644nouveau_uvmm_vm_bind(struct nouveau_uvmm_bind_job_args *args)1645{1646 struct nouveau_uvmm_bind_job *job;1647 int ret;1648 1649 ret = nouveau_uvmm_bind_job_init(&job, args);1650 if (ret)1651 return ret;1652 1653 ret = nouveau_job_submit(&job->base);1654 if (ret)1655 goto err_job_fini;1656 1657 return 0;1658 1659err_job_fini:1660 nouveau_job_fini(&job->base);1661 return ret;1662}1663 1664static int1665nouveau_uvmm_vm_bind_ucopy(struct nouveau_uvmm_bind_job_args *args,1666 struct drm_nouveau_vm_bind *req)1667{1668 struct drm_nouveau_sync **s;1669 u32 inc = req->wait_count;1670 u64 ins = req->wait_ptr;1671 u32 outc = req->sig_count;1672 u64 outs = req->sig_ptr;1673 u32 opc = req->op_count;1674 u64 ops = req->op_ptr;1675 int ret;1676 1677 args->flags = req->flags;1678 1679 if (opc) {1680 args->op.count = opc;1681 args->op.s = u_memcpya(ops, opc,1682 sizeof(*args->op.s));1683 if (IS_ERR(args->op.s))1684 return PTR_ERR(args->op.s);1685 }1686 1687 if (inc) {1688 s = &args->in_sync.s;1689 1690 args->in_sync.count = inc;1691 *s = u_memcpya(ins, inc, sizeof(**s));1692 if (IS_ERR(*s)) {1693 ret = PTR_ERR(*s);1694 goto err_free_ops;1695 }1696 }1697 1698 if (outc) {1699 s = &args->out_sync.s;1700 1701 args->out_sync.count = outc;1702 *s = u_memcpya(outs, outc, sizeof(**s));1703 if (IS_ERR(*s)) {1704 ret = PTR_ERR(*s);1705 goto err_free_ins;1706 }1707 }1708 1709 return 0;1710 1711err_free_ops:1712 u_free(args->op.s);1713err_free_ins:1714 u_free(args->in_sync.s);1715 return ret;1716}1717 1718static void1719nouveau_uvmm_vm_bind_ufree(struct nouveau_uvmm_bind_job_args *args)1720{1721 u_free(args->op.s);1722 u_free(args->in_sync.s);1723 u_free(args->out_sync.s);1724}1725 1726int1727nouveau_uvmm_ioctl_vm_bind(struct drm_device *dev,1728 void *data,1729 struct drm_file *file_priv)1730{1731 struct nouveau_cli *cli = nouveau_cli(file_priv);1732 struct nouveau_uvmm_bind_job_args args = {};1733 struct drm_nouveau_vm_bind *req = data;1734 int ret = 0;1735 1736 if (unlikely(!nouveau_cli_uvmm_locked(cli)))1737 return -ENOSYS;1738 1739 ret = nouveau_uvmm_vm_bind_ucopy(&args, req);1740 if (ret)1741 return ret;1742 1743 args.sched = cli->sched;1744 args.file_priv = file_priv;1745 1746 ret = nouveau_uvmm_vm_bind(&args);1747 if (ret)1748 goto out_free_args;1749 1750out_free_args:1751 nouveau_uvmm_vm_bind_ufree(&args);1752 return ret;1753}1754 1755void1756nouveau_uvmm_bo_map_all(struct nouveau_bo *nvbo, struct nouveau_mem *mem)1757{1758 struct drm_gem_object *obj = &nvbo->bo.base;1759 struct drm_gpuvm_bo *vm_bo;1760 struct drm_gpuva *va;1761 1762 dma_resv_assert_held(obj->resv);1763 1764 drm_gem_for_each_gpuvm_bo(vm_bo, obj) {1765 drm_gpuvm_bo_for_each_va(va, vm_bo) {1766 struct nouveau_uvma *uvma = uvma_from_va(va);1767 1768 nouveau_uvma_map(uvma, mem);1769 drm_gpuva_invalidate(va, false);1770 }1771 }1772}1773 1774void1775nouveau_uvmm_bo_unmap_all(struct nouveau_bo *nvbo)1776{1777 struct drm_gem_object *obj = &nvbo->bo.base;1778 struct drm_gpuvm_bo *vm_bo;1779 struct drm_gpuva *va;1780 1781 dma_resv_assert_held(obj->resv);1782 1783 drm_gem_for_each_gpuvm_bo(vm_bo, obj) {1784 drm_gpuvm_bo_for_each_va(va, vm_bo) {1785 struct nouveau_uvma *uvma = uvma_from_va(va);1786 1787 nouveau_uvma_unmap(uvma);1788 drm_gpuva_invalidate(va, true);1789 }1790 }1791}1792 1793static void1794nouveau_uvmm_free(struct drm_gpuvm *gpuvm)1795{1796 struct nouveau_uvmm *uvmm = uvmm_from_gpuvm(gpuvm);1797 1798 kfree(uvmm);1799}1800 1801static int1802nouveau_uvmm_bo_validate(struct drm_gpuvm_bo *vm_bo, struct drm_exec *exec)1803{1804 struct nouveau_bo *nvbo = nouveau_gem_object(vm_bo->obj);1805 1806 nouveau_bo_placement_set(nvbo, nvbo->valid_domains, 0);1807 return nouveau_bo_validate(nvbo, true, false);1808}1809 1810static const struct drm_gpuvm_ops gpuvm_ops = {1811 .vm_free = nouveau_uvmm_free,1812 .vm_bo_validate = nouveau_uvmm_bo_validate,1813};1814 1815int1816nouveau_uvmm_ioctl_vm_init(struct drm_device *dev,1817 void *data,1818 struct drm_file *file_priv)1819{1820 struct nouveau_uvmm *uvmm;1821 struct nouveau_cli *cli = nouveau_cli(file_priv);1822 struct drm_device *drm = cli->drm->dev;1823 struct drm_gem_object *r_obj;1824 struct drm_nouveau_vm_init *init = data;1825 u64 kernel_managed_end;1826 int ret;1827 1828 if (check_add_overflow(init->kernel_managed_addr,1829 init->kernel_managed_size,1830 &kernel_managed_end))1831 return -EINVAL;1832 1833 if (kernel_managed_end > NOUVEAU_VA_SPACE_END)1834 return -EINVAL;1835 1836 mutex_lock(&cli->mutex);1837 1838 if (unlikely(cli->uvmm.disabled)) {1839 ret = -ENOSYS;1840 goto out_unlock;1841 }1842 1843 uvmm = kzalloc(sizeof(*uvmm), GFP_KERNEL);1844 if (!uvmm) {1845 ret = -ENOMEM;1846 goto out_unlock;1847 }1848 1849 r_obj = drm_gpuvm_resv_object_alloc(drm);1850 if (!r_obj) {1851 kfree(uvmm);1852 ret = -ENOMEM;1853 goto out_unlock;1854 }1855 1856 mutex_init(&uvmm->mutex);1857 mt_init_flags(&uvmm->region_mt, MT_FLAGS_LOCK_EXTERN);1858 mt_set_external_lock(&uvmm->region_mt, &uvmm->mutex);1859 1860 drm_gpuvm_init(&uvmm->base, cli->name, 0, drm, r_obj,1861 NOUVEAU_VA_SPACE_START,1862 NOUVEAU_VA_SPACE_END,1863 init->kernel_managed_addr,1864 init->kernel_managed_size,1865 &gpuvm_ops);1866 /* GPUVM takes care from here on. */1867 drm_gem_object_put(r_obj);1868 1869 ret = nvif_vmm_ctor(&cli->mmu, "uvmm",1870 cli->vmm.vmm.object.oclass, RAW,1871 init->kernel_managed_addr,1872 init->kernel_managed_size,1873 NULL, 0, &uvmm->vmm.vmm);1874 if (ret)1875 goto out_gpuvm_fini;1876 1877 uvmm->vmm.cli = cli;1878 cli->uvmm.ptr = uvmm;1879 mutex_unlock(&cli->mutex);1880 1881 return 0;1882 1883out_gpuvm_fini:1884 drm_gpuvm_put(&uvmm->base);1885out_unlock:1886 mutex_unlock(&cli->mutex);1887 return ret;1888}1889 1890void1891nouveau_uvmm_fini(struct nouveau_uvmm *uvmm)1892{1893 MA_STATE(mas, &uvmm->region_mt, 0, 0);1894 struct nouveau_uvma_region *reg;1895 struct nouveau_cli *cli = uvmm->vmm.cli;1896 struct drm_gpuva *va, *next;1897 1898 nouveau_uvmm_lock(uvmm);1899 drm_gpuvm_for_each_va_safe(va, next, &uvmm->base) {1900 struct nouveau_uvma *uvma = uvma_from_va(va);1901 struct drm_gem_object *obj = va->gem.obj;1902 1903 if (unlikely(va == &uvmm->base.kernel_alloc_node))1904 continue;1905 1906 drm_gpuva_remove(va);1907 1908 dma_resv_lock(obj->resv, NULL);1909 drm_gpuva_unlink(va);1910 dma_resv_unlock(obj->resv);1911 1912 nouveau_uvma_unmap(uvma);1913 nouveau_uvma_vmm_put(uvma);1914 1915 nouveau_uvma_gem_put(uvma);1916 nouveau_uvma_free(uvma);1917 }1918 1919 mas_for_each(&mas, reg, ULONG_MAX) {1920 mas_erase(&mas);1921 nouveau_uvma_region_sparse_unref(reg);1922 nouveau_uvma_region_put(reg);1923 }1924 1925 WARN(!mtree_empty(&uvmm->region_mt),1926 "nouveau_uvma_region tree not empty, potentially leaking memory.");1927 __mt_destroy(&uvmm->region_mt);1928 nouveau_uvmm_unlock(uvmm);1929 1930 mutex_lock(&cli->mutex);1931 nouveau_vmm_fini(&uvmm->vmm);1932 drm_gpuvm_put(&uvmm->base);1933 mutex_unlock(&cli->mutex);1934}1935