1043 lines · c
1/*2 * Copyright 2008 Advanced Micro Devices, Inc.3 * Copyright 2008 Red Hat Inc.4 * Copyright 2009 Jerome Glisse.5 *6 * Permission is hereby granted, free of charge, to any person obtaining a7 * copy of this software and associated documentation files (the "Software"),8 * to deal in the Software without restriction, including without limitation9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,10 * and/or sell copies of the Software, and to permit persons to whom the11 * Software is furnished to do so, subject to the following conditions:12 *13 * The above copyright notice and this permission notice shall be included in14 * all copies or substantial portions of the Software.15 *16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL19 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR22 * OTHER DEALINGS IN THE SOFTWARE.23 *24 * Authors: Dave Airlie25 * Alex Deucher26 * Jerome Glisse27 */28#include <linux/ktime.h>29#include <linux/module.h>30#include <linux/pagemap.h>31#include <linux/pci.h>32#include <linux/dma-buf.h>33 34#include <drm/amdgpu_drm.h>35#include <drm/drm_drv.h>36#include <drm/drm_exec.h>37#include <drm/drm_gem_ttm_helper.h>38#include <drm/ttm/ttm_tt.h>39 40#include "amdgpu.h"41#include "amdgpu_display.h"42#include "amdgpu_dma_buf.h"43#include "amdgpu_hmm.h"44#include "amdgpu_xgmi.h"45 46static vm_fault_t amdgpu_gem_fault(struct vm_fault *vmf)47{48 struct ttm_buffer_object *bo = vmf->vma->vm_private_data;49 struct drm_device *ddev = bo->base.dev;50 vm_fault_t ret;51 int idx;52 53 ret = ttm_bo_vm_reserve(bo, vmf);54 if (ret)55 return ret;56 57 if (drm_dev_enter(ddev, &idx)) {58 ret = amdgpu_bo_fault_reserve_notify(bo);59 if (ret) {60 drm_dev_exit(idx);61 goto unlock;62 }63 64 ret = ttm_bo_vm_fault_reserved(vmf, vmf->vma->vm_page_prot,65 TTM_BO_VM_NUM_PREFAULT);66 67 drm_dev_exit(idx);68 } else {69 ret = ttm_bo_vm_dummy_page(vmf, vmf->vma->vm_page_prot);70 }71 if (ret == VM_FAULT_RETRY && !(vmf->flags & FAULT_FLAG_RETRY_NOWAIT))72 return ret;73 74unlock:75 dma_resv_unlock(bo->base.resv);76 return ret;77}78 79static const struct vm_operations_struct amdgpu_gem_vm_ops = {80 .fault = amdgpu_gem_fault,81 .open = ttm_bo_vm_open,82 .close = ttm_bo_vm_close,83 .access = ttm_bo_vm_access84};85 86static void amdgpu_gem_object_free(struct drm_gem_object *gobj)87{88 struct amdgpu_bo *aobj = gem_to_amdgpu_bo(gobj);89 90 if (aobj) {91 amdgpu_hmm_unregister(aobj);92 ttm_bo_put(&aobj->tbo);93 }94}95 96int amdgpu_gem_object_create(struct amdgpu_device *adev, unsigned long size,97 int alignment, u32 initial_domain,98 u64 flags, enum ttm_bo_type type,99 struct dma_resv *resv,100 struct drm_gem_object **obj, int8_t xcp_id_plus1)101{102 struct amdgpu_bo *bo;103 struct amdgpu_bo_user *ubo;104 struct amdgpu_bo_param bp;105 int r;106 107 memset(&bp, 0, sizeof(bp));108 *obj = NULL;109 flags |= AMDGPU_GEM_CREATE_VRAM_WIPE_ON_RELEASE;110 111 bp.size = size;112 bp.byte_align = alignment;113 bp.type = type;114 bp.resv = resv;115 bp.preferred_domain = initial_domain;116 bp.flags = flags;117 bp.domain = initial_domain;118 bp.bo_ptr_size = sizeof(struct amdgpu_bo);119 bp.xcp_id_plus1 = xcp_id_plus1;120 121 r = amdgpu_bo_create_user(adev, &bp, &ubo);122 if (r)123 return r;124 125 bo = &ubo->bo;126 *obj = &bo->tbo.base;127 128 return 0;129}130 131void amdgpu_gem_force_release(struct amdgpu_device *adev)132{133 struct drm_device *ddev = adev_to_drm(adev);134 struct drm_file *file;135 136 mutex_lock(&ddev->filelist_mutex);137 138 list_for_each_entry(file, &ddev->filelist, lhead) {139 struct drm_gem_object *gobj;140 int handle;141 142 WARN_ONCE(1, "Still active user space clients!\n");143 spin_lock(&file->table_lock);144 idr_for_each_entry(&file->object_idr, gobj, handle) {145 WARN_ONCE(1, "And also active allocations!\n");146 drm_gem_object_put(gobj);147 }148 idr_destroy(&file->object_idr);149 spin_unlock(&file->table_lock);150 }151 152 mutex_unlock(&ddev->filelist_mutex);153}154 155/*156 * Call from drm_gem_handle_create which appear in both new and open ioctl157 * case.158 */159static int amdgpu_gem_object_open(struct drm_gem_object *obj,160 struct drm_file *file_priv)161{162 struct amdgpu_bo *abo = gem_to_amdgpu_bo(obj);163 struct amdgpu_device *adev = amdgpu_ttm_adev(abo->tbo.bdev);164 struct amdgpu_fpriv *fpriv = file_priv->driver_priv;165 struct amdgpu_vm *vm = &fpriv->vm;166 struct amdgpu_bo_va *bo_va;167 struct mm_struct *mm;168 int r;169 170 mm = amdgpu_ttm_tt_get_usermm(abo->tbo.ttm);171 if (mm && mm != current->mm)172 return -EPERM;173 174 if (abo->flags & AMDGPU_GEM_CREATE_VM_ALWAYS_VALID &&175 !amdgpu_vm_is_bo_always_valid(vm, abo))176 return -EPERM;177 178 r = amdgpu_bo_reserve(abo, false);179 if (r)180 return r;181 182 bo_va = amdgpu_vm_bo_find(vm, abo);183 if (!bo_va)184 bo_va = amdgpu_vm_bo_add(adev, vm, abo);185 else186 ++bo_va->ref_count;187 amdgpu_bo_unreserve(abo);188 189 /* Validate and add eviction fence to DMABuf imports with dynamic190 * attachment in compute VMs. Re-validation will be done by191 * amdgpu_vm_validate. Fences are on the reservation shared with the192 * export, which is currently required to be validated and fenced193 * already by amdgpu_amdkfd_gpuvm_restore_process_bos.194 *195 * Nested locking below for the case that a GEM object is opened in196 * kfd_mem_export_dmabuf. Since the lock below is only taken for imports,197 * but not for export, this is a different lock class that cannot lead to198 * circular lock dependencies.199 */200 if (!vm->is_compute_context || !vm->process_info)201 return 0;202 if (!obj->import_attach ||203 !dma_buf_is_dynamic(obj->import_attach->dmabuf))204 return 0;205 mutex_lock_nested(&vm->process_info->lock, 1);206 if (!WARN_ON(!vm->process_info->eviction_fence)) {207 r = amdgpu_amdkfd_bo_validate_and_fence(abo, AMDGPU_GEM_DOMAIN_GTT,208 &vm->process_info->eviction_fence->base);209 if (r) {210 struct amdgpu_task_info *ti = amdgpu_vm_get_task_info_vm(vm);211 212 dev_warn(adev->dev, "validate_and_fence failed: %d\n", r);213 if (ti) {214 dev_warn(adev->dev, "pid %d\n", ti->pid);215 amdgpu_vm_put_task_info(ti);216 }217 }218 }219 mutex_unlock(&vm->process_info->lock);220 221 return r;222}223 224static void amdgpu_gem_object_close(struct drm_gem_object *obj,225 struct drm_file *file_priv)226{227 struct amdgpu_bo *bo = gem_to_amdgpu_bo(obj);228 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);229 struct amdgpu_fpriv *fpriv = file_priv->driver_priv;230 struct amdgpu_vm *vm = &fpriv->vm;231 232 struct dma_fence *fence = NULL;233 struct amdgpu_bo_va *bo_va;234 struct drm_exec exec;235 long r;236 237 drm_exec_init(&exec, DRM_EXEC_IGNORE_DUPLICATES, 0);238 drm_exec_until_all_locked(&exec) {239 r = drm_exec_prepare_obj(&exec, &bo->tbo.base, 1);240 drm_exec_retry_on_contention(&exec);241 if (unlikely(r))242 goto out_unlock;243 244 r = amdgpu_vm_lock_pd(vm, &exec, 0);245 drm_exec_retry_on_contention(&exec);246 if (unlikely(r))247 goto out_unlock;248 }249 250 bo_va = amdgpu_vm_bo_find(vm, bo);251 if (!bo_va || --bo_va->ref_count)252 goto out_unlock;253 254 amdgpu_vm_bo_del(adev, bo_va);255 if (!amdgpu_vm_ready(vm))256 goto out_unlock;257 258 r = amdgpu_vm_clear_freed(adev, vm, &fence);259 if (unlikely(r < 0))260 dev_err(adev->dev, "failed to clear page "261 "tables on GEM object close (%ld)\n", r);262 if (r || !fence)263 goto out_unlock;264 265 amdgpu_bo_fence(bo, fence, true);266 dma_fence_put(fence);267 268out_unlock:269 if (r)270 dev_err(adev->dev, "leaking bo va (%ld)\n", r);271 drm_exec_fini(&exec);272}273 274static int amdgpu_gem_object_mmap(struct drm_gem_object *obj, struct vm_area_struct *vma)275{276 struct amdgpu_bo *bo = gem_to_amdgpu_bo(obj);277 278 if (amdgpu_ttm_tt_get_usermm(bo->tbo.ttm))279 return -EPERM;280 if (bo->flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS)281 return -EPERM;282 283 /* Workaround for Thunk bug creating PROT_NONE,MAP_PRIVATE mappings284 * for debugger access to invisible VRAM. Should have used MAP_SHARED285 * instead. Clearing VM_MAYWRITE prevents the mapping from ever286 * becoming writable and makes is_cow_mapping(vm_flags) false.287 */288 if (is_cow_mapping(vma->vm_flags) &&289 !(vma->vm_flags & VM_ACCESS_FLAGS))290 vm_flags_clear(vma, VM_MAYWRITE);291 292 return drm_gem_ttm_mmap(obj, vma);293}294 295const struct drm_gem_object_funcs amdgpu_gem_object_funcs = {296 .free = amdgpu_gem_object_free,297 .open = amdgpu_gem_object_open,298 .close = amdgpu_gem_object_close,299 .export = amdgpu_gem_prime_export,300 .vmap = drm_gem_ttm_vmap,301 .vunmap = drm_gem_ttm_vunmap,302 .mmap = amdgpu_gem_object_mmap,303 .vm_ops = &amdgpu_gem_vm_ops,304};305 306/*307 * GEM ioctls.308 */309int amdgpu_gem_create_ioctl(struct drm_device *dev, void *data,310 struct drm_file *filp)311{312 struct amdgpu_device *adev = drm_to_adev(dev);313 struct amdgpu_fpriv *fpriv = filp->driver_priv;314 struct amdgpu_vm *vm = &fpriv->vm;315 union drm_amdgpu_gem_create *args = data;316 uint64_t flags = args->in.domain_flags;317 uint64_t size = args->in.bo_size;318 struct dma_resv *resv = NULL;319 struct drm_gem_object *gobj;320 uint32_t handle, initial_domain;321 int r;322 323 /* reject DOORBELLs until userspace code to use it is available */324 if (args->in.domains & AMDGPU_GEM_DOMAIN_DOORBELL)325 return -EINVAL;326 327 /* reject invalid gem flags */328 if (flags & ~(AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED |329 AMDGPU_GEM_CREATE_NO_CPU_ACCESS |330 AMDGPU_GEM_CREATE_CPU_GTT_USWC |331 AMDGPU_GEM_CREATE_VRAM_CLEARED |332 AMDGPU_GEM_CREATE_VM_ALWAYS_VALID |333 AMDGPU_GEM_CREATE_EXPLICIT_SYNC |334 AMDGPU_GEM_CREATE_ENCRYPTED |335 AMDGPU_GEM_CREATE_GFX12_DCC |336 AMDGPU_GEM_CREATE_DISCARDABLE))337 return -EINVAL;338 339 /* reject invalid gem domains */340 if (args->in.domains & ~AMDGPU_GEM_DOMAIN_MASK)341 return -EINVAL;342 343 if (!amdgpu_is_tmz(adev) && (flags & AMDGPU_GEM_CREATE_ENCRYPTED)) {344 DRM_NOTE_ONCE("Cannot allocate secure buffer since TMZ is disabled\n");345 return -EINVAL;346 }347 348 /* always clear VRAM */349 flags |= AMDGPU_GEM_CREATE_VRAM_CLEARED;350 351 /* create a gem object to contain this object in */352 if (args->in.domains & (AMDGPU_GEM_DOMAIN_GDS |353 AMDGPU_GEM_DOMAIN_GWS | AMDGPU_GEM_DOMAIN_OA)) {354 if (flags & AMDGPU_GEM_CREATE_VM_ALWAYS_VALID) {355 /* if gds bo is created from user space, it must be356 * passed to bo list357 */358 DRM_ERROR("GDS bo cannot be per-vm-bo\n");359 return -EINVAL;360 }361 flags |= AMDGPU_GEM_CREATE_NO_CPU_ACCESS;362 }363 364 if (flags & AMDGPU_GEM_CREATE_VM_ALWAYS_VALID) {365 r = amdgpu_bo_reserve(vm->root.bo, false);366 if (r)367 return r;368 369 resv = vm->root.bo->tbo.base.resv;370 }371 372 initial_domain = (u32)(0xffffffff & args->in.domains);373retry:374 r = amdgpu_gem_object_create(adev, size, args->in.alignment,375 initial_domain,376 flags, ttm_bo_type_device, resv, &gobj, fpriv->xcp_id + 1);377 if (r && r != -ERESTARTSYS) {378 if (flags & AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED) {379 flags &= ~AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;380 goto retry;381 }382 383 if (initial_domain == AMDGPU_GEM_DOMAIN_VRAM) {384 initial_domain |= AMDGPU_GEM_DOMAIN_GTT;385 goto retry;386 }387 DRM_DEBUG("Failed to allocate GEM object (%llu, %d, %llu, %d)\n",388 size, initial_domain, args->in.alignment, r);389 }390 391 if (flags & AMDGPU_GEM_CREATE_VM_ALWAYS_VALID) {392 if (!r) {393 struct amdgpu_bo *abo = gem_to_amdgpu_bo(gobj);394 395 abo->parent = amdgpu_bo_ref(vm->root.bo);396 }397 amdgpu_bo_unreserve(vm->root.bo);398 }399 if (r)400 return r;401 402 r = drm_gem_handle_create(filp, gobj, &handle);403 /* drop reference from allocate - handle holds it now */404 drm_gem_object_put(gobj);405 if (r)406 return r;407 408 memset(args, 0, sizeof(*args));409 args->out.handle = handle;410 return 0;411}412 413int amdgpu_gem_userptr_ioctl(struct drm_device *dev, void *data,414 struct drm_file *filp)415{416 struct ttm_operation_ctx ctx = { true, false };417 struct amdgpu_device *adev = drm_to_adev(dev);418 struct drm_amdgpu_gem_userptr *args = data;419 struct amdgpu_fpriv *fpriv = filp->driver_priv;420 struct drm_gem_object *gobj;421 struct hmm_range *range;422 struct amdgpu_bo *bo;423 uint32_t handle;424 int r;425 426 args->addr = untagged_addr(args->addr);427 428 if (offset_in_page(args->addr | args->size))429 return -EINVAL;430 431 /* reject unknown flag values */432 if (args->flags & ~(AMDGPU_GEM_USERPTR_READONLY |433 AMDGPU_GEM_USERPTR_ANONONLY | AMDGPU_GEM_USERPTR_VALIDATE |434 AMDGPU_GEM_USERPTR_REGISTER))435 return -EINVAL;436 437 if (!(args->flags & AMDGPU_GEM_USERPTR_READONLY) &&438 !(args->flags & AMDGPU_GEM_USERPTR_REGISTER)) {439 440 /* if we want to write to it we must install a MMU notifier */441 return -EACCES;442 }443 444 /* create a gem object to contain this object in */445 r = amdgpu_gem_object_create(adev, args->size, 0, AMDGPU_GEM_DOMAIN_CPU,446 0, ttm_bo_type_device, NULL, &gobj, fpriv->xcp_id + 1);447 if (r)448 return r;449 450 bo = gem_to_amdgpu_bo(gobj);451 bo->preferred_domains = AMDGPU_GEM_DOMAIN_GTT;452 bo->allowed_domains = AMDGPU_GEM_DOMAIN_GTT;453 r = amdgpu_ttm_tt_set_userptr(&bo->tbo, args->addr, args->flags);454 if (r)455 goto release_object;456 457 r = amdgpu_hmm_register(bo, args->addr);458 if (r)459 goto release_object;460 461 if (args->flags & AMDGPU_GEM_USERPTR_VALIDATE) {462 r = amdgpu_ttm_tt_get_user_pages(bo, bo->tbo.ttm->pages,463 &range);464 if (r)465 goto release_object;466 467 r = amdgpu_bo_reserve(bo, true);468 if (r)469 goto user_pages_done;470 471 amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_GTT);472 r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);473 amdgpu_bo_unreserve(bo);474 if (r)475 goto user_pages_done;476 }477 478 r = drm_gem_handle_create(filp, gobj, &handle);479 if (r)480 goto user_pages_done;481 482 args->handle = handle;483 484user_pages_done:485 if (args->flags & AMDGPU_GEM_USERPTR_VALIDATE)486 amdgpu_ttm_tt_get_user_pages_done(bo->tbo.ttm, range);487 488release_object:489 drm_gem_object_put(gobj);490 491 return r;492}493 494int amdgpu_mode_dumb_mmap(struct drm_file *filp,495 struct drm_device *dev,496 uint32_t handle, uint64_t *offset_p)497{498 struct drm_gem_object *gobj;499 struct amdgpu_bo *robj;500 501 gobj = drm_gem_object_lookup(filp, handle);502 if (!gobj)503 return -ENOENT;504 505 robj = gem_to_amdgpu_bo(gobj);506 if (amdgpu_ttm_tt_get_usermm(robj->tbo.ttm) ||507 (robj->flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS)) {508 drm_gem_object_put(gobj);509 return -EPERM;510 }511 *offset_p = amdgpu_bo_mmap_offset(robj);512 drm_gem_object_put(gobj);513 return 0;514}515 516int amdgpu_gem_mmap_ioctl(struct drm_device *dev, void *data,517 struct drm_file *filp)518{519 union drm_amdgpu_gem_mmap *args = data;520 uint32_t handle = args->in.handle;521 522 memset(args, 0, sizeof(*args));523 return amdgpu_mode_dumb_mmap(filp, dev, handle, &args->out.addr_ptr);524}525 526/**527 * amdgpu_gem_timeout - calculate jiffies timeout from absolute value528 *529 * @timeout_ns: timeout in ns530 *531 * Calculate the timeout in jiffies from an absolute timeout in ns.532 */533unsigned long amdgpu_gem_timeout(uint64_t timeout_ns)534{535 unsigned long timeout_jiffies;536 ktime_t timeout;537 538 /* clamp timeout if it's to large */539 if (((int64_t)timeout_ns) < 0)540 return MAX_SCHEDULE_TIMEOUT;541 542 timeout = ktime_sub(ns_to_ktime(timeout_ns), ktime_get());543 if (ktime_to_ns(timeout) < 0)544 return 0;545 546 timeout_jiffies = nsecs_to_jiffies(ktime_to_ns(timeout));547 /* clamp timeout to avoid unsigned-> signed overflow */548 if (timeout_jiffies > MAX_SCHEDULE_TIMEOUT)549 return MAX_SCHEDULE_TIMEOUT - 1;550 551 return timeout_jiffies;552}553 554int amdgpu_gem_wait_idle_ioctl(struct drm_device *dev, void *data,555 struct drm_file *filp)556{557 union drm_amdgpu_gem_wait_idle *args = data;558 struct drm_gem_object *gobj;559 struct amdgpu_bo *robj;560 uint32_t handle = args->in.handle;561 unsigned long timeout = amdgpu_gem_timeout(args->in.timeout);562 int r = 0;563 long ret;564 565 gobj = drm_gem_object_lookup(filp, handle);566 if (!gobj)567 return -ENOENT;568 569 robj = gem_to_amdgpu_bo(gobj);570 ret = dma_resv_wait_timeout(robj->tbo.base.resv, DMA_RESV_USAGE_READ,571 true, timeout);572 573 /* ret == 0 means not signaled,574 * ret > 0 means signaled575 * ret < 0 means interrupted before timeout576 */577 if (ret >= 0) {578 memset(args, 0, sizeof(*args));579 args->out.status = (ret == 0);580 } else581 r = ret;582 583 drm_gem_object_put(gobj);584 return r;585}586 587int amdgpu_gem_metadata_ioctl(struct drm_device *dev, void *data,588 struct drm_file *filp)589{590 struct drm_amdgpu_gem_metadata *args = data;591 struct drm_gem_object *gobj;592 struct amdgpu_bo *robj;593 int r = -1;594 595 DRM_DEBUG("%d\n", args->handle);596 gobj = drm_gem_object_lookup(filp, args->handle);597 if (gobj == NULL)598 return -ENOENT;599 robj = gem_to_amdgpu_bo(gobj);600 601 r = amdgpu_bo_reserve(robj, false);602 if (unlikely(r != 0))603 goto out;604 605 if (args->op == AMDGPU_GEM_METADATA_OP_GET_METADATA) {606 amdgpu_bo_get_tiling_flags(robj, &args->data.tiling_info);607 r = amdgpu_bo_get_metadata(robj, args->data.data,608 sizeof(args->data.data),609 &args->data.data_size_bytes,610 &args->data.flags);611 } else if (args->op == AMDGPU_GEM_METADATA_OP_SET_METADATA) {612 if (args->data.data_size_bytes > sizeof(args->data.data)) {613 r = -EINVAL;614 goto unreserve;615 }616 r = amdgpu_bo_set_tiling_flags(robj, args->data.tiling_info);617 if (!r)618 r = amdgpu_bo_set_metadata(robj, args->data.data,619 args->data.data_size_bytes,620 args->data.flags);621 }622 623unreserve:624 amdgpu_bo_unreserve(robj);625out:626 drm_gem_object_put(gobj);627 return r;628}629 630/**631 * amdgpu_gem_va_update_vm -update the bo_va in its VM632 *633 * @adev: amdgpu_device pointer634 * @vm: vm to update635 * @bo_va: bo_va to update636 * @operation: map, unmap or clear637 *638 * Update the bo_va directly after setting its address. Errors are not639 * vital here, so they are not reported back to userspace.640 */641static void amdgpu_gem_va_update_vm(struct amdgpu_device *adev,642 struct amdgpu_vm *vm,643 struct amdgpu_bo_va *bo_va,644 uint32_t operation)645{646 int r;647 648 if (!amdgpu_vm_ready(vm))649 return;650 651 r = amdgpu_vm_clear_freed(adev, vm, NULL);652 if (r)653 goto error;654 655 if (operation == AMDGPU_VA_OP_MAP ||656 operation == AMDGPU_VA_OP_REPLACE) {657 r = amdgpu_vm_bo_update(adev, bo_va, false);658 if (r)659 goto error;660 }661 662 r = amdgpu_vm_update_pdes(adev, vm, false);663 664error:665 if (r && r != -ERESTARTSYS)666 DRM_ERROR("Couldn't update BO_VA (%d)\n", r);667}668 669/**670 * amdgpu_gem_va_map_flags - map GEM UAPI flags into hardware flags671 *672 * @adev: amdgpu_device pointer673 * @flags: GEM UAPI flags674 *675 * Returns the GEM UAPI flags mapped into hardware for the ASIC.676 */677uint64_t amdgpu_gem_va_map_flags(struct amdgpu_device *adev, uint32_t flags)678{679 uint64_t pte_flag = 0;680 681 if (flags & AMDGPU_VM_PAGE_EXECUTABLE)682 pte_flag |= AMDGPU_PTE_EXECUTABLE;683 if (flags & AMDGPU_VM_PAGE_READABLE)684 pte_flag |= AMDGPU_PTE_READABLE;685 if (flags & AMDGPU_VM_PAGE_WRITEABLE)686 pte_flag |= AMDGPU_PTE_WRITEABLE;687 if (flags & AMDGPU_VM_PAGE_PRT)688 pte_flag |= AMDGPU_PTE_PRT_FLAG(adev);689 if (flags & AMDGPU_VM_PAGE_NOALLOC)690 pte_flag |= AMDGPU_PTE_NOALLOC;691 692 if (adev->gmc.gmc_funcs->map_mtype)693 pte_flag |= amdgpu_gmc_map_mtype(adev,694 flags & AMDGPU_VM_MTYPE_MASK);695 696 return pte_flag;697}698 699int amdgpu_gem_va_ioctl(struct drm_device *dev, void *data,700 struct drm_file *filp)701{702 const uint32_t valid_flags = AMDGPU_VM_DELAY_UPDATE |703 AMDGPU_VM_PAGE_READABLE | AMDGPU_VM_PAGE_WRITEABLE |704 AMDGPU_VM_PAGE_EXECUTABLE | AMDGPU_VM_MTYPE_MASK |705 AMDGPU_VM_PAGE_NOALLOC;706 const uint32_t prt_flags = AMDGPU_VM_DELAY_UPDATE |707 AMDGPU_VM_PAGE_PRT;708 709 struct drm_amdgpu_gem_va *args = data;710 struct drm_gem_object *gobj;711 struct amdgpu_device *adev = drm_to_adev(dev);712 struct amdgpu_fpriv *fpriv = filp->driver_priv;713 struct amdgpu_bo *abo;714 struct amdgpu_bo_va *bo_va;715 struct drm_exec exec;716 uint64_t va_flags;717 uint64_t vm_size;718 int r = 0;719 720 if (args->va_address < AMDGPU_VA_RESERVED_BOTTOM) {721 dev_dbg(dev->dev,722 "va_address 0x%llx is in reserved area 0x%llx\n",723 args->va_address, AMDGPU_VA_RESERVED_BOTTOM);724 return -EINVAL;725 }726 727 if (args->va_address >= AMDGPU_GMC_HOLE_START &&728 args->va_address < AMDGPU_GMC_HOLE_END) {729 dev_dbg(dev->dev,730 "va_address 0x%llx is in VA hole 0x%llx-0x%llx\n",731 args->va_address, AMDGPU_GMC_HOLE_START,732 AMDGPU_GMC_HOLE_END);733 return -EINVAL;734 }735 736 args->va_address &= AMDGPU_GMC_HOLE_MASK;737 738 vm_size = adev->vm_manager.max_pfn * AMDGPU_GPU_PAGE_SIZE;739 vm_size -= AMDGPU_VA_RESERVED_TOP;740 if (args->va_address + args->map_size > vm_size) {741 dev_dbg(dev->dev,742 "va_address 0x%llx is in top reserved area 0x%llx\n",743 args->va_address + args->map_size, vm_size);744 return -EINVAL;745 }746 747 if ((args->flags & ~valid_flags) && (args->flags & ~prt_flags)) {748 dev_dbg(dev->dev, "invalid flags combination 0x%08X\n",749 args->flags);750 return -EINVAL;751 }752 753 switch (args->operation) {754 case AMDGPU_VA_OP_MAP:755 case AMDGPU_VA_OP_UNMAP:756 case AMDGPU_VA_OP_CLEAR:757 case AMDGPU_VA_OP_REPLACE:758 break;759 default:760 dev_dbg(dev->dev, "unsupported operation %d\n",761 args->operation);762 return -EINVAL;763 }764 765 if ((args->operation != AMDGPU_VA_OP_CLEAR) &&766 !(args->flags & AMDGPU_VM_PAGE_PRT)) {767 gobj = drm_gem_object_lookup(filp, args->handle);768 if (gobj == NULL)769 return -ENOENT;770 abo = gem_to_amdgpu_bo(gobj);771 } else {772 gobj = NULL;773 abo = NULL;774 }775 776 drm_exec_init(&exec, DRM_EXEC_INTERRUPTIBLE_WAIT |777 DRM_EXEC_IGNORE_DUPLICATES, 0);778 drm_exec_until_all_locked(&exec) {779 if (gobj) {780 r = drm_exec_lock_obj(&exec, gobj);781 drm_exec_retry_on_contention(&exec);782 if (unlikely(r))783 goto error;784 }785 786 r = amdgpu_vm_lock_pd(&fpriv->vm, &exec, 2);787 drm_exec_retry_on_contention(&exec);788 if (unlikely(r))789 goto error;790 }791 792 if (abo) {793 bo_va = amdgpu_vm_bo_find(&fpriv->vm, abo);794 if (!bo_va) {795 r = -ENOENT;796 goto error;797 }798 } else if (args->operation != AMDGPU_VA_OP_CLEAR) {799 bo_va = fpriv->prt_va;800 } else {801 bo_va = NULL;802 }803 804 switch (args->operation) {805 case AMDGPU_VA_OP_MAP:806 va_flags = amdgpu_gem_va_map_flags(adev, args->flags);807 r = amdgpu_vm_bo_map(adev, bo_va, args->va_address,808 args->offset_in_bo, args->map_size,809 va_flags);810 break;811 case AMDGPU_VA_OP_UNMAP:812 r = amdgpu_vm_bo_unmap(adev, bo_va, args->va_address);813 break;814 815 case AMDGPU_VA_OP_CLEAR:816 r = amdgpu_vm_bo_clear_mappings(adev, &fpriv->vm,817 args->va_address,818 args->map_size);819 break;820 case AMDGPU_VA_OP_REPLACE:821 va_flags = amdgpu_gem_va_map_flags(adev, args->flags);822 r = amdgpu_vm_bo_replace_map(adev, bo_va, args->va_address,823 args->offset_in_bo, args->map_size,824 va_flags);825 break;826 default:827 break;828 }829 if (!r && !(args->flags & AMDGPU_VM_DELAY_UPDATE) && !adev->debug_vm)830 amdgpu_gem_va_update_vm(adev, &fpriv->vm, bo_va,831 args->operation);832 833error:834 drm_exec_fini(&exec);835 drm_gem_object_put(gobj);836 return r;837}838 839int amdgpu_gem_op_ioctl(struct drm_device *dev, void *data,840 struct drm_file *filp)841{842 struct amdgpu_device *adev = drm_to_adev(dev);843 struct drm_amdgpu_gem_op *args = data;844 struct drm_gem_object *gobj;845 struct amdgpu_vm_bo_base *base;846 struct amdgpu_bo *robj;847 int r;848 849 gobj = drm_gem_object_lookup(filp, args->handle);850 if (!gobj)851 return -ENOENT;852 853 robj = gem_to_amdgpu_bo(gobj);854 855 r = amdgpu_bo_reserve(robj, false);856 if (unlikely(r))857 goto out;858 859 switch (args->op) {860 case AMDGPU_GEM_OP_GET_GEM_CREATE_INFO: {861 struct drm_amdgpu_gem_create_in info;862 void __user *out = u64_to_user_ptr(args->value);863 864 info.bo_size = robj->tbo.base.size;865 info.alignment = robj->tbo.page_alignment << PAGE_SHIFT;866 info.domains = robj->preferred_domains;867 info.domain_flags = robj->flags;868 amdgpu_bo_unreserve(robj);869 if (copy_to_user(out, &info, sizeof(info)))870 r = -EFAULT;871 break;872 }873 case AMDGPU_GEM_OP_SET_PLACEMENT:874 if (robj->tbo.base.import_attach &&875 args->value & AMDGPU_GEM_DOMAIN_VRAM) {876 r = -EINVAL;877 amdgpu_bo_unreserve(robj);878 break;879 }880 if (amdgpu_ttm_tt_get_usermm(robj->tbo.ttm)) {881 r = -EPERM;882 amdgpu_bo_unreserve(robj);883 break;884 }885 for (base = robj->vm_bo; base; base = base->next)886 if (amdgpu_xgmi_same_hive(amdgpu_ttm_adev(robj->tbo.bdev),887 amdgpu_ttm_adev(base->vm->root.bo->tbo.bdev))) {888 r = -EINVAL;889 amdgpu_bo_unreserve(robj);890 goto out;891 }892 893 894 robj->preferred_domains = args->value & (AMDGPU_GEM_DOMAIN_VRAM |895 AMDGPU_GEM_DOMAIN_GTT |896 AMDGPU_GEM_DOMAIN_CPU);897 robj->allowed_domains = robj->preferred_domains;898 if (robj->allowed_domains == AMDGPU_GEM_DOMAIN_VRAM)899 robj->allowed_domains |= AMDGPU_GEM_DOMAIN_GTT;900 901 if (robj->flags & AMDGPU_GEM_CREATE_VM_ALWAYS_VALID)902 amdgpu_vm_bo_invalidate(adev, robj, true);903 904 amdgpu_bo_unreserve(robj);905 break;906 default:907 amdgpu_bo_unreserve(robj);908 r = -EINVAL;909 }910 911out:912 drm_gem_object_put(gobj);913 return r;914}915 916static int amdgpu_gem_align_pitch(struct amdgpu_device *adev,917 int width,918 int cpp,919 bool tiled)920{921 int aligned = width;922 int pitch_mask = 0;923 924 switch (cpp) {925 case 1:926 pitch_mask = 255;927 break;928 case 2:929 pitch_mask = 127;930 break;931 case 3:932 case 4:933 pitch_mask = 63;934 break;935 }936 937 aligned += pitch_mask;938 aligned &= ~pitch_mask;939 return aligned * cpp;940}941 942int amdgpu_mode_dumb_create(struct drm_file *file_priv,943 struct drm_device *dev,944 struct drm_mode_create_dumb *args)945{946 struct amdgpu_device *adev = drm_to_adev(dev);947 struct amdgpu_fpriv *fpriv = file_priv->driver_priv;948 struct drm_gem_object *gobj;949 uint32_t handle;950 u64 flags = AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED |951 AMDGPU_GEM_CREATE_CPU_GTT_USWC |952 AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS;953 u32 domain;954 int r;955 956 /*957 * The buffer returned from this function should be cleared, but958 * it can only be done if the ring is enabled or we'll fail to959 * create the buffer.960 */961 if (adev->mman.buffer_funcs_enabled)962 flags |= AMDGPU_GEM_CREATE_VRAM_CLEARED;963 964 args->pitch = amdgpu_gem_align_pitch(adev, args->width,965 DIV_ROUND_UP(args->bpp, 8), 0);966 args->size = (u64)args->pitch * args->height;967 args->size = ALIGN(args->size, PAGE_SIZE);968 domain = amdgpu_bo_get_preferred_domain(adev,969 amdgpu_display_supported_domains(adev, flags));970 r = amdgpu_gem_object_create(adev, args->size, 0, domain, flags,971 ttm_bo_type_device, NULL, &gobj, fpriv->xcp_id + 1);972 if (r)973 return -ENOMEM;974 975 r = drm_gem_handle_create(file_priv, gobj, &handle);976 /* drop reference from allocate - handle holds it now */977 drm_gem_object_put(gobj);978 if (r)979 return r;980 981 args->handle = handle;982 return 0;983}984 985#if defined(CONFIG_DEBUG_FS)986static int amdgpu_debugfs_gem_info_show(struct seq_file *m, void *unused)987{988 struct amdgpu_device *adev = m->private;989 struct drm_device *dev = adev_to_drm(adev);990 struct drm_file *file;991 int r;992 993 r = mutex_lock_interruptible(&dev->filelist_mutex);994 if (r)995 return r;996 997 list_for_each_entry(file, &dev->filelist, lhead) {998 struct task_struct *task;999 struct drm_gem_object *gobj;1000 struct pid *pid;1001 int id;1002 1003 /*1004 * Although we have a valid reference on file->pid, that does1005 * not guarantee that the task_struct who called get_pid() is1006 * still alive (e.g. get_pid(current) => fork() => exit()).1007 * Therefore, we need to protect this ->comm access using RCU.1008 */1009 rcu_read_lock();1010 pid = rcu_dereference(file->pid);1011 task = pid_task(pid, PIDTYPE_TGID);1012 seq_printf(m, "pid %8d command %s:\n", pid_nr(pid),1013 task ? task->comm : "<unknown>");1014 rcu_read_unlock();1015 1016 spin_lock(&file->table_lock);1017 idr_for_each_entry(&file->object_idr, gobj, id) {1018 struct amdgpu_bo *bo = gem_to_amdgpu_bo(gobj);1019 1020 amdgpu_bo_print_info(id, bo, m);1021 }1022 spin_unlock(&file->table_lock);1023 }1024 1025 mutex_unlock(&dev->filelist_mutex);1026 return 0;1027}1028 1029DEFINE_SHOW_ATTRIBUTE(amdgpu_debugfs_gem_info);1030 1031#endif1032 1033void amdgpu_debugfs_gem_init(struct amdgpu_device *adev)1034{1035#if defined(CONFIG_DEBUG_FS)1036 struct drm_minor *minor = adev_to_drm(adev)->primary;1037 struct dentry *root = minor->debugfs_root;1038 1039 debugfs_create_file("amdgpu_gem_info", 0444, root, adev,1040 &amdgpu_debugfs_gem_info_fops);1041#endif1042}1043