1814 lines · c
1/*2 * Copyright 2008 Jerome Glisse.3 * All Rights Reserved.4 *5 * Permission is hereby granted, free of charge, to any person obtaining a6 * copy of this software and associated documentation files (the "Software"),7 * to deal in the Software without restriction, including without limitation8 * the rights to use, copy, modify, merge, publish, distribute, sublicense,9 * and/or sell copies of the Software, and to permit persons to whom the10 * Software is furnished to do so, subject to the following conditions:11 *12 * The above copyright notice and this permission notice (including the next13 * paragraph) shall be included in all copies or substantial portions of the14 * Software.15 *16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL19 * PRECISION INSIGHT AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER22 * DEALINGS IN THE SOFTWARE.23 *24 * Authors:25 * Jerome Glisse <glisse@freedesktop.org>26 */27 28#include <linux/file.h>29#include <linux/pagemap.h>30#include <linux/sync_file.h>31#include <linux/dma-buf.h>32 33#include <drm/amdgpu_drm.h>34#include <drm/drm_syncobj.h>35#include <drm/ttm/ttm_tt.h>36 37#include "amdgpu_cs.h"38#include "amdgpu.h"39#include "amdgpu_trace.h"40#include "amdgpu_gmc.h"41#include "amdgpu_gem.h"42#include "amdgpu_ras.h"43 44static int amdgpu_cs_parser_init(struct amdgpu_cs_parser *p,45 struct amdgpu_device *adev,46 struct drm_file *filp,47 union drm_amdgpu_cs *cs)48{49 struct amdgpu_fpriv *fpriv = filp->driver_priv;50 51 if (cs->in.num_chunks == 0)52 return -EINVAL;53 54 memset(p, 0, sizeof(*p));55 p->adev = adev;56 p->filp = filp;57 58 p->ctx = amdgpu_ctx_get(fpriv, cs->in.ctx_id);59 if (!p->ctx)60 return -EINVAL;61 62 if (atomic_read(&p->ctx->guilty)) {63 amdgpu_ctx_put(p->ctx);64 return -ECANCELED;65 }66 67 amdgpu_sync_create(&p->sync);68 drm_exec_init(&p->exec, DRM_EXEC_INTERRUPTIBLE_WAIT |69 DRM_EXEC_IGNORE_DUPLICATES, 0);70 return 0;71}72 73static int amdgpu_cs_job_idx(struct amdgpu_cs_parser *p,74 struct drm_amdgpu_cs_chunk_ib *chunk_ib)75{76 struct drm_sched_entity *entity;77 unsigned int i;78 int r;79 80 r = amdgpu_ctx_get_entity(p->ctx, chunk_ib->ip_type,81 chunk_ib->ip_instance,82 chunk_ib->ring, &entity);83 if (r)84 return r;85 86 /*87 * Abort if there is no run queue associated with this entity.88 * Possibly because of disabled HW IP.89 */90 if (entity->rq == NULL)91 return -EINVAL;92 93 /* Check if we can add this IB to some existing job */94 for (i = 0; i < p->gang_size; ++i)95 if (p->entities[i] == entity)96 return i;97 98 /* If not increase the gang size if possible */99 if (i == AMDGPU_CS_GANG_SIZE)100 return -EINVAL;101 102 p->entities[i] = entity;103 p->gang_size = i + 1;104 return i;105}106 107static int amdgpu_cs_p1_ib(struct amdgpu_cs_parser *p,108 struct drm_amdgpu_cs_chunk_ib *chunk_ib,109 unsigned int *num_ibs)110{111 int r;112 113 r = amdgpu_cs_job_idx(p, chunk_ib);114 if (r < 0)115 return r;116 117 if (num_ibs[r] >= amdgpu_ring_max_ibs(chunk_ib->ip_type))118 return -EINVAL;119 120 ++(num_ibs[r]);121 p->gang_leader_idx = r;122 return 0;123}124 125static int amdgpu_cs_p1_user_fence(struct amdgpu_cs_parser *p,126 struct drm_amdgpu_cs_chunk_fence *data,127 uint32_t *offset)128{129 struct drm_gem_object *gobj;130 unsigned long size;131 132 gobj = drm_gem_object_lookup(p->filp, data->handle);133 if (gobj == NULL)134 return -EINVAL;135 136 p->uf_bo = amdgpu_bo_ref(gem_to_amdgpu_bo(gobj));137 drm_gem_object_put(gobj);138 139 size = amdgpu_bo_size(p->uf_bo);140 if (size != PAGE_SIZE || data->offset > (size - 8))141 return -EINVAL;142 143 if (amdgpu_ttm_tt_get_usermm(p->uf_bo->tbo.ttm))144 return -EINVAL;145 146 *offset = data->offset;147 return 0;148}149 150static int amdgpu_cs_p1_bo_handles(struct amdgpu_cs_parser *p,151 struct drm_amdgpu_bo_list_in *data)152{153 struct drm_amdgpu_bo_list_entry *info;154 int r;155 156 r = amdgpu_bo_create_list_entry_array(data, &info);157 if (r)158 return r;159 160 r = amdgpu_bo_list_create(p->adev, p->filp, info, data->bo_number,161 &p->bo_list);162 if (r)163 goto error_free;164 165 kvfree(info);166 return 0;167 168error_free:169 kvfree(info);170 171 return r;172}173 174/* Copy the data from userspace and go over it the first time */175static int amdgpu_cs_pass1(struct amdgpu_cs_parser *p,176 union drm_amdgpu_cs *cs)177{178 struct amdgpu_fpriv *fpriv = p->filp->driver_priv;179 unsigned int num_ibs[AMDGPU_CS_GANG_SIZE] = { };180 struct amdgpu_vm *vm = &fpriv->vm;181 uint64_t *chunk_array_user;182 uint64_t *chunk_array;183 uint32_t uf_offset = 0;184 size_t size;185 int ret;186 int i;187 188 chunk_array = kvmalloc_array(cs->in.num_chunks, sizeof(uint64_t),189 GFP_KERNEL);190 if (!chunk_array)191 return -ENOMEM;192 193 /* get chunks */194 chunk_array_user = u64_to_user_ptr(cs->in.chunks);195 if (copy_from_user(chunk_array, chunk_array_user,196 sizeof(uint64_t)*cs->in.num_chunks)) {197 ret = -EFAULT;198 goto free_chunk;199 }200 201 p->nchunks = cs->in.num_chunks;202 p->chunks = kvmalloc_array(p->nchunks, sizeof(struct amdgpu_cs_chunk),203 GFP_KERNEL);204 if (!p->chunks) {205 ret = -ENOMEM;206 goto free_chunk;207 }208 209 for (i = 0; i < p->nchunks; i++) {210 struct drm_amdgpu_cs_chunk __user *chunk_ptr = NULL;211 struct drm_amdgpu_cs_chunk user_chunk;212 uint32_t __user *cdata;213 214 chunk_ptr = u64_to_user_ptr(chunk_array[i]);215 if (copy_from_user(&user_chunk, chunk_ptr,216 sizeof(struct drm_amdgpu_cs_chunk))) {217 ret = -EFAULT;218 i--;219 goto free_partial_kdata;220 }221 p->chunks[i].chunk_id = user_chunk.chunk_id;222 p->chunks[i].length_dw = user_chunk.length_dw;223 224 size = p->chunks[i].length_dw;225 cdata = u64_to_user_ptr(user_chunk.chunk_data);226 227 p->chunks[i].kdata = kvmalloc_array(size, sizeof(uint32_t),228 GFP_KERNEL);229 if (p->chunks[i].kdata == NULL) {230 ret = -ENOMEM;231 i--;232 goto free_partial_kdata;233 }234 size *= sizeof(uint32_t);235 if (copy_from_user(p->chunks[i].kdata, cdata, size)) {236 ret = -EFAULT;237 goto free_partial_kdata;238 }239 240 /* Assume the worst on the following checks */241 ret = -EINVAL;242 switch (p->chunks[i].chunk_id) {243 case AMDGPU_CHUNK_ID_IB:244 if (size < sizeof(struct drm_amdgpu_cs_chunk_ib))245 goto free_partial_kdata;246 247 ret = amdgpu_cs_p1_ib(p, p->chunks[i].kdata, num_ibs);248 if (ret)249 goto free_partial_kdata;250 break;251 252 case AMDGPU_CHUNK_ID_FENCE:253 if (size < sizeof(struct drm_amdgpu_cs_chunk_fence))254 goto free_partial_kdata;255 256 ret = amdgpu_cs_p1_user_fence(p, p->chunks[i].kdata,257 &uf_offset);258 if (ret)259 goto free_partial_kdata;260 break;261 262 case AMDGPU_CHUNK_ID_BO_HANDLES:263 if (size < sizeof(struct drm_amdgpu_bo_list_in))264 goto free_partial_kdata;265 266 /* Only a single BO list is allowed to simplify handling. */267 if (p->bo_list)268 goto free_partial_kdata;269 270 ret = amdgpu_cs_p1_bo_handles(p, p->chunks[i].kdata);271 if (ret)272 goto free_partial_kdata;273 break;274 275 case AMDGPU_CHUNK_ID_DEPENDENCIES:276 case AMDGPU_CHUNK_ID_SYNCOBJ_IN:277 case AMDGPU_CHUNK_ID_SYNCOBJ_OUT:278 case AMDGPU_CHUNK_ID_SCHEDULED_DEPENDENCIES:279 case AMDGPU_CHUNK_ID_SYNCOBJ_TIMELINE_WAIT:280 case AMDGPU_CHUNK_ID_SYNCOBJ_TIMELINE_SIGNAL:281 case AMDGPU_CHUNK_ID_CP_GFX_SHADOW:282 break;283 284 default:285 goto free_partial_kdata;286 }287 }288 289 if (!p->gang_size) {290 ret = -EINVAL;291 goto free_all_kdata;292 }293 294 for (i = 0; i < p->gang_size; ++i) {295 ret = amdgpu_job_alloc(p->adev, vm, p->entities[i], vm,296 num_ibs[i], &p->jobs[i]);297 if (ret)298 goto free_all_kdata;299 p->jobs[i]->enforce_isolation = p->adev->enforce_isolation[fpriv->xcp_id];300 }301 p->gang_leader = p->jobs[p->gang_leader_idx];302 303 if (p->ctx->generation != p->gang_leader->generation) {304 ret = -ECANCELED;305 goto free_all_kdata;306 }307 308 if (p->uf_bo)309 p->gang_leader->uf_addr = uf_offset;310 kvfree(chunk_array);311 312 /* Use this opportunity to fill in task info for the vm */313 amdgpu_vm_set_task_info(vm);314 315 return 0;316 317free_all_kdata:318 i = p->nchunks - 1;319free_partial_kdata:320 for (; i >= 0; i--)321 kvfree(p->chunks[i].kdata);322 kvfree(p->chunks);323 p->chunks = NULL;324 p->nchunks = 0;325free_chunk:326 kvfree(chunk_array);327 328 return ret;329}330 331static int amdgpu_cs_p2_ib(struct amdgpu_cs_parser *p,332 struct amdgpu_cs_chunk *chunk,333 unsigned int *ce_preempt,334 unsigned int *de_preempt)335{336 struct drm_amdgpu_cs_chunk_ib *chunk_ib = chunk->kdata;337 struct amdgpu_fpriv *fpriv = p->filp->driver_priv;338 struct amdgpu_vm *vm = &fpriv->vm;339 struct amdgpu_ring *ring;340 struct amdgpu_job *job;341 struct amdgpu_ib *ib;342 int r;343 344 r = amdgpu_cs_job_idx(p, chunk_ib);345 if (r < 0)346 return r;347 348 job = p->jobs[r];349 ring = amdgpu_job_ring(job);350 ib = &job->ibs[job->num_ibs++];351 352 /* MM engine doesn't support user fences */353 if (p->uf_bo && ring->funcs->no_user_fence)354 return -EINVAL;355 356 if (chunk_ib->ip_type == AMDGPU_HW_IP_GFX &&357 chunk_ib->flags & AMDGPU_IB_FLAG_PREEMPT) {358 if (chunk_ib->flags & AMDGPU_IB_FLAG_CE)359 (*ce_preempt)++;360 else361 (*de_preempt)++;362 363 /* Each GFX command submit allows only 1 IB max364 * preemptible for CE & DE */365 if (*ce_preempt > 1 || *de_preempt > 1)366 return -EINVAL;367 }368 369 if (chunk_ib->flags & AMDGPU_IB_FLAG_PREAMBLE)370 job->preamble_status |= AMDGPU_PREAMBLE_IB_PRESENT;371 372 r = amdgpu_ib_get(p->adev, vm, ring->funcs->parse_cs ?373 chunk_ib->ib_bytes : 0,374 AMDGPU_IB_POOL_DELAYED, ib);375 if (r) {376 DRM_ERROR("Failed to get ib !\n");377 return r;378 }379 380 ib->gpu_addr = chunk_ib->va_start;381 ib->length_dw = chunk_ib->ib_bytes / 4;382 ib->flags = chunk_ib->flags;383 return 0;384}385 386static int amdgpu_cs_p2_dependencies(struct amdgpu_cs_parser *p,387 struct amdgpu_cs_chunk *chunk)388{389 struct drm_amdgpu_cs_chunk_dep *deps = chunk->kdata;390 struct amdgpu_fpriv *fpriv = p->filp->driver_priv;391 unsigned int num_deps;392 int i, r;393 394 num_deps = chunk->length_dw * 4 /395 sizeof(struct drm_amdgpu_cs_chunk_dep);396 397 for (i = 0; i < num_deps; ++i) {398 struct amdgpu_ctx *ctx;399 struct drm_sched_entity *entity;400 struct dma_fence *fence;401 402 ctx = amdgpu_ctx_get(fpriv, deps[i].ctx_id);403 if (ctx == NULL)404 return -EINVAL;405 406 r = amdgpu_ctx_get_entity(ctx, deps[i].ip_type,407 deps[i].ip_instance,408 deps[i].ring, &entity);409 if (r) {410 amdgpu_ctx_put(ctx);411 return r;412 }413 414 fence = amdgpu_ctx_get_fence(ctx, entity, deps[i].handle);415 amdgpu_ctx_put(ctx);416 417 if (IS_ERR(fence))418 return PTR_ERR(fence);419 else if (!fence)420 continue;421 422 if (chunk->chunk_id == AMDGPU_CHUNK_ID_SCHEDULED_DEPENDENCIES) {423 struct drm_sched_fence *s_fence;424 struct dma_fence *old = fence;425 426 s_fence = to_drm_sched_fence(fence);427 fence = dma_fence_get(&s_fence->scheduled);428 dma_fence_put(old);429 }430 431 r = amdgpu_sync_fence(&p->sync, fence);432 dma_fence_put(fence);433 if (r)434 return r;435 }436 return 0;437}438 439static int amdgpu_syncobj_lookup_and_add(struct amdgpu_cs_parser *p,440 uint32_t handle, u64 point,441 u64 flags)442{443 struct dma_fence *fence;444 int r;445 446 r = drm_syncobj_find_fence(p->filp, handle, point, flags, &fence);447 if (r) {448 DRM_ERROR("syncobj %u failed to find fence @ %llu (%d)!\n",449 handle, point, r);450 return r;451 }452 453 r = amdgpu_sync_fence(&p->sync, fence);454 dma_fence_put(fence);455 return r;456}457 458static int amdgpu_cs_p2_syncobj_in(struct amdgpu_cs_parser *p,459 struct amdgpu_cs_chunk *chunk)460{461 struct drm_amdgpu_cs_chunk_sem *deps = chunk->kdata;462 unsigned int num_deps;463 int i, r;464 465 num_deps = chunk->length_dw * 4 /466 sizeof(struct drm_amdgpu_cs_chunk_sem);467 for (i = 0; i < num_deps; ++i) {468 r = amdgpu_syncobj_lookup_and_add(p, deps[i].handle, 0, 0);469 if (r)470 return r;471 }472 473 return 0;474}475 476static int amdgpu_cs_p2_syncobj_timeline_wait(struct amdgpu_cs_parser *p,477 struct amdgpu_cs_chunk *chunk)478{479 struct drm_amdgpu_cs_chunk_syncobj *syncobj_deps = chunk->kdata;480 unsigned int num_deps;481 int i, r;482 483 num_deps = chunk->length_dw * 4 /484 sizeof(struct drm_amdgpu_cs_chunk_syncobj);485 for (i = 0; i < num_deps; ++i) {486 r = amdgpu_syncobj_lookup_and_add(p, syncobj_deps[i].handle,487 syncobj_deps[i].point,488 syncobj_deps[i].flags);489 if (r)490 return r;491 }492 493 return 0;494}495 496static int amdgpu_cs_p2_syncobj_out(struct amdgpu_cs_parser *p,497 struct amdgpu_cs_chunk *chunk)498{499 struct drm_amdgpu_cs_chunk_sem *deps = chunk->kdata;500 unsigned int num_deps;501 int i;502 503 num_deps = chunk->length_dw * 4 /504 sizeof(struct drm_amdgpu_cs_chunk_sem);505 506 if (p->post_deps)507 return -EINVAL;508 509 p->post_deps = kmalloc_array(num_deps, sizeof(*p->post_deps),510 GFP_KERNEL);511 p->num_post_deps = 0;512 513 if (!p->post_deps)514 return -ENOMEM;515 516 517 for (i = 0; i < num_deps; ++i) {518 p->post_deps[i].syncobj =519 drm_syncobj_find(p->filp, deps[i].handle);520 if (!p->post_deps[i].syncobj)521 return -EINVAL;522 p->post_deps[i].chain = NULL;523 p->post_deps[i].point = 0;524 p->num_post_deps++;525 }526 527 return 0;528}529 530static int amdgpu_cs_p2_syncobj_timeline_signal(struct amdgpu_cs_parser *p,531 struct amdgpu_cs_chunk *chunk)532{533 struct drm_amdgpu_cs_chunk_syncobj *syncobj_deps = chunk->kdata;534 unsigned int num_deps;535 int i;536 537 num_deps = chunk->length_dw * 4 /538 sizeof(struct drm_amdgpu_cs_chunk_syncobj);539 540 if (p->post_deps)541 return -EINVAL;542 543 p->post_deps = kmalloc_array(num_deps, sizeof(*p->post_deps),544 GFP_KERNEL);545 p->num_post_deps = 0;546 547 if (!p->post_deps)548 return -ENOMEM;549 550 for (i = 0; i < num_deps; ++i) {551 struct amdgpu_cs_post_dep *dep = &p->post_deps[i];552 553 dep->chain = NULL;554 if (syncobj_deps[i].point) {555 dep->chain = dma_fence_chain_alloc();556 if (!dep->chain)557 return -ENOMEM;558 }559 560 dep->syncobj = drm_syncobj_find(p->filp,561 syncobj_deps[i].handle);562 if (!dep->syncobj) {563 dma_fence_chain_free(dep->chain);564 return -EINVAL;565 }566 dep->point = syncobj_deps[i].point;567 p->num_post_deps++;568 }569 570 return 0;571}572 573static int amdgpu_cs_p2_shadow(struct amdgpu_cs_parser *p,574 struct amdgpu_cs_chunk *chunk)575{576 struct drm_amdgpu_cs_chunk_cp_gfx_shadow *shadow = chunk->kdata;577 int i;578 579 if (shadow->flags & ~AMDGPU_CS_CHUNK_CP_GFX_SHADOW_FLAGS_INIT_SHADOW)580 return -EINVAL;581 582 for (i = 0; i < p->gang_size; ++i) {583 p->jobs[i]->shadow_va = shadow->shadow_va;584 p->jobs[i]->csa_va = shadow->csa_va;585 p->jobs[i]->gds_va = shadow->gds_va;586 p->jobs[i]->init_shadow =587 shadow->flags & AMDGPU_CS_CHUNK_CP_GFX_SHADOW_FLAGS_INIT_SHADOW;588 }589 590 return 0;591}592 593static int amdgpu_cs_pass2(struct amdgpu_cs_parser *p)594{595 unsigned int ce_preempt = 0, de_preempt = 0;596 int i, r;597 598 for (i = 0; i < p->nchunks; ++i) {599 struct amdgpu_cs_chunk *chunk;600 601 chunk = &p->chunks[i];602 603 switch (chunk->chunk_id) {604 case AMDGPU_CHUNK_ID_IB:605 r = amdgpu_cs_p2_ib(p, chunk, &ce_preempt, &de_preempt);606 if (r)607 return r;608 break;609 case AMDGPU_CHUNK_ID_DEPENDENCIES:610 case AMDGPU_CHUNK_ID_SCHEDULED_DEPENDENCIES:611 r = amdgpu_cs_p2_dependencies(p, chunk);612 if (r)613 return r;614 break;615 case AMDGPU_CHUNK_ID_SYNCOBJ_IN:616 r = amdgpu_cs_p2_syncobj_in(p, chunk);617 if (r)618 return r;619 break;620 case AMDGPU_CHUNK_ID_SYNCOBJ_OUT:621 r = amdgpu_cs_p2_syncobj_out(p, chunk);622 if (r)623 return r;624 break;625 case AMDGPU_CHUNK_ID_SYNCOBJ_TIMELINE_WAIT:626 r = amdgpu_cs_p2_syncobj_timeline_wait(p, chunk);627 if (r)628 return r;629 break;630 case AMDGPU_CHUNK_ID_SYNCOBJ_TIMELINE_SIGNAL:631 r = amdgpu_cs_p2_syncobj_timeline_signal(p, chunk);632 if (r)633 return r;634 break;635 case AMDGPU_CHUNK_ID_CP_GFX_SHADOW:636 r = amdgpu_cs_p2_shadow(p, chunk);637 if (r)638 return r;639 break;640 }641 }642 643 return 0;644}645 646/* Convert microseconds to bytes. */647static u64 us_to_bytes(struct amdgpu_device *adev, s64 us)648{649 if (us <= 0 || !adev->mm_stats.log2_max_MBps)650 return 0;651 652 /* Since accum_us is incremented by a million per second, just653 * multiply it by the number of MB/s to get the number of bytes.654 */655 return us << adev->mm_stats.log2_max_MBps;656}657 658static s64 bytes_to_us(struct amdgpu_device *adev, u64 bytes)659{660 if (!adev->mm_stats.log2_max_MBps)661 return 0;662 663 return bytes >> adev->mm_stats.log2_max_MBps;664}665 666/* Returns how many bytes TTM can move right now. If no bytes can be moved,667 * it returns 0. If it returns non-zero, it's OK to move at least one buffer,668 * which means it can go over the threshold once. If that happens, the driver669 * will be in debt and no other buffer migrations can be done until that debt670 * is repaid.671 *672 * This approach allows moving a buffer of any size (it's important to allow673 * that).674 *675 * The currency is simply time in microseconds and it increases as the clock676 * ticks. The accumulated microseconds (us) are converted to bytes and677 * returned.678 */679static void amdgpu_cs_get_threshold_for_moves(struct amdgpu_device *adev,680 u64 *max_bytes,681 u64 *max_vis_bytes)682{683 s64 time_us, increment_us;684 u64 free_vram, total_vram, used_vram;685 /* Allow a maximum of 200 accumulated ms. This is basically per-IB686 * throttling.687 *688 * It means that in order to get full max MBps, at least 5 IBs per689 * second must be submitted and not more than 200ms apart from each690 * other.691 */692 const s64 us_upper_bound = 200000;693 694 if (!adev->mm_stats.log2_max_MBps) {695 *max_bytes = 0;696 *max_vis_bytes = 0;697 return;698 }699 700 total_vram = adev->gmc.real_vram_size - atomic64_read(&adev->vram_pin_size);701 used_vram = ttm_resource_manager_usage(&adev->mman.vram_mgr.manager);702 free_vram = used_vram >= total_vram ? 0 : total_vram - used_vram;703 704 spin_lock(&adev->mm_stats.lock);705 706 /* Increase the amount of accumulated us. */707 time_us = ktime_to_us(ktime_get());708 increment_us = time_us - adev->mm_stats.last_update_us;709 adev->mm_stats.last_update_us = time_us;710 adev->mm_stats.accum_us = min(adev->mm_stats.accum_us + increment_us,711 us_upper_bound);712 713 /* This prevents the short period of low performance when the VRAM714 * usage is low and the driver is in debt or doesn't have enough715 * accumulated us to fill VRAM quickly.716 *717 * The situation can occur in these cases:718 * - a lot of VRAM is freed by userspace719 * - the presence of a big buffer causes a lot of evictions720 * (solution: split buffers into smaller ones)721 *722 * If 128 MB or 1/8th of VRAM is free, start filling it now by setting723 * accum_us to a positive number.724 */725 if (free_vram >= 128 * 1024 * 1024 || free_vram >= total_vram / 8) {726 s64 min_us;727 728 /* Be more aggressive on dGPUs. Try to fill a portion of free729 * VRAM now.730 */731 if (!(adev->flags & AMD_IS_APU))732 min_us = bytes_to_us(adev, free_vram / 4);733 else734 min_us = 0; /* Reset accum_us on APUs. */735 736 adev->mm_stats.accum_us = max(min_us, adev->mm_stats.accum_us);737 }738 739 /* This is set to 0 if the driver is in debt to disallow (optional)740 * buffer moves.741 */742 *max_bytes = us_to_bytes(adev, adev->mm_stats.accum_us);743 744 /* Do the same for visible VRAM if half of it is free */745 if (!amdgpu_gmc_vram_full_visible(&adev->gmc)) {746 u64 total_vis_vram = adev->gmc.visible_vram_size;747 u64 used_vis_vram =748 amdgpu_vram_mgr_vis_usage(&adev->mman.vram_mgr);749 750 if (used_vis_vram < total_vis_vram) {751 u64 free_vis_vram = total_vis_vram - used_vis_vram;752 753 adev->mm_stats.accum_us_vis = min(adev->mm_stats.accum_us_vis +754 increment_us, us_upper_bound);755 756 if (free_vis_vram >= total_vis_vram / 2)757 adev->mm_stats.accum_us_vis =758 max(bytes_to_us(adev, free_vis_vram / 2),759 adev->mm_stats.accum_us_vis);760 }761 762 *max_vis_bytes = us_to_bytes(adev, adev->mm_stats.accum_us_vis);763 } else {764 *max_vis_bytes = 0;765 }766 767 spin_unlock(&adev->mm_stats.lock);768}769 770/* Report how many bytes have really been moved for the last command771 * submission. This can result in a debt that can stop buffer migrations772 * temporarily.773 */774void amdgpu_cs_report_moved_bytes(struct amdgpu_device *adev, u64 num_bytes,775 u64 num_vis_bytes)776{777 spin_lock(&adev->mm_stats.lock);778 adev->mm_stats.accum_us -= bytes_to_us(adev, num_bytes);779 adev->mm_stats.accum_us_vis -= bytes_to_us(adev, num_vis_bytes);780 spin_unlock(&adev->mm_stats.lock);781}782 783static int amdgpu_cs_bo_validate(void *param, struct amdgpu_bo *bo)784{785 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);786 struct amdgpu_cs_parser *p = param;787 struct ttm_operation_ctx ctx = {788 .interruptible = true,789 .no_wait_gpu = false,790 .resv = bo->tbo.base.resv791 };792 uint32_t domain;793 int r;794 795 if (bo->tbo.pin_count)796 return 0;797 798 /* Don't move this buffer if we have depleted our allowance799 * to move it. Don't move anything if the threshold is zero.800 */801 if (p->bytes_moved < p->bytes_moved_threshold &&802 (!bo->tbo.base.dma_buf ||803 list_empty(&bo->tbo.base.dma_buf->attachments))) {804 if (!amdgpu_gmc_vram_full_visible(&adev->gmc) &&805 (bo->flags & AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED)) {806 /* And don't move a CPU_ACCESS_REQUIRED BO to limited807 * visible VRAM if we've depleted our allowance to do808 * that.809 */810 if (p->bytes_moved_vis < p->bytes_moved_vis_threshold)811 domain = bo->preferred_domains;812 else813 domain = bo->allowed_domains;814 } else {815 domain = bo->preferred_domains;816 }817 } else {818 domain = bo->allowed_domains;819 }820 821retry:822 amdgpu_bo_placement_from_domain(bo, domain);823 r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);824 825 p->bytes_moved += ctx.bytes_moved;826 if (!amdgpu_gmc_vram_full_visible(&adev->gmc) &&827 amdgpu_res_cpu_visible(adev, bo->tbo.resource))828 p->bytes_moved_vis += ctx.bytes_moved;829 830 if (unlikely(r == -ENOMEM) && domain != bo->allowed_domains) {831 domain = bo->allowed_domains;832 goto retry;833 }834 835 return r;836}837 838static int amdgpu_cs_parser_bos(struct amdgpu_cs_parser *p,839 union drm_amdgpu_cs *cs)840{841 struct amdgpu_fpriv *fpriv = p->filp->driver_priv;842 struct ttm_operation_ctx ctx = { true, false };843 struct amdgpu_vm *vm = &fpriv->vm;844 struct amdgpu_bo_list_entry *e;845 struct drm_gem_object *obj;846 unsigned long index;847 unsigned int i;848 int r;849 850 /* p->bo_list could already be assigned if AMDGPU_CHUNK_ID_BO_HANDLES is present */851 if (cs->in.bo_list_handle) {852 if (p->bo_list)853 return -EINVAL;854 855 r = amdgpu_bo_list_get(fpriv, cs->in.bo_list_handle,856 &p->bo_list);857 if (r)858 return r;859 } else if (!p->bo_list) {860 /* Create a empty bo_list when no handle is provided */861 r = amdgpu_bo_list_create(p->adev, p->filp, NULL, 0,862 &p->bo_list);863 if (r)864 return r;865 }866 867 mutex_lock(&p->bo_list->bo_list_mutex);868 869 /* Get userptr backing pages. If pages are updated after registered870 * in amdgpu_gem_userptr_ioctl(), amdgpu_cs_list_validate() will do871 * amdgpu_ttm_backend_bind() to flush and invalidate new pages872 */873 amdgpu_bo_list_for_each_userptr_entry(e, p->bo_list) {874 bool userpage_invalidated = false;875 struct amdgpu_bo *bo = e->bo;876 int i;877 878 e->user_pages = kvcalloc(bo->tbo.ttm->num_pages,879 sizeof(struct page *),880 GFP_KERNEL);881 if (!e->user_pages) {882 DRM_ERROR("kvmalloc_array failure\n");883 r = -ENOMEM;884 goto out_free_user_pages;885 }886 887 r = amdgpu_ttm_tt_get_user_pages(bo, e->user_pages, &e->range);888 if (r) {889 kvfree(e->user_pages);890 e->user_pages = NULL;891 goto out_free_user_pages;892 }893 894 for (i = 0; i < bo->tbo.ttm->num_pages; i++) {895 if (bo->tbo.ttm->pages[i] != e->user_pages[i]) {896 userpage_invalidated = true;897 break;898 }899 }900 e->user_invalidated = userpage_invalidated;901 }902 903 drm_exec_until_all_locked(&p->exec) {904 r = amdgpu_vm_lock_pd(&fpriv->vm, &p->exec, 1 + p->gang_size);905 drm_exec_retry_on_contention(&p->exec);906 if (unlikely(r))907 goto out_free_user_pages;908 909 amdgpu_bo_list_for_each_entry(e, p->bo_list) {910 /* One fence for TTM and one for each CS job */911 r = drm_exec_prepare_obj(&p->exec, &e->bo->tbo.base,912 1 + p->gang_size);913 drm_exec_retry_on_contention(&p->exec);914 if (unlikely(r))915 goto out_free_user_pages;916 917 e->bo_va = amdgpu_vm_bo_find(vm, e->bo);918 }919 920 if (p->uf_bo) {921 r = drm_exec_prepare_obj(&p->exec, &p->uf_bo->tbo.base,922 1 + p->gang_size);923 drm_exec_retry_on_contention(&p->exec);924 if (unlikely(r))925 goto out_free_user_pages;926 }927 }928 929 amdgpu_bo_list_for_each_userptr_entry(e, p->bo_list) {930 struct mm_struct *usermm;931 932 usermm = amdgpu_ttm_tt_get_usermm(e->bo->tbo.ttm);933 if (usermm && usermm != current->mm) {934 r = -EPERM;935 goto out_free_user_pages;936 }937 938 if (amdgpu_ttm_tt_is_userptr(e->bo->tbo.ttm) &&939 e->user_invalidated && e->user_pages) {940 amdgpu_bo_placement_from_domain(e->bo,941 AMDGPU_GEM_DOMAIN_CPU);942 r = ttm_bo_validate(&e->bo->tbo, &e->bo->placement,943 &ctx);944 if (r)945 goto out_free_user_pages;946 947 amdgpu_ttm_tt_set_user_pages(e->bo->tbo.ttm,948 e->user_pages);949 }950 951 kvfree(e->user_pages);952 e->user_pages = NULL;953 }954 955 amdgpu_cs_get_threshold_for_moves(p->adev, &p->bytes_moved_threshold,956 &p->bytes_moved_vis_threshold);957 p->bytes_moved = 0;958 p->bytes_moved_vis = 0;959 960 r = amdgpu_vm_validate(p->adev, &fpriv->vm, NULL,961 amdgpu_cs_bo_validate, p);962 if (r) {963 DRM_ERROR("amdgpu_vm_validate() failed.\n");964 goto out_free_user_pages;965 }966 967 drm_exec_for_each_locked_object(&p->exec, index, obj) {968 r = amdgpu_cs_bo_validate(p, gem_to_amdgpu_bo(obj));969 if (unlikely(r))970 goto out_free_user_pages;971 }972 973 if (p->uf_bo) {974 r = amdgpu_ttm_alloc_gart(&p->uf_bo->tbo);975 if (unlikely(r))976 goto out_free_user_pages;977 978 p->gang_leader->uf_addr += amdgpu_bo_gpu_offset(p->uf_bo);979 }980 981 amdgpu_cs_report_moved_bytes(p->adev, p->bytes_moved,982 p->bytes_moved_vis);983 984 for (i = 0; i < p->gang_size; ++i)985 amdgpu_job_set_resources(p->jobs[i], p->bo_list->gds_obj,986 p->bo_list->gws_obj,987 p->bo_list->oa_obj);988 return 0;989 990out_free_user_pages:991 amdgpu_bo_list_for_each_userptr_entry(e, p->bo_list) {992 struct amdgpu_bo *bo = e->bo;993 994 if (!e->user_pages)995 continue;996 amdgpu_ttm_tt_get_user_pages_done(bo->tbo.ttm, e->range);997 kvfree(e->user_pages);998 e->user_pages = NULL;999 e->range = NULL;1000 }1001 mutex_unlock(&p->bo_list->bo_list_mutex);1002 return r;1003}1004 1005static void trace_amdgpu_cs_ibs(struct amdgpu_cs_parser *p)1006{1007 int i, j;1008 1009 if (!trace_amdgpu_cs_enabled())1010 return;1011 1012 for (i = 0; i < p->gang_size; ++i) {1013 struct amdgpu_job *job = p->jobs[i];1014 1015 for (j = 0; j < job->num_ibs; ++j)1016 trace_amdgpu_cs(p, job, &job->ibs[j]);1017 }1018}1019 1020static int amdgpu_cs_patch_ibs(struct amdgpu_cs_parser *p,1021 struct amdgpu_job *job)1022{1023 struct amdgpu_ring *ring = amdgpu_job_ring(job);1024 unsigned int i;1025 int r;1026 1027 /* Only for UVD/VCE VM emulation */1028 if (!ring->funcs->parse_cs && !ring->funcs->patch_cs_in_place)1029 return 0;1030 1031 for (i = 0; i < job->num_ibs; ++i) {1032 struct amdgpu_ib *ib = &job->ibs[i];1033 struct amdgpu_bo_va_mapping *m;1034 struct amdgpu_bo *aobj;1035 uint64_t va_start;1036 uint8_t *kptr;1037 1038 va_start = ib->gpu_addr & AMDGPU_GMC_HOLE_MASK;1039 r = amdgpu_cs_find_mapping(p, va_start, &aobj, &m);1040 if (r) {1041 DRM_ERROR("IB va_start is invalid\n");1042 return r;1043 }1044 1045 if ((va_start + ib->length_dw * 4) >1046 (m->last + 1) * AMDGPU_GPU_PAGE_SIZE) {1047 DRM_ERROR("IB va_start+ib_bytes is invalid\n");1048 return -EINVAL;1049 }1050 1051 /* the IB should be reserved at this point */1052 r = amdgpu_bo_kmap(aobj, (void **)&kptr);1053 if (r)1054 return r;1055 1056 kptr += va_start - (m->start * AMDGPU_GPU_PAGE_SIZE);1057 1058 if (ring->funcs->parse_cs) {1059 memcpy(ib->ptr, kptr, ib->length_dw * 4);1060 amdgpu_bo_kunmap(aobj);1061 1062 r = amdgpu_ring_parse_cs(ring, p, job, ib);1063 if (r)1064 return r;1065 1066 if (ib->sa_bo)1067 ib->gpu_addr = amdgpu_sa_bo_gpu_addr(ib->sa_bo);1068 } else {1069 ib->ptr = (uint32_t *)kptr;1070 r = amdgpu_ring_patch_cs_in_place(ring, p, job, ib);1071 amdgpu_bo_kunmap(aobj);1072 if (r)1073 return r;1074 }1075 }1076 1077 return 0;1078}1079 1080static int amdgpu_cs_patch_jobs(struct amdgpu_cs_parser *p)1081{1082 unsigned int i;1083 int r;1084 1085 for (i = 0; i < p->gang_size; ++i) {1086 r = amdgpu_cs_patch_ibs(p, p->jobs[i]);1087 if (r)1088 return r;1089 }1090 return 0;1091}1092 1093static int amdgpu_cs_vm_handling(struct amdgpu_cs_parser *p)1094{1095 struct amdgpu_fpriv *fpriv = p->filp->driver_priv;1096 struct amdgpu_job *job = p->gang_leader;1097 struct amdgpu_device *adev = p->adev;1098 struct amdgpu_vm *vm = &fpriv->vm;1099 struct amdgpu_bo_list_entry *e;1100 struct amdgpu_bo_va *bo_va;1101 unsigned int i;1102 int r;1103 1104 /*1105 * We can't use gang submit on with reserved VMIDs when the VM changes1106 * can't be invalidated by more than one engine at the same time.1107 */1108 if (p->gang_size > 1 && !p->adev->vm_manager.concurrent_flush) {1109 for (i = 0; i < p->gang_size; ++i) {1110 struct drm_sched_entity *entity = p->entities[i];1111 struct drm_gpu_scheduler *sched = entity->rq->sched;1112 struct amdgpu_ring *ring = to_amdgpu_ring(sched);1113 1114 if (amdgpu_vmid_uses_reserved(adev, vm, ring->vm_hub))1115 return -EINVAL;1116 }1117 }1118 1119 r = amdgpu_vm_clear_freed(adev, vm, NULL);1120 if (r)1121 return r;1122 1123 r = amdgpu_vm_bo_update(adev, fpriv->prt_va, false);1124 if (r)1125 return r;1126 1127 r = amdgpu_sync_fence(&p->sync, fpriv->prt_va->last_pt_update);1128 if (r)1129 return r;1130 1131 if (fpriv->csa_va) {1132 bo_va = fpriv->csa_va;1133 BUG_ON(!bo_va);1134 r = amdgpu_vm_bo_update(adev, bo_va, false);1135 if (r)1136 return r;1137 1138 r = amdgpu_sync_fence(&p->sync, bo_va->last_pt_update);1139 if (r)1140 return r;1141 }1142 1143 /* FIXME: In theory this loop shouldn't be needed any more when1144 * amdgpu_vm_handle_moved handles all moved BOs that are reserved1145 * with p->ticket. But removing it caused test regressions, so I'm1146 * leaving it here for now.1147 */1148 amdgpu_bo_list_for_each_entry(e, p->bo_list) {1149 bo_va = e->bo_va;1150 if (bo_va == NULL)1151 continue;1152 1153 r = amdgpu_vm_bo_update(adev, bo_va, false);1154 if (r)1155 return r;1156 1157 r = amdgpu_sync_fence(&p->sync, bo_va->last_pt_update);1158 if (r)1159 return r;1160 }1161 1162 r = amdgpu_vm_handle_moved(adev, vm, &p->exec.ticket);1163 if (r)1164 return r;1165 1166 r = amdgpu_vm_update_pdes(adev, vm, false);1167 if (r)1168 return r;1169 1170 r = amdgpu_sync_fence(&p->sync, vm->last_update);1171 if (r)1172 return r;1173 1174 for (i = 0; i < p->gang_size; ++i) {1175 job = p->jobs[i];1176 1177 if (!job->vm)1178 continue;1179 1180 job->vm_pd_addr = amdgpu_gmc_pd_addr(vm->root.bo);1181 }1182 1183 if (adev->debug_vm) {1184 /* Invalidate all BOs to test for userspace bugs */1185 amdgpu_bo_list_for_each_entry(e, p->bo_list) {1186 struct amdgpu_bo *bo = e->bo;1187 1188 /* ignore duplicates */1189 if (!bo)1190 continue;1191 1192 amdgpu_vm_bo_invalidate(adev, bo, false);1193 }1194 }1195 1196 return 0;1197}1198 1199static int amdgpu_cs_sync_rings(struct amdgpu_cs_parser *p)1200{1201 struct amdgpu_fpriv *fpriv = p->filp->driver_priv;1202 struct drm_gpu_scheduler *sched;1203 struct drm_gem_object *obj;1204 struct dma_fence *fence;1205 unsigned long index;1206 unsigned int i;1207 int r;1208 1209 r = amdgpu_ctx_wait_prev_fence(p->ctx, p->entities[p->gang_leader_idx]);1210 if (r) {1211 if (r != -ERESTARTSYS)1212 DRM_ERROR("amdgpu_ctx_wait_prev_fence failed.\n");1213 return r;1214 }1215 1216 drm_exec_for_each_locked_object(&p->exec, index, obj) {1217 struct amdgpu_bo *bo = gem_to_amdgpu_bo(obj);1218 1219 struct dma_resv *resv = bo->tbo.base.resv;1220 enum amdgpu_sync_mode sync_mode;1221 1222 sync_mode = amdgpu_bo_explicit_sync(bo) ?1223 AMDGPU_SYNC_EXPLICIT : AMDGPU_SYNC_NE_OWNER;1224 r = amdgpu_sync_resv(p->adev, &p->sync, resv, sync_mode,1225 &fpriv->vm);1226 if (r)1227 return r;1228 }1229 1230 for (i = 0; i < p->gang_size; ++i) {1231 r = amdgpu_sync_push_to_job(&p->sync, p->jobs[i]);1232 if (r)1233 return r;1234 }1235 1236 sched = p->gang_leader->base.entity->rq->sched;1237 while ((fence = amdgpu_sync_get_fence(&p->sync))) {1238 struct drm_sched_fence *s_fence = to_drm_sched_fence(fence);1239 1240 /*1241 * When we have an dependency it might be necessary to insert a1242 * pipeline sync to make sure that all caches etc are flushed and the1243 * next job actually sees the results from the previous one1244 * before we start executing on the same scheduler ring.1245 */1246 if (!s_fence || s_fence->sched != sched) {1247 dma_fence_put(fence);1248 continue;1249 }1250 1251 r = amdgpu_sync_fence(&p->gang_leader->explicit_sync, fence);1252 dma_fence_put(fence);1253 if (r)1254 return r;1255 }1256 return 0;1257}1258 1259static void amdgpu_cs_post_dependencies(struct amdgpu_cs_parser *p)1260{1261 int i;1262 1263 for (i = 0; i < p->num_post_deps; ++i) {1264 if (p->post_deps[i].chain && p->post_deps[i].point) {1265 drm_syncobj_add_point(p->post_deps[i].syncobj,1266 p->post_deps[i].chain,1267 p->fence, p->post_deps[i].point);1268 p->post_deps[i].chain = NULL;1269 } else {1270 drm_syncobj_replace_fence(p->post_deps[i].syncobj,1271 p->fence);1272 }1273 }1274}1275 1276static int amdgpu_cs_submit(struct amdgpu_cs_parser *p,1277 union drm_amdgpu_cs *cs)1278{1279 struct amdgpu_fpriv *fpriv = p->filp->driver_priv;1280 struct amdgpu_job *leader = p->gang_leader;1281 struct amdgpu_bo_list_entry *e;1282 struct drm_gem_object *gobj;1283 unsigned long index;1284 unsigned int i;1285 uint64_t seq;1286 int r;1287 1288 for (i = 0; i < p->gang_size; ++i)1289 drm_sched_job_arm(&p->jobs[i]->base);1290 1291 for (i = 0; i < p->gang_size; ++i) {1292 struct dma_fence *fence;1293 1294 if (p->jobs[i] == leader)1295 continue;1296 1297 fence = &p->jobs[i]->base.s_fence->scheduled;1298 dma_fence_get(fence);1299 r = drm_sched_job_add_dependency(&leader->base, fence);1300 if (r) {1301 dma_fence_put(fence);1302 return r;1303 }1304 }1305 1306 if (p->gang_size > 1) {1307 for (i = 0; i < p->gang_size; ++i)1308 amdgpu_job_set_gang_leader(p->jobs[i], leader);1309 }1310 1311 /* No memory allocation is allowed while holding the notifier lock.1312 * The lock is held until amdgpu_cs_submit is finished and fence is1313 * added to BOs.1314 */1315 mutex_lock(&p->adev->notifier_lock);1316 1317 /* If userptr are invalidated after amdgpu_cs_parser_bos(), return1318 * -EAGAIN, drmIoctl in libdrm will restart the amdgpu_cs_ioctl.1319 */1320 r = 0;1321 amdgpu_bo_list_for_each_userptr_entry(e, p->bo_list) {1322 r |= !amdgpu_ttm_tt_get_user_pages_done(e->bo->tbo.ttm,1323 e->range);1324 e->range = NULL;1325 }1326 if (r) {1327 r = -EAGAIN;1328 mutex_unlock(&p->adev->notifier_lock);1329 return r;1330 }1331 1332 p->fence = dma_fence_get(&leader->base.s_fence->finished);1333 drm_exec_for_each_locked_object(&p->exec, index, gobj) {1334 1335 ttm_bo_move_to_lru_tail_unlocked(&gem_to_amdgpu_bo(gobj)->tbo);1336 1337 /* Everybody except for the gang leader uses READ */1338 for (i = 0; i < p->gang_size; ++i) {1339 if (p->jobs[i] == leader)1340 continue;1341 1342 dma_resv_add_fence(gobj->resv,1343 &p->jobs[i]->base.s_fence->finished,1344 DMA_RESV_USAGE_READ);1345 }1346 1347 /* The gang leader as remembered as writer */1348 dma_resv_add_fence(gobj->resv, p->fence, DMA_RESV_USAGE_WRITE);1349 }1350 1351 seq = amdgpu_ctx_add_fence(p->ctx, p->entities[p->gang_leader_idx],1352 p->fence);1353 amdgpu_cs_post_dependencies(p);1354 1355 if ((leader->preamble_status & AMDGPU_PREAMBLE_IB_PRESENT) &&1356 !p->ctx->preamble_presented) {1357 leader->preamble_status |= AMDGPU_PREAMBLE_IB_PRESENT_FIRST;1358 p->ctx->preamble_presented = true;1359 }1360 1361 cs->out.handle = seq;1362 leader->uf_sequence = seq;1363 1364 amdgpu_vm_bo_trace_cs(&fpriv->vm, &p->exec.ticket);1365 for (i = 0; i < p->gang_size; ++i) {1366 amdgpu_job_free_resources(p->jobs[i]);1367 trace_amdgpu_cs_ioctl(p->jobs[i]);1368 drm_sched_entity_push_job(&p->jobs[i]->base);1369 p->jobs[i] = NULL;1370 }1371 1372 amdgpu_vm_move_to_lru_tail(p->adev, &fpriv->vm);1373 1374 mutex_unlock(&p->adev->notifier_lock);1375 mutex_unlock(&p->bo_list->bo_list_mutex);1376 return 0;1377}1378 1379/* Cleanup the parser structure */1380static void amdgpu_cs_parser_fini(struct amdgpu_cs_parser *parser)1381{1382 unsigned int i;1383 1384 amdgpu_sync_free(&parser->sync);1385 drm_exec_fini(&parser->exec);1386 1387 for (i = 0; i < parser->num_post_deps; i++) {1388 drm_syncobj_put(parser->post_deps[i].syncobj);1389 kfree(parser->post_deps[i].chain);1390 }1391 kfree(parser->post_deps);1392 1393 dma_fence_put(parser->fence);1394 1395 if (parser->ctx)1396 amdgpu_ctx_put(parser->ctx);1397 if (parser->bo_list)1398 amdgpu_bo_list_put(parser->bo_list);1399 1400 for (i = 0; i < parser->nchunks; i++)1401 kvfree(parser->chunks[i].kdata);1402 kvfree(parser->chunks);1403 for (i = 0; i < parser->gang_size; ++i) {1404 if (parser->jobs[i])1405 amdgpu_job_free(parser->jobs[i]);1406 }1407 amdgpu_bo_unref(&parser->uf_bo);1408}1409 1410int amdgpu_cs_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)1411{1412 struct amdgpu_device *adev = drm_to_adev(dev);1413 struct amdgpu_cs_parser parser;1414 int r;1415 1416 if (amdgpu_ras_intr_triggered())1417 return -EHWPOISON;1418 1419 if (!adev->accel_working)1420 return -EBUSY;1421 1422 r = amdgpu_cs_parser_init(&parser, adev, filp, data);1423 if (r) {1424 DRM_ERROR_RATELIMITED("Failed to initialize parser %d!\n", r);1425 return r;1426 }1427 1428 r = amdgpu_cs_pass1(&parser, data);1429 if (r)1430 goto error_fini;1431 1432 r = amdgpu_cs_pass2(&parser);1433 if (r)1434 goto error_fini;1435 1436 r = amdgpu_cs_parser_bos(&parser, data);1437 if (r) {1438 if (r == -ENOMEM)1439 DRM_ERROR("Not enough memory for command submission!\n");1440 else if (r != -ERESTARTSYS && r != -EAGAIN)1441 DRM_DEBUG("Failed to process the buffer list %d!\n", r);1442 goto error_fini;1443 }1444 1445 r = amdgpu_cs_patch_jobs(&parser);1446 if (r)1447 goto error_backoff;1448 1449 r = amdgpu_cs_vm_handling(&parser);1450 if (r)1451 goto error_backoff;1452 1453 r = amdgpu_cs_sync_rings(&parser);1454 if (r)1455 goto error_backoff;1456 1457 trace_amdgpu_cs_ibs(&parser);1458 1459 r = amdgpu_cs_submit(&parser, data);1460 if (r)1461 goto error_backoff;1462 1463 amdgpu_cs_parser_fini(&parser);1464 return 0;1465 1466error_backoff:1467 mutex_unlock(&parser.bo_list->bo_list_mutex);1468 1469error_fini:1470 amdgpu_cs_parser_fini(&parser);1471 return r;1472}1473 1474/**1475 * amdgpu_cs_wait_ioctl - wait for a command submission to finish1476 *1477 * @dev: drm device1478 * @data: data from userspace1479 * @filp: file private1480 *1481 * Wait for the command submission identified by handle to finish.1482 */1483int amdgpu_cs_wait_ioctl(struct drm_device *dev, void *data,1484 struct drm_file *filp)1485{1486 union drm_amdgpu_wait_cs *wait = data;1487 unsigned long timeout = amdgpu_gem_timeout(wait->in.timeout);1488 struct drm_sched_entity *entity;1489 struct amdgpu_ctx *ctx;1490 struct dma_fence *fence;1491 long r;1492 1493 ctx = amdgpu_ctx_get(filp->driver_priv, wait->in.ctx_id);1494 if (ctx == NULL)1495 return -EINVAL;1496 1497 r = amdgpu_ctx_get_entity(ctx, wait->in.ip_type, wait->in.ip_instance,1498 wait->in.ring, &entity);1499 if (r) {1500 amdgpu_ctx_put(ctx);1501 return r;1502 }1503 1504 fence = amdgpu_ctx_get_fence(ctx, entity, wait->in.handle);1505 if (IS_ERR(fence))1506 r = PTR_ERR(fence);1507 else if (fence) {1508 r = dma_fence_wait_timeout(fence, true, timeout);1509 if (r > 0 && fence->error)1510 r = fence->error;1511 dma_fence_put(fence);1512 } else1513 r = 1;1514 1515 amdgpu_ctx_put(ctx);1516 if (r < 0)1517 return r;1518 1519 memset(wait, 0, sizeof(*wait));1520 wait->out.status = (r == 0);1521 1522 return 0;1523}1524 1525/**1526 * amdgpu_cs_get_fence - helper to get fence from drm_amdgpu_fence1527 *1528 * @adev: amdgpu device1529 * @filp: file private1530 * @user: drm_amdgpu_fence copied from user space1531 */1532static struct dma_fence *amdgpu_cs_get_fence(struct amdgpu_device *adev,1533 struct drm_file *filp,1534 struct drm_amdgpu_fence *user)1535{1536 struct drm_sched_entity *entity;1537 struct amdgpu_ctx *ctx;1538 struct dma_fence *fence;1539 int r;1540 1541 ctx = amdgpu_ctx_get(filp->driver_priv, user->ctx_id);1542 if (ctx == NULL)1543 return ERR_PTR(-EINVAL);1544 1545 r = amdgpu_ctx_get_entity(ctx, user->ip_type, user->ip_instance,1546 user->ring, &entity);1547 if (r) {1548 amdgpu_ctx_put(ctx);1549 return ERR_PTR(r);1550 }1551 1552 fence = amdgpu_ctx_get_fence(ctx, entity, user->seq_no);1553 amdgpu_ctx_put(ctx);1554 1555 return fence;1556}1557 1558int amdgpu_cs_fence_to_handle_ioctl(struct drm_device *dev, void *data,1559 struct drm_file *filp)1560{1561 struct amdgpu_device *adev = drm_to_adev(dev);1562 union drm_amdgpu_fence_to_handle *info = data;1563 struct dma_fence *fence;1564 struct drm_syncobj *syncobj;1565 struct sync_file *sync_file;1566 int fd, r;1567 1568 fence = amdgpu_cs_get_fence(adev, filp, &info->in.fence);1569 if (IS_ERR(fence))1570 return PTR_ERR(fence);1571 1572 if (!fence)1573 fence = dma_fence_get_stub();1574 1575 switch (info->in.what) {1576 case AMDGPU_FENCE_TO_HANDLE_GET_SYNCOBJ:1577 r = drm_syncobj_create(&syncobj, 0, fence);1578 dma_fence_put(fence);1579 if (r)1580 return r;1581 r = drm_syncobj_get_handle(filp, syncobj, &info->out.handle);1582 drm_syncobj_put(syncobj);1583 return r;1584 1585 case AMDGPU_FENCE_TO_HANDLE_GET_SYNCOBJ_FD:1586 r = drm_syncobj_create(&syncobj, 0, fence);1587 dma_fence_put(fence);1588 if (r)1589 return r;1590 r = drm_syncobj_get_fd(syncobj, (int *)&info->out.handle);1591 drm_syncobj_put(syncobj);1592 return r;1593 1594 case AMDGPU_FENCE_TO_HANDLE_GET_SYNC_FILE_FD:1595 fd = get_unused_fd_flags(O_CLOEXEC);1596 if (fd < 0) {1597 dma_fence_put(fence);1598 return fd;1599 }1600 1601 sync_file = sync_file_create(fence);1602 dma_fence_put(fence);1603 if (!sync_file) {1604 put_unused_fd(fd);1605 return -ENOMEM;1606 }1607 1608 fd_install(fd, sync_file->file);1609 info->out.handle = fd;1610 return 0;1611 1612 default:1613 dma_fence_put(fence);1614 return -EINVAL;1615 }1616}1617 1618/**1619 * amdgpu_cs_wait_all_fences - wait on all fences to signal1620 *1621 * @adev: amdgpu device1622 * @filp: file private1623 * @wait: wait parameters1624 * @fences: array of drm_amdgpu_fence1625 */1626static int amdgpu_cs_wait_all_fences(struct amdgpu_device *adev,1627 struct drm_file *filp,1628 union drm_amdgpu_wait_fences *wait,1629 struct drm_amdgpu_fence *fences)1630{1631 uint32_t fence_count = wait->in.fence_count;1632 unsigned int i;1633 long r = 1;1634 1635 for (i = 0; i < fence_count; i++) {1636 struct dma_fence *fence;1637 unsigned long timeout = amdgpu_gem_timeout(wait->in.timeout_ns);1638 1639 fence = amdgpu_cs_get_fence(adev, filp, &fences[i]);1640 if (IS_ERR(fence))1641 return PTR_ERR(fence);1642 else if (!fence)1643 continue;1644 1645 r = dma_fence_wait_timeout(fence, true, timeout);1646 if (r > 0 && fence->error)1647 r = fence->error;1648 1649 dma_fence_put(fence);1650 if (r < 0)1651 return r;1652 1653 if (r == 0)1654 break;1655 }1656 1657 memset(wait, 0, sizeof(*wait));1658 wait->out.status = (r > 0);1659 1660 return 0;1661}1662 1663/**1664 * amdgpu_cs_wait_any_fence - wait on any fence to signal1665 *1666 * @adev: amdgpu device1667 * @filp: file private1668 * @wait: wait parameters1669 * @fences: array of drm_amdgpu_fence1670 */1671static int amdgpu_cs_wait_any_fence(struct amdgpu_device *adev,1672 struct drm_file *filp,1673 union drm_amdgpu_wait_fences *wait,1674 struct drm_amdgpu_fence *fences)1675{1676 unsigned long timeout = amdgpu_gem_timeout(wait->in.timeout_ns);1677 uint32_t fence_count = wait->in.fence_count;1678 uint32_t first = ~0;1679 struct dma_fence **array;1680 unsigned int i;1681 long r;1682 1683 /* Prepare the fence array */1684 array = kcalloc(fence_count, sizeof(struct dma_fence *), GFP_KERNEL);1685 1686 if (array == NULL)1687 return -ENOMEM;1688 1689 for (i = 0; i < fence_count; i++) {1690 struct dma_fence *fence;1691 1692 fence = amdgpu_cs_get_fence(adev, filp, &fences[i]);1693 if (IS_ERR(fence)) {1694 r = PTR_ERR(fence);1695 goto err_free_fence_array;1696 } else if (fence) {1697 array[i] = fence;1698 } else { /* NULL, the fence has been already signaled */1699 r = 1;1700 first = i;1701 goto out;1702 }1703 }1704 1705 r = dma_fence_wait_any_timeout(array, fence_count, true, timeout,1706 &first);1707 if (r < 0)1708 goto err_free_fence_array;1709 1710out:1711 memset(wait, 0, sizeof(*wait));1712 wait->out.status = (r > 0);1713 wait->out.first_signaled = first;1714 1715 if (first < fence_count && array[first])1716 r = array[first]->error;1717 else1718 r = 0;1719 1720err_free_fence_array:1721 for (i = 0; i < fence_count; i++)1722 dma_fence_put(array[i]);1723 kfree(array);1724 1725 return r;1726}1727 1728/**1729 * amdgpu_cs_wait_fences_ioctl - wait for multiple command submissions to finish1730 *1731 * @dev: drm device1732 * @data: data from userspace1733 * @filp: file private1734 */1735int amdgpu_cs_wait_fences_ioctl(struct drm_device *dev, void *data,1736 struct drm_file *filp)1737{1738 struct amdgpu_device *adev = drm_to_adev(dev);1739 union drm_amdgpu_wait_fences *wait = data;1740 uint32_t fence_count = wait->in.fence_count;1741 struct drm_amdgpu_fence *fences_user;1742 struct drm_amdgpu_fence *fences;1743 int r;1744 1745 /* Get the fences from userspace */1746 fences = kmalloc_array(fence_count, sizeof(struct drm_amdgpu_fence),1747 GFP_KERNEL);1748 if (fences == NULL)1749 return -ENOMEM;1750 1751 fences_user = u64_to_user_ptr(wait->in.fences);1752 if (copy_from_user(fences, fences_user,1753 sizeof(struct drm_amdgpu_fence) * fence_count)) {1754 r = -EFAULT;1755 goto err_free_fences;1756 }1757 1758 if (wait->in.wait_all)1759 r = amdgpu_cs_wait_all_fences(adev, filp, wait, fences);1760 else1761 r = amdgpu_cs_wait_any_fence(adev, filp, wait, fences);1762 1763err_free_fences:1764 kfree(fences);1765 1766 return r;1767}1768 1769/**1770 * amdgpu_cs_find_mapping - find bo_va for VM address1771 *1772 * @parser: command submission parser context1773 * @addr: VM address1774 * @bo: resulting BO of the mapping found1775 * @map: Placeholder to return found BO mapping1776 *1777 * Search the buffer objects in the command submission context for a certain1778 * virtual memory address. Returns allocation structure when found, NULL1779 * otherwise.1780 */1781int amdgpu_cs_find_mapping(struct amdgpu_cs_parser *parser,1782 uint64_t addr, struct amdgpu_bo **bo,1783 struct amdgpu_bo_va_mapping **map)1784{1785 struct amdgpu_fpriv *fpriv = parser->filp->driver_priv;1786 struct ttm_operation_ctx ctx = { false, false };1787 struct amdgpu_vm *vm = &fpriv->vm;1788 struct amdgpu_bo_va_mapping *mapping;1789 int i, r;1790 1791 addr /= AMDGPU_GPU_PAGE_SIZE;1792 1793 mapping = amdgpu_vm_bo_lookup_mapping(vm, addr);1794 if (!mapping || !mapping->bo_va || !mapping->bo_va->base.bo)1795 return -EINVAL;1796 1797 *bo = mapping->bo_va->base.bo;1798 *map = mapping;1799 1800 /* Double check that the BO is reserved by this CS */1801 if (dma_resv_locking_ctx((*bo)->tbo.base.resv) != &parser->exec.ticket)1802 return -EINVAL;1803 1804 (*bo)->flags |= AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS;1805 amdgpu_bo_placement_from_domain(*bo, (*bo)->allowed_domains);1806 for (i = 0; i < (*bo)->placement.num_placement; i++)1807 (*bo)->placements[i].flags |= TTM_PL_FLAG_CONTIGUOUS;1808 r = ttm_bo_validate(&(*bo)->tbo, &(*bo)->placement, &ctx);1809 if (r)1810 return r;1811 1812 return amdgpu_ttm_alloc_gart(&(*bo)->tbo);1813}1814