1401 lines · c
1/*2 * Copyright © 2014 Broadcom3 *4 * Permission is hereby granted, free of charge, to any person obtaining a5 * copy of this software and associated documentation files (the "Software"),6 * to deal in the Software without restriction, including without limitation7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,8 * and/or sell copies of the Software, and to permit persons to whom the9 * Software is furnished to do so, subject to the following conditions:10 *11 * The above copyright notice and this permission notice (including the next12 * paragraph) shall be included in all copies or substantial portions of the13 * Software.14 *15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS21 * IN THE SOFTWARE.22 */23 24#include <linux/module.h>25#include <linux/platform_device.h>26#include <linux/pm_runtime.h>27#include <linux/device.h>28#include <linux/io.h>29#include <linux/sched/signal.h>30#include <linux/dma-fence-array.h>31 32#include <drm/drm_syncobj.h>33 34#include "uapi/drm/vc4_drm.h"35#include "vc4_drv.h"36#include "vc4_regs.h"37#include "vc4_trace.h"38 39static void40vc4_queue_hangcheck(struct drm_device *dev)41{42 struct vc4_dev *vc4 = to_vc4_dev(dev);43 44 mod_timer(&vc4->hangcheck.timer,45 round_jiffies_up(jiffies + msecs_to_jiffies(100)));46}47 48struct vc4_hang_state {49 struct drm_vc4_get_hang_state user_state;50 51 u32 bo_count;52 struct drm_gem_object **bo;53};54 55static void56vc4_free_hang_state(struct drm_device *dev, struct vc4_hang_state *state)57{58 unsigned int i;59 60 for (i = 0; i < state->user_state.bo_count; i++)61 drm_gem_object_put(state->bo[i]);62 63 kfree(state);64}65 66int67vc4_get_hang_state_ioctl(struct drm_device *dev, void *data,68 struct drm_file *file_priv)69{70 struct drm_vc4_get_hang_state *get_state = data;71 struct drm_vc4_get_hang_state_bo *bo_state;72 struct vc4_hang_state *kernel_state;73 struct drm_vc4_get_hang_state *state;74 struct vc4_dev *vc4 = to_vc4_dev(dev);75 unsigned long irqflags;76 u32 i;77 int ret = 0;78 79 if (WARN_ON_ONCE(vc4->is_vc5))80 return -ENODEV;81 82 if (!vc4->v3d) {83 DRM_DEBUG("VC4_GET_HANG_STATE with no VC4 V3D probed\n");84 return -ENODEV;85 }86 87 spin_lock_irqsave(&vc4->job_lock, irqflags);88 kernel_state = vc4->hang_state;89 if (!kernel_state) {90 spin_unlock_irqrestore(&vc4->job_lock, irqflags);91 return -ENOENT;92 }93 state = &kernel_state->user_state;94 95 /* If the user's array isn't big enough, just return the96 * required array size.97 */98 if (get_state->bo_count < state->bo_count) {99 get_state->bo_count = state->bo_count;100 spin_unlock_irqrestore(&vc4->job_lock, irqflags);101 return 0;102 }103 104 vc4->hang_state = NULL;105 spin_unlock_irqrestore(&vc4->job_lock, irqflags);106 107 /* Save the user's BO pointer, so we don't stomp it with the memcpy. */108 state->bo = get_state->bo;109 memcpy(get_state, state, sizeof(*state));110 111 bo_state = kcalloc(state->bo_count, sizeof(*bo_state), GFP_KERNEL);112 if (!bo_state) {113 ret = -ENOMEM;114 goto err_free;115 }116 117 for (i = 0; i < state->bo_count; i++) {118 struct vc4_bo *vc4_bo = to_vc4_bo(kernel_state->bo[i]);119 u32 handle;120 121 ret = drm_gem_handle_create(file_priv, kernel_state->bo[i],122 &handle);123 124 if (ret) {125 state->bo_count = i;126 goto err_delete_handle;127 }128 bo_state[i].handle = handle;129 bo_state[i].paddr = vc4_bo->base.dma_addr;130 bo_state[i].size = vc4_bo->base.base.size;131 }132 133 if (copy_to_user(u64_to_user_ptr(get_state->bo),134 bo_state,135 state->bo_count * sizeof(*bo_state)))136 ret = -EFAULT;137 138err_delete_handle:139 if (ret) {140 for (i = 0; i < state->bo_count; i++)141 drm_gem_handle_delete(file_priv, bo_state[i].handle);142 }143 144err_free:145 vc4_free_hang_state(dev, kernel_state);146 kfree(bo_state);147 148 return ret;149}150 151static void152vc4_save_hang_state(struct drm_device *dev)153{154 struct vc4_dev *vc4 = to_vc4_dev(dev);155 struct drm_vc4_get_hang_state *state;156 struct vc4_hang_state *kernel_state;157 struct vc4_exec_info *exec[2];158 struct vc4_bo *bo;159 unsigned long irqflags;160 unsigned int i, j, k, unref_list_count;161 162 kernel_state = kcalloc(1, sizeof(*kernel_state), GFP_KERNEL);163 if (!kernel_state)164 return;165 166 state = &kernel_state->user_state;167 168 spin_lock_irqsave(&vc4->job_lock, irqflags);169 exec[0] = vc4_first_bin_job(vc4);170 exec[1] = vc4_first_render_job(vc4);171 if (!exec[0] && !exec[1]) {172 spin_unlock_irqrestore(&vc4->job_lock, irqflags);173 return;174 }175 176 /* Get the bos from both binner and renderer into hang state. */177 state->bo_count = 0;178 for (i = 0; i < 2; i++) {179 if (!exec[i])180 continue;181 182 unref_list_count = 0;183 list_for_each_entry(bo, &exec[i]->unref_list, unref_head)184 unref_list_count++;185 state->bo_count += exec[i]->bo_count + unref_list_count;186 }187 188 kernel_state->bo = kcalloc(state->bo_count,189 sizeof(*kernel_state->bo), GFP_ATOMIC);190 191 if (!kernel_state->bo) {192 spin_unlock_irqrestore(&vc4->job_lock, irqflags);193 return;194 }195 196 k = 0;197 for (i = 0; i < 2; i++) {198 if (!exec[i])199 continue;200 201 for (j = 0; j < exec[i]->bo_count; j++) {202 bo = to_vc4_bo(exec[i]->bo[j]);203 204 /* Retain BOs just in case they were marked purgeable.205 * This prevents the BO from being purged before206 * someone had a chance to dump the hang state.207 */208 WARN_ON(!refcount_read(&bo->usecnt));209 refcount_inc(&bo->usecnt);210 drm_gem_object_get(exec[i]->bo[j]);211 kernel_state->bo[k++] = exec[i]->bo[j];212 }213 214 list_for_each_entry(bo, &exec[i]->unref_list, unref_head) {215 /* No need to retain BOs coming from the ->unref_list216 * because they are naturally unpurgeable.217 */218 drm_gem_object_get(&bo->base.base);219 kernel_state->bo[k++] = &bo->base.base;220 }221 }222 223 WARN_ON_ONCE(k != state->bo_count);224 225 if (exec[0])226 state->start_bin = exec[0]->ct0ca;227 if (exec[1])228 state->start_render = exec[1]->ct1ca;229 230 spin_unlock_irqrestore(&vc4->job_lock, irqflags);231 232 state->ct0ca = V3D_READ(V3D_CTNCA(0));233 state->ct0ea = V3D_READ(V3D_CTNEA(0));234 235 state->ct1ca = V3D_READ(V3D_CTNCA(1));236 state->ct1ea = V3D_READ(V3D_CTNEA(1));237 238 state->ct0cs = V3D_READ(V3D_CTNCS(0));239 state->ct1cs = V3D_READ(V3D_CTNCS(1));240 241 state->ct0ra0 = V3D_READ(V3D_CT00RA0);242 state->ct1ra0 = V3D_READ(V3D_CT01RA0);243 244 state->bpca = V3D_READ(V3D_BPCA);245 state->bpcs = V3D_READ(V3D_BPCS);246 state->bpoa = V3D_READ(V3D_BPOA);247 state->bpos = V3D_READ(V3D_BPOS);248 249 state->vpmbase = V3D_READ(V3D_VPMBASE);250 251 state->dbge = V3D_READ(V3D_DBGE);252 state->fdbgo = V3D_READ(V3D_FDBGO);253 state->fdbgb = V3D_READ(V3D_FDBGB);254 state->fdbgr = V3D_READ(V3D_FDBGR);255 state->fdbgs = V3D_READ(V3D_FDBGS);256 state->errstat = V3D_READ(V3D_ERRSTAT);257 258 /* We need to turn purgeable BOs into unpurgeable ones so that259 * userspace has a chance to dump the hang state before the kernel260 * decides to purge those BOs.261 * Note that BO consistency at dump time cannot be guaranteed. For262 * example, if the owner of these BOs decides to re-use them or mark263 * them purgeable again there's nothing we can do to prevent it.264 */265 for (i = 0; i < kernel_state->user_state.bo_count; i++) {266 struct vc4_bo *bo = to_vc4_bo(kernel_state->bo[i]);267 268 if (bo->madv == __VC4_MADV_NOTSUPP)269 continue;270 271 mutex_lock(&bo->madv_lock);272 if (!WARN_ON(bo->madv == __VC4_MADV_PURGED))273 bo->madv = VC4_MADV_WILLNEED;274 refcount_dec(&bo->usecnt);275 mutex_unlock(&bo->madv_lock);276 }277 278 spin_lock_irqsave(&vc4->job_lock, irqflags);279 if (vc4->hang_state) {280 spin_unlock_irqrestore(&vc4->job_lock, irqflags);281 vc4_free_hang_state(dev, kernel_state);282 } else {283 vc4->hang_state = kernel_state;284 spin_unlock_irqrestore(&vc4->job_lock, irqflags);285 }286}287 288static void289vc4_reset(struct drm_device *dev)290{291 struct vc4_dev *vc4 = to_vc4_dev(dev);292 293 DRM_INFO("Resetting GPU.\n");294 295 mutex_lock(&vc4->power_lock);296 if (vc4->power_refcount) {297 /* Power the device off and back on the by dropping the298 * reference on runtime PM.299 */300 pm_runtime_put_sync_suspend(&vc4->v3d->pdev->dev);301 pm_runtime_get_sync(&vc4->v3d->pdev->dev);302 }303 mutex_unlock(&vc4->power_lock);304 305 vc4_irq_reset(dev);306 307 /* Rearm the hangcheck -- another job might have been waiting308 * for our hung one to get kicked off, and vc4_irq_reset()309 * would have started it.310 */311 vc4_queue_hangcheck(dev);312}313 314static void315vc4_reset_work(struct work_struct *work)316{317 struct vc4_dev *vc4 =318 container_of(work, struct vc4_dev, hangcheck.reset_work);319 320 vc4_save_hang_state(&vc4->base);321 322 vc4_reset(&vc4->base);323}324 325static void326vc4_hangcheck_elapsed(struct timer_list *t)327{328 struct vc4_dev *vc4 = from_timer(vc4, t, hangcheck.timer);329 struct drm_device *dev = &vc4->base;330 uint32_t ct0ca, ct1ca;331 unsigned long irqflags;332 struct vc4_exec_info *bin_exec, *render_exec;333 334 spin_lock_irqsave(&vc4->job_lock, irqflags);335 336 bin_exec = vc4_first_bin_job(vc4);337 render_exec = vc4_first_render_job(vc4);338 339 /* If idle, we can stop watching for hangs. */340 if (!bin_exec && !render_exec) {341 spin_unlock_irqrestore(&vc4->job_lock, irqflags);342 return;343 }344 345 ct0ca = V3D_READ(V3D_CTNCA(0));346 ct1ca = V3D_READ(V3D_CTNCA(1));347 348 /* If we've made any progress in execution, rearm the timer349 * and wait.350 */351 if ((bin_exec && ct0ca != bin_exec->last_ct0ca) ||352 (render_exec && ct1ca != render_exec->last_ct1ca)) {353 if (bin_exec)354 bin_exec->last_ct0ca = ct0ca;355 if (render_exec)356 render_exec->last_ct1ca = ct1ca;357 spin_unlock_irqrestore(&vc4->job_lock, irqflags);358 vc4_queue_hangcheck(dev);359 return;360 }361 362 spin_unlock_irqrestore(&vc4->job_lock, irqflags);363 364 /* We've gone too long with no progress, reset. This has to365 * be done from a work struct, since resetting can sleep and366 * this timer hook isn't allowed to.367 */368 schedule_work(&vc4->hangcheck.reset_work);369}370 371static void372submit_cl(struct drm_device *dev, uint32_t thread, uint32_t start, uint32_t end)373{374 struct vc4_dev *vc4 = to_vc4_dev(dev);375 376 /* Set the current and end address of the control list.377 * Writing the end register is what starts the job.378 */379 V3D_WRITE(V3D_CTNCA(thread), start);380 V3D_WRITE(V3D_CTNEA(thread), end);381}382 383int384vc4_wait_for_seqno(struct drm_device *dev, uint64_t seqno, uint64_t timeout_ns,385 bool interruptible)386{387 struct vc4_dev *vc4 = to_vc4_dev(dev);388 int ret = 0;389 unsigned long timeout_expire;390 DEFINE_WAIT(wait);391 392 if (WARN_ON_ONCE(vc4->is_vc5))393 return -ENODEV;394 395 if (vc4->finished_seqno >= seqno)396 return 0;397 398 if (timeout_ns == 0)399 return -ETIME;400 401 timeout_expire = jiffies + nsecs_to_jiffies(timeout_ns);402 403 trace_vc4_wait_for_seqno_begin(dev, seqno, timeout_ns);404 for (;;) {405 prepare_to_wait(&vc4->job_wait_queue, &wait,406 interruptible ? TASK_INTERRUPTIBLE :407 TASK_UNINTERRUPTIBLE);408 409 if (interruptible && signal_pending(current)) {410 ret = -ERESTARTSYS;411 break;412 }413 414 if (vc4->finished_seqno >= seqno)415 break;416 417 if (timeout_ns != ~0ull) {418 if (time_after_eq(jiffies, timeout_expire)) {419 ret = -ETIME;420 break;421 }422 schedule_timeout(timeout_expire - jiffies);423 } else {424 schedule();425 }426 }427 428 finish_wait(&vc4->job_wait_queue, &wait);429 trace_vc4_wait_for_seqno_end(dev, seqno);430 431 return ret;432}433 434static void435vc4_flush_caches(struct drm_device *dev)436{437 struct vc4_dev *vc4 = to_vc4_dev(dev);438 439 /* Flush the GPU L2 caches. These caches sit on top of system440 * L3 (the 128kb or so shared with the CPU), and are441 * non-allocating in the L3.442 */443 V3D_WRITE(V3D_L2CACTL,444 V3D_L2CACTL_L2CCLR);445 446 V3D_WRITE(V3D_SLCACTL,447 VC4_SET_FIELD(0xf, V3D_SLCACTL_T1CC) |448 VC4_SET_FIELD(0xf, V3D_SLCACTL_T0CC) |449 VC4_SET_FIELD(0xf, V3D_SLCACTL_UCC) |450 VC4_SET_FIELD(0xf, V3D_SLCACTL_ICC));451}452 453static void454vc4_flush_texture_caches(struct drm_device *dev)455{456 struct vc4_dev *vc4 = to_vc4_dev(dev);457 458 V3D_WRITE(V3D_L2CACTL,459 V3D_L2CACTL_L2CCLR);460 461 V3D_WRITE(V3D_SLCACTL,462 VC4_SET_FIELD(0xf, V3D_SLCACTL_T1CC) |463 VC4_SET_FIELD(0xf, V3D_SLCACTL_T0CC));464}465 466/* Sets the registers for the next job to be actually be executed in467 * the hardware.468 *469 * The job_lock should be held during this.470 */471void472vc4_submit_next_bin_job(struct drm_device *dev)473{474 struct vc4_dev *vc4 = to_vc4_dev(dev);475 struct vc4_exec_info *exec;476 477 if (WARN_ON_ONCE(vc4->is_vc5))478 return;479 480again:481 exec = vc4_first_bin_job(vc4);482 if (!exec)483 return;484 485 vc4_flush_caches(dev);486 487 /* Only start the perfmon if it was not already started by a previous488 * job.489 */490 if (exec->perfmon && vc4->active_perfmon != exec->perfmon)491 vc4_perfmon_start(vc4, exec->perfmon);492 493 /* Either put the job in the binner if it uses the binner, or494 * immediately move it to the to-be-rendered queue.495 */496 if (exec->ct0ca != exec->ct0ea) {497 trace_vc4_submit_cl(dev, false, exec->seqno, exec->ct0ca,498 exec->ct0ea);499 submit_cl(dev, 0, exec->ct0ca, exec->ct0ea);500 } else {501 struct vc4_exec_info *next;502 503 vc4_move_job_to_render(dev, exec);504 next = vc4_first_bin_job(vc4);505 506 /* We can't start the next bin job if the previous job had a507 * different perfmon instance attached to it. The same goes508 * if one of them had a perfmon attached to it and the other509 * one doesn't.510 */511 if (next && next->perfmon == exec->perfmon)512 goto again;513 }514}515 516void517vc4_submit_next_render_job(struct drm_device *dev)518{519 struct vc4_dev *vc4 = to_vc4_dev(dev);520 struct vc4_exec_info *exec = vc4_first_render_job(vc4);521 522 if (!exec)523 return;524 525 if (WARN_ON_ONCE(vc4->is_vc5))526 return;527 528 /* A previous RCL may have written to one of our textures, and529 * our full cache flush at bin time may have occurred before530 * that RCL completed. Flush the texture cache now, but not531 * the instructions or uniforms (since we don't write those532 * from an RCL).533 */534 vc4_flush_texture_caches(dev);535 536 trace_vc4_submit_cl(dev, true, exec->seqno, exec->ct1ca, exec->ct1ea);537 submit_cl(dev, 1, exec->ct1ca, exec->ct1ea);538}539 540void541vc4_move_job_to_render(struct drm_device *dev, struct vc4_exec_info *exec)542{543 struct vc4_dev *vc4 = to_vc4_dev(dev);544 bool was_empty = list_empty(&vc4->render_job_list);545 546 if (WARN_ON_ONCE(vc4->is_vc5))547 return;548 549 list_move_tail(&exec->head, &vc4->render_job_list);550 if (was_empty)551 vc4_submit_next_render_job(dev);552}553 554static void555vc4_update_bo_seqnos(struct vc4_exec_info *exec, uint64_t seqno)556{557 struct vc4_bo *bo;558 unsigned i;559 560 for (i = 0; i < exec->bo_count; i++) {561 bo = to_vc4_bo(exec->bo[i]);562 bo->seqno = seqno;563 564 dma_resv_add_fence(bo->base.base.resv, exec->fence,565 DMA_RESV_USAGE_READ);566 }567 568 list_for_each_entry(bo, &exec->unref_list, unref_head) {569 bo->seqno = seqno;570 }571 572 for (i = 0; i < exec->rcl_write_bo_count; i++) {573 bo = to_vc4_bo(&exec->rcl_write_bo[i]->base);574 bo->write_seqno = seqno;575 576 dma_resv_add_fence(bo->base.base.resv, exec->fence,577 DMA_RESV_USAGE_WRITE);578 }579}580 581static void582vc4_unlock_bo_reservations(struct drm_device *dev,583 struct vc4_exec_info *exec,584 struct ww_acquire_ctx *acquire_ctx)585{586 int i;587 588 for (i = 0; i < exec->bo_count; i++)589 dma_resv_unlock(exec->bo[i]->resv);590 591 ww_acquire_fini(acquire_ctx);592}593 594/* Takes the reservation lock on all the BOs being referenced, so that595 * at queue submit time we can update the reservations.596 *597 * We don't lock the RCL the tile alloc/state BOs, or overflow memory598 * (all of which are on exec->unref_list). They're entirely private599 * to vc4, so we don't attach dma-buf fences to them.600 */601static int602vc4_lock_bo_reservations(struct drm_device *dev,603 struct vc4_exec_info *exec,604 struct ww_acquire_ctx *acquire_ctx)605{606 int contended_lock = -1;607 int i, ret;608 struct drm_gem_object *bo;609 610 ww_acquire_init(acquire_ctx, &reservation_ww_class);611 612retry:613 if (contended_lock != -1) {614 bo = exec->bo[contended_lock];615 ret = dma_resv_lock_slow_interruptible(bo->resv, acquire_ctx);616 if (ret) {617 ww_acquire_done(acquire_ctx);618 return ret;619 }620 }621 622 for (i = 0; i < exec->bo_count; i++) {623 if (i == contended_lock)624 continue;625 626 bo = exec->bo[i];627 628 ret = dma_resv_lock_interruptible(bo->resv, acquire_ctx);629 if (ret) {630 int j;631 632 for (j = 0; j < i; j++) {633 bo = exec->bo[j];634 dma_resv_unlock(bo->resv);635 }636 637 if (contended_lock != -1 && contended_lock >= i) {638 bo = exec->bo[contended_lock];639 640 dma_resv_unlock(bo->resv);641 }642 643 if (ret == -EDEADLK) {644 contended_lock = i;645 goto retry;646 }647 648 ww_acquire_done(acquire_ctx);649 return ret;650 }651 }652 653 ww_acquire_done(acquire_ctx);654 655 /* Reserve space for our shared (read-only) fence references,656 * before we commit the CL to the hardware.657 */658 for (i = 0; i < exec->bo_count; i++) {659 bo = exec->bo[i];660 661 ret = dma_resv_reserve_fences(bo->resv, 1);662 if (ret) {663 vc4_unlock_bo_reservations(dev, exec, acquire_ctx);664 return ret;665 }666 }667 668 return 0;669}670 671/* Queues a struct vc4_exec_info for execution. If no job is672 * currently executing, then submits it.673 *674 * Unlike most GPUs, our hardware only handles one command list at a675 * time. To queue multiple jobs at once, we'd need to edit the676 * previous command list to have a jump to the new one at the end, and677 * then bump the end address. That's a change for a later date,678 * though.679 */680static int681vc4_queue_submit(struct drm_device *dev, struct vc4_exec_info *exec,682 struct ww_acquire_ctx *acquire_ctx,683 struct drm_syncobj *out_sync)684{685 struct vc4_dev *vc4 = to_vc4_dev(dev);686 struct vc4_exec_info *renderjob;687 uint64_t seqno;688 unsigned long irqflags;689 struct vc4_fence *fence;690 691 fence = kzalloc(sizeof(*fence), GFP_KERNEL);692 if (!fence)693 return -ENOMEM;694 fence->dev = dev;695 696 spin_lock_irqsave(&vc4->job_lock, irqflags);697 698 seqno = ++vc4->emit_seqno;699 exec->seqno = seqno;700 701 dma_fence_init(&fence->base, &vc4_fence_ops, &vc4->job_lock,702 vc4->dma_fence_context, exec->seqno);703 fence->seqno = exec->seqno;704 exec->fence = &fence->base;705 706 if (out_sync)707 drm_syncobj_replace_fence(out_sync, exec->fence);708 709 vc4_update_bo_seqnos(exec, seqno);710 711 vc4_unlock_bo_reservations(dev, exec, acquire_ctx);712 713 list_add_tail(&exec->head, &vc4->bin_job_list);714 715 /* If no bin job was executing and if the render job (if any) has the716 * same perfmon as our job attached to it (or if both jobs don't have717 * perfmon activated), then kick ours off. Otherwise, it'll get718 * started when the previous job's flush/render done interrupt occurs.719 */720 renderjob = vc4_first_render_job(vc4);721 if (vc4_first_bin_job(vc4) == exec &&722 (!renderjob || renderjob->perfmon == exec->perfmon)) {723 vc4_submit_next_bin_job(dev);724 vc4_queue_hangcheck(dev);725 }726 727 spin_unlock_irqrestore(&vc4->job_lock, irqflags);728 729 return 0;730}731 732/**733 * vc4_cl_lookup_bos() - Sets up exec->bo[] with the GEM objects734 * referenced by the job.735 * @dev: DRM device736 * @file_priv: DRM file for this fd737 * @exec: V3D job being set up738 *739 * The command validator needs to reference BOs by their index within740 * the submitted job's BO list. This does the validation of the job's741 * BO list and reference counting for the lifetime of the job.742 */743static int744vc4_cl_lookup_bos(struct drm_device *dev,745 struct drm_file *file_priv,746 struct vc4_exec_info *exec)747{748 struct drm_vc4_submit_cl *args = exec->args;749 int ret = 0;750 int i;751 752 exec->bo_count = args->bo_handle_count;753 754 if (!exec->bo_count) {755 /* See comment on bo_index for why we have to check756 * this.757 */758 DRM_DEBUG("Rendering requires BOs to validate\n");759 return -EINVAL;760 }761 762 ret = drm_gem_objects_lookup(file_priv, u64_to_user_ptr(args->bo_handles),763 exec->bo_count, &exec->bo);764 765 if (ret)766 goto fail_put_bo;767 768 for (i = 0; i < exec->bo_count; i++) {769 ret = vc4_bo_inc_usecnt(to_vc4_bo(exec->bo[i]));770 if (ret)771 goto fail_dec_usecnt;772 }773 774 return 0;775 776fail_dec_usecnt:777 /* Decrease usecnt on acquired objects.778 * We cannot rely on vc4_complete_exec() to release resources here,779 * because vc4_complete_exec() has no information about which BO has780 * had its ->usecnt incremented.781 * To make things easier we just free everything explicitly and set782 * exec->bo to NULL so that vc4_complete_exec() skips the 'BO release'783 * step.784 */785 for (i-- ; i >= 0; i--)786 vc4_bo_dec_usecnt(to_vc4_bo(exec->bo[i]));787 788fail_put_bo:789 /* Release any reference to acquired objects. */790 for (i = 0; i < exec->bo_count && exec->bo[i]; i++)791 drm_gem_object_put(exec->bo[i]);792 793 kvfree(exec->bo);794 exec->bo = NULL;795 return ret;796}797 798static int799vc4_get_bcl(struct drm_device *dev, struct vc4_exec_info *exec)800{801 struct drm_vc4_submit_cl *args = exec->args;802 struct vc4_dev *vc4 = to_vc4_dev(dev);803 void *temp = NULL;804 void *bin;805 int ret = 0;806 uint32_t bin_offset = 0;807 uint32_t shader_rec_offset = roundup(bin_offset + args->bin_cl_size,808 16);809 uint32_t uniforms_offset = shader_rec_offset + args->shader_rec_size;810 uint32_t exec_size = uniforms_offset + args->uniforms_size;811 uint32_t temp_size = exec_size + (sizeof(struct vc4_shader_state) *812 args->shader_rec_count);813 struct vc4_bo *bo;814 815 if (shader_rec_offset < args->bin_cl_size ||816 uniforms_offset < shader_rec_offset ||817 exec_size < uniforms_offset ||818 args->shader_rec_count >= (UINT_MAX /819 sizeof(struct vc4_shader_state)) ||820 temp_size < exec_size) {821 DRM_DEBUG("overflow in exec arguments\n");822 ret = -EINVAL;823 goto fail;824 }825 826 /* Allocate space where we'll store the copied in user command lists827 * and shader records.828 *829 * We don't just copy directly into the BOs because we need to830 * read the contents back for validation, and I think the831 * bo->vaddr is uncached access.832 */833 temp = kvmalloc_array(temp_size, 1, GFP_KERNEL);834 if (!temp) {835 drm_err(dev, "Failed to allocate storage for copying "836 "in bin/render CLs.\n");837 ret = -ENOMEM;838 goto fail;839 }840 bin = temp + bin_offset;841 exec->shader_rec_u = temp + shader_rec_offset;842 exec->uniforms_u = temp + uniforms_offset;843 exec->shader_state = temp + exec_size;844 exec->shader_state_size = args->shader_rec_count;845 846 if (copy_from_user(bin,847 u64_to_user_ptr(args->bin_cl),848 args->bin_cl_size)) {849 ret = -EFAULT;850 goto fail;851 }852 853 if (copy_from_user(exec->shader_rec_u,854 u64_to_user_ptr(args->shader_rec),855 args->shader_rec_size)) {856 ret = -EFAULT;857 goto fail;858 }859 860 if (copy_from_user(exec->uniforms_u,861 u64_to_user_ptr(args->uniforms),862 args->uniforms_size)) {863 ret = -EFAULT;864 goto fail;865 }866 867 bo = vc4_bo_create(dev, exec_size, true, VC4_BO_TYPE_BCL);868 if (IS_ERR(bo)) {869 drm_err(dev, "Couldn't allocate BO for binning\n");870 ret = PTR_ERR(bo);871 goto fail;872 }873 exec->exec_bo = &bo->base;874 875 list_add_tail(&to_vc4_bo(&exec->exec_bo->base)->unref_head,876 &exec->unref_list);877 878 exec->ct0ca = exec->exec_bo->dma_addr + bin_offset;879 880 exec->bin_u = bin;881 882 exec->shader_rec_v = exec->exec_bo->vaddr + shader_rec_offset;883 exec->shader_rec_p = exec->exec_bo->dma_addr + shader_rec_offset;884 exec->shader_rec_size = args->shader_rec_size;885 886 exec->uniforms_v = exec->exec_bo->vaddr + uniforms_offset;887 exec->uniforms_p = exec->exec_bo->dma_addr + uniforms_offset;888 exec->uniforms_size = args->uniforms_size;889 890 ret = vc4_validate_bin_cl(dev,891 exec->exec_bo->vaddr + bin_offset,892 bin,893 exec);894 if (ret)895 goto fail;896 897 ret = vc4_validate_shader_recs(dev, exec);898 if (ret)899 goto fail;900 901 if (exec->found_tile_binning_mode_config_packet) {902 ret = vc4_v3d_bin_bo_get(vc4, &exec->bin_bo_used);903 if (ret)904 goto fail;905 }906 907 /* Block waiting on any previous rendering into the CS's VBO,908 * IB, or textures, so that pixels are actually written by the909 * time we try to read them.910 */911 ret = vc4_wait_for_seqno(dev, exec->bin_dep_seqno, ~0ull, true);912 913fail:914 kvfree(temp);915 return ret;916}917 918static void919vc4_complete_exec(struct drm_device *dev, struct vc4_exec_info *exec)920{921 struct vc4_dev *vc4 = to_vc4_dev(dev);922 unsigned long irqflags;923 unsigned i;924 925 /* If we got force-completed because of GPU reset rather than926 * through our IRQ handler, signal the fence now.927 */928 if (exec->fence) {929 dma_fence_signal(exec->fence);930 dma_fence_put(exec->fence);931 }932 933 if (exec->bo) {934 for (i = 0; i < exec->bo_count; i++) {935 struct vc4_bo *bo = to_vc4_bo(exec->bo[i]);936 937 vc4_bo_dec_usecnt(bo);938 drm_gem_object_put(exec->bo[i]);939 }940 kvfree(exec->bo);941 }942 943 while (!list_empty(&exec->unref_list)) {944 struct vc4_bo *bo = list_first_entry(&exec->unref_list,945 struct vc4_bo, unref_head);946 list_del(&bo->unref_head);947 drm_gem_object_put(&bo->base.base);948 }949 950 /* Free up the allocation of any bin slots we used. */951 spin_lock_irqsave(&vc4->job_lock, irqflags);952 vc4->bin_alloc_used &= ~exec->bin_slots;953 spin_unlock_irqrestore(&vc4->job_lock, irqflags);954 955 /* Release the reference on the binner BO if needed. */956 if (exec->bin_bo_used)957 vc4_v3d_bin_bo_put(vc4);958 959 /* Release the reference we had on the perf monitor. */960 vc4_perfmon_put(exec->perfmon);961 962 vc4_v3d_pm_put(vc4);963 964 kfree(exec);965}966 967void968vc4_job_handle_completed(struct vc4_dev *vc4)969{970 unsigned long irqflags;971 struct vc4_seqno_cb *cb, *cb_temp;972 973 if (WARN_ON_ONCE(vc4->is_vc5))974 return;975 976 spin_lock_irqsave(&vc4->job_lock, irqflags);977 while (!list_empty(&vc4->job_done_list)) {978 struct vc4_exec_info *exec =979 list_first_entry(&vc4->job_done_list,980 struct vc4_exec_info, head);981 list_del(&exec->head);982 983 spin_unlock_irqrestore(&vc4->job_lock, irqflags);984 vc4_complete_exec(&vc4->base, exec);985 spin_lock_irqsave(&vc4->job_lock, irqflags);986 }987 988 list_for_each_entry_safe(cb, cb_temp, &vc4->seqno_cb_list, work.entry) {989 if (cb->seqno <= vc4->finished_seqno) {990 list_del_init(&cb->work.entry);991 schedule_work(&cb->work);992 }993 }994 995 spin_unlock_irqrestore(&vc4->job_lock, irqflags);996}997 998static void vc4_seqno_cb_work(struct work_struct *work)999{1000 struct vc4_seqno_cb *cb = container_of(work, struct vc4_seqno_cb, work);1001 1002 cb->func(cb);1003}1004 1005int vc4_queue_seqno_cb(struct drm_device *dev,1006 struct vc4_seqno_cb *cb, uint64_t seqno,1007 void (*func)(struct vc4_seqno_cb *cb))1008{1009 struct vc4_dev *vc4 = to_vc4_dev(dev);1010 unsigned long irqflags;1011 1012 if (WARN_ON_ONCE(vc4->is_vc5))1013 return -ENODEV;1014 1015 cb->func = func;1016 INIT_WORK(&cb->work, vc4_seqno_cb_work);1017 1018 spin_lock_irqsave(&vc4->job_lock, irqflags);1019 if (seqno > vc4->finished_seqno) {1020 cb->seqno = seqno;1021 list_add_tail(&cb->work.entry, &vc4->seqno_cb_list);1022 } else {1023 schedule_work(&cb->work);1024 }1025 spin_unlock_irqrestore(&vc4->job_lock, irqflags);1026 1027 return 0;1028}1029 1030/* Scheduled when any job has been completed, this walks the list of1031 * jobs that had completed and unrefs their BOs and frees their exec1032 * structs.1033 */1034static void1035vc4_job_done_work(struct work_struct *work)1036{1037 struct vc4_dev *vc4 =1038 container_of(work, struct vc4_dev, job_done_work);1039 1040 vc4_job_handle_completed(vc4);1041}1042 1043static int1044vc4_wait_for_seqno_ioctl_helper(struct drm_device *dev,1045 uint64_t seqno,1046 uint64_t *timeout_ns)1047{1048 unsigned long start = jiffies;1049 int ret = vc4_wait_for_seqno(dev, seqno, *timeout_ns, true);1050 1051 if ((ret == -EINTR || ret == -ERESTARTSYS) && *timeout_ns != ~0ull) {1052 uint64_t delta = jiffies_to_nsecs(jiffies - start);1053 1054 if (*timeout_ns >= delta)1055 *timeout_ns -= delta;1056 }1057 1058 return ret;1059}1060 1061int1062vc4_wait_seqno_ioctl(struct drm_device *dev, void *data,1063 struct drm_file *file_priv)1064{1065 struct vc4_dev *vc4 = to_vc4_dev(dev);1066 struct drm_vc4_wait_seqno *args = data;1067 1068 if (WARN_ON_ONCE(vc4->is_vc5))1069 return -ENODEV;1070 1071 return vc4_wait_for_seqno_ioctl_helper(dev, args->seqno,1072 &args->timeout_ns);1073}1074 1075int1076vc4_wait_bo_ioctl(struct drm_device *dev, void *data,1077 struct drm_file *file_priv)1078{1079 struct vc4_dev *vc4 = to_vc4_dev(dev);1080 int ret;1081 struct drm_vc4_wait_bo *args = data;1082 struct drm_gem_object *gem_obj;1083 struct vc4_bo *bo;1084 1085 if (WARN_ON_ONCE(vc4->is_vc5))1086 return -ENODEV;1087 1088 if (args->pad != 0)1089 return -EINVAL;1090 1091 gem_obj = drm_gem_object_lookup(file_priv, args->handle);1092 if (!gem_obj) {1093 DRM_DEBUG("Failed to look up GEM BO %d\n", args->handle);1094 return -EINVAL;1095 }1096 bo = to_vc4_bo(gem_obj);1097 1098 ret = vc4_wait_for_seqno_ioctl_helper(dev, bo->seqno,1099 &args->timeout_ns);1100 1101 drm_gem_object_put(gem_obj);1102 return ret;1103}1104 1105/**1106 * vc4_submit_cl_ioctl() - Submits a job (frame) to the VC4.1107 * @dev: DRM device1108 * @data: ioctl argument1109 * @file_priv: DRM file for this fd1110 *1111 * This is the main entrypoint for userspace to submit a 3D frame to1112 * the GPU. Userspace provides the binner command list (if1113 * applicable), and the kernel sets up the render command list to draw1114 * to the framebuffer described in the ioctl, using the command lists1115 * that the 3D engine's binner will produce.1116 */1117int1118vc4_submit_cl_ioctl(struct drm_device *dev, void *data,1119 struct drm_file *file_priv)1120{1121 struct vc4_dev *vc4 = to_vc4_dev(dev);1122 struct vc4_file *vc4file = file_priv->driver_priv;1123 struct drm_vc4_submit_cl *args = data;1124 struct drm_syncobj *out_sync = NULL;1125 struct vc4_exec_info *exec;1126 struct ww_acquire_ctx acquire_ctx;1127 struct dma_fence *in_fence;1128 int ret = 0;1129 1130 trace_vc4_submit_cl_ioctl(dev, args->bin_cl_size,1131 args->shader_rec_size,1132 args->bo_handle_count);1133 1134 if (WARN_ON_ONCE(vc4->is_vc5))1135 return -ENODEV;1136 1137 if (!vc4->v3d) {1138 DRM_DEBUG("VC4_SUBMIT_CL with no VC4 V3D probed\n");1139 return -ENODEV;1140 }1141 1142 if ((args->flags & ~(VC4_SUBMIT_CL_USE_CLEAR_COLOR |1143 VC4_SUBMIT_CL_FIXED_RCL_ORDER |1144 VC4_SUBMIT_CL_RCL_ORDER_INCREASING_X |1145 VC4_SUBMIT_CL_RCL_ORDER_INCREASING_Y)) != 0) {1146 DRM_DEBUG("Unknown flags: 0x%02x\n", args->flags);1147 return -EINVAL;1148 }1149 1150 if (args->pad2 != 0) {1151 DRM_DEBUG("Invalid pad: 0x%08x\n", args->pad2);1152 return -EINVAL;1153 }1154 1155 exec = kcalloc(1, sizeof(*exec), GFP_KERNEL);1156 if (!exec)1157 return -ENOMEM;1158 1159 exec->dev = vc4;1160 1161 ret = vc4_v3d_pm_get(vc4);1162 if (ret) {1163 kfree(exec);1164 return ret;1165 }1166 1167 exec->args = args;1168 INIT_LIST_HEAD(&exec->unref_list);1169 1170 ret = vc4_cl_lookup_bos(dev, file_priv, exec);1171 if (ret)1172 goto fail;1173 1174 if (args->perfmonid) {1175 exec->perfmon = vc4_perfmon_find(vc4file,1176 args->perfmonid);1177 if (!exec->perfmon) {1178 ret = -ENOENT;1179 goto fail;1180 }1181 }1182 1183 if (args->in_sync) {1184 ret = drm_syncobj_find_fence(file_priv, args->in_sync,1185 0, 0, &in_fence);1186 if (ret)1187 goto fail;1188 1189 /* When the fence (or fence array) is exclusively from our1190 * context we can skip the wait since jobs are executed in1191 * order of their submission through this ioctl and this can1192 * only have fences from a prior job.1193 */1194 if (!dma_fence_match_context(in_fence,1195 vc4->dma_fence_context)) {1196 ret = dma_fence_wait(in_fence, true);1197 if (ret) {1198 dma_fence_put(in_fence);1199 goto fail;1200 }1201 }1202 1203 dma_fence_put(in_fence);1204 }1205 1206 if (exec->args->bin_cl_size != 0) {1207 ret = vc4_get_bcl(dev, exec);1208 if (ret)1209 goto fail;1210 } else {1211 exec->ct0ca = 0;1212 exec->ct0ea = 0;1213 }1214 1215 ret = vc4_get_rcl(dev, exec);1216 if (ret)1217 goto fail;1218 1219 ret = vc4_lock_bo_reservations(dev, exec, &acquire_ctx);1220 if (ret)1221 goto fail;1222 1223 if (args->out_sync) {1224 out_sync = drm_syncobj_find(file_priv, args->out_sync);1225 if (!out_sync) {1226 ret = -EINVAL;1227 goto fail;1228 }1229 1230 /* We replace the fence in out_sync in vc4_queue_submit since1231 * the render job could execute immediately after that call.1232 * If it finishes before our ioctl processing resumes the1233 * render job fence could already have been freed.1234 */1235 }1236 1237 /* Clear this out of the struct we'll be putting in the queue,1238 * since it's part of our stack.1239 */1240 exec->args = NULL;1241 1242 ret = vc4_queue_submit(dev, exec, &acquire_ctx, out_sync);1243 1244 /* The syncobj isn't part of the exec data and we need to free our1245 * reference even if job submission failed.1246 */1247 if (out_sync)1248 drm_syncobj_put(out_sync);1249 1250 if (ret)1251 goto fail;1252 1253 /* Return the seqno for our job. */1254 args->seqno = vc4->emit_seqno;1255 1256 return 0;1257 1258fail:1259 vc4_complete_exec(&vc4->base, exec);1260 1261 return ret;1262}1263 1264static void vc4_gem_destroy(struct drm_device *dev, void *unused);1265int vc4_gem_init(struct drm_device *dev)1266{1267 struct vc4_dev *vc4 = to_vc4_dev(dev);1268 int ret;1269 1270 if (WARN_ON_ONCE(vc4->is_vc5))1271 return -ENODEV;1272 1273 vc4->dma_fence_context = dma_fence_context_alloc(1);1274 1275 INIT_LIST_HEAD(&vc4->bin_job_list);1276 INIT_LIST_HEAD(&vc4->render_job_list);1277 INIT_LIST_HEAD(&vc4->job_done_list);1278 INIT_LIST_HEAD(&vc4->seqno_cb_list);1279 spin_lock_init(&vc4->job_lock);1280 1281 INIT_WORK(&vc4->hangcheck.reset_work, vc4_reset_work);1282 timer_setup(&vc4->hangcheck.timer, vc4_hangcheck_elapsed, 0);1283 1284 INIT_WORK(&vc4->job_done_work, vc4_job_done_work);1285 1286 ret = drmm_mutex_init(dev, &vc4->power_lock);1287 if (ret)1288 return ret;1289 1290 INIT_LIST_HEAD(&vc4->purgeable.list);1291 1292 ret = drmm_mutex_init(dev, &vc4->purgeable.lock);1293 if (ret)1294 return ret;1295 1296 return drmm_add_action_or_reset(dev, vc4_gem_destroy, NULL);1297}1298 1299static void vc4_gem_destroy(struct drm_device *dev, void *unused)1300{1301 struct vc4_dev *vc4 = to_vc4_dev(dev);1302 1303 /* Waiting for exec to finish would need to be done before1304 * unregistering V3D.1305 */1306 WARN_ON(vc4->emit_seqno != vc4->finished_seqno);1307 1308 /* V3D should already have disabled its interrupt and cleared1309 * the overflow allocation registers. Now free the object.1310 */1311 if (vc4->bin_bo) {1312 drm_gem_object_put(&vc4->bin_bo->base.base);1313 vc4->bin_bo = NULL;1314 }1315 1316 if (vc4->hang_state)1317 vc4_free_hang_state(dev, vc4->hang_state);1318}1319 1320int vc4_gem_madvise_ioctl(struct drm_device *dev, void *data,1321 struct drm_file *file_priv)1322{1323 struct vc4_dev *vc4 = to_vc4_dev(dev);1324 struct drm_vc4_gem_madvise *args = data;1325 struct drm_gem_object *gem_obj;1326 struct vc4_bo *bo;1327 int ret;1328 1329 if (WARN_ON_ONCE(vc4->is_vc5))1330 return -ENODEV;1331 1332 switch (args->madv) {1333 case VC4_MADV_DONTNEED:1334 case VC4_MADV_WILLNEED:1335 break;1336 default:1337 return -EINVAL;1338 }1339 1340 if (args->pad != 0)1341 return -EINVAL;1342 1343 gem_obj = drm_gem_object_lookup(file_priv, args->handle);1344 if (!gem_obj) {1345 DRM_DEBUG("Failed to look up GEM BO %d\n", args->handle);1346 return -ENOENT;1347 }1348 1349 bo = to_vc4_bo(gem_obj);1350 1351 /* Only BOs exposed to userspace can be purged. */1352 if (bo->madv == __VC4_MADV_NOTSUPP) {1353 DRM_DEBUG("madvise not supported on this BO\n");1354 ret = -EINVAL;1355 goto out_put_gem;1356 }1357 1358 /* Not sure it's safe to purge imported BOs. Let's just assume it's1359 * not until proven otherwise.1360 */1361 if (gem_obj->import_attach) {1362 DRM_DEBUG("madvise not supported on imported BOs\n");1363 ret = -EINVAL;1364 goto out_put_gem;1365 }1366 1367 mutex_lock(&bo->madv_lock);1368 1369 if (args->madv == VC4_MADV_DONTNEED && bo->madv == VC4_MADV_WILLNEED &&1370 !refcount_read(&bo->usecnt)) {1371 /* If the BO is about to be marked as purgeable, is not used1372 * and is not already purgeable or purged, add it to the1373 * purgeable list.1374 */1375 vc4_bo_add_to_purgeable_pool(bo);1376 } else if (args->madv == VC4_MADV_WILLNEED &&1377 bo->madv == VC4_MADV_DONTNEED &&1378 !refcount_read(&bo->usecnt)) {1379 /* The BO has not been purged yet, just remove it from1380 * the purgeable list.1381 */1382 vc4_bo_remove_from_purgeable_pool(bo);1383 }1384 1385 /* Save the purged state. */1386 args->retained = bo->madv != __VC4_MADV_PURGED;1387 1388 /* Update internal madv state only if the bo was not purged. */1389 if (bo->madv != __VC4_MADV_PURGED)1390 bo->madv = args->madv;1391 1392 mutex_unlock(&bo->madv_lock);1393 1394 ret = 0;1395 1396out_put_gem:1397 drm_gem_object_put(gem_obj);1398 1399 return ret;1400}1401