862 lines · c
1// SPDX-License-Identifier: GPL-2.0+2/* Copyright (C) 2018 Broadcom */3 4/**5 * DOC: Broadcom V3D scheduling6 *7 * The shared DRM GPU scheduler is used to coordinate submitting jobs8 * to the hardware. Each DRM fd (roughly a client process) gets its9 * own scheduler entity, which will process jobs in order. The GPU10 * scheduler will round-robin between clients to submit the next job.11 *12 * For simplicity, and in order to keep latency low for interactive13 * jobs when bulk background jobs are queued up, we submit a new job14 * to the HW only when it has completed the last one, instead of15 * filling up the CT[01]Q FIFOs with jobs. Similarly, we use16 * drm_sched_job_add_dependency() to manage the dependency between bin and17 * render, instead of having the clients submit jobs using the HW's18 * semaphores to interlock between them.19 */20 21#include <linux/sched/clock.h>22#include <linux/kthread.h>23 24#include <drm/drm_syncobj.h>25 26#include "v3d_drv.h"27#include "v3d_regs.h"28#include "v3d_trace.h"29 30#define V3D_CSD_CFG012_WG_COUNT_SHIFT 1631 32static struct v3d_job *33to_v3d_job(struct drm_sched_job *sched_job)34{35 return container_of(sched_job, struct v3d_job, base);36}37 38static struct v3d_bin_job *39to_bin_job(struct drm_sched_job *sched_job)40{41 return container_of(sched_job, struct v3d_bin_job, base.base);42}43 44static struct v3d_render_job *45to_render_job(struct drm_sched_job *sched_job)46{47 return container_of(sched_job, struct v3d_render_job, base.base);48}49 50static struct v3d_tfu_job *51to_tfu_job(struct drm_sched_job *sched_job)52{53 return container_of(sched_job, struct v3d_tfu_job, base.base);54}55 56static struct v3d_csd_job *57to_csd_job(struct drm_sched_job *sched_job)58{59 return container_of(sched_job, struct v3d_csd_job, base.base);60}61 62static struct v3d_cpu_job *63to_cpu_job(struct drm_sched_job *sched_job)64{65 return container_of(sched_job, struct v3d_cpu_job, base.base);66}67 68static void69v3d_sched_job_free(struct drm_sched_job *sched_job)70{71 struct v3d_job *job = to_v3d_job(sched_job);72 73 v3d_job_cleanup(job);74}75 76void77v3d_timestamp_query_info_free(struct v3d_timestamp_query_info *query_info,78 unsigned int count)79{80 if (query_info->queries) {81 unsigned int i;82 83 for (i = 0; i < count; i++)84 drm_syncobj_put(query_info->queries[i].syncobj);85 86 kvfree(query_info->queries);87 }88}89 90void91v3d_performance_query_info_free(struct v3d_performance_query_info *query_info,92 unsigned int count)93{94 if (query_info->queries) {95 unsigned int i;96 97 for (i = 0; i < count; i++) {98 drm_syncobj_put(query_info->queries[i].syncobj);99 kvfree(query_info->queries[i].kperfmon_ids);100 }101 102 kvfree(query_info->queries);103 }104}105 106static void107v3d_cpu_job_free(struct drm_sched_job *sched_job)108{109 struct v3d_cpu_job *job = to_cpu_job(sched_job);110 111 v3d_timestamp_query_info_free(&job->timestamp_query,112 job->timestamp_query.count);113 114 v3d_performance_query_info_free(&job->performance_query,115 job->performance_query.count);116 117 v3d_job_cleanup(&job->base);118}119 120static void121v3d_switch_perfmon(struct v3d_dev *v3d, struct v3d_job *job)122{123 if (job->perfmon != v3d->active_perfmon)124 v3d_perfmon_stop(v3d, v3d->active_perfmon, true);125 126 if (job->perfmon && v3d->active_perfmon != job->perfmon)127 v3d_perfmon_start(v3d, job->perfmon);128}129 130static void131v3d_job_start_stats(struct v3d_job *job, enum v3d_queue queue)132{133 struct v3d_dev *v3d = job->v3d;134 struct v3d_file_priv *file = job->file->driver_priv;135 struct v3d_stats *global_stats = &v3d->queue[queue].stats;136 struct v3d_stats *local_stats = &file->stats[queue];137 u64 now = local_clock();138 139 preempt_disable();140 141 write_seqcount_begin(&local_stats->lock);142 local_stats->start_ns = now;143 write_seqcount_end(&local_stats->lock);144 145 write_seqcount_begin(&global_stats->lock);146 global_stats->start_ns = now;147 write_seqcount_end(&global_stats->lock);148 149 preempt_enable();150}151 152static void153v3d_stats_update(struct v3d_stats *stats, u64 now)154{155 write_seqcount_begin(&stats->lock);156 stats->enabled_ns += now - stats->start_ns;157 stats->jobs_completed++;158 stats->start_ns = 0;159 write_seqcount_end(&stats->lock);160}161 162void163v3d_job_update_stats(struct v3d_job *job, enum v3d_queue queue)164{165 struct v3d_dev *v3d = job->v3d;166 struct v3d_file_priv *file = job->file->driver_priv;167 struct v3d_stats *global_stats = &v3d->queue[queue].stats;168 struct v3d_stats *local_stats = &file->stats[queue];169 u64 now = local_clock();170 171 preempt_disable();172 v3d_stats_update(local_stats, now);173 v3d_stats_update(global_stats, now);174 preempt_enable();175}176 177static struct dma_fence *v3d_bin_job_run(struct drm_sched_job *sched_job)178{179 struct v3d_bin_job *job = to_bin_job(sched_job);180 struct v3d_dev *v3d = job->base.v3d;181 struct drm_device *dev = &v3d->drm;182 struct dma_fence *fence;183 unsigned long irqflags;184 185 if (unlikely(job->base.base.s_fence->finished.error))186 return NULL;187 188 /* Lock required around bin_job update vs189 * v3d_overflow_mem_work().190 */191 spin_lock_irqsave(&v3d->job_lock, irqflags);192 v3d->bin_job = job;193 /* Clear out the overflow allocation, so we don't194 * reuse the overflow attached to a previous job.195 */196 V3D_CORE_WRITE(0, V3D_PTB_BPOS, 0);197 spin_unlock_irqrestore(&v3d->job_lock, irqflags);198 199 v3d_invalidate_caches(v3d);200 201 fence = v3d_fence_create(v3d, V3D_BIN);202 if (IS_ERR(fence))203 return NULL;204 205 if (job->base.irq_fence)206 dma_fence_put(job->base.irq_fence);207 job->base.irq_fence = dma_fence_get(fence);208 209 trace_v3d_submit_cl(dev, false, to_v3d_fence(fence)->seqno,210 job->start, job->end);211 212 v3d_job_start_stats(&job->base, V3D_BIN);213 v3d_switch_perfmon(v3d, &job->base);214 215 /* Set the current and end address of the control list.216 * Writing the end register is what starts the job.217 */218 if (job->qma) {219 V3D_CORE_WRITE(0, V3D_CLE_CT0QMA, job->qma);220 V3D_CORE_WRITE(0, V3D_CLE_CT0QMS, job->qms);221 }222 if (job->qts) {223 V3D_CORE_WRITE(0, V3D_CLE_CT0QTS,224 V3D_CLE_CT0QTS_ENABLE |225 job->qts);226 }227 V3D_CORE_WRITE(0, V3D_CLE_CT0QBA, job->start);228 V3D_CORE_WRITE(0, V3D_CLE_CT0QEA, job->end);229 230 return fence;231}232 233static struct dma_fence *v3d_render_job_run(struct drm_sched_job *sched_job)234{235 struct v3d_render_job *job = to_render_job(sched_job);236 struct v3d_dev *v3d = job->base.v3d;237 struct drm_device *dev = &v3d->drm;238 struct dma_fence *fence;239 240 if (unlikely(job->base.base.s_fence->finished.error))241 return NULL;242 243 v3d->render_job = job;244 245 /* Can we avoid this flush? We need to be careful of246 * scheduling, though -- imagine job0 rendering to texture and247 * job1 reading, and them being executed as bin0, bin1,248 * render0, render1, so that render1's flush at bin time249 * wasn't enough.250 */251 v3d_invalidate_caches(v3d);252 253 fence = v3d_fence_create(v3d, V3D_RENDER);254 if (IS_ERR(fence))255 return NULL;256 257 if (job->base.irq_fence)258 dma_fence_put(job->base.irq_fence);259 job->base.irq_fence = dma_fence_get(fence);260 261 trace_v3d_submit_cl(dev, true, to_v3d_fence(fence)->seqno,262 job->start, job->end);263 264 v3d_job_start_stats(&job->base, V3D_RENDER);265 v3d_switch_perfmon(v3d, &job->base);266 267 /* XXX: Set the QCFG */268 269 /* Set the current and end address of the control list.270 * Writing the end register is what starts the job.271 */272 V3D_CORE_WRITE(0, V3D_CLE_CT1QBA, job->start);273 V3D_CORE_WRITE(0, V3D_CLE_CT1QEA, job->end);274 275 return fence;276}277 278static struct dma_fence *279v3d_tfu_job_run(struct drm_sched_job *sched_job)280{281 struct v3d_tfu_job *job = to_tfu_job(sched_job);282 struct v3d_dev *v3d = job->base.v3d;283 struct drm_device *dev = &v3d->drm;284 struct dma_fence *fence;285 286 fence = v3d_fence_create(v3d, V3D_TFU);287 if (IS_ERR(fence))288 return NULL;289 290 v3d->tfu_job = job;291 if (job->base.irq_fence)292 dma_fence_put(job->base.irq_fence);293 job->base.irq_fence = dma_fence_get(fence);294 295 trace_v3d_submit_tfu(dev, to_v3d_fence(fence)->seqno);296 297 v3d_job_start_stats(&job->base, V3D_TFU);298 299 V3D_WRITE(V3D_TFU_IIA(v3d->ver), job->args.iia);300 V3D_WRITE(V3D_TFU_IIS(v3d->ver), job->args.iis);301 V3D_WRITE(V3D_TFU_ICA(v3d->ver), job->args.ica);302 V3D_WRITE(V3D_TFU_IUA(v3d->ver), job->args.iua);303 V3D_WRITE(V3D_TFU_IOA(v3d->ver), job->args.ioa);304 if (v3d->ver >= 71)305 V3D_WRITE(V3D_V7_TFU_IOC, job->args.v71.ioc);306 V3D_WRITE(V3D_TFU_IOS(v3d->ver), job->args.ios);307 V3D_WRITE(V3D_TFU_COEF0(v3d->ver), job->args.coef[0]);308 if (v3d->ver >= 71 || (job->args.coef[0] & V3D_TFU_COEF0_USECOEF)) {309 V3D_WRITE(V3D_TFU_COEF1(v3d->ver), job->args.coef[1]);310 V3D_WRITE(V3D_TFU_COEF2(v3d->ver), job->args.coef[2]);311 V3D_WRITE(V3D_TFU_COEF3(v3d->ver), job->args.coef[3]);312 }313 /* ICFG kicks off the job. */314 V3D_WRITE(V3D_TFU_ICFG(v3d->ver), job->args.icfg | V3D_TFU_ICFG_IOC);315 316 return fence;317}318 319static struct dma_fence *320v3d_csd_job_run(struct drm_sched_job *sched_job)321{322 struct v3d_csd_job *job = to_csd_job(sched_job);323 struct v3d_dev *v3d = job->base.v3d;324 struct drm_device *dev = &v3d->drm;325 struct dma_fence *fence;326 int i, csd_cfg0_reg;327 328 v3d->csd_job = job;329 330 v3d_invalidate_caches(v3d);331 332 fence = v3d_fence_create(v3d, V3D_CSD);333 if (IS_ERR(fence))334 return NULL;335 336 if (job->base.irq_fence)337 dma_fence_put(job->base.irq_fence);338 job->base.irq_fence = dma_fence_get(fence);339 340 trace_v3d_submit_csd(dev, to_v3d_fence(fence)->seqno);341 342 v3d_job_start_stats(&job->base, V3D_CSD);343 v3d_switch_perfmon(v3d, &job->base);344 345 csd_cfg0_reg = V3D_CSD_QUEUED_CFG0(v3d->ver);346 for (i = 1; i <= 6; i++)347 V3D_CORE_WRITE(0, csd_cfg0_reg + 4 * i, job->args.cfg[i]);348 349 /* Although V3D 7.1 has an eighth configuration register, we are not350 * using it. Therefore, make sure it remains unused.351 *352 * XXX: Set the CFG7 register353 */354 if (v3d->ver >= 71)355 V3D_CORE_WRITE(0, V3D_V7_CSD_QUEUED_CFG7, 0);356 357 /* CFG0 write kicks off the job. */358 V3D_CORE_WRITE(0, csd_cfg0_reg, job->args.cfg[0]);359 360 return fence;361}362 363static void364v3d_rewrite_csd_job_wg_counts_from_indirect(struct v3d_cpu_job *job)365{366 struct v3d_indirect_csd_info *indirect_csd = &job->indirect_csd;367 struct v3d_bo *bo = to_v3d_bo(job->base.bo[0]);368 struct v3d_bo *indirect = to_v3d_bo(indirect_csd->indirect);369 struct drm_v3d_submit_csd *args = &indirect_csd->job->args;370 u32 *wg_counts;371 372 v3d_get_bo_vaddr(bo);373 v3d_get_bo_vaddr(indirect);374 375 wg_counts = (uint32_t *)(bo->vaddr + indirect_csd->offset);376 377 if (wg_counts[0] == 0 || wg_counts[1] == 0 || wg_counts[2] == 0)378 return;379 380 args->cfg[0] = wg_counts[0] << V3D_CSD_CFG012_WG_COUNT_SHIFT;381 args->cfg[1] = wg_counts[1] << V3D_CSD_CFG012_WG_COUNT_SHIFT;382 args->cfg[2] = wg_counts[2] << V3D_CSD_CFG012_WG_COUNT_SHIFT;383 args->cfg[4] = DIV_ROUND_UP(indirect_csd->wg_size, 16) *384 (wg_counts[0] * wg_counts[1] * wg_counts[2]) - 1;385 386 for (int i = 0; i < 3; i++) {387 /* 0xffffffff indicates that the uniform rewrite is not needed */388 if (indirect_csd->wg_uniform_offsets[i] != 0xffffffff) {389 u32 uniform_idx = indirect_csd->wg_uniform_offsets[i];390 ((uint32_t *)indirect->vaddr)[uniform_idx] = wg_counts[i];391 }392 }393 394 v3d_put_bo_vaddr(indirect);395 v3d_put_bo_vaddr(bo);396}397 398static void399v3d_timestamp_query(struct v3d_cpu_job *job)400{401 struct v3d_timestamp_query_info *timestamp_query = &job->timestamp_query;402 struct v3d_bo *bo = to_v3d_bo(job->base.bo[0]);403 u8 *value_addr;404 405 v3d_get_bo_vaddr(bo);406 407 for (int i = 0; i < timestamp_query->count; i++) {408 value_addr = ((u8 *)bo->vaddr) + timestamp_query->queries[i].offset;409 *((u64 *)value_addr) = i == 0 ? ktime_get_ns() : 0ull;410 411 drm_syncobj_replace_fence(timestamp_query->queries[i].syncobj,412 job->base.done_fence);413 }414 415 v3d_put_bo_vaddr(bo);416}417 418static void419v3d_reset_timestamp_queries(struct v3d_cpu_job *job)420{421 struct v3d_timestamp_query_info *timestamp_query = &job->timestamp_query;422 struct v3d_timestamp_query *queries = timestamp_query->queries;423 struct v3d_bo *bo = to_v3d_bo(job->base.bo[0]);424 u8 *value_addr;425 426 v3d_get_bo_vaddr(bo);427 428 for (int i = 0; i < timestamp_query->count; i++) {429 value_addr = ((u8 *)bo->vaddr) + queries[i].offset;430 *((u64 *)value_addr) = 0;431 432 drm_syncobj_replace_fence(queries[i].syncobj, NULL);433 }434 435 v3d_put_bo_vaddr(bo);436}437 438static void write_to_buffer_32(u32 *dst, unsigned int idx, u32 value)439{440 dst[idx] = value;441}442 443static void write_to_buffer_64(u64 *dst, unsigned int idx, u64 value)444{445 dst[idx] = value;446}447 448static void449write_to_buffer(void *dst, unsigned int idx, bool do_64bit, u64 value)450{451 if (do_64bit)452 write_to_buffer_64(dst, idx, value);453 else454 write_to_buffer_32(dst, idx, value);455}456 457static void458v3d_copy_query_results(struct v3d_cpu_job *job)459{460 struct v3d_timestamp_query_info *timestamp_query = &job->timestamp_query;461 struct v3d_timestamp_query *queries = timestamp_query->queries;462 struct v3d_bo *bo = to_v3d_bo(job->base.bo[0]);463 struct v3d_bo *timestamp = to_v3d_bo(job->base.bo[1]);464 struct v3d_copy_query_results_info *copy = &job->copy;465 struct dma_fence *fence;466 u8 *query_addr;467 bool available, write_result;468 u8 *data;469 int i;470 471 v3d_get_bo_vaddr(bo);472 v3d_get_bo_vaddr(timestamp);473 474 data = ((u8 *)bo->vaddr) + copy->offset;475 476 for (i = 0; i < timestamp_query->count; i++) {477 fence = drm_syncobj_fence_get(queries[i].syncobj);478 available = fence ? dma_fence_is_signaled(fence) : false;479 480 write_result = available || copy->do_partial;481 if (write_result) {482 query_addr = ((u8 *)timestamp->vaddr) + queries[i].offset;483 write_to_buffer(data, 0, copy->do_64bit, *((u64 *)query_addr));484 }485 486 if (copy->availability_bit)487 write_to_buffer(data, 1, copy->do_64bit, available ? 1u : 0u);488 489 data += copy->stride;490 491 dma_fence_put(fence);492 }493 494 v3d_put_bo_vaddr(timestamp);495 v3d_put_bo_vaddr(bo);496}497 498static void499v3d_reset_performance_queries(struct v3d_cpu_job *job)500{501 struct v3d_performance_query_info *performance_query = &job->performance_query;502 struct v3d_file_priv *v3d_priv = job->base.file->driver_priv;503 struct v3d_dev *v3d = job->base.v3d;504 struct v3d_perfmon *perfmon;505 506 for (int i = 0; i < performance_query->count; i++) {507 for (int j = 0; j < performance_query->nperfmons; j++) {508 perfmon = v3d_perfmon_find(v3d_priv,509 performance_query->queries[i].kperfmon_ids[j]);510 if (!perfmon) {511 DRM_DEBUG("Failed to find perfmon.");512 continue;513 }514 515 v3d_perfmon_stop(v3d, perfmon, false);516 517 memset(perfmon->values, 0, perfmon->ncounters * sizeof(u64));518 519 v3d_perfmon_put(perfmon);520 }521 522 drm_syncobj_replace_fence(performance_query->queries[i].syncobj, NULL);523 }524}525 526static void527v3d_write_performance_query_result(struct v3d_cpu_job *job, void *data,528 unsigned int query)529{530 struct v3d_performance_query_info *performance_query =531 &job->performance_query;532 struct v3d_file_priv *v3d_priv = job->base.file->driver_priv;533 struct v3d_performance_query *perf_query =534 &performance_query->queries[query];535 struct v3d_dev *v3d = job->base.v3d;536 unsigned int i, j, offset;537 538 for (i = 0, offset = 0;539 i < performance_query->nperfmons;540 i++, offset += DRM_V3D_MAX_PERF_COUNTERS) {541 struct v3d_perfmon *perfmon;542 543 perfmon = v3d_perfmon_find(v3d_priv,544 perf_query->kperfmon_ids[i]);545 if (!perfmon) {546 DRM_DEBUG("Failed to find perfmon.");547 continue;548 }549 550 v3d_perfmon_stop(v3d, perfmon, true);551 552 if (job->copy.do_64bit) {553 for (j = 0; j < perfmon->ncounters; j++)554 write_to_buffer_64(data, offset + j,555 perfmon->values[j]);556 } else {557 for (j = 0; j < perfmon->ncounters; j++)558 write_to_buffer_32(data, offset + j,559 perfmon->values[j]);560 }561 562 v3d_perfmon_put(perfmon);563 }564}565 566static void567v3d_copy_performance_query(struct v3d_cpu_job *job)568{569 struct v3d_performance_query_info *performance_query = &job->performance_query;570 struct v3d_copy_query_results_info *copy = &job->copy;571 struct v3d_bo *bo = to_v3d_bo(job->base.bo[0]);572 struct dma_fence *fence;573 bool available, write_result;574 u8 *data;575 576 v3d_get_bo_vaddr(bo);577 578 data = ((u8 *)bo->vaddr) + copy->offset;579 580 for (int i = 0; i < performance_query->count; i++) {581 fence = drm_syncobj_fence_get(performance_query->queries[i].syncobj);582 available = fence ? dma_fence_is_signaled(fence) : false;583 584 write_result = available || copy->do_partial;585 if (write_result)586 v3d_write_performance_query_result(job, data, i);587 588 if (copy->availability_bit)589 write_to_buffer(data, performance_query->ncounters,590 copy->do_64bit, available ? 1u : 0u);591 592 data += copy->stride;593 594 dma_fence_put(fence);595 }596 597 v3d_put_bo_vaddr(bo);598}599 600static const v3d_cpu_job_fn cpu_job_function[] = {601 [V3D_CPU_JOB_TYPE_INDIRECT_CSD] = v3d_rewrite_csd_job_wg_counts_from_indirect,602 [V3D_CPU_JOB_TYPE_TIMESTAMP_QUERY] = v3d_timestamp_query,603 [V3D_CPU_JOB_TYPE_RESET_TIMESTAMP_QUERY] = v3d_reset_timestamp_queries,604 [V3D_CPU_JOB_TYPE_COPY_TIMESTAMP_QUERY] = v3d_copy_query_results,605 [V3D_CPU_JOB_TYPE_RESET_PERFORMANCE_QUERY] = v3d_reset_performance_queries,606 [V3D_CPU_JOB_TYPE_COPY_PERFORMANCE_QUERY] = v3d_copy_performance_query,607};608 609static struct dma_fence *610v3d_cpu_job_run(struct drm_sched_job *sched_job)611{612 struct v3d_cpu_job *job = to_cpu_job(sched_job);613 struct v3d_dev *v3d = job->base.v3d;614 615 v3d->cpu_job = job;616 617 if (job->job_type >= ARRAY_SIZE(cpu_job_function)) {618 DRM_DEBUG_DRIVER("Unknown CPU job: %d\n", job->job_type);619 return NULL;620 }621 622 v3d_job_start_stats(&job->base, V3D_CPU);623 trace_v3d_cpu_job_begin(&v3d->drm, job->job_type);624 625 cpu_job_function[job->job_type](job);626 627 trace_v3d_cpu_job_end(&v3d->drm, job->job_type);628 v3d_job_update_stats(&job->base, V3D_CPU);629 630 return NULL;631}632 633static struct dma_fence *634v3d_cache_clean_job_run(struct drm_sched_job *sched_job)635{636 struct v3d_job *job = to_v3d_job(sched_job);637 struct v3d_dev *v3d = job->v3d;638 639 v3d_job_start_stats(job, V3D_CACHE_CLEAN);640 641 v3d_clean_caches(v3d);642 643 v3d_job_update_stats(job, V3D_CACHE_CLEAN);644 645 return NULL;646}647 648static enum drm_gpu_sched_stat649v3d_gpu_reset_for_timeout(struct v3d_dev *v3d, struct drm_sched_job *sched_job)650{651 enum v3d_queue q;652 653 mutex_lock(&v3d->reset_lock);654 655 /* block scheduler */656 for (q = 0; q < V3D_MAX_QUEUES; q++)657 drm_sched_stop(&v3d->queue[q].sched, sched_job);658 659 if (sched_job)660 drm_sched_increase_karma(sched_job);661 662 /* get the GPU back into the init state */663 v3d_reset(v3d);664 665 for (q = 0; q < V3D_MAX_QUEUES; q++)666 drm_sched_resubmit_jobs(&v3d->queue[q].sched);667 668 /* Unblock schedulers and restart their jobs. */669 for (q = 0; q < V3D_MAX_QUEUES; q++) {670 drm_sched_start(&v3d->queue[q].sched);671 }672 673 mutex_unlock(&v3d->reset_lock);674 675 return DRM_GPU_SCHED_STAT_NOMINAL;676}677 678/* If the current address or return address have changed, then the GPU679 * has probably made progress and we should delay the reset. This680 * could fail if the GPU got in an infinite loop in the CL, but that681 * is pretty unlikely outside of an i-g-t testcase.682 */683static enum drm_gpu_sched_stat684v3d_cl_job_timedout(struct drm_sched_job *sched_job, enum v3d_queue q,685 u32 *timedout_ctca, u32 *timedout_ctra)686{687 struct v3d_job *job = to_v3d_job(sched_job);688 struct v3d_dev *v3d = job->v3d;689 u32 ctca = V3D_CORE_READ(0, V3D_CLE_CTNCA(q));690 u32 ctra = V3D_CORE_READ(0, V3D_CLE_CTNRA(q));691 692 if (*timedout_ctca != ctca || *timedout_ctra != ctra) {693 *timedout_ctca = ctca;694 *timedout_ctra = ctra;695 return DRM_GPU_SCHED_STAT_NOMINAL;696 }697 698 return v3d_gpu_reset_for_timeout(v3d, sched_job);699}700 701static enum drm_gpu_sched_stat702v3d_bin_job_timedout(struct drm_sched_job *sched_job)703{704 struct v3d_bin_job *job = to_bin_job(sched_job);705 706 return v3d_cl_job_timedout(sched_job, V3D_BIN,707 &job->timedout_ctca, &job->timedout_ctra);708}709 710static enum drm_gpu_sched_stat711v3d_render_job_timedout(struct drm_sched_job *sched_job)712{713 struct v3d_render_job *job = to_render_job(sched_job);714 715 return v3d_cl_job_timedout(sched_job, V3D_RENDER,716 &job->timedout_ctca, &job->timedout_ctra);717}718 719static enum drm_gpu_sched_stat720v3d_generic_job_timedout(struct drm_sched_job *sched_job)721{722 struct v3d_job *job = to_v3d_job(sched_job);723 724 return v3d_gpu_reset_for_timeout(job->v3d, sched_job);725}726 727static enum drm_gpu_sched_stat728v3d_csd_job_timedout(struct drm_sched_job *sched_job)729{730 struct v3d_csd_job *job = to_csd_job(sched_job);731 struct v3d_dev *v3d = job->base.v3d;732 u32 batches = V3D_CORE_READ(0, V3D_CSD_CURRENT_CFG4(v3d->ver));733 734 /* If we've made progress, skip reset and let the timer get735 * rearmed.736 */737 if (job->timedout_batches != batches) {738 job->timedout_batches = batches;739 return DRM_GPU_SCHED_STAT_NOMINAL;740 }741 742 return v3d_gpu_reset_for_timeout(v3d, sched_job);743}744 745static const struct drm_sched_backend_ops v3d_bin_sched_ops = {746 .run_job = v3d_bin_job_run,747 .timedout_job = v3d_bin_job_timedout,748 .free_job = v3d_sched_job_free,749};750 751static const struct drm_sched_backend_ops v3d_render_sched_ops = {752 .run_job = v3d_render_job_run,753 .timedout_job = v3d_render_job_timedout,754 .free_job = v3d_sched_job_free,755};756 757static const struct drm_sched_backend_ops v3d_tfu_sched_ops = {758 .run_job = v3d_tfu_job_run,759 .timedout_job = v3d_generic_job_timedout,760 .free_job = v3d_sched_job_free,761};762 763static const struct drm_sched_backend_ops v3d_csd_sched_ops = {764 .run_job = v3d_csd_job_run,765 .timedout_job = v3d_csd_job_timedout,766 .free_job = v3d_sched_job_free767};768 769static const struct drm_sched_backend_ops v3d_cache_clean_sched_ops = {770 .run_job = v3d_cache_clean_job_run,771 .timedout_job = v3d_generic_job_timedout,772 .free_job = v3d_sched_job_free773};774 775static const struct drm_sched_backend_ops v3d_cpu_sched_ops = {776 .run_job = v3d_cpu_job_run,777 .timedout_job = v3d_generic_job_timedout,778 .free_job = v3d_cpu_job_free779};780 781int782v3d_sched_init(struct v3d_dev *v3d)783{784 int hw_jobs_limit = 1;785 int job_hang_limit = 0;786 int hang_limit_ms = 500;787 int ret;788 789 ret = drm_sched_init(&v3d->queue[V3D_BIN].sched,790 &v3d_bin_sched_ops, NULL,791 DRM_SCHED_PRIORITY_COUNT,792 hw_jobs_limit, job_hang_limit,793 msecs_to_jiffies(hang_limit_ms), NULL,794 NULL, "v3d_bin", v3d->drm.dev);795 if (ret)796 return ret;797 798 ret = drm_sched_init(&v3d->queue[V3D_RENDER].sched,799 &v3d_render_sched_ops, NULL,800 DRM_SCHED_PRIORITY_COUNT,801 hw_jobs_limit, job_hang_limit,802 msecs_to_jiffies(hang_limit_ms), NULL,803 NULL, "v3d_render", v3d->drm.dev);804 if (ret)805 goto fail;806 807 ret = drm_sched_init(&v3d->queue[V3D_TFU].sched,808 &v3d_tfu_sched_ops, NULL,809 DRM_SCHED_PRIORITY_COUNT,810 hw_jobs_limit, job_hang_limit,811 msecs_to_jiffies(hang_limit_ms), NULL,812 NULL, "v3d_tfu", v3d->drm.dev);813 if (ret)814 goto fail;815 816 if (v3d_has_csd(v3d)) {817 ret = drm_sched_init(&v3d->queue[V3D_CSD].sched,818 &v3d_csd_sched_ops, NULL,819 DRM_SCHED_PRIORITY_COUNT,820 hw_jobs_limit, job_hang_limit,821 msecs_to_jiffies(hang_limit_ms), NULL,822 NULL, "v3d_csd", v3d->drm.dev);823 if (ret)824 goto fail;825 826 ret = drm_sched_init(&v3d->queue[V3D_CACHE_CLEAN].sched,827 &v3d_cache_clean_sched_ops, NULL,828 DRM_SCHED_PRIORITY_COUNT,829 hw_jobs_limit, job_hang_limit,830 msecs_to_jiffies(hang_limit_ms), NULL,831 NULL, "v3d_cache_clean", v3d->drm.dev);832 if (ret)833 goto fail;834 }835 836 ret = drm_sched_init(&v3d->queue[V3D_CPU].sched,837 &v3d_cpu_sched_ops, NULL,838 DRM_SCHED_PRIORITY_COUNT,839 1, job_hang_limit,840 msecs_to_jiffies(hang_limit_ms), NULL,841 NULL, "v3d_cpu", v3d->drm.dev);842 if (ret)843 goto fail;844 845 return 0;846 847fail:848 v3d_sched_fini(v3d);849 return ret;850}851 852void853v3d_sched_fini(struct v3d_dev *v3d)854{855 enum v3d_queue q;856 857 for (q = 0; q < V3D_MAX_QUEUES; q++) {858 if (v3d->queue[q].sched.ready)859 drm_sched_fini(&v3d->queue[q].sched);860 }861}862