1433 lines · c
1// SPDX-License-Identifier: GPL-2.0-only OR MIT2/* Copyright (c) 2023 Imagination Technologies Ltd. */3 4#include <drm/drm_managed.h>5#include <drm/gpu_scheduler.h>6 7#include "pvr_cccb.h"8#include "pvr_context.h"9#include "pvr_device.h"10#include "pvr_drv.h"11#include "pvr_job.h"12#include "pvr_queue.h"13#include "pvr_vm.h"14 15#include "pvr_rogue_fwif_client.h"16 17#define MAX_DEADLINE_MS 3000018 19#define CTX_COMPUTE_CCCB_SIZE_LOG2 1520#define CTX_FRAG_CCCB_SIZE_LOG2 1521#define CTX_GEOM_CCCB_SIZE_LOG2 1522#define CTX_TRANSFER_CCCB_SIZE_LOG2 1523 24static int get_xfer_ctx_state_size(struct pvr_device *pvr_dev)25{26 u32 num_isp_store_registers;27 28 if (PVR_HAS_FEATURE(pvr_dev, xe_memory_hierarchy)) {29 num_isp_store_registers = 1;30 } else {31 int err;32 33 err = PVR_FEATURE_VALUE(pvr_dev, num_isp_ipp_pipes, &num_isp_store_registers);34 if (WARN_ON(err))35 return err;36 }37 38 return sizeof(struct rogue_fwif_frag_ctx_state) +39 (num_isp_store_registers *40 sizeof(((struct rogue_fwif_frag_ctx_state *)0)->frag_reg_isp_store[0]));41}42 43static int get_frag_ctx_state_size(struct pvr_device *pvr_dev)44{45 u32 num_isp_store_registers;46 int err;47 48 if (PVR_HAS_FEATURE(pvr_dev, xe_memory_hierarchy)) {49 err = PVR_FEATURE_VALUE(pvr_dev, num_raster_pipes, &num_isp_store_registers);50 if (WARN_ON(err))51 return err;52 53 if (PVR_HAS_FEATURE(pvr_dev, gpu_multicore_support)) {54 u32 xpu_max_slaves;55 56 err = PVR_FEATURE_VALUE(pvr_dev, xpu_max_slaves, &xpu_max_slaves);57 if (WARN_ON(err))58 return err;59 60 num_isp_store_registers *= (1 + xpu_max_slaves);61 }62 } else {63 err = PVR_FEATURE_VALUE(pvr_dev, num_isp_ipp_pipes, &num_isp_store_registers);64 if (WARN_ON(err))65 return err;66 }67 68 return sizeof(struct rogue_fwif_frag_ctx_state) +69 (num_isp_store_registers *70 sizeof(((struct rogue_fwif_frag_ctx_state *)0)->frag_reg_isp_store[0]));71}72 73static int get_ctx_state_size(struct pvr_device *pvr_dev, enum drm_pvr_job_type type)74{75 switch (type) {76 case DRM_PVR_JOB_TYPE_GEOMETRY:77 return sizeof(struct rogue_fwif_geom_ctx_state);78 case DRM_PVR_JOB_TYPE_FRAGMENT:79 return get_frag_ctx_state_size(pvr_dev);80 case DRM_PVR_JOB_TYPE_COMPUTE:81 return sizeof(struct rogue_fwif_compute_ctx_state);82 case DRM_PVR_JOB_TYPE_TRANSFER_FRAG:83 return get_xfer_ctx_state_size(pvr_dev);84 }85 86 WARN(1, "Invalid queue type");87 return -EINVAL;88}89 90static u32 get_ctx_offset(enum drm_pvr_job_type type)91{92 switch (type) {93 case DRM_PVR_JOB_TYPE_GEOMETRY:94 return offsetof(struct rogue_fwif_fwrendercontext, geom_context);95 case DRM_PVR_JOB_TYPE_FRAGMENT:96 return offsetof(struct rogue_fwif_fwrendercontext, frag_context);97 case DRM_PVR_JOB_TYPE_COMPUTE:98 return offsetof(struct rogue_fwif_fwcomputecontext, cdm_context);99 case DRM_PVR_JOB_TYPE_TRANSFER_FRAG:100 return offsetof(struct rogue_fwif_fwtransfercontext, tq_context);101 }102 103 return 0;104}105 106static const char *107pvr_queue_fence_get_driver_name(struct dma_fence *f)108{109 return PVR_DRIVER_NAME;110}111 112static void pvr_queue_fence_release(struct dma_fence *f)113{114 struct pvr_queue_fence *fence = container_of(f, struct pvr_queue_fence, base);115 116 pvr_context_put(fence->queue->ctx);117 dma_fence_free(f);118}119 120static const char *121pvr_queue_job_fence_get_timeline_name(struct dma_fence *f)122{123 struct pvr_queue_fence *fence = container_of(f, struct pvr_queue_fence, base);124 125 switch (fence->queue->type) {126 case DRM_PVR_JOB_TYPE_GEOMETRY:127 return "geometry";128 129 case DRM_PVR_JOB_TYPE_FRAGMENT:130 return "fragment";131 132 case DRM_PVR_JOB_TYPE_COMPUTE:133 return "compute";134 135 case DRM_PVR_JOB_TYPE_TRANSFER_FRAG:136 return "transfer";137 }138 139 WARN(1, "Invalid queue type");140 return "invalid";141}142 143static const char *144pvr_queue_cccb_fence_get_timeline_name(struct dma_fence *f)145{146 struct pvr_queue_fence *fence = container_of(f, struct pvr_queue_fence, base);147 148 switch (fence->queue->type) {149 case DRM_PVR_JOB_TYPE_GEOMETRY:150 return "geometry-cccb";151 152 case DRM_PVR_JOB_TYPE_FRAGMENT:153 return "fragment-cccb";154 155 case DRM_PVR_JOB_TYPE_COMPUTE:156 return "compute-cccb";157 158 case DRM_PVR_JOB_TYPE_TRANSFER_FRAG:159 return "transfer-cccb";160 }161 162 WARN(1, "Invalid queue type");163 return "invalid";164}165 166static const struct dma_fence_ops pvr_queue_job_fence_ops = {167 .get_driver_name = pvr_queue_fence_get_driver_name,168 .get_timeline_name = pvr_queue_job_fence_get_timeline_name,169 .release = pvr_queue_fence_release,170};171 172/**173 * to_pvr_queue_job_fence() - Return a pvr_queue_fence object if the fence is174 * backed by a UFO.175 * @f: The dma_fence to turn into a pvr_queue_fence.176 *177 * Return:178 * * A non-NULL pvr_queue_fence object if the dma_fence is backed by a UFO, or179 * * NULL otherwise.180 */181static struct pvr_queue_fence *182to_pvr_queue_job_fence(struct dma_fence *f)183{184 struct drm_sched_fence *sched_fence = to_drm_sched_fence(f);185 186 if (sched_fence)187 f = sched_fence->parent;188 189 if (f && f->ops == &pvr_queue_job_fence_ops)190 return container_of(f, struct pvr_queue_fence, base);191 192 return NULL;193}194 195static const struct dma_fence_ops pvr_queue_cccb_fence_ops = {196 .get_driver_name = pvr_queue_fence_get_driver_name,197 .get_timeline_name = pvr_queue_cccb_fence_get_timeline_name,198 .release = pvr_queue_fence_release,199};200 201/**202 * pvr_queue_fence_put() - Put wrapper for pvr_queue_fence objects.203 * @f: The dma_fence object to put.204 *205 * If the pvr_queue_fence has been initialized, we call dma_fence_put(),206 * otherwise we free the object with dma_fence_free(). This allows us207 * to do the right thing before and after pvr_queue_fence_init() had been208 * called.209 */210static void pvr_queue_fence_put(struct dma_fence *f)211{212 if (!f)213 return;214 215 if (WARN_ON(f->ops &&216 f->ops != &pvr_queue_cccb_fence_ops &&217 f->ops != &pvr_queue_job_fence_ops))218 return;219 220 /* If the fence hasn't been initialized yet, free the object directly. */221 if (f->ops)222 dma_fence_put(f);223 else224 dma_fence_free(f);225}226 227/**228 * pvr_queue_fence_alloc() - Allocate a pvr_queue_fence fence object229 *230 * Call this function to allocate job CCCB and done fences. This only231 * allocates the objects. Initialization happens when the underlying232 * dma_fence object is to be returned to drm_sched (in prepare_job() or233 * run_job()).234 *235 * Return:236 * * A valid pointer if the allocation succeeds, or237 * * NULL if the allocation fails.238 */239static struct dma_fence *240pvr_queue_fence_alloc(void)241{242 struct pvr_queue_fence *fence;243 244 fence = kzalloc(sizeof(*fence), GFP_KERNEL);245 if (!fence)246 return NULL;247 248 return &fence->base;249}250 251/**252 * pvr_queue_fence_init() - Initializes a pvr_queue_fence object.253 * @f: The fence to initialize254 * @queue: The queue this fence belongs to.255 * @fence_ops: The fence operations.256 * @fence_ctx: The fence context.257 *258 * Wrapper around dma_fence_init() that takes care of initializing the259 * pvr_queue_fence::queue field too.260 */261static void262pvr_queue_fence_init(struct dma_fence *f,263 struct pvr_queue *queue,264 const struct dma_fence_ops *fence_ops,265 struct pvr_queue_fence_ctx *fence_ctx)266{267 struct pvr_queue_fence *fence = container_of(f, struct pvr_queue_fence, base);268 269 pvr_context_get(queue->ctx);270 fence->queue = queue;271 dma_fence_init(&fence->base, fence_ops,272 &fence_ctx->lock, fence_ctx->id,273 atomic_inc_return(&fence_ctx->seqno));274}275 276/**277 * pvr_queue_cccb_fence_init() - Initializes a CCCB fence object.278 * @fence: The fence to initialize.279 * @queue: The queue this fence belongs to.280 *281 * Initializes a fence that can be used to wait for CCCB space.282 *283 * Should be called in the ::prepare_job() path, so the fence returned to284 * drm_sched is valid.285 */286static void287pvr_queue_cccb_fence_init(struct dma_fence *fence, struct pvr_queue *queue)288{289 pvr_queue_fence_init(fence, queue, &pvr_queue_cccb_fence_ops,290 &queue->cccb_fence_ctx.base);291}292 293/**294 * pvr_queue_job_fence_init() - Initializes a job done fence object.295 * @fence: The fence to initialize.296 * @queue: The queue this fence belongs to.297 *298 * Initializes a fence that will be signaled when the GPU is done executing299 * a job.300 *301 * Should be called *before* the ::run_job() path, so the fence is initialised302 * before being placed in the pending_list.303 */304static void305pvr_queue_job_fence_init(struct dma_fence *fence, struct pvr_queue *queue)306{307 pvr_queue_fence_init(fence, queue, &pvr_queue_job_fence_ops,308 &queue->job_fence_ctx);309}310 311/**312 * pvr_queue_fence_ctx_init() - Queue fence context initialization.313 * @fence_ctx: The context to initialize314 */315static void316pvr_queue_fence_ctx_init(struct pvr_queue_fence_ctx *fence_ctx)317{318 spin_lock_init(&fence_ctx->lock);319 fence_ctx->id = dma_fence_context_alloc(1);320 atomic_set(&fence_ctx->seqno, 0);321}322 323static u32 ufo_cmds_size(u32 elem_count)324{325 /* We can pass at most ROGUE_FWIF_CCB_CMD_MAX_UFOS per UFO-related command. */326 u32 full_cmd_count = elem_count / ROGUE_FWIF_CCB_CMD_MAX_UFOS;327 u32 remaining_elems = elem_count % ROGUE_FWIF_CCB_CMD_MAX_UFOS;328 u32 size = full_cmd_count *329 pvr_cccb_get_size_of_cmd_with_hdr(ROGUE_FWIF_CCB_CMD_MAX_UFOS *330 sizeof(struct rogue_fwif_ufo));331 332 if (remaining_elems) {333 size += pvr_cccb_get_size_of_cmd_with_hdr(remaining_elems *334 sizeof(struct rogue_fwif_ufo));335 }336 337 return size;338}339 340static u32 job_cmds_size(struct pvr_job *job, u32 ufo_wait_count)341{342 /* One UFO cmd for the fence signaling, one UFO cmd per native fence native,343 * and a command for the job itself.344 */345 return ufo_cmds_size(1) + ufo_cmds_size(ufo_wait_count) +346 pvr_cccb_get_size_of_cmd_with_hdr(job->cmd_len);347}348 349/**350 * job_count_remaining_native_deps() - Count the number of non-signaled native dependencies.351 * @job: Job to operate on.352 *353 * Returns: Number of non-signaled native deps remaining.354 */355static unsigned long job_count_remaining_native_deps(struct pvr_job *job)356{357 unsigned long remaining_count = 0;358 struct dma_fence *fence = NULL;359 unsigned long index;360 361 xa_for_each(&job->base.dependencies, index, fence) {362 struct pvr_queue_fence *jfence;363 364 jfence = to_pvr_queue_job_fence(fence);365 if (!jfence)366 continue;367 368 if (!dma_fence_is_signaled(&jfence->base))369 remaining_count++;370 }371 372 return remaining_count;373}374 375/**376 * pvr_queue_get_job_cccb_fence() - Get the CCCB fence attached to a job.377 * @queue: The queue this job will be submitted to.378 * @job: The job to get the CCCB fence on.379 *380 * The CCCB fence is a synchronization primitive allowing us to delay job381 * submission until there's enough space in the CCCB to submit the job.382 *383 * Return:384 * * NULL if there's enough space in the CCCB to submit this job, or385 * * A valid dma_fence object otherwise.386 */387static struct dma_fence *388pvr_queue_get_job_cccb_fence(struct pvr_queue *queue, struct pvr_job *job)389{390 struct pvr_queue_fence *cccb_fence;391 unsigned int native_deps_remaining;392 393 /* If the fence is NULL, that means we already checked that we had394 * enough space in the cccb for our job.395 */396 if (!job->cccb_fence)397 return NULL;398 399 mutex_lock(&queue->cccb_fence_ctx.job_lock);400 401 /* Count remaining native dependencies and check if the job fits in the CCCB. */402 native_deps_remaining = job_count_remaining_native_deps(job);403 if (pvr_cccb_cmdseq_fits(&queue->cccb, job_cmds_size(job, native_deps_remaining))) {404 pvr_queue_fence_put(job->cccb_fence);405 job->cccb_fence = NULL;406 goto out_unlock;407 }408 409 /* There should be no job attached to the CCCB fence context:410 * drm_sched_entity guarantees that jobs are submitted one at a time.411 */412 if (WARN_ON(queue->cccb_fence_ctx.job))413 pvr_job_put(queue->cccb_fence_ctx.job);414 415 queue->cccb_fence_ctx.job = pvr_job_get(job);416 417 /* Initialize the fence before returning it. */418 cccb_fence = container_of(job->cccb_fence, struct pvr_queue_fence, base);419 if (!WARN_ON(cccb_fence->queue))420 pvr_queue_cccb_fence_init(job->cccb_fence, queue);421 422out_unlock:423 mutex_unlock(&queue->cccb_fence_ctx.job_lock);424 425 return dma_fence_get(job->cccb_fence);426}427 428/**429 * pvr_queue_get_job_kccb_fence() - Get the KCCB fence attached to a job.430 * @queue: The queue this job will be submitted to.431 * @job: The job to get the KCCB fence on.432 *433 * The KCCB fence is a synchronization primitive allowing us to delay job434 * submission until there's enough space in the KCCB to submit the job.435 *436 * Return:437 * * NULL if there's enough space in the KCCB to submit this job, or438 * * A valid dma_fence object otherwise.439 */440static struct dma_fence *441pvr_queue_get_job_kccb_fence(struct pvr_queue *queue, struct pvr_job *job)442{443 struct pvr_device *pvr_dev = queue->ctx->pvr_dev;444 struct dma_fence *kccb_fence = NULL;445 446 /* If the fence is NULL, that means we already checked that we had447 * enough space in the KCCB for our job.448 */449 if (!job->kccb_fence)450 return NULL;451 452 if (!WARN_ON(job->kccb_fence->ops)) {453 kccb_fence = pvr_kccb_reserve_slot(pvr_dev, job->kccb_fence);454 job->kccb_fence = NULL;455 }456 457 return kccb_fence;458}459 460static struct dma_fence *461pvr_queue_get_paired_frag_job_dep(struct pvr_queue *queue, struct pvr_job *job)462{463 struct pvr_job *frag_job = job->type == DRM_PVR_JOB_TYPE_GEOMETRY ?464 job->paired_job : NULL;465 struct dma_fence *f;466 unsigned long index;467 468 if (!frag_job)469 return NULL;470 471 xa_for_each(&frag_job->base.dependencies, index, f) {472 /* Skip already signaled fences. */473 if (dma_fence_is_signaled(f))474 continue;475 476 /* Skip our own fence. */477 if (f == &job->base.s_fence->scheduled)478 continue;479 480 return dma_fence_get(f);481 }482 483 return frag_job->base.sched->ops->prepare_job(&frag_job->base, &queue->entity);484}485 486/**487 * pvr_queue_prepare_job() - Return the next internal dependencies expressed as a dma_fence.488 * @sched_job: The job to query the next internal dependency on489 * @s_entity: The entity this job is queue on.490 *491 * After iterating over drm_sched_job::dependencies, drm_sched let the driver return492 * its own internal dependencies. We use this function to return our internal dependencies.493 */494static struct dma_fence *495pvr_queue_prepare_job(struct drm_sched_job *sched_job,496 struct drm_sched_entity *s_entity)497{498 struct pvr_job *job = container_of(sched_job, struct pvr_job, base);499 struct pvr_queue *queue = container_of(s_entity, struct pvr_queue, entity);500 struct dma_fence *internal_dep = NULL;501 502 /*503 * Initialize the done_fence, so we can signal it. This must be done504 * here because otherwise by the time of run_job() the job will end up505 * in the pending list without a valid fence.506 */507 if (job->type == DRM_PVR_JOB_TYPE_FRAGMENT && job->paired_job) {508 /*509 * This will be called on a paired fragment job after being510 * submitted to firmware. We can tell if this is the case and511 * bail early from whether run_job() has been called on the512 * geometry job, which would issue a pm ref.513 */514 if (job->paired_job->has_pm_ref)515 return NULL;516 517 /*518 * In this case we need to use the job's own ctx to initialise519 * the done_fence. The other steps are done in the ctx of the520 * paired geometry job.521 */522 pvr_queue_job_fence_init(job->done_fence,523 job->ctx->queues.fragment);524 } else {525 pvr_queue_job_fence_init(job->done_fence, queue);526 }527 528 /* CCCB fence is used to make sure we have enough space in the CCCB to529 * submit our commands.530 */531 internal_dep = pvr_queue_get_job_cccb_fence(queue, job);532 533 /* KCCB fence is used to make sure we have a KCCB slot to queue our534 * CMD_KICK.535 */536 if (!internal_dep)537 internal_dep = pvr_queue_get_job_kccb_fence(queue, job);538 539 /* Any extra internal dependency should be added here, using the following540 * pattern:541 *542 * if (!internal_dep)543 * internal_dep = pvr_queue_get_job_xxxx_fence(queue, job);544 */545 546 /* The paired job fence should come last, when everything else is ready. */547 if (!internal_dep)548 internal_dep = pvr_queue_get_paired_frag_job_dep(queue, job);549 550 return internal_dep;551}552 553/**554 * pvr_queue_update_active_state_locked() - Update the queue active state.555 * @queue: Queue to update the state on.556 *557 * Locked version of pvr_queue_update_active_state(). Must be called with558 * pvr_device::queue::lock held.559 */560static void pvr_queue_update_active_state_locked(struct pvr_queue *queue)561{562 struct pvr_device *pvr_dev = queue->ctx->pvr_dev;563 564 lockdep_assert_held(&pvr_dev->queues.lock);565 566 /* The queue is temporary out of any list when it's being reset,567 * we don't want a call to pvr_queue_update_active_state_locked()568 * to re-insert it behind our back.569 */570 if (list_empty(&queue->node))571 return;572 573 if (!atomic_read(&queue->in_flight_job_count))574 list_move_tail(&queue->node, &pvr_dev->queues.idle);575 else576 list_move_tail(&queue->node, &pvr_dev->queues.active);577}578 579/**580 * pvr_queue_update_active_state() - Update the queue active state.581 * @queue: Queue to update the state on.582 *583 * Active state is based on the in_flight_job_count value.584 *585 * Updating the active state implies moving the queue in or out of the586 * active queue list, which also defines whether the queue is checked587 * or not when a FW event is received.588 *589 * This function should be called any time a job is submitted or it done590 * fence is signaled.591 */592static void pvr_queue_update_active_state(struct pvr_queue *queue)593{594 struct pvr_device *pvr_dev = queue->ctx->pvr_dev;595 596 mutex_lock(&pvr_dev->queues.lock);597 pvr_queue_update_active_state_locked(queue);598 mutex_unlock(&pvr_dev->queues.lock);599}600 601static void pvr_queue_submit_job_to_cccb(struct pvr_job *job)602{603 struct pvr_queue *queue = container_of(job->base.sched, struct pvr_queue, scheduler);604 struct rogue_fwif_ufo ufos[ROGUE_FWIF_CCB_CMD_MAX_UFOS];605 struct pvr_cccb *cccb = &queue->cccb;606 struct pvr_queue_fence *jfence;607 struct dma_fence *fence;608 unsigned long index;609 u32 ufo_count = 0;610 611 /* We need to add the queue to the active list before updating the CCCB,612 * otherwise we might miss the FW event informing us that something613 * happened on this queue.614 */615 atomic_inc(&queue->in_flight_job_count);616 pvr_queue_update_active_state(queue);617 618 xa_for_each(&job->base.dependencies, index, fence) {619 jfence = to_pvr_queue_job_fence(fence);620 if (!jfence)621 continue;622 623 /* Skip the partial render fence, we will place it at the end. */624 if (job->type == DRM_PVR_JOB_TYPE_FRAGMENT && job->paired_job &&625 &job->paired_job->base.s_fence->scheduled == fence)626 continue;627 628 if (dma_fence_is_signaled(&jfence->base))629 continue;630 631 pvr_fw_object_get_fw_addr(jfence->queue->timeline_ufo.fw_obj,632 &ufos[ufo_count].addr);633 ufos[ufo_count++].value = jfence->base.seqno;634 635 if (ufo_count == ARRAY_SIZE(ufos)) {636 pvr_cccb_write_command_with_header(cccb, ROGUE_FWIF_CCB_CMD_TYPE_FENCE_PR,637 sizeof(ufos), ufos, 0, 0);638 ufo_count = 0;639 }640 }641 642 /* Partial render fence goes last. */643 if (job->type == DRM_PVR_JOB_TYPE_FRAGMENT && job->paired_job) {644 jfence = to_pvr_queue_job_fence(job->paired_job->done_fence);645 if (!WARN_ON(!jfence)) {646 pvr_fw_object_get_fw_addr(jfence->queue->timeline_ufo.fw_obj,647 &ufos[ufo_count].addr);648 ufos[ufo_count++].value = job->paired_job->done_fence->seqno;649 }650 }651 652 if (ufo_count) {653 pvr_cccb_write_command_with_header(cccb, ROGUE_FWIF_CCB_CMD_TYPE_FENCE_PR,654 sizeof(ufos[0]) * ufo_count, ufos, 0, 0);655 }656 657 if (job->type == DRM_PVR_JOB_TYPE_GEOMETRY && job->paired_job) {658 struct rogue_fwif_cmd_geom *cmd = job->cmd;659 660 /* Reference value for the partial render test is the current queue fence661 * seqno minus one.662 */663 pvr_fw_object_get_fw_addr(queue->timeline_ufo.fw_obj,664 &cmd->partial_render_geom_frag_fence.addr);665 cmd->partial_render_geom_frag_fence.value = job->done_fence->seqno - 1;666 }667 668 /* Submit job to FW */669 pvr_cccb_write_command_with_header(cccb, job->fw_ccb_cmd_type, job->cmd_len, job->cmd,670 job->id, job->id);671 672 /* Signal the job fence. */673 pvr_fw_object_get_fw_addr(queue->timeline_ufo.fw_obj, &ufos[0].addr);674 ufos[0].value = job->done_fence->seqno;675 pvr_cccb_write_command_with_header(cccb, ROGUE_FWIF_CCB_CMD_TYPE_UPDATE,676 sizeof(ufos[0]), ufos, 0, 0);677}678 679/**680 * pvr_queue_run_job() - Submit a job to the FW.681 * @sched_job: The job to submit.682 *683 * This function is called when all non-native dependencies have been met and684 * when the commands resulting from this job are guaranteed to fit in the CCCB.685 */686static struct dma_fence *pvr_queue_run_job(struct drm_sched_job *sched_job)687{688 struct pvr_job *job = container_of(sched_job, struct pvr_job, base);689 struct pvr_device *pvr_dev = job->pvr_dev;690 int err;691 692 /* The fragment job is issued along the geometry job when we use combined693 * geom+frag kicks. When we get there, we should simply return the694 * done_fence that's been initialized earlier.695 */696 if (job->paired_job && job->type == DRM_PVR_JOB_TYPE_FRAGMENT &&697 job->done_fence->ops) {698 return dma_fence_get(job->done_fence);699 }700 701 /* The only kind of jobs that can be paired are geometry and fragment, and702 * we bail out early if we see a fragment job that's paired with a geomtry703 * job.704 * Paired jobs must also target the same context and point to the same705 * HWRT.706 */707 if (WARN_ON(job->paired_job &&708 (job->type != DRM_PVR_JOB_TYPE_GEOMETRY ||709 job->paired_job->type != DRM_PVR_JOB_TYPE_FRAGMENT ||710 job->hwrt != job->paired_job->hwrt ||711 job->ctx != job->paired_job->ctx)))712 return ERR_PTR(-EINVAL);713 714 err = pvr_job_get_pm_ref(job);715 if (WARN_ON(err))716 return ERR_PTR(err);717 718 if (job->paired_job) {719 err = pvr_job_get_pm_ref(job->paired_job);720 if (WARN_ON(err))721 return ERR_PTR(err);722 }723 724 /* Submit our job to the CCCB */725 pvr_queue_submit_job_to_cccb(job);726 727 if (job->paired_job) {728 struct pvr_job *geom_job = job;729 struct pvr_job *frag_job = job->paired_job;730 struct pvr_queue *geom_queue = job->ctx->queues.geometry;731 struct pvr_queue *frag_queue = job->ctx->queues.fragment;732 733 /* Submit the fragment job along the geometry job and send a combined kick. */734 pvr_queue_submit_job_to_cccb(frag_job);735 pvr_cccb_send_kccb_combined_kick(pvr_dev,736 &geom_queue->cccb, &frag_queue->cccb,737 pvr_context_get_fw_addr(geom_job->ctx) +738 geom_queue->ctx_offset,739 pvr_context_get_fw_addr(frag_job->ctx) +740 frag_queue->ctx_offset,741 job->hwrt,742 frag_job->fw_ccb_cmd_type ==743 ROGUE_FWIF_CCB_CMD_TYPE_FRAG_PR);744 } else {745 struct pvr_queue *queue = container_of(job->base.sched,746 struct pvr_queue, scheduler);747 748 pvr_cccb_send_kccb_kick(pvr_dev, &queue->cccb,749 pvr_context_get_fw_addr(job->ctx) + queue->ctx_offset,750 job->hwrt);751 }752 753 return dma_fence_get(job->done_fence);754}755 756static void pvr_queue_stop(struct pvr_queue *queue, struct pvr_job *bad_job)757{758 drm_sched_stop(&queue->scheduler, bad_job ? &bad_job->base : NULL);759}760 761static void pvr_queue_start(struct pvr_queue *queue)762{763 struct pvr_job *job;764 765 /* Make sure we CPU-signal the UFO object, so other queues don't get766 * blocked waiting on it.767 */768 *queue->timeline_ufo.value = atomic_read(&queue->job_fence_ctx.seqno);769 770 list_for_each_entry(job, &queue->scheduler.pending_list, base.list) {771 if (dma_fence_is_signaled(job->done_fence)) {772 /* Jobs might have completed after drm_sched_stop() was called.773 * In that case, re-assign the parent field to the done_fence.774 */775 WARN_ON(job->base.s_fence->parent);776 job->base.s_fence->parent = dma_fence_get(job->done_fence);777 } else {778 /* If we had unfinished jobs, flag the entity as guilty so no779 * new job can be submitted.780 */781 atomic_set(&queue->ctx->faulty, 1);782 }783 }784 785 drm_sched_start(&queue->scheduler);786}787 788/**789 * pvr_queue_timedout_job() - Handle a job timeout event.790 * @s_job: The job this timeout occurred on.791 *792 * FIXME: We don't do anything here to unblock the situation, we just stop+start793 * the scheduler, and re-assign parent fences in the middle.794 *795 * Return:796 * * DRM_GPU_SCHED_STAT_NOMINAL.797 */798static enum drm_gpu_sched_stat799pvr_queue_timedout_job(struct drm_sched_job *s_job)800{801 struct drm_gpu_scheduler *sched = s_job->sched;802 struct pvr_queue *queue = container_of(sched, struct pvr_queue, scheduler);803 struct pvr_device *pvr_dev = queue->ctx->pvr_dev;804 struct pvr_job *job;805 u32 job_count = 0;806 807 dev_err(sched->dev, "Job timeout\n");808 809 /* Before we stop the scheduler, make sure the queue is out of any list, so810 * any call to pvr_queue_update_active_state_locked() that might happen811 * until the scheduler is really stopped doesn't end up re-inserting the812 * queue in the active list. This would cause813 * pvr_queue_signal_done_fences() and drm_sched_stop() to race with each814 * other when accessing the pending_list, since drm_sched_stop() doesn't815 * grab the job_list_lock when modifying the list (it's assuming the816 * only other accessor is the scheduler, and it's safe to not grab the817 * lock since it's stopped).818 */819 mutex_lock(&pvr_dev->queues.lock);820 list_del_init(&queue->node);821 mutex_unlock(&pvr_dev->queues.lock);822 823 drm_sched_stop(sched, s_job);824 825 /* Re-assign job parent fences. */826 list_for_each_entry(job, &sched->pending_list, base.list) {827 job->base.s_fence->parent = dma_fence_get(job->done_fence);828 job_count++;829 }830 WARN_ON(atomic_read(&queue->in_flight_job_count) != job_count);831 832 /* Re-insert the queue in the proper list, and kick a queue processing833 * operation if there were jobs pending.834 */835 mutex_lock(&pvr_dev->queues.lock);836 if (!job_count) {837 list_move_tail(&queue->node, &pvr_dev->queues.idle);838 } else {839 atomic_set(&queue->in_flight_job_count, job_count);840 list_move_tail(&queue->node, &pvr_dev->queues.active);841 pvr_queue_process(queue);842 }843 mutex_unlock(&pvr_dev->queues.lock);844 845 drm_sched_start(sched);846 847 return DRM_GPU_SCHED_STAT_NOMINAL;848}849 850/**851 * pvr_queue_free_job() - Release the reference the scheduler had on a job object.852 * @sched_job: Job object to free.853 */854static void pvr_queue_free_job(struct drm_sched_job *sched_job)855{856 struct pvr_job *job = container_of(sched_job, struct pvr_job, base);857 858 drm_sched_job_cleanup(sched_job);859 job->paired_job = NULL;860 pvr_job_put(job);861}862 863static const struct drm_sched_backend_ops pvr_queue_sched_ops = {864 .prepare_job = pvr_queue_prepare_job,865 .run_job = pvr_queue_run_job,866 .timedout_job = pvr_queue_timedout_job,867 .free_job = pvr_queue_free_job,868};869 870/**871 * pvr_queue_fence_is_ufo_backed() - Check if a dma_fence is backed by a UFO object872 * @f: Fence to test.873 *874 * A UFO-backed fence is a fence that can be signaled or waited upon FW-side.875 * pvr_job::done_fence objects are backed by the timeline UFO attached to the queue876 * they are pushed to, but those fences are not directly exposed to the outside877 * world, so we also need to check if the fence we're being passed is a878 * drm_sched_fence that was coming from our driver.879 */880bool pvr_queue_fence_is_ufo_backed(struct dma_fence *f)881{882 struct drm_sched_fence *sched_fence = f ? to_drm_sched_fence(f) : NULL;883 884 if (sched_fence &&885 sched_fence->sched->ops == &pvr_queue_sched_ops)886 return true;887 888 if (f && f->ops == &pvr_queue_job_fence_ops)889 return true;890 891 return false;892}893 894/**895 * pvr_queue_signal_done_fences() - Signal done fences.896 * @queue: Queue to check.897 *898 * Signal done fences of jobs whose seqno is less than the current value of899 * the UFO object attached to the queue.900 */901static void902pvr_queue_signal_done_fences(struct pvr_queue *queue)903{904 struct pvr_job *job, *tmp_job;905 u32 cur_seqno;906 907 spin_lock(&queue->scheduler.job_list_lock);908 cur_seqno = *queue->timeline_ufo.value;909 list_for_each_entry_safe(job, tmp_job, &queue->scheduler.pending_list, base.list) {910 if ((int)(cur_seqno - lower_32_bits(job->done_fence->seqno)) < 0)911 break;912 913 if (!dma_fence_is_signaled(job->done_fence)) {914 dma_fence_signal(job->done_fence);915 pvr_job_release_pm_ref(job);916 atomic_dec(&queue->in_flight_job_count);917 }918 }919 spin_unlock(&queue->scheduler.job_list_lock);920}921 922/**923 * pvr_queue_check_job_waiting_for_cccb_space() - Check if the job waiting for CCCB space924 * can be unblocked925 * pushed to the CCCB926 * @queue: Queue to check927 *928 * If we have a job waiting for CCCB, and this job now fits in the CCCB, we signal929 * its CCCB fence, which should kick drm_sched.930 */931static void932pvr_queue_check_job_waiting_for_cccb_space(struct pvr_queue *queue)933{934 struct pvr_queue_fence *cccb_fence;935 u32 native_deps_remaining;936 struct pvr_job *job;937 938 mutex_lock(&queue->cccb_fence_ctx.job_lock);939 job = queue->cccb_fence_ctx.job;940 if (!job)941 goto out_unlock;942 943 /* If we have a job attached to the CCCB fence context, its CCCB fence944 * shouldn't be NULL.945 */946 if (WARN_ON(!job->cccb_fence)) {947 job = NULL;948 goto out_unlock;949 }950 951 /* If we get there, CCCB fence has to be initialized. */952 cccb_fence = container_of(job->cccb_fence, struct pvr_queue_fence, base);953 if (WARN_ON(!cccb_fence->queue)) {954 job = NULL;955 goto out_unlock;956 }957 958 /* Evict signaled dependencies before checking for CCCB space.959 * If the job fits, signal the CCCB fence, this should unblock960 * the drm_sched_entity.961 */962 native_deps_remaining = job_count_remaining_native_deps(job);963 if (!pvr_cccb_cmdseq_fits(&queue->cccb, job_cmds_size(job, native_deps_remaining))) {964 job = NULL;965 goto out_unlock;966 }967 968 dma_fence_signal(job->cccb_fence);969 pvr_queue_fence_put(job->cccb_fence);970 job->cccb_fence = NULL;971 queue->cccb_fence_ctx.job = NULL;972 973out_unlock:974 mutex_unlock(&queue->cccb_fence_ctx.job_lock);975 976 pvr_job_put(job);977}978 979/**980 * pvr_queue_process() - Process events that happened on a queue.981 * @queue: Queue to check982 *983 * Signal job fences and check if jobs waiting for CCCB space can be unblocked.984 */985void pvr_queue_process(struct pvr_queue *queue)986{987 lockdep_assert_held(&queue->ctx->pvr_dev->queues.lock);988 989 pvr_queue_check_job_waiting_for_cccb_space(queue);990 pvr_queue_signal_done_fences(queue);991 pvr_queue_update_active_state_locked(queue);992}993 994static u32 get_dm_type(struct pvr_queue *queue)995{996 switch (queue->type) {997 case DRM_PVR_JOB_TYPE_GEOMETRY:998 return PVR_FWIF_DM_GEOM;999 case DRM_PVR_JOB_TYPE_TRANSFER_FRAG:1000 case DRM_PVR_JOB_TYPE_FRAGMENT:1001 return PVR_FWIF_DM_FRAG;1002 case DRM_PVR_JOB_TYPE_COMPUTE:1003 return PVR_FWIF_DM_CDM;1004 }1005 1006 return ~0;1007}1008 1009/**1010 * init_fw_context() - Initializes the queue part of a FW context.1011 * @queue: Queue object to initialize the FW context for.1012 * @fw_ctx_map: The FW context CPU mapping.1013 *1014 * FW contexts are containing various states, one of them being a per-queue state1015 * that needs to be initialized for each queue being exposed by a context. This1016 * function takes care of that.1017 */1018static void init_fw_context(struct pvr_queue *queue, void *fw_ctx_map)1019{1020 struct pvr_context *ctx = queue->ctx;1021 struct pvr_fw_object *fw_mem_ctx_obj = pvr_vm_get_fw_mem_context(ctx->vm_ctx);1022 struct rogue_fwif_fwcommoncontext *cctx_fw;1023 struct pvr_cccb *cccb = &queue->cccb;1024 1025 cctx_fw = fw_ctx_map + queue->ctx_offset;1026 cctx_fw->ccbctl_fw_addr = cccb->ctrl_fw_addr;1027 cctx_fw->ccb_fw_addr = cccb->cccb_fw_addr;1028 1029 cctx_fw->dm = get_dm_type(queue);1030 cctx_fw->priority = ctx->priority;1031 cctx_fw->priority_seq_num = 0;1032 cctx_fw->max_deadline_ms = MAX_DEADLINE_MS;1033 cctx_fw->pid = task_tgid_nr(current);1034 cctx_fw->server_common_context_id = ctx->ctx_id;1035 1036 pvr_fw_object_get_fw_addr(fw_mem_ctx_obj, &cctx_fw->fw_mem_context_fw_addr);1037 1038 pvr_fw_object_get_fw_addr(queue->reg_state_obj, &cctx_fw->context_state_addr);1039}1040 1041/**1042 * pvr_queue_cleanup_fw_context() - Wait for the FW context to be idle and clean it up.1043 * @queue: Queue on FW context to clean up.1044 *1045 * Return:1046 * * 0 on success,1047 * * Any error returned by pvr_fw_structure_cleanup() otherwise.1048 */1049static int pvr_queue_cleanup_fw_context(struct pvr_queue *queue)1050{1051 if (!queue->ctx->fw_obj)1052 return 0;1053 1054 return pvr_fw_structure_cleanup(queue->ctx->pvr_dev,1055 ROGUE_FWIF_CLEANUP_FWCOMMONCONTEXT,1056 queue->ctx->fw_obj, queue->ctx_offset);1057}1058 1059/**1060 * pvr_queue_job_init() - Initialize queue related fields in a pvr_job object.1061 * @job: The job to initialize.1062 *1063 * Bind the job to a queue and allocate memory to guarantee pvr_queue_job_arm()1064 * and pvr_queue_job_push() can't fail. We also make sure the context type is1065 * valid and the job can fit in the CCCB.1066 *1067 * Return:1068 * * 0 on success, or1069 * * An error code if something failed.1070 */1071int pvr_queue_job_init(struct pvr_job *job)1072{1073 /* Fragment jobs need at least one native fence wait on the geometry job fence. */1074 u32 min_native_dep_count = job->type == DRM_PVR_JOB_TYPE_FRAGMENT ? 1 : 0;1075 struct pvr_queue *queue;1076 int err;1077 1078 if (atomic_read(&job->ctx->faulty))1079 return -EIO;1080 1081 queue = pvr_context_get_queue_for_job(job->ctx, job->type);1082 if (!queue)1083 return -EINVAL;1084 1085 if (!pvr_cccb_cmdseq_can_fit(&queue->cccb, job_cmds_size(job, min_native_dep_count)))1086 return -E2BIG;1087 1088 err = drm_sched_job_init(&job->base, &queue->entity, 1, THIS_MODULE);1089 if (err)1090 return err;1091 1092 job->cccb_fence = pvr_queue_fence_alloc();1093 job->kccb_fence = pvr_kccb_fence_alloc();1094 job->done_fence = pvr_queue_fence_alloc();1095 if (!job->cccb_fence || !job->kccb_fence || !job->done_fence)1096 return -ENOMEM;1097 1098 return 0;1099}1100 1101/**1102 * pvr_queue_job_arm() - Arm a job object.1103 * @job: The job to arm.1104 *1105 * Initializes fences and return the drm_sched finished fence so it can1106 * be exposed to the outside world. Once this function is called, you should1107 * make sure the job is pushed using pvr_queue_job_push(), or guarantee that1108 * no one grabbed a reference to the returned fence. The latter can happen if1109 * we do multi-job submission, and something failed when creating/initializing1110 * a job. In that case, we know the fence didn't leave the driver, and we1111 * can thus guarantee nobody will wait on an dead fence object.1112 *1113 * Return:1114 * * A dma_fence object.1115 */1116struct dma_fence *pvr_queue_job_arm(struct pvr_job *job)1117{1118 drm_sched_job_arm(&job->base);1119 1120 return &job->base.s_fence->finished;1121}1122 1123/**1124 * pvr_queue_job_cleanup() - Cleanup fence/scheduler related fields in the job object.1125 * @job: The job to cleanup.1126 *1127 * Should be called in the job release path.1128 */1129void pvr_queue_job_cleanup(struct pvr_job *job)1130{1131 pvr_queue_fence_put(job->done_fence);1132 pvr_queue_fence_put(job->cccb_fence);1133 pvr_kccb_fence_put(job->kccb_fence);1134 1135 if (job->base.s_fence)1136 drm_sched_job_cleanup(&job->base);1137}1138 1139/**1140 * pvr_queue_job_push() - Push a job to its queue.1141 * @job: The job to push.1142 *1143 * Must be called after pvr_queue_job_init() and after all dependencies1144 * have been added to the job. This will effectively queue the job to1145 * the drm_sched_entity attached to the queue. We grab a reference on1146 * the job object, so the caller is free to drop its reference when it's1147 * done accessing the job object.1148 */1149void pvr_queue_job_push(struct pvr_job *job)1150{1151 struct pvr_queue *queue = container_of(job->base.sched, struct pvr_queue, scheduler);1152 1153 /* Keep track of the last queued job scheduled fence for combined submit. */1154 dma_fence_put(queue->last_queued_job_scheduled_fence);1155 queue->last_queued_job_scheduled_fence = dma_fence_get(&job->base.s_fence->scheduled);1156 1157 pvr_job_get(job);1158 drm_sched_entity_push_job(&job->base);1159}1160 1161static void reg_state_init(void *cpu_ptr, void *priv)1162{1163 struct pvr_queue *queue = priv;1164 1165 if (queue->type == DRM_PVR_JOB_TYPE_GEOMETRY) {1166 struct rogue_fwif_geom_ctx_state *geom_ctx_state_fw = cpu_ptr;1167 1168 geom_ctx_state_fw->geom_core[0].geom_reg_vdm_call_stack_pointer_init =1169 queue->callstack_addr;1170 }1171}1172 1173/**1174 * pvr_queue_create() - Create a queue object.1175 * @ctx: The context this queue will be attached to.1176 * @type: The type of jobs being pushed to this queue.1177 * @args: The arguments passed to the context creation function.1178 * @fw_ctx_map: CPU mapping of the FW context object.1179 *1180 * Create a queue object that will be used to queue and track jobs.1181 *1182 * Return:1183 * * A valid pointer to a pvr_queue object, or1184 * * An error pointer if the creation/initialization failed.1185 */1186struct pvr_queue *pvr_queue_create(struct pvr_context *ctx,1187 enum drm_pvr_job_type type,1188 struct drm_pvr_ioctl_create_context_args *args,1189 void *fw_ctx_map)1190{1191 static const struct {1192 u32 cccb_size;1193 const char *name;1194 } props[] = {1195 [DRM_PVR_JOB_TYPE_GEOMETRY] = {1196 .cccb_size = CTX_GEOM_CCCB_SIZE_LOG2,1197 .name = "geometry",1198 },1199 [DRM_PVR_JOB_TYPE_FRAGMENT] = {1200 .cccb_size = CTX_FRAG_CCCB_SIZE_LOG2,1201 .name = "fragment"1202 },1203 [DRM_PVR_JOB_TYPE_COMPUTE] = {1204 .cccb_size = CTX_COMPUTE_CCCB_SIZE_LOG2,1205 .name = "compute"1206 },1207 [DRM_PVR_JOB_TYPE_TRANSFER_FRAG] = {1208 .cccb_size = CTX_TRANSFER_CCCB_SIZE_LOG2,1209 .name = "transfer_frag"1210 },1211 };1212 struct pvr_device *pvr_dev = ctx->pvr_dev;1213 struct drm_gpu_scheduler *sched;1214 struct pvr_queue *queue;1215 int ctx_state_size, err;1216 void *cpu_map;1217 1218 if (WARN_ON(type >= sizeof(props)))1219 return ERR_PTR(-EINVAL);1220 1221 switch (ctx->type) {1222 case DRM_PVR_CTX_TYPE_RENDER:1223 if (type != DRM_PVR_JOB_TYPE_GEOMETRY &&1224 type != DRM_PVR_JOB_TYPE_FRAGMENT)1225 return ERR_PTR(-EINVAL);1226 break;1227 case DRM_PVR_CTX_TYPE_COMPUTE:1228 if (type != DRM_PVR_JOB_TYPE_COMPUTE)1229 return ERR_PTR(-EINVAL);1230 break;1231 case DRM_PVR_CTX_TYPE_TRANSFER_FRAG:1232 if (type != DRM_PVR_JOB_TYPE_TRANSFER_FRAG)1233 return ERR_PTR(-EINVAL);1234 break;1235 default:1236 return ERR_PTR(-EINVAL);1237 }1238 1239 ctx_state_size = get_ctx_state_size(pvr_dev, type);1240 if (ctx_state_size < 0)1241 return ERR_PTR(ctx_state_size);1242 1243 queue = kzalloc(sizeof(*queue), GFP_KERNEL);1244 if (!queue)1245 return ERR_PTR(-ENOMEM);1246 1247 queue->type = type;1248 queue->ctx_offset = get_ctx_offset(type);1249 queue->ctx = ctx;1250 queue->callstack_addr = args->callstack_addr;1251 sched = &queue->scheduler;1252 INIT_LIST_HEAD(&queue->node);1253 mutex_init(&queue->cccb_fence_ctx.job_lock);1254 pvr_queue_fence_ctx_init(&queue->cccb_fence_ctx.base);1255 pvr_queue_fence_ctx_init(&queue->job_fence_ctx);1256 1257 err = pvr_cccb_init(pvr_dev, &queue->cccb, props[type].cccb_size, props[type].name);1258 if (err)1259 goto err_free_queue;1260 1261 err = pvr_fw_object_create(pvr_dev, ctx_state_size,1262 PVR_BO_FW_FLAGS_DEVICE_UNCACHED,1263 reg_state_init, queue, &queue->reg_state_obj);1264 if (err)1265 goto err_cccb_fini;1266 1267 init_fw_context(queue, fw_ctx_map);1268 1269 if (type != DRM_PVR_JOB_TYPE_GEOMETRY && type != DRM_PVR_JOB_TYPE_FRAGMENT &&1270 args->callstack_addr) {1271 err = -EINVAL;1272 goto err_release_reg_state;1273 }1274 1275 cpu_map = pvr_fw_object_create_and_map(pvr_dev, sizeof(*queue->timeline_ufo.value),1276 PVR_BO_FW_FLAGS_DEVICE_UNCACHED,1277 NULL, NULL, &queue->timeline_ufo.fw_obj);1278 if (IS_ERR(cpu_map)) {1279 err = PTR_ERR(cpu_map);1280 goto err_release_reg_state;1281 }1282 1283 queue->timeline_ufo.value = cpu_map;1284 1285 err = drm_sched_init(&queue->scheduler,1286 &pvr_queue_sched_ops,1287 pvr_dev->sched_wq, 1, 64 * 1024, 1,1288 msecs_to_jiffies(500),1289 pvr_dev->sched_wq, NULL, "pvr-queue",1290 pvr_dev->base.dev);1291 if (err)1292 goto err_release_ufo;1293 1294 err = drm_sched_entity_init(&queue->entity,1295 DRM_SCHED_PRIORITY_KERNEL,1296 &sched, 1, &ctx->faulty);1297 if (err)1298 goto err_sched_fini;1299 1300 mutex_lock(&pvr_dev->queues.lock);1301 list_add_tail(&queue->node, &pvr_dev->queues.idle);1302 mutex_unlock(&pvr_dev->queues.lock);1303 1304 return queue;1305 1306err_sched_fini:1307 drm_sched_fini(&queue->scheduler);1308 1309err_release_ufo:1310 pvr_fw_object_unmap_and_destroy(queue->timeline_ufo.fw_obj);1311 1312err_release_reg_state:1313 pvr_fw_object_destroy(queue->reg_state_obj);1314 1315err_cccb_fini:1316 pvr_cccb_fini(&queue->cccb);1317 1318err_free_queue:1319 mutex_destroy(&queue->cccb_fence_ctx.job_lock);1320 kfree(queue);1321 1322 return ERR_PTR(err);1323}1324 1325void pvr_queue_device_pre_reset(struct pvr_device *pvr_dev)1326{1327 struct pvr_queue *queue;1328 1329 mutex_lock(&pvr_dev->queues.lock);1330 list_for_each_entry(queue, &pvr_dev->queues.idle, node)1331 pvr_queue_stop(queue, NULL);1332 list_for_each_entry(queue, &pvr_dev->queues.active, node)1333 pvr_queue_stop(queue, NULL);1334 mutex_unlock(&pvr_dev->queues.lock);1335}1336 1337void pvr_queue_device_post_reset(struct pvr_device *pvr_dev)1338{1339 struct pvr_queue *queue;1340 1341 mutex_lock(&pvr_dev->queues.lock);1342 list_for_each_entry(queue, &pvr_dev->queues.active, node)1343 pvr_queue_start(queue);1344 list_for_each_entry(queue, &pvr_dev->queues.idle, node)1345 pvr_queue_start(queue);1346 mutex_unlock(&pvr_dev->queues.lock);1347}1348 1349/**1350 * pvr_queue_kill() - Kill a queue.1351 * @queue: The queue to kill.1352 *1353 * Kill the queue so no new jobs can be pushed. Should be called when the1354 * context handle is destroyed. The queue object might last longer if jobs1355 * are still in flight and holding a reference to the context this queue1356 * belongs to.1357 */1358void pvr_queue_kill(struct pvr_queue *queue)1359{1360 drm_sched_entity_destroy(&queue->entity);1361 dma_fence_put(queue->last_queued_job_scheduled_fence);1362 queue->last_queued_job_scheduled_fence = NULL;1363}1364 1365/**1366 * pvr_queue_destroy() - Destroy a queue.1367 * @queue: The queue to destroy.1368 *1369 * Cleanup the queue and free the resources attached to it. Should be1370 * called from the context release function.1371 */1372void pvr_queue_destroy(struct pvr_queue *queue)1373{1374 if (!queue)1375 return;1376 1377 mutex_lock(&queue->ctx->pvr_dev->queues.lock);1378 list_del_init(&queue->node);1379 mutex_unlock(&queue->ctx->pvr_dev->queues.lock);1380 1381 drm_sched_fini(&queue->scheduler);1382 drm_sched_entity_fini(&queue->entity);1383 1384 if (WARN_ON(queue->last_queued_job_scheduled_fence))1385 dma_fence_put(queue->last_queued_job_scheduled_fence);1386 1387 pvr_queue_cleanup_fw_context(queue);1388 1389 pvr_fw_object_unmap_and_destroy(queue->timeline_ufo.fw_obj);1390 pvr_fw_object_destroy(queue->reg_state_obj);1391 pvr_cccb_fini(&queue->cccb);1392 mutex_destroy(&queue->cccb_fence_ctx.job_lock);1393 kfree(queue);1394}1395 1396/**1397 * pvr_queue_device_init() - Device-level initialization of queue related fields.1398 * @pvr_dev: The device to initialize.1399 *1400 * Initializes all fields related to queue management in pvr_device.1401 *1402 * Return:1403 * * 0 on success, or1404 * * An error code on failure.1405 */1406int pvr_queue_device_init(struct pvr_device *pvr_dev)1407{1408 int err;1409 1410 INIT_LIST_HEAD(&pvr_dev->queues.active);1411 INIT_LIST_HEAD(&pvr_dev->queues.idle);1412 err = drmm_mutex_init(from_pvr_device(pvr_dev), &pvr_dev->queues.lock);1413 if (err)1414 return err;1415 1416 pvr_dev->sched_wq = alloc_workqueue("powervr-sched", WQ_UNBOUND, 0);1417 if (!pvr_dev->sched_wq)1418 return -ENOMEM;1419 1420 return 0;1421}1422 1423/**1424 * pvr_queue_device_fini() - Device-level cleanup of queue related fields.1425 * @pvr_dev: The device to cleanup.1426 *1427 * Cleanup/free all queue-related resources attached to a pvr_device object.1428 */1429void pvr_queue_device_fini(struct pvr_device *pvr_dev)1430{1431 destroy_workqueue(pvr_dev->sched_wq);1432}1433