990 lines · c
1// SPDX-License-Identifier: GPL-2.02/* Copyright 2019 Linaro, Ltd, Rob Herring <robh@kernel.org> */3/* Copyright 2019 Collabora ltd. */4#include <linux/delay.h>5#include <linux/interrupt.h>6#include <linux/io.h>7#include <linux/iopoll.h>8#include <linux/platform_device.h>9#include <linux/pm_runtime.h>10#include <linux/dma-resv.h>11#include <drm/gpu_scheduler.h>12#include <drm/panfrost_drm.h>13 14#include "panfrost_device.h"15#include "panfrost_devfreq.h"16#include "panfrost_job.h"17#include "panfrost_features.h"18#include "panfrost_issues.h"19#include "panfrost_gem.h"20#include "panfrost_regs.h"21#include "panfrost_gpu.h"22#include "panfrost_mmu.h"23#include "panfrost_dump.h"24 25#define JOB_TIMEOUT_MS 50026 27#define job_write(dev, reg, data) writel(data, dev->iomem + (reg))28#define job_read(dev, reg) readl(dev->iomem + (reg))29 30struct panfrost_queue_state {31 struct drm_gpu_scheduler sched;32 u64 fence_context;33 u64 emit_seqno;34};35 36struct panfrost_job_slot {37 struct panfrost_queue_state queue[NUM_JOB_SLOTS];38 spinlock_t job_lock;39 int irq;40};41 42static struct panfrost_job *43to_panfrost_job(struct drm_sched_job *sched_job)44{45 return container_of(sched_job, struct panfrost_job, base);46}47 48struct panfrost_fence {49 struct dma_fence base;50 struct drm_device *dev;51 /* panfrost seqno for signaled() test */52 u64 seqno;53 int queue;54};55 56static inline struct panfrost_fence *57to_panfrost_fence(struct dma_fence *fence)58{59 return (struct panfrost_fence *)fence;60}61 62static const char *panfrost_fence_get_driver_name(struct dma_fence *fence)63{64 return "panfrost";65}66 67static const char *panfrost_fence_get_timeline_name(struct dma_fence *fence)68{69 struct panfrost_fence *f = to_panfrost_fence(fence);70 71 switch (f->queue) {72 case 0:73 return "panfrost-js-0";74 case 1:75 return "panfrost-js-1";76 case 2:77 return "panfrost-js-2";78 default:79 return NULL;80 }81}82 83static const struct dma_fence_ops panfrost_fence_ops = {84 .get_driver_name = panfrost_fence_get_driver_name,85 .get_timeline_name = panfrost_fence_get_timeline_name,86};87 88static struct dma_fence *panfrost_fence_create(struct panfrost_device *pfdev, int js_num)89{90 struct panfrost_fence *fence;91 struct panfrost_job_slot *js = pfdev->js;92 93 fence = kzalloc(sizeof(*fence), GFP_KERNEL);94 if (!fence)95 return ERR_PTR(-ENOMEM);96 97 fence->dev = pfdev->ddev;98 fence->queue = js_num;99 fence->seqno = ++js->queue[js_num].emit_seqno;100 dma_fence_init(&fence->base, &panfrost_fence_ops, &js->job_lock,101 js->queue[js_num].fence_context, fence->seqno);102 103 return &fence->base;104}105 106int panfrost_job_get_slot(struct panfrost_job *job)107{108 /* JS0: fragment jobs.109 * JS1: vertex/tiler jobs110 * JS2: compute jobs111 */112 if (job->requirements & PANFROST_JD_REQ_FS)113 return 0;114 115/* Not exposed to userspace yet */116#if 0117 if (job->requirements & PANFROST_JD_REQ_ONLY_COMPUTE) {118 if ((job->requirements & PANFROST_JD_REQ_CORE_GRP_MASK) &&119 (job->pfdev->features.nr_core_groups == 2))120 return 2;121 if (panfrost_has_hw_issue(job->pfdev, HW_ISSUE_8987))122 return 2;123 }124#endif125 return 1;126}127 128static void panfrost_job_write_affinity(struct panfrost_device *pfdev,129 u32 requirements,130 int js)131{132 u64 affinity;133 134 /*135 * Use all cores for now.136 * Eventually we may need to support tiler only jobs and h/w with137 * multiple (2) coherent core groups138 */139 affinity = pfdev->features.shader_present;140 141 job_write(pfdev, JS_AFFINITY_NEXT_LO(js), lower_32_bits(affinity));142 job_write(pfdev, JS_AFFINITY_NEXT_HI(js), upper_32_bits(affinity));143}144 145static u32146panfrost_get_job_chain_flag(const struct panfrost_job *job)147{148 struct panfrost_fence *f = to_panfrost_fence(job->done_fence);149 150 if (!panfrost_has_hw_feature(job->pfdev, HW_FEATURE_JOBCHAIN_DISAMBIGUATION))151 return 0;152 153 return (f->seqno & 1) ? JS_CONFIG_JOB_CHAIN_FLAG : 0;154}155 156static struct panfrost_job *157panfrost_dequeue_job(struct panfrost_device *pfdev, int slot)158{159 struct panfrost_job *job = pfdev->jobs[slot][0];160 161 WARN_ON(!job);162 if (job->is_profiled) {163 if (job->engine_usage) {164 job->engine_usage->elapsed_ns[slot] +=165 ktime_to_ns(ktime_sub(ktime_get(), job->start_time));166 job->engine_usage->cycles[slot] +=167 panfrost_cycle_counter_read(pfdev) - job->start_cycles;168 }169 panfrost_cycle_counter_put(job->pfdev);170 }171 172 pfdev->jobs[slot][0] = pfdev->jobs[slot][1];173 pfdev->jobs[slot][1] = NULL;174 175 return job;176}177 178static unsigned int179panfrost_enqueue_job(struct panfrost_device *pfdev, int slot,180 struct panfrost_job *job)181{182 if (WARN_ON(!job))183 return 0;184 185 if (!pfdev->jobs[slot][0]) {186 pfdev->jobs[slot][0] = job;187 return 0;188 }189 190 WARN_ON(pfdev->jobs[slot][1]);191 pfdev->jobs[slot][1] = job;192 WARN_ON(panfrost_get_job_chain_flag(job) ==193 panfrost_get_job_chain_flag(pfdev->jobs[slot][0]));194 return 1;195}196 197static void panfrost_job_hw_submit(struct panfrost_job *job, int js)198{199 struct panfrost_device *pfdev = job->pfdev;200 unsigned int subslot;201 u32 cfg;202 u64 jc_head = job->jc;203 int ret;204 205 panfrost_devfreq_record_busy(&pfdev->pfdevfreq);206 207 ret = pm_runtime_get_sync(pfdev->dev);208 if (ret < 0)209 return;210 211 if (WARN_ON(job_read(pfdev, JS_COMMAND_NEXT(js)))) {212 return;213 }214 215 cfg = panfrost_mmu_as_get(pfdev, job->mmu);216 217 job_write(pfdev, JS_HEAD_NEXT_LO(js), lower_32_bits(jc_head));218 job_write(pfdev, JS_HEAD_NEXT_HI(js), upper_32_bits(jc_head));219 220 panfrost_job_write_affinity(pfdev, job->requirements, js);221 222 /* start MMU, medium priority, cache clean/flush on end, clean/flush on223 * start */224 cfg |= JS_CONFIG_THREAD_PRI(8) |225 JS_CONFIG_START_FLUSH_CLEAN_INVALIDATE |226 JS_CONFIG_END_FLUSH_CLEAN_INVALIDATE |227 panfrost_get_job_chain_flag(job);228 229 if (panfrost_has_hw_feature(pfdev, HW_FEATURE_FLUSH_REDUCTION))230 cfg |= JS_CONFIG_ENABLE_FLUSH_REDUCTION;231 232 if (panfrost_has_hw_issue(pfdev, HW_ISSUE_10649))233 cfg |= JS_CONFIG_START_MMU;234 235 job_write(pfdev, JS_CONFIG_NEXT(js), cfg);236 237 if (panfrost_has_hw_feature(pfdev, HW_FEATURE_FLUSH_REDUCTION))238 job_write(pfdev, JS_FLUSH_ID_NEXT(js), job->flush_id);239 240 /* GO ! */241 242 spin_lock(&pfdev->js->job_lock);243 subslot = panfrost_enqueue_job(pfdev, js, job);244 /* Don't queue the job if a reset is in progress */245 if (!atomic_read(&pfdev->reset.pending)) {246 if (pfdev->profile_mode) {247 panfrost_cycle_counter_get(pfdev);248 job->is_profiled = true;249 job->start_time = ktime_get();250 job->start_cycles = panfrost_cycle_counter_read(pfdev);251 }252 253 job_write(pfdev, JS_COMMAND_NEXT(js), JS_COMMAND_START);254 dev_dbg(pfdev->dev,255 "JS: Submitting atom %p to js[%d][%d] with head=0x%llx AS %d",256 job, js, subslot, jc_head, cfg & 0xf);257 }258 spin_unlock(&pfdev->js->job_lock);259}260 261static int panfrost_acquire_object_fences(struct drm_gem_object **bos,262 int bo_count,263 struct drm_sched_job *job)264{265 int i, ret;266 267 for (i = 0; i < bo_count; i++) {268 ret = dma_resv_reserve_fences(bos[i]->resv, 1);269 if (ret)270 return ret;271 272 /* panfrost always uses write mode in its current uapi */273 ret = drm_sched_job_add_implicit_dependencies(job, bos[i],274 true);275 if (ret)276 return ret;277 }278 279 return 0;280}281 282static void panfrost_attach_object_fences(struct drm_gem_object **bos,283 int bo_count,284 struct dma_fence *fence)285{286 int i;287 288 for (i = 0; i < bo_count; i++)289 dma_resv_add_fence(bos[i]->resv, fence, DMA_RESV_USAGE_WRITE);290}291 292int panfrost_job_push(struct panfrost_job *job)293{294 struct panfrost_device *pfdev = job->pfdev;295 struct ww_acquire_ctx acquire_ctx;296 int ret = 0;297 298 ret = drm_gem_lock_reservations(job->bos, job->bo_count,299 &acquire_ctx);300 if (ret)301 return ret;302 303 mutex_lock(&pfdev->sched_lock);304 drm_sched_job_arm(&job->base);305 306 job->render_done_fence = dma_fence_get(&job->base.s_fence->finished);307 308 ret = panfrost_acquire_object_fences(job->bos, job->bo_count,309 &job->base);310 if (ret) {311 mutex_unlock(&pfdev->sched_lock);312 goto unlock;313 }314 315 kref_get(&job->refcount); /* put by scheduler job completion */316 317 drm_sched_entity_push_job(&job->base);318 319 mutex_unlock(&pfdev->sched_lock);320 321 panfrost_attach_object_fences(job->bos, job->bo_count,322 job->render_done_fence);323 324unlock:325 drm_gem_unlock_reservations(job->bos, job->bo_count, &acquire_ctx);326 327 return ret;328}329 330static void panfrost_job_cleanup(struct kref *ref)331{332 struct panfrost_job *job = container_of(ref, struct panfrost_job,333 refcount);334 unsigned int i;335 336 dma_fence_put(job->done_fence);337 dma_fence_put(job->render_done_fence);338 339 if (job->mappings) {340 for (i = 0; i < job->bo_count; i++) {341 if (!job->mappings[i])342 break;343 344 atomic_dec(&job->mappings[i]->obj->gpu_usecount);345 panfrost_gem_mapping_put(job->mappings[i]);346 }347 kvfree(job->mappings);348 }349 350 if (job->bos) {351 for (i = 0; i < job->bo_count; i++)352 drm_gem_object_put(job->bos[i]);353 354 kvfree(job->bos);355 }356 357 kfree(job);358}359 360void panfrost_job_put(struct panfrost_job *job)361{362 kref_put(&job->refcount, panfrost_job_cleanup);363}364 365static void panfrost_job_free(struct drm_sched_job *sched_job)366{367 struct panfrost_job *job = to_panfrost_job(sched_job);368 369 drm_sched_job_cleanup(sched_job);370 371 panfrost_job_put(job);372}373 374static struct dma_fence *panfrost_job_run(struct drm_sched_job *sched_job)375{376 struct panfrost_job *job = to_panfrost_job(sched_job);377 struct panfrost_device *pfdev = job->pfdev;378 int slot = panfrost_job_get_slot(job);379 struct dma_fence *fence = NULL;380 381 if (unlikely(job->base.s_fence->finished.error))382 return NULL;383 384 /* Nothing to execute: can happen if the job has finished while385 * we were resetting the GPU.386 */387 if (!job->jc)388 return NULL;389 390 fence = panfrost_fence_create(pfdev, slot);391 if (IS_ERR(fence))392 return fence;393 394 if (job->done_fence)395 dma_fence_put(job->done_fence);396 job->done_fence = dma_fence_get(fence);397 398 panfrost_job_hw_submit(job, slot);399 400 return fence;401}402 403void panfrost_job_enable_interrupts(struct panfrost_device *pfdev)404{405 int j;406 u32 irq_mask = 0;407 408 clear_bit(PANFROST_COMP_BIT_JOB, pfdev->is_suspended);409 410 for (j = 0; j < NUM_JOB_SLOTS; j++) {411 irq_mask |= MK_JS_MASK(j);412 }413 414 job_write(pfdev, JOB_INT_CLEAR, irq_mask);415 job_write(pfdev, JOB_INT_MASK, irq_mask);416}417 418void panfrost_job_suspend_irq(struct panfrost_device *pfdev)419{420 set_bit(PANFROST_COMP_BIT_JOB, pfdev->is_suspended);421 422 job_write(pfdev, JOB_INT_MASK, 0);423 synchronize_irq(pfdev->js->irq);424}425 426static void panfrost_job_handle_err(struct panfrost_device *pfdev,427 struct panfrost_job *job,428 unsigned int js)429{430 u32 js_status = job_read(pfdev, JS_STATUS(js));431 const char *exception_name = panfrost_exception_name(js_status);432 bool signal_fence = true;433 434 if (!panfrost_exception_is_fault(js_status)) {435 dev_dbg(pfdev->dev, "js event, js=%d, status=%s, head=0x%x, tail=0x%x",436 js, exception_name,437 job_read(pfdev, JS_HEAD_LO(js)),438 job_read(pfdev, JS_TAIL_LO(js)));439 } else {440 dev_err(pfdev->dev, "js fault, js=%d, status=%s, head=0x%x, tail=0x%x",441 js, exception_name,442 job_read(pfdev, JS_HEAD_LO(js)),443 job_read(pfdev, JS_TAIL_LO(js)));444 }445 446 if (js_status == DRM_PANFROST_EXCEPTION_STOPPED) {447 /* Update the job head so we can resume */448 job->jc = job_read(pfdev, JS_TAIL_LO(js)) |449 ((u64)job_read(pfdev, JS_TAIL_HI(js)) << 32);450 451 /* The job will be resumed, don't signal the fence */452 signal_fence = false;453 } else if (js_status == DRM_PANFROST_EXCEPTION_TERMINATED) {454 /* Job has been hard-stopped, flag it as canceled */455 dma_fence_set_error(job->done_fence, -ECANCELED);456 job->jc = 0;457 } else if (panfrost_exception_is_fault(js_status)) {458 /* We might want to provide finer-grained error code based on459 * the exception type, but unconditionally setting to EINVAL460 * is good enough for now.461 */462 dma_fence_set_error(job->done_fence, -EINVAL);463 job->jc = 0;464 }465 466 panfrost_mmu_as_put(pfdev, job->mmu);467 panfrost_devfreq_record_idle(&pfdev->pfdevfreq);468 469 if (signal_fence)470 dma_fence_signal_locked(job->done_fence);471 472 pm_runtime_put_autosuspend(pfdev->dev);473 474 if (panfrost_exception_needs_reset(pfdev, js_status)) {475 atomic_set(&pfdev->reset.pending, 1);476 drm_sched_fault(&pfdev->js->queue[js].sched);477 }478}479 480static void panfrost_job_handle_done(struct panfrost_device *pfdev,481 struct panfrost_job *job)482{483 /* Set ->jc to 0 to avoid re-submitting an already finished job (can484 * happen when we receive the DONE interrupt while doing a GPU reset).485 */486 job->jc = 0;487 panfrost_mmu_as_put(pfdev, job->mmu);488 panfrost_devfreq_record_idle(&pfdev->pfdevfreq);489 490 dma_fence_signal_locked(job->done_fence);491 pm_runtime_put_autosuspend(pfdev->dev);492}493 494static void panfrost_job_handle_irq(struct panfrost_device *pfdev, u32 status)495{496 struct panfrost_job *done[NUM_JOB_SLOTS][2] = {};497 struct panfrost_job *failed[NUM_JOB_SLOTS] = {};498 u32 js_state = 0, js_events = 0;499 unsigned int i, j;500 501 /* First we collect all failed/done jobs. */502 while (status) {503 u32 js_state_mask = 0;504 505 for (j = 0; j < NUM_JOB_SLOTS; j++) {506 if (status & MK_JS_MASK(j))507 js_state_mask |= MK_JS_MASK(j);508 509 if (status & JOB_INT_MASK_DONE(j)) {510 if (done[j][0])511 done[j][1] = panfrost_dequeue_job(pfdev, j);512 else513 done[j][0] = panfrost_dequeue_job(pfdev, j);514 }515 516 if (status & JOB_INT_MASK_ERR(j)) {517 /* Cancel the next submission. Will be submitted518 * after we're done handling this failure if519 * there's no reset pending.520 */521 job_write(pfdev, JS_COMMAND_NEXT(j), JS_COMMAND_NOP);522 failed[j] = panfrost_dequeue_job(pfdev, j);523 }524 }525 526 /* JS_STATE is sampled when JOB_INT_CLEAR is written.527 * For each BIT(slot) or BIT(slot + 16) bit written to528 * JOB_INT_CLEAR, the corresponding bits in JS_STATE529 * (BIT(slot) and BIT(slot + 16)) are updated, but this530 * is racy. If we only have one job done at the time we531 * read JOB_INT_RAWSTAT but the second job fails before we532 * clear the status, we end up with a status containing533 * only the DONE bit and consider both jobs as DONE since534 * JS_STATE reports both NEXT and CURRENT as inactive.535 * To prevent that, let's repeat this clear+read steps536 * until status is 0.537 */538 job_write(pfdev, JOB_INT_CLEAR, status);539 js_state &= ~js_state_mask;540 js_state |= job_read(pfdev, JOB_INT_JS_STATE) & js_state_mask;541 js_events |= status;542 status = job_read(pfdev, JOB_INT_RAWSTAT);543 }544 545 /* Then we handle the dequeued jobs. */546 for (j = 0; j < NUM_JOB_SLOTS; j++) {547 if (!(js_events & MK_JS_MASK(j)))548 continue;549 550 if (failed[j]) {551 panfrost_job_handle_err(pfdev, failed[j], j);552 } else if (pfdev->jobs[j][0] && !(js_state & MK_JS_MASK(j))) {553 /* When the current job doesn't fail, the JM dequeues554 * the next job without waiting for an ACK, this means555 * we can have 2 jobs dequeued and only catch the556 * interrupt when the second one is done. If both slots557 * are inactive, but one job remains in pfdev->jobs[j],558 * consider it done. Of course that doesn't apply if a559 * failure happened since we cancelled execution of the560 * job in _NEXT (see above).561 */562 if (WARN_ON(!done[j][0]))563 done[j][0] = panfrost_dequeue_job(pfdev, j);564 else565 done[j][1] = panfrost_dequeue_job(pfdev, j);566 }567 568 for (i = 0; i < ARRAY_SIZE(done[0]) && done[j][i]; i++)569 panfrost_job_handle_done(pfdev, done[j][i]);570 }571 572 /* And finally we requeue jobs that were waiting in the second slot573 * and have been stopped if we detected a failure on the first slot.574 */575 for (j = 0; j < NUM_JOB_SLOTS; j++) {576 if (!(js_events & MK_JS_MASK(j)))577 continue;578 579 if (!failed[j] || !pfdev->jobs[j][0])580 continue;581 582 if (pfdev->jobs[j][0]->jc == 0) {583 /* The job was cancelled, signal the fence now */584 struct panfrost_job *canceled = panfrost_dequeue_job(pfdev, j);585 586 dma_fence_set_error(canceled->done_fence, -ECANCELED);587 panfrost_job_handle_done(pfdev, canceled);588 } else if (!atomic_read(&pfdev->reset.pending)) {589 /* Requeue the job we removed if no reset is pending */590 job_write(pfdev, JS_COMMAND_NEXT(j), JS_COMMAND_START);591 }592 }593}594 595static void panfrost_job_handle_irqs(struct panfrost_device *pfdev)596{597 u32 status = job_read(pfdev, JOB_INT_RAWSTAT);598 599 while (status) {600 pm_runtime_mark_last_busy(pfdev->dev);601 602 spin_lock(&pfdev->js->job_lock);603 panfrost_job_handle_irq(pfdev, status);604 spin_unlock(&pfdev->js->job_lock);605 status = job_read(pfdev, JOB_INT_RAWSTAT);606 }607}608 609static u32 panfrost_active_slots(struct panfrost_device *pfdev,610 u32 *js_state_mask, u32 js_state)611{612 u32 rawstat;613 614 if (!(js_state & *js_state_mask))615 return 0;616 617 rawstat = job_read(pfdev, JOB_INT_RAWSTAT);618 if (rawstat) {619 unsigned int i;620 621 for (i = 0; i < NUM_JOB_SLOTS; i++) {622 if (rawstat & MK_JS_MASK(i))623 *js_state_mask &= ~MK_JS_MASK(i);624 }625 }626 627 return js_state & *js_state_mask;628}629 630static void631panfrost_reset(struct panfrost_device *pfdev,632 struct drm_sched_job *bad)633{634 u32 js_state, js_state_mask = 0xffffffff;635 unsigned int i, j;636 bool cookie;637 int ret;638 639 if (!atomic_read(&pfdev->reset.pending))640 return;641 642 /* Stop the schedulers.643 *644 * FIXME: We temporarily get out of the dma_fence_signalling section645 * because the cleanup path generate lockdep splats when taking locks646 * to release job resources. We should rework the code to follow this647 * pattern:648 *649 * try_lock650 * if (locked)651 * release652 * else653 * schedule_work_to_release_later654 */655 for (i = 0; i < NUM_JOB_SLOTS; i++)656 drm_sched_stop(&pfdev->js->queue[i].sched, bad);657 658 cookie = dma_fence_begin_signalling();659 660 if (bad)661 drm_sched_increase_karma(bad);662 663 /* Mask job interrupts and synchronize to make sure we won't be664 * interrupted during our reset.665 */666 job_write(pfdev, JOB_INT_MASK, 0);667 synchronize_irq(pfdev->js->irq);668 669 for (i = 0; i < NUM_JOB_SLOTS; i++) {670 /* Cancel the next job and soft-stop the running job. */671 job_write(pfdev, JS_COMMAND_NEXT(i), JS_COMMAND_NOP);672 job_write(pfdev, JS_COMMAND(i), JS_COMMAND_SOFT_STOP);673 }674 675 /* Wait at most 10ms for soft-stops to complete */676 ret = readl_poll_timeout(pfdev->iomem + JOB_INT_JS_STATE, js_state,677 !panfrost_active_slots(pfdev, &js_state_mask, js_state),678 10, 10000);679 680 if (ret)681 dev_err(pfdev->dev, "Soft-stop failed\n");682 683 /* Handle the remaining interrupts before we reset. */684 panfrost_job_handle_irqs(pfdev);685 686 /* Remaining interrupts have been handled, but we might still have687 * stuck jobs. Let's make sure the PM counters stay balanced by688 * manually calling pm_runtime_put_noidle() and689 * panfrost_devfreq_record_idle() for each stuck job.690 * Let's also make sure the cycle counting register's refcnt is691 * kept balanced to prevent it from running forever692 */693 spin_lock(&pfdev->js->job_lock);694 for (i = 0; i < NUM_JOB_SLOTS; i++) {695 for (j = 0; j < ARRAY_SIZE(pfdev->jobs[0]) && pfdev->jobs[i][j]; j++) {696 if (pfdev->jobs[i][j]->is_profiled)697 panfrost_cycle_counter_put(pfdev->jobs[i][j]->pfdev);698 pm_runtime_put_noidle(pfdev->dev);699 panfrost_devfreq_record_idle(&pfdev->pfdevfreq);700 }701 }702 memset(pfdev->jobs, 0, sizeof(pfdev->jobs));703 spin_unlock(&pfdev->js->job_lock);704 705 /* Proceed with reset now. */706 panfrost_device_reset(pfdev);707 708 /* panfrost_device_reset() unmasks job interrupts, but we want to709 * keep them masked a bit longer.710 */711 job_write(pfdev, JOB_INT_MASK, 0);712 713 /* GPU has been reset, we can clear the reset pending bit. */714 atomic_set(&pfdev->reset.pending, 0);715 716 /* Now resubmit jobs that were previously queued but didn't have a717 * chance to finish.718 * FIXME: We temporarily get out of the DMA fence signalling section719 * while resubmitting jobs because the job submission logic will720 * allocate memory with the GFP_KERNEL flag which can trigger memory721 * reclaim and exposes a lock ordering issue.722 */723 dma_fence_end_signalling(cookie);724 for (i = 0; i < NUM_JOB_SLOTS; i++)725 drm_sched_resubmit_jobs(&pfdev->js->queue[i].sched);726 cookie = dma_fence_begin_signalling();727 728 /* Restart the schedulers */729 for (i = 0; i < NUM_JOB_SLOTS; i++)730 drm_sched_start(&pfdev->js->queue[i].sched);731 732 /* Re-enable job interrupts now that everything has been restarted. */733 job_write(pfdev, JOB_INT_MASK,734 GENMASK(16 + NUM_JOB_SLOTS - 1, 16) |735 GENMASK(NUM_JOB_SLOTS - 1, 0));736 737 dma_fence_end_signalling(cookie);738}739 740static enum drm_gpu_sched_stat panfrost_job_timedout(struct drm_sched_job741 *sched_job)742{743 struct panfrost_job *job = to_panfrost_job(sched_job);744 struct panfrost_device *pfdev = job->pfdev;745 int js = panfrost_job_get_slot(job);746 747 /*748 * If the GPU managed to complete this jobs fence, the timeout is749 * spurious. Bail out.750 */751 if (dma_fence_is_signaled(job->done_fence))752 return DRM_GPU_SCHED_STAT_NOMINAL;753 754 /*755 * Panfrost IRQ handler may take a long time to process an interrupt756 * if there is another IRQ handler hogging the processing.757 * For example, the HDMI encoder driver might be stuck in the IRQ758 * handler for a significant time in a case of bad cable connection.759 * In order to catch such cases and not report spurious Panfrost760 * job timeouts, synchronize the IRQ handler and re-check the fence761 * status.762 */763 synchronize_irq(pfdev->js->irq);764 765 if (dma_fence_is_signaled(job->done_fence)) {766 dev_warn(pfdev->dev, "unexpectedly high interrupt latency\n");767 return DRM_GPU_SCHED_STAT_NOMINAL;768 }769 770 dev_err(pfdev->dev, "gpu sched timeout, js=%d, config=0x%x, status=0x%x, head=0x%x, tail=0x%x, sched_job=%p",771 js,772 job_read(pfdev, JS_CONFIG(js)),773 job_read(pfdev, JS_STATUS(js)),774 job_read(pfdev, JS_HEAD_LO(js)),775 job_read(pfdev, JS_TAIL_LO(js)),776 sched_job);777 778 panfrost_core_dump(job);779 780 atomic_set(&pfdev->reset.pending, 1);781 panfrost_reset(pfdev, sched_job);782 783 return DRM_GPU_SCHED_STAT_NOMINAL;784}785 786static void panfrost_reset_work(struct work_struct *work)787{788 struct panfrost_device *pfdev;789 790 pfdev = container_of(work, struct panfrost_device, reset.work);791 panfrost_reset(pfdev, NULL);792}793 794static const struct drm_sched_backend_ops panfrost_sched_ops = {795 .run_job = panfrost_job_run,796 .timedout_job = panfrost_job_timedout,797 .free_job = panfrost_job_free798};799 800static irqreturn_t panfrost_job_irq_handler_thread(int irq, void *data)801{802 struct panfrost_device *pfdev = data;803 804 panfrost_job_handle_irqs(pfdev);805 806 /* Enable interrupts only if we're not about to get suspended */807 if (!test_bit(PANFROST_COMP_BIT_JOB, pfdev->is_suspended))808 job_write(pfdev, JOB_INT_MASK,809 GENMASK(16 + NUM_JOB_SLOTS - 1, 16) |810 GENMASK(NUM_JOB_SLOTS - 1, 0));811 812 return IRQ_HANDLED;813}814 815static irqreturn_t panfrost_job_irq_handler(int irq, void *data)816{817 struct panfrost_device *pfdev = data;818 u32 status;819 820 if (test_bit(PANFROST_COMP_BIT_JOB, pfdev->is_suspended))821 return IRQ_NONE;822 823 status = job_read(pfdev, JOB_INT_STAT);824 if (!status)825 return IRQ_NONE;826 827 job_write(pfdev, JOB_INT_MASK, 0);828 return IRQ_WAKE_THREAD;829}830 831int panfrost_job_init(struct panfrost_device *pfdev)832{833 struct panfrost_job_slot *js;834 unsigned int nentries = 2;835 int ret, j;836 837 /* All GPUs have two entries per queue, but without jobchain838 * disambiguation stopping the right job in the close path is tricky,839 * so let's just advertise one entry in that case.840 */841 if (!panfrost_has_hw_feature(pfdev, HW_FEATURE_JOBCHAIN_DISAMBIGUATION))842 nentries = 1;843 844 pfdev->js = js = devm_kzalloc(pfdev->dev, sizeof(*js), GFP_KERNEL);845 if (!js)846 return -ENOMEM;847 848 INIT_WORK(&pfdev->reset.work, panfrost_reset_work);849 spin_lock_init(&js->job_lock);850 851 js->irq = platform_get_irq_byname(to_platform_device(pfdev->dev), "job");852 if (js->irq < 0)853 return js->irq;854 855 ret = devm_request_threaded_irq(pfdev->dev, js->irq,856 panfrost_job_irq_handler,857 panfrost_job_irq_handler_thread,858 IRQF_SHARED, KBUILD_MODNAME "-job",859 pfdev);860 if (ret) {861 dev_err(pfdev->dev, "failed to request job irq");862 return ret;863 }864 865 pfdev->reset.wq = alloc_ordered_workqueue("panfrost-reset", 0);866 if (!pfdev->reset.wq)867 return -ENOMEM;868 869 for (j = 0; j < NUM_JOB_SLOTS; j++) {870 js->queue[j].fence_context = dma_fence_context_alloc(1);871 872 ret = drm_sched_init(&js->queue[j].sched,873 &panfrost_sched_ops, NULL,874 DRM_SCHED_PRIORITY_COUNT,875 nentries, 0,876 msecs_to_jiffies(JOB_TIMEOUT_MS),877 pfdev->reset.wq,878 NULL, "pan_js", pfdev->dev);879 if (ret) {880 dev_err(pfdev->dev, "Failed to create scheduler: %d.", ret);881 goto err_sched;882 }883 }884 885 panfrost_job_enable_interrupts(pfdev);886 887 return 0;888 889err_sched:890 for (j--; j >= 0; j--)891 drm_sched_fini(&js->queue[j].sched);892 893 destroy_workqueue(pfdev->reset.wq);894 return ret;895}896 897void panfrost_job_fini(struct panfrost_device *pfdev)898{899 struct panfrost_job_slot *js = pfdev->js;900 int j;901 902 job_write(pfdev, JOB_INT_MASK, 0);903 904 for (j = 0; j < NUM_JOB_SLOTS; j++) {905 drm_sched_fini(&js->queue[j].sched);906 }907 908 cancel_work_sync(&pfdev->reset.work);909 destroy_workqueue(pfdev->reset.wq);910}911 912int panfrost_job_open(struct panfrost_file_priv *panfrost_priv)913{914 struct panfrost_device *pfdev = panfrost_priv->pfdev;915 struct panfrost_job_slot *js = pfdev->js;916 struct drm_gpu_scheduler *sched;917 int ret, i;918 919 for (i = 0; i < NUM_JOB_SLOTS; i++) {920 sched = &js->queue[i].sched;921 ret = drm_sched_entity_init(&panfrost_priv->sched_entity[i],922 DRM_SCHED_PRIORITY_NORMAL, &sched,923 1, NULL);924 if (WARN_ON(ret))925 return ret;926 }927 return 0;928}929 930void panfrost_job_close(struct panfrost_file_priv *panfrost_priv)931{932 struct panfrost_device *pfdev = panfrost_priv->pfdev;933 int i;934 935 for (i = 0; i < NUM_JOB_SLOTS; i++)936 drm_sched_entity_destroy(&panfrost_priv->sched_entity[i]);937 938 /* Kill in-flight jobs */939 spin_lock(&pfdev->js->job_lock);940 for (i = 0; i < NUM_JOB_SLOTS; i++) {941 struct drm_sched_entity *entity = &panfrost_priv->sched_entity[i];942 int j;943 944 for (j = ARRAY_SIZE(pfdev->jobs[0]) - 1; j >= 0; j--) {945 struct panfrost_job *job = pfdev->jobs[i][j];946 u32 cmd;947 948 if (!job || job->base.entity != entity)949 continue;950 951 if (j == 1) {952 /* Try to cancel the job before it starts */953 job_write(pfdev, JS_COMMAND_NEXT(i), JS_COMMAND_NOP);954 /* Reset the job head so it doesn't get restarted if955 * the job in the first slot failed.956 */957 job->jc = 0;958 }959 960 if (panfrost_has_hw_feature(pfdev, HW_FEATURE_JOBCHAIN_DISAMBIGUATION)) {961 cmd = panfrost_get_job_chain_flag(job) ?962 JS_COMMAND_HARD_STOP_1 :963 JS_COMMAND_HARD_STOP_0;964 } else {965 cmd = JS_COMMAND_HARD_STOP;966 }967 968 job_write(pfdev, JS_COMMAND(i), cmd);969 970 /* Jobs can outlive their file context */971 job->engine_usage = NULL;972 }973 }974 spin_unlock(&pfdev->js->job_lock);975}976 977int panfrost_job_is_idle(struct panfrost_device *pfdev)978{979 struct panfrost_job_slot *js = pfdev->js;980 int i;981 982 for (i = 0; i < NUM_JOB_SLOTS; i++) {983 /* If there are any jobs in the HW queue, we're not idle */984 if (atomic_read(&js->queue[i].sched.credit_count))985 return false;986 }987 988 return true;989}990