151 lines · c
1// SPDX-License-Identifier: GPL-2.02/*3 * Copyright (C) 2017 Etnaviv Project4 */5 6#include <linux/moduleparam.h>7 8#include "etnaviv_drv.h"9#include "etnaviv_dump.h"10#include "etnaviv_gem.h"11#include "etnaviv_gpu.h"12#include "etnaviv_sched.h"13#include "state.xml.h"14 15static int etnaviv_job_hang_limit = 0;16module_param_named(job_hang_limit, etnaviv_job_hang_limit, int , 0444);17static int etnaviv_hw_jobs_limit = 4;18module_param_named(hw_job_limit, etnaviv_hw_jobs_limit, int , 0444);19 20static struct dma_fence *etnaviv_sched_run_job(struct drm_sched_job *sched_job)21{22 struct etnaviv_gem_submit *submit = to_etnaviv_submit(sched_job);23 struct dma_fence *fence = NULL;24 25 if (likely(!sched_job->s_fence->finished.error))26 fence = etnaviv_gpu_submit(submit);27 else28 dev_dbg(submit->gpu->dev, "skipping bad job\n");29 30 return fence;31}32 33static enum drm_gpu_sched_stat etnaviv_sched_timedout_job(struct drm_sched_job34 *sched_job)35{36 struct etnaviv_gem_submit *submit = to_etnaviv_submit(sched_job);37 struct etnaviv_gpu *gpu = submit->gpu;38 u32 dma_addr;39 int change;40 41 /*42 * If the GPU managed to complete this jobs fence, the timout is43 * spurious. Bail out.44 */45 if (dma_fence_is_signaled(submit->out_fence))46 goto out_no_timeout;47 48 /*49 * If the GPU is still making forward progress on the front-end (which50 * should never loop) we shift out the timeout to give it a chance to51 * finish the job.52 */53 dma_addr = gpu_read(gpu, VIVS_FE_DMA_ADDRESS);54 change = dma_addr - gpu->hangcheck_dma_addr;55 if (gpu->state == ETNA_GPU_STATE_RUNNING &&56 (gpu->completed_fence != gpu->hangcheck_fence ||57 change < 0 || change > 16)) {58 gpu->hangcheck_dma_addr = dma_addr;59 gpu->hangcheck_fence = gpu->completed_fence;60 goto out_no_timeout;61 }62 63 /* block scheduler */64 drm_sched_stop(&gpu->sched, sched_job);65 66 if(sched_job)67 drm_sched_increase_karma(sched_job);68 69 /* get the GPU back into the init state */70 etnaviv_core_dump(submit);71 etnaviv_gpu_recover_hang(submit);72 73 drm_sched_resubmit_jobs(&gpu->sched);74 75 drm_sched_start(&gpu->sched);76 return DRM_GPU_SCHED_STAT_NOMINAL;77 78out_no_timeout:79 list_add(&sched_job->list, &sched_job->sched->pending_list);80 return DRM_GPU_SCHED_STAT_NOMINAL;81}82 83static void etnaviv_sched_free_job(struct drm_sched_job *sched_job)84{85 struct etnaviv_gem_submit *submit = to_etnaviv_submit(sched_job);86 87 drm_sched_job_cleanup(sched_job);88 89 etnaviv_submit_put(submit);90}91 92static const struct drm_sched_backend_ops etnaviv_sched_ops = {93 .run_job = etnaviv_sched_run_job,94 .timedout_job = etnaviv_sched_timedout_job,95 .free_job = etnaviv_sched_free_job,96};97 98int etnaviv_sched_push_job(struct etnaviv_gem_submit *submit)99{100 struct etnaviv_gpu *gpu = submit->gpu;101 int ret;102 103 /*104 * Hold the sched lock across the whole operation to avoid jobs being105 * pushed out of order with regard to their sched fence seqnos as106 * allocated in drm_sched_job_arm.107 */108 mutex_lock(&gpu->sched_lock);109 110 drm_sched_job_arm(&submit->sched_job);111 112 submit->out_fence = dma_fence_get(&submit->sched_job.s_fence->finished);113 ret = xa_alloc_cyclic(&gpu->user_fences, &submit->out_fence_id,114 submit->out_fence, xa_limit_32b,115 &gpu->next_user_fence, GFP_KERNEL);116 if (ret < 0) {117 drm_sched_job_cleanup(&submit->sched_job);118 goto out_unlock;119 }120 121 /* the scheduler holds on to the job now */122 kref_get(&submit->refcount);123 124 drm_sched_entity_push_job(&submit->sched_job);125 126out_unlock:127 mutex_unlock(&gpu->sched_lock);128 129 return ret;130}131 132int etnaviv_sched_init(struct etnaviv_gpu *gpu)133{134 int ret;135 136 ret = drm_sched_init(&gpu->sched, &etnaviv_sched_ops, NULL,137 DRM_SCHED_PRIORITY_COUNT,138 etnaviv_hw_jobs_limit, etnaviv_job_hang_limit,139 msecs_to_jiffies(500), NULL, NULL,140 dev_name(gpu->dev), gpu->dev);141 if (ret)142 return ret;143 144 return 0;145}146 147void etnaviv_sched_fini(struct etnaviv_gpu *gpu)148{149 drm_sched_fini(&gpu->sched);150}151