329 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2/* Copyright (c) 2017 The Linux Foundation. All rights reserved.3 */4 5#include "msm_gem.h"6#include "a5xx_gpu.h"7 8/*9 * Try to transition the preemption state from old to new. Return10 * true on success or false if the original state wasn't 'old'11 */12static inline bool try_preempt_state(struct a5xx_gpu *a5xx_gpu,13 enum preempt_state old, enum preempt_state new)14{15 enum preempt_state cur = atomic_cmpxchg(&a5xx_gpu->preempt_state,16 old, new);17 18 return (cur == old);19}20 21/*22 * Force the preemption state to the specified state. This is used in cases23 * where the current state is known and won't change24 */25static inline void set_preempt_state(struct a5xx_gpu *gpu,26 enum preempt_state new)27{28 /*29 * preempt_state may be read by other cores trying to trigger a30 * preemption or in the interrupt handler so barriers are needed31 * before...32 */33 smp_mb__before_atomic();34 atomic_set(&gpu->preempt_state, new);35 /* ... and after*/36 smp_mb__after_atomic();37}38 39/* Write the most recent wptr for the given ring into the hardware */40static inline void update_wptr(struct msm_gpu *gpu, struct msm_ringbuffer *ring)41{42 unsigned long flags;43 uint32_t wptr;44 45 if (!ring)46 return;47 48 spin_lock_irqsave(&ring->preempt_lock, flags);49 wptr = get_wptr(ring);50 spin_unlock_irqrestore(&ring->preempt_lock, flags);51 52 gpu_write(gpu, REG_A5XX_CP_RB_WPTR, wptr);53}54 55/* Return the highest priority ringbuffer with something in it */56static struct msm_ringbuffer *get_next_ring(struct msm_gpu *gpu)57{58 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);59 struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu);60 unsigned long flags;61 int i;62 63 for (i = 0; i < gpu->nr_rings; i++) {64 bool empty;65 struct msm_ringbuffer *ring = gpu->rb[i];66 67 spin_lock_irqsave(&ring->preempt_lock, flags);68 empty = (get_wptr(ring) == gpu->funcs->get_rptr(gpu, ring));69 if (!empty && ring == a5xx_gpu->cur_ring)70 empty = ring->memptrs->fence == a5xx_gpu->last_seqno[i];71 spin_unlock_irqrestore(&ring->preempt_lock, flags);72 73 if (!empty)74 return ring;75 }76 77 return NULL;78}79 80static void a5xx_preempt_timer(struct timer_list *t)81{82 struct a5xx_gpu *a5xx_gpu = from_timer(a5xx_gpu, t, preempt_timer);83 struct msm_gpu *gpu = &a5xx_gpu->base.base;84 struct drm_device *dev = gpu->dev;85 86 if (!try_preempt_state(a5xx_gpu, PREEMPT_TRIGGERED, PREEMPT_FAULTED))87 return;88 89 DRM_DEV_ERROR(dev->dev, "%s: preemption timed out\n", gpu->name);90 kthread_queue_work(gpu->worker, &gpu->recover_work);91}92 93/* Try to trigger a preemption switch */94void a5xx_preempt_trigger(struct msm_gpu *gpu)95{96 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);97 struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu);98 unsigned long flags;99 struct msm_ringbuffer *ring;100 101 if (gpu->nr_rings == 1)102 return;103 104 /*105 * Serialize preemption start to ensure that we always make106 * decision on latest state. Otherwise we can get stuck in107 * lower priority or empty ring.108 */109 spin_lock_irqsave(&a5xx_gpu->preempt_start_lock, flags);110 111 /*112 * Try to start preemption by moving from NONE to START. If113 * unsuccessful, a preemption is already in flight114 */115 if (!try_preempt_state(a5xx_gpu, PREEMPT_NONE, PREEMPT_START))116 goto out;117 118 /* Get the next ring to preempt to */119 ring = get_next_ring(gpu);120 121 /*122 * If no ring is populated or the highest priority ring is the current123 * one do nothing except to update the wptr to the latest and greatest124 */125 if (!ring || (a5xx_gpu->cur_ring == ring)) {126 /*127 * Its possible that while a preemption request is in progress128 * from an irq context, a user context trying to submit might129 * fail to update the write pointer, because it determines130 * that the preempt state is not PREEMPT_NONE.131 *132 * Close the race by introducing an intermediate133 * state PREEMPT_ABORT to let the submit path134 * know that the ringbuffer is not going to change135 * and can safely update the write pointer.136 */137 138 set_preempt_state(a5xx_gpu, PREEMPT_ABORT);139 update_wptr(gpu, a5xx_gpu->cur_ring);140 set_preempt_state(a5xx_gpu, PREEMPT_NONE);141 goto out;142 }143 144 spin_unlock_irqrestore(&a5xx_gpu->preempt_start_lock, flags);145 146 /* Make sure the wptr doesn't update while we're in motion */147 spin_lock_irqsave(&ring->preempt_lock, flags);148 a5xx_gpu->preempt[ring->id]->wptr = get_wptr(ring);149 spin_unlock_irqrestore(&ring->preempt_lock, flags);150 151 /* Set the address of the incoming preemption record */152 gpu_write64(gpu, REG_A5XX_CP_CONTEXT_SWITCH_RESTORE_ADDR_LO,153 a5xx_gpu->preempt_iova[ring->id]);154 155 a5xx_gpu->next_ring = ring;156 157 /* Start a timer to catch a stuck preemption */158 mod_timer(&a5xx_gpu->preempt_timer, jiffies + msecs_to_jiffies(10000));159 160 /* Set the preemption state to triggered */161 set_preempt_state(a5xx_gpu, PREEMPT_TRIGGERED);162 163 /* Make sure everything is written before hitting the button */164 wmb();165 166 /* And actually start the preemption */167 gpu_write(gpu, REG_A5XX_CP_CONTEXT_SWITCH_CNTL, 1);168 return;169 170out:171 spin_unlock_irqrestore(&a5xx_gpu->preempt_start_lock, flags);172}173 174void a5xx_preempt_irq(struct msm_gpu *gpu)175{176 uint32_t status;177 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);178 struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu);179 struct drm_device *dev = gpu->dev;180 181 if (!try_preempt_state(a5xx_gpu, PREEMPT_TRIGGERED, PREEMPT_PENDING))182 return;183 184 /* Delete the preemption watchdog timer */185 del_timer(&a5xx_gpu->preempt_timer);186 187 /*188 * The hardware should be setting CP_CONTEXT_SWITCH_CNTL to zero before189 * firing the interrupt, but there is a non zero chance of a hardware190 * condition or a software race that could set it again before we have a191 * chance to finish. If that happens, log and go for recovery192 */193 status = gpu_read(gpu, REG_A5XX_CP_CONTEXT_SWITCH_CNTL);194 if (unlikely(status)) {195 set_preempt_state(a5xx_gpu, PREEMPT_FAULTED);196 DRM_DEV_ERROR(dev->dev, "%s: Preemption failed to complete\n",197 gpu->name);198 kthread_queue_work(gpu->worker, &gpu->recover_work);199 return;200 }201 202 a5xx_gpu->cur_ring = a5xx_gpu->next_ring;203 a5xx_gpu->next_ring = NULL;204 205 update_wptr(gpu, a5xx_gpu->cur_ring);206 207 set_preempt_state(a5xx_gpu, PREEMPT_NONE);208 209 /*210 * Try to trigger preemption again in case there was a submit or211 * retire during ring switch212 */213 a5xx_preempt_trigger(gpu);214}215 216void a5xx_preempt_hw_init(struct msm_gpu *gpu)217{218 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);219 struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu);220 int i;221 222 /* Always come up on rb 0 */223 a5xx_gpu->cur_ring = gpu->rb[0];224 225 /* No preemption if we only have one ring */226 if (gpu->nr_rings == 1)227 return;228 229 for (i = 0; i < gpu->nr_rings; i++) {230 a5xx_gpu->preempt[i]->data = 0;231 a5xx_gpu->preempt[i]->info = 0;232 a5xx_gpu->preempt[i]->wptr = 0;233 a5xx_gpu->preempt[i]->rptr = 0;234 a5xx_gpu->preempt[i]->rbase = gpu->rb[i]->iova;235 a5xx_gpu->preempt[i]->rptr_addr = shadowptr(a5xx_gpu, gpu->rb[i]);236 }237 238 /* Write a 0 to signal that we aren't switching pagetables */239 gpu_write64(gpu, REG_A5XX_CP_CONTEXT_SWITCH_SMMU_INFO_LO, 0);240 241 /* Reset the preemption state */242 set_preempt_state(a5xx_gpu, PREEMPT_NONE);243}244 245static int preempt_init_ring(struct a5xx_gpu *a5xx_gpu,246 struct msm_ringbuffer *ring)247{248 struct adreno_gpu *adreno_gpu = &a5xx_gpu->base;249 struct msm_gpu *gpu = &adreno_gpu->base;250 struct a5xx_preempt_record *ptr;251 void *counters;252 struct drm_gem_object *bo = NULL, *counters_bo = NULL;253 u64 iova = 0, counters_iova = 0;254 255 ptr = msm_gem_kernel_new(gpu->dev,256 A5XX_PREEMPT_RECORD_SIZE + A5XX_PREEMPT_COUNTER_SIZE,257 MSM_BO_WC | MSM_BO_MAP_PRIV, gpu->aspace, &bo, &iova);258 259 if (IS_ERR(ptr))260 return PTR_ERR(ptr);261 262 /* The buffer to store counters needs to be unprivileged */263 counters = msm_gem_kernel_new(gpu->dev,264 A5XX_PREEMPT_COUNTER_SIZE,265 MSM_BO_WC, gpu->aspace, &counters_bo, &counters_iova);266 if (IS_ERR(counters)) {267 msm_gem_kernel_put(bo, gpu->aspace);268 return PTR_ERR(counters);269 }270 271 msm_gem_object_set_name(bo, "preempt");272 msm_gem_object_set_name(counters_bo, "preempt_counters");273 274 a5xx_gpu->preempt_bo[ring->id] = bo;275 a5xx_gpu->preempt_counters_bo[ring->id] = counters_bo;276 a5xx_gpu->preempt_iova[ring->id] = iova;277 a5xx_gpu->preempt[ring->id] = ptr;278 279 /* Set up the defaults on the preemption record */280 281 ptr->magic = A5XX_PREEMPT_RECORD_MAGIC;282 ptr->info = 0;283 ptr->data = 0;284 ptr->cntl = MSM_GPU_RB_CNTL_DEFAULT | AXXX_CP_RB_CNTL_NO_UPDATE;285 286 ptr->counter = counters_iova;287 288 return 0;289}290 291void a5xx_preempt_fini(struct msm_gpu *gpu)292{293 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);294 struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu);295 int i;296 297 for (i = 0; i < gpu->nr_rings; i++) {298 msm_gem_kernel_put(a5xx_gpu->preempt_bo[i], gpu->aspace);299 msm_gem_kernel_put(a5xx_gpu->preempt_counters_bo[i], gpu->aspace);300 }301}302 303void a5xx_preempt_init(struct msm_gpu *gpu)304{305 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);306 struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu);307 int i;308 309 /* No preemption if we only have one ring */310 if (gpu->nr_rings <= 1)311 return;312 313 for (i = 0; i < gpu->nr_rings; i++) {314 if (preempt_init_ring(a5xx_gpu, gpu->rb[i])) {315 /*316 * On any failure our adventure is over. Clean up and317 * set nr_rings to 1 to force preemption off318 */319 a5xx_preempt_fini(gpu);320 gpu->nr_rings = 1;321 322 return;323 }324 }325 326 spin_lock_init(&a5xx_gpu->preempt_start_lock);327 timer_setup(&a5xx_gpu->preempt_timer, a5xx_preempt_timer, 0);328}329