359 lines · c
1/*2 * Copyright © 2014 Broadcom3 *4 * Permission is hereby granted, free of charge, to any person obtaining a5 * copy of this software and associated documentation files (the "Software"),6 * to deal in the Software without restriction, including without limitation7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,8 * and/or sell copies of the Software, and to permit persons to whom the9 * Software is furnished to do so, subject to the following conditions:10 *11 * The above copyright notice and this permission notice (including the next12 * paragraph) shall be included in all copies or substantial portions of the13 * Software.14 *15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS21 * IN THE SOFTWARE.22 */23 24/**25 * DOC: Interrupt management for the V3D engine26 *27 * We have an interrupt status register (V3D_INTCTL) which reports28 * interrupts, and where writing 1 bits clears those interrupts.29 * There are also a pair of interrupt registers30 * (V3D_INTENA/V3D_INTDIS) where writing a 1 to their bits enables or31 * disables that specific interrupt, and 0s written are ignored32 * (reading either one returns the set of enabled interrupts).33 *34 * When we take a binning flush done interrupt, we need to submit the35 * next frame for binning and move the finished frame to the render36 * thread.37 *38 * When we take a render frame interrupt, we need to wake the39 * processes waiting for some frame to be done, and get the next frame40 * submitted ASAP (so the hardware doesn't sit idle when there's work41 * to do).42 *43 * When we take the binner out of memory interrupt, we need to44 * allocate some new memory and pass it to the binner so that the45 * current job can make progress.46 */47 48#include <linux/platform_device.h>49 50#include <drm/drm_drv.h>51 52#include "vc4_drv.h"53#include "vc4_regs.h"54#include "vc4_trace.h"55 56#define V3D_DRIVER_IRQS (V3D_INT_OUTOMEM | \57 V3D_INT_FLDONE | \58 V3D_INT_FRDONE)59 60static void61vc4_overflow_mem_work(struct work_struct *work)62{63 struct vc4_dev *vc4 =64 container_of(work, struct vc4_dev, overflow_mem_work);65 struct vc4_bo *bo;66 int bin_bo_slot;67 struct vc4_exec_info *exec;68 unsigned long irqflags;69 70 mutex_lock(&vc4->bin_bo_lock);71 72 if (!vc4->bin_bo)73 goto complete;74 75 bo = vc4->bin_bo;76 77 bin_bo_slot = vc4_v3d_get_bin_slot(vc4);78 if (bin_bo_slot < 0) {79 drm_err(&vc4->base, "Couldn't allocate binner overflow mem\n");80 goto complete;81 }82 83 spin_lock_irqsave(&vc4->job_lock, irqflags);84 85 if (vc4->bin_alloc_overflow) {86 /* If we had overflow memory allocated previously,87 * then that chunk will free when the current bin job88 * is done. If we don't have a bin job running, then89 * the chunk will be done whenever the list of render90 * jobs has drained.91 */92 exec = vc4_first_bin_job(vc4);93 if (!exec)94 exec = vc4_last_render_job(vc4);95 if (exec) {96 exec->bin_slots |= vc4->bin_alloc_overflow;97 } else {98 /* There's nothing queued in the hardware, so99 * the old slot is free immediately.100 */101 vc4->bin_alloc_used &= ~vc4->bin_alloc_overflow;102 }103 }104 vc4->bin_alloc_overflow = BIT(bin_bo_slot);105 106 V3D_WRITE(V3D_BPOA, bo->base.dma_addr + bin_bo_slot * vc4->bin_alloc_size);107 V3D_WRITE(V3D_BPOS, bo->base.base.size);108 V3D_WRITE(V3D_INTCTL, V3D_INT_OUTOMEM);109 V3D_WRITE(V3D_INTENA, V3D_INT_OUTOMEM);110 spin_unlock_irqrestore(&vc4->job_lock, irqflags);111 112complete:113 mutex_unlock(&vc4->bin_bo_lock);114}115 116static void117vc4_irq_finish_bin_job(struct drm_device *dev)118{119 struct vc4_dev *vc4 = to_vc4_dev(dev);120 struct vc4_exec_info *next, *exec = vc4_first_bin_job(vc4);121 122 if (!exec)123 return;124 125 trace_vc4_bcl_end_irq(dev, exec->seqno);126 127 vc4_move_job_to_render(dev, exec);128 next = vc4_first_bin_job(vc4);129 130 /* Only submit the next job in the bin list if it matches the perfmon131 * attached to the one that just finished (or if both jobs don't have132 * perfmon attached to them).133 */134 if (next && next->perfmon == exec->perfmon)135 vc4_submit_next_bin_job(dev);136}137 138static void139vc4_cancel_bin_job(struct drm_device *dev)140{141 struct vc4_dev *vc4 = to_vc4_dev(dev);142 struct vc4_exec_info *exec = vc4_first_bin_job(vc4);143 144 if (!exec)145 return;146 147 /* Stop the perfmon so that the next bin job can be started. */148 if (exec->perfmon)149 vc4_perfmon_stop(vc4, exec->perfmon, false);150 151 list_move_tail(&exec->head, &vc4->bin_job_list);152 vc4_submit_next_bin_job(dev);153}154 155static void156vc4_irq_finish_render_job(struct drm_device *dev)157{158 struct vc4_dev *vc4 = to_vc4_dev(dev);159 struct vc4_exec_info *exec = vc4_first_render_job(vc4);160 struct vc4_exec_info *nextbin, *nextrender;161 162 if (!exec)163 return;164 165 trace_vc4_rcl_end_irq(dev, exec->seqno);166 167 vc4->finished_seqno++;168 list_move_tail(&exec->head, &vc4->job_done_list);169 170 nextbin = vc4_first_bin_job(vc4);171 nextrender = vc4_first_render_job(vc4);172 173 /* Only stop the perfmon if following jobs in the queue don't expect it174 * to be enabled.175 */176 if (exec->perfmon && !nextrender &&177 (!nextbin || nextbin->perfmon != exec->perfmon))178 vc4_perfmon_stop(vc4, exec->perfmon, true);179 180 /* If there's a render job waiting, start it. If this is not the case181 * we may have to unblock the binner if it's been stalled because of182 * perfmon (this can be checked by comparing the perfmon attached to183 * the finished renderjob to the one attached to the next bin job: if184 * they don't match, this means the binner is stalled and should be185 * restarted).186 */187 if (nextrender)188 vc4_submit_next_render_job(dev);189 else if (nextbin && nextbin->perfmon != exec->perfmon)190 vc4_submit_next_bin_job(dev);191 192 if (exec->fence) {193 dma_fence_signal_locked(exec->fence);194 dma_fence_put(exec->fence);195 exec->fence = NULL;196 }197 198 wake_up_all(&vc4->job_wait_queue);199 schedule_work(&vc4->job_done_work);200}201 202static irqreturn_t203vc4_irq(int irq, void *arg)204{205 struct drm_device *dev = arg;206 struct vc4_dev *vc4 = to_vc4_dev(dev);207 uint32_t intctl;208 irqreturn_t status = IRQ_NONE;209 210 barrier();211 intctl = V3D_READ(V3D_INTCTL);212 213 /* Acknowledge the interrupts we're handling here. The binner214 * last flush / render frame done interrupt will be cleared,215 * while OUTOMEM will stay high until the underlying cause is216 * cleared.217 */218 V3D_WRITE(V3D_INTCTL, intctl);219 220 if (intctl & V3D_INT_OUTOMEM) {221 /* Disable OUTOMEM until the work is done. */222 V3D_WRITE(V3D_INTDIS, V3D_INT_OUTOMEM);223 schedule_work(&vc4->overflow_mem_work);224 status = IRQ_HANDLED;225 }226 227 if (intctl & V3D_INT_FLDONE) {228 spin_lock(&vc4->job_lock);229 vc4_irq_finish_bin_job(dev);230 spin_unlock(&vc4->job_lock);231 status = IRQ_HANDLED;232 }233 234 if (intctl & V3D_INT_FRDONE) {235 spin_lock(&vc4->job_lock);236 vc4_irq_finish_render_job(dev);237 spin_unlock(&vc4->job_lock);238 status = IRQ_HANDLED;239 }240 241 return status;242}243 244static void245vc4_irq_prepare(struct drm_device *dev)246{247 struct vc4_dev *vc4 = to_vc4_dev(dev);248 249 if (!vc4->v3d)250 return;251 252 init_waitqueue_head(&vc4->job_wait_queue);253 INIT_WORK(&vc4->overflow_mem_work, vc4_overflow_mem_work);254 255 /* Clear any pending interrupts someone might have left around256 * for us.257 */258 V3D_WRITE(V3D_INTCTL, V3D_DRIVER_IRQS);259}260 261void262vc4_irq_enable(struct drm_device *dev)263{264 struct vc4_dev *vc4 = to_vc4_dev(dev);265 266 if (WARN_ON_ONCE(vc4->is_vc5))267 return;268 269 if (!vc4->v3d)270 return;271 272 /* Enable the render done interrupts. The out-of-memory interrupt is273 * enabled as soon as we have a binner BO allocated.274 */275 V3D_WRITE(V3D_INTENA, V3D_INT_FLDONE | V3D_INT_FRDONE);276}277 278void279vc4_irq_disable(struct drm_device *dev)280{281 struct vc4_dev *vc4 = to_vc4_dev(dev);282 283 if (WARN_ON_ONCE(vc4->is_vc5))284 return;285 286 if (!vc4->v3d)287 return;288 289 /* Disable sending interrupts for our driver's IRQs. */290 V3D_WRITE(V3D_INTDIS, V3D_DRIVER_IRQS);291 292 /* Clear any pending interrupts we might have left. */293 V3D_WRITE(V3D_INTCTL, V3D_DRIVER_IRQS);294 295 /* Finish any interrupt handler still in flight. */296 synchronize_irq(vc4->irq);297 298 cancel_work_sync(&vc4->overflow_mem_work);299}300 301int vc4_irq_install(struct drm_device *dev, int irq)302{303 struct vc4_dev *vc4 = to_vc4_dev(dev);304 int ret;305 306 if (WARN_ON_ONCE(vc4->is_vc5))307 return -ENODEV;308 309 if (irq == IRQ_NOTCONNECTED)310 return -ENOTCONN;311 312 vc4_irq_prepare(dev);313 314 ret = request_irq(irq, vc4_irq, 0, dev->driver->name, dev);315 if (ret)316 return ret;317 318 vc4_irq_enable(dev);319 320 return 0;321}322 323void vc4_irq_uninstall(struct drm_device *dev)324{325 struct vc4_dev *vc4 = to_vc4_dev(dev);326 327 if (WARN_ON_ONCE(vc4->is_vc5))328 return;329 330 vc4_irq_disable(dev);331 free_irq(vc4->irq, dev);332}333 334/** Reinitializes interrupt registers when a GPU reset is performed. */335void vc4_irq_reset(struct drm_device *dev)336{337 struct vc4_dev *vc4 = to_vc4_dev(dev);338 unsigned long irqflags;339 340 if (WARN_ON_ONCE(vc4->is_vc5))341 return;342 343 /* Acknowledge any stale IRQs. */344 V3D_WRITE(V3D_INTCTL, V3D_DRIVER_IRQS);345 346 /*347 * Turn all our interrupts on. Binner out of memory is the348 * only one we expect to trigger at this point, since we've349 * just come from poweron and haven't supplied any overflow350 * memory yet.351 */352 V3D_WRITE(V3D_INTENA, V3D_DRIVER_IRQS);353 354 spin_lock_irqsave(&vc4->job_lock, irqflags);355 vc4_cancel_bin_job(dev);356 vc4_irq_finish_render_job(dev);357 spin_unlock_irqrestore(&vc4->job_lock, irqflags);358}359