389 lines · c
1// SPDX-License-Identifier: GPL-2.0 OR MIT2/* Copyright 2017-2019 Qiang Yu <yuq825@gmail.com> */3 4#include <linux/interrupt.h>5#include <linux/iopoll.h>6#include <linux/device.h>7#include <linux/slab.h>8 9#include <drm/lima_drm.h>10 11#include "lima_device.h"12#include "lima_gp.h"13#include "lima_regs.h"14#include "lima_gem.h"15#include "lima_vm.h"16 17#define gp_write(reg, data) writel(data, ip->iomem + reg)18#define gp_read(reg) readl(ip->iomem + reg)19 20static irqreturn_t lima_gp_irq_handler(int irq, void *data)21{22 struct lima_ip *ip = data;23 struct lima_device *dev = ip->dev;24 struct lima_sched_pipe *pipe = dev->pipe + lima_pipe_gp;25 struct lima_sched_task *task = pipe->current_task;26 u32 state = gp_read(LIMA_GP_INT_STAT);27 u32 status = gp_read(LIMA_GP_STATUS);28 bool done = false;29 30 /* for shared irq case */31 if (!state)32 return IRQ_NONE;33 34 if (state & LIMA_GP_IRQ_MASK_ERROR) {35 if ((state & LIMA_GP_IRQ_MASK_ERROR) ==36 LIMA_GP_IRQ_PLBU_OUT_OF_MEM) {37 dev_dbg(dev->dev, "%s out of heap irq status=%x\n",38 lima_ip_name(ip), status);39 } else {40 dev_err(dev->dev, "%s error irq state=%x status=%x\n",41 lima_ip_name(ip), state, status);42 if (task)43 task->recoverable = false;44 }45 46 /* mask all interrupts before hard reset */47 gp_write(LIMA_GP_INT_MASK, 0);48 49 pipe->error = true;50 done = true;51 } else {52 bool valid = state & (LIMA_GP_IRQ_VS_END_CMD_LST |53 LIMA_GP_IRQ_PLBU_END_CMD_LST);54 bool active = status & (LIMA_GP_STATUS_VS_ACTIVE |55 LIMA_GP_STATUS_PLBU_ACTIVE);56 done = valid && !active;57 pipe->error = false;58 }59 60 gp_write(LIMA_GP_INT_CLEAR, state);61 62 if (done)63 lima_sched_pipe_task_done(pipe);64 65 return IRQ_HANDLED;66}67 68static void lima_gp_soft_reset_async(struct lima_ip *ip)69{70 if (ip->data.async_reset)71 return;72 73 gp_write(LIMA_GP_INT_MASK, 0);74 gp_write(LIMA_GP_INT_CLEAR, LIMA_GP_IRQ_RESET_COMPLETED);75 gp_write(LIMA_GP_CMD, LIMA_GP_CMD_SOFT_RESET);76 ip->data.async_reset = true;77}78 79static int lima_gp_soft_reset_async_wait(struct lima_ip *ip)80{81 struct lima_device *dev = ip->dev;82 int err;83 u32 v;84 85 if (!ip->data.async_reset)86 return 0;87 88 err = readl_poll_timeout(ip->iomem + LIMA_GP_INT_RAWSTAT, v,89 v & LIMA_GP_IRQ_RESET_COMPLETED,90 0, 100);91 if (err) {92 dev_err(dev->dev, "%s soft reset time out\n",93 lima_ip_name(ip));94 return err;95 }96 97 gp_write(LIMA_GP_INT_CLEAR, LIMA_GP_IRQ_MASK_ALL);98 gp_write(LIMA_GP_INT_MASK, LIMA_GP_IRQ_MASK_USED);99 100 ip->data.async_reset = false;101 return 0;102}103 104static int lima_gp_task_validate(struct lima_sched_pipe *pipe,105 struct lima_sched_task *task)106{107 struct drm_lima_gp_frame *frame = task->frame;108 u32 *f = frame->frame;109 (void)pipe;110 111 if (f[LIMA_GP_VSCL_START_ADDR >> 2] >112 f[LIMA_GP_VSCL_END_ADDR >> 2] ||113 f[LIMA_GP_PLBUCL_START_ADDR >> 2] >114 f[LIMA_GP_PLBUCL_END_ADDR >> 2] ||115 f[LIMA_GP_PLBU_ALLOC_START_ADDR >> 2] >116 f[LIMA_GP_PLBU_ALLOC_END_ADDR >> 2])117 return -EINVAL;118 119 if (f[LIMA_GP_VSCL_START_ADDR >> 2] ==120 f[LIMA_GP_VSCL_END_ADDR >> 2] &&121 f[LIMA_GP_PLBUCL_START_ADDR >> 2] ==122 f[LIMA_GP_PLBUCL_END_ADDR >> 2])123 return -EINVAL;124 125 return 0;126}127 128static void lima_gp_task_run(struct lima_sched_pipe *pipe,129 struct lima_sched_task *task)130{131 struct lima_ip *ip = pipe->processor[0];132 struct drm_lima_gp_frame *frame = task->frame;133 u32 *f = frame->frame;134 u32 cmd = 0;135 int i;136 137 /* update real heap buffer size for GP */138 for (i = 0; i < task->num_bos; i++) {139 struct lima_bo *bo = task->bos[i];140 141 if (bo->heap_size &&142 lima_vm_get_va(task->vm, bo) ==143 f[LIMA_GP_PLBU_ALLOC_START_ADDR >> 2]) {144 f[LIMA_GP_PLBU_ALLOC_END_ADDR >> 2] =145 f[LIMA_GP_PLBU_ALLOC_START_ADDR >> 2] +146 bo->heap_size;147 task->recoverable = true;148 task->heap = bo;149 break;150 }151 }152 153 if (f[LIMA_GP_VSCL_START_ADDR >> 2] !=154 f[LIMA_GP_VSCL_END_ADDR >> 2])155 cmd |= LIMA_GP_CMD_START_VS;156 if (f[LIMA_GP_PLBUCL_START_ADDR >> 2] !=157 f[LIMA_GP_PLBUCL_END_ADDR >> 2])158 cmd |= LIMA_GP_CMD_START_PLBU;159 160 /* before any hw ops, wait last success task async soft reset */161 lima_gp_soft_reset_async_wait(ip);162 163 for (i = 0; i < LIMA_GP_FRAME_REG_NUM; i++)164 writel(f[i], ip->iomem + LIMA_GP_VSCL_START_ADDR + i * 4);165 166 gp_write(LIMA_GP_CMD, LIMA_GP_CMD_UPDATE_PLBU_ALLOC);167 gp_write(LIMA_GP_CMD, cmd);168}169 170static int lima_gp_bus_stop_poll(struct lima_ip *ip)171{172 return !!(gp_read(LIMA_GP_STATUS) & LIMA_GP_STATUS_BUS_STOPPED);173}174 175static int lima_gp_hard_reset_poll(struct lima_ip *ip)176{177 gp_write(LIMA_GP_PERF_CNT_0_LIMIT, 0xC01A0000);178 return gp_read(LIMA_GP_PERF_CNT_0_LIMIT) == 0xC01A0000;179}180 181static int lima_gp_hard_reset(struct lima_ip *ip)182{183 struct lima_device *dev = ip->dev;184 int ret;185 186 gp_write(LIMA_GP_PERF_CNT_0_LIMIT, 0xC0FFE000);187 gp_write(LIMA_GP_INT_MASK, 0);188 189 gp_write(LIMA_GP_CMD, LIMA_GP_CMD_STOP_BUS);190 ret = lima_poll_timeout(ip, lima_gp_bus_stop_poll, 10, 100);191 if (ret) {192 dev_err(dev->dev, "%s bus stop timeout\n", lima_ip_name(ip));193 return ret;194 }195 gp_write(LIMA_GP_CMD, LIMA_GP_CMD_RESET);196 ret = lima_poll_timeout(ip, lima_gp_hard_reset_poll, 10, 100);197 if (ret) {198 dev_err(dev->dev, "%s hard reset timeout\n", lima_ip_name(ip));199 return ret;200 }201 202 gp_write(LIMA_GP_PERF_CNT_0_LIMIT, 0);203 gp_write(LIMA_GP_INT_CLEAR, LIMA_GP_IRQ_MASK_ALL);204 gp_write(LIMA_GP_INT_MASK, LIMA_GP_IRQ_MASK_USED);205 206 /*207 * if there was an async soft reset queued,208 * don't wait for it in the next job209 */210 ip->data.async_reset = false;211 212 return 0;213}214 215static void lima_gp_task_fini(struct lima_sched_pipe *pipe)216{217 lima_gp_soft_reset_async(pipe->processor[0]);218}219 220static void lima_gp_task_error(struct lima_sched_pipe *pipe)221{222 struct lima_ip *ip = pipe->processor[0];223 224 dev_err(ip->dev->dev, "%s task error int_state=%x status=%x\n",225 lima_ip_name(ip), gp_read(LIMA_GP_INT_STAT),226 gp_read(LIMA_GP_STATUS));227 228 lima_gp_hard_reset(ip);229}230 231static void lima_gp_task_mmu_error(struct lima_sched_pipe *pipe)232{233 lima_sched_pipe_task_done(pipe);234}235 236static void lima_gp_task_mask_irq(struct lima_sched_pipe *pipe)237{238 struct lima_ip *ip = pipe->processor[0];239 240 gp_write(LIMA_GP_INT_MASK, 0);241}242 243static int lima_gp_task_recover(struct lima_sched_pipe *pipe)244{245 struct lima_ip *ip = pipe->processor[0];246 struct lima_sched_task *task = pipe->current_task;247 struct drm_lima_gp_frame *frame = task->frame;248 u32 *f = frame->frame;249 size_t fail_size =250 f[LIMA_GP_PLBU_ALLOC_END_ADDR >> 2] -251 f[LIMA_GP_PLBU_ALLOC_START_ADDR >> 2];252 253 if (fail_size == task->heap->heap_size) {254 int ret;255 256 ret = lima_heap_alloc(task->heap, task->vm);257 if (ret < 0)258 return ret;259 }260 261 gp_write(LIMA_GP_INT_MASK, LIMA_GP_IRQ_MASK_USED);262 /* Resume from where we stopped, i.e. new start is old end */263 gp_write(LIMA_GP_PLBU_ALLOC_START_ADDR,264 f[LIMA_GP_PLBU_ALLOC_END_ADDR >> 2]);265 f[LIMA_GP_PLBU_ALLOC_END_ADDR >> 2] =266 f[LIMA_GP_PLBU_ALLOC_START_ADDR >> 2] + task->heap->heap_size;267 gp_write(LIMA_GP_PLBU_ALLOC_END_ADDR,268 f[LIMA_GP_PLBU_ALLOC_END_ADDR >> 2]);269 gp_write(LIMA_GP_CMD, LIMA_GP_CMD_UPDATE_PLBU_ALLOC);270 return 0;271}272 273static void lima_gp_print_version(struct lima_ip *ip)274{275 u32 version, major, minor;276 char *name;277 278 version = gp_read(LIMA_GP_VERSION);279 major = (version >> 8) & 0xFF;280 minor = version & 0xFF;281 switch (version >> 16) {282 case 0xA07:283 name = "mali200";284 break;285 case 0xC07:286 name = "mali300";287 break;288 case 0xB07:289 name = "mali400";290 break;291 case 0xD07:292 name = "mali450";293 break;294 default:295 name = "unknown";296 break;297 }298 dev_info(ip->dev->dev, "%s - %s version major %d minor %d\n",299 lima_ip_name(ip), name, major, minor);300}301 302static struct kmem_cache *lima_gp_task_slab;303static int lima_gp_task_slab_refcnt;304 305static int lima_gp_hw_init(struct lima_ip *ip)306{307 ip->data.async_reset = false;308 lima_gp_soft_reset_async(ip);309 return lima_gp_soft_reset_async_wait(ip);310}311 312int lima_gp_resume(struct lima_ip *ip)313{314 return lima_gp_hw_init(ip);315}316 317void lima_gp_suspend(struct lima_ip *ip)318{319 320}321 322int lima_gp_init(struct lima_ip *ip)323{324 struct lima_device *dev = ip->dev;325 int err;326 327 lima_gp_print_version(ip);328 329 err = lima_gp_hw_init(ip);330 if (err)331 return err;332 333 err = devm_request_irq(dev->dev, ip->irq, lima_gp_irq_handler,334 IRQF_SHARED, lima_ip_name(ip), ip);335 if (err) {336 dev_err(dev->dev, "%s fail to request irq\n",337 lima_ip_name(ip));338 return err;339 }340 341 dev->gp_version = gp_read(LIMA_GP_VERSION);342 343 return 0;344}345 346void lima_gp_fini(struct lima_ip *ip)347{348 struct lima_device *dev = ip->dev;349 350 devm_free_irq(dev->dev, ip->irq, ip);351}352 353int lima_gp_pipe_init(struct lima_device *dev)354{355 int frame_size = sizeof(struct drm_lima_gp_frame);356 struct lima_sched_pipe *pipe = dev->pipe + lima_pipe_gp;357 358 if (!lima_gp_task_slab) {359 lima_gp_task_slab = kmem_cache_create_usercopy(360 "lima_gp_task", sizeof(struct lima_sched_task) + frame_size,361 0, SLAB_HWCACHE_ALIGN, sizeof(struct lima_sched_task),362 frame_size, NULL);363 if (!lima_gp_task_slab)364 return -ENOMEM;365 }366 lima_gp_task_slab_refcnt++;367 368 pipe->frame_size = frame_size;369 pipe->task_slab = lima_gp_task_slab;370 371 pipe->task_validate = lima_gp_task_validate;372 pipe->task_run = lima_gp_task_run;373 pipe->task_fini = lima_gp_task_fini;374 pipe->task_error = lima_gp_task_error;375 pipe->task_mmu_error = lima_gp_task_mmu_error;376 pipe->task_recover = lima_gp_task_recover;377 pipe->task_mask_irq = lima_gp_task_mask_irq;378 379 return 0;380}381 382void lima_gp_pipe_fini(struct lima_device *dev)383{384 if (!--lima_gp_task_slab_refcnt) {385 kmem_cache_destroy(lima_gp_task_slab);386 lima_gp_task_slab = NULL;387 }388}389