536 lines · c
1// SPDX-License-Identifier: GPL-2.0 OR MIT2/* Copyright 2017-2019 Qiang Yu <yuq825@gmail.com> */3 4#include <linux/regulator/consumer.h>5#include <linux/reset.h>6#include <linux/clk.h>7#include <linux/slab.h>8#include <linux/dma-mapping.h>9#include <linux/platform_device.h>10 11#include "lima_device.h"12#include "lima_gp.h"13#include "lima_pp.h"14#include "lima_mmu.h"15#include "lima_pmu.h"16#include "lima_l2_cache.h"17#include "lima_dlbu.h"18#include "lima_bcast.h"19#include "lima_vm.h"20 21struct lima_ip_desc {22 char *name;23 char *irq_name;24 bool must_have[lima_gpu_num];25 int offset[lima_gpu_num];26 27 int (*init)(struct lima_ip *ip);28 void (*fini)(struct lima_ip *ip);29 int (*resume)(struct lima_ip *ip);30 void (*suspend)(struct lima_ip *ip);31};32 33#define LIMA_IP_DESC(ipname, mst0, mst1, off0, off1, func, irq) \34 [lima_ip_##ipname] = { \35 .name = #ipname, \36 .irq_name = irq, \37 .must_have = { \38 [lima_gpu_mali400] = mst0, \39 [lima_gpu_mali450] = mst1, \40 }, \41 .offset = { \42 [lima_gpu_mali400] = off0, \43 [lima_gpu_mali450] = off1, \44 }, \45 .init = lima_##func##_init, \46 .fini = lima_##func##_fini, \47 .resume = lima_##func##_resume, \48 .suspend = lima_##func##_suspend, \49 }50 51static struct lima_ip_desc lima_ip_desc[lima_ip_num] = {52 LIMA_IP_DESC(pmu, false, false, 0x02000, 0x02000, pmu, "pmu"),53 LIMA_IP_DESC(l2_cache0, true, true, 0x01000, 0x10000, l2_cache, NULL),54 LIMA_IP_DESC(l2_cache1, false, true, -1, 0x01000, l2_cache, NULL),55 LIMA_IP_DESC(l2_cache2, false, false, -1, 0x11000, l2_cache, NULL),56 LIMA_IP_DESC(gp, true, true, 0x00000, 0x00000, gp, "gp"),57 LIMA_IP_DESC(pp0, true, true, 0x08000, 0x08000, pp, "pp0"),58 LIMA_IP_DESC(pp1, false, false, 0x0A000, 0x0A000, pp, "pp1"),59 LIMA_IP_DESC(pp2, false, false, 0x0C000, 0x0C000, pp, "pp2"),60 LIMA_IP_DESC(pp3, false, false, 0x0E000, 0x0E000, pp, "pp3"),61 LIMA_IP_DESC(pp4, false, false, -1, 0x28000, pp, "pp4"),62 LIMA_IP_DESC(pp5, false, false, -1, 0x2A000, pp, "pp5"),63 LIMA_IP_DESC(pp6, false, false, -1, 0x2C000, pp, "pp6"),64 LIMA_IP_DESC(pp7, false, false, -1, 0x2E000, pp, "pp7"),65 LIMA_IP_DESC(gpmmu, true, true, 0x03000, 0x03000, mmu, "gpmmu"),66 LIMA_IP_DESC(ppmmu0, true, true, 0x04000, 0x04000, mmu, "ppmmu0"),67 LIMA_IP_DESC(ppmmu1, false, false, 0x05000, 0x05000, mmu, "ppmmu1"),68 LIMA_IP_DESC(ppmmu2, false, false, 0x06000, 0x06000, mmu, "ppmmu2"),69 LIMA_IP_DESC(ppmmu3, false, false, 0x07000, 0x07000, mmu, "ppmmu3"),70 LIMA_IP_DESC(ppmmu4, false, false, -1, 0x1C000, mmu, "ppmmu4"),71 LIMA_IP_DESC(ppmmu5, false, false, -1, 0x1D000, mmu, "ppmmu5"),72 LIMA_IP_DESC(ppmmu6, false, false, -1, 0x1E000, mmu, "ppmmu6"),73 LIMA_IP_DESC(ppmmu7, false, false, -1, 0x1F000, mmu, "ppmmu7"),74 LIMA_IP_DESC(dlbu, false, true, -1, 0x14000, dlbu, NULL),75 LIMA_IP_DESC(bcast, false, true, -1, 0x13000, bcast, NULL),76 LIMA_IP_DESC(pp_bcast, false, true, -1, 0x16000, pp_bcast, "pp"),77 LIMA_IP_DESC(ppmmu_bcast, false, true, -1, 0x15000, mmu, NULL),78};79 80const char *lima_ip_name(struct lima_ip *ip)81{82 return lima_ip_desc[ip->id].name;83}84 85static int lima_clk_enable(struct lima_device *dev)86{87 int err;88 89 err = clk_prepare_enable(dev->clk_bus);90 if (err)91 return err;92 93 err = clk_prepare_enable(dev->clk_gpu);94 if (err)95 goto error_out0;96 97 if (dev->reset) {98 err = reset_control_deassert(dev->reset);99 if (err) {100 dev_err(dev->dev,101 "reset controller deassert failed %d\n", err);102 goto error_out1;103 }104 }105 106 return 0;107 108error_out1:109 clk_disable_unprepare(dev->clk_gpu);110error_out0:111 clk_disable_unprepare(dev->clk_bus);112 return err;113}114 115static void lima_clk_disable(struct lima_device *dev)116{117 if (dev->reset)118 reset_control_assert(dev->reset);119 clk_disable_unprepare(dev->clk_gpu);120 clk_disable_unprepare(dev->clk_bus);121}122 123static int lima_clk_init(struct lima_device *dev)124{125 int err;126 127 dev->clk_bus = devm_clk_get(dev->dev, "bus");128 if (IS_ERR(dev->clk_bus)) {129 err = PTR_ERR(dev->clk_bus);130 if (err != -EPROBE_DEFER)131 dev_err(dev->dev, "get bus clk failed %d\n", err);132 dev->clk_bus = NULL;133 return err;134 }135 136 dev->clk_gpu = devm_clk_get(dev->dev, "core");137 if (IS_ERR(dev->clk_gpu)) {138 err = PTR_ERR(dev->clk_gpu);139 if (err != -EPROBE_DEFER)140 dev_err(dev->dev, "get core clk failed %d\n", err);141 dev->clk_gpu = NULL;142 return err;143 }144 145 dev->reset = devm_reset_control_array_get_optional_shared(dev->dev);146 if (IS_ERR(dev->reset)) {147 err = PTR_ERR(dev->reset);148 if (err != -EPROBE_DEFER)149 dev_err(dev->dev, "get reset controller failed %d\n",150 err);151 dev->reset = NULL;152 return err;153 }154 155 return lima_clk_enable(dev);156}157 158static void lima_clk_fini(struct lima_device *dev)159{160 lima_clk_disable(dev);161}162 163static int lima_regulator_enable(struct lima_device *dev)164{165 int ret;166 167 if (!dev->regulator)168 return 0;169 170 ret = regulator_enable(dev->regulator);171 if (ret < 0) {172 dev_err(dev->dev, "failed to enable regulator: %d\n", ret);173 return ret;174 }175 176 return 0;177}178 179static void lima_regulator_disable(struct lima_device *dev)180{181 if (dev->regulator)182 regulator_disable(dev->regulator);183}184 185static int lima_regulator_init(struct lima_device *dev)186{187 int ret;188 189 dev->regulator = devm_regulator_get_optional(dev->dev, "mali");190 if (IS_ERR(dev->regulator)) {191 ret = PTR_ERR(dev->regulator);192 dev->regulator = NULL;193 if (ret == -ENODEV)194 return 0;195 if (ret != -EPROBE_DEFER)196 dev_err(dev->dev, "failed to get regulator: %d\n", ret);197 return ret;198 }199 200 return lima_regulator_enable(dev);201}202 203static void lima_regulator_fini(struct lima_device *dev)204{205 lima_regulator_disable(dev);206}207 208static int lima_init_ip(struct lima_device *dev, int index)209{210 struct platform_device *pdev = to_platform_device(dev->dev);211 struct lima_ip_desc *desc = lima_ip_desc + index;212 struct lima_ip *ip = dev->ip + index;213 const char *irq_name = desc->irq_name;214 int offset = desc->offset[dev->id];215 bool must = desc->must_have[dev->id];216 int err;217 218 if (offset < 0)219 return 0;220 221 ip->dev = dev;222 ip->id = index;223 ip->iomem = dev->iomem + offset;224 if (irq_name) {225 err = must ? platform_get_irq_byname(pdev, irq_name) :226 platform_get_irq_byname_optional(pdev, irq_name);227 if (err < 0)228 goto out;229 ip->irq = err;230 }231 232 err = desc->init(ip);233 if (!err) {234 ip->present = true;235 return 0;236 }237 238out:239 return must ? err : 0;240}241 242static void lima_fini_ip(struct lima_device *ldev, int index)243{244 struct lima_ip_desc *desc = lima_ip_desc + index;245 struct lima_ip *ip = ldev->ip + index;246 247 if (ip->present)248 desc->fini(ip);249}250 251static int lima_resume_ip(struct lima_device *ldev, int index)252{253 struct lima_ip_desc *desc = lima_ip_desc + index;254 struct lima_ip *ip = ldev->ip + index;255 int ret = 0;256 257 if (ip->present)258 ret = desc->resume(ip);259 260 return ret;261}262 263static void lima_suspend_ip(struct lima_device *ldev, int index)264{265 struct lima_ip_desc *desc = lima_ip_desc + index;266 struct lima_ip *ip = ldev->ip + index;267 268 if (ip->present)269 desc->suspend(ip);270}271 272static int lima_init_gp_pipe(struct lima_device *dev)273{274 struct lima_sched_pipe *pipe = dev->pipe + lima_pipe_gp;275 int err;276 277 pipe->ldev = dev;278 279 err = lima_sched_pipe_init(pipe, "gp");280 if (err)281 return err;282 283 pipe->l2_cache[pipe->num_l2_cache++] = dev->ip + lima_ip_l2_cache0;284 pipe->mmu[pipe->num_mmu++] = dev->ip + lima_ip_gpmmu;285 pipe->processor[pipe->num_processor++] = dev->ip + lima_ip_gp;286 287 err = lima_gp_pipe_init(dev);288 if (err) {289 lima_sched_pipe_fini(pipe);290 return err;291 }292 293 return 0;294}295 296static void lima_fini_gp_pipe(struct lima_device *dev)297{298 struct lima_sched_pipe *pipe = dev->pipe + lima_pipe_gp;299 300 lima_gp_pipe_fini(dev);301 lima_sched_pipe_fini(pipe);302}303 304static int lima_init_pp_pipe(struct lima_device *dev)305{306 struct lima_sched_pipe *pipe = dev->pipe + lima_pipe_pp;307 int err, i;308 309 pipe->ldev = dev;310 311 err = lima_sched_pipe_init(pipe, "pp");312 if (err)313 return err;314 315 for (i = 0; i < LIMA_SCHED_PIPE_MAX_PROCESSOR; i++) {316 struct lima_ip *pp = dev->ip + lima_ip_pp0 + i;317 struct lima_ip *ppmmu = dev->ip + lima_ip_ppmmu0 + i;318 struct lima_ip *l2_cache;319 320 if (dev->id == lima_gpu_mali400)321 l2_cache = dev->ip + lima_ip_l2_cache0;322 else323 l2_cache = dev->ip + lima_ip_l2_cache1 + (i >> 2);324 325 if (pp->present && ppmmu->present && l2_cache->present) {326 pipe->mmu[pipe->num_mmu++] = ppmmu;327 pipe->processor[pipe->num_processor++] = pp;328 if (!pipe->l2_cache[i >> 2])329 pipe->l2_cache[pipe->num_l2_cache++] = l2_cache;330 }331 }332 333 if (dev->ip[lima_ip_bcast].present) {334 pipe->bcast_processor = dev->ip + lima_ip_pp_bcast;335 pipe->bcast_mmu = dev->ip + lima_ip_ppmmu_bcast;336 }337 338 err = lima_pp_pipe_init(dev);339 if (err) {340 lima_sched_pipe_fini(pipe);341 return err;342 }343 344 return 0;345}346 347static void lima_fini_pp_pipe(struct lima_device *dev)348{349 struct lima_sched_pipe *pipe = dev->pipe + lima_pipe_pp;350 351 lima_pp_pipe_fini(dev);352 lima_sched_pipe_fini(pipe);353}354 355int lima_device_init(struct lima_device *ldev)356{357 struct platform_device *pdev = to_platform_device(ldev->dev);358 int err, i;359 360 dma_set_coherent_mask(ldev->dev, DMA_BIT_MASK(32));361 dma_set_max_seg_size(ldev->dev, UINT_MAX);362 363 err = lima_clk_init(ldev);364 if (err)365 return err;366 367 err = lima_regulator_init(ldev);368 if (err)369 goto err_out0;370 371 ldev->empty_vm = lima_vm_create(ldev);372 if (!ldev->empty_vm) {373 err = -ENOMEM;374 goto err_out1;375 }376 377 ldev->va_start = 0;378 if (ldev->id == lima_gpu_mali450) {379 ldev->va_end = LIMA_VA_RESERVE_START;380 ldev->dlbu_cpu = dma_alloc_wc(381 ldev->dev, LIMA_PAGE_SIZE,382 &ldev->dlbu_dma, GFP_KERNEL | __GFP_NOWARN);383 if (!ldev->dlbu_cpu) {384 err = -ENOMEM;385 goto err_out2;386 }387 } else388 ldev->va_end = LIMA_VA_RESERVE_END;389 390 ldev->iomem = devm_platform_ioremap_resource(pdev, 0);391 if (IS_ERR(ldev->iomem)) {392 dev_err(ldev->dev, "fail to ioremap iomem\n");393 err = PTR_ERR(ldev->iomem);394 goto err_out3;395 }396 397 for (i = 0; i < lima_ip_num; i++) {398 err = lima_init_ip(ldev, i);399 if (err)400 goto err_out4;401 }402 403 err = lima_init_gp_pipe(ldev);404 if (err)405 goto err_out4;406 407 err = lima_init_pp_pipe(ldev);408 if (err)409 goto err_out5;410 411 ldev->dump.magic = LIMA_DUMP_MAGIC;412 ldev->dump.version_major = LIMA_DUMP_MAJOR;413 ldev->dump.version_minor = LIMA_DUMP_MINOR;414 INIT_LIST_HEAD(&ldev->error_task_list);415 mutex_init(&ldev->error_task_list_lock);416 417 dev_info(ldev->dev, "bus rate = %lu\n", clk_get_rate(ldev->clk_bus));418 dev_info(ldev->dev, "mod rate = %lu", clk_get_rate(ldev->clk_gpu));419 420 return 0;421 422err_out5:423 lima_fini_gp_pipe(ldev);424err_out4:425 while (--i >= 0)426 lima_fini_ip(ldev, i);427err_out3:428 if (ldev->dlbu_cpu)429 dma_free_wc(ldev->dev, LIMA_PAGE_SIZE,430 ldev->dlbu_cpu, ldev->dlbu_dma);431err_out2:432 lima_vm_put(ldev->empty_vm);433err_out1:434 lima_regulator_fini(ldev);435err_out0:436 lima_clk_fini(ldev);437 return err;438}439 440void lima_device_fini(struct lima_device *ldev)441{442 int i;443 struct lima_sched_error_task *et, *tmp;444 445 list_for_each_entry_safe(et, tmp, &ldev->error_task_list, list) {446 list_del(&et->list);447 kvfree(et);448 }449 mutex_destroy(&ldev->error_task_list_lock);450 451 lima_fini_pp_pipe(ldev);452 lima_fini_gp_pipe(ldev);453 454 for (i = lima_ip_num - 1; i >= 0; i--)455 lima_fini_ip(ldev, i);456 457 if (ldev->dlbu_cpu)458 dma_free_wc(ldev->dev, LIMA_PAGE_SIZE,459 ldev->dlbu_cpu, ldev->dlbu_dma);460 461 lima_vm_put(ldev->empty_vm);462 463 lima_regulator_fini(ldev);464 465 lima_clk_fini(ldev);466}467 468int lima_device_resume(struct device *dev)469{470 struct lima_device *ldev = dev_get_drvdata(dev);471 int i, err;472 473 err = lima_clk_enable(ldev);474 if (err) {475 dev_err(dev, "resume clk fail %d\n", err);476 return err;477 }478 479 err = lima_regulator_enable(ldev);480 if (err) {481 dev_err(dev, "resume regulator fail %d\n", err);482 goto err_out0;483 }484 485 for (i = 0; i < lima_ip_num; i++) {486 err = lima_resume_ip(ldev, i);487 if (err) {488 dev_err(dev, "resume ip %d fail\n", i);489 goto err_out1;490 }491 }492 493 err = lima_devfreq_resume(&ldev->devfreq);494 if (err) {495 dev_err(dev, "devfreq resume fail\n");496 goto err_out1;497 }498 499 return 0;500 501err_out1:502 while (--i >= 0)503 lima_suspend_ip(ldev, i);504 lima_regulator_disable(ldev);505err_out0:506 lima_clk_disable(ldev);507 return err;508}509 510int lima_device_suspend(struct device *dev)511{512 struct lima_device *ldev = dev_get_drvdata(dev);513 int i, err;514 515 /* check any task running */516 for (i = 0; i < lima_pipe_num; i++) {517 if (atomic_read(&ldev->pipe[i].base.credit_count))518 return -EBUSY;519 }520 521 err = lima_devfreq_suspend(&ldev->devfreq);522 if (err) {523 dev_err(dev, "devfreq suspend fail\n");524 return err;525 }526 527 for (i = lima_ip_num - 1; i >= 0; i--)528 lima_suspend_ip(ldev, i);529 530 lima_regulator_disable(ldev);531 532 lima_clk_disable(ldev);533 534 return 0;535}536