562 lines · c
1// SPDX-License-Identifier: GPL-2.0 or MIT2/* Copyright 2018 Marty E. Plummer <hanetzer@startmail.com> */3/* Copyright 2019 Linaro, Ltd, Rob Herring <robh@kernel.org> */4/* Copyright 2023 Collabora ltd. */5 6#include <linux/clk.h>7#include <linux/mm.h>8#include <linux/platform_device.h>9#include <linux/pm_domain.h>10#include <linux/pm_runtime.h>11#include <linux/regulator/consumer.h>12#include <linux/reset.h>13 14#include <drm/drm_drv.h>15#include <drm/drm_managed.h>16 17#include "panthor_devfreq.h"18#include "panthor_device.h"19#include "panthor_fw.h"20#include "panthor_gpu.h"21#include "panthor_mmu.h"22#include "panthor_regs.h"23#include "panthor_sched.h"24 25static int panthor_clk_init(struct panthor_device *ptdev)26{27 ptdev->clks.core = devm_clk_get(ptdev->base.dev, NULL);28 if (IS_ERR(ptdev->clks.core))29 return dev_err_probe(ptdev->base.dev,30 PTR_ERR(ptdev->clks.core),31 "get 'core' clock failed");32 33 ptdev->clks.stacks = devm_clk_get_optional(ptdev->base.dev, "stacks");34 if (IS_ERR(ptdev->clks.stacks))35 return dev_err_probe(ptdev->base.dev,36 PTR_ERR(ptdev->clks.stacks),37 "get 'stacks' clock failed");38 39 ptdev->clks.coregroup = devm_clk_get_optional(ptdev->base.dev, "coregroup");40 if (IS_ERR(ptdev->clks.coregroup))41 return dev_err_probe(ptdev->base.dev,42 PTR_ERR(ptdev->clks.coregroup),43 "get 'coregroup' clock failed");44 45 drm_info(&ptdev->base, "clock rate = %lu\n", clk_get_rate(ptdev->clks.core));46 return 0;47}48 49void panthor_device_unplug(struct panthor_device *ptdev)50{51 /* This function can be called from two different path: the reset work52 * and the platform device remove callback. drm_dev_unplug() doesn't53 * deal with concurrent callers, so we have to protect drm_dev_unplug()54 * calls with our own lock, and bail out if the device is already55 * unplugged.56 */57 mutex_lock(&ptdev->unplug.lock);58 if (drm_dev_is_unplugged(&ptdev->base)) {59 /* Someone beat us, release the lock and wait for the unplug60 * operation to be reported as done.61 **/62 mutex_unlock(&ptdev->unplug.lock);63 wait_for_completion(&ptdev->unplug.done);64 return;65 }66 67 /* Call drm_dev_unplug() so any access to HW blocks happening after68 * that point get rejected.69 */70 drm_dev_unplug(&ptdev->base);71 72 /* We do the rest of the unplug with the unplug lock released,73 * future callers will wait on ptdev->unplug.done anyway.74 */75 mutex_unlock(&ptdev->unplug.lock);76 77 drm_WARN_ON(&ptdev->base, pm_runtime_get_sync(ptdev->base.dev) < 0);78 79 /* Now, try to cleanly shutdown the GPU before the device resources80 * get reclaimed.81 */82 panthor_sched_unplug(ptdev);83 panthor_fw_unplug(ptdev);84 panthor_mmu_unplug(ptdev);85 panthor_gpu_unplug(ptdev);86 87 pm_runtime_dont_use_autosuspend(ptdev->base.dev);88 pm_runtime_put_sync_suspend(ptdev->base.dev);89 90 /* If PM is disabled, we need to call the suspend handler manually. */91 if (!IS_ENABLED(CONFIG_PM))92 panthor_device_suspend(ptdev->base.dev);93 94 /* Report the unplug operation as done to unblock concurrent95 * panthor_device_unplug() callers.96 */97 complete_all(&ptdev->unplug.done);98}99 100static void panthor_device_reset_cleanup(struct drm_device *ddev, void *data)101{102 struct panthor_device *ptdev = container_of(ddev, struct panthor_device, base);103 104 cancel_work_sync(&ptdev->reset.work);105 destroy_workqueue(ptdev->reset.wq);106}107 108static void panthor_device_reset_work(struct work_struct *work)109{110 struct panthor_device *ptdev = container_of(work, struct panthor_device, reset.work);111 int ret = 0, cookie;112 113 if (atomic_read(&ptdev->pm.state) != PANTHOR_DEVICE_PM_STATE_ACTIVE) {114 /*115 * No need for a reset as the device has been (or will be)116 * powered down117 */118 atomic_set(&ptdev->reset.pending, 0);119 return;120 }121 122 if (!drm_dev_enter(&ptdev->base, &cookie))123 return;124 125 panthor_sched_pre_reset(ptdev);126 panthor_fw_pre_reset(ptdev, true);127 panthor_mmu_pre_reset(ptdev);128 panthor_gpu_soft_reset(ptdev);129 panthor_gpu_l2_power_on(ptdev);130 panthor_mmu_post_reset(ptdev);131 ret = panthor_fw_post_reset(ptdev);132 atomic_set(&ptdev->reset.pending, 0);133 panthor_sched_post_reset(ptdev, ret != 0);134 drm_dev_exit(cookie);135 136 if (ret) {137 panthor_device_unplug(ptdev);138 drm_err(&ptdev->base, "Failed to boot MCU after reset, making device unusable.");139 }140}141 142static bool panthor_device_is_initialized(struct panthor_device *ptdev)143{144 return !!ptdev->scheduler;145}146 147static void panthor_device_free_page(struct drm_device *ddev, void *data)148{149 __free_page(data);150}151 152int panthor_device_init(struct panthor_device *ptdev)153{154 u32 *dummy_page_virt;155 struct resource *res;156 struct page *p;157 int ret;158 159 ptdev->coherent = device_get_dma_attr(ptdev->base.dev) == DEV_DMA_COHERENT;160 161 init_completion(&ptdev->unplug.done);162 ret = drmm_mutex_init(&ptdev->base, &ptdev->unplug.lock);163 if (ret)164 return ret;165 166 ret = drmm_mutex_init(&ptdev->base, &ptdev->pm.mmio_lock);167 if (ret)168 return ret;169 170 atomic_set(&ptdev->pm.state, PANTHOR_DEVICE_PM_STATE_SUSPENDED);171 p = alloc_page(GFP_KERNEL | __GFP_ZERO);172 if (!p)173 return -ENOMEM;174 175 ptdev->pm.dummy_latest_flush = p;176 dummy_page_virt = page_address(p);177 ret = drmm_add_action_or_reset(&ptdev->base, panthor_device_free_page,178 ptdev->pm.dummy_latest_flush);179 if (ret)180 return ret;181 182 /*183 * Set the dummy page holding the latest flush to 1. This will cause the184 * flush to avoided as we know it isn't necessary if the submission185 * happens while the dummy page is mapped. Zero cannot be used because186 * that means 'always flush'.187 */188 *dummy_page_virt = 1;189 190 INIT_WORK(&ptdev->reset.work, panthor_device_reset_work);191 ptdev->reset.wq = alloc_ordered_workqueue("panthor-reset-wq", 0);192 if (!ptdev->reset.wq)193 return -ENOMEM;194 195 ret = drmm_add_action_or_reset(&ptdev->base, panthor_device_reset_cleanup, NULL);196 if (ret)197 return ret;198 199 ret = panthor_clk_init(ptdev);200 if (ret)201 return ret;202 203 ret = panthor_devfreq_init(ptdev);204 if (ret)205 return ret;206 207 ptdev->iomem = devm_platform_get_and_ioremap_resource(to_platform_device(ptdev->base.dev),208 0, &res);209 if (IS_ERR(ptdev->iomem))210 return PTR_ERR(ptdev->iomem);211 212 ptdev->phys_addr = res->start;213 214 ret = devm_pm_runtime_enable(ptdev->base.dev);215 if (ret)216 return ret;217 218 ret = pm_runtime_resume_and_get(ptdev->base.dev);219 if (ret)220 return ret;221 222 /* If PM is disabled, we need to call panthor_device_resume() manually. */223 if (!IS_ENABLED(CONFIG_PM)) {224 ret = panthor_device_resume(ptdev->base.dev);225 if (ret)226 return ret;227 }228 229 ret = panthor_gpu_init(ptdev);230 if (ret)231 goto err_rpm_put;232 233 ret = panthor_mmu_init(ptdev);234 if (ret)235 goto err_unplug_gpu;236 237 ret = panthor_fw_init(ptdev);238 if (ret)239 goto err_unplug_mmu;240 241 ret = panthor_sched_init(ptdev);242 if (ret)243 goto err_unplug_fw;244 245 /* ~3 frames */246 pm_runtime_set_autosuspend_delay(ptdev->base.dev, 50);247 pm_runtime_use_autosuspend(ptdev->base.dev);248 249 ret = drm_dev_register(&ptdev->base, 0);250 if (ret)251 goto err_disable_autosuspend;252 253 pm_runtime_put_autosuspend(ptdev->base.dev);254 return 0;255 256err_disable_autosuspend:257 pm_runtime_dont_use_autosuspend(ptdev->base.dev);258 panthor_sched_unplug(ptdev);259 260err_unplug_fw:261 panthor_fw_unplug(ptdev);262 263err_unplug_mmu:264 panthor_mmu_unplug(ptdev);265 266err_unplug_gpu:267 panthor_gpu_unplug(ptdev);268 269err_rpm_put:270 pm_runtime_put_sync_suspend(ptdev->base.dev);271 return ret;272}273 274#define PANTHOR_EXCEPTION(id) \275 [DRM_PANTHOR_EXCEPTION_ ## id] = { \276 .name = #id, \277 }278 279struct panthor_exception_info {280 const char *name;281};282 283static const struct panthor_exception_info panthor_exception_infos[] = {284 PANTHOR_EXCEPTION(OK),285 PANTHOR_EXCEPTION(TERMINATED),286 PANTHOR_EXCEPTION(KABOOM),287 PANTHOR_EXCEPTION(EUREKA),288 PANTHOR_EXCEPTION(ACTIVE),289 PANTHOR_EXCEPTION(CS_RES_TERM),290 PANTHOR_EXCEPTION(CS_CONFIG_FAULT),291 PANTHOR_EXCEPTION(CS_UNRECOVERABLE),292 PANTHOR_EXCEPTION(CS_ENDPOINT_FAULT),293 PANTHOR_EXCEPTION(CS_BUS_FAULT),294 PANTHOR_EXCEPTION(CS_INSTR_INVALID),295 PANTHOR_EXCEPTION(CS_CALL_STACK_OVERFLOW),296 PANTHOR_EXCEPTION(CS_INHERIT_FAULT),297 PANTHOR_EXCEPTION(INSTR_INVALID_PC),298 PANTHOR_EXCEPTION(INSTR_INVALID_ENC),299 PANTHOR_EXCEPTION(INSTR_BARRIER_FAULT),300 PANTHOR_EXCEPTION(DATA_INVALID_FAULT),301 PANTHOR_EXCEPTION(TILE_RANGE_FAULT),302 PANTHOR_EXCEPTION(ADDR_RANGE_FAULT),303 PANTHOR_EXCEPTION(IMPRECISE_FAULT),304 PANTHOR_EXCEPTION(OOM),305 PANTHOR_EXCEPTION(CSF_FW_INTERNAL_ERROR),306 PANTHOR_EXCEPTION(CSF_RES_EVICTION_TIMEOUT),307 PANTHOR_EXCEPTION(GPU_BUS_FAULT),308 PANTHOR_EXCEPTION(GPU_SHAREABILITY_FAULT),309 PANTHOR_EXCEPTION(SYS_SHAREABILITY_FAULT),310 PANTHOR_EXCEPTION(GPU_CACHEABILITY_FAULT),311 PANTHOR_EXCEPTION(TRANSLATION_FAULT_0),312 PANTHOR_EXCEPTION(TRANSLATION_FAULT_1),313 PANTHOR_EXCEPTION(TRANSLATION_FAULT_2),314 PANTHOR_EXCEPTION(TRANSLATION_FAULT_3),315 PANTHOR_EXCEPTION(TRANSLATION_FAULT_4),316 PANTHOR_EXCEPTION(PERM_FAULT_0),317 PANTHOR_EXCEPTION(PERM_FAULT_1),318 PANTHOR_EXCEPTION(PERM_FAULT_2),319 PANTHOR_EXCEPTION(PERM_FAULT_3),320 PANTHOR_EXCEPTION(ACCESS_FLAG_1),321 PANTHOR_EXCEPTION(ACCESS_FLAG_2),322 PANTHOR_EXCEPTION(ACCESS_FLAG_3),323 PANTHOR_EXCEPTION(ADDR_SIZE_FAULT_IN),324 PANTHOR_EXCEPTION(ADDR_SIZE_FAULT_OUT0),325 PANTHOR_EXCEPTION(ADDR_SIZE_FAULT_OUT1),326 PANTHOR_EXCEPTION(ADDR_SIZE_FAULT_OUT2),327 PANTHOR_EXCEPTION(ADDR_SIZE_FAULT_OUT3),328 PANTHOR_EXCEPTION(MEM_ATTR_FAULT_0),329 PANTHOR_EXCEPTION(MEM_ATTR_FAULT_1),330 PANTHOR_EXCEPTION(MEM_ATTR_FAULT_2),331 PANTHOR_EXCEPTION(MEM_ATTR_FAULT_3),332};333 334const char *panthor_exception_name(struct panthor_device *ptdev, u32 exception_code)335{336 if (exception_code >= ARRAY_SIZE(panthor_exception_infos) ||337 !panthor_exception_infos[exception_code].name)338 return "Unknown exception type";339 340 return panthor_exception_infos[exception_code].name;341}342 343static vm_fault_t panthor_mmio_vm_fault(struct vm_fault *vmf)344{345 struct vm_area_struct *vma = vmf->vma;346 struct panthor_device *ptdev = vma->vm_private_data;347 u64 offset = (u64)vma->vm_pgoff << PAGE_SHIFT;348 unsigned long pfn;349 pgprot_t pgprot;350 vm_fault_t ret;351 bool active;352 int cookie;353 354 if (!drm_dev_enter(&ptdev->base, &cookie))355 return VM_FAULT_SIGBUS;356 357 mutex_lock(&ptdev->pm.mmio_lock);358 active = atomic_read(&ptdev->pm.state) == PANTHOR_DEVICE_PM_STATE_ACTIVE;359 360 switch (offset) {361 case DRM_PANTHOR_USER_FLUSH_ID_MMIO_OFFSET:362 if (active)363 pfn = __phys_to_pfn(ptdev->phys_addr + CSF_GPU_LATEST_FLUSH_ID);364 else365 pfn = page_to_pfn(ptdev->pm.dummy_latest_flush);366 break;367 368 default:369 ret = VM_FAULT_SIGBUS;370 goto out_unlock;371 }372 373 pgprot = vma->vm_page_prot;374 if (active)375 pgprot = pgprot_noncached(pgprot);376 377 ret = vmf_insert_pfn_prot(vma, vmf->address, pfn, pgprot);378 379out_unlock:380 mutex_unlock(&ptdev->pm.mmio_lock);381 drm_dev_exit(cookie);382 return ret;383}384 385static const struct vm_operations_struct panthor_mmio_vm_ops = {386 .fault = panthor_mmio_vm_fault,387};388 389int panthor_device_mmap_io(struct panthor_device *ptdev, struct vm_area_struct *vma)390{391 u64 offset = (u64)vma->vm_pgoff << PAGE_SHIFT;392 393 if ((vma->vm_flags & VM_SHARED) == 0)394 return -EINVAL;395 396 switch (offset) {397 case DRM_PANTHOR_USER_FLUSH_ID_MMIO_OFFSET:398 if (vma->vm_end - vma->vm_start != PAGE_SIZE ||399 (vma->vm_flags & (VM_WRITE | VM_EXEC)))400 return -EINVAL;401 vm_flags_clear(vma, VM_MAYWRITE);402 403 break;404 405 default:406 return -EINVAL;407 }408 409 /* Defer actual mapping to the fault handler. */410 vma->vm_private_data = ptdev;411 vma->vm_ops = &panthor_mmio_vm_ops;412 vm_flags_set(vma,413 VM_IO | VM_DONTCOPY | VM_DONTEXPAND |414 VM_NORESERVE | VM_DONTDUMP | VM_PFNMAP);415 return 0;416}417 418int panthor_device_resume(struct device *dev)419{420 struct panthor_device *ptdev = dev_get_drvdata(dev);421 int ret, cookie;422 423 if (atomic_read(&ptdev->pm.state) != PANTHOR_DEVICE_PM_STATE_SUSPENDED)424 return -EINVAL;425 426 atomic_set(&ptdev->pm.state, PANTHOR_DEVICE_PM_STATE_RESUMING);427 428 ret = clk_prepare_enable(ptdev->clks.core);429 if (ret)430 goto err_set_suspended;431 432 ret = clk_prepare_enable(ptdev->clks.stacks);433 if (ret)434 goto err_disable_core_clk;435 436 ret = clk_prepare_enable(ptdev->clks.coregroup);437 if (ret)438 goto err_disable_stacks_clk;439 440 ret = panthor_devfreq_resume(ptdev);441 if (ret)442 goto err_disable_coregroup_clk;443 444 if (panthor_device_is_initialized(ptdev) &&445 drm_dev_enter(&ptdev->base, &cookie)) {446 panthor_gpu_resume(ptdev);447 panthor_mmu_resume(ptdev);448 ret = drm_WARN_ON(&ptdev->base, panthor_fw_resume(ptdev));449 if (!ret) {450 panthor_sched_resume(ptdev);451 } else {452 panthor_mmu_suspend(ptdev);453 panthor_gpu_suspend(ptdev);454 }455 456 drm_dev_exit(cookie);457 458 if (ret)459 goto err_suspend_devfreq;460 }461 462 if (atomic_read(&ptdev->reset.pending))463 queue_work(ptdev->reset.wq, &ptdev->reset.work);464 465 /* Clear all IOMEM mappings pointing to this device after we've466 * resumed. This way the fake mappings pointing to the dummy pages467 * are removed and the real iomem mapping will be restored on next468 * access.469 */470 mutex_lock(&ptdev->pm.mmio_lock);471 unmap_mapping_range(ptdev->base.anon_inode->i_mapping,472 DRM_PANTHOR_USER_MMIO_OFFSET, 0, 1);473 atomic_set(&ptdev->pm.state, PANTHOR_DEVICE_PM_STATE_ACTIVE);474 mutex_unlock(&ptdev->pm.mmio_lock);475 return 0;476 477err_suspend_devfreq:478 panthor_devfreq_suspend(ptdev);479 480err_disable_coregroup_clk:481 clk_disable_unprepare(ptdev->clks.coregroup);482 483err_disable_stacks_clk:484 clk_disable_unprepare(ptdev->clks.stacks);485 486err_disable_core_clk:487 clk_disable_unprepare(ptdev->clks.core);488 489err_set_suspended:490 atomic_set(&ptdev->pm.state, PANTHOR_DEVICE_PM_STATE_SUSPENDED);491 return ret;492}493 494int panthor_device_suspend(struct device *dev)495{496 struct panthor_device *ptdev = dev_get_drvdata(dev);497 int ret, cookie;498 499 if (atomic_read(&ptdev->pm.state) != PANTHOR_DEVICE_PM_STATE_ACTIVE)500 return -EINVAL;501 502 /* Clear all IOMEM mappings pointing to this device before we503 * shutdown the power-domain and clocks. Failing to do that results504 * in external aborts when the process accesses the iomem region.505 * We change the state and call unmap_mapping_range() with the506 * mmio_lock held to make sure the vm_fault handler won't set up507 * invalid mappings.508 */509 mutex_lock(&ptdev->pm.mmio_lock);510 atomic_set(&ptdev->pm.state, PANTHOR_DEVICE_PM_STATE_SUSPENDING);511 unmap_mapping_range(ptdev->base.anon_inode->i_mapping,512 DRM_PANTHOR_USER_MMIO_OFFSET, 0, 1);513 mutex_unlock(&ptdev->pm.mmio_lock);514 515 if (panthor_device_is_initialized(ptdev) &&516 drm_dev_enter(&ptdev->base, &cookie)) {517 cancel_work_sync(&ptdev->reset.work);518 519 /* We prepare everything as if we were resetting the GPU.520 * The end of the reset will happen in the resume path though.521 */522 panthor_sched_suspend(ptdev);523 panthor_fw_suspend(ptdev);524 panthor_mmu_suspend(ptdev);525 panthor_gpu_suspend(ptdev);526 drm_dev_exit(cookie);527 }528 529 ret = panthor_devfreq_suspend(ptdev);530 if (ret) {531 if (panthor_device_is_initialized(ptdev) &&532 drm_dev_enter(&ptdev->base, &cookie)) {533 panthor_gpu_resume(ptdev);534 panthor_mmu_resume(ptdev);535 drm_WARN_ON(&ptdev->base, panthor_fw_resume(ptdev));536 panthor_sched_resume(ptdev);537 drm_dev_exit(cookie);538 }539 540 goto err_set_active;541 }542 543 clk_disable_unprepare(ptdev->clks.coregroup);544 clk_disable_unprepare(ptdev->clks.stacks);545 clk_disable_unprepare(ptdev->clks.core);546 atomic_set(&ptdev->pm.state, PANTHOR_DEVICE_PM_STATE_SUSPENDED);547 return 0;548 549err_set_active:550 /* If something failed and we have to revert back to an551 * active state, we also need to clear the MMIO userspace552 * mappings, so any dumb pages that were mapped while we553 * were trying to suspend gets invalidated.554 */555 mutex_lock(&ptdev->pm.mmio_lock);556 atomic_set(&ptdev->pm.state, PANTHOR_DEVICE_PM_STATE_ACTIVE);557 unmap_mapping_range(ptdev->base.anon_inode->i_mapping,558 DRM_PANTHOR_USER_MMIO_OFFSET, 0, 1);559 mutex_unlock(&ptdev->pm.mmio_lock);560 return ret;561}562