1149 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2/*3 * Copyright (C) 2013 Red Hat4 * Author: Rob Clark <robdclark@gmail.com>5 *6 * Copyright (c) 2014 The Linux Foundation. All rights reserved.7 */8 9#include <linux/ascii85.h>10#include <linux/interconnect.h>11#include <linux/firmware/qcom/qcom_scm.h>12#include <linux/kernel.h>13#include <linux/of_address.h>14#include <linux/pm_opp.h>15#include <linux/slab.h>16#include <linux/soc/qcom/mdt_loader.h>17#include <linux/nvmem-consumer.h>18#include <soc/qcom/ocmem.h>19#include "adreno_gpu.h"20#include "a6xx_gpu.h"21#include "msm_gem.h"22#include "msm_mmu.h"23 24static u64 address_space_size = 0;25MODULE_PARM_DESC(address_space_size, "Override for size of processes private GPU address space");26module_param(address_space_size, ullong, 0600);27 28static bool zap_available = true;29 30static int zap_shader_load_mdt(struct msm_gpu *gpu, const char *fwname,31 u32 pasid)32{33 struct device *dev = &gpu->pdev->dev;34 const struct firmware *fw;35 const char *signed_fwname = NULL;36 struct device_node *np, *mem_np;37 struct resource r;38 phys_addr_t mem_phys;39 ssize_t mem_size;40 void *mem_region = NULL;41 int ret;42 43 if (!IS_ENABLED(CONFIG_ARCH_QCOM)) {44 zap_available = false;45 return -EINVAL;46 }47 48 np = of_get_child_by_name(dev->of_node, "zap-shader");49 if (!of_device_is_available(np)) {50 zap_available = false;51 return -ENODEV;52 }53 54 mem_np = of_parse_phandle(np, "memory-region", 0);55 of_node_put(np);56 if (!mem_np) {57 zap_available = false;58 return -EINVAL;59 }60 61 ret = of_address_to_resource(mem_np, 0, &r);62 of_node_put(mem_np);63 if (ret)64 return ret;65 66 mem_phys = r.start;67 68 /*69 * Check for a firmware-name property. This is the new scheme70 * to handle firmware that may be signed with device specific71 * keys, allowing us to have a different zap fw path for different72 * devices.73 *74 * If the firmware-name property is found, we bypass the75 * adreno_request_fw() mechanism, because we don't need to handle76 * the /lib/firmware/qcom/... vs /lib/firmware/... case.77 *78 * If the firmware-name property is not found, for backwards79 * compatibility we fall back to the fwname from the gpulist80 * table.81 */82 of_property_read_string_index(np, "firmware-name", 0, &signed_fwname);83 if (signed_fwname) {84 fwname = signed_fwname;85 ret = request_firmware_direct(&fw, fwname, gpu->dev->dev);86 if (ret)87 fw = ERR_PTR(ret);88 } else if (fwname) {89 /* Request the MDT file from the default location: */90 fw = adreno_request_fw(to_adreno_gpu(gpu), fwname);91 } else {92 /*93 * For new targets, we require the firmware-name property,94 * if a zap-shader is required, rather than falling back95 * to a firmware name specified in gpulist.96 *97 * Because the firmware is signed with a (potentially)98 * device specific key, having the name come from gpulist99 * was a bad idea, and is only provided for backwards100 * compatibility for older targets.101 */102 return -ENOENT;103 }104 105 if (IS_ERR(fw)) {106 DRM_DEV_ERROR(dev, "Unable to load %s\n", fwname);107 return PTR_ERR(fw);108 }109 110 /* Figure out how much memory we need */111 mem_size = qcom_mdt_get_size(fw);112 if (mem_size < 0) {113 ret = mem_size;114 goto out;115 }116 117 if (mem_size > resource_size(&r)) {118 DRM_DEV_ERROR(dev,119 "memory region is too small to load the MDT\n");120 ret = -E2BIG;121 goto out;122 }123 124 /* Allocate memory for the firmware image */125 mem_region = memremap(mem_phys, mem_size, MEMREMAP_WC);126 if (!mem_region) {127 ret = -ENOMEM;128 goto out;129 }130 131 /*132 * Load the rest of the MDT133 *134 * Note that we could be dealing with two different paths, since135 * with upstream linux-firmware it would be in a qcom/ subdir..136 * adreno_request_fw() handles this, but qcom_mdt_load() does137 * not. But since we've already gotten through adreno_request_fw()138 * we know which of the two cases it is:139 */140 if (signed_fwname || (to_adreno_gpu(gpu)->fwloc == FW_LOCATION_LEGACY)) {141 ret = qcom_mdt_load(dev, fw, fwname, pasid,142 mem_region, mem_phys, mem_size, NULL);143 } else {144 char *newname;145 146 newname = kasprintf(GFP_KERNEL, "qcom/%s", fwname);147 148 ret = qcom_mdt_load(dev, fw, newname, pasid,149 mem_region, mem_phys, mem_size, NULL);150 kfree(newname);151 }152 if (ret)153 goto out;154 155 /* Send the image to the secure world */156 ret = qcom_scm_pas_auth_and_reset(pasid);157 158 /*159 * If the scm call returns -EOPNOTSUPP we assume that this target160 * doesn't need/support the zap shader so quietly fail161 */162 if (ret == -EOPNOTSUPP)163 zap_available = false;164 else if (ret)165 DRM_DEV_ERROR(dev, "Unable to authorize the image\n");166 167out:168 if (mem_region)169 memunmap(mem_region);170 171 release_firmware(fw);172 173 return ret;174}175 176int adreno_zap_shader_load(struct msm_gpu *gpu, u32 pasid)177{178 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);179 struct platform_device *pdev = gpu->pdev;180 181 /* Short cut if we determine the zap shader isn't available/needed */182 if (!zap_available)183 return -ENODEV;184 185 /* We need SCM to be able to load the firmware */186 if (!qcom_scm_is_available()) {187 DRM_DEV_ERROR(&pdev->dev, "SCM is not available\n");188 return -EPROBE_DEFER;189 }190 191 return zap_shader_load_mdt(gpu, adreno_gpu->info->zapfw, pasid);192}193 194struct msm_gem_address_space *195adreno_create_address_space(struct msm_gpu *gpu,196 struct platform_device *pdev)197{198 return adreno_iommu_create_address_space(gpu, pdev, 0);199}200 201struct msm_gem_address_space *202adreno_iommu_create_address_space(struct msm_gpu *gpu,203 struct platform_device *pdev,204 unsigned long quirks)205{206 struct iommu_domain_geometry *geometry;207 struct msm_mmu *mmu;208 struct msm_gem_address_space *aspace;209 u64 start, size;210 211 mmu = msm_iommu_gpu_new(&pdev->dev, gpu, quirks);212 if (IS_ERR_OR_NULL(mmu))213 return ERR_CAST(mmu);214 215 geometry = msm_iommu_get_geometry(mmu);216 if (IS_ERR(geometry))217 return ERR_CAST(geometry);218 219 /*220 * Use the aperture start or SZ_16M, whichever is greater. This will221 * ensure that we align with the allocated pagetable range while still222 * allowing room in the lower 32 bits for GMEM and whatnot223 */224 start = max_t(u64, SZ_16M, geometry->aperture_start);225 size = geometry->aperture_end - start + 1;226 227 aspace = msm_gem_address_space_create(mmu, "gpu",228 start & GENMASK_ULL(48, 0), size);229 230 if (IS_ERR(aspace) && !IS_ERR(mmu))231 mmu->funcs->destroy(mmu);232 233 return aspace;234}235 236u64 adreno_private_address_space_size(struct msm_gpu *gpu)237{238 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);239 240 if (address_space_size)241 return address_space_size;242 243 if (adreno_gpu->info->address_space_size)244 return adreno_gpu->info->address_space_size;245 246 return SZ_4G;247}248 249#define ARM_SMMU_FSR_TF BIT(1)250#define ARM_SMMU_FSR_PF BIT(3)251#define ARM_SMMU_FSR_EF BIT(4)252 253int adreno_fault_handler(struct msm_gpu *gpu, unsigned long iova, int flags,254 struct adreno_smmu_fault_info *info, const char *block,255 u32 scratch[4])256{257 const char *type = "UNKNOWN";258 bool do_devcoredump = info && !READ_ONCE(gpu->crashstate);259 260 /*261 * If we aren't going to be resuming later from fault_worker, then do262 * it now.263 */264 if (!do_devcoredump) {265 gpu->aspace->mmu->funcs->resume_translation(gpu->aspace->mmu);266 }267 268 /*269 * Print a default message if we couldn't get the data from the270 * adreno-smmu-priv271 */272 if (!info) {273 pr_warn_ratelimited("*** gpu fault: iova=%.16lx flags=%d (%u,%u,%u,%u)\n",274 iova, flags,275 scratch[0], scratch[1], scratch[2], scratch[3]);276 277 return 0;278 }279 280 if (info->fsr & ARM_SMMU_FSR_TF)281 type = "TRANSLATION";282 else if (info->fsr & ARM_SMMU_FSR_PF)283 type = "PERMISSION";284 else if (info->fsr & ARM_SMMU_FSR_EF)285 type = "EXTERNAL";286 287 pr_warn_ratelimited("*** gpu fault: ttbr0=%.16llx iova=%.16lx dir=%s type=%s source=%s (%u,%u,%u,%u)\n",288 info->ttbr0, iova,289 flags & IOMMU_FAULT_WRITE ? "WRITE" : "READ",290 type, block,291 scratch[0], scratch[1], scratch[2], scratch[3]);292 293 if (do_devcoredump) {294 /* Turn off the hangcheck timer to keep it from bothering us */295 del_timer(&gpu->hangcheck_timer);296 297 gpu->fault_info.ttbr0 = info->ttbr0;298 gpu->fault_info.iova = iova;299 gpu->fault_info.flags = flags;300 gpu->fault_info.type = type;301 gpu->fault_info.block = block;302 303 kthread_queue_work(gpu->worker, &gpu->fault_work);304 }305 306 return 0;307}308 309int adreno_get_param(struct msm_gpu *gpu, struct msm_file_private *ctx,310 uint32_t param, uint64_t *value, uint32_t *len)311{312 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);313 314 /* No pointer params yet */315 if (*len != 0)316 return -EINVAL;317 318 switch (param) {319 case MSM_PARAM_GPU_ID:320 *value = adreno_gpu->info->revn;321 return 0;322 case MSM_PARAM_GMEM_SIZE:323 *value = adreno_gpu->info->gmem;324 return 0;325 case MSM_PARAM_GMEM_BASE:326 if (adreno_is_a650_family(adreno_gpu) ||327 adreno_is_a740_family(adreno_gpu))328 *value = 0;329 else330 *value = 0x100000;331 return 0;332 case MSM_PARAM_CHIP_ID:333 *value = adreno_gpu->chip_id;334 if (!adreno_gpu->info->revn)335 *value |= ((uint64_t) adreno_gpu->speedbin) << 32;336 return 0;337 case MSM_PARAM_MAX_FREQ:338 *value = adreno_gpu->base.fast_rate;339 return 0;340 case MSM_PARAM_TIMESTAMP:341 if (adreno_gpu->funcs->get_timestamp) {342 int ret;343 344 pm_runtime_get_sync(&gpu->pdev->dev);345 ret = adreno_gpu->funcs->get_timestamp(gpu, value);346 pm_runtime_put_autosuspend(&gpu->pdev->dev);347 348 return ret;349 }350 return -EINVAL;351 case MSM_PARAM_PRIORITIES:352 *value = gpu->nr_rings * NR_SCHED_PRIORITIES;353 return 0;354 case MSM_PARAM_PP_PGTABLE:355 *value = 0;356 return 0;357 case MSM_PARAM_FAULTS:358 if (ctx->aspace)359 *value = gpu->global_faults + ctx->aspace->faults;360 else361 *value = gpu->global_faults;362 return 0;363 case MSM_PARAM_SUSPENDS:364 *value = gpu->suspend_count;365 return 0;366 case MSM_PARAM_VA_START:367 if (ctx->aspace == gpu->aspace)368 return -EINVAL;369 *value = ctx->aspace->va_start;370 return 0;371 case MSM_PARAM_VA_SIZE:372 if (ctx->aspace == gpu->aspace)373 return -EINVAL;374 *value = ctx->aspace->va_size;375 return 0;376 case MSM_PARAM_HIGHEST_BANK_BIT:377 *value = adreno_gpu->ubwc_config.highest_bank_bit;378 return 0;379 case MSM_PARAM_RAYTRACING:380 *value = adreno_gpu->has_ray_tracing;381 return 0;382 case MSM_PARAM_UBWC_SWIZZLE:383 *value = adreno_gpu->ubwc_config.ubwc_swizzle;384 return 0;385 case MSM_PARAM_MACROTILE_MODE:386 *value = adreno_gpu->ubwc_config.macrotile_mode;387 return 0;388 default:389 DBG("%s: invalid param: %u", gpu->name, param);390 return -EINVAL;391 }392}393 394int adreno_set_param(struct msm_gpu *gpu, struct msm_file_private *ctx,395 uint32_t param, uint64_t value, uint32_t len)396{397 switch (param) {398 case MSM_PARAM_COMM:399 case MSM_PARAM_CMDLINE:400 /* kstrdup_quotable_cmdline() limits to PAGE_SIZE, so401 * that should be a reasonable upper bound402 */403 if (len > PAGE_SIZE)404 return -EINVAL;405 break;406 default:407 if (len != 0)408 return -EINVAL;409 }410 411 switch (param) {412 case MSM_PARAM_COMM:413 case MSM_PARAM_CMDLINE: {414 char *str, **paramp;415 416 str = memdup_user_nul(u64_to_user_ptr(value), len);417 if (IS_ERR(str))418 return PTR_ERR(str);419 420 mutex_lock(&gpu->lock);421 422 if (param == MSM_PARAM_COMM) {423 paramp = &ctx->comm;424 } else {425 paramp = &ctx->cmdline;426 }427 428 kfree(*paramp);429 *paramp = str;430 431 mutex_unlock(&gpu->lock);432 433 return 0;434 }435 case MSM_PARAM_SYSPROF:436 if (!capable(CAP_SYS_ADMIN))437 return -EPERM;438 return msm_file_private_set_sysprof(ctx, gpu, value);439 default:440 DBG("%s: invalid param: %u", gpu->name, param);441 return -EINVAL;442 }443}444 445const struct firmware *446adreno_request_fw(struct adreno_gpu *adreno_gpu, const char *fwname)447{448 struct drm_device *drm = adreno_gpu->base.dev;449 const struct firmware *fw = NULL;450 char *newname;451 int ret;452 453 newname = kasprintf(GFP_KERNEL, "qcom/%s", fwname);454 if (!newname)455 return ERR_PTR(-ENOMEM);456 457 /*458 * Try first to load from qcom/$fwfile using a direct load (to avoid459 * a potential timeout waiting for usermode helper)460 */461 if ((adreno_gpu->fwloc == FW_LOCATION_UNKNOWN) ||462 (adreno_gpu->fwloc == FW_LOCATION_NEW)) {463 464 ret = request_firmware_direct(&fw, newname, drm->dev);465 if (!ret) {466 DRM_DEV_INFO(drm->dev, "loaded %s from new location\n",467 newname);468 adreno_gpu->fwloc = FW_LOCATION_NEW;469 goto out;470 } else if (adreno_gpu->fwloc != FW_LOCATION_UNKNOWN) {471 DRM_DEV_ERROR(drm->dev, "failed to load %s: %d\n",472 newname, ret);473 fw = ERR_PTR(ret);474 goto out;475 }476 }477 478 /*479 * Then try the legacy location without qcom/ prefix480 */481 if ((adreno_gpu->fwloc == FW_LOCATION_UNKNOWN) ||482 (adreno_gpu->fwloc == FW_LOCATION_LEGACY)) {483 484 ret = request_firmware_direct(&fw, fwname, drm->dev);485 if (!ret) {486 DRM_DEV_INFO(drm->dev, "loaded %s from legacy location\n",487 fwname);488 adreno_gpu->fwloc = FW_LOCATION_LEGACY;489 goto out;490 } else if (adreno_gpu->fwloc != FW_LOCATION_UNKNOWN) {491 DRM_DEV_ERROR(drm->dev, "failed to load %s: %d\n",492 fwname, ret);493 fw = ERR_PTR(ret);494 goto out;495 }496 }497 498 /*499 * Finally fall back to request_firmware() for cases where the500 * usermode helper is needed (I think mainly android)501 */502 if ((adreno_gpu->fwloc == FW_LOCATION_UNKNOWN) ||503 (adreno_gpu->fwloc == FW_LOCATION_HELPER)) {504 505 ret = request_firmware(&fw, newname, drm->dev);506 if (!ret) {507 DRM_DEV_INFO(drm->dev, "loaded %s with helper\n",508 newname);509 adreno_gpu->fwloc = FW_LOCATION_HELPER;510 goto out;511 } else if (adreno_gpu->fwloc != FW_LOCATION_UNKNOWN) {512 DRM_DEV_ERROR(drm->dev, "failed to load %s: %d\n",513 newname, ret);514 fw = ERR_PTR(ret);515 goto out;516 }517 }518 519 DRM_DEV_ERROR(drm->dev, "failed to load %s\n", fwname);520 fw = ERR_PTR(-ENOENT);521out:522 kfree(newname);523 return fw;524}525 526int adreno_load_fw(struct adreno_gpu *adreno_gpu)527{528 int i;529 530 for (i = 0; i < ARRAY_SIZE(adreno_gpu->info->fw); i++) {531 const struct firmware *fw;532 533 if (!adreno_gpu->info->fw[i])534 continue;535 536 /* Skip loading GMU firwmare with GMU Wrapper */537 if (adreno_has_gmu_wrapper(adreno_gpu) && i == ADRENO_FW_GMU)538 continue;539 540 /* Skip if the firmware has already been loaded */541 if (adreno_gpu->fw[i])542 continue;543 544 fw = adreno_request_fw(adreno_gpu, adreno_gpu->info->fw[i]);545 if (IS_ERR(fw))546 return PTR_ERR(fw);547 548 adreno_gpu->fw[i] = fw;549 }550 551 return 0;552}553 554struct drm_gem_object *adreno_fw_create_bo(struct msm_gpu *gpu,555 const struct firmware *fw, u64 *iova)556{557 struct drm_gem_object *bo;558 void *ptr;559 560 ptr = msm_gem_kernel_new(gpu->dev, fw->size - 4,561 MSM_BO_WC | MSM_BO_GPU_READONLY, gpu->aspace, &bo, iova);562 563 if (IS_ERR(ptr))564 return ERR_CAST(ptr);565 566 memcpy(ptr, &fw->data[4], fw->size - 4);567 568 msm_gem_put_vaddr(bo);569 570 return bo;571}572 573int adreno_hw_init(struct msm_gpu *gpu)574{575 VERB("%s", gpu->name);576 577 for (int i = 0; i < gpu->nr_rings; i++) {578 struct msm_ringbuffer *ring = gpu->rb[i];579 580 if (!ring)581 continue;582 583 ring->cur = ring->start;584 ring->next = ring->start;585 ring->memptrs->rptr = 0;586 ring->memptrs->bv_fence = ring->fctx->completed_fence;587 588 /* Detect and clean up an impossible fence, ie. if GPU managed589 * to scribble something invalid, we don't want that to confuse590 * us into mistakingly believing that submits have completed.591 */592 if (fence_before(ring->fctx->last_fence, ring->memptrs->fence)) {593 ring->memptrs->fence = ring->fctx->last_fence;594 }595 }596 597 return 0;598}599 600/* Use this helper to read rptr, since a430 doesn't update rptr in memory */601static uint32_t get_rptr(struct adreno_gpu *adreno_gpu,602 struct msm_ringbuffer *ring)603{604 struct msm_gpu *gpu = &adreno_gpu->base;605 606 return gpu->funcs->get_rptr(gpu, ring);607}608 609struct msm_ringbuffer *adreno_active_ring(struct msm_gpu *gpu)610{611 return gpu->rb[0];612}613 614void adreno_recover(struct msm_gpu *gpu)615{616 struct drm_device *dev = gpu->dev;617 int ret;618 619 // XXX pm-runtime?? we *need* the device to be off after this620 // so maybe continuing to call ->pm_suspend/resume() is better?621 622 gpu->funcs->pm_suspend(gpu);623 gpu->funcs->pm_resume(gpu);624 625 ret = msm_gpu_hw_init(gpu);626 if (ret) {627 DRM_DEV_ERROR(dev->dev, "gpu hw init failed: %d\n", ret);628 /* hmm, oh well? */629 }630}631 632void adreno_flush(struct msm_gpu *gpu, struct msm_ringbuffer *ring, u32 reg)633{634 uint32_t wptr;635 636 /* Copy the shadow to the actual register */637 ring->cur = ring->next;638 639 /*640 * Mask wptr value that we calculate to fit in the HW range. This is641 * to account for the possibility that the last command fit exactly into642 * the ringbuffer and rb->next hasn't wrapped to zero yet643 */644 wptr = get_wptr(ring);645 646 /* ensure writes to ringbuffer have hit system memory: */647 mb();648 649 gpu_write(gpu, reg, wptr);650}651 652bool adreno_idle(struct msm_gpu *gpu, struct msm_ringbuffer *ring)653{654 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);655 uint32_t wptr = get_wptr(ring);656 657 /* wait for CP to drain ringbuffer: */658 if (!spin_until(get_rptr(adreno_gpu, ring) == wptr))659 return true;660 661 /* TODO maybe we need to reset GPU here to recover from hang? */662 DRM_ERROR("%s: timeout waiting to drain ringbuffer %d rptr/wptr = %X/%X\n",663 gpu->name, ring->id, get_rptr(adreno_gpu, ring), wptr);664 665 return false;666}667 668int adreno_gpu_state_get(struct msm_gpu *gpu, struct msm_gpu_state *state)669{670 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);671 int i, count = 0;672 673 WARN_ON(!mutex_is_locked(&gpu->lock));674 675 kref_init(&state->ref);676 677 ktime_get_real_ts64(&state->time);678 679 for (i = 0; i < gpu->nr_rings; i++) {680 int size = 0, j;681 682 state->ring[i].fence = gpu->rb[i]->memptrs->fence;683 state->ring[i].iova = gpu->rb[i]->iova;684 state->ring[i].seqno = gpu->rb[i]->fctx->last_fence;685 state->ring[i].rptr = get_rptr(adreno_gpu, gpu->rb[i]);686 state->ring[i].wptr = get_wptr(gpu->rb[i]);687 688 /* Copy at least 'wptr' dwords of the data */689 size = state->ring[i].wptr;690 691 /* After wptr find the last non zero dword to save space */692 for (j = state->ring[i].wptr; j < MSM_GPU_RINGBUFFER_SZ >> 2; j++)693 if (gpu->rb[i]->start[j])694 size = j + 1;695 696 if (size) {697 state->ring[i].data = kvmemdup(gpu->rb[i]->start, size << 2, GFP_KERNEL);698 if (state->ring[i].data)699 state->ring[i].data_size = size << 2;700 }701 }702 703 /* Some targets prefer to collect their own registers */704 if (!adreno_gpu->registers)705 return 0;706 707 /* Count the number of registers */708 for (i = 0; adreno_gpu->registers[i] != ~0; i += 2)709 count += adreno_gpu->registers[i + 1] -710 adreno_gpu->registers[i] + 1;711 712 state->registers = kcalloc(count * 2, sizeof(u32), GFP_KERNEL);713 if (state->registers) {714 int pos = 0;715 716 for (i = 0; adreno_gpu->registers[i] != ~0; i += 2) {717 u32 start = adreno_gpu->registers[i];718 u32 end = adreno_gpu->registers[i + 1];719 u32 addr;720 721 for (addr = start; addr <= end; addr++) {722 state->registers[pos++] = addr;723 state->registers[pos++] = gpu_read(gpu, addr);724 }725 }726 727 state->nr_registers = count;728 }729 730 return 0;731}732 733void adreno_gpu_state_destroy(struct msm_gpu_state *state)734{735 int i;736 737 for (i = 0; i < ARRAY_SIZE(state->ring); i++)738 kvfree(state->ring[i].data);739 740 for (i = 0; state->bos && i < state->nr_bos; i++)741 kvfree(state->bos[i].data);742 743 kfree(state->bos);744 kfree(state->comm);745 kfree(state->cmd);746 kfree(state->registers);747}748 749static void adreno_gpu_state_kref_destroy(struct kref *kref)750{751 struct msm_gpu_state *state = container_of(kref,752 struct msm_gpu_state, ref);753 754 adreno_gpu_state_destroy(state);755 kfree(state);756}757 758int adreno_gpu_state_put(struct msm_gpu_state *state)759{760 if (IS_ERR_OR_NULL(state))761 return 1;762 763 return kref_put(&state->ref, adreno_gpu_state_kref_destroy);764}765 766#if defined(CONFIG_DEBUG_FS) || defined(CONFIG_DEV_COREDUMP)767 768static char *adreno_gpu_ascii85_encode(u32 *src, size_t len)769{770 void *buf;771 size_t buf_itr = 0, buffer_size;772 char out[ASCII85_BUFSZ];773 long l;774 int i;775 776 if (!src || !len)777 return NULL;778 779 l = ascii85_encode_len(len);780 781 /*782 * Ascii85 outputs either a 5 byte string or a 1 byte string. So we783 * account for the worst case of 5 bytes per dword plus the 1 for '\0'784 */785 buffer_size = (l * 5) + 1;786 787 buf = kvmalloc(buffer_size, GFP_KERNEL);788 if (!buf)789 return NULL;790 791 for (i = 0; i < l; i++)792 buf_itr += scnprintf(buf + buf_itr, buffer_size - buf_itr, "%s",793 ascii85_encode(src[i], out));794 795 return buf;796}797 798/* len is expected to be in bytes799 *800 * WARNING: *ptr should be allocated with kvmalloc or friends. It can be free'd801 * with kvfree() and replaced with a newly kvmalloc'd buffer on the first call802 * when the unencoded raw data is encoded803 */804void adreno_show_object(struct drm_printer *p, void **ptr, int len,805 bool *encoded)806{807 if (!*ptr || !len)808 return;809 810 if (!*encoded) {811 long datalen, i;812 u32 *buf = *ptr;813 814 /*815 * Only dump the non-zero part of the buffer - rarely will816 * any data completely fill the entire allocated size of817 * the buffer.818 */819 for (datalen = 0, i = 0; i < len >> 2; i++)820 if (buf[i])821 datalen = ((i + 1) << 2);822 823 /*824 * If we reach here, then the originally captured binary buffer825 * will be replaced with the ascii85 encoded string826 */827 *ptr = adreno_gpu_ascii85_encode(buf, datalen);828 829 kvfree(buf);830 831 *encoded = true;832 }833 834 if (!*ptr)835 return;836 837 drm_puts(p, " data: !!ascii85 |\n");838 drm_puts(p, " ");839 840 drm_puts(p, *ptr);841 842 drm_puts(p, "\n");843}844 845void adreno_show(struct msm_gpu *gpu, struct msm_gpu_state *state,846 struct drm_printer *p)847{848 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);849 int i;850 851 if (IS_ERR_OR_NULL(state))852 return;853 854 drm_printf(p, "revision: %u (%"ADRENO_CHIPID_FMT")\n",855 adreno_gpu->info->revn,856 ADRENO_CHIPID_ARGS(adreno_gpu->chip_id));857 /*858 * If this is state collected due to iova fault, so fault related info859 *860 * TTBR0 would not be zero, so this is a good way to distinguish861 */862 if (state->fault_info.ttbr0) {863 const struct msm_gpu_fault_info *info = &state->fault_info;864 865 drm_puts(p, "fault-info:\n");866 drm_printf(p, " - ttbr0=%.16llx\n", info->ttbr0);867 drm_printf(p, " - iova=%.16lx\n", info->iova);868 drm_printf(p, " - dir=%s\n", info->flags & IOMMU_FAULT_WRITE ? "WRITE" : "READ");869 drm_printf(p, " - type=%s\n", info->type);870 drm_printf(p, " - source=%s\n", info->block);871 }872 873 drm_printf(p, "rbbm-status: 0x%08x\n", state->rbbm_status);874 875 drm_puts(p, "ringbuffer:\n");876 877 for (i = 0; i < gpu->nr_rings; i++) {878 drm_printf(p, " - id: %d\n", i);879 drm_printf(p, " iova: 0x%016llx\n", state->ring[i].iova);880 drm_printf(p, " last-fence: %u\n", state->ring[i].seqno);881 drm_printf(p, " retired-fence: %u\n", state->ring[i].fence);882 drm_printf(p, " rptr: %u\n", state->ring[i].rptr);883 drm_printf(p, " wptr: %u\n", state->ring[i].wptr);884 drm_printf(p, " size: %u\n", MSM_GPU_RINGBUFFER_SZ);885 886 adreno_show_object(p, &state->ring[i].data,887 state->ring[i].data_size, &state->ring[i].encoded);888 }889 890 if (state->bos) {891 drm_puts(p, "bos:\n");892 893 for (i = 0; i < state->nr_bos; i++) {894 drm_printf(p, " - iova: 0x%016llx\n",895 state->bos[i].iova);896 drm_printf(p, " size: %zd\n", state->bos[i].size);897 drm_printf(p, " flags: 0x%x\n", state->bos[i].flags);898 drm_printf(p, " name: %-32s\n", state->bos[i].name);899 900 adreno_show_object(p, &state->bos[i].data,901 state->bos[i].size, &state->bos[i].encoded);902 }903 }904 905 if (state->nr_registers) {906 drm_puts(p, "registers:\n");907 908 for (i = 0; i < state->nr_registers; i++) {909 drm_printf(p, " - { offset: 0x%04x, value: 0x%08x }\n",910 state->registers[i * 2] << 2,911 state->registers[(i * 2) + 1]);912 }913 }914}915#endif916 917/* Dump common gpu status and scratch registers on any hang, to make918 * the hangcheck logs more useful. The scratch registers seem always919 * safe to read when GPU has hung (unlike some other regs, depending920 * on how the GPU hung), and they are useful to match up to cmdstream921 * dumps when debugging hangs:922 */923void adreno_dump_info(struct msm_gpu *gpu)924{925 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);926 int i;927 928 printk("revision: %u (%"ADRENO_CHIPID_FMT")\n",929 adreno_gpu->info->revn,930 ADRENO_CHIPID_ARGS(adreno_gpu->chip_id));931 932 for (i = 0; i < gpu->nr_rings; i++) {933 struct msm_ringbuffer *ring = gpu->rb[i];934 935 printk("rb %d: fence: %d/%d\n", i,936 ring->memptrs->fence,937 ring->fctx->last_fence);938 939 printk("rptr: %d\n", get_rptr(adreno_gpu, ring));940 printk("rb wptr: %d\n", get_wptr(ring));941 }942}943 944/* would be nice to not have to duplicate the _show() stuff with printk(): */945void adreno_dump(struct msm_gpu *gpu)946{947 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);948 int i;949 950 if (!adreno_gpu->registers)951 return;952 953 /* dump these out in a form that can be parsed by demsm: */954 printk("IO:region %s 00000000 00020000\n", gpu->name);955 for (i = 0; adreno_gpu->registers[i] != ~0; i += 2) {956 uint32_t start = adreno_gpu->registers[i];957 uint32_t end = adreno_gpu->registers[i+1];958 uint32_t addr;959 960 for (addr = start; addr <= end; addr++) {961 uint32_t val = gpu_read(gpu, addr);962 printk("IO:R %08x %08x\n", addr<<2, val);963 }964 }965}966 967static uint32_t ring_freewords(struct msm_ringbuffer *ring)968{969 struct adreno_gpu *adreno_gpu = to_adreno_gpu(ring->gpu);970 uint32_t size = MSM_GPU_RINGBUFFER_SZ >> 2;971 /* Use ring->next to calculate free size */972 uint32_t wptr = ring->next - ring->start;973 uint32_t rptr = get_rptr(adreno_gpu, ring);974 return (rptr + (size - 1) - wptr) % size;975}976 977void adreno_wait_ring(struct msm_ringbuffer *ring, uint32_t ndwords)978{979 if (spin_until(ring_freewords(ring) >= ndwords))980 DRM_DEV_ERROR(ring->gpu->dev->dev,981 "timeout waiting for space in ringbuffer %d\n",982 ring->id);983}984 985static int adreno_get_pwrlevels(struct device *dev,986 struct msm_gpu *gpu)987{988 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);989 unsigned long freq = ULONG_MAX;990 struct dev_pm_opp *opp;991 int ret;992 993 gpu->fast_rate = 0;994 995 /* devm_pm_opp_of_add_table may error out but will still create an OPP table */996 ret = devm_pm_opp_of_add_table(dev);997 if (ret == -ENODEV) {998 /* Special cases for ancient hw with ancient DT bindings */999 if (adreno_is_a2xx(adreno_gpu)) {1000 dev_warn(dev, "Unable to find the OPP table. Falling back to 200 MHz.\n");1001 dev_pm_opp_add(dev, 200000000, 0);1002 } else if (adreno_is_a320(adreno_gpu)) {1003 dev_warn(dev, "Unable to find the OPP table. Falling back to 450 MHz.\n");1004 dev_pm_opp_add(dev, 450000000, 0);1005 } else {1006 DRM_DEV_ERROR(dev, "Unable to find the OPP table\n");1007 return -ENODEV;1008 }1009 } else if (ret) {1010 DRM_DEV_ERROR(dev, "Unable to set the OPP table\n");1011 return ret;1012 }1013 1014 /* Find the fastest defined rate */1015 opp = dev_pm_opp_find_freq_floor(dev, &freq);1016 if (IS_ERR(opp))1017 return PTR_ERR(opp);1018 1019 gpu->fast_rate = freq;1020 dev_pm_opp_put(opp);1021 1022 DBG("fast_rate=%u, slow_rate=27000000", gpu->fast_rate);1023 1024 return 0;1025}1026 1027int adreno_gpu_ocmem_init(struct device *dev, struct adreno_gpu *adreno_gpu,1028 struct adreno_ocmem *adreno_ocmem)1029{1030 struct ocmem_buf *ocmem_hdl;1031 struct ocmem *ocmem;1032 1033 ocmem = of_get_ocmem(dev);1034 if (IS_ERR(ocmem)) {1035 if (PTR_ERR(ocmem) == -ENODEV) {1036 /*1037 * Return success since either the ocmem property was1038 * not specified in device tree, or ocmem support is1039 * not compiled into the kernel.1040 */1041 return 0;1042 }1043 1044 return PTR_ERR(ocmem);1045 }1046 1047 ocmem_hdl = ocmem_allocate(ocmem, OCMEM_GRAPHICS, adreno_gpu->info->gmem);1048 if (IS_ERR(ocmem_hdl))1049 return PTR_ERR(ocmem_hdl);1050 1051 adreno_ocmem->ocmem = ocmem;1052 adreno_ocmem->base = ocmem_hdl->addr;1053 adreno_ocmem->hdl = ocmem_hdl;1054 1055 if (WARN_ON(ocmem_hdl->len != adreno_gpu->info->gmem))1056 return -ENOMEM;1057 1058 return 0;1059}1060 1061void adreno_gpu_ocmem_cleanup(struct adreno_ocmem *adreno_ocmem)1062{1063 if (adreno_ocmem && adreno_ocmem->base)1064 ocmem_free(adreno_ocmem->ocmem, OCMEM_GRAPHICS,1065 adreno_ocmem->hdl);1066}1067 1068int adreno_read_speedbin(struct device *dev, u32 *speedbin)1069{1070 return nvmem_cell_read_variable_le_u32(dev, "speed_bin", speedbin);1071}1072 1073int adreno_gpu_init(struct drm_device *drm, struct platform_device *pdev,1074 struct adreno_gpu *adreno_gpu,1075 const struct adreno_gpu_funcs *funcs, int nr_rings)1076{1077 struct device *dev = &pdev->dev;1078 struct adreno_platform_config *config = dev->platform_data;1079 struct msm_gpu_config adreno_gpu_config = { 0 };1080 struct msm_gpu *gpu = &adreno_gpu->base;1081 const char *gpu_name;1082 u32 speedbin;1083 int ret;1084 1085 adreno_gpu->funcs = funcs;1086 adreno_gpu->info = config->info;1087 adreno_gpu->chip_id = config->chip_id;1088 1089 gpu->allow_relocs = config->info->family < ADRENO_6XX_GEN1;1090 gpu->pdev = pdev;1091 1092 /* Only handle the core clock when GMU is not in use (or is absent). */1093 if (adreno_has_gmu_wrapper(adreno_gpu) ||1094 adreno_gpu->info->family < ADRENO_6XX_GEN1) {1095 /*1096 * This can only be done before devm_pm_opp_of_add_table(), or1097 * dev_pm_opp_set_config() will WARN_ON()1098 */1099 if (IS_ERR(devm_clk_get(dev, "core"))) {1100 /*1101 * If "core" is absent, go for the legacy clock name.1102 * If we got this far in probing, it's a given one of1103 * them exists.1104 */1105 devm_pm_opp_set_clkname(dev, "core_clk");1106 } else1107 devm_pm_opp_set_clkname(dev, "core");1108 }1109 1110 if (adreno_read_speedbin(dev, &speedbin) || !speedbin)1111 speedbin = 0xffff;1112 adreno_gpu->speedbin = (uint16_t) (0xffff & speedbin);1113 1114 gpu_name = devm_kasprintf(dev, GFP_KERNEL, "%"ADRENO_CHIPID_FMT,1115 ADRENO_CHIPID_ARGS(config->chip_id));1116 if (!gpu_name)1117 return -ENOMEM;1118 1119 adreno_gpu_config.ioname = "kgsl_3d0_reg_memory";1120 1121 adreno_gpu_config.nr_rings = nr_rings;1122 1123 ret = adreno_get_pwrlevels(dev, gpu);1124 if (ret)1125 return ret;1126 1127 pm_runtime_set_autosuspend_delay(dev,1128 adreno_gpu->info->inactive_period);1129 pm_runtime_use_autosuspend(dev);1130 1131 return msm_gpu_init(drm, pdev, &adreno_gpu->base, &funcs->base,1132 gpu_name, &adreno_gpu_config);1133}1134 1135void adreno_gpu_cleanup(struct adreno_gpu *adreno_gpu)1136{1137 struct msm_gpu *gpu = &adreno_gpu->base;1138 struct msm_drm_private *priv = gpu->dev ? gpu->dev->dev_private : NULL;1139 unsigned int i;1140 1141 for (i = 0; i < ARRAY_SIZE(adreno_gpu->info->fw); i++)1142 release_firmware(adreno_gpu->fw[i]);1143 1144 if (priv && pm_runtime_enabled(&priv->gpu_pdev->dev))1145 pm_runtime_disable(&priv->gpu_pdev->dev);1146 1147 msm_gpu_cleanup(&adreno_gpu->base);1148}1149