659 lines · c
1// SPDX-License-Identifier: GPL-2.0-only OR MIT2/* Copyright (c) 2023 Imagination Technologies Ltd. */3 4#include "pvr_device.h"5#include "pvr_device_info.h"6 7#include "pvr_fw.h"8#include "pvr_params.h"9#include "pvr_power.h"10#include "pvr_queue.h"11#include "pvr_rogue_cr_defs.h"12#include "pvr_stream.h"13#include "pvr_vm.h"14 15#include <drm/drm_print.h>16 17#include <linux/bitfield.h>18#include <linux/clk.h>19#include <linux/compiler_attributes.h>20#include <linux/compiler_types.h>21#include <linux/dma-mapping.h>22#include <linux/err.h>23#include <linux/firmware.h>24#include <linux/gfp.h>25#include <linux/interrupt.h>26#include <linux/platform_device.h>27#include <linux/pm_runtime.h>28#include <linux/slab.h>29#include <linux/stddef.h>30#include <linux/types.h>31#include <linux/workqueue.h>32 33/* Major number for the supported version of the firmware. */34#define PVR_FW_VERSION_MAJOR 135 36/**37 * pvr_device_reg_init() - Initialize kernel access to a PowerVR device's38 * control registers.39 * @pvr_dev: Target PowerVR device.40 *41 * Sets struct pvr_device->regs.42 *43 * This method of mapping the device control registers into memory ensures that44 * they are unmapped when the driver is detached (i.e. no explicit cleanup is45 * required).46 *47 * Return:48 * * 0 on success, or49 * * Any error returned by devm_platform_ioremap_resource().50 */51static int52pvr_device_reg_init(struct pvr_device *pvr_dev)53{54 struct drm_device *drm_dev = from_pvr_device(pvr_dev);55 struct platform_device *plat_dev = to_platform_device(drm_dev->dev);56 struct resource *regs_resource;57 void __iomem *regs;58 59 pvr_dev->regs_resource = NULL;60 pvr_dev->regs = NULL;61 62 regs = devm_platform_get_and_ioremap_resource(plat_dev, 0, ®s_resource);63 if (IS_ERR(regs))64 return dev_err_probe(drm_dev->dev, PTR_ERR(regs),65 "failed to ioremap gpu registers\n");66 67 pvr_dev->regs = regs;68 pvr_dev->regs_resource = regs_resource;69 70 return 0;71}72 73/**74 * pvr_device_clk_init() - Initialize clocks required by a PowerVR device75 * @pvr_dev: Target PowerVR device.76 *77 * Sets struct pvr_device->core_clk, struct pvr_device->sys_clk and78 * struct pvr_device->mem_clk.79 *80 * Three clocks are required by the PowerVR device: core, sys and mem. On81 * return, this function guarantees that the clocks are in one of the following82 * states:83 *84 * * All successfully initialized,85 * * Core errored, sys and mem uninitialized,86 * * Core deinitialized, sys errored, mem uninitialized, or87 * * Core and sys deinitialized, mem errored.88 *89 * Return:90 * * 0 on success,91 * * Any error returned by devm_clk_get(), or92 * * Any error returned by devm_clk_get_optional().93 */94static int pvr_device_clk_init(struct pvr_device *pvr_dev)95{96 struct drm_device *drm_dev = from_pvr_device(pvr_dev);97 struct clk *core_clk;98 struct clk *sys_clk;99 struct clk *mem_clk;100 101 core_clk = devm_clk_get(drm_dev->dev, "core");102 if (IS_ERR(core_clk))103 return dev_err_probe(drm_dev->dev, PTR_ERR(core_clk),104 "failed to get core clock\n");105 106 sys_clk = devm_clk_get_optional(drm_dev->dev, "sys");107 if (IS_ERR(sys_clk))108 return dev_err_probe(drm_dev->dev, PTR_ERR(sys_clk),109 "failed to get sys clock\n");110 111 mem_clk = devm_clk_get_optional(drm_dev->dev, "mem");112 if (IS_ERR(mem_clk))113 return dev_err_probe(drm_dev->dev, PTR_ERR(mem_clk),114 "failed to get mem clock\n");115 116 pvr_dev->core_clk = core_clk;117 pvr_dev->sys_clk = sys_clk;118 pvr_dev->mem_clk = mem_clk;119 120 return 0;121}122 123/**124 * pvr_device_process_active_queues() - Process all queue related events.125 * @pvr_dev: PowerVR device to check126 *127 * This is called any time we receive a FW event. It iterates over all128 * active queues and calls pvr_queue_process() on them.129 */130static void pvr_device_process_active_queues(struct pvr_device *pvr_dev)131{132 struct pvr_queue *queue, *tmp_queue;133 LIST_HEAD(active_queues);134 135 mutex_lock(&pvr_dev->queues.lock);136 137 /* Move all active queues to a temporary list. Queues that remain138 * active after we're done processing them are re-inserted to139 * the queues.active list by pvr_queue_process().140 */141 list_splice_init(&pvr_dev->queues.active, &active_queues);142 143 list_for_each_entry_safe(queue, tmp_queue, &active_queues, node)144 pvr_queue_process(queue);145 146 mutex_unlock(&pvr_dev->queues.lock);147}148 149static irqreturn_t pvr_device_irq_thread_handler(int irq, void *data)150{151 struct pvr_device *pvr_dev = data;152 irqreturn_t ret = IRQ_NONE;153 154 /* We are in the threaded handler, we can keep dequeuing events until we155 * don't see any. This should allow us to reduce the number of interrupts156 * when the GPU is receiving a massive amount of short jobs.157 */158 while (pvr_fw_irq_pending(pvr_dev)) {159 pvr_fw_irq_clear(pvr_dev);160 161 if (pvr_dev->fw_dev.booted) {162 pvr_fwccb_process(pvr_dev);163 pvr_kccb_wake_up_waiters(pvr_dev);164 pvr_device_process_active_queues(pvr_dev);165 }166 167 pm_runtime_mark_last_busy(from_pvr_device(pvr_dev)->dev);168 169 ret = IRQ_HANDLED;170 }171 172 /* Unmask FW irqs before returning, so new interrupts can be received. */173 pvr_fw_irq_enable(pvr_dev);174 return ret;175}176 177static irqreturn_t pvr_device_irq_handler(int irq, void *data)178{179 struct pvr_device *pvr_dev = data;180 181 if (!pvr_fw_irq_pending(pvr_dev))182 return IRQ_NONE; /* Spurious IRQ - ignore. */183 184 /* Mask the FW interrupts before waking up the thread. Will be unmasked185 * when the thread handler is done processing events.186 */187 pvr_fw_irq_disable(pvr_dev);188 return IRQ_WAKE_THREAD;189}190 191/**192 * pvr_device_irq_init() - Initialise IRQ required by a PowerVR device193 * @pvr_dev: Target PowerVR device.194 *195 * Returns:196 * * 0 on success,197 * * Any error returned by platform_get_irq_byname(), or198 * * Any error returned by request_irq().199 */200static int201pvr_device_irq_init(struct pvr_device *pvr_dev)202{203 struct drm_device *drm_dev = from_pvr_device(pvr_dev);204 struct platform_device *plat_dev = to_platform_device(drm_dev->dev);205 206 init_waitqueue_head(&pvr_dev->kccb.rtn_q);207 208 pvr_dev->irq = platform_get_irq(plat_dev, 0);209 if (pvr_dev->irq < 0)210 return pvr_dev->irq;211 212 /* Clear any pending events before requesting the IRQ line. */213 pvr_fw_irq_clear(pvr_dev);214 pvr_fw_irq_enable(pvr_dev);215 216 return request_threaded_irq(pvr_dev->irq, pvr_device_irq_handler,217 pvr_device_irq_thread_handler,218 IRQF_SHARED, "gpu", pvr_dev);219}220 221/**222 * pvr_device_irq_fini() - Deinitialise IRQ required by a PowerVR device223 * @pvr_dev: Target PowerVR device.224 */225static void226pvr_device_irq_fini(struct pvr_device *pvr_dev)227{228 free_irq(pvr_dev->irq, pvr_dev);229}230 231/**232 * pvr_build_firmware_filename() - Construct a PowerVR firmware filename233 * @pvr_dev: Target PowerVR device.234 * @base: First part of the filename.235 * @major: Major version number.236 *237 * A PowerVR firmware filename consists of three parts separated by underscores238 * (``'_'``) along with a '.fw' file suffix. The first part is the exact value239 * of @base, the second part is the hardware version string derived from @pvr_fw240 * and the final part is the firmware version number constructed from @major with241 * a 'v' prefix, e.g. powervr/rogue_4.40.2.51_v1.fw.242 *243 * The returned string will have been slab allocated and must be freed with244 * kfree().245 *246 * Return:247 * * The constructed filename on success, or248 * * Any error returned by kasprintf().249 */250static char *251pvr_build_firmware_filename(struct pvr_device *pvr_dev, const char *base,252 u8 major)253{254 struct pvr_gpu_id *gpu_id = &pvr_dev->gpu_id;255 256 return kasprintf(GFP_KERNEL, "%s_%d.%d.%d.%d_v%d.fw", base, gpu_id->b,257 gpu_id->v, gpu_id->n, gpu_id->c, major);258}259 260static void261pvr_release_firmware(void *data)262{263 struct pvr_device *pvr_dev = data;264 265 release_firmware(pvr_dev->fw_dev.firmware);266}267 268/**269 * pvr_request_firmware() - Load firmware for a PowerVR device270 * @pvr_dev: Target PowerVR device.271 *272 * See pvr_build_firmware_filename() for details on firmware file naming.273 *274 * Return:275 * * 0 on success,276 * * Any error returned by pvr_build_firmware_filename(), or277 * * Any error returned by request_firmware().278 */279static int280pvr_request_firmware(struct pvr_device *pvr_dev)281{282 struct drm_device *drm_dev = &pvr_dev->base;283 char *filename;284 const struct firmware *fw;285 int err;286 287 filename = pvr_build_firmware_filename(pvr_dev, "powervr/rogue",288 PVR_FW_VERSION_MAJOR);289 if (!filename)290 return -ENOMEM;291 292 /*293 * This function takes a copy of &filename, meaning we can free our294 * instance before returning.295 */296 err = request_firmware(&fw, filename, pvr_dev->base.dev);297 if (err) {298 drm_err(drm_dev, "failed to load firmware %s (err=%d)\n",299 filename, err);300 goto err_free_filename;301 }302 303 drm_info(drm_dev, "loaded firmware %s\n", filename);304 kfree(filename);305 306 pvr_dev->fw_dev.firmware = fw;307 308 return devm_add_action_or_reset(drm_dev->dev, pvr_release_firmware, pvr_dev);309 310err_free_filename:311 kfree(filename);312 313 return err;314}315 316/**317 * pvr_load_gpu_id() - Load a PowerVR device's GPU ID (BVNC) from control registers.318 *319 * Sets struct pvr_dev.gpu_id.320 *321 * @pvr_dev: Target PowerVR device.322 */323static void324pvr_load_gpu_id(struct pvr_device *pvr_dev)325{326 struct pvr_gpu_id *gpu_id = &pvr_dev->gpu_id;327 u64 bvnc;328 329 /*330 * Try reading the BVNC using the newer (cleaner) method first. If the331 * B value is zero, fall back to the older method.332 */333 bvnc = pvr_cr_read64(pvr_dev, ROGUE_CR_CORE_ID__PBVNC);334 335 gpu_id->b = PVR_CR_FIELD_GET(bvnc, CORE_ID__PBVNC__BRANCH_ID);336 if (gpu_id->b != 0) {337 gpu_id->v = PVR_CR_FIELD_GET(bvnc, CORE_ID__PBVNC__VERSION_ID);338 gpu_id->n = PVR_CR_FIELD_GET(bvnc, CORE_ID__PBVNC__NUMBER_OF_SCALABLE_UNITS);339 gpu_id->c = PVR_CR_FIELD_GET(bvnc, CORE_ID__PBVNC__CONFIG_ID);340 } else {341 u32 core_rev = pvr_cr_read32(pvr_dev, ROGUE_CR_CORE_REVISION);342 u32 core_id = pvr_cr_read32(pvr_dev, ROGUE_CR_CORE_ID);343 u16 core_id_config = PVR_CR_FIELD_GET(core_id, CORE_ID_CONFIG);344 345 gpu_id->b = PVR_CR_FIELD_GET(core_rev, CORE_REVISION_MAJOR);346 gpu_id->v = PVR_CR_FIELD_GET(core_rev, CORE_REVISION_MINOR);347 gpu_id->n = FIELD_GET(0xFF00, core_id_config);348 gpu_id->c = FIELD_GET(0x00FF, core_id_config);349 }350}351 352/**353 * pvr_set_dma_info() - Set PowerVR device DMA information354 * @pvr_dev: Target PowerVR device.355 *356 * Sets the DMA mask and max segment size for the PowerVR device.357 *358 * Return:359 * * 0 on success,360 * * Any error returned by PVR_FEATURE_VALUE(), or361 * * Any error returned by dma_set_mask().362 */363 364static int365pvr_set_dma_info(struct pvr_device *pvr_dev)366{367 struct drm_device *drm_dev = from_pvr_device(pvr_dev);368 u16 phys_bus_width;369 int err;370 371 err = PVR_FEATURE_VALUE(pvr_dev, phys_bus_width, &phys_bus_width);372 if (err) {373 drm_err(drm_dev, "Failed to get device physical bus width\n");374 return err;375 }376 377 err = dma_set_mask(drm_dev->dev, DMA_BIT_MASK(phys_bus_width));378 if (err) {379 drm_err(drm_dev, "Failed to set DMA mask (err=%d)\n", err);380 return err;381 }382 383 dma_set_max_seg_size(drm_dev->dev, UINT_MAX);384 385 return 0;386}387 388/**389 * pvr_device_gpu_init() - GPU-specific initialization for a PowerVR device390 * @pvr_dev: Target PowerVR device.391 *392 * The following steps are taken to ensure the device is ready:393 *394 * 1. Read the hardware version information from control registers,395 * 2. Initialise the hardware feature information,396 * 3. Setup the device DMA information,397 * 4. Setup the device-scoped memory context, and398 * 5. Load firmware into the device.399 *400 * Return:401 * * 0 on success,402 * * -%ENODEV if the GPU is not supported,403 * * Any error returned by pvr_set_dma_info(),404 * * Any error returned by pvr_memory_context_init(), or405 * * Any error returned by pvr_request_firmware().406 */407static int408pvr_device_gpu_init(struct pvr_device *pvr_dev)409{410 int err;411 412 pvr_load_gpu_id(pvr_dev);413 414 err = pvr_request_firmware(pvr_dev);415 if (err)416 return err;417 418 err = pvr_fw_validate_init_device_info(pvr_dev);419 if (err)420 return err;421 422 if (PVR_HAS_FEATURE(pvr_dev, meta))423 pvr_dev->fw_dev.processor_type = PVR_FW_PROCESSOR_TYPE_META;424 else if (PVR_HAS_FEATURE(pvr_dev, mips))425 pvr_dev->fw_dev.processor_type = PVR_FW_PROCESSOR_TYPE_MIPS;426 else if (PVR_HAS_FEATURE(pvr_dev, riscv_fw_processor))427 pvr_dev->fw_dev.processor_type = PVR_FW_PROCESSOR_TYPE_RISCV;428 else429 return -EINVAL;430 431 pvr_stream_create_musthave_masks(pvr_dev);432 433 err = pvr_set_dma_info(pvr_dev);434 if (err)435 return err;436 437 if (pvr_dev->fw_dev.processor_type != PVR_FW_PROCESSOR_TYPE_MIPS) {438 pvr_dev->kernel_vm_ctx = pvr_vm_create_context(pvr_dev, false);439 if (IS_ERR(pvr_dev->kernel_vm_ctx))440 return PTR_ERR(pvr_dev->kernel_vm_ctx);441 }442 443 err = pvr_fw_init(pvr_dev);444 if (err)445 goto err_vm_ctx_put;446 447 return 0;448 449err_vm_ctx_put:450 if (pvr_dev->fw_dev.processor_type != PVR_FW_PROCESSOR_TYPE_MIPS) {451 pvr_vm_context_put(pvr_dev->kernel_vm_ctx);452 pvr_dev->kernel_vm_ctx = NULL;453 }454 455 return err;456}457 458/**459 * pvr_device_gpu_fini() - GPU-specific deinitialization for a PowerVR device460 * @pvr_dev: Target PowerVR device.461 */462static void463pvr_device_gpu_fini(struct pvr_device *pvr_dev)464{465 pvr_fw_fini(pvr_dev);466 467 if (pvr_dev->fw_dev.processor_type != PVR_FW_PROCESSOR_TYPE_MIPS) {468 WARN_ON(!pvr_vm_context_put(pvr_dev->kernel_vm_ctx));469 pvr_dev->kernel_vm_ctx = NULL;470 }471}472 473/**474 * pvr_device_init() - Initialize a PowerVR device475 * @pvr_dev: Target PowerVR device.476 *477 * If this function returns successfully, the device will have been fully478 * initialized. Otherwise, any parts of the device initialized before an error479 * occurs will be de-initialized before returning.480 *481 * NOTE: The initialization steps currently taken are the bare minimum required482 * to read from the control registers. The device is unlikely to function483 * until further initialization steps are added. [This note should be484 * removed when that happens.]485 *486 * Return:487 * * 0 on success,488 * * Any error returned by pvr_device_reg_init(),489 * * Any error returned by pvr_device_clk_init(), or490 * * Any error returned by pvr_device_gpu_init().491 */492int493pvr_device_init(struct pvr_device *pvr_dev)494{495 struct drm_device *drm_dev = from_pvr_device(pvr_dev);496 struct device *dev = drm_dev->dev;497 int err;498 499 /*500 * Setup device parameters. We do this first in case other steps501 * depend on them.502 */503 err = pvr_device_params_init(&pvr_dev->params);504 if (err)505 return err;506 507 /* Enable and initialize clocks required for the device to operate. */508 err = pvr_device_clk_init(pvr_dev);509 if (err)510 return err;511 512 /* Explicitly power the GPU so we can access control registers before the FW is booted. */513 err = pm_runtime_resume_and_get(dev);514 if (err)515 return err;516 517 /* Map the control registers into memory. */518 err = pvr_device_reg_init(pvr_dev);519 if (err)520 goto err_pm_runtime_put;521 522 /* Perform GPU-specific initialization steps. */523 err = pvr_device_gpu_init(pvr_dev);524 if (err)525 goto err_pm_runtime_put;526 527 err = pvr_device_irq_init(pvr_dev);528 if (err)529 goto err_device_gpu_fini;530 531 pm_runtime_put(dev);532 533 return 0;534 535err_device_gpu_fini:536 pvr_device_gpu_fini(pvr_dev);537 538err_pm_runtime_put:539 pm_runtime_put_sync_suspend(dev);540 541 return err;542}543 544/**545 * pvr_device_fini() - Deinitialize a PowerVR device546 * @pvr_dev: Target PowerVR device.547 */548void549pvr_device_fini(struct pvr_device *pvr_dev)550{551 /*552 * Deinitialization stages are performed in reverse order compared to553 * the initialization stages in pvr_device_init().554 */555 pvr_device_irq_fini(pvr_dev);556 pvr_device_gpu_fini(pvr_dev);557}558 559bool560pvr_device_has_uapi_quirk(struct pvr_device *pvr_dev, u32 quirk)561{562 switch (quirk) {563 case 47217:564 return PVR_HAS_QUIRK(pvr_dev, 47217);565 case 48545:566 return PVR_HAS_QUIRK(pvr_dev, 48545);567 case 49927:568 return PVR_HAS_QUIRK(pvr_dev, 49927);569 case 51764:570 return PVR_HAS_QUIRK(pvr_dev, 51764);571 case 62269:572 return PVR_HAS_QUIRK(pvr_dev, 62269);573 default:574 return false;575 };576}577 578bool579pvr_device_has_uapi_enhancement(struct pvr_device *pvr_dev, u32 enhancement)580{581 switch (enhancement) {582 case 35421:583 return PVR_HAS_ENHANCEMENT(pvr_dev, 35421);584 case 42064:585 return PVR_HAS_ENHANCEMENT(pvr_dev, 42064);586 default:587 return false;588 };589}590 591/**592 * pvr_device_has_feature() - Look up device feature based on feature definition593 * @pvr_dev: Device pointer.594 * @feature: Feature to look up. Should be one of %PVR_FEATURE_*.595 *596 * Returns:597 * * %true if feature is present on device, or598 * * %false if feature is not present on device.599 */600bool601pvr_device_has_feature(struct pvr_device *pvr_dev, u32 feature)602{603 switch (feature) {604 case PVR_FEATURE_CLUSTER_GROUPING:605 return PVR_HAS_FEATURE(pvr_dev, cluster_grouping);606 607 case PVR_FEATURE_COMPUTE_MORTON_CAPABLE:608 return PVR_HAS_FEATURE(pvr_dev, compute_morton_capable);609 610 case PVR_FEATURE_FB_CDC_V4:611 return PVR_HAS_FEATURE(pvr_dev, fb_cdc_v4);612 613 case PVR_FEATURE_GPU_MULTICORE_SUPPORT:614 return PVR_HAS_FEATURE(pvr_dev, gpu_multicore_support);615 616 case PVR_FEATURE_ISP_ZLS_D24_S8_PACKING_OGL_MODE:617 return PVR_HAS_FEATURE(pvr_dev, isp_zls_d24_s8_packing_ogl_mode);618 619 case PVR_FEATURE_S7_TOP_INFRASTRUCTURE:620 return PVR_HAS_FEATURE(pvr_dev, s7_top_infrastructure);621 622 case PVR_FEATURE_TESSELLATION:623 return PVR_HAS_FEATURE(pvr_dev, tessellation);624 625 case PVR_FEATURE_TPU_DM_GLOBAL_REGISTERS:626 return PVR_HAS_FEATURE(pvr_dev, tpu_dm_global_registers);627 628 case PVR_FEATURE_VDM_DRAWINDIRECT:629 return PVR_HAS_FEATURE(pvr_dev, vdm_drawindirect);630 631 case PVR_FEATURE_VDM_OBJECT_LEVEL_LLS:632 return PVR_HAS_FEATURE(pvr_dev, vdm_object_level_lls);633 634 case PVR_FEATURE_ZLS_SUBTILE:635 return PVR_HAS_FEATURE(pvr_dev, zls_subtile);636 637 /* Derived features. */638 case PVR_FEATURE_CDM_USER_MODE_QUEUE: {639 u8 cdm_control_stream_format = 0;640 641 PVR_FEATURE_VALUE(pvr_dev, cdm_control_stream_format, &cdm_control_stream_format);642 return (cdm_control_stream_format >= 2 && cdm_control_stream_format <= 4);643 }644 645 case PVR_FEATURE_REQUIRES_FB_CDC_ZLS_SETUP:646 if (PVR_HAS_FEATURE(pvr_dev, fbcdc_algorithm)) {647 u8 fbcdc_algorithm = 0;648 649 PVR_FEATURE_VALUE(pvr_dev, fbcdc_algorithm, &fbcdc_algorithm);650 return (fbcdc_algorithm < 3 || PVR_HAS_FEATURE(pvr_dev, fb_cdc_v4));651 }652 return false;653 654 default:655 WARN(true, "Looking up undefined feature %u\n", feature);656 return false;657 }658}659