736 lines · c
1/* SPDX-License-Identifier: GPL-2.0-only OR MIT */2/* Copyright (c) 2023 Imagination Technologies Ltd. */3 4#ifndef PVR_DEVICE_H5#define PVR_DEVICE_H6 7#include "pvr_ccb.h"8#include "pvr_device_info.h"9#include "pvr_fw.h"10#include "pvr_params.h"11#include "pvr_rogue_fwif_stream.h"12#include "pvr_stream.h"13 14#include <drm/drm_device.h>15#include <drm/drm_file.h>16#include <drm/drm_mm.h>17 18#include <linux/bits.h>19#include <linux/compiler_attributes.h>20#include <linux/compiler_types.h>21#include <linux/io.h>22#include <linux/iopoll.h>23#include <linux/kernel.h>24#include <linux/math.h>25#include <linux/mutex.h>26#include <linux/spinlock_types.h>27#include <linux/timer.h>28#include <linux/types.h>29#include <linux/wait.h>30#include <linux/workqueue.h>31#include <linux/xarray.h>32 33/* Forward declaration from <linux/clk.h>. */34struct clk;35 36/* Forward declaration from <linux/firmware.h>. */37struct firmware;38 39/**40 * struct pvr_gpu_id - Hardware GPU ID information for a PowerVR device41 * @b: Branch ID.42 * @v: Version ID.43 * @n: Number of scalable units.44 * @c: Config ID.45 */46struct pvr_gpu_id {47 u16 b, v, n, c;48};49 50/**51 * struct pvr_fw_version - Firmware version information52 * @major: Major version number.53 * @minor: Minor version number.54 */55struct pvr_fw_version {56 u16 major, minor;57};58 59/**60 * struct pvr_device - powervr-specific wrapper for &struct drm_device61 */62struct pvr_device {63 /**64 * @base: The underlying &struct drm_device.65 *66 * Do not access this member directly, instead call67 * from_pvr_device().68 */69 struct drm_device base;70 71 /** @gpu_id: GPU ID detected at runtime. */72 struct pvr_gpu_id gpu_id;73 74 /**75 * @features: Hardware feature information.76 *77 * Do not access this member directly, instead use PVR_HAS_FEATURE()78 * or PVR_FEATURE_VALUE() macros.79 */80 struct pvr_device_features features;81 82 /**83 * @quirks: Hardware quirk information.84 *85 * Do not access this member directly, instead use PVR_HAS_QUIRK().86 */87 struct pvr_device_quirks quirks;88 89 /**90 * @enhancements: Hardware enhancement information.91 *92 * Do not access this member directly, instead use93 * PVR_HAS_ENHANCEMENT().94 */95 struct pvr_device_enhancements enhancements;96 97 /** @fw_version: Firmware version detected at runtime. */98 struct pvr_fw_version fw_version;99 100 /** @regs_resource: Resource representing device control registers. */101 struct resource *regs_resource;102 103 /**104 * @regs: Device control registers.105 *106 * These are mapped into memory when the device is initialized; that107 * location is where this pointer points.108 */109 void __iomem *regs;110 111 /**112 * @core_clk: General core clock.113 *114 * This is the primary clock used by the entire GPU core.115 */116 struct clk *core_clk;117 118 /**119 * @sys_clk: Optional system bus clock.120 *121 * This may be used on some platforms to provide an independent clock to the SoC Interface122 * (SOCIF). If present, this needs to be enabled/disabled together with @core_clk.123 */124 struct clk *sys_clk;125 126 /**127 * @mem_clk: Optional memory clock.128 *129 * This may be used on some platforms to provide an independent clock to the Memory130 * Interface (MEMIF). If present, this needs to be enabled/disabled together with @core_clk.131 */132 struct clk *mem_clk;133 134 /** @irq: IRQ number. */135 int irq;136 137 /** @fwccb: Firmware CCB. */138 struct pvr_ccb fwccb;139 140 /**141 * @kernel_vm_ctx: Virtual memory context used for kernel mappings.142 *143 * This is used for mappings in the firmware address region when a META firmware processor144 * is in use.145 *146 * When a MIPS firmware processor is in use, this will be %NULL.147 */148 struct pvr_vm_context *kernel_vm_ctx;149 150 /** @fw_dev: Firmware related data. */151 struct pvr_fw_device fw_dev;152 153 /**154 * @params: Device-specific parameters.155 *156 * The values of these parameters are initialized from the157 * defaults specified as module parameters. They may be158 * modified at runtime via debugfs (if enabled).159 */160 struct pvr_device_params params;161 162 /** @stream_musthave_quirks: Bit array of "must-have" quirks for stream commands. */163 u32 stream_musthave_quirks[PVR_STREAM_TYPE_MAX][PVR_STREAM_EXTHDR_TYPE_MAX];164 165 /**166 * @mmu_flush_cache_flags: Records which MMU caches require flushing167 * before submitting the next job.168 */169 atomic_t mmu_flush_cache_flags;170 171 /**172 * @ctx_ids: Array of contexts belonging to this device. Array members173 * are of type "struct pvr_context *".174 *175 * This array is used to allocate IDs used by the firmware.176 */177 struct xarray ctx_ids;178 179 /**180 * @free_list_ids: Array of free lists belonging to this device. Array members181 * are of type "struct pvr_free_list *".182 *183 * This array is used to allocate IDs used by the firmware.184 */185 struct xarray free_list_ids;186 187 /**188 * @job_ids: Array of jobs belonging to this device. Array members189 * are of type "struct pvr_job *".190 */191 struct xarray job_ids;192 193 /**194 * @queues: Queue-related fields.195 */196 struct {197 /** @queues.active: Active queue list. */198 struct list_head active;199 200 /** @queues.idle: Idle queue list. */201 struct list_head idle;202 203 /** @queues.lock: Lock protecting access to the active/idle204 * lists. */205 struct mutex lock;206 } queues;207 208 /**209 * @watchdog: Watchdog for communications with firmware.210 */211 struct {212 /** @watchdog.work: Work item for watchdog callback. */213 struct delayed_work work;214 215 /**216 * @watchdog.old_kccb_cmds_executed: KCCB command execution217 * count at last watchdog poll.218 */219 u32 old_kccb_cmds_executed;220 221 /**222 * @watchdog.kccb_stall_count: Number of watchdog polls223 * KCCB has been stalled for.224 */225 u32 kccb_stall_count;226 } watchdog;227 228 /**229 * @kccb: Circular buffer for communications with firmware.230 */231 struct {232 /** @kccb.ccb: Kernel CCB. */233 struct pvr_ccb ccb;234 235 /** @kccb.rtn_q: Waitqueue for KCCB command return waiters. */236 wait_queue_head_t rtn_q;237 238 /** @kccb.rtn_obj: Object representing KCCB return slots. */239 struct pvr_fw_object *rtn_obj;240 241 /**242 * @kccb.rtn: Pointer to CPU mapping of KCCB return slots.243 * Must be accessed by READ_ONCE()/WRITE_ONCE().244 */245 u32 *rtn;246 247 /** @kccb.slot_count: Total number of KCCB slots available. */248 u32 slot_count;249 250 /** @kccb.reserved_count: Number of KCCB slots reserved for251 * future use. */252 u32 reserved_count;253 254 /**255 * @kccb.waiters: List of KCCB slot waiters.256 */257 struct list_head waiters;258 259 /** @kccb.fence_ctx: KCCB fence context. */260 struct {261 /** @kccb.fence_ctx.id: KCCB fence context ID262 * allocated with dma_fence_context_alloc(). */263 u64 id;264 265 /** @kccb.fence_ctx.seqno: Sequence number incremented266 * each time a fence is created. */267 atomic_t seqno;268 269 /**270 * @kccb.fence_ctx.lock: Lock used to synchronize271 * access to fences allocated by this context.272 */273 spinlock_t lock;274 } fence_ctx;275 } kccb;276 277 /**278 * @lost: %true if the device has been lost.279 *280 * This variable is set if the device has become irretrievably unavailable, e.g. if the281 * firmware processor has stopped responding and can not be revived via a hard reset.282 */283 bool lost;284 285 /**286 * @reset_sem: Reset semaphore.287 *288 * GPU reset code will lock this for writing. Any code that submits commands to the firmware289 * that isn't in an IRQ handler or on the scheduler workqueue must lock this for reading.290 * Once this has been successfully locked, &pvr_dev->lost _must_ be checked, and -%EIO must291 * be returned if it is set.292 */293 struct rw_semaphore reset_sem;294 295 /** @sched_wq: Workqueue for schedulers. */296 struct workqueue_struct *sched_wq;297 298 /**299 * @ctx_list_lock: Lock to be held when accessing the context list in300 * struct pvr_file.301 */302 spinlock_t ctx_list_lock;303};304 305/**306 * struct pvr_file - powervr-specific data to be assigned to &struct307 * drm_file.driver_priv308 */309struct pvr_file {310 /**311 * @file: A reference to the parent &struct drm_file.312 *313 * Do not access this member directly, instead call from_pvr_file().314 */315 struct drm_file *file;316 317 /**318 * @pvr_dev: A reference to the powervr-specific wrapper for the319 * associated device. Saves on repeated calls to to_pvr_device().320 */321 struct pvr_device *pvr_dev;322 323 /**324 * @ctx_handles: Array of contexts belonging to this file. Array members325 * are of type "struct pvr_context *".326 *327 * This array is used to allocate handles returned to userspace.328 */329 struct xarray ctx_handles;330 331 /**332 * @free_list_handles: Array of free lists belonging to this file. Array333 * members are of type "struct pvr_free_list *".334 *335 * This array is used to allocate handles returned to userspace.336 */337 struct xarray free_list_handles;338 339 /**340 * @hwrt_handles: Array of HWRT datasets belonging to this file. Array341 * members are of type "struct pvr_hwrt_dataset *".342 *343 * This array is used to allocate handles returned to userspace.344 */345 struct xarray hwrt_handles;346 347 /**348 * @vm_ctx_handles: Array of VM contexts belonging to this file. Array349 * members are of type "struct pvr_vm_context *".350 *351 * This array is used to allocate handles returned to userspace.352 */353 struct xarray vm_ctx_handles;354 355 /** @contexts: PVR context list. */356 struct list_head contexts;357};358 359/**360 * PVR_HAS_FEATURE() - Tests whether a PowerVR device has a given feature361 * @pvr_dev: [IN] Target PowerVR device.362 * @feature: [IN] Hardware feature name.363 *364 * Feature names are derived from those found in &struct pvr_device_features by365 * dropping the 'has_' prefix, which is applied by this macro.366 *367 * Return:368 * * true if the named feature is present in the hardware369 * * false if the named feature is not present in the hardware370 */371#define PVR_HAS_FEATURE(pvr_dev, feature) ((pvr_dev)->features.has_##feature)372 373/**374 * PVR_FEATURE_VALUE() - Gets a PowerVR device feature value375 * @pvr_dev: [IN] Target PowerVR device.376 * @feature: [IN] Feature name.377 * @value_out: [OUT] Feature value.378 *379 * This macro will get a feature value for those features that have values.380 * If the feature is not present, nothing will be stored to @value_out.381 *382 * Feature names are derived from those found in &struct pvr_device_features by383 * dropping the 'has_' prefix.384 *385 * Return:386 * * 0 on success, or387 * * -%EINVAL if the named feature is not present in the hardware388 */389#define PVR_FEATURE_VALUE(pvr_dev, feature, value_out) \390 ({ \391 struct pvr_device *_pvr_dev = pvr_dev; \392 int _ret = -EINVAL; \393 if (_pvr_dev->features.has_##feature) { \394 *(value_out) = _pvr_dev->features.feature; \395 _ret = 0; \396 } \397 _ret; \398 })399 400/**401 * PVR_HAS_QUIRK() - Tests whether a physical device has a given quirk402 * @pvr_dev: [IN] Target PowerVR device.403 * @quirk: [IN] Hardware quirk name.404 *405 * Quirk numbers are derived from those found in #pvr_device_quirks by406 * dropping the 'has_brn' prefix, which is applied by this macro.407 *408 * Returns409 * * true if the quirk is present in the hardware, or410 * * false if the quirk is not present in the hardware.411 */412#define PVR_HAS_QUIRK(pvr_dev, quirk) ((pvr_dev)->quirks.has_brn##quirk)413 414/**415 * PVR_HAS_ENHANCEMENT() - Tests whether a physical device has a given416 * enhancement417 * @pvr_dev: [IN] Target PowerVR device.418 * @enhancement: [IN] Hardware enhancement name.419 *420 * Enhancement numbers are derived from those found in #pvr_device_enhancements421 * by dropping the 'has_ern' prefix, which is applied by this macro.422 *423 * Returns424 * * true if the enhancement is present in the hardware, or425 * * false if the enhancement is not present in the hardware.426 */427#define PVR_HAS_ENHANCEMENT(pvr_dev, enhancement) ((pvr_dev)->enhancements.has_ern##enhancement)428 429#define from_pvr_device(pvr_dev) (&(pvr_dev)->base)430 431#define to_pvr_device(drm_dev) container_of_const(drm_dev, struct pvr_device, base)432 433#define from_pvr_file(pvr_file) ((pvr_file)->file)434 435#define to_pvr_file(file) ((file)->driver_priv)436 437/**438 * PVR_PACKED_BVNC() - Packs B, V, N and C values into a 64-bit unsigned integer439 * @b: Branch ID.440 * @v: Version ID.441 * @n: Number of scalable units.442 * @c: Config ID.443 *444 * The packed layout is as follows:445 *446 * +--------+--------+--------+-------+447 * | 63..48 | 47..32 | 31..16 | 15..0 |448 * +========+========+========+=======+449 * | B | V | N | C |450 * +--------+--------+--------+-------+451 *452 * pvr_gpu_id_to_packed_bvnc() should be used instead of this macro when a453 * &struct pvr_gpu_id is available in order to ensure proper type checking.454 *455 * Return: Packed BVNC.456 */457/* clang-format off */458#define PVR_PACKED_BVNC(b, v, n, c) \459 ((((u64)(b) & GENMASK_ULL(15, 0)) << 48) | \460 (((u64)(v) & GENMASK_ULL(15, 0)) << 32) | \461 (((u64)(n) & GENMASK_ULL(15, 0)) << 16) | \462 (((u64)(c) & GENMASK_ULL(15, 0)) << 0))463/* clang-format on */464 465/**466 * pvr_gpu_id_to_packed_bvnc() - Packs B, V, N and C values into a 64-bit467 * unsigned integer468 * @gpu_id: GPU ID.469 *470 * The packed layout is as follows:471 *472 * +--------+--------+--------+-------+473 * | 63..48 | 47..32 | 31..16 | 15..0 |474 * +========+========+========+=======+475 * | B | V | N | C |476 * +--------+--------+--------+-------+477 *478 * This should be used in preference to PVR_PACKED_BVNC() when a &struct479 * pvr_gpu_id is available in order to ensure proper type checking.480 *481 * Return: Packed BVNC.482 */483static __always_inline u64484pvr_gpu_id_to_packed_bvnc(struct pvr_gpu_id *gpu_id)485{486 return PVR_PACKED_BVNC(gpu_id->b, gpu_id->v, gpu_id->n, gpu_id->c);487}488 489static __always_inline void490packed_bvnc_to_pvr_gpu_id(u64 bvnc, struct pvr_gpu_id *gpu_id)491{492 gpu_id->b = (bvnc & GENMASK_ULL(63, 48)) >> 48;493 gpu_id->v = (bvnc & GENMASK_ULL(47, 32)) >> 32;494 gpu_id->n = (bvnc & GENMASK_ULL(31, 16)) >> 16;495 gpu_id->c = bvnc & GENMASK_ULL(15, 0);496}497 498int pvr_device_init(struct pvr_device *pvr_dev);499void pvr_device_fini(struct pvr_device *pvr_dev);500void pvr_device_reset(struct pvr_device *pvr_dev);501 502bool503pvr_device_has_uapi_quirk(struct pvr_device *pvr_dev, u32 quirk);504bool505pvr_device_has_uapi_enhancement(struct pvr_device *pvr_dev, u32 enhancement);506bool507pvr_device_has_feature(struct pvr_device *pvr_dev, u32 feature);508 509/**510 * PVR_CR_FIELD_GET() - Extract a single field from a PowerVR control register511 * @val: Value of the target register.512 * @field: Field specifier, as defined in "pvr_rogue_cr_defs.h".513 *514 * Return: The extracted field.515 */516#define PVR_CR_FIELD_GET(val, field) FIELD_GET(~ROGUE_CR_##field##_CLRMSK, val)517 518/**519 * pvr_cr_read32() - Read a 32-bit register from a PowerVR device520 * @pvr_dev: Target PowerVR device.521 * @reg: Target register.522 *523 * Return: The value of the requested register.524 */525static __always_inline u32526pvr_cr_read32(struct pvr_device *pvr_dev, u32 reg)527{528 return ioread32(pvr_dev->regs + reg);529}530 531/**532 * pvr_cr_read64() - Read a 64-bit register from a PowerVR device533 * @pvr_dev: Target PowerVR device.534 * @reg: Target register.535 *536 * Return: The value of the requested register.537 */538static __always_inline u64539pvr_cr_read64(struct pvr_device *pvr_dev, u32 reg)540{541 return ioread64(pvr_dev->regs + reg);542}543 544/**545 * pvr_cr_write32() - Write to a 32-bit register in a PowerVR device546 * @pvr_dev: Target PowerVR device.547 * @reg: Target register.548 * @val: Value to write.549 */550static __always_inline void551pvr_cr_write32(struct pvr_device *pvr_dev, u32 reg, u32 val)552{553 iowrite32(val, pvr_dev->regs + reg);554}555 556/**557 * pvr_cr_write64() - Write to a 64-bit register in a PowerVR device558 * @pvr_dev: Target PowerVR device.559 * @reg: Target register.560 * @val: Value to write.561 */562static __always_inline void563pvr_cr_write64(struct pvr_device *pvr_dev, u32 reg, u64 val)564{565 iowrite64(val, pvr_dev->regs + reg);566}567 568/**569 * pvr_cr_poll_reg32() - Wait for a 32-bit register to match a given value by570 * polling571 * @pvr_dev: Target PowerVR device.572 * @reg_addr: Address of register.573 * @reg_value: Expected register value (after masking).574 * @reg_mask: Mask of bits valid for comparison with @reg_value.575 * @timeout_usec: Timeout length, in us.576 *577 * Returns:578 * * 0 on success, or579 * * -%ETIMEDOUT on timeout.580 */581static __always_inline int582pvr_cr_poll_reg32(struct pvr_device *pvr_dev, u32 reg_addr, u32 reg_value,583 u32 reg_mask, u64 timeout_usec)584{585 u32 value;586 587 return readl_poll_timeout(pvr_dev->regs + reg_addr, value,588 (value & reg_mask) == reg_value, 0, timeout_usec);589}590 591/**592 * pvr_cr_poll_reg64() - Wait for a 64-bit register to match a given value by593 * polling594 * @pvr_dev: Target PowerVR device.595 * @reg_addr: Address of register.596 * @reg_value: Expected register value (after masking).597 * @reg_mask: Mask of bits valid for comparison with @reg_value.598 * @timeout_usec: Timeout length, in us.599 *600 * Returns:601 * * 0 on success, or602 * * -%ETIMEDOUT on timeout.603 */604static __always_inline int605pvr_cr_poll_reg64(struct pvr_device *pvr_dev, u32 reg_addr, u64 reg_value,606 u64 reg_mask, u64 timeout_usec)607{608 u64 value;609 610 return readq_poll_timeout(pvr_dev->regs + reg_addr, value,611 (value & reg_mask) == reg_value, 0, timeout_usec);612}613 614/**615 * pvr_round_up_to_cacheline_size() - Round up a provided size to be cacheline616 * aligned617 * @pvr_dev: Target PowerVR device.618 * @size: Initial size, in bytes.619 *620 * Returns:621 * * Size aligned to cacheline size.622 */623static __always_inline size_t624pvr_round_up_to_cacheline_size(struct pvr_device *pvr_dev, size_t size)625{626 u16 slc_cacheline_size_bits = 0;627 u16 slc_cacheline_size_bytes;628 629 WARN_ON(!PVR_HAS_FEATURE(pvr_dev, slc_cache_line_size_bits));630 PVR_FEATURE_VALUE(pvr_dev, slc_cache_line_size_bits,631 &slc_cacheline_size_bits);632 slc_cacheline_size_bytes = slc_cacheline_size_bits / 8;633 634 return round_up(size, slc_cacheline_size_bytes);635}636 637/**638 * DOC: IOCTL validation helpers639 *640 * To validate the constraints imposed on IOCTL argument structs, a collection641 * of macros and helper functions exist in ``pvr_device.h``.642 *643 * Of the current helpers, it should only be necessary to call644 * PVR_IOCTL_UNION_PADDING_CHECK() directly. This macro should be used once in645 * every code path which extracts a union member from a struct passed from646 * userspace.647 */648 649/**650 * pvr_ioctl_union_padding_check() - Validate that the implicit padding between651 * the end of a union member and the end of the union itself is zeroed.652 * @instance: Pointer to the instance of the struct to validate.653 * @union_offset: Offset into the type of @instance of the target union. Must654 * be 64-bit aligned.655 * @union_size: Size of the target union in the type of @instance. Must be656 * 64-bit aligned.657 * @member_size: Size of the target member in the target union specified by658 * @union_offset and @union_size. It is assumed that the offset of the target659 * member is zero relative to @union_offset. Must be 64-bit aligned.660 *661 * You probably want to use PVR_IOCTL_UNION_PADDING_CHECK() instead of calling662 * this function directly, since that macro abstracts away much of the setup,663 * and also provides some static validation. See its docs for details.664 *665 * Return:666 * * %true if every byte between the end of the used member of the union and667 * the end of that union is zeroed, or668 * * %false otherwise.669 */670static __always_inline bool671pvr_ioctl_union_padding_check(void *instance, size_t union_offset,672 size_t union_size, size_t member_size)673{674 /*675 * void pointer arithmetic is technically illegal - cast to a byte676 * pointer so this addition works safely.677 */678 void *padding_start = ((u8 *)instance) + union_offset + member_size;679 size_t padding_size = union_size - member_size;680 681 return mem_is_zero(padding_start, padding_size);682}683 684/**685 * PVR_STATIC_ASSERT_64BIT_ALIGNED() - Inline assertion for 64-bit alignment.686 * @static_expr_: Target expression to evaluate.687 *688 * If @static_expr_ does not evaluate to a constant integer which would be a689 * 64-bit aligned address (i.e. a multiple of 8), compilation will fail.690 *691 * Return:692 * The value of @static_expr_.693 */694#define PVR_STATIC_ASSERT_64BIT_ALIGNED(static_expr_) \695 ({ \696 static_assert(((static_expr_) & (sizeof(u64) - 1)) == 0); \697 (static_expr_); \698 })699 700/**701 * PVR_IOCTL_UNION_PADDING_CHECK() - Validate that the implicit padding between702 * the end of a union member and the end of the union itself is zeroed.703 * @struct_instance_: An expression which evaluates to a pointer to a UAPI data704 * struct.705 * @union_: The name of the union member of @struct_instance_ to check. If the706 * union member is nested within the type of @struct_instance_, this may707 * contain the member access operator (".").708 * @member_: The name of the member of @union_ to assess.709 *710 * This is a wrapper around pvr_ioctl_union_padding_check() which performs711 * alignment checks and simplifies things for the caller.712 *713 * Return:714 * * %true if every byte in @struct_instance_ between the end of @member_ and715 * the end of @union_ is zeroed, or716 * * %false otherwise.717 */718#define PVR_IOCTL_UNION_PADDING_CHECK(struct_instance_, union_, member_) \719 ({ \720 typeof(struct_instance_) __instance = (struct_instance_); \721 size_t __union_offset = PVR_STATIC_ASSERT_64BIT_ALIGNED( \722 offsetof(typeof(*__instance), union_)); \723 size_t __union_size = PVR_STATIC_ASSERT_64BIT_ALIGNED( \724 sizeof(__instance->union_)); \725 size_t __member_size = PVR_STATIC_ASSERT_64BIT_ALIGNED( \726 sizeof(__instance->union_.member_)); \727 pvr_ioctl_union_padding_check(__instance, __union_offset, \728 __union_size, __member_size); \729 })730 731#define PVR_FW_PROCESSOR_TYPE_META 0732#define PVR_FW_PROCESSOR_TYPE_MIPS 1733#define PVR_FW_PROCESSOR_TYPE_RISCV 2734 735#endif /* PVR_DEVICE_H */736