871 lines · c
1// SPDX-License-Identifier: MIT2/*3 * Copyright © 2022 Intel Corporation4 */5 6#include "i915_drv.h"7#include "intel_gt.h"8#include "intel_gt_mcr.h"9#include "intel_gt_print.h"10#include "intel_gt_regs.h"11 12/**13 * DOC: GT Multicast/Replicated (MCR) Register Support14 *15 * Some GT registers are designed as "multicast" or "replicated" registers:16 * multiple instances of the same register share a single MMIO offset. MCR17 * registers are generally used when the hardware needs to potentially track18 * independent values of a register per hardware unit (e.g., per-subslice,19 * per-L3bank, etc.). The specific types of replication that exist vary20 * per-platform.21 *22 * MMIO accesses to MCR registers are controlled according to the settings23 * programmed in the platform's MCR_SELECTOR register(s). MMIO writes to MCR24 * registers can be done in either a (i.e., a single write updates all25 * instances of the register to the same value) or unicast (a write updates only26 * one specific instance). Reads of MCR registers always operate in a unicast27 * manner regardless of how the multicast/unicast bit is set in MCR_SELECTOR.28 * Selection of a specific MCR instance for unicast operations is referred to29 * as "steering."30 *31 * If MCR register operations are steered toward a hardware unit that is32 * fused off or currently powered down due to power gating, the MMIO operation33 * is "terminated" by the hardware. Terminated read operations will return a34 * value of zero and terminated unicast write operations will be silently35 * ignored.36 */37 38#define HAS_MSLICE_STEERING(i915) (INTEL_INFO(i915)->has_mslice_steering)39 40static const char * const intel_steering_types[] = {41 "L3BANK",42 "MSLICE",43 "LNCF",44 "GAM",45 "DSS",46 "OADDRM",47 "INSTANCE 0",48};49 50static const struct intel_mmio_range icl_l3bank_steering_table[] = {51 { 0x00B100, 0x00B3FF },52 {},53};54 55/*56 * Although the bspec lists more "MSLICE" ranges than shown here, some of those57 * are of a "GAM" subclass that has special rules. Thus we use a separate58 * GAM table farther down for those.59 */60static const struct intel_mmio_range dg2_mslice_steering_table[] = {61 { 0x00DD00, 0x00DDFF },62 { 0x00E900, 0x00FFFF }, /* 0xEA00 - OxEFFF is unused */63 {},64};65 66static const struct intel_mmio_range dg2_lncf_steering_table[] = {67 { 0x00B000, 0x00B0FF },68 { 0x00D880, 0x00D8FF },69 {},70};71 72static const struct intel_mmio_range xelpg_instance0_steering_table[] = {73 { 0x000B00, 0x000BFF }, /* SQIDI */74 { 0x001000, 0x001FFF }, /* SQIDI */75 { 0x004000, 0x0048FF }, /* GAM */76 { 0x008700, 0x0087FF }, /* SQIDI */77 { 0x00B000, 0x00B0FF }, /* NODE */78 { 0x00C800, 0x00CFFF }, /* GAM */79 { 0x00D880, 0x00D8FF }, /* NODE */80 { 0x00DD00, 0x00DDFF }, /* OAAL2 */81 {},82};83 84static const struct intel_mmio_range xelpg_l3bank_steering_table[] = {85 { 0x00B100, 0x00B3FF },86 {},87};88 89/* DSS steering is used for SLICE ranges as well */90static const struct intel_mmio_range xelpg_dss_steering_table[] = {91 { 0x005200, 0x0052FF }, /* SLICE */92 { 0x005500, 0x007FFF }, /* SLICE */93 { 0x008140, 0x00815F }, /* SLICE (0x8140-0x814F), DSS (0x8150-0x815F) */94 { 0x0094D0, 0x00955F }, /* SLICE (0x94D0-0x951F), DSS (0x9520-0x955F) */95 { 0x009680, 0x0096FF }, /* DSS */96 { 0x00D800, 0x00D87F }, /* SLICE */97 { 0x00DC00, 0x00DCFF }, /* SLICE */98 { 0x00DE80, 0x00E8FF }, /* DSS (0xE000-0xE0FF reserved) */99 {},100};101 102static const struct intel_mmio_range xelpmp_oaddrm_steering_table[] = {103 { 0x393200, 0x39323F },104 { 0x393400, 0x3934FF },105 {},106};107 108void intel_gt_mcr_init(struct intel_gt *gt)109{110 struct drm_i915_private *i915 = gt->i915;111 unsigned long fuse;112 int i;113 114 spin_lock_init(>->mcr_lock);115 116 /*117 * An mslice is unavailable only if both the meml3 for the slice is118 * disabled *and* all of the DSS in the slice (quadrant) are disabled.119 */120 if (HAS_MSLICE_STEERING(i915)) {121 gt->info.mslice_mask =122 intel_slicemask_from_xehp_dssmask(gt->info.sseu.subslice_mask,123 GEN_DSS_PER_MSLICE);124 gt->info.mslice_mask |=125 (intel_uncore_read(gt->uncore, GEN10_MIRROR_FUSE3) &126 GEN12_MEML3_EN_MASK);127 128 if (!gt->info.mslice_mask) /* should be impossible! */129 gt_warn(gt, "mslice mask all zero!\n");130 }131 132 if (MEDIA_VER(i915) >= 13 && gt->type == GT_MEDIA) {133 gt->steering_table[OADDRM] = xelpmp_oaddrm_steering_table;134 } else if (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 70)) {135 /* Wa_14016747170 */136 if (IS_GFX_GT_IP_STEP(gt, IP_VER(12, 70), STEP_A0, STEP_B0) ||137 IS_GFX_GT_IP_STEP(gt, IP_VER(12, 71), STEP_A0, STEP_B0))138 fuse = REG_FIELD_GET(MTL_GT_L3_EXC_MASK,139 intel_uncore_read(gt->uncore,140 MTL_GT_ACTIVITY_FACTOR));141 else142 fuse = REG_FIELD_GET(GT_L3_EXC_MASK,143 intel_uncore_read(gt->uncore, XEHP_FUSE4));144 145 /*146 * Despite the register field being named "exclude mask" the147 * bits actually represent enabled banks (two banks per bit).148 */149 for_each_set_bit(i, &fuse, 3)150 gt->info.l3bank_mask |= 0x3 << 2 * i;151 152 gt->steering_table[INSTANCE0] = xelpg_instance0_steering_table;153 gt->steering_table[L3BANK] = xelpg_l3bank_steering_table;154 gt->steering_table[DSS] = xelpg_dss_steering_table;155 } else if (IS_DG2(i915)) {156 gt->steering_table[MSLICE] = dg2_mslice_steering_table;157 gt->steering_table[LNCF] = dg2_lncf_steering_table;158 /*159 * No need to hook up the GAM table since it has a dedicated160 * steering control register on DG2 and can use implicit161 * steering.162 */163 } else if (GRAPHICS_VER(i915) >= 11 &&164 GRAPHICS_VER_FULL(i915) < IP_VER(12, 55)) {165 gt->steering_table[L3BANK] = icl_l3bank_steering_table;166 gt->info.l3bank_mask =167 ~intel_uncore_read(gt->uncore, GEN10_MIRROR_FUSE3) &168 GEN10_L3BANK_MASK;169 if (!gt->info.l3bank_mask) /* should be impossible! */170 gt_warn(gt, "L3 bank mask is all zero!\n");171 } else if (GRAPHICS_VER(i915) >= 11) {172 /*173 * We expect all modern platforms to have at least some174 * type of steering that needs to be initialized.175 */176 MISSING_CASE(INTEL_INFO(i915)->platform);177 }178}179 180/*181 * Although the rest of the driver should use MCR-specific functions to182 * read/write MCR registers, we still use the regular intel_uncore_* functions183 * internally to implement those, so we need a way for the functions in this184 * file to "cast" an i915_mcr_reg_t into an i915_reg_t.185 */186static i915_reg_t mcr_reg_cast(const i915_mcr_reg_t mcr)187{188 i915_reg_t r = { .reg = mcr.reg };189 190 return r;191}192 193/*194 * rw_with_mcr_steering_fw - Access a register with specific MCR steering195 * @gt: GT to read register from196 * @reg: register being accessed197 * @rw_flag: FW_REG_READ for read access or FW_REG_WRITE for write access198 * @group: group number (documented as "sliceid" on older platforms)199 * @instance: instance number (documented as "subsliceid" on older platforms)200 * @value: register value to be written (ignored for read)201 *202 * Context: The caller must hold the MCR lock203 * Return: 0 for write access. register value for read access.204 *205 * Caller needs to make sure the relevant forcewake wells are up.206 */207static u32 rw_with_mcr_steering_fw(struct intel_gt *gt,208 i915_mcr_reg_t reg, u8 rw_flag,209 int group, int instance, u32 value)210{211 struct intel_uncore *uncore = gt->uncore;212 u32 mcr_mask, mcr_ss, mcr, old_mcr, val = 0;213 214 lockdep_assert_held(>->mcr_lock);215 216 if (GRAPHICS_VER_FULL(uncore->i915) >= IP_VER(12, 70)) {217 /*218 * Always leave the hardware in multicast mode when doing reads219 * (see comment about Wa_22013088509 below) and only change it220 * to unicast mode when doing writes of a specific instance.221 *222 * No need to save old steering reg value.223 */224 intel_uncore_write_fw(uncore, MTL_MCR_SELECTOR,225 REG_FIELD_PREP(MTL_MCR_GROUPID, group) |226 REG_FIELD_PREP(MTL_MCR_INSTANCEID, instance) |227 (rw_flag == FW_REG_READ ? GEN11_MCR_MULTICAST : 0));228 } else if (GRAPHICS_VER(uncore->i915) >= 11) {229 mcr_mask = GEN11_MCR_SLICE_MASK | GEN11_MCR_SUBSLICE_MASK;230 mcr_ss = GEN11_MCR_SLICE(group) | GEN11_MCR_SUBSLICE(instance);231 232 /*233 * Wa_22013088509234 *235 * The setting of the multicast/unicast bit usually wouldn't236 * matter for read operations (which always return the value237 * from a single register instance regardless of how that bit238 * is set), but some platforms have a workaround requiring us239 * to remain in multicast mode for reads. There's no real240 * downside to this, so we'll just go ahead and do so on all241 * platforms; we'll only clear the multicast bit from the mask242 * when exlicitly doing a write operation.243 */244 if (rw_flag == FW_REG_WRITE)245 mcr_mask |= GEN11_MCR_MULTICAST;246 247 mcr = intel_uncore_read_fw(uncore, GEN8_MCR_SELECTOR);248 old_mcr = mcr;249 250 mcr &= ~mcr_mask;251 mcr |= mcr_ss;252 intel_uncore_write_fw(uncore, GEN8_MCR_SELECTOR, mcr);253 } else {254 mcr_mask = GEN8_MCR_SLICE_MASK | GEN8_MCR_SUBSLICE_MASK;255 mcr_ss = GEN8_MCR_SLICE(group) | GEN8_MCR_SUBSLICE(instance);256 257 mcr = intel_uncore_read_fw(uncore, GEN8_MCR_SELECTOR);258 old_mcr = mcr;259 260 mcr &= ~mcr_mask;261 mcr |= mcr_ss;262 intel_uncore_write_fw(uncore, GEN8_MCR_SELECTOR, mcr);263 }264 265 if (rw_flag == FW_REG_READ)266 val = intel_uncore_read_fw(uncore, mcr_reg_cast(reg));267 else268 intel_uncore_write_fw(uncore, mcr_reg_cast(reg), value);269 270 /*271 * For pre-MTL platforms, we need to restore the old value of the272 * steering control register to ensure that implicit steering continues273 * to behave as expected. For MTL and beyond, we need only reinstate274 * the 'multicast' bit (and only if we did a write that cleared it).275 */276 if (GRAPHICS_VER_FULL(uncore->i915) >= IP_VER(12, 70) && rw_flag == FW_REG_WRITE)277 intel_uncore_write_fw(uncore, MTL_MCR_SELECTOR, GEN11_MCR_MULTICAST);278 else if (GRAPHICS_VER_FULL(uncore->i915) < IP_VER(12, 70))279 intel_uncore_write_fw(uncore, GEN8_MCR_SELECTOR, old_mcr);280 281 return val;282}283 284static u32 rw_with_mcr_steering(struct intel_gt *gt,285 i915_mcr_reg_t reg, u8 rw_flag,286 int group, int instance,287 u32 value)288{289 struct intel_uncore *uncore = gt->uncore;290 enum forcewake_domains fw_domains;291 unsigned long flags;292 u32 val;293 294 fw_domains = intel_uncore_forcewake_for_reg(uncore, mcr_reg_cast(reg),295 rw_flag);296 fw_domains |= intel_uncore_forcewake_for_reg(uncore,297 GEN8_MCR_SELECTOR,298 FW_REG_READ | FW_REG_WRITE);299 300 intel_gt_mcr_lock(gt, &flags);301 spin_lock(&uncore->lock);302 intel_uncore_forcewake_get__locked(uncore, fw_domains);303 304 val = rw_with_mcr_steering_fw(gt, reg, rw_flag, group, instance, value);305 306 intel_uncore_forcewake_put__locked(uncore, fw_domains);307 spin_unlock(&uncore->lock);308 intel_gt_mcr_unlock(gt, flags);309 310 return val;311}312 313/**314 * intel_gt_mcr_lock - Acquire MCR steering lock315 * @gt: GT structure316 * @flags: storage to save IRQ flags to317 *318 * Performs locking to protect the steering for the duration of an MCR319 * operation. On MTL and beyond, a hardware lock will also be taken to320 * serialize access not only for the driver, but also for external hardware and321 * firmware agents.322 *323 * Context: Takes gt->mcr_lock. uncore->lock should *not* be held when this324 * function is called, although it may be acquired after this325 * function call.326 */327void intel_gt_mcr_lock(struct intel_gt *gt, unsigned long *flags)328 __acquires(>->mcr_lock)329{330 unsigned long __flags;331 int err = 0;332 333 lockdep_assert_not_held(>->uncore->lock);334 335 /*336 * Starting with MTL, we need to coordinate not only with other337 * driver threads, but also with hardware/firmware agents. A dedicated338 * locking register is used.339 */340 if (GRAPHICS_VER_FULL(gt->i915) >= IP_VER(12, 70)) {341 /*342 * The steering control and semaphore registers are inside an343 * "always on" power domain with respect to RC6. However there344 * are some issues if higher-level platform sleep states are345 * entering/exiting at the same time these registers are346 * accessed. Grabbing GT forcewake and holding it over the347 * entire lock/steer/unlock cycle ensures that those sleep348 * states have been fully exited before we access these349 * registers. This wakeref will be released in the unlock350 * routine.351 *352 * Wa_22018931422353 */354 intel_uncore_forcewake_get(gt->uncore, FORCEWAKE_GT);355 356 err = wait_for(intel_uncore_read_fw(gt->uncore,357 MTL_STEER_SEMAPHORE) == 0x1, 100);358 }359 360 /*361 * Even on platforms with a hardware lock, we'll continue to grab362 * a software spinlock too for lockdep purposes. If the hardware lock363 * was already acquired, there should never be contention on the364 * software lock.365 */366 spin_lock_irqsave(>->mcr_lock, __flags);367 368 *flags = __flags;369 370 /*371 * In theory we should never fail to acquire the HW semaphore; this372 * would indicate some hardware/firmware is misbehaving and not373 * releasing it properly.374 */375 if (err == -ETIMEDOUT) {376 gt_err_ratelimited(gt, "hardware MCR steering semaphore timed out");377 add_taint_for_CI(gt->i915, TAINT_WARN); /* CI is now unreliable */378 }379}380 381/**382 * intel_gt_mcr_unlock - Release MCR steering lock383 * @gt: GT structure384 * @flags: IRQ flags to restore385 *386 * Releases the lock acquired by intel_gt_mcr_lock().387 *388 * Context: Releases gt->mcr_lock389 */390void intel_gt_mcr_unlock(struct intel_gt *gt, unsigned long flags)391 __releases(>->mcr_lock)392{393 spin_unlock_irqrestore(>->mcr_lock, flags);394 395 if (GRAPHICS_VER_FULL(gt->i915) >= IP_VER(12, 70)) {396 intel_uncore_write_fw(gt->uncore, MTL_STEER_SEMAPHORE, 0x1);397 398 intel_uncore_forcewake_put(gt->uncore, FORCEWAKE_GT);399 }400}401 402/**403 * intel_gt_mcr_lock_sanitize - Sanitize MCR steering lock404 * @gt: GT structure405 *406 * This will be used to sanitize the initial status of the hardware lock407 * during driver load and resume since there won't be any concurrent access408 * from other agents at those times, but it's possible that boot firmware409 * may have left the lock in a bad state.410 *411 */412void intel_gt_mcr_lock_sanitize(struct intel_gt *gt)413{414 /*415 * This gets called at load/resume time, so we shouldn't be416 * racing with other driver threads grabbing the mcr lock.417 */418 lockdep_assert_not_held(>->mcr_lock);419 420 if (GRAPHICS_VER_FULL(gt->i915) >= IP_VER(12, 70))421 intel_uncore_write_fw(gt->uncore, MTL_STEER_SEMAPHORE, 0x1);422}423 424/**425 * intel_gt_mcr_read - read a specific instance of an MCR register426 * @gt: GT structure427 * @reg: the MCR register to read428 * @group: the MCR group429 * @instance: the MCR instance430 *431 * Context: Takes and releases gt->mcr_lock432 *433 * Returns the value read from an MCR register after steering toward a specific434 * group/instance.435 */436u32 intel_gt_mcr_read(struct intel_gt *gt,437 i915_mcr_reg_t reg,438 int group, int instance)439{440 return rw_with_mcr_steering(gt, reg, FW_REG_READ, group, instance, 0);441}442 443/**444 * intel_gt_mcr_unicast_write - write a specific instance of an MCR register445 * @gt: GT structure446 * @reg: the MCR register to write447 * @value: value to write448 * @group: the MCR group449 * @instance: the MCR instance450 *451 * Write an MCR register in unicast mode after steering toward a specific452 * group/instance.453 *454 * Context: Calls a function that takes and releases gt->mcr_lock455 */456void intel_gt_mcr_unicast_write(struct intel_gt *gt, i915_mcr_reg_t reg, u32 value,457 int group, int instance)458{459 rw_with_mcr_steering(gt, reg, FW_REG_WRITE, group, instance, value);460}461 462/**463 * intel_gt_mcr_multicast_write - write a value to all instances of an MCR register464 * @gt: GT structure465 * @reg: the MCR register to write466 * @value: value to write467 *468 * Write an MCR register in multicast mode to update all instances.469 *470 * Context: Takes and releases gt->mcr_lock471 */472void intel_gt_mcr_multicast_write(struct intel_gt *gt,473 i915_mcr_reg_t reg, u32 value)474{475 unsigned long flags;476 477 intel_gt_mcr_lock(gt, &flags);478 479 /*480 * Ensure we have multicast behavior, just in case some non-i915 agent481 * left the hardware in unicast mode.482 */483 if (GRAPHICS_VER_FULL(gt->i915) >= IP_VER(12, 70))484 intel_uncore_write_fw(gt->uncore, MTL_MCR_SELECTOR, GEN11_MCR_MULTICAST);485 486 intel_uncore_write(gt->uncore, mcr_reg_cast(reg), value);487 488 intel_gt_mcr_unlock(gt, flags);489}490 491/**492 * intel_gt_mcr_multicast_write_fw - write a value to all instances of an MCR register493 * @gt: GT structure494 * @reg: the MCR register to write495 * @value: value to write496 *497 * Write an MCR register in multicast mode to update all instances. This498 * function assumes the caller is already holding any necessary forcewake499 * domains; use intel_gt_mcr_multicast_write() in cases where forcewake should500 * be obtained automatically.501 *502 * Context: The caller must hold gt->mcr_lock.503 */504void intel_gt_mcr_multicast_write_fw(struct intel_gt *gt, i915_mcr_reg_t reg, u32 value)505{506 lockdep_assert_held(>->mcr_lock);507 508 /*509 * Ensure we have multicast behavior, just in case some non-i915 agent510 * left the hardware in unicast mode.511 */512 if (GRAPHICS_VER_FULL(gt->i915) >= IP_VER(12, 70))513 intel_uncore_write_fw(gt->uncore, MTL_MCR_SELECTOR, GEN11_MCR_MULTICAST);514 515 intel_uncore_write_fw(gt->uncore, mcr_reg_cast(reg), value);516}517 518/**519 * intel_gt_mcr_multicast_rmw - Performs a multicast RMW operations520 * @gt: GT structure521 * @reg: the MCR register to read and write522 * @clear: bits to clear during RMW523 * @set: bits to set during RMW524 *525 * Performs a read-modify-write on an MCR register in a multicast manner.526 * This operation only makes sense on MCR registers where all instances are527 * expected to have the same value. The read will target any non-terminated528 * instance and the write will be applied to all instances.529 *530 * This function assumes the caller is already holding any necessary forcewake531 * domains; use intel_gt_mcr_multicast_rmw() in cases where forcewake should532 * be obtained automatically.533 *534 * Context: Calls functions that take and release gt->mcr_lock535 *536 * Returns the old (unmodified) value read.537 */538u32 intel_gt_mcr_multicast_rmw(struct intel_gt *gt, i915_mcr_reg_t reg,539 u32 clear, u32 set)540{541 u32 val = intel_gt_mcr_read_any(gt, reg);542 543 intel_gt_mcr_multicast_write(gt, reg, (val & ~clear) | set);544 545 return val;546}547 548/*549 * reg_needs_read_steering - determine whether a register read requires550 * explicit steering551 * @gt: GT structure552 * @reg: the register to check steering requirements for553 * @type: type of multicast steering to check554 *555 * Determines whether @reg needs explicit steering of a specific type for556 * reads.557 *558 * Returns false if @reg does not belong to a register range of the given559 * steering type, or if the default (subslice-based) steering IDs are suitable560 * for @type steering too.561 */562static bool reg_needs_read_steering(struct intel_gt *gt,563 i915_mcr_reg_t reg,564 enum intel_steering_type type)565{566 u32 offset = i915_mmio_reg_offset(reg);567 const struct intel_mmio_range *entry;568 569 if (likely(!gt->steering_table[type]))570 return false;571 572 if (IS_GSI_REG(offset))573 offset += gt->uncore->gsi_offset;574 575 for (entry = gt->steering_table[type]; entry->end; entry++) {576 if (offset >= entry->start && offset <= entry->end)577 return true;578 }579 580 return false;581}582 583/*584 * get_nonterminated_steering - determines valid IDs for a class of MCR steering585 * @gt: GT structure586 * @type: multicast register type587 * @group: Group ID returned588 * @instance: Instance ID returned589 *590 * Determines group and instance values that will steer reads of the specified591 * MCR class to a non-terminated instance.592 */593static void get_nonterminated_steering(struct intel_gt *gt,594 enum intel_steering_type type,595 u8 *group, u8 *instance)596{597 u32 dss;598 599 switch (type) {600 case L3BANK:601 *group = 0; /* unused */602 *instance = __ffs(gt->info.l3bank_mask);603 break;604 case MSLICE:605 GEM_WARN_ON(!HAS_MSLICE_STEERING(gt->i915));606 *group = __ffs(gt->info.mslice_mask);607 *instance = 0; /* unused */608 break;609 case LNCF:610 /*611 * An LNCF is always present if its mslice is present, so we612 * can safely just steer to LNCF 0 in all cases.613 */614 GEM_WARN_ON(!HAS_MSLICE_STEERING(gt->i915));615 *group = __ffs(gt->info.mslice_mask) << 1;616 *instance = 0; /* unused */617 break;618 case GAM:619 *group = IS_DG2(gt->i915) ? 1 : 0;620 *instance = 0;621 break;622 case DSS:623 dss = intel_sseu_find_first_xehp_dss(>->info.sseu, 0, 0);624 *group = dss / GEN_DSS_PER_GSLICE;625 *instance = dss % GEN_DSS_PER_GSLICE;626 break;627 case INSTANCE0:628 /*629 * There are a lot of MCR types for which instance (0, 0)630 * will always provide a non-terminated value.631 */632 *group = 0;633 *instance = 0;634 break;635 case OADDRM:636 if ((VDBOX_MASK(gt) | VEBOX_MASK(gt) | gt->info.sfc_mask) & BIT(0))637 *group = 0;638 else639 *group = 1;640 *instance = 0;641 break;642 default:643 MISSING_CASE(type);644 *group = 0;645 *instance = 0;646 }647}648 649/**650 * intel_gt_mcr_get_nonterminated_steering - find group/instance values that651 * will steer a register to a non-terminated instance652 * @gt: GT structure653 * @reg: register for which the steering is required654 * @group: return variable for group steering655 * @instance: return variable for instance steering656 *657 * This function returns a group/instance pair that is guaranteed to work for658 * read steering of the given register. Note that a value will be returned even659 * if the register is not replicated and therefore does not actually require660 * steering.661 */662void intel_gt_mcr_get_nonterminated_steering(struct intel_gt *gt,663 i915_mcr_reg_t reg,664 u8 *group, u8 *instance)665{666 int type;667 668 for (type = 0; type < NUM_STEERING_TYPES; type++) {669 if (reg_needs_read_steering(gt, reg, type)) {670 get_nonterminated_steering(gt, type, group, instance);671 return;672 }673 }674 675 *group = gt->default_steering.groupid;676 *instance = gt->default_steering.instanceid;677}678 679/**680 * intel_gt_mcr_read_any_fw - reads one instance of an MCR register681 * @gt: GT structure682 * @reg: register to read683 *684 * Reads a GT MCR register. The read will be steered to a non-terminated685 * instance (i.e., one that isn't fused off or powered down by power gating).686 * This function assumes the caller is already holding any necessary forcewake687 * domains; use intel_gt_mcr_read_any() in cases where forcewake should be688 * obtained automatically.689 *690 * Context: The caller must hold gt->mcr_lock.691 *692 * Returns the value from a non-terminated instance of @reg.693 */694u32 intel_gt_mcr_read_any_fw(struct intel_gt *gt, i915_mcr_reg_t reg)695{696 int type;697 u8 group, instance;698 699 lockdep_assert_held(>->mcr_lock);700 701 for (type = 0; type < NUM_STEERING_TYPES; type++) {702 if (reg_needs_read_steering(gt, reg, type)) {703 get_nonterminated_steering(gt, type, &group, &instance);704 return rw_with_mcr_steering_fw(gt, reg,705 FW_REG_READ,706 group, instance, 0);707 }708 }709 710 return intel_uncore_read_fw(gt->uncore, mcr_reg_cast(reg));711}712 713/**714 * intel_gt_mcr_read_any - reads one instance of an MCR register715 * @gt: GT structure716 * @reg: register to read717 *718 * Reads a GT MCR register. The read will be steered to a non-terminated719 * instance (i.e., one that isn't fused off or powered down by power gating).720 *721 * Context: Calls a function that takes and releases gt->mcr_lock.722 *723 * Returns the value from a non-terminated instance of @reg.724 */725u32 intel_gt_mcr_read_any(struct intel_gt *gt, i915_mcr_reg_t reg)726{727 int type;728 u8 group, instance;729 730 for (type = 0; type < NUM_STEERING_TYPES; type++) {731 if (reg_needs_read_steering(gt, reg, type)) {732 get_nonterminated_steering(gt, type, &group, &instance);733 return rw_with_mcr_steering(gt, reg,734 FW_REG_READ,735 group, instance, 0);736 }737 }738 739 return intel_uncore_read(gt->uncore, mcr_reg_cast(reg));740}741 742static void report_steering_type(struct drm_printer *p,743 struct intel_gt *gt,744 enum intel_steering_type type,745 bool dump_table)746{747 const struct intel_mmio_range *entry;748 u8 group, instance;749 750 BUILD_BUG_ON(ARRAY_SIZE(intel_steering_types) != NUM_STEERING_TYPES);751 752 if (!gt->steering_table[type]) {753 drm_printf(p, "%s steering: uses default steering\n",754 intel_steering_types[type]);755 return;756 }757 758 get_nonterminated_steering(gt, type, &group, &instance);759 drm_printf(p, "%s steering: group=0x%x, instance=0x%x\n",760 intel_steering_types[type], group, instance);761 762 if (!dump_table)763 return;764 765 for (entry = gt->steering_table[type]; entry->end; entry++)766 drm_printf(p, "\t0x%06x - 0x%06x\n", entry->start, entry->end);767}768 769void intel_gt_mcr_report_steering(struct drm_printer *p, struct intel_gt *gt,770 bool dump_table)771{772 /*773 * Starting with MTL we no longer have default steering;774 * all ranges are explicitly steered.775 */776 if (GRAPHICS_VER_FULL(gt->i915) < IP_VER(12, 70))777 drm_printf(p, "Default steering: group=0x%x, instance=0x%x\n",778 gt->default_steering.groupid,779 gt->default_steering.instanceid);780 781 if (GRAPHICS_VER_FULL(gt->i915) >= IP_VER(12, 70)) {782 for (int i = 0; i < NUM_STEERING_TYPES; i++)783 if (gt->steering_table[i])784 report_steering_type(p, gt, i, dump_table);785 } else if (HAS_MSLICE_STEERING(gt->i915)) {786 report_steering_type(p, gt, MSLICE, dump_table);787 report_steering_type(p, gt, LNCF, dump_table);788 }789}790 791/**792 * intel_gt_mcr_get_ss_steering - returns the group/instance steering for a SS793 * @gt: GT structure794 * @dss: DSS ID to obtain steering for795 * @group: pointer to storage for steering group ID796 * @instance: pointer to storage for steering instance ID797 *798 * Returns the steering IDs (via the @group and @instance parameters) that799 * correspond to a specific subslice/DSS ID.800 */801void intel_gt_mcr_get_ss_steering(struct intel_gt *gt, unsigned int dss,802 unsigned int *group, unsigned int *instance)803{804 if (GRAPHICS_VER_FULL(gt->i915) >= IP_VER(12, 55)) {805 *group = dss / GEN_DSS_PER_GSLICE;806 *instance = dss % GEN_DSS_PER_GSLICE;807 } else {808 *group = dss / GEN_MAX_SS_PER_HSW_SLICE;809 *instance = dss % GEN_MAX_SS_PER_HSW_SLICE;810 return;811 }812}813 814/**815 * intel_gt_mcr_wait_for_reg - wait until MCR register matches expected state816 * @gt: GT structure817 * @reg: the register to read818 * @mask: mask to apply to register value819 * @value: value to wait for820 * @fast_timeout_us: fast timeout in microsecond for atomic/tight wait821 * @slow_timeout_ms: slow timeout in millisecond822 *823 * This routine waits until the target register @reg contains the expected824 * @value after applying the @mask, i.e. it waits until ::825 *826 * (intel_gt_mcr_read_any_fw(gt, reg) & mask) == value827 *828 * Otherwise, the wait will timeout after @slow_timeout_ms milliseconds.829 * For atomic context @slow_timeout_ms must be zero and @fast_timeout_us830 * must be not larger than 20,0000 microseconds.831 *832 * This function is basically an MCR-friendly version of833 * __intel_wait_for_register_fw(). Generally this function will only be used834 * on GAM registers which are a bit special --- although they're MCR registers,835 * reads (e.g., waiting for status updates) are always directed to the primary836 * instance.837 *838 * Note that this routine assumes the caller holds forcewake asserted, it is839 * not suitable for very long waits.840 *841 * Context: Calls a function that takes and releases gt->mcr_lock842 * Return: 0 if the register matches the desired condition, or -ETIMEDOUT.843 */844int intel_gt_mcr_wait_for_reg(struct intel_gt *gt,845 i915_mcr_reg_t reg,846 u32 mask,847 u32 value,848 unsigned int fast_timeout_us,849 unsigned int slow_timeout_ms)850{851 int ret;852 853 lockdep_assert_not_held(>->mcr_lock);854 855#define done ((intel_gt_mcr_read_any(gt, reg) & mask) == value)856 857 /* Catch any overuse of this function */858 might_sleep_if(slow_timeout_ms);859 GEM_BUG_ON(fast_timeout_us > 20000);860 GEM_BUG_ON(!fast_timeout_us && !slow_timeout_ms);861 862 ret = -ETIMEDOUT;863 if (fast_timeout_us && fast_timeout_us <= 20000)864 ret = _wait_for_atomic(done, fast_timeout_us, 0);865 if (ret && slow_timeout_ms)866 ret = wait_for(done, slow_timeout_ms);867 868 return ret;869#undef done870}871