779 lines · c
1// SPDX-License-Identifier: MIT2/*3 * Copyright © 2022 Intel Corporation4 */5 6#include "xe_gt_mcr.h"7 8#include "regs/xe_gt_regs.h"9#include "xe_assert.h"10#include "xe_gt.h"11#include "xe_gt_printk.h"12#include "xe_gt_topology.h"13#include "xe_gt_types.h"14#include "xe_guc_hwconfig.h"15#include "xe_mmio.h"16#include "xe_sriov.h"17 18/**19 * DOC: GT Multicast/Replicated (MCR) Register Support20 *21 * Some GT registers are designed as "multicast" or "replicated" registers:22 * multiple instances of the same register share a single MMIO offset. MCR23 * registers are generally used when the hardware needs to potentially track24 * independent values of a register per hardware unit (e.g., per-subslice,25 * per-L3bank, etc.). The specific types of replication that exist vary26 * per-platform.27 *28 * MMIO accesses to MCR registers are controlled according to the settings29 * programmed in the platform's MCR_SELECTOR register(s). MMIO writes to MCR30 * registers can be done in either multicast (a single write updates all31 * instances of the register to the same value) or unicast (a write updates only32 * one specific instance) form. Reads of MCR registers always operate in a33 * unicast manner regardless of how the multicast/unicast bit is set in34 * MCR_SELECTOR. Selection of a specific MCR instance for unicast operations is35 * referred to as "steering."36 *37 * If MCR register operations are steered toward a hardware unit that is38 * fused off or currently powered down due to power gating, the MMIO operation39 * is "terminated" by the hardware. Terminated read operations will return a40 * value of zero and terminated unicast write operations will be silently41 * ignored. During device initialization, the goal of the various42 * ``init_steering_*()`` functions is to apply the platform-specific rules for43 * each MCR register type to identify a steering target that will select a44 * non-terminated instance.45 *46 * MCR registers are not available on Virtual Function (VF).47 */48 49#define STEER_SEMAPHORE XE_REG(0xFD0)50 51static inline struct xe_reg to_xe_reg(struct xe_reg_mcr reg_mcr)52{53 return reg_mcr.__reg;54}55 56enum {57 MCR_OP_READ,58 MCR_OP_WRITE59};60 61static const struct xe_mmio_range xelp_l3bank_steering_table[] = {62 { 0x00B100, 0x00B3FF },63 {},64};65 66static const struct xe_mmio_range xehp_l3bank_steering_table[] = {67 { 0x008C80, 0x008CFF },68 { 0x00B100, 0x00B3FF },69 {},70};71 72/*73 * Although the bspec lists more "MSLICE" ranges than shown here, some of those74 * are of a "GAM" subclass that has special rules and doesn't need to be75 * included here.76 */77static const struct xe_mmio_range xehp_mslice_steering_table[] = {78 { 0x00DD00, 0x00DDFF },79 { 0x00E900, 0x00FFFF }, /* 0xEA00 - OxEFFF is unused */80 {},81};82 83static const struct xe_mmio_range xehp_lncf_steering_table[] = {84 { 0x00B000, 0x00B0FF },85 { 0x00D880, 0x00D8FF },86 {},87};88 89/*90 * We have several types of MCR registers where steering to (0,0) will always91 * provide us with a non-terminated value. We'll stick them all in the same92 * table for simplicity.93 */94static const struct xe_mmio_range xehpc_instance0_steering_table[] = {95 { 0x004000, 0x004AFF }, /* HALF-BSLICE */96 { 0x008800, 0x00887F }, /* CC */97 { 0x008A80, 0x008AFF }, /* TILEPSMI */98 { 0x00B000, 0x00B0FF }, /* HALF-BSLICE */99 { 0x00B100, 0x00B3FF }, /* L3BANK */100 { 0x00C800, 0x00CFFF }, /* HALF-BSLICE */101 { 0x00D800, 0x00D8FF }, /* HALF-BSLICE */102 { 0x00DD00, 0x00DDFF }, /* BSLICE */103 { 0x00E900, 0x00E9FF }, /* HALF-BSLICE */104 { 0x00EC00, 0x00EEFF }, /* HALF-BSLICE */105 { 0x00F000, 0x00FFFF }, /* HALF-BSLICE */106 { 0x024180, 0x0241FF }, /* HALF-BSLICE */107 {},108};109 110static const struct xe_mmio_range xelpg_instance0_steering_table[] = {111 { 0x000B00, 0x000BFF }, /* SQIDI */112 { 0x001000, 0x001FFF }, /* SQIDI */113 { 0x004000, 0x0048FF }, /* GAM */114 { 0x008700, 0x0087FF }, /* SQIDI */115 { 0x00B000, 0x00B0FF }, /* NODE */116 { 0x00C800, 0x00CFFF }, /* GAM */117 { 0x00D880, 0x00D8FF }, /* NODE */118 { 0x00DD00, 0x00DDFF }, /* OAAL2 */119 {},120};121 122static const struct xe_mmio_range xelpg_l3bank_steering_table[] = {123 { 0x00B100, 0x00B3FF },124 {},125};126 127static const struct xe_mmio_range xelp_dss_steering_table[] = {128 { 0x008150, 0x00815F },129 { 0x009520, 0x00955F },130 { 0x00DE80, 0x00E8FF },131 { 0x024A00, 0x024A7F },132 {},133};134 135/* DSS steering is used for GSLICE ranges as well */136static const struct xe_mmio_range xehp_dss_steering_table[] = {137 { 0x005200, 0x0052FF }, /* GSLICE */138 { 0x005400, 0x007FFF }, /* GSLICE */139 { 0x008140, 0x00815F }, /* GSLICE (0x8140-0x814F), DSS (0x8150-0x815F) */140 { 0x008D00, 0x008DFF }, /* DSS */141 { 0x0094D0, 0x00955F }, /* GSLICE (0x94D0-0x951F), DSS (0x9520-0x955F) */142 { 0x009680, 0x0096FF }, /* DSS */143 { 0x00D800, 0x00D87F }, /* GSLICE */144 { 0x00DC00, 0x00DCFF }, /* GSLICE */145 { 0x00DE80, 0x00E8FF }, /* DSS (0xE000-0xE0FF reserved ) */146 { 0x017000, 0x017FFF }, /* GSLICE */147 { 0x024A00, 0x024A7F }, /* DSS */148 {},149};150 151/* DSS steering is used for COMPUTE ranges as well */152static const struct xe_mmio_range xehpc_dss_steering_table[] = {153 { 0x008140, 0x00817F }, /* COMPUTE (0x8140-0x814F & 0x8160-0x817F), DSS (0x8150-0x815F) */154 { 0x0094D0, 0x00955F }, /* COMPUTE (0x94D0-0x951F), DSS (0x9520-0x955F) */155 { 0x009680, 0x0096FF }, /* DSS */156 { 0x00DC00, 0x00DCFF }, /* COMPUTE */157 { 0x00DE80, 0x00E7FF }, /* DSS (0xDF00-0xE1FF reserved ) */158 {},159};160 161/* DSS steering is used for SLICE ranges as well */162static const struct xe_mmio_range xelpg_dss_steering_table[] = {163 { 0x005200, 0x0052FF }, /* SLICE */164 { 0x005500, 0x007FFF }, /* SLICE */165 { 0x008140, 0x00815F }, /* SLICE (0x8140-0x814F), DSS (0x8150-0x815F) */166 { 0x0094D0, 0x00955F }, /* SLICE (0x94D0-0x951F), DSS (0x9520-0x955F) */167 { 0x009680, 0x0096FF }, /* DSS */168 { 0x00D800, 0x00D87F }, /* SLICE */169 { 0x00DC00, 0x00DCFF }, /* SLICE */170 { 0x00DE80, 0x00E8FF }, /* DSS (0xE000-0xE0FF reserved) */171 {},172};173 174static const struct xe_mmio_range xelpmp_oaddrm_steering_table[] = {175 { 0x393200, 0x39323F },176 { 0x393400, 0x3934FF },177 {},178};179 180static const struct xe_mmio_range dg2_implicit_steering_table[] = {181 { 0x000B00, 0x000BFF }, /* SF (SQIDI replication) */182 { 0x001000, 0x001FFF }, /* SF (SQIDI replication) */183 { 0x004000, 0x004AFF }, /* GAM (MSLICE replication) */184 { 0x008700, 0x0087FF }, /* MCFG (SQIDI replication) */185 { 0x00C800, 0x00CFFF }, /* GAM (MSLICE replication) */186 { 0x00F000, 0x00FFFF }, /* GAM (MSLICE replication) */187 {},188};189 190static const struct xe_mmio_range xe2lpg_dss_steering_table[] = {191 { 0x005200, 0x0052FF }, /* SLICE */192 { 0x005500, 0x007FFF }, /* SLICE */193 { 0x008140, 0x00815F }, /* SLICE (0x8140-0x814F), DSS (0x8150-0x815F) */194 { 0x0094D0, 0x00955F }, /* SLICE (0x94D0-0x951F), DSS (0x9520-0x955F) */195 { 0x009680, 0x0096FF }, /* DSS */196 { 0x00D800, 0x00D87F }, /* SLICE */197 { 0x00DC00, 0x00DCFF }, /* SLICE */198 { 0x00DE80, 0x00E8FF }, /* DSS (0xE000-0xE0FF reserved) */199 { 0x00E980, 0x00E9FF }, /* SLICE */200 { 0x013000, 0x0133FF }, /* DSS (0x13000-0x131FF), SLICE (0x13200-0x133FF) */201 {},202};203 204static const struct xe_mmio_range xe2lpg_sqidi_psmi_steering_table[] = {205 { 0x000B00, 0x000BFF },206 { 0x001000, 0x001FFF },207 {},208};209 210static const struct xe_mmio_range xe2lpg_instance0_steering_table[] = {211 { 0x004000, 0x004AFF }, /* GAM, rsvd, GAMWKR */212 { 0x008700, 0x00887F }, /* SQIDI, MEMPIPE */213 { 0x00B000, 0x00B3FF }, /* NODE, L3BANK */214 { 0x00C800, 0x00CFFF }, /* GAM */215 { 0x00D880, 0x00D8FF }, /* NODE */216 { 0x00DD00, 0x00DDFF }, /* MEMPIPE */217 { 0x00E900, 0x00E97F }, /* MEMPIPE */218 { 0x00F000, 0x00FFFF }, /* GAM, GAMWKR */219 { 0x013400, 0x0135FF }, /* MEMPIPE */220 {},221};222 223static const struct xe_mmio_range xe2lpm_gpmxmt_steering_table[] = {224 { 0x388160, 0x38817F },225 { 0x389480, 0x3894CF },226 {},227};228 229static const struct xe_mmio_range xe2lpm_instance0_steering_table[] = {230 { 0x384000, 0x3847DF }, /* GAM, rsvd, GAM */231 { 0x384900, 0x384AFF }, /* GAM */232 { 0x389560, 0x3895FF }, /* MEDIAINF */233 { 0x38B600, 0x38B8FF }, /* L3BANK */234 { 0x38C800, 0x38D07F }, /* GAM, MEDIAINF */235 { 0x38F000, 0x38F0FF }, /* GAM */236 { 0x393C00, 0x393C7F }, /* MEDIAINF */237 {},238};239 240static void init_steering_l3bank(struct xe_gt *gt)241{242 if (GRAPHICS_VERx100(gt_to_xe(gt)) >= 1270) {243 u32 mslice_mask = REG_FIELD_GET(MEML3_EN_MASK,244 xe_mmio_read32(gt, MIRROR_FUSE3));245 u32 bank_mask = REG_FIELD_GET(GT_L3_EXC_MASK,246 xe_mmio_read32(gt, XEHP_FUSE4));247 248 /*249 * Group selects mslice, instance selects bank within mslice.250 * Bank 0 is always valid _except_ when the bank mask is 010b.251 */252 gt->steering[L3BANK].group_target = __ffs(mslice_mask);253 gt->steering[L3BANK].instance_target =254 bank_mask & BIT(0) ? 0 : 2;255 } else if (gt_to_xe(gt)->info.platform == XE_DG2) {256 u32 mslice_mask = REG_FIELD_GET(MEML3_EN_MASK,257 xe_mmio_read32(gt, MIRROR_FUSE3));258 u32 bank = __ffs(mslice_mask) * 8;259 260 /*261 * Like mslice registers, look for a valid mslice and steer to262 * the first L3BANK of that quad. Access to the Nth L3 bank is263 * split between the first bits of group and instance264 */265 gt->steering[L3BANK].group_target = (bank >> 2) & 0x7;266 gt->steering[L3BANK].instance_target = bank & 0x3;267 } else {268 u32 fuse = REG_FIELD_GET(L3BANK_MASK,269 ~xe_mmio_read32(gt, MIRROR_FUSE3));270 271 gt->steering[L3BANK].group_target = 0; /* unused */272 gt->steering[L3BANK].instance_target = __ffs(fuse);273 }274}275 276static void init_steering_mslice(struct xe_gt *gt)277{278 u32 mask = REG_FIELD_GET(MEML3_EN_MASK,279 xe_mmio_read32(gt, MIRROR_FUSE3));280 281 /*282 * mslice registers are valid (not terminated) if either the meml3283 * associated with the mslice is present, or at least one DSS associated284 * with the mslice is present. There will always be at least one meml3285 * so we can just use that to find a non-terminated mslice and ignore286 * the DSS fusing.287 */288 gt->steering[MSLICE].group_target = __ffs(mask);289 gt->steering[MSLICE].instance_target = 0; /* unused */290 291 /*292 * LNCF termination is also based on mslice presence, so we'll set293 * it up here. Either LNCF within a non-terminated mslice will work,294 * so we just always pick LNCF 0 here.295 */296 gt->steering[LNCF].group_target = __ffs(mask) << 1;297 gt->steering[LNCF].instance_target = 0; /* unused */298}299 300static unsigned int dss_per_group(struct xe_gt *gt)301{302 struct xe_guc *guc = >->uc.guc;303 u32 max_slices = 0, max_subslices = 0;304 int ret;305 306 /*307 * Try to query the GuC's hwconfig table for the maximum number of308 * slices and subslices. These don't reflect the platform's actual309 * slice/DSS counts, just the physical layout by which we should310 * determine the steering targets. On older platforms with older GuC311 * firmware releases it's possible that these attributes may not be312 * included in the table, so we can always fall back to the old313 * hardcoded layouts.314 */315#define HWCONFIG_ATTR_MAX_SLICES 1316#define HWCONFIG_ATTR_MAX_SUBSLICES 70317 318 ret = xe_guc_hwconfig_lookup_u32(guc, HWCONFIG_ATTR_MAX_SLICES,319 &max_slices);320 if (ret < 0 || max_slices == 0)321 goto fallback;322 323 ret = xe_guc_hwconfig_lookup_u32(guc, HWCONFIG_ATTR_MAX_SUBSLICES,324 &max_subslices);325 if (ret < 0 || max_subslices == 0)326 goto fallback;327 328 return DIV_ROUND_UP(max_subslices, max_slices);329 330fallback:331 xe_gt_dbg(gt, "GuC hwconfig cannot provide dss/slice; using typical fallback values\n");332 if (gt_to_xe(gt)->info.platform == XE_PVC)333 return 8;334 else if (GRAPHICS_VERx100(gt_to_xe(gt)) >= 1250)335 return 4;336 else337 return 6;338}339 340/**341 * xe_gt_mcr_get_dss_steering - Get the group/instance steering for a DSS342 * @gt: GT structure343 * @dss: DSS ID to obtain steering for344 * @group: pointer to storage for steering group ID345 * @instance: pointer to storage for steering instance ID346 */347void xe_gt_mcr_get_dss_steering(struct xe_gt *gt, unsigned int dss, u16 *group, u16 *instance)348{349 xe_gt_assert(gt, dss < XE_MAX_DSS_FUSE_BITS);350 351 *group = dss / gt->steering_dss_per_grp;352 *instance = dss % gt->steering_dss_per_grp;353}354 355static void init_steering_dss(struct xe_gt *gt)356{357 gt->steering_dss_per_grp = dss_per_group(gt);358 359 xe_gt_mcr_get_dss_steering(gt,360 min(xe_dss_mask_group_ffs(gt->fuse_topo.g_dss_mask, 0, 0),361 xe_dss_mask_group_ffs(gt->fuse_topo.c_dss_mask, 0, 0)),362 >->steering[DSS].group_target,363 >->steering[DSS].instance_target);364}365 366static void init_steering_oaddrm(struct xe_gt *gt)367{368 /*369 * First instance is only terminated if the entire first media slice370 * is absent (i.e., no VCS0 or VECS0).371 */372 if (gt->info.engine_mask & (XE_HW_ENGINE_VCS0 | XE_HW_ENGINE_VECS0))373 gt->steering[OADDRM].group_target = 0;374 else375 gt->steering[OADDRM].group_target = 1;376 377 gt->steering[OADDRM].instance_target = 0; /* unused */378}379 380static void init_steering_sqidi_psmi(struct xe_gt *gt)381{382 u32 mask = REG_FIELD_GET(XE2_NODE_ENABLE_MASK,383 xe_mmio_read32(gt, MIRROR_FUSE3));384 u32 select = __ffs(mask);385 386 gt->steering[SQIDI_PSMI].group_target = select >> 1;387 gt->steering[SQIDI_PSMI].instance_target = select & 0x1;388}389 390static void init_steering_inst0(struct xe_gt *gt)391{392 gt->steering[INSTANCE0].group_target = 0; /* unused */393 gt->steering[INSTANCE0].instance_target = 0; /* unused */394}395 396static const struct {397 const char *name;398 void (*init)(struct xe_gt *gt);399} xe_steering_types[] = {400 [L3BANK] = { "L3BANK", init_steering_l3bank },401 [MSLICE] = { "MSLICE", init_steering_mslice },402 [LNCF] = { "LNCF", NULL }, /* initialized by mslice init */403 [DSS] = { "DSS", init_steering_dss },404 [OADDRM] = { "OADDRM / GPMXMT", init_steering_oaddrm },405 [SQIDI_PSMI] = { "SQIDI_PSMI", init_steering_sqidi_psmi },406 [INSTANCE0] = { "INSTANCE 0", init_steering_inst0 },407 [IMPLICIT_STEERING] = { "IMPLICIT", NULL },408};409 410/**411 * xe_gt_mcr_init_early - Early initialization of the MCR support412 * @gt: GT structure413 *414 * Perform early software only initialization of the MCR lock to allow415 * the synchronization on accessing the STEER_SEMAPHORE register and416 * use the xe_gt_mcr_multicast_write() function.417 */418void xe_gt_mcr_init_early(struct xe_gt *gt)419{420 BUILD_BUG_ON(IMPLICIT_STEERING + 1 != NUM_STEERING_TYPES);421 BUILD_BUG_ON(ARRAY_SIZE(xe_steering_types) != NUM_STEERING_TYPES);422 423 spin_lock_init(>->mcr_lock);424}425 426/**427 * xe_gt_mcr_init - Normal initialization of the MCR support428 * @gt: GT structure429 *430 * Perform normal initialization of the MCR for all usages.431 */432void xe_gt_mcr_init(struct xe_gt *gt)433{434 struct xe_device *xe = gt_to_xe(gt);435 436 if (IS_SRIOV_VF(xe))437 return;438 439 if (gt->info.type == XE_GT_TYPE_MEDIA) {440 drm_WARN_ON(&xe->drm, MEDIA_VER(xe) < 13);441 442 if (MEDIA_VERx100(xe) >= 1301) {443 gt->steering[OADDRM].ranges = xe2lpm_gpmxmt_steering_table;444 gt->steering[INSTANCE0].ranges = xe2lpm_instance0_steering_table;445 } else {446 gt->steering[OADDRM].ranges = xelpmp_oaddrm_steering_table;447 }448 } else {449 if (GRAPHICS_VER(xe) >= 20) {450 gt->steering[DSS].ranges = xe2lpg_dss_steering_table;451 gt->steering[SQIDI_PSMI].ranges = xe2lpg_sqidi_psmi_steering_table;452 gt->steering[INSTANCE0].ranges = xe2lpg_instance0_steering_table;453 } else if (GRAPHICS_VERx100(xe) >= 1270) {454 gt->steering[INSTANCE0].ranges = xelpg_instance0_steering_table;455 gt->steering[L3BANK].ranges = xelpg_l3bank_steering_table;456 gt->steering[DSS].ranges = xelpg_dss_steering_table;457 } else if (xe->info.platform == XE_PVC) {458 gt->steering[INSTANCE0].ranges = xehpc_instance0_steering_table;459 gt->steering[DSS].ranges = xehpc_dss_steering_table;460 } else if (xe->info.platform == XE_DG2) {461 gt->steering[L3BANK].ranges = xehp_l3bank_steering_table;462 gt->steering[MSLICE].ranges = xehp_mslice_steering_table;463 gt->steering[LNCF].ranges = xehp_lncf_steering_table;464 gt->steering[DSS].ranges = xehp_dss_steering_table;465 gt->steering[IMPLICIT_STEERING].ranges = dg2_implicit_steering_table;466 } else {467 gt->steering[L3BANK].ranges = xelp_l3bank_steering_table;468 gt->steering[DSS].ranges = xelp_dss_steering_table;469 }470 }471 472 /* Select non-terminated steering target for each type */473 for (int i = 0; i < NUM_STEERING_TYPES; i++)474 if (gt->steering[i].ranges && xe_steering_types[i].init)475 xe_steering_types[i].init(gt);476}477 478/**479 * xe_gt_mcr_set_implicit_defaults - Initialize steer control registers480 * @gt: GT structure481 *482 * Some register ranges don't need to have their steering control registers483 * changed on each access - it's sufficient to set them once on initialization.484 * This function sets those registers for each platform *485 */486void xe_gt_mcr_set_implicit_defaults(struct xe_gt *gt)487{488 struct xe_device *xe = gt_to_xe(gt);489 490 if (IS_SRIOV_VF(xe))491 return;492 493 if (xe->info.platform == XE_DG2) {494 u32 steer_val = REG_FIELD_PREP(MCR_SLICE_MASK, 0) |495 REG_FIELD_PREP(MCR_SUBSLICE_MASK, 2);496 497 xe_mmio_write32(gt, MCFG_MCR_SELECTOR, steer_val);498 xe_mmio_write32(gt, SF_MCR_SELECTOR, steer_val);499 /*500 * For GAM registers, all reads should be directed to instance 1501 * (unicast reads against other instances are not allowed),502 * and instance 1 is already the hardware's default steering503 * target, which we never change504 */505 }506}507 508/*509 * xe_gt_mcr_get_nonterminated_steering - find group/instance values that510 * will steer a register to a non-terminated instance511 * @gt: GT structure512 * @reg: register for which the steering is required513 * @group: return variable for group steering514 * @instance: return variable for instance steering515 *516 * This function returns a group/instance pair that is guaranteed to work for517 * read steering of the given register. Note that a value will be returned even518 * if the register is not replicated and therefore does not actually require519 * steering.520 *521 * Returns true if the caller should steer to the @group/@instance values522 * returned. Returns false if the caller need not perform any steering523 */524static bool xe_gt_mcr_get_nonterminated_steering(struct xe_gt *gt,525 struct xe_reg_mcr reg_mcr,526 u8 *group, u8 *instance)527{528 const struct xe_reg reg = to_xe_reg(reg_mcr);529 const struct xe_mmio_range *implicit_ranges;530 531 for (int type = 0; type < IMPLICIT_STEERING; type++) {532 if (!gt->steering[type].ranges)533 continue;534 535 for (int i = 0; gt->steering[type].ranges[i].end > 0; i++) {536 if (xe_mmio_in_range(gt, >->steering[type].ranges[i], reg)) {537 *group = gt->steering[type].group_target;538 *instance = gt->steering[type].instance_target;539 return true;540 }541 }542 }543 544 implicit_ranges = gt->steering[IMPLICIT_STEERING].ranges;545 if (implicit_ranges)546 for (int i = 0; implicit_ranges[i].end > 0; i++)547 if (xe_mmio_in_range(gt, &implicit_ranges[i], reg))548 return false;549 550 /*551 * Not found in a steering table and not a register with implicit552 * steering. Just steer to 0/0 as a guess and raise a warning.553 */554 drm_WARN(>_to_xe(gt)->drm, true,555 "Did not find MCR register %#x in any MCR steering table\n",556 reg.addr);557 *group = 0;558 *instance = 0;559 560 return true;561}562 563/*564 * Obtain exclusive access to MCR steering. On MTL and beyond we also need565 * to synchronize with external clients (e.g., firmware), so a semaphore566 * register will also need to be taken.567 */568static void mcr_lock(struct xe_gt *gt) __acquires(>->mcr_lock)569{570 struct xe_device *xe = gt_to_xe(gt);571 int ret = 0;572 573 spin_lock(>->mcr_lock);574 575 /*576 * Starting with MTL we also need to grab a semaphore register577 * to synchronize with external agents (e.g., firmware) that now578 * shares the same steering control register. The semaphore is obtained579 * when a read to the relevant register returns 1.580 */581 if (GRAPHICS_VERx100(xe) >= 1270)582 ret = xe_mmio_wait32(gt, STEER_SEMAPHORE, 0x1, 0x1, 10, NULL,583 true);584 585 drm_WARN_ON_ONCE(&xe->drm, ret == -ETIMEDOUT);586}587 588static void mcr_unlock(struct xe_gt *gt) __releases(>->mcr_lock)589{590 /* Release hardware semaphore - this is done by writing 1 to the register */591 if (GRAPHICS_VERx100(gt_to_xe(gt)) >= 1270)592 xe_mmio_write32(gt, STEER_SEMAPHORE, 0x1);593 594 spin_unlock(>->mcr_lock);595}596 597/*598 * Access a register with specific MCR steering599 *600 * Caller needs to make sure the relevant forcewake wells are up.601 */602static u32 rw_with_mcr_steering(struct xe_gt *gt, struct xe_reg_mcr reg_mcr,603 u8 rw_flag, int group, int instance, u32 value)604{605 const struct xe_reg reg = to_xe_reg(reg_mcr);606 struct xe_reg steer_reg;607 u32 steer_val, val = 0;608 609 lockdep_assert_held(>->mcr_lock);610 611 if (GRAPHICS_VERx100(gt_to_xe(gt)) >= 1270) {612 steer_reg = MTL_MCR_SELECTOR;613 steer_val = REG_FIELD_PREP(MTL_MCR_GROUPID, group) |614 REG_FIELD_PREP(MTL_MCR_INSTANCEID, instance);615 } else {616 steer_reg = MCR_SELECTOR;617 steer_val = REG_FIELD_PREP(MCR_SLICE_MASK, group) |618 REG_FIELD_PREP(MCR_SUBSLICE_MASK, instance);619 }620 621 /*622 * Always leave the hardware in multicast mode when doing reads and only623 * change it to unicast mode when doing writes of a specific instance.624 *625 * The setting of the multicast/unicast bit usually wouldn't matter for626 * read operations (which always return the value from a single register627 * instance regardless of how that bit is set), but some platforms may628 * have workarounds requiring us to remain in multicast mode for reads,629 * e.g. Wa_22013088509 on PVC. There's no real downside to this, so630 * we'll just go ahead and do so on all platforms; we'll only clear the631 * multicast bit from the mask when explicitly doing a write operation.632 *633 * No need to save old steering reg value.634 */635 if (rw_flag == MCR_OP_READ)636 steer_val |= MCR_MULTICAST;637 638 xe_mmio_write32(gt, steer_reg, steer_val);639 640 if (rw_flag == MCR_OP_READ)641 val = xe_mmio_read32(gt, reg);642 else643 xe_mmio_write32(gt, reg, value);644 645 /*646 * If we turned off the multicast bit (during a write) we're required647 * to turn it back on before finishing. The group and instance values648 * don't matter since they'll be re-programmed on the next MCR649 * operation.650 */651 if (rw_flag == MCR_OP_WRITE)652 xe_mmio_write32(gt, steer_reg, MCR_MULTICAST);653 654 return val;655}656 657/**658 * xe_gt_mcr_unicast_read_any - reads a non-terminated instance of an MCR register659 * @gt: GT structure660 * @reg_mcr: register to read661 *662 * Reads a GT MCR register. The read will be steered to a non-terminated663 * instance (i.e., one that isn't fused off or powered down by power gating).664 * This function assumes the caller is already holding any necessary forcewake665 * domains.666 *667 * Returns the value from a non-terminated instance of @reg.668 */669u32 xe_gt_mcr_unicast_read_any(struct xe_gt *gt, struct xe_reg_mcr reg_mcr)670{671 const struct xe_reg reg = to_xe_reg(reg_mcr);672 u8 group, instance;673 u32 val;674 bool steer;675 676 xe_gt_assert(gt, !IS_SRIOV_VF(gt_to_xe(gt)));677 678 steer = xe_gt_mcr_get_nonterminated_steering(gt, reg_mcr,679 &group, &instance);680 681 if (steer) {682 mcr_lock(gt);683 val = rw_with_mcr_steering(gt, reg_mcr, MCR_OP_READ,684 group, instance, 0);685 mcr_unlock(gt);686 } else {687 val = xe_mmio_read32(gt, reg);688 }689 690 return val;691}692 693/**694 * xe_gt_mcr_unicast_read - read a specific instance of an MCR register695 * @gt: GT structure696 * @reg_mcr: the MCR register to read697 * @group: the MCR group698 * @instance: the MCR instance699 *700 * Returns the value read from an MCR register after steering toward a specific701 * group/instance.702 */703u32 xe_gt_mcr_unicast_read(struct xe_gt *gt,704 struct xe_reg_mcr reg_mcr,705 int group, int instance)706{707 u32 val;708 709 xe_gt_assert(gt, !IS_SRIOV_VF(gt_to_xe(gt)));710 711 mcr_lock(gt);712 val = rw_with_mcr_steering(gt, reg_mcr, MCR_OP_READ, group, instance, 0);713 mcr_unlock(gt);714 715 return val;716}717 718/**719 * xe_gt_mcr_unicast_write - write a specific instance of an MCR register720 * @gt: GT structure721 * @reg_mcr: the MCR register to write722 * @value: value to write723 * @group: the MCR group724 * @instance: the MCR instance725 *726 * Write an MCR register in unicast mode after steering toward a specific727 * group/instance.728 */729void xe_gt_mcr_unicast_write(struct xe_gt *gt, struct xe_reg_mcr reg_mcr,730 u32 value, int group, int instance)731{732 xe_gt_assert(gt, !IS_SRIOV_VF(gt_to_xe(gt)));733 734 mcr_lock(gt);735 rw_with_mcr_steering(gt, reg_mcr, MCR_OP_WRITE, group, instance, value);736 mcr_unlock(gt);737}738 739/**740 * xe_gt_mcr_multicast_write - write a value to all instances of an MCR register741 * @gt: GT structure742 * @reg_mcr: the MCR register to write743 * @value: value to write744 *745 * Write an MCR register in multicast mode to update all instances.746 */747void xe_gt_mcr_multicast_write(struct xe_gt *gt, struct xe_reg_mcr reg_mcr,748 u32 value)749{750 struct xe_reg reg = to_xe_reg(reg_mcr);751 752 xe_gt_assert(gt, !IS_SRIOV_VF(gt_to_xe(gt)));753 754 /*755 * Synchronize with any unicast operations. Once we have exclusive756 * access, the MULTICAST bit should already be set, so there's no need757 * to touch the steering register.758 */759 mcr_lock(gt);760 xe_mmio_write32(gt, reg, value);761 mcr_unlock(gt);762}763 764void xe_gt_mcr_steering_dump(struct xe_gt *gt, struct drm_printer *p)765{766 for (int i = 0; i < NUM_STEERING_TYPES; i++) {767 if (gt->steering[i].ranges) {768 drm_printf(p, "%s steering: group=%#x, instance=%#x\n",769 xe_steering_types[i].name,770 gt->steering[i].group_target,771 gt->steering[i].instance_target);772 for (int j = 0; gt->steering[i].ranges[j].end; j++)773 drm_printf(p, "\t0x%06x - 0x%06x\n",774 gt->steering[i].ranges[j].start,775 gt->steering[i].ranges[j].end);776 }777 }778}779