801 lines · c
1// SPDX-License-Identifier: MIT2/*3 * Copyright © 2022 Intel Corporation4 */5 6#include "xe_mocs.h"7 8#include "regs/xe_gt_regs.h"9#include "xe_bo.h"10#include "xe_device.h"11#include "xe_exec_queue.h"12#include "xe_force_wake.h"13#include "xe_gt.h"14#include "xe_gt_mcr.h"15#include "xe_gt_printk.h"16#include "xe_mmio.h"17#include "xe_platform_types.h"18#include "xe_pm.h"19#include "xe_sriov.h"20#include "xe_step_types.h"21 22#if IS_ENABLED(CONFIG_DRM_XE_DEBUG)23#define mocs_dbg xe_gt_dbg24#else25__printf(2, 3)26static inline void mocs_dbg(const struct xe_gt *gt,27 const char *format, ...)28{ /* noop */ }29#endif30 31enum {32 HAS_GLOBAL_MOCS = BIT(0),33 HAS_LNCF_MOCS = BIT(1),34};35 36struct xe_mocs_entry {37 u32 control_value;38 u16 l3cc_value;39 u16 used;40};41 42struct xe_mocs_info;43 44struct xe_mocs_ops {45 void (*dump)(struct xe_mocs_info *mocs, unsigned int flags,46 struct xe_gt *gt, struct drm_printer *p);47};48 49struct xe_mocs_info {50 /*51 * Size of the spec's suggested MOCS programming table. The list of52 * table entries from the spec can potentially be smaller than the53 * number of hardware registers used to program the MOCS table; in such54 * cases the registers for the remaining indices will be programmed to55 * match unused_entries_index.56 */57 unsigned int table_size;58 /* Number of MOCS entries supported by the hardware */59 unsigned int num_mocs_regs;60 const struct xe_mocs_entry *table;61 const struct xe_mocs_ops *ops;62 u8 uc_index;63 u8 wb_index;64 u8 unused_entries_index;65};66 67/* Defines for the tables (GLOB_MOCS_0 - GLOB_MOCS_16) */68#define IG_PAT REG_BIT(8)69#define L3_CACHE_POLICY_MASK REG_GENMASK(5, 4)70#define L4_CACHE_POLICY_MASK REG_GENMASK(3, 2)71 72/* Helper defines */73#define XELP_NUM_MOCS_ENTRIES 64 /* 63-64 are reserved, but configured. */74#define PVC_NUM_MOCS_ENTRIES 375#define MTL_NUM_MOCS_ENTRIES 1676#define XE2_NUM_MOCS_ENTRIES 1677 78/* (e)LLC caching options */79/*80 * Note: LE_0_PAGETABLE works only up to Gen11; for newer gens it means81 * the same as LE_UC82 */83#define LE_0_PAGETABLE LE_CACHEABILITY(0)84#define LE_1_UC LE_CACHEABILITY(1)85#define LE_2_WT LE_CACHEABILITY(2)86#define LE_3_WB LE_CACHEABILITY(3)87 88/* Target cache */89#define LE_TC_0_PAGETABLE LE_TGT_CACHE(0)90#define LE_TC_1_LLC LE_TGT_CACHE(1)91#define LE_TC_2_LLC_ELLC LE_TGT_CACHE(2)92#define LE_TC_3_LLC_ELLC_ALT LE_TGT_CACHE(3)93 94/* L3 caching options */95#define L3_0_DIRECT L3_CACHEABILITY(0)96#define L3_1_UC L3_CACHEABILITY(1)97#define L3_2_RESERVED L3_CACHEABILITY(2)98#define L3_3_WB L3_CACHEABILITY(3)99 100/* L4 caching options */101#define L4_0_WB REG_FIELD_PREP(L4_CACHE_POLICY_MASK, 0)102#define L4_1_WT REG_FIELD_PREP(L4_CACHE_POLICY_MASK, 1)103#define L4_3_UC REG_FIELD_PREP(L4_CACHE_POLICY_MASK, 3)104 105#define XE2_L3_0_WB REG_FIELD_PREP(L3_CACHE_POLICY_MASK, 0)106/* XD: WB Transient Display */107#define XE2_L3_1_XD REG_FIELD_PREP(L3_CACHE_POLICY_MASK, 1)108#define XE2_L3_3_UC REG_FIELD_PREP(L3_CACHE_POLICY_MASK, 3)109 110#define XE2_L3_CLOS_MASK REG_GENMASK(7, 6)111 112#define MOCS_ENTRY(__idx, __control_value, __l3cc_value) \113 [__idx] = { \114 .control_value = __control_value, \115 .l3cc_value = __l3cc_value, \116 .used = 1, \117 }118 119/*120 * MOCS tables121 *122 * These are the MOCS tables that are programmed across all the rings.123 * The control value is programmed to all the rings that support the124 * MOCS registers. While the l3cc_values are only programmed to the125 * LNCFCMOCS0 - LNCFCMOCS32 registers.126 *127 * These tables are intended to be kept reasonably consistent across128 * HW platforms, and for ICL+, be identical across OSes. To achieve129 * that, the list of entries is published as part of bspec.130 *131 * Entries not part of the following tables are undefined as far as userspace is132 * concerned and shouldn't be relied upon. The last few entries are reserved by133 * the hardware. They should be initialized according to bspec and never used.134 *135 * NOTE1: These tables are part of bspec and defined as part of the hardware136 * interface. It is expected that, for specific hardware platform, existing137 * entries will remain constant and the table will only be updated by adding new138 * entries, filling unused positions.139 *140 * NOTE2: Reserved and unspecified MOCS indices have been set to L3 WB. These141 * reserved entries should never be used. They may be changed to low performant142 * variants with better coherency in the future if more entries are needed.143 */144 145static const struct xe_mocs_entry gen12_mocs_desc[] = {146 /* Base - L3 + LLC */147 MOCS_ENTRY(2,148 LE_3_WB | LE_TC_1_LLC | LE_LRUM(3),149 L3_3_WB),150 /* Base - Uncached */151 MOCS_ENTRY(3,152 LE_1_UC | LE_TC_1_LLC,153 L3_1_UC),154 /* Base - L3 */155 MOCS_ENTRY(4,156 LE_1_UC | LE_TC_1_LLC,157 L3_3_WB),158 /* Base - LLC */159 MOCS_ENTRY(5,160 LE_3_WB | LE_TC_1_LLC | LE_LRUM(3),161 L3_1_UC),162 /* Age 0 - LLC */163 MOCS_ENTRY(6,164 LE_3_WB | LE_TC_1_LLC | LE_LRUM(1),165 L3_1_UC),166 /* Age 0 - L3 + LLC */167 MOCS_ENTRY(7,168 LE_3_WB | LE_TC_1_LLC | LE_LRUM(1),169 L3_3_WB),170 /* Age: Don't Chg. - LLC */171 MOCS_ENTRY(8,172 LE_3_WB | LE_TC_1_LLC | LE_LRUM(2),173 L3_1_UC),174 /* Age: Don't Chg. - L3 + LLC */175 MOCS_ENTRY(9,176 LE_3_WB | LE_TC_1_LLC | LE_LRUM(2),177 L3_3_WB),178 /* No AOM - LLC */179 MOCS_ENTRY(10,180 LE_3_WB | LE_TC_1_LLC | LE_LRUM(3) | LE_AOM(1),181 L3_1_UC),182 /* No AOM - L3 + LLC */183 MOCS_ENTRY(11,184 LE_3_WB | LE_TC_1_LLC | LE_LRUM(3) | LE_AOM(1),185 L3_3_WB),186 /* No AOM; Age 0 - LLC */187 MOCS_ENTRY(12,188 LE_3_WB | LE_TC_1_LLC | LE_LRUM(1) | LE_AOM(1),189 L3_1_UC),190 /* No AOM; Age 0 - L3 + LLC */191 MOCS_ENTRY(13,192 LE_3_WB | LE_TC_1_LLC | LE_LRUM(1) | LE_AOM(1),193 L3_3_WB),194 /* No AOM; Age:DC - LLC */195 MOCS_ENTRY(14,196 LE_3_WB | LE_TC_1_LLC | LE_LRUM(2) | LE_AOM(1),197 L3_1_UC),198 /* No AOM; Age:DC - L3 + LLC */199 MOCS_ENTRY(15,200 LE_3_WB | LE_TC_1_LLC | LE_LRUM(2) | LE_AOM(1),201 L3_3_WB),202 /* Self-Snoop - L3 + LLC */203 MOCS_ENTRY(18,204 LE_3_WB | LE_TC_1_LLC | LE_LRUM(3) | LE_SSE(3),205 L3_3_WB),206 /* Skip Caching - L3 + LLC(12.5%) */207 MOCS_ENTRY(19,208 LE_3_WB | LE_TC_1_LLC | LE_LRUM(3) | LE_SCC(7),209 L3_3_WB),210 /* Skip Caching - L3 + LLC(25%) */211 MOCS_ENTRY(20,212 LE_3_WB | LE_TC_1_LLC | LE_LRUM(3) | LE_SCC(3),213 L3_3_WB),214 /* Skip Caching - L3 + LLC(50%) */215 MOCS_ENTRY(21,216 LE_3_WB | LE_TC_1_LLC | LE_LRUM(3) | LE_SCC(1),217 L3_3_WB),218 /* Skip Caching - L3 + LLC(75%) */219 MOCS_ENTRY(22,220 LE_3_WB | LE_TC_1_LLC | LE_LRUM(3) | LE_RSC(1) | LE_SCC(3),221 L3_3_WB),222 /* Skip Caching - L3 + LLC(87.5%) */223 MOCS_ENTRY(23,224 LE_3_WB | LE_TC_1_LLC | LE_LRUM(3) | LE_RSC(1) | LE_SCC(7),225 L3_3_WB),226 /* Implicitly enable L1 - HDC:L1 + L3 + LLC */227 MOCS_ENTRY(48,228 LE_3_WB | LE_TC_1_LLC | LE_LRUM(3),229 L3_3_WB),230 /* Implicitly enable L1 - HDC:L1 + L3 */231 MOCS_ENTRY(49,232 LE_1_UC | LE_TC_1_LLC,233 L3_3_WB),234 /* Implicitly enable L1 - HDC:L1 + LLC */235 MOCS_ENTRY(50,236 LE_3_WB | LE_TC_1_LLC | LE_LRUM(3),237 L3_1_UC),238 /* Implicitly enable L1 - HDC:L1 */239 MOCS_ENTRY(51,240 LE_1_UC | LE_TC_1_LLC,241 L3_1_UC),242 /* HW Special Case (CCS) */243 MOCS_ENTRY(60,244 LE_3_WB | LE_TC_1_LLC | LE_LRUM(3),245 L3_1_UC),246 /* HW Special Case (Displayable) */247 MOCS_ENTRY(61,248 LE_1_UC | LE_TC_1_LLC,249 L3_3_WB),250 /* HW Reserved - SW program but never use */251 MOCS_ENTRY(62,252 LE_3_WB | LE_TC_1_LLC | LE_LRUM(3),253 L3_1_UC),254 /* HW Reserved - SW program but never use */255 MOCS_ENTRY(63,256 LE_3_WB | LE_TC_1_LLC | LE_LRUM(3),257 L3_1_UC)258};259 260static bool regs_are_mcr(struct xe_gt *gt)261{262 struct xe_device *xe = gt_to_xe(gt);263 264 if (xe_gt_is_media_type(gt))265 return MEDIA_VER(xe) >= 20;266 else267 return GRAPHICS_VERx100(xe) >= 1250;268}269 270static void xelp_lncf_dump(struct xe_mocs_info *info, struct xe_gt *gt, struct drm_printer *p)271{272 unsigned int i, j;273 u32 reg_val;274 275 drm_printf(p, "LNCFCMOCS[idx] = [ESC, SCC, L3CC] (value)\n\n");276 277 for (i = 0, j = 0; i < (info->num_mocs_regs + 1) / 2; i++, j++) {278 if (regs_are_mcr(gt))279 reg_val = xe_gt_mcr_unicast_read_any(gt, XEHP_LNCFCMOCS(i));280 else281 reg_val = xe_mmio_read32(gt, XELP_LNCFCMOCS(i));282 283 drm_printf(p, "LNCFCMOCS[%2d] = [%u, %u, %u] (%#8x)\n",284 j++,285 !!(reg_val & L3_ESC_MASK),286 REG_FIELD_GET(L3_SCC_MASK, reg_val),287 REG_FIELD_GET(L3_CACHEABILITY_MASK, reg_val),288 reg_val);289 290 drm_printf(p, "LNCFCMOCS[%2d] = [%u, %u, %u] (%#8x)\n",291 j,292 !!(reg_val & L3_UPPER_IDX_ESC_MASK),293 REG_FIELD_GET(L3_UPPER_IDX_SCC_MASK, reg_val),294 REG_FIELD_GET(L3_UPPER_IDX_CACHEABILITY_MASK, reg_val),295 reg_val);296 }297}298 299static void xelp_mocs_dump(struct xe_mocs_info *info, unsigned int flags,300 struct xe_gt *gt, struct drm_printer *p)301{302 unsigned int i;303 u32 reg_val;304 305 if (flags & HAS_GLOBAL_MOCS) {306 drm_printf(p, "Global mocs table configuration:\n");307 drm_printf(p, "GLOB_MOCS[idx] = [LeCC, TC, LRUM, AOM, RSC, SCC, PFM, SCF, CoS, SSE] (value)\n\n");308 309 for (i = 0; i < info->num_mocs_regs; i++) {310 if (regs_are_mcr(gt))311 reg_val = xe_gt_mcr_unicast_read_any(gt, XEHP_GLOBAL_MOCS(i));312 else313 reg_val = xe_mmio_read32(gt, XELP_GLOBAL_MOCS(i));314 315 drm_printf(p, "GLOB_MOCS[%2d] = [%u, %u, %u, %u, %u, %u, %u, %u, %u, %u ] (%#8x)\n",316 i,317 REG_FIELD_GET(LE_CACHEABILITY_MASK, reg_val),318 REG_FIELD_GET(LE_TGT_CACHE_MASK, reg_val),319 REG_FIELD_GET(LE_LRUM_MASK, reg_val),320 !!(reg_val & LE_AOM_MASK),321 !!(reg_val & LE_RSC_MASK),322 REG_FIELD_GET(LE_SCC_MASK, reg_val),323 REG_FIELD_GET(LE_PFM_MASK, reg_val),324 !!(reg_val & LE_SCF_MASK),325 REG_FIELD_GET(LE_COS_MASK, reg_val),326 REG_FIELD_GET(LE_SSE_MASK, reg_val),327 reg_val);328 }329 }330 331 xelp_lncf_dump(info, gt, p);332}333 334static const struct xe_mocs_ops xelp_mocs_ops = {335 .dump = xelp_mocs_dump,336};337 338static const struct xe_mocs_entry dg1_mocs_desc[] = {339 /* UC */340 MOCS_ENTRY(1, 0, L3_1_UC),341 /* WB - L3 */342 MOCS_ENTRY(5, 0, L3_3_WB),343 /* WB - L3 50% */344 MOCS_ENTRY(6, 0, L3_ESC(1) | L3_SCC(1) | L3_3_WB),345 /* WB - L3 25% */346 MOCS_ENTRY(7, 0, L3_ESC(1) | L3_SCC(3) | L3_3_WB),347 /* WB - L3 12.5% */348 MOCS_ENTRY(8, 0, L3_ESC(1) | L3_SCC(7) | L3_3_WB),349 350 /* HDC:L1 + L3 */351 MOCS_ENTRY(48, 0, L3_3_WB),352 /* HDC:L1 */353 MOCS_ENTRY(49, 0, L3_1_UC),354 355 /* HW Reserved */356 MOCS_ENTRY(60, 0, L3_1_UC),357 MOCS_ENTRY(61, 0, L3_1_UC),358 MOCS_ENTRY(62, 0, L3_1_UC),359 MOCS_ENTRY(63, 0, L3_1_UC),360};361 362static const struct xe_mocs_entry dg2_mocs_desc[] = {363 /* UC - Coherent; GO:L3 */364 MOCS_ENTRY(0, 0, L3_1_UC | L3_LKUP(1)),365 /* UC - Coherent; GO:Memory */366 MOCS_ENTRY(1, 0, L3_1_UC | L3_GLBGO(1) | L3_LKUP(1)),367 /* UC - Non-Coherent; GO:Memory */368 MOCS_ENTRY(2, 0, L3_1_UC | L3_GLBGO(1)),369 370 /* WB - LC */371 MOCS_ENTRY(3, 0, L3_3_WB | L3_LKUP(1)),372};373 374static void xehp_lncf_dump(struct xe_mocs_info *info, unsigned int flags,375 struct xe_gt *gt, struct drm_printer *p)376{377 unsigned int i, j;378 u32 reg_val;379 380 drm_printf(p, "LNCFCMOCS[idx] = [UCL3LOOKUP, GLBGO, L3CC] (value)\n\n");381 382 for (i = 0, j = 0; i < (info->num_mocs_regs + 1) / 2; i++, j++) {383 if (regs_are_mcr(gt))384 reg_val = xe_gt_mcr_unicast_read_any(gt, XEHP_LNCFCMOCS(i));385 else386 reg_val = xe_mmio_read32(gt, XELP_LNCFCMOCS(i));387 388 drm_printf(p, "LNCFCMOCS[%2d] = [%u, %u, %u] (%#8x)\n",389 j++,390 !!(reg_val & L3_LKUP_MASK),391 !!(reg_val & L3_GLBGO_MASK),392 REG_FIELD_GET(L3_CACHEABILITY_MASK, reg_val),393 reg_val);394 395 drm_printf(p, "LNCFCMOCS[%2d] = [%u, %u, %u] (%#8x)\n",396 j,397 !!(reg_val & L3_UPPER_LKUP_MASK),398 !!(reg_val & L3_UPPER_GLBGO_MASK),399 REG_FIELD_GET(L3_UPPER_IDX_CACHEABILITY_MASK, reg_val),400 reg_val);401 }402}403 404static const struct xe_mocs_ops xehp_mocs_ops = {405 .dump = xehp_lncf_dump,406};407 408static const struct xe_mocs_entry pvc_mocs_desc[] = {409 /* Error */410 MOCS_ENTRY(0, 0, L3_3_WB),411 412 /* UC */413 MOCS_ENTRY(1, 0, L3_1_UC),414 415 /* WB */416 MOCS_ENTRY(2, 0, L3_3_WB),417};418 419static void pvc_mocs_dump(struct xe_mocs_info *info, unsigned int flags, struct xe_gt *gt,420 struct drm_printer *p)421{422 unsigned int i, j;423 u32 reg_val;424 425 drm_printf(p, "LNCFCMOCS[idx] = [ L3CC ] (value)\n\n");426 427 for (i = 0, j = 0; i < (info->num_mocs_regs + 1) / 2; i++, j++) {428 if (regs_are_mcr(gt))429 reg_val = xe_gt_mcr_unicast_read_any(gt, XEHP_LNCFCMOCS(i));430 else431 reg_val = xe_mmio_read32(gt, XELP_LNCFCMOCS(i));432 433 drm_printf(p, "LNCFCMOCS[%2d] = [ %u ] (%#8x)\n",434 j++,435 REG_FIELD_GET(L3_CACHEABILITY_MASK, reg_val),436 reg_val);437 438 drm_printf(p, "LNCFCMOCS[%2d] = [ %u ] (%#8x)\n",439 j,440 REG_FIELD_GET(L3_UPPER_IDX_CACHEABILITY_MASK, reg_val),441 reg_val);442 }443}444 445static const struct xe_mocs_ops pvc_mocs_ops = {446 .dump = pvc_mocs_dump,447};448 449static const struct xe_mocs_entry mtl_mocs_desc[] = {450 /* Error - Reserved for Non-Use */451 MOCS_ENTRY(0,452 0,453 L3_LKUP(1) | L3_3_WB),454 /* Cached - L3 + L4 */455 MOCS_ENTRY(1,456 IG_PAT,457 L3_LKUP(1) | L3_3_WB),458 /* L4 - GO:L3 */459 MOCS_ENTRY(2,460 IG_PAT,461 L3_LKUP(1) | L3_1_UC),462 /* Uncached - GO:L3 */463 MOCS_ENTRY(3,464 IG_PAT | L4_3_UC,465 L3_LKUP(1) | L3_1_UC),466 /* L4 - GO:Mem */467 MOCS_ENTRY(4,468 IG_PAT,469 L3_LKUP(1) | L3_GLBGO(1) | L3_1_UC),470 /* Uncached - GO:Mem */471 MOCS_ENTRY(5,472 IG_PAT | L4_3_UC,473 L3_LKUP(1) | L3_GLBGO(1) | L3_1_UC),474 /* L4 - L3:NoLKUP; GO:L3 */475 MOCS_ENTRY(6,476 IG_PAT,477 L3_1_UC),478 /* Uncached - L3:NoLKUP; GO:L3 */479 MOCS_ENTRY(7,480 IG_PAT | L4_3_UC,481 L3_1_UC),482 /* L4 - L3:NoLKUP; GO:Mem */483 MOCS_ENTRY(8,484 IG_PAT,485 L3_GLBGO(1) | L3_1_UC),486 /* Uncached - L3:NoLKUP; GO:Mem */487 MOCS_ENTRY(9,488 IG_PAT | L4_3_UC,489 L3_GLBGO(1) | L3_1_UC),490 /* Display - L3; L4:WT */491 MOCS_ENTRY(14,492 IG_PAT | L4_1_WT,493 L3_LKUP(1) | L3_3_WB),494 /* CCS - Non-Displayable */495 MOCS_ENTRY(15,496 IG_PAT,497 L3_GLBGO(1) | L3_1_UC),498};499 500static void mtl_mocs_dump(struct xe_mocs_info *info, unsigned int flags,501 struct xe_gt *gt, struct drm_printer *p)502{503 unsigned int i;504 u32 reg_val;505 506 drm_printf(p, "Global mocs table configuration:\n");507 drm_printf(p, "GLOB_MOCS[idx] = [IG_PAT, L4_CACHE_POLICY] (value)\n\n");508 509 for (i = 0; i < info->num_mocs_regs; i++) {510 if (regs_are_mcr(gt))511 reg_val = xe_gt_mcr_unicast_read_any(gt, XEHP_GLOBAL_MOCS(i));512 else513 reg_val = xe_mmio_read32(gt, XELP_GLOBAL_MOCS(i));514 515 drm_printf(p, "GLOB_MOCS[%2d] = [%u, %u] (%#8x)\n",516 i,517 !!(reg_val & IG_PAT),518 REG_FIELD_GET(L4_CACHE_POLICY_MASK, reg_val),519 reg_val);520 }521 522 /* MTL lncf mocs table pattern is similar to that of xehp */523 xehp_lncf_dump(info, flags, gt, p);524}525 526static const struct xe_mocs_ops mtl_mocs_ops = {527 .dump = mtl_mocs_dump,528};529 530static const struct xe_mocs_entry xe2_mocs_table[] = {531 /* Defer to PAT */532 MOCS_ENTRY(0, XE2_L3_0_WB | L4_3_UC, 0),533 /* Cached L3, Uncached L4 */534 MOCS_ENTRY(1, IG_PAT | XE2_L3_0_WB | L4_3_UC, 0),535 /* Uncached L3, Cached L4 */536 MOCS_ENTRY(2, IG_PAT | XE2_L3_3_UC | L4_0_WB, 0),537 /* Uncached L3 + L4 */538 MOCS_ENTRY(3, IG_PAT | XE2_L3_3_UC | L4_3_UC, 0),539 /* Cached L3 + L4 */540 MOCS_ENTRY(4, IG_PAT | XE2_L3_0_WB | L4_0_WB, 0),541};542 543static void xe2_mocs_dump(struct xe_mocs_info *info, unsigned int flags,544 struct xe_gt *gt, struct drm_printer *p)545{546 unsigned int i;547 u32 reg_val;548 549 drm_printf(p, "Global mocs table configuration:\n");550 drm_printf(p, "GLOB_MOCS[idx] = [IG_PAT, L3_CLOS, L3_CACHE_POLICY, L4_CACHE_POLICY] (value)\n\n");551 552 for (i = 0; i < info->num_mocs_regs; i++) {553 if (regs_are_mcr(gt))554 reg_val = xe_gt_mcr_unicast_read_any(gt, XEHP_GLOBAL_MOCS(i));555 else556 reg_val = xe_mmio_read32(gt, XELP_GLOBAL_MOCS(i));557 558 drm_printf(p, "GLOB_MOCS[%2d] = [%u, %u, %u] (%#8x)\n",559 i,560 !!(reg_val & IG_PAT),561 REG_FIELD_GET(XE2_L3_CLOS_MASK, reg_val),562 REG_FIELD_GET(L4_CACHE_POLICY_MASK, reg_val),563 reg_val);564 }565}566 567static const struct xe_mocs_ops xe2_mocs_ops = {568 .dump = xe2_mocs_dump,569};570 571static unsigned int get_mocs_settings(struct xe_device *xe,572 struct xe_mocs_info *info)573{574 unsigned int flags = 0;575 576 memset(info, 0, sizeof(struct xe_mocs_info));577 578 switch (xe->info.platform) {579 case XE_LUNARLAKE:580 case XE_BATTLEMAGE:581 info->ops = &xe2_mocs_ops;582 info->table_size = ARRAY_SIZE(xe2_mocs_table);583 info->table = xe2_mocs_table;584 info->num_mocs_regs = XE2_NUM_MOCS_ENTRIES;585 info->uc_index = 3;586 info->wb_index = 4;587 info->unused_entries_index = 4;588 break;589 case XE_PVC:590 info->ops = &pvc_mocs_ops;591 info->table_size = ARRAY_SIZE(pvc_mocs_desc);592 info->table = pvc_mocs_desc;593 info->num_mocs_regs = PVC_NUM_MOCS_ENTRIES;594 info->uc_index = 1;595 info->wb_index = 2;596 info->unused_entries_index = 2;597 break;598 case XE_METEORLAKE:599 info->ops = &mtl_mocs_ops;600 info->table_size = ARRAY_SIZE(mtl_mocs_desc);601 info->table = mtl_mocs_desc;602 info->num_mocs_regs = MTL_NUM_MOCS_ENTRIES;603 info->uc_index = 9;604 info->unused_entries_index = 1;605 break;606 case XE_DG2:607 info->ops = &xehp_mocs_ops;608 info->table_size = ARRAY_SIZE(dg2_mocs_desc);609 info->table = dg2_mocs_desc;610 info->uc_index = 1;611 /*612 * Last entry is RO on hardware, don't bother with what was613 * written when checking later614 */615 info->num_mocs_regs = XELP_NUM_MOCS_ENTRIES - 1;616 info->unused_entries_index = 3;617 break;618 case XE_DG1:619 info->ops = &xelp_mocs_ops;620 info->table_size = ARRAY_SIZE(dg1_mocs_desc);621 info->table = dg1_mocs_desc;622 info->uc_index = 1;623 info->num_mocs_regs = XELP_NUM_MOCS_ENTRIES;624 info->unused_entries_index = 5;625 break;626 case XE_TIGERLAKE:627 case XE_ROCKETLAKE:628 case XE_ALDERLAKE_S:629 case XE_ALDERLAKE_P:630 case XE_ALDERLAKE_N:631 info->ops = &xelp_mocs_ops;632 info->table_size = ARRAY_SIZE(gen12_mocs_desc);633 info->table = gen12_mocs_desc;634 info->num_mocs_regs = XELP_NUM_MOCS_ENTRIES;635 info->uc_index = 3;636 info->unused_entries_index = 2;637 break;638 default:639 drm_err(&xe->drm, "Platform that should have a MOCS table does not.\n");640 return 0;641 }642 643 /*644 * Index 0 is a reserved/unused table entry on most platforms, but645 * even on those where it does represent a legitimate MOCS entry, it646 * never represents the "most cached, least coherent" behavior we want647 * to populate undefined table rows with. So if unused_entries_index648 * is still 0 at this point, we'll assume that it was omitted by649 * mistake in the switch statement above.650 */651 xe_assert(xe, info->unused_entries_index != 0);652 653 xe_assert(xe, info->ops && info->ops->dump);654 xe_assert(xe, info->table_size <= info->num_mocs_regs);655 656 if (!IS_DGFX(xe) || GRAPHICS_VER(xe) >= 20)657 flags |= HAS_GLOBAL_MOCS;658 if (GRAPHICS_VER(xe) < 20)659 flags |= HAS_LNCF_MOCS;660 661 return flags;662}663 664/*665 * Get control_value from MOCS entry. If the table entry is not defined, the666 * settings from unused_entries_index will be returned.667 */668static u32 get_entry_control(const struct xe_mocs_info *info,669 unsigned int index)670{671 if (index < info->table_size && info->table[index].used)672 return info->table[index].control_value;673 return info->table[info->unused_entries_index].control_value;674}675 676static void __init_mocs_table(struct xe_gt *gt,677 const struct xe_mocs_info *info)678{679 unsigned int i;680 u32 mocs;681 682 mocs_dbg(gt, "mocs entries: %d\n", info->num_mocs_regs);683 684 for (i = 0; i < info->num_mocs_regs; i++) {685 mocs = get_entry_control(info, i);686 687 mocs_dbg(gt, "GLOB_MOCS[%d] 0x%x 0x%x\n", i,688 XELP_GLOBAL_MOCS(i).addr, mocs);689 690 if (regs_are_mcr(gt))691 xe_gt_mcr_multicast_write(gt, XEHP_GLOBAL_MOCS(i), mocs);692 else693 xe_mmio_write32(gt, XELP_GLOBAL_MOCS(i), mocs);694 }695}696 697/*698 * Get l3cc_value from MOCS entry taking into account when it's not used699 * then if unused_entries_index is not zero then its value will be returned700 * otherwise I915_MOCS_PTE's value is returned in this case.701 */702static u16 get_entry_l3cc(const struct xe_mocs_info *info,703 unsigned int index)704{705 if (index < info->table_size && info->table[index].used)706 return info->table[index].l3cc_value;707 return info->table[info->unused_entries_index].l3cc_value;708}709 710static u32 l3cc_combine(u16 low, u16 high)711{712 return low | (u32)high << 16;713}714 715static void init_l3cc_table(struct xe_gt *gt,716 const struct xe_mocs_info *info)717{718 unsigned int i;719 u32 l3cc;720 721 mocs_dbg(gt, "l3cc entries: %d\n", info->num_mocs_regs);722 723 for (i = 0; i < (info->num_mocs_regs + 1) / 2; i++) {724 l3cc = l3cc_combine(get_entry_l3cc(info, 2 * i),725 get_entry_l3cc(info, 2 * i + 1));726 727 mocs_dbg(gt, "LNCFCMOCS[%d] 0x%x 0x%x\n", i,728 XELP_LNCFCMOCS(i).addr, l3cc);729 730 if (regs_are_mcr(gt))731 xe_gt_mcr_multicast_write(gt, XEHP_LNCFCMOCS(i), l3cc);732 else733 xe_mmio_write32(gt, XELP_LNCFCMOCS(i), l3cc);734 }735}736 737void xe_mocs_init_early(struct xe_gt *gt)738{739 struct xe_mocs_info table;740 741 get_mocs_settings(gt_to_xe(gt), &table);742 gt->mocs.uc_index = table.uc_index;743 gt->mocs.wb_index = table.wb_index;744}745 746void xe_mocs_init(struct xe_gt *gt)747{748 struct xe_mocs_info table;749 unsigned int flags;750 751 if (IS_SRIOV_VF(gt_to_xe(gt)))752 return;753 754 /*755 * MOCS settings are split between "GLOB_MOCS" and/or "LNCFCMOCS"756 * registers depending on platform.757 *758 * These registers should be programmed before GuC initialization759 * since their values will affect some of the memory transactions760 * performed by the GuC.761 */762 flags = get_mocs_settings(gt_to_xe(gt), &table);763 mocs_dbg(gt, "flag:0x%x\n", flags);764 765 if (IS_SRIOV_VF(gt_to_xe(gt)))766 return;767 768 if (flags & HAS_GLOBAL_MOCS)769 __init_mocs_table(gt, &table);770 if (flags & HAS_LNCF_MOCS)771 init_l3cc_table(gt, &table);772}773 774void xe_mocs_dump(struct xe_gt *gt, struct drm_printer *p)775{776 struct xe_mocs_info table;777 unsigned int flags;778 u32 ret;779 struct xe_device *xe = gt_to_xe(gt);780 781 flags = get_mocs_settings(xe, &table);782 783 xe_pm_runtime_get_noresume(xe);784 ret = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);785 786 if (ret)787 goto err_fw;788 789 table.ops->dump(&table, flags, gt, p);790 791 xe_force_wake_put(gt_to_fw(gt), XE_FW_GT);792 793err_fw:794 xe_assert(xe, !ret);795 xe_pm_runtime_put(xe);796}797 798#if IS_ENABLED(CONFIG_DRM_XE_KUNIT_TEST)799#include "tests/xe_mocs.c"800#endif801