476 lines · c
1// SPDX-License-Identifier: MIT2/*3 * Copyright © 2023 Intel Corporation4 */5 6#include "xe_pat.h"7 8#include <uapi/drm/xe_drm.h>9 10#include <generated/xe_wa_oob.h>11 12#include "regs/xe_reg_defs.h"13#include "xe_assert.h"14#include "xe_device.h"15#include "xe_force_wake.h"16#include "xe_gt.h"17#include "xe_gt_mcr.h"18#include "xe_mmio.h"19#include "xe_sriov.h"20#include "xe_wa.h"21 22#define _PAT_ATS 0x47fc23#define _PAT_INDEX(index) _PICK_EVEN_2RANGES(index, 8, \24 0x4800, 0x4804, \25 0x4848, 0x484c)26#define _PAT_PTA 0x482027 28#define XE2_NO_PROMOTE REG_BIT(10)29#define XE2_COMP_EN REG_BIT(9)30#define XE2_L3_CLOS REG_GENMASK(7, 6)31#define XE2_L3_POLICY REG_GENMASK(5, 4)32#define XE2_L4_POLICY REG_GENMASK(3, 2)33#define XE2_COH_MODE REG_GENMASK(1, 0)34 35#define XELPG_L4_POLICY_MASK REG_GENMASK(3, 2)36#define XELPG_PAT_3_UC REG_FIELD_PREP(XELPG_L4_POLICY_MASK, 3)37#define XELPG_PAT_1_WT REG_FIELD_PREP(XELPG_L4_POLICY_MASK, 1)38#define XELPG_PAT_0_WB REG_FIELD_PREP(XELPG_L4_POLICY_MASK, 0)39#define XELPG_INDEX_COH_MODE_MASK REG_GENMASK(1, 0)40#define XELPG_3_COH_2W REG_FIELD_PREP(XELPG_INDEX_COH_MODE_MASK, 3)41#define XELPG_2_COH_1W REG_FIELD_PREP(XELPG_INDEX_COH_MODE_MASK, 2)42#define XELPG_0_COH_NON REG_FIELD_PREP(XELPG_INDEX_COH_MODE_MASK, 0)43 44#define XEHPC_CLOS_LEVEL_MASK REG_GENMASK(3, 2)45#define XEHPC_PAT_CLOS(x) REG_FIELD_PREP(XEHPC_CLOS_LEVEL_MASK, x)46 47#define XELP_MEM_TYPE_MASK REG_GENMASK(1, 0)48#define XELP_PAT_WB REG_FIELD_PREP(XELP_MEM_TYPE_MASK, 3)49#define XELP_PAT_WT REG_FIELD_PREP(XELP_MEM_TYPE_MASK, 2)50#define XELP_PAT_WC REG_FIELD_PREP(XELP_MEM_TYPE_MASK, 1)51#define XELP_PAT_UC REG_FIELD_PREP(XELP_MEM_TYPE_MASK, 0)52 53static const char *XELP_MEM_TYPE_STR_MAP[] = { "UC", "WC", "WT", "WB" };54 55struct xe_pat_ops {56 void (*program_graphics)(struct xe_gt *gt, const struct xe_pat_table_entry table[],57 int n_entries);58 void (*program_media)(struct xe_gt *gt, const struct xe_pat_table_entry table[],59 int n_entries);60 void (*dump)(struct xe_gt *gt, struct drm_printer *p);61};62 63static const struct xe_pat_table_entry xelp_pat_table[] = {64 [0] = { XELP_PAT_WB, XE_COH_AT_LEAST_1WAY },65 [1] = { XELP_PAT_WC, XE_COH_NONE },66 [2] = { XELP_PAT_WT, XE_COH_NONE },67 [3] = { XELP_PAT_UC, XE_COH_NONE },68};69 70static const struct xe_pat_table_entry xehpc_pat_table[] = {71 [0] = { XELP_PAT_UC, XE_COH_NONE },72 [1] = { XELP_PAT_WC, XE_COH_NONE },73 [2] = { XELP_PAT_WT, XE_COH_NONE },74 [3] = { XELP_PAT_WB, XE_COH_AT_LEAST_1WAY },75 [4] = { XEHPC_PAT_CLOS(1) | XELP_PAT_WT, XE_COH_NONE },76 [5] = { XEHPC_PAT_CLOS(1) | XELP_PAT_WB, XE_COH_AT_LEAST_1WAY },77 [6] = { XEHPC_PAT_CLOS(2) | XELP_PAT_WT, XE_COH_NONE },78 [7] = { XEHPC_PAT_CLOS(2) | XELP_PAT_WB, XE_COH_AT_LEAST_1WAY },79};80 81static const struct xe_pat_table_entry xelpg_pat_table[] = {82 [0] = { XELPG_PAT_0_WB, XE_COH_NONE },83 [1] = { XELPG_PAT_1_WT, XE_COH_NONE },84 [2] = { XELPG_PAT_3_UC, XE_COH_NONE },85 [3] = { XELPG_PAT_0_WB | XELPG_2_COH_1W, XE_COH_AT_LEAST_1WAY },86 [4] = { XELPG_PAT_0_WB | XELPG_3_COH_2W, XE_COH_AT_LEAST_1WAY },87};88 89/*90 * The Xe2 table is getting large/complicated so it's easier to review if91 * provided in a form that exactly matches the bspec's formatting. The meaning92 * of the fields here are:93 * - no_promote: 0=promotable, 1=no promote94 * - comp_en: 0=disable, 1=enable95 * - l3clos: L3 class of service (0-3)96 * - l3_policy: 0=WB, 1=XD ("WB - Transient Display"), 3=UC97 * - l4_policy: 0=WB, 1=WT, 3=UC98 * - coh_mode: 0=no snoop, 2=1-way coherent, 3=2-way coherent99 *100 * Reserved entries should be programmed with the maximum caching, minimum101 * coherency (which matches an all-0's encoding), so we can just omit them102 * in the table.103 */104#define XE2_PAT(no_promote, comp_en, l3clos, l3_policy, l4_policy, __coh_mode) \105 { \106 .value = (no_promote ? XE2_NO_PROMOTE : 0) | \107 (comp_en ? XE2_COMP_EN : 0) | \108 REG_FIELD_PREP(XE2_L3_CLOS, l3clos) | \109 REG_FIELD_PREP(XE2_L3_POLICY, l3_policy) | \110 REG_FIELD_PREP(XE2_L4_POLICY, l4_policy) | \111 REG_FIELD_PREP(XE2_COH_MODE, __coh_mode), \112 .coh_mode = __coh_mode ? XE_COH_AT_LEAST_1WAY : XE_COH_NONE \113 }114 115static const struct xe_pat_table_entry xe2_pat_table[] = {116 [ 0] = XE2_PAT( 0, 0, 0, 0, 3, 0 ),117 [ 1] = XE2_PAT( 0, 0, 0, 0, 3, 2 ),118 [ 2] = XE2_PAT( 0, 0, 0, 0, 3, 3 ),119 [ 3] = XE2_PAT( 0, 0, 0, 3, 3, 0 ),120 [ 4] = XE2_PAT( 0, 0, 0, 3, 0, 2 ),121 [ 5] = XE2_PAT( 0, 0, 0, 3, 3, 2 ),122 [ 6] = XE2_PAT( 1, 0, 0, 1, 3, 0 ),123 [ 7] = XE2_PAT( 0, 0, 0, 3, 0, 3 ),124 [ 8] = XE2_PAT( 0, 0, 0, 3, 0, 0 ),125 [ 9] = XE2_PAT( 0, 1, 0, 0, 3, 0 ),126 [10] = XE2_PAT( 0, 1, 0, 3, 0, 0 ),127 [11] = XE2_PAT( 1, 1, 0, 1, 3, 0 ),128 [12] = XE2_PAT( 0, 1, 0, 3, 3, 0 ),129 [13] = XE2_PAT( 0, 0, 0, 0, 0, 0 ),130 [14] = XE2_PAT( 0, 1, 0, 0, 0, 0 ),131 [15] = XE2_PAT( 1, 1, 0, 1, 1, 0 ),132 /* 16..19 are reserved; leave set to all 0's */133 [20] = XE2_PAT( 0, 0, 1, 0, 3, 0 ),134 [21] = XE2_PAT( 0, 1, 1, 0, 3, 0 ),135 [22] = XE2_PAT( 0, 0, 1, 0, 3, 2 ),136 [23] = XE2_PAT( 0, 0, 1, 0, 3, 3 ),137 [24] = XE2_PAT( 0, 0, 2, 0, 3, 0 ),138 [25] = XE2_PAT( 0, 1, 2, 0, 3, 0 ),139 [26] = XE2_PAT( 0, 0, 2, 0, 3, 2 ),140 [27] = XE2_PAT( 0, 0, 2, 0, 3, 3 ),141 [28] = XE2_PAT( 0, 0, 3, 0, 3, 0 ),142 [29] = XE2_PAT( 0, 1, 3, 0, 3, 0 ),143 [30] = XE2_PAT( 0, 0, 3, 0, 3, 2 ),144 [31] = XE2_PAT( 0, 0, 3, 0, 3, 3 ),145};146 147/* Special PAT values programmed outside the main table */148static const struct xe_pat_table_entry xe2_pat_ats = XE2_PAT( 0, 0, 0, 0, 3, 3 );149static const struct xe_pat_table_entry xe2_pat_pta = XE2_PAT( 0, 0, 0, 0, 3, 0 );150 151u16 xe_pat_index_get_coh_mode(struct xe_device *xe, u16 pat_index)152{153 WARN_ON(pat_index >= xe->pat.n_entries);154 return xe->pat.table[pat_index].coh_mode;155}156 157static void program_pat(struct xe_gt *gt, const struct xe_pat_table_entry table[],158 int n_entries)159{160 for (int i = 0; i < n_entries; i++) {161 struct xe_reg reg = XE_REG(_PAT_INDEX(i));162 163 xe_mmio_write32(gt, reg, table[i].value);164 }165}166 167static void program_pat_mcr(struct xe_gt *gt, const struct xe_pat_table_entry table[],168 int n_entries)169{170 for (int i = 0; i < n_entries; i++) {171 struct xe_reg_mcr reg_mcr = XE_REG_MCR(_PAT_INDEX(i));172 173 xe_gt_mcr_multicast_write(gt, reg_mcr, table[i].value);174 }175}176 177static void xelp_dump(struct xe_gt *gt, struct drm_printer *p)178{179 struct xe_device *xe = gt_to_xe(gt);180 int i, err;181 182 err = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);183 if (err)184 goto err_fw;185 186 drm_printf(p, "PAT table:\n");187 188 for (i = 0; i < xe->pat.n_entries; i++) {189 u32 pat = xe_mmio_read32(gt, XE_REG(_PAT_INDEX(i)));190 u8 mem_type = REG_FIELD_GET(XELP_MEM_TYPE_MASK, pat);191 192 drm_printf(p, "PAT[%2d] = %s (%#8x)\n", i,193 XELP_MEM_TYPE_STR_MAP[mem_type], pat);194 }195 196 err = xe_force_wake_put(gt_to_fw(gt), XE_FW_GT);197err_fw:198 xe_assert(xe, !err);199}200 201static const struct xe_pat_ops xelp_pat_ops = {202 .program_graphics = program_pat,203 .dump = xelp_dump,204};205 206static void xehp_dump(struct xe_gt *gt, struct drm_printer *p)207{208 struct xe_device *xe = gt_to_xe(gt);209 int i, err;210 211 err = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);212 if (err)213 goto err_fw;214 215 drm_printf(p, "PAT table:\n");216 217 for (i = 0; i < xe->pat.n_entries; i++) {218 u32 pat = xe_gt_mcr_unicast_read_any(gt, XE_REG_MCR(_PAT_INDEX(i)));219 u8 mem_type;220 221 mem_type = REG_FIELD_GET(XELP_MEM_TYPE_MASK, pat);222 223 drm_printf(p, "PAT[%2d] = %s (%#8x)\n", i,224 XELP_MEM_TYPE_STR_MAP[mem_type], pat);225 }226 227 err = xe_force_wake_put(gt_to_fw(gt), XE_FW_GT);228err_fw:229 xe_assert(xe, !err);230}231 232static const struct xe_pat_ops xehp_pat_ops = {233 .program_graphics = program_pat_mcr,234 .dump = xehp_dump,235};236 237static void xehpc_dump(struct xe_gt *gt, struct drm_printer *p)238{239 struct xe_device *xe = gt_to_xe(gt);240 int i, err;241 242 err = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);243 if (err)244 goto err_fw;245 246 drm_printf(p, "PAT table:\n");247 248 for (i = 0; i < xe->pat.n_entries; i++) {249 u32 pat = xe_gt_mcr_unicast_read_any(gt, XE_REG_MCR(_PAT_INDEX(i)));250 251 drm_printf(p, "PAT[%2d] = [ %u, %u ] (%#8x)\n", i,252 REG_FIELD_GET(XELP_MEM_TYPE_MASK, pat),253 REG_FIELD_GET(XEHPC_CLOS_LEVEL_MASK, pat), pat);254 }255 256 err = xe_force_wake_put(gt_to_fw(gt), XE_FW_GT);257err_fw:258 xe_assert(xe, !err);259}260 261static const struct xe_pat_ops xehpc_pat_ops = {262 .program_graphics = program_pat_mcr,263 .dump = xehpc_dump,264};265 266static void xelpg_dump(struct xe_gt *gt, struct drm_printer *p)267{268 struct xe_device *xe = gt_to_xe(gt);269 int i, err;270 271 err = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);272 if (err)273 goto err_fw;274 275 drm_printf(p, "PAT table:\n");276 277 for (i = 0; i < xe->pat.n_entries; i++) {278 u32 pat;279 280 if (xe_gt_is_media_type(gt))281 pat = xe_mmio_read32(gt, XE_REG(_PAT_INDEX(i)));282 else283 pat = xe_gt_mcr_unicast_read_any(gt, XE_REG_MCR(_PAT_INDEX(i)));284 285 drm_printf(p, "PAT[%2d] = [ %u, %u ] (%#8x)\n", i,286 REG_FIELD_GET(XELPG_L4_POLICY_MASK, pat),287 REG_FIELD_GET(XELPG_INDEX_COH_MODE_MASK, pat), pat);288 }289 290 err = xe_force_wake_put(gt_to_fw(gt), XE_FW_GT);291err_fw:292 xe_assert(xe, !err);293}294 295/*296 * SAMedia register offsets are adjusted by the write methods and they target297 * registers that are not MCR, while for normal GT they are MCR298 */299static const struct xe_pat_ops xelpg_pat_ops = {300 .program_graphics = program_pat,301 .program_media = program_pat_mcr,302 .dump = xelpg_dump,303};304 305static void xe2lpg_program_pat(struct xe_gt *gt, const struct xe_pat_table_entry table[],306 int n_entries)307{308 program_pat_mcr(gt, table, n_entries);309 xe_gt_mcr_multicast_write(gt, XE_REG_MCR(_PAT_ATS), xe2_pat_ats.value);310 311 if (IS_DGFX(gt_to_xe(gt)))312 xe_gt_mcr_multicast_write(gt, XE_REG_MCR(_PAT_PTA), xe2_pat_pta.value);313}314 315static void xe2lpm_program_pat(struct xe_gt *gt, const struct xe_pat_table_entry table[],316 int n_entries)317{318 program_pat(gt, table, n_entries);319 xe_mmio_write32(gt, XE_REG(_PAT_ATS), xe2_pat_ats.value);320 321 if (IS_DGFX(gt_to_xe(gt)))322 xe_mmio_write32(gt, XE_REG(_PAT_PTA), xe2_pat_pta.value);323}324 325static void xe2_dump(struct xe_gt *gt, struct drm_printer *p)326{327 struct xe_device *xe = gt_to_xe(gt);328 int i, err;329 u32 pat;330 331 err = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);332 if (err)333 goto err_fw;334 335 drm_printf(p, "PAT table:\n");336 337 for (i = 0; i < xe->pat.n_entries; i++) {338 if (xe_gt_is_media_type(gt))339 pat = xe_mmio_read32(gt, XE_REG(_PAT_INDEX(i)));340 else341 pat = xe_gt_mcr_unicast_read_any(gt, XE_REG_MCR(_PAT_INDEX(i)));342 343 drm_printf(p, "PAT[%2d] = [ %u, %u, %u, %u, %u, %u ] (%#8x)\n", i,344 !!(pat & XE2_NO_PROMOTE),345 !!(pat & XE2_COMP_EN),346 REG_FIELD_GET(XE2_L3_CLOS, pat),347 REG_FIELD_GET(XE2_L3_POLICY, pat),348 REG_FIELD_GET(XE2_L4_POLICY, pat),349 REG_FIELD_GET(XE2_COH_MODE, pat),350 pat);351 }352 353 /*354 * Also print PTA_MODE, which describes how the hardware accesses355 * PPGTT entries.356 */357 if (xe_gt_is_media_type(gt))358 pat = xe_mmio_read32(gt, XE_REG(_PAT_PTA));359 else360 pat = xe_gt_mcr_unicast_read_any(gt, XE_REG_MCR(_PAT_PTA));361 362 drm_printf(p, "Page Table Access:\n");363 drm_printf(p, "PTA_MODE= [ %u, %u, %u, %u, %u, %u ] (%#8x)\n",364 !!(pat & XE2_NO_PROMOTE),365 !!(pat & XE2_COMP_EN),366 REG_FIELD_GET(XE2_L3_CLOS, pat),367 REG_FIELD_GET(XE2_L3_POLICY, pat),368 REG_FIELD_GET(XE2_L4_POLICY, pat),369 REG_FIELD_GET(XE2_COH_MODE, pat),370 pat);371 372 err = xe_force_wake_put(gt_to_fw(gt), XE_FW_GT);373err_fw:374 xe_assert(xe, !err);375}376 377static const struct xe_pat_ops xe2_pat_ops = {378 .program_graphics = xe2lpg_program_pat,379 .program_media = xe2lpm_program_pat,380 .dump = xe2_dump,381};382 383void xe_pat_init_early(struct xe_device *xe)384{385 if (GRAPHICS_VER(xe) == 20) {386 xe->pat.ops = &xe2_pat_ops;387 xe->pat.table = xe2_pat_table;388 389 /* Wa_16023588340. XXX: Should use XE_WA */390 if (GRAPHICS_VERx100(xe) == 2001)391 xe->pat.n_entries = 28; /* Disable CLOS3 */392 else393 xe->pat.n_entries = ARRAY_SIZE(xe2_pat_table);394 395 xe->pat.idx[XE_CACHE_NONE] = 3;396 xe->pat.idx[XE_CACHE_WT] = 15;397 xe->pat.idx[XE_CACHE_WB] = 2;398 xe->pat.idx[XE_CACHE_NONE_COMPRESSION] = 12; /*Applicable on xe2 and beyond */399 } else if (xe->info.platform == XE_METEORLAKE) {400 xe->pat.ops = &xelpg_pat_ops;401 xe->pat.table = xelpg_pat_table;402 xe->pat.n_entries = ARRAY_SIZE(xelpg_pat_table);403 xe->pat.idx[XE_CACHE_NONE] = 2;404 xe->pat.idx[XE_CACHE_WT] = 1;405 xe->pat.idx[XE_CACHE_WB] = 3;406 } else if (xe->info.platform == XE_PVC) {407 xe->pat.ops = &xehpc_pat_ops;408 xe->pat.table = xehpc_pat_table;409 xe->pat.n_entries = ARRAY_SIZE(xehpc_pat_table);410 xe->pat.idx[XE_CACHE_NONE] = 0;411 xe->pat.idx[XE_CACHE_WT] = 2;412 xe->pat.idx[XE_CACHE_WB] = 3;413 } else if (xe->info.platform == XE_DG2) {414 /*415 * Table is the same as previous platforms, but programming416 * method has changed.417 */418 xe->pat.ops = &xehp_pat_ops;419 xe->pat.table = xelp_pat_table;420 xe->pat.n_entries = ARRAY_SIZE(xelp_pat_table);421 xe->pat.idx[XE_CACHE_NONE] = 3;422 xe->pat.idx[XE_CACHE_WT] = 2;423 xe->pat.idx[XE_CACHE_WB] = 0;424 } else if (GRAPHICS_VERx100(xe) <= 1210) {425 WARN_ON_ONCE(!IS_DGFX(xe) && !xe->info.has_llc);426 xe->pat.ops = &xelp_pat_ops;427 xe->pat.table = xelp_pat_table;428 xe->pat.n_entries = ARRAY_SIZE(xelp_pat_table);429 xe->pat.idx[XE_CACHE_NONE] = 3;430 xe->pat.idx[XE_CACHE_WT] = 2;431 xe->pat.idx[XE_CACHE_WB] = 0;432 } else {433 /*434 * Going forward we expect to need new PAT settings for most435 * new platforms; failure to provide a new table can easily436 * lead to subtle, hard-to-debug problems. If none of the437 * conditions above match the platform we're running on we'll438 * raise an error rather than trying to silently inherit the439 * most recent platform's behavior.440 */441 drm_err(&xe->drm, "Missing PAT table for platform with graphics version %d.%02d!\n",442 GRAPHICS_VER(xe), GRAPHICS_VERx100(xe) % 100);443 }444 445 /* VFs can't program nor dump PAT settings */446 if (IS_SRIOV_VF(xe))447 xe->pat.ops = NULL;448 449 xe_assert(xe, !xe->pat.ops || xe->pat.ops->dump);450 xe_assert(xe, !xe->pat.ops || xe->pat.ops->program_graphics);451 xe_assert(xe, !xe->pat.ops || MEDIA_VER(xe) < 13 || xe->pat.ops->program_media);452}453 454void xe_pat_init(struct xe_gt *gt)455{456 struct xe_device *xe = gt_to_xe(gt);457 458 if (!xe->pat.ops)459 return;460 461 if (xe_gt_is_media_type(gt))462 xe->pat.ops->program_media(gt, xe->pat.table, xe->pat.n_entries);463 else464 xe->pat.ops->program_graphics(gt, xe->pat.table, xe->pat.n_entries);465}466 467void xe_pat_dump(struct xe_gt *gt, struct drm_printer *p)468{469 struct xe_device *xe = gt_to_xe(gt);470 471 if (!xe->pat.ops)472 return;473 474 xe->pat.ops->dump(gt, p);475}476