902 lines · c
1// SPDX-License-Identifier: MIT2/*3 * Copyright © 2022 Intel Corporation4 */5 6#include "xe_wa.h"7 8#include <drm/drm_managed.h>9#include <kunit/visibility.h>10#include <linux/compiler_types.h>11 12#include <generated/xe_wa_oob.h>13 14#include "regs/xe_engine_regs.h"15#include "regs/xe_gt_regs.h"16#include "regs/xe_regs.h"17#include "xe_device_types.h"18#include "xe_force_wake.h"19#include "xe_gt.h"20#include "xe_hw_engine_types.h"21#include "xe_mmio.h"22#include "xe_platform_types.h"23#include "xe_rtp.h"24#include "xe_sriov.h"25#include "xe_step.h"26 27/**28 * DOC: Hardware workarounds29 *30 * Hardware workarounds are register programming documented to be executed in31 * the driver that fall outside of the normal programming sequences for a32 * platform. There are some basic categories of workarounds, depending on33 * how/when they are applied:34 *35 * - LRC workarounds: workarounds that touch registers that are36 * saved/restored to/from the HW context image. The list is emitted (via Load37 * Register Immediate commands) once when initializing the device and saved in38 * the default context. That default context is then used on every context39 * creation to have a "primed golden context", i.e. a context image that40 * already contains the changes needed to all the registers.41 *42 * - Engine workarounds: the list of these WAs is applied whenever the specific43 * engine is reset. It's also possible that a set of engine classes share a44 * common power domain and they are reset together. This happens on some45 * platforms with render and compute engines. In this case (at least) one of46 * them need to keeep the workaround programming: the approach taken in the47 * driver is to tie those workarounds to the first compute/render engine that48 * is registered. When executing with GuC submission, engine resets are49 * outside of kernel driver control, hence the list of registers involved in50 * written once, on engine initialization, and then passed to GuC, that51 * saves/restores their values before/after the reset takes place. See52 * ``drivers/gpu/drm/xe/xe_guc_ads.c`` for reference.53 *54 * - GT workarounds: the list of these WAs is applied whenever these registers55 * revert to their default values: on GPU reset, suspend/resume [1]_, etc.56 *57 * - Register whitelist: some workarounds need to be implemented in userspace,58 * but need to touch privileged registers. The whitelist in the kernel59 * instructs the hardware to allow the access to happen. From the kernel side,60 * this is just a special case of a MMIO workaround (as we write the list of61 * these to/be-whitelisted registers to some special HW registers).62 *63 * - Workaround batchbuffers: buffers that get executed automatically by the64 * hardware on every HW context restore. These buffers are created and65 * programmed in the default context so the hardware always go through those66 * programming sequences when switching contexts. The support for workaround67 * batchbuffers is enabled these hardware mechanisms:68 *69 * #. INDIRECT_CTX: A batchbuffer and an offset are provided in the default70 * context, pointing the hardware to jump to that location when that offset71 * is reached in the context restore. Workaround batchbuffer in the driver72 * currently uses this mechanism for all platforms.73 *74 * #. BB_PER_CTX_PTR: A batchbuffer is provided in the default context,75 * pointing the hardware to a buffer to continue executing after the76 * engine registers are restored in a context restore sequence. This is77 * currently not used in the driver.78 *79 * - Other/OOB: There are WAs that, due to their nature, cannot be applied from80 * a central place. Those are peppered around the rest of the code, as needed.81 * Workarounds related to the display IP are the main example.82 *83 * .. [1] Technically, some registers are powercontext saved & restored, so they84 * survive a suspend/resume. In practice, writing them again is not too85 * costly and simplifies things, so it's the approach taken in the driver.86 *87 * .. note::88 * Hardware workarounds in xe work the same way as in i915, with the89 * difference of how they are maintained in the code. In xe it uses the90 * xe_rtp infrastructure so the workarounds can be kept in tables, following91 * a more declarative approach rather than procedural.92 */93 94#undef XE_REG_MCR95#define XE_REG_MCR(...) XE_REG(__VA_ARGS__, .mcr = 1)96 97__diag_push();98__diag_ignore_all("-Woverride-init", "Allow field overrides in table");99 100static const struct xe_rtp_entry_sr gt_was[] = {101 { XE_RTP_NAME("14011060649"),102 XE_RTP_RULES(MEDIA_VERSION_RANGE(1200, 1255),103 ENGINE_CLASS(VIDEO_DECODE),104 FUNC(xe_rtp_match_even_instance)),105 XE_RTP_ACTIONS(SET(VDBOX_CGCTL3F10(0), IECPUNIT_CLKGATE_DIS)),106 XE_RTP_ENTRY_FLAG(FOREACH_ENGINE),107 },108 { XE_RTP_NAME("14011059788"),109 XE_RTP_RULES(GRAPHICS_VERSION_RANGE(1200, 1210)),110 XE_RTP_ACTIONS(SET(DFR_RATIO_EN_AND_CHICKEN, DFR_DISABLE))111 },112 { XE_RTP_NAME("14015795083"),113 XE_RTP_RULES(GRAPHICS_VERSION_RANGE(1200, 1260)),114 XE_RTP_ACTIONS(CLR(MISCCPCTL, DOP_CLOCK_GATE_RENDER_ENABLE))115 },116 117 /* DG1 */118 119 { XE_RTP_NAME("1409420604"),120 XE_RTP_RULES(PLATFORM(DG1)),121 XE_RTP_ACTIONS(SET(SUBSLICE_UNIT_LEVEL_CLKGATE2, CPSSUNIT_CLKGATE_DIS))122 },123 { XE_RTP_NAME("1408615072"),124 XE_RTP_RULES(PLATFORM(DG1)),125 XE_RTP_ACTIONS(SET(UNSLICE_UNIT_LEVEL_CLKGATE2, VSUNIT_CLKGATE2_DIS))126 },127 128 /* DG2 */129 130 { XE_RTP_NAME("22010523718"),131 XE_RTP_RULES(SUBPLATFORM(DG2, G10)),132 XE_RTP_ACTIONS(SET(UNSLICE_UNIT_LEVEL_CLKGATE, CG3DDISCFEG_CLKGATE_DIS))133 },134 { XE_RTP_NAME("14011006942"),135 XE_RTP_RULES(SUBPLATFORM(DG2, G10)),136 XE_RTP_ACTIONS(SET(SUBSLICE_UNIT_LEVEL_CLKGATE, DSS_ROUTER_CLKGATE_DIS))137 },138 { XE_RTP_NAME("14014830051"),139 XE_RTP_RULES(PLATFORM(DG2)),140 XE_RTP_ACTIONS(CLR(SARB_CHICKEN1, COMP_CKN_IN))141 },142 { XE_RTP_NAME("18018781329"),143 XE_RTP_RULES(PLATFORM(DG2)),144 XE_RTP_ACTIONS(SET(RENDER_MOD_CTRL, FORCE_MISS_FTLB),145 SET(COMP_MOD_CTRL, FORCE_MISS_FTLB),146 SET(XEHP_VDBX_MOD_CTRL, FORCE_MISS_FTLB),147 SET(XEHP_VEBX_MOD_CTRL, FORCE_MISS_FTLB))148 },149 { XE_RTP_NAME("1509235366"),150 XE_RTP_RULES(PLATFORM(DG2)),151 XE_RTP_ACTIONS(SET(XEHP_GAMCNTRL_CTRL,152 INVALIDATION_BROADCAST_MODE_DIS |153 GLOBAL_INVALIDATION_MODE))154 },155 156 /* PVC */157 158 { XE_RTP_NAME("18018781329"),159 XE_RTP_RULES(PLATFORM(PVC)),160 XE_RTP_ACTIONS(SET(RENDER_MOD_CTRL, FORCE_MISS_FTLB),161 SET(COMP_MOD_CTRL, FORCE_MISS_FTLB),162 SET(XEHP_VDBX_MOD_CTRL, FORCE_MISS_FTLB),163 SET(XEHP_VEBX_MOD_CTRL, FORCE_MISS_FTLB))164 },165 { XE_RTP_NAME("16016694945"),166 XE_RTP_RULES(PLATFORM(PVC)),167 XE_RTP_ACTIONS(SET(XEHPC_LNCFMISCCFGREG0, XEHPC_OVRLSCCC))168 },169 170 /* Xe_LPG */171 172 { XE_RTP_NAME("14015795083"),173 XE_RTP_RULES(GRAPHICS_VERSION_RANGE(1270, 1271), GRAPHICS_STEP(A0, B0)),174 XE_RTP_ACTIONS(CLR(MISCCPCTL, DOP_CLOCK_GATE_RENDER_ENABLE))175 },176 { XE_RTP_NAME("14018575942"),177 XE_RTP_RULES(GRAPHICS_VERSION_RANGE(1270, 1274)),178 XE_RTP_ACTIONS(SET(COMP_MOD_CTRL, FORCE_MISS_FTLB))179 },180 { XE_RTP_NAME("22016670082"),181 XE_RTP_RULES(GRAPHICS_VERSION_RANGE(1270, 1274)),182 XE_RTP_ACTIONS(SET(SQCNT1, ENFORCE_RAR))183 },184 185 /* Xe_LPM+ */186 187 { XE_RTP_NAME("16021867713"),188 XE_RTP_RULES(MEDIA_VERSION(1300),189 ENGINE_CLASS(VIDEO_DECODE)),190 XE_RTP_ACTIONS(SET(VDBOX_CGCTL3F1C(0), MFXPIPE_CLKGATE_DIS)),191 XE_RTP_ENTRY_FLAG(FOREACH_ENGINE),192 },193 { XE_RTP_NAME("22016670082"),194 XE_RTP_RULES(MEDIA_VERSION(1300)),195 XE_RTP_ACTIONS(SET(XELPMP_SQCNT1, ENFORCE_RAR))196 },197 198 /* Xe2_LPG */199 200 { XE_RTP_NAME("16020975621"),201 XE_RTP_RULES(GRAPHICS_VERSION(2004), GRAPHICS_STEP(A0, B0)),202 XE_RTP_ACTIONS(SET(XEHP_SLICE_UNIT_LEVEL_CLKGATE, SBEUNIT_CLKGATE_DIS))203 },204 { XE_RTP_NAME("14018157293"),205 XE_RTP_RULES(GRAPHICS_VERSION(2004), GRAPHICS_STEP(A0, B0)),206 XE_RTP_ACTIONS(SET(XEHPC_L3CLOS_MASK(0), ~0),207 SET(XEHPC_L3CLOS_MASK(1), ~0),208 SET(XEHPC_L3CLOS_MASK(2), ~0),209 SET(XEHPC_L3CLOS_MASK(3), ~0))210 },211 212 /* Xe2_LPM */213 214 { XE_RTP_NAME("14017421178"),215 XE_RTP_RULES(MEDIA_VERSION(2000),216 ENGINE_CLASS(VIDEO_DECODE)),217 XE_RTP_ACTIONS(SET(VDBOX_CGCTL3F10(0), IECPUNIT_CLKGATE_DIS)),218 XE_RTP_ENTRY_FLAG(FOREACH_ENGINE),219 },220 { XE_RTP_NAME("16021867713"),221 XE_RTP_RULES(MEDIA_VERSION(2000),222 ENGINE_CLASS(VIDEO_DECODE)),223 XE_RTP_ACTIONS(SET(VDBOX_CGCTL3F1C(0), MFXPIPE_CLKGATE_DIS)),224 XE_RTP_ENTRY_FLAG(FOREACH_ENGINE),225 },226 { XE_RTP_NAME("14019449301"),227 XE_RTP_RULES(MEDIA_VERSION(2000), ENGINE_CLASS(VIDEO_DECODE)),228 XE_RTP_ACTIONS(SET(VDBOX_CGCTL3F08(0), CG3DDISHRS_CLKGATE_DIS)),229 XE_RTP_ENTRY_FLAG(FOREACH_ENGINE),230 },231 232 /* Xe2_HPM */233 234 { XE_RTP_NAME("16021867713"),235 XE_RTP_RULES(MEDIA_VERSION(1301),236 ENGINE_CLASS(VIDEO_DECODE)),237 XE_RTP_ACTIONS(SET(VDBOX_CGCTL3F1C(0), MFXPIPE_CLKGATE_DIS)),238 XE_RTP_ENTRY_FLAG(FOREACH_ENGINE),239 },240 { XE_RTP_NAME("14020316580"),241 XE_RTP_RULES(MEDIA_VERSION(1301)),242 XE_RTP_ACTIONS(CLR(POWERGATE_ENABLE,243 VDN_HCP_POWERGATE_ENABLE(0) |244 VDN_MFXVDENC_POWERGATE_ENABLE(0) |245 VDN_HCP_POWERGATE_ENABLE(2) |246 VDN_MFXVDENC_POWERGATE_ENABLE(2))),247 },248 { XE_RTP_NAME("14019449301"),249 XE_RTP_RULES(MEDIA_VERSION(1301), ENGINE_CLASS(VIDEO_DECODE)),250 XE_RTP_ACTIONS(SET(VDBOX_CGCTL3F08(0), CG3DDISHRS_CLKGATE_DIS)),251 XE_RTP_ENTRY_FLAG(FOREACH_ENGINE),252 },253 254 {}255};256 257static const struct xe_rtp_entry_sr engine_was[] = {258 { XE_RTP_NAME("22010931296, 18011464164, 14010919138"),259 XE_RTP_RULES(GRAPHICS_VERSION(1200), ENGINE_CLASS(RENDER)),260 XE_RTP_ACTIONS(SET(FF_THREAD_MODE(RENDER_RING_BASE),261 FF_TESSELATION_DOP_GATE_DISABLE))262 },263 { XE_RTP_NAME("1409804808"),264 XE_RTP_RULES(GRAPHICS_VERSION(1200),265 ENGINE_CLASS(RENDER),266 IS_INTEGRATED),267 XE_RTP_ACTIONS(SET(ROW_CHICKEN2, PUSH_CONST_DEREF_HOLD_DIS))268 },269 { XE_RTP_NAME("14010229206, 1409085225"),270 XE_RTP_RULES(GRAPHICS_VERSION(1200),271 ENGINE_CLASS(RENDER),272 IS_INTEGRATED),273 XE_RTP_ACTIONS(SET(ROW_CHICKEN4, DISABLE_TDL_PUSH))274 },275 { XE_RTP_NAME("1606931601"),276 XE_RTP_RULES(GRAPHICS_VERSION_RANGE(1200, 1210), ENGINE_CLASS(RENDER)),277 XE_RTP_ACTIONS(SET(ROW_CHICKEN2, DISABLE_EARLY_READ))278 },279 { XE_RTP_NAME("14010826681, 1606700617, 22010271021, 18019627453"),280 XE_RTP_RULES(GRAPHICS_VERSION_RANGE(1200, 1255), ENGINE_CLASS(RENDER)),281 XE_RTP_ACTIONS(SET(CS_DEBUG_MODE1(RENDER_RING_BASE),282 FF_DOP_CLOCK_GATE_DISABLE))283 },284 { XE_RTP_NAME("1406941453"),285 XE_RTP_RULES(GRAPHICS_VERSION_RANGE(1200, 1210), ENGINE_CLASS(RENDER)),286 XE_RTP_ACTIONS(SET(SAMPLER_MODE, ENABLE_SMALLPL))287 },288 { XE_RTP_NAME("FtrPerCtxtPreemptionGranularityControl"),289 XE_RTP_RULES(GRAPHICS_VERSION_RANGE(1200, 1250), ENGINE_CLASS(RENDER)),290 XE_RTP_ACTIONS(SET(FF_SLICE_CS_CHICKEN1(RENDER_RING_BASE),291 FFSC_PERCTX_PREEMPT_CTRL))292 },293 294 /* TGL */295 296 { XE_RTP_NAME("1607297627, 1607030317, 1607186500"),297 XE_RTP_RULES(PLATFORM(TIGERLAKE), ENGINE_CLASS(RENDER)),298 XE_RTP_ACTIONS(SET(RING_PSMI_CTL(RENDER_RING_BASE),299 WAIT_FOR_EVENT_POWER_DOWN_DISABLE |300 RC_SEMA_IDLE_MSG_DISABLE))301 },302 303 /* RKL */304 305 { XE_RTP_NAME("1607297627, 1607030317, 1607186500"),306 XE_RTP_RULES(PLATFORM(ROCKETLAKE), ENGINE_CLASS(RENDER)),307 XE_RTP_ACTIONS(SET(RING_PSMI_CTL(RENDER_RING_BASE),308 WAIT_FOR_EVENT_POWER_DOWN_DISABLE |309 RC_SEMA_IDLE_MSG_DISABLE))310 },311 312 /* ADL-P */313 314 { XE_RTP_NAME("1607297627, 1607030317, 1607186500"),315 XE_RTP_RULES(PLATFORM(ALDERLAKE_P), ENGINE_CLASS(RENDER)),316 XE_RTP_ACTIONS(SET(RING_PSMI_CTL(RENDER_RING_BASE),317 WAIT_FOR_EVENT_POWER_DOWN_DISABLE |318 RC_SEMA_IDLE_MSG_DISABLE))319 },320 321 /* DG2 */322 323 { XE_RTP_NAME("22013037850"),324 XE_RTP_RULES(PLATFORM(DG2), FUNC(xe_rtp_match_first_render_or_compute)),325 XE_RTP_ACTIONS(SET(LSC_CHICKEN_BIT_0_UDW,326 DISABLE_128B_EVICTION_COMMAND_UDW))327 },328 { XE_RTP_NAME("22014226127"),329 XE_RTP_RULES(PLATFORM(DG2), FUNC(xe_rtp_match_first_render_or_compute)),330 XE_RTP_ACTIONS(SET(LSC_CHICKEN_BIT_0, DISABLE_D8_D16_COASLESCE))331 },332 { XE_RTP_NAME("18017747507"),333 XE_RTP_RULES(PLATFORM(DG2), FUNC(xe_rtp_match_first_render_or_compute)),334 XE_RTP_ACTIONS(SET(VFG_PREEMPTION_CHICKEN,335 POLYGON_TRIFAN_LINELOOP_DISABLE))336 },337 { XE_RTP_NAME("22012826095, 22013059131"),338 XE_RTP_RULES(SUBPLATFORM(DG2, G11),339 FUNC(xe_rtp_match_first_render_or_compute)),340 XE_RTP_ACTIONS(FIELD_SET(LSC_CHICKEN_BIT_0_UDW,341 MAXREQS_PER_BANK,342 REG_FIELD_PREP(MAXREQS_PER_BANK, 2)))343 },344 { XE_RTP_NAME("22013059131"),345 XE_RTP_RULES(SUBPLATFORM(DG2, G11),346 FUNC(xe_rtp_match_first_render_or_compute)),347 XE_RTP_ACTIONS(SET(LSC_CHICKEN_BIT_0, FORCE_1_SUB_MESSAGE_PER_FRAGMENT))348 },349 { XE_RTP_NAME("14015227452"),350 XE_RTP_RULES(PLATFORM(DG2),351 FUNC(xe_rtp_match_first_render_or_compute)),352 XE_RTP_ACTIONS(SET(ROW_CHICKEN4, XEHP_DIS_BBL_SYSPIPE))353 },354 { XE_RTP_NAME("18028616096"),355 XE_RTP_RULES(PLATFORM(DG2),356 FUNC(xe_rtp_match_first_render_or_compute)),357 XE_RTP_ACTIONS(SET(LSC_CHICKEN_BIT_0_UDW, UGM_FRAGMENT_THRESHOLD_TO_3))358 },359 { XE_RTP_NAME("22015475538"),360 XE_RTP_RULES(PLATFORM(DG2),361 FUNC(xe_rtp_match_first_render_or_compute)),362 XE_RTP_ACTIONS(SET(LSC_CHICKEN_BIT_0_UDW, DIS_CHAIN_2XSIMD8))363 },364 { XE_RTP_NAME("22012654132"),365 XE_RTP_RULES(SUBPLATFORM(DG2, G11),366 FUNC(xe_rtp_match_first_render_or_compute)),367 XE_RTP_ACTIONS(SET(CACHE_MODE_SS, ENABLE_PREFETCH_INTO_IC,368 /*369 * Register can't be read back for verification on370 * DG2 due to Wa_14012342262371 */372 .read_mask = 0))373 },374 { XE_RTP_NAME("1509727124"),375 XE_RTP_RULES(PLATFORM(DG2), ENGINE_CLASS(RENDER)),376 XE_RTP_ACTIONS(SET(SAMPLER_MODE, SC_DISABLE_POWER_OPTIMIZATION_EBB))377 },378 { XE_RTP_NAME("22012856258"),379 XE_RTP_RULES(PLATFORM(DG2), ENGINE_CLASS(RENDER)),380 XE_RTP_ACTIONS(SET(ROW_CHICKEN2, DISABLE_READ_SUPPRESSION))381 },382 { XE_RTP_NAME("22010960976, 14013347512"),383 XE_RTP_RULES(PLATFORM(DG2), ENGINE_CLASS(RENDER)),384 XE_RTP_ACTIONS(CLR(XEHP_HDC_CHICKEN0,385 LSC_L1_FLUSH_CTL_3D_DATAPORT_FLUSH_EVENTS_MASK))386 },387 { XE_RTP_NAME("14015150844"),388 XE_RTP_RULES(PLATFORM(DG2), FUNC(xe_rtp_match_first_render_or_compute)),389 XE_RTP_ACTIONS(SET(XEHP_HDC_CHICKEN0, DIS_ATOMIC_CHAINING_TYPED_WRITES,390 XE_RTP_NOCHECK))391 },392 393 /* PVC */394 395 { XE_RTP_NAME("22014226127"),396 XE_RTP_RULES(PLATFORM(PVC), FUNC(xe_rtp_match_first_render_or_compute)),397 XE_RTP_ACTIONS(SET(LSC_CHICKEN_BIT_0, DISABLE_D8_D16_COASLESCE))398 },399 { XE_RTP_NAME("14015227452"),400 XE_RTP_RULES(PLATFORM(PVC), FUNC(xe_rtp_match_first_render_or_compute)),401 XE_RTP_ACTIONS(SET(ROW_CHICKEN4, XEHP_DIS_BBL_SYSPIPE))402 },403 { XE_RTP_NAME("18020744125"),404 XE_RTP_RULES(PLATFORM(PVC), FUNC(xe_rtp_match_first_render_or_compute),405 ENGINE_CLASS(COMPUTE)),406 XE_RTP_ACTIONS(SET(RING_HWSTAM(RENDER_RING_BASE), ~0))407 },408 { XE_RTP_NAME("14014999345"),409 XE_RTP_RULES(PLATFORM(PVC), ENGINE_CLASS(COMPUTE),410 GRAPHICS_STEP(B0, C0)),411 XE_RTP_ACTIONS(SET(CACHE_MODE_SS, DISABLE_ECC))412 },413 414 /* Xe_LPG */415 416 { XE_RTP_NAME("14017856879"),417 XE_RTP_RULES(GRAPHICS_VERSION_RANGE(1270, 1274),418 FUNC(xe_rtp_match_first_render_or_compute)),419 XE_RTP_ACTIONS(SET(ROW_CHICKEN3, DIS_FIX_EOT1_FLUSH))420 },421 { XE_RTP_NAME("14015150844"),422 XE_RTP_RULES(GRAPHICS_VERSION_RANGE(1270, 1271),423 FUNC(xe_rtp_match_first_render_or_compute)),424 XE_RTP_ACTIONS(SET(XEHP_HDC_CHICKEN0, DIS_ATOMIC_CHAINING_TYPED_WRITES,425 XE_RTP_NOCHECK))426 },427 { XE_RTP_NAME("14020495402"),428 XE_RTP_RULES(GRAPHICS_VERSION_RANGE(1270, 1274),429 FUNC(xe_rtp_match_first_render_or_compute)),430 XE_RTP_ACTIONS(SET(ROW_CHICKEN2, DISABLE_TDL_SVHS_GATING))431 },432 433 /* Xe2_LPG */434 435 { XE_RTP_NAME("18032247524"),436 XE_RTP_RULES(GRAPHICS_VERSION(2004),437 FUNC(xe_rtp_match_first_render_or_compute)),438 XE_RTP_ACTIONS(SET(LSC_CHICKEN_BIT_0, SEQUENTIAL_ACCESS_UPGRADE_DISABLE))439 },440 { XE_RTP_NAME("16018712365"),441 XE_RTP_RULES(GRAPHICS_VERSION(2004), FUNC(xe_rtp_match_first_render_or_compute)),442 XE_RTP_ACTIONS(SET(LSC_CHICKEN_BIT_0_UDW, XE2_ALLOC_DPA_STARVE_FIX_DIS))443 },444 { XE_RTP_NAME("14018957109"),445 XE_RTP_RULES(GRAPHICS_VERSION(2004), GRAPHICS_STEP(A0, B0),446 FUNC(xe_rtp_match_first_render_or_compute)),447 XE_RTP_ACTIONS(SET(HALF_SLICE_CHICKEN5, DISABLE_SAMPLE_G_PERFORMANCE))448 },449 { XE_RTP_NAME("14020338487"),450 XE_RTP_RULES(GRAPHICS_VERSION(2004), FUNC(xe_rtp_match_first_render_or_compute)),451 XE_RTP_ACTIONS(SET(ROW_CHICKEN3, XE2_EUPEND_CHK_FLUSH_DIS))452 },453 { XE_RTP_NAME("18034896535, 16021540221"), /* 16021540221: GRAPHICS_STEP(A0, B0) */454 XE_RTP_RULES(GRAPHICS_VERSION_RANGE(2001, 2004),455 FUNC(xe_rtp_match_first_render_or_compute)),456 XE_RTP_ACTIONS(SET(ROW_CHICKEN4, DISABLE_TDL_PUSH))457 },458 { XE_RTP_NAME("14019322943"),459 XE_RTP_RULES(GRAPHICS_VERSION(2004), GRAPHICS_STEP(A0, B0),460 FUNC(xe_rtp_match_first_render_or_compute)),461 XE_RTP_ACTIONS(SET(LSC_CHICKEN_BIT_0, TGM_WRITE_EOM_FORCE))462 },463 { XE_RTP_NAME("14018471104"),464 XE_RTP_RULES(GRAPHICS_VERSION(2004), FUNC(xe_rtp_match_first_render_or_compute)),465 XE_RTP_ACTIONS(SET(LSC_CHICKEN_BIT_0_UDW, ENABLE_SMP_LD_RENDER_SURFACE_CONTROL))466 },467 { XE_RTP_NAME("16018737384"),468 XE_RTP_RULES(GRAPHICS_VERSION(2004), FUNC(xe_rtp_match_first_render_or_compute)),469 XE_RTP_ACTIONS(SET(ROW_CHICKEN, EARLY_EOT_DIS))470 },471 /*472 * These two workarounds are the same, just applying to different473 * engines. Although Wa_18032095049 (for the RCS) isn't required on474 * all steppings, disabling these reports has no impact for our475 * driver or the GuC, so we go ahead and treat it the same as476 * Wa_16021639441 which does apply to all steppings.477 */478 { XE_RTP_NAME("18032095049, 16021639441"),479 XE_RTP_RULES(GRAPHICS_VERSION(2004)),480 XE_RTP_ACTIONS(SET(CSFE_CHICKEN1(0),481 GHWSP_CSB_REPORT_DIS |482 PPHWSP_CSB_AND_TIMESTAMP_REPORT_DIS,483 XE_RTP_ACTION_FLAG(ENGINE_BASE)))484 },485 { XE_RTP_NAME("16018610683"),486 XE_RTP_RULES(GRAPHICS_VERSION(2004), FUNC(xe_rtp_match_first_render_or_compute)),487 XE_RTP_ACTIONS(SET(TDL_TSL_CHICKEN, SLM_WMTP_RESTORE))488 },489 { XE_RTP_NAME("14021402888"),490 XE_RTP_RULES(GRAPHICS_VERSION(2004), ENGINE_CLASS(RENDER)),491 XE_RTP_ACTIONS(SET(HALF_SLICE_CHICKEN7, CLEAR_OPTIMIZATION_DISABLE))492 },493 494 /* Xe2_HPG */495 496 { XE_RTP_NAME("16018712365"),497 XE_RTP_RULES(GRAPHICS_VERSION(2001), FUNC(xe_rtp_match_first_render_or_compute)),498 XE_RTP_ACTIONS(SET(LSC_CHICKEN_BIT_0_UDW, XE2_ALLOC_DPA_STARVE_FIX_DIS))499 },500 { XE_RTP_NAME("16018737384"),501 XE_RTP_RULES(GRAPHICS_VERSION(2001), FUNC(xe_rtp_match_first_render_or_compute)),502 XE_RTP_ACTIONS(SET(ROW_CHICKEN, EARLY_EOT_DIS))503 },504 { XE_RTP_NAME("14019988906"),505 XE_RTP_RULES(GRAPHICS_VERSION(2001), FUNC(xe_rtp_match_first_render_or_compute)),506 XE_RTP_ACTIONS(SET(XEHP_PSS_CHICKEN, FLSH_IGNORES_PSD))507 },508 { XE_RTP_NAME("14019877138"),509 XE_RTP_RULES(GRAPHICS_VERSION(2001), FUNC(xe_rtp_match_first_render_or_compute)),510 XE_RTP_ACTIONS(SET(XEHP_PSS_CHICKEN, FD_END_COLLECT))511 },512 { XE_RTP_NAME("14020338487"),513 XE_RTP_RULES(GRAPHICS_VERSION(2001), FUNC(xe_rtp_match_first_render_or_compute)),514 XE_RTP_ACTIONS(SET(ROW_CHICKEN3, XE2_EUPEND_CHK_FLUSH_DIS))515 },516 { XE_RTP_NAME("18032247524"),517 XE_RTP_RULES(GRAPHICS_VERSION(2001), FUNC(xe_rtp_match_first_render_or_compute)),518 XE_RTP_ACTIONS(SET(LSC_CHICKEN_BIT_0, SEQUENTIAL_ACCESS_UPGRADE_DISABLE))519 },520 { XE_RTP_NAME("14018471104"),521 XE_RTP_RULES(GRAPHICS_VERSION(2001), FUNC(xe_rtp_match_first_render_or_compute)),522 XE_RTP_ACTIONS(SET(LSC_CHICKEN_BIT_0_UDW, ENABLE_SMP_LD_RENDER_SURFACE_CONTROL))523 },524 /*525 * Although this workaround isn't required for the RCS, disabling these526 * reports has no impact for our driver or the GuC, so we go ahead and527 * apply this to all engines for simplicity.528 */529 { XE_RTP_NAME("16021639441"),530 XE_RTP_RULES(GRAPHICS_VERSION(2001)),531 XE_RTP_ACTIONS(SET(CSFE_CHICKEN1(0),532 GHWSP_CSB_REPORT_DIS |533 PPHWSP_CSB_AND_TIMESTAMP_REPORT_DIS,534 XE_RTP_ACTION_FLAG(ENGINE_BASE)))535 },536 { XE_RTP_NAME("14019811474"),537 XE_RTP_RULES(GRAPHICS_VERSION(2001),538 FUNC(xe_rtp_match_first_render_or_compute)),539 XE_RTP_ACTIONS(SET(LSC_CHICKEN_BIT_0, WR_REQ_CHAINING_DIS))540 },541 { XE_RTP_NAME("14021402888"),542 XE_RTP_RULES(GRAPHICS_VERSION(2001), ENGINE_CLASS(RENDER)),543 XE_RTP_ACTIONS(SET(HALF_SLICE_CHICKEN7, CLEAR_OPTIMIZATION_DISABLE))544 },545 { XE_RTP_NAME("14021821874"),546 XE_RTP_RULES(GRAPHICS_VERSION(2001), FUNC(xe_rtp_match_first_render_or_compute)),547 XE_RTP_ACTIONS(SET(TDL_TSL_CHICKEN, STK_ID_RESTRICT))548 },549 550 /* Xe2_LPM */551 552 { XE_RTP_NAME("16021639441"),553 XE_RTP_RULES(MEDIA_VERSION(2000)),554 XE_RTP_ACTIONS(SET(CSFE_CHICKEN1(0),555 GHWSP_CSB_REPORT_DIS |556 PPHWSP_CSB_AND_TIMESTAMP_REPORT_DIS,557 XE_RTP_ACTION_FLAG(ENGINE_BASE)))558 },559 560 /* Xe2_HPM */561 562 { XE_RTP_NAME("16021639441"),563 XE_RTP_RULES(MEDIA_VERSION(1301)),564 XE_RTP_ACTIONS(SET(CSFE_CHICKEN1(0),565 GHWSP_CSB_REPORT_DIS |566 PPHWSP_CSB_AND_TIMESTAMP_REPORT_DIS,567 XE_RTP_ACTION_FLAG(ENGINE_BASE)))568 },569 570 {}571};572 573static const struct xe_rtp_entry_sr lrc_was[] = {574 { XE_RTP_NAME("1409342910, 14010698770, 14010443199, 1408979724, 1409178076, 1409207793, 1409217633, 1409252684, 1409347922, 1409142259"),575 XE_RTP_RULES(GRAPHICS_VERSION_RANGE(1200, 1210)),576 XE_RTP_ACTIONS(SET(COMMON_SLICE_CHICKEN3,577 DISABLE_CPS_AWARE_COLOR_PIPE))578 },579 { XE_RTP_NAME("WaDisableGPGPUMidThreadPreemption"),580 XE_RTP_RULES(GRAPHICS_VERSION_RANGE(1200, 1210)),581 XE_RTP_ACTIONS(FIELD_SET(CS_CHICKEN1(RENDER_RING_BASE),582 PREEMPT_GPGPU_LEVEL_MASK,583 PREEMPT_GPGPU_THREAD_GROUP_LEVEL))584 },585 { XE_RTP_NAME("1806527549"),586 XE_RTP_RULES(GRAPHICS_VERSION(1200)),587 XE_RTP_ACTIONS(SET(HIZ_CHICKEN, HZ_DEPTH_TEST_LE_GE_OPT_DISABLE))588 },589 { XE_RTP_NAME("1606376872"),590 XE_RTP_RULES(GRAPHICS_VERSION(1200)),591 XE_RTP_ACTIONS(SET(COMMON_SLICE_CHICKEN4, DISABLE_TDC_LOAD_BALANCING_CALC))592 },593 594 /* DG1 */595 596 { XE_RTP_NAME("1409044764"),597 XE_RTP_RULES(PLATFORM(DG1)),598 XE_RTP_ACTIONS(CLR(COMMON_SLICE_CHICKEN3,599 DG1_FLOAT_POINT_BLEND_OPT_STRICT_MODE_EN))600 },601 { XE_RTP_NAME("22010493298"),602 XE_RTP_RULES(PLATFORM(DG1)),603 XE_RTP_ACTIONS(SET(HIZ_CHICKEN,604 DG1_HZ_READ_SUPPRESSION_OPTIMIZATION_DISABLE))605 },606 607 /* DG2 */608 609 { XE_RTP_NAME("16013271637"),610 XE_RTP_RULES(PLATFORM(DG2)),611 XE_RTP_ACTIONS(SET(XEHP_SLICE_COMMON_ECO_CHICKEN1,612 MSC_MSAA_REODER_BUF_BYPASS_DISABLE))613 },614 { XE_RTP_NAME("14014947963"),615 XE_RTP_RULES(PLATFORM(DG2)),616 XE_RTP_ACTIONS(FIELD_SET(VF_PREEMPTION,617 PREEMPTION_VERTEX_COUNT,618 0x4000))619 },620 { XE_RTP_NAME("18018764978"),621 XE_RTP_RULES(PLATFORM(DG2)),622 XE_RTP_ACTIONS(SET(XEHP_PSS_MODE2,623 SCOREBOARD_STALL_FLUSH_CONTROL))624 },625 { XE_RTP_NAME("18019271663"),626 XE_RTP_RULES(PLATFORM(DG2)),627 XE_RTP_ACTIONS(SET(CACHE_MODE_1, MSAA_OPTIMIZATION_REDUC_DISABLE))628 },629 { XE_RTP_NAME("14019877138"),630 XE_RTP_RULES(PLATFORM(DG2)),631 XE_RTP_ACTIONS(SET(XEHP_PSS_CHICKEN, FD_END_COLLECT))632 },633 634 /* PVC */635 636 { XE_RTP_NAME("16017236439"),637 XE_RTP_RULES(PLATFORM(PVC), ENGINE_CLASS(COPY),638 FUNC(xe_rtp_match_even_instance)),639 XE_RTP_ACTIONS(SET(BCS_SWCTRL(0),640 BCS_SWCTRL_DISABLE_256B,641 XE_RTP_ACTION_FLAG(ENGINE_BASE))),642 },643 644 /* Xe_LPG */645 646 { XE_RTP_NAME("18019271663"),647 XE_RTP_RULES(GRAPHICS_VERSION_RANGE(1270, 1274)),648 XE_RTP_ACTIONS(SET(CACHE_MODE_1, MSAA_OPTIMIZATION_REDUC_DISABLE))649 },650 { XE_RTP_NAME("14019877138"),651 XE_RTP_RULES(GRAPHICS_VERSION_RANGE(1270, 1274), ENGINE_CLASS(RENDER)),652 XE_RTP_ACTIONS(SET(XEHP_PSS_CHICKEN, FD_END_COLLECT))653 },654 655 /* Xe2_LPG */656 657 { XE_RTP_NAME("16020518922"),658 XE_RTP_RULES(GRAPHICS_VERSION(2004), GRAPHICS_STEP(A0, B0),659 ENGINE_CLASS(RENDER)),660 XE_RTP_ACTIONS(SET(FF_MODE,661 DIS_TE_AUTOSTRIP |662 DIS_MESH_PARTIAL_AUTOSTRIP |663 DIS_MESH_AUTOSTRIP),664 SET(VFLSKPD,665 DIS_PARTIAL_AUTOSTRIP |666 DIS_AUTOSTRIP))667 },668 { XE_RTP_NAME("14019386621"),669 XE_RTP_RULES(GRAPHICS_VERSION(2004), ENGINE_CLASS(RENDER)),670 XE_RTP_ACTIONS(SET(VF_SCRATCHPAD, XE2_VFG_TED_CREDIT_INTERFACE_DISABLE))671 },672 { XE_RTP_NAME("14019877138"),673 XE_RTP_RULES(GRAPHICS_VERSION(2004), ENGINE_CLASS(RENDER)),674 XE_RTP_ACTIONS(SET(XEHP_PSS_CHICKEN, FD_END_COLLECT))675 },676 { XE_RTP_NAME("14020013138"),677 XE_RTP_RULES(GRAPHICS_VERSION(2004), GRAPHICS_STEP(A0, B0),678 ENGINE_CLASS(RENDER)),679 XE_RTP_ACTIONS(SET(WM_CHICKEN3, HIZ_PLANE_COMPRESSION_DIS))680 },681 { XE_RTP_NAME("14019988906"),682 XE_RTP_RULES(GRAPHICS_VERSION(2004), ENGINE_CLASS(RENDER)),683 XE_RTP_ACTIONS(SET(XEHP_PSS_CHICKEN, FLSH_IGNORES_PSD))684 },685 { XE_RTP_NAME("16020183090"),686 XE_RTP_RULES(GRAPHICS_VERSION(2004), GRAPHICS_STEP(A0, B0),687 ENGINE_CLASS(RENDER)),688 XE_RTP_ACTIONS(SET(INSTPM(RENDER_RING_BASE), ENABLE_SEMAPHORE_POLL_BIT))689 },690 { XE_RTP_NAME("18033852989"),691 XE_RTP_RULES(GRAPHICS_VERSION_RANGE(2001, 2004), ENGINE_CLASS(RENDER)),692 XE_RTP_ACTIONS(SET(COMMON_SLICE_CHICKEN1, DISABLE_BOTTOM_CLIP_RECTANGLE_TEST))693 },694 { XE_RTP_NAME("14021567978"),695 XE_RTP_RULES(GRAPHICS_VERSION_RANGE(2001, XE_RTP_END_VERSION_UNDEFINED),696 ENGINE_CLASS(RENDER)),697 XE_RTP_ACTIONS(SET(CHICKEN_RASTER_2, TBIMR_FAST_CLIP))698 },699 { XE_RTP_NAME("14020756599"),700 XE_RTP_RULES(GRAPHICS_VERSION(2004), ENGINE_CLASS(RENDER), OR,701 MEDIA_VERSION_ANY_GT(2000), ENGINE_CLASS(RENDER)),702 XE_RTP_ACTIONS(SET(WM_CHICKEN3, HIZ_PLANE_COMPRESSION_DIS))703 },704 { XE_RTP_NAME("14021490052"),705 XE_RTP_RULES(GRAPHICS_VERSION(2004), ENGINE_CLASS(RENDER)),706 XE_RTP_ACTIONS(SET(FF_MODE,707 DIS_MESH_PARTIAL_AUTOSTRIP |708 DIS_MESH_AUTOSTRIP),709 SET(VFLSKPD,710 DIS_PARTIAL_AUTOSTRIP |711 DIS_AUTOSTRIP))712 },713 { XE_RTP_NAME("15016589081"),714 XE_RTP_RULES(GRAPHICS_VERSION(2004), ENGINE_CLASS(RENDER)),715 XE_RTP_ACTIONS(SET(CHICKEN_RASTER_1, DIS_CLIP_NEGATIVE_BOUNDING_BOX))716 },717 718 /* Xe2_HPG */719 { XE_RTP_NAME("15010599737"),720 XE_RTP_RULES(GRAPHICS_VERSION(2001), ENGINE_CLASS(RENDER)),721 XE_RTP_ACTIONS(SET(CHICKEN_RASTER_1, DIS_SF_ROUND_NEAREST_EVEN))722 },723 { XE_RTP_NAME("14019386621"),724 XE_RTP_RULES(GRAPHICS_VERSION(2001), ENGINE_CLASS(RENDER)),725 XE_RTP_ACTIONS(SET(VF_SCRATCHPAD, XE2_VFG_TED_CREDIT_INTERFACE_DISABLE))726 },727 { XE_RTP_NAME("14020756599"),728 XE_RTP_RULES(GRAPHICS_VERSION(2001), ENGINE_CLASS(RENDER)),729 XE_RTP_ACTIONS(SET(WM_CHICKEN3, HIZ_PLANE_COMPRESSION_DIS))730 },731 { XE_RTP_NAME("14021490052"),732 XE_RTP_RULES(GRAPHICS_VERSION(2001), ENGINE_CLASS(RENDER)),733 XE_RTP_ACTIONS(SET(FF_MODE,734 DIS_MESH_PARTIAL_AUTOSTRIP |735 DIS_MESH_AUTOSTRIP),736 SET(VFLSKPD,737 DIS_PARTIAL_AUTOSTRIP |738 DIS_AUTOSTRIP))739 },740 { XE_RTP_NAME("15016589081"),741 XE_RTP_RULES(GRAPHICS_VERSION(2001), ENGINE_CLASS(RENDER)),742 XE_RTP_ACTIONS(SET(CHICKEN_RASTER_1, DIS_CLIP_NEGATIVE_BOUNDING_BOX))743 },744 745 {}746};747 748static __maybe_unused const struct xe_rtp_entry oob_was[] = {749#include <generated/xe_wa_oob.c>750 {}751};752 753static_assert(ARRAY_SIZE(oob_was) - 1 == _XE_WA_OOB_COUNT);754 755__diag_pop();756 757/**758 * xe_wa_process_oob - process OOB workaround table759 * @gt: GT instance to process workarounds for760 *761 * Process OOB workaround table for this platform, marking in @gt the762 * workarounds that are active.763 */764void xe_wa_process_oob(struct xe_gt *gt)765{766 struct xe_rtp_process_ctx ctx = XE_RTP_PROCESS_CTX_INITIALIZER(gt);767 768 xe_rtp_process_ctx_enable_active_tracking(&ctx, gt->wa_active.oob,769 ARRAY_SIZE(oob_was));770 gt->wa_active.oob_initialized = true;771 xe_rtp_process(&ctx, oob_was);772}773 774/**775 * xe_wa_process_gt - process GT workaround table776 * @gt: GT instance to process workarounds for777 *778 * Process GT workaround table for this platform, saving in @gt all the779 * workarounds that need to be applied at the GT level.780 */781void xe_wa_process_gt(struct xe_gt *gt)782{783 struct xe_rtp_process_ctx ctx = XE_RTP_PROCESS_CTX_INITIALIZER(gt);784 785 xe_rtp_process_ctx_enable_active_tracking(&ctx, gt->wa_active.gt,786 ARRAY_SIZE(gt_was));787 xe_rtp_process_to_sr(&ctx, gt_was, >->reg_sr);788}789EXPORT_SYMBOL_IF_KUNIT(xe_wa_process_gt);790 791/**792 * xe_wa_process_engine - process engine workaround table793 * @hwe: engine instance to process workarounds for794 *795 * Process engine workaround table for this platform, saving in @hwe all the796 * workarounds that need to be applied at the engine level that match this797 * engine.798 */799void xe_wa_process_engine(struct xe_hw_engine *hwe)800{801 struct xe_rtp_process_ctx ctx = XE_RTP_PROCESS_CTX_INITIALIZER(hwe);802 803 xe_rtp_process_ctx_enable_active_tracking(&ctx, hwe->gt->wa_active.engine,804 ARRAY_SIZE(engine_was));805 xe_rtp_process_to_sr(&ctx, engine_was, &hwe->reg_sr);806}807 808/**809 * xe_wa_process_lrc - process context workaround table810 * @hwe: engine instance to process workarounds for811 *812 * Process context workaround table for this platform, saving in @hwe all the813 * workarounds that need to be applied on context restore. These are workarounds814 * touching registers that are part of the HW context image.815 */816void xe_wa_process_lrc(struct xe_hw_engine *hwe)817{818 struct xe_rtp_process_ctx ctx = XE_RTP_PROCESS_CTX_INITIALIZER(hwe);819 820 xe_rtp_process_ctx_enable_active_tracking(&ctx, hwe->gt->wa_active.lrc,821 ARRAY_SIZE(lrc_was));822 xe_rtp_process_to_sr(&ctx, lrc_was, &hwe->reg_lrc);823}824 825/**826 * xe_wa_init - initialize gt with workaround bookkeeping827 * @gt: GT instance to initialize828 *829 * Returns 0 for success, negative error code otherwise.830 */831int xe_wa_init(struct xe_gt *gt)832{833 struct xe_device *xe = gt_to_xe(gt);834 size_t n_oob, n_lrc, n_engine, n_gt, total;835 unsigned long *p;836 837 n_gt = BITS_TO_LONGS(ARRAY_SIZE(gt_was));838 n_engine = BITS_TO_LONGS(ARRAY_SIZE(engine_was));839 n_lrc = BITS_TO_LONGS(ARRAY_SIZE(lrc_was));840 n_oob = BITS_TO_LONGS(ARRAY_SIZE(oob_was));841 total = n_gt + n_engine + n_lrc + n_oob;842 843 p = drmm_kzalloc(&xe->drm, sizeof(*p) * total, GFP_KERNEL);844 if (!p)845 return -ENOMEM;846 847 gt->wa_active.gt = p;848 p += n_gt;849 gt->wa_active.engine = p;850 p += n_engine;851 gt->wa_active.lrc = p;852 p += n_lrc;853 gt->wa_active.oob = p;854 855 return 0;856}857 858void xe_wa_dump(struct xe_gt *gt, struct drm_printer *p)859{860 size_t idx;861 862 drm_printf(p, "GT Workarounds\n");863 for_each_set_bit(idx, gt->wa_active.gt, ARRAY_SIZE(gt_was))864 drm_printf_indent(p, 1, "%s\n", gt_was[idx].name);865 866 drm_printf(p, "\nEngine Workarounds\n");867 for_each_set_bit(idx, gt->wa_active.engine, ARRAY_SIZE(engine_was))868 drm_printf_indent(p, 1, "%s\n", engine_was[idx].name);869 870 drm_printf(p, "\nLRC Workarounds\n");871 for_each_set_bit(idx, gt->wa_active.lrc, ARRAY_SIZE(lrc_was))872 drm_printf_indent(p, 1, "%s\n", lrc_was[idx].name);873 874 drm_printf(p, "\nOOB Workarounds\n");875 for_each_set_bit(idx, gt->wa_active.oob, ARRAY_SIZE(oob_was))876 if (oob_was[idx].name)877 drm_printf_indent(p, 1, "%s\n", oob_was[idx].name);878}879 880/*881 * Apply tile (non-GT, non-display) workarounds. Think very carefully before882 * adding anything to this function; most workarounds should be implemented883 * elsewhere. The programming here is primarily for sgunit/soc workarounds,884 * which are relatively rare. Since the registers these workarounds target are885 * outside the GT, they should only need to be applied once at device886 * probe/resume; they will not lose their values on any kind of GT or engine887 * reset.888 *889 * TODO: We may want to move this over to xe_rtp in the future once we have890 * enough workarounds to justify the work.891 */892void xe_wa_apply_tile_workarounds(struct xe_tile *tile)893{894 struct xe_gt *mmio = tile->primary_gt;895 896 if (IS_SRIOV_VF(tile->xe))897 return;898 899 if (XE_WA(mmio, 22010954014))900 xe_mmio_rmw32(mmio, XEHP_CLOCK_GATE_DIS, 0, SGSI_SIDECLK_DIS);901}902