510 lines · c
1// SPDX-License-Identifier: MIT2/*3 * Copyright © 2022 Intel Corporation4 */5 6#include "xe_ring_ops.h"7 8#include <generated/xe_wa_oob.h>9 10#include "instructions/xe_gpu_commands.h"11#include "instructions/xe_mi_commands.h"12#include "regs/xe_engine_regs.h"13#include "regs/xe_gt_regs.h"14#include "regs/xe_lrc_layout.h"15#include "xe_exec_queue_types.h"16#include "xe_gt.h"17#include "xe_lrc.h"18#include "xe_macros.h"19#include "xe_sched_job.h"20#include "xe_sriov.h"21#include "xe_vm_types.h"22#include "xe_vm.h"23#include "xe_wa.h"24 25/*26 * 3D-related flags that can't be set on _engines_ that lack access to the 3D27 * pipeline (i.e., CCS engines).28 */29#define PIPE_CONTROL_3D_ENGINE_FLAGS (\30 PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH | \31 PIPE_CONTROL_DEPTH_CACHE_FLUSH | \32 PIPE_CONTROL_TILE_CACHE_FLUSH | \33 PIPE_CONTROL_DEPTH_STALL | \34 PIPE_CONTROL_STALL_AT_SCOREBOARD | \35 PIPE_CONTROL_PSD_SYNC | \36 PIPE_CONTROL_AMFS_FLUSH | \37 PIPE_CONTROL_VF_CACHE_INVALIDATE | \38 PIPE_CONTROL_GLOBAL_SNAPSHOT_RESET)39 40/* 3D-related flags that can't be set on _platforms_ that lack a 3D pipeline */41#define PIPE_CONTROL_3D_ARCH_FLAGS ( \42 PIPE_CONTROL_3D_ENGINE_FLAGS | \43 PIPE_CONTROL_INDIRECT_STATE_DISABLE | \44 PIPE_CONTROL_FLUSH_ENABLE | \45 PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE | \46 PIPE_CONTROL_DC_FLUSH_ENABLE)47 48static u32 preparser_disable(bool state)49{50 return MI_ARB_CHECK | BIT(8) | state;51}52 53static int emit_aux_table_inv(struct xe_gt *gt, struct xe_reg reg,54 u32 *dw, int i)55{56 dw[i++] = MI_LOAD_REGISTER_IMM | MI_LRI_NUM_REGS(1) | MI_LRI_MMIO_REMAP_EN;57 dw[i++] = reg.addr + gt->mmio.adj_offset;58 dw[i++] = AUX_INV;59 dw[i++] = MI_NOOP;60 61 return i;62}63 64static int emit_user_interrupt(u32 *dw, int i)65{66 dw[i++] = MI_USER_INTERRUPT;67 dw[i++] = MI_ARB_ON_OFF | MI_ARB_ENABLE;68 dw[i++] = MI_ARB_CHECK;69 70 return i;71}72 73static int emit_store_imm_ggtt(u32 addr, u32 value, u32 *dw, int i)74{75 dw[i++] = MI_STORE_DATA_IMM | MI_SDI_GGTT | MI_SDI_NUM_DW(1);76 dw[i++] = addr;77 dw[i++] = 0;78 dw[i++] = value;79 80 return i;81}82 83static int emit_flush_dw(u32 *dw, int i)84{85 dw[i++] = MI_FLUSH_DW | MI_FLUSH_IMM_DW;86 dw[i++] = 0;87 dw[i++] = 0;88 dw[i++] = 0;89 90 return i;91}92 93static int emit_flush_imm_ggtt(u32 addr, u32 value, bool invalidate_tlb,94 u32 *dw, int i)95{96 dw[i++] = MI_FLUSH_DW | MI_FLUSH_DW_OP_STOREDW | MI_FLUSH_IMM_DW |97 (invalidate_tlb ? MI_INVALIDATE_TLB : 0);98 dw[i++] = addr | MI_FLUSH_DW_USE_GTT;99 dw[i++] = 0;100 dw[i++] = value;101 102 return i;103}104 105static int emit_bb_start(u64 batch_addr, u32 ppgtt_flag, u32 *dw, int i)106{107 dw[i++] = MI_BATCH_BUFFER_START | ppgtt_flag | XE_INSTR_NUM_DW(3);108 dw[i++] = lower_32_bits(batch_addr);109 dw[i++] = upper_32_bits(batch_addr);110 111 return i;112}113 114static int emit_flush_invalidate(u32 flag, u32 *dw, int i)115{116 dw[i] = MI_FLUSH_DW;117 dw[i] |= flag;118 dw[i++] |= MI_INVALIDATE_TLB | MI_FLUSH_DW_OP_STOREDW | MI_FLUSH_IMM_DW |119 MI_FLUSH_DW_STORE_INDEX;120 121 dw[i++] = LRC_PPHWSP_SCRATCH_ADDR | MI_FLUSH_DW_USE_GTT;122 dw[i++] = 0;123 dw[i++] = ~0U;124 125 return i;126}127 128static int129emit_pipe_control(u32 *dw, int i, u32 bit_group_0, u32 bit_group_1, u32 offset, u32 value)130{131 dw[i++] = GFX_OP_PIPE_CONTROL(6) | bit_group_0;132 dw[i++] = bit_group_1;133 dw[i++] = offset;134 dw[i++] = 0;135 dw[i++] = value;136 dw[i++] = 0;137 138 return i;139}140 141static int emit_pipe_invalidate(u32 mask_flags, bool invalidate_tlb, u32 *dw,142 int i)143{144 u32 flags = PIPE_CONTROL_CS_STALL |145 PIPE_CONTROL_COMMAND_CACHE_INVALIDATE |146 PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE |147 PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE |148 PIPE_CONTROL_VF_CACHE_INVALIDATE |149 PIPE_CONTROL_CONST_CACHE_INVALIDATE |150 PIPE_CONTROL_STATE_CACHE_INVALIDATE |151 PIPE_CONTROL_QW_WRITE |152 PIPE_CONTROL_STORE_DATA_INDEX;153 154 if (invalidate_tlb)155 flags |= PIPE_CONTROL_TLB_INVALIDATE;156 157 flags &= ~mask_flags;158 159 return emit_pipe_control(dw, i, 0, flags, LRC_PPHWSP_SCRATCH_ADDR, 0);160}161 162static int emit_store_imm_ppgtt_posted(u64 addr, u64 value,163 u32 *dw, int i)164{165 dw[i++] = MI_STORE_DATA_IMM | MI_SDI_NUM_QW(1);166 dw[i++] = lower_32_bits(addr);167 dw[i++] = upper_32_bits(addr);168 dw[i++] = lower_32_bits(value);169 dw[i++] = upper_32_bits(value);170 171 return i;172}173 174static int emit_render_cache_flush(struct xe_sched_job *job, u32 *dw, int i)175{176 struct xe_gt *gt = job->q->gt;177 bool lacks_render = !(gt->info.engine_mask & XE_HW_ENGINE_RCS_MASK);178 u32 flags;179 180 flags = (PIPE_CONTROL_CS_STALL |181 PIPE_CONTROL_TILE_CACHE_FLUSH |182 PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH |183 PIPE_CONTROL_DEPTH_CACHE_FLUSH |184 PIPE_CONTROL_DC_FLUSH_ENABLE |185 PIPE_CONTROL_FLUSH_ENABLE);186 187 if (XE_WA(gt, 1409600907))188 flags |= PIPE_CONTROL_DEPTH_STALL;189 190 if (lacks_render)191 flags &= ~PIPE_CONTROL_3D_ARCH_FLAGS;192 else if (job->q->class == XE_ENGINE_CLASS_COMPUTE)193 flags &= ~PIPE_CONTROL_3D_ENGINE_FLAGS;194 195 return emit_pipe_control(dw, i, PIPE_CONTROL0_HDC_PIPELINE_FLUSH, flags, 0, 0);196}197 198static int emit_pipe_control_to_ring_end(struct xe_hw_engine *hwe, u32 *dw, int i)199{200 if (hwe->class != XE_ENGINE_CLASS_RENDER)201 return i;202 203 if (XE_WA(hwe->gt, 16020292621))204 i = emit_pipe_control(dw, i, 0, PIPE_CONTROL_LRI_POST_SYNC,205 RING_NOPID(hwe->mmio_base).addr, 0);206 207 return i;208}209 210static int emit_pipe_imm_ggtt(u32 addr, u32 value, bool stall_only, u32 *dw,211 int i)212{213 u32 flags = PIPE_CONTROL_CS_STALL | PIPE_CONTROL_GLOBAL_GTT_IVB |214 PIPE_CONTROL_QW_WRITE;215 216 if (!stall_only)217 flags |= PIPE_CONTROL_FLUSH_ENABLE;218 219 return emit_pipe_control(dw, i, 0, flags, addr, value);220}221 222static u32 get_ppgtt_flag(struct xe_sched_job *job)223{224 return job->q->vm ? BIT(8) : 0;225}226 227static int emit_copy_timestamp(struct xe_lrc *lrc, u32 *dw, int i)228{229 dw[i++] = MI_COPY_MEM_MEM | MI_COPY_MEM_MEM_SRC_GGTT |230 MI_COPY_MEM_MEM_DST_GGTT;231 dw[i++] = xe_lrc_ctx_job_timestamp_ggtt_addr(lrc);232 dw[i++] = 0;233 dw[i++] = xe_lrc_ctx_timestamp_ggtt_addr(lrc);234 dw[i++] = 0;235 dw[i++] = MI_NOOP;236 237 return i;238}239 240/* for engines that don't require any special HW handling (no EUs, no aux inval, etc) */241static void __emit_job_gen12_simple(struct xe_sched_job *job, struct xe_lrc *lrc,242 u64 batch_addr, u32 seqno)243{244 u32 dw[MAX_JOB_SIZE_DW], i = 0;245 u32 ppgtt_flag = get_ppgtt_flag(job);246 struct xe_gt *gt = job->q->gt;247 248 i = emit_copy_timestamp(lrc, dw, i);249 250 if (job->ring_ops_flush_tlb) {251 dw[i++] = preparser_disable(true);252 i = emit_flush_imm_ggtt(xe_lrc_start_seqno_ggtt_addr(lrc),253 seqno, true, dw, i);254 dw[i++] = preparser_disable(false);255 } else {256 i = emit_store_imm_ggtt(xe_lrc_start_seqno_ggtt_addr(lrc),257 seqno, dw, i);258 }259 260 i = emit_bb_start(batch_addr, ppgtt_flag, dw, i);261 262 if (job->user_fence.used) {263 i = emit_flush_dw(dw, i);264 i = emit_store_imm_ppgtt_posted(job->user_fence.addr,265 job->user_fence.value,266 dw, i);267 }268 269 i = emit_flush_imm_ggtt(xe_lrc_seqno_ggtt_addr(lrc), seqno, false, dw, i);270 271 i = emit_user_interrupt(dw, i);272 273 xe_gt_assert(gt, i <= MAX_JOB_SIZE_DW);274 275 xe_lrc_write_ring(lrc, dw, i * sizeof(*dw));276}277 278static bool has_aux_ccs(struct xe_device *xe)279{280 /*281 * PVC is a special case that has no compression of either type282 * (FlatCCS or AuxCCS). Also, AuxCCS is no longer used from Xe2283 * onward, so any future platforms with no FlatCCS will not have284 * AuxCCS either.285 */286 if (GRAPHICS_VER(xe) >= 20 || xe->info.platform == XE_PVC)287 return false;288 289 return !xe->info.has_flat_ccs;290}291 292static void __emit_job_gen12_video(struct xe_sched_job *job, struct xe_lrc *lrc,293 u64 batch_addr, u32 seqno)294{295 u32 dw[MAX_JOB_SIZE_DW], i = 0;296 u32 ppgtt_flag = get_ppgtt_flag(job);297 struct xe_gt *gt = job->q->gt;298 struct xe_device *xe = gt_to_xe(gt);299 bool decode = job->q->class == XE_ENGINE_CLASS_VIDEO_DECODE;300 301 i = emit_copy_timestamp(lrc, dw, i);302 303 dw[i++] = preparser_disable(true);304 305 /* hsdes: 1809175790 */306 if (has_aux_ccs(xe)) {307 if (decode)308 i = emit_aux_table_inv(gt, VD0_AUX_INV, dw, i);309 else310 i = emit_aux_table_inv(gt, VE0_AUX_INV, dw, i);311 }312 313 if (job->ring_ops_flush_tlb)314 i = emit_flush_imm_ggtt(xe_lrc_start_seqno_ggtt_addr(lrc),315 seqno, true, dw, i);316 317 dw[i++] = preparser_disable(false);318 319 if (!job->ring_ops_flush_tlb)320 i = emit_store_imm_ggtt(xe_lrc_start_seqno_ggtt_addr(lrc),321 seqno, dw, i);322 323 i = emit_bb_start(batch_addr, ppgtt_flag, dw, i);324 325 if (job->user_fence.used) {326 i = emit_flush_dw(dw, i);327 i = emit_store_imm_ppgtt_posted(job->user_fence.addr,328 job->user_fence.value,329 dw, i);330 }331 332 i = emit_flush_imm_ggtt(xe_lrc_seqno_ggtt_addr(lrc), seqno, false, dw, i);333 334 i = emit_user_interrupt(dw, i);335 336 xe_gt_assert(gt, i <= MAX_JOB_SIZE_DW);337 338 xe_lrc_write_ring(lrc, dw, i * sizeof(*dw));339}340 341static void __emit_job_gen12_render_compute(struct xe_sched_job *job,342 struct xe_lrc *lrc,343 u64 batch_addr, u32 seqno)344{345 u32 dw[MAX_JOB_SIZE_DW], i = 0;346 u32 ppgtt_flag = get_ppgtt_flag(job);347 struct xe_gt *gt = job->q->gt;348 struct xe_device *xe = gt_to_xe(gt);349 bool lacks_render = !(gt->info.engine_mask & XE_HW_ENGINE_RCS_MASK);350 u32 mask_flags = 0;351 352 i = emit_copy_timestamp(lrc, dw, i);353 354 dw[i++] = preparser_disable(true);355 if (lacks_render)356 mask_flags = PIPE_CONTROL_3D_ARCH_FLAGS;357 else if (job->q->class == XE_ENGINE_CLASS_COMPUTE)358 mask_flags = PIPE_CONTROL_3D_ENGINE_FLAGS;359 360 /* See __xe_pt_bind_vma() for a discussion on TLB invalidations. */361 i = emit_pipe_invalidate(mask_flags, job->ring_ops_flush_tlb, dw, i);362 363 /* hsdes: 1809175790 */364 if (has_aux_ccs(xe))365 i = emit_aux_table_inv(gt, CCS_AUX_INV, dw, i);366 367 dw[i++] = preparser_disable(false);368 369 i = emit_store_imm_ggtt(xe_lrc_start_seqno_ggtt_addr(lrc),370 seqno, dw, i);371 372 i = emit_bb_start(batch_addr, ppgtt_flag, dw, i);373 374 i = emit_render_cache_flush(job, dw, i);375 376 if (job->user_fence.used)377 i = emit_store_imm_ppgtt_posted(job->user_fence.addr,378 job->user_fence.value,379 dw, i);380 381 i = emit_pipe_imm_ggtt(xe_lrc_seqno_ggtt_addr(lrc), seqno, lacks_render, dw, i);382 383 i = emit_user_interrupt(dw, i);384 385 i = emit_pipe_control_to_ring_end(job->q->hwe, dw, i);386 387 xe_gt_assert(gt, i <= MAX_JOB_SIZE_DW);388 389 xe_lrc_write_ring(lrc, dw, i * sizeof(*dw));390}391 392static void emit_migration_job_gen12(struct xe_sched_job *job,393 struct xe_lrc *lrc, u32 seqno)394{395 u32 dw[MAX_JOB_SIZE_DW], i = 0;396 397 i = emit_copy_timestamp(lrc, dw, i);398 399 i = emit_store_imm_ggtt(xe_lrc_start_seqno_ggtt_addr(lrc),400 seqno, dw, i);401 402 dw[i++] = MI_ARB_ON_OFF | MI_ARB_DISABLE; /* Enabled again below */403 404 i = emit_bb_start(job->ptrs[0].batch_addr, BIT(8), dw, i);405 406 if (!IS_SRIOV_VF(gt_to_xe(job->q->gt))) {407 /* XXX: Do we need this? Leaving for now. */408 dw[i++] = preparser_disable(true);409 i = emit_flush_invalidate(0, dw, i);410 dw[i++] = preparser_disable(false);411 }412 413 i = emit_bb_start(job->ptrs[1].batch_addr, BIT(8), dw, i);414 415 dw[i++] = MI_FLUSH_DW | MI_INVALIDATE_TLB | job->migrate_flush_flags |416 MI_FLUSH_DW_OP_STOREDW | MI_FLUSH_IMM_DW;417 dw[i++] = xe_lrc_seqno_ggtt_addr(lrc) | MI_FLUSH_DW_USE_GTT;418 dw[i++] = 0;419 dw[i++] = seqno; /* value */420 421 i = emit_user_interrupt(dw, i);422 423 xe_gt_assert(job->q->gt, i <= MAX_JOB_SIZE_DW);424 425 xe_lrc_write_ring(lrc, dw, i * sizeof(*dw));426}427 428static void emit_job_gen12_gsc(struct xe_sched_job *job)429{430 struct xe_gt *gt = job->q->gt;431 432 xe_gt_assert(gt, job->q->width <= 1); /* no parallel submission for GSCCS */433 434 __emit_job_gen12_simple(job, job->q->lrc[0],435 job->ptrs[0].batch_addr,436 xe_sched_job_lrc_seqno(job));437}438 439static void emit_job_gen12_copy(struct xe_sched_job *job)440{441 int i;442 443 if (xe_sched_job_is_migration(job->q)) {444 emit_migration_job_gen12(job, job->q->lrc[0],445 xe_sched_job_lrc_seqno(job));446 return;447 }448 449 for (i = 0; i < job->q->width; ++i)450 __emit_job_gen12_simple(job, job->q->lrc[i],451 job->ptrs[i].batch_addr,452 xe_sched_job_lrc_seqno(job));453}454 455static void emit_job_gen12_video(struct xe_sched_job *job)456{457 int i;458 459 /* FIXME: Not doing parallel handshake for now */460 for (i = 0; i < job->q->width; ++i)461 __emit_job_gen12_video(job, job->q->lrc[i],462 job->ptrs[i].batch_addr,463 xe_sched_job_lrc_seqno(job));464}465 466static void emit_job_gen12_render_compute(struct xe_sched_job *job)467{468 int i;469 470 for (i = 0; i < job->q->width; ++i)471 __emit_job_gen12_render_compute(job, job->q->lrc[i],472 job->ptrs[i].batch_addr,473 xe_sched_job_lrc_seqno(job));474}475 476static const struct xe_ring_ops ring_ops_gen12_gsc = {477 .emit_job = emit_job_gen12_gsc,478};479 480static const struct xe_ring_ops ring_ops_gen12_copy = {481 .emit_job = emit_job_gen12_copy,482};483 484static const struct xe_ring_ops ring_ops_gen12_video = {485 .emit_job = emit_job_gen12_video,486};487 488static const struct xe_ring_ops ring_ops_gen12_render_compute = {489 .emit_job = emit_job_gen12_render_compute,490};491 492const struct xe_ring_ops *493xe_ring_ops_get(struct xe_gt *gt, enum xe_engine_class class)494{495 switch (class) {496 case XE_ENGINE_CLASS_OTHER:497 return &ring_ops_gen12_gsc;498 case XE_ENGINE_CLASS_COPY:499 return &ring_ops_gen12_copy;500 case XE_ENGINE_CLASS_VIDEO_DECODE:501 case XE_ENGINE_CLASS_VIDEO_ENHANCE:502 return &ring_ops_gen12_video;503 case XE_ENGINE_CLASS_RENDER:504 case XE_ENGINE_CLASS_COMPUTE:505 return &ring_ops_gen12_render_compute;506 default:507 return NULL;508 }509}510