1548 lines · c
1// SPDX-License-Identifier: MIT2/*3 * Copyright © 2020 Intel Corporation4 */5 6#include "xe_migrate.h"7 8#include <linux/bitfield.h>9#include <linux/sizes.h>10 11#include <drm/drm_managed.h>12#include <drm/ttm/ttm_tt.h>13#include <uapi/drm/xe_drm.h>14 15#include <generated/xe_wa_oob.h>16 17#include "instructions/xe_gpu_commands.h"18#include "instructions/xe_mi_commands.h"19#include "regs/xe_gtt_defs.h"20#include "tests/xe_test.h"21#include "xe_assert.h"22#include "xe_bb.h"23#include "xe_bo.h"24#include "xe_exec_queue.h"25#include "xe_ggtt.h"26#include "xe_gt.h"27#include "xe_hw_engine.h"28#include "xe_lrc.h"29#include "xe_map.h"30#include "xe_mocs.h"31#include "xe_pt.h"32#include "xe_res_cursor.h"33#include "xe_sched_job.h"34#include "xe_sync.h"35#include "xe_trace_bo.h"36#include "xe_vm.h"37 38/**39 * struct xe_migrate - migrate context.40 */41struct xe_migrate {42 /** @q: Default exec queue used for migration */43 struct xe_exec_queue *q;44 /** @tile: Backpointer to the tile this struct xe_migrate belongs to. */45 struct xe_tile *tile;46 /** @job_mutex: Timeline mutex for @eng. */47 struct mutex job_mutex;48 /** @pt_bo: Page-table buffer object. */49 struct xe_bo *pt_bo;50 /** @batch_base_ofs: VM offset of the migration batch buffer */51 u64 batch_base_ofs;52 /** @usm_batch_base_ofs: VM offset of the usm batch buffer */53 u64 usm_batch_base_ofs;54 /** @cleared_mem_ofs: VM offset of @cleared_bo. */55 u64 cleared_mem_ofs;56 /**57 * @fence: dma-fence representing the last migration job batch.58 * Protected by @job_mutex.59 */60 struct dma_fence *fence;61 /**62 * @vm_update_sa: For integrated, used to suballocate page-tables63 * out of the pt_bo.64 */65 struct drm_suballoc_manager vm_update_sa;66 /** @min_chunk_size: For dgfx, Minimum chunk size */67 u64 min_chunk_size;68};69 70#define MAX_PREEMPTDISABLE_TRANSFER SZ_8M /* Around 1ms. */71#define MAX_CCS_LIMITED_TRANSFER SZ_4M /* XE_PAGE_SIZE * (FIELD_MAX(XE2_CCS_SIZE_MASK) + 1) */72#define NUM_KERNEL_PDE 1573#define NUM_PT_SLOTS 3274#define LEVEL0_PAGE_TABLE_ENCODE_SIZE SZ_2M75#define MAX_NUM_PTE 51276#define IDENTITY_OFFSET 256ULL77 78/*79 * Although MI_STORE_DATA_IMM's "length" field is 10-bits, 0x3FE is the largest80 * legal value accepted. Since that instruction field is always stored in81 * (val-2) format, this translates to 0x400 dwords for the true maximum length82 * of the instruction. Subtracting the instruction header (1 dword) and83 * address (2 dwords), that leaves 0x3FD dwords (0x1FE qwords) for PTE values.84 */85#define MAX_PTE_PER_SDI 0x1FE86 87/**88 * xe_tile_migrate_exec_queue() - Get this tile's migrate exec queue.89 * @tile: The tile.90 *91 * Returns the default migrate exec queue of this tile.92 *93 * Return: The default migrate exec queue94 */95struct xe_exec_queue *xe_tile_migrate_exec_queue(struct xe_tile *tile)96{97 return tile->migrate->q;98}99 100static void xe_migrate_fini(struct drm_device *dev, void *arg)101{102 struct xe_migrate *m = arg;103 104 xe_vm_lock(m->q->vm, false);105 xe_bo_unpin(m->pt_bo);106 xe_vm_unlock(m->q->vm);107 108 dma_fence_put(m->fence);109 xe_bo_put(m->pt_bo);110 drm_suballoc_manager_fini(&m->vm_update_sa);111 mutex_destroy(&m->job_mutex);112 xe_vm_close_and_put(m->q->vm);113 xe_exec_queue_put(m->q);114}115 116static u64 xe_migrate_vm_addr(u64 slot, u32 level)117{118 XE_WARN_ON(slot >= NUM_PT_SLOTS);119 120 /* First slot is reserved for mapping of PT bo and bb, start from 1 */121 return (slot + 1ULL) << xe_pt_shift(level + 1);122}123 124static u64 xe_migrate_vram_ofs(struct xe_device *xe, u64 addr, bool is_comp_pte)125{126 /*127 * Remove the DPA to get a correct offset into identity table for the128 * migrate offset129 */130 u64 identity_offset = IDENTITY_OFFSET;131 132 if (GRAPHICS_VER(xe) >= 20 && is_comp_pte)133 identity_offset += DIV_ROUND_UP_ULL(xe->mem.vram.actual_physical_size, SZ_1G);134 135 addr -= xe->mem.vram.dpa_base;136 return addr + (identity_offset << xe_pt_shift(2));137}138 139static void xe_migrate_program_identity(struct xe_device *xe, struct xe_vm *vm, struct xe_bo *bo,140 u64 map_ofs, u64 vram_offset, u16 pat_index, u64 pt_2m_ofs)141{142 u64 pos, ofs, flags;143 u64 entry;144 /* XXX: Unclear if this should be usable_size? */145 u64 vram_limit = xe->mem.vram.actual_physical_size +146 xe->mem.vram.dpa_base;147 u32 level = 2;148 149 ofs = map_ofs + XE_PAGE_SIZE * level + vram_offset * 8;150 flags = vm->pt_ops->pte_encode_addr(xe, 0, pat_index, level,151 true, 0);152 153 xe_assert(xe, IS_ALIGNED(xe->mem.vram.usable_size, SZ_2M));154 155 /*156 * Use 1GB pages when possible, last chunk always use 2M157 * pages as mixing reserved memory (stolen, WOCPM) with a single158 * mapping is not allowed on certain platforms.159 */160 for (pos = xe->mem.vram.dpa_base; pos < vram_limit;161 pos += SZ_1G, ofs += 8) {162 if (pos + SZ_1G >= vram_limit) {163 entry = vm->pt_ops->pde_encode_bo(bo, pt_2m_ofs,164 pat_index);165 xe_map_wr(xe, &bo->vmap, ofs, u64, entry);166 167 flags = vm->pt_ops->pte_encode_addr(xe, 0,168 pat_index,169 level - 1,170 true, 0);171 172 for (ofs = pt_2m_ofs; pos < vram_limit;173 pos += SZ_2M, ofs += 8)174 xe_map_wr(xe, &bo->vmap, ofs, u64, pos | flags);175 break; /* Ensure pos == vram_limit assert correct */176 }177 178 xe_map_wr(xe, &bo->vmap, ofs, u64, pos | flags);179 }180 181 xe_assert(xe, pos == vram_limit);182}183 184static int xe_migrate_prepare_vm(struct xe_tile *tile, struct xe_migrate *m,185 struct xe_vm *vm)186{187 struct xe_device *xe = tile_to_xe(tile);188 u16 pat_index = xe->pat.idx[XE_CACHE_WB];189 u8 id = tile->id;190 u32 num_entries = NUM_PT_SLOTS, num_level = vm->pt_root[id]->level;191#define VRAM_IDENTITY_MAP_COUNT 2192 u32 num_setup = num_level + VRAM_IDENTITY_MAP_COUNT;193#undef VRAM_IDENTITY_MAP_COUNT194 u32 map_ofs, level, i;195 struct xe_bo *bo, *batch = tile->mem.kernel_bb_pool->bo;196 u64 entry, pt29_ofs;197 198 /* Can't bump NUM_PT_SLOTS too high */199 BUILD_BUG_ON(NUM_PT_SLOTS > SZ_2M/XE_PAGE_SIZE);200 /* Must be a multiple of 64K to support all platforms */201 BUILD_BUG_ON(NUM_PT_SLOTS * XE_PAGE_SIZE % SZ_64K);202 /* And one slot reserved for the 4KiB page table updates */203 BUILD_BUG_ON(!(NUM_KERNEL_PDE & 1));204 205 /* Need to be sure everything fits in the first PT, or create more */206 xe_tile_assert(tile, m->batch_base_ofs + batch->size < SZ_2M);207 208 bo = xe_bo_create_pin_map(vm->xe, tile, vm,209 num_entries * XE_PAGE_SIZE,210 ttm_bo_type_kernel,211 XE_BO_FLAG_VRAM_IF_DGFX(tile) |212 XE_BO_FLAG_PINNED);213 if (IS_ERR(bo))214 return PTR_ERR(bo);215 216 /* PT30 & PT31 reserved for 2M identity map */217 pt29_ofs = bo->size - 3 * XE_PAGE_SIZE;218 entry = vm->pt_ops->pde_encode_bo(bo, pt29_ofs, pat_index);219 xe_pt_write(xe, &vm->pt_root[id]->bo->vmap, 0, entry);220 221 map_ofs = (num_entries - num_setup) * XE_PAGE_SIZE;222 223 /* Map the entire BO in our level 0 pt */224 for (i = 0, level = 0; i < num_entries; level++) {225 entry = vm->pt_ops->pte_encode_bo(bo, i * XE_PAGE_SIZE,226 pat_index, 0);227 228 xe_map_wr(xe, &bo->vmap, map_ofs + level * 8, u64, entry);229 230 if (vm->flags & XE_VM_FLAG_64K)231 i += 16;232 else233 i += 1;234 }235 236 if (!IS_DGFX(xe)) {237 /* Write out batch too */238 m->batch_base_ofs = NUM_PT_SLOTS * XE_PAGE_SIZE;239 for (i = 0; i < batch->size;240 i += vm->flags & XE_VM_FLAG_64K ? XE_64K_PAGE_SIZE :241 XE_PAGE_SIZE) {242 entry = vm->pt_ops->pte_encode_bo(batch, i,243 pat_index, 0);244 245 xe_map_wr(xe, &bo->vmap, map_ofs + level * 8, u64,246 entry);247 level++;248 }249 if (xe->info.has_usm) {250 xe_tile_assert(tile, batch->size == SZ_1M);251 252 batch = tile->primary_gt->usm.bb_pool->bo;253 m->usm_batch_base_ofs = m->batch_base_ofs + SZ_1M;254 xe_tile_assert(tile, batch->size == SZ_512K);255 256 for (i = 0; i < batch->size;257 i += vm->flags & XE_VM_FLAG_64K ? XE_64K_PAGE_SIZE :258 XE_PAGE_SIZE) {259 entry = vm->pt_ops->pte_encode_bo(batch, i,260 pat_index, 0);261 262 xe_map_wr(xe, &bo->vmap, map_ofs + level * 8, u64,263 entry);264 level++;265 }266 }267 } else {268 u64 batch_addr = xe_bo_addr(batch, 0, XE_PAGE_SIZE);269 270 m->batch_base_ofs = xe_migrate_vram_ofs(xe, batch_addr, false);271 272 if (xe->info.has_usm) {273 batch = tile->primary_gt->usm.bb_pool->bo;274 batch_addr = xe_bo_addr(batch, 0, XE_PAGE_SIZE);275 m->usm_batch_base_ofs = xe_migrate_vram_ofs(xe, batch_addr, false);276 }277 }278 279 for (level = 1; level < num_level; level++) {280 u32 flags = 0;281 282 if (vm->flags & XE_VM_FLAG_64K && level == 1)283 flags = XE_PDE_64K;284 285 entry = vm->pt_ops->pde_encode_bo(bo, map_ofs + (u64)(level - 1) *286 XE_PAGE_SIZE, pat_index);287 xe_map_wr(xe, &bo->vmap, map_ofs + XE_PAGE_SIZE * level, u64,288 entry | flags);289 }290 291 /* Write PDE's that point to our BO. */292 for (i = 0; i < map_ofs / PAGE_SIZE; i++) {293 entry = vm->pt_ops->pde_encode_bo(bo, (u64)i * XE_PAGE_SIZE,294 pat_index);295 296 xe_map_wr(xe, &bo->vmap, map_ofs + XE_PAGE_SIZE +297 (i + 1) * 8, u64, entry);298 }299 300 /* Set up a 1GiB NULL mapping at 255GiB offset. */301 level = 2;302 xe_map_wr(xe, &bo->vmap, map_ofs + XE_PAGE_SIZE * level + 255 * 8, u64,303 vm->pt_ops->pte_encode_addr(xe, 0, pat_index, level, IS_DGFX(xe), 0)304 | XE_PTE_NULL);305 m->cleared_mem_ofs = (255ULL << xe_pt_shift(level));306 307 /* Identity map the entire vram at 256GiB offset */308 if (IS_DGFX(xe)) {309 u64 pt30_ofs = bo->size - 2 * XE_PAGE_SIZE;310 311 xe_migrate_program_identity(xe, vm, bo, map_ofs, IDENTITY_OFFSET,312 pat_index, pt30_ofs);313 xe_assert(xe, xe->mem.vram.actual_physical_size <=314 (MAX_NUM_PTE - IDENTITY_OFFSET) * SZ_1G);315 316 /*317 * Identity map the entire vram for compressed pat_index for xe2+318 * if flat ccs is enabled.319 */320 if (GRAPHICS_VER(xe) >= 20 && xe_device_has_flat_ccs(xe)) {321 u16 comp_pat_index = xe->pat.idx[XE_CACHE_NONE_COMPRESSION];322 u64 vram_offset = IDENTITY_OFFSET +323 DIV_ROUND_UP_ULL(xe->mem.vram.actual_physical_size, SZ_1G);324 u64 pt31_ofs = bo->size - XE_PAGE_SIZE;325 326 xe_assert(xe, xe->mem.vram.actual_physical_size <= (MAX_NUM_PTE -327 IDENTITY_OFFSET - IDENTITY_OFFSET / 2) * SZ_1G);328 xe_migrate_program_identity(xe, vm, bo, map_ofs, vram_offset,329 comp_pat_index, pt31_ofs);330 }331 }332 333 /*334 * Example layout created above, with root level = 3:335 * [PT0...PT7]: kernel PT's for copy/clear; 64 or 4KiB PTE's336 * [PT8]: Kernel PT for VM_BIND, 4 KiB PTE's337 * [PT9...PT26]: Userspace PT's for VM_BIND, 4 KiB PTE's338 * [PT27 = PDE 0] [PT28 = PDE 1] [PT29 = PDE 2] [PT30 & PT31 = 2M vram identity map]339 *340 * This makes the lowest part of the VM point to the pagetables.341 * Hence the lowest 2M in the vm should point to itself, with a few writes342 * and flushes, other parts of the VM can be used either for copying and343 * clearing.344 *345 * For performance, the kernel reserves PDE's, so about 20 are left346 * for async VM updates.347 *348 * To make it easier to work, each scratch PT is put in slot (1 + PT #)349 * everywhere, this allows lockless updates to scratch pages by using350 * the different addresses in VM.351 */352#define NUM_VMUSA_UNIT_PER_PAGE 32353#define VM_SA_UPDATE_UNIT_SIZE (XE_PAGE_SIZE / NUM_VMUSA_UNIT_PER_PAGE)354#define NUM_VMUSA_WRITES_PER_UNIT (VM_SA_UPDATE_UNIT_SIZE / sizeof(u64))355 drm_suballoc_manager_init(&m->vm_update_sa,356 (size_t)(map_ofs / XE_PAGE_SIZE - NUM_KERNEL_PDE) *357 NUM_VMUSA_UNIT_PER_PAGE, 0);358 359 m->pt_bo = bo;360 return 0;361}362 363/*364 * Including the reserved copy engine is required to avoid deadlocks due to365 * migrate jobs servicing the faults gets stuck behind the job that faulted.366 */367static u32 xe_migrate_usm_logical_mask(struct xe_gt *gt)368{369 u32 logical_mask = 0;370 struct xe_hw_engine *hwe;371 enum xe_hw_engine_id id;372 373 for_each_hw_engine(hwe, gt, id) {374 if (hwe->class != XE_ENGINE_CLASS_COPY)375 continue;376 377 if (xe_gt_is_usm_hwe(gt, hwe))378 logical_mask |= BIT(hwe->logical_instance);379 }380 381 return logical_mask;382}383 384static bool xe_migrate_needs_ccs_emit(struct xe_device *xe)385{386 return xe_device_has_flat_ccs(xe) && !(GRAPHICS_VER(xe) >= 20 && IS_DGFX(xe));387}388 389/**390 * xe_migrate_init() - Initialize a migrate context391 * @tile: Back-pointer to the tile we're initializing for.392 *393 * Return: Pointer to a migrate context on success. Error pointer on error.394 */395struct xe_migrate *xe_migrate_init(struct xe_tile *tile)396{397 struct xe_device *xe = tile_to_xe(tile);398 struct xe_gt *primary_gt = tile->primary_gt;399 struct xe_migrate *m;400 struct xe_vm *vm;401 int err;402 403 m = drmm_kzalloc(&xe->drm, sizeof(*m), GFP_KERNEL);404 if (!m)405 return ERR_PTR(-ENOMEM);406 407 m->tile = tile;408 409 /* Special layout, prepared below.. */410 vm = xe_vm_create(xe, XE_VM_FLAG_MIGRATION |411 XE_VM_FLAG_SET_TILE_ID(tile));412 if (IS_ERR(vm))413 return ERR_CAST(vm);414 415 xe_vm_lock(vm, false);416 err = xe_migrate_prepare_vm(tile, m, vm);417 xe_vm_unlock(vm);418 if (err) {419 xe_vm_close_and_put(vm);420 return ERR_PTR(err);421 }422 423 if (xe->info.has_usm) {424 struct xe_hw_engine *hwe = xe_gt_hw_engine(primary_gt,425 XE_ENGINE_CLASS_COPY,426 primary_gt->usm.reserved_bcs_instance,427 false);428 u32 logical_mask = xe_migrate_usm_logical_mask(primary_gt);429 430 if (!hwe || !logical_mask)431 return ERR_PTR(-EINVAL);432 433 /*434 * XXX: Currently only reserving 1 (likely slow) BCS instance on435 * PVC, may want to revisit if performance is needed.436 */437 m->q = xe_exec_queue_create(xe, vm, logical_mask, 1, hwe,438 EXEC_QUEUE_FLAG_KERNEL |439 EXEC_QUEUE_FLAG_PERMANENT |440 EXEC_QUEUE_FLAG_HIGH_PRIORITY, 0);441 } else {442 m->q = xe_exec_queue_create_class(xe, primary_gt, vm,443 XE_ENGINE_CLASS_COPY,444 EXEC_QUEUE_FLAG_KERNEL |445 EXEC_QUEUE_FLAG_PERMANENT, 0);446 }447 if (IS_ERR(m->q)) {448 xe_vm_close_and_put(vm);449 return ERR_CAST(m->q);450 }451 452 mutex_init(&m->job_mutex);453 fs_reclaim_acquire(GFP_KERNEL);454 might_lock(&m->job_mutex);455 fs_reclaim_release(GFP_KERNEL);456 457 err = drmm_add_action_or_reset(&xe->drm, xe_migrate_fini, m);458 if (err)459 return ERR_PTR(err);460 461 if (IS_DGFX(xe)) {462 if (xe_migrate_needs_ccs_emit(xe))463 /* min chunk size corresponds to 4K of CCS Metadata */464 m->min_chunk_size = SZ_4K * SZ_64K /465 xe_device_ccs_bytes(xe, SZ_64K);466 else467 /* Somewhat arbitrary to avoid a huge amount of blits */468 m->min_chunk_size = SZ_64K;469 m->min_chunk_size = roundup_pow_of_two(m->min_chunk_size);470 drm_dbg(&xe->drm, "Migrate min chunk size is 0x%08llx\n",471 (unsigned long long)m->min_chunk_size);472 }473 474 return m;475}476 477static u64 max_mem_transfer_per_pass(struct xe_device *xe)478{479 if (!IS_DGFX(xe) && xe_device_has_flat_ccs(xe))480 return MAX_CCS_LIMITED_TRANSFER;481 482 return MAX_PREEMPTDISABLE_TRANSFER;483}484 485static u64 xe_migrate_res_sizes(struct xe_migrate *m, struct xe_res_cursor *cur)486{487 struct xe_device *xe = tile_to_xe(m->tile);488 u64 size = min_t(u64, max_mem_transfer_per_pass(xe), cur->remaining);489 490 if (mem_type_is_vram(cur->mem_type)) {491 /*492 * VRAM we want to blit in chunks with sizes aligned to493 * min_chunk_size in order for the offset to CCS metadata to be494 * page-aligned. If it's the last chunk it may be smaller.495 *496 * Another constraint is that we need to limit the blit to497 * the VRAM block size, unless size is smaller than498 * min_chunk_size.499 */500 u64 chunk = max_t(u64, cur->size, m->min_chunk_size);501 502 size = min_t(u64, size, chunk);503 if (size > m->min_chunk_size)504 size = round_down(size, m->min_chunk_size);505 }506 507 return size;508}509 510static bool xe_migrate_allow_identity(u64 size, const struct xe_res_cursor *cur)511{512 /* If the chunk is not fragmented, allow identity map. */513 return cur->size >= size;514}515 516#define PTE_UPDATE_FLAG_IS_VRAM BIT(0)517#define PTE_UPDATE_FLAG_IS_COMP_PTE BIT(1)518 519static u32 pte_update_size(struct xe_migrate *m,520 u32 flags,521 struct ttm_resource *res,522 struct xe_res_cursor *cur,523 u64 *L0, u64 *L0_ofs, u32 *L0_pt,524 u32 cmd_size, u32 pt_ofs, u32 avail_pts)525{526 u32 cmds = 0;527 bool is_vram = PTE_UPDATE_FLAG_IS_VRAM & flags;528 bool is_comp_pte = PTE_UPDATE_FLAG_IS_COMP_PTE & flags;529 530 *L0_pt = pt_ofs;531 if (is_vram && xe_migrate_allow_identity(*L0, cur)) {532 /* Offset into identity map. */533 *L0_ofs = xe_migrate_vram_ofs(tile_to_xe(m->tile),534 cur->start + vram_region_gpu_offset(res),535 is_comp_pte);536 cmds += cmd_size;537 } else {538 /* Clip L0 to available size */539 u64 size = min(*L0, (u64)avail_pts * SZ_2M);540 u32 num_4k_pages = (size + XE_PAGE_SIZE - 1) >> XE_PTE_SHIFT;541 542 *L0 = size;543 *L0_ofs = xe_migrate_vm_addr(pt_ofs, 0);544 545 /* MI_STORE_DATA_IMM */546 cmds += 3 * DIV_ROUND_UP(num_4k_pages, MAX_PTE_PER_SDI);547 548 /* PDE qwords */549 cmds += num_4k_pages * 2;550 551 /* Each chunk has a single blit command */552 cmds += cmd_size;553 }554 555 return cmds;556}557 558static void emit_pte(struct xe_migrate *m,559 struct xe_bb *bb, u32 at_pt,560 bool is_vram, bool is_comp_pte,561 struct xe_res_cursor *cur,562 u32 size, struct ttm_resource *res)563{564 struct xe_device *xe = tile_to_xe(m->tile);565 struct xe_vm *vm = m->q->vm;566 u16 pat_index;567 u32 ptes;568 u64 ofs = (u64)at_pt * XE_PAGE_SIZE;569 u64 cur_ofs;570 571 /* Indirect access needs compression enabled uncached PAT index */572 if (GRAPHICS_VERx100(xe) >= 2000)573 pat_index = is_comp_pte ? xe->pat.idx[XE_CACHE_NONE_COMPRESSION] :574 xe->pat.idx[XE_CACHE_WB];575 else576 pat_index = xe->pat.idx[XE_CACHE_WB];577 578 ptes = DIV_ROUND_UP(size, XE_PAGE_SIZE);579 580 while (ptes) {581 u32 chunk = min(MAX_PTE_PER_SDI, ptes);582 583 bb->cs[bb->len++] = MI_STORE_DATA_IMM | MI_SDI_NUM_QW(chunk);584 bb->cs[bb->len++] = ofs;585 bb->cs[bb->len++] = 0;586 587 cur_ofs = ofs;588 ofs += chunk * 8;589 ptes -= chunk;590 591 while (chunk--) {592 u64 addr, flags = 0;593 bool devmem = false;594 595 addr = xe_res_dma(cur) & PAGE_MASK;596 if (is_vram) {597 if (vm->flags & XE_VM_FLAG_64K) {598 u64 va = cur_ofs * XE_PAGE_SIZE / 8;599 600 xe_assert(xe, (va & (SZ_64K - 1)) ==601 (addr & (SZ_64K - 1)));602 603 flags |= XE_PTE_PS64;604 }605 606 addr += vram_region_gpu_offset(res);607 devmem = true;608 }609 610 addr = vm->pt_ops->pte_encode_addr(m->tile->xe,611 addr, pat_index,612 0, devmem, flags);613 bb->cs[bb->len++] = lower_32_bits(addr);614 bb->cs[bb->len++] = upper_32_bits(addr);615 616 xe_res_next(cur, min_t(u32, size, PAGE_SIZE));617 cur_ofs += 8;618 }619 }620}621 622#define EMIT_COPY_CCS_DW 5623static void emit_copy_ccs(struct xe_gt *gt, struct xe_bb *bb,624 u64 dst_ofs, bool dst_is_indirect,625 u64 src_ofs, bool src_is_indirect,626 u32 size)627{628 struct xe_device *xe = gt_to_xe(gt);629 u32 *cs = bb->cs + bb->len;630 u32 num_ccs_blks;631 u32 num_pages;632 u32 ccs_copy_size;633 u32 mocs;634 635 if (GRAPHICS_VERx100(xe) >= 2000) {636 num_pages = DIV_ROUND_UP(size, XE_PAGE_SIZE);637 xe_gt_assert(gt, FIELD_FIT(XE2_CCS_SIZE_MASK, num_pages - 1));638 639 ccs_copy_size = REG_FIELD_PREP(XE2_CCS_SIZE_MASK, num_pages - 1);640 mocs = FIELD_PREP(XE2_XY_CTRL_SURF_MOCS_INDEX_MASK, gt->mocs.uc_index);641 642 } else {643 num_ccs_blks = DIV_ROUND_UP(xe_device_ccs_bytes(gt_to_xe(gt), size),644 NUM_CCS_BYTES_PER_BLOCK);645 xe_gt_assert(gt, FIELD_FIT(CCS_SIZE_MASK, num_ccs_blks - 1));646 647 ccs_copy_size = REG_FIELD_PREP(CCS_SIZE_MASK, num_ccs_blks - 1);648 mocs = FIELD_PREP(XY_CTRL_SURF_MOCS_MASK, gt->mocs.uc_index);649 }650 651 *cs++ = XY_CTRL_SURF_COPY_BLT |652 (src_is_indirect ? 0x0 : 0x1) << SRC_ACCESS_TYPE_SHIFT |653 (dst_is_indirect ? 0x0 : 0x1) << DST_ACCESS_TYPE_SHIFT |654 ccs_copy_size;655 *cs++ = lower_32_bits(src_ofs);656 *cs++ = upper_32_bits(src_ofs) | mocs;657 *cs++ = lower_32_bits(dst_ofs);658 *cs++ = upper_32_bits(dst_ofs) | mocs;659 660 bb->len = cs - bb->cs;661}662 663#define EMIT_COPY_DW 10664static void emit_copy(struct xe_gt *gt, struct xe_bb *bb,665 u64 src_ofs, u64 dst_ofs, unsigned int size,666 unsigned int pitch)667{668 struct xe_device *xe = gt_to_xe(gt);669 u32 mocs = 0;670 u32 tile_y = 0;671 672 xe_gt_assert(gt, size / pitch <= S16_MAX);673 xe_gt_assert(gt, pitch / 4 <= S16_MAX);674 xe_gt_assert(gt, pitch <= U16_MAX);675 676 if (GRAPHICS_VER(xe) >= 20)677 mocs = FIELD_PREP(XE2_XY_FAST_COPY_BLT_MOCS_INDEX_MASK, gt->mocs.uc_index);678 679 if (GRAPHICS_VERx100(xe) >= 1250)680 tile_y = XY_FAST_COPY_BLT_D1_SRC_TILE4 | XY_FAST_COPY_BLT_D1_DST_TILE4;681 682 bb->cs[bb->len++] = XY_FAST_COPY_BLT_CMD | (10 - 2);683 bb->cs[bb->len++] = XY_FAST_COPY_BLT_DEPTH_32 | pitch | tile_y | mocs;684 bb->cs[bb->len++] = 0;685 bb->cs[bb->len++] = (size / pitch) << 16 | pitch / 4;686 bb->cs[bb->len++] = lower_32_bits(dst_ofs);687 bb->cs[bb->len++] = upper_32_bits(dst_ofs);688 bb->cs[bb->len++] = 0;689 bb->cs[bb->len++] = pitch | mocs;690 bb->cs[bb->len++] = lower_32_bits(src_ofs);691 bb->cs[bb->len++] = upper_32_bits(src_ofs);692}693 694static u64 xe_migrate_batch_base(struct xe_migrate *m, bool usm)695{696 return usm ? m->usm_batch_base_ofs : m->batch_base_ofs;697}698 699static u32 xe_migrate_ccs_copy(struct xe_migrate *m,700 struct xe_bb *bb,701 u64 src_ofs, bool src_is_indirect,702 u64 dst_ofs, bool dst_is_indirect, u32 dst_size,703 u64 ccs_ofs, bool copy_ccs)704{705 struct xe_gt *gt = m->tile->primary_gt;706 u32 flush_flags = 0;707 708 if (!copy_ccs && dst_is_indirect) {709 /*710 * If the src is already in vram, then it should already711 * have been cleared by us, or has been populated by the712 * user. Make sure we copy the CCS aux state as-is.713 *714 * Otherwise if the bo doesn't have any CCS metadata attached,715 * we still need to clear it for security reasons.716 */717 u64 ccs_src_ofs = src_is_indirect ? src_ofs : m->cleared_mem_ofs;718 719 emit_copy_ccs(gt, bb,720 dst_ofs, true,721 ccs_src_ofs, src_is_indirect, dst_size);722 723 flush_flags = MI_FLUSH_DW_CCS;724 } else if (copy_ccs) {725 if (!src_is_indirect)726 src_ofs = ccs_ofs;727 else if (!dst_is_indirect)728 dst_ofs = ccs_ofs;729 730 xe_gt_assert(gt, src_is_indirect || dst_is_indirect);731 732 emit_copy_ccs(gt, bb, dst_ofs, dst_is_indirect, src_ofs,733 src_is_indirect, dst_size);734 if (dst_is_indirect)735 flush_flags = MI_FLUSH_DW_CCS;736 }737 738 return flush_flags;739}740 741/**742 * xe_migrate_copy() - Copy content of TTM resources.743 * @m: The migration context.744 * @src_bo: The buffer object @src is currently bound to.745 * @dst_bo: If copying between resources created for the same bo, set this to746 * the same value as @src_bo. If copying between buffer objects, set it to747 * the buffer object @dst is currently bound to.748 * @src: The source TTM resource.749 * @dst: The dst TTM resource.750 * @copy_only_ccs: If true copy only CCS metadata751 *752 * Copies the contents of @src to @dst: On flat CCS devices,753 * the CCS metadata is copied as well if needed, or if not present,754 * the CCS metadata of @dst is cleared for security reasons.755 *756 * Return: Pointer to a dma_fence representing the last copy batch, or757 * an error pointer on failure. If there is a failure, any copy operation758 * started by the function call has been synced.759 */760struct dma_fence *xe_migrate_copy(struct xe_migrate *m,761 struct xe_bo *src_bo,762 struct xe_bo *dst_bo,763 struct ttm_resource *src,764 struct ttm_resource *dst,765 bool copy_only_ccs)766{767 struct xe_gt *gt = m->tile->primary_gt;768 struct xe_device *xe = gt_to_xe(gt);769 struct dma_fence *fence = NULL;770 u64 size = src_bo->size;771 struct xe_res_cursor src_it, dst_it, ccs_it;772 u64 src_L0_ofs, dst_L0_ofs;773 u32 src_L0_pt, dst_L0_pt;774 u64 src_L0, dst_L0;775 int pass = 0;776 int err;777 bool src_is_pltt = src->mem_type == XE_PL_TT;778 bool dst_is_pltt = dst->mem_type == XE_PL_TT;779 bool src_is_vram = mem_type_is_vram(src->mem_type);780 bool dst_is_vram = mem_type_is_vram(dst->mem_type);781 bool copy_ccs = xe_device_has_flat_ccs(xe) &&782 xe_bo_needs_ccs_pages(src_bo) && xe_bo_needs_ccs_pages(dst_bo);783 bool copy_system_ccs = copy_ccs && (!src_is_vram || !dst_is_vram);784 bool use_comp_pat = xe_device_has_flat_ccs(xe) &&785 GRAPHICS_VER(xe) >= 20 && src_is_vram && !dst_is_vram;786 787 /* Copying CCS between two different BOs is not supported yet. */788 if (XE_WARN_ON(copy_ccs && src_bo != dst_bo))789 return ERR_PTR(-EINVAL);790 791 if (src_bo != dst_bo && XE_WARN_ON(src_bo->size != dst_bo->size))792 return ERR_PTR(-EINVAL);793 794 if (!src_is_vram)795 xe_res_first_sg(xe_bo_sg(src_bo), 0, size, &src_it);796 else797 xe_res_first(src, 0, size, &src_it);798 if (!dst_is_vram)799 xe_res_first_sg(xe_bo_sg(dst_bo), 0, size, &dst_it);800 else801 xe_res_first(dst, 0, size, &dst_it);802 803 if (copy_system_ccs)804 xe_res_first_sg(xe_bo_sg(src_bo), xe_bo_ccs_pages_start(src_bo),805 PAGE_ALIGN(xe_device_ccs_bytes(xe, size)),806 &ccs_it);807 808 while (size) {809 u32 batch_size = 2; /* arb_clear() + MI_BATCH_BUFFER_END */810 struct xe_sched_job *job;811 struct xe_bb *bb;812 u32 flush_flags = 0;813 u32 update_idx;814 u64 ccs_ofs, ccs_size;815 u32 ccs_pt;816 u32 pte_flags;817 818 bool usm = xe->info.has_usm;819 u32 avail_pts = max_mem_transfer_per_pass(xe) / LEVEL0_PAGE_TABLE_ENCODE_SIZE;820 821 src_L0 = xe_migrate_res_sizes(m, &src_it);822 dst_L0 = xe_migrate_res_sizes(m, &dst_it);823 824 drm_dbg(&xe->drm, "Pass %u, sizes: %llu & %llu\n",825 pass++, src_L0, dst_L0);826 827 src_L0 = min(src_L0, dst_L0);828 829 pte_flags = src_is_vram ? PTE_UPDATE_FLAG_IS_VRAM : 0;830 pte_flags |= use_comp_pat ? PTE_UPDATE_FLAG_IS_COMP_PTE : 0;831 batch_size += pte_update_size(m, pte_flags, src, &src_it, &src_L0,832 &src_L0_ofs, &src_L0_pt, 0, 0,833 avail_pts);834 835 pte_flags = dst_is_vram ? PTE_UPDATE_FLAG_IS_VRAM : 0;836 batch_size += pte_update_size(m, pte_flags, dst, &dst_it, &src_L0,837 &dst_L0_ofs, &dst_L0_pt, 0,838 avail_pts, avail_pts);839 840 if (copy_system_ccs) {841 ccs_size = xe_device_ccs_bytes(xe, src_L0);842 batch_size += pte_update_size(m, 0, NULL, &ccs_it, &ccs_size,843 &ccs_ofs, &ccs_pt, 0,844 2 * avail_pts,845 avail_pts);846 xe_assert(xe, IS_ALIGNED(ccs_it.start, PAGE_SIZE));847 }848 849 /* Add copy commands size here */850 batch_size += ((copy_only_ccs) ? 0 : EMIT_COPY_DW) +851 ((xe_migrate_needs_ccs_emit(xe) ? EMIT_COPY_CCS_DW : 0));852 853 bb = xe_bb_new(gt, batch_size, usm);854 if (IS_ERR(bb)) {855 err = PTR_ERR(bb);856 goto err_sync;857 }858 859 if (src_is_vram && xe_migrate_allow_identity(src_L0, &src_it))860 xe_res_next(&src_it, src_L0);861 else862 emit_pte(m, bb, src_L0_pt, src_is_vram, copy_system_ccs,863 &src_it, src_L0, src);864 865 if (dst_is_vram && xe_migrate_allow_identity(src_L0, &dst_it))866 xe_res_next(&dst_it, src_L0);867 else868 emit_pte(m, bb, dst_L0_pt, dst_is_vram, copy_system_ccs,869 &dst_it, src_L0, dst);870 871 if (copy_system_ccs)872 emit_pte(m, bb, ccs_pt, false, false, &ccs_it, ccs_size, src);873 874 bb->cs[bb->len++] = MI_BATCH_BUFFER_END;875 update_idx = bb->len;876 877 if (!copy_only_ccs)878 emit_copy(gt, bb, src_L0_ofs, dst_L0_ofs, src_L0, XE_PAGE_SIZE);879 880 if (xe_migrate_needs_ccs_emit(xe))881 flush_flags = xe_migrate_ccs_copy(m, bb, src_L0_ofs,882 IS_DGFX(xe) ? src_is_vram : src_is_pltt,883 dst_L0_ofs,884 IS_DGFX(xe) ? dst_is_vram : dst_is_pltt,885 src_L0, ccs_ofs, copy_ccs);886 887 job = xe_bb_create_migration_job(m->q, bb,888 xe_migrate_batch_base(m, usm),889 update_idx);890 if (IS_ERR(job)) {891 err = PTR_ERR(job);892 goto err;893 }894 895 xe_sched_job_add_migrate_flush(job, flush_flags);896 if (!fence) {897 err = xe_sched_job_add_deps(job, src_bo->ttm.base.resv,898 DMA_RESV_USAGE_BOOKKEEP);899 if (!err && src_bo != dst_bo)900 err = xe_sched_job_add_deps(job, dst_bo->ttm.base.resv,901 DMA_RESV_USAGE_BOOKKEEP);902 if (err)903 goto err_job;904 }905 906 mutex_lock(&m->job_mutex);907 xe_sched_job_arm(job);908 dma_fence_put(fence);909 fence = dma_fence_get(&job->drm.s_fence->finished);910 xe_sched_job_push(job);911 912 dma_fence_put(m->fence);913 m->fence = dma_fence_get(fence);914 915 mutex_unlock(&m->job_mutex);916 917 xe_bb_free(bb, fence);918 size -= src_L0;919 continue;920 921err_job:922 xe_sched_job_put(job);923err:924 xe_bb_free(bb, NULL);925 926err_sync:927 /* Sync partial copy if any. FIXME: under job_mutex? */928 if (fence) {929 dma_fence_wait(fence, false);930 dma_fence_put(fence);931 }932 933 return ERR_PTR(err);934 }935 936 return fence;937}938 939static void emit_clear_link_copy(struct xe_gt *gt, struct xe_bb *bb, u64 src_ofs,940 u32 size, u32 pitch)941{942 struct xe_device *xe = gt_to_xe(gt);943 u32 *cs = bb->cs + bb->len;944 u32 len = PVC_MEM_SET_CMD_LEN_DW;945 946 *cs++ = PVC_MEM_SET_CMD | PVC_MEM_SET_MATRIX | (len - 2);947 *cs++ = pitch - 1;948 *cs++ = (size / pitch) - 1;949 *cs++ = pitch - 1;950 *cs++ = lower_32_bits(src_ofs);951 *cs++ = upper_32_bits(src_ofs);952 if (GRAPHICS_VERx100(xe) >= 2000)953 *cs++ = FIELD_PREP(XE2_MEM_SET_MOCS_INDEX_MASK, gt->mocs.uc_index);954 else955 *cs++ = FIELD_PREP(PVC_MEM_SET_MOCS_INDEX_MASK, gt->mocs.uc_index);956 957 xe_gt_assert(gt, cs - bb->cs == len + bb->len);958 959 bb->len += len;960}961 962static void emit_clear_main_copy(struct xe_gt *gt, struct xe_bb *bb,963 u64 src_ofs, u32 size, u32 pitch, bool is_vram)964{965 struct xe_device *xe = gt_to_xe(gt);966 u32 *cs = bb->cs + bb->len;967 u32 len = XY_FAST_COLOR_BLT_DW;968 969 if (GRAPHICS_VERx100(xe) < 1250)970 len = 11;971 972 *cs++ = XY_FAST_COLOR_BLT_CMD | XY_FAST_COLOR_BLT_DEPTH_32 |973 (len - 2);974 if (GRAPHICS_VERx100(xe) >= 2000)975 *cs++ = FIELD_PREP(XE2_XY_FAST_COLOR_BLT_MOCS_INDEX_MASK, gt->mocs.uc_index) |976 (pitch - 1);977 else978 *cs++ = FIELD_PREP(XY_FAST_COLOR_BLT_MOCS_MASK, gt->mocs.uc_index) |979 (pitch - 1);980 *cs++ = 0;981 *cs++ = (size / pitch) << 16 | pitch / 4;982 *cs++ = lower_32_bits(src_ofs);983 *cs++ = upper_32_bits(src_ofs);984 *cs++ = (is_vram ? 0x0 : 0x1) << XY_FAST_COLOR_BLT_MEM_TYPE_SHIFT;985 *cs++ = 0;986 *cs++ = 0;987 *cs++ = 0;988 *cs++ = 0;989 990 if (len > 11) {991 *cs++ = 0;992 *cs++ = 0;993 *cs++ = 0;994 *cs++ = 0;995 *cs++ = 0;996 }997 998 xe_gt_assert(gt, cs - bb->cs == len + bb->len);999 1000 bb->len += len;1001}1002 1003static bool has_service_copy_support(struct xe_gt *gt)1004{1005 /*1006 * What we care about is whether the architecture was designed with1007 * service copy functionality (specifically the new MEM_SET / MEM_COPY1008 * instructions) so check the architectural engine list rather than the1009 * actual list since these instructions are usable on BCS0 even if1010 * all of the actual service copy engines (BCS1-BCS8) have been fused1011 * off.1012 */1013 return gt->info.engine_mask & GENMASK(XE_HW_ENGINE_BCS8,1014 XE_HW_ENGINE_BCS1);1015}1016 1017static u32 emit_clear_cmd_len(struct xe_gt *gt)1018{1019 if (has_service_copy_support(gt))1020 return PVC_MEM_SET_CMD_LEN_DW;1021 else1022 return XY_FAST_COLOR_BLT_DW;1023}1024 1025static void emit_clear(struct xe_gt *gt, struct xe_bb *bb, u64 src_ofs,1026 u32 size, u32 pitch, bool is_vram)1027{1028 if (has_service_copy_support(gt))1029 emit_clear_link_copy(gt, bb, src_ofs, size, pitch);1030 else1031 emit_clear_main_copy(gt, bb, src_ofs, size, pitch,1032 is_vram);1033}1034 1035/**1036 * xe_migrate_clear() - Copy content of TTM resources.1037 * @m: The migration context.1038 * @bo: The buffer object @dst is currently bound to.1039 * @dst: The dst TTM resource to be cleared.1040 * @clear_flags: flags to specify which data to clear: CCS, BO, or both.1041 *1042 * Clear the contents of @dst to zero when XE_MIGRATE_CLEAR_FLAG_BO_DATA is set.1043 * On flat CCS devices, the CCS metadata is cleared to zero with XE_MIGRATE_CLEAR_FLAG_CCS_DATA.1044 * Set XE_MIGRATE_CLEAR_FLAG_FULL to clear bo as well as CCS metadata.1045 * TODO: Eliminate the @bo argument.1046 *1047 * Return: Pointer to a dma_fence representing the last clear batch, or1048 * an error pointer on failure. If there is a failure, any clear operation1049 * started by the function call has been synced.1050 */1051struct dma_fence *xe_migrate_clear(struct xe_migrate *m,1052 struct xe_bo *bo,1053 struct ttm_resource *dst,1054 u32 clear_flags)1055{1056 bool clear_vram = mem_type_is_vram(dst->mem_type);1057 bool clear_bo_data = XE_MIGRATE_CLEAR_FLAG_BO_DATA & clear_flags;1058 bool clear_ccs = XE_MIGRATE_CLEAR_FLAG_CCS_DATA & clear_flags;1059 struct xe_gt *gt = m->tile->primary_gt;1060 struct xe_device *xe = gt_to_xe(gt);1061 bool clear_only_system_ccs = false;1062 struct dma_fence *fence = NULL;1063 u64 size = bo->size;1064 struct xe_res_cursor src_it;1065 struct ttm_resource *src = dst;1066 int err;1067 1068 if (WARN_ON(!clear_bo_data && !clear_ccs))1069 return NULL;1070 1071 if (!clear_bo_data && clear_ccs && !IS_DGFX(xe))1072 clear_only_system_ccs = true;1073 1074 if (!clear_vram)1075 xe_res_first_sg(xe_bo_sg(bo), 0, bo->size, &src_it);1076 else1077 xe_res_first(src, 0, bo->size, &src_it);1078 1079 while (size) {1080 u64 clear_L0_ofs;1081 u32 clear_L0_pt;1082 u32 flush_flags = 0;1083 u64 clear_L0;1084 struct xe_sched_job *job;1085 struct xe_bb *bb;1086 u32 batch_size, update_idx;1087 u32 pte_flags;1088 1089 bool usm = xe->info.has_usm;1090 u32 avail_pts = max_mem_transfer_per_pass(xe) / LEVEL0_PAGE_TABLE_ENCODE_SIZE;1091 1092 clear_L0 = xe_migrate_res_sizes(m, &src_it);1093 1094 /* Calculate final sizes and batch size.. */1095 pte_flags = clear_vram ? PTE_UPDATE_FLAG_IS_VRAM : 0;1096 batch_size = 2 +1097 pte_update_size(m, pte_flags, src, &src_it,1098 &clear_L0, &clear_L0_ofs, &clear_L0_pt,1099 clear_bo_data ? emit_clear_cmd_len(gt) : 0, 0,1100 avail_pts);1101 1102 if (xe_migrate_needs_ccs_emit(xe))1103 batch_size += EMIT_COPY_CCS_DW;1104 1105 /* Clear commands */1106 1107 if (WARN_ON_ONCE(!clear_L0))1108 break;1109 1110 bb = xe_bb_new(gt, batch_size, usm);1111 if (IS_ERR(bb)) {1112 err = PTR_ERR(bb);1113 goto err_sync;1114 }1115 1116 size -= clear_L0;1117 /* Preemption is enabled again by the ring ops. */1118 if (clear_vram && xe_migrate_allow_identity(clear_L0, &src_it))1119 xe_res_next(&src_it, clear_L0);1120 else1121 emit_pte(m, bb, clear_L0_pt, clear_vram, clear_only_system_ccs,1122 &src_it, clear_L0, dst);1123 1124 bb->cs[bb->len++] = MI_BATCH_BUFFER_END;1125 update_idx = bb->len;1126 1127 if (clear_bo_data)1128 emit_clear(gt, bb, clear_L0_ofs, clear_L0, XE_PAGE_SIZE, clear_vram);1129 1130 if (xe_migrate_needs_ccs_emit(xe)) {1131 emit_copy_ccs(gt, bb, clear_L0_ofs, true,1132 m->cleared_mem_ofs, false, clear_L0);1133 flush_flags = MI_FLUSH_DW_CCS;1134 }1135 1136 job = xe_bb_create_migration_job(m->q, bb,1137 xe_migrate_batch_base(m, usm),1138 update_idx);1139 if (IS_ERR(job)) {1140 err = PTR_ERR(job);1141 goto err;1142 }1143 1144 xe_sched_job_add_migrate_flush(job, flush_flags);1145 if (!fence) {1146 /*1147 * There can't be anything userspace related at this1148 * point, so we just need to respect any potential move1149 * fences, which are always tracked as1150 * DMA_RESV_USAGE_KERNEL.1151 */1152 err = xe_sched_job_add_deps(job, bo->ttm.base.resv,1153 DMA_RESV_USAGE_KERNEL);1154 if (err)1155 goto err_job;1156 }1157 1158 mutex_lock(&m->job_mutex);1159 xe_sched_job_arm(job);1160 dma_fence_put(fence);1161 fence = dma_fence_get(&job->drm.s_fence->finished);1162 xe_sched_job_push(job);1163 1164 dma_fence_put(m->fence);1165 m->fence = dma_fence_get(fence);1166 1167 mutex_unlock(&m->job_mutex);1168 1169 xe_bb_free(bb, fence);1170 continue;1171 1172err_job:1173 xe_sched_job_put(job);1174err:1175 xe_bb_free(bb, NULL);1176err_sync:1177 /* Sync partial copies if any. FIXME: job_mutex? */1178 if (fence) {1179 dma_fence_wait(m->fence, false);1180 dma_fence_put(fence);1181 }1182 1183 return ERR_PTR(err);1184 }1185 1186 if (clear_ccs)1187 bo->ccs_cleared = true;1188 1189 return fence;1190}1191 1192static void write_pgtable(struct xe_tile *tile, struct xe_bb *bb, u64 ppgtt_ofs,1193 const struct xe_vm_pgtable_update_op *pt_op,1194 const struct xe_vm_pgtable_update *update,1195 struct xe_migrate_pt_update *pt_update)1196{1197 const struct xe_migrate_pt_update_ops *ops = pt_update->ops;1198 u32 chunk;1199 u32 ofs = update->ofs, size = update->qwords;1200 1201 /*1202 * If we have 512 entries (max), we would populate it ourselves,1203 * and update the PDE above it to the new pointer.1204 * The only time this can only happen if we have to update the top1205 * PDE. This requires a BO that is almost vm->size big.1206 *1207 * This shouldn't be possible in practice.. might change when 16K1208 * pages are used. Hence the assert.1209 */1210 xe_tile_assert(tile, update->qwords < MAX_NUM_PTE);1211 if (!ppgtt_ofs)1212 ppgtt_ofs = xe_migrate_vram_ofs(tile_to_xe(tile),1213 xe_bo_addr(update->pt_bo, 0,1214 XE_PAGE_SIZE), false);1215 1216 do {1217 u64 addr = ppgtt_ofs + ofs * 8;1218 1219 chunk = min(size, MAX_PTE_PER_SDI);1220 1221 /* Ensure populatefn can do memset64 by aligning bb->cs */1222 if (!(bb->len & 1))1223 bb->cs[bb->len++] = MI_NOOP;1224 1225 bb->cs[bb->len++] = MI_STORE_DATA_IMM | MI_SDI_NUM_QW(chunk);1226 bb->cs[bb->len++] = lower_32_bits(addr);1227 bb->cs[bb->len++] = upper_32_bits(addr);1228 if (pt_op->bind)1229 ops->populate(pt_update, tile, NULL, bb->cs + bb->len,1230 ofs, chunk, update);1231 else1232 ops->clear(pt_update, tile, NULL, bb->cs + bb->len,1233 ofs, chunk, update);1234 1235 bb->len += chunk * 2;1236 ofs += chunk;1237 size -= chunk;1238 } while (size);1239}1240 1241struct xe_vm *xe_migrate_get_vm(struct xe_migrate *m)1242{1243 return xe_vm_get(m->q->vm);1244}1245 1246#if IS_ENABLED(CONFIG_DRM_XE_KUNIT_TEST)1247struct migrate_test_params {1248 struct xe_test_priv base;1249 bool force_gpu;1250};1251 1252#define to_migrate_test_params(_priv) \1253 container_of(_priv, struct migrate_test_params, base)1254#endif1255 1256static struct dma_fence *1257xe_migrate_update_pgtables_cpu(struct xe_migrate *m,1258 struct xe_migrate_pt_update *pt_update)1259{1260 XE_TEST_DECLARE(struct migrate_test_params *test =1261 to_migrate_test_params1262 (xe_cur_kunit_priv(XE_TEST_LIVE_MIGRATE));)1263 const struct xe_migrate_pt_update_ops *ops = pt_update->ops;1264 struct xe_vm *vm = pt_update->vops->vm;1265 struct xe_vm_pgtable_update_ops *pt_update_ops =1266 &pt_update->vops->pt_update_ops[pt_update->tile_id];1267 int err;1268 u32 i, j;1269 1270 if (XE_TEST_ONLY(test && test->force_gpu))1271 return ERR_PTR(-ETIME);1272 1273 if (ops->pre_commit) {1274 pt_update->job = NULL;1275 err = ops->pre_commit(pt_update);1276 if (err)1277 return ERR_PTR(err);1278 }1279 1280 for (i = 0; i < pt_update_ops->num_ops; ++i) {1281 const struct xe_vm_pgtable_update_op *pt_op =1282 &pt_update_ops->ops[i];1283 1284 for (j = 0; j < pt_op->num_entries; j++) {1285 const struct xe_vm_pgtable_update *update =1286 &pt_op->entries[j];1287 1288 if (pt_op->bind)1289 ops->populate(pt_update, m->tile,1290 &update->pt_bo->vmap, NULL,1291 update->ofs, update->qwords,1292 update);1293 else1294 ops->clear(pt_update, m->tile,1295 &update->pt_bo->vmap, NULL,1296 update->ofs, update->qwords, update);1297 }1298 }1299 1300 trace_xe_vm_cpu_bind(vm);1301 xe_device_wmb(vm->xe);1302 1303 return dma_fence_get_stub();1304}1305 1306static struct dma_fence *1307__xe_migrate_update_pgtables(struct xe_migrate *m,1308 struct xe_migrate_pt_update *pt_update,1309 struct xe_vm_pgtable_update_ops *pt_update_ops)1310{1311 const struct xe_migrate_pt_update_ops *ops = pt_update->ops;1312 struct xe_tile *tile = m->tile;1313 struct xe_gt *gt = tile->primary_gt;1314 struct xe_device *xe = tile_to_xe(tile);1315 struct xe_sched_job *job;1316 struct dma_fence *fence;1317 struct drm_suballoc *sa_bo = NULL;1318 struct xe_bb *bb;1319 u32 i, j, batch_size = 0, ppgtt_ofs, update_idx, page_ofs = 0;1320 u32 num_updates = 0, current_update = 0;1321 u64 addr;1322 int err = 0;1323 bool is_migrate = pt_update_ops->q == m->q;1324 bool usm = is_migrate && xe->info.has_usm;1325 1326 for (i = 0; i < pt_update_ops->num_ops; ++i) {1327 struct xe_vm_pgtable_update_op *pt_op = &pt_update_ops->ops[i];1328 struct xe_vm_pgtable_update *updates = pt_op->entries;1329 1330 num_updates += pt_op->num_entries;1331 for (j = 0; j < pt_op->num_entries; ++j) {1332 u32 num_cmds = DIV_ROUND_UP(updates[j].qwords,1333 MAX_PTE_PER_SDI);1334 1335 /* align noop + MI_STORE_DATA_IMM cmd prefix */1336 batch_size += 4 * num_cmds + updates[j].qwords * 2;1337 }1338 }1339 1340 /* fixed + PTE entries */1341 if (IS_DGFX(xe))1342 batch_size += 2;1343 else1344 batch_size += 6 * (num_updates / MAX_PTE_PER_SDI + 1) +1345 num_updates * 2;1346 1347 bb = xe_bb_new(gt, batch_size, usm);1348 if (IS_ERR(bb))1349 return ERR_CAST(bb);1350 1351 /* For sysmem PTE's, need to map them in our hole.. */1352 if (!IS_DGFX(xe)) {1353 u32 ptes, ofs;1354 1355 ppgtt_ofs = NUM_KERNEL_PDE - 1;1356 if (!is_migrate) {1357 u32 num_units = DIV_ROUND_UP(num_updates,1358 NUM_VMUSA_WRITES_PER_UNIT);1359 1360 if (num_units > m->vm_update_sa.size) {1361 err = -ENOBUFS;1362 goto err_bb;1363 }1364 sa_bo = drm_suballoc_new(&m->vm_update_sa, num_units,1365 GFP_KERNEL, true, 0);1366 if (IS_ERR(sa_bo)) {1367 err = PTR_ERR(sa_bo);1368 goto err_bb;1369 }1370 1371 ppgtt_ofs = NUM_KERNEL_PDE +1372 (drm_suballoc_soffset(sa_bo) /1373 NUM_VMUSA_UNIT_PER_PAGE);1374 page_ofs = (drm_suballoc_soffset(sa_bo) %1375 NUM_VMUSA_UNIT_PER_PAGE) *1376 VM_SA_UPDATE_UNIT_SIZE;1377 }1378 1379 /* Map our PT's to gtt */1380 i = 0;1381 j = 0;1382 ptes = num_updates;1383 ofs = ppgtt_ofs * XE_PAGE_SIZE + page_ofs;1384 while (ptes) {1385 u32 chunk = min(MAX_PTE_PER_SDI, ptes);1386 u32 idx = 0;1387 1388 bb->cs[bb->len++] = MI_STORE_DATA_IMM |1389 MI_SDI_NUM_QW(chunk);1390 bb->cs[bb->len++] = ofs;1391 bb->cs[bb->len++] = 0; /* upper_32_bits */1392 1393 for (; i < pt_update_ops->num_ops; ++i) {1394 struct xe_vm_pgtable_update_op *pt_op =1395 &pt_update_ops->ops[i];1396 struct xe_vm_pgtable_update *updates = pt_op->entries;1397 1398 for (; j < pt_op->num_entries; ++j, ++current_update, ++idx) {1399 struct xe_vm *vm = pt_update->vops->vm;1400 struct xe_bo *pt_bo = updates[j].pt_bo;1401 1402 if (idx == chunk)1403 goto next_cmd;1404 1405 xe_tile_assert(tile, pt_bo->size == SZ_4K);1406 1407 /* Map a PT at most once */1408 if (pt_bo->update_index < 0)1409 pt_bo->update_index = current_update;1410 1411 addr = vm->pt_ops->pte_encode_bo(pt_bo, 0,1412 XE_CACHE_WB, 0);1413 bb->cs[bb->len++] = lower_32_bits(addr);1414 bb->cs[bb->len++] = upper_32_bits(addr);1415 }1416 1417 j = 0;1418 }1419 1420next_cmd:1421 ptes -= chunk;1422 ofs += chunk * sizeof(u64);1423 }1424 1425 bb->cs[bb->len++] = MI_BATCH_BUFFER_END;1426 update_idx = bb->len;1427 1428 addr = xe_migrate_vm_addr(ppgtt_ofs, 0) +1429 (page_ofs / sizeof(u64)) * XE_PAGE_SIZE;1430 for (i = 0; i < pt_update_ops->num_ops; ++i) {1431 struct xe_vm_pgtable_update_op *pt_op =1432 &pt_update_ops->ops[i];1433 struct xe_vm_pgtable_update *updates = pt_op->entries;1434 1435 for (j = 0; j < pt_op->num_entries; ++j) {1436 struct xe_bo *pt_bo = updates[j].pt_bo;1437 1438 write_pgtable(tile, bb, addr +1439 pt_bo->update_index * XE_PAGE_SIZE,1440 pt_op, &updates[j], pt_update);1441 }1442 }1443 } else {1444 /* phys pages, no preamble required */1445 bb->cs[bb->len++] = MI_BATCH_BUFFER_END;1446 update_idx = bb->len;1447 1448 for (i = 0; i < pt_update_ops->num_ops; ++i) {1449 struct xe_vm_pgtable_update_op *pt_op =1450 &pt_update_ops->ops[i];1451 struct xe_vm_pgtable_update *updates = pt_op->entries;1452 1453 for (j = 0; j < pt_op->num_entries; ++j)1454 write_pgtable(tile, bb, 0, pt_op, &updates[j],1455 pt_update);1456 }1457 }1458 1459 job = xe_bb_create_migration_job(pt_update_ops->q, bb,1460 xe_migrate_batch_base(m, usm),1461 update_idx);1462 if (IS_ERR(job)) {1463 err = PTR_ERR(job);1464 goto err_sa;1465 }1466 1467 if (ops->pre_commit) {1468 pt_update->job = job;1469 err = ops->pre_commit(pt_update);1470 if (err)1471 goto err_job;1472 }1473 if (is_migrate)1474 mutex_lock(&m->job_mutex);1475 1476 xe_sched_job_arm(job);1477 fence = dma_fence_get(&job->drm.s_fence->finished);1478 xe_sched_job_push(job);1479 1480 if (is_migrate)1481 mutex_unlock(&m->job_mutex);1482 1483 xe_bb_free(bb, fence);1484 drm_suballoc_free(sa_bo, fence);1485 1486 return fence;1487 1488err_job:1489 xe_sched_job_put(job);1490err_sa:1491 drm_suballoc_free(sa_bo, NULL);1492err_bb:1493 xe_bb_free(bb, NULL);1494 return ERR_PTR(err);1495}1496 1497/**1498 * xe_migrate_update_pgtables() - Pipelined page-table update1499 * @m: The migrate context.1500 * @pt_update: PT update arguments1501 *1502 * Perform a pipelined page-table update. The update descriptors are typically1503 * built under the same lock critical section as a call to this function. If1504 * using the default engine for the updates, they will be performed in the1505 * order they grab the job_mutex. If different engines are used, external1506 * synchronization is needed for overlapping updates to maintain page-table1507 * consistency. Note that the meaing of "overlapping" is that the updates1508 * touch the same page-table, which might be a higher-level page-directory.1509 * If no pipelining is needed, then updates may be performed by the cpu.1510 *1511 * Return: A dma_fence that, when signaled, indicates the update completion.1512 */1513struct dma_fence *1514xe_migrate_update_pgtables(struct xe_migrate *m,1515 struct xe_migrate_pt_update *pt_update)1516 1517{1518 struct xe_vm_pgtable_update_ops *pt_update_ops =1519 &pt_update->vops->pt_update_ops[pt_update->tile_id];1520 struct dma_fence *fence;1521 1522 fence = xe_migrate_update_pgtables_cpu(m, pt_update);1523 1524 /* -ETIME indicates a job is needed, anything else is legit error */1525 if (!IS_ERR(fence) || PTR_ERR(fence) != -ETIME)1526 return fence;1527 1528 return __xe_migrate_update_pgtables(m, pt_update, pt_update_ops);1529}1530 1531/**1532 * xe_migrate_wait() - Complete all operations using the xe_migrate context1533 * @m: Migrate context to wait for.1534 *1535 * Waits until the GPU no longer uses the migrate context's default engine1536 * or its page-table objects. FIXME: What about separate page-table update1537 * engines?1538 */1539void xe_migrate_wait(struct xe_migrate *m)1540{1541 if (m->fence)1542 dma_fence_wait(m->fence, false);1543}1544 1545#if IS_ENABLED(CONFIG_DRM_XE_KUNIT_TEST)1546#include "tests/xe_migrate.c"1547#endif1548