507 lines · c
1// SPDX-License-Identifier: MIT2/*3 * Copyright © 2023 Intel Corporation4 */5 6#include <linux/align.h>7 8#include <drm/drm_managed.h>9 10#include "regs/xe_gt_regs.h"11 12#include "xe_assert.h"13#include "xe_bo.h"14#include "xe_lmtt.h"15#include "xe_map.h"16#include "xe_mmio.h"17#include "xe_res_cursor.h"18#include "xe_sriov.h"19#include "xe_sriov_printk.h"20 21/**22 * DOC: Local Memory Translation Table23 *24 * The Local Memory Translation Table (LMTT) provides additional abstraction25 * when Virtual Function (VF) is accessing device Local Memory (VRAM).26 *27 * The Root LMTT Page Directory contains one entry for each VF. Entries are28 * indexed by the function number (1-based, index 0 is unused).29 *30 * See `Two-Level LMTT Structure`_ and `Multi-Level LMTT Structure`_.31 */32 33#define lmtt_assert(lmtt, condition) xe_tile_assert(lmtt_to_tile(lmtt), condition)34#define lmtt_debug(lmtt, msg...) xe_sriov_dbg_verbose(lmtt_to_xe(lmtt), "LMTT: " msg)35 36static bool xe_has_multi_level_lmtt(struct xe_device *xe)37{38 return GRAPHICS_VERx100(xe) >= 1260;39}40 41static struct xe_tile *lmtt_to_tile(struct xe_lmtt *lmtt)42{43 return container_of(lmtt, struct xe_tile, sriov.pf.lmtt);44}45 46static struct xe_device *lmtt_to_xe(struct xe_lmtt *lmtt)47{48 return tile_to_xe(lmtt_to_tile(lmtt));49}50 51static u64 lmtt_page_size(struct xe_lmtt *lmtt)52{53 return BIT_ULL(lmtt->ops->lmtt_pte_shift(0));54}55 56static struct xe_lmtt_pt *lmtt_pt_alloc(struct xe_lmtt *lmtt, unsigned int level)57{58 unsigned int num_entries = level ? lmtt->ops->lmtt_pte_num(level) : 0;59 struct xe_lmtt_pt *pt;60 struct xe_bo *bo;61 int err;62 63 pt = kzalloc(struct_size(pt, entries, num_entries), GFP_KERNEL);64 if (!pt) {65 err = -ENOMEM;66 goto out;67 }68 69 bo = xe_bo_create_pin_map(lmtt_to_xe(lmtt), lmtt_to_tile(lmtt), NULL,70 PAGE_ALIGN(lmtt->ops->lmtt_pte_size(level) *71 lmtt->ops->lmtt_pte_num(level)),72 ttm_bo_type_kernel,73 XE_BO_FLAG_VRAM_IF_DGFX(lmtt_to_tile(lmtt)) |74 XE_BO_FLAG_NEEDS_64K | XE_BO_FLAG_PINNED);75 if (IS_ERR(bo)) {76 err = PTR_ERR(bo);77 goto out_free_pt;78 }79 80 lmtt_assert(lmtt, xe_bo_is_vram(bo));81 82 pt->level = level;83 pt->bo = bo;84 return pt;85 86out_free_pt:87 kfree(pt);88out:89 return ERR_PTR(err);90}91 92static void lmtt_pt_free(struct xe_lmtt_pt *pt)93{94 xe_bo_unpin_map_no_vm(pt->bo);95 kfree(pt);96}97 98static int lmtt_init_pd(struct xe_lmtt *lmtt)99{100 struct xe_lmtt_pt *pd;101 102 lmtt_assert(lmtt, !lmtt->pd);103 lmtt_assert(lmtt, lmtt->ops->lmtt_root_pd_level());104 105 pd = lmtt_pt_alloc(lmtt, lmtt->ops->lmtt_root_pd_level());106 if (IS_ERR(pd))107 return PTR_ERR(pd);108 109 lmtt->pd = pd;110 return 0;111}112 113static void lmtt_fini_pd(struct xe_lmtt *lmtt)114{115 struct xe_lmtt_pt *pd = lmtt->pd;116 unsigned int num_entries = lmtt->ops->lmtt_pte_num(pd->level);117 unsigned int n = 0;118 119 /* make sure we don't leak */120 for (n = 0; n < num_entries; n++)121 lmtt_assert(lmtt, !pd->entries[n]);122 123 lmtt->pd = NULL;124 lmtt_pt_free(pd);125}126 127static void fini_lmtt(struct drm_device *drm, void *arg)128{129 struct xe_lmtt *lmtt = arg;130 131 lmtt_assert(lmtt, !(!!lmtt->ops ^ !!lmtt->pd));132 133 if (!lmtt->pd)134 return;135 136 lmtt_fini_pd(lmtt);137 lmtt->ops = NULL;138}139 140/**141 * xe_lmtt_init - LMTT software initialization.142 * @lmtt: the &xe_lmtt to initialize143 *144 * The LMTT initialization requires two steps.145 *146 * The xe_lmtt_init() checks if LMTT is required on current device and selects147 * and initialize proper variant of the LMTT Root Directory. Currently supported148 * variants are `Two-Level LMTT Structure`_ and `Multi-Level LMTT Structure`_.149 *150 * In next step xe_lmtt_init_hw() will register this directory on the hardware.151 *152 * Notes:153 * The LMTT allocations are managed and will be implicitly released on driver unload.154 * This function shall be called only once and only when running as a PF driver.155 * Any LMTT initialization failure should block VFs enabling.156 *157 * Return: 0 on success or a negative error code on failure.158 */159int xe_lmtt_init(struct xe_lmtt *lmtt)160{161 struct xe_device *xe = lmtt_to_xe(lmtt);162 int err;163 164 lmtt_assert(lmtt, IS_SRIOV_PF(xe));165 lmtt_assert(lmtt, !lmtt->ops);166 167 if (!IS_DGFX(xe))168 return 0;169 170 if (xe_has_multi_level_lmtt(xe))171 lmtt->ops = &lmtt_ml_ops;172 else173 lmtt->ops = &lmtt_2l_ops;174 175 err = lmtt_init_pd(lmtt);176 if (unlikely(err))177 goto fail;178 179 return drmm_add_action_or_reset(&xe->drm, fini_lmtt, lmtt);180 181fail:182 lmtt->ops = NULL;183 return err;184}185 186static void lmtt_setup_dir_ptr(struct xe_lmtt *lmtt)187{188 struct xe_tile *tile = lmtt_to_tile(lmtt);189 struct xe_device *xe = tile_to_xe(tile);190 dma_addr_t offset = xe_bo_main_addr(lmtt->pd->bo, XE_PAGE_SIZE);191 192 lmtt_debug(lmtt, "DIR offset %pad\n", &offset);193 lmtt_assert(lmtt, xe_bo_is_vram(lmtt->pd->bo));194 lmtt_assert(lmtt, IS_ALIGNED(offset, SZ_64K));195 196 xe_mmio_write32(tile->primary_gt,197 GRAPHICS_VER(xe) >= 20 ? XE2_LMEM_CFG : LMEM_CFG,198 LMEM_EN | REG_FIELD_PREP(LMTT_DIR_PTR, offset / SZ_64K));199}200 201/**202 * xe_lmtt_init_hw - Perform LMTT hardware initialization.203 * @lmtt: the &xe_lmtt to initialize204 *205 * This function is a second step of the LMTT initialization.206 * This function registers LMTT Root Directory prepared in xe_lmtt_init().207 *208 * This function shall be called after every hardware reset.209 * This function shall be called only when running as a PF driver.210 */211void xe_lmtt_init_hw(struct xe_lmtt *lmtt)212{213 if (!lmtt->pd)214 return;215 216 lmtt_setup_dir_ptr(lmtt);217}218 219static void lmtt_write_pte(struct xe_lmtt *lmtt, struct xe_lmtt_pt *pt,220 u64 pte, unsigned int idx)221{222 unsigned int level = pt->level;223 224 lmtt_assert(lmtt, idx <= lmtt->ops->lmtt_pte_num(level));225 lmtt_debug(lmtt, "WRITE level=%u index=%u pte=%#llx\n", level, idx, pte);226 227 switch (lmtt->ops->lmtt_pte_size(level)) {228 case sizeof(u32):229 xe_map_wr(lmtt_to_xe(lmtt), &pt->bo->vmap, idx * sizeof(u32), u32, pte);230 break;231 case sizeof(u64):232 xe_map_wr(lmtt_to_xe(lmtt), &pt->bo->vmap, idx * sizeof(u64), u64, pte);233 break;234 default:235 lmtt_assert(lmtt, !!!"invalid pte size");236 }237}238 239static void lmtt_destroy_pt(struct xe_lmtt *lmtt, struct xe_lmtt_pt *pd)240{241 unsigned int num_entries = pd->level ? lmtt->ops->lmtt_pte_num(pd->level) : 0;242 struct xe_lmtt_pt *pt;243 unsigned int i;244 245 for (i = 0; i < num_entries; i++) {246 pt = pd->entries[i];247 pd->entries[i] = NULL;248 if (!pt)249 continue;250 251 lmtt_destroy_pt(lmtt, pt);252 }253 254 lmtt_pt_free(pd);255}256 257static void lmtt_drop_pages(struct xe_lmtt *lmtt, unsigned int vfid)258{259 struct xe_lmtt_pt *pd = lmtt->pd;260 struct xe_lmtt_pt *pt;261 262 pt = pd->entries[vfid];263 pd->entries[vfid] = NULL;264 if (!pt)265 return;266 267 lmtt_write_pte(lmtt, pd, LMTT_PTE_INVALID, vfid);268 269 lmtt_assert(lmtt, pd->level > 0);270 lmtt_assert(lmtt, pt->level == pd->level - 1);271 lmtt_destroy_pt(lmtt, pt);272}273 274static int __lmtt_alloc_range(struct xe_lmtt *lmtt, struct xe_lmtt_pt *pd,275 u64 start, u64 end)276{277 u64 pte_addr_shift = BIT_ULL(lmtt->ops->lmtt_pte_shift(pd->level));278 u64 offset;279 int err;280 281 lmtt_assert(lmtt, pd->level > 0);282 283 offset = start;284 while (offset < end) {285 struct xe_lmtt_pt *pt;286 u64 next, pde, pt_addr;287 unsigned int idx;288 289 pt = lmtt_pt_alloc(lmtt, pd->level - 1);290 if (IS_ERR(pt))291 return PTR_ERR(pt);292 293 pt_addr = xe_bo_main_addr(pt->bo, XE_PAGE_SIZE);294 295 idx = lmtt->ops->lmtt_pte_index(offset, pd->level);296 pde = lmtt->ops->lmtt_pte_encode(pt_addr, pd->level);297 298 lmtt_write_pte(lmtt, pd, pde, idx);299 300 pd->entries[idx] = pt;301 302 next = min(end, round_up(offset + 1, pte_addr_shift));303 304 if (pt->level != 0) {305 err = __lmtt_alloc_range(lmtt, pt, offset, next);306 if (err)307 return err;308 }309 310 offset = next;311 }312 313 return 0;314}315 316static int lmtt_alloc_range(struct xe_lmtt *lmtt, unsigned int vfid, u64 start, u64 end)317{318 struct xe_lmtt_pt *pd = lmtt->pd;319 struct xe_lmtt_pt *pt;320 u64 pt_addr;321 u64 pde;322 int err;323 324 lmtt_assert(lmtt, pd->level > 0);325 lmtt_assert(lmtt, vfid <= lmtt->ops->lmtt_pte_num(pd->level));326 lmtt_assert(lmtt, IS_ALIGNED(start, lmtt_page_size(lmtt)));327 lmtt_assert(lmtt, IS_ALIGNED(end, lmtt_page_size(lmtt)));328 329 if (pd->entries[vfid])330 return -ENOTEMPTY;331 332 pt = lmtt_pt_alloc(lmtt, pd->level - 1);333 if (IS_ERR(pt))334 return PTR_ERR(pt);335 336 pt_addr = xe_bo_main_addr(pt->bo, XE_PAGE_SIZE);337 338 pde = lmtt->ops->lmtt_pte_encode(pt_addr, pd->level);339 340 lmtt_write_pte(lmtt, pd, pde, vfid);341 342 pd->entries[vfid] = pt;343 344 if (pt->level != 0) {345 err = __lmtt_alloc_range(lmtt, pt, start, end);346 if (err)347 goto out_free_pt;348 }349 350 return 0;351 352out_free_pt:353 lmtt_pt_free(pt);354 return err;355}356 357static struct xe_lmtt_pt *lmtt_leaf_pt(struct xe_lmtt *lmtt, unsigned int vfid, u64 addr)358{359 struct xe_lmtt_pt *pd = lmtt->pd;360 struct xe_lmtt_pt *pt;361 362 lmtt_assert(lmtt, vfid <= lmtt->ops->lmtt_pte_num(pd->level));363 pt = pd->entries[vfid];364 365 while (pt->level) {366 lmtt_assert(lmtt, lmtt->ops->lmtt_pte_index(addr, pt->level) <=367 lmtt->ops->lmtt_pte_num(pt->level));368 369 pt = pt->entries[lmtt->ops->lmtt_pte_index(addr, pt->level)];370 371 addr >>= lmtt->ops->lmtt_pte_shift(pt->level);372 }373 374 lmtt_assert(lmtt, lmtt->ops->lmtt_pte_index(addr, pt->level) <=375 lmtt->ops->lmtt_pte_num(pt->level));376 lmtt_assert(lmtt, pt->level != pd->level);377 lmtt_assert(lmtt, pt->level == 0);378 return pt;379}380 381static void lmtt_insert_bo(struct xe_lmtt *lmtt, unsigned int vfid, struct xe_bo *bo, u64 start)382{383 u64 page_size = lmtt_page_size(lmtt);384 struct xe_res_cursor cur;385 struct xe_lmtt_pt *pt;386 u64 addr, vram_offset;387 388 lmtt_assert(lmtt, IS_ALIGNED(start, page_size));389 lmtt_assert(lmtt, IS_ALIGNED(bo->size, page_size));390 lmtt_assert(lmtt, xe_bo_is_vram(bo));391 392 vram_offset = vram_region_gpu_offset(bo->ttm.resource);393 xe_res_first(bo->ttm.resource, 0, bo->size, &cur);394 while (cur.remaining) {395 addr = xe_res_dma(&cur);396 addr += vram_offset; /* XXX */397 398 pt = lmtt_leaf_pt(lmtt, vfid, start);399 400 lmtt_write_pte(lmtt, pt, lmtt->ops->lmtt_pte_encode(addr, 0),401 lmtt->ops->lmtt_pte_index(start, 0));402 403 xe_res_next(&cur, page_size);404 start += page_size;405 }406}407 408/**409 * xe_lmtt_prepare_pages - Create VF's LMTT Page Tables.410 * @lmtt: the &xe_lmtt to update411 * @vfid: the VF identifier (1-based)412 * @range: top range of LMEM offset to be supported413 *414 * This function creates empty LMTT page tables for given VF to support415 * up to maximum #range LMEM offset. The LMTT page tables created by this416 * function must be released using xe_lmtt_drop_pages() function.417 *418 * Notes:419 * This function shall be called only after successful LMTT initialization.420 * See xe_lmtt_init().421 *422 * Return: 0 on success or a negative error code on failure.423 */424int xe_lmtt_prepare_pages(struct xe_lmtt *lmtt, unsigned int vfid, u64 range)425{426 lmtt_assert(lmtt, lmtt->pd);427 lmtt_assert(lmtt, vfid);428 429 return lmtt_alloc_range(lmtt, vfid, 0, range);430}431 432/**433 * xe_lmtt_populate_pages - Update VF's LMTT Page Table Entries.434 * @lmtt: the &xe_lmtt to update435 * @vfid: the VF identifier (1-based)436 * @bo: the buffer object with LMEM allocation to be mapped437 * @offset: the offset at which #bo should be mapped438 *439 * This function updates VF's LMTT entries to use given buffer object as a backstore.440 *441 * Notes:442 * This function shall be called only after successful preparation of the443 * VF's LMTT Page Tables. See xe_lmtt_prepare().444 *445 * Return: 0 on success or a negative error code on failure.446 */447int xe_lmtt_populate_pages(struct xe_lmtt *lmtt, unsigned int vfid, struct xe_bo *bo, u64 offset)448{449 lmtt_assert(lmtt, lmtt->pd);450 lmtt_assert(lmtt, vfid);451 452 lmtt_insert_bo(lmtt, vfid, bo, offset);453 return 0;454}455 456/**457 * xe_lmtt_drop_pages - Remove VF's LMTT Pages.458 * @lmtt: the &xe_lmtt to update459 * @vfid: the VF identifier (1-based)460 *461 * This function removes all LMTT Page Tables prepared by xe_lmtt_prepare_pages().462 *463 * This function shall be called only after successful LMTT initialization.464 * See xe_lmtt_init().465 */466void xe_lmtt_drop_pages(struct xe_lmtt *lmtt, unsigned int vfid)467{468 lmtt_assert(lmtt, lmtt->pd);469 lmtt_assert(lmtt, vfid);470 471 lmtt_drop_pages(lmtt, vfid);472}473 474/**475 * xe_lmtt_estimate_pt_size - Estimate size of LMTT PT allocations.476 * @lmtt: the &xe_lmtt477 * @size: the size of the LMEM to be mapped over LMTT (including any offset)478 *479 * This function shall be called only by PF.480 *481 * Return: size of the PT allocation(s) needed to support given LMEM size.482 */483u64 xe_lmtt_estimate_pt_size(struct xe_lmtt *lmtt, u64 size)484{485 unsigned int level = 0;486 u64 pt_size;487 488 lmtt_assert(lmtt, IS_SRIOV_PF(lmtt_to_xe(lmtt)));489 lmtt_assert(lmtt, IS_DGFX(lmtt_to_xe(lmtt)));490 lmtt_assert(lmtt, lmtt->ops);491 492 pt_size = PAGE_ALIGN(lmtt->ops->lmtt_pte_size(level) *493 lmtt->ops->lmtt_pte_num(level));494 495 while (++level < lmtt->ops->lmtt_root_pd_level()) {496 pt_size *= lmtt->ops->lmtt_pte_index(size, level) + 1;497 pt_size += PAGE_ALIGN(lmtt->ops->lmtt_pte_size(level) *498 lmtt->ops->lmtt_pte_num(level));499 }500 501 return pt_size;502}503 504#if IS_BUILTIN(CONFIG_DRM_XE_KUNIT_TEST)505#include "tests/xe_lmtt_test.c"506#endif507