493 lines · c
1// SPDX-License-Identifier: MIT2/*3 * Copyright © 2021-2022 Intel Corporation4 * Copyright (C) 2021-2002 Red Hat5 */6 7#include <drm/drm_managed.h>8 9#include <drm/ttm/ttm_placement.h>10#include <drm/ttm/ttm_range_manager.h>11 12#include "xe_bo.h"13#include "xe_device.h"14#include "xe_gt.h"15#include "xe_res_cursor.h"16#include "xe_ttm_vram_mgr.h"17 18static inline struct drm_buddy_block *19xe_ttm_vram_mgr_first_block(struct list_head *list)20{21 return list_first_entry_or_null(list, struct drm_buddy_block, link);22}23 24static inline bool xe_is_vram_mgr_blocks_contiguous(struct drm_buddy *mm,25 struct list_head *head)26{27 struct drm_buddy_block *block;28 u64 start, size;29 30 block = xe_ttm_vram_mgr_first_block(head);31 if (!block)32 return false;33 34 while (head != block->link.next) {35 start = drm_buddy_block_offset(block);36 size = drm_buddy_block_size(mm, block);37 38 block = list_entry(block->link.next, struct drm_buddy_block,39 link);40 if (start + size != drm_buddy_block_offset(block))41 return false;42 }43 44 return true;45}46 47static int xe_ttm_vram_mgr_new(struct ttm_resource_manager *man,48 struct ttm_buffer_object *tbo,49 const struct ttm_place *place,50 struct ttm_resource **res)51{52 struct xe_ttm_vram_mgr *mgr = to_xe_ttm_vram_mgr(man);53 struct xe_ttm_vram_mgr_resource *vres;54 struct drm_buddy *mm = &mgr->mm;55 u64 size, remaining_size, min_page_size;56 unsigned long lpfn;57 int err;58 59 lpfn = place->lpfn;60 if (!lpfn || lpfn > man->size >> PAGE_SHIFT)61 lpfn = man->size >> PAGE_SHIFT;62 63 if (tbo->base.size >> PAGE_SHIFT > (lpfn - place->fpfn))64 return -E2BIG; /* don't trigger eviction for the impossible */65 66 vres = kzalloc(sizeof(*vres), GFP_KERNEL);67 if (!vres)68 return -ENOMEM;69 70 ttm_resource_init(tbo, place, &vres->base);71 72 /* bail out quickly if there's likely not enough VRAM for this BO */73 if (ttm_resource_manager_usage(man) > man->size) {74 err = -ENOSPC;75 goto error_fini;76 }77 78 INIT_LIST_HEAD(&vres->blocks);79 80 if (place->flags & TTM_PL_FLAG_TOPDOWN)81 vres->flags |= DRM_BUDDY_TOPDOWN_ALLOCATION;82 83 if (place->fpfn || lpfn != man->size >> PAGE_SHIFT)84 vres->flags |= DRM_BUDDY_RANGE_ALLOCATION;85 86 if (WARN_ON(!vres->base.size)) {87 err = -EINVAL;88 goto error_fini;89 }90 size = vres->base.size;91 92 min_page_size = mgr->default_page_size;93 if (tbo->page_alignment)94 min_page_size = (u64)tbo->page_alignment << PAGE_SHIFT;95 96 if (WARN_ON(min_page_size < mm->chunk_size)) {97 err = -EINVAL;98 goto error_fini;99 }100 101 if (WARN_ON(min_page_size > SZ_2G)) { /* FIXME: sg limit */102 err = -EINVAL;103 goto error_fini;104 }105 106 if (WARN_ON((size > SZ_2G &&107 (vres->base.placement & TTM_PL_FLAG_CONTIGUOUS)))) {108 err = -EINVAL;109 goto error_fini;110 }111 112 if (WARN_ON(!IS_ALIGNED(size, min_page_size))) {113 err = -EINVAL;114 goto error_fini;115 }116 117 mutex_lock(&mgr->lock);118 if (lpfn <= mgr->visible_size >> PAGE_SHIFT && size > mgr->visible_avail) {119 mutex_unlock(&mgr->lock);120 err = -ENOSPC;121 goto error_fini;122 }123 124 if (place->fpfn + (size >> PAGE_SHIFT) != place->lpfn &&125 place->flags & TTM_PL_FLAG_CONTIGUOUS) {126 size = roundup_pow_of_two(size);127 min_page_size = size;128 129 lpfn = max_t(unsigned long, place->fpfn + (size >> PAGE_SHIFT), lpfn);130 }131 132 remaining_size = size;133 do {134 /*135 * Limit maximum size to 2GiB due to SG table limitations.136 * FIXME: Should maybe be handled as part of sg construction.137 */138 u64 alloc_size = min_t(u64, remaining_size, SZ_2G);139 140 err = drm_buddy_alloc_blocks(mm, (u64)place->fpfn << PAGE_SHIFT,141 (u64)lpfn << PAGE_SHIFT,142 alloc_size,143 min_page_size,144 &vres->blocks,145 vres->flags);146 if (err)147 goto error_free_blocks;148 149 remaining_size -= alloc_size;150 } while (remaining_size);151 152 if (place->flags & TTM_PL_FLAG_CONTIGUOUS) {153 if (!drm_buddy_block_trim(mm, NULL, vres->base.size, &vres->blocks))154 size = vres->base.size;155 }156 157 if (lpfn <= mgr->visible_size >> PAGE_SHIFT) {158 vres->used_visible_size = size;159 } else {160 struct drm_buddy_block *block;161 162 list_for_each_entry(block, &vres->blocks, link) {163 u64 start = drm_buddy_block_offset(block);164 165 if (start < mgr->visible_size) {166 u64 end = start + drm_buddy_block_size(mm, block);167 168 vres->used_visible_size +=169 min(end, mgr->visible_size) - start;170 }171 }172 }173 174 mgr->visible_avail -= vres->used_visible_size;175 mutex_unlock(&mgr->lock);176 177 if (!(vres->base.placement & TTM_PL_FLAG_CONTIGUOUS) &&178 xe_is_vram_mgr_blocks_contiguous(mm, &vres->blocks))179 vres->base.placement |= TTM_PL_FLAG_CONTIGUOUS;180 181 /*182 * For some kernel objects we still rely on the start when io mapping183 * the object.184 */185 if (vres->base.placement & TTM_PL_FLAG_CONTIGUOUS) {186 struct drm_buddy_block *block = list_first_entry(&vres->blocks,187 typeof(*block),188 link);189 190 vres->base.start = drm_buddy_block_offset(block) >> PAGE_SHIFT;191 } else {192 vres->base.start = XE_BO_INVALID_OFFSET;193 }194 195 *res = &vres->base;196 return 0;197 198error_free_blocks:199 drm_buddy_free_list(mm, &vres->blocks, 0);200 mutex_unlock(&mgr->lock);201error_fini:202 ttm_resource_fini(man, &vres->base);203 kfree(vres);204 205 return err;206}207 208static void xe_ttm_vram_mgr_del(struct ttm_resource_manager *man,209 struct ttm_resource *res)210{211 struct xe_ttm_vram_mgr_resource *vres =212 to_xe_ttm_vram_mgr_resource(res);213 struct xe_ttm_vram_mgr *mgr = to_xe_ttm_vram_mgr(man);214 struct drm_buddy *mm = &mgr->mm;215 216 mutex_lock(&mgr->lock);217 drm_buddy_free_list(mm, &vres->blocks, 0);218 mgr->visible_avail += vres->used_visible_size;219 mutex_unlock(&mgr->lock);220 221 ttm_resource_fini(man, res);222 223 kfree(vres);224}225 226static void xe_ttm_vram_mgr_debug(struct ttm_resource_manager *man,227 struct drm_printer *printer)228{229 struct xe_ttm_vram_mgr *mgr = to_xe_ttm_vram_mgr(man);230 struct drm_buddy *mm = &mgr->mm;231 232 mutex_lock(&mgr->lock);233 drm_printf(printer, "default_page_size: %lluKiB\n",234 mgr->default_page_size >> 10);235 drm_printf(printer, "visible_avail: %lluMiB\n",236 (u64)mgr->visible_avail >> 20);237 drm_printf(printer, "visible_size: %lluMiB\n",238 (u64)mgr->visible_size >> 20);239 240 drm_buddy_print(mm, printer);241 mutex_unlock(&mgr->lock);242 drm_printf(printer, "man size:%llu\n", man->size);243}244 245static bool xe_ttm_vram_mgr_intersects(struct ttm_resource_manager *man,246 struct ttm_resource *res,247 const struct ttm_place *place,248 size_t size)249{250 struct xe_ttm_vram_mgr *mgr = to_xe_ttm_vram_mgr(man);251 struct xe_ttm_vram_mgr_resource *vres =252 to_xe_ttm_vram_mgr_resource(res);253 struct drm_buddy *mm = &mgr->mm;254 struct drm_buddy_block *block;255 256 if (!place->fpfn && !place->lpfn)257 return true;258 259 if (!place->fpfn && place->lpfn == mgr->visible_size >> PAGE_SHIFT)260 return vres->used_visible_size > 0;261 262 list_for_each_entry(block, &vres->blocks, link) {263 unsigned long fpfn =264 drm_buddy_block_offset(block) >> PAGE_SHIFT;265 unsigned long lpfn = fpfn +266 (drm_buddy_block_size(mm, block) >> PAGE_SHIFT);267 268 if (place->fpfn < lpfn && place->lpfn > fpfn)269 return true;270 }271 272 return false;273}274 275static bool xe_ttm_vram_mgr_compatible(struct ttm_resource_manager *man,276 struct ttm_resource *res,277 const struct ttm_place *place,278 size_t size)279{280 struct xe_ttm_vram_mgr *mgr = to_xe_ttm_vram_mgr(man);281 struct xe_ttm_vram_mgr_resource *vres =282 to_xe_ttm_vram_mgr_resource(res);283 struct drm_buddy *mm = &mgr->mm;284 struct drm_buddy_block *block;285 286 if (!place->fpfn && !place->lpfn)287 return true;288 289 if (!place->fpfn && place->lpfn == mgr->visible_size >> PAGE_SHIFT)290 return vres->used_visible_size == size;291 292 list_for_each_entry(block, &vres->blocks, link) {293 unsigned long fpfn =294 drm_buddy_block_offset(block) >> PAGE_SHIFT;295 unsigned long lpfn = fpfn +296 (drm_buddy_block_size(mm, block) >> PAGE_SHIFT);297 298 if (fpfn < place->fpfn || lpfn > place->lpfn)299 return false;300 }301 302 return true;303}304 305static const struct ttm_resource_manager_func xe_ttm_vram_mgr_func = {306 .alloc = xe_ttm_vram_mgr_new,307 .free = xe_ttm_vram_mgr_del,308 .intersects = xe_ttm_vram_mgr_intersects,309 .compatible = xe_ttm_vram_mgr_compatible,310 .debug = xe_ttm_vram_mgr_debug311};312 313static void ttm_vram_mgr_fini(struct drm_device *dev, void *arg)314{315 struct xe_device *xe = to_xe_device(dev);316 struct xe_ttm_vram_mgr *mgr = arg;317 struct ttm_resource_manager *man = &mgr->manager;318 319 ttm_resource_manager_set_used(man, false);320 321 if (ttm_resource_manager_evict_all(&xe->ttm, man))322 return;323 324 WARN_ON_ONCE(mgr->visible_avail != mgr->visible_size);325 326 drm_buddy_fini(&mgr->mm);327 328 ttm_resource_manager_cleanup(&mgr->manager);329 330 ttm_set_driver_manager(&xe->ttm, mgr->mem_type, NULL);331 332 mutex_destroy(&mgr->lock);333}334 335int __xe_ttm_vram_mgr_init(struct xe_device *xe, struct xe_ttm_vram_mgr *mgr,336 u32 mem_type, u64 size, u64 io_size,337 u64 default_page_size)338{339 struct ttm_resource_manager *man = &mgr->manager;340 int err;341 342 man->func = &xe_ttm_vram_mgr_func;343 mgr->mem_type = mem_type;344 mutex_init(&mgr->lock);345 mgr->default_page_size = default_page_size;346 mgr->visible_size = io_size;347 mgr->visible_avail = io_size;348 349 ttm_resource_manager_init(man, &xe->ttm, size);350 err = drm_buddy_init(&mgr->mm, man->size, default_page_size);351 if (err)352 return err;353 354 ttm_set_driver_manager(&xe->ttm, mem_type, &mgr->manager);355 ttm_resource_manager_set_used(&mgr->manager, true);356 357 return drmm_add_action_or_reset(&xe->drm, ttm_vram_mgr_fini, mgr);358}359 360int xe_ttm_vram_mgr_init(struct xe_tile *tile, struct xe_ttm_vram_mgr *mgr)361{362 struct xe_device *xe = tile_to_xe(tile);363 struct xe_mem_region *vram = &tile->mem.vram;364 365 mgr->vram = vram;366 return __xe_ttm_vram_mgr_init(xe, mgr, XE_PL_VRAM0 + tile->id,367 vram->usable_size, vram->io_size,368 PAGE_SIZE);369}370 371int xe_ttm_vram_mgr_alloc_sgt(struct xe_device *xe,372 struct ttm_resource *res,373 u64 offset, u64 length,374 struct device *dev,375 enum dma_data_direction dir,376 struct sg_table **sgt)377{378 struct xe_tile *tile = &xe->tiles[res->mem_type - XE_PL_VRAM0];379 struct xe_ttm_vram_mgr_resource *vres = to_xe_ttm_vram_mgr_resource(res);380 struct xe_res_cursor cursor;381 struct scatterlist *sg;382 int num_entries = 0;383 int i, r;384 385 if (vres->used_visible_size < res->size)386 return -EOPNOTSUPP;387 388 *sgt = kmalloc(sizeof(**sgt), GFP_KERNEL);389 if (!*sgt)390 return -ENOMEM;391 392 /* Determine the number of DRM_BUDDY blocks to export */393 xe_res_first(res, offset, length, &cursor);394 while (cursor.remaining) {395 num_entries++;396 xe_res_next(&cursor, cursor.size);397 }398 399 r = sg_alloc_table(*sgt, num_entries, GFP_KERNEL);400 if (r)401 goto error_free;402 403 /* Initialize scatterlist nodes of sg_table */404 for_each_sgtable_sg((*sgt), sg, i)405 sg->length = 0;406 407 /*408 * Walk down DRM_BUDDY blocks to populate scatterlist nodes409 * @note: Use iterator api to get first the DRM_BUDDY block410 * and the number of bytes from it. Access the following411 * DRM_BUDDY block(s) if more buffer needs to exported412 */413 xe_res_first(res, offset, length, &cursor);414 for_each_sgtable_sg((*sgt), sg, i) {415 phys_addr_t phys = cursor.start + tile->mem.vram.io_start;416 size_t size = cursor.size;417 dma_addr_t addr;418 419 addr = dma_map_resource(dev, phys, size, dir,420 DMA_ATTR_SKIP_CPU_SYNC);421 r = dma_mapping_error(dev, addr);422 if (r)423 goto error_unmap;424 425 sg_set_page(sg, NULL, size, 0);426 sg_dma_address(sg) = addr;427 sg_dma_len(sg) = size;428 429 xe_res_next(&cursor, cursor.size);430 }431 432 return 0;433 434error_unmap:435 for_each_sgtable_sg((*sgt), sg, i) {436 if (!sg->length)437 continue;438 439 dma_unmap_resource(dev, sg->dma_address,440 sg->length, dir,441 DMA_ATTR_SKIP_CPU_SYNC);442 }443 sg_free_table(*sgt);444 445error_free:446 kfree(*sgt);447 return r;448}449 450void xe_ttm_vram_mgr_free_sgt(struct device *dev, enum dma_data_direction dir,451 struct sg_table *sgt)452{453 struct scatterlist *sg;454 int i;455 456 for_each_sgtable_sg(sgt, sg, i)457 dma_unmap_resource(dev, sg->dma_address,458 sg->length, dir,459 DMA_ATTR_SKIP_CPU_SYNC);460 sg_free_table(sgt);461 kfree(sgt);462}463 464u64 xe_ttm_vram_get_cpu_visible_size(struct ttm_resource_manager *man)465{466 struct xe_ttm_vram_mgr *mgr = to_xe_ttm_vram_mgr(man);467 468 return mgr->visible_size;469}470 471void xe_ttm_vram_get_used(struct ttm_resource_manager *man,472 u64 *used, u64 *used_visible)473{474 struct xe_ttm_vram_mgr *mgr = to_xe_ttm_vram_mgr(man);475 476 mutex_lock(&mgr->lock);477 *used = mgr->mm.size - mgr->mm.avail;478 *used_visible = mgr->visible_size - mgr->visible_avail;479 mutex_unlock(&mgr->lock);480}481 482u64 xe_ttm_vram_get_avail(struct ttm_resource_manager *man)483{484 struct xe_ttm_vram_mgr *mgr = to_xe_ttm_vram_mgr(man);485 u64 avail;486 487 mutex_lock(&mgr->lock);488 avail = mgr->mm.avail;489 mutex_unlock(&mgr->lock);490 491 return avail;492}493