2395 lines · c
1// SPDX-License-Identifier: MIT2/*3 * Copyright © 2021 Intel Corporation4 */5 6#include "xe_bo.h"7 8#include <linux/dma-buf.h>9 10#include <drm/drm_drv.h>11#include <drm/drm_gem_ttm_helper.h>12#include <drm/drm_managed.h>13#include <drm/ttm/ttm_device.h>14#include <drm/ttm/ttm_placement.h>15#include <drm/ttm/ttm_tt.h>16#include <uapi/drm/xe_drm.h>17 18#include "xe_device.h"19#include "xe_dma_buf.h"20#include "xe_drm_client.h"21#include "xe_ggtt.h"22#include "xe_gt.h"23#include "xe_map.h"24#include "xe_migrate.h"25#include "xe_pm.h"26#include "xe_preempt_fence.h"27#include "xe_res_cursor.h"28#include "xe_trace_bo.h"29#include "xe_ttm_stolen_mgr.h"30#include "xe_vm.h"31 32const char *const xe_mem_type_to_name[TTM_NUM_MEM_TYPES] = {33 [XE_PL_SYSTEM] = "system",34 [XE_PL_TT] = "gtt",35 [XE_PL_VRAM0] = "vram0",36 [XE_PL_VRAM1] = "vram1",37 [XE_PL_STOLEN] = "stolen"38};39 40static const struct ttm_place sys_placement_flags = {41 .fpfn = 0,42 .lpfn = 0,43 .mem_type = XE_PL_SYSTEM,44 .flags = 0,45};46 47static struct ttm_placement sys_placement = {48 .num_placement = 1,49 .placement = &sys_placement_flags,50};51 52static const struct ttm_place tt_placement_flags[] = {53 {54 .fpfn = 0,55 .lpfn = 0,56 .mem_type = XE_PL_TT,57 .flags = TTM_PL_FLAG_DESIRED,58 },59 {60 .fpfn = 0,61 .lpfn = 0,62 .mem_type = XE_PL_SYSTEM,63 .flags = TTM_PL_FLAG_FALLBACK,64 }65};66 67static struct ttm_placement tt_placement = {68 .num_placement = 2,69 .placement = tt_placement_flags,70};71 72bool mem_type_is_vram(u32 mem_type)73{74 return mem_type >= XE_PL_VRAM0 && mem_type != XE_PL_STOLEN;75}76 77static bool resource_is_stolen_vram(struct xe_device *xe, struct ttm_resource *res)78{79 return res->mem_type == XE_PL_STOLEN && IS_DGFX(xe);80}81 82static bool resource_is_vram(struct ttm_resource *res)83{84 return mem_type_is_vram(res->mem_type);85}86 87bool xe_bo_is_vram(struct xe_bo *bo)88{89 return resource_is_vram(bo->ttm.resource) ||90 resource_is_stolen_vram(xe_bo_device(bo), bo->ttm.resource);91}92 93bool xe_bo_is_stolen(struct xe_bo *bo)94{95 return bo->ttm.resource->mem_type == XE_PL_STOLEN;96}97 98/**99 * xe_bo_has_single_placement - check if BO is placed only in one memory location100 * @bo: The BO101 *102 * This function checks whether a given BO is placed in only one memory location.103 *104 * Returns: true if the BO is placed in a single memory location, false otherwise.105 *106 */107bool xe_bo_has_single_placement(struct xe_bo *bo)108{109 return bo->placement.num_placement == 1;110}111 112/**113 * xe_bo_is_stolen_devmem - check if BO is of stolen type accessed via PCI BAR114 * @bo: The BO115 *116 * The stolen memory is accessed through the PCI BAR for both DGFX and some117 * integrated platforms that have a dedicated bit in the PTE for devmem (DM).118 *119 * Returns: true if it's stolen memory accessed via PCI BAR, false otherwise.120 */121bool xe_bo_is_stolen_devmem(struct xe_bo *bo)122{123 return xe_bo_is_stolen(bo) &&124 GRAPHICS_VERx100(xe_bo_device(bo)) >= 1270;125}126 127static bool xe_bo_is_user(struct xe_bo *bo)128{129 return bo->flags & XE_BO_FLAG_USER;130}131 132static struct xe_migrate *133mem_type_to_migrate(struct xe_device *xe, u32 mem_type)134{135 struct xe_tile *tile;136 137 xe_assert(xe, mem_type == XE_PL_STOLEN || mem_type_is_vram(mem_type));138 tile = &xe->tiles[mem_type == XE_PL_STOLEN ? 0 : (mem_type - XE_PL_VRAM0)];139 return tile->migrate;140}141 142static struct xe_mem_region *res_to_mem_region(struct ttm_resource *res)143{144 struct xe_device *xe = ttm_to_xe_device(res->bo->bdev);145 struct ttm_resource_manager *mgr;146 147 xe_assert(xe, resource_is_vram(res));148 mgr = ttm_manager_type(&xe->ttm, res->mem_type);149 return to_xe_ttm_vram_mgr(mgr)->vram;150}151 152static void try_add_system(struct xe_device *xe, struct xe_bo *bo,153 u32 bo_flags, u32 *c)154{155 if (bo_flags & XE_BO_FLAG_SYSTEM) {156 xe_assert(xe, *c < ARRAY_SIZE(bo->placements));157 158 bo->placements[*c] = (struct ttm_place) {159 .mem_type = XE_PL_TT,160 };161 *c += 1;162 }163}164 165static void add_vram(struct xe_device *xe, struct xe_bo *bo,166 struct ttm_place *places, u32 bo_flags, u32 mem_type, u32 *c)167{168 struct ttm_place place = { .mem_type = mem_type };169 struct xe_mem_region *vram;170 u64 io_size;171 172 xe_assert(xe, *c < ARRAY_SIZE(bo->placements));173 174 vram = to_xe_ttm_vram_mgr(ttm_manager_type(&xe->ttm, mem_type))->vram;175 xe_assert(xe, vram && vram->usable_size);176 io_size = vram->io_size;177 178 /*179 * For eviction / restore on suspend / resume objects180 * pinned in VRAM must be contiguous181 */182 if (bo_flags & (XE_BO_FLAG_PINNED |183 XE_BO_FLAG_GGTT))184 place.flags |= TTM_PL_FLAG_CONTIGUOUS;185 186 if (io_size < vram->usable_size) {187 if (bo_flags & XE_BO_FLAG_NEEDS_CPU_ACCESS) {188 place.fpfn = 0;189 place.lpfn = io_size >> PAGE_SHIFT;190 } else {191 place.flags |= TTM_PL_FLAG_TOPDOWN;192 }193 }194 places[*c] = place;195 *c += 1;196}197 198static void try_add_vram(struct xe_device *xe, struct xe_bo *bo,199 u32 bo_flags, u32 *c)200{201 if (bo_flags & XE_BO_FLAG_VRAM0)202 add_vram(xe, bo, bo->placements, bo_flags, XE_PL_VRAM0, c);203 if (bo_flags & XE_BO_FLAG_VRAM1)204 add_vram(xe, bo, bo->placements, bo_flags, XE_PL_VRAM1, c);205}206 207static void try_add_stolen(struct xe_device *xe, struct xe_bo *bo,208 u32 bo_flags, u32 *c)209{210 if (bo_flags & XE_BO_FLAG_STOLEN) {211 xe_assert(xe, *c < ARRAY_SIZE(bo->placements));212 213 bo->placements[*c] = (struct ttm_place) {214 .mem_type = XE_PL_STOLEN,215 .flags = bo_flags & (XE_BO_FLAG_PINNED |216 XE_BO_FLAG_GGTT) ?217 TTM_PL_FLAG_CONTIGUOUS : 0,218 };219 *c += 1;220 }221}222 223static int __xe_bo_placement_for_flags(struct xe_device *xe, struct xe_bo *bo,224 u32 bo_flags)225{226 u32 c = 0;227 228 try_add_vram(xe, bo, bo_flags, &c);229 try_add_system(xe, bo, bo_flags, &c);230 try_add_stolen(xe, bo, bo_flags, &c);231 232 if (!c)233 return -EINVAL;234 235 bo->placement = (struct ttm_placement) {236 .num_placement = c,237 .placement = bo->placements,238 };239 240 return 0;241}242 243int xe_bo_placement_for_flags(struct xe_device *xe, struct xe_bo *bo,244 u32 bo_flags)245{246 xe_bo_assert_held(bo);247 return __xe_bo_placement_for_flags(xe, bo, bo_flags);248}249 250static void xe_evict_flags(struct ttm_buffer_object *tbo,251 struct ttm_placement *placement)252{253 if (!xe_bo_is_xe_bo(tbo)) {254 /* Don't handle scatter gather BOs */255 if (tbo->type == ttm_bo_type_sg) {256 placement->num_placement = 0;257 return;258 }259 260 *placement = sys_placement;261 return;262 }263 264 /*265 * For xe, sg bos that are evicted to system just triggers a266 * rebind of the sg list upon subsequent validation to XE_PL_TT.267 */268 switch (tbo->resource->mem_type) {269 case XE_PL_VRAM0:270 case XE_PL_VRAM1:271 case XE_PL_STOLEN:272 *placement = tt_placement;273 break;274 case XE_PL_TT:275 default:276 *placement = sys_placement;277 break;278 }279}280 281struct xe_ttm_tt {282 struct ttm_tt ttm;283 struct device *dev;284 struct sg_table sgt;285 struct sg_table *sg;286};287 288static int xe_tt_map_sg(struct ttm_tt *tt)289{290 struct xe_ttm_tt *xe_tt = container_of(tt, struct xe_ttm_tt, ttm);291 unsigned long num_pages = tt->num_pages;292 int ret;293 294 XE_WARN_ON(tt->page_flags & TTM_TT_FLAG_EXTERNAL);295 296 if (xe_tt->sg)297 return 0;298 299 ret = sg_alloc_table_from_pages_segment(&xe_tt->sgt, tt->pages,300 num_pages, 0,301 (u64)num_pages << PAGE_SHIFT,302 xe_sg_segment_size(xe_tt->dev),303 GFP_KERNEL);304 if (ret)305 return ret;306 307 xe_tt->sg = &xe_tt->sgt;308 ret = dma_map_sgtable(xe_tt->dev, xe_tt->sg, DMA_BIDIRECTIONAL,309 DMA_ATTR_SKIP_CPU_SYNC);310 if (ret) {311 sg_free_table(xe_tt->sg);312 xe_tt->sg = NULL;313 return ret;314 }315 316 return 0;317}318 319static void xe_tt_unmap_sg(struct ttm_tt *tt)320{321 struct xe_ttm_tt *xe_tt = container_of(tt, struct xe_ttm_tt, ttm);322 323 if (xe_tt->sg) {324 dma_unmap_sgtable(xe_tt->dev, xe_tt->sg,325 DMA_BIDIRECTIONAL, 0);326 sg_free_table(xe_tt->sg);327 xe_tt->sg = NULL;328 }329}330 331struct sg_table *xe_bo_sg(struct xe_bo *bo)332{333 struct ttm_tt *tt = bo->ttm.ttm;334 struct xe_ttm_tt *xe_tt = container_of(tt, struct xe_ttm_tt, ttm);335 336 return xe_tt->sg;337}338 339static struct ttm_tt *xe_ttm_tt_create(struct ttm_buffer_object *ttm_bo,340 u32 page_flags)341{342 struct xe_bo *bo = ttm_to_xe_bo(ttm_bo);343 struct xe_device *xe = xe_bo_device(bo);344 struct xe_ttm_tt *tt;345 unsigned long extra_pages;346 enum ttm_caching caching = ttm_cached;347 int err;348 349 tt = kzalloc(sizeof(*tt), GFP_KERNEL);350 if (!tt)351 return NULL;352 353 tt->dev = xe->drm.dev;354 355 extra_pages = 0;356 if (xe_bo_needs_ccs_pages(bo))357 extra_pages = DIV_ROUND_UP(xe_device_ccs_bytes(xe, bo->size),358 PAGE_SIZE);359 360 /*361 * DGFX system memory is always WB / ttm_cached, since362 * other caching modes are only supported on x86. DGFX363 * GPU system memory accesses are always coherent with the364 * CPU.365 */366 if (!IS_DGFX(xe)) {367 switch (bo->cpu_caching) {368 case DRM_XE_GEM_CPU_CACHING_WC:369 caching = ttm_write_combined;370 break;371 default:372 caching = ttm_cached;373 break;374 }375 376 WARN_ON((bo->flags & XE_BO_FLAG_USER) && !bo->cpu_caching);377 378 /*379 * Display scanout is always non-coherent with the CPU cache.380 *381 * For Xe_LPG and beyond, PPGTT PTE lookups are also382 * non-coherent and require a CPU:WC mapping.383 */384 if ((!bo->cpu_caching && bo->flags & XE_BO_FLAG_SCANOUT) ||385 (xe->info.graphics_verx100 >= 1270 &&386 bo->flags & XE_BO_FLAG_PAGETABLE))387 caching = ttm_write_combined;388 }389 390 if (bo->flags & XE_BO_FLAG_NEEDS_UC) {391 /*392 * Valid only for internally-created buffers only, for393 * which cpu_caching is never initialized.394 */395 xe_assert(xe, bo->cpu_caching == 0);396 caching = ttm_uncached;397 }398 399 err = ttm_tt_init(&tt->ttm, &bo->ttm, page_flags, caching, extra_pages);400 if (err) {401 kfree(tt);402 return NULL;403 }404 405 return &tt->ttm;406}407 408static int xe_ttm_tt_populate(struct ttm_device *ttm_dev, struct ttm_tt *tt,409 struct ttm_operation_ctx *ctx)410{411 int err;412 413 /*414 * dma-bufs are not populated with pages, and the dma-415 * addresses are set up when moved to XE_PL_TT.416 */417 if (tt->page_flags & TTM_TT_FLAG_EXTERNAL)418 return 0;419 420 err = ttm_pool_alloc(&ttm_dev->pool, tt, ctx);421 if (err)422 return err;423 424 return err;425}426 427static void xe_ttm_tt_unpopulate(struct ttm_device *ttm_dev, struct ttm_tt *tt)428{429 if (tt->page_flags & TTM_TT_FLAG_EXTERNAL)430 return;431 432 xe_tt_unmap_sg(tt);433 434 return ttm_pool_free(&ttm_dev->pool, tt);435}436 437static void xe_ttm_tt_destroy(struct ttm_device *ttm_dev, struct ttm_tt *tt)438{439 ttm_tt_fini(tt);440 kfree(tt);441}442 443static int xe_ttm_io_mem_reserve(struct ttm_device *bdev,444 struct ttm_resource *mem)445{446 struct xe_device *xe = ttm_to_xe_device(bdev);447 448 switch (mem->mem_type) {449 case XE_PL_SYSTEM:450 case XE_PL_TT:451 return 0;452 case XE_PL_VRAM0:453 case XE_PL_VRAM1: {454 struct xe_ttm_vram_mgr_resource *vres =455 to_xe_ttm_vram_mgr_resource(mem);456 struct xe_mem_region *vram = res_to_mem_region(mem);457 458 if (vres->used_visible_size < mem->size)459 return -EINVAL;460 461 mem->bus.offset = mem->start << PAGE_SHIFT;462 463 if (vram->mapping &&464 mem->placement & TTM_PL_FLAG_CONTIGUOUS)465 mem->bus.addr = (u8 __force *)vram->mapping +466 mem->bus.offset;467 468 mem->bus.offset += vram->io_start;469 mem->bus.is_iomem = true;470 471#if !defined(CONFIG_X86)472 mem->bus.caching = ttm_write_combined;473#endif474 return 0;475 } case XE_PL_STOLEN:476 return xe_ttm_stolen_io_mem_reserve(xe, mem);477 default:478 return -EINVAL;479 }480}481 482static int xe_bo_trigger_rebind(struct xe_device *xe, struct xe_bo *bo,483 const struct ttm_operation_ctx *ctx)484{485 struct dma_resv_iter cursor;486 struct dma_fence *fence;487 struct drm_gem_object *obj = &bo->ttm.base;488 struct drm_gpuvm_bo *vm_bo;489 bool idle = false;490 int ret = 0;491 492 dma_resv_assert_held(bo->ttm.base.resv);493 494 if (!list_empty(&bo->ttm.base.gpuva.list)) {495 dma_resv_iter_begin(&cursor, bo->ttm.base.resv,496 DMA_RESV_USAGE_BOOKKEEP);497 dma_resv_for_each_fence_unlocked(&cursor, fence)498 dma_fence_enable_sw_signaling(fence);499 dma_resv_iter_end(&cursor);500 }501 502 drm_gem_for_each_gpuvm_bo(vm_bo, obj) {503 struct xe_vm *vm = gpuvm_to_vm(vm_bo->vm);504 struct drm_gpuva *gpuva;505 506 if (!xe_vm_in_fault_mode(vm)) {507 drm_gpuvm_bo_evict(vm_bo, true);508 continue;509 }510 511 if (!idle) {512 long timeout;513 514 if (ctx->no_wait_gpu &&515 !dma_resv_test_signaled(bo->ttm.base.resv,516 DMA_RESV_USAGE_BOOKKEEP))517 return -EBUSY;518 519 timeout = dma_resv_wait_timeout(bo->ttm.base.resv,520 DMA_RESV_USAGE_BOOKKEEP,521 ctx->interruptible,522 MAX_SCHEDULE_TIMEOUT);523 if (!timeout)524 return -ETIME;525 if (timeout < 0)526 return timeout;527 528 idle = true;529 }530 531 drm_gpuvm_bo_for_each_va(gpuva, vm_bo) {532 struct xe_vma *vma = gpuva_to_vma(gpuva);533 534 trace_xe_vma_evict(vma);535 ret = xe_vm_invalidate_vma(vma);536 if (XE_WARN_ON(ret))537 return ret;538 }539 }540 541 return ret;542}543 544/*545 * The dma-buf map_attachment() / unmap_attachment() is hooked up here.546 * Note that unmapping the attachment is deferred to the next547 * map_attachment time, or to bo destroy (after idling) whichever comes first.548 * This is to avoid syncing before unmap_attachment(), assuming that the549 * caller relies on idling the reservation object before moving the550 * backing store out. Should that assumption not hold, then we will be able551 * to unconditionally call unmap_attachment() when moving out to system.552 */553static int xe_bo_move_dmabuf(struct ttm_buffer_object *ttm_bo,554 struct ttm_resource *new_res)555{556 struct dma_buf_attachment *attach = ttm_bo->base.import_attach;557 struct xe_ttm_tt *xe_tt = container_of(ttm_bo->ttm, struct xe_ttm_tt,558 ttm);559 struct xe_device *xe = ttm_to_xe_device(ttm_bo->bdev);560 struct sg_table *sg;561 562 xe_assert(xe, attach);563 xe_assert(xe, ttm_bo->ttm);564 565 if (new_res->mem_type == XE_PL_SYSTEM)566 goto out;567 568 if (ttm_bo->sg) {569 dma_buf_unmap_attachment(attach, ttm_bo->sg, DMA_BIDIRECTIONAL);570 ttm_bo->sg = NULL;571 }572 573 sg = dma_buf_map_attachment(attach, DMA_BIDIRECTIONAL);574 if (IS_ERR(sg))575 return PTR_ERR(sg);576 577 ttm_bo->sg = sg;578 xe_tt->sg = sg;579 580out:581 ttm_bo_move_null(ttm_bo, new_res);582 583 return 0;584}585 586/**587 * xe_bo_move_notify - Notify subsystems of a pending move588 * @bo: The buffer object589 * @ctx: The struct ttm_operation_ctx controlling locking and waits.590 *591 * This function notifies subsystems of an upcoming buffer move.592 * Upon receiving such a notification, subsystems should schedule593 * halting access to the underlying pages and optionally add a fence594 * to the buffer object's dma_resv object, that signals when access is595 * stopped. The caller will wait on all dma_resv fences before596 * starting the move.597 *598 * A subsystem may commence access to the object after obtaining599 * bindings to the new backing memory under the object lock.600 *601 * Return: 0 on success, -EINTR or -ERESTARTSYS if interrupted in fault mode,602 * negative error code on error.603 */604static int xe_bo_move_notify(struct xe_bo *bo,605 const struct ttm_operation_ctx *ctx)606{607 struct ttm_buffer_object *ttm_bo = &bo->ttm;608 struct xe_device *xe = ttm_to_xe_device(ttm_bo->bdev);609 struct ttm_resource *old_mem = ttm_bo->resource;610 u32 old_mem_type = old_mem ? old_mem->mem_type : XE_PL_SYSTEM;611 int ret;612 613 /*614 * If this starts to call into many components, consider615 * using a notification chain here.616 */617 618 if (xe_bo_is_pinned(bo))619 return -EINVAL;620 621 xe_bo_vunmap(bo);622 ret = xe_bo_trigger_rebind(xe, bo, ctx);623 if (ret)624 return ret;625 626 /* Don't call move_notify() for imported dma-bufs. */627 if (ttm_bo->base.dma_buf && !ttm_bo->base.import_attach)628 dma_buf_move_notify(ttm_bo->base.dma_buf);629 630 /*631 * TTM has already nuked the mmap for us (see ttm_bo_unmap_virtual),632 * so if we moved from VRAM make sure to unlink this from the userfault633 * tracking.634 */635 if (mem_type_is_vram(old_mem_type)) {636 mutex_lock(&xe->mem_access.vram_userfault.lock);637 if (!list_empty(&bo->vram_userfault_link))638 list_del_init(&bo->vram_userfault_link);639 mutex_unlock(&xe->mem_access.vram_userfault.lock);640 }641 642 return 0;643}644 645static int xe_bo_move(struct ttm_buffer_object *ttm_bo, bool evict,646 struct ttm_operation_ctx *ctx,647 struct ttm_resource *new_mem,648 struct ttm_place *hop)649{650 struct xe_device *xe = ttm_to_xe_device(ttm_bo->bdev);651 struct xe_bo *bo = ttm_to_xe_bo(ttm_bo);652 struct ttm_resource *old_mem = ttm_bo->resource;653 u32 old_mem_type = old_mem ? old_mem->mem_type : XE_PL_SYSTEM;654 struct ttm_tt *ttm = ttm_bo->ttm;655 struct xe_migrate *migrate = NULL;656 struct dma_fence *fence;657 bool move_lacks_source;658 bool tt_has_data;659 bool needs_clear;660 bool handle_system_ccs = (!IS_DGFX(xe) && xe_bo_needs_ccs_pages(bo) &&661 ttm && ttm_tt_is_populated(ttm)) ? true : false;662 int ret = 0;663 664 /* Bo creation path, moving to system or TT. */665 if ((!old_mem && ttm) && !handle_system_ccs) {666 if (new_mem->mem_type == XE_PL_TT)667 ret = xe_tt_map_sg(ttm);668 if (!ret)669 ttm_bo_move_null(ttm_bo, new_mem);670 goto out;671 }672 673 if (ttm_bo->type == ttm_bo_type_sg) {674 ret = xe_bo_move_notify(bo, ctx);675 if (!ret)676 ret = xe_bo_move_dmabuf(ttm_bo, new_mem);677 return ret;678 }679 680 tt_has_data = ttm && (ttm_tt_is_populated(ttm) ||681 (ttm->page_flags & TTM_TT_FLAG_SWAPPED));682 683 move_lacks_source = !old_mem || (handle_system_ccs ? (!bo->ccs_cleared) :684 (!mem_type_is_vram(old_mem_type) && !tt_has_data));685 686 needs_clear = (ttm && ttm->page_flags & TTM_TT_FLAG_ZERO_ALLOC) ||687 (!ttm && ttm_bo->type == ttm_bo_type_device);688 689 if (new_mem->mem_type == XE_PL_TT) {690 ret = xe_tt_map_sg(ttm);691 if (ret)692 goto out;693 }694 695 if ((move_lacks_source && !needs_clear)) {696 ttm_bo_move_null(ttm_bo, new_mem);697 goto out;698 }699 700 if (old_mem_type == XE_PL_SYSTEM && new_mem->mem_type == XE_PL_TT && !handle_system_ccs) {701 ttm_bo_move_null(ttm_bo, new_mem);702 goto out;703 }704 705 /*706 * Failed multi-hop where the old_mem is still marked as707 * TTM_PL_FLAG_TEMPORARY, should just be a dummy move.708 */709 if (old_mem_type == XE_PL_TT &&710 new_mem->mem_type == XE_PL_TT) {711 ttm_bo_move_null(ttm_bo, new_mem);712 goto out;713 }714 715 if (!move_lacks_source && !xe_bo_is_pinned(bo)) {716 ret = xe_bo_move_notify(bo, ctx);717 if (ret)718 goto out;719 }720 721 if (old_mem_type == XE_PL_TT &&722 new_mem->mem_type == XE_PL_SYSTEM) {723 long timeout = dma_resv_wait_timeout(ttm_bo->base.resv,724 DMA_RESV_USAGE_BOOKKEEP,725 true,726 MAX_SCHEDULE_TIMEOUT);727 if (timeout < 0) {728 ret = timeout;729 goto out;730 }731 732 if (!handle_system_ccs) {733 ttm_bo_move_null(ttm_bo, new_mem);734 goto out;735 }736 }737 738 if (!move_lacks_source &&739 ((old_mem_type == XE_PL_SYSTEM && resource_is_vram(new_mem)) ||740 (mem_type_is_vram(old_mem_type) &&741 new_mem->mem_type == XE_PL_SYSTEM))) {742 hop->fpfn = 0;743 hop->lpfn = 0;744 hop->mem_type = XE_PL_TT;745 hop->flags = TTM_PL_FLAG_TEMPORARY;746 ret = -EMULTIHOP;747 goto out;748 }749 750 if (bo->tile)751 migrate = bo->tile->migrate;752 else if (resource_is_vram(new_mem))753 migrate = mem_type_to_migrate(xe, new_mem->mem_type);754 else if (mem_type_is_vram(old_mem_type))755 migrate = mem_type_to_migrate(xe, old_mem_type);756 else757 migrate = xe->tiles[0].migrate;758 759 xe_assert(xe, migrate);760 trace_xe_bo_move(bo, new_mem->mem_type, old_mem_type, move_lacks_source);761 if (xe_rpm_reclaim_safe(xe)) {762 /*763 * We might be called through swapout in the validation path of764 * another TTM device, so unconditionally acquire rpm here.765 */766 xe_pm_runtime_get(xe);767 } else {768 drm_WARN_ON(&xe->drm, handle_system_ccs);769 xe_pm_runtime_get_noresume(xe);770 }771 772 if (xe_bo_is_pinned(bo) && !xe_bo_is_user(bo)) {773 /*774 * Kernel memory that is pinned should only be moved on suspend775 * / resume, some of the pinned memory is required for the776 * device to resume / use the GPU to move other evicted memory777 * (user memory) around. This likely could be optimized a bit778 * futher where we find the minimum set of pinned memory779 * required for resume but for simplity doing a memcpy for all780 * pinned memory.781 */782 ret = xe_bo_vmap(bo);783 if (!ret) {784 ret = ttm_bo_move_memcpy(ttm_bo, ctx, new_mem);785 786 /* Create a new VMAP once kernel BO back in VRAM */787 if (!ret && resource_is_vram(new_mem)) {788 struct xe_mem_region *vram = res_to_mem_region(new_mem);789 void __iomem *new_addr = vram->mapping +790 (new_mem->start << PAGE_SHIFT);791 792 if (XE_WARN_ON(new_mem->start == XE_BO_INVALID_OFFSET)) {793 ret = -EINVAL;794 xe_pm_runtime_put(xe);795 goto out;796 }797 798 xe_assert(xe, new_mem->start ==799 bo->placements->fpfn);800 801 iosys_map_set_vaddr_iomem(&bo->vmap, new_addr);802 }803 }804 } else {805 if (move_lacks_source) {806 u32 flags = 0;807 808 if (mem_type_is_vram(new_mem->mem_type))809 flags |= XE_MIGRATE_CLEAR_FLAG_FULL;810 else if (handle_system_ccs)811 flags |= XE_MIGRATE_CLEAR_FLAG_CCS_DATA;812 813 fence = xe_migrate_clear(migrate, bo, new_mem, flags);814 }815 else816 fence = xe_migrate_copy(migrate, bo, bo, old_mem,817 new_mem, handle_system_ccs);818 if (IS_ERR(fence)) {819 ret = PTR_ERR(fence);820 xe_pm_runtime_put(xe);821 goto out;822 }823 if (!move_lacks_source) {824 ret = ttm_bo_move_accel_cleanup(ttm_bo, fence, evict,825 true, new_mem);826 if (ret) {827 dma_fence_wait(fence, false);828 ttm_bo_move_null(ttm_bo, new_mem);829 ret = 0;830 }831 } else {832 /*833 * ttm_bo_move_accel_cleanup() may blow up if834 * bo->resource == NULL, so just attach the835 * fence and set the new resource.836 */837 dma_resv_add_fence(ttm_bo->base.resv, fence,838 DMA_RESV_USAGE_KERNEL);839 ttm_bo_move_null(ttm_bo, new_mem);840 }841 842 dma_fence_put(fence);843 }844 845 xe_pm_runtime_put(xe);846 847out:848 if ((!ttm_bo->resource || ttm_bo->resource->mem_type == XE_PL_SYSTEM) &&849 ttm_bo->ttm)850 xe_tt_unmap_sg(ttm_bo->ttm);851 852 return ret;853}854 855/**856 * xe_bo_evict_pinned() - Evict a pinned VRAM object to system memory857 * @bo: The buffer object to move.858 *859 * On successful completion, the object memory will be moved to sytem memory.860 *861 * This is needed to for special handling of pinned VRAM object during862 * suspend-resume.863 *864 * Return: 0 on success. Negative error code on failure.865 */866int xe_bo_evict_pinned(struct xe_bo *bo)867{868 struct ttm_place place = {869 .mem_type = XE_PL_TT,870 };871 struct ttm_placement placement = {872 .placement = &place,873 .num_placement = 1,874 };875 struct ttm_operation_ctx ctx = {876 .interruptible = false,877 };878 struct ttm_resource *new_mem;879 int ret;880 881 xe_bo_assert_held(bo);882 883 if (WARN_ON(!bo->ttm.resource))884 return -EINVAL;885 886 if (WARN_ON(!xe_bo_is_pinned(bo)))887 return -EINVAL;888 889 if (!xe_bo_is_vram(bo))890 return 0;891 892 ret = ttm_bo_mem_space(&bo->ttm, &placement, &new_mem, &ctx);893 if (ret)894 return ret;895 896 if (!bo->ttm.ttm) {897 bo->ttm.ttm = xe_ttm_tt_create(&bo->ttm, 0);898 if (!bo->ttm.ttm) {899 ret = -ENOMEM;900 goto err_res_free;901 }902 }903 904 ret = ttm_tt_populate(bo->ttm.bdev, bo->ttm.ttm, &ctx);905 if (ret)906 goto err_res_free;907 908 ret = dma_resv_reserve_fences(bo->ttm.base.resv, 1);909 if (ret)910 goto err_res_free;911 912 ret = xe_bo_move(&bo->ttm, false, &ctx, new_mem, NULL);913 if (ret)914 goto err_res_free;915 916 return 0;917 918err_res_free:919 ttm_resource_free(&bo->ttm, &new_mem);920 return ret;921}922 923/**924 * xe_bo_restore_pinned() - Restore a pinned VRAM object925 * @bo: The buffer object to move.926 *927 * On successful completion, the object memory will be moved back to VRAM.928 *929 * This is needed to for special handling of pinned VRAM object during930 * suspend-resume.931 *932 * Return: 0 on success. Negative error code on failure.933 */934int xe_bo_restore_pinned(struct xe_bo *bo)935{936 struct ttm_operation_ctx ctx = {937 .interruptible = false,938 };939 struct ttm_resource *new_mem;940 struct ttm_place *place = &bo->placements[0];941 int ret;942 943 xe_bo_assert_held(bo);944 945 if (WARN_ON(!bo->ttm.resource))946 return -EINVAL;947 948 if (WARN_ON(!xe_bo_is_pinned(bo)))949 return -EINVAL;950 951 if (WARN_ON(xe_bo_is_vram(bo)))952 return -EINVAL;953 954 if (WARN_ON(!bo->ttm.ttm && !xe_bo_is_stolen(bo)))955 return -EINVAL;956 957 if (!mem_type_is_vram(place->mem_type))958 return 0;959 960 ret = ttm_bo_mem_space(&bo->ttm, &bo->placement, &new_mem, &ctx);961 if (ret)962 return ret;963 964 ret = ttm_tt_populate(bo->ttm.bdev, bo->ttm.ttm, &ctx);965 if (ret)966 goto err_res_free;967 968 ret = dma_resv_reserve_fences(bo->ttm.base.resv, 1);969 if (ret)970 goto err_res_free;971 972 ret = xe_bo_move(&bo->ttm, false, &ctx, new_mem, NULL);973 if (ret)974 goto err_res_free;975 976 return 0;977 978err_res_free:979 ttm_resource_free(&bo->ttm, &new_mem);980 return ret;981}982 983static unsigned long xe_ttm_io_mem_pfn(struct ttm_buffer_object *ttm_bo,984 unsigned long page_offset)985{986 struct xe_bo *bo = ttm_to_xe_bo(ttm_bo);987 struct xe_res_cursor cursor;988 struct xe_mem_region *vram;989 990 if (ttm_bo->resource->mem_type == XE_PL_STOLEN)991 return xe_ttm_stolen_io_offset(bo, page_offset << PAGE_SHIFT) >> PAGE_SHIFT;992 993 vram = res_to_mem_region(ttm_bo->resource);994 xe_res_first(ttm_bo->resource, (u64)page_offset << PAGE_SHIFT, 0, &cursor);995 return (vram->io_start + cursor.start) >> PAGE_SHIFT;996}997 998static void __xe_bo_vunmap(struct xe_bo *bo);999 1000/*1001 * TODO: Move this function to TTM so we don't rely on how TTM does its1002 * locking, thereby abusing TTM internals.1003 */1004static bool xe_ttm_bo_lock_in_destructor(struct ttm_buffer_object *ttm_bo)1005{1006 struct xe_device *xe = ttm_to_xe_device(ttm_bo->bdev);1007 bool locked;1008 1009 xe_assert(xe, !kref_read(&ttm_bo->kref));1010 1011 /*1012 * We can typically only race with TTM trylocking under the1013 * lru_lock, which will immediately be unlocked again since1014 * the ttm_bo refcount is zero at this point. So trylocking *should*1015 * always succeed here, as long as we hold the lru lock.1016 */1017 spin_lock(&ttm_bo->bdev->lru_lock);1018 locked = dma_resv_trylock(ttm_bo->base.resv);1019 spin_unlock(&ttm_bo->bdev->lru_lock);1020 xe_assert(xe, locked);1021 1022 return locked;1023}1024 1025static void xe_ttm_bo_release_notify(struct ttm_buffer_object *ttm_bo)1026{1027 struct dma_resv_iter cursor;1028 struct dma_fence *fence;1029 struct dma_fence *replacement = NULL;1030 struct xe_bo *bo;1031 1032 if (!xe_bo_is_xe_bo(ttm_bo))1033 return;1034 1035 bo = ttm_to_xe_bo(ttm_bo);1036 xe_assert(xe_bo_device(bo), !(bo->created && kref_read(&ttm_bo->base.refcount)));1037 1038 /*1039 * Corner case where TTM fails to allocate memory and this BOs resv1040 * still points the VMs resv1041 */1042 if (ttm_bo->base.resv != &ttm_bo->base._resv)1043 return;1044 1045 if (!xe_ttm_bo_lock_in_destructor(ttm_bo))1046 return;1047 1048 /*1049 * Scrub the preempt fences if any. The unbind fence is already1050 * attached to the resv.1051 * TODO: Don't do this for external bos once we scrub them after1052 * unbind.1053 */1054 dma_resv_for_each_fence(&cursor, ttm_bo->base.resv,1055 DMA_RESV_USAGE_BOOKKEEP, fence) {1056 if (xe_fence_is_xe_preempt(fence) &&1057 !dma_fence_is_signaled(fence)) {1058 if (!replacement)1059 replacement = dma_fence_get_stub();1060 1061 dma_resv_replace_fences(ttm_bo->base.resv,1062 fence->context,1063 replacement,1064 DMA_RESV_USAGE_BOOKKEEP);1065 }1066 }1067 dma_fence_put(replacement);1068 1069 dma_resv_unlock(ttm_bo->base.resv);1070}1071 1072static void xe_ttm_bo_delete_mem_notify(struct ttm_buffer_object *ttm_bo)1073{1074 if (!xe_bo_is_xe_bo(ttm_bo))1075 return;1076 1077 /*1078 * Object is idle and about to be destroyed. Release the1079 * dma-buf attachment.1080 */1081 if (ttm_bo->type == ttm_bo_type_sg && ttm_bo->sg) {1082 struct xe_ttm_tt *xe_tt = container_of(ttm_bo->ttm,1083 struct xe_ttm_tt, ttm);1084 1085 dma_buf_unmap_attachment(ttm_bo->base.import_attach, ttm_bo->sg,1086 DMA_BIDIRECTIONAL);1087 ttm_bo->sg = NULL;1088 xe_tt->sg = NULL;1089 }1090}1091 1092const struct ttm_device_funcs xe_ttm_funcs = {1093 .ttm_tt_create = xe_ttm_tt_create,1094 .ttm_tt_populate = xe_ttm_tt_populate,1095 .ttm_tt_unpopulate = xe_ttm_tt_unpopulate,1096 .ttm_tt_destroy = xe_ttm_tt_destroy,1097 .evict_flags = xe_evict_flags,1098 .move = xe_bo_move,1099 .io_mem_reserve = xe_ttm_io_mem_reserve,1100 .io_mem_pfn = xe_ttm_io_mem_pfn,1101 .release_notify = xe_ttm_bo_release_notify,1102 .eviction_valuable = ttm_bo_eviction_valuable,1103 .delete_mem_notify = xe_ttm_bo_delete_mem_notify,1104};1105 1106static void xe_ttm_bo_destroy(struct ttm_buffer_object *ttm_bo)1107{1108 struct xe_bo *bo = ttm_to_xe_bo(ttm_bo);1109 struct xe_device *xe = ttm_to_xe_device(ttm_bo->bdev);1110 1111 if (bo->ttm.base.import_attach)1112 drm_prime_gem_destroy(&bo->ttm.base, NULL);1113 drm_gem_object_release(&bo->ttm.base);1114 1115 xe_assert(xe, list_empty(&ttm_bo->base.gpuva.list));1116 1117 if (bo->ggtt_node && bo->ggtt_node->base.size)1118 xe_ggtt_remove_bo(bo->tile->mem.ggtt, bo);1119 1120#ifdef CONFIG_PROC_FS1121 if (bo->client)1122 xe_drm_client_remove_bo(bo);1123#endif1124 1125 if (bo->vm && xe_bo_is_user(bo))1126 xe_vm_put(bo->vm);1127 1128 mutex_lock(&xe->mem_access.vram_userfault.lock);1129 if (!list_empty(&bo->vram_userfault_link))1130 list_del(&bo->vram_userfault_link);1131 mutex_unlock(&xe->mem_access.vram_userfault.lock);1132 1133 kfree(bo);1134}1135 1136static void xe_gem_object_free(struct drm_gem_object *obj)1137{1138 /* Our BO reference counting scheme works as follows:1139 *1140 * The gem object kref is typically used throughout the driver,1141 * and the gem object holds a ttm_buffer_object refcount, so1142 * that when the last gem object reference is put, which is when1143 * we end up in this function, we put also that ttm_buffer_object1144 * refcount. Anything using gem interfaces is then no longer1145 * allowed to access the object in a way that requires a gem1146 * refcount, including locking the object.1147 *1148 * driver ttm callbacks is allowed to use the ttm_buffer_object1149 * refcount directly if needed.1150 */1151 __xe_bo_vunmap(gem_to_xe_bo(obj));1152 ttm_bo_put(container_of(obj, struct ttm_buffer_object, base));1153}1154 1155static void xe_gem_object_close(struct drm_gem_object *obj,1156 struct drm_file *file_priv)1157{1158 struct xe_bo *bo = gem_to_xe_bo(obj);1159 1160 if (bo->vm && !xe_vm_in_fault_mode(bo->vm)) {1161 xe_assert(xe_bo_device(bo), xe_bo_is_user(bo));1162 1163 xe_bo_lock(bo, false);1164 ttm_bo_set_bulk_move(&bo->ttm, NULL);1165 xe_bo_unlock(bo);1166 }1167}1168 1169static vm_fault_t xe_gem_fault(struct vm_fault *vmf)1170{1171 struct ttm_buffer_object *tbo = vmf->vma->vm_private_data;1172 struct drm_device *ddev = tbo->base.dev;1173 struct xe_device *xe = to_xe_device(ddev);1174 struct xe_bo *bo = ttm_to_xe_bo(tbo);1175 bool needs_rpm = bo->flags & XE_BO_FLAG_VRAM_MASK;1176 vm_fault_t ret;1177 int idx;1178 1179 if (needs_rpm)1180 xe_pm_runtime_get(xe);1181 1182 ret = ttm_bo_vm_reserve(tbo, vmf);1183 if (ret)1184 goto out;1185 1186 if (drm_dev_enter(ddev, &idx)) {1187 trace_xe_bo_cpu_fault(bo);1188 1189 ret = ttm_bo_vm_fault_reserved(vmf, vmf->vma->vm_page_prot,1190 TTM_BO_VM_NUM_PREFAULT);1191 drm_dev_exit(idx);1192 } else {1193 ret = ttm_bo_vm_dummy_page(vmf, vmf->vma->vm_page_prot);1194 }1195 1196 if (ret == VM_FAULT_RETRY && !(vmf->flags & FAULT_FLAG_RETRY_NOWAIT))1197 goto out;1198 /*1199 * ttm_bo_vm_reserve() already has dma_resv_lock.1200 */1201 if (ret == VM_FAULT_NOPAGE && mem_type_is_vram(tbo->resource->mem_type)) {1202 mutex_lock(&xe->mem_access.vram_userfault.lock);1203 if (list_empty(&bo->vram_userfault_link))1204 list_add(&bo->vram_userfault_link, &xe->mem_access.vram_userfault.list);1205 mutex_unlock(&xe->mem_access.vram_userfault.lock);1206 }1207 1208 dma_resv_unlock(tbo->base.resv);1209out:1210 if (needs_rpm)1211 xe_pm_runtime_put(xe);1212 1213 return ret;1214}1215 1216static const struct vm_operations_struct xe_gem_vm_ops = {1217 .fault = xe_gem_fault,1218 .open = ttm_bo_vm_open,1219 .close = ttm_bo_vm_close,1220 .access = ttm_bo_vm_access1221};1222 1223static const struct drm_gem_object_funcs xe_gem_object_funcs = {1224 .free = xe_gem_object_free,1225 .close = xe_gem_object_close,1226 .mmap = drm_gem_ttm_mmap,1227 .export = xe_gem_prime_export,1228 .vm_ops = &xe_gem_vm_ops,1229};1230 1231/**1232 * xe_bo_alloc - Allocate storage for a struct xe_bo1233 *1234 * This funcition is intended to allocate storage to be used for input1235 * to __xe_bo_create_locked(), in the case a pointer to the bo to be1236 * created is needed before the call to __xe_bo_create_locked().1237 * If __xe_bo_create_locked ends up never to be called, then the1238 * storage allocated with this function needs to be freed using1239 * xe_bo_free().1240 *1241 * Return: A pointer to an uninitialized struct xe_bo on success,1242 * ERR_PTR(-ENOMEM) on error.1243 */1244struct xe_bo *xe_bo_alloc(void)1245{1246 struct xe_bo *bo = kzalloc(sizeof(*bo), GFP_KERNEL);1247 1248 if (!bo)1249 return ERR_PTR(-ENOMEM);1250 1251 return bo;1252}1253 1254/**1255 * xe_bo_free - Free storage allocated using xe_bo_alloc()1256 * @bo: The buffer object storage.1257 *1258 * Refer to xe_bo_alloc() documentation for valid use-cases.1259 */1260void xe_bo_free(struct xe_bo *bo)1261{1262 kfree(bo);1263}1264 1265struct xe_bo *___xe_bo_create_locked(struct xe_device *xe, struct xe_bo *bo,1266 struct xe_tile *tile, struct dma_resv *resv,1267 struct ttm_lru_bulk_move *bulk, size_t size,1268 u16 cpu_caching, enum ttm_bo_type type,1269 u32 flags)1270{1271 struct ttm_operation_ctx ctx = {1272 .interruptible = true,1273 .no_wait_gpu = false,1274 };1275 struct ttm_placement *placement;1276 uint32_t alignment;1277 size_t aligned_size;1278 int err;1279 1280 /* Only kernel objects should set GT */1281 xe_assert(xe, !tile || type == ttm_bo_type_kernel);1282 1283 if (XE_WARN_ON(!size)) {1284 xe_bo_free(bo);1285 return ERR_PTR(-EINVAL);1286 }1287 1288 if (flags & (XE_BO_FLAG_VRAM_MASK | XE_BO_FLAG_STOLEN) &&1289 !(flags & XE_BO_FLAG_IGNORE_MIN_PAGE_SIZE) &&1290 ((xe->info.vram_flags & XE_VRAM_FLAGS_NEED64K) ||1291 (flags & (XE_BO_FLAG_NEEDS_64K | XE_BO_FLAG_NEEDS_2M)))) {1292 size_t align = flags & XE_BO_FLAG_NEEDS_2M ? SZ_2M : SZ_64K;1293 1294 aligned_size = ALIGN(size, align);1295 if (type != ttm_bo_type_device)1296 size = ALIGN(size, align);1297 flags |= XE_BO_FLAG_INTERNAL_64K;1298 alignment = align >> PAGE_SHIFT;1299 } else {1300 aligned_size = ALIGN(size, SZ_4K);1301 flags &= ~XE_BO_FLAG_INTERNAL_64K;1302 alignment = SZ_4K >> PAGE_SHIFT;1303 }1304 1305 if (type == ttm_bo_type_device && aligned_size != size)1306 return ERR_PTR(-EINVAL);1307 1308 if (!bo) {1309 bo = xe_bo_alloc();1310 if (IS_ERR(bo))1311 return bo;1312 }1313 1314 bo->ccs_cleared = false;1315 bo->tile = tile;1316 bo->size = size;1317 bo->flags = flags;1318 bo->cpu_caching = cpu_caching;1319 bo->ttm.base.funcs = &xe_gem_object_funcs;1320 bo->ttm.priority = XE_BO_PRIORITY_NORMAL;1321 INIT_LIST_HEAD(&bo->pinned_link);1322#ifdef CONFIG_PROC_FS1323 INIT_LIST_HEAD(&bo->client_link);1324#endif1325 INIT_LIST_HEAD(&bo->vram_userfault_link);1326 1327 drm_gem_private_object_init(&xe->drm, &bo->ttm.base, size);1328 1329 if (resv) {1330 ctx.allow_res_evict = !(flags & XE_BO_FLAG_NO_RESV_EVICT);1331 ctx.resv = resv;1332 }1333 1334 if (!(flags & XE_BO_FLAG_FIXED_PLACEMENT)) {1335 err = __xe_bo_placement_for_flags(xe, bo, bo->flags);1336 if (WARN_ON(err)) {1337 xe_ttm_bo_destroy(&bo->ttm);1338 return ERR_PTR(err);1339 }1340 }1341 1342 /* Defer populating type_sg bos */1343 placement = (type == ttm_bo_type_sg ||1344 bo->flags & XE_BO_FLAG_DEFER_BACKING) ? &sys_placement :1345 &bo->placement;1346 err = ttm_bo_init_reserved(&xe->ttm, &bo->ttm, type,1347 placement, alignment,1348 &ctx, NULL, resv, xe_ttm_bo_destroy);1349 if (err)1350 return ERR_PTR(err);1351 1352 /*1353 * The VRAM pages underneath are potentially still being accessed by the1354 * GPU, as per async GPU clearing and async evictions. However TTM makes1355 * sure to add any corresponding move/clear fences into the objects1356 * dma-resv using the DMA_RESV_USAGE_KERNEL slot.1357 *1358 * For KMD internal buffers we don't care about GPU clearing, however we1359 * still need to handle async evictions, where the VRAM is still being1360 * accessed by the GPU. Most internal callers are not expecting this,1361 * since they are missing the required synchronisation before accessing1362 * the memory. To keep things simple just sync wait any kernel fences1363 * here, if the buffer is designated KMD internal.1364 *1365 * For normal userspace objects we should already have the required1366 * pipelining or sync waiting elsewhere, since we already have to deal1367 * with things like async GPU clearing.1368 */1369 if (type == ttm_bo_type_kernel) {1370 long timeout = dma_resv_wait_timeout(bo->ttm.base.resv,1371 DMA_RESV_USAGE_KERNEL,1372 ctx.interruptible,1373 MAX_SCHEDULE_TIMEOUT);1374 1375 if (timeout < 0) {1376 if (!resv)1377 dma_resv_unlock(bo->ttm.base.resv);1378 xe_bo_put(bo);1379 return ERR_PTR(timeout);1380 }1381 }1382 1383 bo->created = true;1384 if (bulk)1385 ttm_bo_set_bulk_move(&bo->ttm, bulk);1386 else1387 ttm_bo_move_to_lru_tail_unlocked(&bo->ttm);1388 1389 return bo;1390}1391 1392static int __xe_bo_fixed_placement(struct xe_device *xe,1393 struct xe_bo *bo,1394 u32 flags,1395 u64 start, u64 end, u64 size)1396{1397 struct ttm_place *place = bo->placements;1398 1399 if (flags & (XE_BO_FLAG_USER | XE_BO_FLAG_SYSTEM))1400 return -EINVAL;1401 1402 place->flags = TTM_PL_FLAG_CONTIGUOUS;1403 place->fpfn = start >> PAGE_SHIFT;1404 place->lpfn = end >> PAGE_SHIFT;1405 1406 switch (flags & (XE_BO_FLAG_STOLEN | XE_BO_FLAG_VRAM_MASK)) {1407 case XE_BO_FLAG_VRAM0:1408 place->mem_type = XE_PL_VRAM0;1409 break;1410 case XE_BO_FLAG_VRAM1:1411 place->mem_type = XE_PL_VRAM1;1412 break;1413 case XE_BO_FLAG_STOLEN:1414 place->mem_type = XE_PL_STOLEN;1415 break;1416 1417 default:1418 /* 0 or multiple of the above set */1419 return -EINVAL;1420 }1421 1422 bo->placement = (struct ttm_placement) {1423 .num_placement = 1,1424 .placement = place,1425 };1426 1427 return 0;1428}1429 1430static struct xe_bo *1431__xe_bo_create_locked(struct xe_device *xe,1432 struct xe_tile *tile, struct xe_vm *vm,1433 size_t size, u64 start, u64 end,1434 u16 cpu_caching, enum ttm_bo_type type, u32 flags)1435{1436 struct xe_bo *bo = NULL;1437 int err;1438 1439 if (vm)1440 xe_vm_assert_held(vm);1441 1442 if (start || end != ~0ULL) {1443 bo = xe_bo_alloc();1444 if (IS_ERR(bo))1445 return bo;1446 1447 flags |= XE_BO_FLAG_FIXED_PLACEMENT;1448 err = __xe_bo_fixed_placement(xe, bo, flags, start, end, size);1449 if (err) {1450 xe_bo_free(bo);1451 return ERR_PTR(err);1452 }1453 }1454 1455 bo = ___xe_bo_create_locked(xe, bo, tile, vm ? xe_vm_resv(vm) : NULL,1456 vm && !xe_vm_in_fault_mode(vm) &&1457 flags & XE_BO_FLAG_USER ?1458 &vm->lru_bulk_move : NULL, size,1459 cpu_caching, type, flags);1460 if (IS_ERR(bo))1461 return bo;1462 1463 /*1464 * Note that instead of taking a reference no the drm_gpuvm_resv_bo(),1465 * to ensure the shared resv doesn't disappear under the bo, the bo1466 * will keep a reference to the vm, and avoid circular references1467 * by having all the vm's bo refereferences released at vm close1468 * time.1469 */1470 if (vm && xe_bo_is_user(bo))1471 xe_vm_get(vm);1472 bo->vm = vm;1473 1474 if (bo->flags & XE_BO_FLAG_GGTT) {1475 if (!tile && flags & XE_BO_FLAG_STOLEN)1476 tile = xe_device_get_root_tile(xe);1477 1478 xe_assert(xe, tile);1479 1480 if (flags & XE_BO_FLAG_FIXED_PLACEMENT) {1481 err = xe_ggtt_insert_bo_at(tile->mem.ggtt, bo,1482 start + bo->size, U64_MAX);1483 } else {1484 err = xe_ggtt_insert_bo(tile->mem.ggtt, bo);1485 }1486 if (err)1487 goto err_unlock_put_bo;1488 }1489 1490 return bo;1491 1492err_unlock_put_bo:1493 __xe_bo_unset_bulk_move(bo);1494 xe_bo_unlock_vm_held(bo);1495 xe_bo_put(bo);1496 return ERR_PTR(err);1497}1498 1499struct xe_bo *1500xe_bo_create_locked_range(struct xe_device *xe,1501 struct xe_tile *tile, struct xe_vm *vm,1502 size_t size, u64 start, u64 end,1503 enum ttm_bo_type type, u32 flags)1504{1505 return __xe_bo_create_locked(xe, tile, vm, size, start, end, 0, type, flags);1506}1507 1508struct xe_bo *xe_bo_create_locked(struct xe_device *xe, struct xe_tile *tile,1509 struct xe_vm *vm, size_t size,1510 enum ttm_bo_type type, u32 flags)1511{1512 return __xe_bo_create_locked(xe, tile, vm, size, 0, ~0ULL, 0, type, flags);1513}1514 1515struct xe_bo *xe_bo_create_user(struct xe_device *xe, struct xe_tile *tile,1516 struct xe_vm *vm, size_t size,1517 u16 cpu_caching,1518 u32 flags)1519{1520 struct xe_bo *bo = __xe_bo_create_locked(xe, tile, vm, size, 0, ~0ULL,1521 cpu_caching, ttm_bo_type_device,1522 flags | XE_BO_FLAG_USER);1523 if (!IS_ERR(bo))1524 xe_bo_unlock_vm_held(bo);1525 1526 return bo;1527}1528 1529struct xe_bo *xe_bo_create(struct xe_device *xe, struct xe_tile *tile,1530 struct xe_vm *vm, size_t size,1531 enum ttm_bo_type type, u32 flags)1532{1533 struct xe_bo *bo = xe_bo_create_locked(xe, tile, vm, size, type, flags);1534 1535 if (!IS_ERR(bo))1536 xe_bo_unlock_vm_held(bo);1537 1538 return bo;1539}1540 1541struct xe_bo *xe_bo_create_pin_map_at(struct xe_device *xe, struct xe_tile *tile,1542 struct xe_vm *vm,1543 size_t size, u64 offset,1544 enum ttm_bo_type type, u32 flags)1545{1546 struct xe_bo *bo;1547 int err;1548 u64 start = offset == ~0ull ? 0 : offset;1549 u64 end = offset == ~0ull ? offset : start + size;1550 1551 if (flags & XE_BO_FLAG_STOLEN &&1552 xe_ttm_stolen_cpu_access_needs_ggtt(xe))1553 flags |= XE_BO_FLAG_GGTT;1554 1555 bo = xe_bo_create_locked_range(xe, tile, vm, size, start, end, type,1556 flags | XE_BO_FLAG_NEEDS_CPU_ACCESS);1557 if (IS_ERR(bo))1558 return bo;1559 1560 err = xe_bo_pin(bo);1561 if (err)1562 goto err_put;1563 1564 err = xe_bo_vmap(bo);1565 if (err)1566 goto err_unpin;1567 1568 xe_bo_unlock_vm_held(bo);1569 1570 return bo;1571 1572err_unpin:1573 xe_bo_unpin(bo);1574err_put:1575 xe_bo_unlock_vm_held(bo);1576 xe_bo_put(bo);1577 return ERR_PTR(err);1578}1579 1580struct xe_bo *xe_bo_create_pin_map(struct xe_device *xe, struct xe_tile *tile,1581 struct xe_vm *vm, size_t size,1582 enum ttm_bo_type type, u32 flags)1583{1584 return xe_bo_create_pin_map_at(xe, tile, vm, size, ~0ull, type, flags);1585}1586 1587struct xe_bo *xe_bo_create_from_data(struct xe_device *xe, struct xe_tile *tile,1588 const void *data, size_t size,1589 enum ttm_bo_type type, u32 flags)1590{1591 struct xe_bo *bo = xe_bo_create_pin_map(xe, tile, NULL,1592 ALIGN(size, PAGE_SIZE),1593 type, flags);1594 if (IS_ERR(bo))1595 return bo;1596 1597 xe_map_memcpy_to(xe, &bo->vmap, 0, data, size);1598 1599 return bo;1600}1601 1602static void __xe_bo_unpin_map_no_vm(void *arg)1603{1604 xe_bo_unpin_map_no_vm(arg);1605}1606 1607struct xe_bo *xe_managed_bo_create_pin_map(struct xe_device *xe, struct xe_tile *tile,1608 size_t size, u32 flags)1609{1610 struct xe_bo *bo;1611 int ret;1612 1613 bo = xe_bo_create_pin_map(xe, tile, NULL, size, ttm_bo_type_kernel, flags);1614 if (IS_ERR(bo))1615 return bo;1616 1617 ret = devm_add_action_or_reset(xe->drm.dev, __xe_bo_unpin_map_no_vm, bo);1618 if (ret)1619 return ERR_PTR(ret);1620 1621 return bo;1622}1623 1624struct xe_bo *xe_managed_bo_create_from_data(struct xe_device *xe, struct xe_tile *tile,1625 const void *data, size_t size, u32 flags)1626{1627 struct xe_bo *bo = xe_managed_bo_create_pin_map(xe, tile, ALIGN(size, PAGE_SIZE), flags);1628 1629 if (IS_ERR(bo))1630 return bo;1631 1632 xe_map_memcpy_to(xe, &bo->vmap, 0, data, size);1633 1634 return bo;1635}1636 1637/**1638 * xe_managed_bo_reinit_in_vram1639 * @xe: xe device1640 * @tile: Tile where the new buffer will be created1641 * @src: Managed buffer object allocated in system memory1642 *1643 * Replace a managed src buffer object allocated in system memory with a new1644 * one allocated in vram, copying the data between them.1645 * Buffer object in VRAM is not going to have the same GGTT address, the caller1646 * is responsible for making sure that any old references to it are updated.1647 *1648 * Returns 0 for success, negative error code otherwise.1649 */1650int xe_managed_bo_reinit_in_vram(struct xe_device *xe, struct xe_tile *tile, struct xe_bo **src)1651{1652 struct xe_bo *bo;1653 u32 dst_flags = XE_BO_FLAG_VRAM_IF_DGFX(tile) | XE_BO_FLAG_GGTT;1654 1655 dst_flags |= (*src)->flags & XE_BO_FLAG_GGTT_INVALIDATE;1656 1657 xe_assert(xe, IS_DGFX(xe));1658 xe_assert(xe, !(*src)->vmap.is_iomem);1659 1660 bo = xe_managed_bo_create_from_data(xe, tile, (*src)->vmap.vaddr,1661 (*src)->size, dst_flags);1662 if (IS_ERR(bo))1663 return PTR_ERR(bo);1664 1665 devm_release_action(xe->drm.dev, __xe_bo_unpin_map_no_vm, *src);1666 *src = bo;1667 1668 return 0;1669}1670 1671/*1672 * XXX: This is in the VM bind data path, likely should calculate this once and1673 * store, with a recalculation if the BO is moved.1674 */1675uint64_t vram_region_gpu_offset(struct ttm_resource *res)1676{1677 struct xe_device *xe = ttm_to_xe_device(res->bo->bdev);1678 1679 if (res->mem_type == XE_PL_STOLEN)1680 return xe_ttm_stolen_gpu_offset(xe);1681 1682 return res_to_mem_region(res)->dpa_base;1683}1684 1685/**1686 * xe_bo_pin_external - pin an external BO1687 * @bo: buffer object to be pinned1688 *1689 * Pin an external (not tied to a VM, can be exported via dma-buf / prime FD)1690 * BO. Unique call compared to xe_bo_pin as this function has it own set of1691 * asserts and code to ensure evict / restore on suspend / resume.1692 *1693 * Returns 0 for success, negative error code otherwise.1694 */1695int xe_bo_pin_external(struct xe_bo *bo)1696{1697 struct xe_device *xe = xe_bo_device(bo);1698 int err;1699 1700 xe_assert(xe, !bo->vm);1701 xe_assert(xe, xe_bo_is_user(bo));1702 1703 if (!xe_bo_is_pinned(bo)) {1704 err = xe_bo_validate(bo, NULL, false);1705 if (err)1706 return err;1707 1708 if (xe_bo_is_vram(bo)) {1709 spin_lock(&xe->pinned.lock);1710 list_add_tail(&bo->pinned_link,1711 &xe->pinned.external_vram);1712 spin_unlock(&xe->pinned.lock);1713 }1714 }1715 1716 ttm_bo_pin(&bo->ttm);1717 1718 /*1719 * FIXME: If we always use the reserve / unreserve functions for locking1720 * we do not need this.1721 */1722 ttm_bo_move_to_lru_tail_unlocked(&bo->ttm);1723 1724 return 0;1725}1726 1727int xe_bo_pin(struct xe_bo *bo)1728{1729 struct ttm_place *place = &bo->placements[0];1730 struct xe_device *xe = xe_bo_device(bo);1731 int err;1732 1733 /* We currently don't expect user BO to be pinned */1734 xe_assert(xe, !xe_bo_is_user(bo));1735 1736 /* Pinned object must be in GGTT or have pinned flag */1737 xe_assert(xe, bo->flags & (XE_BO_FLAG_PINNED |1738 XE_BO_FLAG_GGTT));1739 1740 /*1741 * No reason we can't support pinning imported dma-bufs we just don't1742 * expect to pin an imported dma-buf.1743 */1744 xe_assert(xe, !bo->ttm.base.import_attach);1745 1746 /* We only expect at most 1 pin */1747 xe_assert(xe, !xe_bo_is_pinned(bo));1748 1749 err = xe_bo_validate(bo, NULL, false);1750 if (err)1751 return err;1752 1753 /*1754 * For pinned objects in on DGFX, which are also in vram, we expect1755 * these to be in contiguous VRAM memory. Required eviction / restore1756 * during suspend / resume (force restore to same physical address).1757 */1758 if (IS_DGFX(xe) && !(IS_ENABLED(CONFIG_DRM_XE_DEBUG) &&1759 bo->flags & XE_BO_FLAG_INTERNAL_TEST)) {1760 if (mem_type_is_vram(place->mem_type)) {1761 xe_assert(xe, place->flags & TTM_PL_FLAG_CONTIGUOUS);1762 1763 place->fpfn = (xe_bo_addr(bo, 0, PAGE_SIZE) -1764 vram_region_gpu_offset(bo->ttm.resource)) >> PAGE_SHIFT;1765 place->lpfn = place->fpfn + (bo->size >> PAGE_SHIFT);1766 }1767 }1768 1769 if (mem_type_is_vram(place->mem_type) || bo->flags & XE_BO_FLAG_GGTT) {1770 spin_lock(&xe->pinned.lock);1771 list_add_tail(&bo->pinned_link, &xe->pinned.kernel_bo_present);1772 spin_unlock(&xe->pinned.lock);1773 }1774 1775 ttm_bo_pin(&bo->ttm);1776 1777 /*1778 * FIXME: If we always use the reserve / unreserve functions for locking1779 * we do not need this.1780 */1781 ttm_bo_move_to_lru_tail_unlocked(&bo->ttm);1782 1783 return 0;1784}1785 1786/**1787 * xe_bo_unpin_external - unpin an external BO1788 * @bo: buffer object to be unpinned1789 *1790 * Unpin an external (not tied to a VM, can be exported via dma-buf / prime FD)1791 * BO. Unique call compared to xe_bo_unpin as this function has it own set of1792 * asserts and code to ensure evict / restore on suspend / resume.1793 *1794 * Returns 0 for success, negative error code otherwise.1795 */1796void xe_bo_unpin_external(struct xe_bo *bo)1797{1798 struct xe_device *xe = xe_bo_device(bo);1799 1800 xe_assert(xe, !bo->vm);1801 xe_assert(xe, xe_bo_is_pinned(bo));1802 xe_assert(xe, xe_bo_is_user(bo));1803 1804 spin_lock(&xe->pinned.lock);1805 if (bo->ttm.pin_count == 1 && !list_empty(&bo->pinned_link))1806 list_del_init(&bo->pinned_link);1807 spin_unlock(&xe->pinned.lock);1808 1809 ttm_bo_unpin(&bo->ttm);1810 1811 /*1812 * FIXME: If we always use the reserve / unreserve functions for locking1813 * we do not need this.1814 */1815 ttm_bo_move_to_lru_tail_unlocked(&bo->ttm);1816}1817 1818void xe_bo_unpin(struct xe_bo *bo)1819{1820 struct ttm_place *place = &bo->placements[0];1821 struct xe_device *xe = xe_bo_device(bo);1822 1823 xe_assert(xe, !bo->ttm.base.import_attach);1824 xe_assert(xe, xe_bo_is_pinned(bo));1825 1826 if (mem_type_is_vram(place->mem_type) || bo->flags & XE_BO_FLAG_GGTT) {1827 spin_lock(&xe->pinned.lock);1828 xe_assert(xe, !list_empty(&bo->pinned_link));1829 list_del_init(&bo->pinned_link);1830 spin_unlock(&xe->pinned.lock);1831 }1832 ttm_bo_unpin(&bo->ttm);1833}1834 1835/**1836 * xe_bo_validate() - Make sure the bo is in an allowed placement1837 * @bo: The bo,1838 * @vm: Pointer to a the vm the bo shares a locked dma_resv object with, or1839 * NULL. Used together with @allow_res_evict.1840 * @allow_res_evict: Whether it's allowed to evict bos sharing @vm's1841 * reservation object.1842 *1843 * Make sure the bo is in allowed placement, migrating it if necessary. If1844 * needed, other bos will be evicted. If bos selected for eviction shares1845 * the @vm's reservation object, they can be evicted iff @allow_res_evict is1846 * set to true, otherwise they will be bypassed.1847 *1848 * Return: 0 on success, negative error code on failure. May return1849 * -EINTR or -ERESTARTSYS if internal waits are interrupted by a signal.1850 */1851int xe_bo_validate(struct xe_bo *bo, struct xe_vm *vm, bool allow_res_evict)1852{1853 struct ttm_operation_ctx ctx = {1854 .interruptible = true,1855 .no_wait_gpu = false,1856 };1857 1858 if (vm) {1859 lockdep_assert_held(&vm->lock);1860 xe_vm_assert_held(vm);1861 1862 ctx.allow_res_evict = allow_res_evict;1863 ctx.resv = xe_vm_resv(vm);1864 }1865 1866 return ttm_bo_validate(&bo->ttm, &bo->placement, &ctx);1867}1868 1869bool xe_bo_is_xe_bo(struct ttm_buffer_object *bo)1870{1871 if (bo->destroy == &xe_ttm_bo_destroy)1872 return true;1873 1874 return false;1875}1876 1877/*1878 * Resolve a BO address. There is no assert to check if the proper lock is held1879 * so it should only be used in cases where it is not fatal to get the wrong1880 * address, such as printing debug information, but not in cases where memory is1881 * written based on this result.1882 */1883dma_addr_t __xe_bo_addr(struct xe_bo *bo, u64 offset, size_t page_size)1884{1885 struct xe_device *xe = xe_bo_device(bo);1886 struct xe_res_cursor cur;1887 u64 page;1888 1889 xe_assert(xe, page_size <= PAGE_SIZE);1890 page = offset >> PAGE_SHIFT;1891 offset &= (PAGE_SIZE - 1);1892 1893 if (!xe_bo_is_vram(bo) && !xe_bo_is_stolen(bo)) {1894 xe_assert(xe, bo->ttm.ttm);1895 1896 xe_res_first_sg(xe_bo_sg(bo), page << PAGE_SHIFT,1897 page_size, &cur);1898 return xe_res_dma(&cur) + offset;1899 } else {1900 struct xe_res_cursor cur;1901 1902 xe_res_first(bo->ttm.resource, page << PAGE_SHIFT,1903 page_size, &cur);1904 return cur.start + offset + vram_region_gpu_offset(bo->ttm.resource);1905 }1906}1907 1908dma_addr_t xe_bo_addr(struct xe_bo *bo, u64 offset, size_t page_size)1909{1910 if (!READ_ONCE(bo->ttm.pin_count))1911 xe_bo_assert_held(bo);1912 return __xe_bo_addr(bo, offset, page_size);1913}1914 1915int xe_bo_vmap(struct xe_bo *bo)1916{1917 void *virtual;1918 bool is_iomem;1919 int ret;1920 1921 xe_bo_assert_held(bo);1922 1923 if (!(bo->flags & XE_BO_FLAG_NEEDS_CPU_ACCESS))1924 return -EINVAL;1925 1926 if (!iosys_map_is_null(&bo->vmap))1927 return 0;1928 1929 /*1930 * We use this more or less deprecated interface for now since1931 * ttm_bo_vmap() doesn't offer the optimization of kmapping1932 * single page bos, which is done here.1933 * TODO: Fix up ttm_bo_vmap to do that, or fix up ttm_bo_kmap1934 * to use struct iosys_map.1935 */1936 ret = ttm_bo_kmap(&bo->ttm, 0, bo->size >> PAGE_SHIFT, &bo->kmap);1937 if (ret)1938 return ret;1939 1940 virtual = ttm_kmap_obj_virtual(&bo->kmap, &is_iomem);1941 if (is_iomem)1942 iosys_map_set_vaddr_iomem(&bo->vmap, (void __iomem *)virtual);1943 else1944 iosys_map_set_vaddr(&bo->vmap, virtual);1945 1946 return 0;1947}1948 1949static void __xe_bo_vunmap(struct xe_bo *bo)1950{1951 if (!iosys_map_is_null(&bo->vmap)) {1952 iosys_map_clear(&bo->vmap);1953 ttm_bo_kunmap(&bo->kmap);1954 }1955}1956 1957void xe_bo_vunmap(struct xe_bo *bo)1958{1959 xe_bo_assert_held(bo);1960 __xe_bo_vunmap(bo);1961}1962 1963int xe_gem_create_ioctl(struct drm_device *dev, void *data,1964 struct drm_file *file)1965{1966 struct xe_device *xe = to_xe_device(dev);1967 struct xe_file *xef = to_xe_file(file);1968 struct drm_xe_gem_create *args = data;1969 struct xe_vm *vm = NULL;1970 struct xe_bo *bo;1971 unsigned int bo_flags;1972 u32 handle;1973 int err;1974 1975 if (XE_IOCTL_DBG(xe, args->extensions) ||1976 XE_IOCTL_DBG(xe, args->pad[0] || args->pad[1] || args->pad[2]) ||1977 XE_IOCTL_DBG(xe, args->reserved[0] || args->reserved[1]))1978 return -EINVAL;1979 1980 /* at least one valid memory placement must be specified */1981 if (XE_IOCTL_DBG(xe, (args->placement & ~xe->info.mem_region_mask) ||1982 !args->placement))1983 return -EINVAL;1984 1985 if (XE_IOCTL_DBG(xe, args->flags &1986 ~(DRM_XE_GEM_CREATE_FLAG_DEFER_BACKING |1987 DRM_XE_GEM_CREATE_FLAG_SCANOUT |1988 DRM_XE_GEM_CREATE_FLAG_NEEDS_VISIBLE_VRAM)))1989 return -EINVAL;1990 1991 if (XE_IOCTL_DBG(xe, args->handle))1992 return -EINVAL;1993 1994 if (XE_IOCTL_DBG(xe, !args->size))1995 return -EINVAL;1996 1997 if (XE_IOCTL_DBG(xe, args->size > SIZE_MAX))1998 return -EINVAL;1999 2000 if (XE_IOCTL_DBG(xe, args->size & ~PAGE_MASK))2001 return -EINVAL;2002 2003 bo_flags = 0;2004 if (args->flags & DRM_XE_GEM_CREATE_FLAG_DEFER_BACKING)2005 bo_flags |= XE_BO_FLAG_DEFER_BACKING;2006 2007 if (args->flags & DRM_XE_GEM_CREATE_FLAG_SCANOUT)2008 bo_flags |= XE_BO_FLAG_SCANOUT;2009 2010 bo_flags |= args->placement << (ffs(XE_BO_FLAG_SYSTEM) - 1);2011 2012 /* CCS formats need physical placement at a 64K alignment in VRAM. */2013 if ((bo_flags & XE_BO_FLAG_VRAM_MASK) &&2014 (bo_flags & XE_BO_FLAG_SCANOUT) &&2015 !(xe->info.vram_flags & XE_VRAM_FLAGS_NEED64K) &&2016 IS_ALIGNED(args->size, SZ_64K))2017 bo_flags |= XE_BO_FLAG_NEEDS_64K;2018 2019 if (args->flags & DRM_XE_GEM_CREATE_FLAG_NEEDS_VISIBLE_VRAM) {2020 if (XE_IOCTL_DBG(xe, !(bo_flags & XE_BO_FLAG_VRAM_MASK)))2021 return -EINVAL;2022 2023 bo_flags |= XE_BO_FLAG_NEEDS_CPU_ACCESS;2024 }2025 2026 if (XE_IOCTL_DBG(xe, !args->cpu_caching ||2027 args->cpu_caching > DRM_XE_GEM_CPU_CACHING_WC))2028 return -EINVAL;2029 2030 if (XE_IOCTL_DBG(xe, bo_flags & XE_BO_FLAG_VRAM_MASK &&2031 args->cpu_caching != DRM_XE_GEM_CPU_CACHING_WC))2032 return -EINVAL;2033 2034 if (XE_IOCTL_DBG(xe, bo_flags & XE_BO_FLAG_SCANOUT &&2035 args->cpu_caching == DRM_XE_GEM_CPU_CACHING_WB))2036 return -EINVAL;2037 2038 if (args->vm_id) {2039 vm = xe_vm_lookup(xef, args->vm_id);2040 if (XE_IOCTL_DBG(xe, !vm))2041 return -ENOENT;2042 err = xe_vm_lock(vm, true);2043 if (err)2044 goto out_vm;2045 }2046 2047 bo = xe_bo_create_user(xe, NULL, vm, args->size, args->cpu_caching,2048 bo_flags);2049 2050 if (vm)2051 xe_vm_unlock(vm);2052 2053 if (IS_ERR(bo)) {2054 err = PTR_ERR(bo);2055 goto out_vm;2056 }2057 2058 err = drm_gem_handle_create(file, &bo->ttm.base, &handle);2059 if (err)2060 goto out_bulk;2061 2062 args->handle = handle;2063 goto out_put;2064 2065out_bulk:2066 if (vm && !xe_vm_in_fault_mode(vm)) {2067 xe_vm_lock(vm, false);2068 __xe_bo_unset_bulk_move(bo);2069 xe_vm_unlock(vm);2070 }2071out_put:2072 xe_bo_put(bo);2073out_vm:2074 if (vm)2075 xe_vm_put(vm);2076 2077 return err;2078}2079 2080int xe_gem_mmap_offset_ioctl(struct drm_device *dev, void *data,2081 struct drm_file *file)2082{2083 struct xe_device *xe = to_xe_device(dev);2084 struct drm_xe_gem_mmap_offset *args = data;2085 struct drm_gem_object *gem_obj;2086 2087 if (XE_IOCTL_DBG(xe, args->extensions) ||2088 XE_IOCTL_DBG(xe, args->reserved[0] || args->reserved[1]))2089 return -EINVAL;2090 2091 if (XE_IOCTL_DBG(xe, args->flags))2092 return -EINVAL;2093 2094 gem_obj = drm_gem_object_lookup(file, args->handle);2095 if (XE_IOCTL_DBG(xe, !gem_obj))2096 return -ENOENT;2097 2098 /* The mmap offset was set up at BO allocation time. */2099 args->offset = drm_vma_node_offset_addr(&gem_obj->vma_node);2100 2101 xe_bo_put(gem_to_xe_bo(gem_obj));2102 return 0;2103}2104 2105/**2106 * xe_bo_lock() - Lock the buffer object's dma_resv object2107 * @bo: The struct xe_bo whose lock is to be taken2108 * @intr: Whether to perform any wait interruptible2109 *2110 * Locks the buffer object's dma_resv object. If the buffer object is2111 * pointing to a shared dma_resv object, that shared lock is locked.2112 *2113 * Return: 0 on success, -EINTR if @intr is true and the wait for a2114 * contended lock was interrupted. If @intr is set to false, the2115 * function always returns 0.2116 */2117int xe_bo_lock(struct xe_bo *bo, bool intr)2118{2119 if (intr)2120 return dma_resv_lock_interruptible(bo->ttm.base.resv, NULL);2121 2122 dma_resv_lock(bo->ttm.base.resv, NULL);2123 2124 return 0;2125}2126 2127/**2128 * xe_bo_unlock() - Unlock the buffer object's dma_resv object2129 * @bo: The struct xe_bo whose lock is to be released.2130 *2131 * Unlock a buffer object lock that was locked by xe_bo_lock().2132 */2133void xe_bo_unlock(struct xe_bo *bo)2134{2135 dma_resv_unlock(bo->ttm.base.resv);2136}2137 2138/**2139 * xe_bo_can_migrate - Whether a buffer object likely can be migrated2140 * @bo: The buffer object to migrate2141 * @mem_type: The TTM memory type intended to migrate to2142 *2143 * Check whether the buffer object supports migration to the2144 * given memory type. Note that pinning may affect the ability to migrate as2145 * returned by this function.2146 *2147 * This function is primarily intended as a helper for checking the2148 * possibility to migrate buffer objects and can be called without2149 * the object lock held.2150 *2151 * Return: true if migration is possible, false otherwise.2152 */2153bool xe_bo_can_migrate(struct xe_bo *bo, u32 mem_type)2154{2155 unsigned int cur_place;2156 2157 if (bo->ttm.type == ttm_bo_type_kernel)2158 return true;2159 2160 if (bo->ttm.type == ttm_bo_type_sg)2161 return false;2162 2163 for (cur_place = 0; cur_place < bo->placement.num_placement;2164 cur_place++) {2165 if (bo->placements[cur_place].mem_type == mem_type)2166 return true;2167 }2168 2169 return false;2170}2171 2172static void xe_place_from_ttm_type(u32 mem_type, struct ttm_place *place)2173{2174 memset(place, 0, sizeof(*place));2175 place->mem_type = mem_type;2176}2177 2178/**2179 * xe_bo_migrate - Migrate an object to the desired region id2180 * @bo: The buffer object to migrate.2181 * @mem_type: The TTM region type to migrate to.2182 *2183 * Attempt to migrate the buffer object to the desired memory region. The2184 * buffer object may not be pinned, and must be locked.2185 * On successful completion, the object memory type will be updated,2186 * but an async migration task may not have completed yet, and to2187 * accomplish that, the object's kernel fences must be signaled with2188 * the object lock held.2189 *2190 * Return: 0 on success. Negative error code on failure. In particular may2191 * return -EINTR or -ERESTARTSYS if signal pending.2192 */2193int xe_bo_migrate(struct xe_bo *bo, u32 mem_type)2194{2195 struct xe_device *xe = ttm_to_xe_device(bo->ttm.bdev);2196 struct ttm_operation_ctx ctx = {2197 .interruptible = true,2198 .no_wait_gpu = false,2199 };2200 struct ttm_placement placement;2201 struct ttm_place requested;2202 2203 xe_bo_assert_held(bo);2204 2205 if (bo->ttm.resource->mem_type == mem_type)2206 return 0;2207 2208 if (xe_bo_is_pinned(bo))2209 return -EBUSY;2210 2211 if (!xe_bo_can_migrate(bo, mem_type))2212 return -EINVAL;2213 2214 xe_place_from_ttm_type(mem_type, &requested);2215 placement.num_placement = 1;2216 placement.placement = &requested;2217 2218 /*2219 * Stolen needs to be handled like below VRAM handling if we ever need2220 * to support it.2221 */2222 drm_WARN_ON(&xe->drm, mem_type == XE_PL_STOLEN);2223 2224 if (mem_type_is_vram(mem_type)) {2225 u32 c = 0;2226 2227 add_vram(xe, bo, &requested, bo->flags, mem_type, &c);2228 }2229 2230 return ttm_bo_validate(&bo->ttm, &placement, &ctx);2231}2232 2233/**2234 * xe_bo_evict - Evict an object to evict placement2235 * @bo: The buffer object to migrate.2236 * @force_alloc: Set force_alloc in ttm_operation_ctx2237 *2238 * On successful completion, the object memory will be moved to evict2239 * placement. Ths function blocks until the object has been fully moved.2240 *2241 * Return: 0 on success. Negative error code on failure.2242 */2243int xe_bo_evict(struct xe_bo *bo, bool force_alloc)2244{2245 struct ttm_operation_ctx ctx = {2246 .interruptible = false,2247 .no_wait_gpu = false,2248 .force_alloc = force_alloc,2249 };2250 struct ttm_placement placement;2251 int ret;2252 2253 xe_evict_flags(&bo->ttm, &placement);2254 ret = ttm_bo_validate(&bo->ttm, &placement, &ctx);2255 if (ret)2256 return ret;2257 2258 dma_resv_wait_timeout(bo->ttm.base.resv, DMA_RESV_USAGE_KERNEL,2259 false, MAX_SCHEDULE_TIMEOUT);2260 2261 return 0;2262}2263 2264/**2265 * xe_bo_needs_ccs_pages - Whether a bo needs to back up CCS pages when2266 * placed in system memory.2267 * @bo: The xe_bo2268 *2269 * Return: true if extra pages need to be allocated, false otherwise.2270 */2271bool xe_bo_needs_ccs_pages(struct xe_bo *bo)2272{2273 struct xe_device *xe = xe_bo_device(bo);2274 2275 if (GRAPHICS_VER(xe) >= 20 && IS_DGFX(xe))2276 return false;2277 2278 if (!xe_device_has_flat_ccs(xe) || bo->ttm.type != ttm_bo_type_device)2279 return false;2280 2281 /* On discrete GPUs, if the GPU can access this buffer from2282 * system memory (i.e., it allows XE_PL_TT placement), FlatCCS2283 * can't be used since there's no CCS storage associated with2284 * non-VRAM addresses.2285 */2286 if (IS_DGFX(xe) && (bo->flags & XE_BO_FLAG_SYSTEM))2287 return false;2288 2289 return true;2290}2291 2292/**2293 * __xe_bo_release_dummy() - Dummy kref release function2294 * @kref: The embedded struct kref.2295 *2296 * Dummy release function for xe_bo_put_deferred(). Keep off.2297 */2298void __xe_bo_release_dummy(struct kref *kref)2299{2300}2301 2302/**2303 * xe_bo_put_commit() - Put bos whose put was deferred by xe_bo_put_deferred().2304 * @deferred: The lockless list used for the call to xe_bo_put_deferred().2305 *2306 * Puts all bos whose put was deferred by xe_bo_put_deferred().2307 * The @deferred list can be either an onstack local list or a global2308 * shared list used by a workqueue.2309 */2310void xe_bo_put_commit(struct llist_head *deferred)2311{2312 struct llist_node *freed;2313 struct xe_bo *bo, *next;2314 2315 if (!deferred)2316 return;2317 2318 freed = llist_del_all(deferred);2319 if (!freed)2320 return;2321 2322 llist_for_each_entry_safe(bo, next, freed, freed)2323 drm_gem_object_free(&bo->ttm.base.refcount);2324}2325 2326void xe_bo_put(struct xe_bo *bo)2327{2328 might_sleep();2329 if (bo) {2330#ifdef CONFIG_PROC_FS2331 if (bo->client)2332 might_lock(&bo->client->bos_lock);2333#endif2334 if (bo->ggtt_node && bo->ggtt_node->ggtt)2335 might_lock(&bo->ggtt_node->ggtt->lock);2336 drm_gem_object_put(&bo->ttm.base);2337 }2338}2339 2340/**2341 * xe_bo_dumb_create - Create a dumb bo as backing for a fb2342 * @file_priv: ...2343 * @dev: ...2344 * @args: ...2345 *2346 * See dumb_create() hook in include/drm/drm_drv.h2347 *2348 * Return: ...2349 */2350int xe_bo_dumb_create(struct drm_file *file_priv,2351 struct drm_device *dev,2352 struct drm_mode_create_dumb *args)2353{2354 struct xe_device *xe = to_xe_device(dev);2355 struct xe_bo *bo;2356 uint32_t handle;2357 int cpp = DIV_ROUND_UP(args->bpp, 8);2358 int err;2359 u32 page_size = max_t(u32, PAGE_SIZE,2360 xe->info.vram_flags & XE_VRAM_FLAGS_NEED64K ? SZ_64K : SZ_4K);2361 2362 args->pitch = ALIGN(args->width * cpp, 64);2363 args->size = ALIGN(mul_u32_u32(args->pitch, args->height),2364 page_size);2365 2366 bo = xe_bo_create_user(xe, NULL, NULL, args->size,2367 DRM_XE_GEM_CPU_CACHING_WC,2368 XE_BO_FLAG_VRAM_IF_DGFX(xe_device_get_root_tile(xe)) |2369 XE_BO_FLAG_SCANOUT |2370 XE_BO_FLAG_NEEDS_CPU_ACCESS);2371 if (IS_ERR(bo))2372 return PTR_ERR(bo);2373 2374 err = drm_gem_handle_create(file_priv, &bo->ttm.base, &handle);2375 /* drop reference from allocate - handle holds it now */2376 drm_gem_object_put(&bo->ttm.base);2377 if (!err)2378 args->handle = handle;2379 return err;2380}2381 2382void xe_bo_runtime_pm_release_mmap_offset(struct xe_bo *bo)2383{2384 struct ttm_buffer_object *tbo = &bo->ttm;2385 struct ttm_device *bdev = tbo->bdev;2386 2387 drm_vma_node_unmap(&tbo->base.vma_node, bdev->dev_mapping);2388 2389 list_del_init(&bo->vram_userfault_link);2390}2391 2392#if IS_ENABLED(CONFIG_DRM_XE_KUNIT_TEST)2393#include "tests/xe_bo.c"2394#endif2395