333 lines · c
1/* SPDX-License-Identifier: MIT */2/*3 * Copyright © 2021 Intel Corporation4 */5 6#ifndef _XE_BO_H_7#define _XE_BO_H_8 9#include <drm/ttm/ttm_tt.h>10 11#include "xe_bo_types.h"12#include "xe_macros.h"13#include "xe_vm_types.h"14#include "xe_vm.h"15 16#define XE_DEFAULT_GTT_SIZE_MB 3072ULL /* 3GB by default */17 18#define XE_BO_FLAG_USER BIT(0)19/* The bits below need to be contiguous, or things break */20#define XE_BO_FLAG_SYSTEM BIT(1)21#define XE_BO_FLAG_VRAM0 BIT(2)22#define XE_BO_FLAG_VRAM1 BIT(3)23#define XE_BO_FLAG_VRAM_MASK (XE_BO_FLAG_VRAM0 | XE_BO_FLAG_VRAM1)24/* -- */25#define XE_BO_FLAG_STOLEN BIT(4)26#define XE_BO_FLAG_VRAM_IF_DGFX(tile) (IS_DGFX(tile_to_xe(tile)) ? \27 XE_BO_FLAG_VRAM0 << (tile)->id : \28 XE_BO_FLAG_SYSTEM)29#define XE_BO_FLAG_GGTT BIT(5)30#define XE_BO_FLAG_IGNORE_MIN_PAGE_SIZE BIT(6)31#define XE_BO_FLAG_PINNED BIT(7)32#define XE_BO_FLAG_NO_RESV_EVICT BIT(8)33#define XE_BO_FLAG_DEFER_BACKING BIT(9)34#define XE_BO_FLAG_SCANOUT BIT(10)35#define XE_BO_FLAG_FIXED_PLACEMENT BIT(11)36#define XE_BO_FLAG_PAGETABLE BIT(12)37#define XE_BO_FLAG_NEEDS_CPU_ACCESS BIT(13)38#define XE_BO_FLAG_NEEDS_UC BIT(14)39#define XE_BO_FLAG_NEEDS_64K BIT(15)40#define XE_BO_FLAG_NEEDS_2M BIT(16)41#define XE_BO_FLAG_GGTT_INVALIDATE BIT(17)42/* this one is trigger internally only */43#define XE_BO_FLAG_INTERNAL_TEST BIT(30)44#define XE_BO_FLAG_INTERNAL_64K BIT(31)45 46#define XE_PTE_SHIFT 1247#define XE_PAGE_SIZE (1 << XE_PTE_SHIFT)48#define XE_PTE_MASK (XE_PAGE_SIZE - 1)49#define XE_PDE_SHIFT (XE_PTE_SHIFT - 3)50#define XE_PDES (1 << XE_PDE_SHIFT)51#define XE_PDE_MASK (XE_PDES - 1)52 53#define XE_64K_PTE_SHIFT 1654#define XE_64K_PAGE_SIZE (1 << XE_64K_PTE_SHIFT)55#define XE_64K_PTE_MASK (XE_64K_PAGE_SIZE - 1)56#define XE_64K_PDE_MASK (XE_PDE_MASK >> 4)57 58#define XE_PL_SYSTEM TTM_PL_SYSTEM59#define XE_PL_TT TTM_PL_TT60#define XE_PL_VRAM0 TTM_PL_VRAM61#define XE_PL_VRAM1 (XE_PL_VRAM0 + 1)62#define XE_PL_STOLEN (TTM_NUM_MEM_TYPES - 1)63 64#define XE_BO_PROPS_INVALID (-1)65 66struct sg_table;67 68struct xe_bo *xe_bo_alloc(void);69void xe_bo_free(struct xe_bo *bo);70 71struct xe_bo *___xe_bo_create_locked(struct xe_device *xe, struct xe_bo *bo,72 struct xe_tile *tile, struct dma_resv *resv,73 struct ttm_lru_bulk_move *bulk, size_t size,74 u16 cpu_caching, enum ttm_bo_type type,75 u32 flags);76struct xe_bo *77xe_bo_create_locked_range(struct xe_device *xe,78 struct xe_tile *tile, struct xe_vm *vm,79 size_t size, u64 start, u64 end,80 enum ttm_bo_type type, u32 flags);81struct xe_bo *xe_bo_create_locked(struct xe_device *xe, struct xe_tile *tile,82 struct xe_vm *vm, size_t size,83 enum ttm_bo_type type, u32 flags);84struct xe_bo *xe_bo_create(struct xe_device *xe, struct xe_tile *tile,85 struct xe_vm *vm, size_t size,86 enum ttm_bo_type type, u32 flags);87struct xe_bo *xe_bo_create_user(struct xe_device *xe, struct xe_tile *tile,88 struct xe_vm *vm, size_t size,89 u16 cpu_caching,90 u32 flags);91struct xe_bo *xe_bo_create_pin_map(struct xe_device *xe, struct xe_tile *tile,92 struct xe_vm *vm, size_t size,93 enum ttm_bo_type type, u32 flags);94struct xe_bo *xe_bo_create_pin_map_at(struct xe_device *xe, struct xe_tile *tile,95 struct xe_vm *vm, size_t size, u64 offset,96 enum ttm_bo_type type, u32 flags);97struct xe_bo *xe_bo_create_from_data(struct xe_device *xe, struct xe_tile *tile,98 const void *data, size_t size,99 enum ttm_bo_type type, u32 flags);100struct xe_bo *xe_managed_bo_create_pin_map(struct xe_device *xe, struct xe_tile *tile,101 size_t size, u32 flags);102struct xe_bo *xe_managed_bo_create_from_data(struct xe_device *xe, struct xe_tile *tile,103 const void *data, size_t size, u32 flags);104int xe_managed_bo_reinit_in_vram(struct xe_device *xe, struct xe_tile *tile, struct xe_bo **src);105 106int xe_bo_placement_for_flags(struct xe_device *xe, struct xe_bo *bo,107 u32 bo_flags);108 109static inline struct xe_bo *ttm_to_xe_bo(const struct ttm_buffer_object *bo)110{111 return container_of(bo, struct xe_bo, ttm);112}113 114static inline struct xe_bo *gem_to_xe_bo(const struct drm_gem_object *obj)115{116 return container_of(obj, struct xe_bo, ttm.base);117}118 119#define xe_bo_device(bo) ttm_to_xe_device((bo)->ttm.bdev)120 121static inline struct xe_bo *xe_bo_get(struct xe_bo *bo)122{123 if (bo)124 drm_gem_object_get(&bo->ttm.base);125 126 return bo;127}128 129void xe_bo_put(struct xe_bo *bo);130 131static inline void __xe_bo_unset_bulk_move(struct xe_bo *bo)132{133 if (bo)134 ttm_bo_set_bulk_move(&bo->ttm, NULL);135}136 137static inline void xe_bo_assert_held(struct xe_bo *bo)138{139 if (bo)140 dma_resv_assert_held((bo)->ttm.base.resv);141}142 143int xe_bo_lock(struct xe_bo *bo, bool intr);144 145void xe_bo_unlock(struct xe_bo *bo);146 147static inline void xe_bo_unlock_vm_held(struct xe_bo *bo)148{149 if (bo) {150 XE_WARN_ON(bo->vm && bo->ttm.base.resv != xe_vm_resv(bo->vm));151 if (bo->vm)152 xe_vm_assert_held(bo->vm);153 else154 dma_resv_unlock(bo->ttm.base.resv);155 }156}157 158int xe_bo_pin_external(struct xe_bo *bo);159int xe_bo_pin(struct xe_bo *bo);160void xe_bo_unpin_external(struct xe_bo *bo);161void xe_bo_unpin(struct xe_bo *bo);162int xe_bo_validate(struct xe_bo *bo, struct xe_vm *vm, bool allow_res_evict);163 164static inline bool xe_bo_is_pinned(struct xe_bo *bo)165{166 return bo->ttm.pin_count;167}168 169static inline void xe_bo_unpin_map_no_vm(struct xe_bo *bo)170{171 if (likely(bo)) {172 xe_bo_lock(bo, false);173 xe_bo_unpin(bo);174 xe_bo_unlock(bo);175 176 xe_bo_put(bo);177 }178}179 180bool xe_bo_is_xe_bo(struct ttm_buffer_object *bo);181dma_addr_t __xe_bo_addr(struct xe_bo *bo, u64 offset, size_t page_size);182dma_addr_t xe_bo_addr(struct xe_bo *bo, u64 offset, size_t page_size);183 184static inline dma_addr_t185xe_bo_main_addr(struct xe_bo *bo, size_t page_size)186{187 return xe_bo_addr(bo, 0, page_size);188}189 190static inline u32191xe_bo_ggtt_addr(struct xe_bo *bo)192{193 if (XE_WARN_ON(!bo->ggtt_node))194 return 0;195 196 XE_WARN_ON(bo->ggtt_node->base.size > bo->size);197 XE_WARN_ON(bo->ggtt_node->base.start + bo->ggtt_node->base.size > (1ull << 32));198 return bo->ggtt_node->base.start;199}200 201int xe_bo_vmap(struct xe_bo *bo);202void xe_bo_vunmap(struct xe_bo *bo);203 204bool mem_type_is_vram(u32 mem_type);205bool xe_bo_is_vram(struct xe_bo *bo);206bool xe_bo_is_stolen(struct xe_bo *bo);207bool xe_bo_is_stolen_devmem(struct xe_bo *bo);208bool xe_bo_has_single_placement(struct xe_bo *bo);209uint64_t vram_region_gpu_offset(struct ttm_resource *res);210 211bool xe_bo_can_migrate(struct xe_bo *bo, u32 mem_type);212 213int xe_bo_migrate(struct xe_bo *bo, u32 mem_type);214int xe_bo_evict(struct xe_bo *bo, bool force_alloc);215 216int xe_bo_evict_pinned(struct xe_bo *bo);217int xe_bo_restore_pinned(struct xe_bo *bo);218 219extern const struct ttm_device_funcs xe_ttm_funcs;220extern const char *const xe_mem_type_to_name[];221 222int xe_gem_create_ioctl(struct drm_device *dev, void *data,223 struct drm_file *file);224int xe_gem_mmap_offset_ioctl(struct drm_device *dev, void *data,225 struct drm_file *file);226void xe_bo_runtime_pm_release_mmap_offset(struct xe_bo *bo);227 228int xe_bo_dumb_create(struct drm_file *file_priv,229 struct drm_device *dev,230 struct drm_mode_create_dumb *args);231 232bool xe_bo_needs_ccs_pages(struct xe_bo *bo);233 234static inline size_t xe_bo_ccs_pages_start(struct xe_bo *bo)235{236 return PAGE_ALIGN(bo->ttm.base.size);237}238 239static inline bool xe_bo_has_pages(struct xe_bo *bo)240{241 if ((bo->ttm.ttm && ttm_tt_is_populated(bo->ttm.ttm)) ||242 xe_bo_is_vram(bo))243 return true;244 245 return false;246}247 248void __xe_bo_release_dummy(struct kref *kref);249 250/**251 * xe_bo_put_deferred() - Put a buffer object with delayed final freeing252 * @bo: The bo to put.253 * @deferred: List to which to add the buffer object if we cannot put, or254 * NULL if the function is to put unconditionally.255 *256 * Since the final freeing of an object includes both sleeping and (!)257 * memory allocation in the dma_resv individualization, it's not ok258 * to put an object from atomic context nor from within a held lock259 * tainted by reclaim. In such situations we want to defer the final260 * freeing until we've exited the restricting context, or in the worst261 * case to a workqueue.262 * This function either puts the object if possible without the refcount263 * reaching zero, or adds it to the @deferred list if that was not possible.264 * The caller needs to follow up with a call to xe_bo_put_commit() to actually265 * put the bo iff this function returns true. It's safe to always266 * follow up with a call to xe_bo_put_commit().267 * TODO: It's TTM that is the villain here. Perhaps TTM should add an268 * interface like this.269 *270 * Return: true if @bo was the first object put on the @freed list,271 * false otherwise.272 */273static inline bool274xe_bo_put_deferred(struct xe_bo *bo, struct llist_head *deferred)275{276 if (!deferred) {277 xe_bo_put(bo);278 return false;279 }280 281 if (!kref_put(&bo->ttm.base.refcount, __xe_bo_release_dummy))282 return false;283 284 return llist_add(&bo->freed, deferred);285}286 287void xe_bo_put_commit(struct llist_head *deferred);288 289struct sg_table *xe_bo_sg(struct xe_bo *bo);290 291/*292 * xe_sg_segment_size() - Provides upper limit for sg segment size.293 * @dev: device pointer294 *295 * Returns the maximum segment size for the 'struct scatterlist'296 * elements.297 */298static inline unsigned int xe_sg_segment_size(struct device *dev)299{300 struct scatterlist __maybe_unused sg;301 size_t max = BIT_ULL(sizeof(sg.length) * 8) - 1;302 303 max = min_t(size_t, max, dma_max_mapping_size(dev));304 305 /*306 * The iommu_dma_map_sg() function ensures iova allocation doesn't307 * cross dma segment boundary. It does so by padding some sg elements.308 * This can cause overflow, ending up with sg->length being set to 0.309 * Avoid this by ensuring maximum segment size is half of 'max'310 * rounded down to PAGE_SIZE.311 */312 return round_down(max / 2, PAGE_SIZE);313}314 315#define i915_gem_object_flush_if_display(obj) ((void)(obj))316 317#if IS_ENABLED(CONFIG_DRM_XE_KUNIT_TEST)318/**319 * xe_bo_is_mem_type - Whether the bo currently resides in the given320 * TTM memory type321 * @bo: The bo to check.322 * @mem_type: The TTM memory type.323 *324 * Return: true iff the bo resides in @mem_type, false otherwise.325 */326static inline bool xe_bo_is_mem_type(struct xe_bo *bo, u32 mem_type)327{328 xe_bo_assert_held(bo);329 return bo->ttm.resource->mem_type == mem_type;330}331#endif332#endif333