380 lines · c
1/*2 * Copyright 2008 Advanced Micro Devices, Inc.3 * Copyright 2008 Red Hat Inc.4 * Copyright 2009 Jerome Glisse.5 *6 * Permission is hereby granted, free of charge, to any person obtaining a7 * copy of this software and associated documentation files (the "Software"),8 * to deal in the Software without restriction, including without limitation9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,10 * and/or sell copies of the Software, and to permit persons to whom the11 * Software is furnished to do so, subject to the following conditions:12 *13 * The above copyright notice and this permission notice shall be included in14 * all copies or substantial portions of the Software.15 *16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL19 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR22 * OTHER DEALINGS IN THE SOFTWARE.23 *24 * Authors: Dave Airlie25 * Alex Deucher26 * Jerome Glisse27 */28#ifndef __AMDGPU_OBJECT_H__29#define __AMDGPU_OBJECT_H__30 31#include <drm/amdgpu_drm.h>32#include "amdgpu.h"33#include "amdgpu_res_cursor.h"34 35#ifdef CONFIG_MMU_NOTIFIER36#include <linux/mmu_notifier.h>37#endif38 39#define AMDGPU_BO_INVALID_OFFSET LONG_MAX40#define AMDGPU_BO_MAX_PLACEMENTS 341 42/* BO flag to indicate a KFD userptr BO */43#define AMDGPU_AMDKFD_CREATE_USERPTR_BO (1ULL << 63)44 45#define to_amdgpu_bo_user(abo) container_of((abo), struct amdgpu_bo_user, bo)46#define to_amdgpu_bo_vm(abo) container_of((abo), struct amdgpu_bo_vm, bo)47 48struct amdgpu_bo_param {49 unsigned long size;50 int byte_align;51 u32 bo_ptr_size;52 u32 domain;53 u32 preferred_domain;54 u64 flags;55 enum ttm_bo_type type;56 bool no_wait_gpu;57 struct dma_resv *resv;58 void (*destroy)(struct ttm_buffer_object *bo);59 /* xcp partition number plus 1, 0 means any partition */60 int8_t xcp_id_plus1;61};62 63/* bo virtual addresses in a vm */64struct amdgpu_bo_va_mapping {65 struct amdgpu_bo_va *bo_va;66 struct list_head list;67 struct rb_node rb;68 uint64_t start;69 uint64_t last;70 uint64_t __subtree_last;71 uint64_t offset;72 uint64_t flags;73};74 75/* User space allocated BO in a VM */76struct amdgpu_bo_va {77 struct amdgpu_vm_bo_base base;78 79 /* protected by bo being reserved */80 unsigned ref_count;81 82 /* all other members protected by the VM PD being reserved */83 struct dma_fence *last_pt_update;84 85 /* mappings for this bo_va */86 struct list_head invalids;87 struct list_head valids;88 89 /* If the mappings are cleared or filled */90 bool cleared;91 92 bool is_xgmi;93 94 /*95 * protected by vm reservation lock96 * if non-zero, cannot unmap from GPU because user queues may still access it97 */98 unsigned int queue_refcount;99};100 101struct amdgpu_bo {102 /* Protected by tbo.reserved */103 u32 preferred_domains;104 u32 allowed_domains;105 struct ttm_place placements[AMDGPU_BO_MAX_PLACEMENTS];106 struct ttm_placement placement;107 struct ttm_buffer_object tbo;108 struct ttm_bo_kmap_obj kmap;109 u64 flags;110 /* per VM structure for page tables and with virtual addresses */111 struct amdgpu_vm_bo_base *vm_bo;112 /* Constant after initialization */113 struct amdgpu_bo *parent;114 115#ifdef CONFIG_MMU_NOTIFIER116 struct mmu_interval_notifier notifier;117#endif118 struct kgd_mem *kfd_bo;119 120 /*121 * For GPUs with spatial partitioning, xcp partition number, -1 means122 * any partition. For other ASICs without spatial partition, always 0123 * for memory accounting.124 */125 int8_t xcp_id;126};127 128struct amdgpu_bo_user {129 struct amdgpu_bo bo;130 u64 tiling_flags;131 u64 metadata_flags;132 void *metadata;133 u32 metadata_size;134 135};136 137struct amdgpu_bo_vm {138 struct amdgpu_bo bo;139 struct amdgpu_vm_bo_base entries[];140};141 142struct amdgpu_mem_stats {143 /* current VRAM usage, includes visible VRAM */144 uint64_t vram;145 /* current shared VRAM usage, includes visible VRAM */146 uint64_t vram_shared;147 /* current visible VRAM usage */148 uint64_t visible_vram;149 /* current GTT usage */150 uint64_t gtt;151 /* current shared GTT usage */152 uint64_t gtt_shared;153 /* current system memory usage */154 uint64_t cpu;155 /* current shared system memory usage */156 uint64_t cpu_shared;157 /* sum of evicted buffers, includes visible VRAM */158 uint64_t evicted_vram;159 /* sum of evicted buffers due to CPU access */160 uint64_t evicted_visible_vram;161 /* how much userspace asked for, includes vis.VRAM */162 uint64_t requested_vram;163 /* how much userspace asked for */164 uint64_t requested_visible_vram;165 /* how much userspace asked for */166 uint64_t requested_gtt;167};168 169static inline struct amdgpu_bo *ttm_to_amdgpu_bo(struct ttm_buffer_object *tbo)170{171 return container_of(tbo, struct amdgpu_bo, tbo);172}173 174/**175 * amdgpu_mem_type_to_domain - return domain corresponding to mem_type176 * @mem_type: ttm memory type177 *178 * Returns corresponding domain of the ttm mem_type179 */180static inline unsigned amdgpu_mem_type_to_domain(u32 mem_type)181{182 switch (mem_type) {183 case TTM_PL_VRAM:184 return AMDGPU_GEM_DOMAIN_VRAM;185 case TTM_PL_TT:186 return AMDGPU_GEM_DOMAIN_GTT;187 case TTM_PL_SYSTEM:188 return AMDGPU_GEM_DOMAIN_CPU;189 case AMDGPU_PL_GDS:190 return AMDGPU_GEM_DOMAIN_GDS;191 case AMDGPU_PL_GWS:192 return AMDGPU_GEM_DOMAIN_GWS;193 case AMDGPU_PL_OA:194 return AMDGPU_GEM_DOMAIN_OA;195 case AMDGPU_PL_DOORBELL:196 return AMDGPU_GEM_DOMAIN_DOORBELL;197 default:198 break;199 }200 return 0;201}202 203/**204 * amdgpu_bo_reserve - reserve bo205 * @bo: bo structure206 * @no_intr: don't return -ERESTARTSYS on pending signal207 *208 * Returns:209 * -ERESTARTSYS: A wait for the buffer to become unreserved was interrupted by210 * a signal. Release all buffer reservations and return to user-space.211 */212static inline int amdgpu_bo_reserve(struct amdgpu_bo *bo, bool no_intr)213{214 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);215 int r;216 217 r = ttm_bo_reserve(&bo->tbo, !no_intr, false, NULL);218 if (unlikely(r != 0)) {219 if (r != -ERESTARTSYS)220 dev_err(adev->dev, "%p reserve failed\n", bo);221 return r;222 }223 return 0;224}225 226static inline void amdgpu_bo_unreserve(struct amdgpu_bo *bo)227{228 ttm_bo_unreserve(&bo->tbo);229}230 231static inline unsigned long amdgpu_bo_size(struct amdgpu_bo *bo)232{233 return bo->tbo.base.size;234}235 236static inline unsigned amdgpu_bo_ngpu_pages(struct amdgpu_bo *bo)237{238 return bo->tbo.base.size / AMDGPU_GPU_PAGE_SIZE;239}240 241static inline unsigned amdgpu_bo_gpu_page_alignment(struct amdgpu_bo *bo)242{243 return (bo->tbo.page_alignment << PAGE_SHIFT) / AMDGPU_GPU_PAGE_SIZE;244}245 246/**247 * amdgpu_bo_mmap_offset - return mmap offset of bo248 * @bo: amdgpu object for which we query the offset249 *250 * Returns mmap offset of the object.251 */252static inline u64 amdgpu_bo_mmap_offset(struct amdgpu_bo *bo)253{254 return drm_vma_node_offset_addr(&bo->tbo.base.vma_node);255}256 257/**258 * amdgpu_bo_explicit_sync - return whether the bo is explicitly synced259 */260static inline bool amdgpu_bo_explicit_sync(struct amdgpu_bo *bo)261{262 return bo->flags & AMDGPU_GEM_CREATE_EXPLICIT_SYNC;263}264 265/**266 * amdgpu_bo_encrypted - test if the BO is encrypted267 * @bo: pointer to a buffer object268 *269 * Return true if the buffer object is encrypted, false otherwise.270 */271static inline bool amdgpu_bo_encrypted(struct amdgpu_bo *bo)272{273 return bo->flags & AMDGPU_GEM_CREATE_ENCRYPTED;274}275 276bool amdgpu_bo_is_amdgpu_bo(struct ttm_buffer_object *bo);277void amdgpu_bo_placement_from_domain(struct amdgpu_bo *abo, u32 domain);278 279int amdgpu_bo_create(struct amdgpu_device *adev,280 struct amdgpu_bo_param *bp,281 struct amdgpu_bo **bo_ptr);282int amdgpu_bo_create_reserved(struct amdgpu_device *adev,283 unsigned long size, int align,284 u32 domain, struct amdgpu_bo **bo_ptr,285 u64 *gpu_addr, void **cpu_addr);286int amdgpu_bo_create_kernel(struct amdgpu_device *adev,287 unsigned long size, int align,288 u32 domain, struct amdgpu_bo **bo_ptr,289 u64 *gpu_addr, void **cpu_addr);290int amdgpu_bo_create_kernel_at(struct amdgpu_device *adev,291 uint64_t offset, uint64_t size,292 struct amdgpu_bo **bo_ptr, void **cpu_addr);293int amdgpu_bo_create_user(struct amdgpu_device *adev,294 struct amdgpu_bo_param *bp,295 struct amdgpu_bo_user **ubo_ptr);296int amdgpu_bo_create_vm(struct amdgpu_device *adev,297 struct amdgpu_bo_param *bp,298 struct amdgpu_bo_vm **ubo_ptr);299void amdgpu_bo_free_kernel(struct amdgpu_bo **bo, u64 *gpu_addr,300 void **cpu_addr);301int amdgpu_bo_kmap(struct amdgpu_bo *bo, void **ptr);302void *amdgpu_bo_kptr(struct amdgpu_bo *bo);303void amdgpu_bo_kunmap(struct amdgpu_bo *bo);304struct amdgpu_bo *amdgpu_bo_ref(struct amdgpu_bo *bo);305void amdgpu_bo_unref(struct amdgpu_bo **bo);306int amdgpu_bo_pin(struct amdgpu_bo *bo, u32 domain);307void amdgpu_bo_unpin(struct amdgpu_bo *bo);308int amdgpu_bo_init(struct amdgpu_device *adev);309void amdgpu_bo_fini(struct amdgpu_device *adev);310int amdgpu_bo_set_tiling_flags(struct amdgpu_bo *bo, u64 tiling_flags);311void amdgpu_bo_get_tiling_flags(struct amdgpu_bo *bo, u64 *tiling_flags);312int amdgpu_bo_set_metadata (struct amdgpu_bo *bo, void *metadata,313 uint32_t metadata_size, uint64_t flags);314int amdgpu_bo_get_metadata(struct amdgpu_bo *bo, void *buffer,315 size_t buffer_size, uint32_t *metadata_size,316 uint64_t *flags);317void amdgpu_bo_move_notify(struct ttm_buffer_object *bo,318 bool evict,319 struct ttm_resource *new_mem);320void amdgpu_bo_release_notify(struct ttm_buffer_object *bo);321vm_fault_t amdgpu_bo_fault_reserve_notify(struct ttm_buffer_object *bo);322void amdgpu_bo_fence(struct amdgpu_bo *bo, struct dma_fence *fence,323 bool shared);324int amdgpu_bo_sync_wait_resv(struct amdgpu_device *adev, struct dma_resv *resv,325 enum amdgpu_sync_mode sync_mode, void *owner,326 bool intr);327int amdgpu_bo_sync_wait(struct amdgpu_bo *bo, void *owner, bool intr);328u64 amdgpu_bo_gpu_offset(struct amdgpu_bo *bo);329u64 amdgpu_bo_gpu_offset_no_check(struct amdgpu_bo *bo);330void amdgpu_bo_get_memory(struct amdgpu_bo *bo,331 struct amdgpu_mem_stats *stats);332uint32_t amdgpu_bo_get_preferred_domain(struct amdgpu_device *adev,333 uint32_t domain);334 335/*336 * sub allocation337 */338static inline struct amdgpu_sa_manager *339to_amdgpu_sa_manager(struct drm_suballoc_manager *manager)340{341 return container_of(manager, struct amdgpu_sa_manager, base);342}343 344static inline uint64_t amdgpu_sa_bo_gpu_addr(struct drm_suballoc *sa_bo)345{346 return to_amdgpu_sa_manager(sa_bo->manager)->gpu_addr +347 drm_suballoc_soffset(sa_bo);348}349 350static inline void *amdgpu_sa_bo_cpu_addr(struct drm_suballoc *sa_bo)351{352 return to_amdgpu_sa_manager(sa_bo->manager)->cpu_ptr +353 drm_suballoc_soffset(sa_bo);354}355 356int amdgpu_sa_bo_manager_init(struct amdgpu_device *adev,357 struct amdgpu_sa_manager *sa_manager,358 unsigned size, u32 align, u32 domain);359void amdgpu_sa_bo_manager_fini(struct amdgpu_device *adev,360 struct amdgpu_sa_manager *sa_manager);361int amdgpu_sa_bo_manager_start(struct amdgpu_device *adev,362 struct amdgpu_sa_manager *sa_manager);363int amdgpu_sa_bo_new(struct amdgpu_sa_manager *sa_manager,364 struct drm_suballoc **sa_bo,365 unsigned int size);366void amdgpu_sa_bo_free(struct amdgpu_device *adev,367 struct drm_suballoc **sa_bo,368 struct dma_fence *fence);369#if defined(CONFIG_DEBUG_FS)370void amdgpu_sa_bo_dump_debug_info(struct amdgpu_sa_manager *sa_manager,371 struct seq_file *m);372u64 amdgpu_bo_print_info(int id, struct amdgpu_bo *bo, struct seq_file *m);373#endif374void amdgpu_debugfs_sa_init(struct amdgpu_device *adev);375 376bool amdgpu_bo_support_uswc(u64 bo_flags);377 378 379#endif380