232 lines · c
1/*2 * Copyright 2016 Advanced Micro Devices, Inc.3 *4 * Permission is hereby granted, free of charge, to any person obtaining a5 * copy of this software and associated documentation files (the "Software"),6 * to deal in the Software without restriction, including without limitation7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,8 * and/or sell copies of the Software, and to permit persons to whom the9 * Software is furnished to do so, subject to the following conditions:10 *11 * The above copyright notice and this permission notice shall be included in12 * all copies or substantial portions of the Software.13 *14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR20 * OTHER DEALINGS IN THE SOFTWARE.21 *22 */23 24#ifndef __AMDGPU_TTM_H__25#define __AMDGPU_TTM_H__26 27#include <linux/dma-direction.h>28#include <drm/gpu_scheduler.h>29#include "amdgpu_vram_mgr.h"30#include "amdgpu.h"31 32#define AMDGPU_PL_GDS (TTM_PL_PRIV + 0)33#define AMDGPU_PL_GWS (TTM_PL_PRIV + 1)34#define AMDGPU_PL_OA (TTM_PL_PRIV + 2)35#define AMDGPU_PL_PREEMPT (TTM_PL_PRIV + 3)36#define AMDGPU_PL_DOORBELL (TTM_PL_PRIV + 4)37 38#define AMDGPU_GTT_MAX_TRANSFER_SIZE 51239#define AMDGPU_GTT_NUM_TRANSFER_WINDOWS 240 41extern const struct attribute_group amdgpu_vram_mgr_attr_group;42extern const struct attribute_group amdgpu_gtt_mgr_attr_group;43 44struct hmm_range;45 46struct amdgpu_gtt_mgr {47 struct ttm_resource_manager manager;48 struct drm_mm mm;49 spinlock_t lock;50};51 52struct amdgpu_mman {53 struct ttm_device bdev;54 struct ttm_pool *ttm_pools;55 bool initialized;56 void __iomem *aper_base_kaddr;57 58 /* buffer handling */59 const struct amdgpu_buffer_funcs *buffer_funcs;60 struct amdgpu_ring *buffer_funcs_ring;61 bool buffer_funcs_enabled;62 63 struct mutex gtt_window_lock;64 /* High priority scheduler entity for buffer moves */65 struct drm_sched_entity high_pr;66 /* Low priority scheduler entity for VRAM clearing */67 struct drm_sched_entity low_pr;68 69 struct amdgpu_vram_mgr vram_mgr;70 struct amdgpu_gtt_mgr gtt_mgr;71 struct ttm_resource_manager preempt_mgr;72 73 uint64_t stolen_vga_size;74 struct amdgpu_bo *stolen_vga_memory;75 uint64_t stolen_extended_size;76 struct amdgpu_bo *stolen_extended_memory;77 bool keep_stolen_vga_memory;78 79 struct amdgpu_bo *stolen_reserved_memory;80 uint64_t stolen_reserved_offset;81 uint64_t stolen_reserved_size;82 83 /* discovery */84 uint8_t *discovery_bin;85 uint32_t discovery_tmr_size;86 /* fw reserved memory */87 struct amdgpu_bo *fw_reserved_memory;88 89 /* firmware VRAM reservation */90 u64 fw_vram_usage_start_offset;91 u64 fw_vram_usage_size;92 struct amdgpu_bo *fw_vram_usage_reserved_bo;93 void *fw_vram_usage_va;94 95 /* driver VRAM reservation */96 u64 drv_vram_usage_start_offset;97 u64 drv_vram_usage_size;98 struct amdgpu_bo *drv_vram_usage_reserved_bo;99 void *drv_vram_usage_va;100 101 /* PAGE_SIZE'd BO for process memory r/w over SDMA. */102 struct amdgpu_bo *sdma_access_bo;103 void *sdma_access_ptr;104};105 106struct amdgpu_copy_mem {107 struct ttm_buffer_object *bo;108 struct ttm_resource *mem;109 unsigned long offset;110};111 112#define AMDGPU_COPY_FLAGS_TMZ (1 << 0)113#define AMDGPU_COPY_FLAGS_READ_DECOMPRESSED (1 << 1)114#define AMDGPU_COPY_FLAGS_WRITE_COMPRESSED (1 << 2)115#define AMDGPU_COPY_FLAGS_MAX_COMPRESSED_SHIFT 3116#define AMDGPU_COPY_FLAGS_MAX_COMPRESSED_MASK 0x03117#define AMDGPU_COPY_FLAGS_NUMBER_TYPE_SHIFT 5118#define AMDGPU_COPY_FLAGS_NUMBER_TYPE_MASK 0x07119#define AMDGPU_COPY_FLAGS_DATA_FORMAT_SHIFT 8120#define AMDGPU_COPY_FLAGS_DATA_FORMAT_MASK 0x3f121 122#define AMDGPU_COPY_FLAGS_SET(field, value) \123 (((__u32)(value) & AMDGPU_COPY_FLAGS_##field##_MASK) << AMDGPU_COPY_FLAGS_##field##_SHIFT)124#define AMDGPU_COPY_FLAGS_GET(value, field) \125 (((__u32)(value) >> AMDGPU_COPY_FLAGS_##field##_SHIFT) & AMDGPU_COPY_FLAGS_##field##_MASK)126 127int amdgpu_gtt_mgr_init(struct amdgpu_device *adev, uint64_t gtt_size);128void amdgpu_gtt_mgr_fini(struct amdgpu_device *adev);129int amdgpu_preempt_mgr_init(struct amdgpu_device *adev);130void amdgpu_preempt_mgr_fini(struct amdgpu_device *adev);131int amdgpu_vram_mgr_init(struct amdgpu_device *adev);132void amdgpu_vram_mgr_fini(struct amdgpu_device *adev);133 134bool amdgpu_gtt_mgr_has_gart_addr(struct ttm_resource *mem);135void amdgpu_gtt_mgr_recover(struct amdgpu_gtt_mgr *mgr);136 137uint64_t amdgpu_preempt_mgr_usage(struct ttm_resource_manager *man);138 139u64 amdgpu_vram_mgr_bo_visible_size(struct amdgpu_bo *bo);140int amdgpu_vram_mgr_alloc_sgt(struct amdgpu_device *adev,141 struct ttm_resource *mem,142 u64 offset, u64 size,143 struct device *dev,144 enum dma_data_direction dir,145 struct sg_table **sgt);146void amdgpu_vram_mgr_free_sgt(struct device *dev,147 enum dma_data_direction dir,148 struct sg_table *sgt);149uint64_t amdgpu_vram_mgr_vis_usage(struct amdgpu_vram_mgr *mgr);150int amdgpu_vram_mgr_reserve_range(struct amdgpu_vram_mgr *mgr,151 uint64_t start, uint64_t size);152int amdgpu_vram_mgr_query_page_status(struct amdgpu_vram_mgr *mgr,153 uint64_t start);154 155bool amdgpu_res_cpu_visible(struct amdgpu_device *adev,156 struct ttm_resource *res);157 158int amdgpu_ttm_init(struct amdgpu_device *adev);159void amdgpu_ttm_fini(struct amdgpu_device *adev);160void amdgpu_ttm_set_buffer_funcs_status(struct amdgpu_device *adev,161 bool enable);162int amdgpu_copy_buffer(struct amdgpu_ring *ring, uint64_t src_offset,163 uint64_t dst_offset, uint32_t byte_count,164 struct dma_resv *resv,165 struct dma_fence **fence, bool direct_submit,166 bool vm_needs_flush, uint32_t copy_flags);167int amdgpu_ttm_copy_mem_to_mem(struct amdgpu_device *adev,168 const struct amdgpu_copy_mem *src,169 const struct amdgpu_copy_mem *dst,170 uint64_t size, bool tmz,171 struct dma_resv *resv,172 struct dma_fence **f);173int amdgpu_ttm_clear_buffer(struct amdgpu_bo *bo,174 struct dma_resv *resv,175 struct dma_fence **fence);176int amdgpu_fill_buffer(struct amdgpu_bo *bo,177 uint32_t src_data,178 struct dma_resv *resv,179 struct dma_fence **fence,180 bool delayed);181 182int amdgpu_ttm_alloc_gart(struct ttm_buffer_object *bo);183void amdgpu_ttm_recover_gart(struct ttm_buffer_object *tbo);184uint64_t amdgpu_ttm_domain_start(struct amdgpu_device *adev, uint32_t type);185 186#if IS_ENABLED(CONFIG_DRM_AMDGPU_USERPTR)187int amdgpu_ttm_tt_get_user_pages(struct amdgpu_bo *bo, struct page **pages,188 struct hmm_range **range);189void amdgpu_ttm_tt_discard_user_pages(struct ttm_tt *ttm,190 struct hmm_range *range);191bool amdgpu_ttm_tt_get_user_pages_done(struct ttm_tt *ttm,192 struct hmm_range *range);193#else194static inline int amdgpu_ttm_tt_get_user_pages(struct amdgpu_bo *bo,195 struct page **pages,196 struct hmm_range **range)197{198 return -EPERM;199}200static inline void amdgpu_ttm_tt_discard_user_pages(struct ttm_tt *ttm,201 struct hmm_range *range)202{203}204static inline bool amdgpu_ttm_tt_get_user_pages_done(struct ttm_tt *ttm,205 struct hmm_range *range)206{207 return false;208}209#endif210 211void amdgpu_ttm_tt_set_user_pages(struct ttm_tt *ttm, struct page **pages);212int amdgpu_ttm_tt_get_userptr(const struct ttm_buffer_object *tbo,213 uint64_t *user_addr);214int amdgpu_ttm_tt_set_userptr(struct ttm_buffer_object *bo,215 uint64_t addr, uint32_t flags);216bool amdgpu_ttm_tt_has_userptr(struct ttm_tt *ttm);217struct mm_struct *amdgpu_ttm_tt_get_usermm(struct ttm_tt *ttm);218bool amdgpu_ttm_tt_affect_userptr(struct ttm_tt *ttm, unsigned long start,219 unsigned long end, unsigned long *userptr);220bool amdgpu_ttm_tt_userptr_invalidated(struct ttm_tt *ttm,221 int *last_invalidated);222bool amdgpu_ttm_tt_is_userptr(struct ttm_tt *ttm);223bool amdgpu_ttm_tt_is_readonly(struct ttm_tt *ttm);224uint64_t amdgpu_ttm_tt_pde_flags(struct ttm_tt *ttm, struct ttm_resource *mem);225uint64_t amdgpu_ttm_tt_pte_flags(struct amdgpu_device *adev, struct ttm_tt *ttm,226 struct ttm_resource *mem);227int amdgpu_ttm_evict_resources(struct amdgpu_device *adev, int mem_type);228 229void amdgpu_ttm_debugfs_init(struct amdgpu_device *adev);230 231#endif232