513 lines · c
1/*2 * Copyright 2014 Advanced Micro Devices, Inc.3 *4 * Permission is hereby granted, free of charge, to any person obtaining a5 * copy of this software and associated documentation files (the "Software"),6 * to deal in the Software without restriction, including without limitation7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,8 * and/or sell copies of the Software, and to permit persons to whom the9 * Software is furnished to do so, subject to the following conditions:10 *11 * The above copyright notice and this permission notice shall be included in12 * all copies or substantial portions of the Software.13 *14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR20 * OTHER DEALINGS IN THE SOFTWARE.21 */22 23/* amdgpu_amdkfd.h defines the private interface between amdgpu and amdkfd. */24 25#ifndef AMDGPU_AMDKFD_H_INCLUDED26#define AMDGPU_AMDKFD_H_INCLUDED27 28#include <linux/list.h>29#include <linux/types.h>30#include <linux/mm.h>31#include <linux/kthread.h>32#include <linux/workqueue.h>33#include <linux/mmu_notifier.h>34#include <linux/memremap.h>35#include <kgd_kfd_interface.h>36#include <drm/drm_client.h>37#include "amdgpu_sync.h"38#include "amdgpu_vm.h"39#include "amdgpu_xcp.h"40 41extern uint64_t amdgpu_amdkfd_total_mem_size;42 43enum TLB_FLUSH_TYPE {44 TLB_FLUSH_LEGACY = 0,45 TLB_FLUSH_LIGHTWEIGHT,46 TLB_FLUSH_HEAVYWEIGHT47};48 49struct amdgpu_device;50struct amdgpu_reset_context;51 52enum kfd_mem_attachment_type {53 KFD_MEM_ATT_SHARED, /* Share kgd_mem->bo or another attachment's */54 KFD_MEM_ATT_USERPTR, /* SG bo to DMA map pages from a userptr bo */55 KFD_MEM_ATT_DMABUF, /* DMAbuf to DMA map TTM BOs */56 KFD_MEM_ATT_SG /* Tag to DMA map SG BOs */57};58 59struct kfd_mem_attachment {60 struct list_head list;61 enum kfd_mem_attachment_type type;62 bool is_mapped;63 struct amdgpu_bo_va *bo_va;64 struct amdgpu_device *adev;65 uint64_t va;66 uint64_t pte_flags;67};68 69struct kgd_mem {70 struct mutex lock;71 struct amdgpu_bo *bo;72 struct dma_buf *dmabuf;73 struct hmm_range *range;74 struct list_head attachments;75 /* protected by amdkfd_process_info.lock */76 struct list_head validate_list;77 uint32_t domain;78 unsigned int mapped_to_gpu_memory;79 uint64_t va;80 81 uint32_t alloc_flags;82 83 uint32_t invalid;84 struct amdkfd_process_info *process_info;85 86 struct amdgpu_sync sync;87 88 uint32_t gem_handle;89 bool aql_queue;90 bool is_imported;91};92 93/* KFD Memory Eviction */94struct amdgpu_amdkfd_fence {95 struct dma_fence base;96 struct mm_struct *mm;97 spinlock_t lock;98 char timeline_name[TASK_COMM_LEN];99 struct svm_range_bo *svm_bo;100};101 102struct amdgpu_kfd_dev {103 struct kfd_dev *dev;104 int64_t vram_used[MAX_XCP];105 uint64_t vram_used_aligned[MAX_XCP];106 bool init_complete;107 struct work_struct reset_work;108 109 /* HMM page migration MEMORY_DEVICE_PRIVATE mapping */110 struct dev_pagemap pgmap;111 112 /* Client for KFD BO GEM handle allocations */113 struct drm_client_dev client;114};115 116enum kgd_engine_type {117 KGD_ENGINE_PFP = 1,118 KGD_ENGINE_ME,119 KGD_ENGINE_CE,120 KGD_ENGINE_MEC1,121 KGD_ENGINE_MEC2,122 KGD_ENGINE_RLC,123 KGD_ENGINE_SDMA1,124 KGD_ENGINE_SDMA2,125 KGD_ENGINE_MAX126};127 128 129struct amdkfd_process_info {130 /* List head of all VMs that belong to a KFD process */131 struct list_head vm_list_head;132 /* List head for all KFD BOs that belong to a KFD process. */133 struct list_head kfd_bo_list;134 /* List of userptr BOs that are valid or invalid */135 struct list_head userptr_valid_list;136 struct list_head userptr_inval_list;137 /* Lock to protect kfd_bo_list */138 struct mutex lock;139 140 /* Number of VMs */141 unsigned int n_vms;142 /* Eviction Fence */143 struct amdgpu_amdkfd_fence *eviction_fence;144 145 /* MMU-notifier related fields */146 struct mutex notifier_lock;147 uint32_t evicted_bos;148 struct delayed_work restore_userptr_work;149 struct pid *pid;150 bool block_mmu_notifications;151};152 153int amdgpu_amdkfd_init(void);154void amdgpu_amdkfd_fini(void);155 156void amdgpu_amdkfd_suspend(struct amdgpu_device *adev, bool run_pm);157int amdgpu_amdkfd_resume(struct amdgpu_device *adev, bool run_pm);158void amdgpu_amdkfd_interrupt(struct amdgpu_device *adev,159 const void *ih_ring_entry);160void amdgpu_amdkfd_device_probe(struct amdgpu_device *adev);161void amdgpu_amdkfd_device_init(struct amdgpu_device *adev);162void amdgpu_amdkfd_device_fini_sw(struct amdgpu_device *adev);163int amdgpu_amdkfd_check_and_lock_kfd(struct amdgpu_device *adev);164void amdgpu_amdkfd_unlock_kfd(struct amdgpu_device *adev);165int amdgpu_amdkfd_submit_ib(struct amdgpu_device *adev,166 enum kgd_engine_type engine,167 uint32_t vmid, uint64_t gpu_addr,168 uint32_t *ib_cmd, uint32_t ib_len);169void amdgpu_amdkfd_set_compute_idle(struct amdgpu_device *adev, bool idle);170bool amdgpu_amdkfd_have_atomics_support(struct amdgpu_device *adev);171 172bool amdgpu_amdkfd_is_kfd_vmid(struct amdgpu_device *adev, u32 vmid);173 174int amdgpu_amdkfd_pre_reset(struct amdgpu_device *adev,175 struct amdgpu_reset_context *reset_context);176 177int amdgpu_amdkfd_post_reset(struct amdgpu_device *adev);178 179void amdgpu_amdkfd_gpu_reset(struct amdgpu_device *adev);180 181int amdgpu_queue_mask_bit_to_set_resource_bit(struct amdgpu_device *adev,182 int queue_bit);183 184struct amdgpu_amdkfd_fence *amdgpu_amdkfd_fence_create(u64 context,185 struct mm_struct *mm,186 struct svm_range_bo *svm_bo);187 188int amdgpu_amdkfd_drm_client_create(struct amdgpu_device *adev);189#if defined(CONFIG_DEBUG_FS)190int kfd_debugfs_kfd_mem_limits(struct seq_file *m, void *data);191#endif192#if IS_ENABLED(CONFIG_HSA_AMD)193bool amdkfd_fence_check_mm(struct dma_fence *f, struct mm_struct *mm);194struct amdgpu_amdkfd_fence *to_amdgpu_amdkfd_fence(struct dma_fence *f);195int amdgpu_amdkfd_remove_fence_on_pt_pd_bos(struct amdgpu_bo *bo);196int amdgpu_amdkfd_evict_userptr(struct mmu_interval_notifier *mni,197 unsigned long cur_seq, struct kgd_mem *mem);198int amdgpu_amdkfd_bo_validate_and_fence(struct amdgpu_bo *bo,199 uint32_t domain,200 struct dma_fence *fence);201#else202static inline203bool amdkfd_fence_check_mm(struct dma_fence *f, struct mm_struct *mm)204{205 return false;206}207 208static inline209struct amdgpu_amdkfd_fence *to_amdgpu_amdkfd_fence(struct dma_fence *f)210{211 return NULL;212}213 214static inline215int amdgpu_amdkfd_remove_fence_on_pt_pd_bos(struct amdgpu_bo *bo)216{217 return 0;218}219 220static inline221int amdgpu_amdkfd_evict_userptr(struct mmu_interval_notifier *mni,222 unsigned long cur_seq, struct kgd_mem *mem)223{224 return 0;225}226static inline227int amdgpu_amdkfd_bo_validate_and_fence(struct amdgpu_bo *bo,228 uint32_t domain,229 struct dma_fence *fence)230{231 return 0;232}233#endif234/* Shared API */235int amdgpu_amdkfd_alloc_gtt_mem(struct amdgpu_device *adev, size_t size,236 void **mem_obj, uint64_t *gpu_addr,237 void **cpu_ptr, bool mqd_gfx9);238void amdgpu_amdkfd_free_gtt_mem(struct amdgpu_device *adev, void **mem_obj);239int amdgpu_amdkfd_alloc_gws(struct amdgpu_device *adev, size_t size,240 void **mem_obj);241void amdgpu_amdkfd_free_gws(struct amdgpu_device *adev, void *mem_obj);242int amdgpu_amdkfd_add_gws_to_process(void *info, void *gws, struct kgd_mem **mem);243int amdgpu_amdkfd_remove_gws_from_process(void *info, void *mem);244uint32_t amdgpu_amdkfd_get_fw_version(struct amdgpu_device *adev,245 enum kgd_engine_type type);246void amdgpu_amdkfd_get_local_mem_info(struct amdgpu_device *adev,247 struct kfd_local_mem_info *mem_info,248 struct amdgpu_xcp *xcp);249uint64_t amdgpu_amdkfd_get_gpu_clock_counter(struct amdgpu_device *adev);250 251uint32_t amdgpu_amdkfd_get_max_engine_clock_in_mhz(struct amdgpu_device *adev);252int amdgpu_amdkfd_get_dmabuf_info(struct amdgpu_device *adev, int dma_buf_fd,253 struct amdgpu_device **dmabuf_adev,254 uint64_t *bo_size, void *metadata_buffer,255 size_t buffer_size, uint32_t *metadata_size,256 uint32_t *flags, int8_t *xcp_id);257uint8_t amdgpu_amdkfd_get_xgmi_hops_count(struct amdgpu_device *dst,258 struct amdgpu_device *src);259int amdgpu_amdkfd_get_xgmi_bandwidth_mbytes(struct amdgpu_device *dst,260 struct amdgpu_device *src,261 bool is_min);262int amdgpu_amdkfd_get_pcie_bandwidth_mbytes(struct amdgpu_device *adev, bool is_min);263int amdgpu_amdkfd_send_close_event_drain_irq(struct amdgpu_device *adev,264 uint32_t *payload);265int amdgpu_amdkfd_unmap_hiq(struct amdgpu_device *adev, u32 doorbell_off,266 u32 inst);267int amdgpu_amdkfd_start_sched(struct amdgpu_device *adev, uint32_t node_id);268int amdgpu_amdkfd_stop_sched(struct amdgpu_device *adev, uint32_t node_id);269 270/* Read user wptr from a specified user address space with page fault271 * disabled. The memory must be pinned and mapped to the hardware when272 * this is called in hqd_load functions, so it should never fault in273 * the first place. This resolves a circular lock dependency involving274 * four locks, including the DQM lock and mmap_lock.275 */276#define read_user_wptr(mmptr, wptr, dst) \277 ({ \278 bool valid = false; \279 if ((mmptr) && (wptr)) { \280 pagefault_disable(); \281 if ((mmptr) == current->mm) { \282 valid = !get_user((dst), (wptr)); \283 } else if (current->flags & PF_KTHREAD) { \284 kthread_use_mm(mmptr); \285 valid = !get_user((dst), (wptr)); \286 kthread_unuse_mm(mmptr); \287 } \288 pagefault_enable(); \289 } \290 valid; \291 })292 293/* GPUVM API */294#define drm_priv_to_vm(drm_priv) \295 (&((struct amdgpu_fpriv *) \296 ((struct drm_file *)(drm_priv))->driver_priv)->vm)297 298int amdgpu_amdkfd_gpuvm_set_vm_pasid(struct amdgpu_device *adev,299 struct amdgpu_vm *avm, u32 pasid);300int amdgpu_amdkfd_gpuvm_acquire_process_vm(struct amdgpu_device *adev,301 struct amdgpu_vm *avm,302 void **process_info,303 struct dma_fence **ef);304void amdgpu_amdkfd_gpuvm_release_process_vm(struct amdgpu_device *adev,305 void *drm_priv);306uint64_t amdgpu_amdkfd_gpuvm_get_process_page_dir(void *drm_priv);307size_t amdgpu_amdkfd_get_available_memory(struct amdgpu_device *adev,308 uint8_t xcp_id);309int amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu(310 struct amdgpu_device *adev, uint64_t va, uint64_t size,311 void *drm_priv, struct kgd_mem **mem,312 uint64_t *offset, uint32_t flags, bool criu_resume);313int amdgpu_amdkfd_gpuvm_free_memory_of_gpu(314 struct amdgpu_device *adev, struct kgd_mem *mem, void *drm_priv,315 uint64_t *size);316int amdgpu_amdkfd_gpuvm_map_memory_to_gpu(struct amdgpu_device *adev,317 struct kgd_mem *mem, void *drm_priv);318int amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu(319 struct amdgpu_device *adev, struct kgd_mem *mem, void *drm_priv);320int amdgpu_amdkfd_gpuvm_dmaunmap_mem(struct kgd_mem *mem, void *drm_priv);321int amdgpu_amdkfd_gpuvm_sync_memory(322 struct amdgpu_device *adev, struct kgd_mem *mem, bool intr);323int amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel(struct kgd_mem *mem,324 void **kptr, uint64_t *size);325void amdgpu_amdkfd_gpuvm_unmap_gtt_bo_from_kernel(struct kgd_mem *mem);326 327int amdgpu_amdkfd_map_gtt_bo_to_gart(struct amdgpu_bo *bo, struct amdgpu_bo **bo_gart);328 329int amdgpu_amdkfd_gpuvm_restore_process_bos(void *process_info,330 struct dma_fence __rcu **ef);331int amdgpu_amdkfd_gpuvm_get_vm_fault_info(struct amdgpu_device *adev,332 struct kfd_vm_fault_info *info);333int amdgpu_amdkfd_gpuvm_import_dmabuf_fd(struct amdgpu_device *adev, int fd,334 uint64_t va, void *drm_priv,335 struct kgd_mem **mem, uint64_t *size,336 uint64_t *mmap_offset);337int amdgpu_amdkfd_gpuvm_export_dmabuf(struct kgd_mem *mem,338 struct dma_buf **dmabuf);339void amdgpu_amdkfd_debug_mem_fence(struct amdgpu_device *adev);340int amdgpu_amdkfd_get_tile_config(struct amdgpu_device *adev,341 struct tile_config *config);342void amdgpu_amdkfd_ras_poison_consumption_handler(struct amdgpu_device *adev,343 enum amdgpu_ras_block block, uint32_t reset);344 345void amdgpu_amdkfd_ras_pasid_poison_consumption_handler(struct amdgpu_device *adev,346 enum amdgpu_ras_block block, uint16_t pasid,347 pasid_notify pasid_fn, void *data, uint32_t reset);348 349bool amdgpu_amdkfd_is_fed(struct amdgpu_device *adev);350bool amdgpu_amdkfd_bo_mapped_to_dev(void *drm_priv, struct kgd_mem *mem);351void amdgpu_amdkfd_block_mmu_notifications(void *p);352int amdgpu_amdkfd_criu_resume(void *p);353int amdgpu_amdkfd_reserve_mem_limit(struct amdgpu_device *adev,354 uint64_t size, u32 alloc_flag, int8_t xcp_id);355void amdgpu_amdkfd_unreserve_mem_limit(struct amdgpu_device *adev,356 uint64_t size, u32 alloc_flag, int8_t xcp_id);357 358u64 amdgpu_amdkfd_xcp_memory_size(struct amdgpu_device *adev, int xcp_id);359 360#define KFD_XCP_MEM_ID(adev, xcp_id) \361 ((adev)->xcp_mgr && (xcp_id) >= 0 ?\362 (adev)->xcp_mgr->xcp[(xcp_id)].mem_id : -1)363 364#define KFD_XCP_MEMORY_SIZE(adev, xcp_id) amdgpu_amdkfd_xcp_memory_size((adev), (xcp_id))365 366 367#if IS_ENABLED(CONFIG_HSA_AMD)368void amdgpu_amdkfd_gpuvm_init_mem_limits(void);369void amdgpu_amdkfd_gpuvm_destroy_cb(struct amdgpu_device *adev,370 struct amdgpu_vm *vm);371 372/**373 * @amdgpu_amdkfd_release_notify() - Notify KFD when GEM object is released374 *375 * Allows KFD to release its resources associated with the GEM object.376 */377void amdgpu_amdkfd_release_notify(struct amdgpu_bo *bo);378void amdgpu_amdkfd_reserve_system_mem(uint64_t size);379#else380static inline381void amdgpu_amdkfd_gpuvm_init_mem_limits(void)382{383}384 385static inline386void amdgpu_amdkfd_gpuvm_destroy_cb(struct amdgpu_device *adev,387 struct amdgpu_vm *vm)388{389}390 391static inline392void amdgpu_amdkfd_release_notify(struct amdgpu_bo *bo)393{394}395#endif396 397#if IS_ENABLED(CONFIG_HSA_AMD_SVM)398int kgd2kfd_init_zone_device(struct amdgpu_device *adev);399#else400static inline401int kgd2kfd_init_zone_device(struct amdgpu_device *adev)402{403 return 0;404}405#endif406 407/* KGD2KFD callbacks */408int kgd2kfd_quiesce_mm(struct mm_struct *mm, uint32_t trigger);409int kgd2kfd_resume_mm(struct mm_struct *mm);410int kgd2kfd_schedule_evict_and_restore_process(struct mm_struct *mm,411 struct dma_fence *fence);412#if IS_ENABLED(CONFIG_HSA_AMD)413int kgd2kfd_init(void);414void kgd2kfd_exit(void);415struct kfd_dev *kgd2kfd_probe(struct amdgpu_device *adev, bool vf);416bool kgd2kfd_device_init(struct kfd_dev *kfd,417 const struct kgd2kfd_shared_resources *gpu_resources);418void kgd2kfd_device_exit(struct kfd_dev *kfd);419void kgd2kfd_suspend(struct kfd_dev *kfd, bool run_pm);420int kgd2kfd_resume(struct kfd_dev *kfd, bool run_pm);421int kgd2kfd_pre_reset(struct kfd_dev *kfd,422 struct amdgpu_reset_context *reset_context);423int kgd2kfd_post_reset(struct kfd_dev *kfd);424void kgd2kfd_interrupt(struct kfd_dev *kfd, const void *ih_ring_entry);425void kgd2kfd_set_sram_ecc_flag(struct kfd_dev *kfd);426void kgd2kfd_smi_event_throttle(struct kfd_dev *kfd, uint64_t throttle_bitmask);427int kgd2kfd_check_and_lock_kfd(void);428void kgd2kfd_unlock_kfd(void);429int kgd2kfd_start_sched(struct kfd_dev *kfd, uint32_t node_id);430int kgd2kfd_stop_sched(struct kfd_dev *kfd, uint32_t node_id);431#else432static inline int kgd2kfd_init(void)433{434 return -ENOENT;435}436 437static inline void kgd2kfd_exit(void)438{439}440 441static inline442struct kfd_dev *kgd2kfd_probe(struct amdgpu_device *adev, bool vf)443{444 return NULL;445}446 447static inline448bool kgd2kfd_device_init(struct kfd_dev *kfd,449 const struct kgd2kfd_shared_resources *gpu_resources)450{451 return false;452}453 454static inline void kgd2kfd_device_exit(struct kfd_dev *kfd)455{456}457 458static inline void kgd2kfd_suspend(struct kfd_dev *kfd, bool run_pm)459{460}461 462static inline int kgd2kfd_resume(struct kfd_dev *kfd, bool run_pm)463{464 return 0;465}466 467static inline int kgd2kfd_pre_reset(struct kfd_dev *kfd,468 struct amdgpu_reset_context *reset_context)469{470 return 0;471}472 473static inline int kgd2kfd_post_reset(struct kfd_dev *kfd)474{475 return 0;476}477 478static inline479void kgd2kfd_interrupt(struct kfd_dev *kfd, const void *ih_ring_entry)480{481}482 483static inline484void kgd2kfd_set_sram_ecc_flag(struct kfd_dev *kfd)485{486}487 488static inline489void kgd2kfd_smi_event_throttle(struct kfd_dev *kfd, uint64_t throttle_bitmask)490{491}492 493static inline int kgd2kfd_check_and_lock_kfd(void)494{495 return 0;496}497 498static inline void kgd2kfd_unlock_kfd(void)499{500}501 502static inline int kgd2kfd_start_sched(struct kfd_dev *kfd, uint32_t node_id)503{504 return 0;505}506 507static inline int kgd2kfd_stop_sched(struct kfd_dev *kfd, uint32_t node_id)508{509 return 0;510}511#endif512#endif /* AMDGPU_AMDKFD_H_INCLUDED */513