3238 lines · c
1// SPDX-License-Identifier: MIT2/*3 * Copyright 2014-2018 Advanced Micro Devices, Inc.4 *5 * Permission is hereby granted, free of charge, to any person obtaining a6 * copy of this software and associated documentation files (the "Software"),7 * to deal in the Software without restriction, including without limitation8 * the rights to use, copy, modify, merge, publish, distribute, sublicense,9 * and/or sell copies of the Software, and to permit persons to whom the10 * Software is furnished to do so, subject to the following conditions:11 *12 * The above copyright notice and this permission notice shall be included in13 * all copies or substantial portions of the Software.14 *15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL18 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR19 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,20 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR21 * OTHER DEALINGS IN THE SOFTWARE.22 */23#include <linux/dma-buf.h>24#include <linux/list.h>25#include <linux/pagemap.h>26#include <linux/sched/mm.h>27#include <linux/sched/task.h>28#include <drm/ttm/ttm_tt.h>29 30#include <drm/drm_exec.h>31 32#include "amdgpu_object.h"33#include "amdgpu_gem.h"34#include "amdgpu_vm.h"35#include "amdgpu_hmm.h"36#include "amdgpu_amdkfd.h"37#include "amdgpu_dma_buf.h"38#include <uapi/linux/kfd_ioctl.h>39#include "amdgpu_xgmi.h"40#include "kfd_priv.h"41#include "kfd_smi_events.h"42 43/* Userptr restore delay, just long enough to allow consecutive VM44 * changes to accumulate45 */46#define AMDGPU_USERPTR_RESTORE_DELAY_MS 147#define AMDGPU_RESERVE_MEM_LIMIT (3UL << 29)48 49/*50 * Align VRAM availability to 2MB to avoid fragmentation caused by 4K allocations in the tail 2MB51 * BO chunk52 */53#define VRAM_AVAILABLITY_ALIGN (1 << 21)54 55/* Impose limit on how much memory KFD can use */56static struct {57 uint64_t max_system_mem_limit;58 uint64_t max_ttm_mem_limit;59 int64_t system_mem_used;60 int64_t ttm_mem_used;61 spinlock_t mem_limit_lock;62} kfd_mem_limit;63 64static const char * const domain_bit_to_string[] = {65 "CPU",66 "GTT",67 "VRAM",68 "GDS",69 "GWS",70 "OA"71};72 73#define domain_string(domain) domain_bit_to_string[ffs(domain)-1]74 75static void amdgpu_amdkfd_restore_userptr_worker(struct work_struct *work);76 77static bool kfd_mem_is_attached(struct amdgpu_vm *avm,78 struct kgd_mem *mem)79{80 struct kfd_mem_attachment *entry;81 82 list_for_each_entry(entry, &mem->attachments, list)83 if (entry->bo_va->base.vm == avm)84 return true;85 86 return false;87}88 89/**90 * reuse_dmamap() - Check whether adev can share the original91 * userptr BO92 *93 * If both adev and bo_adev are in direct mapping or94 * in the same iommu group, they can share the original BO.95 *96 * @adev: Device to which can or cannot share the original BO97 * @bo_adev: Device to which allocated BO belongs to98 *99 * Return: returns true if adev can share original userptr BO,100 * false otherwise.101 */102static bool reuse_dmamap(struct amdgpu_device *adev, struct amdgpu_device *bo_adev)103{104 return (adev->ram_is_direct_mapped && bo_adev->ram_is_direct_mapped) ||105 (adev->dev->iommu_group == bo_adev->dev->iommu_group);106}107 108/* Set memory usage limits. Current, limits are109 * System (TTM + userptr) memory - 15/16th System RAM110 * TTM memory - 3/8th System RAM111 */112void amdgpu_amdkfd_gpuvm_init_mem_limits(void)113{114 struct sysinfo si;115 uint64_t mem;116 117 if (kfd_mem_limit.max_system_mem_limit)118 return;119 120 si_meminfo(&si);121 mem = si.totalram - si.totalhigh;122 mem *= si.mem_unit;123 124 spin_lock_init(&kfd_mem_limit.mem_limit_lock);125 kfd_mem_limit.max_system_mem_limit = mem - (mem >> 6);126 if (kfd_mem_limit.max_system_mem_limit < 2 * AMDGPU_RESERVE_MEM_LIMIT)127 kfd_mem_limit.max_system_mem_limit >>= 1;128 else129 kfd_mem_limit.max_system_mem_limit -= AMDGPU_RESERVE_MEM_LIMIT;130 131 kfd_mem_limit.max_ttm_mem_limit = ttm_tt_pages_limit() << PAGE_SHIFT;132 pr_debug("Kernel memory limit %lluM, TTM limit %lluM\n",133 (kfd_mem_limit.max_system_mem_limit >> 20),134 (kfd_mem_limit.max_ttm_mem_limit >> 20));135}136 137void amdgpu_amdkfd_reserve_system_mem(uint64_t size)138{139 kfd_mem_limit.system_mem_used += size;140}141 142/* Estimate page table size needed to represent a given memory size143 *144 * With 4KB pages, we need one 8 byte PTE for each 4KB of memory145 * (factor 512, >> 9). With 2MB pages, we need one 8 byte PTE for 2MB146 * of memory (factor 256K, >> 18). ROCm user mode tries to optimize147 * for 2MB pages for TLB efficiency. However, small allocations and148 * fragmented system memory still need some 4KB pages. We choose a149 * compromise that should work in most cases without reserving too150 * much memory for page tables unnecessarily (factor 16K, >> 14).151 */152 153#define ESTIMATE_PT_SIZE(mem_size) max(((mem_size) >> 14), AMDGPU_VM_RESERVED_VRAM)154 155/**156 * amdgpu_amdkfd_reserve_mem_limit() - Decrease available memory by size157 * of buffer.158 *159 * @adev: Device to which allocated BO belongs to160 * @size: Size of buffer, in bytes, encapsulated by B0. This should be161 * equivalent to amdgpu_bo_size(BO)162 * @alloc_flag: Flag used in allocating a BO as noted above163 * @xcp_id: xcp_id is used to get xcp from xcp manager, one xcp is164 * managed as one compute node in driver for app165 *166 * Return:167 * returns -ENOMEM in case of error, ZERO otherwise168 */169int amdgpu_amdkfd_reserve_mem_limit(struct amdgpu_device *adev,170 uint64_t size, u32 alloc_flag, int8_t xcp_id)171{172 uint64_t reserved_for_pt =173 ESTIMATE_PT_SIZE(amdgpu_amdkfd_total_mem_size);174 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);175 uint64_t reserved_for_ras = (con ? con->reserved_pages_in_bytes : 0);176 size_t system_mem_needed, ttm_mem_needed, vram_needed;177 int ret = 0;178 uint64_t vram_size = 0;179 180 system_mem_needed = 0;181 ttm_mem_needed = 0;182 vram_needed = 0;183 if (alloc_flag & KFD_IOC_ALLOC_MEM_FLAGS_GTT) {184 system_mem_needed = size;185 ttm_mem_needed = size;186 } else if (alloc_flag & KFD_IOC_ALLOC_MEM_FLAGS_VRAM) {187 /*188 * Conservatively round up the allocation requirement to 2 MB189 * to avoid fragmentation caused by 4K allocations in the tail190 * 2M BO chunk.191 */192 vram_needed = size;193 /*194 * For GFX 9.4.3, get the VRAM size from XCP structs195 */196 if (WARN_ONCE(xcp_id < 0, "invalid XCP ID %d", xcp_id))197 return -EINVAL;198 199 vram_size = KFD_XCP_MEMORY_SIZE(adev, xcp_id);200 if (adev->flags & AMD_IS_APU) {201 system_mem_needed = size;202 ttm_mem_needed = size;203 }204 } else if (alloc_flag & KFD_IOC_ALLOC_MEM_FLAGS_USERPTR) {205 system_mem_needed = size;206 } else if (!(alloc_flag &207 (KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL |208 KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP))) {209 pr_err("%s: Invalid BO type %#x\n", __func__, alloc_flag);210 return -ENOMEM;211 }212 213 spin_lock(&kfd_mem_limit.mem_limit_lock);214 215 if (kfd_mem_limit.system_mem_used + system_mem_needed >216 kfd_mem_limit.max_system_mem_limit)217 pr_debug("Set no_system_mem_limit=1 if using shared memory\n");218 219 if ((kfd_mem_limit.system_mem_used + system_mem_needed >220 kfd_mem_limit.max_system_mem_limit && !no_system_mem_limit) ||221 (kfd_mem_limit.ttm_mem_used + ttm_mem_needed >222 kfd_mem_limit.max_ttm_mem_limit) ||223 (adev && xcp_id >= 0 && adev->kfd.vram_used[xcp_id] + vram_needed >224 vram_size - reserved_for_pt - reserved_for_ras - atomic64_read(&adev->vram_pin_size))) {225 ret = -ENOMEM;226 goto release;227 }228 229 /* Update memory accounting by decreasing available system230 * memory, TTM memory and GPU memory as computed above231 */232 WARN_ONCE(vram_needed && !adev,233 "adev reference can't be null when vram is used");234 if (adev && xcp_id >= 0) {235 adev->kfd.vram_used[xcp_id] += vram_needed;236 adev->kfd.vram_used_aligned[xcp_id] +=237 (adev->flags & AMD_IS_APU) ?238 vram_needed :239 ALIGN(vram_needed, VRAM_AVAILABLITY_ALIGN);240 }241 kfd_mem_limit.system_mem_used += system_mem_needed;242 kfd_mem_limit.ttm_mem_used += ttm_mem_needed;243 244release:245 spin_unlock(&kfd_mem_limit.mem_limit_lock);246 return ret;247}248 249void amdgpu_amdkfd_unreserve_mem_limit(struct amdgpu_device *adev,250 uint64_t size, u32 alloc_flag, int8_t xcp_id)251{252 spin_lock(&kfd_mem_limit.mem_limit_lock);253 254 if (alloc_flag & KFD_IOC_ALLOC_MEM_FLAGS_GTT) {255 kfd_mem_limit.system_mem_used -= size;256 kfd_mem_limit.ttm_mem_used -= size;257 } else if (alloc_flag & KFD_IOC_ALLOC_MEM_FLAGS_VRAM) {258 WARN_ONCE(!adev,259 "adev reference can't be null when alloc mem flags vram is set");260 if (WARN_ONCE(xcp_id < 0, "invalid XCP ID %d", xcp_id))261 goto release;262 263 if (adev) {264 adev->kfd.vram_used[xcp_id] -= size;265 if (adev->flags & AMD_IS_APU) {266 adev->kfd.vram_used_aligned[xcp_id] -= size;267 kfd_mem_limit.system_mem_used -= size;268 kfd_mem_limit.ttm_mem_used -= size;269 } else {270 adev->kfd.vram_used_aligned[xcp_id] -=271 ALIGN(size, VRAM_AVAILABLITY_ALIGN);272 }273 }274 } else if (alloc_flag & KFD_IOC_ALLOC_MEM_FLAGS_USERPTR) {275 kfd_mem_limit.system_mem_used -= size;276 } else if (!(alloc_flag &277 (KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL |278 KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP))) {279 pr_err("%s: Invalid BO type %#x\n", __func__, alloc_flag);280 goto release;281 }282 WARN_ONCE(adev && xcp_id >= 0 && adev->kfd.vram_used[xcp_id] < 0,283 "KFD VRAM memory accounting unbalanced for xcp: %d", xcp_id);284 WARN_ONCE(kfd_mem_limit.ttm_mem_used < 0,285 "KFD TTM memory accounting unbalanced");286 WARN_ONCE(kfd_mem_limit.system_mem_used < 0,287 "KFD system memory accounting unbalanced");288 289release:290 spin_unlock(&kfd_mem_limit.mem_limit_lock);291}292 293void amdgpu_amdkfd_release_notify(struct amdgpu_bo *bo)294{295 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);296 u32 alloc_flags = bo->kfd_bo->alloc_flags;297 u64 size = amdgpu_bo_size(bo);298 299 amdgpu_amdkfd_unreserve_mem_limit(adev, size, alloc_flags,300 bo->xcp_id);301 302 kfree(bo->kfd_bo);303}304 305/**306 * create_dmamap_sg_bo() - Creates a amdgpu_bo object to reflect information307 * about USERPTR or DOOREBELL or MMIO BO.308 *309 * @adev: Device for which dmamap BO is being created310 * @mem: BO of peer device that is being DMA mapped. Provides parameters311 * in building the dmamap BO312 * @bo_out: Output parameter updated with handle of dmamap BO313 */314static int315create_dmamap_sg_bo(struct amdgpu_device *adev,316 struct kgd_mem *mem, struct amdgpu_bo **bo_out)317{318 struct drm_gem_object *gem_obj;319 int ret;320 uint64_t flags = 0;321 322 ret = amdgpu_bo_reserve(mem->bo, false);323 if (ret)324 return ret;325 326 if (mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_USERPTR)327 flags |= mem->bo->flags & (AMDGPU_GEM_CREATE_COHERENT |328 AMDGPU_GEM_CREATE_UNCACHED);329 330 ret = amdgpu_gem_object_create(adev, mem->bo->tbo.base.size, 1,331 AMDGPU_GEM_DOMAIN_CPU, AMDGPU_GEM_CREATE_PREEMPTIBLE | flags,332 ttm_bo_type_sg, mem->bo->tbo.base.resv, &gem_obj, 0);333 334 amdgpu_bo_unreserve(mem->bo);335 336 if (ret) {337 pr_err("Error in creating DMA mappable SG BO on domain: %d\n", ret);338 return -EINVAL;339 }340 341 *bo_out = gem_to_amdgpu_bo(gem_obj);342 (*bo_out)->parent = amdgpu_bo_ref(mem->bo);343 return ret;344}345 346/* amdgpu_amdkfd_remove_eviction_fence - Removes eviction fence from BO's347 * reservation object.348 *349 * @bo: [IN] Remove eviction fence(s) from this BO350 * @ef: [IN] This eviction fence is removed if it351 * is present in the shared list.352 *353 * NOTE: Must be called with BO reserved i.e. bo->tbo.resv->lock held.354 */355static int amdgpu_amdkfd_remove_eviction_fence(struct amdgpu_bo *bo,356 struct amdgpu_amdkfd_fence *ef)357{358 struct dma_fence *replacement;359 360 if (!ef)361 return -EINVAL;362 363 /* TODO: Instead of block before we should use the fence of the page364 * table update and TLB flush here directly.365 */366 replacement = dma_fence_get_stub();367 dma_resv_replace_fences(bo->tbo.base.resv, ef->base.context,368 replacement, DMA_RESV_USAGE_BOOKKEEP);369 dma_fence_put(replacement);370 return 0;371}372 373int amdgpu_amdkfd_remove_fence_on_pt_pd_bos(struct amdgpu_bo *bo)374{375 struct amdgpu_bo *root = bo;376 struct amdgpu_vm_bo_base *vm_bo;377 struct amdgpu_vm *vm;378 struct amdkfd_process_info *info;379 struct amdgpu_amdkfd_fence *ef;380 int ret;381 382 /* we can always get vm_bo from root PD bo.*/383 while (root->parent)384 root = root->parent;385 386 vm_bo = root->vm_bo;387 if (!vm_bo)388 return 0;389 390 vm = vm_bo->vm;391 if (!vm)392 return 0;393 394 info = vm->process_info;395 if (!info || !info->eviction_fence)396 return 0;397 398 ef = container_of(dma_fence_get(&info->eviction_fence->base),399 struct amdgpu_amdkfd_fence, base);400 401 BUG_ON(!dma_resv_trylock(bo->tbo.base.resv));402 ret = amdgpu_amdkfd_remove_eviction_fence(bo, ef);403 dma_resv_unlock(bo->tbo.base.resv);404 405 dma_fence_put(&ef->base);406 return ret;407}408 409static int amdgpu_amdkfd_bo_validate(struct amdgpu_bo *bo, uint32_t domain,410 bool wait)411{412 struct ttm_operation_ctx ctx = { false, false };413 int ret;414 415 if (WARN(amdgpu_ttm_tt_get_usermm(bo->tbo.ttm),416 "Called with userptr BO"))417 return -EINVAL;418 419 /* bo has been pinned, not need validate it */420 if (bo->tbo.pin_count)421 return 0;422 423 amdgpu_bo_placement_from_domain(bo, domain);424 425 ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);426 if (ret)427 goto validate_fail;428 if (wait)429 amdgpu_bo_sync_wait(bo, AMDGPU_FENCE_OWNER_KFD, false);430 431validate_fail:432 return ret;433}434 435int amdgpu_amdkfd_bo_validate_and_fence(struct amdgpu_bo *bo,436 uint32_t domain,437 struct dma_fence *fence)438{439 int ret = amdgpu_bo_reserve(bo, false);440 441 if (ret)442 return ret;443 444 ret = amdgpu_amdkfd_bo_validate(bo, domain, true);445 if (ret)446 goto unreserve_out;447 448 ret = dma_resv_reserve_fences(bo->tbo.base.resv, 1);449 if (ret)450 goto unreserve_out;451 452 dma_resv_add_fence(bo->tbo.base.resv, fence,453 DMA_RESV_USAGE_BOOKKEEP);454 455unreserve_out:456 amdgpu_bo_unreserve(bo);457 458 return ret;459}460 461static int amdgpu_amdkfd_validate_vm_bo(void *_unused, struct amdgpu_bo *bo)462{463 return amdgpu_amdkfd_bo_validate(bo, bo->allowed_domains, false);464}465 466/* vm_validate_pt_pd_bos - Validate page table and directory BOs467 *468 * Page directories are not updated here because huge page handling469 * during page table updates can invalidate page directory entries470 * again. Page directories are only updated after updating page471 * tables.472 */473static int vm_validate_pt_pd_bos(struct amdgpu_vm *vm,474 struct ww_acquire_ctx *ticket)475{476 struct amdgpu_bo *pd = vm->root.bo;477 struct amdgpu_device *adev = amdgpu_ttm_adev(pd->tbo.bdev);478 int ret;479 480 ret = amdgpu_vm_validate(adev, vm, ticket,481 amdgpu_amdkfd_validate_vm_bo, NULL);482 if (ret) {483 pr_err("failed to validate PT BOs\n");484 return ret;485 }486 487 vm->pd_phys_addr = amdgpu_gmc_pd_addr(vm->root.bo);488 489 return 0;490}491 492static int vm_update_pds(struct amdgpu_vm *vm, struct amdgpu_sync *sync)493{494 struct amdgpu_bo *pd = vm->root.bo;495 struct amdgpu_device *adev = amdgpu_ttm_adev(pd->tbo.bdev);496 int ret;497 498 ret = amdgpu_vm_update_pdes(adev, vm, false);499 if (ret)500 return ret;501 502 return amdgpu_sync_fence(sync, vm->last_update);503}504 505static uint64_t get_pte_flags(struct amdgpu_device *adev, struct kgd_mem *mem)506{507 uint32_t mapping_flags = AMDGPU_VM_PAGE_READABLE |508 AMDGPU_VM_MTYPE_DEFAULT;509 510 if (mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE)511 mapping_flags |= AMDGPU_VM_PAGE_WRITEABLE;512 if (mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_EXECUTABLE)513 mapping_flags |= AMDGPU_VM_PAGE_EXECUTABLE;514 515 return amdgpu_gem_va_map_flags(adev, mapping_flags);516}517 518/**519 * create_sg_table() - Create an sg_table for a contiguous DMA addr range520 * @addr: The starting address to point to521 * @size: Size of memory area in bytes being pointed to522 *523 * Allocates an instance of sg_table and initializes it to point to memory524 * area specified by input parameters. The address used to build is assumed525 * to be DMA mapped, if needed.526 *527 * DOORBELL or MMIO BOs use only one scatterlist node in their sg_table528 * because they are physically contiguous.529 *530 * Return: Initialized instance of SG Table or NULL531 */532static struct sg_table *create_sg_table(uint64_t addr, uint32_t size)533{534 struct sg_table *sg = kmalloc(sizeof(*sg), GFP_KERNEL);535 536 if (!sg)537 return NULL;538 if (sg_alloc_table(sg, 1, GFP_KERNEL)) {539 kfree(sg);540 return NULL;541 }542 sg_dma_address(sg->sgl) = addr;543 sg->sgl->length = size;544#ifdef CONFIG_NEED_SG_DMA_LENGTH545 sg->sgl->dma_length = size;546#endif547 return sg;548}549 550static int551kfd_mem_dmamap_userptr(struct kgd_mem *mem,552 struct kfd_mem_attachment *attachment)553{554 enum dma_data_direction direction =555 mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE ?556 DMA_BIDIRECTIONAL : DMA_TO_DEVICE;557 struct ttm_operation_ctx ctx = {.interruptible = true};558 struct amdgpu_bo *bo = attachment->bo_va->base.bo;559 struct amdgpu_device *adev = attachment->adev;560 struct ttm_tt *src_ttm = mem->bo->tbo.ttm;561 struct ttm_tt *ttm = bo->tbo.ttm;562 int ret;563 564 if (WARN_ON(ttm->num_pages != src_ttm->num_pages))565 return -EINVAL;566 567 ttm->sg = kmalloc(sizeof(*ttm->sg), GFP_KERNEL);568 if (unlikely(!ttm->sg))569 return -ENOMEM;570 571 /* Same sequence as in amdgpu_ttm_tt_pin_userptr */572 ret = sg_alloc_table_from_pages(ttm->sg, src_ttm->pages,573 ttm->num_pages, 0,574 (u64)ttm->num_pages << PAGE_SHIFT,575 GFP_KERNEL);576 if (unlikely(ret))577 goto free_sg;578 579 ret = dma_map_sgtable(adev->dev, ttm->sg, direction, 0);580 if (unlikely(ret))581 goto release_sg;582 583 amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_GTT);584 ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);585 if (ret)586 goto unmap_sg;587 588 return 0;589 590unmap_sg:591 dma_unmap_sgtable(adev->dev, ttm->sg, direction, 0);592release_sg:593 pr_err("DMA map userptr failed: %d\n", ret);594 sg_free_table(ttm->sg);595free_sg:596 kfree(ttm->sg);597 ttm->sg = NULL;598 return ret;599}600 601static int602kfd_mem_dmamap_dmabuf(struct kfd_mem_attachment *attachment)603{604 struct ttm_operation_ctx ctx = {.interruptible = true};605 struct amdgpu_bo *bo = attachment->bo_va->base.bo;606 int ret;607 608 amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_CPU);609 ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);610 if (ret)611 return ret;612 613 amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_GTT);614 return ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);615}616 617/**618 * kfd_mem_dmamap_sg_bo() - Create DMA mapped sg_table to access DOORBELL or MMIO BO619 * @mem: SG BO of the DOORBELL or MMIO resource on the owning device620 * @attachment: Virtual address attachment of the BO on accessing device621 *622 * An access request from the device that owns DOORBELL does not require DMA mapping.623 * This is because the request doesn't go through PCIe root complex i.e. it instead624 * loops back. The need to DMA map arises only when accessing peer device's DOORBELL625 *626 * In contrast, all access requests for MMIO need to be DMA mapped without regard to627 * device ownership. This is because access requests for MMIO go through PCIe root628 * complex.629 *630 * This is accomplished in two steps:631 * - Obtain DMA mapped address of DOORBELL or MMIO memory that could be used632 * in updating requesting device's page table633 * - Signal TTM to mark memory pointed to by requesting device's BO as GPU634 * accessible. This allows an update of requesting device's page table635 * with entries associated with DOOREBELL or MMIO memory636 *637 * This method is invoked in the following contexts:638 * - Mapping of DOORBELL or MMIO BO of same or peer device639 * - Validating an evicted DOOREBELL or MMIO BO on device seeking access640 *641 * Return: ZERO if successful, NON-ZERO otherwise642 */643static int644kfd_mem_dmamap_sg_bo(struct kgd_mem *mem,645 struct kfd_mem_attachment *attachment)646{647 struct ttm_operation_ctx ctx = {.interruptible = true};648 struct amdgpu_bo *bo = attachment->bo_va->base.bo;649 struct amdgpu_device *adev = attachment->adev;650 struct ttm_tt *ttm = bo->tbo.ttm;651 enum dma_data_direction dir;652 dma_addr_t dma_addr;653 bool mmio;654 int ret;655 656 /* Expect SG Table of dmapmap BO to be NULL */657 mmio = (mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP);658 if (unlikely(ttm->sg)) {659 pr_err("SG Table of %d BO for peer device is UNEXPECTEDLY NON-NULL", mmio);660 return -EINVAL;661 }662 663 dir = mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE ?664 DMA_BIDIRECTIONAL : DMA_TO_DEVICE;665 dma_addr = mem->bo->tbo.sg->sgl->dma_address;666 pr_debug("%d BO size: %d\n", mmio, mem->bo->tbo.sg->sgl->length);667 pr_debug("%d BO address before DMA mapping: %llx\n", mmio, dma_addr);668 dma_addr = dma_map_resource(adev->dev, dma_addr,669 mem->bo->tbo.sg->sgl->length, dir, DMA_ATTR_SKIP_CPU_SYNC);670 ret = dma_mapping_error(adev->dev, dma_addr);671 if (unlikely(ret))672 return ret;673 pr_debug("%d BO address after DMA mapping: %llx\n", mmio, dma_addr);674 675 ttm->sg = create_sg_table(dma_addr, mem->bo->tbo.sg->sgl->length);676 if (unlikely(!ttm->sg)) {677 ret = -ENOMEM;678 goto unmap_sg;679 }680 681 amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_GTT);682 ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);683 if (unlikely(ret))684 goto free_sg;685 686 return ret;687 688free_sg:689 sg_free_table(ttm->sg);690 kfree(ttm->sg);691 ttm->sg = NULL;692unmap_sg:693 dma_unmap_resource(adev->dev, dma_addr, mem->bo->tbo.sg->sgl->length,694 dir, DMA_ATTR_SKIP_CPU_SYNC);695 return ret;696}697 698static int699kfd_mem_dmamap_attachment(struct kgd_mem *mem,700 struct kfd_mem_attachment *attachment)701{702 switch (attachment->type) {703 case KFD_MEM_ATT_SHARED:704 return 0;705 case KFD_MEM_ATT_USERPTR:706 return kfd_mem_dmamap_userptr(mem, attachment);707 case KFD_MEM_ATT_DMABUF:708 return kfd_mem_dmamap_dmabuf(attachment);709 case KFD_MEM_ATT_SG:710 return kfd_mem_dmamap_sg_bo(mem, attachment);711 default:712 WARN_ON_ONCE(1);713 }714 return -EINVAL;715}716 717static void718kfd_mem_dmaunmap_userptr(struct kgd_mem *mem,719 struct kfd_mem_attachment *attachment)720{721 enum dma_data_direction direction =722 mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE ?723 DMA_BIDIRECTIONAL : DMA_TO_DEVICE;724 struct ttm_operation_ctx ctx = {.interruptible = false};725 struct amdgpu_bo *bo = attachment->bo_va->base.bo;726 struct amdgpu_device *adev = attachment->adev;727 struct ttm_tt *ttm = bo->tbo.ttm;728 729 if (unlikely(!ttm->sg))730 return;731 732 amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_CPU);733 ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);734 735 dma_unmap_sgtable(adev->dev, ttm->sg, direction, 0);736 sg_free_table(ttm->sg);737 kfree(ttm->sg);738 ttm->sg = NULL;739}740 741static void742kfd_mem_dmaunmap_dmabuf(struct kfd_mem_attachment *attachment)743{744 /* This is a no-op. We don't want to trigger eviction fences when745 * unmapping DMABufs. Therefore the invalidation (moving to system746 * domain) is done in kfd_mem_dmamap_dmabuf.747 */748}749 750/**751 * kfd_mem_dmaunmap_sg_bo() - Free DMA mapped sg_table of DOORBELL or MMIO BO752 * @mem: SG BO of the DOORBELL or MMIO resource on the owning device753 * @attachment: Virtual address attachment of the BO on accessing device754 *755 * The method performs following steps:756 * - Signal TTM to mark memory pointed to by BO as GPU inaccessible757 * - Free SG Table that is used to encapsulate DMA mapped memory of758 * peer device's DOORBELL or MMIO memory759 *760 * This method is invoked in the following contexts:761 * UNMapping of DOORBELL or MMIO BO on a device having access to its memory762 * Eviction of DOOREBELL or MMIO BO on device having access to its memory763 *764 * Return: void765 */766static void767kfd_mem_dmaunmap_sg_bo(struct kgd_mem *mem,768 struct kfd_mem_attachment *attachment)769{770 struct ttm_operation_ctx ctx = {.interruptible = true};771 struct amdgpu_bo *bo = attachment->bo_va->base.bo;772 struct amdgpu_device *adev = attachment->adev;773 struct ttm_tt *ttm = bo->tbo.ttm;774 enum dma_data_direction dir;775 776 if (unlikely(!ttm->sg)) {777 pr_debug("SG Table of BO is NULL");778 return;779 }780 781 amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_CPU);782 ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);783 784 dir = mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE ?785 DMA_BIDIRECTIONAL : DMA_TO_DEVICE;786 dma_unmap_resource(adev->dev, ttm->sg->sgl->dma_address,787 ttm->sg->sgl->length, dir, DMA_ATTR_SKIP_CPU_SYNC);788 sg_free_table(ttm->sg);789 kfree(ttm->sg);790 ttm->sg = NULL;791 bo->tbo.sg = NULL;792}793 794static void795kfd_mem_dmaunmap_attachment(struct kgd_mem *mem,796 struct kfd_mem_attachment *attachment)797{798 switch (attachment->type) {799 case KFD_MEM_ATT_SHARED:800 break;801 case KFD_MEM_ATT_USERPTR:802 kfd_mem_dmaunmap_userptr(mem, attachment);803 break;804 case KFD_MEM_ATT_DMABUF:805 kfd_mem_dmaunmap_dmabuf(attachment);806 break;807 case KFD_MEM_ATT_SG:808 kfd_mem_dmaunmap_sg_bo(mem, attachment);809 break;810 default:811 WARN_ON_ONCE(1);812 }813}814 815static int kfd_mem_export_dmabuf(struct kgd_mem *mem)816{817 if (!mem->dmabuf) {818 struct amdgpu_device *bo_adev;819 struct dma_buf *dmabuf;820 821 bo_adev = amdgpu_ttm_adev(mem->bo->tbo.bdev);822 dmabuf = drm_gem_prime_handle_to_dmabuf(&bo_adev->ddev, bo_adev->kfd.client.file,823 mem->gem_handle,824 mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE ?825 DRM_RDWR : 0);826 if (IS_ERR(dmabuf))827 return PTR_ERR(dmabuf);828 mem->dmabuf = dmabuf;829 }830 831 return 0;832}833 834static int835kfd_mem_attach_dmabuf(struct amdgpu_device *adev, struct kgd_mem *mem,836 struct amdgpu_bo **bo)837{838 struct drm_gem_object *gobj;839 int ret;840 841 ret = kfd_mem_export_dmabuf(mem);842 if (ret)843 return ret;844 845 gobj = amdgpu_gem_prime_import(adev_to_drm(adev), mem->dmabuf);846 if (IS_ERR(gobj))847 return PTR_ERR(gobj);848 849 *bo = gem_to_amdgpu_bo(gobj);850 (*bo)->flags |= AMDGPU_GEM_CREATE_PREEMPTIBLE;851 852 return 0;853}854 855/* kfd_mem_attach - Add a BO to a VM856 *857 * Everything that needs to bo done only once when a BO is first added858 * to a VM. It can later be mapped and unmapped many times without859 * repeating these steps.860 *861 * 0. Create BO for DMA mapping, if needed862 * 1. Allocate and initialize BO VA entry data structure863 * 2. Add BO to the VM864 * 3. Determine ASIC-specific PTE flags865 * 4. Alloc page tables and directories if needed866 * 4a. Validate new page tables and directories867 */868static int kfd_mem_attach(struct amdgpu_device *adev, struct kgd_mem *mem,869 struct amdgpu_vm *vm, bool is_aql)870{871 struct amdgpu_device *bo_adev = amdgpu_ttm_adev(mem->bo->tbo.bdev);872 unsigned long bo_size = mem->bo->tbo.base.size;873 uint64_t va = mem->va;874 struct kfd_mem_attachment *attachment[2] = {NULL, NULL};875 struct amdgpu_bo *bo[2] = {NULL, NULL};876 struct amdgpu_bo_va *bo_va;877 bool same_hive = false;878 int i, ret;879 880 if (!va) {881 pr_err("Invalid VA when adding BO to VM\n");882 return -EINVAL;883 }884 885 /* Determine access to VRAM, MMIO and DOORBELL BOs of peer devices886 *887 * The access path of MMIO and DOORBELL BOs of is always over PCIe.888 * In contrast the access path of VRAM BOs depens upon the type of889 * link that connects the peer device. Access over PCIe is allowed890 * if peer device has large BAR. In contrast, access over xGMI is891 * allowed for both small and large BAR configurations of peer device892 */893 if ((adev != bo_adev && !(adev->flags & AMD_IS_APU)) &&894 ((mem->domain == AMDGPU_GEM_DOMAIN_VRAM) ||895 (mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL) ||896 (mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP))) {897 if (mem->domain == AMDGPU_GEM_DOMAIN_VRAM)898 same_hive = amdgpu_xgmi_same_hive(adev, bo_adev);899 if (!same_hive && !amdgpu_device_is_peer_accessible(bo_adev, adev))900 return -EINVAL;901 }902 903 for (i = 0; i <= is_aql; i++) {904 attachment[i] = kzalloc(sizeof(*attachment[i]), GFP_KERNEL);905 if (unlikely(!attachment[i])) {906 ret = -ENOMEM;907 goto unwind;908 }909 910 pr_debug("\t add VA 0x%llx - 0x%llx to vm %p\n", va,911 va + bo_size, vm);912 913 if ((adev == bo_adev && !(mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP)) ||914 (amdgpu_ttm_tt_get_usermm(mem->bo->tbo.ttm) && reuse_dmamap(adev, bo_adev)) ||915 (mem->domain == AMDGPU_GEM_DOMAIN_GTT && reuse_dmamap(adev, bo_adev)) ||916 same_hive) {917 /* Mappings on the local GPU, or VRAM mappings in the918 * local hive, or userptr, or GTT mapping can reuse dma map919 * address space share the original BO920 */921 attachment[i]->type = KFD_MEM_ATT_SHARED;922 bo[i] = mem->bo;923 drm_gem_object_get(&bo[i]->tbo.base);924 } else if (i > 0) {925 /* Multiple mappings on the same GPU share the BO */926 attachment[i]->type = KFD_MEM_ATT_SHARED;927 bo[i] = bo[0];928 drm_gem_object_get(&bo[i]->tbo.base);929 } else if (amdgpu_ttm_tt_get_usermm(mem->bo->tbo.ttm)) {930 /* Create an SG BO to DMA-map userptrs on other GPUs */931 attachment[i]->type = KFD_MEM_ATT_USERPTR;932 ret = create_dmamap_sg_bo(adev, mem, &bo[i]);933 if (ret)934 goto unwind;935 /* Handle DOORBELL BOs of peer devices and MMIO BOs of local and peer devices */936 } else if (mem->bo->tbo.type == ttm_bo_type_sg) {937 WARN_ONCE(!(mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL ||938 mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP),939 "Handing invalid SG BO in ATTACH request");940 attachment[i]->type = KFD_MEM_ATT_SG;941 ret = create_dmamap_sg_bo(adev, mem, &bo[i]);942 if (ret)943 goto unwind;944 /* Enable acces to GTT and VRAM BOs of peer devices */945 } else if (mem->domain == AMDGPU_GEM_DOMAIN_GTT ||946 mem->domain == AMDGPU_GEM_DOMAIN_VRAM) {947 attachment[i]->type = KFD_MEM_ATT_DMABUF;948 ret = kfd_mem_attach_dmabuf(adev, mem, &bo[i]);949 if (ret)950 goto unwind;951 pr_debug("Employ DMABUF mechanism to enable peer GPU access\n");952 } else {953 WARN_ONCE(true, "Handling invalid ATTACH request");954 ret = -EINVAL;955 goto unwind;956 }957 958 /* Add BO to VM internal data structures */959 ret = amdgpu_bo_reserve(bo[i], false);960 if (ret) {961 pr_debug("Unable to reserve BO during memory attach");962 goto unwind;963 }964 bo_va = amdgpu_vm_bo_find(vm, bo[i]);965 if (!bo_va)966 bo_va = amdgpu_vm_bo_add(adev, vm, bo[i]);967 else968 ++bo_va->ref_count;969 attachment[i]->bo_va = bo_va;970 amdgpu_bo_unreserve(bo[i]);971 if (unlikely(!attachment[i]->bo_va)) {972 ret = -ENOMEM;973 pr_err("Failed to add BO object to VM. ret == %d\n",974 ret);975 goto unwind;976 }977 attachment[i]->va = va;978 attachment[i]->pte_flags = get_pte_flags(adev, mem);979 attachment[i]->adev = adev;980 list_add(&attachment[i]->list, &mem->attachments);981 982 va += bo_size;983 }984 985 return 0;986 987unwind:988 for (; i >= 0; i--) {989 if (!attachment[i])990 continue;991 if (attachment[i]->bo_va) {992 amdgpu_bo_reserve(bo[i], true);993 if (--attachment[i]->bo_va->ref_count == 0)994 amdgpu_vm_bo_del(adev, attachment[i]->bo_va);995 amdgpu_bo_unreserve(bo[i]);996 list_del(&attachment[i]->list);997 }998 if (bo[i])999 drm_gem_object_put(&bo[i]->tbo.base);1000 kfree(attachment[i]);1001 }1002 return ret;1003}1004 1005static void kfd_mem_detach(struct kfd_mem_attachment *attachment)1006{1007 struct amdgpu_bo *bo = attachment->bo_va->base.bo;1008 1009 pr_debug("\t remove VA 0x%llx in entry %p\n",1010 attachment->va, attachment);1011 if (--attachment->bo_va->ref_count == 0)1012 amdgpu_vm_bo_del(attachment->adev, attachment->bo_va);1013 drm_gem_object_put(&bo->tbo.base);1014 list_del(&attachment->list);1015 kfree(attachment);1016}1017 1018static void add_kgd_mem_to_kfd_bo_list(struct kgd_mem *mem,1019 struct amdkfd_process_info *process_info,1020 bool userptr)1021{1022 mutex_lock(&process_info->lock);1023 if (userptr)1024 list_add_tail(&mem->validate_list,1025 &process_info->userptr_valid_list);1026 else1027 list_add_tail(&mem->validate_list, &process_info->kfd_bo_list);1028 mutex_unlock(&process_info->lock);1029}1030 1031static void remove_kgd_mem_from_kfd_bo_list(struct kgd_mem *mem,1032 struct amdkfd_process_info *process_info)1033{1034 mutex_lock(&process_info->lock);1035 list_del(&mem->validate_list);1036 mutex_unlock(&process_info->lock);1037}1038 1039/* Initializes user pages. It registers the MMU notifier and validates1040 * the userptr BO in the GTT domain.1041 *1042 * The BO must already be on the userptr_valid_list. Otherwise an1043 * eviction and restore may happen that leaves the new BO unmapped1044 * with the user mode queues running.1045 *1046 * Takes the process_info->lock to protect against concurrent restore1047 * workers.1048 *1049 * Returns 0 for success, negative errno for errors.1050 */1051static int init_user_pages(struct kgd_mem *mem, uint64_t user_addr,1052 bool criu_resume)1053{1054 struct amdkfd_process_info *process_info = mem->process_info;1055 struct amdgpu_bo *bo = mem->bo;1056 struct ttm_operation_ctx ctx = { true, false };1057 struct hmm_range *range;1058 int ret = 0;1059 1060 mutex_lock(&process_info->lock);1061 1062 ret = amdgpu_ttm_tt_set_userptr(&bo->tbo, user_addr, 0);1063 if (ret) {1064 pr_err("%s: Failed to set userptr: %d\n", __func__, ret);1065 goto out;1066 }1067 1068 ret = amdgpu_hmm_register(bo, user_addr);1069 if (ret) {1070 pr_err("%s: Failed to register MMU notifier: %d\n",1071 __func__, ret);1072 goto out;1073 }1074 1075 if (criu_resume) {1076 /*1077 * During a CRIU restore operation, the userptr buffer objects1078 * will be validated in the restore_userptr_work worker at a1079 * later stage when it is scheduled by another ioctl called by1080 * CRIU master process for the target pid for restore.1081 */1082 mutex_lock(&process_info->notifier_lock);1083 mem->invalid++;1084 mutex_unlock(&process_info->notifier_lock);1085 mutex_unlock(&process_info->lock);1086 return 0;1087 }1088 1089 ret = amdgpu_ttm_tt_get_user_pages(bo, bo->tbo.ttm->pages, &range);1090 if (ret) {1091 if (ret == -EAGAIN)1092 pr_debug("Failed to get user pages, try again\n");1093 else1094 pr_err("%s: Failed to get user pages: %d\n", __func__, ret);1095 goto unregister_out;1096 }1097 1098 ret = amdgpu_bo_reserve(bo, true);1099 if (ret) {1100 pr_err("%s: Failed to reserve BO\n", __func__);1101 goto release_out;1102 }1103 amdgpu_bo_placement_from_domain(bo, mem->domain);1104 ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);1105 if (ret)1106 pr_err("%s: failed to validate BO\n", __func__);1107 amdgpu_bo_unreserve(bo);1108 1109release_out:1110 amdgpu_ttm_tt_get_user_pages_done(bo->tbo.ttm, range);1111unregister_out:1112 if (ret)1113 amdgpu_hmm_unregister(bo);1114out:1115 mutex_unlock(&process_info->lock);1116 return ret;1117}1118 1119/* Reserving a BO and its page table BOs must happen atomically to1120 * avoid deadlocks. Some operations update multiple VMs at once. Track1121 * all the reservation info in a context structure. Optionally a sync1122 * object can track VM updates.1123 */1124struct bo_vm_reservation_context {1125 /* DRM execution context for the reservation */1126 struct drm_exec exec;1127 /* Number of VMs reserved */1128 unsigned int n_vms;1129 /* Pointer to sync object */1130 struct amdgpu_sync *sync;1131};1132 1133enum bo_vm_match {1134 BO_VM_NOT_MAPPED = 0, /* Match VMs where a BO is not mapped */1135 BO_VM_MAPPED, /* Match VMs where a BO is mapped */1136 BO_VM_ALL, /* Match all VMs a BO was added to */1137};1138 1139/**1140 * reserve_bo_and_vm - reserve a BO and a VM unconditionally.1141 * @mem: KFD BO structure.1142 * @vm: the VM to reserve.1143 * @ctx: the struct that will be used in unreserve_bo_and_vms().1144 */1145static int reserve_bo_and_vm(struct kgd_mem *mem,1146 struct amdgpu_vm *vm,1147 struct bo_vm_reservation_context *ctx)1148{1149 struct amdgpu_bo *bo = mem->bo;1150 int ret;1151 1152 WARN_ON(!vm);1153 1154 ctx->n_vms = 1;1155 ctx->sync = &mem->sync;1156 drm_exec_init(&ctx->exec, DRM_EXEC_INTERRUPTIBLE_WAIT, 0);1157 drm_exec_until_all_locked(&ctx->exec) {1158 ret = amdgpu_vm_lock_pd(vm, &ctx->exec, 2);1159 drm_exec_retry_on_contention(&ctx->exec);1160 if (unlikely(ret))1161 goto error;1162 1163 ret = drm_exec_prepare_obj(&ctx->exec, &bo->tbo.base, 1);1164 drm_exec_retry_on_contention(&ctx->exec);1165 if (unlikely(ret))1166 goto error;1167 }1168 return 0;1169 1170error:1171 pr_err("Failed to reserve buffers in ttm.\n");1172 drm_exec_fini(&ctx->exec);1173 return ret;1174}1175 1176/**1177 * reserve_bo_and_cond_vms - reserve a BO and some VMs conditionally1178 * @mem: KFD BO structure.1179 * @vm: the VM to reserve. If NULL, then all VMs associated with the BO1180 * is used. Otherwise, a single VM associated with the BO.1181 * @map_type: the mapping status that will be used to filter the VMs.1182 * @ctx: the struct that will be used in unreserve_bo_and_vms().1183 *1184 * Returns 0 for success, negative for failure.1185 */1186static int reserve_bo_and_cond_vms(struct kgd_mem *mem,1187 struct amdgpu_vm *vm, enum bo_vm_match map_type,1188 struct bo_vm_reservation_context *ctx)1189{1190 struct kfd_mem_attachment *entry;1191 struct amdgpu_bo *bo = mem->bo;1192 int ret;1193 1194 ctx->sync = &mem->sync;1195 drm_exec_init(&ctx->exec, DRM_EXEC_INTERRUPTIBLE_WAIT |1196 DRM_EXEC_IGNORE_DUPLICATES, 0);1197 drm_exec_until_all_locked(&ctx->exec) {1198 ctx->n_vms = 0;1199 list_for_each_entry(entry, &mem->attachments, list) {1200 if ((vm && vm != entry->bo_va->base.vm) ||1201 (entry->is_mapped != map_type1202 && map_type != BO_VM_ALL))1203 continue;1204 1205 ret = amdgpu_vm_lock_pd(entry->bo_va->base.vm,1206 &ctx->exec, 2);1207 drm_exec_retry_on_contention(&ctx->exec);1208 if (unlikely(ret))1209 goto error;1210 ++ctx->n_vms;1211 }1212 1213 ret = drm_exec_prepare_obj(&ctx->exec, &bo->tbo.base, 1);1214 drm_exec_retry_on_contention(&ctx->exec);1215 if (unlikely(ret))1216 goto error;1217 }1218 return 0;1219 1220error:1221 pr_err("Failed to reserve buffers in ttm.\n");1222 drm_exec_fini(&ctx->exec);1223 return ret;1224}1225 1226/**1227 * unreserve_bo_and_vms - Unreserve BO and VMs from a reservation context1228 * @ctx: Reservation context to unreserve1229 * @wait: Optionally wait for a sync object representing pending VM updates1230 * @intr: Whether the wait is interruptible1231 *1232 * Also frees any resources allocated in1233 * reserve_bo_and_(cond_)vm(s). Returns the status from1234 * amdgpu_sync_wait.1235 */1236static int unreserve_bo_and_vms(struct bo_vm_reservation_context *ctx,1237 bool wait, bool intr)1238{1239 int ret = 0;1240 1241 if (wait)1242 ret = amdgpu_sync_wait(ctx->sync, intr);1243 1244 drm_exec_fini(&ctx->exec);1245 ctx->sync = NULL;1246 return ret;1247}1248 1249static int unmap_bo_from_gpuvm(struct kgd_mem *mem,1250 struct kfd_mem_attachment *entry,1251 struct amdgpu_sync *sync)1252{1253 struct amdgpu_bo_va *bo_va = entry->bo_va;1254 struct amdgpu_device *adev = entry->adev;1255 struct amdgpu_vm *vm = bo_va->base.vm;1256 1257 if (bo_va->queue_refcount) {1258 pr_debug("bo_va->queue_refcount %d\n", bo_va->queue_refcount);1259 return -EBUSY;1260 }1261 1262 amdgpu_vm_bo_unmap(adev, bo_va, entry->va);1263 1264 amdgpu_vm_clear_freed(adev, vm, &bo_va->last_pt_update);1265 1266 amdgpu_sync_fence(sync, bo_va->last_pt_update);1267 1268 return 0;1269}1270 1271static int update_gpuvm_pte(struct kgd_mem *mem,1272 struct kfd_mem_attachment *entry,1273 struct amdgpu_sync *sync)1274{1275 struct amdgpu_bo_va *bo_va = entry->bo_va;1276 struct amdgpu_device *adev = entry->adev;1277 int ret;1278 1279 ret = kfd_mem_dmamap_attachment(mem, entry);1280 if (ret)1281 return ret;1282 1283 /* Update the page tables */1284 ret = amdgpu_vm_bo_update(adev, bo_va, false);1285 if (ret) {1286 pr_err("amdgpu_vm_bo_update failed\n");1287 return ret;1288 }1289 1290 return amdgpu_sync_fence(sync, bo_va->last_pt_update);1291}1292 1293static int map_bo_to_gpuvm(struct kgd_mem *mem,1294 struct kfd_mem_attachment *entry,1295 struct amdgpu_sync *sync,1296 bool no_update_pte)1297{1298 int ret;1299 1300 /* Set virtual address for the allocation */1301 ret = amdgpu_vm_bo_map(entry->adev, entry->bo_va, entry->va, 0,1302 amdgpu_bo_size(entry->bo_va->base.bo),1303 entry->pte_flags);1304 if (ret) {1305 pr_err("Failed to map VA 0x%llx in vm. ret %d\n",1306 entry->va, ret);1307 return ret;1308 }1309 1310 if (no_update_pte)1311 return 0;1312 1313 ret = update_gpuvm_pte(mem, entry, sync);1314 if (ret) {1315 pr_err("update_gpuvm_pte() failed\n");1316 goto update_gpuvm_pte_failed;1317 }1318 1319 return 0;1320 1321update_gpuvm_pte_failed:1322 unmap_bo_from_gpuvm(mem, entry, sync);1323 kfd_mem_dmaunmap_attachment(mem, entry);1324 return ret;1325}1326 1327static int process_validate_vms(struct amdkfd_process_info *process_info,1328 struct ww_acquire_ctx *ticket)1329{1330 struct amdgpu_vm *peer_vm;1331 int ret;1332 1333 list_for_each_entry(peer_vm, &process_info->vm_list_head,1334 vm_list_node) {1335 ret = vm_validate_pt_pd_bos(peer_vm, ticket);1336 if (ret)1337 return ret;1338 }1339 1340 return 0;1341}1342 1343static int process_sync_pds_resv(struct amdkfd_process_info *process_info,1344 struct amdgpu_sync *sync)1345{1346 struct amdgpu_vm *peer_vm;1347 int ret;1348 1349 list_for_each_entry(peer_vm, &process_info->vm_list_head,1350 vm_list_node) {1351 struct amdgpu_bo *pd = peer_vm->root.bo;1352 1353 ret = amdgpu_sync_resv(NULL, sync, pd->tbo.base.resv,1354 AMDGPU_SYNC_NE_OWNER,1355 AMDGPU_FENCE_OWNER_KFD);1356 if (ret)1357 return ret;1358 }1359 1360 return 0;1361}1362 1363static int process_update_pds(struct amdkfd_process_info *process_info,1364 struct amdgpu_sync *sync)1365{1366 struct amdgpu_vm *peer_vm;1367 int ret;1368 1369 list_for_each_entry(peer_vm, &process_info->vm_list_head,1370 vm_list_node) {1371 ret = vm_update_pds(peer_vm, sync);1372 if (ret)1373 return ret;1374 }1375 1376 return 0;1377}1378 1379static int init_kfd_vm(struct amdgpu_vm *vm, void **process_info,1380 struct dma_fence **ef)1381{1382 struct amdkfd_process_info *info = NULL;1383 int ret;1384 1385 if (!*process_info) {1386 info = kzalloc(sizeof(*info), GFP_KERNEL);1387 if (!info)1388 return -ENOMEM;1389 1390 mutex_init(&info->lock);1391 mutex_init(&info->notifier_lock);1392 INIT_LIST_HEAD(&info->vm_list_head);1393 INIT_LIST_HEAD(&info->kfd_bo_list);1394 INIT_LIST_HEAD(&info->userptr_valid_list);1395 INIT_LIST_HEAD(&info->userptr_inval_list);1396 1397 info->eviction_fence =1398 amdgpu_amdkfd_fence_create(dma_fence_context_alloc(1),1399 current->mm,1400 NULL);1401 if (!info->eviction_fence) {1402 pr_err("Failed to create eviction fence\n");1403 ret = -ENOMEM;1404 goto create_evict_fence_fail;1405 }1406 1407 info->pid = get_task_pid(current->group_leader, PIDTYPE_PID);1408 INIT_DELAYED_WORK(&info->restore_userptr_work,1409 amdgpu_amdkfd_restore_userptr_worker);1410 1411 *process_info = info;1412 }1413 1414 vm->process_info = *process_info;1415 1416 /* Validate page directory and attach eviction fence */1417 ret = amdgpu_bo_reserve(vm->root.bo, true);1418 if (ret)1419 goto reserve_pd_fail;1420 ret = vm_validate_pt_pd_bos(vm, NULL);1421 if (ret) {1422 pr_err("validate_pt_pd_bos() failed\n");1423 goto validate_pd_fail;1424 }1425 ret = amdgpu_bo_sync_wait(vm->root.bo,1426 AMDGPU_FENCE_OWNER_KFD, false);1427 if (ret)1428 goto wait_pd_fail;1429 ret = dma_resv_reserve_fences(vm->root.bo->tbo.base.resv, 1);1430 if (ret)1431 goto reserve_shared_fail;1432 dma_resv_add_fence(vm->root.bo->tbo.base.resv,1433 &vm->process_info->eviction_fence->base,1434 DMA_RESV_USAGE_BOOKKEEP);1435 amdgpu_bo_unreserve(vm->root.bo);1436 1437 /* Update process info */1438 mutex_lock(&vm->process_info->lock);1439 list_add_tail(&vm->vm_list_node,1440 &(vm->process_info->vm_list_head));1441 vm->process_info->n_vms++;1442 if (ef)1443 *ef = dma_fence_get(&vm->process_info->eviction_fence->base);1444 mutex_unlock(&vm->process_info->lock);1445 1446 return 0;1447 1448reserve_shared_fail:1449wait_pd_fail:1450validate_pd_fail:1451 amdgpu_bo_unreserve(vm->root.bo);1452reserve_pd_fail:1453 vm->process_info = NULL;1454 if (info) {1455 dma_fence_put(&info->eviction_fence->base);1456 *process_info = NULL;1457 put_pid(info->pid);1458create_evict_fence_fail:1459 mutex_destroy(&info->lock);1460 mutex_destroy(&info->notifier_lock);1461 kfree(info);1462 }1463 return ret;1464}1465 1466/**1467 * amdgpu_amdkfd_gpuvm_pin_bo() - Pins a BO using following criteria1468 * @bo: Handle of buffer object being pinned1469 * @domain: Domain into which BO should be pinned1470 *1471 * - USERPTR BOs are UNPINNABLE and will return error1472 * - All other BO types (GTT, VRAM, MMIO and DOORBELL) will have their1473 * PIN count incremented. It is valid to PIN a BO multiple times1474 *1475 * Return: ZERO if successful in pinning, Non-Zero in case of error.1476 */1477static int amdgpu_amdkfd_gpuvm_pin_bo(struct amdgpu_bo *bo, u32 domain)1478{1479 int ret = 0;1480 1481 ret = amdgpu_bo_reserve(bo, false);1482 if (unlikely(ret))1483 return ret;1484 1485 if (bo->flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS) {1486 /*1487 * If bo is not contiguous on VRAM, move to system memory first to ensure1488 * we can get contiguous VRAM space after evicting other BOs.1489 */1490 if (!(bo->tbo.resource->placement & TTM_PL_FLAG_CONTIGUOUS)) {1491 struct ttm_operation_ctx ctx = { true, false };1492 1493 amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_GTT);1494 ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);1495 if (unlikely(ret)) {1496 pr_debug("validate bo 0x%p to GTT failed %d\n", &bo->tbo, ret);1497 goto out;1498 }1499 }1500 }1501 1502 ret = amdgpu_bo_pin(bo, domain);1503 if (ret)1504 pr_err("Error in Pinning BO to domain: %d\n", domain);1505 1506 amdgpu_bo_sync_wait(bo, AMDGPU_FENCE_OWNER_KFD, false);1507out:1508 amdgpu_bo_unreserve(bo);1509 return ret;1510}1511 1512/**1513 * amdgpu_amdkfd_gpuvm_unpin_bo() - Unpins BO using following criteria1514 * @bo: Handle of buffer object being unpinned1515 *1516 * - Is a illegal request for USERPTR BOs and is ignored1517 * - All other BO types (GTT, VRAM, MMIO and DOORBELL) will have their1518 * PIN count decremented. Calls to UNPIN must balance calls to PIN1519 */1520static void amdgpu_amdkfd_gpuvm_unpin_bo(struct amdgpu_bo *bo)1521{1522 int ret = 0;1523 1524 ret = amdgpu_bo_reserve(bo, false);1525 if (unlikely(ret))1526 return;1527 1528 amdgpu_bo_unpin(bo);1529 amdgpu_bo_unreserve(bo);1530}1531 1532int amdgpu_amdkfd_gpuvm_set_vm_pasid(struct amdgpu_device *adev,1533 struct amdgpu_vm *avm, u32 pasid)1534 1535{1536 int ret;1537 1538 /* Free the original amdgpu allocated pasid,1539 * will be replaced with kfd allocated pasid.1540 */1541 if (avm->pasid) {1542 amdgpu_pasid_free(avm->pasid);1543 amdgpu_vm_set_pasid(adev, avm, 0);1544 }1545 1546 ret = amdgpu_vm_set_pasid(adev, avm, pasid);1547 if (ret)1548 return ret;1549 1550 return 0;1551}1552 1553int amdgpu_amdkfd_gpuvm_acquire_process_vm(struct amdgpu_device *adev,1554 struct amdgpu_vm *avm,1555 void **process_info,1556 struct dma_fence **ef)1557{1558 int ret;1559 1560 /* Already a compute VM? */1561 if (avm->process_info)1562 return -EINVAL;1563 1564 /* Convert VM into a compute VM */1565 ret = amdgpu_vm_make_compute(adev, avm);1566 if (ret)1567 return ret;1568 1569 /* Initialize KFD part of the VM and process info */1570 ret = init_kfd_vm(avm, process_info, ef);1571 if (ret)1572 return ret;1573 1574 amdgpu_vm_set_task_info(avm);1575 1576 return 0;1577}1578 1579void amdgpu_amdkfd_gpuvm_destroy_cb(struct amdgpu_device *adev,1580 struct amdgpu_vm *vm)1581{1582 struct amdkfd_process_info *process_info = vm->process_info;1583 1584 if (!process_info)1585 return;1586 1587 /* Update process info */1588 mutex_lock(&process_info->lock);1589 process_info->n_vms--;1590 list_del(&vm->vm_list_node);1591 mutex_unlock(&process_info->lock);1592 1593 vm->process_info = NULL;1594 1595 /* Release per-process resources when last compute VM is destroyed */1596 if (!process_info->n_vms) {1597 WARN_ON(!list_empty(&process_info->kfd_bo_list));1598 WARN_ON(!list_empty(&process_info->userptr_valid_list));1599 WARN_ON(!list_empty(&process_info->userptr_inval_list));1600 1601 dma_fence_put(&process_info->eviction_fence->base);1602 cancel_delayed_work_sync(&process_info->restore_userptr_work);1603 put_pid(process_info->pid);1604 mutex_destroy(&process_info->lock);1605 mutex_destroy(&process_info->notifier_lock);1606 kfree(process_info);1607 }1608}1609 1610void amdgpu_amdkfd_gpuvm_release_process_vm(struct amdgpu_device *adev,1611 void *drm_priv)1612{1613 struct amdgpu_vm *avm;1614 1615 if (WARN_ON(!adev || !drm_priv))1616 return;1617 1618 avm = drm_priv_to_vm(drm_priv);1619 1620 pr_debug("Releasing process vm %p\n", avm);1621 1622 /* The original pasid of amdgpu vm has already been1623 * released during making a amdgpu vm to a compute vm1624 * The current pasid is managed by kfd and will be1625 * released on kfd process destroy. Set amdgpu pasid1626 * to 0 to avoid duplicate release.1627 */1628 amdgpu_vm_release_compute(adev, avm);1629}1630 1631uint64_t amdgpu_amdkfd_gpuvm_get_process_page_dir(void *drm_priv)1632{1633 struct amdgpu_vm *avm = drm_priv_to_vm(drm_priv);1634 struct amdgpu_bo *pd = avm->root.bo;1635 struct amdgpu_device *adev = amdgpu_ttm_adev(pd->tbo.bdev);1636 1637 if (adev->asic_type < CHIP_VEGA10)1638 return avm->pd_phys_addr >> AMDGPU_GPU_PAGE_SHIFT;1639 return avm->pd_phys_addr;1640}1641 1642void amdgpu_amdkfd_block_mmu_notifications(void *p)1643{1644 struct amdkfd_process_info *pinfo = (struct amdkfd_process_info *)p;1645 1646 mutex_lock(&pinfo->lock);1647 WRITE_ONCE(pinfo->block_mmu_notifications, true);1648 mutex_unlock(&pinfo->lock);1649}1650 1651int amdgpu_amdkfd_criu_resume(void *p)1652{1653 int ret = 0;1654 struct amdkfd_process_info *pinfo = (struct amdkfd_process_info *)p;1655 1656 mutex_lock(&pinfo->lock);1657 pr_debug("scheduling work\n");1658 mutex_lock(&pinfo->notifier_lock);1659 pinfo->evicted_bos++;1660 mutex_unlock(&pinfo->notifier_lock);1661 if (!READ_ONCE(pinfo->block_mmu_notifications)) {1662 ret = -EINVAL;1663 goto out_unlock;1664 }1665 WRITE_ONCE(pinfo->block_mmu_notifications, false);1666 queue_delayed_work(system_freezable_wq,1667 &pinfo->restore_userptr_work, 0);1668 1669out_unlock:1670 mutex_unlock(&pinfo->lock);1671 return ret;1672}1673 1674size_t amdgpu_amdkfd_get_available_memory(struct amdgpu_device *adev,1675 uint8_t xcp_id)1676{1677 uint64_t reserved_for_pt =1678 ESTIMATE_PT_SIZE(amdgpu_amdkfd_total_mem_size);1679 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);1680 uint64_t reserved_for_ras = (con ? con->reserved_pages_in_bytes : 0);1681 ssize_t available;1682 uint64_t vram_available, system_mem_available, ttm_mem_available;1683 1684 spin_lock(&kfd_mem_limit.mem_limit_lock);1685 vram_available = KFD_XCP_MEMORY_SIZE(adev, xcp_id)1686 - adev->kfd.vram_used_aligned[xcp_id]1687 - atomic64_read(&adev->vram_pin_size)1688 - reserved_for_pt1689 - reserved_for_ras;1690 1691 if (adev->flags & AMD_IS_APU) {1692 system_mem_available = no_system_mem_limit ?1693 kfd_mem_limit.max_system_mem_limit :1694 kfd_mem_limit.max_system_mem_limit -1695 kfd_mem_limit.system_mem_used;1696 1697 ttm_mem_available = kfd_mem_limit.max_ttm_mem_limit -1698 kfd_mem_limit.ttm_mem_used;1699 1700 available = min3(system_mem_available, ttm_mem_available,1701 vram_available);1702 available = ALIGN_DOWN(available, PAGE_SIZE);1703 } else {1704 available = ALIGN_DOWN(vram_available, VRAM_AVAILABLITY_ALIGN);1705 }1706 1707 spin_unlock(&kfd_mem_limit.mem_limit_lock);1708 1709 if (available < 0)1710 available = 0;1711 1712 return available;1713}1714 1715int amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu(1716 struct amdgpu_device *adev, uint64_t va, uint64_t size,1717 void *drm_priv, struct kgd_mem **mem,1718 uint64_t *offset, uint32_t flags, bool criu_resume)1719{1720 struct amdgpu_vm *avm = drm_priv_to_vm(drm_priv);1721 struct amdgpu_fpriv *fpriv = container_of(avm, struct amdgpu_fpriv, vm);1722 enum ttm_bo_type bo_type = ttm_bo_type_device;1723 struct sg_table *sg = NULL;1724 uint64_t user_addr = 0;1725 struct amdgpu_bo *bo;1726 struct drm_gem_object *gobj = NULL;1727 u32 domain, alloc_domain;1728 uint64_t aligned_size;1729 int8_t xcp_id = -1;1730 u64 alloc_flags;1731 int ret;1732 1733 /*1734 * Check on which domain to allocate BO1735 */1736 if (flags & KFD_IOC_ALLOC_MEM_FLAGS_VRAM) {1737 domain = alloc_domain = AMDGPU_GEM_DOMAIN_VRAM;1738 1739 if (adev->flags & AMD_IS_APU) {1740 domain = AMDGPU_GEM_DOMAIN_GTT;1741 alloc_domain = AMDGPU_GEM_DOMAIN_GTT;1742 alloc_flags = 0;1743 } else {1744 alloc_flags = AMDGPU_GEM_CREATE_VRAM_WIPE_ON_RELEASE;1745 alloc_flags |= (flags & KFD_IOC_ALLOC_MEM_FLAGS_PUBLIC) ?1746 AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED : 0;1747 1748 /* For contiguous VRAM allocation */1749 if (flags & KFD_IOC_ALLOC_MEM_FLAGS_CONTIGUOUS)1750 alloc_flags |= AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS;1751 }1752 xcp_id = fpriv->xcp_id == AMDGPU_XCP_NO_PARTITION ?1753 0 : fpriv->xcp_id;1754 } else if (flags & KFD_IOC_ALLOC_MEM_FLAGS_GTT) {1755 domain = alloc_domain = AMDGPU_GEM_DOMAIN_GTT;1756 alloc_flags = 0;1757 } else {1758 domain = AMDGPU_GEM_DOMAIN_GTT;1759 alloc_domain = AMDGPU_GEM_DOMAIN_CPU;1760 alloc_flags = AMDGPU_GEM_CREATE_PREEMPTIBLE;1761 1762 if (flags & KFD_IOC_ALLOC_MEM_FLAGS_USERPTR) {1763 if (!offset || !*offset)1764 return -EINVAL;1765 user_addr = untagged_addr(*offset);1766 } else if (flags & (KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL |1767 KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP)) {1768 bo_type = ttm_bo_type_sg;1769 if (size > UINT_MAX)1770 return -EINVAL;1771 sg = create_sg_table(*offset, size);1772 if (!sg)1773 return -ENOMEM;1774 } else {1775 return -EINVAL;1776 }1777 }1778 1779 if (flags & KFD_IOC_ALLOC_MEM_FLAGS_COHERENT)1780 alloc_flags |= AMDGPU_GEM_CREATE_COHERENT;1781 if (flags & KFD_IOC_ALLOC_MEM_FLAGS_EXT_COHERENT)1782 alloc_flags |= AMDGPU_GEM_CREATE_EXT_COHERENT;1783 if (flags & KFD_IOC_ALLOC_MEM_FLAGS_UNCACHED)1784 alloc_flags |= AMDGPU_GEM_CREATE_UNCACHED;1785 1786 *mem = kzalloc(sizeof(struct kgd_mem), GFP_KERNEL);1787 if (!*mem) {1788 ret = -ENOMEM;1789 goto err;1790 }1791 INIT_LIST_HEAD(&(*mem)->attachments);1792 mutex_init(&(*mem)->lock);1793 (*mem)->aql_queue = !!(flags & KFD_IOC_ALLOC_MEM_FLAGS_AQL_QUEUE_MEM);1794 1795 /* Workaround for AQL queue wraparound bug. Map the same1796 * memory twice. That means we only actually allocate half1797 * the memory.1798 */1799 if ((*mem)->aql_queue)1800 size >>= 1;1801 aligned_size = PAGE_ALIGN(size);1802 1803 (*mem)->alloc_flags = flags;1804 1805 amdgpu_sync_create(&(*mem)->sync);1806 1807 ret = amdgpu_amdkfd_reserve_mem_limit(adev, aligned_size, flags,1808 xcp_id);1809 if (ret) {1810 pr_debug("Insufficient memory\n");1811 goto err_reserve_limit;1812 }1813 1814 pr_debug("\tcreate BO VA 0x%llx size 0x%llx domain %s xcp_id %d\n",1815 va, (*mem)->aql_queue ? size << 1 : size,1816 domain_string(alloc_domain), xcp_id);1817 1818 ret = amdgpu_gem_object_create(adev, aligned_size, 1, alloc_domain, alloc_flags,1819 bo_type, NULL, &gobj, xcp_id + 1);1820 if (ret) {1821 pr_debug("Failed to create BO on domain %s. ret %d\n",1822 domain_string(alloc_domain), ret);1823 goto err_bo_create;1824 }1825 ret = drm_vma_node_allow(&gobj->vma_node, drm_priv);1826 if (ret) {1827 pr_debug("Failed to allow vma node access. ret %d\n", ret);1828 goto err_node_allow;1829 }1830 ret = drm_gem_handle_create(adev->kfd.client.file, gobj, &(*mem)->gem_handle);1831 if (ret)1832 goto err_gem_handle_create;1833 bo = gem_to_amdgpu_bo(gobj);1834 if (bo_type == ttm_bo_type_sg) {1835 bo->tbo.sg = sg;1836 bo->tbo.ttm->sg = sg;1837 }1838 bo->kfd_bo = *mem;1839 (*mem)->bo = bo;1840 if (user_addr)1841 bo->flags |= AMDGPU_AMDKFD_CREATE_USERPTR_BO;1842 1843 (*mem)->va = va;1844 (*mem)->domain = domain;1845 (*mem)->mapped_to_gpu_memory = 0;1846 (*mem)->process_info = avm->process_info;1847 1848 add_kgd_mem_to_kfd_bo_list(*mem, avm->process_info, user_addr);1849 1850 if (user_addr) {1851 pr_debug("creating userptr BO for user_addr = %llx\n", user_addr);1852 ret = init_user_pages(*mem, user_addr, criu_resume);1853 if (ret)1854 goto allocate_init_user_pages_failed;1855 } else if (flags & (KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL |1856 KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP)) {1857 ret = amdgpu_amdkfd_gpuvm_pin_bo(bo, AMDGPU_GEM_DOMAIN_GTT);1858 if (ret) {1859 pr_err("Pinning MMIO/DOORBELL BO during ALLOC FAILED\n");1860 goto err_pin_bo;1861 }1862 bo->allowed_domains = AMDGPU_GEM_DOMAIN_GTT;1863 bo->preferred_domains = AMDGPU_GEM_DOMAIN_GTT;1864 } else {1865 mutex_lock(&avm->process_info->lock);1866 if (avm->process_info->eviction_fence &&1867 !dma_fence_is_signaled(&avm->process_info->eviction_fence->base))1868 ret = amdgpu_amdkfd_bo_validate_and_fence(bo, domain,1869 &avm->process_info->eviction_fence->base);1870 mutex_unlock(&avm->process_info->lock);1871 if (ret)1872 goto err_validate_bo;1873 }1874 1875 if (offset)1876 *offset = amdgpu_bo_mmap_offset(bo);1877 1878 return 0;1879 1880allocate_init_user_pages_failed:1881err_pin_bo:1882err_validate_bo:1883 remove_kgd_mem_from_kfd_bo_list(*mem, avm->process_info);1884 drm_gem_handle_delete(adev->kfd.client.file, (*mem)->gem_handle);1885err_gem_handle_create:1886 drm_vma_node_revoke(&gobj->vma_node, drm_priv);1887err_node_allow:1888 /* Don't unreserve system mem limit twice */1889 goto err_reserve_limit;1890err_bo_create:1891 amdgpu_amdkfd_unreserve_mem_limit(adev, aligned_size, flags, xcp_id);1892err_reserve_limit:1893 amdgpu_sync_free(&(*mem)->sync);1894 mutex_destroy(&(*mem)->lock);1895 if (gobj)1896 drm_gem_object_put(gobj);1897 else1898 kfree(*mem);1899err:1900 if (sg) {1901 sg_free_table(sg);1902 kfree(sg);1903 }1904 return ret;1905}1906 1907int amdgpu_amdkfd_gpuvm_free_memory_of_gpu(1908 struct amdgpu_device *adev, struct kgd_mem *mem, void *drm_priv,1909 uint64_t *size)1910{1911 struct amdkfd_process_info *process_info = mem->process_info;1912 unsigned long bo_size = mem->bo->tbo.base.size;1913 bool use_release_notifier = (mem->bo->kfd_bo == mem);1914 struct kfd_mem_attachment *entry, *tmp;1915 struct bo_vm_reservation_context ctx;1916 unsigned int mapped_to_gpu_memory;1917 int ret;1918 bool is_imported = false;1919 1920 mutex_lock(&mem->lock);1921 1922 /* Unpin MMIO/DOORBELL BO's that were pinned during allocation */1923 if (mem->alloc_flags &1924 (KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL |1925 KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP)) {1926 amdgpu_amdkfd_gpuvm_unpin_bo(mem->bo);1927 }1928 1929 mapped_to_gpu_memory = mem->mapped_to_gpu_memory;1930 is_imported = mem->is_imported;1931 mutex_unlock(&mem->lock);1932 /* lock is not needed after this, since mem is unused and will1933 * be freed anyway1934 */1935 1936 if (mapped_to_gpu_memory > 0) {1937 pr_debug("BO VA 0x%llx size 0x%lx is still mapped.\n",1938 mem->va, bo_size);1939 return -EBUSY;1940 }1941 1942 /* Make sure restore workers don't access the BO any more */1943 mutex_lock(&process_info->lock);1944 list_del(&mem->validate_list);1945 mutex_unlock(&process_info->lock);1946 1947 /* Cleanup user pages and MMU notifiers */1948 if (amdgpu_ttm_tt_get_usermm(mem->bo->tbo.ttm)) {1949 amdgpu_hmm_unregister(mem->bo);1950 mutex_lock(&process_info->notifier_lock);1951 amdgpu_ttm_tt_discard_user_pages(mem->bo->tbo.ttm, mem->range);1952 mutex_unlock(&process_info->notifier_lock);1953 }1954 1955 ret = reserve_bo_and_cond_vms(mem, NULL, BO_VM_ALL, &ctx);1956 if (unlikely(ret))1957 return ret;1958 1959 amdgpu_amdkfd_remove_eviction_fence(mem->bo,1960 process_info->eviction_fence);1961 pr_debug("Release VA 0x%llx - 0x%llx\n", mem->va,1962 mem->va + bo_size * (1 + mem->aql_queue));1963 1964 /* Remove from VM internal data structures */1965 list_for_each_entry_safe(entry, tmp, &mem->attachments, list) {1966 kfd_mem_dmaunmap_attachment(mem, entry);1967 kfd_mem_detach(entry);1968 }1969 1970 ret = unreserve_bo_and_vms(&ctx, false, false);1971 1972 /* Free the sync object */1973 amdgpu_sync_free(&mem->sync);1974 1975 /* If the SG is not NULL, it's one we created for a doorbell or mmio1976 * remap BO. We need to free it.1977 */1978 if (mem->bo->tbo.sg) {1979 sg_free_table(mem->bo->tbo.sg);1980 kfree(mem->bo->tbo.sg);1981 }1982 1983 /* Update the size of the BO being freed if it was allocated from1984 * VRAM and is not imported. For APP APU VRAM allocations are done1985 * in GTT domain1986 */1987 if (size) {1988 if (!is_imported &&1989 (mem->bo->preferred_domains == AMDGPU_GEM_DOMAIN_VRAM ||1990 ((adev->flags & AMD_IS_APU) &&1991 mem->bo->preferred_domains == AMDGPU_GEM_DOMAIN_GTT)))1992 *size = bo_size;1993 else1994 *size = 0;1995 }1996 1997 /* Free the BO*/1998 drm_vma_node_revoke(&mem->bo->tbo.base.vma_node, drm_priv);1999 drm_gem_handle_delete(adev->kfd.client.file, mem->gem_handle);2000 if (mem->dmabuf) {2001 dma_buf_put(mem->dmabuf);2002 mem->dmabuf = NULL;2003 }2004 mutex_destroy(&mem->lock);2005 2006 /* If this releases the last reference, it will end up calling2007 * amdgpu_amdkfd_release_notify and kfree the mem struct. That's why2008 * this needs to be the last call here.2009 */2010 drm_gem_object_put(&mem->bo->tbo.base);2011 2012 /*2013 * For kgd_mem allocated in amdgpu_amdkfd_gpuvm_import_dmabuf(),2014 * explicitly free it here.2015 */2016 if (!use_release_notifier)2017 kfree(mem);2018 2019 return ret;2020}2021 2022int amdgpu_amdkfd_gpuvm_map_memory_to_gpu(2023 struct amdgpu_device *adev, struct kgd_mem *mem,2024 void *drm_priv)2025{2026 struct amdgpu_vm *avm = drm_priv_to_vm(drm_priv);2027 int ret;2028 struct amdgpu_bo *bo;2029 uint32_t domain;2030 struct kfd_mem_attachment *entry;2031 struct bo_vm_reservation_context ctx;2032 unsigned long bo_size;2033 bool is_invalid_userptr = false;2034 2035 bo = mem->bo;2036 if (!bo) {2037 pr_err("Invalid BO when mapping memory to GPU\n");2038 return -EINVAL;2039 }2040 2041 /* Make sure restore is not running concurrently. Since we2042 * don't map invalid userptr BOs, we rely on the next restore2043 * worker to do the mapping2044 */2045 mutex_lock(&mem->process_info->lock);2046 2047 /* Lock notifier lock. If we find an invalid userptr BO, we can be2048 * sure that the MMU notifier is no longer running2049 * concurrently and the queues are actually stopped2050 */2051 if (amdgpu_ttm_tt_get_usermm(bo->tbo.ttm)) {2052 mutex_lock(&mem->process_info->notifier_lock);2053 is_invalid_userptr = !!mem->invalid;2054 mutex_unlock(&mem->process_info->notifier_lock);2055 }2056 2057 mutex_lock(&mem->lock);2058 2059 domain = mem->domain;2060 bo_size = bo->tbo.base.size;2061 2062 pr_debug("Map VA 0x%llx - 0x%llx to vm %p domain %s\n",2063 mem->va,2064 mem->va + bo_size * (1 + mem->aql_queue),2065 avm, domain_string(domain));2066 2067 if (!kfd_mem_is_attached(avm, mem)) {2068 ret = kfd_mem_attach(adev, mem, avm, mem->aql_queue);2069 if (ret)2070 goto out;2071 }2072 2073 ret = reserve_bo_and_vm(mem, avm, &ctx);2074 if (unlikely(ret))2075 goto out;2076 2077 /* Userptr can be marked as "not invalid", but not actually be2078 * validated yet (still in the system domain). In that case2079 * the queues are still stopped and we can leave mapping for2080 * the next restore worker2081 */2082 if (amdgpu_ttm_tt_get_usermm(bo->tbo.ttm) &&2083 bo->tbo.resource->mem_type == TTM_PL_SYSTEM)2084 is_invalid_userptr = true;2085 2086 ret = vm_validate_pt_pd_bos(avm, NULL);2087 if (unlikely(ret))2088 goto out_unreserve;2089 2090 list_for_each_entry(entry, &mem->attachments, list) {2091 if (entry->bo_va->base.vm != avm || entry->is_mapped)2092 continue;2093 2094 pr_debug("\t map VA 0x%llx - 0x%llx in entry %p\n",2095 entry->va, entry->va + bo_size, entry);2096 2097 ret = map_bo_to_gpuvm(mem, entry, ctx.sync,2098 is_invalid_userptr);2099 if (ret) {2100 pr_err("Failed to map bo to gpuvm\n");2101 goto out_unreserve;2102 }2103 2104 ret = vm_update_pds(avm, ctx.sync);2105 if (ret) {2106 pr_err("Failed to update page directories\n");2107 goto out_unreserve;2108 }2109 2110 entry->is_mapped = true;2111 mem->mapped_to_gpu_memory++;2112 pr_debug("\t INC mapping count %d\n",2113 mem->mapped_to_gpu_memory);2114 }2115 2116 ret = unreserve_bo_and_vms(&ctx, false, false);2117 2118 goto out;2119 2120out_unreserve:2121 unreserve_bo_and_vms(&ctx, false, false);2122out:2123 mutex_unlock(&mem->process_info->lock);2124 mutex_unlock(&mem->lock);2125 return ret;2126}2127 2128int amdgpu_amdkfd_gpuvm_dmaunmap_mem(struct kgd_mem *mem, void *drm_priv)2129{2130 struct kfd_mem_attachment *entry;2131 struct amdgpu_vm *vm;2132 int ret;2133 2134 vm = drm_priv_to_vm(drm_priv);2135 2136 mutex_lock(&mem->lock);2137 2138 ret = amdgpu_bo_reserve(mem->bo, true);2139 if (ret)2140 goto out;2141 2142 list_for_each_entry(entry, &mem->attachments, list) {2143 if (entry->bo_va->base.vm != vm)2144 continue;2145 if (entry->bo_va->base.bo->tbo.ttm &&2146 !entry->bo_va->base.bo->tbo.ttm->sg)2147 continue;2148 2149 kfd_mem_dmaunmap_attachment(mem, entry);2150 }2151 2152 amdgpu_bo_unreserve(mem->bo);2153out:2154 mutex_unlock(&mem->lock);2155 2156 return ret;2157}2158 2159int amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu(2160 struct amdgpu_device *adev, struct kgd_mem *mem, void *drm_priv)2161{2162 struct amdgpu_vm *avm = drm_priv_to_vm(drm_priv);2163 unsigned long bo_size = mem->bo->tbo.base.size;2164 struct kfd_mem_attachment *entry;2165 struct bo_vm_reservation_context ctx;2166 int ret;2167 2168 mutex_lock(&mem->lock);2169 2170 ret = reserve_bo_and_cond_vms(mem, avm, BO_VM_MAPPED, &ctx);2171 if (unlikely(ret))2172 goto out;2173 /* If no VMs were reserved, it means the BO wasn't actually mapped */2174 if (ctx.n_vms == 0) {2175 ret = -EINVAL;2176 goto unreserve_out;2177 }2178 2179 ret = vm_validate_pt_pd_bos(avm, NULL);2180 if (unlikely(ret))2181 goto unreserve_out;2182 2183 pr_debug("Unmap VA 0x%llx - 0x%llx from vm %p\n",2184 mem->va,2185 mem->va + bo_size * (1 + mem->aql_queue),2186 avm);2187 2188 list_for_each_entry(entry, &mem->attachments, list) {2189 if (entry->bo_va->base.vm != avm || !entry->is_mapped)2190 continue;2191 2192 pr_debug("\t unmap VA 0x%llx - 0x%llx from entry %p\n",2193 entry->va, entry->va + bo_size, entry);2194 2195 ret = unmap_bo_from_gpuvm(mem, entry, ctx.sync);2196 if (ret)2197 goto unreserve_out;2198 2199 entry->is_mapped = false;2200 2201 mem->mapped_to_gpu_memory--;2202 pr_debug("\t DEC mapping count %d\n",2203 mem->mapped_to_gpu_memory);2204 }2205 2206unreserve_out:2207 unreserve_bo_and_vms(&ctx, false, false);2208out:2209 mutex_unlock(&mem->lock);2210 return ret;2211}2212 2213int amdgpu_amdkfd_gpuvm_sync_memory(2214 struct amdgpu_device *adev, struct kgd_mem *mem, bool intr)2215{2216 struct amdgpu_sync sync;2217 int ret;2218 2219 amdgpu_sync_create(&sync);2220 2221 mutex_lock(&mem->lock);2222 amdgpu_sync_clone(&mem->sync, &sync);2223 mutex_unlock(&mem->lock);2224 2225 ret = amdgpu_sync_wait(&sync, intr);2226 amdgpu_sync_free(&sync);2227 return ret;2228}2229 2230/**2231 * amdgpu_amdkfd_map_gtt_bo_to_gart - Map BO to GART and increment reference count2232 * @bo: Buffer object to be mapped2233 * @bo_gart: Return bo reference2234 *2235 * Before return, bo reference count is incremented. To release the reference and unpin/2236 * unmap the BO, call amdgpu_amdkfd_free_gtt_mem.2237 */2238int amdgpu_amdkfd_map_gtt_bo_to_gart(struct amdgpu_bo *bo, struct amdgpu_bo **bo_gart)2239{2240 int ret;2241 2242 ret = amdgpu_bo_reserve(bo, true);2243 if (ret) {2244 pr_err("Failed to reserve bo. ret %d\n", ret);2245 goto err_reserve_bo_failed;2246 }2247 2248 ret = amdgpu_bo_pin(bo, AMDGPU_GEM_DOMAIN_GTT);2249 if (ret) {2250 pr_err("Failed to pin bo. ret %d\n", ret);2251 goto err_pin_bo_failed;2252 }2253 2254 ret = amdgpu_ttm_alloc_gart(&bo->tbo);2255 if (ret) {2256 pr_err("Failed to bind bo to GART. ret %d\n", ret);2257 goto err_map_bo_gart_failed;2258 }2259 2260 amdgpu_amdkfd_remove_eviction_fence(2261 bo, bo->vm_bo->vm->process_info->eviction_fence);2262 2263 amdgpu_bo_unreserve(bo);2264 2265 *bo_gart = amdgpu_bo_ref(bo);2266 2267 return 0;2268 2269err_map_bo_gart_failed:2270 amdgpu_bo_unpin(bo);2271err_pin_bo_failed:2272 amdgpu_bo_unreserve(bo);2273err_reserve_bo_failed:2274 2275 return ret;2276}2277 2278/** amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel() - Map a GTT BO for kernel CPU access2279 *2280 * @mem: Buffer object to be mapped for CPU access2281 * @kptr[out]: pointer in kernel CPU address space2282 * @size[out]: size of the buffer2283 *2284 * Pins the BO and maps it for kernel CPU access. The eviction fence is removed2285 * from the BO, since pinned BOs cannot be evicted. The bo must remain on the2286 * validate_list, so the GPU mapping can be restored after a page table was2287 * evicted.2288 *2289 * Return: 0 on success, error code on failure2290 */2291int amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel(struct kgd_mem *mem,2292 void **kptr, uint64_t *size)2293{2294 int ret;2295 struct amdgpu_bo *bo = mem->bo;2296 2297 if (amdgpu_ttm_tt_get_usermm(bo->tbo.ttm)) {2298 pr_err("userptr can't be mapped to kernel\n");2299 return -EINVAL;2300 }2301 2302 mutex_lock(&mem->process_info->lock);2303 2304 ret = amdgpu_bo_reserve(bo, true);2305 if (ret) {2306 pr_err("Failed to reserve bo. ret %d\n", ret);2307 goto bo_reserve_failed;2308 }2309 2310 ret = amdgpu_bo_pin(bo, AMDGPU_GEM_DOMAIN_GTT);2311 if (ret) {2312 pr_err("Failed to pin bo. ret %d\n", ret);2313 goto pin_failed;2314 }2315 2316 ret = amdgpu_bo_kmap(bo, kptr);2317 if (ret) {2318 pr_err("Failed to map bo to kernel. ret %d\n", ret);2319 goto kmap_failed;2320 }2321 2322 amdgpu_amdkfd_remove_eviction_fence(2323 bo, mem->process_info->eviction_fence);2324 2325 if (size)2326 *size = amdgpu_bo_size(bo);2327 2328 amdgpu_bo_unreserve(bo);2329 2330 mutex_unlock(&mem->process_info->lock);2331 return 0;2332 2333kmap_failed:2334 amdgpu_bo_unpin(bo);2335pin_failed:2336 amdgpu_bo_unreserve(bo);2337bo_reserve_failed:2338 mutex_unlock(&mem->process_info->lock);2339 2340 return ret;2341}2342 2343/** amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel() - Unmap a GTT BO for kernel CPU access2344 *2345 * @mem: Buffer object to be unmapped for CPU access2346 *2347 * Removes the kernel CPU mapping and unpins the BO. It does not restore the2348 * eviction fence, so this function should only be used for cleanup before the2349 * BO is destroyed.2350 */2351void amdgpu_amdkfd_gpuvm_unmap_gtt_bo_from_kernel(struct kgd_mem *mem)2352{2353 struct amdgpu_bo *bo = mem->bo;2354 2355 amdgpu_bo_reserve(bo, true);2356 amdgpu_bo_kunmap(bo);2357 amdgpu_bo_unpin(bo);2358 amdgpu_bo_unreserve(bo);2359}2360 2361int amdgpu_amdkfd_gpuvm_get_vm_fault_info(struct amdgpu_device *adev,2362 struct kfd_vm_fault_info *mem)2363{2364 if (atomic_read(&adev->gmc.vm_fault_info_updated) == 1) {2365 *mem = *adev->gmc.vm_fault_info;2366 mb(); /* make sure read happened */2367 atomic_set(&adev->gmc.vm_fault_info_updated, 0);2368 }2369 return 0;2370}2371 2372static int import_obj_create(struct amdgpu_device *adev,2373 struct dma_buf *dma_buf,2374 struct drm_gem_object *obj,2375 uint64_t va, void *drm_priv,2376 struct kgd_mem **mem, uint64_t *size,2377 uint64_t *mmap_offset)2378{2379 struct amdgpu_vm *avm = drm_priv_to_vm(drm_priv);2380 struct amdgpu_bo *bo;2381 int ret;2382 2383 bo = gem_to_amdgpu_bo(obj);2384 if (!(bo->preferred_domains & (AMDGPU_GEM_DOMAIN_VRAM |2385 AMDGPU_GEM_DOMAIN_GTT)))2386 /* Only VRAM and GTT BOs are supported */2387 return -EINVAL;2388 2389 *mem = kzalloc(sizeof(struct kgd_mem), GFP_KERNEL);2390 if (!*mem)2391 return -ENOMEM;2392 2393 ret = drm_vma_node_allow(&obj->vma_node, drm_priv);2394 if (ret)2395 goto err_free_mem;2396 2397 if (size)2398 *size = amdgpu_bo_size(bo);2399 2400 if (mmap_offset)2401 *mmap_offset = amdgpu_bo_mmap_offset(bo);2402 2403 INIT_LIST_HEAD(&(*mem)->attachments);2404 mutex_init(&(*mem)->lock);2405 2406 (*mem)->alloc_flags =2407 ((bo->preferred_domains & AMDGPU_GEM_DOMAIN_VRAM) ?2408 KFD_IOC_ALLOC_MEM_FLAGS_VRAM : KFD_IOC_ALLOC_MEM_FLAGS_GTT)2409 | KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE2410 | KFD_IOC_ALLOC_MEM_FLAGS_EXECUTABLE;2411 2412 get_dma_buf(dma_buf);2413 (*mem)->dmabuf = dma_buf;2414 (*mem)->bo = bo;2415 (*mem)->va = va;2416 (*mem)->domain = (bo->preferred_domains & AMDGPU_GEM_DOMAIN_VRAM) &&2417 !(adev->flags & AMD_IS_APU) ?2418 AMDGPU_GEM_DOMAIN_VRAM : AMDGPU_GEM_DOMAIN_GTT;2419 2420 (*mem)->mapped_to_gpu_memory = 0;2421 (*mem)->process_info = avm->process_info;2422 add_kgd_mem_to_kfd_bo_list(*mem, avm->process_info, false);2423 amdgpu_sync_create(&(*mem)->sync);2424 (*mem)->is_imported = true;2425 2426 mutex_lock(&avm->process_info->lock);2427 if (avm->process_info->eviction_fence &&2428 !dma_fence_is_signaled(&avm->process_info->eviction_fence->base))2429 ret = amdgpu_amdkfd_bo_validate_and_fence(bo, (*mem)->domain,2430 &avm->process_info->eviction_fence->base);2431 mutex_unlock(&avm->process_info->lock);2432 if (ret)2433 goto err_remove_mem;2434 2435 return 0;2436 2437err_remove_mem:2438 remove_kgd_mem_from_kfd_bo_list(*mem, avm->process_info);2439 drm_vma_node_revoke(&obj->vma_node, drm_priv);2440err_free_mem:2441 kfree(*mem);2442 return ret;2443}2444 2445int amdgpu_amdkfd_gpuvm_import_dmabuf_fd(struct amdgpu_device *adev, int fd,2446 uint64_t va, void *drm_priv,2447 struct kgd_mem **mem, uint64_t *size,2448 uint64_t *mmap_offset)2449{2450 struct drm_gem_object *obj;2451 uint32_t handle;2452 int ret;2453 2454 ret = drm_gem_prime_fd_to_handle(&adev->ddev, adev->kfd.client.file, fd,2455 &handle);2456 if (ret)2457 return ret;2458 obj = drm_gem_object_lookup(adev->kfd.client.file, handle);2459 if (!obj) {2460 ret = -EINVAL;2461 goto err_release_handle;2462 }2463 2464 ret = import_obj_create(adev, obj->dma_buf, obj, va, drm_priv, mem, size,2465 mmap_offset);2466 if (ret)2467 goto err_put_obj;2468 2469 (*mem)->gem_handle = handle;2470 2471 return 0;2472 2473err_put_obj:2474 drm_gem_object_put(obj);2475err_release_handle:2476 drm_gem_handle_delete(adev->kfd.client.file, handle);2477 return ret;2478}2479 2480int amdgpu_amdkfd_gpuvm_export_dmabuf(struct kgd_mem *mem,2481 struct dma_buf **dma_buf)2482{2483 int ret;2484 2485 mutex_lock(&mem->lock);2486 ret = kfd_mem_export_dmabuf(mem);2487 if (ret)2488 goto out;2489 2490 get_dma_buf(mem->dmabuf);2491 *dma_buf = mem->dmabuf;2492out:2493 mutex_unlock(&mem->lock);2494 return ret;2495}2496 2497/* Evict a userptr BO by stopping the queues if necessary2498 *2499 * Runs in MMU notifier, may be in RECLAIM_FS context. This means it2500 * cannot do any memory allocations, and cannot take any locks that2501 * are held elsewhere while allocating memory.2502 *2503 * It doesn't do anything to the BO itself. The real work happens in2504 * restore, where we get updated page addresses. This function only2505 * ensures that GPU access to the BO is stopped.2506 */2507int amdgpu_amdkfd_evict_userptr(struct mmu_interval_notifier *mni,2508 unsigned long cur_seq, struct kgd_mem *mem)2509{2510 struct amdkfd_process_info *process_info = mem->process_info;2511 int r = 0;2512 2513 /* Do not process MMU notifications during CRIU restore until2514 * KFD_CRIU_OP_RESUME IOCTL is received2515 */2516 if (READ_ONCE(process_info->block_mmu_notifications))2517 return 0;2518 2519 mutex_lock(&process_info->notifier_lock);2520 mmu_interval_set_seq(mni, cur_seq);2521 2522 mem->invalid++;2523 if (++process_info->evicted_bos == 1) {2524 /* First eviction, stop the queues */2525 r = kgd2kfd_quiesce_mm(mni->mm,2526 KFD_QUEUE_EVICTION_TRIGGER_USERPTR);2527 if (r)2528 pr_err("Failed to quiesce KFD\n");2529 queue_delayed_work(system_freezable_wq,2530 &process_info->restore_userptr_work,2531 msecs_to_jiffies(AMDGPU_USERPTR_RESTORE_DELAY_MS));2532 }2533 mutex_unlock(&process_info->notifier_lock);2534 2535 return r;2536}2537 2538/* Update invalid userptr BOs2539 *2540 * Moves invalidated (evicted) userptr BOs from userptr_valid_list to2541 * userptr_inval_list and updates user pages for all BOs that have2542 * been invalidated since their last update.2543 */2544static int update_invalid_user_pages(struct amdkfd_process_info *process_info,2545 struct mm_struct *mm)2546{2547 struct kgd_mem *mem, *tmp_mem;2548 struct amdgpu_bo *bo;2549 struct ttm_operation_ctx ctx = { false, false };2550 uint32_t invalid;2551 int ret = 0;2552 2553 mutex_lock(&process_info->notifier_lock);2554 2555 /* Move all invalidated BOs to the userptr_inval_list */2556 list_for_each_entry_safe(mem, tmp_mem,2557 &process_info->userptr_valid_list,2558 validate_list)2559 if (mem->invalid)2560 list_move_tail(&mem->validate_list,2561 &process_info->userptr_inval_list);2562 2563 /* Go through userptr_inval_list and update any invalid user_pages */2564 list_for_each_entry(mem, &process_info->userptr_inval_list,2565 validate_list) {2566 invalid = mem->invalid;2567 if (!invalid)2568 /* BO hasn't been invalidated since the last2569 * revalidation attempt. Keep its page list.2570 */2571 continue;2572 2573 bo = mem->bo;2574 2575 amdgpu_ttm_tt_discard_user_pages(bo->tbo.ttm, mem->range);2576 mem->range = NULL;2577 2578 /* BO reservations and getting user pages (hmm_range_fault)2579 * must happen outside the notifier lock2580 */2581 mutex_unlock(&process_info->notifier_lock);2582 2583 /* Move the BO to system (CPU) domain if necessary to unmap2584 * and free the SG table2585 */2586 if (bo->tbo.resource->mem_type != TTM_PL_SYSTEM) {2587 if (amdgpu_bo_reserve(bo, true))2588 return -EAGAIN;2589 amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_CPU);2590 ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);2591 amdgpu_bo_unreserve(bo);2592 if (ret) {2593 pr_err("%s: Failed to invalidate userptr BO\n",2594 __func__);2595 return -EAGAIN;2596 }2597 }2598 2599 /* Get updated user pages */2600 ret = amdgpu_ttm_tt_get_user_pages(bo, bo->tbo.ttm->pages,2601 &mem->range);2602 if (ret) {2603 pr_debug("Failed %d to get user pages\n", ret);2604 2605 /* Return -EFAULT bad address error as success. It will2606 * fail later with a VM fault if the GPU tries to access2607 * it. Better than hanging indefinitely with stalled2608 * user mode queues.2609 *2610 * Return other error -EBUSY or -ENOMEM to retry restore2611 */2612 if (ret != -EFAULT)2613 return ret;2614 2615 ret = 0;2616 }2617 2618 mutex_lock(&process_info->notifier_lock);2619 2620 /* Mark the BO as valid unless it was invalidated2621 * again concurrently.2622 */2623 if (mem->invalid != invalid) {2624 ret = -EAGAIN;2625 goto unlock_out;2626 }2627 /* set mem valid if mem has hmm range associated */2628 if (mem->range)2629 mem->invalid = 0;2630 }2631 2632unlock_out:2633 mutex_unlock(&process_info->notifier_lock);2634 2635 return ret;2636}2637 2638/* Validate invalid userptr BOs2639 *2640 * Validates BOs on the userptr_inval_list. Also updates GPUVM page tables2641 * with new page addresses and waits for the page table updates to complete.2642 */2643static int validate_invalid_user_pages(struct amdkfd_process_info *process_info)2644{2645 struct ttm_operation_ctx ctx = { false, false };2646 struct amdgpu_sync sync;2647 struct drm_exec exec;2648 2649 struct amdgpu_vm *peer_vm;2650 struct kgd_mem *mem, *tmp_mem;2651 struct amdgpu_bo *bo;2652 int ret;2653 2654 amdgpu_sync_create(&sync);2655 2656 drm_exec_init(&exec, 0, 0);2657 /* Reserve all BOs and page tables for validation */2658 drm_exec_until_all_locked(&exec) {2659 /* Reserve all the page directories */2660 list_for_each_entry(peer_vm, &process_info->vm_list_head,2661 vm_list_node) {2662 ret = amdgpu_vm_lock_pd(peer_vm, &exec, 2);2663 drm_exec_retry_on_contention(&exec);2664 if (unlikely(ret))2665 goto unreserve_out;2666 }2667 2668 /* Reserve the userptr_inval_list entries to resv_list */2669 list_for_each_entry(mem, &process_info->userptr_inval_list,2670 validate_list) {2671 struct drm_gem_object *gobj;2672 2673 gobj = &mem->bo->tbo.base;2674 ret = drm_exec_prepare_obj(&exec, gobj, 1);2675 drm_exec_retry_on_contention(&exec);2676 if (unlikely(ret))2677 goto unreserve_out;2678 }2679 }2680 2681 ret = process_validate_vms(process_info, NULL);2682 if (ret)2683 goto unreserve_out;2684 2685 /* Validate BOs and update GPUVM page tables */2686 list_for_each_entry_safe(mem, tmp_mem,2687 &process_info->userptr_inval_list,2688 validate_list) {2689 struct kfd_mem_attachment *attachment;2690 2691 bo = mem->bo;2692 2693 /* Validate the BO if we got user pages */2694 if (bo->tbo.ttm->pages[0]) {2695 amdgpu_bo_placement_from_domain(bo, mem->domain);2696 ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);2697 if (ret) {2698 pr_err("%s: failed to validate BO\n", __func__);2699 goto unreserve_out;2700 }2701 }2702 2703 /* Update mapping. If the BO was not validated2704 * (because we couldn't get user pages), this will2705 * clear the page table entries, which will result in2706 * VM faults if the GPU tries to access the invalid2707 * memory.2708 */2709 list_for_each_entry(attachment, &mem->attachments, list) {2710 if (!attachment->is_mapped)2711 continue;2712 2713 kfd_mem_dmaunmap_attachment(mem, attachment);2714 ret = update_gpuvm_pte(mem, attachment, &sync);2715 if (ret) {2716 pr_err("%s: update PTE failed\n", __func__);2717 /* make sure this gets validated again */2718 mutex_lock(&process_info->notifier_lock);2719 mem->invalid++;2720 mutex_unlock(&process_info->notifier_lock);2721 goto unreserve_out;2722 }2723 }2724 }2725 2726 /* Update page directories */2727 ret = process_update_pds(process_info, &sync);2728 2729unreserve_out:2730 drm_exec_fini(&exec);2731 amdgpu_sync_wait(&sync, false);2732 amdgpu_sync_free(&sync);2733 2734 return ret;2735}2736 2737/* Confirm that all user pages are valid while holding the notifier lock2738 *2739 * Moves valid BOs from the userptr_inval_list back to userptr_val_list.2740 */2741static int confirm_valid_user_pages_locked(struct amdkfd_process_info *process_info)2742{2743 struct kgd_mem *mem, *tmp_mem;2744 int ret = 0;2745 2746 list_for_each_entry_safe(mem, tmp_mem,2747 &process_info->userptr_inval_list,2748 validate_list) {2749 bool valid;2750 2751 /* keep mem without hmm range at userptr_inval_list */2752 if (!mem->range)2753 continue;2754 2755 /* Only check mem with hmm range associated */2756 valid = amdgpu_ttm_tt_get_user_pages_done(2757 mem->bo->tbo.ttm, mem->range);2758 2759 mem->range = NULL;2760 if (!valid) {2761 WARN(!mem->invalid, "Invalid BO not marked invalid");2762 ret = -EAGAIN;2763 continue;2764 }2765 2766 if (mem->invalid) {2767 WARN(1, "Valid BO is marked invalid");2768 ret = -EAGAIN;2769 continue;2770 }2771 2772 list_move_tail(&mem->validate_list,2773 &process_info->userptr_valid_list);2774 }2775 2776 return ret;2777}2778 2779/* Worker callback to restore evicted userptr BOs2780 *2781 * Tries to update and validate all userptr BOs. If successful and no2782 * concurrent evictions happened, the queues are restarted. Otherwise,2783 * reschedule for another attempt later.2784 */2785static void amdgpu_amdkfd_restore_userptr_worker(struct work_struct *work)2786{2787 struct delayed_work *dwork = to_delayed_work(work);2788 struct amdkfd_process_info *process_info =2789 container_of(dwork, struct amdkfd_process_info,2790 restore_userptr_work);2791 struct task_struct *usertask;2792 struct mm_struct *mm;2793 uint32_t evicted_bos;2794 2795 mutex_lock(&process_info->notifier_lock);2796 evicted_bos = process_info->evicted_bos;2797 mutex_unlock(&process_info->notifier_lock);2798 if (!evicted_bos)2799 return;2800 2801 /* Reference task and mm in case of concurrent process termination */2802 usertask = get_pid_task(process_info->pid, PIDTYPE_PID);2803 if (!usertask)2804 return;2805 mm = get_task_mm(usertask);2806 if (!mm) {2807 put_task_struct(usertask);2808 return;2809 }2810 2811 mutex_lock(&process_info->lock);2812 2813 if (update_invalid_user_pages(process_info, mm))2814 goto unlock_out;2815 /* userptr_inval_list can be empty if all evicted userptr BOs2816 * have been freed. In that case there is nothing to validate2817 * and we can just restart the queues.2818 */2819 if (!list_empty(&process_info->userptr_inval_list)) {2820 if (validate_invalid_user_pages(process_info))2821 goto unlock_out;2822 }2823 /* Final check for concurrent evicton and atomic update. If2824 * another eviction happens after successful update, it will2825 * be a first eviction that calls quiesce_mm. The eviction2826 * reference counting inside KFD will handle this case.2827 */2828 mutex_lock(&process_info->notifier_lock);2829 if (process_info->evicted_bos != evicted_bos)2830 goto unlock_notifier_out;2831 2832 if (confirm_valid_user_pages_locked(process_info)) {2833 WARN(1, "User pages unexpectedly invalid");2834 goto unlock_notifier_out;2835 }2836 2837 process_info->evicted_bos = evicted_bos = 0;2838 2839 if (kgd2kfd_resume_mm(mm)) {2840 pr_err("%s: Failed to resume KFD\n", __func__);2841 /* No recovery from this failure. Probably the CP is2842 * hanging. No point trying again.2843 */2844 }2845 2846unlock_notifier_out:2847 mutex_unlock(&process_info->notifier_lock);2848unlock_out:2849 mutex_unlock(&process_info->lock);2850 2851 /* If validation failed, reschedule another attempt */2852 if (evicted_bos) {2853 queue_delayed_work(system_freezable_wq,2854 &process_info->restore_userptr_work,2855 msecs_to_jiffies(AMDGPU_USERPTR_RESTORE_DELAY_MS));2856 2857 kfd_smi_event_queue_restore_rescheduled(mm);2858 }2859 mmput(mm);2860 put_task_struct(usertask);2861}2862 2863static void replace_eviction_fence(struct dma_fence __rcu **ef,2864 struct dma_fence *new_ef)2865{2866 struct dma_fence *old_ef = rcu_replace_pointer(*ef, new_ef, true2867 /* protected by process_info->lock */);2868 2869 /* If we're replacing an unsignaled eviction fence, that fence will2870 * never be signaled, and if anyone is still waiting on that fence,2871 * they will hang forever. This should never happen. We should only2872 * replace the fence in restore_work that only gets scheduled after2873 * eviction work signaled the fence.2874 */2875 WARN_ONCE(!dma_fence_is_signaled(old_ef),2876 "Replacing unsignaled eviction fence");2877 dma_fence_put(old_ef);2878}2879 2880/** amdgpu_amdkfd_gpuvm_restore_process_bos - Restore all BOs for the given2881 * KFD process identified by process_info2882 *2883 * @process_info: amdkfd_process_info of the KFD process2884 *2885 * After memory eviction, restore thread calls this function. The function2886 * should be called when the Process is still valid. BO restore involves -2887 *2888 * 1. Release old eviction fence and create new one2889 * 2. Get two copies of PD BO list from all the VMs. Keep one copy as pd_list.2890 * 3 Use the second PD list and kfd_bo_list to create a list (ctx.list) of2891 * BOs that need to be reserved.2892 * 4. Reserve all the BOs2893 * 5. Validate of PD and PT BOs.2894 * 6. Validate all KFD BOs using kfd_bo_list and Map them and add new fence2895 * 7. Add fence to all PD and PT BOs.2896 * 8. Unreserve all BOs2897 */2898int amdgpu_amdkfd_gpuvm_restore_process_bos(void *info, struct dma_fence __rcu **ef)2899{2900 struct amdkfd_process_info *process_info = info;2901 struct amdgpu_vm *peer_vm;2902 struct kgd_mem *mem;2903 struct list_head duplicate_save;2904 struct amdgpu_sync sync_obj;2905 unsigned long failed_size = 0;2906 unsigned long total_size = 0;2907 struct drm_exec exec;2908 int ret;2909 2910 INIT_LIST_HEAD(&duplicate_save);2911 2912 mutex_lock(&process_info->lock);2913 2914 drm_exec_init(&exec, DRM_EXEC_IGNORE_DUPLICATES, 0);2915 drm_exec_until_all_locked(&exec) {2916 list_for_each_entry(peer_vm, &process_info->vm_list_head,2917 vm_list_node) {2918 ret = amdgpu_vm_lock_pd(peer_vm, &exec, 2);2919 drm_exec_retry_on_contention(&exec);2920 if (unlikely(ret)) {2921 pr_err("Locking VM PD failed, ret: %d\n", ret);2922 goto ttm_reserve_fail;2923 }2924 }2925 2926 /* Reserve all BOs and page tables/directory. Add all BOs from2927 * kfd_bo_list to ctx.list2928 */2929 list_for_each_entry(mem, &process_info->kfd_bo_list,2930 validate_list) {2931 struct drm_gem_object *gobj;2932 2933 gobj = &mem->bo->tbo.base;2934 ret = drm_exec_prepare_obj(&exec, gobj, 1);2935 drm_exec_retry_on_contention(&exec);2936 if (unlikely(ret)) {2937 pr_err("drm_exec_prepare_obj failed, ret: %d\n", ret);2938 goto ttm_reserve_fail;2939 }2940 }2941 }2942 2943 amdgpu_sync_create(&sync_obj);2944 2945 /* Validate BOs managed by KFD */2946 list_for_each_entry(mem, &process_info->kfd_bo_list,2947 validate_list) {2948 2949 struct amdgpu_bo *bo = mem->bo;2950 uint32_t domain = mem->domain;2951 struct dma_resv_iter cursor;2952 struct dma_fence *fence;2953 2954 total_size += amdgpu_bo_size(bo);2955 2956 ret = amdgpu_amdkfd_bo_validate(bo, domain, false);2957 if (ret) {2958 pr_debug("Memory eviction: Validate BOs failed\n");2959 failed_size += amdgpu_bo_size(bo);2960 ret = amdgpu_amdkfd_bo_validate(bo,2961 AMDGPU_GEM_DOMAIN_GTT, false);2962 if (ret) {2963 pr_debug("Memory eviction: Try again\n");2964 goto validate_map_fail;2965 }2966 }2967 dma_resv_for_each_fence(&cursor, bo->tbo.base.resv,2968 DMA_RESV_USAGE_KERNEL, fence) {2969 ret = amdgpu_sync_fence(&sync_obj, fence);2970 if (ret) {2971 pr_debug("Memory eviction: Sync BO fence failed. Try again\n");2972 goto validate_map_fail;2973 }2974 }2975 }2976 2977 if (failed_size)2978 pr_debug("0x%lx/0x%lx in system\n", failed_size, total_size);2979 2980 /* Validate PDs, PTs and evicted DMABuf imports last. Otherwise BO2981 * validations above would invalidate DMABuf imports again.2982 */2983 ret = process_validate_vms(process_info, &exec.ticket);2984 if (ret) {2985 pr_debug("Validating VMs failed, ret: %d\n", ret);2986 goto validate_map_fail;2987 }2988 2989 /* Update mappings managed by KFD. */2990 list_for_each_entry(mem, &process_info->kfd_bo_list,2991 validate_list) {2992 struct kfd_mem_attachment *attachment;2993 2994 list_for_each_entry(attachment, &mem->attachments, list) {2995 if (!attachment->is_mapped)2996 continue;2997 2998 kfd_mem_dmaunmap_attachment(mem, attachment);2999 ret = update_gpuvm_pte(mem, attachment, &sync_obj);3000 if (ret) {3001 pr_debug("Memory eviction: update PTE failed. Try again\n");3002 goto validate_map_fail;3003 }3004 }3005 }3006 3007 /* Update mappings not managed by KFD */3008 list_for_each_entry(peer_vm, &process_info->vm_list_head,3009 vm_list_node) {3010 struct amdgpu_device *adev = amdgpu_ttm_adev(3011 peer_vm->root.bo->tbo.bdev);3012 3013 ret = amdgpu_vm_handle_moved(adev, peer_vm, &exec.ticket);3014 if (ret) {3015 pr_debug("Memory eviction: handle moved failed. Try again\n");3016 goto validate_map_fail;3017 }3018 }3019 3020 /* Update page directories */3021 ret = process_update_pds(process_info, &sync_obj);3022 if (ret) {3023 pr_debug("Memory eviction: update PDs failed. Try again\n");3024 goto validate_map_fail;3025 }3026 3027 /* Sync with fences on all the page tables. They implicitly depend on any3028 * move fences from amdgpu_vm_handle_moved above.3029 */3030 ret = process_sync_pds_resv(process_info, &sync_obj);3031 if (ret) {3032 pr_debug("Memory eviction: Failed to sync to PD BO moving fence. Try again\n");3033 goto validate_map_fail;3034 }3035 3036 /* Wait for validate and PT updates to finish */3037 amdgpu_sync_wait(&sync_obj, false);3038 3039 /* The old eviction fence may be unsignaled if restore happens3040 * after a GPU reset or suspend/resume. Keep the old fence in that3041 * case. Otherwise release the old eviction fence and create new3042 * one, because fence only goes from unsignaled to signaled once3043 * and cannot be reused. Use context and mm from the old fence.3044 *3045 * If an old eviction fence signals after this check, that's OK.3046 * Anyone signaling an eviction fence must stop the queues first3047 * and schedule another restore worker.3048 */3049 if (dma_fence_is_signaled(&process_info->eviction_fence->base)) {3050 struct amdgpu_amdkfd_fence *new_fence =3051 amdgpu_amdkfd_fence_create(3052 process_info->eviction_fence->base.context,3053 process_info->eviction_fence->mm,3054 NULL);3055 3056 if (!new_fence) {3057 pr_err("Failed to create eviction fence\n");3058 ret = -ENOMEM;3059 goto validate_map_fail;3060 }3061 dma_fence_put(&process_info->eviction_fence->base);3062 process_info->eviction_fence = new_fence;3063 replace_eviction_fence(ef, dma_fence_get(&new_fence->base));3064 } else {3065 WARN_ONCE(*ef != &process_info->eviction_fence->base,3066 "KFD eviction fence doesn't match KGD process_info");3067 }3068 3069 /* Attach new eviction fence to all BOs except pinned ones */3070 list_for_each_entry(mem, &process_info->kfd_bo_list, validate_list) {3071 if (mem->bo->tbo.pin_count)3072 continue;3073 3074 dma_resv_add_fence(mem->bo->tbo.base.resv,3075 &process_info->eviction_fence->base,3076 DMA_RESV_USAGE_BOOKKEEP);3077 }3078 /* Attach eviction fence to PD / PT BOs and DMABuf imports */3079 list_for_each_entry(peer_vm, &process_info->vm_list_head,3080 vm_list_node) {3081 struct amdgpu_bo *bo = peer_vm->root.bo;3082 3083 dma_resv_add_fence(bo->tbo.base.resv,3084 &process_info->eviction_fence->base,3085 DMA_RESV_USAGE_BOOKKEEP);3086 }3087 3088validate_map_fail:3089 amdgpu_sync_free(&sync_obj);3090ttm_reserve_fail:3091 drm_exec_fini(&exec);3092 mutex_unlock(&process_info->lock);3093 return ret;3094}3095 3096int amdgpu_amdkfd_add_gws_to_process(void *info, void *gws, struct kgd_mem **mem)3097{3098 struct amdkfd_process_info *process_info = (struct amdkfd_process_info *)info;3099 struct amdgpu_bo *gws_bo = (struct amdgpu_bo *)gws;3100 int ret;3101 3102 if (!info || !gws)3103 return -EINVAL;3104 3105 *mem = kzalloc(sizeof(struct kgd_mem), GFP_KERNEL);3106 if (!*mem)3107 return -ENOMEM;3108 3109 mutex_init(&(*mem)->lock);3110 INIT_LIST_HEAD(&(*mem)->attachments);3111 (*mem)->bo = amdgpu_bo_ref(gws_bo);3112 (*mem)->domain = AMDGPU_GEM_DOMAIN_GWS;3113 (*mem)->process_info = process_info;3114 add_kgd_mem_to_kfd_bo_list(*mem, process_info, false);3115 amdgpu_sync_create(&(*mem)->sync);3116 3117 3118 /* Validate gws bo the first time it is added to process */3119 mutex_lock(&(*mem)->process_info->lock);3120 ret = amdgpu_bo_reserve(gws_bo, false);3121 if (unlikely(ret)) {3122 pr_err("Reserve gws bo failed %d\n", ret);3123 goto bo_reservation_failure;3124 }3125 3126 ret = amdgpu_amdkfd_bo_validate(gws_bo, AMDGPU_GEM_DOMAIN_GWS, true);3127 if (ret) {3128 pr_err("GWS BO validate failed %d\n", ret);3129 goto bo_validation_failure;3130 }3131 /* GWS resource is shared b/t amdgpu and amdkfd3132 * Add process eviction fence to bo so they can3133 * evict each other.3134 */3135 ret = dma_resv_reserve_fences(gws_bo->tbo.base.resv, 1);3136 if (ret)3137 goto reserve_shared_fail;3138 dma_resv_add_fence(gws_bo->tbo.base.resv,3139 &process_info->eviction_fence->base,3140 DMA_RESV_USAGE_BOOKKEEP);3141 amdgpu_bo_unreserve(gws_bo);3142 mutex_unlock(&(*mem)->process_info->lock);3143 3144 return ret;3145 3146reserve_shared_fail:3147bo_validation_failure:3148 amdgpu_bo_unreserve(gws_bo);3149bo_reservation_failure:3150 mutex_unlock(&(*mem)->process_info->lock);3151 amdgpu_sync_free(&(*mem)->sync);3152 remove_kgd_mem_from_kfd_bo_list(*mem, process_info);3153 amdgpu_bo_unref(&gws_bo);3154 mutex_destroy(&(*mem)->lock);3155 kfree(*mem);3156 *mem = NULL;3157 return ret;3158}3159 3160int amdgpu_amdkfd_remove_gws_from_process(void *info, void *mem)3161{3162 int ret;3163 struct amdkfd_process_info *process_info = (struct amdkfd_process_info *)info;3164 struct kgd_mem *kgd_mem = (struct kgd_mem *)mem;3165 struct amdgpu_bo *gws_bo = kgd_mem->bo;3166 3167 /* Remove BO from process's validate list so restore worker won't touch3168 * it anymore3169 */3170 remove_kgd_mem_from_kfd_bo_list(kgd_mem, process_info);3171 3172 ret = amdgpu_bo_reserve(gws_bo, false);3173 if (unlikely(ret)) {3174 pr_err("Reserve gws bo failed %d\n", ret);3175 //TODO add BO back to validate_list?3176 return ret;3177 }3178 amdgpu_amdkfd_remove_eviction_fence(gws_bo,3179 process_info->eviction_fence);3180 amdgpu_bo_unreserve(gws_bo);3181 amdgpu_sync_free(&kgd_mem->sync);3182 amdgpu_bo_unref(&gws_bo);3183 mutex_destroy(&kgd_mem->lock);3184 kfree(mem);3185 return 0;3186}3187 3188/* Returns GPU-specific tiling mode information */3189int amdgpu_amdkfd_get_tile_config(struct amdgpu_device *adev,3190 struct tile_config *config)3191{3192 config->gb_addr_config = adev->gfx.config.gb_addr_config;3193 config->tile_config_ptr = adev->gfx.config.tile_mode_array;3194 config->num_tile_configs =3195 ARRAY_SIZE(adev->gfx.config.tile_mode_array);3196 config->macro_tile_config_ptr =3197 adev->gfx.config.macrotile_mode_array;3198 config->num_macro_tile_configs =3199 ARRAY_SIZE(adev->gfx.config.macrotile_mode_array);3200 3201 /* Those values are not set from GFX9 onwards */3202 config->num_banks = adev->gfx.config.num_banks;3203 config->num_ranks = adev->gfx.config.num_ranks;3204 3205 return 0;3206}3207 3208bool amdgpu_amdkfd_bo_mapped_to_dev(void *drm_priv, struct kgd_mem *mem)3209{3210 struct amdgpu_vm *vm = drm_priv_to_vm(drm_priv);3211 struct kfd_mem_attachment *entry;3212 3213 list_for_each_entry(entry, &mem->attachments, list) {3214 if (entry->is_mapped && entry->bo_va->base.vm == vm)3215 return true;3216 }3217 return false;3218}3219 3220#if defined(CONFIG_DEBUG_FS)3221 3222int kfd_debugfs_kfd_mem_limits(struct seq_file *m, void *data)3223{3224 3225 spin_lock(&kfd_mem_limit.mem_limit_lock);3226 seq_printf(m, "System mem used %lldM out of %lluM\n",3227 (kfd_mem_limit.system_mem_used >> 20),3228 (kfd_mem_limit.max_system_mem_limit >> 20));3229 seq_printf(m, "TTM mem used %lldM out of %lluM\n",3230 (kfd_mem_limit.ttm_mem_used >> 20),3231 (kfd_mem_limit.max_ttm_mem_limit >> 20));3232 spin_unlock(&kfd_mem_limit.mem_limit_lock);3233 3234 return 0;3235}3236 3237#endif3238