brintos / linux-shallow public Read only

1/*2 * Copyright 2008 Advanced Micro Devices, Inc.3 * Copyright 2008 Red Hat Inc.4 * Copyright 2009 Jerome Glisse.5 *6 * Permission is hereby granted, free of charge, to any person obtaining a7 * copy of this software and associated documentation files (the "Software"),8 * to deal in the Software without restriction, including without limitation9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,10 * and/or sell copies of the Software, and to permit persons to whom the11 * Software is furnished to do so, subject to the following conditions:12 *13 * The above copyright notice and this permission notice shall be included in14 * all copies or substantial portions of the Software.15 *16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL19 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR22 * OTHER DEALINGS IN THE SOFTWARE.23 *24 * Authors: Dave Airlie25 *          Alex Deucher26 *          Jerome Glisse27 */28 29#include <linux/dma-fence-array.h>30#include <linux/interval_tree_generic.h>31#include <linux/idr.h>32#include <linux/dma-buf.h>33 34#include <drm/amdgpu_drm.h>35#include <drm/drm_drv.h>36#include <drm/ttm/ttm_tt.h>37#include <drm/drm_exec.h>38#include "amdgpu.h"39#include "amdgpu_trace.h"40#include "amdgpu_amdkfd.h"41#include "amdgpu_gmc.h"42#include "amdgpu_xgmi.h"43#include "amdgpu_dma_buf.h"44#include "amdgpu_res_cursor.h"45#include "kfd_svm.h"46 47/**48 * DOC: GPUVM49 *50 * GPUVM is the MMU functionality provided on the GPU.51 * GPUVM is similar to the legacy GART on older asics, however52 * rather than there being a single global GART table53 * for the entire GPU, there can be multiple GPUVM page tables active54 * at any given time.  The GPUVM page tables can contain a mix55 * VRAM pages and system pages (both memory and MMIO) and system pages56 * can be mapped as snooped (cached system pages) or unsnooped57 * (uncached system pages).58 *59 * Each active GPUVM has an ID associated with it and there is a page table60 * linked with each VMID.  When executing a command buffer,61 * the kernel tells the engine what VMID to use for that command62 * buffer.  VMIDs are allocated dynamically as commands are submitted.63 * The userspace drivers maintain their own address space and the kernel64 * sets up their pages tables accordingly when they submit their65 * command buffers and a VMID is assigned.66 * The hardware supports up to 16 active GPUVMs at any given time.67 *68 * Each GPUVM is represented by a 1-2 or 1-5 level page table, depending69 * on the ASIC family.  GPUVM supports RWX attributes on each page as well70 * as other features such as encryption and caching attributes.71 *72 * VMID 0 is special.  It is the GPUVM used for the kernel driver.  In73 * addition to an aperture managed by a page table, VMID 0 also has74 * several other apertures.  There is an aperture for direct access to VRAM75 * and there is a legacy AGP aperture which just forwards accesses directly76 * to the matching system physical addresses (or IOVAs when an IOMMU is77 * present).  These apertures provide direct access to these memories without78 * incurring the overhead of a page table.  VMID 0 is used by the kernel79 * driver for tasks like memory management.80 *81 * GPU clients (i.e., engines on the GPU) use GPUVM VMIDs to access memory.82 * For user applications, each application can have their own unique GPUVM83 * address space.  The application manages the address space and the kernel84 * driver manages the GPUVM page tables for each process.  If an GPU client85 * accesses an invalid page, it will generate a GPU page fault, similar to86 * accessing an invalid page on a CPU.87 */88 89#define START(node) ((node)->start)90#define LAST(node) ((node)->last)91 92INTERVAL_TREE_DEFINE(struct amdgpu_bo_va_mapping, rb, uint64_t, __subtree_last,93		     START, LAST, static, amdgpu_vm_it)94 95#undef START96#undef LAST97 98/**99 * struct amdgpu_prt_cb - Helper to disable partial resident texture feature from a fence callback100 */101struct amdgpu_prt_cb {102 103	/**104	 * @adev: amdgpu device105	 */106	struct amdgpu_device *adev;107 108	/**109	 * @cb: callback110	 */111	struct dma_fence_cb cb;112};113 114/**115 * struct amdgpu_vm_tlb_seq_struct - Helper to increment the TLB flush sequence116 */117struct amdgpu_vm_tlb_seq_struct {118	/**119	 * @vm: pointer to the amdgpu_vm structure to set the fence sequence on120	 */121	struct amdgpu_vm *vm;122 123	/**124	 * @cb: callback125	 */126	struct dma_fence_cb cb;127};128 129/**130 * amdgpu_vm_set_pasid - manage pasid and vm ptr mapping131 *132 * @adev: amdgpu_device pointer133 * @vm: amdgpu_vm pointer134 * @pasid: the pasid the VM is using on this GPU135 *136 * Set the pasid this VM is using on this GPU, can also be used to remove the137 * pasid by passing in zero.138 *139 */140int amdgpu_vm_set_pasid(struct amdgpu_device *adev, struct amdgpu_vm *vm,141			u32 pasid)142{143	int r;144 145	if (vm->pasid == pasid)146		return 0;147 148	if (vm->pasid) {149		r = xa_err(xa_erase_irq(&adev->vm_manager.pasids, vm->pasid));150		if (r < 0)151			return r;152 153		vm->pasid = 0;154	}155 156	if (pasid) {157		r = xa_err(xa_store_irq(&adev->vm_manager.pasids, pasid, vm,158					GFP_KERNEL));159		if (r < 0)160			return r;161 162		vm->pasid = pasid;163	}164 165 166	return 0;167}168 169/**170 * amdgpu_vm_bo_evicted - vm_bo is evicted171 *172 * @vm_bo: vm_bo which is evicted173 *174 * State for PDs/PTs and per VM BOs which are not at the location they should175 * be.176 */177static void amdgpu_vm_bo_evicted(struct amdgpu_vm_bo_base *vm_bo)178{179	struct amdgpu_vm *vm = vm_bo->vm;180	struct amdgpu_bo *bo = vm_bo->bo;181 182	vm_bo->moved = true;183	spin_lock(&vm_bo->vm->status_lock);184	if (bo->tbo.type == ttm_bo_type_kernel)185		list_move(&vm_bo->vm_status, &vm->evicted);186	else187		list_move_tail(&vm_bo->vm_status, &vm->evicted);188	spin_unlock(&vm_bo->vm->status_lock);189}190/**191 * amdgpu_vm_bo_moved - vm_bo is moved192 *193 * @vm_bo: vm_bo which is moved194 *195 * State for per VM BOs which are moved, but that change is not yet reflected196 * in the page tables.197 */198static void amdgpu_vm_bo_moved(struct amdgpu_vm_bo_base *vm_bo)199{200	spin_lock(&vm_bo->vm->status_lock);201	list_move(&vm_bo->vm_status, &vm_bo->vm->moved);202	spin_unlock(&vm_bo->vm->status_lock);203}204 205/**206 * amdgpu_vm_bo_idle - vm_bo is idle207 *208 * @vm_bo: vm_bo which is now idle209 *210 * State for PDs/PTs and per VM BOs which have gone through the state machine211 * and are now idle.212 */213static void amdgpu_vm_bo_idle(struct amdgpu_vm_bo_base *vm_bo)214{215	spin_lock(&vm_bo->vm->status_lock);216	list_move(&vm_bo->vm_status, &vm_bo->vm->idle);217	spin_unlock(&vm_bo->vm->status_lock);218	vm_bo->moved = false;219}220 221/**222 * amdgpu_vm_bo_invalidated - vm_bo is invalidated223 *224 * @vm_bo: vm_bo which is now invalidated225 *226 * State for normal BOs which are invalidated and that change not yet reflected227 * in the PTs.228 */229static void amdgpu_vm_bo_invalidated(struct amdgpu_vm_bo_base *vm_bo)230{231	spin_lock(&vm_bo->vm->status_lock);232	list_move(&vm_bo->vm_status, &vm_bo->vm->invalidated);233	spin_unlock(&vm_bo->vm->status_lock);234}235 236/**237 * amdgpu_vm_bo_evicted_user - vm_bo is evicted238 *239 * @vm_bo: vm_bo which is evicted240 *241 * State for BOs used by user mode queues which are not at the location they242 * should be.243 */244static void amdgpu_vm_bo_evicted_user(struct amdgpu_vm_bo_base *vm_bo)245{246	vm_bo->moved = true;247	spin_lock(&vm_bo->vm->status_lock);248	list_move(&vm_bo->vm_status, &vm_bo->vm->evicted_user);249	spin_unlock(&vm_bo->vm->status_lock);250}251 252/**253 * amdgpu_vm_bo_relocated - vm_bo is reloacted254 *255 * @vm_bo: vm_bo which is relocated256 *257 * State for PDs/PTs which needs to update their parent PD.258 * For the root PD, just move to idle state.259 */260static void amdgpu_vm_bo_relocated(struct amdgpu_vm_bo_base *vm_bo)261{262	if (vm_bo->bo->parent) {263		spin_lock(&vm_bo->vm->status_lock);264		list_move(&vm_bo->vm_status, &vm_bo->vm->relocated);265		spin_unlock(&vm_bo->vm->status_lock);266	} else {267		amdgpu_vm_bo_idle(vm_bo);268	}269}270 271/**272 * amdgpu_vm_bo_done - vm_bo is done273 *274 * @vm_bo: vm_bo which is now done275 *276 * State for normal BOs which are invalidated and that change has been updated277 * in the PTs.278 */279static void amdgpu_vm_bo_done(struct amdgpu_vm_bo_base *vm_bo)280{281	spin_lock(&vm_bo->vm->status_lock);282	list_move(&vm_bo->vm_status, &vm_bo->vm->done);283	spin_unlock(&vm_bo->vm->status_lock);284}285 286/**287 * amdgpu_vm_bo_reset_state_machine - reset the vm_bo state machine288 * @vm: the VM which state machine to reset289 *290 * Move all vm_bo object in the VM into a state where they will be updated291 * again during validation.292 */293static void amdgpu_vm_bo_reset_state_machine(struct amdgpu_vm *vm)294{295	struct amdgpu_vm_bo_base *vm_bo, *tmp;296 297	spin_lock(&vm->status_lock);298	list_splice_init(&vm->done, &vm->invalidated);299	list_for_each_entry(vm_bo, &vm->invalidated, vm_status)300		vm_bo->moved = true;301	list_for_each_entry_safe(vm_bo, tmp, &vm->idle, vm_status) {302		struct amdgpu_bo *bo = vm_bo->bo;303 304		vm_bo->moved = true;305		if (!bo || bo->tbo.type != ttm_bo_type_kernel)306			list_move(&vm_bo->vm_status, &vm_bo->vm->moved);307		else if (bo->parent)308			list_move(&vm_bo->vm_status, &vm_bo->vm->relocated);309	}310	spin_unlock(&vm->status_lock);311}312 313/**314 * amdgpu_vm_bo_base_init - Adds bo to the list of bos associated with the vm315 *316 * @base: base structure for tracking BO usage in a VM317 * @vm: vm to which bo is to be added318 * @bo: amdgpu buffer object319 *320 * Initialize a bo_va_base structure and add it to the appropriate lists321 *322 */323void amdgpu_vm_bo_base_init(struct amdgpu_vm_bo_base *base,324			    struct amdgpu_vm *vm, struct amdgpu_bo *bo)325{326	base->vm = vm;327	base->bo = bo;328	base->next = NULL;329	INIT_LIST_HEAD(&base->vm_status);330 331	if (!bo)332		return;333	base->next = bo->vm_bo;334	bo->vm_bo = base;335 336	if (!amdgpu_vm_is_bo_always_valid(vm, bo))337		return;338 339	dma_resv_assert_held(vm->root.bo->tbo.base.resv);340 341	ttm_bo_set_bulk_move(&bo->tbo, &vm->lru_bulk_move);342	if (bo->tbo.type == ttm_bo_type_kernel && bo->parent)343		amdgpu_vm_bo_relocated(base);344	else345		amdgpu_vm_bo_idle(base);346 347	if (bo->preferred_domains &348	    amdgpu_mem_type_to_domain(bo->tbo.resource->mem_type))349		return;350 351	/*352	 * we checked all the prerequisites, but it looks like this per vm bo353	 * is currently evicted. add the bo to the evicted list to make sure it354	 * is validated on next vm use to avoid fault.355	 * */356	amdgpu_vm_bo_evicted(base);357}358 359/**360 * amdgpu_vm_lock_pd - lock PD in drm_exec361 *362 * @vm: vm providing the BOs363 * @exec: drm execution context364 * @num_fences: number of extra fences to reserve365 *366 * Lock the VM root PD in the DRM execution context.367 */368int amdgpu_vm_lock_pd(struct amdgpu_vm *vm, struct drm_exec *exec,369		      unsigned int num_fences)370{371	/* We need at least two fences for the VM PD/PT updates */372	return drm_exec_prepare_obj(exec, &vm->root.bo->tbo.base,373				    2 + num_fences);374}375 376/**377 * amdgpu_vm_move_to_lru_tail - move all BOs to the end of LRU378 *379 * @adev: amdgpu device pointer380 * @vm: vm providing the BOs381 *382 * Move all BOs to the end of LRU and remember their positions to put them383 * together.384 */385void amdgpu_vm_move_to_lru_tail(struct amdgpu_device *adev,386				struct amdgpu_vm *vm)387{388	spin_lock(&adev->mman.bdev.lru_lock);389	ttm_lru_bulk_move_tail(&vm->lru_bulk_move);390	spin_unlock(&adev->mman.bdev.lru_lock);391}392 393/* Create scheduler entities for page table updates */394static int amdgpu_vm_init_entities(struct amdgpu_device *adev,395				   struct amdgpu_vm *vm)396{397	int r;398 399	r = drm_sched_entity_init(&vm->immediate, DRM_SCHED_PRIORITY_NORMAL,400				  adev->vm_manager.vm_pte_scheds,401				  adev->vm_manager.vm_pte_num_scheds, NULL);402	if (r)403		goto error;404 405	return drm_sched_entity_init(&vm->delayed, DRM_SCHED_PRIORITY_NORMAL,406				     adev->vm_manager.vm_pte_scheds,407				     adev->vm_manager.vm_pte_num_scheds, NULL);408 409error:410	drm_sched_entity_destroy(&vm->immediate);411	return r;412}413 414/* Destroy the entities for page table updates again */415static void amdgpu_vm_fini_entities(struct amdgpu_vm *vm)416{417	drm_sched_entity_destroy(&vm->immediate);418	drm_sched_entity_destroy(&vm->delayed);419}420 421/**422 * amdgpu_vm_generation - return the page table re-generation counter423 * @adev: the amdgpu_device424 * @vm: optional VM to check, might be NULL425 *426 * Returns a page table re-generation token to allow checking if submissions427 * are still valid to use this VM. The VM parameter might be NULL in which case428 * just the VRAM lost counter will be used.429 */430uint64_t amdgpu_vm_generation(struct amdgpu_device *adev, struct amdgpu_vm *vm)431{432	uint64_t result = (u64)atomic_read(&adev->vram_lost_counter) << 32;433 434	if (!vm)435		return result;436 437	result += lower_32_bits(vm->generation);438	/* Add one if the page tables will be re-generated on next CS */439	if (drm_sched_entity_error(&vm->delayed))440		++result;441 442	return result;443}444 445/**446 * amdgpu_vm_validate - validate evicted BOs tracked in the VM447 *448 * @adev: amdgpu device pointer449 * @vm: vm providing the BOs450 * @ticket: optional reservation ticket used to reserve the VM451 * @validate: callback to do the validation452 * @param: parameter for the validation callback453 *454 * Validate the page table BOs and per-VM BOs on command submission if455 * necessary. If a ticket is given, also try to validate evicted user queue456 * BOs. They must already be reserved with the given ticket.457 *458 * Returns:459 * Validation result.460 */461int amdgpu_vm_validate(struct amdgpu_device *adev, struct amdgpu_vm *vm,462		       struct ww_acquire_ctx *ticket,463		       int (*validate)(void *p, struct amdgpu_bo *bo),464		       void *param)465{466	uint64_t new_vm_generation = amdgpu_vm_generation(adev, vm);467	struct amdgpu_vm_bo_base *bo_base;468	struct amdgpu_bo *bo;469	int r;470 471	if (vm->generation != new_vm_generation) {472		vm->generation = new_vm_generation;473		amdgpu_vm_bo_reset_state_machine(vm);474		amdgpu_vm_fini_entities(vm);475		r = amdgpu_vm_init_entities(adev, vm);476		if (r)477			return r;478	}479 480	spin_lock(&vm->status_lock);481	while (!list_empty(&vm->evicted)) {482		bo_base = list_first_entry(&vm->evicted,483					   struct amdgpu_vm_bo_base,484					   vm_status);485		spin_unlock(&vm->status_lock);486 487		bo = bo_base->bo;488 489		r = validate(param, bo);490		if (r)491			return r;492 493		if (bo->tbo.type != ttm_bo_type_kernel) {494			amdgpu_vm_bo_moved(bo_base);495		} else {496			vm->update_funcs->map_table(to_amdgpu_bo_vm(bo));497			amdgpu_vm_bo_relocated(bo_base);498		}499		spin_lock(&vm->status_lock);500	}501	while (ticket && !list_empty(&vm->evicted_user)) {502		bo_base = list_first_entry(&vm->evicted_user,503					   struct amdgpu_vm_bo_base,504					   vm_status);505		spin_unlock(&vm->status_lock);506 507		bo = bo_base->bo;508 509		if (dma_resv_locking_ctx(bo->tbo.base.resv) != ticket) {510			struct amdgpu_task_info *ti = amdgpu_vm_get_task_info_vm(vm);511 512			pr_warn_ratelimited("Evicted user BO is not reserved\n");513			if (ti) {514				pr_warn_ratelimited("pid %d\n", ti->pid);515				amdgpu_vm_put_task_info(ti);516			}517 518			return -EINVAL;519		}520 521		r = validate(param, bo);522		if (r)523			return r;524 525		amdgpu_vm_bo_invalidated(bo_base);526 527		spin_lock(&vm->status_lock);528	}529	spin_unlock(&vm->status_lock);530 531	amdgpu_vm_eviction_lock(vm);532	vm->evicting = false;533	amdgpu_vm_eviction_unlock(vm);534 535	return 0;536}537 538/**539 * amdgpu_vm_ready - check VM is ready for updates540 *541 * @vm: VM to check542 *543 * Check if all VM PDs/PTs are ready for updates544 *545 * Returns:546 * True if VM is not evicting.547 */548bool amdgpu_vm_ready(struct amdgpu_vm *vm)549{550	bool empty;551	bool ret;552 553	amdgpu_vm_eviction_lock(vm);554	ret = !vm->evicting;555	amdgpu_vm_eviction_unlock(vm);556 557	spin_lock(&vm->status_lock);558	empty = list_empty(&vm->evicted);559	spin_unlock(&vm->status_lock);560 561	return ret && empty;562}563 564/**565 * amdgpu_vm_check_compute_bug - check whether asic has compute vm bug566 *567 * @adev: amdgpu_device pointer568 */569void amdgpu_vm_check_compute_bug(struct amdgpu_device *adev)570{571	const struct amdgpu_ip_block *ip_block;572	bool has_compute_vm_bug;573	struct amdgpu_ring *ring;574	int i;575 576	has_compute_vm_bug = false;577 578	ip_block = amdgpu_device_ip_get_ip_block(adev, AMD_IP_BLOCK_TYPE_GFX);579	if (ip_block) {580		/* Compute has a VM bug for GFX version < 7.581		   Compute has a VM bug for GFX 8 MEC firmware version < 673.*/582		if (ip_block->version->major <= 7)583			has_compute_vm_bug = true;584		else if (ip_block->version->major == 8)585			if (adev->gfx.mec_fw_version < 673)586				has_compute_vm_bug = true;587	}588 589	for (i = 0; i < adev->num_rings; i++) {590		ring = adev->rings[i];591		if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE)592			/* only compute rings */593			ring->has_compute_vm_bug = has_compute_vm_bug;594		else595			ring->has_compute_vm_bug = false;596	}597}598 599/**600 * amdgpu_vm_need_pipeline_sync - Check if pipe sync is needed for job.601 *602 * @ring: ring on which the job will be submitted603 * @job: job to submit604 *605 * Returns:606 * True if sync is needed.607 */608bool amdgpu_vm_need_pipeline_sync(struct amdgpu_ring *ring,609				  struct amdgpu_job *job)610{611	struct amdgpu_device *adev = ring->adev;612	unsigned vmhub = ring->vm_hub;613	struct amdgpu_vmid_mgr *id_mgr = &adev->vm_manager.id_mgr[vmhub];614 615	if (job->vmid == 0)616		return false;617 618	if (job->vm_needs_flush || ring->has_compute_vm_bug)619		return true;620 621	if (ring->funcs->emit_gds_switch && job->gds_switch_needed)622		return true;623 624	if (amdgpu_vmid_had_gpu_reset(adev, &id_mgr->ids[job->vmid]))625		return true;626 627	return false;628}629 630/**631 * amdgpu_vm_flush - hardware flush the vm632 *633 * @ring: ring to use for flush634 * @job:  related job635 * @need_pipe_sync: is pipe sync needed636 *637 * Emit a VM flush when it is necessary.638 *639 * Returns:640 * 0 on success, errno otherwise.641 */642int amdgpu_vm_flush(struct amdgpu_ring *ring, struct amdgpu_job *job,643		    bool need_pipe_sync)644{645	struct amdgpu_device *adev = ring->adev;646	unsigned vmhub = ring->vm_hub;647	struct amdgpu_vmid_mgr *id_mgr = &adev->vm_manager.id_mgr[vmhub];648	struct amdgpu_vmid *id = &id_mgr->ids[job->vmid];649	bool spm_update_needed = job->spm_update_needed;650	bool gds_switch_needed = ring->funcs->emit_gds_switch &&651		job->gds_switch_needed;652	bool vm_flush_needed = job->vm_needs_flush;653	struct dma_fence *fence = NULL;654	bool pasid_mapping_needed = false;655	unsigned int patch;656	int r;657 658	if (amdgpu_vmid_had_gpu_reset(adev, id)) {659		gds_switch_needed = true;660		vm_flush_needed = true;661		pasid_mapping_needed = true;662		spm_update_needed = true;663	}664 665	mutex_lock(&id_mgr->lock);666	if (id->pasid != job->pasid || !id->pasid_mapping ||667	    !dma_fence_is_signaled(id->pasid_mapping))668		pasid_mapping_needed = true;669	mutex_unlock(&id_mgr->lock);670 671	gds_switch_needed &= !!ring->funcs->emit_gds_switch;672	vm_flush_needed &= !!ring->funcs->emit_vm_flush  &&673			job->vm_pd_addr != AMDGPU_BO_INVALID_OFFSET;674	pasid_mapping_needed &= adev->gmc.gmc_funcs->emit_pasid_mapping &&675		ring->funcs->emit_wreg;676 677	if (adev->gfx.enable_cleaner_shader &&678	    ring->funcs->emit_cleaner_shader &&679	    job->enforce_isolation)680		ring->funcs->emit_cleaner_shader(ring);681 682	if (!vm_flush_needed && !gds_switch_needed && !need_pipe_sync)683		return 0;684 685	amdgpu_ring_ib_begin(ring);686	if (ring->funcs->init_cond_exec)687		patch = amdgpu_ring_init_cond_exec(ring,688						   ring->cond_exe_gpu_addr);689 690	if (need_pipe_sync)691		amdgpu_ring_emit_pipeline_sync(ring);692 693	if (vm_flush_needed) {694		trace_amdgpu_vm_flush(ring, job->vmid, job->vm_pd_addr);695		amdgpu_ring_emit_vm_flush(ring, job->vmid, job->vm_pd_addr);696	}697 698	if (pasid_mapping_needed)699		amdgpu_gmc_emit_pasid_mapping(ring, job->vmid, job->pasid);700 701	if (spm_update_needed && adev->gfx.rlc.funcs->update_spm_vmid)702		adev->gfx.rlc.funcs->update_spm_vmid(adev, ring, job->vmid);703 704	if (!ring->is_mes_queue && ring->funcs->emit_gds_switch &&705	    gds_switch_needed) {706		amdgpu_ring_emit_gds_switch(ring, job->vmid, job->gds_base,707					    job->gds_size, job->gws_base,708					    job->gws_size, job->oa_base,709					    job->oa_size);710	}711 712	if (vm_flush_needed || pasid_mapping_needed) {713		r = amdgpu_fence_emit(ring, &fence, NULL, 0);714		if (r)715			return r;716	}717 718	if (vm_flush_needed) {719		mutex_lock(&id_mgr->lock);720		dma_fence_put(id->last_flush);721		id->last_flush = dma_fence_get(fence);722		id->current_gpu_reset_count =723			atomic_read(&adev->gpu_reset_counter);724		mutex_unlock(&id_mgr->lock);725	}726 727	if (pasid_mapping_needed) {728		mutex_lock(&id_mgr->lock);729		id->pasid = job->pasid;730		dma_fence_put(id->pasid_mapping);731		id->pasid_mapping = dma_fence_get(fence);732		mutex_unlock(&id_mgr->lock);733	}734	dma_fence_put(fence);735 736	amdgpu_ring_patch_cond_exec(ring, patch);737 738	/* the double SWITCH_BUFFER here *cannot* be skipped by COND_EXEC */739	if (ring->funcs->emit_switch_buffer) {740		amdgpu_ring_emit_switch_buffer(ring);741		amdgpu_ring_emit_switch_buffer(ring);742	}743 744	amdgpu_ring_ib_end(ring);745	return 0;746}747 748/**749 * amdgpu_vm_bo_find - find the bo_va for a specific vm & bo750 *751 * @vm: requested vm752 * @bo: requested buffer object753 *754 * Find @bo inside the requested vm.755 * Search inside the @bos vm list for the requested vm756 * Returns the found bo_va or NULL if none is found757 *758 * Object has to be reserved!759 *760 * Returns:761 * Found bo_va or NULL.762 */763struct amdgpu_bo_va *amdgpu_vm_bo_find(struct amdgpu_vm *vm,764				       struct amdgpu_bo *bo)765{766	struct amdgpu_vm_bo_base *base;767 768	for (base = bo->vm_bo; base; base = base->next) {769		if (base->vm != vm)770			continue;771 772		return container_of(base, struct amdgpu_bo_va, base);773	}774	return NULL;775}776 777/**778 * amdgpu_vm_map_gart - Resolve gart mapping of addr779 *780 * @pages_addr: optional DMA address to use for lookup781 * @addr: the unmapped addr782 *783 * Look up the physical address of the page that the pte resolves784 * to.785 *786 * Returns:787 * The pointer for the page table entry.788 */789uint64_t amdgpu_vm_map_gart(const dma_addr_t *pages_addr, uint64_t addr)790{791	uint64_t result;792 793	/* page table offset */794	result = pages_addr[addr >> PAGE_SHIFT];795 796	/* in case cpu page size != gpu page size*/797	result |= addr & (~PAGE_MASK);798 799	result &= 0xFFFFFFFFFFFFF000ULL;800 801	return result;802}803 804/**805 * amdgpu_vm_update_pdes - make sure that all directories are valid806 *807 * @adev: amdgpu_device pointer808 * @vm: requested vm809 * @immediate: submit immediately to the paging queue810 *811 * Makes sure all directories are up to date.812 *813 * Returns:814 * 0 for success, error for failure.815 */816int amdgpu_vm_update_pdes(struct amdgpu_device *adev,817			  struct amdgpu_vm *vm, bool immediate)818{819	struct amdgpu_vm_update_params params;820	struct amdgpu_vm_bo_base *entry;821	bool flush_tlb_needed = false;822	LIST_HEAD(relocated);823	int r, idx;824 825	spin_lock(&vm->status_lock);826	list_splice_init(&vm->relocated, &relocated);827	spin_unlock(&vm->status_lock);828 829	if (list_empty(&relocated))830		return 0;831 832	if (!drm_dev_enter(adev_to_drm(adev), &idx))833		return -ENODEV;834 835	memset(&params, 0, sizeof(params));836	params.adev = adev;837	params.vm = vm;838	params.immediate = immediate;839 840	r = vm->update_funcs->prepare(&params, NULL);841	if (r)842		goto error;843 844	list_for_each_entry(entry, &relocated, vm_status) {845		/* vm_flush_needed after updating moved PDEs */846		flush_tlb_needed |= entry->moved;847 848		r = amdgpu_vm_pde_update(&params, entry);849		if (r)850			goto error;851	}852 853	r = vm->update_funcs->commit(&params, &vm->last_update);854	if (r)855		goto error;856 857	if (flush_tlb_needed)858		atomic64_inc(&vm->tlb_seq);859 860	while (!list_empty(&relocated)) {861		entry = list_first_entry(&relocated, struct amdgpu_vm_bo_base,862					 vm_status);863		amdgpu_vm_bo_idle(entry);864	}865 866error:867	drm_dev_exit(idx);868	return r;869}870 871/**872 * amdgpu_vm_tlb_seq_cb - make sure to increment tlb sequence873 * @fence: unused874 * @cb: the callback structure875 *876 * Increments the tlb sequence to make sure that future CS execute a VM flush.877 */878static void amdgpu_vm_tlb_seq_cb(struct dma_fence *fence,879				 struct dma_fence_cb *cb)880{881	struct amdgpu_vm_tlb_seq_struct *tlb_cb;882 883	tlb_cb = container_of(cb, typeof(*tlb_cb), cb);884	atomic64_inc(&tlb_cb->vm->tlb_seq);885	kfree(tlb_cb);886}887 888/**889 * amdgpu_vm_tlb_flush - prepare TLB flush890 *891 * @params: parameters for update892 * @fence: input fence to sync TLB flush with893 * @tlb_cb: the callback structure894 *895 * Increments the tlb sequence to make sure that future CS execute a VM flush.896 */897static void898amdgpu_vm_tlb_flush(struct amdgpu_vm_update_params *params,899		    struct dma_fence **fence,900		    struct amdgpu_vm_tlb_seq_struct *tlb_cb)901{902	struct amdgpu_vm *vm = params->vm;903 904	tlb_cb->vm = vm;905	if (!fence || !*fence) {906		amdgpu_vm_tlb_seq_cb(NULL, &tlb_cb->cb);907		return;908	}909 910	if (!dma_fence_add_callback(*fence, &tlb_cb->cb,911				    amdgpu_vm_tlb_seq_cb)) {912		dma_fence_put(vm->last_tlb_flush);913		vm->last_tlb_flush = dma_fence_get(*fence);914	} else {915		amdgpu_vm_tlb_seq_cb(NULL, &tlb_cb->cb);916	}917 918	/* Prepare a TLB flush fence to be attached to PTs */919	if (!params->unlocked && vm->is_compute_context) {920		amdgpu_vm_tlb_fence_create(params->adev, vm, fence);921 922		/* Makes sure no PD/PT is freed before the flush */923		dma_resv_add_fence(vm->root.bo->tbo.base.resv, *fence,924				   DMA_RESV_USAGE_BOOKKEEP);925	}926}927 928/**929 * amdgpu_vm_update_range - update a range in the vm page table930 *931 * @adev: amdgpu_device pointer to use for commands932 * @vm: the VM to update the range933 * @immediate: immediate submission in a page fault934 * @unlocked: unlocked invalidation during MM callback935 * @flush_tlb: trigger tlb invalidation after update completed936 * @allow_override: change MTYPE for local NUMA nodes937 * @sync: fences we need to sync to938 * @start: start of mapped range939 * @last: last mapped entry940 * @flags: flags for the entries941 * @offset: offset into nodes and pages_addr942 * @vram_base: base for vram mappings943 * @res: ttm_resource to map944 * @pages_addr: DMA addresses to use for mapping945 * @fence: optional resulting fence946 *947 * Fill in the page table entries between @start and @last.948 *949 * Returns:950 * 0 for success, negative erro code for failure.951 */952int amdgpu_vm_update_range(struct amdgpu_device *adev, struct amdgpu_vm *vm,953			   bool immediate, bool unlocked, bool flush_tlb,954			   bool allow_override, struct amdgpu_sync *sync,955			   uint64_t start, uint64_t last, uint64_t flags,956			   uint64_t offset, uint64_t vram_base,957			   struct ttm_resource *res, dma_addr_t *pages_addr,958			   struct dma_fence **fence)959{960	struct amdgpu_vm_tlb_seq_struct *tlb_cb;961	struct amdgpu_vm_update_params params;962	struct amdgpu_res_cursor cursor;963	int r, idx;964 965	if (!drm_dev_enter(adev_to_drm(adev), &idx))966		return -ENODEV;967 968	tlb_cb = kmalloc(sizeof(*tlb_cb), GFP_KERNEL);969	if (!tlb_cb) {970		drm_dev_exit(idx);971		return -ENOMEM;972	}973 974	/* Vega20+XGMI where PTEs get inadvertently cached in L2 texture cache,975	 * heavy-weight flush TLB unconditionally.976	 */977	flush_tlb |= adev->gmc.xgmi.num_physical_nodes &&978		     amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 0);979 980	/*981	 * On GFX8 and older any 8 PTE block with a valid bit set enters the TLB982	 */983	flush_tlb |= amdgpu_ip_version(adev, GC_HWIP, 0) < IP_VERSION(9, 0, 0);984 985	memset(&params, 0, sizeof(params));986	params.adev = adev;987	params.vm = vm;988	params.immediate = immediate;989	params.pages_addr = pages_addr;990	params.unlocked = unlocked;991	params.needs_flush = flush_tlb;992	params.allow_override = allow_override;993	INIT_LIST_HEAD(&params.tlb_flush_waitlist);994 995	amdgpu_vm_eviction_lock(vm);996	if (vm->evicting) {997		r = -EBUSY;998		goto error_free;999	}1000 1001	if (!unlocked && !dma_fence_is_signaled(vm->last_unlocked)) {1002		struct dma_fence *tmp = dma_fence_get_stub();1003 1004		amdgpu_bo_fence(vm->root.bo, vm->last_unlocked, true);1005		swap(vm->last_unlocked, tmp);1006		dma_fence_put(tmp);1007	}1008 1009	r = vm->update_funcs->prepare(&params, sync);1010	if (r)1011		goto error_free;1012 1013	amdgpu_res_first(pages_addr ? NULL : res, offset,1014			 (last - start + 1) * AMDGPU_GPU_PAGE_SIZE, &cursor);1015	while (cursor.remaining) {1016		uint64_t tmp, num_entries, addr;1017 1018		num_entries = cursor.size >> AMDGPU_GPU_PAGE_SHIFT;1019		if (pages_addr) {1020			bool contiguous = true;1021 1022			if (num_entries > AMDGPU_GPU_PAGES_IN_CPU_PAGE) {1023				uint64_t pfn = cursor.start >> PAGE_SHIFT;1024				uint64_t count;1025 1026				contiguous = pages_addr[pfn + 1] ==1027					pages_addr[pfn] + PAGE_SIZE;1028 1029				tmp = num_entries /1030					AMDGPU_GPU_PAGES_IN_CPU_PAGE;1031				for (count = 2; count < tmp; ++count) {1032					uint64_t idx = pfn + count;1033 1034					if (contiguous != (pages_addr[idx] ==1035					    pages_addr[idx - 1] + PAGE_SIZE))1036						break;1037				}1038				if (!contiguous)1039					count--;1040				num_entries = count *1041					AMDGPU_GPU_PAGES_IN_CPU_PAGE;1042			}1043 1044			if (!contiguous) {1045				addr = cursor.start;1046				params.pages_addr = pages_addr;1047			} else {1048				addr = pages_addr[cursor.start >> PAGE_SHIFT];1049				params.pages_addr = NULL;1050			}1051 1052		} else if (flags & (AMDGPU_PTE_VALID | AMDGPU_PTE_PRT_FLAG(adev))) {1053			addr = vram_base + cursor.start;1054		} else {1055			addr = 0;1056		}1057 1058		tmp = start + num_entries;1059		r = amdgpu_vm_ptes_update(&params, start, tmp, addr, flags);1060		if (r)1061			goto error_free;1062 1063		amdgpu_res_next(&cursor, num_entries * AMDGPU_GPU_PAGE_SIZE);1064		start = tmp;1065	}1066 1067	r = vm->update_funcs->commit(&params, fence);1068	if (r)1069		goto error_free;1070 1071	if (params.needs_flush) {1072		amdgpu_vm_tlb_flush(&params, fence, tlb_cb);1073		tlb_cb = NULL;1074	}1075 1076	amdgpu_vm_pt_free_list(adev, &params);1077 1078error_free:1079	kfree(tlb_cb);1080	amdgpu_vm_eviction_unlock(vm);1081	drm_dev_exit(idx);1082	return r;1083}1084 1085static void amdgpu_vm_bo_get_memory(struct amdgpu_bo_va *bo_va,1086				    struct amdgpu_mem_stats *stats)1087{1088	struct amdgpu_vm *vm = bo_va->base.vm;1089	struct amdgpu_bo *bo = bo_va->base.bo;1090 1091	if (!bo)1092		return;1093 1094	/*1095	 * For now ignore BOs which are currently locked and potentially1096	 * changing their location.1097	 */1098	if (!amdgpu_vm_is_bo_always_valid(vm, bo) &&1099	    !dma_resv_trylock(bo->tbo.base.resv))1100		return;1101 1102	amdgpu_bo_get_memory(bo, stats);1103	if (!amdgpu_vm_is_bo_always_valid(vm, bo))1104		dma_resv_unlock(bo->tbo.base.resv);1105}1106 1107void amdgpu_vm_get_memory(struct amdgpu_vm *vm,1108			  struct amdgpu_mem_stats *stats)1109{1110	struct amdgpu_bo_va *bo_va, *tmp;1111 1112	spin_lock(&vm->status_lock);1113	list_for_each_entry_safe(bo_va, tmp, &vm->idle, base.vm_status)1114		amdgpu_vm_bo_get_memory(bo_va, stats);1115 1116	list_for_each_entry_safe(bo_va, tmp, &vm->evicted, base.vm_status)1117		amdgpu_vm_bo_get_memory(bo_va, stats);1118 1119	list_for_each_entry_safe(bo_va, tmp, &vm->relocated, base.vm_status)1120		amdgpu_vm_bo_get_memory(bo_va, stats);1121 1122	list_for_each_entry_safe(bo_va, tmp, &vm->moved, base.vm_status)1123		amdgpu_vm_bo_get_memory(bo_va, stats);1124 1125	list_for_each_entry_safe(bo_va, tmp, &vm->invalidated, base.vm_status)1126		amdgpu_vm_bo_get_memory(bo_va, stats);1127 1128	list_for_each_entry_safe(bo_va, tmp, &vm->done, base.vm_status)1129		amdgpu_vm_bo_get_memory(bo_va, stats);1130	spin_unlock(&vm->status_lock);1131}1132 1133/**1134 * amdgpu_vm_bo_update - update all BO mappings in the vm page table1135 *1136 * @adev: amdgpu_device pointer1137 * @bo_va: requested BO and VM object1138 * @clear: if true clear the entries1139 *1140 * Fill in the page table entries for @bo_va.1141 *1142 * Returns:1143 * 0 for success, -EINVAL for failure.1144 */1145int amdgpu_vm_bo_update(struct amdgpu_device *adev, struct amdgpu_bo_va *bo_va,1146			bool clear)1147{1148	struct amdgpu_bo *bo = bo_va->base.bo;1149	struct amdgpu_vm *vm = bo_va->base.vm;1150	struct amdgpu_bo_va_mapping *mapping;1151	struct dma_fence **last_update;1152	dma_addr_t *pages_addr = NULL;1153	struct ttm_resource *mem;1154	struct amdgpu_sync sync;1155	bool flush_tlb = clear;1156	uint64_t vram_base;1157	uint64_t flags;1158	bool uncached;1159	int r;1160 1161	amdgpu_sync_create(&sync);1162	if (clear || !bo) {1163		mem = NULL;1164 1165		/* Implicitly sync to command submissions in the same VM before1166		 * unmapping.1167		 */1168		r = amdgpu_sync_resv(adev, &sync, vm->root.bo->tbo.base.resv,1169				     AMDGPU_SYNC_EQ_OWNER, vm);1170		if (r)1171			goto error_free;1172		if (bo) {1173			r = amdgpu_sync_kfd(&sync, bo->tbo.base.resv);1174			if (r)1175				goto error_free;1176		}1177 1178	} else {1179		struct drm_gem_object *obj = &bo->tbo.base;1180 1181		if (obj->import_attach && bo_va->is_xgmi) {1182			struct dma_buf *dma_buf = obj->import_attach->dmabuf;1183			struct drm_gem_object *gobj = dma_buf->priv;1184			struct amdgpu_bo *abo = gem_to_amdgpu_bo(gobj);1185 1186			if (abo->tbo.resource &&1187			    abo->tbo.resource->mem_type == TTM_PL_VRAM)1188				bo = gem_to_amdgpu_bo(gobj);1189		}1190		mem = bo->tbo.resource;1191		if (mem && (mem->mem_type == TTM_PL_TT ||1192			    mem->mem_type == AMDGPU_PL_PREEMPT))1193			pages_addr = bo->tbo.ttm->dma_address;1194 1195		/* Implicitly sync to moving fences before mapping anything */1196		r = amdgpu_sync_resv(adev, &sync, bo->tbo.base.resv,1197				     AMDGPU_SYNC_EXPLICIT, vm);1198		if (r)1199			goto error_free;1200	}1201 1202	if (bo) {1203		struct amdgpu_device *bo_adev;1204 1205		flags = amdgpu_ttm_tt_pte_flags(adev, bo->tbo.ttm, mem);1206 1207		if (amdgpu_bo_encrypted(bo))1208			flags |= AMDGPU_PTE_TMZ;1209 1210		bo_adev = amdgpu_ttm_adev(bo->tbo.bdev);1211		vram_base = bo_adev->vm_manager.vram_base_offset;1212		uncached = (bo->flags & AMDGPU_GEM_CREATE_UNCACHED) != 0;1213	} else {1214		flags = 0x0;1215		vram_base = 0;1216		uncached = false;1217	}1218 1219	if (clear || amdgpu_vm_is_bo_always_valid(vm, bo))1220		last_update = &vm->last_update;1221	else1222		last_update = &bo_va->last_pt_update;1223 1224	if (!clear && bo_va->base.moved) {1225		flush_tlb = true;1226		list_splice_init(&bo_va->valids, &bo_va->invalids);1227 1228	} else if (bo_va->cleared != clear) {1229		list_splice_init(&bo_va->valids, &bo_va->invalids);1230	}1231 1232	list_for_each_entry(mapping, &bo_va->invalids, list) {1233		uint64_t update_flags = flags;1234 1235		/* normally,bo_va->flags only contians READABLE and WIRTEABLE bit go here1236		 * but in case of something, we filter the flags in first place1237		 */1238		if (!(mapping->flags & AMDGPU_PTE_READABLE))1239			update_flags &= ~AMDGPU_PTE_READABLE;1240		if (!(mapping->flags & AMDGPU_PTE_WRITEABLE))1241			update_flags &= ~AMDGPU_PTE_WRITEABLE;1242 1243		/* Apply ASIC specific mapping flags */1244		amdgpu_gmc_get_vm_pte(adev, mapping, &update_flags);1245 1246		trace_amdgpu_vm_bo_update(mapping);1247 1248		r = amdgpu_vm_update_range(adev, vm, false, false, flush_tlb,1249					   !uncached, &sync, mapping->start,1250					   mapping->last, update_flags,1251					   mapping->offset, vram_base, mem,1252					   pages_addr, last_update);1253		if (r)1254			goto error_free;1255	}1256 1257	/* If the BO is not in its preferred location add it back to1258	 * the evicted list so that it gets validated again on the1259	 * next command submission.1260	 */1261	if (amdgpu_vm_is_bo_always_valid(vm, bo)) {1262		uint32_t mem_type = bo->tbo.resource->mem_type;1263 1264		if (!(bo->preferred_domains &1265		      amdgpu_mem_type_to_domain(mem_type)))1266			amdgpu_vm_bo_evicted(&bo_va->base);1267		else1268			amdgpu_vm_bo_idle(&bo_va->base);1269	} else {1270		amdgpu_vm_bo_done(&bo_va->base);1271	}1272 1273	list_splice_init(&bo_va->invalids, &bo_va->valids);1274	bo_va->cleared = clear;1275	bo_va->base.moved = false;1276 1277	if (trace_amdgpu_vm_bo_mapping_enabled()) {1278		list_for_each_entry(mapping, &bo_va->valids, list)1279			trace_amdgpu_vm_bo_mapping(mapping);1280	}1281 1282error_free:1283	amdgpu_sync_free(&sync);1284	return r;1285}1286 1287/**1288 * amdgpu_vm_update_prt_state - update the global PRT state1289 *1290 * @adev: amdgpu_device pointer1291 */1292static void amdgpu_vm_update_prt_state(struct amdgpu_device *adev)1293{1294	unsigned long flags;1295	bool enable;1296 1297	spin_lock_irqsave(&adev->vm_manager.prt_lock, flags);1298	enable = !!atomic_read(&adev->vm_manager.num_prt_users);1299	adev->gmc.gmc_funcs->set_prt(adev, enable);1300	spin_unlock_irqrestore(&adev->vm_manager.prt_lock, flags);1301}1302 1303/**1304 * amdgpu_vm_prt_get - add a PRT user1305 *1306 * @adev: amdgpu_device pointer1307 */1308static void amdgpu_vm_prt_get(struct amdgpu_device *adev)1309{1310	if (!adev->gmc.gmc_funcs->set_prt)1311		return;1312 1313	if (atomic_inc_return(&adev->vm_manager.num_prt_users) == 1)1314		amdgpu_vm_update_prt_state(adev);1315}1316 1317/**1318 * amdgpu_vm_prt_put - drop a PRT user1319 *1320 * @adev: amdgpu_device pointer1321 */1322static void amdgpu_vm_prt_put(struct amdgpu_device *adev)1323{1324	if (atomic_dec_return(&adev->vm_manager.num_prt_users) == 0)1325		amdgpu_vm_update_prt_state(adev);1326}1327 1328/**1329 * amdgpu_vm_prt_cb - callback for updating the PRT status1330 *1331 * @fence: fence for the callback1332 * @_cb: the callback function1333 */1334static void amdgpu_vm_prt_cb(struct dma_fence *fence, struct dma_fence_cb *_cb)1335{1336	struct amdgpu_prt_cb *cb = container_of(_cb, struct amdgpu_prt_cb, cb);1337 1338	amdgpu_vm_prt_put(cb->adev);1339	kfree(cb);1340}1341 1342/**1343 * amdgpu_vm_add_prt_cb - add callback for updating the PRT status1344 *1345 * @adev: amdgpu_device pointer1346 * @fence: fence for the callback1347 */1348static void amdgpu_vm_add_prt_cb(struct amdgpu_device *adev,1349				 struct dma_fence *fence)1350{1351	struct amdgpu_prt_cb *cb;1352 1353	if (!adev->gmc.gmc_funcs->set_prt)1354		return;1355 1356	cb = kmalloc(sizeof(struct amdgpu_prt_cb), GFP_KERNEL);1357	if (!cb) {1358		/* Last resort when we are OOM */1359		if (fence)1360			dma_fence_wait(fence, false);1361 1362		amdgpu_vm_prt_put(adev);1363	} else {1364		cb->adev = adev;1365		if (!fence || dma_fence_add_callback(fence, &cb->cb,1366						     amdgpu_vm_prt_cb))1367			amdgpu_vm_prt_cb(fence, &cb->cb);1368	}1369}1370 1371/**1372 * amdgpu_vm_free_mapping - free a mapping1373 *1374 * @adev: amdgpu_device pointer1375 * @vm: requested vm1376 * @mapping: mapping to be freed1377 * @fence: fence of the unmap operation1378 *1379 * Free a mapping and make sure we decrease the PRT usage count if applicable.1380 */1381static void amdgpu_vm_free_mapping(struct amdgpu_device *adev,1382				   struct amdgpu_vm *vm,1383				   struct amdgpu_bo_va_mapping *mapping,1384				   struct dma_fence *fence)1385{1386	if (mapping->flags & AMDGPU_PTE_PRT_FLAG(adev))1387		amdgpu_vm_add_prt_cb(adev, fence);1388	kfree(mapping);1389}1390 1391/**1392 * amdgpu_vm_prt_fini - finish all prt mappings1393 *1394 * @adev: amdgpu_device pointer1395 * @vm: requested vm1396 *1397 * Register a cleanup callback to disable PRT support after VM dies.1398 */1399static void amdgpu_vm_prt_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm)1400{1401	struct dma_resv *resv = vm->root.bo->tbo.base.resv;1402	struct dma_resv_iter cursor;1403	struct dma_fence *fence;1404 1405	dma_resv_for_each_fence(&cursor, resv, DMA_RESV_USAGE_BOOKKEEP, fence) {1406		/* Add a callback for each fence in the reservation object */1407		amdgpu_vm_prt_get(adev);1408		amdgpu_vm_add_prt_cb(adev, fence);1409	}1410}1411 1412/**1413 * amdgpu_vm_clear_freed - clear freed BOs in the PT1414 *1415 * @adev: amdgpu_device pointer1416 * @vm: requested vm1417 * @fence: optional resulting fence (unchanged if no work needed to be done1418 * or if an error occurred)1419 *1420 * Make sure all freed BOs are cleared in the PT.1421 * PTs have to be reserved and mutex must be locked!1422 *1423 * Returns:1424 * 0 for success.1425 *1426 */1427int amdgpu_vm_clear_freed(struct amdgpu_device *adev,1428			  struct amdgpu_vm *vm,1429			  struct dma_fence **fence)1430{1431	struct amdgpu_bo_va_mapping *mapping;1432	struct dma_fence *f = NULL;1433	struct amdgpu_sync sync;1434	int r;1435 1436 1437	/*1438	 * Implicitly sync to command submissions in the same VM before1439	 * unmapping.1440	 */1441	amdgpu_sync_create(&sync);1442	r = amdgpu_sync_resv(adev, &sync, vm->root.bo->tbo.base.resv,1443			     AMDGPU_SYNC_EQ_OWNER, vm);1444	if (r)1445		goto error_free;1446 1447	while (!list_empty(&vm->freed)) {1448		mapping = list_first_entry(&vm->freed,1449			struct amdgpu_bo_va_mapping, list);1450		list_del(&mapping->list);1451 1452		r = amdgpu_vm_update_range(adev, vm, false, false, true, false,1453					   &sync, mapping->start, mapping->last,1454					   0, 0, 0, NULL, NULL, &f);1455		amdgpu_vm_free_mapping(adev, vm, mapping, f);1456		if (r) {1457			dma_fence_put(f);1458			goto error_free;1459		}1460	}1461 1462	if (fence && f) {1463		dma_fence_put(*fence);1464		*fence = f;1465	} else {1466		dma_fence_put(f);1467	}1468 1469error_free:1470	amdgpu_sync_free(&sync);1471	return r;1472 1473}1474 1475/**1476 * amdgpu_vm_handle_moved - handle moved BOs in the PT1477 *1478 * @adev: amdgpu_device pointer1479 * @vm: requested vm1480 * @ticket: optional reservation ticket used to reserve the VM1481 *1482 * Make sure all BOs which are moved are updated in the PTs.1483 *1484 * Returns:1485 * 0 for success.1486 *1487 * PTs have to be reserved!1488 */1489int amdgpu_vm_handle_moved(struct amdgpu_device *adev,1490			   struct amdgpu_vm *vm,1491			   struct ww_acquire_ctx *ticket)1492{1493	struct amdgpu_bo_va *bo_va;1494	struct dma_resv *resv;1495	bool clear, unlock;1496	int r;1497 1498	spin_lock(&vm->status_lock);1499	while (!list_empty(&vm->moved)) {1500		bo_va = list_first_entry(&vm->moved, struct amdgpu_bo_va,1501					 base.vm_status);1502		spin_unlock(&vm->status_lock);1503 1504		/* Per VM BOs never need to bo cleared in the page tables */1505		r = amdgpu_vm_bo_update(adev, bo_va, false);1506		if (r)1507			return r;1508		spin_lock(&vm->status_lock);1509	}1510 1511	while (!list_empty(&vm->invalidated)) {1512		bo_va = list_first_entry(&vm->invalidated, struct amdgpu_bo_va,1513					 base.vm_status);1514		resv = bo_va->base.bo->tbo.base.resv;1515		spin_unlock(&vm->status_lock);1516 1517		/* Try to reserve the BO to avoid clearing its ptes */1518		if (!adev->debug_vm && dma_resv_trylock(resv)) {1519			clear = false;1520			unlock = true;1521		/* The caller is already holding the reservation lock */1522		} else if (ticket && dma_resv_locking_ctx(resv) == ticket) {1523			clear = false;1524			unlock = false;1525		/* Somebody else is using the BO right now */1526		} else {1527			clear = true;1528			unlock = false;1529		}1530 1531		r = amdgpu_vm_bo_update(adev, bo_va, clear);1532 1533		if (unlock)1534			dma_resv_unlock(resv);1535		if (r)1536			return r;1537 1538		/* Remember evicted DMABuf imports in compute VMs for later1539		 * validation1540		 */1541		if (vm->is_compute_context &&1542		    bo_va->base.bo->tbo.base.import_attach &&1543		    (!bo_va->base.bo->tbo.resource ||1544		     bo_va->base.bo->tbo.resource->mem_type == TTM_PL_SYSTEM))1545			amdgpu_vm_bo_evicted_user(&bo_va->base);1546 1547		spin_lock(&vm->status_lock);1548	}1549	spin_unlock(&vm->status_lock);1550 1551	return 0;1552}1553 1554/**1555 * amdgpu_vm_flush_compute_tlb - Flush TLB on compute VM1556 *1557 * @adev: amdgpu_device pointer1558 * @vm: requested vm1559 * @flush_type: flush type1560 * @xcc_mask: mask of XCCs that belong to the compute partition in need of a TLB flush.1561 *1562 * Flush TLB if needed for a compute VM.1563 *1564 * Returns:1565 * 0 for success.1566 */1567int amdgpu_vm_flush_compute_tlb(struct amdgpu_device *adev,1568				struct amdgpu_vm *vm,1569				uint32_t flush_type,1570				uint32_t xcc_mask)1571{1572	uint64_t tlb_seq = amdgpu_vm_tlb_seq(vm);1573	bool all_hub = false;1574	int xcc = 0, r = 0;1575 1576	WARN_ON_ONCE(!vm->is_compute_context);1577 1578	/*1579	 * It can be that we race and lose here, but that is extremely unlikely1580	 * and the worst thing which could happen is that we flush the changes1581	 * into the TLB once more which is harmless.1582	 */1583	if (atomic64_xchg(&vm->kfd_last_flushed_seq, tlb_seq) == tlb_seq)1584		return 0;1585 1586	if (adev->family == AMDGPU_FAMILY_AI ||1587	    adev->family == AMDGPU_FAMILY_RV)1588		all_hub = true;1589 1590	for_each_inst(xcc, xcc_mask) {1591		r = amdgpu_gmc_flush_gpu_tlb_pasid(adev, vm->pasid, flush_type,1592						   all_hub, xcc);1593		if (r)1594			break;1595	}1596	return r;1597}1598 1599/**1600 * amdgpu_vm_bo_add - add a bo to a specific vm1601 *1602 * @adev: amdgpu_device pointer1603 * @vm: requested vm1604 * @bo: amdgpu buffer object1605 *1606 * Add @bo into the requested vm.1607 * Add @bo to the list of bos associated with the vm1608 *1609 * Returns:1610 * Newly added bo_va or NULL for failure1611 *1612 * Object has to be reserved!1613 */1614struct amdgpu_bo_va *amdgpu_vm_bo_add(struct amdgpu_device *adev,1615				      struct amdgpu_vm *vm,1616				      struct amdgpu_bo *bo)1617{1618	struct amdgpu_bo_va *bo_va;1619 1620	bo_va = kzalloc(sizeof(struct amdgpu_bo_va), GFP_KERNEL);1621	if (bo_va == NULL) {1622		return NULL;1623	}1624	amdgpu_vm_bo_base_init(&bo_va->base, vm, bo);1625 1626	bo_va->ref_count = 1;1627	bo_va->last_pt_update = dma_fence_get_stub();1628	INIT_LIST_HEAD(&bo_va->valids);1629	INIT_LIST_HEAD(&bo_va->invalids);1630 1631	if (!bo)1632		return bo_va;1633 1634	dma_resv_assert_held(bo->tbo.base.resv);1635	if (amdgpu_dmabuf_is_xgmi_accessible(adev, bo)) {1636		bo_va->is_xgmi = true;1637		/* Power up XGMI if it can be potentially used */1638		amdgpu_xgmi_set_pstate(adev, AMDGPU_XGMI_PSTATE_MAX_VEGA20);1639	}1640 1641	return bo_va;1642}1643 1644 1645/**1646 * amdgpu_vm_bo_insert_map - insert a new mapping1647 *1648 * @adev: amdgpu_device pointer1649 * @bo_va: bo_va to store the address1650 * @mapping: the mapping to insert1651 *1652 * Insert a new mapping into all structures.1653 */1654static void amdgpu_vm_bo_insert_map(struct amdgpu_device *adev,1655				    struct amdgpu_bo_va *bo_va,1656				    struct amdgpu_bo_va_mapping *mapping)1657{1658	struct amdgpu_vm *vm = bo_va->base.vm;1659	struct amdgpu_bo *bo = bo_va->base.bo;1660 1661	mapping->bo_va = bo_va;1662	list_add(&mapping->list, &bo_va->invalids);1663	amdgpu_vm_it_insert(mapping, &vm->va);1664 1665	if (mapping->flags & AMDGPU_PTE_PRT_FLAG(adev))1666		amdgpu_vm_prt_get(adev);1667 1668	if (amdgpu_vm_is_bo_always_valid(vm, bo) && !bo_va->base.moved)1669		amdgpu_vm_bo_moved(&bo_va->base);1670 1671	trace_amdgpu_vm_bo_map(bo_va, mapping);1672}1673 1674/* Validate operation parameters to prevent potential abuse */1675static int amdgpu_vm_verify_parameters(struct amdgpu_device *adev,1676					  struct amdgpu_bo *bo,1677					  uint64_t saddr,1678					  uint64_t offset,1679					  uint64_t size)1680{1681	uint64_t tmp, lpfn;1682 1683	if (saddr & AMDGPU_GPU_PAGE_MASK1684	    || offset & AMDGPU_GPU_PAGE_MASK1685	    || size & AMDGPU_GPU_PAGE_MASK)1686		return -EINVAL;1687 1688	if (check_add_overflow(saddr, size, &tmp)1689	    || check_add_overflow(offset, size, &tmp)1690	    || size == 0 /* which also leads to end < begin */)1691		return -EINVAL;1692 1693	/* make sure object fit at this offset */1694	if (bo && offset + size > amdgpu_bo_size(bo))1695		return -EINVAL;1696 1697	/* Ensure last pfn not exceed max_pfn */1698	lpfn = (saddr + size - 1) >> AMDGPU_GPU_PAGE_SHIFT;1699	if (lpfn >= adev->vm_manager.max_pfn)1700		return -EINVAL;1701 1702	return 0;1703}1704 1705/**1706 * amdgpu_vm_bo_map - map bo inside a vm1707 *1708 * @adev: amdgpu_device pointer1709 * @bo_va: bo_va to store the address1710 * @saddr: where to map the BO1711 * @offset: requested offset in the BO1712 * @size: BO size in bytes1713 * @flags: attributes of pages (read/write/valid/etc.)1714 *1715 * Add a mapping of the BO at the specefied addr into the VM.1716 *1717 * Returns:1718 * 0 for success, error for failure.1719 *1720 * Object has to be reserved and unreserved outside!1721 */1722int amdgpu_vm_bo_map(struct amdgpu_device *adev,1723		     struct amdgpu_bo_va *bo_va,1724		     uint64_t saddr, uint64_t offset,1725		     uint64_t size, uint64_t flags)1726{1727	struct amdgpu_bo_va_mapping *mapping, *tmp;1728	struct amdgpu_bo *bo = bo_va->base.bo;1729	struct amdgpu_vm *vm = bo_va->base.vm;1730	uint64_t eaddr;1731	int r;1732 1733	r = amdgpu_vm_verify_parameters(adev, bo, saddr, offset, size);1734	if (r)1735		return r;1736 1737	saddr /= AMDGPU_GPU_PAGE_SIZE;1738	eaddr = saddr + (size - 1) / AMDGPU_GPU_PAGE_SIZE;1739 1740	tmp = amdgpu_vm_it_iter_first(&vm->va, saddr, eaddr);1741	if (tmp) {1742		/* bo and tmp overlap, invalid addr */1743		dev_err(adev->dev, "bo %p va 0x%010Lx-0x%010Lx conflict with "1744			"0x%010Lx-0x%010Lx\n", bo, saddr, eaddr,1745			tmp->start, tmp->last + 1);1746		return -EINVAL;1747	}1748 1749	mapping = kmalloc(sizeof(*mapping), GFP_KERNEL);1750	if (!mapping)1751		return -ENOMEM;1752 1753	mapping->start = saddr;1754	mapping->last = eaddr;1755	mapping->offset = offset;1756	mapping->flags = flags;1757 1758	amdgpu_vm_bo_insert_map(adev, bo_va, mapping);1759 1760	return 0;1761}1762 1763/**1764 * amdgpu_vm_bo_replace_map - map bo inside a vm, replacing existing mappings1765 *1766 * @adev: amdgpu_device pointer1767 * @bo_va: bo_va to store the address1768 * @saddr: where to map the BO1769 * @offset: requested offset in the BO1770 * @size: BO size in bytes1771 * @flags: attributes of pages (read/write/valid/etc.)1772 *1773 * Add a mapping of the BO at the specefied addr into the VM. Replace existing1774 * mappings as we do so.1775 *1776 * Returns:1777 * 0 for success, error for failure.1778 *1779 * Object has to be reserved and unreserved outside!1780 */1781int amdgpu_vm_bo_replace_map(struct amdgpu_device *adev,1782			     struct amdgpu_bo_va *bo_va,1783			     uint64_t saddr, uint64_t offset,1784			     uint64_t size, uint64_t flags)1785{1786	struct amdgpu_bo_va_mapping *mapping;1787	struct amdgpu_bo *bo = bo_va->base.bo;1788	uint64_t eaddr;1789	int r;1790 1791	r = amdgpu_vm_verify_parameters(adev, bo, saddr, offset, size);1792	if (r)1793		return r;1794 1795	/* Allocate all the needed memory */1796	mapping = kmalloc(sizeof(*mapping), GFP_KERNEL);1797	if (!mapping)1798		return -ENOMEM;1799 1800	r = amdgpu_vm_bo_clear_mappings(adev, bo_va->base.vm, saddr, size);1801	if (r) {1802		kfree(mapping);1803		return r;1804	}1805 1806	saddr /= AMDGPU_GPU_PAGE_SIZE;1807	eaddr = saddr + (size - 1) / AMDGPU_GPU_PAGE_SIZE;1808 1809	mapping->start = saddr;1810	mapping->last = eaddr;1811	mapping->offset = offset;1812	mapping->flags = flags;1813 1814	amdgpu_vm_bo_insert_map(adev, bo_va, mapping);1815 1816	return 0;1817}1818 1819/**1820 * amdgpu_vm_bo_unmap - remove bo mapping from vm1821 *1822 * @adev: amdgpu_device pointer1823 * @bo_va: bo_va to remove the address from1824 * @saddr: where to the BO is mapped1825 *1826 * Remove a mapping of the BO at the specefied addr from the VM.1827 *1828 * Returns:1829 * 0 for success, error for failure.1830 *1831 * Object has to be reserved and unreserved outside!1832 */1833int amdgpu_vm_bo_unmap(struct amdgpu_device *adev,1834		       struct amdgpu_bo_va *bo_va,1835		       uint64_t saddr)1836{1837	struct amdgpu_bo_va_mapping *mapping;1838	struct amdgpu_vm *vm = bo_va->base.vm;1839	bool valid = true;1840 1841	saddr /= AMDGPU_GPU_PAGE_SIZE;1842 1843	list_for_each_entry(mapping, &bo_va->valids, list) {1844		if (mapping->start == saddr)1845			break;1846	}1847 1848	if (&mapping->list == &bo_va->valids) {1849		valid = false;1850 1851		list_for_each_entry(mapping, &bo_va->invalids, list) {1852			if (mapping->start == saddr)1853				break;1854		}1855 1856		if (&mapping->list == &bo_va->invalids)1857			return -ENOENT;1858	}1859 1860	list_del(&mapping->list);1861	amdgpu_vm_it_remove(mapping, &vm->va);1862	mapping->bo_va = NULL;1863	trace_amdgpu_vm_bo_unmap(bo_va, mapping);1864 1865	if (valid)1866		list_add(&mapping->list, &vm->freed);1867	else1868		amdgpu_vm_free_mapping(adev, vm, mapping,1869				       bo_va->last_pt_update);1870 1871	return 0;1872}1873 1874/**1875 * amdgpu_vm_bo_clear_mappings - remove all mappings in a specific range1876 *1877 * @adev: amdgpu_device pointer1878 * @vm: VM structure to use1879 * @saddr: start of the range1880 * @size: size of the range1881 *1882 * Remove all mappings in a range, split them as appropriate.1883 *1884 * Returns:1885 * 0 for success, error for failure.1886 */1887int amdgpu_vm_bo_clear_mappings(struct amdgpu_device *adev,1888				struct amdgpu_vm *vm,1889				uint64_t saddr, uint64_t size)1890{1891	struct amdgpu_bo_va_mapping *before, *after, *tmp, *next;1892	LIST_HEAD(removed);1893	uint64_t eaddr;1894	int r;1895 1896	r = amdgpu_vm_verify_parameters(adev, NULL, saddr, 0, size);1897	if (r)1898		return r;1899 1900	saddr /= AMDGPU_GPU_PAGE_SIZE;1901	eaddr = saddr + (size - 1) / AMDGPU_GPU_PAGE_SIZE;1902 1903	/* Allocate all the needed memory */1904	before = kzalloc(sizeof(*before), GFP_KERNEL);1905	if (!before)1906		return -ENOMEM;1907	INIT_LIST_HEAD(&before->list);1908 1909	after = kzalloc(sizeof(*after), GFP_KERNEL);1910	if (!after) {1911		kfree(before);1912		return -ENOMEM;1913	}1914	INIT_LIST_HEAD(&after->list);1915 1916	/* Now gather all removed mappings */1917	tmp = amdgpu_vm_it_iter_first(&vm->va, saddr, eaddr);1918	while (tmp) {1919		/* Remember mapping split at the start */1920		if (tmp->start < saddr) {1921			before->start = tmp->start;1922			before->last = saddr - 1;1923			before->offset = tmp->offset;1924			before->flags = tmp->flags;1925			before->bo_va = tmp->bo_va;1926			list_add(&before->list, &tmp->bo_va->invalids);1927		}1928 1929		/* Remember mapping split at the end */1930		if (tmp->last > eaddr) {1931			after->start = eaddr + 1;1932			after->last = tmp->last;1933			after->offset = tmp->offset;1934			after->offset += (after->start - tmp->start) << PAGE_SHIFT;1935			after->flags = tmp->flags;1936			after->bo_va = tmp->bo_va;1937			list_add(&after->list, &tmp->bo_va->invalids);1938		}1939 1940		list_del(&tmp->list);1941		list_add(&tmp->list, &removed);1942 1943		tmp = amdgpu_vm_it_iter_next(tmp, saddr, eaddr);1944	}1945 1946	/* And free them up */1947	list_for_each_entry_safe(tmp, next, &removed, list) {1948		amdgpu_vm_it_remove(tmp, &vm->va);1949		list_del(&tmp->list);1950 1951		if (tmp->start < saddr)1952		    tmp->start = saddr;1953		if (tmp->last > eaddr)1954		    tmp->last = eaddr;1955 1956		tmp->bo_va = NULL;1957		list_add(&tmp->list, &vm->freed);1958		trace_amdgpu_vm_bo_unmap(NULL, tmp);1959	}1960 1961	/* Insert partial mapping before the range */1962	if (!list_empty(&before->list)) {1963		struct amdgpu_bo *bo = before->bo_va->base.bo;1964 1965		amdgpu_vm_it_insert(before, &vm->va);1966		if (before->flags & AMDGPU_PTE_PRT_FLAG(adev))1967			amdgpu_vm_prt_get(adev);1968 1969		if (amdgpu_vm_is_bo_always_valid(vm, bo) &&1970		    !before->bo_va->base.moved)1971			amdgpu_vm_bo_moved(&before->bo_va->base);1972	} else {1973		kfree(before);1974	}1975 1976	/* Insert partial mapping after the range */1977	if (!list_empty(&after->list)) {1978		struct amdgpu_bo *bo = after->bo_va->base.bo;1979 1980		amdgpu_vm_it_insert(after, &vm->va);1981		if (after->flags & AMDGPU_PTE_PRT_FLAG(adev))1982			amdgpu_vm_prt_get(adev);1983 1984		if (amdgpu_vm_is_bo_always_valid(vm, bo) &&1985		    !after->bo_va->base.moved)1986			amdgpu_vm_bo_moved(&after->bo_va->base);1987	} else {1988		kfree(after);1989	}1990 1991	return 0;1992}1993 1994/**1995 * amdgpu_vm_bo_lookup_mapping - find mapping by address1996 *1997 * @vm: the requested VM1998 * @addr: the address1999 *2000 * Find a mapping by it's address.2001 *2002 * Returns:2003 * The amdgpu_bo_va_mapping matching for addr or NULL2004 *2005 */2006struct amdgpu_bo_va_mapping *amdgpu_vm_bo_lookup_mapping(struct amdgpu_vm *vm,2007							 uint64_t addr)2008{2009	return amdgpu_vm_it_iter_first(&vm->va, addr, addr);2010}2011 2012/**2013 * amdgpu_vm_bo_trace_cs - trace all reserved mappings2014 *2015 * @vm: the requested vm2016 * @ticket: CS ticket2017 *2018 * Trace all mappings of BOs reserved during a command submission.2019 */2020void amdgpu_vm_bo_trace_cs(struct amdgpu_vm *vm, struct ww_acquire_ctx *ticket)2021{2022	struct amdgpu_bo_va_mapping *mapping;2023 2024	if (!trace_amdgpu_vm_bo_cs_enabled())2025		return;2026 2027	for (mapping = amdgpu_vm_it_iter_first(&vm->va, 0, U64_MAX); mapping;2028	     mapping = amdgpu_vm_it_iter_next(mapping, 0, U64_MAX)) {2029		if (mapping->bo_va && mapping->bo_va->base.bo) {2030			struct amdgpu_bo *bo;2031 2032			bo = mapping->bo_va->base.bo;2033			if (dma_resv_locking_ctx(bo->tbo.base.resv) !=2034			    ticket)2035				continue;2036		}2037 2038		trace_amdgpu_vm_bo_cs(mapping);2039	}2040}2041 2042/**2043 * amdgpu_vm_bo_del - remove a bo from a specific vm2044 *2045 * @adev: amdgpu_device pointer2046 * @bo_va: requested bo_va2047 *2048 * Remove @bo_va->bo from the requested vm.2049 *2050 * Object have to be reserved!2051 */2052void amdgpu_vm_bo_del(struct amdgpu_device *adev,2053		      struct amdgpu_bo_va *bo_va)2054{2055	struct amdgpu_bo_va_mapping *mapping, *next;2056	struct amdgpu_bo *bo = bo_va->base.bo;2057	struct amdgpu_vm *vm = bo_va->base.vm;2058	struct amdgpu_vm_bo_base **base;2059 2060	dma_resv_assert_held(vm->root.bo->tbo.base.resv);2061 2062	if (bo) {2063		dma_resv_assert_held(bo->tbo.base.resv);2064		if (amdgpu_vm_is_bo_always_valid(vm, bo))2065			ttm_bo_set_bulk_move(&bo->tbo, NULL);2066 2067		for (base = &bo_va->base.bo->vm_bo; *base;2068		     base = &(*base)->next) {2069			if (*base != &bo_va->base)2070				continue;2071 2072			*base = bo_va->base.next;2073			break;2074		}2075	}2076 2077	spin_lock(&vm->status_lock);2078	list_del(&bo_va->base.vm_status);2079	spin_unlock(&vm->status_lock);2080 2081	list_for_each_entry_safe(mapping, next, &bo_va->valids, list) {2082		list_del(&mapping->list);2083		amdgpu_vm_it_remove(mapping, &vm->va);2084		mapping->bo_va = NULL;2085		trace_amdgpu_vm_bo_unmap(bo_va, mapping);2086		list_add(&mapping->list, &vm->freed);2087	}2088	list_for_each_entry_safe(mapping, next, &bo_va->invalids, list) {2089		list_del(&mapping->list);2090		amdgpu_vm_it_remove(mapping, &vm->va);2091		amdgpu_vm_free_mapping(adev, vm, mapping,2092				       bo_va->last_pt_update);2093	}2094 2095	dma_fence_put(bo_va->last_pt_update);2096 2097	if (bo && bo_va->is_xgmi)2098		amdgpu_xgmi_set_pstate(adev, AMDGPU_XGMI_PSTATE_MIN);2099 2100	kfree(bo_va);2101}2102 2103/**2104 * amdgpu_vm_evictable - check if we can evict a VM2105 *2106 * @bo: A page table of the VM.2107 *2108 * Check if it is possible to evict a VM.2109 */2110bool amdgpu_vm_evictable(struct amdgpu_bo *bo)2111{2112	struct amdgpu_vm_bo_base *bo_base = bo->vm_bo;2113 2114	/* Page tables of a destroyed VM can go away immediately */2115	if (!bo_base || !bo_base->vm)2116		return true;2117 2118	/* Don't evict VM page tables while they are busy */2119	if (!dma_resv_test_signaled(bo->tbo.base.resv, DMA_RESV_USAGE_BOOKKEEP))2120		return false;2121 2122	/* Try to block ongoing updates */2123	if (!amdgpu_vm_eviction_trylock(bo_base->vm))2124		return false;2125 2126	/* Don't evict VM page tables while they are updated */2127	if (!dma_fence_is_signaled(bo_base->vm->last_unlocked)) {2128		amdgpu_vm_eviction_unlock(bo_base->vm);2129		return false;2130	}2131 2132	bo_base->vm->evicting = true;2133	amdgpu_vm_eviction_unlock(bo_base->vm);2134	return true;2135}2136 2137/**2138 * amdgpu_vm_bo_invalidate - mark the bo as invalid2139 *2140 * @adev: amdgpu_device pointer2141 * @bo: amdgpu buffer object2142 * @evicted: is the BO evicted2143 *2144 * Mark @bo as invalid.2145 */2146void amdgpu_vm_bo_invalidate(struct amdgpu_device *adev,2147			     struct amdgpu_bo *bo, bool evicted)2148{2149	struct amdgpu_vm_bo_base *bo_base;2150 2151	for (bo_base = bo->vm_bo; bo_base; bo_base = bo_base->next) {2152		struct amdgpu_vm *vm = bo_base->vm;2153 2154		if (evicted && amdgpu_vm_is_bo_always_valid(vm, bo)) {2155			amdgpu_vm_bo_evicted(bo_base);2156			continue;2157		}2158 2159		if (bo_base->moved)2160			continue;2161		bo_base->moved = true;2162 2163		if (bo->tbo.type == ttm_bo_type_kernel)2164			amdgpu_vm_bo_relocated(bo_base);2165		else if (amdgpu_vm_is_bo_always_valid(vm, bo))2166			amdgpu_vm_bo_moved(bo_base);2167		else2168			amdgpu_vm_bo_invalidated(bo_base);2169	}2170}2171 2172/**2173 * amdgpu_vm_get_block_size - calculate VM page table size as power of two2174 *2175 * @vm_size: VM size2176 *2177 * Returns:2178 * VM page table as power of two2179 */2180static uint32_t amdgpu_vm_get_block_size(uint64_t vm_size)2181{2182	/* Total bits covered by PD + PTs */2183	unsigned bits = ilog2(vm_size) + 18;2184 2185	/* Make sure the PD is 4K in size up to 8GB address space.2186	   Above that split equal between PD and PTs */2187	if (vm_size <= 8)2188		return (bits - 9);2189	else2190		return ((bits + 3) / 2);2191}2192 2193/**2194 * amdgpu_vm_adjust_size - adjust vm size, block size and fragment size2195 *2196 * @adev: amdgpu_device pointer2197 * @min_vm_size: the minimum vm size in GB if it's set auto2198 * @fragment_size_default: Default PTE fragment size2199 * @max_level: max VMPT level2200 * @max_bits: max address space size in bits2201 *2202 */2203void amdgpu_vm_adjust_size(struct amdgpu_device *adev, uint32_t min_vm_size,2204			   uint32_t fragment_size_default, unsigned max_level,2205			   unsigned max_bits)2206{2207	unsigned int max_size = 1 << (max_bits - 30);2208	unsigned int vm_size;2209	uint64_t tmp;2210 2211	/* adjust vm size first */2212	if (amdgpu_vm_size != -1) {2213		vm_size = amdgpu_vm_size;2214		if (vm_size > max_size) {2215			dev_warn(adev->dev, "VM size (%d) too large, max is %u GB\n",2216				 amdgpu_vm_size, max_size);2217			vm_size = max_size;2218		}2219	} else {2220		struct sysinfo si;2221		unsigned int phys_ram_gb;2222 2223		/* Optimal VM size depends on the amount of physical2224		 * RAM available. Underlying requirements and2225		 * assumptions:2226		 *2227		 *  - Need to map system memory and VRAM from all GPUs2228		 *     - VRAM from other GPUs not known here2229		 *     - Assume VRAM <= system memory2230		 *  - On GFX8 and older, VM space can be segmented for2231		 *    different MTYPEs2232		 *  - Need to allow room for fragmentation, guard pages etc.2233		 *2234		 * This adds up to a rough guess of system memory x3.2235		 * Round up to power of two to maximize the available2236		 * VM size with the given page table size.2237		 */2238		si_meminfo(&si);2239		phys_ram_gb = ((uint64_t)si.totalram * si.mem_unit +2240			       (1 << 30) - 1) >> 30;2241		vm_size = roundup_pow_of_two(2242			clamp(phys_ram_gb * 3, min_vm_size, max_size));2243	}2244 2245	adev->vm_manager.max_pfn = (uint64_t)vm_size << 18;2246 2247	tmp = roundup_pow_of_two(adev->vm_manager.max_pfn);2248	if (amdgpu_vm_block_size != -1)2249		tmp >>= amdgpu_vm_block_size - 9;2250	tmp = DIV_ROUND_UP(fls64(tmp) - 1, 9) - 1;2251	adev->vm_manager.num_level = min_t(unsigned int, max_level, tmp);2252	switch (adev->vm_manager.num_level) {2253	case 3:2254		adev->vm_manager.root_level = AMDGPU_VM_PDB2;2255		break;2256	case 2:2257		adev->vm_manager.root_level = AMDGPU_VM_PDB1;2258		break;2259	case 1:2260		adev->vm_manager.root_level = AMDGPU_VM_PDB0;2261		break;2262	default:2263		dev_err(adev->dev, "VMPT only supports 2~4+1 levels\n");2264	}2265	/* block size depends on vm size and hw setup*/2266	if (amdgpu_vm_block_size != -1)2267		adev->vm_manager.block_size =2268			min((unsigned)amdgpu_vm_block_size, max_bits2269			    - AMDGPU_GPU_PAGE_SHIFT2270			    - 9 * adev->vm_manager.num_level);2271	else if (adev->vm_manager.num_level > 1)2272		adev->vm_manager.block_size = 9;2273	else2274		adev->vm_manager.block_size = amdgpu_vm_get_block_size(tmp);2275 2276	if (amdgpu_vm_fragment_size == -1)2277		adev->vm_manager.fragment_size = fragment_size_default;2278	else2279		adev->vm_manager.fragment_size = amdgpu_vm_fragment_size;2280 2281	DRM_INFO("vm size is %u GB, %u levels, block size is %u-bit, fragment size is %u-bit\n",2282		 vm_size, adev->vm_manager.num_level + 1,2283		 adev->vm_manager.block_size,2284		 adev->vm_manager.fragment_size);2285}2286 2287/**2288 * amdgpu_vm_wait_idle - wait for the VM to become idle2289 *2290 * @vm: VM object to wait for2291 * @timeout: timeout to wait for VM to become idle2292 */2293long amdgpu_vm_wait_idle(struct amdgpu_vm *vm, long timeout)2294{2295	timeout = dma_resv_wait_timeout(vm->root.bo->tbo.base.resv,2296					DMA_RESV_USAGE_BOOKKEEP,2297					true, timeout);2298	if (timeout <= 0)2299		return timeout;2300 2301	return dma_fence_wait_timeout(vm->last_unlocked, true, timeout);2302}2303 2304static void amdgpu_vm_destroy_task_info(struct kref *kref)2305{2306	struct amdgpu_task_info *ti = container_of(kref, struct amdgpu_task_info, refcount);2307 2308	kfree(ti);2309}2310 2311static inline struct amdgpu_vm *2312amdgpu_vm_get_vm_from_pasid(struct amdgpu_device *adev, u32 pasid)2313{2314	struct amdgpu_vm *vm;2315	unsigned long flags;2316 2317	xa_lock_irqsave(&adev->vm_manager.pasids, flags);2318	vm = xa_load(&adev->vm_manager.pasids, pasid);2319	xa_unlock_irqrestore(&adev->vm_manager.pasids, flags);2320 2321	return vm;2322}2323 2324/**2325 * amdgpu_vm_put_task_info - reference down the vm task_info ptr2326 *2327 * @task_info: task_info struct under discussion.2328 *2329 * frees the vm task_info ptr at the last put2330 */2331void amdgpu_vm_put_task_info(struct amdgpu_task_info *task_info)2332{2333	kref_put(&task_info->refcount, amdgpu_vm_destroy_task_info);2334}2335 2336/**2337 * amdgpu_vm_get_task_info_vm - Extracts task info for a vm.2338 *2339 * @vm: VM to get info from2340 *2341 * Returns the reference counted task_info structure, which must be2342 * referenced down with amdgpu_vm_put_task_info.2343 */2344struct amdgpu_task_info *2345amdgpu_vm_get_task_info_vm(struct amdgpu_vm *vm)2346{2347	struct amdgpu_task_info *ti = NULL;2348 2349	if (vm) {2350		ti = vm->task_info;2351		kref_get(&vm->task_info->refcount);2352	}2353 2354	return ti;2355}2356 2357/**2358 * amdgpu_vm_get_task_info_pasid - Extracts task info for a PASID.2359 *2360 * @adev: drm device pointer2361 * @pasid: PASID identifier for VM2362 *2363 * Returns the reference counted task_info structure, which must be2364 * referenced down with amdgpu_vm_put_task_info.2365 */2366struct amdgpu_task_info *2367amdgpu_vm_get_task_info_pasid(struct amdgpu_device *adev, u32 pasid)2368{2369	return amdgpu_vm_get_task_info_vm(2370			amdgpu_vm_get_vm_from_pasid(adev, pasid));2371}2372 2373static int amdgpu_vm_create_task_info(struct amdgpu_vm *vm)2374{2375	vm->task_info = kzalloc(sizeof(struct amdgpu_task_info), GFP_KERNEL);2376	if (!vm->task_info)2377		return -ENOMEM;2378 2379	kref_init(&vm->task_info->refcount);2380	return 0;2381}2382 2383/**2384 * amdgpu_vm_set_task_info - Sets VMs task info.2385 *2386 * @vm: vm for which to set the info2387 */2388void amdgpu_vm_set_task_info(struct amdgpu_vm *vm)2389{2390	if (!vm->task_info)2391		return;2392 2393	if (vm->task_info->pid == current->pid)2394		return;2395 2396	vm->task_info->pid = current->pid;2397	get_task_comm(vm->task_info->task_name, current);2398 2399	if (current->group_leader->mm != current->mm)2400		return;2401 2402	vm->task_info->tgid = current->group_leader->pid;2403	get_task_comm(vm->task_info->process_name, current->group_leader);2404}2405 2406/**2407 * amdgpu_vm_init - initialize a vm instance2408 *2409 * @adev: amdgpu_device pointer2410 * @vm: requested vm2411 * @xcp_id: GPU partition selection id2412 *2413 * Init @vm fields.2414 *2415 * Returns:2416 * 0 for success, error for failure.2417 */2418int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm,2419		   int32_t xcp_id)2420{2421	struct amdgpu_bo *root_bo;2422	struct amdgpu_bo_vm *root;2423	int r, i;2424 2425	vm->va = RB_ROOT_CACHED;2426	for (i = 0; i < AMDGPU_MAX_VMHUBS; i++)2427		vm->reserved_vmid[i] = NULL;2428	INIT_LIST_HEAD(&vm->evicted);2429	INIT_LIST_HEAD(&vm->evicted_user);2430	INIT_LIST_HEAD(&vm->relocated);2431	INIT_LIST_HEAD(&vm->moved);2432	INIT_LIST_HEAD(&vm->idle);2433	INIT_LIST_HEAD(&vm->invalidated);2434	spin_lock_init(&vm->status_lock);2435	INIT_LIST_HEAD(&vm->freed);2436	INIT_LIST_HEAD(&vm->done);2437	INIT_LIST_HEAD(&vm->pt_freed);2438	INIT_WORK(&vm->pt_free_work, amdgpu_vm_pt_free_work);2439	INIT_KFIFO(vm->faults);2440 2441	r = amdgpu_vm_init_entities(adev, vm);2442	if (r)2443		return r;2444 2445	ttm_lru_bulk_move_init(&vm->lru_bulk_move);2446 2447	vm->is_compute_context = false;2448 2449	vm->use_cpu_for_update = !!(adev->vm_manager.vm_update_mode &2450				    AMDGPU_VM_USE_CPU_FOR_GFX);2451 2452	DRM_DEBUG_DRIVER("VM update mode is %s\n",2453			 vm->use_cpu_for_update ? "CPU" : "SDMA");2454	WARN_ONCE((vm->use_cpu_for_update &&2455		   !amdgpu_gmc_vram_full_visible(&adev->gmc)),2456		  "CPU update of VM recommended only for large BAR system\n");2457 2458	if (vm->use_cpu_for_update)2459		vm->update_funcs = &amdgpu_vm_cpu_funcs;2460	else2461		vm->update_funcs = &amdgpu_vm_sdma_funcs;2462 2463	vm->last_update = dma_fence_get_stub();2464	vm->last_unlocked = dma_fence_get_stub();2465	vm->last_tlb_flush = dma_fence_get_stub();2466	vm->generation = amdgpu_vm_generation(adev, NULL);2467 2468	mutex_init(&vm->eviction_lock);2469	vm->evicting = false;2470	vm->tlb_fence_context = dma_fence_context_alloc(1);2471 2472	r = amdgpu_vm_pt_create(adev, vm, adev->vm_manager.root_level,2473				false, &root, xcp_id);2474	if (r)2475		goto error_free_delayed;2476 2477	root_bo = amdgpu_bo_ref(&root->bo);2478	r = amdgpu_bo_reserve(root_bo, true);2479	if (r) {2480		amdgpu_bo_unref(&root_bo);2481		goto error_free_delayed;2482	}2483 2484	amdgpu_vm_bo_base_init(&vm->root, vm, root_bo);2485	r = dma_resv_reserve_fences(root_bo->tbo.base.resv, 1);2486	if (r)2487		goto error_free_root;2488 2489	r = amdgpu_vm_pt_clear(adev, vm, root, false);2490	if (r)2491		goto error_free_root;2492 2493	r = amdgpu_vm_create_task_info(vm);2494	if (r)2495		DRM_DEBUG("Failed to create task info for VM\n");2496 2497	amdgpu_bo_unreserve(vm->root.bo);2498	amdgpu_bo_unref(&root_bo);2499 2500	return 0;2501 2502error_free_root:2503	amdgpu_vm_pt_free_root(adev, vm);2504	amdgpu_bo_unreserve(vm->root.bo);2505	amdgpu_bo_unref(&root_bo);2506 2507error_free_delayed:2508	dma_fence_put(vm->last_tlb_flush);2509	dma_fence_put(vm->last_unlocked);2510	ttm_lru_bulk_move_fini(&adev->mman.bdev, &vm->lru_bulk_move);2511	amdgpu_vm_fini_entities(vm);2512 2513	return r;2514}2515 2516/**2517 * amdgpu_vm_make_compute - Turn a GFX VM into a compute VM2518 *2519 * @adev: amdgpu_device pointer2520 * @vm: requested vm2521 *2522 * This only works on GFX VMs that don't have any BOs added and no2523 * page tables allocated yet.2524 *2525 * Changes the following VM parameters:2526 * - use_cpu_for_update2527 * - pte_supports_ats2528 *2529 * Reinitializes the page directory to reflect the changed ATS2530 * setting.2531 *2532 * Returns:2533 * 0 for success, -errno for errors.2534 */2535int amdgpu_vm_make_compute(struct amdgpu_device *adev, struct amdgpu_vm *vm)2536{2537	int r;2538 2539	r = amdgpu_bo_reserve(vm->root.bo, true);2540	if (r)2541		return r;2542 2543	/* Update VM state */2544	vm->use_cpu_for_update = !!(adev->vm_manager.vm_update_mode &2545				    AMDGPU_VM_USE_CPU_FOR_COMPUTE);2546	DRM_DEBUG_DRIVER("VM update mode is %s\n",2547			 vm->use_cpu_for_update ? "CPU" : "SDMA");2548	WARN_ONCE((vm->use_cpu_for_update &&2549		   !amdgpu_gmc_vram_full_visible(&adev->gmc)),2550		  "CPU update of VM recommended only for large BAR system\n");2551 2552	if (vm->use_cpu_for_update) {2553		/* Sync with last SDMA update/clear before switching to CPU */2554		r = amdgpu_bo_sync_wait(vm->root.bo,2555					AMDGPU_FENCE_OWNER_UNDEFINED, true);2556		if (r)2557			goto unreserve_bo;2558 2559		vm->update_funcs = &amdgpu_vm_cpu_funcs;2560		r = amdgpu_vm_pt_map_tables(adev, vm);2561		if (r)2562			goto unreserve_bo;2563 2564	} else {2565		vm->update_funcs = &amdgpu_vm_sdma_funcs;2566	}2567 2568	dma_fence_put(vm->last_update);2569	vm->last_update = dma_fence_get_stub();2570	vm->is_compute_context = true;2571 2572unreserve_bo:2573	amdgpu_bo_unreserve(vm->root.bo);2574	return r;2575}2576 2577/**2578 * amdgpu_vm_release_compute - release a compute vm2579 * @adev: amdgpu_device pointer2580 * @vm: a vm turned into compute vm by calling amdgpu_vm_make_compute2581 *2582 * This is a correspondant of amdgpu_vm_make_compute. It decouples compute2583 * pasid from vm. Compute should stop use of vm after this call.2584 */2585void amdgpu_vm_release_compute(struct amdgpu_device *adev, struct amdgpu_vm *vm)2586{2587	amdgpu_vm_set_pasid(adev, vm, 0);2588	vm->is_compute_context = false;2589}2590 2591/**2592 * amdgpu_vm_fini - tear down a vm instance2593 *2594 * @adev: amdgpu_device pointer2595 * @vm: requested vm2596 *2597 * Tear down @vm.2598 * Unbind the VM and remove all bos from the vm bo list2599 */2600void amdgpu_vm_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm)2601{2602	struct amdgpu_bo_va_mapping *mapping, *tmp;2603	bool prt_fini_needed = !!adev->gmc.gmc_funcs->set_prt;2604	struct amdgpu_bo *root;2605	unsigned long flags;2606	int i;2607 2608	amdgpu_amdkfd_gpuvm_destroy_cb(adev, vm);2609 2610	flush_work(&vm->pt_free_work);2611 2612	root = amdgpu_bo_ref(vm->root.bo);2613	amdgpu_bo_reserve(root, true);2614	amdgpu_vm_put_task_info(vm->task_info);2615	amdgpu_vm_set_pasid(adev, vm, 0);2616	dma_fence_wait(vm->last_unlocked, false);2617	dma_fence_put(vm->last_unlocked);2618	dma_fence_wait(vm->last_tlb_flush, false);2619	/* Make sure that all fence callbacks have completed */2620	spin_lock_irqsave(vm->last_tlb_flush->lock, flags);2621	spin_unlock_irqrestore(vm->last_tlb_flush->lock, flags);2622	dma_fence_put(vm->last_tlb_flush);2623 2624	list_for_each_entry_safe(mapping, tmp, &vm->freed, list) {2625		if (mapping->flags & AMDGPU_PTE_PRT_FLAG(adev) && prt_fini_needed) {2626			amdgpu_vm_prt_fini(adev, vm);2627			prt_fini_needed = false;2628		}2629 2630		list_del(&mapping->list);2631		amdgpu_vm_free_mapping(adev, vm, mapping, NULL);2632	}2633 2634	amdgpu_vm_pt_free_root(adev, vm);2635	amdgpu_bo_unreserve(root);2636	amdgpu_bo_unref(&root);2637	WARN_ON(vm->root.bo);2638 2639	amdgpu_vm_fini_entities(vm);2640 2641	if (!RB_EMPTY_ROOT(&vm->va.rb_root)) {2642		dev_err(adev->dev, "still active bo inside vm\n");2643	}2644	rbtree_postorder_for_each_entry_safe(mapping, tmp,2645					     &vm->va.rb_root, rb) {2646		/* Don't remove the mapping here, we don't want to trigger a2647		 * rebalance and the tree is about to be destroyed anyway.2648		 */2649		list_del(&mapping->list);2650		kfree(mapping);2651	}2652 2653	dma_fence_put(vm->last_update);2654 2655	for (i = 0; i < AMDGPU_MAX_VMHUBS; i++) {2656		if (vm->reserved_vmid[i]) {2657			amdgpu_vmid_free_reserved(adev, i);2658			vm->reserved_vmid[i] = false;2659		}2660	}2661 2662	ttm_lru_bulk_move_fini(&adev->mman.bdev, &vm->lru_bulk_move);2663}2664 2665/**2666 * amdgpu_vm_manager_init - init the VM manager2667 *2668 * @adev: amdgpu_device pointer2669 *2670 * Initialize the VM manager structures2671 */2672void amdgpu_vm_manager_init(struct amdgpu_device *adev)2673{2674	unsigned i;2675 2676	/* Concurrent flushes are only possible starting with Vega10 and2677	 * are broken on Navi10 and Navi14.2678	 */2679	adev->vm_manager.concurrent_flush = !(adev->asic_type < CHIP_VEGA10 ||2680					      adev->asic_type == CHIP_NAVI10 ||2681					      adev->asic_type == CHIP_NAVI14);2682	amdgpu_vmid_mgr_init(adev);2683 2684	adev->vm_manager.fence_context =2685		dma_fence_context_alloc(AMDGPU_MAX_RINGS);2686	for (i = 0; i < AMDGPU_MAX_RINGS; ++i)2687		adev->vm_manager.seqno[i] = 0;2688 2689	spin_lock_init(&adev->vm_manager.prt_lock);2690	atomic_set(&adev->vm_manager.num_prt_users, 0);2691 2692	/* If not overridden by the user, by default, only in large BAR systems2693	 * Compute VM tables will be updated by CPU2694	 */2695#ifdef CONFIG_X86_642696	if (amdgpu_vm_update_mode == -1) {2697		/* For asic with VF MMIO access protection2698		 * avoid using CPU for VM table updates2699		 */2700		if (amdgpu_gmc_vram_full_visible(&adev->gmc) &&2701		    !amdgpu_sriov_vf_mmio_access_protection(adev))2702			adev->vm_manager.vm_update_mode =2703				AMDGPU_VM_USE_CPU_FOR_COMPUTE;2704		else2705			adev->vm_manager.vm_update_mode = 0;2706	} else2707		adev->vm_manager.vm_update_mode = amdgpu_vm_update_mode;2708#else2709	adev->vm_manager.vm_update_mode = 0;2710#endif2711 2712	xa_init_flags(&adev->vm_manager.pasids, XA_FLAGS_LOCK_IRQ);2713}2714 2715/**2716 * amdgpu_vm_manager_fini - cleanup VM manager2717 *2718 * @adev: amdgpu_device pointer2719 *2720 * Cleanup the VM manager and free resources.2721 */2722void amdgpu_vm_manager_fini(struct amdgpu_device *adev)2723{2724	WARN_ON(!xa_empty(&adev->vm_manager.pasids));2725	xa_destroy(&adev->vm_manager.pasids);2726 2727	amdgpu_vmid_mgr_fini(adev);2728}2729 2730/**2731 * amdgpu_vm_ioctl - Manages VMID reservation for vm hubs.2732 *2733 * @dev: drm device pointer2734 * @data: drm_amdgpu_vm2735 * @filp: drm file pointer2736 *2737 * Returns:2738 * 0 for success, -errno for errors.2739 */2740int amdgpu_vm_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)2741{2742	union drm_amdgpu_vm *args = data;2743	struct amdgpu_device *adev = drm_to_adev(dev);2744	struct amdgpu_fpriv *fpriv = filp->driver_priv;2745 2746	/* No valid flags defined yet */2747	if (args->in.flags)2748		return -EINVAL;2749 2750	switch (args->in.op) {2751	case AMDGPU_VM_OP_RESERVE_VMID:2752		/* We only have requirement to reserve vmid from gfxhub */2753		if (!fpriv->vm.reserved_vmid[AMDGPU_GFXHUB(0)]) {2754			amdgpu_vmid_alloc_reserved(adev, AMDGPU_GFXHUB(0));2755			fpriv->vm.reserved_vmid[AMDGPU_GFXHUB(0)] = true;2756		}2757 2758		break;2759	case AMDGPU_VM_OP_UNRESERVE_VMID:2760		if (fpriv->vm.reserved_vmid[AMDGPU_GFXHUB(0)]) {2761			amdgpu_vmid_free_reserved(adev, AMDGPU_GFXHUB(0));2762			fpriv->vm.reserved_vmid[AMDGPU_GFXHUB(0)] = false;2763		}2764		break;2765	default:2766		return -EINVAL;2767	}2768 2769	return 0;2770}2771 2772/**2773 * amdgpu_vm_handle_fault - graceful handling of VM faults.2774 * @adev: amdgpu device pointer2775 * @pasid: PASID of the VM2776 * @ts: Timestamp of the fault2777 * @vmid: VMID, only used for GFX 9.4.3.2778 * @node_id: Node_id received in IH cookie. Only applicable for2779 *           GFX 9.4.3.2780 * @addr: Address of the fault2781 * @write_fault: true is write fault, false is read fault2782 *2783 * Try to gracefully handle a VM fault. Return true if the fault was handled and2784 * shouldn't be reported any more.2785 */2786bool amdgpu_vm_handle_fault(struct amdgpu_device *adev, u32 pasid,2787			    u32 vmid, u32 node_id, uint64_t addr, uint64_t ts,2788			    bool write_fault)2789{2790	bool is_compute_context = false;2791	struct amdgpu_bo *root;2792	unsigned long irqflags;2793	uint64_t value, flags;2794	struct amdgpu_vm *vm;2795	int r;2796 2797	xa_lock_irqsave(&adev->vm_manager.pasids, irqflags);2798	vm = xa_load(&adev->vm_manager.pasids, pasid);2799	if (vm) {2800		root = amdgpu_bo_ref(vm->root.bo);2801		is_compute_context = vm->is_compute_context;2802	} else {2803		root = NULL;2804	}2805	xa_unlock_irqrestore(&adev->vm_manager.pasids, irqflags);2806 2807	if (!root)2808		return false;2809 2810	addr /= AMDGPU_GPU_PAGE_SIZE;2811 2812	if (is_compute_context && !svm_range_restore_pages(adev, pasid, vmid,2813	    node_id, addr, ts, write_fault)) {2814		amdgpu_bo_unref(&root);2815		return true;2816	}2817 2818	r = amdgpu_bo_reserve(root, true);2819	if (r)2820		goto error_unref;2821 2822	/* Double check that the VM still exists */2823	xa_lock_irqsave(&adev->vm_manager.pasids, irqflags);2824	vm = xa_load(&adev->vm_manager.pasids, pasid);2825	if (vm && vm->root.bo != root)2826		vm = NULL;2827	xa_unlock_irqrestore(&adev->vm_manager.pasids, irqflags);2828	if (!vm)2829		goto error_unlock;2830 2831	flags = AMDGPU_PTE_VALID | AMDGPU_PTE_SNOOPED |2832		AMDGPU_PTE_SYSTEM;2833 2834	if (is_compute_context) {2835		/* Intentionally setting invalid PTE flag2836		 * combination to force a no-retry-fault2837		 */2838		flags = AMDGPU_VM_NORETRY_FLAGS;2839		value = 0;2840	} else if (amdgpu_vm_fault_stop == AMDGPU_VM_FAULT_STOP_NEVER) {2841		/* Redirect the access to the dummy page */2842		value = adev->dummy_page_addr;2843		flags |= AMDGPU_PTE_EXECUTABLE | AMDGPU_PTE_READABLE |2844			AMDGPU_PTE_WRITEABLE;2845 2846	} else {2847		/* Let the hw retry silently on the PTE */2848		value = 0;2849	}2850 2851	r = dma_resv_reserve_fences(root->tbo.base.resv, 1);2852	if (r) {2853		pr_debug("failed %d to reserve fence slot\n", r);2854		goto error_unlock;2855	}2856 2857	r = amdgpu_vm_update_range(adev, vm, true, false, false, false,2858				   NULL, addr, addr, flags, value, 0, NULL, NULL, NULL);2859	if (r)2860		goto error_unlock;2861 2862	r = amdgpu_vm_update_pdes(adev, vm, true);2863 2864error_unlock:2865	amdgpu_bo_unreserve(root);2866	if (r < 0)2867		DRM_ERROR("Can't handle page fault (%d)\n", r);2868 2869error_unref:2870	amdgpu_bo_unref(&root);2871 2872	return false;2873}2874 2875#if defined(CONFIG_DEBUG_FS)2876/**2877 * amdgpu_debugfs_vm_bo_info  - print BO info for the VM2878 *2879 * @vm: Requested VM for printing BO info2880 * @m: debugfs file2881 *2882 * Print BO information in debugfs file for the VM2883 */2884void amdgpu_debugfs_vm_bo_info(struct amdgpu_vm *vm, struct seq_file *m)2885{2886	struct amdgpu_bo_va *bo_va, *tmp;2887	u64 total_idle = 0;2888	u64 total_evicted = 0;2889	u64 total_relocated = 0;2890	u64 total_moved = 0;2891	u64 total_invalidated = 0;2892	u64 total_done = 0;2893	unsigned int total_idle_objs = 0;2894	unsigned int total_evicted_objs = 0;2895	unsigned int total_relocated_objs = 0;2896	unsigned int total_moved_objs = 0;2897	unsigned int total_invalidated_objs = 0;2898	unsigned int total_done_objs = 0;2899	unsigned int id = 0;2900 2901	spin_lock(&vm->status_lock);2902	seq_puts(m, "\tIdle BOs:\n");2903	list_for_each_entry_safe(bo_va, tmp, &vm->idle, base.vm_status) {2904		if (!bo_va->base.bo)2905			continue;2906		total_idle += amdgpu_bo_print_info(id++, bo_va->base.bo, m);2907	}2908	total_idle_objs = id;2909	id = 0;2910 2911	seq_puts(m, "\tEvicted BOs:\n");2912	list_for_each_entry_safe(bo_va, tmp, &vm->evicted, base.vm_status) {2913		if (!bo_va->base.bo)2914			continue;2915		total_evicted += amdgpu_bo_print_info(id++, bo_va->base.bo, m);2916	}2917	total_evicted_objs = id;2918	id = 0;2919 2920	seq_puts(m, "\tRelocated BOs:\n");2921	list_for_each_entry_safe(bo_va, tmp, &vm->relocated, base.vm_status) {2922		if (!bo_va->base.bo)2923			continue;2924		total_relocated += amdgpu_bo_print_info(id++, bo_va->base.bo, m);2925	}2926	total_relocated_objs = id;2927	id = 0;2928 2929	seq_puts(m, "\tMoved BOs:\n");2930	list_for_each_entry_safe(bo_va, tmp, &vm->moved, base.vm_status) {2931		if (!bo_va->base.bo)2932			continue;2933		total_moved += amdgpu_bo_print_info(id++, bo_va->base.bo, m);2934	}2935	total_moved_objs = id;2936	id = 0;2937 2938	seq_puts(m, "\tInvalidated BOs:\n");2939	list_for_each_entry_safe(bo_va, tmp, &vm->invalidated, base.vm_status) {2940		if (!bo_va->base.bo)2941			continue;2942		total_invalidated += amdgpu_bo_print_info(id++,	bo_va->base.bo, m);2943	}2944	total_invalidated_objs = id;2945	id = 0;2946 2947	seq_puts(m, "\tDone BOs:\n");2948	list_for_each_entry_safe(bo_va, tmp, &vm->done, base.vm_status) {2949		if (!bo_va->base.bo)2950			continue;2951		total_done += amdgpu_bo_print_info(id++, bo_va->base.bo, m);2952	}2953	spin_unlock(&vm->status_lock);2954	total_done_objs = id;2955 2956	seq_printf(m, "\tTotal idle size:        %12lld\tobjs:\t%d\n", total_idle,2957		   total_idle_objs);2958	seq_printf(m, "\tTotal evicted size:     %12lld\tobjs:\t%d\n", total_evicted,2959		   total_evicted_objs);2960	seq_printf(m, "\tTotal relocated size:   %12lld\tobjs:\t%d\n", total_relocated,2961		   total_relocated_objs);2962	seq_printf(m, "\tTotal moved size:       %12lld\tobjs:\t%d\n", total_moved,2963		   total_moved_objs);2964	seq_printf(m, "\tTotal invalidated size: %12lld\tobjs:\t%d\n", total_invalidated,2965		   total_invalidated_objs);2966	seq_printf(m, "\tTotal done size:        %12lld\tobjs:\t%d\n", total_done,2967		   total_done_objs);2968}2969#endif2970 2971/**2972 * amdgpu_vm_update_fault_cache - update cached fault into.2973 * @adev: amdgpu device pointer2974 * @pasid: PASID of the VM2975 * @addr: Address of the fault2976 * @status: GPUVM fault status register2977 * @vmhub: which vmhub got the fault2978 *2979 * Cache the fault info for later use by userspace in debugging.2980 */2981void amdgpu_vm_update_fault_cache(struct amdgpu_device *adev,2982				  unsigned int pasid,2983				  uint64_t addr,2984				  uint32_t status,2985				  unsigned int vmhub)2986{2987	struct amdgpu_vm *vm;2988	unsigned long flags;2989 2990	xa_lock_irqsave(&adev->vm_manager.pasids, flags);2991 2992	vm = xa_load(&adev->vm_manager.pasids, pasid);2993	/* Don't update the fault cache if status is 0.  In the multiple2994	 * fault case, subsequent faults will return a 0 status which is2995	 * useless for userspace and replaces the useful fault status, so2996	 * only update if status is non-0.2997	 */2998	if (vm && status) {2999		vm->fault_info.addr = addr;3000		vm->fault_info.status = status;3001		/*3002		 * Update the fault information globally for later usage3003		 * when vm could be stale or freed.3004		 */3005		adev->vm_manager.fault_info.addr = addr;3006		adev->vm_manager.fault_info.vmhub = vmhub;3007		adev->vm_manager.fault_info.status = status;3008 3009		if (AMDGPU_IS_GFXHUB(vmhub)) {3010			vm->fault_info.vmhub = AMDGPU_VMHUB_TYPE_GFX;3011			vm->fault_info.vmhub |=3012				(vmhub - AMDGPU_GFXHUB_START) << AMDGPU_VMHUB_IDX_SHIFT;3013		} else if (AMDGPU_IS_MMHUB0(vmhub)) {3014			vm->fault_info.vmhub = AMDGPU_VMHUB_TYPE_MM0;3015			vm->fault_info.vmhub |=3016				(vmhub - AMDGPU_MMHUB0_START) << AMDGPU_VMHUB_IDX_SHIFT;3017		} else if (AMDGPU_IS_MMHUB1(vmhub)) {3018			vm->fault_info.vmhub = AMDGPU_VMHUB_TYPE_MM1;3019			vm->fault_info.vmhub |=3020				(vmhub - AMDGPU_MMHUB1_START) << AMDGPU_VMHUB_IDX_SHIFT;3021		} else {3022			WARN_ONCE(1, "Invalid vmhub %u\n", vmhub);3023		}3024	}3025	xa_unlock_irqrestore(&adev->vm_manager.pasids, flags);3026}3027 3028/**3029 * amdgpu_vm_is_bo_always_valid - check if the BO is VM always valid3030 *3031 * @vm: VM to test against.3032 * @bo: BO to be tested.3033 *3034 * Returns true if the BO shares the dma_resv object with the root PD and is3035 * always guaranteed to be valid inside the VM.3036 */3037bool amdgpu_vm_is_bo_always_valid(struct amdgpu_vm *vm, struct amdgpu_bo *bo)3038{3039	return bo && bo->tbo.base.resv == vm->root.bo->tbo.base.resv;3040}3041