brintos / linux-shallow public Read only

1// SPDX-License-Identifier: GPL-2.0 OR MIT2/*3 * Copyright 2020-2021 Advanced Micro Devices, Inc.4 *5 * Permission is hereby granted, free of charge, to any person obtaining a6 * copy of this software and associated documentation files (the "Software"),7 * to deal in the Software without restriction, including without limitation8 * the rights to use, copy, modify, merge, publish, distribute, sublicense,9 * and/or sell copies of the Software, and to permit persons to whom the10 * Software is furnished to do so, subject to the following conditions:11 *12 * The above copyright notice and this permission notice shall be included in13 * all copies or substantial portions of the Software.14 *15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL18 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR19 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,20 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR21 * OTHER DEALINGS IN THE SOFTWARE.22 */23 24#include <linux/types.h>25#include <linux/sched/task.h>26#include <linux/dynamic_debug.h>27#include <drm/ttm/ttm_tt.h>28#include <drm/drm_exec.h>29 30#include "amdgpu_sync.h"31#include "amdgpu_object.h"32#include "amdgpu_vm.h"33#include "amdgpu_hmm.h"34#include "amdgpu.h"35#include "amdgpu_xgmi.h"36#include "kfd_priv.h"37#include "kfd_svm.h"38#include "kfd_migrate.h"39#include "kfd_smi_events.h"40 41#ifdef dev_fmt42#undef dev_fmt43#endif44#define dev_fmt(fmt) "kfd_svm: %s: " fmt, __func__45 46#define AMDGPU_SVM_RANGE_RESTORE_DELAY_MS 147 48/* Long enough to ensure no retry fault comes after svm range is restored and49 * page table is updated.50 */51#define AMDGPU_SVM_RANGE_RETRY_FAULT_PENDING	(2UL * NSEC_PER_MSEC)52#if IS_ENABLED(CONFIG_DYNAMIC_DEBUG)53#define dynamic_svm_range_dump(svms) \54	_dynamic_func_call_no_desc("svm_range_dump", svm_range_debug_dump, svms)55#else56#define dynamic_svm_range_dump(svms) \57	do { if (0) svm_range_debug_dump(svms); } while (0)58#endif59 60/* Giant svm range split into smaller ranges based on this, it is decided using61 * minimum of all dGPU/APU 1/32 VRAM size, between 2MB to 1GB and alignment to62 * power of 2MB.63 */64static uint64_t max_svm_range_pages;65 66struct criu_svm_metadata {67	struct list_head list;68	struct kfd_criu_svm_range_priv_data data;69};70 71static void svm_range_evict_svm_bo_worker(struct work_struct *work);72static bool73svm_range_cpu_invalidate_pagetables(struct mmu_interval_notifier *mni,74				    const struct mmu_notifier_range *range,75				    unsigned long cur_seq);76static int77svm_range_check_vm(struct kfd_process *p, uint64_t start, uint64_t last,78		   uint64_t *bo_s, uint64_t *bo_l);79static const struct mmu_interval_notifier_ops svm_range_mn_ops = {80	.invalidate = svm_range_cpu_invalidate_pagetables,81};82 83/**84 * svm_range_unlink - unlink svm_range from lists and interval tree85 * @prange: svm range structure to be removed86 *87 * Remove the svm_range from the svms and svm_bo lists and the svms88 * interval tree.89 *90 * Context: The caller must hold svms->lock91 */92static void svm_range_unlink(struct svm_range *prange)93{94	pr_debug("svms 0x%p prange 0x%p [0x%lx 0x%lx]\n", prange->svms,95		 prange, prange->start, prange->last);96 97	if (prange->svm_bo) {98		spin_lock(&prange->svm_bo->list_lock);99		list_del(&prange->svm_bo_list);100		spin_unlock(&prange->svm_bo->list_lock);101	}102 103	list_del(&prange->list);104	if (prange->it_node.start != 0 && prange->it_node.last != 0)105		interval_tree_remove(&prange->it_node, &prange->svms->objects);106}107 108static void109svm_range_add_notifier_locked(struct mm_struct *mm, struct svm_range *prange)110{111	pr_debug("svms 0x%p prange 0x%p [0x%lx 0x%lx]\n", prange->svms,112		 prange, prange->start, prange->last);113 114	mmu_interval_notifier_insert_locked(&prange->notifier, mm,115				     prange->start << PAGE_SHIFT,116				     prange->npages << PAGE_SHIFT,117				     &svm_range_mn_ops);118}119 120/**121 * svm_range_add_to_svms - add svm range to svms122 * @prange: svm range structure to be added123 *124 * Add the svm range to svms interval tree and link list125 *126 * Context: The caller must hold svms->lock127 */128static void svm_range_add_to_svms(struct svm_range *prange)129{130	pr_debug("svms 0x%p prange 0x%p [0x%lx 0x%lx]\n", prange->svms,131		 prange, prange->start, prange->last);132 133	list_move_tail(&prange->list, &prange->svms->list);134	prange->it_node.start = prange->start;135	prange->it_node.last = prange->last;136	interval_tree_insert(&prange->it_node, &prange->svms->objects);137}138 139static void svm_range_remove_notifier(struct svm_range *prange)140{141	pr_debug("remove notifier svms 0x%p prange 0x%p [0x%lx 0x%lx]\n",142		 prange->svms, prange,143		 prange->notifier.interval_tree.start >> PAGE_SHIFT,144		 prange->notifier.interval_tree.last >> PAGE_SHIFT);145 146	if (prange->notifier.interval_tree.start != 0 &&147	    prange->notifier.interval_tree.last != 0)148		mmu_interval_notifier_remove(&prange->notifier);149}150 151static bool152svm_is_valid_dma_mapping_addr(struct device *dev, dma_addr_t dma_addr)153{154	return dma_addr && !dma_mapping_error(dev, dma_addr) &&155	       !(dma_addr & SVM_RANGE_VRAM_DOMAIN);156}157 158static int159svm_range_dma_map_dev(struct amdgpu_device *adev, struct svm_range *prange,160		      unsigned long offset, unsigned long npages,161		      unsigned long *hmm_pfns, uint32_t gpuidx)162{163	enum dma_data_direction dir = DMA_BIDIRECTIONAL;164	dma_addr_t *addr = prange->dma_addr[gpuidx];165	struct device *dev = adev->dev;166	struct page *page;167	int i, r;168 169	if (!addr) {170		addr = kvcalloc(prange->npages, sizeof(*addr), GFP_KERNEL);171		if (!addr)172			return -ENOMEM;173		prange->dma_addr[gpuidx] = addr;174	}175 176	addr += offset;177	for (i = 0; i < npages; i++) {178		if (svm_is_valid_dma_mapping_addr(dev, addr[i]))179			dma_unmap_page(dev, addr[i], PAGE_SIZE, dir);180 181		page = hmm_pfn_to_page(hmm_pfns[i]);182		if (is_zone_device_page(page)) {183			struct amdgpu_device *bo_adev = prange->svm_bo->node->adev;184 185			addr[i] = (hmm_pfns[i] << PAGE_SHIFT) +186				   bo_adev->vm_manager.vram_base_offset -187				   bo_adev->kfd.pgmap.range.start;188			addr[i] |= SVM_RANGE_VRAM_DOMAIN;189			pr_debug_ratelimited("vram address: 0x%llx\n", addr[i]);190			continue;191		}192		addr[i] = dma_map_page(dev, page, 0, PAGE_SIZE, dir);193		r = dma_mapping_error(dev, addr[i]);194		if (r) {195			dev_err(dev, "failed %d dma_map_page\n", r);196			return r;197		}198		pr_debug_ratelimited("dma mapping 0x%llx for page addr 0x%lx\n",199				     addr[i] >> PAGE_SHIFT, page_to_pfn(page));200	}201 202	return 0;203}204 205static int206svm_range_dma_map(struct svm_range *prange, unsigned long *bitmap,207		  unsigned long offset, unsigned long npages,208		  unsigned long *hmm_pfns)209{210	struct kfd_process *p;211	uint32_t gpuidx;212	int r;213 214	p = container_of(prange->svms, struct kfd_process, svms);215 216	for_each_set_bit(gpuidx, bitmap, MAX_GPU_INSTANCE) {217		struct kfd_process_device *pdd;218 219		pr_debug("mapping to gpu idx 0x%x\n", gpuidx);220		pdd = kfd_process_device_from_gpuidx(p, gpuidx);221		if (!pdd) {222			pr_debug("failed to find device idx %d\n", gpuidx);223			return -EINVAL;224		}225 226		r = svm_range_dma_map_dev(pdd->dev->adev, prange, offset, npages,227					  hmm_pfns, gpuidx);228		if (r)229			break;230	}231 232	return r;233}234 235void svm_range_dma_unmap_dev(struct device *dev, dma_addr_t *dma_addr,236			 unsigned long offset, unsigned long npages)237{238	enum dma_data_direction dir = DMA_BIDIRECTIONAL;239	int i;240 241	if (!dma_addr)242		return;243 244	for (i = offset; i < offset + npages; i++) {245		if (!svm_is_valid_dma_mapping_addr(dev, dma_addr[i]))246			continue;247		pr_debug_ratelimited("unmap 0x%llx\n", dma_addr[i] >> PAGE_SHIFT);248		dma_unmap_page(dev, dma_addr[i], PAGE_SIZE, dir);249		dma_addr[i] = 0;250	}251}252 253void svm_range_dma_unmap(struct svm_range *prange)254{255	struct kfd_process_device *pdd;256	dma_addr_t *dma_addr;257	struct device *dev;258	struct kfd_process *p;259	uint32_t gpuidx;260 261	p = container_of(prange->svms, struct kfd_process, svms);262 263	for (gpuidx = 0; gpuidx < MAX_GPU_INSTANCE; gpuidx++) {264		dma_addr = prange->dma_addr[gpuidx];265		if (!dma_addr)266			continue;267 268		pdd = kfd_process_device_from_gpuidx(p, gpuidx);269		if (!pdd) {270			pr_debug("failed to find device idx %d\n", gpuidx);271			continue;272		}273		dev = &pdd->dev->adev->pdev->dev;274 275		svm_range_dma_unmap_dev(dev, dma_addr, 0, prange->npages);276	}277}278 279static void svm_range_free(struct svm_range *prange, bool do_unmap)280{281	uint64_t size = (prange->last - prange->start + 1) << PAGE_SHIFT;282	struct kfd_process *p = container_of(prange->svms, struct kfd_process, svms);283	uint32_t gpuidx;284 285	pr_debug("svms 0x%p prange 0x%p [0x%lx 0x%lx]\n", prange->svms, prange,286		 prange->start, prange->last);287 288	svm_range_vram_node_free(prange);289	if (do_unmap)290		svm_range_dma_unmap(prange);291 292	if (do_unmap && !p->xnack_enabled) {293		pr_debug("unreserve prange 0x%p size: 0x%llx\n", prange, size);294		amdgpu_amdkfd_unreserve_mem_limit(NULL, size,295					KFD_IOC_ALLOC_MEM_FLAGS_USERPTR, 0);296	}297 298	/* free dma_addr array for each gpu */299	for (gpuidx = 0; gpuidx < MAX_GPU_INSTANCE; gpuidx++) {300		if (prange->dma_addr[gpuidx]) {301			kvfree(prange->dma_addr[gpuidx]);302			prange->dma_addr[gpuidx] = NULL;303		}304	}305 306	mutex_destroy(&prange->lock);307	mutex_destroy(&prange->migrate_mutex);308	kfree(prange);309}310 311static void312svm_range_set_default_attributes(struct svm_range_list *svms, int32_t *location,313				 int32_t *prefetch_loc, uint8_t *granularity,314				 uint32_t *flags)315{316	*location = KFD_IOCTL_SVM_LOCATION_UNDEFINED;317	*prefetch_loc = KFD_IOCTL_SVM_LOCATION_UNDEFINED;318	*granularity = svms->default_granularity;319	*flags =320		KFD_IOCTL_SVM_FLAG_HOST_ACCESS | KFD_IOCTL_SVM_FLAG_COHERENT;321}322 323static struct324svm_range *svm_range_new(struct svm_range_list *svms, uint64_t start,325			 uint64_t last, bool update_mem_usage)326{327	uint64_t size = last - start + 1;328	struct svm_range *prange;329	struct kfd_process *p;330 331	prange = kzalloc(sizeof(*prange), GFP_KERNEL);332	if (!prange)333		return NULL;334 335	p = container_of(svms, struct kfd_process, svms);336	if (!p->xnack_enabled && update_mem_usage &&337	    amdgpu_amdkfd_reserve_mem_limit(NULL, size << PAGE_SHIFT,338				    KFD_IOC_ALLOC_MEM_FLAGS_USERPTR, 0)) {339		pr_info("SVM mapping failed, exceeds resident system memory limit\n");340		kfree(prange);341		return NULL;342	}343	prange->npages = size;344	prange->svms = svms;345	prange->start = start;346	prange->last = last;347	INIT_LIST_HEAD(&prange->list);348	INIT_LIST_HEAD(&prange->update_list);349	INIT_LIST_HEAD(&prange->svm_bo_list);350	INIT_LIST_HEAD(&prange->deferred_list);351	INIT_LIST_HEAD(&prange->child_list);352	atomic_set(&prange->invalid, 0);353	prange->validate_timestamp = 0;354	prange->vram_pages = 0;355	mutex_init(&prange->migrate_mutex);356	mutex_init(&prange->lock);357 358	if (p->xnack_enabled)359		bitmap_copy(prange->bitmap_access, svms->bitmap_supported,360			    MAX_GPU_INSTANCE);361 362	svm_range_set_default_attributes(svms, &prange->preferred_loc,363					 &prange->prefetch_loc,364					 &prange->granularity, &prange->flags);365 366	pr_debug("svms 0x%p [0x%llx 0x%llx]\n", svms, start, last);367 368	return prange;369}370 371static bool svm_bo_ref_unless_zero(struct svm_range_bo *svm_bo)372{373	if (!svm_bo || !kref_get_unless_zero(&svm_bo->kref))374		return false;375 376	return true;377}378 379static void svm_range_bo_release(struct kref *kref)380{381	struct svm_range_bo *svm_bo;382 383	svm_bo = container_of(kref, struct svm_range_bo, kref);384	pr_debug("svm_bo 0x%p\n", svm_bo);385 386	spin_lock(&svm_bo->list_lock);387	while (!list_empty(&svm_bo->range_list)) {388		struct svm_range *prange =389				list_first_entry(&svm_bo->range_list,390						struct svm_range, svm_bo_list);391		/* list_del_init tells a concurrent svm_range_vram_node_new when392		 * it's safe to reuse the svm_bo pointer and svm_bo_list head.393		 */394		list_del_init(&prange->svm_bo_list);395		spin_unlock(&svm_bo->list_lock);396 397		pr_debug("svms 0x%p [0x%lx 0x%lx]\n", prange->svms,398			 prange->start, prange->last);399		mutex_lock(&prange->lock);400		prange->svm_bo = NULL;401		/* prange should not hold vram page now */402		WARN_ONCE(prange->actual_loc, "prange should not hold vram page");403		mutex_unlock(&prange->lock);404 405		spin_lock(&svm_bo->list_lock);406	}407	spin_unlock(&svm_bo->list_lock);408 409	if (mmget_not_zero(svm_bo->eviction_fence->mm)) {410		struct kfd_process_device *pdd;411		struct kfd_process *p;412		struct mm_struct *mm;413 414		mm = svm_bo->eviction_fence->mm;415		/*416		 * The forked child process takes svm_bo device pages ref, svm_bo could be417		 * released after parent process is gone.418		 */419		p = kfd_lookup_process_by_mm(mm);420		if (p) {421			pdd = kfd_get_process_device_data(svm_bo->node, p);422			if (pdd)423				atomic64_sub(amdgpu_bo_size(svm_bo->bo), &pdd->vram_usage);424			kfd_unref_process(p);425		}426		mmput(mm);427	}428 429	if (!dma_fence_is_signaled(&svm_bo->eviction_fence->base))430		/* We're not in the eviction worker. Signal the fence. */431		dma_fence_signal(&svm_bo->eviction_fence->base);432	dma_fence_put(&svm_bo->eviction_fence->base);433	amdgpu_bo_unref(&svm_bo->bo);434	kfree(svm_bo);435}436 437static void svm_range_bo_wq_release(struct work_struct *work)438{439	struct svm_range_bo *svm_bo;440 441	svm_bo = container_of(work, struct svm_range_bo, release_work);442	svm_range_bo_release(&svm_bo->kref);443}444 445static void svm_range_bo_release_async(struct kref *kref)446{447	struct svm_range_bo *svm_bo;448 449	svm_bo = container_of(kref, struct svm_range_bo, kref);450	pr_debug("svm_bo 0x%p\n", svm_bo);451	INIT_WORK(&svm_bo->release_work, svm_range_bo_wq_release);452	schedule_work(&svm_bo->release_work);453}454 455void svm_range_bo_unref_async(struct svm_range_bo *svm_bo)456{457	kref_put(&svm_bo->kref, svm_range_bo_release_async);458}459 460static void svm_range_bo_unref(struct svm_range_bo *svm_bo)461{462	if (svm_bo)463		kref_put(&svm_bo->kref, svm_range_bo_release);464}465 466static bool467svm_range_validate_svm_bo(struct kfd_node *node, struct svm_range *prange)468{469	mutex_lock(&prange->lock);470	if (!prange->svm_bo) {471		mutex_unlock(&prange->lock);472		return false;473	}474	if (prange->ttm_res) {475		/* We still have a reference, all is well */476		mutex_unlock(&prange->lock);477		return true;478	}479	if (svm_bo_ref_unless_zero(prange->svm_bo)) {480		/*481		 * Migrate from GPU to GPU, remove range from source svm_bo->node482		 * range list, and return false to allocate svm_bo from destination483		 * node.484		 */485		if (prange->svm_bo->node != node) {486			mutex_unlock(&prange->lock);487 488			spin_lock(&prange->svm_bo->list_lock);489			list_del_init(&prange->svm_bo_list);490			spin_unlock(&prange->svm_bo->list_lock);491 492			svm_range_bo_unref(prange->svm_bo);493			return false;494		}495		if (READ_ONCE(prange->svm_bo->evicting)) {496			struct dma_fence *f;497			struct svm_range_bo *svm_bo;498			/* The BO is getting evicted,499			 * we need to get a new one500			 */501			mutex_unlock(&prange->lock);502			svm_bo = prange->svm_bo;503			f = dma_fence_get(&svm_bo->eviction_fence->base);504			svm_range_bo_unref(prange->svm_bo);505			/* wait for the fence to avoid long spin-loop506			 * at list_empty_careful507			 */508			dma_fence_wait(f, false);509			dma_fence_put(f);510		} else {511			/* The BO was still around and we got512			 * a new reference to it513			 */514			mutex_unlock(&prange->lock);515			pr_debug("reuse old bo svms 0x%p [0x%lx 0x%lx]\n",516				 prange->svms, prange->start, prange->last);517 518			prange->ttm_res = prange->svm_bo->bo->tbo.resource;519			return true;520		}521 522	} else {523		mutex_unlock(&prange->lock);524	}525 526	/* We need a new svm_bo. Spin-loop to wait for concurrent527	 * svm_range_bo_release to finish removing this range from528	 * its range list and set prange->svm_bo to null. After this,529	 * it is safe to reuse the svm_bo pointer and svm_bo_list head.530	 */531	while (!list_empty_careful(&prange->svm_bo_list) || prange->svm_bo)532		cond_resched();533 534	return false;535}536 537static struct svm_range_bo *svm_range_bo_new(void)538{539	struct svm_range_bo *svm_bo;540 541	svm_bo = kzalloc(sizeof(*svm_bo), GFP_KERNEL);542	if (!svm_bo)543		return NULL;544 545	kref_init(&svm_bo->kref);546	INIT_LIST_HEAD(&svm_bo->range_list);547	spin_lock_init(&svm_bo->list_lock);548 549	return svm_bo;550}551 552int553svm_range_vram_node_new(struct kfd_node *node, struct svm_range *prange,554			bool clear)555{556	struct kfd_process_device *pdd;557	struct amdgpu_bo_param bp;558	struct svm_range_bo *svm_bo;559	struct amdgpu_bo_user *ubo;560	struct amdgpu_bo *bo;561	struct kfd_process *p;562	struct mm_struct *mm;563	int r;564 565	p = container_of(prange->svms, struct kfd_process, svms);566	pr_debug("pasid: %x svms 0x%p [0x%lx 0x%lx]\n", p->pasid, prange->svms,567		 prange->start, prange->last);568 569	if (svm_range_validate_svm_bo(node, prange))570		return 0;571 572	svm_bo = svm_range_bo_new();573	if (!svm_bo) {574		pr_debug("failed to alloc svm bo\n");575		return -ENOMEM;576	}577	mm = get_task_mm(p->lead_thread);578	if (!mm) {579		pr_debug("failed to get mm\n");580		kfree(svm_bo);581		return -ESRCH;582	}583	svm_bo->node = node;584	svm_bo->eviction_fence =585		amdgpu_amdkfd_fence_create(dma_fence_context_alloc(1),586					   mm,587					   svm_bo);588	mmput(mm);589	INIT_WORK(&svm_bo->eviction_work, svm_range_evict_svm_bo_worker);590	svm_bo->evicting = 0;591	memset(&bp, 0, sizeof(bp));592	bp.size = prange->npages * PAGE_SIZE;593	bp.byte_align = PAGE_SIZE;594	bp.domain = AMDGPU_GEM_DOMAIN_VRAM;595	bp.flags = AMDGPU_GEM_CREATE_NO_CPU_ACCESS;596	bp.flags |= clear ? AMDGPU_GEM_CREATE_VRAM_CLEARED : 0;597	bp.flags |= AMDGPU_GEM_CREATE_DISCARDABLE;598	bp.type = ttm_bo_type_device;599	bp.resv = NULL;600	if (node->xcp)601		bp.xcp_id_plus1 = node->xcp->id + 1;602 603	r = amdgpu_bo_create_user(node->adev, &bp, &ubo);604	if (r) {605		pr_debug("failed %d to create bo\n", r);606		goto create_bo_failed;607	}608	bo = &ubo->bo;609 610	pr_debug("alloc bo at offset 0x%lx size 0x%lx on partition %d\n",611		 bo->tbo.resource->start << PAGE_SHIFT, bp.size,612		 bp.xcp_id_plus1 - 1);613 614	r = amdgpu_bo_reserve(bo, true);615	if (r) {616		pr_debug("failed %d to reserve bo\n", r);617		goto reserve_bo_failed;618	}619 620	if (clear) {621		r = amdgpu_bo_sync_wait(bo, AMDGPU_FENCE_OWNER_KFD, false);622		if (r) {623			pr_debug("failed %d to sync bo\n", r);624			amdgpu_bo_unreserve(bo);625			goto reserve_bo_failed;626		}627	}628 629	r = dma_resv_reserve_fences(bo->tbo.base.resv, 1);630	if (r) {631		pr_debug("failed %d to reserve bo\n", r);632		amdgpu_bo_unreserve(bo);633		goto reserve_bo_failed;634	}635	amdgpu_bo_fence(bo, &svm_bo->eviction_fence->base, true);636 637	amdgpu_bo_unreserve(bo);638 639	svm_bo->bo = bo;640	prange->svm_bo = svm_bo;641	prange->ttm_res = bo->tbo.resource;642	prange->offset = 0;643 644	spin_lock(&svm_bo->list_lock);645	list_add(&prange->svm_bo_list, &svm_bo->range_list);646	spin_unlock(&svm_bo->list_lock);647 648	pdd = svm_range_get_pdd_by_node(prange, node);649	if (pdd)650		atomic64_add(amdgpu_bo_size(bo), &pdd->vram_usage);651 652	return 0;653 654reserve_bo_failed:655	amdgpu_bo_unref(&bo);656create_bo_failed:657	dma_fence_put(&svm_bo->eviction_fence->base);658	kfree(svm_bo);659	prange->ttm_res = NULL;660 661	return r;662}663 664void svm_range_vram_node_free(struct svm_range *prange)665{666	/* serialize prange->svm_bo unref */667	mutex_lock(&prange->lock);668	/* prange->svm_bo has not been unref */669	if (prange->ttm_res) {670		prange->ttm_res = NULL;671		mutex_unlock(&prange->lock);672		svm_range_bo_unref(prange->svm_bo);673	} else674		mutex_unlock(&prange->lock);675}676 677struct kfd_node *678svm_range_get_node_by_id(struct svm_range *prange, uint32_t gpu_id)679{680	struct kfd_process *p;681	struct kfd_process_device *pdd;682 683	p = container_of(prange->svms, struct kfd_process, svms);684	pdd = kfd_process_device_data_by_id(p, gpu_id);685	if (!pdd) {686		pr_debug("failed to get kfd process device by id 0x%x\n", gpu_id);687		return NULL;688	}689 690	return pdd->dev;691}692 693struct kfd_process_device *694svm_range_get_pdd_by_node(struct svm_range *prange, struct kfd_node *node)695{696	struct kfd_process *p;697 698	p = container_of(prange->svms, struct kfd_process, svms);699 700	return kfd_get_process_device_data(node, p);701}702 703static int svm_range_bo_validate(void *param, struct amdgpu_bo *bo)704{705	struct ttm_operation_ctx ctx = { false, false };706 707	amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_VRAM);708 709	return ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);710}711 712static int713svm_range_check_attr(struct kfd_process *p,714		     uint32_t nattr, struct kfd_ioctl_svm_attribute *attrs)715{716	uint32_t i;717 718	for (i = 0; i < nattr; i++) {719		uint32_t val = attrs[i].value;720		int gpuidx = MAX_GPU_INSTANCE;721 722		switch (attrs[i].type) {723		case KFD_IOCTL_SVM_ATTR_PREFERRED_LOC:724			if (val != KFD_IOCTL_SVM_LOCATION_SYSMEM &&725			    val != KFD_IOCTL_SVM_LOCATION_UNDEFINED)726				gpuidx = kfd_process_gpuidx_from_gpuid(p, val);727			break;728		case KFD_IOCTL_SVM_ATTR_PREFETCH_LOC:729			if (val != KFD_IOCTL_SVM_LOCATION_SYSMEM)730				gpuidx = kfd_process_gpuidx_from_gpuid(p, val);731			break;732		case KFD_IOCTL_SVM_ATTR_ACCESS:733		case KFD_IOCTL_SVM_ATTR_ACCESS_IN_PLACE:734		case KFD_IOCTL_SVM_ATTR_NO_ACCESS:735			gpuidx = kfd_process_gpuidx_from_gpuid(p, val);736			break;737		case KFD_IOCTL_SVM_ATTR_SET_FLAGS:738			break;739		case KFD_IOCTL_SVM_ATTR_CLR_FLAGS:740			break;741		case KFD_IOCTL_SVM_ATTR_GRANULARITY:742			break;743		default:744			pr_debug("unknown attr type 0x%x\n", attrs[i].type);745			return -EINVAL;746		}747 748		if (gpuidx < 0) {749			pr_debug("no GPU 0x%x found\n", val);750			return -EINVAL;751		} else if (gpuidx < MAX_GPU_INSTANCE &&752			   !test_bit(gpuidx, p->svms.bitmap_supported)) {753			pr_debug("GPU 0x%x not supported\n", val);754			return -EINVAL;755		}756	}757 758	return 0;759}760 761static void762svm_range_apply_attrs(struct kfd_process *p, struct svm_range *prange,763		      uint32_t nattr, struct kfd_ioctl_svm_attribute *attrs,764		      bool *update_mapping)765{766	uint32_t i;767	int gpuidx;768 769	for (i = 0; i < nattr; i++) {770		switch (attrs[i].type) {771		case KFD_IOCTL_SVM_ATTR_PREFERRED_LOC:772			prange->preferred_loc = attrs[i].value;773			break;774		case KFD_IOCTL_SVM_ATTR_PREFETCH_LOC:775			prange->prefetch_loc = attrs[i].value;776			break;777		case KFD_IOCTL_SVM_ATTR_ACCESS:778		case KFD_IOCTL_SVM_ATTR_ACCESS_IN_PLACE:779		case KFD_IOCTL_SVM_ATTR_NO_ACCESS:780			if (!p->xnack_enabled)781				*update_mapping = true;782 783			gpuidx = kfd_process_gpuidx_from_gpuid(p,784							       attrs[i].value);785			if (attrs[i].type == KFD_IOCTL_SVM_ATTR_NO_ACCESS) {786				bitmap_clear(prange->bitmap_access, gpuidx, 1);787				bitmap_clear(prange->bitmap_aip, gpuidx, 1);788			} else if (attrs[i].type == KFD_IOCTL_SVM_ATTR_ACCESS) {789				bitmap_set(prange->bitmap_access, gpuidx, 1);790				bitmap_clear(prange->bitmap_aip, gpuidx, 1);791			} else {792				bitmap_clear(prange->bitmap_access, gpuidx, 1);793				bitmap_set(prange->bitmap_aip, gpuidx, 1);794			}795			break;796		case KFD_IOCTL_SVM_ATTR_SET_FLAGS:797			*update_mapping = true;798			prange->flags |= attrs[i].value;799			break;800		case KFD_IOCTL_SVM_ATTR_CLR_FLAGS:801			*update_mapping = true;802			prange->flags &= ~attrs[i].value;803			break;804		case KFD_IOCTL_SVM_ATTR_GRANULARITY:805			prange->granularity = min_t(uint32_t, attrs[i].value, 0x3F);806			break;807		default:808			WARN_ONCE(1, "svm_range_check_attrs wasn't called?");809		}810	}811}812 813static bool814svm_range_is_same_attrs(struct kfd_process *p, struct svm_range *prange,815			uint32_t nattr, struct kfd_ioctl_svm_attribute *attrs)816{817	uint32_t i;818	int gpuidx;819 820	for (i = 0; i < nattr; i++) {821		switch (attrs[i].type) {822		case KFD_IOCTL_SVM_ATTR_PREFERRED_LOC:823			if (prange->preferred_loc != attrs[i].value)824				return false;825			break;826		case KFD_IOCTL_SVM_ATTR_PREFETCH_LOC:827			/* Prefetch should always trigger a migration even828			 * if the value of the attribute didn't change.829			 */830			return false;831		case KFD_IOCTL_SVM_ATTR_ACCESS:832		case KFD_IOCTL_SVM_ATTR_ACCESS_IN_PLACE:833		case KFD_IOCTL_SVM_ATTR_NO_ACCESS:834			gpuidx = kfd_process_gpuidx_from_gpuid(p,835							       attrs[i].value);836			if (attrs[i].type == KFD_IOCTL_SVM_ATTR_NO_ACCESS) {837				if (test_bit(gpuidx, prange->bitmap_access) ||838				    test_bit(gpuidx, prange->bitmap_aip))839					return false;840			} else if (attrs[i].type == KFD_IOCTL_SVM_ATTR_ACCESS) {841				if (!test_bit(gpuidx, prange->bitmap_access))842					return false;843			} else {844				if (!test_bit(gpuidx, prange->bitmap_aip))845					return false;846			}847			break;848		case KFD_IOCTL_SVM_ATTR_SET_FLAGS:849			if ((prange->flags & attrs[i].value) != attrs[i].value)850				return false;851			break;852		case KFD_IOCTL_SVM_ATTR_CLR_FLAGS:853			if ((prange->flags & attrs[i].value) != 0)854				return false;855			break;856		case KFD_IOCTL_SVM_ATTR_GRANULARITY:857			if (prange->granularity != attrs[i].value)858				return false;859			break;860		default:861			WARN_ONCE(1, "svm_range_check_attrs wasn't called?");862		}863	}864 865	return true;866}867 868/**869 * svm_range_debug_dump - print all range information from svms870 * @svms: svm range list header871 *872 * debug output svm range start, end, prefetch location from svms873 * interval tree and link list874 *875 * Context: The caller must hold svms->lock876 */877static void svm_range_debug_dump(struct svm_range_list *svms)878{879	struct interval_tree_node *node;880	struct svm_range *prange;881 882	pr_debug("dump svms 0x%p list\n", svms);883	pr_debug("range\tstart\tpage\tend\t\tlocation\n");884 885	list_for_each_entry(prange, &svms->list, list) {886		pr_debug("0x%p 0x%lx\t0x%llx\t0x%llx\t0x%x\n",887			 prange, prange->start, prange->npages,888			 prange->start + prange->npages - 1,889			 prange->actual_loc);890	}891 892	pr_debug("dump svms 0x%p interval tree\n", svms);893	pr_debug("range\tstart\tpage\tend\t\tlocation\n");894	node = interval_tree_iter_first(&svms->objects, 0, ~0ULL);895	while (node) {896		prange = container_of(node, struct svm_range, it_node);897		pr_debug("0x%p 0x%lx\t0x%llx\t0x%llx\t0x%x\n",898			 prange, prange->start, prange->npages,899			 prange->start + prange->npages - 1,900			 prange->actual_loc);901		node = interval_tree_iter_next(node, 0, ~0ULL);902	}903}904 905static void *906svm_range_copy_array(void *psrc, size_t size, uint64_t num_elements,907		     uint64_t offset, uint64_t *vram_pages)908{909	unsigned char *src = (unsigned char *)psrc + offset;910	unsigned char *dst;911	uint64_t i;912 913	dst = kvmalloc_array(num_elements, size, GFP_KERNEL);914	if (!dst)915		return NULL;916 917	if (!vram_pages) {918		memcpy(dst, src, num_elements * size);919		return (void *)dst;920	}921 922	*vram_pages = 0;923	for (i = 0; i < num_elements; i++) {924		dma_addr_t *temp;925		temp = (dma_addr_t *)dst + i;926		*temp = *((dma_addr_t *)src + i);927		if (*temp&SVM_RANGE_VRAM_DOMAIN)928			(*vram_pages)++;929	}930 931	return (void *)dst;932}933 934static int935svm_range_copy_dma_addrs(struct svm_range *dst, struct svm_range *src)936{937	int i;938 939	for (i = 0; i < MAX_GPU_INSTANCE; i++) {940		if (!src->dma_addr[i])941			continue;942		dst->dma_addr[i] = svm_range_copy_array(src->dma_addr[i],943					sizeof(*src->dma_addr[i]), src->npages, 0, NULL);944		if (!dst->dma_addr[i])945			return -ENOMEM;946	}947 948	return 0;949}950 951static int952svm_range_split_array(void *ppnew, void *ppold, size_t size,953		      uint64_t old_start, uint64_t old_n,954		      uint64_t new_start, uint64_t new_n, uint64_t *new_vram_pages)955{956	unsigned char *new, *old, *pold;957	uint64_t d;958 959	if (!ppold)960		return 0;961	pold = *(unsigned char **)ppold;962	if (!pold)963		return 0;964 965	d = (new_start - old_start) * size;966	/* get dma addr array for new range and calculte its vram page number */967	new = svm_range_copy_array(pold, size, new_n, d, new_vram_pages);968	if (!new)969		return -ENOMEM;970	d = (new_start == old_start) ? new_n * size : 0;971	old = svm_range_copy_array(pold, size, old_n, d, NULL);972	if (!old) {973		kvfree(new);974		return -ENOMEM;975	}976	kvfree(pold);977	*(void **)ppold = old;978	*(void **)ppnew = new;979 980	return 0;981}982 983static int984svm_range_split_pages(struct svm_range *new, struct svm_range *old,985		      uint64_t start, uint64_t last)986{987	uint64_t npages = last - start + 1;988	int i, r;989 990	for (i = 0; i < MAX_GPU_INSTANCE; i++) {991		r = svm_range_split_array(&new->dma_addr[i], &old->dma_addr[i],992					  sizeof(*old->dma_addr[i]), old->start,993					  npages, new->start, new->npages,994					  old->actual_loc ? &new->vram_pages : NULL);995		if (r)996			return r;997	}998	if (old->actual_loc)999		old->vram_pages -= new->vram_pages;1000 1001	return 0;1002}1003 1004static int1005svm_range_split_nodes(struct svm_range *new, struct svm_range *old,1006		      uint64_t start, uint64_t last)1007{1008	uint64_t npages = last - start + 1;1009 1010	pr_debug("svms 0x%p new prange 0x%p start 0x%lx [0x%llx 0x%llx]\n",1011		 new->svms, new, new->start, start, last);1012 1013	if (new->start == old->start) {1014		new->offset = old->offset;1015		old->offset += new->npages;1016	} else {1017		new->offset = old->offset + npages;1018	}1019 1020	new->svm_bo = svm_range_bo_ref(old->svm_bo);1021	new->ttm_res = old->ttm_res;1022 1023	spin_lock(&new->svm_bo->list_lock);1024	list_add(&new->svm_bo_list, &new->svm_bo->range_list);1025	spin_unlock(&new->svm_bo->list_lock);1026 1027	return 0;1028}1029 1030/**1031 * svm_range_split_adjust - split range and adjust1032 *1033 * @new: new range1034 * @old: the old range1035 * @start: the old range adjust to start address in pages1036 * @last: the old range adjust to last address in pages1037 *1038 * Copy system memory dma_addr or vram ttm_res in old range to new1039 * range from new_start up to size new->npages, the remaining old range is from1040 * start to last1041 *1042 * Return:1043 * 0 - OK, -ENOMEM - out of memory1044 */1045static int1046svm_range_split_adjust(struct svm_range *new, struct svm_range *old,1047		      uint64_t start, uint64_t last)1048{1049	int r;1050 1051	pr_debug("svms 0x%p new 0x%lx old [0x%lx 0x%lx] => [0x%llx 0x%llx]\n",1052		 new->svms, new->start, old->start, old->last, start, last);1053 1054	if (new->start < old->start ||1055	    new->last > old->last) {1056		WARN_ONCE(1, "invalid new range start or last\n");1057		return -EINVAL;1058	}1059 1060	r = svm_range_split_pages(new, old, start, last);1061	if (r)1062		return r;1063 1064	if (old->actual_loc && old->ttm_res) {1065		r = svm_range_split_nodes(new, old, start, last);1066		if (r)1067			return r;1068	}1069 1070	old->npages = last - start + 1;1071	old->start = start;1072	old->last = last;1073	new->flags = old->flags;1074	new->preferred_loc = old->preferred_loc;1075	new->prefetch_loc = old->prefetch_loc;1076	new->actual_loc = old->actual_loc;1077	new->granularity = old->granularity;1078	new->mapped_to_gpu = old->mapped_to_gpu;1079	bitmap_copy(new->bitmap_access, old->bitmap_access, MAX_GPU_INSTANCE);1080	bitmap_copy(new->bitmap_aip, old->bitmap_aip, MAX_GPU_INSTANCE);1081	atomic_set(&new->queue_refcount, atomic_read(&old->queue_refcount));1082 1083	return 0;1084}1085 1086/**1087 * svm_range_split - split a range in 2 ranges1088 *1089 * @prange: the svm range to split1090 * @start: the remaining range start address in pages1091 * @last: the remaining range last address in pages1092 * @new: the result new range generated1093 *1094 * Two cases only:1095 * case 1: if start == prange->start1096 *         prange ==> prange[start, last]1097 *         new range [last + 1, prange->last]1098 *1099 * case 2: if last == prange->last1100 *         prange ==> prange[start, last]1101 *         new range [prange->start, start - 1]1102 *1103 * Return:1104 * 0 - OK, -ENOMEM - out of memory, -EINVAL - invalid start, last1105 */1106static int1107svm_range_split(struct svm_range *prange, uint64_t start, uint64_t last,1108		struct svm_range **new)1109{1110	uint64_t old_start = prange->start;1111	uint64_t old_last = prange->last;1112	struct svm_range_list *svms;1113	int r = 0;1114 1115	pr_debug("svms 0x%p [0x%llx 0x%llx] to [0x%llx 0x%llx]\n", prange->svms,1116		 old_start, old_last, start, last);1117 1118	if (old_start != start && old_last != last)1119		return -EINVAL;1120	if (start < old_start || last > old_last)1121		return -EINVAL;1122 1123	svms = prange->svms;1124	if (old_start == start)1125		*new = svm_range_new(svms, last + 1, old_last, false);1126	else1127		*new = svm_range_new(svms, old_start, start - 1, false);1128	if (!*new)1129		return -ENOMEM;1130 1131	r = svm_range_split_adjust(*new, prange, start, last);1132	if (r) {1133		pr_debug("failed %d split [0x%llx 0x%llx] to [0x%llx 0x%llx]\n",1134			 r, old_start, old_last, start, last);1135		svm_range_free(*new, false);1136		*new = NULL;1137	}1138 1139	return r;1140}1141 1142static int1143svm_range_split_tail(struct svm_range *prange, uint64_t new_last,1144		     struct list_head *insert_list, struct list_head *remap_list)1145{1146	struct svm_range *tail = NULL;1147	int r = svm_range_split(prange, prange->start, new_last, &tail);1148 1149	if (!r) {1150		list_add(&tail->list, insert_list);1151		if (!IS_ALIGNED(new_last + 1, 1UL << prange->granularity))1152			list_add(&tail->update_list, remap_list);1153	}1154	return r;1155}1156 1157static int1158svm_range_split_head(struct svm_range *prange, uint64_t new_start,1159		     struct list_head *insert_list, struct list_head *remap_list)1160{1161	struct svm_range *head = NULL;1162	int r = svm_range_split(prange, new_start, prange->last, &head);1163 1164	if (!r) {1165		list_add(&head->list, insert_list);1166		if (!IS_ALIGNED(new_start, 1UL << prange->granularity))1167			list_add(&head->update_list, remap_list);1168	}1169	return r;1170}1171 1172static void1173svm_range_add_child(struct svm_range *prange, struct mm_struct *mm,1174		    struct svm_range *pchild, enum svm_work_list_ops op)1175{1176	pr_debug("add child 0x%p [0x%lx 0x%lx] to prange 0x%p child list %d\n",1177		 pchild, pchild->start, pchild->last, prange, op);1178 1179	pchild->work_item.mm = mm;1180	pchild->work_item.op = op;1181	list_add_tail(&pchild->child_list, &prange->child_list);1182}1183 1184static bool1185svm_nodes_in_same_hive(struct kfd_node *node_a, struct kfd_node *node_b)1186{1187	return (node_a->adev == node_b->adev ||1188		amdgpu_xgmi_same_hive(node_a->adev, node_b->adev));1189}1190 1191static uint64_t1192svm_range_get_pte_flags(struct kfd_node *node,1193			struct svm_range *prange, int domain)1194{1195	struct kfd_node *bo_node;1196	uint32_t flags = prange->flags;1197	uint32_t mapping_flags = 0;1198	uint64_t pte_flags;1199	bool snoop = (domain != SVM_RANGE_VRAM_DOMAIN);1200	bool coherent = flags & (KFD_IOCTL_SVM_FLAG_COHERENT | KFD_IOCTL_SVM_FLAG_EXT_COHERENT);1201	bool ext_coherent = flags & KFD_IOCTL_SVM_FLAG_EXT_COHERENT;1202	unsigned int mtype_local;1203 1204	if (domain == SVM_RANGE_VRAM_DOMAIN)1205		bo_node = prange->svm_bo->node;1206 1207	switch (amdgpu_ip_version(node->adev, GC_HWIP, 0)) {1208	case IP_VERSION(9, 4, 1):1209		if (domain == SVM_RANGE_VRAM_DOMAIN) {1210			if (bo_node == node) {1211				mapping_flags |= coherent ?1212					AMDGPU_VM_MTYPE_CC : AMDGPU_VM_MTYPE_RW;1213			} else {1214				mapping_flags |= coherent ?1215					AMDGPU_VM_MTYPE_UC : AMDGPU_VM_MTYPE_NC;1216				if (svm_nodes_in_same_hive(node, bo_node))1217					snoop = true;1218			}1219		} else {1220			mapping_flags |= coherent ?1221				AMDGPU_VM_MTYPE_UC : AMDGPU_VM_MTYPE_NC;1222		}1223		break;1224	case IP_VERSION(9, 4, 2):1225		if (domain == SVM_RANGE_VRAM_DOMAIN) {1226			if (bo_node == node) {1227				mapping_flags |= coherent ?1228					AMDGPU_VM_MTYPE_CC : AMDGPU_VM_MTYPE_RW;1229				if (node->adev->gmc.xgmi.connected_to_cpu)1230					snoop = true;1231			} else {1232				mapping_flags |= coherent ?1233					AMDGPU_VM_MTYPE_UC : AMDGPU_VM_MTYPE_NC;1234				if (svm_nodes_in_same_hive(node, bo_node))1235					snoop = true;1236			}1237		} else {1238			mapping_flags |= coherent ?1239				AMDGPU_VM_MTYPE_UC : AMDGPU_VM_MTYPE_NC;1240		}1241		break;1242	case IP_VERSION(9, 4, 3):1243	case IP_VERSION(9, 4, 4):1244		if (ext_coherent)1245			mtype_local = node->adev->rev_id ? AMDGPU_VM_MTYPE_CC : AMDGPU_VM_MTYPE_UC;1246		else1247			mtype_local = amdgpu_mtype_local == 1 ? AMDGPU_VM_MTYPE_NC :1248				amdgpu_mtype_local == 2 ? AMDGPU_VM_MTYPE_CC : AMDGPU_VM_MTYPE_RW;1249		snoop = true;1250		if (domain == SVM_RANGE_VRAM_DOMAIN) {1251			/* local HBM region close to partition */1252			if (bo_node->adev == node->adev &&1253			    (!bo_node->xcp || !node->xcp || bo_node->xcp->mem_id == node->xcp->mem_id))1254				mapping_flags |= mtype_local;1255			/* local HBM region far from partition or remote XGMI GPU1256			 * with regular system scope coherence1257			 */1258			else if (svm_nodes_in_same_hive(bo_node, node) && !ext_coherent)1259				mapping_flags |= AMDGPU_VM_MTYPE_NC;1260			/* PCIe P2P or extended system scope coherence */1261			else1262				mapping_flags |= AMDGPU_VM_MTYPE_UC;1263		/* system memory accessed by the APU */1264		} else if (node->adev->flags & AMD_IS_APU) {1265			/* On NUMA systems, locality is determined per-page1266			 * in amdgpu_gmc_override_vm_pte_flags1267			 */1268			if (num_possible_nodes() <= 1)1269				mapping_flags |= mtype_local;1270			else1271				mapping_flags |= ext_coherent ? AMDGPU_VM_MTYPE_UC : AMDGPU_VM_MTYPE_NC;1272		/* system memory accessed by the dGPU */1273		} else {1274			mapping_flags |= AMDGPU_VM_MTYPE_UC;1275		}1276		break;1277	case IP_VERSION(12, 0, 0):1278	case IP_VERSION(12, 0, 1):1279		if (domain == SVM_RANGE_VRAM_DOMAIN) {1280			if (bo_node != node)1281				mapping_flags |= AMDGPU_VM_MTYPE_NC;1282		} else {1283			mapping_flags |= coherent ?1284				AMDGPU_VM_MTYPE_UC : AMDGPU_VM_MTYPE_NC;1285		}1286		break;1287	default:1288		mapping_flags |= coherent ?1289			AMDGPU_VM_MTYPE_UC : AMDGPU_VM_MTYPE_NC;1290	}1291 1292	mapping_flags |= AMDGPU_VM_PAGE_READABLE | AMDGPU_VM_PAGE_WRITEABLE;1293 1294	if (flags & KFD_IOCTL_SVM_FLAG_GPU_RO)1295		mapping_flags &= ~AMDGPU_VM_PAGE_WRITEABLE;1296	if (flags & KFD_IOCTL_SVM_FLAG_GPU_EXEC)1297		mapping_flags |= AMDGPU_VM_PAGE_EXECUTABLE;1298 1299	pte_flags = AMDGPU_PTE_VALID;1300	pte_flags |= (domain == SVM_RANGE_VRAM_DOMAIN) ? 0 : AMDGPU_PTE_SYSTEM;1301	pte_flags |= snoop ? AMDGPU_PTE_SNOOPED : 0;1302	if (KFD_GC_VERSION(node) >= IP_VERSION(12, 0, 0))1303		pte_flags |= AMDGPU_PTE_IS_PTE;1304 1305	pte_flags |= amdgpu_gem_va_map_flags(node->adev, mapping_flags);1306	return pte_flags;1307}1308 1309static int1310svm_range_unmap_from_gpu(struct amdgpu_device *adev, struct amdgpu_vm *vm,1311			 uint64_t start, uint64_t last,1312			 struct dma_fence **fence)1313{1314	uint64_t init_pte_value = 0;1315 1316	pr_debug("[0x%llx 0x%llx]\n", start, last);1317 1318	return amdgpu_vm_update_range(adev, vm, false, true, true, false, NULL, start,1319				      last, init_pte_value, 0, 0, NULL, NULL,1320				      fence);1321}1322 1323static int1324svm_range_unmap_from_gpus(struct svm_range *prange, unsigned long start,1325			  unsigned long last, uint32_t trigger)1326{1327	DECLARE_BITMAP(bitmap, MAX_GPU_INSTANCE);1328	struct kfd_process_device *pdd;1329	struct dma_fence *fence = NULL;1330	struct kfd_process *p;1331	uint32_t gpuidx;1332	int r = 0;1333 1334	if (!prange->mapped_to_gpu) {1335		pr_debug("prange 0x%p [0x%lx 0x%lx] not mapped to GPU\n",1336			 prange, prange->start, prange->last);1337		return 0;1338	}1339 1340	if (prange->start == start && prange->last == last) {1341		pr_debug("unmap svms 0x%p prange 0x%p\n", prange->svms, prange);1342		prange->mapped_to_gpu = false;1343	}1344 1345	bitmap_or(bitmap, prange->bitmap_access, prange->bitmap_aip,1346		  MAX_GPU_INSTANCE);1347	p = container_of(prange->svms, struct kfd_process, svms);1348 1349	for_each_set_bit(gpuidx, bitmap, MAX_GPU_INSTANCE) {1350		pr_debug("unmap from gpu idx 0x%x\n", gpuidx);1351		pdd = kfd_process_device_from_gpuidx(p, gpuidx);1352		if (!pdd) {1353			pr_debug("failed to find device idx %d\n", gpuidx);1354			return -EINVAL;1355		}1356 1357		kfd_smi_event_unmap_from_gpu(pdd->dev, p->lead_thread->pid,1358					     start, last, trigger);1359 1360		r = svm_range_unmap_from_gpu(pdd->dev->adev,1361					     drm_priv_to_vm(pdd->drm_priv),1362					     start, last, &fence);1363		if (r)1364			break;1365 1366		if (fence) {1367			r = dma_fence_wait(fence, false);1368			dma_fence_put(fence);1369			fence = NULL;1370			if (r)1371				break;1372		}1373		kfd_flush_tlb(pdd, TLB_FLUSH_HEAVYWEIGHT);1374	}1375 1376	return r;1377}1378 1379static int1380svm_range_map_to_gpu(struct kfd_process_device *pdd, struct svm_range *prange,1381		     unsigned long offset, unsigned long npages, bool readonly,1382		     dma_addr_t *dma_addr, struct amdgpu_device *bo_adev,1383		     struct dma_fence **fence, bool flush_tlb)1384{1385	struct amdgpu_device *adev = pdd->dev->adev;1386	struct amdgpu_vm *vm = drm_priv_to_vm(pdd->drm_priv);1387	uint64_t pte_flags;1388	unsigned long last_start;1389	int last_domain;1390	int r = 0;1391	int64_t i, j;1392 1393	last_start = prange->start + offset;1394 1395	pr_debug("svms 0x%p [0x%lx 0x%lx] readonly %d\n", prange->svms,1396		 last_start, last_start + npages - 1, readonly);1397 1398	for (i = offset; i < offset + npages; i++) {1399		last_domain = dma_addr[i] & SVM_RANGE_VRAM_DOMAIN;1400		dma_addr[i] &= ~SVM_RANGE_VRAM_DOMAIN;1401 1402		/* Collect all pages in the same address range and memory domain1403		 * that can be mapped with a single call to update mapping.1404		 */1405		if (i < offset + npages - 1 &&1406		    last_domain == (dma_addr[i + 1] & SVM_RANGE_VRAM_DOMAIN))1407			continue;1408 1409		pr_debug("Mapping range [0x%lx 0x%llx] on domain: %s\n",1410			 last_start, prange->start + i, last_domain ? "GPU" : "CPU");1411 1412		pte_flags = svm_range_get_pte_flags(pdd->dev, prange, last_domain);1413		if (readonly)1414			pte_flags &= ~AMDGPU_PTE_WRITEABLE;1415 1416		pr_debug("svms 0x%p map [0x%lx 0x%llx] vram %d PTE 0x%llx\n",1417			 prange->svms, last_start, prange->start + i,1418			 (last_domain == SVM_RANGE_VRAM_DOMAIN) ? 1 : 0,1419			 pte_flags);1420 1421		/* For dGPU mode, we use same vm_manager to allocate VRAM for1422		 * different memory partition based on fpfn/lpfn, we should use1423		 * same vm_manager.vram_base_offset regardless memory partition.1424		 */1425		r = amdgpu_vm_update_range(adev, vm, false, false, flush_tlb, true,1426					   NULL, last_start, prange->start + i,1427					   pte_flags,1428					   (last_start - prange->start) << PAGE_SHIFT,1429					   bo_adev ? bo_adev->vm_manager.vram_base_offset : 0,1430					   NULL, dma_addr, &vm->last_update);1431 1432		for (j = last_start - prange->start; j <= i; j++)1433			dma_addr[j] |= last_domain;1434 1435		if (r) {1436			pr_debug("failed %d to map to gpu 0x%lx\n", r, prange->start);1437			goto out;1438		}1439		last_start = prange->start + i + 1;1440	}1441 1442	r = amdgpu_vm_update_pdes(adev, vm, false);1443	if (r) {1444		pr_debug("failed %d to update directories 0x%lx\n", r,1445			 prange->start);1446		goto out;1447	}1448 1449	if (fence)1450		*fence = dma_fence_get(vm->last_update);1451 1452out:1453	return r;1454}1455 1456static int1457svm_range_map_to_gpus(struct svm_range *prange, unsigned long offset,1458		      unsigned long npages, bool readonly,1459		      unsigned long *bitmap, bool wait, bool flush_tlb)1460{1461	struct kfd_process_device *pdd;1462	struct amdgpu_device *bo_adev = NULL;1463	struct kfd_process *p;1464	struct dma_fence *fence = NULL;1465	uint32_t gpuidx;1466	int r = 0;1467 1468	if (prange->svm_bo && prange->ttm_res)1469		bo_adev = prange->svm_bo->node->adev;1470 1471	p = container_of(prange->svms, struct kfd_process, svms);1472	for_each_set_bit(gpuidx, bitmap, MAX_GPU_INSTANCE) {1473		pr_debug("mapping to gpu idx 0x%x\n", gpuidx);1474		pdd = kfd_process_device_from_gpuidx(p, gpuidx);1475		if (!pdd) {1476			pr_debug("failed to find device idx %d\n", gpuidx);1477			return -EINVAL;1478		}1479 1480		pdd = kfd_bind_process_to_device(pdd->dev, p);1481		if (IS_ERR(pdd))1482			return -EINVAL;1483 1484		if (bo_adev && pdd->dev->adev != bo_adev &&1485		    !amdgpu_xgmi_same_hive(pdd->dev->adev, bo_adev)) {1486			pr_debug("cannot map to device idx %d\n", gpuidx);1487			continue;1488		}1489 1490		r = svm_range_map_to_gpu(pdd, prange, offset, npages, readonly,1491					 prange->dma_addr[gpuidx],1492					 bo_adev, wait ? &fence : NULL,1493					 flush_tlb);1494		if (r)1495			break;1496 1497		if (fence) {1498			r = dma_fence_wait(fence, false);1499			dma_fence_put(fence);1500			fence = NULL;1501			if (r) {1502				pr_debug("failed %d to dma fence wait\n", r);1503				break;1504			}1505		}1506 1507		kfd_flush_tlb(pdd, TLB_FLUSH_LEGACY);1508	}1509 1510	return r;1511}1512 1513struct svm_validate_context {1514	struct kfd_process *process;1515	struct svm_range *prange;1516	bool intr;1517	DECLARE_BITMAP(bitmap, MAX_GPU_INSTANCE);1518	struct drm_exec exec;1519};1520 1521static int svm_range_reserve_bos(struct svm_validate_context *ctx, bool intr)1522{1523	struct kfd_process_device *pdd;1524	struct amdgpu_vm *vm;1525	uint32_t gpuidx;1526	int r;1527 1528	drm_exec_init(&ctx->exec, intr ? DRM_EXEC_INTERRUPTIBLE_WAIT: 0, 0);1529	drm_exec_until_all_locked(&ctx->exec) {1530		for_each_set_bit(gpuidx, ctx->bitmap, MAX_GPU_INSTANCE) {1531			pdd = kfd_process_device_from_gpuidx(ctx->process, gpuidx);1532			if (!pdd) {1533				pr_debug("failed to find device idx %d\n", gpuidx);1534				r = -EINVAL;1535				goto unreserve_out;1536			}1537			vm = drm_priv_to_vm(pdd->drm_priv);1538 1539			r = amdgpu_vm_lock_pd(vm, &ctx->exec, 2);1540			drm_exec_retry_on_contention(&ctx->exec);1541			if (unlikely(r)) {1542				pr_debug("failed %d to reserve bo\n", r);1543				goto unreserve_out;1544			}1545		}1546	}1547 1548	for_each_set_bit(gpuidx, ctx->bitmap, MAX_GPU_INSTANCE) {1549		pdd = kfd_process_device_from_gpuidx(ctx->process, gpuidx);1550		if (!pdd) {1551			pr_debug("failed to find device idx %d\n", gpuidx);1552			r = -EINVAL;1553			goto unreserve_out;1554		}1555 1556		r = amdgpu_vm_validate(pdd->dev->adev,1557				       drm_priv_to_vm(pdd->drm_priv), NULL,1558				       svm_range_bo_validate, NULL);1559		if (r) {1560			pr_debug("failed %d validate pt bos\n", r);1561			goto unreserve_out;1562		}1563	}1564 1565	return 0;1566 1567unreserve_out:1568	drm_exec_fini(&ctx->exec);1569	return r;1570}1571 1572static void svm_range_unreserve_bos(struct svm_validate_context *ctx)1573{1574	drm_exec_fini(&ctx->exec);1575}1576 1577static void *kfd_svm_page_owner(struct kfd_process *p, int32_t gpuidx)1578{1579	struct kfd_process_device *pdd;1580 1581	pdd = kfd_process_device_from_gpuidx(p, gpuidx);1582	if (!pdd)1583		return NULL;1584 1585	return SVM_ADEV_PGMAP_OWNER(pdd->dev->adev);1586}1587 1588/*1589 * Validation+GPU mapping with concurrent invalidation (MMU notifiers)1590 *1591 * To prevent concurrent destruction or change of range attributes, the1592 * svm_read_lock must be held. The caller must not hold the svm_write_lock1593 * because that would block concurrent evictions and lead to deadlocks. To1594 * serialize concurrent migrations or validations of the same range, the1595 * prange->migrate_mutex must be held.1596 *1597 * For VRAM ranges, the SVM BO must be allocated and valid (protected by its1598 * eviction fence.1599 *1600 * The following sequence ensures race-free validation and GPU mapping:1601 *1602 * 1. Reserve page table (and SVM BO if range is in VRAM)1603 * 2. hmm_range_fault to get page addresses (if system memory)1604 * 3. DMA-map pages (if system memory)1605 * 4-a. Take notifier lock1606 * 4-b. Check that pages still valid (mmu_interval_read_retry)1607 * 4-c. Check that the range was not split or otherwise invalidated1608 * 4-d. Update GPU page table1609 * 4.e. Release notifier lock1610 * 5. Release page table (and SVM BO) reservation1611 */1612static int svm_range_validate_and_map(struct mm_struct *mm,1613				      unsigned long map_start, unsigned long map_last,1614				      struct svm_range *prange, int32_t gpuidx,1615				      bool intr, bool wait, bool flush_tlb)1616{1617	struct svm_validate_context *ctx;1618	unsigned long start, end, addr;1619	struct kfd_process *p;1620	void *owner;1621	int32_t idx;1622	int r = 0;1623 1624	ctx = kzalloc(sizeof(struct svm_validate_context), GFP_KERNEL);1625	if (!ctx)1626		return -ENOMEM;1627	ctx->process = container_of(prange->svms, struct kfd_process, svms);1628	ctx->prange = prange;1629	ctx->intr = intr;1630 1631	if (gpuidx < MAX_GPU_INSTANCE) {1632		bitmap_zero(ctx->bitmap, MAX_GPU_INSTANCE);1633		bitmap_set(ctx->bitmap, gpuidx, 1);1634	} else if (ctx->process->xnack_enabled) {1635		bitmap_copy(ctx->bitmap, prange->bitmap_aip, MAX_GPU_INSTANCE);1636 1637		/* If prefetch range to GPU, or GPU retry fault migrate range to1638		 * GPU, which has ACCESS attribute to the range, create mapping1639		 * on that GPU.1640		 */1641		if (prange->actual_loc) {1642			gpuidx = kfd_process_gpuidx_from_gpuid(ctx->process,1643							prange->actual_loc);1644			if (gpuidx < 0) {1645				WARN_ONCE(1, "failed get device by id 0x%x\n",1646					 prange->actual_loc);1647				r = -EINVAL;1648				goto free_ctx;1649			}1650			if (test_bit(gpuidx, prange->bitmap_access))1651				bitmap_set(ctx->bitmap, gpuidx, 1);1652		}1653 1654		/*1655		 * If prange is already mapped or with always mapped flag,1656		 * update mapping on GPUs with ACCESS attribute1657		 */1658		if (bitmap_empty(ctx->bitmap, MAX_GPU_INSTANCE)) {1659			if (prange->mapped_to_gpu ||1660			    prange->flags & KFD_IOCTL_SVM_FLAG_GPU_ALWAYS_MAPPED)1661				bitmap_copy(ctx->bitmap, prange->bitmap_access, MAX_GPU_INSTANCE);1662		}1663	} else {1664		bitmap_or(ctx->bitmap, prange->bitmap_access,1665			  prange->bitmap_aip, MAX_GPU_INSTANCE);1666	}1667 1668	if (bitmap_empty(ctx->bitmap, MAX_GPU_INSTANCE)) {1669		r = 0;1670		goto free_ctx;1671	}1672 1673	if (prange->actual_loc && !prange->ttm_res) {1674		/* This should never happen. actual_loc gets set by1675		 * svm_migrate_ram_to_vram after allocating a BO.1676		 */1677		WARN_ONCE(1, "VRAM BO missing during validation\n");1678		r = -EINVAL;1679		goto free_ctx;1680	}1681 1682	r = svm_range_reserve_bos(ctx, intr);1683	if (r)1684		goto free_ctx;1685 1686	p = container_of(prange->svms, struct kfd_process, svms);1687	owner = kfd_svm_page_owner(p, find_first_bit(ctx->bitmap,1688						MAX_GPU_INSTANCE));1689	for_each_set_bit(idx, ctx->bitmap, MAX_GPU_INSTANCE) {1690		if (kfd_svm_page_owner(p, idx) != owner) {1691			owner = NULL;1692			break;1693		}1694	}1695 1696	start = map_start << PAGE_SHIFT;1697	end = (map_last + 1) << PAGE_SHIFT;1698	for (addr = start; !r && addr < end; ) {1699		struct hmm_range *hmm_range = NULL;1700		unsigned long map_start_vma;1701		unsigned long map_last_vma;1702		struct vm_area_struct *vma;1703		unsigned long next = 0;1704		unsigned long offset;1705		unsigned long npages;1706		bool readonly;1707 1708		vma = vma_lookup(mm, addr);1709		if (vma) {1710			readonly = !(vma->vm_flags & VM_WRITE);1711 1712			next = min(vma->vm_end, end);1713			npages = (next - addr) >> PAGE_SHIFT;1714			WRITE_ONCE(p->svms.faulting_task, current);1715			r = amdgpu_hmm_range_get_pages(&prange->notifier, addr, npages,1716						       readonly, owner, NULL,1717						       &hmm_range);1718			WRITE_ONCE(p->svms.faulting_task, NULL);1719			if (r)1720				pr_debug("failed %d to get svm range pages\n", r);1721		} else {1722			r = -EFAULT;1723		}1724 1725		if (!r) {1726			offset = (addr >> PAGE_SHIFT) - prange->start;1727			r = svm_range_dma_map(prange, ctx->bitmap, offset, npages,1728					      hmm_range->hmm_pfns);1729			if (r)1730				pr_debug("failed %d to dma map range\n", r);1731		}1732 1733		svm_range_lock(prange);1734 1735		/* Free backing memory of hmm_range if it was initialized1736		 * Overrride return value to TRY AGAIN only if prior returns1737		 * were successful1738		 */1739		if (hmm_range && amdgpu_hmm_range_get_pages_done(hmm_range) && !r) {1740			pr_debug("hmm update the range, need validate again\n");1741			r = -EAGAIN;1742		}1743 1744		if (!r && !list_empty(&prange->child_list)) {1745			pr_debug("range split by unmap in parallel, validate again\n");1746			r = -EAGAIN;1747		}1748 1749		if (!r) {1750			map_start_vma = max(map_start, prange->start + offset);1751			map_last_vma = min(map_last, prange->start + offset + npages - 1);1752			if (map_start_vma <= map_last_vma) {1753				offset = map_start_vma - prange->start;1754				npages = map_last_vma - map_start_vma + 1;1755				r = svm_range_map_to_gpus(prange, offset, npages, readonly,1756							  ctx->bitmap, wait, flush_tlb);1757			}1758		}1759 1760		if (!r && next == end)1761			prange->mapped_to_gpu = true;1762 1763		svm_range_unlock(prange);1764 1765		addr = next;1766	}1767 1768	svm_range_unreserve_bos(ctx);1769	if (!r)1770		prange->validate_timestamp = ktime_get_boottime();1771 1772free_ctx:1773	kfree(ctx);1774 1775	return r;1776}1777 1778/**1779 * svm_range_list_lock_and_flush_work - flush pending deferred work1780 *1781 * @svms: the svm range list1782 * @mm: the mm structure1783 *1784 * Context: Returns with mmap write lock held, pending deferred work flushed1785 *1786 */1787void1788svm_range_list_lock_and_flush_work(struct svm_range_list *svms,1789				   struct mm_struct *mm)1790{1791retry_flush_work:1792	flush_work(&svms->deferred_list_work);1793	mmap_write_lock(mm);1794 1795	if (list_empty(&svms->deferred_range_list))1796		return;1797	mmap_write_unlock(mm);1798	pr_debug("retry flush\n");1799	goto retry_flush_work;1800}1801 1802static void svm_range_restore_work(struct work_struct *work)1803{1804	struct delayed_work *dwork = to_delayed_work(work);1805	struct amdkfd_process_info *process_info;1806	struct svm_range_list *svms;1807	struct svm_range *prange;1808	struct kfd_process *p;1809	struct mm_struct *mm;1810	int evicted_ranges;1811	int invalid;1812	int r;1813 1814	svms = container_of(dwork, struct svm_range_list, restore_work);1815	evicted_ranges = atomic_read(&svms->evicted_ranges);1816	if (!evicted_ranges)1817		return;1818 1819	pr_debug("restore svm ranges\n");1820 1821	p = container_of(svms, struct kfd_process, svms);1822	process_info = p->kgd_process_info;1823 1824	/* Keep mm reference when svm_range_validate_and_map ranges */1825	mm = get_task_mm(p->lead_thread);1826	if (!mm) {1827		pr_debug("svms 0x%p process mm gone\n", svms);1828		return;1829	}1830 1831	mutex_lock(&process_info->lock);1832	svm_range_list_lock_and_flush_work(svms, mm);1833	mutex_lock(&svms->lock);1834 1835	evicted_ranges = atomic_read(&svms->evicted_ranges);1836 1837	list_for_each_entry(prange, &svms->list, list) {1838		invalid = atomic_read(&prange->invalid);1839		if (!invalid)1840			continue;1841 1842		pr_debug("restoring svms 0x%p prange 0x%p [0x%lx %lx] inv %d\n",1843			 prange->svms, prange, prange->start, prange->last,1844			 invalid);1845 1846		/*1847		 * If range is migrating, wait for migration is done.1848		 */1849		mutex_lock(&prange->migrate_mutex);1850 1851		r = svm_range_validate_and_map(mm, prange->start, prange->last, prange,1852					       MAX_GPU_INSTANCE, false, true, false);1853		if (r)1854			pr_debug("failed %d to map 0x%lx to gpus\n", r,1855				 prange->start);1856 1857		mutex_unlock(&prange->migrate_mutex);1858		if (r)1859			goto out_reschedule;1860 1861		if (atomic_cmpxchg(&prange->invalid, invalid, 0) != invalid)1862			goto out_reschedule;1863	}1864 1865	if (atomic_cmpxchg(&svms->evicted_ranges, evicted_ranges, 0) !=1866	    evicted_ranges)1867		goto out_reschedule;1868 1869	evicted_ranges = 0;1870 1871	r = kgd2kfd_resume_mm(mm);1872	if (r) {1873		/* No recovery from this failure. Probably the CP is1874		 * hanging. No point trying again.1875		 */1876		pr_debug("failed %d to resume KFD\n", r);1877	}1878 1879	pr_debug("restore svm ranges successfully\n");1880 1881out_reschedule:1882	mutex_unlock(&svms->lock);1883	mmap_write_unlock(mm);1884	mutex_unlock(&process_info->lock);1885 1886	/* If validation failed, reschedule another attempt */1887	if (evicted_ranges) {1888		pr_debug("reschedule to restore svm range\n");1889		queue_delayed_work(system_freezable_wq, &svms->restore_work,1890			msecs_to_jiffies(AMDGPU_SVM_RANGE_RESTORE_DELAY_MS));1891 1892		kfd_smi_event_queue_restore_rescheduled(mm);1893	}1894	mmput(mm);1895}1896 1897/**1898 * svm_range_evict - evict svm range1899 * @prange: svm range structure1900 * @mm: current process mm_struct1901 * @start: starting process queue number1902 * @last: last process queue number1903 * @event: mmu notifier event when range is evicted or migrated1904 *1905 * Stop all queues of the process to ensure GPU doesn't access the memory, then1906 * return to let CPU evict the buffer and proceed CPU pagetable update.1907 *1908 * Don't need use lock to sync cpu pagetable invalidation with GPU execution.1909 * If invalidation happens while restore work is running, restore work will1910 * restart to ensure to get the latest CPU pages mapping to GPU, then start1911 * the queues.1912 */1913static int1914svm_range_evict(struct svm_range *prange, struct mm_struct *mm,1915		unsigned long start, unsigned long last,1916		enum mmu_notifier_event event)1917{1918	struct svm_range_list *svms = prange->svms;1919	struct svm_range *pchild;1920	struct kfd_process *p;1921	int r = 0;1922 1923	p = container_of(svms, struct kfd_process, svms);1924 1925	pr_debug("invalidate svms 0x%p prange [0x%lx 0x%lx] [0x%lx 0x%lx]\n",1926		 svms, prange->start, prange->last, start, last);1927 1928	if (!p->xnack_enabled ||1929	    (prange->flags & KFD_IOCTL_SVM_FLAG_GPU_ALWAYS_MAPPED)) {1930		int evicted_ranges;1931		bool mapped = prange->mapped_to_gpu;1932 1933		list_for_each_entry(pchild, &prange->child_list, child_list) {1934			if (!pchild->mapped_to_gpu)1935				continue;1936			mapped = true;1937			mutex_lock_nested(&pchild->lock, 1);1938			if (pchild->start <= last && pchild->last >= start) {1939				pr_debug("increment pchild invalid [0x%lx 0x%lx]\n",1940					 pchild->start, pchild->last);1941				atomic_inc(&pchild->invalid);1942			}1943			mutex_unlock(&pchild->lock);1944		}1945 1946		if (!mapped)1947			return r;1948 1949		if (prange->start <= last && prange->last >= start)1950			atomic_inc(&prange->invalid);1951 1952		evicted_ranges = atomic_inc_return(&svms->evicted_ranges);1953		if (evicted_ranges != 1)1954			return r;1955 1956		pr_debug("evicting svms 0x%p range [0x%lx 0x%lx]\n",1957			 prange->svms, prange->start, prange->last);1958 1959		/* First eviction, stop the queues */1960		r = kgd2kfd_quiesce_mm(mm, KFD_QUEUE_EVICTION_TRIGGER_SVM);1961		if (r)1962			pr_debug("failed to quiesce KFD\n");1963 1964		pr_debug("schedule to restore svm %p ranges\n", svms);1965		queue_delayed_work(system_freezable_wq, &svms->restore_work,1966			msecs_to_jiffies(AMDGPU_SVM_RANGE_RESTORE_DELAY_MS));1967	} else {1968		unsigned long s, l;1969		uint32_t trigger;1970 1971		if (event == MMU_NOTIFY_MIGRATE)1972			trigger = KFD_SVM_UNMAP_TRIGGER_MMU_NOTIFY_MIGRATE;1973		else1974			trigger = KFD_SVM_UNMAP_TRIGGER_MMU_NOTIFY;1975 1976		pr_debug("invalidate unmap svms 0x%p [0x%lx 0x%lx] from GPUs\n",1977			 prange->svms, start, last);1978		list_for_each_entry(pchild, &prange->child_list, child_list) {1979			mutex_lock_nested(&pchild->lock, 1);1980			s = max(start, pchild->start);1981			l = min(last, pchild->last);1982			if (l >= s)1983				svm_range_unmap_from_gpus(pchild, s, l, trigger);1984			mutex_unlock(&pchild->lock);1985		}1986		s = max(start, prange->start);1987		l = min(last, prange->last);1988		if (l >= s)1989			svm_range_unmap_from_gpus(prange, s, l, trigger);1990	}1991 1992	return r;1993}1994 1995static struct svm_range *svm_range_clone(struct svm_range *old)1996{1997	struct svm_range *new;1998 1999	new = svm_range_new(old->svms, old->start, old->last, false);2000	if (!new)2001		return NULL;2002	if (svm_range_copy_dma_addrs(new, old)) {2003		svm_range_free(new, false);2004		return NULL;2005	}2006	if (old->svm_bo) {2007		new->ttm_res = old->ttm_res;2008		new->offset = old->offset;2009		new->svm_bo = svm_range_bo_ref(old->svm_bo);2010		spin_lock(&new->svm_bo->list_lock);2011		list_add(&new->svm_bo_list, &new->svm_bo->range_list);2012		spin_unlock(&new->svm_bo->list_lock);2013	}2014	new->flags = old->flags;2015	new->preferred_loc = old->preferred_loc;2016	new->prefetch_loc = old->prefetch_loc;2017	new->actual_loc = old->actual_loc;2018	new->granularity = old->granularity;2019	new->mapped_to_gpu = old->mapped_to_gpu;2020	new->vram_pages = old->vram_pages;2021	bitmap_copy(new->bitmap_access, old->bitmap_access, MAX_GPU_INSTANCE);2022	bitmap_copy(new->bitmap_aip, old->bitmap_aip, MAX_GPU_INSTANCE);2023	atomic_set(&new->queue_refcount, atomic_read(&old->queue_refcount));2024 2025	return new;2026}2027 2028void svm_range_set_max_pages(struct amdgpu_device *adev)2029{2030	uint64_t max_pages;2031	uint64_t pages, _pages;2032	uint64_t min_pages = 0;2033	int i, id;2034 2035	for (i = 0; i < adev->kfd.dev->num_nodes; i++) {2036		if (adev->kfd.dev->nodes[i]->xcp)2037			id = adev->kfd.dev->nodes[i]->xcp->id;2038		else2039			id = -1;2040		pages = KFD_XCP_MEMORY_SIZE(adev, id) >> 17;2041		pages = clamp(pages, 1ULL << 9, 1ULL << 18);2042		pages = rounddown_pow_of_two(pages);2043		min_pages = min_not_zero(min_pages, pages);2044	}2045 2046	do {2047		max_pages = READ_ONCE(max_svm_range_pages);2048		_pages = min_not_zero(max_pages, min_pages);2049	} while (cmpxchg(&max_svm_range_pages, max_pages, _pages) != max_pages);2050}2051 2052static int2053svm_range_split_new(struct svm_range_list *svms, uint64_t start, uint64_t last,2054		    uint64_t max_pages, struct list_head *insert_list,2055		    struct list_head *update_list)2056{2057	struct svm_range *prange;2058	uint64_t l;2059 2060	pr_debug("max_svm_range_pages 0x%llx adding [0x%llx 0x%llx]\n",2061		 max_pages, start, last);2062 2063	while (last >= start) {2064		l = min(last, ALIGN_DOWN(start + max_pages, max_pages) - 1);2065 2066		prange = svm_range_new(svms, start, l, true);2067		if (!prange)2068			return -ENOMEM;2069		list_add(&prange->list, insert_list);2070		list_add(&prange->update_list, update_list);2071 2072		start = l + 1;2073	}2074	return 0;2075}2076 2077/**2078 * svm_range_add - add svm range and handle overlap2079 * @p: the range add to this process svms2080 * @start: page size aligned2081 * @size: page size aligned2082 * @nattr: number of attributes2083 * @attrs: array of attributes2084 * @update_list: output, the ranges need validate and update GPU mapping2085 * @insert_list: output, the ranges need insert to svms2086 * @remove_list: output, the ranges are replaced and need remove from svms2087 * @remap_list: output, remap unaligned svm ranges2088 *2089 * Check if the virtual address range has overlap with any existing ranges,2090 * split partly overlapping ranges and add new ranges in the gaps. All changes2091 * should be applied to the range_list and interval tree transactionally. If2092 * any range split or allocation fails, the entire update fails. Therefore any2093 * existing overlapping svm_ranges are cloned and the original svm_ranges left2094 * unchanged.2095 *2096 * If the transaction succeeds, the caller can update and insert clones and2097 * new ranges, then free the originals.2098 *2099 * Otherwise the caller can free the clones and new ranges, while the old2100 * svm_ranges remain unchanged.2101 *2102 * Context: Process context, caller must hold svms->lock2103 *2104 * Return:2105 * 0 - OK, otherwise error code2106 */2107static int2108svm_range_add(struct kfd_process *p, uint64_t start, uint64_t size,2109	      uint32_t nattr, struct kfd_ioctl_svm_attribute *attrs,2110	      struct list_head *update_list, struct list_head *insert_list,2111	      struct list_head *remove_list, struct list_head *remap_list)2112{2113	unsigned long last = start + size - 1UL;2114	struct svm_range_list *svms = &p->svms;2115	struct interval_tree_node *node;2116	struct svm_range *prange;2117	struct svm_range *tmp;2118	struct list_head new_list;2119	int r = 0;2120 2121	pr_debug("svms 0x%p [0x%llx 0x%lx]\n", &p->svms, start, last);2122 2123	INIT_LIST_HEAD(update_list);2124	INIT_LIST_HEAD(insert_list);2125	INIT_LIST_HEAD(remove_list);2126	INIT_LIST_HEAD(&new_list);2127	INIT_LIST_HEAD(remap_list);2128 2129	node = interval_tree_iter_first(&svms->objects, start, last);2130	while (node) {2131		struct interval_tree_node *next;2132		unsigned long next_start;2133 2134		pr_debug("found overlap node [0x%lx 0x%lx]\n", node->start,2135			 node->last);2136 2137		prange = container_of(node, struct svm_range, it_node);2138		next = interval_tree_iter_next(node, start, last);2139		next_start = min(node->last, last) + 1;2140 2141		if (svm_range_is_same_attrs(p, prange, nattr, attrs) &&2142		    prange->mapped_to_gpu) {2143			/* nothing to do */2144		} else if (node->start < start || node->last > last) {2145			/* node intersects the update range and its attributes2146			 * will change. Clone and split it, apply updates only2147			 * to the overlapping part2148			 */2149			struct svm_range *old = prange;2150 2151			prange = svm_range_clone(old);2152			if (!prange) {2153				r = -ENOMEM;2154				goto out;2155			}2156 2157			list_add(&old->update_list, remove_list);2158			list_add(&prange->list, insert_list);2159			list_add(&prange->update_list, update_list);2160 2161			if (node->start < start) {2162				pr_debug("change old range start\n");2163				r = svm_range_split_head(prange, start,2164							 insert_list, remap_list);2165				if (r)2166					goto out;2167			}2168			if (node->last > last) {2169				pr_debug("change old range last\n");2170				r = svm_range_split_tail(prange, last,2171							 insert_list, remap_list);2172				if (r)2173					goto out;2174			}2175		} else {2176			/* The node is contained within start..last,2177			 * just update it2178			 */2179			list_add(&prange->update_list, update_list);2180		}2181 2182		/* insert a new node if needed */2183		if (node->start > start) {2184			r = svm_range_split_new(svms, start, node->start - 1,2185						READ_ONCE(max_svm_range_pages),2186						&new_list, update_list);2187			if (r)2188				goto out;2189		}2190 2191		node = next;2192		start = next_start;2193	}2194 2195	/* add a final range at the end if needed */2196	if (start <= last)2197		r = svm_range_split_new(svms, start, last,2198					READ_ONCE(max_svm_range_pages),2199					&new_list, update_list);2200 2201out:2202	if (r) {2203		list_for_each_entry_safe(prange, tmp, insert_list, list)2204			svm_range_free(prange, false);2205		list_for_each_entry_safe(prange, tmp, &new_list, list)2206			svm_range_free(prange, true);2207	} else {2208		list_splice(&new_list, insert_list);2209	}2210 2211	return r;2212}2213 2214static void2215svm_range_update_notifier_and_interval_tree(struct mm_struct *mm,2216					    struct svm_range *prange)2217{2218	unsigned long start;2219	unsigned long last;2220 2221	start = prange->notifier.interval_tree.start >> PAGE_SHIFT;2222	last = prange->notifier.interval_tree.last >> PAGE_SHIFT;2223 2224	if (prange->start == start && prange->last == last)2225		return;2226 2227	pr_debug("up notifier 0x%p prange 0x%p [0x%lx 0x%lx] [0x%lx 0x%lx]\n",2228		  prange->svms, prange, start, last, prange->start,2229		  prange->last);2230 2231	if (start != 0 && last != 0) {2232		interval_tree_remove(&prange->it_node, &prange->svms->objects);2233		svm_range_remove_notifier(prange);2234	}2235	prange->it_node.start = prange->start;2236	prange->it_node.last = prange->last;2237 2238	interval_tree_insert(&prange->it_node, &prange->svms->objects);2239	svm_range_add_notifier_locked(mm, prange);2240}2241 2242static void2243svm_range_handle_list_op(struct svm_range_list *svms, struct svm_range *prange,2244			 struct mm_struct *mm)2245{2246	switch (prange->work_item.op) {2247	case SVM_OP_NULL:2248		pr_debug("NULL OP 0x%p prange 0x%p [0x%lx 0x%lx]\n",2249			 svms, prange, prange->start, prange->last);2250		break;2251	case SVM_OP_UNMAP_RANGE:2252		pr_debug("remove 0x%p prange 0x%p [0x%lx 0x%lx]\n",2253			 svms, prange, prange->start, prange->last);2254		svm_range_unlink(prange);2255		svm_range_remove_notifier(prange);2256		svm_range_free(prange, true);2257		break;2258	case SVM_OP_UPDATE_RANGE_NOTIFIER:2259		pr_debug("update notifier 0x%p prange 0x%p [0x%lx 0x%lx]\n",2260			 svms, prange, prange->start, prange->last);2261		svm_range_update_notifier_and_interval_tree(mm, prange);2262		break;2263	case SVM_OP_UPDATE_RANGE_NOTIFIER_AND_MAP:2264		pr_debug("update and map 0x%p prange 0x%p [0x%lx 0x%lx]\n",2265			 svms, prange, prange->start, prange->last);2266		svm_range_update_notifier_and_interval_tree(mm, prange);2267		/* TODO: implement deferred validation and mapping */2268		break;2269	case SVM_OP_ADD_RANGE:2270		pr_debug("add 0x%p prange 0x%p [0x%lx 0x%lx]\n", svms, prange,2271			 prange->start, prange->last);2272		svm_range_add_to_svms(prange);2273		svm_range_add_notifier_locked(mm, prange);2274		break;2275	case SVM_OP_ADD_RANGE_AND_MAP:2276		pr_debug("add and map 0x%p prange 0x%p [0x%lx 0x%lx]\n", svms,2277			 prange, prange->start, prange->last);2278		svm_range_add_to_svms(prange);2279		svm_range_add_notifier_locked(mm, prange);2280		/* TODO: implement deferred validation and mapping */2281		break;2282	default:2283		WARN_ONCE(1, "Unknown prange 0x%p work op %d\n", prange,2284			 prange->work_item.op);2285	}2286}2287 2288static void svm_range_drain_retry_fault(struct svm_range_list *svms)2289{2290	struct kfd_process_device *pdd;2291	struct kfd_process *p;2292	uint32_t i;2293 2294	p = container_of(svms, struct kfd_process, svms);2295 2296	for_each_set_bit(i, svms->bitmap_supported, p->n_pdds) {2297		pdd = p->pdds[i];2298		if (!pdd)2299			continue;2300 2301		pr_debug("drain retry fault gpu %d svms %p\n", i, svms);2302 2303		amdgpu_ih_wait_on_checkpoint_process_ts(pdd->dev->adev,2304				pdd->dev->adev->irq.retry_cam_enabled ?2305				&pdd->dev->adev->irq.ih :2306				&pdd->dev->adev->irq.ih1);2307 2308		if (pdd->dev->adev->irq.retry_cam_enabled)2309			amdgpu_ih_wait_on_checkpoint_process_ts(pdd->dev->adev,2310				&pdd->dev->adev->irq.ih_soft);2311 2312 2313		pr_debug("drain retry fault gpu %d svms 0x%p done\n", i, svms);2314	}2315}2316 2317static void svm_range_deferred_list_work(struct work_struct *work)2318{2319	struct svm_range_list *svms;2320	struct svm_range *prange;2321	struct mm_struct *mm;2322 2323	svms = container_of(work, struct svm_range_list, deferred_list_work);2324	pr_debug("enter svms 0x%p\n", svms);2325 2326	spin_lock(&svms->deferred_list_lock);2327	while (!list_empty(&svms->deferred_range_list)) {2328		prange = list_first_entry(&svms->deferred_range_list,2329					  struct svm_range, deferred_list);2330		spin_unlock(&svms->deferred_list_lock);2331 2332		pr_debug("prange 0x%p [0x%lx 0x%lx] op %d\n", prange,2333			 prange->start, prange->last, prange->work_item.op);2334 2335		mm = prange->work_item.mm;2336 2337		mmap_write_lock(mm);2338 2339		/* Remove from deferred_list must be inside mmap write lock, for2340		 * two race cases:2341		 * 1. unmap_from_cpu may change work_item.op and add the range2342		 *    to deferred_list again, cause use after free bug.2343		 * 2. svm_range_list_lock_and_flush_work may hold mmap write2344		 *    lock and continue because deferred_list is empty, but2345		 *    deferred_list work is actually waiting for mmap lock.2346		 */2347		spin_lock(&svms->deferred_list_lock);2348		list_del_init(&prange->deferred_list);2349		spin_unlock(&svms->deferred_list_lock);2350 2351		mutex_lock(&svms->lock);2352		mutex_lock(&prange->migrate_mutex);2353		while (!list_empty(&prange->child_list)) {2354			struct svm_range *pchild;2355 2356			pchild = list_first_entry(&prange->child_list,2357						struct svm_range, child_list);2358			pr_debug("child prange 0x%p op %d\n", pchild,2359				 pchild->work_item.op);2360			list_del_init(&pchild->child_list);2361			svm_range_handle_list_op(svms, pchild, mm);2362		}2363		mutex_unlock(&prange->migrate_mutex);2364 2365		svm_range_handle_list_op(svms, prange, mm);2366		mutex_unlock(&svms->lock);2367		mmap_write_unlock(mm);2368 2369		/* Pairs with mmget in svm_range_add_list_work. If dropping the2370		 * last mm refcount, schedule release work to avoid circular locking2371		 */2372		mmput_async(mm);2373 2374		spin_lock(&svms->deferred_list_lock);2375	}2376	spin_unlock(&svms->deferred_list_lock);2377	pr_debug("exit svms 0x%p\n", svms);2378}2379 2380void2381svm_range_add_list_work(struct svm_range_list *svms, struct svm_range *prange,2382			struct mm_struct *mm, enum svm_work_list_ops op)2383{2384	spin_lock(&svms->deferred_list_lock);2385	/* if prange is on the deferred list */2386	if (!list_empty(&prange->deferred_list)) {2387		pr_debug("update exist prange 0x%p work op %d\n", prange, op);2388		WARN_ONCE(prange->work_item.mm != mm, "unmatch mm\n");2389		if (op != SVM_OP_NULL &&2390		    prange->work_item.op != SVM_OP_UNMAP_RANGE)2391			prange->work_item.op = op;2392	} else {2393		prange->work_item.op = op;2394 2395		/* Pairs with mmput in deferred_list_work */2396		mmget(mm);2397		prange->work_item.mm = mm;2398		list_add_tail(&prange->deferred_list,2399			      &prange->svms->deferred_range_list);2400		pr_debug("add prange 0x%p [0x%lx 0x%lx] to work list op %d\n",2401			 prange, prange->start, prange->last, op);2402	}2403	spin_unlock(&svms->deferred_list_lock);2404}2405 2406void schedule_deferred_list_work(struct svm_range_list *svms)2407{2408	spin_lock(&svms->deferred_list_lock);2409	if (!list_empty(&svms->deferred_range_list))2410		schedule_work(&svms->deferred_list_work);2411	spin_unlock(&svms->deferred_list_lock);2412}2413 2414static void2415svm_range_unmap_split(struct mm_struct *mm, struct svm_range *parent,2416		      struct svm_range *prange, unsigned long start,2417		      unsigned long last)2418{2419	struct svm_range *head;2420	struct svm_range *tail;2421 2422	if (prange->work_item.op == SVM_OP_UNMAP_RANGE) {2423		pr_debug("prange 0x%p [0x%lx 0x%lx] is already freed\n", prange,2424			 prange->start, prange->last);2425		return;2426	}2427	if (start > prange->last || last < prange->start)2428		return;2429 2430	head = tail = prange;2431	if (start > prange->start)2432		svm_range_split(prange, prange->start, start - 1, &tail);2433	if (last < tail->last)2434		svm_range_split(tail, last + 1, tail->last, &head);2435 2436	if (head != prange && tail != prange) {2437		svm_range_add_child(parent, mm, head, SVM_OP_UNMAP_RANGE);2438		svm_range_add_child(parent, mm, tail, SVM_OP_ADD_RANGE);2439	} else if (tail != prange) {2440		svm_range_add_child(parent, mm, tail, SVM_OP_UNMAP_RANGE);2441	} else if (head != prange) {2442		svm_range_add_child(parent, mm, head, SVM_OP_UNMAP_RANGE);2443	} else if (parent != prange) {2444		prange->work_item.op = SVM_OP_UNMAP_RANGE;2445	}2446}2447 2448static void2449svm_range_unmap_from_cpu(struct mm_struct *mm, struct svm_range *prange,2450			 unsigned long start, unsigned long last)2451{2452	uint32_t trigger = KFD_SVM_UNMAP_TRIGGER_UNMAP_FROM_CPU;2453	struct svm_range_list *svms;2454	struct svm_range *pchild;2455	struct kfd_process *p;2456	unsigned long s, l;2457	bool unmap_parent;2458	uint32_t i;2459 2460	if (atomic_read(&prange->queue_refcount)) {2461		int r;2462 2463		pr_warn("Freeing queue vital buffer 0x%lx, queue evicted\n",2464			prange->start << PAGE_SHIFT);2465		r = kgd2kfd_quiesce_mm(mm, KFD_QUEUE_EVICTION_TRIGGER_SVM);2466		if (r)2467			pr_debug("failed %d to quiesce KFD queues\n", r);2468	}2469 2470	p = kfd_lookup_process_by_mm(mm);2471	if (!p)2472		return;2473	svms = &p->svms;2474 2475	pr_debug("svms 0x%p prange 0x%p [0x%lx 0x%lx] [0x%lx 0x%lx]\n", svms,2476		 prange, prange->start, prange->last, start, last);2477 2478	/* calculate time stamps that are used to decide which page faults need be2479	 * dropped or handled before unmap pages from gpu vm2480	 */2481	for_each_set_bit(i, svms->bitmap_supported, p->n_pdds) {2482		struct kfd_process_device *pdd;2483		struct amdgpu_device *adev;2484		struct amdgpu_ih_ring *ih;2485		uint32_t checkpoint_wptr;2486 2487		pdd = p->pdds[i];2488		if (!pdd)2489			continue;2490 2491		adev = pdd->dev->adev;2492 2493		/* Check and drain ih1 ring if cam not available */2494		if (adev->irq.ih1.ring_size) {2495			ih = &adev->irq.ih1;2496			checkpoint_wptr = amdgpu_ih_get_wptr(adev, ih);2497			if (ih->rptr != checkpoint_wptr) {2498				svms->checkpoint_ts[i] =2499					amdgpu_ih_decode_iv_ts(adev, ih, checkpoint_wptr, -1);2500				continue;2501			}2502		}2503 2504		/* check if dev->irq.ih_soft is not empty */2505		ih = &adev->irq.ih_soft;2506		checkpoint_wptr = amdgpu_ih_get_wptr(adev, ih);2507		if (ih->rptr != checkpoint_wptr)2508			svms->checkpoint_ts[i] = amdgpu_ih_decode_iv_ts(adev, ih, checkpoint_wptr, -1);2509	}2510 2511	unmap_parent = start <= prange->start && last >= prange->last;2512 2513	list_for_each_entry(pchild, &prange->child_list, child_list) {2514		mutex_lock_nested(&pchild->lock, 1);2515		s = max(start, pchild->start);2516		l = min(last, pchild->last);2517		if (l >= s)2518			svm_range_unmap_from_gpus(pchild, s, l, trigger);2519		svm_range_unmap_split(mm, prange, pchild, start, last);2520		mutex_unlock(&pchild->lock);2521	}2522	s = max(start, prange->start);2523	l = min(last, prange->last);2524	if (l >= s)2525		svm_range_unmap_from_gpus(prange, s, l, trigger);2526	svm_range_unmap_split(mm, prange, prange, start, last);2527 2528	if (unmap_parent)2529		svm_range_add_list_work(svms, prange, mm, SVM_OP_UNMAP_RANGE);2530	else2531		svm_range_add_list_work(svms, prange, mm,2532					SVM_OP_UPDATE_RANGE_NOTIFIER);2533	schedule_deferred_list_work(svms);2534 2535	kfd_unref_process(p);2536}2537 2538/**2539 * svm_range_cpu_invalidate_pagetables - interval notifier callback2540 * @mni: mmu_interval_notifier struct2541 * @range: mmu_notifier_range struct2542 * @cur_seq: value to pass to mmu_interval_set_seq()2543 *2544 * If event is MMU_NOTIFY_UNMAP, this is from CPU unmap range, otherwise, it2545 * is from migration, or CPU page invalidation callback.2546 *2547 * For unmap event, unmap range from GPUs, remove prange from svms in a delayed2548 * work thread, and split prange if only part of prange is unmapped.2549 *2550 * For invalidation event, if GPU retry fault is not enabled, evict the queues,2551 * then schedule svm_range_restore_work to update GPU mapping and resume queues.2552 * If GPU retry fault is enabled, unmap the svm range from GPU, retry fault will2553 * update GPU mapping to recover.2554 *2555 * Context: mmap lock, notifier_invalidate_start lock are held2556 *          for invalidate event, prange lock is held if this is from migration2557 */2558static bool2559svm_range_cpu_invalidate_pagetables(struct mmu_interval_notifier *mni,2560				    const struct mmu_notifier_range *range,2561				    unsigned long cur_seq)2562{2563	struct svm_range *prange;2564	unsigned long start;2565	unsigned long last;2566 2567	if (range->event == MMU_NOTIFY_RELEASE)2568		return true;2569	if (!mmget_not_zero(mni->mm))2570		return true;2571 2572	start = mni->interval_tree.start;2573	last = mni->interval_tree.last;2574	start = max(start, range->start) >> PAGE_SHIFT;2575	last = min(last, range->end - 1) >> PAGE_SHIFT;2576	pr_debug("[0x%lx 0x%lx] range[0x%lx 0x%lx] notifier[0x%lx 0x%lx] %d\n",2577		 start, last, range->start >> PAGE_SHIFT,2578		 (range->end - 1) >> PAGE_SHIFT,2579		 mni->interval_tree.start >> PAGE_SHIFT,2580		 mni->interval_tree.last >> PAGE_SHIFT, range->event);2581 2582	prange = container_of(mni, struct svm_range, notifier);2583 2584	svm_range_lock(prange);2585	mmu_interval_set_seq(mni, cur_seq);2586 2587	switch (range->event) {2588	case MMU_NOTIFY_UNMAP:2589		svm_range_unmap_from_cpu(mni->mm, prange, start, last);2590		break;2591	default:2592		svm_range_evict(prange, mni->mm, start, last, range->event);2593		break;2594	}2595 2596	svm_range_unlock(prange);2597	mmput(mni->mm);2598 2599	return true;2600}2601 2602/**2603 * svm_range_from_addr - find svm range from fault address2604 * @svms: svm range list header2605 * @addr: address to search range interval tree, in pages2606 * @parent: parent range if range is on child list2607 *2608 * Context: The caller must hold svms->lock2609 *2610 * Return: the svm_range found or NULL2611 */2612struct svm_range *2613svm_range_from_addr(struct svm_range_list *svms, unsigned long addr,2614		    struct svm_range **parent)2615{2616	struct interval_tree_node *node;2617	struct svm_range *prange;2618	struct svm_range *pchild;2619 2620	node = interval_tree_iter_first(&svms->objects, addr, addr);2621	if (!node)2622		return NULL;2623 2624	prange = container_of(node, struct svm_range, it_node);2625	pr_debug("address 0x%lx prange [0x%lx 0x%lx] node [0x%lx 0x%lx]\n",2626		 addr, prange->start, prange->last, node->start, node->last);2627 2628	if (addr >= prange->start && addr <= prange->last) {2629		if (parent)2630			*parent = prange;2631		return prange;2632	}2633	list_for_each_entry(pchild, &prange->child_list, child_list)2634		if (addr >= pchild->start && addr <= pchild->last) {2635			pr_debug("found address 0x%lx pchild [0x%lx 0x%lx]\n",2636				 addr, pchild->start, pchild->last);2637			if (parent)2638				*parent = prange;2639			return pchild;2640		}2641 2642	return NULL;2643}2644 2645/* svm_range_best_restore_location - decide the best fault restore location2646 * @prange: svm range structure2647 * @adev: the GPU on which vm fault happened2648 *2649 * This is only called when xnack is on, to decide the best location to restore2650 * the range mapping after GPU vm fault. Caller uses the best location to do2651 * migration if actual loc is not best location, then update GPU page table2652 * mapping to the best location.2653 *2654 * If the preferred loc is accessible by faulting GPU, use preferred loc.2655 * If vm fault gpu idx is on range ACCESSIBLE bitmap, best_loc is vm fault gpu2656 * If vm fault gpu idx is on range ACCESSIBLE_IN_PLACE bitmap, then2657 *    if range actual loc is cpu, best_loc is cpu2658 *    if vm fault gpu is on xgmi same hive of range actual loc gpu, best_loc is2659 *    range actual loc.2660 * Otherwise, GPU no access, best_loc is -1.2661 *2662 * Return:2663 * -1 means vm fault GPU no access2664 * 0 for CPU or GPU id2665 */2666static int32_t2667svm_range_best_restore_location(struct svm_range *prange,2668				struct kfd_node *node,2669				int32_t *gpuidx)2670{2671	struct kfd_node *bo_node, *preferred_node;2672	struct kfd_process *p;2673	uint32_t gpuid;2674	int r;2675 2676	p = container_of(prange->svms, struct kfd_process, svms);2677 2678	r = kfd_process_gpuid_from_node(p, node, &gpuid, gpuidx);2679	if (r < 0) {2680		pr_debug("failed to get gpuid from kgd\n");2681		return -1;2682	}2683 2684	if (node->adev->flags & AMD_IS_APU)2685		return 0;2686 2687	if (prange->preferred_loc == gpuid ||2688	    prange->preferred_loc == KFD_IOCTL_SVM_LOCATION_SYSMEM) {2689		return prange->preferred_loc;2690	} else if (prange->preferred_loc != KFD_IOCTL_SVM_LOCATION_UNDEFINED) {2691		preferred_node = svm_range_get_node_by_id(prange, prange->preferred_loc);2692		if (preferred_node && svm_nodes_in_same_hive(node, preferred_node))2693			return prange->preferred_loc;2694		/* fall through */2695	}2696 2697	if (test_bit(*gpuidx, prange->bitmap_access))2698		return gpuid;2699 2700	if (test_bit(*gpuidx, prange->bitmap_aip)) {2701		if (!prange->actual_loc)2702			return 0;2703 2704		bo_node = svm_range_get_node_by_id(prange, prange->actual_loc);2705		if (bo_node && svm_nodes_in_same_hive(node, bo_node))2706			return prange->actual_loc;2707		else2708			return 0;2709	}2710 2711	return -1;2712}2713 2714static int2715svm_range_get_range_boundaries(struct kfd_process *p, int64_t addr,2716			       unsigned long *start, unsigned long *last,2717			       bool *is_heap_stack)2718{2719	struct vm_area_struct *vma;2720	struct interval_tree_node *node;2721	struct rb_node *rb_node;2722	unsigned long start_limit, end_limit;2723 2724	vma = vma_lookup(p->mm, addr << PAGE_SHIFT);2725	if (!vma) {2726		pr_debug("VMA does not exist in address [0x%llx]\n", addr);2727		return -EFAULT;2728	}2729 2730	*is_heap_stack = vma_is_initial_heap(vma) || vma_is_initial_stack(vma);2731 2732	start_limit = max(vma->vm_start >> PAGE_SHIFT,2733		      (unsigned long)ALIGN_DOWN(addr, 1UL << p->svms.default_granularity));2734	end_limit = min(vma->vm_end >> PAGE_SHIFT,2735		    (unsigned long)ALIGN(addr + 1, 1UL << p->svms.default_granularity));2736 2737	/* First range that starts after the fault address */2738	node = interval_tree_iter_first(&p->svms.objects, addr + 1, ULONG_MAX);2739	if (node) {2740		end_limit = min(end_limit, node->start);2741		/* Last range that ends before the fault address */2742		rb_node = rb_prev(&node->rb);2743	} else {2744		/* Last range must end before addr because2745		 * there was no range after addr2746		 */2747		rb_node = rb_last(&p->svms.objects.rb_root);2748	}2749	if (rb_node) {2750		node = container_of(rb_node, struct interval_tree_node, rb);2751		if (node->last >= addr) {2752			WARN(1, "Overlap with prev node and page fault addr\n");2753			return -EFAULT;2754		}2755		start_limit = max(start_limit, node->last + 1);2756	}2757 2758	*start = start_limit;2759	*last = end_limit - 1;2760 2761	pr_debug("vma [0x%lx 0x%lx] range [0x%lx 0x%lx] is_heap_stack %d\n",2762		 vma->vm_start >> PAGE_SHIFT, vma->vm_end >> PAGE_SHIFT,2763		 *start, *last, *is_heap_stack);2764 2765	return 0;2766}2767 2768static int2769svm_range_check_vm_userptr(struct kfd_process *p, uint64_t start, uint64_t last,2770			   uint64_t *bo_s, uint64_t *bo_l)2771{2772	struct amdgpu_bo_va_mapping *mapping;2773	struct interval_tree_node *node;2774	struct amdgpu_bo *bo = NULL;2775	unsigned long userptr;2776	uint32_t i;2777	int r;2778 2779	for (i = 0; i < p->n_pdds; i++) {2780		struct amdgpu_vm *vm;2781 2782		if (!p->pdds[i]->drm_priv)2783			continue;2784 2785		vm = drm_priv_to_vm(p->pdds[i]->drm_priv);2786		r = amdgpu_bo_reserve(vm->root.bo, false);2787		if (r)2788			return r;2789 2790		/* Check userptr by searching entire vm->va interval tree */2791		node = interval_tree_iter_first(&vm->va, 0, ~0ULL);2792		while (node) {2793			mapping = container_of((struct rb_node *)node,2794					       struct amdgpu_bo_va_mapping, rb);2795			bo = mapping->bo_va->base.bo;2796 2797			if (!amdgpu_ttm_tt_affect_userptr(bo->tbo.ttm,2798							 start << PAGE_SHIFT,2799							 last << PAGE_SHIFT,2800							 &userptr)) {2801				node = interval_tree_iter_next(node, 0, ~0ULL);2802				continue;2803			}2804 2805			pr_debug("[0x%llx 0x%llx] already userptr mapped\n",2806				 start, last);2807			if (bo_s && bo_l) {2808				*bo_s = userptr >> PAGE_SHIFT;2809				*bo_l = *bo_s + bo->tbo.ttm->num_pages - 1;2810			}2811			amdgpu_bo_unreserve(vm->root.bo);2812			return -EADDRINUSE;2813		}2814		amdgpu_bo_unreserve(vm->root.bo);2815	}2816	return 0;2817}2818 2819static struct2820svm_range *svm_range_create_unregistered_range(struct kfd_node *node,2821						struct kfd_process *p,2822						struct mm_struct *mm,2823						int64_t addr)2824{2825	struct svm_range *prange = NULL;2826	unsigned long start, last;2827	uint32_t gpuid, gpuidx;2828	bool is_heap_stack;2829	uint64_t bo_s = 0;2830	uint64_t bo_l = 0;2831	int r;2832 2833	if (svm_range_get_range_boundaries(p, addr, &start, &last,2834					   &is_heap_stack))2835		return NULL;2836 2837	r = svm_range_check_vm(p, start, last, &bo_s, &bo_l);2838	if (r != -EADDRINUSE)2839		r = svm_range_check_vm_userptr(p, start, last, &bo_s, &bo_l);2840 2841	if (r == -EADDRINUSE) {2842		if (addr >= bo_s && addr <= bo_l)2843			return NULL;2844 2845		/* Create one page svm range if 2MB range overlapping */2846		start = addr;2847		last = addr;2848	}2849 2850	prange = svm_range_new(&p->svms, start, last, true);2851	if (!prange) {2852		pr_debug("Failed to create prange in address [0x%llx]\n", addr);2853		return NULL;2854	}2855	if (kfd_process_gpuid_from_node(p, node, &gpuid, &gpuidx)) {2856		pr_debug("failed to get gpuid from kgd\n");2857		svm_range_free(prange, true);2858		return NULL;2859	}2860 2861	if (is_heap_stack)2862		prange->preferred_loc = KFD_IOCTL_SVM_LOCATION_SYSMEM;2863 2864	svm_range_add_to_svms(prange);2865	svm_range_add_notifier_locked(mm, prange);2866 2867	return prange;2868}2869 2870/* svm_range_skip_recover - decide if prange can be recovered2871 * @prange: svm range structure2872 *2873 * GPU vm retry fault handle skip recover the range for cases:2874 * 1. prange is on deferred list to be removed after unmap, it is stale fault,2875 *    deferred list work will drain the stale fault before free the prange.2876 * 2. prange is on deferred list to add interval notifier after split, or2877 * 3. prange is child range, it is split from parent prange, recover later2878 *    after interval notifier is added.2879 *2880 * Return: true to skip recover, false to recover2881 */2882static bool svm_range_skip_recover(struct svm_range *prange)2883{2884	struct svm_range_list *svms = prange->svms;2885 2886	spin_lock(&svms->deferred_list_lock);2887	if (list_empty(&prange->deferred_list) &&2888	    list_empty(&prange->child_list)) {2889		spin_unlock(&svms->deferred_list_lock);2890		return false;2891	}2892	spin_unlock(&svms->deferred_list_lock);2893 2894	if (prange->work_item.op == SVM_OP_UNMAP_RANGE) {2895		pr_debug("svms 0x%p prange 0x%p [0x%lx 0x%lx] unmapped\n",2896			 svms, prange, prange->start, prange->last);2897		return true;2898	}2899	if (prange->work_item.op == SVM_OP_ADD_RANGE_AND_MAP ||2900	    prange->work_item.op == SVM_OP_ADD_RANGE) {2901		pr_debug("svms 0x%p prange 0x%p [0x%lx 0x%lx] not added yet\n",2902			 svms, prange, prange->start, prange->last);2903		return true;2904	}2905	return false;2906}2907 2908static void2909svm_range_count_fault(struct kfd_node *node, struct kfd_process *p,2910		      int32_t gpuidx)2911{2912	struct kfd_process_device *pdd;2913 2914	/* fault is on different page of same range2915	 * or fault is skipped to recover later2916	 * or fault is on invalid virtual address2917	 */2918	if (gpuidx == MAX_GPU_INSTANCE) {2919		uint32_t gpuid;2920		int r;2921 2922		r = kfd_process_gpuid_from_node(p, node, &gpuid, &gpuidx);2923		if (r < 0)2924			return;2925	}2926 2927	/* fault is recovered2928	 * or fault cannot recover because GPU no access on the range2929	 */2930	pdd = kfd_process_device_from_gpuidx(p, gpuidx);2931	if (pdd)2932		WRITE_ONCE(pdd->faults, pdd->faults + 1);2933}2934 2935static bool2936svm_fault_allowed(struct vm_area_struct *vma, bool write_fault)2937{2938	unsigned long requested = VM_READ;2939 2940	if (write_fault)2941		requested |= VM_WRITE;2942 2943	pr_debug("requested 0x%lx, vma permission flags 0x%lx\n", requested,2944		vma->vm_flags);2945	return (vma->vm_flags & requested) == requested;2946}2947 2948int2949svm_range_restore_pages(struct amdgpu_device *adev, unsigned int pasid,2950			uint32_t vmid, uint32_t node_id,2951			uint64_t addr, uint64_t ts, bool write_fault)2952{2953	unsigned long start, last, size;2954	struct mm_struct *mm = NULL;2955	struct svm_range_list *svms;2956	struct svm_range *prange;2957	struct kfd_process *p;2958	ktime_t timestamp = ktime_get_boottime();2959	struct kfd_node *node;2960	int32_t best_loc;2961	int32_t gpuid, gpuidx = MAX_GPU_INSTANCE;2962	bool write_locked = false;2963	struct vm_area_struct *vma;2964	bool migration = false;2965	int r = 0;2966 2967	if (!KFD_IS_SVM_API_SUPPORTED(adev)) {2968		pr_debug("device does not support SVM\n");2969		return -EFAULT;2970	}2971 2972	p = kfd_lookup_process_by_pasid(pasid);2973	if (!p) {2974		pr_debug("kfd process not founded pasid 0x%x\n", pasid);2975		return 0;2976	}2977	svms = &p->svms;2978 2979	pr_debug("restoring svms 0x%p fault address 0x%llx\n", svms, addr);2980 2981	if (atomic_read(&svms->drain_pagefaults)) {2982		pr_debug("page fault handling disabled, drop fault 0x%llx\n", addr);2983		r = 0;2984		goto out;2985	}2986 2987	node = kfd_node_by_irq_ids(adev, node_id, vmid);2988	if (!node) {2989		pr_debug("kfd node does not exist node_id: %d, vmid: %d\n", node_id,2990			 vmid);2991		r = -EFAULT;2992		goto out;2993	}2994 2995	if (kfd_process_gpuid_from_node(p, node, &gpuid, &gpuidx)) {2996		pr_debug("failed to get gpuid/gpuidex for node_id: %d\n", node_id);2997		r = -EFAULT;2998		goto out;2999	}3000 3001	/* check if this page fault time stamp is before svms->checkpoint_ts */3002	if (svms->checkpoint_ts[gpuidx] != 0) {3003		if (amdgpu_ih_ts_after(ts,  svms->checkpoint_ts[gpuidx])) {3004			pr_debug("draining retry fault, drop fault 0x%llx\n", addr);3005			r = 0;3006			goto out;3007		} else3008			/* ts is after svms->checkpoint_ts now, reset svms->checkpoint_ts3009			 * to zero to avoid following ts wrap around give wrong comparing3010			 */3011			svms->checkpoint_ts[gpuidx] = 0;3012	}3013 3014	if (!p->xnack_enabled) {3015		pr_debug("XNACK not enabled for pasid 0x%x\n", pasid);3016		r = -EFAULT;3017		goto out;3018	}3019 3020	/* p->lead_thread is available as kfd_process_wq_release flush the work3021	 * before releasing task ref.3022	 */3023	mm = get_task_mm(p->lead_thread);3024	if (!mm) {3025		pr_debug("svms 0x%p failed to get mm\n", svms);3026		r = 0;3027		goto out;3028	}3029 3030	mmap_read_lock(mm);3031retry_write_locked:3032	mutex_lock(&svms->lock);3033	prange = svm_range_from_addr(svms, addr, NULL);3034	if (!prange) {3035		pr_debug("failed to find prange svms 0x%p address [0x%llx]\n",3036			 svms, addr);3037		if (!write_locked) {3038			/* Need the write lock to create new range with MMU notifier.3039			 * Also flush pending deferred work to make sure the interval3040			 * tree is up to date before we add a new range3041			 */3042			mutex_unlock(&svms->lock);3043			mmap_read_unlock(mm);3044			mmap_write_lock(mm);3045			write_locked = true;3046			goto retry_write_locked;3047		}3048		prange = svm_range_create_unregistered_range(node, p, mm, addr);3049		if (!prange) {3050			pr_debug("failed to create unregistered range svms 0x%p address [0x%llx]\n",3051				 svms, addr);3052			mmap_write_downgrade(mm);3053			r = -EFAULT;3054			goto out_unlock_svms;3055		}3056	}3057	if (write_locked)3058		mmap_write_downgrade(mm);3059 3060	mutex_lock(&prange->migrate_mutex);3061 3062	if (svm_range_skip_recover(prange)) {3063		amdgpu_gmc_filter_faults_remove(node->adev, addr, pasid);3064		r = 0;3065		goto out_unlock_range;3066	}3067 3068	/* skip duplicate vm fault on different pages of same range */3069	if (ktime_before(timestamp, ktime_add_ns(prange->validate_timestamp,3070				AMDGPU_SVM_RANGE_RETRY_FAULT_PENDING))) {3071		pr_debug("svms 0x%p [0x%lx %lx] already restored\n",3072			 svms, prange->start, prange->last);3073		r = 0;3074		goto out_unlock_range;3075	}3076 3077	/* __do_munmap removed VMA, return success as we are handling stale3078	 * retry fault.3079	 */3080	vma = vma_lookup(mm, addr << PAGE_SHIFT);3081	if (!vma) {3082		pr_debug("address 0x%llx VMA is removed\n", addr);3083		r = 0;3084		goto out_unlock_range;3085	}3086 3087	if (!svm_fault_allowed(vma, write_fault)) {3088		pr_debug("fault addr 0x%llx no %s permission\n", addr,3089			write_fault ? "write" : "read");3090		r = -EPERM;3091		goto out_unlock_range;3092	}3093 3094	best_loc = svm_range_best_restore_location(prange, node, &gpuidx);3095	if (best_loc == -1) {3096		pr_debug("svms %p failed get best restore loc [0x%lx 0x%lx]\n",3097			 svms, prange->start, prange->last);3098		r = -EACCES;3099		goto out_unlock_range;3100	}3101 3102	pr_debug("svms %p [0x%lx 0x%lx] best restore 0x%x, actual loc 0x%x\n",3103		 svms, prange->start, prange->last, best_loc,3104		 prange->actual_loc);3105 3106	kfd_smi_event_page_fault_start(node, p->lead_thread->pid, addr,3107				       write_fault, timestamp);3108 3109	/* Align migration range start and size to granularity size */3110	size = 1UL << prange->granularity;3111	start = max_t(unsigned long, ALIGN_DOWN(addr, size), prange->start);3112	last = min_t(unsigned long, ALIGN(addr + 1, size) - 1, prange->last);3113	if (prange->actual_loc != 0 || best_loc != 0) {3114		migration = true;3115 3116		if (best_loc) {3117			r = svm_migrate_to_vram(prange, best_loc, start, last,3118					mm, KFD_MIGRATE_TRIGGER_PAGEFAULT_GPU);3119			if (r) {3120				pr_debug("svm_migrate_to_vram failed (%d) at %llx, falling back to system memory\n",3121					 r, addr);3122				/* Fallback to system memory if migration to3123				 * VRAM failed3124				 */3125				if (prange->actual_loc && prange->actual_loc != best_loc)3126					r = svm_migrate_vram_to_ram(prange, mm, start, last,3127						KFD_MIGRATE_TRIGGER_PAGEFAULT_GPU, NULL);3128				else3129					r = 0;3130			}3131		} else {3132			r = svm_migrate_vram_to_ram(prange, mm, start, last,3133					KFD_MIGRATE_TRIGGER_PAGEFAULT_GPU, NULL);3134		}3135		if (r) {3136			pr_debug("failed %d to migrate svms %p [0x%lx 0x%lx]\n",3137				 r, svms, start, last);3138			goto out_unlock_range;3139		}3140	}3141 3142	r = svm_range_validate_and_map(mm, start, last, prange, gpuidx, false,3143				       false, false);3144	if (r)3145		pr_debug("failed %d to map svms 0x%p [0x%lx 0x%lx] to gpus\n",3146			 r, svms, start, last);3147 3148	kfd_smi_event_page_fault_end(node, p->lead_thread->pid, addr,3149				     migration);3150 3151out_unlock_range:3152	mutex_unlock(&prange->migrate_mutex);3153out_unlock_svms:3154	mutex_unlock(&svms->lock);3155	mmap_read_unlock(mm);3156 3157	svm_range_count_fault(node, p, gpuidx);3158 3159	mmput(mm);3160out:3161	kfd_unref_process(p);3162 3163	if (r == -EAGAIN) {3164		pr_debug("recover vm fault later\n");3165		amdgpu_gmc_filter_faults_remove(node->adev, addr, pasid);3166		r = 0;3167	}3168	return r;3169}3170 3171int3172svm_range_switch_xnack_reserve_mem(struct kfd_process *p, bool xnack_enabled)3173{3174	struct svm_range *prange, *pchild;3175	uint64_t reserved_size = 0;3176	uint64_t size;3177	int r = 0;3178 3179	pr_debug("switching xnack from %d to %d\n", p->xnack_enabled, xnack_enabled);3180 3181	mutex_lock(&p->svms.lock);3182 3183	list_for_each_entry(prange, &p->svms.list, list) {3184		svm_range_lock(prange);3185		list_for_each_entry(pchild, &prange->child_list, child_list) {3186			size = (pchild->last - pchild->start + 1) << PAGE_SHIFT;3187			if (xnack_enabled) {3188				amdgpu_amdkfd_unreserve_mem_limit(NULL, size,3189					KFD_IOC_ALLOC_MEM_FLAGS_USERPTR, 0);3190			} else {3191				r = amdgpu_amdkfd_reserve_mem_limit(NULL, size,3192					KFD_IOC_ALLOC_MEM_FLAGS_USERPTR, 0);3193				if (r)3194					goto out_unlock;3195				reserved_size += size;3196			}3197		}3198 3199		size = (prange->last - prange->start + 1) << PAGE_SHIFT;3200		if (xnack_enabled) {3201			amdgpu_amdkfd_unreserve_mem_limit(NULL, size,3202					KFD_IOC_ALLOC_MEM_FLAGS_USERPTR, 0);3203		} else {3204			r = amdgpu_amdkfd_reserve_mem_limit(NULL, size,3205					KFD_IOC_ALLOC_MEM_FLAGS_USERPTR, 0);3206			if (r)3207				goto out_unlock;3208			reserved_size += size;3209		}3210out_unlock:3211		svm_range_unlock(prange);3212		if (r)3213			break;3214	}3215 3216	if (r)3217		amdgpu_amdkfd_unreserve_mem_limit(NULL, reserved_size,3218					KFD_IOC_ALLOC_MEM_FLAGS_USERPTR, 0);3219	else3220		/* Change xnack mode must be inside svms lock, to avoid race with3221		 * svm_range_deferred_list_work unreserve memory in parallel.3222		 */3223		p->xnack_enabled = xnack_enabled;3224 3225	mutex_unlock(&p->svms.lock);3226	return r;3227}3228 3229void svm_range_list_fini(struct kfd_process *p)3230{3231	struct svm_range *prange;3232	struct svm_range *next;3233 3234	pr_debug("pasid 0x%x svms 0x%p\n", p->pasid, &p->svms);3235 3236	cancel_delayed_work_sync(&p->svms.restore_work);3237 3238	/* Ensure list work is finished before process is destroyed */3239	flush_work(&p->svms.deferred_list_work);3240 3241	/*3242	 * Ensure no retry fault comes in afterwards, as page fault handler will3243	 * not find kfd process and take mm lock to recover fault.3244	 * stop kfd page fault handing, then wait pending page faults got drained3245	 */3246	atomic_set(&p->svms.drain_pagefaults, 1);3247	svm_range_drain_retry_fault(&p->svms);3248 3249	list_for_each_entry_safe(prange, next, &p->svms.list, list) {3250		svm_range_unlink(prange);3251		svm_range_remove_notifier(prange);3252		svm_range_free(prange, true);3253	}3254 3255	mutex_destroy(&p->svms.lock);3256 3257	pr_debug("pasid 0x%x svms 0x%p done\n", p->pasid, &p->svms);3258}3259 3260int svm_range_list_init(struct kfd_process *p)3261{3262	struct svm_range_list *svms = &p->svms;3263	int i;3264 3265	svms->objects = RB_ROOT_CACHED;3266	mutex_init(&svms->lock);3267	INIT_LIST_HEAD(&svms->list);3268	atomic_set(&svms->evicted_ranges, 0);3269	atomic_set(&svms->drain_pagefaults, 0);3270	INIT_DELAYED_WORK(&svms->restore_work, svm_range_restore_work);3271	INIT_WORK(&svms->deferred_list_work, svm_range_deferred_list_work);3272	INIT_LIST_HEAD(&svms->deferred_range_list);3273	INIT_LIST_HEAD(&svms->criu_svm_metadata_list);3274	spin_lock_init(&svms->deferred_list_lock);3275 3276	for (i = 0; i < p->n_pdds; i++)3277		if (KFD_IS_SVM_API_SUPPORTED(p->pdds[i]->dev->adev))3278			bitmap_set(svms->bitmap_supported, i, 1);3279 3280	 /* Value of default granularity cannot exceed 0x1B, the3281	  * number of pages supported by a 4-level paging table3282	  */3283	svms->default_granularity = min_t(u8, amdgpu_svm_default_granularity, 0x1B);3284	pr_debug("Default SVM Granularity to use: %d\n", svms->default_granularity);3285 3286	return 0;3287}3288 3289/**3290 * svm_range_check_vm - check if virtual address range mapped already3291 * @p: current kfd_process3292 * @start: range start address, in pages3293 * @last: range last address, in pages3294 * @bo_s: mapping start address in pages if address range already mapped3295 * @bo_l: mapping last address in pages if address range already mapped3296 *3297 * The purpose is to avoid virtual address ranges already allocated by3298 * kfd_ioctl_alloc_memory_of_gpu ioctl.3299 * It looks for each pdd in the kfd_process.3300 *3301 * Context: Process context3302 *3303 * Return 0 - OK, if the range is not mapped.3304 * Otherwise error code:3305 * -EADDRINUSE - if address is mapped already by kfd_ioctl_alloc_memory_of_gpu3306 * -ERESTARTSYS - A wait for the buffer to become unreserved was interrupted by3307 * a signal. Release all buffer reservations and return to user-space.3308 */3309static int3310svm_range_check_vm(struct kfd_process *p, uint64_t start, uint64_t last,3311		   uint64_t *bo_s, uint64_t *bo_l)3312{3313	struct amdgpu_bo_va_mapping *mapping;3314	struct interval_tree_node *node;3315	uint32_t i;3316	int r;3317 3318	for (i = 0; i < p->n_pdds; i++) {3319		struct amdgpu_vm *vm;3320 3321		if (!p->pdds[i]->drm_priv)3322			continue;3323 3324		vm = drm_priv_to_vm(p->pdds[i]->drm_priv);3325		r = amdgpu_bo_reserve(vm->root.bo, false);3326		if (r)3327			return r;3328 3329		node = interval_tree_iter_first(&vm->va, start, last);3330		if (node) {3331			pr_debug("range [0x%llx 0x%llx] already TTM mapped\n",3332				 start, last);3333			mapping = container_of((struct rb_node *)node,3334					       struct amdgpu_bo_va_mapping, rb);3335			if (bo_s && bo_l) {3336				*bo_s = mapping->start;3337				*bo_l = mapping->last;3338			}3339			amdgpu_bo_unreserve(vm->root.bo);3340			return -EADDRINUSE;3341		}3342		amdgpu_bo_unreserve(vm->root.bo);3343	}3344 3345	return 0;3346}3347 3348/**3349 * svm_range_is_valid - check if virtual address range is valid3350 * @p: current kfd_process3351 * @start: range start address, in pages3352 * @size: range size, in pages3353 *3354 * Valid virtual address range means it belongs to one or more VMAs3355 *3356 * Context: Process context3357 *3358 * Return:3359 *  0 - OK, otherwise error code3360 */3361static int3362svm_range_is_valid(struct kfd_process *p, uint64_t start, uint64_t size)3363{3364	const unsigned long device_vma = VM_IO | VM_PFNMAP | VM_MIXEDMAP;3365	struct vm_area_struct *vma;3366	unsigned long end;3367	unsigned long start_unchg = start;3368 3369	start <<= PAGE_SHIFT;3370	end = start + (size << PAGE_SHIFT);3371	do {3372		vma = vma_lookup(p->mm, start);3373		if (!vma || (vma->vm_flags & device_vma))3374			return -EFAULT;3375		start = min(end, vma->vm_end);3376	} while (start < end);3377 3378	return svm_range_check_vm(p, start_unchg, (end - 1) >> PAGE_SHIFT, NULL,3379				  NULL);3380}3381 3382/**3383 * svm_range_best_prefetch_location - decide the best prefetch location3384 * @prange: svm range structure3385 *3386 * For xnack off:3387 * If range map to single GPU, the best prefetch location is prefetch_loc, which3388 * can be CPU or GPU.3389 *3390 * If range is ACCESS or ACCESS_IN_PLACE by mGPUs, only if mGPU connection on3391 * XGMI same hive, the best prefetch location is prefetch_loc GPU, othervise3392 * the best prefetch location is always CPU, because GPU can not have coherent3393 * mapping VRAM of other GPUs even with large-BAR PCIe connection.3394 *3395 * For xnack on:3396 * If range is not ACCESS_IN_PLACE by mGPUs, the best prefetch location is3397 * prefetch_loc, other GPU access will generate vm fault and trigger migration.3398 *3399 * If range is ACCESS_IN_PLACE by mGPUs, only if mGPU connection on XGMI same3400 * hive, the best prefetch location is prefetch_loc GPU, otherwise the best3401 * prefetch location is always CPU.3402 *3403 * Context: Process context3404 *3405 * Return:3406 * 0 for CPU or GPU id3407 */3408static uint32_t3409svm_range_best_prefetch_location(struct svm_range *prange)3410{3411	DECLARE_BITMAP(bitmap, MAX_GPU_INSTANCE);3412	uint32_t best_loc = prange->prefetch_loc;3413	struct kfd_process_device *pdd;3414	struct kfd_node *bo_node;3415	struct kfd_process *p;3416	uint32_t gpuidx;3417 3418	p = container_of(prange->svms, struct kfd_process, svms);3419 3420	if (!best_loc || best_loc == KFD_IOCTL_SVM_LOCATION_UNDEFINED)3421		goto out;3422 3423	bo_node = svm_range_get_node_by_id(prange, best_loc);3424	if (!bo_node) {3425		WARN_ONCE(1, "failed to get valid kfd node at id%x\n", best_loc);3426		best_loc = 0;3427		goto out;3428	}3429 3430	if (bo_node->adev->flags & AMD_IS_APU) {3431		best_loc = 0;3432		goto out;3433	}3434 3435	if (p->xnack_enabled)3436		bitmap_copy(bitmap, prange->bitmap_aip, MAX_GPU_INSTANCE);3437	else3438		bitmap_or(bitmap, prange->bitmap_access, prange->bitmap_aip,3439			  MAX_GPU_INSTANCE);3440 3441	for_each_set_bit(gpuidx, bitmap, MAX_GPU_INSTANCE) {3442		pdd = kfd_process_device_from_gpuidx(p, gpuidx);3443		if (!pdd) {3444			pr_debug("failed to get device by idx 0x%x\n", gpuidx);3445			continue;3446		}3447 3448		if (pdd->dev->adev == bo_node->adev)3449			continue;3450 3451		if (!svm_nodes_in_same_hive(pdd->dev, bo_node)) {3452			best_loc = 0;3453			break;3454		}3455	}3456 3457out:3458	pr_debug("xnack %d svms 0x%p [0x%lx 0x%lx] best loc 0x%x\n",3459		 p->xnack_enabled, &p->svms, prange->start, prange->last,3460		 best_loc);3461 3462	return best_loc;3463}3464 3465/* svm_range_trigger_migration - start page migration if prefetch loc changed3466 * @mm: current process mm_struct3467 * @prange: svm range structure3468 * @migrated: output, true if migration is triggered3469 *3470 * If range perfetch_loc is GPU, actual loc is cpu 0, then migrate the range3471 * from ram to vram.3472 * If range prefetch_loc is cpu 0, actual loc is GPU, then migrate the range3473 * from vram to ram.3474 *3475 * If GPU vm fault retry is not enabled, migration interact with MMU notifier3476 * and restore work:3477 * 1. migrate_vma_setup invalidate pages, MMU notifier callback svm_range_evict3478 *    stops all queues, schedule restore work3479 * 2. svm_range_restore_work wait for migration is done by3480 *    a. svm_range_validate_vram takes prange->migrate_mutex3481 *    b. svm_range_validate_ram HMM get pages wait for CPU fault handle returns3482 * 3. restore work update mappings of GPU, resume all queues.3483 *3484 * Context: Process context3485 *3486 * Return:3487 * 0 - OK, otherwise - error code of migration3488 */3489static int3490svm_range_trigger_migration(struct mm_struct *mm, struct svm_range *prange,3491			    bool *migrated)3492{3493	uint32_t best_loc;3494	int r = 0;3495 3496	*migrated = false;3497	best_loc = svm_range_best_prefetch_location(prange);3498 3499	/* when best_loc is a gpu node and same as prange->actual_loc3500	 * we still need do migration as prange->actual_loc !=0 does3501	 * not mean all pages in prange are vram. hmm migrate will pick3502	 * up right pages during migration.3503	 */3504	if ((best_loc == KFD_IOCTL_SVM_LOCATION_UNDEFINED) ||3505	    (best_loc == 0 && prange->actual_loc == 0))3506		return 0;3507 3508	if (!best_loc) {3509		r = svm_migrate_vram_to_ram(prange, mm, prange->start, prange->last,3510					KFD_MIGRATE_TRIGGER_PREFETCH, NULL);3511		*migrated = !r;3512		return r;3513	}3514 3515	r = svm_migrate_to_vram(prange, best_loc, prange->start, prange->last,3516				mm, KFD_MIGRATE_TRIGGER_PREFETCH);3517	*migrated = !r;3518 3519	return 0;3520}3521 3522int svm_range_schedule_evict_svm_bo(struct amdgpu_amdkfd_fence *fence)3523{3524	/* Dereferencing fence->svm_bo is safe here because the fence hasn't3525	 * signaled yet and we're under the protection of the fence->lock.3526	 * After the fence is signaled in svm_range_bo_release, we cannot get3527	 * here any more.3528	 *3529	 * Reference is dropped in svm_range_evict_svm_bo_worker.3530	 */3531	if (svm_bo_ref_unless_zero(fence->svm_bo)) {3532		WRITE_ONCE(fence->svm_bo->evicting, 1);3533		schedule_work(&fence->svm_bo->eviction_work);3534	}3535 3536	return 0;3537}3538 3539static void svm_range_evict_svm_bo_worker(struct work_struct *work)3540{3541	struct svm_range_bo *svm_bo;3542	struct mm_struct *mm;3543	int r = 0;3544 3545	svm_bo = container_of(work, struct svm_range_bo, eviction_work);3546 3547	if (mmget_not_zero(svm_bo->eviction_fence->mm)) {3548		mm = svm_bo->eviction_fence->mm;3549	} else {3550		svm_range_bo_unref(svm_bo);3551		return;3552	}3553 3554	mmap_read_lock(mm);3555	spin_lock(&svm_bo->list_lock);3556	while (!list_empty(&svm_bo->range_list) && !r) {3557		struct svm_range *prange =3558				list_first_entry(&svm_bo->range_list,3559						struct svm_range, svm_bo_list);3560		int retries = 3;3561 3562		list_del_init(&prange->svm_bo_list);3563		spin_unlock(&svm_bo->list_lock);3564 3565		pr_debug("svms 0x%p [0x%lx 0x%lx]\n", prange->svms,3566			 prange->start, prange->last);3567 3568		mutex_lock(&prange->migrate_mutex);3569		do {3570			/* migrate all vram pages in this prange to sys ram3571			 * after that prange->actual_loc should be zero3572			 */3573			r = svm_migrate_vram_to_ram(prange, mm,3574					prange->start, prange->last,3575					KFD_MIGRATE_TRIGGER_TTM_EVICTION, NULL);3576		} while (!r && prange->actual_loc && --retries);3577 3578		if (!r && prange->actual_loc)3579			pr_info_once("Migration failed during eviction");3580 3581		if (!prange->actual_loc) {3582			mutex_lock(&prange->lock);3583			prange->svm_bo = NULL;3584			mutex_unlock(&prange->lock);3585		}3586		mutex_unlock(&prange->migrate_mutex);3587 3588		spin_lock(&svm_bo->list_lock);3589	}3590	spin_unlock(&svm_bo->list_lock);3591	mmap_read_unlock(mm);3592	mmput(mm);3593 3594	dma_fence_signal(&svm_bo->eviction_fence->base);3595 3596	/* This is the last reference to svm_bo, after svm_range_vram_node_free3597	 * has been called in svm_migrate_vram_to_ram3598	 */3599	WARN_ONCE(!r && kref_read(&svm_bo->kref) != 1, "This was not the last reference\n");3600	svm_range_bo_unref(svm_bo);3601}3602 3603static int3604svm_range_set_attr(struct kfd_process *p, struct mm_struct *mm,3605		   uint64_t start, uint64_t size, uint32_t nattr,3606		   struct kfd_ioctl_svm_attribute *attrs)3607{3608	struct amdkfd_process_info *process_info = p->kgd_process_info;3609	struct list_head update_list;3610	struct list_head insert_list;3611	struct list_head remove_list;3612	struct list_head remap_list;3613	struct svm_range_list *svms;3614	struct svm_range *prange;3615	struct svm_range *next;3616	bool update_mapping = false;3617	bool flush_tlb;3618	int r, ret = 0;3619 3620	pr_debug("pasid 0x%x svms 0x%p [0x%llx 0x%llx] pages 0x%llx\n",3621		 p->pasid, &p->svms, start, start + size - 1, size);3622 3623	r = svm_range_check_attr(p, nattr, attrs);3624	if (r)3625		return r;3626 3627	svms = &p->svms;3628 3629	mutex_lock(&process_info->lock);3630 3631	svm_range_list_lock_and_flush_work(svms, mm);3632 3633	r = svm_range_is_valid(p, start, size);3634	if (r) {3635		pr_debug("invalid range r=%d\n", r);3636		mmap_write_unlock(mm);3637		goto out;3638	}3639 3640	mutex_lock(&svms->lock);3641 3642	/* Add new range and split existing ranges as needed */3643	r = svm_range_add(p, start, size, nattr, attrs, &update_list,3644			  &insert_list, &remove_list, &remap_list);3645	if (r) {3646		mutex_unlock(&svms->lock);3647		mmap_write_unlock(mm);3648		goto out;3649	}3650	/* Apply changes as a transaction */3651	list_for_each_entry_safe(prange, next, &insert_list, list) {3652		svm_range_add_to_svms(prange);3653		svm_range_add_notifier_locked(mm, prange);3654	}3655	list_for_each_entry(prange, &update_list, update_list) {3656		svm_range_apply_attrs(p, prange, nattr, attrs, &update_mapping);3657		/* TODO: unmap ranges from GPU that lost access */3658	}3659	list_for_each_entry_safe(prange, next, &remove_list, update_list) {3660		pr_debug("unlink old 0x%p prange 0x%p [0x%lx 0x%lx]\n",3661			 prange->svms, prange, prange->start,3662			 prange->last);3663		svm_range_unlink(prange);3664		svm_range_remove_notifier(prange);3665		svm_range_free(prange, false);3666	}3667 3668	mmap_write_downgrade(mm);3669	/* Trigger migrations and revalidate and map to GPUs as needed. If3670	 * this fails we may be left with partially completed actions. There3671	 * is no clean way of rolling back to the previous state in such a3672	 * case because the rollback wouldn't be guaranteed to work either.3673	 */3674	list_for_each_entry(prange, &update_list, update_list) {3675		bool migrated;3676 3677		mutex_lock(&prange->migrate_mutex);3678 3679		r = svm_range_trigger_migration(mm, prange, &migrated);3680		if (r)3681			goto out_unlock_range;3682 3683		if (migrated && (!p->xnack_enabled ||3684		    (prange->flags & KFD_IOCTL_SVM_FLAG_GPU_ALWAYS_MAPPED)) &&3685		    prange->mapped_to_gpu) {3686			pr_debug("restore_work will update mappings of GPUs\n");3687			mutex_unlock(&prange->migrate_mutex);3688			continue;3689		}3690 3691		if (!migrated && !update_mapping) {3692			mutex_unlock(&prange->migrate_mutex);3693			continue;3694		}3695 3696		flush_tlb = !migrated && update_mapping && prange->mapped_to_gpu;3697 3698		r = svm_range_validate_and_map(mm, prange->start, prange->last, prange,3699					       MAX_GPU_INSTANCE, true, true, flush_tlb);3700		if (r)3701			pr_debug("failed %d to map svm range\n", r);3702 3703out_unlock_range:3704		mutex_unlock(&prange->migrate_mutex);3705		if (r)3706			ret = r;3707	}3708 3709	list_for_each_entry(prange, &remap_list, update_list) {3710		pr_debug("Remapping prange 0x%p [0x%lx 0x%lx]\n",3711			 prange, prange->start, prange->last);3712		mutex_lock(&prange->migrate_mutex);3713		r = svm_range_validate_and_map(mm,  prange->start, prange->last, prange,3714					       MAX_GPU_INSTANCE, true, true, prange->mapped_to_gpu);3715		if (r)3716			pr_debug("failed %d on remap svm range\n", r);3717		mutex_unlock(&prange->migrate_mutex);3718		if (r)3719			ret = r;3720	}3721 3722	dynamic_svm_range_dump(svms);3723 3724	mutex_unlock(&svms->lock);3725	mmap_read_unlock(mm);3726out:3727	mutex_unlock(&process_info->lock);3728 3729	pr_debug("pasid 0x%x svms 0x%p [0x%llx 0x%llx] done, r=%d\n", p->pasid,3730		 &p->svms, start, start + size - 1, r);3731 3732	return ret ? ret : r;3733}3734 3735static int3736svm_range_get_attr(struct kfd_process *p, struct mm_struct *mm,3737		   uint64_t start, uint64_t size, uint32_t nattr,3738		   struct kfd_ioctl_svm_attribute *attrs)3739{3740	DECLARE_BITMAP(bitmap_access, MAX_GPU_INSTANCE);3741	DECLARE_BITMAP(bitmap_aip, MAX_GPU_INSTANCE);3742	bool get_preferred_loc = false;3743	bool get_prefetch_loc = false;3744	bool get_granularity = false;3745	bool get_accessible = false;3746	bool get_flags = false;3747	uint64_t last = start + size - 1UL;3748	uint8_t granularity = 0xff;3749	struct interval_tree_node *node;3750	struct svm_range_list *svms;3751	struct svm_range *prange;3752	uint32_t prefetch_loc = KFD_IOCTL_SVM_LOCATION_UNDEFINED;3753	uint32_t location = KFD_IOCTL_SVM_LOCATION_UNDEFINED;3754	uint32_t flags_and = 0xffffffff;3755	uint32_t flags_or = 0;3756	int gpuidx;3757	uint32_t i;3758	int r = 0;3759 3760	pr_debug("svms 0x%p [0x%llx 0x%llx] nattr 0x%x\n", &p->svms, start,3761		 start + size - 1, nattr);3762 3763	/* Flush pending deferred work to avoid racing with deferred actions from3764	 * previous memory map changes (e.g. munmap). Concurrent memory map changes3765	 * can still race with get_attr because we don't hold the mmap lock. But that3766	 * would be a race condition in the application anyway, and undefined3767	 * behaviour is acceptable in that case.3768	 */3769	flush_work(&p->svms.deferred_list_work);3770 3771	mmap_read_lock(mm);3772	r = svm_range_is_valid(p, start, size);3773	mmap_read_unlock(mm);3774	if (r) {3775		pr_debug("invalid range r=%d\n", r);3776		return r;3777	}3778 3779	for (i = 0; i < nattr; i++) {3780		switch (attrs[i].type) {3781		case KFD_IOCTL_SVM_ATTR_PREFERRED_LOC:3782			get_preferred_loc = true;3783			break;3784		case KFD_IOCTL_SVM_ATTR_PREFETCH_LOC:3785			get_prefetch_loc = true;3786			break;3787		case KFD_IOCTL_SVM_ATTR_ACCESS:3788			get_accessible = true;3789			break;3790		case KFD_IOCTL_SVM_ATTR_SET_FLAGS:3791		case KFD_IOCTL_SVM_ATTR_CLR_FLAGS:3792			get_flags = true;3793			break;3794		case KFD_IOCTL_SVM_ATTR_GRANULARITY:3795			get_granularity = true;3796			break;3797		case KFD_IOCTL_SVM_ATTR_ACCESS_IN_PLACE:3798		case KFD_IOCTL_SVM_ATTR_NO_ACCESS:3799			fallthrough;3800		default:3801			pr_debug("get invalid attr type 0x%x\n", attrs[i].type);3802			return -EINVAL;3803		}3804	}3805 3806	svms = &p->svms;3807 3808	mutex_lock(&svms->lock);3809 3810	node = interval_tree_iter_first(&svms->objects, start, last);3811	if (!node) {3812		pr_debug("range attrs not found return default values\n");3813		svm_range_set_default_attributes(svms, &location, &prefetch_loc,3814						 &granularity, &flags_and);3815		flags_or = flags_and;3816		if (p->xnack_enabled)3817			bitmap_copy(bitmap_access, svms->bitmap_supported,3818				    MAX_GPU_INSTANCE);3819		else3820			bitmap_zero(bitmap_access, MAX_GPU_INSTANCE);3821		bitmap_zero(bitmap_aip, MAX_GPU_INSTANCE);3822		goto fill_values;3823	}3824	bitmap_copy(bitmap_access, svms->bitmap_supported, MAX_GPU_INSTANCE);3825	bitmap_copy(bitmap_aip, svms->bitmap_supported, MAX_GPU_INSTANCE);3826 3827	while (node) {3828		struct interval_tree_node *next;3829 3830		prange = container_of(node, struct svm_range, it_node);3831		next = interval_tree_iter_next(node, start, last);3832 3833		if (get_preferred_loc) {3834			if (prange->preferred_loc ==3835					KFD_IOCTL_SVM_LOCATION_UNDEFINED ||3836			    (location != KFD_IOCTL_SVM_LOCATION_UNDEFINED &&3837			     location != prange->preferred_loc)) {3838				location = KFD_IOCTL_SVM_LOCATION_UNDEFINED;3839				get_preferred_loc = false;3840			} else {3841				location = prange->preferred_loc;3842			}3843		}3844		if (get_prefetch_loc) {3845			if (prange->prefetch_loc ==3846					KFD_IOCTL_SVM_LOCATION_UNDEFINED ||3847			    (prefetch_loc != KFD_IOCTL_SVM_LOCATION_UNDEFINED &&3848			     prefetch_loc != prange->prefetch_loc)) {3849				prefetch_loc = KFD_IOCTL_SVM_LOCATION_UNDEFINED;3850				get_prefetch_loc = false;3851			} else {3852				prefetch_loc = prange->prefetch_loc;3853			}3854		}3855		if (get_accessible) {3856			bitmap_and(bitmap_access, bitmap_access,3857				   prange->bitmap_access, MAX_GPU_INSTANCE);3858			bitmap_and(bitmap_aip, bitmap_aip,3859				   prange->bitmap_aip, MAX_GPU_INSTANCE);3860		}3861		if (get_flags) {3862			flags_and &= prange->flags;3863			flags_or |= prange->flags;3864		}3865 3866		if (get_granularity && prange->granularity < granularity)3867			granularity = prange->granularity;3868 3869		node = next;3870	}3871fill_values:3872	mutex_unlock(&svms->lock);3873 3874	for (i = 0; i < nattr; i++) {3875		switch (attrs[i].type) {3876		case KFD_IOCTL_SVM_ATTR_PREFERRED_LOC:3877			attrs[i].value = location;3878			break;3879		case KFD_IOCTL_SVM_ATTR_PREFETCH_LOC:3880			attrs[i].value = prefetch_loc;3881			break;3882		case KFD_IOCTL_SVM_ATTR_ACCESS:3883			gpuidx = kfd_process_gpuidx_from_gpuid(p,3884							       attrs[i].value);3885			if (gpuidx < 0) {3886				pr_debug("invalid gpuid %x\n", attrs[i].value);3887				return -EINVAL;3888			}3889			if (test_bit(gpuidx, bitmap_access))3890				attrs[i].type = KFD_IOCTL_SVM_ATTR_ACCESS;3891			else if (test_bit(gpuidx, bitmap_aip))3892				attrs[i].type =3893					KFD_IOCTL_SVM_ATTR_ACCESS_IN_PLACE;3894			else3895				attrs[i].type = KFD_IOCTL_SVM_ATTR_NO_ACCESS;3896			break;3897		case KFD_IOCTL_SVM_ATTR_SET_FLAGS:3898			attrs[i].value = flags_and;3899			break;3900		case KFD_IOCTL_SVM_ATTR_CLR_FLAGS:3901			attrs[i].value = ~flags_or;3902			break;3903		case KFD_IOCTL_SVM_ATTR_GRANULARITY:3904			attrs[i].value = (uint32_t)granularity;3905			break;3906		}3907	}3908 3909	return 0;3910}3911 3912int kfd_criu_resume_svm(struct kfd_process *p)3913{3914	struct kfd_ioctl_svm_attribute *set_attr_new, *set_attr = NULL;3915	int nattr_common = 4, nattr_accessibility = 1;3916	struct criu_svm_metadata *criu_svm_md = NULL;3917	struct svm_range_list *svms = &p->svms;3918	struct criu_svm_metadata *next = NULL;3919	uint32_t set_flags = 0xffffffff;3920	int i, j, num_attrs, ret = 0;3921	uint64_t set_attr_size;3922	struct mm_struct *mm;3923 3924	if (list_empty(&svms->criu_svm_metadata_list)) {3925		pr_debug("No SVM data from CRIU restore stage 2\n");3926		return ret;3927	}3928 3929	mm = get_task_mm(p->lead_thread);3930	if (!mm) {3931		pr_err("failed to get mm for the target process\n");3932		return -ESRCH;3933	}3934 3935	num_attrs = nattr_common + (nattr_accessibility * p->n_pdds);3936 3937	i = j = 0;3938	list_for_each_entry(criu_svm_md, &svms->criu_svm_metadata_list, list) {3939		pr_debug("criu_svm_md[%d]\n\tstart: 0x%llx size: 0x%llx (npages)\n",3940			 i, criu_svm_md->data.start_addr, criu_svm_md->data.size);3941 3942		for (j = 0; j < num_attrs; j++) {3943			pr_debug("\ncriu_svm_md[%d]->attrs[%d].type : 0x%x\ncriu_svm_md[%d]->attrs[%d].value : 0x%x\n",3944				 i, j, criu_svm_md->data.attrs[j].type,3945				 i, j, criu_svm_md->data.attrs[j].value);3946			switch (criu_svm_md->data.attrs[j].type) {3947			/* During Checkpoint operation, the query for3948			 * KFD_IOCTL_SVM_ATTR_PREFETCH_LOC attribute might3949			 * return KFD_IOCTL_SVM_LOCATION_UNDEFINED if they were3950			 * not used by the range which was checkpointed. Care3951			 * must be taken to not restore with an invalid value3952			 * otherwise the gpuidx value will be invalid and3953			 * set_attr would eventually fail so just replace those3954			 * with another dummy attribute such as3955			 * KFD_IOCTL_SVM_ATTR_SET_FLAGS.3956			 */3957			case KFD_IOCTL_SVM_ATTR_PREFETCH_LOC:3958				if (criu_svm_md->data.attrs[j].value ==3959				    KFD_IOCTL_SVM_LOCATION_UNDEFINED) {3960					criu_svm_md->data.attrs[j].type =3961						KFD_IOCTL_SVM_ATTR_SET_FLAGS;3962					criu_svm_md->data.attrs[j].value = 0;3963				}3964				break;3965			case KFD_IOCTL_SVM_ATTR_SET_FLAGS:3966				set_flags = criu_svm_md->data.attrs[j].value;3967				break;3968			default:3969				break;3970			}3971		}3972 3973		/* CLR_FLAGS is not available via get_attr during checkpoint but3974		 * it needs to be inserted before restoring the ranges so3975		 * allocate extra space for it before calling set_attr3976		 */3977		set_attr_size = sizeof(struct kfd_ioctl_svm_attribute) *3978						(num_attrs + 1);3979		set_attr_new = krealloc(set_attr, set_attr_size,3980					    GFP_KERNEL);3981		if (!set_attr_new) {3982			ret = -ENOMEM;3983			goto exit;3984		}3985		set_attr = set_attr_new;3986 3987		memcpy(set_attr, criu_svm_md->data.attrs, num_attrs *3988					sizeof(struct kfd_ioctl_svm_attribute));3989		set_attr[num_attrs].type = KFD_IOCTL_SVM_ATTR_CLR_FLAGS;3990		set_attr[num_attrs].value = ~set_flags;3991 3992		ret = svm_range_set_attr(p, mm, criu_svm_md->data.start_addr,3993					 criu_svm_md->data.size, num_attrs + 1,3994					 set_attr);3995		if (ret) {3996			pr_err("CRIU: failed to set range attributes\n");3997			goto exit;3998		}3999 4000		i++;4001	}4002exit:4003	kfree(set_attr);4004	list_for_each_entry_safe(criu_svm_md, next, &svms->criu_svm_metadata_list, list) {4005		pr_debug("freeing criu_svm_md[]\n\tstart: 0x%llx\n",4006						criu_svm_md->data.start_addr);4007		kfree(criu_svm_md);4008	}4009 4010	mmput(mm);4011	return ret;4012 4013}4014 4015int kfd_criu_restore_svm(struct kfd_process *p,4016			 uint8_t __user *user_priv_ptr,4017			 uint64_t *priv_data_offset,4018			 uint64_t max_priv_data_size)4019{4020	uint64_t svm_priv_data_size, svm_object_md_size, svm_attrs_size;4021	int nattr_common = 4, nattr_accessibility = 1;4022	struct criu_svm_metadata *criu_svm_md = NULL;4023	struct svm_range_list *svms = &p->svms;4024	uint32_t num_devices;4025	int ret = 0;4026 4027	num_devices = p->n_pdds;4028	/* Handle one SVM range object at a time, also the number of gpus are4029	 * assumed to be same on the restore node, checking must be done while4030	 * evaluating the topology earlier4031	 */4032 4033	svm_attrs_size = sizeof(struct kfd_ioctl_svm_attribute) *4034		(nattr_common + nattr_accessibility * num_devices);4035	svm_object_md_size = sizeof(struct criu_svm_metadata) + svm_attrs_size;4036 4037	svm_priv_data_size = sizeof(struct kfd_criu_svm_range_priv_data) +4038								svm_attrs_size;4039 4040	criu_svm_md = kzalloc(svm_object_md_size, GFP_KERNEL);4041	if (!criu_svm_md) {4042		pr_err("failed to allocate memory to store svm metadata\n");4043		return -ENOMEM;4044	}4045	if (*priv_data_offset + svm_priv_data_size > max_priv_data_size) {4046		ret = -EINVAL;4047		goto exit;4048	}4049 4050	ret = copy_from_user(&criu_svm_md->data, user_priv_ptr + *priv_data_offset,4051			     svm_priv_data_size);4052	if (ret) {4053		ret = -EFAULT;4054		goto exit;4055	}4056	*priv_data_offset += svm_priv_data_size;4057 4058	list_add_tail(&criu_svm_md->list, &svms->criu_svm_metadata_list);4059 4060	return 0;4061 4062 4063exit:4064	kfree(criu_svm_md);4065	return ret;4066}4067 4068int svm_range_get_info(struct kfd_process *p, uint32_t *num_svm_ranges,4069		       uint64_t *svm_priv_data_size)4070{4071	uint64_t total_size, accessibility_size, common_attr_size;4072	int nattr_common = 4, nattr_accessibility = 1;4073	int num_devices = p->n_pdds;4074	struct svm_range_list *svms;4075	struct svm_range *prange;4076	uint32_t count = 0;4077 4078	*svm_priv_data_size = 0;4079 4080	svms = &p->svms;4081	if (!svms)4082		return -EINVAL;4083 4084	mutex_lock(&svms->lock);4085	list_for_each_entry(prange, &svms->list, list) {4086		pr_debug("prange: 0x%p start: 0x%lx\t npages: 0x%llx\t end: 0x%llx\n",4087			 prange, prange->start, prange->npages,4088			 prange->start + prange->npages - 1);4089		count++;4090	}4091	mutex_unlock(&svms->lock);4092 4093	*num_svm_ranges = count;4094	/* Only the accessbility attributes need to be queried for all the gpus4095	 * individually, remaining ones are spanned across the entire process4096	 * regardless of the various gpu nodes. Of the remaining attributes,4097	 * KFD_IOCTL_SVM_ATTR_CLR_FLAGS need not be saved.4098	 *4099	 * KFD_IOCTL_SVM_ATTR_PREFERRED_LOC4100	 * KFD_IOCTL_SVM_ATTR_PREFETCH_LOC4101	 * KFD_IOCTL_SVM_ATTR_SET_FLAGS4102	 * KFD_IOCTL_SVM_ATTR_GRANULARITY4103	 *4104	 * ** ACCESSBILITY ATTRIBUTES **4105	 * (Considered as one, type is altered during query, value is gpuid)4106	 * KFD_IOCTL_SVM_ATTR_ACCESS4107	 * KFD_IOCTL_SVM_ATTR_ACCESS_IN_PLACE4108	 * KFD_IOCTL_SVM_ATTR_NO_ACCESS4109	 */4110	if (*num_svm_ranges > 0) {4111		common_attr_size = sizeof(struct kfd_ioctl_svm_attribute) *4112			nattr_common;4113		accessibility_size = sizeof(struct kfd_ioctl_svm_attribute) *4114			nattr_accessibility * num_devices;4115 4116		total_size = sizeof(struct kfd_criu_svm_range_priv_data) +4117			common_attr_size + accessibility_size;4118 4119		*svm_priv_data_size = *num_svm_ranges * total_size;4120	}4121 4122	pr_debug("num_svm_ranges %u total_priv_size %llu\n", *num_svm_ranges,4123		 *svm_priv_data_size);4124	return 0;4125}4126 4127int kfd_criu_checkpoint_svm(struct kfd_process *p,4128			    uint8_t __user *user_priv_data,4129			    uint64_t *priv_data_offset)4130{4131	struct kfd_criu_svm_range_priv_data *svm_priv = NULL;4132	struct kfd_ioctl_svm_attribute *query_attr = NULL;4133	uint64_t svm_priv_data_size, query_attr_size = 0;4134	int index, nattr_common = 4, ret = 0;4135	struct svm_range_list *svms;4136	int num_devices = p->n_pdds;4137	struct svm_range *prange;4138	struct mm_struct *mm;4139 4140	svms = &p->svms;4141	if (!svms)4142		return -EINVAL;4143 4144	mm = get_task_mm(p->lead_thread);4145	if (!mm) {4146		pr_err("failed to get mm for the target process\n");4147		return -ESRCH;4148	}4149 4150	query_attr_size = sizeof(struct kfd_ioctl_svm_attribute) *4151				(nattr_common + num_devices);4152 4153	query_attr = kzalloc(query_attr_size, GFP_KERNEL);4154	if (!query_attr) {4155		ret = -ENOMEM;4156		goto exit;4157	}4158 4159	query_attr[0].type = KFD_IOCTL_SVM_ATTR_PREFERRED_LOC;4160	query_attr[1].type = KFD_IOCTL_SVM_ATTR_PREFETCH_LOC;4161	query_attr[2].type = KFD_IOCTL_SVM_ATTR_SET_FLAGS;4162	query_attr[3].type = KFD_IOCTL_SVM_ATTR_GRANULARITY;4163 4164	for (index = 0; index < num_devices; index++) {4165		struct kfd_process_device *pdd = p->pdds[index];4166 4167		query_attr[index + nattr_common].type =4168			KFD_IOCTL_SVM_ATTR_ACCESS;4169		query_attr[index + nattr_common].value = pdd->user_gpu_id;4170	}4171 4172	svm_priv_data_size = sizeof(*svm_priv) + query_attr_size;4173 4174	svm_priv = kzalloc(svm_priv_data_size, GFP_KERNEL);4175	if (!svm_priv) {4176		ret = -ENOMEM;4177		goto exit_query;4178	}4179 4180	index = 0;4181	list_for_each_entry(prange, &svms->list, list) {4182 4183		svm_priv->object_type = KFD_CRIU_OBJECT_TYPE_SVM_RANGE;4184		svm_priv->start_addr = prange->start;4185		svm_priv->size = prange->npages;4186		memcpy(&svm_priv->attrs, query_attr, query_attr_size);4187		pr_debug("CRIU: prange: 0x%p start: 0x%lx\t npages: 0x%llx end: 0x%llx\t size: 0x%llx\n",4188			 prange, prange->start, prange->npages,4189			 prange->start + prange->npages - 1,4190			 prange->npages * PAGE_SIZE);4191 4192		ret = svm_range_get_attr(p, mm, svm_priv->start_addr,4193					 svm_priv->size,4194					 (nattr_common + num_devices),4195					 svm_priv->attrs);4196		if (ret) {4197			pr_err("CRIU: failed to obtain range attributes\n");4198			goto exit_priv;4199		}4200 4201		if (copy_to_user(user_priv_data + *priv_data_offset, svm_priv,4202				 svm_priv_data_size)) {4203			pr_err("Failed to copy svm priv to user\n");4204			ret = -EFAULT;4205			goto exit_priv;4206		}4207 4208		*priv_data_offset += svm_priv_data_size;4209 4210	}4211 4212 4213exit_priv:4214	kfree(svm_priv);4215exit_query:4216	kfree(query_attr);4217exit:4218	mmput(mm);4219	return ret;4220}4221 4222int4223svm_ioctl(struct kfd_process *p, enum kfd_ioctl_svm_op op, uint64_t start,4224	  uint64_t size, uint32_t nattrs, struct kfd_ioctl_svm_attribute *attrs)4225{4226	struct mm_struct *mm = current->mm;4227	int r;4228 4229	start >>= PAGE_SHIFT;4230	size >>= PAGE_SHIFT;4231 4232	switch (op) {4233	case KFD_IOCTL_SVM_OP_SET_ATTR:4234		r = svm_range_set_attr(p, mm, start, size, nattrs, attrs);4235		break;4236	case KFD_IOCTL_SVM_OP_GET_ATTR:4237		r = svm_range_get_attr(p, mm, start, size, nattrs, attrs);4238		break;4239	default:4240		r = EINVAL;4241		break;4242	}4243 4244	return r;4245}4246