brintos / linux-shallow public Read only

1// SPDX-License-Identifier: GPL-2.0 OR MIT2/*3 * Copyright 2014-2022 Advanced Micro Devices, Inc.4 *5 * Permission is hereby granted, free of charge, to any person obtaining a6 * copy of this software and associated documentation files (the "Software"),7 * to deal in the Software without restriction, including without limitation8 * the rights to use, copy, modify, merge, publish, distribute, sublicense,9 * and/or sell copies of the Software, and to permit persons to whom the10 * Software is furnished to do so, subject to the following conditions:11 *12 * The above copyright notice and this permission notice shall be included in13 * all copies or substantial portions of the Software.14 *15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL18 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR19 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,20 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR21 * OTHER DEALINGS IN THE SOFTWARE.22 *23 */24 25#include <linux/ratelimit.h>26#include <linux/printk.h>27#include <linux/slab.h>28#include <linux/list.h>29#include <linux/types.h>30#include <linux/bitops.h>31#include <linux/sched.h>32#include "kfd_priv.h"33#include "kfd_device_queue_manager.h"34#include "kfd_mqd_manager.h"35#include "cik_regs.h"36#include "kfd_kernel_queue.h"37#include "amdgpu_amdkfd.h"38#include "amdgpu_reset.h"39#include "mes_v11_api_def.h"40#include "kfd_debug.h"41 42/* Size of the per-pipe EOP queue */43#define CIK_HPD_EOP_BYTES_LOG2 1144#define CIK_HPD_EOP_BYTES (1U << CIK_HPD_EOP_BYTES_LOG2)45 46static int set_pasid_vmid_mapping(struct device_queue_manager *dqm,47				  u32 pasid, unsigned int vmid);48 49static int execute_queues_cpsch(struct device_queue_manager *dqm,50				enum kfd_unmap_queues_filter filter,51				uint32_t filter_param,52				uint32_t grace_period);53static int unmap_queues_cpsch(struct device_queue_manager *dqm,54				enum kfd_unmap_queues_filter filter,55				uint32_t filter_param,56				uint32_t grace_period,57				bool reset);58 59static int map_queues_cpsch(struct device_queue_manager *dqm);60 61static void deallocate_sdma_queue(struct device_queue_manager *dqm,62				struct queue *q);63 64static inline void deallocate_hqd(struct device_queue_manager *dqm,65				struct queue *q);66static int allocate_hqd(struct device_queue_manager *dqm, struct queue *q);67static int allocate_sdma_queue(struct device_queue_manager *dqm,68				struct queue *q, const uint32_t *restore_sdma_id);69static void kfd_process_hw_exception(struct work_struct *work);70 71static inline72enum KFD_MQD_TYPE get_mqd_type_from_queue_type(enum kfd_queue_type type)73{74	if (type == KFD_QUEUE_TYPE_SDMA || type == KFD_QUEUE_TYPE_SDMA_XGMI)75		return KFD_MQD_TYPE_SDMA;76	return KFD_MQD_TYPE_CP;77}78 79static bool is_pipe_enabled(struct device_queue_manager *dqm, int mec, int pipe)80{81	int i;82	int pipe_offset = (mec * dqm->dev->kfd->shared_resources.num_pipe_per_mec83		+ pipe) * dqm->dev->kfd->shared_resources.num_queue_per_pipe;84 85	/* queue is available for KFD usage if bit is 1 */86	for (i = 0; i <  dqm->dev->kfd->shared_resources.num_queue_per_pipe; ++i)87		if (test_bit(pipe_offset + i,88			      dqm->dev->kfd->shared_resources.cp_queue_bitmap))89			return true;90	return false;91}92 93unsigned int get_cp_queues_num(struct device_queue_manager *dqm)94{95	return bitmap_weight(dqm->dev->kfd->shared_resources.cp_queue_bitmap,96				AMDGPU_MAX_QUEUES);97}98 99unsigned int get_queues_per_pipe(struct device_queue_manager *dqm)100{101	return dqm->dev->kfd->shared_resources.num_queue_per_pipe;102}103 104unsigned int get_pipes_per_mec(struct device_queue_manager *dqm)105{106	return dqm->dev->kfd->shared_resources.num_pipe_per_mec;107}108 109static unsigned int get_num_all_sdma_engines(struct device_queue_manager *dqm)110{111	return kfd_get_num_sdma_engines(dqm->dev) +112		kfd_get_num_xgmi_sdma_engines(dqm->dev);113}114 115unsigned int get_num_sdma_queues(struct device_queue_manager *dqm)116{117	return kfd_get_num_sdma_engines(dqm->dev) *118		dqm->dev->kfd->device_info.num_sdma_queues_per_engine;119}120 121unsigned int get_num_xgmi_sdma_queues(struct device_queue_manager *dqm)122{123	return kfd_get_num_xgmi_sdma_engines(dqm->dev) *124		dqm->dev->kfd->device_info.num_sdma_queues_per_engine;125}126 127static void init_sdma_bitmaps(struct device_queue_manager *dqm)128{129	bitmap_zero(dqm->sdma_bitmap, KFD_MAX_SDMA_QUEUES);130	bitmap_set(dqm->sdma_bitmap, 0, get_num_sdma_queues(dqm));131 132	bitmap_zero(dqm->xgmi_sdma_bitmap, KFD_MAX_SDMA_QUEUES);133	bitmap_set(dqm->xgmi_sdma_bitmap, 0, get_num_xgmi_sdma_queues(dqm));134 135	/* Mask out the reserved queues */136	bitmap_andnot(dqm->sdma_bitmap, dqm->sdma_bitmap,137		      dqm->dev->kfd->device_info.reserved_sdma_queues_bitmap,138		      KFD_MAX_SDMA_QUEUES);139}140 141void program_sh_mem_settings(struct device_queue_manager *dqm,142					struct qcm_process_device *qpd)143{144	uint32_t xcc_mask = dqm->dev->xcc_mask;145	int xcc_id;146 147	for_each_inst(xcc_id, xcc_mask)148		dqm->dev->kfd2kgd->program_sh_mem_settings(149			dqm->dev->adev, qpd->vmid, qpd->sh_mem_config,150			qpd->sh_mem_ape1_base, qpd->sh_mem_ape1_limit,151			qpd->sh_mem_bases, xcc_id);152}153 154static void kfd_hws_hang(struct device_queue_manager *dqm)155{156	struct device_process_node *cur;157	struct qcm_process_device *qpd;158	struct queue *q;159 160	/* Mark all device queues as reset. */161	list_for_each_entry(cur, &dqm->queues, list) {162		qpd = cur->qpd;163		list_for_each_entry(q, &qpd->queues_list, list) {164			struct kfd_process_device *pdd = qpd_to_pdd(qpd);165 166			pdd->has_reset_queue = true;167		}168	}169 170	/*171	 * Issue a GPU reset if HWS is unresponsive172	 */173	schedule_work(&dqm->hw_exception_work);174}175 176static int convert_to_mes_queue_type(int queue_type)177{178	int mes_queue_type;179 180	switch (queue_type) {181	case KFD_QUEUE_TYPE_COMPUTE:182		mes_queue_type = MES_QUEUE_TYPE_COMPUTE;183		break;184	case KFD_QUEUE_TYPE_SDMA:185		mes_queue_type = MES_QUEUE_TYPE_SDMA;186		break;187	default:188		WARN(1, "Invalid queue type %d", queue_type);189		mes_queue_type = -EINVAL;190		break;191	}192 193	return mes_queue_type;194}195 196static int add_queue_mes(struct device_queue_manager *dqm, struct queue *q,197			 struct qcm_process_device *qpd)198{199	struct amdgpu_device *adev = (struct amdgpu_device *)dqm->dev->adev;200	struct kfd_process_device *pdd = qpd_to_pdd(qpd);201	struct mes_add_queue_input queue_input;202	int r, queue_type;203	uint64_t wptr_addr_off;204 205	if (!down_read_trylock(&adev->reset_domain->sem))206		return -EIO;207 208	memset(&queue_input, 0x0, sizeof(struct mes_add_queue_input));209	queue_input.process_id = qpd->pqm->process->pasid;210	queue_input.page_table_base_addr =  qpd->page_table_base;211	queue_input.process_va_start = 0;212	queue_input.process_va_end = adev->vm_manager.max_pfn - 1;213	/* MES unit for quantum is 100ns */214	queue_input.process_quantum = KFD_MES_PROCESS_QUANTUM;  /* Equivalent to 10ms. */215	queue_input.process_context_addr = pdd->proc_ctx_gpu_addr;216	queue_input.gang_quantum = KFD_MES_GANG_QUANTUM; /* Equivalent to 1ms */217	queue_input.gang_context_addr = q->gang_ctx_gpu_addr;218	queue_input.inprocess_gang_priority = q->properties.priority;219	queue_input.gang_global_priority_level =220					AMDGPU_MES_PRIORITY_LEVEL_NORMAL;221	queue_input.doorbell_offset = q->properties.doorbell_off;222	queue_input.mqd_addr = q->gart_mqd_addr;223	queue_input.wptr_addr = (uint64_t)q->properties.write_ptr;224 225	wptr_addr_off = (uint64_t)q->properties.write_ptr & (PAGE_SIZE - 1);226	queue_input.wptr_mc_addr = amdgpu_bo_gpu_offset(q->properties.wptr_bo) + wptr_addr_off;227 228	queue_input.is_kfd_process = 1;229	queue_input.is_aql_queue = (q->properties.format == KFD_QUEUE_FORMAT_AQL);230	queue_input.queue_size = q->properties.queue_size >> 2;231 232	queue_input.paging = false;233	queue_input.tba_addr = qpd->tba_addr;234	queue_input.tma_addr = qpd->tma_addr;235	queue_input.trap_en = !kfd_dbg_has_cwsr_workaround(q->device);236	queue_input.skip_process_ctx_clear =237		qpd->pqm->process->runtime_info.runtime_state == DEBUG_RUNTIME_STATE_ENABLED &&238						(qpd->pqm->process->debug_trap_enabled ||239						 kfd_dbg_has_ttmps_always_setup(q->device));240 241	queue_type = convert_to_mes_queue_type(q->properties.type);242	if (queue_type < 0) {243		dev_err(adev->dev, "Queue type not supported with MES, queue:%d\n",244			q->properties.type);245		up_read(&adev->reset_domain->sem);246		return -EINVAL;247	}248	queue_input.queue_type = (uint32_t)queue_type;249 250	queue_input.exclusively_scheduled = q->properties.is_gws;251 252	amdgpu_mes_lock(&adev->mes);253	r = adev->mes.funcs->add_hw_queue(&adev->mes, &queue_input);254	amdgpu_mes_unlock(&adev->mes);255	up_read(&adev->reset_domain->sem);256	if (r) {257		dev_err(adev->dev, "failed to add hardware queue to MES, doorbell=0x%x\n",258			q->properties.doorbell_off);259		dev_err(adev->dev, "MES might be in unrecoverable state, issue a GPU reset\n");260		kfd_hws_hang(dqm);261	}262 263	return r;264}265 266static int remove_queue_mes(struct device_queue_manager *dqm, struct queue *q,267			struct qcm_process_device *qpd)268{269	struct amdgpu_device *adev = (struct amdgpu_device *)dqm->dev->adev;270	int r;271	struct mes_remove_queue_input queue_input;272 273	if (!down_read_trylock(&adev->reset_domain->sem))274		return -EIO;275 276	memset(&queue_input, 0x0, sizeof(struct mes_remove_queue_input));277	queue_input.doorbell_offset = q->properties.doorbell_off;278	queue_input.gang_context_addr = q->gang_ctx_gpu_addr;279 280	amdgpu_mes_lock(&adev->mes);281	r = adev->mes.funcs->remove_hw_queue(&adev->mes, &queue_input);282	amdgpu_mes_unlock(&adev->mes);283	up_read(&adev->reset_domain->sem);284 285	if (r) {286		dev_err(adev->dev, "failed to remove hardware queue from MES, doorbell=0x%x\n",287			q->properties.doorbell_off);288		dev_err(adev->dev, "MES might be in unrecoverable state, issue a GPU reset\n");289		kfd_hws_hang(dqm);290	}291 292	return r;293}294 295static int remove_all_queues_mes(struct device_queue_manager *dqm)296{297	struct device_process_node *cur;298	struct device *dev = dqm->dev->adev->dev;299	struct qcm_process_device *qpd;300	struct queue *q;301	int retval = 0;302 303	list_for_each_entry(cur, &dqm->queues, list) {304		qpd = cur->qpd;305		list_for_each_entry(q, &qpd->queues_list, list) {306			if (q->properties.is_active) {307				retval = remove_queue_mes(dqm, q, qpd);308				if (retval) {309					dev_err(dev, "%s: Failed to remove queue %d for dev %d",310						__func__,311						q->properties.queue_id,312						dqm->dev->id);313					return retval;314				}315			}316		}317	}318 319	return retval;320}321 322static int suspend_all_queues_mes(struct device_queue_manager *dqm)323{324	struct amdgpu_device *adev = (struct amdgpu_device *)dqm->dev->adev;325	int r = 0;326 327	if (!down_read_trylock(&adev->reset_domain->sem))328		return -EIO;329 330	r = amdgpu_mes_suspend(adev);331	up_read(&adev->reset_domain->sem);332 333	if (r) {334		dev_err(adev->dev, "failed to suspend gangs from MES\n");335		dev_err(adev->dev, "MES might be in unrecoverable state, issue a GPU reset\n");336		kfd_hws_hang(dqm);337	}338 339	return r;340}341 342static int resume_all_queues_mes(struct device_queue_manager *dqm)343{344	struct amdgpu_device *adev = (struct amdgpu_device *)dqm->dev->adev;345	int r = 0;346 347	if (!down_read_trylock(&adev->reset_domain->sem))348		return -EIO;349 350	r = amdgpu_mes_resume(adev);351	up_read(&adev->reset_domain->sem);352 353	if (r) {354		dev_err(adev->dev, "failed to resume gangs from MES\n");355		dev_err(adev->dev, "MES might be in unrecoverable state, issue a GPU reset\n");356		kfd_hws_hang(dqm);357	}358 359	return r;360}361 362static void increment_queue_count(struct device_queue_manager *dqm,363				  struct qcm_process_device *qpd,364				  struct queue *q)365{366	dqm->active_queue_count++;367	if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE ||368	    q->properties.type == KFD_QUEUE_TYPE_DIQ)369		dqm->active_cp_queue_count++;370 371	if (q->properties.is_gws) {372		dqm->gws_queue_count++;373		qpd->mapped_gws_queue = true;374	}375}376 377static void decrement_queue_count(struct device_queue_manager *dqm,378				  struct qcm_process_device *qpd,379				  struct queue *q)380{381	dqm->active_queue_count--;382	if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE ||383	    q->properties.type == KFD_QUEUE_TYPE_DIQ)384		dqm->active_cp_queue_count--;385 386	if (q->properties.is_gws) {387		dqm->gws_queue_count--;388		qpd->mapped_gws_queue = false;389	}390}391 392/*393 * Allocate a doorbell ID to this queue.394 * If doorbell_id is passed in, make sure requested ID is valid then allocate it.395 */396static int allocate_doorbell(struct qcm_process_device *qpd,397			     struct queue *q,398			     uint32_t const *restore_id)399{400	struct kfd_node *dev = qpd->dqm->dev;401 402	if (!KFD_IS_SOC15(dev)) {403		/* On pre-SOC15 chips we need to use the queue ID to404		 * preserve the user mode ABI.405		 */406 407		if (restore_id && *restore_id != q->properties.queue_id)408			return -EINVAL;409 410		q->doorbell_id = q->properties.queue_id;411	} else if (q->properties.type == KFD_QUEUE_TYPE_SDMA ||412			q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) {413		/* For SDMA queues on SOC15 with 8-byte doorbell, use static414		 * doorbell assignments based on the engine and queue id.415		 * The doobell index distance between RLC (2*i) and (2*i+1)416		 * for a SDMA engine is 512.417		 */418 419		uint32_t *idx_offset = dev->kfd->shared_resources.sdma_doorbell_idx;420 421		/*422		 * q->properties.sdma_engine_id corresponds to the virtual423		 * sdma engine number. However, for doorbell allocation,424		 * we need the physical sdma engine id in order to get the425		 * correct doorbell offset.426		 */427		uint32_t valid_id = idx_offset[qpd->dqm->dev->node_id *428					       get_num_all_sdma_engines(qpd->dqm) +429					       q->properties.sdma_engine_id]430						+ (q->properties.sdma_queue_id & 1)431						* KFD_QUEUE_DOORBELL_MIRROR_OFFSET432						+ (q->properties.sdma_queue_id >> 1);433 434		if (restore_id && *restore_id != valid_id)435			return -EINVAL;436		q->doorbell_id = valid_id;437	} else {438		/* For CP queues on SOC15 */439		if (restore_id) {440			/* make sure that ID is free  */441			if (__test_and_set_bit(*restore_id, qpd->doorbell_bitmap))442				return -EINVAL;443 444			q->doorbell_id = *restore_id;445		} else {446			/* or reserve a free doorbell ID */447			unsigned int found;448 449			found = find_first_zero_bit(qpd->doorbell_bitmap,450						    KFD_MAX_NUM_OF_QUEUES_PER_PROCESS);451			if (found >= KFD_MAX_NUM_OF_QUEUES_PER_PROCESS) {452				pr_debug("No doorbells available");453				return -EBUSY;454			}455			set_bit(found, qpd->doorbell_bitmap);456			q->doorbell_id = found;457		}458	}459 460	q->properties.doorbell_off = amdgpu_doorbell_index_on_bar(dev->adev,461								  qpd->proc_doorbells,462								  q->doorbell_id,463								  dev->kfd->device_info.doorbell_size);464	return 0;465}466 467static void deallocate_doorbell(struct qcm_process_device *qpd,468				struct queue *q)469{470	unsigned int old;471	struct kfd_node *dev = qpd->dqm->dev;472 473	if (!KFD_IS_SOC15(dev) ||474	    q->properties.type == KFD_QUEUE_TYPE_SDMA ||475	    q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)476		return;477 478	old = test_and_clear_bit(q->doorbell_id, qpd->doorbell_bitmap);479	WARN_ON(!old);480}481 482static void program_trap_handler_settings(struct device_queue_manager *dqm,483				struct qcm_process_device *qpd)484{485	uint32_t xcc_mask = dqm->dev->xcc_mask;486	int xcc_id;487 488	if (dqm->dev->kfd2kgd->program_trap_handler_settings)489		for_each_inst(xcc_id, xcc_mask)490			dqm->dev->kfd2kgd->program_trap_handler_settings(491				dqm->dev->adev, qpd->vmid, qpd->tba_addr,492				qpd->tma_addr, xcc_id);493}494 495static int allocate_vmid(struct device_queue_manager *dqm,496			struct qcm_process_device *qpd,497			struct queue *q)498{499	struct device *dev = dqm->dev->adev->dev;500	int allocated_vmid = -1, i;501 502	for (i = dqm->dev->vm_info.first_vmid_kfd;503			i <= dqm->dev->vm_info.last_vmid_kfd; i++) {504		if (!dqm->vmid_pasid[i]) {505			allocated_vmid = i;506			break;507		}508	}509 510	if (allocated_vmid < 0) {511		dev_err(dev, "no more vmid to allocate\n");512		return -ENOSPC;513	}514 515	pr_debug("vmid allocated: %d\n", allocated_vmid);516 517	dqm->vmid_pasid[allocated_vmid] = q->process->pasid;518 519	set_pasid_vmid_mapping(dqm, q->process->pasid, allocated_vmid);520 521	qpd->vmid = allocated_vmid;522	q->properties.vmid = allocated_vmid;523 524	program_sh_mem_settings(dqm, qpd);525 526	if (KFD_IS_SOC15(dqm->dev) && dqm->dev->kfd->cwsr_enabled)527		program_trap_handler_settings(dqm, qpd);528 529	/* qpd->page_table_base is set earlier when register_process()530	 * is called, i.e. when the first queue is created.531	 */532	dqm->dev->kfd2kgd->set_vm_context_page_table_base(dqm->dev->adev,533			qpd->vmid,534			qpd->page_table_base);535	/* invalidate the VM context after pasid and vmid mapping is set up */536	kfd_flush_tlb(qpd_to_pdd(qpd), TLB_FLUSH_LEGACY);537 538	if (dqm->dev->kfd2kgd->set_scratch_backing_va)539		dqm->dev->kfd2kgd->set_scratch_backing_va(dqm->dev->adev,540				qpd->sh_hidden_private_base, qpd->vmid);541 542	return 0;543}544 545static int flush_texture_cache_nocpsch(struct kfd_node *kdev,546				struct qcm_process_device *qpd)547{548	const struct packet_manager_funcs *pmf = qpd->dqm->packet_mgr.pmf;549	int ret;550 551	if (!qpd->ib_kaddr)552		return -ENOMEM;553 554	ret = pmf->release_mem(qpd->ib_base, (uint32_t *)qpd->ib_kaddr);555	if (ret)556		return ret;557 558	return amdgpu_amdkfd_submit_ib(kdev->adev, KGD_ENGINE_MEC1, qpd->vmid,559				qpd->ib_base, (uint32_t *)qpd->ib_kaddr,560				pmf->release_mem_size / sizeof(uint32_t));561}562 563static void deallocate_vmid(struct device_queue_manager *dqm,564				struct qcm_process_device *qpd,565				struct queue *q)566{567	struct device *dev = dqm->dev->adev->dev;568 569	/* On GFX v7, CP doesn't flush TC at dequeue */570	if (q->device->adev->asic_type == CHIP_HAWAII)571		if (flush_texture_cache_nocpsch(q->device, qpd))572			dev_err(dev, "Failed to flush TC\n");573 574	kfd_flush_tlb(qpd_to_pdd(qpd), TLB_FLUSH_LEGACY);575 576	/* Release the vmid mapping */577	set_pasid_vmid_mapping(dqm, 0, qpd->vmid);578	dqm->vmid_pasid[qpd->vmid] = 0;579 580	qpd->vmid = 0;581	q->properties.vmid = 0;582}583 584static int create_queue_nocpsch(struct device_queue_manager *dqm,585				struct queue *q,586				struct qcm_process_device *qpd,587				const struct kfd_criu_queue_priv_data *qd,588				const void *restore_mqd, const void *restore_ctl_stack)589{590	struct mqd_manager *mqd_mgr;591	int retval;592 593	dqm_lock(dqm);594 595	if (dqm->total_queue_count >= max_num_of_queues_per_device) {596		pr_warn("Can't create new usermode queue because %d queues were already created\n",597				dqm->total_queue_count);598		retval = -EPERM;599		goto out_unlock;600	}601 602	if (list_empty(&qpd->queues_list)) {603		retval = allocate_vmid(dqm, qpd, q);604		if (retval)605			goto out_unlock;606	}607	q->properties.vmid = qpd->vmid;608	/*609	 * Eviction state logic: mark all queues as evicted, even ones610	 * not currently active. Restoring inactive queues later only611	 * updates the is_evicted flag but is a no-op otherwise.612	 */613	q->properties.is_evicted = !!qpd->evicted;614 615	q->properties.tba_addr = qpd->tba_addr;616	q->properties.tma_addr = qpd->tma_addr;617 618	mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type(619			q->properties.type)];620	if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE) {621		retval = allocate_hqd(dqm, q);622		if (retval)623			goto deallocate_vmid;624		pr_debug("Loading mqd to hqd on pipe %d, queue %d\n",625			q->pipe, q->queue);626	} else if (q->properties.type == KFD_QUEUE_TYPE_SDMA ||627		q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) {628		retval = allocate_sdma_queue(dqm, q, qd ? &qd->sdma_id : NULL);629		if (retval)630			goto deallocate_vmid;631		dqm->asic_ops.init_sdma_vm(dqm, q, qpd);632	}633 634	retval = allocate_doorbell(qpd, q, qd ? &qd->doorbell_id : NULL);635	if (retval)636		goto out_deallocate_hqd;637 638	/* Temporarily release dqm lock to avoid a circular lock dependency */639	dqm_unlock(dqm);640	q->mqd_mem_obj = mqd_mgr->allocate_mqd(mqd_mgr->dev, &q->properties);641	dqm_lock(dqm);642 643	if (!q->mqd_mem_obj) {644		retval = -ENOMEM;645		goto out_deallocate_doorbell;646	}647 648	if (qd)649		mqd_mgr->restore_mqd(mqd_mgr, &q->mqd, q->mqd_mem_obj, &q->gart_mqd_addr,650				     &q->properties, restore_mqd, restore_ctl_stack,651				     qd->ctl_stack_size);652	else653		mqd_mgr->init_mqd(mqd_mgr, &q->mqd, q->mqd_mem_obj,654					&q->gart_mqd_addr, &q->properties);655 656	if (q->properties.is_active) {657		if (!dqm->sched_running) {658			WARN_ONCE(1, "Load non-HWS mqd while stopped\n");659			goto add_queue_to_list;660		}661 662		if (WARN(q->process->mm != current->mm,663					"should only run in user thread"))664			retval = -EFAULT;665		else666			retval = mqd_mgr->load_mqd(mqd_mgr, q->mqd, q->pipe,667					q->queue, &q->properties, current->mm);668		if (retval)669			goto out_free_mqd;670	}671 672add_queue_to_list:673	list_add(&q->list, &qpd->queues_list);674	qpd->queue_count++;675	if (q->properties.is_active)676		increment_queue_count(dqm, qpd, q);677 678	/*679	 * Unconditionally increment this counter, regardless of the queue's680	 * type or whether the queue is active.681	 */682	dqm->total_queue_count++;683	pr_debug("Total of %d queues are accountable so far\n",684			dqm->total_queue_count);685	goto out_unlock;686 687out_free_mqd:688	mqd_mgr->free_mqd(mqd_mgr, q->mqd, q->mqd_mem_obj);689out_deallocate_doorbell:690	deallocate_doorbell(qpd, q);691out_deallocate_hqd:692	if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE)693		deallocate_hqd(dqm, q);694	else if (q->properties.type == KFD_QUEUE_TYPE_SDMA ||695		q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)696		deallocate_sdma_queue(dqm, q);697deallocate_vmid:698	if (list_empty(&qpd->queues_list))699		deallocate_vmid(dqm, qpd, q);700out_unlock:701	dqm_unlock(dqm);702	return retval;703}704 705static int allocate_hqd(struct device_queue_manager *dqm, struct queue *q)706{707	bool set;708	int pipe, bit, i;709 710	set = false;711 712	for (pipe = dqm->next_pipe_to_allocate, i = 0;713			i < get_pipes_per_mec(dqm);714			pipe = ((pipe + 1) % get_pipes_per_mec(dqm)), ++i) {715 716		if (!is_pipe_enabled(dqm, 0, pipe))717			continue;718 719		if (dqm->allocated_queues[pipe] != 0) {720			bit = ffs(dqm->allocated_queues[pipe]) - 1;721			dqm->allocated_queues[pipe] &= ~(1 << bit);722			q->pipe = pipe;723			q->queue = bit;724			set = true;725			break;726		}727	}728 729	if (!set)730		return -EBUSY;731 732	pr_debug("hqd slot - pipe %d, queue %d\n", q->pipe, q->queue);733	/* horizontal hqd allocation */734	dqm->next_pipe_to_allocate = (pipe + 1) % get_pipes_per_mec(dqm);735 736	return 0;737}738 739static inline void deallocate_hqd(struct device_queue_manager *dqm,740				struct queue *q)741{742	dqm->allocated_queues[q->pipe] |= (1 << q->queue);743}744 745#define SQ_IND_CMD_CMD_KILL		0x00000003746#define SQ_IND_CMD_MODE_BROADCAST	0x00000001747 748static int dbgdev_wave_reset_wavefronts(struct kfd_node *dev, struct kfd_process *p)749{750	int status = 0;751	unsigned int vmid;752	uint16_t queried_pasid;753	union SQ_CMD_BITS reg_sq_cmd;754	union GRBM_GFX_INDEX_BITS reg_gfx_index;755	struct kfd_process_device *pdd;756	int first_vmid_to_scan = dev->vm_info.first_vmid_kfd;757	int last_vmid_to_scan = dev->vm_info.last_vmid_kfd;758	uint32_t xcc_mask = dev->xcc_mask;759	int xcc_id;760 761	reg_sq_cmd.u32All = 0;762	reg_gfx_index.u32All = 0;763 764	pr_debug("Killing all process wavefronts\n");765 766	if (!dev->kfd2kgd->get_atc_vmid_pasid_mapping_info) {767		dev_err(dev->adev->dev, "no vmid pasid mapping supported\n");768		return -EOPNOTSUPP;769	}770 771	/* Scan all registers in the range ATC_VMID8_PASID_MAPPING ..772	 * ATC_VMID15_PASID_MAPPING773	 * to check which VMID the current process is mapped to.774	 */775 776	for (vmid = first_vmid_to_scan; vmid <= last_vmid_to_scan; vmid++) {777		status = dev->kfd2kgd->get_atc_vmid_pasid_mapping_info778				(dev->adev, vmid, &queried_pasid);779 780		if (status && queried_pasid == p->pasid) {781			pr_debug("Killing wave fronts of vmid %d and pasid 0x%x\n",782					vmid, p->pasid);783			break;784		}785	}786 787	if (vmid > last_vmid_to_scan) {788		dev_err(dev->adev->dev, "Didn't find vmid for pasid 0x%x\n", p->pasid);789		return -EFAULT;790	}791 792	/* taking the VMID for that process on the safe way using PDD */793	pdd = kfd_get_process_device_data(dev, p);794	if (!pdd)795		return -EFAULT;796 797	reg_gfx_index.bits.sh_broadcast_writes = 1;798	reg_gfx_index.bits.se_broadcast_writes = 1;799	reg_gfx_index.bits.instance_broadcast_writes = 1;800	reg_sq_cmd.bits.mode = SQ_IND_CMD_MODE_BROADCAST;801	reg_sq_cmd.bits.cmd = SQ_IND_CMD_CMD_KILL;802	reg_sq_cmd.bits.vm_id = vmid;803 804	for_each_inst(xcc_id, xcc_mask)805		dev->kfd2kgd->wave_control_execute(806			dev->adev, reg_gfx_index.u32All,807			reg_sq_cmd.u32All, xcc_id);808 809	return 0;810}811 812/* Access to DQM has to be locked before calling destroy_queue_nocpsch_locked813 * to avoid asynchronized access814 */815static int destroy_queue_nocpsch_locked(struct device_queue_manager *dqm,816				struct qcm_process_device *qpd,817				struct queue *q)818{819	int retval;820	struct mqd_manager *mqd_mgr;821 822	mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type(823			q->properties.type)];824 825	if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE)826		deallocate_hqd(dqm, q);827	else if (q->properties.type == KFD_QUEUE_TYPE_SDMA)828		deallocate_sdma_queue(dqm, q);829	else if (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)830		deallocate_sdma_queue(dqm, q);831	else {832		pr_debug("q->properties.type %d is invalid\n",833				q->properties.type);834		return -EINVAL;835	}836	dqm->total_queue_count--;837 838	deallocate_doorbell(qpd, q);839 840	if (!dqm->sched_running) {841		WARN_ONCE(1, "Destroy non-HWS queue while stopped\n");842		return 0;843	}844 845	retval = mqd_mgr->destroy_mqd(mqd_mgr, q->mqd,846				KFD_PREEMPT_TYPE_WAVEFRONT_RESET,847				KFD_UNMAP_LATENCY_MS,848				q->pipe, q->queue);849	if (retval == -ETIME)850		qpd->reset_wavefronts = true;851 852	list_del(&q->list);853	if (list_empty(&qpd->queues_list)) {854		if (qpd->reset_wavefronts) {855			pr_warn("Resetting wave fronts (nocpsch) on dev %p\n",856					dqm->dev);857			/* dbgdev_wave_reset_wavefronts has to be called before858			 * deallocate_vmid(), i.e. when vmid is still in use.859			 */860			dbgdev_wave_reset_wavefronts(dqm->dev,861					qpd->pqm->process);862			qpd->reset_wavefronts = false;863		}864 865		deallocate_vmid(dqm, qpd, q);866	}867	qpd->queue_count--;868	if (q->properties.is_active)869		decrement_queue_count(dqm, qpd, q);870 871	return retval;872}873 874static int destroy_queue_nocpsch(struct device_queue_manager *dqm,875				struct qcm_process_device *qpd,876				struct queue *q)877{878	int retval;879	uint64_t sdma_val = 0;880	struct device *dev = dqm->dev->adev->dev;881	struct kfd_process_device *pdd = qpd_to_pdd(qpd);882	struct mqd_manager *mqd_mgr =883		dqm->mqd_mgrs[get_mqd_type_from_queue_type(q->properties.type)];884 885	/* Get the SDMA queue stats */886	if ((q->properties.type == KFD_QUEUE_TYPE_SDMA) ||887	    (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)) {888		retval = read_sdma_queue_counter((uint64_t __user *)q->properties.read_ptr,889							&sdma_val);890		if (retval)891			dev_err(dev, "Failed to read SDMA queue counter for queue: %d\n",892				q->properties.queue_id);893	}894 895	dqm_lock(dqm);896	retval = destroy_queue_nocpsch_locked(dqm, qpd, q);897	if (!retval)898		pdd->sdma_past_activity_counter += sdma_val;899	dqm_unlock(dqm);900 901	mqd_mgr->free_mqd(mqd_mgr, q->mqd, q->mqd_mem_obj);902 903	return retval;904}905 906static int update_queue(struct device_queue_manager *dqm, struct queue *q,907			struct mqd_update_info *minfo)908{909	int retval = 0;910	struct device *dev = dqm->dev->adev->dev;911	struct mqd_manager *mqd_mgr;912	struct kfd_process_device *pdd;913	bool prev_active = false;914 915	dqm_lock(dqm);916	pdd = kfd_get_process_device_data(q->device, q->process);917	if (!pdd) {918		retval = -ENODEV;919		goto out_unlock;920	}921	mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type(922			q->properties.type)];923 924	/* Save previous activity state for counters */925	prev_active = q->properties.is_active;926 927	/* Make sure the queue is unmapped before updating the MQD */928	if (dqm->sched_policy != KFD_SCHED_POLICY_NO_HWS) {929		if (!dqm->dev->kfd->shared_resources.enable_mes)930			retval = unmap_queues_cpsch(dqm,931						    KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0, USE_DEFAULT_GRACE_PERIOD, false);932		else if (prev_active)933			retval = remove_queue_mes(dqm, q, &pdd->qpd);934 935		/* queue is reset so inaccessable  */936		if (pdd->has_reset_queue) {937			retval = -EACCES;938			goto out_unlock;939		}940 941		if (retval) {942			dev_err(dev, "unmap queue failed\n");943			goto out_unlock;944		}945	} else if (prev_active &&946		   (q->properties.type == KFD_QUEUE_TYPE_COMPUTE ||947		    q->properties.type == KFD_QUEUE_TYPE_SDMA ||948		    q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)) {949 950		if (!dqm->sched_running) {951			WARN_ONCE(1, "Update non-HWS queue while stopped\n");952			goto out_unlock;953		}954 955		retval = mqd_mgr->destroy_mqd(mqd_mgr, q->mqd,956				(dqm->dev->kfd->cwsr_enabled ?957				 KFD_PREEMPT_TYPE_WAVEFRONT_SAVE :958				 KFD_PREEMPT_TYPE_WAVEFRONT_DRAIN),959				KFD_UNMAP_LATENCY_MS, q->pipe, q->queue);960		if (retval) {961			dev_err(dev, "destroy mqd failed\n");962			goto out_unlock;963		}964	}965 966	mqd_mgr->update_mqd(mqd_mgr, q->mqd, &q->properties, minfo);967 968	/*969	 * check active state vs. the previous state and modify970	 * counter accordingly. map_queues_cpsch uses the971	 * dqm->active_queue_count to determine whether a new runlist must be972	 * uploaded.973	 */974	if (q->properties.is_active && !prev_active) {975		increment_queue_count(dqm, &pdd->qpd, q);976	} else if (!q->properties.is_active && prev_active) {977		decrement_queue_count(dqm, &pdd->qpd, q);978	} else if (q->gws && !q->properties.is_gws) {979		if (q->properties.is_active) {980			dqm->gws_queue_count++;981			pdd->qpd.mapped_gws_queue = true;982		}983		q->properties.is_gws = true;984	} else if (!q->gws && q->properties.is_gws) {985		if (q->properties.is_active) {986			dqm->gws_queue_count--;987			pdd->qpd.mapped_gws_queue = false;988		}989		q->properties.is_gws = false;990	}991 992	if (dqm->sched_policy != KFD_SCHED_POLICY_NO_HWS) {993		if (!dqm->dev->kfd->shared_resources.enable_mes)994			retval = map_queues_cpsch(dqm);995		else if (q->properties.is_active)996			retval = add_queue_mes(dqm, q, &pdd->qpd);997	} else if (q->properties.is_active &&998		 (q->properties.type == KFD_QUEUE_TYPE_COMPUTE ||999		  q->properties.type == KFD_QUEUE_TYPE_SDMA ||1000		  q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)) {1001		if (WARN(q->process->mm != current->mm,1002			 "should only run in user thread"))1003			retval = -EFAULT;1004		else1005			retval = mqd_mgr->load_mqd(mqd_mgr, q->mqd,1006						   q->pipe, q->queue,1007						   &q->properties, current->mm);1008	}1009 1010out_unlock:1011	dqm_unlock(dqm);1012	return retval;1013}1014 1015/* suspend_single_queue does not lock the dqm like the1016 * evict_process_queues_cpsch or evict_process_queues_nocpsch. You should1017 * lock the dqm before calling, and unlock after calling.1018 *1019 * The reason we don't lock the dqm is because this function may be1020 * called on multiple queues in a loop, so rather than locking/unlocking1021 * multiple times, we will just keep the dqm locked for all of the calls.1022 */1023static int suspend_single_queue(struct device_queue_manager *dqm,1024				      struct kfd_process_device *pdd,1025				      struct queue *q)1026{1027	bool is_new;1028 1029	if (q->properties.is_suspended)1030		return 0;1031 1032	pr_debug("Suspending PASID %u queue [%i]\n",1033			pdd->process->pasid,1034			q->properties.queue_id);1035 1036	is_new = q->properties.exception_status & KFD_EC_MASK(EC_QUEUE_NEW);1037 1038	if (is_new || q->properties.is_being_destroyed) {1039		pr_debug("Suspend: skip %s queue id %i\n",1040				is_new ? "new" : "destroyed",1041				q->properties.queue_id);1042		return -EBUSY;1043	}1044 1045	q->properties.is_suspended = true;1046	if (q->properties.is_active) {1047		if (dqm->dev->kfd->shared_resources.enable_mes) {1048			int r = remove_queue_mes(dqm, q, &pdd->qpd);1049 1050			if (r)1051				return r;1052		}1053 1054		decrement_queue_count(dqm, &pdd->qpd, q);1055		q->properties.is_active = false;1056	}1057 1058	return 0;1059}1060 1061/* resume_single_queue does not lock the dqm like the functions1062 * restore_process_queues_cpsch or restore_process_queues_nocpsch. You should1063 * lock the dqm before calling, and unlock after calling.1064 *1065 * The reason we don't lock the dqm is because this function may be1066 * called on multiple queues in a loop, so rather than locking/unlocking1067 * multiple times, we will just keep the dqm locked for all of the calls.1068 */1069static int resume_single_queue(struct device_queue_manager *dqm,1070				      struct qcm_process_device *qpd,1071				      struct queue *q)1072{1073	struct kfd_process_device *pdd;1074 1075	if (!q->properties.is_suspended)1076		return 0;1077 1078	pdd = qpd_to_pdd(qpd);1079 1080	pr_debug("Restoring from suspend PASID %u queue [%i]\n",1081			    pdd->process->pasid,1082			    q->properties.queue_id);1083 1084	q->properties.is_suspended = false;1085 1086	if (QUEUE_IS_ACTIVE(q->properties)) {1087		if (dqm->dev->kfd->shared_resources.enable_mes) {1088			int r = add_queue_mes(dqm, q, &pdd->qpd);1089 1090			if (r)1091				return r;1092		}1093 1094		q->properties.is_active = true;1095		increment_queue_count(dqm, qpd, q);1096	}1097 1098	return 0;1099}1100 1101static int evict_process_queues_nocpsch(struct device_queue_manager *dqm,1102					struct qcm_process_device *qpd)1103{1104	struct queue *q;1105	struct mqd_manager *mqd_mgr;1106	struct kfd_process_device *pdd;1107	int retval, ret = 0;1108 1109	dqm_lock(dqm);1110	if (qpd->evicted++ > 0) /* already evicted, do nothing */1111		goto out;1112 1113	pdd = qpd_to_pdd(qpd);1114	pr_debug_ratelimited("Evicting PASID 0x%x queues\n",1115			    pdd->process->pasid);1116 1117	pdd->last_evict_timestamp = get_jiffies_64();1118	/* Mark all queues as evicted. Deactivate all active queues on1119	 * the qpd.1120	 */1121	list_for_each_entry(q, &qpd->queues_list, list) {1122		q->properties.is_evicted = true;1123		if (!q->properties.is_active)1124			continue;1125 1126		mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type(1127				q->properties.type)];1128		q->properties.is_active = false;1129		decrement_queue_count(dqm, qpd, q);1130 1131		if (WARN_ONCE(!dqm->sched_running, "Evict when stopped\n"))1132			continue;1133 1134		retval = mqd_mgr->destroy_mqd(mqd_mgr, q->mqd,1135				(dqm->dev->kfd->cwsr_enabled ?1136				 KFD_PREEMPT_TYPE_WAVEFRONT_SAVE :1137				 KFD_PREEMPT_TYPE_WAVEFRONT_DRAIN),1138				KFD_UNMAP_LATENCY_MS, q->pipe, q->queue);1139		if (retval && !ret)1140			/* Return the first error, but keep going to1141			 * maintain a consistent eviction state1142			 */1143			ret = retval;1144	}1145 1146out:1147	dqm_unlock(dqm);1148	return ret;1149}1150 1151static int evict_process_queues_cpsch(struct device_queue_manager *dqm,1152				      struct qcm_process_device *qpd)1153{1154	struct queue *q;1155	struct device *dev = dqm->dev->adev->dev;1156	struct kfd_process_device *pdd;1157	int retval = 0;1158 1159	dqm_lock(dqm);1160	if (qpd->evicted++ > 0) /* already evicted, do nothing */1161		goto out;1162 1163	pdd = qpd_to_pdd(qpd);1164 1165	/* The debugger creates processes that temporarily have not acquired1166	 * all VMs for all devices and has no VMs itself.1167	 * Skip queue eviction on process eviction.1168	 */1169	if (!pdd->drm_priv)1170		goto out;1171 1172	pr_debug_ratelimited("Evicting PASID 0x%x queues\n",1173			    pdd->process->pasid);1174 1175	/* Mark all queues as evicted. Deactivate all active queues on1176	 * the qpd.1177	 */1178	list_for_each_entry(q, &qpd->queues_list, list) {1179		q->properties.is_evicted = true;1180		if (!q->properties.is_active)1181			continue;1182 1183		q->properties.is_active = false;1184		decrement_queue_count(dqm, qpd, q);1185 1186		if (dqm->dev->kfd->shared_resources.enable_mes) {1187			retval = remove_queue_mes(dqm, q, qpd);1188			if (retval) {1189				dev_err(dev, "Failed to evict queue %d\n",1190					q->properties.queue_id);1191				goto out;1192			}1193		}1194	}1195	pdd->last_evict_timestamp = get_jiffies_64();1196	if (!dqm->dev->kfd->shared_resources.enable_mes)1197		retval = execute_queues_cpsch(dqm,1198					      qpd->is_debug ?1199					      KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES :1200					      KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0,1201					      USE_DEFAULT_GRACE_PERIOD);1202 1203out:1204	dqm_unlock(dqm);1205	return retval;1206}1207 1208static int restore_process_queues_nocpsch(struct device_queue_manager *dqm,1209					  struct qcm_process_device *qpd)1210{1211	struct mm_struct *mm = NULL;1212	struct queue *q;1213	struct mqd_manager *mqd_mgr;1214	struct kfd_process_device *pdd;1215	uint64_t pd_base;1216	uint64_t eviction_duration;1217	int retval, ret = 0;1218 1219	pdd = qpd_to_pdd(qpd);1220	/* Retrieve PD base */1221	pd_base = amdgpu_amdkfd_gpuvm_get_process_page_dir(pdd->drm_priv);1222 1223	dqm_lock(dqm);1224	if (WARN_ON_ONCE(!qpd->evicted)) /* already restored, do nothing */1225		goto out;1226	if (qpd->evicted > 1) { /* ref count still > 0, decrement & quit */1227		qpd->evicted--;1228		goto out;1229	}1230 1231	pr_debug_ratelimited("Restoring PASID 0x%x queues\n",1232			    pdd->process->pasid);1233 1234	/* Update PD Base in QPD */1235	qpd->page_table_base = pd_base;1236	pr_debug("Updated PD address to 0x%llx\n", pd_base);1237 1238	if (!list_empty(&qpd->queues_list)) {1239		dqm->dev->kfd2kgd->set_vm_context_page_table_base(1240				dqm->dev->adev,1241				qpd->vmid,1242				qpd->page_table_base);1243		kfd_flush_tlb(pdd, TLB_FLUSH_LEGACY);1244	}1245 1246	/* Take a safe reference to the mm_struct, which may otherwise1247	 * disappear even while the kfd_process is still referenced.1248	 */1249	mm = get_task_mm(pdd->process->lead_thread);1250	if (!mm) {1251		ret = -EFAULT;1252		goto out;1253	}1254 1255	/* Remove the eviction flags. Activate queues that are not1256	 * inactive for other reasons.1257	 */1258	list_for_each_entry(q, &qpd->queues_list, list) {1259		q->properties.is_evicted = false;1260		if (!QUEUE_IS_ACTIVE(q->properties))1261			continue;1262 1263		mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type(1264				q->properties.type)];1265		q->properties.is_active = true;1266		increment_queue_count(dqm, qpd, q);1267 1268		if (WARN_ONCE(!dqm->sched_running, "Restore when stopped\n"))1269			continue;1270 1271		retval = mqd_mgr->load_mqd(mqd_mgr, q->mqd, q->pipe,1272				       q->queue, &q->properties, mm);1273		if (retval && !ret)1274			/* Return the first error, but keep going to1275			 * maintain a consistent eviction state1276			 */1277			ret = retval;1278	}1279	qpd->evicted = 0;1280	eviction_duration = get_jiffies_64() - pdd->last_evict_timestamp;1281	atomic64_add(eviction_duration, &pdd->evict_duration_counter);1282out:1283	if (mm)1284		mmput(mm);1285	dqm_unlock(dqm);1286	return ret;1287}1288 1289static int restore_process_queues_cpsch(struct device_queue_manager *dqm,1290					struct qcm_process_device *qpd)1291{1292	struct queue *q;1293	struct device *dev = dqm->dev->adev->dev;1294	struct kfd_process_device *pdd;1295	uint64_t eviction_duration;1296	int retval = 0;1297 1298	pdd = qpd_to_pdd(qpd);1299 1300	dqm_lock(dqm);1301	if (WARN_ON_ONCE(!qpd->evicted)) /* already restored, do nothing */1302		goto out;1303	if (qpd->evicted > 1) { /* ref count still > 0, decrement & quit */1304		qpd->evicted--;1305		goto out;1306	}1307 1308	/* The debugger creates processes that temporarily have not acquired1309	 * all VMs for all devices and has no VMs itself.1310	 * Skip queue restore on process restore.1311	 */1312	if (!pdd->drm_priv)1313		goto vm_not_acquired;1314 1315	pr_debug_ratelimited("Restoring PASID 0x%x queues\n",1316			    pdd->process->pasid);1317 1318	/* Update PD Base in QPD */1319	qpd->page_table_base = amdgpu_amdkfd_gpuvm_get_process_page_dir(pdd->drm_priv);1320	pr_debug("Updated PD address to 0x%llx\n", qpd->page_table_base);1321 1322	/* activate all active queues on the qpd */1323	list_for_each_entry(q, &qpd->queues_list, list) {1324		q->properties.is_evicted = false;1325		if (!QUEUE_IS_ACTIVE(q->properties))1326			continue;1327 1328		q->properties.is_active = true;1329		increment_queue_count(dqm, &pdd->qpd, q);1330 1331		if (dqm->dev->kfd->shared_resources.enable_mes) {1332			retval = add_queue_mes(dqm, q, qpd);1333			if (retval) {1334				dev_err(dev, "Failed to restore queue %d\n",1335					q->properties.queue_id);1336				goto out;1337			}1338		}1339	}1340	if (!dqm->dev->kfd->shared_resources.enable_mes)1341		retval = execute_queues_cpsch(dqm,1342					      KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0, USE_DEFAULT_GRACE_PERIOD);1343	eviction_duration = get_jiffies_64() - pdd->last_evict_timestamp;1344	atomic64_add(eviction_duration, &pdd->evict_duration_counter);1345vm_not_acquired:1346	qpd->evicted = 0;1347out:1348	dqm_unlock(dqm);1349	return retval;1350}1351 1352static int register_process(struct device_queue_manager *dqm,1353					struct qcm_process_device *qpd)1354{1355	struct device_process_node *n;1356	struct kfd_process_device *pdd;1357	uint64_t pd_base;1358	int retval;1359 1360	n = kzalloc(sizeof(*n), GFP_KERNEL);1361	if (!n)1362		return -ENOMEM;1363 1364	n->qpd = qpd;1365 1366	pdd = qpd_to_pdd(qpd);1367	/* Retrieve PD base */1368	pd_base = amdgpu_amdkfd_gpuvm_get_process_page_dir(pdd->drm_priv);1369 1370	dqm_lock(dqm);1371	list_add(&n->list, &dqm->queues);1372 1373	/* Update PD Base in QPD */1374	qpd->page_table_base = pd_base;1375	pr_debug("Updated PD address to 0x%llx\n", pd_base);1376 1377	retval = dqm->asic_ops.update_qpd(dqm, qpd);1378 1379	dqm->processes_count++;1380 1381	dqm_unlock(dqm);1382 1383	/* Outside the DQM lock because under the DQM lock we can't do1384	 * reclaim or take other locks that others hold while reclaiming.1385	 */1386	kfd_inc_compute_active(dqm->dev);1387 1388	return retval;1389}1390 1391static int unregister_process(struct device_queue_manager *dqm,1392					struct qcm_process_device *qpd)1393{1394	int retval;1395	struct device_process_node *cur, *next;1396 1397	pr_debug("qpd->queues_list is %s\n",1398			list_empty(&qpd->queues_list) ? "empty" : "not empty");1399 1400	retval = 0;1401	dqm_lock(dqm);1402 1403	list_for_each_entry_safe(cur, next, &dqm->queues, list) {1404		if (qpd == cur->qpd) {1405			list_del(&cur->list);1406			kfree(cur);1407			dqm->processes_count--;1408			goto out;1409		}1410	}1411	/* qpd not found in dqm list */1412	retval = 1;1413out:1414	dqm_unlock(dqm);1415 1416	/* Outside the DQM lock because under the DQM lock we can't do1417	 * reclaim or take other locks that others hold while reclaiming.1418	 */1419	if (!retval)1420		kfd_dec_compute_active(dqm->dev);1421 1422	return retval;1423}1424 1425static int1426set_pasid_vmid_mapping(struct device_queue_manager *dqm, u32 pasid,1427			unsigned int vmid)1428{1429	uint32_t xcc_mask = dqm->dev->xcc_mask;1430	int xcc_id, ret;1431 1432	for_each_inst(xcc_id, xcc_mask) {1433		ret = dqm->dev->kfd2kgd->set_pasid_vmid_mapping(1434			dqm->dev->adev, pasid, vmid, xcc_id);1435		if (ret)1436			break;1437	}1438 1439	return ret;1440}1441 1442static void init_interrupts(struct device_queue_manager *dqm)1443{1444	uint32_t xcc_mask = dqm->dev->xcc_mask;1445	unsigned int i, xcc_id;1446 1447	for_each_inst(xcc_id, xcc_mask) {1448		for (i = 0 ; i < get_pipes_per_mec(dqm) ; i++) {1449			if (is_pipe_enabled(dqm, 0, i)) {1450				dqm->dev->kfd2kgd->init_interrupts(1451					dqm->dev->adev, i, xcc_id);1452			}1453		}1454	}1455}1456 1457static int initialize_nocpsch(struct device_queue_manager *dqm)1458{1459	int pipe, queue;1460 1461	pr_debug("num of pipes: %d\n", get_pipes_per_mec(dqm));1462 1463	dqm->allocated_queues = kcalloc(get_pipes_per_mec(dqm),1464					sizeof(unsigned int), GFP_KERNEL);1465	if (!dqm->allocated_queues)1466		return -ENOMEM;1467 1468	mutex_init(&dqm->lock_hidden);1469	INIT_LIST_HEAD(&dqm->queues);1470	dqm->active_queue_count = dqm->next_pipe_to_allocate = 0;1471	dqm->active_cp_queue_count = 0;1472	dqm->gws_queue_count = 0;1473 1474	for (pipe = 0; pipe < get_pipes_per_mec(dqm); pipe++) {1475		int pipe_offset = pipe * get_queues_per_pipe(dqm);1476 1477		for (queue = 0; queue < get_queues_per_pipe(dqm); queue++)1478			if (test_bit(pipe_offset + queue,1479				     dqm->dev->kfd->shared_resources.cp_queue_bitmap))1480				dqm->allocated_queues[pipe] |= 1 << queue;1481	}1482 1483	memset(dqm->vmid_pasid, 0, sizeof(dqm->vmid_pasid));1484 1485	init_sdma_bitmaps(dqm);1486 1487	return 0;1488}1489 1490static void uninitialize(struct device_queue_manager *dqm)1491{1492	int i;1493 1494	WARN_ON(dqm->active_queue_count > 0 || dqm->processes_count > 0);1495 1496	kfree(dqm->allocated_queues);1497	for (i = 0 ; i < KFD_MQD_TYPE_MAX ; i++)1498		kfree(dqm->mqd_mgrs[i]);1499	mutex_destroy(&dqm->lock_hidden);1500}1501 1502static int start_nocpsch(struct device_queue_manager *dqm)1503{1504	int r = 0;1505 1506	pr_info("SW scheduler is used");1507	init_interrupts(dqm);1508 1509	if (dqm->dev->adev->asic_type == CHIP_HAWAII)1510		r = pm_init(&dqm->packet_mgr, dqm);1511	if (!r)1512		dqm->sched_running = true;1513 1514	return r;1515}1516 1517static int stop_nocpsch(struct device_queue_manager *dqm)1518{1519	dqm_lock(dqm);1520	if (!dqm->sched_running) {1521		dqm_unlock(dqm);1522		return 0;1523	}1524 1525	if (dqm->dev->adev->asic_type == CHIP_HAWAII)1526		pm_uninit(&dqm->packet_mgr);1527	dqm->sched_running = false;1528	dqm_unlock(dqm);1529 1530	return 0;1531}1532 1533static int allocate_sdma_queue(struct device_queue_manager *dqm,1534				struct queue *q, const uint32_t *restore_sdma_id)1535{1536	struct device *dev = dqm->dev->adev->dev;1537	int bit;1538 1539	if (q->properties.type == KFD_QUEUE_TYPE_SDMA) {1540		if (bitmap_empty(dqm->sdma_bitmap, KFD_MAX_SDMA_QUEUES)) {1541			dev_err(dev, "No more SDMA queue to allocate\n");1542			return -ENOMEM;1543		}1544 1545		if (restore_sdma_id) {1546			/* Re-use existing sdma_id */1547			if (!test_bit(*restore_sdma_id, dqm->sdma_bitmap)) {1548				dev_err(dev, "SDMA queue already in use\n");1549				return -EBUSY;1550			}1551			clear_bit(*restore_sdma_id, dqm->sdma_bitmap);1552			q->sdma_id = *restore_sdma_id;1553		} else {1554			/* Find first available sdma_id */1555			bit = find_first_bit(dqm->sdma_bitmap,1556					     get_num_sdma_queues(dqm));1557			clear_bit(bit, dqm->sdma_bitmap);1558			q->sdma_id = bit;1559		}1560 1561		q->properties.sdma_engine_id =1562			q->sdma_id % kfd_get_num_sdma_engines(dqm->dev);1563		q->properties.sdma_queue_id = q->sdma_id /1564				kfd_get_num_sdma_engines(dqm->dev);1565	} else if (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) {1566		if (bitmap_empty(dqm->xgmi_sdma_bitmap, KFD_MAX_SDMA_QUEUES)) {1567			dev_err(dev, "No more XGMI SDMA queue to allocate\n");1568			return -ENOMEM;1569		}1570		if (restore_sdma_id) {1571			/* Re-use existing sdma_id */1572			if (!test_bit(*restore_sdma_id, dqm->xgmi_sdma_bitmap)) {1573				dev_err(dev, "SDMA queue already in use\n");1574				return -EBUSY;1575			}1576			clear_bit(*restore_sdma_id, dqm->xgmi_sdma_bitmap);1577			q->sdma_id = *restore_sdma_id;1578		} else {1579			bit = find_first_bit(dqm->xgmi_sdma_bitmap,1580					     get_num_xgmi_sdma_queues(dqm));1581			clear_bit(bit, dqm->xgmi_sdma_bitmap);1582			q->sdma_id = bit;1583		}1584		/* sdma_engine_id is sdma id including1585		 * both PCIe-optimized SDMAs and XGMI-1586		 * optimized SDMAs. The calculation below1587		 * assumes the first N engines are always1588		 * PCIe-optimized ones1589		 */1590		q->properties.sdma_engine_id =1591			kfd_get_num_sdma_engines(dqm->dev) +1592			q->sdma_id % kfd_get_num_xgmi_sdma_engines(dqm->dev);1593		q->properties.sdma_queue_id = q->sdma_id /1594			kfd_get_num_xgmi_sdma_engines(dqm->dev);1595	} else if (q->properties.type == KFD_QUEUE_TYPE_SDMA_BY_ENG_ID) {1596		int i, num_queues, num_engines, eng_offset = 0, start_engine;1597		bool free_bit_found = false, is_xgmi = false;1598 1599		if (q->properties.sdma_engine_id < kfd_get_num_sdma_engines(dqm->dev)) {1600			num_queues = get_num_sdma_queues(dqm);1601			num_engines = kfd_get_num_sdma_engines(dqm->dev);1602			q->properties.type = KFD_QUEUE_TYPE_SDMA;1603		} else {1604			num_queues = get_num_xgmi_sdma_queues(dqm);1605			num_engines = kfd_get_num_xgmi_sdma_engines(dqm->dev);1606			eng_offset = kfd_get_num_sdma_engines(dqm->dev);1607			q->properties.type = KFD_QUEUE_TYPE_SDMA_XGMI;1608			is_xgmi = true;1609		}1610 1611		/* Scan available bit based on target engine ID. */1612		start_engine = q->properties.sdma_engine_id - eng_offset;1613		for (i = start_engine; i < num_queues; i += num_engines) {1614 1615			if (!test_bit(i, is_xgmi ? dqm->xgmi_sdma_bitmap : dqm->sdma_bitmap))1616				continue;1617 1618			clear_bit(i, is_xgmi ? dqm->xgmi_sdma_bitmap : dqm->sdma_bitmap);1619			q->sdma_id = i;1620			q->properties.sdma_queue_id = q->sdma_id / num_engines;1621			free_bit_found = true;1622			break;1623		}1624 1625		if (!free_bit_found) {1626			dev_err(dev, "No more SDMA queue to allocate for target ID %i\n",1627				q->properties.sdma_engine_id);1628			return -ENOMEM;1629		}1630	}1631 1632	pr_debug("SDMA engine id: %d\n", q->properties.sdma_engine_id);1633	pr_debug("SDMA queue id: %d\n", q->properties.sdma_queue_id);1634 1635	return 0;1636}1637 1638static void deallocate_sdma_queue(struct device_queue_manager *dqm,1639				struct queue *q)1640{1641	if (q->properties.type == KFD_QUEUE_TYPE_SDMA) {1642		if (q->sdma_id >= get_num_sdma_queues(dqm))1643			return;1644		set_bit(q->sdma_id, dqm->sdma_bitmap);1645	} else if (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) {1646		if (q->sdma_id >= get_num_xgmi_sdma_queues(dqm))1647			return;1648		set_bit(q->sdma_id, dqm->xgmi_sdma_bitmap);1649	}1650}1651 1652/*1653 * Device Queue Manager implementation for cp scheduler1654 */1655 1656static int set_sched_resources(struct device_queue_manager *dqm)1657{1658	int i, mec;1659	struct scheduling_resources res;1660	struct device *dev = dqm->dev->adev->dev;1661 1662	res.vmid_mask = dqm->dev->compute_vmid_bitmap;1663 1664	res.queue_mask = 0;1665	for (i = 0; i < AMDGPU_MAX_QUEUES; ++i) {1666		mec = (i / dqm->dev->kfd->shared_resources.num_queue_per_pipe)1667			/ dqm->dev->kfd->shared_resources.num_pipe_per_mec;1668 1669		if (!test_bit(i, dqm->dev->kfd->shared_resources.cp_queue_bitmap))1670			continue;1671 1672		/* only acquire queues from the first MEC */1673		if (mec > 0)1674			continue;1675 1676		/* This situation may be hit in the future if a new HW1677		 * generation exposes more than 64 queues. If so, the1678		 * definition of res.queue_mask needs updating1679		 */1680		if (WARN_ON(i >= (sizeof(res.queue_mask)*8))) {1681			dev_err(dev, "Invalid queue enabled by amdgpu: %d\n", i);1682			break;1683		}1684 1685		res.queue_mask |= 1ull1686			<< amdgpu_queue_mask_bit_to_set_resource_bit(1687				dqm->dev->adev, i);1688	}1689	res.gws_mask = ~0ull;1690	res.oac_mask = res.gds_heap_base = res.gds_heap_size = 0;1691 1692	pr_debug("Scheduling resources:\n"1693			"vmid mask: 0x%8X\n"1694			"queue mask: 0x%8llX\n",1695			res.vmid_mask, res.queue_mask);1696 1697	return pm_send_set_resources(&dqm->packet_mgr, &res);1698}1699 1700static int initialize_cpsch(struct device_queue_manager *dqm)1701{1702	pr_debug("num of pipes: %d\n", get_pipes_per_mec(dqm));1703 1704	mutex_init(&dqm->lock_hidden);1705	INIT_LIST_HEAD(&dqm->queues);1706	dqm->active_queue_count = dqm->processes_count = 0;1707	dqm->active_cp_queue_count = 0;1708	dqm->gws_queue_count = 0;1709	dqm->active_runlist = false;1710	INIT_WORK(&dqm->hw_exception_work, kfd_process_hw_exception);1711	dqm->trap_debug_vmid = 0;1712 1713	init_sdma_bitmaps(dqm);1714 1715	if (dqm->dev->kfd2kgd->get_iq_wait_times)1716		dqm->dev->kfd2kgd->get_iq_wait_times(dqm->dev->adev,1717					&dqm->wait_times,1718					ffs(dqm->dev->xcc_mask) - 1);1719	return 0;1720}1721 1722/* halt_cpsch:1723 * Unmap queues so the schedule doesn't continue remaining jobs in the queue.1724 * Then set dqm->sched_halt so queues don't map to runlist until unhalt_cpsch1725 * is called.1726 */1727static int halt_cpsch(struct device_queue_manager *dqm)1728{1729	int ret = 0;1730 1731	dqm_lock(dqm);1732	if (!dqm->sched_running) {1733		dqm_unlock(dqm);1734		return 0;1735	}1736 1737	WARN_ONCE(dqm->sched_halt, "Scheduling is already on halt\n");1738 1739	if (!dqm->is_hws_hang) {1740		if (!dqm->dev->kfd->shared_resources.enable_mes)1741			ret = unmap_queues_cpsch(dqm,1742						 KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES, 0,1743				USE_DEFAULT_GRACE_PERIOD, false);1744		else1745			ret = remove_all_queues_mes(dqm);1746	}1747	dqm->sched_halt = true;1748	dqm_unlock(dqm);1749 1750	return ret;1751}1752 1753/* unhalt_cpsch1754 * Unset dqm->sched_halt and map queues back to runlist1755 */1756static int unhalt_cpsch(struct device_queue_manager *dqm)1757{1758	int ret = 0;1759 1760	dqm_lock(dqm);1761	if (!dqm->sched_running || !dqm->sched_halt) {1762		WARN_ONCE(!dqm->sched_halt, "Scheduling is not on halt.\n");1763		dqm_unlock(dqm);1764		return 0;1765	}1766	dqm->sched_halt = false;1767	if (!dqm->dev->kfd->shared_resources.enable_mes)1768		ret = execute_queues_cpsch(dqm,1769					   KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES,1770			0, USE_DEFAULT_GRACE_PERIOD);1771	dqm_unlock(dqm);1772 1773	return ret;1774}1775 1776static int start_cpsch(struct device_queue_manager *dqm)1777{1778	struct device *dev = dqm->dev->adev->dev;1779	int retval, num_hw_queue_slots;1780 1781	retval = 0;1782 1783	dqm_lock(dqm);1784 1785	if (!dqm->dev->kfd->shared_resources.enable_mes) {1786		retval = pm_init(&dqm->packet_mgr, dqm);1787		if (retval)1788			goto fail_packet_manager_init;1789 1790		retval = set_sched_resources(dqm);1791		if (retval)1792			goto fail_set_sched_resources;1793	}1794	pr_debug("Allocating fence memory\n");1795 1796	/* allocate fence memory on the gart */1797	retval = kfd_gtt_sa_allocate(dqm->dev, sizeof(*dqm->fence_addr),1798					&dqm->fence_mem);1799 1800	if (retval)1801		goto fail_allocate_vidmem;1802 1803	dqm->fence_addr = (uint64_t *)dqm->fence_mem->cpu_ptr;1804	dqm->fence_gpu_addr = dqm->fence_mem->gpu_addr;1805 1806	init_interrupts(dqm);1807 1808	/* clear hang status when driver try to start the hw scheduler */1809	dqm->sched_running = true;1810 1811	if (!dqm->dev->kfd->shared_resources.enable_mes)1812		execute_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0, USE_DEFAULT_GRACE_PERIOD);1813 1814	/* Set CWSR grace period to 1x1000 cycle for GFX9.4.3 APU */1815	if (amdgpu_emu_mode == 0 && dqm->dev->adev->gmc.is_app_apu &&1816	    (KFD_GC_VERSION(dqm->dev) == IP_VERSION(9, 4, 3))) {1817		uint32_t reg_offset = 0;1818		uint32_t grace_period = 1;1819 1820		retval = pm_update_grace_period(&dqm->packet_mgr,1821						grace_period);1822		if (retval)1823			dev_err(dev, "Setting grace timeout failed\n");1824		else if (dqm->dev->kfd2kgd->build_grace_period_packet_info)1825			/* Update dqm->wait_times maintained in software */1826			dqm->dev->kfd2kgd->build_grace_period_packet_info(1827					dqm->dev->adev,	dqm->wait_times,1828					grace_period, &reg_offset,1829					&dqm->wait_times);1830	}1831 1832	/* setup per-queue reset detection buffer  */1833	num_hw_queue_slots =  dqm->dev->kfd->shared_resources.num_queue_per_pipe *1834			      dqm->dev->kfd->shared_resources.num_pipe_per_mec *1835			      NUM_XCC(dqm->dev->xcc_mask);1836 1837	dqm->detect_hang_info_size = num_hw_queue_slots * sizeof(struct dqm_detect_hang_info);1838	dqm->detect_hang_info = kzalloc(dqm->detect_hang_info_size, GFP_KERNEL);1839 1840	if (!dqm->detect_hang_info) {1841		retval = -ENOMEM;1842		goto fail_detect_hang_buffer;1843	}1844 1845	dqm_unlock(dqm);1846 1847	return 0;1848fail_detect_hang_buffer:1849	kfd_gtt_sa_free(dqm->dev, dqm->fence_mem);1850fail_allocate_vidmem:1851fail_set_sched_resources:1852	if (!dqm->dev->kfd->shared_resources.enable_mes)1853		pm_uninit(&dqm->packet_mgr);1854fail_packet_manager_init:1855	dqm_unlock(dqm);1856	return retval;1857}1858 1859static int stop_cpsch(struct device_queue_manager *dqm)1860{1861	dqm_lock(dqm);1862	if (!dqm->sched_running) {1863		dqm_unlock(dqm);1864		return 0;1865	}1866 1867	if (!dqm->dev->kfd->shared_resources.enable_mes)1868		unmap_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES, 0, USE_DEFAULT_GRACE_PERIOD, false);1869	else1870		remove_all_queues_mes(dqm);1871 1872	dqm->sched_running = false;1873 1874	if (!dqm->dev->kfd->shared_resources.enable_mes)1875		pm_release_ib(&dqm->packet_mgr);1876 1877	kfd_gtt_sa_free(dqm->dev, dqm->fence_mem);1878	if (!dqm->dev->kfd->shared_resources.enable_mes)1879		pm_uninit(&dqm->packet_mgr);1880	kfree(dqm->detect_hang_info);1881	dqm->detect_hang_info = NULL;1882	dqm_unlock(dqm);1883 1884	return 0;1885}1886 1887static int create_kernel_queue_cpsch(struct device_queue_manager *dqm,1888					struct kernel_queue *kq,1889					struct qcm_process_device *qpd)1890{1891	dqm_lock(dqm);1892	if (dqm->total_queue_count >= max_num_of_queues_per_device) {1893		pr_warn("Can't create new kernel queue because %d queues were already created\n",1894				dqm->total_queue_count);1895		dqm_unlock(dqm);1896		return -EPERM;1897	}1898 1899	/*1900	 * Unconditionally increment this counter, regardless of the queue's1901	 * type or whether the queue is active.1902	 */1903	dqm->total_queue_count++;1904	pr_debug("Total of %d queues are accountable so far\n",1905			dqm->total_queue_count);1906 1907	list_add(&kq->list, &qpd->priv_queue_list);1908	increment_queue_count(dqm, qpd, kq->queue);1909	qpd->is_debug = true;1910	execute_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0,1911			USE_DEFAULT_GRACE_PERIOD);1912	dqm_unlock(dqm);1913 1914	return 0;1915}1916 1917static void destroy_kernel_queue_cpsch(struct device_queue_manager *dqm,1918					struct kernel_queue *kq,1919					struct qcm_process_device *qpd)1920{1921	dqm_lock(dqm);1922	list_del(&kq->list);1923	decrement_queue_count(dqm, qpd, kq->queue);1924	qpd->is_debug = false;1925	execute_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES, 0,1926			USE_DEFAULT_GRACE_PERIOD);1927	/*1928	 * Unconditionally decrement this counter, regardless of the queue's1929	 * type.1930	 */1931	dqm->total_queue_count--;1932	pr_debug("Total of %d queues are accountable so far\n",1933			dqm->total_queue_count);1934	dqm_unlock(dqm);1935}1936 1937static int create_queue_cpsch(struct device_queue_manager *dqm, struct queue *q,1938			struct qcm_process_device *qpd,1939			const struct kfd_criu_queue_priv_data *qd,1940			const void *restore_mqd, const void *restore_ctl_stack)1941{1942	int retval;1943	struct mqd_manager *mqd_mgr;1944 1945	if (dqm->total_queue_count >= max_num_of_queues_per_device) {1946		pr_warn("Can't create new usermode queue because %d queues were already created\n",1947				dqm->total_queue_count);1948		retval = -EPERM;1949		goto out;1950	}1951 1952	if (q->properties.type == KFD_QUEUE_TYPE_SDMA ||1953		q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI ||1954		q->properties.type == KFD_QUEUE_TYPE_SDMA_BY_ENG_ID) {1955		dqm_lock(dqm);1956		retval = allocate_sdma_queue(dqm, q, qd ? &qd->sdma_id : NULL);1957		dqm_unlock(dqm);1958		if (retval)1959			goto out;1960	}1961 1962	retval = allocate_doorbell(qpd, q, qd ? &qd->doorbell_id : NULL);1963	if (retval)1964		goto out_deallocate_sdma_queue;1965 1966	mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type(1967			q->properties.type)];1968 1969	if (q->properties.type == KFD_QUEUE_TYPE_SDMA ||1970		q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)1971		dqm->asic_ops.init_sdma_vm(dqm, q, qpd);1972	q->properties.tba_addr = qpd->tba_addr;1973	q->properties.tma_addr = qpd->tma_addr;1974	q->mqd_mem_obj = mqd_mgr->allocate_mqd(mqd_mgr->dev, &q->properties);1975	if (!q->mqd_mem_obj) {1976		retval = -ENOMEM;1977		goto out_deallocate_doorbell;1978	}1979 1980	dqm_lock(dqm);1981	/*1982	 * Eviction state logic: mark all queues as evicted, even ones1983	 * not currently active. Restoring inactive queues later only1984	 * updates the is_evicted flag but is a no-op otherwise.1985	 */1986	q->properties.is_evicted = !!qpd->evicted;1987	q->properties.is_dbg_wa = qpd->pqm->process->debug_trap_enabled &&1988				  kfd_dbg_has_cwsr_workaround(q->device);1989 1990	if (qd)1991		mqd_mgr->restore_mqd(mqd_mgr, &q->mqd, q->mqd_mem_obj, &q->gart_mqd_addr,1992				     &q->properties, restore_mqd, restore_ctl_stack,1993				     qd->ctl_stack_size);1994	else1995		mqd_mgr->init_mqd(mqd_mgr, &q->mqd, q->mqd_mem_obj,1996					&q->gart_mqd_addr, &q->properties);1997 1998	list_add(&q->list, &qpd->queues_list);1999	qpd->queue_count++;2000 2001	if (q->properties.is_active) {2002		increment_queue_count(dqm, qpd, q);2003 2004		if (!dqm->dev->kfd->shared_resources.enable_mes)2005			retval = execute_queues_cpsch(dqm,2006					KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0, USE_DEFAULT_GRACE_PERIOD);2007		else2008			retval = add_queue_mes(dqm, q, qpd);2009		if (retval)2010			goto cleanup_queue;2011	}2012 2013	/*2014	 * Unconditionally increment this counter, regardless of the queue's2015	 * type or whether the queue is active.2016	 */2017	dqm->total_queue_count++;2018 2019	pr_debug("Total of %d queues are accountable so far\n",2020			dqm->total_queue_count);2021 2022	dqm_unlock(dqm);2023	return retval;2024 2025cleanup_queue:2026	qpd->queue_count--;2027	list_del(&q->list);2028	if (q->properties.is_active)2029		decrement_queue_count(dqm, qpd, q);2030	mqd_mgr->free_mqd(mqd_mgr, q->mqd, q->mqd_mem_obj);2031	dqm_unlock(dqm);2032out_deallocate_doorbell:2033	deallocate_doorbell(qpd, q);2034out_deallocate_sdma_queue:2035	if (q->properties.type == KFD_QUEUE_TYPE_SDMA ||2036		q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) {2037		dqm_lock(dqm);2038		deallocate_sdma_queue(dqm, q);2039		dqm_unlock(dqm);2040	}2041out:2042	return retval;2043}2044 2045int amdkfd_fence_wait_timeout(struct device_queue_manager *dqm,2046			      uint64_t fence_value,2047			      unsigned int timeout_ms)2048{2049	unsigned long end_jiffies = msecs_to_jiffies(timeout_ms) + jiffies;2050	struct device *dev = dqm->dev->adev->dev;2051	uint64_t *fence_addr =  dqm->fence_addr;2052 2053	while (*fence_addr != fence_value) {2054		/* Fatal err detected, this response won't come */2055		if (amdgpu_amdkfd_is_fed(dqm->dev->adev))2056			return -EIO;2057 2058		if (time_after(jiffies, end_jiffies)) {2059			dev_err(dev, "qcm fence wait loop timeout expired\n");2060			/* In HWS case, this is used to halt the driver thread2061			 * in order not to mess up CP states before doing2062			 * scandumps for FW debugging.2063			 */2064			while (halt_if_hws_hang)2065				schedule();2066 2067			return -ETIME;2068		}2069		schedule();2070	}2071 2072	return 0;2073}2074 2075/* dqm->lock mutex has to be locked before calling this function */2076static int map_queues_cpsch(struct device_queue_manager *dqm)2077{2078	struct device *dev = dqm->dev->adev->dev;2079	int retval;2080 2081	if (!dqm->sched_running || dqm->sched_halt)2082		return 0;2083	if (dqm->active_queue_count <= 0 || dqm->processes_count <= 0)2084		return 0;2085	if (dqm->active_runlist)2086		return 0;2087 2088	retval = pm_send_runlist(&dqm->packet_mgr, &dqm->queues);2089	pr_debug("%s sent runlist\n", __func__);2090	if (retval) {2091		dev_err(dev, "failed to execute runlist\n");2092		return retval;2093	}2094	dqm->active_runlist = true;2095 2096	return retval;2097}2098 2099static void set_queue_as_reset(struct device_queue_manager *dqm, struct queue *q,2100			       struct qcm_process_device *qpd)2101{2102	struct kfd_process_device *pdd = qpd_to_pdd(qpd);2103 2104	dev_err(dqm->dev->adev->dev, "queue id 0x%0x at pasid 0x%0x is reset\n",2105		q->properties.queue_id, q->process->pasid);2106 2107	pdd->has_reset_queue = true;2108	if (q->properties.is_active) {2109		q->properties.is_active = false;2110		decrement_queue_count(dqm, qpd, q);2111	}2112}2113 2114static int detect_queue_hang(struct device_queue_manager *dqm)2115{2116	int i;2117 2118	/* detect should be used only in dqm locked queue reset */2119	if (WARN_ON(dqm->detect_hang_count > 0))2120		return 0;2121 2122	memset(dqm->detect_hang_info, 0, dqm->detect_hang_info_size);2123 2124	for (i = 0; i < AMDGPU_MAX_QUEUES; ++i) {2125		uint32_t mec, pipe, queue;2126		int xcc_id;2127 2128		mec = (i / dqm->dev->kfd->shared_resources.num_queue_per_pipe)2129			/ dqm->dev->kfd->shared_resources.num_pipe_per_mec;2130 2131		if (mec || !test_bit(i, dqm->dev->kfd->shared_resources.cp_queue_bitmap))2132			continue;2133 2134		amdgpu_queue_mask_bit_to_mec_queue(dqm->dev->adev, i, &mec, &pipe, &queue);2135 2136		for_each_inst(xcc_id, dqm->dev->xcc_mask) {2137			uint64_t queue_addr = dqm->dev->kfd2kgd->hqd_get_pq_addr(2138						dqm->dev->adev, pipe, queue, xcc_id);2139			struct dqm_detect_hang_info hang_info;2140 2141			if (!queue_addr)2142				continue;2143 2144			hang_info.pipe_id = pipe;2145			hang_info.queue_id = queue;2146			hang_info.xcc_id = xcc_id;2147			hang_info.queue_address = queue_addr;2148 2149			dqm->detect_hang_info[dqm->detect_hang_count] = hang_info;2150			dqm->detect_hang_count++;2151		}2152	}2153 2154	return dqm->detect_hang_count;2155}2156 2157static struct queue *find_queue_by_address(struct device_queue_manager *dqm, uint64_t queue_address)2158{2159	struct device_process_node *cur;2160	struct qcm_process_device *qpd;2161	struct queue *q;2162 2163	list_for_each_entry(cur, &dqm->queues, list) {2164		qpd = cur->qpd;2165		list_for_each_entry(q, &qpd->queues_list, list) {2166			if (queue_address == q->properties.queue_address)2167				return q;2168		}2169	}2170 2171	return NULL;2172}2173 2174/* only for compute queue */2175static int reset_queues_on_hws_hang(struct device_queue_manager *dqm)2176{2177	int r = 0, reset_count = 0, i;2178 2179	if (!dqm->detect_hang_info || dqm->is_hws_hang)2180		return -EIO;2181 2182	/* assume dqm locked. */2183	if (!detect_queue_hang(dqm))2184		return -ENOTRECOVERABLE;2185 2186	for (i = 0; i < dqm->detect_hang_count; i++) {2187		struct dqm_detect_hang_info hang_info = dqm->detect_hang_info[i];2188		struct queue *q = find_queue_by_address(dqm, hang_info.queue_address);2189		struct kfd_process_device *pdd;2190		uint64_t queue_addr = 0;2191 2192		if (!q) {2193			r = -ENOTRECOVERABLE;2194			goto reset_fail;2195		}2196 2197		pdd = kfd_get_process_device_data(dqm->dev, q->process);2198		if (!pdd) {2199			r = -ENOTRECOVERABLE;2200			goto reset_fail;2201		}2202 2203		queue_addr = dqm->dev->kfd2kgd->hqd_reset(dqm->dev->adev,2204				hang_info.pipe_id, hang_info.queue_id, hang_info.xcc_id,2205				KFD_UNMAP_LATENCY_MS);2206 2207		/* either reset failed or we reset an unexpected queue. */2208		if (queue_addr != q->properties.queue_address) {2209			r = -ENOTRECOVERABLE;2210			goto reset_fail;2211		}2212 2213		set_queue_as_reset(dqm, q, &pdd->qpd);2214		reset_count++;2215	}2216 2217	if (reset_count == dqm->detect_hang_count)2218		kfd_signal_reset_event(dqm->dev);2219	else2220		r = -ENOTRECOVERABLE;2221 2222reset_fail:2223	dqm->detect_hang_count = 0;2224 2225	return r;2226}2227 2228/* dqm->lock mutex has to be locked before calling this function */2229static int unmap_queues_cpsch(struct device_queue_manager *dqm,2230				enum kfd_unmap_queues_filter filter,2231				uint32_t filter_param,2232				uint32_t grace_period,2233				bool reset)2234{2235	struct device *dev = dqm->dev->adev->dev;2236	struct mqd_manager *mqd_mgr;2237	int retval;2238 2239	if (!dqm->sched_running)2240		return 0;2241	if (!dqm->active_runlist)2242		return 0;2243	if (!down_read_trylock(&dqm->dev->adev->reset_domain->sem))2244		return -EIO;2245 2246	if (grace_period != USE_DEFAULT_GRACE_PERIOD) {2247		retval = pm_update_grace_period(&dqm->packet_mgr, grace_period);2248		if (retval)2249			goto out;2250	}2251 2252	retval = pm_send_unmap_queue(&dqm->packet_mgr, filter, filter_param, reset);2253	if (retval)2254		goto out;2255 2256	*dqm->fence_addr = KFD_FENCE_INIT;2257	pm_send_query_status(&dqm->packet_mgr, dqm->fence_gpu_addr,2258				KFD_FENCE_COMPLETED);2259	/* should be timed out */2260	retval = amdkfd_fence_wait_timeout(dqm, KFD_FENCE_COMPLETED,2261					   queue_preemption_timeout_ms);2262	if (retval) {2263		dev_err(dev, "The cp might be in an unrecoverable state due to an unsuccessful queues preemption\n");2264		kfd_hws_hang(dqm);2265		goto out;2266	}2267 2268	/* In the current MEC firmware implementation, if compute queue2269	 * doesn't response to the preemption request in time, HIQ will2270	 * abandon the unmap request without returning any timeout error2271	 * to driver. Instead, MEC firmware will log the doorbell of the2272	 * unresponding compute queue to HIQ.MQD.queue_doorbell_id fields.2273	 * To make sure the queue unmap was successful, driver need to2274	 * check those fields2275	 */2276	mqd_mgr = dqm->mqd_mgrs[KFD_MQD_TYPE_HIQ];2277	if (mqd_mgr->check_preemption_failed(mqd_mgr, dqm->packet_mgr.priv_queue->queue->mqd)) {2278		if (reset_queues_on_hws_hang(dqm)) {2279			while (halt_if_hws_hang)2280				schedule();2281			dqm->is_hws_hang = true;2282			kfd_hws_hang(dqm);2283			retval = -ETIME;2284			goto out;2285		}2286	}2287 2288	/* We need to reset the grace period value for this device */2289	if (grace_period != USE_DEFAULT_GRACE_PERIOD) {2290		if (pm_update_grace_period(&dqm->packet_mgr,2291					USE_DEFAULT_GRACE_PERIOD))2292			dev_err(dev, "Failed to reset grace period\n");2293	}2294 2295	pm_release_ib(&dqm->packet_mgr);2296	dqm->active_runlist = false;2297 2298out:2299	up_read(&dqm->dev->adev->reset_domain->sem);2300	return retval;2301}2302 2303/* only for compute queue */2304static int reset_queues_cpsch(struct device_queue_manager *dqm, uint16_t pasid)2305{2306	int retval;2307 2308	dqm_lock(dqm);2309 2310	retval = unmap_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_BY_PASID,2311			pasid, USE_DEFAULT_GRACE_PERIOD, true);2312 2313	dqm_unlock(dqm);2314	return retval;2315}2316 2317/* dqm->lock mutex has to be locked before calling this function */2318static int execute_queues_cpsch(struct device_queue_manager *dqm,2319				enum kfd_unmap_queues_filter filter,2320				uint32_t filter_param,2321				uint32_t grace_period)2322{2323	int retval;2324 2325	if (!down_read_trylock(&dqm->dev->adev->reset_domain->sem))2326		return -EIO;2327	retval = unmap_queues_cpsch(dqm, filter, filter_param, grace_period, false);2328	if (!retval)2329		retval = map_queues_cpsch(dqm);2330	up_read(&dqm->dev->adev->reset_domain->sem);2331	return retval;2332}2333 2334static int wait_on_destroy_queue(struct device_queue_manager *dqm,2335				 struct queue *q)2336{2337	struct kfd_process_device *pdd = kfd_get_process_device_data(q->device,2338								q->process);2339	int ret = 0;2340 2341	if (pdd->qpd.is_debug)2342		return ret;2343 2344	q->properties.is_being_destroyed = true;2345 2346	if (pdd->process->debug_trap_enabled && q->properties.is_suspended) {2347		dqm_unlock(dqm);2348		mutex_unlock(&q->process->mutex);2349		ret = wait_event_interruptible(dqm->destroy_wait,2350						!q->properties.is_suspended);2351 2352		mutex_lock(&q->process->mutex);2353		dqm_lock(dqm);2354	}2355 2356	return ret;2357}2358 2359static int destroy_queue_cpsch(struct device_queue_manager *dqm,2360				struct qcm_process_device *qpd,2361				struct queue *q)2362{2363	int retval;2364	struct mqd_manager *mqd_mgr;2365	uint64_t sdma_val = 0;2366	struct kfd_process_device *pdd = qpd_to_pdd(qpd);2367	struct device *dev = dqm->dev->adev->dev;2368 2369	/* Get the SDMA queue stats */2370	if ((q->properties.type == KFD_QUEUE_TYPE_SDMA) ||2371	    (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)) {2372		retval = read_sdma_queue_counter((uint64_t __user *)q->properties.read_ptr,2373							&sdma_val);2374		if (retval)2375			dev_err(dev, "Failed to read SDMA queue counter for queue: %d\n",2376				q->properties.queue_id);2377	}2378 2379	/* remove queue from list to prevent rescheduling after preemption */2380	dqm_lock(dqm);2381 2382	retval = wait_on_destroy_queue(dqm, q);2383 2384	if (retval) {2385		dqm_unlock(dqm);2386		return retval;2387	}2388 2389	if (qpd->is_debug) {2390		/*2391		 * error, currently we do not allow to destroy a queue2392		 * of a currently debugged process2393		 */2394		retval = -EBUSY;2395		goto failed_try_destroy_debugged_queue;2396 2397	}2398 2399	mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type(2400			q->properties.type)];2401 2402	deallocate_doorbell(qpd, q);2403 2404	if ((q->properties.type == KFD_QUEUE_TYPE_SDMA) ||2405	    (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)) {2406		deallocate_sdma_queue(dqm, q);2407		pdd->sdma_past_activity_counter += sdma_val;2408	}2409 2410	if (q->properties.is_active) {2411		decrement_queue_count(dqm, qpd, q);2412		q->properties.is_active = false;2413		if (!dqm->dev->kfd->shared_resources.enable_mes) {2414			retval = execute_queues_cpsch(dqm,2415						      KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0,2416						      USE_DEFAULT_GRACE_PERIOD);2417			if (retval == -ETIME)2418				qpd->reset_wavefronts = true;2419		} else {2420			retval = remove_queue_mes(dqm, q, qpd);2421		}2422	}2423	list_del(&q->list);2424	qpd->queue_count--;2425 2426	/*2427	 * Unconditionally decrement this counter, regardless of the queue's2428	 * type2429	 */2430	dqm->total_queue_count--;2431	pr_debug("Total of %d queues are accountable so far\n",2432			dqm->total_queue_count);2433 2434	dqm_unlock(dqm);2435 2436	/*2437	 * Do free_mqd and raise delete event after dqm_unlock(dqm) to avoid2438	 * circular locking2439	 */2440	kfd_dbg_ev_raise(KFD_EC_MASK(EC_DEVICE_QUEUE_DELETE),2441				qpd->pqm->process, q->device,2442				-1, false, NULL, 0);2443 2444	mqd_mgr->free_mqd(mqd_mgr, q->mqd, q->mqd_mem_obj);2445 2446	return retval;2447 2448failed_try_destroy_debugged_queue:2449 2450	dqm_unlock(dqm);2451	return retval;2452}2453 2454/*2455 * Low bits must be 0000/FFFF as required by HW, high bits must be 0 to2456 * stay in user mode.2457 */2458#define APE1_FIXED_BITS_MASK 0xFFFF80000000FFFFULL2459/* APE1 limit is inclusive and 64K aligned. */2460#define APE1_LIMIT_ALIGNMENT 0xFFFF2461 2462static bool set_cache_memory_policy(struct device_queue_manager *dqm,2463				   struct qcm_process_device *qpd,2464				   enum cache_policy default_policy,2465				   enum cache_policy alternate_policy,2466				   void __user *alternate_aperture_base,2467				   uint64_t alternate_aperture_size)2468{2469	bool retval = true;2470 2471	if (!dqm->asic_ops.set_cache_memory_policy)2472		return retval;2473 2474	dqm_lock(dqm);2475 2476	if (alternate_aperture_size == 0) {2477		/* base > limit disables APE1 */2478		qpd->sh_mem_ape1_base = 1;2479		qpd->sh_mem_ape1_limit = 0;2480	} else {2481		/*2482		 * In FSA64, APE1_Base[63:0] = { 16{SH_MEM_APE1_BASE[31]},2483		 *			SH_MEM_APE1_BASE[31:0], 0x0000 }2484		 * APE1_Limit[63:0] = { 16{SH_MEM_APE1_LIMIT[31]},2485		 *			SH_MEM_APE1_LIMIT[31:0], 0xFFFF }2486		 * Verify that the base and size parameters can be2487		 * represented in this format and convert them.2488		 * Additionally restrict APE1 to user-mode addresses.2489		 */2490 2491		uint64_t base = (uintptr_t)alternate_aperture_base;2492		uint64_t limit = base + alternate_aperture_size - 1;2493 2494		if (limit <= base || (base & APE1_FIXED_BITS_MASK) != 0 ||2495		   (limit & APE1_FIXED_BITS_MASK) != APE1_LIMIT_ALIGNMENT) {2496			retval = false;2497			goto out;2498		}2499 2500		qpd->sh_mem_ape1_base = base >> 16;2501		qpd->sh_mem_ape1_limit = limit >> 16;2502	}2503 2504	retval = dqm->asic_ops.set_cache_memory_policy(2505			dqm,2506			qpd,2507			default_policy,2508			alternate_policy,2509			alternate_aperture_base,2510			alternate_aperture_size);2511 2512	if ((dqm->sched_policy == KFD_SCHED_POLICY_NO_HWS) && (qpd->vmid != 0))2513		program_sh_mem_settings(dqm, qpd);2514 2515	pr_debug("sh_mem_config: 0x%x, ape1_base: 0x%x, ape1_limit: 0x%x\n",2516		qpd->sh_mem_config, qpd->sh_mem_ape1_base,2517		qpd->sh_mem_ape1_limit);2518 2519out:2520	dqm_unlock(dqm);2521	return retval;2522}2523 2524static int process_termination_nocpsch(struct device_queue_manager *dqm,2525		struct qcm_process_device *qpd)2526{2527	struct queue *q;2528	struct device_process_node *cur, *next_dpn;2529	int retval = 0;2530	bool found = false;2531 2532	dqm_lock(dqm);2533 2534	/* Clear all user mode queues */2535	while (!list_empty(&qpd->queues_list)) {2536		struct mqd_manager *mqd_mgr;2537		int ret;2538 2539		q = list_first_entry(&qpd->queues_list, struct queue, list);2540		mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type(2541				q->properties.type)];2542		ret = destroy_queue_nocpsch_locked(dqm, qpd, q);2543		if (ret)2544			retval = ret;2545		dqm_unlock(dqm);2546		mqd_mgr->free_mqd(mqd_mgr, q->mqd, q->mqd_mem_obj);2547		dqm_lock(dqm);2548	}2549 2550	/* Unregister process */2551	list_for_each_entry_safe(cur, next_dpn, &dqm->queues, list) {2552		if (qpd == cur->qpd) {2553			list_del(&cur->list);2554			kfree(cur);2555			dqm->processes_count--;2556			found = true;2557			break;2558		}2559	}2560 2561	dqm_unlock(dqm);2562 2563	/* Outside the DQM lock because under the DQM lock we can't do2564	 * reclaim or take other locks that others hold while reclaiming.2565	 */2566	if (found)2567		kfd_dec_compute_active(dqm->dev);2568 2569	return retval;2570}2571 2572static int get_wave_state(struct device_queue_manager *dqm,2573			  struct queue *q,2574			  void __user *ctl_stack,2575			  u32 *ctl_stack_used_size,2576			  u32 *save_area_used_size)2577{2578	struct mqd_manager *mqd_mgr;2579 2580	dqm_lock(dqm);2581 2582	mqd_mgr = dqm->mqd_mgrs[KFD_MQD_TYPE_CP];2583 2584	if (q->properties.type != KFD_QUEUE_TYPE_COMPUTE ||2585	    q->properties.is_active || !q->device->kfd->cwsr_enabled ||2586	    !mqd_mgr->get_wave_state) {2587		dqm_unlock(dqm);2588		return -EINVAL;2589	}2590 2591	dqm_unlock(dqm);2592 2593	/*2594	 * get_wave_state is outside the dqm lock to prevent circular locking2595	 * and the queue should be protected against destruction by the process2596	 * lock.2597	 */2598	return mqd_mgr->get_wave_state(mqd_mgr, q->mqd, &q->properties,2599			ctl_stack, ctl_stack_used_size, save_area_used_size);2600}2601 2602static void get_queue_checkpoint_info(struct device_queue_manager *dqm,2603			const struct queue *q,2604			u32 *mqd_size,2605			u32 *ctl_stack_size)2606{2607	struct mqd_manager *mqd_mgr;2608	enum KFD_MQD_TYPE mqd_type =2609			get_mqd_type_from_queue_type(q->properties.type);2610 2611	dqm_lock(dqm);2612	mqd_mgr = dqm->mqd_mgrs[mqd_type];2613	*mqd_size = mqd_mgr->mqd_size;2614	*ctl_stack_size = 0;2615 2616	if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE && mqd_mgr->get_checkpoint_info)2617		mqd_mgr->get_checkpoint_info(mqd_mgr, q->mqd, ctl_stack_size);2618 2619	dqm_unlock(dqm);2620}2621 2622static int checkpoint_mqd(struct device_queue_manager *dqm,2623			  const struct queue *q,2624			  void *mqd,2625			  void *ctl_stack)2626{2627	struct mqd_manager *mqd_mgr;2628	int r = 0;2629	enum KFD_MQD_TYPE mqd_type =2630			get_mqd_type_from_queue_type(q->properties.type);2631 2632	dqm_lock(dqm);2633 2634	if (q->properties.is_active || !q->device->kfd->cwsr_enabled) {2635		r = -EINVAL;2636		goto dqm_unlock;2637	}2638 2639	mqd_mgr = dqm->mqd_mgrs[mqd_type];2640	if (!mqd_mgr->checkpoint_mqd) {2641		r = -EOPNOTSUPP;2642		goto dqm_unlock;2643	}2644 2645	mqd_mgr->checkpoint_mqd(mqd_mgr, q->mqd, mqd, ctl_stack);2646 2647dqm_unlock:2648	dqm_unlock(dqm);2649	return r;2650}2651 2652static int process_termination_cpsch(struct device_queue_manager *dqm,2653		struct qcm_process_device *qpd)2654{2655	int retval;2656	struct queue *q;2657	struct device *dev = dqm->dev->adev->dev;2658	struct kernel_queue *kq, *kq_next;2659	struct mqd_manager *mqd_mgr;2660	struct device_process_node *cur, *next_dpn;2661	enum kfd_unmap_queues_filter filter =2662		KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES;2663	bool found = false;2664 2665	retval = 0;2666 2667	dqm_lock(dqm);2668 2669	/* Clean all kernel queues */2670	list_for_each_entry_safe(kq, kq_next, &qpd->priv_queue_list, list) {2671		list_del(&kq->list);2672		decrement_queue_count(dqm, qpd, kq->queue);2673		qpd->is_debug = false;2674		dqm->total_queue_count--;2675		filter = KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES;2676	}2677 2678	/* Clear all user mode queues */2679	list_for_each_entry(q, &qpd->queues_list, list) {2680		if (q->properties.type == KFD_QUEUE_TYPE_SDMA)2681			deallocate_sdma_queue(dqm, q);2682		else if (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)2683			deallocate_sdma_queue(dqm, q);2684 2685		if (q->properties.is_active) {2686			decrement_queue_count(dqm, qpd, q);2687 2688			if (dqm->dev->kfd->shared_resources.enable_mes) {2689				retval = remove_queue_mes(dqm, q, qpd);2690				if (retval)2691					dev_err(dev, "Failed to remove queue %d\n",2692						q->properties.queue_id);2693			}2694		}2695 2696		dqm->total_queue_count--;2697	}2698 2699	/* Unregister process */2700	list_for_each_entry_safe(cur, next_dpn, &dqm->queues, list) {2701		if (qpd == cur->qpd) {2702			list_del(&cur->list);2703			kfree(cur);2704			dqm->processes_count--;2705			found = true;2706			break;2707		}2708	}2709 2710	if (!dqm->dev->kfd->shared_resources.enable_mes)2711		retval = execute_queues_cpsch(dqm, filter, 0, USE_DEFAULT_GRACE_PERIOD);2712 2713	if ((retval || qpd->reset_wavefronts) &&2714	    down_read_trylock(&dqm->dev->adev->reset_domain->sem)) {2715		pr_warn("Resetting wave fronts (cpsch) on dev %p\n", dqm->dev);2716		dbgdev_wave_reset_wavefronts(dqm->dev, qpd->pqm->process);2717		qpd->reset_wavefronts = false;2718		up_read(&dqm->dev->adev->reset_domain->sem);2719	}2720 2721	/* Lastly, free mqd resources.2722	 * Do free_mqd() after dqm_unlock to avoid circular locking.2723	 */2724	while (!list_empty(&qpd->queues_list)) {2725		q = list_first_entry(&qpd->queues_list, struct queue, list);2726		mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type(2727				q->properties.type)];2728		list_del(&q->list);2729		qpd->queue_count--;2730		dqm_unlock(dqm);2731		mqd_mgr->free_mqd(mqd_mgr, q->mqd, q->mqd_mem_obj);2732		dqm_lock(dqm);2733	}2734	dqm_unlock(dqm);2735 2736	/* Outside the DQM lock because under the DQM lock we can't do2737	 * reclaim or take other locks that others hold while reclaiming.2738	 */2739	if (found)2740		kfd_dec_compute_active(dqm->dev);2741 2742	return retval;2743}2744 2745static int init_mqd_managers(struct device_queue_manager *dqm)2746{2747	int i, j;2748	struct device *dev = dqm->dev->adev->dev;2749	struct mqd_manager *mqd_mgr;2750 2751	for (i = 0; i < KFD_MQD_TYPE_MAX; i++) {2752		mqd_mgr = dqm->asic_ops.mqd_manager_init(i, dqm->dev);2753		if (!mqd_mgr) {2754			dev_err(dev, "mqd manager [%d] initialization failed\n", i);2755			goto out_free;2756		}2757		dqm->mqd_mgrs[i] = mqd_mgr;2758	}2759 2760	return 0;2761 2762out_free:2763	for (j = 0; j < i; j++) {2764		kfree(dqm->mqd_mgrs[j]);2765		dqm->mqd_mgrs[j] = NULL;2766	}2767 2768	return -ENOMEM;2769}2770 2771/* Allocate one hiq mqd (HWS) and all SDMA mqd in a continuous trunk*/2772static int allocate_hiq_sdma_mqd(struct device_queue_manager *dqm)2773{2774	int retval;2775	struct kfd_node *dev = dqm->dev;2776	struct kfd_mem_obj *mem_obj = &dqm->hiq_sdma_mqd;2777	uint32_t size = dqm->mqd_mgrs[KFD_MQD_TYPE_SDMA]->mqd_size *2778		get_num_all_sdma_engines(dqm) *2779		dev->kfd->device_info.num_sdma_queues_per_engine +2780		(dqm->mqd_mgrs[KFD_MQD_TYPE_HIQ]->mqd_size *2781		NUM_XCC(dqm->dev->xcc_mask));2782 2783	retval = amdgpu_amdkfd_alloc_gtt_mem(dev->adev, size,2784		&(mem_obj->gtt_mem), &(mem_obj->gpu_addr),2785		(void *)&(mem_obj->cpu_ptr), false);2786 2787	return retval;2788}2789 2790struct device_queue_manager *device_queue_manager_init(struct kfd_node *dev)2791{2792	struct device_queue_manager *dqm;2793 2794	pr_debug("Loading device queue manager\n");2795 2796	dqm = kzalloc(sizeof(*dqm), GFP_KERNEL);2797	if (!dqm)2798		return NULL;2799 2800	switch (dev->adev->asic_type) {2801	/* HWS is not available on Hawaii. */2802	case CHIP_HAWAII:2803	/* HWS depends on CWSR for timely dequeue. CWSR is not2804	 * available on Tonga.2805	 *2806	 * FIXME: This argument also applies to Kaveri.2807	 */2808	case CHIP_TONGA:2809		dqm->sched_policy = KFD_SCHED_POLICY_NO_HWS;2810		break;2811	default:2812		dqm->sched_policy = sched_policy;2813		break;2814	}2815 2816	dqm->dev = dev;2817	switch (dqm->sched_policy) {2818	case KFD_SCHED_POLICY_HWS:2819	case KFD_SCHED_POLICY_HWS_NO_OVERSUBSCRIPTION:2820		/* initialize dqm for cp scheduling */2821		dqm->ops.create_queue = create_queue_cpsch;2822		dqm->ops.initialize = initialize_cpsch;2823		dqm->ops.start = start_cpsch;2824		dqm->ops.stop = stop_cpsch;2825		dqm->ops.halt = halt_cpsch;2826		dqm->ops.unhalt = unhalt_cpsch;2827		dqm->ops.destroy_queue = destroy_queue_cpsch;2828		dqm->ops.update_queue = update_queue;2829		dqm->ops.register_process = register_process;2830		dqm->ops.unregister_process = unregister_process;2831		dqm->ops.uninitialize = uninitialize;2832		dqm->ops.create_kernel_queue = create_kernel_queue_cpsch;2833		dqm->ops.destroy_kernel_queue = destroy_kernel_queue_cpsch;2834		dqm->ops.set_cache_memory_policy = set_cache_memory_policy;2835		dqm->ops.process_termination = process_termination_cpsch;2836		dqm->ops.evict_process_queues = evict_process_queues_cpsch;2837		dqm->ops.restore_process_queues = restore_process_queues_cpsch;2838		dqm->ops.get_wave_state = get_wave_state;2839		dqm->ops.reset_queues = reset_queues_cpsch;2840		dqm->ops.get_queue_checkpoint_info = get_queue_checkpoint_info;2841		dqm->ops.checkpoint_mqd = checkpoint_mqd;2842		break;2843	case KFD_SCHED_POLICY_NO_HWS:2844		/* initialize dqm for no cp scheduling */2845		dqm->ops.start = start_nocpsch;2846		dqm->ops.stop = stop_nocpsch;2847		dqm->ops.create_queue = create_queue_nocpsch;2848		dqm->ops.destroy_queue = destroy_queue_nocpsch;2849		dqm->ops.update_queue = update_queue;2850		dqm->ops.register_process = register_process;2851		dqm->ops.unregister_process = unregister_process;2852		dqm->ops.initialize = initialize_nocpsch;2853		dqm->ops.uninitialize = uninitialize;2854		dqm->ops.set_cache_memory_policy = set_cache_memory_policy;2855		dqm->ops.process_termination = process_termination_nocpsch;2856		dqm->ops.evict_process_queues = evict_process_queues_nocpsch;2857		dqm->ops.restore_process_queues =2858			restore_process_queues_nocpsch;2859		dqm->ops.get_wave_state = get_wave_state;2860		dqm->ops.get_queue_checkpoint_info = get_queue_checkpoint_info;2861		dqm->ops.checkpoint_mqd = checkpoint_mqd;2862		break;2863	default:2864		dev_err(dev->adev->dev, "Invalid scheduling policy %d\n", dqm->sched_policy);2865		goto out_free;2866	}2867 2868	switch (dev->adev->asic_type) {2869	case CHIP_KAVERI:2870	case CHIP_HAWAII:2871		device_queue_manager_init_cik(&dqm->asic_ops);2872		break;2873 2874	case CHIP_CARRIZO:2875	case CHIP_TONGA:2876	case CHIP_FIJI:2877	case CHIP_POLARIS10:2878	case CHIP_POLARIS11:2879	case CHIP_POLARIS12:2880	case CHIP_VEGAM:2881		device_queue_manager_init_vi(&dqm->asic_ops);2882		break;2883 2884	default:2885		if (KFD_GC_VERSION(dev) >= IP_VERSION(12, 0, 0))2886			device_queue_manager_init_v12(&dqm->asic_ops);2887		else if (KFD_GC_VERSION(dev) >= IP_VERSION(11, 0, 0))2888			device_queue_manager_init_v11(&dqm->asic_ops);2889		else if (KFD_GC_VERSION(dev) >= IP_VERSION(10, 1, 1))2890			device_queue_manager_init_v10(&dqm->asic_ops);2891		else if (KFD_GC_VERSION(dev) >= IP_VERSION(9, 0, 1))2892			device_queue_manager_init_v9(&dqm->asic_ops);2893		else {2894			WARN(1, "Unexpected ASIC family %u",2895			     dev->adev->asic_type);2896			goto out_free;2897		}2898	}2899 2900	if (init_mqd_managers(dqm))2901		goto out_free;2902 2903	if (!dev->kfd->shared_resources.enable_mes && allocate_hiq_sdma_mqd(dqm)) {2904		dev_err(dev->adev->dev, "Failed to allocate hiq sdma mqd trunk buffer\n");2905		goto out_free;2906	}2907 2908	if (!dqm->ops.initialize(dqm)) {2909		init_waitqueue_head(&dqm->destroy_wait);2910		return dqm;2911	}2912 2913out_free:2914	kfree(dqm);2915	return NULL;2916}2917 2918static void deallocate_hiq_sdma_mqd(struct kfd_node *dev,2919				    struct kfd_mem_obj *mqd)2920{2921	WARN(!mqd, "No hiq sdma mqd trunk to free");2922 2923	amdgpu_amdkfd_free_gtt_mem(dev->adev, &mqd->gtt_mem);2924}2925 2926void device_queue_manager_uninit(struct device_queue_manager *dqm)2927{2928	dqm->ops.stop(dqm);2929	dqm->ops.uninitialize(dqm);2930	if (!dqm->dev->kfd->shared_resources.enable_mes)2931		deallocate_hiq_sdma_mqd(dqm->dev, &dqm->hiq_sdma_mqd);2932	kfree(dqm);2933}2934 2935int kfd_dqm_suspend_bad_queue_mes(struct kfd_node *knode, u32 pasid, u32 doorbell_id)2936{2937	struct kfd_process_device *pdd;2938	struct kfd_process *p = kfd_lookup_process_by_pasid(pasid);2939	struct device_queue_manager *dqm = knode->dqm;2940	struct device *dev = dqm->dev->adev->dev;2941	struct qcm_process_device *qpd;2942	struct queue *q = NULL;2943	int ret = 0;2944 2945	if (!p)2946		return -EINVAL;2947 2948	dqm_lock(dqm);2949 2950	pdd = kfd_get_process_device_data(dqm->dev, p);2951	if (pdd) {2952		qpd = &pdd->qpd;2953 2954		list_for_each_entry(q, &qpd->queues_list, list) {2955			if (q->doorbell_id == doorbell_id && q->properties.is_active) {2956				ret = suspend_all_queues_mes(dqm);2957				if (ret) {2958					dev_err(dev, "Suspending all queues failed");2959					goto out;2960				}2961 2962				q->properties.is_evicted = true;2963				q->properties.is_active = false;2964				decrement_queue_count(dqm, qpd, q);2965 2966				ret = remove_queue_mes(dqm, q, qpd);2967				if (ret) {2968					dev_err(dev, "Removing bad queue failed");2969					goto out;2970				}2971 2972				ret = resume_all_queues_mes(dqm);2973				if (ret)2974					dev_err(dev, "Resuming all queues failed");2975 2976				break;2977			}2978		}2979	}2980 2981out:2982	dqm_unlock(dqm);2983	return ret;2984}2985 2986static int kfd_dqm_evict_pasid_mes(struct device_queue_manager *dqm,2987				   struct qcm_process_device *qpd)2988{2989	struct device *dev = dqm->dev->adev->dev;2990	int ret = 0;2991 2992	/* Check if process is already evicted */2993	dqm_lock(dqm);2994	if (qpd->evicted) {2995		/* Increment the evicted count to make sure the2996		 * process stays evicted before its terminated.2997		 */2998		qpd->evicted++;2999		dqm_unlock(dqm);3000		goto out;3001	}3002	dqm_unlock(dqm);3003 3004	ret = suspend_all_queues_mes(dqm);3005	if (ret) {3006		dev_err(dev, "Suspending all queues failed");3007		goto out;3008	}3009 3010	ret = dqm->ops.evict_process_queues(dqm, qpd);3011	if (ret) {3012		dev_err(dev, "Evicting process queues failed");3013		goto out;3014	}3015 3016	ret = resume_all_queues_mes(dqm);3017	if (ret)3018		dev_err(dev, "Resuming all queues failed");3019 3020out:3021	return ret;3022}3023 3024int kfd_dqm_evict_pasid(struct device_queue_manager *dqm, u32 pasid)3025{3026	struct kfd_process_device *pdd;3027	struct kfd_process *p = kfd_lookup_process_by_pasid(pasid);3028	int ret = 0;3029 3030	if (!p)3031		return -EINVAL;3032	WARN(debug_evictions, "Evicting pid %d", p->lead_thread->pid);3033	pdd = kfd_get_process_device_data(dqm->dev, p);3034	if (pdd) {3035		if (dqm->dev->kfd->shared_resources.enable_mes)3036			ret = kfd_dqm_evict_pasid_mes(dqm, &pdd->qpd);3037		else3038			ret = dqm->ops.evict_process_queues(dqm, &pdd->qpd);3039	}3040 3041	kfd_unref_process(p);3042 3043	return ret;3044}3045 3046static void kfd_process_hw_exception(struct work_struct *work)3047{3048	struct device_queue_manager *dqm = container_of(work,3049			struct device_queue_manager, hw_exception_work);3050	amdgpu_amdkfd_gpu_reset(dqm->dev->adev);3051}3052 3053int reserve_debug_trap_vmid(struct device_queue_manager *dqm,3054				struct qcm_process_device *qpd)3055{3056	int r;3057	struct device *dev = dqm->dev->adev->dev;3058	int updated_vmid_mask;3059 3060	if (dqm->sched_policy == KFD_SCHED_POLICY_NO_HWS) {3061		dev_err(dev, "Unsupported on sched_policy: %i\n", dqm->sched_policy);3062		return -EINVAL;3063	}3064 3065	dqm_lock(dqm);3066 3067	if (dqm->trap_debug_vmid != 0) {3068		dev_err(dev, "Trap debug id already reserved\n");3069		r = -EBUSY;3070		goto out_unlock;3071	}3072 3073	r = unmap_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES, 0,3074			USE_DEFAULT_GRACE_PERIOD, false);3075	if (r)3076		goto out_unlock;3077 3078	updated_vmid_mask = dqm->dev->kfd->shared_resources.compute_vmid_bitmap;3079	updated_vmid_mask &= ~(1 << dqm->dev->vm_info.last_vmid_kfd);3080 3081	dqm->dev->kfd->shared_resources.compute_vmid_bitmap = updated_vmid_mask;3082	dqm->trap_debug_vmid = dqm->dev->vm_info.last_vmid_kfd;3083	r = set_sched_resources(dqm);3084	if (r)3085		goto out_unlock;3086 3087	r = map_queues_cpsch(dqm);3088	if (r)3089		goto out_unlock;3090 3091	pr_debug("Reserved VMID for trap debug: %i\n", dqm->trap_debug_vmid);3092 3093out_unlock:3094	dqm_unlock(dqm);3095	return r;3096}3097 3098/*3099 * Releases vmid for the trap debugger3100 */3101int release_debug_trap_vmid(struct device_queue_manager *dqm,3102			struct qcm_process_device *qpd)3103{3104	struct device *dev = dqm->dev->adev->dev;3105	int r;3106	int updated_vmid_mask;3107	uint32_t trap_debug_vmid;3108 3109	if (dqm->sched_policy == KFD_SCHED_POLICY_NO_HWS) {3110		dev_err(dev, "Unsupported on sched_policy: %i\n", dqm->sched_policy);3111		return -EINVAL;3112	}3113 3114	dqm_lock(dqm);3115	trap_debug_vmid = dqm->trap_debug_vmid;3116	if (dqm->trap_debug_vmid == 0) {3117		dev_err(dev, "Trap debug id is not reserved\n");3118		r = -EINVAL;3119		goto out_unlock;3120	}3121 3122	r = unmap_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES, 0,3123			USE_DEFAULT_GRACE_PERIOD, false);3124	if (r)3125		goto out_unlock;3126 3127	updated_vmid_mask = dqm->dev->kfd->shared_resources.compute_vmid_bitmap;3128	updated_vmid_mask |= (1 << dqm->dev->vm_info.last_vmid_kfd);3129 3130	dqm->dev->kfd->shared_resources.compute_vmid_bitmap = updated_vmid_mask;3131	dqm->trap_debug_vmid = 0;3132	r = set_sched_resources(dqm);3133	if (r)3134		goto out_unlock;3135 3136	r = map_queues_cpsch(dqm);3137	if (r)3138		goto out_unlock;3139 3140	pr_debug("Released VMID for trap debug: %i\n", trap_debug_vmid);3141 3142out_unlock:3143	dqm_unlock(dqm);3144	return r;3145}3146 3147#define QUEUE_NOT_FOUND		-13148/* invalidate queue operation in array */3149static void q_array_invalidate(uint32_t num_queues, uint32_t *queue_ids)3150{3151	int i;3152 3153	for (i = 0; i < num_queues; i++)3154		queue_ids[i] |= KFD_DBG_QUEUE_INVALID_MASK;3155}3156 3157/* find queue index in array */3158static int q_array_get_index(unsigned int queue_id,3159		uint32_t num_queues,3160		uint32_t *queue_ids)3161{3162	int i;3163 3164	for (i = 0; i < num_queues; i++)3165		if (queue_id == (queue_ids[i] & ~KFD_DBG_QUEUE_INVALID_MASK))3166			return i;3167 3168	return QUEUE_NOT_FOUND;3169}3170 3171struct copy_context_work_handler_workarea {3172	struct work_struct copy_context_work;3173	struct kfd_process *p;3174};3175 3176static void copy_context_work_handler (struct work_struct *work)3177{3178	struct copy_context_work_handler_workarea *workarea;3179	struct mqd_manager *mqd_mgr;3180	struct queue *q;3181	struct mm_struct *mm;3182	struct kfd_process *p;3183	uint32_t tmp_ctl_stack_used_size, tmp_save_area_used_size;3184	int i;3185 3186	workarea = container_of(work,3187			struct copy_context_work_handler_workarea,3188			copy_context_work);3189 3190	p = workarea->p;3191	mm = get_task_mm(p->lead_thread);3192 3193	if (!mm)3194		return;3195 3196	kthread_use_mm(mm);3197	for (i = 0; i < p->n_pdds; i++) {3198		struct kfd_process_device *pdd = p->pdds[i];3199		struct device_queue_manager *dqm = pdd->dev->dqm;3200		struct qcm_process_device *qpd = &pdd->qpd;3201 3202		list_for_each_entry(q, &qpd->queues_list, list) {3203			mqd_mgr = dqm->mqd_mgrs[KFD_MQD_TYPE_CP];3204 3205			/* We ignore the return value from get_wave_state3206			 * because3207			 * i) right now, it always returns 0, and3208			 * ii) if we hit an error, we would continue to the3209			 *      next queue anyway.3210			 */3211			mqd_mgr->get_wave_state(mqd_mgr,3212					q->mqd,3213					&q->properties,3214					(void __user *)	q->properties.ctx_save_restore_area_address,3215					&tmp_ctl_stack_used_size,3216					&tmp_save_area_used_size);3217		}3218	}3219	kthread_unuse_mm(mm);3220	mmput(mm);3221}3222 3223static uint32_t *get_queue_ids(uint32_t num_queues, uint32_t *usr_queue_id_array)3224{3225	size_t array_size = num_queues * sizeof(uint32_t);3226 3227	if (!usr_queue_id_array)3228		return NULL;3229 3230	return memdup_user(usr_queue_id_array, array_size);3231}3232 3233int resume_queues(struct kfd_process *p,3234		uint32_t num_queues,3235		uint32_t *usr_queue_id_array)3236{3237	uint32_t *queue_ids = NULL;3238	int total_resumed = 0;3239	int i;3240 3241	if (usr_queue_id_array) {3242		queue_ids = get_queue_ids(num_queues, usr_queue_id_array);3243 3244		if (IS_ERR(queue_ids))3245			return PTR_ERR(queue_ids);3246 3247		/* mask all queues as invalid.  unmask per successful request */3248		q_array_invalidate(num_queues, queue_ids);3249	}3250 3251	for (i = 0; i < p->n_pdds; i++) {3252		struct kfd_process_device *pdd = p->pdds[i];3253		struct device_queue_manager *dqm = pdd->dev->dqm;3254		struct device *dev = dqm->dev->adev->dev;3255		struct qcm_process_device *qpd = &pdd->qpd;3256		struct queue *q;3257		int r, per_device_resumed = 0;3258 3259		dqm_lock(dqm);3260 3261		/* unmask queues that resume or already resumed as valid */3262		list_for_each_entry(q, &qpd->queues_list, list) {3263			int q_idx = QUEUE_NOT_FOUND;3264 3265			if (queue_ids)3266				q_idx = q_array_get_index(3267						q->properties.queue_id,3268						num_queues,3269						queue_ids);3270 3271			if (!queue_ids || q_idx != QUEUE_NOT_FOUND) {3272				int err = resume_single_queue(dqm, &pdd->qpd, q);3273 3274				if (queue_ids) {3275					if (!err) {3276						queue_ids[q_idx] &=3277							~KFD_DBG_QUEUE_INVALID_MASK;3278					} else {3279						queue_ids[q_idx] |=3280							KFD_DBG_QUEUE_ERROR_MASK;3281						break;3282					}3283				}3284 3285				if (dqm->dev->kfd->shared_resources.enable_mes) {3286					wake_up_all(&dqm->destroy_wait);3287					if (!err)3288						total_resumed++;3289				} else {3290					per_device_resumed++;3291				}3292			}3293		}3294 3295		if (!per_device_resumed) {3296			dqm_unlock(dqm);3297			continue;3298		}3299 3300		r = execute_queues_cpsch(dqm,3301					KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES,3302					0,3303					USE_DEFAULT_GRACE_PERIOD);3304		if (r) {3305			dev_err(dev, "Failed to resume process queues\n");3306			if (queue_ids) {3307				list_for_each_entry(q, &qpd->queues_list, list) {3308					int q_idx = q_array_get_index(3309							q->properties.queue_id,3310							num_queues,3311							queue_ids);3312 3313					/* mask queue as error on resume fail */3314					if (q_idx != QUEUE_NOT_FOUND)3315						queue_ids[q_idx] |=3316							KFD_DBG_QUEUE_ERROR_MASK;3317				}3318			}3319		} else {3320			wake_up_all(&dqm->destroy_wait);3321			total_resumed += per_device_resumed;3322		}3323 3324		dqm_unlock(dqm);3325	}3326 3327	if (queue_ids) {3328		if (copy_to_user((void __user *)usr_queue_id_array, queue_ids,3329				num_queues * sizeof(uint32_t)))3330			pr_err("copy_to_user failed on queue resume\n");3331 3332		kfree(queue_ids);3333	}3334 3335	return total_resumed;3336}3337 3338int suspend_queues(struct kfd_process *p,3339			uint32_t num_queues,3340			uint32_t grace_period,3341			uint64_t exception_clear_mask,3342			uint32_t *usr_queue_id_array)3343{3344	uint32_t *queue_ids = get_queue_ids(num_queues, usr_queue_id_array);3345	int total_suspended = 0;3346	int i;3347 3348	if (IS_ERR(queue_ids))3349		return PTR_ERR(queue_ids);3350 3351	/* mask all queues as invalid.  umask on successful request */3352	q_array_invalidate(num_queues, queue_ids);3353 3354	for (i = 0; i < p->n_pdds; i++) {3355		struct kfd_process_device *pdd = p->pdds[i];3356		struct device_queue_manager *dqm = pdd->dev->dqm;3357		struct device *dev = dqm->dev->adev->dev;3358		struct qcm_process_device *qpd = &pdd->qpd;3359		struct queue *q;3360		int r, per_device_suspended = 0;3361 3362		mutex_lock(&p->event_mutex);3363		dqm_lock(dqm);3364 3365		/* unmask queues that suspend or already suspended */3366		list_for_each_entry(q, &qpd->queues_list, list) {3367			int q_idx = q_array_get_index(q->properties.queue_id,3368							num_queues,3369							queue_ids);3370 3371			if (q_idx != QUEUE_NOT_FOUND) {3372				int err = suspend_single_queue(dqm, pdd, q);3373				bool is_mes = dqm->dev->kfd->shared_resources.enable_mes;3374 3375				if (!err) {3376					queue_ids[q_idx] &= ~KFD_DBG_QUEUE_INVALID_MASK;3377					if (exception_clear_mask && is_mes)3378						q->properties.exception_status &=3379							~exception_clear_mask;3380 3381					if (is_mes)3382						total_suspended++;3383					else3384						per_device_suspended++;3385				} else if (err != -EBUSY) {3386					r = err;3387					queue_ids[q_idx] |= KFD_DBG_QUEUE_ERROR_MASK;3388					break;3389				}3390			}3391		}3392 3393		if (!per_device_suspended) {3394			dqm_unlock(dqm);3395			mutex_unlock(&p->event_mutex);3396			if (total_suspended)3397				amdgpu_amdkfd_debug_mem_fence(dqm->dev->adev);3398			continue;3399		}3400 3401		r = execute_queues_cpsch(dqm,3402			KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0,3403			grace_period);3404 3405		if (r)3406			dev_err(dev, "Failed to suspend process queues.\n");3407		else3408			total_suspended += per_device_suspended;3409 3410		list_for_each_entry(q, &qpd->queues_list, list) {3411			int q_idx = q_array_get_index(q->properties.queue_id,3412						num_queues, queue_ids);3413 3414			if (q_idx == QUEUE_NOT_FOUND)3415				continue;3416 3417			/* mask queue as error on suspend fail */3418			if (r)3419				queue_ids[q_idx] |= KFD_DBG_QUEUE_ERROR_MASK;3420			else if (exception_clear_mask)3421				q->properties.exception_status &=3422							~exception_clear_mask;3423		}3424 3425		dqm_unlock(dqm);3426		mutex_unlock(&p->event_mutex);3427		amdgpu_device_flush_hdp(dqm->dev->adev, NULL);3428	}3429 3430	if (total_suspended) {3431		struct copy_context_work_handler_workarea copy_context_worker;3432 3433		INIT_WORK_ONSTACK(3434				&copy_context_worker.copy_context_work,3435				copy_context_work_handler);3436 3437		copy_context_worker.p = p;3438 3439		schedule_work(&copy_context_worker.copy_context_work);3440 3441 3442		flush_work(&copy_context_worker.copy_context_work);3443		destroy_work_on_stack(&copy_context_worker.copy_context_work);3444	}3445 3446	if (copy_to_user((void __user *)usr_queue_id_array, queue_ids,3447			num_queues * sizeof(uint32_t)))3448		pr_err("copy_to_user failed on queue suspend\n");3449 3450	kfree(queue_ids);3451 3452	return total_suspended;3453}3454 3455static uint32_t set_queue_type_for_user(struct queue_properties *q_props)3456{3457	switch (q_props->type) {3458	case KFD_QUEUE_TYPE_COMPUTE:3459		return q_props->format == KFD_QUEUE_FORMAT_PM43460					? KFD_IOC_QUEUE_TYPE_COMPUTE3461					: KFD_IOC_QUEUE_TYPE_COMPUTE_AQL;3462	case KFD_QUEUE_TYPE_SDMA:3463		return KFD_IOC_QUEUE_TYPE_SDMA;3464	case KFD_QUEUE_TYPE_SDMA_XGMI:3465		return KFD_IOC_QUEUE_TYPE_SDMA_XGMI;3466	default:3467		WARN_ONCE(true, "queue type not recognized!");3468		return 0xffffffff;3469	};3470}3471 3472void set_queue_snapshot_entry(struct queue *q,3473			      uint64_t exception_clear_mask,3474			      struct kfd_queue_snapshot_entry *qss_entry)3475{3476	qss_entry->ring_base_address = q->properties.queue_address;3477	qss_entry->write_pointer_address = (uint64_t)q->properties.write_ptr;3478	qss_entry->read_pointer_address = (uint64_t)q->properties.read_ptr;3479	qss_entry->ctx_save_restore_address =3480				q->properties.ctx_save_restore_area_address;3481	qss_entry->ctx_save_restore_area_size =3482				q->properties.ctx_save_restore_area_size;3483	qss_entry->exception_status = q->properties.exception_status;3484	qss_entry->queue_id = q->properties.queue_id;3485	qss_entry->gpu_id = q->device->id;3486	qss_entry->ring_size = (uint32_t)q->properties.queue_size;3487	qss_entry->queue_type = set_queue_type_for_user(&q->properties);3488	q->properties.exception_status &= ~exception_clear_mask;3489}3490 3491int debug_lock_and_unmap(struct device_queue_manager *dqm)3492{3493	struct device *dev = dqm->dev->adev->dev;3494	int r;3495 3496	if (dqm->sched_policy == KFD_SCHED_POLICY_NO_HWS) {3497		dev_err(dev, "Unsupported on sched_policy: %i\n", dqm->sched_policy);3498		return -EINVAL;3499	}3500 3501	if (!kfd_dbg_is_per_vmid_supported(dqm->dev))3502		return 0;3503 3504	dqm_lock(dqm);3505 3506	r = unmap_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES, 0, 0, false);3507	if (r)3508		dqm_unlock(dqm);3509 3510	return r;3511}3512 3513int debug_map_and_unlock(struct device_queue_manager *dqm)3514{3515	struct device *dev = dqm->dev->adev->dev;3516	int r;3517 3518	if (dqm->sched_policy == KFD_SCHED_POLICY_NO_HWS) {3519		dev_err(dev, "Unsupported on sched_policy: %i\n", dqm->sched_policy);3520		return -EINVAL;3521	}3522 3523	if (!kfd_dbg_is_per_vmid_supported(dqm->dev))3524		return 0;3525 3526	r = map_queues_cpsch(dqm);3527 3528	dqm_unlock(dqm);3529 3530	return r;3531}3532 3533int debug_refresh_runlist(struct device_queue_manager *dqm)3534{3535	int r = debug_lock_and_unmap(dqm);3536 3537	if (r)3538		return r;3539 3540	return debug_map_and_unlock(dqm);3541}3542 3543bool kfd_dqm_is_queue_in_process(struct device_queue_manager *dqm,3544				 struct qcm_process_device *qpd,3545				 int doorbell_off, u32 *queue_format)3546{3547	struct queue *q;3548	bool r = false;3549 3550	if (!queue_format)3551		return r;3552 3553	dqm_lock(dqm);3554 3555	list_for_each_entry(q, &qpd->queues_list, list) {3556		if (q->properties.doorbell_off == doorbell_off) {3557			*queue_format = q->properties.format;3558			r = true;3559			goto out;3560		}3561	}3562 3563out:3564	dqm_unlock(dqm);3565	return r;3566}3567#if defined(CONFIG_DEBUG_FS)3568 3569static void seq_reg_dump(struct seq_file *m,3570			 uint32_t (*dump)[2], uint32_t n_regs)3571{3572	uint32_t i, count;3573 3574	for (i = 0, count = 0; i < n_regs; i++) {3575		if (count == 0 ||3576		    dump[i-1][0] + sizeof(uint32_t) != dump[i][0]) {3577			seq_printf(m, "%s    %08x: %08x",3578				   i ? "\n" : "",3579				   dump[i][0], dump[i][1]);3580			count = 7;3581		} else {3582			seq_printf(m, " %08x", dump[i][1]);3583			count--;3584		}3585	}3586 3587	seq_puts(m, "\n");3588}3589 3590int dqm_debugfs_hqds(struct seq_file *m, void *data)3591{3592	struct device_queue_manager *dqm = data;3593	uint32_t xcc_mask = dqm->dev->xcc_mask;3594	uint32_t (*dump)[2], n_regs;3595	int pipe, queue;3596	int r = 0, xcc_id;3597	uint32_t sdma_engine_start;3598 3599	if (!dqm->sched_running) {3600		seq_puts(m, " Device is stopped\n");3601		return 0;3602	}3603 3604	for_each_inst(xcc_id, xcc_mask) {3605		r = dqm->dev->kfd2kgd->hqd_dump(dqm->dev->adev,3606						KFD_CIK_HIQ_PIPE,3607						KFD_CIK_HIQ_QUEUE, &dump,3608						&n_regs, xcc_id);3609		if (!r) {3610			seq_printf(3611				m,3612				"   Inst %d, HIQ on MEC %d Pipe %d Queue %d\n",3613				xcc_id,3614				KFD_CIK_HIQ_PIPE / get_pipes_per_mec(dqm) + 1,3615				KFD_CIK_HIQ_PIPE % get_pipes_per_mec(dqm),3616				KFD_CIK_HIQ_QUEUE);3617			seq_reg_dump(m, dump, n_regs);3618 3619			kfree(dump);3620		}3621 3622		for (pipe = 0; pipe < get_pipes_per_mec(dqm); pipe++) {3623			int pipe_offset = pipe * get_queues_per_pipe(dqm);3624 3625			for (queue = 0; queue < get_queues_per_pipe(dqm); queue++) {3626				if (!test_bit(pipe_offset + queue,3627				      dqm->dev->kfd->shared_resources.cp_queue_bitmap))3628					continue;3629 3630				r = dqm->dev->kfd2kgd->hqd_dump(dqm->dev->adev,3631								pipe, queue,3632								&dump, &n_regs,3633								xcc_id);3634				if (r)3635					break;3636 3637				seq_printf(m,3638					   " Inst %d,  CP Pipe %d, Queue %d\n",3639					   xcc_id, pipe, queue);3640				seq_reg_dump(m, dump, n_regs);3641 3642				kfree(dump);3643			}3644		}3645	}3646 3647	sdma_engine_start = dqm->dev->node_id * get_num_all_sdma_engines(dqm);3648	for (pipe = sdma_engine_start;3649	     pipe < (sdma_engine_start + get_num_all_sdma_engines(dqm));3650	     pipe++) {3651		for (queue = 0;3652		     queue < dqm->dev->kfd->device_info.num_sdma_queues_per_engine;3653		     queue++) {3654			r = dqm->dev->kfd2kgd->hqd_sdma_dump(3655				dqm->dev->adev, pipe, queue, &dump, &n_regs);3656			if (r)3657				break;3658 3659			seq_printf(m, "  SDMA Engine %d, RLC %d\n",3660				  pipe, queue);3661			seq_reg_dump(m, dump, n_regs);3662 3663			kfree(dump);3664		}3665	}3666 3667	return r;3668}3669 3670int dqm_debugfs_hang_hws(struct device_queue_manager *dqm)3671{3672	int r = 0;3673 3674	dqm_lock(dqm);3675	r = pm_debugfs_hang_hws(&dqm->packet_mgr);3676	if (r) {3677		dqm_unlock(dqm);3678		return r;3679	}3680	dqm->active_runlist = true;3681	r = execute_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES,3682				0, USE_DEFAULT_GRACE_PERIOD);3683	dqm_unlock(dqm);3684 3685	return r;3686}3687 3688#endif3689