brintos / linux-shallow public Read only

1/*2 * Copyright 2019 Advanced Micro Devices, Inc.3 *4 * Permission is hereby granted, free of charge, to any person obtaining a5 * copy of this software and associated documentation files (the "Software"),6 * to deal in the Software without restriction, including without limitation7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,8 * and/or sell copies of the Software, and to permit persons to whom the9 * Software is furnished to do so, subject to the following conditions:10 *11 * The above copyright notice and this permission notice shall be included in12 * all copies or substantial portions of the Software.13 *14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR20 * OTHER DEALINGS IN THE SOFTWARE.21 *22 */23 24#include <linux/firmware.h>25#include "amdgpu.h"26#include "amdgpu_vcn.h"27#include "amdgpu_pm.h"28#include "amdgpu_cs.h"29#include "soc15.h"30#include "soc15d.h"31#include "vcn_v2_0.h"32#include "mmsch_v3_0.h"33#include "vcn_sw_ring.h"34 35#include "vcn/vcn_3_0_0_offset.h"36#include "vcn/vcn_3_0_0_sh_mask.h"37#include "ivsrcid/vcn/irqsrcs_vcn_2_0.h"38 39#include <drm/drm_drv.h>40 41#define VCN_VID_SOC_ADDRESS_2_0					0x1fa0042#define VCN1_VID_SOC_ADDRESS_3_0				0x4820043 44#define mmUVD_CONTEXT_ID_INTERNAL_OFFSET			0x2745#define mmUVD_GPCOM_VCPU_CMD_INTERNAL_OFFSET			0x0f46#define mmUVD_GPCOM_VCPU_DATA0_INTERNAL_OFFSET			0x1047#define mmUVD_GPCOM_VCPU_DATA1_INTERNAL_OFFSET			0x1148#define mmUVD_NO_OP_INTERNAL_OFFSET				0x2949#define mmUVD_GP_SCRATCH8_INTERNAL_OFFSET			0x6650#define mmUVD_SCRATCH9_INTERNAL_OFFSET				0xc01d51 52#define mmUVD_LMI_RBC_IB_VMID_INTERNAL_OFFSET			0x43153#define mmUVD_LMI_RBC_IB_64BIT_BAR_LOW_INTERNAL_OFFSET		0x3b454#define mmUVD_LMI_RBC_IB_64BIT_BAR_HIGH_INTERNAL_OFFSET		0x3b555#define mmUVD_RBC_IB_SIZE_INTERNAL_OFFSET			0x25c56 57#define VCN_INSTANCES_SIENNA_CICHLID				258#define DEC_SW_RING_ENABLED					FALSE59 60#define RDECODE_MSG_CREATE					0x0000000061#define RDECODE_MESSAGE_CREATE					0x0000000162 63static const struct amdgpu_hwip_reg_entry vcn_reg_list_3_0[] = {64	SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_POWER_STATUS),65	SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_STATUS),66	SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_CONTEXT_ID),67	SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_CONTEXT_ID2),68	SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_GPCOM_VCPU_DATA0),69	SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_GPCOM_VCPU_DATA1),70	SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_GPCOM_VCPU_CMD),71	SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_RB_BASE_HI),72	SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_RB_BASE_LO),73	SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_RB_BASE_HI2),74	SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_RB_BASE_LO2),75	SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_RB_BASE_HI3),76	SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_RB_BASE_LO3),77	SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_RB_BASE_HI4),78	SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_RB_BASE_LO4),79	SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_RB_RPTR),80	SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_RB_WPTR),81	SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_RB_RPTR2),82	SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_RB_WPTR2),83	SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_RB_RPTR3),84	SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_RB_WPTR3),85	SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_RB_RPTR4),86	SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_RB_WPTR4),87	SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_RB_SIZE),88	SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_RB_SIZE2),89	SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_RB_SIZE3),90	SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_RB_SIZE4),91	SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_PGFSM_CONFIG),92	SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_PGFSM_STATUS),93	SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_DPG_LMA_CTL),94	SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_DPG_LMA_DATA),95	SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_DPG_LMA_MASK),96	SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_DPG_PAUSE)97};98 99static int amdgpu_ih_clientid_vcns[] = {100	SOC15_IH_CLIENTID_VCN,101	SOC15_IH_CLIENTID_VCN1102};103 104static int vcn_v3_0_start_sriov(struct amdgpu_device *adev);105static void vcn_v3_0_set_dec_ring_funcs(struct amdgpu_device *adev);106static void vcn_v3_0_set_enc_ring_funcs(struct amdgpu_device *adev);107static void vcn_v3_0_set_irq_funcs(struct amdgpu_device *adev);108static int vcn_v3_0_set_powergating_state(void *handle,109			enum amd_powergating_state state);110static int vcn_v3_0_pause_dpg_mode(struct amdgpu_device *adev,111			int inst_idx, struct dpg_pause_state *new_state);112 113static void vcn_v3_0_dec_ring_set_wptr(struct amdgpu_ring *ring);114static void vcn_v3_0_enc_ring_set_wptr(struct amdgpu_ring *ring);115 116/**117 * vcn_v3_0_early_init - set function pointers and load microcode118 *119 * @handle: amdgpu_device pointer120 *121 * Set ring and irq function pointers122 * Load microcode from filesystem123 */124static int vcn_v3_0_early_init(void *handle)125{126	struct amdgpu_device *adev = (struct amdgpu_device *)handle;127 128	if (amdgpu_sriov_vf(adev)) {129		adev->vcn.num_vcn_inst = VCN_INSTANCES_SIENNA_CICHLID;130		adev->vcn.harvest_config = 0;131		adev->vcn.num_enc_rings = 1;132 133	} else {134		if (adev->vcn.harvest_config == (AMDGPU_VCN_HARVEST_VCN0 |135						 AMDGPU_VCN_HARVEST_VCN1))136			/* both instances are harvested, disable the block */137			return -ENOENT;138 139		if (amdgpu_ip_version(adev, UVD_HWIP, 0) ==140		    IP_VERSION(3, 0, 33))141			adev->vcn.num_enc_rings = 0;142		else143			adev->vcn.num_enc_rings = 2;144	}145 146	vcn_v3_0_set_dec_ring_funcs(adev);147	vcn_v3_0_set_enc_ring_funcs(adev);148	vcn_v3_0_set_irq_funcs(adev);149 150	return amdgpu_vcn_early_init(adev);151}152 153/**154 * vcn_v3_0_sw_init - sw init for VCN block155 *156 * @handle: amdgpu_device pointer157 *158 * Load firmware and sw initialization159 */160static int vcn_v3_0_sw_init(void *handle)161{162	struct amdgpu_ring *ring;163	int i, j, r;164	int vcn_doorbell_index = 0;165	uint32_t reg_count = ARRAY_SIZE(vcn_reg_list_3_0);166	uint32_t *ptr;167	struct amdgpu_device *adev = (struct amdgpu_device *)handle;168 169	r = amdgpu_vcn_sw_init(adev);170	if (r)171		return r;172 173	amdgpu_vcn_setup_ucode(adev);174 175	r = amdgpu_vcn_resume(adev);176	if (r)177		return r;178 179	/*180	 * Note: doorbell assignment is fixed for SRIOV multiple VCN engines181	 * Formula:182	 *   vcn_db_base  = adev->doorbell_index.vcn.vcn_ring0_1 << 1;183	 *   dec_ring_i   = vcn_db_base + i * (adev->vcn.num_enc_rings + 1)184	 *   enc_ring_i,j = vcn_db_base + i * (adev->vcn.num_enc_rings + 1) + 1 + j185	 */186	if (amdgpu_sriov_vf(adev)) {187		vcn_doorbell_index = adev->doorbell_index.vcn.vcn_ring0_1;188		/* get DWORD offset */189		vcn_doorbell_index = vcn_doorbell_index << 1;190	}191 192	for (i = 0; i < adev->vcn.num_vcn_inst; i++) {193		volatile struct amdgpu_fw_shared *fw_shared;194 195		if (adev->vcn.harvest_config & (1 << i))196			continue;197 198		adev->vcn.internal.context_id = mmUVD_CONTEXT_ID_INTERNAL_OFFSET;199		adev->vcn.internal.ib_vmid = mmUVD_LMI_RBC_IB_VMID_INTERNAL_OFFSET;200		adev->vcn.internal.ib_bar_low = mmUVD_LMI_RBC_IB_64BIT_BAR_LOW_INTERNAL_OFFSET;201		adev->vcn.internal.ib_bar_high = mmUVD_LMI_RBC_IB_64BIT_BAR_HIGH_INTERNAL_OFFSET;202		adev->vcn.internal.ib_size = mmUVD_RBC_IB_SIZE_INTERNAL_OFFSET;203		adev->vcn.internal.gp_scratch8 = mmUVD_GP_SCRATCH8_INTERNAL_OFFSET;204 205		adev->vcn.internal.scratch9 = mmUVD_SCRATCH9_INTERNAL_OFFSET;206		adev->vcn.inst[i].external.scratch9 = SOC15_REG_OFFSET(VCN, i, mmUVD_SCRATCH9);207		adev->vcn.internal.data0 = mmUVD_GPCOM_VCPU_DATA0_INTERNAL_OFFSET;208		adev->vcn.inst[i].external.data0 = SOC15_REG_OFFSET(VCN, i, mmUVD_GPCOM_VCPU_DATA0);209		adev->vcn.internal.data1 = mmUVD_GPCOM_VCPU_DATA1_INTERNAL_OFFSET;210		adev->vcn.inst[i].external.data1 = SOC15_REG_OFFSET(VCN, i, mmUVD_GPCOM_VCPU_DATA1);211		adev->vcn.internal.cmd = mmUVD_GPCOM_VCPU_CMD_INTERNAL_OFFSET;212		adev->vcn.inst[i].external.cmd = SOC15_REG_OFFSET(VCN, i, mmUVD_GPCOM_VCPU_CMD);213		adev->vcn.internal.nop = mmUVD_NO_OP_INTERNAL_OFFSET;214		adev->vcn.inst[i].external.nop = SOC15_REG_OFFSET(VCN, i, mmUVD_NO_OP);215 216		/* VCN DEC TRAP */217		r = amdgpu_irq_add_id(adev, amdgpu_ih_clientid_vcns[i],218				VCN_2_0__SRCID__UVD_SYSTEM_MESSAGE_INTERRUPT, &adev->vcn.inst[i].irq);219		if (r)220			return r;221 222		atomic_set(&adev->vcn.inst[i].sched_score, 0);223 224		ring = &adev->vcn.inst[i].ring_dec;225		ring->use_doorbell = true;226		if (amdgpu_sriov_vf(adev)) {227			ring->doorbell_index = vcn_doorbell_index + i * (adev->vcn.num_enc_rings + 1);228		} else {229			ring->doorbell_index = (adev->doorbell_index.vcn.vcn_ring0_1 << 1) + 8 * i;230		}231		ring->vm_hub = AMDGPU_MMHUB0(0);232		sprintf(ring->name, "vcn_dec_%d", i);233		r = amdgpu_ring_init(adev, ring, 512, &adev->vcn.inst[i].irq, 0,234				     AMDGPU_RING_PRIO_DEFAULT,235				     &adev->vcn.inst[i].sched_score);236		if (r)237			return r;238 239		for (j = 0; j < adev->vcn.num_enc_rings; ++j) {240			enum amdgpu_ring_priority_level hw_prio = amdgpu_vcn_get_enc_ring_prio(j);241 242			/* VCN ENC TRAP */243			r = amdgpu_irq_add_id(adev, amdgpu_ih_clientid_vcns[i],244				j + VCN_2_0__SRCID__UVD_ENC_GENERAL_PURPOSE, &adev->vcn.inst[i].irq);245			if (r)246				return r;247 248			ring = &adev->vcn.inst[i].ring_enc[j];249			ring->use_doorbell = true;250			if (amdgpu_sriov_vf(adev)) {251				ring->doorbell_index = vcn_doorbell_index + i * (adev->vcn.num_enc_rings + 1) + 1 + j;252			} else {253				ring->doorbell_index = (adev->doorbell_index.vcn.vcn_ring0_1 << 1) + 2 + j + 8 * i;254			}255			ring->vm_hub = AMDGPU_MMHUB0(0);256			sprintf(ring->name, "vcn_enc_%d.%d", i, j);257			r = amdgpu_ring_init(adev, ring, 512, &adev->vcn.inst[i].irq, 0,258					     hw_prio, &adev->vcn.inst[i].sched_score);259			if (r)260				return r;261		}262 263		fw_shared = adev->vcn.inst[i].fw_shared.cpu_addr;264		fw_shared->present_flag_0 |= cpu_to_le32(AMDGPU_VCN_SW_RING_FLAG) |265					     cpu_to_le32(AMDGPU_VCN_MULTI_QUEUE_FLAG) |266					     cpu_to_le32(AMDGPU_VCN_FW_SHARED_FLAG_0_RB);267		fw_shared->sw_ring.is_enabled = cpu_to_le32(DEC_SW_RING_ENABLED);268		fw_shared->present_flag_0 |= AMDGPU_VCN_SMU_VERSION_INFO_FLAG;269		if (amdgpu_ip_version(adev, UVD_HWIP, 0) == IP_VERSION(3, 1, 2))270			fw_shared->smu_interface_info.smu_interface_type = 2;271		else if (amdgpu_ip_version(adev, UVD_HWIP, 0) ==272			 IP_VERSION(3, 1, 1))273			fw_shared->smu_interface_info.smu_interface_type = 1;274 275		if (amdgpu_vcnfw_log)276			amdgpu_vcn_fwlog_init(&adev->vcn.inst[i]);277	}278 279	if (amdgpu_sriov_vf(adev)) {280		r = amdgpu_virt_alloc_mm_table(adev);281		if (r)282			return r;283	}284	if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG)285		adev->vcn.pause_dpg_mode = vcn_v3_0_pause_dpg_mode;286 287	/* Allocate memory for VCN IP Dump buffer */288	ptr = kcalloc(adev->vcn.num_vcn_inst * reg_count, sizeof(uint32_t), GFP_KERNEL);289	if (ptr == NULL) {290		DRM_ERROR("Failed to allocate memory for VCN IP Dump\n");291		adev->vcn.ip_dump = NULL;292	} else {293		adev->vcn.ip_dump = ptr;294	}295 296	return 0;297}298 299/**300 * vcn_v3_0_sw_fini - sw fini for VCN block301 *302 * @handle: amdgpu_device pointer303 *304 * VCN suspend and free up sw allocation305 */306static int vcn_v3_0_sw_fini(void *handle)307{308	struct amdgpu_device *adev = (struct amdgpu_device *)handle;309	int i, r, idx;310 311	if (drm_dev_enter(adev_to_drm(adev), &idx)) {312		for (i = 0; i < adev->vcn.num_vcn_inst; i++) {313			volatile struct amdgpu_fw_shared *fw_shared;314 315			if (adev->vcn.harvest_config & (1 << i))316				continue;317			fw_shared = adev->vcn.inst[i].fw_shared.cpu_addr;318			fw_shared->present_flag_0 = 0;319			fw_shared->sw_ring.is_enabled = false;320		}321 322		drm_dev_exit(idx);323	}324 325	if (amdgpu_sriov_vf(adev))326		amdgpu_virt_free_mm_table(adev);327 328	r = amdgpu_vcn_suspend(adev);329	if (r)330		return r;331 332	r = amdgpu_vcn_sw_fini(adev);333 334	kfree(adev->vcn.ip_dump);335	return r;336}337 338/**339 * vcn_v3_0_hw_init - start and test VCN block340 *341 * @handle: amdgpu_device pointer342 *343 * Initialize the hardware, boot up the VCPU and do some testing344 */345static int vcn_v3_0_hw_init(void *handle)346{347	struct amdgpu_device *adev = (struct amdgpu_device *)handle;348	struct amdgpu_ring *ring;349	int i, j, r;350 351	if (amdgpu_sriov_vf(adev)) {352		r = vcn_v3_0_start_sriov(adev);353		if (r)354			return r;355 356		/* initialize VCN dec and enc ring buffers */357		for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {358			if (adev->vcn.harvest_config & (1 << i))359				continue;360 361			ring = &adev->vcn.inst[i].ring_dec;362			if (amdgpu_vcn_is_disabled_vcn(adev, VCN_DECODE_RING, i)) {363				ring->sched.ready = false;364				ring->no_scheduler = true;365				dev_info(adev->dev, "ring %s is disabled by hypervisor\n", ring->name);366			} else {367				ring->wptr = 0;368				ring->wptr_old = 0;369				vcn_v3_0_dec_ring_set_wptr(ring);370				ring->sched.ready = true;371			}372 373			for (j = 0; j < adev->vcn.num_enc_rings; ++j) {374				ring = &adev->vcn.inst[i].ring_enc[j];375				if (amdgpu_vcn_is_disabled_vcn(adev, VCN_ENCODE_RING, i)) {376					ring->sched.ready = false;377					ring->no_scheduler = true;378					dev_info(adev->dev, "ring %s is disabled by hypervisor\n", ring->name);379				} else {380					ring->wptr = 0;381					ring->wptr_old = 0;382					vcn_v3_0_enc_ring_set_wptr(ring);383					ring->sched.ready = true;384				}385			}386		}387	} else {388		for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {389			if (adev->vcn.harvest_config & (1 << i))390				continue;391 392			ring = &adev->vcn.inst[i].ring_dec;393 394			adev->nbio.funcs->vcn_doorbell_range(adev, ring->use_doorbell,395						     ring->doorbell_index, i);396 397			r = amdgpu_ring_test_helper(ring);398			if (r)399				return r;400 401			for (j = 0; j < adev->vcn.num_enc_rings; ++j) {402				ring = &adev->vcn.inst[i].ring_enc[j];403				r = amdgpu_ring_test_helper(ring);404				if (r)405					return r;406			}407		}408	}409 410	return 0;411}412 413/**414 * vcn_v3_0_hw_fini - stop the hardware block415 *416 * @handle: amdgpu_device pointer417 *418 * Stop the VCN block, mark ring as not ready any more419 */420static int vcn_v3_0_hw_fini(void *handle)421{422	struct amdgpu_device *adev = (struct amdgpu_device *)handle;423	int i;424 425	cancel_delayed_work_sync(&adev->vcn.idle_work);426 427	for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {428		if (adev->vcn.harvest_config & (1 << i))429			continue;430 431		if (!amdgpu_sriov_vf(adev)) {432			if ((adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG) ||433					(adev->vcn.cur_state != AMD_PG_STATE_GATE &&434					 RREG32_SOC15(VCN, i, mmUVD_STATUS))) {435				vcn_v3_0_set_powergating_state(adev, AMD_PG_STATE_GATE);436			}437		}438	}439 440	return 0;441}442 443/**444 * vcn_v3_0_suspend - suspend VCN block445 *446 * @handle: amdgpu_device pointer447 *448 * HW fini and suspend VCN block449 */450static int vcn_v3_0_suspend(void *handle)451{452	int r;453	struct amdgpu_device *adev = (struct amdgpu_device *)handle;454 455	r = vcn_v3_0_hw_fini(adev);456	if (r)457		return r;458 459	r = amdgpu_vcn_suspend(adev);460 461	return r;462}463 464/**465 * vcn_v3_0_resume - resume VCN block466 *467 * @handle: amdgpu_device pointer468 *469 * Resume firmware and hw init VCN block470 */471static int vcn_v3_0_resume(void *handle)472{473	int r;474	struct amdgpu_device *adev = (struct amdgpu_device *)handle;475 476	r = amdgpu_vcn_resume(adev);477	if (r)478		return r;479 480	r = vcn_v3_0_hw_init(adev);481 482	return r;483}484 485/**486 * vcn_v3_0_mc_resume - memory controller programming487 *488 * @adev: amdgpu_device pointer489 * @inst: instance number490 *491 * Let the VCN memory controller know it's offsets492 */493static void vcn_v3_0_mc_resume(struct amdgpu_device *adev, int inst)494{495	uint32_t size = AMDGPU_GPU_PAGE_ALIGN(adev->vcn.fw[inst]->size + 4);496	uint32_t offset;497 498	/* cache window 0: fw */499	if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {500		WREG32_SOC15(VCN, inst, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW,501			(adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + inst].tmr_mc_addr_lo));502		WREG32_SOC15(VCN, inst, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH,503			(adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + inst].tmr_mc_addr_hi));504		WREG32_SOC15(VCN, inst, mmUVD_VCPU_CACHE_OFFSET0, 0);505		offset = 0;506	} else {507		WREG32_SOC15(VCN, inst, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW,508			lower_32_bits(adev->vcn.inst[inst].gpu_addr));509		WREG32_SOC15(VCN, inst, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH,510			upper_32_bits(adev->vcn.inst[inst].gpu_addr));511		offset = size;512		WREG32_SOC15(VCN, inst, mmUVD_VCPU_CACHE_OFFSET0,513			AMDGPU_UVD_FIRMWARE_OFFSET >> 3);514	}515	WREG32_SOC15(VCN, inst, mmUVD_VCPU_CACHE_SIZE0, size);516 517	/* cache window 1: stack */518	WREG32_SOC15(VCN, inst, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW,519		lower_32_bits(adev->vcn.inst[inst].gpu_addr + offset));520	WREG32_SOC15(VCN, inst, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH,521		upper_32_bits(adev->vcn.inst[inst].gpu_addr + offset));522	WREG32_SOC15(VCN, inst, mmUVD_VCPU_CACHE_OFFSET1, 0);523	WREG32_SOC15(VCN, inst, mmUVD_VCPU_CACHE_SIZE1, AMDGPU_VCN_STACK_SIZE);524 525	/* cache window 2: context */526	WREG32_SOC15(VCN, inst, mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_LOW,527		lower_32_bits(adev->vcn.inst[inst].gpu_addr + offset + AMDGPU_VCN_STACK_SIZE));528	WREG32_SOC15(VCN, inst, mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_HIGH,529		upper_32_bits(adev->vcn.inst[inst].gpu_addr + offset + AMDGPU_VCN_STACK_SIZE));530	WREG32_SOC15(VCN, inst, mmUVD_VCPU_CACHE_OFFSET2, 0);531	WREG32_SOC15(VCN, inst, mmUVD_VCPU_CACHE_SIZE2, AMDGPU_VCN_CONTEXT_SIZE);532 533	/* non-cache window */534	WREG32_SOC15(VCN, inst, mmUVD_LMI_VCPU_NC0_64BIT_BAR_LOW,535		lower_32_bits(adev->vcn.inst[inst].fw_shared.gpu_addr));536	WREG32_SOC15(VCN, inst, mmUVD_LMI_VCPU_NC0_64BIT_BAR_HIGH,537		upper_32_bits(adev->vcn.inst[inst].fw_shared.gpu_addr));538	WREG32_SOC15(VCN, inst, mmUVD_VCPU_NONCACHE_OFFSET0, 0);539	WREG32_SOC15(VCN, inst, mmUVD_VCPU_NONCACHE_SIZE0,540		AMDGPU_GPU_PAGE_ALIGN(sizeof(struct amdgpu_fw_shared)));541}542 543static void vcn_v3_0_mc_resume_dpg_mode(struct amdgpu_device *adev, int inst_idx, bool indirect)544{545	uint32_t size = AMDGPU_GPU_PAGE_ALIGN(adev->vcn.fw[inst_idx]->size + 4);546	uint32_t offset;547 548	/* cache window 0: fw */549	if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {550		if (!indirect) {551			WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(552				VCN, inst_idx, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW),553				(adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + inst_idx].tmr_mc_addr_lo), 0, indirect);554			WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(555				VCN, inst_idx, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH),556				(adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + inst_idx].tmr_mc_addr_hi), 0, indirect);557			WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(558				VCN, inst_idx, mmUVD_VCPU_CACHE_OFFSET0), 0, 0, indirect);559		} else {560			WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(561				VCN, inst_idx, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW), 0, 0, indirect);562			WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(563				VCN, inst_idx, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH), 0, 0, indirect);564			WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(565				VCN, inst_idx, mmUVD_VCPU_CACHE_OFFSET0), 0, 0, indirect);566		}567		offset = 0;568	} else {569		WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(570			VCN, inst_idx, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW),571			lower_32_bits(adev->vcn.inst[inst_idx].gpu_addr), 0, indirect);572		WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(573			VCN, inst_idx, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH),574			upper_32_bits(adev->vcn.inst[inst_idx].gpu_addr), 0, indirect);575		offset = size;576		WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(577			VCN, inst_idx, mmUVD_VCPU_CACHE_OFFSET0),578			AMDGPU_UVD_FIRMWARE_OFFSET >> 3, 0, indirect);579	}580 581	if (!indirect)582		WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(583			VCN, inst_idx, mmUVD_VCPU_CACHE_SIZE0), size, 0, indirect);584	else585		WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(586			VCN, inst_idx, mmUVD_VCPU_CACHE_SIZE0), 0, 0, indirect);587 588	/* cache window 1: stack */589	if (!indirect) {590		WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(591			VCN, inst_idx, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW),592			lower_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset), 0, indirect);593		WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(594			VCN, inst_idx, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH),595			upper_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset), 0, indirect);596		WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(597			VCN, inst_idx, mmUVD_VCPU_CACHE_OFFSET1), 0, 0, indirect);598	} else {599		WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(600			VCN, inst_idx, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW), 0, 0, indirect);601		WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(602			VCN, inst_idx, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH), 0, 0, indirect);603		WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(604			VCN, inst_idx, mmUVD_VCPU_CACHE_OFFSET1), 0, 0, indirect);605	}606	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(607			VCN, inst_idx, mmUVD_VCPU_CACHE_SIZE1), AMDGPU_VCN_STACK_SIZE, 0, indirect);608 609	/* cache window 2: context */610	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(611			VCN, inst_idx, mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_LOW),612			lower_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset + AMDGPU_VCN_STACK_SIZE), 0, indirect);613	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(614			VCN, inst_idx, mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_HIGH),615			upper_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset + AMDGPU_VCN_STACK_SIZE), 0, indirect);616	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(617			VCN, inst_idx, mmUVD_VCPU_CACHE_OFFSET2), 0, 0, indirect);618	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(619			VCN, inst_idx, mmUVD_VCPU_CACHE_SIZE2), AMDGPU_VCN_CONTEXT_SIZE, 0, indirect);620 621	/* non-cache window */622	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(623			VCN, inst_idx, mmUVD_LMI_VCPU_NC0_64BIT_BAR_LOW),624			lower_32_bits(adev->vcn.inst[inst_idx].fw_shared.gpu_addr), 0, indirect);625	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(626			VCN, inst_idx, mmUVD_LMI_VCPU_NC0_64BIT_BAR_HIGH),627			upper_32_bits(adev->vcn.inst[inst_idx].fw_shared.gpu_addr), 0, indirect);628	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(629			VCN, inst_idx, mmUVD_VCPU_NONCACHE_OFFSET0), 0, 0, indirect);630	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(631			VCN, inst_idx, mmUVD_VCPU_NONCACHE_SIZE0),632			AMDGPU_GPU_PAGE_ALIGN(sizeof(struct amdgpu_fw_shared)), 0, indirect);633 634	/* VCN global tiling registers */635	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(636		UVD, inst_idx, mmUVD_GFX10_ADDR_CONFIG), adev->gfx.config.gb_addr_config, 0, indirect);637}638 639static void vcn_v3_0_disable_static_power_gating(struct amdgpu_device *adev, int inst)640{641	uint32_t data = 0;642 643	if (adev->pg_flags & AMD_PG_SUPPORT_VCN) {644		data = (1 << UVD_PGFSM_CONFIG__UVDM_PWR_CONFIG__SHIFT645			| 1 << UVD_PGFSM_CONFIG__UVDU_PWR_CONFIG__SHIFT646			| 2 << UVD_PGFSM_CONFIG__UVDF_PWR_CONFIG__SHIFT647			| 2 << UVD_PGFSM_CONFIG__UVDC_PWR_CONFIG__SHIFT648			| 2 << UVD_PGFSM_CONFIG__UVDB_PWR_CONFIG__SHIFT649			| 2 << UVD_PGFSM_CONFIG__UVDIRL_PWR_CONFIG__SHIFT650			| 1 << UVD_PGFSM_CONFIG__UVDLM_PWR_CONFIG__SHIFT651			| 2 << UVD_PGFSM_CONFIG__UVDTD_PWR_CONFIG__SHIFT652			| 2 << UVD_PGFSM_CONFIG__UVDTE_PWR_CONFIG__SHIFT653			| 2 << UVD_PGFSM_CONFIG__UVDE_PWR_CONFIG__SHIFT654			| 2 << UVD_PGFSM_CONFIG__UVDAB_PWR_CONFIG__SHIFT655			| 2 << UVD_PGFSM_CONFIG__UVDATD_PWR_CONFIG__SHIFT656			| 2 << UVD_PGFSM_CONFIG__UVDNA_PWR_CONFIG__SHIFT657			| 2 << UVD_PGFSM_CONFIG__UVDNB_PWR_CONFIG__SHIFT);658 659		WREG32_SOC15(VCN, inst, mmUVD_PGFSM_CONFIG, data);660		SOC15_WAIT_ON_RREG(VCN, inst, mmUVD_PGFSM_STATUS,661			UVD_PGFSM_STATUS__UVDM_UVDU_UVDLM_PWR_ON_3_0, 0x3F3FFFFF);662	} else {663		data = (1 << UVD_PGFSM_CONFIG__UVDM_PWR_CONFIG__SHIFT664			| 1 << UVD_PGFSM_CONFIG__UVDU_PWR_CONFIG__SHIFT665			| 1 << UVD_PGFSM_CONFIG__UVDF_PWR_CONFIG__SHIFT666			| 1 << UVD_PGFSM_CONFIG__UVDC_PWR_CONFIG__SHIFT667			| 1 << UVD_PGFSM_CONFIG__UVDB_PWR_CONFIG__SHIFT668			| 1 << UVD_PGFSM_CONFIG__UVDIRL_PWR_CONFIG__SHIFT669			| 1 << UVD_PGFSM_CONFIG__UVDLM_PWR_CONFIG__SHIFT670			| 1 << UVD_PGFSM_CONFIG__UVDTD_PWR_CONFIG__SHIFT671			| 1 << UVD_PGFSM_CONFIG__UVDTE_PWR_CONFIG__SHIFT672			| 1 << UVD_PGFSM_CONFIG__UVDE_PWR_CONFIG__SHIFT673			| 1 << UVD_PGFSM_CONFIG__UVDAB_PWR_CONFIG__SHIFT674			| 1 << UVD_PGFSM_CONFIG__UVDATD_PWR_CONFIG__SHIFT675			| 1 << UVD_PGFSM_CONFIG__UVDNA_PWR_CONFIG__SHIFT676			| 1 << UVD_PGFSM_CONFIG__UVDNB_PWR_CONFIG__SHIFT);677		WREG32_SOC15(VCN, inst, mmUVD_PGFSM_CONFIG, data);678		SOC15_WAIT_ON_RREG(VCN, inst, mmUVD_PGFSM_STATUS, 0,  0x3F3FFFFF);679	}680 681	data = RREG32_SOC15(VCN, inst, mmUVD_POWER_STATUS);682	data &= ~0x103;683	if (adev->pg_flags & AMD_PG_SUPPORT_VCN)684		data |= UVD_PGFSM_CONFIG__UVDM_UVDU_PWR_ON |685			UVD_POWER_STATUS__UVD_PG_EN_MASK;686 687	WREG32_SOC15(VCN, inst, mmUVD_POWER_STATUS, data);688}689 690static void vcn_v3_0_enable_static_power_gating(struct amdgpu_device *adev, int inst)691{692	uint32_t data;693 694	if (adev->pg_flags & AMD_PG_SUPPORT_VCN) {695		/* Before power off, this indicator has to be turned on */696		data = RREG32_SOC15(VCN, inst, mmUVD_POWER_STATUS);697		data &= ~UVD_POWER_STATUS__UVD_POWER_STATUS_MASK;698		data |= UVD_POWER_STATUS__UVD_POWER_STATUS_TILES_OFF;699		WREG32_SOC15(VCN, inst, mmUVD_POWER_STATUS, data);700 701		data = (2 << UVD_PGFSM_CONFIG__UVDM_PWR_CONFIG__SHIFT702			| 2 << UVD_PGFSM_CONFIG__UVDU_PWR_CONFIG__SHIFT703			| 2 << UVD_PGFSM_CONFIG__UVDF_PWR_CONFIG__SHIFT704			| 2 << UVD_PGFSM_CONFIG__UVDC_PWR_CONFIG__SHIFT705			| 2 << UVD_PGFSM_CONFIG__UVDB_PWR_CONFIG__SHIFT706			| 2 << UVD_PGFSM_CONFIG__UVDIRL_PWR_CONFIG__SHIFT707			| 2 << UVD_PGFSM_CONFIG__UVDLM_PWR_CONFIG__SHIFT708			| 2 << UVD_PGFSM_CONFIG__UVDTD_PWR_CONFIG__SHIFT709			| 2 << UVD_PGFSM_CONFIG__UVDTE_PWR_CONFIG__SHIFT710			| 2 << UVD_PGFSM_CONFIG__UVDE_PWR_CONFIG__SHIFT711			| 2 << UVD_PGFSM_CONFIG__UVDAB_PWR_CONFIG__SHIFT712			| 2 << UVD_PGFSM_CONFIG__UVDATD_PWR_CONFIG__SHIFT713			| 2 << UVD_PGFSM_CONFIG__UVDNA_PWR_CONFIG__SHIFT714			| 2 << UVD_PGFSM_CONFIG__UVDNB_PWR_CONFIG__SHIFT);715		WREG32_SOC15(VCN, inst, mmUVD_PGFSM_CONFIG, data);716 717		data = (2 << UVD_PGFSM_STATUS__UVDM_PWR_STATUS__SHIFT718			| 2 << UVD_PGFSM_STATUS__UVDU_PWR_STATUS__SHIFT719			| 2 << UVD_PGFSM_STATUS__UVDF_PWR_STATUS__SHIFT720			| 2 << UVD_PGFSM_STATUS__UVDC_PWR_STATUS__SHIFT721			| 2 << UVD_PGFSM_STATUS__UVDB_PWR_STATUS__SHIFT722			| 2 << UVD_PGFSM_STATUS__UVDIRL_PWR_STATUS__SHIFT723			| 2 << UVD_PGFSM_STATUS__UVDLM_PWR_STATUS__SHIFT724			| 2 << UVD_PGFSM_STATUS__UVDTD_PWR_STATUS__SHIFT725			| 2 << UVD_PGFSM_STATUS__UVDTE_PWR_STATUS__SHIFT726			| 2 << UVD_PGFSM_STATUS__UVDE_PWR_STATUS__SHIFT727			| 2 << UVD_PGFSM_STATUS__UVDAB_PWR_STATUS__SHIFT728			| 2 << UVD_PGFSM_STATUS__UVDATD_PWR_STATUS__SHIFT729			| 2 << UVD_PGFSM_STATUS__UVDNA_PWR_STATUS__SHIFT730			| 2 << UVD_PGFSM_STATUS__UVDNB_PWR_STATUS__SHIFT);731		SOC15_WAIT_ON_RREG(VCN, inst, mmUVD_PGFSM_STATUS, data, 0x3F3FFFFF);732	}733}734 735/**736 * vcn_v3_0_disable_clock_gating - disable VCN clock gating737 *738 * @adev: amdgpu_device pointer739 * @inst: instance number740 *741 * Disable clock gating for VCN block742 */743static void vcn_v3_0_disable_clock_gating(struct amdgpu_device *adev, int inst)744{745	uint32_t data;746 747	/* VCN disable CGC */748	data = RREG32_SOC15(VCN, inst, mmUVD_CGC_CTRL);749	if (adev->cg_flags & AMD_CG_SUPPORT_VCN_MGCG)750		data |= 1 << UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;751	else752		data &= ~UVD_CGC_CTRL__DYN_CLOCK_MODE_MASK;753	data |= 1 << UVD_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT;754	data |= 4 << UVD_CGC_CTRL__CLK_OFF_DELAY__SHIFT;755	WREG32_SOC15(VCN, inst, mmUVD_CGC_CTRL, data);756 757	data = RREG32_SOC15(VCN, inst, mmUVD_CGC_GATE);758	data &= ~(UVD_CGC_GATE__SYS_MASK759		| UVD_CGC_GATE__UDEC_MASK760		| UVD_CGC_GATE__MPEG2_MASK761		| UVD_CGC_GATE__REGS_MASK762		| UVD_CGC_GATE__RBC_MASK763		| UVD_CGC_GATE__LMI_MC_MASK764		| UVD_CGC_GATE__LMI_UMC_MASK765		| UVD_CGC_GATE__IDCT_MASK766		| UVD_CGC_GATE__MPRD_MASK767		| UVD_CGC_GATE__MPC_MASK768		| UVD_CGC_GATE__LBSI_MASK769		| UVD_CGC_GATE__LRBBM_MASK770		| UVD_CGC_GATE__UDEC_RE_MASK771		| UVD_CGC_GATE__UDEC_CM_MASK772		| UVD_CGC_GATE__UDEC_IT_MASK773		| UVD_CGC_GATE__UDEC_DB_MASK774		| UVD_CGC_GATE__UDEC_MP_MASK775		| UVD_CGC_GATE__WCB_MASK776		| UVD_CGC_GATE__VCPU_MASK777		| UVD_CGC_GATE__MMSCH_MASK);778 779	WREG32_SOC15(VCN, inst, mmUVD_CGC_GATE, data);780 781	SOC15_WAIT_ON_RREG(VCN, inst, mmUVD_CGC_GATE, 0,  0xFFFFFFFF);782 783	data = RREG32_SOC15(VCN, inst, mmUVD_CGC_CTRL);784	data &= ~(UVD_CGC_CTRL__UDEC_RE_MODE_MASK785		| UVD_CGC_CTRL__UDEC_CM_MODE_MASK786		| UVD_CGC_CTRL__UDEC_IT_MODE_MASK787		| UVD_CGC_CTRL__UDEC_DB_MODE_MASK788		| UVD_CGC_CTRL__UDEC_MP_MODE_MASK789		| UVD_CGC_CTRL__SYS_MODE_MASK790		| UVD_CGC_CTRL__UDEC_MODE_MASK791		| UVD_CGC_CTRL__MPEG2_MODE_MASK792		| UVD_CGC_CTRL__REGS_MODE_MASK793		| UVD_CGC_CTRL__RBC_MODE_MASK794		| UVD_CGC_CTRL__LMI_MC_MODE_MASK795		| UVD_CGC_CTRL__LMI_UMC_MODE_MASK796		| UVD_CGC_CTRL__IDCT_MODE_MASK797		| UVD_CGC_CTRL__MPRD_MODE_MASK798		| UVD_CGC_CTRL__MPC_MODE_MASK799		| UVD_CGC_CTRL__LBSI_MODE_MASK800		| UVD_CGC_CTRL__LRBBM_MODE_MASK801		| UVD_CGC_CTRL__WCB_MODE_MASK802		| UVD_CGC_CTRL__VCPU_MODE_MASK803		| UVD_CGC_CTRL__MMSCH_MODE_MASK);804	WREG32_SOC15(VCN, inst, mmUVD_CGC_CTRL, data);805 806	data = RREG32_SOC15(VCN, inst, mmUVD_SUVD_CGC_GATE);807	data |= (UVD_SUVD_CGC_GATE__SRE_MASK808		| UVD_SUVD_CGC_GATE__SIT_MASK809		| UVD_SUVD_CGC_GATE__SMP_MASK810		| UVD_SUVD_CGC_GATE__SCM_MASK811		| UVD_SUVD_CGC_GATE__SDB_MASK812		| UVD_SUVD_CGC_GATE__SRE_H264_MASK813		| UVD_SUVD_CGC_GATE__SRE_HEVC_MASK814		| UVD_SUVD_CGC_GATE__SIT_H264_MASK815		| UVD_SUVD_CGC_GATE__SIT_HEVC_MASK816		| UVD_SUVD_CGC_GATE__SCM_H264_MASK817		| UVD_SUVD_CGC_GATE__SCM_HEVC_MASK818		| UVD_SUVD_CGC_GATE__SDB_H264_MASK819		| UVD_SUVD_CGC_GATE__SDB_HEVC_MASK820		| UVD_SUVD_CGC_GATE__SCLR_MASK821		| UVD_SUVD_CGC_GATE__ENT_MASK822		| UVD_SUVD_CGC_GATE__IME_MASK823		| UVD_SUVD_CGC_GATE__SIT_HEVC_DEC_MASK824		| UVD_SUVD_CGC_GATE__SIT_HEVC_ENC_MASK825		| UVD_SUVD_CGC_GATE__SITE_MASK826		| UVD_SUVD_CGC_GATE__SRE_VP9_MASK827		| UVD_SUVD_CGC_GATE__SCM_VP9_MASK828		| UVD_SUVD_CGC_GATE__SIT_VP9_DEC_MASK829		| UVD_SUVD_CGC_GATE__SDB_VP9_MASK830		| UVD_SUVD_CGC_GATE__IME_HEVC_MASK831		| UVD_SUVD_CGC_GATE__EFC_MASK832		| UVD_SUVD_CGC_GATE__SAOE_MASK833		| UVD_SUVD_CGC_GATE__SRE_AV1_MASK834		| UVD_SUVD_CGC_GATE__FBC_PCLK_MASK835		| UVD_SUVD_CGC_GATE__FBC_CCLK_MASK836		| UVD_SUVD_CGC_GATE__SCM_AV1_MASK837		| UVD_SUVD_CGC_GATE__SMPA_MASK);838	WREG32_SOC15(VCN, inst, mmUVD_SUVD_CGC_GATE, data);839 840	data = RREG32_SOC15(VCN, inst, mmUVD_SUVD_CGC_GATE2);841	data |= (UVD_SUVD_CGC_GATE2__MPBE0_MASK842		| UVD_SUVD_CGC_GATE2__MPBE1_MASK843		| UVD_SUVD_CGC_GATE2__SIT_AV1_MASK844		| UVD_SUVD_CGC_GATE2__SDB_AV1_MASK845		| UVD_SUVD_CGC_GATE2__MPC1_MASK);846	WREG32_SOC15(VCN, inst, mmUVD_SUVD_CGC_GATE2, data);847 848	data = RREG32_SOC15(VCN, inst, mmUVD_SUVD_CGC_CTRL);849	data &= ~(UVD_SUVD_CGC_CTRL__SRE_MODE_MASK850		| UVD_SUVD_CGC_CTRL__SIT_MODE_MASK851		| UVD_SUVD_CGC_CTRL__SMP_MODE_MASK852		| UVD_SUVD_CGC_CTRL__SCM_MODE_MASK853		| UVD_SUVD_CGC_CTRL__SDB_MODE_MASK854		| UVD_SUVD_CGC_CTRL__SCLR_MODE_MASK855		| UVD_SUVD_CGC_CTRL__ENT_MODE_MASK856		| UVD_SUVD_CGC_CTRL__IME_MODE_MASK857		| UVD_SUVD_CGC_CTRL__SITE_MODE_MASK858		| UVD_SUVD_CGC_CTRL__EFC_MODE_MASK859		| UVD_SUVD_CGC_CTRL__SAOE_MODE_MASK860		| UVD_SUVD_CGC_CTRL__SMPA_MODE_MASK861		| UVD_SUVD_CGC_CTRL__MPBE0_MODE_MASK862		| UVD_SUVD_CGC_CTRL__MPBE1_MODE_MASK863		| UVD_SUVD_CGC_CTRL__SIT_AV1_MODE_MASK864		| UVD_SUVD_CGC_CTRL__SDB_AV1_MODE_MASK865		| UVD_SUVD_CGC_CTRL__MPC1_MODE_MASK866		| UVD_SUVD_CGC_CTRL__FBC_PCLK_MASK867		| UVD_SUVD_CGC_CTRL__FBC_CCLK_MASK);868	WREG32_SOC15(VCN, inst, mmUVD_SUVD_CGC_CTRL, data);869}870 871static void vcn_v3_0_clock_gating_dpg_mode(struct amdgpu_device *adev,872		uint8_t sram_sel, int inst_idx, uint8_t indirect)873{874	uint32_t reg_data = 0;875 876	/* enable sw clock gating control */877	if (adev->cg_flags & AMD_CG_SUPPORT_VCN_MGCG)878		reg_data = 1 << UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;879	else880		reg_data = 0 << UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;881	reg_data |= 1 << UVD_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT;882	reg_data |= 4 << UVD_CGC_CTRL__CLK_OFF_DELAY__SHIFT;883	reg_data &= ~(UVD_CGC_CTRL__UDEC_RE_MODE_MASK |884		 UVD_CGC_CTRL__UDEC_CM_MODE_MASK |885		 UVD_CGC_CTRL__UDEC_IT_MODE_MASK |886		 UVD_CGC_CTRL__UDEC_DB_MODE_MASK |887		 UVD_CGC_CTRL__UDEC_MP_MODE_MASK |888		 UVD_CGC_CTRL__SYS_MODE_MASK |889		 UVD_CGC_CTRL__UDEC_MODE_MASK |890		 UVD_CGC_CTRL__MPEG2_MODE_MASK |891		 UVD_CGC_CTRL__REGS_MODE_MASK |892		 UVD_CGC_CTRL__RBC_MODE_MASK |893		 UVD_CGC_CTRL__LMI_MC_MODE_MASK |894		 UVD_CGC_CTRL__LMI_UMC_MODE_MASK |895		 UVD_CGC_CTRL__IDCT_MODE_MASK |896		 UVD_CGC_CTRL__MPRD_MODE_MASK |897		 UVD_CGC_CTRL__MPC_MODE_MASK |898		 UVD_CGC_CTRL__LBSI_MODE_MASK |899		 UVD_CGC_CTRL__LRBBM_MODE_MASK |900		 UVD_CGC_CTRL__WCB_MODE_MASK |901		 UVD_CGC_CTRL__VCPU_MODE_MASK |902		 UVD_CGC_CTRL__MMSCH_MODE_MASK);903	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(904		VCN, inst_idx, mmUVD_CGC_CTRL), reg_data, sram_sel, indirect);905 906	/* turn off clock gating */907	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(908		VCN, inst_idx, mmUVD_CGC_GATE), 0, sram_sel, indirect);909 910	/* turn on SUVD clock gating */911	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(912		VCN, inst_idx, mmUVD_SUVD_CGC_GATE), 1, sram_sel, indirect);913 914	/* turn on sw mode in UVD_SUVD_CGC_CTRL */915	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(916		VCN, inst_idx, mmUVD_SUVD_CGC_CTRL), 0, sram_sel, indirect);917}918 919/**920 * vcn_v3_0_enable_clock_gating - enable VCN clock gating921 *922 * @adev: amdgpu_device pointer923 * @inst: instance number924 *925 * Enable clock gating for VCN block926 */927static void vcn_v3_0_enable_clock_gating(struct amdgpu_device *adev, int inst)928{929	uint32_t data;930 931	/* enable VCN CGC */932	data = RREG32_SOC15(VCN, inst, mmUVD_CGC_CTRL);933	if (adev->cg_flags & AMD_CG_SUPPORT_VCN_MGCG)934		data |= 1 << UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;935	else936		data |= 0 << UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;937	data |= 1 << UVD_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT;938	data |= 4 << UVD_CGC_CTRL__CLK_OFF_DELAY__SHIFT;939	WREG32_SOC15(VCN, inst, mmUVD_CGC_CTRL, data);940 941	data = RREG32_SOC15(VCN, inst, mmUVD_CGC_CTRL);942	data |= (UVD_CGC_CTRL__UDEC_RE_MODE_MASK943		| UVD_CGC_CTRL__UDEC_CM_MODE_MASK944		| UVD_CGC_CTRL__UDEC_IT_MODE_MASK945		| UVD_CGC_CTRL__UDEC_DB_MODE_MASK946		| UVD_CGC_CTRL__UDEC_MP_MODE_MASK947		| UVD_CGC_CTRL__SYS_MODE_MASK948		| UVD_CGC_CTRL__UDEC_MODE_MASK949		| UVD_CGC_CTRL__MPEG2_MODE_MASK950		| UVD_CGC_CTRL__REGS_MODE_MASK951		| UVD_CGC_CTRL__RBC_MODE_MASK952		| UVD_CGC_CTRL__LMI_MC_MODE_MASK953		| UVD_CGC_CTRL__LMI_UMC_MODE_MASK954		| UVD_CGC_CTRL__IDCT_MODE_MASK955		| UVD_CGC_CTRL__MPRD_MODE_MASK956		| UVD_CGC_CTRL__MPC_MODE_MASK957		| UVD_CGC_CTRL__LBSI_MODE_MASK958		| UVD_CGC_CTRL__LRBBM_MODE_MASK959		| UVD_CGC_CTRL__WCB_MODE_MASK960		| UVD_CGC_CTRL__VCPU_MODE_MASK961		| UVD_CGC_CTRL__MMSCH_MODE_MASK);962	WREG32_SOC15(VCN, inst, mmUVD_CGC_CTRL, data);963 964	data = RREG32_SOC15(VCN, inst, mmUVD_SUVD_CGC_CTRL);965	data |= (UVD_SUVD_CGC_CTRL__SRE_MODE_MASK966		| UVD_SUVD_CGC_CTRL__SIT_MODE_MASK967		| UVD_SUVD_CGC_CTRL__SMP_MODE_MASK968		| UVD_SUVD_CGC_CTRL__SCM_MODE_MASK969		| UVD_SUVD_CGC_CTRL__SDB_MODE_MASK970		| UVD_SUVD_CGC_CTRL__SCLR_MODE_MASK971		| UVD_SUVD_CGC_CTRL__ENT_MODE_MASK972		| UVD_SUVD_CGC_CTRL__IME_MODE_MASK973		| UVD_SUVD_CGC_CTRL__SITE_MODE_MASK974		| UVD_SUVD_CGC_CTRL__EFC_MODE_MASK975		| UVD_SUVD_CGC_CTRL__SAOE_MODE_MASK976		| UVD_SUVD_CGC_CTRL__SMPA_MODE_MASK977		| UVD_SUVD_CGC_CTRL__MPBE0_MODE_MASK978		| UVD_SUVD_CGC_CTRL__MPBE1_MODE_MASK979		| UVD_SUVD_CGC_CTRL__SIT_AV1_MODE_MASK980		| UVD_SUVD_CGC_CTRL__SDB_AV1_MODE_MASK981		| UVD_SUVD_CGC_CTRL__MPC1_MODE_MASK982		| UVD_SUVD_CGC_CTRL__FBC_PCLK_MASK983		| UVD_SUVD_CGC_CTRL__FBC_CCLK_MASK);984	WREG32_SOC15(VCN, inst, mmUVD_SUVD_CGC_CTRL, data);985}986 987static int vcn_v3_0_start_dpg_mode(struct amdgpu_device *adev, int inst_idx, bool indirect)988{989	volatile struct amdgpu_fw_shared *fw_shared = adev->vcn.inst[inst_idx].fw_shared.cpu_addr;990	struct amdgpu_ring *ring;991	uint32_t rb_bufsz, tmp;992 993	/* disable register anti-hang mechanism */994	WREG32_P(SOC15_REG_OFFSET(VCN, inst_idx, mmUVD_POWER_STATUS), 1,995		~UVD_POWER_STATUS__UVD_POWER_STATUS_MASK);996	/* enable dynamic power gating mode */997	tmp = RREG32_SOC15(VCN, inst_idx, mmUVD_POWER_STATUS);998	tmp |= UVD_POWER_STATUS__UVD_PG_MODE_MASK;999	tmp |= UVD_POWER_STATUS__UVD_PG_EN_MASK;1000	WREG32_SOC15(VCN, inst_idx, mmUVD_POWER_STATUS, tmp);1001 1002	if (indirect)1003		adev->vcn.inst[inst_idx].dpg_sram_curr_addr = (uint32_t *)adev->vcn.inst[inst_idx].dpg_sram_cpu_addr;1004 1005	/* enable clock gating */1006	vcn_v3_0_clock_gating_dpg_mode(adev, 0, inst_idx, indirect);1007 1008	/* enable VCPU clock */1009	tmp = (0xFF << UVD_VCPU_CNTL__PRB_TIMEOUT_VAL__SHIFT);1010	tmp |= UVD_VCPU_CNTL__CLK_EN_MASK;1011	tmp |= UVD_VCPU_CNTL__BLK_RST_MASK;1012	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(1013		VCN, inst_idx, mmUVD_VCPU_CNTL), tmp, 0, indirect);1014 1015	/* disable master interupt */1016	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(1017		VCN, inst_idx, mmUVD_MASTINT_EN), 0, 0, indirect);1018 1019	/* setup mmUVD_LMI_CTRL */1020	tmp = (0x8 | UVD_LMI_CTRL__WRITE_CLEAN_TIMER_EN_MASK |1021		UVD_LMI_CTRL__REQ_MODE_MASK |1022		UVD_LMI_CTRL__CRC_RESET_MASK |1023		UVD_LMI_CTRL__MASK_MC_URGENT_MASK |1024		UVD_LMI_CTRL__DATA_COHERENCY_EN_MASK |1025		UVD_LMI_CTRL__VCPU_DATA_COHERENCY_EN_MASK |1026		(8 << UVD_LMI_CTRL__WRITE_CLEAN_TIMER__SHIFT) |1027		0x00100000L);1028	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(1029		VCN, inst_idx, mmUVD_LMI_CTRL), tmp, 0, indirect);1030 1031	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(1032		VCN, inst_idx, mmUVD_MPC_CNTL),1033		0x2 << UVD_MPC_CNTL__REPLACEMENT_MODE__SHIFT, 0, indirect);1034 1035	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(1036		VCN, inst_idx, mmUVD_MPC_SET_MUXA0),1037		((0x1 << UVD_MPC_SET_MUXA0__VARA_1__SHIFT) |1038		 (0x2 << UVD_MPC_SET_MUXA0__VARA_2__SHIFT) |1039		 (0x3 << UVD_MPC_SET_MUXA0__VARA_3__SHIFT) |1040		 (0x4 << UVD_MPC_SET_MUXA0__VARA_4__SHIFT)), 0, indirect);1041 1042	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(1043		VCN, inst_idx, mmUVD_MPC_SET_MUXB0),1044		 ((0x1 << UVD_MPC_SET_MUXB0__VARB_1__SHIFT) |1045		 (0x2 << UVD_MPC_SET_MUXB0__VARB_2__SHIFT) |1046		 (0x3 << UVD_MPC_SET_MUXB0__VARB_3__SHIFT) |1047		 (0x4 << UVD_MPC_SET_MUXB0__VARB_4__SHIFT)), 0, indirect);1048 1049	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(1050		VCN, inst_idx, mmUVD_MPC_SET_MUX),1051		((0x0 << UVD_MPC_SET_MUX__SET_0__SHIFT) |1052		 (0x1 << UVD_MPC_SET_MUX__SET_1__SHIFT) |1053		 (0x2 << UVD_MPC_SET_MUX__SET_2__SHIFT)), 0, indirect);1054 1055	vcn_v3_0_mc_resume_dpg_mode(adev, inst_idx, indirect);1056 1057	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(1058		VCN, inst_idx, mmUVD_REG_XX_MASK), 0x10, 0, indirect);1059	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(1060		VCN, inst_idx, mmUVD_RBC_XX_IB_REG_CHECK), 0x3, 0, indirect);1061 1062	/* enable LMI MC and UMC channels */1063	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(1064		VCN, inst_idx, mmUVD_LMI_CTRL2), 0, 0, indirect);1065 1066	/* unblock VCPU register access */1067	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(1068		VCN, inst_idx, mmUVD_RB_ARB_CTRL), 0, 0, indirect);1069 1070	tmp = (0xFF << UVD_VCPU_CNTL__PRB_TIMEOUT_VAL__SHIFT);1071	tmp |= UVD_VCPU_CNTL__CLK_EN_MASK;1072	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(1073		VCN, inst_idx, mmUVD_VCPU_CNTL), tmp, 0, indirect);1074 1075	/* enable master interrupt */1076	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(1077		VCN, inst_idx, mmUVD_MASTINT_EN),1078		UVD_MASTINT_EN__VCPU_EN_MASK, 0, indirect);1079 1080	/* add nop to workaround PSP size check */1081	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(1082		VCN, inst_idx, mmUVD_VCPU_CNTL), tmp, 0, indirect);1083 1084	if (indirect)1085		amdgpu_vcn_psp_update_sram(adev, inst_idx, 0);1086 1087	ring = &adev->vcn.inst[inst_idx].ring_dec;1088	/* force RBC into idle state */1089	rb_bufsz = order_base_2(ring->ring_size);1090	tmp = REG_SET_FIELD(0, UVD_RBC_RB_CNTL, RB_BUFSZ, rb_bufsz);1091	tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_BLKSZ, 1);1092	tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_NO_FETCH, 1);1093	tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_NO_UPDATE, 1);1094	tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_RPTR_WR_EN, 1);1095	WREG32_SOC15(VCN, inst_idx, mmUVD_RBC_RB_CNTL, tmp);1096 1097	/* Stall DPG before WPTR/RPTR reset */1098	WREG32_P(SOC15_REG_OFFSET(VCN, inst_idx, mmUVD_POWER_STATUS),1099		UVD_POWER_STATUS__STALL_DPG_POWER_UP_MASK,1100		~UVD_POWER_STATUS__STALL_DPG_POWER_UP_MASK);1101	fw_shared->multi_queue.decode_queue_mode |= cpu_to_le32(FW_QUEUE_RING_RESET);1102 1103	/* set the write pointer delay */1104	WREG32_SOC15(VCN, inst_idx, mmUVD_RBC_RB_WPTR_CNTL, 0);1105 1106	/* set the wb address */1107	WREG32_SOC15(VCN, inst_idx, mmUVD_RBC_RB_RPTR_ADDR,1108		(upper_32_bits(ring->gpu_addr) >> 2));1109 1110	/* programm the RB_BASE for ring buffer */1111	WREG32_SOC15(VCN, inst_idx, mmUVD_LMI_RBC_RB_64BIT_BAR_LOW,1112		lower_32_bits(ring->gpu_addr));1113	WREG32_SOC15(VCN, inst_idx, mmUVD_LMI_RBC_RB_64BIT_BAR_HIGH,1114		upper_32_bits(ring->gpu_addr));1115 1116	/* Initialize the ring buffer's read and write pointers */1117	WREG32_SOC15(VCN, inst_idx, mmUVD_RBC_RB_RPTR, 0);1118 1119	WREG32_SOC15(VCN, inst_idx, mmUVD_SCRATCH2, 0);1120 1121	ring->wptr = RREG32_SOC15(VCN, inst_idx, mmUVD_RBC_RB_RPTR);1122	WREG32_SOC15(VCN, inst_idx, mmUVD_RBC_RB_WPTR,1123		lower_32_bits(ring->wptr));1124 1125	/* Reset FW shared memory RBC WPTR/RPTR */1126	fw_shared->rb.rptr = 0;1127	fw_shared->rb.wptr = lower_32_bits(ring->wptr);1128 1129	/*resetting done, fw can check RB ring */1130	fw_shared->multi_queue.decode_queue_mode &= cpu_to_le32(~FW_QUEUE_RING_RESET);1131 1132	/* Unstall DPG */1133	WREG32_P(SOC15_REG_OFFSET(VCN, inst_idx, mmUVD_POWER_STATUS),1134		0, ~UVD_POWER_STATUS__STALL_DPG_POWER_UP_MASK);1135 1136	return 0;1137}1138 1139static int vcn_v3_0_start(struct amdgpu_device *adev)1140{1141	volatile struct amdgpu_fw_shared *fw_shared;1142	struct amdgpu_ring *ring;1143	uint32_t rb_bufsz, tmp;1144	int i, j, k, r;1145 1146	if (adev->pm.dpm_enabled)1147		amdgpu_dpm_enable_uvd(adev, true);1148 1149	for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {1150		if (adev->vcn.harvest_config & (1 << i))1151			continue;1152 1153		if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG) {1154			r = vcn_v3_0_start_dpg_mode(adev, i, adev->vcn.indirect_sram);1155			continue;1156		}1157 1158		/* disable VCN power gating */1159		vcn_v3_0_disable_static_power_gating(adev, i);1160 1161		/* set VCN status busy */1162		tmp = RREG32_SOC15(VCN, i, mmUVD_STATUS) | UVD_STATUS__UVD_BUSY;1163		WREG32_SOC15(VCN, i, mmUVD_STATUS, tmp);1164 1165		/*SW clock gating */1166		vcn_v3_0_disable_clock_gating(adev, i);1167 1168		/* enable VCPU clock */1169		WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_VCPU_CNTL),1170			UVD_VCPU_CNTL__CLK_EN_MASK, ~UVD_VCPU_CNTL__CLK_EN_MASK);1171 1172		/* disable master interrupt */1173		WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_MASTINT_EN), 0,1174			~UVD_MASTINT_EN__VCPU_EN_MASK);1175 1176		/* enable LMI MC and UMC channels */1177		WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_LMI_CTRL2), 0,1178			~UVD_LMI_CTRL2__STALL_ARB_UMC_MASK);1179 1180		tmp = RREG32_SOC15(VCN, i, mmUVD_SOFT_RESET);1181		tmp &= ~UVD_SOFT_RESET__LMI_SOFT_RESET_MASK;1182		tmp &= ~UVD_SOFT_RESET__LMI_UMC_SOFT_RESET_MASK;1183		WREG32_SOC15(VCN, i, mmUVD_SOFT_RESET, tmp);1184 1185		/* setup mmUVD_LMI_CTRL */1186		tmp = RREG32_SOC15(VCN, i, mmUVD_LMI_CTRL);1187		WREG32_SOC15(VCN, i, mmUVD_LMI_CTRL, tmp |1188			UVD_LMI_CTRL__WRITE_CLEAN_TIMER_EN_MASK	|1189			UVD_LMI_CTRL__MASK_MC_URGENT_MASK |1190			UVD_LMI_CTRL__DATA_COHERENCY_EN_MASK |1191			UVD_LMI_CTRL__VCPU_DATA_COHERENCY_EN_MASK);1192 1193		/* setup mmUVD_MPC_CNTL */1194		tmp = RREG32_SOC15(VCN, i, mmUVD_MPC_CNTL);1195		tmp &= ~UVD_MPC_CNTL__REPLACEMENT_MODE_MASK;1196		tmp |= 0x2 << UVD_MPC_CNTL__REPLACEMENT_MODE__SHIFT;1197		WREG32_SOC15(VCN, i, mmUVD_MPC_CNTL, tmp);1198 1199		/* setup UVD_MPC_SET_MUXA0 */1200		WREG32_SOC15(VCN, i, mmUVD_MPC_SET_MUXA0,1201			((0x1 << UVD_MPC_SET_MUXA0__VARA_1__SHIFT) |1202			(0x2 << UVD_MPC_SET_MUXA0__VARA_2__SHIFT) |1203			(0x3 << UVD_MPC_SET_MUXA0__VARA_3__SHIFT) |1204			(0x4 << UVD_MPC_SET_MUXA0__VARA_4__SHIFT)));1205 1206		/* setup UVD_MPC_SET_MUXB0 */1207		WREG32_SOC15(VCN, i, mmUVD_MPC_SET_MUXB0,1208			((0x1 << UVD_MPC_SET_MUXB0__VARB_1__SHIFT) |1209			(0x2 << UVD_MPC_SET_MUXB0__VARB_2__SHIFT) |1210			(0x3 << UVD_MPC_SET_MUXB0__VARB_3__SHIFT) |1211			(0x4 << UVD_MPC_SET_MUXB0__VARB_4__SHIFT)));1212 1213		/* setup mmUVD_MPC_SET_MUX */1214		WREG32_SOC15(VCN, i, mmUVD_MPC_SET_MUX,1215			((0x0 << UVD_MPC_SET_MUX__SET_0__SHIFT) |1216			(0x1 << UVD_MPC_SET_MUX__SET_1__SHIFT) |1217			(0x2 << UVD_MPC_SET_MUX__SET_2__SHIFT)));1218 1219		vcn_v3_0_mc_resume(adev, i);1220 1221		/* VCN global tiling registers */1222		WREG32_SOC15(VCN, i, mmUVD_GFX10_ADDR_CONFIG,1223			adev->gfx.config.gb_addr_config);1224 1225		/* unblock VCPU register access */1226		WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_RB_ARB_CTRL), 0,1227			~UVD_RB_ARB_CTRL__VCPU_DIS_MASK);1228 1229		/* release VCPU reset to boot */1230		WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_VCPU_CNTL), 0,1231			~UVD_VCPU_CNTL__BLK_RST_MASK);1232 1233		for (j = 0; j < 10; ++j) {1234			uint32_t status;1235 1236			for (k = 0; k < 100; ++k) {1237				status = RREG32_SOC15(VCN, i, mmUVD_STATUS);1238				if (status & 2)1239					break;1240				mdelay(10);1241			}1242			r = 0;1243			if (status & 2)1244				break;1245 1246			DRM_ERROR("VCN[%d] decode not responding, trying to reset the VCPU!!!\n", i);1247			WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_VCPU_CNTL),1248				UVD_VCPU_CNTL__BLK_RST_MASK,1249				~UVD_VCPU_CNTL__BLK_RST_MASK);1250			mdelay(10);1251			WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_VCPU_CNTL), 0,1252				~UVD_VCPU_CNTL__BLK_RST_MASK);1253 1254			mdelay(10);1255			r = -1;1256		}1257 1258		if (r) {1259			DRM_ERROR("VCN[%d] decode not responding, giving up!!!\n", i);1260			return r;1261		}1262 1263		/* enable master interrupt */1264		WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_MASTINT_EN),1265			UVD_MASTINT_EN__VCPU_EN_MASK,1266			~UVD_MASTINT_EN__VCPU_EN_MASK);1267 1268		/* clear the busy bit of VCN_STATUS */1269		WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_STATUS), 0,1270			~(2 << UVD_STATUS__VCPU_REPORT__SHIFT));1271 1272		WREG32_SOC15(VCN, i, mmUVD_LMI_RBC_RB_VMID, 0);1273 1274		ring = &adev->vcn.inst[i].ring_dec;1275		/* force RBC into idle state */1276		rb_bufsz = order_base_2(ring->ring_size);1277		tmp = REG_SET_FIELD(0, UVD_RBC_RB_CNTL, RB_BUFSZ, rb_bufsz);1278		tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_BLKSZ, 1);1279		tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_NO_FETCH, 1);1280		tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_NO_UPDATE, 1);1281		tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_RPTR_WR_EN, 1);1282		WREG32_SOC15(VCN, i, mmUVD_RBC_RB_CNTL, tmp);1283 1284		fw_shared = adev->vcn.inst[i].fw_shared.cpu_addr;1285		fw_shared->multi_queue.decode_queue_mode |= cpu_to_le32(FW_QUEUE_RING_RESET);1286 1287		/* programm the RB_BASE for ring buffer */1288		WREG32_SOC15(VCN, i, mmUVD_LMI_RBC_RB_64BIT_BAR_LOW,1289			lower_32_bits(ring->gpu_addr));1290		WREG32_SOC15(VCN, i, mmUVD_LMI_RBC_RB_64BIT_BAR_HIGH,1291			upper_32_bits(ring->gpu_addr));1292 1293		/* Initialize the ring buffer's read and write pointers */1294		WREG32_SOC15(VCN, i, mmUVD_RBC_RB_RPTR, 0);1295 1296		WREG32_SOC15(VCN, i, mmUVD_SCRATCH2, 0);1297		ring->wptr = RREG32_SOC15(VCN, i, mmUVD_RBC_RB_RPTR);1298		WREG32_SOC15(VCN, i, mmUVD_RBC_RB_WPTR,1299			lower_32_bits(ring->wptr));1300		fw_shared->rb.wptr = lower_32_bits(ring->wptr);1301		fw_shared->multi_queue.decode_queue_mode &= cpu_to_le32(~FW_QUEUE_RING_RESET);1302 1303		if (amdgpu_ip_version(adev, UVD_HWIP, 0) !=1304		    IP_VERSION(3, 0, 33)) {1305			fw_shared->multi_queue.encode_generalpurpose_queue_mode |= cpu_to_le32(FW_QUEUE_RING_RESET);1306			ring = &adev->vcn.inst[i].ring_enc[0];1307			WREG32_SOC15(VCN, i, mmUVD_RB_RPTR, lower_32_bits(ring->wptr));1308			WREG32_SOC15(VCN, i, mmUVD_RB_WPTR, lower_32_bits(ring->wptr));1309			WREG32_SOC15(VCN, i, mmUVD_RB_BASE_LO, ring->gpu_addr);1310			WREG32_SOC15(VCN, i, mmUVD_RB_BASE_HI, upper_32_bits(ring->gpu_addr));1311			WREG32_SOC15(VCN, i, mmUVD_RB_SIZE, ring->ring_size / 4);1312			fw_shared->multi_queue.encode_generalpurpose_queue_mode &= cpu_to_le32(~FW_QUEUE_RING_RESET);1313 1314			fw_shared->multi_queue.encode_lowlatency_queue_mode |= cpu_to_le32(FW_QUEUE_RING_RESET);1315			ring = &adev->vcn.inst[i].ring_enc[1];1316			WREG32_SOC15(VCN, i, mmUVD_RB_RPTR2, lower_32_bits(ring->wptr));1317			WREG32_SOC15(VCN, i, mmUVD_RB_WPTR2, lower_32_bits(ring->wptr));1318			WREG32_SOC15(VCN, i, mmUVD_RB_BASE_LO2, ring->gpu_addr);1319			WREG32_SOC15(VCN, i, mmUVD_RB_BASE_HI2, upper_32_bits(ring->gpu_addr));1320			WREG32_SOC15(VCN, i, mmUVD_RB_SIZE2, ring->ring_size / 4);1321			fw_shared->multi_queue.encode_lowlatency_queue_mode &= cpu_to_le32(~FW_QUEUE_RING_RESET);1322		}1323	}1324 1325	return 0;1326}1327 1328static int vcn_v3_0_start_sriov(struct amdgpu_device *adev)1329{1330	int i, j;1331	struct amdgpu_ring *ring;1332	uint64_t cache_addr;1333	uint64_t rb_addr;1334	uint64_t ctx_addr;1335	uint32_t param, resp, expected;1336	uint32_t offset, cache_size;1337	uint32_t tmp, timeout;1338 1339	struct amdgpu_mm_table *table = &adev->virt.mm_table;1340	uint32_t *table_loc;1341	uint32_t table_size;1342	uint32_t size, size_dw;1343 1344	struct mmsch_v3_0_cmd_direct_write1345		direct_wt = { {0} };1346	struct mmsch_v3_0_cmd_direct_read_modify_write1347		direct_rd_mod_wt = { {0} };1348	struct mmsch_v3_0_cmd_end end = { {0} };1349	struct mmsch_v3_0_init_header header;1350 1351	direct_wt.cmd_header.command_type =1352		MMSCH_COMMAND__DIRECT_REG_WRITE;1353	direct_rd_mod_wt.cmd_header.command_type =1354		MMSCH_COMMAND__DIRECT_REG_READ_MODIFY_WRITE;1355	end.cmd_header.command_type =1356		MMSCH_COMMAND__END;1357 1358	header.version = MMSCH_VERSION;1359	header.total_size = sizeof(struct mmsch_v3_0_init_header) >> 2;1360	for (i = 0; i < MMSCH_V3_0_VCN_INSTANCES; i++) {1361		header.inst[i].init_status = 0;1362		header.inst[i].table_offset = 0;1363		header.inst[i].table_size = 0;1364	}1365 1366	table_loc = (uint32_t *)table->cpu_addr;1367	table_loc += header.total_size;1368	for (i = 0; i < adev->vcn.num_vcn_inst; i++) {1369		if (adev->vcn.harvest_config & (1 << i))1370			continue;1371 1372		table_size = 0;1373 1374		MMSCH_V3_0_INSERT_DIRECT_RD_MOD_WT(SOC15_REG_OFFSET(VCN, i,1375			mmUVD_STATUS),1376			~UVD_STATUS__UVD_BUSY, UVD_STATUS__UVD_BUSY);1377 1378		cache_size = AMDGPU_GPU_PAGE_ALIGN(adev->vcn.fw[i]->size + 4);1379 1380		if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {1381			MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,1382				mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW),1383				adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + i].tmr_mc_addr_lo);1384			MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,1385				mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH),1386				adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + i].tmr_mc_addr_hi);1387			offset = 0;1388			MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,1389				mmUVD_VCPU_CACHE_OFFSET0),1390				0);1391		} else {1392			MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,1393				mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW),1394				lower_32_bits(adev->vcn.inst[i].gpu_addr));1395			MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,1396				mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH),1397				upper_32_bits(adev->vcn.inst[i].gpu_addr));1398			offset = cache_size;1399			MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,1400				mmUVD_VCPU_CACHE_OFFSET0),1401				AMDGPU_UVD_FIRMWARE_OFFSET >> 3);1402		}1403 1404		MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,1405			mmUVD_VCPU_CACHE_SIZE0),1406			cache_size);1407 1408		cache_addr = adev->vcn.inst[i].gpu_addr + offset;1409		MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,1410			mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW),1411			lower_32_bits(cache_addr));1412		MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,1413			mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH),1414			upper_32_bits(cache_addr));1415		MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,1416			mmUVD_VCPU_CACHE_OFFSET1),1417			0);1418		MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,1419			mmUVD_VCPU_CACHE_SIZE1),1420			AMDGPU_VCN_STACK_SIZE);1421 1422		cache_addr = adev->vcn.inst[i].gpu_addr + offset +1423			AMDGPU_VCN_STACK_SIZE;1424		MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,1425			mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_LOW),1426			lower_32_bits(cache_addr));1427		MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,1428			mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_HIGH),1429			upper_32_bits(cache_addr));1430		MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,1431			mmUVD_VCPU_CACHE_OFFSET2),1432			0);1433		MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,1434			mmUVD_VCPU_CACHE_SIZE2),1435			AMDGPU_VCN_CONTEXT_SIZE);1436 1437		for (j = 0; j < adev->vcn.num_enc_rings; ++j) {1438			ring = &adev->vcn.inst[i].ring_enc[j];1439			ring->wptr = 0;1440			rb_addr = ring->gpu_addr;1441			MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,1442				mmUVD_RB_BASE_LO),1443				lower_32_bits(rb_addr));1444			MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,1445				mmUVD_RB_BASE_HI),1446				upper_32_bits(rb_addr));1447			MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,1448				mmUVD_RB_SIZE),1449				ring->ring_size / 4);1450		}1451 1452		ring = &adev->vcn.inst[i].ring_dec;1453		ring->wptr = 0;1454		rb_addr = ring->gpu_addr;1455		MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,1456			mmUVD_LMI_RBC_RB_64BIT_BAR_LOW),1457			lower_32_bits(rb_addr));1458		MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,1459			mmUVD_LMI_RBC_RB_64BIT_BAR_HIGH),1460			upper_32_bits(rb_addr));1461		/* force RBC into idle state */1462		tmp = order_base_2(ring->ring_size);1463		tmp = REG_SET_FIELD(0, UVD_RBC_RB_CNTL, RB_BUFSZ, tmp);1464		tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_BLKSZ, 1);1465		tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_NO_FETCH, 1);1466		tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_NO_UPDATE, 1);1467		tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_RPTR_WR_EN, 1);1468		MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,1469			mmUVD_RBC_RB_CNTL),1470			tmp);1471 1472		/* add end packet */1473		MMSCH_V3_0_INSERT_END();1474 1475		/* refine header */1476		header.inst[i].init_status = 0;1477		header.inst[i].table_offset = header.total_size;1478		header.inst[i].table_size = table_size;1479		header.total_size += table_size;1480	}1481 1482	/* Update init table header in memory */1483	size = sizeof(struct mmsch_v3_0_init_header);1484	table_loc = (uint32_t *)table->cpu_addr;1485	memcpy((void *)table_loc, &header, size);1486 1487	/* message MMSCH (in VCN[0]) to initialize this client1488	 * 1, write to mmsch_vf_ctx_addr_lo/hi register with GPU mc addr1489	 * of memory descriptor location1490	 */1491	ctx_addr = table->gpu_addr;1492	WREG32_SOC15(VCN, 0, mmMMSCH_VF_CTX_ADDR_LO, lower_32_bits(ctx_addr));1493	WREG32_SOC15(VCN, 0, mmMMSCH_VF_CTX_ADDR_HI, upper_32_bits(ctx_addr));1494 1495	/* 2, update vmid of descriptor */1496	tmp = RREG32_SOC15(VCN, 0, mmMMSCH_VF_VMID);1497	tmp &= ~MMSCH_VF_VMID__VF_CTX_VMID_MASK;1498	/* use domain0 for MM scheduler */1499	tmp |= (0 << MMSCH_VF_VMID__VF_CTX_VMID__SHIFT);1500	WREG32_SOC15(VCN, 0, mmMMSCH_VF_VMID, tmp);1501 1502	/* 3, notify mmsch about the size of this descriptor */1503	size = header.total_size;1504	WREG32_SOC15(VCN, 0, mmMMSCH_VF_CTX_SIZE, size);1505 1506	/* 4, set resp to zero */1507	WREG32_SOC15(VCN, 0, mmMMSCH_VF_MAILBOX_RESP, 0);1508 1509	/* 5, kick off the initialization and wait until1510	 * MMSCH_VF_MAILBOX_RESP becomes non-zero1511	 */1512	param = 0x10000001;1513	WREG32_SOC15(VCN, 0, mmMMSCH_VF_MAILBOX_HOST, param);1514	tmp = 0;1515	timeout = 1000;1516	resp = 0;1517	expected = param + 1;1518	while (resp != expected) {1519		resp = RREG32_SOC15(VCN, 0, mmMMSCH_VF_MAILBOX_RESP);1520		if (resp == expected)1521			break;1522 1523		udelay(10);1524		tmp = tmp + 10;1525		if (tmp >= timeout) {1526			DRM_ERROR("failed to init MMSCH. TIME-OUT after %d usec"\1527				" waiting for mmMMSCH_VF_MAILBOX_RESP "\1528				"(expected=0x%08x, readback=0x%08x)\n",1529				tmp, expected, resp);1530			return -EBUSY;1531		}1532	}1533 1534	return 0;1535}1536 1537static int vcn_v3_0_stop_dpg_mode(struct amdgpu_device *adev, int inst_idx)1538{1539	struct dpg_pause_state state = {.fw_based = VCN_DPG_STATE__UNPAUSE};1540	uint32_t tmp;1541 1542	vcn_v3_0_pause_dpg_mode(adev, inst_idx, &state);1543 1544	/* Wait for power status to be 1 */1545	SOC15_WAIT_ON_RREG(VCN, inst_idx, mmUVD_POWER_STATUS, 1,1546		UVD_POWER_STATUS__UVD_POWER_STATUS_MASK);1547 1548	/* wait for read ptr to be equal to write ptr */1549	tmp = RREG32_SOC15(VCN, inst_idx, mmUVD_RB_WPTR);1550	SOC15_WAIT_ON_RREG(VCN, inst_idx, mmUVD_RB_RPTR, tmp, 0xFFFFFFFF);1551 1552	tmp = RREG32_SOC15(VCN, inst_idx, mmUVD_RB_WPTR2);1553	SOC15_WAIT_ON_RREG(VCN, inst_idx, mmUVD_RB_RPTR2, tmp, 0xFFFFFFFF);1554 1555	tmp = RREG32_SOC15(VCN, inst_idx, mmUVD_RBC_RB_WPTR) & 0x7FFFFFFF;1556	SOC15_WAIT_ON_RREG(VCN, inst_idx, mmUVD_RBC_RB_RPTR, tmp, 0xFFFFFFFF);1557 1558	SOC15_WAIT_ON_RREG(VCN, inst_idx, mmUVD_POWER_STATUS, 1,1559		UVD_POWER_STATUS__UVD_POWER_STATUS_MASK);1560 1561	/* disable dynamic power gating mode */1562	WREG32_P(SOC15_REG_OFFSET(VCN, inst_idx, mmUVD_POWER_STATUS), 0,1563		~UVD_POWER_STATUS__UVD_PG_MODE_MASK);1564 1565	return 0;1566}1567 1568static int vcn_v3_0_stop(struct amdgpu_device *adev)1569{1570	uint32_t tmp;1571	int i, r = 0;1572 1573	for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {1574		if (adev->vcn.harvest_config & (1 << i))1575			continue;1576 1577		if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG) {1578			r = vcn_v3_0_stop_dpg_mode(adev, i);1579			continue;1580		}1581 1582		/* wait for vcn idle */1583		r = SOC15_WAIT_ON_RREG(VCN, i, mmUVD_STATUS, UVD_STATUS__IDLE, 0x7);1584		if (r)1585			return r;1586 1587		tmp = UVD_LMI_STATUS__VCPU_LMI_WRITE_CLEAN_MASK |1588			UVD_LMI_STATUS__READ_CLEAN_MASK |1589			UVD_LMI_STATUS__WRITE_CLEAN_MASK |1590			UVD_LMI_STATUS__WRITE_CLEAN_RAW_MASK;1591		r = SOC15_WAIT_ON_RREG(VCN, i, mmUVD_LMI_STATUS, tmp, tmp);1592		if (r)1593			return r;1594 1595		/* disable LMI UMC channel */1596		tmp = RREG32_SOC15(VCN, i, mmUVD_LMI_CTRL2);1597		tmp |= UVD_LMI_CTRL2__STALL_ARB_UMC_MASK;1598		WREG32_SOC15(VCN, i, mmUVD_LMI_CTRL2, tmp);1599		tmp = UVD_LMI_STATUS__UMC_READ_CLEAN_RAW_MASK|1600			UVD_LMI_STATUS__UMC_WRITE_CLEAN_RAW_MASK;1601		r = SOC15_WAIT_ON_RREG(VCN, i, mmUVD_LMI_STATUS, tmp, tmp);1602		if (r)1603			return r;1604 1605		/* block VCPU register access */1606		WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_RB_ARB_CTRL),1607			UVD_RB_ARB_CTRL__VCPU_DIS_MASK,1608			~UVD_RB_ARB_CTRL__VCPU_DIS_MASK);1609 1610		/* reset VCPU */1611		WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_VCPU_CNTL),1612			UVD_VCPU_CNTL__BLK_RST_MASK,1613			~UVD_VCPU_CNTL__BLK_RST_MASK);1614 1615		/* disable VCPU clock */1616		WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_VCPU_CNTL), 0,1617			~(UVD_VCPU_CNTL__CLK_EN_MASK));1618 1619		/* apply soft reset */1620		tmp = RREG32_SOC15(VCN, i, mmUVD_SOFT_RESET);1621		tmp |= UVD_SOFT_RESET__LMI_UMC_SOFT_RESET_MASK;1622		WREG32_SOC15(VCN, i, mmUVD_SOFT_RESET, tmp);1623		tmp = RREG32_SOC15(VCN, i, mmUVD_SOFT_RESET);1624		tmp |= UVD_SOFT_RESET__LMI_SOFT_RESET_MASK;1625		WREG32_SOC15(VCN, i, mmUVD_SOFT_RESET, tmp);1626 1627		/* clear status */1628		WREG32_SOC15(VCN, i, mmUVD_STATUS, 0);1629 1630		/* apply HW clock gating */1631		vcn_v3_0_enable_clock_gating(adev, i);1632 1633		/* enable VCN power gating */1634		vcn_v3_0_enable_static_power_gating(adev, i);1635	}1636 1637	if (adev->pm.dpm_enabled)1638		amdgpu_dpm_enable_uvd(adev, false);1639 1640	return 0;1641}1642 1643static int vcn_v3_0_pause_dpg_mode(struct amdgpu_device *adev,1644		   int inst_idx, struct dpg_pause_state *new_state)1645{1646	volatile struct amdgpu_fw_shared *fw_shared;1647	struct amdgpu_ring *ring;1648	uint32_t reg_data = 0;1649	int ret_code;1650 1651	/* pause/unpause if state is changed */1652	if (adev->vcn.inst[inst_idx].pause_state.fw_based != new_state->fw_based) {1653		DRM_DEBUG("dpg pause state changed %d -> %d",1654			adev->vcn.inst[inst_idx].pause_state.fw_based,	new_state->fw_based);1655		reg_data = RREG32_SOC15(VCN, inst_idx, mmUVD_DPG_PAUSE) &1656			(~UVD_DPG_PAUSE__NJ_PAUSE_DPG_ACK_MASK);1657 1658		if (new_state->fw_based == VCN_DPG_STATE__PAUSE) {1659			ret_code = SOC15_WAIT_ON_RREG(VCN, inst_idx, mmUVD_POWER_STATUS, 0x1,1660				UVD_POWER_STATUS__UVD_POWER_STATUS_MASK);1661 1662			if (!ret_code) {1663				/* pause DPG */1664				reg_data |= UVD_DPG_PAUSE__NJ_PAUSE_DPG_REQ_MASK;1665				WREG32_SOC15(VCN, inst_idx, mmUVD_DPG_PAUSE, reg_data);1666 1667				/* wait for ACK */1668				SOC15_WAIT_ON_RREG(VCN, inst_idx, mmUVD_DPG_PAUSE,1669					UVD_DPG_PAUSE__NJ_PAUSE_DPG_ACK_MASK,1670					UVD_DPG_PAUSE__NJ_PAUSE_DPG_ACK_MASK);1671 1672				/* Stall DPG before WPTR/RPTR reset */1673				WREG32_P(SOC15_REG_OFFSET(VCN, inst_idx, mmUVD_POWER_STATUS),1674					UVD_POWER_STATUS__STALL_DPG_POWER_UP_MASK,1675					~UVD_POWER_STATUS__STALL_DPG_POWER_UP_MASK);1676 1677				if (amdgpu_ip_version(adev, UVD_HWIP, 0) !=1678				    IP_VERSION(3, 0, 33)) {1679					/* Restore */1680					fw_shared = adev->vcn.inst[inst_idx].fw_shared.cpu_addr;1681					fw_shared->multi_queue.encode_generalpurpose_queue_mode |= cpu_to_le32(FW_QUEUE_RING_RESET);1682					ring = &adev->vcn.inst[inst_idx].ring_enc[0];1683					ring->wptr = 0;1684					WREG32_SOC15(VCN, inst_idx, mmUVD_RB_BASE_LO, ring->gpu_addr);1685					WREG32_SOC15(VCN, inst_idx, mmUVD_RB_BASE_HI, upper_32_bits(ring->gpu_addr));1686					WREG32_SOC15(VCN, inst_idx, mmUVD_RB_SIZE, ring->ring_size / 4);1687					WREG32_SOC15(VCN, inst_idx, mmUVD_RB_RPTR, lower_32_bits(ring->wptr));1688					WREG32_SOC15(VCN, inst_idx, mmUVD_RB_WPTR, lower_32_bits(ring->wptr));1689					fw_shared->multi_queue.encode_generalpurpose_queue_mode &= cpu_to_le32(~FW_QUEUE_RING_RESET);1690 1691					fw_shared->multi_queue.encode_lowlatency_queue_mode |= cpu_to_le32(FW_QUEUE_RING_RESET);1692					ring = &adev->vcn.inst[inst_idx].ring_enc[1];1693					ring->wptr = 0;1694					WREG32_SOC15(VCN, inst_idx, mmUVD_RB_BASE_LO2, ring->gpu_addr);1695					WREG32_SOC15(VCN, inst_idx, mmUVD_RB_BASE_HI2, upper_32_bits(ring->gpu_addr));1696					WREG32_SOC15(VCN, inst_idx, mmUVD_RB_SIZE2, ring->ring_size / 4);1697					WREG32_SOC15(VCN, inst_idx, mmUVD_RB_RPTR2, lower_32_bits(ring->wptr));1698					WREG32_SOC15(VCN, inst_idx, mmUVD_RB_WPTR2, lower_32_bits(ring->wptr));1699					fw_shared->multi_queue.encode_lowlatency_queue_mode &= cpu_to_le32(~FW_QUEUE_RING_RESET);1700 1701					/* restore wptr/rptr with pointers saved in FW shared memory*/1702					WREG32_SOC15(VCN, inst_idx, mmUVD_RBC_RB_RPTR, fw_shared->rb.rptr);1703					WREG32_SOC15(VCN, inst_idx, mmUVD_RBC_RB_WPTR, fw_shared->rb.wptr);1704				}1705 1706				/* Unstall DPG */1707				WREG32_P(SOC15_REG_OFFSET(VCN, inst_idx, mmUVD_POWER_STATUS),1708					0, ~UVD_POWER_STATUS__STALL_DPG_POWER_UP_MASK);1709 1710				SOC15_WAIT_ON_RREG(VCN, inst_idx, mmUVD_POWER_STATUS,1711					UVD_PGFSM_CONFIG__UVDM_UVDU_PWR_ON, UVD_POWER_STATUS__UVD_POWER_STATUS_MASK);1712			}1713		} else {1714			/* unpause dpg, no need to wait */1715			reg_data &= ~UVD_DPG_PAUSE__NJ_PAUSE_DPG_REQ_MASK;1716			WREG32_SOC15(VCN, inst_idx, mmUVD_DPG_PAUSE, reg_data);1717		}1718		adev->vcn.inst[inst_idx].pause_state.fw_based = new_state->fw_based;1719	}1720 1721	return 0;1722}1723 1724/**1725 * vcn_v3_0_dec_ring_get_rptr - get read pointer1726 *1727 * @ring: amdgpu_ring pointer1728 *1729 * Returns the current hardware read pointer1730 */1731static uint64_t vcn_v3_0_dec_ring_get_rptr(struct amdgpu_ring *ring)1732{1733	struct amdgpu_device *adev = ring->adev;1734 1735	return RREG32_SOC15(VCN, ring->me, mmUVD_RBC_RB_RPTR);1736}1737 1738/**1739 * vcn_v3_0_dec_ring_get_wptr - get write pointer1740 *1741 * @ring: amdgpu_ring pointer1742 *1743 * Returns the current hardware write pointer1744 */1745static uint64_t vcn_v3_0_dec_ring_get_wptr(struct amdgpu_ring *ring)1746{1747	struct amdgpu_device *adev = ring->adev;1748 1749	if (ring->use_doorbell)1750		return *ring->wptr_cpu_addr;1751	else1752		return RREG32_SOC15(VCN, ring->me, mmUVD_RBC_RB_WPTR);1753}1754 1755/**1756 * vcn_v3_0_dec_ring_set_wptr - set write pointer1757 *1758 * @ring: amdgpu_ring pointer1759 *1760 * Commits the write pointer to the hardware1761 */1762static void vcn_v3_0_dec_ring_set_wptr(struct amdgpu_ring *ring)1763{1764	struct amdgpu_device *adev = ring->adev;1765	volatile struct amdgpu_fw_shared *fw_shared;1766 1767	if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG) {1768		/*whenever update RBC_RB_WPTR, we save the wptr in shared rb.wptr and scratch2 */1769		fw_shared = adev->vcn.inst[ring->me].fw_shared.cpu_addr;1770		fw_shared->rb.wptr = lower_32_bits(ring->wptr);1771		WREG32_SOC15(VCN, ring->me, mmUVD_SCRATCH2,1772			lower_32_bits(ring->wptr));1773	}1774 1775	if (ring->use_doorbell) {1776		*ring->wptr_cpu_addr = lower_32_bits(ring->wptr);1777		WDOORBELL32(ring->doorbell_index, lower_32_bits(ring->wptr));1778	} else {1779		WREG32_SOC15(VCN, ring->me, mmUVD_RBC_RB_WPTR, lower_32_bits(ring->wptr));1780	}1781}1782 1783static const struct amdgpu_ring_funcs vcn_v3_0_dec_sw_ring_vm_funcs = {1784	.type = AMDGPU_RING_TYPE_VCN_DEC,1785	.align_mask = 0x3f,1786	.nop = VCN_DEC_SW_CMD_NO_OP,1787	.secure_submission_supported = true,1788	.get_rptr = vcn_v3_0_dec_ring_get_rptr,1789	.get_wptr = vcn_v3_0_dec_ring_get_wptr,1790	.set_wptr = vcn_v3_0_dec_ring_set_wptr,1791	.emit_frame_size =1792		SOC15_FLUSH_GPU_TLB_NUM_WREG * 3 +1793		SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 4 +1794		VCN_SW_RING_EMIT_FRAME_SIZE,1795	.emit_ib_size = 5, /* vcn_dec_sw_ring_emit_ib */1796	.emit_ib = vcn_dec_sw_ring_emit_ib,1797	.emit_fence = vcn_dec_sw_ring_emit_fence,1798	.emit_vm_flush = vcn_dec_sw_ring_emit_vm_flush,1799	.test_ring = amdgpu_vcn_dec_sw_ring_test_ring,1800	.test_ib = NULL,//amdgpu_vcn_dec_sw_ring_test_ib,1801	.insert_nop = amdgpu_ring_insert_nop,1802	.insert_end = vcn_dec_sw_ring_insert_end,1803	.pad_ib = amdgpu_ring_generic_pad_ib,1804	.begin_use = amdgpu_vcn_ring_begin_use,1805	.end_use = amdgpu_vcn_ring_end_use,1806	.emit_wreg = vcn_dec_sw_ring_emit_wreg,1807	.emit_reg_wait = vcn_dec_sw_ring_emit_reg_wait,1808	.emit_reg_write_reg_wait = amdgpu_ring_emit_reg_write_reg_wait_helper,1809};1810 1811static int vcn_v3_0_limit_sched(struct amdgpu_cs_parser *p,1812				struct amdgpu_job *job)1813{1814	struct drm_gpu_scheduler **scheds;1815 1816	/* The create msg must be in the first IB submitted */1817	if (atomic_read(&job->base.entity->fence_seq))1818		return -EINVAL;1819 1820	/* if VCN0 is harvested, we can't support AV1 */1821	if (p->adev->vcn.harvest_config & AMDGPU_VCN_HARVEST_VCN0)1822		return -EINVAL;1823 1824	scheds = p->adev->gpu_sched[AMDGPU_HW_IP_VCN_DEC]1825		[AMDGPU_RING_PRIO_DEFAULT].sched;1826	drm_sched_entity_modify_sched(job->base.entity, scheds, 1);1827	return 0;1828}1829 1830static int vcn_v3_0_dec_msg(struct amdgpu_cs_parser *p, struct amdgpu_job *job,1831			    uint64_t addr)1832{1833	struct ttm_operation_ctx ctx = { false, false };1834	struct amdgpu_bo_va_mapping *map;1835	uint32_t *msg, num_buffers;1836	struct amdgpu_bo *bo;1837	uint64_t start, end;1838	unsigned int i;1839	void *ptr;1840	int r;1841 1842	addr &= AMDGPU_GMC_HOLE_MASK;1843	r = amdgpu_cs_find_mapping(p, addr, &bo, &map);1844	if (r) {1845		DRM_ERROR("Can't find BO for addr 0x%08Lx\n", addr);1846		return r;1847	}1848 1849	start = map->start * AMDGPU_GPU_PAGE_SIZE;1850	end = (map->last + 1) * AMDGPU_GPU_PAGE_SIZE;1851	if (addr & 0x7) {1852		DRM_ERROR("VCN messages must be 8 byte aligned!\n");1853		return -EINVAL;1854	}1855 1856	bo->flags |= AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;1857	amdgpu_bo_placement_from_domain(bo, bo->allowed_domains);1858	r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);1859	if (r) {1860		DRM_ERROR("Failed validating the VCN message BO (%d)!\n", r);1861		return r;1862	}1863 1864	r = amdgpu_bo_kmap(bo, &ptr);1865	if (r) {1866		DRM_ERROR("Failed mapping the VCN message (%d)!\n", r);1867		return r;1868	}1869 1870	msg = ptr + addr - start;1871 1872	/* Check length */1873	if (msg[1] > end - addr) {1874		r = -EINVAL;1875		goto out;1876	}1877 1878	if (msg[3] != RDECODE_MSG_CREATE)1879		goto out;1880 1881	num_buffers = msg[2];1882	for (i = 0, msg = &msg[6]; i < num_buffers; ++i, msg += 4) {1883		uint32_t offset, size, *create;1884 1885		if (msg[0] != RDECODE_MESSAGE_CREATE)1886			continue;1887 1888		offset = msg[1];1889		size = msg[2];1890 1891		if (offset + size > end) {1892			r = -EINVAL;1893			goto out;1894		}1895 1896		create = ptr + addr + offset - start;1897 1898		/* H246, HEVC and VP9 can run on any instance */1899		if (create[0] == 0x7 || create[0] == 0x10 || create[0] == 0x11)1900			continue;1901 1902		r = vcn_v3_0_limit_sched(p, job);1903		if (r)1904			goto out;1905	}1906 1907out:1908	amdgpu_bo_kunmap(bo);1909	return r;1910}1911 1912static int vcn_v3_0_ring_patch_cs_in_place(struct amdgpu_cs_parser *p,1913					   struct amdgpu_job *job,1914					   struct amdgpu_ib *ib)1915{1916	struct amdgpu_ring *ring = amdgpu_job_ring(job);1917	uint32_t msg_lo = 0, msg_hi = 0;1918	unsigned i;1919	int r;1920 1921	/* The first instance can decode anything */1922	if (!ring->me)1923		return 0;1924 1925	for (i = 0; i < ib->length_dw; i += 2) {1926		uint32_t reg = amdgpu_ib_get_value(ib, i);1927		uint32_t val = amdgpu_ib_get_value(ib, i + 1);1928 1929		if (reg == PACKET0(p->adev->vcn.internal.data0, 0)) {1930			msg_lo = val;1931		} else if (reg == PACKET0(p->adev->vcn.internal.data1, 0)) {1932			msg_hi = val;1933		} else if (reg == PACKET0(p->adev->vcn.internal.cmd, 0) &&1934			   val == 0) {1935			r = vcn_v3_0_dec_msg(p, job,1936					     ((u64)msg_hi) << 32 | msg_lo);1937			if (r)1938				return r;1939		}1940	}1941	return 0;1942}1943 1944static const struct amdgpu_ring_funcs vcn_v3_0_dec_ring_vm_funcs = {1945	.type = AMDGPU_RING_TYPE_VCN_DEC,1946	.align_mask = 0xf,1947	.secure_submission_supported = true,1948	.get_rptr = vcn_v3_0_dec_ring_get_rptr,1949	.get_wptr = vcn_v3_0_dec_ring_get_wptr,1950	.set_wptr = vcn_v3_0_dec_ring_set_wptr,1951	.patch_cs_in_place = vcn_v3_0_ring_patch_cs_in_place,1952	.emit_frame_size =1953		SOC15_FLUSH_GPU_TLB_NUM_WREG * 6 +1954		SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 8 +1955		8 + /* vcn_v2_0_dec_ring_emit_vm_flush */1956		14 + 14 + /* vcn_v2_0_dec_ring_emit_fence x2 vm fence */1957		6,1958	.emit_ib_size = 8, /* vcn_v2_0_dec_ring_emit_ib */1959	.emit_ib = vcn_v2_0_dec_ring_emit_ib,1960	.emit_fence = vcn_v2_0_dec_ring_emit_fence,1961	.emit_vm_flush = vcn_v2_0_dec_ring_emit_vm_flush,1962	.test_ring = vcn_v2_0_dec_ring_test_ring,1963	.test_ib = amdgpu_vcn_dec_ring_test_ib,1964	.insert_nop = vcn_v2_0_dec_ring_insert_nop,1965	.insert_start = vcn_v2_0_dec_ring_insert_start,1966	.insert_end = vcn_v2_0_dec_ring_insert_end,1967	.pad_ib = amdgpu_ring_generic_pad_ib,1968	.begin_use = amdgpu_vcn_ring_begin_use,1969	.end_use = amdgpu_vcn_ring_end_use,1970	.emit_wreg = vcn_v2_0_dec_ring_emit_wreg,1971	.emit_reg_wait = vcn_v2_0_dec_ring_emit_reg_wait,1972	.emit_reg_write_reg_wait = amdgpu_ring_emit_reg_write_reg_wait_helper,1973};1974 1975/**1976 * vcn_v3_0_enc_ring_get_rptr - get enc read pointer1977 *1978 * @ring: amdgpu_ring pointer1979 *1980 * Returns the current hardware enc read pointer1981 */1982static uint64_t vcn_v3_0_enc_ring_get_rptr(struct amdgpu_ring *ring)1983{1984	struct amdgpu_device *adev = ring->adev;1985 1986	if (ring == &adev->vcn.inst[ring->me].ring_enc[0])1987		return RREG32_SOC15(VCN, ring->me, mmUVD_RB_RPTR);1988	else1989		return RREG32_SOC15(VCN, ring->me, mmUVD_RB_RPTR2);1990}1991 1992/**1993 * vcn_v3_0_enc_ring_get_wptr - get enc write pointer1994 *1995 * @ring: amdgpu_ring pointer1996 *1997 * Returns the current hardware enc write pointer1998 */1999static uint64_t vcn_v3_0_enc_ring_get_wptr(struct amdgpu_ring *ring)2000{2001	struct amdgpu_device *adev = ring->adev;2002 2003	if (ring == &adev->vcn.inst[ring->me].ring_enc[0]) {2004		if (ring->use_doorbell)2005			return *ring->wptr_cpu_addr;2006		else2007			return RREG32_SOC15(VCN, ring->me, mmUVD_RB_WPTR);2008	} else {2009		if (ring->use_doorbell)2010			return *ring->wptr_cpu_addr;2011		else2012			return RREG32_SOC15(VCN, ring->me, mmUVD_RB_WPTR2);2013	}2014}2015 2016/**2017 * vcn_v3_0_enc_ring_set_wptr - set enc write pointer2018 *2019 * @ring: amdgpu_ring pointer2020 *2021 * Commits the enc write pointer to the hardware2022 */2023static void vcn_v3_0_enc_ring_set_wptr(struct amdgpu_ring *ring)2024{2025	struct amdgpu_device *adev = ring->adev;2026 2027	if (ring == &adev->vcn.inst[ring->me].ring_enc[0]) {2028		if (ring->use_doorbell) {2029			*ring->wptr_cpu_addr = lower_32_bits(ring->wptr);2030			WDOORBELL32(ring->doorbell_index, lower_32_bits(ring->wptr));2031		} else {2032			WREG32_SOC15(VCN, ring->me, mmUVD_RB_WPTR, lower_32_bits(ring->wptr));2033		}2034	} else {2035		if (ring->use_doorbell) {2036			*ring->wptr_cpu_addr = lower_32_bits(ring->wptr);2037			WDOORBELL32(ring->doorbell_index, lower_32_bits(ring->wptr));2038		} else {2039			WREG32_SOC15(VCN, ring->me, mmUVD_RB_WPTR2, lower_32_bits(ring->wptr));2040		}2041	}2042}2043 2044static const struct amdgpu_ring_funcs vcn_v3_0_enc_ring_vm_funcs = {2045	.type = AMDGPU_RING_TYPE_VCN_ENC,2046	.align_mask = 0x3f,2047	.nop = VCN_ENC_CMD_NO_OP,2048	.get_rptr = vcn_v3_0_enc_ring_get_rptr,2049	.get_wptr = vcn_v3_0_enc_ring_get_wptr,2050	.set_wptr = vcn_v3_0_enc_ring_set_wptr,2051	.emit_frame_size =2052		SOC15_FLUSH_GPU_TLB_NUM_WREG * 3 +2053		SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 4 +2054		4 + /* vcn_v2_0_enc_ring_emit_vm_flush */2055		5 + 5 + /* vcn_v2_0_enc_ring_emit_fence x2 vm fence */2056		1, /* vcn_v2_0_enc_ring_insert_end */2057	.emit_ib_size = 5, /* vcn_v2_0_enc_ring_emit_ib */2058	.emit_ib = vcn_v2_0_enc_ring_emit_ib,2059	.emit_fence = vcn_v2_0_enc_ring_emit_fence,2060	.emit_vm_flush = vcn_v2_0_enc_ring_emit_vm_flush,2061	.test_ring = amdgpu_vcn_enc_ring_test_ring,2062	.test_ib = amdgpu_vcn_enc_ring_test_ib,2063	.insert_nop = amdgpu_ring_insert_nop,2064	.insert_end = vcn_v2_0_enc_ring_insert_end,2065	.pad_ib = amdgpu_ring_generic_pad_ib,2066	.begin_use = amdgpu_vcn_ring_begin_use,2067	.end_use = amdgpu_vcn_ring_end_use,2068	.emit_wreg = vcn_v2_0_enc_ring_emit_wreg,2069	.emit_reg_wait = vcn_v2_0_enc_ring_emit_reg_wait,2070	.emit_reg_write_reg_wait = amdgpu_ring_emit_reg_write_reg_wait_helper,2071};2072 2073static void vcn_v3_0_set_dec_ring_funcs(struct amdgpu_device *adev)2074{2075	int i;2076 2077	for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {2078		if (adev->vcn.harvest_config & (1 << i))2079			continue;2080 2081		if (!DEC_SW_RING_ENABLED)2082			adev->vcn.inst[i].ring_dec.funcs = &vcn_v3_0_dec_ring_vm_funcs;2083		else2084			adev->vcn.inst[i].ring_dec.funcs = &vcn_v3_0_dec_sw_ring_vm_funcs;2085		adev->vcn.inst[i].ring_dec.me = i;2086	}2087}2088 2089static void vcn_v3_0_set_enc_ring_funcs(struct amdgpu_device *adev)2090{2091	int i, j;2092 2093	for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {2094		if (adev->vcn.harvest_config & (1 << i))2095			continue;2096 2097		for (j = 0; j < adev->vcn.num_enc_rings; ++j) {2098			adev->vcn.inst[i].ring_enc[j].funcs = &vcn_v3_0_enc_ring_vm_funcs;2099			adev->vcn.inst[i].ring_enc[j].me = i;2100		}2101	}2102}2103 2104static bool vcn_v3_0_is_idle(void *handle)2105{2106	struct amdgpu_device *adev = (struct amdgpu_device *)handle;2107	int i, ret = 1;2108 2109	for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {2110		if (adev->vcn.harvest_config & (1 << i))2111			continue;2112 2113		ret &= (RREG32_SOC15(VCN, i, mmUVD_STATUS) == UVD_STATUS__IDLE);2114	}2115 2116	return ret;2117}2118 2119static int vcn_v3_0_wait_for_idle(void *handle)2120{2121	struct amdgpu_device *adev = (struct amdgpu_device *)handle;2122	int i, ret = 0;2123 2124	for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {2125		if (adev->vcn.harvest_config & (1 << i))2126			continue;2127 2128		ret = SOC15_WAIT_ON_RREG(VCN, i, mmUVD_STATUS, UVD_STATUS__IDLE,2129			UVD_STATUS__IDLE);2130		if (ret)2131			return ret;2132	}2133 2134	return ret;2135}2136 2137static int vcn_v3_0_set_clockgating_state(void *handle,2138					  enum amd_clockgating_state state)2139{2140	struct amdgpu_device *adev = (struct amdgpu_device *)handle;2141	bool enable = state == AMD_CG_STATE_GATE;2142	int i;2143 2144	for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {2145		if (adev->vcn.harvest_config & (1 << i))2146			continue;2147 2148		if (enable) {2149			if (RREG32_SOC15(VCN, i, mmUVD_STATUS) != UVD_STATUS__IDLE)2150				return -EBUSY;2151			vcn_v3_0_enable_clock_gating(adev, i);2152		} else {2153			vcn_v3_0_disable_clock_gating(adev, i);2154		}2155	}2156 2157	return 0;2158}2159 2160static int vcn_v3_0_set_powergating_state(void *handle,2161					  enum amd_powergating_state state)2162{2163	struct amdgpu_device *adev = (struct amdgpu_device *)handle;2164	int ret;2165 2166	/* for SRIOV, guest should not control VCN Power-gating2167	 * MMSCH FW should control Power-gating and clock-gating2168	 * guest should avoid touching CGC and PG2169	 */2170	if (amdgpu_sriov_vf(adev)) {2171		adev->vcn.cur_state = AMD_PG_STATE_UNGATE;2172		return 0;2173	}2174 2175	if (state == adev->vcn.cur_state)2176		return 0;2177 2178	if (state == AMD_PG_STATE_GATE)2179		ret = vcn_v3_0_stop(adev);2180	else2181		ret = vcn_v3_0_start(adev);2182 2183	if (!ret)2184		adev->vcn.cur_state = state;2185 2186	return ret;2187}2188 2189static int vcn_v3_0_set_interrupt_state(struct amdgpu_device *adev,2190					struct amdgpu_irq_src *source,2191					unsigned type,2192					enum amdgpu_interrupt_state state)2193{2194	return 0;2195}2196 2197static int vcn_v3_0_process_interrupt(struct amdgpu_device *adev,2198				      struct amdgpu_irq_src *source,2199				      struct amdgpu_iv_entry *entry)2200{2201	uint32_t ip_instance;2202 2203	switch (entry->client_id) {2204	case SOC15_IH_CLIENTID_VCN:2205		ip_instance = 0;2206		break;2207	case SOC15_IH_CLIENTID_VCN1:2208		ip_instance = 1;2209		break;2210	default:2211		DRM_ERROR("Unhandled client id: %d\n", entry->client_id);2212		return 0;2213	}2214 2215	DRM_DEBUG("IH: VCN TRAP\n");2216 2217	switch (entry->src_id) {2218	case VCN_2_0__SRCID__UVD_SYSTEM_MESSAGE_INTERRUPT:2219		amdgpu_fence_process(&adev->vcn.inst[ip_instance].ring_dec);2220		break;2221	case VCN_2_0__SRCID__UVD_ENC_GENERAL_PURPOSE:2222		amdgpu_fence_process(&adev->vcn.inst[ip_instance].ring_enc[0]);2223		break;2224	case VCN_2_0__SRCID__UVD_ENC_LOW_LATENCY:2225		amdgpu_fence_process(&adev->vcn.inst[ip_instance].ring_enc[1]);2226		break;2227	default:2228		DRM_ERROR("Unhandled interrupt: %d %d\n",2229			  entry->src_id, entry->src_data[0]);2230		break;2231	}2232 2233	return 0;2234}2235 2236static const struct amdgpu_irq_src_funcs vcn_v3_0_irq_funcs = {2237	.set = vcn_v3_0_set_interrupt_state,2238	.process = vcn_v3_0_process_interrupt,2239};2240 2241static void vcn_v3_0_set_irq_funcs(struct amdgpu_device *adev)2242{2243	int i;2244 2245	for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {2246		if (adev->vcn.harvest_config & (1 << i))2247			continue;2248 2249		adev->vcn.inst[i].irq.num_types = adev->vcn.num_enc_rings + 1;2250		adev->vcn.inst[i].irq.funcs = &vcn_v3_0_irq_funcs;2251	}2252}2253 2254static void vcn_v3_0_print_ip_state(void *handle, struct drm_printer *p)2255{2256	struct amdgpu_device *adev = (struct amdgpu_device *)handle;2257	int i, j;2258	uint32_t reg_count = ARRAY_SIZE(vcn_reg_list_3_0);2259	uint32_t inst_off;2260	bool is_powered;2261 2262	if (!adev->vcn.ip_dump)2263		return;2264 2265	drm_printf(p, "num_instances:%d\n", adev->vcn.num_vcn_inst);2266	for (i = 0; i < adev->vcn.num_vcn_inst; i++) {2267		if (adev->vcn.harvest_config & (1 << i)) {2268			drm_printf(p, "\nHarvested Instance:VCN%d Skipping dump\n", i);2269			continue;2270		}2271 2272		inst_off = i * reg_count;2273		is_powered = (adev->vcn.ip_dump[inst_off] &2274			      UVD_POWER_STATUS__UVD_POWER_STATUS_MASK) != 1;2275 2276		if (is_powered) {2277			drm_printf(p, "\nActive Instance:VCN%d\n", i);2278			for (j = 0; j < reg_count; j++)2279				drm_printf(p, "%-50s \t 0x%08x\n", vcn_reg_list_3_0[j].reg_name,2280					   adev->vcn.ip_dump[inst_off + j]);2281		} else {2282			drm_printf(p, "\nInactive Instance:VCN%d\n", i);2283		}2284	}2285}2286 2287static void vcn_v3_0_dump_ip_state(void *handle)2288{2289	struct amdgpu_device *adev = (struct amdgpu_device *)handle;2290	int i, j;2291	bool is_powered;2292	uint32_t inst_off;2293	uint32_t reg_count = ARRAY_SIZE(vcn_reg_list_3_0);2294 2295	if (!adev->vcn.ip_dump)2296		return;2297 2298	for (i = 0; i < adev->vcn.num_vcn_inst; i++) {2299		if (adev->vcn.harvest_config & (1 << i))2300			continue;2301 2302		inst_off = i * reg_count;2303		/* mmUVD_POWER_STATUS is always readable and is first element of the array */2304		adev->vcn.ip_dump[inst_off] = RREG32_SOC15(VCN, i, mmUVD_POWER_STATUS);2305		is_powered = (adev->vcn.ip_dump[inst_off] &2306			      UVD_POWER_STATUS__UVD_POWER_STATUS_MASK) != 1;2307 2308		if (is_powered)2309			for (j = 1; j < reg_count; j++)2310				adev->vcn.ip_dump[inst_off + j] =2311					RREG32(SOC15_REG_ENTRY_OFFSET_INST(vcn_reg_list_3_0[j], i));2312	}2313}2314 2315static const struct amd_ip_funcs vcn_v3_0_ip_funcs = {2316	.name = "vcn_v3_0",2317	.early_init = vcn_v3_0_early_init,2318	.late_init = NULL,2319	.sw_init = vcn_v3_0_sw_init,2320	.sw_fini = vcn_v3_0_sw_fini,2321	.hw_init = vcn_v3_0_hw_init,2322	.hw_fini = vcn_v3_0_hw_fini,2323	.suspend = vcn_v3_0_suspend,2324	.resume = vcn_v3_0_resume,2325	.is_idle = vcn_v3_0_is_idle,2326	.wait_for_idle = vcn_v3_0_wait_for_idle,2327	.check_soft_reset = NULL,2328	.pre_soft_reset = NULL,2329	.soft_reset = NULL,2330	.post_soft_reset = NULL,2331	.set_clockgating_state = vcn_v3_0_set_clockgating_state,2332	.set_powergating_state = vcn_v3_0_set_powergating_state,2333	.dump_ip_state = vcn_v3_0_dump_ip_state,2334	.print_ip_state = vcn_v3_0_print_ip_state,2335};2336 2337const struct amdgpu_ip_block_version vcn_v3_0_ip_block = {2338	.type = AMD_IP_BLOCK_TYPE_VCN,2339	.major = 3,2340	.minor = 0,2341	.rev = 0,2342	.funcs = &vcn_v3_0_ip_funcs,2343};2344