2307 lines · c
1/*2 * Copyright 2021 Advanced Micro Devices, Inc.3 *4 * Permission is hereby granted, free of charge, to any person obtaining a5 * copy of this software and associated documentation files (the "Software"),6 * to deal in the Software without restriction, including without limitation7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,8 * and/or sell copies of the Software, and to permit persons to whom the9 * Software is furnished to do so, subject to the following conditions:10 *11 * The above copyright notice and this permission notice shall be included in12 * all copies or substantial portions of the Software.13 *14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR20 * OTHER DEALINGS IN THE SOFTWARE.21 *22 */23 24#include <linux/firmware.h>25#include "amdgpu.h"26#include "amdgpu_vcn.h"27#include "amdgpu_pm.h"28#include "amdgpu_cs.h"29#include "soc15.h"30#include "soc15d.h"31#include "soc15_hw_ip.h"32#include "vcn_v2_0.h"33#include "mmsch_v4_0.h"34#include "vcn_v4_0.h"35 36#include "vcn/vcn_4_0_0_offset.h"37#include "vcn/vcn_4_0_0_sh_mask.h"38#include "ivsrcid/vcn/irqsrcs_vcn_4_0.h"39 40#include <drm/drm_drv.h>41 42#define mmUVD_DPG_LMA_CTL regUVD_DPG_LMA_CTL43#define mmUVD_DPG_LMA_CTL_BASE_IDX regUVD_DPG_LMA_CTL_BASE_IDX44#define mmUVD_DPG_LMA_DATA regUVD_DPG_LMA_DATA45#define mmUVD_DPG_LMA_DATA_BASE_IDX regUVD_DPG_LMA_DATA_BASE_IDX46 47#define VCN_VID_SOC_ADDRESS_2_0 0x1fb0048#define VCN1_VID_SOC_ADDRESS_3_0 0x4830049 50#define VCN_HARVEST_MMSCH 051 52#define RDECODE_MSG_CREATE 0x0000000053#define RDECODE_MESSAGE_CREATE 0x0000000154 55static const struct amdgpu_hwip_reg_entry vcn_reg_list_4_0[] = {56 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_POWER_STATUS),57 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_STATUS),58 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_CONTEXT_ID),59 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_CONTEXT_ID2),60 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_GPCOM_VCPU_DATA0),61 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_GPCOM_VCPU_DATA1),62 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_GPCOM_VCPU_CMD),63 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_BASE_HI),64 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_BASE_LO),65 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_BASE_HI2),66 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_BASE_LO2),67 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_BASE_HI3),68 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_BASE_LO3),69 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_BASE_HI4),70 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_BASE_LO4),71 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_RPTR),72 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_WPTR),73 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_RPTR2),74 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_WPTR2),75 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_RPTR3),76 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_WPTR3),77 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_RPTR4),78 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_WPTR4),79 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_SIZE),80 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_SIZE2),81 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_SIZE3),82 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_SIZE4),83 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_PGFSM_CONFIG),84 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_PGFSM_STATUS),85 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_DPG_LMA_CTL),86 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_DPG_LMA_DATA),87 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_DPG_LMA_MASK),88 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_DPG_PAUSE)89};90 91static int amdgpu_ih_clientid_vcns[] = {92 SOC15_IH_CLIENTID_VCN,93 SOC15_IH_CLIENTID_VCN194};95 96static int vcn_v4_0_start_sriov(struct amdgpu_device *adev);97static void vcn_v4_0_set_unified_ring_funcs(struct amdgpu_device *adev);98static void vcn_v4_0_set_irq_funcs(struct amdgpu_device *adev);99static int vcn_v4_0_set_powergating_state(void *handle,100 enum amd_powergating_state state);101static int vcn_v4_0_pause_dpg_mode(struct amdgpu_device *adev,102 int inst_idx, struct dpg_pause_state *new_state);103static void vcn_v4_0_unified_ring_set_wptr(struct amdgpu_ring *ring);104static void vcn_v4_0_set_ras_funcs(struct amdgpu_device *adev);105 106/**107 * vcn_v4_0_early_init - set function pointers and load microcode108 *109 * @handle: amdgpu_device pointer110 *111 * Set ring and irq function pointers112 * Load microcode from filesystem113 */114static int vcn_v4_0_early_init(void *handle)115{116 struct amdgpu_device *adev = (struct amdgpu_device *)handle;117 int i;118 119 if (amdgpu_sriov_vf(adev)) {120 adev->vcn.harvest_config = VCN_HARVEST_MMSCH;121 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {122 if (amdgpu_vcn_is_disabled_vcn(adev, VCN_ENCODE_RING, i)) {123 adev->vcn.harvest_config |= 1 << i;124 dev_info(adev->dev, "VCN%d is disabled by hypervisor\n", i);125 }126 }127 }128 129 /* re-use enc ring as unified ring */130 adev->vcn.num_enc_rings = 1;131 132 vcn_v4_0_set_unified_ring_funcs(adev);133 vcn_v4_0_set_irq_funcs(adev);134 vcn_v4_0_set_ras_funcs(adev);135 136 return amdgpu_vcn_early_init(adev);137}138 139static int vcn_v4_0_fw_shared_init(struct amdgpu_device *adev, int inst_idx)140{141 volatile struct amdgpu_vcn4_fw_shared *fw_shared;142 143 fw_shared = adev->vcn.inst[inst_idx].fw_shared.cpu_addr;144 fw_shared->present_flag_0 = cpu_to_le32(AMDGPU_FW_SHARED_FLAG_0_UNIFIED_QUEUE);145 fw_shared->sq.is_enabled = 1;146 147 fw_shared->present_flag_0 |= cpu_to_le32(AMDGPU_VCN_SMU_DPM_INTERFACE_FLAG);148 fw_shared->smu_dpm_interface.smu_interface_type = (adev->flags & AMD_IS_APU) ?149 AMDGPU_VCN_SMU_DPM_INTERFACE_APU : AMDGPU_VCN_SMU_DPM_INTERFACE_DGPU;150 151 if (amdgpu_ip_version(adev, VCN_HWIP, 0) ==152 IP_VERSION(4, 0, 2)) {153 fw_shared->present_flag_0 |= AMDGPU_FW_SHARED_FLAG_0_DRM_KEY_INJECT;154 fw_shared->drm_key_wa.method =155 AMDGPU_DRM_KEY_INJECT_WORKAROUND_VCNFW_ASD_HANDSHAKING;156 }157 158 if (amdgpu_vcnfw_log)159 amdgpu_vcn_fwlog_init(&adev->vcn.inst[inst_idx]);160 161 return 0;162}163 164/**165 * vcn_v4_0_sw_init - sw init for VCN block166 *167 * @handle: amdgpu_device pointer168 *169 * Load firmware and sw initialization170 */171static int vcn_v4_0_sw_init(void *handle)172{173 struct amdgpu_ring *ring;174 struct amdgpu_device *adev = (struct amdgpu_device *)handle;175 int i, r;176 uint32_t reg_count = ARRAY_SIZE(vcn_reg_list_4_0);177 uint32_t *ptr;178 179 r = amdgpu_vcn_sw_init(adev);180 if (r)181 return r;182 183 amdgpu_vcn_setup_ucode(adev);184 185 r = amdgpu_vcn_resume(adev);186 if (r)187 return r;188 189 for (i = 0; i < adev->vcn.num_vcn_inst; i++) {190 if (adev->vcn.harvest_config & (1 << i))191 continue;192 193 /* Init instance 0 sched_score to 1, so it's scheduled after other instances */194 if (i == 0)195 atomic_set(&adev->vcn.inst[i].sched_score, 1);196 else197 atomic_set(&adev->vcn.inst[i].sched_score, 0);198 199 /* VCN UNIFIED TRAP */200 r = amdgpu_irq_add_id(adev, amdgpu_ih_clientid_vcns[i],201 VCN_4_0__SRCID__UVD_ENC_GENERAL_PURPOSE, &adev->vcn.inst[i].irq);202 if (r)203 return r;204 205 /* VCN POISON TRAP */206 r = amdgpu_irq_add_id(adev, amdgpu_ih_clientid_vcns[i],207 VCN_4_0__SRCID_UVD_POISON, &adev->vcn.inst[i].ras_poison_irq);208 if (r)209 return r;210 211 ring = &adev->vcn.inst[i].ring_enc[0];212 ring->use_doorbell = true;213 if (amdgpu_sriov_vf(adev))214 ring->doorbell_index = (adev->doorbell_index.vcn.vcn_ring0_1 << 1) + i * (adev->vcn.num_enc_rings + 1) + 1;215 else216 ring->doorbell_index = (adev->doorbell_index.vcn.vcn_ring0_1 << 1) + 2 + 8 * i;217 ring->vm_hub = AMDGPU_MMHUB0(0);218 sprintf(ring->name, "vcn_unified_%d", i);219 220 r = amdgpu_ring_init(adev, ring, 512, &adev->vcn.inst[i].irq, 0,221 AMDGPU_RING_PRIO_0, &adev->vcn.inst[i].sched_score);222 if (r)223 return r;224 225 vcn_v4_0_fw_shared_init(adev, i);226 }227 228 if (amdgpu_sriov_vf(adev)) {229 r = amdgpu_virt_alloc_mm_table(adev);230 if (r)231 return r;232 }233 234 if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG)235 adev->vcn.pause_dpg_mode = vcn_v4_0_pause_dpg_mode;236 237 r = amdgpu_vcn_ras_sw_init(adev);238 if (r)239 return r;240 241 /* Allocate memory for VCN IP Dump buffer */242 ptr = kcalloc(adev->vcn.num_vcn_inst * reg_count, sizeof(uint32_t), GFP_KERNEL);243 if (!ptr) {244 DRM_ERROR("Failed to allocate memory for VCN IP Dump\n");245 adev->vcn.ip_dump = NULL;246 } else {247 adev->vcn.ip_dump = ptr;248 }249 250 return 0;251}252 253/**254 * vcn_v4_0_sw_fini - sw fini for VCN block255 *256 * @handle: amdgpu_device pointer257 *258 * VCN suspend and free up sw allocation259 */260static int vcn_v4_0_sw_fini(void *handle)261{262 struct amdgpu_device *adev = (struct amdgpu_device *)handle;263 int i, r, idx;264 265 if (drm_dev_enter(adev_to_drm(adev), &idx)) {266 for (i = 0; i < adev->vcn.num_vcn_inst; i++) {267 volatile struct amdgpu_vcn4_fw_shared *fw_shared;268 269 if (adev->vcn.harvest_config & (1 << i))270 continue;271 272 fw_shared = adev->vcn.inst[i].fw_shared.cpu_addr;273 fw_shared->present_flag_0 = 0;274 fw_shared->sq.is_enabled = 0;275 }276 277 drm_dev_exit(idx);278 }279 280 if (amdgpu_sriov_vf(adev))281 amdgpu_virt_free_mm_table(adev);282 283 r = amdgpu_vcn_suspend(adev);284 if (r)285 return r;286 287 r = amdgpu_vcn_sw_fini(adev);288 289 kfree(adev->vcn.ip_dump);290 291 return r;292}293 294/**295 * vcn_v4_0_hw_init - start and test VCN block296 *297 * @handle: amdgpu_device pointer298 *299 * Initialize the hardware, boot up the VCPU and do some testing300 */301static int vcn_v4_0_hw_init(void *handle)302{303 struct amdgpu_device *adev = (struct amdgpu_device *)handle;304 struct amdgpu_ring *ring;305 int i, r;306 307 if (amdgpu_sriov_vf(adev)) {308 r = vcn_v4_0_start_sriov(adev);309 if (r)310 return r;311 312 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {313 if (adev->vcn.harvest_config & (1 << i))314 continue;315 316 ring = &adev->vcn.inst[i].ring_enc[0];317 ring->wptr = 0;318 ring->wptr_old = 0;319 vcn_v4_0_unified_ring_set_wptr(ring);320 ring->sched.ready = true;321 }322 } else {323 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {324 if (adev->vcn.harvest_config & (1 << i))325 continue;326 327 ring = &adev->vcn.inst[i].ring_enc[0];328 329 adev->nbio.funcs->vcn_doorbell_range(adev, ring->use_doorbell,330 ((adev->doorbell_index.vcn.vcn_ring0_1 << 1) + 8 * i), i);331 332 r = amdgpu_ring_test_helper(ring);333 if (r)334 return r;335 }336 }337 338 return 0;339}340 341/**342 * vcn_v4_0_hw_fini - stop the hardware block343 *344 * @handle: amdgpu_device pointer345 *346 * Stop the VCN block, mark ring as not ready any more347 */348static int vcn_v4_0_hw_fini(void *handle)349{350 struct amdgpu_device *adev = (struct amdgpu_device *)handle;351 int i;352 353 cancel_delayed_work_sync(&adev->vcn.idle_work);354 355 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {356 if (adev->vcn.harvest_config & (1 << i))357 continue;358 if (!amdgpu_sriov_vf(adev)) {359 if ((adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG) ||360 (adev->vcn.cur_state != AMD_PG_STATE_GATE &&361 RREG32_SOC15(VCN, i, regUVD_STATUS))) {362 vcn_v4_0_set_powergating_state(adev, AMD_PG_STATE_GATE);363 }364 }365 if (amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__VCN))366 amdgpu_irq_put(adev, &adev->vcn.inst[i].ras_poison_irq, 0);367 }368 369 return 0;370}371 372/**373 * vcn_v4_0_suspend - suspend VCN block374 *375 * @handle: amdgpu_device pointer376 *377 * HW fini and suspend VCN block378 */379static int vcn_v4_0_suspend(void *handle)380{381 int r;382 struct amdgpu_device *adev = (struct amdgpu_device *)handle;383 384 r = vcn_v4_0_hw_fini(adev);385 if (r)386 return r;387 388 r = amdgpu_vcn_suspend(adev);389 390 return r;391}392 393/**394 * vcn_v4_0_resume - resume VCN block395 *396 * @handle: amdgpu_device pointer397 *398 * Resume firmware and hw init VCN block399 */400static int vcn_v4_0_resume(void *handle)401{402 int r;403 struct amdgpu_device *adev = (struct amdgpu_device *)handle;404 405 r = amdgpu_vcn_resume(adev);406 if (r)407 return r;408 409 r = vcn_v4_0_hw_init(adev);410 411 return r;412}413 414/**415 * vcn_v4_0_mc_resume - memory controller programming416 *417 * @adev: amdgpu_device pointer418 * @inst: instance number419 *420 * Let the VCN memory controller know it's offsets421 */422static void vcn_v4_0_mc_resume(struct amdgpu_device *adev, int inst)423{424 uint32_t offset, size;425 const struct common_firmware_header *hdr;426 427 hdr = (const struct common_firmware_header *)adev->vcn.fw[inst]->data;428 size = AMDGPU_GPU_PAGE_ALIGN(le32_to_cpu(hdr->ucode_size_bytes) + 8);429 430 /* cache window 0: fw */431 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {432 WREG32_SOC15(VCN, inst, regUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW,433 (adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + inst].tmr_mc_addr_lo));434 WREG32_SOC15(VCN, inst, regUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH,435 (adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + inst].tmr_mc_addr_hi));436 WREG32_SOC15(VCN, inst, regUVD_VCPU_CACHE_OFFSET0, 0);437 offset = 0;438 } else {439 WREG32_SOC15(VCN, inst, regUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW,440 lower_32_bits(adev->vcn.inst[inst].gpu_addr));441 WREG32_SOC15(VCN, inst, regUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH,442 upper_32_bits(adev->vcn.inst[inst].gpu_addr));443 offset = size;444 WREG32_SOC15(VCN, inst, regUVD_VCPU_CACHE_OFFSET0, AMDGPU_UVD_FIRMWARE_OFFSET >> 3);445 }446 WREG32_SOC15(VCN, inst, regUVD_VCPU_CACHE_SIZE0, size);447 448 /* cache window 1: stack */449 WREG32_SOC15(VCN, inst, regUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW,450 lower_32_bits(adev->vcn.inst[inst].gpu_addr + offset));451 WREG32_SOC15(VCN, inst, regUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH,452 upper_32_bits(adev->vcn.inst[inst].gpu_addr + offset));453 WREG32_SOC15(VCN, inst, regUVD_VCPU_CACHE_OFFSET1, 0);454 WREG32_SOC15(VCN, inst, regUVD_VCPU_CACHE_SIZE1, AMDGPU_VCN_STACK_SIZE);455 456 /* cache window 2: context */457 WREG32_SOC15(VCN, inst, regUVD_LMI_VCPU_CACHE2_64BIT_BAR_LOW,458 lower_32_bits(adev->vcn.inst[inst].gpu_addr + offset + AMDGPU_VCN_STACK_SIZE));459 WREG32_SOC15(VCN, inst, regUVD_LMI_VCPU_CACHE2_64BIT_BAR_HIGH,460 upper_32_bits(adev->vcn.inst[inst].gpu_addr + offset + AMDGPU_VCN_STACK_SIZE));461 WREG32_SOC15(VCN, inst, regUVD_VCPU_CACHE_OFFSET2, 0);462 WREG32_SOC15(VCN, inst, regUVD_VCPU_CACHE_SIZE2, AMDGPU_VCN_CONTEXT_SIZE);463 464 /* non-cache window */465 WREG32_SOC15(VCN, inst, regUVD_LMI_VCPU_NC0_64BIT_BAR_LOW,466 lower_32_bits(adev->vcn.inst[inst].fw_shared.gpu_addr));467 WREG32_SOC15(VCN, inst, regUVD_LMI_VCPU_NC0_64BIT_BAR_HIGH,468 upper_32_bits(adev->vcn.inst[inst].fw_shared.gpu_addr));469 WREG32_SOC15(VCN, inst, regUVD_VCPU_NONCACHE_OFFSET0, 0);470 WREG32_SOC15(VCN, inst, regUVD_VCPU_NONCACHE_SIZE0,471 AMDGPU_GPU_PAGE_ALIGN(sizeof(struct amdgpu_vcn4_fw_shared)));472}473 474/**475 * vcn_v4_0_mc_resume_dpg_mode - memory controller programming for dpg mode476 *477 * @adev: amdgpu_device pointer478 * @inst_idx: instance number index479 * @indirect: indirectly write sram480 *481 * Let the VCN memory controller know it's offsets with dpg mode482 */483static void vcn_v4_0_mc_resume_dpg_mode(struct amdgpu_device *adev, int inst_idx, bool indirect)484{485 uint32_t offset, size;486 const struct common_firmware_header *hdr;487 hdr = (const struct common_firmware_header *)adev->vcn.fw[inst_idx]->data;488 size = AMDGPU_GPU_PAGE_ALIGN(le32_to_cpu(hdr->ucode_size_bytes) + 8);489 490 /* cache window 0: fw */491 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {492 if (!indirect) {493 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(494 VCN, inst_idx, regUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW),495 (adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + inst_idx].tmr_mc_addr_lo), 0, indirect);496 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(497 VCN, inst_idx, regUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH),498 (adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + inst_idx].tmr_mc_addr_hi), 0, indirect);499 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(500 VCN, inst_idx, regUVD_VCPU_CACHE_OFFSET0), 0, 0, indirect);501 } else {502 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(503 VCN, inst_idx, regUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW), 0, 0, indirect);504 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(505 VCN, inst_idx, regUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH), 0, 0, indirect);506 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(507 VCN, inst_idx, regUVD_VCPU_CACHE_OFFSET0), 0, 0, indirect);508 }509 offset = 0;510 } else {511 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(512 VCN, inst_idx, regUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW),513 lower_32_bits(adev->vcn.inst[inst_idx].gpu_addr), 0, indirect);514 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(515 VCN, inst_idx, regUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH),516 upper_32_bits(adev->vcn.inst[inst_idx].gpu_addr), 0, indirect);517 offset = size;518 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(519 VCN, inst_idx, regUVD_VCPU_CACHE_OFFSET0),520 AMDGPU_UVD_FIRMWARE_OFFSET >> 3, 0, indirect);521 }522 523 if (!indirect)524 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(525 VCN, inst_idx, regUVD_VCPU_CACHE_SIZE0), size, 0, indirect);526 else527 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(528 VCN, inst_idx, regUVD_VCPU_CACHE_SIZE0), 0, 0, indirect);529 530 /* cache window 1: stack */531 if (!indirect) {532 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(533 VCN, inst_idx, regUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW),534 lower_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset), 0, indirect);535 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(536 VCN, inst_idx, regUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH),537 upper_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset), 0, indirect);538 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(539 VCN, inst_idx, regUVD_VCPU_CACHE_OFFSET1), 0, 0, indirect);540 } else {541 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(542 VCN, inst_idx, regUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW), 0, 0, indirect);543 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(544 VCN, inst_idx, regUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH), 0, 0, indirect);545 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(546 VCN, inst_idx, regUVD_VCPU_CACHE_OFFSET1), 0, 0, indirect);547 }548 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(549 VCN, inst_idx, regUVD_VCPU_CACHE_SIZE1), AMDGPU_VCN_STACK_SIZE, 0, indirect);550 551 /* cache window 2: context */552 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(553 VCN, inst_idx, regUVD_LMI_VCPU_CACHE2_64BIT_BAR_LOW),554 lower_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset + AMDGPU_VCN_STACK_SIZE), 0, indirect);555 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(556 VCN, inst_idx, regUVD_LMI_VCPU_CACHE2_64BIT_BAR_HIGH),557 upper_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset + AMDGPU_VCN_STACK_SIZE), 0, indirect);558 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(559 VCN, inst_idx, regUVD_VCPU_CACHE_OFFSET2), 0, 0, indirect);560 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(561 VCN, inst_idx, regUVD_VCPU_CACHE_SIZE2), AMDGPU_VCN_CONTEXT_SIZE, 0, indirect);562 563 /* non-cache window */564 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(565 VCN, inst_idx, regUVD_LMI_VCPU_NC0_64BIT_BAR_LOW),566 lower_32_bits(adev->vcn.inst[inst_idx].fw_shared.gpu_addr), 0, indirect);567 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(568 VCN, inst_idx, regUVD_LMI_VCPU_NC0_64BIT_BAR_HIGH),569 upper_32_bits(adev->vcn.inst[inst_idx].fw_shared.gpu_addr), 0, indirect);570 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(571 VCN, inst_idx, regUVD_VCPU_NONCACHE_OFFSET0), 0, 0, indirect);572 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(573 VCN, inst_idx, regUVD_VCPU_NONCACHE_SIZE0),574 AMDGPU_GPU_PAGE_ALIGN(sizeof(struct amdgpu_vcn4_fw_shared)), 0, indirect);575 576 /* VCN global tiling registers */577 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(578 VCN, 0, regUVD_GFX10_ADDR_CONFIG), adev->gfx.config.gb_addr_config, 0, indirect);579}580 581/**582 * vcn_v4_0_disable_static_power_gating - disable VCN static power gating583 *584 * @adev: amdgpu_device pointer585 * @inst: instance number586 *587 * Disable static power gating for VCN block588 */589static void vcn_v4_0_disable_static_power_gating(struct amdgpu_device *adev, int inst)590{591 uint32_t data = 0;592 593 if (adev->pg_flags & AMD_PG_SUPPORT_VCN) {594 data = (1 << UVD_PGFSM_CONFIG__UVDM_PWR_CONFIG__SHIFT595 | 1 << UVD_PGFSM_CONFIG__UVDS_PWR_CONFIG__SHIFT596 | 1 << UVD_PGFSM_CONFIG__UVDLM_PWR_CONFIG__SHIFT597 | 2 << UVD_PGFSM_CONFIG__UVDF_PWR_CONFIG__SHIFT598 | 2 << UVD_PGFSM_CONFIG__UVDTC_PWR_CONFIG__SHIFT599 | 2 << UVD_PGFSM_CONFIG__UVDB_PWR_CONFIG__SHIFT600 | 2 << UVD_PGFSM_CONFIG__UVDTA_PWR_CONFIG__SHIFT601 | 2 << UVD_PGFSM_CONFIG__UVDTD_PWR_CONFIG__SHIFT602 | 2 << UVD_PGFSM_CONFIG__UVDTE_PWR_CONFIG__SHIFT603 | 2 << UVD_PGFSM_CONFIG__UVDE_PWR_CONFIG__SHIFT604 | 2 << UVD_PGFSM_CONFIG__UVDAB_PWR_CONFIG__SHIFT605 | 2 << UVD_PGFSM_CONFIG__UVDTB_PWR_CONFIG__SHIFT606 | 2 << UVD_PGFSM_CONFIG__UVDNA_PWR_CONFIG__SHIFT607 | 2 << UVD_PGFSM_CONFIG__UVDNB_PWR_CONFIG__SHIFT);608 609 WREG32_SOC15(VCN, inst, regUVD_PGFSM_CONFIG, data);610 SOC15_WAIT_ON_RREG(VCN, inst, regUVD_PGFSM_STATUS,611 UVD_PGFSM_STATUS__UVDM_UVDU_UVDLM_PWR_ON_3_0, 0x3F3FFFFF);612 } else {613 uint32_t value;614 615 value = (inst) ? 0x2200800 : 0;616 data = (1 << UVD_PGFSM_CONFIG__UVDM_PWR_CONFIG__SHIFT617 | 1 << UVD_PGFSM_CONFIG__UVDS_PWR_CONFIG__SHIFT618 | 1 << UVD_PGFSM_CONFIG__UVDLM_PWR_CONFIG__SHIFT619 | 1 << UVD_PGFSM_CONFIG__UVDF_PWR_CONFIG__SHIFT620 | 1 << UVD_PGFSM_CONFIG__UVDTC_PWR_CONFIG__SHIFT621 | 1 << UVD_PGFSM_CONFIG__UVDB_PWR_CONFIG__SHIFT622 | 1 << UVD_PGFSM_CONFIG__UVDTA_PWR_CONFIG__SHIFT623 | 1 << UVD_PGFSM_CONFIG__UVDTD_PWR_CONFIG__SHIFT624 | 1 << UVD_PGFSM_CONFIG__UVDTE_PWR_CONFIG__SHIFT625 | 1 << UVD_PGFSM_CONFIG__UVDE_PWR_CONFIG__SHIFT626 | 1 << UVD_PGFSM_CONFIG__UVDAB_PWR_CONFIG__SHIFT627 | 1 << UVD_PGFSM_CONFIG__UVDTB_PWR_CONFIG__SHIFT628 | 1 << UVD_PGFSM_CONFIG__UVDNA_PWR_CONFIG__SHIFT629 | 1 << UVD_PGFSM_CONFIG__UVDNB_PWR_CONFIG__SHIFT);630 631 WREG32_SOC15(VCN, inst, regUVD_PGFSM_CONFIG, data);632 SOC15_WAIT_ON_RREG(VCN, inst, regUVD_PGFSM_STATUS, value, 0x3F3FFFFF);633 }634 635 data = RREG32_SOC15(VCN, inst, regUVD_POWER_STATUS);636 data &= ~0x103;637 if (adev->pg_flags & AMD_PG_SUPPORT_VCN)638 data |= UVD_PGFSM_CONFIG__UVDM_UVDU_PWR_ON |639 UVD_POWER_STATUS__UVD_PG_EN_MASK;640 641 WREG32_SOC15(VCN, inst, regUVD_POWER_STATUS, data);642 643 return;644}645 646/**647 * vcn_v4_0_enable_static_power_gating - enable VCN static power gating648 *649 * @adev: amdgpu_device pointer650 * @inst: instance number651 *652 * Enable static power gating for VCN block653 */654static void vcn_v4_0_enable_static_power_gating(struct amdgpu_device *adev, int inst)655{656 uint32_t data;657 658 if (adev->pg_flags & AMD_PG_SUPPORT_VCN) {659 /* Before power off, this indicator has to be turned on */660 data = RREG32_SOC15(VCN, inst, regUVD_POWER_STATUS);661 data &= ~UVD_POWER_STATUS__UVD_POWER_STATUS_MASK;662 data |= UVD_POWER_STATUS__UVD_POWER_STATUS_TILES_OFF;663 WREG32_SOC15(VCN, inst, regUVD_POWER_STATUS, data);664 665 data = (2 << UVD_PGFSM_CONFIG__UVDM_PWR_CONFIG__SHIFT666 | 2 << UVD_PGFSM_CONFIG__UVDS_PWR_CONFIG__SHIFT667 | 2 << UVD_PGFSM_CONFIG__UVDF_PWR_CONFIG__SHIFT668 | 2 << UVD_PGFSM_CONFIG__UVDTC_PWR_CONFIG__SHIFT669 | 2 << UVD_PGFSM_CONFIG__UVDB_PWR_CONFIG__SHIFT670 | 2 << UVD_PGFSM_CONFIG__UVDTA_PWR_CONFIG__SHIFT671 | 2 << UVD_PGFSM_CONFIG__UVDLM_PWR_CONFIG__SHIFT672 | 2 << UVD_PGFSM_CONFIG__UVDTD_PWR_CONFIG__SHIFT673 | 2 << UVD_PGFSM_CONFIG__UVDTE_PWR_CONFIG__SHIFT674 | 2 << UVD_PGFSM_CONFIG__UVDE_PWR_CONFIG__SHIFT675 | 2 << UVD_PGFSM_CONFIG__UVDAB_PWR_CONFIG__SHIFT676 | 2 << UVD_PGFSM_CONFIG__UVDTB_PWR_CONFIG__SHIFT677 | 2 << UVD_PGFSM_CONFIG__UVDNA_PWR_CONFIG__SHIFT678 | 2 << UVD_PGFSM_CONFIG__UVDNB_PWR_CONFIG__SHIFT);679 WREG32_SOC15(VCN, inst, regUVD_PGFSM_CONFIG, data);680 681 data = (2 << UVD_PGFSM_STATUS__UVDM_PWR_STATUS__SHIFT682 | 2 << UVD_PGFSM_STATUS__UVDS_PWR_STATUS__SHIFT683 | 2 << UVD_PGFSM_STATUS__UVDF_PWR_STATUS__SHIFT684 | 2 << UVD_PGFSM_STATUS__UVDTC_PWR_STATUS__SHIFT685 | 2 << UVD_PGFSM_STATUS__UVDB_PWR_STATUS__SHIFT686 | 2 << UVD_PGFSM_STATUS__UVDTA_PWR_STATUS__SHIFT687 | 2 << UVD_PGFSM_STATUS__UVDLM_PWR_STATUS__SHIFT688 | 2 << UVD_PGFSM_STATUS__UVDTD_PWR_STATUS__SHIFT689 | 2 << UVD_PGFSM_STATUS__UVDTE_PWR_STATUS__SHIFT690 | 2 << UVD_PGFSM_STATUS__UVDE_PWR_STATUS__SHIFT691 | 2 << UVD_PGFSM_STATUS__UVDAB_PWR_STATUS__SHIFT692 | 2 << UVD_PGFSM_STATUS__UVDTB_PWR_STATUS__SHIFT693 | 2 << UVD_PGFSM_STATUS__UVDNA_PWR_STATUS__SHIFT694 | 2 << UVD_PGFSM_STATUS__UVDNB_PWR_STATUS__SHIFT);695 SOC15_WAIT_ON_RREG(VCN, inst, regUVD_PGFSM_STATUS, data, 0x3F3FFFFF);696 }697 698 return;699}700 701/**702 * vcn_v4_0_disable_clock_gating - disable VCN clock gating703 *704 * @adev: amdgpu_device pointer705 * @inst: instance number706 *707 * Disable clock gating for VCN block708 */709static void vcn_v4_0_disable_clock_gating(struct amdgpu_device *adev, int inst)710{711 uint32_t data;712 713 if (adev->cg_flags & AMD_CG_SUPPORT_VCN_MGCG)714 return;715 716 /* VCN disable CGC */717 data = RREG32_SOC15(VCN, inst, regUVD_CGC_CTRL);718 data &= ~UVD_CGC_CTRL__DYN_CLOCK_MODE_MASK;719 data |= 1 << UVD_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT;720 data |= 4 << UVD_CGC_CTRL__CLK_OFF_DELAY__SHIFT;721 WREG32_SOC15(VCN, inst, regUVD_CGC_CTRL, data);722 723 data = RREG32_SOC15(VCN, inst, regUVD_CGC_GATE);724 data &= ~(UVD_CGC_GATE__SYS_MASK725 | UVD_CGC_GATE__UDEC_MASK726 | UVD_CGC_GATE__MPEG2_MASK727 | UVD_CGC_GATE__REGS_MASK728 | UVD_CGC_GATE__RBC_MASK729 | UVD_CGC_GATE__LMI_MC_MASK730 | UVD_CGC_GATE__LMI_UMC_MASK731 | UVD_CGC_GATE__IDCT_MASK732 | UVD_CGC_GATE__MPRD_MASK733 | UVD_CGC_GATE__MPC_MASK734 | UVD_CGC_GATE__LBSI_MASK735 | UVD_CGC_GATE__LRBBM_MASK736 | UVD_CGC_GATE__UDEC_RE_MASK737 | UVD_CGC_GATE__UDEC_CM_MASK738 | UVD_CGC_GATE__UDEC_IT_MASK739 | UVD_CGC_GATE__UDEC_DB_MASK740 | UVD_CGC_GATE__UDEC_MP_MASK741 | UVD_CGC_GATE__WCB_MASK742 | UVD_CGC_GATE__VCPU_MASK743 | UVD_CGC_GATE__MMSCH_MASK);744 745 WREG32_SOC15(VCN, inst, regUVD_CGC_GATE, data);746 SOC15_WAIT_ON_RREG(VCN, inst, regUVD_CGC_GATE, 0, 0xFFFFFFFF);747 748 data = RREG32_SOC15(VCN, inst, regUVD_CGC_CTRL);749 data &= ~(UVD_CGC_CTRL__UDEC_RE_MODE_MASK750 | UVD_CGC_CTRL__UDEC_CM_MODE_MASK751 | UVD_CGC_CTRL__UDEC_IT_MODE_MASK752 | UVD_CGC_CTRL__UDEC_DB_MODE_MASK753 | UVD_CGC_CTRL__UDEC_MP_MODE_MASK754 | UVD_CGC_CTRL__SYS_MODE_MASK755 | UVD_CGC_CTRL__UDEC_MODE_MASK756 | UVD_CGC_CTRL__MPEG2_MODE_MASK757 | UVD_CGC_CTRL__REGS_MODE_MASK758 | UVD_CGC_CTRL__RBC_MODE_MASK759 | UVD_CGC_CTRL__LMI_MC_MODE_MASK760 | UVD_CGC_CTRL__LMI_UMC_MODE_MASK761 | UVD_CGC_CTRL__IDCT_MODE_MASK762 | UVD_CGC_CTRL__MPRD_MODE_MASK763 | UVD_CGC_CTRL__MPC_MODE_MASK764 | UVD_CGC_CTRL__LBSI_MODE_MASK765 | UVD_CGC_CTRL__LRBBM_MODE_MASK766 | UVD_CGC_CTRL__WCB_MODE_MASK767 | UVD_CGC_CTRL__VCPU_MODE_MASK768 | UVD_CGC_CTRL__MMSCH_MODE_MASK);769 WREG32_SOC15(VCN, inst, regUVD_CGC_CTRL, data);770 771 data = RREG32_SOC15(VCN, inst, regUVD_SUVD_CGC_GATE);772 data |= (UVD_SUVD_CGC_GATE__SRE_MASK773 | UVD_SUVD_CGC_GATE__SIT_MASK774 | UVD_SUVD_CGC_GATE__SMP_MASK775 | UVD_SUVD_CGC_GATE__SCM_MASK776 | UVD_SUVD_CGC_GATE__SDB_MASK777 | UVD_SUVD_CGC_GATE__SRE_H264_MASK778 | UVD_SUVD_CGC_GATE__SRE_HEVC_MASK779 | UVD_SUVD_CGC_GATE__SIT_H264_MASK780 | UVD_SUVD_CGC_GATE__SIT_HEVC_MASK781 | UVD_SUVD_CGC_GATE__SCM_H264_MASK782 | UVD_SUVD_CGC_GATE__SCM_HEVC_MASK783 | UVD_SUVD_CGC_GATE__SDB_H264_MASK784 | UVD_SUVD_CGC_GATE__SDB_HEVC_MASK785 | UVD_SUVD_CGC_GATE__SCLR_MASK786 | UVD_SUVD_CGC_GATE__UVD_SC_MASK787 | UVD_SUVD_CGC_GATE__ENT_MASK788 | UVD_SUVD_CGC_GATE__SIT_HEVC_DEC_MASK789 | UVD_SUVD_CGC_GATE__SIT_HEVC_ENC_MASK790 | UVD_SUVD_CGC_GATE__SITE_MASK791 | UVD_SUVD_CGC_GATE__SRE_VP9_MASK792 | UVD_SUVD_CGC_GATE__SCM_VP9_MASK793 | UVD_SUVD_CGC_GATE__SIT_VP9_DEC_MASK794 | UVD_SUVD_CGC_GATE__SDB_VP9_MASK795 | UVD_SUVD_CGC_GATE__IME_HEVC_MASK);796 WREG32_SOC15(VCN, inst, regUVD_SUVD_CGC_GATE, data);797 798 data = RREG32_SOC15(VCN, inst, regUVD_SUVD_CGC_CTRL);799 data &= ~(UVD_SUVD_CGC_CTRL__SRE_MODE_MASK800 | UVD_SUVD_CGC_CTRL__SIT_MODE_MASK801 | UVD_SUVD_CGC_CTRL__SMP_MODE_MASK802 | UVD_SUVD_CGC_CTRL__SCM_MODE_MASK803 | UVD_SUVD_CGC_CTRL__SDB_MODE_MASK804 | UVD_SUVD_CGC_CTRL__SCLR_MODE_MASK805 | UVD_SUVD_CGC_CTRL__UVD_SC_MODE_MASK806 | UVD_SUVD_CGC_CTRL__ENT_MODE_MASK807 | UVD_SUVD_CGC_CTRL__IME_MODE_MASK808 | UVD_SUVD_CGC_CTRL__SITE_MODE_MASK);809 WREG32_SOC15(VCN, inst, regUVD_SUVD_CGC_CTRL, data);810}811 812/**813 * vcn_v4_0_disable_clock_gating_dpg_mode - disable VCN clock gating dpg mode814 *815 * @adev: amdgpu_device pointer816 * @sram_sel: sram select817 * @inst_idx: instance number index818 * @indirect: indirectly write sram819 *820 * Disable clock gating for VCN block with dpg mode821 */822static void vcn_v4_0_disable_clock_gating_dpg_mode(struct amdgpu_device *adev, uint8_t sram_sel,823 int inst_idx, uint8_t indirect)824{825 uint32_t reg_data = 0;826 827 if (adev->cg_flags & AMD_CG_SUPPORT_VCN_MGCG)828 return;829 830 /* enable sw clock gating control */831 reg_data = 0 << UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;832 reg_data |= 1 << UVD_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT;833 reg_data |= 4 << UVD_CGC_CTRL__CLK_OFF_DELAY__SHIFT;834 reg_data &= ~(UVD_CGC_CTRL__UDEC_RE_MODE_MASK |835 UVD_CGC_CTRL__UDEC_CM_MODE_MASK |836 UVD_CGC_CTRL__UDEC_IT_MODE_MASK |837 UVD_CGC_CTRL__UDEC_DB_MODE_MASK |838 UVD_CGC_CTRL__UDEC_MP_MODE_MASK |839 UVD_CGC_CTRL__SYS_MODE_MASK |840 UVD_CGC_CTRL__UDEC_MODE_MASK |841 UVD_CGC_CTRL__MPEG2_MODE_MASK |842 UVD_CGC_CTRL__REGS_MODE_MASK |843 UVD_CGC_CTRL__RBC_MODE_MASK |844 UVD_CGC_CTRL__LMI_MC_MODE_MASK |845 UVD_CGC_CTRL__LMI_UMC_MODE_MASK |846 UVD_CGC_CTRL__IDCT_MODE_MASK |847 UVD_CGC_CTRL__MPRD_MODE_MASK |848 UVD_CGC_CTRL__MPC_MODE_MASK |849 UVD_CGC_CTRL__LBSI_MODE_MASK |850 UVD_CGC_CTRL__LRBBM_MODE_MASK |851 UVD_CGC_CTRL__WCB_MODE_MASK |852 UVD_CGC_CTRL__VCPU_MODE_MASK);853 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(854 VCN, inst_idx, regUVD_CGC_CTRL), reg_data, sram_sel, indirect);855 856 /* turn off clock gating */857 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(858 VCN, inst_idx, regUVD_CGC_GATE), 0, sram_sel, indirect);859 860 /* turn on SUVD clock gating */861 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(862 VCN, inst_idx, regUVD_SUVD_CGC_GATE), 1, sram_sel, indirect);863 864 /* turn on sw mode in UVD_SUVD_CGC_CTRL */865 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(866 VCN, inst_idx, regUVD_SUVD_CGC_CTRL), 0, sram_sel, indirect);867}868 869/**870 * vcn_v4_0_enable_clock_gating - enable VCN clock gating871 *872 * @adev: amdgpu_device pointer873 * @inst: instance number874 *875 * Enable clock gating for VCN block876 */877static void vcn_v4_0_enable_clock_gating(struct amdgpu_device *adev, int inst)878{879 uint32_t data;880 881 if (adev->cg_flags & AMD_CG_SUPPORT_VCN_MGCG)882 return;883 884 /* enable VCN CGC */885 data = RREG32_SOC15(VCN, inst, regUVD_CGC_CTRL);886 data |= 0 << UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;887 data |= 1 << UVD_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT;888 data |= 4 << UVD_CGC_CTRL__CLK_OFF_DELAY__SHIFT;889 WREG32_SOC15(VCN, inst, regUVD_CGC_CTRL, data);890 891 data = RREG32_SOC15(VCN, inst, regUVD_CGC_CTRL);892 data |= (UVD_CGC_CTRL__UDEC_RE_MODE_MASK893 | UVD_CGC_CTRL__UDEC_CM_MODE_MASK894 | UVD_CGC_CTRL__UDEC_IT_MODE_MASK895 | UVD_CGC_CTRL__UDEC_DB_MODE_MASK896 | UVD_CGC_CTRL__UDEC_MP_MODE_MASK897 | UVD_CGC_CTRL__SYS_MODE_MASK898 | UVD_CGC_CTRL__UDEC_MODE_MASK899 | UVD_CGC_CTRL__MPEG2_MODE_MASK900 | UVD_CGC_CTRL__REGS_MODE_MASK901 | UVD_CGC_CTRL__RBC_MODE_MASK902 | UVD_CGC_CTRL__LMI_MC_MODE_MASK903 | UVD_CGC_CTRL__LMI_UMC_MODE_MASK904 | UVD_CGC_CTRL__IDCT_MODE_MASK905 | UVD_CGC_CTRL__MPRD_MODE_MASK906 | UVD_CGC_CTRL__MPC_MODE_MASK907 | UVD_CGC_CTRL__LBSI_MODE_MASK908 | UVD_CGC_CTRL__LRBBM_MODE_MASK909 | UVD_CGC_CTRL__WCB_MODE_MASK910 | UVD_CGC_CTRL__VCPU_MODE_MASK911 | UVD_CGC_CTRL__MMSCH_MODE_MASK);912 WREG32_SOC15(VCN, inst, regUVD_CGC_CTRL, data);913 914 data = RREG32_SOC15(VCN, inst, regUVD_SUVD_CGC_CTRL);915 data |= (UVD_SUVD_CGC_CTRL__SRE_MODE_MASK916 | UVD_SUVD_CGC_CTRL__SIT_MODE_MASK917 | UVD_SUVD_CGC_CTRL__SMP_MODE_MASK918 | UVD_SUVD_CGC_CTRL__SCM_MODE_MASK919 | UVD_SUVD_CGC_CTRL__SDB_MODE_MASK920 | UVD_SUVD_CGC_CTRL__SCLR_MODE_MASK921 | UVD_SUVD_CGC_CTRL__UVD_SC_MODE_MASK922 | UVD_SUVD_CGC_CTRL__ENT_MODE_MASK923 | UVD_SUVD_CGC_CTRL__IME_MODE_MASK924 | UVD_SUVD_CGC_CTRL__SITE_MODE_MASK);925 WREG32_SOC15(VCN, inst, regUVD_SUVD_CGC_CTRL, data);926}927 928static void vcn_v4_0_enable_ras(struct amdgpu_device *adev, int inst_idx,929 bool indirect)930{931 uint32_t tmp;932 933 if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__VCN))934 return;935 936 tmp = VCN_RAS_CNTL__VCPU_VCODEC_REARM_MASK |937 VCN_RAS_CNTL__VCPU_VCODEC_IH_EN_MASK |938 VCN_RAS_CNTL__VCPU_VCODEC_PMI_EN_MASK |939 VCN_RAS_CNTL__VCPU_VCODEC_STALL_EN_MASK;940 WREG32_SOC15_DPG_MODE(inst_idx,941 SOC15_DPG_MODE_OFFSET(VCN, 0, regVCN_RAS_CNTL),942 tmp, 0, indirect);943 944 tmp = UVD_SYS_INT_EN__RASCNTL_VCPU_VCODEC_EN_MASK;945 WREG32_SOC15_DPG_MODE(inst_idx,946 SOC15_DPG_MODE_OFFSET(VCN, 0, regUVD_SYS_INT_EN),947 tmp, 0, indirect);948}949 950/**951 * vcn_v4_0_start_dpg_mode - VCN start with dpg mode952 *953 * @adev: amdgpu_device pointer954 * @inst_idx: instance number index955 * @indirect: indirectly write sram956 *957 * Start VCN block with dpg mode958 */959static int vcn_v4_0_start_dpg_mode(struct amdgpu_device *adev, int inst_idx, bool indirect)960{961 volatile struct amdgpu_vcn4_fw_shared *fw_shared = adev->vcn.inst[inst_idx].fw_shared.cpu_addr;962 struct amdgpu_ring *ring;963 uint32_t tmp;964 965 /* disable register anti-hang mechanism */966 WREG32_P(SOC15_REG_OFFSET(VCN, inst_idx, regUVD_POWER_STATUS), 1,967 ~UVD_POWER_STATUS__UVD_POWER_STATUS_MASK);968 /* enable dynamic power gating mode */969 tmp = RREG32_SOC15(VCN, inst_idx, regUVD_POWER_STATUS);970 tmp |= UVD_POWER_STATUS__UVD_PG_MODE_MASK;971 tmp |= UVD_POWER_STATUS__UVD_PG_EN_MASK;972 WREG32_SOC15(VCN, inst_idx, regUVD_POWER_STATUS, tmp);973 974 if (indirect)975 adev->vcn.inst[inst_idx].dpg_sram_curr_addr = (uint32_t *)adev->vcn.inst[inst_idx].dpg_sram_cpu_addr;976 977 /* enable clock gating */978 vcn_v4_0_disable_clock_gating_dpg_mode(adev, 0, inst_idx, indirect);979 980 /* enable VCPU clock */981 tmp = (0xFF << UVD_VCPU_CNTL__PRB_TIMEOUT_VAL__SHIFT);982 tmp |= UVD_VCPU_CNTL__CLK_EN_MASK | UVD_VCPU_CNTL__BLK_RST_MASK;983 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(984 VCN, inst_idx, regUVD_VCPU_CNTL), tmp, 0, indirect);985 986 /* disable master interupt */987 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(988 VCN, inst_idx, regUVD_MASTINT_EN), 0, 0, indirect);989 990 /* setup regUVD_LMI_CTRL */991 tmp = (UVD_LMI_CTRL__WRITE_CLEAN_TIMER_EN_MASK |992 UVD_LMI_CTRL__REQ_MODE_MASK |993 UVD_LMI_CTRL__CRC_RESET_MASK |994 UVD_LMI_CTRL__MASK_MC_URGENT_MASK |995 UVD_LMI_CTRL__DATA_COHERENCY_EN_MASK |996 UVD_LMI_CTRL__VCPU_DATA_COHERENCY_EN_MASK |997 (8 << UVD_LMI_CTRL__WRITE_CLEAN_TIMER__SHIFT) |998 0x00100000L);999 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(1000 VCN, inst_idx, regUVD_LMI_CTRL), tmp, 0, indirect);1001 1002 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(1003 VCN, inst_idx, regUVD_MPC_CNTL),1004 0x2 << UVD_MPC_CNTL__REPLACEMENT_MODE__SHIFT, 0, indirect);1005 1006 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(1007 VCN, inst_idx, regUVD_MPC_SET_MUXA0),1008 ((0x1 << UVD_MPC_SET_MUXA0__VARA_1__SHIFT) |1009 (0x2 << UVD_MPC_SET_MUXA0__VARA_2__SHIFT) |1010 (0x3 << UVD_MPC_SET_MUXA0__VARA_3__SHIFT) |1011 (0x4 << UVD_MPC_SET_MUXA0__VARA_4__SHIFT)), 0, indirect);1012 1013 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(1014 VCN, inst_idx, regUVD_MPC_SET_MUXB0),1015 ((0x1 << UVD_MPC_SET_MUXB0__VARB_1__SHIFT) |1016 (0x2 << UVD_MPC_SET_MUXB0__VARB_2__SHIFT) |1017 (0x3 << UVD_MPC_SET_MUXB0__VARB_3__SHIFT) |1018 (0x4 << UVD_MPC_SET_MUXB0__VARB_4__SHIFT)), 0, indirect);1019 1020 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(1021 VCN, inst_idx, regUVD_MPC_SET_MUX),1022 ((0x0 << UVD_MPC_SET_MUX__SET_0__SHIFT) |1023 (0x1 << UVD_MPC_SET_MUX__SET_1__SHIFT) |1024 (0x2 << UVD_MPC_SET_MUX__SET_2__SHIFT)), 0, indirect);1025 1026 vcn_v4_0_mc_resume_dpg_mode(adev, inst_idx, indirect);1027 1028 tmp = (0xFF << UVD_VCPU_CNTL__PRB_TIMEOUT_VAL__SHIFT);1029 tmp |= UVD_VCPU_CNTL__CLK_EN_MASK;1030 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(1031 VCN, inst_idx, regUVD_VCPU_CNTL), tmp, 0, indirect);1032 1033 /* enable LMI MC and UMC channels */1034 tmp = 0x1f << UVD_LMI_CTRL2__RE_OFLD_MIF_WR_REQ_NUM__SHIFT;1035 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(1036 VCN, inst_idx, regUVD_LMI_CTRL2), tmp, 0, indirect);1037 1038 vcn_v4_0_enable_ras(adev, inst_idx, indirect);1039 1040 /* enable master interrupt */1041 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(1042 VCN, inst_idx, regUVD_MASTINT_EN),1043 UVD_MASTINT_EN__VCPU_EN_MASK, 0, indirect);1044 1045 1046 if (indirect)1047 amdgpu_vcn_psp_update_sram(adev, inst_idx, 0);1048 1049 ring = &adev->vcn.inst[inst_idx].ring_enc[0];1050 1051 WREG32_SOC15(VCN, inst_idx, regUVD_RB_BASE_LO, ring->gpu_addr);1052 WREG32_SOC15(VCN, inst_idx, regUVD_RB_BASE_HI, upper_32_bits(ring->gpu_addr));1053 WREG32_SOC15(VCN, inst_idx, regUVD_RB_SIZE, ring->ring_size / 4);1054 1055 tmp = RREG32_SOC15(VCN, inst_idx, regVCN_RB_ENABLE);1056 tmp &= ~(VCN_RB_ENABLE__RB1_EN_MASK);1057 WREG32_SOC15(VCN, inst_idx, regVCN_RB_ENABLE, tmp);1058 fw_shared->sq.queue_mode |= FW_QUEUE_RING_RESET;1059 WREG32_SOC15(VCN, inst_idx, regUVD_RB_RPTR, 0);1060 WREG32_SOC15(VCN, inst_idx, regUVD_RB_WPTR, 0);1061 1062 tmp = RREG32_SOC15(VCN, inst_idx, regUVD_RB_RPTR);1063 WREG32_SOC15(VCN, inst_idx, regUVD_RB_WPTR, tmp);1064 ring->wptr = RREG32_SOC15(VCN, inst_idx, regUVD_RB_WPTR);1065 1066 tmp = RREG32_SOC15(VCN, inst_idx, regVCN_RB_ENABLE);1067 tmp |= VCN_RB_ENABLE__RB1_EN_MASK;1068 WREG32_SOC15(VCN, inst_idx, regVCN_RB_ENABLE, tmp);1069 fw_shared->sq.queue_mode &= ~(FW_QUEUE_RING_RESET | FW_QUEUE_DPG_HOLD_OFF);1070 1071 WREG32_SOC15(VCN, inst_idx, regVCN_RB1_DB_CTRL,1072 ring->doorbell_index << VCN_RB1_DB_CTRL__OFFSET__SHIFT |1073 VCN_RB1_DB_CTRL__EN_MASK);1074 1075 return 0;1076}1077 1078 1079/**1080 * vcn_v4_0_start - VCN start1081 *1082 * @adev: amdgpu_device pointer1083 *1084 * Start VCN block1085 */1086static int vcn_v4_0_start(struct amdgpu_device *adev)1087{1088 volatile struct amdgpu_vcn4_fw_shared *fw_shared;1089 struct amdgpu_ring *ring;1090 uint32_t tmp;1091 int i, j, k, r;1092 1093 if (adev->pm.dpm_enabled)1094 amdgpu_dpm_enable_uvd(adev, true);1095 1096 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {1097 if (adev->vcn.harvest_config & (1 << i))1098 continue;1099 1100 fw_shared = adev->vcn.inst[i].fw_shared.cpu_addr;1101 1102 if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG) {1103 r = vcn_v4_0_start_dpg_mode(adev, i, adev->vcn.indirect_sram);1104 continue;1105 }1106 1107 /* disable VCN power gating */1108 vcn_v4_0_disable_static_power_gating(adev, i);1109 1110 /* set VCN status busy */1111 tmp = RREG32_SOC15(VCN, i, regUVD_STATUS) | UVD_STATUS__UVD_BUSY;1112 WREG32_SOC15(VCN, i, regUVD_STATUS, tmp);1113 1114 /*SW clock gating */1115 vcn_v4_0_disable_clock_gating(adev, i);1116 1117 /* enable VCPU clock */1118 WREG32_P(SOC15_REG_OFFSET(VCN, i, regUVD_VCPU_CNTL),1119 UVD_VCPU_CNTL__CLK_EN_MASK, ~UVD_VCPU_CNTL__CLK_EN_MASK);1120 1121 /* disable master interrupt */1122 WREG32_P(SOC15_REG_OFFSET(VCN, i, regUVD_MASTINT_EN), 0,1123 ~UVD_MASTINT_EN__VCPU_EN_MASK);1124 1125 /* enable LMI MC and UMC channels */1126 WREG32_P(SOC15_REG_OFFSET(VCN, i, regUVD_LMI_CTRL2), 0,1127 ~UVD_LMI_CTRL2__STALL_ARB_UMC_MASK);1128 1129 tmp = RREG32_SOC15(VCN, i, regUVD_SOFT_RESET);1130 tmp &= ~UVD_SOFT_RESET__LMI_SOFT_RESET_MASK;1131 tmp &= ~UVD_SOFT_RESET__LMI_UMC_SOFT_RESET_MASK;1132 WREG32_SOC15(VCN, i, regUVD_SOFT_RESET, tmp);1133 1134 /* setup regUVD_LMI_CTRL */1135 tmp = RREG32_SOC15(VCN, i, regUVD_LMI_CTRL);1136 WREG32_SOC15(VCN, i, regUVD_LMI_CTRL, tmp |1137 UVD_LMI_CTRL__WRITE_CLEAN_TIMER_EN_MASK |1138 UVD_LMI_CTRL__MASK_MC_URGENT_MASK |1139 UVD_LMI_CTRL__DATA_COHERENCY_EN_MASK |1140 UVD_LMI_CTRL__VCPU_DATA_COHERENCY_EN_MASK);1141 1142 /* setup regUVD_MPC_CNTL */1143 tmp = RREG32_SOC15(VCN, i, regUVD_MPC_CNTL);1144 tmp &= ~UVD_MPC_CNTL__REPLACEMENT_MODE_MASK;1145 tmp |= 0x2 << UVD_MPC_CNTL__REPLACEMENT_MODE__SHIFT;1146 WREG32_SOC15(VCN, i, regUVD_MPC_CNTL, tmp);1147 1148 /* setup UVD_MPC_SET_MUXA0 */1149 WREG32_SOC15(VCN, i, regUVD_MPC_SET_MUXA0,1150 ((0x1 << UVD_MPC_SET_MUXA0__VARA_1__SHIFT) |1151 (0x2 << UVD_MPC_SET_MUXA0__VARA_2__SHIFT) |1152 (0x3 << UVD_MPC_SET_MUXA0__VARA_3__SHIFT) |1153 (0x4 << UVD_MPC_SET_MUXA0__VARA_4__SHIFT)));1154 1155 /* setup UVD_MPC_SET_MUXB0 */1156 WREG32_SOC15(VCN, i, regUVD_MPC_SET_MUXB0,1157 ((0x1 << UVD_MPC_SET_MUXB0__VARB_1__SHIFT) |1158 (0x2 << UVD_MPC_SET_MUXB0__VARB_2__SHIFT) |1159 (0x3 << UVD_MPC_SET_MUXB0__VARB_3__SHIFT) |1160 (0x4 << UVD_MPC_SET_MUXB0__VARB_4__SHIFT)));1161 1162 /* setup UVD_MPC_SET_MUX */1163 WREG32_SOC15(VCN, i, regUVD_MPC_SET_MUX,1164 ((0x0 << UVD_MPC_SET_MUX__SET_0__SHIFT) |1165 (0x1 << UVD_MPC_SET_MUX__SET_1__SHIFT) |1166 (0x2 << UVD_MPC_SET_MUX__SET_2__SHIFT)));1167 1168 vcn_v4_0_mc_resume(adev, i);1169 1170 /* VCN global tiling registers */1171 WREG32_SOC15(VCN, i, regUVD_GFX10_ADDR_CONFIG,1172 adev->gfx.config.gb_addr_config);1173 1174 /* unblock VCPU register access */1175 WREG32_P(SOC15_REG_OFFSET(VCN, i, regUVD_RB_ARB_CTRL), 0,1176 ~UVD_RB_ARB_CTRL__VCPU_DIS_MASK);1177 1178 /* release VCPU reset to boot */1179 WREG32_P(SOC15_REG_OFFSET(VCN, i, regUVD_VCPU_CNTL), 0,1180 ~UVD_VCPU_CNTL__BLK_RST_MASK);1181 1182 for (j = 0; j < 10; ++j) {1183 uint32_t status;1184 1185 for (k = 0; k < 100; ++k) {1186 status = RREG32_SOC15(VCN, i, regUVD_STATUS);1187 if (status & 2)1188 break;1189 mdelay(10);1190 if (amdgpu_emu_mode == 1)1191 msleep(1);1192 }1193 1194 if (amdgpu_emu_mode == 1) {1195 r = -1;1196 if (status & 2) {1197 r = 0;1198 break;1199 }1200 } else {1201 r = 0;1202 if (status & 2)1203 break;1204 1205 dev_err(adev->dev, "VCN[%d] is not responding, trying to reset the VCPU!!!\n", i);1206 WREG32_P(SOC15_REG_OFFSET(VCN, i, regUVD_VCPU_CNTL),1207 UVD_VCPU_CNTL__BLK_RST_MASK,1208 ~UVD_VCPU_CNTL__BLK_RST_MASK);1209 mdelay(10);1210 WREG32_P(SOC15_REG_OFFSET(VCN, i, regUVD_VCPU_CNTL), 0,1211 ~UVD_VCPU_CNTL__BLK_RST_MASK);1212 1213 mdelay(10);1214 r = -1;1215 }1216 }1217 1218 if (r) {1219 dev_err(adev->dev, "VCN[%d] is not responding, giving up!!!\n", i);1220 return r;1221 }1222 1223 /* enable master interrupt */1224 WREG32_P(SOC15_REG_OFFSET(VCN, i, regUVD_MASTINT_EN),1225 UVD_MASTINT_EN__VCPU_EN_MASK,1226 ~UVD_MASTINT_EN__VCPU_EN_MASK);1227 1228 /* clear the busy bit of VCN_STATUS */1229 WREG32_P(SOC15_REG_OFFSET(VCN, i, regUVD_STATUS), 0,1230 ~(2 << UVD_STATUS__VCPU_REPORT__SHIFT));1231 1232 ring = &adev->vcn.inst[i].ring_enc[0];1233 WREG32_SOC15(VCN, i, regVCN_RB1_DB_CTRL,1234 ring->doorbell_index << VCN_RB1_DB_CTRL__OFFSET__SHIFT |1235 VCN_RB1_DB_CTRL__EN_MASK);1236 1237 WREG32_SOC15(VCN, i, regUVD_RB_BASE_LO, ring->gpu_addr);1238 WREG32_SOC15(VCN, i, regUVD_RB_BASE_HI, upper_32_bits(ring->gpu_addr));1239 WREG32_SOC15(VCN, i, regUVD_RB_SIZE, ring->ring_size / 4);1240 1241 tmp = RREG32_SOC15(VCN, i, regVCN_RB_ENABLE);1242 tmp &= ~(VCN_RB_ENABLE__RB1_EN_MASK);1243 WREG32_SOC15(VCN, i, regVCN_RB_ENABLE, tmp);1244 fw_shared->sq.queue_mode |= FW_QUEUE_RING_RESET;1245 WREG32_SOC15(VCN, i, regUVD_RB_RPTR, 0);1246 WREG32_SOC15(VCN, i, regUVD_RB_WPTR, 0);1247 1248 tmp = RREG32_SOC15(VCN, i, regUVD_RB_RPTR);1249 WREG32_SOC15(VCN, i, regUVD_RB_WPTR, tmp);1250 ring->wptr = RREG32_SOC15(VCN, i, regUVD_RB_WPTR);1251 1252 tmp = RREG32_SOC15(VCN, i, regVCN_RB_ENABLE);1253 tmp |= VCN_RB_ENABLE__RB1_EN_MASK;1254 WREG32_SOC15(VCN, i, regVCN_RB_ENABLE, tmp);1255 fw_shared->sq.queue_mode &= ~(FW_QUEUE_RING_RESET | FW_QUEUE_DPG_HOLD_OFF);1256 }1257 1258 return 0;1259}1260 1261static int vcn_v4_0_init_ring_metadata(struct amdgpu_device *adev, uint32_t vcn_inst, struct amdgpu_ring *ring_enc)1262{1263 struct amdgpu_vcn_rb_metadata *rb_metadata = NULL;1264 uint8_t *rb_ptr = (uint8_t *)ring_enc->ring;1265 1266 rb_ptr += ring_enc->ring_size;1267 rb_metadata = (struct amdgpu_vcn_rb_metadata *)rb_ptr;1268 1269 memset(rb_metadata, 0, sizeof(struct amdgpu_vcn_rb_metadata));1270 rb_metadata->size = sizeof(struct amdgpu_vcn_rb_metadata);1271 rb_metadata->present_flag_0 |= cpu_to_le32(AMDGPU_VCN_VF_RB_SETUP_FLAG);1272 rb_metadata->present_flag_0 |= cpu_to_le32(AMDGPU_VCN_VF_RB_DECOUPLE_FLAG);1273 rb_metadata->version = 1;1274 rb_metadata->ring_id = vcn_inst & 0xFF;1275 1276 return 0;1277}1278 1279static int vcn_v4_0_start_sriov(struct amdgpu_device *adev)1280{1281 int i;1282 struct amdgpu_ring *ring_enc;1283 uint64_t cache_addr;1284 uint64_t rb_enc_addr;1285 uint64_t ctx_addr;1286 uint32_t param, resp, expected;1287 uint32_t offset, cache_size;1288 uint32_t tmp, timeout;1289 1290 struct amdgpu_mm_table *table = &adev->virt.mm_table;1291 uint32_t *table_loc;1292 uint32_t table_size;1293 uint32_t size, size_dw;1294 uint32_t init_status;1295 uint32_t enabled_vcn;1296 1297 struct mmsch_v4_0_cmd_direct_write1298 direct_wt = { {0} };1299 struct mmsch_v4_0_cmd_direct_read_modify_write1300 direct_rd_mod_wt = { {0} };1301 struct mmsch_v4_0_cmd_end end = { {0} };1302 struct mmsch_v4_0_init_header header;1303 1304 volatile struct amdgpu_vcn4_fw_shared *fw_shared;1305 volatile struct amdgpu_fw_shared_rb_setup *rb_setup;1306 1307 direct_wt.cmd_header.command_type =1308 MMSCH_COMMAND__DIRECT_REG_WRITE;1309 direct_rd_mod_wt.cmd_header.command_type =1310 MMSCH_COMMAND__DIRECT_REG_READ_MODIFY_WRITE;1311 end.cmd_header.command_type =1312 MMSCH_COMMAND__END;1313 1314 header.version = MMSCH_VERSION;1315 header.total_size = sizeof(struct mmsch_v4_0_init_header) >> 2;1316 for (i = 0; i < MMSCH_V4_0_VCN_INSTANCES; i++) {1317 header.inst[i].init_status = 0;1318 header.inst[i].table_offset = 0;1319 header.inst[i].table_size = 0;1320 }1321 1322 table_loc = (uint32_t *)table->cpu_addr;1323 table_loc += header.total_size;1324 for (i = 0; i < adev->vcn.num_vcn_inst; i++) {1325 if (adev->vcn.harvest_config & (1 << i))1326 continue;1327 1328 // Must re/init fw_shared at beginning1329 vcn_v4_0_fw_shared_init(adev, i);1330 1331 table_size = 0;1332 1333 MMSCH_V4_0_INSERT_DIRECT_RD_MOD_WT(SOC15_REG_OFFSET(VCN, i,1334 regUVD_STATUS),1335 ~UVD_STATUS__UVD_BUSY, UVD_STATUS__UVD_BUSY);1336 1337 cache_size = AMDGPU_GPU_PAGE_ALIGN(adev->vcn.fw[i]->size + 4);1338 1339 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {1340 MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,1341 regUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW),1342 adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + i].tmr_mc_addr_lo);1343 MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,1344 regUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH),1345 adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + i].tmr_mc_addr_hi);1346 offset = 0;1347 MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,1348 regUVD_VCPU_CACHE_OFFSET0),1349 0);1350 } else {1351 MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,1352 regUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW),1353 lower_32_bits(adev->vcn.inst[i].gpu_addr));1354 MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,1355 regUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH),1356 upper_32_bits(adev->vcn.inst[i].gpu_addr));1357 offset = cache_size;1358 MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,1359 regUVD_VCPU_CACHE_OFFSET0),1360 AMDGPU_UVD_FIRMWARE_OFFSET >> 3);1361 }1362 1363 MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,1364 regUVD_VCPU_CACHE_SIZE0),1365 cache_size);1366 1367 cache_addr = adev->vcn.inst[i].gpu_addr + offset;1368 MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,1369 regUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW),1370 lower_32_bits(cache_addr));1371 MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,1372 regUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH),1373 upper_32_bits(cache_addr));1374 MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,1375 regUVD_VCPU_CACHE_OFFSET1),1376 0);1377 MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,1378 regUVD_VCPU_CACHE_SIZE1),1379 AMDGPU_VCN_STACK_SIZE);1380 1381 cache_addr = adev->vcn.inst[i].gpu_addr + offset +1382 AMDGPU_VCN_STACK_SIZE;1383 MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,1384 regUVD_LMI_VCPU_CACHE2_64BIT_BAR_LOW),1385 lower_32_bits(cache_addr));1386 MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,1387 regUVD_LMI_VCPU_CACHE2_64BIT_BAR_HIGH),1388 upper_32_bits(cache_addr));1389 MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,1390 regUVD_VCPU_CACHE_OFFSET2),1391 0);1392 MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,1393 regUVD_VCPU_CACHE_SIZE2),1394 AMDGPU_VCN_CONTEXT_SIZE);1395 1396 fw_shared = adev->vcn.inst[i].fw_shared.cpu_addr;1397 rb_setup = &fw_shared->rb_setup;1398 1399 ring_enc = &adev->vcn.inst[i].ring_enc[0];1400 ring_enc->wptr = 0;1401 rb_enc_addr = ring_enc->gpu_addr;1402 1403 rb_setup->is_rb_enabled_flags |= RB_ENABLED;1404 fw_shared->present_flag_0 |= cpu_to_le32(AMDGPU_VCN_VF_RB_SETUP_FLAG);1405 1406 if (amdgpu_sriov_is_vcn_rb_decouple(adev)) {1407 vcn_v4_0_init_ring_metadata(adev, i, ring_enc);1408 1409 memset((void *)&rb_setup->rb_info, 0, sizeof(struct amdgpu_vcn_rb_setup_info) * MAX_NUM_VCN_RB_SETUP);1410 if (!(adev->vcn.harvest_config & (1 << 0))) {1411 rb_setup->rb_info[0].rb_addr_lo = lower_32_bits(adev->vcn.inst[0].ring_enc[0].gpu_addr);1412 rb_setup->rb_info[0].rb_addr_hi = upper_32_bits(adev->vcn.inst[0].ring_enc[0].gpu_addr);1413 rb_setup->rb_info[0].rb_size = adev->vcn.inst[0].ring_enc[0].ring_size / 4;1414 }1415 if (!(adev->vcn.harvest_config & (1 << 1))) {1416 rb_setup->rb_info[2].rb_addr_lo = lower_32_bits(adev->vcn.inst[1].ring_enc[0].gpu_addr);1417 rb_setup->rb_info[2].rb_addr_hi = upper_32_bits(adev->vcn.inst[1].ring_enc[0].gpu_addr);1418 rb_setup->rb_info[2].rb_size = adev->vcn.inst[1].ring_enc[0].ring_size / 4;1419 }1420 fw_shared->decouple.is_enabled = 1;1421 fw_shared->present_flag_0 |= cpu_to_le32(AMDGPU_VCN_VF_RB_DECOUPLE_FLAG);1422 } else {1423 rb_setup->rb_addr_lo = lower_32_bits(rb_enc_addr);1424 rb_setup->rb_addr_hi = upper_32_bits(rb_enc_addr);1425 rb_setup->rb_size = ring_enc->ring_size / 4;1426 }1427 1428 MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,1429 regUVD_LMI_VCPU_NC0_64BIT_BAR_LOW),1430 lower_32_bits(adev->vcn.inst[i].fw_shared.gpu_addr));1431 MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,1432 regUVD_LMI_VCPU_NC0_64BIT_BAR_HIGH),1433 upper_32_bits(adev->vcn.inst[i].fw_shared.gpu_addr));1434 MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,1435 regUVD_VCPU_NONCACHE_SIZE0),1436 AMDGPU_GPU_PAGE_ALIGN(sizeof(struct amdgpu_vcn4_fw_shared)));1437 1438 /* add end packet */1439 MMSCH_V4_0_INSERT_END();1440 1441 /* refine header */1442 header.inst[i].init_status = 0;1443 header.inst[i].table_offset = header.total_size;1444 header.inst[i].table_size = table_size;1445 header.total_size += table_size;1446 }1447 1448 /* Update init table header in memory */1449 size = sizeof(struct mmsch_v4_0_init_header);1450 table_loc = (uint32_t *)table->cpu_addr;1451 memcpy((void *)table_loc, &header, size);1452 1453 /* message MMSCH (in VCN[0]) to initialize this client1454 * 1, write to mmsch_vf_ctx_addr_lo/hi register with GPU mc addr1455 * of memory descriptor location1456 */1457 ctx_addr = table->gpu_addr;1458 WREG32_SOC15(VCN, 0, regMMSCH_VF_CTX_ADDR_LO, lower_32_bits(ctx_addr));1459 WREG32_SOC15(VCN, 0, regMMSCH_VF_CTX_ADDR_HI, upper_32_bits(ctx_addr));1460 1461 /* 2, update vmid of descriptor */1462 tmp = RREG32_SOC15(VCN, 0, regMMSCH_VF_VMID);1463 tmp &= ~MMSCH_VF_VMID__VF_CTX_VMID_MASK;1464 /* use domain0 for MM scheduler */1465 tmp |= (0 << MMSCH_VF_VMID__VF_CTX_VMID__SHIFT);1466 WREG32_SOC15(VCN, 0, regMMSCH_VF_VMID, tmp);1467 1468 /* 3, notify mmsch about the size of this descriptor */1469 size = header.total_size;1470 WREG32_SOC15(VCN, 0, regMMSCH_VF_CTX_SIZE, size);1471 1472 /* 4, set resp to zero */1473 WREG32_SOC15(VCN, 0, regMMSCH_VF_MAILBOX_RESP, 0);1474 1475 /* 5, kick off the initialization and wait until1476 * MMSCH_VF_MAILBOX_RESP becomes non-zero1477 */1478 param = 0x00000001;1479 WREG32_SOC15(VCN, 0, regMMSCH_VF_MAILBOX_HOST, param);1480 tmp = 0;1481 timeout = 1000;1482 resp = 0;1483 expected = MMSCH_VF_MAILBOX_RESP__OK;1484 while (resp != expected) {1485 resp = RREG32_SOC15(VCN, 0, regMMSCH_VF_MAILBOX_RESP);1486 if (resp != 0)1487 break;1488 1489 udelay(10);1490 tmp = tmp + 10;1491 if (tmp >= timeout) {1492 DRM_ERROR("failed to init MMSCH. TIME-OUT after %d usec"\1493 " waiting for regMMSCH_VF_MAILBOX_RESP "\1494 "(expected=0x%08x, readback=0x%08x)\n",1495 tmp, expected, resp);1496 return -EBUSY;1497 }1498 }1499 enabled_vcn = amdgpu_vcn_is_disabled_vcn(adev, VCN_DECODE_RING, 0) ? 1 : 0;1500 init_status = ((struct mmsch_v4_0_init_header *)(table_loc))->inst[enabled_vcn].init_status;1501 if (resp != expected && resp != MMSCH_VF_MAILBOX_RESP__INCOMPLETE1502 && init_status != MMSCH_VF_ENGINE_STATUS__PASS)1503 DRM_ERROR("MMSCH init status is incorrect! readback=0x%08x, header init "\1504 "status for VCN%x: 0x%x\n", resp, enabled_vcn, init_status);1505 1506 return 0;1507}1508 1509/**1510 * vcn_v4_0_stop_dpg_mode - VCN stop with dpg mode1511 *1512 * @adev: amdgpu_device pointer1513 * @inst_idx: instance number index1514 *1515 * Stop VCN block with dpg mode1516 */1517static void vcn_v4_0_stop_dpg_mode(struct amdgpu_device *adev, int inst_idx)1518{1519 struct dpg_pause_state state = {.fw_based = VCN_DPG_STATE__UNPAUSE};1520 uint32_t tmp;1521 1522 vcn_v4_0_pause_dpg_mode(adev, inst_idx, &state);1523 /* Wait for power status to be 1 */1524 SOC15_WAIT_ON_RREG(VCN, inst_idx, regUVD_POWER_STATUS, 1,1525 UVD_POWER_STATUS__UVD_POWER_STATUS_MASK);1526 1527 /* wait for read ptr to be equal to write ptr */1528 tmp = RREG32_SOC15(VCN, inst_idx, regUVD_RB_WPTR);1529 SOC15_WAIT_ON_RREG(VCN, inst_idx, regUVD_RB_RPTR, tmp, 0xFFFFFFFF);1530 1531 SOC15_WAIT_ON_RREG(VCN, inst_idx, regUVD_POWER_STATUS, 1,1532 UVD_POWER_STATUS__UVD_POWER_STATUS_MASK);1533 1534 /* disable dynamic power gating mode */1535 WREG32_P(SOC15_REG_OFFSET(VCN, inst_idx, regUVD_POWER_STATUS), 0,1536 ~UVD_POWER_STATUS__UVD_PG_MODE_MASK);1537}1538 1539/**1540 * vcn_v4_0_stop - VCN stop1541 *1542 * @adev: amdgpu_device pointer1543 *1544 * Stop VCN block1545 */1546static int vcn_v4_0_stop(struct amdgpu_device *adev)1547{1548 volatile struct amdgpu_vcn4_fw_shared *fw_shared;1549 uint32_t tmp;1550 int i, r = 0;1551 1552 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {1553 if (adev->vcn.harvest_config & (1 << i))1554 continue;1555 1556 fw_shared = adev->vcn.inst[i].fw_shared.cpu_addr;1557 fw_shared->sq.queue_mode |= FW_QUEUE_DPG_HOLD_OFF;1558 1559 if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG) {1560 vcn_v4_0_stop_dpg_mode(adev, i);1561 continue;1562 }1563 1564 /* wait for vcn idle */1565 r = SOC15_WAIT_ON_RREG(VCN, i, regUVD_STATUS, UVD_STATUS__IDLE, 0x7);1566 if (r)1567 return r;1568 1569 tmp = UVD_LMI_STATUS__VCPU_LMI_WRITE_CLEAN_MASK |1570 UVD_LMI_STATUS__READ_CLEAN_MASK |1571 UVD_LMI_STATUS__WRITE_CLEAN_MASK |1572 UVD_LMI_STATUS__WRITE_CLEAN_RAW_MASK;1573 r = SOC15_WAIT_ON_RREG(VCN, i, regUVD_LMI_STATUS, tmp, tmp);1574 if (r)1575 return r;1576 1577 /* disable LMI UMC channel */1578 tmp = RREG32_SOC15(VCN, i, regUVD_LMI_CTRL2);1579 tmp |= UVD_LMI_CTRL2__STALL_ARB_UMC_MASK;1580 WREG32_SOC15(VCN, i, regUVD_LMI_CTRL2, tmp);1581 tmp = UVD_LMI_STATUS__UMC_READ_CLEAN_RAW_MASK |1582 UVD_LMI_STATUS__UMC_WRITE_CLEAN_RAW_MASK;1583 r = SOC15_WAIT_ON_RREG(VCN, i, regUVD_LMI_STATUS, tmp, tmp);1584 if (r)1585 return r;1586 1587 /* block VCPU register access */1588 WREG32_P(SOC15_REG_OFFSET(VCN, i, regUVD_RB_ARB_CTRL),1589 UVD_RB_ARB_CTRL__VCPU_DIS_MASK,1590 ~UVD_RB_ARB_CTRL__VCPU_DIS_MASK);1591 1592 /* reset VCPU */1593 WREG32_P(SOC15_REG_OFFSET(VCN, i, regUVD_VCPU_CNTL),1594 UVD_VCPU_CNTL__BLK_RST_MASK,1595 ~UVD_VCPU_CNTL__BLK_RST_MASK);1596 1597 /* disable VCPU clock */1598 WREG32_P(SOC15_REG_OFFSET(VCN, i, regUVD_VCPU_CNTL), 0,1599 ~(UVD_VCPU_CNTL__CLK_EN_MASK));1600 1601 /* apply soft reset */1602 tmp = RREG32_SOC15(VCN, i, regUVD_SOFT_RESET);1603 tmp |= UVD_SOFT_RESET__LMI_UMC_SOFT_RESET_MASK;1604 WREG32_SOC15(VCN, i, regUVD_SOFT_RESET, tmp);1605 tmp = RREG32_SOC15(VCN, i, regUVD_SOFT_RESET);1606 tmp |= UVD_SOFT_RESET__LMI_SOFT_RESET_MASK;1607 WREG32_SOC15(VCN, i, regUVD_SOFT_RESET, tmp);1608 1609 /* clear status */1610 WREG32_SOC15(VCN, i, regUVD_STATUS, 0);1611 1612 /* apply HW clock gating */1613 vcn_v4_0_enable_clock_gating(adev, i);1614 1615 /* enable VCN power gating */1616 vcn_v4_0_enable_static_power_gating(adev, i);1617 }1618 1619 if (adev->pm.dpm_enabled)1620 amdgpu_dpm_enable_uvd(adev, false);1621 1622 return 0;1623}1624 1625/**1626 * vcn_v4_0_pause_dpg_mode - VCN pause with dpg mode1627 *1628 * @adev: amdgpu_device pointer1629 * @inst_idx: instance number index1630 * @new_state: pause state1631 *1632 * Pause dpg mode for VCN block1633 */1634static int vcn_v4_0_pause_dpg_mode(struct amdgpu_device *adev, int inst_idx,1635 struct dpg_pause_state *new_state)1636{1637 uint32_t reg_data = 0;1638 int ret_code;1639 1640 /* pause/unpause if state is changed */1641 if (adev->vcn.inst[inst_idx].pause_state.fw_based != new_state->fw_based) {1642 DRM_DEV_DEBUG(adev->dev, "dpg pause state changed %d -> %d",1643 adev->vcn.inst[inst_idx].pause_state.fw_based, new_state->fw_based);1644 reg_data = RREG32_SOC15(VCN, inst_idx, regUVD_DPG_PAUSE) &1645 (~UVD_DPG_PAUSE__NJ_PAUSE_DPG_ACK_MASK);1646 1647 if (new_state->fw_based == VCN_DPG_STATE__PAUSE) {1648 ret_code = SOC15_WAIT_ON_RREG(VCN, inst_idx, regUVD_POWER_STATUS, 0x1,1649 UVD_POWER_STATUS__UVD_POWER_STATUS_MASK);1650 1651 if (!ret_code) {1652 /* pause DPG */1653 reg_data |= UVD_DPG_PAUSE__NJ_PAUSE_DPG_REQ_MASK;1654 WREG32_SOC15(VCN, inst_idx, regUVD_DPG_PAUSE, reg_data);1655 1656 /* wait for ACK */1657 SOC15_WAIT_ON_RREG(VCN, inst_idx, regUVD_DPG_PAUSE,1658 UVD_DPG_PAUSE__NJ_PAUSE_DPG_ACK_MASK,1659 UVD_DPG_PAUSE__NJ_PAUSE_DPG_ACK_MASK);1660 1661 SOC15_WAIT_ON_RREG(VCN, inst_idx, regUVD_POWER_STATUS,1662 UVD_PGFSM_CONFIG__UVDM_UVDU_PWR_ON, UVD_POWER_STATUS__UVD_POWER_STATUS_MASK);1663 }1664 } else {1665 /* unpause dpg, no need to wait */1666 reg_data &= ~UVD_DPG_PAUSE__NJ_PAUSE_DPG_REQ_MASK;1667 WREG32_SOC15(VCN, inst_idx, regUVD_DPG_PAUSE, reg_data);1668 }1669 adev->vcn.inst[inst_idx].pause_state.fw_based = new_state->fw_based;1670 }1671 1672 return 0;1673}1674 1675/**1676 * vcn_v4_0_unified_ring_get_rptr - get unified read pointer1677 *1678 * @ring: amdgpu_ring pointer1679 *1680 * Returns the current hardware unified read pointer1681 */1682static uint64_t vcn_v4_0_unified_ring_get_rptr(struct amdgpu_ring *ring)1683{1684 struct amdgpu_device *adev = ring->adev;1685 1686 if (ring != &adev->vcn.inst[ring->me].ring_enc[0])1687 DRM_ERROR("wrong ring id is identified in %s", __func__);1688 1689 return RREG32_SOC15(VCN, ring->me, regUVD_RB_RPTR);1690}1691 1692/**1693 * vcn_v4_0_unified_ring_get_wptr - get unified write pointer1694 *1695 * @ring: amdgpu_ring pointer1696 *1697 * Returns the current hardware unified write pointer1698 */1699static uint64_t vcn_v4_0_unified_ring_get_wptr(struct amdgpu_ring *ring)1700{1701 struct amdgpu_device *adev = ring->adev;1702 1703 if (ring != &adev->vcn.inst[ring->me].ring_enc[0])1704 DRM_ERROR("wrong ring id is identified in %s", __func__);1705 1706 if (ring->use_doorbell)1707 return *ring->wptr_cpu_addr;1708 else1709 return RREG32_SOC15(VCN, ring->me, regUVD_RB_WPTR);1710}1711 1712/**1713 * vcn_v4_0_unified_ring_set_wptr - set enc write pointer1714 *1715 * @ring: amdgpu_ring pointer1716 *1717 * Commits the enc write pointer to the hardware1718 */1719static void vcn_v4_0_unified_ring_set_wptr(struct amdgpu_ring *ring)1720{1721 struct amdgpu_device *adev = ring->adev;1722 1723 if (ring != &adev->vcn.inst[ring->me].ring_enc[0])1724 DRM_ERROR("wrong ring id is identified in %s", __func__);1725 1726 if (ring->use_doorbell) {1727 *ring->wptr_cpu_addr = lower_32_bits(ring->wptr);1728 WDOORBELL32(ring->doorbell_index, lower_32_bits(ring->wptr));1729 } else {1730 WREG32_SOC15(VCN, ring->me, regUVD_RB_WPTR, lower_32_bits(ring->wptr));1731 }1732}1733 1734static int vcn_v4_0_limit_sched(struct amdgpu_cs_parser *p,1735 struct amdgpu_job *job)1736{1737 struct drm_gpu_scheduler **scheds;1738 1739 /* The create msg must be in the first IB submitted */1740 if (atomic_read(&job->base.entity->fence_seq))1741 return -EINVAL;1742 1743 /* if VCN0 is harvested, we can't support AV1 */1744 if (p->adev->vcn.harvest_config & AMDGPU_VCN_HARVEST_VCN0)1745 return -EINVAL;1746 1747 scheds = p->adev->gpu_sched[AMDGPU_HW_IP_VCN_ENC]1748 [AMDGPU_RING_PRIO_0].sched;1749 drm_sched_entity_modify_sched(job->base.entity, scheds, 1);1750 return 0;1751}1752 1753static int vcn_v4_0_dec_msg(struct amdgpu_cs_parser *p, struct amdgpu_job *job,1754 uint64_t addr)1755{1756 struct ttm_operation_ctx ctx = { false, false };1757 struct amdgpu_bo_va_mapping *map;1758 uint32_t *msg, num_buffers;1759 struct amdgpu_bo *bo;1760 uint64_t start, end;1761 unsigned int i;1762 void *ptr;1763 int r;1764 1765 addr &= AMDGPU_GMC_HOLE_MASK;1766 r = amdgpu_cs_find_mapping(p, addr, &bo, &map);1767 if (r) {1768 DRM_ERROR("Can't find BO for addr 0x%08llx\n", addr);1769 return r;1770 }1771 1772 start = map->start * AMDGPU_GPU_PAGE_SIZE;1773 end = (map->last + 1) * AMDGPU_GPU_PAGE_SIZE;1774 if (addr & 0x7) {1775 DRM_ERROR("VCN messages must be 8 byte aligned!\n");1776 return -EINVAL;1777 }1778 1779 bo->flags |= AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;1780 amdgpu_bo_placement_from_domain(bo, bo->allowed_domains);1781 r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);1782 if (r) {1783 DRM_ERROR("Failed validating the VCN message BO (%d)!\n", r);1784 return r;1785 }1786 1787 r = amdgpu_bo_kmap(bo, &ptr);1788 if (r) {1789 DRM_ERROR("Failed mapping the VCN message (%d)!\n", r);1790 return r;1791 }1792 1793 msg = ptr + addr - start;1794 1795 /* Check length */1796 if (msg[1] > end - addr) {1797 r = -EINVAL;1798 goto out;1799 }1800 1801 if (msg[3] != RDECODE_MSG_CREATE)1802 goto out;1803 1804 num_buffers = msg[2];1805 for (i = 0, msg = &msg[6]; i < num_buffers; ++i, msg += 4) {1806 uint32_t offset, size, *create;1807 1808 if (msg[0] != RDECODE_MESSAGE_CREATE)1809 continue;1810 1811 offset = msg[1];1812 size = msg[2];1813 1814 if (offset + size > end) {1815 r = -EINVAL;1816 goto out;1817 }1818 1819 create = ptr + addr + offset - start;1820 1821 /* H264, HEVC and VP9 can run on any instance */1822 if (create[0] == 0x7 || create[0] == 0x10 || create[0] == 0x11)1823 continue;1824 1825 r = vcn_v4_0_limit_sched(p, job);1826 if (r)1827 goto out;1828 }1829 1830out:1831 amdgpu_bo_kunmap(bo);1832 return r;1833}1834 1835#define RADEON_VCN_ENGINE_TYPE_ENCODE (0x00000002)1836#define RADEON_VCN_ENGINE_TYPE_DECODE (0x00000003)1837 1838#define RADEON_VCN_ENGINE_INFO (0x30000001)1839#define RADEON_VCN_ENGINE_INFO_MAX_OFFSET 161840 1841#define RENCODE_ENCODE_STANDARD_AV1 21842#define RENCODE_IB_PARAM_SESSION_INIT 0x000000031843#define RENCODE_IB_PARAM_SESSION_INIT_MAX_OFFSET 641844 1845/* return the offset in ib if id is found, -1 otherwise1846 * to speed up the searching we only search upto max_offset1847 */1848static int vcn_v4_0_enc_find_ib_param(struct amdgpu_ib *ib, uint32_t id, int max_offset)1849{1850 int i;1851 1852 for (i = 0; i < ib->length_dw && i < max_offset && ib->ptr[i] >= 8; i += ib->ptr[i]/4) {1853 if (ib->ptr[i + 1] == id)1854 return i;1855 }1856 return -1;1857}1858 1859static int vcn_v4_0_ring_patch_cs_in_place(struct amdgpu_cs_parser *p,1860 struct amdgpu_job *job,1861 struct amdgpu_ib *ib)1862{1863 struct amdgpu_ring *ring = amdgpu_job_ring(job);1864 struct amdgpu_vcn_decode_buffer *decode_buffer;1865 uint64_t addr;1866 uint32_t val;1867 int idx;1868 1869 /* The first instance can decode anything */1870 if (!ring->me)1871 return 0;1872 1873 /* RADEON_VCN_ENGINE_INFO is at the top of ib block */1874 idx = vcn_v4_0_enc_find_ib_param(ib, RADEON_VCN_ENGINE_INFO,1875 RADEON_VCN_ENGINE_INFO_MAX_OFFSET);1876 if (idx < 0) /* engine info is missing */1877 return 0;1878 1879 val = amdgpu_ib_get_value(ib, idx + 2); /* RADEON_VCN_ENGINE_TYPE */1880 if (val == RADEON_VCN_ENGINE_TYPE_DECODE) {1881 decode_buffer = (struct amdgpu_vcn_decode_buffer *)&ib->ptr[idx + 6];1882 1883 if (!(decode_buffer->valid_buf_flag & 0x1))1884 return 0;1885 1886 addr = ((u64)decode_buffer->msg_buffer_address_hi) << 32 |1887 decode_buffer->msg_buffer_address_lo;1888 return vcn_v4_0_dec_msg(p, job, addr);1889 } else if (val == RADEON_VCN_ENGINE_TYPE_ENCODE) {1890 idx = vcn_v4_0_enc_find_ib_param(ib, RENCODE_IB_PARAM_SESSION_INIT,1891 RENCODE_IB_PARAM_SESSION_INIT_MAX_OFFSET);1892 if (idx >= 0 && ib->ptr[idx + 2] == RENCODE_ENCODE_STANDARD_AV1)1893 return vcn_v4_0_limit_sched(p, job);1894 }1895 return 0;1896}1897 1898static struct amdgpu_ring_funcs vcn_v4_0_unified_ring_vm_funcs = {1899 .type = AMDGPU_RING_TYPE_VCN_ENC,1900 .align_mask = 0x3f,1901 .nop = VCN_ENC_CMD_NO_OP,1902 .extra_dw = sizeof(struct amdgpu_vcn_rb_metadata),1903 .get_rptr = vcn_v4_0_unified_ring_get_rptr,1904 .get_wptr = vcn_v4_0_unified_ring_get_wptr,1905 .set_wptr = vcn_v4_0_unified_ring_set_wptr,1906 .patch_cs_in_place = vcn_v4_0_ring_patch_cs_in_place,1907 .emit_frame_size =1908 SOC15_FLUSH_GPU_TLB_NUM_WREG * 3 +1909 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 4 +1910 4 + /* vcn_v2_0_enc_ring_emit_vm_flush */1911 5 + 5 + /* vcn_v2_0_enc_ring_emit_fence x2 vm fence */1912 1, /* vcn_v2_0_enc_ring_insert_end */1913 .emit_ib_size = 5, /* vcn_v2_0_enc_ring_emit_ib */1914 .emit_ib = vcn_v2_0_enc_ring_emit_ib,1915 .emit_fence = vcn_v2_0_enc_ring_emit_fence,1916 .emit_vm_flush = vcn_v2_0_enc_ring_emit_vm_flush,1917 .test_ring = amdgpu_vcn_enc_ring_test_ring,1918 .test_ib = amdgpu_vcn_unified_ring_test_ib,1919 .insert_nop = amdgpu_ring_insert_nop,1920 .insert_end = vcn_v2_0_enc_ring_insert_end,1921 .pad_ib = amdgpu_ring_generic_pad_ib,1922 .begin_use = amdgpu_vcn_ring_begin_use,1923 .end_use = amdgpu_vcn_ring_end_use,1924 .emit_wreg = vcn_v2_0_enc_ring_emit_wreg,1925 .emit_reg_wait = vcn_v2_0_enc_ring_emit_reg_wait,1926 .emit_reg_write_reg_wait = amdgpu_ring_emit_reg_write_reg_wait_helper,1927};1928 1929/**1930 * vcn_v4_0_set_unified_ring_funcs - set unified ring functions1931 *1932 * @adev: amdgpu_device pointer1933 *1934 * Set unified ring functions1935 */1936static void vcn_v4_0_set_unified_ring_funcs(struct amdgpu_device *adev)1937{1938 int i;1939 1940 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {1941 if (adev->vcn.harvest_config & (1 << i))1942 continue;1943 1944 if (amdgpu_ip_version(adev, VCN_HWIP, 0) == IP_VERSION(4, 0, 2))1945 vcn_v4_0_unified_ring_vm_funcs.secure_submission_supported = true;1946 1947 adev->vcn.inst[i].ring_enc[0].funcs =1948 (const struct amdgpu_ring_funcs *)&vcn_v4_0_unified_ring_vm_funcs;1949 adev->vcn.inst[i].ring_enc[0].me = i;1950 }1951}1952 1953/**1954 * vcn_v4_0_is_idle - check VCN block is idle1955 *1956 * @handle: amdgpu_device pointer1957 *1958 * Check whether VCN block is idle1959 */1960static bool vcn_v4_0_is_idle(void *handle)1961{1962 struct amdgpu_device *adev = (struct amdgpu_device *)handle;1963 int i, ret = 1;1964 1965 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {1966 if (adev->vcn.harvest_config & (1 << i))1967 continue;1968 1969 ret &= (RREG32_SOC15(VCN, i, regUVD_STATUS) == UVD_STATUS__IDLE);1970 }1971 1972 return ret;1973}1974 1975/**1976 * vcn_v4_0_wait_for_idle - wait for VCN block idle1977 *1978 * @handle: amdgpu_device pointer1979 *1980 * Wait for VCN block idle1981 */1982static int vcn_v4_0_wait_for_idle(void *handle)1983{1984 struct amdgpu_device *adev = (struct amdgpu_device *)handle;1985 int i, ret = 0;1986 1987 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {1988 if (adev->vcn.harvest_config & (1 << i))1989 continue;1990 1991 ret = SOC15_WAIT_ON_RREG(VCN, i, regUVD_STATUS, UVD_STATUS__IDLE,1992 UVD_STATUS__IDLE);1993 if (ret)1994 return ret;1995 }1996 1997 return ret;1998}1999 2000/**2001 * vcn_v4_0_set_clockgating_state - set VCN block clockgating state2002 *2003 * @handle: amdgpu_device pointer2004 * @state: clock gating state2005 *2006 * Set VCN block clockgating state2007 */2008static int vcn_v4_0_set_clockgating_state(void *handle, enum amd_clockgating_state state)2009{2010 struct amdgpu_device *adev = (struct amdgpu_device *)handle;2011 bool enable = state == AMD_CG_STATE_GATE;2012 int i;2013 2014 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {2015 if (adev->vcn.harvest_config & (1 << i))2016 continue;2017 2018 if (enable) {2019 if (RREG32_SOC15(VCN, i, regUVD_STATUS) != UVD_STATUS__IDLE)2020 return -EBUSY;2021 vcn_v4_0_enable_clock_gating(adev, i);2022 } else {2023 vcn_v4_0_disable_clock_gating(adev, i);2024 }2025 }2026 2027 return 0;2028}2029 2030/**2031 * vcn_v4_0_set_powergating_state - set VCN block powergating state2032 *2033 * @handle: amdgpu_device pointer2034 * @state: power gating state2035 *2036 * Set VCN block powergating state2037 */2038static int vcn_v4_0_set_powergating_state(void *handle, enum amd_powergating_state state)2039{2040 struct amdgpu_device *adev = (struct amdgpu_device *)handle;2041 int ret;2042 2043 /* for SRIOV, guest should not control VCN Power-gating2044 * MMSCH FW should control Power-gating and clock-gating2045 * guest should avoid touching CGC and PG2046 */2047 if (amdgpu_sriov_vf(adev)) {2048 adev->vcn.cur_state = AMD_PG_STATE_UNGATE;2049 return 0;2050 }2051 2052 if (state == adev->vcn.cur_state)2053 return 0;2054 2055 if (state == AMD_PG_STATE_GATE)2056 ret = vcn_v4_0_stop(adev);2057 else2058 ret = vcn_v4_0_start(adev);2059 2060 if (!ret)2061 adev->vcn.cur_state = state;2062 2063 return ret;2064}2065 2066/**2067 * vcn_v4_0_set_ras_interrupt_state - set VCN block RAS interrupt state2068 *2069 * @adev: amdgpu_device pointer2070 * @source: interrupt sources2071 * @type: interrupt types2072 * @state: interrupt states2073 *2074 * Set VCN block RAS interrupt state2075 */2076static int vcn_v4_0_set_ras_interrupt_state(struct amdgpu_device *adev,2077 struct amdgpu_irq_src *source,2078 unsigned int type,2079 enum amdgpu_interrupt_state state)2080{2081 return 0;2082}2083 2084/**2085 * vcn_v4_0_process_interrupt - process VCN block interrupt2086 *2087 * @adev: amdgpu_device pointer2088 * @source: interrupt sources2089 * @entry: interrupt entry from clients and sources2090 *2091 * Process VCN block interrupt2092 */2093static int vcn_v4_0_process_interrupt(struct amdgpu_device *adev, struct amdgpu_irq_src *source,2094 struct amdgpu_iv_entry *entry)2095{2096 uint32_t ip_instance;2097 2098 if (amdgpu_sriov_is_vcn_rb_decouple(adev)) {2099 ip_instance = entry->ring_id;2100 } else {2101 switch (entry->client_id) {2102 case SOC15_IH_CLIENTID_VCN:2103 ip_instance = 0;2104 break;2105 case SOC15_IH_CLIENTID_VCN1:2106 ip_instance = 1;2107 break;2108 default:2109 DRM_ERROR("Unhandled client id: %d\n", entry->client_id);2110 return 0;2111 }2112 }2113 2114 DRM_DEBUG("IH: VCN TRAP\n");2115 2116 switch (entry->src_id) {2117 case VCN_4_0__SRCID__UVD_ENC_GENERAL_PURPOSE:2118 amdgpu_fence_process(&adev->vcn.inst[ip_instance].ring_enc[0]);2119 break;2120 default:2121 DRM_ERROR("Unhandled interrupt: %d %d\n",2122 entry->src_id, entry->src_data[0]);2123 break;2124 }2125 2126 return 0;2127}2128 2129static const struct amdgpu_irq_src_funcs vcn_v4_0_irq_funcs = {2130 .process = vcn_v4_0_process_interrupt,2131};2132 2133static const struct amdgpu_irq_src_funcs vcn_v4_0_ras_irq_funcs = {2134 .set = vcn_v4_0_set_ras_interrupt_state,2135 .process = amdgpu_vcn_process_poison_irq,2136};2137 2138/**2139 * vcn_v4_0_set_irq_funcs - set VCN block interrupt irq functions2140 *2141 * @adev: amdgpu_device pointer2142 *2143 * Set VCN block interrupt irq functions2144 */2145static void vcn_v4_0_set_irq_funcs(struct amdgpu_device *adev)2146{2147 int i;2148 2149 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {2150 if (adev->vcn.harvest_config & (1 << i))2151 continue;2152 2153 adev->vcn.inst[i].irq.num_types = adev->vcn.num_enc_rings + 1;2154 adev->vcn.inst[i].irq.funcs = &vcn_v4_0_irq_funcs;2155 2156 adev->vcn.inst[i].ras_poison_irq.num_types = adev->vcn.num_enc_rings + 1;2157 adev->vcn.inst[i].ras_poison_irq.funcs = &vcn_v4_0_ras_irq_funcs;2158 }2159}2160 2161static void vcn_v4_0_print_ip_state(void *handle, struct drm_printer *p)2162{2163 struct amdgpu_device *adev = (struct amdgpu_device *)handle;2164 int i, j;2165 uint32_t reg_count = ARRAY_SIZE(vcn_reg_list_4_0);2166 uint32_t inst_off, is_powered;2167 2168 if (!adev->vcn.ip_dump)2169 return;2170 2171 drm_printf(p, "num_instances:%d\n", adev->vcn.num_vcn_inst);2172 for (i = 0; i < adev->vcn.num_vcn_inst; i++) {2173 if (adev->vcn.harvest_config & (1 << i)) {2174 drm_printf(p, "\nHarvested Instance:VCN%d Skipping dump\n", i);2175 continue;2176 }2177 2178 inst_off = i * reg_count;2179 is_powered = (adev->vcn.ip_dump[inst_off] &2180 UVD_POWER_STATUS__UVD_POWER_STATUS_MASK) != 1;2181 2182 if (is_powered) {2183 drm_printf(p, "\nActive Instance:VCN%d\n", i);2184 for (j = 0; j < reg_count; j++)2185 drm_printf(p, "%-50s \t 0x%08x\n", vcn_reg_list_4_0[j].reg_name,2186 adev->vcn.ip_dump[inst_off + j]);2187 } else {2188 drm_printf(p, "\nInactive Instance:VCN%d\n", i);2189 }2190 }2191}2192 2193static void vcn_v4_0_dump_ip_state(void *handle)2194{2195 struct amdgpu_device *adev = (struct amdgpu_device *)handle;2196 int i, j;2197 bool is_powered;2198 uint32_t inst_off;2199 uint32_t reg_count = ARRAY_SIZE(vcn_reg_list_4_0);2200 2201 if (!adev->vcn.ip_dump)2202 return;2203 2204 for (i = 0; i < adev->vcn.num_vcn_inst; i++) {2205 if (adev->vcn.harvest_config & (1 << i))2206 continue;2207 2208 inst_off = i * reg_count;2209 /* mmUVD_POWER_STATUS is always readable and is first element of the array */2210 adev->vcn.ip_dump[inst_off] = RREG32_SOC15(VCN, i, regUVD_POWER_STATUS);2211 is_powered = (adev->vcn.ip_dump[inst_off] &2212 UVD_POWER_STATUS__UVD_POWER_STATUS_MASK) != 1;2213 2214 if (is_powered)2215 for (j = 1; j < reg_count; j++)2216 adev->vcn.ip_dump[inst_off + j] =2217 RREG32(SOC15_REG_ENTRY_OFFSET_INST(vcn_reg_list_4_0[j],2218 i));2219 }2220}2221 2222static const struct amd_ip_funcs vcn_v4_0_ip_funcs = {2223 .name = "vcn_v4_0",2224 .early_init = vcn_v4_0_early_init,2225 .late_init = NULL,2226 .sw_init = vcn_v4_0_sw_init,2227 .sw_fini = vcn_v4_0_sw_fini,2228 .hw_init = vcn_v4_0_hw_init,2229 .hw_fini = vcn_v4_0_hw_fini,2230 .suspend = vcn_v4_0_suspend,2231 .resume = vcn_v4_0_resume,2232 .is_idle = vcn_v4_0_is_idle,2233 .wait_for_idle = vcn_v4_0_wait_for_idle,2234 .check_soft_reset = NULL,2235 .pre_soft_reset = NULL,2236 .soft_reset = NULL,2237 .post_soft_reset = NULL,2238 .set_clockgating_state = vcn_v4_0_set_clockgating_state,2239 .set_powergating_state = vcn_v4_0_set_powergating_state,2240 .dump_ip_state = vcn_v4_0_dump_ip_state,2241 .print_ip_state = vcn_v4_0_print_ip_state,2242};2243 2244const struct amdgpu_ip_block_version vcn_v4_0_ip_block = {2245 .type = AMD_IP_BLOCK_TYPE_VCN,2246 .major = 4,2247 .minor = 0,2248 .rev = 0,2249 .funcs = &vcn_v4_0_ip_funcs,2250};2251 2252static uint32_t vcn_v4_0_query_poison_by_instance(struct amdgpu_device *adev,2253 uint32_t instance, uint32_t sub_block)2254{2255 uint32_t poison_stat = 0, reg_value = 0;2256 2257 switch (sub_block) {2258 case AMDGPU_VCN_V4_0_VCPU_VCODEC:2259 reg_value = RREG32_SOC15(VCN, instance, regUVD_RAS_VCPU_VCODEC_STATUS);2260 poison_stat = REG_GET_FIELD(reg_value, UVD_RAS_VCPU_VCODEC_STATUS, POISONED_PF);2261 break;2262 default:2263 break;2264 }2265 2266 if (poison_stat)2267 dev_info(adev->dev, "Poison detected in VCN%d, sub_block%d\n",2268 instance, sub_block);2269 2270 return poison_stat;2271}2272 2273static bool vcn_v4_0_query_ras_poison_status(struct amdgpu_device *adev)2274{2275 uint32_t inst, sub;2276 uint32_t poison_stat = 0;2277 2278 for (inst = 0; inst < adev->vcn.num_vcn_inst; inst++)2279 for (sub = 0; sub < AMDGPU_VCN_V4_0_MAX_SUB_BLOCK; sub++)2280 poison_stat +=2281 vcn_v4_0_query_poison_by_instance(adev, inst, sub);2282 2283 return !!poison_stat;2284}2285 2286const struct amdgpu_ras_block_hw_ops vcn_v4_0_ras_hw_ops = {2287 .query_poison_status = vcn_v4_0_query_ras_poison_status,2288};2289 2290static struct amdgpu_vcn_ras vcn_v4_0_ras = {2291 .ras_block = {2292 .hw_ops = &vcn_v4_0_ras_hw_ops,2293 .ras_late_init = amdgpu_vcn_ras_late_init,2294 },2295};2296 2297static void vcn_v4_0_set_ras_funcs(struct amdgpu_device *adev)2298{2299 switch (amdgpu_ip_version(adev, VCN_HWIP, 0)) {2300 case IP_VERSION(4, 0, 0):2301 adev->vcn.ras = &vcn_v4_0_ras;2302 break;2303 default:2304 break;2305 }2306}2307