2106 lines · c
1/*2 * Copyright 2019 Advanced Micro Devices, Inc.3 *4 * Permission is hereby granted, free of charge, to any person obtaining a5 * copy of this software and associated documentation files (the "Software"),6 * to deal in the Software without restriction, including without limitation7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,8 * and/or sell copies of the Software, and to permit persons to whom the9 * Software is furnished to do so, subject to the following conditions:10 *11 * The above copyright notice and this permission notice shall be included in12 * all copies or substantial portions of the Software.13 *14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR20 * OTHER DEALINGS IN THE SOFTWARE.21 *22 */23 24#include <linux/firmware.h>25#include <drm/drm_drv.h>26 27#include "amdgpu.h"28#include "amdgpu_vcn.h"29#include "amdgpu_pm.h"30#include "soc15.h"31#include "soc15d.h"32#include "vcn_v2_0.h"33#include "mmsch_v1_0.h"34#include "vcn_v2_5.h"35 36#include "vcn/vcn_2_5_offset.h"37#include "vcn/vcn_2_5_sh_mask.h"38#include "ivsrcid/vcn/irqsrcs_vcn_2_0.h"39 40#define VCN_VID_SOC_ADDRESS_2_0 0x1fa0041#define VCN1_VID_SOC_ADDRESS_3_0 0x4820042 43#define mmUVD_CONTEXT_ID_INTERNAL_OFFSET 0x2744#define mmUVD_GPCOM_VCPU_CMD_INTERNAL_OFFSET 0x0f45#define mmUVD_GPCOM_VCPU_DATA0_INTERNAL_OFFSET 0x1046#define mmUVD_GPCOM_VCPU_DATA1_INTERNAL_OFFSET 0x1147#define mmUVD_NO_OP_INTERNAL_OFFSET 0x2948#define mmUVD_GP_SCRATCH8_INTERNAL_OFFSET 0x6649#define mmUVD_SCRATCH9_INTERNAL_OFFSET 0xc01d50 51#define mmUVD_LMI_RBC_IB_VMID_INTERNAL_OFFSET 0x43152#define mmUVD_LMI_RBC_IB_64BIT_BAR_LOW_INTERNAL_OFFSET 0x3b453#define mmUVD_LMI_RBC_IB_64BIT_BAR_HIGH_INTERNAL_OFFSET 0x3b554#define mmUVD_RBC_IB_SIZE_INTERNAL_OFFSET 0x25c55 56#define VCN25_MAX_HW_INSTANCES_ARCTURUS 257 58static const struct amdgpu_hwip_reg_entry vcn_reg_list_2_5[] = {59 SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_POWER_STATUS),60 SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_POWER_STATUS),61 SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_STATUS),62 SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_CONTEXT_ID),63 SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_CONTEXT_ID2),64 SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_GPCOM_VCPU_DATA0),65 SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_GPCOM_VCPU_DATA1),66 SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_GPCOM_VCPU_CMD),67 SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_RB_BASE_HI),68 SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_RB_BASE_LO),69 SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_RB_BASE_HI2),70 SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_RB_BASE_LO2),71 SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_RB_BASE_HI3),72 SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_RB_BASE_LO3),73 SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_RB_BASE_HI4),74 SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_RB_BASE_LO4),75 SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_RB_RPTR),76 SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_RB_WPTR),77 SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_RB_RPTR2),78 SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_RB_WPTR2),79 SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_RB_RPTR3),80 SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_RB_WPTR3),81 SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_RB_RPTR4),82 SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_RB_WPTR4),83 SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_RB_SIZE),84 SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_RB_SIZE2),85 SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_RB_SIZE3),86 SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_RB_SIZE4),87 SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_PGFSM_CONFIG),88 SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_PGFSM_STATUS),89 SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_DPG_LMA_CTL),90 SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_DPG_LMA_DATA),91 SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_DPG_LMA_MASK),92 SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_DPG_PAUSE)93};94 95static void vcn_v2_5_set_dec_ring_funcs(struct amdgpu_device *adev);96static void vcn_v2_5_set_enc_ring_funcs(struct amdgpu_device *adev);97static void vcn_v2_5_set_irq_funcs(struct amdgpu_device *adev);98static int vcn_v2_5_set_powergating_state(void *handle,99 enum amd_powergating_state state);100static int vcn_v2_5_pause_dpg_mode(struct amdgpu_device *adev,101 int inst_idx, struct dpg_pause_state *new_state);102static int vcn_v2_5_sriov_start(struct amdgpu_device *adev);103static void vcn_v2_5_set_ras_funcs(struct amdgpu_device *adev);104 105static int amdgpu_ih_clientid_vcns[] = {106 SOC15_IH_CLIENTID_VCN,107 SOC15_IH_CLIENTID_VCN1108};109 110/**111 * vcn_v2_5_early_init - set function pointers and load microcode112 *113 * @handle: amdgpu_device pointer114 *115 * Set ring and irq function pointers116 * Load microcode from filesystem117 */118static int vcn_v2_5_early_init(void *handle)119{120 struct amdgpu_device *adev = (struct amdgpu_device *)handle;121 122 if (amdgpu_sriov_vf(adev)) {123 adev->vcn.num_vcn_inst = 2;124 adev->vcn.harvest_config = 0;125 adev->vcn.num_enc_rings = 1;126 } else {127 u32 harvest;128 int i;129 130 for (i = 0; i < adev->vcn.num_vcn_inst; i++) {131 harvest = RREG32_SOC15(VCN, i, mmCC_UVD_HARVESTING);132 if (harvest & CC_UVD_HARVESTING__UVD_DISABLE_MASK)133 adev->vcn.harvest_config |= 1 << i;134 }135 if (adev->vcn.harvest_config == (AMDGPU_VCN_HARVEST_VCN0 |136 AMDGPU_VCN_HARVEST_VCN1))137 /* both instances are harvested, disable the block */138 return -ENOENT;139 140 adev->vcn.num_enc_rings = 2;141 }142 143 vcn_v2_5_set_dec_ring_funcs(adev);144 vcn_v2_5_set_enc_ring_funcs(adev);145 vcn_v2_5_set_irq_funcs(adev);146 vcn_v2_5_set_ras_funcs(adev);147 148 return amdgpu_vcn_early_init(adev);149}150 151/**152 * vcn_v2_5_sw_init - sw init for VCN block153 *154 * @handle: amdgpu_device pointer155 *156 * Load firmware and sw initialization157 */158static int vcn_v2_5_sw_init(void *handle)159{160 struct amdgpu_ring *ring;161 int i, j, r;162 uint32_t reg_count = ARRAY_SIZE(vcn_reg_list_2_5);163 uint32_t *ptr;164 struct amdgpu_device *adev = (struct amdgpu_device *)handle;165 166 for (j = 0; j < adev->vcn.num_vcn_inst; j++) {167 if (adev->vcn.harvest_config & (1 << j))168 continue;169 /* VCN DEC TRAP */170 r = amdgpu_irq_add_id(adev, amdgpu_ih_clientid_vcns[j],171 VCN_2_0__SRCID__UVD_SYSTEM_MESSAGE_INTERRUPT, &adev->vcn.inst[j].irq);172 if (r)173 return r;174 175 /* VCN ENC TRAP */176 for (i = 0; i < adev->vcn.num_enc_rings; ++i) {177 r = amdgpu_irq_add_id(adev, amdgpu_ih_clientid_vcns[j],178 i + VCN_2_0__SRCID__UVD_ENC_GENERAL_PURPOSE, &adev->vcn.inst[j].irq);179 if (r)180 return r;181 }182 183 /* VCN POISON TRAP */184 r = amdgpu_irq_add_id(adev, amdgpu_ih_clientid_vcns[j],185 VCN_2_6__SRCID_UVD_POISON, &adev->vcn.inst[j].ras_poison_irq);186 if (r)187 return r;188 }189 190 r = amdgpu_vcn_sw_init(adev);191 if (r)192 return r;193 194 amdgpu_vcn_setup_ucode(adev);195 196 r = amdgpu_vcn_resume(adev);197 if (r)198 return r;199 200 for (j = 0; j < adev->vcn.num_vcn_inst; j++) {201 volatile struct amdgpu_fw_shared *fw_shared;202 203 if (adev->vcn.harvest_config & (1 << j))204 continue;205 adev->vcn.internal.context_id = mmUVD_CONTEXT_ID_INTERNAL_OFFSET;206 adev->vcn.internal.ib_vmid = mmUVD_LMI_RBC_IB_VMID_INTERNAL_OFFSET;207 adev->vcn.internal.ib_bar_low = mmUVD_LMI_RBC_IB_64BIT_BAR_LOW_INTERNAL_OFFSET;208 adev->vcn.internal.ib_bar_high = mmUVD_LMI_RBC_IB_64BIT_BAR_HIGH_INTERNAL_OFFSET;209 adev->vcn.internal.ib_size = mmUVD_RBC_IB_SIZE_INTERNAL_OFFSET;210 adev->vcn.internal.gp_scratch8 = mmUVD_GP_SCRATCH8_INTERNAL_OFFSET;211 212 adev->vcn.internal.scratch9 = mmUVD_SCRATCH9_INTERNAL_OFFSET;213 adev->vcn.inst[j].external.scratch9 = SOC15_REG_OFFSET(VCN, j, mmUVD_SCRATCH9);214 adev->vcn.internal.data0 = mmUVD_GPCOM_VCPU_DATA0_INTERNAL_OFFSET;215 adev->vcn.inst[j].external.data0 = SOC15_REG_OFFSET(VCN, j, mmUVD_GPCOM_VCPU_DATA0);216 adev->vcn.internal.data1 = mmUVD_GPCOM_VCPU_DATA1_INTERNAL_OFFSET;217 adev->vcn.inst[j].external.data1 = SOC15_REG_OFFSET(VCN, j, mmUVD_GPCOM_VCPU_DATA1);218 adev->vcn.internal.cmd = mmUVD_GPCOM_VCPU_CMD_INTERNAL_OFFSET;219 adev->vcn.inst[j].external.cmd = SOC15_REG_OFFSET(VCN, j, mmUVD_GPCOM_VCPU_CMD);220 adev->vcn.internal.nop = mmUVD_NO_OP_INTERNAL_OFFSET;221 adev->vcn.inst[j].external.nop = SOC15_REG_OFFSET(VCN, j, mmUVD_NO_OP);222 223 ring = &adev->vcn.inst[j].ring_dec;224 ring->use_doorbell = true;225 226 ring->doorbell_index = (adev->doorbell_index.vcn.vcn_ring0_1 << 1) +227 (amdgpu_sriov_vf(adev) ? 2*j : 8*j);228 229 if (amdgpu_ip_version(adev, UVD_HWIP, 0) == IP_VERSION(2, 5, 0))230 ring->vm_hub = AMDGPU_MMHUB1(0);231 else232 ring->vm_hub = AMDGPU_MMHUB0(0);233 234 sprintf(ring->name, "vcn_dec_%d", j);235 r = amdgpu_ring_init(adev, ring, 512, &adev->vcn.inst[j].irq,236 0, AMDGPU_RING_PRIO_DEFAULT, NULL);237 if (r)238 return r;239 240 for (i = 0; i < adev->vcn.num_enc_rings; ++i) {241 enum amdgpu_ring_priority_level hw_prio = amdgpu_vcn_get_enc_ring_prio(i);242 243 ring = &adev->vcn.inst[j].ring_enc[i];244 ring->use_doorbell = true;245 246 ring->doorbell_index = (adev->doorbell_index.vcn.vcn_ring0_1 << 1) +247 (amdgpu_sriov_vf(adev) ? (1 + i + 2*j) : (2 + i + 8*j));248 249 if (amdgpu_ip_version(adev, UVD_HWIP, 0) ==250 IP_VERSION(2, 5, 0))251 ring->vm_hub = AMDGPU_MMHUB1(0);252 else253 ring->vm_hub = AMDGPU_MMHUB0(0);254 255 sprintf(ring->name, "vcn_enc_%d.%d", j, i);256 r = amdgpu_ring_init(adev, ring, 512,257 &adev->vcn.inst[j].irq, 0,258 hw_prio, NULL);259 if (r)260 return r;261 }262 263 fw_shared = adev->vcn.inst[j].fw_shared.cpu_addr;264 fw_shared->present_flag_0 = cpu_to_le32(AMDGPU_VCN_MULTI_QUEUE_FLAG);265 266 if (amdgpu_vcnfw_log)267 amdgpu_vcn_fwlog_init(&adev->vcn.inst[i]);268 }269 270 if (amdgpu_sriov_vf(adev)) {271 r = amdgpu_virt_alloc_mm_table(adev);272 if (r)273 return r;274 }275 276 if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG)277 adev->vcn.pause_dpg_mode = vcn_v2_5_pause_dpg_mode;278 279 r = amdgpu_vcn_ras_sw_init(adev);280 if (r)281 return r;282 283 /* Allocate memory for VCN IP Dump buffer */284 ptr = kcalloc(adev->vcn.num_vcn_inst * reg_count, sizeof(uint32_t), GFP_KERNEL);285 if (!ptr) {286 DRM_ERROR("Failed to allocate memory for VCN IP Dump\n");287 adev->vcn.ip_dump = NULL;288 } else {289 adev->vcn.ip_dump = ptr;290 }291 292 return 0;293}294 295/**296 * vcn_v2_5_sw_fini - sw fini for VCN block297 *298 * @handle: amdgpu_device pointer299 *300 * VCN suspend and free up sw allocation301 */302static int vcn_v2_5_sw_fini(void *handle)303{304 int i, r, idx;305 struct amdgpu_device *adev = (struct amdgpu_device *)handle;306 volatile struct amdgpu_fw_shared *fw_shared;307 308 if (drm_dev_enter(adev_to_drm(adev), &idx)) {309 for (i = 0; i < adev->vcn.num_vcn_inst; i++) {310 if (adev->vcn.harvest_config & (1 << i))311 continue;312 fw_shared = adev->vcn.inst[i].fw_shared.cpu_addr;313 fw_shared->present_flag_0 = 0;314 }315 drm_dev_exit(idx);316 }317 318 319 if (amdgpu_sriov_vf(adev))320 amdgpu_virt_free_mm_table(adev);321 322 r = amdgpu_vcn_suspend(adev);323 if (r)324 return r;325 326 r = amdgpu_vcn_sw_fini(adev);327 328 kfree(adev->vcn.ip_dump);329 330 return r;331}332 333/**334 * vcn_v2_5_hw_init - start and test VCN block335 *336 * @handle: amdgpu_device pointer337 *338 * Initialize the hardware, boot up the VCPU and do some testing339 */340static int vcn_v2_5_hw_init(void *handle)341{342 struct amdgpu_device *adev = (struct amdgpu_device *)handle;343 struct amdgpu_ring *ring;344 int i, j, r = 0;345 346 if (amdgpu_sriov_vf(adev))347 r = vcn_v2_5_sriov_start(adev);348 349 for (j = 0; j < adev->vcn.num_vcn_inst; ++j) {350 if (adev->vcn.harvest_config & (1 << j))351 continue;352 353 if (amdgpu_sriov_vf(adev)) {354 adev->vcn.inst[j].ring_enc[0].sched.ready = true;355 adev->vcn.inst[j].ring_enc[1].sched.ready = false;356 adev->vcn.inst[j].ring_enc[2].sched.ready = false;357 adev->vcn.inst[j].ring_dec.sched.ready = true;358 } else {359 360 ring = &adev->vcn.inst[j].ring_dec;361 362 adev->nbio.funcs->vcn_doorbell_range(adev, ring->use_doorbell,363 ring->doorbell_index, j);364 365 r = amdgpu_ring_test_helper(ring);366 if (r)367 return r;368 369 for (i = 0; i < adev->vcn.num_enc_rings; ++i) {370 ring = &adev->vcn.inst[j].ring_enc[i];371 r = amdgpu_ring_test_helper(ring);372 if (r)373 return r;374 }375 }376 }377 378 return r;379}380 381/**382 * vcn_v2_5_hw_fini - stop the hardware block383 *384 * @handle: amdgpu_device pointer385 *386 * Stop the VCN block, mark ring as not ready any more387 */388static int vcn_v2_5_hw_fini(void *handle)389{390 struct amdgpu_device *adev = (struct amdgpu_device *)handle;391 int i;392 393 cancel_delayed_work_sync(&adev->vcn.idle_work);394 395 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {396 if (adev->vcn.harvest_config & (1 << i))397 continue;398 399 if ((adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG) ||400 (adev->vcn.cur_state != AMD_PG_STATE_GATE &&401 RREG32_SOC15(VCN, i, mmUVD_STATUS)))402 vcn_v2_5_set_powergating_state(adev, AMD_PG_STATE_GATE);403 404 if (amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__VCN))405 amdgpu_irq_put(adev, &adev->vcn.inst[i].ras_poison_irq, 0);406 }407 408 return 0;409}410 411/**412 * vcn_v2_5_suspend - suspend VCN block413 *414 * @handle: amdgpu_device pointer415 *416 * HW fini and suspend VCN block417 */418static int vcn_v2_5_suspend(void *handle)419{420 int r;421 struct amdgpu_device *adev = (struct amdgpu_device *)handle;422 423 r = vcn_v2_5_hw_fini(adev);424 if (r)425 return r;426 427 r = amdgpu_vcn_suspend(adev);428 429 return r;430}431 432/**433 * vcn_v2_5_resume - resume VCN block434 *435 * @handle: amdgpu_device pointer436 *437 * Resume firmware and hw init VCN block438 */439static int vcn_v2_5_resume(void *handle)440{441 int r;442 struct amdgpu_device *adev = (struct amdgpu_device *)handle;443 444 r = amdgpu_vcn_resume(adev);445 if (r)446 return r;447 448 r = vcn_v2_5_hw_init(adev);449 450 return r;451}452 453/**454 * vcn_v2_5_mc_resume - memory controller programming455 *456 * @adev: amdgpu_device pointer457 *458 * Let the VCN memory controller know it's offsets459 */460static void vcn_v2_5_mc_resume(struct amdgpu_device *adev)461{462 uint32_t size;463 uint32_t offset;464 int i;465 466 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {467 if (adev->vcn.harvest_config & (1 << i))468 continue;469 470 size = AMDGPU_GPU_PAGE_ALIGN(adev->vcn.fw[i]->size + 4);471 /* cache window 0: fw */472 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {473 WREG32_SOC15(VCN, i, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW,474 (adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + i].tmr_mc_addr_lo));475 WREG32_SOC15(VCN, i, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH,476 (adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + i].tmr_mc_addr_hi));477 WREG32_SOC15(VCN, i, mmUVD_VCPU_CACHE_OFFSET0, 0);478 offset = 0;479 } else {480 WREG32_SOC15(VCN, i, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW,481 lower_32_bits(adev->vcn.inst[i].gpu_addr));482 WREG32_SOC15(VCN, i, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH,483 upper_32_bits(adev->vcn.inst[i].gpu_addr));484 offset = size;485 WREG32_SOC15(VCN, i, mmUVD_VCPU_CACHE_OFFSET0,486 AMDGPU_UVD_FIRMWARE_OFFSET >> 3);487 }488 WREG32_SOC15(VCN, i, mmUVD_VCPU_CACHE_SIZE0, size);489 490 /* cache window 1: stack */491 WREG32_SOC15(VCN, i, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW,492 lower_32_bits(adev->vcn.inst[i].gpu_addr + offset));493 WREG32_SOC15(VCN, i, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH,494 upper_32_bits(adev->vcn.inst[i].gpu_addr + offset));495 WREG32_SOC15(VCN, i, mmUVD_VCPU_CACHE_OFFSET1, 0);496 WREG32_SOC15(VCN, i, mmUVD_VCPU_CACHE_SIZE1, AMDGPU_VCN_STACK_SIZE);497 498 /* cache window 2: context */499 WREG32_SOC15(VCN, i, mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_LOW,500 lower_32_bits(adev->vcn.inst[i].gpu_addr + offset + AMDGPU_VCN_STACK_SIZE));501 WREG32_SOC15(VCN, i, mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_HIGH,502 upper_32_bits(adev->vcn.inst[i].gpu_addr + offset + AMDGPU_VCN_STACK_SIZE));503 WREG32_SOC15(VCN, i, mmUVD_VCPU_CACHE_OFFSET2, 0);504 WREG32_SOC15(VCN, i, mmUVD_VCPU_CACHE_SIZE2, AMDGPU_VCN_CONTEXT_SIZE);505 506 /* non-cache window */507 WREG32_SOC15(VCN, i, mmUVD_LMI_VCPU_NC0_64BIT_BAR_LOW,508 lower_32_bits(adev->vcn.inst[i].fw_shared.gpu_addr));509 WREG32_SOC15(VCN, i, mmUVD_LMI_VCPU_NC0_64BIT_BAR_HIGH,510 upper_32_bits(adev->vcn.inst[i].fw_shared.gpu_addr));511 WREG32_SOC15(VCN, i, mmUVD_VCPU_NONCACHE_OFFSET0, 0);512 WREG32_SOC15(VCN, i, mmUVD_VCPU_NONCACHE_SIZE0,513 AMDGPU_GPU_PAGE_ALIGN(sizeof(struct amdgpu_fw_shared)));514 }515}516 517static void vcn_v2_5_mc_resume_dpg_mode(struct amdgpu_device *adev, int inst_idx, bool indirect)518{519 uint32_t size = AMDGPU_GPU_PAGE_ALIGN(adev->vcn.fw[inst_idx]->size + 4);520 uint32_t offset;521 522 /* cache window 0: fw */523 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {524 if (!indirect) {525 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(526 VCN, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW),527 (adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + inst_idx].tmr_mc_addr_lo), 0, indirect);528 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(529 VCN, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH),530 (adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + inst_idx].tmr_mc_addr_hi), 0, indirect);531 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(532 VCN, 0, mmUVD_VCPU_CACHE_OFFSET0), 0, 0, indirect);533 } else {534 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(535 VCN, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW), 0, 0, indirect);536 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(537 VCN, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH), 0, 0, indirect);538 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(539 VCN, 0, mmUVD_VCPU_CACHE_OFFSET0), 0, 0, indirect);540 }541 offset = 0;542 } else {543 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(544 VCN, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW),545 lower_32_bits(adev->vcn.inst[inst_idx].gpu_addr), 0, indirect);546 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(547 VCN, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH),548 upper_32_bits(adev->vcn.inst[inst_idx].gpu_addr), 0, indirect);549 offset = size;550 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(551 VCN, 0, mmUVD_VCPU_CACHE_OFFSET0),552 AMDGPU_UVD_FIRMWARE_OFFSET >> 3, 0, indirect);553 }554 555 if (!indirect)556 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(557 VCN, 0, mmUVD_VCPU_CACHE_SIZE0), size, 0, indirect);558 else559 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(560 VCN, 0, mmUVD_VCPU_CACHE_SIZE0), 0, 0, indirect);561 562 /* cache window 1: stack */563 if (!indirect) {564 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(565 VCN, 0, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW),566 lower_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset), 0, indirect);567 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(568 VCN, 0, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH),569 upper_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset), 0, indirect);570 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(571 VCN, 0, mmUVD_VCPU_CACHE_OFFSET1), 0, 0, indirect);572 } else {573 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(574 VCN, 0, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW), 0, 0, indirect);575 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(576 VCN, 0, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH), 0, 0, indirect);577 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(578 VCN, 0, mmUVD_VCPU_CACHE_OFFSET1), 0, 0, indirect);579 }580 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(581 VCN, 0, mmUVD_VCPU_CACHE_SIZE1), AMDGPU_VCN_STACK_SIZE, 0, indirect);582 583 /* cache window 2: context */584 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(585 VCN, 0, mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_LOW),586 lower_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset + AMDGPU_VCN_STACK_SIZE), 0, indirect);587 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(588 VCN, 0, mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_HIGH),589 upper_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset + AMDGPU_VCN_STACK_SIZE), 0, indirect);590 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(591 VCN, 0, mmUVD_VCPU_CACHE_OFFSET2), 0, 0, indirect);592 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(593 VCN, 0, mmUVD_VCPU_CACHE_SIZE2), AMDGPU_VCN_CONTEXT_SIZE, 0, indirect);594 595 /* non-cache window */596 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(597 VCN, 0, mmUVD_LMI_VCPU_NC0_64BIT_BAR_LOW),598 lower_32_bits(adev->vcn.inst[inst_idx].fw_shared.gpu_addr), 0, indirect);599 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(600 VCN, 0, mmUVD_LMI_VCPU_NC0_64BIT_BAR_HIGH),601 upper_32_bits(adev->vcn.inst[inst_idx].fw_shared.gpu_addr), 0, indirect);602 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(603 VCN, 0, mmUVD_VCPU_NONCACHE_OFFSET0), 0, 0, indirect);604 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(605 VCN, 0, mmUVD_VCPU_NONCACHE_SIZE0),606 AMDGPU_GPU_PAGE_ALIGN(sizeof(struct amdgpu_fw_shared)), 0, indirect);607 608 /* VCN global tiling registers */609 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(610 VCN, 0, mmUVD_GFX8_ADDR_CONFIG), adev->gfx.config.gb_addr_config, 0, indirect);611}612 613/**614 * vcn_v2_5_disable_clock_gating - disable VCN clock gating615 *616 * @adev: amdgpu_device pointer617 *618 * Disable clock gating for VCN block619 */620static void vcn_v2_5_disable_clock_gating(struct amdgpu_device *adev)621{622 uint32_t data;623 int i;624 625 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {626 if (adev->vcn.harvest_config & (1 << i))627 continue;628 /* UVD disable CGC */629 data = RREG32_SOC15(VCN, i, mmUVD_CGC_CTRL);630 if (adev->cg_flags & AMD_CG_SUPPORT_VCN_MGCG)631 data |= 1 << UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;632 else633 data &= ~UVD_CGC_CTRL__DYN_CLOCK_MODE_MASK;634 data |= 1 << UVD_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT;635 data |= 4 << UVD_CGC_CTRL__CLK_OFF_DELAY__SHIFT;636 WREG32_SOC15(VCN, i, mmUVD_CGC_CTRL, data);637 638 data = RREG32_SOC15(VCN, i, mmUVD_CGC_GATE);639 data &= ~(UVD_CGC_GATE__SYS_MASK640 | UVD_CGC_GATE__UDEC_MASK641 | UVD_CGC_GATE__MPEG2_MASK642 | UVD_CGC_GATE__REGS_MASK643 | UVD_CGC_GATE__RBC_MASK644 | UVD_CGC_GATE__LMI_MC_MASK645 | UVD_CGC_GATE__LMI_UMC_MASK646 | UVD_CGC_GATE__IDCT_MASK647 | UVD_CGC_GATE__MPRD_MASK648 | UVD_CGC_GATE__MPC_MASK649 | UVD_CGC_GATE__LBSI_MASK650 | UVD_CGC_GATE__LRBBM_MASK651 | UVD_CGC_GATE__UDEC_RE_MASK652 | UVD_CGC_GATE__UDEC_CM_MASK653 | UVD_CGC_GATE__UDEC_IT_MASK654 | UVD_CGC_GATE__UDEC_DB_MASK655 | UVD_CGC_GATE__UDEC_MP_MASK656 | UVD_CGC_GATE__WCB_MASK657 | UVD_CGC_GATE__VCPU_MASK658 | UVD_CGC_GATE__MMSCH_MASK);659 660 WREG32_SOC15(VCN, i, mmUVD_CGC_GATE, data);661 662 SOC15_WAIT_ON_RREG(VCN, i, mmUVD_CGC_GATE, 0, 0xFFFFFFFF);663 664 data = RREG32_SOC15(VCN, i, mmUVD_CGC_CTRL);665 data &= ~(UVD_CGC_CTRL__UDEC_RE_MODE_MASK666 | UVD_CGC_CTRL__UDEC_CM_MODE_MASK667 | UVD_CGC_CTRL__UDEC_IT_MODE_MASK668 | UVD_CGC_CTRL__UDEC_DB_MODE_MASK669 | UVD_CGC_CTRL__UDEC_MP_MODE_MASK670 | UVD_CGC_CTRL__SYS_MODE_MASK671 | UVD_CGC_CTRL__UDEC_MODE_MASK672 | UVD_CGC_CTRL__MPEG2_MODE_MASK673 | UVD_CGC_CTRL__REGS_MODE_MASK674 | UVD_CGC_CTRL__RBC_MODE_MASK675 | UVD_CGC_CTRL__LMI_MC_MODE_MASK676 | UVD_CGC_CTRL__LMI_UMC_MODE_MASK677 | UVD_CGC_CTRL__IDCT_MODE_MASK678 | UVD_CGC_CTRL__MPRD_MODE_MASK679 | UVD_CGC_CTRL__MPC_MODE_MASK680 | UVD_CGC_CTRL__LBSI_MODE_MASK681 | UVD_CGC_CTRL__LRBBM_MODE_MASK682 | UVD_CGC_CTRL__WCB_MODE_MASK683 | UVD_CGC_CTRL__VCPU_MODE_MASK684 | UVD_CGC_CTRL__MMSCH_MODE_MASK);685 WREG32_SOC15(VCN, i, mmUVD_CGC_CTRL, data);686 687 /* turn on */688 data = RREG32_SOC15(VCN, i, mmUVD_SUVD_CGC_GATE);689 data |= (UVD_SUVD_CGC_GATE__SRE_MASK690 | UVD_SUVD_CGC_GATE__SIT_MASK691 | UVD_SUVD_CGC_GATE__SMP_MASK692 | UVD_SUVD_CGC_GATE__SCM_MASK693 | UVD_SUVD_CGC_GATE__SDB_MASK694 | UVD_SUVD_CGC_GATE__SRE_H264_MASK695 | UVD_SUVD_CGC_GATE__SRE_HEVC_MASK696 | UVD_SUVD_CGC_GATE__SIT_H264_MASK697 | UVD_SUVD_CGC_GATE__SIT_HEVC_MASK698 | UVD_SUVD_CGC_GATE__SCM_H264_MASK699 | UVD_SUVD_CGC_GATE__SCM_HEVC_MASK700 | UVD_SUVD_CGC_GATE__SDB_H264_MASK701 | UVD_SUVD_CGC_GATE__SDB_HEVC_MASK702 | UVD_SUVD_CGC_GATE__SCLR_MASK703 | UVD_SUVD_CGC_GATE__UVD_SC_MASK704 | UVD_SUVD_CGC_GATE__ENT_MASK705 | UVD_SUVD_CGC_GATE__SIT_HEVC_DEC_MASK706 | UVD_SUVD_CGC_GATE__SIT_HEVC_ENC_MASK707 | UVD_SUVD_CGC_GATE__SITE_MASK708 | UVD_SUVD_CGC_GATE__SRE_VP9_MASK709 | UVD_SUVD_CGC_GATE__SCM_VP9_MASK710 | UVD_SUVD_CGC_GATE__SIT_VP9_DEC_MASK711 | UVD_SUVD_CGC_GATE__SDB_VP9_MASK712 | UVD_SUVD_CGC_GATE__IME_HEVC_MASK);713 WREG32_SOC15(VCN, i, mmUVD_SUVD_CGC_GATE, data);714 715 data = RREG32_SOC15(VCN, i, mmUVD_SUVD_CGC_CTRL);716 data &= ~(UVD_SUVD_CGC_CTRL__SRE_MODE_MASK717 | UVD_SUVD_CGC_CTRL__SIT_MODE_MASK718 | UVD_SUVD_CGC_CTRL__SMP_MODE_MASK719 | UVD_SUVD_CGC_CTRL__SCM_MODE_MASK720 | UVD_SUVD_CGC_CTRL__SDB_MODE_MASK721 | UVD_SUVD_CGC_CTRL__SCLR_MODE_MASK722 | UVD_SUVD_CGC_CTRL__UVD_SC_MODE_MASK723 | UVD_SUVD_CGC_CTRL__ENT_MODE_MASK724 | UVD_SUVD_CGC_CTRL__IME_MODE_MASK725 | UVD_SUVD_CGC_CTRL__SITE_MODE_MASK);726 WREG32_SOC15(VCN, i, mmUVD_SUVD_CGC_CTRL, data);727 }728}729 730static void vcn_v2_5_clock_gating_dpg_mode(struct amdgpu_device *adev,731 uint8_t sram_sel, int inst_idx, uint8_t indirect)732{733 uint32_t reg_data = 0;734 735 /* enable sw clock gating control */736 if (adev->cg_flags & AMD_CG_SUPPORT_VCN_MGCG)737 reg_data = 1 << UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;738 else739 reg_data = 0 << UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;740 reg_data |= 1 << UVD_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT;741 reg_data |= 4 << UVD_CGC_CTRL__CLK_OFF_DELAY__SHIFT;742 reg_data &= ~(UVD_CGC_CTRL__UDEC_RE_MODE_MASK |743 UVD_CGC_CTRL__UDEC_CM_MODE_MASK |744 UVD_CGC_CTRL__UDEC_IT_MODE_MASK |745 UVD_CGC_CTRL__UDEC_DB_MODE_MASK |746 UVD_CGC_CTRL__UDEC_MP_MODE_MASK |747 UVD_CGC_CTRL__SYS_MODE_MASK |748 UVD_CGC_CTRL__UDEC_MODE_MASK |749 UVD_CGC_CTRL__MPEG2_MODE_MASK |750 UVD_CGC_CTRL__REGS_MODE_MASK |751 UVD_CGC_CTRL__RBC_MODE_MASK |752 UVD_CGC_CTRL__LMI_MC_MODE_MASK |753 UVD_CGC_CTRL__LMI_UMC_MODE_MASK |754 UVD_CGC_CTRL__IDCT_MODE_MASK |755 UVD_CGC_CTRL__MPRD_MODE_MASK |756 UVD_CGC_CTRL__MPC_MODE_MASK |757 UVD_CGC_CTRL__LBSI_MODE_MASK |758 UVD_CGC_CTRL__LRBBM_MODE_MASK |759 UVD_CGC_CTRL__WCB_MODE_MASK |760 UVD_CGC_CTRL__VCPU_MODE_MASK |761 UVD_CGC_CTRL__MMSCH_MODE_MASK);762 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(763 VCN, 0, mmUVD_CGC_CTRL), reg_data, sram_sel, indirect);764 765 /* turn off clock gating */766 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(767 VCN, 0, mmUVD_CGC_GATE), 0, sram_sel, indirect);768 769 /* turn on SUVD clock gating */770 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(771 VCN, 0, mmUVD_SUVD_CGC_GATE), 1, sram_sel, indirect);772 773 /* turn on sw mode in UVD_SUVD_CGC_CTRL */774 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(775 VCN, 0, mmUVD_SUVD_CGC_CTRL), 0, sram_sel, indirect);776}777 778/**779 * vcn_v2_5_enable_clock_gating - enable VCN clock gating780 *781 * @adev: amdgpu_device pointer782 *783 * Enable clock gating for VCN block784 */785static void vcn_v2_5_enable_clock_gating(struct amdgpu_device *adev)786{787 uint32_t data = 0;788 int i;789 790 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {791 if (adev->vcn.harvest_config & (1 << i))792 continue;793 /* enable UVD CGC */794 data = RREG32_SOC15(VCN, i, mmUVD_CGC_CTRL);795 if (adev->cg_flags & AMD_CG_SUPPORT_VCN_MGCG)796 data |= 1 << UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;797 else798 data |= 0 << UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;799 data |= 1 << UVD_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT;800 data |= 4 << UVD_CGC_CTRL__CLK_OFF_DELAY__SHIFT;801 WREG32_SOC15(VCN, i, mmUVD_CGC_CTRL, data);802 803 data = RREG32_SOC15(VCN, i, mmUVD_CGC_CTRL);804 data |= (UVD_CGC_CTRL__UDEC_RE_MODE_MASK805 | UVD_CGC_CTRL__UDEC_CM_MODE_MASK806 | UVD_CGC_CTRL__UDEC_IT_MODE_MASK807 | UVD_CGC_CTRL__UDEC_DB_MODE_MASK808 | UVD_CGC_CTRL__UDEC_MP_MODE_MASK809 | UVD_CGC_CTRL__SYS_MODE_MASK810 | UVD_CGC_CTRL__UDEC_MODE_MASK811 | UVD_CGC_CTRL__MPEG2_MODE_MASK812 | UVD_CGC_CTRL__REGS_MODE_MASK813 | UVD_CGC_CTRL__RBC_MODE_MASK814 | UVD_CGC_CTRL__LMI_MC_MODE_MASK815 | UVD_CGC_CTRL__LMI_UMC_MODE_MASK816 | UVD_CGC_CTRL__IDCT_MODE_MASK817 | UVD_CGC_CTRL__MPRD_MODE_MASK818 | UVD_CGC_CTRL__MPC_MODE_MASK819 | UVD_CGC_CTRL__LBSI_MODE_MASK820 | UVD_CGC_CTRL__LRBBM_MODE_MASK821 | UVD_CGC_CTRL__WCB_MODE_MASK822 | UVD_CGC_CTRL__VCPU_MODE_MASK);823 WREG32_SOC15(VCN, i, mmUVD_CGC_CTRL, data);824 825 data = RREG32_SOC15(VCN, i, mmUVD_SUVD_CGC_CTRL);826 data |= (UVD_SUVD_CGC_CTRL__SRE_MODE_MASK827 | UVD_SUVD_CGC_CTRL__SIT_MODE_MASK828 | UVD_SUVD_CGC_CTRL__SMP_MODE_MASK829 | UVD_SUVD_CGC_CTRL__SCM_MODE_MASK830 | UVD_SUVD_CGC_CTRL__SDB_MODE_MASK831 | UVD_SUVD_CGC_CTRL__SCLR_MODE_MASK832 | UVD_SUVD_CGC_CTRL__UVD_SC_MODE_MASK833 | UVD_SUVD_CGC_CTRL__ENT_MODE_MASK834 | UVD_SUVD_CGC_CTRL__IME_MODE_MASK835 | UVD_SUVD_CGC_CTRL__SITE_MODE_MASK);836 WREG32_SOC15(VCN, i, mmUVD_SUVD_CGC_CTRL, data);837 }838}839 840static void vcn_v2_6_enable_ras(struct amdgpu_device *adev, int inst_idx,841 bool indirect)842{843 uint32_t tmp;844 845 if (amdgpu_ip_version(adev, UVD_HWIP, 0) != IP_VERSION(2, 6, 0))846 return;847 848 tmp = VCN_RAS_CNTL__VCPU_VCODEC_REARM_MASK |849 VCN_RAS_CNTL__VCPU_VCODEC_IH_EN_MASK |850 VCN_RAS_CNTL__VCPU_VCODEC_PMI_EN_MASK |851 VCN_RAS_CNTL__VCPU_VCODEC_STALL_EN_MASK;852 WREG32_SOC15_DPG_MODE(inst_idx,853 SOC15_DPG_MODE_OFFSET(VCN, 0, mmVCN_RAS_CNTL),854 tmp, 0, indirect);855 856 tmp = UVD_VCPU_INT_EN__RASCNTL_VCPU_VCODEC_EN_MASK;857 WREG32_SOC15_DPG_MODE(inst_idx,858 SOC15_DPG_MODE_OFFSET(VCN, 0, mmUVD_VCPU_INT_EN),859 tmp, 0, indirect);860 861 tmp = UVD_SYS_INT_EN__RASCNTL_VCPU_VCODEC_EN_MASK;862 WREG32_SOC15_DPG_MODE(inst_idx,863 SOC15_DPG_MODE_OFFSET(VCN, 0, mmUVD_SYS_INT_EN),864 tmp, 0, indirect);865}866 867static int vcn_v2_5_start_dpg_mode(struct amdgpu_device *adev, int inst_idx, bool indirect)868{869 volatile struct amdgpu_fw_shared *fw_shared = adev->vcn.inst[inst_idx].fw_shared.cpu_addr;870 struct amdgpu_ring *ring;871 uint32_t rb_bufsz, tmp;872 873 /* disable register anti-hang mechanism */874 WREG32_P(SOC15_REG_OFFSET(VCN, inst_idx, mmUVD_POWER_STATUS), 1,875 ~UVD_POWER_STATUS__UVD_POWER_STATUS_MASK);876 /* enable dynamic power gating mode */877 tmp = RREG32_SOC15(VCN, inst_idx, mmUVD_POWER_STATUS);878 tmp |= UVD_POWER_STATUS__UVD_PG_MODE_MASK;879 tmp |= UVD_POWER_STATUS__UVD_PG_EN_MASK;880 WREG32_SOC15(VCN, inst_idx, mmUVD_POWER_STATUS, tmp);881 882 if (indirect)883 adev->vcn.inst[inst_idx].dpg_sram_curr_addr = (uint32_t *)adev->vcn.inst[inst_idx].dpg_sram_cpu_addr;884 885 /* enable clock gating */886 vcn_v2_5_clock_gating_dpg_mode(adev, 0, inst_idx, indirect);887 888 /* enable VCPU clock */889 tmp = (0xFF << UVD_VCPU_CNTL__PRB_TIMEOUT_VAL__SHIFT);890 tmp |= UVD_VCPU_CNTL__CLK_EN_MASK;891 tmp |= UVD_VCPU_CNTL__BLK_RST_MASK;892 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(893 VCN, 0, mmUVD_VCPU_CNTL), tmp, 0, indirect);894 895 /* disable master interupt */896 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(897 VCN, 0, mmUVD_MASTINT_EN), 0, 0, indirect);898 899 /* setup mmUVD_LMI_CTRL */900 tmp = (0x8 | UVD_LMI_CTRL__WRITE_CLEAN_TIMER_EN_MASK |901 UVD_LMI_CTRL__REQ_MODE_MASK |902 UVD_LMI_CTRL__CRC_RESET_MASK |903 UVD_LMI_CTRL__MASK_MC_URGENT_MASK |904 UVD_LMI_CTRL__DATA_COHERENCY_EN_MASK |905 UVD_LMI_CTRL__VCPU_DATA_COHERENCY_EN_MASK |906 (8 << UVD_LMI_CTRL__WRITE_CLEAN_TIMER__SHIFT) |907 0x00100000L);908 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(909 VCN, 0, mmUVD_LMI_CTRL), tmp, 0, indirect);910 911 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(912 VCN, 0, mmUVD_MPC_CNTL),913 0x2 << UVD_MPC_CNTL__REPLACEMENT_MODE__SHIFT, 0, indirect);914 915 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(916 VCN, 0, mmUVD_MPC_SET_MUXA0),917 ((0x1 << UVD_MPC_SET_MUXA0__VARA_1__SHIFT) |918 (0x2 << UVD_MPC_SET_MUXA0__VARA_2__SHIFT) |919 (0x3 << UVD_MPC_SET_MUXA0__VARA_3__SHIFT) |920 (0x4 << UVD_MPC_SET_MUXA0__VARA_4__SHIFT)), 0, indirect);921 922 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(923 VCN, 0, mmUVD_MPC_SET_MUXB0),924 ((0x1 << UVD_MPC_SET_MUXB0__VARB_1__SHIFT) |925 (0x2 << UVD_MPC_SET_MUXB0__VARB_2__SHIFT) |926 (0x3 << UVD_MPC_SET_MUXB0__VARB_3__SHIFT) |927 (0x4 << UVD_MPC_SET_MUXB0__VARB_4__SHIFT)), 0, indirect);928 929 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(930 VCN, 0, mmUVD_MPC_SET_MUX),931 ((0x0 << UVD_MPC_SET_MUX__SET_0__SHIFT) |932 (0x1 << UVD_MPC_SET_MUX__SET_1__SHIFT) |933 (0x2 << UVD_MPC_SET_MUX__SET_2__SHIFT)), 0, indirect);934 935 vcn_v2_5_mc_resume_dpg_mode(adev, inst_idx, indirect);936 937 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(938 VCN, 0, mmUVD_REG_XX_MASK), 0x10, 0, indirect);939 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(940 VCN, 0, mmUVD_RBC_XX_IB_REG_CHECK), 0x3, 0, indirect);941 942 /* enable LMI MC and UMC channels */943 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(944 VCN, 0, mmUVD_LMI_CTRL2), 0, 0, indirect);945 946 vcn_v2_6_enable_ras(adev, inst_idx, indirect);947 948 /* unblock VCPU register access */949 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(950 VCN, 0, mmUVD_RB_ARB_CTRL), 0, 0, indirect);951 952 tmp = (0xFF << UVD_VCPU_CNTL__PRB_TIMEOUT_VAL__SHIFT);953 tmp |= UVD_VCPU_CNTL__CLK_EN_MASK;954 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(955 VCN, 0, mmUVD_VCPU_CNTL), tmp, 0, indirect);956 957 /* enable master interrupt */958 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(959 VCN, 0, mmUVD_MASTINT_EN),960 UVD_MASTINT_EN__VCPU_EN_MASK, 0, indirect);961 962 if (indirect)963 amdgpu_vcn_psp_update_sram(adev, inst_idx, 0);964 965 ring = &adev->vcn.inst[inst_idx].ring_dec;966 /* force RBC into idle state */967 rb_bufsz = order_base_2(ring->ring_size);968 tmp = REG_SET_FIELD(0, UVD_RBC_RB_CNTL, RB_BUFSZ, rb_bufsz);969 tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_BLKSZ, 1);970 tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_NO_FETCH, 1);971 tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_NO_UPDATE, 1);972 tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_RPTR_WR_EN, 1);973 WREG32_SOC15(VCN, inst_idx, mmUVD_RBC_RB_CNTL, tmp);974 975 /* Stall DPG before WPTR/RPTR reset */976 WREG32_P(SOC15_REG_OFFSET(VCN, inst_idx, mmUVD_POWER_STATUS),977 UVD_POWER_STATUS__STALL_DPG_POWER_UP_MASK,978 ~UVD_POWER_STATUS__STALL_DPG_POWER_UP_MASK);979 fw_shared->multi_queue.decode_queue_mode |= FW_QUEUE_RING_RESET;980 981 /* set the write pointer delay */982 WREG32_SOC15(VCN, inst_idx, mmUVD_RBC_RB_WPTR_CNTL, 0);983 984 /* set the wb address */985 WREG32_SOC15(VCN, inst_idx, mmUVD_RBC_RB_RPTR_ADDR,986 (upper_32_bits(ring->gpu_addr) >> 2));987 988 /* program the RB_BASE for ring buffer */989 WREG32_SOC15(VCN, inst_idx, mmUVD_LMI_RBC_RB_64BIT_BAR_LOW,990 lower_32_bits(ring->gpu_addr));991 WREG32_SOC15(VCN, inst_idx, mmUVD_LMI_RBC_RB_64BIT_BAR_HIGH,992 upper_32_bits(ring->gpu_addr));993 994 /* Initialize the ring buffer's read and write pointers */995 WREG32_SOC15(VCN, inst_idx, mmUVD_RBC_RB_RPTR, 0);996 997 WREG32_SOC15(VCN, inst_idx, mmUVD_SCRATCH2, 0);998 999 ring->wptr = RREG32_SOC15(VCN, inst_idx, mmUVD_RBC_RB_RPTR);1000 WREG32_SOC15(VCN, inst_idx, mmUVD_RBC_RB_WPTR,1001 lower_32_bits(ring->wptr));1002 1003 fw_shared->multi_queue.decode_queue_mode &= ~FW_QUEUE_RING_RESET;1004 /* Unstall DPG */1005 WREG32_P(SOC15_REG_OFFSET(VCN, inst_idx, mmUVD_POWER_STATUS),1006 0, ~UVD_POWER_STATUS__STALL_DPG_POWER_UP_MASK);1007 1008 return 0;1009}1010 1011static int vcn_v2_5_start(struct amdgpu_device *adev)1012{1013 struct amdgpu_ring *ring;1014 uint32_t rb_bufsz, tmp;1015 int i, j, k, r;1016 1017 if (adev->pm.dpm_enabled)1018 amdgpu_dpm_enable_uvd(adev, true);1019 1020 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {1021 if (adev->vcn.harvest_config & (1 << i))1022 continue;1023 if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG) {1024 r = vcn_v2_5_start_dpg_mode(adev, i, adev->vcn.indirect_sram);1025 continue;1026 }1027 1028 /* disable register anti-hang mechanism */1029 WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_POWER_STATUS), 0,1030 ~UVD_POWER_STATUS__UVD_POWER_STATUS_MASK);1031 1032 /* set uvd status busy */1033 tmp = RREG32_SOC15(VCN, i, mmUVD_STATUS) | UVD_STATUS__UVD_BUSY;1034 WREG32_SOC15(VCN, i, mmUVD_STATUS, tmp);1035 }1036 1037 if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG)1038 return 0;1039 1040 /*SW clock gating */1041 vcn_v2_5_disable_clock_gating(adev);1042 1043 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {1044 if (adev->vcn.harvest_config & (1 << i))1045 continue;1046 /* enable VCPU clock */1047 WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_VCPU_CNTL),1048 UVD_VCPU_CNTL__CLK_EN_MASK, ~UVD_VCPU_CNTL__CLK_EN_MASK);1049 1050 /* disable master interrupt */1051 WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_MASTINT_EN), 0,1052 ~UVD_MASTINT_EN__VCPU_EN_MASK);1053 1054 /* setup mmUVD_LMI_CTRL */1055 tmp = RREG32_SOC15(VCN, i, mmUVD_LMI_CTRL);1056 tmp &= ~0xff;1057 WREG32_SOC15(VCN, i, mmUVD_LMI_CTRL, tmp | 0x8|1058 UVD_LMI_CTRL__WRITE_CLEAN_TIMER_EN_MASK |1059 UVD_LMI_CTRL__MASK_MC_URGENT_MASK |1060 UVD_LMI_CTRL__DATA_COHERENCY_EN_MASK |1061 UVD_LMI_CTRL__VCPU_DATA_COHERENCY_EN_MASK);1062 1063 /* setup mmUVD_MPC_CNTL */1064 tmp = RREG32_SOC15(VCN, i, mmUVD_MPC_CNTL);1065 tmp &= ~UVD_MPC_CNTL__REPLACEMENT_MODE_MASK;1066 tmp |= 0x2 << UVD_MPC_CNTL__REPLACEMENT_MODE__SHIFT;1067 WREG32_SOC15(VCN, i, mmUVD_MPC_CNTL, tmp);1068 1069 /* setup UVD_MPC_SET_MUXA0 */1070 WREG32_SOC15(VCN, i, mmUVD_MPC_SET_MUXA0,1071 ((0x1 << UVD_MPC_SET_MUXA0__VARA_1__SHIFT) |1072 (0x2 << UVD_MPC_SET_MUXA0__VARA_2__SHIFT) |1073 (0x3 << UVD_MPC_SET_MUXA0__VARA_3__SHIFT) |1074 (0x4 << UVD_MPC_SET_MUXA0__VARA_4__SHIFT)));1075 1076 /* setup UVD_MPC_SET_MUXB0 */1077 WREG32_SOC15(VCN, i, mmUVD_MPC_SET_MUXB0,1078 ((0x1 << UVD_MPC_SET_MUXB0__VARB_1__SHIFT) |1079 (0x2 << UVD_MPC_SET_MUXB0__VARB_2__SHIFT) |1080 (0x3 << UVD_MPC_SET_MUXB0__VARB_3__SHIFT) |1081 (0x4 << UVD_MPC_SET_MUXB0__VARB_4__SHIFT)));1082 1083 /* setup mmUVD_MPC_SET_MUX */1084 WREG32_SOC15(VCN, i, mmUVD_MPC_SET_MUX,1085 ((0x0 << UVD_MPC_SET_MUX__SET_0__SHIFT) |1086 (0x1 << UVD_MPC_SET_MUX__SET_1__SHIFT) |1087 (0x2 << UVD_MPC_SET_MUX__SET_2__SHIFT)));1088 }1089 1090 vcn_v2_5_mc_resume(adev);1091 1092 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {1093 volatile struct amdgpu_fw_shared *fw_shared = adev->vcn.inst[i].fw_shared.cpu_addr;1094 if (adev->vcn.harvest_config & (1 << i))1095 continue;1096 /* VCN global tiling registers */1097 WREG32_SOC15(VCN, i, mmUVD_GFX8_ADDR_CONFIG,1098 adev->gfx.config.gb_addr_config);1099 WREG32_SOC15(VCN, i, mmUVD_GFX8_ADDR_CONFIG,1100 adev->gfx.config.gb_addr_config);1101 1102 /* enable LMI MC and UMC channels */1103 WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_LMI_CTRL2), 0,1104 ~UVD_LMI_CTRL2__STALL_ARB_UMC_MASK);1105 1106 /* unblock VCPU register access */1107 WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_RB_ARB_CTRL), 0,1108 ~UVD_RB_ARB_CTRL__VCPU_DIS_MASK);1109 1110 WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_VCPU_CNTL), 0,1111 ~UVD_VCPU_CNTL__BLK_RST_MASK);1112 1113 for (k = 0; k < 10; ++k) {1114 uint32_t status;1115 1116 for (j = 0; j < 100; ++j) {1117 status = RREG32_SOC15(VCN, i, mmUVD_STATUS);1118 if (status & 2)1119 break;1120 if (amdgpu_emu_mode == 1)1121 msleep(500);1122 else1123 mdelay(10);1124 }1125 r = 0;1126 if (status & 2)1127 break;1128 1129 DRM_ERROR("VCN decode not responding, trying to reset the VCPU!!!\n");1130 WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_VCPU_CNTL),1131 UVD_VCPU_CNTL__BLK_RST_MASK,1132 ~UVD_VCPU_CNTL__BLK_RST_MASK);1133 mdelay(10);1134 WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_VCPU_CNTL), 0,1135 ~UVD_VCPU_CNTL__BLK_RST_MASK);1136 1137 mdelay(10);1138 r = -1;1139 }1140 1141 if (r) {1142 DRM_ERROR("VCN decode not responding, giving up!!!\n");1143 return r;1144 }1145 1146 /* enable master interrupt */1147 WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_MASTINT_EN),1148 UVD_MASTINT_EN__VCPU_EN_MASK,1149 ~UVD_MASTINT_EN__VCPU_EN_MASK);1150 1151 /* clear the busy bit of VCN_STATUS */1152 WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_STATUS), 0,1153 ~(2 << UVD_STATUS__VCPU_REPORT__SHIFT));1154 1155 WREG32_SOC15(VCN, i, mmUVD_LMI_RBC_RB_VMID, 0);1156 1157 ring = &adev->vcn.inst[i].ring_dec;1158 /* force RBC into idle state */1159 rb_bufsz = order_base_2(ring->ring_size);1160 tmp = REG_SET_FIELD(0, UVD_RBC_RB_CNTL, RB_BUFSZ, rb_bufsz);1161 tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_BLKSZ, 1);1162 tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_NO_FETCH, 1);1163 tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_NO_UPDATE, 1);1164 tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_RPTR_WR_EN, 1);1165 WREG32_SOC15(VCN, i, mmUVD_RBC_RB_CNTL, tmp);1166 1167 fw_shared->multi_queue.decode_queue_mode |= FW_QUEUE_RING_RESET;1168 /* program the RB_BASE for ring buffer */1169 WREG32_SOC15(VCN, i, mmUVD_LMI_RBC_RB_64BIT_BAR_LOW,1170 lower_32_bits(ring->gpu_addr));1171 WREG32_SOC15(VCN, i, mmUVD_LMI_RBC_RB_64BIT_BAR_HIGH,1172 upper_32_bits(ring->gpu_addr));1173 1174 /* Initialize the ring buffer's read and write pointers */1175 WREG32_SOC15(VCN, i, mmUVD_RBC_RB_RPTR, 0);1176 1177 ring->wptr = RREG32_SOC15(VCN, i, mmUVD_RBC_RB_RPTR);1178 WREG32_SOC15(VCN, i, mmUVD_RBC_RB_WPTR,1179 lower_32_bits(ring->wptr));1180 fw_shared->multi_queue.decode_queue_mode &= ~FW_QUEUE_RING_RESET;1181 1182 fw_shared->multi_queue.encode_generalpurpose_queue_mode |= FW_QUEUE_RING_RESET;1183 ring = &adev->vcn.inst[i].ring_enc[0];1184 WREG32_SOC15(VCN, i, mmUVD_RB_RPTR, lower_32_bits(ring->wptr));1185 WREG32_SOC15(VCN, i, mmUVD_RB_WPTR, lower_32_bits(ring->wptr));1186 WREG32_SOC15(VCN, i, mmUVD_RB_BASE_LO, ring->gpu_addr);1187 WREG32_SOC15(VCN, i, mmUVD_RB_BASE_HI, upper_32_bits(ring->gpu_addr));1188 WREG32_SOC15(VCN, i, mmUVD_RB_SIZE, ring->ring_size / 4);1189 fw_shared->multi_queue.encode_generalpurpose_queue_mode &= ~FW_QUEUE_RING_RESET;1190 1191 fw_shared->multi_queue.encode_lowlatency_queue_mode |= FW_QUEUE_RING_RESET;1192 ring = &adev->vcn.inst[i].ring_enc[1];1193 WREG32_SOC15(VCN, i, mmUVD_RB_RPTR2, lower_32_bits(ring->wptr));1194 WREG32_SOC15(VCN, i, mmUVD_RB_WPTR2, lower_32_bits(ring->wptr));1195 WREG32_SOC15(VCN, i, mmUVD_RB_BASE_LO2, ring->gpu_addr);1196 WREG32_SOC15(VCN, i, mmUVD_RB_BASE_HI2, upper_32_bits(ring->gpu_addr));1197 WREG32_SOC15(VCN, i, mmUVD_RB_SIZE2, ring->ring_size / 4);1198 fw_shared->multi_queue.encode_lowlatency_queue_mode &= ~FW_QUEUE_RING_RESET;1199 }1200 1201 return 0;1202}1203 1204static int vcn_v2_5_mmsch_start(struct amdgpu_device *adev,1205 struct amdgpu_mm_table *table)1206{1207 uint32_t data = 0, loop = 0, size = 0;1208 uint64_t addr = table->gpu_addr;1209 struct mmsch_v1_1_init_header *header = NULL;1210 1211 header = (struct mmsch_v1_1_init_header *)table->cpu_addr;1212 size = header->total_size;1213 1214 /*1215 * 1, write to vce_mmsch_vf_ctx_addr_lo/hi register with GPU mc addr of1216 * memory descriptor location1217 */1218 WREG32_SOC15(VCN, 0, mmMMSCH_VF_CTX_ADDR_LO, lower_32_bits(addr));1219 WREG32_SOC15(VCN, 0, mmMMSCH_VF_CTX_ADDR_HI, upper_32_bits(addr));1220 1221 /* 2, update vmid of descriptor */1222 data = RREG32_SOC15(VCN, 0, mmMMSCH_VF_VMID);1223 data &= ~MMSCH_VF_VMID__VF_CTX_VMID_MASK;1224 /* use domain0 for MM scheduler */1225 data |= (0 << MMSCH_VF_VMID__VF_CTX_VMID__SHIFT);1226 WREG32_SOC15(VCN, 0, mmMMSCH_VF_VMID, data);1227 1228 /* 3, notify mmsch about the size of this descriptor */1229 WREG32_SOC15(VCN, 0, mmMMSCH_VF_CTX_SIZE, size);1230 1231 /* 4, set resp to zero */1232 WREG32_SOC15(VCN, 0, mmMMSCH_VF_MAILBOX_RESP, 0);1233 1234 /*1235 * 5, kick off the initialization and wait until1236 * VCE_MMSCH_VF_MAILBOX_RESP becomes non-zero1237 */1238 WREG32_SOC15(VCN, 0, mmMMSCH_VF_MAILBOX_HOST, 0x10000001);1239 1240 data = RREG32_SOC15(VCN, 0, mmMMSCH_VF_MAILBOX_RESP);1241 loop = 10;1242 while ((data & 0x10000002) != 0x10000002) {1243 udelay(100);1244 data = RREG32_SOC15(VCN, 0, mmMMSCH_VF_MAILBOX_RESP);1245 loop--;1246 if (!loop)1247 break;1248 }1249 1250 if (!loop) {1251 dev_err(adev->dev,1252 "failed to init MMSCH, mmMMSCH_VF_MAILBOX_RESP = %x\n",1253 data);1254 return -EBUSY;1255 }1256 1257 return 0;1258}1259 1260static int vcn_v2_5_sriov_start(struct amdgpu_device *adev)1261{1262 struct amdgpu_ring *ring;1263 uint32_t offset, size, tmp, i, rb_bufsz;1264 uint32_t table_size = 0;1265 struct mmsch_v1_0_cmd_direct_write direct_wt = { { 0 } };1266 struct mmsch_v1_0_cmd_direct_read_modify_write direct_rd_mod_wt = { { 0 } };1267 struct mmsch_v1_0_cmd_end end = { { 0 } };1268 uint32_t *init_table = adev->virt.mm_table.cpu_addr;1269 struct mmsch_v1_1_init_header *header = (struct mmsch_v1_1_init_header *)init_table;1270 1271 direct_wt.cmd_header.command_type = MMSCH_COMMAND__DIRECT_REG_WRITE;1272 direct_rd_mod_wt.cmd_header.command_type = MMSCH_COMMAND__DIRECT_REG_READ_MODIFY_WRITE;1273 end.cmd_header.command_type = MMSCH_COMMAND__END;1274 1275 header->version = MMSCH_VERSION;1276 header->total_size = sizeof(struct mmsch_v1_1_init_header) >> 2;1277 init_table += header->total_size;1278 1279 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {1280 header->eng[i].table_offset = header->total_size;1281 header->eng[i].init_status = 0;1282 header->eng[i].table_size = 0;1283 1284 table_size = 0;1285 1286 MMSCH_V1_0_INSERT_DIRECT_RD_MOD_WT(1287 SOC15_REG_OFFSET(VCN, i, mmUVD_STATUS),1288 ~UVD_STATUS__UVD_BUSY, UVD_STATUS__UVD_BUSY);1289 1290 size = AMDGPU_GPU_PAGE_ALIGN(adev->vcn.fw[i]->size + 4);1291 /* mc resume*/1292 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {1293 MMSCH_V1_0_INSERT_DIRECT_WT(1294 SOC15_REG_OFFSET(VCN, i,1295 mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW),1296 adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + i].tmr_mc_addr_lo);1297 MMSCH_V1_0_INSERT_DIRECT_WT(1298 SOC15_REG_OFFSET(VCN, i,1299 mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH),1300 adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + i].tmr_mc_addr_hi);1301 offset = 0;1302 MMSCH_V1_0_INSERT_DIRECT_WT(1303 SOC15_REG_OFFSET(VCN, i, mmUVD_VCPU_CACHE_OFFSET0), 0);1304 } else {1305 MMSCH_V1_0_INSERT_DIRECT_WT(1306 SOC15_REG_OFFSET(VCN, i,1307 mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW),1308 lower_32_bits(adev->vcn.inst[i].gpu_addr));1309 MMSCH_V1_0_INSERT_DIRECT_WT(1310 SOC15_REG_OFFSET(VCN, i,1311 mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH),1312 upper_32_bits(adev->vcn.inst[i].gpu_addr));1313 offset = size;1314 MMSCH_V1_0_INSERT_DIRECT_WT(1315 SOC15_REG_OFFSET(VCN, i, mmUVD_VCPU_CACHE_OFFSET0),1316 AMDGPU_UVD_FIRMWARE_OFFSET >> 3);1317 }1318 1319 MMSCH_V1_0_INSERT_DIRECT_WT(1320 SOC15_REG_OFFSET(VCN, i, mmUVD_VCPU_CACHE_SIZE0),1321 size);1322 MMSCH_V1_0_INSERT_DIRECT_WT(1323 SOC15_REG_OFFSET(VCN, i,1324 mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW),1325 lower_32_bits(adev->vcn.inst[i].gpu_addr + offset));1326 MMSCH_V1_0_INSERT_DIRECT_WT(1327 SOC15_REG_OFFSET(VCN, i,1328 mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH),1329 upper_32_bits(adev->vcn.inst[i].gpu_addr + offset));1330 MMSCH_V1_0_INSERT_DIRECT_WT(1331 SOC15_REG_OFFSET(VCN, i, mmUVD_VCPU_CACHE_OFFSET1),1332 0);1333 MMSCH_V1_0_INSERT_DIRECT_WT(1334 SOC15_REG_OFFSET(VCN, i, mmUVD_VCPU_CACHE_SIZE1),1335 AMDGPU_VCN_STACK_SIZE);1336 MMSCH_V1_0_INSERT_DIRECT_WT(1337 SOC15_REG_OFFSET(VCN, i,1338 mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_LOW),1339 lower_32_bits(adev->vcn.inst[i].gpu_addr + offset +1340 AMDGPU_VCN_STACK_SIZE));1341 MMSCH_V1_0_INSERT_DIRECT_WT(1342 SOC15_REG_OFFSET(VCN, i,1343 mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_HIGH),1344 upper_32_bits(adev->vcn.inst[i].gpu_addr + offset +1345 AMDGPU_VCN_STACK_SIZE));1346 MMSCH_V1_0_INSERT_DIRECT_WT(1347 SOC15_REG_OFFSET(VCN, i, mmUVD_VCPU_CACHE_OFFSET2),1348 0);1349 MMSCH_V1_0_INSERT_DIRECT_WT(1350 SOC15_REG_OFFSET(VCN, i, mmUVD_VCPU_CACHE_SIZE2),1351 AMDGPU_VCN_CONTEXT_SIZE);1352 1353 ring = &adev->vcn.inst[i].ring_enc[0];1354 ring->wptr = 0;1355 1356 MMSCH_V1_0_INSERT_DIRECT_WT(1357 SOC15_REG_OFFSET(VCN, i, mmUVD_RB_BASE_LO),1358 lower_32_bits(ring->gpu_addr));1359 MMSCH_V1_0_INSERT_DIRECT_WT(1360 SOC15_REG_OFFSET(VCN, i, mmUVD_RB_BASE_HI),1361 upper_32_bits(ring->gpu_addr));1362 MMSCH_V1_0_INSERT_DIRECT_WT(1363 SOC15_REG_OFFSET(VCN, i, mmUVD_RB_SIZE),1364 ring->ring_size / 4);1365 1366 ring = &adev->vcn.inst[i].ring_dec;1367 ring->wptr = 0;1368 MMSCH_V1_0_INSERT_DIRECT_WT(1369 SOC15_REG_OFFSET(VCN, i,1370 mmUVD_LMI_RBC_RB_64BIT_BAR_LOW),1371 lower_32_bits(ring->gpu_addr));1372 MMSCH_V1_0_INSERT_DIRECT_WT(1373 SOC15_REG_OFFSET(VCN, i,1374 mmUVD_LMI_RBC_RB_64BIT_BAR_HIGH),1375 upper_32_bits(ring->gpu_addr));1376 1377 /* force RBC into idle state */1378 rb_bufsz = order_base_2(ring->ring_size);1379 tmp = REG_SET_FIELD(0, UVD_RBC_RB_CNTL, RB_BUFSZ, rb_bufsz);1380 tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_BLKSZ, 1);1381 tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_NO_FETCH, 1);1382 tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_NO_UPDATE, 1);1383 tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_RPTR_WR_EN, 1);1384 MMSCH_V1_0_INSERT_DIRECT_WT(1385 SOC15_REG_OFFSET(VCN, i, mmUVD_RBC_RB_CNTL), tmp);1386 1387 /* add end packet */1388 memcpy((void *)init_table, &end, sizeof(struct mmsch_v1_0_cmd_end));1389 table_size += sizeof(struct mmsch_v1_0_cmd_end) / 4;1390 init_table += sizeof(struct mmsch_v1_0_cmd_end) / 4;1391 1392 /* refine header */1393 header->eng[i].table_size = table_size;1394 header->total_size += table_size;1395 }1396 1397 return vcn_v2_5_mmsch_start(adev, &adev->virt.mm_table);1398}1399 1400static int vcn_v2_5_stop_dpg_mode(struct amdgpu_device *adev, int inst_idx)1401{1402 uint32_t tmp;1403 1404 /* Wait for power status to be 1 */1405 SOC15_WAIT_ON_RREG(VCN, inst_idx, mmUVD_POWER_STATUS, 1,1406 UVD_POWER_STATUS__UVD_POWER_STATUS_MASK);1407 1408 /* wait for read ptr to be equal to write ptr */1409 tmp = RREG32_SOC15(VCN, inst_idx, mmUVD_RB_WPTR);1410 SOC15_WAIT_ON_RREG(VCN, inst_idx, mmUVD_RB_RPTR, tmp, 0xFFFFFFFF);1411 1412 tmp = RREG32_SOC15(VCN, inst_idx, mmUVD_RB_WPTR2);1413 SOC15_WAIT_ON_RREG(VCN, inst_idx, mmUVD_RB_RPTR2, tmp, 0xFFFFFFFF);1414 1415 tmp = RREG32_SOC15(VCN, inst_idx, mmUVD_RBC_RB_WPTR) & 0x7FFFFFFF;1416 SOC15_WAIT_ON_RREG(VCN, inst_idx, mmUVD_RBC_RB_RPTR, tmp, 0xFFFFFFFF);1417 1418 SOC15_WAIT_ON_RREG(VCN, inst_idx, mmUVD_POWER_STATUS, 1,1419 UVD_POWER_STATUS__UVD_POWER_STATUS_MASK);1420 1421 /* disable dynamic power gating mode */1422 WREG32_P(SOC15_REG_OFFSET(VCN, inst_idx, mmUVD_POWER_STATUS), 0,1423 ~UVD_POWER_STATUS__UVD_PG_MODE_MASK);1424 1425 return 0;1426}1427 1428static int vcn_v2_5_stop(struct amdgpu_device *adev)1429{1430 uint32_t tmp;1431 int i, r = 0;1432 1433 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {1434 if (adev->vcn.harvest_config & (1 << i))1435 continue;1436 if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG) {1437 r = vcn_v2_5_stop_dpg_mode(adev, i);1438 continue;1439 }1440 1441 /* wait for vcn idle */1442 r = SOC15_WAIT_ON_RREG(VCN, i, mmUVD_STATUS, UVD_STATUS__IDLE, 0x7);1443 if (r)1444 return r;1445 1446 tmp = UVD_LMI_STATUS__VCPU_LMI_WRITE_CLEAN_MASK |1447 UVD_LMI_STATUS__READ_CLEAN_MASK |1448 UVD_LMI_STATUS__WRITE_CLEAN_MASK |1449 UVD_LMI_STATUS__WRITE_CLEAN_RAW_MASK;1450 r = SOC15_WAIT_ON_RREG(VCN, i, mmUVD_LMI_STATUS, tmp, tmp);1451 if (r)1452 return r;1453 1454 /* block LMI UMC channel */1455 tmp = RREG32_SOC15(VCN, i, mmUVD_LMI_CTRL2);1456 tmp |= UVD_LMI_CTRL2__STALL_ARB_UMC_MASK;1457 WREG32_SOC15(VCN, i, mmUVD_LMI_CTRL2, tmp);1458 1459 tmp = UVD_LMI_STATUS__UMC_READ_CLEAN_RAW_MASK|1460 UVD_LMI_STATUS__UMC_WRITE_CLEAN_RAW_MASK;1461 r = SOC15_WAIT_ON_RREG(VCN, i, mmUVD_LMI_STATUS, tmp, tmp);1462 if (r)1463 return r;1464 1465 /* block VCPU register access */1466 WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_RB_ARB_CTRL),1467 UVD_RB_ARB_CTRL__VCPU_DIS_MASK,1468 ~UVD_RB_ARB_CTRL__VCPU_DIS_MASK);1469 1470 /* reset VCPU */1471 WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_VCPU_CNTL),1472 UVD_VCPU_CNTL__BLK_RST_MASK,1473 ~UVD_VCPU_CNTL__BLK_RST_MASK);1474 1475 /* disable VCPU clock */1476 WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_VCPU_CNTL), 0,1477 ~(UVD_VCPU_CNTL__CLK_EN_MASK));1478 1479 /* clear status */1480 WREG32_SOC15(VCN, i, mmUVD_STATUS, 0);1481 1482 vcn_v2_5_enable_clock_gating(adev);1483 1484 /* enable register anti-hang mechanism */1485 WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_POWER_STATUS),1486 UVD_POWER_STATUS__UVD_POWER_STATUS_MASK,1487 ~UVD_POWER_STATUS__UVD_POWER_STATUS_MASK);1488 }1489 1490 if (adev->pm.dpm_enabled)1491 amdgpu_dpm_enable_uvd(adev, false);1492 1493 return 0;1494}1495 1496static int vcn_v2_5_pause_dpg_mode(struct amdgpu_device *adev,1497 int inst_idx, struct dpg_pause_state *new_state)1498{1499 struct amdgpu_ring *ring;1500 uint32_t reg_data = 0;1501 int ret_code = 0;1502 1503 /* pause/unpause if state is changed */1504 if (adev->vcn.inst[inst_idx].pause_state.fw_based != new_state->fw_based) {1505 DRM_DEBUG("dpg pause state changed %d -> %d",1506 adev->vcn.inst[inst_idx].pause_state.fw_based, new_state->fw_based);1507 reg_data = RREG32_SOC15(VCN, inst_idx, mmUVD_DPG_PAUSE) &1508 (~UVD_DPG_PAUSE__NJ_PAUSE_DPG_ACK_MASK);1509 1510 if (new_state->fw_based == VCN_DPG_STATE__PAUSE) {1511 ret_code = SOC15_WAIT_ON_RREG(VCN, inst_idx, mmUVD_POWER_STATUS, 0x1,1512 UVD_POWER_STATUS__UVD_POWER_STATUS_MASK);1513 1514 if (!ret_code) {1515 volatile struct amdgpu_fw_shared *fw_shared = adev->vcn.inst[inst_idx].fw_shared.cpu_addr;1516 1517 /* pause DPG */1518 reg_data |= UVD_DPG_PAUSE__NJ_PAUSE_DPG_REQ_MASK;1519 WREG32_SOC15(VCN, inst_idx, mmUVD_DPG_PAUSE, reg_data);1520 1521 /* wait for ACK */1522 SOC15_WAIT_ON_RREG(VCN, inst_idx, mmUVD_DPG_PAUSE,1523 UVD_DPG_PAUSE__NJ_PAUSE_DPG_ACK_MASK,1524 UVD_DPG_PAUSE__NJ_PAUSE_DPG_ACK_MASK);1525 1526 /* Stall DPG before WPTR/RPTR reset */1527 WREG32_P(SOC15_REG_OFFSET(VCN, inst_idx, mmUVD_POWER_STATUS),1528 UVD_POWER_STATUS__STALL_DPG_POWER_UP_MASK,1529 ~UVD_POWER_STATUS__STALL_DPG_POWER_UP_MASK);1530 1531 /* Restore */1532 fw_shared->multi_queue.encode_generalpurpose_queue_mode |= FW_QUEUE_RING_RESET;1533 ring = &adev->vcn.inst[inst_idx].ring_enc[0];1534 ring->wptr = 0;1535 WREG32_SOC15(VCN, inst_idx, mmUVD_RB_BASE_LO, ring->gpu_addr);1536 WREG32_SOC15(VCN, inst_idx, mmUVD_RB_BASE_HI, upper_32_bits(ring->gpu_addr));1537 WREG32_SOC15(VCN, inst_idx, mmUVD_RB_SIZE, ring->ring_size / 4);1538 WREG32_SOC15(VCN, inst_idx, mmUVD_RB_RPTR, lower_32_bits(ring->wptr));1539 WREG32_SOC15(VCN, inst_idx, mmUVD_RB_WPTR, lower_32_bits(ring->wptr));1540 fw_shared->multi_queue.encode_generalpurpose_queue_mode &= ~FW_QUEUE_RING_RESET;1541 1542 fw_shared->multi_queue.encode_lowlatency_queue_mode |= FW_QUEUE_RING_RESET;1543 ring = &adev->vcn.inst[inst_idx].ring_enc[1];1544 ring->wptr = 0;1545 WREG32_SOC15(VCN, inst_idx, mmUVD_RB_BASE_LO2, ring->gpu_addr);1546 WREG32_SOC15(VCN, inst_idx, mmUVD_RB_BASE_HI2, upper_32_bits(ring->gpu_addr));1547 WREG32_SOC15(VCN, inst_idx, mmUVD_RB_SIZE2, ring->ring_size / 4);1548 WREG32_SOC15(VCN, inst_idx, mmUVD_RB_RPTR2, lower_32_bits(ring->wptr));1549 WREG32_SOC15(VCN, inst_idx, mmUVD_RB_WPTR2, lower_32_bits(ring->wptr));1550 fw_shared->multi_queue.encode_lowlatency_queue_mode &= ~FW_QUEUE_RING_RESET;1551 1552 /* Unstall DPG */1553 WREG32_P(SOC15_REG_OFFSET(VCN, inst_idx, mmUVD_POWER_STATUS),1554 0, ~UVD_POWER_STATUS__STALL_DPG_POWER_UP_MASK);1555 1556 SOC15_WAIT_ON_RREG(VCN, inst_idx, mmUVD_POWER_STATUS,1557 UVD_PGFSM_CONFIG__UVDM_UVDU_PWR_ON, UVD_POWER_STATUS__UVD_POWER_STATUS_MASK);1558 }1559 } else {1560 reg_data &= ~UVD_DPG_PAUSE__NJ_PAUSE_DPG_REQ_MASK;1561 WREG32_SOC15(VCN, inst_idx, mmUVD_DPG_PAUSE, reg_data);1562 SOC15_WAIT_ON_RREG(VCN, inst_idx, mmUVD_POWER_STATUS, 0x1,1563 UVD_POWER_STATUS__UVD_POWER_STATUS_MASK);1564 }1565 adev->vcn.inst[inst_idx].pause_state.fw_based = new_state->fw_based;1566 }1567 1568 return 0;1569}1570 1571/**1572 * vcn_v2_5_dec_ring_get_rptr - get read pointer1573 *1574 * @ring: amdgpu_ring pointer1575 *1576 * Returns the current hardware read pointer1577 */1578static uint64_t vcn_v2_5_dec_ring_get_rptr(struct amdgpu_ring *ring)1579{1580 struct amdgpu_device *adev = ring->adev;1581 1582 return RREG32_SOC15(VCN, ring->me, mmUVD_RBC_RB_RPTR);1583}1584 1585/**1586 * vcn_v2_5_dec_ring_get_wptr - get write pointer1587 *1588 * @ring: amdgpu_ring pointer1589 *1590 * Returns the current hardware write pointer1591 */1592static uint64_t vcn_v2_5_dec_ring_get_wptr(struct amdgpu_ring *ring)1593{1594 struct amdgpu_device *adev = ring->adev;1595 1596 if (ring->use_doorbell)1597 return *ring->wptr_cpu_addr;1598 else1599 return RREG32_SOC15(VCN, ring->me, mmUVD_RBC_RB_WPTR);1600}1601 1602/**1603 * vcn_v2_5_dec_ring_set_wptr - set write pointer1604 *1605 * @ring: amdgpu_ring pointer1606 *1607 * Commits the write pointer to the hardware1608 */1609static void vcn_v2_5_dec_ring_set_wptr(struct amdgpu_ring *ring)1610{1611 struct amdgpu_device *adev = ring->adev;1612 1613 if (ring->use_doorbell) {1614 *ring->wptr_cpu_addr = lower_32_bits(ring->wptr);1615 WDOORBELL32(ring->doorbell_index, lower_32_bits(ring->wptr));1616 } else {1617 WREG32_SOC15(VCN, ring->me, mmUVD_RBC_RB_WPTR, lower_32_bits(ring->wptr));1618 }1619}1620 1621static const struct amdgpu_ring_funcs vcn_v2_5_dec_ring_vm_funcs = {1622 .type = AMDGPU_RING_TYPE_VCN_DEC,1623 .align_mask = 0xf,1624 .secure_submission_supported = true,1625 .get_rptr = vcn_v2_5_dec_ring_get_rptr,1626 .get_wptr = vcn_v2_5_dec_ring_get_wptr,1627 .set_wptr = vcn_v2_5_dec_ring_set_wptr,1628 .emit_frame_size =1629 SOC15_FLUSH_GPU_TLB_NUM_WREG * 6 +1630 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 8 +1631 8 + /* vcn_v2_0_dec_ring_emit_vm_flush */1632 14 + 14 + /* vcn_v2_0_dec_ring_emit_fence x2 vm fence */1633 6,1634 .emit_ib_size = 8, /* vcn_v2_0_dec_ring_emit_ib */1635 .emit_ib = vcn_v2_0_dec_ring_emit_ib,1636 .emit_fence = vcn_v2_0_dec_ring_emit_fence,1637 .emit_vm_flush = vcn_v2_0_dec_ring_emit_vm_flush,1638 .test_ring = vcn_v2_0_dec_ring_test_ring,1639 .test_ib = amdgpu_vcn_dec_ring_test_ib,1640 .insert_nop = vcn_v2_0_dec_ring_insert_nop,1641 .insert_start = vcn_v2_0_dec_ring_insert_start,1642 .insert_end = vcn_v2_0_dec_ring_insert_end,1643 .pad_ib = amdgpu_ring_generic_pad_ib,1644 .begin_use = amdgpu_vcn_ring_begin_use,1645 .end_use = amdgpu_vcn_ring_end_use,1646 .emit_wreg = vcn_v2_0_dec_ring_emit_wreg,1647 .emit_reg_wait = vcn_v2_0_dec_ring_emit_reg_wait,1648 .emit_reg_write_reg_wait = amdgpu_ring_emit_reg_write_reg_wait_helper,1649};1650 1651/**1652 * vcn_v2_5_enc_ring_get_rptr - get enc read pointer1653 *1654 * @ring: amdgpu_ring pointer1655 *1656 * Returns the current hardware enc read pointer1657 */1658static uint64_t vcn_v2_5_enc_ring_get_rptr(struct amdgpu_ring *ring)1659{1660 struct amdgpu_device *adev = ring->adev;1661 1662 if (ring == &adev->vcn.inst[ring->me].ring_enc[0])1663 return RREG32_SOC15(VCN, ring->me, mmUVD_RB_RPTR);1664 else1665 return RREG32_SOC15(VCN, ring->me, mmUVD_RB_RPTR2);1666}1667 1668/**1669 * vcn_v2_5_enc_ring_get_wptr - get enc write pointer1670 *1671 * @ring: amdgpu_ring pointer1672 *1673 * Returns the current hardware enc write pointer1674 */1675static uint64_t vcn_v2_5_enc_ring_get_wptr(struct amdgpu_ring *ring)1676{1677 struct amdgpu_device *adev = ring->adev;1678 1679 if (ring == &adev->vcn.inst[ring->me].ring_enc[0]) {1680 if (ring->use_doorbell)1681 return *ring->wptr_cpu_addr;1682 else1683 return RREG32_SOC15(VCN, ring->me, mmUVD_RB_WPTR);1684 } else {1685 if (ring->use_doorbell)1686 return *ring->wptr_cpu_addr;1687 else1688 return RREG32_SOC15(VCN, ring->me, mmUVD_RB_WPTR2);1689 }1690}1691 1692/**1693 * vcn_v2_5_enc_ring_set_wptr - set enc write pointer1694 *1695 * @ring: amdgpu_ring pointer1696 *1697 * Commits the enc write pointer to the hardware1698 */1699static void vcn_v2_5_enc_ring_set_wptr(struct amdgpu_ring *ring)1700{1701 struct amdgpu_device *adev = ring->adev;1702 1703 if (ring == &adev->vcn.inst[ring->me].ring_enc[0]) {1704 if (ring->use_doorbell) {1705 *ring->wptr_cpu_addr = lower_32_bits(ring->wptr);1706 WDOORBELL32(ring->doorbell_index, lower_32_bits(ring->wptr));1707 } else {1708 WREG32_SOC15(VCN, ring->me, mmUVD_RB_WPTR, lower_32_bits(ring->wptr));1709 }1710 } else {1711 if (ring->use_doorbell) {1712 *ring->wptr_cpu_addr = lower_32_bits(ring->wptr);1713 WDOORBELL32(ring->doorbell_index, lower_32_bits(ring->wptr));1714 } else {1715 WREG32_SOC15(VCN, ring->me, mmUVD_RB_WPTR2, lower_32_bits(ring->wptr));1716 }1717 }1718}1719 1720static const struct amdgpu_ring_funcs vcn_v2_5_enc_ring_vm_funcs = {1721 .type = AMDGPU_RING_TYPE_VCN_ENC,1722 .align_mask = 0x3f,1723 .nop = VCN_ENC_CMD_NO_OP,1724 .get_rptr = vcn_v2_5_enc_ring_get_rptr,1725 .get_wptr = vcn_v2_5_enc_ring_get_wptr,1726 .set_wptr = vcn_v2_5_enc_ring_set_wptr,1727 .emit_frame_size =1728 SOC15_FLUSH_GPU_TLB_NUM_WREG * 3 +1729 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 4 +1730 4 + /* vcn_v2_0_enc_ring_emit_vm_flush */1731 5 + 5 + /* vcn_v2_0_enc_ring_emit_fence x2 vm fence */1732 1, /* vcn_v2_0_enc_ring_insert_end */1733 .emit_ib_size = 5, /* vcn_v2_0_enc_ring_emit_ib */1734 .emit_ib = vcn_v2_0_enc_ring_emit_ib,1735 .emit_fence = vcn_v2_0_enc_ring_emit_fence,1736 .emit_vm_flush = vcn_v2_0_enc_ring_emit_vm_flush,1737 .test_ring = amdgpu_vcn_enc_ring_test_ring,1738 .test_ib = amdgpu_vcn_enc_ring_test_ib,1739 .insert_nop = amdgpu_ring_insert_nop,1740 .insert_end = vcn_v2_0_enc_ring_insert_end,1741 .pad_ib = amdgpu_ring_generic_pad_ib,1742 .begin_use = amdgpu_vcn_ring_begin_use,1743 .end_use = amdgpu_vcn_ring_end_use,1744 .emit_wreg = vcn_v2_0_enc_ring_emit_wreg,1745 .emit_reg_wait = vcn_v2_0_enc_ring_emit_reg_wait,1746 .emit_reg_write_reg_wait = amdgpu_ring_emit_reg_write_reg_wait_helper,1747};1748 1749static void vcn_v2_5_set_dec_ring_funcs(struct amdgpu_device *adev)1750{1751 int i;1752 1753 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {1754 if (adev->vcn.harvest_config & (1 << i))1755 continue;1756 adev->vcn.inst[i].ring_dec.funcs = &vcn_v2_5_dec_ring_vm_funcs;1757 adev->vcn.inst[i].ring_dec.me = i;1758 }1759}1760 1761static void vcn_v2_5_set_enc_ring_funcs(struct amdgpu_device *adev)1762{1763 int i, j;1764 1765 for (j = 0; j < adev->vcn.num_vcn_inst; ++j) {1766 if (adev->vcn.harvest_config & (1 << j))1767 continue;1768 for (i = 0; i < adev->vcn.num_enc_rings; ++i) {1769 adev->vcn.inst[j].ring_enc[i].funcs = &vcn_v2_5_enc_ring_vm_funcs;1770 adev->vcn.inst[j].ring_enc[i].me = j;1771 }1772 }1773}1774 1775static bool vcn_v2_5_is_idle(void *handle)1776{1777 struct amdgpu_device *adev = (struct amdgpu_device *)handle;1778 int i, ret = 1;1779 1780 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {1781 if (adev->vcn.harvest_config & (1 << i))1782 continue;1783 ret &= (RREG32_SOC15(VCN, i, mmUVD_STATUS) == UVD_STATUS__IDLE);1784 }1785 1786 return ret;1787}1788 1789static int vcn_v2_5_wait_for_idle(void *handle)1790{1791 struct amdgpu_device *adev = (struct amdgpu_device *)handle;1792 int i, ret = 0;1793 1794 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {1795 if (adev->vcn.harvest_config & (1 << i))1796 continue;1797 ret = SOC15_WAIT_ON_RREG(VCN, i, mmUVD_STATUS, UVD_STATUS__IDLE,1798 UVD_STATUS__IDLE);1799 if (ret)1800 return ret;1801 }1802 1803 return ret;1804}1805 1806static int vcn_v2_5_set_clockgating_state(void *handle,1807 enum amd_clockgating_state state)1808{1809 struct amdgpu_device *adev = (struct amdgpu_device *)handle;1810 bool enable = (state == AMD_CG_STATE_GATE);1811 1812 if (amdgpu_sriov_vf(adev))1813 return 0;1814 1815 if (enable) {1816 if (!vcn_v2_5_is_idle(handle))1817 return -EBUSY;1818 vcn_v2_5_enable_clock_gating(adev);1819 } else {1820 vcn_v2_5_disable_clock_gating(adev);1821 }1822 1823 return 0;1824}1825 1826static int vcn_v2_5_set_powergating_state(void *handle,1827 enum amd_powergating_state state)1828{1829 struct amdgpu_device *adev = (struct amdgpu_device *)handle;1830 int ret;1831 1832 if (amdgpu_sriov_vf(adev))1833 return 0;1834 1835 if(state == adev->vcn.cur_state)1836 return 0;1837 1838 if (state == AMD_PG_STATE_GATE)1839 ret = vcn_v2_5_stop(adev);1840 else1841 ret = vcn_v2_5_start(adev);1842 1843 if(!ret)1844 adev->vcn.cur_state = state;1845 1846 return ret;1847}1848 1849static int vcn_v2_5_set_interrupt_state(struct amdgpu_device *adev,1850 struct amdgpu_irq_src *source,1851 unsigned type,1852 enum amdgpu_interrupt_state state)1853{1854 return 0;1855}1856 1857static int vcn_v2_6_set_ras_interrupt_state(struct amdgpu_device *adev,1858 struct amdgpu_irq_src *source,1859 unsigned int type,1860 enum amdgpu_interrupt_state state)1861{1862 return 0;1863}1864 1865static int vcn_v2_5_process_interrupt(struct amdgpu_device *adev,1866 struct amdgpu_irq_src *source,1867 struct amdgpu_iv_entry *entry)1868{1869 uint32_t ip_instance;1870 1871 switch (entry->client_id) {1872 case SOC15_IH_CLIENTID_VCN:1873 ip_instance = 0;1874 break;1875 case SOC15_IH_CLIENTID_VCN1:1876 ip_instance = 1;1877 break;1878 default:1879 DRM_ERROR("Unhandled client id: %d\n", entry->client_id);1880 return 0;1881 }1882 1883 DRM_DEBUG("IH: VCN TRAP\n");1884 1885 switch (entry->src_id) {1886 case VCN_2_0__SRCID__UVD_SYSTEM_MESSAGE_INTERRUPT:1887 amdgpu_fence_process(&adev->vcn.inst[ip_instance].ring_dec);1888 break;1889 case VCN_2_0__SRCID__UVD_ENC_GENERAL_PURPOSE:1890 amdgpu_fence_process(&adev->vcn.inst[ip_instance].ring_enc[0]);1891 break;1892 case VCN_2_0__SRCID__UVD_ENC_LOW_LATENCY:1893 amdgpu_fence_process(&adev->vcn.inst[ip_instance].ring_enc[1]);1894 break;1895 default:1896 DRM_ERROR("Unhandled interrupt: %d %d\n",1897 entry->src_id, entry->src_data[0]);1898 break;1899 }1900 1901 return 0;1902}1903 1904static const struct amdgpu_irq_src_funcs vcn_v2_5_irq_funcs = {1905 .set = vcn_v2_5_set_interrupt_state,1906 .process = vcn_v2_5_process_interrupt,1907};1908 1909static const struct amdgpu_irq_src_funcs vcn_v2_6_ras_irq_funcs = {1910 .set = vcn_v2_6_set_ras_interrupt_state,1911 .process = amdgpu_vcn_process_poison_irq,1912};1913 1914static void vcn_v2_5_set_irq_funcs(struct amdgpu_device *adev)1915{1916 int i;1917 1918 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {1919 if (adev->vcn.harvest_config & (1 << i))1920 continue;1921 adev->vcn.inst[i].irq.num_types = adev->vcn.num_enc_rings + 1;1922 adev->vcn.inst[i].irq.funcs = &vcn_v2_5_irq_funcs;1923 1924 adev->vcn.inst[i].ras_poison_irq.num_types = adev->vcn.num_enc_rings + 1;1925 adev->vcn.inst[i].ras_poison_irq.funcs = &vcn_v2_6_ras_irq_funcs;1926 }1927}1928 1929static void vcn_v2_5_print_ip_state(void *handle, struct drm_printer *p)1930{1931 struct amdgpu_device *adev = (struct amdgpu_device *)handle;1932 int i, j;1933 uint32_t reg_count = ARRAY_SIZE(vcn_reg_list_2_5);1934 uint32_t inst_off, is_powered;1935 1936 if (!adev->vcn.ip_dump)1937 return;1938 1939 drm_printf(p, "num_instances:%d\n", adev->vcn.num_vcn_inst);1940 for (i = 0; i < adev->vcn.num_vcn_inst; i++) {1941 if (adev->vcn.harvest_config & (1 << i)) {1942 drm_printf(p, "\nHarvested Instance:VCN%d Skipping dump\n", i);1943 continue;1944 }1945 1946 inst_off = i * reg_count;1947 is_powered = (adev->vcn.ip_dump[inst_off] &1948 UVD_POWER_STATUS__UVD_POWER_STATUS_MASK) != 1;1949 1950 if (is_powered) {1951 drm_printf(p, "\nActive Instance:VCN%d\n", i);1952 for (j = 0; j < reg_count; j++)1953 drm_printf(p, "%-50s \t 0x%08x\n", vcn_reg_list_2_5[j].reg_name,1954 adev->vcn.ip_dump[inst_off + j]);1955 } else {1956 drm_printf(p, "\nInactive Instance:VCN%d\n", i);1957 }1958 }1959}1960 1961static void vcn_v2_5_dump_ip_state(void *handle)1962{1963 struct amdgpu_device *adev = (struct amdgpu_device *)handle;1964 int i, j;1965 bool is_powered;1966 uint32_t inst_off;1967 uint32_t reg_count = ARRAY_SIZE(vcn_reg_list_2_5);1968 1969 if (!adev->vcn.ip_dump)1970 return;1971 1972 for (i = 0; i < adev->vcn.num_vcn_inst; i++) {1973 if (adev->vcn.harvest_config & (1 << i))1974 continue;1975 1976 inst_off = i * reg_count;1977 /* mmUVD_POWER_STATUS is always readable and is first element of the array */1978 adev->vcn.ip_dump[inst_off] = RREG32_SOC15(VCN, i, mmUVD_POWER_STATUS);1979 is_powered = (adev->vcn.ip_dump[inst_off] &1980 UVD_POWER_STATUS__UVD_POWER_STATUS_MASK) != 1;1981 1982 if (is_powered)1983 for (j = 1; j < reg_count; j++)1984 adev->vcn.ip_dump[inst_off + j] =1985 RREG32(SOC15_REG_ENTRY_OFFSET_INST(vcn_reg_list_2_5[j], i));1986 }1987}1988 1989static const struct amd_ip_funcs vcn_v2_5_ip_funcs = {1990 .name = "vcn_v2_5",1991 .early_init = vcn_v2_5_early_init,1992 .late_init = NULL,1993 .sw_init = vcn_v2_5_sw_init,1994 .sw_fini = vcn_v2_5_sw_fini,1995 .hw_init = vcn_v2_5_hw_init,1996 .hw_fini = vcn_v2_5_hw_fini,1997 .suspend = vcn_v2_5_suspend,1998 .resume = vcn_v2_5_resume,1999 .is_idle = vcn_v2_5_is_idle,2000 .wait_for_idle = vcn_v2_5_wait_for_idle,2001 .check_soft_reset = NULL,2002 .pre_soft_reset = NULL,2003 .soft_reset = NULL,2004 .post_soft_reset = NULL,2005 .set_clockgating_state = vcn_v2_5_set_clockgating_state,2006 .set_powergating_state = vcn_v2_5_set_powergating_state,2007 .dump_ip_state = vcn_v2_5_dump_ip_state,2008 .print_ip_state = vcn_v2_5_print_ip_state,2009};2010 2011static const struct amd_ip_funcs vcn_v2_6_ip_funcs = {2012 .name = "vcn_v2_6",2013 .early_init = vcn_v2_5_early_init,2014 .late_init = NULL,2015 .sw_init = vcn_v2_5_sw_init,2016 .sw_fini = vcn_v2_5_sw_fini,2017 .hw_init = vcn_v2_5_hw_init,2018 .hw_fini = vcn_v2_5_hw_fini,2019 .suspend = vcn_v2_5_suspend,2020 .resume = vcn_v2_5_resume,2021 .is_idle = vcn_v2_5_is_idle,2022 .wait_for_idle = vcn_v2_5_wait_for_idle,2023 .check_soft_reset = NULL,2024 .pre_soft_reset = NULL,2025 .soft_reset = NULL,2026 .post_soft_reset = NULL,2027 .set_clockgating_state = vcn_v2_5_set_clockgating_state,2028 .set_powergating_state = vcn_v2_5_set_powergating_state,2029 .dump_ip_state = vcn_v2_5_dump_ip_state,2030 .print_ip_state = vcn_v2_5_print_ip_state,2031};2032 2033const struct amdgpu_ip_block_version vcn_v2_5_ip_block =2034{2035 .type = AMD_IP_BLOCK_TYPE_VCN,2036 .major = 2,2037 .minor = 5,2038 .rev = 0,2039 .funcs = &vcn_v2_5_ip_funcs,2040};2041 2042const struct amdgpu_ip_block_version vcn_v2_6_ip_block =2043{2044 .type = AMD_IP_BLOCK_TYPE_VCN,2045 .major = 2,2046 .minor = 6,2047 .rev = 0,2048 .funcs = &vcn_v2_6_ip_funcs,2049};2050 2051static uint32_t vcn_v2_6_query_poison_by_instance(struct amdgpu_device *adev,2052 uint32_t instance, uint32_t sub_block)2053{2054 uint32_t poison_stat = 0, reg_value = 0;2055 2056 switch (sub_block) {2057 case AMDGPU_VCN_V2_6_VCPU_VCODEC:2058 reg_value = RREG32_SOC15(VCN, instance, mmUVD_RAS_VCPU_VCODEC_STATUS);2059 poison_stat = REG_GET_FIELD(reg_value, UVD_RAS_VCPU_VCODEC_STATUS, POISONED_PF);2060 break;2061 default:2062 break;2063 }2064 2065 if (poison_stat)2066 dev_info(adev->dev, "Poison detected in VCN%d, sub_block%d\n",2067 instance, sub_block);2068 2069 return poison_stat;2070}2071 2072static bool vcn_v2_6_query_poison_status(struct amdgpu_device *adev)2073{2074 uint32_t inst, sub;2075 uint32_t poison_stat = 0;2076 2077 for (inst = 0; inst < adev->vcn.num_vcn_inst; inst++)2078 for (sub = 0; sub < AMDGPU_VCN_V2_6_MAX_SUB_BLOCK; sub++)2079 poison_stat +=2080 vcn_v2_6_query_poison_by_instance(adev, inst, sub);2081 2082 return !!poison_stat;2083}2084 2085const struct amdgpu_ras_block_hw_ops vcn_v2_6_ras_hw_ops = {2086 .query_poison_status = vcn_v2_6_query_poison_status,2087};2088 2089static struct amdgpu_vcn_ras vcn_v2_6_ras = {2090 .ras_block = {2091 .hw_ops = &vcn_v2_6_ras_hw_ops,2092 .ras_late_init = amdgpu_vcn_ras_late_init,2093 },2094};2095 2096static void vcn_v2_5_set_ras_funcs(struct amdgpu_device *adev)2097{2098 switch (amdgpu_ip_version(adev, VCN_HWIP, 0)) {2099 case IP_VERSION(2, 6, 0):2100 adev->vcn.ras = &vcn_v2_6_ras;2101 break;2102 default:2103 break;2104 }2105}2106