1158 lines · c
1/*2 * Copyright 2016 Advanced Micro Devices, Inc.3 * All Rights Reserved.4 *5 * Permission is hereby granted, free of charge, to any person obtaining a6 * copy of this software and associated documentation files (the7 * "Software"), to deal in the Software without restriction, including8 * without limitation the rights to use, copy, modify, merge, publish,9 * distribute, sub license, and/or sell copies of the Software, and to10 * permit persons to whom the Software is furnished to do so, subject to11 * the following conditions:12 *13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,15 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL16 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,17 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR18 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE19 * USE OR OTHER DEALINGS IN THE SOFTWARE.20 *21 * The above copyright notice and this permission notice (including the22 * next paragraph) shall be included in all copies or substantial portions23 * of the Software.24 *25 */26 27#include <linux/firmware.h>28#include <drm/drm_drv.h>29 30#include "amdgpu.h"31#include "amdgpu_vce.h"32#include "soc15.h"33#include "soc15d.h"34#include "soc15_common.h"35#include "mmsch_v1_0.h"36 37#include "vce/vce_4_0_offset.h"38#include "vce/vce_4_0_default.h"39#include "vce/vce_4_0_sh_mask.h"40#include "mmhub/mmhub_1_0_offset.h"41#include "mmhub/mmhub_1_0_sh_mask.h"42 43#include "ivsrcid/vce/irqsrcs_vce_4_0.h"44 45#define VCE_STATUS_VCPU_REPORT_FW_LOADED_MASK 0x0246 47#define VCE_V4_0_FW_SIZE (384 * 1024)48#define VCE_V4_0_STACK_SIZE (64 * 1024)49#define VCE_V4_0_DATA_SIZE ((16 * 1024 * AMDGPU_MAX_VCE_HANDLES) + (52 * 1024))50 51static void vce_v4_0_mc_resume(struct amdgpu_device *adev);52static void vce_v4_0_set_ring_funcs(struct amdgpu_device *adev);53static void vce_v4_0_set_irq_funcs(struct amdgpu_device *adev);54 55/**56 * vce_v4_0_ring_get_rptr - get read pointer57 *58 * @ring: amdgpu_ring pointer59 *60 * Returns the current hardware read pointer61 */62static uint64_t vce_v4_0_ring_get_rptr(struct amdgpu_ring *ring)63{64 struct amdgpu_device *adev = ring->adev;65 66 if (ring->me == 0)67 return RREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_RB_RPTR));68 else if (ring->me == 1)69 return RREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_RB_RPTR2));70 else71 return RREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_RB_RPTR3));72}73 74/**75 * vce_v4_0_ring_get_wptr - get write pointer76 *77 * @ring: amdgpu_ring pointer78 *79 * Returns the current hardware write pointer80 */81static uint64_t vce_v4_0_ring_get_wptr(struct amdgpu_ring *ring)82{83 struct amdgpu_device *adev = ring->adev;84 85 if (ring->use_doorbell)86 return *ring->wptr_cpu_addr;87 88 if (ring->me == 0)89 return RREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_RB_WPTR));90 else if (ring->me == 1)91 return RREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_RB_WPTR2));92 else93 return RREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_RB_WPTR3));94}95 96/**97 * vce_v4_0_ring_set_wptr - set write pointer98 *99 * @ring: amdgpu_ring pointer100 *101 * Commits the write pointer to the hardware102 */103static void vce_v4_0_ring_set_wptr(struct amdgpu_ring *ring)104{105 struct amdgpu_device *adev = ring->adev;106 107 if (ring->use_doorbell) {108 /* XXX check if swapping is necessary on BE */109 *ring->wptr_cpu_addr = lower_32_bits(ring->wptr);110 WDOORBELL32(ring->doorbell_index, lower_32_bits(ring->wptr));111 return;112 }113 114 if (ring->me == 0)115 WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_RB_WPTR),116 lower_32_bits(ring->wptr));117 else if (ring->me == 1)118 WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_RB_WPTR2),119 lower_32_bits(ring->wptr));120 else121 WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_RB_WPTR3),122 lower_32_bits(ring->wptr));123}124 125static int vce_v4_0_firmware_loaded(struct amdgpu_device *adev)126{127 int i, j;128 129 for (i = 0; i < 10; ++i) {130 for (j = 0; j < 100; ++j) {131 uint32_t status =132 RREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_STATUS));133 134 if (status & VCE_STATUS_VCPU_REPORT_FW_LOADED_MASK)135 return 0;136 mdelay(10);137 }138 139 DRM_ERROR("VCE not responding, trying to reset the ECPU!!!\n");140 WREG32_P(SOC15_REG_OFFSET(VCE, 0, mmVCE_SOFT_RESET),141 VCE_SOFT_RESET__ECPU_SOFT_RESET_MASK,142 ~VCE_SOFT_RESET__ECPU_SOFT_RESET_MASK);143 mdelay(10);144 WREG32_P(SOC15_REG_OFFSET(VCE, 0, mmVCE_SOFT_RESET), 0,145 ~VCE_SOFT_RESET__ECPU_SOFT_RESET_MASK);146 mdelay(10);147 148 }149 150 return -ETIMEDOUT;151}152 153static int vce_v4_0_mmsch_start(struct amdgpu_device *adev,154 struct amdgpu_mm_table *table)155{156 uint32_t data = 0, loop;157 uint64_t addr = table->gpu_addr;158 struct mmsch_v1_0_init_header *header = (struct mmsch_v1_0_init_header *)table->cpu_addr;159 uint32_t size;160 161 size = header->header_size + header->vce_table_size + header->uvd_table_size;162 163 /* 1, write to vce_mmsch_vf_ctx_addr_lo/hi register with GPU mc addr of memory descriptor location */164 WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_MMSCH_VF_CTX_ADDR_LO), lower_32_bits(addr));165 WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_MMSCH_VF_CTX_ADDR_HI), upper_32_bits(addr));166 167 /* 2, update vmid of descriptor */168 data = RREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_MMSCH_VF_VMID));169 data &= ~VCE_MMSCH_VF_VMID__VF_CTX_VMID_MASK;170 data |= (0 << VCE_MMSCH_VF_VMID__VF_CTX_VMID__SHIFT); /* use domain0 for MM scheduler */171 WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_MMSCH_VF_VMID), data);172 173 /* 3, notify mmsch about the size of this descriptor */174 WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_MMSCH_VF_CTX_SIZE), size);175 176 /* 4, set resp to zero */177 WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_MMSCH_VF_MAILBOX_RESP), 0);178 179 WDOORBELL32(adev->vce.ring[0].doorbell_index, 0);180 *adev->vce.ring[0].wptr_cpu_addr = 0;181 adev->vce.ring[0].wptr = 0;182 adev->vce.ring[0].wptr_old = 0;183 184 /* 5, kick off the initialization and wait until VCE_MMSCH_VF_MAILBOX_RESP becomes non-zero */185 WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_MMSCH_VF_MAILBOX_HOST), 0x10000001);186 187 data = RREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_MMSCH_VF_MAILBOX_RESP));188 loop = 1000;189 while ((data & 0x10000002) != 0x10000002) {190 udelay(10);191 data = RREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_MMSCH_VF_MAILBOX_RESP));192 loop--;193 if (!loop)194 break;195 }196 197 if (!loop) {198 dev_err(adev->dev, "failed to init MMSCH, mmVCE_MMSCH_VF_MAILBOX_RESP = %x\n", data);199 return -EBUSY;200 }201 202 return 0;203}204 205static int vce_v4_0_sriov_start(struct amdgpu_device *adev)206{207 struct amdgpu_ring *ring;208 uint32_t offset, size;209 uint32_t table_size = 0;210 struct mmsch_v1_0_cmd_direct_write direct_wt = { { 0 } };211 struct mmsch_v1_0_cmd_direct_read_modify_write direct_rd_mod_wt = { { 0 } };212 struct mmsch_v1_0_cmd_direct_polling direct_poll = { { 0 } };213 struct mmsch_v1_0_cmd_end end = { { 0 } };214 uint32_t *init_table = adev->virt.mm_table.cpu_addr;215 struct mmsch_v1_0_init_header *header = (struct mmsch_v1_0_init_header *)init_table;216 217 direct_wt.cmd_header.command_type = MMSCH_COMMAND__DIRECT_REG_WRITE;218 direct_rd_mod_wt.cmd_header.command_type = MMSCH_COMMAND__DIRECT_REG_READ_MODIFY_WRITE;219 direct_poll.cmd_header.command_type = MMSCH_COMMAND__DIRECT_REG_POLLING;220 end.cmd_header.command_type = MMSCH_COMMAND__END;221 222 if (header->vce_table_offset == 0 && header->vce_table_size == 0) {223 header->version = MMSCH_VERSION;224 header->header_size = sizeof(struct mmsch_v1_0_init_header) >> 2;225 226 if (header->uvd_table_offset == 0 && header->uvd_table_size == 0)227 header->vce_table_offset = header->header_size;228 else229 header->vce_table_offset = header->uvd_table_size + header->uvd_table_offset;230 231 init_table += header->vce_table_offset;232 233 ring = &adev->vce.ring[0];234 MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCE, 0, mmVCE_RB_BASE_LO),235 lower_32_bits(ring->gpu_addr));236 MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCE, 0, mmVCE_RB_BASE_HI),237 upper_32_bits(ring->gpu_addr));238 MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCE, 0, mmVCE_RB_SIZE),239 ring->ring_size / 4);240 241 /* BEGING OF MC_RESUME */242 MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCE, 0, mmVCE_LMI_CTRL), 0x398000);243 MMSCH_V1_0_INSERT_DIRECT_RD_MOD_WT(SOC15_REG_OFFSET(VCE, 0, mmVCE_LMI_CACHE_CTRL), ~0x1, 0);244 MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCE, 0, mmVCE_LMI_SWAP_CNTL), 0);245 MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCE, 0, mmVCE_LMI_SWAP_CNTL1), 0);246 MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCE, 0, mmVCE_LMI_VM_CTRL), 0);247 248 offset = AMDGPU_VCE_FIRMWARE_OFFSET;249 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {250 uint32_t low = adev->firmware.ucode[AMDGPU_UCODE_ID_VCE].tmr_mc_addr_lo;251 uint32_t hi = adev->firmware.ucode[AMDGPU_UCODE_ID_VCE].tmr_mc_addr_hi;252 uint64_t tmr_mc_addr = (uint64_t)(hi) << 32 | low;253 254 MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCE, 0,255 mmVCE_LMI_VCPU_CACHE_40BIT_BAR0), tmr_mc_addr >> 8);256 MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCE, 0,257 mmVCE_LMI_VCPU_CACHE_64BIT_BAR0),258 (tmr_mc_addr >> 40) & 0xff);259 MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCE, 0, mmVCE_VCPU_CACHE_OFFSET0), 0);260 } else {261 MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCE, 0,262 mmVCE_LMI_VCPU_CACHE_40BIT_BAR0),263 adev->vce.gpu_addr >> 8);264 MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCE, 0,265 mmVCE_LMI_VCPU_CACHE_64BIT_BAR0),266 (adev->vce.gpu_addr >> 40) & 0xff);267 MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCE, 0, mmVCE_VCPU_CACHE_OFFSET0),268 offset & ~0x0f000000);269 270 }271 MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCE, 0,272 mmVCE_LMI_VCPU_CACHE_40BIT_BAR1),273 adev->vce.gpu_addr >> 8);274 MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCE, 0,275 mmVCE_LMI_VCPU_CACHE_64BIT_BAR1),276 (adev->vce.gpu_addr >> 40) & 0xff);277 MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCE, 0,278 mmVCE_LMI_VCPU_CACHE_40BIT_BAR2),279 adev->vce.gpu_addr >> 8);280 MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCE, 0,281 mmVCE_LMI_VCPU_CACHE_64BIT_BAR2),282 (adev->vce.gpu_addr >> 40) & 0xff);283 284 size = VCE_V4_0_FW_SIZE;285 MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCE, 0, mmVCE_VCPU_CACHE_SIZE0), size);286 287 offset = (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP) ? offset + size : 0;288 size = VCE_V4_0_STACK_SIZE;289 MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCE, 0, mmVCE_VCPU_CACHE_OFFSET1),290 (offset & ~0x0f000000) | (1 << 24));291 MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCE, 0, mmVCE_VCPU_CACHE_SIZE1), size);292 293 offset += size;294 size = VCE_V4_0_DATA_SIZE;295 MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCE, 0, mmVCE_VCPU_CACHE_OFFSET2),296 (offset & ~0x0f000000) | (2 << 24));297 MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCE, 0, mmVCE_VCPU_CACHE_SIZE2), size);298 299 MMSCH_V1_0_INSERT_DIRECT_RD_MOD_WT(SOC15_REG_OFFSET(VCE, 0, mmVCE_LMI_CTRL2), ~0x100, 0);300 MMSCH_V1_0_INSERT_DIRECT_RD_MOD_WT(SOC15_REG_OFFSET(VCE, 0, mmVCE_SYS_INT_EN),301 VCE_SYS_INT_EN__VCE_SYS_INT_TRAP_INTERRUPT_EN_MASK,302 VCE_SYS_INT_EN__VCE_SYS_INT_TRAP_INTERRUPT_EN_MASK);303 304 /* end of MC_RESUME */305 MMSCH_V1_0_INSERT_DIRECT_RD_MOD_WT(SOC15_REG_OFFSET(VCE, 0, mmVCE_STATUS),306 VCE_STATUS__JOB_BUSY_MASK, ~VCE_STATUS__JOB_BUSY_MASK);307 MMSCH_V1_0_INSERT_DIRECT_RD_MOD_WT(SOC15_REG_OFFSET(VCE, 0, mmVCE_VCPU_CNTL),308 ~0x200001, VCE_VCPU_CNTL__CLK_EN_MASK);309 MMSCH_V1_0_INSERT_DIRECT_RD_MOD_WT(SOC15_REG_OFFSET(VCE, 0, mmVCE_SOFT_RESET),310 ~VCE_SOFT_RESET__ECPU_SOFT_RESET_MASK, 0);311 312 MMSCH_V1_0_INSERT_DIRECT_POLL(SOC15_REG_OFFSET(VCE, 0, mmVCE_STATUS),313 VCE_STATUS_VCPU_REPORT_FW_LOADED_MASK,314 VCE_STATUS_VCPU_REPORT_FW_LOADED_MASK);315 316 /* clear BUSY flag */317 MMSCH_V1_0_INSERT_DIRECT_RD_MOD_WT(SOC15_REG_OFFSET(VCE, 0, mmVCE_STATUS),318 ~VCE_STATUS__JOB_BUSY_MASK, 0);319 320 /* add end packet */321 memcpy((void *)init_table, &end, sizeof(struct mmsch_v1_0_cmd_end));322 table_size += sizeof(struct mmsch_v1_0_cmd_end) / 4;323 header->vce_table_size = table_size;324 }325 326 return vce_v4_0_mmsch_start(adev, &adev->virt.mm_table);327}328 329/**330 * vce_v4_0_start - start VCE block331 *332 * @adev: amdgpu_device pointer333 *334 * Setup and start the VCE block335 */336static int vce_v4_0_start(struct amdgpu_device *adev)337{338 struct amdgpu_ring *ring;339 int r;340 341 ring = &adev->vce.ring[0];342 343 WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_RB_RPTR), lower_32_bits(ring->wptr));344 WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_RB_WPTR), lower_32_bits(ring->wptr));345 WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_RB_BASE_LO), ring->gpu_addr);346 WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_RB_BASE_HI), upper_32_bits(ring->gpu_addr));347 WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_RB_SIZE), ring->ring_size / 4);348 349 ring = &adev->vce.ring[1];350 351 WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_RB_RPTR2), lower_32_bits(ring->wptr));352 WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_RB_WPTR2), lower_32_bits(ring->wptr));353 WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_RB_BASE_LO2), ring->gpu_addr);354 WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_RB_BASE_HI2), upper_32_bits(ring->gpu_addr));355 WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_RB_SIZE2), ring->ring_size / 4);356 357 ring = &adev->vce.ring[2];358 359 WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_RB_RPTR3), lower_32_bits(ring->wptr));360 WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_RB_WPTR3), lower_32_bits(ring->wptr));361 WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_RB_BASE_LO3), ring->gpu_addr);362 WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_RB_BASE_HI3), upper_32_bits(ring->gpu_addr));363 WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_RB_SIZE3), ring->ring_size / 4);364 365 vce_v4_0_mc_resume(adev);366 WREG32_P(SOC15_REG_OFFSET(VCE, 0, mmVCE_STATUS), VCE_STATUS__JOB_BUSY_MASK,367 ~VCE_STATUS__JOB_BUSY_MASK);368 369 WREG32_P(SOC15_REG_OFFSET(VCE, 0, mmVCE_VCPU_CNTL), 1, ~0x200001);370 371 WREG32_P(SOC15_REG_OFFSET(VCE, 0, mmVCE_SOFT_RESET), 0,372 ~VCE_SOFT_RESET__ECPU_SOFT_RESET_MASK);373 mdelay(100);374 375 r = vce_v4_0_firmware_loaded(adev);376 377 /* clear BUSY flag */378 WREG32_P(SOC15_REG_OFFSET(VCE, 0, mmVCE_STATUS), 0, ~VCE_STATUS__JOB_BUSY_MASK);379 380 if (r) {381 DRM_ERROR("VCE not responding, giving up!!!\n");382 return r;383 }384 385 return 0;386}387 388static int vce_v4_0_stop(struct amdgpu_device *adev)389{390 391 /* Disable VCPU */392 WREG32_P(SOC15_REG_OFFSET(VCE, 0, mmVCE_VCPU_CNTL), 0, ~0x200001);393 394 /* hold on ECPU */395 WREG32_P(SOC15_REG_OFFSET(VCE, 0, mmVCE_SOFT_RESET),396 VCE_SOFT_RESET__ECPU_SOFT_RESET_MASK,397 ~VCE_SOFT_RESET__ECPU_SOFT_RESET_MASK);398 399 /* clear VCE_STATUS */400 WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_STATUS), 0);401 402 /* Set Clock-Gating off */403 /* if (adev->cg_flags & AMD_CG_SUPPORT_VCE_MGCG)404 vce_v4_0_set_vce_sw_clock_gating(adev, false);405 */406 407 return 0;408}409 410static int vce_v4_0_early_init(void *handle)411{412 struct amdgpu_device *adev = (struct amdgpu_device *)handle;413 414 if (amdgpu_sriov_vf(adev)) /* currently only VCN0 support SRIOV */415 adev->vce.num_rings = 1;416 else417 adev->vce.num_rings = 3;418 419 vce_v4_0_set_ring_funcs(adev);420 vce_v4_0_set_irq_funcs(adev);421 422 return 0;423}424 425static int vce_v4_0_sw_init(void *handle)426{427 struct amdgpu_device *adev = (struct amdgpu_device *)handle;428 struct amdgpu_ring *ring;429 430 unsigned size;431 int r, i;432 433 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_VCE0, 167, &adev->vce.irq);434 if (r)435 return r;436 437 size = VCE_V4_0_STACK_SIZE + VCE_V4_0_DATA_SIZE;438 if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP)439 size += VCE_V4_0_FW_SIZE;440 441 r = amdgpu_vce_sw_init(adev, size);442 if (r)443 return r;444 445 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {446 const struct common_firmware_header *hdr;447 unsigned size = amdgpu_bo_size(adev->vce.vcpu_bo);448 449 adev->vce.saved_bo = kvmalloc(size, GFP_KERNEL);450 if (!adev->vce.saved_bo)451 return -ENOMEM;452 453 hdr = (const struct common_firmware_header *)adev->vce.fw->data;454 adev->firmware.ucode[AMDGPU_UCODE_ID_VCE].ucode_id = AMDGPU_UCODE_ID_VCE;455 adev->firmware.ucode[AMDGPU_UCODE_ID_VCE].fw = adev->vce.fw;456 adev->firmware.fw_size +=457 ALIGN(le32_to_cpu(hdr->ucode_size_bytes), PAGE_SIZE);458 DRM_INFO("PSP loading VCE firmware\n");459 } else {460 r = amdgpu_vce_resume(adev);461 if (r)462 return r;463 }464 465 for (i = 0; i < adev->vce.num_rings; i++) {466 enum amdgpu_ring_priority_level hw_prio = amdgpu_vce_get_ring_prio(i);467 468 ring = &adev->vce.ring[i];469 ring->vm_hub = AMDGPU_MMHUB0(0);470 sprintf(ring->name, "vce%d", i);471 if (amdgpu_sriov_vf(adev)) {472 /* DOORBELL only works under SRIOV */473 ring->use_doorbell = true;474 475 /* currently only use the first encoding ring for sriov,476 * so set unused location for other unused rings.477 */478 if (i == 0)479 ring->doorbell_index = adev->doorbell_index.uvd_vce.vce_ring0_1 * 2;480 else481 ring->doorbell_index = adev->doorbell_index.uvd_vce.vce_ring2_3 * 2 + 1;482 }483 r = amdgpu_ring_init(adev, ring, 512, &adev->vce.irq, 0,484 hw_prio, NULL);485 if (r)486 return r;487 }488 489 r = amdgpu_virt_alloc_mm_table(adev);490 if (r)491 return r;492 493 return r;494}495 496static int vce_v4_0_sw_fini(void *handle)497{498 int r;499 struct amdgpu_device *adev = (struct amdgpu_device *)handle;500 501 /* free MM table */502 amdgpu_virt_free_mm_table(adev);503 504 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {505 kvfree(adev->vce.saved_bo);506 adev->vce.saved_bo = NULL;507 }508 509 r = amdgpu_vce_suspend(adev);510 if (r)511 return r;512 513 return amdgpu_vce_sw_fini(adev);514}515 516static int vce_v4_0_hw_init(void *handle)517{518 int r, i;519 struct amdgpu_device *adev = (struct amdgpu_device *)handle;520 521 if (amdgpu_sriov_vf(adev))522 r = vce_v4_0_sriov_start(adev);523 else524 r = vce_v4_0_start(adev);525 if (r)526 return r;527 528 for (i = 0; i < adev->vce.num_rings; i++) {529 r = amdgpu_ring_test_helper(&adev->vce.ring[i]);530 if (r)531 return r;532 }533 534 DRM_INFO("VCE initialized successfully.\n");535 536 return 0;537}538 539static int vce_v4_0_hw_fini(void *handle)540{541 struct amdgpu_device *adev = (struct amdgpu_device *)handle;542 543 cancel_delayed_work_sync(&adev->vce.idle_work);544 545 if (!amdgpu_sriov_vf(adev)) {546 /* vce_v4_0_wait_for_idle(handle); */547 vce_v4_0_stop(adev);548 } else {549 /* full access mode, so don't touch any VCE register */550 DRM_DEBUG("For SRIOV client, shouldn't do anything.\n");551 }552 553 return 0;554}555 556static int vce_v4_0_suspend(void *handle)557{558 struct amdgpu_device *adev = (struct amdgpu_device *)handle;559 int r, idx;560 561 if (adev->vce.vcpu_bo == NULL)562 return 0;563 564 if (drm_dev_enter(adev_to_drm(adev), &idx)) {565 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {566 unsigned size = amdgpu_bo_size(adev->vce.vcpu_bo);567 void *ptr = adev->vce.cpu_addr;568 569 memcpy_fromio(adev->vce.saved_bo, ptr, size);570 }571 drm_dev_exit(idx);572 }573 574 /*575 * Proper cleanups before halting the HW engine:576 * - cancel the delayed idle work577 * - enable powergating578 * - enable clockgating579 * - disable dpm580 *581 * TODO: to align with the VCN implementation, move the582 * jobs for clockgating/powergating/dpm setting to583 * ->set_powergating_state().584 */585 cancel_delayed_work_sync(&adev->vce.idle_work);586 587 if (adev->pm.dpm_enabled) {588 amdgpu_dpm_enable_vce(adev, false);589 } else {590 amdgpu_asic_set_vce_clocks(adev, 0, 0);591 amdgpu_device_ip_set_powergating_state(adev, AMD_IP_BLOCK_TYPE_VCE,592 AMD_PG_STATE_GATE);593 amdgpu_device_ip_set_clockgating_state(adev, AMD_IP_BLOCK_TYPE_VCE,594 AMD_CG_STATE_GATE);595 }596 597 r = vce_v4_0_hw_fini(adev);598 if (r)599 return r;600 601 return amdgpu_vce_suspend(adev);602}603 604static int vce_v4_0_resume(void *handle)605{606 struct amdgpu_device *adev = (struct amdgpu_device *)handle;607 int r, idx;608 609 if (adev->vce.vcpu_bo == NULL)610 return -EINVAL;611 612 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {613 614 if (drm_dev_enter(adev_to_drm(adev), &idx)) {615 unsigned size = amdgpu_bo_size(adev->vce.vcpu_bo);616 void *ptr = adev->vce.cpu_addr;617 618 memcpy_toio(ptr, adev->vce.saved_bo, size);619 drm_dev_exit(idx);620 }621 } else {622 r = amdgpu_vce_resume(adev);623 if (r)624 return r;625 }626 627 return vce_v4_0_hw_init(adev);628}629 630static void vce_v4_0_mc_resume(struct amdgpu_device *adev)631{632 uint32_t offset, size;633 uint64_t tmr_mc_addr;634 635 WREG32_P(SOC15_REG_OFFSET(VCE, 0, mmVCE_CLOCK_GATING_A), 0, ~(1 << 16));636 WREG32_P(SOC15_REG_OFFSET(VCE, 0, mmVCE_UENC_CLOCK_GATING), 0x1FF000, ~0xFF9FF000);637 WREG32_P(SOC15_REG_OFFSET(VCE, 0, mmVCE_UENC_REG_CLOCK_GATING), 0x3F, ~0x3F);638 WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_CLOCK_GATING_B), 0x1FF);639 640 WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_LMI_CTRL), 0x00398000);641 WREG32_P(SOC15_REG_OFFSET(VCE, 0, mmVCE_LMI_CACHE_CTRL), 0x0, ~0x1);642 WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_LMI_SWAP_CNTL), 0);643 WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_LMI_SWAP_CNTL1), 0);644 WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_LMI_VM_CTRL), 0);645 646 offset = AMDGPU_VCE_FIRMWARE_OFFSET;647 648 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {649 tmr_mc_addr = (uint64_t)(adev->firmware.ucode[AMDGPU_UCODE_ID_VCE].tmr_mc_addr_hi) << 32 |650 adev->firmware.ucode[AMDGPU_UCODE_ID_VCE].tmr_mc_addr_lo;651 WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_LMI_VCPU_CACHE_40BIT_BAR0),652 (tmr_mc_addr >> 8));653 WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_LMI_VCPU_CACHE_64BIT_BAR0),654 (tmr_mc_addr >> 40) & 0xff);655 WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_VCPU_CACHE_OFFSET0), 0);656 } else {657 WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_LMI_VCPU_CACHE_40BIT_BAR0),658 (adev->vce.gpu_addr >> 8));659 WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_LMI_VCPU_CACHE_64BIT_BAR0),660 (adev->vce.gpu_addr >> 40) & 0xff);661 WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_VCPU_CACHE_OFFSET0), offset & ~0x0f000000);662 }663 664 size = VCE_V4_0_FW_SIZE;665 WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_VCPU_CACHE_SIZE0), size);666 667 WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_LMI_VCPU_CACHE_40BIT_BAR1), (adev->vce.gpu_addr >> 8));668 WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_LMI_VCPU_CACHE_64BIT_BAR1), (adev->vce.gpu_addr >> 40) & 0xff);669 offset = (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP) ? offset + size : 0;670 size = VCE_V4_0_STACK_SIZE;671 WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_VCPU_CACHE_OFFSET1), (offset & ~0x0f000000) | (1 << 24));672 WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_VCPU_CACHE_SIZE1), size);673 674 WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_LMI_VCPU_CACHE_40BIT_BAR2), (adev->vce.gpu_addr >> 8));675 WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_LMI_VCPU_CACHE_64BIT_BAR2), (adev->vce.gpu_addr >> 40) & 0xff);676 offset += size;677 size = VCE_V4_0_DATA_SIZE;678 WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_VCPU_CACHE_OFFSET2), (offset & ~0x0f000000) | (2 << 24));679 WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_VCPU_CACHE_SIZE2), size);680 681 WREG32_P(SOC15_REG_OFFSET(VCE, 0, mmVCE_LMI_CTRL2), 0x0, ~0x100);682 WREG32_P(SOC15_REG_OFFSET(VCE, 0, mmVCE_SYS_INT_EN),683 VCE_SYS_INT_EN__VCE_SYS_INT_TRAP_INTERRUPT_EN_MASK,684 ~VCE_SYS_INT_EN__VCE_SYS_INT_TRAP_INTERRUPT_EN_MASK);685}686 687static int vce_v4_0_set_clockgating_state(void *handle,688 enum amd_clockgating_state state)689{690 /* needed for driver unload*/691 return 0;692}693 694#if 0695static bool vce_v4_0_is_idle(void *handle)696{697 struct amdgpu_device *adev = (struct amdgpu_device *)handle;698 u32 mask = 0;699 700 mask |= (adev->vce.harvest_config & AMDGPU_VCE_HARVEST_VCE0) ? 0 : SRBM_STATUS2__VCE0_BUSY_MASK;701 mask |= (adev->vce.harvest_config & AMDGPU_VCE_HARVEST_VCE1) ? 0 : SRBM_STATUS2__VCE1_BUSY_MASK;702 703 return !(RREG32(mmSRBM_STATUS2) & mask);704}705 706static int vce_v4_0_wait_for_idle(void *handle)707{708 unsigned i;709 struct amdgpu_device *adev = (struct amdgpu_device *)handle;710 711 for (i = 0; i < adev->usec_timeout; i++)712 if (vce_v4_0_is_idle(handle))713 return 0;714 715 return -ETIMEDOUT;716}717 718#define VCE_STATUS_VCPU_REPORT_AUTO_BUSY_MASK 0x00000008L /* AUTO_BUSY */719#define VCE_STATUS_VCPU_REPORT_RB0_BUSY_MASK 0x00000010L /* RB0_BUSY */720#define VCE_STATUS_VCPU_REPORT_RB1_BUSY_MASK 0x00000020L /* RB1_BUSY */721#define AMDGPU_VCE_STATUS_BUSY_MASK (VCE_STATUS_VCPU_REPORT_AUTO_BUSY_MASK | \722 VCE_STATUS_VCPU_REPORT_RB0_BUSY_MASK)723 724static bool vce_v4_0_check_soft_reset(void *handle)725{726 struct amdgpu_device *adev = (struct amdgpu_device *)handle;727 u32 srbm_soft_reset = 0;728 729 /* According to VCE team , we should use VCE_STATUS instead730 * SRBM_STATUS.VCE_BUSY bit for busy status checking.731 * GRBM_GFX_INDEX.INSTANCE_INDEX is used to specify which VCE732 * instance's registers are accessed733 * (0 for 1st instance, 10 for 2nd instance).734 *735 *VCE_STATUS736 *|UENC|ACPI|AUTO ACTIVE|RB1 |RB0 |RB2 | |FW_LOADED|JOB |737 *|----+----+-----------+----+----+----+----------+---------+----|738 *|bit8|bit7| bit6 |bit5|bit4|bit3| bit2 | bit1 |bit0|739 *740 * VCE team suggest use bit 3--bit 6 for busy status check741 */742 mutex_lock(&adev->grbm_idx_mutex);743 WREG32_FIELD(GRBM_GFX_INDEX, INSTANCE_INDEX, 0);744 if (RREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_STATUS) & AMDGPU_VCE_STATUS_BUSY_MASK) {745 srbm_soft_reset = REG_SET_FIELD(srbm_soft_reset, SRBM_SOFT_RESET, SOFT_RESET_VCE0, 1);746 srbm_soft_reset = REG_SET_FIELD(srbm_soft_reset, SRBM_SOFT_RESET, SOFT_RESET_VCE1, 1);747 }748 WREG32_FIELD(GRBM_GFX_INDEX, INSTANCE_INDEX, 0x10);749 if (RREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_STATUS) & AMDGPU_VCE_STATUS_BUSY_MASK) {750 srbm_soft_reset = REG_SET_FIELD(srbm_soft_reset, SRBM_SOFT_RESET, SOFT_RESET_VCE0, 1);751 srbm_soft_reset = REG_SET_FIELD(srbm_soft_reset, SRBM_SOFT_RESET, SOFT_RESET_VCE1, 1);752 }753 WREG32_FIELD(GRBM_GFX_INDEX, INSTANCE_INDEX, 0);754 mutex_unlock(&adev->grbm_idx_mutex);755 756 if (srbm_soft_reset) {757 adev->vce.srbm_soft_reset = srbm_soft_reset;758 return true;759 } else {760 adev->vce.srbm_soft_reset = 0;761 return false;762 }763}764 765static int vce_v4_0_soft_reset(void *handle)766{767 struct amdgpu_device *adev = (struct amdgpu_device *)handle;768 u32 srbm_soft_reset;769 770 if (!adev->vce.srbm_soft_reset)771 return 0;772 srbm_soft_reset = adev->vce.srbm_soft_reset;773 774 if (srbm_soft_reset) {775 u32 tmp;776 777 tmp = RREG32(mmSRBM_SOFT_RESET);778 tmp |= srbm_soft_reset;779 dev_info(adev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);780 WREG32(mmSRBM_SOFT_RESET, tmp);781 tmp = RREG32(mmSRBM_SOFT_RESET);782 783 udelay(50);784 785 tmp &= ~srbm_soft_reset;786 WREG32(mmSRBM_SOFT_RESET, tmp);787 tmp = RREG32(mmSRBM_SOFT_RESET);788 789 /* Wait a little for things to settle down */790 udelay(50);791 }792 793 return 0;794}795 796static int vce_v4_0_pre_soft_reset(void *handle)797{798 struct amdgpu_device *adev = (struct amdgpu_device *)handle;799 800 if (!adev->vce.srbm_soft_reset)801 return 0;802 803 mdelay(5);804 805 return vce_v4_0_suspend(adev);806}807 808 809static int vce_v4_0_post_soft_reset(void *handle)810{811 struct amdgpu_device *adev = (struct amdgpu_device *)handle;812 813 if (!adev->vce.srbm_soft_reset)814 return 0;815 816 mdelay(5);817 818 return vce_v4_0_resume(adev);819}820 821static void vce_v4_0_override_vce_clock_gating(struct amdgpu_device *adev, bool override)822{823 u32 tmp, data;824 825 tmp = data = RREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_RB_ARB_CTRL));826 if (override)827 data |= VCE_RB_ARB_CTRL__VCE_CGTT_OVERRIDE_MASK;828 else829 data &= ~VCE_RB_ARB_CTRL__VCE_CGTT_OVERRIDE_MASK;830 831 if (tmp != data)832 WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_RB_ARB_CTRL), data);833}834 835static void vce_v4_0_set_vce_sw_clock_gating(struct amdgpu_device *adev,836 bool gated)837{838 u32 data;839 840 /* Set Override to disable Clock Gating */841 vce_v4_0_override_vce_clock_gating(adev, true);842 843 /* This function enables MGCG which is controlled by firmware.844 With the clocks in the gated state the core is still845 accessible but the firmware will throttle the clocks on the846 fly as necessary.847 */848 if (gated) {849 data = RREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_CLOCK_GATING_B));850 data |= 0x1ff;851 data &= ~0xef0000;852 WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_CLOCK_GATING_B), data);853 854 data = RREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_UENC_CLOCK_GATING));855 data |= 0x3ff000;856 data &= ~0xffc00000;857 WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_UENC_CLOCK_GATING), data);858 859 data = RREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_UENC_CLOCK_GATING_2));860 data |= 0x2;861 data &= ~0x00010000;862 WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_UENC_CLOCK_GATING_2), data);863 864 data = RREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_UENC_REG_CLOCK_GATING));865 data |= 0x37f;866 WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_UENC_REG_CLOCK_GATING), data);867 868 data = RREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_UENC_DMA_DCLK_CTRL));869 data |= VCE_UENC_DMA_DCLK_CTRL__WRDMCLK_FORCEON_MASK |870 VCE_UENC_DMA_DCLK_CTRL__RDDMCLK_FORCEON_MASK |871 VCE_UENC_DMA_DCLK_CTRL__REGCLK_FORCEON_MASK |872 0x8;873 WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_UENC_DMA_DCLK_CTRL), data);874 } else {875 data = RREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_CLOCK_GATING_B));876 data &= ~0x80010;877 data |= 0xe70008;878 WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_CLOCK_GATING_B), data);879 880 data = RREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_UENC_CLOCK_GATING));881 data |= 0xffc00000;882 WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_UENC_CLOCK_GATING), data);883 884 data = RREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_UENC_CLOCK_GATING_2));885 data |= 0x10000;886 WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_UENC_CLOCK_GATING_2), data);887 888 data = RREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_UENC_REG_CLOCK_GATING));889 data &= ~0xffc00000;890 WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_UENC_REG_CLOCK_GATING), data);891 892 data = RREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_UENC_DMA_DCLK_CTRL));893 data &= ~(VCE_UENC_DMA_DCLK_CTRL__WRDMCLK_FORCEON_MASK |894 VCE_UENC_DMA_DCLK_CTRL__RDDMCLK_FORCEON_MASK |895 VCE_UENC_DMA_DCLK_CTRL__REGCLK_FORCEON_MASK |896 0x8);897 WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_UENC_DMA_DCLK_CTRL), data);898 }899 vce_v4_0_override_vce_clock_gating(adev, false);900}901 902static void vce_v4_0_set_bypass_mode(struct amdgpu_device *adev, bool enable)903{904 u32 tmp = RREG32_SMC(ixGCK_DFS_BYPASS_CNTL);905 906 if (enable)907 tmp |= GCK_DFS_BYPASS_CNTL__BYPASSECLK_MASK;908 else909 tmp &= ~GCK_DFS_BYPASS_CNTL__BYPASSECLK_MASK;910 911 WREG32_SMC(ixGCK_DFS_BYPASS_CNTL, tmp);912}913 914static int vce_v4_0_set_clockgating_state(void *handle,915 enum amd_clockgating_state state)916{917 struct amdgpu_device *adev = (struct amdgpu_device *)handle;918 bool enable = (state == AMD_CG_STATE_GATE);919 int i;920 921 if ((adev->asic_type == CHIP_POLARIS10) ||922 (adev->asic_type == CHIP_TONGA) ||923 (adev->asic_type == CHIP_FIJI))924 vce_v4_0_set_bypass_mode(adev, enable);925 926 if (!(adev->cg_flags & AMD_CG_SUPPORT_VCE_MGCG))927 return 0;928 929 mutex_lock(&adev->grbm_idx_mutex);930 for (i = 0; i < 2; i++) {931 /* Program VCE Instance 0 or 1 if not harvested */932 if (adev->vce.harvest_config & (1 << i))933 continue;934 935 WREG32_FIELD(GRBM_GFX_INDEX, VCE_INSTANCE, i);936 937 if (enable) {938 /* initialize VCE_CLOCK_GATING_A: Clock ON/OFF delay */939 uint32_t data = RREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_CLOCK_GATING_A);940 data &= ~(0xf | 0xff0);941 data |= ((0x0 << 0) | (0x04 << 4));942 WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_CLOCK_GATING_A, data);943 944 /* initialize VCE_UENC_CLOCK_GATING: Clock ON/OFF delay */945 data = RREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_UENC_CLOCK_GATING);946 data &= ~(0xf | 0xff0);947 data |= ((0x0 << 0) | (0x04 << 4));948 WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_UENC_CLOCK_GATING, data);949 }950 951 vce_v4_0_set_vce_sw_clock_gating(adev, enable);952 }953 954 WREG32_FIELD(GRBM_GFX_INDEX, VCE_INSTANCE, 0);955 mutex_unlock(&adev->grbm_idx_mutex);956 957 return 0;958}959#endif960 961static int vce_v4_0_set_powergating_state(void *handle,962 enum amd_powergating_state state)963{964 /* This doesn't actually powergate the VCE block.965 * That's done in the dpm code via the SMC. This966 * just re-inits the block as necessary. The actual967 * gating still happens in the dpm code. We should968 * revisit this when there is a cleaner line between969 * the smc and the hw blocks970 */971 struct amdgpu_device *adev = (struct amdgpu_device *)handle;972 973 if (state == AMD_PG_STATE_GATE)974 return vce_v4_0_stop(adev);975 else976 return vce_v4_0_start(adev);977}978 979static void vce_v4_0_ring_emit_ib(struct amdgpu_ring *ring, struct amdgpu_job *job,980 struct amdgpu_ib *ib, uint32_t flags)981{982 unsigned vmid = AMDGPU_JOB_GET_VMID(job);983 984 amdgpu_ring_write(ring, VCE_CMD_IB_VM);985 amdgpu_ring_write(ring, vmid);986 amdgpu_ring_write(ring, lower_32_bits(ib->gpu_addr));987 amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr));988 amdgpu_ring_write(ring, ib->length_dw);989}990 991static void vce_v4_0_ring_emit_fence(struct amdgpu_ring *ring, u64 addr,992 u64 seq, unsigned flags)993{994 WARN_ON(flags & AMDGPU_FENCE_FLAG_64BIT);995 996 amdgpu_ring_write(ring, VCE_CMD_FENCE);997 amdgpu_ring_write(ring, addr);998 amdgpu_ring_write(ring, upper_32_bits(addr));999 amdgpu_ring_write(ring, seq);1000 amdgpu_ring_write(ring, VCE_CMD_TRAP);1001}1002 1003static void vce_v4_0_ring_insert_end(struct amdgpu_ring *ring)1004{1005 amdgpu_ring_write(ring, VCE_CMD_END);1006}1007 1008static void vce_v4_0_emit_reg_wait(struct amdgpu_ring *ring, uint32_t reg,1009 uint32_t val, uint32_t mask)1010{1011 amdgpu_ring_write(ring, VCE_CMD_REG_WAIT);1012 amdgpu_ring_write(ring, reg << 2);1013 amdgpu_ring_write(ring, mask);1014 amdgpu_ring_write(ring, val);1015}1016 1017static void vce_v4_0_emit_vm_flush(struct amdgpu_ring *ring,1018 unsigned int vmid, uint64_t pd_addr)1019{1020 struct amdgpu_vmhub *hub = &ring->adev->vmhub[ring->vm_hub];1021 1022 pd_addr = amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr);1023 1024 /* wait for reg writes */1025 vce_v4_0_emit_reg_wait(ring, hub->ctx0_ptb_addr_lo32 +1026 vmid * hub->ctx_addr_distance,1027 lower_32_bits(pd_addr), 0xffffffff);1028}1029 1030static void vce_v4_0_emit_wreg(struct amdgpu_ring *ring,1031 uint32_t reg, uint32_t val)1032{1033 amdgpu_ring_write(ring, VCE_CMD_REG_WRITE);1034 amdgpu_ring_write(ring, reg << 2);1035 amdgpu_ring_write(ring, val);1036}1037 1038static int vce_v4_0_set_interrupt_state(struct amdgpu_device *adev,1039 struct amdgpu_irq_src *source,1040 unsigned type,1041 enum amdgpu_interrupt_state state)1042{1043 uint32_t val = 0;1044 1045 if (!amdgpu_sriov_vf(adev)) {1046 if (state == AMDGPU_IRQ_STATE_ENABLE)1047 val |= VCE_SYS_INT_EN__VCE_SYS_INT_TRAP_INTERRUPT_EN_MASK;1048 1049 WREG32_P(SOC15_REG_OFFSET(VCE, 0, mmVCE_SYS_INT_EN), val,1050 ~VCE_SYS_INT_EN__VCE_SYS_INT_TRAP_INTERRUPT_EN_MASK);1051 }1052 return 0;1053}1054 1055static int vce_v4_0_process_interrupt(struct amdgpu_device *adev,1056 struct amdgpu_irq_src *source,1057 struct amdgpu_iv_entry *entry)1058{1059 DRM_DEBUG("IH: VCE\n");1060 1061 switch (entry->src_data[0]) {1062 case 0:1063 case 1:1064 case 2:1065 amdgpu_fence_process(&adev->vce.ring[entry->src_data[0]]);1066 break;1067 default:1068 DRM_ERROR("Unhandled interrupt: %d %d\n",1069 entry->src_id, entry->src_data[0]);1070 break;1071 }1072 1073 return 0;1074}1075 1076const struct amd_ip_funcs vce_v4_0_ip_funcs = {1077 .name = "vce_v4_0",1078 .early_init = vce_v4_0_early_init,1079 .late_init = NULL,1080 .sw_init = vce_v4_0_sw_init,1081 .sw_fini = vce_v4_0_sw_fini,1082 .hw_init = vce_v4_0_hw_init,1083 .hw_fini = vce_v4_0_hw_fini,1084 .suspend = vce_v4_0_suspend,1085 .resume = vce_v4_0_resume,1086 .is_idle = NULL /* vce_v4_0_is_idle */,1087 .wait_for_idle = NULL /* vce_v4_0_wait_for_idle */,1088 .check_soft_reset = NULL /* vce_v4_0_check_soft_reset */,1089 .pre_soft_reset = NULL /* vce_v4_0_pre_soft_reset */,1090 .soft_reset = NULL /* vce_v4_0_soft_reset */,1091 .post_soft_reset = NULL /* vce_v4_0_post_soft_reset */,1092 .set_clockgating_state = vce_v4_0_set_clockgating_state,1093 .set_powergating_state = vce_v4_0_set_powergating_state,1094};1095 1096static const struct amdgpu_ring_funcs vce_v4_0_ring_vm_funcs = {1097 .type = AMDGPU_RING_TYPE_VCE,1098 .align_mask = 0x3f,1099 .nop = VCE_CMD_NO_OP,1100 .support_64bit_ptrs = false,1101 .no_user_fence = true,1102 .get_rptr = vce_v4_0_ring_get_rptr,1103 .get_wptr = vce_v4_0_ring_get_wptr,1104 .set_wptr = vce_v4_0_ring_set_wptr,1105 .patch_cs_in_place = amdgpu_vce_ring_parse_cs_vm,1106 .emit_frame_size =1107 SOC15_FLUSH_GPU_TLB_NUM_WREG * 3 +1108 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 4 +1109 4 + /* vce_v4_0_emit_vm_flush */1110 5 + 5 + /* amdgpu_vce_ring_emit_fence x2 vm fence */1111 1, /* vce_v4_0_ring_insert_end */1112 .emit_ib_size = 5, /* vce_v4_0_ring_emit_ib */1113 .emit_ib = vce_v4_0_ring_emit_ib,1114 .emit_vm_flush = vce_v4_0_emit_vm_flush,1115 .emit_fence = vce_v4_0_ring_emit_fence,1116 .test_ring = amdgpu_vce_ring_test_ring,1117 .test_ib = amdgpu_vce_ring_test_ib,1118 .insert_nop = amdgpu_ring_insert_nop,1119 .insert_end = vce_v4_0_ring_insert_end,1120 .pad_ib = amdgpu_ring_generic_pad_ib,1121 .begin_use = amdgpu_vce_ring_begin_use,1122 .end_use = amdgpu_vce_ring_end_use,1123 .emit_wreg = vce_v4_0_emit_wreg,1124 .emit_reg_wait = vce_v4_0_emit_reg_wait,1125 .emit_reg_write_reg_wait = amdgpu_ring_emit_reg_write_reg_wait_helper,1126};1127 1128static void vce_v4_0_set_ring_funcs(struct amdgpu_device *adev)1129{1130 int i;1131 1132 for (i = 0; i < adev->vce.num_rings; i++) {1133 adev->vce.ring[i].funcs = &vce_v4_0_ring_vm_funcs;1134 adev->vce.ring[i].me = i;1135 }1136 DRM_INFO("VCE enabled in VM mode\n");1137}1138 1139static const struct amdgpu_irq_src_funcs vce_v4_0_irq_funcs = {1140 .set = vce_v4_0_set_interrupt_state,1141 .process = vce_v4_0_process_interrupt,1142};1143 1144static void vce_v4_0_set_irq_funcs(struct amdgpu_device *adev)1145{1146 adev->vce.irq.num_types = 1;1147 adev->vce.irq.funcs = &vce_v4_0_irq_funcs;1148};1149 1150const struct amdgpu_ip_block_version vce_v4_0_ip_block =1151{1152 .type = AMD_IP_BLOCK_TYPE_VCE,1153 .major = 4,1154 .minor = 0,1155 .rev = 0,1156 .funcs = &vce_v4_0_ip_funcs,1157};1158