1922 lines · c
1/*2 * Copyright 2016 Advanced Micro Devices, Inc.3 *4 * Permission is hereby granted, free of charge, to any person obtaining a5 * copy of this software and associated documentation files (the "Software"),6 * to deal in the Software without restriction, including without limitation7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,8 * and/or sell copies of the Software, and to permit persons to whom the9 * Software is furnished to do so, subject to the following conditions:10 *11 * The above copyright notice and this permission notice shall be included in12 * all copies or substantial portions of the Software.13 *14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR20 * OTHER DEALINGS IN THE SOFTWARE.21 *22 */23 24#include <linux/firmware.h>25 26#include "amdgpu.h"27#include "amdgpu_uvd.h"28#include "amdgpu_cs.h"29#include "soc15.h"30#include "soc15d.h"31#include "soc15_common.h"32#include "mmsch_v1_0.h"33 34#include "uvd/uvd_7_0_offset.h"35#include "uvd/uvd_7_0_sh_mask.h"36#include "vce/vce_4_0_offset.h"37#include "vce/vce_4_0_default.h"38#include "vce/vce_4_0_sh_mask.h"39#include "nbif/nbif_6_1_offset.h"40#include "mmhub/mmhub_1_0_offset.h"41#include "mmhub/mmhub_1_0_sh_mask.h"42#include "ivsrcid/uvd/irqsrcs_uvd_7_0.h"43 44#define mmUVD_PG0_CC_UVD_HARVESTING 0x00c745#define mmUVD_PG0_CC_UVD_HARVESTING_BASE_IDX 146//UVD_PG0_CC_UVD_HARVESTING47#define UVD_PG0_CC_UVD_HARVESTING__UVD_DISABLE__SHIFT 0x148#define UVD_PG0_CC_UVD_HARVESTING__UVD_DISABLE_MASK 0x00000002L49 50#define UVD7_MAX_HW_INSTANCES_VEGA20 251 52static void uvd_v7_0_set_ring_funcs(struct amdgpu_device *adev);53static void uvd_v7_0_set_enc_ring_funcs(struct amdgpu_device *adev);54static void uvd_v7_0_set_irq_funcs(struct amdgpu_device *adev);55static int uvd_v7_0_start(struct amdgpu_device *adev);56static void uvd_v7_0_stop(struct amdgpu_device *adev);57static int uvd_v7_0_sriov_start(struct amdgpu_device *adev);58 59static int amdgpu_ih_clientid_uvds[] = {60 SOC15_IH_CLIENTID_UVD,61 SOC15_IH_CLIENTID_UVD162};63 64/**65 * uvd_v7_0_ring_get_rptr - get read pointer66 *67 * @ring: amdgpu_ring pointer68 *69 * Returns the current hardware read pointer70 */71static uint64_t uvd_v7_0_ring_get_rptr(struct amdgpu_ring *ring)72{73 struct amdgpu_device *adev = ring->adev;74 75 return RREG32_SOC15(UVD, ring->me, mmUVD_RBC_RB_RPTR);76}77 78/**79 * uvd_v7_0_enc_ring_get_rptr - get enc read pointer80 *81 * @ring: amdgpu_ring pointer82 *83 * Returns the current hardware enc read pointer84 */85static uint64_t uvd_v7_0_enc_ring_get_rptr(struct amdgpu_ring *ring)86{87 struct amdgpu_device *adev = ring->adev;88 89 if (ring == &adev->uvd.inst[ring->me].ring_enc[0])90 return RREG32_SOC15(UVD, ring->me, mmUVD_RB_RPTR);91 else92 return RREG32_SOC15(UVD, ring->me, mmUVD_RB_RPTR2);93}94 95/**96 * uvd_v7_0_ring_get_wptr - get write pointer97 *98 * @ring: amdgpu_ring pointer99 *100 * Returns the current hardware write pointer101 */102static uint64_t uvd_v7_0_ring_get_wptr(struct amdgpu_ring *ring)103{104 struct amdgpu_device *adev = ring->adev;105 106 return RREG32_SOC15(UVD, ring->me, mmUVD_RBC_RB_WPTR);107}108 109/**110 * uvd_v7_0_enc_ring_get_wptr - get enc write pointer111 *112 * @ring: amdgpu_ring pointer113 *114 * Returns the current hardware enc write pointer115 */116static uint64_t uvd_v7_0_enc_ring_get_wptr(struct amdgpu_ring *ring)117{118 struct amdgpu_device *adev = ring->adev;119 120 if (ring->use_doorbell)121 return *ring->wptr_cpu_addr;122 123 if (ring == &adev->uvd.inst[ring->me].ring_enc[0])124 return RREG32_SOC15(UVD, ring->me, mmUVD_RB_WPTR);125 else126 return RREG32_SOC15(UVD, ring->me, mmUVD_RB_WPTR2);127}128 129/**130 * uvd_v7_0_ring_set_wptr - set write pointer131 *132 * @ring: amdgpu_ring pointer133 *134 * Commits the write pointer to the hardware135 */136static void uvd_v7_0_ring_set_wptr(struct amdgpu_ring *ring)137{138 struct amdgpu_device *adev = ring->adev;139 140 WREG32_SOC15(UVD, ring->me, mmUVD_RBC_RB_WPTR, lower_32_bits(ring->wptr));141}142 143/**144 * uvd_v7_0_enc_ring_set_wptr - set enc write pointer145 *146 * @ring: amdgpu_ring pointer147 *148 * Commits the enc write pointer to the hardware149 */150static void uvd_v7_0_enc_ring_set_wptr(struct amdgpu_ring *ring)151{152 struct amdgpu_device *adev = ring->adev;153 154 if (ring->use_doorbell) {155 /* XXX check if swapping is necessary on BE */156 *ring->wptr_cpu_addr = lower_32_bits(ring->wptr);157 WDOORBELL32(ring->doorbell_index, lower_32_bits(ring->wptr));158 return;159 }160 161 if (ring == &adev->uvd.inst[ring->me].ring_enc[0])162 WREG32_SOC15(UVD, ring->me, mmUVD_RB_WPTR,163 lower_32_bits(ring->wptr));164 else165 WREG32_SOC15(UVD, ring->me, mmUVD_RB_WPTR2,166 lower_32_bits(ring->wptr));167}168 169/**170 * uvd_v7_0_enc_ring_test_ring - test if UVD ENC ring is working171 *172 * @ring: the engine to test on173 *174 */175static int uvd_v7_0_enc_ring_test_ring(struct amdgpu_ring *ring)176{177 struct amdgpu_device *adev = ring->adev;178 uint32_t rptr;179 unsigned i;180 int r;181 182 if (amdgpu_sriov_vf(adev))183 return 0;184 185 r = amdgpu_ring_alloc(ring, 16);186 if (r)187 return r;188 189 rptr = amdgpu_ring_get_rptr(ring);190 191 amdgpu_ring_write(ring, HEVC_ENC_CMD_END);192 amdgpu_ring_commit(ring);193 194 for (i = 0; i < adev->usec_timeout; i++) {195 if (amdgpu_ring_get_rptr(ring) != rptr)196 break;197 udelay(1);198 }199 200 if (i >= adev->usec_timeout)201 r = -ETIMEDOUT;202 203 return r;204}205 206/**207 * uvd_v7_0_enc_get_create_msg - generate a UVD ENC create msg208 *209 * @ring: ring we should submit the msg to210 * @handle: session handle to use211 * @bo: amdgpu object for which we query the offset212 * @fence: optional fence to return213 *214 * Open up a stream for HW test215 */216static int uvd_v7_0_enc_get_create_msg(struct amdgpu_ring *ring, u32 handle,217 struct amdgpu_bo *bo,218 struct dma_fence **fence)219{220 const unsigned ib_size_dw = 16;221 struct amdgpu_job *job;222 struct amdgpu_ib *ib;223 struct dma_fence *f = NULL;224 uint64_t addr;225 int i, r;226 227 r = amdgpu_job_alloc_with_ib(ring->adev, NULL, NULL, ib_size_dw * 4,228 AMDGPU_IB_POOL_DIRECT, &job);229 if (r)230 return r;231 232 ib = &job->ibs[0];233 addr = amdgpu_bo_gpu_offset(bo);234 235 ib->length_dw = 0;236 ib->ptr[ib->length_dw++] = 0x00000018;237 ib->ptr[ib->length_dw++] = 0x00000001; /* session info */238 ib->ptr[ib->length_dw++] = handle;239 ib->ptr[ib->length_dw++] = 0x00000000;240 ib->ptr[ib->length_dw++] = upper_32_bits(addr);241 ib->ptr[ib->length_dw++] = addr;242 243 ib->ptr[ib->length_dw++] = 0x00000014;244 ib->ptr[ib->length_dw++] = 0x00000002; /* task info */245 ib->ptr[ib->length_dw++] = 0x0000001c;246 ib->ptr[ib->length_dw++] = 0x00000000;247 ib->ptr[ib->length_dw++] = 0x00000000;248 249 ib->ptr[ib->length_dw++] = 0x00000008;250 ib->ptr[ib->length_dw++] = 0x08000001; /* op initialize */251 252 for (i = ib->length_dw; i < ib_size_dw; ++i)253 ib->ptr[i] = 0x0;254 255 r = amdgpu_job_submit_direct(job, ring, &f);256 if (r)257 goto err;258 259 if (fence)260 *fence = dma_fence_get(f);261 dma_fence_put(f);262 return 0;263 264err:265 amdgpu_job_free(job);266 return r;267}268 269/**270 * uvd_v7_0_enc_get_destroy_msg - generate a UVD ENC destroy msg271 *272 * @ring: ring we should submit the msg to273 * @handle: session handle to use274 * @bo: amdgpu object for which we query the offset275 * @fence: optional fence to return276 *277 * Close up a stream for HW test or if userspace failed to do so278 */279static int uvd_v7_0_enc_get_destroy_msg(struct amdgpu_ring *ring, u32 handle,280 struct amdgpu_bo *bo,281 struct dma_fence **fence)282{283 const unsigned ib_size_dw = 16;284 struct amdgpu_job *job;285 struct amdgpu_ib *ib;286 struct dma_fence *f = NULL;287 uint64_t addr;288 int i, r;289 290 r = amdgpu_job_alloc_with_ib(ring->adev, NULL, NULL, ib_size_dw * 4,291 AMDGPU_IB_POOL_DIRECT, &job);292 if (r)293 return r;294 295 ib = &job->ibs[0];296 addr = amdgpu_bo_gpu_offset(bo);297 298 ib->length_dw = 0;299 ib->ptr[ib->length_dw++] = 0x00000018;300 ib->ptr[ib->length_dw++] = 0x00000001;301 ib->ptr[ib->length_dw++] = handle;302 ib->ptr[ib->length_dw++] = 0x00000000;303 ib->ptr[ib->length_dw++] = upper_32_bits(addr);304 ib->ptr[ib->length_dw++] = addr;305 306 ib->ptr[ib->length_dw++] = 0x00000014;307 ib->ptr[ib->length_dw++] = 0x00000002;308 ib->ptr[ib->length_dw++] = 0x0000001c;309 ib->ptr[ib->length_dw++] = 0x00000000;310 ib->ptr[ib->length_dw++] = 0x00000000;311 312 ib->ptr[ib->length_dw++] = 0x00000008;313 ib->ptr[ib->length_dw++] = 0x08000002; /* op close session */314 315 for (i = ib->length_dw; i < ib_size_dw; ++i)316 ib->ptr[i] = 0x0;317 318 r = amdgpu_job_submit_direct(job, ring, &f);319 if (r)320 goto err;321 322 if (fence)323 *fence = dma_fence_get(f);324 dma_fence_put(f);325 return 0;326 327err:328 amdgpu_job_free(job);329 return r;330}331 332/**333 * uvd_v7_0_enc_ring_test_ib - test if UVD ENC IBs are working334 *335 * @ring: the engine to test on336 * @timeout: timeout value in jiffies, or MAX_SCHEDULE_TIMEOUT337 *338 */339static int uvd_v7_0_enc_ring_test_ib(struct amdgpu_ring *ring, long timeout)340{341 struct dma_fence *fence = NULL;342 struct amdgpu_bo *bo = ring->adev->uvd.ib_bo;343 long r;344 345 r = uvd_v7_0_enc_get_create_msg(ring, 1, bo, NULL);346 if (r)347 goto error;348 349 r = uvd_v7_0_enc_get_destroy_msg(ring, 1, bo, &fence);350 if (r)351 goto error;352 353 r = dma_fence_wait_timeout(fence, false, timeout);354 if (r == 0)355 r = -ETIMEDOUT;356 else if (r > 0)357 r = 0;358 359error:360 dma_fence_put(fence);361 return r;362}363 364static int uvd_v7_0_early_init(void *handle)365{366 struct amdgpu_device *adev = (struct amdgpu_device *)handle;367 368 if (adev->asic_type == CHIP_VEGA20) {369 u32 harvest;370 int i;371 372 adev->uvd.num_uvd_inst = UVD7_MAX_HW_INSTANCES_VEGA20;373 for (i = 0; i < adev->uvd.num_uvd_inst; i++) {374 harvest = RREG32_SOC15(UVD, i, mmUVD_PG0_CC_UVD_HARVESTING);375 if (harvest & UVD_PG0_CC_UVD_HARVESTING__UVD_DISABLE_MASK) {376 adev->uvd.harvest_config |= 1 << i;377 }378 }379 if (adev->uvd.harvest_config == (AMDGPU_UVD_HARVEST_UVD0 |380 AMDGPU_UVD_HARVEST_UVD1))381 /* both instances are harvested, disable the block */382 return -ENOENT;383 } else {384 adev->uvd.num_uvd_inst = 1;385 }386 387 if (amdgpu_sriov_vf(adev))388 adev->uvd.num_enc_rings = 1;389 else390 adev->uvd.num_enc_rings = 2;391 uvd_v7_0_set_ring_funcs(adev);392 uvd_v7_0_set_enc_ring_funcs(adev);393 uvd_v7_0_set_irq_funcs(adev);394 395 return 0;396}397 398static int uvd_v7_0_sw_init(void *handle)399{400 struct amdgpu_ring *ring;401 402 int i, j, r;403 struct amdgpu_device *adev = (struct amdgpu_device *)handle;404 405 for (j = 0; j < adev->uvd.num_uvd_inst; j++) {406 if (adev->uvd.harvest_config & (1 << j))407 continue;408 /* UVD TRAP */409 r = amdgpu_irq_add_id(adev, amdgpu_ih_clientid_uvds[j], UVD_7_0__SRCID__UVD_SYSTEM_MESSAGE_INTERRUPT, &adev->uvd.inst[j].irq);410 if (r)411 return r;412 413 /* UVD ENC TRAP */414 for (i = 0; i < adev->uvd.num_enc_rings; ++i) {415 r = amdgpu_irq_add_id(adev, amdgpu_ih_clientid_uvds[j], i + UVD_7_0__SRCID__UVD_ENC_GEN_PURP, &adev->uvd.inst[j].irq);416 if (r)417 return r;418 }419 }420 421 r = amdgpu_uvd_sw_init(adev);422 if (r)423 return r;424 425 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {426 const struct common_firmware_header *hdr;427 hdr = (const struct common_firmware_header *)adev->uvd.fw->data;428 adev->firmware.ucode[AMDGPU_UCODE_ID_UVD].ucode_id = AMDGPU_UCODE_ID_UVD;429 adev->firmware.ucode[AMDGPU_UCODE_ID_UVD].fw = adev->uvd.fw;430 adev->firmware.fw_size +=431 ALIGN(le32_to_cpu(hdr->ucode_size_bytes), PAGE_SIZE);432 433 if (adev->uvd.num_uvd_inst == UVD7_MAX_HW_INSTANCES_VEGA20) {434 adev->firmware.ucode[AMDGPU_UCODE_ID_UVD1].ucode_id = AMDGPU_UCODE_ID_UVD1;435 adev->firmware.ucode[AMDGPU_UCODE_ID_UVD1].fw = adev->uvd.fw;436 adev->firmware.fw_size +=437 ALIGN(le32_to_cpu(hdr->ucode_size_bytes), PAGE_SIZE);438 }439 DRM_INFO("PSP loading UVD firmware\n");440 }441 442 for (j = 0; j < adev->uvd.num_uvd_inst; j++) {443 if (adev->uvd.harvest_config & (1 << j))444 continue;445 if (!amdgpu_sriov_vf(adev)) {446 ring = &adev->uvd.inst[j].ring;447 ring->vm_hub = AMDGPU_MMHUB0(0);448 sprintf(ring->name, "uvd_%d", ring->me);449 r = amdgpu_ring_init(adev, ring, 512,450 &adev->uvd.inst[j].irq, 0,451 AMDGPU_RING_PRIO_DEFAULT, NULL);452 if (r)453 return r;454 }455 456 for (i = 0; i < adev->uvd.num_enc_rings; ++i) {457 ring = &adev->uvd.inst[j].ring_enc[i];458 ring->vm_hub = AMDGPU_MMHUB0(0);459 sprintf(ring->name, "uvd_enc_%d.%d", ring->me, i);460 if (amdgpu_sriov_vf(adev)) {461 ring->use_doorbell = true;462 463 /* currently only use the first enconding ring for464 * sriov, so set unused location for other unused rings.465 */466 if (i == 0)467 ring->doorbell_index = adev->doorbell_index.uvd_vce.uvd_ring0_1 * 2;468 else469 ring->doorbell_index = adev->doorbell_index.uvd_vce.uvd_ring2_3 * 2 + 1;470 }471 r = amdgpu_ring_init(adev, ring, 512,472 &adev->uvd.inst[j].irq, 0,473 AMDGPU_RING_PRIO_DEFAULT, NULL);474 if (r)475 return r;476 }477 }478 479 r = amdgpu_uvd_resume(adev);480 if (r)481 return r;482 483 r = amdgpu_virt_alloc_mm_table(adev);484 if (r)485 return r;486 487 return r;488}489 490static int uvd_v7_0_sw_fini(void *handle)491{492 int i, j, r;493 struct amdgpu_device *adev = (struct amdgpu_device *)handle;494 495 amdgpu_virt_free_mm_table(adev);496 497 r = amdgpu_uvd_suspend(adev);498 if (r)499 return r;500 501 for (j = 0; j < adev->uvd.num_uvd_inst; ++j) {502 if (adev->uvd.harvest_config & (1 << j))503 continue;504 for (i = 0; i < adev->uvd.num_enc_rings; ++i)505 amdgpu_ring_fini(&adev->uvd.inst[j].ring_enc[i]);506 }507 return amdgpu_uvd_sw_fini(adev);508}509 510/**511 * uvd_v7_0_hw_init - start and test UVD block512 *513 * @handle: handle used to pass amdgpu_device pointer514 *515 * Initialize the hardware, boot up the VCPU and do some testing516 */517static int uvd_v7_0_hw_init(void *handle)518{519 struct amdgpu_device *adev = (struct amdgpu_device *)handle;520 struct amdgpu_ring *ring;521 uint32_t tmp;522 int i, j, r;523 524 if (amdgpu_sriov_vf(adev))525 r = uvd_v7_0_sriov_start(adev);526 else527 r = uvd_v7_0_start(adev);528 if (r)529 goto done;530 531 for (j = 0; j < adev->uvd.num_uvd_inst; ++j) {532 if (adev->uvd.harvest_config & (1 << j))533 continue;534 ring = &adev->uvd.inst[j].ring;535 536 if (!amdgpu_sriov_vf(adev)) {537 r = amdgpu_ring_test_helper(ring);538 if (r)539 goto done;540 541 r = amdgpu_ring_alloc(ring, 10);542 if (r) {543 DRM_ERROR("amdgpu: (%d)ring failed to lock UVD ring (%d).\n", j, r);544 goto done;545 }546 547 tmp = PACKET0(SOC15_REG_OFFSET(UVD, j,548 mmUVD_SEMA_WAIT_FAULT_TIMEOUT_CNTL), 0);549 amdgpu_ring_write(ring, tmp);550 amdgpu_ring_write(ring, 0xFFFFF);551 552 tmp = PACKET0(SOC15_REG_OFFSET(UVD, j,553 mmUVD_SEMA_WAIT_INCOMPLETE_TIMEOUT_CNTL), 0);554 amdgpu_ring_write(ring, tmp);555 amdgpu_ring_write(ring, 0xFFFFF);556 557 tmp = PACKET0(SOC15_REG_OFFSET(UVD, j,558 mmUVD_SEMA_SIGNAL_INCOMPLETE_TIMEOUT_CNTL), 0);559 amdgpu_ring_write(ring, tmp);560 amdgpu_ring_write(ring, 0xFFFFF);561 562 /* Clear timeout status bits */563 amdgpu_ring_write(ring, PACKET0(SOC15_REG_OFFSET(UVD, j,564 mmUVD_SEMA_TIMEOUT_STATUS), 0));565 amdgpu_ring_write(ring, 0x8);566 567 amdgpu_ring_write(ring, PACKET0(SOC15_REG_OFFSET(UVD, j,568 mmUVD_SEMA_CNTL), 0));569 amdgpu_ring_write(ring, 3);570 571 amdgpu_ring_commit(ring);572 }573 574 for (i = 0; i < adev->uvd.num_enc_rings; ++i) {575 ring = &adev->uvd.inst[j].ring_enc[i];576 r = amdgpu_ring_test_helper(ring);577 if (r)578 goto done;579 }580 }581done:582 if (!r)583 DRM_INFO("UVD and UVD ENC initialized successfully.\n");584 585 return r;586}587 588/**589 * uvd_v7_0_hw_fini - stop the hardware block590 *591 * @handle: handle used to pass amdgpu_device pointer592 *593 * Stop the UVD block, mark ring as not ready any more594 */595static int uvd_v7_0_hw_fini(void *handle)596{597 struct amdgpu_device *adev = (struct amdgpu_device *)handle;598 599 cancel_delayed_work_sync(&adev->uvd.idle_work);600 601 if (!amdgpu_sriov_vf(adev))602 uvd_v7_0_stop(adev);603 else {604 /* full access mode, so don't touch any UVD register */605 DRM_DEBUG("For SRIOV client, shouldn't do anything.\n");606 }607 608 return 0;609}610 611static int uvd_v7_0_prepare_suspend(void *handle)612{613 struct amdgpu_device *adev = (struct amdgpu_device *)handle;614 615 return amdgpu_uvd_prepare_suspend(adev);616}617 618static int uvd_v7_0_suspend(void *handle)619{620 int r;621 struct amdgpu_device *adev = (struct amdgpu_device *)handle;622 623 /*624 * Proper cleanups before halting the HW engine:625 * - cancel the delayed idle work626 * - enable powergating627 * - enable clockgating628 * - disable dpm629 *630 * TODO: to align with the VCN implementation, move the631 * jobs for clockgating/powergating/dpm setting to632 * ->set_powergating_state().633 */634 cancel_delayed_work_sync(&adev->uvd.idle_work);635 636 if (adev->pm.dpm_enabled) {637 amdgpu_dpm_enable_uvd(adev, false);638 } else {639 amdgpu_asic_set_uvd_clocks(adev, 0, 0);640 /* shutdown the UVD block */641 amdgpu_device_ip_set_powergating_state(adev, AMD_IP_BLOCK_TYPE_UVD,642 AMD_PG_STATE_GATE);643 amdgpu_device_ip_set_clockgating_state(adev, AMD_IP_BLOCK_TYPE_UVD,644 AMD_CG_STATE_GATE);645 }646 647 r = uvd_v7_0_hw_fini(adev);648 if (r)649 return r;650 651 return amdgpu_uvd_suspend(adev);652}653 654static int uvd_v7_0_resume(void *handle)655{656 int r;657 struct amdgpu_device *adev = (struct amdgpu_device *)handle;658 659 r = amdgpu_uvd_resume(adev);660 if (r)661 return r;662 663 return uvd_v7_0_hw_init(adev);664}665 666/**667 * uvd_v7_0_mc_resume - memory controller programming668 *669 * @adev: amdgpu_device pointer670 *671 * Let the UVD memory controller know it's offsets672 */673static void uvd_v7_0_mc_resume(struct amdgpu_device *adev)674{675 uint32_t size = AMDGPU_UVD_FIRMWARE_SIZE(adev);676 uint32_t offset;677 int i;678 679 for (i = 0; i < adev->uvd.num_uvd_inst; ++i) {680 if (adev->uvd.harvest_config & (1 << i))681 continue;682 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {683 WREG32_SOC15(UVD, i, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW,684 i == 0 ?685 adev->firmware.ucode[AMDGPU_UCODE_ID_UVD].tmr_mc_addr_lo :686 adev->firmware.ucode[AMDGPU_UCODE_ID_UVD1].tmr_mc_addr_lo);687 WREG32_SOC15(UVD, i, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH,688 i == 0 ?689 adev->firmware.ucode[AMDGPU_UCODE_ID_UVD].tmr_mc_addr_hi :690 adev->firmware.ucode[AMDGPU_UCODE_ID_UVD1].tmr_mc_addr_hi);691 WREG32_SOC15(UVD, i, mmUVD_VCPU_CACHE_OFFSET0, 0);692 offset = 0;693 } else {694 WREG32_SOC15(UVD, i, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW,695 lower_32_bits(adev->uvd.inst[i].gpu_addr));696 WREG32_SOC15(UVD, i, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH,697 upper_32_bits(adev->uvd.inst[i].gpu_addr));698 offset = size;699 WREG32_SOC15(UVD, i, mmUVD_VCPU_CACHE_OFFSET0,700 AMDGPU_UVD_FIRMWARE_OFFSET >> 3);701 }702 703 WREG32_SOC15(UVD, i, mmUVD_VCPU_CACHE_SIZE0, size);704 705 WREG32_SOC15(UVD, i, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW,706 lower_32_bits(adev->uvd.inst[i].gpu_addr + offset));707 WREG32_SOC15(UVD, i, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH,708 upper_32_bits(adev->uvd.inst[i].gpu_addr + offset));709 WREG32_SOC15(UVD, i, mmUVD_VCPU_CACHE_OFFSET1, (1 << 21));710 WREG32_SOC15(UVD, i, mmUVD_VCPU_CACHE_SIZE1, AMDGPU_UVD_HEAP_SIZE);711 712 WREG32_SOC15(UVD, i, mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_LOW,713 lower_32_bits(adev->uvd.inst[i].gpu_addr + offset + AMDGPU_UVD_HEAP_SIZE));714 WREG32_SOC15(UVD, i, mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_HIGH,715 upper_32_bits(adev->uvd.inst[i].gpu_addr + offset + AMDGPU_UVD_HEAP_SIZE));716 WREG32_SOC15(UVD, i, mmUVD_VCPU_CACHE_OFFSET2, (2 << 21));717 WREG32_SOC15(UVD, i, mmUVD_VCPU_CACHE_SIZE2,718 AMDGPU_UVD_STACK_SIZE + (AMDGPU_UVD_SESSION_SIZE * 40));719 720 WREG32_SOC15(UVD, i, mmUVD_UDEC_ADDR_CONFIG,721 adev->gfx.config.gb_addr_config);722 WREG32_SOC15(UVD, i, mmUVD_UDEC_DB_ADDR_CONFIG,723 adev->gfx.config.gb_addr_config);724 WREG32_SOC15(UVD, i, mmUVD_UDEC_DBW_ADDR_CONFIG,725 adev->gfx.config.gb_addr_config);726 727 WREG32_SOC15(UVD, i, mmUVD_GP_SCRATCH4, adev->uvd.max_handles);728 }729}730 731static int uvd_v7_0_mmsch_start(struct amdgpu_device *adev,732 struct amdgpu_mm_table *table)733{734 uint32_t data = 0, loop;735 uint64_t addr = table->gpu_addr;736 struct mmsch_v1_0_init_header *header = (struct mmsch_v1_0_init_header *)table->cpu_addr;737 uint32_t size;738 int i;739 740 size = header->header_size + header->vce_table_size + header->uvd_table_size;741 742 /* 1, write to vce_mmsch_vf_ctx_addr_lo/hi register with GPU mc addr of memory descriptor location */743 WREG32_SOC15(VCE, 0, mmVCE_MMSCH_VF_CTX_ADDR_LO, lower_32_bits(addr));744 WREG32_SOC15(VCE, 0, mmVCE_MMSCH_VF_CTX_ADDR_HI, upper_32_bits(addr));745 746 /* 2, update vmid of descriptor */747 data = RREG32_SOC15(VCE, 0, mmVCE_MMSCH_VF_VMID);748 data &= ~VCE_MMSCH_VF_VMID__VF_CTX_VMID_MASK;749 data |= (0 << VCE_MMSCH_VF_VMID__VF_CTX_VMID__SHIFT); /* use domain0 for MM scheduler */750 WREG32_SOC15(VCE, 0, mmVCE_MMSCH_VF_VMID, data);751 752 /* 3, notify mmsch about the size of this descriptor */753 WREG32_SOC15(VCE, 0, mmVCE_MMSCH_VF_CTX_SIZE, size);754 755 /* 4, set resp to zero */756 WREG32_SOC15(VCE, 0, mmVCE_MMSCH_VF_MAILBOX_RESP, 0);757 758 for (i = 0; i < adev->uvd.num_uvd_inst; ++i) {759 if (adev->uvd.harvest_config & (1 << i))760 continue;761 WDOORBELL32(adev->uvd.inst[i].ring_enc[0].doorbell_index, 0);762 *adev->uvd.inst[i].ring_enc[0].wptr_cpu_addr = 0;763 adev->uvd.inst[i].ring_enc[0].wptr = 0;764 adev->uvd.inst[i].ring_enc[0].wptr_old = 0;765 }766 /* 5, kick off the initialization and wait until VCE_MMSCH_VF_MAILBOX_RESP becomes non-zero */767 WREG32_SOC15(VCE, 0, mmVCE_MMSCH_VF_MAILBOX_HOST, 0x10000001);768 769 data = RREG32_SOC15(VCE, 0, mmVCE_MMSCH_VF_MAILBOX_RESP);770 loop = 1000;771 while ((data & 0x10000002) != 0x10000002) {772 udelay(10);773 data = RREG32_SOC15(VCE, 0, mmVCE_MMSCH_VF_MAILBOX_RESP);774 loop--;775 if (!loop)776 break;777 }778 779 if (!loop) {780 dev_err(adev->dev, "failed to init MMSCH, mmVCE_MMSCH_VF_MAILBOX_RESP = %x\n", data);781 return -EBUSY;782 }783 784 return 0;785}786 787static int uvd_v7_0_sriov_start(struct amdgpu_device *adev)788{789 struct amdgpu_ring *ring;790 uint32_t offset, size, tmp;791 uint32_t table_size = 0;792 struct mmsch_v1_0_cmd_direct_write direct_wt = { {0} };793 struct mmsch_v1_0_cmd_direct_read_modify_write direct_rd_mod_wt = { {0} };794 struct mmsch_v1_0_cmd_direct_polling direct_poll = { {0} };795 struct mmsch_v1_0_cmd_end end = { {0} };796 uint32_t *init_table = adev->virt.mm_table.cpu_addr;797 struct mmsch_v1_0_init_header *header = (struct mmsch_v1_0_init_header *)init_table;798 uint8_t i = 0;799 800 direct_wt.cmd_header.command_type = MMSCH_COMMAND__DIRECT_REG_WRITE;801 direct_rd_mod_wt.cmd_header.command_type = MMSCH_COMMAND__DIRECT_REG_READ_MODIFY_WRITE;802 direct_poll.cmd_header.command_type = MMSCH_COMMAND__DIRECT_REG_POLLING;803 end.cmd_header.command_type = MMSCH_COMMAND__END;804 805 if (header->uvd_table_offset == 0 && header->uvd_table_size == 0) {806 header->version = MMSCH_VERSION;807 header->header_size = sizeof(struct mmsch_v1_0_init_header) >> 2;808 809 if (header->vce_table_offset == 0 && header->vce_table_size == 0)810 header->uvd_table_offset = header->header_size;811 else812 header->uvd_table_offset = header->vce_table_size + header->vce_table_offset;813 814 init_table += header->uvd_table_offset;815 816 for (i = 0; i < adev->uvd.num_uvd_inst; ++i) {817 if (adev->uvd.harvest_config & (1 << i))818 continue;819 ring = &adev->uvd.inst[i].ring;820 ring->wptr = 0;821 size = AMDGPU_GPU_PAGE_ALIGN(adev->uvd.fw->size + 4);822 823 MMSCH_V1_0_INSERT_DIRECT_RD_MOD_WT(SOC15_REG_OFFSET(UVD, i, mmUVD_STATUS),824 0xFFFFFFFF, 0x00000004);825 /* mc resume*/826 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {827 MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, i,828 mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW),829 adev->firmware.ucode[AMDGPU_UCODE_ID_UVD].tmr_mc_addr_lo);830 MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, i,831 mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH),832 adev->firmware.ucode[AMDGPU_UCODE_ID_UVD].tmr_mc_addr_hi);833 MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, 0, mmUVD_VCPU_CACHE_OFFSET0), 0);834 offset = 0;835 } else {836 MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, i, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW),837 lower_32_bits(adev->uvd.inst[i].gpu_addr));838 MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, i, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH),839 upper_32_bits(adev->uvd.inst[i].gpu_addr));840 offset = size;841 MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, 0, mmUVD_VCPU_CACHE_OFFSET0),842 AMDGPU_UVD_FIRMWARE_OFFSET >> 3);843 844 }845 846 MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, i, mmUVD_VCPU_CACHE_SIZE0), size);847 848 MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, i, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW),849 lower_32_bits(adev->uvd.inst[i].gpu_addr + offset));850 MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, i, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH),851 upper_32_bits(adev->uvd.inst[i].gpu_addr + offset));852 MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, i, mmUVD_VCPU_CACHE_OFFSET1), (1 << 21));853 MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, i, mmUVD_VCPU_CACHE_SIZE1), AMDGPU_UVD_HEAP_SIZE);854 855 MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, i, mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_LOW),856 lower_32_bits(adev->uvd.inst[i].gpu_addr + offset + AMDGPU_UVD_HEAP_SIZE));857 MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, i, mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_HIGH),858 upper_32_bits(adev->uvd.inst[i].gpu_addr + offset + AMDGPU_UVD_HEAP_SIZE));859 MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, i, mmUVD_VCPU_CACHE_OFFSET2), (2 << 21));860 MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, i, mmUVD_VCPU_CACHE_SIZE2),861 AMDGPU_UVD_STACK_SIZE + (AMDGPU_UVD_SESSION_SIZE * 40));862 863 MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, i, mmUVD_GP_SCRATCH4), adev->uvd.max_handles);864 /* mc resume end*/865 866 /* disable clock gating */867 MMSCH_V1_0_INSERT_DIRECT_RD_MOD_WT(SOC15_REG_OFFSET(UVD, i, mmUVD_CGC_CTRL),868 ~UVD_CGC_CTRL__DYN_CLOCK_MODE_MASK, 0);869 870 /* disable interupt */871 MMSCH_V1_0_INSERT_DIRECT_RD_MOD_WT(SOC15_REG_OFFSET(UVD, i, mmUVD_MASTINT_EN),872 ~UVD_MASTINT_EN__VCPU_EN_MASK, 0);873 874 /* stall UMC and register bus before resetting VCPU */875 MMSCH_V1_0_INSERT_DIRECT_RD_MOD_WT(SOC15_REG_OFFSET(UVD, i, mmUVD_LMI_CTRL2),876 ~UVD_LMI_CTRL2__STALL_ARB_UMC_MASK,877 UVD_LMI_CTRL2__STALL_ARB_UMC_MASK);878 879 /* put LMI, VCPU, RBC etc... into reset */880 MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, i, mmUVD_SOFT_RESET),881 (uint32_t)(UVD_SOFT_RESET__LMI_SOFT_RESET_MASK |882 UVD_SOFT_RESET__VCPU_SOFT_RESET_MASK |883 UVD_SOFT_RESET__LBSI_SOFT_RESET_MASK |884 UVD_SOFT_RESET__RBC_SOFT_RESET_MASK |885 UVD_SOFT_RESET__CSM_SOFT_RESET_MASK |886 UVD_SOFT_RESET__CXW_SOFT_RESET_MASK |887 UVD_SOFT_RESET__TAP_SOFT_RESET_MASK |888 UVD_SOFT_RESET__LMI_UMC_SOFT_RESET_MASK));889 890 /* initialize UVD memory controller */891 MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, i, mmUVD_LMI_CTRL),892 (uint32_t)((0x40 << UVD_LMI_CTRL__WRITE_CLEAN_TIMER__SHIFT) |893 UVD_LMI_CTRL__WRITE_CLEAN_TIMER_EN_MASK |894 UVD_LMI_CTRL__DATA_COHERENCY_EN_MASK |895 UVD_LMI_CTRL__VCPU_DATA_COHERENCY_EN_MASK |896 UVD_LMI_CTRL__REQ_MODE_MASK |897 0x00100000L));898 899 /* take all subblocks out of reset, except VCPU */900 MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, i, mmUVD_SOFT_RESET),901 UVD_SOFT_RESET__VCPU_SOFT_RESET_MASK);902 903 /* enable VCPU clock */904 MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, i, mmUVD_VCPU_CNTL),905 UVD_VCPU_CNTL__CLK_EN_MASK);906 907 /* enable master interrupt */908 MMSCH_V1_0_INSERT_DIRECT_RD_MOD_WT(SOC15_REG_OFFSET(UVD, i, mmUVD_MASTINT_EN),909 ~(UVD_MASTINT_EN__VCPU_EN_MASK|UVD_MASTINT_EN__SYS_EN_MASK),910 (UVD_MASTINT_EN__VCPU_EN_MASK|UVD_MASTINT_EN__SYS_EN_MASK));911 912 /* clear the bit 4 of UVD_STATUS */913 MMSCH_V1_0_INSERT_DIRECT_RD_MOD_WT(SOC15_REG_OFFSET(UVD, i, mmUVD_STATUS),914 ~(2 << UVD_STATUS__VCPU_REPORT__SHIFT), 0);915 916 /* force RBC into idle state */917 size = order_base_2(ring->ring_size);918 tmp = REG_SET_FIELD(0, UVD_RBC_RB_CNTL, RB_BUFSZ, size);919 tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_NO_FETCH, 1);920 MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, i, mmUVD_RBC_RB_CNTL), tmp);921 922 ring = &adev->uvd.inst[i].ring_enc[0];923 ring->wptr = 0;924 MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, i, mmUVD_RB_BASE_LO), ring->gpu_addr);925 MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, i, mmUVD_RB_BASE_HI), upper_32_bits(ring->gpu_addr));926 MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, i, mmUVD_RB_SIZE), ring->ring_size / 4);927 928 /* boot up the VCPU */929 MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, i, mmUVD_SOFT_RESET), 0);930 931 /* enable UMC */932 MMSCH_V1_0_INSERT_DIRECT_RD_MOD_WT(SOC15_REG_OFFSET(UVD, i, mmUVD_LMI_CTRL2),933 ~UVD_LMI_CTRL2__STALL_ARB_UMC_MASK, 0);934 935 MMSCH_V1_0_INSERT_DIRECT_POLL(SOC15_REG_OFFSET(UVD, i, mmUVD_STATUS), 0x02, 0x02);936 }937 /* add end packet */938 memcpy((void *)init_table, &end, sizeof(struct mmsch_v1_0_cmd_end));939 table_size += sizeof(struct mmsch_v1_0_cmd_end) / 4;940 header->uvd_table_size = table_size;941 942 }943 return uvd_v7_0_mmsch_start(adev, &adev->virt.mm_table);944}945 946/**947 * uvd_v7_0_start - start UVD block948 *949 * @adev: amdgpu_device pointer950 *951 * Setup and start the UVD block952 */953static int uvd_v7_0_start(struct amdgpu_device *adev)954{955 struct amdgpu_ring *ring;956 uint32_t rb_bufsz, tmp;957 uint32_t lmi_swap_cntl;958 uint32_t mp_swap_cntl;959 int i, j, k, r;960 961 for (k = 0; k < adev->uvd.num_uvd_inst; ++k) {962 if (adev->uvd.harvest_config & (1 << k))963 continue;964 /* disable DPG */965 WREG32_P(SOC15_REG_OFFSET(UVD, k, mmUVD_POWER_STATUS), 0,966 ~UVD_POWER_STATUS__UVD_PG_MODE_MASK);967 }968 969 /* disable byte swapping */970 lmi_swap_cntl = 0;971 mp_swap_cntl = 0;972 973 uvd_v7_0_mc_resume(adev);974 975 for (k = 0; k < adev->uvd.num_uvd_inst; ++k) {976 if (adev->uvd.harvest_config & (1 << k))977 continue;978 ring = &adev->uvd.inst[k].ring;979 /* disable clock gating */980 WREG32_P(SOC15_REG_OFFSET(UVD, k, mmUVD_CGC_CTRL), 0,981 ~UVD_CGC_CTRL__DYN_CLOCK_MODE_MASK);982 983 /* disable interupt */984 WREG32_P(SOC15_REG_OFFSET(UVD, k, mmUVD_MASTINT_EN), 0,985 ~UVD_MASTINT_EN__VCPU_EN_MASK);986 987 /* stall UMC and register bus before resetting VCPU */988 WREG32_P(SOC15_REG_OFFSET(UVD, k, mmUVD_LMI_CTRL2),989 UVD_LMI_CTRL2__STALL_ARB_UMC_MASK,990 ~UVD_LMI_CTRL2__STALL_ARB_UMC_MASK);991 mdelay(1);992 993 /* put LMI, VCPU, RBC etc... into reset */994 WREG32_SOC15(UVD, k, mmUVD_SOFT_RESET,995 UVD_SOFT_RESET__LMI_SOFT_RESET_MASK |996 UVD_SOFT_RESET__VCPU_SOFT_RESET_MASK |997 UVD_SOFT_RESET__LBSI_SOFT_RESET_MASK |998 UVD_SOFT_RESET__RBC_SOFT_RESET_MASK |999 UVD_SOFT_RESET__CSM_SOFT_RESET_MASK |1000 UVD_SOFT_RESET__CXW_SOFT_RESET_MASK |1001 UVD_SOFT_RESET__TAP_SOFT_RESET_MASK |1002 UVD_SOFT_RESET__LMI_UMC_SOFT_RESET_MASK);1003 mdelay(5);1004 1005 /* initialize UVD memory controller */1006 WREG32_SOC15(UVD, k, mmUVD_LMI_CTRL,1007 (0x40 << UVD_LMI_CTRL__WRITE_CLEAN_TIMER__SHIFT) |1008 UVD_LMI_CTRL__WRITE_CLEAN_TIMER_EN_MASK |1009 UVD_LMI_CTRL__DATA_COHERENCY_EN_MASK |1010 UVD_LMI_CTRL__VCPU_DATA_COHERENCY_EN_MASK |1011 UVD_LMI_CTRL__REQ_MODE_MASK |1012 0x00100000L);1013 1014#ifdef __BIG_ENDIAN1015 /* swap (8 in 32) RB and IB */1016 lmi_swap_cntl = 0xa;1017 mp_swap_cntl = 0;1018#endif1019 WREG32_SOC15(UVD, k, mmUVD_LMI_SWAP_CNTL, lmi_swap_cntl);1020 WREG32_SOC15(UVD, k, mmUVD_MP_SWAP_CNTL, mp_swap_cntl);1021 1022 WREG32_SOC15(UVD, k, mmUVD_MPC_SET_MUXA0, 0x40c2040);1023 WREG32_SOC15(UVD, k, mmUVD_MPC_SET_MUXA1, 0x0);1024 WREG32_SOC15(UVD, k, mmUVD_MPC_SET_MUXB0, 0x40c2040);1025 WREG32_SOC15(UVD, k, mmUVD_MPC_SET_MUXB1, 0x0);1026 WREG32_SOC15(UVD, k, mmUVD_MPC_SET_ALU, 0);1027 WREG32_SOC15(UVD, k, mmUVD_MPC_SET_MUX, 0x88);1028 1029 /* take all subblocks out of reset, except VCPU */1030 WREG32_SOC15(UVD, k, mmUVD_SOFT_RESET,1031 UVD_SOFT_RESET__VCPU_SOFT_RESET_MASK);1032 mdelay(5);1033 1034 /* enable VCPU clock */1035 WREG32_SOC15(UVD, k, mmUVD_VCPU_CNTL,1036 UVD_VCPU_CNTL__CLK_EN_MASK);1037 1038 /* enable UMC */1039 WREG32_P(SOC15_REG_OFFSET(UVD, k, mmUVD_LMI_CTRL2), 0,1040 ~UVD_LMI_CTRL2__STALL_ARB_UMC_MASK);1041 1042 /* boot up the VCPU */1043 WREG32_SOC15(UVD, k, mmUVD_SOFT_RESET, 0);1044 mdelay(10);1045 1046 for (i = 0; i < 10; ++i) {1047 uint32_t status;1048 1049 for (j = 0; j < 100; ++j) {1050 status = RREG32_SOC15(UVD, k, mmUVD_STATUS);1051 if (status & 2)1052 break;1053 mdelay(10);1054 }1055 r = 0;1056 if (status & 2)1057 break;1058 1059 DRM_ERROR("UVD(%d) not responding, trying to reset the VCPU!!!\n", k);1060 WREG32_P(SOC15_REG_OFFSET(UVD, k, mmUVD_SOFT_RESET),1061 UVD_SOFT_RESET__VCPU_SOFT_RESET_MASK,1062 ~UVD_SOFT_RESET__VCPU_SOFT_RESET_MASK);1063 mdelay(10);1064 WREG32_P(SOC15_REG_OFFSET(UVD, k, mmUVD_SOFT_RESET), 0,1065 ~UVD_SOFT_RESET__VCPU_SOFT_RESET_MASK);1066 mdelay(10);1067 r = -1;1068 }1069 1070 if (r) {1071 DRM_ERROR("UVD(%d) not responding, giving up!!!\n", k);1072 return r;1073 }1074 /* enable master interrupt */1075 WREG32_P(SOC15_REG_OFFSET(UVD, k, mmUVD_MASTINT_EN),1076 (UVD_MASTINT_EN__VCPU_EN_MASK|UVD_MASTINT_EN__SYS_EN_MASK),1077 ~(UVD_MASTINT_EN__VCPU_EN_MASK|UVD_MASTINT_EN__SYS_EN_MASK));1078 1079 /* clear the bit 4 of UVD_STATUS */1080 WREG32_P(SOC15_REG_OFFSET(UVD, k, mmUVD_STATUS), 0,1081 ~(2 << UVD_STATUS__VCPU_REPORT__SHIFT));1082 1083 /* force RBC into idle state */1084 rb_bufsz = order_base_2(ring->ring_size);1085 tmp = REG_SET_FIELD(0, UVD_RBC_RB_CNTL, RB_BUFSZ, rb_bufsz);1086 tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_BLKSZ, 1);1087 tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_NO_FETCH, 1);1088 tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_WPTR_POLL_EN, 0);1089 tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_NO_UPDATE, 1);1090 tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_RPTR_WR_EN, 1);1091 WREG32_SOC15(UVD, k, mmUVD_RBC_RB_CNTL, tmp);1092 1093 /* set the write pointer delay */1094 WREG32_SOC15(UVD, k, mmUVD_RBC_RB_WPTR_CNTL, 0);1095 1096 /* set the wb address */1097 WREG32_SOC15(UVD, k, mmUVD_RBC_RB_RPTR_ADDR,1098 (upper_32_bits(ring->gpu_addr) >> 2));1099 1100 /* program the RB_BASE for ring buffer */1101 WREG32_SOC15(UVD, k, mmUVD_LMI_RBC_RB_64BIT_BAR_LOW,1102 lower_32_bits(ring->gpu_addr));1103 WREG32_SOC15(UVD, k, mmUVD_LMI_RBC_RB_64BIT_BAR_HIGH,1104 upper_32_bits(ring->gpu_addr));1105 1106 /* Initialize the ring buffer's read and write pointers */1107 WREG32_SOC15(UVD, k, mmUVD_RBC_RB_RPTR, 0);1108 1109 ring->wptr = RREG32_SOC15(UVD, k, mmUVD_RBC_RB_RPTR);1110 WREG32_SOC15(UVD, k, mmUVD_RBC_RB_WPTR,1111 lower_32_bits(ring->wptr));1112 1113 WREG32_P(SOC15_REG_OFFSET(UVD, k, mmUVD_RBC_RB_CNTL), 0,1114 ~UVD_RBC_RB_CNTL__RB_NO_FETCH_MASK);1115 1116 ring = &adev->uvd.inst[k].ring_enc[0];1117 WREG32_SOC15(UVD, k, mmUVD_RB_RPTR, lower_32_bits(ring->wptr));1118 WREG32_SOC15(UVD, k, mmUVD_RB_WPTR, lower_32_bits(ring->wptr));1119 WREG32_SOC15(UVD, k, mmUVD_RB_BASE_LO, ring->gpu_addr);1120 WREG32_SOC15(UVD, k, mmUVD_RB_BASE_HI, upper_32_bits(ring->gpu_addr));1121 WREG32_SOC15(UVD, k, mmUVD_RB_SIZE, ring->ring_size / 4);1122 1123 ring = &adev->uvd.inst[k].ring_enc[1];1124 WREG32_SOC15(UVD, k, mmUVD_RB_RPTR2, lower_32_bits(ring->wptr));1125 WREG32_SOC15(UVD, k, mmUVD_RB_WPTR2, lower_32_bits(ring->wptr));1126 WREG32_SOC15(UVD, k, mmUVD_RB_BASE_LO2, ring->gpu_addr);1127 WREG32_SOC15(UVD, k, mmUVD_RB_BASE_HI2, upper_32_bits(ring->gpu_addr));1128 WREG32_SOC15(UVD, k, mmUVD_RB_SIZE2, ring->ring_size / 4);1129 }1130 return 0;1131}1132 1133/**1134 * uvd_v7_0_stop - stop UVD block1135 *1136 * @adev: amdgpu_device pointer1137 *1138 * stop the UVD block1139 */1140static void uvd_v7_0_stop(struct amdgpu_device *adev)1141{1142 uint8_t i = 0;1143 1144 for (i = 0; i < adev->uvd.num_uvd_inst; ++i) {1145 if (adev->uvd.harvest_config & (1 << i))1146 continue;1147 /* force RBC into idle state */1148 WREG32_SOC15(UVD, i, mmUVD_RBC_RB_CNTL, 0x11010101);1149 1150 /* Stall UMC and register bus before resetting VCPU */1151 WREG32_P(SOC15_REG_OFFSET(UVD, i, mmUVD_LMI_CTRL2),1152 UVD_LMI_CTRL2__STALL_ARB_UMC_MASK,1153 ~UVD_LMI_CTRL2__STALL_ARB_UMC_MASK);1154 mdelay(1);1155 1156 /* put VCPU into reset */1157 WREG32_SOC15(UVD, i, mmUVD_SOFT_RESET,1158 UVD_SOFT_RESET__VCPU_SOFT_RESET_MASK);1159 mdelay(5);1160 1161 /* disable VCPU clock */1162 WREG32_SOC15(UVD, i, mmUVD_VCPU_CNTL, 0x0);1163 1164 /* Unstall UMC and register bus */1165 WREG32_P(SOC15_REG_OFFSET(UVD, i, mmUVD_LMI_CTRL2), 0,1166 ~UVD_LMI_CTRL2__STALL_ARB_UMC_MASK);1167 }1168}1169 1170/**1171 * uvd_v7_0_ring_emit_fence - emit an fence & trap command1172 *1173 * @ring: amdgpu_ring pointer1174 * @addr: address1175 * @seq: sequence number1176 * @flags: fence related flags1177 *1178 * Write a fence and a trap command to the ring.1179 */1180static void uvd_v7_0_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, u64 seq,1181 unsigned flags)1182{1183 struct amdgpu_device *adev = ring->adev;1184 1185 WARN_ON(flags & AMDGPU_FENCE_FLAG_64BIT);1186 1187 amdgpu_ring_write(ring,1188 PACKET0(SOC15_REG_OFFSET(UVD, ring->me, mmUVD_CONTEXT_ID), 0));1189 amdgpu_ring_write(ring, seq);1190 amdgpu_ring_write(ring,1191 PACKET0(SOC15_REG_OFFSET(UVD, ring->me, mmUVD_GPCOM_VCPU_DATA0), 0));1192 amdgpu_ring_write(ring, addr & 0xffffffff);1193 amdgpu_ring_write(ring,1194 PACKET0(SOC15_REG_OFFSET(UVD, ring->me, mmUVD_GPCOM_VCPU_DATA1), 0));1195 amdgpu_ring_write(ring, upper_32_bits(addr) & 0xff);1196 amdgpu_ring_write(ring,1197 PACKET0(SOC15_REG_OFFSET(UVD, ring->me, mmUVD_GPCOM_VCPU_CMD), 0));1198 amdgpu_ring_write(ring, 0);1199 1200 amdgpu_ring_write(ring,1201 PACKET0(SOC15_REG_OFFSET(UVD, ring->me, mmUVD_GPCOM_VCPU_DATA0), 0));1202 amdgpu_ring_write(ring, 0);1203 amdgpu_ring_write(ring,1204 PACKET0(SOC15_REG_OFFSET(UVD, ring->me, mmUVD_GPCOM_VCPU_DATA1), 0));1205 amdgpu_ring_write(ring, 0);1206 amdgpu_ring_write(ring,1207 PACKET0(SOC15_REG_OFFSET(UVD, ring->me, mmUVD_GPCOM_VCPU_CMD), 0));1208 amdgpu_ring_write(ring, 2);1209}1210 1211/**1212 * uvd_v7_0_enc_ring_emit_fence - emit an enc fence & trap command1213 *1214 * @ring: amdgpu_ring pointer1215 * @addr: address1216 * @seq: sequence number1217 * @flags: fence related flags1218 *1219 * Write enc a fence and a trap command to the ring.1220 */1221static void uvd_v7_0_enc_ring_emit_fence(struct amdgpu_ring *ring, u64 addr,1222 u64 seq, unsigned flags)1223{1224 1225 WARN_ON(flags & AMDGPU_FENCE_FLAG_64BIT);1226 1227 amdgpu_ring_write(ring, HEVC_ENC_CMD_FENCE);1228 amdgpu_ring_write(ring, addr);1229 amdgpu_ring_write(ring, upper_32_bits(addr));1230 amdgpu_ring_write(ring, seq);1231 amdgpu_ring_write(ring, HEVC_ENC_CMD_TRAP);1232}1233 1234/**1235 * uvd_v7_0_ring_emit_hdp_flush - skip HDP flushing1236 *1237 * @ring: amdgpu_ring pointer1238 */1239static void uvd_v7_0_ring_emit_hdp_flush(struct amdgpu_ring *ring)1240{1241 /* The firmware doesn't seem to like touching registers at this point. */1242}1243 1244/**1245 * uvd_v7_0_ring_test_ring - register write test1246 *1247 * @ring: amdgpu_ring pointer1248 *1249 * Test if we can successfully write to the context register1250 */1251static int uvd_v7_0_ring_test_ring(struct amdgpu_ring *ring)1252{1253 struct amdgpu_device *adev = ring->adev;1254 uint32_t tmp = 0;1255 unsigned i;1256 int r;1257 1258 WREG32_SOC15(UVD, ring->me, mmUVD_CONTEXT_ID, 0xCAFEDEAD);1259 r = amdgpu_ring_alloc(ring, 3);1260 if (r)1261 return r;1262 1263 amdgpu_ring_write(ring,1264 PACKET0(SOC15_REG_OFFSET(UVD, ring->me, mmUVD_CONTEXT_ID), 0));1265 amdgpu_ring_write(ring, 0xDEADBEEF);1266 amdgpu_ring_commit(ring);1267 for (i = 0; i < adev->usec_timeout; i++) {1268 tmp = RREG32_SOC15(UVD, ring->me, mmUVD_CONTEXT_ID);1269 if (tmp == 0xDEADBEEF)1270 break;1271 udelay(1);1272 }1273 1274 if (i >= adev->usec_timeout)1275 r = -ETIMEDOUT;1276 1277 return r;1278}1279 1280/**1281 * uvd_v7_0_ring_patch_cs_in_place - Patch the IB for command submission.1282 *1283 * @p: the CS parser with the IBs1284 * @job: which job this ib is in1285 * @ib: which IB to patch1286 *1287 */1288static int uvd_v7_0_ring_patch_cs_in_place(struct amdgpu_cs_parser *p,1289 struct amdgpu_job *job,1290 struct amdgpu_ib *ib)1291{1292 struct amdgpu_ring *ring = to_amdgpu_ring(job->base.sched);1293 unsigned i;1294 1295 /* No patching necessary for the first instance */1296 if (!ring->me)1297 return 0;1298 1299 for (i = 0; i < ib->length_dw; i += 2) {1300 uint32_t reg = amdgpu_ib_get_value(ib, i);1301 1302 reg -= p->adev->reg_offset[UVD_HWIP][0][1];1303 reg += p->adev->reg_offset[UVD_HWIP][1][1];1304 1305 amdgpu_ib_set_value(ib, i, reg);1306 }1307 return 0;1308}1309 1310/**1311 * uvd_v7_0_ring_emit_ib - execute indirect buffer1312 *1313 * @ring: amdgpu_ring pointer1314 * @job: job to retrieve vmid from1315 * @ib: indirect buffer to execute1316 * @flags: unused1317 *1318 * Write ring commands to execute the indirect buffer1319 */1320static void uvd_v7_0_ring_emit_ib(struct amdgpu_ring *ring,1321 struct amdgpu_job *job,1322 struct amdgpu_ib *ib,1323 uint32_t flags)1324{1325 struct amdgpu_device *adev = ring->adev;1326 unsigned vmid = AMDGPU_JOB_GET_VMID(job);1327 1328 amdgpu_ring_write(ring,1329 PACKET0(SOC15_REG_OFFSET(UVD, ring->me, mmUVD_LMI_RBC_IB_VMID), 0));1330 amdgpu_ring_write(ring, vmid);1331 1332 amdgpu_ring_write(ring,1333 PACKET0(SOC15_REG_OFFSET(UVD, ring->me, mmUVD_LMI_RBC_IB_64BIT_BAR_LOW), 0));1334 amdgpu_ring_write(ring, lower_32_bits(ib->gpu_addr));1335 amdgpu_ring_write(ring,1336 PACKET0(SOC15_REG_OFFSET(UVD, ring->me, mmUVD_LMI_RBC_IB_64BIT_BAR_HIGH), 0));1337 amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr));1338 amdgpu_ring_write(ring,1339 PACKET0(SOC15_REG_OFFSET(UVD, ring->me, mmUVD_RBC_IB_SIZE), 0));1340 amdgpu_ring_write(ring, ib->length_dw);1341}1342 1343/**1344 * uvd_v7_0_enc_ring_emit_ib - enc execute indirect buffer1345 *1346 * @ring: amdgpu_ring pointer1347 * @job: job to retrive vmid from1348 * @ib: indirect buffer to execute1349 * @flags: unused1350 *1351 * Write enc ring commands to execute the indirect buffer1352 */1353static void uvd_v7_0_enc_ring_emit_ib(struct amdgpu_ring *ring,1354 struct amdgpu_job *job,1355 struct amdgpu_ib *ib,1356 uint32_t flags)1357{1358 unsigned vmid = AMDGPU_JOB_GET_VMID(job);1359 1360 amdgpu_ring_write(ring, HEVC_ENC_CMD_IB_VM);1361 amdgpu_ring_write(ring, vmid);1362 amdgpu_ring_write(ring, lower_32_bits(ib->gpu_addr));1363 amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr));1364 amdgpu_ring_write(ring, ib->length_dw);1365}1366 1367static void uvd_v7_0_ring_emit_wreg(struct amdgpu_ring *ring,1368 uint32_t reg, uint32_t val)1369{1370 struct amdgpu_device *adev = ring->adev;1371 1372 amdgpu_ring_write(ring,1373 PACKET0(SOC15_REG_OFFSET(UVD, ring->me, mmUVD_GPCOM_VCPU_DATA0), 0));1374 amdgpu_ring_write(ring, reg << 2);1375 amdgpu_ring_write(ring,1376 PACKET0(SOC15_REG_OFFSET(UVD, ring->me, mmUVD_GPCOM_VCPU_DATA1), 0));1377 amdgpu_ring_write(ring, val);1378 amdgpu_ring_write(ring,1379 PACKET0(SOC15_REG_OFFSET(UVD, ring->me, mmUVD_GPCOM_VCPU_CMD), 0));1380 amdgpu_ring_write(ring, 8);1381}1382 1383static void uvd_v7_0_ring_emit_reg_wait(struct amdgpu_ring *ring, uint32_t reg,1384 uint32_t val, uint32_t mask)1385{1386 struct amdgpu_device *adev = ring->adev;1387 1388 amdgpu_ring_write(ring,1389 PACKET0(SOC15_REG_OFFSET(UVD, ring->me, mmUVD_GPCOM_VCPU_DATA0), 0));1390 amdgpu_ring_write(ring, reg << 2);1391 amdgpu_ring_write(ring,1392 PACKET0(SOC15_REG_OFFSET(UVD, ring->me, mmUVD_GPCOM_VCPU_DATA1), 0));1393 amdgpu_ring_write(ring, val);1394 amdgpu_ring_write(ring,1395 PACKET0(SOC15_REG_OFFSET(UVD, ring->me, mmUVD_GP_SCRATCH8), 0));1396 amdgpu_ring_write(ring, mask);1397 amdgpu_ring_write(ring,1398 PACKET0(SOC15_REG_OFFSET(UVD, ring->me, mmUVD_GPCOM_VCPU_CMD), 0));1399 amdgpu_ring_write(ring, 12);1400}1401 1402static void uvd_v7_0_ring_emit_vm_flush(struct amdgpu_ring *ring,1403 unsigned vmid, uint64_t pd_addr)1404{1405 struct amdgpu_vmhub *hub = &ring->adev->vmhub[ring->vm_hub];1406 uint32_t data0, data1, mask;1407 1408 pd_addr = amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr);1409 1410 /* wait for reg writes */1411 data0 = hub->ctx0_ptb_addr_lo32 + vmid * hub->ctx_addr_distance;1412 data1 = lower_32_bits(pd_addr);1413 mask = 0xffffffff;1414 uvd_v7_0_ring_emit_reg_wait(ring, data0, data1, mask);1415}1416 1417static void uvd_v7_0_ring_insert_nop(struct amdgpu_ring *ring, uint32_t count)1418{1419 struct amdgpu_device *adev = ring->adev;1420 int i;1421 1422 WARN_ON(ring->wptr % 2 || count % 2);1423 1424 for (i = 0; i < count / 2; i++) {1425 amdgpu_ring_write(ring, PACKET0(SOC15_REG_OFFSET(UVD, ring->me, mmUVD_NO_OP), 0));1426 amdgpu_ring_write(ring, 0);1427 }1428}1429 1430static void uvd_v7_0_enc_ring_insert_end(struct amdgpu_ring *ring)1431{1432 amdgpu_ring_write(ring, HEVC_ENC_CMD_END);1433}1434 1435static void uvd_v7_0_enc_ring_emit_reg_wait(struct amdgpu_ring *ring,1436 uint32_t reg, uint32_t val,1437 uint32_t mask)1438{1439 amdgpu_ring_write(ring, HEVC_ENC_CMD_REG_WAIT);1440 amdgpu_ring_write(ring, reg << 2);1441 amdgpu_ring_write(ring, mask);1442 amdgpu_ring_write(ring, val);1443}1444 1445static void uvd_v7_0_enc_ring_emit_vm_flush(struct amdgpu_ring *ring,1446 unsigned int vmid, uint64_t pd_addr)1447{1448 struct amdgpu_vmhub *hub = &ring->adev->vmhub[ring->vm_hub];1449 1450 pd_addr = amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr);1451 1452 /* wait for reg writes */1453 uvd_v7_0_enc_ring_emit_reg_wait(ring, hub->ctx0_ptb_addr_lo32 +1454 vmid * hub->ctx_addr_distance,1455 lower_32_bits(pd_addr), 0xffffffff);1456}1457 1458static void uvd_v7_0_enc_ring_emit_wreg(struct amdgpu_ring *ring,1459 uint32_t reg, uint32_t val)1460{1461 amdgpu_ring_write(ring, HEVC_ENC_CMD_REG_WRITE);1462 amdgpu_ring_write(ring, reg << 2);1463 amdgpu_ring_write(ring, val);1464}1465 1466#if 01467static bool uvd_v7_0_is_idle(void *handle)1468{1469 struct amdgpu_device *adev = (struct amdgpu_device *)handle;1470 1471 return !(RREG32(mmSRBM_STATUS) & SRBM_STATUS__UVD_BUSY_MASK);1472}1473 1474static int uvd_v7_0_wait_for_idle(void *handle)1475{1476 unsigned i;1477 struct amdgpu_device *adev = (struct amdgpu_device *)handle;1478 1479 for (i = 0; i < adev->usec_timeout; i++) {1480 if (uvd_v7_0_is_idle(handle))1481 return 0;1482 }1483 return -ETIMEDOUT;1484}1485 1486#define AMDGPU_UVD_STATUS_BUSY_MASK 0xfd1487static bool uvd_v7_0_check_soft_reset(void *handle)1488{1489 struct amdgpu_device *adev = (struct amdgpu_device *)handle;1490 u32 srbm_soft_reset = 0;1491 u32 tmp = RREG32(mmSRBM_STATUS);1492 1493 if (REG_GET_FIELD(tmp, SRBM_STATUS, UVD_RQ_PENDING) ||1494 REG_GET_FIELD(tmp, SRBM_STATUS, UVD_BUSY) ||1495 (RREG32_SOC15(UVD, ring->me, mmUVD_STATUS) &1496 AMDGPU_UVD_STATUS_BUSY_MASK))1497 srbm_soft_reset = REG_SET_FIELD(srbm_soft_reset,1498 SRBM_SOFT_RESET, SOFT_RESET_UVD, 1);1499 1500 if (srbm_soft_reset) {1501 adev->uvd.inst[ring->me].srbm_soft_reset = srbm_soft_reset;1502 return true;1503 } else {1504 adev->uvd.inst[ring->me].srbm_soft_reset = 0;1505 return false;1506 }1507}1508 1509static int uvd_v7_0_pre_soft_reset(void *handle)1510{1511 struct amdgpu_device *adev = (struct amdgpu_device *)handle;1512 1513 if (!adev->uvd.inst[ring->me].srbm_soft_reset)1514 return 0;1515 1516 uvd_v7_0_stop(adev);1517 return 0;1518}1519 1520static int uvd_v7_0_soft_reset(void *handle)1521{1522 struct amdgpu_device *adev = (struct amdgpu_device *)handle;1523 u32 srbm_soft_reset;1524 1525 if (!adev->uvd.inst[ring->me].srbm_soft_reset)1526 return 0;1527 srbm_soft_reset = adev->uvd.inst[ring->me].srbm_soft_reset;1528 1529 if (srbm_soft_reset) {1530 u32 tmp;1531 1532 tmp = RREG32(mmSRBM_SOFT_RESET);1533 tmp |= srbm_soft_reset;1534 dev_info(adev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);1535 WREG32(mmSRBM_SOFT_RESET, tmp);1536 tmp = RREG32(mmSRBM_SOFT_RESET);1537 1538 udelay(50);1539 1540 tmp &= ~srbm_soft_reset;1541 WREG32(mmSRBM_SOFT_RESET, tmp);1542 tmp = RREG32(mmSRBM_SOFT_RESET);1543 1544 /* Wait a little for things to settle down */1545 udelay(50);1546 }1547 1548 return 0;1549}1550 1551static int uvd_v7_0_post_soft_reset(void *handle)1552{1553 struct amdgpu_device *adev = (struct amdgpu_device *)handle;1554 1555 if (!adev->uvd.inst[ring->me].srbm_soft_reset)1556 return 0;1557 1558 mdelay(5);1559 1560 return uvd_v7_0_start(adev);1561}1562#endif1563 1564static int uvd_v7_0_set_interrupt_state(struct amdgpu_device *adev,1565 struct amdgpu_irq_src *source,1566 unsigned type,1567 enum amdgpu_interrupt_state state)1568{1569 // TODO1570 return 0;1571}1572 1573static int uvd_v7_0_process_interrupt(struct amdgpu_device *adev,1574 struct amdgpu_irq_src *source,1575 struct amdgpu_iv_entry *entry)1576{1577 uint32_t ip_instance;1578 1579 switch (entry->client_id) {1580 case SOC15_IH_CLIENTID_UVD:1581 ip_instance = 0;1582 break;1583 case SOC15_IH_CLIENTID_UVD1:1584 ip_instance = 1;1585 break;1586 default:1587 DRM_ERROR("Unhandled client id: %d\n", entry->client_id);1588 return 0;1589 }1590 1591 DRM_DEBUG("IH: UVD TRAP\n");1592 1593 switch (entry->src_id) {1594 case 124:1595 amdgpu_fence_process(&adev->uvd.inst[ip_instance].ring);1596 break;1597 case 119:1598 amdgpu_fence_process(&adev->uvd.inst[ip_instance].ring_enc[0]);1599 break;1600 case 120:1601 if (!amdgpu_sriov_vf(adev))1602 amdgpu_fence_process(&adev->uvd.inst[ip_instance].ring_enc[1]);1603 break;1604 default:1605 DRM_ERROR("Unhandled interrupt: %d %d\n",1606 entry->src_id, entry->src_data[0]);1607 break;1608 }1609 1610 return 0;1611}1612 1613#if 01614static void uvd_v7_0_set_sw_clock_gating(struct amdgpu_device *adev)1615{1616 uint32_t data, data1, data2, suvd_flags;1617 1618 data = RREG32_SOC15(UVD, ring->me, mmUVD_CGC_CTRL);1619 data1 = RREG32_SOC15(UVD, ring->me, mmUVD_SUVD_CGC_GATE);1620 data2 = RREG32_SOC15(UVD, ring->me, mmUVD_SUVD_CGC_CTRL);1621 1622 data &= ~(UVD_CGC_CTRL__CLK_OFF_DELAY_MASK |1623 UVD_CGC_CTRL__CLK_GATE_DLY_TIMER_MASK);1624 1625 suvd_flags = UVD_SUVD_CGC_GATE__SRE_MASK |1626 UVD_SUVD_CGC_GATE__SIT_MASK |1627 UVD_SUVD_CGC_GATE__SMP_MASK |1628 UVD_SUVD_CGC_GATE__SCM_MASK |1629 UVD_SUVD_CGC_GATE__SDB_MASK;1630 1631 data |= UVD_CGC_CTRL__DYN_CLOCK_MODE_MASK |1632 (1 << REG_FIELD_SHIFT(UVD_CGC_CTRL, CLK_GATE_DLY_TIMER)) |1633 (4 << REG_FIELD_SHIFT(UVD_CGC_CTRL, CLK_OFF_DELAY));1634 1635 data &= ~(UVD_CGC_CTRL__UDEC_RE_MODE_MASK |1636 UVD_CGC_CTRL__UDEC_CM_MODE_MASK |1637 UVD_CGC_CTRL__UDEC_IT_MODE_MASK |1638 UVD_CGC_CTRL__UDEC_DB_MODE_MASK |1639 UVD_CGC_CTRL__UDEC_MP_MODE_MASK |1640 UVD_CGC_CTRL__SYS_MODE_MASK |1641 UVD_CGC_CTRL__UDEC_MODE_MASK |1642 UVD_CGC_CTRL__MPEG2_MODE_MASK |1643 UVD_CGC_CTRL__REGS_MODE_MASK |1644 UVD_CGC_CTRL__RBC_MODE_MASK |1645 UVD_CGC_CTRL__LMI_MC_MODE_MASK |1646 UVD_CGC_CTRL__LMI_UMC_MODE_MASK |1647 UVD_CGC_CTRL__IDCT_MODE_MASK |1648 UVD_CGC_CTRL__MPRD_MODE_MASK |1649 UVD_CGC_CTRL__MPC_MODE_MASK |1650 UVD_CGC_CTRL__LBSI_MODE_MASK |1651 UVD_CGC_CTRL__LRBBM_MODE_MASK |1652 UVD_CGC_CTRL__WCB_MODE_MASK |1653 UVD_CGC_CTRL__VCPU_MODE_MASK |1654 UVD_CGC_CTRL__JPEG_MODE_MASK |1655 UVD_CGC_CTRL__JPEG2_MODE_MASK |1656 UVD_CGC_CTRL__SCPU_MODE_MASK);1657 data2 &= ~(UVD_SUVD_CGC_CTRL__SRE_MODE_MASK |1658 UVD_SUVD_CGC_CTRL__SIT_MODE_MASK |1659 UVD_SUVD_CGC_CTRL__SMP_MODE_MASK |1660 UVD_SUVD_CGC_CTRL__SCM_MODE_MASK |1661 UVD_SUVD_CGC_CTRL__SDB_MODE_MASK);1662 data1 |= suvd_flags;1663 1664 WREG32_SOC15(UVD, ring->me, mmUVD_CGC_CTRL, data);1665 WREG32_SOC15(UVD, ring->me, mmUVD_CGC_GATE, 0);1666 WREG32_SOC15(UVD, ring->me, mmUVD_SUVD_CGC_GATE, data1);1667 WREG32_SOC15(UVD, ring->me, mmUVD_SUVD_CGC_CTRL, data2);1668}1669 1670static void uvd_v7_0_set_hw_clock_gating(struct amdgpu_device *adev)1671{1672 uint32_t data, data1, cgc_flags, suvd_flags;1673 1674 data = RREG32_SOC15(UVD, ring->me, mmUVD_CGC_GATE);1675 data1 = RREG32_SOC15(UVD, ring->me, mmUVD_SUVD_CGC_GATE);1676 1677 cgc_flags = UVD_CGC_GATE__SYS_MASK |1678 UVD_CGC_GATE__UDEC_MASK |1679 UVD_CGC_GATE__MPEG2_MASK |1680 UVD_CGC_GATE__RBC_MASK |1681 UVD_CGC_GATE__LMI_MC_MASK |1682 UVD_CGC_GATE__IDCT_MASK |1683 UVD_CGC_GATE__MPRD_MASK |1684 UVD_CGC_GATE__MPC_MASK |1685 UVD_CGC_GATE__LBSI_MASK |1686 UVD_CGC_GATE__LRBBM_MASK |1687 UVD_CGC_GATE__UDEC_RE_MASK |1688 UVD_CGC_GATE__UDEC_CM_MASK |1689 UVD_CGC_GATE__UDEC_IT_MASK |1690 UVD_CGC_GATE__UDEC_DB_MASK |1691 UVD_CGC_GATE__UDEC_MP_MASK |1692 UVD_CGC_GATE__WCB_MASK |1693 UVD_CGC_GATE__VCPU_MASK |1694 UVD_CGC_GATE__SCPU_MASK |1695 UVD_CGC_GATE__JPEG_MASK |1696 UVD_CGC_GATE__JPEG2_MASK;1697 1698 suvd_flags = UVD_SUVD_CGC_GATE__SRE_MASK |1699 UVD_SUVD_CGC_GATE__SIT_MASK |1700 UVD_SUVD_CGC_GATE__SMP_MASK |1701 UVD_SUVD_CGC_GATE__SCM_MASK |1702 UVD_SUVD_CGC_GATE__SDB_MASK;1703 1704 data |= cgc_flags;1705 data1 |= suvd_flags;1706 1707 WREG32_SOC15(UVD, ring->me, mmUVD_CGC_GATE, data);1708 WREG32_SOC15(UVD, ring->me, mmUVD_SUVD_CGC_GATE, data1);1709}1710 1711static void uvd_v7_0_set_bypass_mode(struct amdgpu_device *adev, bool enable)1712{1713 u32 tmp = RREG32_SMC(ixGCK_DFS_BYPASS_CNTL);1714 1715 if (enable)1716 tmp |= (GCK_DFS_BYPASS_CNTL__BYPASSDCLK_MASK |1717 GCK_DFS_BYPASS_CNTL__BYPASSVCLK_MASK);1718 else1719 tmp &= ~(GCK_DFS_BYPASS_CNTL__BYPASSDCLK_MASK |1720 GCK_DFS_BYPASS_CNTL__BYPASSVCLK_MASK);1721 1722 WREG32_SMC(ixGCK_DFS_BYPASS_CNTL, tmp);1723}1724 1725 1726static int uvd_v7_0_set_clockgating_state(void *handle,1727 enum amd_clockgating_state state)1728{1729 struct amdgpu_device *adev = (struct amdgpu_device *)handle;1730 bool enable = (state == AMD_CG_STATE_GATE);1731 1732 uvd_v7_0_set_bypass_mode(adev, enable);1733 1734 if (!(adev->cg_flags & AMD_CG_SUPPORT_UVD_MGCG))1735 return 0;1736 1737 if (enable) {1738 /* disable HW gating and enable Sw gating */1739 uvd_v7_0_set_sw_clock_gating(adev);1740 } else {1741 /* wait for STATUS to clear */1742 if (uvd_v7_0_wait_for_idle(handle))1743 return -EBUSY;1744 1745 /* enable HW gates because UVD is idle */1746 /* uvd_v7_0_set_hw_clock_gating(adev); */1747 }1748 1749 return 0;1750}1751 1752static int uvd_v7_0_set_powergating_state(void *handle,1753 enum amd_powergating_state state)1754{1755 /* This doesn't actually powergate the UVD block.1756 * That's done in the dpm code via the SMC. This1757 * just re-inits the block as necessary. The actual1758 * gating still happens in the dpm code. We should1759 * revisit this when there is a cleaner line between1760 * the smc and the hw blocks1761 */1762 struct amdgpu_device *adev = (struct amdgpu_device *)handle;1763 1764 if (!(adev->pg_flags & AMD_PG_SUPPORT_UVD))1765 return 0;1766 1767 WREG32_SOC15(UVD, ring->me, mmUVD_POWER_STATUS, UVD_POWER_STATUS__UVD_PG_EN_MASK);1768 1769 if (state == AMD_PG_STATE_GATE) {1770 uvd_v7_0_stop(adev);1771 return 0;1772 } else {1773 return uvd_v7_0_start(adev);1774 }1775}1776#endif1777 1778static int uvd_v7_0_set_clockgating_state(void *handle,1779 enum amd_clockgating_state state)1780{1781 /* needed for driver unload*/1782 return 0;1783}1784 1785const struct amd_ip_funcs uvd_v7_0_ip_funcs = {1786 .name = "uvd_v7_0",1787 .early_init = uvd_v7_0_early_init,1788 .late_init = NULL,1789 .sw_init = uvd_v7_0_sw_init,1790 .sw_fini = uvd_v7_0_sw_fini,1791 .hw_init = uvd_v7_0_hw_init,1792 .hw_fini = uvd_v7_0_hw_fini,1793 .prepare_suspend = uvd_v7_0_prepare_suspend,1794 .suspend = uvd_v7_0_suspend,1795 .resume = uvd_v7_0_resume,1796 .is_idle = NULL /* uvd_v7_0_is_idle */,1797 .wait_for_idle = NULL /* uvd_v7_0_wait_for_idle */,1798 .check_soft_reset = NULL /* uvd_v7_0_check_soft_reset */,1799 .pre_soft_reset = NULL /* uvd_v7_0_pre_soft_reset */,1800 .soft_reset = NULL /* uvd_v7_0_soft_reset */,1801 .post_soft_reset = NULL /* uvd_v7_0_post_soft_reset */,1802 .set_clockgating_state = uvd_v7_0_set_clockgating_state,1803 .set_powergating_state = NULL /* uvd_v7_0_set_powergating_state */,1804};1805 1806static const struct amdgpu_ring_funcs uvd_v7_0_ring_vm_funcs = {1807 .type = AMDGPU_RING_TYPE_UVD,1808 .align_mask = 0xf,1809 .support_64bit_ptrs = false,1810 .no_user_fence = true,1811 .get_rptr = uvd_v7_0_ring_get_rptr,1812 .get_wptr = uvd_v7_0_ring_get_wptr,1813 .set_wptr = uvd_v7_0_ring_set_wptr,1814 .patch_cs_in_place = uvd_v7_0_ring_patch_cs_in_place,1815 .emit_frame_size =1816 6 + /* hdp invalidate */1817 SOC15_FLUSH_GPU_TLB_NUM_WREG * 6 +1818 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 8 +1819 8 + /* uvd_v7_0_ring_emit_vm_flush */1820 14 + 14, /* uvd_v7_0_ring_emit_fence x2 vm fence */1821 .emit_ib_size = 8, /* uvd_v7_0_ring_emit_ib */1822 .emit_ib = uvd_v7_0_ring_emit_ib,1823 .emit_fence = uvd_v7_0_ring_emit_fence,1824 .emit_vm_flush = uvd_v7_0_ring_emit_vm_flush,1825 .emit_hdp_flush = uvd_v7_0_ring_emit_hdp_flush,1826 .test_ring = uvd_v7_0_ring_test_ring,1827 .test_ib = amdgpu_uvd_ring_test_ib,1828 .insert_nop = uvd_v7_0_ring_insert_nop,1829 .pad_ib = amdgpu_ring_generic_pad_ib,1830 .begin_use = amdgpu_uvd_ring_begin_use,1831 .end_use = amdgpu_uvd_ring_end_use,1832 .emit_wreg = uvd_v7_0_ring_emit_wreg,1833 .emit_reg_wait = uvd_v7_0_ring_emit_reg_wait,1834 .emit_reg_write_reg_wait = amdgpu_ring_emit_reg_write_reg_wait_helper,1835};1836 1837static const struct amdgpu_ring_funcs uvd_v7_0_enc_ring_vm_funcs = {1838 .type = AMDGPU_RING_TYPE_UVD_ENC,1839 .align_mask = 0x3f,1840 .nop = HEVC_ENC_CMD_NO_OP,1841 .support_64bit_ptrs = false,1842 .no_user_fence = true,1843 .get_rptr = uvd_v7_0_enc_ring_get_rptr,1844 .get_wptr = uvd_v7_0_enc_ring_get_wptr,1845 .set_wptr = uvd_v7_0_enc_ring_set_wptr,1846 .emit_frame_size =1847 3 + 3 + /* hdp flush / invalidate */1848 SOC15_FLUSH_GPU_TLB_NUM_WREG * 3 +1849 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 4 +1850 4 + /* uvd_v7_0_enc_ring_emit_vm_flush */1851 5 + 5 + /* uvd_v7_0_enc_ring_emit_fence x2 vm fence */1852 1, /* uvd_v7_0_enc_ring_insert_end */1853 .emit_ib_size = 5, /* uvd_v7_0_enc_ring_emit_ib */1854 .emit_ib = uvd_v7_0_enc_ring_emit_ib,1855 .emit_fence = uvd_v7_0_enc_ring_emit_fence,1856 .emit_vm_flush = uvd_v7_0_enc_ring_emit_vm_flush,1857 .test_ring = uvd_v7_0_enc_ring_test_ring,1858 .test_ib = uvd_v7_0_enc_ring_test_ib,1859 .insert_nop = amdgpu_ring_insert_nop,1860 .insert_end = uvd_v7_0_enc_ring_insert_end,1861 .pad_ib = amdgpu_ring_generic_pad_ib,1862 .begin_use = amdgpu_uvd_ring_begin_use,1863 .end_use = amdgpu_uvd_ring_end_use,1864 .emit_wreg = uvd_v7_0_enc_ring_emit_wreg,1865 .emit_reg_wait = uvd_v7_0_enc_ring_emit_reg_wait,1866 .emit_reg_write_reg_wait = amdgpu_ring_emit_reg_write_reg_wait_helper,1867};1868 1869static void uvd_v7_0_set_ring_funcs(struct amdgpu_device *adev)1870{1871 int i;1872 1873 for (i = 0; i < adev->uvd.num_uvd_inst; i++) {1874 if (adev->uvd.harvest_config & (1 << i))1875 continue;1876 adev->uvd.inst[i].ring.funcs = &uvd_v7_0_ring_vm_funcs;1877 adev->uvd.inst[i].ring.me = i;1878 DRM_INFO("UVD(%d) is enabled in VM mode\n", i);1879 }1880}1881 1882static void uvd_v7_0_set_enc_ring_funcs(struct amdgpu_device *adev)1883{1884 int i, j;1885 1886 for (j = 0; j < adev->uvd.num_uvd_inst; j++) {1887 if (adev->uvd.harvest_config & (1 << j))1888 continue;1889 for (i = 0; i < adev->uvd.num_enc_rings; ++i) {1890 adev->uvd.inst[j].ring_enc[i].funcs = &uvd_v7_0_enc_ring_vm_funcs;1891 adev->uvd.inst[j].ring_enc[i].me = j;1892 }1893 1894 DRM_INFO("UVD(%d) ENC is enabled in VM mode\n", j);1895 }1896}1897 1898static const struct amdgpu_irq_src_funcs uvd_v7_0_irq_funcs = {1899 .set = uvd_v7_0_set_interrupt_state,1900 .process = uvd_v7_0_process_interrupt,1901};1902 1903static void uvd_v7_0_set_irq_funcs(struct amdgpu_device *adev)1904{1905 int i;1906 1907 for (i = 0; i < adev->uvd.num_uvd_inst; i++) {1908 if (adev->uvd.harvest_config & (1 << i))1909 continue;1910 adev->uvd.inst[i].irq.num_types = adev->uvd.num_enc_rings + 1;1911 adev->uvd.inst[i].irq.funcs = &uvd_v7_0_irq_funcs;1912 }1913}1914 1915const struct amdgpu_ip_block_version uvd_v7_0_ip_block = {1916 .type = AMD_IP_BLOCK_TYPE_UVD,1917 .major = 7,1918 .minor = 0,1919 .rev = 0,1920 .funcs = &uvd_v7_0_ip_funcs,1921};1922