1269 lines · c
1/*2 * Copyright 2014 Advanced Micro Devices, Inc.3 *4 * Permission is hereby granted, free of charge, to any person obtaining a5 * copy of this software and associated documentation files (the "Software"),6 * to deal in the Software without restriction, including without limitation7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,8 * and/or sell copies of the Software, and to permit persons to whom the9 * Software is furnished to do so, subject to the following conditions:10 *11 * The above copyright notice and this permission notice shall be included in12 * all copies or substantial portions of the Software.13 *14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR20 * OTHER DEALINGS IN THE SOFTWARE.21 *22 * Authors: Alex Deucher23 */24 25#include <linux/delay.h>26#include <linux/firmware.h>27#include <linux/module.h>28 29#include "amdgpu.h"30#include "amdgpu_ucode.h"31#include "amdgpu_trace.h"32#include "vi.h"33#include "vid.h"34 35#include "oss/oss_2_4_d.h"36#include "oss/oss_2_4_sh_mask.h"37 38#include "gmc/gmc_7_1_d.h"39#include "gmc/gmc_7_1_sh_mask.h"40 41#include "gca/gfx_8_0_d.h"42#include "gca/gfx_8_0_enum.h"43#include "gca/gfx_8_0_sh_mask.h"44 45#include "bif/bif_5_0_d.h"46#include "bif/bif_5_0_sh_mask.h"47 48#include "iceland_sdma_pkt_open.h"49 50#include "ivsrcid/ivsrcid_vislands30.h"51 52static void sdma_v2_4_set_ring_funcs(struct amdgpu_device *adev);53static void sdma_v2_4_set_buffer_funcs(struct amdgpu_device *adev);54static void sdma_v2_4_set_vm_pte_funcs(struct amdgpu_device *adev);55static void sdma_v2_4_set_irq_funcs(struct amdgpu_device *adev);56 57MODULE_FIRMWARE("amdgpu/topaz_sdma.bin");58MODULE_FIRMWARE("amdgpu/topaz_sdma1.bin");59 60static const u32 sdma_offsets[SDMA_MAX_INSTANCE] = {61 SDMA0_REGISTER_OFFSET,62 SDMA1_REGISTER_OFFSET63};64 65static const u32 golden_settings_iceland_a11[] = {66 mmSDMA0_CHICKEN_BITS, 0xfc910007, 0x00810007,67 mmSDMA0_CLK_CTRL, 0xff000fff, 0x00000000,68 mmSDMA1_CHICKEN_BITS, 0xfc910007, 0x00810007,69 mmSDMA1_CLK_CTRL, 0xff000fff, 0x00000000,70};71 72static const u32 iceland_mgcg_cgcg_init[] = {73 mmSDMA0_CLK_CTRL, 0xff000ff0, 0x00000100,74 mmSDMA1_CLK_CTRL, 0xff000ff0, 0x0000010075};76 77/*78 * sDMA - System DMA79 * Starting with CIK, the GPU has new asynchronous80 * DMA engines. These engines are used for compute81 * and gfx. There are two DMA engines (SDMA0, SDMA1)82 * and each one supports 1 ring buffer used for gfx83 * and 2 queues used for compute.84 *85 * The programming model is very similar to the CP86 * (ring buffer, IBs, etc.), but sDMA has it's own87 * packet format that is different from the PM4 format88 * used by the CP. sDMA supports copying data, writing89 * embedded data, solid fills, and a number of other90 * things. It also has support for tiling/detiling of91 * buffers.92 */93 94static void sdma_v2_4_init_golden_registers(struct amdgpu_device *adev)95{96 switch (adev->asic_type) {97 case CHIP_TOPAZ:98 amdgpu_device_program_register_sequence(adev,99 iceland_mgcg_cgcg_init,100 ARRAY_SIZE(iceland_mgcg_cgcg_init));101 amdgpu_device_program_register_sequence(adev,102 golden_settings_iceland_a11,103 ARRAY_SIZE(golden_settings_iceland_a11));104 break;105 default:106 break;107 }108}109 110static void sdma_v2_4_free_microcode(struct amdgpu_device *adev)111{112 int i;113 114 for (i = 0; i < adev->sdma.num_instances; i++)115 amdgpu_ucode_release(&adev->sdma.instance[i].fw);116}117 118/**119 * sdma_v2_4_init_microcode - load ucode images from disk120 *121 * @adev: amdgpu_device pointer122 *123 * Use the firmware interface to load the ucode images into124 * the driver (not loaded into hw).125 * Returns 0 on success, error on failure.126 */127static int sdma_v2_4_init_microcode(struct amdgpu_device *adev)128{129 const char *chip_name;130 int err = 0, i;131 struct amdgpu_firmware_info *info = NULL;132 const struct common_firmware_header *header = NULL;133 const struct sdma_firmware_header_v1_0 *hdr;134 135 DRM_DEBUG("\n");136 137 switch (adev->asic_type) {138 case CHIP_TOPAZ:139 chip_name = "topaz";140 break;141 default:142 BUG();143 }144 145 for (i = 0; i < adev->sdma.num_instances; i++) {146 if (i == 0)147 err = amdgpu_ucode_request(adev, &adev->sdma.instance[i].fw,148 "amdgpu/%s_sdma.bin", chip_name);149 else150 err = amdgpu_ucode_request(adev, &adev->sdma.instance[i].fw,151 "amdgpu/%s_sdma1.bin", chip_name);152 if (err)153 goto out;154 hdr = (const struct sdma_firmware_header_v1_0 *)adev->sdma.instance[i].fw->data;155 adev->sdma.instance[i].fw_version = le32_to_cpu(hdr->header.ucode_version);156 adev->sdma.instance[i].feature_version = le32_to_cpu(hdr->ucode_feature_version);157 if (adev->sdma.instance[i].feature_version >= 20)158 adev->sdma.instance[i].burst_nop = true;159 160 if (adev->firmware.load_type == AMDGPU_FW_LOAD_SMU) {161 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_SDMA0 + i];162 info->ucode_id = AMDGPU_UCODE_ID_SDMA0 + i;163 info->fw = adev->sdma.instance[i].fw;164 header = (const struct common_firmware_header *)info->fw->data;165 adev->firmware.fw_size +=166 ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE);167 }168 }169 170out:171 if (err) {172 pr_err("sdma_v2_4: Failed to load firmware \"%s_sdma%s.bin\"\n",173 chip_name, i == 0 ? "" : "1");174 for (i = 0; i < adev->sdma.num_instances; i++)175 amdgpu_ucode_release(&adev->sdma.instance[i].fw);176 }177 return err;178}179 180/**181 * sdma_v2_4_ring_get_rptr - get the current read pointer182 *183 * @ring: amdgpu ring pointer184 *185 * Get the current rptr from the hardware (VI+).186 */187static uint64_t sdma_v2_4_ring_get_rptr(struct amdgpu_ring *ring)188{189 /* XXX check if swapping is necessary on BE */190 return *ring->rptr_cpu_addr >> 2;191}192 193/**194 * sdma_v2_4_ring_get_wptr - get the current write pointer195 *196 * @ring: amdgpu ring pointer197 *198 * Get the current wptr from the hardware (VI+).199 */200static uint64_t sdma_v2_4_ring_get_wptr(struct amdgpu_ring *ring)201{202 struct amdgpu_device *adev = ring->adev;203 u32 wptr = RREG32(mmSDMA0_GFX_RB_WPTR + sdma_offsets[ring->me]) >> 2;204 205 return wptr;206}207 208/**209 * sdma_v2_4_ring_set_wptr - commit the write pointer210 *211 * @ring: amdgpu ring pointer212 *213 * Write the wptr back to the hardware (VI+).214 */215static void sdma_v2_4_ring_set_wptr(struct amdgpu_ring *ring)216{217 struct amdgpu_device *adev = ring->adev;218 219 WREG32(mmSDMA0_GFX_RB_WPTR + sdma_offsets[ring->me], ring->wptr << 2);220}221 222static void sdma_v2_4_ring_insert_nop(struct amdgpu_ring *ring, uint32_t count)223{224 struct amdgpu_sdma_instance *sdma = amdgpu_sdma_get_instance_from_ring(ring);225 int i;226 227 for (i = 0; i < count; i++)228 if (sdma && sdma->burst_nop && (i == 0))229 amdgpu_ring_write(ring, ring->funcs->nop |230 SDMA_PKT_NOP_HEADER_COUNT(count - 1));231 else232 amdgpu_ring_write(ring, ring->funcs->nop);233}234 235/**236 * sdma_v2_4_ring_emit_ib - Schedule an IB on the DMA engine237 *238 * @ring: amdgpu ring pointer239 * @job: job to retrieve vmid from240 * @ib: IB object to schedule241 * @flags: unused242 *243 * Schedule an IB in the DMA ring (VI).244 */245static void sdma_v2_4_ring_emit_ib(struct amdgpu_ring *ring,246 struct amdgpu_job *job,247 struct amdgpu_ib *ib,248 uint32_t flags)249{250 unsigned vmid = AMDGPU_JOB_GET_VMID(job);251 252 /* IB packet must end on a 8 DW boundary */253 sdma_v2_4_ring_insert_nop(ring, (2 - lower_32_bits(ring->wptr)) & 7);254 255 amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_INDIRECT) |256 SDMA_PKT_INDIRECT_HEADER_VMID(vmid & 0xf));257 /* base must be 32 byte aligned */258 amdgpu_ring_write(ring, lower_32_bits(ib->gpu_addr) & 0xffffffe0);259 amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr));260 amdgpu_ring_write(ring, ib->length_dw);261 amdgpu_ring_write(ring, 0);262 amdgpu_ring_write(ring, 0);263 264}265 266/**267 * sdma_v2_4_ring_emit_hdp_flush - emit an hdp flush on the DMA ring268 *269 * @ring: amdgpu ring pointer270 *271 * Emit an hdp flush packet on the requested DMA ring.272 */273static void sdma_v2_4_ring_emit_hdp_flush(struct amdgpu_ring *ring)274{275 u32 ref_and_mask = 0;276 277 if (ring->me == 0)278 ref_and_mask = REG_SET_FIELD(ref_and_mask, GPU_HDP_FLUSH_DONE, SDMA0, 1);279 else280 ref_and_mask = REG_SET_FIELD(ref_and_mask, GPU_HDP_FLUSH_DONE, SDMA1, 1);281 282 amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_POLL_REGMEM) |283 SDMA_PKT_POLL_REGMEM_HEADER_HDP_FLUSH(1) |284 SDMA_PKT_POLL_REGMEM_HEADER_FUNC(3)); /* == */285 amdgpu_ring_write(ring, mmGPU_HDP_FLUSH_DONE << 2);286 amdgpu_ring_write(ring, mmGPU_HDP_FLUSH_REQ << 2);287 amdgpu_ring_write(ring, ref_and_mask); /* reference */288 amdgpu_ring_write(ring, ref_and_mask); /* mask */289 amdgpu_ring_write(ring, SDMA_PKT_POLL_REGMEM_DW5_RETRY_COUNT(0xfff) |290 SDMA_PKT_POLL_REGMEM_DW5_INTERVAL(10)); /* retry count, poll interval */291}292 293/**294 * sdma_v2_4_ring_emit_fence - emit a fence on the DMA ring295 *296 * @ring: amdgpu ring pointer297 * @addr: address298 * @seq: sequence number299 * @flags: fence related flags300 *301 * Add a DMA fence packet to the ring to write302 * the fence seq number and DMA trap packet to generate303 * an interrupt if needed (VI).304 */305static void sdma_v2_4_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, u64 seq,306 unsigned flags)307{308 bool write64bit = flags & AMDGPU_FENCE_FLAG_64BIT;309 /* write the fence */310 amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_FENCE));311 amdgpu_ring_write(ring, lower_32_bits(addr));312 amdgpu_ring_write(ring, upper_32_bits(addr));313 amdgpu_ring_write(ring, lower_32_bits(seq));314 315 /* optionally write high bits as well */316 if (write64bit) {317 addr += 4;318 amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_FENCE));319 amdgpu_ring_write(ring, lower_32_bits(addr));320 amdgpu_ring_write(ring, upper_32_bits(addr));321 amdgpu_ring_write(ring, upper_32_bits(seq));322 }323 324 /* generate an interrupt */325 amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_TRAP));326 amdgpu_ring_write(ring, SDMA_PKT_TRAP_INT_CONTEXT_INT_CONTEXT(0));327}328 329/**330 * sdma_v2_4_gfx_stop - stop the gfx async dma engines331 *332 * @adev: amdgpu_device pointer333 *334 * Stop the gfx async dma ring buffers (VI).335 */336static void sdma_v2_4_gfx_stop(struct amdgpu_device *adev)337{338 u32 rb_cntl, ib_cntl;339 int i;340 341 for (i = 0; i < adev->sdma.num_instances; i++) {342 rb_cntl = RREG32(mmSDMA0_GFX_RB_CNTL + sdma_offsets[i]);343 rb_cntl = REG_SET_FIELD(rb_cntl, SDMA0_GFX_RB_CNTL, RB_ENABLE, 0);344 WREG32(mmSDMA0_GFX_RB_CNTL + sdma_offsets[i], rb_cntl);345 ib_cntl = RREG32(mmSDMA0_GFX_IB_CNTL + sdma_offsets[i]);346 ib_cntl = REG_SET_FIELD(ib_cntl, SDMA0_GFX_IB_CNTL, IB_ENABLE, 0);347 WREG32(mmSDMA0_GFX_IB_CNTL + sdma_offsets[i], ib_cntl);348 }349}350 351/**352 * sdma_v2_4_rlc_stop - stop the compute async dma engines353 *354 * @adev: amdgpu_device pointer355 *356 * Stop the compute async dma queues (VI).357 */358static void sdma_v2_4_rlc_stop(struct amdgpu_device *adev)359{360 /* XXX todo */361}362 363/**364 * sdma_v2_4_enable - stop the async dma engines365 *366 * @adev: amdgpu_device pointer367 * @enable: enable/disable the DMA MEs.368 *369 * Halt or unhalt the async dma engines (VI).370 */371static void sdma_v2_4_enable(struct amdgpu_device *adev, bool enable)372{373 u32 f32_cntl;374 int i;375 376 if (!enable) {377 sdma_v2_4_gfx_stop(adev);378 sdma_v2_4_rlc_stop(adev);379 }380 381 for (i = 0; i < adev->sdma.num_instances; i++) {382 f32_cntl = RREG32(mmSDMA0_F32_CNTL + sdma_offsets[i]);383 if (enable)384 f32_cntl = REG_SET_FIELD(f32_cntl, SDMA0_F32_CNTL, HALT, 0);385 else386 f32_cntl = REG_SET_FIELD(f32_cntl, SDMA0_F32_CNTL, HALT, 1);387 WREG32(mmSDMA0_F32_CNTL + sdma_offsets[i], f32_cntl);388 }389}390 391/**392 * sdma_v2_4_gfx_resume - setup and start the async dma engines393 *394 * @adev: amdgpu_device pointer395 *396 * Set up the gfx DMA ring buffers and enable them (VI).397 * Returns 0 for success, error for failure.398 */399static int sdma_v2_4_gfx_resume(struct amdgpu_device *adev)400{401 struct amdgpu_ring *ring;402 u32 rb_cntl, ib_cntl;403 u32 rb_bufsz;404 int i, j, r;405 406 for (i = 0; i < adev->sdma.num_instances; i++) {407 ring = &adev->sdma.instance[i].ring;408 409 mutex_lock(&adev->srbm_mutex);410 for (j = 0; j < 16; j++) {411 vi_srbm_select(adev, 0, 0, 0, j);412 /* SDMA GFX */413 WREG32(mmSDMA0_GFX_VIRTUAL_ADDR + sdma_offsets[i], 0);414 WREG32(mmSDMA0_GFX_APE1_CNTL + sdma_offsets[i], 0);415 }416 vi_srbm_select(adev, 0, 0, 0, 0);417 mutex_unlock(&adev->srbm_mutex);418 419 WREG32(mmSDMA0_TILING_CONFIG + sdma_offsets[i],420 adev->gfx.config.gb_addr_config & 0x70);421 422 WREG32(mmSDMA0_SEM_WAIT_FAIL_TIMER_CNTL + sdma_offsets[i], 0);423 424 /* Set ring buffer size in dwords */425 rb_bufsz = order_base_2(ring->ring_size / 4);426 rb_cntl = RREG32(mmSDMA0_GFX_RB_CNTL + sdma_offsets[i]);427 rb_cntl = REG_SET_FIELD(rb_cntl, SDMA0_GFX_RB_CNTL, RB_SIZE, rb_bufsz);428#ifdef __BIG_ENDIAN429 rb_cntl = REG_SET_FIELD(rb_cntl, SDMA0_GFX_RB_CNTL, RB_SWAP_ENABLE, 1);430 rb_cntl = REG_SET_FIELD(rb_cntl, SDMA0_GFX_RB_CNTL,431 RPTR_WRITEBACK_SWAP_ENABLE, 1);432#endif433 WREG32(mmSDMA0_GFX_RB_CNTL + sdma_offsets[i], rb_cntl);434 435 /* Initialize the ring buffer's read and write pointers */436 WREG32(mmSDMA0_GFX_RB_RPTR + sdma_offsets[i], 0);437 WREG32(mmSDMA0_GFX_RB_WPTR + sdma_offsets[i], 0);438 WREG32(mmSDMA0_GFX_IB_RPTR + sdma_offsets[i], 0);439 WREG32(mmSDMA0_GFX_IB_OFFSET + sdma_offsets[i], 0);440 441 /* set the wb address whether it's enabled or not */442 WREG32(mmSDMA0_GFX_RB_RPTR_ADDR_HI + sdma_offsets[i],443 upper_32_bits(ring->rptr_gpu_addr) & 0xFFFFFFFF);444 WREG32(mmSDMA0_GFX_RB_RPTR_ADDR_LO + sdma_offsets[i],445 lower_32_bits(ring->rptr_gpu_addr) & 0xFFFFFFFC);446 447 rb_cntl = REG_SET_FIELD(rb_cntl, SDMA0_GFX_RB_CNTL, RPTR_WRITEBACK_ENABLE, 1);448 449 WREG32(mmSDMA0_GFX_RB_BASE + sdma_offsets[i], ring->gpu_addr >> 8);450 WREG32(mmSDMA0_GFX_RB_BASE_HI + sdma_offsets[i], ring->gpu_addr >> 40);451 452 ring->wptr = 0;453 WREG32(mmSDMA0_GFX_RB_WPTR + sdma_offsets[i], ring->wptr << 2);454 455 /* enable DMA RB */456 rb_cntl = REG_SET_FIELD(rb_cntl, SDMA0_GFX_RB_CNTL, RB_ENABLE, 1);457 WREG32(mmSDMA0_GFX_RB_CNTL + sdma_offsets[i], rb_cntl);458 459 ib_cntl = RREG32(mmSDMA0_GFX_IB_CNTL + sdma_offsets[i]);460 ib_cntl = REG_SET_FIELD(ib_cntl, SDMA0_GFX_IB_CNTL, IB_ENABLE, 1);461#ifdef __BIG_ENDIAN462 ib_cntl = REG_SET_FIELD(ib_cntl, SDMA0_GFX_IB_CNTL, IB_SWAP_ENABLE, 1);463#endif464 /* enable DMA IBs */465 WREG32(mmSDMA0_GFX_IB_CNTL + sdma_offsets[i], ib_cntl);466 }467 468 sdma_v2_4_enable(adev, true);469 for (i = 0; i < adev->sdma.num_instances; i++) {470 ring = &adev->sdma.instance[i].ring;471 r = amdgpu_ring_test_helper(ring);472 if (r)473 return r;474 }475 476 return 0;477}478 479/**480 * sdma_v2_4_rlc_resume - setup and start the async dma engines481 *482 * @adev: amdgpu_device pointer483 *484 * Set up the compute DMA queues and enable them (VI).485 * Returns 0 for success, error for failure.486 */487static int sdma_v2_4_rlc_resume(struct amdgpu_device *adev)488{489 /* XXX todo */490 return 0;491}492 493 494/**495 * sdma_v2_4_start - setup and start the async dma engines496 *497 * @adev: amdgpu_device pointer498 *499 * Set up the DMA engines and enable them (VI).500 * Returns 0 for success, error for failure.501 */502static int sdma_v2_4_start(struct amdgpu_device *adev)503{504 int r;505 506 /* halt the engine before programing */507 sdma_v2_4_enable(adev, false);508 509 /* start the gfx rings and rlc compute queues */510 r = sdma_v2_4_gfx_resume(adev);511 if (r)512 return r;513 r = sdma_v2_4_rlc_resume(adev);514 if (r)515 return r;516 517 return 0;518}519 520/**521 * sdma_v2_4_ring_test_ring - simple async dma engine test522 *523 * @ring: amdgpu_ring structure holding ring information524 *525 * Test the DMA engine by writing using it to write an526 * value to memory. (VI).527 * Returns 0 for success, error for failure.528 */529static int sdma_v2_4_ring_test_ring(struct amdgpu_ring *ring)530{531 struct amdgpu_device *adev = ring->adev;532 unsigned i;533 unsigned index;534 int r;535 u32 tmp;536 u64 gpu_addr;537 538 r = amdgpu_device_wb_get(adev, &index);539 if (r)540 return r;541 542 gpu_addr = adev->wb.gpu_addr + (index * 4);543 tmp = 0xCAFEDEAD;544 adev->wb.wb[index] = cpu_to_le32(tmp);545 546 r = amdgpu_ring_alloc(ring, 5);547 if (r)548 goto error_free_wb;549 550 amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_WRITE) |551 SDMA_PKT_HEADER_SUB_OP(SDMA_SUBOP_WRITE_LINEAR));552 amdgpu_ring_write(ring, lower_32_bits(gpu_addr));553 amdgpu_ring_write(ring, upper_32_bits(gpu_addr));554 amdgpu_ring_write(ring, SDMA_PKT_WRITE_UNTILED_DW_3_COUNT(1));555 amdgpu_ring_write(ring, 0xDEADBEEF);556 amdgpu_ring_commit(ring);557 558 for (i = 0; i < adev->usec_timeout; i++) {559 tmp = le32_to_cpu(adev->wb.wb[index]);560 if (tmp == 0xDEADBEEF)561 break;562 udelay(1);563 }564 565 if (i >= adev->usec_timeout)566 r = -ETIMEDOUT;567 568error_free_wb:569 amdgpu_device_wb_free(adev, index);570 return r;571}572 573/**574 * sdma_v2_4_ring_test_ib - test an IB on the DMA engine575 *576 * @ring: amdgpu_ring structure holding ring information577 * @timeout: timeout value in jiffies, or MAX_SCHEDULE_TIMEOUT578 *579 * Test a simple IB in the DMA ring (VI).580 * Returns 0 on success, error on failure.581 */582static int sdma_v2_4_ring_test_ib(struct amdgpu_ring *ring, long timeout)583{584 struct amdgpu_device *adev = ring->adev;585 struct amdgpu_ib ib;586 struct dma_fence *f = NULL;587 unsigned index;588 u32 tmp = 0;589 u64 gpu_addr;590 long r;591 592 r = amdgpu_device_wb_get(adev, &index);593 if (r)594 return r;595 596 gpu_addr = adev->wb.gpu_addr + (index * 4);597 tmp = 0xCAFEDEAD;598 adev->wb.wb[index] = cpu_to_le32(tmp);599 memset(&ib, 0, sizeof(ib));600 r = amdgpu_ib_get(adev, NULL, 256,601 AMDGPU_IB_POOL_DIRECT, &ib);602 if (r)603 goto err0;604 605 ib.ptr[0] = SDMA_PKT_HEADER_OP(SDMA_OP_WRITE) |606 SDMA_PKT_HEADER_SUB_OP(SDMA_SUBOP_WRITE_LINEAR);607 ib.ptr[1] = lower_32_bits(gpu_addr);608 ib.ptr[2] = upper_32_bits(gpu_addr);609 ib.ptr[3] = SDMA_PKT_WRITE_UNTILED_DW_3_COUNT(1);610 ib.ptr[4] = 0xDEADBEEF;611 ib.ptr[5] = SDMA_PKT_HEADER_OP(SDMA_OP_NOP);612 ib.ptr[6] = SDMA_PKT_HEADER_OP(SDMA_OP_NOP);613 ib.ptr[7] = SDMA_PKT_HEADER_OP(SDMA_OP_NOP);614 ib.length_dw = 8;615 616 r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f);617 if (r)618 goto err1;619 620 r = dma_fence_wait_timeout(f, false, timeout);621 if (r == 0) {622 r = -ETIMEDOUT;623 goto err1;624 } else if (r < 0) {625 goto err1;626 }627 tmp = le32_to_cpu(adev->wb.wb[index]);628 if (tmp == 0xDEADBEEF)629 r = 0;630 else631 r = -EINVAL;632 633err1:634 amdgpu_ib_free(adev, &ib, NULL);635 dma_fence_put(f);636err0:637 amdgpu_device_wb_free(adev, index);638 return r;639}640 641/**642 * sdma_v2_4_vm_copy_pte - update PTEs by copying them from the GART643 *644 * @ib: indirect buffer to fill with commands645 * @pe: addr of the page entry646 * @src: src addr to copy from647 * @count: number of page entries to update648 *649 * Update PTEs by copying them from the GART using sDMA (CIK).650 */651static void sdma_v2_4_vm_copy_pte(struct amdgpu_ib *ib,652 uint64_t pe, uint64_t src,653 unsigned count)654{655 unsigned bytes = count * 8;656 657 ib->ptr[ib->length_dw++] = SDMA_PKT_HEADER_OP(SDMA_OP_COPY) |658 SDMA_PKT_HEADER_SUB_OP(SDMA_SUBOP_COPY_LINEAR);659 ib->ptr[ib->length_dw++] = bytes;660 ib->ptr[ib->length_dw++] = 0; /* src/dst endian swap */661 ib->ptr[ib->length_dw++] = lower_32_bits(src);662 ib->ptr[ib->length_dw++] = upper_32_bits(src);663 ib->ptr[ib->length_dw++] = lower_32_bits(pe);664 ib->ptr[ib->length_dw++] = upper_32_bits(pe);665}666 667/**668 * sdma_v2_4_vm_write_pte - update PTEs by writing them manually669 *670 * @ib: indirect buffer to fill with commands671 * @pe: addr of the page entry672 * @value: dst addr to write into pe673 * @count: number of page entries to update674 * @incr: increase next addr by incr bytes675 *676 * Update PTEs by writing them manually using sDMA (CIK).677 */678static void sdma_v2_4_vm_write_pte(struct amdgpu_ib *ib, uint64_t pe,679 uint64_t value, unsigned count,680 uint32_t incr)681{682 unsigned ndw = count * 2;683 684 ib->ptr[ib->length_dw++] = SDMA_PKT_HEADER_OP(SDMA_OP_WRITE) |685 SDMA_PKT_HEADER_SUB_OP(SDMA_SUBOP_WRITE_LINEAR);686 ib->ptr[ib->length_dw++] = pe;687 ib->ptr[ib->length_dw++] = upper_32_bits(pe);688 ib->ptr[ib->length_dw++] = ndw;689 for (; ndw > 0; ndw -= 2) {690 ib->ptr[ib->length_dw++] = lower_32_bits(value);691 ib->ptr[ib->length_dw++] = upper_32_bits(value);692 value += incr;693 }694}695 696/**697 * sdma_v2_4_vm_set_pte_pde - update the page tables using sDMA698 *699 * @ib: indirect buffer to fill with commands700 * @pe: addr of the page entry701 * @addr: dst addr to write into pe702 * @count: number of page entries to update703 * @incr: increase next addr by incr bytes704 * @flags: access flags705 *706 * Update the page tables using sDMA (CIK).707 */708static void sdma_v2_4_vm_set_pte_pde(struct amdgpu_ib *ib, uint64_t pe,709 uint64_t addr, unsigned count,710 uint32_t incr, uint64_t flags)711{712 /* for physically contiguous pages (vram) */713 ib->ptr[ib->length_dw++] = SDMA_PKT_HEADER_OP(SDMA_OP_GEN_PTEPDE);714 ib->ptr[ib->length_dw++] = lower_32_bits(pe); /* dst addr */715 ib->ptr[ib->length_dw++] = upper_32_bits(pe);716 ib->ptr[ib->length_dw++] = lower_32_bits(flags); /* mask */717 ib->ptr[ib->length_dw++] = upper_32_bits(flags);718 ib->ptr[ib->length_dw++] = lower_32_bits(addr); /* value */719 ib->ptr[ib->length_dw++] = upper_32_bits(addr);720 ib->ptr[ib->length_dw++] = incr; /* increment size */721 ib->ptr[ib->length_dw++] = 0;722 ib->ptr[ib->length_dw++] = count; /* number of entries */723}724 725/**726 * sdma_v2_4_ring_pad_ib - pad the IB to the required number of dw727 *728 * @ring: amdgpu_ring structure holding ring information729 * @ib: indirect buffer to fill with padding730 *731 */732static void sdma_v2_4_ring_pad_ib(struct amdgpu_ring *ring, struct amdgpu_ib *ib)733{734 struct amdgpu_sdma_instance *sdma = amdgpu_sdma_get_instance_from_ring(ring);735 u32 pad_count;736 int i;737 738 pad_count = (-ib->length_dw) & 7;739 for (i = 0; i < pad_count; i++)740 if (sdma && sdma->burst_nop && (i == 0))741 ib->ptr[ib->length_dw++] =742 SDMA_PKT_HEADER_OP(SDMA_OP_NOP) |743 SDMA_PKT_NOP_HEADER_COUNT(pad_count - 1);744 else745 ib->ptr[ib->length_dw++] =746 SDMA_PKT_HEADER_OP(SDMA_OP_NOP);747}748 749/**750 * sdma_v2_4_ring_emit_pipeline_sync - sync the pipeline751 *752 * @ring: amdgpu_ring pointer753 *754 * Make sure all previous operations are completed (CIK).755 */756static void sdma_v2_4_ring_emit_pipeline_sync(struct amdgpu_ring *ring)757{758 uint32_t seq = ring->fence_drv.sync_seq;759 uint64_t addr = ring->fence_drv.gpu_addr;760 761 /* wait for idle */762 amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_POLL_REGMEM) |763 SDMA_PKT_POLL_REGMEM_HEADER_HDP_FLUSH(0) |764 SDMA_PKT_POLL_REGMEM_HEADER_FUNC(3) | /* equal */765 SDMA_PKT_POLL_REGMEM_HEADER_MEM_POLL(1));766 amdgpu_ring_write(ring, addr & 0xfffffffc);767 amdgpu_ring_write(ring, upper_32_bits(addr) & 0xffffffff);768 amdgpu_ring_write(ring, seq); /* reference */769 amdgpu_ring_write(ring, 0xffffffff); /* mask */770 amdgpu_ring_write(ring, SDMA_PKT_POLL_REGMEM_DW5_RETRY_COUNT(0xfff) |771 SDMA_PKT_POLL_REGMEM_DW5_INTERVAL(4)); /* retry count, poll interval */772}773 774/**775 * sdma_v2_4_ring_emit_vm_flush - cik vm flush using sDMA776 *777 * @ring: amdgpu_ring pointer778 * @vmid: vmid number to use779 * @pd_addr: address780 *781 * Update the page table base and flush the VM TLB782 * using sDMA (VI).783 */784static void sdma_v2_4_ring_emit_vm_flush(struct amdgpu_ring *ring,785 unsigned vmid, uint64_t pd_addr)786{787 amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr);788 789 /* wait for flush */790 amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_POLL_REGMEM) |791 SDMA_PKT_POLL_REGMEM_HEADER_HDP_FLUSH(0) |792 SDMA_PKT_POLL_REGMEM_HEADER_FUNC(0)); /* always */793 amdgpu_ring_write(ring, mmVM_INVALIDATE_REQUEST << 2);794 amdgpu_ring_write(ring, 0);795 amdgpu_ring_write(ring, 0); /* reference */796 amdgpu_ring_write(ring, 0); /* mask */797 amdgpu_ring_write(ring, SDMA_PKT_POLL_REGMEM_DW5_RETRY_COUNT(0xfff) |798 SDMA_PKT_POLL_REGMEM_DW5_INTERVAL(10)); /* retry count, poll interval */799}800 801static void sdma_v2_4_ring_emit_wreg(struct amdgpu_ring *ring,802 uint32_t reg, uint32_t val)803{804 amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_SRBM_WRITE) |805 SDMA_PKT_SRBM_WRITE_HEADER_BYTE_EN(0xf));806 amdgpu_ring_write(ring, reg);807 amdgpu_ring_write(ring, val);808}809 810static int sdma_v2_4_early_init(void *handle)811{812 struct amdgpu_device *adev = (struct amdgpu_device *)handle;813 int r;814 815 adev->sdma.num_instances = SDMA_MAX_INSTANCE;816 817 r = sdma_v2_4_init_microcode(adev);818 if (r)819 return r;820 821 sdma_v2_4_set_ring_funcs(adev);822 sdma_v2_4_set_buffer_funcs(adev);823 sdma_v2_4_set_vm_pte_funcs(adev);824 sdma_v2_4_set_irq_funcs(adev);825 826 return 0;827}828 829static int sdma_v2_4_sw_init(void *handle)830{831 struct amdgpu_ring *ring;832 int r, i;833 struct amdgpu_device *adev = (struct amdgpu_device *)handle;834 835 /* SDMA trap event */836 r = amdgpu_irq_add_id(adev, AMDGPU_IRQ_CLIENTID_LEGACY, VISLANDS30_IV_SRCID_SDMA_TRAP,837 &adev->sdma.trap_irq);838 if (r)839 return r;840 841 /* SDMA Privileged inst */842 r = amdgpu_irq_add_id(adev, AMDGPU_IRQ_CLIENTID_LEGACY, 241,843 &adev->sdma.illegal_inst_irq);844 if (r)845 return r;846 847 /* SDMA Privileged inst */848 r = amdgpu_irq_add_id(adev, AMDGPU_IRQ_CLIENTID_LEGACY, VISLANDS30_IV_SRCID_SDMA_SRBM_WRITE,849 &adev->sdma.illegal_inst_irq);850 if (r)851 return r;852 853 for (i = 0; i < adev->sdma.num_instances; i++) {854 ring = &adev->sdma.instance[i].ring;855 ring->ring_obj = NULL;856 ring->use_doorbell = false;857 sprintf(ring->name, "sdma%d", i);858 r = amdgpu_ring_init(adev, ring, 1024, &adev->sdma.trap_irq,859 (i == 0) ? AMDGPU_SDMA_IRQ_INSTANCE0 :860 AMDGPU_SDMA_IRQ_INSTANCE1,861 AMDGPU_RING_PRIO_DEFAULT, NULL);862 if (r)863 return r;864 }865 866 return r;867}868 869static int sdma_v2_4_sw_fini(void *handle)870{871 struct amdgpu_device *adev = (struct amdgpu_device *)handle;872 int i;873 874 for (i = 0; i < adev->sdma.num_instances; i++)875 amdgpu_ring_fini(&adev->sdma.instance[i].ring);876 877 sdma_v2_4_free_microcode(adev);878 return 0;879}880 881static int sdma_v2_4_hw_init(void *handle)882{883 int r;884 struct amdgpu_device *adev = (struct amdgpu_device *)handle;885 886 sdma_v2_4_init_golden_registers(adev);887 888 r = sdma_v2_4_start(adev);889 if (r)890 return r;891 892 return r;893}894 895static int sdma_v2_4_hw_fini(void *handle)896{897 struct amdgpu_device *adev = (struct amdgpu_device *)handle;898 899 sdma_v2_4_enable(adev, false);900 901 return 0;902}903 904static int sdma_v2_4_suspend(void *handle)905{906 struct amdgpu_device *adev = (struct amdgpu_device *)handle;907 908 return sdma_v2_4_hw_fini(adev);909}910 911static int sdma_v2_4_resume(void *handle)912{913 struct amdgpu_device *adev = (struct amdgpu_device *)handle;914 915 return sdma_v2_4_hw_init(adev);916}917 918static bool sdma_v2_4_is_idle(void *handle)919{920 struct amdgpu_device *adev = (struct amdgpu_device *)handle;921 u32 tmp = RREG32(mmSRBM_STATUS2);922 923 if (tmp & (SRBM_STATUS2__SDMA_BUSY_MASK |924 SRBM_STATUS2__SDMA1_BUSY_MASK))925 return false;926 927 return true;928}929 930static int sdma_v2_4_wait_for_idle(void *handle)931{932 unsigned i;933 u32 tmp;934 struct amdgpu_device *adev = (struct amdgpu_device *)handle;935 936 for (i = 0; i < adev->usec_timeout; i++) {937 tmp = RREG32(mmSRBM_STATUS2) & (SRBM_STATUS2__SDMA_BUSY_MASK |938 SRBM_STATUS2__SDMA1_BUSY_MASK);939 940 if (!tmp)941 return 0;942 udelay(1);943 }944 return -ETIMEDOUT;945}946 947static int sdma_v2_4_soft_reset(void *handle)948{949 u32 srbm_soft_reset = 0;950 struct amdgpu_device *adev = (struct amdgpu_device *)handle;951 u32 tmp = RREG32(mmSRBM_STATUS2);952 953 if (tmp & SRBM_STATUS2__SDMA_BUSY_MASK) {954 /* sdma0 */955 tmp = RREG32(mmSDMA0_F32_CNTL + SDMA0_REGISTER_OFFSET);956 tmp = REG_SET_FIELD(tmp, SDMA0_F32_CNTL, HALT, 0);957 WREG32(mmSDMA0_F32_CNTL + SDMA0_REGISTER_OFFSET, tmp);958 srbm_soft_reset |= SRBM_SOFT_RESET__SOFT_RESET_SDMA_MASK;959 }960 if (tmp & SRBM_STATUS2__SDMA1_BUSY_MASK) {961 /* sdma1 */962 tmp = RREG32(mmSDMA0_F32_CNTL + SDMA1_REGISTER_OFFSET);963 tmp = REG_SET_FIELD(tmp, SDMA0_F32_CNTL, HALT, 0);964 WREG32(mmSDMA0_F32_CNTL + SDMA1_REGISTER_OFFSET, tmp);965 srbm_soft_reset |= SRBM_SOFT_RESET__SOFT_RESET_SDMA1_MASK;966 }967 968 if (srbm_soft_reset) {969 tmp = RREG32(mmSRBM_SOFT_RESET);970 tmp |= srbm_soft_reset;971 dev_info(adev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);972 WREG32(mmSRBM_SOFT_RESET, tmp);973 tmp = RREG32(mmSRBM_SOFT_RESET);974 975 udelay(50);976 977 tmp &= ~srbm_soft_reset;978 WREG32(mmSRBM_SOFT_RESET, tmp);979 tmp = RREG32(mmSRBM_SOFT_RESET);980 981 /* Wait a little for things to settle down */982 udelay(50);983 }984 985 return 0;986}987 988static int sdma_v2_4_set_trap_irq_state(struct amdgpu_device *adev,989 struct amdgpu_irq_src *src,990 unsigned type,991 enum amdgpu_interrupt_state state)992{993 u32 sdma_cntl;994 995 switch (type) {996 case AMDGPU_SDMA_IRQ_INSTANCE0:997 switch (state) {998 case AMDGPU_IRQ_STATE_DISABLE:999 sdma_cntl = RREG32(mmSDMA0_CNTL + SDMA0_REGISTER_OFFSET);1000 sdma_cntl = REG_SET_FIELD(sdma_cntl, SDMA0_CNTL, TRAP_ENABLE, 0);1001 WREG32(mmSDMA0_CNTL + SDMA0_REGISTER_OFFSET, sdma_cntl);1002 break;1003 case AMDGPU_IRQ_STATE_ENABLE:1004 sdma_cntl = RREG32(mmSDMA0_CNTL + SDMA0_REGISTER_OFFSET);1005 sdma_cntl = REG_SET_FIELD(sdma_cntl, SDMA0_CNTL, TRAP_ENABLE, 1);1006 WREG32(mmSDMA0_CNTL + SDMA0_REGISTER_OFFSET, sdma_cntl);1007 break;1008 default:1009 break;1010 }1011 break;1012 case AMDGPU_SDMA_IRQ_INSTANCE1:1013 switch (state) {1014 case AMDGPU_IRQ_STATE_DISABLE:1015 sdma_cntl = RREG32(mmSDMA0_CNTL + SDMA1_REGISTER_OFFSET);1016 sdma_cntl = REG_SET_FIELD(sdma_cntl, SDMA0_CNTL, TRAP_ENABLE, 0);1017 WREG32(mmSDMA0_CNTL + SDMA1_REGISTER_OFFSET, sdma_cntl);1018 break;1019 case AMDGPU_IRQ_STATE_ENABLE:1020 sdma_cntl = RREG32(mmSDMA0_CNTL + SDMA1_REGISTER_OFFSET);1021 sdma_cntl = REG_SET_FIELD(sdma_cntl, SDMA0_CNTL, TRAP_ENABLE, 1);1022 WREG32(mmSDMA0_CNTL + SDMA1_REGISTER_OFFSET, sdma_cntl);1023 break;1024 default:1025 break;1026 }1027 break;1028 default:1029 break;1030 }1031 return 0;1032}1033 1034static int sdma_v2_4_process_trap_irq(struct amdgpu_device *adev,1035 struct amdgpu_irq_src *source,1036 struct amdgpu_iv_entry *entry)1037{1038 u8 instance_id, queue_id;1039 1040 instance_id = (entry->ring_id & 0x3) >> 0;1041 queue_id = (entry->ring_id & 0xc) >> 2;1042 DRM_DEBUG("IH: SDMA trap\n");1043 switch (instance_id) {1044 case 0:1045 switch (queue_id) {1046 case 0:1047 amdgpu_fence_process(&adev->sdma.instance[0].ring);1048 break;1049 case 1:1050 /* XXX compute */1051 break;1052 case 2:1053 /* XXX compute */1054 break;1055 }1056 break;1057 case 1:1058 switch (queue_id) {1059 case 0:1060 amdgpu_fence_process(&adev->sdma.instance[1].ring);1061 break;1062 case 1:1063 /* XXX compute */1064 break;1065 case 2:1066 /* XXX compute */1067 break;1068 }1069 break;1070 }1071 return 0;1072}1073 1074static int sdma_v2_4_process_illegal_inst_irq(struct amdgpu_device *adev,1075 struct amdgpu_irq_src *source,1076 struct amdgpu_iv_entry *entry)1077{1078 u8 instance_id, queue_id;1079 1080 DRM_ERROR("Illegal instruction in SDMA command stream\n");1081 instance_id = (entry->ring_id & 0x3) >> 0;1082 queue_id = (entry->ring_id & 0xc) >> 2;1083 1084 if (instance_id <= 1 && queue_id == 0)1085 drm_sched_fault(&adev->sdma.instance[instance_id].ring.sched);1086 return 0;1087}1088 1089static int sdma_v2_4_set_clockgating_state(void *handle,1090 enum amd_clockgating_state state)1091{1092 /* XXX handled via the smc on VI */1093 return 0;1094}1095 1096static int sdma_v2_4_set_powergating_state(void *handle,1097 enum amd_powergating_state state)1098{1099 return 0;1100}1101 1102static const struct amd_ip_funcs sdma_v2_4_ip_funcs = {1103 .name = "sdma_v2_4",1104 .early_init = sdma_v2_4_early_init,1105 .late_init = NULL,1106 .sw_init = sdma_v2_4_sw_init,1107 .sw_fini = sdma_v2_4_sw_fini,1108 .hw_init = sdma_v2_4_hw_init,1109 .hw_fini = sdma_v2_4_hw_fini,1110 .suspend = sdma_v2_4_suspend,1111 .resume = sdma_v2_4_resume,1112 .is_idle = sdma_v2_4_is_idle,1113 .wait_for_idle = sdma_v2_4_wait_for_idle,1114 .soft_reset = sdma_v2_4_soft_reset,1115 .set_clockgating_state = sdma_v2_4_set_clockgating_state,1116 .set_powergating_state = sdma_v2_4_set_powergating_state,1117 .dump_ip_state = NULL,1118 .print_ip_state = NULL,1119};1120 1121static const struct amdgpu_ring_funcs sdma_v2_4_ring_funcs = {1122 .type = AMDGPU_RING_TYPE_SDMA,1123 .align_mask = 0xf,1124 .nop = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP),1125 .support_64bit_ptrs = false,1126 .secure_submission_supported = true,1127 .get_rptr = sdma_v2_4_ring_get_rptr,1128 .get_wptr = sdma_v2_4_ring_get_wptr,1129 .set_wptr = sdma_v2_4_ring_set_wptr,1130 .emit_frame_size =1131 6 + /* sdma_v2_4_ring_emit_hdp_flush */1132 3 + /* hdp invalidate */1133 6 + /* sdma_v2_4_ring_emit_pipeline_sync */1134 VI_FLUSH_GPU_TLB_NUM_WREG * 3 + 6 + /* sdma_v2_4_ring_emit_vm_flush */1135 10 + 10 + 10, /* sdma_v2_4_ring_emit_fence x3 for user fence, vm fence */1136 .emit_ib_size = 7 + 6, /* sdma_v2_4_ring_emit_ib */1137 .emit_ib = sdma_v2_4_ring_emit_ib,1138 .emit_fence = sdma_v2_4_ring_emit_fence,1139 .emit_pipeline_sync = sdma_v2_4_ring_emit_pipeline_sync,1140 .emit_vm_flush = sdma_v2_4_ring_emit_vm_flush,1141 .emit_hdp_flush = sdma_v2_4_ring_emit_hdp_flush,1142 .test_ring = sdma_v2_4_ring_test_ring,1143 .test_ib = sdma_v2_4_ring_test_ib,1144 .insert_nop = sdma_v2_4_ring_insert_nop,1145 .pad_ib = sdma_v2_4_ring_pad_ib,1146 .emit_wreg = sdma_v2_4_ring_emit_wreg,1147};1148 1149static void sdma_v2_4_set_ring_funcs(struct amdgpu_device *adev)1150{1151 int i;1152 1153 for (i = 0; i < adev->sdma.num_instances; i++) {1154 adev->sdma.instance[i].ring.funcs = &sdma_v2_4_ring_funcs;1155 adev->sdma.instance[i].ring.me = i;1156 }1157}1158 1159static const struct amdgpu_irq_src_funcs sdma_v2_4_trap_irq_funcs = {1160 .set = sdma_v2_4_set_trap_irq_state,1161 .process = sdma_v2_4_process_trap_irq,1162};1163 1164static const struct amdgpu_irq_src_funcs sdma_v2_4_illegal_inst_irq_funcs = {1165 .process = sdma_v2_4_process_illegal_inst_irq,1166};1167 1168static void sdma_v2_4_set_irq_funcs(struct amdgpu_device *adev)1169{1170 adev->sdma.trap_irq.num_types = AMDGPU_SDMA_IRQ_LAST;1171 adev->sdma.trap_irq.funcs = &sdma_v2_4_trap_irq_funcs;1172 adev->sdma.illegal_inst_irq.funcs = &sdma_v2_4_illegal_inst_irq_funcs;1173}1174 1175/**1176 * sdma_v2_4_emit_copy_buffer - copy buffer using the sDMA engine1177 *1178 * @ib: indirect buffer to copy to1179 * @src_offset: src GPU address1180 * @dst_offset: dst GPU address1181 * @byte_count: number of bytes to xfer1182 * @copy_flags: unused1183 *1184 * Copy GPU buffers using the DMA engine (VI).1185 * Used by the amdgpu ttm implementation to move pages if1186 * registered as the asic copy callback.1187 */1188static void sdma_v2_4_emit_copy_buffer(struct amdgpu_ib *ib,1189 uint64_t src_offset,1190 uint64_t dst_offset,1191 uint32_t byte_count,1192 uint32_t copy_flags)1193{1194 ib->ptr[ib->length_dw++] = SDMA_PKT_HEADER_OP(SDMA_OP_COPY) |1195 SDMA_PKT_HEADER_SUB_OP(SDMA_SUBOP_COPY_LINEAR);1196 ib->ptr[ib->length_dw++] = byte_count;1197 ib->ptr[ib->length_dw++] = 0; /* src/dst endian swap */1198 ib->ptr[ib->length_dw++] = lower_32_bits(src_offset);1199 ib->ptr[ib->length_dw++] = upper_32_bits(src_offset);1200 ib->ptr[ib->length_dw++] = lower_32_bits(dst_offset);1201 ib->ptr[ib->length_dw++] = upper_32_bits(dst_offset);1202}1203 1204/**1205 * sdma_v2_4_emit_fill_buffer - fill buffer using the sDMA engine1206 *1207 * @ib: indirect buffer to copy to1208 * @src_data: value to write to buffer1209 * @dst_offset: dst GPU address1210 * @byte_count: number of bytes to xfer1211 *1212 * Fill GPU buffers using the DMA engine (VI).1213 */1214static void sdma_v2_4_emit_fill_buffer(struct amdgpu_ib *ib,1215 uint32_t src_data,1216 uint64_t dst_offset,1217 uint32_t byte_count)1218{1219 ib->ptr[ib->length_dw++] = SDMA_PKT_HEADER_OP(SDMA_OP_CONST_FILL);1220 ib->ptr[ib->length_dw++] = lower_32_bits(dst_offset);1221 ib->ptr[ib->length_dw++] = upper_32_bits(dst_offset);1222 ib->ptr[ib->length_dw++] = src_data;1223 ib->ptr[ib->length_dw++] = byte_count;1224}1225 1226static const struct amdgpu_buffer_funcs sdma_v2_4_buffer_funcs = {1227 .copy_max_bytes = 0x1fffff,1228 .copy_num_dw = 7,1229 .emit_copy_buffer = sdma_v2_4_emit_copy_buffer,1230 1231 .fill_max_bytes = 0x1fffff,1232 .fill_num_dw = 7,1233 .emit_fill_buffer = sdma_v2_4_emit_fill_buffer,1234};1235 1236static void sdma_v2_4_set_buffer_funcs(struct amdgpu_device *adev)1237{1238 adev->mman.buffer_funcs = &sdma_v2_4_buffer_funcs;1239 adev->mman.buffer_funcs_ring = &adev->sdma.instance[0].ring;1240}1241 1242static const struct amdgpu_vm_pte_funcs sdma_v2_4_vm_pte_funcs = {1243 .copy_pte_num_dw = 7,1244 .copy_pte = sdma_v2_4_vm_copy_pte,1245 1246 .write_pte = sdma_v2_4_vm_write_pte,1247 .set_pte_pde = sdma_v2_4_vm_set_pte_pde,1248};1249 1250static void sdma_v2_4_set_vm_pte_funcs(struct amdgpu_device *adev)1251{1252 unsigned i;1253 1254 adev->vm_manager.vm_pte_funcs = &sdma_v2_4_vm_pte_funcs;1255 for (i = 0; i < adev->sdma.num_instances; i++) {1256 adev->vm_manager.vm_pte_scheds[i] =1257 &adev->sdma.instance[i].ring.sched;1258 }1259 adev->vm_manager.vm_pte_num_scheds = adev->sdma.num_instances;1260}1261 1262const struct amdgpu_ip_block_version sdma_v2_4_ip_block = {1263 .type = AMD_IP_BLOCK_TYPE_SDMA,1264 .major = 2,1265 .minor = 4,1266 .rev = 0,1267 .funcs = &sdma_v2_4_ip_funcs,1268};1269