1383 lines · c
1/*2 * Copyright 2013 Advanced Micro Devices, Inc.3 *4 * Permission is hereby granted, free of charge, to any person obtaining a5 * copy of this software and associated documentation files (the "Software"),6 * to deal in the Software without restriction, including without limitation7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,8 * and/or sell copies of the Software, and to permit persons to whom the9 * Software is furnished to do so, subject to the following conditions:10 *11 * The above copyright notice and this permission notice shall be included in12 * all copies or substantial portions of the Software.13 *14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR20 * OTHER DEALINGS IN THE SOFTWARE.21 *22 * Authors: Alex Deucher23 */24 25#include <linux/firmware.h>26#include <linux/module.h>27 28#include "amdgpu.h"29#include "amdgpu_ucode.h"30#include "amdgpu_trace.h"31#include "cikd.h"32#include "cik.h"33 34#include "bif/bif_4_1_d.h"35#include "bif/bif_4_1_sh_mask.h"36 37#include "gca/gfx_7_2_d.h"38#include "gca/gfx_7_2_enum.h"39#include "gca/gfx_7_2_sh_mask.h"40 41#include "gmc/gmc_7_1_d.h"42#include "gmc/gmc_7_1_sh_mask.h"43 44#include "oss/oss_2_0_d.h"45#include "oss/oss_2_0_sh_mask.h"46 47static const u32 sdma_offsets[SDMA_MAX_INSTANCE] =48{49 SDMA0_REGISTER_OFFSET,50 SDMA1_REGISTER_OFFSET51};52 53static void cik_sdma_set_ring_funcs(struct amdgpu_device *adev);54static void cik_sdma_set_irq_funcs(struct amdgpu_device *adev);55static void cik_sdma_set_buffer_funcs(struct amdgpu_device *adev);56static void cik_sdma_set_vm_pte_funcs(struct amdgpu_device *adev);57static int cik_sdma_soft_reset(void *handle);58 59MODULE_FIRMWARE("amdgpu/bonaire_sdma.bin");60MODULE_FIRMWARE("amdgpu/bonaire_sdma1.bin");61MODULE_FIRMWARE("amdgpu/hawaii_sdma.bin");62MODULE_FIRMWARE("amdgpu/hawaii_sdma1.bin");63MODULE_FIRMWARE("amdgpu/kaveri_sdma.bin");64MODULE_FIRMWARE("amdgpu/kaveri_sdma1.bin");65MODULE_FIRMWARE("amdgpu/kabini_sdma.bin");66MODULE_FIRMWARE("amdgpu/kabini_sdma1.bin");67MODULE_FIRMWARE("amdgpu/mullins_sdma.bin");68MODULE_FIRMWARE("amdgpu/mullins_sdma1.bin");69 70u32 amdgpu_cik_gpu_check_soft_reset(struct amdgpu_device *adev);71 72 73static void cik_sdma_free_microcode(struct amdgpu_device *adev)74{75 int i;76 77 for (i = 0; i < adev->sdma.num_instances; i++)78 amdgpu_ucode_release(&adev->sdma.instance[i].fw);79}80 81/*82 * sDMA - System DMA83 * Starting with CIK, the GPU has new asynchronous84 * DMA engines. These engines are used for compute85 * and gfx. There are two DMA engines (SDMA0, SDMA1)86 * and each one supports 1 ring buffer used for gfx87 * and 2 queues used for compute.88 *89 * The programming model is very similar to the CP90 * (ring buffer, IBs, etc.), but sDMA has it's own91 * packet format that is different from the PM4 format92 * used by the CP. sDMA supports copying data, writing93 * embedded data, solid fills, and a number of other94 * things. It also has support for tiling/detiling of95 * buffers.96 */97 98/**99 * cik_sdma_init_microcode - load ucode images from disk100 *101 * @adev: amdgpu_device pointer102 *103 * Use the firmware interface to load the ucode images into104 * the driver (not loaded into hw).105 * Returns 0 on success, error on failure.106 */107static int cik_sdma_init_microcode(struct amdgpu_device *adev)108{109 const char *chip_name;110 int err = 0, i;111 112 DRM_DEBUG("\n");113 114 switch (adev->asic_type) {115 case CHIP_BONAIRE:116 chip_name = "bonaire";117 break;118 case CHIP_HAWAII:119 chip_name = "hawaii";120 break;121 case CHIP_KAVERI:122 chip_name = "kaveri";123 break;124 case CHIP_KABINI:125 chip_name = "kabini";126 break;127 case CHIP_MULLINS:128 chip_name = "mullins";129 break;130 default: BUG();131 }132 133 for (i = 0; i < adev->sdma.num_instances; i++) {134 if (i == 0)135 err = amdgpu_ucode_request(adev, &adev->sdma.instance[i].fw,136 "amdgpu/%s_sdma.bin", chip_name);137 else138 err = amdgpu_ucode_request(adev, &adev->sdma.instance[i].fw,139 "amdgpu/%s_sdma1.bin", chip_name);140 if (err)141 goto out;142 }143out:144 if (err) {145 pr_err("cik_sdma: Failed to load firmware \"%s_sdma%s.bin\"\n",146 chip_name, i == 0 ? "" : "1");147 for (i = 0; i < adev->sdma.num_instances; i++)148 amdgpu_ucode_release(&adev->sdma.instance[i].fw);149 }150 return err;151}152 153/**154 * cik_sdma_ring_get_rptr - get the current read pointer155 *156 * @ring: amdgpu ring pointer157 *158 * Get the current rptr from the hardware (CIK+).159 */160static uint64_t cik_sdma_ring_get_rptr(struct amdgpu_ring *ring)161{162 u32 rptr;163 164 rptr = *ring->rptr_cpu_addr;165 166 return (rptr & 0x3fffc) >> 2;167}168 169/**170 * cik_sdma_ring_get_wptr - get the current write pointer171 *172 * @ring: amdgpu ring pointer173 *174 * Get the current wptr from the hardware (CIK+).175 */176static uint64_t cik_sdma_ring_get_wptr(struct amdgpu_ring *ring)177{178 struct amdgpu_device *adev = ring->adev;179 180 return (RREG32(mmSDMA0_GFX_RB_WPTR + sdma_offsets[ring->me]) & 0x3fffc) >> 2;181}182 183/**184 * cik_sdma_ring_set_wptr - commit the write pointer185 *186 * @ring: amdgpu ring pointer187 *188 * Write the wptr back to the hardware (CIK+).189 */190static void cik_sdma_ring_set_wptr(struct amdgpu_ring *ring)191{192 struct amdgpu_device *adev = ring->adev;193 194 WREG32(mmSDMA0_GFX_RB_WPTR + sdma_offsets[ring->me],195 (ring->wptr << 2) & 0x3fffc);196}197 198static void cik_sdma_ring_insert_nop(struct amdgpu_ring *ring, uint32_t count)199{200 struct amdgpu_sdma_instance *sdma = amdgpu_sdma_get_instance_from_ring(ring);201 int i;202 203 for (i = 0; i < count; i++)204 if (sdma && sdma->burst_nop && (i == 0))205 amdgpu_ring_write(ring, ring->funcs->nop |206 SDMA_NOP_COUNT(count - 1));207 else208 amdgpu_ring_write(ring, ring->funcs->nop);209}210 211/**212 * cik_sdma_ring_emit_ib - Schedule an IB on the DMA engine213 *214 * @ring: amdgpu ring pointer215 * @job: job to retrive vmid from216 * @ib: IB object to schedule217 * @flags: unused218 *219 * Schedule an IB in the DMA ring (CIK).220 */221static void cik_sdma_ring_emit_ib(struct amdgpu_ring *ring,222 struct amdgpu_job *job,223 struct amdgpu_ib *ib,224 uint32_t flags)225{226 unsigned vmid = AMDGPU_JOB_GET_VMID(job);227 u32 extra_bits = vmid & 0xf;228 229 /* IB packet must end on a 8 DW boundary */230 cik_sdma_ring_insert_nop(ring, (4 - lower_32_bits(ring->wptr)) & 7);231 232 amdgpu_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_INDIRECT_BUFFER, 0, extra_bits));233 amdgpu_ring_write(ring, ib->gpu_addr & 0xffffffe0); /* base must be 32 byte aligned */234 amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xffffffff);235 amdgpu_ring_write(ring, ib->length_dw);236 237}238 239/**240 * cik_sdma_ring_emit_hdp_flush - emit an hdp flush on the DMA ring241 *242 * @ring: amdgpu ring pointer243 *244 * Emit an hdp flush packet on the requested DMA ring.245 */246static void cik_sdma_ring_emit_hdp_flush(struct amdgpu_ring *ring)247{248 u32 extra_bits = (SDMA_POLL_REG_MEM_EXTRA_OP(1) |249 SDMA_POLL_REG_MEM_EXTRA_FUNC(3)); /* == */250 u32 ref_and_mask;251 252 if (ring->me == 0)253 ref_and_mask = GPU_HDP_FLUSH_DONE__SDMA0_MASK;254 else255 ref_and_mask = GPU_HDP_FLUSH_DONE__SDMA1_MASK;256 257 amdgpu_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_POLL_REG_MEM, 0, extra_bits));258 amdgpu_ring_write(ring, mmGPU_HDP_FLUSH_DONE << 2);259 amdgpu_ring_write(ring, mmGPU_HDP_FLUSH_REQ << 2);260 amdgpu_ring_write(ring, ref_and_mask); /* reference */261 amdgpu_ring_write(ring, ref_and_mask); /* mask */262 amdgpu_ring_write(ring, (0xfff << 16) | 10); /* retry count, poll interval */263}264 265/**266 * cik_sdma_ring_emit_fence - emit a fence on the DMA ring267 *268 * @ring: amdgpu ring pointer269 * @addr: address270 * @seq: sequence number271 * @flags: fence related flags272 *273 * Add a DMA fence packet to the ring to write274 * the fence seq number and DMA trap packet to generate275 * an interrupt if needed (CIK).276 */277static void cik_sdma_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, u64 seq,278 unsigned flags)279{280 bool write64bit = flags & AMDGPU_FENCE_FLAG_64BIT;281 /* write the fence */282 amdgpu_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_FENCE, 0, 0));283 amdgpu_ring_write(ring, lower_32_bits(addr));284 amdgpu_ring_write(ring, upper_32_bits(addr));285 amdgpu_ring_write(ring, lower_32_bits(seq));286 287 /* optionally write high bits as well */288 if (write64bit) {289 addr += 4;290 amdgpu_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_FENCE, 0, 0));291 amdgpu_ring_write(ring, lower_32_bits(addr));292 amdgpu_ring_write(ring, upper_32_bits(addr));293 amdgpu_ring_write(ring, upper_32_bits(seq));294 }295 296 /* generate an interrupt */297 amdgpu_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_TRAP, 0, 0));298}299 300/**301 * cik_sdma_gfx_stop - stop the gfx async dma engines302 *303 * @adev: amdgpu_device pointer304 *305 * Stop the gfx async dma ring buffers (CIK).306 */307static void cik_sdma_gfx_stop(struct amdgpu_device *adev)308{309 u32 rb_cntl;310 int i;311 312 for (i = 0; i < adev->sdma.num_instances; i++) {313 rb_cntl = RREG32(mmSDMA0_GFX_RB_CNTL + sdma_offsets[i]);314 rb_cntl &= ~SDMA0_GFX_RB_CNTL__RB_ENABLE_MASK;315 WREG32(mmSDMA0_GFX_RB_CNTL + sdma_offsets[i], rb_cntl);316 WREG32(mmSDMA0_GFX_IB_CNTL + sdma_offsets[i], 0);317 }318}319 320/**321 * cik_sdma_rlc_stop - stop the compute async dma engines322 *323 * @adev: amdgpu_device pointer324 *325 * Stop the compute async dma queues (CIK).326 */327static void cik_sdma_rlc_stop(struct amdgpu_device *adev)328{329 /* XXX todo */330}331 332/**333 * cik_ctx_switch_enable - stop the async dma engines context switch334 *335 * @adev: amdgpu_device pointer336 * @enable: enable/disable the DMA MEs context switch.337 *338 * Halt or unhalt the async dma engines context switch (VI).339 */340static void cik_ctx_switch_enable(struct amdgpu_device *adev, bool enable)341{342 u32 f32_cntl, phase_quantum = 0;343 int i;344 345 if (amdgpu_sdma_phase_quantum) {346 unsigned value = amdgpu_sdma_phase_quantum;347 unsigned unit = 0;348 349 while (value > (SDMA0_PHASE0_QUANTUM__VALUE_MASK >>350 SDMA0_PHASE0_QUANTUM__VALUE__SHIFT)) {351 value = (value + 1) >> 1;352 unit++;353 }354 if (unit > (SDMA0_PHASE0_QUANTUM__UNIT_MASK >>355 SDMA0_PHASE0_QUANTUM__UNIT__SHIFT)) {356 value = (SDMA0_PHASE0_QUANTUM__VALUE_MASK >>357 SDMA0_PHASE0_QUANTUM__VALUE__SHIFT);358 unit = (SDMA0_PHASE0_QUANTUM__UNIT_MASK >>359 SDMA0_PHASE0_QUANTUM__UNIT__SHIFT);360 WARN_ONCE(1,361 "clamping sdma_phase_quantum to %uK clock cycles\n",362 value << unit);363 }364 phase_quantum =365 value << SDMA0_PHASE0_QUANTUM__VALUE__SHIFT |366 unit << SDMA0_PHASE0_QUANTUM__UNIT__SHIFT;367 }368 369 for (i = 0; i < adev->sdma.num_instances; i++) {370 f32_cntl = RREG32(mmSDMA0_CNTL + sdma_offsets[i]);371 if (enable) {372 f32_cntl = REG_SET_FIELD(f32_cntl, SDMA0_CNTL,373 AUTO_CTXSW_ENABLE, 1);374 if (amdgpu_sdma_phase_quantum) {375 WREG32(mmSDMA0_PHASE0_QUANTUM + sdma_offsets[i],376 phase_quantum);377 WREG32(mmSDMA0_PHASE1_QUANTUM + sdma_offsets[i],378 phase_quantum);379 }380 } else {381 f32_cntl = REG_SET_FIELD(f32_cntl, SDMA0_CNTL,382 AUTO_CTXSW_ENABLE, 0);383 }384 385 WREG32(mmSDMA0_CNTL + sdma_offsets[i], f32_cntl);386 }387}388 389/**390 * cik_sdma_enable - stop the async dma engines391 *392 * @adev: amdgpu_device pointer393 * @enable: enable/disable the DMA MEs.394 *395 * Halt or unhalt the async dma engines (CIK).396 */397static void cik_sdma_enable(struct amdgpu_device *adev, bool enable)398{399 u32 me_cntl;400 int i;401 402 if (!enable) {403 cik_sdma_gfx_stop(adev);404 cik_sdma_rlc_stop(adev);405 }406 407 for (i = 0; i < adev->sdma.num_instances; i++) {408 me_cntl = RREG32(mmSDMA0_F32_CNTL + sdma_offsets[i]);409 if (enable)410 me_cntl &= ~SDMA0_F32_CNTL__HALT_MASK;411 else412 me_cntl |= SDMA0_F32_CNTL__HALT_MASK;413 WREG32(mmSDMA0_F32_CNTL + sdma_offsets[i], me_cntl);414 }415}416 417/**418 * cik_sdma_gfx_resume - setup and start the async dma engines419 *420 * @adev: amdgpu_device pointer421 *422 * Set up the gfx DMA ring buffers and enable them (CIK).423 * Returns 0 for success, error for failure.424 */425static int cik_sdma_gfx_resume(struct amdgpu_device *adev)426{427 struct amdgpu_ring *ring;428 u32 rb_cntl, ib_cntl;429 u32 rb_bufsz;430 int i, j, r;431 432 for (i = 0; i < adev->sdma.num_instances; i++) {433 ring = &adev->sdma.instance[i].ring;434 435 mutex_lock(&adev->srbm_mutex);436 for (j = 0; j < 16; j++) {437 cik_srbm_select(adev, 0, 0, 0, j);438 /* SDMA GFX */439 WREG32(mmSDMA0_GFX_VIRTUAL_ADDR + sdma_offsets[i], 0);440 WREG32(mmSDMA0_GFX_APE1_CNTL + sdma_offsets[i], 0);441 /* XXX SDMA RLC - todo */442 }443 cik_srbm_select(adev, 0, 0, 0, 0);444 mutex_unlock(&adev->srbm_mutex);445 446 WREG32(mmSDMA0_TILING_CONFIG + sdma_offsets[i],447 adev->gfx.config.gb_addr_config & 0x70);448 449 WREG32(mmSDMA0_SEM_INCOMPLETE_TIMER_CNTL + sdma_offsets[i], 0);450 WREG32(mmSDMA0_SEM_WAIT_FAIL_TIMER_CNTL + sdma_offsets[i], 0);451 452 /* Set ring buffer size in dwords */453 rb_bufsz = order_base_2(ring->ring_size / 4);454 rb_cntl = rb_bufsz << 1;455#ifdef __BIG_ENDIAN456 rb_cntl |= SDMA0_GFX_RB_CNTL__RB_SWAP_ENABLE_MASK |457 SDMA0_GFX_RB_CNTL__RPTR_WRITEBACK_SWAP_ENABLE_MASK;458#endif459 WREG32(mmSDMA0_GFX_RB_CNTL + sdma_offsets[i], rb_cntl);460 461 /* Initialize the ring buffer's read and write pointers */462 WREG32(mmSDMA0_GFX_RB_RPTR + sdma_offsets[i], 0);463 WREG32(mmSDMA0_GFX_RB_WPTR + sdma_offsets[i], 0);464 WREG32(mmSDMA0_GFX_IB_RPTR + sdma_offsets[i], 0);465 WREG32(mmSDMA0_GFX_IB_OFFSET + sdma_offsets[i], 0);466 467 /* set the wb address whether it's enabled or not */468 WREG32(mmSDMA0_GFX_RB_RPTR_ADDR_HI + sdma_offsets[i],469 upper_32_bits(ring->rptr_gpu_addr) & 0xFFFFFFFF);470 WREG32(mmSDMA0_GFX_RB_RPTR_ADDR_LO + sdma_offsets[i],471 ((ring->rptr_gpu_addr) & 0xFFFFFFFC));472 473 rb_cntl |= SDMA0_GFX_RB_CNTL__RPTR_WRITEBACK_ENABLE_MASK;474 475 WREG32(mmSDMA0_GFX_RB_BASE + sdma_offsets[i], ring->gpu_addr >> 8);476 WREG32(mmSDMA0_GFX_RB_BASE_HI + sdma_offsets[i], ring->gpu_addr >> 40);477 478 ring->wptr = 0;479 WREG32(mmSDMA0_GFX_RB_WPTR + sdma_offsets[i], ring->wptr << 2);480 481 /* enable DMA RB */482 WREG32(mmSDMA0_GFX_RB_CNTL + sdma_offsets[i],483 rb_cntl | SDMA0_GFX_RB_CNTL__RB_ENABLE_MASK);484 485 ib_cntl = SDMA0_GFX_IB_CNTL__IB_ENABLE_MASK;486#ifdef __BIG_ENDIAN487 ib_cntl |= SDMA0_GFX_IB_CNTL__IB_SWAP_ENABLE_MASK;488#endif489 /* enable DMA IBs */490 WREG32(mmSDMA0_GFX_IB_CNTL + sdma_offsets[i], ib_cntl);491 }492 493 cik_sdma_enable(adev, true);494 495 for (i = 0; i < adev->sdma.num_instances; i++) {496 ring = &adev->sdma.instance[i].ring;497 r = amdgpu_ring_test_helper(ring);498 if (r)499 return r;500 }501 502 return 0;503}504 505/**506 * cik_sdma_rlc_resume - setup and start the async dma engines507 *508 * @adev: amdgpu_device pointer509 *510 * Set up the compute DMA queues and enable them (CIK).511 * Returns 0 for success, error for failure.512 */513static int cik_sdma_rlc_resume(struct amdgpu_device *adev)514{515 /* XXX todo */516 return 0;517}518 519/**520 * cik_sdma_load_microcode - load the sDMA ME ucode521 *522 * @adev: amdgpu_device pointer523 *524 * Loads the sDMA0/1 ucode.525 * Returns 0 for success, -EINVAL if the ucode is not available.526 */527static int cik_sdma_load_microcode(struct amdgpu_device *adev)528{529 const struct sdma_firmware_header_v1_0 *hdr;530 const __le32 *fw_data;531 u32 fw_size;532 int i, j;533 534 /* halt the MEs */535 cik_sdma_enable(adev, false);536 537 for (i = 0; i < adev->sdma.num_instances; i++) {538 if (!adev->sdma.instance[i].fw)539 return -EINVAL;540 hdr = (const struct sdma_firmware_header_v1_0 *)adev->sdma.instance[i].fw->data;541 amdgpu_ucode_print_sdma_hdr(&hdr->header);542 fw_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;543 adev->sdma.instance[i].fw_version = le32_to_cpu(hdr->header.ucode_version);544 adev->sdma.instance[i].feature_version = le32_to_cpu(hdr->ucode_feature_version);545 if (adev->sdma.instance[i].feature_version >= 20)546 adev->sdma.instance[i].burst_nop = true;547 fw_data = (const __le32 *)548 (adev->sdma.instance[i].fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));549 WREG32(mmSDMA0_UCODE_ADDR + sdma_offsets[i], 0);550 for (j = 0; j < fw_size; j++)551 WREG32(mmSDMA0_UCODE_DATA + sdma_offsets[i], le32_to_cpup(fw_data++));552 WREG32(mmSDMA0_UCODE_ADDR + sdma_offsets[i], adev->sdma.instance[i].fw_version);553 }554 555 return 0;556}557 558/**559 * cik_sdma_start - setup and start the async dma engines560 *561 * @adev: amdgpu_device pointer562 *563 * Set up the DMA engines and enable them (CIK).564 * Returns 0 for success, error for failure.565 */566static int cik_sdma_start(struct amdgpu_device *adev)567{568 int r;569 570 r = cik_sdma_load_microcode(adev);571 if (r)572 return r;573 574 /* halt the engine before programing */575 cik_sdma_enable(adev, false);576 /* enable sdma ring preemption */577 cik_ctx_switch_enable(adev, true);578 579 /* start the gfx rings and rlc compute queues */580 r = cik_sdma_gfx_resume(adev);581 if (r)582 return r;583 r = cik_sdma_rlc_resume(adev);584 if (r)585 return r;586 587 return 0;588}589 590/**591 * cik_sdma_ring_test_ring - simple async dma engine test592 *593 * @ring: amdgpu_ring structure holding ring information594 *595 * Test the DMA engine by writing using it to write an596 * value to memory. (CIK).597 * Returns 0 for success, error for failure.598 */599static int cik_sdma_ring_test_ring(struct amdgpu_ring *ring)600{601 struct amdgpu_device *adev = ring->adev;602 unsigned i;603 unsigned index;604 int r;605 u32 tmp;606 u64 gpu_addr;607 608 r = amdgpu_device_wb_get(adev, &index);609 if (r)610 return r;611 612 gpu_addr = adev->wb.gpu_addr + (index * 4);613 tmp = 0xCAFEDEAD;614 adev->wb.wb[index] = cpu_to_le32(tmp);615 616 r = amdgpu_ring_alloc(ring, 5);617 if (r)618 goto error_free_wb;619 620 amdgpu_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_WRITE, SDMA_WRITE_SUB_OPCODE_LINEAR, 0));621 amdgpu_ring_write(ring, lower_32_bits(gpu_addr));622 amdgpu_ring_write(ring, upper_32_bits(gpu_addr));623 amdgpu_ring_write(ring, 1); /* number of DWs to follow */624 amdgpu_ring_write(ring, 0xDEADBEEF);625 amdgpu_ring_commit(ring);626 627 for (i = 0; i < adev->usec_timeout; i++) {628 tmp = le32_to_cpu(adev->wb.wb[index]);629 if (tmp == 0xDEADBEEF)630 break;631 udelay(1);632 }633 634 if (i >= adev->usec_timeout)635 r = -ETIMEDOUT;636 637error_free_wb:638 amdgpu_device_wb_free(adev, index);639 return r;640}641 642/**643 * cik_sdma_ring_test_ib - test an IB on the DMA engine644 *645 * @ring: amdgpu_ring structure holding ring information646 * @timeout: timeout value in jiffies, or MAX_SCHEDULE_TIMEOUT647 *648 * Test a simple IB in the DMA ring (CIK).649 * Returns 0 on success, error on failure.650 */651static int cik_sdma_ring_test_ib(struct amdgpu_ring *ring, long timeout)652{653 struct amdgpu_device *adev = ring->adev;654 struct amdgpu_ib ib;655 struct dma_fence *f = NULL;656 unsigned index;657 u32 tmp = 0;658 u64 gpu_addr;659 long r;660 661 r = amdgpu_device_wb_get(adev, &index);662 if (r)663 return r;664 665 gpu_addr = adev->wb.gpu_addr + (index * 4);666 tmp = 0xCAFEDEAD;667 adev->wb.wb[index] = cpu_to_le32(tmp);668 memset(&ib, 0, sizeof(ib));669 r = amdgpu_ib_get(adev, NULL, 256,670 AMDGPU_IB_POOL_DIRECT, &ib);671 if (r)672 goto err0;673 674 ib.ptr[0] = SDMA_PACKET(SDMA_OPCODE_WRITE,675 SDMA_WRITE_SUB_OPCODE_LINEAR, 0);676 ib.ptr[1] = lower_32_bits(gpu_addr);677 ib.ptr[2] = upper_32_bits(gpu_addr);678 ib.ptr[3] = 1;679 ib.ptr[4] = 0xDEADBEEF;680 ib.length_dw = 5;681 r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f);682 if (r)683 goto err1;684 685 r = dma_fence_wait_timeout(f, false, timeout);686 if (r == 0) {687 r = -ETIMEDOUT;688 goto err1;689 } else if (r < 0) {690 goto err1;691 }692 tmp = le32_to_cpu(adev->wb.wb[index]);693 if (tmp == 0xDEADBEEF)694 r = 0;695 else696 r = -EINVAL;697 698err1:699 amdgpu_ib_free(adev, &ib, NULL);700 dma_fence_put(f);701err0:702 amdgpu_device_wb_free(adev, index);703 return r;704}705 706/**707 * cik_sdma_vm_copy_pte - update PTEs by copying them from the GART708 *709 * @ib: indirect buffer to fill with commands710 * @pe: addr of the page entry711 * @src: src addr to copy from712 * @count: number of page entries to update713 *714 * Update PTEs by copying them from the GART using sDMA (CIK).715 */716static void cik_sdma_vm_copy_pte(struct amdgpu_ib *ib,717 uint64_t pe, uint64_t src,718 unsigned count)719{720 unsigned bytes = count * 8;721 722 ib->ptr[ib->length_dw++] = SDMA_PACKET(SDMA_OPCODE_COPY,723 SDMA_WRITE_SUB_OPCODE_LINEAR, 0);724 ib->ptr[ib->length_dw++] = bytes;725 ib->ptr[ib->length_dw++] = 0; /* src/dst endian swap */726 ib->ptr[ib->length_dw++] = lower_32_bits(src);727 ib->ptr[ib->length_dw++] = upper_32_bits(src);728 ib->ptr[ib->length_dw++] = lower_32_bits(pe);729 ib->ptr[ib->length_dw++] = upper_32_bits(pe);730}731 732/**733 * cik_sdma_vm_write_pte - update PTEs by writing them manually734 *735 * @ib: indirect buffer to fill with commands736 * @pe: addr of the page entry737 * @value: dst addr to write into pe738 * @count: number of page entries to update739 * @incr: increase next addr by incr bytes740 *741 * Update PTEs by writing them manually using sDMA (CIK).742 */743static void cik_sdma_vm_write_pte(struct amdgpu_ib *ib, uint64_t pe,744 uint64_t value, unsigned count,745 uint32_t incr)746{747 unsigned ndw = count * 2;748 749 ib->ptr[ib->length_dw++] = SDMA_PACKET(SDMA_OPCODE_WRITE,750 SDMA_WRITE_SUB_OPCODE_LINEAR, 0);751 ib->ptr[ib->length_dw++] = lower_32_bits(pe);752 ib->ptr[ib->length_dw++] = upper_32_bits(pe);753 ib->ptr[ib->length_dw++] = ndw;754 for (; ndw > 0; ndw -= 2) {755 ib->ptr[ib->length_dw++] = lower_32_bits(value);756 ib->ptr[ib->length_dw++] = upper_32_bits(value);757 value += incr;758 }759}760 761/**762 * cik_sdma_vm_set_pte_pde - update the page tables using sDMA763 *764 * @ib: indirect buffer to fill with commands765 * @pe: addr of the page entry766 * @addr: dst addr to write into pe767 * @count: number of page entries to update768 * @incr: increase next addr by incr bytes769 * @flags: access flags770 *771 * Update the page tables using sDMA (CIK).772 */773static void cik_sdma_vm_set_pte_pde(struct amdgpu_ib *ib, uint64_t pe,774 uint64_t addr, unsigned count,775 uint32_t incr, uint64_t flags)776{777 /* for physically contiguous pages (vram) */778 ib->ptr[ib->length_dw++] = SDMA_PACKET(SDMA_OPCODE_GENERATE_PTE_PDE, 0, 0);779 ib->ptr[ib->length_dw++] = lower_32_bits(pe); /* dst addr */780 ib->ptr[ib->length_dw++] = upper_32_bits(pe);781 ib->ptr[ib->length_dw++] = lower_32_bits(flags); /* mask */782 ib->ptr[ib->length_dw++] = upper_32_bits(flags);783 ib->ptr[ib->length_dw++] = lower_32_bits(addr); /* value */784 ib->ptr[ib->length_dw++] = upper_32_bits(addr);785 ib->ptr[ib->length_dw++] = incr; /* increment size */786 ib->ptr[ib->length_dw++] = 0;787 ib->ptr[ib->length_dw++] = count; /* number of entries */788}789 790/**791 * cik_sdma_ring_pad_ib - pad the IB to the required number of dw792 *793 * @ring: amdgpu_ring structure holding ring information794 * @ib: indirect buffer to fill with padding795 *796 */797static void cik_sdma_ring_pad_ib(struct amdgpu_ring *ring, struct amdgpu_ib *ib)798{799 struct amdgpu_sdma_instance *sdma = amdgpu_sdma_get_instance_from_ring(ring);800 u32 pad_count;801 int i;802 803 pad_count = (-ib->length_dw) & 7;804 for (i = 0; i < pad_count; i++)805 if (sdma && sdma->burst_nop && (i == 0))806 ib->ptr[ib->length_dw++] =807 SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0) |808 SDMA_NOP_COUNT(pad_count - 1);809 else810 ib->ptr[ib->length_dw++] =811 SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0);812}813 814/**815 * cik_sdma_ring_emit_pipeline_sync - sync the pipeline816 *817 * @ring: amdgpu_ring pointer818 *819 * Make sure all previous operations are completed (CIK).820 */821static void cik_sdma_ring_emit_pipeline_sync(struct amdgpu_ring *ring)822{823 uint32_t seq = ring->fence_drv.sync_seq;824 uint64_t addr = ring->fence_drv.gpu_addr;825 826 /* wait for idle */827 amdgpu_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_POLL_REG_MEM, 0,828 SDMA_POLL_REG_MEM_EXTRA_OP(0) |829 SDMA_POLL_REG_MEM_EXTRA_FUNC(3) | /* equal */830 SDMA_POLL_REG_MEM_EXTRA_M));831 amdgpu_ring_write(ring, addr & 0xfffffffc);832 amdgpu_ring_write(ring, upper_32_bits(addr) & 0xffffffff);833 amdgpu_ring_write(ring, seq); /* reference */834 amdgpu_ring_write(ring, 0xffffffff); /* mask */835 amdgpu_ring_write(ring, (0xfff << 16) | 4); /* retry count, poll interval */836}837 838/**839 * cik_sdma_ring_emit_vm_flush - cik vm flush using sDMA840 *841 * @ring: amdgpu_ring pointer842 * @vmid: vmid number to use843 * @pd_addr: address844 *845 * Update the page table base and flush the VM TLB846 * using sDMA (CIK).847 */848static void cik_sdma_ring_emit_vm_flush(struct amdgpu_ring *ring,849 unsigned vmid, uint64_t pd_addr)850{851 u32 extra_bits = (SDMA_POLL_REG_MEM_EXTRA_OP(0) |852 SDMA_POLL_REG_MEM_EXTRA_FUNC(0)); /* always */853 854 amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr);855 856 amdgpu_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_POLL_REG_MEM, 0, extra_bits));857 amdgpu_ring_write(ring, mmVM_INVALIDATE_REQUEST << 2);858 amdgpu_ring_write(ring, 0);859 amdgpu_ring_write(ring, 0); /* reference */860 amdgpu_ring_write(ring, 0); /* mask */861 amdgpu_ring_write(ring, (0xfff << 16) | 10); /* retry count, poll interval */862}863 864static void cik_sdma_ring_emit_wreg(struct amdgpu_ring *ring,865 uint32_t reg, uint32_t val)866{867 amdgpu_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_SRBM_WRITE, 0, 0xf000));868 amdgpu_ring_write(ring, reg);869 amdgpu_ring_write(ring, val);870}871 872static void cik_enable_sdma_mgcg(struct amdgpu_device *adev,873 bool enable)874{875 u32 orig, data;876 877 if (enable && (adev->cg_flags & AMD_CG_SUPPORT_SDMA_MGCG)) {878 WREG32(mmSDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET, 0x00000100);879 WREG32(mmSDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET, 0x00000100);880 } else {881 orig = data = RREG32(mmSDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET);882 data |= 0xff000000;883 if (data != orig)884 WREG32(mmSDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET, data);885 886 orig = data = RREG32(mmSDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET);887 data |= 0xff000000;888 if (data != orig)889 WREG32(mmSDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET, data);890 }891}892 893static void cik_enable_sdma_mgls(struct amdgpu_device *adev,894 bool enable)895{896 u32 orig, data;897 898 if (enable && (adev->cg_flags & AMD_CG_SUPPORT_SDMA_LS)) {899 orig = data = RREG32(mmSDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET);900 data |= 0x100;901 if (orig != data)902 WREG32(mmSDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET, data);903 904 orig = data = RREG32(mmSDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET);905 data |= 0x100;906 if (orig != data)907 WREG32(mmSDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET, data);908 } else {909 orig = data = RREG32(mmSDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET);910 data &= ~0x100;911 if (orig != data)912 WREG32(mmSDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET, data);913 914 orig = data = RREG32(mmSDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET);915 data &= ~0x100;916 if (orig != data)917 WREG32(mmSDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET, data);918 }919}920 921static int cik_sdma_early_init(void *handle)922{923 struct amdgpu_device *adev = (struct amdgpu_device *)handle;924 int r;925 926 adev->sdma.num_instances = SDMA_MAX_INSTANCE;927 928 r = cik_sdma_init_microcode(adev);929 if (r)930 return r;931 932 cik_sdma_set_ring_funcs(adev);933 cik_sdma_set_irq_funcs(adev);934 cik_sdma_set_buffer_funcs(adev);935 cik_sdma_set_vm_pte_funcs(adev);936 937 return 0;938}939 940static int cik_sdma_sw_init(void *handle)941{942 struct amdgpu_ring *ring;943 struct amdgpu_device *adev = (struct amdgpu_device *)handle;944 int r, i;945 946 /* SDMA trap event */947 r = amdgpu_irq_add_id(adev, AMDGPU_IRQ_CLIENTID_LEGACY, 224,948 &adev->sdma.trap_irq);949 if (r)950 return r;951 952 /* SDMA Privileged inst */953 r = amdgpu_irq_add_id(adev, AMDGPU_IRQ_CLIENTID_LEGACY, 241,954 &adev->sdma.illegal_inst_irq);955 if (r)956 return r;957 958 /* SDMA Privileged inst */959 r = amdgpu_irq_add_id(adev, AMDGPU_IRQ_CLIENTID_LEGACY, 247,960 &adev->sdma.illegal_inst_irq);961 if (r)962 return r;963 964 for (i = 0; i < adev->sdma.num_instances; i++) {965 ring = &adev->sdma.instance[i].ring;966 ring->ring_obj = NULL;967 sprintf(ring->name, "sdma%d", i);968 r = amdgpu_ring_init(adev, ring, 1024,969 &adev->sdma.trap_irq,970 (i == 0) ? AMDGPU_SDMA_IRQ_INSTANCE0 :971 AMDGPU_SDMA_IRQ_INSTANCE1,972 AMDGPU_RING_PRIO_DEFAULT, NULL);973 if (r)974 return r;975 }976 977 return r;978}979 980static int cik_sdma_sw_fini(void *handle)981{982 struct amdgpu_device *adev = (struct amdgpu_device *)handle;983 int i;984 985 for (i = 0; i < adev->sdma.num_instances; i++)986 amdgpu_ring_fini(&adev->sdma.instance[i].ring);987 988 cik_sdma_free_microcode(adev);989 return 0;990}991 992static int cik_sdma_hw_init(void *handle)993{994 int r;995 struct amdgpu_device *adev = (struct amdgpu_device *)handle;996 997 r = cik_sdma_start(adev);998 if (r)999 return r;1000 1001 return r;1002}1003 1004static int cik_sdma_hw_fini(void *handle)1005{1006 struct amdgpu_device *adev = (struct amdgpu_device *)handle;1007 1008 cik_ctx_switch_enable(adev, false);1009 cik_sdma_enable(adev, false);1010 1011 return 0;1012}1013 1014static int cik_sdma_suspend(void *handle)1015{1016 struct amdgpu_device *adev = (struct amdgpu_device *)handle;1017 1018 return cik_sdma_hw_fini(adev);1019}1020 1021static int cik_sdma_resume(void *handle)1022{1023 struct amdgpu_device *adev = (struct amdgpu_device *)handle;1024 1025 cik_sdma_soft_reset(handle);1026 1027 return cik_sdma_hw_init(adev);1028}1029 1030static bool cik_sdma_is_idle(void *handle)1031{1032 struct amdgpu_device *adev = (struct amdgpu_device *)handle;1033 u32 tmp = RREG32(mmSRBM_STATUS2);1034 1035 if (tmp & (SRBM_STATUS2__SDMA_BUSY_MASK |1036 SRBM_STATUS2__SDMA1_BUSY_MASK))1037 return false;1038 1039 return true;1040}1041 1042static int cik_sdma_wait_for_idle(void *handle)1043{1044 unsigned i;1045 u32 tmp;1046 struct amdgpu_device *adev = (struct amdgpu_device *)handle;1047 1048 for (i = 0; i < adev->usec_timeout; i++) {1049 tmp = RREG32(mmSRBM_STATUS2) & (SRBM_STATUS2__SDMA_BUSY_MASK |1050 SRBM_STATUS2__SDMA1_BUSY_MASK);1051 1052 if (!tmp)1053 return 0;1054 udelay(1);1055 }1056 return -ETIMEDOUT;1057}1058 1059static int cik_sdma_soft_reset(void *handle)1060{1061 u32 srbm_soft_reset = 0;1062 struct amdgpu_device *adev = (struct amdgpu_device *)handle;1063 u32 tmp;1064 1065 /* sdma0 */1066 tmp = RREG32(mmSDMA0_F32_CNTL + SDMA0_REGISTER_OFFSET);1067 tmp |= SDMA0_F32_CNTL__HALT_MASK;1068 WREG32(mmSDMA0_F32_CNTL + SDMA0_REGISTER_OFFSET, tmp);1069 srbm_soft_reset |= SRBM_SOFT_RESET__SOFT_RESET_SDMA_MASK;1070 1071 /* sdma1 */1072 tmp = RREG32(mmSDMA0_F32_CNTL + SDMA1_REGISTER_OFFSET);1073 tmp |= SDMA0_F32_CNTL__HALT_MASK;1074 WREG32(mmSDMA0_F32_CNTL + SDMA1_REGISTER_OFFSET, tmp);1075 srbm_soft_reset |= SRBM_SOFT_RESET__SOFT_RESET_SDMA1_MASK;1076 1077 if (srbm_soft_reset) {1078 tmp = RREG32(mmSRBM_SOFT_RESET);1079 tmp |= srbm_soft_reset;1080 dev_info(adev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);1081 WREG32(mmSRBM_SOFT_RESET, tmp);1082 tmp = RREG32(mmSRBM_SOFT_RESET);1083 1084 udelay(50);1085 1086 tmp &= ~srbm_soft_reset;1087 WREG32(mmSRBM_SOFT_RESET, tmp);1088 tmp = RREG32(mmSRBM_SOFT_RESET);1089 1090 /* Wait a little for things to settle down */1091 udelay(50);1092 }1093 1094 return 0;1095}1096 1097static int cik_sdma_set_trap_irq_state(struct amdgpu_device *adev,1098 struct amdgpu_irq_src *src,1099 unsigned type,1100 enum amdgpu_interrupt_state state)1101{1102 u32 sdma_cntl;1103 1104 switch (type) {1105 case AMDGPU_SDMA_IRQ_INSTANCE0:1106 switch (state) {1107 case AMDGPU_IRQ_STATE_DISABLE:1108 sdma_cntl = RREG32(mmSDMA0_CNTL + SDMA0_REGISTER_OFFSET);1109 sdma_cntl &= ~SDMA0_CNTL__TRAP_ENABLE_MASK;1110 WREG32(mmSDMA0_CNTL + SDMA0_REGISTER_OFFSET, sdma_cntl);1111 break;1112 case AMDGPU_IRQ_STATE_ENABLE:1113 sdma_cntl = RREG32(mmSDMA0_CNTL + SDMA0_REGISTER_OFFSET);1114 sdma_cntl |= SDMA0_CNTL__TRAP_ENABLE_MASK;1115 WREG32(mmSDMA0_CNTL + SDMA0_REGISTER_OFFSET, sdma_cntl);1116 break;1117 default:1118 break;1119 }1120 break;1121 case AMDGPU_SDMA_IRQ_INSTANCE1:1122 switch (state) {1123 case AMDGPU_IRQ_STATE_DISABLE:1124 sdma_cntl = RREG32(mmSDMA0_CNTL + SDMA1_REGISTER_OFFSET);1125 sdma_cntl &= ~SDMA0_CNTL__TRAP_ENABLE_MASK;1126 WREG32(mmSDMA0_CNTL + SDMA1_REGISTER_OFFSET, sdma_cntl);1127 break;1128 case AMDGPU_IRQ_STATE_ENABLE:1129 sdma_cntl = RREG32(mmSDMA0_CNTL + SDMA1_REGISTER_OFFSET);1130 sdma_cntl |= SDMA0_CNTL__TRAP_ENABLE_MASK;1131 WREG32(mmSDMA0_CNTL + SDMA1_REGISTER_OFFSET, sdma_cntl);1132 break;1133 default:1134 break;1135 }1136 break;1137 default:1138 break;1139 }1140 return 0;1141}1142 1143static int cik_sdma_process_trap_irq(struct amdgpu_device *adev,1144 struct amdgpu_irq_src *source,1145 struct amdgpu_iv_entry *entry)1146{1147 u8 instance_id, queue_id;1148 1149 instance_id = (entry->ring_id & 0x3) >> 0;1150 queue_id = (entry->ring_id & 0xc) >> 2;1151 DRM_DEBUG("IH: SDMA trap\n");1152 switch (instance_id) {1153 case 0:1154 switch (queue_id) {1155 case 0:1156 amdgpu_fence_process(&adev->sdma.instance[0].ring);1157 break;1158 case 1:1159 /* XXX compute */1160 break;1161 case 2:1162 /* XXX compute */1163 break;1164 }1165 break;1166 case 1:1167 switch (queue_id) {1168 case 0:1169 amdgpu_fence_process(&adev->sdma.instance[1].ring);1170 break;1171 case 1:1172 /* XXX compute */1173 break;1174 case 2:1175 /* XXX compute */1176 break;1177 }1178 break;1179 }1180 1181 return 0;1182}1183 1184static int cik_sdma_process_illegal_inst_irq(struct amdgpu_device *adev,1185 struct amdgpu_irq_src *source,1186 struct amdgpu_iv_entry *entry)1187{1188 u8 instance_id;1189 1190 DRM_ERROR("Illegal instruction in SDMA command stream\n");1191 instance_id = (entry->ring_id & 0x3) >> 0;1192 drm_sched_fault(&adev->sdma.instance[instance_id].ring.sched);1193 return 0;1194}1195 1196static int cik_sdma_set_clockgating_state(void *handle,1197 enum amd_clockgating_state state)1198{1199 bool gate = false;1200 struct amdgpu_device *adev = (struct amdgpu_device *)handle;1201 1202 if (state == AMD_CG_STATE_GATE)1203 gate = true;1204 1205 cik_enable_sdma_mgcg(adev, gate);1206 cik_enable_sdma_mgls(adev, gate);1207 1208 return 0;1209}1210 1211static int cik_sdma_set_powergating_state(void *handle,1212 enum amd_powergating_state state)1213{1214 return 0;1215}1216 1217static const struct amd_ip_funcs cik_sdma_ip_funcs = {1218 .name = "cik_sdma",1219 .early_init = cik_sdma_early_init,1220 .late_init = NULL,1221 .sw_init = cik_sdma_sw_init,1222 .sw_fini = cik_sdma_sw_fini,1223 .hw_init = cik_sdma_hw_init,1224 .hw_fini = cik_sdma_hw_fini,1225 .suspend = cik_sdma_suspend,1226 .resume = cik_sdma_resume,1227 .is_idle = cik_sdma_is_idle,1228 .wait_for_idle = cik_sdma_wait_for_idle,1229 .soft_reset = cik_sdma_soft_reset,1230 .set_clockgating_state = cik_sdma_set_clockgating_state,1231 .set_powergating_state = cik_sdma_set_powergating_state,1232 .dump_ip_state = NULL,1233 .print_ip_state = NULL,1234};1235 1236static const struct amdgpu_ring_funcs cik_sdma_ring_funcs = {1237 .type = AMDGPU_RING_TYPE_SDMA,1238 .align_mask = 0xf,1239 .nop = SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0),1240 .support_64bit_ptrs = false,1241 .get_rptr = cik_sdma_ring_get_rptr,1242 .get_wptr = cik_sdma_ring_get_wptr,1243 .set_wptr = cik_sdma_ring_set_wptr,1244 .emit_frame_size =1245 6 + /* cik_sdma_ring_emit_hdp_flush */1246 3 + /* hdp invalidate */1247 6 + /* cik_sdma_ring_emit_pipeline_sync */1248 CIK_FLUSH_GPU_TLB_NUM_WREG * 3 + 6 + /* cik_sdma_ring_emit_vm_flush */1249 9 + 9 + 9, /* cik_sdma_ring_emit_fence x3 for user fence, vm fence */1250 .emit_ib_size = 7 + 4, /* cik_sdma_ring_emit_ib */1251 .emit_ib = cik_sdma_ring_emit_ib,1252 .emit_fence = cik_sdma_ring_emit_fence,1253 .emit_pipeline_sync = cik_sdma_ring_emit_pipeline_sync,1254 .emit_vm_flush = cik_sdma_ring_emit_vm_flush,1255 .emit_hdp_flush = cik_sdma_ring_emit_hdp_flush,1256 .test_ring = cik_sdma_ring_test_ring,1257 .test_ib = cik_sdma_ring_test_ib,1258 .insert_nop = cik_sdma_ring_insert_nop,1259 .pad_ib = cik_sdma_ring_pad_ib,1260 .emit_wreg = cik_sdma_ring_emit_wreg,1261};1262 1263static void cik_sdma_set_ring_funcs(struct amdgpu_device *adev)1264{1265 int i;1266 1267 for (i = 0; i < adev->sdma.num_instances; i++) {1268 adev->sdma.instance[i].ring.funcs = &cik_sdma_ring_funcs;1269 adev->sdma.instance[i].ring.me = i;1270 }1271}1272 1273static const struct amdgpu_irq_src_funcs cik_sdma_trap_irq_funcs = {1274 .set = cik_sdma_set_trap_irq_state,1275 .process = cik_sdma_process_trap_irq,1276};1277 1278static const struct amdgpu_irq_src_funcs cik_sdma_illegal_inst_irq_funcs = {1279 .process = cik_sdma_process_illegal_inst_irq,1280};1281 1282static void cik_sdma_set_irq_funcs(struct amdgpu_device *adev)1283{1284 adev->sdma.trap_irq.num_types = AMDGPU_SDMA_IRQ_LAST;1285 adev->sdma.trap_irq.funcs = &cik_sdma_trap_irq_funcs;1286 adev->sdma.illegal_inst_irq.funcs = &cik_sdma_illegal_inst_irq_funcs;1287}1288 1289/**1290 * cik_sdma_emit_copy_buffer - copy buffer using the sDMA engine1291 *1292 * @ib: indirect buffer to copy to1293 * @src_offset: src GPU address1294 * @dst_offset: dst GPU address1295 * @byte_count: number of bytes to xfer1296 * @copy_flags: unused1297 *1298 * Copy GPU buffers using the DMA engine (CIK).1299 * Used by the amdgpu ttm implementation to move pages if1300 * registered as the asic copy callback.1301 */1302static void cik_sdma_emit_copy_buffer(struct amdgpu_ib *ib,1303 uint64_t src_offset,1304 uint64_t dst_offset,1305 uint32_t byte_count,1306 uint32_t copy_flags)1307{1308 ib->ptr[ib->length_dw++] = SDMA_PACKET(SDMA_OPCODE_COPY, SDMA_COPY_SUB_OPCODE_LINEAR, 0);1309 ib->ptr[ib->length_dw++] = byte_count;1310 ib->ptr[ib->length_dw++] = 0; /* src/dst endian swap */1311 ib->ptr[ib->length_dw++] = lower_32_bits(src_offset);1312 ib->ptr[ib->length_dw++] = upper_32_bits(src_offset);1313 ib->ptr[ib->length_dw++] = lower_32_bits(dst_offset);1314 ib->ptr[ib->length_dw++] = upper_32_bits(dst_offset);1315}1316 1317/**1318 * cik_sdma_emit_fill_buffer - fill buffer using the sDMA engine1319 *1320 * @ib: indirect buffer to fill1321 * @src_data: value to write to buffer1322 * @dst_offset: dst GPU address1323 * @byte_count: number of bytes to xfer1324 *1325 * Fill GPU buffers using the DMA engine (CIK).1326 */1327static void cik_sdma_emit_fill_buffer(struct amdgpu_ib *ib,1328 uint32_t src_data,1329 uint64_t dst_offset,1330 uint32_t byte_count)1331{1332 ib->ptr[ib->length_dw++] = SDMA_PACKET(SDMA_OPCODE_CONSTANT_FILL, 0, 0);1333 ib->ptr[ib->length_dw++] = lower_32_bits(dst_offset);1334 ib->ptr[ib->length_dw++] = upper_32_bits(dst_offset);1335 ib->ptr[ib->length_dw++] = src_data;1336 ib->ptr[ib->length_dw++] = byte_count;1337}1338 1339static const struct amdgpu_buffer_funcs cik_sdma_buffer_funcs = {1340 .copy_max_bytes = 0x1fffff,1341 .copy_num_dw = 7,1342 .emit_copy_buffer = cik_sdma_emit_copy_buffer,1343 1344 .fill_max_bytes = 0x1fffff,1345 .fill_num_dw = 5,1346 .emit_fill_buffer = cik_sdma_emit_fill_buffer,1347};1348 1349static void cik_sdma_set_buffer_funcs(struct amdgpu_device *adev)1350{1351 adev->mman.buffer_funcs = &cik_sdma_buffer_funcs;1352 adev->mman.buffer_funcs_ring = &adev->sdma.instance[0].ring;1353}1354 1355static const struct amdgpu_vm_pte_funcs cik_sdma_vm_pte_funcs = {1356 .copy_pte_num_dw = 7,1357 .copy_pte = cik_sdma_vm_copy_pte,1358 1359 .write_pte = cik_sdma_vm_write_pte,1360 .set_pte_pde = cik_sdma_vm_set_pte_pde,1361};1362 1363static void cik_sdma_set_vm_pte_funcs(struct amdgpu_device *adev)1364{1365 unsigned i;1366 1367 adev->vm_manager.vm_pte_funcs = &cik_sdma_vm_pte_funcs;1368 for (i = 0; i < adev->sdma.num_instances; i++) {1369 adev->vm_manager.vm_pte_scheds[i] =1370 &adev->sdma.instance[i].ring.sched;1371 }1372 adev->vm_manager.vm_pte_num_scheds = adev->sdma.num_instances;1373}1374 1375const struct amdgpu_ip_block_version cik_sdma_ip_block =1376{1377 .type = AMD_IP_BLOCK_TYPE_SDMA,1378 .major = 2,1379 .minor = 0,1380 .rev = 0,1381 .funcs = &cik_sdma_ip_funcs,1382};1383