459 lines · c
1/*2 * Copyright 2014 Advanced Micro Devices, Inc.3 *4 * Permission is hereby granted, free of charge, to any person obtaining a5 * copy of this software and associated documentation files (the "Software"),6 * to deal in the Software without restriction, including without limitation7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,8 * and/or sell copies of the Software, and to permit persons to whom the9 * Software is furnished to do so, subject to the following conditions:10 *11 * The above copyright notice and this permission notice shall be included in12 * all copies or substantial portions of the Software.13 *14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR20 * OTHER DEALINGS IN THE SOFTWARE.21 *22 */23 24#include <linux/pci.h>25 26#include "amdgpu.h"27#include "amdgpu_ih.h"28#include "vid.h"29 30#include "oss/oss_3_0_1_d.h"31#include "oss/oss_3_0_1_sh_mask.h"32 33#include "bif/bif_5_1_d.h"34#include "bif/bif_5_1_sh_mask.h"35 36/*37 * Interrupts38 * Starting with r6xx, interrupts are handled via a ring buffer.39 * Ring buffers are areas of GPU accessible memory that the GPU40 * writes interrupt vectors into and the host reads vectors out of.41 * There is a rptr (read pointer) that determines where the42 * host is currently reading, and a wptr (write pointer)43 * which determines where the GPU has written. When the44 * pointers are equal, the ring is idle. When the GPU45 * writes vectors to the ring buffer, it increments the46 * wptr. When there is an interrupt, the host then starts47 * fetching commands and processing them until the pointers are48 * equal again at which point it updates the rptr.49 */50 51static void cz_ih_set_interrupt_funcs(struct amdgpu_device *adev);52 53/**54 * cz_ih_enable_interrupts - Enable the interrupt ring buffer55 *56 * @adev: amdgpu_device pointer57 *58 * Enable the interrupt ring buffer (VI).59 */60static void cz_ih_enable_interrupts(struct amdgpu_device *adev)61{62 u32 ih_cntl = RREG32(mmIH_CNTL);63 u32 ih_rb_cntl = RREG32(mmIH_RB_CNTL);64 65 ih_cntl = REG_SET_FIELD(ih_cntl, IH_CNTL, ENABLE_INTR, 1);66 ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, RB_ENABLE, 1);67 WREG32(mmIH_CNTL, ih_cntl);68 WREG32(mmIH_RB_CNTL, ih_rb_cntl);69 adev->irq.ih.enabled = true;70}71 72/**73 * cz_ih_disable_interrupts - Disable the interrupt ring buffer74 *75 * @adev: amdgpu_device pointer76 *77 * Disable the interrupt ring buffer (VI).78 */79static void cz_ih_disable_interrupts(struct amdgpu_device *adev)80{81 u32 ih_rb_cntl = RREG32(mmIH_RB_CNTL);82 u32 ih_cntl = RREG32(mmIH_CNTL);83 84 ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, RB_ENABLE, 0);85 ih_cntl = REG_SET_FIELD(ih_cntl, IH_CNTL, ENABLE_INTR, 0);86 WREG32(mmIH_RB_CNTL, ih_rb_cntl);87 WREG32(mmIH_CNTL, ih_cntl);88 /* set rptr, wptr to 0 */89 WREG32(mmIH_RB_RPTR, 0);90 WREG32(mmIH_RB_WPTR, 0);91 adev->irq.ih.enabled = false;92 adev->irq.ih.rptr = 0;93}94 95/**96 * cz_ih_irq_init - init and enable the interrupt ring97 *98 * @adev: amdgpu_device pointer99 *100 * Allocate a ring buffer for the interrupt controller,101 * enable the RLC, disable interrupts, enable the IH102 * ring buffer and enable it (VI).103 * Called at device load and reume.104 * Returns 0 for success, errors for failure.105 */106static int cz_ih_irq_init(struct amdgpu_device *adev)107{108 struct amdgpu_ih_ring *ih = &adev->irq.ih;109 u32 interrupt_cntl, ih_cntl, ih_rb_cntl;110 int rb_bufsz;111 112 /* disable irqs */113 cz_ih_disable_interrupts(adev);114 115 /* setup interrupt control */116 WREG32(mmINTERRUPT_CNTL2, adev->dummy_page_addr >> 8);117 interrupt_cntl = RREG32(mmINTERRUPT_CNTL);118 /* INTERRUPT_CNTL__IH_DUMMY_RD_OVERRIDE_MASK=0 - dummy read disabled with msi, enabled without msi119 * INTERRUPT_CNTL__IH_DUMMY_RD_OVERRIDE_MASK=1 - dummy read controlled by IH_DUMMY_RD_EN120 */121 interrupt_cntl = REG_SET_FIELD(interrupt_cntl, INTERRUPT_CNTL, IH_DUMMY_RD_OVERRIDE, 0);122 /* INTERRUPT_CNTL__IH_REQ_NONSNOOP_EN_MASK=1 if ring is in non-cacheable memory, e.g., vram */123 interrupt_cntl = REG_SET_FIELD(interrupt_cntl, INTERRUPT_CNTL, IH_REQ_NONSNOOP_EN, 0);124 WREG32(mmINTERRUPT_CNTL, interrupt_cntl);125 126 /* Ring Buffer base. [39:8] of 40-bit address of the beginning of the ring buffer*/127 WREG32(mmIH_RB_BASE, adev->irq.ih.gpu_addr >> 8);128 129 rb_bufsz = order_base_2(adev->irq.ih.ring_size / 4);130 ih_rb_cntl = REG_SET_FIELD(0, IH_RB_CNTL, WPTR_OVERFLOW_ENABLE, 1);131 ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, WPTR_OVERFLOW_CLEAR, 1);132 ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, RB_SIZE, rb_bufsz);133 134 /* Ring Buffer write pointer writeback. If enabled, IH_RB_WPTR register value is written to memory */135 ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, WPTR_WRITEBACK_ENABLE, 1);136 137 /* set the writeback address whether it's enabled or not */138 WREG32(mmIH_RB_WPTR_ADDR_LO, lower_32_bits(ih->wptr_addr));139 WREG32(mmIH_RB_WPTR_ADDR_HI, upper_32_bits(ih->wptr_addr) & 0xFF);140 141 WREG32(mmIH_RB_CNTL, ih_rb_cntl);142 143 /* set rptr, wptr to 0 */144 WREG32(mmIH_RB_RPTR, 0);145 WREG32(mmIH_RB_WPTR, 0);146 147 /* Default settings for IH_CNTL (disabled at first) */148 ih_cntl = RREG32(mmIH_CNTL);149 ih_cntl = REG_SET_FIELD(ih_cntl, IH_CNTL, MC_VMID, 0);150 151 if (adev->irq.msi_enabled)152 ih_cntl = REG_SET_FIELD(ih_cntl, IH_CNTL, RPTR_REARM, 1);153 WREG32(mmIH_CNTL, ih_cntl);154 155 pci_set_master(adev->pdev);156 157 /* enable interrupts */158 cz_ih_enable_interrupts(adev);159 160 return 0;161}162 163/**164 * cz_ih_irq_disable - disable interrupts165 *166 * @adev: amdgpu_device pointer167 *168 * Disable interrupts on the hw (VI).169 */170static void cz_ih_irq_disable(struct amdgpu_device *adev)171{172 cz_ih_disable_interrupts(adev);173 174 /* Wait and acknowledge irq */175 mdelay(1);176}177 178/**179 * cz_ih_get_wptr - get the IH ring buffer wptr180 *181 * @adev: amdgpu_device pointer182 * @ih: IH ring buffer to fetch wptr183 *184 * Get the IH ring buffer wptr from either the register185 * or the writeback memory buffer (VI). Also check for186 * ring buffer overflow and deal with it.187 * Used by cz_irq_process(VI).188 * Returns the value of the wptr.189 */190static u32 cz_ih_get_wptr(struct amdgpu_device *adev,191 struct amdgpu_ih_ring *ih)192{193 u32 wptr, tmp;194 195 wptr = le32_to_cpu(*ih->wptr_cpu);196 197 if (!REG_GET_FIELD(wptr, IH_RB_WPTR, RB_OVERFLOW))198 goto out;199 200 /* Double check that the overflow wasn't already cleared. */201 wptr = RREG32(mmIH_RB_WPTR);202 203 if (!REG_GET_FIELD(wptr, IH_RB_WPTR, RB_OVERFLOW))204 goto out;205 206 wptr = REG_SET_FIELD(wptr, IH_RB_WPTR, RB_OVERFLOW, 0);207 208 /* When a ring buffer overflow happen start parsing interrupt209 * from the last not overwritten vector (wptr + 16). Hopefully210 * this should allow us to catchup.211 */212 dev_warn(adev->dev, "IH ring buffer overflow (0x%08X, 0x%08X, 0x%08X)\n",213 wptr, ih->rptr, (wptr + 16) & ih->ptr_mask);214 ih->rptr = (wptr + 16) & ih->ptr_mask;215 tmp = RREG32(mmIH_RB_CNTL);216 tmp = REG_SET_FIELD(tmp, IH_RB_CNTL, WPTR_OVERFLOW_CLEAR, 1);217 WREG32(mmIH_RB_CNTL, tmp);218 219 /* Unset the CLEAR_OVERFLOW bit immediately so new overflows220 * can be detected.221 */222 tmp = REG_SET_FIELD(tmp, IH_RB_CNTL, WPTR_OVERFLOW_CLEAR, 0);223 WREG32(mmIH_RB_CNTL, tmp);224 225out:226 return (wptr & ih->ptr_mask);227}228 229/**230 * cz_ih_decode_iv - decode an interrupt vector231 *232 * @adev: amdgpu_device pointer233 * @ih: IH ring buffer to decode234 * @entry: IV entry to place decoded information into235 *236 * Decodes the interrupt vector at the current rptr237 * position and also advance the position.238 */239static void cz_ih_decode_iv(struct amdgpu_device *adev,240 struct amdgpu_ih_ring *ih,241 struct amdgpu_iv_entry *entry)242{243 /* wptr/rptr are in bytes! */244 u32 ring_index = ih->rptr >> 2;245 uint32_t dw[4];246 247 dw[0] = le32_to_cpu(ih->ring[ring_index + 0]);248 dw[1] = le32_to_cpu(ih->ring[ring_index + 1]);249 dw[2] = le32_to_cpu(ih->ring[ring_index + 2]);250 dw[3] = le32_to_cpu(ih->ring[ring_index + 3]);251 252 entry->client_id = AMDGPU_IRQ_CLIENTID_LEGACY;253 entry->src_id = dw[0] & 0xff;254 entry->src_data[0] = dw[1] & 0xfffffff;255 entry->ring_id = dw[2] & 0xff;256 entry->vmid = (dw[2] >> 8) & 0xff;257 entry->pasid = (dw[2] >> 16) & 0xffff;258 259 /* wptr/rptr are in bytes! */260 ih->rptr += 16;261}262 263/**264 * cz_ih_set_rptr - set the IH ring buffer rptr265 *266 * @adev: amdgpu_device pointer267 * @ih: IH ring buffer to set rptr268 *269 * Set the IH ring buffer rptr.270 */271static void cz_ih_set_rptr(struct amdgpu_device *adev,272 struct amdgpu_ih_ring *ih)273{274 WREG32(mmIH_RB_RPTR, ih->rptr);275}276 277static int cz_ih_early_init(void *handle)278{279 struct amdgpu_device *adev = (struct amdgpu_device *)handle;280 int ret;281 282 ret = amdgpu_irq_add_domain(adev);283 if (ret)284 return ret;285 286 cz_ih_set_interrupt_funcs(adev);287 288 return 0;289}290 291static int cz_ih_sw_init(void *handle)292{293 int r;294 struct amdgpu_device *adev = (struct amdgpu_device *)handle;295 296 r = amdgpu_ih_ring_init(adev, &adev->irq.ih, 64 * 1024, false);297 if (r)298 return r;299 300 r = amdgpu_irq_init(adev);301 302 return r;303}304 305static int cz_ih_sw_fini(void *handle)306{307 struct amdgpu_device *adev = (struct amdgpu_device *)handle;308 309 amdgpu_irq_fini_sw(adev);310 amdgpu_irq_remove_domain(adev);311 312 return 0;313}314 315static int cz_ih_hw_init(void *handle)316{317 int r;318 struct amdgpu_device *adev = (struct amdgpu_device *)handle;319 320 r = cz_ih_irq_init(adev);321 if (r)322 return r;323 324 return 0;325}326 327static int cz_ih_hw_fini(void *handle)328{329 struct amdgpu_device *adev = (struct amdgpu_device *)handle;330 331 cz_ih_irq_disable(adev);332 333 return 0;334}335 336static int cz_ih_suspend(void *handle)337{338 struct amdgpu_device *adev = (struct amdgpu_device *)handle;339 340 return cz_ih_hw_fini(adev);341}342 343static int cz_ih_resume(void *handle)344{345 struct amdgpu_device *adev = (struct amdgpu_device *)handle;346 347 return cz_ih_hw_init(adev);348}349 350static bool cz_ih_is_idle(void *handle)351{352 struct amdgpu_device *adev = (struct amdgpu_device *)handle;353 u32 tmp = RREG32(mmSRBM_STATUS);354 355 if (REG_GET_FIELD(tmp, SRBM_STATUS, IH_BUSY))356 return false;357 358 return true;359}360 361static int cz_ih_wait_for_idle(void *handle)362{363 unsigned i;364 u32 tmp;365 struct amdgpu_device *adev = (struct amdgpu_device *)handle;366 367 for (i = 0; i < adev->usec_timeout; i++) {368 /* read MC_STATUS */369 tmp = RREG32(mmSRBM_STATUS);370 if (!REG_GET_FIELD(tmp, SRBM_STATUS, IH_BUSY))371 return 0;372 udelay(1);373 }374 return -ETIMEDOUT;375}376 377static int cz_ih_soft_reset(void *handle)378{379 u32 srbm_soft_reset = 0;380 struct amdgpu_device *adev = (struct amdgpu_device *)handle;381 u32 tmp = RREG32(mmSRBM_STATUS);382 383 if (tmp & SRBM_STATUS__IH_BUSY_MASK)384 srbm_soft_reset = REG_SET_FIELD(srbm_soft_reset, SRBM_SOFT_RESET,385 SOFT_RESET_IH, 1);386 387 if (srbm_soft_reset) {388 tmp = RREG32(mmSRBM_SOFT_RESET);389 tmp |= srbm_soft_reset;390 dev_info(adev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);391 WREG32(mmSRBM_SOFT_RESET, tmp);392 tmp = RREG32(mmSRBM_SOFT_RESET);393 394 udelay(50);395 396 tmp &= ~srbm_soft_reset;397 WREG32(mmSRBM_SOFT_RESET, tmp);398 tmp = RREG32(mmSRBM_SOFT_RESET);399 400 /* Wait a little for things to settle down */401 udelay(50);402 }403 404 return 0;405}406 407static int cz_ih_set_clockgating_state(void *handle,408 enum amd_clockgating_state state)409{410 // TODO411 return 0;412}413 414static int cz_ih_set_powergating_state(void *handle,415 enum amd_powergating_state state)416{417 // TODO418 return 0;419}420 421static const struct amd_ip_funcs cz_ih_ip_funcs = {422 .name = "cz_ih",423 .early_init = cz_ih_early_init,424 .late_init = NULL,425 .sw_init = cz_ih_sw_init,426 .sw_fini = cz_ih_sw_fini,427 .hw_init = cz_ih_hw_init,428 .hw_fini = cz_ih_hw_fini,429 .suspend = cz_ih_suspend,430 .resume = cz_ih_resume,431 .is_idle = cz_ih_is_idle,432 .wait_for_idle = cz_ih_wait_for_idle,433 .soft_reset = cz_ih_soft_reset,434 .set_clockgating_state = cz_ih_set_clockgating_state,435 .set_powergating_state = cz_ih_set_powergating_state,436 .dump_ip_state = NULL,437 .print_ip_state = NULL,438};439 440static const struct amdgpu_ih_funcs cz_ih_funcs = {441 .get_wptr = cz_ih_get_wptr,442 .decode_iv = cz_ih_decode_iv,443 .set_rptr = cz_ih_set_rptr444};445 446static void cz_ih_set_interrupt_funcs(struct amdgpu_device *adev)447{448 adev->irq.ih_funcs = &cz_ih_funcs;449}450 451const struct amdgpu_ip_block_version cz_ih_ip_block =452{453 .type = AMD_IP_BLOCK_TYPE_IH,454 .major = 3,455 .minor = 0,456 .rev = 0,457 .funcs = &cz_ih_ip_funcs,458};459