451 lines · c
1/*2 * Copyright 2014 Advanced Micro Devices, Inc.3 *4 * Permission is hereby granted, free of charge, to any person obtaining a5 * copy of this software and associated documentation files (the "Software"),6 * to deal in the Software without restriction, including without limitation7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,8 * and/or sell copies of the Software, and to permit persons to whom the9 * Software is furnished to do so, subject to the following conditions:10 *11 * The above copyright notice and this permission notice shall be included in12 * all copies or substantial portions of the Software.13 *14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR20 * OTHER DEALINGS IN THE SOFTWARE.21 *22 */23 24#include <linux/pci.h>25 26#include "amdgpu.h"27#include "amdgpu_ih.h"28#include "vid.h"29 30#include "oss/oss_2_4_d.h"31#include "oss/oss_2_4_sh_mask.h"32 33#include "bif/bif_5_1_d.h"34#include "bif/bif_5_1_sh_mask.h"35 36/*37 * Interrupts38 * Starting with r6xx, interrupts are handled via a ring buffer.39 * Ring buffers are areas of GPU accessible memory that the GPU40 * writes interrupt vectors into and the host reads vectors out of.41 * There is a rptr (read pointer) that determines where the42 * host is currently reading, and a wptr (write pointer)43 * which determines where the GPU has written. When the44 * pointers are equal, the ring is idle. When the GPU45 * writes vectors to the ring buffer, it increments the46 * wptr. When there is an interrupt, the host then starts47 * fetching commands and processing them until the pointers are48 * equal again at which point it updates the rptr.49 */50 51static void iceland_ih_set_interrupt_funcs(struct amdgpu_device *adev);52 53/**54 * iceland_ih_enable_interrupts - Enable the interrupt ring buffer55 *56 * @adev: amdgpu_device pointer57 *58 * Enable the interrupt ring buffer (VI).59 */60static void iceland_ih_enable_interrupts(struct amdgpu_device *adev)61{62 u32 ih_cntl = RREG32(mmIH_CNTL);63 u32 ih_rb_cntl = RREG32(mmIH_RB_CNTL);64 65 ih_cntl = REG_SET_FIELD(ih_cntl, IH_CNTL, ENABLE_INTR, 1);66 ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, RB_ENABLE, 1);67 WREG32(mmIH_CNTL, ih_cntl);68 WREG32(mmIH_RB_CNTL, ih_rb_cntl);69 adev->irq.ih.enabled = true;70}71 72/**73 * iceland_ih_disable_interrupts - Disable the interrupt ring buffer74 *75 * @adev: amdgpu_device pointer76 *77 * Disable the interrupt ring buffer (VI).78 */79static void iceland_ih_disable_interrupts(struct amdgpu_device *adev)80{81 u32 ih_rb_cntl = RREG32(mmIH_RB_CNTL);82 u32 ih_cntl = RREG32(mmIH_CNTL);83 84 ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, RB_ENABLE, 0);85 ih_cntl = REG_SET_FIELD(ih_cntl, IH_CNTL, ENABLE_INTR, 0);86 WREG32(mmIH_RB_CNTL, ih_rb_cntl);87 WREG32(mmIH_CNTL, ih_cntl);88 /* set rptr, wptr to 0 */89 WREG32(mmIH_RB_RPTR, 0);90 WREG32(mmIH_RB_WPTR, 0);91 adev->irq.ih.enabled = false;92 adev->irq.ih.rptr = 0;93}94 95/**96 * iceland_ih_irq_init - init and enable the interrupt ring97 *98 * @adev: amdgpu_device pointer99 *100 * Allocate a ring buffer for the interrupt controller,101 * enable the RLC, disable interrupts, enable the IH102 * ring buffer and enable it (VI).103 * Called at device load and reume.104 * Returns 0 for success, errors for failure.105 */106static int iceland_ih_irq_init(struct amdgpu_device *adev)107{108 struct amdgpu_ih_ring *ih = &adev->irq.ih;109 int rb_bufsz;110 u32 interrupt_cntl, ih_cntl, ih_rb_cntl;111 112 /* disable irqs */113 iceland_ih_disable_interrupts(adev);114 115 /* setup interrupt control */116 WREG32(mmINTERRUPT_CNTL2, adev->dummy_page_addr >> 8);117 interrupt_cntl = RREG32(mmINTERRUPT_CNTL);118 /* INTERRUPT_CNTL__IH_DUMMY_RD_OVERRIDE_MASK=0 - dummy read disabled with msi, enabled without msi119 * INTERRUPT_CNTL__IH_DUMMY_RD_OVERRIDE_MASK=1 - dummy read controlled by IH_DUMMY_RD_EN120 */121 interrupt_cntl = REG_SET_FIELD(interrupt_cntl, INTERRUPT_CNTL, IH_DUMMY_RD_OVERRIDE, 0);122 /* INTERRUPT_CNTL__IH_REQ_NONSNOOP_EN_MASK=1 if ring is in non-cacheable memory, e.g., vram */123 interrupt_cntl = REG_SET_FIELD(interrupt_cntl, INTERRUPT_CNTL, IH_REQ_NONSNOOP_EN, 0);124 WREG32(mmINTERRUPT_CNTL, interrupt_cntl);125 126 /* Ring Buffer base. [39:8] of 40-bit address of the beginning of the ring buffer*/127 WREG32(mmIH_RB_BASE, adev->irq.ih.gpu_addr >> 8);128 129 rb_bufsz = order_base_2(adev->irq.ih.ring_size / 4);130 ih_rb_cntl = REG_SET_FIELD(0, IH_RB_CNTL, WPTR_OVERFLOW_ENABLE, 1);131 ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, WPTR_OVERFLOW_CLEAR, 1);132 ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, RB_SIZE, rb_bufsz);133 134 /* Ring Buffer write pointer writeback. If enabled, IH_RB_WPTR register value is written to memory */135 ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, WPTR_WRITEBACK_ENABLE, 1);136 137 /* set the writeback address whether it's enabled or not */138 WREG32(mmIH_RB_WPTR_ADDR_LO, lower_32_bits(ih->wptr_addr));139 WREG32(mmIH_RB_WPTR_ADDR_HI, upper_32_bits(ih->wptr_addr) & 0xFF);140 141 WREG32(mmIH_RB_CNTL, ih_rb_cntl);142 143 /* set rptr, wptr to 0 */144 WREG32(mmIH_RB_RPTR, 0);145 WREG32(mmIH_RB_WPTR, 0);146 147 /* Default settings for IH_CNTL (disabled at first) */148 ih_cntl = RREG32(mmIH_CNTL);149 ih_cntl = REG_SET_FIELD(ih_cntl, IH_CNTL, MC_VMID, 0);150 151 if (adev->irq.msi_enabled)152 ih_cntl = REG_SET_FIELD(ih_cntl, IH_CNTL, RPTR_REARM, 1);153 WREG32(mmIH_CNTL, ih_cntl);154 155 pci_set_master(adev->pdev);156 157 /* enable interrupts */158 iceland_ih_enable_interrupts(adev);159 160 return 0;161}162 163/**164 * iceland_ih_irq_disable - disable interrupts165 *166 * @adev: amdgpu_device pointer167 *168 * Disable interrupts on the hw (VI).169 */170static void iceland_ih_irq_disable(struct amdgpu_device *adev)171{172 iceland_ih_disable_interrupts(adev);173 174 /* Wait and acknowledge irq */175 mdelay(1);176}177 178/**179 * iceland_ih_get_wptr - get the IH ring buffer wptr180 *181 * @adev: amdgpu_device pointer182 * @ih: IH ring buffer to fetch wptr183 *184 * Get the IH ring buffer wptr from either the register185 * or the writeback memory buffer (VI). Also check for186 * ring buffer overflow and deal with it.187 * Used by cz_irq_process(VI).188 * Returns the value of the wptr.189 */190static u32 iceland_ih_get_wptr(struct amdgpu_device *adev,191 struct amdgpu_ih_ring *ih)192{193 u32 wptr, tmp;194 195 wptr = le32_to_cpu(*ih->wptr_cpu);196 197 if (!REG_GET_FIELD(wptr, IH_RB_WPTR, RB_OVERFLOW))198 goto out;199 200 /* Double check that the overflow wasn't already cleared. */201 wptr = RREG32(mmIH_RB_WPTR);202 203 if (!REG_GET_FIELD(wptr, IH_RB_WPTR, RB_OVERFLOW))204 goto out;205 206 wptr = REG_SET_FIELD(wptr, IH_RB_WPTR, RB_OVERFLOW, 0);207 /* When a ring buffer overflow happen start parsing interrupt208 * from the last not overwritten vector (wptr + 16). Hopefully209 * this should allow us to catchup.210 */211 dev_warn(adev->dev, "IH ring buffer overflow (0x%08X, 0x%08X, 0x%08X)\n",212 wptr, ih->rptr, (wptr + 16) & ih->ptr_mask);213 ih->rptr = (wptr + 16) & ih->ptr_mask;214 tmp = RREG32(mmIH_RB_CNTL);215 tmp = REG_SET_FIELD(tmp, IH_RB_CNTL, WPTR_OVERFLOW_CLEAR, 1);216 WREG32(mmIH_RB_CNTL, tmp);217 218 /* Unset the CLEAR_OVERFLOW bit immediately so new overflows219 * can be detected.220 */221 tmp = REG_SET_FIELD(tmp, IH_RB_CNTL, WPTR_OVERFLOW_CLEAR, 0);222 WREG32(mmIH_RB_CNTL, tmp);223 224out:225 return (wptr & ih->ptr_mask);226}227 228/**229 * iceland_ih_decode_iv - decode an interrupt vector230 *231 * @adev: amdgpu_device pointer232 * @ih: IH ring buffer to decode233 * @entry: IV entry to place decoded information into234 *235 * Decodes the interrupt vector at the current rptr236 * position and also advance the position.237 */238static void iceland_ih_decode_iv(struct amdgpu_device *adev,239 struct amdgpu_ih_ring *ih,240 struct amdgpu_iv_entry *entry)241{242 /* wptr/rptr are in bytes! */243 u32 ring_index = ih->rptr >> 2;244 uint32_t dw[4];245 246 dw[0] = le32_to_cpu(ih->ring[ring_index + 0]);247 dw[1] = le32_to_cpu(ih->ring[ring_index + 1]);248 dw[2] = le32_to_cpu(ih->ring[ring_index + 2]);249 dw[3] = le32_to_cpu(ih->ring[ring_index + 3]);250 251 entry->client_id = AMDGPU_IRQ_CLIENTID_LEGACY;252 entry->src_id = dw[0] & 0xff;253 entry->src_data[0] = dw[1] & 0xfffffff;254 entry->ring_id = dw[2] & 0xff;255 entry->vmid = (dw[2] >> 8) & 0xff;256 entry->pasid = (dw[2] >> 16) & 0xffff;257 258 /* wptr/rptr are in bytes! */259 ih->rptr += 16;260}261 262/**263 * iceland_ih_set_rptr - set the IH ring buffer rptr264 *265 * @adev: amdgpu_device pointer266 * @ih: IH ring buffer to set rptr267 *268 * Set the IH ring buffer rptr.269 */270static void iceland_ih_set_rptr(struct amdgpu_device *adev,271 struct amdgpu_ih_ring *ih)272{273 WREG32(mmIH_RB_RPTR, ih->rptr);274}275 276static int iceland_ih_early_init(void *handle)277{278 struct amdgpu_device *adev = (struct amdgpu_device *)handle;279 int ret;280 281 ret = amdgpu_irq_add_domain(adev);282 if (ret)283 return ret;284 285 iceland_ih_set_interrupt_funcs(adev);286 287 return 0;288}289 290static int iceland_ih_sw_init(void *handle)291{292 int r;293 struct amdgpu_device *adev = (struct amdgpu_device *)handle;294 295 r = amdgpu_ih_ring_init(adev, &adev->irq.ih, 64 * 1024, false);296 if (r)297 return r;298 299 r = amdgpu_irq_init(adev);300 301 return r;302}303 304static int iceland_ih_sw_fini(void *handle)305{306 struct amdgpu_device *adev = (struct amdgpu_device *)handle;307 308 amdgpu_irq_fini_sw(adev);309 amdgpu_irq_remove_domain(adev);310 311 return 0;312}313 314static int iceland_ih_hw_init(void *handle)315{316 struct amdgpu_device *adev = (struct amdgpu_device *)handle;317 318 return iceland_ih_irq_init(adev);319}320 321static int iceland_ih_hw_fini(void *handle)322{323 struct amdgpu_device *adev = (struct amdgpu_device *)handle;324 325 iceland_ih_irq_disable(adev);326 327 return 0;328}329 330static int iceland_ih_suspend(void *handle)331{332 struct amdgpu_device *adev = (struct amdgpu_device *)handle;333 334 return iceland_ih_hw_fini(adev);335}336 337static int iceland_ih_resume(void *handle)338{339 struct amdgpu_device *adev = (struct amdgpu_device *)handle;340 341 return iceland_ih_hw_init(adev);342}343 344static bool iceland_ih_is_idle(void *handle)345{346 struct amdgpu_device *adev = (struct amdgpu_device *)handle;347 u32 tmp = RREG32(mmSRBM_STATUS);348 349 if (REG_GET_FIELD(tmp, SRBM_STATUS, IH_BUSY))350 return false;351 352 return true;353}354 355static int iceland_ih_wait_for_idle(void *handle)356{357 unsigned i;358 u32 tmp;359 struct amdgpu_device *adev = (struct amdgpu_device *)handle;360 361 for (i = 0; i < adev->usec_timeout; i++) {362 /* read MC_STATUS */363 tmp = RREG32(mmSRBM_STATUS);364 if (!REG_GET_FIELD(tmp, SRBM_STATUS, IH_BUSY))365 return 0;366 udelay(1);367 }368 return -ETIMEDOUT;369}370 371static int iceland_ih_soft_reset(void *handle)372{373 u32 srbm_soft_reset = 0;374 struct amdgpu_device *adev = (struct amdgpu_device *)handle;375 u32 tmp = RREG32(mmSRBM_STATUS);376 377 if (tmp & SRBM_STATUS__IH_BUSY_MASK)378 srbm_soft_reset = REG_SET_FIELD(srbm_soft_reset, SRBM_SOFT_RESET,379 SOFT_RESET_IH, 1);380 381 if (srbm_soft_reset) {382 tmp = RREG32(mmSRBM_SOFT_RESET);383 tmp |= srbm_soft_reset;384 dev_info(adev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);385 WREG32(mmSRBM_SOFT_RESET, tmp);386 tmp = RREG32(mmSRBM_SOFT_RESET);387 388 udelay(50);389 390 tmp &= ~srbm_soft_reset;391 WREG32(mmSRBM_SOFT_RESET, tmp);392 tmp = RREG32(mmSRBM_SOFT_RESET);393 394 /* Wait a little for things to settle down */395 udelay(50);396 }397 398 return 0;399}400 401static int iceland_ih_set_clockgating_state(void *handle,402 enum amd_clockgating_state state)403{404 return 0;405}406 407static int iceland_ih_set_powergating_state(void *handle,408 enum amd_powergating_state state)409{410 return 0;411}412 413static const struct amd_ip_funcs iceland_ih_ip_funcs = {414 .name = "iceland_ih",415 .early_init = iceland_ih_early_init,416 .late_init = NULL,417 .sw_init = iceland_ih_sw_init,418 .sw_fini = iceland_ih_sw_fini,419 .hw_init = iceland_ih_hw_init,420 .hw_fini = iceland_ih_hw_fini,421 .suspend = iceland_ih_suspend,422 .resume = iceland_ih_resume,423 .is_idle = iceland_ih_is_idle,424 .wait_for_idle = iceland_ih_wait_for_idle,425 .soft_reset = iceland_ih_soft_reset,426 .set_clockgating_state = iceland_ih_set_clockgating_state,427 .set_powergating_state = iceland_ih_set_powergating_state,428 .dump_ip_state = NULL,429 .print_ip_state = NULL,430};431 432static const struct amdgpu_ih_funcs iceland_ih_funcs = {433 .get_wptr = iceland_ih_get_wptr,434 .decode_iv = iceland_ih_decode_iv,435 .set_rptr = iceland_ih_set_rptr436};437 438static void iceland_ih_set_interrupt_funcs(struct amdgpu_device *adev)439{440 adev->irq.ih_funcs = &iceland_ih_funcs;441}442 443const struct amdgpu_ip_block_version iceland_ih_ip_block =444{445 .type = AMD_IP_BLOCK_TYPE_IH,446 .major = 2,447 .minor = 4,448 .rev = 0,449 .funcs = &iceland_ih_ip_funcs,450};451