1170 lines · c
1/*2 * Copyright 2019 Advanced Micro Devices, Inc.3 *4 * Permission is hereby granted, free of charge, to any person obtaining a5 * copy of this software and associated documentation files (the "Software"),6 * to deal in the Software without restriction, including without limitation7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,8 * and/or sell copies of the Software, and to permit persons to whom the9 * Software is furnished to do so, subject to the following conditions:10 *11 * The above copyright notice and this permission notice shall be included in12 * all copies or substantial portions of the Software.13 *14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR20 * OTHER DEALINGS IN THE SOFTWARE.21 *22 */23#include <linux/firmware.h>24#include <linux/pci.h>25 26#include <drm/drm_cache.h>27 28#include "amdgpu.h"29#include "amdgpu_atomfirmware.h"30#include "gmc_v10_0.h"31#include "umc_v8_7.h"32 33#include "athub/athub_2_0_0_sh_mask.h"34#include "athub/athub_2_0_0_offset.h"35#include "dcn/dcn_2_0_0_offset.h"36#include "dcn/dcn_2_0_0_sh_mask.h"37#include "oss/osssys_5_0_0_offset.h"38#include "ivsrcid/vmc/irqsrcs_vmc_1_0.h"39#include "navi10_enum.h"40 41#include "soc15.h"42#include "soc15d.h"43#include "soc15_common.h"44 45#include "nbio_v2_3.h"46 47#include "gfxhub_v2_0.h"48#include "gfxhub_v2_1.h"49#include "mmhub_v2_0.h"50#include "mmhub_v2_3.h"51#include "athub_v2_0.h"52#include "athub_v2_1.h"53 54static int gmc_v10_0_ecc_interrupt_state(struct amdgpu_device *adev,55 struct amdgpu_irq_src *src,56 unsigned int type,57 enum amdgpu_interrupt_state state)58{59 return 0;60}61 62static int63gmc_v10_0_vm_fault_interrupt_state(struct amdgpu_device *adev,64 struct amdgpu_irq_src *src, unsigned int type,65 enum amdgpu_interrupt_state state)66{67 switch (state) {68 case AMDGPU_IRQ_STATE_DISABLE:69 /* MM HUB */70 amdgpu_gmc_set_vm_fault_masks(adev, AMDGPU_MMHUB0(0), false);71 /* GFX HUB */72 /* This works because this interrupt is only73 * enabled at init/resume and disabled in74 * fini/suspend, so the overall state doesn't75 * change over the course of suspend/resume.76 */77 if (!adev->in_s0ix)78 amdgpu_gmc_set_vm_fault_masks(adev, AMDGPU_GFXHUB(0), false);79 break;80 case AMDGPU_IRQ_STATE_ENABLE:81 /* MM HUB */82 amdgpu_gmc_set_vm_fault_masks(adev, AMDGPU_MMHUB0(0), true);83 /* GFX HUB */84 /* This works because this interrupt is only85 * enabled at init/resume and disabled in86 * fini/suspend, so the overall state doesn't87 * change over the course of suspend/resume.88 */89 if (!adev->in_s0ix)90 amdgpu_gmc_set_vm_fault_masks(adev, AMDGPU_GFXHUB(0), true);91 break;92 default:93 break;94 }95 96 return 0;97}98 99static int gmc_v10_0_process_interrupt(struct amdgpu_device *adev,100 struct amdgpu_irq_src *source,101 struct amdgpu_iv_entry *entry)102{103 uint32_t vmhub_index = entry->client_id == SOC15_IH_CLIENTID_VMC ?104 AMDGPU_MMHUB0(0) : AMDGPU_GFXHUB(0);105 struct amdgpu_vmhub *hub = &adev->vmhub[vmhub_index];106 bool retry_fault = !!(entry->src_data[1] & 0x80);107 bool write_fault = !!(entry->src_data[1] & 0x20);108 struct amdgpu_task_info *task_info;109 uint32_t status = 0;110 u64 addr;111 112 addr = (u64)entry->src_data[0] << 12;113 addr |= ((u64)entry->src_data[1] & 0xf) << 44;114 115 if (retry_fault) {116 /* Returning 1 here also prevents sending the IV to the KFD */117 118 /* Process it onyl if it's the first fault for this address */119 if (entry->ih != &adev->irq.ih_soft &&120 amdgpu_gmc_filter_faults(adev, entry->ih, addr, entry->pasid,121 entry->timestamp))122 return 1;123 124 /* Delegate it to a different ring if the hardware hasn't125 * already done it.126 */127 if (entry->ih == &adev->irq.ih) {128 amdgpu_irq_delegate(adev, entry, 8);129 return 1;130 }131 132 /* Try to handle the recoverable page faults by filling page133 * tables134 */135 if (amdgpu_vm_handle_fault(adev, entry->pasid, 0, 0, addr,136 entry->timestamp, write_fault))137 return 1;138 }139 140 if (!amdgpu_sriov_vf(adev)) {141 /*142 * Issue a dummy read to wait for the status register to143 * be updated to avoid reading an incorrect value due to144 * the new fast GRBM interface.145 */146 if ((entry->vmid_src == AMDGPU_GFXHUB(0)) &&147 (amdgpu_ip_version(adev, GC_HWIP, 0) <148 IP_VERSION(10, 3, 0)))149 RREG32(hub->vm_l2_pro_fault_status);150 151 status = RREG32(hub->vm_l2_pro_fault_status);152 WREG32_P(hub->vm_l2_pro_fault_cntl, 1, ~1);153 154 amdgpu_vm_update_fault_cache(adev, entry->pasid, addr, status,155 entry->vmid_src ? AMDGPU_MMHUB0(0) : AMDGPU_GFXHUB(0));156 }157 158 if (!printk_ratelimit())159 return 0;160 161 dev_err(adev->dev,162 "[%s] page fault (src_id:%u ring:%u vmid:%u pasid:%u)\n",163 entry->vmid_src ? "mmhub" : "gfxhub",164 entry->src_id, entry->ring_id, entry->vmid, entry->pasid);165 task_info = amdgpu_vm_get_task_info_pasid(adev, entry->pasid);166 if (task_info) {167 dev_err(adev->dev,168 " in process %s pid %d thread %s pid %d\n",169 task_info->process_name, task_info->tgid,170 task_info->task_name, task_info->pid);171 amdgpu_vm_put_task_info(task_info);172 }173 174 dev_err(adev->dev, " in page starting at address 0x%016llx from client 0x%x (%s)\n",175 addr, entry->client_id,176 soc15_ih_clientid_name[entry->client_id]);177 178 if (!amdgpu_sriov_vf(adev))179 hub->vmhub_funcs->print_l2_protection_fault_status(adev,180 status);181 182 return 0;183}184 185static const struct amdgpu_irq_src_funcs gmc_v10_0_irq_funcs = {186 .set = gmc_v10_0_vm_fault_interrupt_state,187 .process = gmc_v10_0_process_interrupt,188};189 190static const struct amdgpu_irq_src_funcs gmc_v10_0_ecc_funcs = {191 .set = gmc_v10_0_ecc_interrupt_state,192 .process = amdgpu_umc_process_ecc_irq,193};194 195static void gmc_v10_0_set_irq_funcs(struct amdgpu_device *adev)196{197 adev->gmc.vm_fault.num_types = 1;198 adev->gmc.vm_fault.funcs = &gmc_v10_0_irq_funcs;199 200 if (!amdgpu_sriov_vf(adev)) {201 adev->gmc.ecc_irq.num_types = 1;202 adev->gmc.ecc_irq.funcs = &gmc_v10_0_ecc_funcs;203 }204}205 206/**207 * gmc_v10_0_use_invalidate_semaphore - judge whether to use semaphore208 *209 * @adev: amdgpu_device pointer210 * @vmhub: vmhub type211 *212 */213static bool gmc_v10_0_use_invalidate_semaphore(struct amdgpu_device *adev,214 uint32_t vmhub)215{216 return ((vmhub == AMDGPU_MMHUB0(0)) &&217 (!amdgpu_sriov_vf(adev)));218}219 220static bool gmc_v10_0_get_atc_vmid_pasid_mapping_info(221 struct amdgpu_device *adev,222 uint8_t vmid, uint16_t *p_pasid)223{224 uint32_t value;225 226 value = RREG32(SOC15_REG_OFFSET(ATHUB, 0, mmATC_VMID0_PASID_MAPPING)227 + vmid);228 *p_pasid = value & ATC_VMID0_PASID_MAPPING__PASID_MASK;229 230 return !!(value & ATC_VMID0_PASID_MAPPING__VALID_MASK);231}232 233/*234 * GART235 * VMID 0 is the physical GPU addresses as used by the kernel.236 * VMIDs 1-15 are used for userspace clients and are handled237 * by the amdgpu vm/hsa code.238 */239 240/**241 * gmc_v10_0_flush_gpu_tlb - gart tlb flush callback242 *243 * @adev: amdgpu_device pointer244 * @vmid: vm instance to flush245 * @vmhub: vmhub type246 * @flush_type: the flush type247 *248 * Flush the TLB for the requested page table.249 */250static void gmc_v10_0_flush_gpu_tlb(struct amdgpu_device *adev, uint32_t vmid,251 uint32_t vmhub, uint32_t flush_type)252{253 bool use_semaphore = gmc_v10_0_use_invalidate_semaphore(adev, vmhub);254 struct amdgpu_vmhub *hub = &adev->vmhub[vmhub];255 u32 inv_req = hub->vmhub_funcs->get_invalidate_req(vmid, flush_type);256 /* Use register 17 for GART */257 const unsigned int eng = 17;258 unsigned char hub_ip = 0;259 u32 sem, req, ack;260 unsigned int i;261 u32 tmp;262 263 sem = hub->vm_inv_eng0_sem + hub->eng_distance * eng;264 req = hub->vm_inv_eng0_req + hub->eng_distance * eng;265 ack = hub->vm_inv_eng0_ack + hub->eng_distance * eng;266 267 /* flush hdp cache */268 adev->hdp.funcs->flush_hdp(adev, NULL);269 270 /* This is necessary for SRIOV as well as for GFXOFF to function271 * properly under bare metal272 */273 if (adev->gfx.kiq[0].ring.sched.ready && !adev->enable_mes &&274 (amdgpu_sriov_runtime(adev) || !amdgpu_sriov_vf(adev))) {275 amdgpu_gmc_fw_reg_write_reg_wait(adev, req, ack, inv_req,276 1 << vmid, GET_INST(GC, 0));277 return;278 }279 280 /* This path is needed before KIQ/MES/GFXOFF are set up */281 hub_ip = (vmhub == AMDGPU_GFXHUB(0)) ? GC_HWIP : MMHUB_HWIP;282 283 spin_lock(&adev->gmc.invalidate_lock);284 /*285 * It may lose gpuvm invalidate acknowldege state across power-gating286 * off cycle, add semaphore acquire before invalidation and semaphore287 * release after invalidation to avoid entering power gated state288 * to WA the Issue289 */290 291 /* TODO: It needs to continue working on debugging with semaphore for GFXHUB as well. */292 if (use_semaphore) {293 for (i = 0; i < adev->usec_timeout; i++) {294 /* a read return value of 1 means semaphore acuqire */295 tmp = RREG32_RLC_NO_KIQ(sem, hub_ip);296 if (tmp & 0x1)297 break;298 udelay(1);299 }300 301 if (i >= adev->usec_timeout)302 DRM_ERROR("Timeout waiting for sem acquire in VM flush!\n");303 }304 305 WREG32_RLC_NO_KIQ(req, inv_req, hub_ip);306 307 /*308 * Issue a dummy read to wait for the ACK register to be cleared309 * to avoid a false ACK due to the new fast GRBM interface.310 */311 if ((vmhub == AMDGPU_GFXHUB(0)) &&312 (amdgpu_ip_version(adev, GC_HWIP, 0) < IP_VERSION(10, 3, 0)))313 RREG32_RLC_NO_KIQ(req, hub_ip);314 315 /* Wait for ACK with a delay.*/316 for (i = 0; i < adev->usec_timeout; i++) {317 tmp = RREG32_RLC_NO_KIQ(ack, hub_ip);318 tmp &= 1 << vmid;319 if (tmp)320 break;321 322 udelay(1);323 }324 325 /* TODO: It needs to continue working on debugging with semaphore for GFXHUB as well. */326 if (use_semaphore)327 WREG32_RLC_NO_KIQ(sem, 0, hub_ip);328 329 spin_unlock(&adev->gmc.invalidate_lock);330 331 if (i >= adev->usec_timeout)332 dev_err(adev->dev, "Timeout waiting for VM flush hub: %d!\n",333 vmhub);334}335 336/**337 * gmc_v10_0_flush_gpu_tlb_pasid - tlb flush via pasid338 *339 * @adev: amdgpu_device pointer340 * @pasid: pasid to be flush341 * @flush_type: the flush type342 * @all_hub: Used with PACKET3_INVALIDATE_TLBS_ALL_HUB()343 * @inst: is used to select which instance of KIQ to use for the invalidation344 *345 * Flush the TLB for the requested pasid.346 */347static void gmc_v10_0_flush_gpu_tlb_pasid(struct amdgpu_device *adev,348 uint16_t pasid, uint32_t flush_type,349 bool all_hub, uint32_t inst)350{351 uint16_t queried;352 int vmid, i;353 354 for (vmid = 1; vmid < AMDGPU_NUM_VMID; vmid++) {355 bool valid;356 357 valid = gmc_v10_0_get_atc_vmid_pasid_mapping_info(adev, vmid,358 &queried);359 if (!valid || queried != pasid)360 continue;361 362 if (all_hub) {363 for_each_set_bit(i, adev->vmhubs_mask,364 AMDGPU_MAX_VMHUBS)365 gmc_v10_0_flush_gpu_tlb(adev, vmid, i,366 flush_type);367 } else {368 gmc_v10_0_flush_gpu_tlb(adev, vmid, AMDGPU_GFXHUB(0),369 flush_type);370 }371 }372}373 374static uint64_t gmc_v10_0_emit_flush_gpu_tlb(struct amdgpu_ring *ring,375 unsigned int vmid, uint64_t pd_addr)376{377 bool use_semaphore = gmc_v10_0_use_invalidate_semaphore(ring->adev, ring->vm_hub);378 struct amdgpu_vmhub *hub = &ring->adev->vmhub[ring->vm_hub];379 uint32_t req = hub->vmhub_funcs->get_invalidate_req(vmid, 0);380 unsigned int eng = ring->vm_inv_eng;381 382 /*383 * It may lose gpuvm invalidate acknowldege state across power-gating384 * off cycle, add semaphore acquire before invalidation and semaphore385 * release after invalidation to avoid entering power gated state386 * to WA the Issue387 */388 389 /* TODO: It needs to continue working on debugging with semaphore for GFXHUB as well. */390 if (use_semaphore)391 /* a read return value of 1 means semaphore acuqire */392 amdgpu_ring_emit_reg_wait(ring,393 hub->vm_inv_eng0_sem +394 hub->eng_distance * eng, 0x1, 0x1);395 396 amdgpu_ring_emit_wreg(ring, hub->ctx0_ptb_addr_lo32 +397 (hub->ctx_addr_distance * vmid),398 lower_32_bits(pd_addr));399 400 amdgpu_ring_emit_wreg(ring, hub->ctx0_ptb_addr_hi32 +401 (hub->ctx_addr_distance * vmid),402 upper_32_bits(pd_addr));403 404 amdgpu_ring_emit_reg_write_reg_wait(ring, hub->vm_inv_eng0_req +405 hub->eng_distance * eng,406 hub->vm_inv_eng0_ack +407 hub->eng_distance * eng,408 req, 1 << vmid);409 410 /* TODO: It needs to continue working on debugging with semaphore for GFXHUB as well. */411 if (use_semaphore)412 /*413 * add semaphore release after invalidation,414 * write with 0 means semaphore release415 */416 amdgpu_ring_emit_wreg(ring, hub->vm_inv_eng0_sem +417 hub->eng_distance * eng, 0);418 419 return pd_addr;420}421 422static void gmc_v10_0_emit_pasid_mapping(struct amdgpu_ring *ring, unsigned int vmid,423 unsigned int pasid)424{425 struct amdgpu_device *adev = ring->adev;426 uint32_t reg;427 428 /* MES fw manages IH_VMID_x_LUT updating */429 if (ring->is_mes_queue)430 return;431 432 if (ring->vm_hub == AMDGPU_GFXHUB(0))433 reg = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_VMID_0_LUT) + vmid;434 else435 reg = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_VMID_0_LUT_MM) + vmid;436 437 amdgpu_ring_emit_wreg(ring, reg, pasid);438}439 440/*441 * PTE format on NAVI 10:442 * 63:59 reserved443 * 58 reserved and for sienna_cichlid is used for MALL noalloc444 * 57 reserved445 * 56 F446 * 55 L447 * 54 reserved448 * 53:52 SW449 * 51 T450 * 50:48 mtype451 * 47:12 4k physical page base address452 * 11:7 fragment453 * 6 write454 * 5 read455 * 4 exe456 * 3 Z457 * 2 snooped458 * 1 system459 * 0 valid460 *461 * PDE format on NAVI 10:462 * 63:59 block fragment size463 * 58:55 reserved464 * 54 P465 * 53:48 reserved466 * 47:6 physical base address of PD or PTE467 * 5:3 reserved468 * 2 C469 * 1 system470 * 0 valid471 */472 473static uint64_t gmc_v10_0_map_mtype(struct amdgpu_device *adev, uint32_t flags)474{475 switch (flags) {476 case AMDGPU_VM_MTYPE_DEFAULT:477 return AMDGPU_PTE_MTYPE_NV10(0ULL, MTYPE_NC);478 case AMDGPU_VM_MTYPE_NC:479 return AMDGPU_PTE_MTYPE_NV10(0ULL, MTYPE_NC);480 case AMDGPU_VM_MTYPE_WC:481 return AMDGPU_PTE_MTYPE_NV10(0ULL, MTYPE_WC);482 case AMDGPU_VM_MTYPE_CC:483 return AMDGPU_PTE_MTYPE_NV10(0ULL, MTYPE_CC);484 case AMDGPU_VM_MTYPE_UC:485 return AMDGPU_PTE_MTYPE_NV10(0ULL, MTYPE_UC);486 default:487 return AMDGPU_PTE_MTYPE_NV10(0ULL, MTYPE_NC);488 }489}490 491static void gmc_v10_0_get_vm_pde(struct amdgpu_device *adev, int level,492 uint64_t *addr, uint64_t *flags)493{494 if (!(*flags & AMDGPU_PDE_PTE) && !(*flags & AMDGPU_PTE_SYSTEM))495 *addr = amdgpu_gmc_vram_mc2pa(adev, *addr);496 BUG_ON(*addr & 0xFFFF00000000003FULL);497 498 if (!adev->gmc.translate_further)499 return;500 501 if (level == AMDGPU_VM_PDB1) {502 /* Set the block fragment size */503 if (!(*flags & AMDGPU_PDE_PTE))504 *flags |= AMDGPU_PDE_BFS(0x9);505 506 } else if (level == AMDGPU_VM_PDB0) {507 if (*flags & AMDGPU_PDE_PTE)508 *flags &= ~AMDGPU_PDE_PTE;509 else510 *flags |= AMDGPU_PTE_TF;511 }512}513 514static void gmc_v10_0_get_vm_pte(struct amdgpu_device *adev,515 struct amdgpu_bo_va_mapping *mapping,516 uint64_t *flags)517{518 struct amdgpu_bo *bo = mapping->bo_va->base.bo;519 520 *flags &= ~AMDGPU_PTE_EXECUTABLE;521 *flags |= mapping->flags & AMDGPU_PTE_EXECUTABLE;522 523 *flags &= ~AMDGPU_PTE_MTYPE_NV10_MASK;524 *flags |= (mapping->flags & AMDGPU_PTE_MTYPE_NV10_MASK);525 526 *flags &= ~AMDGPU_PTE_NOALLOC;527 *flags |= (mapping->flags & AMDGPU_PTE_NOALLOC);528 529 if (mapping->flags & AMDGPU_PTE_PRT) {530 *flags |= AMDGPU_PTE_PRT;531 *flags |= AMDGPU_PTE_SNOOPED;532 *flags |= AMDGPU_PTE_LOG;533 *flags |= AMDGPU_PTE_SYSTEM;534 *flags &= ~AMDGPU_PTE_VALID;535 }536 537 if (bo && bo->flags & (AMDGPU_GEM_CREATE_COHERENT |538 AMDGPU_GEM_CREATE_EXT_COHERENT |539 AMDGPU_GEM_CREATE_UNCACHED))540 *flags = AMDGPU_PTE_MTYPE_NV10(*flags, MTYPE_UC);541}542 543static unsigned int gmc_v10_0_get_vbios_fb_size(struct amdgpu_device *adev)544{545 u32 d1vga_control = RREG32_SOC15(DCE, 0, mmD1VGA_CONTROL);546 unsigned int size;547 548 if (REG_GET_FIELD(d1vga_control, D1VGA_CONTROL, D1VGA_MODE_ENABLE)) {549 size = AMDGPU_VBIOS_VGA_ALLOCATION;550 } else {551 u32 viewport;552 u32 pitch;553 554 viewport = RREG32_SOC15(DCE, 0, mmHUBP0_DCSURF_PRI_VIEWPORT_DIMENSION);555 pitch = RREG32_SOC15(DCE, 0, mmHUBPREQ0_DCSURF_SURFACE_PITCH);556 size = (REG_GET_FIELD(viewport,557 HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION, PRI_VIEWPORT_HEIGHT) *558 REG_GET_FIELD(pitch, HUBPREQ0_DCSURF_SURFACE_PITCH, PITCH) *559 4);560 }561 562 return size;563}564 565static const struct amdgpu_gmc_funcs gmc_v10_0_gmc_funcs = {566 .flush_gpu_tlb = gmc_v10_0_flush_gpu_tlb,567 .flush_gpu_tlb_pasid = gmc_v10_0_flush_gpu_tlb_pasid,568 .emit_flush_gpu_tlb = gmc_v10_0_emit_flush_gpu_tlb,569 .emit_pasid_mapping = gmc_v10_0_emit_pasid_mapping,570 .map_mtype = gmc_v10_0_map_mtype,571 .get_vm_pde = gmc_v10_0_get_vm_pde,572 .get_vm_pte = gmc_v10_0_get_vm_pte,573 .get_vbios_fb_size = gmc_v10_0_get_vbios_fb_size,574};575 576static void gmc_v10_0_set_gmc_funcs(struct amdgpu_device *adev)577{578 if (adev->gmc.gmc_funcs == NULL)579 adev->gmc.gmc_funcs = &gmc_v10_0_gmc_funcs;580}581 582static void gmc_v10_0_set_umc_funcs(struct amdgpu_device *adev)583{584 switch (amdgpu_ip_version(adev, UMC_HWIP, 0)) {585 case IP_VERSION(8, 7, 0):586 adev->umc.max_ras_err_cnt_per_query = UMC_V8_7_TOTAL_CHANNEL_NUM;587 adev->umc.channel_inst_num = UMC_V8_7_CHANNEL_INSTANCE_NUM;588 adev->umc.umc_inst_num = UMC_V8_7_UMC_INSTANCE_NUM;589 adev->umc.channel_offs = UMC_V8_7_PER_CHANNEL_OFFSET_SIENNA;590 adev->umc.retire_unit = 1;591 adev->umc.channel_idx_tbl = &umc_v8_7_channel_idx_tbl[0][0];592 adev->umc.ras = &umc_v8_7_ras;593 break;594 default:595 break;596 }597}598 599static void gmc_v10_0_set_mmhub_funcs(struct amdgpu_device *adev)600{601 switch (amdgpu_ip_version(adev, MMHUB_HWIP, 0)) {602 case IP_VERSION(2, 3, 0):603 case IP_VERSION(2, 4, 0):604 case IP_VERSION(2, 4, 1):605 adev->mmhub.funcs = &mmhub_v2_3_funcs;606 break;607 default:608 adev->mmhub.funcs = &mmhub_v2_0_funcs;609 break;610 }611}612 613static void gmc_v10_0_set_gfxhub_funcs(struct amdgpu_device *adev)614{615 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {616 case IP_VERSION(10, 3, 0):617 case IP_VERSION(10, 3, 2):618 case IP_VERSION(10, 3, 1):619 case IP_VERSION(10, 3, 4):620 case IP_VERSION(10, 3, 5):621 case IP_VERSION(10, 3, 6):622 case IP_VERSION(10, 3, 3):623 case IP_VERSION(10, 3, 7):624 adev->gfxhub.funcs = &gfxhub_v2_1_funcs;625 break;626 default:627 adev->gfxhub.funcs = &gfxhub_v2_0_funcs;628 break;629 }630}631 632 633static int gmc_v10_0_early_init(void *handle)634{635 struct amdgpu_device *adev = (struct amdgpu_device *)handle;636 637 gmc_v10_0_set_mmhub_funcs(adev);638 gmc_v10_0_set_gfxhub_funcs(adev);639 gmc_v10_0_set_gmc_funcs(adev);640 gmc_v10_0_set_irq_funcs(adev);641 gmc_v10_0_set_umc_funcs(adev);642 643 adev->gmc.shared_aperture_start = 0x2000000000000000ULL;644 adev->gmc.shared_aperture_end =645 adev->gmc.shared_aperture_start + (4ULL << 30) - 1;646 adev->gmc.private_aperture_start = 0x1000000000000000ULL;647 adev->gmc.private_aperture_end =648 adev->gmc.private_aperture_start + (4ULL << 30) - 1;649 adev->gmc.noretry_flags = AMDGPU_VM_NORETRY_FLAGS_TF;650 651 return 0;652}653 654static int gmc_v10_0_late_init(void *handle)655{656 struct amdgpu_device *adev = (struct amdgpu_device *)handle;657 int r;658 659 r = amdgpu_gmc_allocate_vm_inv_eng(adev);660 if (r)661 return r;662 663 r = amdgpu_gmc_ras_late_init(adev);664 if (r)665 return r;666 667 return amdgpu_irq_get(adev, &adev->gmc.vm_fault, 0);668}669 670static void gmc_v10_0_vram_gtt_location(struct amdgpu_device *adev,671 struct amdgpu_gmc *mc)672{673 u64 base = 0;674 675 base = adev->gfxhub.funcs->get_fb_location(adev);676 677 /* add the xgmi offset of the physical node */678 base += adev->gmc.xgmi.physical_node_id * adev->gmc.xgmi.node_segment_size;679 680 amdgpu_gmc_set_agp_default(adev, mc);681 amdgpu_gmc_vram_location(adev, &adev->gmc, base);682 amdgpu_gmc_gart_location(adev, mc, AMDGPU_GART_PLACEMENT_BEST_FIT);683 if (!amdgpu_sriov_vf(adev) && (amdgpu_agp == 1))684 amdgpu_gmc_agp_location(adev, mc);685 686 /* base offset of vram pages */687 adev->vm_manager.vram_base_offset = adev->gfxhub.funcs->get_mc_fb_offset(adev);688 689 /* add the xgmi offset of the physical node */690 adev->vm_manager.vram_base_offset +=691 adev->gmc.xgmi.physical_node_id * adev->gmc.xgmi.node_segment_size;692}693 694/**695 * gmc_v10_0_mc_init - initialize the memory controller driver params696 *697 * @adev: amdgpu_device pointer698 *699 * Look up the amount of vram, vram width, and decide how to place700 * vram and gart within the GPU's physical address space.701 * Returns 0 for success.702 */703static int gmc_v10_0_mc_init(struct amdgpu_device *adev)704{705 int r;706 707 /* size in MB on si */708 adev->gmc.mc_vram_size =709 adev->nbio.funcs->get_memsize(adev) * 1024ULL * 1024ULL;710 adev->gmc.real_vram_size = adev->gmc.mc_vram_size;711 712 if (!(adev->flags & AMD_IS_APU)) {713 r = amdgpu_device_resize_fb_bar(adev);714 if (r)715 return r;716 }717 adev->gmc.aper_base = pci_resource_start(adev->pdev, 0);718 adev->gmc.aper_size = pci_resource_len(adev->pdev, 0);719 720#ifdef CONFIG_X86_64721 if ((adev->flags & AMD_IS_APU) && !amdgpu_passthrough(adev)) {722 adev->gmc.aper_base = adev->gfxhub.funcs->get_mc_fb_offset(adev);723 adev->gmc.aper_size = adev->gmc.real_vram_size;724 }725#endif726 727 adev->gmc.visible_vram_size = adev->gmc.aper_size;728 729 /* set the gart size */730 if (amdgpu_gart_size == -1) {731 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {732 default:733 adev->gmc.gart_size = 512ULL << 20;734 break;735 case IP_VERSION(10, 3, 1): /* DCE SG support */736 case IP_VERSION(10, 3, 3): /* DCE SG support */737 case IP_VERSION(10, 3, 6): /* DCE SG support */738 case IP_VERSION(10, 3, 7): /* DCE SG support */739 adev->gmc.gart_size = 1024ULL << 20;740 break;741 }742 } else {743 adev->gmc.gart_size = (u64)amdgpu_gart_size << 20;744 }745 746 gmc_v10_0_vram_gtt_location(adev, &adev->gmc);747 748 return 0;749}750 751static int gmc_v10_0_gart_init(struct amdgpu_device *adev)752{753 int r;754 755 if (adev->gart.bo) {756 WARN(1, "NAVI10 PCIE GART already initialized\n");757 return 0;758 }759 760 /* Initialize common gart structure */761 r = amdgpu_gart_init(adev);762 if (r)763 return r;764 765 adev->gart.table_size = adev->gart.num_gpu_pages * 8;766 adev->gart.gart_pte_flags = AMDGPU_PTE_MTYPE_NV10(0ULL, MTYPE_UC) |767 AMDGPU_PTE_EXECUTABLE;768 769 return amdgpu_gart_table_vram_alloc(adev);770}771 772static int gmc_v10_0_sw_init(void *handle)773{774 int r, vram_width = 0, vram_type = 0, vram_vendor = 0;775 struct amdgpu_device *adev = (struct amdgpu_device *)handle;776 777 adev->gfxhub.funcs->init(adev);778 779 adev->mmhub.funcs->init(adev);780 781 spin_lock_init(&adev->gmc.invalidate_lock);782 783 if ((adev->flags & AMD_IS_APU) && amdgpu_emu_mode == 1) {784 adev->gmc.vram_type = AMDGPU_VRAM_TYPE_DDR4;785 adev->gmc.vram_width = 64;786 } else if (amdgpu_emu_mode == 1) {787 adev->gmc.vram_type = AMDGPU_VRAM_TYPE_GDDR6;788 adev->gmc.vram_width = 1 * 128; /* numchan * chansize */789 } else {790 r = amdgpu_atomfirmware_get_vram_info(adev,791 &vram_width, &vram_type, &vram_vendor);792 adev->gmc.vram_width = vram_width;793 794 adev->gmc.vram_type = vram_type;795 adev->gmc.vram_vendor = vram_vendor;796 }797 798 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {799 case IP_VERSION(10, 3, 0):800 adev->gmc.mall_size = 128 * 1024 * 1024;801 break;802 case IP_VERSION(10, 3, 2):803 adev->gmc.mall_size = 96 * 1024 * 1024;804 break;805 case IP_VERSION(10, 3, 4):806 adev->gmc.mall_size = 32 * 1024 * 1024;807 break;808 case IP_VERSION(10, 3, 5):809 adev->gmc.mall_size = 16 * 1024 * 1024;810 break;811 default:812 adev->gmc.mall_size = 0;813 break;814 }815 816 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {817 case IP_VERSION(10, 1, 10):818 case IP_VERSION(10, 1, 1):819 case IP_VERSION(10, 1, 2):820 case IP_VERSION(10, 1, 3):821 case IP_VERSION(10, 1, 4):822 case IP_VERSION(10, 3, 0):823 case IP_VERSION(10, 3, 2):824 case IP_VERSION(10, 3, 1):825 case IP_VERSION(10, 3, 4):826 case IP_VERSION(10, 3, 5):827 case IP_VERSION(10, 3, 6):828 case IP_VERSION(10, 3, 3):829 case IP_VERSION(10, 3, 7):830 set_bit(AMDGPU_GFXHUB(0), adev->vmhubs_mask);831 set_bit(AMDGPU_MMHUB0(0), adev->vmhubs_mask);832 /*833 * To fulfill 4-level page support,834 * vm size is 256TB (48bit), maximum size of Navi10/Navi14/Navi12,835 * block size 512 (9bit)836 */837 amdgpu_vm_adjust_size(adev, 256 * 1024, 9, 3, 48);838 break;839 default:840 break;841 }842 843 /* This interrupt is VMC page fault.*/844 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_VMC,845 VMC_1_0__SRCID__VM_FAULT,846 &adev->gmc.vm_fault);847 848 if (r)849 return r;850 851 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_UTCL2,852 UTCL2_1_0__SRCID__FAULT,853 &adev->gmc.vm_fault);854 if (r)855 return r;856 857 if (!amdgpu_sriov_vf(adev)) {858 /* interrupt sent to DF. */859 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_DF, 0,860 &adev->gmc.ecc_irq);861 if (r)862 return r;863 }864 865 /*866 * Set the internal MC address mask This is the max address of the GPU's867 * internal address space.868 */869 adev->gmc.mc_mask = 0xffffffffffffULL; /* 48 bit MC */870 871 r = dma_set_mask_and_coherent(adev->dev, DMA_BIT_MASK(44));872 if (r) {873 dev_warn(adev->dev, "amdgpu: No suitable DMA available.\n");874 return r;875 }876 877 adev->need_swiotlb = drm_need_swiotlb(44);878 879 r = gmc_v10_0_mc_init(adev);880 if (r)881 return r;882 883 amdgpu_gmc_get_vbios_allocations(adev);884 885 /* Memory manager */886 r = amdgpu_bo_init(adev);887 if (r)888 return r;889 890 r = gmc_v10_0_gart_init(adev);891 if (r)892 return r;893 894 /*895 * number of VMs896 * VMID 0 is reserved for System897 * amdgpu graphics/compute will use VMIDs 1-7898 * amdkfd will use VMIDs 8-15899 */900 adev->vm_manager.first_kfd_vmid = 8;901 902 amdgpu_vm_manager_init(adev);903 904 r = amdgpu_gmc_ras_sw_init(adev);905 if (r)906 return r;907 908 return 0;909}910 911/**912 * gmc_v10_0_gart_fini - vm fini callback913 *914 * @adev: amdgpu_device pointer915 *916 * Tears down the driver GART/VM setup (CIK).917 */918static void gmc_v10_0_gart_fini(struct amdgpu_device *adev)919{920 amdgpu_gart_table_vram_free(adev);921}922 923static int gmc_v10_0_sw_fini(void *handle)924{925 struct amdgpu_device *adev = (struct amdgpu_device *)handle;926 927 amdgpu_vm_manager_fini(adev);928 gmc_v10_0_gart_fini(adev);929 amdgpu_gem_force_release(adev);930 amdgpu_bo_fini(adev);931 932 return 0;933}934 935static void gmc_v10_0_init_golden_registers(struct amdgpu_device *adev)936{937}938 939/**940 * gmc_v10_0_gart_enable - gart enable941 *942 * @adev: amdgpu_device pointer943 */944static int gmc_v10_0_gart_enable(struct amdgpu_device *adev)945{946 int r;947 bool value;948 949 if (adev->gart.bo == NULL) {950 dev_err(adev->dev, "No VRAM object for PCIE GART.\n");951 return -EINVAL;952 }953 954 amdgpu_gtt_mgr_recover(&adev->mman.gtt_mgr);955 956 if (!adev->in_s0ix) {957 r = adev->gfxhub.funcs->gart_enable(adev);958 if (r)959 return r;960 }961 962 r = adev->mmhub.funcs->gart_enable(adev);963 if (r)964 return r;965 966 adev->hdp.funcs->init_registers(adev);967 968 /* Flush HDP after it is initialized */969 adev->hdp.funcs->flush_hdp(adev, NULL);970 971 value = (amdgpu_vm_fault_stop == AMDGPU_VM_FAULT_STOP_ALWAYS) ?972 false : true;973 974 if (!adev->in_s0ix)975 adev->gfxhub.funcs->set_fault_enable_default(adev, value);976 adev->mmhub.funcs->set_fault_enable_default(adev, value);977 gmc_v10_0_flush_gpu_tlb(adev, 0, AMDGPU_MMHUB0(0), 0);978 if (!adev->in_s0ix)979 gmc_v10_0_flush_gpu_tlb(adev, 0, AMDGPU_GFXHUB(0), 0);980 981 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",982 (unsigned int)(adev->gmc.gart_size >> 20),983 (unsigned long long)amdgpu_bo_gpu_offset(adev->gart.bo));984 985 return 0;986}987 988static int gmc_v10_0_hw_init(void *handle)989{990 struct amdgpu_device *adev = (struct amdgpu_device *)handle;991 int r;992 993 adev->gmc.flush_pasid_uses_kiq = !amdgpu_emu_mode;994 995 /* The sequence of these two function calls matters.*/996 gmc_v10_0_init_golden_registers(adev);997 998 /*999 * harvestable groups in gc_utcl2 need to be programmed before any GFX block1000 * register setup within GMC, or else system hang when harvesting SA.1001 */1002 if (!adev->in_s0ix && adev->gfxhub.funcs && adev->gfxhub.funcs->utcl2_harvest)1003 adev->gfxhub.funcs->utcl2_harvest(adev);1004 1005 r = gmc_v10_0_gart_enable(adev);1006 if (r)1007 return r;1008 1009 if (amdgpu_emu_mode == 1) {1010 r = amdgpu_gmc_vram_checking(adev);1011 if (r)1012 return r;1013 }1014 1015 if (adev->umc.funcs && adev->umc.funcs->init_registers)1016 adev->umc.funcs->init_registers(adev);1017 1018 return 0;1019}1020 1021/**1022 * gmc_v10_0_gart_disable - gart disable1023 *1024 * @adev: amdgpu_device pointer1025 *1026 * This disables all VM page table.1027 */1028static void gmc_v10_0_gart_disable(struct amdgpu_device *adev)1029{1030 if (!adev->in_s0ix)1031 adev->gfxhub.funcs->gart_disable(adev);1032 adev->mmhub.funcs->gart_disable(adev);1033}1034 1035static int gmc_v10_0_hw_fini(void *handle)1036{1037 struct amdgpu_device *adev = (struct amdgpu_device *)handle;1038 1039 gmc_v10_0_gart_disable(adev);1040 1041 if (amdgpu_sriov_vf(adev)) {1042 /* full access mode, so don't touch any GMC register */1043 DRM_DEBUG("For SRIOV client, shouldn't do anything.\n");1044 return 0;1045 }1046 1047 amdgpu_irq_put(adev, &adev->gmc.vm_fault, 0);1048 1049 if (adev->gmc.ecc_irq.funcs &&1050 amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__UMC))1051 amdgpu_irq_put(adev, &adev->gmc.ecc_irq, 0);1052 1053 return 0;1054}1055 1056static int gmc_v10_0_suspend(void *handle)1057{1058 struct amdgpu_device *adev = (struct amdgpu_device *)handle;1059 1060 gmc_v10_0_hw_fini(adev);1061 1062 return 0;1063}1064 1065static int gmc_v10_0_resume(void *handle)1066{1067 int r;1068 struct amdgpu_device *adev = (struct amdgpu_device *)handle;1069 1070 r = gmc_v10_0_hw_init(adev);1071 if (r)1072 return r;1073 1074 amdgpu_vmid_reset_all(adev);1075 1076 return 0;1077}1078 1079static bool gmc_v10_0_is_idle(void *handle)1080{1081 /* MC is always ready in GMC v10.*/1082 return true;1083}1084 1085static int gmc_v10_0_wait_for_idle(void *handle)1086{1087 /* There is no need to wait for MC idle in GMC v10.*/1088 return 0;1089}1090 1091static int gmc_v10_0_soft_reset(void *handle)1092{1093 return 0;1094}1095 1096static int gmc_v10_0_set_clockgating_state(void *handle,1097 enum amd_clockgating_state state)1098{1099 int r;1100 struct amdgpu_device *adev = (struct amdgpu_device *)handle;1101 1102 /*1103 * The issue mmhub can't disconnect from DF with MMHUB clock gating being disabled1104 * is a new problem observed at DF 3.0.3, however with the same suspend sequence not1105 * seen any issue on the DF 3.0.2 series platform.1106 */1107 if (adev->in_s0ix &&1108 amdgpu_ip_version(adev, DF_HWIP, 0) > IP_VERSION(3, 0, 2)) {1109 dev_dbg(adev->dev, "keep mmhub clock gating being enabled for s0ix\n");1110 return 0;1111 }1112 1113 r = adev->mmhub.funcs->set_clockgating(adev, state);1114 if (r)1115 return r;1116 1117 if (amdgpu_ip_version(adev, ATHUB_HWIP, 0) >= IP_VERSION(2, 1, 0))1118 return athub_v2_1_set_clockgating(adev, state);1119 else1120 return athub_v2_0_set_clockgating(adev, state);1121}1122 1123static void gmc_v10_0_get_clockgating_state(void *handle, u64 *flags)1124{1125 struct amdgpu_device *adev = (struct amdgpu_device *)handle;1126 1127 if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(10, 1, 3) ||1128 amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(10, 1, 4))1129 return;1130 1131 adev->mmhub.funcs->get_clockgating(adev, flags);1132 1133 if (amdgpu_ip_version(adev, ATHUB_HWIP, 0) >= IP_VERSION(2, 1, 0))1134 athub_v2_1_get_clockgating(adev, flags);1135 else1136 athub_v2_0_get_clockgating(adev, flags);1137}1138 1139static int gmc_v10_0_set_powergating_state(void *handle,1140 enum amd_powergating_state state)1141{1142 return 0;1143}1144 1145const struct amd_ip_funcs gmc_v10_0_ip_funcs = {1146 .name = "gmc_v10_0",1147 .early_init = gmc_v10_0_early_init,1148 .late_init = gmc_v10_0_late_init,1149 .sw_init = gmc_v10_0_sw_init,1150 .sw_fini = gmc_v10_0_sw_fini,1151 .hw_init = gmc_v10_0_hw_init,1152 .hw_fini = gmc_v10_0_hw_fini,1153 .suspend = gmc_v10_0_suspend,1154 .resume = gmc_v10_0_resume,1155 .is_idle = gmc_v10_0_is_idle,1156 .wait_for_idle = gmc_v10_0_wait_for_idle,1157 .soft_reset = gmc_v10_0_soft_reset,1158 .set_clockgating_state = gmc_v10_0_set_clockgating_state,1159 .set_powergating_state = gmc_v10_0_set_powergating_state,1160 .get_clockgating_state = gmc_v10_0_get_clockgating_state,1161};1162 1163const struct amdgpu_ip_block_version gmc_v10_0_ip_block = {1164 .type = AMD_IP_BLOCK_TYPE_GMC,1165 .major = 10,1166 .minor = 0,1167 .rev = 0,1168 .funcs = &gmc_v10_0_ip_funcs,1169};1170