1772 lines · c
1/*2 * Copyright 2014 Advanced Micro Devices, Inc.3 *4 * Permission is hereby granted, free of charge, to any person obtaining a5 * copy of this software and associated documentation files (the "Software"),6 * to deal in the Software without restriction, including without limitation7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,8 * and/or sell copies of the Software, and to permit persons to whom the9 * Software is furnished to do so, subject to the following conditions:10 *11 * The above copyright notice and this permission notice shall be included in12 * all copies or substantial portions of the Software.13 *14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR20 * OTHER DEALINGS IN THE SOFTWARE.21 *22 */23 24#include <linux/firmware.h>25#include <linux/module.h>26#include <linux/pci.h>27 28#include <drm/drm_cache.h>29#include "amdgpu.h"30#include "gmc_v8_0.h"31#include "amdgpu_ucode.h"32#include "amdgpu_amdkfd.h"33#include "amdgpu_gem.h"34 35#include "gmc/gmc_8_1_d.h"36#include "gmc/gmc_8_1_sh_mask.h"37 38#include "bif/bif_5_0_d.h"39#include "bif/bif_5_0_sh_mask.h"40 41#include "oss/oss_3_0_d.h"42#include "oss/oss_3_0_sh_mask.h"43 44#include "dce/dce_10_0_d.h"45#include "dce/dce_10_0_sh_mask.h"46 47#include "vid.h"48#include "vi.h"49 50#include "amdgpu_atombios.h"51 52#include "ivsrcid/ivsrcid_vislands30.h"53 54static void gmc_v8_0_set_gmc_funcs(struct amdgpu_device *adev);55static void gmc_v8_0_set_irq_funcs(struct amdgpu_device *adev);56static int gmc_v8_0_wait_for_idle(void *handle);57 58MODULE_FIRMWARE("amdgpu/tonga_mc.bin");59MODULE_FIRMWARE("amdgpu/polaris11_mc.bin");60MODULE_FIRMWARE("amdgpu/polaris10_mc.bin");61MODULE_FIRMWARE("amdgpu/polaris12_mc.bin");62MODULE_FIRMWARE("amdgpu/polaris12_32_mc.bin");63MODULE_FIRMWARE("amdgpu/polaris11_k_mc.bin");64MODULE_FIRMWARE("amdgpu/polaris10_k_mc.bin");65MODULE_FIRMWARE("amdgpu/polaris12_k_mc.bin");66 67static const u32 golden_settings_tonga_a11[] = {68 mmMC_ARB_WTM_GRPWT_RD, 0x00000003, 0x00000000,69 mmMC_HUB_RDREQ_DMIF_LIMIT, 0x0000007f, 0x00000028,70 mmMC_HUB_WDP_UMC, 0x00007fb6, 0x00000991,71 mmVM_PRT_APERTURE0_LOW_ADDR, 0x0fffffff, 0x0fffffff,72 mmVM_PRT_APERTURE1_LOW_ADDR, 0x0fffffff, 0x0fffffff,73 mmVM_PRT_APERTURE2_LOW_ADDR, 0x0fffffff, 0x0fffffff,74 mmVM_PRT_APERTURE3_LOW_ADDR, 0x0fffffff, 0x0fffffff,75};76 77static const u32 tonga_mgcg_cgcg_init[] = {78 mmMC_MEM_POWER_LS, 0xffffffff, 0x0000010479};80 81static const u32 golden_settings_fiji_a10[] = {82 mmVM_PRT_APERTURE0_LOW_ADDR, 0x0fffffff, 0x0fffffff,83 mmVM_PRT_APERTURE1_LOW_ADDR, 0x0fffffff, 0x0fffffff,84 mmVM_PRT_APERTURE2_LOW_ADDR, 0x0fffffff, 0x0fffffff,85 mmVM_PRT_APERTURE3_LOW_ADDR, 0x0fffffff, 0x0fffffff,86};87 88static const u32 fiji_mgcg_cgcg_init[] = {89 mmMC_MEM_POWER_LS, 0xffffffff, 0x0000010490};91 92static const u32 golden_settings_polaris11_a11[] = {93 mmVM_PRT_APERTURE0_LOW_ADDR, 0x0fffffff, 0x0fffffff,94 mmVM_PRT_APERTURE1_LOW_ADDR, 0x0fffffff, 0x0fffffff,95 mmVM_PRT_APERTURE2_LOW_ADDR, 0x0fffffff, 0x0fffffff,96 mmVM_PRT_APERTURE3_LOW_ADDR, 0x0fffffff, 0x0fffffff97};98 99static const u32 golden_settings_polaris10_a11[] = {100 mmMC_ARB_WTM_GRPWT_RD, 0x00000003, 0x00000000,101 mmVM_PRT_APERTURE0_LOW_ADDR, 0x0fffffff, 0x0fffffff,102 mmVM_PRT_APERTURE1_LOW_ADDR, 0x0fffffff, 0x0fffffff,103 mmVM_PRT_APERTURE2_LOW_ADDR, 0x0fffffff, 0x0fffffff,104 mmVM_PRT_APERTURE3_LOW_ADDR, 0x0fffffff, 0x0fffffff105};106 107static const u32 cz_mgcg_cgcg_init[] = {108 mmMC_MEM_POWER_LS, 0xffffffff, 0x00000104109};110 111static const u32 stoney_mgcg_cgcg_init[] = {112 mmATC_MISC_CG, 0xffffffff, 0x000c0200,113 mmMC_MEM_POWER_LS, 0xffffffff, 0x00000104114};115 116static const u32 golden_settings_stoney_common[] = {117 mmMC_HUB_RDREQ_UVD, MC_HUB_RDREQ_UVD__PRESCALE_MASK, 0x00000004,118 mmMC_RD_GRP_OTH, MC_RD_GRP_OTH__UVD_MASK, 0x00600000119};120 121static void gmc_v8_0_init_golden_registers(struct amdgpu_device *adev)122{123 switch (adev->asic_type) {124 case CHIP_FIJI:125 amdgpu_device_program_register_sequence(adev,126 fiji_mgcg_cgcg_init,127 ARRAY_SIZE(fiji_mgcg_cgcg_init));128 amdgpu_device_program_register_sequence(adev,129 golden_settings_fiji_a10,130 ARRAY_SIZE(golden_settings_fiji_a10));131 break;132 case CHIP_TONGA:133 amdgpu_device_program_register_sequence(adev,134 tonga_mgcg_cgcg_init,135 ARRAY_SIZE(tonga_mgcg_cgcg_init));136 amdgpu_device_program_register_sequence(adev,137 golden_settings_tonga_a11,138 ARRAY_SIZE(golden_settings_tonga_a11));139 break;140 case CHIP_POLARIS11:141 case CHIP_POLARIS12:142 case CHIP_VEGAM:143 amdgpu_device_program_register_sequence(adev,144 golden_settings_polaris11_a11,145 ARRAY_SIZE(golden_settings_polaris11_a11));146 break;147 case CHIP_POLARIS10:148 amdgpu_device_program_register_sequence(adev,149 golden_settings_polaris10_a11,150 ARRAY_SIZE(golden_settings_polaris10_a11));151 break;152 case CHIP_CARRIZO:153 amdgpu_device_program_register_sequence(adev,154 cz_mgcg_cgcg_init,155 ARRAY_SIZE(cz_mgcg_cgcg_init));156 break;157 case CHIP_STONEY:158 amdgpu_device_program_register_sequence(adev,159 stoney_mgcg_cgcg_init,160 ARRAY_SIZE(stoney_mgcg_cgcg_init));161 amdgpu_device_program_register_sequence(adev,162 golden_settings_stoney_common,163 ARRAY_SIZE(golden_settings_stoney_common));164 break;165 default:166 break;167 }168}169 170static void gmc_v8_0_mc_stop(struct amdgpu_device *adev)171{172 u32 blackout;173 174 gmc_v8_0_wait_for_idle(adev);175 176 blackout = RREG32(mmMC_SHARED_BLACKOUT_CNTL);177 if (REG_GET_FIELD(blackout, MC_SHARED_BLACKOUT_CNTL, BLACKOUT_MODE) != 1) {178 /* Block CPU access */179 WREG32(mmBIF_FB_EN, 0);180 /* blackout the MC */181 blackout = REG_SET_FIELD(blackout,182 MC_SHARED_BLACKOUT_CNTL, BLACKOUT_MODE, 1);183 WREG32(mmMC_SHARED_BLACKOUT_CNTL, blackout);184 }185 /* wait for the MC to settle */186 udelay(100);187}188 189static void gmc_v8_0_mc_resume(struct amdgpu_device *adev)190{191 u32 tmp;192 193 /* unblackout the MC */194 tmp = RREG32(mmMC_SHARED_BLACKOUT_CNTL);195 tmp = REG_SET_FIELD(tmp, MC_SHARED_BLACKOUT_CNTL, BLACKOUT_MODE, 0);196 WREG32(mmMC_SHARED_BLACKOUT_CNTL, tmp);197 /* allow CPU access */198 tmp = REG_SET_FIELD(0, BIF_FB_EN, FB_READ_EN, 1);199 tmp = REG_SET_FIELD(tmp, BIF_FB_EN, FB_WRITE_EN, 1);200 WREG32(mmBIF_FB_EN, tmp);201}202 203/**204 * gmc_v8_0_init_microcode - load ucode images from disk205 *206 * @adev: amdgpu_device pointer207 *208 * Use the firmware interface to load the ucode images into209 * the driver (not loaded into hw).210 * Returns 0 on success, error on failure.211 */212static int gmc_v8_0_init_microcode(struct amdgpu_device *adev)213{214 const char *chip_name;215 int err;216 217 DRM_DEBUG("\n");218 219 switch (adev->asic_type) {220 case CHIP_TONGA:221 chip_name = "tonga";222 break;223 case CHIP_POLARIS11:224 if (ASICID_IS_P21(adev->pdev->device, adev->pdev->revision) ||225 ASICID_IS_P31(adev->pdev->device, adev->pdev->revision))226 chip_name = "polaris11_k";227 else228 chip_name = "polaris11";229 break;230 case CHIP_POLARIS10:231 if (ASICID_IS_P30(adev->pdev->device, adev->pdev->revision))232 chip_name = "polaris10_k";233 else234 chip_name = "polaris10";235 break;236 case CHIP_POLARIS12:237 if (ASICID_IS_P23(adev->pdev->device, adev->pdev->revision)) {238 chip_name = "polaris12_k";239 } else {240 WREG32(mmMC_SEQ_IO_DEBUG_INDEX, ixMC_IO_DEBUG_UP_159);241 /* Polaris12 32bit ASIC needs a special MC firmware */242 if (RREG32(mmMC_SEQ_IO_DEBUG_DATA) == 0x05b4dc40)243 chip_name = "polaris12_32";244 else245 chip_name = "polaris12";246 }247 break;248 case CHIP_FIJI:249 case CHIP_CARRIZO:250 case CHIP_STONEY:251 case CHIP_VEGAM:252 return 0;253 default:254 return -EINVAL;255 }256 257 err = amdgpu_ucode_request(adev, &adev->gmc.fw, "amdgpu/%s_mc.bin", chip_name);258 if (err) {259 pr_err("mc: Failed to load firmware \"%s_mc.bin\"\n", chip_name);260 amdgpu_ucode_release(&adev->gmc.fw);261 }262 return err;263}264 265/**266 * gmc_v8_0_tonga_mc_load_microcode - load tonga MC ucode into the hw267 *268 * @adev: amdgpu_device pointer269 *270 * Load the GDDR MC ucode into the hw (VI).271 * Returns 0 on success, error on failure.272 */273static int gmc_v8_0_tonga_mc_load_microcode(struct amdgpu_device *adev)274{275 const struct mc_firmware_header_v1_0 *hdr;276 const __le32 *fw_data = NULL;277 const __le32 *io_mc_regs = NULL;278 u32 running;279 int i, ucode_size, regs_size;280 281 /* Skip MC ucode loading on SR-IOV capable boards.282 * vbios does this for us in asic_init in that case.283 * Skip MC ucode loading on VF, because hypervisor will do that284 * for this adaptor.285 */286 if (amdgpu_sriov_bios(adev))287 return 0;288 289 if (!adev->gmc.fw)290 return -EINVAL;291 292 hdr = (const struct mc_firmware_header_v1_0 *)adev->gmc.fw->data;293 amdgpu_ucode_print_mc_hdr(&hdr->header);294 295 adev->gmc.fw_version = le32_to_cpu(hdr->header.ucode_version);296 regs_size = le32_to_cpu(hdr->io_debug_size_bytes) / (4 * 2);297 io_mc_regs = (const __le32 *)298 (adev->gmc.fw->data + le32_to_cpu(hdr->io_debug_array_offset_bytes));299 ucode_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;300 fw_data = (const __le32 *)301 (adev->gmc.fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));302 303 running = REG_GET_FIELD(RREG32(mmMC_SEQ_SUP_CNTL), MC_SEQ_SUP_CNTL, RUN);304 305 if (running == 0) {306 /* reset the engine and set to writable */307 WREG32(mmMC_SEQ_SUP_CNTL, 0x00000008);308 WREG32(mmMC_SEQ_SUP_CNTL, 0x00000010);309 310 /* load mc io regs */311 for (i = 0; i < regs_size; i++) {312 WREG32(mmMC_SEQ_IO_DEBUG_INDEX, le32_to_cpup(io_mc_regs++));313 WREG32(mmMC_SEQ_IO_DEBUG_DATA, le32_to_cpup(io_mc_regs++));314 }315 /* load the MC ucode */316 for (i = 0; i < ucode_size; i++)317 WREG32(mmMC_SEQ_SUP_PGM, le32_to_cpup(fw_data++));318 319 /* put the engine back into the active state */320 WREG32(mmMC_SEQ_SUP_CNTL, 0x00000008);321 WREG32(mmMC_SEQ_SUP_CNTL, 0x00000004);322 WREG32(mmMC_SEQ_SUP_CNTL, 0x00000001);323 324 /* wait for training to complete */325 for (i = 0; i < adev->usec_timeout; i++) {326 if (REG_GET_FIELD(RREG32(mmMC_SEQ_TRAIN_WAKEUP_CNTL),327 MC_SEQ_TRAIN_WAKEUP_CNTL, TRAIN_DONE_D0))328 break;329 udelay(1);330 }331 for (i = 0; i < adev->usec_timeout; i++) {332 if (REG_GET_FIELD(RREG32(mmMC_SEQ_TRAIN_WAKEUP_CNTL),333 MC_SEQ_TRAIN_WAKEUP_CNTL, TRAIN_DONE_D1))334 break;335 udelay(1);336 }337 }338 339 return 0;340}341 342static int gmc_v8_0_polaris_mc_load_microcode(struct amdgpu_device *adev)343{344 const struct mc_firmware_header_v1_0 *hdr;345 const __le32 *fw_data = NULL;346 const __le32 *io_mc_regs = NULL;347 u32 data;348 int i, ucode_size, regs_size;349 350 /* Skip MC ucode loading on SR-IOV capable boards.351 * vbios does this for us in asic_init in that case.352 * Skip MC ucode loading on VF, because hypervisor will do that353 * for this adaptor.354 */355 if (amdgpu_sriov_bios(adev))356 return 0;357 358 if (!adev->gmc.fw)359 return -EINVAL;360 361 hdr = (const struct mc_firmware_header_v1_0 *)adev->gmc.fw->data;362 amdgpu_ucode_print_mc_hdr(&hdr->header);363 364 adev->gmc.fw_version = le32_to_cpu(hdr->header.ucode_version);365 regs_size = le32_to_cpu(hdr->io_debug_size_bytes) / (4 * 2);366 io_mc_regs = (const __le32 *)367 (adev->gmc.fw->data + le32_to_cpu(hdr->io_debug_array_offset_bytes));368 ucode_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;369 fw_data = (const __le32 *)370 (adev->gmc.fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));371 372 data = RREG32(mmMC_SEQ_MISC0);373 data &= ~(0x40);374 WREG32(mmMC_SEQ_MISC0, data);375 376 /* load mc io regs */377 for (i = 0; i < regs_size; i++) {378 WREG32(mmMC_SEQ_IO_DEBUG_INDEX, le32_to_cpup(io_mc_regs++));379 WREG32(mmMC_SEQ_IO_DEBUG_DATA, le32_to_cpup(io_mc_regs++));380 }381 382 WREG32(mmMC_SEQ_SUP_CNTL, 0x00000008);383 WREG32(mmMC_SEQ_SUP_CNTL, 0x00000010);384 385 /* load the MC ucode */386 for (i = 0; i < ucode_size; i++)387 WREG32(mmMC_SEQ_SUP_PGM, le32_to_cpup(fw_data++));388 389 /* put the engine back into the active state */390 WREG32(mmMC_SEQ_SUP_CNTL, 0x00000008);391 WREG32(mmMC_SEQ_SUP_CNTL, 0x00000004);392 WREG32(mmMC_SEQ_SUP_CNTL, 0x00000001);393 394 /* wait for training to complete */395 for (i = 0; i < adev->usec_timeout; i++) {396 data = RREG32(mmMC_SEQ_MISC0);397 if (data & 0x80)398 break;399 udelay(1);400 }401 402 return 0;403}404 405static void gmc_v8_0_vram_gtt_location(struct amdgpu_device *adev,406 struct amdgpu_gmc *mc)407{408 u64 base = 0;409 410 if (!amdgpu_sriov_vf(adev))411 base = RREG32(mmMC_VM_FB_LOCATION) & 0xFFFF;412 base <<= 24;413 414 amdgpu_gmc_set_agp_default(adev, mc);415 amdgpu_gmc_vram_location(adev, mc, base);416 amdgpu_gmc_gart_location(adev, mc, AMDGPU_GART_PLACEMENT_BEST_FIT);417}418 419/**420 * gmc_v8_0_mc_program - program the GPU memory controller421 *422 * @adev: amdgpu_device pointer423 *424 * Set the location of vram, gart, and AGP in the GPU's425 * physical address space (VI).426 */427static void gmc_v8_0_mc_program(struct amdgpu_device *adev)428{429 u32 tmp;430 int i, j;431 432 /* Initialize HDP */433 for (i = 0, j = 0; i < 32; i++, j += 0x6) {434 WREG32((0xb05 + j), 0x00000000);435 WREG32((0xb06 + j), 0x00000000);436 WREG32((0xb07 + j), 0x00000000);437 WREG32((0xb08 + j), 0x00000000);438 WREG32((0xb09 + j), 0x00000000);439 }440 WREG32(mmHDP_REG_COHERENCY_FLUSH_CNTL, 0);441 442 if (gmc_v8_0_wait_for_idle((void *)adev))443 dev_warn(adev->dev, "Wait for MC idle timedout !\n");444 445 if (adev->mode_info.num_crtc) {446 /* Lockout access through VGA aperture*/447 tmp = RREG32(mmVGA_HDP_CONTROL);448 tmp = REG_SET_FIELD(tmp, VGA_HDP_CONTROL, VGA_MEMORY_DISABLE, 1);449 WREG32(mmVGA_HDP_CONTROL, tmp);450 451 /* disable VGA render */452 tmp = RREG32(mmVGA_RENDER_CONTROL);453 tmp = REG_SET_FIELD(tmp, VGA_RENDER_CONTROL, VGA_VSTATUS_CNTL, 0);454 WREG32(mmVGA_RENDER_CONTROL, tmp);455 }456 /* Update configuration */457 WREG32(mmMC_VM_SYSTEM_APERTURE_LOW_ADDR,458 adev->gmc.vram_start >> 12);459 WREG32(mmMC_VM_SYSTEM_APERTURE_HIGH_ADDR,460 adev->gmc.vram_end >> 12);461 WREG32(mmMC_VM_SYSTEM_APERTURE_DEFAULT_ADDR,462 adev->mem_scratch.gpu_addr >> 12);463 464 if (amdgpu_sriov_vf(adev)) {465 tmp = ((adev->gmc.vram_end >> 24) & 0xFFFF) << 16;466 tmp |= ((adev->gmc.vram_start >> 24) & 0xFFFF);467 WREG32(mmMC_VM_FB_LOCATION, tmp);468 /* XXX double check these! */469 WREG32(mmHDP_NONSURFACE_BASE, (adev->gmc.vram_start >> 8));470 WREG32(mmHDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));471 WREG32(mmHDP_NONSURFACE_SIZE, 0x3FFFFFFF);472 }473 474 WREG32(mmMC_VM_AGP_BASE, 0);475 WREG32(mmMC_VM_AGP_TOP, adev->gmc.agp_end >> 22);476 WREG32(mmMC_VM_AGP_BOT, adev->gmc.agp_start >> 22);477 if (gmc_v8_0_wait_for_idle((void *)adev))478 dev_warn(adev->dev, "Wait for MC idle timedout !\n");479 480 WREG32(mmBIF_FB_EN, BIF_FB_EN__FB_READ_EN_MASK | BIF_FB_EN__FB_WRITE_EN_MASK);481 482 tmp = RREG32(mmHDP_MISC_CNTL);483 tmp = REG_SET_FIELD(tmp, HDP_MISC_CNTL, FLUSH_INVALIDATE_CACHE, 0);484 WREG32(mmHDP_MISC_CNTL, tmp);485 486 tmp = RREG32(mmHDP_HOST_PATH_CNTL);487 WREG32(mmHDP_HOST_PATH_CNTL, tmp);488}489 490/**491 * gmc_v8_0_mc_init - initialize the memory controller driver params492 *493 * @adev: amdgpu_device pointer494 *495 * Look up the amount of vram, vram width, and decide how to place496 * vram and gart within the GPU's physical address space (VI).497 * Returns 0 for success.498 */499static int gmc_v8_0_mc_init(struct amdgpu_device *adev)500{501 int r;502 u32 tmp;503 504 adev->gmc.vram_width = amdgpu_atombios_get_vram_width(adev);505 if (!adev->gmc.vram_width) {506 int chansize, numchan;507 508 /* Get VRAM informations */509 tmp = RREG32(mmMC_ARB_RAMCFG);510 if (REG_GET_FIELD(tmp, MC_ARB_RAMCFG, CHANSIZE))511 chansize = 64;512 else513 chansize = 32;514 515 tmp = RREG32(mmMC_SHARED_CHMAP);516 switch (REG_GET_FIELD(tmp, MC_SHARED_CHMAP, NOOFCHAN)) {517 case 0:518 default:519 numchan = 1;520 break;521 case 1:522 numchan = 2;523 break;524 case 2:525 numchan = 4;526 break;527 case 3:528 numchan = 8;529 break;530 case 4:531 numchan = 3;532 break;533 case 5:534 numchan = 6;535 break;536 case 6:537 numchan = 10;538 break;539 case 7:540 numchan = 12;541 break;542 case 8:543 numchan = 16;544 break;545 }546 adev->gmc.vram_width = numchan * chansize;547 }548 /* size in MB on si */549 tmp = RREG32(mmCONFIG_MEMSIZE);550 /* some boards may have garbage in the upper 16 bits */551 if (tmp & 0xffff0000) {552 DRM_INFO("Probable bad vram size: 0x%08x\n", tmp);553 if (tmp & 0xffff)554 tmp &= 0xffff;555 }556 adev->gmc.mc_vram_size = tmp * 1024ULL * 1024ULL;557 adev->gmc.real_vram_size = adev->gmc.mc_vram_size;558 559 if (!(adev->flags & AMD_IS_APU)) {560 r = amdgpu_device_resize_fb_bar(adev);561 if (r)562 return r;563 }564 adev->gmc.aper_base = pci_resource_start(adev->pdev, 0);565 adev->gmc.aper_size = pci_resource_len(adev->pdev, 0);566 567#ifdef CONFIG_X86_64568 if ((adev->flags & AMD_IS_APU) && !amdgpu_passthrough(adev)) {569 adev->gmc.aper_base = ((u64)RREG32(mmMC_VM_FB_OFFSET)) << 22;570 adev->gmc.aper_size = adev->gmc.real_vram_size;571 }572#endif573 574 adev->gmc.visible_vram_size = adev->gmc.aper_size;575 576 /* set the gart size */577 if (amdgpu_gart_size == -1) {578 switch (adev->asic_type) {579 case CHIP_POLARIS10: /* all engines support GPUVM */580 case CHIP_POLARIS11: /* all engines support GPUVM */581 case CHIP_POLARIS12: /* all engines support GPUVM */582 case CHIP_VEGAM: /* all engines support GPUVM */583 default:584 adev->gmc.gart_size = 256ULL << 20;585 break;586 case CHIP_TONGA: /* UVD, VCE do not support GPUVM */587 case CHIP_FIJI: /* UVD, VCE do not support GPUVM */588 case CHIP_CARRIZO: /* UVD, VCE do not support GPUVM, DCE SG support */589 case CHIP_STONEY: /* UVD does not support GPUVM, DCE SG support */590 adev->gmc.gart_size = 1024ULL << 20;591 break;592 }593 } else {594 adev->gmc.gart_size = (u64)amdgpu_gart_size << 20;595 }596 597 adev->gmc.gart_size += adev->pm.smu_prv_buffer_size;598 gmc_v8_0_vram_gtt_location(adev, &adev->gmc);599 600 return 0;601}602 603/**604 * gmc_v8_0_flush_gpu_tlb_pasid - tlb flush via pasid605 *606 * @adev: amdgpu_device pointer607 * @pasid: pasid to be flush608 * @flush_type: type of flush609 * @all_hub: flush all hubs610 * @inst: is used to select which instance of KIQ to use for the invalidation611 *612 * Flush the TLB for the requested pasid.613 */614static void gmc_v8_0_flush_gpu_tlb_pasid(struct amdgpu_device *adev,615 uint16_t pasid, uint32_t flush_type,616 bool all_hub, uint32_t inst)617{618 u32 mask = 0x0;619 int vmid;620 621 for (vmid = 1; vmid < 16; vmid++) {622 u32 tmp = RREG32(mmATC_VMID0_PASID_MAPPING + vmid);623 624 if ((tmp & ATC_VMID0_PASID_MAPPING__VALID_MASK) &&625 (tmp & ATC_VMID0_PASID_MAPPING__PASID_MASK) == pasid)626 mask |= 1 << vmid;627 }628 629 WREG32(mmVM_INVALIDATE_REQUEST, mask);630 RREG32(mmVM_INVALIDATE_RESPONSE);631}632 633/*634 * GART635 * VMID 0 is the physical GPU addresses as used by the kernel.636 * VMIDs 1-15 are used for userspace clients and are handled637 * by the amdgpu vm/hsa code.638 */639 640/**641 * gmc_v8_0_flush_gpu_tlb - gart tlb flush callback642 *643 * @adev: amdgpu_device pointer644 * @vmid: vm instance to flush645 * @vmhub: which hub to flush646 * @flush_type: type of flush647 *648 * Flush the TLB for the requested page table (VI).649 */650static void gmc_v8_0_flush_gpu_tlb(struct amdgpu_device *adev, uint32_t vmid,651 uint32_t vmhub, uint32_t flush_type)652{653 /* bits 0-15 are the VM contexts0-15 */654 WREG32(mmVM_INVALIDATE_REQUEST, 1 << vmid);655}656 657static uint64_t gmc_v8_0_emit_flush_gpu_tlb(struct amdgpu_ring *ring,658 unsigned int vmid, uint64_t pd_addr)659{660 uint32_t reg;661 662 if (vmid < 8)663 reg = mmVM_CONTEXT0_PAGE_TABLE_BASE_ADDR + vmid;664 else665 reg = mmVM_CONTEXT8_PAGE_TABLE_BASE_ADDR + vmid - 8;666 amdgpu_ring_emit_wreg(ring, reg, pd_addr >> 12);667 668 /* bits 0-15 are the VM contexts0-15 */669 amdgpu_ring_emit_wreg(ring, mmVM_INVALIDATE_REQUEST, 1 << vmid);670 671 return pd_addr;672}673 674static void gmc_v8_0_emit_pasid_mapping(struct amdgpu_ring *ring, unsigned int vmid,675 unsigned int pasid)676{677 amdgpu_ring_emit_wreg(ring, mmIH_VMID_0_LUT + vmid, pasid);678}679 680/*681 * PTE format on VI:682 * 63:40 reserved683 * 39:12 4k physical page base address684 * 11:7 fragment685 * 6 write686 * 5 read687 * 4 exe688 * 3 reserved689 * 2 snooped690 * 1 system691 * 0 valid692 *693 * PDE format on VI:694 * 63:59 block fragment size695 * 58:40 reserved696 * 39:1 physical base address of PTE697 * bits 5:1 must be 0.698 * 0 valid699 */700 701static void gmc_v8_0_get_vm_pde(struct amdgpu_device *adev, int level,702 uint64_t *addr, uint64_t *flags)703{704 BUG_ON(*addr & 0xFFFFFF0000000FFFULL);705}706 707static void gmc_v8_0_get_vm_pte(struct amdgpu_device *adev,708 struct amdgpu_bo_va_mapping *mapping,709 uint64_t *flags)710{711 *flags &= ~AMDGPU_PTE_EXECUTABLE;712 *flags |= mapping->flags & AMDGPU_PTE_EXECUTABLE;713 *flags &= ~AMDGPU_PTE_PRT;714}715 716/**717 * gmc_v8_0_set_fault_enable_default - update VM fault handling718 *719 * @adev: amdgpu_device pointer720 * @value: true redirects VM faults to the default page721 */722static void gmc_v8_0_set_fault_enable_default(struct amdgpu_device *adev,723 bool value)724{725 u32 tmp;726 727 tmp = RREG32(mmVM_CONTEXT1_CNTL);728 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,729 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT, value);730 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,731 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT, value);732 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,733 PDE0_PROTECTION_FAULT_ENABLE_DEFAULT, value);734 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,735 VALID_PROTECTION_FAULT_ENABLE_DEFAULT, value);736 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,737 READ_PROTECTION_FAULT_ENABLE_DEFAULT, value);738 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,739 WRITE_PROTECTION_FAULT_ENABLE_DEFAULT, value);740 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,741 EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT, value);742 WREG32(mmVM_CONTEXT1_CNTL, tmp);743}744 745/**746 * gmc_v8_0_set_prt() - set PRT VM fault747 *748 * @adev: amdgpu_device pointer749 * @enable: enable/disable VM fault handling for PRT750 */751static void gmc_v8_0_set_prt(struct amdgpu_device *adev, bool enable)752{753 u32 tmp;754 755 if (enable && !adev->gmc.prt_warning) {756 dev_warn(adev->dev, "Disabling VM faults because of PRT request!\n");757 adev->gmc.prt_warning = true;758 }759 760 tmp = RREG32(mmVM_PRT_CNTL);761 tmp = REG_SET_FIELD(tmp, VM_PRT_CNTL,762 CB_DISABLE_READ_FAULT_ON_UNMAPPED_ACCESS, enable);763 tmp = REG_SET_FIELD(tmp, VM_PRT_CNTL,764 CB_DISABLE_WRITE_FAULT_ON_UNMAPPED_ACCESS, enable);765 tmp = REG_SET_FIELD(tmp, VM_PRT_CNTL,766 TC_DISABLE_READ_FAULT_ON_UNMAPPED_ACCESS, enable);767 tmp = REG_SET_FIELD(tmp, VM_PRT_CNTL,768 TC_DISABLE_WRITE_FAULT_ON_UNMAPPED_ACCESS, enable);769 tmp = REG_SET_FIELD(tmp, VM_PRT_CNTL,770 L2_CACHE_STORE_INVALID_ENTRIES, enable);771 tmp = REG_SET_FIELD(tmp, VM_PRT_CNTL,772 L1_TLB_STORE_INVALID_ENTRIES, enable);773 tmp = REG_SET_FIELD(tmp, VM_PRT_CNTL,774 MASK_PDE0_FAULT, enable);775 WREG32(mmVM_PRT_CNTL, tmp);776 777 if (enable) {778 uint32_t low = AMDGPU_VA_RESERVED_BOTTOM >>779 AMDGPU_GPU_PAGE_SHIFT;780 uint32_t high = adev->vm_manager.max_pfn -781 (AMDGPU_VA_RESERVED_TOP >> AMDGPU_GPU_PAGE_SHIFT);782 783 WREG32(mmVM_PRT_APERTURE0_LOW_ADDR, low);784 WREG32(mmVM_PRT_APERTURE1_LOW_ADDR, low);785 WREG32(mmVM_PRT_APERTURE2_LOW_ADDR, low);786 WREG32(mmVM_PRT_APERTURE3_LOW_ADDR, low);787 WREG32(mmVM_PRT_APERTURE0_HIGH_ADDR, high);788 WREG32(mmVM_PRT_APERTURE1_HIGH_ADDR, high);789 WREG32(mmVM_PRT_APERTURE2_HIGH_ADDR, high);790 WREG32(mmVM_PRT_APERTURE3_HIGH_ADDR, high);791 } else {792 WREG32(mmVM_PRT_APERTURE0_LOW_ADDR, 0xfffffff);793 WREG32(mmVM_PRT_APERTURE1_LOW_ADDR, 0xfffffff);794 WREG32(mmVM_PRT_APERTURE2_LOW_ADDR, 0xfffffff);795 WREG32(mmVM_PRT_APERTURE3_LOW_ADDR, 0xfffffff);796 WREG32(mmVM_PRT_APERTURE0_HIGH_ADDR, 0x0);797 WREG32(mmVM_PRT_APERTURE1_HIGH_ADDR, 0x0);798 WREG32(mmVM_PRT_APERTURE2_HIGH_ADDR, 0x0);799 WREG32(mmVM_PRT_APERTURE3_HIGH_ADDR, 0x0);800 }801}802 803/**804 * gmc_v8_0_gart_enable - gart enable805 *806 * @adev: amdgpu_device pointer807 *808 * This sets up the TLBs, programs the page tables for VMID0,809 * sets up the hw for VMIDs 1-15 which are allocated on810 * demand, and sets up the global locations for the LDS, GDS,811 * and GPUVM for FSA64 clients (VI).812 * Returns 0 for success, errors for failure.813 */814static int gmc_v8_0_gart_enable(struct amdgpu_device *adev)815{816 uint64_t table_addr;817 u32 tmp, field;818 int i;819 820 if (adev->gart.bo == NULL) {821 dev_err(adev->dev, "No VRAM object for PCIE GART.\n");822 return -EINVAL;823 }824 amdgpu_gtt_mgr_recover(&adev->mman.gtt_mgr);825 table_addr = amdgpu_bo_gpu_offset(adev->gart.bo);826 827 /* Setup TLB control */828 tmp = RREG32(mmMC_VM_MX_L1_TLB_CNTL);829 tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, ENABLE_L1_TLB, 1);830 tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, ENABLE_L1_FRAGMENT_PROCESSING, 1);831 tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE, 3);832 tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, ENABLE_ADVANCED_DRIVER_MODEL, 1);833 tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, SYSTEM_APERTURE_UNMAPPED_ACCESS, 0);834 WREG32(mmMC_VM_MX_L1_TLB_CNTL, tmp);835 /* Setup L2 cache */836 tmp = RREG32(mmVM_L2_CNTL);837 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, ENABLE_L2_CACHE, 1);838 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, ENABLE_L2_FRAGMENT_PROCESSING, 1);839 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE, 1);840 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE, 1);841 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, EFFECTIVE_L2_QUEUE_SIZE, 7);842 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, CONTEXT1_IDENTITY_ACCESS_MODE, 1);843 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, ENABLE_DEFAULT_PAGE_OUT_TO_SYSTEM_MEMORY, 1);844 WREG32(mmVM_L2_CNTL, tmp);845 tmp = RREG32(mmVM_L2_CNTL2);846 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS, 1);847 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL2, INVALIDATE_L2_CACHE, 1);848 WREG32(mmVM_L2_CNTL2, tmp);849 850 field = adev->vm_manager.fragment_size;851 tmp = RREG32(mmVM_L2_CNTL3);852 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY, 1);853 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL3, BANK_SELECT, field);854 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL3, L2_CACHE_BIGK_FRAGMENT_SIZE, field);855 WREG32(mmVM_L2_CNTL3, tmp);856 /* XXX: set to enable PTE/PDE in system memory */857 tmp = RREG32(mmVM_L2_CNTL4);858 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL4, VMC_TAP_CONTEXT0_PDE_REQUEST_PHYSICAL, 0);859 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL4, VMC_TAP_CONTEXT0_PDE_REQUEST_SHARED, 0);860 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL4, VMC_TAP_CONTEXT0_PDE_REQUEST_SNOOP, 0);861 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL4, VMC_TAP_CONTEXT0_PTE_REQUEST_PHYSICAL, 0);862 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL4, VMC_TAP_CONTEXT0_PTE_REQUEST_SHARED, 0);863 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL4, VMC_TAP_CONTEXT0_PTE_REQUEST_SNOOP, 0);864 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL4, VMC_TAP_CONTEXT1_PDE_REQUEST_PHYSICAL, 0);865 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL4, VMC_TAP_CONTEXT1_PDE_REQUEST_SHARED, 0);866 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL4, VMC_TAP_CONTEXT1_PDE_REQUEST_SNOOP, 0);867 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL4, VMC_TAP_CONTEXT1_PTE_REQUEST_PHYSICAL, 0);868 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL4, VMC_TAP_CONTEXT1_PTE_REQUEST_SHARED, 0);869 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL4, VMC_TAP_CONTEXT1_PTE_REQUEST_SNOOP, 0);870 WREG32(mmVM_L2_CNTL4, tmp);871 /* setup context0 */872 WREG32(mmVM_CONTEXT0_PAGE_TABLE_START_ADDR, adev->gmc.gart_start >> 12);873 WREG32(mmVM_CONTEXT0_PAGE_TABLE_END_ADDR, adev->gmc.gart_end >> 12);874 WREG32(mmVM_CONTEXT0_PAGE_TABLE_BASE_ADDR, table_addr >> 12);875 WREG32(mmVM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,876 (u32)(adev->dummy_page_addr >> 12));877 WREG32(mmVM_CONTEXT0_CNTL2, 0);878 tmp = RREG32(mmVM_CONTEXT0_CNTL);879 tmp = REG_SET_FIELD(tmp, VM_CONTEXT0_CNTL, ENABLE_CONTEXT, 1);880 tmp = REG_SET_FIELD(tmp, VM_CONTEXT0_CNTL, PAGE_TABLE_DEPTH, 0);881 tmp = REG_SET_FIELD(tmp, VM_CONTEXT0_CNTL, RANGE_PROTECTION_FAULT_ENABLE_DEFAULT, 1);882 WREG32(mmVM_CONTEXT0_CNTL, tmp);883 884 WREG32(mmVM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR, 0);885 WREG32(mmVM_L2_CONTEXT1_IDENTITY_APERTURE_HIGH_ADDR, 0);886 WREG32(mmVM_L2_CONTEXT_IDENTITY_PHYSICAL_OFFSET, 0);887 888 /* empty context1-15 */889 /* FIXME start with 4G, once using 2 level pt switch to full890 * vm size space891 */892 /* set vm size, must be a multiple of 4 */893 WREG32(mmVM_CONTEXT1_PAGE_TABLE_START_ADDR, 0);894 WREG32(mmVM_CONTEXT1_PAGE_TABLE_END_ADDR, adev->vm_manager.max_pfn - 1);895 for (i = 1; i < AMDGPU_NUM_VMID; i++) {896 if (i < 8)897 WREG32(mmVM_CONTEXT0_PAGE_TABLE_BASE_ADDR + i,898 table_addr >> 12);899 else900 WREG32(mmVM_CONTEXT8_PAGE_TABLE_BASE_ADDR + i - 8,901 table_addr >> 12);902 }903 904 /* enable context1-15 */905 WREG32(mmVM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR,906 (u32)(adev->dummy_page_addr >> 12));907 WREG32(mmVM_CONTEXT1_CNTL2, 4);908 tmp = RREG32(mmVM_CONTEXT1_CNTL);909 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, ENABLE_CONTEXT, 1);910 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, PAGE_TABLE_DEPTH, 1);911 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, RANGE_PROTECTION_FAULT_ENABLE_DEFAULT, 1);912 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT, 1);913 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, PDE0_PROTECTION_FAULT_ENABLE_DEFAULT, 1);914 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, VALID_PROTECTION_FAULT_ENABLE_DEFAULT, 1);915 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, READ_PROTECTION_FAULT_ENABLE_DEFAULT, 1);916 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, WRITE_PROTECTION_FAULT_ENABLE_DEFAULT, 1);917 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT, 1);918 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, PAGE_TABLE_BLOCK_SIZE,919 adev->vm_manager.block_size - 9);920 WREG32(mmVM_CONTEXT1_CNTL, tmp);921 if (amdgpu_vm_fault_stop == AMDGPU_VM_FAULT_STOP_ALWAYS)922 gmc_v8_0_set_fault_enable_default(adev, false);923 else924 gmc_v8_0_set_fault_enable_default(adev, true);925 926 gmc_v8_0_flush_gpu_tlb(adev, 0, 0, 0);927 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",928 (unsigned int)(adev->gmc.gart_size >> 20),929 (unsigned long long)table_addr);930 return 0;931}932 933static int gmc_v8_0_gart_init(struct amdgpu_device *adev)934{935 int r;936 937 if (adev->gart.bo) {938 WARN(1, "R600 PCIE GART already initialized\n");939 return 0;940 }941 /* Initialize common gart structure */942 r = amdgpu_gart_init(adev);943 if (r)944 return r;945 adev->gart.table_size = adev->gart.num_gpu_pages * 8;946 adev->gart.gart_pte_flags = AMDGPU_PTE_EXECUTABLE;947 return amdgpu_gart_table_vram_alloc(adev);948}949 950/**951 * gmc_v8_0_gart_disable - gart disable952 *953 * @adev: amdgpu_device pointer954 *955 * This disables all VM page table (VI).956 */957static void gmc_v8_0_gart_disable(struct amdgpu_device *adev)958{959 u32 tmp;960 961 /* Disable all tables */962 WREG32(mmVM_CONTEXT0_CNTL, 0);963 WREG32(mmVM_CONTEXT1_CNTL, 0);964 /* Setup TLB control */965 tmp = RREG32(mmMC_VM_MX_L1_TLB_CNTL);966 tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, ENABLE_L1_TLB, 0);967 tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, ENABLE_L1_FRAGMENT_PROCESSING, 0);968 tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, ENABLE_ADVANCED_DRIVER_MODEL, 0);969 WREG32(mmMC_VM_MX_L1_TLB_CNTL, tmp);970 /* Setup L2 cache */971 tmp = RREG32(mmVM_L2_CNTL);972 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, ENABLE_L2_CACHE, 0);973 WREG32(mmVM_L2_CNTL, tmp);974 WREG32(mmVM_L2_CNTL2, 0);975}976 977/**978 * gmc_v8_0_vm_decode_fault - print human readable fault info979 *980 * @adev: amdgpu_device pointer981 * @status: VM_CONTEXT1_PROTECTION_FAULT_STATUS register value982 * @addr: VM_CONTEXT1_PROTECTION_FAULT_ADDR register value983 * @mc_client: VM_CONTEXT1_PROTECTION_FAULT_MCCLIENT register value984 * @pasid: debug logging only - no functional use985 *986 * Print human readable fault information (VI).987 */988static void gmc_v8_0_vm_decode_fault(struct amdgpu_device *adev, u32 status,989 u32 addr, u32 mc_client, unsigned int pasid)990{991 u32 vmid = REG_GET_FIELD(status, VM_CONTEXT1_PROTECTION_FAULT_STATUS, VMID);992 u32 protections = REG_GET_FIELD(status, VM_CONTEXT1_PROTECTION_FAULT_STATUS,993 PROTECTIONS);994 char block[5] = { mc_client >> 24, (mc_client >> 16) & 0xff,995 (mc_client >> 8) & 0xff, mc_client & 0xff, 0 };996 u32 mc_id;997 998 mc_id = REG_GET_FIELD(status, VM_CONTEXT1_PROTECTION_FAULT_STATUS,999 MEMORY_CLIENT_ID);1000 1001 dev_err(adev->dev, "VM fault (0x%02x, vmid %d, pasid %d) at page %u, %s from '%s' (0x%08x) (%d)\n",1002 protections, vmid, pasid, addr,1003 REG_GET_FIELD(status, VM_CONTEXT1_PROTECTION_FAULT_STATUS,1004 MEMORY_CLIENT_RW) ?1005 "write" : "read", block, mc_client, mc_id);1006}1007 1008static int gmc_v8_0_convert_vram_type(int mc_seq_vram_type)1009{1010 switch (mc_seq_vram_type) {1011 case MC_SEQ_MISC0__MT__GDDR1:1012 return AMDGPU_VRAM_TYPE_GDDR1;1013 case MC_SEQ_MISC0__MT__DDR2:1014 return AMDGPU_VRAM_TYPE_DDR2;1015 case MC_SEQ_MISC0__MT__GDDR3:1016 return AMDGPU_VRAM_TYPE_GDDR3;1017 case MC_SEQ_MISC0__MT__GDDR4:1018 return AMDGPU_VRAM_TYPE_GDDR4;1019 case MC_SEQ_MISC0__MT__GDDR5:1020 return AMDGPU_VRAM_TYPE_GDDR5;1021 case MC_SEQ_MISC0__MT__HBM:1022 return AMDGPU_VRAM_TYPE_HBM;1023 case MC_SEQ_MISC0__MT__DDR3:1024 return AMDGPU_VRAM_TYPE_DDR3;1025 default:1026 return AMDGPU_VRAM_TYPE_UNKNOWN;1027 }1028}1029 1030static int gmc_v8_0_early_init(void *handle)1031{1032 struct amdgpu_device *adev = (struct amdgpu_device *)handle;1033 1034 gmc_v8_0_set_gmc_funcs(adev);1035 gmc_v8_0_set_irq_funcs(adev);1036 1037 adev->gmc.shared_aperture_start = 0x2000000000000000ULL;1038 adev->gmc.shared_aperture_end =1039 adev->gmc.shared_aperture_start + (4ULL << 30) - 1;1040 adev->gmc.private_aperture_start =1041 adev->gmc.shared_aperture_end + 1;1042 adev->gmc.private_aperture_end =1043 adev->gmc.private_aperture_start + (4ULL << 30) - 1;1044 adev->gmc.noretry_flags = AMDGPU_VM_NORETRY_FLAGS_TF;1045 1046 return 0;1047}1048 1049static int gmc_v8_0_late_init(void *handle)1050{1051 struct amdgpu_device *adev = (struct amdgpu_device *)handle;1052 1053 if (amdgpu_vm_fault_stop != AMDGPU_VM_FAULT_STOP_ALWAYS)1054 return amdgpu_irq_get(adev, &adev->gmc.vm_fault, 0);1055 else1056 return 0;1057}1058 1059static unsigned int gmc_v8_0_get_vbios_fb_size(struct amdgpu_device *adev)1060{1061 u32 d1vga_control = RREG32(mmD1VGA_CONTROL);1062 unsigned int size;1063 1064 if (REG_GET_FIELD(d1vga_control, D1VGA_CONTROL, D1VGA_MODE_ENABLE)) {1065 size = AMDGPU_VBIOS_VGA_ALLOCATION;1066 } else {1067 u32 viewport = RREG32(mmVIEWPORT_SIZE);1068 1069 size = (REG_GET_FIELD(viewport, VIEWPORT_SIZE, VIEWPORT_HEIGHT) *1070 REG_GET_FIELD(viewport, VIEWPORT_SIZE, VIEWPORT_WIDTH) *1071 4);1072 }1073 1074 return size;1075}1076 1077#define mmMC_SEQ_MISC0_FIJI 0xA711078 1079static int gmc_v8_0_sw_init(void *handle)1080{1081 int r;1082 struct amdgpu_device *adev = (struct amdgpu_device *)handle;1083 1084 set_bit(AMDGPU_GFXHUB(0), adev->vmhubs_mask);1085 1086 if (adev->flags & AMD_IS_APU) {1087 adev->gmc.vram_type = AMDGPU_VRAM_TYPE_UNKNOWN;1088 } else {1089 u32 tmp;1090 1091 if ((adev->asic_type == CHIP_FIJI) ||1092 (adev->asic_type == CHIP_VEGAM))1093 tmp = RREG32(mmMC_SEQ_MISC0_FIJI);1094 else1095 tmp = RREG32(mmMC_SEQ_MISC0);1096 tmp &= MC_SEQ_MISC0__MT__MASK;1097 adev->gmc.vram_type = gmc_v8_0_convert_vram_type(tmp);1098 }1099 1100 r = amdgpu_irq_add_id(adev, AMDGPU_IRQ_CLIENTID_LEGACY, VISLANDS30_IV_SRCID_GFX_PAGE_INV_FAULT, &adev->gmc.vm_fault);1101 if (r)1102 return r;1103 1104 r = amdgpu_irq_add_id(adev, AMDGPU_IRQ_CLIENTID_LEGACY, VISLANDS30_IV_SRCID_GFX_MEM_PROT_FAULT, &adev->gmc.vm_fault);1105 if (r)1106 return r;1107 1108 /* Adjust VM size here.1109 * Currently set to 4GB ((1 << 20) 4k pages).1110 * Max GPUVM size for cayman and SI is 40 bits.1111 */1112 amdgpu_vm_adjust_size(adev, 64, 9, 1, 40);1113 1114 /* Set the internal MC address mask1115 * This is the max address of the GPU's1116 * internal address space.1117 */1118 adev->gmc.mc_mask = 0xffffffffffULL; /* 40 bit MC */1119 1120 r = dma_set_mask_and_coherent(adev->dev, DMA_BIT_MASK(40));1121 if (r) {1122 pr_warn("No suitable DMA available\n");1123 return r;1124 }1125 adev->need_swiotlb = drm_need_swiotlb(40);1126 1127 r = gmc_v8_0_init_microcode(adev);1128 if (r) {1129 DRM_ERROR("Failed to load mc firmware!\n");1130 return r;1131 }1132 1133 r = gmc_v8_0_mc_init(adev);1134 if (r)1135 return r;1136 1137 amdgpu_gmc_get_vbios_allocations(adev);1138 1139 /* Memory manager */1140 r = amdgpu_bo_init(adev);1141 if (r)1142 return r;1143 1144 r = gmc_v8_0_gart_init(adev);1145 if (r)1146 return r;1147 1148 /*1149 * number of VMs1150 * VMID 0 is reserved for System1151 * amdgpu graphics/compute will use VMIDs 1-71152 * amdkfd will use VMIDs 8-151153 */1154 adev->vm_manager.first_kfd_vmid = 8;1155 amdgpu_vm_manager_init(adev);1156 1157 /* base offset of vram pages */1158 if (adev->flags & AMD_IS_APU) {1159 u64 tmp = RREG32(mmMC_VM_FB_OFFSET);1160 1161 tmp <<= 22;1162 adev->vm_manager.vram_base_offset = tmp;1163 } else {1164 adev->vm_manager.vram_base_offset = 0;1165 }1166 1167 adev->gmc.vm_fault_info = kmalloc(sizeof(struct kfd_vm_fault_info),1168 GFP_KERNEL);1169 if (!adev->gmc.vm_fault_info)1170 return -ENOMEM;1171 atomic_set(&adev->gmc.vm_fault_info_updated, 0);1172 1173 return 0;1174}1175 1176static int gmc_v8_0_sw_fini(void *handle)1177{1178 struct amdgpu_device *adev = (struct amdgpu_device *)handle;1179 1180 amdgpu_gem_force_release(adev);1181 amdgpu_vm_manager_fini(adev);1182 kfree(adev->gmc.vm_fault_info);1183 amdgpu_gart_table_vram_free(adev);1184 amdgpu_bo_fini(adev);1185 amdgpu_ucode_release(&adev->gmc.fw);1186 1187 return 0;1188}1189 1190static int gmc_v8_0_hw_init(void *handle)1191{1192 int r;1193 struct amdgpu_device *adev = (struct amdgpu_device *)handle;1194 1195 gmc_v8_0_init_golden_registers(adev);1196 1197 gmc_v8_0_mc_program(adev);1198 1199 if (adev->asic_type == CHIP_TONGA) {1200 r = gmc_v8_0_tonga_mc_load_microcode(adev);1201 if (r) {1202 DRM_ERROR("Failed to load MC firmware!\n");1203 return r;1204 }1205 } else if (adev->asic_type == CHIP_POLARIS11 ||1206 adev->asic_type == CHIP_POLARIS10 ||1207 adev->asic_type == CHIP_POLARIS12) {1208 r = gmc_v8_0_polaris_mc_load_microcode(adev);1209 if (r) {1210 DRM_ERROR("Failed to load MC firmware!\n");1211 return r;1212 }1213 }1214 1215 r = gmc_v8_0_gart_enable(adev);1216 if (r)1217 return r;1218 1219 if (amdgpu_emu_mode == 1)1220 return amdgpu_gmc_vram_checking(adev);1221 1222 return 0;1223}1224 1225static int gmc_v8_0_hw_fini(void *handle)1226{1227 struct amdgpu_device *adev = (struct amdgpu_device *)handle;1228 1229 amdgpu_irq_put(adev, &adev->gmc.vm_fault, 0);1230 gmc_v8_0_gart_disable(adev);1231 1232 return 0;1233}1234 1235static int gmc_v8_0_suspend(void *handle)1236{1237 struct amdgpu_device *adev = (struct amdgpu_device *)handle;1238 1239 gmc_v8_0_hw_fini(adev);1240 1241 return 0;1242}1243 1244static int gmc_v8_0_resume(void *handle)1245{1246 int r;1247 struct amdgpu_device *adev = (struct amdgpu_device *)handle;1248 1249 r = gmc_v8_0_hw_init(adev);1250 if (r)1251 return r;1252 1253 amdgpu_vmid_reset_all(adev);1254 1255 return 0;1256}1257 1258static bool gmc_v8_0_is_idle(void *handle)1259{1260 struct amdgpu_device *adev = (struct amdgpu_device *)handle;1261 u32 tmp = RREG32(mmSRBM_STATUS);1262 1263 if (tmp & (SRBM_STATUS__MCB_BUSY_MASK | SRBM_STATUS__MCB_NON_DISPLAY_BUSY_MASK |1264 SRBM_STATUS__MCC_BUSY_MASK | SRBM_STATUS__MCD_BUSY_MASK | SRBM_STATUS__VMC_BUSY_MASK))1265 return false;1266 1267 return true;1268}1269 1270static int gmc_v8_0_wait_for_idle(void *handle)1271{1272 unsigned int i;1273 u32 tmp;1274 struct amdgpu_device *adev = (struct amdgpu_device *)handle;1275 1276 for (i = 0; i < adev->usec_timeout; i++) {1277 /* read MC_STATUS */1278 tmp = RREG32(mmSRBM_STATUS) & (SRBM_STATUS__MCB_BUSY_MASK |1279 SRBM_STATUS__MCB_NON_DISPLAY_BUSY_MASK |1280 SRBM_STATUS__MCC_BUSY_MASK |1281 SRBM_STATUS__MCD_BUSY_MASK |1282 SRBM_STATUS__VMC_BUSY_MASK |1283 SRBM_STATUS__VMC1_BUSY_MASK);1284 if (!tmp)1285 return 0;1286 udelay(1);1287 }1288 return -ETIMEDOUT;1289 1290}1291 1292static bool gmc_v8_0_check_soft_reset(void *handle)1293{1294 u32 srbm_soft_reset = 0;1295 struct amdgpu_device *adev = (struct amdgpu_device *)handle;1296 u32 tmp = RREG32(mmSRBM_STATUS);1297 1298 if (tmp & SRBM_STATUS__VMC_BUSY_MASK)1299 srbm_soft_reset = REG_SET_FIELD(srbm_soft_reset,1300 SRBM_SOFT_RESET, SOFT_RESET_VMC, 1);1301 1302 if (tmp & (SRBM_STATUS__MCB_BUSY_MASK | SRBM_STATUS__MCB_NON_DISPLAY_BUSY_MASK |1303 SRBM_STATUS__MCC_BUSY_MASK | SRBM_STATUS__MCD_BUSY_MASK)) {1304 if (!(adev->flags & AMD_IS_APU))1305 srbm_soft_reset = REG_SET_FIELD(srbm_soft_reset,1306 SRBM_SOFT_RESET, SOFT_RESET_MC, 1);1307 }1308 1309 if (srbm_soft_reset) {1310 adev->gmc.srbm_soft_reset = srbm_soft_reset;1311 return true;1312 }1313 1314 adev->gmc.srbm_soft_reset = 0;1315 1316 return false;1317}1318 1319static int gmc_v8_0_pre_soft_reset(void *handle)1320{1321 struct amdgpu_device *adev = (struct amdgpu_device *)handle;1322 1323 if (!adev->gmc.srbm_soft_reset)1324 return 0;1325 1326 gmc_v8_0_mc_stop(adev);1327 if (gmc_v8_0_wait_for_idle(adev))1328 dev_warn(adev->dev, "Wait for GMC idle timed out !\n");1329 1330 return 0;1331}1332 1333static int gmc_v8_0_soft_reset(void *handle)1334{1335 struct amdgpu_device *adev = (struct amdgpu_device *)handle;1336 u32 srbm_soft_reset;1337 1338 if (!adev->gmc.srbm_soft_reset)1339 return 0;1340 srbm_soft_reset = adev->gmc.srbm_soft_reset;1341 1342 if (srbm_soft_reset) {1343 u32 tmp;1344 1345 tmp = RREG32(mmSRBM_SOFT_RESET);1346 tmp |= srbm_soft_reset;1347 dev_info(adev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);1348 WREG32(mmSRBM_SOFT_RESET, tmp);1349 tmp = RREG32(mmSRBM_SOFT_RESET);1350 1351 udelay(50);1352 1353 tmp &= ~srbm_soft_reset;1354 WREG32(mmSRBM_SOFT_RESET, tmp);1355 tmp = RREG32(mmSRBM_SOFT_RESET);1356 1357 /* Wait a little for things to settle down */1358 udelay(50);1359 }1360 1361 return 0;1362}1363 1364static int gmc_v8_0_post_soft_reset(void *handle)1365{1366 struct amdgpu_device *adev = (struct amdgpu_device *)handle;1367 1368 if (!adev->gmc.srbm_soft_reset)1369 return 0;1370 1371 gmc_v8_0_mc_resume(adev);1372 return 0;1373}1374 1375static int gmc_v8_0_vm_fault_interrupt_state(struct amdgpu_device *adev,1376 struct amdgpu_irq_src *src,1377 unsigned int type,1378 enum amdgpu_interrupt_state state)1379{1380 u32 tmp;1381 u32 bits = (VM_CONTEXT1_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |1382 VM_CONTEXT1_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |1383 VM_CONTEXT1_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |1384 VM_CONTEXT1_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |1385 VM_CONTEXT1_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |1386 VM_CONTEXT1_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |1387 VM_CONTEXT1_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK);1388 1389 switch (state) {1390 case AMDGPU_IRQ_STATE_DISABLE:1391 /* system context */1392 tmp = RREG32(mmVM_CONTEXT0_CNTL);1393 tmp &= ~bits;1394 WREG32(mmVM_CONTEXT0_CNTL, tmp);1395 /* VMs */1396 tmp = RREG32(mmVM_CONTEXT1_CNTL);1397 tmp &= ~bits;1398 WREG32(mmVM_CONTEXT1_CNTL, tmp);1399 break;1400 case AMDGPU_IRQ_STATE_ENABLE:1401 /* system context */1402 tmp = RREG32(mmVM_CONTEXT0_CNTL);1403 tmp |= bits;1404 WREG32(mmVM_CONTEXT0_CNTL, tmp);1405 /* VMs */1406 tmp = RREG32(mmVM_CONTEXT1_CNTL);1407 tmp |= bits;1408 WREG32(mmVM_CONTEXT1_CNTL, tmp);1409 break;1410 default:1411 break;1412 }1413 1414 return 0;1415}1416 1417static int gmc_v8_0_process_interrupt(struct amdgpu_device *adev,1418 struct amdgpu_irq_src *source,1419 struct amdgpu_iv_entry *entry)1420{1421 u32 addr, status, mc_client, vmid;1422 1423 if (amdgpu_sriov_vf(adev)) {1424 dev_err(adev->dev, "GPU fault detected: %d 0x%08x\n",1425 entry->src_id, entry->src_data[0]);1426 dev_err(adev->dev, " Can't decode VM fault info here on SRIOV VF\n");1427 return 0;1428 }1429 1430 addr = RREG32(mmVM_CONTEXT1_PROTECTION_FAULT_ADDR);1431 status = RREG32(mmVM_CONTEXT1_PROTECTION_FAULT_STATUS);1432 mc_client = RREG32(mmVM_CONTEXT1_PROTECTION_FAULT_MCCLIENT);1433 /* reset addr and status */1434 WREG32_P(mmVM_CONTEXT1_CNTL2, 1, ~1);1435 1436 if (!addr && !status)1437 return 0;1438 1439 amdgpu_vm_update_fault_cache(adev, entry->pasid,1440 ((u64)addr) << AMDGPU_GPU_PAGE_SHIFT, status, AMDGPU_GFXHUB(0));1441 1442 if (amdgpu_vm_fault_stop == AMDGPU_VM_FAULT_STOP_FIRST)1443 gmc_v8_0_set_fault_enable_default(adev, false);1444 1445 if (printk_ratelimit()) {1446 struct amdgpu_task_info *task_info;1447 1448 dev_err(adev->dev, "GPU fault detected: %d 0x%08x\n",1449 entry->src_id, entry->src_data[0]);1450 1451 task_info = amdgpu_vm_get_task_info_pasid(adev, entry->pasid);1452 if (task_info) {1453 dev_err(adev->dev, " for process %s pid %d thread %s pid %d\n",1454 task_info->process_name, task_info->tgid,1455 task_info->task_name, task_info->pid);1456 amdgpu_vm_put_task_info(task_info);1457 }1458 1459 dev_err(adev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",1460 addr);1461 dev_err(adev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",1462 status);1463 1464 gmc_v8_0_vm_decode_fault(adev, status, addr, mc_client,1465 entry->pasid);1466 }1467 1468 vmid = REG_GET_FIELD(status, VM_CONTEXT1_PROTECTION_FAULT_STATUS,1469 VMID);1470 if (amdgpu_amdkfd_is_kfd_vmid(adev, vmid)1471 && !atomic_read(&adev->gmc.vm_fault_info_updated)) {1472 struct kfd_vm_fault_info *info = adev->gmc.vm_fault_info;1473 u32 protections = REG_GET_FIELD(status,1474 VM_CONTEXT1_PROTECTION_FAULT_STATUS,1475 PROTECTIONS);1476 1477 info->vmid = vmid;1478 info->mc_id = REG_GET_FIELD(status,1479 VM_CONTEXT1_PROTECTION_FAULT_STATUS,1480 MEMORY_CLIENT_ID);1481 info->status = status;1482 info->page_addr = addr;1483 info->prot_valid = protections & 0x7 ? true : false;1484 info->prot_read = protections & 0x8 ? true : false;1485 info->prot_write = protections & 0x10 ? true : false;1486 info->prot_exec = protections & 0x20 ? true : false;1487 mb();1488 atomic_set(&adev->gmc.vm_fault_info_updated, 1);1489 }1490 1491 return 0;1492}1493 1494static void fiji_update_mc_medium_grain_clock_gating(struct amdgpu_device *adev,1495 bool enable)1496{1497 uint32_t data;1498 1499 if (enable && (adev->cg_flags & AMD_CG_SUPPORT_MC_MGCG)) {1500 data = RREG32(mmMC_HUB_MISC_HUB_CG);1501 data |= MC_HUB_MISC_HUB_CG__ENABLE_MASK;1502 WREG32(mmMC_HUB_MISC_HUB_CG, data);1503 1504 data = RREG32(mmMC_HUB_MISC_SIP_CG);1505 data |= MC_HUB_MISC_SIP_CG__ENABLE_MASK;1506 WREG32(mmMC_HUB_MISC_SIP_CG, data);1507 1508 data = RREG32(mmMC_HUB_MISC_VM_CG);1509 data |= MC_HUB_MISC_VM_CG__ENABLE_MASK;1510 WREG32(mmMC_HUB_MISC_VM_CG, data);1511 1512 data = RREG32(mmMC_XPB_CLK_GAT);1513 data |= MC_XPB_CLK_GAT__ENABLE_MASK;1514 WREG32(mmMC_XPB_CLK_GAT, data);1515 1516 data = RREG32(mmATC_MISC_CG);1517 data |= ATC_MISC_CG__ENABLE_MASK;1518 WREG32(mmATC_MISC_CG, data);1519 1520 data = RREG32(mmMC_CITF_MISC_WR_CG);1521 data |= MC_CITF_MISC_WR_CG__ENABLE_MASK;1522 WREG32(mmMC_CITF_MISC_WR_CG, data);1523 1524 data = RREG32(mmMC_CITF_MISC_RD_CG);1525 data |= MC_CITF_MISC_RD_CG__ENABLE_MASK;1526 WREG32(mmMC_CITF_MISC_RD_CG, data);1527 1528 data = RREG32(mmMC_CITF_MISC_VM_CG);1529 data |= MC_CITF_MISC_VM_CG__ENABLE_MASK;1530 WREG32(mmMC_CITF_MISC_VM_CG, data);1531 1532 data = RREG32(mmVM_L2_CG);1533 data |= VM_L2_CG__ENABLE_MASK;1534 WREG32(mmVM_L2_CG, data);1535 } else {1536 data = RREG32(mmMC_HUB_MISC_HUB_CG);1537 data &= ~MC_HUB_MISC_HUB_CG__ENABLE_MASK;1538 WREG32(mmMC_HUB_MISC_HUB_CG, data);1539 1540 data = RREG32(mmMC_HUB_MISC_SIP_CG);1541 data &= ~MC_HUB_MISC_SIP_CG__ENABLE_MASK;1542 WREG32(mmMC_HUB_MISC_SIP_CG, data);1543 1544 data = RREG32(mmMC_HUB_MISC_VM_CG);1545 data &= ~MC_HUB_MISC_VM_CG__ENABLE_MASK;1546 WREG32(mmMC_HUB_MISC_VM_CG, data);1547 1548 data = RREG32(mmMC_XPB_CLK_GAT);1549 data &= ~MC_XPB_CLK_GAT__ENABLE_MASK;1550 WREG32(mmMC_XPB_CLK_GAT, data);1551 1552 data = RREG32(mmATC_MISC_CG);1553 data &= ~ATC_MISC_CG__ENABLE_MASK;1554 WREG32(mmATC_MISC_CG, data);1555 1556 data = RREG32(mmMC_CITF_MISC_WR_CG);1557 data &= ~MC_CITF_MISC_WR_CG__ENABLE_MASK;1558 WREG32(mmMC_CITF_MISC_WR_CG, data);1559 1560 data = RREG32(mmMC_CITF_MISC_RD_CG);1561 data &= ~MC_CITF_MISC_RD_CG__ENABLE_MASK;1562 WREG32(mmMC_CITF_MISC_RD_CG, data);1563 1564 data = RREG32(mmMC_CITF_MISC_VM_CG);1565 data &= ~MC_CITF_MISC_VM_CG__ENABLE_MASK;1566 WREG32(mmMC_CITF_MISC_VM_CG, data);1567 1568 data = RREG32(mmVM_L2_CG);1569 data &= ~VM_L2_CG__ENABLE_MASK;1570 WREG32(mmVM_L2_CG, data);1571 }1572}1573 1574static void fiji_update_mc_light_sleep(struct amdgpu_device *adev,1575 bool enable)1576{1577 uint32_t data;1578 1579 if (enable && (adev->cg_flags & AMD_CG_SUPPORT_MC_LS)) {1580 data = RREG32(mmMC_HUB_MISC_HUB_CG);1581 data |= MC_HUB_MISC_HUB_CG__MEM_LS_ENABLE_MASK;1582 WREG32(mmMC_HUB_MISC_HUB_CG, data);1583 1584 data = RREG32(mmMC_HUB_MISC_SIP_CG);1585 data |= MC_HUB_MISC_SIP_CG__MEM_LS_ENABLE_MASK;1586 WREG32(mmMC_HUB_MISC_SIP_CG, data);1587 1588 data = RREG32(mmMC_HUB_MISC_VM_CG);1589 data |= MC_HUB_MISC_VM_CG__MEM_LS_ENABLE_MASK;1590 WREG32(mmMC_HUB_MISC_VM_CG, data);1591 1592 data = RREG32(mmMC_XPB_CLK_GAT);1593 data |= MC_XPB_CLK_GAT__MEM_LS_ENABLE_MASK;1594 WREG32(mmMC_XPB_CLK_GAT, data);1595 1596 data = RREG32(mmATC_MISC_CG);1597 data |= ATC_MISC_CG__MEM_LS_ENABLE_MASK;1598 WREG32(mmATC_MISC_CG, data);1599 1600 data = RREG32(mmMC_CITF_MISC_WR_CG);1601 data |= MC_CITF_MISC_WR_CG__MEM_LS_ENABLE_MASK;1602 WREG32(mmMC_CITF_MISC_WR_CG, data);1603 1604 data = RREG32(mmMC_CITF_MISC_RD_CG);1605 data |= MC_CITF_MISC_RD_CG__MEM_LS_ENABLE_MASK;1606 WREG32(mmMC_CITF_MISC_RD_CG, data);1607 1608 data = RREG32(mmMC_CITF_MISC_VM_CG);1609 data |= MC_CITF_MISC_VM_CG__MEM_LS_ENABLE_MASK;1610 WREG32(mmMC_CITF_MISC_VM_CG, data);1611 1612 data = RREG32(mmVM_L2_CG);1613 data |= VM_L2_CG__MEM_LS_ENABLE_MASK;1614 WREG32(mmVM_L2_CG, data);1615 } else {1616 data = RREG32(mmMC_HUB_MISC_HUB_CG);1617 data &= ~MC_HUB_MISC_HUB_CG__MEM_LS_ENABLE_MASK;1618 WREG32(mmMC_HUB_MISC_HUB_CG, data);1619 1620 data = RREG32(mmMC_HUB_MISC_SIP_CG);1621 data &= ~MC_HUB_MISC_SIP_CG__MEM_LS_ENABLE_MASK;1622 WREG32(mmMC_HUB_MISC_SIP_CG, data);1623 1624 data = RREG32(mmMC_HUB_MISC_VM_CG);1625 data &= ~MC_HUB_MISC_VM_CG__MEM_LS_ENABLE_MASK;1626 WREG32(mmMC_HUB_MISC_VM_CG, data);1627 1628 data = RREG32(mmMC_XPB_CLK_GAT);1629 data &= ~MC_XPB_CLK_GAT__MEM_LS_ENABLE_MASK;1630 WREG32(mmMC_XPB_CLK_GAT, data);1631 1632 data = RREG32(mmATC_MISC_CG);1633 data &= ~ATC_MISC_CG__MEM_LS_ENABLE_MASK;1634 WREG32(mmATC_MISC_CG, data);1635 1636 data = RREG32(mmMC_CITF_MISC_WR_CG);1637 data &= ~MC_CITF_MISC_WR_CG__MEM_LS_ENABLE_MASK;1638 WREG32(mmMC_CITF_MISC_WR_CG, data);1639 1640 data = RREG32(mmMC_CITF_MISC_RD_CG);1641 data &= ~MC_CITF_MISC_RD_CG__MEM_LS_ENABLE_MASK;1642 WREG32(mmMC_CITF_MISC_RD_CG, data);1643 1644 data = RREG32(mmMC_CITF_MISC_VM_CG);1645 data &= ~MC_CITF_MISC_VM_CG__MEM_LS_ENABLE_MASK;1646 WREG32(mmMC_CITF_MISC_VM_CG, data);1647 1648 data = RREG32(mmVM_L2_CG);1649 data &= ~VM_L2_CG__MEM_LS_ENABLE_MASK;1650 WREG32(mmVM_L2_CG, data);1651 }1652}1653 1654static int gmc_v8_0_set_clockgating_state(void *handle,1655 enum amd_clockgating_state state)1656{1657 struct amdgpu_device *adev = (struct amdgpu_device *)handle;1658 1659 if (amdgpu_sriov_vf(adev))1660 return 0;1661 1662 switch (adev->asic_type) {1663 case CHIP_FIJI:1664 fiji_update_mc_medium_grain_clock_gating(adev,1665 state == AMD_CG_STATE_GATE);1666 fiji_update_mc_light_sleep(adev,1667 state == AMD_CG_STATE_GATE);1668 break;1669 default:1670 break;1671 }1672 return 0;1673}1674 1675static int gmc_v8_0_set_powergating_state(void *handle,1676 enum amd_powergating_state state)1677{1678 return 0;1679}1680 1681static void gmc_v8_0_get_clockgating_state(void *handle, u64 *flags)1682{1683 struct amdgpu_device *adev = (struct amdgpu_device *)handle;1684 int data;1685 1686 if (amdgpu_sriov_vf(adev))1687 *flags = 0;1688 1689 /* AMD_CG_SUPPORT_MC_MGCG */1690 data = RREG32(mmMC_HUB_MISC_HUB_CG);1691 if (data & MC_HUB_MISC_HUB_CG__ENABLE_MASK)1692 *flags |= AMD_CG_SUPPORT_MC_MGCG;1693 1694 /* AMD_CG_SUPPORT_MC_LS */1695 if (data & MC_HUB_MISC_HUB_CG__MEM_LS_ENABLE_MASK)1696 *flags |= AMD_CG_SUPPORT_MC_LS;1697}1698 1699static const struct amd_ip_funcs gmc_v8_0_ip_funcs = {1700 .name = "gmc_v8_0",1701 .early_init = gmc_v8_0_early_init,1702 .late_init = gmc_v8_0_late_init,1703 .sw_init = gmc_v8_0_sw_init,1704 .sw_fini = gmc_v8_0_sw_fini,1705 .hw_init = gmc_v8_0_hw_init,1706 .hw_fini = gmc_v8_0_hw_fini,1707 .suspend = gmc_v8_0_suspend,1708 .resume = gmc_v8_0_resume,1709 .is_idle = gmc_v8_0_is_idle,1710 .wait_for_idle = gmc_v8_0_wait_for_idle,1711 .check_soft_reset = gmc_v8_0_check_soft_reset,1712 .pre_soft_reset = gmc_v8_0_pre_soft_reset,1713 .soft_reset = gmc_v8_0_soft_reset,1714 .post_soft_reset = gmc_v8_0_post_soft_reset,1715 .set_clockgating_state = gmc_v8_0_set_clockgating_state,1716 .set_powergating_state = gmc_v8_0_set_powergating_state,1717 .get_clockgating_state = gmc_v8_0_get_clockgating_state,1718 .dump_ip_state = NULL,1719 .print_ip_state = NULL,1720};1721 1722static const struct amdgpu_gmc_funcs gmc_v8_0_gmc_funcs = {1723 .flush_gpu_tlb = gmc_v8_0_flush_gpu_tlb,1724 .flush_gpu_tlb_pasid = gmc_v8_0_flush_gpu_tlb_pasid,1725 .emit_flush_gpu_tlb = gmc_v8_0_emit_flush_gpu_tlb,1726 .emit_pasid_mapping = gmc_v8_0_emit_pasid_mapping,1727 .set_prt = gmc_v8_0_set_prt,1728 .get_vm_pde = gmc_v8_0_get_vm_pde,1729 .get_vm_pte = gmc_v8_0_get_vm_pte,1730 .get_vbios_fb_size = gmc_v8_0_get_vbios_fb_size,1731};1732 1733static const struct amdgpu_irq_src_funcs gmc_v8_0_irq_funcs = {1734 .set = gmc_v8_0_vm_fault_interrupt_state,1735 .process = gmc_v8_0_process_interrupt,1736};1737 1738static void gmc_v8_0_set_gmc_funcs(struct amdgpu_device *adev)1739{1740 adev->gmc.gmc_funcs = &gmc_v8_0_gmc_funcs;1741}1742 1743static void gmc_v8_0_set_irq_funcs(struct amdgpu_device *adev)1744{1745 adev->gmc.vm_fault.num_types = 1;1746 adev->gmc.vm_fault.funcs = &gmc_v8_0_irq_funcs;1747}1748 1749const struct amdgpu_ip_block_version gmc_v8_0_ip_block = {1750 .type = AMD_IP_BLOCK_TYPE_GMC,1751 .major = 8,1752 .minor = 0,1753 .rev = 0,1754 .funcs = &gmc_v8_0_ip_funcs,1755};1756 1757const struct amdgpu_ip_block_version gmc_v8_1_ip_block = {1758 .type = AMD_IP_BLOCK_TYPE_GMC,1759 .major = 8,1760 .minor = 1,1761 .rev = 0,1762 .funcs = &gmc_v8_0_ip_funcs,1763};1764 1765const struct amdgpu_ip_block_version gmc_v8_5_ip_block = {1766 .type = AMD_IP_BLOCK_TYPE_GMC,1767 .major = 8,1768 .minor = 5,1769 .rev = 0,1770 .funcs = &gmc_v8_0_ip_funcs,1771};1772