424 lines · c
1/*2 * Copyright 2019 Advanced Micro Devices, Inc.3 *4 * Permission is hereby granted, free of charge, to any person obtaining a5 * copy of this software and associated documentation files (the "Software"),6 * to deal in the Software without restriction, including without limitation7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,8 * and/or sell copies of the Software, and to permit persons to whom the9 * Software is furnished to do so, subject to the following conditions:10 *11 * The above copyright notice and this permission notice shall be included in12 * all copies or substantial portions of the Software.13 *14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR20 * OTHER DEALINGS IN THE SOFTWARE.21 */22#include <linux/module.h>23#include <linux/uaccess.h>24#include <linux/firmware.h>25#include "amdgpu.h"26#include "amdgpu_amdkfd.h"27#include "amdgpu_amdkfd_arcturus.h"28#include "amdgpu_reset.h"29#include "sdma0/sdma0_4_2_2_offset.h"30#include "sdma0/sdma0_4_2_2_sh_mask.h"31#include "sdma1/sdma1_4_2_2_offset.h"32#include "sdma1/sdma1_4_2_2_sh_mask.h"33#include "sdma2/sdma2_4_2_2_offset.h"34#include "sdma2/sdma2_4_2_2_sh_mask.h"35#include "sdma3/sdma3_4_2_2_offset.h"36#include "sdma3/sdma3_4_2_2_sh_mask.h"37#include "sdma4/sdma4_4_2_2_offset.h"38#include "sdma4/sdma4_4_2_2_sh_mask.h"39#include "sdma5/sdma5_4_2_2_offset.h"40#include "sdma5/sdma5_4_2_2_sh_mask.h"41#include "sdma6/sdma6_4_2_2_offset.h"42#include "sdma6/sdma6_4_2_2_sh_mask.h"43#include "sdma7/sdma7_4_2_2_offset.h"44#include "sdma7/sdma7_4_2_2_sh_mask.h"45#include "v9_structs.h"46#include "soc15.h"47#include "soc15d.h"48#include "amdgpu_amdkfd_gfx_v9.h"49#include "gfxhub_v1_0.h"50#include "mmhub_v9_4.h"51#include "gc/gc_9_0_offset.h"52#include "gc/gc_9_0_sh_mask.h"53 54#define HQD_N_REGS 5655#define DUMP_REG(addr) do { \56 if (WARN_ON_ONCE(i >= HQD_N_REGS)) \57 break; \58 (*dump)[i][0] = (addr) << 2; \59 (*dump)[i++][1] = RREG32(addr); \60 } while (0)61 62static inline struct v9_sdma_mqd *get_sdma_mqd(void *mqd)63{64 return (struct v9_sdma_mqd *)mqd;65}66 67static uint32_t get_sdma_rlc_reg_offset(struct amdgpu_device *adev,68 unsigned int engine_id,69 unsigned int queue_id)70{71 uint32_t sdma_engine_reg_base = 0;72 uint32_t sdma_rlc_reg_offset;73 74 switch (engine_id) {75 default:76 dev_warn(adev->dev,77 "Invalid sdma engine id (%d), using engine id 0\n",78 engine_id);79 fallthrough;80 case 0:81 sdma_engine_reg_base = SOC15_REG_OFFSET(SDMA0, 0,82 mmSDMA0_RLC0_RB_CNTL) - mmSDMA0_RLC0_RB_CNTL;83 break;84 case 1:85 sdma_engine_reg_base = SOC15_REG_OFFSET(SDMA1, 0,86 mmSDMA1_RLC0_RB_CNTL) - mmSDMA1_RLC0_RB_CNTL;87 break;88 case 2:89 sdma_engine_reg_base = SOC15_REG_OFFSET(SDMA2, 0,90 mmSDMA2_RLC0_RB_CNTL) - mmSDMA2_RLC0_RB_CNTL;91 break;92 case 3:93 sdma_engine_reg_base = SOC15_REG_OFFSET(SDMA3, 0,94 mmSDMA3_RLC0_RB_CNTL) - mmSDMA3_RLC0_RB_CNTL;95 break;96 case 4:97 sdma_engine_reg_base = SOC15_REG_OFFSET(SDMA4, 0,98 mmSDMA4_RLC0_RB_CNTL) - mmSDMA4_RLC0_RB_CNTL;99 break;100 case 5:101 sdma_engine_reg_base = SOC15_REG_OFFSET(SDMA5, 0,102 mmSDMA5_RLC0_RB_CNTL) - mmSDMA5_RLC0_RB_CNTL;103 break;104 case 6:105 sdma_engine_reg_base = SOC15_REG_OFFSET(SDMA6, 0,106 mmSDMA6_RLC0_RB_CNTL) - mmSDMA6_RLC0_RB_CNTL;107 break;108 case 7:109 sdma_engine_reg_base = SOC15_REG_OFFSET(SDMA7, 0,110 mmSDMA7_RLC0_RB_CNTL) - mmSDMA7_RLC0_RB_CNTL;111 break;112 }113 114 sdma_rlc_reg_offset = sdma_engine_reg_base115 + queue_id * (mmSDMA0_RLC1_RB_CNTL - mmSDMA0_RLC0_RB_CNTL);116 117 pr_debug("RLC register offset for SDMA%d RLC%d: 0x%x\n", engine_id,118 queue_id, sdma_rlc_reg_offset);119 120 return sdma_rlc_reg_offset;121}122 123int kgd_arcturus_hqd_sdma_load(struct amdgpu_device *adev, void *mqd,124 uint32_t __user *wptr, struct mm_struct *mm)125{126 struct v9_sdma_mqd *m;127 uint32_t sdma_rlc_reg_offset;128 unsigned long end_jiffies;129 uint32_t data;130 uint64_t data64;131 uint64_t __user *wptr64 = (uint64_t __user *)wptr;132 133 m = get_sdma_mqd(mqd);134 sdma_rlc_reg_offset = get_sdma_rlc_reg_offset(adev, m->sdma_engine_id,135 m->sdma_queue_id);136 137 WREG32(sdma_rlc_reg_offset + mmSDMA0_RLC0_RB_CNTL,138 m->sdmax_rlcx_rb_cntl & (~SDMA0_RLC0_RB_CNTL__RB_ENABLE_MASK));139 140 end_jiffies = msecs_to_jiffies(2000) + jiffies;141 while (true) {142 data = RREG32(sdma_rlc_reg_offset + mmSDMA0_RLC0_CONTEXT_STATUS);143 if (data & SDMA0_RLC0_CONTEXT_STATUS__IDLE_MASK)144 break;145 if (time_after(jiffies, end_jiffies)) {146 pr_err("SDMA RLC not idle in %s\n", __func__);147 return -ETIME;148 }149 usleep_range(500, 1000);150 }151 152 WREG32(sdma_rlc_reg_offset + mmSDMA0_RLC0_DOORBELL_OFFSET,153 m->sdmax_rlcx_doorbell_offset);154 155 data = REG_SET_FIELD(m->sdmax_rlcx_doorbell, SDMA0_RLC0_DOORBELL,156 ENABLE, 1);157 WREG32(sdma_rlc_reg_offset + mmSDMA0_RLC0_DOORBELL, data);158 WREG32(sdma_rlc_reg_offset + mmSDMA0_RLC0_RB_RPTR,159 m->sdmax_rlcx_rb_rptr);160 WREG32(sdma_rlc_reg_offset + mmSDMA0_RLC0_RB_RPTR_HI,161 m->sdmax_rlcx_rb_rptr_hi);162 163 WREG32(sdma_rlc_reg_offset + mmSDMA0_RLC0_MINOR_PTR_UPDATE, 1);164 if (read_user_wptr(mm, wptr64, data64)) {165 WREG32(sdma_rlc_reg_offset + mmSDMA0_RLC0_RB_WPTR,166 lower_32_bits(data64));167 WREG32(sdma_rlc_reg_offset + mmSDMA0_RLC0_RB_WPTR_HI,168 upper_32_bits(data64));169 } else {170 WREG32(sdma_rlc_reg_offset + mmSDMA0_RLC0_RB_WPTR,171 m->sdmax_rlcx_rb_rptr);172 WREG32(sdma_rlc_reg_offset + mmSDMA0_RLC0_RB_WPTR_HI,173 m->sdmax_rlcx_rb_rptr_hi);174 }175 WREG32(sdma_rlc_reg_offset + mmSDMA0_RLC0_MINOR_PTR_UPDATE, 0);176 177 WREG32(sdma_rlc_reg_offset + mmSDMA0_RLC0_RB_BASE, m->sdmax_rlcx_rb_base);178 WREG32(sdma_rlc_reg_offset + mmSDMA0_RLC0_RB_BASE_HI,179 m->sdmax_rlcx_rb_base_hi);180 WREG32(sdma_rlc_reg_offset + mmSDMA0_RLC0_RB_RPTR_ADDR_LO,181 m->sdmax_rlcx_rb_rptr_addr_lo);182 WREG32(sdma_rlc_reg_offset + mmSDMA0_RLC0_RB_RPTR_ADDR_HI,183 m->sdmax_rlcx_rb_rptr_addr_hi);184 185 data = REG_SET_FIELD(m->sdmax_rlcx_rb_cntl, SDMA0_RLC0_RB_CNTL,186 RB_ENABLE, 1);187 WREG32(sdma_rlc_reg_offset + mmSDMA0_RLC0_RB_CNTL, data);188 189 return 0;190}191 192int kgd_arcturus_hqd_sdma_dump(struct amdgpu_device *adev,193 uint32_t engine_id, uint32_t queue_id,194 uint32_t (**dump)[2], uint32_t *n_regs)195{196 uint32_t sdma_rlc_reg_offset = get_sdma_rlc_reg_offset(adev,197 engine_id, queue_id);198 uint32_t i = 0, reg;199#undef HQD_N_REGS200#define HQD_N_REGS (19+6+7+10)201 202 *dump = kmalloc_array(HQD_N_REGS, sizeof(**dump), GFP_KERNEL);203 if (*dump == NULL)204 return -ENOMEM;205 206 for (reg = mmSDMA0_RLC0_RB_CNTL; reg <= mmSDMA0_RLC0_DOORBELL; reg++)207 DUMP_REG(sdma_rlc_reg_offset + reg);208 for (reg = mmSDMA0_RLC0_STATUS; reg <= mmSDMA0_RLC0_CSA_ADDR_HI; reg++)209 DUMP_REG(sdma_rlc_reg_offset + reg);210 for (reg = mmSDMA0_RLC0_IB_SUB_REMAIN;211 reg <= mmSDMA0_RLC0_MINOR_PTR_UPDATE; reg++)212 DUMP_REG(sdma_rlc_reg_offset + reg);213 for (reg = mmSDMA0_RLC0_MIDCMD_DATA0;214 reg <= mmSDMA0_RLC0_MIDCMD_CNTL; reg++)215 DUMP_REG(sdma_rlc_reg_offset + reg);216 217 WARN_ON_ONCE(i != HQD_N_REGS);218 *n_regs = i;219 220 return 0;221}222 223bool kgd_arcturus_hqd_sdma_is_occupied(struct amdgpu_device *adev,224 void *mqd)225{226 struct v9_sdma_mqd *m;227 uint32_t sdma_rlc_reg_offset;228 uint32_t sdma_rlc_rb_cntl;229 230 m = get_sdma_mqd(mqd);231 sdma_rlc_reg_offset = get_sdma_rlc_reg_offset(adev, m->sdma_engine_id,232 m->sdma_queue_id);233 234 sdma_rlc_rb_cntl = RREG32(sdma_rlc_reg_offset + mmSDMA0_RLC0_RB_CNTL);235 236 if (sdma_rlc_rb_cntl & SDMA0_RLC0_RB_CNTL__RB_ENABLE_MASK)237 return true;238 239 return false;240}241 242int kgd_arcturus_hqd_sdma_destroy(struct amdgpu_device *adev, void *mqd,243 unsigned int utimeout)244{245 struct v9_sdma_mqd *m;246 uint32_t sdma_rlc_reg_offset;247 uint32_t temp;248 unsigned long end_jiffies = (utimeout * HZ / 1000) + jiffies;249 250 m = get_sdma_mqd(mqd);251 sdma_rlc_reg_offset = get_sdma_rlc_reg_offset(adev, m->sdma_engine_id,252 m->sdma_queue_id);253 254 temp = RREG32(sdma_rlc_reg_offset + mmSDMA0_RLC0_RB_CNTL);255 temp = temp & ~SDMA0_RLC0_RB_CNTL__RB_ENABLE_MASK;256 WREG32(sdma_rlc_reg_offset + mmSDMA0_RLC0_RB_CNTL, temp);257 258 while (true) {259 temp = RREG32(sdma_rlc_reg_offset + mmSDMA0_RLC0_CONTEXT_STATUS);260 if (temp & SDMA0_RLC0_CONTEXT_STATUS__IDLE_MASK)261 break;262 if (time_after(jiffies, end_jiffies)) {263 pr_err("SDMA RLC not idle in %s\n", __func__);264 return -ETIME;265 }266 usleep_range(500, 1000);267 }268 269 WREG32(sdma_rlc_reg_offset + mmSDMA0_RLC0_DOORBELL, 0);270 WREG32(sdma_rlc_reg_offset + mmSDMA0_RLC0_RB_CNTL,271 RREG32(sdma_rlc_reg_offset + mmSDMA0_RLC0_RB_CNTL) |272 SDMA0_RLC0_RB_CNTL__RB_ENABLE_MASK);273 274 m->sdmax_rlcx_rb_rptr = RREG32(sdma_rlc_reg_offset + mmSDMA0_RLC0_RB_RPTR);275 m->sdmax_rlcx_rb_rptr_hi =276 RREG32(sdma_rlc_reg_offset + mmSDMA0_RLC0_RB_RPTR_HI);277 278 return 0;279}280 281/*282 * Helper used to suspend/resume gfx pipe for image post process work to set283 * barrier behaviour.284 */285static int suspend_resume_compute_scheduler(struct amdgpu_device *adev, bool suspend)286{287 int i, r = 0;288 289 for (i = 0; i < adev->gfx.num_compute_rings; i++) {290 struct amdgpu_ring *ring = &adev->gfx.compute_ring[i];291 292 if (!amdgpu_ring_sched_ready(ring))293 continue;294 295 /* stop secheduler and drain ring. */296 if (suspend) {297 drm_sched_stop(&ring->sched, NULL);298 r = amdgpu_fence_wait_empty(ring);299 if (r)300 goto out;301 } else {302 drm_sched_start(&ring->sched);303 }304 }305 306out:307 /* return on resume or failure to drain rings. */308 if (!suspend || r)309 return r;310 311 return amdgpu_device_ip_wait_for_idle(adev, AMD_IP_BLOCK_TYPE_GFX);312}313 314static void set_barrier_auto_waitcnt(struct amdgpu_device *adev, bool enable_waitcnt)315{316 uint32_t data;317 318 WRITE_ONCE(adev->barrier_has_auto_waitcnt, enable_waitcnt);319 320 if (!down_read_trylock(&adev->reset_domain->sem))321 return;322 323 amdgpu_amdkfd_suspend(adev, false);324 325 if (suspend_resume_compute_scheduler(adev, true))326 goto out;327 328 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmSQ_CONFIG));329 data = REG_SET_FIELD(data, SQ_CONFIG, DISABLE_BARRIER_WAITCNT,330 !enable_waitcnt);331 WREG32(SOC15_REG_OFFSET(GC, 0, mmSQ_CONFIG), data);332 333out:334 suspend_resume_compute_scheduler(adev, false);335 336 amdgpu_amdkfd_resume(adev, false);337 338 up_read(&adev->reset_domain->sem);339}340 341/*342 * restore_dbg_registers is ignored here but is a general interface requirement343 * for devices that support GFXOFF and where the RLC save/restore list344 * does not support hw registers for debugging i.e. the driver has to manually345 * initialize the debug mode registers after it has disabled GFX off during the346 * debug session.347 */348static uint32_t kgd_arcturus_enable_debug_trap(struct amdgpu_device *adev,349 bool restore_dbg_registers,350 uint32_t vmid)351{352 mutex_lock(&adev->grbm_idx_mutex);353 354 kgd_gfx_v9_set_wave_launch_stall(adev, vmid, true);355 356 set_barrier_auto_waitcnt(adev, true);357 358 WREG32(SOC15_REG_OFFSET(GC, 0, mmSPI_GDBG_TRAP_MASK), 0);359 360 kgd_gfx_v9_set_wave_launch_stall(adev, vmid, false);361 362 mutex_unlock(&adev->grbm_idx_mutex);363 364 return 0;365}366 367/*368 * keep_trap_enabled is ignored here but is a general interface requirement369 * for devices that support multi-process debugging where the performance370 * overhead from trap temporary setup needs to be bypassed when the debug371 * session has ended.372 */373static uint32_t kgd_arcturus_disable_debug_trap(struct amdgpu_device *adev,374 bool keep_trap_enabled,375 uint32_t vmid)376{377 378 mutex_lock(&adev->grbm_idx_mutex);379 380 kgd_gfx_v9_set_wave_launch_stall(adev, vmid, true);381 382 set_barrier_auto_waitcnt(adev, false);383 384 WREG32(SOC15_REG_OFFSET(GC, 0, mmSPI_GDBG_TRAP_MASK), 0);385 386 kgd_gfx_v9_set_wave_launch_stall(adev, vmid, false);387 388 mutex_unlock(&adev->grbm_idx_mutex);389 390 return 0;391}392const struct kfd2kgd_calls arcturus_kfd2kgd = {393 .program_sh_mem_settings = kgd_gfx_v9_program_sh_mem_settings,394 .set_pasid_vmid_mapping = kgd_gfx_v9_set_pasid_vmid_mapping,395 .init_interrupts = kgd_gfx_v9_init_interrupts,396 .hqd_load = kgd_gfx_v9_hqd_load,397 .hiq_mqd_load = kgd_gfx_v9_hiq_mqd_load,398 .hqd_sdma_load = kgd_arcturus_hqd_sdma_load,399 .hqd_dump = kgd_gfx_v9_hqd_dump,400 .hqd_sdma_dump = kgd_arcturus_hqd_sdma_dump,401 .hqd_is_occupied = kgd_gfx_v9_hqd_is_occupied,402 .hqd_sdma_is_occupied = kgd_arcturus_hqd_sdma_is_occupied,403 .hqd_destroy = kgd_gfx_v9_hqd_destroy,404 .hqd_sdma_destroy = kgd_arcturus_hqd_sdma_destroy,405 .wave_control_execute = kgd_gfx_v9_wave_control_execute,406 .get_atc_vmid_pasid_mapping_info =407 kgd_gfx_v9_get_atc_vmid_pasid_mapping_info,408 .set_vm_context_page_table_base =409 kgd_gfx_v9_set_vm_context_page_table_base,410 .enable_debug_trap = kgd_arcturus_enable_debug_trap,411 .disable_debug_trap = kgd_arcturus_disable_debug_trap,412 .validate_trap_override_request = kgd_gfx_v9_validate_trap_override_request,413 .set_wave_launch_trap_override = kgd_gfx_v9_set_wave_launch_trap_override,414 .set_wave_launch_mode = kgd_gfx_v9_set_wave_launch_mode,415 .set_address_watch = kgd_gfx_v9_set_address_watch,416 .clear_address_watch = kgd_gfx_v9_clear_address_watch,417 .get_iq_wait_times = kgd_gfx_v9_get_iq_wait_times,418 .build_grace_period_packet_info = kgd_gfx_v9_build_grace_period_packet_info,419 .get_cu_occupancy = kgd_gfx_v9_get_cu_occupancy,420 .program_trap_handler_settings = kgd_gfx_v9_program_trap_handler_settings,421 .hqd_get_pq_addr = kgd_gfx_v9_hqd_get_pq_addr,422 .hqd_reset = kgd_gfx_v9_hqd_reset423};424