1680 lines · c
1/*2 * Copyright 2019 Advanced Micro Devices, Inc.3 *4 * Permission is hereby granted, free of charge, to any person obtaining a5 * copy of this software and associated documentation files (the "Software"),6 * to deal in the Software without restriction, including without limitation7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,8 * and/or sell copies of the Software, and to permit persons to whom the9 * Software is furnished to do so, subject to the following conditions:10 *11 * The above copyright notice and this permission notice shall be included in12 * all copies or substantial portions of the Software.13 *14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR20 * OTHER DEALINGS IN THE SOFTWARE.21 *22 */23 24#include <linux/firmware.h>25#include <linux/module.h>26#include "amdgpu.h"27#include "soc15_common.h"28#include "soc21.h"29#include "gfx_v11_0.h"30#include "gc/gc_11_0_0_offset.h"31#include "gc/gc_11_0_0_sh_mask.h"32#include "gc/gc_11_0_0_default.h"33#include "v11_structs.h"34#include "mes_v11_api_def.h"35 36MODULE_FIRMWARE("amdgpu/gc_11_0_0_mes.bin");37MODULE_FIRMWARE("amdgpu/gc_11_0_0_mes_2.bin");38MODULE_FIRMWARE("amdgpu/gc_11_0_0_mes1.bin");39MODULE_FIRMWARE("amdgpu/gc_11_0_1_mes.bin");40MODULE_FIRMWARE("amdgpu/gc_11_0_1_mes_2.bin");41MODULE_FIRMWARE("amdgpu/gc_11_0_1_mes1.bin");42MODULE_FIRMWARE("amdgpu/gc_11_0_2_mes.bin");43MODULE_FIRMWARE("amdgpu/gc_11_0_2_mes_2.bin");44MODULE_FIRMWARE("amdgpu/gc_11_0_2_mes1.bin");45MODULE_FIRMWARE("amdgpu/gc_11_0_3_mes.bin");46MODULE_FIRMWARE("amdgpu/gc_11_0_3_mes_2.bin");47MODULE_FIRMWARE("amdgpu/gc_11_0_3_mes1.bin");48MODULE_FIRMWARE("amdgpu/gc_11_0_4_mes.bin");49MODULE_FIRMWARE("amdgpu/gc_11_0_4_mes_2.bin");50MODULE_FIRMWARE("amdgpu/gc_11_0_4_mes1.bin");51MODULE_FIRMWARE("amdgpu/gc_11_5_0_mes_2.bin");52MODULE_FIRMWARE("amdgpu/gc_11_5_0_mes1.bin");53MODULE_FIRMWARE("amdgpu/gc_11_5_1_mes_2.bin");54MODULE_FIRMWARE("amdgpu/gc_11_5_1_mes1.bin");55MODULE_FIRMWARE("amdgpu/gc_11_5_2_mes_2.bin");56MODULE_FIRMWARE("amdgpu/gc_11_5_2_mes1.bin");57 58static int mes_v11_0_hw_init(void *handle);59static int mes_v11_0_hw_fini(void *handle);60static int mes_v11_0_kiq_hw_init(struct amdgpu_device *adev);61static int mes_v11_0_kiq_hw_fini(struct amdgpu_device *adev);62 63#define MES_EOP_SIZE 204864#define GFX_MES_DRAM_SIZE 0x8000065 66static void mes_v11_0_ring_set_wptr(struct amdgpu_ring *ring)67{68 struct amdgpu_device *adev = ring->adev;69 70 if (ring->use_doorbell) {71 atomic64_set((atomic64_t *)ring->wptr_cpu_addr,72 ring->wptr);73 WDOORBELL64(ring->doorbell_index, ring->wptr);74 } else {75 BUG();76 }77}78 79static u64 mes_v11_0_ring_get_rptr(struct amdgpu_ring *ring)80{81 return *ring->rptr_cpu_addr;82}83 84static u64 mes_v11_0_ring_get_wptr(struct amdgpu_ring *ring)85{86 u64 wptr;87 88 if (ring->use_doorbell)89 wptr = atomic64_read((atomic64_t *)ring->wptr_cpu_addr);90 else91 BUG();92 return wptr;93}94 95static const struct amdgpu_ring_funcs mes_v11_0_ring_funcs = {96 .type = AMDGPU_RING_TYPE_MES,97 .align_mask = 1,98 .nop = 0,99 .support_64bit_ptrs = true,100 .get_rptr = mes_v11_0_ring_get_rptr,101 .get_wptr = mes_v11_0_ring_get_wptr,102 .set_wptr = mes_v11_0_ring_set_wptr,103 .insert_nop = amdgpu_ring_insert_nop,104};105 106static const char *mes_v11_0_opcodes[] = {107 "SET_HW_RSRC",108 "SET_SCHEDULING_CONFIG",109 "ADD_QUEUE",110 "REMOVE_QUEUE",111 "PERFORM_YIELD",112 "SET_GANG_PRIORITY_LEVEL",113 "SUSPEND",114 "RESUME",115 "RESET",116 "SET_LOG_BUFFER",117 "CHANGE_GANG_PRORITY",118 "QUERY_SCHEDULER_STATUS",119 "PROGRAM_GDS",120 "SET_DEBUG_VMID",121 "MISC",122 "UPDATE_ROOT_PAGE_TABLE",123 "AMD_LOG",124 "unused",125 "unused",126 "SET_HW_RSRC_1",127};128 129static const char *mes_v11_0_misc_opcodes[] = {130 "WRITE_REG",131 "INV_GART",132 "QUERY_STATUS",133 "READ_REG",134 "WAIT_REG_MEM",135 "SET_SHADER_DEBUGGER",136};137 138static const char *mes_v11_0_get_op_string(union MESAPI__MISC *x_pkt)139{140 const char *op_str = NULL;141 142 if (x_pkt->header.opcode < ARRAY_SIZE(mes_v11_0_opcodes))143 op_str = mes_v11_0_opcodes[x_pkt->header.opcode];144 145 return op_str;146}147 148static const char *mes_v11_0_get_misc_op_string(union MESAPI__MISC *x_pkt)149{150 const char *op_str = NULL;151 152 if ((x_pkt->header.opcode == MES_SCH_API_MISC) &&153 (x_pkt->opcode < ARRAY_SIZE(mes_v11_0_misc_opcodes)))154 op_str = mes_v11_0_misc_opcodes[x_pkt->opcode];155 156 return op_str;157}158 159static int mes_v11_0_submit_pkt_and_poll_completion(struct amdgpu_mes *mes,160 void *pkt, int size,161 int api_status_off)162{163 union MESAPI__QUERY_MES_STATUS mes_status_pkt;164 signed long timeout = 2100000; /* 2100 ms */165 struct amdgpu_device *adev = mes->adev;166 struct amdgpu_ring *ring = &mes->ring[0];167 struct MES_API_STATUS *api_status;168 union MESAPI__MISC *x_pkt = pkt;169 const char *op_str, *misc_op_str;170 unsigned long flags;171 u64 status_gpu_addr;172 u32 seq, status_offset;173 u64 *status_ptr;174 signed long r;175 int ret;176 177 if (x_pkt->header.opcode >= MES_SCH_API_MAX)178 return -EINVAL;179 180 if (amdgpu_emu_mode) {181 timeout *= 100;182 } else if (amdgpu_sriov_vf(adev)) {183 /* Worst case in sriov where all other 15 VF timeout, each VF needs about 600ms */184 timeout = 15 * 600 * 1000;185 }186 187 ret = amdgpu_device_wb_get(adev, &status_offset);188 if (ret)189 return ret;190 191 status_gpu_addr = adev->wb.gpu_addr + (status_offset * 4);192 status_ptr = (u64 *)&adev->wb.wb[status_offset];193 *status_ptr = 0;194 195 spin_lock_irqsave(&mes->ring_lock[0], flags);196 r = amdgpu_ring_alloc(ring, (size + sizeof(mes_status_pkt)) / 4);197 if (r)198 goto error_unlock_free;199 200 seq = ++ring->fence_drv.sync_seq;201 r = amdgpu_fence_wait_polling(ring,202 seq - ring->fence_drv.num_fences_mask,203 timeout);204 if (r < 1)205 goto error_undo;206 207 api_status = (struct MES_API_STATUS *)((char *)pkt + api_status_off);208 api_status->api_completion_fence_addr = status_gpu_addr;209 api_status->api_completion_fence_value = 1;210 211 amdgpu_ring_write_multiple(ring, pkt, size / 4);212 213 memset(&mes_status_pkt, 0, sizeof(mes_status_pkt));214 mes_status_pkt.header.type = MES_API_TYPE_SCHEDULER;215 mes_status_pkt.header.opcode = MES_SCH_API_QUERY_SCHEDULER_STATUS;216 mes_status_pkt.header.dwsize = API_FRAME_SIZE_IN_DWORDS;217 mes_status_pkt.api_status.api_completion_fence_addr =218 ring->fence_drv.gpu_addr;219 mes_status_pkt.api_status.api_completion_fence_value = seq;220 221 amdgpu_ring_write_multiple(ring, &mes_status_pkt,222 sizeof(mes_status_pkt) / 4);223 224 amdgpu_ring_commit(ring);225 spin_unlock_irqrestore(&mes->ring_lock[0], flags);226 227 op_str = mes_v11_0_get_op_string(x_pkt);228 misc_op_str = mes_v11_0_get_misc_op_string(x_pkt);229 230 if (misc_op_str)231 dev_dbg(adev->dev, "MES msg=%s (%s) was emitted\n", op_str,232 misc_op_str);233 else if (op_str)234 dev_dbg(adev->dev, "MES msg=%s was emitted\n", op_str);235 else236 dev_dbg(adev->dev, "MES msg=%d was emitted\n",237 x_pkt->header.opcode);238 239 r = amdgpu_fence_wait_polling(ring, seq, timeout);240 if (r < 1 || !*status_ptr) {241 242 if (misc_op_str)243 dev_err(adev->dev, "MES failed to respond to msg=%s (%s)\n",244 op_str, misc_op_str);245 else if (op_str)246 dev_err(adev->dev, "MES failed to respond to msg=%s\n",247 op_str);248 else249 dev_err(adev->dev, "MES failed to respond to msg=%d\n",250 x_pkt->header.opcode);251 252 while (halt_if_hws_hang)253 schedule();254 255 r = -ETIMEDOUT;256 goto error_wb_free;257 }258 259 amdgpu_device_wb_free(adev, status_offset);260 return 0;261 262error_undo:263 dev_err(adev->dev, "MES ring buffer is full.\n");264 amdgpu_ring_undo(ring);265 266error_unlock_free:267 spin_unlock_irqrestore(&mes->ring_lock[0], flags);268 269error_wb_free:270 amdgpu_device_wb_free(adev, status_offset);271 return r;272}273 274static int convert_to_mes_queue_type(int queue_type)275{276 if (queue_type == AMDGPU_RING_TYPE_GFX)277 return MES_QUEUE_TYPE_GFX;278 else if (queue_type == AMDGPU_RING_TYPE_COMPUTE)279 return MES_QUEUE_TYPE_COMPUTE;280 else if (queue_type == AMDGPU_RING_TYPE_SDMA)281 return MES_QUEUE_TYPE_SDMA;282 else283 BUG();284 return -1;285}286 287static int mes_v11_0_add_hw_queue(struct amdgpu_mes *mes,288 struct mes_add_queue_input *input)289{290 struct amdgpu_device *adev = mes->adev;291 union MESAPI__ADD_QUEUE mes_add_queue_pkt;292 struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_GFXHUB(0)];293 uint32_t vm_cntx_cntl = hub->vm_cntx_cntl;294 295 memset(&mes_add_queue_pkt, 0, sizeof(mes_add_queue_pkt));296 297 mes_add_queue_pkt.header.type = MES_API_TYPE_SCHEDULER;298 mes_add_queue_pkt.header.opcode = MES_SCH_API_ADD_QUEUE;299 mes_add_queue_pkt.header.dwsize = API_FRAME_SIZE_IN_DWORDS;300 301 mes_add_queue_pkt.process_id = input->process_id;302 mes_add_queue_pkt.page_table_base_addr = input->page_table_base_addr;303 mes_add_queue_pkt.process_va_start = input->process_va_start;304 mes_add_queue_pkt.process_va_end = input->process_va_end;305 mes_add_queue_pkt.process_quantum = input->process_quantum;306 mes_add_queue_pkt.process_context_addr = input->process_context_addr;307 mes_add_queue_pkt.gang_quantum = input->gang_quantum;308 mes_add_queue_pkt.gang_context_addr = input->gang_context_addr;309 mes_add_queue_pkt.inprocess_gang_priority =310 input->inprocess_gang_priority;311 mes_add_queue_pkt.gang_global_priority_level =312 input->gang_global_priority_level;313 mes_add_queue_pkt.doorbell_offset = input->doorbell_offset;314 mes_add_queue_pkt.mqd_addr = input->mqd_addr;315 316 if (((adev->mes.sched_version & AMDGPU_MES_API_VERSION_MASK) >>317 AMDGPU_MES_API_VERSION_SHIFT) >= 2)318 mes_add_queue_pkt.wptr_addr = input->wptr_mc_addr;319 else320 mes_add_queue_pkt.wptr_addr = input->wptr_addr;321 322 mes_add_queue_pkt.queue_type =323 convert_to_mes_queue_type(input->queue_type);324 mes_add_queue_pkt.paging = input->paging;325 mes_add_queue_pkt.vm_context_cntl = vm_cntx_cntl;326 mes_add_queue_pkt.gws_base = input->gws_base;327 mes_add_queue_pkt.gws_size = input->gws_size;328 mes_add_queue_pkt.trap_handler_addr = input->tba_addr;329 mes_add_queue_pkt.tma_addr = input->tma_addr;330 mes_add_queue_pkt.trap_en = input->trap_en;331 mes_add_queue_pkt.skip_process_ctx_clear = input->skip_process_ctx_clear;332 mes_add_queue_pkt.is_kfd_process = input->is_kfd_process;333 334 /* For KFD, gds_size is re-used for queue size (needed in MES for AQL queues) */335 mes_add_queue_pkt.is_aql_queue = input->is_aql_queue;336 mes_add_queue_pkt.gds_size = input->queue_size;337 338 mes_add_queue_pkt.exclusively_scheduled = input->exclusively_scheduled;339 340 return mes_v11_0_submit_pkt_and_poll_completion(mes,341 &mes_add_queue_pkt, sizeof(mes_add_queue_pkt),342 offsetof(union MESAPI__ADD_QUEUE, api_status));343}344 345static int mes_v11_0_remove_hw_queue(struct amdgpu_mes *mes,346 struct mes_remove_queue_input *input)347{348 union MESAPI__REMOVE_QUEUE mes_remove_queue_pkt;349 350 memset(&mes_remove_queue_pkt, 0, sizeof(mes_remove_queue_pkt));351 352 mes_remove_queue_pkt.header.type = MES_API_TYPE_SCHEDULER;353 mes_remove_queue_pkt.header.opcode = MES_SCH_API_REMOVE_QUEUE;354 mes_remove_queue_pkt.header.dwsize = API_FRAME_SIZE_IN_DWORDS;355 356 mes_remove_queue_pkt.doorbell_offset = input->doorbell_offset;357 mes_remove_queue_pkt.gang_context_addr = input->gang_context_addr;358 359 return mes_v11_0_submit_pkt_and_poll_completion(mes,360 &mes_remove_queue_pkt, sizeof(mes_remove_queue_pkt),361 offsetof(union MESAPI__REMOVE_QUEUE, api_status));362}363 364static int mes_v11_0_reset_queue_mmio(struct amdgpu_mes *mes, uint32_t queue_type,365 uint32_t me_id, uint32_t pipe_id,366 uint32_t queue_id, uint32_t vmid)367{368 struct amdgpu_device *adev = mes->adev;369 uint32_t value;370 int i, r = 0;371 372 amdgpu_gfx_rlc_enter_safe_mode(adev, 0);373 374 if (queue_type == AMDGPU_RING_TYPE_GFX) {375 dev_info(adev->dev, "reset gfx queue (%d:%d:%d: vmid:%d)\n",376 me_id, pipe_id, queue_id, vmid);377 378 mutex_lock(&adev->gfx.reset_sem_mutex);379 gfx_v11_0_request_gfx_index_mutex(adev, true);380 /* all se allow writes */381 WREG32_SOC15(GC, 0, regGRBM_GFX_INDEX,382 (uint32_t)(0x1 << GRBM_GFX_INDEX__SE_BROADCAST_WRITES__SHIFT));383 value = REG_SET_FIELD(0, CP_VMID_RESET, RESET_REQUEST, 1 << vmid);384 if (pipe_id == 0)385 value = REG_SET_FIELD(value, CP_VMID_RESET, PIPE0_QUEUES, 1 << queue_id);386 else387 value = REG_SET_FIELD(value, CP_VMID_RESET, PIPE1_QUEUES, 1 << queue_id);388 WREG32_SOC15(GC, 0, regCP_VMID_RESET, value);389 gfx_v11_0_request_gfx_index_mutex(adev, false);390 mutex_unlock(&adev->gfx.reset_sem_mutex);391 392 mutex_lock(&adev->srbm_mutex);393 soc21_grbm_select(adev, me_id, pipe_id, queue_id, 0);394 /* wait till dequeue take effects */395 for (i = 0; i < adev->usec_timeout; i++) {396 if (!(RREG32_SOC15(GC, 0, regCP_GFX_HQD_ACTIVE) & 1))397 break;398 udelay(1);399 }400 if (i >= adev->usec_timeout) {401 dev_err(adev->dev, "failed to wait on gfx hqd deactivate\n");402 r = -ETIMEDOUT;403 }404 405 soc21_grbm_select(adev, 0, 0, 0, 0);406 mutex_unlock(&adev->srbm_mutex);407 } else if (queue_type == AMDGPU_RING_TYPE_COMPUTE) {408 dev_info(adev->dev, "reset compute queue (%d:%d:%d)\n",409 me_id, pipe_id, queue_id);410 mutex_lock(&adev->srbm_mutex);411 soc21_grbm_select(adev, me_id, pipe_id, queue_id, 0);412 WREG32_SOC15(GC, 0, regCP_HQD_DEQUEUE_REQUEST, 0x2);413 WREG32_SOC15(GC, 0, regSPI_COMPUTE_QUEUE_RESET, 0x1);414 415 /* wait till dequeue take effects */416 for (i = 0; i < adev->usec_timeout; i++) {417 if (!(RREG32_SOC15(GC, 0, regCP_HQD_ACTIVE) & 1))418 break;419 udelay(1);420 }421 if (i >= adev->usec_timeout) {422 dev_err(adev->dev, "failed to wait on hqd deactivate\n");423 r = -ETIMEDOUT;424 }425 soc21_grbm_select(adev, 0, 0, 0, 0);426 mutex_unlock(&adev->srbm_mutex);427 }428 429 amdgpu_gfx_rlc_exit_safe_mode(adev, 0);430 return r;431}432 433static int mes_v11_0_reset_hw_queue(struct amdgpu_mes *mes,434 struct mes_reset_queue_input *input)435{436 if (input->use_mmio)437 return mes_v11_0_reset_queue_mmio(mes, input->queue_type,438 input->me_id, input->pipe_id,439 input->queue_id, input->vmid);440 441 union MESAPI__RESET mes_reset_queue_pkt;442 443 memset(&mes_reset_queue_pkt, 0, sizeof(mes_reset_queue_pkt));444 445 mes_reset_queue_pkt.header.type = MES_API_TYPE_SCHEDULER;446 mes_reset_queue_pkt.header.opcode = MES_SCH_API_RESET;447 mes_reset_queue_pkt.header.dwsize = API_FRAME_SIZE_IN_DWORDS;448 449 mes_reset_queue_pkt.doorbell_offset = input->doorbell_offset;450 mes_reset_queue_pkt.gang_context_addr = input->gang_context_addr;451 /*mes_reset_queue_pkt.reset_queue_only = 1;*/452 453 return mes_v11_0_submit_pkt_and_poll_completion(mes,454 &mes_reset_queue_pkt, sizeof(mes_reset_queue_pkt),455 offsetof(union MESAPI__REMOVE_QUEUE, api_status));456}457 458static int mes_v11_0_map_legacy_queue(struct amdgpu_mes *mes,459 struct mes_map_legacy_queue_input *input)460{461 union MESAPI__ADD_QUEUE mes_add_queue_pkt;462 463 memset(&mes_add_queue_pkt, 0, sizeof(mes_add_queue_pkt));464 465 mes_add_queue_pkt.header.type = MES_API_TYPE_SCHEDULER;466 mes_add_queue_pkt.header.opcode = MES_SCH_API_ADD_QUEUE;467 mes_add_queue_pkt.header.dwsize = API_FRAME_SIZE_IN_DWORDS;468 469 mes_add_queue_pkt.pipe_id = input->pipe_id;470 mes_add_queue_pkt.queue_id = input->queue_id;471 mes_add_queue_pkt.doorbell_offset = input->doorbell_offset;472 mes_add_queue_pkt.mqd_addr = input->mqd_addr;473 mes_add_queue_pkt.wptr_addr = input->wptr_addr;474 mes_add_queue_pkt.queue_type =475 convert_to_mes_queue_type(input->queue_type);476 mes_add_queue_pkt.map_legacy_kq = 1;477 478 return mes_v11_0_submit_pkt_and_poll_completion(mes,479 &mes_add_queue_pkt, sizeof(mes_add_queue_pkt),480 offsetof(union MESAPI__ADD_QUEUE, api_status));481}482 483static int mes_v11_0_unmap_legacy_queue(struct amdgpu_mes *mes,484 struct mes_unmap_legacy_queue_input *input)485{486 union MESAPI__REMOVE_QUEUE mes_remove_queue_pkt;487 488 memset(&mes_remove_queue_pkt, 0, sizeof(mes_remove_queue_pkt));489 490 mes_remove_queue_pkt.header.type = MES_API_TYPE_SCHEDULER;491 mes_remove_queue_pkt.header.opcode = MES_SCH_API_REMOVE_QUEUE;492 mes_remove_queue_pkt.header.dwsize = API_FRAME_SIZE_IN_DWORDS;493 494 mes_remove_queue_pkt.doorbell_offset = input->doorbell_offset;495 mes_remove_queue_pkt.gang_context_addr = 0;496 497 mes_remove_queue_pkt.pipe_id = input->pipe_id;498 mes_remove_queue_pkt.queue_id = input->queue_id;499 500 if (input->action == PREEMPT_QUEUES_NO_UNMAP) {501 mes_remove_queue_pkt.preempt_legacy_gfx_queue = 1;502 mes_remove_queue_pkt.tf_addr = input->trail_fence_addr;503 mes_remove_queue_pkt.tf_data =504 lower_32_bits(input->trail_fence_data);505 } else {506 mes_remove_queue_pkt.unmap_legacy_queue = 1;507 mes_remove_queue_pkt.queue_type =508 convert_to_mes_queue_type(input->queue_type);509 }510 511 return mes_v11_0_submit_pkt_and_poll_completion(mes,512 &mes_remove_queue_pkt, sizeof(mes_remove_queue_pkt),513 offsetof(union MESAPI__REMOVE_QUEUE, api_status));514}515 516static int mes_v11_0_suspend_gang(struct amdgpu_mes *mes,517 struct mes_suspend_gang_input *input)518{519 union MESAPI__SUSPEND mes_suspend_gang_pkt;520 521 memset(&mes_suspend_gang_pkt, 0, sizeof(mes_suspend_gang_pkt));522 523 mes_suspend_gang_pkt.header.type = MES_API_TYPE_SCHEDULER;524 mes_suspend_gang_pkt.header.opcode = MES_SCH_API_SUSPEND;525 mes_suspend_gang_pkt.header.dwsize = API_FRAME_SIZE_IN_DWORDS;526 527 mes_suspend_gang_pkt.suspend_all_gangs = input->suspend_all_gangs;528 mes_suspend_gang_pkt.gang_context_addr = input->gang_context_addr;529 mes_suspend_gang_pkt.suspend_fence_addr = input->suspend_fence_addr;530 mes_suspend_gang_pkt.suspend_fence_value = input->suspend_fence_value;531 532 return mes_v11_0_submit_pkt_and_poll_completion(mes,533 &mes_suspend_gang_pkt, sizeof(mes_suspend_gang_pkt),534 offsetof(union MESAPI__SUSPEND, api_status));535}536 537static int mes_v11_0_resume_gang(struct amdgpu_mes *mes,538 struct mes_resume_gang_input *input)539{540 union MESAPI__RESUME mes_resume_gang_pkt;541 542 memset(&mes_resume_gang_pkt, 0, sizeof(mes_resume_gang_pkt));543 544 mes_resume_gang_pkt.header.type = MES_API_TYPE_SCHEDULER;545 mes_resume_gang_pkt.header.opcode = MES_SCH_API_RESUME;546 mes_resume_gang_pkt.header.dwsize = API_FRAME_SIZE_IN_DWORDS;547 548 mes_resume_gang_pkt.resume_all_gangs = input->resume_all_gangs;549 mes_resume_gang_pkt.gang_context_addr = input->gang_context_addr;550 551 return mes_v11_0_submit_pkt_and_poll_completion(mes,552 &mes_resume_gang_pkt, sizeof(mes_resume_gang_pkt),553 offsetof(union MESAPI__RESUME, api_status));554}555 556static int mes_v11_0_query_sched_status(struct amdgpu_mes *mes)557{558 union MESAPI__QUERY_MES_STATUS mes_status_pkt;559 560 memset(&mes_status_pkt, 0, sizeof(mes_status_pkt));561 562 mes_status_pkt.header.type = MES_API_TYPE_SCHEDULER;563 mes_status_pkt.header.opcode = MES_SCH_API_QUERY_SCHEDULER_STATUS;564 mes_status_pkt.header.dwsize = API_FRAME_SIZE_IN_DWORDS;565 566 return mes_v11_0_submit_pkt_and_poll_completion(mes,567 &mes_status_pkt, sizeof(mes_status_pkt),568 offsetof(union MESAPI__QUERY_MES_STATUS, api_status));569}570 571static int mes_v11_0_misc_op(struct amdgpu_mes *mes,572 struct mes_misc_op_input *input)573{574 union MESAPI__MISC misc_pkt;575 576 memset(&misc_pkt, 0, sizeof(misc_pkt));577 578 misc_pkt.header.type = MES_API_TYPE_SCHEDULER;579 misc_pkt.header.opcode = MES_SCH_API_MISC;580 misc_pkt.header.dwsize = API_FRAME_SIZE_IN_DWORDS;581 582 switch (input->op) {583 case MES_MISC_OP_READ_REG:584 misc_pkt.opcode = MESAPI_MISC__READ_REG;585 misc_pkt.read_reg.reg_offset = input->read_reg.reg_offset;586 misc_pkt.read_reg.buffer_addr = input->read_reg.buffer_addr;587 break;588 case MES_MISC_OP_WRITE_REG:589 misc_pkt.opcode = MESAPI_MISC__WRITE_REG;590 misc_pkt.write_reg.reg_offset = input->write_reg.reg_offset;591 misc_pkt.write_reg.reg_value = input->write_reg.reg_value;592 break;593 case MES_MISC_OP_WRM_REG_WAIT:594 misc_pkt.opcode = MESAPI_MISC__WAIT_REG_MEM;595 misc_pkt.wait_reg_mem.op = WRM_OPERATION__WAIT_REG_MEM;596 misc_pkt.wait_reg_mem.reference = input->wrm_reg.ref;597 misc_pkt.wait_reg_mem.mask = input->wrm_reg.mask;598 misc_pkt.wait_reg_mem.reg_offset1 = input->wrm_reg.reg0;599 misc_pkt.wait_reg_mem.reg_offset2 = 0;600 break;601 case MES_MISC_OP_WRM_REG_WR_WAIT:602 misc_pkt.opcode = MESAPI_MISC__WAIT_REG_MEM;603 misc_pkt.wait_reg_mem.op = WRM_OPERATION__WR_WAIT_WR_REG;604 misc_pkt.wait_reg_mem.reference = input->wrm_reg.ref;605 misc_pkt.wait_reg_mem.mask = input->wrm_reg.mask;606 misc_pkt.wait_reg_mem.reg_offset1 = input->wrm_reg.reg0;607 misc_pkt.wait_reg_mem.reg_offset2 = input->wrm_reg.reg1;608 break;609 case MES_MISC_OP_SET_SHADER_DEBUGGER:610 misc_pkt.opcode = MESAPI_MISC__SET_SHADER_DEBUGGER;611 misc_pkt.set_shader_debugger.process_context_addr =612 input->set_shader_debugger.process_context_addr;613 misc_pkt.set_shader_debugger.flags.u32all =614 input->set_shader_debugger.flags.u32all;615 misc_pkt.set_shader_debugger.spi_gdbg_per_vmid_cntl =616 input->set_shader_debugger.spi_gdbg_per_vmid_cntl;617 memcpy(misc_pkt.set_shader_debugger.tcp_watch_cntl,618 input->set_shader_debugger.tcp_watch_cntl,619 sizeof(misc_pkt.set_shader_debugger.tcp_watch_cntl));620 misc_pkt.set_shader_debugger.trap_en = input->set_shader_debugger.trap_en;621 break;622 default:623 DRM_ERROR("unsupported misc op (%d) \n", input->op);624 return -EINVAL;625 }626 627 return mes_v11_0_submit_pkt_and_poll_completion(mes,628 &misc_pkt, sizeof(misc_pkt),629 offsetof(union MESAPI__MISC, api_status));630}631 632static int mes_v11_0_set_hw_resources(struct amdgpu_mes *mes)633{634 int i;635 struct amdgpu_device *adev = mes->adev;636 union MESAPI_SET_HW_RESOURCES mes_set_hw_res_pkt;637 638 memset(&mes_set_hw_res_pkt, 0, sizeof(mes_set_hw_res_pkt));639 640 mes_set_hw_res_pkt.header.type = MES_API_TYPE_SCHEDULER;641 mes_set_hw_res_pkt.header.opcode = MES_SCH_API_SET_HW_RSRC;642 mes_set_hw_res_pkt.header.dwsize = API_FRAME_SIZE_IN_DWORDS;643 644 mes_set_hw_res_pkt.vmid_mask_mmhub = mes->vmid_mask_mmhub;645 mes_set_hw_res_pkt.vmid_mask_gfxhub = mes->vmid_mask_gfxhub;646 mes_set_hw_res_pkt.gds_size = adev->gds.gds_size;647 mes_set_hw_res_pkt.paging_vmid = 0;648 mes_set_hw_res_pkt.g_sch_ctx_gpu_mc_ptr = mes->sch_ctx_gpu_addr[0];649 mes_set_hw_res_pkt.query_status_fence_gpu_mc_ptr =650 mes->query_status_fence_gpu_addr[0];651 652 for (i = 0; i < MAX_COMPUTE_PIPES; i++)653 mes_set_hw_res_pkt.compute_hqd_mask[i] =654 mes->compute_hqd_mask[i];655 656 for (i = 0; i < MAX_GFX_PIPES; i++)657 mes_set_hw_res_pkt.gfx_hqd_mask[i] = mes->gfx_hqd_mask[i];658 659 for (i = 0; i < MAX_SDMA_PIPES; i++)660 mes_set_hw_res_pkt.sdma_hqd_mask[i] = mes->sdma_hqd_mask[i];661 662 for (i = 0; i < AMD_PRIORITY_NUM_LEVELS; i++)663 mes_set_hw_res_pkt.aggregated_doorbells[i] =664 mes->aggregated_doorbells[i];665 666 for (i = 0; i < 5; i++) {667 mes_set_hw_res_pkt.gc_base[i] = adev->reg_offset[GC_HWIP][0][i];668 mes_set_hw_res_pkt.mmhub_base[i] =669 adev->reg_offset[MMHUB_HWIP][0][i];670 mes_set_hw_res_pkt.osssys_base[i] =671 adev->reg_offset[OSSSYS_HWIP][0][i];672 }673 674 mes_set_hw_res_pkt.disable_reset = 1;675 mes_set_hw_res_pkt.disable_mes_log = 1;676 mes_set_hw_res_pkt.use_different_vmid_compute = 1;677 mes_set_hw_res_pkt.enable_reg_active_poll = 1;678 mes_set_hw_res_pkt.enable_level_process_quantum_check = 1;679 mes_set_hw_res_pkt.oversubscription_timer = 50;680 if (amdgpu_mes_log_enable) {681 mes_set_hw_res_pkt.enable_mes_event_int_logging = 1;682 mes_set_hw_res_pkt.event_intr_history_gpu_mc_ptr =683 mes->event_log_gpu_addr;684 }685 686 return mes_v11_0_submit_pkt_and_poll_completion(mes,687 &mes_set_hw_res_pkt, sizeof(mes_set_hw_res_pkt),688 offsetof(union MESAPI_SET_HW_RESOURCES, api_status));689}690 691static int mes_v11_0_set_hw_resources_1(struct amdgpu_mes *mes)692{693 int size = 128 * PAGE_SIZE;694 int ret = 0;695 struct amdgpu_device *adev = mes->adev;696 union MESAPI_SET_HW_RESOURCES_1 mes_set_hw_res_pkt;697 memset(&mes_set_hw_res_pkt, 0, sizeof(mes_set_hw_res_pkt));698 699 mes_set_hw_res_pkt.header.type = MES_API_TYPE_SCHEDULER;700 mes_set_hw_res_pkt.header.opcode = MES_SCH_API_SET_HW_RSRC_1;701 mes_set_hw_res_pkt.header.dwsize = API_FRAME_SIZE_IN_DWORDS;702 mes_set_hw_res_pkt.enable_mes_info_ctx = 1;703 704 ret = amdgpu_bo_create_kernel(adev, size, PAGE_SIZE,705 AMDGPU_GEM_DOMAIN_VRAM,706 &mes->resource_1,707 &mes->resource_1_gpu_addr,708 &mes->resource_1_addr);709 if (ret) {710 dev_err(adev->dev, "(%d) failed to create mes resource_1 bo\n", ret);711 return ret;712 }713 714 mes_set_hw_res_pkt.mes_info_ctx_mc_addr = mes->resource_1_gpu_addr;715 mes_set_hw_res_pkt.mes_info_ctx_size = mes->resource_1->tbo.base.size;716 return mes_v11_0_submit_pkt_and_poll_completion(mes,717 &mes_set_hw_res_pkt, sizeof(mes_set_hw_res_pkt),718 offsetof(union MESAPI_SET_HW_RESOURCES_1, api_status));719}720 721static int mes_v11_0_reset_legacy_queue(struct amdgpu_mes *mes,722 struct mes_reset_legacy_queue_input *input)723{724 union MESAPI__RESET mes_reset_queue_pkt;725 726 if (input->use_mmio)727 return mes_v11_0_reset_queue_mmio(mes, input->queue_type,728 input->me_id, input->pipe_id,729 input->queue_id, input->vmid);730 731 memset(&mes_reset_queue_pkt, 0, sizeof(mes_reset_queue_pkt));732 733 mes_reset_queue_pkt.header.type = MES_API_TYPE_SCHEDULER;734 mes_reset_queue_pkt.header.opcode = MES_SCH_API_RESET;735 mes_reset_queue_pkt.header.dwsize = API_FRAME_SIZE_IN_DWORDS;736 737 mes_reset_queue_pkt.queue_type =738 convert_to_mes_queue_type(input->queue_type);739 740 if (mes_reset_queue_pkt.queue_type == MES_QUEUE_TYPE_GFX) {741 mes_reset_queue_pkt.reset_legacy_gfx = 1;742 mes_reset_queue_pkt.pipe_id_lp = input->pipe_id;743 mes_reset_queue_pkt.queue_id_lp = input->queue_id;744 mes_reset_queue_pkt.mqd_mc_addr_lp = input->mqd_addr;745 mes_reset_queue_pkt.doorbell_offset_lp = input->doorbell_offset;746 mes_reset_queue_pkt.wptr_addr_lp = input->wptr_addr;747 mes_reset_queue_pkt.vmid_id_lp = input->vmid;748 } else {749 mes_reset_queue_pkt.reset_queue_only = 1;750 mes_reset_queue_pkt.doorbell_offset = input->doorbell_offset;751 }752 753 return mes_v11_0_submit_pkt_and_poll_completion(mes,754 &mes_reset_queue_pkt, sizeof(mes_reset_queue_pkt),755 offsetof(union MESAPI__RESET, api_status));756}757 758static const struct amdgpu_mes_funcs mes_v11_0_funcs = {759 .add_hw_queue = mes_v11_0_add_hw_queue,760 .remove_hw_queue = mes_v11_0_remove_hw_queue,761 .map_legacy_queue = mes_v11_0_map_legacy_queue,762 .unmap_legacy_queue = mes_v11_0_unmap_legacy_queue,763 .suspend_gang = mes_v11_0_suspend_gang,764 .resume_gang = mes_v11_0_resume_gang,765 .misc_op = mes_v11_0_misc_op,766 .reset_legacy_queue = mes_v11_0_reset_legacy_queue,767 .reset_hw_queue = mes_v11_0_reset_hw_queue,768};769 770static int mes_v11_0_allocate_ucode_buffer(struct amdgpu_device *adev,771 enum admgpu_mes_pipe pipe)772{773 int r;774 const struct mes_firmware_header_v1_0 *mes_hdr;775 const __le32 *fw_data;776 unsigned fw_size;777 778 mes_hdr = (const struct mes_firmware_header_v1_0 *)779 adev->mes.fw[pipe]->data;780 781 fw_data = (const __le32 *)(adev->mes.fw[pipe]->data +782 le32_to_cpu(mes_hdr->mes_ucode_offset_bytes));783 fw_size = le32_to_cpu(mes_hdr->mes_ucode_size_bytes);784 785 r = amdgpu_bo_create_reserved(adev, fw_size,786 PAGE_SIZE,787 AMDGPU_GEM_DOMAIN_VRAM |788 AMDGPU_GEM_DOMAIN_GTT,789 &adev->mes.ucode_fw_obj[pipe],790 &adev->mes.ucode_fw_gpu_addr[pipe],791 (void **)&adev->mes.ucode_fw_ptr[pipe]);792 if (r) {793 dev_err(adev->dev, "(%d) failed to create mes fw bo\n", r);794 return r;795 }796 797 memcpy(adev->mes.ucode_fw_ptr[pipe], fw_data, fw_size);798 799 amdgpu_bo_kunmap(adev->mes.ucode_fw_obj[pipe]);800 amdgpu_bo_unreserve(adev->mes.ucode_fw_obj[pipe]);801 802 return 0;803}804 805static int mes_v11_0_allocate_ucode_data_buffer(struct amdgpu_device *adev,806 enum admgpu_mes_pipe pipe)807{808 int r;809 const struct mes_firmware_header_v1_0 *mes_hdr;810 const __le32 *fw_data;811 unsigned fw_size;812 813 mes_hdr = (const struct mes_firmware_header_v1_0 *)814 adev->mes.fw[pipe]->data;815 816 fw_data = (const __le32 *)(adev->mes.fw[pipe]->data +817 le32_to_cpu(mes_hdr->mes_ucode_data_offset_bytes));818 fw_size = le32_to_cpu(mes_hdr->mes_ucode_data_size_bytes);819 820 if (fw_size > GFX_MES_DRAM_SIZE) {821 dev_err(adev->dev, "PIPE%d ucode data fw size (%d) is greater than dram size (%d)\n",822 pipe, fw_size, GFX_MES_DRAM_SIZE);823 return -EINVAL;824 }825 826 r = amdgpu_bo_create_reserved(adev, GFX_MES_DRAM_SIZE,827 64 * 1024,828 AMDGPU_GEM_DOMAIN_VRAM |829 AMDGPU_GEM_DOMAIN_GTT,830 &adev->mes.data_fw_obj[pipe],831 &adev->mes.data_fw_gpu_addr[pipe],832 (void **)&adev->mes.data_fw_ptr[pipe]);833 if (r) {834 dev_err(adev->dev, "(%d) failed to create mes data fw bo\n", r);835 return r;836 }837 838 memcpy(adev->mes.data_fw_ptr[pipe], fw_data, fw_size);839 840 amdgpu_bo_kunmap(adev->mes.data_fw_obj[pipe]);841 amdgpu_bo_unreserve(adev->mes.data_fw_obj[pipe]);842 843 return 0;844}845 846static void mes_v11_0_free_ucode_buffers(struct amdgpu_device *adev,847 enum admgpu_mes_pipe pipe)848{849 amdgpu_bo_free_kernel(&adev->mes.data_fw_obj[pipe],850 &adev->mes.data_fw_gpu_addr[pipe],851 (void **)&adev->mes.data_fw_ptr[pipe]);852 853 amdgpu_bo_free_kernel(&adev->mes.ucode_fw_obj[pipe],854 &adev->mes.ucode_fw_gpu_addr[pipe],855 (void **)&adev->mes.ucode_fw_ptr[pipe]);856}857 858static void mes_v11_0_get_fw_version(struct amdgpu_device *adev)859{860 int pipe;861 862 /* get MES scheduler/KIQ versions */863 mutex_lock(&adev->srbm_mutex);864 865 for (pipe = 0; pipe < AMDGPU_MAX_MES_PIPES; pipe++) {866 soc21_grbm_select(adev, 3, pipe, 0, 0);867 868 if (pipe == AMDGPU_MES_SCHED_PIPE)869 adev->mes.sched_version =870 RREG32_SOC15(GC, 0, regCP_MES_GP3_LO);871 else if (pipe == AMDGPU_MES_KIQ_PIPE && adev->enable_mes_kiq)872 adev->mes.kiq_version =873 RREG32_SOC15(GC, 0, regCP_MES_GP3_LO);874 }875 876 soc21_grbm_select(adev, 0, 0, 0, 0);877 mutex_unlock(&adev->srbm_mutex);878}879 880static void mes_v11_0_enable(struct amdgpu_device *adev, bool enable)881{882 uint64_t ucode_addr;883 uint32_t pipe, data = 0;884 885 if (enable) {886 data = RREG32_SOC15(GC, 0, regCP_MES_CNTL);887 data = REG_SET_FIELD(data, CP_MES_CNTL, MES_PIPE0_RESET, 1);888 data = REG_SET_FIELD(data, CP_MES_CNTL,889 MES_PIPE1_RESET, adev->enable_mes_kiq ? 1 : 0);890 WREG32_SOC15(GC, 0, regCP_MES_CNTL, data);891 892 mutex_lock(&adev->srbm_mutex);893 for (pipe = 0; pipe < AMDGPU_MAX_MES_PIPES; pipe++) {894 if (!adev->enable_mes_kiq &&895 pipe == AMDGPU_MES_KIQ_PIPE)896 continue;897 898 soc21_grbm_select(adev, 3, pipe, 0, 0);899 900 ucode_addr = adev->mes.uc_start_addr[pipe] >> 2;901 WREG32_SOC15(GC, 0, regCP_MES_PRGRM_CNTR_START,902 lower_32_bits(ucode_addr));903 WREG32_SOC15(GC, 0, regCP_MES_PRGRM_CNTR_START_HI,904 upper_32_bits(ucode_addr));905 }906 soc21_grbm_select(adev, 0, 0, 0, 0);907 mutex_unlock(&adev->srbm_mutex);908 909 /* unhalt MES and activate pipe0 */910 data = REG_SET_FIELD(0, CP_MES_CNTL, MES_PIPE0_ACTIVE, 1);911 data = REG_SET_FIELD(data, CP_MES_CNTL, MES_PIPE1_ACTIVE,912 adev->enable_mes_kiq ? 1 : 0);913 WREG32_SOC15(GC, 0, regCP_MES_CNTL, data);914 915 if (amdgpu_emu_mode)916 msleep(100);917 else918 udelay(500);919 } else {920 data = RREG32_SOC15(GC, 0, regCP_MES_CNTL);921 data = REG_SET_FIELD(data, CP_MES_CNTL, MES_PIPE0_ACTIVE, 0);922 data = REG_SET_FIELD(data, CP_MES_CNTL, MES_PIPE1_ACTIVE, 0);923 data = REG_SET_FIELD(data, CP_MES_CNTL,924 MES_INVALIDATE_ICACHE, 1);925 data = REG_SET_FIELD(data, CP_MES_CNTL, MES_PIPE0_RESET, 1);926 data = REG_SET_FIELD(data, CP_MES_CNTL, MES_PIPE1_RESET,927 adev->enable_mes_kiq ? 1 : 0);928 data = REG_SET_FIELD(data, CP_MES_CNTL, MES_HALT, 1);929 WREG32_SOC15(GC, 0, regCP_MES_CNTL, data);930 }931}932 933/* This function is for backdoor MES firmware */934static int mes_v11_0_load_microcode(struct amdgpu_device *adev,935 enum admgpu_mes_pipe pipe, bool prime_icache)936{937 int r;938 uint32_t data;939 uint64_t ucode_addr;940 941 mes_v11_0_enable(adev, false);942 943 if (!adev->mes.fw[pipe])944 return -EINVAL;945 946 r = mes_v11_0_allocate_ucode_buffer(adev, pipe);947 if (r)948 return r;949 950 r = mes_v11_0_allocate_ucode_data_buffer(adev, pipe);951 if (r) {952 mes_v11_0_free_ucode_buffers(adev, pipe);953 return r;954 }955 956 mutex_lock(&adev->srbm_mutex);957 /* me=3, pipe=0, queue=0 */958 soc21_grbm_select(adev, 3, pipe, 0, 0);959 960 WREG32_SOC15(GC, 0, regCP_MES_IC_BASE_CNTL, 0);961 962 /* set ucode start address */963 ucode_addr = adev->mes.uc_start_addr[pipe] >> 2;964 WREG32_SOC15(GC, 0, regCP_MES_PRGRM_CNTR_START,965 lower_32_bits(ucode_addr));966 WREG32_SOC15(GC, 0, regCP_MES_PRGRM_CNTR_START_HI,967 upper_32_bits(ucode_addr));968 969 /* set ucode fimrware address */970 WREG32_SOC15(GC, 0, regCP_MES_IC_BASE_LO,971 lower_32_bits(adev->mes.ucode_fw_gpu_addr[pipe]));972 WREG32_SOC15(GC, 0, regCP_MES_IC_BASE_HI,973 upper_32_bits(adev->mes.ucode_fw_gpu_addr[pipe]));974 975 /* set ucode instruction cache boundary to 2M-1 */976 WREG32_SOC15(GC, 0, regCP_MES_MIBOUND_LO, 0x1FFFFF);977 978 /* set ucode data firmware address */979 WREG32_SOC15(GC, 0, regCP_MES_MDBASE_LO,980 lower_32_bits(adev->mes.data_fw_gpu_addr[pipe]));981 WREG32_SOC15(GC, 0, regCP_MES_MDBASE_HI,982 upper_32_bits(adev->mes.data_fw_gpu_addr[pipe]));983 984 /* Set 0x7FFFF (512K-1) to CP_MES_MDBOUND_LO */985 WREG32_SOC15(GC, 0, regCP_MES_MDBOUND_LO, 0x7FFFF);986 987 if (prime_icache) {988 /* invalidate ICACHE */989 data = RREG32_SOC15(GC, 0, regCP_MES_IC_OP_CNTL);990 data = REG_SET_FIELD(data, CP_MES_IC_OP_CNTL, PRIME_ICACHE, 0);991 data = REG_SET_FIELD(data, CP_MES_IC_OP_CNTL, INVALIDATE_CACHE, 1);992 WREG32_SOC15(GC, 0, regCP_MES_IC_OP_CNTL, data);993 994 /* prime the ICACHE. */995 data = RREG32_SOC15(GC, 0, regCP_MES_IC_OP_CNTL);996 data = REG_SET_FIELD(data, CP_MES_IC_OP_CNTL, PRIME_ICACHE, 1);997 WREG32_SOC15(GC, 0, regCP_MES_IC_OP_CNTL, data);998 }999 1000 soc21_grbm_select(adev, 0, 0, 0, 0);1001 mutex_unlock(&adev->srbm_mutex);1002 1003 return 0;1004}1005 1006static int mes_v11_0_allocate_eop_buf(struct amdgpu_device *adev,1007 enum admgpu_mes_pipe pipe)1008{1009 int r;1010 u32 *eop;1011 1012 r = amdgpu_bo_create_reserved(adev, MES_EOP_SIZE, PAGE_SIZE,1013 AMDGPU_GEM_DOMAIN_GTT,1014 &adev->mes.eop_gpu_obj[pipe],1015 &adev->mes.eop_gpu_addr[pipe],1016 (void **)&eop);1017 if (r) {1018 dev_warn(adev->dev, "(%d) create EOP bo failed\n", r);1019 return r;1020 }1021 1022 memset(eop, 0,1023 adev->mes.eop_gpu_obj[pipe]->tbo.base.size);1024 1025 amdgpu_bo_kunmap(adev->mes.eop_gpu_obj[pipe]);1026 amdgpu_bo_unreserve(adev->mes.eop_gpu_obj[pipe]);1027 1028 return 0;1029}1030 1031static int mes_v11_0_mqd_init(struct amdgpu_ring *ring)1032{1033 struct v11_compute_mqd *mqd = ring->mqd_ptr;1034 uint64_t hqd_gpu_addr, wb_gpu_addr, eop_base_addr;1035 uint32_t tmp;1036 1037 memset(mqd, 0, sizeof(*mqd));1038 1039 mqd->header = 0xC0310800;1040 mqd->compute_pipelinestat_enable = 0x00000001;1041 mqd->compute_static_thread_mgmt_se0 = 0xffffffff;1042 mqd->compute_static_thread_mgmt_se1 = 0xffffffff;1043 mqd->compute_static_thread_mgmt_se2 = 0xffffffff;1044 mqd->compute_static_thread_mgmt_se3 = 0xffffffff;1045 mqd->compute_misc_reserved = 0x00000007;1046 1047 eop_base_addr = ring->eop_gpu_addr >> 8;1048 1049 /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */1050 tmp = regCP_HQD_EOP_CONTROL_DEFAULT;1051 tmp = REG_SET_FIELD(tmp, CP_HQD_EOP_CONTROL, EOP_SIZE,1052 (order_base_2(MES_EOP_SIZE / 4) - 1));1053 1054 mqd->cp_hqd_eop_base_addr_lo = lower_32_bits(eop_base_addr);1055 mqd->cp_hqd_eop_base_addr_hi = upper_32_bits(eop_base_addr);1056 mqd->cp_hqd_eop_control = tmp;1057 1058 /* disable the queue if it's active */1059 ring->wptr = 0;1060 mqd->cp_hqd_pq_rptr = 0;1061 mqd->cp_hqd_pq_wptr_lo = 0;1062 mqd->cp_hqd_pq_wptr_hi = 0;1063 1064 /* set the pointer to the MQD */1065 mqd->cp_mqd_base_addr_lo = ring->mqd_gpu_addr & 0xfffffffc;1066 mqd->cp_mqd_base_addr_hi = upper_32_bits(ring->mqd_gpu_addr);1067 1068 /* set MQD vmid to 0 */1069 tmp = regCP_MQD_CONTROL_DEFAULT;1070 tmp = REG_SET_FIELD(tmp, CP_MQD_CONTROL, VMID, 0);1071 mqd->cp_mqd_control = tmp;1072 1073 /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */1074 hqd_gpu_addr = ring->gpu_addr >> 8;1075 mqd->cp_hqd_pq_base_lo = lower_32_bits(hqd_gpu_addr);1076 mqd->cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr);1077 1078 /* set the wb address whether it's enabled or not */1079 wb_gpu_addr = ring->rptr_gpu_addr;1080 mqd->cp_hqd_pq_rptr_report_addr_lo = wb_gpu_addr & 0xfffffffc;1081 mqd->cp_hqd_pq_rptr_report_addr_hi =1082 upper_32_bits(wb_gpu_addr) & 0xffff;1083 1084 /* only used if CP_PQ_WPTR_POLL_CNTL.CP_PQ_WPTR_POLL_CNTL__EN_MASK=1 */1085 wb_gpu_addr = ring->wptr_gpu_addr;1086 mqd->cp_hqd_pq_wptr_poll_addr_lo = wb_gpu_addr & 0xfffffff8;1087 mqd->cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff;1088 1089 /* set up the HQD, this is similar to CP_RB0_CNTL */1090 tmp = regCP_HQD_PQ_CONTROL_DEFAULT;1091 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, QUEUE_SIZE,1092 (order_base_2(ring->ring_size / 4) - 1));1093 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, RPTR_BLOCK_SIZE,1094 ((order_base_2(AMDGPU_GPU_PAGE_SIZE / 4) - 1) << 8));1095 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, UNORD_DISPATCH, 1);1096 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, TUNNEL_DISPATCH, 0);1097 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, PRIV_STATE, 1);1098 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, KMD_QUEUE, 1);1099 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, NO_UPDATE_RPTR, 1);1100 mqd->cp_hqd_pq_control = tmp;1101 1102 /* enable doorbell */1103 tmp = 0;1104 if (ring->use_doorbell) {1105 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,1106 DOORBELL_OFFSET, ring->doorbell_index);1107 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,1108 DOORBELL_EN, 1);1109 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,1110 DOORBELL_SOURCE, 0);1111 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,1112 DOORBELL_HIT, 0);1113 } else1114 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,1115 DOORBELL_EN, 0);1116 mqd->cp_hqd_pq_doorbell_control = tmp;1117 1118 mqd->cp_hqd_vmid = 0;1119 /* activate the queue */1120 mqd->cp_hqd_active = 1;1121 1122 tmp = regCP_HQD_PERSISTENT_STATE_DEFAULT;1123 tmp = REG_SET_FIELD(tmp, CP_HQD_PERSISTENT_STATE,1124 PRELOAD_SIZE, 0x55);1125 mqd->cp_hqd_persistent_state = tmp;1126 1127 mqd->cp_hqd_ib_control = regCP_HQD_IB_CONTROL_DEFAULT;1128 mqd->cp_hqd_iq_timer = regCP_HQD_IQ_TIMER_DEFAULT;1129 mqd->cp_hqd_quantum = regCP_HQD_QUANTUM_DEFAULT;1130 1131 amdgpu_device_flush_hdp(ring->adev, NULL);1132 return 0;1133}1134 1135static void mes_v11_0_queue_init_register(struct amdgpu_ring *ring)1136{1137 struct v11_compute_mqd *mqd = ring->mqd_ptr;1138 struct amdgpu_device *adev = ring->adev;1139 uint32_t data = 0;1140 1141 mutex_lock(&adev->srbm_mutex);1142 soc21_grbm_select(adev, 3, ring->pipe, 0, 0);1143 1144 /* set CP_HQD_VMID.VMID = 0. */1145 data = RREG32_SOC15(GC, 0, regCP_HQD_VMID);1146 data = REG_SET_FIELD(data, CP_HQD_VMID, VMID, 0);1147 WREG32_SOC15(GC, 0, regCP_HQD_VMID, data);1148 1149 /* set CP_HQD_PQ_DOORBELL_CONTROL.DOORBELL_EN=0 */1150 data = RREG32_SOC15(GC, 0, regCP_HQD_PQ_DOORBELL_CONTROL);1151 data = REG_SET_FIELD(data, CP_HQD_PQ_DOORBELL_CONTROL,1152 DOORBELL_EN, 0);1153 WREG32_SOC15(GC, 0, regCP_HQD_PQ_DOORBELL_CONTROL, data);1154 1155 /* set CP_MQD_BASE_ADDR/HI with the MQD base address */1156 WREG32_SOC15(GC, 0, regCP_MQD_BASE_ADDR, mqd->cp_mqd_base_addr_lo);1157 WREG32_SOC15(GC, 0, regCP_MQD_BASE_ADDR_HI, mqd->cp_mqd_base_addr_hi);1158 1159 /* set CP_MQD_CONTROL.VMID=0 */1160 data = RREG32_SOC15(GC, 0, regCP_MQD_CONTROL);1161 data = REG_SET_FIELD(data, CP_MQD_CONTROL, VMID, 0);1162 WREG32_SOC15(GC, 0, regCP_MQD_CONTROL, 0);1163 1164 /* set CP_HQD_PQ_BASE/HI with the ring buffer base address */1165 WREG32_SOC15(GC, 0, regCP_HQD_PQ_BASE, mqd->cp_hqd_pq_base_lo);1166 WREG32_SOC15(GC, 0, regCP_HQD_PQ_BASE_HI, mqd->cp_hqd_pq_base_hi);1167 1168 /* set CP_HQD_PQ_RPTR_REPORT_ADDR/HI */1169 WREG32_SOC15(GC, 0, regCP_HQD_PQ_RPTR_REPORT_ADDR,1170 mqd->cp_hqd_pq_rptr_report_addr_lo);1171 WREG32_SOC15(GC, 0, regCP_HQD_PQ_RPTR_REPORT_ADDR_HI,1172 mqd->cp_hqd_pq_rptr_report_addr_hi);1173 1174 /* set CP_HQD_PQ_CONTROL */1175 WREG32_SOC15(GC, 0, regCP_HQD_PQ_CONTROL, mqd->cp_hqd_pq_control);1176 1177 /* set CP_HQD_PQ_WPTR_POLL_ADDR/HI */1178 WREG32_SOC15(GC, 0, regCP_HQD_PQ_WPTR_POLL_ADDR,1179 mqd->cp_hqd_pq_wptr_poll_addr_lo);1180 WREG32_SOC15(GC, 0, regCP_HQD_PQ_WPTR_POLL_ADDR_HI,1181 mqd->cp_hqd_pq_wptr_poll_addr_hi);1182 1183 /* set CP_HQD_PQ_DOORBELL_CONTROL */1184 WREG32_SOC15(GC, 0, regCP_HQD_PQ_DOORBELL_CONTROL,1185 mqd->cp_hqd_pq_doorbell_control);1186 1187 /* set CP_HQD_PERSISTENT_STATE.PRELOAD_SIZE=0x53 */1188 WREG32_SOC15(GC, 0, regCP_HQD_PERSISTENT_STATE, mqd->cp_hqd_persistent_state);1189 1190 /* set CP_HQD_ACTIVE.ACTIVE=1 */1191 WREG32_SOC15(GC, 0, regCP_HQD_ACTIVE, mqd->cp_hqd_active);1192 1193 soc21_grbm_select(adev, 0, 0, 0, 0);1194 mutex_unlock(&adev->srbm_mutex);1195}1196 1197static int mes_v11_0_kiq_enable_queue(struct amdgpu_device *adev)1198{1199 struct amdgpu_kiq *kiq = &adev->gfx.kiq[0];1200 struct amdgpu_ring *kiq_ring = &adev->gfx.kiq[0].ring;1201 int r;1202 1203 if (!kiq->pmf || !kiq->pmf->kiq_map_queues)1204 return -EINVAL;1205 1206 r = amdgpu_ring_alloc(kiq_ring, kiq->pmf->map_queues_size);1207 if (r) {1208 DRM_ERROR("Failed to lock KIQ (%d).\n", r);1209 return r;1210 }1211 1212 kiq->pmf->kiq_map_queues(kiq_ring, &adev->mes.ring[0]);1213 1214 return amdgpu_ring_test_helper(kiq_ring);1215}1216 1217static int mes_v11_0_queue_init(struct amdgpu_device *adev,1218 enum admgpu_mes_pipe pipe)1219{1220 struct amdgpu_ring *ring;1221 int r;1222 1223 if (pipe == AMDGPU_MES_KIQ_PIPE)1224 ring = &adev->gfx.kiq[0].ring;1225 else if (pipe == AMDGPU_MES_SCHED_PIPE)1226 ring = &adev->mes.ring[0];1227 else1228 BUG();1229 1230 if ((pipe == AMDGPU_MES_SCHED_PIPE) &&1231 (amdgpu_in_reset(adev) || adev->in_suspend)) {1232 *(ring->wptr_cpu_addr) = 0;1233 *(ring->rptr_cpu_addr) = 0;1234 amdgpu_ring_clear_ring(ring);1235 }1236 1237 r = mes_v11_0_mqd_init(ring);1238 if (r)1239 return r;1240 1241 if (pipe == AMDGPU_MES_SCHED_PIPE) {1242 r = mes_v11_0_kiq_enable_queue(adev);1243 if (r)1244 return r;1245 } else {1246 mes_v11_0_queue_init_register(ring);1247 }1248 1249 return 0;1250}1251 1252static int mes_v11_0_ring_init(struct amdgpu_device *adev)1253{1254 struct amdgpu_ring *ring;1255 1256 ring = &adev->mes.ring[0];1257 1258 ring->funcs = &mes_v11_0_ring_funcs;1259 1260 ring->me = 3;1261 ring->pipe = 0;1262 ring->queue = 0;1263 1264 ring->ring_obj = NULL;1265 ring->use_doorbell = true;1266 ring->doorbell_index = adev->doorbell_index.mes_ring0 << 1;1267 ring->eop_gpu_addr = adev->mes.eop_gpu_addr[AMDGPU_MES_SCHED_PIPE];1268 ring->no_scheduler = true;1269 sprintf(ring->name, "mes_%d.%d.%d", ring->me, ring->pipe, ring->queue);1270 1271 return amdgpu_ring_init(adev, ring, 1024, NULL, 0,1272 AMDGPU_RING_PRIO_DEFAULT, NULL);1273}1274 1275static int mes_v11_0_kiq_ring_init(struct amdgpu_device *adev)1276{1277 struct amdgpu_ring *ring;1278 1279 spin_lock_init(&adev->gfx.kiq[0].ring_lock);1280 1281 ring = &adev->gfx.kiq[0].ring;1282 1283 ring->me = 3;1284 ring->pipe = 1;1285 ring->queue = 0;1286 1287 ring->adev = NULL;1288 ring->ring_obj = NULL;1289 ring->use_doorbell = true;1290 ring->doorbell_index = adev->doorbell_index.mes_ring1 << 1;1291 ring->eop_gpu_addr = adev->mes.eop_gpu_addr[AMDGPU_MES_KIQ_PIPE];1292 ring->no_scheduler = true;1293 sprintf(ring->name, "mes_kiq_%d.%d.%d",1294 ring->me, ring->pipe, ring->queue);1295 1296 return amdgpu_ring_init(adev, ring, 1024, NULL, 0,1297 AMDGPU_RING_PRIO_DEFAULT, NULL);1298}1299 1300static int mes_v11_0_mqd_sw_init(struct amdgpu_device *adev,1301 enum admgpu_mes_pipe pipe)1302{1303 int r, mqd_size = sizeof(struct v11_compute_mqd);1304 struct amdgpu_ring *ring;1305 1306 if (pipe == AMDGPU_MES_KIQ_PIPE)1307 ring = &adev->gfx.kiq[0].ring;1308 else if (pipe == AMDGPU_MES_SCHED_PIPE)1309 ring = &adev->mes.ring[0];1310 else1311 BUG();1312 1313 if (ring->mqd_obj)1314 return 0;1315 1316 r = amdgpu_bo_create_kernel(adev, mqd_size, PAGE_SIZE,1317 AMDGPU_GEM_DOMAIN_VRAM |1318 AMDGPU_GEM_DOMAIN_GTT, &ring->mqd_obj,1319 &ring->mqd_gpu_addr, &ring->mqd_ptr);1320 if (r) {1321 dev_warn(adev->dev, "failed to create ring mqd bo (%d)", r);1322 return r;1323 }1324 1325 memset(ring->mqd_ptr, 0, mqd_size);1326 1327 /* prepare MQD backup */1328 adev->mes.mqd_backup[pipe] = kmalloc(mqd_size, GFP_KERNEL);1329 if (!adev->mes.mqd_backup[pipe]) {1330 dev_warn(adev->dev,1331 "no memory to create MQD backup for ring %s\n",1332 ring->name);1333 return -ENOMEM;1334 }1335 1336 return 0;1337}1338 1339static int mes_v11_0_sw_init(void *handle)1340{1341 struct amdgpu_device *adev = (struct amdgpu_device *)handle;1342 int pipe, r;1343 1344 adev->mes.funcs = &mes_v11_0_funcs;1345 adev->mes.kiq_hw_init = &mes_v11_0_kiq_hw_init;1346 adev->mes.kiq_hw_fini = &mes_v11_0_kiq_hw_fini;1347 1348 adev->mes.event_log_size = AMDGPU_MES_LOG_BUFFER_SIZE;1349 1350 r = amdgpu_mes_init(adev);1351 if (r)1352 return r;1353 1354 for (pipe = 0; pipe < AMDGPU_MAX_MES_PIPES; pipe++) {1355 if (!adev->enable_mes_kiq && pipe == AMDGPU_MES_KIQ_PIPE)1356 continue;1357 1358 r = mes_v11_0_allocate_eop_buf(adev, pipe);1359 if (r)1360 return r;1361 1362 r = mes_v11_0_mqd_sw_init(adev, pipe);1363 if (r)1364 return r;1365 }1366 1367 if (adev->enable_mes_kiq) {1368 r = mes_v11_0_kiq_ring_init(adev);1369 if (r)1370 return r;1371 }1372 1373 r = mes_v11_0_ring_init(adev);1374 if (r)1375 return r;1376 1377 return 0;1378}1379 1380static int mes_v11_0_sw_fini(void *handle)1381{1382 struct amdgpu_device *adev = (struct amdgpu_device *)handle;1383 int pipe;1384 1385 for (pipe = 0; pipe < AMDGPU_MAX_MES_PIPES; pipe++) {1386 kfree(adev->mes.mqd_backup[pipe]);1387 1388 amdgpu_bo_free_kernel(&adev->mes.eop_gpu_obj[pipe],1389 &adev->mes.eop_gpu_addr[pipe],1390 NULL);1391 amdgpu_ucode_release(&adev->mes.fw[pipe]);1392 }1393 1394 amdgpu_bo_free_kernel(&adev->gfx.kiq[0].ring.mqd_obj,1395 &adev->gfx.kiq[0].ring.mqd_gpu_addr,1396 &adev->gfx.kiq[0].ring.mqd_ptr);1397 1398 amdgpu_bo_free_kernel(&adev->mes.ring[0].mqd_obj,1399 &adev->mes.ring[0].mqd_gpu_addr,1400 &adev->mes.ring[0].mqd_ptr);1401 1402 amdgpu_ring_fini(&adev->gfx.kiq[0].ring);1403 amdgpu_ring_fini(&adev->mes.ring[0]);1404 1405 if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) {1406 mes_v11_0_free_ucode_buffers(adev, AMDGPU_MES_KIQ_PIPE);1407 mes_v11_0_free_ucode_buffers(adev, AMDGPU_MES_SCHED_PIPE);1408 }1409 1410 amdgpu_mes_fini(adev);1411 return 0;1412}1413 1414static void mes_v11_0_kiq_dequeue(struct amdgpu_ring *ring)1415{1416 uint32_t data;1417 int i;1418 struct amdgpu_device *adev = ring->adev;1419 1420 mutex_lock(&adev->srbm_mutex);1421 soc21_grbm_select(adev, 3, ring->pipe, 0, 0);1422 1423 /* disable the queue if it's active */1424 if (RREG32_SOC15(GC, 0, regCP_HQD_ACTIVE) & 1) {1425 WREG32_SOC15(GC, 0, regCP_HQD_DEQUEUE_REQUEST, 1);1426 for (i = 0; i < adev->usec_timeout; i++) {1427 if (!(RREG32_SOC15(GC, 0, regCP_HQD_ACTIVE) & 1))1428 break;1429 udelay(1);1430 }1431 }1432 data = RREG32_SOC15(GC, 0, regCP_HQD_PQ_DOORBELL_CONTROL);1433 data = REG_SET_FIELD(data, CP_HQD_PQ_DOORBELL_CONTROL,1434 DOORBELL_EN, 0);1435 data = REG_SET_FIELD(data, CP_HQD_PQ_DOORBELL_CONTROL,1436 DOORBELL_HIT, 1);1437 WREG32_SOC15(GC, 0, regCP_HQD_PQ_DOORBELL_CONTROL, data);1438 1439 WREG32_SOC15(GC, 0, regCP_HQD_PQ_DOORBELL_CONTROL, 0);1440 1441 WREG32_SOC15(GC, 0, regCP_HQD_PQ_WPTR_LO, 0);1442 WREG32_SOC15(GC, 0, regCP_HQD_PQ_WPTR_HI, 0);1443 WREG32_SOC15(GC, 0, regCP_HQD_PQ_RPTR, 0);1444 1445 soc21_grbm_select(adev, 0, 0, 0, 0);1446 mutex_unlock(&adev->srbm_mutex);1447}1448 1449static void mes_v11_0_kiq_setting(struct amdgpu_ring *ring)1450{1451 uint32_t tmp;1452 struct amdgpu_device *adev = ring->adev;1453 1454 /* tell RLC which is KIQ queue */1455 tmp = RREG32_SOC15(GC, 0, regRLC_CP_SCHEDULERS);1456 tmp &= 0xffffff00;1457 tmp |= (ring->me << 5) | (ring->pipe << 3) | (ring->queue);1458 WREG32_SOC15(GC, 0, regRLC_CP_SCHEDULERS, tmp);1459 tmp |= 0x80;1460 WREG32_SOC15(GC, 0, regRLC_CP_SCHEDULERS, tmp);1461}1462 1463static void mes_v11_0_kiq_clear(struct amdgpu_device *adev)1464{1465 uint32_t tmp;1466 1467 /* tell RLC which is KIQ dequeue */1468 tmp = RREG32_SOC15(GC, 0, regRLC_CP_SCHEDULERS);1469 tmp &= ~RLC_CP_SCHEDULERS__scheduler0_MASK;1470 WREG32_SOC15(GC, 0, regRLC_CP_SCHEDULERS, tmp);1471}1472 1473static int mes_v11_0_kiq_hw_init(struct amdgpu_device *adev)1474{1475 int r = 0;1476 1477 if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) {1478 1479 r = mes_v11_0_load_microcode(adev, AMDGPU_MES_SCHED_PIPE, false);1480 if (r) {1481 DRM_ERROR("failed to load MES fw, r=%d\n", r);1482 return r;1483 }1484 1485 r = mes_v11_0_load_microcode(adev, AMDGPU_MES_KIQ_PIPE, true);1486 if (r) {1487 DRM_ERROR("failed to load MES kiq fw, r=%d\n", r);1488 return r;1489 }1490 1491 }1492 1493 mes_v11_0_enable(adev, true);1494 1495 mes_v11_0_get_fw_version(adev);1496 1497 mes_v11_0_kiq_setting(&adev->gfx.kiq[0].ring);1498 1499 r = mes_v11_0_queue_init(adev, AMDGPU_MES_KIQ_PIPE);1500 if (r)1501 goto failure;1502 1503 if ((adev->mes.sched_version & AMDGPU_MES_VERSION_MASK) >= 0x47)1504 adev->mes.enable_legacy_queue_map = true;1505 else1506 adev->mes.enable_legacy_queue_map = false;1507 1508 if (adev->mes.enable_legacy_queue_map) {1509 r = mes_v11_0_hw_init(adev);1510 if (r)1511 goto failure;1512 }1513 1514 return r;1515 1516failure:1517 mes_v11_0_hw_fini(adev);1518 return r;1519}1520 1521static int mes_v11_0_kiq_hw_fini(struct amdgpu_device *adev)1522{1523 if (adev->mes.ring[0].sched.ready) {1524 mes_v11_0_kiq_dequeue(&adev->mes.ring[0]);1525 adev->mes.ring[0].sched.ready = false;1526 }1527 1528 if (amdgpu_sriov_vf(adev)) {1529 mes_v11_0_kiq_dequeue(&adev->gfx.kiq[0].ring);1530 mes_v11_0_kiq_clear(adev);1531 }1532 1533 mes_v11_0_enable(adev, false);1534 1535 return 0;1536}1537 1538static int mes_v11_0_hw_init(void *handle)1539{1540 int r;1541 struct amdgpu_device *adev = (struct amdgpu_device *)handle;1542 1543 if (adev->mes.ring[0].sched.ready)1544 goto out;1545 1546 if (!adev->enable_mes_kiq) {1547 if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) {1548 r = mes_v11_0_load_microcode(adev,1549 AMDGPU_MES_SCHED_PIPE, true);1550 if (r) {1551 DRM_ERROR("failed to MES fw, r=%d\n", r);1552 return r;1553 }1554 }1555 1556 mes_v11_0_enable(adev, true);1557 }1558 1559 r = mes_v11_0_queue_init(adev, AMDGPU_MES_SCHED_PIPE);1560 if (r)1561 goto failure;1562 1563 r = mes_v11_0_set_hw_resources(&adev->mes);1564 if (r)1565 goto failure;1566 1567 if (amdgpu_sriov_is_mes_info_enable(adev)) {1568 r = mes_v11_0_set_hw_resources_1(&adev->mes);1569 if (r) {1570 DRM_ERROR("failed mes_v11_0_set_hw_resources_1, r=%d\n", r);1571 goto failure;1572 }1573 }1574 1575 r = mes_v11_0_query_sched_status(&adev->mes);1576 if (r) {1577 DRM_ERROR("MES is busy\n");1578 goto failure;1579 }1580 1581out:1582 /*1583 * Disable KIQ ring usage from the driver once MES is enabled.1584 * MES uses KIQ ring exclusively so driver cannot access KIQ ring1585 * with MES enabled.1586 */1587 adev->gfx.kiq[0].ring.sched.ready = false;1588 adev->mes.ring[0].sched.ready = true;1589 1590 return 0;1591 1592failure:1593 mes_v11_0_hw_fini(adev);1594 return r;1595}1596 1597static int mes_v11_0_hw_fini(void *handle)1598{1599 struct amdgpu_device *adev = (struct amdgpu_device *)handle;1600 if (amdgpu_sriov_is_mes_info_enable(adev)) {1601 amdgpu_bo_free_kernel(&adev->mes.resource_1, &adev->mes.resource_1_gpu_addr,1602 &adev->mes.resource_1_addr);1603 }1604 return 0;1605}1606 1607static int mes_v11_0_suspend(void *handle)1608{1609 int r;1610 struct amdgpu_device *adev = (struct amdgpu_device *)handle;1611 1612 r = amdgpu_mes_suspend(adev);1613 if (r)1614 return r;1615 1616 return mes_v11_0_hw_fini(adev);1617}1618 1619static int mes_v11_0_resume(void *handle)1620{1621 int r;1622 struct amdgpu_device *adev = (struct amdgpu_device *)handle;1623 1624 r = mes_v11_0_hw_init(adev);1625 if (r)1626 return r;1627 1628 return amdgpu_mes_resume(adev);1629}1630 1631static int mes_v11_0_early_init(void *handle)1632{1633 struct amdgpu_device *adev = (struct amdgpu_device *)handle;1634 int pipe, r;1635 1636 for (pipe = 0; pipe < AMDGPU_MAX_MES_PIPES; pipe++) {1637 if (!adev->enable_mes_kiq && pipe == AMDGPU_MES_KIQ_PIPE)1638 continue;1639 r = amdgpu_mes_init_microcode(adev, pipe);1640 if (r)1641 return r;1642 }1643 1644 return 0;1645}1646 1647static int mes_v11_0_late_init(void *handle)1648{1649 struct amdgpu_device *adev = (struct amdgpu_device *)handle;1650 1651 /* it's only intended for use in mes_self_test case, not for s0ix and reset */1652 if (!amdgpu_in_reset(adev) && !adev->in_s0ix && !adev->in_suspend &&1653 (amdgpu_ip_version(adev, GC_HWIP, 0) != IP_VERSION(11, 0, 3)))1654 amdgpu_mes_self_test(adev);1655 1656 return 0;1657}1658 1659static const struct amd_ip_funcs mes_v11_0_ip_funcs = {1660 .name = "mes_v11_0",1661 .early_init = mes_v11_0_early_init,1662 .late_init = mes_v11_0_late_init,1663 .sw_init = mes_v11_0_sw_init,1664 .sw_fini = mes_v11_0_sw_fini,1665 .hw_init = mes_v11_0_hw_init,1666 .hw_fini = mes_v11_0_hw_fini,1667 .suspend = mes_v11_0_suspend,1668 .resume = mes_v11_0_resume,1669 .dump_ip_state = NULL,1670 .print_ip_state = NULL,1671};1672 1673const struct amdgpu_ip_block_version mes_v11_0_ip_block = {1674 .type = AMD_IP_BLOCK_TYPE_MES,1675 .major = 11,1676 .minor = 0,1677 .rev = 0,1678 .funcs = &mes_v11_0_ip_funcs,1679};1680