449 lines · c
1// SPDX-License-Identifier: GPL-2.0 OR MIT2/*3 * Copyright 2016-2022 Advanced Micro Devices, Inc.4 *5 * Permission is hereby granted, free of charge, to any person obtaining a6 * copy of this software and associated documentation files (the "Software"),7 * to deal in the Software without restriction, including without limitation8 * the rights to use, copy, modify, merge, publish, distribute, sublicense,9 * and/or sell copies of the Software, and to permit persons to whom the10 * Software is furnished to do so, subject to the following conditions:11 *12 * The above copyright notice and this permission notice shall be included in13 * all copies or substantial portions of the Software.14 *15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL18 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR19 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,20 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR21 * OTHER DEALINGS IN THE SOFTWARE.22 *23 */24 25#include "kfd_kernel_queue.h"26#include "kfd_device_queue_manager.h"27#include "kfd_pm4_headers_ai.h"28#include "kfd_pm4_headers_aldebaran.h"29#include "kfd_pm4_opcodes.h"30#include "gc/gc_10_1_0_sh_mask.h"31 32static int pm_map_process_v9(struct packet_manager *pm,33 uint32_t *buffer, struct qcm_process_device *qpd)34{35 struct pm4_mes_map_process *packet;36 uint64_t vm_page_table_base_addr = qpd->page_table_base;37 struct kfd_node *kfd = pm->dqm->dev;38 struct kfd_process_device *pdd =39 container_of(qpd, struct kfd_process_device, qpd);40 struct amdgpu_device *adev = kfd->adev;41 42 packet = (struct pm4_mes_map_process *)buffer;43 memset(buffer, 0, sizeof(struct pm4_mes_map_process));44 packet->header.u32All = pm_build_pm4_header(IT_MAP_PROCESS,45 sizeof(struct pm4_mes_map_process));46 if (adev->enforce_isolation[kfd->node_id])47 packet->bitfields2.exec_cleaner_shader = 1;48 packet->bitfields2.diq_enable = (qpd->is_debug) ? 1 : 0;49 packet->bitfields2.process_quantum = 10;50 packet->bitfields2.pasid = qpd->pqm->process->pasid;51 packet->bitfields14.gds_size = qpd->gds_size & 0x3F;52 packet->bitfields14.gds_size_hi = (qpd->gds_size >> 6) & 0xF;53 packet->bitfields14.num_gws = (qpd->mapped_gws_queue) ? qpd->num_gws : 0;54 packet->bitfields14.num_oac = qpd->num_oac;55 packet->bitfields14.sdma_enable = 1;56 packet->bitfields14.num_queues = (qpd->is_debug) ? 0 : qpd->queue_count;57 58 if (kfd->dqm->trap_debug_vmid && pdd->process->debug_trap_enabled &&59 pdd->process->runtime_info.runtime_state == DEBUG_RUNTIME_STATE_ENABLED) {60 packet->bitfields2.debug_vmid = kfd->dqm->trap_debug_vmid;61 packet->bitfields2.new_debug = 1;62 }63 64 packet->sh_mem_config = qpd->sh_mem_config;65 packet->sh_mem_bases = qpd->sh_mem_bases;66 if (qpd->tba_addr) {67 packet->sq_shader_tba_lo = lower_32_bits(qpd->tba_addr >> 8);68 /* On GFX9, unlike GFX10, bit TRAP_EN of SQ_SHADER_TBA_HI is69 * not defined, so setting it won't do any harm.70 */71 packet->sq_shader_tba_hi = upper_32_bits(qpd->tba_addr >> 8)72 | 1 << SQ_SHADER_TBA_HI__TRAP_EN__SHIFT;73 74 packet->sq_shader_tma_lo = lower_32_bits(qpd->tma_addr >> 8);75 packet->sq_shader_tma_hi = upper_32_bits(qpd->tma_addr >> 8);76 }77 78 packet->gds_addr_lo = lower_32_bits(qpd->gds_context_area);79 packet->gds_addr_hi = upper_32_bits(qpd->gds_context_area);80 81 packet->vm_context_page_table_base_addr_lo32 =82 lower_32_bits(vm_page_table_base_addr);83 packet->vm_context_page_table_base_addr_hi32 =84 upper_32_bits(vm_page_table_base_addr);85 86 return 0;87}88 89static int pm_map_process_aldebaran(struct packet_manager *pm,90 uint32_t *buffer, struct qcm_process_device *qpd)91{92 struct pm4_mes_map_process_aldebaran *packet;93 uint64_t vm_page_table_base_addr = qpd->page_table_base;94 struct kfd_dev *kfd = pm->dqm->dev->kfd;95 struct kfd_node *knode = pm->dqm->dev;96 struct kfd_process_device *pdd =97 container_of(qpd, struct kfd_process_device, qpd);98 int i;99 struct amdgpu_device *adev = kfd->adev;100 101 packet = (struct pm4_mes_map_process_aldebaran *)buffer;102 memset(buffer, 0, sizeof(struct pm4_mes_map_process_aldebaran));103 packet->header.u32All = pm_build_pm4_header(IT_MAP_PROCESS,104 sizeof(struct pm4_mes_map_process_aldebaran));105 if (adev->enforce_isolation[knode->node_id])106 packet->bitfields2.exec_cleaner_shader = 1;107 packet->bitfields2.diq_enable = (qpd->is_debug) ? 1 : 0;108 packet->bitfields2.process_quantum = 10;109 packet->bitfields2.pasid = qpd->pqm->process->pasid;110 packet->bitfields14.gds_size = qpd->gds_size & 0x3F;111 packet->bitfields14.gds_size_hi = (qpd->gds_size >> 6) & 0xF;112 packet->bitfields14.num_gws = (qpd->mapped_gws_queue) ? qpd->num_gws : 0;113 packet->bitfields14.num_oac = qpd->num_oac;114 packet->bitfields14.sdma_enable = 1;115 packet->bitfields14.num_queues = (qpd->is_debug) ? 0 : qpd->queue_count;116 packet->spi_gdbg_per_vmid_cntl = pdd->spi_dbg_override |117 pdd->spi_dbg_launch_mode;118 119 if (pdd->process->debug_trap_enabled) {120 for (i = 0; i < kfd->device_info.num_of_watch_points; i++)121 packet->tcp_watch_cntl[i] = pdd->watch_points[i];122 123 packet->bitfields2.single_memops =124 !!(pdd->process->dbg_flags & KFD_DBG_TRAP_FLAG_SINGLE_MEM_OP);125 }126 127 packet->sh_mem_config = qpd->sh_mem_config;128 packet->sh_mem_bases = qpd->sh_mem_bases;129 if (qpd->tba_addr) {130 packet->sq_shader_tba_lo = lower_32_bits(qpd->tba_addr >> 8);131 packet->sq_shader_tba_hi = upper_32_bits(qpd->tba_addr >> 8);132 packet->sq_shader_tma_lo = lower_32_bits(qpd->tma_addr >> 8);133 packet->sq_shader_tma_hi = upper_32_bits(qpd->tma_addr >> 8);134 }135 136 packet->gds_addr_lo = lower_32_bits(qpd->gds_context_area);137 packet->gds_addr_hi = upper_32_bits(qpd->gds_context_area);138 139 packet->vm_context_page_table_base_addr_lo32 =140 lower_32_bits(vm_page_table_base_addr);141 packet->vm_context_page_table_base_addr_hi32 =142 upper_32_bits(vm_page_table_base_addr);143 144 return 0;145}146 147static int pm_runlist_v9(struct packet_manager *pm, uint32_t *buffer,148 uint64_t ib, size_t ib_size_in_dwords, bool chain)149{150 struct pm4_mes_runlist *packet;151 152 int concurrent_proc_cnt = 0;153 struct kfd_node *kfd = pm->dqm->dev;154 struct amdgpu_device *adev = kfd->adev;155 156 /* Determine the number of processes to map together to HW:157 * it can not exceed the number of VMIDs available to the158 * scheduler, and it is determined by the smaller of the number159 * of processes in the runlist and kfd module parameter160 * hws_max_conc_proc.161 * However, if enforce_isolation is set (toggle LDS/VGPRs/SGPRs162 * cleaner between process switch), enable single-process mode163 * in HWS.164 * Note: the arbitration between the number of VMIDs and165 * hws_max_conc_proc has been done in166 * kgd2kfd_device_init().167 */168 concurrent_proc_cnt = adev->enforce_isolation[kfd->node_id] ?169 1 : min(pm->dqm->processes_count,170 kfd->max_proc_per_quantum);171 172 packet = (struct pm4_mes_runlist *)buffer;173 174 memset(buffer, 0, sizeof(struct pm4_mes_runlist));175 packet->header.u32All = pm_build_pm4_header(IT_RUN_LIST,176 sizeof(struct pm4_mes_runlist));177 178 packet->bitfields4.ib_size = ib_size_in_dwords;179 packet->bitfields4.chain = chain ? 1 : 0;180 packet->bitfields4.offload_polling = 0;181 packet->bitfields4.chained_runlist_idle_disable = chain ? 1 : 0;182 packet->bitfields4.valid = 1;183 packet->bitfields4.process_cnt = concurrent_proc_cnt;184 packet->ordinal2 = lower_32_bits(ib);185 packet->ib_base_hi = upper_32_bits(ib);186 187 return 0;188}189 190static int pm_set_resources_v9(struct packet_manager *pm, uint32_t *buffer,191 struct scheduling_resources *res)192{193 struct pm4_mes_set_resources *packet;194 195 packet = (struct pm4_mes_set_resources *)buffer;196 memset(buffer, 0, sizeof(struct pm4_mes_set_resources));197 198 packet->header.u32All = pm_build_pm4_header(IT_SET_RESOURCES,199 sizeof(struct pm4_mes_set_resources));200 201 packet->bitfields2.queue_type =202 queue_type__mes_set_resources__hsa_interface_queue_hiq;203 packet->bitfields2.vmid_mask = res->vmid_mask;204 packet->bitfields2.unmap_latency = KFD_UNMAP_LATENCY_MS / 100;205 packet->bitfields7.oac_mask = res->oac_mask;206 packet->bitfields8.gds_heap_base = res->gds_heap_base;207 packet->bitfields8.gds_heap_size = res->gds_heap_size;208 209 packet->gws_mask_lo = lower_32_bits(res->gws_mask);210 packet->gws_mask_hi = upper_32_bits(res->gws_mask);211 212 packet->queue_mask_lo = lower_32_bits(res->queue_mask);213 packet->queue_mask_hi = upper_32_bits(res->queue_mask);214 215 return 0;216}217 218static inline bool pm_use_ext_eng(struct kfd_dev *dev)219{220 return amdgpu_ip_version(dev->adev, SDMA0_HWIP, 0) >=221 IP_VERSION(5, 2, 0);222}223 224static int pm_map_queues_v9(struct packet_manager *pm, uint32_t *buffer,225 struct queue *q, bool is_static)226{227 struct pm4_mes_map_queues *packet;228 229 packet = (struct pm4_mes_map_queues *)buffer;230 memset(buffer, 0, sizeof(struct pm4_mes_map_queues));231 232 packet->header.u32All = pm_build_pm4_header(IT_MAP_QUEUES,233 sizeof(struct pm4_mes_map_queues));234 packet->bitfields2.num_queues = 1;235 packet->bitfields2.queue_sel =236 queue_sel__mes_map_queues__map_to_hws_determined_queue_slots_vi;237 238 packet->bitfields2.engine_sel =239 engine_sel__mes_map_queues__compute_vi;240 packet->bitfields2.gws_control_queue = q->gws ? 1 : 0;241 packet->bitfields2.extended_engine_sel =242 extended_engine_sel__mes_map_queues__legacy_engine_sel;243 packet->bitfields2.queue_type =244 queue_type__mes_map_queues__normal_compute_vi;245 246 switch (q->properties.type) {247 case KFD_QUEUE_TYPE_COMPUTE:248 if (is_static)249 packet->bitfields2.queue_type =250 queue_type__mes_map_queues__normal_latency_static_queue_vi;251 break;252 case KFD_QUEUE_TYPE_DIQ:253 packet->bitfields2.queue_type =254 queue_type__mes_map_queues__debug_interface_queue_vi;255 break;256 case KFD_QUEUE_TYPE_SDMA:257 case KFD_QUEUE_TYPE_SDMA_XGMI:258 if (q->properties.sdma_engine_id < 2 &&259 !pm_use_ext_eng(q->device->kfd))260 packet->bitfields2.engine_sel = q->properties.sdma_engine_id +261 engine_sel__mes_map_queues__sdma0_vi;262 else {263 /*264 * For GFX9.4.3, SDMA engine id can be greater than 8.265 * For such cases, set extended_engine_sel to 2 and266 * ensure engine_sel lies between 0-7.267 */268 if (q->properties.sdma_engine_id >= 8)269 packet->bitfields2.extended_engine_sel =270 extended_engine_sel__mes_map_queues__sdma8_to_15_sel;271 else272 packet->bitfields2.extended_engine_sel =273 extended_engine_sel__mes_map_queues__sdma0_to_7_sel;274 275 packet->bitfields2.engine_sel = q->properties.sdma_engine_id % 8;276 }277 break;278 default:279 WARN(1, "queue type %d", q->properties.type);280 return -EINVAL;281 }282 packet->bitfields3.doorbell_offset =283 q->properties.doorbell_off;284 285 packet->mqd_addr_lo =286 lower_32_bits(q->gart_mqd_addr);287 288 packet->mqd_addr_hi =289 upper_32_bits(q->gart_mqd_addr);290 291 packet->wptr_addr_lo =292 lower_32_bits((uint64_t)q->properties.write_ptr);293 294 packet->wptr_addr_hi =295 upper_32_bits((uint64_t)q->properties.write_ptr);296 297 return 0;298}299 300static int pm_set_grace_period_v9(struct packet_manager *pm,301 uint32_t *buffer,302 uint32_t grace_period)303{304 struct pm4_mec_write_data_mmio *packet;305 uint32_t reg_offset = 0;306 uint32_t reg_data = 0;307 308 pm->dqm->dev->kfd2kgd->build_grace_period_packet_info(309 pm->dqm->dev->adev,310 pm->dqm->wait_times,311 grace_period,312 ®_offset,313 ®_data);314 315 if (grace_period == USE_DEFAULT_GRACE_PERIOD)316 reg_data = pm->dqm->wait_times;317 318 packet = (struct pm4_mec_write_data_mmio *)buffer;319 memset(buffer, 0, sizeof(struct pm4_mec_write_data_mmio));320 321 packet->header.u32All = pm_build_pm4_header(IT_WRITE_DATA,322 sizeof(struct pm4_mec_write_data_mmio));323 324 packet->bitfields2.dst_sel = dst_sel___write_data__mem_mapped_register;325 packet->bitfields2.addr_incr =326 addr_incr___write_data__do_not_increment_address;327 328 packet->bitfields3.dst_mmreg_addr = reg_offset;329 330 packet->data = reg_data;331 332 return 0;333}334 335static int pm_unmap_queues_v9(struct packet_manager *pm, uint32_t *buffer,336 enum kfd_unmap_queues_filter filter,337 uint32_t filter_param, bool reset)338{339 struct pm4_mes_unmap_queues *packet;340 341 packet = (struct pm4_mes_unmap_queues *)buffer;342 memset(buffer, 0, sizeof(struct pm4_mes_unmap_queues));343 344 packet->header.u32All = pm_build_pm4_header(IT_UNMAP_QUEUES,345 sizeof(struct pm4_mes_unmap_queues));346 347 packet->bitfields2.extended_engine_sel =348 pm_use_ext_eng(pm->dqm->dev->kfd) ?349 extended_engine_sel__mes_unmap_queues__sdma0_to_7_sel :350 extended_engine_sel__mes_unmap_queues__legacy_engine_sel;351 352 packet->bitfields2.engine_sel =353 engine_sel__mes_unmap_queues__compute;354 355 if (reset)356 packet->bitfields2.action =357 action__mes_unmap_queues__reset_queues;358 else359 packet->bitfields2.action =360 action__mes_unmap_queues__preempt_queues;361 362 switch (filter) {363 case KFD_UNMAP_QUEUES_FILTER_BY_PASID:364 packet->bitfields2.queue_sel =365 queue_sel__mes_unmap_queues__perform_request_on_pasid_queues;366 packet->bitfields3a.pasid = filter_param;367 break;368 case KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES:369 packet->bitfields2.queue_sel =370 queue_sel__mes_unmap_queues__unmap_all_queues;371 break;372 case KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES:373 /* in this case, we do not preempt static queues */374 packet->bitfields2.queue_sel =375 queue_sel__mes_unmap_queues__unmap_all_non_static_queues;376 break;377 default:378 WARN(1, "filter %d", filter);379 return -EINVAL;380 }381 382 return 0;383 384}385 386static int pm_query_status_v9(struct packet_manager *pm, uint32_t *buffer,387 uint64_t fence_address, uint64_t fence_value)388{389 struct pm4_mes_query_status *packet;390 391 packet = (struct pm4_mes_query_status *)buffer;392 memset(buffer, 0, sizeof(struct pm4_mes_query_status));393 394 395 packet->header.u32All = pm_build_pm4_header(IT_QUERY_STATUS,396 sizeof(struct pm4_mes_query_status));397 398 packet->bitfields2.context_id = 0;399 packet->bitfields2.interrupt_sel =400 interrupt_sel__mes_query_status__completion_status;401 packet->bitfields2.command =402 command__mes_query_status__fence_only_after_write_ack;403 404 packet->addr_hi = upper_32_bits((uint64_t)fence_address);405 packet->addr_lo = lower_32_bits((uint64_t)fence_address);406 packet->data_hi = upper_32_bits((uint64_t)fence_value);407 packet->data_lo = lower_32_bits((uint64_t)fence_value);408 409 return 0;410}411 412const struct packet_manager_funcs kfd_v9_pm_funcs = {413 .map_process = pm_map_process_v9,414 .runlist = pm_runlist_v9,415 .set_resources = pm_set_resources_v9,416 .map_queues = pm_map_queues_v9,417 .unmap_queues = pm_unmap_queues_v9,418 .set_grace_period = pm_set_grace_period_v9,419 .query_status = pm_query_status_v9,420 .release_mem = NULL,421 .map_process_size = sizeof(struct pm4_mes_map_process),422 .runlist_size = sizeof(struct pm4_mes_runlist),423 .set_resources_size = sizeof(struct pm4_mes_set_resources),424 .map_queues_size = sizeof(struct pm4_mes_map_queues),425 .unmap_queues_size = sizeof(struct pm4_mes_unmap_queues),426 .set_grace_period_size = sizeof(struct pm4_mec_write_data_mmio),427 .query_status_size = sizeof(struct pm4_mes_query_status),428 .release_mem_size = 0,429};430 431const struct packet_manager_funcs kfd_aldebaran_pm_funcs = {432 .map_process = pm_map_process_aldebaran,433 .runlist = pm_runlist_v9,434 .set_resources = pm_set_resources_v9,435 .map_queues = pm_map_queues_v9,436 .unmap_queues = pm_unmap_queues_v9,437 .set_grace_period = pm_set_grace_period_v9,438 .query_status = pm_query_status_v9,439 .release_mem = NULL,440 .map_process_size = sizeof(struct pm4_mes_map_process_aldebaran),441 .runlist_size = sizeof(struct pm4_mes_runlist),442 .set_resources_size = sizeof(struct pm4_mes_set_resources),443 .map_queues_size = sizeof(struct pm4_mes_map_queues),444 .unmap_queues_size = sizeof(struct pm4_mes_unmap_queues),445 .set_grace_period_size = sizeof(struct pm4_mec_write_data_mmio),446 .query_status_size = sizeof(struct pm4_mes_query_status),447 .release_mem_size = 0,448};449