569 lines · c
1/*2 * Copyright 2021 Advanced Micro Devices, Inc.3 *4 * Permission is hereby granted, free of charge, to any person obtaining a5 * copy of this software and associated documentation files (the "Software"),6 * to deal in the Software without restriction, including without limitation7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,8 * and/or sell copies of the Software, and to permit persons to whom the9 * Software is furnished to do so, subject to the following conditions:10 *11 * The above copyright notice and this permission notice shall be included in12 * all copies or substantial portions of the Software.13 *14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR20 * OTHER DEALINGS IN THE SOFTWARE.21 *22 */23 24#include <linux/printk.h>25#include <linux/slab.h>26#include <linux/uaccess.h>27#include "kfd_priv.h"28#include "kfd_mqd_manager.h"29#include "v11_structs.h"30#include "gc/gc_11_0_0_offset.h"31#include "gc/gc_11_0_0_sh_mask.h"32#include "amdgpu_amdkfd.h"33 34static inline struct v11_compute_mqd *get_mqd(void *mqd)35{36 return (struct v11_compute_mqd *)mqd;37}38 39static inline struct v11_sdma_mqd *get_sdma_mqd(void *mqd)40{41 return (struct v11_sdma_mqd *)mqd;42}43 44static void update_cu_mask(struct mqd_manager *mm, void *mqd,45 struct mqd_update_info *minfo)46{47 struct v11_compute_mqd *m;48 uint32_t se_mask[KFD_MAX_NUM_SE] = {0};49 bool has_wa_flag = minfo && (minfo->update_flag & (UPDATE_FLAG_DBG_WA_ENABLE |50 UPDATE_FLAG_DBG_WA_DISABLE));51 52 if (!minfo || !(has_wa_flag || minfo->cu_mask.ptr))53 return;54 55 m = get_mqd(mqd);56 57 if (has_wa_flag) {58 uint32_t wa_mask =59 (minfo->update_flag & UPDATE_FLAG_DBG_WA_ENABLE) ? 0xffff : 0xffffffff;60 61 m->compute_static_thread_mgmt_se0 = wa_mask;62 m->compute_static_thread_mgmt_se1 = wa_mask;63 m->compute_static_thread_mgmt_se2 = wa_mask;64 m->compute_static_thread_mgmt_se3 = wa_mask;65 m->compute_static_thread_mgmt_se4 = wa_mask;66 m->compute_static_thread_mgmt_se5 = wa_mask;67 m->compute_static_thread_mgmt_se6 = wa_mask;68 m->compute_static_thread_mgmt_se7 = wa_mask;69 70 return;71 }72 73 mqd_symmetrically_map_cu_mask(mm,74 minfo->cu_mask.ptr, minfo->cu_mask.count, se_mask, 0);75 76 m->compute_static_thread_mgmt_se0 = se_mask[0];77 m->compute_static_thread_mgmt_se1 = se_mask[1];78 m->compute_static_thread_mgmt_se2 = se_mask[2];79 m->compute_static_thread_mgmt_se3 = se_mask[3];80 m->compute_static_thread_mgmt_se4 = se_mask[4];81 m->compute_static_thread_mgmt_se5 = se_mask[5];82 m->compute_static_thread_mgmt_se6 = se_mask[6];83 m->compute_static_thread_mgmt_se7 = se_mask[7];84 85 pr_debug("update cu mask to %#x %#x %#x %#x %#x %#x %#x %#x\n",86 m->compute_static_thread_mgmt_se0,87 m->compute_static_thread_mgmt_se1,88 m->compute_static_thread_mgmt_se2,89 m->compute_static_thread_mgmt_se3,90 m->compute_static_thread_mgmt_se4,91 m->compute_static_thread_mgmt_se5,92 m->compute_static_thread_mgmt_se6,93 m->compute_static_thread_mgmt_se7);94}95 96static void set_priority(struct v11_compute_mqd *m, struct queue_properties *q)97{98 m->cp_hqd_pipe_priority = pipe_priority_map[q->priority];99 m->cp_hqd_queue_priority = q->priority;100}101 102static struct kfd_mem_obj *allocate_mqd(struct kfd_node *node,103 struct queue_properties *q)104{105 struct kfd_mem_obj *mqd_mem_obj;106 int size;107 108 /*109 * MES write to areas beyond MQD size. So allocate110 * 1 PAGE_SIZE memory for MQD is MES is enabled.111 */112 if (node->kfd->shared_resources.enable_mes)113 size = PAGE_SIZE;114 else115 size = sizeof(struct v11_compute_mqd);116 117 if (kfd_gtt_sa_allocate(node, size, &mqd_mem_obj))118 return NULL;119 120 return mqd_mem_obj;121}122 123static void init_mqd(struct mqd_manager *mm, void **mqd,124 struct kfd_mem_obj *mqd_mem_obj, uint64_t *gart_addr,125 struct queue_properties *q)126{127 uint64_t addr;128 struct v11_compute_mqd *m;129 int size;130 uint32_t wa_mask = q->is_dbg_wa ? 0xffff : 0xffffffff;131 132 m = (struct v11_compute_mqd *) mqd_mem_obj->cpu_ptr;133 addr = mqd_mem_obj->gpu_addr;134 135 if (mm->dev->kfd->shared_resources.enable_mes)136 size = PAGE_SIZE;137 else138 size = sizeof(struct v11_compute_mqd);139 140 memset(m, 0, size);141 142 m->header = 0xC0310800;143 m->compute_pipelinestat_enable = 1;144 145 m->compute_static_thread_mgmt_se0 = wa_mask;146 m->compute_static_thread_mgmt_se1 = wa_mask;147 m->compute_static_thread_mgmt_se2 = wa_mask;148 m->compute_static_thread_mgmt_se3 = wa_mask;149 m->compute_static_thread_mgmt_se4 = wa_mask;150 m->compute_static_thread_mgmt_se5 = wa_mask;151 m->compute_static_thread_mgmt_se6 = wa_mask;152 m->compute_static_thread_mgmt_se7 = wa_mask;153 154 m->cp_hqd_persistent_state = CP_HQD_PERSISTENT_STATE__PRELOAD_REQ_MASK |155 0x55 << CP_HQD_PERSISTENT_STATE__PRELOAD_SIZE__SHIFT;156 157 m->cp_mqd_control = 1 << CP_MQD_CONTROL__PRIV_STATE__SHIFT;158 159 m->cp_mqd_base_addr_lo = lower_32_bits(addr);160 m->cp_mqd_base_addr_hi = upper_32_bits(addr);161 162 m->cp_hqd_quantum = 1 << CP_HQD_QUANTUM__QUANTUM_EN__SHIFT |163 1 << CP_HQD_QUANTUM__QUANTUM_SCALE__SHIFT |164 1 << CP_HQD_QUANTUM__QUANTUM_DURATION__SHIFT;165 166 /* Set cp_hqd_hq_scheduler0 bit 14 to 1 to have the CP set up the167 * DISPATCH_PTR. This is required for the kfd debugger168 */169 m->cp_hqd_hq_status0 = 1 << 14;170 171 /*172 * GFX11 RS64 CPFW version >= 509 supports PCIe atomics support173 * acknowledgment.174 */175 if (amdgpu_amdkfd_have_atomics_support(mm->dev->adev))176 m->cp_hqd_hq_status0 |= 1 << 29;177 178 if (q->format == KFD_QUEUE_FORMAT_AQL) {179 m->cp_hqd_aql_control =180 1 << CP_HQD_AQL_CONTROL__CONTROL0__SHIFT;181 }182 183 if (mm->dev->kfd->cwsr_enabled) {184 m->cp_hqd_persistent_state |=185 (1 << CP_HQD_PERSISTENT_STATE__QSWITCH_MODE__SHIFT);186 m->cp_hqd_ctx_save_base_addr_lo =187 lower_32_bits(q->ctx_save_restore_area_address);188 m->cp_hqd_ctx_save_base_addr_hi =189 upper_32_bits(q->ctx_save_restore_area_address);190 m->cp_hqd_ctx_save_size = q->ctx_save_restore_area_size;191 m->cp_hqd_cntl_stack_size = q->ctl_stack_size;192 m->cp_hqd_cntl_stack_offset = q->ctl_stack_size;193 m->cp_hqd_wg_state_offset = q->ctl_stack_size;194 }195 196 *mqd = m;197 if (gart_addr)198 *gart_addr = addr;199 mm->update_mqd(mm, m, q, NULL);200}201 202static int load_mqd(struct mqd_manager *mm, void *mqd,203 uint32_t pipe_id, uint32_t queue_id,204 struct queue_properties *p, struct mm_struct *mms)205{206 int r = 0;207 /* AQL write pointer counts in 64B packets, PM4/CP counts in dwords. */208 uint32_t wptr_shift = (p->format == KFD_QUEUE_FORMAT_AQL ? 4 : 0);209 210 r = mm->dev->kfd2kgd->hqd_load(mm->dev->adev, mqd, pipe_id, queue_id,211 (uint32_t __user *)p->write_ptr,212 wptr_shift, 0, mms, 0);213 return r;214}215 216static void update_mqd(struct mqd_manager *mm, void *mqd,217 struct queue_properties *q,218 struct mqd_update_info *minfo)219{220 struct v11_compute_mqd *m;221 222 m = get_mqd(mqd);223 224 m->cp_hqd_pq_control = 5 << CP_HQD_PQ_CONTROL__RPTR_BLOCK_SIZE__SHIFT;225 m->cp_hqd_pq_control |=226 ffs(q->queue_size / sizeof(unsigned int)) - 1 - 1;227 m->cp_hqd_pq_control |= CP_HQD_PQ_CONTROL__UNORD_DISPATCH_MASK;228 pr_debug("cp_hqd_pq_control 0x%x\n", m->cp_hqd_pq_control);229 230 m->cp_hqd_pq_base_lo = lower_32_bits((uint64_t)q->queue_address >> 8);231 m->cp_hqd_pq_base_hi = upper_32_bits((uint64_t)q->queue_address >> 8);232 233 m->cp_hqd_pq_rptr_report_addr_lo = lower_32_bits((uint64_t)q->read_ptr);234 m->cp_hqd_pq_rptr_report_addr_hi = upper_32_bits((uint64_t)q->read_ptr);235 m->cp_hqd_pq_wptr_poll_addr_lo = lower_32_bits((uint64_t)q->write_ptr);236 m->cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits((uint64_t)q->write_ptr);237 238 m->cp_hqd_pq_doorbell_control =239 q->doorbell_off <<240 CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_OFFSET__SHIFT;241 pr_debug("cp_hqd_pq_doorbell_control 0x%x\n",242 m->cp_hqd_pq_doorbell_control);243 244 m->cp_hqd_ib_control = 3 << CP_HQD_IB_CONTROL__MIN_IB_AVAIL_SIZE__SHIFT;245 246 /*247 * HW does not clamp this field correctly. Maximum EOP queue size248 * is constrained by per-SE EOP done signal count, which is 8-bit.249 * Limit is 0xFF EOP entries (= 0x7F8 dwords). CP will not submit250 * more than (EOP entry count - 1) so a queue size of 0x800 dwords251 * is safe, giving a maximum field value of 0xA.252 */253 m->cp_hqd_eop_control = min(0xA,254 ffs(q->eop_ring_buffer_size / sizeof(unsigned int)) - 1 - 1);255 m->cp_hqd_eop_base_addr_lo =256 lower_32_bits(q->eop_ring_buffer_address >> 8);257 m->cp_hqd_eop_base_addr_hi =258 upper_32_bits(q->eop_ring_buffer_address >> 8);259 260 m->cp_hqd_iq_timer = 0;261 262 m->cp_hqd_vmid = q->vmid;263 264 if (q->format == KFD_QUEUE_FORMAT_AQL) {265 /* GC 10 removed WPP_CLAMP from PQ Control */266 m->cp_hqd_pq_control |= CP_HQD_PQ_CONTROL__NO_UPDATE_RPTR_MASK |267 2 << CP_HQD_PQ_CONTROL__SLOT_BASED_WPTR__SHIFT |268 1 << CP_HQD_PQ_CONTROL__QUEUE_FULL_EN__SHIFT ;269 m->cp_hqd_pq_doorbell_control |=270 1 << CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_BIF_DROP__SHIFT;271 }272 if (mm->dev->kfd->cwsr_enabled)273 m->cp_hqd_ctx_save_control = 0;274 275 update_cu_mask(mm, mqd, minfo);276 set_priority(m, q);277 278 q->is_active = QUEUE_IS_ACTIVE(*q);279}280 281static bool check_preemption_failed(struct mqd_manager *mm, void *mqd)282{283 struct v11_compute_mqd *m = (struct v11_compute_mqd *)mqd;284 285 return kfd_check_hiq_mqd_doorbell_id(mm->dev, m->queue_doorbell_id0, 0);286}287 288static int get_wave_state(struct mqd_manager *mm, void *mqd,289 struct queue_properties *q,290 void __user *ctl_stack,291 u32 *ctl_stack_used_size,292 u32 *save_area_used_size)293{294 struct v11_compute_mqd *m;295 struct kfd_context_save_area_header header;296 297 m = get_mqd(mqd);298 299 /* Control stack is written backwards, while workgroup context data300 * is written forwards. Both starts from m->cp_hqd_cntl_stack_size.301 * Current position is at m->cp_hqd_cntl_stack_offset and302 * m->cp_hqd_wg_state_offset, respectively.303 */304 *ctl_stack_used_size = m->cp_hqd_cntl_stack_size -305 m->cp_hqd_cntl_stack_offset;306 *save_area_used_size = m->cp_hqd_wg_state_offset -307 m->cp_hqd_cntl_stack_size;308 309 /* Control stack is not copied to user mode for GFXv11 because310 * it's part of the context save area that is already311 * accessible to user mode312 */313 header.wave_state.control_stack_size = *ctl_stack_used_size;314 header.wave_state.wave_state_size = *save_area_used_size;315 316 header.wave_state.wave_state_offset = m->cp_hqd_wg_state_offset;317 header.wave_state.control_stack_offset = m->cp_hqd_cntl_stack_offset;318 319 if (copy_to_user(ctl_stack, &header, sizeof(header.wave_state)))320 return -EFAULT;321 322 return 0;323}324 325static void checkpoint_mqd(struct mqd_manager *mm, void *mqd, void *mqd_dst, void *ctl_stack_dst)326{327 struct v11_compute_mqd *m;328 329 m = get_mqd(mqd);330 331 memcpy(mqd_dst, m, sizeof(struct v11_compute_mqd));332}333 334static void restore_mqd(struct mqd_manager *mm, void **mqd,335 struct kfd_mem_obj *mqd_mem_obj, uint64_t *gart_addr,336 struct queue_properties *qp,337 const void *mqd_src,338 const void *ctl_stack_src, const u32 ctl_stack_size)339{340 uint64_t addr;341 struct v11_compute_mqd *m;342 343 m = (struct v11_compute_mqd *) mqd_mem_obj->cpu_ptr;344 addr = mqd_mem_obj->gpu_addr;345 346 memcpy(m, mqd_src, sizeof(*m));347 348 *mqd = m;349 if (gart_addr)350 *gart_addr = addr;351 352 m->cp_hqd_pq_doorbell_control =353 qp->doorbell_off <<354 CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_OFFSET__SHIFT;355 pr_debug("cp_hqd_pq_doorbell_control 0x%x\n",356 m->cp_hqd_pq_doorbell_control);357 358 qp->is_active = 0;359}360 361 362static void init_mqd_hiq(struct mqd_manager *mm, void **mqd,363 struct kfd_mem_obj *mqd_mem_obj, uint64_t *gart_addr,364 struct queue_properties *q)365{366 struct v11_compute_mqd *m;367 368 init_mqd(mm, mqd, mqd_mem_obj, gart_addr, q);369 370 m = get_mqd(*mqd);371 372 m->cp_hqd_pq_control |= 1 << CP_HQD_PQ_CONTROL__PRIV_STATE__SHIFT |373 1 << CP_HQD_PQ_CONTROL__KMD_QUEUE__SHIFT;374}375 376static int destroy_hiq_mqd(struct mqd_manager *mm, void *mqd,377 enum kfd_preempt_type type, unsigned int timeout,378 uint32_t pipe_id, uint32_t queue_id)379{380 int err;381 struct v11_compute_mqd *m;382 u32 doorbell_off;383 384 m = get_mqd(mqd);385 386 doorbell_off = m->cp_hqd_pq_doorbell_control >>387 CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_OFFSET__SHIFT;388 389 err = amdgpu_amdkfd_unmap_hiq(mm->dev->adev, doorbell_off, 0);390 if (err)391 pr_debug("Destroy HIQ MQD failed: %d\n", err);392 393 return err;394}395 396static void init_mqd_sdma(struct mqd_manager *mm, void **mqd,397 struct kfd_mem_obj *mqd_mem_obj, uint64_t *gart_addr,398 struct queue_properties *q)399{400 struct v11_sdma_mqd *m;401 int size;402 403 m = (struct v11_sdma_mqd *) mqd_mem_obj->cpu_ptr;404 405 if (mm->dev->kfd->shared_resources.enable_mes)406 size = PAGE_SIZE;407 else408 size = sizeof(struct v11_sdma_mqd);409 410 memset(m, 0, size);411 *mqd = m;412 if (gart_addr)413 *gart_addr = mqd_mem_obj->gpu_addr;414 415 mm->update_mqd(mm, m, q, NULL);416}417 418#define SDMA_RLC_DUMMY_DEFAULT 0xf419 420static void update_mqd_sdma(struct mqd_manager *mm, void *mqd,421 struct queue_properties *q,422 struct mqd_update_info *minfo)423{424 struct v11_sdma_mqd *m;425 426 m = get_sdma_mqd(mqd);427 m->sdmax_rlcx_rb_cntl = (ffs(q->queue_size / sizeof(unsigned int)) - 1)428 << SDMA0_QUEUE0_RB_CNTL__RB_SIZE__SHIFT |429 q->vmid << SDMA0_QUEUE0_RB_CNTL__RB_VMID__SHIFT |430 1 << SDMA0_QUEUE0_RB_CNTL__RPTR_WRITEBACK_ENABLE__SHIFT |431 6 << SDMA0_QUEUE0_RB_CNTL__RPTR_WRITEBACK_TIMER__SHIFT |432 1 << SDMA0_QUEUE0_RB_CNTL__F32_WPTR_POLL_ENABLE__SHIFT;433 434 m->sdmax_rlcx_rb_base = lower_32_bits(q->queue_address >> 8);435 m->sdmax_rlcx_rb_base_hi = upper_32_bits(q->queue_address >> 8);436 m->sdmax_rlcx_rb_rptr_addr_lo = lower_32_bits((uint64_t)q->read_ptr);437 m->sdmax_rlcx_rb_rptr_addr_hi = upper_32_bits((uint64_t)q->read_ptr);438 m->sdmax_rlcx_rb_wptr_poll_addr_lo = lower_32_bits((uint64_t)q->write_ptr);439 m->sdmax_rlcx_rb_wptr_poll_addr_hi = upper_32_bits((uint64_t)q->write_ptr);440 m->sdmax_rlcx_doorbell_offset =441 q->doorbell_off << SDMA0_QUEUE0_DOORBELL_OFFSET__OFFSET__SHIFT;442 443 m->sdmax_rlcx_sched_cntl = (amdgpu_sdma_phase_quantum444 << SDMA0_QUEUE0_SCHEDULE_CNTL__CONTEXT_QUANTUM__SHIFT)445 & SDMA0_QUEUE0_SCHEDULE_CNTL__CONTEXT_QUANTUM_MASK;446 447 m->sdma_engine_id = q->sdma_engine_id;448 m->sdma_queue_id = q->sdma_queue_id;449 m->sdmax_rlcx_dummy_reg = SDMA_RLC_DUMMY_DEFAULT;450 451 q->is_active = QUEUE_IS_ACTIVE(*q);452}453 454#if defined(CONFIG_DEBUG_FS)455 456static int debugfs_show_mqd(struct seq_file *m, void *data)457{458 seq_hex_dump(m, " ", DUMP_PREFIX_OFFSET, 32, 4,459 data, sizeof(struct v11_compute_mqd), false);460 return 0;461}462 463static int debugfs_show_mqd_sdma(struct seq_file *m, void *data)464{465 seq_hex_dump(m, " ", DUMP_PREFIX_OFFSET, 32, 4,466 data, sizeof(struct v11_sdma_mqd), false);467 return 0;468}469 470#endif471 472struct mqd_manager *mqd_manager_init_v11(enum KFD_MQD_TYPE type,473 struct kfd_node *dev)474{475 struct mqd_manager *mqd;476 477 if (WARN_ON(type >= KFD_MQD_TYPE_MAX))478 return NULL;479 480 mqd = kzalloc(sizeof(*mqd), GFP_KERNEL);481 if (!mqd)482 return NULL;483 484 mqd->dev = dev;485 486 switch (type) {487 case KFD_MQD_TYPE_CP:488 pr_debug("%s@%i\n", __func__, __LINE__);489 mqd->allocate_mqd = allocate_mqd;490 mqd->init_mqd = init_mqd;491 mqd->free_mqd = kfd_free_mqd_cp;492 mqd->load_mqd = load_mqd;493 mqd->update_mqd = update_mqd;494 mqd->destroy_mqd = kfd_destroy_mqd_cp;495 mqd->is_occupied = kfd_is_occupied_cp;496 mqd->mqd_size = sizeof(struct v11_compute_mqd);497 mqd->get_wave_state = get_wave_state;498 mqd->mqd_stride = kfd_mqd_stride;499 mqd->checkpoint_mqd = checkpoint_mqd;500 mqd->restore_mqd = restore_mqd;501#if defined(CONFIG_DEBUG_FS)502 mqd->debugfs_show_mqd = debugfs_show_mqd;503#endif504 pr_debug("%s@%i\n", __func__, __LINE__);505 break;506 case KFD_MQD_TYPE_HIQ:507 pr_debug("%s@%i\n", __func__, __LINE__);508 mqd->allocate_mqd = allocate_hiq_mqd;509 mqd->init_mqd = init_mqd_hiq;510 mqd->free_mqd = free_mqd_hiq_sdma;511 mqd->load_mqd = kfd_hiq_load_mqd_kiq;512 mqd->update_mqd = update_mqd;513 mqd->destroy_mqd = destroy_hiq_mqd;514 mqd->is_occupied = kfd_is_occupied_cp;515 mqd->mqd_size = sizeof(struct v11_compute_mqd);516 mqd->mqd_stride = kfd_mqd_stride;517#if defined(CONFIG_DEBUG_FS)518 mqd->debugfs_show_mqd = debugfs_show_mqd;519#endif520 mqd->check_preemption_failed = check_preemption_failed;521 pr_debug("%s@%i\n", __func__, __LINE__);522 break;523 case KFD_MQD_TYPE_DIQ:524 mqd->allocate_mqd = allocate_mqd;525 mqd->init_mqd = init_mqd_hiq;526 mqd->free_mqd = kfd_free_mqd_cp;527 mqd->load_mqd = load_mqd;528 mqd->update_mqd = update_mqd;529 mqd->destroy_mqd = kfd_destroy_mqd_cp;530 mqd->is_occupied = kfd_is_occupied_cp;531 mqd->mqd_size = sizeof(struct v11_compute_mqd);532#if defined(CONFIG_DEBUG_FS)533 mqd->debugfs_show_mqd = debugfs_show_mqd;534#endif535 break;536 case KFD_MQD_TYPE_SDMA:537 pr_debug("%s@%i\n", __func__, __LINE__);538 mqd->allocate_mqd = allocate_sdma_mqd;539 mqd->init_mqd = init_mqd_sdma;540 mqd->free_mqd = free_mqd_hiq_sdma;541 mqd->load_mqd = kfd_load_mqd_sdma;542 mqd->update_mqd = update_mqd_sdma;543 mqd->destroy_mqd = kfd_destroy_mqd_sdma;544 mqd->is_occupied = kfd_is_occupied_sdma;545 mqd->checkpoint_mqd = checkpoint_mqd;546 mqd->restore_mqd = restore_mqd;547 mqd->mqd_size = sizeof(struct v11_sdma_mqd);548 mqd->mqd_stride = kfd_mqd_stride;549#if defined(CONFIG_DEBUG_FS)550 mqd->debugfs_show_mqd = debugfs_show_mqd_sdma;551#endif552 /*553 * To allocate SDMA MQDs by generic functions554 * when MES is enabled.555 */556 if (dev->kfd->shared_resources.enable_mes) {557 mqd->allocate_mqd = allocate_mqd;558 mqd->free_mqd = kfd_free_mqd_cp;559 }560 pr_debug("%s@%i\n", __func__, __LINE__);561 break;562 default:563 kfree(mqd);564 return NULL;565 }566 567 return mqd;568}569