1149 lines · c
1// SPDX-License-Identifier: GPL-2.0 OR MIT2/*3 * Copyright 2014-2022 Advanced Micro Devices, Inc.4 *5 * Permission is hereby granted, free of charge, to any person obtaining a6 * copy of this software and associated documentation files (the "Software"),7 * to deal in the Software without restriction, including without limitation8 * the rights to use, copy, modify, merge, publish, distribute, sublicense,9 * and/or sell copies of the Software, and to permit persons to whom the10 * Software is furnished to do so, subject to the following conditions:11 *12 * The above copyright notice and this permission notice shall be included in13 * all copies or substantial portions of the Software.14 *15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL18 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR19 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,20 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR21 * OTHER DEALINGS IN THE SOFTWARE.22 *23 */24 25#include <linux/slab.h>26#include <linux/list.h>27#include "kfd_device_queue_manager.h"28#include "kfd_priv.h"29#include "kfd_kernel_queue.h"30#include "amdgpu_amdkfd.h"31#include "amdgpu_reset.h"32 33static inline struct process_queue_node *get_queue_by_qid(34 struct process_queue_manager *pqm, unsigned int qid)35{36 struct process_queue_node *pqn;37 38 list_for_each_entry(pqn, &pqm->queues, process_queue_list) {39 if ((pqn->q && pqn->q->properties.queue_id == qid) ||40 (pqn->kq && pqn->kq->queue->properties.queue_id == qid))41 return pqn;42 }43 44 return NULL;45}46 47static int assign_queue_slot_by_qid(struct process_queue_manager *pqm,48 unsigned int qid)49{50 if (qid >= KFD_MAX_NUM_OF_QUEUES_PER_PROCESS)51 return -EINVAL;52 53 if (__test_and_set_bit(qid, pqm->queue_slot_bitmap)) {54 pr_err("Cannot create new queue because requested qid(%u) is in use\n", qid);55 return -ENOSPC;56 }57 58 return 0;59}60 61static int find_available_queue_slot(struct process_queue_manager *pqm,62 unsigned int *qid)63{64 unsigned long found;65 66 found = find_first_zero_bit(pqm->queue_slot_bitmap,67 KFD_MAX_NUM_OF_QUEUES_PER_PROCESS);68 69 pr_debug("The new slot id %lu\n", found);70 71 if (found >= KFD_MAX_NUM_OF_QUEUES_PER_PROCESS) {72 pr_info("Cannot open more queues for process with pasid 0x%x\n",73 pqm->process->pasid);74 return -ENOMEM;75 }76 77 set_bit(found, pqm->queue_slot_bitmap);78 *qid = found;79 80 return 0;81}82 83void kfd_process_dequeue_from_device(struct kfd_process_device *pdd)84{85 struct kfd_node *dev = pdd->dev;86 87 if (pdd->already_dequeued)88 return;89 90 dev->dqm->ops.process_termination(dev->dqm, &pdd->qpd);91 if (dev->kfd->shared_resources.enable_mes &&92 down_read_trylock(&dev->adev->reset_domain->sem)) {93 amdgpu_mes_flush_shader_debugger(dev->adev,94 pdd->proc_ctx_gpu_addr);95 up_read(&dev->adev->reset_domain->sem);96 }97 pdd->already_dequeued = true;98}99 100int pqm_set_gws(struct process_queue_manager *pqm, unsigned int qid,101 void *gws)102{103 struct mqd_update_info minfo = {0};104 struct kfd_node *dev = NULL;105 struct process_queue_node *pqn;106 struct kfd_process_device *pdd;107 struct kgd_mem *mem = NULL;108 int ret;109 110 pqn = get_queue_by_qid(pqm, qid);111 if (!pqn) {112 pr_err("Queue id does not match any known queue\n");113 return -EINVAL;114 }115 116 if (pqn->q)117 dev = pqn->q->device;118 if (WARN_ON(!dev))119 return -ENODEV;120 121 pdd = kfd_get_process_device_data(dev, pqm->process);122 if (!pdd) {123 pr_err("Process device data doesn't exist\n");124 return -EINVAL;125 }126 127 /* Only allow one queue per process can have GWS assigned */128 if (gws && pdd->qpd.num_gws)129 return -EBUSY;130 131 if (!gws && pdd->qpd.num_gws == 0)132 return -EINVAL;133 134 if (KFD_GC_VERSION(dev) != IP_VERSION(9, 4, 3) &&135 KFD_GC_VERSION(dev) != IP_VERSION(9, 4, 4) &&136 !dev->kfd->shared_resources.enable_mes) {137 if (gws)138 ret = amdgpu_amdkfd_add_gws_to_process(pdd->process->kgd_process_info,139 gws, &mem);140 else141 ret = amdgpu_amdkfd_remove_gws_from_process(pdd->process->kgd_process_info,142 pqn->q->gws);143 if (unlikely(ret))144 return ret;145 pqn->q->gws = mem;146 } else {147 /*148 * Intentionally set GWS to a non-NULL value149 * for devices that do not use GWS for global wave150 * synchronization but require the formality151 * of setting GWS for cooperative groups.152 */153 pqn->q->gws = gws ? ERR_PTR(-ENOMEM) : NULL;154 }155 156 pdd->qpd.num_gws = gws ? dev->adev->gds.gws_size : 0;157 minfo.update_flag = gws ? UPDATE_FLAG_IS_GWS : 0;158 159 return pqn->q->device->dqm->ops.update_queue(pqn->q->device->dqm,160 pqn->q, &minfo);161}162 163void kfd_process_dequeue_from_all_devices(struct kfd_process *p)164{165 int i;166 167 for (i = 0; i < p->n_pdds; i++)168 kfd_process_dequeue_from_device(p->pdds[i]);169}170 171int pqm_init(struct process_queue_manager *pqm, struct kfd_process *p)172{173 INIT_LIST_HEAD(&pqm->queues);174 pqm->queue_slot_bitmap = bitmap_zalloc(KFD_MAX_NUM_OF_QUEUES_PER_PROCESS,175 GFP_KERNEL);176 if (!pqm->queue_slot_bitmap)177 return -ENOMEM;178 pqm->process = p;179 180 return 0;181}182 183static void pqm_clean_queue_resource(struct process_queue_manager *pqm,184 struct process_queue_node *pqn)185{186 struct kfd_node *dev;187 struct kfd_process_device *pdd;188 189 dev = pqn->q->device;190 191 pdd = kfd_get_process_device_data(dev, pqm->process);192 if (!pdd) {193 pr_err("Process device data doesn't exist\n");194 return;195 }196 197 if (pqn->q->gws) {198 if (KFD_GC_VERSION(pqn->q->device) != IP_VERSION(9, 4, 3) &&199 KFD_GC_VERSION(pqn->q->device) != IP_VERSION(9, 4, 4) &&200 !dev->kfd->shared_resources.enable_mes)201 amdgpu_amdkfd_remove_gws_from_process(202 pqm->process->kgd_process_info, pqn->q->gws);203 pdd->qpd.num_gws = 0;204 }205 206 if (dev->kfd->shared_resources.enable_mes) {207 amdgpu_amdkfd_free_gtt_mem(dev->adev, &pqn->q->gang_ctx_bo);208 amdgpu_amdkfd_free_gtt_mem(dev->adev, (void **)&pqn->q->wptr_bo_gart);209 }210}211 212void pqm_uninit(struct process_queue_manager *pqm)213{214 struct process_queue_node *pqn, *next;215 struct kfd_process_device *pdd;216 217 list_for_each_entry_safe(pqn, next, &pqm->queues, process_queue_list) {218 if (pqn->q) {219 pdd = kfd_get_process_device_data(pqn->q->device, pqm->process);220 kfd_queue_unref_bo_vas(pdd, &pqn->q->properties);221 kfd_queue_release_buffers(pdd, &pqn->q->properties);222 pqm_clean_queue_resource(pqm, pqn);223 }224 225 kfd_procfs_del_queue(pqn->q);226 uninit_queue(pqn->q);227 list_del(&pqn->process_queue_list);228 kfree(pqn);229 }230 231 bitmap_free(pqm->queue_slot_bitmap);232 pqm->queue_slot_bitmap = NULL;233}234 235static int init_user_queue(struct process_queue_manager *pqm,236 struct kfd_node *dev, struct queue **q,237 struct queue_properties *q_properties,238 struct file *f, unsigned int qid)239{240 int retval;241 242 /* Doorbell initialized in user space*/243 q_properties->doorbell_ptr = NULL;244 q_properties->exception_status = KFD_EC_MASK(EC_QUEUE_NEW);245 246 /* let DQM handle it*/247 q_properties->vmid = 0;248 q_properties->queue_id = qid;249 250 retval = init_queue(q, q_properties);251 if (retval != 0)252 return retval;253 254 (*q)->device = dev;255 (*q)->process = pqm->process;256 257 if (dev->kfd->shared_resources.enable_mes) {258 retval = amdgpu_amdkfd_alloc_gtt_mem(dev->adev,259 AMDGPU_MES_GANG_CTX_SIZE,260 &(*q)->gang_ctx_bo,261 &(*q)->gang_ctx_gpu_addr,262 &(*q)->gang_ctx_cpu_ptr,263 false);264 if (retval) {265 pr_err("failed to allocate gang context bo\n");266 goto cleanup;267 }268 memset((*q)->gang_ctx_cpu_ptr, 0, AMDGPU_MES_GANG_CTX_SIZE);269 270 /* Starting with GFX11, wptr BOs must be mapped to GART for MES to determine work271 * on unmapped queues for usermode queue oversubscription (no aggregated doorbell)272 */273 if (((dev->adev->mes.sched_version & AMDGPU_MES_API_VERSION_MASK)274 >> AMDGPU_MES_API_VERSION_SHIFT) >= 2) {275 if (dev->adev != amdgpu_ttm_adev(q_properties->wptr_bo->tbo.bdev)) {276 pr_err("Queue memory allocated to wrong device\n");277 retval = -EINVAL;278 goto free_gang_ctx_bo;279 }280 281 retval = amdgpu_amdkfd_map_gtt_bo_to_gart(q_properties->wptr_bo,282 &(*q)->wptr_bo_gart);283 if (retval) {284 pr_err("Failed to map wptr bo to GART\n");285 goto free_gang_ctx_bo;286 }287 }288 }289 290 pr_debug("PQM After init queue");291 return 0;292 293free_gang_ctx_bo:294 amdgpu_amdkfd_free_gtt_mem(dev->adev, (*q)->gang_ctx_bo);295cleanup:296 uninit_queue(*q);297 *q = NULL;298 return retval;299}300 301int pqm_create_queue(struct process_queue_manager *pqm,302 struct kfd_node *dev,303 struct file *f,304 struct queue_properties *properties,305 unsigned int *qid,306 const struct kfd_criu_queue_priv_data *q_data,307 const void *restore_mqd,308 const void *restore_ctl_stack,309 uint32_t *p_doorbell_offset_in_process)310{311 int retval;312 struct kfd_process_device *pdd;313 struct queue *q;314 struct process_queue_node *pqn;315 struct kernel_queue *kq;316 enum kfd_queue_type type = properties->type;317 unsigned int max_queues = 127; /* HWS limit */318 319 /*320 * On GFX 9.4.3, increase the number of queues that321 * can be created to 255. No HWS limit on GFX 9.4.3.322 */323 if (KFD_GC_VERSION(dev) == IP_VERSION(9, 4, 3) ||324 KFD_GC_VERSION(dev) == IP_VERSION(9, 4, 4))325 max_queues = 255;326 327 q = NULL;328 kq = NULL;329 330 pdd = kfd_get_process_device_data(dev, pqm->process);331 if (!pdd) {332 pr_err("Process device data doesn't exist\n");333 return -1;334 }335 336 /*337 * for debug process, verify that it is within the static queues limit338 * currently limit is set to half of the total avail HQD slots339 * If we are just about to create DIQ, the is_debug flag is not set yet340 * Hence we also check the type as well341 */342 if ((pdd->qpd.is_debug) || (type == KFD_QUEUE_TYPE_DIQ))343 max_queues = dev->kfd->device_info.max_no_of_hqd/2;344 345 if (pdd->qpd.queue_count >= max_queues)346 return -ENOSPC;347 348 if (q_data) {349 retval = assign_queue_slot_by_qid(pqm, q_data->q_id);350 *qid = q_data->q_id;351 } else352 retval = find_available_queue_slot(pqm, qid);353 354 if (retval != 0)355 return retval;356 357 if (list_empty(&pdd->qpd.queues_list) &&358 list_empty(&pdd->qpd.priv_queue_list))359 dev->dqm->ops.register_process(dev->dqm, &pdd->qpd);360 361 pqn = kzalloc(sizeof(*pqn), GFP_KERNEL);362 if (!pqn) {363 retval = -ENOMEM;364 goto err_allocate_pqn;365 }366 367 switch (type) {368 case KFD_QUEUE_TYPE_SDMA:369 case KFD_QUEUE_TYPE_SDMA_XGMI:370 case KFD_QUEUE_TYPE_SDMA_BY_ENG_ID:371 /* SDMA queues are always allocated statically no matter372 * which scheduler mode is used. We also do not need to373 * check whether a SDMA queue can be allocated here, because374 * allocate_sdma_queue() in create_queue() has the375 * corresponding check logic.376 */377 retval = init_user_queue(pqm, dev, &q, properties, f, *qid);378 if (retval != 0)379 goto err_create_queue;380 pqn->q = q;381 pqn->kq = NULL;382 retval = dev->dqm->ops.create_queue(dev->dqm, q, &pdd->qpd, q_data,383 restore_mqd, restore_ctl_stack);384 print_queue(q);385 break;386 387 case KFD_QUEUE_TYPE_COMPUTE:388 /* check if there is over subscription */389 if ((dev->dqm->sched_policy ==390 KFD_SCHED_POLICY_HWS_NO_OVERSUBSCRIPTION) &&391 ((dev->dqm->processes_count >= dev->vm_info.vmid_num_kfd) ||392 (dev->dqm->active_queue_count >= get_cp_queues_num(dev->dqm)))) {393 pr_debug("Over-subscription is not allowed when amdkfd.sched_policy == 1\n");394 retval = -EPERM;395 goto err_create_queue;396 }397 398 retval = init_user_queue(pqm, dev, &q, properties, f, *qid);399 if (retval != 0)400 goto err_create_queue;401 pqn->q = q;402 pqn->kq = NULL;403 retval = dev->dqm->ops.create_queue(dev->dqm, q, &pdd->qpd, q_data,404 restore_mqd, restore_ctl_stack);405 print_queue(q);406 break;407 case KFD_QUEUE_TYPE_DIQ:408 kq = kernel_queue_init(dev, KFD_QUEUE_TYPE_DIQ);409 if (!kq) {410 retval = -ENOMEM;411 goto err_create_queue;412 }413 kq->queue->properties.queue_id = *qid;414 pqn->kq = kq;415 pqn->q = NULL;416 retval = kfd_process_drain_interrupts(pdd);417 if (retval)418 break;419 420 retval = dev->dqm->ops.create_kernel_queue(dev->dqm,421 kq, &pdd->qpd);422 break;423 default:424 WARN(1, "Invalid queue type %d", type);425 retval = -EINVAL;426 }427 428 if (retval != 0) {429 pr_err("Pasid 0x%x DQM create queue type %d failed. ret %d\n",430 pqm->process->pasid, type, retval);431 goto err_create_queue;432 }433 434 if (q && p_doorbell_offset_in_process) {435 /* Return the doorbell offset within the doorbell page436 * to the caller so it can be passed up to user mode437 * (in bytes).438 * relative doorbell index = Absolute doorbell index -439 * absolute index of first doorbell in the page.440 */441 uint32_t first_db_index = amdgpu_doorbell_index_on_bar(pdd->dev->adev,442 pdd->qpd.proc_doorbells,443 0,444 pdd->dev->kfd->device_info.doorbell_size);445 446 *p_doorbell_offset_in_process = (q->properties.doorbell_off447 - first_db_index) * sizeof(uint32_t);448 }449 450 pr_debug("PQM After DQM create queue\n");451 452 list_add(&pqn->process_queue_list, &pqm->queues);453 454 if (q) {455 pr_debug("PQM done creating queue\n");456 kfd_procfs_add_queue(q);457 print_queue_properties(&q->properties);458 }459 460 return retval;461 462err_create_queue:463 uninit_queue(q);464 if (kq)465 kernel_queue_uninit(kq);466 kfree(pqn);467err_allocate_pqn:468 /* check if queues list is empty unregister process from device */469 clear_bit(*qid, pqm->queue_slot_bitmap);470 if (list_empty(&pdd->qpd.queues_list) &&471 list_empty(&pdd->qpd.priv_queue_list))472 dev->dqm->ops.unregister_process(dev->dqm, &pdd->qpd);473 return retval;474}475 476int pqm_destroy_queue(struct process_queue_manager *pqm, unsigned int qid)477{478 struct process_queue_node *pqn;479 struct kfd_process_device *pdd;480 struct device_queue_manager *dqm;481 struct kfd_node *dev;482 int retval;483 484 dqm = NULL;485 486 retval = 0;487 488 pqn = get_queue_by_qid(pqm, qid);489 if (!pqn) {490 pr_err("Queue id does not match any known queue\n");491 return -EINVAL;492 }493 494 dev = NULL;495 if (pqn->kq)496 dev = pqn->kq->dev;497 if (pqn->q)498 dev = pqn->q->device;499 if (WARN_ON(!dev))500 return -ENODEV;501 502 pdd = kfd_get_process_device_data(dev, pqm->process);503 if (!pdd) {504 pr_err("Process device data doesn't exist\n");505 return -1;506 }507 508 if (pqn->kq) {509 /* destroy kernel queue (DIQ) */510 dqm = pqn->kq->dev->dqm;511 dqm->ops.destroy_kernel_queue(dqm, pqn->kq, &pdd->qpd);512 kernel_queue_uninit(pqn->kq);513 }514 515 if (pqn->q) {516 retval = kfd_queue_unref_bo_vas(pdd, &pqn->q->properties);517 if (retval)518 goto err_destroy_queue;519 520 dqm = pqn->q->device->dqm;521 retval = dqm->ops.destroy_queue(dqm, &pdd->qpd, pqn->q);522 if (retval) {523 pr_err("Pasid 0x%x destroy queue %d failed, ret %d\n",524 pqm->process->pasid,525 pqn->q->properties.queue_id, retval);526 if (retval != -ETIME)527 goto err_destroy_queue;528 }529 kfd_procfs_del_queue(pqn->q);530 kfd_queue_release_buffers(pdd, &pqn->q->properties);531 pqm_clean_queue_resource(pqm, pqn);532 uninit_queue(pqn->q);533 }534 535 list_del(&pqn->process_queue_list);536 kfree(pqn);537 clear_bit(qid, pqm->queue_slot_bitmap);538 539 if (list_empty(&pdd->qpd.queues_list) &&540 list_empty(&pdd->qpd.priv_queue_list))541 dqm->ops.unregister_process(dqm, &pdd->qpd);542 543err_destroy_queue:544 return retval;545}546 547int pqm_update_queue_properties(struct process_queue_manager *pqm,548 unsigned int qid, struct queue_properties *p)549{550 int retval;551 struct process_queue_node *pqn;552 553 pqn = get_queue_by_qid(pqm, qid);554 if (!pqn || !pqn->q) {555 pr_debug("No queue %d exists for update operation\n", qid);556 return -EFAULT;557 }558 559 /*560 * Update with NULL ring address is used to disable the queue561 */562 if (p->queue_address && p->queue_size) {563 struct kfd_process_device *pdd;564 struct amdgpu_vm *vm;565 struct queue *q = pqn->q;566 int err;567 568 pdd = kfd_get_process_device_data(q->device, q->process);569 if (!pdd)570 return -ENODEV;571 vm = drm_priv_to_vm(pdd->drm_priv);572 err = amdgpu_bo_reserve(vm->root.bo, false);573 if (err)574 return err;575 576 if (kfd_queue_buffer_get(vm, (void *)p->queue_address, &p->ring_bo,577 p->queue_size)) {578 pr_debug("ring buf 0x%llx size 0x%llx not mapped on GPU\n",579 p->queue_address, p->queue_size);580 return -EFAULT;581 }582 583 kfd_queue_unref_bo_va(vm, &pqn->q->properties.ring_bo);584 kfd_queue_buffer_put(&pqn->q->properties.ring_bo);585 amdgpu_bo_unreserve(vm->root.bo);586 587 pqn->q->properties.ring_bo = p->ring_bo;588 }589 590 pqn->q->properties.queue_address = p->queue_address;591 pqn->q->properties.queue_size = p->queue_size;592 pqn->q->properties.queue_percent = p->queue_percent;593 pqn->q->properties.priority = p->priority;594 pqn->q->properties.pm4_target_xcc = p->pm4_target_xcc;595 596 retval = pqn->q->device->dqm->ops.update_queue(pqn->q->device->dqm,597 pqn->q, NULL);598 if (retval != 0)599 return retval;600 601 return 0;602}603 604int pqm_update_mqd(struct process_queue_manager *pqm,605 unsigned int qid, struct mqd_update_info *minfo)606{607 int retval;608 struct process_queue_node *pqn;609 610 pqn = get_queue_by_qid(pqm, qid);611 if (!pqn) {612 pr_debug("No queue %d exists for update operation\n", qid);613 return -EFAULT;614 }615 616 /* CUs are masked for debugger requirements so deny user mask */617 if (pqn->q->properties.is_dbg_wa && minfo && minfo->cu_mask.ptr)618 return -EBUSY;619 620 /* ASICs that have WGPs must enforce pairwise enabled mask checks. */621 if (minfo && minfo->cu_mask.ptr &&622 KFD_GC_VERSION(pqn->q->device) >= IP_VERSION(10, 0, 0)) {623 int i;624 625 for (i = 0; i < minfo->cu_mask.count; i += 2) {626 uint32_t cu_pair = (minfo->cu_mask.ptr[i / 32] >> (i % 32)) & 0x3;627 628 if (cu_pair && cu_pair != 0x3) {629 pr_debug("CUs must be adjacent pairwise enabled.\n");630 return -EINVAL;631 }632 }633 }634 635 retval = pqn->q->device->dqm->ops.update_queue(pqn->q->device->dqm,636 pqn->q, minfo);637 if (retval != 0)638 return retval;639 640 if (minfo && minfo->cu_mask.ptr)641 pqn->q->properties.is_user_cu_masked = true;642 643 return 0;644}645 646struct kernel_queue *pqm_get_kernel_queue(647 struct process_queue_manager *pqm,648 unsigned int qid)649{650 struct process_queue_node *pqn;651 652 pqn = get_queue_by_qid(pqm, qid);653 if (pqn && pqn->kq)654 return pqn->kq;655 656 return NULL;657}658 659struct queue *pqm_get_user_queue(struct process_queue_manager *pqm,660 unsigned int qid)661{662 struct process_queue_node *pqn;663 664 pqn = get_queue_by_qid(pqm, qid);665 return pqn ? pqn->q : NULL;666}667 668int pqm_get_wave_state(struct process_queue_manager *pqm,669 unsigned int qid,670 void __user *ctl_stack,671 u32 *ctl_stack_used_size,672 u32 *save_area_used_size)673{674 struct process_queue_node *pqn;675 676 pqn = get_queue_by_qid(pqm, qid);677 if (!pqn) {678 pr_debug("amdkfd: No queue %d exists for operation\n",679 qid);680 return -EFAULT;681 }682 683 return pqn->q->device->dqm->ops.get_wave_state(pqn->q->device->dqm,684 pqn->q,685 ctl_stack,686 ctl_stack_used_size,687 save_area_used_size);688}689 690int pqm_get_queue_snapshot(struct process_queue_manager *pqm,691 uint64_t exception_clear_mask,692 void __user *buf,693 int *num_qss_entries,694 uint32_t *entry_size)695{696 struct process_queue_node *pqn;697 struct kfd_queue_snapshot_entry src;698 uint32_t tmp_entry_size = *entry_size, tmp_qss_entries = *num_qss_entries;699 int r = 0;700 701 *num_qss_entries = 0;702 if (!(*entry_size))703 return -EINVAL;704 705 *entry_size = min_t(size_t, *entry_size, sizeof(struct kfd_queue_snapshot_entry));706 mutex_lock(&pqm->process->event_mutex);707 708 memset(&src, 0, sizeof(src));709 710 list_for_each_entry(pqn, &pqm->queues, process_queue_list) {711 if (!pqn->q)712 continue;713 714 if (*num_qss_entries < tmp_qss_entries) {715 set_queue_snapshot_entry(pqn->q, exception_clear_mask, &src);716 717 if (copy_to_user(buf, &src, *entry_size)) {718 r = -EFAULT;719 break;720 }721 buf += tmp_entry_size;722 }723 *num_qss_entries += 1;724 }725 726 mutex_unlock(&pqm->process->event_mutex);727 return r;728}729 730static int get_queue_data_sizes(struct kfd_process_device *pdd,731 struct queue *q,732 uint32_t *mqd_size,733 uint32_t *ctl_stack_size)734{735 int ret;736 737 ret = pqm_get_queue_checkpoint_info(&pdd->process->pqm,738 q->properties.queue_id,739 mqd_size,740 ctl_stack_size);741 if (ret)742 pr_err("Failed to get queue dump info (%d)\n", ret);743 744 return ret;745}746 747int kfd_process_get_queue_info(struct kfd_process *p,748 uint32_t *num_queues,749 uint64_t *priv_data_sizes)750{751 uint32_t extra_data_sizes = 0;752 struct queue *q;753 int i;754 int ret;755 756 *num_queues = 0;757 758 /* Run over all PDDs of the process */759 for (i = 0; i < p->n_pdds; i++) {760 struct kfd_process_device *pdd = p->pdds[i];761 762 list_for_each_entry(q, &pdd->qpd.queues_list, list) {763 if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE ||764 q->properties.type == KFD_QUEUE_TYPE_SDMA ||765 q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) {766 uint32_t mqd_size, ctl_stack_size;767 768 *num_queues = *num_queues + 1;769 770 ret = get_queue_data_sizes(pdd, q, &mqd_size, &ctl_stack_size);771 if (ret)772 return ret;773 774 extra_data_sizes += mqd_size + ctl_stack_size;775 } else {776 pr_err("Unsupported queue type (%d)\n", q->properties.type);777 return -EOPNOTSUPP;778 }779 }780 }781 *priv_data_sizes = extra_data_sizes +782 (*num_queues * sizeof(struct kfd_criu_queue_priv_data));783 784 return 0;785}786 787static int pqm_checkpoint_mqd(struct process_queue_manager *pqm,788 unsigned int qid,789 void *mqd,790 void *ctl_stack)791{792 struct process_queue_node *pqn;793 794 pqn = get_queue_by_qid(pqm, qid);795 if (!pqn) {796 pr_debug("amdkfd: No queue %d exists for operation\n", qid);797 return -EFAULT;798 }799 800 if (!pqn->q->device->dqm->ops.checkpoint_mqd) {801 pr_err("amdkfd: queue dumping not supported on this device\n");802 return -EOPNOTSUPP;803 }804 805 return pqn->q->device->dqm->ops.checkpoint_mqd(pqn->q->device->dqm,806 pqn->q, mqd, ctl_stack);807}808 809static int criu_checkpoint_queue(struct kfd_process_device *pdd,810 struct queue *q,811 struct kfd_criu_queue_priv_data *q_data)812{813 uint8_t *mqd, *ctl_stack;814 int ret;815 816 mqd = (void *)(q_data + 1);817 ctl_stack = mqd + q_data->mqd_size;818 819 q_data->gpu_id = pdd->user_gpu_id;820 q_data->type = q->properties.type;821 q_data->format = q->properties.format;822 q_data->q_id = q->properties.queue_id;823 q_data->q_address = q->properties.queue_address;824 q_data->q_size = q->properties.queue_size;825 q_data->priority = q->properties.priority;826 q_data->q_percent = q->properties.queue_percent;827 q_data->read_ptr_addr = (uint64_t)q->properties.read_ptr;828 q_data->write_ptr_addr = (uint64_t)q->properties.write_ptr;829 q_data->doorbell_id = q->doorbell_id;830 831 q_data->sdma_id = q->sdma_id;832 833 q_data->eop_ring_buffer_address =834 q->properties.eop_ring_buffer_address;835 836 q_data->eop_ring_buffer_size = q->properties.eop_ring_buffer_size;837 838 q_data->ctx_save_restore_area_address =839 q->properties.ctx_save_restore_area_address;840 841 q_data->ctx_save_restore_area_size =842 q->properties.ctx_save_restore_area_size;843 844 q_data->gws = !!q->gws;845 846 ret = pqm_checkpoint_mqd(&pdd->process->pqm, q->properties.queue_id, mqd, ctl_stack);847 if (ret) {848 pr_err("Failed checkpoint queue_mqd (%d)\n", ret);849 return ret;850 }851 852 pr_debug("Dumping Queue: gpu_id:%x queue_id:%u\n", q_data->gpu_id, q_data->q_id);853 return ret;854}855 856static int criu_checkpoint_queues_device(struct kfd_process_device *pdd,857 uint8_t __user *user_priv,858 unsigned int *q_index,859 uint64_t *queues_priv_data_offset)860{861 unsigned int q_private_data_size = 0;862 uint8_t *q_private_data = NULL; /* Local buffer to store individual queue private data */863 struct queue *q;864 int ret = 0;865 866 list_for_each_entry(q, &pdd->qpd.queues_list, list) {867 struct kfd_criu_queue_priv_data *q_data;868 uint64_t q_data_size;869 uint32_t mqd_size;870 uint32_t ctl_stack_size;871 872 if (q->properties.type != KFD_QUEUE_TYPE_COMPUTE &&873 q->properties.type != KFD_QUEUE_TYPE_SDMA &&874 q->properties.type != KFD_QUEUE_TYPE_SDMA_XGMI) {875 876 pr_err("Unsupported queue type (%d)\n", q->properties.type);877 ret = -EOPNOTSUPP;878 break;879 }880 881 ret = get_queue_data_sizes(pdd, q, &mqd_size, &ctl_stack_size);882 if (ret)883 break;884 885 q_data_size = sizeof(*q_data) + mqd_size + ctl_stack_size;886 887 /* Increase local buffer space if needed */888 if (q_private_data_size < q_data_size) {889 kfree(q_private_data);890 891 q_private_data = kzalloc(q_data_size, GFP_KERNEL);892 if (!q_private_data) {893 ret = -ENOMEM;894 break;895 }896 q_private_data_size = q_data_size;897 }898 899 q_data = (struct kfd_criu_queue_priv_data *)q_private_data;900 901 /* data stored in this order: priv_data, mqd, ctl_stack */902 q_data->mqd_size = mqd_size;903 q_data->ctl_stack_size = ctl_stack_size;904 905 ret = criu_checkpoint_queue(pdd, q, q_data);906 if (ret)907 break;908 909 q_data->object_type = KFD_CRIU_OBJECT_TYPE_QUEUE;910 911 ret = copy_to_user(user_priv + *queues_priv_data_offset,912 q_data, q_data_size);913 if (ret) {914 ret = -EFAULT;915 break;916 }917 *queues_priv_data_offset += q_data_size;918 *q_index = *q_index + 1;919 }920 921 kfree(q_private_data);922 923 return ret;924}925 926int kfd_criu_checkpoint_queues(struct kfd_process *p,927 uint8_t __user *user_priv_data,928 uint64_t *priv_data_offset)929{930 int ret = 0, pdd_index, q_index = 0;931 932 for (pdd_index = 0; pdd_index < p->n_pdds; pdd_index++) {933 struct kfd_process_device *pdd = p->pdds[pdd_index];934 935 /*936 * criu_checkpoint_queues_device will copy data to user and update q_index and937 * queues_priv_data_offset938 */939 ret = criu_checkpoint_queues_device(pdd, user_priv_data, &q_index,940 priv_data_offset);941 942 if (ret)943 break;944 }945 946 return ret;947}948 949static void set_queue_properties_from_criu(struct queue_properties *qp,950 struct kfd_criu_queue_priv_data *q_data)951{952 qp->is_interop = false;953 qp->queue_percent = q_data->q_percent;954 qp->priority = q_data->priority;955 qp->queue_address = q_data->q_address;956 qp->queue_size = q_data->q_size;957 qp->read_ptr = (uint32_t *) q_data->read_ptr_addr;958 qp->write_ptr = (uint32_t *) q_data->write_ptr_addr;959 qp->eop_ring_buffer_address = q_data->eop_ring_buffer_address;960 qp->eop_ring_buffer_size = q_data->eop_ring_buffer_size;961 qp->ctx_save_restore_area_address = q_data->ctx_save_restore_area_address;962 qp->ctx_save_restore_area_size = q_data->ctx_save_restore_area_size;963 qp->ctl_stack_size = q_data->ctl_stack_size;964 qp->type = q_data->type;965 qp->format = q_data->format;966}967 968int kfd_criu_restore_queue(struct kfd_process *p,969 uint8_t __user *user_priv_ptr,970 uint64_t *priv_data_offset,971 uint64_t max_priv_data_size)972{973 uint8_t *mqd, *ctl_stack, *q_extra_data = NULL;974 struct kfd_criu_queue_priv_data *q_data;975 struct kfd_process_device *pdd;976 uint64_t q_extra_data_size;977 struct queue_properties qp;978 unsigned int queue_id;979 int ret = 0;980 981 if (*priv_data_offset + sizeof(*q_data) > max_priv_data_size)982 return -EINVAL;983 984 q_data = kmalloc(sizeof(*q_data), GFP_KERNEL);985 if (!q_data)986 return -ENOMEM;987 988 ret = copy_from_user(q_data, user_priv_ptr + *priv_data_offset, sizeof(*q_data));989 if (ret) {990 ret = -EFAULT;991 goto exit;992 }993 994 *priv_data_offset += sizeof(*q_data);995 q_extra_data_size = (uint64_t)q_data->ctl_stack_size + q_data->mqd_size;996 997 if (*priv_data_offset + q_extra_data_size > max_priv_data_size) {998 ret = -EINVAL;999 goto exit;1000 }1001 1002 q_extra_data = kmalloc(q_extra_data_size, GFP_KERNEL);1003 if (!q_extra_data) {1004 ret = -ENOMEM;1005 goto exit;1006 }1007 1008 ret = copy_from_user(q_extra_data, user_priv_ptr + *priv_data_offset, q_extra_data_size);1009 if (ret) {1010 ret = -EFAULT;1011 goto exit;1012 }1013 1014 *priv_data_offset += q_extra_data_size;1015 1016 pdd = kfd_process_device_data_by_id(p, q_data->gpu_id);1017 if (!pdd) {1018 pr_err("Failed to get pdd\n");1019 ret = -EINVAL;1020 goto exit;1021 }1022 1023 /* data stored in this order: mqd, ctl_stack */1024 mqd = q_extra_data;1025 ctl_stack = mqd + q_data->mqd_size;1026 1027 memset(&qp, 0, sizeof(qp));1028 set_queue_properties_from_criu(&qp, q_data);1029 1030 print_queue_properties(&qp);1031 1032 ret = pqm_create_queue(&p->pqm, pdd->dev, NULL, &qp, &queue_id, q_data, mqd, ctl_stack,1033 NULL);1034 if (ret) {1035 pr_err("Failed to create new queue err:%d\n", ret);1036 goto exit;1037 }1038 1039 if (q_data->gws)1040 ret = pqm_set_gws(&p->pqm, q_data->q_id, pdd->dev->gws);1041 1042exit:1043 if (ret)1044 pr_err("Failed to restore queue (%d)\n", ret);1045 else1046 pr_debug("Queue id %d was restored successfully\n", queue_id);1047 1048 kfree(q_data);1049 kfree(q_extra_data);1050 1051 return ret;1052}1053 1054int pqm_get_queue_checkpoint_info(struct process_queue_manager *pqm,1055 unsigned int qid,1056 uint32_t *mqd_size,1057 uint32_t *ctl_stack_size)1058{1059 struct process_queue_node *pqn;1060 1061 pqn = get_queue_by_qid(pqm, qid);1062 if (!pqn) {1063 pr_debug("amdkfd: No queue %d exists for operation\n", qid);1064 return -EFAULT;1065 }1066 1067 if (!pqn->q->device->dqm->ops.get_queue_checkpoint_info) {1068 pr_err("amdkfd: queue dumping not supported on this device\n");1069 return -EOPNOTSUPP;1070 }1071 1072 pqn->q->device->dqm->ops.get_queue_checkpoint_info(pqn->q->device->dqm,1073 pqn->q, mqd_size,1074 ctl_stack_size);1075 return 0;1076}1077 1078#if defined(CONFIG_DEBUG_FS)1079 1080int pqm_debugfs_mqds(struct seq_file *m, void *data)1081{1082 struct process_queue_manager *pqm = data;1083 struct process_queue_node *pqn;1084 struct queue *q;1085 enum KFD_MQD_TYPE mqd_type;1086 struct mqd_manager *mqd_mgr;1087 int r = 0, xcc, num_xccs = 1;1088 void *mqd;1089 uint64_t size = 0;1090 1091 list_for_each_entry(pqn, &pqm->queues, process_queue_list) {1092 if (pqn->q) {1093 q = pqn->q;1094 switch (q->properties.type) {1095 case KFD_QUEUE_TYPE_SDMA:1096 case KFD_QUEUE_TYPE_SDMA_XGMI:1097 seq_printf(m, " SDMA queue on device %x\n",1098 q->device->id);1099 mqd_type = KFD_MQD_TYPE_SDMA;1100 break;1101 case KFD_QUEUE_TYPE_COMPUTE:1102 seq_printf(m, " Compute queue on device %x\n",1103 q->device->id);1104 mqd_type = KFD_MQD_TYPE_CP;1105 num_xccs = NUM_XCC(q->device->xcc_mask);1106 break;1107 default:1108 seq_printf(m,1109 " Bad user queue type %d on device %x\n",1110 q->properties.type, q->device->id);1111 continue;1112 }1113 mqd_mgr = q->device->dqm->mqd_mgrs[mqd_type];1114 size = mqd_mgr->mqd_stride(mqd_mgr,1115 &q->properties);1116 } else if (pqn->kq) {1117 q = pqn->kq->queue;1118 mqd_mgr = pqn->kq->mqd_mgr;1119 switch (q->properties.type) {1120 case KFD_QUEUE_TYPE_DIQ:1121 seq_printf(m, " DIQ on device %x\n",1122 pqn->kq->dev->id);1123 break;1124 default:1125 seq_printf(m,1126 " Bad kernel queue type %d on device %x\n",1127 q->properties.type,1128 pqn->kq->dev->id);1129 continue;1130 }1131 } else {1132 seq_printf(m,1133 " Weird: Queue node with neither kernel nor user queue\n");1134 continue;1135 }1136 1137 for (xcc = 0; xcc < num_xccs; xcc++) {1138 mqd = q->mqd + size * xcc;1139 r = mqd_mgr->debugfs_show_mqd(m, mqd);1140 if (r != 0)1141 break;1142 }1143 }1144 1145 return r;1146}1147 1148#endif1149