2312 lines · c
1// SPDX-License-Identifier: GPL-2.0 OR MIT2/*3 * Copyright 2014-2022 Advanced Micro Devices, Inc.4 *5 * Permission is hereby granted, free of charge, to any person obtaining a6 * copy of this software and associated documentation files (the "Software"),7 * to deal in the Software without restriction, including without limitation8 * the rights to use, copy, modify, merge, publish, distribute, sublicense,9 * and/or sell copies of the Software, and to permit persons to whom the10 * Software is furnished to do so, subject to the following conditions:11 *12 * The above copyright notice and this permission notice shall be included in13 * all copies or substantial portions of the Software.14 *15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL18 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR19 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,20 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR21 * OTHER DEALINGS IN THE SOFTWARE.22 */23 24#include <linux/mutex.h>25#include <linux/log2.h>26#include <linux/sched.h>27#include <linux/sched/mm.h>28#include <linux/sched/task.h>29#include <linux/mmu_context.h>30#include <linux/slab.h>31#include <linux/notifier.h>32#include <linux/compat.h>33#include <linux/mman.h>34#include <linux/file.h>35#include <linux/pm_runtime.h>36#include "amdgpu_amdkfd.h"37#include "amdgpu.h"38 39struct mm_struct;40 41#include "kfd_priv.h"42#include "kfd_device_queue_manager.h"43#include "kfd_svm.h"44#include "kfd_smi_events.h"45#include "kfd_debug.h"46 47/*48 * List of struct kfd_process (field kfd_process).49 * Unique/indexed by mm_struct*50 */51DEFINE_HASHTABLE(kfd_processes_table, KFD_PROCESS_TABLE_SIZE);52DEFINE_MUTEX(kfd_processes_mutex);53 54DEFINE_SRCU(kfd_processes_srcu);55 56/* For process termination handling */57static struct workqueue_struct *kfd_process_wq;58 59/* Ordered, single-threaded workqueue for restoring evicted60 * processes. Restoring multiple processes concurrently under memory61 * pressure can lead to processes blocking each other from validating62 * their BOs and result in a live-lock situation where processes63 * remain evicted indefinitely.64 */65static struct workqueue_struct *kfd_restore_wq;66 67static struct kfd_process *find_process(const struct task_struct *thread,68 bool ref);69static void kfd_process_ref_release(struct kref *ref);70static struct kfd_process *create_process(const struct task_struct *thread);71 72static void evict_process_worker(struct work_struct *work);73static void restore_process_worker(struct work_struct *work);74 75static void kfd_process_device_destroy_cwsr_dgpu(struct kfd_process_device *pdd);76 77struct kfd_procfs_tree {78 struct kobject *kobj;79};80 81static struct kfd_procfs_tree procfs;82 83/*84 * Structure for SDMA activity tracking85 */86struct kfd_sdma_activity_handler_workarea {87 struct work_struct sdma_activity_work;88 struct kfd_process_device *pdd;89 uint64_t sdma_activity_counter;90};91 92struct temp_sdma_queue_list {93 uint64_t __user *rptr;94 uint64_t sdma_val;95 unsigned int queue_id;96 struct list_head list;97};98 99static void kfd_sdma_activity_worker(struct work_struct *work)100{101 struct kfd_sdma_activity_handler_workarea *workarea;102 struct kfd_process_device *pdd;103 uint64_t val;104 struct mm_struct *mm;105 struct queue *q;106 struct qcm_process_device *qpd;107 struct device_queue_manager *dqm;108 int ret = 0;109 struct temp_sdma_queue_list sdma_q_list;110 struct temp_sdma_queue_list *sdma_q, *next;111 112 workarea = container_of(work, struct kfd_sdma_activity_handler_workarea,113 sdma_activity_work);114 115 pdd = workarea->pdd;116 if (!pdd)117 return;118 dqm = pdd->dev->dqm;119 qpd = &pdd->qpd;120 if (!dqm || !qpd)121 return;122 /*123 * Total SDMA activity is current SDMA activity + past SDMA activity124 * Past SDMA count is stored in pdd.125 * To get the current activity counters for all active SDMA queues,126 * we loop over all SDMA queues and get their counts from user-space.127 *128 * We cannot call get_user() with dqm_lock held as it can cause129 * a circular lock dependency situation. To read the SDMA stats,130 * we need to do the following:131 *132 * 1. Create a temporary list of SDMA queue nodes from the qpd->queues_list,133 * with dqm_lock/dqm_unlock().134 * 2. Call get_user() for each node in temporary list without dqm_lock.135 * Save the SDMA count for each node and also add the count to the total136 * SDMA count counter.137 * Its possible, during this step, a few SDMA queue nodes got deleted138 * from the qpd->queues_list.139 * 3. Do a second pass over qpd->queues_list to check if any nodes got deleted.140 * If any node got deleted, its SDMA count would be captured in the sdma141 * past activity counter. So subtract the SDMA counter stored in step 2142 * for this node from the total SDMA count.143 */144 INIT_LIST_HEAD(&sdma_q_list.list);145 146 /*147 * Create the temp list of all SDMA queues148 */149 dqm_lock(dqm);150 151 list_for_each_entry(q, &qpd->queues_list, list) {152 if ((q->properties.type != KFD_QUEUE_TYPE_SDMA) &&153 (q->properties.type != KFD_QUEUE_TYPE_SDMA_XGMI))154 continue;155 156 sdma_q = kzalloc(sizeof(struct temp_sdma_queue_list), GFP_KERNEL);157 if (!sdma_q) {158 dqm_unlock(dqm);159 goto cleanup;160 }161 162 INIT_LIST_HEAD(&sdma_q->list);163 sdma_q->rptr = (uint64_t __user *)q->properties.read_ptr;164 sdma_q->queue_id = q->properties.queue_id;165 list_add_tail(&sdma_q->list, &sdma_q_list.list);166 }167 168 /*169 * If the temp list is empty, then no SDMA queues nodes were found in170 * qpd->queues_list. Return the past activity count as the total sdma171 * count172 */173 if (list_empty(&sdma_q_list.list)) {174 workarea->sdma_activity_counter = pdd->sdma_past_activity_counter;175 dqm_unlock(dqm);176 return;177 }178 179 dqm_unlock(dqm);180 181 /*182 * Get the usage count for each SDMA queue in temp_list.183 */184 mm = get_task_mm(pdd->process->lead_thread);185 if (!mm)186 goto cleanup;187 188 kthread_use_mm(mm);189 190 list_for_each_entry(sdma_q, &sdma_q_list.list, list) {191 val = 0;192 ret = read_sdma_queue_counter(sdma_q->rptr, &val);193 if (ret) {194 pr_debug("Failed to read SDMA queue active counter for queue id: %d",195 sdma_q->queue_id);196 } else {197 sdma_q->sdma_val = val;198 workarea->sdma_activity_counter += val;199 }200 }201 202 kthread_unuse_mm(mm);203 mmput(mm);204 205 /*206 * Do a second iteration over qpd_queues_list to check if any SDMA207 * nodes got deleted while fetching SDMA counter.208 */209 dqm_lock(dqm);210 211 workarea->sdma_activity_counter += pdd->sdma_past_activity_counter;212 213 list_for_each_entry(q, &qpd->queues_list, list) {214 if (list_empty(&sdma_q_list.list))215 break;216 217 if ((q->properties.type != KFD_QUEUE_TYPE_SDMA) &&218 (q->properties.type != KFD_QUEUE_TYPE_SDMA_XGMI))219 continue;220 221 list_for_each_entry_safe(sdma_q, next, &sdma_q_list.list, list) {222 if (((uint64_t __user *)q->properties.read_ptr == sdma_q->rptr) &&223 (sdma_q->queue_id == q->properties.queue_id)) {224 list_del(&sdma_q->list);225 kfree(sdma_q);226 break;227 }228 }229 }230 231 dqm_unlock(dqm);232 233 /*234 * If temp list is not empty, it implies some queues got deleted235 * from qpd->queues_list during SDMA usage read. Subtract the SDMA236 * count for each node from the total SDMA count.237 */238 list_for_each_entry_safe(sdma_q, next, &sdma_q_list.list, list) {239 workarea->sdma_activity_counter -= sdma_q->sdma_val;240 list_del(&sdma_q->list);241 kfree(sdma_q);242 }243 244 return;245 246cleanup:247 list_for_each_entry_safe(sdma_q, next, &sdma_q_list.list, list) {248 list_del(&sdma_q->list);249 kfree(sdma_q);250 }251}252 253/**254 * kfd_get_cu_occupancy - Collect number of waves in-flight on this device255 * by current process. Translates acquired wave count into number of compute units256 * that are occupied.257 *258 * @attr: Handle of attribute that allows reporting of wave count. The attribute259 * handle encapsulates GPU device it is associated with, thereby allowing collection260 * of waves in flight, etc261 * @buffer: Handle of user provided buffer updated with wave count262 *263 * Return: Number of bytes written to user buffer or an error value264 */265static int kfd_get_cu_occupancy(struct attribute *attr, char *buffer)266{267 int cu_cnt;268 int wave_cnt;269 int max_waves_per_cu;270 struct kfd_node *dev = NULL;271 struct kfd_process *proc = NULL;272 struct kfd_process_device *pdd = NULL;273 int i;274 struct kfd_cu_occupancy cu_occupancy[AMDGPU_MAX_QUEUES];275 u32 queue_format;276 277 memset(cu_occupancy, 0x0, sizeof(cu_occupancy));278 279 pdd = container_of(attr, struct kfd_process_device, attr_cu_occupancy);280 dev = pdd->dev;281 if (dev->kfd2kgd->get_cu_occupancy == NULL)282 return -EINVAL;283 284 cu_cnt = 0;285 proc = pdd->process;286 if (pdd->qpd.queue_count == 0) {287 pr_debug("Gpu-Id: %d has no active queues for process %d\n",288 dev->id, proc->pasid);289 return snprintf(buffer, PAGE_SIZE, "%d\n", cu_cnt);290 }291 292 /* Collect wave count from device if it supports */293 wave_cnt = 0;294 max_waves_per_cu = 0;295 296 /*297 * For GFX 9.4.3, fetch the CU occupancy from the first XCC in the partition.298 * For AQL queues, because of cooperative dispatch we multiply the wave count299 * by number of XCCs in the partition to get the total wave counts across all300 * XCCs in the partition.301 * For PM4 queues, there is no cooperative dispatch so wave_cnt stay as it is.302 */303 dev->kfd2kgd->get_cu_occupancy(dev->adev, cu_occupancy,304 &max_waves_per_cu, ffs(dev->xcc_mask) - 1);305 306 for (i = 0; i < AMDGPU_MAX_QUEUES; i++) {307 if (cu_occupancy[i].wave_cnt != 0 &&308 kfd_dqm_is_queue_in_process(dev->dqm, &pdd->qpd,309 cu_occupancy[i].doorbell_off,310 &queue_format)) {311 if (unlikely(queue_format == KFD_QUEUE_FORMAT_PM4))312 wave_cnt += cu_occupancy[i].wave_cnt;313 else314 wave_cnt += (NUM_XCC(dev->xcc_mask) *315 cu_occupancy[i].wave_cnt);316 }317 }318 319 /* Translate wave count to number of compute units */320 cu_cnt = (wave_cnt + (max_waves_per_cu - 1)) / max_waves_per_cu;321 return snprintf(buffer, PAGE_SIZE, "%d\n", cu_cnt);322}323 324static ssize_t kfd_procfs_show(struct kobject *kobj, struct attribute *attr,325 char *buffer)326{327 if (strcmp(attr->name, "pasid") == 0) {328 struct kfd_process *p = container_of(attr, struct kfd_process,329 attr_pasid);330 331 return snprintf(buffer, PAGE_SIZE, "%d\n", p->pasid);332 } else if (strncmp(attr->name, "vram_", 5) == 0) {333 struct kfd_process_device *pdd = container_of(attr, struct kfd_process_device,334 attr_vram);335 return snprintf(buffer, PAGE_SIZE, "%llu\n", atomic64_read(&pdd->vram_usage));336 } else if (strncmp(attr->name, "sdma_", 5) == 0) {337 struct kfd_process_device *pdd = container_of(attr, struct kfd_process_device,338 attr_sdma);339 struct kfd_sdma_activity_handler_workarea sdma_activity_work_handler;340 341 INIT_WORK(&sdma_activity_work_handler.sdma_activity_work,342 kfd_sdma_activity_worker);343 344 sdma_activity_work_handler.pdd = pdd;345 sdma_activity_work_handler.sdma_activity_counter = 0;346 347 schedule_work(&sdma_activity_work_handler.sdma_activity_work);348 349 flush_work(&sdma_activity_work_handler.sdma_activity_work);350 351 return snprintf(buffer, PAGE_SIZE, "%llu\n",352 (sdma_activity_work_handler.sdma_activity_counter)/353 SDMA_ACTIVITY_DIVISOR);354 } else {355 pr_err("Invalid attribute");356 return -EINVAL;357 }358 359 return 0;360}361 362static void kfd_procfs_kobj_release(struct kobject *kobj)363{364 kfree(kobj);365}366 367static const struct sysfs_ops kfd_procfs_ops = {368 .show = kfd_procfs_show,369};370 371static const struct kobj_type procfs_type = {372 .release = kfd_procfs_kobj_release,373 .sysfs_ops = &kfd_procfs_ops,374};375 376void kfd_procfs_init(void)377{378 int ret = 0;379 380 procfs.kobj = kfd_alloc_struct(procfs.kobj);381 if (!procfs.kobj)382 return;383 384 ret = kobject_init_and_add(procfs.kobj, &procfs_type,385 &kfd_device->kobj, "proc");386 if (ret) {387 pr_warn("Could not create procfs proc folder");388 /* If we fail to create the procfs, clean up */389 kfd_procfs_shutdown();390 }391}392 393void kfd_procfs_shutdown(void)394{395 if (procfs.kobj) {396 kobject_del(procfs.kobj);397 kobject_put(procfs.kobj);398 procfs.kobj = NULL;399 }400}401 402static ssize_t kfd_procfs_queue_show(struct kobject *kobj,403 struct attribute *attr, char *buffer)404{405 struct queue *q = container_of(kobj, struct queue, kobj);406 407 if (!strcmp(attr->name, "size"))408 return snprintf(buffer, PAGE_SIZE, "%llu",409 q->properties.queue_size);410 else if (!strcmp(attr->name, "type"))411 return snprintf(buffer, PAGE_SIZE, "%d", q->properties.type);412 else if (!strcmp(attr->name, "gpuid"))413 return snprintf(buffer, PAGE_SIZE, "%u", q->device->id);414 else415 pr_err("Invalid attribute");416 417 return 0;418}419 420static ssize_t kfd_procfs_stats_show(struct kobject *kobj,421 struct attribute *attr, char *buffer)422{423 if (strcmp(attr->name, "evicted_ms") == 0) {424 struct kfd_process_device *pdd = container_of(attr,425 struct kfd_process_device,426 attr_evict);427 uint64_t evict_jiffies;428 429 evict_jiffies = atomic64_read(&pdd->evict_duration_counter);430 431 return snprintf(buffer,432 PAGE_SIZE,433 "%llu\n",434 jiffies64_to_msecs(evict_jiffies));435 436 /* Sysfs handle that gets CU occupancy is per device */437 } else if (strcmp(attr->name, "cu_occupancy") == 0) {438 return kfd_get_cu_occupancy(attr, buffer);439 } else {440 pr_err("Invalid attribute");441 }442 443 return 0;444}445 446static ssize_t kfd_sysfs_counters_show(struct kobject *kobj,447 struct attribute *attr, char *buf)448{449 struct kfd_process_device *pdd;450 451 if (!strcmp(attr->name, "faults")) {452 pdd = container_of(attr, struct kfd_process_device,453 attr_faults);454 return sysfs_emit(buf, "%llu\n", READ_ONCE(pdd->faults));455 }456 if (!strcmp(attr->name, "page_in")) {457 pdd = container_of(attr, struct kfd_process_device,458 attr_page_in);459 return sysfs_emit(buf, "%llu\n", READ_ONCE(pdd->page_in));460 }461 if (!strcmp(attr->name, "page_out")) {462 pdd = container_of(attr, struct kfd_process_device,463 attr_page_out);464 return sysfs_emit(buf, "%llu\n", READ_ONCE(pdd->page_out));465 }466 return 0;467}468 469static struct attribute attr_queue_size = {470 .name = "size",471 .mode = KFD_SYSFS_FILE_MODE472};473 474static struct attribute attr_queue_type = {475 .name = "type",476 .mode = KFD_SYSFS_FILE_MODE477};478 479static struct attribute attr_queue_gpuid = {480 .name = "gpuid",481 .mode = KFD_SYSFS_FILE_MODE482};483 484static struct attribute *procfs_queue_attrs[] = {485 &attr_queue_size,486 &attr_queue_type,487 &attr_queue_gpuid,488 NULL489};490ATTRIBUTE_GROUPS(procfs_queue);491 492static const struct sysfs_ops procfs_queue_ops = {493 .show = kfd_procfs_queue_show,494};495 496static const struct kobj_type procfs_queue_type = {497 .sysfs_ops = &procfs_queue_ops,498 .default_groups = procfs_queue_groups,499};500 501static const struct sysfs_ops procfs_stats_ops = {502 .show = kfd_procfs_stats_show,503};504 505static const struct kobj_type procfs_stats_type = {506 .sysfs_ops = &procfs_stats_ops,507 .release = kfd_procfs_kobj_release,508};509 510static const struct sysfs_ops sysfs_counters_ops = {511 .show = kfd_sysfs_counters_show,512};513 514static const struct kobj_type sysfs_counters_type = {515 .sysfs_ops = &sysfs_counters_ops,516 .release = kfd_procfs_kobj_release,517};518 519int kfd_procfs_add_queue(struct queue *q)520{521 struct kfd_process *proc;522 int ret;523 524 if (!q || !q->process)525 return -EINVAL;526 proc = q->process;527 528 /* Create proc/<pid>/queues/<queue id> folder */529 if (!proc->kobj_queues)530 return -EFAULT;531 ret = kobject_init_and_add(&q->kobj, &procfs_queue_type,532 proc->kobj_queues, "%u", q->properties.queue_id);533 if (ret < 0) {534 pr_warn("Creating proc/<pid>/queues/%u failed",535 q->properties.queue_id);536 kobject_put(&q->kobj);537 return ret;538 }539 540 return 0;541}542 543static void kfd_sysfs_create_file(struct kobject *kobj, struct attribute *attr,544 char *name)545{546 int ret;547 548 if (!kobj || !attr || !name)549 return;550 551 attr->name = name;552 attr->mode = KFD_SYSFS_FILE_MODE;553 sysfs_attr_init(attr);554 555 ret = sysfs_create_file(kobj, attr);556 if (ret)557 pr_warn("Create sysfs %s/%s failed %d", kobj->name, name, ret);558}559 560static void kfd_procfs_add_sysfs_stats(struct kfd_process *p)561{562 int ret;563 int i;564 char stats_dir_filename[MAX_SYSFS_FILENAME_LEN];565 566 if (!p || !p->kobj)567 return;568 569 /*570 * Create sysfs files for each GPU:571 * - proc/<pid>/stats_<gpuid>/572 * - proc/<pid>/stats_<gpuid>/evicted_ms573 * - proc/<pid>/stats_<gpuid>/cu_occupancy574 */575 for (i = 0; i < p->n_pdds; i++) {576 struct kfd_process_device *pdd = p->pdds[i];577 578 snprintf(stats_dir_filename, MAX_SYSFS_FILENAME_LEN,579 "stats_%u", pdd->dev->id);580 pdd->kobj_stats = kfd_alloc_struct(pdd->kobj_stats);581 if (!pdd->kobj_stats)582 return;583 584 ret = kobject_init_and_add(pdd->kobj_stats,585 &procfs_stats_type,586 p->kobj,587 stats_dir_filename);588 589 if (ret) {590 pr_warn("Creating KFD proc/stats_%s folder failed",591 stats_dir_filename);592 kobject_put(pdd->kobj_stats);593 pdd->kobj_stats = NULL;594 return;595 }596 597 kfd_sysfs_create_file(pdd->kobj_stats, &pdd->attr_evict,598 "evicted_ms");599 /* Add sysfs file to report compute unit occupancy */600 if (pdd->dev->kfd2kgd->get_cu_occupancy)601 kfd_sysfs_create_file(pdd->kobj_stats,602 &pdd->attr_cu_occupancy,603 "cu_occupancy");604 }605}606 607static void kfd_procfs_add_sysfs_counters(struct kfd_process *p)608{609 int ret = 0;610 int i;611 char counters_dir_filename[MAX_SYSFS_FILENAME_LEN];612 613 if (!p || !p->kobj)614 return;615 616 /*617 * Create sysfs files for each GPU which supports SVM618 * - proc/<pid>/counters_<gpuid>/619 * - proc/<pid>/counters_<gpuid>/faults620 * - proc/<pid>/counters_<gpuid>/page_in621 * - proc/<pid>/counters_<gpuid>/page_out622 */623 for_each_set_bit(i, p->svms.bitmap_supported, p->n_pdds) {624 struct kfd_process_device *pdd = p->pdds[i];625 struct kobject *kobj_counters;626 627 snprintf(counters_dir_filename, MAX_SYSFS_FILENAME_LEN,628 "counters_%u", pdd->dev->id);629 kobj_counters = kfd_alloc_struct(kobj_counters);630 if (!kobj_counters)631 return;632 633 ret = kobject_init_and_add(kobj_counters, &sysfs_counters_type,634 p->kobj, counters_dir_filename);635 if (ret) {636 pr_warn("Creating KFD proc/%s folder failed",637 counters_dir_filename);638 kobject_put(kobj_counters);639 return;640 }641 642 pdd->kobj_counters = kobj_counters;643 kfd_sysfs_create_file(kobj_counters, &pdd->attr_faults,644 "faults");645 kfd_sysfs_create_file(kobj_counters, &pdd->attr_page_in,646 "page_in");647 kfd_sysfs_create_file(kobj_counters, &pdd->attr_page_out,648 "page_out");649 }650}651 652static void kfd_procfs_add_sysfs_files(struct kfd_process *p)653{654 int i;655 656 if (!p || !p->kobj)657 return;658 659 /*660 * Create sysfs files for each GPU:661 * - proc/<pid>/vram_<gpuid>662 * - proc/<pid>/sdma_<gpuid>663 */664 for (i = 0; i < p->n_pdds; i++) {665 struct kfd_process_device *pdd = p->pdds[i];666 667 snprintf(pdd->vram_filename, MAX_SYSFS_FILENAME_LEN, "vram_%u",668 pdd->dev->id);669 kfd_sysfs_create_file(p->kobj, &pdd->attr_vram,670 pdd->vram_filename);671 672 snprintf(pdd->sdma_filename, MAX_SYSFS_FILENAME_LEN, "sdma_%u",673 pdd->dev->id);674 kfd_sysfs_create_file(p->kobj, &pdd->attr_sdma,675 pdd->sdma_filename);676 }677}678 679void kfd_procfs_del_queue(struct queue *q)680{681 if (!q)682 return;683 684 kobject_del(&q->kobj);685 kobject_put(&q->kobj);686}687 688int kfd_process_create_wq(void)689{690 if (!kfd_process_wq)691 kfd_process_wq = alloc_workqueue("kfd_process_wq", 0, 0);692 if (!kfd_restore_wq)693 kfd_restore_wq = alloc_ordered_workqueue("kfd_restore_wq",694 WQ_FREEZABLE);695 696 if (!kfd_process_wq || !kfd_restore_wq) {697 kfd_process_destroy_wq();698 return -ENOMEM;699 }700 701 return 0;702}703 704void kfd_process_destroy_wq(void)705{706 if (kfd_process_wq) {707 destroy_workqueue(kfd_process_wq);708 kfd_process_wq = NULL;709 }710 if (kfd_restore_wq) {711 destroy_workqueue(kfd_restore_wq);712 kfd_restore_wq = NULL;713 }714}715 716static void kfd_process_free_gpuvm(struct kgd_mem *mem,717 struct kfd_process_device *pdd, void **kptr)718{719 struct kfd_node *dev = pdd->dev;720 721 if (kptr && *kptr) {722 amdgpu_amdkfd_gpuvm_unmap_gtt_bo_from_kernel(mem);723 *kptr = NULL;724 }725 726 amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu(dev->adev, mem, pdd->drm_priv);727 amdgpu_amdkfd_gpuvm_free_memory_of_gpu(dev->adev, mem, pdd->drm_priv,728 NULL);729}730 731/* kfd_process_alloc_gpuvm - Allocate GPU VM for the KFD process732 * This function should be only called right after the process733 * is created and when kfd_processes_mutex is still being held734 * to avoid concurrency. Because of that exclusiveness, we do735 * not need to take p->mutex.736 */737static int kfd_process_alloc_gpuvm(struct kfd_process_device *pdd,738 uint64_t gpu_va, uint32_t size,739 uint32_t flags, struct kgd_mem **mem, void **kptr)740{741 struct kfd_node *kdev = pdd->dev;742 int err;743 744 err = amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu(kdev->adev, gpu_va, size,745 pdd->drm_priv, mem, NULL,746 flags, false);747 if (err)748 goto err_alloc_mem;749 750 err = amdgpu_amdkfd_gpuvm_map_memory_to_gpu(kdev->adev, *mem,751 pdd->drm_priv);752 if (err)753 goto err_map_mem;754 755 err = amdgpu_amdkfd_gpuvm_sync_memory(kdev->adev, *mem, true);756 if (err) {757 pr_debug("Sync memory failed, wait interrupted by user signal\n");758 goto sync_memory_failed;759 }760 761 if (kptr) {762 err = amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel(763 (struct kgd_mem *)*mem, kptr, NULL);764 if (err) {765 pr_debug("Map GTT BO to kernel failed\n");766 goto sync_memory_failed;767 }768 }769 770 return err;771 772sync_memory_failed:773 amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu(kdev->adev, *mem, pdd->drm_priv);774 775err_map_mem:776 amdgpu_amdkfd_gpuvm_free_memory_of_gpu(kdev->adev, *mem, pdd->drm_priv,777 NULL);778err_alloc_mem:779 *mem = NULL;780 *kptr = NULL;781 return err;782}783 784/* kfd_process_device_reserve_ib_mem - Reserve memory inside the785 * process for IB usage The memory reserved is for KFD to submit786 * IB to AMDGPU from kernel. If the memory is reserved787 * successfully, ib_kaddr will have the CPU/kernel788 * address. Check ib_kaddr before accessing the memory.789 */790static int kfd_process_device_reserve_ib_mem(struct kfd_process_device *pdd)791{792 struct qcm_process_device *qpd = &pdd->qpd;793 uint32_t flags = KFD_IOC_ALLOC_MEM_FLAGS_GTT |794 KFD_IOC_ALLOC_MEM_FLAGS_NO_SUBSTITUTE |795 KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE |796 KFD_IOC_ALLOC_MEM_FLAGS_EXECUTABLE;797 struct kgd_mem *mem;798 void *kaddr;799 int ret;800 801 if (qpd->ib_kaddr || !qpd->ib_base)802 return 0;803 804 /* ib_base is only set for dGPU */805 ret = kfd_process_alloc_gpuvm(pdd, qpd->ib_base, PAGE_SIZE, flags,806 &mem, &kaddr);807 if (ret)808 return ret;809 810 qpd->ib_mem = mem;811 qpd->ib_kaddr = kaddr;812 813 return 0;814}815 816static void kfd_process_device_destroy_ib_mem(struct kfd_process_device *pdd)817{818 struct qcm_process_device *qpd = &pdd->qpd;819 820 if (!qpd->ib_kaddr || !qpd->ib_base)821 return;822 823 kfd_process_free_gpuvm(qpd->ib_mem, pdd, &qpd->ib_kaddr);824}825 826struct kfd_process *kfd_create_process(struct task_struct *thread)827{828 struct kfd_process *process;829 int ret;830 831 if (!(thread->mm && mmget_not_zero(thread->mm)))832 return ERR_PTR(-EINVAL);833 834 /* Only the pthreads threading model is supported. */835 if (thread->group_leader->mm != thread->mm) {836 mmput(thread->mm);837 return ERR_PTR(-EINVAL);838 }839 840 /*841 * take kfd processes mutex before starting of process creation842 * so there won't be a case where two threads of the same process843 * create two kfd_process structures844 */845 mutex_lock(&kfd_processes_mutex);846 847 if (kfd_is_locked()) {848 pr_debug("KFD is locked! Cannot create process");849 process = ERR_PTR(-EINVAL);850 goto out;851 }852 853 /* A prior open of /dev/kfd could have already created the process. */854 process = find_process(thread, false);855 if (process) {856 pr_debug("Process already found\n");857 } else {858 /* If the process just called exec(3), it is possible that the859 * cleanup of the kfd_process (following the release of the mm860 * of the old process image) is still in the cleanup work queue.861 * Make sure to drain any job before trying to recreate any862 * resource for this process.863 */864 flush_workqueue(kfd_process_wq);865 866 process = create_process(thread);867 if (IS_ERR(process))868 goto out;869 870 if (!procfs.kobj)871 goto out;872 873 process->kobj = kfd_alloc_struct(process->kobj);874 if (!process->kobj) {875 pr_warn("Creating procfs kobject failed");876 goto out;877 }878 ret = kobject_init_and_add(process->kobj, &procfs_type,879 procfs.kobj, "%d",880 (int)process->lead_thread->pid);881 if (ret) {882 pr_warn("Creating procfs pid directory failed");883 kobject_put(process->kobj);884 goto out;885 }886 887 kfd_sysfs_create_file(process->kobj, &process->attr_pasid,888 "pasid");889 890 process->kobj_queues = kobject_create_and_add("queues",891 process->kobj);892 if (!process->kobj_queues)893 pr_warn("Creating KFD proc/queues folder failed");894 895 kfd_procfs_add_sysfs_stats(process);896 kfd_procfs_add_sysfs_files(process);897 kfd_procfs_add_sysfs_counters(process);898 899 init_waitqueue_head(&process->wait_irq_drain);900 }901out:902 if (!IS_ERR(process))903 kref_get(&process->ref);904 mutex_unlock(&kfd_processes_mutex);905 mmput(thread->mm);906 907 return process;908}909 910struct kfd_process *kfd_get_process(const struct task_struct *thread)911{912 struct kfd_process *process;913 914 if (!thread->mm)915 return ERR_PTR(-EINVAL);916 917 /* Only the pthreads threading model is supported. */918 if (thread->group_leader->mm != thread->mm)919 return ERR_PTR(-EINVAL);920 921 process = find_process(thread, false);922 if (!process)923 return ERR_PTR(-EINVAL);924 925 return process;926}927 928static struct kfd_process *find_process_by_mm(const struct mm_struct *mm)929{930 struct kfd_process *process;931 932 hash_for_each_possible_rcu(kfd_processes_table, process,933 kfd_processes, (uintptr_t)mm)934 if (process->mm == mm)935 return process;936 937 return NULL;938}939 940static struct kfd_process *find_process(const struct task_struct *thread,941 bool ref)942{943 struct kfd_process *p;944 int idx;945 946 idx = srcu_read_lock(&kfd_processes_srcu);947 p = find_process_by_mm(thread->mm);948 if (p && ref)949 kref_get(&p->ref);950 srcu_read_unlock(&kfd_processes_srcu, idx);951 952 return p;953}954 955void kfd_unref_process(struct kfd_process *p)956{957 kref_put(&p->ref, kfd_process_ref_release);958}959 960/* This increments the process->ref counter. */961struct kfd_process *kfd_lookup_process_by_pid(struct pid *pid)962{963 struct task_struct *task = NULL;964 struct kfd_process *p = NULL;965 966 if (!pid) {967 task = current;968 get_task_struct(task);969 } else {970 task = get_pid_task(pid, PIDTYPE_PID);971 }972 973 if (task) {974 p = find_process(task, true);975 put_task_struct(task);976 }977 978 return p;979}980 981static void kfd_process_device_free_bos(struct kfd_process_device *pdd)982{983 struct kfd_process *p = pdd->process;984 void *mem;985 int id;986 int i;987 988 /*989 * Remove all handles from idr and release appropriate990 * local memory object991 */992 idr_for_each_entry(&pdd->alloc_idr, mem, id) {993 994 for (i = 0; i < p->n_pdds; i++) {995 struct kfd_process_device *peer_pdd = p->pdds[i];996 997 if (!peer_pdd->drm_priv)998 continue;999 amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu(1000 peer_pdd->dev->adev, mem, peer_pdd->drm_priv);1001 }1002 1003 amdgpu_amdkfd_gpuvm_free_memory_of_gpu(pdd->dev->adev, mem,1004 pdd->drm_priv, NULL);1005 kfd_process_device_remove_obj_handle(pdd, id);1006 }1007}1008 1009/*1010 * Just kunmap and unpin signal BO here. It will be freed in1011 * kfd_process_free_outstanding_kfd_bos()1012 */1013static void kfd_process_kunmap_signal_bo(struct kfd_process *p)1014{1015 struct kfd_process_device *pdd;1016 struct kfd_node *kdev;1017 void *mem;1018 1019 kdev = kfd_device_by_id(GET_GPU_ID(p->signal_handle));1020 if (!kdev)1021 return;1022 1023 mutex_lock(&p->mutex);1024 1025 pdd = kfd_get_process_device_data(kdev, p);1026 if (!pdd)1027 goto out;1028 1029 mem = kfd_process_device_translate_handle(1030 pdd, GET_IDR_HANDLE(p->signal_handle));1031 if (!mem)1032 goto out;1033 1034 amdgpu_amdkfd_gpuvm_unmap_gtt_bo_from_kernel(mem);1035 1036out:1037 mutex_unlock(&p->mutex);1038}1039 1040static void kfd_process_free_outstanding_kfd_bos(struct kfd_process *p)1041{1042 int i;1043 1044 for (i = 0; i < p->n_pdds; i++)1045 kfd_process_device_free_bos(p->pdds[i]);1046}1047 1048static void kfd_process_destroy_pdds(struct kfd_process *p)1049{1050 int i;1051 1052 for (i = 0; i < p->n_pdds; i++) {1053 struct kfd_process_device *pdd = p->pdds[i];1054 1055 pr_debug("Releasing pdd (topology id %d) for process (pasid 0x%x)\n",1056 pdd->dev->id, p->pasid);1057 1058 kfd_process_device_destroy_cwsr_dgpu(pdd);1059 kfd_process_device_destroy_ib_mem(pdd);1060 1061 if (pdd->drm_file) {1062 amdgpu_amdkfd_gpuvm_release_process_vm(1063 pdd->dev->adev, pdd->drm_priv);1064 fput(pdd->drm_file);1065 }1066 1067 if (pdd->qpd.cwsr_kaddr && !pdd->qpd.cwsr_base)1068 free_pages((unsigned long)pdd->qpd.cwsr_kaddr,1069 get_order(KFD_CWSR_TBA_TMA_SIZE));1070 1071 idr_destroy(&pdd->alloc_idr);1072 1073 kfd_free_process_doorbells(pdd->dev->kfd, pdd);1074 1075 if (pdd->dev->kfd->shared_resources.enable_mes)1076 amdgpu_amdkfd_free_gtt_mem(pdd->dev->adev,1077 &pdd->proc_ctx_bo);1078 /*1079 * before destroying pdd, make sure to report availability1080 * for auto suspend1081 */1082 if (pdd->runtime_inuse) {1083 pm_runtime_mark_last_busy(adev_to_drm(pdd->dev->adev)->dev);1084 pm_runtime_put_autosuspend(adev_to_drm(pdd->dev->adev)->dev);1085 pdd->runtime_inuse = false;1086 }1087 1088 kfree(pdd);1089 p->pdds[i] = NULL;1090 }1091 p->n_pdds = 0;1092}1093 1094static void kfd_process_remove_sysfs(struct kfd_process *p)1095{1096 struct kfd_process_device *pdd;1097 int i;1098 1099 if (!p->kobj)1100 return;1101 1102 sysfs_remove_file(p->kobj, &p->attr_pasid);1103 kobject_del(p->kobj_queues);1104 kobject_put(p->kobj_queues);1105 p->kobj_queues = NULL;1106 1107 for (i = 0; i < p->n_pdds; i++) {1108 pdd = p->pdds[i];1109 1110 sysfs_remove_file(p->kobj, &pdd->attr_vram);1111 sysfs_remove_file(p->kobj, &pdd->attr_sdma);1112 1113 sysfs_remove_file(pdd->kobj_stats, &pdd->attr_evict);1114 if (pdd->dev->kfd2kgd->get_cu_occupancy)1115 sysfs_remove_file(pdd->kobj_stats,1116 &pdd->attr_cu_occupancy);1117 kobject_del(pdd->kobj_stats);1118 kobject_put(pdd->kobj_stats);1119 pdd->kobj_stats = NULL;1120 }1121 1122 for_each_set_bit(i, p->svms.bitmap_supported, p->n_pdds) {1123 pdd = p->pdds[i];1124 1125 sysfs_remove_file(pdd->kobj_counters, &pdd->attr_faults);1126 sysfs_remove_file(pdd->kobj_counters, &pdd->attr_page_in);1127 sysfs_remove_file(pdd->kobj_counters, &pdd->attr_page_out);1128 kobject_del(pdd->kobj_counters);1129 kobject_put(pdd->kobj_counters);1130 pdd->kobj_counters = NULL;1131 }1132 1133 kobject_del(p->kobj);1134 kobject_put(p->kobj);1135 p->kobj = NULL;1136}1137 1138/* No process locking is needed in this function, because the process1139 * is not findable any more. We must assume that no other thread is1140 * using it any more, otherwise we couldn't safely free the process1141 * structure in the end.1142 */1143static void kfd_process_wq_release(struct work_struct *work)1144{1145 struct kfd_process *p = container_of(work, struct kfd_process,1146 release_work);1147 struct dma_fence *ef;1148 1149 kfd_process_dequeue_from_all_devices(p);1150 pqm_uninit(&p->pqm);1151 1152 /* Signal the eviction fence after user mode queues are1153 * destroyed. This allows any BOs to be freed without1154 * triggering pointless evictions or waiting for fences.1155 */1156 synchronize_rcu();1157 ef = rcu_access_pointer(p->ef);1158 dma_fence_signal(ef);1159 1160 kfd_process_remove_sysfs(p);1161 1162 kfd_process_kunmap_signal_bo(p);1163 kfd_process_free_outstanding_kfd_bos(p);1164 svm_range_list_fini(p);1165 1166 kfd_process_destroy_pdds(p);1167 dma_fence_put(ef);1168 1169 kfd_event_free_process(p);1170 1171 kfd_pasid_free(p->pasid);1172 mutex_destroy(&p->mutex);1173 1174 put_task_struct(p->lead_thread);1175 1176 kfree(p);1177}1178 1179static void kfd_process_ref_release(struct kref *ref)1180{1181 struct kfd_process *p = container_of(ref, struct kfd_process, ref);1182 1183 INIT_WORK(&p->release_work, kfd_process_wq_release);1184 queue_work(kfd_process_wq, &p->release_work);1185}1186 1187static struct mmu_notifier *kfd_process_alloc_notifier(struct mm_struct *mm)1188{1189 int idx = srcu_read_lock(&kfd_processes_srcu);1190 struct kfd_process *p = find_process_by_mm(mm);1191 1192 srcu_read_unlock(&kfd_processes_srcu, idx);1193 1194 return p ? &p->mmu_notifier : ERR_PTR(-ESRCH);1195}1196 1197static void kfd_process_free_notifier(struct mmu_notifier *mn)1198{1199 kfd_unref_process(container_of(mn, struct kfd_process, mmu_notifier));1200}1201 1202static void kfd_process_notifier_release_internal(struct kfd_process *p)1203{1204 int i;1205 1206 cancel_delayed_work_sync(&p->eviction_work);1207 cancel_delayed_work_sync(&p->restore_work);1208 1209 for (i = 0; i < p->n_pdds; i++) {1210 struct kfd_process_device *pdd = p->pdds[i];1211 1212 /* re-enable GFX OFF since runtime enable with ttmp setup disabled it. */1213 if (!kfd_dbg_is_rlc_restore_supported(pdd->dev) && p->runtime_info.ttmp_setup)1214 amdgpu_gfx_off_ctrl(pdd->dev->adev, true);1215 }1216 1217 /* Indicate to other users that MM is no longer valid */1218 p->mm = NULL;1219 kfd_dbg_trap_disable(p);1220 1221 if (atomic_read(&p->debugged_process_count) > 0) {1222 struct kfd_process *target;1223 unsigned int temp;1224 int idx = srcu_read_lock(&kfd_processes_srcu);1225 1226 hash_for_each_rcu(kfd_processes_table, temp, target, kfd_processes) {1227 if (target->debugger_process && target->debugger_process == p) {1228 mutex_lock_nested(&target->mutex, 1);1229 kfd_dbg_trap_disable(target);1230 mutex_unlock(&target->mutex);1231 if (atomic_read(&p->debugged_process_count) == 0)1232 break;1233 }1234 }1235 1236 srcu_read_unlock(&kfd_processes_srcu, idx);1237 }1238 1239 mmu_notifier_put(&p->mmu_notifier);1240}1241 1242static void kfd_process_notifier_release(struct mmu_notifier *mn,1243 struct mm_struct *mm)1244{1245 struct kfd_process *p;1246 1247 /*1248 * The kfd_process structure can not be free because the1249 * mmu_notifier srcu is read locked1250 */1251 p = container_of(mn, struct kfd_process, mmu_notifier);1252 if (WARN_ON(p->mm != mm))1253 return;1254 1255 mutex_lock(&kfd_processes_mutex);1256 /*1257 * Do early return if table is empty.1258 *1259 * This could potentially happen if this function is called concurrently1260 * by mmu_notifier and by kfd_cleanup_pocesses.1261 *1262 */1263 if (hash_empty(kfd_processes_table)) {1264 mutex_unlock(&kfd_processes_mutex);1265 return;1266 }1267 hash_del_rcu(&p->kfd_processes);1268 mutex_unlock(&kfd_processes_mutex);1269 synchronize_srcu(&kfd_processes_srcu);1270 1271 kfd_process_notifier_release_internal(p);1272}1273 1274static const struct mmu_notifier_ops kfd_process_mmu_notifier_ops = {1275 .release = kfd_process_notifier_release,1276 .alloc_notifier = kfd_process_alloc_notifier,1277 .free_notifier = kfd_process_free_notifier,1278};1279 1280/*1281 * This code handles the case when driver is being unloaded before all1282 * mm_struct are released. We need to safely free the kfd_process and1283 * avoid race conditions with mmu_notifier that might try to free them.1284 *1285 */1286void kfd_cleanup_processes(void)1287{1288 struct kfd_process *p;1289 struct hlist_node *p_temp;1290 unsigned int temp;1291 HLIST_HEAD(cleanup_list);1292 1293 /*1294 * Move all remaining kfd_process from the process table to a1295 * temp list for processing. Once done, callback from mmu_notifier1296 * release will not see the kfd_process in the table and do early return,1297 * avoiding double free issues.1298 */1299 mutex_lock(&kfd_processes_mutex);1300 hash_for_each_safe(kfd_processes_table, temp, p_temp, p, kfd_processes) {1301 hash_del_rcu(&p->kfd_processes);1302 synchronize_srcu(&kfd_processes_srcu);1303 hlist_add_head(&p->kfd_processes, &cleanup_list);1304 }1305 mutex_unlock(&kfd_processes_mutex);1306 1307 hlist_for_each_entry_safe(p, p_temp, &cleanup_list, kfd_processes)1308 kfd_process_notifier_release_internal(p);1309 1310 /*1311 * Ensures that all outstanding free_notifier get called, triggering1312 * the release of the kfd_process struct.1313 */1314 mmu_notifier_synchronize();1315}1316 1317int kfd_process_init_cwsr_apu(struct kfd_process *p, struct file *filep)1318{1319 unsigned long offset;1320 int i;1321 1322 if (p->has_cwsr)1323 return 0;1324 1325 for (i = 0; i < p->n_pdds; i++) {1326 struct kfd_node *dev = p->pdds[i]->dev;1327 struct qcm_process_device *qpd = &p->pdds[i]->qpd;1328 1329 if (!dev->kfd->cwsr_enabled || qpd->cwsr_kaddr || qpd->cwsr_base)1330 continue;1331 1332 offset = KFD_MMAP_TYPE_RESERVED_MEM | KFD_MMAP_GPU_ID(dev->id);1333 qpd->tba_addr = (int64_t)vm_mmap(filep, 0,1334 KFD_CWSR_TBA_TMA_SIZE, PROT_READ | PROT_EXEC,1335 MAP_SHARED, offset);1336 1337 if (IS_ERR_VALUE(qpd->tba_addr)) {1338 int err = qpd->tba_addr;1339 1340 dev_err(dev->adev->dev,1341 "Failure to set tba address. error %d.\n", err);1342 qpd->tba_addr = 0;1343 qpd->cwsr_kaddr = NULL;1344 return err;1345 }1346 1347 memcpy(qpd->cwsr_kaddr, dev->kfd->cwsr_isa, dev->kfd->cwsr_isa_size);1348 1349 kfd_process_set_trap_debug_flag(qpd, p->debug_trap_enabled);1350 1351 qpd->tma_addr = qpd->tba_addr + KFD_CWSR_TMA_OFFSET;1352 pr_debug("set tba :0x%llx, tma:0x%llx, cwsr_kaddr:%p for pqm.\n",1353 qpd->tba_addr, qpd->tma_addr, qpd->cwsr_kaddr);1354 }1355 1356 p->has_cwsr = true;1357 1358 return 0;1359}1360 1361static int kfd_process_device_init_cwsr_dgpu(struct kfd_process_device *pdd)1362{1363 struct kfd_node *dev = pdd->dev;1364 struct qcm_process_device *qpd = &pdd->qpd;1365 uint32_t flags = KFD_IOC_ALLOC_MEM_FLAGS_GTT1366 | KFD_IOC_ALLOC_MEM_FLAGS_NO_SUBSTITUTE1367 | KFD_IOC_ALLOC_MEM_FLAGS_EXECUTABLE;1368 struct kgd_mem *mem;1369 void *kaddr;1370 int ret;1371 1372 if (!dev->kfd->cwsr_enabled || qpd->cwsr_kaddr || !qpd->cwsr_base)1373 return 0;1374 1375 /* cwsr_base is only set for dGPU */1376 ret = kfd_process_alloc_gpuvm(pdd, qpd->cwsr_base,1377 KFD_CWSR_TBA_TMA_SIZE, flags, &mem, &kaddr);1378 if (ret)1379 return ret;1380 1381 qpd->cwsr_mem = mem;1382 qpd->cwsr_kaddr = kaddr;1383 qpd->tba_addr = qpd->cwsr_base;1384 1385 memcpy(qpd->cwsr_kaddr, dev->kfd->cwsr_isa, dev->kfd->cwsr_isa_size);1386 1387 kfd_process_set_trap_debug_flag(&pdd->qpd,1388 pdd->process->debug_trap_enabled);1389 1390 qpd->tma_addr = qpd->tba_addr + KFD_CWSR_TMA_OFFSET;1391 pr_debug("set tba :0x%llx, tma:0x%llx, cwsr_kaddr:%p for pqm.\n",1392 qpd->tba_addr, qpd->tma_addr, qpd->cwsr_kaddr);1393 1394 return 0;1395}1396 1397static void kfd_process_device_destroy_cwsr_dgpu(struct kfd_process_device *pdd)1398{1399 struct kfd_node *dev = pdd->dev;1400 struct qcm_process_device *qpd = &pdd->qpd;1401 1402 if (!dev->kfd->cwsr_enabled || !qpd->cwsr_kaddr || !qpd->cwsr_base)1403 return;1404 1405 kfd_process_free_gpuvm(qpd->cwsr_mem, pdd, &qpd->cwsr_kaddr);1406}1407 1408void kfd_process_set_trap_handler(struct qcm_process_device *qpd,1409 uint64_t tba_addr,1410 uint64_t tma_addr)1411{1412 if (qpd->cwsr_kaddr) {1413 /* KFD trap handler is bound, record as second-level TBA/TMA1414 * in first-level TMA. First-level trap will jump to second.1415 */1416 uint64_t *tma =1417 (uint64_t *)(qpd->cwsr_kaddr + KFD_CWSR_TMA_OFFSET);1418 tma[0] = tba_addr;1419 tma[1] = tma_addr;1420 } else {1421 /* No trap handler bound, bind as first-level TBA/TMA. */1422 qpd->tba_addr = tba_addr;1423 qpd->tma_addr = tma_addr;1424 }1425}1426 1427bool kfd_process_xnack_mode(struct kfd_process *p, bool supported)1428{1429 int i;1430 1431 /* On most GFXv9 GPUs, the retry mode in the SQ must match the1432 * boot time retry setting. Mixing processes with different1433 * XNACK/retry settings can hang the GPU.1434 *1435 * Different GPUs can have different noretry settings depending1436 * on HW bugs or limitations. We need to find at least one1437 * XNACK mode for this process that's compatible with all GPUs.1438 * Fortunately GPUs with retry enabled (noretry=0) can run code1439 * built for XNACK-off. On GFXv9 it may perform slower.1440 *1441 * Therefore applications built for XNACK-off can always be1442 * supported and will be our fallback if any GPU does not1443 * support retry.1444 */1445 for (i = 0; i < p->n_pdds; i++) {1446 struct kfd_node *dev = p->pdds[i]->dev;1447 1448 /* Only consider GFXv9 and higher GPUs. Older GPUs don't1449 * support the SVM APIs and don't need to be considered1450 * for the XNACK mode selection.1451 */1452 if (!KFD_IS_SOC15(dev))1453 continue;1454 /* Aldebaran can always support XNACK because it can support1455 * per-process XNACK mode selection. But let the dev->noretry1456 * setting still influence the default XNACK mode.1457 */1458 if (supported && KFD_SUPPORT_XNACK_PER_PROCESS(dev)) {1459 if (!amdgpu_sriov_xnack_support(dev->kfd->adev)) {1460 pr_debug("SRIOV platform xnack not supported\n");1461 return false;1462 }1463 continue;1464 }1465 1466 /* GFXv10 and later GPUs do not support shader preemption1467 * during page faults. This can lead to poor QoS for queue1468 * management and memory-manager-related preemptions or1469 * even deadlocks.1470 */1471 if (KFD_GC_VERSION(dev) >= IP_VERSION(10, 1, 1))1472 return false;1473 1474 if (dev->kfd->noretry)1475 return false;1476 }1477 1478 return true;1479}1480 1481void kfd_process_set_trap_debug_flag(struct qcm_process_device *qpd,1482 bool enabled)1483{1484 if (qpd->cwsr_kaddr) {1485 uint64_t *tma =1486 (uint64_t *)(qpd->cwsr_kaddr + KFD_CWSR_TMA_OFFSET);1487 tma[2] = enabled;1488 }1489}1490 1491/*1492 * On return the kfd_process is fully operational and will be freed when the1493 * mm is released1494 */1495static struct kfd_process *create_process(const struct task_struct *thread)1496{1497 struct kfd_process *process;1498 struct mmu_notifier *mn;1499 int err = -ENOMEM;1500 1501 process = kzalloc(sizeof(*process), GFP_KERNEL);1502 if (!process)1503 goto err_alloc_process;1504 1505 kref_init(&process->ref);1506 mutex_init(&process->mutex);1507 process->mm = thread->mm;1508 process->lead_thread = thread->group_leader;1509 process->n_pdds = 0;1510 process->queues_paused = false;1511 INIT_DELAYED_WORK(&process->eviction_work, evict_process_worker);1512 INIT_DELAYED_WORK(&process->restore_work, restore_process_worker);1513 process->last_restore_timestamp = get_jiffies_64();1514 err = kfd_event_init_process(process);1515 if (err)1516 goto err_event_init;1517 process->is_32bit_user_mode = in_compat_syscall();1518 process->debug_trap_enabled = false;1519 process->debugger_process = NULL;1520 process->exception_enable_mask = 0;1521 atomic_set(&process->debugged_process_count, 0);1522 sema_init(&process->runtime_enable_sema, 0);1523 1524 process->pasid = kfd_pasid_alloc();1525 if (process->pasid == 0) {1526 err = -ENOSPC;1527 goto err_alloc_pasid;1528 }1529 1530 err = pqm_init(&process->pqm, process);1531 if (err != 0)1532 goto err_process_pqm_init;1533 1534 /* init process apertures*/1535 err = kfd_init_apertures(process);1536 if (err != 0)1537 goto err_init_apertures;1538 1539 /* Check XNACK support after PDDs are created in kfd_init_apertures */1540 process->xnack_enabled = kfd_process_xnack_mode(process, false);1541 1542 err = svm_range_list_init(process);1543 if (err)1544 goto err_init_svm_range_list;1545 1546 /* alloc_notifier needs to find the process in the hash table */1547 hash_add_rcu(kfd_processes_table, &process->kfd_processes,1548 (uintptr_t)process->mm);1549 1550 /* Avoid free_notifier to start kfd_process_wq_release if1551 * mmu_notifier_get failed because of pending signal.1552 */1553 kref_get(&process->ref);1554 1555 /* MMU notifier registration must be the last call that can fail1556 * because after this point we cannot unwind the process creation.1557 * After this point, mmu_notifier_put will trigger the cleanup by1558 * dropping the last process reference in the free_notifier.1559 */1560 mn = mmu_notifier_get(&kfd_process_mmu_notifier_ops, process->mm);1561 if (IS_ERR(mn)) {1562 err = PTR_ERR(mn);1563 goto err_register_notifier;1564 }1565 BUG_ON(mn != &process->mmu_notifier);1566 1567 kfd_unref_process(process);1568 get_task_struct(process->lead_thread);1569 1570 INIT_WORK(&process->debug_event_workarea, debug_event_write_work_handler);1571 1572 return process;1573 1574err_register_notifier:1575 hash_del_rcu(&process->kfd_processes);1576 svm_range_list_fini(process);1577err_init_svm_range_list:1578 kfd_process_free_outstanding_kfd_bos(process);1579 kfd_process_destroy_pdds(process);1580err_init_apertures:1581 pqm_uninit(&process->pqm);1582err_process_pqm_init:1583 kfd_pasid_free(process->pasid);1584err_alloc_pasid:1585 kfd_event_free_process(process);1586err_event_init:1587 mutex_destroy(&process->mutex);1588 kfree(process);1589err_alloc_process:1590 return ERR_PTR(err);1591}1592 1593struct kfd_process_device *kfd_get_process_device_data(struct kfd_node *dev,1594 struct kfd_process *p)1595{1596 int i;1597 1598 for (i = 0; i < p->n_pdds; i++)1599 if (p->pdds[i]->dev == dev)1600 return p->pdds[i];1601 1602 return NULL;1603}1604 1605struct kfd_process_device *kfd_create_process_device_data(struct kfd_node *dev,1606 struct kfd_process *p)1607{1608 struct kfd_process_device *pdd = NULL;1609 int retval = 0;1610 1611 if (WARN_ON_ONCE(p->n_pdds >= MAX_GPU_INSTANCE))1612 return NULL;1613 pdd = kzalloc(sizeof(*pdd), GFP_KERNEL);1614 if (!pdd)1615 return NULL;1616 1617 pdd->dev = dev;1618 INIT_LIST_HEAD(&pdd->qpd.queues_list);1619 INIT_LIST_HEAD(&pdd->qpd.priv_queue_list);1620 pdd->qpd.dqm = dev->dqm;1621 pdd->qpd.pqm = &p->pqm;1622 pdd->qpd.evicted = 0;1623 pdd->qpd.mapped_gws_queue = false;1624 pdd->process = p;1625 pdd->bound = PDD_UNBOUND;1626 pdd->already_dequeued = false;1627 pdd->runtime_inuse = false;1628 atomic64_set(&pdd->vram_usage, 0);1629 pdd->sdma_past_activity_counter = 0;1630 pdd->user_gpu_id = dev->id;1631 atomic64_set(&pdd->evict_duration_counter, 0);1632 1633 if (dev->kfd->shared_resources.enable_mes) {1634 retval = amdgpu_amdkfd_alloc_gtt_mem(dev->adev,1635 AMDGPU_MES_PROC_CTX_SIZE,1636 &pdd->proc_ctx_bo,1637 &pdd->proc_ctx_gpu_addr,1638 &pdd->proc_ctx_cpu_ptr,1639 false);1640 if (retval) {1641 dev_err(dev->adev->dev,1642 "failed to allocate process context bo\n");1643 goto err_free_pdd;1644 }1645 memset(pdd->proc_ctx_cpu_ptr, 0, AMDGPU_MES_PROC_CTX_SIZE);1646 }1647 1648 p->pdds[p->n_pdds++] = pdd;1649 if (kfd_dbg_is_per_vmid_supported(pdd->dev))1650 pdd->spi_dbg_override = pdd->dev->kfd2kgd->disable_debug_trap(1651 pdd->dev->adev,1652 false,1653 0);1654 1655 /* Init idr used for memory handle translation */1656 idr_init(&pdd->alloc_idr);1657 1658 return pdd;1659 1660err_free_pdd:1661 kfree(pdd);1662 return NULL;1663}1664 1665/**1666 * kfd_process_device_init_vm - Initialize a VM for a process-device1667 *1668 * @pdd: The process-device1669 * @drm_file: Optional pointer to a DRM file descriptor1670 *1671 * If @drm_file is specified, it will be used to acquire the VM from1672 * that file descriptor. If successful, the @pdd takes ownership of1673 * the file descriptor.1674 *1675 * If @drm_file is NULL, a new VM is created.1676 *1677 * Returns 0 on success, -errno on failure.1678 */1679int kfd_process_device_init_vm(struct kfd_process_device *pdd,1680 struct file *drm_file)1681{1682 struct amdgpu_fpriv *drv_priv;1683 struct amdgpu_vm *avm;1684 struct kfd_process *p;1685 struct dma_fence *ef;1686 struct kfd_node *dev;1687 int ret;1688 1689 if (!drm_file)1690 return -EINVAL;1691 1692 if (pdd->drm_priv)1693 return -EBUSY;1694 1695 ret = amdgpu_file_to_fpriv(drm_file, &drv_priv);1696 if (ret)1697 return ret;1698 avm = &drv_priv->vm;1699 1700 p = pdd->process;1701 dev = pdd->dev;1702 1703 ret = amdgpu_amdkfd_gpuvm_acquire_process_vm(dev->adev, avm,1704 &p->kgd_process_info,1705 p->ef ? NULL : &ef);1706 if (ret) {1707 dev_err(dev->adev->dev, "Failed to create process VM object\n");1708 return ret;1709 }1710 1711 if (!p->ef)1712 RCU_INIT_POINTER(p->ef, ef);1713 1714 pdd->drm_priv = drm_file->private_data;1715 1716 ret = kfd_process_device_reserve_ib_mem(pdd);1717 if (ret)1718 goto err_reserve_ib_mem;1719 ret = kfd_process_device_init_cwsr_dgpu(pdd);1720 if (ret)1721 goto err_init_cwsr;1722 1723 ret = amdgpu_amdkfd_gpuvm_set_vm_pasid(dev->adev, avm, p->pasid);1724 if (ret)1725 goto err_set_pasid;1726 1727 pdd->drm_file = drm_file;1728 1729 return 0;1730 1731err_set_pasid:1732 kfd_process_device_destroy_cwsr_dgpu(pdd);1733err_init_cwsr:1734 kfd_process_device_destroy_ib_mem(pdd);1735err_reserve_ib_mem:1736 pdd->drm_priv = NULL;1737 amdgpu_amdkfd_gpuvm_destroy_cb(dev->adev, avm);1738 1739 return ret;1740}1741 1742/*1743 * Direct the IOMMU to bind the process (specifically the pasid->mm)1744 * to the device.1745 * Unbinding occurs when the process dies or the device is removed.1746 *1747 * Assumes that the process lock is held.1748 */1749struct kfd_process_device *kfd_bind_process_to_device(struct kfd_node *dev,1750 struct kfd_process *p)1751{1752 struct kfd_process_device *pdd;1753 int err;1754 1755 pdd = kfd_get_process_device_data(dev, p);1756 if (!pdd) {1757 dev_err(dev->adev->dev, "Process device data doesn't exist\n");1758 return ERR_PTR(-ENOMEM);1759 }1760 1761 if (!pdd->drm_priv)1762 return ERR_PTR(-ENODEV);1763 1764 /*1765 * signal runtime-pm system to auto resume and prevent1766 * further runtime suspend once device pdd is created until1767 * pdd is destroyed.1768 */1769 if (!pdd->runtime_inuse) {1770 err = pm_runtime_get_sync(adev_to_drm(dev->adev)->dev);1771 if (err < 0) {1772 pm_runtime_put_autosuspend(adev_to_drm(dev->adev)->dev);1773 return ERR_PTR(err);1774 }1775 }1776 1777 /*1778 * make sure that runtime_usage counter is incremented just once1779 * per pdd1780 */1781 pdd->runtime_inuse = true;1782 1783 return pdd;1784}1785 1786/* Create specific handle mapped to mem from process local memory idr1787 * Assumes that the process lock is held.1788 */1789int kfd_process_device_create_obj_handle(struct kfd_process_device *pdd,1790 void *mem)1791{1792 return idr_alloc(&pdd->alloc_idr, mem, 0, 0, GFP_KERNEL);1793}1794 1795/* Translate specific handle from process local memory idr1796 * Assumes that the process lock is held.1797 */1798void *kfd_process_device_translate_handle(struct kfd_process_device *pdd,1799 int handle)1800{1801 if (handle < 0)1802 return NULL;1803 1804 return idr_find(&pdd->alloc_idr, handle);1805}1806 1807/* Remove specific handle from process local memory idr1808 * Assumes that the process lock is held.1809 */1810void kfd_process_device_remove_obj_handle(struct kfd_process_device *pdd,1811 int handle)1812{1813 if (handle >= 0)1814 idr_remove(&pdd->alloc_idr, handle);1815}1816 1817/* This increments the process->ref counter. */1818struct kfd_process *kfd_lookup_process_by_pasid(u32 pasid)1819{1820 struct kfd_process *p, *ret_p = NULL;1821 unsigned int temp;1822 1823 int idx = srcu_read_lock(&kfd_processes_srcu);1824 1825 hash_for_each_rcu(kfd_processes_table, temp, p, kfd_processes) {1826 if (p->pasid == pasid) {1827 kref_get(&p->ref);1828 ret_p = p;1829 break;1830 }1831 }1832 1833 srcu_read_unlock(&kfd_processes_srcu, idx);1834 1835 return ret_p;1836}1837 1838/* This increments the process->ref counter. */1839struct kfd_process *kfd_lookup_process_by_mm(const struct mm_struct *mm)1840{1841 struct kfd_process *p;1842 1843 int idx = srcu_read_lock(&kfd_processes_srcu);1844 1845 p = find_process_by_mm(mm);1846 if (p)1847 kref_get(&p->ref);1848 1849 srcu_read_unlock(&kfd_processes_srcu, idx);1850 1851 return p;1852}1853 1854/* kfd_process_evict_queues - Evict all user queues of a process1855 *1856 * Eviction is reference-counted per process-device. This means multiple1857 * evictions from different sources can be nested safely.1858 */1859int kfd_process_evict_queues(struct kfd_process *p, uint32_t trigger)1860{1861 int r = 0;1862 int i;1863 unsigned int n_evicted = 0;1864 1865 for (i = 0; i < p->n_pdds; i++) {1866 struct kfd_process_device *pdd = p->pdds[i];1867 struct device *dev = pdd->dev->adev->dev;1868 1869 kfd_smi_event_queue_eviction(pdd->dev, p->lead_thread->pid,1870 trigger);1871 1872 r = pdd->dev->dqm->ops.evict_process_queues(pdd->dev->dqm,1873 &pdd->qpd);1874 /* evict return -EIO if HWS is hang or asic is resetting, in this case1875 * we would like to set all the queues to be in evicted state to prevent1876 * them been add back since they actually not be saved right now.1877 */1878 if (r && r != -EIO) {1879 dev_err(dev, "Failed to evict process queues\n");1880 goto fail;1881 }1882 n_evicted++;1883 1884 pdd->dev->dqm->is_hws_hang = false;1885 }1886 1887 return r;1888 1889fail:1890 /* To keep state consistent, roll back partial eviction by1891 * restoring queues1892 */1893 for (i = 0; i < p->n_pdds; i++) {1894 struct kfd_process_device *pdd = p->pdds[i];1895 1896 if (n_evicted == 0)1897 break;1898 1899 kfd_smi_event_queue_restore(pdd->dev, p->lead_thread->pid);1900 1901 if (pdd->dev->dqm->ops.restore_process_queues(pdd->dev->dqm,1902 &pdd->qpd))1903 dev_err(pdd->dev->adev->dev,1904 "Failed to restore queues\n");1905 1906 n_evicted--;1907 }1908 1909 return r;1910}1911 1912/* kfd_process_restore_queues - Restore all user queues of a process */1913int kfd_process_restore_queues(struct kfd_process *p)1914{1915 int r, ret = 0;1916 int i;1917 1918 for (i = 0; i < p->n_pdds; i++) {1919 struct kfd_process_device *pdd = p->pdds[i];1920 struct device *dev = pdd->dev->adev->dev;1921 1922 kfd_smi_event_queue_restore(pdd->dev, p->lead_thread->pid);1923 1924 r = pdd->dev->dqm->ops.restore_process_queues(pdd->dev->dqm,1925 &pdd->qpd);1926 if (r) {1927 dev_err(dev, "Failed to restore process queues\n");1928 if (!ret)1929 ret = r;1930 }1931 }1932 1933 return ret;1934}1935 1936int kfd_process_gpuidx_from_gpuid(struct kfd_process *p, uint32_t gpu_id)1937{1938 int i;1939 1940 for (i = 0; i < p->n_pdds; i++)1941 if (p->pdds[i] && gpu_id == p->pdds[i]->user_gpu_id)1942 return i;1943 return -EINVAL;1944}1945 1946int1947kfd_process_gpuid_from_node(struct kfd_process *p, struct kfd_node *node,1948 uint32_t *gpuid, uint32_t *gpuidx)1949{1950 int i;1951 1952 for (i = 0; i < p->n_pdds; i++)1953 if (p->pdds[i] && p->pdds[i]->dev == node) {1954 *gpuid = p->pdds[i]->user_gpu_id;1955 *gpuidx = i;1956 return 0;1957 }1958 return -EINVAL;1959}1960 1961static int signal_eviction_fence(struct kfd_process *p)1962{1963 struct dma_fence *ef;1964 int ret;1965 1966 rcu_read_lock();1967 ef = dma_fence_get_rcu_safe(&p->ef);1968 rcu_read_unlock();1969 if (!ef)1970 return -EINVAL;1971 1972 ret = dma_fence_signal(ef);1973 dma_fence_put(ef);1974 1975 return ret;1976}1977 1978static void evict_process_worker(struct work_struct *work)1979{1980 int ret;1981 struct kfd_process *p;1982 struct delayed_work *dwork;1983 1984 dwork = to_delayed_work(work);1985 1986 /* Process termination destroys this worker thread. So during the1987 * lifetime of this thread, kfd_process p will be valid1988 */1989 p = container_of(dwork, struct kfd_process, eviction_work);1990 1991 pr_debug("Started evicting pasid 0x%x\n", p->pasid);1992 ret = kfd_process_evict_queues(p, KFD_QUEUE_EVICTION_TRIGGER_TTM);1993 if (!ret) {1994 /* If another thread already signaled the eviction fence,1995 * they are responsible stopping the queues and scheduling1996 * the restore work.1997 */1998 if (signal_eviction_fence(p) ||1999 mod_delayed_work(kfd_restore_wq, &p->restore_work,2000 msecs_to_jiffies(PROCESS_RESTORE_TIME_MS)))2001 kfd_process_restore_queues(p);2002 2003 pr_debug("Finished evicting pasid 0x%x\n", p->pasid);2004 } else2005 pr_err("Failed to evict queues of pasid 0x%x\n", p->pasid);2006}2007 2008static int restore_process_helper(struct kfd_process *p)2009{2010 int ret = 0;2011 2012 /* VMs may not have been acquired yet during debugging. */2013 if (p->kgd_process_info) {2014 ret = amdgpu_amdkfd_gpuvm_restore_process_bos(2015 p->kgd_process_info, &p->ef);2016 if (ret)2017 return ret;2018 }2019 2020 ret = kfd_process_restore_queues(p);2021 if (!ret)2022 pr_debug("Finished restoring pasid 0x%x\n", p->pasid);2023 else2024 pr_err("Failed to restore queues of pasid 0x%x\n", p->pasid);2025 2026 return ret;2027}2028 2029static void restore_process_worker(struct work_struct *work)2030{2031 struct delayed_work *dwork;2032 struct kfd_process *p;2033 int ret = 0;2034 2035 dwork = to_delayed_work(work);2036 2037 /* Process termination destroys this worker thread. So during the2038 * lifetime of this thread, kfd_process p will be valid2039 */2040 p = container_of(dwork, struct kfd_process, restore_work);2041 pr_debug("Started restoring pasid 0x%x\n", p->pasid);2042 2043 /* Setting last_restore_timestamp before successful restoration.2044 * Otherwise this would have to be set by KGD (restore_process_bos)2045 * before KFD BOs are unreserved. If not, the process can be evicted2046 * again before the timestamp is set.2047 * If restore fails, the timestamp will be set again in the next2048 * attempt. This would mean that the minimum GPU quanta would be2049 * PROCESS_ACTIVE_TIME_MS - (time to execute the following two2050 * functions)2051 */2052 2053 p->last_restore_timestamp = get_jiffies_64();2054 2055 ret = restore_process_helper(p);2056 if (ret) {2057 pr_debug("Failed to restore BOs of pasid 0x%x, retry after %d ms\n",2058 p->pasid, PROCESS_BACK_OFF_TIME_MS);2059 if (mod_delayed_work(kfd_restore_wq, &p->restore_work,2060 msecs_to_jiffies(PROCESS_RESTORE_TIME_MS)))2061 kfd_process_restore_queues(p);2062 }2063}2064 2065void kfd_suspend_all_processes(void)2066{2067 struct kfd_process *p;2068 unsigned int temp;2069 int idx = srcu_read_lock(&kfd_processes_srcu);2070 2071 WARN(debug_evictions, "Evicting all processes");2072 hash_for_each_rcu(kfd_processes_table, temp, p, kfd_processes) {2073 if (kfd_process_evict_queues(p, KFD_QUEUE_EVICTION_TRIGGER_SUSPEND))2074 pr_err("Failed to suspend process 0x%x\n", p->pasid);2075 signal_eviction_fence(p);2076 }2077 srcu_read_unlock(&kfd_processes_srcu, idx);2078}2079 2080int kfd_resume_all_processes(void)2081{2082 struct kfd_process *p;2083 unsigned int temp;2084 int ret = 0, idx = srcu_read_lock(&kfd_processes_srcu);2085 2086 hash_for_each_rcu(kfd_processes_table, temp, p, kfd_processes) {2087 if (restore_process_helper(p)) {2088 pr_err("Restore process %d failed during resume\n",2089 p->pasid);2090 ret = -EFAULT;2091 }2092 }2093 srcu_read_unlock(&kfd_processes_srcu, idx);2094 return ret;2095}2096 2097int kfd_reserved_mem_mmap(struct kfd_node *dev, struct kfd_process *process,2098 struct vm_area_struct *vma)2099{2100 struct kfd_process_device *pdd;2101 struct qcm_process_device *qpd;2102 2103 if ((vma->vm_end - vma->vm_start) != KFD_CWSR_TBA_TMA_SIZE) {2104 dev_err(dev->adev->dev, "Incorrect CWSR mapping size.\n");2105 return -EINVAL;2106 }2107 2108 pdd = kfd_get_process_device_data(dev, process);2109 if (!pdd)2110 return -EINVAL;2111 qpd = &pdd->qpd;2112 2113 qpd->cwsr_kaddr = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO,2114 get_order(KFD_CWSR_TBA_TMA_SIZE));2115 if (!qpd->cwsr_kaddr) {2116 dev_err(dev->adev->dev,2117 "Error allocating per process CWSR buffer.\n");2118 return -ENOMEM;2119 }2120 2121 vm_flags_set(vma, VM_IO | VM_DONTCOPY | VM_DONTEXPAND2122 | VM_NORESERVE | VM_DONTDUMP | VM_PFNMAP);2123 /* Mapping pages to user process */2124 return remap_pfn_range(vma, vma->vm_start,2125 PFN_DOWN(__pa(qpd->cwsr_kaddr)),2126 KFD_CWSR_TBA_TMA_SIZE, vma->vm_page_prot);2127}2128 2129/* assumes caller holds process lock. */2130int kfd_process_drain_interrupts(struct kfd_process_device *pdd)2131{2132 uint32_t irq_drain_fence[8];2133 uint8_t node_id = 0;2134 int r = 0;2135 2136 if (!KFD_IS_SOC15(pdd->dev))2137 return 0;2138 2139 pdd->process->irq_drain_is_open = true;2140 2141 memset(irq_drain_fence, 0, sizeof(irq_drain_fence));2142 irq_drain_fence[0] = (KFD_IRQ_FENCE_SOURCEID << 8) |2143 KFD_IRQ_FENCE_CLIENTID;2144 irq_drain_fence[3] = pdd->process->pasid;2145 2146 /*2147 * For GFX 9.4.3, send the NodeId also in IH cookie DW[3]2148 */2149 if (KFD_GC_VERSION(pdd->dev->kfd) == IP_VERSION(9, 4, 3) ||2150 KFD_GC_VERSION(pdd->dev->kfd) == IP_VERSION(9, 4, 4)) {2151 node_id = ffs(pdd->dev->interrupt_bitmap) - 1;2152 irq_drain_fence[3] |= node_id << 16;2153 }2154 2155 /* ensure stale irqs scheduled KFD interrupts and send drain fence. */2156 if (amdgpu_amdkfd_send_close_event_drain_irq(pdd->dev->adev,2157 irq_drain_fence)) {2158 pdd->process->irq_drain_is_open = false;2159 return 0;2160 }2161 2162 r = wait_event_interruptible(pdd->process->wait_irq_drain,2163 !READ_ONCE(pdd->process->irq_drain_is_open));2164 if (r)2165 pdd->process->irq_drain_is_open = false;2166 2167 return r;2168}2169 2170void kfd_process_close_interrupt_drain(unsigned int pasid)2171{2172 struct kfd_process *p;2173 2174 p = kfd_lookup_process_by_pasid(pasid);2175 2176 if (!p)2177 return;2178 2179 WRITE_ONCE(p->irq_drain_is_open, false);2180 wake_up_all(&p->wait_irq_drain);2181 kfd_unref_process(p);2182}2183 2184struct send_exception_work_handler_workarea {2185 struct work_struct work;2186 struct kfd_process *p;2187 unsigned int queue_id;2188 uint64_t error_reason;2189};2190 2191static void send_exception_work_handler(struct work_struct *work)2192{2193 struct send_exception_work_handler_workarea *workarea;2194 struct kfd_process *p;2195 struct queue *q;2196 struct mm_struct *mm;2197 struct kfd_context_save_area_header __user *csa_header;2198 uint64_t __user *err_payload_ptr;2199 uint64_t cur_err;2200 uint32_t ev_id;2201 2202 workarea = container_of(work,2203 struct send_exception_work_handler_workarea,2204 work);2205 p = workarea->p;2206 2207 mm = get_task_mm(p->lead_thread);2208 2209 if (!mm)2210 return;2211 2212 kthread_use_mm(mm);2213 2214 q = pqm_get_user_queue(&p->pqm, workarea->queue_id);2215 2216 if (!q)2217 goto out;2218 2219 csa_header = (void __user *)q->properties.ctx_save_restore_area_address;2220 2221 get_user(err_payload_ptr, (uint64_t __user **)&csa_header->err_payload_addr);2222 get_user(cur_err, err_payload_ptr);2223 cur_err |= workarea->error_reason;2224 put_user(cur_err, err_payload_ptr);2225 get_user(ev_id, &csa_header->err_event_id);2226 2227 kfd_set_event(p, ev_id);2228 2229out:2230 kthread_unuse_mm(mm);2231 mmput(mm);2232}2233 2234int kfd_send_exception_to_runtime(struct kfd_process *p,2235 unsigned int queue_id,2236 uint64_t error_reason)2237{2238 struct send_exception_work_handler_workarea worker;2239 2240 INIT_WORK_ONSTACK(&worker.work, send_exception_work_handler);2241 2242 worker.p = p;2243 worker.queue_id = queue_id;2244 worker.error_reason = error_reason;2245 2246 schedule_work(&worker.work);2247 flush_work(&worker.work);2248 destroy_work_on_stack(&worker.work);2249 2250 return 0;2251}2252 2253struct kfd_process_device *kfd_process_device_data_by_id(struct kfd_process *p, uint32_t gpu_id)2254{2255 int i;2256 2257 if (gpu_id) {2258 for (i = 0; i < p->n_pdds; i++) {2259 struct kfd_process_device *pdd = p->pdds[i];2260 2261 if (pdd->user_gpu_id == gpu_id)2262 return pdd;2263 }2264 }2265 return NULL;2266}2267 2268int kfd_process_get_user_gpu_id(struct kfd_process *p, uint32_t actual_gpu_id)2269{2270 int i;2271 2272 if (!actual_gpu_id)2273 return 0;2274 2275 for (i = 0; i < p->n_pdds; i++) {2276 struct kfd_process_device *pdd = p->pdds[i];2277 2278 if (pdd->dev->id == actual_gpu_id)2279 return pdd->user_gpu_id;2280 }2281 return -EINVAL;2282}2283 2284#if defined(CONFIG_DEBUG_FS)2285 2286int kfd_debugfs_mqds_by_process(struct seq_file *m, void *data)2287{2288 struct kfd_process *p;2289 unsigned int temp;2290 int r = 0;2291 2292 int idx = srcu_read_lock(&kfd_processes_srcu);2293 2294 hash_for_each_rcu(kfd_processes_table, temp, p, kfd_processes) {2295 seq_printf(m, "Process %d PASID 0x%x:\n",2296 p->lead_thread->tgid, p->pasid);2297 2298 mutex_lock(&p->mutex);2299 r = pqm_debugfs_mqds(m, &p->pqm);2300 mutex_unlock(&p->mutex);2301 2302 if (r)2303 break;2304 }2305 2306 srcu_read_unlock(&kfd_processes_srcu, idx);2307 2308 return r;2309}2310 2311#endif2312