391 lines · c
1// SPDX-License-Identifier: GPL-2.0 OR MIT2/*3 * Copyright 2020-2022 Advanced Micro Devices, Inc.4 *5 * Permission is hereby granted, free of charge, to any person obtaining a6 * copy of this software and associated documentation files (the "Software"),7 * to deal in the Software without restriction, including without limitation8 * the rights to use, copy, modify, merge, publish, distribute, sublicense,9 * and/or sell copies of the Software, and to permit persons to whom the10 * Software is furnished to do so, subject to the following conditions:11 *12 * The above copyright notice and this permission notice shall be included in13 * all copies or substantial portions of the Software.14 *15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL18 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR19 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,20 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR21 * OTHER DEALINGS IN THE SOFTWARE.22 */23 24#include <linux/poll.h>25#include <linux/wait.h>26#include <linux/anon_inodes.h>27#include <uapi/linux/kfd_ioctl.h>28#include "amdgpu.h"29#include "amdgpu_vm.h"30#include "kfd_priv.h"31#include "kfd_smi_events.h"32#include "amdgpu_reset.h"33 34struct kfd_smi_client {35 struct list_head list;36 struct kfifo fifo;37 wait_queue_head_t wait_queue;38 /* events enabled */39 uint64_t events;40 struct kfd_node *dev;41 spinlock_t lock;42 struct rcu_head rcu;43 pid_t pid;44 bool suser;45};46 47#define MAX_KFIFO_SIZE 102448 49static __poll_t kfd_smi_ev_poll(struct file *, struct poll_table_struct *);50static ssize_t kfd_smi_ev_read(struct file *, char __user *, size_t, loff_t *);51static ssize_t kfd_smi_ev_write(struct file *, const char __user *, size_t,52 loff_t *);53static int kfd_smi_ev_release(struct inode *, struct file *);54 55static const char kfd_smi_name[] = "kfd_smi_ev";56 57static const struct file_operations kfd_smi_ev_fops = {58 .owner = THIS_MODULE,59 .poll = kfd_smi_ev_poll,60 .read = kfd_smi_ev_read,61 .write = kfd_smi_ev_write,62 .release = kfd_smi_ev_release63};64 65static __poll_t kfd_smi_ev_poll(struct file *filep,66 struct poll_table_struct *wait)67{68 struct kfd_smi_client *client = filep->private_data;69 __poll_t mask = 0;70 71 poll_wait(filep, &client->wait_queue, wait);72 73 spin_lock(&client->lock);74 if (!kfifo_is_empty(&client->fifo))75 mask = EPOLLIN | EPOLLRDNORM;76 spin_unlock(&client->lock);77 78 return mask;79}80 81static ssize_t kfd_smi_ev_read(struct file *filep, char __user *user,82 size_t size, loff_t *offset)83{84 int ret;85 size_t to_copy;86 struct kfd_smi_client *client = filep->private_data;87 unsigned char *buf;88 89 size = min_t(size_t, size, MAX_KFIFO_SIZE);90 buf = kmalloc(size, GFP_KERNEL);91 if (!buf)92 return -ENOMEM;93 94 /* kfifo_to_user can sleep so we can't use spinlock protection around95 * it. Instead, we kfifo out as spinlocked then copy them to the user.96 */97 spin_lock(&client->lock);98 to_copy = kfifo_len(&client->fifo);99 if (!to_copy) {100 spin_unlock(&client->lock);101 ret = -EAGAIN;102 goto ret_err;103 }104 to_copy = min(size, to_copy);105 ret = kfifo_out(&client->fifo, buf, to_copy);106 spin_unlock(&client->lock);107 if (ret <= 0) {108 ret = -EAGAIN;109 goto ret_err;110 }111 112 ret = copy_to_user(user, buf, to_copy);113 if (ret) {114 ret = -EFAULT;115 goto ret_err;116 }117 118 kfree(buf);119 return to_copy;120 121ret_err:122 kfree(buf);123 return ret;124}125 126static ssize_t kfd_smi_ev_write(struct file *filep, const char __user *user,127 size_t size, loff_t *offset)128{129 struct kfd_smi_client *client = filep->private_data;130 uint64_t events;131 132 if (!access_ok(user, size) || size < sizeof(events))133 return -EFAULT;134 if (copy_from_user(&events, user, sizeof(events)))135 return -EFAULT;136 137 WRITE_ONCE(client->events, events);138 139 return sizeof(events);140}141 142static void kfd_smi_ev_client_free(struct rcu_head *p)143{144 struct kfd_smi_client *ev = container_of(p, struct kfd_smi_client, rcu);145 146 kfifo_free(&ev->fifo);147 kfree(ev);148}149 150static int kfd_smi_ev_release(struct inode *inode, struct file *filep)151{152 struct kfd_smi_client *client = filep->private_data;153 struct kfd_node *dev = client->dev;154 155 spin_lock(&dev->smi_lock);156 list_del_rcu(&client->list);157 spin_unlock(&dev->smi_lock);158 159 call_rcu(&client->rcu, kfd_smi_ev_client_free);160 return 0;161}162 163static bool kfd_smi_ev_enabled(pid_t pid, struct kfd_smi_client *client,164 unsigned int event)165{166 uint64_t all = KFD_SMI_EVENT_MASK_FROM_INDEX(KFD_SMI_EVENT_ALL_PROCESS);167 uint64_t events = READ_ONCE(client->events);168 169 if (pid && client->pid != pid && !(client->suser && (events & all)))170 return false;171 172 return events & KFD_SMI_EVENT_MASK_FROM_INDEX(event);173}174 175static void add_event_to_kfifo(pid_t pid, struct kfd_node *dev,176 unsigned int smi_event, char *event_msg, int len)177{178 struct kfd_smi_client *client;179 180 rcu_read_lock();181 182 list_for_each_entry_rcu(client, &dev->smi_clients, list) {183 if (!kfd_smi_ev_enabled(pid, client, smi_event))184 continue;185 spin_lock(&client->lock);186 if (kfifo_avail(&client->fifo) >= len) {187 kfifo_in(&client->fifo, event_msg, len);188 wake_up_all(&client->wait_queue);189 } else {190 pr_debug("smi_event(EventID: %u): no space left\n",191 smi_event);192 }193 spin_unlock(&client->lock);194 }195 196 rcu_read_unlock();197}198 199__printf(4, 5)200static void kfd_smi_event_add(pid_t pid, struct kfd_node *dev,201 unsigned int event, char *fmt, ...)202{203 char fifo_in[KFD_SMI_EVENT_MSG_SIZE];204 int len;205 va_list args;206 207 if (list_empty(&dev->smi_clients))208 return;209 210 len = snprintf(fifo_in, sizeof(fifo_in), "%x ", event);211 212 va_start(args, fmt);213 len += vsnprintf(fifo_in + len, sizeof(fifo_in) - len, fmt, args);214 va_end(args);215 216 add_event_to_kfifo(pid, dev, event, fifo_in, len);217}218 219void kfd_smi_event_update_gpu_reset(struct kfd_node *dev, bool post_reset,220 struct amdgpu_reset_context *reset_context)221{222 unsigned int event;223 char reset_cause[64];224 225 if (post_reset) {226 event = KFD_SMI_EVENT_GPU_POST_RESET;227 } else {228 event = KFD_SMI_EVENT_GPU_PRE_RESET;229 ++(dev->reset_seq_num);230 }231 232 memset(reset_cause, 0, sizeof(reset_cause));233 234 if (reset_context)235 amdgpu_reset_get_desc(reset_context, reset_cause,236 sizeof(reset_cause));237 238 kfd_smi_event_add(0, dev, event, KFD_EVENT_FMT_UPDATE_GPU_RESET(239 dev->reset_seq_num, reset_cause));240}241 242void kfd_smi_event_update_thermal_throttling(struct kfd_node *dev,243 uint64_t throttle_bitmask)244{245 kfd_smi_event_add(0, dev, KFD_SMI_EVENT_THERMAL_THROTTLE, KFD_EVENT_FMT_THERMAL_THROTTLING(246 throttle_bitmask,247 amdgpu_dpm_get_thermal_throttling_counter(dev->adev)));248}249 250void kfd_smi_event_update_vmfault(struct kfd_node *dev, uint16_t pasid)251{252 struct amdgpu_task_info *task_info;253 254 task_info = amdgpu_vm_get_task_info_pasid(dev->adev, pasid);255 if (task_info) {256 /* Report VM faults from user applications, not retry from kernel */257 if (task_info->pid)258 kfd_smi_event_add(0, dev, KFD_SMI_EVENT_VMFAULT, KFD_EVENT_FMT_VMFAULT(259 task_info->pid, task_info->task_name));260 amdgpu_vm_put_task_info(task_info);261 }262}263 264void kfd_smi_event_page_fault_start(struct kfd_node *node, pid_t pid,265 unsigned long address, bool write_fault,266 ktime_t ts)267{268 kfd_smi_event_add(pid, node, KFD_SMI_EVENT_PAGE_FAULT_START,269 KFD_EVENT_FMT_PAGEFAULT_START(ktime_to_ns(ts), pid,270 address, node->id, write_fault ? 'W' : 'R'));271}272 273void kfd_smi_event_page_fault_end(struct kfd_node *node, pid_t pid,274 unsigned long address, bool migration)275{276 kfd_smi_event_add(pid, node, KFD_SMI_EVENT_PAGE_FAULT_END,277 KFD_EVENT_FMT_PAGEFAULT_END(ktime_get_boottime_ns(),278 pid, address, node->id, migration ? 'M' : 'U'));279}280 281void kfd_smi_event_migration_start(struct kfd_node *node, pid_t pid,282 unsigned long start, unsigned long end,283 uint32_t from, uint32_t to,284 uint32_t prefetch_loc, uint32_t preferred_loc,285 uint32_t trigger)286{287 kfd_smi_event_add(pid, node, KFD_SMI_EVENT_MIGRATE_START,288 KFD_EVENT_FMT_MIGRATE_START(289 ktime_get_boottime_ns(), pid, start, end - start,290 from, to, prefetch_loc, preferred_loc, trigger));291}292 293void kfd_smi_event_migration_end(struct kfd_node *node, pid_t pid,294 unsigned long start, unsigned long end,295 uint32_t from, uint32_t to, uint32_t trigger)296{297 kfd_smi_event_add(pid, node, KFD_SMI_EVENT_MIGRATE_END,298 KFD_EVENT_FMT_MIGRATE_END(299 ktime_get_boottime_ns(), pid, start, end - start,300 from, to, trigger));301}302 303void kfd_smi_event_queue_eviction(struct kfd_node *node, pid_t pid,304 uint32_t trigger)305{306 kfd_smi_event_add(pid, node, KFD_SMI_EVENT_QUEUE_EVICTION,307 KFD_EVENT_FMT_QUEUE_EVICTION(ktime_get_boottime_ns(), pid,308 node->id, trigger));309}310 311void kfd_smi_event_queue_restore(struct kfd_node *node, pid_t pid)312{313 kfd_smi_event_add(pid, node, KFD_SMI_EVENT_QUEUE_RESTORE,314 KFD_EVENT_FMT_QUEUE_RESTORE(ktime_get_boottime_ns(), pid,315 node->id, 0));316}317 318void kfd_smi_event_queue_restore_rescheduled(struct mm_struct *mm)319{320 struct kfd_process *p;321 int i;322 323 p = kfd_lookup_process_by_mm(mm);324 if (!p)325 return;326 327 for (i = 0; i < p->n_pdds; i++) {328 struct kfd_process_device *pdd = p->pdds[i];329 330 kfd_smi_event_add(p->lead_thread->pid, pdd->dev,331 KFD_SMI_EVENT_QUEUE_RESTORE,332 KFD_EVENT_FMT_QUEUE_RESTORE(ktime_get_boottime_ns(),333 p->lead_thread->pid, pdd->dev->id, 'R'));334 }335 kfd_unref_process(p);336}337 338void kfd_smi_event_unmap_from_gpu(struct kfd_node *node, pid_t pid,339 unsigned long address, unsigned long last,340 uint32_t trigger)341{342 kfd_smi_event_add(pid, node, KFD_SMI_EVENT_UNMAP_FROM_GPU,343 KFD_EVENT_FMT_UNMAP_FROM_GPU(ktime_get_boottime_ns(),344 pid, address, last - address + 1, node->id, trigger));345}346 347int kfd_smi_event_open(struct kfd_node *dev, uint32_t *fd)348{349 struct kfd_smi_client *client;350 int ret;351 352 client = kzalloc(sizeof(struct kfd_smi_client), GFP_KERNEL);353 if (!client)354 return -ENOMEM;355 INIT_LIST_HEAD(&client->list);356 357 ret = kfifo_alloc(&client->fifo, MAX_KFIFO_SIZE, GFP_KERNEL);358 if (ret) {359 kfree(client);360 return ret;361 }362 363 init_waitqueue_head(&client->wait_queue);364 spin_lock_init(&client->lock);365 client->events = 0;366 client->dev = dev;367 client->pid = current->tgid;368 client->suser = capable(CAP_SYS_ADMIN);369 370 spin_lock(&dev->smi_lock);371 list_add_rcu(&client->list, &dev->smi_clients);372 spin_unlock(&dev->smi_lock);373 374 ret = anon_inode_getfd(kfd_smi_name, &kfd_smi_ev_fops, (void *)client,375 O_RDWR);376 if (ret < 0) {377 spin_lock(&dev->smi_lock);378 list_del_rcu(&client->list);379 spin_unlock(&dev->smi_lock);380 381 synchronize_rcu();382 383 kfifo_free(&client->fifo);384 kfree(client);385 return ret;386 }387 *fd = ret;388 389 return 0;390}391