652 lines · c
1// SPDX-License-Identifier: GPL-2.0-or-later2#include <linux/compat.h>3#include <linux/dma-mapping.h>4#include <linux/iommu.h>5#include <linux/module.h>6#include <linux/poll.h>7#include <linux/slab.h>8#include <linux/uacce.h>9 10static dev_t uacce_devt;11static DEFINE_XARRAY_ALLOC(uacce_xa);12 13static const struct class uacce_class = {14 .name = UACCE_NAME,15};16 17/*18 * If the parent driver or the device disappears, the queue state is invalid and19 * ops are not usable anymore.20 */21static bool uacce_queue_is_valid(struct uacce_queue *q)22{23 return q->state == UACCE_Q_INIT || q->state == UACCE_Q_STARTED;24}25 26static int uacce_start_queue(struct uacce_queue *q)27{28 int ret;29 30 if (q->state != UACCE_Q_INIT)31 return -EINVAL;32 33 if (q->uacce->ops->start_queue) {34 ret = q->uacce->ops->start_queue(q);35 if (ret < 0)36 return ret;37 }38 39 q->state = UACCE_Q_STARTED;40 return 0;41}42 43static int uacce_put_queue(struct uacce_queue *q)44{45 struct uacce_device *uacce = q->uacce;46 47 if ((q->state == UACCE_Q_STARTED) && uacce->ops->stop_queue)48 uacce->ops->stop_queue(q);49 50 if ((q->state == UACCE_Q_INIT || q->state == UACCE_Q_STARTED) &&51 uacce->ops->put_queue)52 uacce->ops->put_queue(q);53 54 q->state = UACCE_Q_ZOMBIE;55 56 return 0;57}58 59static long uacce_fops_unl_ioctl(struct file *filep,60 unsigned int cmd, unsigned long arg)61{62 struct uacce_queue *q = filep->private_data;63 struct uacce_device *uacce = q->uacce;64 long ret = -ENXIO;65 66 /*67 * uacce->ops->ioctl() may take the mmap_lock when copying arg to/from68 * user. Avoid a circular lock dependency with uacce_fops_mmap(), which69 * gets called with mmap_lock held, by taking uacce->mutex instead of70 * q->mutex. Doing this in uacce_fops_mmap() is not possible because71 * uacce_fops_open() calls iommu_sva_bind_device(), which takes72 * mmap_lock, while holding uacce->mutex.73 */74 mutex_lock(&uacce->mutex);75 if (!uacce_queue_is_valid(q))76 goto out_unlock;77 78 switch (cmd) {79 case UACCE_CMD_START_Q:80 ret = uacce_start_queue(q);81 break;82 case UACCE_CMD_PUT_Q:83 ret = uacce_put_queue(q);84 break;85 default:86 if (uacce->ops->ioctl)87 ret = uacce->ops->ioctl(q, cmd, arg);88 else89 ret = -EINVAL;90 }91out_unlock:92 mutex_unlock(&uacce->mutex);93 return ret;94}95 96#ifdef CONFIG_COMPAT97static long uacce_fops_compat_ioctl(struct file *filep,98 unsigned int cmd, unsigned long arg)99{100 arg = (unsigned long)compat_ptr(arg);101 102 return uacce_fops_unl_ioctl(filep, cmd, arg);103}104#endif105 106static int uacce_bind_queue(struct uacce_device *uacce, struct uacce_queue *q)107{108 u32 pasid;109 struct iommu_sva *handle;110 111 if (!(uacce->flags & UACCE_DEV_SVA))112 return 0;113 114 handle = iommu_sva_bind_device(uacce->parent, current->mm);115 if (IS_ERR(handle))116 return PTR_ERR(handle);117 118 pasid = iommu_sva_get_pasid(handle);119 if (pasid == IOMMU_PASID_INVALID) {120 iommu_sva_unbind_device(handle);121 return -ENODEV;122 }123 124 q->handle = handle;125 q->pasid = pasid;126 return 0;127}128 129static void uacce_unbind_queue(struct uacce_queue *q)130{131 if (!q->handle)132 return;133 iommu_sva_unbind_device(q->handle);134 q->handle = NULL;135}136 137static int uacce_fops_open(struct inode *inode, struct file *filep)138{139 struct uacce_device *uacce;140 struct uacce_queue *q;141 int ret;142 143 uacce = xa_load(&uacce_xa, iminor(inode));144 if (!uacce)145 return -ENODEV;146 147 q = kzalloc(sizeof(struct uacce_queue), GFP_KERNEL);148 if (!q)149 return -ENOMEM;150 151 mutex_lock(&uacce->mutex);152 153 if (!uacce->parent) {154 ret = -EINVAL;155 goto out_with_mem;156 }157 158 ret = uacce_bind_queue(uacce, q);159 if (ret)160 goto out_with_mem;161 162 q->uacce = uacce;163 164 if (uacce->ops->get_queue) {165 ret = uacce->ops->get_queue(uacce, q->pasid, q);166 if (ret < 0)167 goto out_with_bond;168 }169 170 init_waitqueue_head(&q->wait);171 filep->private_data = q;172 q->state = UACCE_Q_INIT;173 q->mapping = filep->f_mapping;174 mutex_init(&q->mutex);175 list_add(&q->list, &uacce->queues);176 mutex_unlock(&uacce->mutex);177 178 return 0;179 180out_with_bond:181 uacce_unbind_queue(q);182out_with_mem:183 kfree(q);184 mutex_unlock(&uacce->mutex);185 return ret;186}187 188static int uacce_fops_release(struct inode *inode, struct file *filep)189{190 struct uacce_queue *q = filep->private_data;191 struct uacce_device *uacce = q->uacce;192 193 mutex_lock(&uacce->mutex);194 uacce_put_queue(q);195 uacce_unbind_queue(q);196 list_del(&q->list);197 mutex_unlock(&uacce->mutex);198 kfree(q);199 200 return 0;201}202 203static void uacce_vma_close(struct vm_area_struct *vma)204{205 struct uacce_queue *q = vma->vm_private_data;206 207 if (vma->vm_pgoff < UACCE_MAX_REGION) {208 struct uacce_qfile_region *qfr = q->qfrs[vma->vm_pgoff];209 210 mutex_lock(&q->mutex);211 q->qfrs[vma->vm_pgoff] = NULL;212 mutex_unlock(&q->mutex);213 kfree(qfr);214 }215}216 217static const struct vm_operations_struct uacce_vm_ops = {218 .close = uacce_vma_close,219};220 221static int uacce_fops_mmap(struct file *filep, struct vm_area_struct *vma)222{223 struct uacce_queue *q = filep->private_data;224 struct uacce_device *uacce = q->uacce;225 struct uacce_qfile_region *qfr;226 enum uacce_qfrt type = UACCE_MAX_REGION;227 int ret = 0;228 229 if (vma->vm_pgoff < UACCE_MAX_REGION)230 type = vma->vm_pgoff;231 else232 return -EINVAL;233 234 qfr = kzalloc(sizeof(*qfr), GFP_KERNEL);235 if (!qfr)236 return -ENOMEM;237 238 vm_flags_set(vma, VM_DONTCOPY | VM_DONTEXPAND | VM_WIPEONFORK);239 vma->vm_ops = &uacce_vm_ops;240 vma->vm_private_data = q;241 qfr->type = type;242 243 mutex_lock(&q->mutex);244 if (!uacce_queue_is_valid(q)) {245 ret = -ENXIO;246 goto out_with_lock;247 }248 249 if (q->qfrs[type]) {250 ret = -EEXIST;251 goto out_with_lock;252 }253 254 switch (type) {255 case UACCE_QFRT_MMIO:256 case UACCE_QFRT_DUS:257 if (!uacce->ops->mmap) {258 ret = -EINVAL;259 goto out_with_lock;260 }261 262 ret = uacce->ops->mmap(q, vma, qfr);263 if (ret)264 goto out_with_lock;265 break;266 267 default:268 ret = -EINVAL;269 goto out_with_lock;270 }271 272 q->qfrs[type] = qfr;273 mutex_unlock(&q->mutex);274 275 return ret;276 277out_with_lock:278 mutex_unlock(&q->mutex);279 kfree(qfr);280 return ret;281}282 283static __poll_t uacce_fops_poll(struct file *file, poll_table *wait)284{285 struct uacce_queue *q = file->private_data;286 struct uacce_device *uacce = q->uacce;287 __poll_t ret = 0;288 289 mutex_lock(&q->mutex);290 if (!uacce_queue_is_valid(q))291 goto out_unlock;292 293 poll_wait(file, &q->wait, wait);294 295 if (uacce->ops->is_q_updated && uacce->ops->is_q_updated(q))296 ret = EPOLLIN | EPOLLRDNORM;297 298out_unlock:299 mutex_unlock(&q->mutex);300 return ret;301}302 303static const struct file_operations uacce_fops = {304 .owner = THIS_MODULE,305 .open = uacce_fops_open,306 .release = uacce_fops_release,307 .unlocked_ioctl = uacce_fops_unl_ioctl,308#ifdef CONFIG_COMPAT309 .compat_ioctl = uacce_fops_compat_ioctl,310#endif311 .mmap = uacce_fops_mmap,312 .poll = uacce_fops_poll,313};314 315#define to_uacce_device(dev) container_of(dev, struct uacce_device, dev)316 317static ssize_t api_show(struct device *dev,318 struct device_attribute *attr, char *buf)319{320 struct uacce_device *uacce = to_uacce_device(dev);321 322 return sysfs_emit(buf, "%s\n", uacce->api_ver);323}324 325static ssize_t flags_show(struct device *dev,326 struct device_attribute *attr, char *buf)327{328 struct uacce_device *uacce = to_uacce_device(dev);329 330 return sysfs_emit(buf, "%u\n", uacce->flags);331}332 333static ssize_t available_instances_show(struct device *dev,334 struct device_attribute *attr,335 char *buf)336{337 struct uacce_device *uacce = to_uacce_device(dev);338 339 if (!uacce->ops->get_available_instances)340 return -ENODEV;341 342 return sysfs_emit(buf, "%d\n",343 uacce->ops->get_available_instances(uacce));344}345 346static ssize_t algorithms_show(struct device *dev,347 struct device_attribute *attr, char *buf)348{349 struct uacce_device *uacce = to_uacce_device(dev);350 351 return sysfs_emit(buf, "%s\n", uacce->algs);352}353 354static ssize_t region_mmio_size_show(struct device *dev,355 struct device_attribute *attr, char *buf)356{357 struct uacce_device *uacce = to_uacce_device(dev);358 359 return sysfs_emit(buf, "%lu\n",360 uacce->qf_pg_num[UACCE_QFRT_MMIO] << PAGE_SHIFT);361}362 363static ssize_t region_dus_size_show(struct device *dev,364 struct device_attribute *attr, char *buf)365{366 struct uacce_device *uacce = to_uacce_device(dev);367 368 return sysfs_emit(buf, "%lu\n",369 uacce->qf_pg_num[UACCE_QFRT_DUS] << PAGE_SHIFT);370}371 372static ssize_t isolate_show(struct device *dev,373 struct device_attribute *attr, char *buf)374{375 struct uacce_device *uacce = to_uacce_device(dev);376 377 return sysfs_emit(buf, "%d\n", uacce->ops->get_isolate_state(uacce));378}379 380static ssize_t isolate_strategy_show(struct device *dev, struct device_attribute *attr, char *buf)381{382 struct uacce_device *uacce = to_uacce_device(dev);383 u32 val;384 385 val = uacce->ops->isolate_err_threshold_read(uacce);386 387 return sysfs_emit(buf, "%u\n", val);388}389 390static ssize_t isolate_strategy_store(struct device *dev, struct device_attribute *attr,391 const char *buf, size_t count)392{393 struct uacce_device *uacce = to_uacce_device(dev);394 unsigned long val;395 int ret;396 397 if (kstrtoul(buf, 0, &val) < 0)398 return -EINVAL;399 400 if (val > UACCE_MAX_ERR_THRESHOLD)401 return -EINVAL;402 403 ret = uacce->ops->isolate_err_threshold_write(uacce, val);404 if (ret)405 return ret;406 407 return count;408}409 410static DEVICE_ATTR_RO(api);411static DEVICE_ATTR_RO(flags);412static DEVICE_ATTR_RO(available_instances);413static DEVICE_ATTR_RO(algorithms);414static DEVICE_ATTR_RO(region_mmio_size);415static DEVICE_ATTR_RO(region_dus_size);416static DEVICE_ATTR_RO(isolate);417static DEVICE_ATTR_RW(isolate_strategy);418 419static struct attribute *uacce_dev_attrs[] = {420 &dev_attr_api.attr,421 &dev_attr_flags.attr,422 &dev_attr_available_instances.attr,423 &dev_attr_algorithms.attr,424 &dev_attr_region_mmio_size.attr,425 &dev_attr_region_dus_size.attr,426 &dev_attr_isolate.attr,427 &dev_attr_isolate_strategy.attr,428 NULL,429};430 431static umode_t uacce_dev_is_visible(struct kobject *kobj,432 struct attribute *attr, int n)433{434 struct device *dev = kobj_to_dev(kobj);435 struct uacce_device *uacce = to_uacce_device(dev);436 437 if (((attr == &dev_attr_region_mmio_size.attr) &&438 (!uacce->qf_pg_num[UACCE_QFRT_MMIO])) ||439 ((attr == &dev_attr_region_dus_size.attr) &&440 (!uacce->qf_pg_num[UACCE_QFRT_DUS])))441 return 0;442 443 if (attr == &dev_attr_isolate_strategy.attr &&444 (!uacce->ops->isolate_err_threshold_read &&445 !uacce->ops->isolate_err_threshold_write))446 return 0;447 448 if (attr == &dev_attr_isolate.attr && !uacce->ops->get_isolate_state)449 return 0;450 451 return attr->mode;452}453 454static struct attribute_group uacce_dev_group = {455 .is_visible = uacce_dev_is_visible,456 .attrs = uacce_dev_attrs,457};458 459__ATTRIBUTE_GROUPS(uacce_dev);460 461static void uacce_release(struct device *dev)462{463 struct uacce_device *uacce = to_uacce_device(dev);464 465 kfree(uacce);466}467 468static unsigned int uacce_enable_sva(struct device *parent, unsigned int flags)469{470 int ret;471 472 if (!(flags & UACCE_DEV_SVA))473 return flags;474 475 flags &= ~UACCE_DEV_SVA;476 477 ret = iommu_dev_enable_feature(parent, IOMMU_DEV_FEAT_IOPF);478 if (ret) {479 dev_err(parent, "failed to enable IOPF feature! ret = %pe\n", ERR_PTR(ret));480 return flags;481 }482 483 ret = iommu_dev_enable_feature(parent, IOMMU_DEV_FEAT_SVA);484 if (ret) {485 dev_err(parent, "failed to enable SVA feature! ret = %pe\n", ERR_PTR(ret));486 iommu_dev_disable_feature(parent, IOMMU_DEV_FEAT_IOPF);487 return flags;488 }489 490 return flags | UACCE_DEV_SVA;491}492 493static void uacce_disable_sva(struct uacce_device *uacce)494{495 if (!(uacce->flags & UACCE_DEV_SVA))496 return;497 498 iommu_dev_disable_feature(uacce->parent, IOMMU_DEV_FEAT_SVA);499 iommu_dev_disable_feature(uacce->parent, IOMMU_DEV_FEAT_IOPF);500}501 502/**503 * uacce_alloc() - alloc an accelerator504 * @parent: pointer of uacce parent device505 * @interface: pointer of uacce_interface for register506 *507 * Returns uacce pointer if success and ERR_PTR if not508 * Need check returned negotiated uacce->flags509 */510struct uacce_device *uacce_alloc(struct device *parent,511 struct uacce_interface *interface)512{513 unsigned int flags = interface->flags;514 struct uacce_device *uacce;515 int ret;516 517 uacce = kzalloc(sizeof(struct uacce_device), GFP_KERNEL);518 if (!uacce)519 return ERR_PTR(-ENOMEM);520 521 flags = uacce_enable_sva(parent, flags);522 523 uacce->parent = parent;524 uacce->flags = flags;525 uacce->ops = interface->ops;526 527 ret = xa_alloc(&uacce_xa, &uacce->dev_id, uacce, xa_limit_32b,528 GFP_KERNEL);529 if (ret < 0)530 goto err_with_uacce;531 532 INIT_LIST_HEAD(&uacce->queues);533 mutex_init(&uacce->mutex);534 device_initialize(&uacce->dev);535 uacce->dev.devt = MKDEV(MAJOR(uacce_devt), uacce->dev_id);536 uacce->dev.class = &uacce_class;537 uacce->dev.groups = uacce_dev_groups;538 uacce->dev.parent = uacce->parent;539 uacce->dev.release = uacce_release;540 dev_set_name(&uacce->dev, "%s-%d", interface->name, uacce->dev_id);541 542 return uacce;543 544err_with_uacce:545 uacce_disable_sva(uacce);546 kfree(uacce);547 return ERR_PTR(ret);548}549EXPORT_SYMBOL_GPL(uacce_alloc);550 551/**552 * uacce_register() - add the accelerator to cdev and export to user space553 * @uacce: The initialized uacce device554 *555 * Return 0 if register succeeded, or an error.556 */557int uacce_register(struct uacce_device *uacce)558{559 if (!uacce)560 return -ENODEV;561 562 uacce->cdev = cdev_alloc();563 if (!uacce->cdev)564 return -ENOMEM;565 566 uacce->cdev->ops = &uacce_fops;567 uacce->cdev->owner = THIS_MODULE;568 569 return cdev_device_add(uacce->cdev, &uacce->dev);570}571EXPORT_SYMBOL_GPL(uacce_register);572 573/**574 * uacce_remove() - remove the accelerator575 * @uacce: the accelerator to remove576 */577void uacce_remove(struct uacce_device *uacce)578{579 struct uacce_queue *q, *next_q;580 581 if (!uacce)582 return;583 584 /*585 * uacce_fops_open() may be running concurrently, even after we remove586 * the cdev. Holding uacce->mutex ensures that open() does not obtain a587 * removed uacce device.588 */589 mutex_lock(&uacce->mutex);590 /* ensure no open queue remains */591 list_for_each_entry_safe(q, next_q, &uacce->queues, list) {592 /*593 * Taking q->mutex ensures that fops do not use the defunct594 * uacce->ops after the queue is disabled.595 */596 mutex_lock(&q->mutex);597 uacce_put_queue(q);598 mutex_unlock(&q->mutex);599 uacce_unbind_queue(q);600 601 /*602 * unmap remaining mapping from user space, preventing user still603 * access the mmaped area while parent device is already removed604 */605 unmap_mapping_range(q->mapping, 0, 0, 1);606 }607 608 /* disable sva now since no opened queues */609 uacce_disable_sva(uacce);610 611 if (uacce->cdev)612 cdev_device_del(uacce->cdev, &uacce->dev);613 xa_erase(&uacce_xa, uacce->dev_id);614 /*615 * uacce exists as long as there are open fds, but ops will be freed616 * now. Ensure that bugs cause NULL deref rather than use-after-free.617 */618 uacce->ops = NULL;619 uacce->parent = NULL;620 mutex_unlock(&uacce->mutex);621 put_device(&uacce->dev);622}623EXPORT_SYMBOL_GPL(uacce_remove);624 625static int __init uacce_init(void)626{627 int ret;628 629 ret = class_register(&uacce_class);630 if (ret)631 return ret;632 633 ret = alloc_chrdev_region(&uacce_devt, 0, MINORMASK, UACCE_NAME);634 if (ret)635 class_unregister(&uacce_class);636 637 return ret;638}639 640static __exit void uacce_exit(void)641{642 unregister_chrdev_region(uacce_devt, MINORMASK);643 class_unregister(&uacce_class);644}645 646subsys_initcall(uacce_init);647module_exit(uacce_exit);648 649MODULE_LICENSE("GPL");650MODULE_AUTHOR("HiSilicon Tech. Co., Ltd.");651MODULE_DESCRIPTION("Accelerator interface for Userland applications");652