2224 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2/*3 * VDUSE: vDPA Device in Userspace4 *5 * Copyright (C) 2020-2021 Bytedance Inc. and/or its affiliates. All rights reserved.6 *7 * Author: Xie Yongji <xieyongji@bytedance.com>8 *9 */10 11#include "linux/virtio_net.h"12#include <linux/init.h>13#include <linux/module.h>14#include <linux/cdev.h>15#include <linux/device.h>16#include <linux/eventfd.h>17#include <linux/slab.h>18#include <linux/wait.h>19#include <linux/dma-map-ops.h>20#include <linux/poll.h>21#include <linux/file.h>22#include <linux/uio.h>23#include <linux/vdpa.h>24#include <linux/nospec.h>25#include <linux/vmalloc.h>26#include <linux/sched/mm.h>27#include <uapi/linux/vduse.h>28#include <uapi/linux/vdpa.h>29#include <uapi/linux/virtio_config.h>30#include <uapi/linux/virtio_ids.h>31#include <uapi/linux/virtio_blk.h>32#include <uapi/linux/virtio_ring.h>33#include <linux/mod_devicetable.h>34 35#include "iova_domain.h"36 37#define DRV_AUTHOR "Yongji Xie <xieyongji@bytedance.com>"38#define DRV_DESC "vDPA Device in Userspace"39#define DRV_LICENSE "GPL v2"40 41#define VDUSE_DEV_MAX (1U << MINORBITS)42#define VDUSE_MAX_BOUNCE_SIZE (1024 * 1024 * 1024)43#define VDUSE_MIN_BOUNCE_SIZE (1024 * 1024)44#define VDUSE_BOUNCE_SIZE (64 * 1024 * 1024)45/* 128 MB reserved for virtqueue creation */46#define VDUSE_IOVA_SIZE (VDUSE_MAX_BOUNCE_SIZE + 128 * 1024 * 1024)47#define VDUSE_MSG_DEFAULT_TIMEOUT 3048 49#define IRQ_UNBOUND -150 51struct vduse_virtqueue {52 u16 index;53 u16 num_max;54 u32 num;55 u64 desc_addr;56 u64 driver_addr;57 u64 device_addr;58 struct vdpa_vq_state state;59 bool ready;60 bool kicked;61 spinlock_t kick_lock;62 spinlock_t irq_lock;63 struct eventfd_ctx *kickfd;64 struct vdpa_callback cb;65 struct work_struct inject;66 struct work_struct kick;67 int irq_effective_cpu;68 struct cpumask irq_affinity;69 struct kobject kobj;70};71 72struct vduse_dev;73 74struct vduse_vdpa {75 struct vdpa_device vdpa;76 struct vduse_dev *dev;77};78 79struct vduse_umem {80 unsigned long iova;81 unsigned long npages;82 struct page **pages;83 struct mm_struct *mm;84};85 86struct vduse_dev {87 struct vduse_vdpa *vdev;88 struct device *dev;89 struct vduse_virtqueue **vqs;90 struct vduse_iova_domain *domain;91 char *name;92 struct mutex lock;93 spinlock_t msg_lock;94 u64 msg_unique;95 u32 msg_timeout;96 wait_queue_head_t waitq;97 struct list_head send_list;98 struct list_head recv_list;99 struct vdpa_callback config_cb;100 struct work_struct inject;101 spinlock_t irq_lock;102 struct rw_semaphore rwsem;103 int minor;104 bool broken;105 bool connected;106 u64 api_version;107 u64 device_features;108 u64 driver_features;109 u32 device_id;110 u32 vendor_id;111 u32 generation;112 u32 config_size;113 void *config;114 u8 status;115 u32 vq_num;116 u32 vq_align;117 struct vduse_umem *umem;118 struct mutex mem_lock;119 unsigned int bounce_size;120 struct mutex domain_lock;121};122 123struct vduse_dev_msg {124 struct vduse_dev_request req;125 struct vduse_dev_response resp;126 struct list_head list;127 wait_queue_head_t waitq;128 bool completed;129};130 131struct vduse_control {132 u64 api_version;133};134 135static DEFINE_MUTEX(vduse_lock);136static DEFINE_IDR(vduse_idr);137 138static dev_t vduse_major;139static struct cdev vduse_ctrl_cdev;140static struct cdev vduse_cdev;141static struct workqueue_struct *vduse_irq_wq;142static struct workqueue_struct *vduse_irq_bound_wq;143 144static u32 allowed_device_id[] = {145 VIRTIO_ID_BLOCK,146 VIRTIO_ID_NET,147};148 149static inline struct vduse_dev *vdpa_to_vduse(struct vdpa_device *vdpa)150{151 struct vduse_vdpa *vdev = container_of(vdpa, struct vduse_vdpa, vdpa);152 153 return vdev->dev;154}155 156static inline struct vduse_dev *dev_to_vduse(struct device *dev)157{158 struct vdpa_device *vdpa = dev_to_vdpa(dev);159 160 return vdpa_to_vduse(vdpa);161}162 163static struct vduse_dev_msg *vduse_find_msg(struct list_head *head,164 uint32_t request_id)165{166 struct vduse_dev_msg *msg;167 168 list_for_each_entry(msg, head, list) {169 if (msg->req.request_id == request_id) {170 list_del(&msg->list);171 return msg;172 }173 }174 175 return NULL;176}177 178static struct vduse_dev_msg *vduse_dequeue_msg(struct list_head *head)179{180 struct vduse_dev_msg *msg = NULL;181 182 if (!list_empty(head)) {183 msg = list_first_entry(head, struct vduse_dev_msg, list);184 list_del(&msg->list);185 }186 187 return msg;188}189 190static void vduse_enqueue_msg(struct list_head *head,191 struct vduse_dev_msg *msg)192{193 list_add_tail(&msg->list, head);194}195 196static void vduse_dev_broken(struct vduse_dev *dev)197{198 struct vduse_dev_msg *msg, *tmp;199 200 if (unlikely(dev->broken))201 return;202 203 list_splice_init(&dev->recv_list, &dev->send_list);204 list_for_each_entry_safe(msg, tmp, &dev->send_list, list) {205 list_del(&msg->list);206 msg->completed = 1;207 msg->resp.result = VDUSE_REQ_RESULT_FAILED;208 wake_up(&msg->waitq);209 }210 dev->broken = true;211 wake_up(&dev->waitq);212}213 214static int vduse_dev_msg_sync(struct vduse_dev *dev,215 struct vduse_dev_msg *msg)216{217 int ret;218 219 if (unlikely(dev->broken))220 return -EIO;221 222 init_waitqueue_head(&msg->waitq);223 spin_lock(&dev->msg_lock);224 if (unlikely(dev->broken)) {225 spin_unlock(&dev->msg_lock);226 return -EIO;227 }228 msg->req.request_id = dev->msg_unique++;229 vduse_enqueue_msg(&dev->send_list, msg);230 wake_up(&dev->waitq);231 spin_unlock(&dev->msg_lock);232 if (dev->msg_timeout)233 ret = wait_event_killable_timeout(msg->waitq, msg->completed,234 (long)dev->msg_timeout * HZ);235 else236 ret = wait_event_killable(msg->waitq, msg->completed);237 238 spin_lock(&dev->msg_lock);239 if (!msg->completed) {240 list_del(&msg->list);241 msg->resp.result = VDUSE_REQ_RESULT_FAILED;242 /* Mark the device as malfunction when there is a timeout */243 if (!ret)244 vduse_dev_broken(dev);245 }246 ret = (msg->resp.result == VDUSE_REQ_RESULT_OK) ? 0 : -EIO;247 spin_unlock(&dev->msg_lock);248 249 return ret;250}251 252static int vduse_dev_get_vq_state_packed(struct vduse_dev *dev,253 struct vduse_virtqueue *vq,254 struct vdpa_vq_state_packed *packed)255{256 struct vduse_dev_msg msg = { 0 };257 int ret;258 259 msg.req.type = VDUSE_GET_VQ_STATE;260 msg.req.vq_state.index = vq->index;261 262 ret = vduse_dev_msg_sync(dev, &msg);263 if (ret)264 return ret;265 266 packed->last_avail_counter =267 msg.resp.vq_state.packed.last_avail_counter & 0x0001;268 packed->last_avail_idx =269 msg.resp.vq_state.packed.last_avail_idx & 0x7FFF;270 packed->last_used_counter =271 msg.resp.vq_state.packed.last_used_counter & 0x0001;272 packed->last_used_idx =273 msg.resp.vq_state.packed.last_used_idx & 0x7FFF;274 275 return 0;276}277 278static int vduse_dev_get_vq_state_split(struct vduse_dev *dev,279 struct vduse_virtqueue *vq,280 struct vdpa_vq_state_split *split)281{282 struct vduse_dev_msg msg = { 0 };283 int ret;284 285 msg.req.type = VDUSE_GET_VQ_STATE;286 msg.req.vq_state.index = vq->index;287 288 ret = vduse_dev_msg_sync(dev, &msg);289 if (ret)290 return ret;291 292 split->avail_index = msg.resp.vq_state.split.avail_index;293 294 return 0;295}296 297static int vduse_dev_set_status(struct vduse_dev *dev, u8 status)298{299 struct vduse_dev_msg msg = { 0 };300 301 msg.req.type = VDUSE_SET_STATUS;302 msg.req.s.status = status;303 304 return vduse_dev_msg_sync(dev, &msg);305}306 307static int vduse_dev_update_iotlb(struct vduse_dev *dev,308 u64 start, u64 last)309{310 struct vduse_dev_msg msg = { 0 };311 312 if (last < start)313 return -EINVAL;314 315 msg.req.type = VDUSE_UPDATE_IOTLB;316 msg.req.iova.start = start;317 msg.req.iova.last = last;318 319 return vduse_dev_msg_sync(dev, &msg);320}321 322static ssize_t vduse_dev_read_iter(struct kiocb *iocb, struct iov_iter *to)323{324 struct file *file = iocb->ki_filp;325 struct vduse_dev *dev = file->private_data;326 struct vduse_dev_msg *msg;327 int size = sizeof(struct vduse_dev_request);328 ssize_t ret;329 330 if (iov_iter_count(to) < size)331 return -EINVAL;332 333 spin_lock(&dev->msg_lock);334 while (1) {335 msg = vduse_dequeue_msg(&dev->send_list);336 if (msg)337 break;338 339 ret = -EAGAIN;340 if (file->f_flags & O_NONBLOCK)341 goto unlock;342 343 spin_unlock(&dev->msg_lock);344 ret = wait_event_interruptible_exclusive(dev->waitq,345 !list_empty(&dev->send_list));346 if (ret)347 return ret;348 349 spin_lock(&dev->msg_lock);350 }351 spin_unlock(&dev->msg_lock);352 ret = copy_to_iter(&msg->req, size, to);353 spin_lock(&dev->msg_lock);354 if (ret != size) {355 ret = -EFAULT;356 vduse_enqueue_msg(&dev->send_list, msg);357 goto unlock;358 }359 vduse_enqueue_msg(&dev->recv_list, msg);360unlock:361 spin_unlock(&dev->msg_lock);362 363 return ret;364}365 366static bool is_mem_zero(const char *ptr, int size)367{368 int i;369 370 for (i = 0; i < size; i++) {371 if (ptr[i])372 return false;373 }374 return true;375}376 377static ssize_t vduse_dev_write_iter(struct kiocb *iocb, struct iov_iter *from)378{379 struct file *file = iocb->ki_filp;380 struct vduse_dev *dev = file->private_data;381 struct vduse_dev_response resp;382 struct vduse_dev_msg *msg;383 size_t ret;384 385 ret = copy_from_iter(&resp, sizeof(resp), from);386 if (ret != sizeof(resp))387 return -EINVAL;388 389 if (!is_mem_zero((const char *)resp.reserved, sizeof(resp.reserved)))390 return -EINVAL;391 392 spin_lock(&dev->msg_lock);393 msg = vduse_find_msg(&dev->recv_list, resp.request_id);394 if (!msg) {395 ret = -ENOENT;396 goto unlock;397 }398 399 memcpy(&msg->resp, &resp, sizeof(resp));400 msg->completed = 1;401 wake_up(&msg->waitq);402unlock:403 spin_unlock(&dev->msg_lock);404 405 return ret;406}407 408static __poll_t vduse_dev_poll(struct file *file, poll_table *wait)409{410 struct vduse_dev *dev = file->private_data;411 __poll_t mask = 0;412 413 poll_wait(file, &dev->waitq, wait);414 415 spin_lock(&dev->msg_lock);416 417 if (unlikely(dev->broken))418 mask |= EPOLLERR;419 if (!list_empty(&dev->send_list))420 mask |= EPOLLIN | EPOLLRDNORM;421 if (!list_empty(&dev->recv_list))422 mask |= EPOLLOUT | EPOLLWRNORM;423 424 spin_unlock(&dev->msg_lock);425 426 return mask;427}428 429static void vduse_dev_reset(struct vduse_dev *dev)430{431 int i;432 struct vduse_iova_domain *domain = dev->domain;433 434 /* The coherent mappings are handled in vduse_dev_free_coherent() */435 if (domain && domain->bounce_map)436 vduse_domain_reset_bounce_map(domain);437 438 down_write(&dev->rwsem);439 440 dev->status = 0;441 dev->driver_features = 0;442 dev->generation++;443 spin_lock(&dev->irq_lock);444 dev->config_cb.callback = NULL;445 dev->config_cb.private = NULL;446 spin_unlock(&dev->irq_lock);447 flush_work(&dev->inject);448 449 for (i = 0; i < dev->vq_num; i++) {450 struct vduse_virtqueue *vq = dev->vqs[i];451 452 vq->ready = false;453 vq->desc_addr = 0;454 vq->driver_addr = 0;455 vq->device_addr = 0;456 vq->num = 0;457 memset(&vq->state, 0, sizeof(vq->state));458 459 spin_lock(&vq->kick_lock);460 vq->kicked = false;461 if (vq->kickfd)462 eventfd_ctx_put(vq->kickfd);463 vq->kickfd = NULL;464 spin_unlock(&vq->kick_lock);465 466 spin_lock(&vq->irq_lock);467 vq->cb.callback = NULL;468 vq->cb.private = NULL;469 vq->cb.trigger = NULL;470 spin_unlock(&vq->irq_lock);471 flush_work(&vq->inject);472 flush_work(&vq->kick);473 }474 475 up_write(&dev->rwsem);476}477 478static int vduse_vdpa_set_vq_address(struct vdpa_device *vdpa, u16 idx,479 u64 desc_area, u64 driver_area,480 u64 device_area)481{482 struct vduse_dev *dev = vdpa_to_vduse(vdpa);483 struct vduse_virtqueue *vq = dev->vqs[idx];484 485 vq->desc_addr = desc_area;486 vq->driver_addr = driver_area;487 vq->device_addr = device_area;488 489 return 0;490}491 492static void vduse_vq_kick(struct vduse_virtqueue *vq)493{494 spin_lock(&vq->kick_lock);495 if (!vq->ready)496 goto unlock;497 498 if (vq->kickfd)499 eventfd_signal(vq->kickfd);500 else501 vq->kicked = true;502unlock:503 spin_unlock(&vq->kick_lock);504}505 506static void vduse_vq_kick_work(struct work_struct *work)507{508 struct vduse_virtqueue *vq = container_of(work,509 struct vduse_virtqueue, kick);510 511 vduse_vq_kick(vq);512}513 514static void vduse_vdpa_kick_vq(struct vdpa_device *vdpa, u16 idx)515{516 struct vduse_dev *dev = vdpa_to_vduse(vdpa);517 struct vduse_virtqueue *vq = dev->vqs[idx];518 519 if (!eventfd_signal_allowed()) {520 schedule_work(&vq->kick);521 return;522 }523 vduse_vq_kick(vq);524}525 526static void vduse_vdpa_set_vq_cb(struct vdpa_device *vdpa, u16 idx,527 struct vdpa_callback *cb)528{529 struct vduse_dev *dev = vdpa_to_vduse(vdpa);530 struct vduse_virtqueue *vq = dev->vqs[idx];531 532 spin_lock(&vq->irq_lock);533 vq->cb.callback = cb->callback;534 vq->cb.private = cb->private;535 vq->cb.trigger = cb->trigger;536 spin_unlock(&vq->irq_lock);537}538 539static void vduse_vdpa_set_vq_num(struct vdpa_device *vdpa, u16 idx, u32 num)540{541 struct vduse_dev *dev = vdpa_to_vduse(vdpa);542 struct vduse_virtqueue *vq = dev->vqs[idx];543 544 vq->num = num;545}546 547static u16 vduse_vdpa_get_vq_size(struct vdpa_device *vdpa, u16 idx)548{549 struct vduse_dev *dev = vdpa_to_vduse(vdpa);550 struct vduse_virtqueue *vq = dev->vqs[idx];551 552 if (vq->num)553 return vq->num;554 else555 return vq->num_max;556}557 558static void vduse_vdpa_set_vq_ready(struct vdpa_device *vdpa,559 u16 idx, bool ready)560{561 struct vduse_dev *dev = vdpa_to_vduse(vdpa);562 struct vduse_virtqueue *vq = dev->vqs[idx];563 564 vq->ready = ready;565}566 567static bool vduse_vdpa_get_vq_ready(struct vdpa_device *vdpa, u16 idx)568{569 struct vduse_dev *dev = vdpa_to_vduse(vdpa);570 struct vduse_virtqueue *vq = dev->vqs[idx];571 572 return vq->ready;573}574 575static int vduse_vdpa_set_vq_state(struct vdpa_device *vdpa, u16 idx,576 const struct vdpa_vq_state *state)577{578 struct vduse_dev *dev = vdpa_to_vduse(vdpa);579 struct vduse_virtqueue *vq = dev->vqs[idx];580 581 if (dev->driver_features & BIT_ULL(VIRTIO_F_RING_PACKED)) {582 vq->state.packed.last_avail_counter =583 state->packed.last_avail_counter;584 vq->state.packed.last_avail_idx = state->packed.last_avail_idx;585 vq->state.packed.last_used_counter =586 state->packed.last_used_counter;587 vq->state.packed.last_used_idx = state->packed.last_used_idx;588 } else589 vq->state.split.avail_index = state->split.avail_index;590 591 return 0;592}593 594static int vduse_vdpa_get_vq_state(struct vdpa_device *vdpa, u16 idx,595 struct vdpa_vq_state *state)596{597 struct vduse_dev *dev = vdpa_to_vduse(vdpa);598 struct vduse_virtqueue *vq = dev->vqs[idx];599 600 if (dev->driver_features & BIT_ULL(VIRTIO_F_RING_PACKED))601 return vduse_dev_get_vq_state_packed(dev, vq, &state->packed);602 603 return vduse_dev_get_vq_state_split(dev, vq, &state->split);604}605 606static u32 vduse_vdpa_get_vq_align(struct vdpa_device *vdpa)607{608 struct vduse_dev *dev = vdpa_to_vduse(vdpa);609 610 return dev->vq_align;611}612 613static u64 vduse_vdpa_get_device_features(struct vdpa_device *vdpa)614{615 struct vduse_dev *dev = vdpa_to_vduse(vdpa);616 617 return dev->device_features;618}619 620static int vduse_vdpa_set_driver_features(struct vdpa_device *vdpa, u64 features)621{622 struct vduse_dev *dev = vdpa_to_vduse(vdpa);623 624 dev->driver_features = features;625 return 0;626}627 628static u64 vduse_vdpa_get_driver_features(struct vdpa_device *vdpa)629{630 struct vduse_dev *dev = vdpa_to_vduse(vdpa);631 632 return dev->driver_features;633}634 635static void vduse_vdpa_set_config_cb(struct vdpa_device *vdpa,636 struct vdpa_callback *cb)637{638 struct vduse_dev *dev = vdpa_to_vduse(vdpa);639 640 spin_lock(&dev->irq_lock);641 dev->config_cb.callback = cb->callback;642 dev->config_cb.private = cb->private;643 spin_unlock(&dev->irq_lock);644}645 646static u16 vduse_vdpa_get_vq_num_max(struct vdpa_device *vdpa)647{648 struct vduse_dev *dev = vdpa_to_vduse(vdpa);649 u16 num_max = 0;650 int i;651 652 for (i = 0; i < dev->vq_num; i++)653 if (num_max < dev->vqs[i]->num_max)654 num_max = dev->vqs[i]->num_max;655 656 return num_max;657}658 659static u32 vduse_vdpa_get_device_id(struct vdpa_device *vdpa)660{661 struct vduse_dev *dev = vdpa_to_vduse(vdpa);662 663 return dev->device_id;664}665 666static u32 vduse_vdpa_get_vendor_id(struct vdpa_device *vdpa)667{668 struct vduse_dev *dev = vdpa_to_vduse(vdpa);669 670 return dev->vendor_id;671}672 673static u8 vduse_vdpa_get_status(struct vdpa_device *vdpa)674{675 struct vduse_dev *dev = vdpa_to_vduse(vdpa);676 677 return dev->status;678}679 680static void vduse_vdpa_set_status(struct vdpa_device *vdpa, u8 status)681{682 struct vduse_dev *dev = vdpa_to_vduse(vdpa);683 684 if (vduse_dev_set_status(dev, status))685 return;686 687 dev->status = status;688}689 690static size_t vduse_vdpa_get_config_size(struct vdpa_device *vdpa)691{692 struct vduse_dev *dev = vdpa_to_vduse(vdpa);693 694 return dev->config_size;695}696 697static void vduse_vdpa_get_config(struct vdpa_device *vdpa, unsigned int offset,698 void *buf, unsigned int len)699{700 struct vduse_dev *dev = vdpa_to_vduse(vdpa);701 702 /* Initialize the buffer in case of partial copy. */703 memset(buf, 0, len);704 705 if (offset > dev->config_size)706 return;707 708 if (len > dev->config_size - offset)709 len = dev->config_size - offset;710 711 memcpy(buf, dev->config + offset, len);712}713 714static void vduse_vdpa_set_config(struct vdpa_device *vdpa, unsigned int offset,715 const void *buf, unsigned int len)716{717 /* Now we only support read-only configuration space */718}719 720static int vduse_vdpa_reset(struct vdpa_device *vdpa)721{722 struct vduse_dev *dev = vdpa_to_vduse(vdpa);723 int ret = vduse_dev_set_status(dev, 0);724 725 vduse_dev_reset(dev);726 727 return ret;728}729 730static u32 vduse_vdpa_get_generation(struct vdpa_device *vdpa)731{732 struct vduse_dev *dev = vdpa_to_vduse(vdpa);733 734 return dev->generation;735}736 737static int vduse_vdpa_set_vq_affinity(struct vdpa_device *vdpa, u16 idx,738 const struct cpumask *cpu_mask)739{740 struct vduse_dev *dev = vdpa_to_vduse(vdpa);741 742 if (cpu_mask)743 cpumask_copy(&dev->vqs[idx]->irq_affinity, cpu_mask);744 else745 cpumask_setall(&dev->vqs[idx]->irq_affinity);746 747 return 0;748}749 750static const struct cpumask *751vduse_vdpa_get_vq_affinity(struct vdpa_device *vdpa, u16 idx)752{753 struct vduse_dev *dev = vdpa_to_vduse(vdpa);754 755 return &dev->vqs[idx]->irq_affinity;756}757 758static int vduse_vdpa_set_map(struct vdpa_device *vdpa,759 unsigned int asid,760 struct vhost_iotlb *iotlb)761{762 struct vduse_dev *dev = vdpa_to_vduse(vdpa);763 int ret;764 765 ret = vduse_domain_set_map(dev->domain, iotlb);766 if (ret)767 return ret;768 769 ret = vduse_dev_update_iotlb(dev, 0ULL, ULLONG_MAX);770 if (ret) {771 vduse_domain_clear_map(dev->domain, iotlb);772 return ret;773 }774 775 return 0;776}777 778static void vduse_vdpa_free(struct vdpa_device *vdpa)779{780 struct vduse_dev *dev = vdpa_to_vduse(vdpa);781 782 dev->vdev = NULL;783}784 785static const struct vdpa_config_ops vduse_vdpa_config_ops = {786 .set_vq_address = vduse_vdpa_set_vq_address,787 .kick_vq = vduse_vdpa_kick_vq,788 .set_vq_cb = vduse_vdpa_set_vq_cb,789 .set_vq_num = vduse_vdpa_set_vq_num,790 .get_vq_size = vduse_vdpa_get_vq_size,791 .set_vq_ready = vduse_vdpa_set_vq_ready,792 .get_vq_ready = vduse_vdpa_get_vq_ready,793 .set_vq_state = vduse_vdpa_set_vq_state,794 .get_vq_state = vduse_vdpa_get_vq_state,795 .get_vq_align = vduse_vdpa_get_vq_align,796 .get_device_features = vduse_vdpa_get_device_features,797 .set_driver_features = vduse_vdpa_set_driver_features,798 .get_driver_features = vduse_vdpa_get_driver_features,799 .set_config_cb = vduse_vdpa_set_config_cb,800 .get_vq_num_max = vduse_vdpa_get_vq_num_max,801 .get_device_id = vduse_vdpa_get_device_id,802 .get_vendor_id = vduse_vdpa_get_vendor_id,803 .get_status = vduse_vdpa_get_status,804 .set_status = vduse_vdpa_set_status,805 .get_config_size = vduse_vdpa_get_config_size,806 .get_config = vduse_vdpa_get_config,807 .set_config = vduse_vdpa_set_config,808 .get_generation = vduse_vdpa_get_generation,809 .set_vq_affinity = vduse_vdpa_set_vq_affinity,810 .get_vq_affinity = vduse_vdpa_get_vq_affinity,811 .reset = vduse_vdpa_reset,812 .set_map = vduse_vdpa_set_map,813 .free = vduse_vdpa_free,814};815 816static void vduse_dev_sync_single_for_device(struct device *dev,817 dma_addr_t dma_addr, size_t size,818 enum dma_data_direction dir)819{820 struct vduse_dev *vdev = dev_to_vduse(dev);821 struct vduse_iova_domain *domain = vdev->domain;822 823 vduse_domain_sync_single_for_device(domain, dma_addr, size, dir);824}825 826static void vduse_dev_sync_single_for_cpu(struct device *dev,827 dma_addr_t dma_addr, size_t size,828 enum dma_data_direction dir)829{830 struct vduse_dev *vdev = dev_to_vduse(dev);831 struct vduse_iova_domain *domain = vdev->domain;832 833 vduse_domain_sync_single_for_cpu(domain, dma_addr, size, dir);834}835 836static dma_addr_t vduse_dev_map_page(struct device *dev, struct page *page,837 unsigned long offset, size_t size,838 enum dma_data_direction dir,839 unsigned long attrs)840{841 struct vduse_dev *vdev = dev_to_vduse(dev);842 struct vduse_iova_domain *domain = vdev->domain;843 844 return vduse_domain_map_page(domain, page, offset, size, dir, attrs);845}846 847static void vduse_dev_unmap_page(struct device *dev, dma_addr_t dma_addr,848 size_t size, enum dma_data_direction dir,849 unsigned long attrs)850{851 struct vduse_dev *vdev = dev_to_vduse(dev);852 struct vduse_iova_domain *domain = vdev->domain;853 854 return vduse_domain_unmap_page(domain, dma_addr, size, dir, attrs);855}856 857static void *vduse_dev_alloc_coherent(struct device *dev, size_t size,858 dma_addr_t *dma_addr, gfp_t flag,859 unsigned long attrs)860{861 struct vduse_dev *vdev = dev_to_vduse(dev);862 struct vduse_iova_domain *domain = vdev->domain;863 unsigned long iova;864 void *addr;865 866 *dma_addr = DMA_MAPPING_ERROR;867 addr = vduse_domain_alloc_coherent(domain, size,868 (dma_addr_t *)&iova, flag, attrs);869 if (!addr)870 return NULL;871 872 *dma_addr = (dma_addr_t)iova;873 874 return addr;875}876 877static void vduse_dev_free_coherent(struct device *dev, size_t size,878 void *vaddr, dma_addr_t dma_addr,879 unsigned long attrs)880{881 struct vduse_dev *vdev = dev_to_vduse(dev);882 struct vduse_iova_domain *domain = vdev->domain;883 884 vduse_domain_free_coherent(domain, size, vaddr, dma_addr, attrs);885}886 887static size_t vduse_dev_max_mapping_size(struct device *dev)888{889 struct vduse_dev *vdev = dev_to_vduse(dev);890 struct vduse_iova_domain *domain = vdev->domain;891 892 return domain->bounce_size;893}894 895static const struct dma_map_ops vduse_dev_dma_ops = {896 .sync_single_for_device = vduse_dev_sync_single_for_device,897 .sync_single_for_cpu = vduse_dev_sync_single_for_cpu,898 .map_page = vduse_dev_map_page,899 .unmap_page = vduse_dev_unmap_page,900 .alloc = vduse_dev_alloc_coherent,901 .free = vduse_dev_free_coherent,902 .max_mapping_size = vduse_dev_max_mapping_size,903};904 905static unsigned int perm_to_file_flags(u8 perm)906{907 unsigned int flags = 0;908 909 switch (perm) {910 case VDUSE_ACCESS_WO:911 flags |= O_WRONLY;912 break;913 case VDUSE_ACCESS_RO:914 flags |= O_RDONLY;915 break;916 case VDUSE_ACCESS_RW:917 flags |= O_RDWR;918 break;919 default:920 WARN(1, "invalidate vhost IOTLB permission\n");921 break;922 }923 924 return flags;925}926 927static int vduse_kickfd_setup(struct vduse_dev *dev,928 struct vduse_vq_eventfd *eventfd)929{930 struct eventfd_ctx *ctx = NULL;931 struct vduse_virtqueue *vq;932 u32 index;933 934 if (eventfd->index >= dev->vq_num)935 return -EINVAL;936 937 index = array_index_nospec(eventfd->index, dev->vq_num);938 vq = dev->vqs[index];939 if (eventfd->fd >= 0) {940 ctx = eventfd_ctx_fdget(eventfd->fd);941 if (IS_ERR(ctx))942 return PTR_ERR(ctx);943 } else if (eventfd->fd != VDUSE_EVENTFD_DEASSIGN)944 return 0;945 946 spin_lock(&vq->kick_lock);947 if (vq->kickfd)948 eventfd_ctx_put(vq->kickfd);949 vq->kickfd = ctx;950 if (vq->ready && vq->kicked && vq->kickfd) {951 eventfd_signal(vq->kickfd);952 vq->kicked = false;953 }954 spin_unlock(&vq->kick_lock);955 956 return 0;957}958 959static bool vduse_dev_is_ready(struct vduse_dev *dev)960{961 int i;962 963 for (i = 0; i < dev->vq_num; i++)964 if (!dev->vqs[i]->num_max)965 return false;966 967 return true;968}969 970static void vduse_dev_irq_inject(struct work_struct *work)971{972 struct vduse_dev *dev = container_of(work, struct vduse_dev, inject);973 974 spin_lock_bh(&dev->irq_lock);975 if (dev->config_cb.callback)976 dev->config_cb.callback(dev->config_cb.private);977 spin_unlock_bh(&dev->irq_lock);978}979 980static void vduse_vq_irq_inject(struct work_struct *work)981{982 struct vduse_virtqueue *vq = container_of(work,983 struct vduse_virtqueue, inject);984 985 spin_lock_bh(&vq->irq_lock);986 if (vq->ready && vq->cb.callback)987 vq->cb.callback(vq->cb.private);988 spin_unlock_bh(&vq->irq_lock);989}990 991static bool vduse_vq_signal_irqfd(struct vduse_virtqueue *vq)992{993 bool signal = false;994 995 if (!vq->cb.trigger)996 return false;997 998 spin_lock_irq(&vq->irq_lock);999 if (vq->ready && vq->cb.trigger) {1000 eventfd_signal(vq->cb.trigger);1001 signal = true;1002 }1003 spin_unlock_irq(&vq->irq_lock);1004 1005 return signal;1006}1007 1008static int vduse_dev_queue_irq_work(struct vduse_dev *dev,1009 struct work_struct *irq_work,1010 int irq_effective_cpu)1011{1012 int ret = -EINVAL;1013 1014 down_read(&dev->rwsem);1015 if (!(dev->status & VIRTIO_CONFIG_S_DRIVER_OK))1016 goto unlock;1017 1018 ret = 0;1019 if (irq_effective_cpu == IRQ_UNBOUND)1020 queue_work(vduse_irq_wq, irq_work);1021 else1022 queue_work_on(irq_effective_cpu,1023 vduse_irq_bound_wq, irq_work);1024unlock:1025 up_read(&dev->rwsem);1026 1027 return ret;1028}1029 1030static int vduse_dev_dereg_umem(struct vduse_dev *dev,1031 u64 iova, u64 size)1032{1033 int ret;1034 1035 mutex_lock(&dev->mem_lock);1036 ret = -ENOENT;1037 if (!dev->umem)1038 goto unlock;1039 1040 ret = -EINVAL;1041 if (!dev->domain)1042 goto unlock;1043 1044 if (dev->umem->iova != iova || size != dev->domain->bounce_size)1045 goto unlock;1046 1047 vduse_domain_remove_user_bounce_pages(dev->domain);1048 unpin_user_pages_dirty_lock(dev->umem->pages,1049 dev->umem->npages, true);1050 atomic64_sub(dev->umem->npages, &dev->umem->mm->pinned_vm);1051 mmdrop(dev->umem->mm);1052 vfree(dev->umem->pages);1053 kfree(dev->umem);1054 dev->umem = NULL;1055 ret = 0;1056unlock:1057 mutex_unlock(&dev->mem_lock);1058 return ret;1059}1060 1061static int vduse_dev_reg_umem(struct vduse_dev *dev,1062 u64 iova, u64 uaddr, u64 size)1063{1064 struct page **page_list = NULL;1065 struct vduse_umem *umem = NULL;1066 long pinned = 0;1067 unsigned long npages, lock_limit;1068 int ret;1069 1070 if (!dev->domain || !dev->domain->bounce_map ||1071 size != dev->domain->bounce_size ||1072 iova != 0 || uaddr & ~PAGE_MASK)1073 return -EINVAL;1074 1075 mutex_lock(&dev->mem_lock);1076 ret = -EEXIST;1077 if (dev->umem)1078 goto unlock;1079 1080 ret = -ENOMEM;1081 npages = size >> PAGE_SHIFT;1082 page_list = __vmalloc(array_size(npages, sizeof(struct page *)),1083 GFP_KERNEL_ACCOUNT);1084 umem = kzalloc(sizeof(*umem), GFP_KERNEL);1085 if (!page_list || !umem)1086 goto unlock;1087 1088 mmap_read_lock(current->mm);1089 1090 lock_limit = PFN_DOWN(rlimit(RLIMIT_MEMLOCK));1091 if (npages + atomic64_read(¤t->mm->pinned_vm) > lock_limit)1092 goto out;1093 1094 pinned = pin_user_pages(uaddr, npages, FOLL_LONGTERM | FOLL_WRITE,1095 page_list);1096 if (pinned != npages) {1097 ret = pinned < 0 ? pinned : -ENOMEM;1098 goto out;1099 }1100 1101 ret = vduse_domain_add_user_bounce_pages(dev->domain,1102 page_list, pinned);1103 if (ret)1104 goto out;1105 1106 atomic64_add(npages, ¤t->mm->pinned_vm);1107 1108 umem->pages = page_list;1109 umem->npages = pinned;1110 umem->iova = iova;1111 umem->mm = current->mm;1112 mmgrab(current->mm);1113 1114 dev->umem = umem;1115out:1116 if (ret && pinned > 0)1117 unpin_user_pages(page_list, pinned);1118 1119 mmap_read_unlock(current->mm);1120unlock:1121 if (ret) {1122 vfree(page_list);1123 kfree(umem);1124 }1125 mutex_unlock(&dev->mem_lock);1126 return ret;1127}1128 1129static void vduse_vq_update_effective_cpu(struct vduse_virtqueue *vq)1130{1131 int curr_cpu = vq->irq_effective_cpu;1132 1133 while (true) {1134 curr_cpu = cpumask_next(curr_cpu, &vq->irq_affinity);1135 if (cpu_online(curr_cpu))1136 break;1137 1138 if (curr_cpu >= nr_cpu_ids)1139 curr_cpu = IRQ_UNBOUND;1140 }1141 1142 vq->irq_effective_cpu = curr_cpu;1143}1144 1145static long vduse_dev_ioctl(struct file *file, unsigned int cmd,1146 unsigned long arg)1147{1148 struct vduse_dev *dev = file->private_data;1149 void __user *argp = (void __user *)arg;1150 int ret;1151 1152 if (unlikely(dev->broken))1153 return -EPERM;1154 1155 switch (cmd) {1156 case VDUSE_IOTLB_GET_FD: {1157 struct vduse_iotlb_entry entry;1158 struct vhost_iotlb_map *map;1159 struct vdpa_map_file *map_file;1160 struct file *f = NULL;1161 1162 ret = -EFAULT;1163 if (copy_from_user(&entry, argp, sizeof(entry)))1164 break;1165 1166 ret = -EINVAL;1167 if (entry.start > entry.last)1168 break;1169 1170 mutex_lock(&dev->domain_lock);1171 if (!dev->domain) {1172 mutex_unlock(&dev->domain_lock);1173 break;1174 }1175 spin_lock(&dev->domain->iotlb_lock);1176 map = vhost_iotlb_itree_first(dev->domain->iotlb,1177 entry.start, entry.last);1178 if (map) {1179 map_file = (struct vdpa_map_file *)map->opaque;1180 f = get_file(map_file->file);1181 entry.offset = map_file->offset;1182 entry.start = map->start;1183 entry.last = map->last;1184 entry.perm = map->perm;1185 }1186 spin_unlock(&dev->domain->iotlb_lock);1187 mutex_unlock(&dev->domain_lock);1188 ret = -EINVAL;1189 if (!f)1190 break;1191 1192 ret = -EFAULT;1193 if (copy_to_user(argp, &entry, sizeof(entry))) {1194 fput(f);1195 break;1196 }1197 ret = receive_fd(f, NULL, perm_to_file_flags(entry.perm));1198 fput(f);1199 break;1200 }1201 case VDUSE_DEV_GET_FEATURES:1202 /*1203 * Just mirror what driver wrote here.1204 * The driver is expected to check FEATURE_OK later.1205 */1206 ret = put_user(dev->driver_features, (u64 __user *)argp);1207 break;1208 case VDUSE_DEV_SET_CONFIG: {1209 struct vduse_config_data config;1210 unsigned long size = offsetof(struct vduse_config_data,1211 buffer);1212 1213 ret = -EFAULT;1214 if (copy_from_user(&config, argp, size))1215 break;1216 1217 ret = -EINVAL;1218 if (config.offset > dev->config_size ||1219 config.length == 0 ||1220 config.length > dev->config_size - config.offset)1221 break;1222 1223 ret = -EFAULT;1224 if (copy_from_user(dev->config + config.offset, argp + size,1225 config.length))1226 break;1227 1228 ret = 0;1229 break;1230 }1231 case VDUSE_DEV_INJECT_CONFIG_IRQ:1232 ret = vduse_dev_queue_irq_work(dev, &dev->inject, IRQ_UNBOUND);1233 break;1234 case VDUSE_VQ_SETUP: {1235 struct vduse_vq_config config;1236 u32 index;1237 1238 ret = -EFAULT;1239 if (copy_from_user(&config, argp, sizeof(config)))1240 break;1241 1242 ret = -EINVAL;1243 if (config.index >= dev->vq_num)1244 break;1245 1246 if (!is_mem_zero((const char *)config.reserved,1247 sizeof(config.reserved)))1248 break;1249 1250 index = array_index_nospec(config.index, dev->vq_num);1251 dev->vqs[index]->num_max = config.max_size;1252 ret = 0;1253 break;1254 }1255 case VDUSE_VQ_GET_INFO: {1256 struct vduse_vq_info vq_info;1257 struct vduse_virtqueue *vq;1258 u32 index;1259 1260 ret = -EFAULT;1261 if (copy_from_user(&vq_info, argp, sizeof(vq_info)))1262 break;1263 1264 ret = -EINVAL;1265 if (vq_info.index >= dev->vq_num)1266 break;1267 1268 index = array_index_nospec(vq_info.index, dev->vq_num);1269 vq = dev->vqs[index];1270 vq_info.desc_addr = vq->desc_addr;1271 vq_info.driver_addr = vq->driver_addr;1272 vq_info.device_addr = vq->device_addr;1273 vq_info.num = vq->num;1274 1275 if (dev->driver_features & BIT_ULL(VIRTIO_F_RING_PACKED)) {1276 vq_info.packed.last_avail_counter =1277 vq->state.packed.last_avail_counter;1278 vq_info.packed.last_avail_idx =1279 vq->state.packed.last_avail_idx;1280 vq_info.packed.last_used_counter =1281 vq->state.packed.last_used_counter;1282 vq_info.packed.last_used_idx =1283 vq->state.packed.last_used_idx;1284 } else1285 vq_info.split.avail_index =1286 vq->state.split.avail_index;1287 1288 vq_info.ready = vq->ready;1289 1290 ret = -EFAULT;1291 if (copy_to_user(argp, &vq_info, sizeof(vq_info)))1292 break;1293 1294 ret = 0;1295 break;1296 }1297 case VDUSE_VQ_SETUP_KICKFD: {1298 struct vduse_vq_eventfd eventfd;1299 1300 ret = -EFAULT;1301 if (copy_from_user(&eventfd, argp, sizeof(eventfd)))1302 break;1303 1304 ret = vduse_kickfd_setup(dev, &eventfd);1305 break;1306 }1307 case VDUSE_VQ_INJECT_IRQ: {1308 u32 index;1309 1310 ret = -EFAULT;1311 if (get_user(index, (u32 __user *)argp))1312 break;1313 1314 ret = -EINVAL;1315 if (index >= dev->vq_num)1316 break;1317 1318 ret = 0;1319 index = array_index_nospec(index, dev->vq_num);1320 if (!vduse_vq_signal_irqfd(dev->vqs[index])) {1321 vduse_vq_update_effective_cpu(dev->vqs[index]);1322 ret = vduse_dev_queue_irq_work(dev,1323 &dev->vqs[index]->inject,1324 dev->vqs[index]->irq_effective_cpu);1325 }1326 break;1327 }1328 case VDUSE_IOTLB_REG_UMEM: {1329 struct vduse_iova_umem umem;1330 1331 ret = -EFAULT;1332 if (copy_from_user(&umem, argp, sizeof(umem)))1333 break;1334 1335 ret = -EINVAL;1336 if (!is_mem_zero((const char *)umem.reserved,1337 sizeof(umem.reserved)))1338 break;1339 1340 mutex_lock(&dev->domain_lock);1341 ret = vduse_dev_reg_umem(dev, umem.iova,1342 umem.uaddr, umem.size);1343 mutex_unlock(&dev->domain_lock);1344 break;1345 }1346 case VDUSE_IOTLB_DEREG_UMEM: {1347 struct vduse_iova_umem umem;1348 1349 ret = -EFAULT;1350 if (copy_from_user(&umem, argp, sizeof(umem)))1351 break;1352 1353 ret = -EINVAL;1354 if (!is_mem_zero((const char *)umem.reserved,1355 sizeof(umem.reserved)))1356 break;1357 mutex_lock(&dev->domain_lock);1358 ret = vduse_dev_dereg_umem(dev, umem.iova,1359 umem.size);1360 mutex_unlock(&dev->domain_lock);1361 break;1362 }1363 case VDUSE_IOTLB_GET_INFO: {1364 struct vduse_iova_info info;1365 struct vhost_iotlb_map *map;1366 1367 ret = -EFAULT;1368 if (copy_from_user(&info, argp, sizeof(info)))1369 break;1370 1371 ret = -EINVAL;1372 if (info.start > info.last)1373 break;1374 1375 if (!is_mem_zero((const char *)info.reserved,1376 sizeof(info.reserved)))1377 break;1378 1379 mutex_lock(&dev->domain_lock);1380 if (!dev->domain) {1381 mutex_unlock(&dev->domain_lock);1382 break;1383 }1384 spin_lock(&dev->domain->iotlb_lock);1385 map = vhost_iotlb_itree_first(dev->domain->iotlb,1386 info.start, info.last);1387 if (map) {1388 info.start = map->start;1389 info.last = map->last;1390 info.capability = 0;1391 if (dev->domain->bounce_map && map->start == 0 &&1392 map->last == dev->domain->bounce_size - 1)1393 info.capability |= VDUSE_IOVA_CAP_UMEM;1394 }1395 spin_unlock(&dev->domain->iotlb_lock);1396 mutex_unlock(&dev->domain_lock);1397 if (!map)1398 break;1399 1400 ret = -EFAULT;1401 if (copy_to_user(argp, &info, sizeof(info)))1402 break;1403 1404 ret = 0;1405 break;1406 }1407 default:1408 ret = -ENOIOCTLCMD;1409 break;1410 }1411 1412 return ret;1413}1414 1415static int vduse_dev_release(struct inode *inode, struct file *file)1416{1417 struct vduse_dev *dev = file->private_data;1418 1419 mutex_lock(&dev->domain_lock);1420 if (dev->domain)1421 vduse_dev_dereg_umem(dev, 0, dev->domain->bounce_size);1422 mutex_unlock(&dev->domain_lock);1423 spin_lock(&dev->msg_lock);1424 /* Make sure the inflight messages can processed after reconncection */1425 list_splice_init(&dev->recv_list, &dev->send_list);1426 spin_unlock(&dev->msg_lock);1427 dev->connected = false;1428 1429 return 0;1430}1431 1432static struct vduse_dev *vduse_dev_get_from_minor(int minor)1433{1434 struct vduse_dev *dev;1435 1436 mutex_lock(&vduse_lock);1437 dev = idr_find(&vduse_idr, minor);1438 mutex_unlock(&vduse_lock);1439 1440 return dev;1441}1442 1443static int vduse_dev_open(struct inode *inode, struct file *file)1444{1445 int ret;1446 struct vduse_dev *dev = vduse_dev_get_from_minor(iminor(inode));1447 1448 if (!dev)1449 return -ENODEV;1450 1451 ret = -EBUSY;1452 mutex_lock(&dev->lock);1453 if (dev->connected)1454 goto unlock;1455 1456 ret = 0;1457 dev->connected = true;1458 file->private_data = dev;1459unlock:1460 mutex_unlock(&dev->lock);1461 1462 return ret;1463}1464 1465static const struct file_operations vduse_dev_fops = {1466 .owner = THIS_MODULE,1467 .open = vduse_dev_open,1468 .release = vduse_dev_release,1469 .read_iter = vduse_dev_read_iter,1470 .write_iter = vduse_dev_write_iter,1471 .poll = vduse_dev_poll,1472 .unlocked_ioctl = vduse_dev_ioctl,1473 .compat_ioctl = compat_ptr_ioctl,1474 .llseek = noop_llseek,1475};1476 1477static ssize_t irq_cb_affinity_show(struct vduse_virtqueue *vq, char *buf)1478{1479 return sprintf(buf, "%*pb\n", cpumask_pr_args(&vq->irq_affinity));1480}1481 1482static ssize_t irq_cb_affinity_store(struct vduse_virtqueue *vq,1483 const char *buf, size_t count)1484{1485 cpumask_var_t new_value;1486 int ret;1487 1488 if (!zalloc_cpumask_var(&new_value, GFP_KERNEL))1489 return -ENOMEM;1490 1491 ret = cpumask_parse(buf, new_value);1492 if (ret)1493 goto free_mask;1494 1495 ret = -EINVAL;1496 if (!cpumask_intersects(new_value, cpu_online_mask))1497 goto free_mask;1498 1499 cpumask_copy(&vq->irq_affinity, new_value);1500 ret = count;1501free_mask:1502 free_cpumask_var(new_value);1503 return ret;1504}1505 1506struct vq_sysfs_entry {1507 struct attribute attr;1508 ssize_t (*show)(struct vduse_virtqueue *vq, char *buf);1509 ssize_t (*store)(struct vduse_virtqueue *vq, const char *buf,1510 size_t count);1511};1512 1513static struct vq_sysfs_entry irq_cb_affinity_attr = __ATTR_RW(irq_cb_affinity);1514 1515static struct attribute *vq_attrs[] = {1516 &irq_cb_affinity_attr.attr,1517 NULL,1518};1519ATTRIBUTE_GROUPS(vq);1520 1521static ssize_t vq_attr_show(struct kobject *kobj, struct attribute *attr,1522 char *buf)1523{1524 struct vduse_virtqueue *vq = container_of(kobj,1525 struct vduse_virtqueue, kobj);1526 struct vq_sysfs_entry *entry = container_of(attr,1527 struct vq_sysfs_entry, attr);1528 1529 if (!entry->show)1530 return -EIO;1531 1532 return entry->show(vq, buf);1533}1534 1535static ssize_t vq_attr_store(struct kobject *kobj, struct attribute *attr,1536 const char *buf, size_t count)1537{1538 struct vduse_virtqueue *vq = container_of(kobj,1539 struct vduse_virtqueue, kobj);1540 struct vq_sysfs_entry *entry = container_of(attr,1541 struct vq_sysfs_entry, attr);1542 1543 if (!entry->store)1544 return -EIO;1545 1546 return entry->store(vq, buf, count);1547}1548 1549static const struct sysfs_ops vq_sysfs_ops = {1550 .show = vq_attr_show,1551 .store = vq_attr_store,1552};1553 1554static void vq_release(struct kobject *kobj)1555{1556 struct vduse_virtqueue *vq = container_of(kobj,1557 struct vduse_virtqueue, kobj);1558 kfree(vq);1559}1560 1561static const struct kobj_type vq_type = {1562 .release = vq_release,1563 .sysfs_ops = &vq_sysfs_ops,1564 .default_groups = vq_groups,1565};1566 1567static char *vduse_devnode(const struct device *dev, umode_t *mode)1568{1569 return kasprintf(GFP_KERNEL, "vduse/%s", dev_name(dev));1570}1571 1572static const struct class vduse_class = {1573 .name = "vduse",1574 .devnode = vduse_devnode,1575};1576 1577static void vduse_dev_deinit_vqs(struct vduse_dev *dev)1578{1579 int i;1580 1581 if (!dev->vqs)1582 return;1583 1584 for (i = 0; i < dev->vq_num; i++)1585 kobject_put(&dev->vqs[i]->kobj);1586 kfree(dev->vqs);1587}1588 1589static int vduse_dev_init_vqs(struct vduse_dev *dev, u32 vq_align, u32 vq_num)1590{1591 int ret, i;1592 1593 dev->vq_align = vq_align;1594 dev->vq_num = vq_num;1595 dev->vqs = kcalloc(dev->vq_num, sizeof(*dev->vqs), GFP_KERNEL);1596 if (!dev->vqs)1597 return -ENOMEM;1598 1599 for (i = 0; i < vq_num; i++) {1600 dev->vqs[i] = kzalloc(sizeof(*dev->vqs[i]), GFP_KERNEL);1601 if (!dev->vqs[i]) {1602 ret = -ENOMEM;1603 goto err;1604 }1605 1606 dev->vqs[i]->index = i;1607 dev->vqs[i]->irq_effective_cpu = IRQ_UNBOUND;1608 INIT_WORK(&dev->vqs[i]->inject, vduse_vq_irq_inject);1609 INIT_WORK(&dev->vqs[i]->kick, vduse_vq_kick_work);1610 spin_lock_init(&dev->vqs[i]->kick_lock);1611 spin_lock_init(&dev->vqs[i]->irq_lock);1612 cpumask_setall(&dev->vqs[i]->irq_affinity);1613 1614 kobject_init(&dev->vqs[i]->kobj, &vq_type);1615 ret = kobject_add(&dev->vqs[i]->kobj,1616 &dev->dev->kobj, "vq%d", i);1617 if (ret) {1618 kfree(dev->vqs[i]);1619 goto err;1620 }1621 }1622 1623 return 0;1624err:1625 while (i--)1626 kobject_put(&dev->vqs[i]->kobj);1627 kfree(dev->vqs);1628 dev->vqs = NULL;1629 return ret;1630}1631 1632static struct vduse_dev *vduse_dev_create(void)1633{1634 struct vduse_dev *dev = kzalloc(sizeof(*dev), GFP_KERNEL);1635 1636 if (!dev)1637 return NULL;1638 1639 mutex_init(&dev->lock);1640 mutex_init(&dev->mem_lock);1641 mutex_init(&dev->domain_lock);1642 spin_lock_init(&dev->msg_lock);1643 INIT_LIST_HEAD(&dev->send_list);1644 INIT_LIST_HEAD(&dev->recv_list);1645 spin_lock_init(&dev->irq_lock);1646 init_rwsem(&dev->rwsem);1647 1648 INIT_WORK(&dev->inject, vduse_dev_irq_inject);1649 init_waitqueue_head(&dev->waitq);1650 1651 return dev;1652}1653 1654static void vduse_dev_destroy(struct vduse_dev *dev)1655{1656 kfree(dev);1657}1658 1659static struct vduse_dev *vduse_find_dev(const char *name)1660{1661 struct vduse_dev *dev;1662 int id;1663 1664 idr_for_each_entry(&vduse_idr, dev, id)1665 if (!strcmp(dev->name, name))1666 return dev;1667 1668 return NULL;1669}1670 1671static int vduse_destroy_dev(char *name)1672{1673 struct vduse_dev *dev = vduse_find_dev(name);1674 1675 if (!dev)1676 return -EINVAL;1677 1678 mutex_lock(&dev->lock);1679 if (dev->vdev || dev->connected) {1680 mutex_unlock(&dev->lock);1681 return -EBUSY;1682 }1683 dev->connected = true;1684 mutex_unlock(&dev->lock);1685 1686 vduse_dev_reset(dev);1687 device_destroy(&vduse_class, MKDEV(MAJOR(vduse_major), dev->minor));1688 idr_remove(&vduse_idr, dev->minor);1689 kvfree(dev->config);1690 vduse_dev_deinit_vqs(dev);1691 if (dev->domain)1692 vduse_domain_destroy(dev->domain);1693 kfree(dev->name);1694 vduse_dev_destroy(dev);1695 module_put(THIS_MODULE);1696 1697 return 0;1698}1699 1700static bool device_is_allowed(u32 device_id)1701{1702 int i;1703 1704 for (i = 0; i < ARRAY_SIZE(allowed_device_id); i++)1705 if (allowed_device_id[i] == device_id)1706 return true;1707 1708 return false;1709}1710 1711static bool features_is_valid(struct vduse_dev_config *config)1712{1713 if (!(config->features & BIT_ULL(VIRTIO_F_ACCESS_PLATFORM)))1714 return false;1715 1716 /* Now we only support read-only configuration space */1717 if ((config->device_id == VIRTIO_ID_BLOCK) &&1718 (config->features & BIT_ULL(VIRTIO_BLK_F_CONFIG_WCE)))1719 return false;1720 else if ((config->device_id == VIRTIO_ID_NET) &&1721 (config->features & BIT_ULL(VIRTIO_NET_F_CTRL_VQ)))1722 return false;1723 1724 if ((config->device_id == VIRTIO_ID_NET) &&1725 !(config->features & BIT_ULL(VIRTIO_F_VERSION_1)))1726 return false;1727 1728 return true;1729}1730 1731static bool vduse_validate_config(struct vduse_dev_config *config)1732{1733 if (!is_mem_zero((const char *)config->reserved,1734 sizeof(config->reserved)))1735 return false;1736 1737 if (config->vq_align > PAGE_SIZE)1738 return false;1739 1740 if (config->config_size > PAGE_SIZE)1741 return false;1742 1743 if (config->vq_num > 0xffff)1744 return false;1745 1746 if (!config->name[0])1747 return false;1748 1749 if (!device_is_allowed(config->device_id))1750 return false;1751 1752 if (!features_is_valid(config))1753 return false;1754 1755 return true;1756}1757 1758static ssize_t msg_timeout_show(struct device *device,1759 struct device_attribute *attr, char *buf)1760{1761 struct vduse_dev *dev = dev_get_drvdata(device);1762 1763 return sysfs_emit(buf, "%u\n", dev->msg_timeout);1764}1765 1766static ssize_t msg_timeout_store(struct device *device,1767 struct device_attribute *attr,1768 const char *buf, size_t count)1769{1770 struct vduse_dev *dev = dev_get_drvdata(device);1771 int ret;1772 1773 ret = kstrtouint(buf, 10, &dev->msg_timeout);1774 if (ret < 0)1775 return ret;1776 1777 return count;1778}1779 1780static DEVICE_ATTR_RW(msg_timeout);1781 1782static ssize_t bounce_size_show(struct device *device,1783 struct device_attribute *attr, char *buf)1784{1785 struct vduse_dev *dev = dev_get_drvdata(device);1786 1787 return sysfs_emit(buf, "%u\n", dev->bounce_size);1788}1789 1790static ssize_t bounce_size_store(struct device *device,1791 struct device_attribute *attr,1792 const char *buf, size_t count)1793{1794 struct vduse_dev *dev = dev_get_drvdata(device);1795 unsigned int bounce_size;1796 int ret;1797 1798 ret = -EPERM;1799 mutex_lock(&dev->domain_lock);1800 if (dev->domain)1801 goto unlock;1802 1803 ret = kstrtouint(buf, 10, &bounce_size);1804 if (ret < 0)1805 goto unlock;1806 1807 ret = -EINVAL;1808 if (bounce_size > VDUSE_MAX_BOUNCE_SIZE ||1809 bounce_size < VDUSE_MIN_BOUNCE_SIZE)1810 goto unlock;1811 1812 dev->bounce_size = bounce_size & PAGE_MASK;1813 ret = count;1814unlock:1815 mutex_unlock(&dev->domain_lock);1816 return ret;1817}1818 1819static DEVICE_ATTR_RW(bounce_size);1820 1821static struct attribute *vduse_dev_attrs[] = {1822 &dev_attr_msg_timeout.attr,1823 &dev_attr_bounce_size.attr,1824 NULL1825};1826 1827ATTRIBUTE_GROUPS(vduse_dev);1828 1829static int vduse_create_dev(struct vduse_dev_config *config,1830 void *config_buf, u64 api_version)1831{1832 int ret;1833 struct vduse_dev *dev;1834 1835 ret = -EPERM;1836 if ((config->device_id == VIRTIO_ID_NET) && !capable(CAP_NET_ADMIN))1837 goto err;1838 1839 ret = -EEXIST;1840 if (vduse_find_dev(config->name))1841 goto err;1842 1843 ret = -ENOMEM;1844 dev = vduse_dev_create();1845 if (!dev)1846 goto err;1847 1848 dev->api_version = api_version;1849 dev->device_features = config->features;1850 dev->device_id = config->device_id;1851 dev->vendor_id = config->vendor_id;1852 dev->name = kstrdup(config->name, GFP_KERNEL);1853 if (!dev->name)1854 goto err_str;1855 1856 dev->bounce_size = VDUSE_BOUNCE_SIZE;1857 dev->config = config_buf;1858 dev->config_size = config->config_size;1859 1860 ret = idr_alloc(&vduse_idr, dev, 1, VDUSE_DEV_MAX, GFP_KERNEL);1861 if (ret < 0)1862 goto err_idr;1863 1864 dev->minor = ret;1865 dev->msg_timeout = VDUSE_MSG_DEFAULT_TIMEOUT;1866 dev->dev = device_create_with_groups(&vduse_class, NULL,1867 MKDEV(MAJOR(vduse_major), dev->minor),1868 dev, vduse_dev_groups, "%s", config->name);1869 if (IS_ERR(dev->dev)) {1870 ret = PTR_ERR(dev->dev);1871 goto err_dev;1872 }1873 1874 ret = vduse_dev_init_vqs(dev, config->vq_align, config->vq_num);1875 if (ret)1876 goto err_vqs;1877 1878 __module_get(THIS_MODULE);1879 1880 return 0;1881err_vqs:1882 device_destroy(&vduse_class, MKDEV(MAJOR(vduse_major), dev->minor));1883err_dev:1884 idr_remove(&vduse_idr, dev->minor);1885err_idr:1886 kfree(dev->name);1887err_str:1888 vduse_dev_destroy(dev);1889err:1890 return ret;1891}1892 1893static long vduse_ioctl(struct file *file, unsigned int cmd,1894 unsigned long arg)1895{1896 int ret;1897 void __user *argp = (void __user *)arg;1898 struct vduse_control *control = file->private_data;1899 1900 mutex_lock(&vduse_lock);1901 switch (cmd) {1902 case VDUSE_GET_API_VERSION:1903 ret = put_user(control->api_version, (u64 __user *)argp);1904 break;1905 case VDUSE_SET_API_VERSION: {1906 u64 api_version;1907 1908 ret = -EFAULT;1909 if (get_user(api_version, (u64 __user *)argp))1910 break;1911 1912 ret = -EINVAL;1913 if (api_version > VDUSE_API_VERSION)1914 break;1915 1916 ret = 0;1917 control->api_version = api_version;1918 break;1919 }1920 case VDUSE_CREATE_DEV: {1921 struct vduse_dev_config config;1922 unsigned long size = offsetof(struct vduse_dev_config, config);1923 void *buf;1924 1925 ret = -EFAULT;1926 if (copy_from_user(&config, argp, size))1927 break;1928 1929 ret = -EINVAL;1930 if (vduse_validate_config(&config) == false)1931 break;1932 1933 buf = vmemdup_user(argp + size, config.config_size);1934 if (IS_ERR(buf)) {1935 ret = PTR_ERR(buf);1936 break;1937 }1938 config.name[VDUSE_NAME_MAX - 1] = '\0';1939 ret = vduse_create_dev(&config, buf, control->api_version);1940 if (ret)1941 kvfree(buf);1942 break;1943 }1944 case VDUSE_DESTROY_DEV: {1945 char name[VDUSE_NAME_MAX];1946 1947 ret = -EFAULT;1948 if (copy_from_user(name, argp, VDUSE_NAME_MAX))1949 break;1950 1951 name[VDUSE_NAME_MAX - 1] = '\0';1952 ret = vduse_destroy_dev(name);1953 break;1954 }1955 default:1956 ret = -EINVAL;1957 break;1958 }1959 mutex_unlock(&vduse_lock);1960 1961 return ret;1962}1963 1964static int vduse_release(struct inode *inode, struct file *file)1965{1966 struct vduse_control *control = file->private_data;1967 1968 kfree(control);1969 return 0;1970}1971 1972static int vduse_open(struct inode *inode, struct file *file)1973{1974 struct vduse_control *control;1975 1976 control = kmalloc(sizeof(struct vduse_control), GFP_KERNEL);1977 if (!control)1978 return -ENOMEM;1979 1980 control->api_version = VDUSE_API_VERSION;1981 file->private_data = control;1982 1983 return 0;1984}1985 1986static const struct file_operations vduse_ctrl_fops = {1987 .owner = THIS_MODULE,1988 .open = vduse_open,1989 .release = vduse_release,1990 .unlocked_ioctl = vduse_ioctl,1991 .compat_ioctl = compat_ptr_ioctl,1992 .llseek = noop_llseek,1993};1994 1995struct vduse_mgmt_dev {1996 struct vdpa_mgmt_dev mgmt_dev;1997 struct device dev;1998};1999 2000static struct vduse_mgmt_dev *vduse_mgmt;2001 2002static int vduse_dev_init_vdpa(struct vduse_dev *dev, const char *name)2003{2004 struct vduse_vdpa *vdev;2005 int ret;2006 2007 if (dev->vdev)2008 return -EEXIST;2009 2010 vdev = vdpa_alloc_device(struct vduse_vdpa, vdpa, dev->dev,2011 &vduse_vdpa_config_ops, 1, 1, name, true);2012 if (IS_ERR(vdev))2013 return PTR_ERR(vdev);2014 2015 dev->vdev = vdev;2016 vdev->dev = dev;2017 vdev->vdpa.dev.dma_mask = &vdev->vdpa.dev.coherent_dma_mask;2018 ret = dma_set_mask_and_coherent(&vdev->vdpa.dev, DMA_BIT_MASK(64));2019 if (ret) {2020 put_device(&vdev->vdpa.dev);2021 return ret;2022 }2023 set_dma_ops(&vdev->vdpa.dev, &vduse_dev_dma_ops);2024 vdev->vdpa.dma_dev = &vdev->vdpa.dev;2025 vdev->vdpa.mdev = &vduse_mgmt->mgmt_dev;2026 2027 return 0;2028}2029 2030static int vdpa_dev_add(struct vdpa_mgmt_dev *mdev, const char *name,2031 const struct vdpa_dev_set_config *config)2032{2033 struct vduse_dev *dev;2034 int ret;2035 2036 mutex_lock(&vduse_lock);2037 dev = vduse_find_dev(name);2038 if (!dev || !vduse_dev_is_ready(dev)) {2039 mutex_unlock(&vduse_lock);2040 return -EINVAL;2041 }2042 ret = vduse_dev_init_vdpa(dev, name);2043 mutex_unlock(&vduse_lock);2044 if (ret)2045 return ret;2046 2047 mutex_lock(&dev->domain_lock);2048 if (!dev->domain)2049 dev->domain = vduse_domain_create(VDUSE_IOVA_SIZE - 1,2050 dev->bounce_size);2051 mutex_unlock(&dev->domain_lock);2052 if (!dev->domain) {2053 put_device(&dev->vdev->vdpa.dev);2054 return -ENOMEM;2055 }2056 2057 ret = _vdpa_register_device(&dev->vdev->vdpa, dev->vq_num);2058 if (ret) {2059 put_device(&dev->vdev->vdpa.dev);2060 mutex_lock(&dev->domain_lock);2061 vduse_domain_destroy(dev->domain);2062 dev->domain = NULL;2063 mutex_unlock(&dev->domain_lock);2064 return ret;2065 }2066 2067 return 0;2068}2069 2070static void vdpa_dev_del(struct vdpa_mgmt_dev *mdev, struct vdpa_device *dev)2071{2072 _vdpa_unregister_device(dev);2073}2074 2075static const struct vdpa_mgmtdev_ops vdpa_dev_mgmtdev_ops = {2076 .dev_add = vdpa_dev_add,2077 .dev_del = vdpa_dev_del,2078};2079 2080static struct virtio_device_id id_table[] = {2081 { VIRTIO_ID_BLOCK, VIRTIO_DEV_ANY_ID },2082 { VIRTIO_ID_NET, VIRTIO_DEV_ANY_ID },2083 { 0 },2084};2085 2086static void vduse_mgmtdev_release(struct device *dev)2087{2088 struct vduse_mgmt_dev *mgmt_dev;2089 2090 mgmt_dev = container_of(dev, struct vduse_mgmt_dev, dev);2091 kfree(mgmt_dev);2092}2093 2094static int vduse_mgmtdev_init(void)2095{2096 int ret;2097 2098 vduse_mgmt = kzalloc(sizeof(*vduse_mgmt), GFP_KERNEL);2099 if (!vduse_mgmt)2100 return -ENOMEM;2101 2102 ret = dev_set_name(&vduse_mgmt->dev, "vduse");2103 if (ret) {2104 kfree(vduse_mgmt);2105 return ret;2106 }2107 2108 vduse_mgmt->dev.release = vduse_mgmtdev_release;2109 2110 ret = device_register(&vduse_mgmt->dev);2111 if (ret)2112 goto dev_reg_err;2113 2114 vduse_mgmt->mgmt_dev.id_table = id_table;2115 vduse_mgmt->mgmt_dev.ops = &vdpa_dev_mgmtdev_ops;2116 vduse_mgmt->mgmt_dev.device = &vduse_mgmt->dev;2117 ret = vdpa_mgmtdev_register(&vduse_mgmt->mgmt_dev);2118 if (ret)2119 device_unregister(&vduse_mgmt->dev);2120 2121 return ret;2122 2123dev_reg_err:2124 put_device(&vduse_mgmt->dev);2125 return ret;2126}2127 2128static void vduse_mgmtdev_exit(void)2129{2130 vdpa_mgmtdev_unregister(&vduse_mgmt->mgmt_dev);2131 device_unregister(&vduse_mgmt->dev);2132}2133 2134static int vduse_init(void)2135{2136 int ret;2137 struct device *dev;2138 2139 ret = class_register(&vduse_class);2140 if (ret)2141 return ret;2142 2143 ret = alloc_chrdev_region(&vduse_major, 0, VDUSE_DEV_MAX, "vduse");2144 if (ret)2145 goto err_chardev_region;2146 2147 /* /dev/vduse/control */2148 cdev_init(&vduse_ctrl_cdev, &vduse_ctrl_fops);2149 vduse_ctrl_cdev.owner = THIS_MODULE;2150 ret = cdev_add(&vduse_ctrl_cdev, vduse_major, 1);2151 if (ret)2152 goto err_ctrl_cdev;2153 2154 dev = device_create(&vduse_class, NULL, vduse_major, NULL, "control");2155 if (IS_ERR(dev)) {2156 ret = PTR_ERR(dev);2157 goto err_device;2158 }2159 2160 /* /dev/vduse/$DEVICE */2161 cdev_init(&vduse_cdev, &vduse_dev_fops);2162 vduse_cdev.owner = THIS_MODULE;2163 ret = cdev_add(&vduse_cdev, MKDEV(MAJOR(vduse_major), 1),2164 VDUSE_DEV_MAX - 1);2165 if (ret)2166 goto err_cdev;2167 2168 ret = -ENOMEM;2169 vduse_irq_wq = alloc_workqueue("vduse-irq",2170 WQ_HIGHPRI | WQ_SYSFS | WQ_UNBOUND, 0);2171 if (!vduse_irq_wq)2172 goto err_wq;2173 2174 vduse_irq_bound_wq = alloc_workqueue("vduse-irq-bound", WQ_HIGHPRI, 0);2175 if (!vduse_irq_bound_wq)2176 goto err_bound_wq;2177 2178 ret = vduse_domain_init();2179 if (ret)2180 goto err_domain;2181 2182 ret = vduse_mgmtdev_init();2183 if (ret)2184 goto err_mgmtdev;2185 2186 return 0;2187err_mgmtdev:2188 vduse_domain_exit();2189err_domain:2190 destroy_workqueue(vduse_irq_bound_wq);2191err_bound_wq:2192 destroy_workqueue(vduse_irq_wq);2193err_wq:2194 cdev_del(&vduse_cdev);2195err_cdev:2196 device_destroy(&vduse_class, vduse_major);2197err_device:2198 cdev_del(&vduse_ctrl_cdev);2199err_ctrl_cdev:2200 unregister_chrdev_region(vduse_major, VDUSE_DEV_MAX);2201err_chardev_region:2202 class_unregister(&vduse_class);2203 return ret;2204}2205module_init(vduse_init);2206 2207static void vduse_exit(void)2208{2209 vduse_mgmtdev_exit();2210 vduse_domain_exit();2211 destroy_workqueue(vduse_irq_bound_wq);2212 destroy_workqueue(vduse_irq_wq);2213 cdev_del(&vduse_cdev);2214 device_destroy(&vduse_class, vduse_major);2215 cdev_del(&vduse_ctrl_cdev);2216 unregister_chrdev_region(vduse_major, VDUSE_DEV_MAX);2217 class_unregister(&vduse_class);2218}2219module_exit(vduse_exit);2220 2221MODULE_LICENSE(DRV_LICENSE);2222MODULE_AUTHOR(DRV_AUTHOR);2223MODULE_DESCRIPTION(DRV_DESC);2224