855 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2/*3 * VDPA device simulator core.4 *5 * Copyright (c) 2020, Red Hat Inc. All rights reserved.6 * Author: Jason Wang <jasowang@redhat.com>7 *8 */9 10#include <linux/init.h>11#include <linux/module.h>12#include <linux/device.h>13#include <linux/kernel.h>14#include <linux/kthread.h>15#include <linux/slab.h>16#include <linux/dma-map-ops.h>17#include <linux/vringh.h>18#include <linux/vdpa.h>19#include <linux/vhost_iotlb.h>20#include <uapi/linux/vdpa.h>21#include <uapi/linux/vhost_types.h>22 23#include "vdpa_sim.h"24 25#define DRV_VERSION "0.1"26#define DRV_AUTHOR "Jason Wang <jasowang@redhat.com>"27#define DRV_DESC "vDPA Device Simulator core"28#define DRV_LICENSE "GPL v2"29 30static int batch_mapping = 1;31module_param(batch_mapping, int, 0444);32MODULE_PARM_DESC(batch_mapping, "Batched mapping 1 -Enable; 0 - Disable");33 34static int max_iotlb_entries = 2048;35module_param(max_iotlb_entries, int, 0444);36MODULE_PARM_DESC(max_iotlb_entries,37 "Maximum number of iotlb entries for each address space. 0 means unlimited. (default: 2048)");38 39static bool use_va = true;40module_param(use_va, bool, 0444);41MODULE_PARM_DESC(use_va, "Enable/disable the device's ability to use VA");42 43#define VDPASIM_QUEUE_ALIGN PAGE_SIZE44#define VDPASIM_QUEUE_MAX 25645#define VDPASIM_VENDOR_ID 046 47struct vdpasim_mm_work {48 struct kthread_work work;49 struct vdpasim *vdpasim;50 struct mm_struct *mm_to_bind;51 int ret;52};53 54static void vdpasim_mm_work_fn(struct kthread_work *work)55{56 struct vdpasim_mm_work *mm_work =57 container_of(work, struct vdpasim_mm_work, work);58 struct vdpasim *vdpasim = mm_work->vdpasim;59 60 mm_work->ret = 0;61 62 //TODO: should we attach the cgroup of the mm owner?63 vdpasim->mm_bound = mm_work->mm_to_bind;64}65 66static void vdpasim_worker_change_mm_sync(struct vdpasim *vdpasim,67 struct vdpasim_mm_work *mm_work)68{69 struct kthread_work *work = &mm_work->work;70 71 kthread_init_work(work, vdpasim_mm_work_fn);72 kthread_queue_work(vdpasim->worker, work);73 74 kthread_flush_work(work);75}76 77static struct vdpasim *vdpa_to_sim(struct vdpa_device *vdpa)78{79 return container_of(vdpa, struct vdpasim, vdpa);80}81 82static void vdpasim_vq_notify(struct vringh *vring)83{84 struct vdpasim_virtqueue *vq =85 container_of(vring, struct vdpasim_virtqueue, vring);86 87 if (!vq->cb)88 return;89 90 vq->cb(vq->private);91}92 93static void vdpasim_queue_ready(struct vdpasim *vdpasim, unsigned int idx)94{95 struct vdpasim_virtqueue *vq = &vdpasim->vqs[idx];96 uint16_t last_avail_idx = vq->vring.last_avail_idx;97 struct vring_desc *desc = (struct vring_desc *)98 (uintptr_t)vq->desc_addr;99 struct vring_avail *avail = (struct vring_avail *)100 (uintptr_t)vq->driver_addr;101 struct vring_used *used = (struct vring_used *)102 (uintptr_t)vq->device_addr;103 104 if (use_va && vdpasim->mm_bound) {105 vringh_init_iotlb_va(&vq->vring, vdpasim->features, vq->num,106 true, desc, avail, used);107 } else {108 vringh_init_iotlb(&vq->vring, vdpasim->features, vq->num,109 true, desc, avail, used);110 }111 112 vq->vring.last_avail_idx = last_avail_idx;113 114 /*115 * Since vdpa_sim does not support receive inflight descriptors as a116 * destination of a migration, let's set both avail_idx and used_idx117 * the same at vq start. This is how vhost-user works in a118 * VHOST_SET_VRING_BASE call.119 *120 * Although the simple fix is to set last_used_idx at121 * vdpasim_set_vq_state, it would be reset at vdpasim_queue_ready.122 */123 vq->vring.last_used_idx = last_avail_idx;124 vq->vring.notify = vdpasim_vq_notify;125}126 127static void vdpasim_vq_reset(struct vdpasim *vdpasim,128 struct vdpasim_virtqueue *vq)129{130 vq->ready = false;131 vq->desc_addr = 0;132 vq->driver_addr = 0;133 vq->device_addr = 0;134 vq->cb = NULL;135 vq->private = NULL;136 vringh_init_iotlb(&vq->vring, vdpasim->dev_attr.supported_features,137 VDPASIM_QUEUE_MAX, false, NULL, NULL, NULL);138 139 vq->vring.notify = NULL;140}141 142static void vdpasim_do_reset(struct vdpasim *vdpasim, u32 flags)143{144 int i;145 146 spin_lock(&vdpasim->iommu_lock);147 148 for (i = 0; i < vdpasim->dev_attr.nvqs; i++) {149 vdpasim_vq_reset(vdpasim, &vdpasim->vqs[i]);150 vringh_set_iotlb(&vdpasim->vqs[i].vring, &vdpasim->iommu[0],151 &vdpasim->iommu_lock);152 }153 154 if (flags & VDPA_RESET_F_CLEAN_MAP) {155 for (i = 0; i < vdpasim->dev_attr.nas; i++) {156 vhost_iotlb_reset(&vdpasim->iommu[i]);157 vhost_iotlb_add_range(&vdpasim->iommu[i], 0, ULONG_MAX,158 0, VHOST_MAP_RW);159 vdpasim->iommu_pt[i] = true;160 }161 }162 163 vdpasim->running = false;164 spin_unlock(&vdpasim->iommu_lock);165 166 vdpasim->features = 0;167 vdpasim->status = 0;168 ++vdpasim->generation;169}170 171static const struct vdpa_config_ops vdpasim_config_ops;172static const struct vdpa_config_ops vdpasim_batch_config_ops;173 174static void vdpasim_work_fn(struct kthread_work *work)175{176 struct vdpasim *vdpasim = container_of(work, struct vdpasim, work);177 struct mm_struct *mm = vdpasim->mm_bound;178 179 if (use_va && mm) {180 if (!mmget_not_zero(mm))181 return;182 kthread_use_mm(mm);183 }184 185 vdpasim->dev_attr.work_fn(vdpasim);186 187 if (use_va && mm) {188 kthread_unuse_mm(mm);189 mmput(mm);190 }191}192 193struct vdpasim *vdpasim_create(struct vdpasim_dev_attr *dev_attr,194 const struct vdpa_dev_set_config *config)195{196 const struct vdpa_config_ops *ops;197 struct vdpa_device *vdpa;198 struct vdpasim *vdpasim;199 struct device *dev;200 int i, ret = -ENOMEM;201 202 if (!dev_attr->alloc_size)203 return ERR_PTR(-EINVAL);204 205 if (config->mask & BIT_ULL(VDPA_ATTR_DEV_FEATURES)) {206 if (config->device_features &207 ~dev_attr->supported_features)208 return ERR_PTR(-EINVAL);209 dev_attr->supported_features =210 config->device_features;211 }212 213 if (batch_mapping)214 ops = &vdpasim_batch_config_ops;215 else216 ops = &vdpasim_config_ops;217 218 vdpa = __vdpa_alloc_device(NULL, ops,219 dev_attr->ngroups, dev_attr->nas,220 dev_attr->alloc_size,221 dev_attr->name, use_va);222 if (IS_ERR(vdpa)) {223 ret = PTR_ERR(vdpa);224 goto err_alloc;225 }226 227 vdpasim = vdpa_to_sim(vdpa);228 vdpasim->dev_attr = *dev_attr;229 dev = &vdpasim->vdpa.dev;230 231 kthread_init_work(&vdpasim->work, vdpasim_work_fn);232 vdpasim->worker = kthread_create_worker(0, "vDPA sim worker: %s",233 dev_attr->name);234 if (IS_ERR(vdpasim->worker))235 goto err_iommu;236 237 mutex_init(&vdpasim->mutex);238 spin_lock_init(&vdpasim->iommu_lock);239 240 dev->dma_mask = &dev->coherent_dma_mask;241 if (dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64)))242 goto err_iommu;243 vdpasim->vdpa.mdev = dev_attr->mgmt_dev;244 245 vdpasim->config = kzalloc(dev_attr->config_size, GFP_KERNEL);246 if (!vdpasim->config)247 goto err_iommu;248 249 vdpasim->vqs = kcalloc(dev_attr->nvqs, sizeof(struct vdpasim_virtqueue),250 GFP_KERNEL);251 if (!vdpasim->vqs)252 goto err_iommu;253 254 vdpasim->iommu = kmalloc_array(vdpasim->dev_attr.nas,255 sizeof(*vdpasim->iommu), GFP_KERNEL);256 if (!vdpasim->iommu)257 goto err_iommu;258 259 vdpasim->iommu_pt = kmalloc_array(vdpasim->dev_attr.nas,260 sizeof(*vdpasim->iommu_pt), GFP_KERNEL);261 if (!vdpasim->iommu_pt)262 goto err_iommu;263 264 for (i = 0; i < vdpasim->dev_attr.nas; i++) {265 vhost_iotlb_init(&vdpasim->iommu[i], max_iotlb_entries, 0);266 vhost_iotlb_add_range(&vdpasim->iommu[i], 0, ULONG_MAX, 0,267 VHOST_MAP_RW);268 vdpasim->iommu_pt[i] = true;269 }270 271 for (i = 0; i < dev_attr->nvqs; i++)272 vringh_set_iotlb(&vdpasim->vqs[i].vring, &vdpasim->iommu[0],273 &vdpasim->iommu_lock);274 275 vdpasim->vdpa.dma_dev = dev;276 277 return vdpasim;278 279err_iommu:280 put_device(dev);281err_alloc:282 return ERR_PTR(ret);283}284EXPORT_SYMBOL_GPL(vdpasim_create);285 286void vdpasim_schedule_work(struct vdpasim *vdpasim)287{288 kthread_queue_work(vdpasim->worker, &vdpasim->work);289}290EXPORT_SYMBOL_GPL(vdpasim_schedule_work);291 292static int vdpasim_set_vq_address(struct vdpa_device *vdpa, u16 idx,293 u64 desc_area, u64 driver_area,294 u64 device_area)295{296 struct vdpasim *vdpasim = vdpa_to_sim(vdpa);297 struct vdpasim_virtqueue *vq = &vdpasim->vqs[idx];298 299 vq->desc_addr = desc_area;300 vq->driver_addr = driver_area;301 vq->device_addr = device_area;302 303 return 0;304}305 306static void vdpasim_set_vq_num(struct vdpa_device *vdpa, u16 idx, u32 num)307{308 struct vdpasim *vdpasim = vdpa_to_sim(vdpa);309 struct vdpasim_virtqueue *vq = &vdpasim->vqs[idx];310 311 vq->num = num;312}313 314static u16 vdpasim_get_vq_size(struct vdpa_device *vdpa, u16 idx)315{316 struct vdpasim *vdpasim = vdpa_to_sim(vdpa);317 struct vdpasim_virtqueue *vq = &vdpasim->vqs[idx];318 319 if (vdpasim->status & VIRTIO_CONFIG_S_DRIVER_OK)320 return vq->num;321 else322 return VDPASIM_QUEUE_MAX;323}324 325static void vdpasim_kick_vq(struct vdpa_device *vdpa, u16 idx)326{327 struct vdpasim *vdpasim = vdpa_to_sim(vdpa);328 struct vdpasim_virtqueue *vq = &vdpasim->vqs[idx];329 330 if (!vdpasim->running &&331 (vdpasim->status & VIRTIO_CONFIG_S_DRIVER_OK)) {332 vdpasim->pending_kick = true;333 return;334 }335 336 if (vq->ready)337 vdpasim_schedule_work(vdpasim);338}339 340static void vdpasim_set_vq_cb(struct vdpa_device *vdpa, u16 idx,341 struct vdpa_callback *cb)342{343 struct vdpasim *vdpasim = vdpa_to_sim(vdpa);344 struct vdpasim_virtqueue *vq = &vdpasim->vqs[idx];345 346 vq->cb = cb->callback;347 vq->private = cb->private;348}349 350static void vdpasim_set_vq_ready(struct vdpa_device *vdpa, u16 idx, bool ready)351{352 struct vdpasim *vdpasim = vdpa_to_sim(vdpa);353 struct vdpasim_virtqueue *vq = &vdpasim->vqs[idx];354 bool old_ready;355 356 mutex_lock(&vdpasim->mutex);357 old_ready = vq->ready;358 vq->ready = ready;359 if (vq->ready && !old_ready) {360 vdpasim_queue_ready(vdpasim, idx);361 }362 mutex_unlock(&vdpasim->mutex);363}364 365static bool vdpasim_get_vq_ready(struct vdpa_device *vdpa, u16 idx)366{367 struct vdpasim *vdpasim = vdpa_to_sim(vdpa);368 struct vdpasim_virtqueue *vq = &vdpasim->vqs[idx];369 370 return vq->ready;371}372 373static int vdpasim_set_vq_state(struct vdpa_device *vdpa, u16 idx,374 const struct vdpa_vq_state *state)375{376 struct vdpasim *vdpasim = vdpa_to_sim(vdpa);377 struct vdpasim_virtqueue *vq = &vdpasim->vqs[idx];378 struct vringh *vrh = &vq->vring;379 380 mutex_lock(&vdpasim->mutex);381 vrh->last_avail_idx = state->split.avail_index;382 mutex_unlock(&vdpasim->mutex);383 384 return 0;385}386 387static int vdpasim_get_vq_state(struct vdpa_device *vdpa, u16 idx,388 struct vdpa_vq_state *state)389{390 struct vdpasim *vdpasim = vdpa_to_sim(vdpa);391 struct vdpasim_virtqueue *vq = &vdpasim->vqs[idx];392 struct vringh *vrh = &vq->vring;393 394 state->split.avail_index = vrh->last_avail_idx;395 return 0;396}397 398static int vdpasim_get_vq_stats(struct vdpa_device *vdpa, u16 idx,399 struct sk_buff *msg,400 struct netlink_ext_ack *extack)401{402 struct vdpasim *vdpasim = vdpa_to_sim(vdpa);403 404 if (vdpasim->dev_attr.get_stats)405 return vdpasim->dev_attr.get_stats(vdpasim, idx,406 msg, extack);407 return -EOPNOTSUPP;408}409 410static u32 vdpasim_get_vq_align(struct vdpa_device *vdpa)411{412 return VDPASIM_QUEUE_ALIGN;413}414 415static u32 vdpasim_get_vq_group(struct vdpa_device *vdpa, u16 idx)416{417 /* RX and TX belongs to group 0, CVQ belongs to group 1 */418 if (idx == 2)419 return 1;420 else421 return 0;422}423 424static u64 vdpasim_get_device_features(struct vdpa_device *vdpa)425{426 struct vdpasim *vdpasim = vdpa_to_sim(vdpa);427 428 return vdpasim->dev_attr.supported_features;429}430 431static u64 vdpasim_get_backend_features(const struct vdpa_device *vdpa)432{433 return BIT_ULL(VHOST_BACKEND_F_ENABLE_AFTER_DRIVER_OK);434}435 436static int vdpasim_set_driver_features(struct vdpa_device *vdpa, u64 features)437{438 struct vdpasim *vdpasim = vdpa_to_sim(vdpa);439 440 /* DMA mapping must be done by driver */441 if (!(features & (1ULL << VIRTIO_F_ACCESS_PLATFORM)))442 return -EINVAL;443 444 vdpasim->features = features & vdpasim->dev_attr.supported_features;445 446 return 0;447}448 449static u64 vdpasim_get_driver_features(struct vdpa_device *vdpa)450{451 struct vdpasim *vdpasim = vdpa_to_sim(vdpa);452 453 return vdpasim->features;454}455 456static void vdpasim_set_config_cb(struct vdpa_device *vdpa,457 struct vdpa_callback *cb)458{459 /* We don't support config interrupt */460}461 462static u16 vdpasim_get_vq_num_max(struct vdpa_device *vdpa)463{464 return VDPASIM_QUEUE_MAX;465}466 467static u32 vdpasim_get_device_id(struct vdpa_device *vdpa)468{469 struct vdpasim *vdpasim = vdpa_to_sim(vdpa);470 471 return vdpasim->dev_attr.id;472}473 474static u32 vdpasim_get_vendor_id(struct vdpa_device *vdpa)475{476 return VDPASIM_VENDOR_ID;477}478 479static u8 vdpasim_get_status(struct vdpa_device *vdpa)480{481 struct vdpasim *vdpasim = vdpa_to_sim(vdpa);482 u8 status;483 484 mutex_lock(&vdpasim->mutex);485 status = vdpasim->status;486 mutex_unlock(&vdpasim->mutex);487 488 return status;489}490 491static void vdpasim_set_status(struct vdpa_device *vdpa, u8 status)492{493 struct vdpasim *vdpasim = vdpa_to_sim(vdpa);494 495 mutex_lock(&vdpasim->mutex);496 vdpasim->status = status;497 vdpasim->running = (status & VIRTIO_CONFIG_S_DRIVER_OK) != 0;498 mutex_unlock(&vdpasim->mutex);499}500 501static int vdpasim_compat_reset(struct vdpa_device *vdpa, u32 flags)502{503 struct vdpasim *vdpasim = vdpa_to_sim(vdpa);504 505 mutex_lock(&vdpasim->mutex);506 vdpasim->status = 0;507 vdpasim_do_reset(vdpasim, flags);508 mutex_unlock(&vdpasim->mutex);509 510 return 0;511}512 513static int vdpasim_reset(struct vdpa_device *vdpa)514{515 return vdpasim_compat_reset(vdpa, 0);516}517 518static int vdpasim_suspend(struct vdpa_device *vdpa)519{520 struct vdpasim *vdpasim = vdpa_to_sim(vdpa);521 522 mutex_lock(&vdpasim->mutex);523 vdpasim->running = false;524 mutex_unlock(&vdpasim->mutex);525 526 return 0;527}528 529static int vdpasim_resume(struct vdpa_device *vdpa)530{531 struct vdpasim *vdpasim = vdpa_to_sim(vdpa);532 int i;533 534 mutex_lock(&vdpasim->mutex);535 vdpasim->running = true;536 537 if (vdpasim->pending_kick) {538 /* Process pending descriptors */539 for (i = 0; i < vdpasim->dev_attr.nvqs; ++i)540 vdpasim_kick_vq(vdpa, i);541 542 vdpasim->pending_kick = false;543 }544 545 mutex_unlock(&vdpasim->mutex);546 547 return 0;548}549 550static size_t vdpasim_get_config_size(struct vdpa_device *vdpa)551{552 struct vdpasim *vdpasim = vdpa_to_sim(vdpa);553 554 return vdpasim->dev_attr.config_size;555}556 557static void vdpasim_get_config(struct vdpa_device *vdpa, unsigned int offset,558 void *buf, unsigned int len)559{560 struct vdpasim *vdpasim = vdpa_to_sim(vdpa);561 562 if (offset + len > vdpasim->dev_attr.config_size)563 return;564 565 if (vdpasim->dev_attr.get_config)566 vdpasim->dev_attr.get_config(vdpasim, vdpasim->config);567 568 memcpy(buf, vdpasim->config + offset, len);569}570 571static void vdpasim_set_config(struct vdpa_device *vdpa, unsigned int offset,572 const void *buf, unsigned int len)573{574 struct vdpasim *vdpasim = vdpa_to_sim(vdpa);575 576 if (offset + len > vdpasim->dev_attr.config_size)577 return;578 579 memcpy(vdpasim->config + offset, buf, len);580 581 if (vdpasim->dev_attr.set_config)582 vdpasim->dev_attr.set_config(vdpasim, vdpasim->config);583}584 585static u32 vdpasim_get_generation(struct vdpa_device *vdpa)586{587 struct vdpasim *vdpasim = vdpa_to_sim(vdpa);588 589 return vdpasim->generation;590}591 592static struct vdpa_iova_range vdpasim_get_iova_range(struct vdpa_device *vdpa)593{594 struct vdpa_iova_range range = {595 .first = 0ULL,596 .last = ULLONG_MAX,597 };598 599 return range;600}601 602static int vdpasim_set_group_asid(struct vdpa_device *vdpa, unsigned int group,603 unsigned int asid)604{605 struct vdpasim *vdpasim = vdpa_to_sim(vdpa);606 struct vhost_iotlb *iommu;607 int i;608 609 if (group > vdpasim->dev_attr.ngroups)610 return -EINVAL;611 612 if (asid >= vdpasim->dev_attr.nas)613 return -EINVAL;614 615 iommu = &vdpasim->iommu[asid];616 617 mutex_lock(&vdpasim->mutex);618 619 for (i = 0; i < vdpasim->dev_attr.nvqs; i++)620 if (vdpasim_get_vq_group(vdpa, i) == group)621 vringh_set_iotlb(&vdpasim->vqs[i].vring, iommu,622 &vdpasim->iommu_lock);623 624 mutex_unlock(&vdpasim->mutex);625 626 return 0;627}628 629static int vdpasim_set_map(struct vdpa_device *vdpa, unsigned int asid,630 struct vhost_iotlb *iotlb)631{632 struct vdpasim *vdpasim = vdpa_to_sim(vdpa);633 struct vhost_iotlb_map *map;634 struct vhost_iotlb *iommu;635 u64 start = 0ULL, last = 0ULL - 1;636 int ret;637 638 if (asid >= vdpasim->dev_attr.nas)639 return -EINVAL;640 641 spin_lock(&vdpasim->iommu_lock);642 643 iommu = &vdpasim->iommu[asid];644 vhost_iotlb_reset(iommu);645 vdpasim->iommu_pt[asid] = false;646 647 for (map = vhost_iotlb_itree_first(iotlb, start, last); map;648 map = vhost_iotlb_itree_next(map, start, last)) {649 ret = vhost_iotlb_add_range(iommu, map->start,650 map->last, map->addr, map->perm);651 if (ret)652 goto err;653 }654 spin_unlock(&vdpasim->iommu_lock);655 return 0;656 657err:658 vhost_iotlb_reset(iommu);659 spin_unlock(&vdpasim->iommu_lock);660 return ret;661}662 663static int vdpasim_reset_map(struct vdpa_device *vdpa, unsigned int asid)664{665 struct vdpasim *vdpasim = vdpa_to_sim(vdpa);666 667 if (asid >= vdpasim->dev_attr.nas)668 return -EINVAL;669 670 spin_lock(&vdpasim->iommu_lock);671 if (vdpasim->iommu_pt[asid])672 goto out;673 vhost_iotlb_reset(&vdpasim->iommu[asid]);674 vhost_iotlb_add_range(&vdpasim->iommu[asid], 0, ULONG_MAX,675 0, VHOST_MAP_RW);676 vdpasim->iommu_pt[asid] = true;677out:678 spin_unlock(&vdpasim->iommu_lock);679 return 0;680}681 682static int vdpasim_bind_mm(struct vdpa_device *vdpa, struct mm_struct *mm)683{684 struct vdpasim *vdpasim = vdpa_to_sim(vdpa);685 struct vdpasim_mm_work mm_work;686 687 mm_work.vdpasim = vdpasim;688 mm_work.mm_to_bind = mm;689 690 vdpasim_worker_change_mm_sync(vdpasim, &mm_work);691 692 return mm_work.ret;693}694 695static void vdpasim_unbind_mm(struct vdpa_device *vdpa)696{697 struct vdpasim *vdpasim = vdpa_to_sim(vdpa);698 struct vdpasim_mm_work mm_work;699 700 mm_work.vdpasim = vdpasim;701 mm_work.mm_to_bind = NULL;702 703 vdpasim_worker_change_mm_sync(vdpasim, &mm_work);704}705 706static int vdpasim_dma_map(struct vdpa_device *vdpa, unsigned int asid,707 u64 iova, u64 size,708 u64 pa, u32 perm, void *opaque)709{710 struct vdpasim *vdpasim = vdpa_to_sim(vdpa);711 int ret;712 713 if (asid >= vdpasim->dev_attr.nas)714 return -EINVAL;715 716 spin_lock(&vdpasim->iommu_lock);717 if (vdpasim->iommu_pt[asid]) {718 vhost_iotlb_reset(&vdpasim->iommu[asid]);719 vdpasim->iommu_pt[asid] = false;720 }721 ret = vhost_iotlb_add_range_ctx(&vdpasim->iommu[asid], iova,722 iova + size - 1, pa, perm, opaque);723 spin_unlock(&vdpasim->iommu_lock);724 725 return ret;726}727 728static int vdpasim_dma_unmap(struct vdpa_device *vdpa, unsigned int asid,729 u64 iova, u64 size)730{731 struct vdpasim *vdpasim = vdpa_to_sim(vdpa);732 733 if (asid >= vdpasim->dev_attr.nas)734 return -EINVAL;735 736 if (vdpasim->iommu_pt[asid]) {737 vhost_iotlb_reset(&vdpasim->iommu[asid]);738 vdpasim->iommu_pt[asid] = false;739 }740 741 spin_lock(&vdpasim->iommu_lock);742 vhost_iotlb_del_range(&vdpasim->iommu[asid], iova, iova + size - 1);743 spin_unlock(&vdpasim->iommu_lock);744 745 return 0;746}747 748static void vdpasim_free(struct vdpa_device *vdpa)749{750 struct vdpasim *vdpasim = vdpa_to_sim(vdpa);751 int i;752 753 kthread_cancel_work_sync(&vdpasim->work);754 kthread_destroy_worker(vdpasim->worker);755 756 for (i = 0; i < vdpasim->dev_attr.nvqs; i++) {757 vringh_kiov_cleanup(&vdpasim->vqs[i].out_iov);758 vringh_kiov_cleanup(&vdpasim->vqs[i].in_iov);759 }760 761 vdpasim->dev_attr.free(vdpasim);762 763 for (i = 0; i < vdpasim->dev_attr.nas; i++)764 vhost_iotlb_reset(&vdpasim->iommu[i]);765 kfree(vdpasim->iommu);766 kfree(vdpasim->iommu_pt);767 kfree(vdpasim->vqs);768 kfree(vdpasim->config);769}770 771static const struct vdpa_config_ops vdpasim_config_ops = {772 .set_vq_address = vdpasim_set_vq_address,773 .set_vq_num = vdpasim_set_vq_num,774 .kick_vq = vdpasim_kick_vq,775 .set_vq_cb = vdpasim_set_vq_cb,776 .set_vq_ready = vdpasim_set_vq_ready,777 .get_vq_ready = vdpasim_get_vq_ready,778 .set_vq_state = vdpasim_set_vq_state,779 .get_vendor_vq_stats = vdpasim_get_vq_stats,780 .get_vq_state = vdpasim_get_vq_state,781 .get_vq_align = vdpasim_get_vq_align,782 .get_vq_group = vdpasim_get_vq_group,783 .get_device_features = vdpasim_get_device_features,784 .get_backend_features = vdpasim_get_backend_features,785 .set_driver_features = vdpasim_set_driver_features,786 .get_driver_features = vdpasim_get_driver_features,787 .set_config_cb = vdpasim_set_config_cb,788 .get_vq_num_max = vdpasim_get_vq_num_max,789 .get_vq_size = vdpasim_get_vq_size,790 .get_device_id = vdpasim_get_device_id,791 .get_vendor_id = vdpasim_get_vendor_id,792 .get_status = vdpasim_get_status,793 .set_status = vdpasim_set_status,794 .reset = vdpasim_reset,795 .compat_reset = vdpasim_compat_reset,796 .suspend = vdpasim_suspend,797 .resume = vdpasim_resume,798 .get_config_size = vdpasim_get_config_size,799 .get_config = vdpasim_get_config,800 .set_config = vdpasim_set_config,801 .get_generation = vdpasim_get_generation,802 .get_iova_range = vdpasim_get_iova_range,803 .set_group_asid = vdpasim_set_group_asid,804 .dma_map = vdpasim_dma_map,805 .dma_unmap = vdpasim_dma_unmap,806 .reset_map = vdpasim_reset_map,807 .bind_mm = vdpasim_bind_mm,808 .unbind_mm = vdpasim_unbind_mm,809 .free = vdpasim_free,810};811 812static const struct vdpa_config_ops vdpasim_batch_config_ops = {813 .set_vq_address = vdpasim_set_vq_address,814 .set_vq_num = vdpasim_set_vq_num,815 .kick_vq = vdpasim_kick_vq,816 .set_vq_cb = vdpasim_set_vq_cb,817 .set_vq_ready = vdpasim_set_vq_ready,818 .get_vq_ready = vdpasim_get_vq_ready,819 .set_vq_state = vdpasim_set_vq_state,820 .get_vendor_vq_stats = vdpasim_get_vq_stats,821 .get_vq_state = vdpasim_get_vq_state,822 .get_vq_align = vdpasim_get_vq_align,823 .get_vq_group = vdpasim_get_vq_group,824 .get_device_features = vdpasim_get_device_features,825 .get_backend_features = vdpasim_get_backend_features,826 .set_driver_features = vdpasim_set_driver_features,827 .get_driver_features = vdpasim_get_driver_features,828 .set_config_cb = vdpasim_set_config_cb,829 .get_vq_num_max = vdpasim_get_vq_num_max,830 .get_device_id = vdpasim_get_device_id,831 .get_vendor_id = vdpasim_get_vendor_id,832 .get_status = vdpasim_get_status,833 .set_status = vdpasim_set_status,834 .reset = vdpasim_reset,835 .compat_reset = vdpasim_compat_reset,836 .suspend = vdpasim_suspend,837 .resume = vdpasim_resume,838 .get_config_size = vdpasim_get_config_size,839 .get_config = vdpasim_get_config,840 .set_config = vdpasim_set_config,841 .get_generation = vdpasim_get_generation,842 .get_iova_range = vdpasim_get_iova_range,843 .set_group_asid = vdpasim_set_group_asid,844 .set_map = vdpasim_set_map,845 .reset_map = vdpasim_reset_map,846 .bind_mm = vdpasim_bind_mm,847 .unbind_mm = vdpasim_unbind_mm,848 .free = vdpasim_free,849};850 851MODULE_VERSION(DRV_VERSION);852MODULE_LICENSE(DRV_LICENSE);853MODULE_AUTHOR(DRV_AUTHOR);854MODULE_DESCRIPTION(DRV_DESC);855