551 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2/*3 * VIRTIO based driver for vDPA device4 *5 * Copyright (c) 2020, Red Hat. All rights reserved.6 * Author: Jason Wang <jasowang@redhat.com>7 *8 */9 10#include <linux/init.h>11#include <linux/module.h>12#include <linux/device.h>13#include <linux/kernel.h>14#include <linux/slab.h>15#include <linux/uuid.h>16#include <linux/group_cpus.h>17#include <linux/virtio.h>18#include <linux/vdpa.h>19#include <linux/virtio_config.h>20#include <linux/virtio_ring.h>21 22#define MOD_VERSION "0.1"23#define MOD_AUTHOR "Jason Wang <jasowang@redhat.com>"24#define MOD_DESC "vDPA bus driver for virtio devices"25#define MOD_LICENSE "GPL v2"26 27struct virtio_vdpa_device {28 struct virtio_device vdev;29 struct vdpa_device *vdpa;30 u64 features;31 32 /* The lock to protect virtqueue list */33 spinlock_t lock;34 /* List of virtio_vdpa_vq_info */35 struct list_head virtqueues;36};37 38struct virtio_vdpa_vq_info {39 /* the actual virtqueue */40 struct virtqueue *vq;41 42 /* the list node for the virtqueues list */43 struct list_head node;44};45 46static inline struct virtio_vdpa_device *47to_virtio_vdpa_device(struct virtio_device *dev)48{49 return container_of(dev, struct virtio_vdpa_device, vdev);50}51 52static struct vdpa_device *vd_get_vdpa(struct virtio_device *vdev)53{54 return to_virtio_vdpa_device(vdev)->vdpa;55}56 57static void virtio_vdpa_get(struct virtio_device *vdev, unsigned int offset,58 void *buf, unsigned int len)59{60 struct vdpa_device *vdpa = vd_get_vdpa(vdev);61 62 vdpa_get_config(vdpa, offset, buf, len);63}64 65static void virtio_vdpa_set(struct virtio_device *vdev, unsigned int offset,66 const void *buf, unsigned int len)67{68 struct vdpa_device *vdpa = vd_get_vdpa(vdev);69 70 vdpa_set_config(vdpa, offset, buf, len);71}72 73static u32 virtio_vdpa_generation(struct virtio_device *vdev)74{75 struct vdpa_device *vdpa = vd_get_vdpa(vdev);76 const struct vdpa_config_ops *ops = vdpa->config;77 78 if (ops->get_generation)79 return ops->get_generation(vdpa);80 81 return 0;82}83 84static u8 virtio_vdpa_get_status(struct virtio_device *vdev)85{86 struct vdpa_device *vdpa = vd_get_vdpa(vdev);87 const struct vdpa_config_ops *ops = vdpa->config;88 89 return ops->get_status(vdpa);90}91 92static void virtio_vdpa_set_status(struct virtio_device *vdev, u8 status)93{94 struct vdpa_device *vdpa = vd_get_vdpa(vdev);95 96 return vdpa_set_status(vdpa, status);97}98 99static void virtio_vdpa_reset(struct virtio_device *vdev)100{101 struct vdpa_device *vdpa = vd_get_vdpa(vdev);102 103 vdpa_reset(vdpa, 0);104}105 106static bool virtio_vdpa_notify(struct virtqueue *vq)107{108 struct vdpa_device *vdpa = vd_get_vdpa(vq->vdev);109 const struct vdpa_config_ops *ops = vdpa->config;110 111 ops->kick_vq(vdpa, vq->index);112 113 return true;114}115 116static bool virtio_vdpa_notify_with_data(struct virtqueue *vq)117{118 struct vdpa_device *vdpa = vd_get_vdpa(vq->vdev);119 const struct vdpa_config_ops *ops = vdpa->config;120 u32 data = vring_notification_data(vq);121 122 ops->kick_vq_with_data(vdpa, data);123 124 return true;125}126 127static irqreturn_t virtio_vdpa_config_cb(void *private)128{129 struct virtio_vdpa_device *vd_dev = private;130 131 virtio_config_changed(&vd_dev->vdev);132 133 return IRQ_HANDLED;134}135 136static irqreturn_t virtio_vdpa_virtqueue_cb(void *private)137{138 struct virtio_vdpa_vq_info *info = private;139 140 return vring_interrupt(0, info->vq);141}142 143static struct virtqueue *144virtio_vdpa_setup_vq(struct virtio_device *vdev, unsigned int index,145 void (*callback)(struct virtqueue *vq),146 const char *name, bool ctx)147{148 struct virtio_vdpa_device *vd_dev = to_virtio_vdpa_device(vdev);149 struct vdpa_device *vdpa = vd_get_vdpa(vdev);150 struct device *dma_dev;151 const struct vdpa_config_ops *ops = vdpa->config;152 struct virtio_vdpa_vq_info *info;153 bool (*notify)(struct virtqueue *vq) = virtio_vdpa_notify;154 struct vdpa_callback cb;155 struct virtqueue *vq;156 u64 desc_addr, driver_addr, device_addr;157 /* Assume split virtqueue, switch to packed if necessary */158 struct vdpa_vq_state state = {0};159 unsigned long flags;160 u32 align, max_num, min_num = 1;161 bool may_reduce_num = true;162 int err;163 164 if (!name)165 return NULL;166 167 if (index >= vdpa->nvqs)168 return ERR_PTR(-ENOENT);169 170 /* We cannot accept VIRTIO_F_NOTIFICATION_DATA without kick_vq_with_data */171 if (__virtio_test_bit(vdev, VIRTIO_F_NOTIFICATION_DATA)) {172 if (ops->kick_vq_with_data)173 notify = virtio_vdpa_notify_with_data;174 else175 __virtio_clear_bit(vdev, VIRTIO_F_NOTIFICATION_DATA);176 }177 178 /* Queue shouldn't already be set up. */179 if (ops->get_vq_ready(vdpa, index))180 return ERR_PTR(-ENOENT);181 182 /* Allocate and fill out our active queue description */183 info = kmalloc(sizeof(*info), GFP_KERNEL);184 if (!info)185 return ERR_PTR(-ENOMEM);186 if (ops->get_vq_size)187 max_num = ops->get_vq_size(vdpa, index);188 else189 max_num = ops->get_vq_num_max(vdpa);190 191 if (max_num == 0) {192 err = -ENOENT;193 goto error_new_virtqueue;194 }195 196 if (ops->get_vq_num_min)197 min_num = ops->get_vq_num_min(vdpa);198 199 may_reduce_num = (max_num == min_num) ? false : true;200 201 /* Create the vring */202 align = ops->get_vq_align(vdpa);203 204 if (ops->get_vq_dma_dev)205 dma_dev = ops->get_vq_dma_dev(vdpa, index);206 else207 dma_dev = vdpa_get_dma_dev(vdpa);208 vq = vring_create_virtqueue_dma(index, max_num, align, vdev,209 true, may_reduce_num, ctx,210 notify, callback, name, dma_dev);211 if (!vq) {212 err = -ENOMEM;213 goto error_new_virtqueue;214 }215 216 vq->num_max = max_num;217 218 /* Setup virtqueue callback */219 cb.callback = callback ? virtio_vdpa_virtqueue_cb : NULL;220 cb.private = info;221 cb.trigger = NULL;222 ops->set_vq_cb(vdpa, index, &cb);223 ops->set_vq_num(vdpa, index, virtqueue_get_vring_size(vq));224 225 desc_addr = virtqueue_get_desc_addr(vq);226 driver_addr = virtqueue_get_avail_addr(vq);227 device_addr = virtqueue_get_used_addr(vq);228 229 if (ops->set_vq_address(vdpa, index,230 desc_addr, driver_addr,231 device_addr)) {232 err = -EINVAL;233 goto err_vq;234 }235 236 /* reset virtqueue state index */237 if (virtio_has_feature(vdev, VIRTIO_F_RING_PACKED)) {238 struct vdpa_vq_state_packed *s = &state.packed;239 240 s->last_avail_counter = 1;241 s->last_avail_idx = 0;242 s->last_used_counter = 1;243 s->last_used_idx = 0;244 }245 err = ops->set_vq_state(vdpa, index, &state);246 if (err)247 goto err_vq;248 249 ops->set_vq_ready(vdpa, index, 1);250 251 vq->priv = info;252 info->vq = vq;253 254 spin_lock_irqsave(&vd_dev->lock, flags);255 list_add(&info->node, &vd_dev->virtqueues);256 spin_unlock_irqrestore(&vd_dev->lock, flags);257 258 return vq;259 260err_vq:261 vring_del_virtqueue(vq);262error_new_virtqueue:263 ops->set_vq_ready(vdpa, index, 0);264 /* VDPA driver should make sure vq is stopeed here */265 WARN_ON(ops->get_vq_ready(vdpa, index));266 kfree(info);267 return ERR_PTR(err);268}269 270static void virtio_vdpa_del_vq(struct virtqueue *vq)271{272 struct virtio_vdpa_device *vd_dev = to_virtio_vdpa_device(vq->vdev);273 struct vdpa_device *vdpa = vd_dev->vdpa;274 const struct vdpa_config_ops *ops = vdpa->config;275 struct virtio_vdpa_vq_info *info = vq->priv;276 unsigned int index = vq->index;277 unsigned long flags;278 279 spin_lock_irqsave(&vd_dev->lock, flags);280 list_del(&info->node);281 spin_unlock_irqrestore(&vd_dev->lock, flags);282 283 /* Select and deactivate the queue (best effort) */284 ops->set_vq_ready(vdpa, index, 0);285 286 vring_del_virtqueue(vq);287 288 kfree(info);289}290 291static void virtio_vdpa_del_vqs(struct virtio_device *vdev)292{293 struct virtqueue *vq, *n;294 295 list_for_each_entry_safe(vq, n, &vdev->vqs, list)296 virtio_vdpa_del_vq(vq);297}298 299static void default_calc_sets(struct irq_affinity *affd, unsigned int affvecs)300{301 affd->nr_sets = 1;302 affd->set_size[0] = affvecs;303}304 305static struct cpumask *306create_affinity_masks(unsigned int nvecs, struct irq_affinity *affd)307{308 unsigned int affvecs = 0, curvec, usedvecs, i;309 struct cpumask *masks = NULL;310 311 if (nvecs > affd->pre_vectors + affd->post_vectors)312 affvecs = nvecs - affd->pre_vectors - affd->post_vectors;313 314 if (!affd->calc_sets)315 affd->calc_sets = default_calc_sets;316 317 affd->calc_sets(affd, affvecs);318 319 if (!affvecs)320 return NULL;321 322 masks = kcalloc(nvecs, sizeof(*masks), GFP_KERNEL);323 if (!masks)324 return NULL;325 326 /* Fill out vectors at the beginning that don't need affinity */327 for (curvec = 0; curvec < affd->pre_vectors; curvec++)328 cpumask_setall(&masks[curvec]);329 330 for (i = 0, usedvecs = 0; i < affd->nr_sets; i++) {331 unsigned int this_vecs = affd->set_size[i];332 int j;333 struct cpumask *result = group_cpus_evenly(this_vecs);334 335 if (!result) {336 kfree(masks);337 return NULL;338 }339 340 for (j = 0; j < this_vecs; j++)341 cpumask_copy(&masks[curvec + j], &result[j]);342 kfree(result);343 344 curvec += this_vecs;345 usedvecs += this_vecs;346 }347 348 /* Fill out vectors at the end that don't need affinity */349 if (usedvecs >= affvecs)350 curvec = affd->pre_vectors + affvecs;351 else352 curvec = affd->pre_vectors + usedvecs;353 for (; curvec < nvecs; curvec++)354 cpumask_setall(&masks[curvec]);355 356 return masks;357}358 359static int virtio_vdpa_find_vqs(struct virtio_device *vdev, unsigned int nvqs,360 struct virtqueue *vqs[],361 struct virtqueue_info vqs_info[],362 struct irq_affinity *desc)363{364 struct virtio_vdpa_device *vd_dev = to_virtio_vdpa_device(vdev);365 struct vdpa_device *vdpa = vd_get_vdpa(vdev);366 const struct vdpa_config_ops *ops = vdpa->config;367 struct irq_affinity default_affd = { 0 };368 struct cpumask *masks;369 struct vdpa_callback cb;370 bool has_affinity = desc && ops->set_vq_affinity;371 int i, err, queue_idx = 0;372 373 if (has_affinity) {374 masks = create_affinity_masks(nvqs, desc ? desc : &default_affd);375 if (!masks)376 return -ENOMEM;377 }378 379 for (i = 0; i < nvqs; ++i) {380 struct virtqueue_info *vqi = &vqs_info[i];381 382 if (!vqi->name) {383 vqs[i] = NULL;384 continue;385 }386 387 vqs[i] = virtio_vdpa_setup_vq(vdev, queue_idx++, vqi->callback,388 vqi->name, vqi->ctx);389 if (IS_ERR(vqs[i])) {390 err = PTR_ERR(vqs[i]);391 goto err_setup_vq;392 }393 394 if (has_affinity)395 ops->set_vq_affinity(vdpa, i, &masks[i]);396 }397 398 cb.callback = virtio_vdpa_config_cb;399 cb.private = vd_dev;400 ops->set_config_cb(vdpa, &cb);401 if (has_affinity)402 kfree(masks);403 404 return 0;405 406err_setup_vq:407 virtio_vdpa_del_vqs(vdev);408 if (has_affinity)409 kfree(masks);410 return err;411}412 413static u64 virtio_vdpa_get_features(struct virtio_device *vdev)414{415 struct vdpa_device *vdpa = vd_get_vdpa(vdev);416 const struct vdpa_config_ops *ops = vdpa->config;417 418 return ops->get_device_features(vdpa);419}420 421static int virtio_vdpa_finalize_features(struct virtio_device *vdev)422{423 struct vdpa_device *vdpa = vd_get_vdpa(vdev);424 425 /* Give virtio_ring a chance to accept features. */426 vring_transport_features(vdev);427 428 return vdpa_set_features(vdpa, vdev->features);429}430 431static const char *virtio_vdpa_bus_name(struct virtio_device *vdev)432{433 struct virtio_vdpa_device *vd_dev = to_virtio_vdpa_device(vdev);434 struct vdpa_device *vdpa = vd_dev->vdpa;435 436 return dev_name(&vdpa->dev);437}438 439static int virtio_vdpa_set_vq_affinity(struct virtqueue *vq,440 const struct cpumask *cpu_mask)441{442 struct virtio_vdpa_device *vd_dev = to_virtio_vdpa_device(vq->vdev);443 struct vdpa_device *vdpa = vd_dev->vdpa;444 const struct vdpa_config_ops *ops = vdpa->config;445 unsigned int index = vq->index;446 447 if (ops->set_vq_affinity)448 return ops->set_vq_affinity(vdpa, index, cpu_mask);449 450 return 0;451}452 453static const struct cpumask *454virtio_vdpa_get_vq_affinity(struct virtio_device *vdev, int index)455{456 struct vdpa_device *vdpa = vd_get_vdpa(vdev);457 const struct vdpa_config_ops *ops = vdpa->config;458 459 if (ops->get_vq_affinity)460 return ops->get_vq_affinity(vdpa, index);461 462 return NULL;463}464 465static const struct virtio_config_ops virtio_vdpa_config_ops = {466 .get = virtio_vdpa_get,467 .set = virtio_vdpa_set,468 .generation = virtio_vdpa_generation,469 .get_status = virtio_vdpa_get_status,470 .set_status = virtio_vdpa_set_status,471 .reset = virtio_vdpa_reset,472 .find_vqs = virtio_vdpa_find_vqs,473 .del_vqs = virtio_vdpa_del_vqs,474 .get_features = virtio_vdpa_get_features,475 .finalize_features = virtio_vdpa_finalize_features,476 .bus_name = virtio_vdpa_bus_name,477 .set_vq_affinity = virtio_vdpa_set_vq_affinity,478 .get_vq_affinity = virtio_vdpa_get_vq_affinity,479};480 481static void virtio_vdpa_release_dev(struct device *_d)482{483 struct virtio_device *vdev =484 container_of(_d, struct virtio_device, dev);485 struct virtio_vdpa_device *vd_dev =486 container_of(vdev, struct virtio_vdpa_device, vdev);487 488 kfree(vd_dev);489}490 491static int virtio_vdpa_probe(struct vdpa_device *vdpa)492{493 const struct vdpa_config_ops *ops = vdpa->config;494 struct virtio_vdpa_device *vd_dev, *reg_dev = NULL;495 int ret = -EINVAL;496 497 vd_dev = kzalloc(sizeof(*vd_dev), GFP_KERNEL);498 if (!vd_dev)499 return -ENOMEM;500 501 vd_dev->vdev.dev.parent = vdpa_get_dma_dev(vdpa);502 vd_dev->vdev.dev.release = virtio_vdpa_release_dev;503 vd_dev->vdev.config = &virtio_vdpa_config_ops;504 vd_dev->vdpa = vdpa;505 INIT_LIST_HEAD(&vd_dev->virtqueues);506 spin_lock_init(&vd_dev->lock);507 508 vd_dev->vdev.id.device = ops->get_device_id(vdpa);509 if (vd_dev->vdev.id.device == 0)510 goto err;511 512 vd_dev->vdev.id.vendor = ops->get_vendor_id(vdpa);513 ret = register_virtio_device(&vd_dev->vdev);514 reg_dev = vd_dev;515 if (ret)516 goto err;517 518 vdpa_set_drvdata(vdpa, vd_dev);519 520 return 0;521 522err:523 if (reg_dev)524 put_device(&vd_dev->vdev.dev);525 else526 kfree(vd_dev);527 return ret;528}529 530static void virtio_vdpa_remove(struct vdpa_device *vdpa)531{532 struct virtio_vdpa_device *vd_dev = vdpa_get_drvdata(vdpa);533 534 unregister_virtio_device(&vd_dev->vdev);535}536 537static struct vdpa_driver virtio_vdpa_driver = {538 .driver = {539 .name = "virtio_vdpa",540 },541 .probe = virtio_vdpa_probe,542 .remove = virtio_vdpa_remove,543};544 545module_vdpa_driver(virtio_vdpa_driver);546 547MODULE_VERSION(MOD_VERSION);548MODULE_LICENSE(MOD_LICENSE);549MODULE_AUTHOR(MOD_AUTHOR);550MODULE_DESCRIPTION(MOD_DESC);551