584 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2/*3 * VDPA simulator for networking device.4 *5 * Copyright (c) 2020, Red Hat Inc. All rights reserved.6 * Author: Jason Wang <jasowang@redhat.com>7 *8 */9 10#include <linux/init.h>11#include <linux/module.h>12#include <linux/device.h>13#include <linux/kernel.h>14#include <linux/etherdevice.h>15#include <linux/vringh.h>16#include <linux/vdpa.h>17#include <net/netlink.h>18#include <uapi/linux/virtio_net.h>19#include <uapi/linux/vdpa.h>20 21#include "vdpa_sim.h"22 23#define DRV_VERSION "0.1"24#define DRV_AUTHOR "Jason Wang <jasowang@redhat.com>"25#define DRV_DESC "vDPA Device Simulator for networking device"26#define DRV_LICENSE "GPL v2"27 28#define VDPASIM_NET_FEATURES (VDPASIM_FEATURES | \29 (1ULL << VIRTIO_NET_F_MAC) | \30 (1ULL << VIRTIO_NET_F_STATUS) | \31 (1ULL << VIRTIO_NET_F_MTU) | \32 (1ULL << VIRTIO_NET_F_CTRL_VQ) | \33 (1ULL << VIRTIO_NET_F_CTRL_MAC_ADDR))34 35/* 3 virtqueues, 2 address spaces, 2 virtqueue groups */36#define VDPASIM_NET_VQ_NUM 337#define VDPASIM_NET_AS_NUM 238#define VDPASIM_NET_GROUP_NUM 239 40struct vdpasim_dataq_stats {41 struct u64_stats_sync syncp;42 u64 pkts;43 u64 bytes;44 u64 drops;45 u64 errors;46 u64 overruns;47};48 49struct vdpasim_cq_stats {50 struct u64_stats_sync syncp;51 u64 requests;52 u64 successes;53 u64 errors;54};55 56struct vdpasim_net{57 struct vdpasim vdpasim;58 struct vdpasim_dataq_stats tx_stats;59 struct vdpasim_dataq_stats rx_stats;60 struct vdpasim_cq_stats cq_stats;61 void *buffer;62};63 64static struct vdpasim_net *sim_to_net(struct vdpasim *vdpasim)65{66 return container_of(vdpasim, struct vdpasim_net, vdpasim);67}68 69static void vdpasim_net_complete(struct vdpasim_virtqueue *vq, size_t len)70{71 /* Make sure data is wrote before advancing index */72 smp_wmb();73 74 vringh_complete_iotlb(&vq->vring, vq->head, len);75 76 /* Make sure used is visible before rasing the interrupt. */77 smp_wmb();78 79 local_bh_disable();80 if (vringh_need_notify_iotlb(&vq->vring) > 0)81 vringh_notify(&vq->vring);82 local_bh_enable();83}84 85static bool receive_filter(struct vdpasim *vdpasim, size_t len)86{87 bool modern = vdpasim->features & (1ULL << VIRTIO_F_VERSION_1);88 size_t hdr_len = modern ? sizeof(struct virtio_net_hdr_v1) :89 sizeof(struct virtio_net_hdr);90 struct virtio_net_config *vio_config = vdpasim->config;91 struct vdpasim_net *net = sim_to_net(vdpasim);92 93 if (len < ETH_ALEN + hdr_len)94 return false;95 96 if (is_broadcast_ether_addr(net->buffer + hdr_len) ||97 is_multicast_ether_addr(net->buffer + hdr_len))98 return true;99 if (!strncmp(net->buffer + hdr_len, vio_config->mac, ETH_ALEN))100 return true;101 102 return false;103}104 105static virtio_net_ctrl_ack vdpasim_handle_ctrl_mac(struct vdpasim *vdpasim,106 u8 cmd)107{108 struct virtio_net_config *vio_config = vdpasim->config;109 struct vdpasim_virtqueue *cvq = &vdpasim->vqs[2];110 virtio_net_ctrl_ack status = VIRTIO_NET_ERR;111 size_t read;112 113 switch (cmd) {114 case VIRTIO_NET_CTRL_MAC_ADDR_SET:115 read = vringh_iov_pull_iotlb(&cvq->vring, &cvq->in_iov,116 vio_config->mac, ETH_ALEN);117 if (read == ETH_ALEN)118 status = VIRTIO_NET_OK;119 break;120 default:121 break;122 }123 124 return status;125}126 127static void vdpasim_handle_cvq(struct vdpasim *vdpasim)128{129 struct vdpasim_virtqueue *cvq = &vdpasim->vqs[2];130 struct vdpasim_net *net = sim_to_net(vdpasim);131 virtio_net_ctrl_ack status = VIRTIO_NET_ERR;132 struct virtio_net_ctrl_hdr ctrl;133 size_t read, write;134 u64 requests = 0, errors = 0, successes = 0;135 int err;136 137 if (!(vdpasim->features & (1ULL << VIRTIO_NET_F_CTRL_VQ)))138 return;139 140 if (!cvq->ready)141 return;142 143 while (true) {144 err = vringh_getdesc_iotlb(&cvq->vring, &cvq->in_iov,145 &cvq->out_iov,146 &cvq->head, GFP_ATOMIC);147 if (err <= 0)148 break;149 150 ++requests;151 read = vringh_iov_pull_iotlb(&cvq->vring, &cvq->in_iov, &ctrl,152 sizeof(ctrl));153 if (read != sizeof(ctrl)) {154 ++errors;155 break;156 }157 158 switch (ctrl.class) {159 case VIRTIO_NET_CTRL_MAC:160 status = vdpasim_handle_ctrl_mac(vdpasim, ctrl.cmd);161 break;162 default:163 break;164 }165 166 if (status == VIRTIO_NET_OK)167 ++successes;168 else169 ++errors;170 171 /* Make sure data is wrote before advancing index */172 smp_wmb();173 174 write = vringh_iov_push_iotlb(&cvq->vring, &cvq->out_iov,175 &status, sizeof(status));176 vringh_complete_iotlb(&cvq->vring, cvq->head, write);177 vringh_kiov_cleanup(&cvq->in_iov);178 vringh_kiov_cleanup(&cvq->out_iov);179 180 /* Make sure used is visible before rasing the interrupt. */181 smp_wmb();182 183 local_bh_disable();184 if (cvq->cb)185 cvq->cb(cvq->private);186 local_bh_enable();187 }188 189 u64_stats_update_begin(&net->cq_stats.syncp);190 net->cq_stats.requests += requests;191 net->cq_stats.errors += errors;192 net->cq_stats.successes += successes;193 u64_stats_update_end(&net->cq_stats.syncp);194}195 196static void vdpasim_net_work(struct vdpasim *vdpasim)197{198 struct vdpasim_virtqueue *txq = &vdpasim->vqs[1];199 struct vdpasim_virtqueue *rxq = &vdpasim->vqs[0];200 struct vdpasim_net *net = sim_to_net(vdpasim);201 ssize_t read, write;202 u64 tx_pkts = 0, rx_pkts = 0, tx_bytes = 0, rx_bytes = 0;203 u64 rx_drops = 0, rx_overruns = 0, rx_errors = 0, tx_errors = 0;204 int err;205 206 mutex_lock(&vdpasim->mutex);207 208 if (!vdpasim->running)209 goto out;210 211 if (!(vdpasim->status & VIRTIO_CONFIG_S_DRIVER_OK))212 goto out;213 214 vdpasim_handle_cvq(vdpasim);215 216 if (!txq->ready || !rxq->ready)217 goto out;218 219 while (true) {220 err = vringh_getdesc_iotlb(&txq->vring, &txq->out_iov, NULL,221 &txq->head, GFP_ATOMIC);222 if (err <= 0) {223 if (err)224 ++tx_errors;225 break;226 }227 228 ++tx_pkts;229 read = vringh_iov_pull_iotlb(&txq->vring, &txq->out_iov,230 net->buffer, PAGE_SIZE);231 232 tx_bytes += read;233 234 if (!receive_filter(vdpasim, read)) {235 ++rx_drops;236 vdpasim_net_complete(txq, 0);237 continue;238 }239 240 err = vringh_getdesc_iotlb(&rxq->vring, NULL, &rxq->in_iov,241 &rxq->head, GFP_ATOMIC);242 if (err <= 0) {243 ++rx_overruns;244 vdpasim_net_complete(txq, 0);245 break;246 }247 248 write = vringh_iov_push_iotlb(&rxq->vring, &rxq->in_iov,249 net->buffer, read);250 if (write <= 0) {251 ++rx_errors;252 break;253 }254 255 ++rx_pkts;256 rx_bytes += write;257 258 vdpasim_net_complete(txq, 0);259 vdpasim_net_complete(rxq, write);260 261 if (tx_pkts > 4) {262 vdpasim_schedule_work(vdpasim);263 goto out;264 }265 }266 267out:268 mutex_unlock(&vdpasim->mutex);269 270 u64_stats_update_begin(&net->tx_stats.syncp);271 net->tx_stats.pkts += tx_pkts;272 net->tx_stats.bytes += tx_bytes;273 net->tx_stats.errors += tx_errors;274 u64_stats_update_end(&net->tx_stats.syncp);275 276 u64_stats_update_begin(&net->rx_stats.syncp);277 net->rx_stats.pkts += rx_pkts;278 net->rx_stats.bytes += rx_bytes;279 net->rx_stats.drops += rx_drops;280 net->rx_stats.errors += rx_errors;281 net->rx_stats.overruns += rx_overruns;282 u64_stats_update_end(&net->rx_stats.syncp);283}284 285static int vdpasim_net_get_stats(struct vdpasim *vdpasim, u16 idx,286 struct sk_buff *msg,287 struct netlink_ext_ack *extack)288{289 struct vdpasim_net *net = sim_to_net(vdpasim);290 u64 rx_pkts, rx_bytes, rx_errors, rx_overruns, rx_drops;291 u64 tx_pkts, tx_bytes, tx_errors, tx_drops;292 u64 cq_requests, cq_successes, cq_errors;293 unsigned int start;294 int err = -EMSGSIZE;295 296 switch(idx) {297 case 0:298 do {299 start = u64_stats_fetch_begin(&net->rx_stats.syncp);300 rx_pkts = net->rx_stats.pkts;301 rx_bytes = net->rx_stats.bytes;302 rx_errors = net->rx_stats.errors;303 rx_overruns = net->rx_stats.overruns;304 rx_drops = net->rx_stats.drops;305 } while (u64_stats_fetch_retry(&net->rx_stats.syncp, start));306 307 if (nla_put_string(msg, VDPA_ATTR_DEV_VENDOR_ATTR_NAME,308 "rx packets"))309 break;310 if (nla_put_u64_64bit(msg, VDPA_ATTR_DEV_VENDOR_ATTR_VALUE,311 rx_pkts, VDPA_ATTR_PAD))312 break;313 if (nla_put_string(msg, VDPA_ATTR_DEV_VENDOR_ATTR_NAME,314 "rx bytes"))315 break;316 if (nla_put_u64_64bit(msg, VDPA_ATTR_DEV_VENDOR_ATTR_VALUE,317 rx_bytes, VDPA_ATTR_PAD))318 break;319 if (nla_put_string(msg, VDPA_ATTR_DEV_VENDOR_ATTR_NAME,320 "rx errors"))321 break;322 if (nla_put_u64_64bit(msg, VDPA_ATTR_DEV_VENDOR_ATTR_VALUE,323 rx_errors, VDPA_ATTR_PAD))324 break;325 if (nla_put_string(msg, VDPA_ATTR_DEV_VENDOR_ATTR_NAME,326 "rx overruns"))327 break;328 if (nla_put_u64_64bit(msg, VDPA_ATTR_DEV_VENDOR_ATTR_VALUE,329 rx_overruns, VDPA_ATTR_PAD))330 break;331 if (nla_put_string(msg, VDPA_ATTR_DEV_VENDOR_ATTR_NAME,332 "rx drops"))333 break;334 if (nla_put_u64_64bit(msg, VDPA_ATTR_DEV_VENDOR_ATTR_VALUE,335 rx_drops, VDPA_ATTR_PAD))336 break;337 err = 0;338 break;339 case 1:340 do {341 start = u64_stats_fetch_begin(&net->tx_stats.syncp);342 tx_pkts = net->tx_stats.pkts;343 tx_bytes = net->tx_stats.bytes;344 tx_errors = net->tx_stats.errors;345 tx_drops = net->tx_stats.drops;346 } while (u64_stats_fetch_retry(&net->tx_stats.syncp, start));347 348 if (nla_put_string(msg, VDPA_ATTR_DEV_VENDOR_ATTR_NAME,349 "tx packets"))350 break;351 if (nla_put_u64_64bit(msg, VDPA_ATTR_DEV_VENDOR_ATTR_VALUE,352 tx_pkts, VDPA_ATTR_PAD))353 break;354 if (nla_put_string(msg, VDPA_ATTR_DEV_VENDOR_ATTR_NAME,355 "tx bytes"))356 break;357 if (nla_put_u64_64bit(msg, VDPA_ATTR_DEV_VENDOR_ATTR_VALUE,358 tx_bytes, VDPA_ATTR_PAD))359 break;360 if (nla_put_string(msg, VDPA_ATTR_DEV_VENDOR_ATTR_NAME,361 "tx errors"))362 break;363 if (nla_put_u64_64bit(msg, VDPA_ATTR_DEV_VENDOR_ATTR_VALUE,364 tx_errors, VDPA_ATTR_PAD))365 break;366 if (nla_put_string(msg, VDPA_ATTR_DEV_VENDOR_ATTR_NAME,367 "tx drops"))368 break;369 if (nla_put_u64_64bit(msg, VDPA_ATTR_DEV_VENDOR_ATTR_VALUE,370 tx_drops, VDPA_ATTR_PAD))371 break;372 err = 0;373 break;374 case 2:375 do {376 start = u64_stats_fetch_begin(&net->cq_stats.syncp);377 cq_requests = net->cq_stats.requests;378 cq_successes = net->cq_stats.successes;379 cq_errors = net->cq_stats.errors;380 } while (u64_stats_fetch_retry(&net->cq_stats.syncp, start));381 382 if (nla_put_string(msg, VDPA_ATTR_DEV_VENDOR_ATTR_NAME,383 "cvq requests"))384 break;385 if (nla_put_u64_64bit(msg, VDPA_ATTR_DEV_VENDOR_ATTR_VALUE,386 cq_requests, VDPA_ATTR_PAD))387 break;388 if (nla_put_string(msg, VDPA_ATTR_DEV_VENDOR_ATTR_NAME,389 "cvq successes"))390 break;391 if (nla_put_u64_64bit(msg, VDPA_ATTR_DEV_VENDOR_ATTR_VALUE,392 cq_successes, VDPA_ATTR_PAD))393 break;394 if (nla_put_string(msg, VDPA_ATTR_DEV_VENDOR_ATTR_NAME,395 "cvq errors"))396 break;397 if (nla_put_u64_64bit(msg, VDPA_ATTR_DEV_VENDOR_ATTR_VALUE,398 cq_errors, VDPA_ATTR_PAD))399 break;400 err = 0;401 break;402 default:403 err = -EINVAL;404 break;405 }406 407 return err;408}409 410static void vdpasim_net_get_config(struct vdpasim *vdpasim, void *config)411{412 struct virtio_net_config *net_config = config;413 414 net_config->status = cpu_to_vdpasim16(vdpasim, VIRTIO_NET_S_LINK_UP);415}416 417static int vdpasim_net_set_attr(struct vdpa_mgmt_dev *mdev, struct vdpa_device *dev,418 const struct vdpa_dev_set_config *config)419{420 struct vdpasim *vdpasim = container_of(dev, struct vdpasim, vdpa);421 struct virtio_net_config *vio_config = vdpasim->config;422 423 mutex_lock(&vdpasim->mutex);424 425 if (config->mask & (1 << VDPA_ATTR_DEV_NET_CFG_MACADDR)) {426 ether_addr_copy(vio_config->mac, config->net.mac);427 mutex_unlock(&vdpasim->mutex);428 return 0;429 }430 431 mutex_unlock(&vdpasim->mutex);432 return -EOPNOTSUPP;433}434 435static void vdpasim_net_setup_config(struct vdpasim *vdpasim,436 const struct vdpa_dev_set_config *config)437{438 struct virtio_net_config *vio_config = vdpasim->config;439 440 if (config->mask & (1 << VDPA_ATTR_DEV_NET_CFG_MACADDR))441 memcpy(vio_config->mac, config->net.mac, ETH_ALEN);442 if (config->mask & (1 << VDPA_ATTR_DEV_NET_CFG_MTU))443 vio_config->mtu = cpu_to_vdpasim16(vdpasim, config->net.mtu);444 else445 /* Setup default MTU to be 1500 */446 vio_config->mtu = cpu_to_vdpasim16(vdpasim, 1500);447}448 449static void vdpasim_net_free(struct vdpasim *vdpasim)450{451 struct vdpasim_net *net = sim_to_net(vdpasim);452 453 kvfree(net->buffer);454}455 456static void vdpasim_net_mgmtdev_release(struct device *dev)457{458}459 460static struct device vdpasim_net_mgmtdev = {461 .init_name = "vdpasim_net",462 .release = vdpasim_net_mgmtdev_release,463};464 465static int vdpasim_net_dev_add(struct vdpa_mgmt_dev *mdev, const char *name,466 const struct vdpa_dev_set_config *config)467{468 struct vdpasim_dev_attr dev_attr = {};469 struct vdpasim_net *net;470 struct vdpasim *simdev;471 int ret;472 473 dev_attr.mgmt_dev = mdev;474 dev_attr.name = name;475 dev_attr.id = VIRTIO_ID_NET;476 dev_attr.supported_features = VDPASIM_NET_FEATURES;477 dev_attr.nvqs = VDPASIM_NET_VQ_NUM;478 dev_attr.ngroups = VDPASIM_NET_GROUP_NUM;479 dev_attr.nas = VDPASIM_NET_AS_NUM;480 dev_attr.alloc_size = sizeof(struct vdpasim_net);481 dev_attr.config_size = sizeof(struct virtio_net_config);482 dev_attr.get_config = vdpasim_net_get_config;483 dev_attr.work_fn = vdpasim_net_work;484 dev_attr.get_stats = vdpasim_net_get_stats;485 dev_attr.free = vdpasim_net_free;486 487 simdev = vdpasim_create(&dev_attr, config);488 if (IS_ERR(simdev))489 return PTR_ERR(simdev);490 491 vdpasim_net_setup_config(simdev, config);492 493 net = sim_to_net(simdev);494 495 u64_stats_init(&net->tx_stats.syncp);496 u64_stats_init(&net->rx_stats.syncp);497 u64_stats_init(&net->cq_stats.syncp);498 499 net->buffer = kvmalloc(PAGE_SIZE, GFP_KERNEL);500 if (!net->buffer) {501 ret = -ENOMEM;502 goto reg_err;503 }504 505 /*506 * Initialization must be completed before this call, since it can507 * connect the device to the vDPA bus, so requests can arrive after508 * this call.509 */510 ret = _vdpa_register_device(&simdev->vdpa, VDPASIM_NET_VQ_NUM);511 if (ret)512 goto reg_err;513 514 return 0;515 516reg_err:517 put_device(&simdev->vdpa.dev);518 return ret;519}520 521static void vdpasim_net_dev_del(struct vdpa_mgmt_dev *mdev,522 struct vdpa_device *dev)523{524 struct vdpasim *simdev = container_of(dev, struct vdpasim, vdpa);525 526 _vdpa_unregister_device(&simdev->vdpa);527}528 529static const struct vdpa_mgmtdev_ops vdpasim_net_mgmtdev_ops = {530 .dev_add = vdpasim_net_dev_add,531 .dev_del = vdpasim_net_dev_del,532 .dev_set_attr = vdpasim_net_set_attr533};534 535static struct virtio_device_id id_table[] = {536 { VIRTIO_ID_NET, VIRTIO_DEV_ANY_ID },537 { 0 },538};539 540static struct vdpa_mgmt_dev mgmt_dev = {541 .device = &vdpasim_net_mgmtdev,542 .id_table = id_table,543 .ops = &vdpasim_net_mgmtdev_ops,544 .config_attr_mask = (1 << VDPA_ATTR_DEV_NET_CFG_MACADDR |545 1 << VDPA_ATTR_DEV_NET_CFG_MTU |546 1 << VDPA_ATTR_DEV_FEATURES),547 .max_supported_vqs = VDPASIM_NET_VQ_NUM,548 .supported_features = VDPASIM_NET_FEATURES,549};550 551static int __init vdpasim_net_init(void)552{553 int ret;554 555 ret = device_register(&vdpasim_net_mgmtdev);556 if (ret) {557 put_device(&vdpasim_net_mgmtdev);558 return ret;559 }560 561 ret = vdpa_mgmtdev_register(&mgmt_dev);562 if (ret)563 goto parent_err;564 return 0;565 566parent_err:567 device_unregister(&vdpasim_net_mgmtdev);568 return ret;569}570 571static void __exit vdpasim_net_exit(void)572{573 vdpa_mgmtdev_unregister(&mgmt_dev);574 device_unregister(&vdpasim_net_mgmtdev);575}576 577module_init(vdpasim_net_init);578module_exit(vdpasim_net_exit);579 580MODULE_VERSION(DRV_VERSION);581MODULE_LICENSE(DRV_LICENSE);582MODULE_AUTHOR(DRV_AUTHOR);583MODULE_DESCRIPTION(DRV_DESC);584