404 lines · c
1// SPDX-License-Identifier: GPL-2.02#define _GNU_SOURCE3#include <getopt.h>4#include <limits.h>5#include <string.h>6#include <poll.h>7#include <sys/eventfd.h>8#include <stdlib.h>9#include <assert.h>10#include <unistd.h>11#include <sys/ioctl.h>12#include <sys/stat.h>13#include <sys/types.h>14#include <fcntl.h>15#include <stdbool.h>16#include <linux/virtio_types.h>17#include <linux/vhost.h>18#include <linux/virtio.h>19#include <linux/virtio_ring.h>20#include "../../drivers/vhost/test.h"21 22#define RANDOM_BATCH -123 24/* Unused */25void *__kmalloc_fake, *__kfree_ignore_start, *__kfree_ignore_end;26 27struct vq_info {28 int kick;29 int call;30 int num;31 int idx;32 void *ring;33 /* copy used for control */34 struct vring vring;35 struct virtqueue *vq;36};37 38struct vdev_info {39 struct virtio_device vdev;40 int control;41 struct pollfd fds[1];42 struct vq_info vqs[1];43 int nvqs;44 void *buf;45 size_t buf_size;46 struct vhost_memory *mem;47};48 49static const struct vhost_vring_file no_backend = { .fd = -1 },50 backend = { .fd = 1 };51static const struct vhost_vring_state null_state = {};52 53bool vq_notify(struct virtqueue *vq)54{55 struct vq_info *info = vq->priv;56 unsigned long long v = 1;57 int r;58 r = write(info->kick, &v, sizeof v);59 assert(r == sizeof v);60 return true;61}62 63void vq_callback(struct virtqueue *vq)64{65}66 67 68void vhost_vq_setup(struct vdev_info *dev, struct vq_info *info)69{70 struct vhost_vring_state state = { .index = info->idx };71 struct vhost_vring_file file = { .index = info->idx };72 unsigned long long features = dev->vdev.features;73 struct vhost_vring_addr addr = {74 .index = info->idx,75 .desc_user_addr = (uint64_t)(unsigned long)info->vring.desc,76 .avail_user_addr = (uint64_t)(unsigned long)info->vring.avail,77 .used_user_addr = (uint64_t)(unsigned long)info->vring.used,78 };79 int r;80 r = ioctl(dev->control, VHOST_SET_FEATURES, &features);81 assert(r >= 0);82 state.num = info->vring.num;83 r = ioctl(dev->control, VHOST_SET_VRING_NUM, &state);84 assert(r >= 0);85 state.num = 0;86 r = ioctl(dev->control, VHOST_SET_VRING_BASE, &state);87 assert(r >= 0);88 r = ioctl(dev->control, VHOST_SET_VRING_ADDR, &addr);89 assert(r >= 0);90 file.fd = info->kick;91 r = ioctl(dev->control, VHOST_SET_VRING_KICK, &file);92 assert(r >= 0);93 file.fd = info->call;94 r = ioctl(dev->control, VHOST_SET_VRING_CALL, &file);95 assert(r >= 0);96}97 98static void vq_reset(struct vq_info *info, int num, struct virtio_device *vdev)99{100 if (info->vq)101 vring_del_virtqueue(info->vq);102 103 memset(info->ring, 0, vring_size(num, 4096));104 vring_init(&info->vring, num, info->ring, 4096);105 info->vq = vring_new_virtqueue(info->idx, num, 4096, vdev, true, false,106 info->ring, vq_notify, vq_callback, "test");107 assert(info->vq);108 info->vq->priv = info;109}110 111static void vq_info_add(struct vdev_info *dev, int num)112{113 struct vq_info *info = &dev->vqs[dev->nvqs];114 int r;115 info->idx = dev->nvqs;116 info->kick = eventfd(0, EFD_NONBLOCK);117 info->call = eventfd(0, EFD_NONBLOCK);118 r = posix_memalign(&info->ring, 4096, vring_size(num, 4096));119 assert(r >= 0);120 vq_reset(info, num, &dev->vdev);121 vhost_vq_setup(dev, info);122 dev->fds[info->idx].fd = info->call;123 dev->fds[info->idx].events = POLLIN;124 dev->nvqs++;125}126 127static void vdev_info_init(struct vdev_info* dev, unsigned long long features)128{129 int r;130 memset(dev, 0, sizeof *dev);131 dev->vdev.features = features;132 INIT_LIST_HEAD(&dev->vdev.vqs);133 spin_lock_init(&dev->vdev.vqs_list_lock);134 dev->buf_size = 1024;135 dev->buf = malloc(dev->buf_size);136 assert(dev->buf);137 dev->control = open("/dev/vhost-test", O_RDWR);138 assert(dev->control >= 0);139 r = ioctl(dev->control, VHOST_SET_OWNER, NULL);140 assert(r >= 0);141 dev->mem = malloc(offsetof(struct vhost_memory, regions) +142 sizeof dev->mem->regions[0]);143 assert(dev->mem);144 memset(dev->mem, 0, offsetof(struct vhost_memory, regions) +145 sizeof dev->mem->regions[0]);146 dev->mem->nregions = 1;147 dev->mem->regions[0].guest_phys_addr = (long)dev->buf;148 dev->mem->regions[0].userspace_addr = (long)dev->buf;149 dev->mem->regions[0].memory_size = dev->buf_size;150 r = ioctl(dev->control, VHOST_SET_MEM_TABLE, dev->mem);151 assert(r >= 0);152}153 154/* TODO: this is pretty bad: we get a cache line bounce155 * for the wait queue on poll and another one on read,156 * plus the read which is there just to clear the157 * current state. */158static void wait_for_interrupt(struct vdev_info *dev)159{160 int i;161 unsigned long long val;162 poll(dev->fds, dev->nvqs, -1);163 for (i = 0; i < dev->nvqs; ++i)164 if (dev->fds[i].revents & POLLIN) {165 read(dev->fds[i].fd, &val, sizeof val);166 }167}168 169static void run_test(struct vdev_info *dev, struct vq_info *vq,170 bool delayed, int batch, int reset_n, int bufs)171{172 struct scatterlist sl;173 long started = 0, completed = 0, next_reset = reset_n;174 long completed_before, started_before;175 int r, test = 1;176 unsigned int len;177 long long spurious = 0;178 const bool random_batch = batch == RANDOM_BATCH;179 180 r = ioctl(dev->control, VHOST_TEST_RUN, &test);181 assert(r >= 0);182 if (!reset_n) {183 next_reset = INT_MAX;184 }185 186 for (;;) {187 virtqueue_disable_cb(vq->vq);188 completed_before = completed;189 started_before = started;190 do {191 const bool reset = completed > next_reset;192 if (random_batch)193 batch = (random() % vq->vring.num) + 1;194 195 while (started < bufs &&196 (started - completed) < batch) {197 sg_init_one(&sl, dev->buf, dev->buf_size);198 r = virtqueue_add_outbuf(vq->vq, &sl, 1,199 dev->buf + started,200 GFP_ATOMIC);201 if (unlikely(r != 0)) {202 if (r == -ENOSPC &&203 started > started_before)204 r = 0;205 else206 r = -1;207 break;208 }209 210 ++started;211 212 if (unlikely(!virtqueue_kick(vq->vq))) {213 r = -1;214 break;215 }216 }217 218 if (started >= bufs)219 r = -1;220 221 if (reset) {222 r = ioctl(dev->control, VHOST_TEST_SET_BACKEND,223 &no_backend);224 assert(!r);225 }226 227 /* Flush out completed bufs if any */228 while (virtqueue_get_buf(vq->vq, &len)) {229 ++completed;230 r = 0;231 }232 233 if (reset) {234 struct vhost_vring_state s = { .index = 0 };235 236 vq_reset(vq, vq->vring.num, &dev->vdev);237 238 r = ioctl(dev->control, VHOST_GET_VRING_BASE,239 &s);240 assert(!r);241 242 s.num = 0;243 r = ioctl(dev->control, VHOST_SET_VRING_BASE,244 &null_state);245 assert(!r);246 247 r = ioctl(dev->control, VHOST_TEST_SET_BACKEND,248 &backend);249 assert(!r);250 251 started = completed;252 while (completed > next_reset)253 next_reset += completed;254 }255 } while (r == 0);256 if (completed == completed_before && started == started_before)257 ++spurious;258 assert(completed <= bufs);259 assert(started <= bufs);260 if (completed == bufs)261 break;262 if (delayed) {263 if (virtqueue_enable_cb_delayed(vq->vq))264 wait_for_interrupt(dev);265 } else {266 if (virtqueue_enable_cb(vq->vq))267 wait_for_interrupt(dev);268 }269 }270 test = 0;271 r = ioctl(dev->control, VHOST_TEST_RUN, &test);272 assert(r >= 0);273 fprintf(stderr,274 "spurious wakeups: 0x%llx started=0x%lx completed=0x%lx\n",275 spurious, started, completed);276}277 278const char optstring[] = "h";279const struct option longopts[] = {280 {281 .name = "help",282 .val = 'h',283 },284 {285 .name = "event-idx",286 .val = 'E',287 },288 {289 .name = "no-event-idx",290 .val = 'e',291 },292 {293 .name = "indirect",294 .val = 'I',295 },296 {297 .name = "no-indirect",298 .val = 'i',299 },300 {301 .name = "virtio-1",302 .val = '1',303 },304 {305 .name = "no-virtio-1",306 .val = '0',307 },308 {309 .name = "delayed-interrupt",310 .val = 'D',311 },312 {313 .name = "no-delayed-interrupt",314 .val = 'd',315 },316 {317 .name = "batch",318 .val = 'b',319 .has_arg = required_argument,320 },321 {322 .name = "reset",323 .val = 'r',324 .has_arg = optional_argument,325 },326 {327 }328};329 330static void help(int status)331{332 fprintf(stderr, "Usage: virtio_test [--help]"333 " [--no-indirect]"334 " [--no-event-idx]"335 " [--no-virtio-1]"336 " [--delayed-interrupt]"337 " [--batch=random/N]"338 " [--reset=N]"339 "\n");340 341 exit(status);342}343 344int main(int argc, char **argv)345{346 struct vdev_info dev;347 unsigned long long features = (1ULL << VIRTIO_RING_F_INDIRECT_DESC) |348 (1ULL << VIRTIO_RING_F_EVENT_IDX) | (1ULL << VIRTIO_F_VERSION_1);349 long batch = 1, reset = 0;350 int o;351 bool delayed = false;352 353 for (;;) {354 o = getopt_long(argc, argv, optstring, longopts, NULL);355 switch (o) {356 case -1:357 goto done;358 case '?':359 help(2);360 case 'e':361 features &= ~(1ULL << VIRTIO_RING_F_EVENT_IDX);362 break;363 case 'h':364 help(0);365 case 'i':366 features &= ~(1ULL << VIRTIO_RING_F_INDIRECT_DESC);367 break;368 case '0':369 features &= ~(1ULL << VIRTIO_F_VERSION_1);370 break;371 case 'D':372 delayed = true;373 break;374 case 'b':375 if (0 == strcmp(optarg, "random")) {376 batch = RANDOM_BATCH;377 } else {378 batch = strtol(optarg, NULL, 10);379 assert(batch > 0);380 assert(batch < (long)INT_MAX + 1);381 }382 break;383 case 'r':384 if (!optarg) {385 reset = 1;386 } else {387 reset = strtol(optarg, NULL, 10);388 assert(reset > 0);389 assert(reset < (long)INT_MAX + 1);390 }391 break;392 default:393 assert(0);394 break;395 }396 }397 398done:399 vdev_info_init(&dev, features);400 vq_info_add(&dev, 256);401 run_test(&dev, &dev.vqs[0], delayed, batch, reset, 0x100000);402 return 0;403}404