334 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2/*3 * Copyright (C) 2016 Red Hat, Inc.4 * Author: Michael S. Tsirkin <mst@redhat.com>5 *6 * Partial implementation of virtio 0.9. event index is used for signalling,7 * unconditionally. Design roughly follows linux kernel implementation in order8 * to be able to judge its performance.9 */10#define _GNU_SOURCE11#include "main.h"12#include <stdlib.h>13#include <stdio.h>14#include <assert.h>15#include <string.h>16#include <linux/virtio_ring.h>17 18struct data {19 void *data;20} *data;21 22struct vring ring;23 24/* enabling the below activates experimental ring polling code25 * (which skips index reads on consumer in favor of looking at26 * high bits of ring id ^ 0x8000).27 */28/* #ifdef RING_POLL */29/* enabling the below activates experimental in-order code30 * (which skips ring updates and reads and writes len in descriptor).31 */32/* #ifdef INORDER */33 34#if defined(RING_POLL) && defined(INORDER)35#error "RING_POLL and INORDER are mutually exclusive"36#endif37 38/* how much padding is needed to avoid false cache sharing */39#define HOST_GUEST_PADDING 0x8040 41struct guest {42 unsigned short avail_idx;43 unsigned short last_used_idx;44 unsigned short num_free;45 unsigned short kicked_avail_idx;46#ifndef INORDER47 unsigned short free_head;48#else49 unsigned short reserved_free_head;50#endif51 unsigned char reserved[HOST_GUEST_PADDING - 10];52} guest;53 54struct host {55 /* we do not need to track last avail index56 * unless we have more than one in flight.57 */58 unsigned short used_idx;59 unsigned short called_used_idx;60 unsigned char reserved[HOST_GUEST_PADDING - 4];61} host;62 63/* implemented by ring */64void alloc_ring(void)65{66 int ret;67 int i;68 void *p;69 70 ret = posix_memalign(&p, 0x1000, vring_size(ring_size, 0x1000));71 if (ret) {72 perror("Unable to allocate ring buffer.\n");73 exit(3);74 }75 memset(p, 0, vring_size(ring_size, 0x1000));76 vring_init(&ring, ring_size, p, 0x1000);77 78 guest.avail_idx = 0;79 guest.kicked_avail_idx = -1;80 guest.last_used_idx = 0;81#ifndef INORDER82 /* Put everything in free lists. */83 guest.free_head = 0;84#endif85 for (i = 0; i < ring_size - 1; i++)86 ring.desc[i].next = i + 1;87 host.used_idx = 0;88 host.called_used_idx = -1;89 guest.num_free = ring_size;90 data = malloc(ring_size * sizeof *data);91 if (!data) {92 perror("Unable to allocate data buffer.\n");93 exit(3);94 }95 memset(data, 0, ring_size * sizeof *data);96}97 98/* guest side */99int add_inbuf(unsigned len, void *buf, void *datap)100{101 unsigned head;102#ifndef INORDER103 unsigned avail;104#endif105 struct vring_desc *desc;106 107 if (!guest.num_free)108 return -1;109 110#ifdef INORDER111 head = (ring_size - 1) & (guest.avail_idx++);112#else113 head = guest.free_head;114#endif115 guest.num_free--;116 117 desc = ring.desc;118 desc[head].flags = VRING_DESC_F_NEXT;119 desc[head].addr = (unsigned long)(void *)buf;120 desc[head].len = len;121 /* We do it like this to simulate the way122 * we'd have to flip it if we had multiple123 * descriptors.124 */125 desc[head].flags &= ~VRING_DESC_F_NEXT;126#ifndef INORDER127 guest.free_head = desc[head].next;128#endif129 130 data[head].data = datap;131 132#ifdef RING_POLL133 /* Barrier A (for pairing) */134 smp_release();135 avail = guest.avail_idx++;136 ring.avail->ring[avail & (ring_size - 1)] =137 (head | (avail & ~(ring_size - 1))) ^ 0x8000;138#else139#ifndef INORDER140 /* Barrier A (for pairing) */141 smp_release();142 avail = (ring_size - 1) & (guest.avail_idx++);143 ring.avail->ring[avail] = head;144#endif145 /* Barrier A (for pairing) */146 smp_release();147#endif148 ring.avail->idx = guest.avail_idx;149 return 0;150}151 152void *get_buf(unsigned *lenp, void **bufp)153{154 unsigned head;155 unsigned index;156 void *datap;157 158#ifdef RING_POLL159 head = (ring_size - 1) & guest.last_used_idx;160 index = ring.used->ring[head].id;161 if ((index ^ guest.last_used_idx ^ 0x8000) & ~(ring_size - 1))162 return NULL;163 /* Barrier B (for pairing) */164 smp_acquire();165 index &= ring_size - 1;166#else167 if (ring.used->idx == guest.last_used_idx)168 return NULL;169 /* Barrier B (for pairing) */170 smp_acquire();171#ifdef INORDER172 head = (ring_size - 1) & guest.last_used_idx;173 index = head;174#else175 head = (ring_size - 1) & guest.last_used_idx;176 index = ring.used->ring[head].id;177#endif178 179#endif180#ifdef INORDER181 *lenp = ring.desc[index].len;182#else183 *lenp = ring.used->ring[head].len;184#endif185 datap = data[index].data;186 *bufp = (void*)(unsigned long)ring.desc[index].addr;187 data[index].data = NULL;188#ifndef INORDER189 ring.desc[index].next = guest.free_head;190 guest.free_head = index;191#endif192 guest.num_free++;193 guest.last_used_idx++;194 return datap;195}196 197bool used_empty()198{199 unsigned short last_used_idx = guest.last_used_idx;200#ifdef RING_POLL201 unsigned short head = last_used_idx & (ring_size - 1);202 unsigned index = ring.used->ring[head].id;203 204 return (index ^ last_used_idx ^ 0x8000) & ~(ring_size - 1);205#else206 return ring.used->idx == last_used_idx;207#endif208}209 210void disable_call()211{212 /* Doing nothing to disable calls might cause213 * extra interrupts, but reduces the number of cache misses.214 */215}216 217bool enable_call()218{219 vring_used_event(&ring) = guest.last_used_idx;220 /* Flush call index write */221 /* Barrier D (for pairing) */222 smp_mb();223 return used_empty();224}225 226void kick_available(void)227{228 bool need;229 230 /* Flush in previous flags write */231 /* Barrier C (for pairing) */232 smp_mb();233 need = vring_need_event(vring_avail_event(&ring),234 guest.avail_idx,235 guest.kicked_avail_idx);236 237 guest.kicked_avail_idx = guest.avail_idx;238 if (need)239 kick();240}241 242/* host side */243void disable_kick()244{245 /* Doing nothing to disable kicks might cause246 * extra interrupts, but reduces the number of cache misses.247 */248}249 250bool enable_kick()251{252 vring_avail_event(&ring) = host.used_idx;253 /* Barrier C (for pairing) */254 smp_mb();255 return avail_empty();256}257 258bool avail_empty()259{260 unsigned head = host.used_idx;261#ifdef RING_POLL262 unsigned index = ring.avail->ring[head & (ring_size - 1)];263 264 return ((index ^ head ^ 0x8000) & ~(ring_size - 1));265#else266 return head == ring.avail->idx;267#endif268}269 270bool use_buf(unsigned *lenp, void **bufp)271{272 unsigned used_idx = host.used_idx;273 struct vring_desc *desc;274 unsigned head;275 276#ifdef RING_POLL277 head = ring.avail->ring[used_idx & (ring_size - 1)];278 if ((used_idx ^ head ^ 0x8000) & ~(ring_size - 1))279 return false;280 /* Barrier A (for pairing) */281 smp_acquire();282 283 used_idx &= ring_size - 1;284 desc = &ring.desc[head & (ring_size - 1)];285#else286 if (used_idx == ring.avail->idx)287 return false;288 289 /* Barrier A (for pairing) */290 smp_acquire();291 292 used_idx &= ring_size - 1;293#ifdef INORDER294 head = used_idx;295#else296 head = ring.avail->ring[used_idx];297#endif298 desc = &ring.desc[head];299#endif300 301 *lenp = desc->len;302 *bufp = (void *)(unsigned long)desc->addr;303 304#ifdef INORDER305 desc->len = desc->len - 1;306#else307 /* now update used ring */308 ring.used->ring[used_idx].id = head;309 ring.used->ring[used_idx].len = desc->len - 1;310#endif311 /* Barrier B (for pairing) */312 smp_release();313 host.used_idx++;314 ring.used->idx = host.used_idx;315 316 return true;317}318 319void call_used(void)320{321 bool need;322 323 /* Flush in previous flags write */324 /* Barrier D (for pairing) */325 smp_mb();326 need = vring_need_event(vring_used_event(&ring),327 host.used_idx,328 host.called_used_idx);329 330 host.called_used_idx = host.used_idx;331 if (need)332 call();333}334