271 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2/*3 * Copyright (C) 2016 Red Hat, Inc.4 * Author: Michael S. Tsirkin <mst@redhat.com>5 *6 * Simple descriptor-based ring. virtio 0.9 compatible event index is used for7 * signalling, unconditionally.8 */9#define _GNU_SOURCE10#include "main.h"11#include <stdlib.h>12#include <stdio.h>13#include <string.h>14 15/* Next - Where next entry will be written.16 * Prev - "Next" value when event triggered previously.17 * Event - Peer requested event after writing this entry.18 */19static inline bool need_event(unsigned short event,20 unsigned short next,21 unsigned short prev)22{23 return (unsigned short)(next - event - 1) < (unsigned short)(next - prev);24}25 26/* Design:27 * Guest adds descriptors with unique index values and DESC_HW in flags.28 * Host overwrites used descriptors with correct len, index, and DESC_HW clear.29 * Flags are always set last.30 */31#define DESC_HW 0x132 33struct desc {34 unsigned short flags;35 unsigned short index;36 unsigned len;37 unsigned long long addr;38};39 40/* how much padding is needed to avoid false cache sharing */41#define HOST_GUEST_PADDING 0x8042 43/* Mostly read */44struct event {45 unsigned short kick_index;46 unsigned char reserved0[HOST_GUEST_PADDING - 2];47 unsigned short call_index;48 unsigned char reserved1[HOST_GUEST_PADDING - 2];49};50 51struct data {52 void *buf; /* descriptor is writeable, we can't get buf from there */53 void *data;54} *data;55 56struct desc *ring;57struct event *event;58 59struct guest {60 unsigned avail_idx;61 unsigned last_used_idx;62 unsigned num_free;63 unsigned kicked_avail_idx;64 unsigned char reserved[HOST_GUEST_PADDING - 12];65} guest;66 67struct host {68 /* we do not need to track last avail index69 * unless we have more than one in flight.70 */71 unsigned used_idx;72 unsigned called_used_idx;73 unsigned char reserved[HOST_GUEST_PADDING - 4];74} host;75 76/* implemented by ring */77void alloc_ring(void)78{79 int ret;80 int i;81 82 ret = posix_memalign((void **)&ring, 0x1000, ring_size * sizeof *ring);83 if (ret) {84 perror("Unable to allocate ring buffer.\n");85 exit(3);86 }87 event = calloc(1, sizeof(*event));88 if (!event) {89 perror("Unable to allocate event buffer.\n");90 exit(3);91 }92 guest.avail_idx = 0;93 guest.kicked_avail_idx = -1;94 guest.last_used_idx = 0;95 host.used_idx = 0;96 host.called_used_idx = -1;97 for (i = 0; i < ring_size; ++i) {98 struct desc desc = {99 .index = i,100 };101 ring[i] = desc;102 }103 guest.num_free = ring_size;104 data = calloc(ring_size, sizeof(*data));105 if (!data) {106 perror("Unable to allocate data buffer.\n");107 exit(3);108 }109}110 111/* guest side */112int add_inbuf(unsigned len, void *buf, void *datap)113{114 unsigned head, index;115 116 if (!guest.num_free)117 return -1;118 119 guest.num_free--;120 head = (ring_size - 1) & (guest.avail_idx++);121 122 /* Start with a write. On MESI architectures this helps123 * avoid a shared state with consumer that is polling this descriptor.124 */125 ring[head].addr = (unsigned long)(void*)buf;126 ring[head].len = len;127 /* read below might bypass write above. That is OK because it's just an128 * optimization. If this happens, we will get the cache line in a129 * shared state which is unfortunate, but probably not worth it to130 * add an explicit full barrier to avoid this.131 */132 barrier();133 index = ring[head].index;134 data[index].buf = buf;135 data[index].data = datap;136 /* Barrier A (for pairing) */137 smp_release();138 ring[head].flags = DESC_HW;139 140 return 0;141}142 143void *get_buf(unsigned *lenp, void **bufp)144{145 unsigned head = (ring_size - 1) & guest.last_used_idx;146 unsigned index;147 void *datap;148 149 if (ring[head].flags & DESC_HW)150 return NULL;151 /* Barrier B (for pairing) */152 smp_acquire();153 *lenp = ring[head].len;154 index = ring[head].index & (ring_size - 1);155 datap = data[index].data;156 *bufp = data[index].buf;157 data[index].buf = NULL;158 data[index].data = NULL;159 guest.num_free++;160 guest.last_used_idx++;161 return datap;162}163 164bool used_empty()165{166 unsigned head = (ring_size - 1) & guest.last_used_idx;167 168 return (ring[head].flags & DESC_HW);169}170 171void disable_call()172{173 /* Doing nothing to disable calls might cause174 * extra interrupts, but reduces the number of cache misses.175 */176}177 178bool enable_call()179{180 event->call_index = guest.last_used_idx;181 /* Flush call index write */182 /* Barrier D (for pairing) */183 smp_mb();184 return used_empty();185}186 187void kick_available(void)188{189 bool need;190 191 /* Flush in previous flags write */192 /* Barrier C (for pairing) */193 smp_mb();194 need = need_event(event->kick_index,195 guest.avail_idx,196 guest.kicked_avail_idx);197 198 guest.kicked_avail_idx = guest.avail_idx;199 if (need)200 kick();201}202 203/* host side */204void disable_kick()205{206 /* Doing nothing to disable kicks might cause207 * extra interrupts, but reduces the number of cache misses.208 */209}210 211bool enable_kick()212{213 event->kick_index = host.used_idx;214 /* Barrier C (for pairing) */215 smp_mb();216 return avail_empty();217}218 219bool avail_empty()220{221 unsigned head = (ring_size - 1) & host.used_idx;222 223 return !(ring[head].flags & DESC_HW);224}225 226bool use_buf(unsigned *lenp, void **bufp)227{228 unsigned head = (ring_size - 1) & host.used_idx;229 230 if (!(ring[head].flags & DESC_HW))231 return false;232 233 /* make sure length read below is not speculated */234 /* Barrier A (for pairing) */235 smp_acquire();236 237 /* simple in-order completion: we don't need238 * to touch index at all. This also means we239 * can just modify the descriptor in-place.240 */241 ring[head].len--;242 /* Make sure len is valid before flags.243 * Note: alternative is to write len and flags in one access -244 * possible on 64 bit architectures but wmb is free on Intel anyway245 * so I have no way to test whether it's a gain.246 */247 /* Barrier B (for pairing) */248 smp_release();249 ring[head].flags = 0;250 host.used_idx++;251 return true;252}253 254void call_used(void)255{256 bool need;257 258 /* Flush in previous flags write */259 /* Barrier D (for pairing) */260 smp_mb();261 262 need = need_event(event->call_index,263 host.used_idx,264 host.called_used_idx);265 266 host.called_used_idx = host.used_idx;267 268 if (need)269 call();270}271