283 lines · c
1/* SPDX-License-Identifier: MIT */2 3#include <linux/io_uring.h>4#include <sys/mman.h>5#include <sys/syscall.h>6#include <stdio.h>7#include <string.h>8#include <unistd.h>9 10struct io_sq_ring {11 unsigned int *head;12 unsigned int *tail;13 unsigned int *ring_mask;14 unsigned int *ring_entries;15 unsigned int *flags;16 unsigned int *array;17};18 19struct io_cq_ring {20 unsigned int *head;21 unsigned int *tail;22 unsigned int *ring_mask;23 unsigned int *ring_entries;24 struct io_uring_cqe *cqes;25};26 27struct io_uring_sq {28 unsigned int *khead;29 unsigned int *ktail;30 unsigned int *kring_mask;31 unsigned int *kring_entries;32 unsigned int *kflags;33 unsigned int *kdropped;34 unsigned int *array;35 struct io_uring_sqe *sqes;36 37 unsigned int sqe_head;38 unsigned int sqe_tail;39 40 size_t ring_sz;41};42 43struct io_uring_cq {44 unsigned int *khead;45 unsigned int *ktail;46 unsigned int *kring_mask;47 unsigned int *kring_entries;48 unsigned int *koverflow;49 struct io_uring_cqe *cqes;50 51 size_t ring_sz;52};53 54struct io_uring {55 struct io_uring_sq sq;56 struct io_uring_cq cq;57 int ring_fd;58};59 60#if defined(__x86_64) || defined(__i386__)61#define read_barrier() __asm__ __volatile__("":::"memory")62#define write_barrier() __asm__ __volatile__("":::"memory")63#else64#define read_barrier() __sync_synchronize()65#define write_barrier() __sync_synchronize()66#endif67 68static inline int io_uring_mmap(int fd, struct io_uring_params *p,69 struct io_uring_sq *sq, struct io_uring_cq *cq)70{71 size_t size;72 void *ptr;73 int ret;74 75 sq->ring_sz = p->sq_off.array + p->sq_entries * sizeof(unsigned int);76 ptr = mmap(0, sq->ring_sz, PROT_READ | PROT_WRITE,77 MAP_SHARED | MAP_POPULATE, fd, IORING_OFF_SQ_RING);78 if (ptr == MAP_FAILED)79 return -errno;80 sq->khead = ptr + p->sq_off.head;81 sq->ktail = ptr + p->sq_off.tail;82 sq->kring_mask = ptr + p->sq_off.ring_mask;83 sq->kring_entries = ptr + p->sq_off.ring_entries;84 sq->kflags = ptr + p->sq_off.flags;85 sq->kdropped = ptr + p->sq_off.dropped;86 sq->array = ptr + p->sq_off.array;87 88 size = p->sq_entries * sizeof(struct io_uring_sqe);89 sq->sqes = mmap(0, size, PROT_READ | PROT_WRITE,90 MAP_SHARED | MAP_POPULATE, fd, IORING_OFF_SQES);91 if (sq->sqes == MAP_FAILED) {92 ret = -errno;93err:94 munmap(sq->khead, sq->ring_sz);95 return ret;96 }97 98 cq->ring_sz = p->cq_off.cqes + p->cq_entries * sizeof(struct io_uring_cqe);99 ptr = mmap(0, cq->ring_sz, PROT_READ | PROT_WRITE,100 MAP_SHARED | MAP_POPULATE, fd, IORING_OFF_CQ_RING);101 if (ptr == MAP_FAILED) {102 ret = -errno;103 munmap(sq->sqes, p->sq_entries * sizeof(struct io_uring_sqe));104 goto err;105 }106 cq->khead = ptr + p->cq_off.head;107 cq->ktail = ptr + p->cq_off.tail;108 cq->kring_mask = ptr + p->cq_off.ring_mask;109 cq->kring_entries = ptr + p->cq_off.ring_entries;110 cq->koverflow = ptr + p->cq_off.overflow;111 cq->cqes = ptr + p->cq_off.cqes;112 return 0;113}114 115static inline int io_uring_setup(unsigned int entries,116 struct io_uring_params *p)117{118 return syscall(__NR_io_uring_setup, entries, p);119}120 121static inline int io_uring_enter(int fd, unsigned int to_submit,122 unsigned int min_complete,123 unsigned int flags, sigset_t *sig)124{125 return syscall(__NR_io_uring_enter, fd, to_submit, min_complete,126 flags, sig, _NSIG / 8);127}128 129static inline int io_uring_queue_init(unsigned int entries,130 struct io_uring *ring,131 unsigned int flags)132{133 struct io_uring_params p;134 int fd, ret;135 136 memset(ring, 0, sizeof(*ring));137 memset(&p, 0, sizeof(p));138 p.flags = flags;139 140 fd = io_uring_setup(entries, &p);141 if (fd < 0)142 return fd;143 ret = io_uring_mmap(fd, &p, &ring->sq, &ring->cq);144 if (!ret)145 ring->ring_fd = fd;146 else147 close(fd);148 return ret;149}150 151/* Get a sqe */152static inline struct io_uring_sqe *io_uring_get_sqe(struct io_uring *ring)153{154 struct io_uring_sq *sq = &ring->sq;155 156 if (sq->sqe_tail + 1 - sq->sqe_head > *sq->kring_entries)157 return NULL;158 return &sq->sqes[sq->sqe_tail++ & *sq->kring_mask];159}160 161static inline int io_uring_wait_cqe(struct io_uring *ring,162 struct io_uring_cqe **cqe_ptr)163{164 struct io_uring_cq *cq = &ring->cq;165 const unsigned int mask = *cq->kring_mask;166 unsigned int head = *cq->khead;167 int ret;168 169 *cqe_ptr = NULL;170 do {171 read_barrier();172 if (head != *cq->ktail) {173 *cqe_ptr = &cq->cqes[head & mask];174 break;175 }176 ret = io_uring_enter(ring->ring_fd, 0, 1,177 IORING_ENTER_GETEVENTS, NULL);178 if (ret < 0)179 return -errno;180 } while (1);181 182 return 0;183}184 185static inline int io_uring_submit(struct io_uring *ring)186{187 struct io_uring_sq *sq = &ring->sq;188 const unsigned int mask = *sq->kring_mask;189 unsigned int ktail, submitted, to_submit;190 int ret;191 192 read_barrier();193 if (*sq->khead != *sq->ktail) {194 submitted = *sq->kring_entries;195 goto submit;196 }197 if (sq->sqe_head == sq->sqe_tail)198 return 0;199 200 ktail = *sq->ktail;201 to_submit = sq->sqe_tail - sq->sqe_head;202 for (submitted = 0; submitted < to_submit; submitted++) {203 read_barrier();204 sq->array[ktail++ & mask] = sq->sqe_head++ & mask;205 }206 if (!submitted)207 return 0;208 209 if (*sq->ktail != ktail) {210 write_barrier();211 *sq->ktail = ktail;212 write_barrier();213 }214submit:215 ret = io_uring_enter(ring->ring_fd, submitted, 0,216 IORING_ENTER_GETEVENTS, NULL);217 return ret < 0 ? -errno : ret;218}219 220static inline void io_uring_queue_exit(struct io_uring *ring)221{222 struct io_uring_sq *sq = &ring->sq;223 224 munmap(sq->sqes, *sq->kring_entries * sizeof(struct io_uring_sqe));225 munmap(sq->khead, sq->ring_sz);226 close(ring->ring_fd);227}228 229/* Prepare and send the SQE */230static inline void io_uring_prep_cmd(struct io_uring_sqe *sqe, int op,231 int sockfd,232 int level, int optname,233 const void *optval,234 int optlen)235{236 memset(sqe, 0, sizeof(*sqe));237 sqe->opcode = (__u8)IORING_OP_URING_CMD;238 sqe->fd = sockfd;239 sqe->cmd_op = op;240 241 sqe->level = level;242 sqe->optname = optname;243 sqe->optval = (unsigned long long)optval;244 sqe->optlen = optlen;245}246 247static inline int io_uring_register_buffers(struct io_uring *ring,248 const struct iovec *iovecs,249 unsigned int nr_iovecs)250{251 int ret;252 253 ret = syscall(__NR_io_uring_register, ring->ring_fd,254 IORING_REGISTER_BUFFERS, iovecs, nr_iovecs);255 return (ret < 0) ? -errno : ret;256}257 258static inline void io_uring_prep_send(struct io_uring_sqe *sqe, int sockfd,259 const void *buf, size_t len, int flags)260{261 memset(sqe, 0, sizeof(*sqe));262 sqe->opcode = (__u8)IORING_OP_SEND;263 sqe->fd = sockfd;264 sqe->addr = (unsigned long)buf;265 sqe->len = len;266 sqe->msg_flags = (__u32)flags;267}268 269static inline void io_uring_prep_sendzc(struct io_uring_sqe *sqe, int sockfd,270 const void *buf, size_t len, int flags,271 unsigned int zc_flags)272{273 io_uring_prep_send(sqe, sockfd, buf, len, flags);274 sqe->opcode = (__u8)IORING_OP_SEND_ZC;275 sqe->ioprio = zc_flags;276}277 278static inline void io_uring_cqe_seen(struct io_uring *ring)279{280 *(&ring->cq)->khead += 1;281 write_barrier();282}283