321 lines · c
1/* SPDX-License-Identifier: MIT */2/* based on linux-kernel/tools/testing/selftests/net/msg_zerocopy.c */3#include <assert.h>4#include <errno.h>5#include <error.h>6#include <fcntl.h>7#include <limits.h>8#include <stdbool.h>9#include <stdint.h>10#include <stdio.h>11#include <stdlib.h>12#include <string.h>13#include <unistd.h>14 15#include <arpa/inet.h>16#include <linux/errqueue.h>17#include <linux/if_packet.h>18#include <linux/io_uring.h>19#include <linux/ipv6.h>20#include <linux/socket.h>21#include <linux/sockios.h>22#include <net/ethernet.h>23#include <net/if.h>24#include <netinet/in.h>25#include <netinet/ip.h>26#include <netinet/ip6.h>27#include <netinet/tcp.h>28#include <netinet/udp.h>29#include <sys/ioctl.h>30#include <sys/mman.h>31#include <sys/resource.h>32#include <sys/socket.h>33#include <sys/stat.h>34#include <sys/time.h>35#include <sys/types.h>36#include <sys/un.h>37#include <sys/wait.h>38 39#include <io_uring/mini_liburing.h>40 41#define NOTIF_TAG 0xfffffffULL42#define NONZC_TAG 043#define ZC_TAG 144 45enum {46 MODE_NONZC = 0,47 MODE_ZC = 1,48 MODE_ZC_FIXED = 2,49 MODE_MIXED = 3,50};51 52static bool cfg_cork = false;53static int cfg_mode = MODE_ZC_FIXED;54static int cfg_nr_reqs = 8;55static int cfg_family = PF_UNSPEC;56static int cfg_payload_len;57static int cfg_port = 8000;58static int cfg_runtime_ms = 4200;59 60static socklen_t cfg_alen;61static struct sockaddr_storage cfg_dst_addr;62 63static char payload[IP_MAXPACKET] __attribute__((aligned(4096)));64 65static unsigned long gettimeofday_ms(void)66{67 struct timeval tv;68 69 gettimeofday(&tv, NULL);70 return (tv.tv_sec * 1000) + (tv.tv_usec / 1000);71}72 73static void do_setsockopt(int fd, int level, int optname, int val)74{75 if (setsockopt(fd, level, optname, &val, sizeof(val)))76 error(1, errno, "setsockopt %d.%d: %d", level, optname, val);77}78 79static int do_setup_tx(int domain, int type, int protocol)80{81 int fd;82 83 fd = socket(domain, type, protocol);84 if (fd == -1)85 error(1, errno, "socket t");86 87 do_setsockopt(fd, SOL_SOCKET, SO_SNDBUF, 1 << 21);88 89 if (connect(fd, (void *) &cfg_dst_addr, cfg_alen))90 error(1, errno, "connect");91 return fd;92}93 94static void do_tx(int domain, int type, int protocol)95{96 struct io_uring_sqe *sqe;97 struct io_uring_cqe *cqe;98 unsigned long packets = 0, bytes = 0;99 struct io_uring ring;100 struct iovec iov;101 uint64_t tstop;102 int i, fd, ret;103 int compl_cqes = 0;104 105 fd = do_setup_tx(domain, type, protocol);106 107 ret = io_uring_queue_init(512, &ring, 0);108 if (ret)109 error(1, ret, "io_uring: queue init");110 111 iov.iov_base = payload;112 iov.iov_len = cfg_payload_len;113 114 ret = io_uring_register_buffers(&ring, &iov, 1);115 if (ret)116 error(1, ret, "io_uring: buffer registration");117 118 tstop = gettimeofday_ms() + cfg_runtime_ms;119 do {120 if (cfg_cork)121 do_setsockopt(fd, IPPROTO_UDP, UDP_CORK, 1);122 123 for (i = 0; i < cfg_nr_reqs; i++) {124 unsigned zc_flags = 0;125 unsigned buf_idx = 0;126 unsigned mode = cfg_mode;127 unsigned msg_flags = MSG_WAITALL;128 129 if (cfg_mode == MODE_MIXED)130 mode = rand() % 3;131 132 sqe = io_uring_get_sqe(&ring);133 134 if (mode == MODE_NONZC) {135 io_uring_prep_send(sqe, fd, payload,136 cfg_payload_len, msg_flags);137 sqe->user_data = NONZC_TAG;138 } else {139 io_uring_prep_sendzc(sqe, fd, payload,140 cfg_payload_len,141 msg_flags, zc_flags);142 if (mode == MODE_ZC_FIXED) {143 sqe->ioprio |= IORING_RECVSEND_FIXED_BUF;144 sqe->buf_index = buf_idx;145 }146 sqe->user_data = ZC_TAG;147 }148 }149 150 ret = io_uring_submit(&ring);151 if (ret != cfg_nr_reqs)152 error(1, ret, "submit");153 154 if (cfg_cork)155 do_setsockopt(fd, IPPROTO_UDP, UDP_CORK, 0);156 for (i = 0; i < cfg_nr_reqs; i++) {157 ret = io_uring_wait_cqe(&ring, &cqe);158 if (ret)159 error(1, ret, "wait cqe");160 161 if (cqe->user_data != NONZC_TAG &&162 cqe->user_data != ZC_TAG)163 error(1, -EINVAL, "invalid cqe->user_data");164 165 if (cqe->flags & IORING_CQE_F_NOTIF) {166 if (cqe->flags & IORING_CQE_F_MORE)167 error(1, -EINVAL, "invalid notif flags");168 if (compl_cqes <= 0)169 error(1, -EINVAL, "notification mismatch");170 compl_cqes--;171 i--;172 io_uring_cqe_seen(&ring);173 continue;174 }175 if (cqe->flags & IORING_CQE_F_MORE) {176 if (cqe->user_data != ZC_TAG)177 error(1, cqe->res, "unexpected F_MORE");178 compl_cqes++;179 }180 if (cqe->res >= 0) {181 packets++;182 bytes += cqe->res;183 } else if (cqe->res != -EAGAIN) {184 error(1, cqe->res, "send failed");185 }186 io_uring_cqe_seen(&ring);187 }188 } while (gettimeofday_ms() < tstop);189 190 while (compl_cqes) {191 ret = io_uring_wait_cqe(&ring, &cqe);192 if (ret)193 error(1, ret, "wait cqe");194 if (cqe->flags & IORING_CQE_F_MORE)195 error(1, -EINVAL, "invalid notif flags");196 if (!(cqe->flags & IORING_CQE_F_NOTIF))197 error(1, -EINVAL, "missing notif flag");198 199 io_uring_cqe_seen(&ring);200 compl_cqes--;201 }202 203 fprintf(stderr, "tx=%lu (MB=%lu), tx/s=%lu (MB/s=%lu)\n",204 packets, bytes >> 20,205 packets / (cfg_runtime_ms / 1000),206 (bytes >> 20) / (cfg_runtime_ms / 1000));207 208 if (close(fd))209 error(1, errno, "close");210}211 212static void do_test(int domain, int type, int protocol)213{214 int i;215 216 for (i = 0; i < IP_MAXPACKET; i++)217 payload[i] = 'a' + (i % 26);218 do_tx(domain, type, protocol);219}220 221static void usage(const char *filepath)222{223 error(1, 0, "Usage: %s (-4|-6) (udp|tcp) -D<dst_ip> [-s<payload size>] "224 "[-t<time s>] [-n<batch>] [-p<port>] [-m<mode>]", filepath);225}226 227static void parse_opts(int argc, char **argv)228{229 const int max_payload_len = sizeof(payload) -230 sizeof(struct ipv6hdr) -231 sizeof(struct tcphdr) -232 40 /* max tcp options */;233 struct sockaddr_in6 *addr6 = (void *) &cfg_dst_addr;234 struct sockaddr_in *addr4 = (void *) &cfg_dst_addr;235 char *daddr = NULL;236 int c;237 238 if (argc <= 1)239 usage(argv[0]);240 cfg_payload_len = max_payload_len;241 242 while ((c = getopt(argc, argv, "46D:p:s:t:n:c:m:")) != -1) {243 switch (c) {244 case '4':245 if (cfg_family != PF_UNSPEC)246 error(1, 0, "Pass one of -4 or -6");247 cfg_family = PF_INET;248 cfg_alen = sizeof(struct sockaddr_in);249 break;250 case '6':251 if (cfg_family != PF_UNSPEC)252 error(1, 0, "Pass one of -4 or -6");253 cfg_family = PF_INET6;254 cfg_alen = sizeof(struct sockaddr_in6);255 break;256 case 'D':257 daddr = optarg;258 break;259 case 'p':260 cfg_port = strtoul(optarg, NULL, 0);261 break;262 case 's':263 cfg_payload_len = strtoul(optarg, NULL, 0);264 break;265 case 't':266 cfg_runtime_ms = 200 + strtoul(optarg, NULL, 10) * 1000;267 break;268 case 'n':269 cfg_nr_reqs = strtoul(optarg, NULL, 0);270 break;271 case 'c':272 cfg_cork = strtol(optarg, NULL, 0);273 break;274 case 'm':275 cfg_mode = strtol(optarg, NULL, 0);276 break;277 }278 }279 280 switch (cfg_family) {281 case PF_INET:282 memset(addr4, 0, sizeof(*addr4));283 addr4->sin_family = AF_INET;284 addr4->sin_port = htons(cfg_port);285 if (daddr &&286 inet_pton(AF_INET, daddr, &(addr4->sin_addr)) != 1)287 error(1, 0, "ipv4 parse error: %s", daddr);288 break;289 case PF_INET6:290 memset(addr6, 0, sizeof(*addr6));291 addr6->sin6_family = AF_INET6;292 addr6->sin6_port = htons(cfg_port);293 if (daddr &&294 inet_pton(AF_INET6, daddr, &(addr6->sin6_addr)) != 1)295 error(1, 0, "ipv6 parse error: %s", daddr);296 break;297 default:298 error(1, 0, "illegal domain");299 }300 301 if (cfg_payload_len > max_payload_len)302 error(1, 0, "-s: payload exceeds max (%d)", max_payload_len);303 if (optind != argc - 1)304 usage(argv[0]);305}306 307int main(int argc, char **argv)308{309 const char *cfg_test = argv[argc - 1];310 311 parse_opts(argc, argv);312 313 if (!strcmp(cfg_test, "tcp"))314 do_test(cfg_family, SOCK_STREAM, 0);315 else if (!strcmp(cfg_test, "udp"))316 do_test(cfg_family, SOCK_DGRAM, 0);317 else318 error(1, 0, "unknown cfg_test %s", cfg_test);319 return 0;320}321