735 lines · c
1// SPDX-License-Identifier: GPL-2.02 3#define _GNU_SOURCE4 5#include <arpa/inet.h>6#include <errno.h>7#include <error.h>8#include <linux/errqueue.h>9#include <linux/net_tstamp.h>10#include <netinet/if_ether.h>11#include <netinet/in.h>12#include <netinet/ip.h>13#include <netinet/ip6.h>14#include <netinet/udp.h>15#include <poll.h>16#include <sched.h>17#include <signal.h>18#include <stdbool.h>19#include <stdio.h>20#include <stdlib.h>21#include <string.h>22#include <sys/socket.h>23#include <sys/time.h>24#include <sys/poll.h>25#include <sys/types.h>26#include <unistd.h>27 28#include "../kselftest.h"29 30#ifndef ETH_MAX_MTU31#define ETH_MAX_MTU 0xFFFFU32#endif33 34#ifndef UDP_SEGMENT35#define UDP_SEGMENT 10336#endif37 38#ifndef SO_ZEROCOPY39#define SO_ZEROCOPY 6040#endif41 42#ifndef SO_EE_ORIGIN_ZEROCOPY43#define SO_EE_ORIGIN_ZEROCOPY 544#endif45 46#ifndef MSG_ZEROCOPY47#define MSG_ZEROCOPY 0x400000048#endif49 50#ifndef ENOTSUPP51#define ENOTSUPP 52452#endif53 54#define NUM_PKT 10055 56static bool cfg_cache_trash;57static int cfg_cpu = -1;58static int cfg_connected = true;59static int cfg_family = PF_UNSPEC;60static uint16_t cfg_mss;61static int cfg_payload_len = (1472 * 42);62static int cfg_port = 8000;63static int cfg_runtime_ms = -1;64static bool cfg_poll;65static int cfg_poll_loop_timeout_ms = 2000;66static bool cfg_segment;67static bool cfg_sendmmsg;68static bool cfg_tcp;69static uint32_t cfg_tx_ts = SOF_TIMESTAMPING_TX_SOFTWARE;70static bool cfg_tx_tstamp;71static bool cfg_audit;72static bool cfg_verbose;73static bool cfg_zerocopy;74static int cfg_msg_nr;75static uint16_t cfg_gso_size;76static unsigned long total_num_msgs;77static unsigned long total_num_sends;78static unsigned long stat_tx_ts;79static unsigned long stat_tx_ts_errors;80static unsigned long tstart;81static unsigned long tend;82static unsigned long stat_zcopies;83 84static socklen_t cfg_alen;85static struct sockaddr_storage cfg_dst_addr;86 87static bool interrupted;88static char buf[NUM_PKT][ETH_MAX_MTU];89 90static void sigint_handler(int signum)91{92 if (signum == SIGINT)93 interrupted = true;94}95 96static unsigned long gettimeofday_ms(void)97{98 struct timeval tv;99 100 gettimeofday(&tv, NULL);101 return (tv.tv_sec * 1000) + (tv.tv_usec / 1000);102}103 104static int set_cpu(int cpu)105{106 cpu_set_t mask;107 108 CPU_ZERO(&mask);109 CPU_SET(cpu, &mask);110 if (sched_setaffinity(0, sizeof(mask), &mask))111 error(1, 0, "setaffinity %d", cpu);112 113 return 0;114}115 116static void setup_sockaddr(int domain, const char *str_addr, void *sockaddr)117{118 struct sockaddr_in6 *addr6 = (void *) sockaddr;119 struct sockaddr_in *addr4 = (void *) sockaddr;120 121 switch (domain) {122 case PF_INET:123 addr4->sin_family = AF_INET;124 addr4->sin_port = htons(cfg_port);125 if (inet_pton(AF_INET, str_addr, &(addr4->sin_addr)) != 1)126 error(1, 0, "ipv4 parse error: %s", str_addr);127 break;128 case PF_INET6:129 addr6->sin6_family = AF_INET6;130 addr6->sin6_port = htons(cfg_port);131 if (inet_pton(AF_INET6, str_addr, &(addr6->sin6_addr)) != 1)132 error(1, 0, "ipv6 parse error: %s", str_addr);133 break;134 default:135 error(1, 0, "illegal domain");136 }137}138 139static void flush_cmsg(struct cmsghdr *cmsg)140{141 struct sock_extended_err *err;142 struct scm_timestamping *tss;143 __u32 lo;144 __u32 hi;145 int i;146 147 switch (cmsg->cmsg_level) {148 case SOL_SOCKET:149 if (cmsg->cmsg_type == SO_TIMESTAMPING) {150 i = (cfg_tx_ts == SOF_TIMESTAMPING_TX_HARDWARE) ? 2 : 0;151 tss = (struct scm_timestamping *)CMSG_DATA(cmsg);152 if (tss->ts[i].tv_sec == 0)153 stat_tx_ts_errors++;154 } else {155 error(1, 0, "unknown SOL_SOCKET cmsg type=%u\n",156 cmsg->cmsg_type);157 }158 break;159 case SOL_IP:160 case SOL_IPV6:161 switch (cmsg->cmsg_type) {162 case IP_RECVERR:163 case IPV6_RECVERR:164 {165 err = (struct sock_extended_err *)CMSG_DATA(cmsg);166 switch (err->ee_origin) {167 case SO_EE_ORIGIN_TIMESTAMPING:168 /* Got a TX timestamp from error queue */169 stat_tx_ts++;170 break;171 case SO_EE_ORIGIN_ICMP:172 case SO_EE_ORIGIN_ICMP6:173 if (cfg_verbose)174 fprintf(stderr,175 "received ICMP error: type=%u, code=%u\n",176 err->ee_type, err->ee_code);177 break;178 case SO_EE_ORIGIN_ZEROCOPY:179 {180 lo = err->ee_info;181 hi = err->ee_data;182 /* range of IDs acknowledged */183 stat_zcopies += hi - lo + 1;184 break;185 }186 case SO_EE_ORIGIN_LOCAL:187 if (cfg_verbose)188 fprintf(stderr,189 "received packet with local origin: %u\n",190 err->ee_origin);191 break;192 default:193 error(0, 1, "received packet with origin: %u",194 err->ee_origin);195 }196 break;197 }198 default:199 error(0, 1, "unknown IP msg type=%u\n",200 cmsg->cmsg_type);201 break;202 }203 break;204 default:205 error(0, 1, "unknown cmsg level=%u\n",206 cmsg->cmsg_level);207 }208}209 210static void flush_errqueue_recv(int fd)211{212 char control[CMSG_SPACE(sizeof(struct scm_timestamping)) +213 CMSG_SPACE(sizeof(struct sock_extended_err)) +214 CMSG_SPACE(sizeof(struct sockaddr_in6))] = {0};215 struct msghdr msg = {0};216 struct cmsghdr *cmsg;217 int ret;218 219 while (1) {220 msg.msg_control = control;221 msg.msg_controllen = sizeof(control);222 ret = recvmsg(fd, &msg, MSG_ERRQUEUE);223 if (ret == -1 && errno == EAGAIN)224 break;225 if (ret == -1)226 error(1, errno, "errqueue");227 if (msg.msg_flags != MSG_ERRQUEUE)228 error(1, 0, "errqueue: flags 0x%x\n", msg.msg_flags);229 if (cfg_audit) {230 for (cmsg = CMSG_FIRSTHDR(&msg);231 cmsg;232 cmsg = CMSG_NXTHDR(&msg, cmsg))233 flush_cmsg(cmsg);234 }235 msg.msg_flags = 0;236 }237}238 239static void flush_errqueue(int fd, const bool do_poll,240 unsigned long poll_timeout, const bool poll_err)241{242 if (do_poll) {243 struct pollfd fds = {0};244 int ret;245 246 fds.fd = fd;247 ret = poll(&fds, 1, poll_timeout);248 if (ret == 0) {249 if ((cfg_verbose) && (poll_err))250 fprintf(stderr, "poll timeout\n");251 } else if (ret < 0) {252 error(1, errno, "poll");253 }254 }255 256 flush_errqueue_recv(fd);257}258 259static void flush_errqueue_retry(int fd, unsigned long num_sends)260{261 unsigned long tnow, tstop;262 bool first_try = true;263 264 tnow = gettimeofday_ms();265 tstop = tnow + cfg_poll_loop_timeout_ms;266 do {267 flush_errqueue(fd, true, tstop - tnow, first_try);268 first_try = false;269 tnow = gettimeofday_ms();270 } while ((stat_zcopies != num_sends) && (tnow < tstop));271}272 273static int send_tcp(int fd, char *data)274{275 int ret, done = 0, count = 0;276 277 while (done < cfg_payload_len) {278 ret = send(fd, data + done, cfg_payload_len - done,279 cfg_zerocopy ? MSG_ZEROCOPY : 0);280 if (ret == -1)281 error(1, errno, "write");282 283 done += ret;284 count++;285 }286 287 return count;288}289 290static int send_udp(int fd, char *data)291{292 int ret, total_len, len, count = 0;293 294 total_len = cfg_payload_len;295 296 while (total_len) {297 len = total_len < cfg_mss ? total_len : cfg_mss;298 299 ret = sendto(fd, data, len, cfg_zerocopy ? MSG_ZEROCOPY : 0,300 cfg_connected ? NULL : (void *)&cfg_dst_addr,301 cfg_connected ? 0 : cfg_alen);302 if (ret == -1)303 error(1, errno, "write");304 if (ret != len)305 error(1, errno, "write: %uB != %uB\n", ret, len);306 307 total_len -= len;308 count++;309 }310 311 return count;312}313 314static void send_ts_cmsg(struct cmsghdr *cm)315{316 uint32_t *valp;317 318 cm->cmsg_level = SOL_SOCKET;319 cm->cmsg_type = SO_TIMESTAMPING;320 cm->cmsg_len = CMSG_LEN(sizeof(cfg_tx_ts));321 valp = (void *)CMSG_DATA(cm);322 *valp = cfg_tx_ts;323}324 325static int send_udp_sendmmsg(int fd, char *data)326{327 char control[CMSG_SPACE(sizeof(cfg_tx_ts))] = {0};328 const int max_nr_msg = ETH_MAX_MTU / ETH_DATA_LEN;329 struct mmsghdr mmsgs[max_nr_msg];330 struct iovec iov[max_nr_msg];331 unsigned int off = 0, left;332 size_t msg_controllen = 0;333 int i = 0, ret;334 335 memset(mmsgs, 0, sizeof(mmsgs));336 337 if (cfg_tx_tstamp) {338 struct msghdr msg = {0};339 struct cmsghdr *cmsg;340 341 msg.msg_control = control;342 msg.msg_controllen = sizeof(control);343 cmsg = CMSG_FIRSTHDR(&msg);344 send_ts_cmsg(cmsg);345 msg_controllen += CMSG_SPACE(sizeof(cfg_tx_ts));346 }347 348 left = cfg_payload_len;349 while (left) {350 if (i == max_nr_msg)351 error(1, 0, "sendmmsg: exceeds max_nr_msg");352 353 iov[i].iov_base = data + off;354 iov[i].iov_len = cfg_mss < left ? cfg_mss : left;355 356 mmsgs[i].msg_hdr.msg_iov = iov + i;357 mmsgs[i].msg_hdr.msg_iovlen = 1;358 359 mmsgs[i].msg_hdr.msg_name = (void *)&cfg_dst_addr;360 mmsgs[i].msg_hdr.msg_namelen = cfg_alen;361 if (msg_controllen) {362 mmsgs[i].msg_hdr.msg_control = control;363 mmsgs[i].msg_hdr.msg_controllen = msg_controllen;364 }365 366 off += iov[i].iov_len;367 left -= iov[i].iov_len;368 i++;369 }370 371 ret = sendmmsg(fd, mmsgs, i, cfg_zerocopy ? MSG_ZEROCOPY : 0);372 if (ret == -1)373 error(1, errno, "sendmmsg");374 375 return ret;376}377 378static void send_udp_segment_cmsg(struct cmsghdr *cm)379{380 uint16_t *valp;381 382 cm->cmsg_level = SOL_UDP;383 cm->cmsg_type = UDP_SEGMENT;384 cm->cmsg_len = CMSG_LEN(sizeof(cfg_gso_size));385 valp = (void *)CMSG_DATA(cm);386 *valp = cfg_gso_size;387}388 389static int send_udp_segment(int fd, char *data)390{391 char control[CMSG_SPACE(sizeof(cfg_gso_size)) +392 CMSG_SPACE(sizeof(cfg_tx_ts))] = {0};393 struct msghdr msg = {0};394 struct iovec iov = {0};395 size_t msg_controllen;396 struct cmsghdr *cmsg;397 int ret;398 399 iov.iov_base = data;400 iov.iov_len = cfg_payload_len;401 402 msg.msg_iov = &iov;403 msg.msg_iovlen = 1;404 405 msg.msg_control = control;406 msg.msg_controllen = sizeof(control);407 cmsg = CMSG_FIRSTHDR(&msg);408 send_udp_segment_cmsg(cmsg);409 msg_controllen = CMSG_SPACE(sizeof(cfg_mss));410 if (cfg_tx_tstamp) {411 cmsg = CMSG_NXTHDR(&msg, cmsg);412 send_ts_cmsg(cmsg);413 msg_controllen += CMSG_SPACE(sizeof(cfg_tx_ts));414 }415 416 msg.msg_controllen = msg_controllen;417 msg.msg_name = (void *)&cfg_dst_addr;418 msg.msg_namelen = cfg_alen;419 420 ret = sendmsg(fd, &msg, cfg_zerocopy ? MSG_ZEROCOPY : 0);421 if (ret == -1)422 error(1, errno, "sendmsg");423 if (ret != iov.iov_len)424 error(1, 0, "sendmsg: %u != %llu\n", ret,425 (unsigned long long)iov.iov_len);426 427 return 1;428}429 430static void usage(const char *filepath)431{432 error(1, 0, "Usage: %s [-46acmHPtTuvz] [-C cpu] [-D dst ip] [-l secs] "433 "[-L secs] [-M messagenr] [-p port] [-s sendsize] [-S gsosize]",434 filepath);435}436 437static void parse_opts(int argc, char **argv)438{439 const char *bind_addr = NULL;440 int max_len, hdrlen;441 int c;442 443 while ((c = getopt(argc, argv, "46acC:D:Hl:L:mM:p:s:PS:tTuvz")) != -1) {444 switch (c) {445 case '4':446 if (cfg_family != PF_UNSPEC)447 error(1, 0, "Pass one of -4 or -6");448 cfg_family = PF_INET;449 cfg_alen = sizeof(struct sockaddr_in);450 break;451 case '6':452 if (cfg_family != PF_UNSPEC)453 error(1, 0, "Pass one of -4 or -6");454 cfg_family = PF_INET6;455 cfg_alen = sizeof(struct sockaddr_in6);456 break;457 case 'a':458 cfg_audit = true;459 break;460 case 'c':461 cfg_cache_trash = true;462 break;463 case 'C':464 cfg_cpu = strtol(optarg, NULL, 0);465 break;466 case 'D':467 bind_addr = optarg;468 break;469 case 'l':470 cfg_runtime_ms = strtoul(optarg, NULL, 10) * 1000;471 break;472 case 'L':473 cfg_poll_loop_timeout_ms = strtoul(optarg, NULL, 10) * 1000;474 break;475 case 'm':476 cfg_sendmmsg = true;477 break;478 case 'M':479 cfg_msg_nr = strtoul(optarg, NULL, 10);480 break;481 case 'p':482 cfg_port = strtoul(optarg, NULL, 0);483 break;484 case 'P':485 cfg_poll = true;486 break;487 case 's':488 cfg_payload_len = strtoul(optarg, NULL, 0);489 break;490 case 'S':491 cfg_gso_size = strtoul(optarg, NULL, 0);492 cfg_segment = true;493 break;494 case 'H':495 cfg_tx_ts = SOF_TIMESTAMPING_TX_HARDWARE;496 cfg_tx_tstamp = true;497 break;498 case 't':499 cfg_tcp = true;500 break;501 case 'T':502 cfg_tx_tstamp = true;503 break;504 case 'u':505 cfg_connected = false;506 break;507 case 'v':508 cfg_verbose = true;509 break;510 case 'z':511 cfg_zerocopy = true;512 break;513 default:514 exit(1);515 }516 }517 518 if (!bind_addr)519 bind_addr = cfg_family == PF_INET6 ? "::" : "0.0.0.0";520 521 setup_sockaddr(cfg_family, bind_addr, &cfg_dst_addr);522 523 if (optind != argc)524 usage(argv[0]);525 526 if (cfg_family == PF_UNSPEC)527 error(1, 0, "must pass one of -4 or -6");528 if (cfg_tcp && !cfg_connected)529 error(1, 0, "connectionless tcp makes no sense");530 if (cfg_segment && cfg_sendmmsg)531 error(1, 0, "cannot combine segment offload and sendmmsg");532 if (cfg_tx_tstamp && !(cfg_segment || cfg_sendmmsg))533 error(1, 0, "Options -T and -H require either -S or -m option");534 535 if (cfg_family == PF_INET)536 hdrlen = sizeof(struct iphdr) + sizeof(struct udphdr);537 else538 hdrlen = sizeof(struct ip6_hdr) + sizeof(struct udphdr);539 540 cfg_mss = ETH_DATA_LEN - hdrlen;541 max_len = ETH_MAX_MTU - hdrlen;542 if (!cfg_gso_size)543 cfg_gso_size = cfg_mss;544 545 if (cfg_payload_len > max_len)546 error(1, 0, "payload length %u exceeds max %u",547 cfg_payload_len, max_len);548}549 550static void set_pmtu_discover(int fd, bool is_ipv4)551{552 int level, name, val;553 554 if (is_ipv4) {555 level = SOL_IP;556 name = IP_MTU_DISCOVER;557 val = IP_PMTUDISC_DO;558 } else {559 level = SOL_IPV6;560 name = IPV6_MTU_DISCOVER;561 val = IPV6_PMTUDISC_DO;562 }563 564 if (setsockopt(fd, level, name, &val, sizeof(val)))565 error(1, errno, "setsockopt path mtu");566}567 568static void set_tx_timestamping(int fd)569{570 int val = SOF_TIMESTAMPING_OPT_CMSG | SOF_TIMESTAMPING_OPT_ID |571 SOF_TIMESTAMPING_OPT_TSONLY;572 573 if (cfg_tx_ts == SOF_TIMESTAMPING_TX_SOFTWARE)574 val |= SOF_TIMESTAMPING_SOFTWARE;575 else576 val |= SOF_TIMESTAMPING_RAW_HARDWARE;577 578 if (setsockopt(fd, SOL_SOCKET, SO_TIMESTAMPING, &val, sizeof(val)))579 error(1, errno, "setsockopt tx timestamping");580}581 582static void print_audit_report(unsigned long num_msgs, unsigned long num_sends)583{584 unsigned long tdelta;585 586 tdelta = tend - tstart;587 if (!tdelta)588 return;589 590 fprintf(stderr, "Summary over %lu.%03lu seconds...\n",591 tdelta / 1000, tdelta % 1000);592 fprintf(stderr,593 "sum %s tx: %6lu MB/s %10lu calls (%lu/s) %10lu msgs (%lu/s)\n",594 cfg_tcp ? "tcp" : "udp",595 ((num_msgs * cfg_payload_len) >> 10) / tdelta,596 num_sends, num_sends * 1000 / tdelta,597 num_msgs, num_msgs * 1000 / tdelta);598 599 if (cfg_tx_tstamp) {600 if (stat_tx_ts_errors)601 error(1, 0,602 "Expected clean TX Timestamps: %9lu msgs received %6lu errors",603 stat_tx_ts, stat_tx_ts_errors);604 if (stat_tx_ts != num_sends)605 error(1, 0,606 "Unexpected number of TX Timestamps: %9lu expected %9lu received",607 num_sends, stat_tx_ts);608 fprintf(stderr,609 "Tx Timestamps: %19lu received %17lu errors\n",610 stat_tx_ts, stat_tx_ts_errors);611 }612 613 if (cfg_zerocopy) {614 if (stat_zcopies != num_sends)615 error(1, 0, "Unexpected number of Zerocopy completions: %9lu expected %9lu received",616 num_sends, stat_zcopies);617 fprintf(stderr,618 "Zerocopy acks: %19lu\n",619 stat_zcopies);620 }621}622 623static void print_report(unsigned long num_msgs, unsigned long num_sends)624{625 fprintf(stderr,626 "%s tx: %6lu MB/s %8lu calls/s %6lu msg/s\n",627 cfg_tcp ? "tcp" : "udp",628 (num_msgs * cfg_payload_len) >> 20,629 num_sends, num_msgs);630 631 if (cfg_audit) {632 total_num_msgs += num_msgs;633 total_num_sends += num_sends;634 }635}636 637int main(int argc, char **argv)638{639 unsigned long num_msgs, num_sends;640 unsigned long tnow, treport, tstop;641 int fd, i, val, ret;642 643 parse_opts(argc, argv);644 645 if (cfg_cpu > 0)646 set_cpu(cfg_cpu);647 648 for (i = 0; i < sizeof(buf[0]); i++)649 buf[0][i] = 'a' + (i % 26);650 for (i = 1; i < NUM_PKT; i++)651 memcpy(buf[i], buf[0], sizeof(buf[0]));652 653 signal(SIGINT, sigint_handler);654 655 fd = socket(cfg_family, cfg_tcp ? SOCK_STREAM : SOCK_DGRAM, 0);656 if (fd == -1)657 error(1, errno, "socket");658 659 if (cfg_zerocopy) {660 val = 1;661 662 ret = setsockopt(fd, SOL_SOCKET, SO_ZEROCOPY,663 &val, sizeof(val));664 if (ret) {665 if (errno == ENOPROTOOPT || errno == ENOTSUPP) {666 fprintf(stderr, "SO_ZEROCOPY not supported");667 exit(KSFT_SKIP);668 }669 error(1, errno, "setsockopt zerocopy");670 }671 }672 673 if (cfg_connected &&674 connect(fd, (void *)&cfg_dst_addr, cfg_alen))675 error(1, errno, "connect");676 677 if (cfg_segment)678 set_pmtu_discover(fd, cfg_family == PF_INET);679 680 if (cfg_tx_tstamp)681 set_tx_timestamping(fd);682 683 num_msgs = num_sends = 0;684 tnow = gettimeofday_ms();685 tstart = tnow;686 tend = tnow;687 tstop = tnow + cfg_runtime_ms;688 treport = tnow + 1000;689 690 i = 0;691 do {692 if (cfg_tcp)693 num_sends += send_tcp(fd, buf[i]);694 else if (cfg_segment)695 num_sends += send_udp_segment(fd, buf[i]);696 else if (cfg_sendmmsg)697 num_sends += send_udp_sendmmsg(fd, buf[i]);698 else699 num_sends += send_udp(fd, buf[i]);700 num_msgs++;701 if ((cfg_zerocopy && ((num_msgs & 0xF) == 0)) || cfg_tx_tstamp)702 flush_errqueue(fd, cfg_poll, 500, true);703 704 if (cfg_msg_nr && num_msgs >= cfg_msg_nr)705 break;706 707 tnow = gettimeofday_ms();708 if (tnow >= treport) {709 print_report(num_msgs, num_sends);710 num_msgs = num_sends = 0;711 treport = tnow + 1000;712 }713 714 /* cold cache when writing buffer */715 if (cfg_cache_trash)716 i = ++i < NUM_PKT ? i : 0;717 718 } while (!interrupted && (cfg_runtime_ms == -1 || tnow < tstop));719 720 if (cfg_zerocopy || cfg_tx_tstamp)721 flush_errqueue_retry(fd, num_sends);722 723 if (close(fd))724 error(1, errno, "close");725 726 if (cfg_audit) {727 tend = tnow;728 total_num_msgs += num_msgs;729 total_num_sends += num_sends;730 print_audit_report(total_num_msgs, total_num_sends);731 }732 733 return 0;734}735