2088 lines · c
1// SPDX-License-Identifier: GPL-2.02// Copyright (c) 2017-2018 Covalent IO, Inc. http://covalent.io3#include <stdio.h>4#include <stdlib.h>5#include <sys/socket.h>6#include <sys/ioctl.h>7#include <sys/select.h>8#include <netinet/in.h>9#include <arpa/inet.h>10#include <unistd.h>11#include <string.h>12#include <errno.h>13#include <stdbool.h>14#include <signal.h>15#include <fcntl.h>16#include <sys/wait.h>17#include <time.h>18#include <sched.h>19 20#include <sys/time.h>21#include <sys/types.h>22#include <sys/sendfile.h>23 24#include <linux/netlink.h>25#include <linux/socket.h>26#include <linux/sock_diag.h>27#include <linux/bpf.h>28#include <linux/if_link.h>29#include <linux/tls.h>30#include <assert.h>31#include <libgen.h>32 33#include <getopt.h>34 35#include <bpf/bpf.h>36#include <bpf/libbpf.h>37 38#include "bpf_util.h"39#include "cgroup_helpers.h"40 41int running;42static void running_handler(int a);43 44#ifndef TCP_ULP45# define TCP_ULP 3146#endif47#ifndef SOL_TLS48# define SOL_TLS 28249#endif50 51/* randomly selected ports for testing on lo */52#define S1_PORT 1000053#define S2_PORT 1000154 55#define BPF_SOCKMAP_FILENAME "test_sockmap_kern.bpf.o"56#define BPF_SOCKHASH_FILENAME "test_sockhash_kern.bpf.o"57#define CG_PATH "/sockmap"58 59/* global sockets */60int s1, s2, c1, c2, p1, p2;61int test_cnt;62int passed;63int failed;64int map_fd[9];65struct bpf_map *maps[9];66struct bpf_program *progs[9];67struct bpf_link *links[9];68 69int txmsg_pass;70int txmsg_redir;71int txmsg_drop;72int txmsg_apply;73int txmsg_cork;74int txmsg_start;75int txmsg_end;76int txmsg_start_push;77int txmsg_end_push;78int txmsg_start_pop;79int txmsg_pop;80int txmsg_ingress;81int txmsg_redir_skb;82int txmsg_ktls_skb;83int txmsg_ktls_skb_drop;84int txmsg_ktls_skb_redir;85int ktls;86int peek_flag;87int skb_use_parser;88int txmsg_omit_skb_parser;89 90static const struct option long_options[] = {91 {"help", no_argument, NULL, 'h' },92 {"cgroup", required_argument, NULL, 'c' },93 {"rate", required_argument, NULL, 'r' },94 {"verbose", optional_argument, NULL, 'v' },95 {"iov_count", required_argument, NULL, 'i' },96 {"length", required_argument, NULL, 'l' },97 {"test", required_argument, NULL, 't' },98 {"data_test", no_argument, NULL, 'd' },99 {"txmsg", no_argument, &txmsg_pass, 1 },100 {"txmsg_redir", no_argument, &txmsg_redir, 1 },101 {"txmsg_drop", no_argument, &txmsg_drop, 1 },102 {"txmsg_apply", required_argument, NULL, 'a'},103 {"txmsg_cork", required_argument, NULL, 'k'},104 {"txmsg_start", required_argument, NULL, 's'},105 {"txmsg_end", required_argument, NULL, 'e'},106 {"txmsg_start_push", required_argument, NULL, 'p'},107 {"txmsg_end_push", required_argument, NULL, 'q'},108 {"txmsg_start_pop", required_argument, NULL, 'w'},109 {"txmsg_pop", required_argument, NULL, 'x'},110 {"txmsg_ingress", no_argument, &txmsg_ingress, 1 },111 {"txmsg_redir_skb", no_argument, &txmsg_redir_skb, 1 },112 {"ktls", no_argument, &ktls, 1 },113 {"peek", no_argument, &peek_flag, 1 },114 {"txmsg_omit_skb_parser", no_argument, &txmsg_omit_skb_parser, 1},115 {"whitelist", required_argument, NULL, 'n' },116 {"blacklist", required_argument, NULL, 'b' },117 {0, 0, NULL, 0 }118};119 120struct test_env {121 const char *type;122 const char *subtest;123 const char *prepend;124 125 int test_num;126 int subtest_num;127 128 int succ_cnt;129 int fail_cnt;130 int fail_last;131};132 133struct test_env env;134 135struct sockmap_options {136 int verbose;137 bool base;138 bool sendpage;139 bool data_test;140 bool drop_expected;141 bool check_recved_len;142 bool tx_wait_mem;143 int iov_count;144 int iov_length;145 int rate;146 char *map;147 char *whitelist;148 char *blacklist;149 char *prepend;150};151 152struct _test {153 char *title;154 void (*tester)(int cg_fd, struct sockmap_options *opt);155};156 157static void test_start(void)158{159 env.subtest_num++;160}161 162static void test_fail(void)163{164 env.fail_cnt++;165}166 167static void test_pass(void)168{169 env.succ_cnt++;170}171 172static void test_reset(void)173{174 txmsg_start = txmsg_end = 0;175 txmsg_start_pop = txmsg_pop = 0;176 txmsg_start_push = txmsg_end_push = 0;177 txmsg_pass = txmsg_drop = txmsg_redir = 0;178 txmsg_apply = txmsg_cork = 0;179 txmsg_ingress = txmsg_redir_skb = 0;180 txmsg_ktls_skb = txmsg_ktls_skb_drop = txmsg_ktls_skb_redir = 0;181 txmsg_omit_skb_parser = 0;182 skb_use_parser = 0;183}184 185static int test_start_subtest(const struct _test *t, struct sockmap_options *o)186{187 env.type = o->map;188 env.subtest = t->title;189 env.prepend = o->prepend;190 env.test_num++;191 env.subtest_num = 0;192 env.fail_last = env.fail_cnt;193 test_reset();194 return 0;195}196 197static void test_end_subtest(void)198{199 int error = env.fail_cnt - env.fail_last;200 int type = strcmp(env.type, BPF_SOCKMAP_FILENAME);201 202 if (!error)203 test_pass();204 205 fprintf(stdout, "#%2d/%2d %8s:%s:%s:%s\n",206 env.test_num, env.subtest_num,207 !type ? "sockmap" : "sockhash",208 env.prepend ? : "",209 env.subtest, error ? "FAIL" : "OK");210}211 212static void test_print_results(void)213{214 fprintf(stdout, "Pass: %d Fail: %d\n",215 env.succ_cnt, env.fail_cnt);216}217 218static void usage(char *argv[])219{220 int i;221 222 printf(" Usage: %s --cgroup <cgroup_path>\n", argv[0]);223 printf(" options:\n");224 for (i = 0; long_options[i].name != 0; i++) {225 printf(" --%-12s", long_options[i].name);226 if (long_options[i].flag != NULL)227 printf(" flag (internal value:%d)\n",228 *long_options[i].flag);229 else230 printf(" -%c\n", long_options[i].val);231 }232 printf("\n");233}234 235char *sock_to_string(int s)236{237 if (s == c1)238 return "client1";239 else if (s == c2)240 return "client2";241 else if (s == s1)242 return "server1";243 else if (s == s2)244 return "server2";245 else if (s == p1)246 return "peer1";247 else if (s == p2)248 return "peer2";249 else250 return "unknown";251}252 253static int sockmap_init_ktls(int verbose, int s)254{255 struct tls12_crypto_info_aes_gcm_128 tls_tx = {256 .info = {257 .version = TLS_1_2_VERSION,258 .cipher_type = TLS_CIPHER_AES_GCM_128,259 },260 };261 struct tls12_crypto_info_aes_gcm_128 tls_rx = {262 .info = {263 .version = TLS_1_2_VERSION,264 .cipher_type = TLS_CIPHER_AES_GCM_128,265 },266 };267 int so_buf = 6553500;268 int err;269 270 err = setsockopt(s, 6, TCP_ULP, "tls", sizeof("tls"));271 if (err) {272 fprintf(stderr, "setsockopt: TCP_ULP(%s) failed with error %i\n", sock_to_string(s), err);273 return -EINVAL;274 }275 err = setsockopt(s, SOL_TLS, TLS_TX, (void *)&tls_tx, sizeof(tls_tx));276 if (err) {277 fprintf(stderr, "setsockopt: TLS_TX(%s) failed with error %i\n", sock_to_string(s), err);278 return -EINVAL;279 }280 err = setsockopt(s, SOL_TLS, TLS_RX, (void *)&tls_rx, sizeof(tls_rx));281 if (err) {282 fprintf(stderr, "setsockopt: TLS_RX(%s) failed with error %i\n", sock_to_string(s), err);283 return -EINVAL;284 }285 err = setsockopt(s, SOL_SOCKET, SO_SNDBUF, &so_buf, sizeof(so_buf));286 if (err) {287 fprintf(stderr, "setsockopt: (%s) failed sndbuf with error %i\n", sock_to_string(s), err);288 return -EINVAL;289 }290 err = setsockopt(s, SOL_SOCKET, SO_RCVBUF, &so_buf, sizeof(so_buf));291 if (err) {292 fprintf(stderr, "setsockopt: (%s) failed rcvbuf with error %i\n", sock_to_string(s), err);293 return -EINVAL;294 }295 296 if (verbose)297 fprintf(stdout, "socket(%s) kTLS enabled\n", sock_to_string(s));298 return 0;299}300static int sockmap_init_sockets(int verbose)301{302 int i, err, one = 1;303 struct sockaddr_in addr;304 int *fds[4] = {&s1, &s2, &c1, &c2};305 306 s1 = s2 = p1 = p2 = c1 = c2 = 0;307 308 /* Init sockets */309 for (i = 0; i < 4; i++) {310 *fds[i] = socket(AF_INET, SOCK_STREAM, 0);311 if (*fds[i] < 0) {312 perror("socket s1 failed()");313 return errno;314 }315 }316 317 /* Allow reuse */318 for (i = 0; i < 2; i++) {319 err = setsockopt(*fds[i], SOL_SOCKET, SO_REUSEADDR,320 (char *)&one, sizeof(one));321 if (err) {322 perror("setsockopt failed()");323 return errno;324 }325 }326 327 /* Non-blocking sockets */328 for (i = 0; i < 2; i++) {329 err = ioctl(*fds[i], FIONBIO, (char *)&one);330 if (err < 0) {331 perror("ioctl s1 failed()");332 return errno;333 }334 }335 336 /* Bind server sockets */337 memset(&addr, 0, sizeof(struct sockaddr_in));338 addr.sin_family = AF_INET;339 addr.sin_addr.s_addr = inet_addr("127.0.0.1");340 341 addr.sin_port = htons(S1_PORT);342 err = bind(s1, (struct sockaddr *)&addr, sizeof(addr));343 if (err < 0) {344 perror("bind s1 failed()");345 return errno;346 }347 348 addr.sin_port = htons(S2_PORT);349 err = bind(s2, (struct sockaddr *)&addr, sizeof(addr));350 if (err < 0) {351 perror("bind s2 failed()");352 return errno;353 }354 355 /* Listen server sockets */356 addr.sin_port = htons(S1_PORT);357 err = listen(s1, 32);358 if (err < 0) {359 perror("listen s1 failed()");360 return errno;361 }362 363 addr.sin_port = htons(S2_PORT);364 err = listen(s2, 32);365 if (err < 0) {366 perror("listen s1 failed()");367 return errno;368 }369 370 /* Initiate Connect */371 addr.sin_port = htons(S1_PORT);372 err = connect(c1, (struct sockaddr *)&addr, sizeof(addr));373 if (err < 0 && errno != EINPROGRESS) {374 perror("connect c1 failed()");375 return errno;376 }377 378 addr.sin_port = htons(S2_PORT);379 err = connect(c2, (struct sockaddr *)&addr, sizeof(addr));380 if (err < 0 && errno != EINPROGRESS) {381 perror("connect c2 failed()");382 return errno;383 } else if (err < 0) {384 err = 0;385 }386 387 /* Accept Connecrtions */388 p1 = accept(s1, NULL, NULL);389 if (p1 < 0) {390 perror("accept s1 failed()");391 return errno;392 }393 394 p2 = accept(s2, NULL, NULL);395 if (p2 < 0) {396 perror("accept s1 failed()");397 return errno;398 }399 400 if (verbose > 1) {401 printf("connected sockets: c1 <-> p1, c2 <-> p2\n");402 printf("cgroups binding: c1(%i) <-> s1(%i) - - - c2(%i) <-> s2(%i)\n",403 c1, s1, c2, s2);404 }405 return 0;406}407 408struct msg_stats {409 size_t bytes_sent;410 size_t bytes_recvd;411 struct timespec start;412 struct timespec end;413};414 415static int msg_loop_sendpage(int fd, int iov_length, int cnt,416 struct msg_stats *s,417 struct sockmap_options *opt)418{419 bool drop = opt->drop_expected;420 unsigned char k = 0;421 FILE *file;422 int i, fp;423 424 file = tmpfile();425 if (!file) {426 perror("create file for sendpage");427 return 1;428 }429 for (i = 0; i < iov_length * cnt; i++, k++)430 fwrite(&k, sizeof(char), 1, file);431 fflush(file);432 fseek(file, 0, SEEK_SET);433 434 fp = fileno(file);435 436 clock_gettime(CLOCK_MONOTONIC, &s->start);437 for (i = 0; i < cnt; i++) {438 int sent;439 440 errno = 0;441 sent = sendfile(fd, fp, NULL, iov_length);442 443 if (!drop && sent < 0) {444 perror("sendpage loop error");445 fclose(file);446 return sent;447 } else if (drop && sent >= 0) {448 printf("sendpage loop error expected: %i errno %i\n",449 sent, errno);450 fclose(file);451 return -EIO;452 }453 454 if (sent > 0)455 s->bytes_sent += sent;456 }457 clock_gettime(CLOCK_MONOTONIC, &s->end);458 fclose(file);459 return 0;460}461 462static void msg_free_iov(struct msghdr *msg)463{464 int i;465 466 for (i = 0; i < msg->msg_iovlen; i++)467 free(msg->msg_iov[i].iov_base);468 free(msg->msg_iov);469 msg->msg_iov = NULL;470 msg->msg_iovlen = 0;471}472 473static int msg_alloc_iov(struct msghdr *msg,474 int iov_count, int iov_length,475 bool data, bool xmit)476{477 unsigned char k = 0;478 struct iovec *iov;479 int i;480 481 iov = calloc(iov_count, sizeof(struct iovec));482 if (!iov)483 return errno;484 485 for (i = 0; i < iov_count; i++) {486 unsigned char *d = calloc(iov_length, sizeof(char));487 488 if (!d) {489 fprintf(stderr, "iov_count %i/%i OOM\n", i, iov_count);490 goto unwind_iov;491 }492 iov[i].iov_base = d;493 iov[i].iov_len = iov_length;494 495 if (data && xmit) {496 int j;497 498 for (j = 0; j < iov_length; j++)499 d[j] = k++;500 }501 }502 503 msg->msg_iov = iov;504 msg->msg_iovlen = iov_count;505 506 return 0;507unwind_iov:508 for (i--; i >= 0 ; i--)509 free(msg->msg_iov[i].iov_base);510 return -ENOMEM;511}512 513static int msg_verify_data(struct msghdr *msg, int size, int chunk_sz)514{515 int i, j = 0, bytes_cnt = 0;516 unsigned char k = 0;517 518 for (i = 0; i < msg->msg_iovlen; i++) {519 unsigned char *d = msg->msg_iov[i].iov_base;520 521 /* Special case test for skb ingress + ktls */522 if (i == 0 && txmsg_ktls_skb) {523 if (msg->msg_iov[i].iov_len < 4)524 return -EIO;525 if (memcmp(d, "PASS", 4) != 0) {526 fprintf(stderr,527 "detected skb data error with skb ingress update @iov[%i]:%i \"%02x %02x %02x %02x\" != \"PASS\"\n",528 i, 0, d[0], d[1], d[2], d[3]);529 return -EIO;530 }531 j = 4; /* advance index past PASS header */532 }533 534 for (; j < msg->msg_iov[i].iov_len && size; j++) {535 if (d[j] != k++) {536 fprintf(stderr,537 "detected data corruption @iov[%i]:%i %02x != %02x, %02x ?= %02x\n",538 i, j, d[j], k - 1, d[j+1], k);539 return -EIO;540 }541 bytes_cnt++;542 if (bytes_cnt == chunk_sz) {543 k = 0;544 bytes_cnt = 0;545 }546 size--;547 }548 }549 return 0;550}551 552static int msg_loop(int fd, int iov_count, int iov_length, int cnt,553 struct msg_stats *s, bool tx,554 struct sockmap_options *opt)555{556 struct msghdr msg = {0}, msg_peek = {0};557 int err, i, flags = MSG_NOSIGNAL;558 bool drop = opt->drop_expected;559 bool data = opt->data_test;560 int iov_alloc_length = iov_length;561 562 if (!tx && opt->check_recved_len)563 iov_alloc_length *= 2;564 565 err = msg_alloc_iov(&msg, iov_count, iov_alloc_length, data, tx);566 if (err)567 goto out_errno;568 if (peek_flag) {569 err = msg_alloc_iov(&msg_peek, iov_count, iov_length, data, tx);570 if (err)571 goto out_errno;572 }573 574 if (tx) {575 clock_gettime(CLOCK_MONOTONIC, &s->start);576 for (i = 0; i < cnt; i++) {577 int sent;578 579 errno = 0;580 sent = sendmsg(fd, &msg, flags);581 582 if (!drop && sent < 0) {583 if (opt->tx_wait_mem && errno == EACCES) {584 errno = 0;585 goto out_errno;586 }587 perror("sendmsg loop error");588 goto out_errno;589 } else if (drop && sent >= 0) {590 fprintf(stderr,591 "sendmsg loop error expected: %i errno %i\n",592 sent, errno);593 errno = -EIO;594 goto out_errno;595 }596 if (sent > 0)597 s->bytes_sent += sent;598 }599 clock_gettime(CLOCK_MONOTONIC, &s->end);600 } else {601 int slct, recvp = 0, recv, max_fd = fd;602 float total_bytes, txmsg_pop_total;603 int fd_flags = O_NONBLOCK;604 struct timeval timeout;605 fd_set w;606 607 fcntl(fd, fd_flags);608 /* Account for pop bytes noting each iteration of apply will609 * call msg_pop_data helper so we need to account for this610 * by calculating the number of apply iterations. Note user611 * of the tool can create cases where no data is sent by612 * manipulating pop/push/pull/etc. For example txmsg_apply 1613 * with txmsg_pop 1 will try to apply 1B at a time but each614 * iteration will then pop 1B so no data will ever be sent.615 * This is really only useful for testing edge cases in code616 * paths.617 */618 total_bytes = (float)iov_count * (float)iov_length * (float)cnt;619 if (txmsg_apply)620 txmsg_pop_total = txmsg_pop * (total_bytes / txmsg_apply);621 else622 txmsg_pop_total = txmsg_pop * cnt;623 total_bytes -= txmsg_pop_total;624 err = clock_gettime(CLOCK_MONOTONIC, &s->start);625 if (err < 0)626 perror("recv start time");627 while (s->bytes_recvd < total_bytes) {628 if (txmsg_cork) {629 timeout.tv_sec = 0;630 timeout.tv_usec = 300000;631 } else {632 timeout.tv_sec = 3;633 timeout.tv_usec = 0;634 }635 636 /* FD sets */637 FD_ZERO(&w);638 FD_SET(fd, &w);639 640 slct = select(max_fd + 1, &w, NULL, NULL, &timeout);641 if (slct == -1) {642 perror("select()");643 clock_gettime(CLOCK_MONOTONIC, &s->end);644 goto out_errno;645 } else if (!slct) {646 if (opt->verbose)647 fprintf(stderr, "unexpected timeout: recved %zu/%f pop_total %f\n", s->bytes_recvd, total_bytes, txmsg_pop_total);648 errno = -EIO;649 clock_gettime(CLOCK_MONOTONIC, &s->end);650 goto out_errno;651 }652 653 if (opt->tx_wait_mem) {654 FD_ZERO(&w);655 FD_SET(fd, &w);656 slct = select(max_fd + 1, NULL, NULL, &w, &timeout);657 errno = 0;658 close(fd);659 goto out_errno;660 }661 662 errno = 0;663 if (peek_flag) {664 flags |= MSG_PEEK;665 recvp = recvmsg(fd, &msg_peek, flags);666 if (recvp < 0) {667 if (errno != EWOULDBLOCK) {668 clock_gettime(CLOCK_MONOTONIC, &s->end);669 goto out_errno;670 }671 }672 flags = 0;673 }674 675 recv = recvmsg(fd, &msg, flags);676 if (recv < 0) {677 if (errno != EWOULDBLOCK) {678 clock_gettime(CLOCK_MONOTONIC, &s->end);679 perror("recv failed()");680 goto out_errno;681 }682 }683 684 if (recv > 0)685 s->bytes_recvd += recv;686 687 if (opt->check_recved_len && s->bytes_recvd > total_bytes) {688 errno = EMSGSIZE;689 fprintf(stderr, "recv failed(), bytes_recvd:%zd, total_bytes:%f\n",690 s->bytes_recvd, total_bytes);691 goto out_errno;692 }693 694 if (data) {695 int chunk_sz = opt->sendpage ?696 iov_length * cnt :697 iov_length * iov_count;698 699 errno = msg_verify_data(&msg, recv, chunk_sz);700 if (errno) {701 perror("data verify msg failed");702 goto out_errno;703 }704 if (recvp) {705 errno = msg_verify_data(&msg_peek,706 recvp,707 chunk_sz);708 if (errno) {709 perror("data verify msg_peek failed");710 goto out_errno;711 }712 }713 }714 }715 clock_gettime(CLOCK_MONOTONIC, &s->end);716 }717 718 msg_free_iov(&msg);719 msg_free_iov(&msg_peek);720 return err;721out_errno:722 msg_free_iov(&msg);723 msg_free_iov(&msg_peek);724 return errno;725}726 727static float giga = 1000000000;728 729static inline float sentBps(struct msg_stats s)730{731 return s.bytes_sent / (s.end.tv_sec - s.start.tv_sec);732}733 734static inline float recvdBps(struct msg_stats s)735{736 return s.bytes_recvd / (s.end.tv_sec - s.start.tv_sec);737}738 739static int sendmsg_test(struct sockmap_options *opt)740{741 float sent_Bps = 0, recvd_Bps = 0;742 int rx_fd, txpid, rxpid, err = 0;743 struct msg_stats s = {0};744 int iov_count = opt->iov_count;745 int iov_buf = opt->iov_length;746 int rx_status, tx_status;747 int cnt = opt->rate;748 749 errno = 0;750 751 if (opt->base)752 rx_fd = p1;753 else754 rx_fd = p2;755 756 if (ktls) {757 /* Redirecting into non-TLS socket which sends into a TLS758 * socket is not a valid test. So in this case lets not759 * enable kTLS but still run the test.760 */761 if (!txmsg_redir || txmsg_ingress) {762 err = sockmap_init_ktls(opt->verbose, rx_fd);763 if (err)764 return err;765 }766 err = sockmap_init_ktls(opt->verbose, c1);767 if (err)768 return err;769 }770 771 if (opt->tx_wait_mem) {772 struct timeval timeout;773 int rxtx_buf_len = 1024;774 775 timeout.tv_sec = 3;776 timeout.tv_usec = 0;777 778 err = setsockopt(c2, SOL_SOCKET, SO_SNDTIMEO, &timeout, sizeof(struct timeval));779 err |= setsockopt(c2, SOL_SOCKET, SO_SNDBUFFORCE, &rxtx_buf_len, sizeof(int));780 err |= setsockopt(p2, SOL_SOCKET, SO_RCVBUFFORCE, &rxtx_buf_len, sizeof(int));781 if (err) {782 perror("setsockopt failed()");783 return errno;784 }785 }786 787 rxpid = fork();788 if (rxpid == 0) {789 if (txmsg_pop || txmsg_start_pop)790 iov_buf -= (txmsg_pop - txmsg_start_pop + 1);791 if (opt->drop_expected || txmsg_ktls_skb_drop)792 _exit(0);793 794 if (!iov_buf) /* zero bytes sent case */795 _exit(0);796 797 if (opt->sendpage)798 iov_count = 1;799 err = msg_loop(rx_fd, iov_count, iov_buf,800 cnt, &s, false, opt);801 if (opt->verbose > 1)802 fprintf(stderr,803 "msg_loop_rx: iov_count %i iov_buf %i cnt %i err %i\n",804 iov_count, iov_buf, cnt, err);805 if (s.end.tv_sec - s.start.tv_sec) {806 sent_Bps = sentBps(s);807 recvd_Bps = recvdBps(s);808 }809 if (opt->verbose > 1)810 fprintf(stdout,811 "rx_sendmsg: TX: %zuB %fB/s %fGB/s RX: %zuB %fB/s %fGB/s %s\n",812 s.bytes_sent, sent_Bps, sent_Bps/giga,813 s.bytes_recvd, recvd_Bps, recvd_Bps/giga,814 peek_flag ? "(peek_msg)" : "");815 if (err && txmsg_cork)816 err = 0;817 exit(err ? 1 : 0);818 } else if (rxpid == -1) {819 perror("msg_loop_rx");820 return errno;821 }822 823 if (opt->tx_wait_mem)824 close(c2);825 826 txpid = fork();827 if (txpid == 0) {828 if (opt->sendpage)829 err = msg_loop_sendpage(c1, iov_buf, cnt, &s, opt);830 else831 err = msg_loop(c1, iov_count, iov_buf,832 cnt, &s, true, opt);833 834 if (err)835 fprintf(stderr,836 "msg_loop_tx: iov_count %i iov_buf %i cnt %i err %i\n",837 iov_count, iov_buf, cnt, err);838 if (s.end.tv_sec - s.start.tv_sec) {839 sent_Bps = sentBps(s);840 recvd_Bps = recvdBps(s);841 }842 if (opt->verbose > 1)843 fprintf(stdout,844 "tx_sendmsg: TX: %zuB %fB/s %f GB/s RX: %zuB %fB/s %fGB/s\n",845 s.bytes_sent, sent_Bps, sent_Bps/giga,846 s.bytes_recvd, recvd_Bps, recvd_Bps/giga);847 exit(err ? 1 : 0);848 } else if (txpid == -1) {849 perror("msg_loop_tx");850 return errno;851 }852 853 assert(waitpid(rxpid, &rx_status, 0) == rxpid);854 assert(waitpid(txpid, &tx_status, 0) == txpid);855 if (WIFEXITED(rx_status)) {856 err = WEXITSTATUS(rx_status);857 if (err) {858 fprintf(stderr, "rx thread exited with err %d.\n", err);859 goto out;860 }861 }862 if (WIFEXITED(tx_status)) {863 err = WEXITSTATUS(tx_status);864 if (err)865 fprintf(stderr, "tx thread exited with err %d.\n", err);866 }867out:868 return err;869}870 871static int forever_ping_pong(int rate, struct sockmap_options *opt)872{873 struct timeval timeout;874 char buf[1024] = {0};875 int sc;876 877 timeout.tv_sec = 10;878 timeout.tv_usec = 0;879 880 /* Ping/Pong data from client to server */881 sc = send(c1, buf, sizeof(buf), 0);882 if (sc < 0) {883 perror("send failed()");884 return sc;885 }886 887 do {888 int s, rc, i, max_fd = p2;889 fd_set w;890 891 /* FD sets */892 FD_ZERO(&w);893 FD_SET(c1, &w);894 FD_SET(c2, &w);895 FD_SET(p1, &w);896 FD_SET(p2, &w);897 898 s = select(max_fd + 1, &w, NULL, NULL, &timeout);899 if (s == -1) {900 perror("select()");901 break;902 } else if (!s) {903 fprintf(stderr, "unexpected timeout\n");904 break;905 }906 907 for (i = 0; i <= max_fd && s > 0; ++i) {908 if (!FD_ISSET(i, &w))909 continue;910 911 s--;912 913 rc = recv(i, buf, sizeof(buf), 0);914 if (rc < 0) {915 if (errno != EWOULDBLOCK) {916 perror("recv failed()");917 return rc;918 }919 }920 921 if (rc == 0) {922 close(i);923 break;924 }925 926 sc = send(i, buf, rc, 0);927 if (sc < 0) {928 perror("send failed()");929 return sc;930 }931 }932 933 if (rate)934 sleep(rate);935 936 if (opt->verbose) {937 printf(".");938 fflush(stdout);939 940 }941 } while (running);942 943 return 0;944}945 946enum {947 SELFTESTS,948 PING_PONG,949 SENDMSG,950 BASE,951 BASE_SENDPAGE,952 SENDPAGE,953};954 955static int run_options(struct sockmap_options *options, int cg_fd, int test)956{957 int i, key, next_key, err, zero = 0;958 struct bpf_program *tx_prog;959 960 /* If base test skip BPF setup */961 if (test == BASE || test == BASE_SENDPAGE)962 goto run;963 964 /* Attach programs to sockmap */965 if (!txmsg_omit_skb_parser) {966 links[0] = bpf_program__attach_sockmap(progs[0], map_fd[0]);967 if (!links[0]) {968 fprintf(stderr,969 "ERROR: bpf_program__attach_sockmap (sockmap %i->%i): (%s)\n",970 bpf_program__fd(progs[0]), map_fd[0], strerror(errno));971 return -1;972 }973 }974 975 links[1] = bpf_program__attach_sockmap(progs[1], map_fd[0]);976 if (!links[1]) {977 fprintf(stderr, "ERROR: bpf_program__attach_sockmap (sockmap): (%s)\n",978 strerror(errno));979 return -1;980 }981 982 /* Attach programs to TLS sockmap */983 if (txmsg_ktls_skb) {984 if (!txmsg_omit_skb_parser) {985 links[2] = bpf_program__attach_sockmap(progs[0], map_fd[8]);986 if (!links[2]) {987 fprintf(stderr,988 "ERROR: bpf_program__attach_sockmap (TLS sockmap %i->%i): (%s)\n",989 bpf_program__fd(progs[0]), map_fd[8], strerror(errno));990 return -1;991 }992 }993 994 links[3] = bpf_program__attach_sockmap(progs[2], map_fd[8]);995 if (!links[3]) {996 fprintf(stderr, "ERROR: bpf_program__attach_sockmap (TLS sockmap): (%s)\n",997 strerror(errno));998 return -1;999 }1000 }1001 1002 /* Attach to cgroups */1003 err = bpf_prog_attach(bpf_program__fd(progs[3]), cg_fd, BPF_CGROUP_SOCK_OPS, 0);1004 if (err) {1005 fprintf(stderr, "ERROR: bpf_prog_attach (groups): %d (%s)\n",1006 err, strerror(errno));1007 return err;1008 }1009 1010run:1011 err = sockmap_init_sockets(options->verbose);1012 if (err) {1013 fprintf(stderr, "ERROR: test socket failed: %d\n", err);1014 goto out;1015 }1016 1017 /* Attach txmsg program to sockmap */1018 if (txmsg_pass)1019 tx_prog = progs[4];1020 else if (txmsg_redir)1021 tx_prog = progs[5];1022 else if (txmsg_apply)1023 tx_prog = progs[6];1024 else if (txmsg_cork)1025 tx_prog = progs[7];1026 else if (txmsg_drop)1027 tx_prog = progs[8];1028 else1029 tx_prog = NULL;1030 1031 if (tx_prog) {1032 int redir_fd;1033 1034 links[4] = bpf_program__attach_sockmap(tx_prog, map_fd[1]);1035 if (!links[4]) {1036 fprintf(stderr,1037 "ERROR: bpf_program__attach_sockmap (txmsg): (%s)\n",1038 strerror(errno));1039 err = -1;1040 goto out;1041 }1042 1043 i = 0;1044 err = bpf_map_update_elem(map_fd[1], &i, &c1, BPF_ANY);1045 if (err) {1046 fprintf(stderr,1047 "ERROR: bpf_map_update_elem (txmsg): %d (%s\n",1048 err, strerror(errno));1049 goto out;1050 }1051 1052 if (txmsg_redir)1053 redir_fd = c2;1054 else1055 redir_fd = c1;1056 1057 err = bpf_map_update_elem(map_fd[2], &i, &redir_fd, BPF_ANY);1058 if (err) {1059 fprintf(stderr,1060 "ERROR: bpf_map_update_elem (txmsg): %d (%s\n",1061 err, strerror(errno));1062 goto out;1063 }1064 1065 if (txmsg_apply) {1066 err = bpf_map_update_elem(map_fd[3],1067 &i, &txmsg_apply, BPF_ANY);1068 if (err) {1069 fprintf(stderr,1070 "ERROR: bpf_map_update_elem (apply_bytes): %d (%s\n",1071 err, strerror(errno));1072 goto out;1073 }1074 }1075 1076 if (txmsg_cork) {1077 err = bpf_map_update_elem(map_fd[4],1078 &i, &txmsg_cork, BPF_ANY);1079 if (err) {1080 fprintf(stderr,1081 "ERROR: bpf_map_update_elem (cork_bytes): %d (%s\n",1082 err, strerror(errno));1083 goto out;1084 }1085 }1086 1087 if (txmsg_start) {1088 err = bpf_map_update_elem(map_fd[5],1089 &i, &txmsg_start, BPF_ANY);1090 if (err) {1091 fprintf(stderr,1092 "ERROR: bpf_map_update_elem (txmsg_start): %d (%s)\n",1093 err, strerror(errno));1094 goto out;1095 }1096 }1097 1098 if (txmsg_end) {1099 i = 1;1100 err = bpf_map_update_elem(map_fd[5],1101 &i, &txmsg_end, BPF_ANY);1102 if (err) {1103 fprintf(stderr,1104 "ERROR: bpf_map_update_elem (txmsg_end): %d (%s)\n",1105 err, strerror(errno));1106 goto out;1107 }1108 }1109 1110 if (txmsg_start_push) {1111 i = 2;1112 err = bpf_map_update_elem(map_fd[5],1113 &i, &txmsg_start_push, BPF_ANY);1114 if (err) {1115 fprintf(stderr,1116 "ERROR: bpf_map_update_elem (txmsg_start_push): %d (%s)\n",1117 err, strerror(errno));1118 goto out;1119 }1120 }1121 1122 if (txmsg_end_push) {1123 i = 3;1124 err = bpf_map_update_elem(map_fd[5],1125 &i, &txmsg_end_push, BPF_ANY);1126 if (err) {1127 fprintf(stderr,1128 "ERROR: bpf_map_update_elem %i@%i (txmsg_end_push): %d (%s)\n",1129 txmsg_end_push, i, err, strerror(errno));1130 goto out;1131 }1132 }1133 1134 if (txmsg_start_pop) {1135 i = 4;1136 err = bpf_map_update_elem(map_fd[5],1137 &i, &txmsg_start_pop, BPF_ANY);1138 if (err) {1139 fprintf(stderr,1140 "ERROR: bpf_map_update_elem %i@%i (txmsg_start_pop): %d (%s)\n",1141 txmsg_start_pop, i, err, strerror(errno));1142 goto out;1143 }1144 } else {1145 i = 4;1146 bpf_map_update_elem(map_fd[5],1147 &i, &txmsg_start_pop, BPF_ANY);1148 }1149 1150 if (txmsg_pop) {1151 i = 5;1152 err = bpf_map_update_elem(map_fd[5],1153 &i, &txmsg_pop, BPF_ANY);1154 if (err) {1155 fprintf(stderr,1156 "ERROR: bpf_map_update_elem %i@%i (txmsg_pop): %d (%s)\n",1157 txmsg_pop, i, err, strerror(errno));1158 goto out;1159 }1160 } else {1161 i = 5;1162 bpf_map_update_elem(map_fd[5],1163 &i, &txmsg_pop, BPF_ANY);1164 1165 }1166 1167 if (txmsg_ingress) {1168 int in = BPF_F_INGRESS;1169 1170 i = 0;1171 err = bpf_map_update_elem(map_fd[6], &i, &in, BPF_ANY);1172 if (err) {1173 fprintf(stderr,1174 "ERROR: bpf_map_update_elem (txmsg_ingress): %d (%s)\n",1175 err, strerror(errno));1176 }1177 i = 1;1178 err = bpf_map_update_elem(map_fd[1], &i, &p1, BPF_ANY);1179 if (err) {1180 fprintf(stderr,1181 "ERROR: bpf_map_update_elem (p1 txmsg): %d (%s)\n",1182 err, strerror(errno));1183 }1184 err = bpf_map_update_elem(map_fd[2], &i, &p1, BPF_ANY);1185 if (err) {1186 fprintf(stderr,1187 "ERROR: bpf_map_update_elem (p1 redir): %d (%s)\n",1188 err, strerror(errno));1189 }1190 1191 i = 2;1192 err = bpf_map_update_elem(map_fd[2], &i, &p2, BPF_ANY);1193 if (err) {1194 fprintf(stderr,1195 "ERROR: bpf_map_update_elem (p2 txmsg): %d (%s)\n",1196 err, strerror(errno));1197 }1198 }1199 1200 if (txmsg_ktls_skb) {1201 int ingress = BPF_F_INGRESS;1202 1203 i = 0;1204 err = bpf_map_update_elem(map_fd[8], &i, &p2, BPF_ANY);1205 if (err) {1206 fprintf(stderr,1207 "ERROR: bpf_map_update_elem (c1 sockmap): %d (%s)\n",1208 err, strerror(errno));1209 }1210 1211 if (txmsg_ktls_skb_redir) {1212 i = 1;1213 err = bpf_map_update_elem(map_fd[7],1214 &i, &ingress, BPF_ANY);1215 if (err) {1216 fprintf(stderr,1217 "ERROR: bpf_map_update_elem (txmsg_ingress): %d (%s)\n",1218 err, strerror(errno));1219 }1220 }1221 1222 if (txmsg_ktls_skb_drop) {1223 i = 1;1224 err = bpf_map_update_elem(map_fd[7], &i, &i, BPF_ANY);1225 }1226 }1227 1228 if (txmsg_redir_skb) {1229 int skb_fd = (test == SENDMSG || test == SENDPAGE) ?1230 p2 : p1;1231 int ingress = BPF_F_INGRESS;1232 1233 i = 0;1234 err = bpf_map_update_elem(map_fd[7],1235 &i, &ingress, BPF_ANY);1236 if (err) {1237 fprintf(stderr,1238 "ERROR: bpf_map_update_elem (txmsg_ingress): %d (%s)\n",1239 err, strerror(errno));1240 }1241 1242 i = 3;1243 err = bpf_map_update_elem(map_fd[0], &i, &skb_fd, BPF_ANY);1244 if (err) {1245 fprintf(stderr,1246 "ERROR: bpf_map_update_elem (c1 sockmap): %d (%s)\n",1247 err, strerror(errno));1248 }1249 }1250 }1251 1252 if (skb_use_parser) {1253 i = 2;1254 err = bpf_map_update_elem(map_fd[7], &i, &skb_use_parser, BPF_ANY);1255 }1256 1257 if (txmsg_drop)1258 options->drop_expected = true;1259 1260 if (test == PING_PONG)1261 err = forever_ping_pong(options->rate, options);1262 else if (test == SENDMSG) {1263 options->base = false;1264 options->sendpage = false;1265 err = sendmsg_test(options);1266 } else if (test == SENDPAGE) {1267 options->base = false;1268 options->sendpage = true;1269 err = sendmsg_test(options);1270 } else if (test == BASE) {1271 options->base = true;1272 options->sendpage = false;1273 err = sendmsg_test(options);1274 } else if (test == BASE_SENDPAGE) {1275 options->base = true;1276 options->sendpage = true;1277 err = sendmsg_test(options);1278 } else1279 fprintf(stderr, "unknown test\n");1280out:1281 /* Detatch and zero all the maps */1282 bpf_prog_detach2(bpf_program__fd(progs[3]), cg_fd, BPF_CGROUP_SOCK_OPS);1283 1284 for (i = 0; i < ARRAY_SIZE(links); i++) {1285 if (links[i])1286 bpf_link__detach(links[i]);1287 }1288 1289 for (i = 0; i < ARRAY_SIZE(map_fd); i++) {1290 key = next_key = 0;1291 bpf_map_update_elem(map_fd[i], &key, &zero, BPF_ANY);1292 while (bpf_map_get_next_key(map_fd[i], &key, &next_key) == 0) {1293 bpf_map_update_elem(map_fd[i], &key, &zero, BPF_ANY);1294 key = next_key;1295 }1296 }1297 1298 close(s1);1299 close(s2);1300 close(p1);1301 close(p2);1302 close(c1);1303 close(c2);1304 return err;1305}1306 1307static char *test_to_str(int test)1308{1309 switch (test) {1310 case SENDMSG:1311 return "sendmsg";1312 case SENDPAGE:1313 return "sendpage";1314 }1315 return "unknown";1316}1317 1318static void append_str(char *dst, const char *src, size_t dst_cap)1319{1320 size_t avail = dst_cap - strlen(dst);1321 1322 if (avail <= 1) /* just zero byte could be written */1323 return;1324 1325 strncat(dst, src, avail - 1); /* strncat() adds + 1 for zero byte */1326}1327 1328#define OPTSTRING 601329static void test_options(char *options)1330{1331 char tstr[OPTSTRING];1332 1333 memset(options, 0, OPTSTRING);1334 1335 if (txmsg_pass)1336 append_str(options, "pass,", OPTSTRING);1337 if (txmsg_redir)1338 append_str(options, "redir,", OPTSTRING);1339 if (txmsg_drop)1340 append_str(options, "drop,", OPTSTRING);1341 if (txmsg_apply) {1342 snprintf(tstr, OPTSTRING, "apply %d,", txmsg_apply);1343 append_str(options, tstr, OPTSTRING);1344 }1345 if (txmsg_cork) {1346 snprintf(tstr, OPTSTRING, "cork %d,", txmsg_cork);1347 append_str(options, tstr, OPTSTRING);1348 }1349 if (txmsg_start) {1350 snprintf(tstr, OPTSTRING, "start %d,", txmsg_start);1351 append_str(options, tstr, OPTSTRING);1352 }1353 if (txmsg_end) {1354 snprintf(tstr, OPTSTRING, "end %d,", txmsg_end);1355 append_str(options, tstr, OPTSTRING);1356 }1357 if (txmsg_start_pop) {1358 snprintf(tstr, OPTSTRING, "pop (%d,%d),",1359 txmsg_start_pop, txmsg_start_pop + txmsg_pop);1360 append_str(options, tstr, OPTSTRING);1361 }1362 if (txmsg_ingress)1363 append_str(options, "ingress,", OPTSTRING);1364 if (txmsg_redir_skb)1365 append_str(options, "redir_skb,", OPTSTRING);1366 if (txmsg_ktls_skb)1367 append_str(options, "ktls_skb,", OPTSTRING);1368 if (ktls)1369 append_str(options, "ktls,", OPTSTRING);1370 if (peek_flag)1371 append_str(options, "peek,", OPTSTRING);1372}1373 1374static int __test_exec(int cgrp, int test, struct sockmap_options *opt)1375{1376 char *options = calloc(OPTSTRING, sizeof(char));1377 int err;1378 1379 if (test == SENDPAGE)1380 opt->sendpage = true;1381 else1382 opt->sendpage = false;1383 1384 if (txmsg_drop)1385 opt->drop_expected = true;1386 else1387 opt->drop_expected = false;1388 1389 test_options(options);1390 1391 if (opt->verbose) {1392 fprintf(stdout,1393 " [TEST %i]: (%i, %i, %i, %s, %s): ",1394 test_cnt, opt->rate, opt->iov_count, opt->iov_length,1395 test_to_str(test), options);1396 fflush(stdout);1397 }1398 err = run_options(opt, cgrp, test);1399 if (opt->verbose)1400 fprintf(stdout, " %s\n", !err ? "PASS" : "FAILED");1401 test_cnt++;1402 !err ? passed++ : failed++;1403 free(options);1404 return err;1405}1406 1407static void test_exec(int cgrp, struct sockmap_options *opt)1408{1409 int type = strcmp(opt->map, BPF_SOCKMAP_FILENAME);1410 int err;1411 1412 if (type == 0) {1413 test_start();1414 err = __test_exec(cgrp, SENDMSG, opt);1415 if (err)1416 test_fail();1417 } else {1418 test_start();1419 err = __test_exec(cgrp, SENDPAGE, opt);1420 if (err)1421 test_fail();1422 }1423}1424 1425static void test_send_one(struct sockmap_options *opt, int cgrp)1426{1427 opt->iov_length = 1;1428 opt->iov_count = 1;1429 opt->rate = 1;1430 test_exec(cgrp, opt);1431 1432 opt->iov_length = 1;1433 opt->iov_count = 1024;1434 opt->rate = 1;1435 test_exec(cgrp, opt);1436 1437 opt->iov_length = 1024;1438 opt->iov_count = 1;1439 opt->rate = 1;1440 test_exec(cgrp, opt);1441 1442}1443 1444static void test_send_many(struct sockmap_options *opt, int cgrp)1445{1446 opt->iov_length = 3;1447 opt->iov_count = 1;1448 opt->rate = 512;1449 test_exec(cgrp, opt);1450 1451 opt->rate = 100;1452 opt->iov_count = 1;1453 opt->iov_length = 5;1454 test_exec(cgrp, opt);1455}1456 1457static void test_send_large(struct sockmap_options *opt, int cgrp)1458{1459 opt->iov_length = 256;1460 opt->iov_count = 1024;1461 opt->rate = 2;1462 test_exec(cgrp, opt);1463}1464 1465static void test_send(struct sockmap_options *opt, int cgrp)1466{1467 test_send_one(opt, cgrp);1468 test_send_many(opt, cgrp);1469 test_send_large(opt, cgrp);1470 sched_yield();1471}1472 1473static void test_txmsg_pass(int cgrp, struct sockmap_options *opt)1474{1475 /* Test small and large iov_count values with pass/redir/apply/cork */1476 txmsg_pass = 1;1477 test_send(opt, cgrp);1478}1479 1480static void test_txmsg_redir(int cgrp, struct sockmap_options *opt)1481{1482 txmsg_redir = 1;1483 test_send(opt, cgrp);1484}1485 1486static void test_txmsg_redir_wait_sndmem(int cgrp, struct sockmap_options *opt)1487{1488 txmsg_redir = 1;1489 opt->tx_wait_mem = true;1490 test_send_large(opt, cgrp);1491 opt->tx_wait_mem = false;1492}1493 1494static void test_txmsg_drop(int cgrp, struct sockmap_options *opt)1495{1496 txmsg_drop = 1;1497 test_send(opt, cgrp);1498}1499 1500static void test_txmsg_ingress_redir(int cgrp, struct sockmap_options *opt)1501{1502 txmsg_pass = txmsg_drop = 0;1503 txmsg_ingress = txmsg_redir = 1;1504 test_send(opt, cgrp);1505}1506 1507static void test_txmsg_skb(int cgrp, struct sockmap_options *opt)1508{1509 bool data = opt->data_test;1510 int k = ktls;1511 1512 opt->data_test = true;1513 ktls = 1;1514 1515 txmsg_pass = txmsg_drop = 0;1516 txmsg_ingress = txmsg_redir = 0;1517 txmsg_ktls_skb = 1;1518 txmsg_pass = 1;1519 1520 /* Using data verification so ensure iov layout is1521 * expected from test receiver side. e.g. has enough1522 * bytes to write test code.1523 */1524 opt->iov_length = 100;1525 opt->iov_count = 1;1526 opt->rate = 1;1527 test_exec(cgrp, opt);1528 1529 txmsg_ktls_skb_drop = 1;1530 test_exec(cgrp, opt);1531 1532 txmsg_ktls_skb_drop = 0;1533 txmsg_ktls_skb_redir = 1;1534 test_exec(cgrp, opt);1535 txmsg_ktls_skb_redir = 0;1536 1537 /* Tests that omit skb_parser */1538 txmsg_omit_skb_parser = 1;1539 ktls = 0;1540 txmsg_ktls_skb = 0;1541 test_exec(cgrp, opt);1542 1543 txmsg_ktls_skb_drop = 1;1544 test_exec(cgrp, opt);1545 txmsg_ktls_skb_drop = 0;1546 1547 txmsg_ktls_skb_redir = 1;1548 test_exec(cgrp, opt);1549 1550 ktls = 1;1551 test_exec(cgrp, opt);1552 txmsg_omit_skb_parser = 0;1553 1554 opt->data_test = data;1555 ktls = k;1556}1557 1558/* Test cork with hung data. This tests poor usage patterns where1559 * cork can leave data on the ring if user program is buggy and1560 * doesn't flush them somehow. They do take some time however1561 * because they wait for a timeout. Test pass, redir and cork with1562 * apply logic. Use cork size of 4097 with send_large to avoid1563 * aligning cork size with send size.1564 */1565static void test_txmsg_cork_hangs(int cgrp, struct sockmap_options *opt)1566{1567 txmsg_pass = 1;1568 txmsg_redir = 0;1569 txmsg_cork = 4097;1570 txmsg_apply = 4097;1571 test_send_large(opt, cgrp);1572 1573 txmsg_pass = 0;1574 txmsg_redir = 1;1575 txmsg_apply = 0;1576 txmsg_cork = 4097;1577 test_send_large(opt, cgrp);1578 1579 txmsg_pass = 0;1580 txmsg_redir = 1;1581 txmsg_apply = 4097;1582 txmsg_cork = 4097;1583 test_send_large(opt, cgrp);1584}1585 1586static void test_txmsg_pull(int cgrp, struct sockmap_options *opt)1587{1588 /* Test basic start/end */1589 txmsg_start = 1;1590 txmsg_end = 2;1591 test_send(opt, cgrp);1592 1593 /* Test >4k pull */1594 txmsg_start = 4096;1595 txmsg_end = 9182;1596 test_send_large(opt, cgrp);1597 1598 /* Test pull + redirect */1599 txmsg_redir = 0;1600 txmsg_start = 1;1601 txmsg_end = 2;1602 test_send(opt, cgrp);1603 1604 /* Test pull + cork */1605 txmsg_redir = 0;1606 txmsg_cork = 512;1607 txmsg_start = 1;1608 txmsg_end = 2;1609 test_send_many(opt, cgrp);1610 1611 /* Test pull + cork + redirect */1612 txmsg_redir = 1;1613 txmsg_cork = 512;1614 txmsg_start = 1;1615 txmsg_end = 2;1616 test_send_many(opt, cgrp);1617}1618 1619static void test_txmsg_pop(int cgrp, struct sockmap_options *opt)1620{1621 /* Test basic pop */1622 txmsg_start_pop = 1;1623 txmsg_pop = 2;1624 test_send_many(opt, cgrp);1625 1626 /* Test pop with >4k */1627 txmsg_start_pop = 4096;1628 txmsg_pop = 4096;1629 test_send_large(opt, cgrp);1630 1631 /* Test pop + redirect */1632 txmsg_redir = 1;1633 txmsg_start_pop = 1;1634 txmsg_pop = 2;1635 test_send_many(opt, cgrp);1636 1637 /* Test pop + cork */1638 txmsg_redir = 0;1639 txmsg_cork = 512;1640 txmsg_start_pop = 1;1641 txmsg_pop = 2;1642 test_send_many(opt, cgrp);1643 1644 /* Test pop + redirect + cork */1645 txmsg_redir = 1;1646 txmsg_cork = 4;1647 txmsg_start_pop = 1;1648 txmsg_pop = 2;1649 test_send_many(opt, cgrp);1650}1651 1652static void test_txmsg_push(int cgrp, struct sockmap_options *opt)1653{1654 /* Test basic push */1655 txmsg_start_push = 1;1656 txmsg_end_push = 1;1657 test_send(opt, cgrp);1658 1659 /* Test push 4kB >4k */1660 txmsg_start_push = 4096;1661 txmsg_end_push = 4096;1662 test_send_large(opt, cgrp);1663 1664 /* Test push + redirect */1665 txmsg_redir = 1;1666 txmsg_start_push = 1;1667 txmsg_end_push = 2;1668 test_send_many(opt, cgrp);1669 1670 /* Test push + cork */1671 txmsg_redir = 0;1672 txmsg_cork = 512;1673 txmsg_start_push = 1;1674 txmsg_end_push = 2;1675 test_send_many(opt, cgrp);1676}1677 1678static void test_txmsg_push_pop(int cgrp, struct sockmap_options *opt)1679{1680 txmsg_start_push = 1;1681 txmsg_end_push = 10;1682 txmsg_start_pop = 5;1683 txmsg_pop = 4;1684 test_send_large(opt, cgrp);1685}1686 1687static void test_txmsg_apply(int cgrp, struct sockmap_options *opt)1688{1689 txmsg_pass = 1;1690 txmsg_redir = 0;1691 txmsg_ingress = 0;1692 txmsg_apply = 1;1693 txmsg_cork = 0;1694 test_send_one(opt, cgrp);1695 1696 txmsg_pass = 0;1697 txmsg_redir = 1;1698 txmsg_ingress = 0;1699 txmsg_apply = 1;1700 txmsg_cork = 0;1701 test_send_one(opt, cgrp);1702 1703 txmsg_pass = 0;1704 txmsg_redir = 1;1705 txmsg_ingress = 1;1706 txmsg_apply = 1;1707 txmsg_cork = 0;1708 test_send_one(opt, cgrp);1709 1710 txmsg_pass = 1;1711 txmsg_redir = 0;1712 txmsg_ingress = 0;1713 txmsg_apply = 1024;1714 txmsg_cork = 0;1715 test_send_large(opt, cgrp);1716 1717 txmsg_pass = 0;1718 txmsg_redir = 1;1719 txmsg_ingress = 0;1720 txmsg_apply = 1024;1721 txmsg_cork = 0;1722 test_send_large(opt, cgrp);1723 1724 txmsg_pass = 0;1725 txmsg_redir = 1;1726 txmsg_ingress = 1;1727 txmsg_apply = 1024;1728 txmsg_cork = 0;1729 test_send_large(opt, cgrp);1730}1731 1732static void test_txmsg_cork(int cgrp, struct sockmap_options *opt)1733{1734 txmsg_pass = 1;1735 txmsg_redir = 0;1736 txmsg_apply = 0;1737 txmsg_cork = 1;1738 test_send(opt, cgrp);1739 1740 txmsg_pass = 1;1741 txmsg_redir = 0;1742 txmsg_apply = 1;1743 txmsg_cork = 1;1744 test_send(opt, cgrp);1745}1746 1747static void test_txmsg_ingress_parser(int cgrp, struct sockmap_options *opt)1748{1749 txmsg_pass = 1;1750 skb_use_parser = 512;1751 if (ktls == 1)1752 skb_use_parser = 570;1753 opt->iov_length = 256;1754 opt->iov_count = 1;1755 opt->rate = 2;1756 test_exec(cgrp, opt);1757}1758 1759static void test_txmsg_ingress_parser2(int cgrp, struct sockmap_options *opt)1760{1761 if (ktls == 1)1762 return;1763 skb_use_parser = 10;1764 opt->iov_length = 20;1765 opt->iov_count = 1;1766 opt->rate = 1;1767 opt->check_recved_len = true;1768 test_exec(cgrp, opt);1769 opt->check_recved_len = false;1770}1771 1772char *map_names[] = {1773 "sock_map",1774 "sock_map_txmsg",1775 "sock_map_redir",1776 "sock_apply_bytes",1777 "sock_cork_bytes",1778 "sock_bytes",1779 "sock_redir_flags",1780 "sock_skb_opts",1781 "tls_sock_map",1782};1783 1784static int populate_progs(char *bpf_file)1785{1786 struct bpf_program *prog;1787 struct bpf_object *obj;1788 int i = 0;1789 long err;1790 1791 obj = bpf_object__open(bpf_file);1792 err = libbpf_get_error(obj);1793 if (err) {1794 char err_buf[256];1795 1796 libbpf_strerror(err, err_buf, sizeof(err_buf));1797 printf("Unable to load eBPF objects in file '%s' : %s\n",1798 bpf_file, err_buf);1799 return -1;1800 }1801 1802 i = bpf_object__load(obj);1803 i = 0;1804 bpf_object__for_each_program(prog, obj) {1805 progs[i] = prog;1806 i++;1807 }1808 1809 for (i = 0; i < ARRAY_SIZE(map_fd); i++) {1810 maps[i] = bpf_object__find_map_by_name(obj, map_names[i]);1811 map_fd[i] = bpf_map__fd(maps[i]);1812 if (map_fd[i] < 0) {1813 fprintf(stderr, "load_bpf_file: (%i) %s\n",1814 map_fd[i], strerror(errno));1815 return -1;1816 }1817 }1818 1819 for (i = 0; i < ARRAY_SIZE(links); i++)1820 links[i] = NULL;1821 1822 return 0;1823}1824 1825struct _test test[] = {1826 {"txmsg test passthrough", test_txmsg_pass},1827 {"txmsg test redirect", test_txmsg_redir},1828 {"txmsg test redirect wait send mem", test_txmsg_redir_wait_sndmem},1829 {"txmsg test drop", test_txmsg_drop},1830 {"txmsg test ingress redirect", test_txmsg_ingress_redir},1831 {"txmsg test skb", test_txmsg_skb},1832 {"txmsg test apply", test_txmsg_apply},1833 {"txmsg test cork", test_txmsg_cork},1834 {"txmsg test hanging corks", test_txmsg_cork_hangs},1835 {"txmsg test push_data", test_txmsg_push},1836 {"txmsg test pull-data", test_txmsg_pull},1837 {"txmsg test pop-data", test_txmsg_pop},1838 {"txmsg test push/pop data", test_txmsg_push_pop},1839 {"txmsg test ingress parser", test_txmsg_ingress_parser},1840 {"txmsg test ingress parser2", test_txmsg_ingress_parser2},1841};1842 1843static int check_whitelist(struct _test *t, struct sockmap_options *opt)1844{1845 char *entry, *ptr;1846 1847 if (!opt->whitelist)1848 return 0;1849 ptr = strdup(opt->whitelist);1850 if (!ptr)1851 return -ENOMEM;1852 entry = strtok(ptr, ",");1853 while (entry) {1854 if ((opt->prepend && strstr(opt->prepend, entry) != 0) ||1855 strstr(opt->map, entry) != 0 ||1856 strstr(t->title, entry) != 0) {1857 free(ptr);1858 return 0;1859 }1860 entry = strtok(NULL, ",");1861 }1862 free(ptr);1863 return -EINVAL;1864}1865 1866static int check_blacklist(struct _test *t, struct sockmap_options *opt)1867{1868 char *entry, *ptr;1869 1870 if (!opt->blacklist)1871 return -EINVAL;1872 ptr = strdup(opt->blacklist);1873 if (!ptr)1874 return -ENOMEM;1875 entry = strtok(ptr, ",");1876 while (entry) {1877 if ((opt->prepend && strstr(opt->prepend, entry) != 0) ||1878 strstr(opt->map, entry) != 0 ||1879 strstr(t->title, entry) != 0) {1880 free(ptr);1881 return 0;1882 }1883 entry = strtok(NULL, ",");1884 }1885 free(ptr);1886 return -EINVAL;1887}1888 1889static int __test_selftests(int cg_fd, struct sockmap_options *opt)1890{1891 int i, err;1892 1893 err = populate_progs(opt->map);1894 if (err < 0) {1895 fprintf(stderr, "ERROR: (%i) load bpf failed\n", err);1896 return err;1897 }1898 1899 /* Tests basic commands and APIs */1900 for (i = 0; i < ARRAY_SIZE(test); i++) {1901 struct _test t = test[i];1902 1903 if (check_whitelist(&t, opt) != 0)1904 continue;1905 if (check_blacklist(&t, opt) == 0)1906 continue;1907 1908 test_start_subtest(&t, opt);1909 t.tester(cg_fd, opt);1910 test_end_subtest();1911 }1912 1913 return err;1914}1915 1916static void test_selftests_sockmap(int cg_fd, struct sockmap_options *opt)1917{1918 opt->map = BPF_SOCKMAP_FILENAME;1919 __test_selftests(cg_fd, opt);1920}1921 1922static void test_selftests_sockhash(int cg_fd, struct sockmap_options *opt)1923{1924 opt->map = BPF_SOCKHASH_FILENAME;1925 __test_selftests(cg_fd, opt);1926}1927 1928static void test_selftests_ktls(int cg_fd, struct sockmap_options *opt)1929{1930 opt->map = BPF_SOCKHASH_FILENAME;1931 opt->prepend = "ktls";1932 ktls = 1;1933 __test_selftests(cg_fd, opt);1934 ktls = 0;1935}1936 1937static int test_selftest(int cg_fd, struct sockmap_options *opt)1938{1939 test_selftests_sockmap(cg_fd, opt);1940 test_selftests_sockhash(cg_fd, opt);1941 test_selftests_ktls(cg_fd, opt);1942 test_print_results();1943 return 0;1944}1945 1946int main(int argc, char **argv)1947{1948 int iov_count = 1, length = 1024, rate = 1;1949 struct sockmap_options options = {0};1950 int opt, longindex, err, cg_fd = 0;1951 char *bpf_file = BPF_SOCKMAP_FILENAME;1952 int test = SELFTESTS;1953 bool cg_created = 0;1954 1955 while ((opt = getopt_long(argc, argv, ":dhv:c:r:i:l:t:p:q:n:b:",1956 long_options, &longindex)) != -1) {1957 switch (opt) {1958 case 's':1959 txmsg_start = atoi(optarg);1960 break;1961 case 'e':1962 txmsg_end = atoi(optarg);1963 break;1964 case 'p':1965 txmsg_start_push = atoi(optarg);1966 break;1967 case 'q':1968 txmsg_end_push = atoi(optarg);1969 break;1970 case 'w':1971 txmsg_start_pop = atoi(optarg);1972 break;1973 case 'x':1974 txmsg_pop = atoi(optarg);1975 break;1976 case 'a':1977 txmsg_apply = atoi(optarg);1978 break;1979 case 'k':1980 txmsg_cork = atoi(optarg);1981 break;1982 case 'c':1983 cg_fd = open(optarg, O_DIRECTORY, O_RDONLY);1984 if (cg_fd < 0) {1985 fprintf(stderr,1986 "ERROR: (%i) open cg path failed: %s\n",1987 cg_fd, optarg);1988 return cg_fd;1989 }1990 break;1991 case 'r':1992 rate = atoi(optarg);1993 break;1994 case 'v':1995 options.verbose = 1;1996 if (optarg)1997 options.verbose = atoi(optarg);1998 break;1999 case 'i':2000 iov_count = atoi(optarg);2001 break;2002 case 'l':2003 length = atoi(optarg);2004 break;2005 case 'd':2006 options.data_test = true;2007 break;2008 case 't':2009 if (strcmp(optarg, "ping") == 0) {2010 test = PING_PONG;2011 } else if (strcmp(optarg, "sendmsg") == 0) {2012 test = SENDMSG;2013 } else if (strcmp(optarg, "base") == 0) {2014 test = BASE;2015 } else if (strcmp(optarg, "base_sendpage") == 0) {2016 test = BASE_SENDPAGE;2017 } else if (strcmp(optarg, "sendpage") == 0) {2018 test = SENDPAGE;2019 } else {2020 usage(argv);2021 return -1;2022 }2023 break;2024 case 'n':2025 options.whitelist = strdup(optarg);2026 if (!options.whitelist)2027 return -ENOMEM;2028 break;2029 case 'b':2030 options.blacklist = strdup(optarg);2031 if (!options.blacklist)2032 return -ENOMEM;2033 case 0:2034 break;2035 case 'h':2036 default:2037 usage(argv);2038 return -1;2039 }2040 }2041 2042 if (!cg_fd) {2043 cg_fd = cgroup_setup_and_join(CG_PATH);2044 if (cg_fd < 0)2045 return cg_fd;2046 cg_created = 1;2047 }2048 2049 /* Use libbpf 1.0 API mode */2050 libbpf_set_strict_mode(LIBBPF_STRICT_ALL);2051 2052 if (test == SELFTESTS) {2053 err = test_selftest(cg_fd, &options);2054 goto out;2055 }2056 2057 err = populate_progs(bpf_file);2058 if (err) {2059 fprintf(stderr, "populate program: (%s) %s\n",2060 bpf_file, strerror(errno));2061 return 1;2062 }2063 running = 1;2064 2065 /* catch SIGINT */2066 signal(SIGINT, running_handler);2067 2068 options.iov_count = iov_count;2069 options.iov_length = length;2070 options.rate = rate;2071 2072 err = run_options(&options, cg_fd, test);2073out:2074 if (options.whitelist)2075 free(options.whitelist);2076 if (options.blacklist)2077 free(options.blacklist);2078 close(cg_fd);2079 if (cg_created)2080 cleanup_cgroup_environment();2081 return err;2082}2083 2084void running_handler(int a)2085{2086 running = 0;2087}2088