brintos / linux-shallow public Read only

1// SPDX-License-Identifier: GPL-2.02/* Copyright(c) 2020 Intel Corporation. */3 4/*5 * Some functions in this program are taken from6 * Linux kernel samples/bpf/xdpsock* and modified7 * for use.8 *9 * See test_xsk.sh for detailed information on test topology10 * and prerequisite network setup.11 *12 * This test program contains two threads, each thread is single socket with13 * a unique UMEM. It validates in-order packet delivery and packet content14 * by sending packets to each other.15 *16 * Tests Information:17 * ------------------18 * These selftests test AF_XDP SKB and Native/DRV modes using veth19 * Virtual Ethernet interfaces.20 *21 * For each mode, the following tests are run:22 *    a. nopoll - soft-irq processing in run-to-completion mode23 *    b. poll - using poll() syscall24 *    c. Socket Teardown25 *       Create a Tx and a Rx socket, Tx from one socket, Rx on another. Destroy26 *       both sockets, then repeat multiple times. Only nopoll mode is used27 *    d. Bi-directional sockets28 *       Configure sockets as bi-directional tx/rx sockets, sets up fill and29 *       completion rings on each socket, tx/rx in both directions. Only nopoll30 *       mode is used31 *    e. Statistics32 *       Trigger some error conditions and ensure that the appropriate statistics33 *       are incremented. Within this test, the following statistics are tested:34 *       i.   rx dropped35 *            Increase the UMEM frame headroom to a value which results in36 *            insufficient space in the rx buffer for both the packet and the headroom.37 *       ii.  tx invalid38 *            Set the 'len' field of tx descriptors to an invalid value (umem frame39 *            size + 1).40 *       iii. rx ring full41 *            Reduce the size of the RX ring to a fraction of the fill ring size.42 *       iv.  fill queue empty43 *            Do not populate the fill queue and then try to receive pkts.44 *    f. bpf_link resource persistence45 *       Configure sockets at indexes 0 and 1, run a traffic on queue ids 0,46 *       then remove xsk sockets from queue 0 on both veth interfaces and47 *       finally run a traffic on queues ids 148 *    g. unaligned mode49 *    h. tests for invalid and corner case Tx descriptors so that the correct ones50 *       are discarded and let through, respectively.51 *    i. 2K frame size tests52 *    j. If multi-buffer is supported, send 9k packets divided into 3 frames53 *    k. If multi-buffer and huge pages are supported, send 9k packets in a single frame54 *       using unaligned mode55 *    l. If multi-buffer is supported, try various nasty combinations of descriptors to56 *       check if they pass the validation or not57 *58 * Flow:59 * -----60 * - Single process spawns two threads: Tx and Rx61 * - Each of these two threads attach to a veth interface62 * - Each thread creates one AF_XDP socket connected to a unique umem for each63 *   veth interface64 * - Tx thread Transmits a number of packets from veth<xxxx> to veth<yyyy>65 * - Rx thread verifies if all packets were received and delivered in-order,66 *   and have the right content67 *68 * Enable/disable packet dump mode:69 * --------------------------70 * To enable L2 - L4 headers and payload dump of each packet on STDOUT, add71 * parameter -D to params array in test_xsk.sh, i.e. params=("-S" "-D")72 */73 74#define _GNU_SOURCE75#include <assert.h>76#include <fcntl.h>77#include <errno.h>78#include <getopt.h>79#include <linux/if_link.h>80#include <linux/if_ether.h>81#include <linux/mman.h>82#include <linux/netdev.h>83#include <linux/bitmap.h>84#include <linux/ethtool.h>85#include <arpa/inet.h>86#include <net/if.h>87#include <locale.h>88#include <poll.h>89#include <pthread.h>90#include <signal.h>91#include <stdio.h>92#include <stdlib.h>93#include <libgen.h>94#include <string.h>95#include <stddef.h>96#include <sys/mman.h>97#include <sys/socket.h>98#include <sys/time.h>99#include <sys/types.h>100#include <unistd.h>101 102#include "xsk_xdp_progs.skel.h"103#include "xsk.h"104#include "xskxceiver.h"105#include <bpf/bpf.h>106#include <linux/filter.h>107#include "../kselftest.h"108#include "xsk_xdp_common.h"109 110#include <network_helpers.h>111 112static bool opt_verbose;113static bool opt_print_tests;114static enum test_mode opt_mode = TEST_MODE_ALL;115static u32 opt_run_test = RUN_ALL_TESTS;116 117void test__fail(void) { /* for network_helpers.c */ }118 119static void __exit_with_error(int error, const char *file, const char *func, int line)120{121	ksft_test_result_fail("[%s:%s:%i]: ERROR: %d/\"%s\"\n", file, func, line, error,122			      strerror(error));123	ksft_exit_xfail();124}125 126#define exit_with_error(error) __exit_with_error(error, __FILE__, __func__, __LINE__)127#define busy_poll_string(test) (test)->ifobj_tx->busy_poll ? "BUSY-POLL " : ""128static char *mode_string(struct test_spec *test)129{130	switch (test->mode) {131	case TEST_MODE_SKB:132		return "SKB";133	case TEST_MODE_DRV:134		return "DRV";135	case TEST_MODE_ZC:136		return "ZC";137	default:138		return "BOGUS";139	}140}141 142static void report_failure(struct test_spec *test)143{144	if (test->fail)145		return;146 147	ksft_test_result_fail("FAIL: %s %s%s\n", mode_string(test), busy_poll_string(test),148			      test->name);149	test->fail = true;150}151 152/* The payload is a word consisting of a packet sequence number in the upper153 * 16-bits and a intra packet data sequence number in the lower 16 bits. So the 3rd packet's154 * 5th word of data will contain the number (2<<16) | 4 as they are numbered from 0.155 */156static void write_payload(void *dest, u32 pkt_nb, u32 start, u32 size)157{158	u32 *ptr = (u32 *)dest, i;159 160	start /= sizeof(*ptr);161	size /= sizeof(*ptr);162	for (i = 0; i < size; i++)163		ptr[i] = htonl(pkt_nb << 16 | (i + start));164}165 166static void gen_eth_hdr(struct xsk_socket_info *xsk, struct ethhdr *eth_hdr)167{168	memcpy(eth_hdr->h_dest, xsk->dst_mac, ETH_ALEN);169	memcpy(eth_hdr->h_source, xsk->src_mac, ETH_ALEN);170	eth_hdr->h_proto = htons(ETH_P_LOOPBACK);171}172 173static bool is_umem_valid(struct ifobject *ifobj)174{175	return !!ifobj->umem->umem;176}177 178static u32 mode_to_xdp_flags(enum test_mode mode)179{180	return (mode == TEST_MODE_SKB) ? XDP_FLAGS_SKB_MODE : XDP_FLAGS_DRV_MODE;181}182 183static u64 umem_size(struct xsk_umem_info *umem)184{185	return umem->num_frames * umem->frame_size;186}187 188static int xsk_configure_umem(struct ifobject *ifobj, struct xsk_umem_info *umem, void *buffer,189			      u64 size)190{191	struct xsk_umem_config cfg = {192		.fill_size = XSK_RING_PROD__DEFAULT_NUM_DESCS,193		.comp_size = XSK_RING_CONS__DEFAULT_NUM_DESCS,194		.frame_size = umem->frame_size,195		.frame_headroom = umem->frame_headroom,196		.flags = XSK_UMEM__DEFAULT_FLAGS197	};198	int ret;199 200	if (umem->fill_size)201		cfg.fill_size = umem->fill_size;202 203	if (umem->comp_size)204		cfg.comp_size = umem->comp_size;205 206	if (umem->unaligned_mode)207		cfg.flags |= XDP_UMEM_UNALIGNED_CHUNK_FLAG;208 209	ret = xsk_umem__create(&umem->umem, buffer, size,210			       &umem->fq, &umem->cq, &cfg);211	if (ret)212		return ret;213 214	umem->buffer = buffer;215	if (ifobj->shared_umem && ifobj->rx_on) {216		umem->base_addr = umem_size(umem);217		umem->next_buffer = umem_size(umem);218	}219 220	return 0;221}222 223static u64 umem_alloc_buffer(struct xsk_umem_info *umem)224{225	u64 addr;226 227	addr = umem->next_buffer;228	umem->next_buffer += umem->frame_size;229	if (umem->next_buffer >= umem->base_addr + umem_size(umem))230		umem->next_buffer = umem->base_addr;231 232	return addr;233}234 235static void umem_reset_alloc(struct xsk_umem_info *umem)236{237	umem->next_buffer = 0;238}239 240static void enable_busy_poll(struct xsk_socket_info *xsk)241{242	int sock_opt;243 244	sock_opt = 1;245	if (setsockopt(xsk_socket__fd(xsk->xsk), SOL_SOCKET, SO_PREFER_BUSY_POLL,246		       (void *)&sock_opt, sizeof(sock_opt)) < 0)247		exit_with_error(errno);248 249	sock_opt = 20;250	if (setsockopt(xsk_socket__fd(xsk->xsk), SOL_SOCKET, SO_BUSY_POLL,251		       (void *)&sock_opt, sizeof(sock_opt)) < 0)252		exit_with_error(errno);253 254	sock_opt = xsk->batch_size;255	if (setsockopt(xsk_socket__fd(xsk->xsk), SOL_SOCKET, SO_BUSY_POLL_BUDGET,256		       (void *)&sock_opt, sizeof(sock_opt)) < 0)257		exit_with_error(errno);258}259 260static int __xsk_configure_socket(struct xsk_socket_info *xsk, struct xsk_umem_info *umem,261				  struct ifobject *ifobject, bool shared)262{263	struct xsk_socket_config cfg = {};264	struct xsk_ring_cons *rxr;265	struct xsk_ring_prod *txr;266 267	xsk->umem = umem;268	cfg.rx_size = xsk->rxqsize;269	cfg.tx_size = XSK_RING_PROD__DEFAULT_NUM_DESCS;270	cfg.bind_flags = ifobject->bind_flags;271	if (shared)272		cfg.bind_flags |= XDP_SHARED_UMEM;273	if (ifobject->mtu > MAX_ETH_PKT_SIZE)274		cfg.bind_flags |= XDP_USE_SG;275	if (umem->comp_size)276		cfg.tx_size = umem->comp_size;277	if (umem->fill_size)278		cfg.rx_size = umem->fill_size;279 280	txr = ifobject->tx_on ? &xsk->tx : NULL;281	rxr = ifobject->rx_on ? &xsk->rx : NULL;282	return xsk_socket__create(&xsk->xsk, ifobject->ifindex, 0, umem->umem, rxr, txr, &cfg);283}284 285static bool ifobj_zc_avail(struct ifobject *ifobject)286{287	size_t umem_sz = DEFAULT_UMEM_BUFFERS * XSK_UMEM__DEFAULT_FRAME_SIZE;288	int mmap_flags = MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE;289	struct xsk_socket_info *xsk;290	struct xsk_umem_info *umem;291	bool zc_avail = false;292	void *bufs;293	int ret;294 295	bufs = mmap(NULL, umem_sz, PROT_READ | PROT_WRITE, mmap_flags, -1, 0);296	if (bufs == MAP_FAILED)297		exit_with_error(errno);298 299	umem = calloc(1, sizeof(struct xsk_umem_info));300	if (!umem) {301		munmap(bufs, umem_sz);302		exit_with_error(ENOMEM);303	}304	umem->frame_size = XSK_UMEM__DEFAULT_FRAME_SIZE;305	ret = xsk_configure_umem(ifobject, umem, bufs, umem_sz);306	if (ret)307		exit_with_error(-ret);308 309	xsk = calloc(1, sizeof(struct xsk_socket_info));310	if (!xsk)311		goto out;312	ifobject->bind_flags = XDP_USE_NEED_WAKEUP | XDP_ZEROCOPY;313	ifobject->rx_on = true;314	xsk->rxqsize = XSK_RING_CONS__DEFAULT_NUM_DESCS;315	ret = __xsk_configure_socket(xsk, umem, ifobject, false);316	if (!ret)317		zc_avail = true;318 319	xsk_socket__delete(xsk->xsk);320	free(xsk);321out:322	munmap(umem->buffer, umem_sz);323	xsk_umem__delete(umem->umem);324	free(umem);325	return zc_avail;326}327 328#define MAX_SKB_FRAGS_PATH "/proc/sys/net/core/max_skb_frags"329static unsigned int get_max_skb_frags(void)330{331	unsigned int max_skb_frags = 0;332	FILE *file;333 334	file = fopen(MAX_SKB_FRAGS_PATH, "r");335	if (!file) {336		ksft_print_msg("Error opening %s\n", MAX_SKB_FRAGS_PATH);337		return 0;338	}339 340	if (fscanf(file, "%u", &max_skb_frags) != 1)341		ksft_print_msg("Error reading %s\n", MAX_SKB_FRAGS_PATH);342 343	fclose(file);344	return max_skb_frags;345}346 347static struct option long_options[] = {348	{"interface", required_argument, 0, 'i'},349	{"busy-poll", no_argument, 0, 'b'},350	{"verbose", no_argument, 0, 'v'},351	{"mode", required_argument, 0, 'm'},352	{"list", no_argument, 0, 'l'},353	{"test", required_argument, 0, 't'},354	{"help", no_argument, 0, 'h'},355	{0, 0, 0, 0}356};357 358static void print_usage(char **argv)359{360	const char *str =361		"  Usage: xskxceiver [OPTIONS]\n"362		"  Options:\n"363		"  -i, --interface      Use interface\n"364		"  -v, --verbose        Verbose output\n"365		"  -b, --busy-poll      Enable busy poll\n"366		"  -m, --mode           Run only mode skb, drv, or zc\n"367		"  -l, --list           List all available tests\n"368		"  -t, --test           Run a specific test. Enter number from -l option.\n"369		"  -h, --help           Display this help and exit\n";370 371	ksft_print_msg(str, basename(argv[0]));372	ksft_exit_xfail();373}374 375static bool validate_interface(struct ifobject *ifobj)376{377	if (!strcmp(ifobj->ifname, ""))378		return false;379	return true;380}381 382static void parse_command_line(struct ifobject *ifobj_tx, struct ifobject *ifobj_rx, int argc,383			       char **argv)384{385	struct ifobject *ifobj;386	u32 interface_nb = 0;387	int option_index, c;388 389	opterr = 0;390 391	for (;;) {392		c = getopt_long(argc, argv, "i:vbm:lt:", long_options, &option_index);393		if (c == -1)394			break;395 396		switch (c) {397		case 'i':398			if (interface_nb == 0)399				ifobj = ifobj_tx;400			else if (interface_nb == 1)401				ifobj = ifobj_rx;402			else403				break;404 405			memcpy(ifobj->ifname, optarg,406			       min_t(size_t, MAX_INTERFACE_NAME_CHARS, strlen(optarg)));407 408			ifobj->ifindex = if_nametoindex(ifobj->ifname);409			if (!ifobj->ifindex)410				exit_with_error(errno);411 412			interface_nb++;413			break;414		case 'v':415			opt_verbose = true;416			break;417		case 'b':418			ifobj_tx->busy_poll = true;419			ifobj_rx->busy_poll = true;420			break;421		case 'm':422			if (!strncmp("skb", optarg, strlen(optarg)))423				opt_mode = TEST_MODE_SKB;424			else if (!strncmp("drv", optarg, strlen(optarg)))425				opt_mode = TEST_MODE_DRV;426			else if (!strncmp("zc", optarg, strlen(optarg)))427				opt_mode = TEST_MODE_ZC;428			else429				print_usage(argv);430			break;431		case 'l':432			opt_print_tests = true;433			break;434		case 't':435			errno = 0;436			opt_run_test = strtol(optarg, NULL, 0);437			if (errno)438				print_usage(argv);439			break;440		case 'h':441		default:442			print_usage(argv);443		}444	}445}446 447static int set_ring_size(struct ifobject *ifobj)448{449	int ret;450	u32 ctr = 0;451 452	while (ctr++ < SOCK_RECONF_CTR) {453		ret = set_hw_ring_size(ifobj->ifname, &ifobj->ring);454		if (!ret)455			break;456 457		/* Retry if it fails */458		if (ctr >= SOCK_RECONF_CTR || errno != EBUSY)459			return -errno;460 461		usleep(USLEEP_MAX);462	}463 464	return ret;465}466 467static int hw_ring_size_reset(struct ifobject *ifobj)468{469	ifobj->ring.tx_pending = ifobj->set_ring.default_tx;470	ifobj->ring.rx_pending = ifobj->set_ring.default_rx;471	return set_ring_size(ifobj);472}473 474static void __test_spec_init(struct test_spec *test, struct ifobject *ifobj_tx,475			     struct ifobject *ifobj_rx)476{477	u32 i, j;478 479	for (i = 0; i < MAX_INTERFACES; i++) {480		struct ifobject *ifobj = i ? ifobj_rx : ifobj_tx;481 482		ifobj->xsk = &ifobj->xsk_arr[0];483		ifobj->use_poll = false;484		ifobj->use_fill_ring = true;485		ifobj->release_rx = true;486		ifobj->validation_func = NULL;487		ifobj->use_metadata = false;488 489		if (i == 0) {490			ifobj->rx_on = false;491			ifobj->tx_on = true;492		} else {493			ifobj->rx_on = true;494			ifobj->tx_on = false;495		}496 497		memset(ifobj->umem, 0, sizeof(*ifobj->umem));498		ifobj->umem->num_frames = DEFAULT_UMEM_BUFFERS;499		ifobj->umem->frame_size = XSK_UMEM__DEFAULT_FRAME_SIZE;500 501		for (j = 0; j < MAX_SOCKETS; j++) {502			memset(&ifobj->xsk_arr[j], 0, sizeof(ifobj->xsk_arr[j]));503			ifobj->xsk_arr[j].rxqsize = XSK_RING_CONS__DEFAULT_NUM_DESCS;504			ifobj->xsk_arr[j].batch_size = DEFAULT_BATCH_SIZE;505			if (i == 0)506				ifobj->xsk_arr[j].pkt_stream = test->tx_pkt_stream_default;507			else508				ifobj->xsk_arr[j].pkt_stream = test->rx_pkt_stream_default;509 510			memcpy(ifobj->xsk_arr[j].src_mac, g_mac, ETH_ALEN);511			memcpy(ifobj->xsk_arr[j].dst_mac, g_mac, ETH_ALEN);512			ifobj->xsk_arr[j].src_mac[5] += ((j * 2) + 0);513			ifobj->xsk_arr[j].dst_mac[5] += ((j * 2) + 1);514		}515	}516 517	if (ifobj_tx->hw_ring_size_supp)518		hw_ring_size_reset(ifobj_tx);519 520	test->ifobj_tx = ifobj_tx;521	test->ifobj_rx = ifobj_rx;522	test->current_step = 0;523	test->total_steps = 1;524	test->nb_sockets = 1;525	test->fail = false;526	test->set_ring = false;527	test->mtu = MAX_ETH_PKT_SIZE;528	test->xdp_prog_rx = ifobj_rx->xdp_progs->progs.xsk_def_prog;529	test->xskmap_rx = ifobj_rx->xdp_progs->maps.xsk;530	test->xdp_prog_tx = ifobj_tx->xdp_progs->progs.xsk_def_prog;531	test->xskmap_tx = ifobj_tx->xdp_progs->maps.xsk;532}533 534static void test_spec_init(struct test_spec *test, struct ifobject *ifobj_tx,535			   struct ifobject *ifobj_rx, enum test_mode mode,536			   const struct test_spec *test_to_run)537{538	struct pkt_stream *tx_pkt_stream;539	struct pkt_stream *rx_pkt_stream;540	u32 i;541 542	tx_pkt_stream = test->tx_pkt_stream_default;543	rx_pkt_stream = test->rx_pkt_stream_default;544	memset(test, 0, sizeof(*test));545	test->tx_pkt_stream_default = tx_pkt_stream;546	test->rx_pkt_stream_default = rx_pkt_stream;547 548	for (i = 0; i < MAX_INTERFACES; i++) {549		struct ifobject *ifobj = i ? ifobj_rx : ifobj_tx;550 551		ifobj->bind_flags = XDP_USE_NEED_WAKEUP;552		if (mode == TEST_MODE_ZC)553			ifobj->bind_flags |= XDP_ZEROCOPY;554		else555			ifobj->bind_flags |= XDP_COPY;556	}557 558	strncpy(test->name, test_to_run->name, MAX_TEST_NAME_SIZE);559	test->test_func = test_to_run->test_func;560	test->mode = mode;561	__test_spec_init(test, ifobj_tx, ifobj_rx);562}563 564static void test_spec_reset(struct test_spec *test)565{566	__test_spec_init(test, test->ifobj_tx, test->ifobj_rx);567}568 569static void test_spec_set_xdp_prog(struct test_spec *test, struct bpf_program *xdp_prog_rx,570				   struct bpf_program *xdp_prog_tx, struct bpf_map *xskmap_rx,571				   struct bpf_map *xskmap_tx)572{573	test->xdp_prog_rx = xdp_prog_rx;574	test->xdp_prog_tx = xdp_prog_tx;575	test->xskmap_rx = xskmap_rx;576	test->xskmap_tx = xskmap_tx;577}578 579static int test_spec_set_mtu(struct test_spec *test, int mtu)580{581	int err;582 583	if (test->ifobj_rx->mtu != mtu) {584		err = xsk_set_mtu(test->ifobj_rx->ifindex, mtu);585		if (err)586			return err;587		test->ifobj_rx->mtu = mtu;588	}589	if (test->ifobj_tx->mtu != mtu) {590		err = xsk_set_mtu(test->ifobj_tx->ifindex, mtu);591		if (err)592			return err;593		test->ifobj_tx->mtu = mtu;594	}595 596	return 0;597}598 599static void pkt_stream_reset(struct pkt_stream *pkt_stream)600{601	if (pkt_stream) {602		pkt_stream->current_pkt_nb = 0;603		pkt_stream->nb_rx_pkts = 0;604	}605}606 607static struct pkt *pkt_stream_get_next_tx_pkt(struct pkt_stream *pkt_stream)608{609	if (pkt_stream->current_pkt_nb >= pkt_stream->nb_pkts)610		return NULL;611 612	return &pkt_stream->pkts[pkt_stream->current_pkt_nb++];613}614 615static struct pkt *pkt_stream_get_next_rx_pkt(struct pkt_stream *pkt_stream, u32 *pkts_sent)616{617	while (pkt_stream->current_pkt_nb < pkt_stream->nb_pkts) {618		(*pkts_sent)++;619		if (pkt_stream->pkts[pkt_stream->current_pkt_nb].valid)620			return &pkt_stream->pkts[pkt_stream->current_pkt_nb++];621		pkt_stream->current_pkt_nb++;622	}623	return NULL;624}625 626static void pkt_stream_delete(struct pkt_stream *pkt_stream)627{628	free(pkt_stream->pkts);629	free(pkt_stream);630}631 632static void pkt_stream_restore_default(struct test_spec *test)633{634	struct pkt_stream *tx_pkt_stream = test->ifobj_tx->xsk->pkt_stream;635	struct pkt_stream *rx_pkt_stream = test->ifobj_rx->xsk->pkt_stream;636 637	if (tx_pkt_stream != test->tx_pkt_stream_default) {638		pkt_stream_delete(test->ifobj_tx->xsk->pkt_stream);639		test->ifobj_tx->xsk->pkt_stream = test->tx_pkt_stream_default;640	}641 642	if (rx_pkt_stream != test->rx_pkt_stream_default) {643		pkt_stream_delete(test->ifobj_rx->xsk->pkt_stream);644		test->ifobj_rx->xsk->pkt_stream = test->rx_pkt_stream_default;645	}646}647 648static struct pkt_stream *__pkt_stream_alloc(u32 nb_pkts)649{650	struct pkt_stream *pkt_stream;651 652	pkt_stream = calloc(1, sizeof(*pkt_stream));653	if (!pkt_stream)654		return NULL;655 656	pkt_stream->pkts = calloc(nb_pkts, sizeof(*pkt_stream->pkts));657	if (!pkt_stream->pkts) {658		free(pkt_stream);659		return NULL;660	}661 662	pkt_stream->nb_pkts = nb_pkts;663	return pkt_stream;664}665 666static bool pkt_continues(u32 options)667{668	return options & XDP_PKT_CONTD;669}670 671static u32 ceil_u32(u32 a, u32 b)672{673	return (a + b - 1) / b;674}675 676static u32 pkt_nb_frags(u32 frame_size, struct pkt_stream *pkt_stream, struct pkt *pkt)677{678	u32 nb_frags = 1, next_frag;679 680	if (!pkt)681		return 1;682 683	if (!pkt_stream->verbatim) {684		if (!pkt->valid || !pkt->len)685			return 1;686		return ceil_u32(pkt->len, frame_size);687	}688 689	/* Search for the end of the packet in verbatim mode */690	if (!pkt_continues(pkt->options))691		return nb_frags;692 693	next_frag = pkt_stream->current_pkt_nb;694	pkt++;695	while (next_frag++ < pkt_stream->nb_pkts) {696		nb_frags++;697		if (!pkt_continues(pkt->options) || !pkt->valid)698			break;699		pkt++;700	}701	return nb_frags;702}703 704static bool set_pkt_valid(int offset, u32 len)705{706	return len <= MAX_ETH_JUMBO_SIZE;707}708 709static void pkt_set(struct pkt_stream *pkt_stream, struct pkt *pkt, int offset, u32 len)710{711	pkt->offset = offset;712	pkt->len = len;713	pkt->valid = set_pkt_valid(offset, len);714}715 716static void pkt_stream_pkt_set(struct pkt_stream *pkt_stream, struct pkt *pkt, int offset, u32 len)717{718	bool prev_pkt_valid = pkt->valid;719 720	pkt_set(pkt_stream, pkt, offset, len);721	pkt_stream->nb_valid_entries += pkt->valid - prev_pkt_valid;722}723 724static u32 pkt_get_buffer_len(struct xsk_umem_info *umem, u32 len)725{726	return ceil_u32(len, umem->frame_size) * umem->frame_size;727}728 729static struct pkt_stream *__pkt_stream_generate(u32 nb_pkts, u32 pkt_len, u32 nb_start, u32 nb_off)730{731	struct pkt_stream *pkt_stream;732	u32 i;733 734	pkt_stream = __pkt_stream_alloc(nb_pkts);735	if (!pkt_stream)736		exit_with_error(ENOMEM);737 738	pkt_stream->nb_pkts = nb_pkts;739	pkt_stream->max_pkt_len = pkt_len;740	for (i = 0; i < nb_pkts; i++) {741		struct pkt *pkt = &pkt_stream->pkts[i];742 743		pkt_stream_pkt_set(pkt_stream, pkt, 0, pkt_len);744		pkt->pkt_nb = nb_start + i * nb_off;745	}746 747	return pkt_stream;748}749 750static struct pkt_stream *pkt_stream_generate(u32 nb_pkts, u32 pkt_len)751{752	return __pkt_stream_generate(nb_pkts, pkt_len, 0, 1);753}754 755static struct pkt_stream *pkt_stream_clone(struct pkt_stream *pkt_stream)756{757	return pkt_stream_generate(pkt_stream->nb_pkts, pkt_stream->pkts[0].len);758}759 760static void pkt_stream_replace(struct test_spec *test, u32 nb_pkts, u32 pkt_len)761{762	struct pkt_stream *pkt_stream;763 764	pkt_stream = pkt_stream_generate(nb_pkts, pkt_len);765	test->ifobj_tx->xsk->pkt_stream = pkt_stream;766	pkt_stream = pkt_stream_generate(nb_pkts, pkt_len);767	test->ifobj_rx->xsk->pkt_stream = pkt_stream;768}769 770static void __pkt_stream_replace_half(struct ifobject *ifobj, u32 pkt_len,771				      int offset)772{773	struct pkt_stream *pkt_stream;774	u32 i;775 776	pkt_stream = pkt_stream_clone(ifobj->xsk->pkt_stream);777	for (i = 1; i < ifobj->xsk->pkt_stream->nb_pkts; i += 2)778		pkt_stream_pkt_set(pkt_stream, &pkt_stream->pkts[i], offset, pkt_len);779 780	ifobj->xsk->pkt_stream = pkt_stream;781}782 783static void pkt_stream_replace_half(struct test_spec *test, u32 pkt_len, int offset)784{785	__pkt_stream_replace_half(test->ifobj_tx, pkt_len, offset);786	__pkt_stream_replace_half(test->ifobj_rx, pkt_len, offset);787}788 789static void pkt_stream_receive_half(struct test_spec *test)790{791	struct pkt_stream *pkt_stream = test->ifobj_tx->xsk->pkt_stream;792	u32 i;793 794	test->ifobj_rx->xsk->pkt_stream = pkt_stream_generate(pkt_stream->nb_pkts,795							      pkt_stream->pkts[0].len);796	pkt_stream = test->ifobj_rx->xsk->pkt_stream;797	for (i = 1; i < pkt_stream->nb_pkts; i += 2)798		pkt_stream->pkts[i].valid = false;799 800	pkt_stream->nb_valid_entries /= 2;801}802 803static void pkt_stream_even_odd_sequence(struct test_spec *test)804{805	struct pkt_stream *pkt_stream;806	u32 i;807 808	for (i = 0; i < test->nb_sockets; i++) {809		pkt_stream = test->ifobj_tx->xsk_arr[i].pkt_stream;810		pkt_stream = __pkt_stream_generate(pkt_stream->nb_pkts / 2,811						   pkt_stream->pkts[0].len, i, 2);812		test->ifobj_tx->xsk_arr[i].pkt_stream = pkt_stream;813 814		pkt_stream = test->ifobj_rx->xsk_arr[i].pkt_stream;815		pkt_stream = __pkt_stream_generate(pkt_stream->nb_pkts / 2,816						   pkt_stream->pkts[0].len, i, 2);817		test->ifobj_rx->xsk_arr[i].pkt_stream = pkt_stream;818	}819}820 821static u64 pkt_get_addr(struct pkt *pkt, struct xsk_umem_info *umem)822{823	if (!pkt->valid)824		return pkt->offset;825	return pkt->offset + umem_alloc_buffer(umem);826}827 828static void pkt_stream_cancel(struct pkt_stream *pkt_stream)829{830	pkt_stream->current_pkt_nb--;831}832 833static void pkt_generate(struct xsk_socket_info *xsk, struct xsk_umem_info *umem, u64 addr, u32 len,834			 u32 pkt_nb, u32 bytes_written)835{836	void *data = xsk_umem__get_data(umem->buffer, addr);837 838	if (len < MIN_PKT_SIZE)839		return;840 841	if (!bytes_written) {842		gen_eth_hdr(xsk, data);843 844		len -= PKT_HDR_SIZE;845		data += PKT_HDR_SIZE;846	} else {847		bytes_written -= PKT_HDR_SIZE;848	}849 850	write_payload(data, pkt_nb, bytes_written, len);851}852 853static struct pkt_stream *__pkt_stream_generate_custom(struct ifobject *ifobj, struct pkt *frames,854						       u32 nb_frames, bool verbatim)855{856	u32 i, len = 0, pkt_nb = 0, payload = 0;857	struct pkt_stream *pkt_stream;858 859	pkt_stream = __pkt_stream_alloc(nb_frames);860	if (!pkt_stream)861		exit_with_error(ENOMEM);862 863	for (i = 0; i < nb_frames; i++) {864		struct pkt *pkt = &pkt_stream->pkts[pkt_nb];865		struct pkt *frame = &frames[i];866 867		pkt->offset = frame->offset;868		if (verbatim) {869			*pkt = *frame;870			pkt->pkt_nb = payload;871			if (!frame->valid || !pkt_continues(frame->options))872				payload++;873		} else {874			if (frame->valid)875				len += frame->len;876			if (frame->valid && pkt_continues(frame->options))877				continue;878 879			pkt->pkt_nb = pkt_nb;880			pkt->len = len;881			pkt->valid = frame->valid;882			pkt->options = 0;883 884			len = 0;885		}886 887		print_verbose("offset: %d len: %u valid: %u options: %u pkt_nb: %u\n",888			      pkt->offset, pkt->len, pkt->valid, pkt->options, pkt->pkt_nb);889 890		if (pkt->valid && pkt->len > pkt_stream->max_pkt_len)891			pkt_stream->max_pkt_len = pkt->len;892 893		if (pkt->valid)894			pkt_stream->nb_valid_entries++;895 896		pkt_nb++;897	}898 899	pkt_stream->nb_pkts = pkt_nb;900	pkt_stream->verbatim = verbatim;901	return pkt_stream;902}903 904static void pkt_stream_generate_custom(struct test_spec *test, struct pkt *pkts, u32 nb_pkts)905{906	struct pkt_stream *pkt_stream;907 908	pkt_stream = __pkt_stream_generate_custom(test->ifobj_tx, pkts, nb_pkts, true);909	test->ifobj_tx->xsk->pkt_stream = pkt_stream;910 911	pkt_stream = __pkt_stream_generate_custom(test->ifobj_rx, pkts, nb_pkts, false);912	test->ifobj_rx->xsk->pkt_stream = pkt_stream;913}914 915static void pkt_print_data(u32 *data, u32 cnt)916{917	u32 i;918 919	for (i = 0; i < cnt; i++) {920		u32 seqnum, pkt_nb;921 922		seqnum = ntohl(*data) & 0xffff;923		pkt_nb = ntohl(*data) >> 16;924		ksft_print_msg("%u:%u ", pkt_nb, seqnum);925		data++;926	}927}928 929static void pkt_dump(void *pkt, u32 len, bool eth_header)930{931	struct ethhdr *ethhdr = pkt;932	u32 i, *data;933 934	if (eth_header) {935		/*extract L2 frame */936		ksft_print_msg("DEBUG>> L2: dst mac: ");937		for (i = 0; i < ETH_ALEN; i++)938			ksft_print_msg("%02X", ethhdr->h_dest[i]);939 940		ksft_print_msg("\nDEBUG>> L2: src mac: ");941		for (i = 0; i < ETH_ALEN; i++)942			ksft_print_msg("%02X", ethhdr->h_source[i]);943 944		data = pkt + PKT_HDR_SIZE;945	} else {946		data = pkt;947	}948 949	/*extract L5 frame */950	ksft_print_msg("\nDEBUG>> L5: seqnum: ");951	pkt_print_data(data, PKT_DUMP_NB_TO_PRINT);952	ksft_print_msg("....");953	if (len > PKT_DUMP_NB_TO_PRINT * sizeof(u32)) {954		ksft_print_msg("\n.... ");955		pkt_print_data(data + len / sizeof(u32) - PKT_DUMP_NB_TO_PRINT,956			       PKT_DUMP_NB_TO_PRINT);957	}958	ksft_print_msg("\n---------------------------------------\n");959}960 961static bool is_offset_correct(struct xsk_umem_info *umem, struct pkt *pkt, u64 addr)962{963	u32 headroom = umem->unaligned_mode ? 0 : umem->frame_headroom;964	u32 offset = addr % umem->frame_size, expected_offset;965	int pkt_offset = pkt->valid ? pkt->offset : 0;966 967	if (!umem->unaligned_mode)968		pkt_offset = 0;969 970	expected_offset = (pkt_offset + headroom + XDP_PACKET_HEADROOM) % umem->frame_size;971 972	if (offset == expected_offset)973		return true;974 975	ksft_print_msg("[%s] expected [%u], got [%u]\n", __func__, expected_offset, offset);976	return false;977}978 979static bool is_metadata_correct(struct pkt *pkt, void *buffer, u64 addr)980{981	void *data = xsk_umem__get_data(buffer, addr);982	struct xdp_info *meta = data - sizeof(struct xdp_info);983 984	if (meta->count != pkt->pkt_nb) {985		ksft_print_msg("[%s] expected meta_count [%d], got meta_count [%llu]\n",986			       __func__, pkt->pkt_nb,987			       (unsigned long long)meta->count);988		return false;989	}990 991	return true;992}993 994static bool is_frag_valid(struct xsk_umem_info *umem, u64 addr, u32 len, u32 expected_pkt_nb,995			  u32 bytes_processed)996{997	u32 seqnum, pkt_nb, *pkt_data, words_to_end, expected_seqnum;998	void *data = xsk_umem__get_data(umem->buffer, addr);999 1000	addr -= umem->base_addr;1001 1002	if (addr >= umem->num_frames * umem->frame_size ||1003	    addr + len > umem->num_frames * umem->frame_size) {1004		ksft_print_msg("Frag invalid addr: %llx len: %u\n",1005			       (unsigned long long)addr, len);1006		return false;1007	}1008	if (!umem->unaligned_mode && addr % umem->frame_size + len > umem->frame_size) {1009		ksft_print_msg("Frag crosses frame boundary addr: %llx len: %u\n",1010			       (unsigned long long)addr, len);1011		return false;1012	}1013 1014	pkt_data = data;1015	if (!bytes_processed) {1016		pkt_data += PKT_HDR_SIZE / sizeof(*pkt_data);1017		len -= PKT_HDR_SIZE;1018	} else {1019		bytes_processed -= PKT_HDR_SIZE;1020	}1021 1022	expected_seqnum = bytes_processed / sizeof(*pkt_data);1023	seqnum = ntohl(*pkt_data) & 0xffff;1024	pkt_nb = ntohl(*pkt_data) >> 16;1025 1026	if (expected_pkt_nb != pkt_nb) {1027		ksft_print_msg("[%s] expected pkt_nb [%u], got pkt_nb [%u]\n",1028			       __func__, expected_pkt_nb, pkt_nb);1029		goto error;1030	}1031	if (expected_seqnum != seqnum) {1032		ksft_print_msg("[%s] expected seqnum at start [%u], got seqnum [%u]\n",1033			       __func__, expected_seqnum, seqnum);1034		goto error;1035	}1036 1037	words_to_end = len / sizeof(*pkt_data) - 1;1038	pkt_data += words_to_end;1039	seqnum = ntohl(*pkt_data) & 0xffff;1040	expected_seqnum += words_to_end;1041	if (expected_seqnum != seqnum) {1042		ksft_print_msg("[%s] expected seqnum at end [%u], got seqnum [%u]\n",1043			       __func__, expected_seqnum, seqnum);1044		goto error;1045	}1046 1047	return true;1048 1049error:1050	pkt_dump(data, len, !bytes_processed);1051	return false;1052}1053 1054static bool is_pkt_valid(struct pkt *pkt, void *buffer, u64 addr, u32 len)1055{1056	if (pkt->len != len) {1057		ksft_print_msg("[%s] expected packet length [%d], got length [%d]\n",1058			       __func__, pkt->len, len);1059		pkt_dump(xsk_umem__get_data(buffer, addr), len, true);1060		return false;1061	}1062 1063	return true;1064}1065 1066static int kick_tx(struct xsk_socket_info *xsk)1067{1068	int ret;1069 1070	ret = sendto(xsk_socket__fd(xsk->xsk), NULL, 0, MSG_DONTWAIT, NULL, 0);1071	if (ret >= 0)1072		return TEST_PASS;1073	if (errno == ENOBUFS || errno == EAGAIN || errno == EBUSY || errno == ENETDOWN) {1074		usleep(100);1075		return TEST_PASS;1076	}1077	return TEST_FAILURE;1078}1079 1080static int kick_rx(struct xsk_socket_info *xsk)1081{1082	int ret;1083 1084	ret = recvfrom(xsk_socket__fd(xsk->xsk), NULL, 0, MSG_DONTWAIT, NULL, NULL);1085	if (ret < 0)1086		return TEST_FAILURE;1087 1088	return TEST_PASS;1089}1090 1091static int complete_pkts(struct xsk_socket_info *xsk, int batch_size)1092{1093	unsigned int rcvd;1094	u32 idx;1095	int ret;1096 1097	if (xsk_ring_prod__needs_wakeup(&xsk->tx)) {1098		ret = kick_tx(xsk);1099		if (ret)1100			return TEST_FAILURE;1101	}1102 1103	rcvd = xsk_ring_cons__peek(&xsk->umem->cq, batch_size, &idx);1104	if (rcvd) {1105		if (rcvd > xsk->outstanding_tx) {1106			u64 addr = *xsk_ring_cons__comp_addr(&xsk->umem->cq, idx + rcvd - 1);1107 1108			ksft_print_msg("[%s] Too many packets completed\n", __func__);1109			ksft_print_msg("Last completion address: %llx\n",1110				       (unsigned long long)addr);1111			return TEST_FAILURE;1112		}1113 1114		xsk_ring_cons__release(&xsk->umem->cq, rcvd);1115		xsk->outstanding_tx -= rcvd;1116	}1117 1118	return TEST_PASS;1119}1120 1121static int __receive_pkts(struct test_spec *test, struct xsk_socket_info *xsk)1122{1123	u32 frags_processed = 0, nb_frags = 0, pkt_len = 0;1124	u32 idx_rx = 0, idx_fq = 0, rcvd, pkts_sent = 0;1125	struct pkt_stream *pkt_stream = xsk->pkt_stream;1126	struct ifobject *ifobj = test->ifobj_rx;1127	struct xsk_umem_info *umem = xsk->umem;1128	struct pollfd fds = { };1129	struct pkt *pkt;1130	u64 first_addr = 0;1131	int ret;1132 1133	fds.fd = xsk_socket__fd(xsk->xsk);1134	fds.events = POLLIN;1135 1136	ret = kick_rx(xsk);1137	if (ret)1138		return TEST_FAILURE;1139 1140	if (ifobj->use_poll) {1141		ret = poll(&fds, 1, POLL_TMOUT);1142		if (ret < 0)1143			return TEST_FAILURE;1144 1145		if (!ret) {1146			if (!is_umem_valid(test->ifobj_tx))1147				return TEST_PASS;1148 1149			ksft_print_msg("ERROR: [%s] Poll timed out\n", __func__);1150			return TEST_CONTINUE;1151		}1152 1153		if (!(fds.revents & POLLIN))1154			return TEST_CONTINUE;1155	}1156 1157	rcvd = xsk_ring_cons__peek(&xsk->rx, xsk->batch_size, &idx_rx);1158	if (!rcvd)1159		return TEST_CONTINUE;1160 1161	if (ifobj->use_fill_ring) {1162		ret = xsk_ring_prod__reserve(&umem->fq, rcvd, &idx_fq);1163		while (ret != rcvd) {1164			if (xsk_ring_prod__needs_wakeup(&umem->fq)) {1165				ret = poll(&fds, 1, POLL_TMOUT);1166				if (ret < 0)1167					return TEST_FAILURE;1168			}1169			ret = xsk_ring_prod__reserve(&umem->fq, rcvd, &idx_fq);1170		}1171	}1172 1173	while (frags_processed < rcvd) {1174		const struct xdp_desc *desc = xsk_ring_cons__rx_desc(&xsk->rx, idx_rx++);1175		u64 addr = desc->addr, orig;1176 1177		orig = xsk_umem__extract_addr(addr);1178		addr = xsk_umem__add_offset_to_addr(addr);1179 1180		if (!nb_frags) {1181			pkt = pkt_stream_get_next_rx_pkt(pkt_stream, &pkts_sent);1182			if (!pkt) {1183				ksft_print_msg("[%s] received too many packets addr: %lx len %u\n",1184					       __func__, addr, desc->len);1185				return TEST_FAILURE;1186			}1187		}1188 1189		print_verbose("Rx: addr: %lx len: %u options: %u pkt_nb: %u valid: %u\n",1190			      addr, desc->len, desc->options, pkt->pkt_nb, pkt->valid);1191 1192		if (!is_frag_valid(umem, addr, desc->len, pkt->pkt_nb, pkt_len) ||1193		    !is_offset_correct(umem, pkt, addr) || (ifobj->use_metadata &&1194		    !is_metadata_correct(pkt, umem->buffer, addr)))1195			return TEST_FAILURE;1196 1197		if (!nb_frags++)1198			first_addr = addr;1199		frags_processed++;1200		pkt_len += desc->len;1201		if (ifobj->use_fill_ring)1202			*xsk_ring_prod__fill_addr(&umem->fq, idx_fq++) = orig;1203 1204		if (pkt_continues(desc->options))1205			continue;1206 1207		/* The complete packet has been received */1208		if (!is_pkt_valid(pkt, umem->buffer, first_addr, pkt_len) ||1209		    !is_offset_correct(umem, pkt, addr))1210			return TEST_FAILURE;1211 1212		pkt_stream->nb_rx_pkts++;1213		nb_frags = 0;1214		pkt_len = 0;1215	}1216 1217	if (nb_frags) {1218		/* In the middle of a packet. Start over from beginning of packet. */1219		idx_rx -= nb_frags;1220		xsk_ring_cons__cancel(&xsk->rx, nb_frags);1221		if (ifobj->use_fill_ring) {1222			idx_fq -= nb_frags;1223			xsk_ring_prod__cancel(&umem->fq, nb_frags);1224		}1225		frags_processed -= nb_frags;1226	}1227 1228	if (ifobj->use_fill_ring)1229		xsk_ring_prod__submit(&umem->fq, frags_processed);1230	if (ifobj->release_rx)1231		xsk_ring_cons__release(&xsk->rx, frags_processed);1232 1233	pthread_mutex_lock(&pacing_mutex);1234	pkts_in_flight -= pkts_sent;1235	pthread_mutex_unlock(&pacing_mutex);1236	pkts_sent = 0;1237 1238return TEST_CONTINUE;1239}1240 1241bool all_packets_received(struct test_spec *test, struct xsk_socket_info *xsk, u32 sock_num,1242			  unsigned long *bitmap)1243{1244	struct pkt_stream *pkt_stream = xsk->pkt_stream;1245 1246	if (!pkt_stream) {1247		__set_bit(sock_num, bitmap);1248		return false;1249	}1250 1251	if (pkt_stream->nb_rx_pkts == pkt_stream->nb_valid_entries) {1252		__set_bit(sock_num, bitmap);1253		if (bitmap_full(bitmap, test->nb_sockets))1254			return true;1255	}1256 1257	return false;1258}1259 1260static int receive_pkts(struct test_spec *test)1261{1262	struct timeval tv_end, tv_now, tv_timeout = {THREAD_TMOUT, 0};1263	DECLARE_BITMAP(bitmap, test->nb_sockets);1264	struct xsk_socket_info *xsk;1265	u32 sock_num = 0;1266	int res, ret;1267 1268	ret = gettimeofday(&tv_now, NULL);1269	if (ret)1270		exit_with_error(errno);1271 1272	timeradd(&tv_now, &tv_timeout, &tv_end);1273 1274	while (1) {1275		xsk = &test->ifobj_rx->xsk_arr[sock_num];1276 1277		if ((all_packets_received(test, xsk, sock_num, bitmap)))1278			break;1279 1280		res = __receive_pkts(test, xsk);1281		if (!(res == TEST_PASS || res == TEST_CONTINUE))1282			return res;1283 1284		ret = gettimeofday(&tv_now, NULL);1285		if (ret)1286			exit_with_error(errno);1287 1288		if (timercmp(&tv_now, &tv_end, >)) {1289			ksft_print_msg("ERROR: [%s] Receive loop timed out\n", __func__);1290			return TEST_FAILURE;1291		}1292		sock_num = (sock_num + 1) % test->nb_sockets;1293	}1294 1295	return TEST_PASS;1296}1297 1298static int __send_pkts(struct ifobject *ifobject, struct xsk_socket_info *xsk, bool timeout)1299{1300	u32 i, idx = 0, valid_pkts = 0, valid_frags = 0, buffer_len;1301	struct pkt_stream *pkt_stream = xsk->pkt_stream;1302	struct xsk_umem_info *umem = ifobject->umem;1303	bool use_poll = ifobject->use_poll;1304	struct pollfd fds = { };1305	int ret;1306 1307	buffer_len = pkt_get_buffer_len(umem, pkt_stream->max_pkt_len);1308	/* pkts_in_flight might be negative if many invalid packets are sent */1309	if (pkts_in_flight >= (int)((umem_size(umem) - xsk->batch_size * buffer_len) /1310	    buffer_len)) {1311		ret = kick_tx(xsk);1312		if (ret)1313			return TEST_FAILURE;1314		return TEST_CONTINUE;1315	}1316 1317	fds.fd = xsk_socket__fd(xsk->xsk);1318	fds.events = POLLOUT;1319 1320	while (xsk_ring_prod__reserve(&xsk->tx, xsk->batch_size, &idx) < xsk->batch_size) {1321		if (use_poll) {1322			ret = poll(&fds, 1, POLL_TMOUT);1323			if (timeout) {1324				if (ret < 0) {1325					ksft_print_msg("ERROR: [%s] Poll error %d\n",1326						       __func__, errno);1327					return TEST_FAILURE;1328				}1329				if (ret == 0)1330					return TEST_PASS;1331				break;1332			}1333			if (ret <= 0) {1334				ksft_print_msg("ERROR: [%s] Poll error %d\n",1335					       __func__, errno);1336				return TEST_FAILURE;1337			}1338		}1339 1340		complete_pkts(xsk, xsk->batch_size);1341	}1342 1343	for (i = 0; i < xsk->batch_size; i++) {1344		struct pkt *pkt = pkt_stream_get_next_tx_pkt(pkt_stream);1345		u32 nb_frags_left, nb_frags, bytes_written = 0;1346 1347		if (!pkt)1348			break;1349 1350		nb_frags = pkt_nb_frags(umem->frame_size, pkt_stream, pkt);1351		if (nb_frags > xsk->batch_size - i) {1352			pkt_stream_cancel(pkt_stream);1353			xsk_ring_prod__cancel(&xsk->tx, xsk->batch_size - i);1354			break;1355		}1356		nb_frags_left = nb_frags;1357 1358		while (nb_frags_left--) {1359			struct xdp_desc *tx_desc = xsk_ring_prod__tx_desc(&xsk->tx, idx + i);1360 1361			tx_desc->addr = pkt_get_addr(pkt, ifobject->umem);1362			if (pkt_stream->verbatim) {1363				tx_desc->len = pkt->len;1364				tx_desc->options = pkt->options;1365			} else if (nb_frags_left) {1366				tx_desc->len = umem->frame_size;1367				tx_desc->options = XDP_PKT_CONTD;1368			} else {1369				tx_desc->len = pkt->len - bytes_written;1370				tx_desc->options = 0;1371			}1372			if (pkt->valid)1373				pkt_generate(xsk, umem, tx_desc->addr, tx_desc->len, pkt->pkt_nb,1374					     bytes_written);1375			bytes_written += tx_desc->len;1376 1377			print_verbose("Tx addr: %llx len: %u options: %u pkt_nb: %u\n",1378				      tx_desc->addr, tx_desc->len, tx_desc->options, pkt->pkt_nb);1379 1380			if (nb_frags_left) {1381				i++;1382				if (pkt_stream->verbatim)1383					pkt = pkt_stream_get_next_tx_pkt(pkt_stream);1384			}1385		}1386 1387		if (pkt && pkt->valid) {1388			valid_pkts++;1389			valid_frags += nb_frags;1390		}1391	}1392 1393	pthread_mutex_lock(&pacing_mutex);1394	pkts_in_flight += valid_pkts;1395	pthread_mutex_unlock(&pacing_mutex);1396 1397	xsk_ring_prod__submit(&xsk->tx, i);1398	xsk->outstanding_tx += valid_frags;1399 1400	if (use_poll) {1401		ret = poll(&fds, 1, POLL_TMOUT);1402		if (ret <= 0) {1403			if (ret == 0 && timeout)1404				return TEST_PASS;1405 1406			ksft_print_msg("ERROR: [%s] Poll error %d\n", __func__, ret);1407			return TEST_FAILURE;1408		}1409	}1410 1411	if (!timeout) {1412		if (complete_pkts(xsk, i))1413			return TEST_FAILURE;1414 1415		usleep(10);1416		return TEST_PASS;1417	}1418 1419	return TEST_CONTINUE;1420}1421 1422static int wait_for_tx_completion(struct xsk_socket_info *xsk)1423{1424	struct timeval tv_end, tv_now, tv_timeout = {THREAD_TMOUT, 0};1425	int ret;1426 1427	ret = gettimeofday(&tv_now, NULL);1428	if (ret)1429		exit_with_error(errno);1430	timeradd(&tv_now, &tv_timeout, &tv_end);1431 1432	while (xsk->outstanding_tx) {1433		ret = gettimeofday(&tv_now, NULL);1434		if (ret)1435			exit_with_error(errno);1436		if (timercmp(&tv_now, &tv_end, >)) {1437			ksft_print_msg("ERROR: [%s] Transmission loop timed out\n", __func__);1438			return TEST_FAILURE;1439		}1440 1441		complete_pkts(xsk, xsk->batch_size);1442	}1443 1444	return TEST_PASS;1445}1446 1447bool all_packets_sent(struct test_spec *test, unsigned long *bitmap)1448{1449	return bitmap_full(bitmap, test->nb_sockets);1450}1451 1452static int send_pkts(struct test_spec *test, struct ifobject *ifobject)1453{1454	bool timeout = !is_umem_valid(test->ifobj_rx);1455	DECLARE_BITMAP(bitmap, test->nb_sockets);1456	u32 i, ret;1457 1458	while (!(all_packets_sent(test, bitmap))) {1459		for (i = 0; i < test->nb_sockets; i++) {1460			struct pkt_stream *pkt_stream;1461 1462			pkt_stream = ifobject->xsk_arr[i].pkt_stream;1463			if (!pkt_stream || pkt_stream->current_pkt_nb >= pkt_stream->nb_pkts) {1464				__set_bit(i, bitmap);1465				continue;1466			}1467			ret = __send_pkts(ifobject, &ifobject->xsk_arr[i], timeout);1468			if (ret == TEST_CONTINUE && !test->fail)1469				continue;1470 1471			if ((ret || test->fail) && !timeout)1472				return TEST_FAILURE;1473 1474			if (ret == TEST_PASS && timeout)1475				return ret;1476 1477			ret = wait_for_tx_completion(&ifobject->xsk_arr[i]);1478			if (ret)1479				return TEST_FAILURE;1480		}1481	}1482 1483	return TEST_PASS;1484}1485 1486static int get_xsk_stats(struct xsk_socket *xsk, struct xdp_statistics *stats)1487{1488	int fd = xsk_socket__fd(xsk), err;1489	socklen_t optlen, expected_len;1490 1491	optlen = sizeof(*stats);1492	err = getsockopt(fd, SOL_XDP, XDP_STATISTICS, stats, &optlen);1493	if (err) {1494		ksft_print_msg("[%s] getsockopt(XDP_STATISTICS) error %u %s\n",1495			       __func__, -err, strerror(-err));1496		return TEST_FAILURE;1497	}1498 1499	expected_len = sizeof(struct xdp_statistics);1500	if (optlen != expected_len) {1501		ksft_print_msg("[%s] getsockopt optlen error. Expected: %u got: %u\n",1502			       __func__, expected_len, optlen);1503		return TEST_FAILURE;1504	}1505 1506	return TEST_PASS;1507}1508 1509static int validate_rx_dropped(struct ifobject *ifobject)1510{1511	struct xsk_socket *xsk = ifobject->xsk->xsk;1512	struct xdp_statistics stats;1513	int err;1514 1515	err = kick_rx(ifobject->xsk);1516	if (err)1517		return TEST_FAILURE;1518 1519	err = get_xsk_stats(xsk, &stats);1520	if (err)1521		return TEST_FAILURE;1522 1523	/* The receiver calls getsockopt after receiving the last (valid)1524	 * packet which is not the final packet sent in this test (valid and1525	 * invalid packets are sent in alternating fashion with the final1526	 * packet being invalid). Since the last packet may or may not have1527	 * been dropped already, both outcomes must be allowed.1528	 */1529	if (stats.rx_dropped == ifobject->xsk->pkt_stream->nb_pkts / 2 ||1530	    stats.rx_dropped == ifobject->xsk->pkt_stream->nb_pkts / 2 - 1)1531		return TEST_PASS;1532 1533	return TEST_FAILURE;1534}1535 1536static int validate_rx_full(struct ifobject *ifobject)1537{1538	struct xsk_socket *xsk = ifobject->xsk->xsk;1539	struct xdp_statistics stats;1540	int err;1541 1542	usleep(1000);1543	err = kick_rx(ifobject->xsk);1544	if (err)1545		return TEST_FAILURE;1546 1547	err = get_xsk_stats(xsk, &stats);1548	if (err)1549		return TEST_FAILURE;1550 1551	if (stats.rx_ring_full)1552		return TEST_PASS;1553 1554	return TEST_FAILURE;1555}1556 1557static int validate_fill_empty(struct ifobject *ifobject)1558{1559	struct xsk_socket *xsk = ifobject->xsk->xsk;1560	struct xdp_statistics stats;1561	int err;1562 1563	usleep(1000);1564	err = kick_rx(ifobject->xsk);1565	if (err)1566		return TEST_FAILURE;1567 1568	err = get_xsk_stats(xsk, &stats);1569	if (err)1570		return TEST_FAILURE;1571 1572	if (stats.rx_fill_ring_empty_descs)1573		return TEST_PASS;1574 1575	return TEST_FAILURE;1576}1577 1578static int validate_tx_invalid_descs(struct ifobject *ifobject)1579{1580	struct xsk_socket *xsk = ifobject->xsk->xsk;1581	int fd = xsk_socket__fd(xsk);1582	struct xdp_statistics stats;1583	socklen_t optlen;1584	int err;1585 1586	optlen = sizeof(stats);1587	err = getsockopt(fd, SOL_XDP, XDP_STATISTICS, &stats, &optlen);1588	if (err) {1589		ksft_print_msg("[%s] getsockopt(XDP_STATISTICS) error %u %s\n",1590			       __func__, -err, strerror(-err));1591		return TEST_FAILURE;1592	}1593 1594	if (stats.tx_invalid_descs != ifobject->xsk->pkt_stream->nb_pkts / 2) {1595		ksft_print_msg("[%s] tx_invalid_descs incorrect. Got [%llu] expected [%u]\n",1596			       __func__,1597			       (unsigned long long)stats.tx_invalid_descs,1598			       ifobject->xsk->pkt_stream->nb_pkts);1599		return TEST_FAILURE;1600	}1601 1602	return TEST_PASS;1603}1604 1605static void xsk_configure_socket(struct test_spec *test, struct ifobject *ifobject,1606				 struct xsk_umem_info *umem, bool tx)1607{1608	int i, ret;1609 1610	for (i = 0; i < test->nb_sockets; i++) {1611		bool shared = (ifobject->shared_umem && tx) ? true : !!i;1612		u32 ctr = 0;1613 1614		while (ctr++ < SOCK_RECONF_CTR) {1615			ret = __xsk_configure_socket(&ifobject->xsk_arr[i], umem,1616						     ifobject, shared);1617			if (!ret)1618				break;1619 1620			/* Retry if it fails as xsk_socket__create() is asynchronous */1621			if (ctr >= SOCK_RECONF_CTR)1622				exit_with_error(-ret);1623			usleep(USLEEP_MAX);1624		}1625		if (ifobject->busy_poll)1626			enable_busy_poll(&ifobject->xsk_arr[i]);1627	}1628}1629 1630static void thread_common_ops_tx(struct test_spec *test, struct ifobject *ifobject)1631{1632	xsk_configure_socket(test, ifobject, test->ifobj_rx->umem, true);1633	ifobject->xsk = &ifobject->xsk_arr[0];1634	ifobject->xskmap = test->ifobj_rx->xskmap;1635	memcpy(ifobject->umem, test->ifobj_rx->umem, sizeof(struct xsk_umem_info));1636	ifobject->umem->base_addr = 0;1637}1638 1639static void xsk_populate_fill_ring(struct xsk_umem_info *umem, struct pkt_stream *pkt_stream,1640				   bool fill_up)1641{1642	u32 rx_frame_size = umem->frame_size - XDP_PACKET_HEADROOM;1643	u32 idx = 0, filled = 0, buffers_to_fill, nb_pkts;1644	int ret;1645 1646	if (umem->num_frames < XSK_RING_PROD__DEFAULT_NUM_DESCS)1647		buffers_to_fill = umem->num_frames;1648	else1649		buffers_to_fill = umem->fill_size;1650 1651	ret = xsk_ring_prod__reserve(&umem->fq, buffers_to_fill, &idx);1652	if (ret != buffers_to_fill)1653		exit_with_error(ENOSPC);1654 1655	while (filled < buffers_to_fill) {1656		struct pkt *pkt = pkt_stream_get_next_rx_pkt(pkt_stream, &nb_pkts);1657		u64 addr;1658		u32 i;1659 1660		for (i = 0; i < pkt_nb_frags(rx_frame_size, pkt_stream, pkt); i++) {1661			if (!pkt) {1662				if (!fill_up)1663					break;1664				addr = filled * umem->frame_size + umem->base_addr;1665			} else if (pkt->offset >= 0) {1666				addr = pkt->offset % umem->frame_size + umem_alloc_buffer(umem);1667			} else {1668				addr = pkt->offset + umem_alloc_buffer(umem);1669			}1670 1671			*xsk_ring_prod__fill_addr(&umem->fq, idx++) = addr;1672			if (++filled >= buffers_to_fill)1673				break;1674		}1675	}1676	xsk_ring_prod__submit(&umem->fq, filled);1677	xsk_ring_prod__cancel(&umem->fq, buffers_to_fill - filled);1678 1679	pkt_stream_reset(pkt_stream);1680	umem_reset_alloc(umem);1681}1682 1683static void thread_common_ops(struct test_spec *test, struct ifobject *ifobject)1684{1685	u64 umem_sz = ifobject->umem->num_frames * ifobject->umem->frame_size;1686	int mmap_flags = MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE;1687	LIBBPF_OPTS(bpf_xdp_query_opts, opts);1688	void *bufs;1689	int ret;1690	u32 i;1691 1692	if (ifobject->umem->unaligned_mode)1693		mmap_flags |= MAP_HUGETLB | MAP_HUGE_2MB;1694 1695	if (ifobject->shared_umem)1696		umem_sz *= 2;1697 1698	bufs = mmap(NULL, umem_sz, PROT_READ | PROT_WRITE, mmap_flags, -1, 0);1699	if (bufs == MAP_FAILED)1700		exit_with_error(errno);1701 1702	ret = xsk_configure_umem(ifobject, ifobject->umem, bufs, umem_sz);1703	if (ret)1704		exit_with_error(-ret);1705 1706	xsk_configure_socket(test, ifobject, ifobject->umem, false);1707 1708	ifobject->xsk = &ifobject->xsk_arr[0];1709 1710	if (!ifobject->rx_on)1711		return;1712 1713	xsk_populate_fill_ring(ifobject->umem, ifobject->xsk->pkt_stream, ifobject->use_fill_ring);1714 1715	for (i = 0; i < test->nb_sockets; i++) {1716		ifobject->xsk = &ifobject->xsk_arr[i];1717		ret = xsk_update_xskmap(ifobject->xskmap, ifobject->xsk->xsk, i);1718		if (ret)1719			exit_with_error(errno);1720	}1721}1722 1723static void *worker_testapp_validate_tx(void *arg)1724{1725	struct test_spec *test = (struct test_spec *)arg;1726	struct ifobject *ifobject = test->ifobj_tx;1727	int err;1728 1729	if (test->current_step == 1) {1730		if (!ifobject->shared_umem)1731			thread_common_ops(test, ifobject);1732		else1733			thread_common_ops_tx(test, ifobject);1734	}1735 1736	err = send_pkts(test, ifobject);1737 1738	if (!err && ifobject->validation_func)1739		err = ifobject->validation_func(ifobject);1740	if (err)1741		report_failure(test);1742 1743	pthread_exit(NULL);1744}1745 1746static void *worker_testapp_validate_rx(void *arg)1747{1748	struct test_spec *test = (struct test_spec *)arg;1749	struct ifobject *ifobject = test->ifobj_rx;1750	int err;1751 1752	if (test->current_step == 1) {1753		thread_common_ops(test, ifobject);1754	} else {1755		xsk_clear_xskmap(ifobject->xskmap);1756		err = xsk_update_xskmap(ifobject->xskmap, ifobject->xsk->xsk, 0);1757		if (err) {1758			ksft_print_msg("Error: Failed to update xskmap, error %s\n",1759				       strerror(-err));1760			exit_with_error(-err);1761		}1762	}1763 1764	pthread_barrier_wait(&barr);1765 1766	err = receive_pkts(test);1767 1768	if (!err && ifobject->validation_func)1769		err = ifobject->validation_func(ifobject);1770	if (err)1771		report_failure(test);1772 1773	pthread_exit(NULL);1774}1775 1776static u64 ceil_u64(u64 a, u64 b)1777{1778	return (a + b - 1) / b;1779}1780 1781static void testapp_clean_xsk_umem(struct ifobject *ifobj)1782{1783	u64 umem_sz = ifobj->umem->num_frames * ifobj->umem->frame_size;1784 1785	if (ifobj->shared_umem)1786		umem_sz *= 2;1787 1788	umem_sz = ceil_u64(umem_sz, HUGEPAGE_SIZE) * HUGEPAGE_SIZE;1789	xsk_umem__delete(ifobj->umem->umem);1790	munmap(ifobj->umem->buffer, umem_sz);1791}1792 1793static void handler(int signum)1794{1795	pthread_exit(NULL);1796}1797 1798static bool xdp_prog_changed_rx(struct test_spec *test)1799{1800	struct ifobject *ifobj = test->ifobj_rx;1801 1802	return ifobj->xdp_prog != test->xdp_prog_rx || ifobj->mode != test->mode;1803}1804 1805static bool xdp_prog_changed_tx(struct test_spec *test)1806{1807	struct ifobject *ifobj = test->ifobj_tx;1808 1809	return ifobj->xdp_prog != test->xdp_prog_tx || ifobj->mode != test->mode;1810}1811 1812static void xsk_reattach_xdp(struct ifobject *ifobj, struct bpf_program *xdp_prog,1813			     struct bpf_map *xskmap, enum test_mode mode)1814{1815	int err;1816 1817	xsk_detach_xdp_program(ifobj->ifindex, mode_to_xdp_flags(ifobj->mode));1818	err = xsk_attach_xdp_program(xdp_prog, ifobj->ifindex, mode_to_xdp_flags(mode));1819	if (err) {1820		ksft_print_msg("Error attaching XDP program\n");1821		exit_with_error(-err);1822	}1823 1824	if (ifobj->mode != mode && (mode == TEST_MODE_DRV || mode == TEST_MODE_ZC))1825		if (!xsk_is_in_mode(ifobj->ifindex, XDP_FLAGS_DRV_MODE)) {1826			ksft_print_msg("ERROR: XDP prog not in DRV mode\n");1827			exit_with_error(EINVAL);1828		}1829 1830	ifobj->xdp_prog = xdp_prog;1831	ifobj->xskmap = xskmap;1832	ifobj->mode = mode;1833}1834 1835static void xsk_attach_xdp_progs(struct test_spec *test, struct ifobject *ifobj_rx,1836				 struct ifobject *ifobj_tx)1837{1838	if (xdp_prog_changed_rx(test))1839		xsk_reattach_xdp(ifobj_rx, test->xdp_prog_rx, test->xskmap_rx, test->mode);1840 1841	if (!ifobj_tx || ifobj_tx->shared_umem)1842		return;1843 1844	if (xdp_prog_changed_tx(test))1845		xsk_reattach_xdp(ifobj_tx, test->xdp_prog_tx, test->xskmap_tx, test->mode);1846}1847 1848static int __testapp_validate_traffic(struct test_spec *test, struct ifobject *ifobj1,1849				      struct ifobject *ifobj2)1850{1851	pthread_t t0, t1;1852	int err;1853 1854	if (test->mtu > MAX_ETH_PKT_SIZE) {1855		if (test->mode == TEST_MODE_ZC && (!ifobj1->multi_buff_zc_supp ||1856						   (ifobj2 && !ifobj2->multi_buff_zc_supp))) {1857			ksft_test_result_skip("Multi buffer for zero-copy not supported.\n");1858			return TEST_SKIP;1859		}1860		if (test->mode != TEST_MODE_ZC && (!ifobj1->multi_buff_supp ||1861						   (ifobj2 && !ifobj2->multi_buff_supp))) {1862			ksft_test_result_skip("Multi buffer not supported.\n");1863			return TEST_SKIP;1864		}1865	}1866	err = test_spec_set_mtu(test, test->mtu);1867	if (err) {1868		ksft_print_msg("Error, could not set mtu.\n");1869		exit_with_error(err);1870	}1871 1872	if (ifobj2) {1873		if (pthread_barrier_init(&barr, NULL, 2))1874			exit_with_error(errno);1875		pkt_stream_reset(ifobj2->xsk->pkt_stream);1876	}1877 1878	test->current_step++;1879	pkt_stream_reset(ifobj1->xsk->pkt_stream);1880	pkts_in_flight = 0;1881 1882	signal(SIGUSR1, handler);1883	/*Spawn RX thread */1884	pthread_create(&t0, NULL, ifobj1->func_ptr, test);1885 1886	if (ifobj2) {1887		pthread_barrier_wait(&barr);1888		if (pthread_barrier_destroy(&barr))1889			exit_with_error(errno);1890 1891		/*Spawn TX thread */1892		pthread_create(&t1, NULL, ifobj2->func_ptr, test);1893 1894		pthread_join(t1, NULL);1895	}1896 1897	if (!ifobj2)1898		pthread_kill(t0, SIGUSR1);1899	else1900		pthread_join(t0, NULL);1901 1902	if (test->total_steps == test->current_step || test->fail) {1903		u32 i;1904 1905		if (ifobj2)1906			for (i = 0; i < test->nb_sockets; i++)1907				xsk_socket__delete(ifobj2->xsk_arr[i].xsk);1908 1909		for (i = 0; i < test->nb_sockets; i++)1910			xsk_socket__delete(ifobj1->xsk_arr[i].xsk);1911 1912		testapp_clean_xsk_umem(ifobj1);1913		if (ifobj2 && !ifobj2->shared_umem)1914			testapp_clean_xsk_umem(ifobj2);1915	}1916 1917	return !!test->fail;1918}1919 1920static int testapp_validate_traffic(struct test_spec *test)1921{1922	struct ifobject *ifobj_rx = test->ifobj_rx;1923	struct ifobject *ifobj_tx = test->ifobj_tx;1924 1925	if ((ifobj_rx->umem->unaligned_mode && !ifobj_rx->unaligned_supp) ||1926	    (ifobj_tx->umem->unaligned_mode && !ifobj_tx->unaligned_supp)) {1927		ksft_test_result_skip("No huge pages present.\n");1928		return TEST_SKIP;1929	}1930 1931	if (test->set_ring) {1932		if (ifobj_tx->hw_ring_size_supp) {1933			if (set_ring_size(ifobj_tx)) {1934				ksft_test_result_skip("Failed to change HW ring size.\n");1935				return TEST_FAILURE;1936			}1937		} else {1938			ksft_test_result_skip("Changing HW ring size not supported.\n");1939			return TEST_SKIP;1940		}1941	}1942 1943	xsk_attach_xdp_progs(test, ifobj_rx, ifobj_tx);1944	return __testapp_validate_traffic(test, ifobj_rx, ifobj_tx);1945}1946 1947static int testapp_validate_traffic_single_thread(struct test_spec *test, struct ifobject *ifobj)1948{1949	return __testapp_validate_traffic(test, ifobj, NULL);1950}1951 1952static int testapp_teardown(struct test_spec *test)1953{1954	int i;1955 1956	for (i = 0; i < MAX_TEARDOWN_ITER; i++) {1957		if (testapp_validate_traffic(test))1958			return TEST_FAILURE;1959		test_spec_reset(test);1960	}1961 1962	return TEST_PASS;1963}1964 1965static void swap_directions(struct ifobject **ifobj1, struct ifobject **ifobj2)1966{1967	thread_func_t tmp_func_ptr = (*ifobj1)->func_ptr;1968	struct ifobject *tmp_ifobj = (*ifobj1);1969 1970	(*ifobj1)->func_ptr = (*ifobj2)->func_ptr;1971	(*ifobj2)->func_ptr = tmp_func_ptr;1972 1973	*ifobj1 = *ifobj2;1974	*ifobj2 = tmp_ifobj;1975}1976 1977static int testapp_bidirectional(struct test_spec *test)1978{1979	int res;1980 1981	test->ifobj_tx->rx_on = true;1982	test->ifobj_rx->tx_on = true;1983	test->total_steps = 2;1984	if (testapp_validate_traffic(test))1985		return TEST_FAILURE;1986 1987	print_verbose("Switching Tx/Rx direction\n");1988	swap_directions(&test->ifobj_rx, &test->ifobj_tx);1989	res = __testapp_validate_traffic(test, test->ifobj_rx, test->ifobj_tx);1990 1991	swap_directions(&test->ifobj_rx, &test->ifobj_tx);1992	return res;1993}1994 1995static int swap_xsk_resources(struct test_spec *test)1996{1997	int ret;1998 1999	test->ifobj_tx->xsk_arr[0].pkt_stream = NULL;2000	test->ifobj_rx->xsk_arr[0].pkt_stream = NULL;2001	test->ifobj_tx->xsk_arr[1].pkt_stream = test->tx_pkt_stream_default;2002	test->ifobj_rx->xsk_arr[1].pkt_stream = test->rx_pkt_stream_default;2003	test->ifobj_tx->xsk = &test->ifobj_tx->xsk_arr[1];2004	test->ifobj_rx->xsk = &test->ifobj_rx->xsk_arr[1];2005 2006	ret = xsk_update_xskmap(test->ifobj_rx->xskmap, test->ifobj_rx->xsk->xsk, 0);2007	if (ret)2008		return TEST_FAILURE;2009 2010	return TEST_PASS;2011}2012 2013static int testapp_xdp_prog_cleanup(struct test_spec *test)2014{2015	test->total_steps = 2;2016	test->nb_sockets = 2;2017	if (testapp_validate_traffic(test))2018		return TEST_FAILURE;2019 2020	if (swap_xsk_resources(test))2021		return TEST_FAILURE;2022	return testapp_validate_traffic(test);2023}2024 2025static int testapp_headroom(struct test_spec *test)2026{2027	test->ifobj_rx->umem->frame_headroom = UMEM_HEADROOM_TEST_SIZE;2028	return testapp_validate_traffic(test);2029}2030 2031static int testapp_stats_rx_dropped(struct test_spec *test)2032{2033	if (test->mode == TEST_MODE_ZC) {2034		ksft_test_result_skip("Can not run RX_DROPPED test for ZC mode\n");2035		return TEST_SKIP;2036	}2037 2038	pkt_stream_replace_half(test, MIN_PKT_SIZE * 4, 0);2039	test->ifobj_rx->umem->frame_headroom = test->ifobj_rx->umem->frame_size -2040		XDP_PACKET_HEADROOM - MIN_PKT_SIZE * 3;2041	pkt_stream_receive_half(test);2042	test->ifobj_rx->validation_func = validate_rx_dropped;2043	return testapp_validate_traffic(test);2044}2045 2046static int testapp_stats_tx_invalid_descs(struct test_spec *test)2047{2048	pkt_stream_replace_half(test, XSK_UMEM__INVALID_FRAME_SIZE, 0);2049	test->ifobj_tx->validation_func = validate_tx_invalid_descs;2050	return testapp_validate_traffic(test);2051}2052 2053static int testapp_stats_rx_full(struct test_spec *test)2054{2055	pkt_stream_replace(test, DEFAULT_UMEM_BUFFERS + DEFAULT_UMEM_BUFFERS / 2, MIN_PKT_SIZE);2056	test->ifobj_rx->xsk->pkt_stream = pkt_stream_generate(DEFAULT_UMEM_BUFFERS, MIN_PKT_SIZE);2057 2058	test->ifobj_rx->xsk->rxqsize = DEFAULT_UMEM_BUFFERS;2059	test->ifobj_rx->release_rx = false;2060	test->ifobj_rx->validation_func = validate_rx_full;2061	return testapp_validate_traffic(test);2062}2063 2064static int testapp_stats_fill_empty(struct test_spec *test)2065{2066	pkt_stream_replace(test, DEFAULT_UMEM_BUFFERS + DEFAULT_UMEM_BUFFERS / 2, MIN_PKT_SIZE);2067	test->ifobj_rx->xsk->pkt_stream = pkt_stream_generate(DEFAULT_UMEM_BUFFERS, MIN_PKT_SIZE);2068 2069	test->ifobj_rx->use_fill_ring = false;2070	test->ifobj_rx->validation_func = validate_fill_empty;2071	return testapp_validate_traffic(test);2072}2073 2074static int testapp_send_receive_unaligned(struct test_spec *test)2075{2076	test->ifobj_tx->umem->unaligned_mode = true;2077	test->ifobj_rx->umem->unaligned_mode = true;2078	/* Let half of the packets straddle a 4K buffer boundary */2079	pkt_stream_replace_half(test, MIN_PKT_SIZE, -MIN_PKT_SIZE / 2);2080 2081	return testapp_validate_traffic(test);2082}2083 2084static int testapp_send_receive_unaligned_mb(struct test_spec *test)2085{2086	test->mtu = MAX_ETH_JUMBO_SIZE;2087	test->ifobj_tx->umem->unaligned_mode = true;2088	test->ifobj_rx->umem->unaligned_mode = true;2089	pkt_stream_replace(test, DEFAULT_PKT_CNT, MAX_ETH_JUMBO_SIZE);2090	return testapp_validate_traffic(test);2091}2092 2093static int testapp_single_pkt(struct test_spec *test)2094{2095	struct pkt pkts[] = {{0, MIN_PKT_SIZE, 0, true}};2096 2097	pkt_stream_generate_custom(test, pkts, ARRAY_SIZE(pkts));2098	return testapp_validate_traffic(test);2099}2100 2101static int testapp_send_receive_mb(struct test_spec *test)2102{2103	test->mtu = MAX_ETH_JUMBO_SIZE;2104	pkt_stream_replace(test, DEFAULT_PKT_CNT, MAX_ETH_JUMBO_SIZE);2105 2106	return testapp_validate_traffic(test);2107}2108 2109static int testapp_invalid_desc_mb(struct test_spec *test)2110{2111	struct xsk_umem_info *umem = test->ifobj_tx->umem;2112	u64 umem_size = umem->num_frames * umem->frame_size;2113	struct pkt pkts[] = {2114		/* Valid packet for synch to start with */2115		{0, MIN_PKT_SIZE, 0, true, 0},2116		/* Zero frame len is not legal */2117		{0, XSK_UMEM__LARGE_FRAME_SIZE, 0, false, XDP_PKT_CONTD},2118		{0, XSK_UMEM__LARGE_FRAME_SIZE, 0, false, XDP_PKT_CONTD},2119		{0, 0, 0, false, 0},2120		/* Invalid address in the second frame */2121		{0, XSK_UMEM__LARGE_FRAME_SIZE, 0, false, XDP_PKT_CONTD},2122		{umem_size, XSK_UMEM__LARGE_FRAME_SIZE, 0, false, XDP_PKT_CONTD},2123		/* Invalid len in the middle */2124		{0, XSK_UMEM__LARGE_FRAME_SIZE, 0, false, XDP_PKT_CONTD},2125		{0, XSK_UMEM__INVALID_FRAME_SIZE, 0, false, XDP_PKT_CONTD},2126		/* Invalid options in the middle */2127		{0, XSK_UMEM__LARGE_FRAME_SIZE, 0, false, XDP_PKT_CONTD},2128		{0, XSK_UMEM__LARGE_FRAME_SIZE, 0, false, XSK_DESC__INVALID_OPTION},2129		/* Transmit 2 frags, receive 3 */2130		{0, XSK_UMEM__MAX_FRAME_SIZE, 0, true, XDP_PKT_CONTD},2131		{0, XSK_UMEM__MAX_FRAME_SIZE, 0, true, 0},2132		/* Middle frame crosses chunk boundary with small length */2133		{0, XSK_UMEM__LARGE_FRAME_SIZE, 0, false, XDP_PKT_CONTD},2134		{-MIN_PKT_SIZE / 2, MIN_PKT_SIZE, 0, false, 0},2135		/* Valid packet for synch so that something is received */2136		{0, MIN_PKT_SIZE, 0, true, 0}};2137 2138	if (umem->unaligned_mode) {2139		/* Crossing a chunk boundary allowed */2140		pkts[12].valid = true;2141		pkts[13].valid = true;2142	}2143 2144	test->mtu = MAX_ETH_JUMBO_SIZE;2145	pkt_stream_generate_custom(test, pkts, ARRAY_SIZE(pkts));2146	return testapp_validate_traffic(test);2147}2148 2149static int testapp_invalid_desc(struct test_spec *test)2150{2151	struct xsk_umem_info *umem = test->ifobj_tx->umem;2152	u64 umem_size = umem->num_frames * umem->frame_size;2153	struct pkt pkts[] = {2154		/* Zero packet address allowed */2155		{0, MIN_PKT_SIZE, 0, true},2156		/* Allowed packet */2157		{0, MIN_PKT_SIZE, 0, true},2158		/* Straddling the start of umem */2159		{-2, MIN_PKT_SIZE, 0, false},2160		/* Packet too large */2161		{0, XSK_UMEM__INVALID_FRAME_SIZE, 0, false},2162		/* Up to end of umem allowed */2163		{umem_size - MIN_PKT_SIZE - 2 * umem->frame_size, MIN_PKT_SIZE, 0, true},2164		/* After umem ends */2165		{umem_size, MIN_PKT_SIZE, 0, false},2166		/* Straddle the end of umem */2167		{umem_size - MIN_PKT_SIZE / 2, MIN_PKT_SIZE, 0, false},2168		/* Straddle a 4K boundary */2169		{0x1000 - MIN_PKT_SIZE / 2, MIN_PKT_SIZE, 0, false},2170		/* Straddle a 2K boundary */2171		{0x800 - MIN_PKT_SIZE / 2, MIN_PKT_SIZE, 0, true},2172		/* Valid packet for synch so that something is received */2173		{0, MIN_PKT_SIZE, 0, true}};2174 2175	if (umem->unaligned_mode) {2176		/* Crossing a page boundary allowed */2177		pkts[7].valid = true;2178	}2179	if (umem->frame_size == XSK_UMEM__DEFAULT_FRAME_SIZE / 2) {2180		/* Crossing a 2K frame size boundary not allowed */2181		pkts[8].valid = false;2182	}2183 2184	if (test->ifobj_tx->shared_umem) {2185		pkts[4].offset += umem_size;2186		pkts[5].offset += umem_size;2187		pkts[6].offset += umem_size;2188	}2189 2190	pkt_stream_generate_custom(test, pkts, ARRAY_SIZE(pkts));2191	return testapp_validate_traffic(test);2192}2193 2194static int testapp_xdp_drop(struct test_spec *test)2195{2196	struct xsk_xdp_progs *skel_rx = test->ifobj_rx->xdp_progs;2197	struct xsk_xdp_progs *skel_tx = test->ifobj_tx->xdp_progs;2198 2199	test_spec_set_xdp_prog(test, skel_rx->progs.xsk_xdp_drop, skel_tx->progs.xsk_xdp_drop,2200			       skel_rx->maps.xsk, skel_tx->maps.xsk);2201 2202	pkt_stream_receive_half(test);2203	return testapp_validate_traffic(test);2204}2205 2206static int testapp_xdp_metadata_copy(struct test_spec *test)2207{2208	struct xsk_xdp_progs *skel_rx = test->ifobj_rx->xdp_progs;2209	struct xsk_xdp_progs *skel_tx = test->ifobj_tx->xdp_progs;2210	struct bpf_map *data_map;2211	int count = 0;2212	int key = 0;2213 2214	test_spec_set_xdp_prog(test, skel_rx->progs.xsk_xdp_populate_metadata,2215			       skel_tx->progs.xsk_xdp_populate_metadata,2216			       skel_rx->maps.xsk, skel_tx->maps.xsk);2217	test->ifobj_rx->use_metadata = true;2218 2219	data_map = bpf_object__find_map_by_name(skel_rx->obj, "xsk_xdp_.bss");2220	if (!data_map || !bpf_map__is_internal(data_map)) {2221		ksft_print_msg("Error: could not find bss section of XDP program\n");2222		return TEST_FAILURE;2223	}2224 2225	if (bpf_map_update_elem(bpf_map__fd(data_map), &key, &count, BPF_ANY)) {2226		ksft_print_msg("Error: could not update count element\n");2227		return TEST_FAILURE;2228	}2229 2230	return testapp_validate_traffic(test);2231}2232 2233static int testapp_xdp_shared_umem(struct test_spec *test)2234{2235	struct xsk_xdp_progs *skel_rx = test->ifobj_rx->xdp_progs;2236	struct xsk_xdp_progs *skel_tx = test->ifobj_tx->xdp_progs;2237 2238	test->total_steps = 1;2239	test->nb_sockets = 2;2240 2241	test_spec_set_xdp_prog(test, skel_rx->progs.xsk_xdp_shared_umem,2242			       skel_tx->progs.xsk_xdp_shared_umem,2243			       skel_rx->maps.xsk, skel_tx->maps.xsk);2244 2245	pkt_stream_even_odd_sequence(test);2246 2247	return testapp_validate_traffic(test);2248}2249 2250static int testapp_poll_txq_tmout(struct test_spec *test)2251{2252	test->ifobj_tx->use_poll = true;2253	/* create invalid frame by set umem frame_size and pkt length equal to 2048 */2254	test->ifobj_tx->umem->frame_size = 2048;2255	pkt_stream_replace(test, 2 * DEFAULT_PKT_CNT, 2048);2256	return testapp_validate_traffic_single_thread(test, test->ifobj_tx);2257}2258 2259static int testapp_poll_rxq_tmout(struct test_spec *test)2260{2261	test->ifobj_rx->use_poll = true;2262	return testapp_validate_traffic_single_thread(test, test->ifobj_rx);2263}2264 2265static int testapp_too_many_frags(struct test_spec *test)2266{2267	struct pkt *pkts;2268	u32 max_frags, i;2269	int ret;2270 2271	if (test->mode == TEST_MODE_ZC) {2272		max_frags = test->ifobj_tx->xdp_zc_max_segs;2273	} else {2274		max_frags = get_max_skb_frags();2275		if (!max_frags) {2276			ksft_print_msg("Couldn't retrieve MAX_SKB_FRAGS from system, using default (17) value\n");2277			max_frags = 17;2278		}2279		max_frags += 1;2280	}2281 2282	pkts = calloc(2 * max_frags + 2, sizeof(struct pkt));2283	if (!pkts)2284		return TEST_FAILURE;2285 2286	test->mtu = MAX_ETH_JUMBO_SIZE;2287 2288	/* Valid packet for synch */2289	pkts[0].len = MIN_PKT_SIZE;2290	pkts[0].valid = true;2291 2292	/* One valid packet with the max amount of frags */2293	for (i = 1; i < max_frags + 1; i++) {2294		pkts[i].len = MIN_PKT_SIZE;2295		pkts[i].options = XDP_PKT_CONTD;2296		pkts[i].valid = true;2297	}2298	pkts[max_frags].options = 0;2299 2300	/* An invalid packet with the max amount of frags but signals packet2301	 * continues on the last frag2302	 */2303	for (i = max_frags + 1; i < 2 * max_frags + 1; i++) {2304		pkts[i].len = MIN_PKT_SIZE;2305		pkts[i].options = XDP_PKT_CONTD;2306		pkts[i].valid = false;2307	}2308 2309	/* Valid packet for synch */2310	pkts[2 * max_frags + 1].len = MIN_PKT_SIZE;2311	pkts[2 * max_frags + 1].valid = true;2312 2313	pkt_stream_generate_custom(test, pkts, 2 * max_frags + 2);2314	ret = testapp_validate_traffic(test);2315 2316	free(pkts);2317	return ret;2318}2319 2320static int xsk_load_xdp_programs(struct ifobject *ifobj)2321{2322	ifobj->xdp_progs = xsk_xdp_progs__open_and_load();2323	if (libbpf_get_error(ifobj->xdp_progs))2324		return libbpf_get_error(ifobj->xdp_progs);2325 2326	return 0;2327}2328 2329static void xsk_unload_xdp_programs(struct ifobject *ifobj)2330{2331	xsk_xdp_progs__destroy(ifobj->xdp_progs);2332}2333 2334/* Simple test */2335static bool hugepages_present(void)2336{2337	size_t mmap_sz = 2 * DEFAULT_UMEM_BUFFERS * XSK_UMEM__DEFAULT_FRAME_SIZE;2338	void *bufs;2339 2340	bufs = mmap(NULL, mmap_sz, PROT_READ | PROT_WRITE,2341		    MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB, -1, MAP_HUGE_2MB);2342	if (bufs == MAP_FAILED)2343		return false;2344 2345	mmap_sz = ceil_u64(mmap_sz, HUGEPAGE_SIZE) * HUGEPAGE_SIZE;2346	munmap(bufs, mmap_sz);2347	return true;2348}2349 2350static void init_iface(struct ifobject *ifobj, thread_func_t func_ptr)2351{2352	LIBBPF_OPTS(bpf_xdp_query_opts, query_opts);2353	int err;2354 2355	ifobj->func_ptr = func_ptr;2356 2357	err = xsk_load_xdp_programs(ifobj);2358	if (err) {2359		ksft_print_msg("Error loading XDP program\n");2360		exit_with_error(err);2361	}2362 2363	if (hugepages_present())2364		ifobj->unaligned_supp = true;2365 2366	err = bpf_xdp_query(ifobj->ifindex, XDP_FLAGS_DRV_MODE, &query_opts);2367	if (err) {2368		ksft_print_msg("Error querying XDP capabilities\n");2369		exit_with_error(-err);2370	}2371	if (query_opts.feature_flags & NETDEV_XDP_ACT_RX_SG)2372		ifobj->multi_buff_supp = true;2373	if (query_opts.feature_flags & NETDEV_XDP_ACT_XSK_ZEROCOPY) {2374		if (query_opts.xdp_zc_max_segs > 1) {2375			ifobj->multi_buff_zc_supp = true;2376			ifobj->xdp_zc_max_segs = query_opts.xdp_zc_max_segs;2377		} else {2378			ifobj->xdp_zc_max_segs = 0;2379		}2380	}2381}2382 2383static int testapp_send_receive(struct test_spec *test)2384{2385	return testapp_validate_traffic(test);2386}2387 2388static int testapp_send_receive_2k_frame(struct test_spec *test)2389{2390	test->ifobj_tx->umem->frame_size = 2048;2391	test->ifobj_rx->umem->frame_size = 2048;2392	pkt_stream_replace(test, DEFAULT_PKT_CNT, MIN_PKT_SIZE);2393	return testapp_validate_traffic(test);2394}2395 2396static int testapp_poll_rx(struct test_spec *test)2397{2398	test->ifobj_rx->use_poll = true;2399	return testapp_validate_traffic(test);2400}2401 2402static int testapp_poll_tx(struct test_spec *test)2403{2404	test->ifobj_tx->use_poll = true;2405	return testapp_validate_traffic(test);2406}2407 2408static int testapp_aligned_inv_desc(struct test_spec *test)2409{2410	return testapp_invalid_desc(test);2411}2412 2413static int testapp_aligned_inv_desc_2k_frame(struct test_spec *test)2414{2415	test->ifobj_tx->umem->frame_size = 2048;2416	test->ifobj_rx->umem->frame_size = 2048;2417	return testapp_invalid_desc(test);2418}2419 2420static int testapp_unaligned_inv_desc(struct test_spec *test)2421{2422	test->ifobj_tx->umem->unaligned_mode = true;2423	test->ifobj_rx->umem->unaligned_mode = true;2424	return testapp_invalid_desc(test);2425}2426 2427static int testapp_unaligned_inv_desc_4001_frame(struct test_spec *test)2428{2429	u64 page_size, umem_size;2430 2431	/* Odd frame size so the UMEM doesn't end near a page boundary. */2432	test->ifobj_tx->umem->frame_size = 4001;2433	test->ifobj_rx->umem->frame_size = 4001;2434	test->ifobj_tx->umem->unaligned_mode = true;2435	test->ifobj_rx->umem->unaligned_mode = true;2436	/* This test exists to test descriptors that staddle the end of2437	 * the UMEM but not a page.2438	 */2439	page_size = sysconf(_SC_PAGESIZE);2440	umem_size = test->ifobj_tx->umem->num_frames * test->ifobj_tx->umem->frame_size;2441	assert(umem_size % page_size > MIN_PKT_SIZE);2442	assert(umem_size % page_size < page_size - MIN_PKT_SIZE);2443 2444	return testapp_invalid_desc(test);2445}2446 2447static int testapp_aligned_inv_desc_mb(struct test_spec *test)2448{2449	return testapp_invalid_desc_mb(test);2450}2451 2452static int testapp_unaligned_inv_desc_mb(struct test_spec *test)2453{2454	test->ifobj_tx->umem->unaligned_mode = true;2455	test->ifobj_rx->umem->unaligned_mode = true;2456	return testapp_invalid_desc_mb(test);2457}2458 2459static int testapp_xdp_metadata(struct test_spec *test)2460{2461	return testapp_xdp_metadata_copy(test);2462}2463 2464static int testapp_xdp_metadata_mb(struct test_spec *test)2465{2466	test->mtu = MAX_ETH_JUMBO_SIZE;2467	return testapp_xdp_metadata_copy(test);2468}2469 2470static int testapp_hw_sw_min_ring_size(struct test_spec *test)2471{2472	int ret;2473 2474	test->set_ring = true;2475	test->total_steps = 2;2476	test->ifobj_tx->ring.tx_pending = DEFAULT_BATCH_SIZE;2477	test->ifobj_tx->ring.rx_pending = DEFAULT_BATCH_SIZE * 2;2478	test->ifobj_tx->xsk->batch_size = 1;2479	test->ifobj_rx->xsk->batch_size = 1;2480	ret = testapp_validate_traffic(test);2481	if (ret)2482		return ret;2483 2484	/* Set batch size to hw_ring_size - 1 */2485	test->ifobj_tx->xsk->batch_size = DEFAULT_BATCH_SIZE - 1;2486	test->ifobj_rx->xsk->batch_size = DEFAULT_BATCH_SIZE - 1;2487	return testapp_validate_traffic(test);2488}2489 2490static int testapp_hw_sw_max_ring_size(struct test_spec *test)2491{2492	u32 max_descs = XSK_RING_PROD__DEFAULT_NUM_DESCS * 4;2493	int ret;2494 2495	test->set_ring = true;2496	test->total_steps = 2;2497	test->ifobj_tx->ring.tx_pending = test->ifobj_tx->ring.tx_max_pending;2498	test->ifobj_tx->ring.rx_pending  = test->ifobj_tx->ring.rx_max_pending;2499	test->ifobj_rx->umem->num_frames = max_descs;2500	test->ifobj_rx->umem->fill_size = max_descs;2501	test->ifobj_rx->umem->comp_size = max_descs;2502	test->ifobj_tx->xsk->batch_size = XSK_RING_PROD__DEFAULT_NUM_DESCS;2503	test->ifobj_rx->xsk->batch_size = XSK_RING_PROD__DEFAULT_NUM_DESCS;2504 2505	ret = testapp_validate_traffic(test);2506	if (ret)2507		return ret;2508 2509	/* Set batch_size to 8152 for testing, as the ice HW ignores the 3 lowest bits when2510	 * updating the Rx HW tail register.2511	 */2512	test->ifobj_tx->xsk->batch_size = test->ifobj_tx->ring.tx_max_pending - 8;2513	test->ifobj_rx->xsk->batch_size = test->ifobj_tx->ring.tx_max_pending - 8;2514	pkt_stream_replace(test, max_descs, MIN_PKT_SIZE);2515	return testapp_validate_traffic(test);2516}2517 2518static void run_pkt_test(struct test_spec *test)2519{2520	int ret;2521 2522	ret = test->test_func(test);2523 2524	if (ret == TEST_PASS)2525		ksft_test_result_pass("PASS: %s %s%s\n", mode_string(test), busy_poll_string(test),2526				      test->name);2527	pkt_stream_restore_default(test);2528}2529 2530static struct ifobject *ifobject_create(void)2531{2532	struct ifobject *ifobj;2533 2534	ifobj = calloc(1, sizeof(struct ifobject));2535	if (!ifobj)2536		return NULL;2537 2538	ifobj->xsk_arr = calloc(MAX_SOCKETS, sizeof(*ifobj->xsk_arr));2539	if (!ifobj->xsk_arr)2540		goto out_xsk_arr;2541 2542	ifobj->umem = calloc(1, sizeof(*ifobj->umem));2543	if (!ifobj->umem)2544		goto out_umem;2545 2546	return ifobj;2547 2548out_umem:2549	free(ifobj->xsk_arr);2550out_xsk_arr:2551	free(ifobj);2552	return NULL;2553}2554 2555static void ifobject_delete(struct ifobject *ifobj)2556{2557	free(ifobj->umem);2558	free(ifobj->xsk_arr);2559	free(ifobj);2560}2561 2562static bool is_xdp_supported(int ifindex)2563{2564	int flags = XDP_FLAGS_DRV_MODE;2565 2566	LIBBPF_OPTS(bpf_link_create_opts, opts, .flags = flags);2567	struct bpf_insn insns[2] = {2568		BPF_MOV64_IMM(BPF_REG_0, XDP_PASS),2569		BPF_EXIT_INSN()2570	};2571	int prog_fd, insn_cnt = ARRAY_SIZE(insns);2572	int err;2573 2574	prog_fd = bpf_prog_load(BPF_PROG_TYPE_XDP, NULL, "GPL", insns, insn_cnt, NULL);2575	if (prog_fd < 0)2576		return false;2577 2578	err = bpf_xdp_attach(ifindex, prog_fd, flags, NULL);2579	if (err) {2580		close(prog_fd);2581		return false;2582	}2583 2584	bpf_xdp_detach(ifindex, flags, NULL);2585	close(prog_fd);2586 2587	return true;2588}2589 2590static const struct test_spec tests[] = {2591	{.name = "SEND_RECEIVE", .test_func = testapp_send_receive},2592	{.name = "SEND_RECEIVE_2K_FRAME", .test_func = testapp_send_receive_2k_frame},2593	{.name = "SEND_RECEIVE_SINGLE_PKT", .test_func = testapp_single_pkt},2594	{.name = "POLL_RX", .test_func = testapp_poll_rx},2595	{.name = "POLL_TX", .test_func = testapp_poll_tx},2596	{.name = "POLL_RXQ_FULL", .test_func = testapp_poll_rxq_tmout},2597	{.name = "POLL_TXQ_FULL", .test_func = testapp_poll_txq_tmout},2598	{.name = "SEND_RECEIVE_UNALIGNED", .test_func = testapp_send_receive_unaligned},2599	{.name = "ALIGNED_INV_DESC", .test_func = testapp_aligned_inv_desc},2600	{.name = "ALIGNED_INV_DESC_2K_FRAME_SIZE", .test_func = testapp_aligned_inv_desc_2k_frame},2601	{.name = "UNALIGNED_INV_DESC", .test_func = testapp_unaligned_inv_desc},2602	{.name = "UNALIGNED_INV_DESC_4001_FRAME_SIZE",2603	 .test_func = testapp_unaligned_inv_desc_4001_frame},2604	{.name = "UMEM_HEADROOM", .test_func = testapp_headroom},2605	{.name = "TEARDOWN", .test_func = testapp_teardown},2606	{.name = "BIDIRECTIONAL", .test_func = testapp_bidirectional},2607	{.name = "STAT_RX_DROPPED", .test_func = testapp_stats_rx_dropped},2608	{.name = "STAT_TX_INVALID", .test_func = testapp_stats_tx_invalid_descs},2609	{.name = "STAT_RX_FULL", .test_func = testapp_stats_rx_full},2610	{.name = "STAT_FILL_EMPTY", .test_func = testapp_stats_fill_empty},2611	{.name = "XDP_PROG_CLEANUP", .test_func = testapp_xdp_prog_cleanup},2612	{.name = "XDP_DROP_HALF", .test_func = testapp_xdp_drop},2613	{.name = "XDP_SHARED_UMEM", .test_func = testapp_xdp_shared_umem},2614	{.name = "XDP_METADATA_COPY", .test_func = testapp_xdp_metadata},2615	{.name = "XDP_METADATA_COPY_MULTI_BUFF", .test_func = testapp_xdp_metadata_mb},2616	{.name = "SEND_RECEIVE_9K_PACKETS", .test_func = testapp_send_receive_mb},2617	{.name = "SEND_RECEIVE_UNALIGNED_9K_PACKETS",2618	 .test_func = testapp_send_receive_unaligned_mb},2619	{.name = "ALIGNED_INV_DESC_MULTI_BUFF", .test_func = testapp_aligned_inv_desc_mb},2620	{.name = "UNALIGNED_INV_DESC_MULTI_BUFF", .test_func = testapp_unaligned_inv_desc_mb},2621	{.name = "TOO_MANY_FRAGS", .test_func = testapp_too_many_frags},2622	{.name = "HW_SW_MIN_RING_SIZE", .test_func = testapp_hw_sw_min_ring_size},2623	{.name = "HW_SW_MAX_RING_SIZE", .test_func = testapp_hw_sw_max_ring_size},2624	};2625 2626static void print_tests(void)2627{2628	u32 i;2629 2630	printf("Tests:\n");2631	for (i = 0; i < ARRAY_SIZE(tests); i++)2632		printf("%u: %s\n", i, tests[i].name);2633}2634 2635int main(int argc, char **argv)2636{2637	struct pkt_stream *rx_pkt_stream_default;2638	struct pkt_stream *tx_pkt_stream_default;2639	struct ifobject *ifobj_tx, *ifobj_rx;2640	u32 i, j, failed_tests = 0, nb_tests;2641	int modes = TEST_MODE_SKB + 1;2642	struct test_spec test;2643	bool shared_netdev;2644	int ret;2645 2646	/* Use libbpf 1.0 API mode */2647	libbpf_set_strict_mode(LIBBPF_STRICT_ALL);2648 2649	ifobj_tx = ifobject_create();2650	if (!ifobj_tx)2651		exit_with_error(ENOMEM);2652	ifobj_rx = ifobject_create();2653	if (!ifobj_rx)2654		exit_with_error(ENOMEM);2655 2656	setlocale(LC_ALL, "");2657 2658	parse_command_line(ifobj_tx, ifobj_rx, argc, argv);2659 2660	if (opt_print_tests) {2661		print_tests();2662		ksft_exit_xpass();2663	}2664	if (opt_run_test != RUN_ALL_TESTS && opt_run_test >= ARRAY_SIZE(tests)) {2665		ksft_print_msg("Error: test %u does not exist.\n", opt_run_test);2666		ksft_exit_xfail();2667	}2668 2669	shared_netdev = (ifobj_tx->ifindex == ifobj_rx->ifindex);2670	ifobj_tx->shared_umem = shared_netdev;2671	ifobj_rx->shared_umem = shared_netdev;2672 2673	if (!validate_interface(ifobj_tx) || !validate_interface(ifobj_rx))2674		print_usage(argv);2675 2676	if (is_xdp_supported(ifobj_tx->ifindex)) {2677		modes++;2678		if (ifobj_zc_avail(ifobj_tx))2679			modes++;2680	}2681 2682	ret = get_hw_ring_size(ifobj_tx->ifname, &ifobj_tx->ring);2683	if (!ret) {2684		ifobj_tx->hw_ring_size_supp = true;2685		ifobj_tx->set_ring.default_tx = ifobj_tx->ring.tx_pending;2686		ifobj_tx->set_ring.default_rx = ifobj_tx->ring.rx_pending;2687	}2688 2689	init_iface(ifobj_rx, worker_testapp_validate_rx);2690	init_iface(ifobj_tx, worker_testapp_validate_tx);2691 2692	test_spec_init(&test, ifobj_tx, ifobj_rx, 0, &tests[0]);2693	tx_pkt_stream_default = pkt_stream_generate(DEFAULT_PKT_CNT, MIN_PKT_SIZE);2694	rx_pkt_stream_default = pkt_stream_generate(DEFAULT_PKT_CNT, MIN_PKT_SIZE);2695	if (!tx_pkt_stream_default || !rx_pkt_stream_default)2696		exit_with_error(ENOMEM);2697	test.tx_pkt_stream_default = tx_pkt_stream_default;2698	test.rx_pkt_stream_default = rx_pkt_stream_default;2699 2700	if (opt_run_test == RUN_ALL_TESTS)2701		nb_tests = ARRAY_SIZE(tests);2702	else2703		nb_tests = 1;2704	if (opt_mode == TEST_MODE_ALL) {2705		ksft_set_plan(modes * nb_tests);2706	} else {2707		if (opt_mode == TEST_MODE_DRV && modes <= TEST_MODE_DRV) {2708			ksft_print_msg("Error: XDP_DRV mode not supported.\n");2709			ksft_exit_xfail();2710		}2711		if (opt_mode == TEST_MODE_ZC && modes <= TEST_MODE_ZC) {2712			ksft_print_msg("Error: zero-copy mode not supported.\n");2713			ksft_exit_xfail();2714		}2715 2716		ksft_set_plan(nb_tests);2717	}2718 2719	for (i = 0; i < modes; i++) {2720		if (opt_mode != TEST_MODE_ALL && i != opt_mode)2721			continue;2722 2723		for (j = 0; j < ARRAY_SIZE(tests); j++) {2724			if (opt_run_test != RUN_ALL_TESTS && j != opt_run_test)2725				continue;2726 2727			test_spec_init(&test, ifobj_tx, ifobj_rx, i, &tests[j]);2728			run_pkt_test(&test);2729			usleep(USLEEP_MAX);2730 2731			if (test.fail)2732				failed_tests++;2733		}2734	}2735 2736	if (ifobj_tx->hw_ring_size_supp)2737		hw_ring_size_reset(ifobj_tx);2738 2739	pkt_stream_delete(tx_pkt_stream_default);2740	pkt_stream_delete(rx_pkt_stream_default);2741	xsk_unload_xdp_programs(ifobj_tx);2742	xsk_unload_xdp_programs(ifobj_rx);2743	ifobject_delete(ifobj_tx);2744	ifobject_delete(ifobj_rx);2745 2746	if (failed_tests)2747		ksft_exit_fail();2748	else2749		ksft_exit_pass();2750}2751