543 lines · c
1// SPDX-License-Identifier: GPL-2.02#include <test_progs.h>3#include <network_helpers.h>4#include "xdp_metadata.skel.h"5#include "xdp_metadata2.skel.h"6#include "xdp_metadata.h"7#include "xsk.h"8 9#include <bpf/btf.h>10#include <linux/errqueue.h>11#include <linux/if_link.h>12#include <linux/net_tstamp.h>13#include <linux/udp.h>14#include <sys/mman.h>15#include <net/if.h>16#include <poll.h>17 18#define TX_NAME "veTX"19#define RX_NAME "veRX"20 21#define UDP_PAYLOAD_BYTES 422 23#define UDP_SOURCE_PORT 123424#define AF_XDP_CONSUMER_PORT 808025 26#define UMEM_NUM 1627#define UMEM_FRAME_SIZE XSK_UMEM__DEFAULT_FRAME_SIZE28#define UMEM_SIZE (UMEM_FRAME_SIZE * UMEM_NUM)29#define XDP_FLAGS XDP_FLAGS_DRV_MODE30#define QUEUE_ID 031 32#define TX_ADDR "10.0.0.1"33#define RX_ADDR "10.0.0.2"34#define PREFIX_LEN "8"35#define FAMILY AF_INET36#define TX_NETNS_NAME "xdp_metadata_tx"37#define RX_NETNS_NAME "xdp_metadata_rx"38#define TX_MAC "00:00:00:00:00:01"39#define RX_MAC "00:00:00:00:00:02"40 41#define VLAN_ID 5942#define VLAN_PROTO "802.1Q"43#define VLAN_PID htons(ETH_P_8021Q)44#define TX_NAME_VLAN TX_NAME "." TO_STR(VLAN_ID)45 46#define XDP_RSS_TYPE_L4 BIT(3)47#define VLAN_VID_MASK 0xfff48 49struct xsk {50 void *umem_area;51 struct xsk_umem *umem;52 struct xsk_ring_prod fill;53 struct xsk_ring_cons comp;54 struct xsk_ring_prod tx;55 struct xsk_ring_cons rx;56 struct xsk_socket *socket;57};58 59static int open_xsk(int ifindex, struct xsk *xsk)60{61 int mmap_flags = MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE;62 const struct xsk_socket_config socket_config = {63 .rx_size = XSK_RING_PROD__DEFAULT_NUM_DESCS,64 .tx_size = XSK_RING_PROD__DEFAULT_NUM_DESCS,65 .bind_flags = XDP_COPY,66 };67 const struct xsk_umem_config umem_config = {68 .fill_size = XSK_RING_PROD__DEFAULT_NUM_DESCS,69 .comp_size = XSK_RING_CONS__DEFAULT_NUM_DESCS,70 .frame_size = XSK_UMEM__DEFAULT_FRAME_SIZE,71 .flags = XDP_UMEM_UNALIGNED_CHUNK_FLAG | XDP_UMEM_TX_SW_CSUM |72 XDP_UMEM_TX_METADATA_LEN,73 .tx_metadata_len = sizeof(struct xsk_tx_metadata),74 };75 __u32 idx;76 u64 addr;77 int ret;78 int i;79 80 xsk->umem_area = mmap(NULL, UMEM_SIZE, PROT_READ | PROT_WRITE, mmap_flags, -1, 0);81 if (!ASSERT_NEQ(xsk->umem_area, MAP_FAILED, "mmap"))82 return -1;83 84 ret = xsk_umem__create(&xsk->umem,85 xsk->umem_area, UMEM_SIZE,86 &xsk->fill,87 &xsk->comp,88 &umem_config);89 if (!ASSERT_OK(ret, "xsk_umem__create"))90 return ret;91 92 ret = xsk_socket__create(&xsk->socket, ifindex, QUEUE_ID,93 xsk->umem,94 &xsk->rx,95 &xsk->tx,96 &socket_config);97 if (!ASSERT_OK(ret, "xsk_socket__create"))98 return ret;99 100 /* First half of umem is for TX. This way address matches 1-to-1101 * to the completion queue index.102 */103 104 for (i = 0; i < UMEM_NUM / 2; i++) {105 addr = i * UMEM_FRAME_SIZE;106 printf("%p: tx_desc[%d] -> %lx\n", xsk, i, addr);107 }108 109 /* Second half of umem is for RX. */110 111 ret = xsk_ring_prod__reserve(&xsk->fill, UMEM_NUM / 2, &idx);112 if (!ASSERT_EQ(UMEM_NUM / 2, ret, "xsk_ring_prod__reserve"))113 return ret;114 if (!ASSERT_EQ(idx, 0, "fill idx != 0"))115 return -1;116 117 for (i = 0; i < UMEM_NUM / 2; i++) {118 addr = (UMEM_NUM / 2 + i) * UMEM_FRAME_SIZE;119 printf("%p: rx_desc[%d] -> %lx\n", xsk, i, addr);120 *xsk_ring_prod__fill_addr(&xsk->fill, i) = addr;121 }122 xsk_ring_prod__submit(&xsk->fill, ret);123 124 return 0;125}126 127static void close_xsk(struct xsk *xsk)128{129 if (xsk->umem)130 xsk_umem__delete(xsk->umem);131 if (xsk->socket)132 xsk_socket__delete(xsk->socket);133 munmap(xsk->umem_area, UMEM_SIZE);134}135 136static void ip_csum(struct iphdr *iph)137{138 __u32 sum = 0;139 __u16 *p;140 int i;141 142 iph->check = 0;143 p = (void *)iph;144 for (i = 0; i < sizeof(*iph) / sizeof(*p); i++)145 sum += p[i];146 147 while (sum >> 16)148 sum = (sum & 0xffff) + (sum >> 16);149 150 iph->check = ~sum;151}152 153static int generate_packet(struct xsk *xsk, __u16 dst_port)154{155 struct xsk_tx_metadata *meta;156 struct xdp_desc *tx_desc;157 struct udphdr *udph;158 struct ethhdr *eth;159 struct iphdr *iph;160 void *data;161 __u32 idx;162 int ret;163 164 ret = xsk_ring_prod__reserve(&xsk->tx, 1, &idx);165 if (!ASSERT_EQ(ret, 1, "xsk_ring_prod__reserve"))166 return -1;167 168 tx_desc = xsk_ring_prod__tx_desc(&xsk->tx, idx);169 tx_desc->addr = idx % (UMEM_NUM / 2) * UMEM_FRAME_SIZE + sizeof(struct xsk_tx_metadata);170 printf("%p: tx_desc[%u]->addr=%llx\n", xsk, idx, tx_desc->addr);171 data = xsk_umem__get_data(xsk->umem_area, tx_desc->addr);172 173 meta = data - sizeof(struct xsk_tx_metadata);174 memset(meta, 0, sizeof(*meta));175 meta->flags = XDP_TXMD_FLAGS_TIMESTAMP;176 177 eth = data;178 iph = (void *)(eth + 1);179 udph = (void *)(iph + 1);180 181 memcpy(eth->h_dest, "\x00\x00\x00\x00\x00\x02", ETH_ALEN);182 memcpy(eth->h_source, "\x00\x00\x00\x00\x00\x01", ETH_ALEN);183 eth->h_proto = htons(ETH_P_IP);184 185 iph->version = 0x4;186 iph->ihl = 0x5;187 iph->tos = 0x9;188 iph->tot_len = htons(sizeof(*iph) + sizeof(*udph) + UDP_PAYLOAD_BYTES);189 iph->id = 0;190 iph->frag_off = 0;191 iph->ttl = 0;192 iph->protocol = IPPROTO_UDP;193 ASSERT_EQ(inet_pton(FAMILY, TX_ADDR, &iph->saddr), 1, "inet_pton(TX_ADDR)");194 ASSERT_EQ(inet_pton(FAMILY, RX_ADDR, &iph->daddr), 1, "inet_pton(RX_ADDR)");195 ip_csum(iph);196 197 udph->source = htons(UDP_SOURCE_PORT);198 udph->dest = htons(dst_port);199 udph->len = htons(sizeof(*udph) + UDP_PAYLOAD_BYTES);200 udph->check = ~csum_tcpudp_magic(iph->saddr, iph->daddr,201 ntohs(udph->len), IPPROTO_UDP, 0);202 203 memset(udph + 1, 0xAA, UDP_PAYLOAD_BYTES);204 205 meta->flags |= XDP_TXMD_FLAGS_CHECKSUM;206 meta->request.csum_start = sizeof(*eth) + sizeof(*iph);207 meta->request.csum_offset = offsetof(struct udphdr, check);208 209 tx_desc->len = sizeof(*eth) + sizeof(*iph) + sizeof(*udph) + UDP_PAYLOAD_BYTES;210 tx_desc->options |= XDP_TX_METADATA;211 xsk_ring_prod__submit(&xsk->tx, 1);212 213 ret = sendto(xsk_socket__fd(xsk->socket), NULL, 0, MSG_DONTWAIT, NULL, 0);214 if (!ASSERT_GE(ret, 0, "sendto"))215 return ret;216 217 return 0;218}219 220static int generate_packet_inet(void)221{222 char udp_payload[UDP_PAYLOAD_BYTES];223 struct sockaddr_in rx_addr;224 int sock_fd, err = 0;225 226 /* Build a packet */227 memset(udp_payload, 0xAA, UDP_PAYLOAD_BYTES);228 rx_addr.sin_addr.s_addr = inet_addr(RX_ADDR);229 rx_addr.sin_family = AF_INET;230 rx_addr.sin_port = htons(AF_XDP_CONSUMER_PORT);231 232 sock_fd = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP);233 if (!ASSERT_GE(sock_fd, 0, "socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP)"))234 return sock_fd;235 236 err = sendto(sock_fd, udp_payload, UDP_PAYLOAD_BYTES, MSG_DONTWAIT,237 (void *)&rx_addr, sizeof(rx_addr));238 ASSERT_GE(err, 0, "sendto");239 240 close(sock_fd);241 return err;242}243 244static void complete_tx(struct xsk *xsk)245{246 struct xsk_tx_metadata *meta;247 __u64 addr;248 void *data;249 __u32 idx;250 251 if (ASSERT_EQ(xsk_ring_cons__peek(&xsk->comp, 1, &idx), 1, "xsk_ring_cons__peek")) {252 addr = *xsk_ring_cons__comp_addr(&xsk->comp, idx);253 254 printf("%p: complete tx idx=%u addr=%llx\n", xsk, idx, addr);255 256 data = xsk_umem__get_data(xsk->umem_area, addr);257 meta = data - sizeof(struct xsk_tx_metadata);258 259 ASSERT_NEQ(meta->completion.tx_timestamp, 0, "tx_timestamp");260 261 xsk_ring_cons__release(&xsk->comp, 1);262 }263}264 265static void refill_rx(struct xsk *xsk, __u64 addr)266{267 __u32 idx;268 269 if (ASSERT_EQ(xsk_ring_prod__reserve(&xsk->fill, 1, &idx), 1, "xsk_ring_prod__reserve")) {270 printf("%p: complete idx=%u addr=%llx\n", xsk, idx, addr);271 *xsk_ring_prod__fill_addr(&xsk->fill, idx) = addr;272 xsk_ring_prod__submit(&xsk->fill, 1);273 }274}275 276static int verify_xsk_metadata(struct xsk *xsk, bool sent_from_af_xdp)277{278 const struct xdp_desc *rx_desc;279 struct pollfd fds = {};280 struct xdp_meta *meta;281 struct udphdr *udph;282 struct ethhdr *eth;283 struct iphdr *iph;284 __u64 comp_addr;285 void *data;286 __u64 addr;287 __u32 idx = 0;288 int ret;289 290 ret = recvfrom(xsk_socket__fd(xsk->socket), NULL, 0, MSG_DONTWAIT, NULL, NULL);291 if (!ASSERT_EQ(ret, 0, "recvfrom"))292 return -1;293 294 fds.fd = xsk_socket__fd(xsk->socket);295 fds.events = POLLIN;296 297 ret = poll(&fds, 1, 1000);298 if (!ASSERT_GT(ret, 0, "poll"))299 return -1;300 301 ret = xsk_ring_cons__peek(&xsk->rx, 1, &idx);302 if (!ASSERT_EQ(ret, 1, "xsk_ring_cons__peek"))303 return -2;304 305 rx_desc = xsk_ring_cons__rx_desc(&xsk->rx, idx);306 comp_addr = xsk_umem__extract_addr(rx_desc->addr);307 addr = xsk_umem__add_offset_to_addr(rx_desc->addr);308 printf("%p: rx_desc[%u]->addr=%llx addr=%llx comp_addr=%llx\n",309 xsk, idx, rx_desc->addr, addr, comp_addr);310 data = xsk_umem__get_data(xsk->umem_area, addr);311 312 /* Make sure we got the packet offset correctly. */313 314 eth = data;315 ASSERT_EQ(eth->h_proto, htons(ETH_P_IP), "eth->h_proto");316 iph = (void *)(eth + 1);317 ASSERT_EQ((int)iph->version, 4, "iph->version");318 udph = (void *)(iph + 1);319 320 /* custom metadata */321 322 meta = data - sizeof(struct xdp_meta);323 324 if (!ASSERT_NEQ(meta->rx_timestamp, 0, "rx_timestamp"))325 return -1;326 327 if (!ASSERT_NEQ(meta->rx_hash, 0, "rx_hash"))328 return -1;329 330 if (!sent_from_af_xdp) {331 if (!ASSERT_NEQ(meta->rx_hash_type & XDP_RSS_TYPE_L4, 0, "rx_hash_type"))332 return -1;333 334 if (!ASSERT_EQ(meta->rx_vlan_tci & VLAN_VID_MASK, VLAN_ID, "rx_vlan_tci"))335 return -1;336 337 if (!ASSERT_EQ(meta->rx_vlan_proto, VLAN_PID, "rx_vlan_proto"))338 return -1;339 goto done;340 }341 342 ASSERT_EQ(meta->rx_hash_type, 0, "rx_hash_type");343 344 /* checksum offload */345 ASSERT_EQ(udph->check, htons(0x721c), "csum");346 347done:348 xsk_ring_cons__release(&xsk->rx, 1);349 refill_rx(xsk, comp_addr);350 351 return 0;352}353 354static void switch_ns_to_rx(struct nstoken **tok)355{356 close_netns(*tok);357 *tok = open_netns(RX_NETNS_NAME);358}359 360static void switch_ns_to_tx(struct nstoken **tok)361{362 close_netns(*tok);363 *tok = open_netns(TX_NETNS_NAME);364}365 366void test_xdp_metadata(void)367{368 struct xdp_metadata2 *bpf_obj2 = NULL;369 struct xdp_metadata *bpf_obj = NULL;370 struct bpf_program *new_prog, *prog;371 struct nstoken *tok = NULL;372 __u32 queue_id = QUEUE_ID;373 struct bpf_map *prog_arr;374 struct xsk tx_xsk = {};375 struct xsk rx_xsk = {};376 __u32 val, key = 0;377 int retries = 10;378 int rx_ifindex;379 int tx_ifindex;380 int sock_fd;381 int ret;382 383 /* Setup new networking namespaces, with a veth pair. */384 SYS(out, "ip netns add " TX_NETNS_NAME);385 SYS(out, "ip netns add " RX_NETNS_NAME);386 387 tok = open_netns(TX_NETNS_NAME);388 if (!ASSERT_OK_PTR(tok, "setns"))389 goto out;390 SYS(out, "ip link add numtxqueues 1 numrxqueues 1 " TX_NAME391 " type veth peer " RX_NAME " numtxqueues 1 numrxqueues 1");392 SYS(out, "ip link set " RX_NAME " netns " RX_NETNS_NAME);393 394 SYS(out, "ip link set dev " TX_NAME " address " TX_MAC);395 SYS(out, "ip link set dev " TX_NAME " up");396 397 SYS(out, "ip link add link " TX_NAME " " TX_NAME_VLAN398 " type vlan proto " VLAN_PROTO " id " TO_STR(VLAN_ID));399 SYS(out, "ip link set dev " TX_NAME_VLAN " up");400 SYS(out, "ip addr add " TX_ADDR "/" PREFIX_LEN " dev " TX_NAME_VLAN);401 402 /* Avoid ARP calls */403 SYS(out, "ip -4 neigh add " RX_ADDR " lladdr " RX_MAC " dev " TX_NAME_VLAN);404 405 switch_ns_to_rx(&tok);406 if (!ASSERT_OK_PTR(tok, "setns rx"))407 goto out;408 409 SYS(out, "ip link set dev " RX_NAME " address " RX_MAC);410 SYS(out, "ip link set dev " RX_NAME " up");411 SYS(out, "ip addr add " RX_ADDR "/" PREFIX_LEN " dev " RX_NAME);412 413 rx_ifindex = if_nametoindex(RX_NAME);414 415 /* Setup separate AF_XDP for RX interface. */416 417 ret = open_xsk(rx_ifindex, &rx_xsk);418 if (!ASSERT_OK(ret, "open_xsk(RX_NAME)"))419 goto out;420 421 bpf_obj = xdp_metadata__open();422 if (!ASSERT_OK_PTR(bpf_obj, "open skeleton"))423 goto out;424 425 prog = bpf_object__find_program_by_name(bpf_obj->obj, "rx");426 bpf_program__set_ifindex(prog, rx_ifindex);427 bpf_program__set_flags(prog, BPF_F_XDP_DEV_BOUND_ONLY);428 429 if (!ASSERT_OK(xdp_metadata__load(bpf_obj), "load skeleton"))430 goto out;431 432 /* Make sure we can't add dev-bound programs to prog maps. */433 prog_arr = bpf_object__find_map_by_name(bpf_obj->obj, "prog_arr");434 if (!ASSERT_OK_PTR(prog_arr, "no prog_arr map"))435 goto out;436 437 val = bpf_program__fd(prog);438 if (!ASSERT_ERR(bpf_map__update_elem(prog_arr, &key, sizeof(key),439 &val, sizeof(val), BPF_ANY),440 "update prog_arr"))441 goto out;442 443 /* Attach BPF program to RX interface. */444 445 ret = bpf_xdp_attach(rx_ifindex,446 bpf_program__fd(bpf_obj->progs.rx),447 XDP_FLAGS, NULL);448 if (!ASSERT_GE(ret, 0, "bpf_xdp_attach"))449 goto out;450 451 sock_fd = xsk_socket__fd(rx_xsk.socket);452 ret = bpf_map_update_elem(bpf_map__fd(bpf_obj->maps.xsk), &queue_id, &sock_fd, 0);453 if (!ASSERT_GE(ret, 0, "bpf_map_update_elem"))454 goto out;455 456 switch_ns_to_tx(&tok);457 if (!ASSERT_OK_PTR(tok, "setns tx"))458 goto out;459 460 /* Setup separate AF_XDP for TX interface nad send packet to the RX socket. */461 tx_ifindex = if_nametoindex(TX_NAME);462 ret = open_xsk(tx_ifindex, &tx_xsk);463 if (!ASSERT_OK(ret, "open_xsk(TX_NAME)"))464 goto out;465 466 if (!ASSERT_GE(generate_packet(&tx_xsk, AF_XDP_CONSUMER_PORT), 0,467 "generate AF_XDP_CONSUMER_PORT"))468 goto out;469 470 switch_ns_to_rx(&tok);471 if (!ASSERT_OK_PTR(tok, "setns rx"))472 goto out;473 474 /* Verify packet sent from AF_XDP has proper metadata. */475 if (!ASSERT_GE(verify_xsk_metadata(&rx_xsk, true), 0,476 "verify_xsk_metadata"))477 goto out;478 479 switch_ns_to_tx(&tok);480 if (!ASSERT_OK_PTR(tok, "setns tx"))481 goto out;482 complete_tx(&tx_xsk);483 484 /* Now check metadata of packet, generated with network stack */485 if (!ASSERT_GE(generate_packet_inet(), 0, "generate UDP packet"))486 goto out;487 488 switch_ns_to_rx(&tok);489 if (!ASSERT_OK_PTR(tok, "setns rx"))490 goto out;491 492 if (!ASSERT_GE(verify_xsk_metadata(&rx_xsk, false), 0,493 "verify_xsk_metadata"))494 goto out;495 496 /* Make sure freplace correctly picks up original bound device497 * and doesn't crash.498 */499 500 bpf_obj2 = xdp_metadata2__open();501 if (!ASSERT_OK_PTR(bpf_obj2, "open skeleton"))502 goto out;503 504 new_prog = bpf_object__find_program_by_name(bpf_obj2->obj, "freplace_rx");505 bpf_program__set_attach_target(new_prog, bpf_program__fd(prog), "rx");506 507 if (!ASSERT_OK(xdp_metadata2__load(bpf_obj2), "load freplace skeleton"))508 goto out;509 510 if (!ASSERT_OK(xdp_metadata2__attach(bpf_obj2), "attach freplace"))511 goto out;512 513 switch_ns_to_tx(&tok);514 if (!ASSERT_OK_PTR(tok, "setns tx"))515 goto out;516 517 /* Send packet to trigger . */518 if (!ASSERT_GE(generate_packet(&tx_xsk, AF_XDP_CONSUMER_PORT), 0,519 "generate freplace packet"))520 goto out;521 522 switch_ns_to_rx(&tok);523 if (!ASSERT_OK_PTR(tok, "setns rx"))524 goto out;525 526 while (!retries--) {527 if (bpf_obj2->bss->called)528 break;529 usleep(10);530 }531 ASSERT_GT(bpf_obj2->bss->called, 0, "not called");532 533out:534 close_xsk(&rx_xsk);535 close_xsk(&tx_xsk);536 xdp_metadata2__destroy(bpf_obj2);537 xdp_metadata__destroy(bpf_obj);538 if (tok)539 close_netns(tok);540 SYS_NOFAIL("ip netns del " RX_NETNS_NAME);541 SYS_NOFAIL("ip netns del " TX_NETNS_NAME);542}543