867 lines · c
1// SPDX-License-Identifier: LGPL-2.1 OR BSD-2-Clause2/* Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. */3 4#define BPF_NO_KFUNC_PROTOTYPES5#include "vmlinux.h"6 7#include <bpf/bpf_helpers.h>8#include <bpf/bpf_endian.h>9#include <asm/errno.h>10 11#include "bpf_compiler.h"12 13#define TC_ACT_OK 014#define TC_ACT_SHOT 215 16#define NSEC_PER_SEC 1000000000L17 18#define ETH_ALEN 619#define ETH_P_IP 0x080020#define ETH_P_IPV6 0x86DD21 22#define tcp_flag_word(tp) (((union tcp_word_hdr *)(tp))->words[3])23 24#define IP_DF 0x400025#define IP_MF 0x200026#define IP_OFFSET 0x1fff27 28#define NEXTHDR_TCP 629 30#define TCPOPT_NOP 131#define TCPOPT_EOL 032#define TCPOPT_MSS 233#define TCPOPT_WINDOW 334#define TCPOPT_SACK_PERM 435#define TCPOPT_TIMESTAMP 836 37#define TCPOLEN_MSS 438#define TCPOLEN_WINDOW 339#define TCPOLEN_SACK_PERM 240#define TCPOLEN_TIMESTAMP 1041 42#define TCP_TS_HZ 100043#define TS_OPT_WSCALE_MASK 0xf44#define TS_OPT_SACK (1 << 4)45#define TS_OPT_ECN (1 << 5)46#define TSBITS 647#define TSMASK (((__u32)1 << TSBITS) - 1)48#define TCP_MAX_WSCALE 14U49 50#define IPV4_MAXLEN 6051#define TCP_MAXLEN 6052 53#define DEFAULT_MSS4 146054#define DEFAULT_MSS6 144055#define DEFAULT_WSCALE 756#define DEFAULT_TTL 6457#define MAX_ALLOWED_PORTS 858 59#define MAX_PACKET_OFF 0xffff60 61#define swap(a, b) \62 do { typeof(a) __tmp = (a); (a) = (b); (b) = __tmp; } while (0)63 64#define __get_unaligned_t(type, ptr) ({ \65 const struct { type x; } __attribute__((__packed__)) *__pptr = (typeof(__pptr))(ptr); \66 __pptr->x; \67})68 69#define get_unaligned(ptr) __get_unaligned_t(typeof(*(ptr)), (ptr))70 71struct {72 __uint(type, BPF_MAP_TYPE_ARRAY);73 __type(key, __u32);74 __type(value, __u64);75 __uint(max_entries, 2);76} values SEC(".maps");77 78struct {79 __uint(type, BPF_MAP_TYPE_ARRAY);80 __type(key, __u32);81 __type(value, __u16);82 __uint(max_entries, MAX_ALLOWED_PORTS);83} allowed_ports SEC(".maps");84 85/* Some symbols defined in net/netfilter/nf_conntrack_bpf.c are unavailable in86 * vmlinux.h if CONFIG_NF_CONNTRACK=m, so they are redefined locally.87 */88 89struct bpf_ct_opts___local {90 s32 netns_id;91 s32 error;92 u8 l4proto;93 u8 dir;94 u8 reserved[2];95} __attribute__((preserve_access_index));96 97#define BPF_F_CURRENT_NETNS (-1)98 99extern struct nf_conn *bpf_xdp_ct_lookup(struct xdp_md *xdp_ctx,100 struct bpf_sock_tuple *bpf_tuple,101 __u32 len_tuple,102 struct bpf_ct_opts___local *opts,103 __u32 len_opts) __ksym;104 105extern struct nf_conn *bpf_skb_ct_lookup(struct __sk_buff *skb_ctx,106 struct bpf_sock_tuple *bpf_tuple,107 u32 len_tuple,108 struct bpf_ct_opts___local *opts,109 u32 len_opts) __ksym;110 111extern void bpf_ct_release(struct nf_conn *ct) __ksym;112 113static __always_inline void swap_eth_addr(__u8 *a, __u8 *b)114{115 __u8 tmp[ETH_ALEN];116 117 __builtin_memcpy(tmp, a, ETH_ALEN);118 __builtin_memcpy(a, b, ETH_ALEN);119 __builtin_memcpy(b, tmp, ETH_ALEN);120}121 122static __always_inline __u16 csum_fold(__u32 csum)123{124 csum = (csum & 0xffff) + (csum >> 16);125 csum = (csum & 0xffff) + (csum >> 16);126 return (__u16)~csum;127}128 129static __always_inline __u16 csum_tcpudp_magic(__be32 saddr, __be32 daddr,130 __u32 len, __u8 proto,131 __u32 csum)132{133 __u64 s = csum;134 135 s += (__u32)saddr;136 s += (__u32)daddr;137#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__138 s += proto + len;139#elif __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__140 s += (proto + len) << 8;141#else142#error Unknown endian143#endif144 s = (s & 0xffffffff) + (s >> 32);145 s = (s & 0xffffffff) + (s >> 32);146 147 return csum_fold((__u32)s);148}149 150static __always_inline __u16 csum_ipv6_magic(const struct in6_addr *saddr,151 const struct in6_addr *daddr,152 __u32 len, __u8 proto, __u32 csum)153{154 __u64 sum = csum;155 int i;156 157 __pragma_loop_unroll158 for (i = 0; i < 4; i++)159 sum += (__u32)saddr->in6_u.u6_addr32[i];160 161 __pragma_loop_unroll162 for (i = 0; i < 4; i++)163 sum += (__u32)daddr->in6_u.u6_addr32[i];164 165 /* Don't combine additions to avoid 32-bit overflow. */166 sum += bpf_htonl(len);167 sum += bpf_htonl(proto);168 169 sum = (sum & 0xffffffff) + (sum >> 32);170 sum = (sum & 0xffffffff) + (sum >> 32);171 172 return csum_fold((__u32)sum);173}174 175static __always_inline __u64 tcp_clock_ns(void)176{177 return bpf_ktime_get_ns();178}179 180static __always_inline __u32 tcp_ns_to_ts(__u64 ns)181{182 return ns / (NSEC_PER_SEC / TCP_TS_HZ);183}184 185static __always_inline __u32 tcp_clock_ms(void)186{187 return tcp_ns_to_ts(tcp_clock_ns());188}189 190struct tcpopt_context {191 void *data;192 void *data_end;193 __be32 *tsecr;194 __u8 wscale;195 bool option_timestamp;196 bool option_sack;197 __u32 off;198};199 200static __always_inline u8 *next(struct tcpopt_context *ctx, __u32 sz)201{202 __u64 off = ctx->off;203 __u8 *data;204 205 /* Verifier forbids access to packet when offset exceeds MAX_PACKET_OFF */206 if (off > MAX_PACKET_OFF - sz)207 return NULL;208 209 data = ctx->data + off;210 barrier_var(data);211 if (data + sz >= ctx->data_end)212 return NULL;213 214 ctx->off += sz;215 return data;216}217 218static int tscookie_tcpopt_parse(struct tcpopt_context *ctx)219{220 __u8 *opcode, *opsize, *wscale, *tsecr;221 __u32 off = ctx->off;222 223 opcode = next(ctx, 1);224 if (!opcode)225 return 1;226 227 if (*opcode == TCPOPT_EOL)228 return 1;229 if (*opcode == TCPOPT_NOP)230 return 0;231 232 opsize = next(ctx, 1);233 if (!opsize || *opsize < 2)234 return 1;235 236 switch (*opcode) {237 case TCPOPT_WINDOW:238 wscale = next(ctx, 1);239 if (!wscale)240 return 1;241 if (*opsize == TCPOLEN_WINDOW)242 ctx->wscale = *wscale < TCP_MAX_WSCALE ? *wscale : TCP_MAX_WSCALE;243 break;244 case TCPOPT_TIMESTAMP:245 tsecr = next(ctx, 4);246 if (!tsecr)247 return 1;248 if (*opsize == TCPOLEN_TIMESTAMP) {249 ctx->option_timestamp = true;250 /* Client's tsval becomes our tsecr. */251 *ctx->tsecr = get_unaligned((__be32 *)tsecr);252 }253 break;254 case TCPOPT_SACK_PERM:255 if (*opsize == TCPOLEN_SACK_PERM)256 ctx->option_sack = true;257 break;258 }259 260 ctx->off = off + *opsize;261 262 return 0;263}264 265static int tscookie_tcpopt_parse_batch(__u32 index, void *context)266{267 int i;268 269 for (i = 0; i < 7; i++)270 if (tscookie_tcpopt_parse(context))271 return 1;272 return 0;273}274 275static __always_inline bool tscookie_init(struct tcphdr *tcp_header,276 __u16 tcp_len, __be32 *tsval,277 __be32 *tsecr, void *data, void *data_end)278{279 struct tcpopt_context loop_ctx = {280 .data = data,281 .data_end = data_end,282 .tsecr = tsecr,283 .wscale = TS_OPT_WSCALE_MASK,284 .option_timestamp = false,285 .option_sack = false,286 /* Note: currently verifier would track .off as unbound scalar.287 * In case if verifier would at some point get smarter and288 * compute bounded value for this var, beware that it might289 * hinder bpf_loop() convergence validation.290 */291 .off = (__u8 *)(tcp_header + 1) - (__u8 *)data,292 };293 u32 cookie;294 295 bpf_loop(6, tscookie_tcpopt_parse_batch, &loop_ctx, 0);296 297 if (!loop_ctx.option_timestamp)298 return false;299 300 cookie = tcp_clock_ms() & ~TSMASK;301 cookie |= loop_ctx.wscale & TS_OPT_WSCALE_MASK;302 if (loop_ctx.option_sack)303 cookie |= TS_OPT_SACK;304 if (tcp_header->ece && tcp_header->cwr)305 cookie |= TS_OPT_ECN;306 *tsval = bpf_htonl(cookie);307 308 return true;309}310 311static __always_inline void values_get_tcpipopts(__u16 *mss, __u8 *wscale,312 __u8 *ttl, bool ipv6)313{314 __u32 key = 0;315 __u64 *value;316 317 value = bpf_map_lookup_elem(&values, &key);318 if (value && *value != 0) {319 if (ipv6)320 *mss = (*value >> 32) & 0xffff;321 else322 *mss = *value & 0xffff;323 *wscale = (*value >> 16) & 0xf;324 *ttl = (*value >> 24) & 0xff;325 return;326 }327 328 *mss = ipv6 ? DEFAULT_MSS6 : DEFAULT_MSS4;329 *wscale = DEFAULT_WSCALE;330 *ttl = DEFAULT_TTL;331}332 333static __always_inline void values_inc_synacks(void)334{335 __u32 key = 1;336 __u64 *value;337 338 value = bpf_map_lookup_elem(&values, &key);339 if (value)340 __sync_fetch_and_add(value, 1);341}342 343static __always_inline bool check_port_allowed(__u16 port)344{345 __u32 i;346 347 for (i = 0; i < MAX_ALLOWED_PORTS; i++) {348 __u32 key = i;349 __u16 *value;350 351 value = bpf_map_lookup_elem(&allowed_ports, &key);352 353 if (!value)354 break;355 /* 0 is a terminator value. Check it first to avoid matching on356 * a forbidden port == 0 and returning true.357 */358 if (*value == 0)359 break;360 361 if (*value == port)362 return true;363 }364 365 return false;366}367 368struct header_pointers {369 struct ethhdr *eth;370 struct iphdr *ipv4;371 struct ipv6hdr *ipv6;372 struct tcphdr *tcp;373 __u16 tcp_len;374};375 376static __always_inline int tcp_dissect(void *data, void *data_end,377 struct header_pointers *hdr)378{379 hdr->eth = data;380 if (hdr->eth + 1 > data_end)381 return XDP_DROP;382 383 switch (bpf_ntohs(hdr->eth->h_proto)) {384 case ETH_P_IP:385 hdr->ipv6 = NULL;386 387 hdr->ipv4 = (void *)hdr->eth + sizeof(*hdr->eth);388 if (hdr->ipv4 + 1 > data_end)389 return XDP_DROP;390 if (hdr->ipv4->ihl * 4 < sizeof(*hdr->ipv4))391 return XDP_DROP;392 if (hdr->ipv4->version != 4)393 return XDP_DROP;394 395 if (hdr->ipv4->protocol != IPPROTO_TCP)396 return XDP_PASS;397 398 hdr->tcp = (void *)hdr->ipv4 + hdr->ipv4->ihl * 4;399 break;400 case ETH_P_IPV6:401 hdr->ipv4 = NULL;402 403 hdr->ipv6 = (void *)hdr->eth + sizeof(*hdr->eth);404 if (hdr->ipv6 + 1 > data_end)405 return XDP_DROP;406 if (hdr->ipv6->version != 6)407 return XDP_DROP;408 409 /* XXX: Extension headers are not supported and could circumvent410 * XDP SYN flood protection.411 */412 if (hdr->ipv6->nexthdr != NEXTHDR_TCP)413 return XDP_PASS;414 415 hdr->tcp = (void *)hdr->ipv6 + sizeof(*hdr->ipv6);416 break;417 default:418 /* XXX: VLANs will circumvent XDP SYN flood protection. */419 return XDP_PASS;420 }421 422 if (hdr->tcp + 1 > data_end)423 return XDP_DROP;424 hdr->tcp_len = hdr->tcp->doff * 4;425 if (hdr->tcp_len < sizeof(*hdr->tcp))426 return XDP_DROP;427 428 return XDP_TX;429}430 431static __always_inline int tcp_lookup(void *ctx, struct header_pointers *hdr, bool xdp)432{433 struct bpf_ct_opts___local ct_lookup_opts = {434 .netns_id = BPF_F_CURRENT_NETNS,435 .l4proto = IPPROTO_TCP,436 };437 struct bpf_sock_tuple tup = {};438 struct nf_conn *ct;439 __u32 tup_size;440 441 if (hdr->ipv4) {442 /* TCP doesn't normally use fragments, and XDP can't reassemble443 * them.444 */445 if ((hdr->ipv4->frag_off & bpf_htons(IP_DF | IP_MF | IP_OFFSET)) != bpf_htons(IP_DF))446 return XDP_DROP;447 448 tup.ipv4.saddr = hdr->ipv4->saddr;449 tup.ipv4.daddr = hdr->ipv4->daddr;450 tup.ipv4.sport = hdr->tcp->source;451 tup.ipv4.dport = hdr->tcp->dest;452 tup_size = sizeof(tup.ipv4);453 } else if (hdr->ipv6) {454 __builtin_memcpy(tup.ipv6.saddr, &hdr->ipv6->saddr, sizeof(tup.ipv6.saddr));455 __builtin_memcpy(tup.ipv6.daddr, &hdr->ipv6->daddr, sizeof(tup.ipv6.daddr));456 tup.ipv6.sport = hdr->tcp->source;457 tup.ipv6.dport = hdr->tcp->dest;458 tup_size = sizeof(tup.ipv6);459 } else {460 /* The verifier can't track that either ipv4 or ipv6 is not461 * NULL.462 */463 return XDP_ABORTED;464 }465 if (xdp)466 ct = bpf_xdp_ct_lookup(ctx, &tup, tup_size, &ct_lookup_opts, sizeof(ct_lookup_opts));467 else468 ct = bpf_skb_ct_lookup(ctx, &tup, tup_size, &ct_lookup_opts, sizeof(ct_lookup_opts));469 if (ct) {470 unsigned long status = ct->status;471 472 bpf_ct_release(ct);473 if (status & IPS_CONFIRMED)474 return XDP_PASS;475 } else if (ct_lookup_opts.error != -ENOENT) {476 return XDP_ABORTED;477 }478 479 /* error == -ENOENT || !(status & IPS_CONFIRMED) */480 return XDP_TX;481}482 483static __always_inline __u8 tcp_mkoptions(__be32 *buf, __be32 *tsopt, __u16 mss,484 __u8 wscale)485{486 __be32 *start = buf;487 488 *buf++ = bpf_htonl((TCPOPT_MSS << 24) | (TCPOLEN_MSS << 16) | mss);489 490 if (!tsopt)491 return buf - start;492 493 if (tsopt[0] & bpf_htonl(1 << 4))494 *buf++ = bpf_htonl((TCPOPT_SACK_PERM << 24) |495 (TCPOLEN_SACK_PERM << 16) |496 (TCPOPT_TIMESTAMP << 8) |497 TCPOLEN_TIMESTAMP);498 else499 *buf++ = bpf_htonl((TCPOPT_NOP << 24) |500 (TCPOPT_NOP << 16) |501 (TCPOPT_TIMESTAMP << 8) |502 TCPOLEN_TIMESTAMP);503 *buf++ = tsopt[0];504 *buf++ = tsopt[1];505 506 if ((tsopt[0] & bpf_htonl(0xf)) != bpf_htonl(0xf))507 *buf++ = bpf_htonl((TCPOPT_NOP << 24) |508 (TCPOPT_WINDOW << 16) |509 (TCPOLEN_WINDOW << 8) |510 wscale);511 512 return buf - start;513}514 515static __always_inline void tcp_gen_synack(struct tcphdr *tcp_header,516 __u32 cookie, __be32 *tsopt,517 __u16 mss, __u8 wscale)518{519 void *tcp_options;520 521 tcp_flag_word(tcp_header) = TCP_FLAG_SYN | TCP_FLAG_ACK;522 if (tsopt && (tsopt[0] & bpf_htonl(1 << 5)))523 tcp_flag_word(tcp_header) |= TCP_FLAG_ECE;524 tcp_header->doff = 5; /* doff is part of tcp_flag_word. */525 swap(tcp_header->source, tcp_header->dest);526 tcp_header->ack_seq = bpf_htonl(bpf_ntohl(tcp_header->seq) + 1);527 tcp_header->seq = bpf_htonl(cookie);528 tcp_header->window = 0;529 tcp_header->urg_ptr = 0;530 tcp_header->check = 0; /* Calculate checksum later. */531 532 tcp_options = (void *)(tcp_header + 1);533 tcp_header->doff += tcp_mkoptions(tcp_options, tsopt, mss, wscale);534}535 536static __always_inline void tcpv4_gen_synack(struct header_pointers *hdr,537 __u32 cookie, __be32 *tsopt)538{539 __u8 wscale;540 __u16 mss;541 __u8 ttl;542 543 values_get_tcpipopts(&mss, &wscale, &ttl, false);544 545 swap_eth_addr(hdr->eth->h_source, hdr->eth->h_dest);546 547 swap(hdr->ipv4->saddr, hdr->ipv4->daddr);548 hdr->ipv4->check = 0; /* Calculate checksum later. */549 hdr->ipv4->tos = 0;550 hdr->ipv4->id = 0;551 hdr->ipv4->ttl = ttl;552 553 tcp_gen_synack(hdr->tcp, cookie, tsopt, mss, wscale);554 555 hdr->tcp_len = hdr->tcp->doff * 4;556 hdr->ipv4->tot_len = bpf_htons(sizeof(*hdr->ipv4) + hdr->tcp_len);557}558 559static __always_inline void tcpv6_gen_synack(struct header_pointers *hdr,560 __u32 cookie, __be32 *tsopt)561{562 __u8 wscale;563 __u16 mss;564 __u8 ttl;565 566 values_get_tcpipopts(&mss, &wscale, &ttl, true);567 568 swap_eth_addr(hdr->eth->h_source, hdr->eth->h_dest);569 570 swap(hdr->ipv6->saddr, hdr->ipv6->daddr);571 *(__be32 *)hdr->ipv6 = bpf_htonl(0x60000000);572 hdr->ipv6->hop_limit = ttl;573 574 tcp_gen_synack(hdr->tcp, cookie, tsopt, mss, wscale);575 576 hdr->tcp_len = hdr->tcp->doff * 4;577 hdr->ipv6->payload_len = bpf_htons(hdr->tcp_len);578}579 580static __always_inline int syncookie_handle_syn(struct header_pointers *hdr,581 void *ctx,582 void *data, void *data_end,583 bool xdp)584{585 __u32 old_pkt_size, new_pkt_size;586 /* Unlike clang 10, clang 11 and 12 generate code that doesn't pass the587 * BPF verifier if tsopt is not volatile. Volatile forces it to store588 * the pointer value and use it directly, otherwise tcp_mkoptions is589 * (mis)compiled like this:590 * if (!tsopt)591 * return buf - start;592 * reg = stored_return_value_of_tscookie_init;593 * if (reg)594 * tsopt = tsopt_buf;595 * else596 * tsopt = NULL;597 * ...598 * *buf++ = tsopt[1];599 * It creates a dead branch where tsopt is assigned NULL, but the600 * verifier can't prove it's dead and blocks the program.601 */602 __be32 * volatile tsopt = NULL;603 __be32 tsopt_buf[2] = {};604 __u16 ip_len;605 __u32 cookie;606 __s64 value;607 608 /* Checksum is not yet verified, but both checksum failure and TCP609 * header checks return XDP_DROP, so the order doesn't matter.610 */611 if (hdr->tcp->fin || hdr->tcp->rst)612 return XDP_DROP;613 614 /* Issue SYN cookies on allowed ports, drop SYN packets on blocked615 * ports.616 */617 if (!check_port_allowed(bpf_ntohs(hdr->tcp->dest)))618 return XDP_DROP;619 620 if (hdr->ipv4) {621 /* Check the IPv4 and TCP checksums before creating a SYNACK. */622 value = bpf_csum_diff(0, 0, (void *)hdr->ipv4, hdr->ipv4->ihl * 4, 0);623 if (value < 0)624 return XDP_ABORTED;625 if (csum_fold(value) != 0)626 return XDP_DROP; /* Bad IPv4 checksum. */627 628 value = bpf_csum_diff(0, 0, (void *)hdr->tcp, hdr->tcp_len, 0);629 if (value < 0)630 return XDP_ABORTED;631 if (csum_tcpudp_magic(hdr->ipv4->saddr, hdr->ipv4->daddr,632 hdr->tcp_len, IPPROTO_TCP, value) != 0)633 return XDP_DROP; /* Bad TCP checksum. */634 635 ip_len = sizeof(*hdr->ipv4);636 637 value = bpf_tcp_raw_gen_syncookie_ipv4(hdr->ipv4, hdr->tcp,638 hdr->tcp_len);639 } else if (hdr->ipv6) {640 /* Check the TCP checksum before creating a SYNACK. */641 value = bpf_csum_diff(0, 0, (void *)hdr->tcp, hdr->tcp_len, 0);642 if (value < 0)643 return XDP_ABORTED;644 if (csum_ipv6_magic(&hdr->ipv6->saddr, &hdr->ipv6->daddr,645 hdr->tcp_len, IPPROTO_TCP, value) != 0)646 return XDP_DROP; /* Bad TCP checksum. */647 648 ip_len = sizeof(*hdr->ipv6);649 650 value = bpf_tcp_raw_gen_syncookie_ipv6(hdr->ipv6, hdr->tcp,651 hdr->tcp_len);652 } else {653 return XDP_ABORTED;654 }655 656 if (value < 0)657 return XDP_ABORTED;658 cookie = (__u32)value;659 660 if (tscookie_init((void *)hdr->tcp, hdr->tcp_len,661 &tsopt_buf[0], &tsopt_buf[1], data, data_end))662 tsopt = tsopt_buf;663 664 /* Check that there is enough space for a SYNACK. It also covers665 * the check that the destination of the __builtin_memmove below666 * doesn't overflow.667 */668 if (data + sizeof(*hdr->eth) + ip_len + TCP_MAXLEN > data_end)669 return XDP_ABORTED;670 671 if (hdr->ipv4) {672 if (hdr->ipv4->ihl * 4 > sizeof(*hdr->ipv4)) {673 struct tcphdr *new_tcp_header;674 675 new_tcp_header = data + sizeof(*hdr->eth) + sizeof(*hdr->ipv4);676 __builtin_memmove(new_tcp_header, hdr->tcp, sizeof(*hdr->tcp));677 hdr->tcp = new_tcp_header;678 679 hdr->ipv4->ihl = sizeof(*hdr->ipv4) / 4;680 }681 682 tcpv4_gen_synack(hdr, cookie, tsopt);683 } else if (hdr->ipv6) {684 tcpv6_gen_synack(hdr, cookie, tsopt);685 } else {686 return XDP_ABORTED;687 }688 689 /* Recalculate checksums. */690 hdr->tcp->check = 0;691 value = bpf_csum_diff(0, 0, (void *)hdr->tcp, hdr->tcp_len, 0);692 if (value < 0)693 return XDP_ABORTED;694 if (hdr->ipv4) {695 hdr->tcp->check = csum_tcpudp_magic(hdr->ipv4->saddr,696 hdr->ipv4->daddr,697 hdr->tcp_len,698 IPPROTO_TCP,699 value);700 701 hdr->ipv4->check = 0;702 value = bpf_csum_diff(0, 0, (void *)hdr->ipv4, sizeof(*hdr->ipv4), 0);703 if (value < 0)704 return XDP_ABORTED;705 hdr->ipv4->check = csum_fold(value);706 } else if (hdr->ipv6) {707 hdr->tcp->check = csum_ipv6_magic(&hdr->ipv6->saddr,708 &hdr->ipv6->daddr,709 hdr->tcp_len,710 IPPROTO_TCP,711 value);712 } else {713 return XDP_ABORTED;714 }715 716 /* Set the new packet size. */717 old_pkt_size = data_end - data;718 new_pkt_size = sizeof(*hdr->eth) + ip_len + hdr->tcp->doff * 4;719 if (xdp) {720 if (bpf_xdp_adjust_tail(ctx, new_pkt_size - old_pkt_size))721 return XDP_ABORTED;722 } else {723 if (bpf_skb_change_tail(ctx, new_pkt_size, 0))724 return XDP_ABORTED;725 }726 727 values_inc_synacks();728 729 return XDP_TX;730}731 732static __always_inline int syncookie_handle_ack(struct header_pointers *hdr)733{734 int err;735 736 if (hdr->tcp->rst)737 return XDP_DROP;738 739 if (hdr->ipv4)740 err = bpf_tcp_raw_check_syncookie_ipv4(hdr->ipv4, hdr->tcp);741 else if (hdr->ipv6)742 err = bpf_tcp_raw_check_syncookie_ipv6(hdr->ipv6, hdr->tcp);743 else744 return XDP_ABORTED;745 if (err)746 return XDP_DROP;747 748 return XDP_PASS;749}750 751static __always_inline int syncookie_part1(void *ctx, void *data, void *data_end,752 struct header_pointers *hdr, bool xdp)753{754 int ret;755 756 ret = tcp_dissect(data, data_end, hdr);757 if (ret != XDP_TX)758 return ret;759 760 ret = tcp_lookup(ctx, hdr, xdp);761 if (ret != XDP_TX)762 return ret;763 764 /* Packet is TCP and doesn't belong to an established connection. */765 766 if ((hdr->tcp->syn ^ hdr->tcp->ack) != 1)767 return XDP_DROP;768 769 /* Grow the TCP header to TCP_MAXLEN to be able to pass any hdr->tcp_len770 * to bpf_tcp_raw_gen_syncookie_ipv{4,6} and pass the verifier.771 */772 if (xdp) {773 if (bpf_xdp_adjust_tail(ctx, TCP_MAXLEN - hdr->tcp_len))774 return XDP_ABORTED;775 } else {776 /* Without volatile the verifier throws this error:777 * R9 32-bit pointer arithmetic prohibited778 */779 volatile u64 old_len = data_end - data;780 781 if (bpf_skb_change_tail(ctx, old_len + TCP_MAXLEN - hdr->tcp_len, 0))782 return XDP_ABORTED;783 }784 785 return XDP_TX;786}787 788static __always_inline int syncookie_part2(void *ctx, void *data, void *data_end,789 struct header_pointers *hdr, bool xdp)790{791 if (hdr->ipv4) {792 hdr->eth = data;793 hdr->ipv4 = (void *)hdr->eth + sizeof(*hdr->eth);794 /* IPV4_MAXLEN is needed when calculating checksum.795 * At least sizeof(struct iphdr) is needed here to access ihl.796 */797 if ((void *)hdr->ipv4 + IPV4_MAXLEN > data_end)798 return XDP_ABORTED;799 hdr->tcp = (void *)hdr->ipv4 + hdr->ipv4->ihl * 4;800 } else if (hdr->ipv6) {801 hdr->eth = data;802 hdr->ipv6 = (void *)hdr->eth + sizeof(*hdr->eth);803 hdr->tcp = (void *)hdr->ipv6 + sizeof(*hdr->ipv6);804 } else {805 return XDP_ABORTED;806 }807 808 if ((void *)hdr->tcp + TCP_MAXLEN > data_end)809 return XDP_ABORTED;810 811 /* We run out of registers, tcp_len gets spilled to the stack, and the812 * verifier forgets its min and max values checked above in tcp_dissect.813 */814 hdr->tcp_len = hdr->tcp->doff * 4;815 if (hdr->tcp_len < sizeof(*hdr->tcp))816 return XDP_ABORTED;817 818 return hdr->tcp->syn ? syncookie_handle_syn(hdr, ctx, data, data_end, xdp) :819 syncookie_handle_ack(hdr);820}821 822SEC("xdp")823int syncookie_xdp(struct xdp_md *ctx)824{825 void *data_end = (void *)(long)ctx->data_end;826 void *data = (void *)(long)ctx->data;827 struct header_pointers hdr;828 int ret;829 830 ret = syncookie_part1(ctx, data, data_end, &hdr, true);831 if (ret != XDP_TX)832 return ret;833 834 data_end = (void *)(long)ctx->data_end;835 data = (void *)(long)ctx->data;836 837 return syncookie_part2(ctx, data, data_end, &hdr, true);838}839 840SEC("tc")841int syncookie_tc(struct __sk_buff *skb)842{843 void *data_end = (void *)(long)skb->data_end;844 void *data = (void *)(long)skb->data;845 struct header_pointers hdr;846 int ret;847 848 ret = syncookie_part1(skb, data, data_end, &hdr, false);849 if (ret != XDP_TX)850 return ret == XDP_PASS ? TC_ACT_OK : TC_ACT_SHOT;851 852 data_end = (void *)(long)skb->data_end;853 data = (void *)(long)skb->data;854 855 ret = syncookie_part2(skb, data, data_end, &hdr, false);856 switch (ret) {857 case XDP_PASS:858 return TC_ACT_OK;859 case XDP_TX:860 return bpf_redirect(skb->ifindex, 0);861 default:862 return TC_ACT_SHOT;863 }864}865 866char _license[] SEC("license") = "GPL";867