627 lines · c
1// SPDX-License-Identifier: GPL-2.02/* Copyright (c) 2020 Facebook */3 4#include <stddef.h>5#include <errno.h>6#include <stdbool.h>7#include <sys/types.h>8#include <sys/socket.h>9#include <linux/tcp.h>10#include <linux/socket.h>11#include <linux/bpf.h>12#include <linux/types.h>13#include <bpf/bpf_helpers.h>14#include <bpf/bpf_endian.h>15#define BPF_PROG_TEST_TCP_HDR_OPTIONS16#include "test_tcp_hdr_options.h"17 18#ifndef sizeof_field19#define sizeof_field(TYPE, MEMBER) sizeof((((TYPE *)0)->MEMBER))20#endif21 22__u8 test_kind = TCPOPT_EXP;23__u16 test_magic = 0xeB9F;24__u32 inherit_cb_flags = 0;25 26struct bpf_test_option passive_synack_out = {};27struct bpf_test_option passive_fin_out = {};28 29struct bpf_test_option passive_estab_in = {};30struct bpf_test_option passive_fin_in = {};31 32struct bpf_test_option active_syn_out = {};33struct bpf_test_option active_fin_out = {};34 35struct bpf_test_option active_estab_in = {};36struct bpf_test_option active_fin_in = {};37 38struct {39 __uint(type, BPF_MAP_TYPE_SK_STORAGE);40 __uint(map_flags, BPF_F_NO_PREALLOC);41 __type(key, int);42 __type(value, struct hdr_stg);43} hdr_stg_map SEC(".maps");44 45static bool skops_want_cookie(const struct bpf_sock_ops *skops)46{47 return skops->args[0] == BPF_WRITE_HDR_TCP_SYNACK_COOKIE;48}49 50static bool skops_current_mss(const struct bpf_sock_ops *skops)51{52 return skops->args[0] == BPF_WRITE_HDR_TCP_CURRENT_MSS;53}54 55static __u8 option_total_len(__u8 flags)56{57 __u8 i, len = 1; /* +1 for flags */58 59 if (!flags)60 return 0;61 62 /* RESEND bit does not use a byte */63 for (i = OPTION_RESEND + 1; i < __NR_OPTION_FLAGS; i++)64 len += !!TEST_OPTION_FLAGS(flags, i);65 66 if (test_kind == TCPOPT_EXP)67 return len + TCP_BPF_EXPOPT_BASE_LEN;68 else69 return len + 2; /* +1 kind, +1 kind-len */70}71 72static void write_test_option(const struct bpf_test_option *test_opt,73 __u8 *data)74{75 __u8 offset = 0;76 77 data[offset++] = test_opt->flags;78 if (TEST_OPTION_FLAGS(test_opt->flags, OPTION_MAX_DELACK_MS))79 data[offset++] = test_opt->max_delack_ms;80 81 if (TEST_OPTION_FLAGS(test_opt->flags, OPTION_RAND))82 data[offset++] = test_opt->rand;83}84 85static int store_option(struct bpf_sock_ops *skops,86 const struct bpf_test_option *test_opt)87{88 union {89 struct tcp_exprm_opt exprm;90 struct tcp_opt regular;91 } write_opt;92 int err;93 94 if (test_kind == TCPOPT_EXP) {95 write_opt.exprm.kind = TCPOPT_EXP;96 write_opt.exprm.len = option_total_len(test_opt->flags);97 write_opt.exprm.magic = __bpf_htons(test_magic);98 write_opt.exprm.data32 = 0;99 write_test_option(test_opt, write_opt.exprm.data);100 err = bpf_store_hdr_opt(skops, &write_opt.exprm,101 sizeof(write_opt.exprm), 0);102 } else {103 write_opt.regular.kind = test_kind;104 write_opt.regular.len = option_total_len(test_opt->flags);105 write_opt.regular.data32 = 0;106 write_test_option(test_opt, write_opt.regular.data);107 err = bpf_store_hdr_opt(skops, &write_opt.regular,108 sizeof(write_opt.regular), 0);109 }110 111 if (err)112 RET_CG_ERR(err);113 114 return CG_OK;115}116 117static int parse_test_option(struct bpf_test_option *opt, const __u8 *start)118{119 opt->flags = *start++;120 121 if (TEST_OPTION_FLAGS(opt->flags, OPTION_MAX_DELACK_MS))122 opt->max_delack_ms = *start++;123 124 if (TEST_OPTION_FLAGS(opt->flags, OPTION_RAND))125 opt->rand = *start++;126 127 return 0;128}129 130static int load_option(struct bpf_sock_ops *skops,131 struct bpf_test_option *test_opt, bool from_syn)132{133 union {134 struct tcp_exprm_opt exprm;135 struct tcp_opt regular;136 } search_opt;137 int ret, load_flags = from_syn ? BPF_LOAD_HDR_OPT_TCP_SYN : 0;138 139 if (test_kind == TCPOPT_EXP) {140 search_opt.exprm.kind = TCPOPT_EXP;141 search_opt.exprm.len = 4;142 search_opt.exprm.magic = __bpf_htons(test_magic);143 search_opt.exprm.data32 = 0;144 ret = bpf_load_hdr_opt(skops, &search_opt.exprm,145 sizeof(search_opt.exprm), load_flags);146 if (ret < 0)147 return ret;148 return parse_test_option(test_opt, search_opt.exprm.data);149 } else {150 search_opt.regular.kind = test_kind;151 search_opt.regular.len = 0;152 search_opt.regular.data32 = 0;153 ret = bpf_load_hdr_opt(skops, &search_opt.regular,154 sizeof(search_opt.regular), load_flags);155 if (ret < 0)156 return ret;157 return parse_test_option(test_opt, search_opt.regular.data);158 }159}160 161static int synack_opt_len(struct bpf_sock_ops *skops)162{163 struct bpf_test_option test_opt = {};164 __u8 optlen;165 int err;166 167 if (!passive_synack_out.flags)168 return CG_OK;169 170 err = load_option(skops, &test_opt, true);171 172 /* bpf_test_option is not found */173 if (err == -ENOMSG)174 return CG_OK;175 176 if (err)177 RET_CG_ERR(err);178 179 optlen = option_total_len(passive_synack_out.flags);180 if (optlen) {181 err = bpf_reserve_hdr_opt(skops, optlen, 0);182 if (err)183 RET_CG_ERR(err);184 }185 186 return CG_OK;187}188 189static int write_synack_opt(struct bpf_sock_ops *skops)190{191 struct bpf_test_option opt;192 193 if (!passive_synack_out.flags)194 /* We should not even be called since no header195 * space has been reserved.196 */197 RET_CG_ERR(0);198 199 opt = passive_synack_out;200 if (skops_want_cookie(skops))201 SET_OPTION_FLAGS(opt.flags, OPTION_RESEND);202 203 return store_option(skops, &opt);204}205 206static int syn_opt_len(struct bpf_sock_ops *skops)207{208 __u8 optlen;209 int err;210 211 if (!active_syn_out.flags)212 return CG_OK;213 214 optlen = option_total_len(active_syn_out.flags);215 if (optlen) {216 err = bpf_reserve_hdr_opt(skops, optlen, 0);217 if (err)218 RET_CG_ERR(err);219 }220 221 return CG_OK;222}223 224static int write_syn_opt(struct bpf_sock_ops *skops)225{226 if (!active_syn_out.flags)227 RET_CG_ERR(0);228 229 return store_option(skops, &active_syn_out);230}231 232static int fin_opt_len(struct bpf_sock_ops *skops)233{234 struct bpf_test_option *opt;235 struct hdr_stg *hdr_stg;236 __u8 optlen;237 int err;238 239 if (!skops->sk)240 RET_CG_ERR(0);241 242 hdr_stg = bpf_sk_storage_get(&hdr_stg_map, skops->sk, NULL, 0);243 if (!hdr_stg)244 RET_CG_ERR(0);245 246 if (hdr_stg->active)247 opt = &active_fin_out;248 else249 opt = &passive_fin_out;250 251 optlen = option_total_len(opt->flags);252 if (optlen) {253 err = bpf_reserve_hdr_opt(skops, optlen, 0);254 if (err)255 RET_CG_ERR(err);256 }257 258 return CG_OK;259}260 261static int write_fin_opt(struct bpf_sock_ops *skops)262{263 struct bpf_test_option *opt;264 struct hdr_stg *hdr_stg;265 266 if (!skops->sk)267 RET_CG_ERR(0);268 269 hdr_stg = bpf_sk_storage_get(&hdr_stg_map, skops->sk, NULL, 0);270 if (!hdr_stg)271 RET_CG_ERR(0);272 273 if (hdr_stg->active)274 opt = &active_fin_out;275 else276 opt = &passive_fin_out;277 278 if (!opt->flags)279 RET_CG_ERR(0);280 281 return store_option(skops, opt);282}283 284static int resend_in_ack(struct bpf_sock_ops *skops)285{286 struct hdr_stg *hdr_stg;287 288 if (!skops->sk)289 return -1;290 291 hdr_stg = bpf_sk_storage_get(&hdr_stg_map, skops->sk, NULL, 0);292 if (!hdr_stg)293 return -1;294 295 return !!hdr_stg->resend_syn;296}297 298static int nodata_opt_len(struct bpf_sock_ops *skops)299{300 int resend;301 302 resend = resend_in_ack(skops);303 if (resend < 0)304 RET_CG_ERR(0);305 306 if (resend)307 return syn_opt_len(skops);308 309 return CG_OK;310}311 312static int write_nodata_opt(struct bpf_sock_ops *skops)313{314 int resend;315 316 resend = resend_in_ack(skops);317 if (resend < 0)318 RET_CG_ERR(0);319 320 if (resend)321 return write_syn_opt(skops);322 323 return CG_OK;324}325 326static int data_opt_len(struct bpf_sock_ops *skops)327{328 /* Same as the nodata version. Mostly to show329 * an example usage on skops->skb_len.330 */331 return nodata_opt_len(skops);332}333 334static int write_data_opt(struct bpf_sock_ops *skops)335{336 return write_nodata_opt(skops);337}338 339static int current_mss_opt_len(struct bpf_sock_ops *skops)340{341 /* Reserve maximum that may be needed */342 int err;343 344 err = bpf_reserve_hdr_opt(skops, option_total_len(OPTION_MASK), 0);345 if (err)346 RET_CG_ERR(err);347 348 return CG_OK;349}350 351static int handle_hdr_opt_len(struct bpf_sock_ops *skops)352{353 __u8 tcp_flags = skops_tcp_flags(skops);354 355 if ((tcp_flags & TCPHDR_SYNACK) == TCPHDR_SYNACK)356 return synack_opt_len(skops);357 358 if (tcp_flags & TCPHDR_SYN)359 return syn_opt_len(skops);360 361 if (tcp_flags & TCPHDR_FIN)362 return fin_opt_len(skops);363 364 if (skops_current_mss(skops))365 /* The kernel is calculating the MSS */366 return current_mss_opt_len(skops);367 368 if (skops->skb_len)369 return data_opt_len(skops);370 371 return nodata_opt_len(skops);372}373 374static int handle_write_hdr_opt(struct bpf_sock_ops *skops)375{376 __u8 tcp_flags = skops_tcp_flags(skops);377 struct tcphdr *th;378 379 if ((tcp_flags & TCPHDR_SYNACK) == TCPHDR_SYNACK)380 return write_synack_opt(skops);381 382 if (tcp_flags & TCPHDR_SYN)383 return write_syn_opt(skops);384 385 if (tcp_flags & TCPHDR_FIN)386 return write_fin_opt(skops);387 388 th = skops->skb_data;389 if (th + 1 > skops->skb_data_end)390 RET_CG_ERR(0);391 392 if (skops->skb_len > tcp_hdrlen(th))393 return write_data_opt(skops);394 395 return write_nodata_opt(skops);396}397 398static int set_delack_max(struct bpf_sock_ops *skops, __u8 max_delack_ms)399{400 __u32 max_delack_us = max_delack_ms * 1000;401 402 return bpf_setsockopt(skops, SOL_TCP, TCP_BPF_DELACK_MAX,403 &max_delack_us, sizeof(max_delack_us));404}405 406static int set_rto_min(struct bpf_sock_ops *skops, __u8 peer_max_delack_ms)407{408 __u32 min_rto_us = peer_max_delack_ms * 1000;409 410 return bpf_setsockopt(skops, SOL_TCP, TCP_BPF_RTO_MIN, &min_rto_us,411 sizeof(min_rto_us));412}413 414static int handle_active_estab(struct bpf_sock_ops *skops)415{416 struct hdr_stg init_stg = {417 .active = true,418 };419 int err;420 421 err = load_option(skops, &active_estab_in, false);422 if (err && err != -ENOMSG)423 RET_CG_ERR(err);424 425 init_stg.resend_syn = TEST_OPTION_FLAGS(active_estab_in.flags,426 OPTION_RESEND);427 if (!skops->sk || !bpf_sk_storage_get(&hdr_stg_map, skops->sk,428 &init_stg,429 BPF_SK_STORAGE_GET_F_CREATE))430 RET_CG_ERR(0);431 432 if (init_stg.resend_syn)433 /* Don't clear the write_hdr cb now because434 * the ACK may get lost and retransmit may435 * be needed.436 *437 * PARSE_ALL_HDR cb flag is set to learn if this438 * resend_syn option has received by the peer.439 *440 * The header option will be resent until a valid441 * packet is received at handle_parse_hdr()442 * and all hdr cb flags will be cleared in443 * handle_parse_hdr().444 */445 set_parse_all_hdr_cb_flags(skops);446 else if (!active_fin_out.flags)447 /* No options will be written from now */448 clear_hdr_cb_flags(skops);449 450 if (active_syn_out.max_delack_ms) {451 err = set_delack_max(skops, active_syn_out.max_delack_ms);452 if (err)453 RET_CG_ERR(err);454 }455 456 if (active_estab_in.max_delack_ms) {457 err = set_rto_min(skops, active_estab_in.max_delack_ms);458 if (err)459 RET_CG_ERR(err);460 }461 462 return CG_OK;463}464 465static int handle_passive_estab(struct bpf_sock_ops *skops)466{467 struct hdr_stg init_stg = {};468 struct tcphdr *th;469 int err;470 471 inherit_cb_flags = skops->bpf_sock_ops_cb_flags;472 473 err = load_option(skops, &passive_estab_in, true);474 if (err == -ENOENT) {475 /* saved_syn is not found. It was in syncookie mode.476 * We have asked the active side to resend the options477 * in ACK, so try to find the bpf_test_option from ACK now.478 */479 err = load_option(skops, &passive_estab_in, false);480 init_stg.syncookie = true;481 }482 483 /* ENOMSG: The bpf_test_option is not found which is fine.484 * Bail out now for all other errors.485 */486 if (err && err != -ENOMSG)487 RET_CG_ERR(err);488 489 th = skops->skb_data;490 if (th + 1 > skops->skb_data_end)491 RET_CG_ERR(0);492 493 if (th->syn) {494 /* Fastopen */495 496 /* Cannot clear cb_flags to stop write_hdr cb.497 * synack is not sent yet for fast open.498 * Even it was, the synack may need to be retransmitted.499 *500 * PARSE_ALL_HDR cb flag is set to learn501 * if synack has reached the peer.502 * All cb_flags will be cleared in handle_parse_hdr().503 */504 set_parse_all_hdr_cb_flags(skops);505 init_stg.fastopen = true;506 } else if (!passive_fin_out.flags) {507 /* No options will be written from now */508 clear_hdr_cb_flags(skops);509 }510 511 if (!skops->sk ||512 !bpf_sk_storage_get(&hdr_stg_map, skops->sk, &init_stg,513 BPF_SK_STORAGE_GET_F_CREATE))514 RET_CG_ERR(0);515 516 if (passive_synack_out.max_delack_ms) {517 err = set_delack_max(skops, passive_synack_out.max_delack_ms);518 if (err)519 RET_CG_ERR(err);520 }521 522 if (passive_estab_in.max_delack_ms) {523 err = set_rto_min(skops, passive_estab_in.max_delack_ms);524 if (err)525 RET_CG_ERR(err);526 }527 528 return CG_OK;529}530 531static int handle_parse_hdr(struct bpf_sock_ops *skops)532{533 struct hdr_stg *hdr_stg;534 struct tcphdr *th;535 536 if (!skops->sk)537 RET_CG_ERR(0);538 539 th = skops->skb_data;540 if (th + 1 > skops->skb_data_end)541 RET_CG_ERR(0);542 543 hdr_stg = bpf_sk_storage_get(&hdr_stg_map, skops->sk, NULL, 0);544 if (!hdr_stg)545 RET_CG_ERR(0);546 547 if (hdr_stg->resend_syn || hdr_stg->fastopen)548 /* The PARSE_ALL_HDR cb flag was turned on549 * to ensure that the previously written550 * options have reached the peer.551 * Those previously written option includes:552 * - Active side: resend_syn in ACK during syncookie553 * or554 * - Passive side: SYNACK during fastopen555 *556 * A valid packet has been received here after557 * the 3WHS, so the PARSE_ALL_HDR cb flag558 * can be cleared now.559 */560 clear_parse_all_hdr_cb_flags(skops);561 562 if (hdr_stg->resend_syn && !active_fin_out.flags)563 /* Active side resent the syn option in ACK564 * because the server was in syncookie mode.565 * A valid packet has been received, so566 * clear header cb flags if there is no567 * more option to send.568 */569 clear_hdr_cb_flags(skops);570 571 if (hdr_stg->fastopen && !passive_fin_out.flags)572 /* Passive side was in fastopen.573 * A valid packet has been received, so574 * the SYNACK has reached the peer.575 * Clear header cb flags if there is no more576 * option to send.577 */578 clear_hdr_cb_flags(skops);579 580 if (th->fin) {581 struct bpf_test_option *fin_opt;582 int err;583 584 if (hdr_stg->active)585 fin_opt = &active_fin_in;586 else587 fin_opt = &passive_fin_in;588 589 err = load_option(skops, fin_opt, false);590 if (err && err != -ENOMSG)591 RET_CG_ERR(err);592 }593 594 return CG_OK;595}596 597SEC("sockops")598int estab(struct bpf_sock_ops *skops)599{600 int true_val = 1;601 602 switch (skops->op) {603 case BPF_SOCK_OPS_TCP_LISTEN_CB:604 bpf_setsockopt(skops, SOL_TCP, TCP_SAVE_SYN,605 &true_val, sizeof(true_val));606 set_hdr_cb_flags(skops, BPF_SOCK_OPS_STATE_CB_FLAG);607 break;608 case BPF_SOCK_OPS_TCP_CONNECT_CB:609 set_hdr_cb_flags(skops, 0);610 break;611 case BPF_SOCK_OPS_PARSE_HDR_OPT_CB:612 return handle_parse_hdr(skops);613 case BPF_SOCK_OPS_HDR_OPT_LEN_CB:614 return handle_hdr_opt_len(skops);615 case BPF_SOCK_OPS_WRITE_HDR_OPT_CB:616 return handle_write_hdr_opt(skops);617 case BPF_SOCK_OPS_PASSIVE_ESTABLISHED_CB:618 return handle_passive_estab(skops);619 case BPF_SOCK_OPS_ACTIVE_ESTABLISHED_CB:620 return handle_active_estab(skops);621 }622 623 return CG_OK;624}625 626char _license[] SEC("license") = "GPL";627