1610 lines · c
1// SPDX-License-Identifier: GPL-2.02#include <unistd.h>3#include <test_progs.h>4#include <network_helpers.h>5#include "tailcall_poke.skel.h"6#include "tailcall_bpf2bpf_hierarchy2.skel.h"7#include "tailcall_bpf2bpf_hierarchy3.skel.h"8#include "tailcall_freplace.skel.h"9#include "tc_bpf2bpf.skel.h"10 11/* test_tailcall_1 checks basic functionality by patching multiple locations12 * in a single program for a single tail call slot with nop->jmp, jmp->nop13 * and jmp->jmp rewrites. Also checks for nop->nop.14 */15static void test_tailcall_1(void)16{17 int err, map_fd, prog_fd, main_fd, i, j;18 struct bpf_map *prog_array;19 struct bpf_program *prog;20 struct bpf_object *obj;21 char prog_name[32];22 char buff[128] = {};23 LIBBPF_OPTS(bpf_test_run_opts, topts,24 .data_in = buff,25 .data_size_in = sizeof(buff),26 .repeat = 1,27 );28 29 err = bpf_prog_test_load("tailcall1.bpf.o", BPF_PROG_TYPE_SCHED_CLS, &obj,30 &prog_fd);31 if (CHECK_FAIL(err))32 return;33 34 prog = bpf_object__find_program_by_name(obj, "entry");35 if (CHECK_FAIL(!prog))36 goto out;37 38 main_fd = bpf_program__fd(prog);39 if (CHECK_FAIL(main_fd < 0))40 goto out;41 42 prog_array = bpf_object__find_map_by_name(obj, "jmp_table");43 if (CHECK_FAIL(!prog_array))44 goto out;45 46 map_fd = bpf_map__fd(prog_array);47 if (CHECK_FAIL(map_fd < 0))48 goto out;49 50 for (i = 0; i < bpf_map__max_entries(prog_array); i++) {51 snprintf(prog_name, sizeof(prog_name), "classifier_%d", i);52 53 prog = bpf_object__find_program_by_name(obj, prog_name);54 if (CHECK_FAIL(!prog))55 goto out;56 57 prog_fd = bpf_program__fd(prog);58 if (CHECK_FAIL(prog_fd < 0))59 goto out;60 61 err = bpf_map_update_elem(map_fd, &i, &prog_fd, BPF_ANY);62 if (CHECK_FAIL(err))63 goto out;64 }65 66 for (i = 0; i < bpf_map__max_entries(prog_array); i++) {67 err = bpf_prog_test_run_opts(main_fd, &topts);68 ASSERT_OK(err, "tailcall");69 ASSERT_EQ(topts.retval, i, "tailcall retval");70 71 err = bpf_map_delete_elem(map_fd, &i);72 if (CHECK_FAIL(err))73 goto out;74 }75 76 err = bpf_prog_test_run_opts(main_fd, &topts);77 ASSERT_OK(err, "tailcall");78 ASSERT_EQ(topts.retval, 3, "tailcall retval");79 80 for (i = 0; i < bpf_map__max_entries(prog_array); i++) {81 snprintf(prog_name, sizeof(prog_name), "classifier_%d", i);82 83 prog = bpf_object__find_program_by_name(obj, prog_name);84 if (CHECK_FAIL(!prog))85 goto out;86 87 prog_fd = bpf_program__fd(prog);88 if (CHECK_FAIL(prog_fd < 0))89 goto out;90 91 err = bpf_map_update_elem(map_fd, &i, &prog_fd, BPF_ANY);92 if (CHECK_FAIL(err))93 goto out;94 }95 96 err = bpf_prog_test_run_opts(main_fd, &topts);97 ASSERT_OK(err, "tailcall");98 ASSERT_OK(topts.retval, "tailcall retval");99 100 for (i = 0; i < bpf_map__max_entries(prog_array); i++) {101 j = bpf_map__max_entries(prog_array) - 1 - i;102 snprintf(prog_name, sizeof(prog_name), "classifier_%d", j);103 104 prog = bpf_object__find_program_by_name(obj, prog_name);105 if (CHECK_FAIL(!prog))106 goto out;107 108 prog_fd = bpf_program__fd(prog);109 if (CHECK_FAIL(prog_fd < 0))110 goto out;111 112 err = bpf_map_update_elem(map_fd, &i, &prog_fd, BPF_ANY);113 if (CHECK_FAIL(err))114 goto out;115 }116 117 for (i = 0; i < bpf_map__max_entries(prog_array); i++) {118 j = bpf_map__max_entries(prog_array) - 1 - i;119 120 err = bpf_prog_test_run_opts(main_fd, &topts);121 ASSERT_OK(err, "tailcall");122 ASSERT_EQ(topts.retval, j, "tailcall retval");123 124 err = bpf_map_delete_elem(map_fd, &i);125 if (CHECK_FAIL(err))126 goto out;127 }128 129 err = bpf_prog_test_run_opts(main_fd, &topts);130 ASSERT_OK(err, "tailcall");131 ASSERT_EQ(topts.retval, 3, "tailcall retval");132 133 for (i = 0; i < bpf_map__max_entries(prog_array); i++) {134 err = bpf_map_delete_elem(map_fd, &i);135 if (CHECK_FAIL(err >= 0 || errno != ENOENT))136 goto out;137 138 err = bpf_prog_test_run_opts(main_fd, &topts);139 ASSERT_OK(err, "tailcall");140 ASSERT_EQ(topts.retval, 3, "tailcall retval");141 }142 143out:144 bpf_object__close(obj);145}146 147/* test_tailcall_2 checks that patching multiple programs for a single148 * tail call slot works. It also jumps through several programs and tests149 * the tail call limit counter.150 */151static void test_tailcall_2(void)152{153 int err, map_fd, prog_fd, main_fd, i;154 struct bpf_map *prog_array;155 struct bpf_program *prog;156 struct bpf_object *obj;157 char prog_name[32];158 char buff[128] = {};159 LIBBPF_OPTS(bpf_test_run_opts, topts,160 .data_in = buff,161 .data_size_in = sizeof(buff),162 .repeat = 1,163 );164 165 err = bpf_prog_test_load("tailcall2.bpf.o", BPF_PROG_TYPE_SCHED_CLS, &obj,166 &prog_fd);167 if (CHECK_FAIL(err))168 return;169 170 prog = bpf_object__find_program_by_name(obj, "entry");171 if (CHECK_FAIL(!prog))172 goto out;173 174 main_fd = bpf_program__fd(prog);175 if (CHECK_FAIL(main_fd < 0))176 goto out;177 178 prog_array = bpf_object__find_map_by_name(obj, "jmp_table");179 if (CHECK_FAIL(!prog_array))180 goto out;181 182 map_fd = bpf_map__fd(prog_array);183 if (CHECK_FAIL(map_fd < 0))184 goto out;185 186 for (i = 0; i < bpf_map__max_entries(prog_array); i++) {187 snprintf(prog_name, sizeof(prog_name), "classifier_%d", i);188 189 prog = bpf_object__find_program_by_name(obj, prog_name);190 if (CHECK_FAIL(!prog))191 goto out;192 193 prog_fd = bpf_program__fd(prog);194 if (CHECK_FAIL(prog_fd < 0))195 goto out;196 197 err = bpf_map_update_elem(map_fd, &i, &prog_fd, BPF_ANY);198 if (CHECK_FAIL(err))199 goto out;200 }201 202 err = bpf_prog_test_run_opts(main_fd, &topts);203 ASSERT_OK(err, "tailcall");204 ASSERT_EQ(topts.retval, 2, "tailcall retval");205 206 i = 2;207 err = bpf_map_delete_elem(map_fd, &i);208 if (CHECK_FAIL(err))209 goto out;210 211 err = bpf_prog_test_run_opts(main_fd, &topts);212 ASSERT_OK(err, "tailcall");213 ASSERT_EQ(topts.retval, 1, "tailcall retval");214 215 i = 0;216 err = bpf_map_delete_elem(map_fd, &i);217 if (CHECK_FAIL(err))218 goto out;219 220 err = bpf_prog_test_run_opts(main_fd, &topts);221 ASSERT_OK(err, "tailcall");222 ASSERT_EQ(topts.retval, 3, "tailcall retval");223out:224 bpf_object__close(obj);225}226 227static void test_tailcall_count(const char *which, bool test_fentry,228 bool test_fexit)229{230 struct bpf_object *obj = NULL, *fentry_obj = NULL, *fexit_obj = NULL;231 struct bpf_link *fentry_link = NULL, *fexit_link = NULL;232 int err, map_fd, prog_fd, main_fd, data_fd, i, val;233 struct bpf_map *prog_array, *data_map;234 struct bpf_program *prog;235 char buff[128] = {};236 LIBBPF_OPTS(bpf_test_run_opts, topts,237 .data_in = buff,238 .data_size_in = sizeof(buff),239 .repeat = 1,240 );241 242 err = bpf_prog_test_load(which, BPF_PROG_TYPE_SCHED_CLS, &obj,243 &prog_fd);244 if (CHECK_FAIL(err))245 return;246 247 prog = bpf_object__find_program_by_name(obj, "entry");248 if (CHECK_FAIL(!prog))249 goto out;250 251 main_fd = bpf_program__fd(prog);252 if (CHECK_FAIL(main_fd < 0))253 goto out;254 255 prog_array = bpf_object__find_map_by_name(obj, "jmp_table");256 if (CHECK_FAIL(!prog_array))257 goto out;258 259 map_fd = bpf_map__fd(prog_array);260 if (CHECK_FAIL(map_fd < 0))261 goto out;262 263 prog = bpf_object__find_program_by_name(obj, "classifier_0");264 if (CHECK_FAIL(!prog))265 goto out;266 267 prog_fd = bpf_program__fd(prog);268 if (CHECK_FAIL(prog_fd < 0))269 goto out;270 271 i = 0;272 err = bpf_map_update_elem(map_fd, &i, &prog_fd, BPF_ANY);273 if (CHECK_FAIL(err))274 goto out;275 276 if (test_fentry) {277 fentry_obj = bpf_object__open_file("tailcall_bpf2bpf_fentry.bpf.o",278 NULL);279 if (!ASSERT_OK_PTR(fentry_obj, "open fentry_obj file"))280 goto out;281 282 prog = bpf_object__find_program_by_name(fentry_obj, "fentry");283 if (!ASSERT_OK_PTR(prog, "find fentry prog"))284 goto out;285 286 err = bpf_program__set_attach_target(prog, prog_fd,287 "subprog_tail");288 if (!ASSERT_OK(err, "set_attach_target subprog_tail"))289 goto out;290 291 err = bpf_object__load(fentry_obj);292 if (!ASSERT_OK(err, "load fentry_obj"))293 goto out;294 295 fentry_link = bpf_program__attach_trace(prog);296 if (!ASSERT_OK_PTR(fentry_link, "attach_trace"))297 goto out;298 }299 300 if (test_fexit) {301 fexit_obj = bpf_object__open_file("tailcall_bpf2bpf_fexit.bpf.o",302 NULL);303 if (!ASSERT_OK_PTR(fexit_obj, "open fexit_obj file"))304 goto out;305 306 prog = bpf_object__find_program_by_name(fexit_obj, "fexit");307 if (!ASSERT_OK_PTR(prog, "find fexit prog"))308 goto out;309 310 err = bpf_program__set_attach_target(prog, prog_fd,311 "subprog_tail");312 if (!ASSERT_OK(err, "set_attach_target subprog_tail"))313 goto out;314 315 err = bpf_object__load(fexit_obj);316 if (!ASSERT_OK(err, "load fexit_obj"))317 goto out;318 319 fexit_link = bpf_program__attach_trace(prog);320 if (!ASSERT_OK_PTR(fexit_link, "attach_trace"))321 goto out;322 }323 324 err = bpf_prog_test_run_opts(main_fd, &topts);325 ASSERT_OK(err, "tailcall");326 ASSERT_EQ(topts.retval, 1, "tailcall retval");327 328 data_map = bpf_object__find_map_by_name(obj, "tailcall.bss");329 if (CHECK_FAIL(!data_map || !bpf_map__is_internal(data_map)))330 goto out;331 332 data_fd = bpf_map__fd(data_map);333 if (CHECK_FAIL(data_fd < 0))334 goto out;335 336 i = 0;337 err = bpf_map_lookup_elem(data_fd, &i, &val);338 ASSERT_OK(err, "tailcall count");339 ASSERT_EQ(val, 33, "tailcall count");340 341 if (test_fentry) {342 data_map = bpf_object__find_map_by_name(fentry_obj, ".bss");343 if (!ASSERT_FALSE(!data_map || !bpf_map__is_internal(data_map),344 "find tailcall_bpf2bpf_fentry.bss map"))345 goto out;346 347 data_fd = bpf_map__fd(data_map);348 if (!ASSERT_FALSE(data_fd < 0,349 "find tailcall_bpf2bpf_fentry.bss map fd"))350 goto out;351 352 i = 0;353 err = bpf_map_lookup_elem(data_fd, &i, &val);354 ASSERT_OK(err, "fentry count");355 ASSERT_EQ(val, 33, "fentry count");356 }357 358 if (test_fexit) {359 data_map = bpf_object__find_map_by_name(fexit_obj, ".bss");360 if (!ASSERT_FALSE(!data_map || !bpf_map__is_internal(data_map),361 "find tailcall_bpf2bpf_fexit.bss map"))362 goto out;363 364 data_fd = bpf_map__fd(data_map);365 if (!ASSERT_FALSE(data_fd < 0,366 "find tailcall_bpf2bpf_fexit.bss map fd"))367 goto out;368 369 i = 0;370 err = bpf_map_lookup_elem(data_fd, &i, &val);371 ASSERT_OK(err, "fexit count");372 ASSERT_EQ(val, 33, "fexit count");373 }374 375 i = 0;376 err = bpf_map_delete_elem(map_fd, &i);377 if (CHECK_FAIL(err))378 goto out;379 380 err = bpf_prog_test_run_opts(main_fd, &topts);381 ASSERT_OK(err, "tailcall");382 ASSERT_OK(topts.retval, "tailcall retval");383out:384 bpf_link__destroy(fentry_link);385 bpf_link__destroy(fexit_link);386 bpf_object__close(fentry_obj);387 bpf_object__close(fexit_obj);388 bpf_object__close(obj);389}390 391/* test_tailcall_3 checks that the count value of the tail call limit392 * enforcement matches with expectations. JIT uses direct jump.393 */394static void test_tailcall_3(void)395{396 test_tailcall_count("tailcall3.bpf.o", false, false);397}398 399/* test_tailcall_6 checks that the count value of the tail call limit400 * enforcement matches with expectations. JIT uses indirect jump.401 */402static void test_tailcall_6(void)403{404 test_tailcall_count("tailcall6.bpf.o", false, false);405}406 407/* test_tailcall_4 checks that the kernel properly selects indirect jump408 * for the case where the key is not known. Latter is passed via global409 * data to select different targets we can compare return value of.410 */411static void test_tailcall_4(void)412{413 int err, map_fd, prog_fd, main_fd, data_fd, i;414 struct bpf_map *prog_array, *data_map;415 struct bpf_program *prog;416 struct bpf_object *obj;417 static const int zero = 0;418 char buff[128] = {};419 char prog_name[32];420 LIBBPF_OPTS(bpf_test_run_opts, topts,421 .data_in = buff,422 .data_size_in = sizeof(buff),423 .repeat = 1,424 );425 426 err = bpf_prog_test_load("tailcall4.bpf.o", BPF_PROG_TYPE_SCHED_CLS, &obj,427 &prog_fd);428 if (CHECK_FAIL(err))429 return;430 431 prog = bpf_object__find_program_by_name(obj, "entry");432 if (CHECK_FAIL(!prog))433 goto out;434 435 main_fd = bpf_program__fd(prog);436 if (CHECK_FAIL(main_fd < 0))437 goto out;438 439 prog_array = bpf_object__find_map_by_name(obj, "jmp_table");440 if (CHECK_FAIL(!prog_array))441 goto out;442 443 map_fd = bpf_map__fd(prog_array);444 if (CHECK_FAIL(map_fd < 0))445 goto out;446 447 data_map = bpf_object__find_map_by_name(obj, "tailcall.bss");448 if (CHECK_FAIL(!data_map || !bpf_map__is_internal(data_map)))449 goto out;450 451 data_fd = bpf_map__fd(data_map);452 if (CHECK_FAIL(data_fd < 0))453 goto out;454 455 for (i = 0; i < bpf_map__max_entries(prog_array); i++) {456 snprintf(prog_name, sizeof(prog_name), "classifier_%d", i);457 458 prog = bpf_object__find_program_by_name(obj, prog_name);459 if (CHECK_FAIL(!prog))460 goto out;461 462 prog_fd = bpf_program__fd(prog);463 if (CHECK_FAIL(prog_fd < 0))464 goto out;465 466 err = bpf_map_update_elem(map_fd, &i, &prog_fd, BPF_ANY);467 if (CHECK_FAIL(err))468 goto out;469 }470 471 for (i = 0; i < bpf_map__max_entries(prog_array); i++) {472 err = bpf_map_update_elem(data_fd, &zero, &i, BPF_ANY);473 if (CHECK_FAIL(err))474 goto out;475 476 err = bpf_prog_test_run_opts(main_fd, &topts);477 ASSERT_OK(err, "tailcall");478 ASSERT_EQ(topts.retval, i, "tailcall retval");479 }480 481 for (i = 0; i < bpf_map__max_entries(prog_array); i++) {482 err = bpf_map_update_elem(data_fd, &zero, &i, BPF_ANY);483 if (CHECK_FAIL(err))484 goto out;485 486 err = bpf_map_delete_elem(map_fd, &i);487 if (CHECK_FAIL(err))488 goto out;489 490 err = bpf_prog_test_run_opts(main_fd, &topts);491 ASSERT_OK(err, "tailcall");492 ASSERT_EQ(topts.retval, 3, "tailcall retval");493 }494out:495 bpf_object__close(obj);496}497 498/* test_tailcall_5 probes similarly to test_tailcall_4 that the kernel generates499 * an indirect jump when the keys are const but different from different branches.500 */501static void test_tailcall_5(void)502{503 int err, map_fd, prog_fd, main_fd, data_fd, i, key[] = { 1111, 1234, 5678 };504 struct bpf_map *prog_array, *data_map;505 struct bpf_program *prog;506 struct bpf_object *obj;507 static const int zero = 0;508 char buff[128] = {};509 char prog_name[32];510 LIBBPF_OPTS(bpf_test_run_opts, topts,511 .data_in = buff,512 .data_size_in = sizeof(buff),513 .repeat = 1,514 );515 516 err = bpf_prog_test_load("tailcall5.bpf.o", BPF_PROG_TYPE_SCHED_CLS, &obj,517 &prog_fd);518 if (CHECK_FAIL(err))519 return;520 521 prog = bpf_object__find_program_by_name(obj, "entry");522 if (CHECK_FAIL(!prog))523 goto out;524 525 main_fd = bpf_program__fd(prog);526 if (CHECK_FAIL(main_fd < 0))527 goto out;528 529 prog_array = bpf_object__find_map_by_name(obj, "jmp_table");530 if (CHECK_FAIL(!prog_array))531 goto out;532 533 map_fd = bpf_map__fd(prog_array);534 if (CHECK_FAIL(map_fd < 0))535 goto out;536 537 data_map = bpf_object__find_map_by_name(obj, "tailcall.bss");538 if (CHECK_FAIL(!data_map || !bpf_map__is_internal(data_map)))539 goto out;540 541 data_fd = bpf_map__fd(data_map);542 if (CHECK_FAIL(data_fd < 0))543 goto out;544 545 for (i = 0; i < bpf_map__max_entries(prog_array); i++) {546 snprintf(prog_name, sizeof(prog_name), "classifier_%d", i);547 548 prog = bpf_object__find_program_by_name(obj, prog_name);549 if (CHECK_FAIL(!prog))550 goto out;551 552 prog_fd = bpf_program__fd(prog);553 if (CHECK_FAIL(prog_fd < 0))554 goto out;555 556 err = bpf_map_update_elem(map_fd, &i, &prog_fd, BPF_ANY);557 if (CHECK_FAIL(err))558 goto out;559 }560 561 for (i = 0; i < bpf_map__max_entries(prog_array); i++) {562 err = bpf_map_update_elem(data_fd, &zero, &key[i], BPF_ANY);563 if (CHECK_FAIL(err))564 goto out;565 566 err = bpf_prog_test_run_opts(main_fd, &topts);567 ASSERT_OK(err, "tailcall");568 ASSERT_EQ(topts.retval, i, "tailcall retval");569 }570 571 for (i = 0; i < bpf_map__max_entries(prog_array); i++) {572 err = bpf_map_update_elem(data_fd, &zero, &key[i], BPF_ANY);573 if (CHECK_FAIL(err))574 goto out;575 576 err = bpf_map_delete_elem(map_fd, &i);577 if (CHECK_FAIL(err))578 goto out;579 580 err = bpf_prog_test_run_opts(main_fd, &topts);581 ASSERT_OK(err, "tailcall");582 ASSERT_EQ(topts.retval, 3, "tailcall retval");583 }584out:585 bpf_object__close(obj);586}587 588/* test_tailcall_bpf2bpf_1 purpose is to make sure that tailcalls are working589 * correctly in correlation with BPF subprograms590 */591static void test_tailcall_bpf2bpf_1(void)592{593 int err, map_fd, prog_fd, main_fd, i;594 struct bpf_map *prog_array;595 struct bpf_program *prog;596 struct bpf_object *obj;597 char prog_name[32];598 LIBBPF_OPTS(bpf_test_run_opts, topts,599 .data_in = &pkt_v4,600 .data_size_in = sizeof(pkt_v4),601 .repeat = 1,602 );603 604 err = bpf_prog_test_load("tailcall_bpf2bpf1.bpf.o", BPF_PROG_TYPE_SCHED_CLS,605 &obj, &prog_fd);606 if (CHECK_FAIL(err))607 return;608 609 prog = bpf_object__find_program_by_name(obj, "entry");610 if (CHECK_FAIL(!prog))611 goto out;612 613 main_fd = bpf_program__fd(prog);614 if (CHECK_FAIL(main_fd < 0))615 goto out;616 617 prog_array = bpf_object__find_map_by_name(obj, "jmp_table");618 if (CHECK_FAIL(!prog_array))619 goto out;620 621 map_fd = bpf_map__fd(prog_array);622 if (CHECK_FAIL(map_fd < 0))623 goto out;624 625 /* nop -> jmp */626 for (i = 0; i < bpf_map__max_entries(prog_array); i++) {627 snprintf(prog_name, sizeof(prog_name), "classifier_%d", i);628 629 prog = bpf_object__find_program_by_name(obj, prog_name);630 if (CHECK_FAIL(!prog))631 goto out;632 633 prog_fd = bpf_program__fd(prog);634 if (CHECK_FAIL(prog_fd < 0))635 goto out;636 637 err = bpf_map_update_elem(map_fd, &i, &prog_fd, BPF_ANY);638 if (CHECK_FAIL(err))639 goto out;640 }641 642 err = bpf_prog_test_run_opts(main_fd, &topts);643 ASSERT_OK(err, "tailcall");644 ASSERT_EQ(topts.retval, 1, "tailcall retval");645 646 /* jmp -> nop, call subprog that will do tailcall */647 i = 1;648 err = bpf_map_delete_elem(map_fd, &i);649 if (CHECK_FAIL(err))650 goto out;651 652 err = bpf_prog_test_run_opts(main_fd, &topts);653 ASSERT_OK(err, "tailcall");654 ASSERT_OK(topts.retval, "tailcall retval");655 656 /* make sure that subprog can access ctx and entry prog that657 * called this subprog can properly return658 */659 i = 0;660 err = bpf_map_delete_elem(map_fd, &i);661 if (CHECK_FAIL(err))662 goto out;663 664 err = bpf_prog_test_run_opts(main_fd, &topts);665 ASSERT_OK(err, "tailcall");666 ASSERT_EQ(topts.retval, sizeof(pkt_v4) * 2, "tailcall retval");667out:668 bpf_object__close(obj);669}670 671/* test_tailcall_bpf2bpf_2 checks that the count value of the tail call limit672 * enforcement matches with expectations when tailcall is preceded with673 * bpf2bpf call.674 */675static void test_tailcall_bpf2bpf_2(void)676{677 int err, map_fd, prog_fd, main_fd, data_fd, i, val;678 struct bpf_map *prog_array, *data_map;679 struct bpf_program *prog;680 struct bpf_object *obj;681 char buff[128] = {};682 LIBBPF_OPTS(bpf_test_run_opts, topts,683 .data_in = buff,684 .data_size_in = sizeof(buff),685 .repeat = 1,686 );687 688 err = bpf_prog_test_load("tailcall_bpf2bpf2.bpf.o", BPF_PROG_TYPE_SCHED_CLS,689 &obj, &prog_fd);690 if (CHECK_FAIL(err))691 return;692 693 prog = bpf_object__find_program_by_name(obj, "entry");694 if (CHECK_FAIL(!prog))695 goto out;696 697 main_fd = bpf_program__fd(prog);698 if (CHECK_FAIL(main_fd < 0))699 goto out;700 701 prog_array = bpf_object__find_map_by_name(obj, "jmp_table");702 if (CHECK_FAIL(!prog_array))703 goto out;704 705 map_fd = bpf_map__fd(prog_array);706 if (CHECK_FAIL(map_fd < 0))707 goto out;708 709 prog = bpf_object__find_program_by_name(obj, "classifier_0");710 if (CHECK_FAIL(!prog))711 goto out;712 713 prog_fd = bpf_program__fd(prog);714 if (CHECK_FAIL(prog_fd < 0))715 goto out;716 717 i = 0;718 err = bpf_map_update_elem(map_fd, &i, &prog_fd, BPF_ANY);719 if (CHECK_FAIL(err))720 goto out;721 722 err = bpf_prog_test_run_opts(main_fd, &topts);723 ASSERT_OK(err, "tailcall");724 ASSERT_EQ(topts.retval, 1, "tailcall retval");725 726 data_map = bpf_object__find_map_by_name(obj, "tailcall.bss");727 if (CHECK_FAIL(!data_map || !bpf_map__is_internal(data_map)))728 goto out;729 730 data_fd = bpf_map__fd(data_map);731 if (CHECK_FAIL(data_fd < 0))732 goto out;733 734 i = 0;735 err = bpf_map_lookup_elem(data_fd, &i, &val);736 ASSERT_OK(err, "tailcall count");737 ASSERT_EQ(val, 33, "tailcall count");738 739 i = 0;740 err = bpf_map_delete_elem(map_fd, &i);741 if (CHECK_FAIL(err))742 goto out;743 744 err = bpf_prog_test_run_opts(main_fd, &topts);745 ASSERT_OK(err, "tailcall");746 ASSERT_OK(topts.retval, "tailcall retval");747out:748 bpf_object__close(obj);749}750 751/* test_tailcall_bpf2bpf_3 checks that non-trivial amount of stack (up to752 * 256 bytes) can be used within bpf subprograms that have the tailcalls753 * in them754 */755static void test_tailcall_bpf2bpf_3(void)756{757 int err, map_fd, prog_fd, main_fd, i;758 struct bpf_map *prog_array;759 struct bpf_program *prog;760 struct bpf_object *obj;761 char prog_name[32];762 LIBBPF_OPTS(bpf_test_run_opts, topts,763 .data_in = &pkt_v4,764 .data_size_in = sizeof(pkt_v4),765 .repeat = 1,766 );767 768 err = bpf_prog_test_load("tailcall_bpf2bpf3.bpf.o", BPF_PROG_TYPE_SCHED_CLS,769 &obj, &prog_fd);770 if (CHECK_FAIL(err))771 return;772 773 prog = bpf_object__find_program_by_name(obj, "entry");774 if (CHECK_FAIL(!prog))775 goto out;776 777 main_fd = bpf_program__fd(prog);778 if (CHECK_FAIL(main_fd < 0))779 goto out;780 781 prog_array = bpf_object__find_map_by_name(obj, "jmp_table");782 if (CHECK_FAIL(!prog_array))783 goto out;784 785 map_fd = bpf_map__fd(prog_array);786 if (CHECK_FAIL(map_fd < 0))787 goto out;788 789 for (i = 0; i < bpf_map__max_entries(prog_array); i++) {790 snprintf(prog_name, sizeof(prog_name), "classifier_%d", i);791 792 prog = bpf_object__find_program_by_name(obj, prog_name);793 if (CHECK_FAIL(!prog))794 goto out;795 796 prog_fd = bpf_program__fd(prog);797 if (CHECK_FAIL(prog_fd < 0))798 goto out;799 800 err = bpf_map_update_elem(map_fd, &i, &prog_fd, BPF_ANY);801 if (CHECK_FAIL(err))802 goto out;803 }804 805 err = bpf_prog_test_run_opts(main_fd, &topts);806 ASSERT_OK(err, "tailcall");807 ASSERT_EQ(topts.retval, sizeof(pkt_v4) * 3, "tailcall retval");808 809 i = 1;810 err = bpf_map_delete_elem(map_fd, &i);811 if (CHECK_FAIL(err))812 goto out;813 814 err = bpf_prog_test_run_opts(main_fd, &topts);815 ASSERT_OK(err, "tailcall");816 ASSERT_EQ(topts.retval, sizeof(pkt_v4), "tailcall retval");817 818 i = 0;819 err = bpf_map_delete_elem(map_fd, &i);820 if (CHECK_FAIL(err))821 goto out;822 823 err = bpf_prog_test_run_opts(main_fd, &topts);824 ASSERT_OK(err, "tailcall");825 ASSERT_EQ(topts.retval, sizeof(pkt_v4) * 2, "tailcall retval");826out:827 bpf_object__close(obj);828}829 830#include "tailcall_bpf2bpf4.skel.h"831 832/* test_tailcall_bpf2bpf_4 checks that tailcall counter is correctly preserved833 * across tailcalls combined with bpf2bpf calls. for making sure that tailcall834 * counter behaves correctly, bpf program will go through following flow:835 *836 * entry -> entry_subprog -> tailcall0 -> bpf_func0 -> subprog0 ->837 * -> tailcall1 -> bpf_func1 -> subprog1 -> tailcall2 -> bpf_func2 ->838 * subprog2 [here bump global counter] --------^839 *840 * We go through first two tailcalls and start counting from the subprog2 where841 * the loop begins. At the end of the test make sure that the global counter is842 * equal to 31, because tailcall counter includes the first two tailcalls843 * whereas global counter is incremented only on loop presented on flow above.844 *845 * The noise parameter is used to insert bpf_map_update calls into the logic846 * to force verifier to patch instructions. This allows us to ensure jump847 * logic remains correct with instruction movement.848 */849static void test_tailcall_bpf2bpf_4(bool noise)850{851 int err, map_fd, prog_fd, main_fd, data_fd, i;852 struct tailcall_bpf2bpf4__bss val;853 struct bpf_map *prog_array, *data_map;854 struct bpf_program *prog;855 struct bpf_object *obj;856 char prog_name[32];857 LIBBPF_OPTS(bpf_test_run_opts, topts,858 .data_in = &pkt_v4,859 .data_size_in = sizeof(pkt_v4),860 .repeat = 1,861 );862 863 err = bpf_prog_test_load("tailcall_bpf2bpf4.bpf.o", BPF_PROG_TYPE_SCHED_CLS,864 &obj, &prog_fd);865 if (CHECK_FAIL(err))866 return;867 868 prog = bpf_object__find_program_by_name(obj, "entry");869 if (CHECK_FAIL(!prog))870 goto out;871 872 main_fd = bpf_program__fd(prog);873 if (CHECK_FAIL(main_fd < 0))874 goto out;875 876 prog_array = bpf_object__find_map_by_name(obj, "jmp_table");877 if (CHECK_FAIL(!prog_array))878 goto out;879 880 map_fd = bpf_map__fd(prog_array);881 if (CHECK_FAIL(map_fd < 0))882 goto out;883 884 for (i = 0; i < bpf_map__max_entries(prog_array); i++) {885 snprintf(prog_name, sizeof(prog_name), "classifier_%d", i);886 887 prog = bpf_object__find_program_by_name(obj, prog_name);888 if (CHECK_FAIL(!prog))889 goto out;890 891 prog_fd = bpf_program__fd(prog);892 if (CHECK_FAIL(prog_fd < 0))893 goto out;894 895 err = bpf_map_update_elem(map_fd, &i, &prog_fd, BPF_ANY);896 if (CHECK_FAIL(err))897 goto out;898 }899 900 data_map = bpf_object__find_map_by_name(obj, "tailcall.bss");901 if (CHECK_FAIL(!data_map || !bpf_map__is_internal(data_map)))902 goto out;903 904 data_fd = bpf_map__fd(data_map);905 if (CHECK_FAIL(data_fd < 0))906 goto out;907 908 i = 0;909 val.noise = noise;910 val.count = 0;911 err = bpf_map_update_elem(data_fd, &i, &val, BPF_ANY);912 if (CHECK_FAIL(err))913 goto out;914 915 err = bpf_prog_test_run_opts(main_fd, &topts);916 ASSERT_OK(err, "tailcall");917 ASSERT_EQ(topts.retval, sizeof(pkt_v4) * 3, "tailcall retval");918 919 i = 0;920 err = bpf_map_lookup_elem(data_fd, &i, &val);921 ASSERT_OK(err, "tailcall count");922 ASSERT_EQ(val.count, 31, "tailcall count");923 924out:925 bpf_object__close(obj);926}927 928#include "tailcall_bpf2bpf6.skel.h"929 930/* Tail call counting works even when there is data on stack which is931 * not aligned to 8 bytes.932 */933static void test_tailcall_bpf2bpf_6(void)934{935 struct tailcall_bpf2bpf6 *obj;936 int err, map_fd, prog_fd, main_fd, data_fd, i, val;937 LIBBPF_OPTS(bpf_test_run_opts, topts,938 .data_in = &pkt_v4,939 .data_size_in = sizeof(pkt_v4),940 .repeat = 1,941 );942 943 obj = tailcall_bpf2bpf6__open_and_load();944 if (!ASSERT_OK_PTR(obj, "open and load"))945 return;946 947 main_fd = bpf_program__fd(obj->progs.entry);948 if (!ASSERT_GE(main_fd, 0, "entry prog fd"))949 goto out;950 951 map_fd = bpf_map__fd(obj->maps.jmp_table);952 if (!ASSERT_GE(map_fd, 0, "jmp_table map fd"))953 goto out;954 955 prog_fd = bpf_program__fd(obj->progs.classifier_0);956 if (!ASSERT_GE(prog_fd, 0, "classifier_0 prog fd"))957 goto out;958 959 i = 0;960 err = bpf_map_update_elem(map_fd, &i, &prog_fd, BPF_ANY);961 if (!ASSERT_OK(err, "jmp_table map update"))962 goto out;963 964 err = bpf_prog_test_run_opts(main_fd, &topts);965 ASSERT_OK(err, "entry prog test run");966 ASSERT_EQ(topts.retval, 0, "tailcall retval");967 968 data_fd = bpf_map__fd(obj->maps.bss);969 if (!ASSERT_GE(data_fd, 0, "bss map fd"))970 goto out;971 972 i = 0;973 err = bpf_map_lookup_elem(data_fd, &i, &val);974 ASSERT_OK(err, "bss map lookup");975 ASSERT_EQ(val, 1, "done flag is set");976 977out:978 tailcall_bpf2bpf6__destroy(obj);979}980 981/* test_tailcall_bpf2bpf_fentry checks that the count value of the tail call982 * limit enforcement matches with expectations when tailcall is preceded with983 * bpf2bpf call, and the bpf2bpf call is traced by fentry.984 */985static void test_tailcall_bpf2bpf_fentry(void)986{987 test_tailcall_count("tailcall_bpf2bpf2.bpf.o", true, false);988}989 990/* test_tailcall_bpf2bpf_fexit checks that the count value of the tail call991 * limit enforcement matches with expectations when tailcall is preceded with992 * bpf2bpf call, and the bpf2bpf call is traced by fexit.993 */994static void test_tailcall_bpf2bpf_fexit(void)995{996 test_tailcall_count("tailcall_bpf2bpf2.bpf.o", false, true);997}998 999/* test_tailcall_bpf2bpf_fentry_fexit checks that the count value of the tail1000 * call limit enforcement matches with expectations when tailcall is preceded1001 * with bpf2bpf call, and the bpf2bpf call is traced by both fentry and fexit.1002 */1003static void test_tailcall_bpf2bpf_fentry_fexit(void)1004{1005 test_tailcall_count("tailcall_bpf2bpf2.bpf.o", true, true);1006}1007 1008/* test_tailcall_bpf2bpf_fentry_entry checks that the count value of the tail1009 * call limit enforcement matches with expectations when tailcall is preceded1010 * with bpf2bpf call, and the bpf2bpf caller is traced by fentry.1011 */1012static void test_tailcall_bpf2bpf_fentry_entry(void)1013{1014 struct bpf_object *tgt_obj = NULL, *fentry_obj = NULL;1015 int err, map_fd, prog_fd, data_fd, i, val;1016 struct bpf_map *prog_array, *data_map;1017 struct bpf_link *fentry_link = NULL;1018 struct bpf_program *prog;1019 char buff[128] = {};1020 1021 LIBBPF_OPTS(bpf_test_run_opts, topts,1022 .data_in = buff,1023 .data_size_in = sizeof(buff),1024 .repeat = 1,1025 );1026 1027 err = bpf_prog_test_load("tailcall_bpf2bpf2.bpf.o",1028 BPF_PROG_TYPE_SCHED_CLS,1029 &tgt_obj, &prog_fd);1030 if (!ASSERT_OK(err, "load tgt_obj"))1031 return;1032 1033 prog_array = bpf_object__find_map_by_name(tgt_obj, "jmp_table");1034 if (!ASSERT_OK_PTR(prog_array, "find jmp_table map"))1035 goto out;1036 1037 map_fd = bpf_map__fd(prog_array);1038 if (!ASSERT_FALSE(map_fd < 0, "find jmp_table map fd"))1039 goto out;1040 1041 prog = bpf_object__find_program_by_name(tgt_obj, "classifier_0");1042 if (!ASSERT_OK_PTR(prog, "find classifier_0 prog"))1043 goto out;1044 1045 prog_fd = bpf_program__fd(prog);1046 if (!ASSERT_FALSE(prog_fd < 0, "find classifier_0 prog fd"))1047 goto out;1048 1049 i = 0;1050 err = bpf_map_update_elem(map_fd, &i, &prog_fd, BPF_ANY);1051 if (!ASSERT_OK(err, "update jmp_table"))1052 goto out;1053 1054 fentry_obj = bpf_object__open_file("tailcall_bpf2bpf_fentry.bpf.o",1055 NULL);1056 if (!ASSERT_OK_PTR(fentry_obj, "open fentry_obj file"))1057 goto out;1058 1059 prog = bpf_object__find_program_by_name(fentry_obj, "fentry");1060 if (!ASSERT_OK_PTR(prog, "find fentry prog"))1061 goto out;1062 1063 err = bpf_program__set_attach_target(prog, prog_fd, "classifier_0");1064 if (!ASSERT_OK(err, "set_attach_target classifier_0"))1065 goto out;1066 1067 err = bpf_object__load(fentry_obj);1068 if (!ASSERT_OK(err, "load fentry_obj"))1069 goto out;1070 1071 fentry_link = bpf_program__attach_trace(prog);1072 if (!ASSERT_OK_PTR(fentry_link, "attach_trace"))1073 goto out;1074 1075 err = bpf_prog_test_run_opts(prog_fd, &topts);1076 ASSERT_OK(err, "tailcall");1077 ASSERT_EQ(topts.retval, 1, "tailcall retval");1078 1079 data_map = bpf_object__find_map_by_name(tgt_obj, "tailcall.bss");1080 if (!ASSERT_FALSE(!data_map || !bpf_map__is_internal(data_map),1081 "find tailcall.bss map"))1082 goto out;1083 1084 data_fd = bpf_map__fd(data_map);1085 if (!ASSERT_FALSE(data_fd < 0, "find tailcall.bss map fd"))1086 goto out;1087 1088 i = 0;1089 err = bpf_map_lookup_elem(data_fd, &i, &val);1090 ASSERT_OK(err, "tailcall count");1091 ASSERT_EQ(val, 34, "tailcall count");1092 1093 data_map = bpf_object__find_map_by_name(fentry_obj, ".bss");1094 if (!ASSERT_FALSE(!data_map || !bpf_map__is_internal(data_map),1095 "find tailcall_bpf2bpf_fentry.bss map"))1096 goto out;1097 1098 data_fd = bpf_map__fd(data_map);1099 if (!ASSERT_FALSE(data_fd < 0,1100 "find tailcall_bpf2bpf_fentry.bss map fd"))1101 goto out;1102 1103 i = 0;1104 err = bpf_map_lookup_elem(data_fd, &i, &val);1105 ASSERT_OK(err, "fentry count");1106 ASSERT_EQ(val, 1, "fentry count");1107 1108out:1109 bpf_link__destroy(fentry_link);1110 bpf_object__close(fentry_obj);1111 bpf_object__close(tgt_obj);1112}1113 1114#define JMP_TABLE "/sys/fs/bpf/jmp_table"1115 1116static int poke_thread_exit;1117 1118static void *poke_update(void *arg)1119{1120 __u32 zero = 0, prog1_fd, prog2_fd, map_fd;1121 struct tailcall_poke *call = arg;1122 1123 map_fd = bpf_map__fd(call->maps.jmp_table);1124 prog1_fd = bpf_program__fd(call->progs.call1);1125 prog2_fd = bpf_program__fd(call->progs.call2);1126 1127 while (!poke_thread_exit) {1128 bpf_map_update_elem(map_fd, &zero, &prog1_fd, BPF_ANY);1129 bpf_map_update_elem(map_fd, &zero, &prog2_fd, BPF_ANY);1130 }1131 1132 return NULL;1133}1134 1135/*1136 * We are trying to hit prog array update during another program load1137 * that shares the same prog array map.1138 *1139 * For that we share the jmp_table map between two skeleton instances1140 * by pinning the jmp_table to same path. Then first skeleton instance1141 * periodically updates jmp_table in 'poke update' thread while we load1142 * the second skeleton instance in the main thread.1143 */1144static void test_tailcall_poke(void)1145{1146 struct tailcall_poke *call, *test;1147 int err, cnt = 10;1148 pthread_t thread;1149 1150 unlink(JMP_TABLE);1151 1152 call = tailcall_poke__open_and_load();1153 if (!ASSERT_OK_PTR(call, "tailcall_poke__open"))1154 return;1155 1156 err = bpf_map__pin(call->maps.jmp_table, JMP_TABLE);1157 if (!ASSERT_OK(err, "bpf_map__pin"))1158 goto out;1159 1160 err = pthread_create(&thread, NULL, poke_update, call);1161 if (!ASSERT_OK(err, "new toggler"))1162 goto out;1163 1164 while (cnt--) {1165 test = tailcall_poke__open();1166 if (!ASSERT_OK_PTR(test, "tailcall_poke__open"))1167 break;1168 1169 err = bpf_map__set_pin_path(test->maps.jmp_table, JMP_TABLE);1170 if (!ASSERT_OK(err, "bpf_map__pin")) {1171 tailcall_poke__destroy(test);1172 break;1173 }1174 1175 bpf_program__set_autoload(test->progs.test, true);1176 bpf_program__set_autoload(test->progs.call1, false);1177 bpf_program__set_autoload(test->progs.call2, false);1178 1179 err = tailcall_poke__load(test);1180 tailcall_poke__destroy(test);1181 if (!ASSERT_OK(err, "tailcall_poke__load"))1182 break;1183 }1184 1185 poke_thread_exit = 1;1186 ASSERT_OK(pthread_join(thread, NULL), "pthread_join");1187 1188out:1189 bpf_map__unpin(call->maps.jmp_table, JMP_TABLE);1190 tailcall_poke__destroy(call);1191}1192 1193static void test_tailcall_hierarchy_count(const char *which, bool test_fentry,1194 bool test_fexit,1195 bool test_fentry_entry)1196{1197 int err, map_fd, prog_fd, main_data_fd, fentry_data_fd, fexit_data_fd, i, val;1198 struct bpf_object *obj = NULL, *fentry_obj = NULL, *fexit_obj = NULL;1199 struct bpf_link *fentry_link = NULL, *fexit_link = NULL;1200 struct bpf_program *prog, *fentry_prog;1201 struct bpf_map *prog_array, *data_map;1202 int fentry_prog_fd;1203 char buff[128] = {};1204 1205 LIBBPF_OPTS(bpf_test_run_opts, topts,1206 .data_in = buff,1207 .data_size_in = sizeof(buff),1208 .repeat = 1,1209 );1210 1211 err = bpf_prog_test_load(which, BPF_PROG_TYPE_SCHED_CLS, &obj,1212 &prog_fd);1213 if (!ASSERT_OK(err, "load obj"))1214 return;1215 1216 prog = bpf_object__find_program_by_name(obj, "entry");1217 if (!ASSERT_OK_PTR(prog, "find entry prog"))1218 goto out;1219 1220 prog_fd = bpf_program__fd(prog);1221 if (!ASSERT_GE(prog_fd, 0, "prog_fd"))1222 goto out;1223 1224 if (test_fentry_entry) {1225 fentry_obj = bpf_object__open_file("tailcall_bpf2bpf_hierarchy_fentry.bpf.o",1226 NULL);1227 if (!ASSERT_OK_PTR(fentry_obj, "open fentry_obj file"))1228 goto out;1229 1230 fentry_prog = bpf_object__find_program_by_name(fentry_obj,1231 "fentry");1232 if (!ASSERT_OK_PTR(prog, "find fentry prog"))1233 goto out;1234 1235 err = bpf_program__set_attach_target(fentry_prog, prog_fd,1236 "entry");1237 if (!ASSERT_OK(err, "set_attach_target entry"))1238 goto out;1239 1240 err = bpf_object__load(fentry_obj);1241 if (!ASSERT_OK(err, "load fentry_obj"))1242 goto out;1243 1244 fentry_link = bpf_program__attach_trace(fentry_prog);1245 if (!ASSERT_OK_PTR(fentry_link, "attach_trace"))1246 goto out;1247 1248 fentry_prog_fd = bpf_program__fd(fentry_prog);1249 if (!ASSERT_GE(fentry_prog_fd, 0, "fentry_prog_fd"))1250 goto out;1251 1252 prog_array = bpf_object__find_map_by_name(fentry_obj, "jmp_table");1253 if (!ASSERT_OK_PTR(prog_array, "find jmp_table"))1254 goto out;1255 1256 map_fd = bpf_map__fd(prog_array);1257 if (!ASSERT_GE(map_fd, 0, "map_fd"))1258 goto out;1259 1260 i = 0;1261 err = bpf_map_update_elem(map_fd, &i, &fentry_prog_fd, BPF_ANY);1262 if (!ASSERT_OK(err, "update jmp_table"))1263 goto out;1264 1265 data_map = bpf_object__find_map_by_name(fentry_obj, ".bss");1266 if (!ASSERT_FALSE(!data_map || !bpf_map__is_internal(data_map),1267 "find data_map"))1268 goto out;1269 1270 } else {1271 prog_array = bpf_object__find_map_by_name(obj, "jmp_table");1272 if (!ASSERT_OK_PTR(prog_array, "find jmp_table"))1273 goto out;1274 1275 map_fd = bpf_map__fd(prog_array);1276 if (!ASSERT_GE(map_fd, 0, "map_fd"))1277 goto out;1278 1279 i = 0;1280 err = bpf_map_update_elem(map_fd, &i, &prog_fd, BPF_ANY);1281 if (!ASSERT_OK(err, "update jmp_table"))1282 goto out;1283 1284 data_map = bpf_object__find_map_by_name(obj, ".bss");1285 if (!ASSERT_FALSE(!data_map || !bpf_map__is_internal(data_map),1286 "find data_map"))1287 goto out;1288 }1289 1290 if (test_fentry) {1291 fentry_obj = bpf_object__open_file("tailcall_bpf2bpf_fentry.bpf.o",1292 NULL);1293 if (!ASSERT_OK_PTR(fentry_obj, "open fentry_obj file"))1294 goto out;1295 1296 prog = bpf_object__find_program_by_name(fentry_obj, "fentry");1297 if (!ASSERT_OK_PTR(prog, "find fentry prog"))1298 goto out;1299 1300 err = bpf_program__set_attach_target(prog, prog_fd,1301 "subprog_tail");1302 if (!ASSERT_OK(err, "set_attach_target subprog_tail"))1303 goto out;1304 1305 err = bpf_object__load(fentry_obj);1306 if (!ASSERT_OK(err, "load fentry_obj"))1307 goto out;1308 1309 fentry_link = bpf_program__attach_trace(prog);1310 if (!ASSERT_OK_PTR(fentry_link, "attach_trace"))1311 goto out;1312 }1313 1314 if (test_fexit) {1315 fexit_obj = bpf_object__open_file("tailcall_bpf2bpf_fexit.bpf.o",1316 NULL);1317 if (!ASSERT_OK_PTR(fexit_obj, "open fexit_obj file"))1318 goto out;1319 1320 prog = bpf_object__find_program_by_name(fexit_obj, "fexit");1321 if (!ASSERT_OK_PTR(prog, "find fexit prog"))1322 goto out;1323 1324 err = bpf_program__set_attach_target(prog, prog_fd,1325 "subprog_tail");1326 if (!ASSERT_OK(err, "set_attach_target subprog_tail"))1327 goto out;1328 1329 err = bpf_object__load(fexit_obj);1330 if (!ASSERT_OK(err, "load fexit_obj"))1331 goto out;1332 1333 fexit_link = bpf_program__attach_trace(prog);1334 if (!ASSERT_OK_PTR(fexit_link, "attach_trace"))1335 goto out;1336 }1337 1338 err = bpf_prog_test_run_opts(prog_fd, &topts);1339 ASSERT_OK(err, "tailcall");1340 ASSERT_EQ(topts.retval, 1, "tailcall retval");1341 1342 main_data_fd = bpf_map__fd(data_map);1343 if (!ASSERT_GE(main_data_fd, 0, "main_data_fd"))1344 goto out;1345 1346 i = 0;1347 err = bpf_map_lookup_elem(main_data_fd, &i, &val);1348 ASSERT_OK(err, "tailcall count");1349 ASSERT_EQ(val, 34, "tailcall count");1350 1351 if (test_fentry) {1352 data_map = bpf_object__find_map_by_name(fentry_obj, ".bss");1353 if (!ASSERT_FALSE(!data_map || !bpf_map__is_internal(data_map),1354 "find tailcall_bpf2bpf_fentry.bss map"))1355 goto out;1356 1357 fentry_data_fd = bpf_map__fd(data_map);1358 if (!ASSERT_GE(fentry_data_fd, 0,1359 "find tailcall_bpf2bpf_fentry.bss map fd"))1360 goto out;1361 1362 i = 0;1363 err = bpf_map_lookup_elem(fentry_data_fd, &i, &val);1364 ASSERT_OK(err, "fentry count");1365 ASSERT_EQ(val, 68, "fentry count");1366 }1367 1368 if (test_fexit) {1369 data_map = bpf_object__find_map_by_name(fexit_obj, ".bss");1370 if (!ASSERT_FALSE(!data_map || !bpf_map__is_internal(data_map),1371 "find tailcall_bpf2bpf_fexit.bss map"))1372 goto out;1373 1374 fexit_data_fd = bpf_map__fd(data_map);1375 if (!ASSERT_GE(fexit_data_fd, 0,1376 "find tailcall_bpf2bpf_fexit.bss map fd"))1377 goto out;1378 1379 i = 0;1380 err = bpf_map_lookup_elem(fexit_data_fd, &i, &val);1381 ASSERT_OK(err, "fexit count");1382 ASSERT_EQ(val, 68, "fexit count");1383 }1384 1385 i = 0;1386 err = bpf_map_delete_elem(map_fd, &i);1387 if (!ASSERT_OK(err, "delete_elem from jmp_table"))1388 goto out;1389 1390 err = bpf_prog_test_run_opts(prog_fd, &topts);1391 ASSERT_OK(err, "tailcall");1392 ASSERT_EQ(topts.retval, 1, "tailcall retval");1393 1394 i = 0;1395 err = bpf_map_lookup_elem(main_data_fd, &i, &val);1396 ASSERT_OK(err, "tailcall count");1397 ASSERT_EQ(val, 35, "tailcall count");1398 1399 if (test_fentry) {1400 i = 0;1401 err = bpf_map_lookup_elem(fentry_data_fd, &i, &val);1402 ASSERT_OK(err, "fentry count");1403 ASSERT_EQ(val, 70, "fentry count");1404 }1405 1406 if (test_fexit) {1407 i = 0;1408 err = bpf_map_lookup_elem(fexit_data_fd, &i, &val);1409 ASSERT_OK(err, "fexit count");1410 ASSERT_EQ(val, 70, "fexit count");1411 }1412 1413out:1414 bpf_link__destroy(fentry_link);1415 bpf_link__destroy(fexit_link);1416 bpf_object__close(fentry_obj);1417 bpf_object__close(fexit_obj);1418 bpf_object__close(obj);1419}1420 1421/* test_tailcall_bpf2bpf_hierarchy_1 checks that the count value of the tail1422 * call limit enforcement matches with expectations when tailcalls are preceded1423 * with two bpf2bpf calls.1424 *1425 * subprog --tailcall-> entry1426 * entry <1427 * subprog --tailcall-> entry1428 */1429static void test_tailcall_bpf2bpf_hierarchy_1(void)1430{1431 test_tailcall_hierarchy_count("tailcall_bpf2bpf_hierarchy1.bpf.o",1432 false, false, false);1433}1434 1435/* test_tailcall_bpf2bpf_hierarchy_fentry checks that the count value of the1436 * tail call limit enforcement matches with expectations when tailcalls are1437 * preceded with two bpf2bpf calls, and the two subprogs are traced by fentry.1438 */1439static void test_tailcall_bpf2bpf_hierarchy_fentry(void)1440{1441 test_tailcall_hierarchy_count("tailcall_bpf2bpf_hierarchy1.bpf.o",1442 true, false, false);1443}1444 1445/* test_tailcall_bpf2bpf_hierarchy_fexit checks that the count value of the tail1446 * call limit enforcement matches with expectations when tailcalls are preceded1447 * with two bpf2bpf calls, and the two subprogs are traced by fexit.1448 */1449static void test_tailcall_bpf2bpf_hierarchy_fexit(void)1450{1451 test_tailcall_hierarchy_count("tailcall_bpf2bpf_hierarchy1.bpf.o",1452 false, true, false);1453}1454 1455/* test_tailcall_bpf2bpf_hierarchy_fentry_fexit checks that the count value of1456 * the tail call limit enforcement matches with expectations when tailcalls are1457 * preceded with two bpf2bpf calls, and the two subprogs are traced by both1458 * fentry and fexit.1459 */1460static void test_tailcall_bpf2bpf_hierarchy_fentry_fexit(void)1461{1462 test_tailcall_hierarchy_count("tailcall_bpf2bpf_hierarchy1.bpf.o",1463 true, true, false);1464}1465 1466/* test_tailcall_bpf2bpf_hierarchy_fentry_entry checks that the count value of1467 * the tail call limit enforcement matches with expectations when tailcalls are1468 * preceded with two bpf2bpf calls in fentry.1469 */1470static void test_tailcall_bpf2bpf_hierarchy_fentry_entry(void)1471{1472 test_tailcall_hierarchy_count("tc_dummy.bpf.o", false, false, true);1473}1474 1475/* test_tailcall_bpf2bpf_hierarchy_2 checks that the count value of the tail1476 * call limit enforcement matches with expectations:1477 *1478 * subprog_tail0 --tailcall-> classifier_0 -> subprog_tail01479 * entry <1480 * subprog_tail1 --tailcall-> classifier_1 -> subprog_tail11481 */1482static void test_tailcall_bpf2bpf_hierarchy_2(void)1483{1484 RUN_TESTS(tailcall_bpf2bpf_hierarchy2);1485}1486 1487/* test_tailcall_bpf2bpf_hierarchy_3 checks that the count value of the tail1488 * call limit enforcement matches with expectations:1489 *1490 * subprog with jmp_table0 to classifier_01491 * entry --tailcall-> classifier_0 <1492 * subprog with jmp_table1 to classifier_01493 */1494static void test_tailcall_bpf2bpf_hierarchy_3(void)1495{1496 RUN_TESTS(tailcall_bpf2bpf_hierarchy3);1497}1498 1499/* test_tailcall_freplace checks that the attached freplace prog is OK to1500 * update the prog_array map.1501 */1502static void test_tailcall_freplace(void)1503{1504 struct tailcall_freplace *freplace_skel = NULL;1505 struct bpf_link *freplace_link = NULL;1506 struct bpf_program *freplace_prog;1507 struct tc_bpf2bpf *tc_skel = NULL;1508 int prog_fd, map_fd;1509 char buff[128] = {};1510 int err, key;1511 1512 LIBBPF_OPTS(bpf_test_run_opts, topts,1513 .data_in = buff,1514 .data_size_in = sizeof(buff),1515 .repeat = 1,1516 );1517 1518 freplace_skel = tailcall_freplace__open();1519 if (!ASSERT_OK_PTR(freplace_skel, "tailcall_freplace__open"))1520 return;1521 1522 tc_skel = tc_bpf2bpf__open_and_load();1523 if (!ASSERT_OK_PTR(tc_skel, "tc_bpf2bpf__open_and_load"))1524 goto out;1525 1526 prog_fd = bpf_program__fd(tc_skel->progs.entry_tc);1527 freplace_prog = freplace_skel->progs.entry_freplace;1528 err = bpf_program__set_attach_target(freplace_prog, prog_fd, "subprog");1529 if (!ASSERT_OK(err, "set_attach_target"))1530 goto out;1531 1532 err = tailcall_freplace__load(freplace_skel);1533 if (!ASSERT_OK(err, "tailcall_freplace__load"))1534 goto out;1535 1536 freplace_link = bpf_program__attach_freplace(freplace_prog, prog_fd,1537 "subprog");1538 if (!ASSERT_OK_PTR(freplace_link, "attach_freplace"))1539 goto out;1540 1541 map_fd = bpf_map__fd(freplace_skel->maps.jmp_table);1542 prog_fd = bpf_program__fd(freplace_prog);1543 key = 0;1544 err = bpf_map_update_elem(map_fd, &key, &prog_fd, BPF_ANY);1545 if (!ASSERT_OK(err, "update jmp_table"))1546 goto out;1547 1548 prog_fd = bpf_program__fd(tc_skel->progs.entry_tc);1549 err = bpf_prog_test_run_opts(prog_fd, &topts);1550 ASSERT_OK(err, "test_run");1551 ASSERT_EQ(topts.retval, 34, "test_run retval");1552 1553out:1554 bpf_link__destroy(freplace_link);1555 tc_bpf2bpf__destroy(tc_skel);1556 tailcall_freplace__destroy(freplace_skel);1557}1558 1559void test_tailcalls(void)1560{1561 if (test__start_subtest("tailcall_1"))1562 test_tailcall_1();1563 if (test__start_subtest("tailcall_2"))1564 test_tailcall_2();1565 if (test__start_subtest("tailcall_3"))1566 test_tailcall_3();1567 if (test__start_subtest("tailcall_4"))1568 test_tailcall_4();1569 if (test__start_subtest("tailcall_5"))1570 test_tailcall_5();1571 if (test__start_subtest("tailcall_6"))1572 test_tailcall_6();1573 if (test__start_subtest("tailcall_bpf2bpf_1"))1574 test_tailcall_bpf2bpf_1();1575 if (test__start_subtest("tailcall_bpf2bpf_2"))1576 test_tailcall_bpf2bpf_2();1577 if (test__start_subtest("tailcall_bpf2bpf_3"))1578 test_tailcall_bpf2bpf_3();1579 if (test__start_subtest("tailcall_bpf2bpf_4"))1580 test_tailcall_bpf2bpf_4(false);1581 if (test__start_subtest("tailcall_bpf2bpf_5"))1582 test_tailcall_bpf2bpf_4(true);1583 if (test__start_subtest("tailcall_bpf2bpf_6"))1584 test_tailcall_bpf2bpf_6();1585 if (test__start_subtest("tailcall_bpf2bpf_fentry"))1586 test_tailcall_bpf2bpf_fentry();1587 if (test__start_subtest("tailcall_bpf2bpf_fexit"))1588 test_tailcall_bpf2bpf_fexit();1589 if (test__start_subtest("tailcall_bpf2bpf_fentry_fexit"))1590 test_tailcall_bpf2bpf_fentry_fexit();1591 if (test__start_subtest("tailcall_bpf2bpf_fentry_entry"))1592 test_tailcall_bpf2bpf_fentry_entry();1593 if (test__start_subtest("tailcall_poke"))1594 test_tailcall_poke();1595 if (test__start_subtest("tailcall_bpf2bpf_hierarchy_1"))1596 test_tailcall_bpf2bpf_hierarchy_1();1597 if (test__start_subtest("tailcall_bpf2bpf_hierarchy_fentry"))1598 test_tailcall_bpf2bpf_hierarchy_fentry();1599 if (test__start_subtest("tailcall_bpf2bpf_hierarchy_fexit"))1600 test_tailcall_bpf2bpf_hierarchy_fexit();1601 if (test__start_subtest("tailcall_bpf2bpf_hierarchy_fentry_fexit"))1602 test_tailcall_bpf2bpf_hierarchy_fentry_fexit();1603 if (test__start_subtest("tailcall_bpf2bpf_hierarchy_fentry_entry"))1604 test_tailcall_bpf2bpf_hierarchy_fentry_entry();1605 test_tailcall_bpf2bpf_hierarchy_2();1606 test_tailcall_bpf2bpf_hierarchy_3();1607 if (test__start_subtest("tailcall_freplace"))1608 test_tailcall_freplace();1609}1610