1118 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2/* Copyright (c) 2022 Meta Platforms, Inc. and affiliates. */3#include <linux/capability.h>4#include <stdlib.h>5#include <regex.h>6#include <test_progs.h>7#include <bpf/btf.h>8 9#include "autoconf_helper.h"10#include "disasm_helpers.h"11#include "unpriv_helpers.h"12#include "cap_helpers.h"13#include "jit_disasm_helpers.h"14 15#define str_has_pfx(str, pfx) \16 (strncmp(str, pfx, __builtin_constant_p(pfx) ? sizeof(pfx) - 1 : strlen(pfx)) == 0)17 18#define TEST_LOADER_LOG_BUF_SZ 209715219 20#define TEST_TAG_EXPECT_FAILURE "comment:test_expect_failure"21#define TEST_TAG_EXPECT_SUCCESS "comment:test_expect_success"22#define TEST_TAG_EXPECT_MSG_PFX "comment:test_expect_msg="23#define TEST_TAG_EXPECT_XLATED_PFX "comment:test_expect_xlated="24#define TEST_TAG_EXPECT_FAILURE_UNPRIV "comment:test_expect_failure_unpriv"25#define TEST_TAG_EXPECT_SUCCESS_UNPRIV "comment:test_expect_success_unpriv"26#define TEST_TAG_EXPECT_MSG_PFX_UNPRIV "comment:test_expect_msg_unpriv="27#define TEST_TAG_EXPECT_XLATED_PFX_UNPRIV "comment:test_expect_xlated_unpriv="28#define TEST_TAG_LOG_LEVEL_PFX "comment:test_log_level="29#define TEST_TAG_PROG_FLAGS_PFX "comment:test_prog_flags="30#define TEST_TAG_DESCRIPTION_PFX "comment:test_description="31#define TEST_TAG_RETVAL_PFX "comment:test_retval="32#define TEST_TAG_RETVAL_PFX_UNPRIV "comment:test_retval_unpriv="33#define TEST_TAG_AUXILIARY "comment:test_auxiliary"34#define TEST_TAG_AUXILIARY_UNPRIV "comment:test_auxiliary_unpriv"35#define TEST_BTF_PATH "comment:test_btf_path="36#define TEST_TAG_ARCH "comment:test_arch="37#define TEST_TAG_JITED_PFX "comment:test_jited="38#define TEST_TAG_JITED_PFX_UNPRIV "comment:test_jited_unpriv="39 40/* Warning: duplicated in bpf_misc.h */41#define POINTER_VALUE 0xcafe4all42#define TEST_DATA_LEN 6443 44#ifdef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS45#define EFFICIENT_UNALIGNED_ACCESS 146#else47#define EFFICIENT_UNALIGNED_ACCESS 048#endif49 50static int sysctl_unpriv_disabled = -1;51 52enum mode {53 PRIV = 1,54 UNPRIV = 255};56 57struct expect_msg {58 const char *substr; /* substring match */59 regex_t regex;60 bool is_regex;61 bool on_next_line;62};63 64struct expected_msgs {65 struct expect_msg *patterns;66 size_t cnt;67};68 69struct test_subspec {70 char *name;71 bool expect_failure;72 struct expected_msgs expect_msgs;73 struct expected_msgs expect_xlated;74 struct expected_msgs jited;75 int retval;76 bool execute;77};78 79struct test_spec {80 const char *prog_name;81 struct test_subspec priv;82 struct test_subspec unpriv;83 const char *btf_custom_path;84 int log_level;85 int prog_flags;86 int mode_mask;87 int arch_mask;88 bool auxiliary;89 bool valid;90};91 92static int tester_init(struct test_loader *tester)93{94 if (!tester->log_buf) {95 tester->log_buf_sz = TEST_LOADER_LOG_BUF_SZ;96 tester->log_buf = calloc(tester->log_buf_sz, 1);97 if (!ASSERT_OK_PTR(tester->log_buf, "tester_log_buf"))98 return -ENOMEM;99 }100 101 return 0;102}103 104void test_loader_fini(struct test_loader *tester)105{106 if (!tester)107 return;108 109 free(tester->log_buf);110}111 112static void free_msgs(struct expected_msgs *msgs)113{114 int i;115 116 for (i = 0; i < msgs->cnt; i++)117 if (msgs->patterns[i].is_regex)118 regfree(&msgs->patterns[i].regex);119 free(msgs->patterns);120 msgs->patterns = NULL;121 msgs->cnt = 0;122}123 124static void free_test_spec(struct test_spec *spec)125{126 /* Deallocate expect_msgs arrays. */127 free_msgs(&spec->priv.expect_msgs);128 free_msgs(&spec->unpriv.expect_msgs);129 free_msgs(&spec->priv.expect_xlated);130 free_msgs(&spec->unpriv.expect_xlated);131 free_msgs(&spec->priv.jited);132 free_msgs(&spec->unpriv.jited);133 134 free(spec->priv.name);135 free(spec->unpriv.name);136 spec->priv.name = NULL;137 spec->unpriv.name = NULL;138}139 140/* Compiles regular expression matching pattern.141 * Pattern has a special syntax:142 *143 * pattern := (<verbatim text> | regex)*144 * regex := "{{" <posix extended regular expression> "}}"145 *146 * In other words, pattern is a verbatim text with inclusion147 * of regular expressions enclosed in "{{" "}}" pairs.148 * For example, pattern "foo{{[0-9]+}}" matches strings like149 * "foo0", "foo007", etc.150 */151static int compile_regex(const char *pattern, regex_t *regex)152{153 char err_buf[256], buf[256] = {}, *ptr, *buf_end;154 const char *original_pattern = pattern;155 bool in_regex = false;156 int err;157 158 buf_end = buf + sizeof(buf);159 ptr = buf;160 while (*pattern && ptr < buf_end - 2) {161 if (!in_regex && str_has_pfx(pattern, "{{")) {162 in_regex = true;163 pattern += 2;164 continue;165 }166 if (in_regex && str_has_pfx(pattern, "}}")) {167 in_regex = false;168 pattern += 2;169 continue;170 }171 if (in_regex) {172 *ptr++ = *pattern++;173 continue;174 }175 /* list of characters that need escaping for extended posix regex */176 if (strchr(".[]\\()*+?{}|^$", *pattern)) {177 *ptr++ = '\\';178 *ptr++ = *pattern++;179 continue;180 }181 *ptr++ = *pattern++;182 }183 if (*pattern) {184 PRINT_FAIL("Regexp too long: '%s'\n", original_pattern);185 return -EINVAL;186 }187 if (in_regex) {188 PRINT_FAIL("Regexp has open '{{' but no closing '}}': '%s'\n", original_pattern);189 return -EINVAL;190 }191 err = regcomp(regex, buf, REG_EXTENDED | REG_NEWLINE);192 if (err != 0) {193 regerror(err, regex, err_buf, sizeof(err_buf));194 PRINT_FAIL("Regexp compilation error in '%s': '%s'\n", buf, err_buf);195 return -EINVAL;196 }197 return 0;198}199 200static int __push_msg(const char *pattern, bool on_next_line, struct expected_msgs *msgs)201{202 struct expect_msg *msg;203 void *tmp;204 int err;205 206 tmp = realloc(msgs->patterns,207 (1 + msgs->cnt) * sizeof(struct expect_msg));208 if (!tmp) {209 ASSERT_FAIL("failed to realloc memory for messages\n");210 return -ENOMEM;211 }212 msgs->patterns = tmp;213 msg = &msgs->patterns[msgs->cnt];214 msg->on_next_line = on_next_line;215 msg->substr = pattern;216 msg->is_regex = false;217 if (strstr(pattern, "{{")) {218 err = compile_regex(pattern, &msg->regex);219 if (err)220 return err;221 msg->is_regex = true;222 }223 msgs->cnt += 1;224 return 0;225}226 227static int clone_msgs(struct expected_msgs *from, struct expected_msgs *to)228{229 struct expect_msg *msg;230 int i, err;231 232 for (i = 0; i < from->cnt; i++) {233 msg = &from->patterns[i];234 err = __push_msg(msg->substr, msg->on_next_line, to);235 if (err)236 return err;237 }238 return 0;239}240 241static int push_msg(const char *substr, struct expected_msgs *msgs)242{243 return __push_msg(substr, false, msgs);244}245 246static int push_disasm_msg(const char *regex_str, bool *on_next_line, struct expected_msgs *msgs)247{248 int err;249 250 if (strcmp(regex_str, "...") == 0) {251 *on_next_line = false;252 return 0;253 }254 err = __push_msg(regex_str, *on_next_line, msgs);255 if (err)256 return err;257 *on_next_line = true;258 return 0;259}260 261static int parse_int(const char *str, int *val, const char *name)262{263 char *end;264 long tmp;265 266 errno = 0;267 if (str_has_pfx(str, "0x"))268 tmp = strtol(str + 2, &end, 16);269 else270 tmp = strtol(str, &end, 10);271 if (errno || end[0] != '\0') {272 PRINT_FAIL("failed to parse %s from '%s'\n", name, str);273 return -EINVAL;274 }275 *val = tmp;276 return 0;277}278 279static int parse_retval(const char *str, int *val, const char *name)280{281 struct {282 char *name;283 int val;284 } named_values[] = {285 { "INT_MIN" , INT_MIN },286 { "POINTER_VALUE", POINTER_VALUE },287 { "TEST_DATA_LEN", TEST_DATA_LEN },288 };289 int i;290 291 for (i = 0; i < ARRAY_SIZE(named_values); ++i) {292 if (strcmp(str, named_values[i].name) != 0)293 continue;294 *val = named_values[i].val;295 return 0;296 }297 298 return parse_int(str, val, name);299}300 301static void update_flags(int *flags, int flag, bool clear)302{303 if (clear)304 *flags &= ~flag;305 else306 *flags |= flag;307}308 309/* Matches a string of form '<pfx>[^=]=.*' and returns it's suffix.310 * Used to parse btf_decl_tag values.311 * Such values require unique prefix because compiler does not add312 * same __attribute__((btf_decl_tag(...))) twice.313 * Test suite uses two-component tags for such cases:314 *315 * <pfx> __COUNTER__ '='316 *317 * For example, two consecutive __msg tags '__msg("foo") __msg("foo")'318 * would be encoded as:319 *320 * [18] DECL_TAG 'comment:test_expect_msg=0=foo' type_id=15 component_idx=-1321 * [19] DECL_TAG 'comment:test_expect_msg=1=foo' type_id=15 component_idx=-1322 *323 * And the purpose of this function is to extract 'foo' from the above.324 */325static const char *skip_dynamic_pfx(const char *s, const char *pfx)326{327 const char *msg;328 329 if (strncmp(s, pfx, strlen(pfx)) != 0)330 return NULL;331 msg = s + strlen(pfx);332 msg = strchr(msg, '=');333 if (!msg)334 return NULL;335 return msg + 1;336}337 338enum arch {339 ARCH_UNKNOWN = 0x1,340 ARCH_X86_64 = 0x2,341 ARCH_ARM64 = 0x4,342 ARCH_RISCV64 = 0x8,343};344 345static int get_current_arch(void)346{347#if defined(__x86_64__)348 return ARCH_X86_64;349#elif defined(__aarch64__)350 return ARCH_ARM64;351#elif defined(__riscv) && __riscv_xlen == 64352 return ARCH_RISCV64;353#endif354 return ARCH_UNKNOWN;355}356 357/* Uses btf_decl_tag attributes to describe the expected test358 * behavior, see bpf_misc.h for detailed description of each attribute359 * and attribute combinations.360 */361static int parse_test_spec(struct test_loader *tester,362 struct bpf_object *obj,363 struct bpf_program *prog,364 struct test_spec *spec)365{366 const char *description = NULL;367 bool has_unpriv_result = false;368 bool has_unpriv_retval = false;369 bool unpriv_xlated_on_next_line = true;370 bool xlated_on_next_line = true;371 bool unpriv_jit_on_next_line;372 bool jit_on_next_line;373 bool collect_jit = false;374 int func_id, i, err = 0;375 u32 arch_mask = 0;376 struct btf *btf;377 enum arch arch;378 379 memset(spec, 0, sizeof(*spec));380 381 spec->prog_name = bpf_program__name(prog);382 spec->prog_flags = testing_prog_flags();383 384 btf = bpf_object__btf(obj);385 if (!btf) {386 ASSERT_FAIL("BPF object has no BTF");387 return -EINVAL;388 }389 390 func_id = btf__find_by_name_kind(btf, spec->prog_name, BTF_KIND_FUNC);391 if (func_id < 0) {392 ASSERT_FAIL("failed to find FUNC BTF type for '%s'", spec->prog_name);393 return -EINVAL;394 }395 396 for (i = 1; i < btf__type_cnt(btf); i++) {397 const char *s, *val, *msg;398 const struct btf_type *t;399 bool clear;400 int flags;401 402 t = btf__type_by_id(btf, i);403 if (!btf_is_decl_tag(t))404 continue;405 406 if (t->type != func_id || btf_decl_tag(t)->component_idx != -1)407 continue;408 409 s = btf__str_by_offset(btf, t->name_off);410 if (str_has_pfx(s, TEST_TAG_DESCRIPTION_PFX)) {411 description = s + sizeof(TEST_TAG_DESCRIPTION_PFX) - 1;412 } else if (strcmp(s, TEST_TAG_EXPECT_FAILURE) == 0) {413 spec->priv.expect_failure = true;414 spec->mode_mask |= PRIV;415 } else if (strcmp(s, TEST_TAG_EXPECT_SUCCESS) == 0) {416 spec->priv.expect_failure = false;417 spec->mode_mask |= PRIV;418 } else if (strcmp(s, TEST_TAG_EXPECT_FAILURE_UNPRIV) == 0) {419 spec->unpriv.expect_failure = true;420 spec->mode_mask |= UNPRIV;421 has_unpriv_result = true;422 } else if (strcmp(s, TEST_TAG_EXPECT_SUCCESS_UNPRIV) == 0) {423 spec->unpriv.expect_failure = false;424 spec->mode_mask |= UNPRIV;425 has_unpriv_result = true;426 } else if (strcmp(s, TEST_TAG_AUXILIARY) == 0) {427 spec->auxiliary = true;428 spec->mode_mask |= PRIV;429 } else if (strcmp(s, TEST_TAG_AUXILIARY_UNPRIV) == 0) {430 spec->auxiliary = true;431 spec->mode_mask |= UNPRIV;432 } else if ((msg = skip_dynamic_pfx(s, TEST_TAG_EXPECT_MSG_PFX))) {433 err = push_msg(msg, &spec->priv.expect_msgs);434 if (err)435 goto cleanup;436 spec->mode_mask |= PRIV;437 } else if ((msg = skip_dynamic_pfx(s, TEST_TAG_EXPECT_MSG_PFX_UNPRIV))) {438 err = push_msg(msg, &spec->unpriv.expect_msgs);439 if (err)440 goto cleanup;441 spec->mode_mask |= UNPRIV;442 } else if ((msg = skip_dynamic_pfx(s, TEST_TAG_JITED_PFX))) {443 if (arch_mask == 0) {444 PRINT_FAIL("__jited used before __arch_*");445 goto cleanup;446 }447 if (collect_jit) {448 err = push_disasm_msg(msg, &jit_on_next_line,449 &spec->priv.jited);450 if (err)451 goto cleanup;452 spec->mode_mask |= PRIV;453 }454 } else if ((msg = skip_dynamic_pfx(s, TEST_TAG_JITED_PFX_UNPRIV))) {455 if (arch_mask == 0) {456 PRINT_FAIL("__unpriv_jited used before __arch_*");457 goto cleanup;458 }459 if (collect_jit) {460 err = push_disasm_msg(msg, &unpriv_jit_on_next_line,461 &spec->unpriv.jited);462 if (err)463 goto cleanup;464 spec->mode_mask |= UNPRIV;465 }466 } else if ((msg = skip_dynamic_pfx(s, TEST_TAG_EXPECT_XLATED_PFX))) {467 err = push_disasm_msg(msg, &xlated_on_next_line,468 &spec->priv.expect_xlated);469 if (err)470 goto cleanup;471 spec->mode_mask |= PRIV;472 } else if ((msg = skip_dynamic_pfx(s, TEST_TAG_EXPECT_XLATED_PFX_UNPRIV))) {473 err = push_disasm_msg(msg, &unpriv_xlated_on_next_line,474 &spec->unpriv.expect_xlated);475 if (err)476 goto cleanup;477 spec->mode_mask |= UNPRIV;478 } else if (str_has_pfx(s, TEST_TAG_RETVAL_PFX)) {479 val = s + sizeof(TEST_TAG_RETVAL_PFX) - 1;480 err = parse_retval(val, &spec->priv.retval, "__retval");481 if (err)482 goto cleanup;483 spec->priv.execute = true;484 spec->mode_mask |= PRIV;485 } else if (str_has_pfx(s, TEST_TAG_RETVAL_PFX_UNPRIV)) {486 val = s + sizeof(TEST_TAG_RETVAL_PFX_UNPRIV) - 1;487 err = parse_retval(val, &spec->unpriv.retval, "__retval_unpriv");488 if (err)489 goto cleanup;490 spec->mode_mask |= UNPRIV;491 spec->unpriv.execute = true;492 has_unpriv_retval = true;493 } else if (str_has_pfx(s, TEST_TAG_LOG_LEVEL_PFX)) {494 val = s + sizeof(TEST_TAG_LOG_LEVEL_PFX) - 1;495 err = parse_int(val, &spec->log_level, "test log level");496 if (err)497 goto cleanup;498 } else if (str_has_pfx(s, TEST_TAG_PROG_FLAGS_PFX)) {499 val = s + sizeof(TEST_TAG_PROG_FLAGS_PFX) - 1;500 501 clear = val[0] == '!';502 if (clear)503 val++;504 505 if (strcmp(val, "BPF_F_STRICT_ALIGNMENT") == 0) {506 update_flags(&spec->prog_flags, BPF_F_STRICT_ALIGNMENT, clear);507 } else if (strcmp(val, "BPF_F_ANY_ALIGNMENT") == 0) {508 update_flags(&spec->prog_flags, BPF_F_ANY_ALIGNMENT, clear);509 } else if (strcmp(val, "BPF_F_TEST_RND_HI32") == 0) {510 update_flags(&spec->prog_flags, BPF_F_TEST_RND_HI32, clear);511 } else if (strcmp(val, "BPF_F_TEST_STATE_FREQ") == 0) {512 update_flags(&spec->prog_flags, BPF_F_TEST_STATE_FREQ, clear);513 } else if (strcmp(val, "BPF_F_SLEEPABLE") == 0) {514 update_flags(&spec->prog_flags, BPF_F_SLEEPABLE, clear);515 } else if (strcmp(val, "BPF_F_XDP_HAS_FRAGS") == 0) {516 update_flags(&spec->prog_flags, BPF_F_XDP_HAS_FRAGS, clear);517 } else if (strcmp(val, "BPF_F_TEST_REG_INVARIANTS") == 0) {518 update_flags(&spec->prog_flags, BPF_F_TEST_REG_INVARIANTS, clear);519 } else /* assume numeric value */ {520 err = parse_int(val, &flags, "test prog flags");521 if (err)522 goto cleanup;523 update_flags(&spec->prog_flags, flags, clear);524 }525 } else if (str_has_pfx(s, TEST_TAG_ARCH)) {526 val = s + sizeof(TEST_TAG_ARCH) - 1;527 if (strcmp(val, "X86_64") == 0) {528 arch = ARCH_X86_64;529 } else if (strcmp(val, "ARM64") == 0) {530 arch = ARCH_ARM64;531 } else if (strcmp(val, "RISCV64") == 0) {532 arch = ARCH_RISCV64;533 } else {534 PRINT_FAIL("bad arch spec: '%s'", val);535 err = -EINVAL;536 goto cleanup;537 }538 arch_mask |= arch;539 collect_jit = get_current_arch() == arch;540 unpriv_jit_on_next_line = true;541 jit_on_next_line = true;542 } else if (str_has_pfx(s, TEST_BTF_PATH)) {543 spec->btf_custom_path = s + sizeof(TEST_BTF_PATH) - 1;544 }545 }546 547 spec->arch_mask = arch_mask ?: -1;548 549 if (spec->mode_mask == 0)550 spec->mode_mask = PRIV;551 552 if (!description)553 description = spec->prog_name;554 555 if (spec->mode_mask & PRIV) {556 spec->priv.name = strdup(description);557 if (!spec->priv.name) {558 PRINT_FAIL("failed to allocate memory for priv.name\n");559 err = -ENOMEM;560 goto cleanup;561 }562 }563 564 if (spec->mode_mask & UNPRIV) {565 int descr_len = strlen(description);566 const char *suffix = " @unpriv";567 char *name;568 569 name = malloc(descr_len + strlen(suffix) + 1);570 if (!name) {571 PRINT_FAIL("failed to allocate memory for unpriv.name\n");572 err = -ENOMEM;573 goto cleanup;574 }575 576 strcpy(name, description);577 strcpy(&name[descr_len], suffix);578 spec->unpriv.name = name;579 }580 581 if (spec->mode_mask & (PRIV | UNPRIV)) {582 if (!has_unpriv_result)583 spec->unpriv.expect_failure = spec->priv.expect_failure;584 585 if (!has_unpriv_retval) {586 spec->unpriv.retval = spec->priv.retval;587 spec->unpriv.execute = spec->priv.execute;588 }589 590 if (spec->unpriv.expect_msgs.cnt == 0)591 clone_msgs(&spec->priv.expect_msgs, &spec->unpriv.expect_msgs);592 if (spec->unpriv.expect_xlated.cnt == 0)593 clone_msgs(&spec->priv.expect_xlated, &spec->unpriv.expect_xlated);594 if (spec->unpriv.jited.cnt == 0)595 clone_msgs(&spec->priv.jited, &spec->unpriv.jited);596 }597 598 spec->valid = true;599 600 return 0;601 602cleanup:603 free_test_spec(spec);604 return err;605}606 607static void prepare_case(struct test_loader *tester,608 struct test_spec *spec,609 struct bpf_object *obj,610 struct bpf_program *prog)611{612 int min_log_level = 0, prog_flags;613 614 if (env.verbosity > VERBOSE_NONE)615 min_log_level = 1;616 if (env.verbosity > VERBOSE_VERY)617 min_log_level = 2;618 619 bpf_program__set_log_buf(prog, tester->log_buf, tester->log_buf_sz);620 621 /* Make sure we set at least minimal log level, unless test requires622 * even higher level already. Make sure to preserve independent log623 * level 4 (verifier stats), though.624 */625 if ((spec->log_level & 3) < min_log_level)626 bpf_program__set_log_level(prog, (spec->log_level & 4) | min_log_level);627 else628 bpf_program__set_log_level(prog, spec->log_level);629 630 prog_flags = bpf_program__flags(prog);631 bpf_program__set_flags(prog, prog_flags | spec->prog_flags);632 633 tester->log_buf[0] = '\0';634}635 636static void emit_verifier_log(const char *log_buf, bool force)637{638 if (!force && env.verbosity == VERBOSE_NONE)639 return;640 fprintf(stdout, "VERIFIER LOG:\n=============\n%s=============\n", log_buf);641}642 643static void emit_xlated(const char *xlated, bool force)644{645 if (!force && env.verbosity == VERBOSE_NONE)646 return;647 fprintf(stdout, "XLATED:\n=============\n%s=============\n", xlated);648}649 650static void emit_jited(const char *jited, bool force)651{652 if (!force && env.verbosity == VERBOSE_NONE)653 return;654 fprintf(stdout, "JITED:\n=============\n%s=============\n", jited);655}656 657static void validate_msgs(char *log_buf, struct expected_msgs *msgs,658 void (*emit_fn)(const char *buf, bool force))659{660 const char *log = log_buf, *prev_match;661 regmatch_t reg_match[1];662 int prev_match_line;663 int match_line;664 int i, j, err;665 666 prev_match_line = -1;667 match_line = 0;668 prev_match = log;669 for (i = 0; i < msgs->cnt; i++) {670 struct expect_msg *msg = &msgs->patterns[i];671 const char *match = NULL, *pat_status;672 bool wrong_line = false;673 674 if (!msg->is_regex) {675 match = strstr(log, msg->substr);676 if (match)677 log = match + strlen(msg->substr);678 } else {679 err = regexec(&msg->regex, log, 1, reg_match, 0);680 if (err == 0) {681 match = log + reg_match[0].rm_so;682 log += reg_match[0].rm_eo;683 }684 }685 686 if (match) {687 for (; prev_match < match; ++prev_match)688 if (*prev_match == '\n')689 ++match_line;690 wrong_line = msg->on_next_line && prev_match_line >= 0 &&691 prev_match_line + 1 != match_line;692 }693 694 if (!match || wrong_line) {695 PRINT_FAIL("expect_msg\n");696 if (env.verbosity == VERBOSE_NONE)697 emit_fn(log_buf, true /*force*/);698 for (j = 0; j <= i; j++) {699 msg = &msgs->patterns[j];700 if (j < i)701 pat_status = "MATCHED ";702 else if (wrong_line)703 pat_status = "WRONG LINE";704 else705 pat_status = "EXPECTED ";706 msg = &msgs->patterns[j];707 fprintf(stderr, "%s %s: '%s'\n",708 pat_status,709 msg->is_regex ? " REGEX" : "SUBSTR",710 msg->substr);711 }712 if (wrong_line) {713 fprintf(stderr,714 "expecting match at line %d, actual match is at line %d\n",715 prev_match_line + 1, match_line);716 }717 break;718 }719 720 prev_match_line = match_line;721 }722}723 724struct cap_state {725 __u64 old_caps;726 bool initialized;727};728 729static int drop_capabilities(struct cap_state *caps)730{731 const __u64 caps_to_drop = (1ULL << CAP_SYS_ADMIN | 1ULL << CAP_NET_ADMIN |732 1ULL << CAP_PERFMON | 1ULL << CAP_BPF);733 int err;734 735 err = cap_disable_effective(caps_to_drop, &caps->old_caps);736 if (err) {737 PRINT_FAIL("failed to drop capabilities: %i, %s\n", err, strerror(err));738 return err;739 }740 741 caps->initialized = true;742 return 0;743}744 745static int restore_capabilities(struct cap_state *caps)746{747 int err;748 749 if (!caps->initialized)750 return 0;751 752 err = cap_enable_effective(caps->old_caps, NULL);753 if (err)754 PRINT_FAIL("failed to restore capabilities: %i, %s\n", err, strerror(err));755 caps->initialized = false;756 return err;757}758 759static bool can_execute_unpriv(struct test_loader *tester, struct test_spec *spec)760{761 if (sysctl_unpriv_disabled < 0)762 sysctl_unpriv_disabled = get_unpriv_disabled() ? 1 : 0;763 if (sysctl_unpriv_disabled)764 return false;765 if ((spec->prog_flags & BPF_F_ANY_ALIGNMENT) && !EFFICIENT_UNALIGNED_ACCESS)766 return false;767 return true;768}769 770static bool is_unpriv_capable_map(struct bpf_map *map)771{772 enum bpf_map_type type;773 __u32 flags;774 775 type = bpf_map__type(map);776 777 switch (type) {778 case BPF_MAP_TYPE_HASH:779 case BPF_MAP_TYPE_PERCPU_HASH:780 case BPF_MAP_TYPE_HASH_OF_MAPS:781 flags = bpf_map__map_flags(map);782 return !(flags & BPF_F_ZERO_SEED);783 case BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE:784 case BPF_MAP_TYPE_ARRAY:785 case BPF_MAP_TYPE_RINGBUF:786 case BPF_MAP_TYPE_PROG_ARRAY:787 case BPF_MAP_TYPE_CGROUP_ARRAY:788 case BPF_MAP_TYPE_PERCPU_ARRAY:789 case BPF_MAP_TYPE_USER_RINGBUF:790 case BPF_MAP_TYPE_ARRAY_OF_MAPS:791 case BPF_MAP_TYPE_CGROUP_STORAGE:792 case BPF_MAP_TYPE_PERF_EVENT_ARRAY:793 return true;794 default:795 return false;796 }797}798 799static int do_prog_test_run(int fd_prog, int *retval, bool empty_opts)800{801 __u8 tmp_out[TEST_DATA_LEN << 2] = {};802 __u8 tmp_in[TEST_DATA_LEN] = {};803 int err, saved_errno;804 LIBBPF_OPTS(bpf_test_run_opts, topts,805 .data_in = tmp_in,806 .data_size_in = sizeof(tmp_in),807 .data_out = tmp_out,808 .data_size_out = sizeof(tmp_out),809 .repeat = 1,810 );811 812 if (empty_opts) {813 memset(&topts, 0, sizeof(struct bpf_test_run_opts));814 topts.sz = sizeof(struct bpf_test_run_opts);815 }816 err = bpf_prog_test_run_opts(fd_prog, &topts);817 saved_errno = errno;818 819 if (err) {820 PRINT_FAIL("FAIL: Unexpected bpf_prog_test_run error: %d (%s) ",821 saved_errno, strerror(saved_errno));822 return err;823 }824 825 ASSERT_OK(0, "bpf_prog_test_run");826 *retval = topts.retval;827 828 return 0;829}830 831static bool should_do_test_run(struct test_spec *spec, struct test_subspec *subspec)832{833 if (!subspec->execute)834 return false;835 836 if (subspec->expect_failure)837 return false;838 839 if ((spec->prog_flags & BPF_F_ANY_ALIGNMENT) && !EFFICIENT_UNALIGNED_ACCESS) {840 if (env.verbosity != VERBOSE_NONE)841 printf("alignment prevents execution\n");842 return false;843 }844 845 return true;846}847 848/* Get a disassembly of BPF program after verifier applies all rewrites */849static int get_xlated_program_text(int prog_fd, char *text, size_t text_sz)850{851 struct bpf_insn *insn_start = NULL, *insn, *insn_end;852 __u32 insns_cnt = 0, i;853 char buf[64];854 FILE *out = NULL;855 int err;856 857 err = get_xlated_program(prog_fd, &insn_start, &insns_cnt);858 if (!ASSERT_OK(err, "get_xlated_program"))859 goto out;860 out = fmemopen(text, text_sz, "w");861 if (!ASSERT_OK_PTR(out, "open_memstream"))862 goto out;863 insn_end = insn_start + insns_cnt;864 insn = insn_start;865 while (insn < insn_end) {866 i = insn - insn_start;867 insn = disasm_insn(insn, buf, sizeof(buf));868 fprintf(out, "%d: %s\n", i, buf);869 }870 fflush(out);871 872out:873 free(insn_start);874 if (out)875 fclose(out);876 return err;877}878 879/* this function is forced noinline and has short generic name to look better880 * in test_progs output (in case of a failure)881 */882static noinline883void run_subtest(struct test_loader *tester,884 struct bpf_object_open_opts *open_opts,885 const void *obj_bytes,886 size_t obj_byte_cnt,887 struct test_spec *specs,888 struct test_spec *spec,889 bool unpriv)890{891 struct test_subspec *subspec = unpriv ? &spec->unpriv : &spec->priv;892 struct bpf_program *tprog = NULL, *tprog_iter;893 struct bpf_link *link, *links[32] = {};894 struct test_spec *spec_iter;895 struct cap_state caps = {};896 struct bpf_object *tobj;897 struct bpf_map *map;898 int retval, err, i;899 int links_cnt = 0;900 bool should_load;901 902 if (!test__start_subtest(subspec->name))903 return;904 905 if ((get_current_arch() & spec->arch_mask) == 0) {906 test__skip();907 return;908 }909 910 if (unpriv) {911 if (!can_execute_unpriv(tester, spec)) {912 test__skip();913 test__end_subtest();914 return;915 }916 if (drop_capabilities(&caps)) {917 test__end_subtest();918 return;919 }920 }921 922 /* Implicitly reset to NULL if next test case doesn't specify */923 open_opts->btf_custom_path = spec->btf_custom_path;924 925 tobj = bpf_object__open_mem(obj_bytes, obj_byte_cnt, open_opts);926 if (!ASSERT_OK_PTR(tobj, "obj_open_mem")) /* shouldn't happen */927 goto subtest_cleanup;928 929 i = 0;930 bpf_object__for_each_program(tprog_iter, tobj) {931 spec_iter = &specs[i++];932 should_load = false;933 934 if (spec_iter->valid) {935 if (strcmp(bpf_program__name(tprog_iter), spec->prog_name) == 0) {936 tprog = tprog_iter;937 should_load = true;938 }939 940 if (spec_iter->auxiliary &&941 spec_iter->mode_mask & (unpriv ? UNPRIV : PRIV))942 should_load = true;943 }944 945 bpf_program__set_autoload(tprog_iter, should_load);946 }947 948 prepare_case(tester, spec, tobj, tprog);949 950 /* By default bpf_object__load() automatically creates all951 * maps declared in the skeleton. Some map types are only952 * allowed in priv mode. Disable autoload for such maps in953 * unpriv mode.954 */955 bpf_object__for_each_map(map, tobj)956 bpf_map__set_autocreate(map, !unpriv || is_unpriv_capable_map(map));957 958 err = bpf_object__load(tobj);959 if (subspec->expect_failure) {960 if (!ASSERT_ERR(err, "unexpected_load_success")) {961 emit_verifier_log(tester->log_buf, false /*force*/);962 goto tobj_cleanup;963 }964 } else {965 if (!ASSERT_OK(err, "unexpected_load_failure")) {966 emit_verifier_log(tester->log_buf, true /*force*/);967 goto tobj_cleanup;968 }969 }970 emit_verifier_log(tester->log_buf, false /*force*/);971 validate_msgs(tester->log_buf, &subspec->expect_msgs, emit_verifier_log);972 973 if (subspec->expect_xlated.cnt) {974 err = get_xlated_program_text(bpf_program__fd(tprog),975 tester->log_buf, tester->log_buf_sz);976 if (err)977 goto tobj_cleanup;978 emit_xlated(tester->log_buf, false /*force*/);979 validate_msgs(tester->log_buf, &subspec->expect_xlated, emit_xlated);980 }981 982 if (subspec->jited.cnt) {983 err = get_jited_program_text(bpf_program__fd(tprog),984 tester->log_buf, tester->log_buf_sz);985 if (err == -EOPNOTSUPP) {986 printf("%s:SKIP: jited programs disassembly is not supported,\n", __func__);987 printf("%s:SKIP: tests are built w/o LLVM development libs\n", __func__);988 test__skip();989 goto tobj_cleanup;990 }991 if (!ASSERT_EQ(err, 0, "get_jited_program_text"))992 goto tobj_cleanup;993 emit_jited(tester->log_buf, false /*force*/);994 validate_msgs(tester->log_buf, &subspec->jited, emit_jited);995 }996 997 if (should_do_test_run(spec, subspec)) {998 /* For some reason test_verifier executes programs999 * with all capabilities restored. Do the same here.1000 */1001 if (restore_capabilities(&caps))1002 goto tobj_cleanup;1003 1004 /* Do bpf_map__attach_struct_ops() for each struct_ops map.1005 * This should trigger bpf_struct_ops->reg callback on kernel side.1006 */1007 bpf_object__for_each_map(map, tobj) {1008 if (!bpf_map__autocreate(map) ||1009 bpf_map__type(map) != BPF_MAP_TYPE_STRUCT_OPS)1010 continue;1011 if (links_cnt >= ARRAY_SIZE(links)) {1012 PRINT_FAIL("too many struct_ops maps");1013 goto tobj_cleanup;1014 }1015 link = bpf_map__attach_struct_ops(map);1016 if (!link) {1017 PRINT_FAIL("bpf_map__attach_struct_ops failed for map %s: err=%d\n",1018 bpf_map__name(map), err);1019 goto tobj_cleanup;1020 }1021 links[links_cnt++] = link;1022 }1023 1024 if (tester->pre_execution_cb) {1025 err = tester->pre_execution_cb(tobj);1026 if (err) {1027 PRINT_FAIL("pre_execution_cb failed: %d\n", err);1028 goto tobj_cleanup;1029 }1030 }1031 1032 do_prog_test_run(bpf_program__fd(tprog), &retval,1033 bpf_program__type(tprog) == BPF_PROG_TYPE_SYSCALL ? true : false);1034 if (retval != subspec->retval && subspec->retval != POINTER_VALUE) {1035 PRINT_FAIL("Unexpected retval: %d != %d\n", retval, subspec->retval);1036 goto tobj_cleanup;1037 }1038 /* redo bpf_map__attach_struct_ops for each test */1039 while (links_cnt > 0)1040 bpf_link__destroy(links[--links_cnt]);1041 }1042 1043tobj_cleanup:1044 while (links_cnt > 0)1045 bpf_link__destroy(links[--links_cnt]);1046 bpf_object__close(tobj);1047subtest_cleanup:1048 test__end_subtest();1049 restore_capabilities(&caps);1050}1051 1052static void process_subtest(struct test_loader *tester,1053 const char *skel_name,1054 skel_elf_bytes_fn elf_bytes_factory)1055{1056 LIBBPF_OPTS(bpf_object_open_opts, open_opts, .object_name = skel_name);1057 struct test_spec *specs = NULL;1058 struct bpf_object *obj = NULL;1059 struct bpf_program *prog;1060 const void *obj_bytes;1061 int err, i, nr_progs;1062 size_t obj_byte_cnt;1063 1064 if (tester_init(tester) < 0)1065 return; /* failed to initialize tester */1066 1067 obj_bytes = elf_bytes_factory(&obj_byte_cnt);1068 obj = bpf_object__open_mem(obj_bytes, obj_byte_cnt, &open_opts);1069 if (!ASSERT_OK_PTR(obj, "obj_open_mem"))1070 return;1071 1072 nr_progs = 0;1073 bpf_object__for_each_program(prog, obj)1074 ++nr_progs;1075 1076 specs = calloc(nr_progs, sizeof(struct test_spec));1077 if (!ASSERT_OK_PTR(specs, "specs_alloc"))1078 return;1079 1080 i = 0;1081 bpf_object__for_each_program(prog, obj) {1082 /* ignore tests for which we can't derive test specification */1083 err = parse_test_spec(tester, obj, prog, &specs[i++]);1084 if (err)1085 PRINT_FAIL("Can't parse test spec for program '%s'\n",1086 bpf_program__name(prog));1087 }1088 1089 i = 0;1090 bpf_object__for_each_program(prog, obj) {1091 struct test_spec *spec = &specs[i++];1092 1093 if (!spec->valid || spec->auxiliary)1094 continue;1095 1096 if (spec->mode_mask & PRIV)1097 run_subtest(tester, &open_opts, obj_bytes, obj_byte_cnt,1098 specs, spec, false);1099 if (spec->mode_mask & UNPRIV)1100 run_subtest(tester, &open_opts, obj_bytes, obj_byte_cnt,1101 specs, spec, true);1102 1103 }1104 1105 for (i = 0; i < nr_progs; ++i)1106 free_test_spec(&specs[i]);1107 free(specs);1108 bpf_object__close(obj);1109}1110 1111void test_loader__run_subtests(struct test_loader *tester,1112 const char *skel_name,1113 skel_elf_bytes_fn elf_bytes_factory)1114{1115 /* see comment in run_subtest() for why we do this function nesting */1116 process_subtest(tester, skel_name, elf_bytes_factory);1117}1118