1688 lines · c
1// SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause)2/* Copyright (c) 2019 Facebook */3 4#ifdef __KERNEL__5#include <linux/bpf.h>6#include <linux/btf.h>7#include <linux/string.h>8#include <linux/bpf_verifier.h>9#include "relo_core.h"10 11static const char *btf_kind_str(const struct btf_type *t)12{13 return btf_type_str(t);14}15 16static bool is_ldimm64_insn(struct bpf_insn *insn)17{18 return insn->code == (BPF_LD | BPF_IMM | BPF_DW);19}20 21static const struct btf_type *22skip_mods_and_typedefs(const struct btf *btf, u32 id, u32 *res_id)23{24 return btf_type_skip_modifiers(btf, id, res_id);25}26 27static const char *btf__name_by_offset(const struct btf *btf, u32 offset)28{29 return btf_name_by_offset(btf, offset);30}31 32static s64 btf__resolve_size(const struct btf *btf, u32 type_id)33{34 const struct btf_type *t;35 int size;36 37 t = btf_type_by_id(btf, type_id);38 t = btf_resolve_size(btf, t, &size);39 if (IS_ERR(t))40 return PTR_ERR(t);41 return size;42}43 44enum libbpf_print_level {45 LIBBPF_WARN,46 LIBBPF_INFO,47 LIBBPF_DEBUG,48};49 50#undef pr_warn51#undef pr_info52#undef pr_debug53#define pr_warn(fmt, log, ...) bpf_log((void *)log, fmt, "", ##__VA_ARGS__)54#define pr_info(fmt, log, ...) bpf_log((void *)log, fmt, "", ##__VA_ARGS__)55#define pr_debug(fmt, log, ...) bpf_log((void *)log, fmt, "", ##__VA_ARGS__)56#define libbpf_print(level, fmt, ...) bpf_log((void *)prog_name, fmt, ##__VA_ARGS__)57#else58#include <stdio.h>59#include <string.h>60#include <errno.h>61#include <ctype.h>62#include <linux/err.h>63 64#include "libbpf.h"65#include "bpf.h"66#include "btf.h"67#include "str_error.h"68#include "libbpf_internal.h"69#endif70 71static bool is_flex_arr(const struct btf *btf,72 const struct bpf_core_accessor *acc,73 const struct btf_array *arr)74{75 const struct btf_type *t;76 77 /* not a flexible array, if not inside a struct or has non-zero size */78 if (!acc->name || arr->nelems > 0)79 return false;80 81 /* has to be the last member of enclosing struct */82 t = btf_type_by_id(btf, acc->type_id);83 return acc->idx == btf_vlen(t) - 1;84}85 86static const char *core_relo_kind_str(enum bpf_core_relo_kind kind)87{88 switch (kind) {89 case BPF_CORE_FIELD_BYTE_OFFSET: return "byte_off";90 case BPF_CORE_FIELD_BYTE_SIZE: return "byte_sz";91 case BPF_CORE_FIELD_EXISTS: return "field_exists";92 case BPF_CORE_FIELD_SIGNED: return "signed";93 case BPF_CORE_FIELD_LSHIFT_U64: return "lshift_u64";94 case BPF_CORE_FIELD_RSHIFT_U64: return "rshift_u64";95 case BPF_CORE_TYPE_ID_LOCAL: return "local_type_id";96 case BPF_CORE_TYPE_ID_TARGET: return "target_type_id";97 case BPF_CORE_TYPE_EXISTS: return "type_exists";98 case BPF_CORE_TYPE_MATCHES: return "type_matches";99 case BPF_CORE_TYPE_SIZE: return "type_size";100 case BPF_CORE_ENUMVAL_EXISTS: return "enumval_exists";101 case BPF_CORE_ENUMVAL_VALUE: return "enumval_value";102 default: return "unknown";103 }104}105 106static bool core_relo_is_field_based(enum bpf_core_relo_kind kind)107{108 switch (kind) {109 case BPF_CORE_FIELD_BYTE_OFFSET:110 case BPF_CORE_FIELD_BYTE_SIZE:111 case BPF_CORE_FIELD_EXISTS:112 case BPF_CORE_FIELD_SIGNED:113 case BPF_CORE_FIELD_LSHIFT_U64:114 case BPF_CORE_FIELD_RSHIFT_U64:115 return true;116 default:117 return false;118 }119}120 121static bool core_relo_is_type_based(enum bpf_core_relo_kind kind)122{123 switch (kind) {124 case BPF_CORE_TYPE_ID_LOCAL:125 case BPF_CORE_TYPE_ID_TARGET:126 case BPF_CORE_TYPE_EXISTS:127 case BPF_CORE_TYPE_MATCHES:128 case BPF_CORE_TYPE_SIZE:129 return true;130 default:131 return false;132 }133}134 135static bool core_relo_is_enumval_based(enum bpf_core_relo_kind kind)136{137 switch (kind) {138 case BPF_CORE_ENUMVAL_EXISTS:139 case BPF_CORE_ENUMVAL_VALUE:140 return true;141 default:142 return false;143 }144}145 146int __bpf_core_types_are_compat(const struct btf *local_btf, __u32 local_id,147 const struct btf *targ_btf, __u32 targ_id, int level)148{149 const struct btf_type *local_type, *targ_type;150 int depth = 32; /* max recursion depth */151 152 /* caller made sure that names match (ignoring flavor suffix) */153 local_type = btf_type_by_id(local_btf, local_id);154 targ_type = btf_type_by_id(targ_btf, targ_id);155 if (!btf_kind_core_compat(local_type, targ_type))156 return 0;157 158recur:159 depth--;160 if (depth < 0)161 return -EINVAL;162 163 local_type = skip_mods_and_typedefs(local_btf, local_id, &local_id);164 targ_type = skip_mods_and_typedefs(targ_btf, targ_id, &targ_id);165 if (!local_type || !targ_type)166 return -EINVAL;167 168 if (!btf_kind_core_compat(local_type, targ_type))169 return 0;170 171 switch (btf_kind(local_type)) {172 case BTF_KIND_UNKN:173 case BTF_KIND_STRUCT:174 case BTF_KIND_UNION:175 case BTF_KIND_ENUM:176 case BTF_KIND_FWD:177 case BTF_KIND_ENUM64:178 return 1;179 case BTF_KIND_INT:180 /* just reject deprecated bitfield-like integers; all other181 * integers are by default compatible between each other182 */183 return btf_int_offset(local_type) == 0 && btf_int_offset(targ_type) == 0;184 case BTF_KIND_PTR:185 local_id = local_type->type;186 targ_id = targ_type->type;187 goto recur;188 case BTF_KIND_ARRAY:189 local_id = btf_array(local_type)->type;190 targ_id = btf_array(targ_type)->type;191 goto recur;192 case BTF_KIND_FUNC_PROTO: {193 struct btf_param *local_p = btf_params(local_type);194 struct btf_param *targ_p = btf_params(targ_type);195 __u16 local_vlen = btf_vlen(local_type);196 __u16 targ_vlen = btf_vlen(targ_type);197 int i, err;198 199 if (local_vlen != targ_vlen)200 return 0;201 202 for (i = 0; i < local_vlen; i++, local_p++, targ_p++) {203 if (level <= 0)204 return -EINVAL;205 206 skip_mods_and_typedefs(local_btf, local_p->type, &local_id);207 skip_mods_and_typedefs(targ_btf, targ_p->type, &targ_id);208 err = __bpf_core_types_are_compat(local_btf, local_id, targ_btf, targ_id,209 level - 1);210 if (err <= 0)211 return err;212 }213 214 /* tail recurse for return type check */215 skip_mods_and_typedefs(local_btf, local_type->type, &local_id);216 skip_mods_and_typedefs(targ_btf, targ_type->type, &targ_id);217 goto recur;218 }219 default:220 pr_warn("unexpected kind %s relocated, local [%d], target [%d]\n",221 btf_kind_str(local_type), local_id, targ_id);222 return 0;223 }224}225 226/*227 * Turn bpf_core_relo into a low- and high-level spec representation,228 * validating correctness along the way, as well as calculating resulting229 * field bit offset, specified by accessor string. Low-level spec captures230 * every single level of nestedness, including traversing anonymous231 * struct/union members. High-level one only captures semantically meaningful232 * "turning points": named fields and array indicies.233 * E.g., for this case:234 *235 * struct sample {236 * int __unimportant;237 * struct {238 * int __1;239 * int __2;240 * int a[7];241 * };242 * };243 *244 * struct sample *s = ...;245 *246 * int x = &s->a[3]; // access string = '0:1:2:3'247 *248 * Low-level spec has 1:1 mapping with each element of access string (it's249 * just a parsed access string representation): [0, 1, 2, 3].250 *251 * High-level spec will capture only 3 points:252 * - initial zero-index access by pointer (&s->... is the same as &s[0]...);253 * - field 'a' access (corresponds to '2' in low-level spec);254 * - array element #3 access (corresponds to '3' in low-level spec).255 *256 * Type-based relocations (TYPE_EXISTS/TYPE_MATCHES/TYPE_SIZE,257 * TYPE_ID_LOCAL/TYPE_ID_TARGET) don't capture any field information. Their258 * spec and raw_spec are kept empty.259 *260 * Enum value-based relocations (ENUMVAL_EXISTS/ENUMVAL_VALUE) use access261 * string to specify enumerator's value index that need to be relocated.262 */263int bpf_core_parse_spec(const char *prog_name, const struct btf *btf,264 const struct bpf_core_relo *relo,265 struct bpf_core_spec *spec)266{267 int access_idx, parsed_len, i;268 struct bpf_core_accessor *acc;269 const struct btf_type *t;270 const char *name, *spec_str;271 __u32 id, name_off;272 __s64 sz;273 274 spec_str = btf__name_by_offset(btf, relo->access_str_off);275 if (str_is_empty(spec_str) || *spec_str == ':')276 return -EINVAL;277 278 memset(spec, 0, sizeof(*spec));279 spec->btf = btf;280 spec->root_type_id = relo->type_id;281 spec->relo_kind = relo->kind;282 283 /* type-based relocations don't have a field access string */284 if (core_relo_is_type_based(relo->kind)) {285 if (strcmp(spec_str, "0"))286 return -EINVAL;287 return 0;288 }289 290 /* parse spec_str="0:1:2:3:4" into array raw_spec=[0, 1, 2, 3, 4] */291 while (*spec_str) {292 if (*spec_str == ':')293 ++spec_str;294 if (sscanf(spec_str, "%d%n", &access_idx, &parsed_len) != 1)295 return -EINVAL;296 if (spec->raw_len == BPF_CORE_SPEC_MAX_LEN)297 return -E2BIG;298 spec_str += parsed_len;299 spec->raw_spec[spec->raw_len++] = access_idx;300 }301 302 if (spec->raw_len == 0)303 return -EINVAL;304 305 t = skip_mods_and_typedefs(btf, relo->type_id, &id);306 if (!t)307 return -EINVAL;308 309 access_idx = spec->raw_spec[0];310 acc = &spec->spec[0];311 acc->type_id = id;312 acc->idx = access_idx;313 spec->len++;314 315 if (core_relo_is_enumval_based(relo->kind)) {316 if (!btf_is_any_enum(t) || spec->raw_len > 1 || access_idx >= btf_vlen(t))317 return -EINVAL;318 319 /* record enumerator name in a first accessor */320 name_off = btf_is_enum(t) ? btf_enum(t)[access_idx].name_off321 : btf_enum64(t)[access_idx].name_off;322 acc->name = btf__name_by_offset(btf, name_off);323 return 0;324 }325 326 if (!core_relo_is_field_based(relo->kind))327 return -EINVAL;328 329 sz = btf__resolve_size(btf, id);330 if (sz < 0)331 return sz;332 spec->bit_offset = access_idx * sz * 8;333 334 for (i = 1; i < spec->raw_len; i++) {335 t = skip_mods_and_typedefs(btf, id, &id);336 if (!t)337 return -EINVAL;338 339 access_idx = spec->raw_spec[i];340 acc = &spec->spec[spec->len];341 342 if (btf_is_composite(t)) {343 const struct btf_member *m;344 __u32 bit_offset;345 346 if (access_idx >= btf_vlen(t))347 return -EINVAL;348 349 bit_offset = btf_member_bit_offset(t, access_idx);350 spec->bit_offset += bit_offset;351 352 m = btf_members(t) + access_idx;353 if (m->name_off) {354 name = btf__name_by_offset(btf, m->name_off);355 if (str_is_empty(name))356 return -EINVAL;357 358 acc->type_id = id;359 acc->idx = access_idx;360 acc->name = name;361 spec->len++;362 }363 364 id = m->type;365 } else if (btf_is_array(t)) {366 const struct btf_array *a = btf_array(t);367 bool flex;368 369 t = skip_mods_and_typedefs(btf, a->type, &id);370 if (!t)371 return -EINVAL;372 373 flex = is_flex_arr(btf, acc - 1, a);374 if (!flex && access_idx >= a->nelems)375 return -EINVAL;376 377 spec->spec[spec->len].type_id = id;378 spec->spec[spec->len].idx = access_idx;379 spec->len++;380 381 sz = btf__resolve_size(btf, id);382 if (sz < 0)383 return sz;384 spec->bit_offset += access_idx * sz * 8;385 } else {386 pr_warn("prog '%s': relo for [%u] %s (at idx %d) captures type [%d] of unexpected kind %s\n",387 prog_name, relo->type_id, spec_str, i, id, btf_kind_str(t));388 return -EINVAL;389 }390 }391 392 return 0;393}394 395/* Check two types for compatibility for the purpose of field access396 * relocation. const/volatile/restrict and typedefs are skipped to ensure we397 * are relocating semantically compatible entities:398 * - any two STRUCTs/UNIONs are compatible and can be mixed;399 * - any two FWDs are compatible, if their names match (modulo flavor suffix);400 * - any two PTRs are always compatible;401 * - for ENUMs, names should be the same (ignoring flavor suffix) or at402 * least one of enums should be anonymous;403 * - for ENUMs, check sizes, names are ignored;404 * - for INT, size and signedness are ignored;405 * - any two FLOATs are always compatible;406 * - for ARRAY, dimensionality is ignored, element types are checked for407 * compatibility recursively;408 * - everything else shouldn't be ever a target of relocation.409 * These rules are not set in stone and probably will be adjusted as we get410 * more experience with using BPF CO-RE relocations.411 */412static int bpf_core_fields_are_compat(const struct btf *local_btf,413 __u32 local_id,414 const struct btf *targ_btf,415 __u32 targ_id)416{417 const struct btf_type *local_type, *targ_type;418 419recur:420 local_type = skip_mods_and_typedefs(local_btf, local_id, &local_id);421 targ_type = skip_mods_and_typedefs(targ_btf, targ_id, &targ_id);422 if (!local_type || !targ_type)423 return -EINVAL;424 425 if (btf_is_composite(local_type) && btf_is_composite(targ_type))426 return 1;427 if (!btf_kind_core_compat(local_type, targ_type))428 return 0;429 430 switch (btf_kind(local_type)) {431 case BTF_KIND_PTR:432 case BTF_KIND_FLOAT:433 return 1;434 case BTF_KIND_FWD:435 case BTF_KIND_ENUM64:436 case BTF_KIND_ENUM: {437 const char *local_name, *targ_name;438 size_t local_len, targ_len;439 440 local_name = btf__name_by_offset(local_btf,441 local_type->name_off);442 targ_name = btf__name_by_offset(targ_btf, targ_type->name_off);443 local_len = bpf_core_essential_name_len(local_name);444 targ_len = bpf_core_essential_name_len(targ_name);445 /* one of them is anonymous or both w/ same flavor-less names */446 return local_len == 0 || targ_len == 0 ||447 (local_len == targ_len &&448 strncmp(local_name, targ_name, local_len) == 0);449 }450 case BTF_KIND_INT:451 /* just reject deprecated bitfield-like integers; all other452 * integers are by default compatible between each other453 */454 return btf_int_offset(local_type) == 0 &&455 btf_int_offset(targ_type) == 0;456 case BTF_KIND_ARRAY:457 local_id = btf_array(local_type)->type;458 targ_id = btf_array(targ_type)->type;459 goto recur;460 default:461 return 0;462 }463}464 465/*466 * Given single high-level named field accessor in local type, find467 * corresponding high-level accessor for a target type. Along the way,468 * maintain low-level spec for target as well. Also keep updating target469 * bit offset.470 *471 * Searching is performed through recursive exhaustive enumeration of all472 * fields of a struct/union. If there are any anonymous (embedded)473 * structs/unions, they are recursively searched as well. If field with474 * desired name is found, check compatibility between local and target types,475 * before returning result.476 *477 * 1 is returned, if field is found.478 * 0 is returned if no compatible field is found.479 * <0 is returned on error.480 */481static int bpf_core_match_member(const struct btf *local_btf,482 const struct bpf_core_accessor *local_acc,483 const struct btf *targ_btf,484 __u32 targ_id,485 struct bpf_core_spec *spec,486 __u32 *next_targ_id)487{488 const struct btf_type *local_type, *targ_type;489 const struct btf_member *local_member, *m;490 const char *local_name, *targ_name;491 __u32 local_id;492 int i, n, found;493 494 targ_type = skip_mods_and_typedefs(targ_btf, targ_id, &targ_id);495 if (!targ_type)496 return -EINVAL;497 if (!btf_is_composite(targ_type))498 return 0;499 500 local_id = local_acc->type_id;501 local_type = btf_type_by_id(local_btf, local_id);502 local_member = btf_members(local_type) + local_acc->idx;503 local_name = btf__name_by_offset(local_btf, local_member->name_off);504 505 n = btf_vlen(targ_type);506 m = btf_members(targ_type);507 for (i = 0; i < n; i++, m++) {508 __u32 bit_offset;509 510 bit_offset = btf_member_bit_offset(targ_type, i);511 512 /* too deep struct/union/array nesting */513 if (spec->raw_len == BPF_CORE_SPEC_MAX_LEN)514 return -E2BIG;515 516 /* speculate this member will be the good one */517 spec->bit_offset += bit_offset;518 spec->raw_spec[spec->raw_len++] = i;519 520 targ_name = btf__name_by_offset(targ_btf, m->name_off);521 if (str_is_empty(targ_name)) {522 /* embedded struct/union, we need to go deeper */523 found = bpf_core_match_member(local_btf, local_acc,524 targ_btf, m->type,525 spec, next_targ_id);526 if (found) /* either found or error */527 return found;528 } else if (strcmp(local_name, targ_name) == 0) {529 /* matching named field */530 struct bpf_core_accessor *targ_acc;531 532 targ_acc = &spec->spec[spec->len++];533 targ_acc->type_id = targ_id;534 targ_acc->idx = i;535 targ_acc->name = targ_name;536 537 *next_targ_id = m->type;538 found = bpf_core_fields_are_compat(local_btf,539 local_member->type,540 targ_btf, m->type);541 if (!found)542 spec->len--; /* pop accessor */543 return found;544 }545 /* member turned out not to be what we looked for */546 spec->bit_offset -= bit_offset;547 spec->raw_len--;548 }549 550 return 0;551}552 553/*554 * Try to match local spec to a target type and, if successful, produce full555 * target spec (high-level, low-level + bit offset).556 */557static int bpf_core_spec_match(struct bpf_core_spec *local_spec,558 const struct btf *targ_btf, __u32 targ_id,559 struct bpf_core_spec *targ_spec)560{561 const struct btf_type *targ_type;562 const struct bpf_core_accessor *local_acc;563 struct bpf_core_accessor *targ_acc;564 int i, sz, matched;565 __u32 name_off;566 567 memset(targ_spec, 0, sizeof(*targ_spec));568 targ_spec->btf = targ_btf;569 targ_spec->root_type_id = targ_id;570 targ_spec->relo_kind = local_spec->relo_kind;571 572 if (core_relo_is_type_based(local_spec->relo_kind)) {573 if (local_spec->relo_kind == BPF_CORE_TYPE_MATCHES)574 return bpf_core_types_match(local_spec->btf,575 local_spec->root_type_id,576 targ_btf, targ_id);577 else578 return bpf_core_types_are_compat(local_spec->btf,579 local_spec->root_type_id,580 targ_btf, targ_id);581 }582 583 local_acc = &local_spec->spec[0];584 targ_acc = &targ_spec->spec[0];585 586 if (core_relo_is_enumval_based(local_spec->relo_kind)) {587 size_t local_essent_len, targ_essent_len;588 const char *targ_name;589 590 /* has to resolve to an enum */591 targ_type = skip_mods_and_typedefs(targ_spec->btf, targ_id, &targ_id);592 if (!btf_is_any_enum(targ_type))593 return 0;594 595 local_essent_len = bpf_core_essential_name_len(local_acc->name);596 597 for (i = 0; i < btf_vlen(targ_type); i++) {598 if (btf_is_enum(targ_type))599 name_off = btf_enum(targ_type)[i].name_off;600 else601 name_off = btf_enum64(targ_type)[i].name_off;602 603 targ_name = btf__name_by_offset(targ_spec->btf, name_off);604 targ_essent_len = bpf_core_essential_name_len(targ_name);605 if (targ_essent_len != local_essent_len)606 continue;607 if (strncmp(local_acc->name, targ_name, local_essent_len) == 0) {608 targ_acc->type_id = targ_id;609 targ_acc->idx = i;610 targ_acc->name = targ_name;611 targ_spec->len++;612 targ_spec->raw_spec[targ_spec->raw_len] = targ_acc->idx;613 targ_spec->raw_len++;614 return 1;615 }616 }617 return 0;618 }619 620 if (!core_relo_is_field_based(local_spec->relo_kind))621 return -EINVAL;622 623 for (i = 0; i < local_spec->len; i++, local_acc++, targ_acc++) {624 targ_type = skip_mods_and_typedefs(targ_spec->btf, targ_id,625 &targ_id);626 if (!targ_type)627 return -EINVAL;628 629 if (local_acc->name) {630 matched = bpf_core_match_member(local_spec->btf,631 local_acc,632 targ_btf, targ_id,633 targ_spec, &targ_id);634 if (matched <= 0)635 return matched;636 } else {637 /* for i=0, targ_id is already treated as array element638 * type (because it's the original struct), for others639 * we should find array element type first640 */641 if (i > 0) {642 const struct btf_array *a;643 bool flex;644 645 if (!btf_is_array(targ_type))646 return 0;647 648 a = btf_array(targ_type);649 flex = is_flex_arr(targ_btf, targ_acc - 1, a);650 if (!flex && local_acc->idx >= a->nelems)651 return 0;652 if (!skip_mods_and_typedefs(targ_btf, a->type,653 &targ_id))654 return -EINVAL;655 }656 657 /* too deep struct/union/array nesting */658 if (targ_spec->raw_len == BPF_CORE_SPEC_MAX_LEN)659 return -E2BIG;660 661 targ_acc->type_id = targ_id;662 targ_acc->idx = local_acc->idx;663 targ_acc->name = NULL;664 targ_spec->len++;665 targ_spec->raw_spec[targ_spec->raw_len] = targ_acc->idx;666 targ_spec->raw_len++;667 668 sz = btf__resolve_size(targ_btf, targ_id);669 if (sz < 0)670 return sz;671 targ_spec->bit_offset += local_acc->idx * sz * 8;672 }673 }674 675 return 1;676}677 678static int bpf_core_calc_field_relo(const char *prog_name,679 const struct bpf_core_relo *relo,680 const struct bpf_core_spec *spec,681 __u64 *val, __u32 *field_sz, __u32 *type_id,682 bool *validate)683{684 const struct bpf_core_accessor *acc;685 const struct btf_type *t;686 __u32 byte_off, byte_sz, bit_off, bit_sz, field_type_id;687 const struct btf_member *m;688 const struct btf_type *mt;689 bool bitfield;690 __s64 sz;691 692 *field_sz = 0;693 694 if (relo->kind == BPF_CORE_FIELD_EXISTS) {695 *val = spec ? 1 : 0;696 return 0;697 }698 699 if (!spec)700 return -EUCLEAN; /* request instruction poisoning */701 702 acc = &spec->spec[spec->len - 1];703 t = btf_type_by_id(spec->btf, acc->type_id);704 705 /* a[n] accessor needs special handling */706 if (!acc->name) {707 if (relo->kind == BPF_CORE_FIELD_BYTE_OFFSET) {708 *val = spec->bit_offset / 8;709 /* remember field size for load/store mem size */710 sz = btf__resolve_size(spec->btf, acc->type_id);711 if (sz < 0)712 return -EINVAL;713 *field_sz = sz;714 *type_id = acc->type_id;715 } else if (relo->kind == BPF_CORE_FIELD_BYTE_SIZE) {716 sz = btf__resolve_size(spec->btf, acc->type_id);717 if (sz < 0)718 return -EINVAL;719 *val = sz;720 } else {721 pr_warn("prog '%s': relo %d at insn #%d can't be applied to array access\n",722 prog_name, relo->kind, relo->insn_off / 8);723 return -EINVAL;724 }725 if (validate)726 *validate = true;727 return 0;728 }729 730 m = btf_members(t) + acc->idx;731 mt = skip_mods_and_typedefs(spec->btf, m->type, &field_type_id);732 bit_off = spec->bit_offset;733 bit_sz = btf_member_bitfield_size(t, acc->idx);734 735 bitfield = bit_sz > 0;736 if (bitfield) {737 byte_sz = mt->size;738 byte_off = bit_off / 8 / byte_sz * byte_sz;739 /* figure out smallest int size necessary for bitfield load */740 while (bit_off + bit_sz - byte_off * 8 > byte_sz * 8) {741 if (byte_sz >= 8) {742 /* bitfield can't be read with 64-bit read */743 pr_warn("prog '%s': relo %d at insn #%d can't be satisfied for bitfield\n",744 prog_name, relo->kind, relo->insn_off / 8);745 return -E2BIG;746 }747 byte_sz *= 2;748 byte_off = bit_off / 8 / byte_sz * byte_sz;749 }750 } else {751 sz = btf__resolve_size(spec->btf, field_type_id);752 if (sz < 0)753 return -EINVAL;754 byte_sz = sz;755 byte_off = spec->bit_offset / 8;756 bit_sz = byte_sz * 8;757 }758 759 /* for bitfields, all the relocatable aspects are ambiguous and we760 * might disagree with compiler, so turn off validation of expected761 * value, except for signedness762 */763 if (validate)764 *validate = !bitfield;765 766 switch (relo->kind) {767 case BPF_CORE_FIELD_BYTE_OFFSET:768 *val = byte_off;769 if (!bitfield) {770 *field_sz = byte_sz;771 *type_id = field_type_id;772 }773 break;774 case BPF_CORE_FIELD_BYTE_SIZE:775 *val = byte_sz;776 break;777 case BPF_CORE_FIELD_SIGNED:778 *val = (btf_is_any_enum(mt) && BTF_INFO_KFLAG(mt->info)) ||779 (btf_is_int(mt) && (btf_int_encoding(mt) & BTF_INT_SIGNED));780 if (validate)781 *validate = true; /* signedness is never ambiguous */782 break;783 case BPF_CORE_FIELD_LSHIFT_U64:784#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__785 *val = 64 - (bit_off + bit_sz - byte_off * 8);786#else787 *val = (8 - byte_sz) * 8 + (bit_off - byte_off * 8);788#endif789 break;790 case BPF_CORE_FIELD_RSHIFT_U64:791 *val = 64 - bit_sz;792 if (validate)793 *validate = true; /* right shift is never ambiguous */794 break;795 case BPF_CORE_FIELD_EXISTS:796 default:797 return -EOPNOTSUPP;798 }799 800 return 0;801}802 803static int bpf_core_calc_type_relo(const struct bpf_core_relo *relo,804 const struct bpf_core_spec *spec,805 __u64 *val, bool *validate)806{807 __s64 sz;808 809 /* by default, always check expected value in bpf_insn */810 if (validate)811 *validate = true;812 813 /* type-based relos return zero when target type is not found */814 if (!spec) {815 *val = 0;816 return 0;817 }818 819 switch (relo->kind) {820 case BPF_CORE_TYPE_ID_TARGET:821 *val = spec->root_type_id;822 /* type ID, embedded in bpf_insn, might change during linking,823 * so enforcing it is pointless824 */825 if (validate)826 *validate = false;827 break;828 case BPF_CORE_TYPE_EXISTS:829 case BPF_CORE_TYPE_MATCHES:830 *val = 1;831 break;832 case BPF_CORE_TYPE_SIZE:833 sz = btf__resolve_size(spec->btf, spec->root_type_id);834 if (sz < 0)835 return -EINVAL;836 *val = sz;837 break;838 case BPF_CORE_TYPE_ID_LOCAL:839 /* BPF_CORE_TYPE_ID_LOCAL is handled specially and shouldn't get here */840 default:841 return -EOPNOTSUPP;842 }843 844 return 0;845}846 847static int bpf_core_calc_enumval_relo(const struct bpf_core_relo *relo,848 const struct bpf_core_spec *spec,849 __u64 *val)850{851 const struct btf_type *t;852 853 switch (relo->kind) {854 case BPF_CORE_ENUMVAL_EXISTS:855 *val = spec ? 1 : 0;856 break;857 case BPF_CORE_ENUMVAL_VALUE:858 if (!spec)859 return -EUCLEAN; /* request instruction poisoning */860 t = btf_type_by_id(spec->btf, spec->spec[0].type_id);861 if (btf_is_enum(t))862 *val = btf_enum(t)[spec->spec[0].idx].val;863 else864 *val = btf_enum64_value(btf_enum64(t) + spec->spec[0].idx);865 break;866 default:867 return -EOPNOTSUPP;868 }869 870 return 0;871}872 873/* Calculate original and target relocation values, given local and target874 * specs and relocation kind. These values are calculated for each candidate.875 * If there are multiple candidates, resulting values should all be consistent876 * with each other. Otherwise, libbpf will refuse to proceed due to ambiguity.877 * If instruction has to be poisoned, *poison will be set to true.878 */879static int bpf_core_calc_relo(const char *prog_name,880 const struct bpf_core_relo *relo,881 int relo_idx,882 const struct bpf_core_spec *local_spec,883 const struct bpf_core_spec *targ_spec,884 struct bpf_core_relo_res *res)885{886 int err = -EOPNOTSUPP;887 888 res->orig_val = 0;889 res->new_val = 0;890 res->poison = false;891 res->validate = true;892 res->fail_memsz_adjust = false;893 res->orig_sz = res->new_sz = 0;894 res->orig_type_id = res->new_type_id = 0;895 896 if (core_relo_is_field_based(relo->kind)) {897 err = bpf_core_calc_field_relo(prog_name, relo, local_spec,898 &res->orig_val, &res->orig_sz,899 &res->orig_type_id, &res->validate);900 err = err ?: bpf_core_calc_field_relo(prog_name, relo, targ_spec,901 &res->new_val, &res->new_sz,902 &res->new_type_id, NULL);903 if (err)904 goto done;905 /* Validate if it's safe to adjust load/store memory size.906 * Adjustments are performed only if original and new memory907 * sizes differ.908 */909 res->fail_memsz_adjust = false;910 if (res->orig_sz != res->new_sz) {911 const struct btf_type *orig_t, *new_t;912 913 orig_t = btf_type_by_id(local_spec->btf, res->orig_type_id);914 new_t = btf_type_by_id(targ_spec->btf, res->new_type_id);915 916 /* There are two use cases in which it's safe to917 * adjust load/store's mem size:918 * - reading a 32-bit kernel pointer, while on BPF919 * size pointers are always 64-bit; in this case920 * it's safe to "downsize" instruction size due to921 * pointer being treated as unsigned integer with922 * zero-extended upper 32-bits;923 * - reading unsigned integers, again due to924 * zero-extension is preserving the value correctly.925 *926 * In all other cases it's incorrect to attempt to927 * load/store field because read value will be928 * incorrect, so we poison relocated instruction.929 */930 if (btf_is_ptr(orig_t) && btf_is_ptr(new_t))931 goto done;932 if (btf_is_int(orig_t) && btf_is_int(new_t) &&933 btf_int_encoding(orig_t) != BTF_INT_SIGNED &&934 btf_int_encoding(new_t) != BTF_INT_SIGNED)935 goto done;936 937 /* mark as invalid mem size adjustment, but this will938 * only be checked for LDX/STX/ST insns939 */940 res->fail_memsz_adjust = true;941 }942 } else if (core_relo_is_type_based(relo->kind)) {943 err = bpf_core_calc_type_relo(relo, local_spec, &res->orig_val, &res->validate);944 err = err ?: bpf_core_calc_type_relo(relo, targ_spec, &res->new_val, NULL);945 } else if (core_relo_is_enumval_based(relo->kind)) {946 err = bpf_core_calc_enumval_relo(relo, local_spec, &res->orig_val);947 err = err ?: bpf_core_calc_enumval_relo(relo, targ_spec, &res->new_val);948 }949 950done:951 if (err == -EUCLEAN) {952 /* EUCLEAN is used to signal instruction poisoning request */953 res->poison = true;954 err = 0;955 } else if (err == -EOPNOTSUPP) {956 /* EOPNOTSUPP means unknown/unsupported relocation */957 pr_warn("prog '%s': relo #%d: unrecognized CO-RE relocation %s (%d) at insn #%d\n",958 prog_name, relo_idx, core_relo_kind_str(relo->kind),959 relo->kind, relo->insn_off / 8);960 }961 962 return err;963}964 965/*966 * Turn instruction for which CO_RE relocation failed into invalid one with967 * distinct signature.968 */969static void bpf_core_poison_insn(const char *prog_name, int relo_idx,970 int insn_idx, struct bpf_insn *insn)971{972 pr_debug("prog '%s': relo #%d: substituting insn #%d w/ invalid insn\n",973 prog_name, relo_idx, insn_idx);974 insn->code = BPF_JMP | BPF_CALL;975 insn->dst_reg = 0;976 insn->src_reg = 0;977 insn->off = 0;978 /* if this instruction is reachable (not a dead code),979 * verifier will complain with the following message:980 * invalid func unknown#195896080981 */982 insn->imm = 195896080; /* => 0xbad2310 => "bad relo" */983}984 985static int insn_bpf_size_to_bytes(struct bpf_insn *insn)986{987 switch (BPF_SIZE(insn->code)) {988 case BPF_DW: return 8;989 case BPF_W: return 4;990 case BPF_H: return 2;991 case BPF_B: return 1;992 default: return -1;993 }994}995 996static int insn_bytes_to_bpf_size(__u32 sz)997{998 switch (sz) {999 case 8: return BPF_DW;1000 case 4: return BPF_W;1001 case 2: return BPF_H;1002 case 1: return BPF_B;1003 default: return -1;1004 }1005}1006 1007/*1008 * Patch relocatable BPF instruction.1009 *1010 * Patched value is determined by relocation kind and target specification.1011 * For existence relocations target spec will be NULL if field/type is not found.1012 * Expected insn->imm value is determined using relocation kind and local1013 * spec, and is checked before patching instruction. If actual insn->imm value1014 * is wrong, bail out with error.1015 *1016 * Currently supported classes of BPF instruction are:1017 * 1. rX = <imm> (assignment with immediate operand);1018 * 2. rX += <imm> (arithmetic operations with immediate operand);1019 * 3. rX = <imm64> (load with 64-bit immediate value);1020 * 4. rX = *(T *)(rY + <off>), where T is one of {u8, u16, u32, u64};1021 * 5. *(T *)(rX + <off>) = rY, where T is one of {u8, u16, u32, u64};1022 * 6. *(T *)(rX + <off>) = <imm>, where T is one of {u8, u16, u32, u64}.1023 */1024int bpf_core_patch_insn(const char *prog_name, struct bpf_insn *insn,1025 int insn_idx, const struct bpf_core_relo *relo,1026 int relo_idx, const struct bpf_core_relo_res *res)1027{1028 __u64 orig_val, new_val;1029 __u8 class;1030 1031 class = BPF_CLASS(insn->code);1032 1033 if (res->poison) {1034poison:1035 /* poison second part of ldimm64 to avoid confusing error from1036 * verifier about "unknown opcode 00"1037 */1038 if (is_ldimm64_insn(insn))1039 bpf_core_poison_insn(prog_name, relo_idx, insn_idx + 1, insn + 1);1040 bpf_core_poison_insn(prog_name, relo_idx, insn_idx, insn);1041 return 0;1042 }1043 1044 orig_val = res->orig_val;1045 new_val = res->new_val;1046 1047 switch (class) {1048 case BPF_ALU:1049 case BPF_ALU64:1050 if (BPF_SRC(insn->code) != BPF_K)1051 return -EINVAL;1052 if (res->validate && insn->imm != orig_val) {1053 pr_warn("prog '%s': relo #%d: unexpected insn #%d (ALU/ALU64) value: got %u, exp %llu -> %llu\n",1054 prog_name, relo_idx,1055 insn_idx, insn->imm, (unsigned long long)orig_val,1056 (unsigned long long)new_val);1057 return -EINVAL;1058 }1059 orig_val = insn->imm;1060 insn->imm = new_val;1061 pr_debug("prog '%s': relo #%d: patched insn #%d (ALU/ALU64) imm %llu -> %llu\n",1062 prog_name, relo_idx, insn_idx,1063 (unsigned long long)orig_val, (unsigned long long)new_val);1064 break;1065 case BPF_LDX:1066 case BPF_ST:1067 case BPF_STX:1068 if (res->validate && insn->off != orig_val) {1069 pr_warn("prog '%s': relo #%d: unexpected insn #%d (LDX/ST/STX) value: got %u, exp %llu -> %llu\n",1070 prog_name, relo_idx, insn_idx, insn->off, (unsigned long long)orig_val,1071 (unsigned long long)new_val);1072 return -EINVAL;1073 }1074 if (new_val > SHRT_MAX) {1075 pr_warn("prog '%s': relo #%d: insn #%d (LDX/ST/STX) value too big: %llu\n",1076 prog_name, relo_idx, insn_idx, (unsigned long long)new_val);1077 return -ERANGE;1078 }1079 if (res->fail_memsz_adjust) {1080 pr_warn("prog '%s': relo #%d: insn #%d (LDX/ST/STX) accesses field incorrectly. "1081 "Make sure you are accessing pointers, unsigned integers, or fields of matching type and size.\n",1082 prog_name, relo_idx, insn_idx);1083 goto poison;1084 }1085 1086 orig_val = insn->off;1087 insn->off = new_val;1088 pr_debug("prog '%s': relo #%d: patched insn #%d (LDX/ST/STX) off %llu -> %llu\n",1089 prog_name, relo_idx, insn_idx, (unsigned long long)orig_val,1090 (unsigned long long)new_val);1091 1092 if (res->new_sz != res->orig_sz) {1093 int insn_bytes_sz, insn_bpf_sz;1094 1095 insn_bytes_sz = insn_bpf_size_to_bytes(insn);1096 if (insn_bytes_sz != res->orig_sz) {1097 pr_warn("prog '%s': relo #%d: insn #%d (LDX/ST/STX) unexpected mem size: got %d, exp %u\n",1098 prog_name, relo_idx, insn_idx, insn_bytes_sz, res->orig_sz);1099 return -EINVAL;1100 }1101 1102 insn_bpf_sz = insn_bytes_to_bpf_size(res->new_sz);1103 if (insn_bpf_sz < 0) {1104 pr_warn("prog '%s': relo #%d: insn #%d (LDX/ST/STX) invalid new mem size: %u\n",1105 prog_name, relo_idx, insn_idx, res->new_sz);1106 return -EINVAL;1107 }1108 1109 insn->code = BPF_MODE(insn->code) | insn_bpf_sz | BPF_CLASS(insn->code);1110 pr_debug("prog '%s': relo #%d: patched insn #%d (LDX/ST/STX) mem_sz %u -> %u\n",1111 prog_name, relo_idx, insn_idx, res->orig_sz, res->new_sz);1112 }1113 break;1114 case BPF_LD: {1115 __u64 imm;1116 1117 if (!is_ldimm64_insn(insn) ||1118 insn[0].src_reg != 0 || insn[0].off != 0 ||1119 insn[1].code != 0 || insn[1].dst_reg != 0 ||1120 insn[1].src_reg != 0 || insn[1].off != 0) {1121 pr_warn("prog '%s': relo #%d: insn #%d (LDIMM64) has unexpected form\n",1122 prog_name, relo_idx, insn_idx);1123 return -EINVAL;1124 }1125 1126 imm = (__u32)insn[0].imm | ((__u64)insn[1].imm << 32);1127 if (res->validate && imm != orig_val) {1128 pr_warn("prog '%s': relo #%d: unexpected insn #%d (LDIMM64) value: got %llu, exp %llu -> %llu\n",1129 prog_name, relo_idx,1130 insn_idx, (unsigned long long)imm,1131 (unsigned long long)orig_val, (unsigned long long)new_val);1132 return -EINVAL;1133 }1134 1135 insn[0].imm = new_val;1136 insn[1].imm = new_val >> 32;1137 pr_debug("prog '%s': relo #%d: patched insn #%d (LDIMM64) imm64 %llu -> %llu\n",1138 prog_name, relo_idx, insn_idx,1139 (unsigned long long)imm, (unsigned long long)new_val);1140 break;1141 }1142 default:1143 pr_warn("prog '%s': relo #%d: trying to relocate unrecognized insn #%d, code:0x%x, src:0x%x, dst:0x%x, off:0x%x, imm:0x%x\n",1144 prog_name, relo_idx, insn_idx, insn->code,1145 insn->src_reg, insn->dst_reg, insn->off, insn->imm);1146 return -EINVAL;1147 }1148 1149 return 0;1150}1151 1152/* Output spec definition in the format:1153 * [<type-id>] (<type-name>) + <raw-spec> => <offset>@<spec>,1154 * where <spec> is a C-syntax view of recorded field access, e.g.: x.a[3].b1155 */1156int bpf_core_format_spec(char *buf, size_t buf_sz, const struct bpf_core_spec *spec)1157{1158 const struct btf_type *t;1159 const char *s;1160 __u32 type_id;1161 int i, len = 0;1162 1163#define append_buf(fmt, args...) \1164 ({ \1165 int r; \1166 r = snprintf(buf, buf_sz, fmt, ##args); \1167 len += r; \1168 if (r >= buf_sz) \1169 r = buf_sz; \1170 buf += r; \1171 buf_sz -= r; \1172 })1173 1174 type_id = spec->root_type_id;1175 t = btf_type_by_id(spec->btf, type_id);1176 s = btf__name_by_offset(spec->btf, t->name_off);1177 1178 append_buf("<%s> [%u] %s %s",1179 core_relo_kind_str(spec->relo_kind),1180 type_id, btf_kind_str(t), str_is_empty(s) ? "<anon>" : s);1181 1182 if (core_relo_is_type_based(spec->relo_kind))1183 return len;1184 1185 if (core_relo_is_enumval_based(spec->relo_kind)) {1186 t = skip_mods_and_typedefs(spec->btf, type_id, NULL);1187 if (btf_is_enum(t)) {1188 const struct btf_enum *e;1189 const char *fmt_str;1190 1191 e = btf_enum(t) + spec->raw_spec[0];1192 s = btf__name_by_offset(spec->btf, e->name_off);1193 fmt_str = BTF_INFO_KFLAG(t->info) ? "::%s = %d" : "::%s = %u";1194 append_buf(fmt_str, s, e->val);1195 } else {1196 const struct btf_enum64 *e;1197 const char *fmt_str;1198 1199 e = btf_enum64(t) + spec->raw_spec[0];1200 s = btf__name_by_offset(spec->btf, e->name_off);1201 fmt_str = BTF_INFO_KFLAG(t->info) ? "::%s = %lld" : "::%s = %llu";1202 append_buf(fmt_str, s, (unsigned long long)btf_enum64_value(e));1203 }1204 return len;1205 }1206 1207 if (core_relo_is_field_based(spec->relo_kind)) {1208 for (i = 0; i < spec->len; i++) {1209 if (spec->spec[i].name)1210 append_buf(".%s", spec->spec[i].name);1211 else if (i > 0 || spec->spec[i].idx > 0)1212 append_buf("[%u]", spec->spec[i].idx);1213 }1214 1215 append_buf(" (");1216 for (i = 0; i < spec->raw_len; i++)1217 append_buf("%s%d", i == 0 ? "" : ":", spec->raw_spec[i]);1218 1219 if (spec->bit_offset % 8)1220 append_buf(" @ offset %u.%u)", spec->bit_offset / 8, spec->bit_offset % 8);1221 else1222 append_buf(" @ offset %u)", spec->bit_offset / 8);1223 return len;1224 }1225 1226 return len;1227#undef append_buf1228}1229 1230/*1231 * Calculate CO-RE relocation target result.1232 *1233 * The outline and important points of the algorithm:1234 * 1. For given local type, find corresponding candidate target types.1235 * Candidate type is a type with the same "essential" name, ignoring1236 * everything after last triple underscore (___). E.g., `sample`,1237 * `sample___flavor_one`, `sample___flavor_another_one`, are all candidates1238 * for each other. Names with triple underscore are referred to as1239 * "flavors" and are useful, among other things, to allow to1240 * specify/support incompatible variations of the same kernel struct, which1241 * might differ between different kernel versions and/or build1242 * configurations.1243 *1244 * N.B. Struct "flavors" could be generated by bpftool's BTF-to-C1245 * converter, when deduplicated BTF of a kernel still contains more than1246 * one different types with the same name. In that case, ___2, ___3, etc1247 * are appended starting from second name conflict. But start flavors are1248 * also useful to be defined "locally", in BPF program, to extract same1249 * data from incompatible changes between different kernel1250 * versions/configurations. For instance, to handle field renames between1251 * kernel versions, one can use two flavors of the struct name with the1252 * same common name and use conditional relocations to extract that field,1253 * depending on target kernel version.1254 * 2. For each candidate type, try to match local specification to this1255 * candidate target type. Matching involves finding corresponding1256 * high-level spec accessors, meaning that all named fields should match,1257 * as well as all array accesses should be within the actual bounds. Also,1258 * types should be compatible (see bpf_core_fields_are_compat for details).1259 * 3. It is supported and expected that there might be multiple flavors1260 * matching the spec. As long as all the specs resolve to the same set of1261 * offsets across all candidates, there is no error. If there is any1262 * ambiguity, CO-RE relocation will fail. This is necessary to accommodate1263 * imperfection of BTF deduplication, which can cause slight duplication of1264 * the same BTF type, if some directly or indirectly referenced (by1265 * pointer) type gets resolved to different actual types in different1266 * object files. If such a situation occurs, deduplicated BTF will end up1267 * with two (or more) structurally identical types, which differ only in1268 * types they refer to through pointer. This should be OK in most cases and1269 * is not an error.1270 * 4. Candidate types search is performed by linearly scanning through all1271 * types in target BTF. It is anticipated that this is overall more1272 * efficient memory-wise and not significantly worse (if not better)1273 * CPU-wise compared to prebuilding a map from all local type names to1274 * a list of candidate type names. It's also sped up by caching resolved1275 * list of matching candidates per each local "root" type ID, that has at1276 * least one bpf_core_relo associated with it. This list is shared1277 * between multiple relocations for the same type ID and is updated as some1278 * of the candidates are pruned due to structural incompatibility.1279 */1280int bpf_core_calc_relo_insn(const char *prog_name,1281 const struct bpf_core_relo *relo,1282 int relo_idx,1283 const struct btf *local_btf,1284 struct bpf_core_cand_list *cands,1285 struct bpf_core_spec *specs_scratch,1286 struct bpf_core_relo_res *targ_res)1287{1288 struct bpf_core_spec *local_spec = &specs_scratch[0];1289 struct bpf_core_spec *cand_spec = &specs_scratch[1];1290 struct bpf_core_spec *targ_spec = &specs_scratch[2];1291 struct bpf_core_relo_res cand_res;1292 const struct btf_type *local_type;1293 const char *local_name;1294 __u32 local_id;1295 char spec_buf[256];1296 int i, j, err;1297 1298 local_id = relo->type_id;1299 local_type = btf_type_by_id(local_btf, local_id);1300 local_name = btf__name_by_offset(local_btf, local_type->name_off);1301 if (!local_name)1302 return -EINVAL;1303 1304 err = bpf_core_parse_spec(prog_name, local_btf, relo, local_spec);1305 if (err) {1306 const char *spec_str;1307 1308 spec_str = btf__name_by_offset(local_btf, relo->access_str_off);1309 pr_warn("prog '%s': relo #%d: parsing [%d] %s %s + %s failed: %d\n",1310 prog_name, relo_idx, local_id, btf_kind_str(local_type),1311 str_is_empty(local_name) ? "<anon>" : local_name,1312 spec_str ?: "<?>", err);1313 return -EINVAL;1314 }1315 1316 bpf_core_format_spec(spec_buf, sizeof(spec_buf), local_spec);1317 pr_debug("prog '%s': relo #%d: %s\n", prog_name, relo_idx, spec_buf);1318 1319 /* TYPE_ID_LOCAL relo is special and doesn't need candidate search */1320 if (relo->kind == BPF_CORE_TYPE_ID_LOCAL) {1321 /* bpf_insn's imm value could get out of sync during linking */1322 memset(targ_res, 0, sizeof(*targ_res));1323 targ_res->validate = false;1324 targ_res->poison = false;1325 targ_res->orig_val = local_spec->root_type_id;1326 targ_res->new_val = local_spec->root_type_id;1327 return 0;1328 }1329 1330 /* libbpf doesn't support candidate search for anonymous types */1331 if (str_is_empty(local_name)) {1332 pr_warn("prog '%s': relo #%d: <%s> (%d) relocation doesn't support anonymous types\n",1333 prog_name, relo_idx, core_relo_kind_str(relo->kind), relo->kind);1334 return -EOPNOTSUPP;1335 }1336 1337 for (i = 0, j = 0; i < cands->len; i++) {1338 err = bpf_core_spec_match(local_spec, cands->cands[i].btf,1339 cands->cands[i].id, cand_spec);1340 if (err < 0) {1341 bpf_core_format_spec(spec_buf, sizeof(spec_buf), cand_spec);1342 pr_warn("prog '%s': relo #%d: error matching candidate #%d %s: %d\n ",1343 prog_name, relo_idx, i, spec_buf, err);1344 return err;1345 }1346 1347 bpf_core_format_spec(spec_buf, sizeof(spec_buf), cand_spec);1348 pr_debug("prog '%s': relo #%d: %s candidate #%d %s\n", prog_name,1349 relo_idx, err == 0 ? "non-matching" : "matching", i, spec_buf);1350 1351 if (err == 0)1352 continue;1353 1354 err = bpf_core_calc_relo(prog_name, relo, relo_idx, local_spec, cand_spec, &cand_res);1355 if (err)1356 return err;1357 1358 if (j == 0) {1359 *targ_res = cand_res;1360 *targ_spec = *cand_spec;1361 } else if (cand_spec->bit_offset != targ_spec->bit_offset) {1362 /* if there are many field relo candidates, they1363 * should all resolve to the same bit offset1364 */1365 pr_warn("prog '%s': relo #%d: field offset ambiguity: %u != %u\n",1366 prog_name, relo_idx, cand_spec->bit_offset,1367 targ_spec->bit_offset);1368 return -EINVAL;1369 } else if (cand_res.poison != targ_res->poison ||1370 cand_res.new_val != targ_res->new_val) {1371 /* all candidates should result in the same relocation1372 * decision and value, otherwise it's dangerous to1373 * proceed due to ambiguity1374 */1375 pr_warn("prog '%s': relo #%d: relocation decision ambiguity: %s %llu != %s %llu\n",1376 prog_name, relo_idx,1377 cand_res.poison ? "failure" : "success",1378 (unsigned long long)cand_res.new_val,1379 targ_res->poison ? "failure" : "success",1380 (unsigned long long)targ_res->new_val);1381 return -EINVAL;1382 }1383 1384 cands->cands[j++] = cands->cands[i];1385 }1386 1387 /*1388 * For BPF_CORE_FIELD_EXISTS relo or when used BPF program has field1389 * existence checks or kernel version/config checks, it's expected1390 * that we might not find any candidates. In this case, if field1391 * wasn't found in any candidate, the list of candidates shouldn't1392 * change at all, we'll just handle relocating appropriately,1393 * depending on relo's kind.1394 */1395 if (j > 0)1396 cands->len = j;1397 1398 /*1399 * If no candidates were found, it might be both a programmer error,1400 * as well as expected case, depending whether instruction w/1401 * relocation is guarded in some way that makes it unreachable (dead1402 * code) if relocation can't be resolved. This is handled in1403 * bpf_core_patch_insn() uniformly by replacing that instruction with1404 * BPF helper call insn (using invalid helper ID). If that instruction1405 * is indeed unreachable, then it will be ignored and eliminated by1406 * verifier. If it was an error, then verifier will complain and point1407 * to a specific instruction number in its log.1408 */1409 if (j == 0) {1410 pr_debug("prog '%s': relo #%d: no matching targets found\n",1411 prog_name, relo_idx);1412 1413 /* calculate single target relo result explicitly */1414 err = bpf_core_calc_relo(prog_name, relo, relo_idx, local_spec, NULL, targ_res);1415 if (err)1416 return err;1417 }1418 1419 return 0;1420}1421 1422static bool bpf_core_names_match(const struct btf *local_btf, size_t local_name_off,1423 const struct btf *targ_btf, size_t targ_name_off)1424{1425 const char *local_n, *targ_n;1426 size_t local_len, targ_len;1427 1428 local_n = btf__name_by_offset(local_btf, local_name_off);1429 targ_n = btf__name_by_offset(targ_btf, targ_name_off);1430 1431 if (str_is_empty(targ_n))1432 return str_is_empty(local_n);1433 1434 targ_len = bpf_core_essential_name_len(targ_n);1435 local_len = bpf_core_essential_name_len(local_n);1436 1437 return targ_len == local_len && strncmp(local_n, targ_n, local_len) == 0;1438}1439 1440static int bpf_core_enums_match(const struct btf *local_btf, const struct btf_type *local_t,1441 const struct btf *targ_btf, const struct btf_type *targ_t)1442{1443 __u16 local_vlen = btf_vlen(local_t);1444 __u16 targ_vlen = btf_vlen(targ_t);1445 int i, j;1446 1447 if (local_t->size != targ_t->size)1448 return 0;1449 1450 if (local_vlen > targ_vlen)1451 return 0;1452 1453 /* iterate over the local enum's variants and make sure each has1454 * a symbolic name correspondent in the target1455 */1456 for (i = 0; i < local_vlen; i++) {1457 bool matched = false;1458 __u32 local_n_off, targ_n_off;1459 1460 local_n_off = btf_is_enum(local_t) ? btf_enum(local_t)[i].name_off :1461 btf_enum64(local_t)[i].name_off;1462 1463 for (j = 0; j < targ_vlen; j++) {1464 targ_n_off = btf_is_enum(targ_t) ? btf_enum(targ_t)[j].name_off :1465 btf_enum64(targ_t)[j].name_off;1466 1467 if (bpf_core_names_match(local_btf, local_n_off, targ_btf, targ_n_off)) {1468 matched = true;1469 break;1470 }1471 }1472 1473 if (!matched)1474 return 0;1475 }1476 return 1;1477}1478 1479static int bpf_core_composites_match(const struct btf *local_btf, const struct btf_type *local_t,1480 const struct btf *targ_btf, const struct btf_type *targ_t,1481 bool behind_ptr, int level)1482{1483 const struct btf_member *local_m = btf_members(local_t);1484 __u16 local_vlen = btf_vlen(local_t);1485 __u16 targ_vlen = btf_vlen(targ_t);1486 int i, j, err;1487 1488 if (local_vlen > targ_vlen)1489 return 0;1490 1491 /* check that all local members have a match in the target */1492 for (i = 0; i < local_vlen; i++, local_m++) {1493 const struct btf_member *targ_m = btf_members(targ_t);1494 bool matched = false;1495 1496 for (j = 0; j < targ_vlen; j++, targ_m++) {1497 if (!bpf_core_names_match(local_btf, local_m->name_off,1498 targ_btf, targ_m->name_off))1499 continue;1500 1501 err = __bpf_core_types_match(local_btf, local_m->type, targ_btf,1502 targ_m->type, behind_ptr, level - 1);1503 if (err < 0)1504 return err;1505 if (err > 0) {1506 matched = true;1507 break;1508 }1509 }1510 1511 if (!matched)1512 return 0;1513 }1514 return 1;1515}1516 1517/* Check that two types "match". This function assumes that root types were1518 * already checked for name match.1519 *1520 * The matching relation is defined as follows:1521 * - modifiers and typedefs are stripped (and, hence, effectively ignored)1522 * - generally speaking types need to be of same kind (struct vs. struct, union1523 * vs. union, etc.)1524 * - exceptions are struct/union behind a pointer which could also match a1525 * forward declaration of a struct or union, respectively, and enum vs.1526 * enum64 (see below)1527 * Then, depending on type:1528 * - integers:1529 * - match if size and signedness match1530 * - arrays & pointers:1531 * - target types are recursively matched1532 * - structs & unions:1533 * - local members need to exist in target with the same name1534 * - for each member we recursively check match unless it is already behind a1535 * pointer, in which case we only check matching names and compatible kind1536 * - enums:1537 * - local variants have to have a match in target by symbolic name (but not1538 * numeric value)1539 * - size has to match (but enum may match enum64 and vice versa)1540 * - function pointers:1541 * - number and position of arguments in local type has to match target1542 * - for each argument and the return value we recursively check match1543 */1544int __bpf_core_types_match(const struct btf *local_btf, __u32 local_id, const struct btf *targ_btf,1545 __u32 targ_id, bool behind_ptr, int level)1546{1547 const struct btf_type *local_t, *targ_t;1548 int depth = 32; /* max recursion depth */1549 __u16 local_k, targ_k;1550 1551 if (level <= 0)1552 return -EINVAL;1553 1554recur:1555 depth--;1556 if (depth < 0)1557 return -EINVAL;1558 1559 local_t = skip_mods_and_typedefs(local_btf, local_id, &local_id);1560 targ_t = skip_mods_and_typedefs(targ_btf, targ_id, &targ_id);1561 if (!local_t || !targ_t)1562 return -EINVAL;1563 1564 /* While the name check happens after typedefs are skipped, root-level1565 * typedefs would still be name-matched as that's the contract with1566 * callers.1567 */1568 if (!bpf_core_names_match(local_btf, local_t->name_off, targ_btf, targ_t->name_off))1569 return 0;1570 1571 local_k = btf_kind(local_t);1572 targ_k = btf_kind(targ_t);1573 1574 switch (local_k) {1575 case BTF_KIND_UNKN:1576 return local_k == targ_k;1577 case BTF_KIND_FWD: {1578 bool local_f = BTF_INFO_KFLAG(local_t->info);1579 1580 if (behind_ptr) {1581 if (local_k == targ_k)1582 return local_f == BTF_INFO_KFLAG(targ_t->info);1583 1584 /* for forward declarations kflag dictates whether the1585 * target is a struct (0) or union (1)1586 */1587 return (targ_k == BTF_KIND_STRUCT && !local_f) ||1588 (targ_k == BTF_KIND_UNION && local_f);1589 } else {1590 if (local_k != targ_k)1591 return 0;1592 1593 /* match if the forward declaration is for the same kind */1594 return local_f == BTF_INFO_KFLAG(targ_t->info);1595 }1596 }1597 case BTF_KIND_ENUM:1598 case BTF_KIND_ENUM64:1599 if (!btf_is_any_enum(targ_t))1600 return 0;1601 1602 return bpf_core_enums_match(local_btf, local_t, targ_btf, targ_t);1603 case BTF_KIND_STRUCT:1604 case BTF_KIND_UNION:1605 if (behind_ptr) {1606 bool targ_f = BTF_INFO_KFLAG(targ_t->info);1607 1608 if (local_k == targ_k)1609 return 1;1610 1611 if (targ_k != BTF_KIND_FWD)1612 return 0;1613 1614 return (local_k == BTF_KIND_UNION) == targ_f;1615 } else {1616 if (local_k != targ_k)1617 return 0;1618 1619 return bpf_core_composites_match(local_btf, local_t, targ_btf, targ_t,1620 behind_ptr, level);1621 }1622 case BTF_KIND_INT: {1623 __u8 local_sgn;1624 __u8 targ_sgn;1625 1626 if (local_k != targ_k)1627 return 0;1628 1629 local_sgn = btf_int_encoding(local_t) & BTF_INT_SIGNED;1630 targ_sgn = btf_int_encoding(targ_t) & BTF_INT_SIGNED;1631 1632 return local_t->size == targ_t->size && local_sgn == targ_sgn;1633 }1634 case BTF_KIND_PTR:1635 if (local_k != targ_k)1636 return 0;1637 1638 behind_ptr = true;1639 1640 local_id = local_t->type;1641 targ_id = targ_t->type;1642 goto recur;1643 case BTF_KIND_ARRAY: {1644 const struct btf_array *local_array = btf_array(local_t);1645 const struct btf_array *targ_array = btf_array(targ_t);1646 1647 if (local_k != targ_k)1648 return 0;1649 1650 if (local_array->nelems != targ_array->nelems)1651 return 0;1652 1653 local_id = local_array->type;1654 targ_id = targ_array->type;1655 goto recur;1656 }1657 case BTF_KIND_FUNC_PROTO: {1658 struct btf_param *local_p = btf_params(local_t);1659 struct btf_param *targ_p = btf_params(targ_t);1660 __u16 local_vlen = btf_vlen(local_t);1661 __u16 targ_vlen = btf_vlen(targ_t);1662 int i, err;1663 1664 if (local_k != targ_k)1665 return 0;1666 1667 if (local_vlen != targ_vlen)1668 return 0;1669 1670 for (i = 0; i < local_vlen; i++, local_p++, targ_p++) {1671 err = __bpf_core_types_match(local_btf, local_p->type, targ_btf,1672 targ_p->type, behind_ptr, level - 1);1673 if (err <= 0)1674 return err;1675 }1676 1677 /* tail recurse for return type check */1678 local_id = local_t->type;1679 targ_id = targ_t->type;1680 goto recur;1681 }1682 default:1683 pr_warn("unexpected kind %s relocated, local [%d], target [%d]\n",1684 btf_kind_str(local_t), local_id, targ_id);1685 return 0;1686 }1687}1688