424 lines · c
1/* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */2#ifndef __BPF_HELPERS__3#define __BPF_HELPERS__4 5/*6 * Note that bpf programs need to include either7 * vmlinux.h (auto-generated from BTF) or linux/types.h8 * in advance since bpf_helper_defs.h uses such types9 * as __u64.10 */11#include "bpf_helper_defs.h"12 13#define __uint(name, val) int (*name)[val]14#define __type(name, val) typeof(val) *name15#define __array(name, val) typeof(val) *name[]16#define __ulong(name, val) enum { ___bpf_concat(__unique_value, __COUNTER__) = val } name17 18/*19 * Helper macro to place programs, maps, license in20 * different sections in elf_bpf file. Section names21 * are interpreted by libbpf depending on the context (BPF programs, BPF maps,22 * extern variables, etc).23 * To allow use of SEC() with externs (e.g., for extern .maps declarations),24 * make sure __attribute__((unused)) doesn't trigger compilation warning.25 */26#if __GNUC__ && !__clang__27 28/*29 * Pragma macros are broken on GCC30 * https://gcc.gnu.org/bugzilla/show_bug.cgi?id=5557831 * https://gcc.gnu.org/bugzilla/show_bug.cgi?id=9040032 */33#define SEC(name) __attribute__((section(name), used))34 35#else36 37#define SEC(name) \38 _Pragma("GCC diagnostic push") \39 _Pragma("GCC diagnostic ignored \"-Wignored-attributes\"") \40 __attribute__((section(name), used)) \41 _Pragma("GCC diagnostic pop") \42 43#endif44 45/* Avoid 'linux/stddef.h' definition of '__always_inline'. */46#undef __always_inline47#define __always_inline inline __attribute__((always_inline))48 49#ifndef __noinline50#define __noinline __attribute__((noinline))51#endif52#ifndef __weak53#define __weak __attribute__((weak))54#endif55 56/*57 * Use __hidden attribute to mark a non-static BPF subprogram effectively58 * static for BPF verifier's verification algorithm purposes, allowing more59 * extensive and permissive BPF verification process, taking into account60 * subprogram's caller context.61 */62#define __hidden __attribute__((visibility("hidden")))63 64/* When utilizing vmlinux.h with BPF CO-RE, user BPF programs can't include65 * any system-level headers (such as stddef.h, linux/version.h, etc), and66 * commonly-used macros like NULL and KERNEL_VERSION aren't available through67 * vmlinux.h. This just adds unnecessary hurdles and forces users to re-define68 * them on their own. So as a convenience, provide such definitions here.69 */70#ifndef NULL71#define NULL ((void *)0)72#endif73 74#ifndef KERNEL_VERSION75#define KERNEL_VERSION(a, b, c) (((a) << 16) + ((b) << 8) + ((c) > 255 ? 255 : (c)))76#endif77 78/*79 * Helper macros to manipulate data structures80 */81 82/* offsetof() definition that uses __builtin_offset() might not preserve field83 * offset CO-RE relocation properly, so force-redefine offsetof() using84 * old-school approach which works with CO-RE correctly85 */86#undef offsetof87#define offsetof(type, member) ((unsigned long)&((type *)0)->member)88 89/* redefined container_of() to ensure we use the above offsetof() macro */90#undef container_of91#define container_of(ptr, type, member) \92 ({ \93 void *__mptr = (void *)(ptr); \94 ((type *)(__mptr - offsetof(type, member))); \95 })96 97/*98 * Compiler (optimization) barrier.99 */100#ifndef barrier101#define barrier() asm volatile("" ::: "memory")102#endif103 104/* Variable-specific compiler (optimization) barrier. It's a no-op which makes105 * compiler believe that there is some black box modification of a given106 * variable and thus prevents compiler from making extra assumption about its107 * value and potential simplifications and optimizations on this variable.108 *109 * E.g., compiler might often delay or even omit 32-bit to 64-bit casting of110 * a variable, making some code patterns unverifiable. Putting barrier_var()111 * in place will ensure that cast is performed before the barrier_var()112 * invocation, because compiler has to pessimistically assume that embedded113 * asm section might perform some extra operations on that variable.114 *115 * This is a variable-specific variant of more global barrier().116 */117#ifndef barrier_var118#define barrier_var(var) asm volatile("" : "+r"(var))119#endif120 121/*122 * Helper macro to throw a compilation error if __bpf_unreachable() gets123 * built into the resulting code. This works given BPF back end does not124 * implement __builtin_trap(). This is useful to assert that certain paths125 * of the program code are never used and hence eliminated by the compiler.126 *127 * For example, consider a switch statement that covers known cases used by128 * the program. __bpf_unreachable() can then reside in the default case. If129 * the program gets extended such that a case is not covered in the switch130 * statement, then it will throw a build error due to the default case not131 * being compiled out.132 */133#ifndef __bpf_unreachable134# define __bpf_unreachable() __builtin_trap()135#endif136 137/*138 * Helper function to perform a tail call with a constant/immediate map slot.139 */140#if (defined(__clang__) && __clang_major__ >= 8) || (!defined(__clang__) && __GNUC__ > 12)141#if defined(__bpf__)142static __always_inline void143bpf_tail_call_static(void *ctx, const void *map, const __u32 slot)144{145 if (!__builtin_constant_p(slot))146 __bpf_unreachable();147 148 /*149 * Provide a hard guarantee that LLVM won't optimize setting r2 (map150 * pointer) and r3 (constant map index) from _different paths_ ending151 * up at the _same_ call insn as otherwise we won't be able to use the152 * jmpq/nopl retpoline-free patching by the x86-64 JIT in the kernel153 * given they mismatch. See also d2e4c1e6c294 ("bpf: Constant map key154 * tracking for prog array pokes") for details on verifier tracking.155 *156 * Note on clobber list: we need to stay in-line with BPF calling157 * convention, so even if we don't end up using r0, r4, r5, we need158 * to mark them as clobber so that LLVM doesn't end up using them159 * before / after the call.160 */161 asm volatile("r1 = %[ctx]\n\t"162 "r2 = %[map]\n\t"163 "r3 = %[slot]\n\t"164 "call 12"165 :: [ctx]"r"(ctx), [map]"r"(map), [slot]"i"(slot)166 : "r0", "r1", "r2", "r3", "r4", "r5");167}168#endif169#endif170 171enum libbpf_pin_type {172 LIBBPF_PIN_NONE,173 /* PIN_BY_NAME: pin maps by name (in /sys/fs/bpf by default) */174 LIBBPF_PIN_BY_NAME,175};176 177enum libbpf_tristate {178 TRI_NO = 0,179 TRI_YES = 1,180 TRI_MODULE = 2,181};182 183#define __kconfig __attribute__((section(".kconfig")))184#define __ksym __attribute__((section(".ksyms")))185#define __kptr_untrusted __attribute__((btf_type_tag("kptr_untrusted")))186#define __kptr __attribute__((btf_type_tag("kptr")))187#define __percpu_kptr __attribute__((btf_type_tag("percpu_kptr")))188 189#if defined (__clang__)190#define bpf_ksym_exists(sym) ({ \191 _Static_assert(!__builtin_constant_p(!!sym), \192 #sym " should be marked as __weak"); \193 !!sym; \194})195#elif __GNUC__ > 8196#define bpf_ksym_exists(sym) ({ \197 _Static_assert(__builtin_has_attribute (*sym, __weak__), \198 #sym " should be marked as __weak"); \199 !!sym; \200})201#else202#define bpf_ksym_exists(sym) !!sym203#endif204 205#define __arg_ctx __attribute__((btf_decl_tag("arg:ctx")))206#define __arg_nonnull __attribute((btf_decl_tag("arg:nonnull")))207#define __arg_nullable __attribute((btf_decl_tag("arg:nullable")))208#define __arg_trusted __attribute((btf_decl_tag("arg:trusted")))209#define __arg_arena __attribute((btf_decl_tag("arg:arena")))210 211#ifndef ___bpf_concat212#define ___bpf_concat(a, b) a ## b213#endif214#ifndef ___bpf_apply215#define ___bpf_apply(fn, n) ___bpf_concat(fn, n)216#endif217#ifndef ___bpf_nth218#define ___bpf_nth(_, _1, _2, _3, _4, _5, _6, _7, _8, _9, _a, _b, _c, N, ...) N219#endif220#ifndef ___bpf_narg221#define ___bpf_narg(...) \222 ___bpf_nth(_, ##__VA_ARGS__, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0)223#endif224 225#define ___bpf_fill0(arr, p, x) do {} while (0)226#define ___bpf_fill1(arr, p, x) arr[p] = x227#define ___bpf_fill2(arr, p, x, args...) arr[p] = x; ___bpf_fill1(arr, p + 1, args)228#define ___bpf_fill3(arr, p, x, args...) arr[p] = x; ___bpf_fill2(arr, p + 1, args)229#define ___bpf_fill4(arr, p, x, args...) arr[p] = x; ___bpf_fill3(arr, p + 1, args)230#define ___bpf_fill5(arr, p, x, args...) arr[p] = x; ___bpf_fill4(arr, p + 1, args)231#define ___bpf_fill6(arr, p, x, args...) arr[p] = x; ___bpf_fill5(arr, p + 1, args)232#define ___bpf_fill7(arr, p, x, args...) arr[p] = x; ___bpf_fill6(arr, p + 1, args)233#define ___bpf_fill8(arr, p, x, args...) arr[p] = x; ___bpf_fill7(arr, p + 1, args)234#define ___bpf_fill9(arr, p, x, args...) arr[p] = x; ___bpf_fill8(arr, p + 1, args)235#define ___bpf_fill10(arr, p, x, args...) arr[p] = x; ___bpf_fill9(arr, p + 1, args)236#define ___bpf_fill11(arr, p, x, args...) arr[p] = x; ___bpf_fill10(arr, p + 1, args)237#define ___bpf_fill12(arr, p, x, args...) arr[p] = x; ___bpf_fill11(arr, p + 1, args)238#define ___bpf_fill(arr, args...) \239 ___bpf_apply(___bpf_fill, ___bpf_narg(args))(arr, 0, args)240 241/*242 * BPF_SEQ_PRINTF to wrap bpf_seq_printf to-be-printed values243 * in a structure.244 */245#define BPF_SEQ_PRINTF(seq, fmt, args...) \246({ \247 static const char ___fmt[] = fmt; \248 unsigned long long ___param[___bpf_narg(args)]; \249 \250 _Pragma("GCC diagnostic push") \251 _Pragma("GCC diagnostic ignored \"-Wint-conversion\"") \252 ___bpf_fill(___param, args); \253 _Pragma("GCC diagnostic pop") \254 \255 bpf_seq_printf(seq, ___fmt, sizeof(___fmt), \256 ___param, sizeof(___param)); \257})258 259/*260 * BPF_SNPRINTF wraps the bpf_snprintf helper with variadic arguments instead of261 * an array of u64.262 */263#define BPF_SNPRINTF(out, out_size, fmt, args...) \264({ \265 static const char ___fmt[] = fmt; \266 unsigned long long ___param[___bpf_narg(args)]; \267 \268 _Pragma("GCC diagnostic push") \269 _Pragma("GCC diagnostic ignored \"-Wint-conversion\"") \270 ___bpf_fill(___param, args); \271 _Pragma("GCC diagnostic pop") \272 \273 bpf_snprintf(out, out_size, ___fmt, \274 ___param, sizeof(___param)); \275})276 277#ifdef BPF_NO_GLOBAL_DATA278#define BPF_PRINTK_FMT_MOD279#else280#define BPF_PRINTK_FMT_MOD static const281#endif282 283#define __bpf_printk(fmt, ...) \284({ \285 BPF_PRINTK_FMT_MOD char ____fmt[] = fmt; \286 bpf_trace_printk(____fmt, sizeof(____fmt), \287 ##__VA_ARGS__); \288})289 290/*291 * __bpf_vprintk wraps the bpf_trace_vprintk helper with variadic arguments292 * instead of an array of u64.293 */294#define __bpf_vprintk(fmt, args...) \295({ \296 static const char ___fmt[] = fmt; \297 unsigned long long ___param[___bpf_narg(args)]; \298 \299 _Pragma("GCC diagnostic push") \300 _Pragma("GCC diagnostic ignored \"-Wint-conversion\"") \301 ___bpf_fill(___param, args); \302 _Pragma("GCC diagnostic pop") \303 \304 bpf_trace_vprintk(___fmt, sizeof(___fmt), \305 ___param, sizeof(___param)); \306})307 308/* Use __bpf_printk when bpf_printk call has 3 or fewer fmt args309 * Otherwise use __bpf_vprintk310 */311#define ___bpf_pick_printk(...) \312 ___bpf_nth(_, ##__VA_ARGS__, __bpf_vprintk, __bpf_vprintk, __bpf_vprintk, \313 __bpf_vprintk, __bpf_vprintk, __bpf_vprintk, __bpf_vprintk, \314 __bpf_vprintk, __bpf_vprintk, __bpf_printk /*3*/, __bpf_printk /*2*/,\315 __bpf_printk /*1*/, __bpf_printk /*0*/)316 317/* Helper macro to print out debug messages */318#define bpf_printk(fmt, args...) ___bpf_pick_printk(args)(fmt, ##args)319 320struct bpf_iter_num;321 322extern int bpf_iter_num_new(struct bpf_iter_num *it, int start, int end) __weak __ksym;323extern int *bpf_iter_num_next(struct bpf_iter_num *it) __weak __ksym;324extern void bpf_iter_num_destroy(struct bpf_iter_num *it) __weak __ksym;325 326#ifndef bpf_for_each327/* bpf_for_each(iter_type, cur_elem, args...) provides generic construct for328 * using BPF open-coded iterators without having to write mundane explicit329 * low-level loop logic. Instead, it provides for()-like generic construct330 * that can be used pretty naturally. E.g., for some hypothetical cgroup331 * iterator, you'd write:332 *333 * struct cgroup *cg, *parent_cg = <...>;334 *335 * bpf_for_each(cgroup, cg, parent_cg, CG_ITER_CHILDREN) {336 * bpf_printk("Child cgroup id = %d", cg->cgroup_id);337 * if (cg->cgroup_id == 123)338 * break;339 * }340 *341 * I.e., it looks almost like high-level for each loop in other languages,342 * supports continue/break, and is verifiable by BPF verifier.343 *344 * For iterating integers, the difference between bpf_for_each(num, i, N, M)345 * and bpf_for(i, N, M) is in that bpf_for() provides additional proof to346 * verifier that i is in [N, M) range, and in bpf_for_each() case i is `int347 * *`, not just `int`. So for integers bpf_for() is more convenient.348 *349 * Note: this macro relies on C99 feature of allowing to declare variables350 * inside for() loop, bound to for() loop lifetime. It also utilizes GCC351 * extension: __attribute__((cleanup(<func>))), supported by both GCC and352 * Clang.353 */354#define bpf_for_each(type, cur, args...) for ( \355 /* initialize and define destructor */ \356 struct bpf_iter_##type ___it __attribute__((aligned(8), /* enforce, just in case */, \357 cleanup(bpf_iter_##type##_destroy))), \358 /* ___p pointer is just to call bpf_iter_##type##_new() *once* to init ___it */ \359 *___p __attribute__((unused)) = ( \360 bpf_iter_##type##_new(&___it, ##args), \361 /* this is a workaround for Clang bug: it currently doesn't emit BTF */ \362 /* for bpf_iter_##type##_destroy() when used from cleanup() attribute */ \363 (void)bpf_iter_##type##_destroy, (void *)0); \364 /* iteration and termination check */ \365 (((cur) = bpf_iter_##type##_next(&___it))); \366)367#endif /* bpf_for_each */368 369#ifndef bpf_for370/* bpf_for(i, start, end) implements a for()-like looping construct that sets371 * provided integer variable *i* to values starting from *start* through,372 * but not including, *end*. It also proves to BPF verifier that *i* belongs373 * to range [start, end), so this can be used for accessing arrays without374 * extra checks.375 *376 * Note: *start* and *end* are assumed to be expressions with no side effects377 * and whose values do not change throughout bpf_for() loop execution. They do378 * not have to be statically known or constant, though.379 *380 * Note: similarly to bpf_for_each(), it relies on C99 feature of declaring for()381 * loop bound variables and cleanup attribute, supported by GCC and Clang.382 */383#define bpf_for(i, start, end) for ( \384 /* initialize and define destructor */ \385 struct bpf_iter_num ___it __attribute__((aligned(8), /* enforce, just in case */ \386 cleanup(bpf_iter_num_destroy))), \387 /* ___p pointer is necessary to call bpf_iter_num_new() *once* to init ___it */ \388 *___p __attribute__((unused)) = ( \389 bpf_iter_num_new(&___it, (start), (end)), \390 /* this is a workaround for Clang bug: it currently doesn't emit BTF */ \391 /* for bpf_iter_num_destroy() when used from cleanup() attribute */ \392 (void)bpf_iter_num_destroy, (void *)0); \393 ({ \394 /* iteration step */ \395 int *___t = bpf_iter_num_next(&___it); \396 /* termination and bounds check */ \397 (___t && ((i) = *___t, (i) >= (start) && (i) < (end))); \398 }); \399)400#endif /* bpf_for */401 402#ifndef bpf_repeat403/* bpf_repeat(N) performs N iterations without exposing iteration number404 *405 * Note: similarly to bpf_for_each(), it relies on C99 feature of declaring for()406 * loop bound variables and cleanup attribute, supported by GCC and Clang.407 */408#define bpf_repeat(N) for ( \409 /* initialize and define destructor */ \410 struct bpf_iter_num ___it __attribute__((aligned(8), /* enforce, just in case */ \411 cleanup(bpf_iter_num_destroy))), \412 /* ___p pointer is necessary to call bpf_iter_num_new() *once* to init ___it */ \413 *___p __attribute__((unused)) = ( \414 bpf_iter_num_new(&___it, 0, (N)), \415 /* this is a workaround for Clang bug: it currently doesn't emit BTF */ \416 /* for bpf_iter_num_destroy() when used from cleanup() attribute */ \417 (void)bpf_iter_num_destroy, (void *)0); \418 bpf_iter_num_next(&___it); \419 /* nothing here */ \420)421#endif /* bpf_repeat */422 423#endif424