251 lines · c
1/* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */2/* Copyright (c) 2022 Meta Platforms, Inc. and affiliates. */3#ifndef __USDT_BPF_H__4#define __USDT_BPF_H__5 6#include <linux/errno.h>7#include "bpf_helpers.h"8#include "bpf_tracing.h"9 10/* Below types and maps are internal implementation details of libbpf's USDT11 * support and are subjects to change. Also, bpf_usdt_xxx() API helpers should12 * be considered an unstable API as well and might be adjusted based on user13 * feedback from using libbpf's USDT support in production.14 */15 16/* User can override BPF_USDT_MAX_SPEC_CNT to change default size of internal17 * map that keeps track of USDT argument specifications. This might be18 * necessary if there are a lot of USDT attachments.19 */20#ifndef BPF_USDT_MAX_SPEC_CNT21#define BPF_USDT_MAX_SPEC_CNT 25622#endif23/* User can override BPF_USDT_MAX_IP_CNT to change default size of internal24 * map that keeps track of IP (memory address) mapping to USDT argument25 * specification.26 * Note, if kernel supports BPF cookies, this map is not used and could be27 * resized all the way to 1 to save a bit of memory.28 */29#ifndef BPF_USDT_MAX_IP_CNT30#define BPF_USDT_MAX_IP_CNT (4 * BPF_USDT_MAX_SPEC_CNT)31#endif32 33enum __bpf_usdt_arg_type {34 BPF_USDT_ARG_CONST,35 BPF_USDT_ARG_REG,36 BPF_USDT_ARG_REG_DEREF,37};38 39struct __bpf_usdt_arg_spec {40 /* u64 scalar interpreted depending on arg_type, see below */41 __u64 val_off;42 /* arg location case, see bpf_usdt_arg() for details */43 enum __bpf_usdt_arg_type arg_type;44 /* offset of referenced register within struct pt_regs */45 short reg_off;46 /* whether arg should be interpreted as signed value */47 bool arg_signed;48 /* number of bits that need to be cleared and, optionally,49 * sign-extended to cast arguments that are 1, 2, or 4 bytes50 * long into final 8-byte u64/s64 value returned to user51 */52 char arg_bitshift;53};54 55/* should match USDT_MAX_ARG_CNT in usdt.c exactly */56#define BPF_USDT_MAX_ARG_CNT 1257struct __bpf_usdt_spec {58 struct __bpf_usdt_arg_spec args[BPF_USDT_MAX_ARG_CNT];59 __u64 usdt_cookie;60 short arg_cnt;61};62 63struct {64 __uint(type, BPF_MAP_TYPE_ARRAY);65 __uint(max_entries, BPF_USDT_MAX_SPEC_CNT);66 __type(key, int);67 __type(value, struct __bpf_usdt_spec);68} __bpf_usdt_specs SEC(".maps") __weak;69 70struct {71 __uint(type, BPF_MAP_TYPE_HASH);72 __uint(max_entries, BPF_USDT_MAX_IP_CNT);73 __type(key, long);74 __type(value, __u32);75} __bpf_usdt_ip_to_spec_id SEC(".maps") __weak;76 77extern const _Bool LINUX_HAS_BPF_COOKIE __kconfig;78 79static __always_inline80int __bpf_usdt_spec_id(struct pt_regs *ctx)81{82 if (!LINUX_HAS_BPF_COOKIE) {83 long ip = PT_REGS_IP(ctx);84 int *spec_id_ptr;85 86 spec_id_ptr = bpf_map_lookup_elem(&__bpf_usdt_ip_to_spec_id, &ip);87 return spec_id_ptr ? *spec_id_ptr : -ESRCH;88 }89 90 return bpf_get_attach_cookie(ctx);91}92 93/* Return number of USDT arguments defined for currently traced USDT. */94__weak __hidden95int bpf_usdt_arg_cnt(struct pt_regs *ctx)96{97 struct __bpf_usdt_spec *spec;98 int spec_id;99 100 spec_id = __bpf_usdt_spec_id(ctx);101 if (spec_id < 0)102 return -ESRCH;103 104 spec = bpf_map_lookup_elem(&__bpf_usdt_specs, &spec_id);105 if (!spec)106 return -ESRCH;107 108 return spec->arg_cnt;109}110 111/* Fetch USDT argument #*arg_num* (zero-indexed) and put its value into *res.112 * Returns 0 on success; negative error, otherwise.113 * On error *res is guaranteed to be set to zero.114 */115__weak __hidden116int bpf_usdt_arg(struct pt_regs *ctx, __u64 arg_num, long *res)117{118 struct __bpf_usdt_spec *spec;119 struct __bpf_usdt_arg_spec *arg_spec;120 unsigned long val;121 int err, spec_id;122 123 *res = 0;124 125 spec_id = __bpf_usdt_spec_id(ctx);126 if (spec_id < 0)127 return -ESRCH;128 129 spec = bpf_map_lookup_elem(&__bpf_usdt_specs, &spec_id);130 if (!spec)131 return -ESRCH;132 133 if (arg_num >= BPF_USDT_MAX_ARG_CNT)134 return -ENOENT;135 barrier_var(arg_num);136 if (arg_num >= spec->arg_cnt)137 return -ENOENT;138 139 arg_spec = &spec->args[arg_num];140 switch (arg_spec->arg_type) {141 case BPF_USDT_ARG_CONST:142 /* Arg is just a constant ("-4@$-9" in USDT arg spec).143 * value is recorded in arg_spec->val_off directly.144 */145 val = arg_spec->val_off;146 break;147 case BPF_USDT_ARG_REG:148 /* Arg is in a register (e.g, "8@%rax" in USDT arg spec),149 * so we read the contents of that register directly from150 * struct pt_regs. To keep things simple user-space parts151 * record offsetof(struct pt_regs, <regname>) in arg_spec->reg_off.152 */153 err = bpf_probe_read_kernel(&val, sizeof(val), (void *)ctx + arg_spec->reg_off);154 if (err)155 return err;156 break;157 case BPF_USDT_ARG_REG_DEREF:158 /* Arg is in memory addressed by register, plus some offset159 * (e.g., "-4@-1204(%rbp)" in USDT arg spec). Register is160 * identified like with BPF_USDT_ARG_REG case, and the offset161 * is in arg_spec->val_off. We first fetch register contents162 * from pt_regs, then do another user-space probe read to163 * fetch argument value itself.164 */165 err = bpf_probe_read_kernel(&val, sizeof(val), (void *)ctx + arg_spec->reg_off);166 if (err)167 return err;168 err = bpf_probe_read_user(&val, sizeof(val), (void *)val + arg_spec->val_off);169 if (err)170 return err;171#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__172 val >>= arg_spec->arg_bitshift;173#endif174 break;175 default:176 return -EINVAL;177 }178 179 /* cast arg from 1, 2, or 4 bytes to final 8 byte size clearing180 * necessary upper arg_bitshift bits, with sign extension if argument181 * is signed182 */183 val <<= arg_spec->arg_bitshift;184 if (arg_spec->arg_signed)185 val = ((long)val) >> arg_spec->arg_bitshift;186 else187 val = val >> arg_spec->arg_bitshift;188 *res = val;189 return 0;190}191 192/* Retrieve user-specified cookie value provided during attach as193 * bpf_usdt_opts.usdt_cookie. This serves the same purpose as BPF cookie194 * returned by bpf_get_attach_cookie(). Libbpf's support for USDT is itself195 * utilizing BPF cookies internally, so user can't use BPF cookie directly196 * for USDT programs and has to use bpf_usdt_cookie() API instead.197 */198__weak __hidden199long bpf_usdt_cookie(struct pt_regs *ctx)200{201 struct __bpf_usdt_spec *spec;202 int spec_id;203 204 spec_id = __bpf_usdt_spec_id(ctx);205 if (spec_id < 0)206 return 0;207 208 spec = bpf_map_lookup_elem(&__bpf_usdt_specs, &spec_id);209 if (!spec)210 return 0;211 212 return spec->usdt_cookie;213}214 215/* we rely on ___bpf_apply() and ___bpf_narg() macros already defined in bpf_tracing.h */216#define ___bpf_usdt_args0() ctx217#define ___bpf_usdt_args1(x) ___bpf_usdt_args0(), ({ long _x; bpf_usdt_arg(ctx, 0, &_x); _x; })218#define ___bpf_usdt_args2(x, args...) ___bpf_usdt_args1(args), ({ long _x; bpf_usdt_arg(ctx, 1, &_x); _x; })219#define ___bpf_usdt_args3(x, args...) ___bpf_usdt_args2(args), ({ long _x; bpf_usdt_arg(ctx, 2, &_x); _x; })220#define ___bpf_usdt_args4(x, args...) ___bpf_usdt_args3(args), ({ long _x; bpf_usdt_arg(ctx, 3, &_x); _x; })221#define ___bpf_usdt_args5(x, args...) ___bpf_usdt_args4(args), ({ long _x; bpf_usdt_arg(ctx, 4, &_x); _x; })222#define ___bpf_usdt_args6(x, args...) ___bpf_usdt_args5(args), ({ long _x; bpf_usdt_arg(ctx, 5, &_x); _x; })223#define ___bpf_usdt_args7(x, args...) ___bpf_usdt_args6(args), ({ long _x; bpf_usdt_arg(ctx, 6, &_x); _x; })224#define ___bpf_usdt_args8(x, args...) ___bpf_usdt_args7(args), ({ long _x; bpf_usdt_arg(ctx, 7, &_x); _x; })225#define ___bpf_usdt_args9(x, args...) ___bpf_usdt_args8(args), ({ long _x; bpf_usdt_arg(ctx, 8, &_x); _x; })226#define ___bpf_usdt_args10(x, args...) ___bpf_usdt_args9(args), ({ long _x; bpf_usdt_arg(ctx, 9, &_x); _x; })227#define ___bpf_usdt_args11(x, args...) ___bpf_usdt_args10(args), ({ long _x; bpf_usdt_arg(ctx, 10, &_x); _x; })228#define ___bpf_usdt_args12(x, args...) ___bpf_usdt_args11(args), ({ long _x; bpf_usdt_arg(ctx, 11, &_x); _x; })229#define ___bpf_usdt_args(args...) ___bpf_apply(___bpf_usdt_args, ___bpf_narg(args))(args)230 231/*232 * BPF_USDT serves the same purpose for USDT handlers as BPF_PROG for233 * tp_btf/fentry/fexit BPF programs and BPF_KPROBE for kprobes.234 * Original struct pt_regs * context is preserved as 'ctx' argument.235 */236#define BPF_USDT(name, args...) \237name(struct pt_regs *ctx); \238static __always_inline typeof(name(0)) \239____##name(struct pt_regs *ctx, ##args); \240typeof(name(0)) name(struct pt_regs *ctx) \241{ \242 _Pragma("GCC diagnostic push") \243 _Pragma("GCC diagnostic ignored \"-Wint-conversion\"") \244 return ____##name(___bpf_usdt_args(args)); \245 _Pragma("GCC diagnostic pop") \246} \247static __always_inline typeof(name(0)) \248____##name(struct pt_regs *ctx, ##args)249 250#endif /* __USDT_BPF_H__ */251