1601 lines · c
1// SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause)2/* Copyright (c) 2022 Meta Platforms, Inc. and affiliates. */3#include <ctype.h>4#include <stdio.h>5#include <stdlib.h>6#include <string.h>7#include <libelf.h>8#include <gelf.h>9#include <unistd.h>10#include <linux/ptrace.h>11#include <linux/kernel.h>12 13/* s8 will be marked as poison while it's a reg of riscv */14#if defined(__riscv)15#define rv_s8 s816#endif17 18#include "bpf.h"19#include "libbpf.h"20#include "libbpf_common.h"21#include "libbpf_internal.h"22#include "hashmap.h"23 24/* libbpf's USDT support consists of BPF-side state/code and user-space25 * state/code working together in concert. BPF-side parts are defined in26 * usdt.bpf.h header library. User-space state is encapsulated by struct27 * usdt_manager and all the supporting code centered around usdt_manager.28 *29 * usdt.bpf.h defines two BPF maps that usdt_manager expects: USDT spec map30 * and IP-to-spec-ID map, which is auxiliary map necessary for kernels that31 * don't support BPF cookie (see below). These two maps are implicitly32 * embedded into user's end BPF object file when user's code included33 * usdt.bpf.h. This means that libbpf doesn't do anything special to create34 * these USDT support maps. They are created by normal libbpf logic of35 * instantiating BPF maps when opening and loading BPF object.36 *37 * As such, libbpf is basically unaware of the need to do anything38 * USDT-related until the very first call to bpf_program__attach_usdt(), which39 * can be called by user explicitly or happen automatically during skeleton40 * attach (or, equivalently, through generic bpf_program__attach() call). At41 * this point, libbpf will instantiate and initialize struct usdt_manager and42 * store it in bpf_object. USDT manager is per-BPF object construct, as each43 * independent BPF object might or might not have USDT programs, and thus all44 * the expected USDT-related state. There is no coordination between two45 * bpf_object in parts of USDT attachment, they are oblivious of each other's46 * existence and libbpf is just oblivious, dealing with bpf_object-specific47 * USDT state.48 *49 * Quick crash course on USDTs.50 *51 * From user-space application's point of view, USDT is essentially just52 * a slightly special function call that normally has zero overhead, unless it53 * is being traced by some external entity (e.g, BPF-based tool). Here's how54 * a typical application can trigger USDT probe:55 *56 * #include <sys/sdt.h> // provided by systemtap-sdt-devel package57 * // folly also provide similar functionality in folly/tracing/StaticTracepoint.h58 *59 * STAP_PROBE3(my_usdt_provider, my_usdt_probe_name, 123, x, &y);60 *61 * USDT is identified by it's <provider-name>:<probe-name> pair of names. Each62 * individual USDT has a fixed number of arguments (3 in the above example)63 * and specifies values of each argument as if it was a function call.64 *65 * USDT call is actually not a function call, but is instead replaced by66 * a single NOP instruction (thus zero overhead, effectively). But in addition67 * to that, those USDT macros generate special SHT_NOTE ELF records in68 * .note.stapsdt ELF section. Here's an example USDT definition as emitted by69 * `readelf -n <binary>`:70 *71 * stapsdt 0x00000089 NT_STAPSDT (SystemTap probe descriptors)72 * Provider: test73 * Name: usdt1274 * Location: 0x0000000000549df3, Base: 0x00000000008effa4, Semaphore: 0x0000000000a4606e75 * Arguments: -4@-1204(%rbp) -4@%edi -8@-1216(%rbp) -8@%r8 -4@$5 -8@%r9 8@%rdx 8@%r10 -4@$-9 -2@%cx -2@%ax -1@%sil76 *77 * In this case we have USDT test:usdt12 with 12 arguments.78 *79 * Location and base are offsets used to calculate absolute IP address of that80 * NOP instruction that kernel can replace with an interrupt instruction to81 * trigger instrumentation code (BPF program for all that we care about).82 *83 * Semaphore above is and optional feature. It records an address of a 2-byte84 * refcount variable (normally in '.probes' ELF section) used for signaling if85 * there is anything that is attached to USDT. This is useful for user86 * applications if, for example, they need to prepare some arguments that are87 * passed only to USDTs and preparation is expensive. By checking if USDT is88 * "activated", an application can avoid paying those costs unnecessarily.89 * Recent enough kernel has built-in support for automatically managing this90 * refcount, which libbpf expects and relies on. If USDT is defined without91 * associated semaphore, this value will be zero. See selftests for semaphore92 * examples.93 *94 * Arguments is the most interesting part. This USDT specification string is95 * providing information about all the USDT arguments and their locations. The96 * part before @ sign defined byte size of the argument (1, 2, 4, or 8) and97 * whether the argument is signed or unsigned (negative size means signed).98 * The part after @ sign is assembly-like definition of argument location99 * (see [0] for more details). Technically, assembler can provide some pretty100 * advanced definitions, but libbpf is currently supporting three most common101 * cases:102 * 1) immediate constant, see 5th and 9th args above (-4@$5 and -4@-9);103 * 2) register value, e.g., 8@%rdx, which means "unsigned 8-byte integer104 * whose value is in register %rdx";105 * 3) memory dereference addressed by register, e.g., -4@-1204(%rbp), which106 * specifies signed 32-bit integer stored at offset -1204 bytes from107 * memory address stored in %rbp.108 *109 * [0] https://sourceware.org/systemtap/wiki/UserSpaceProbeImplementation110 *111 * During attachment, libbpf parses all the relevant USDT specifications and112 * prepares `struct usdt_spec` (USDT spec), which is then provided to BPF-side113 * code through spec map. This allows BPF applications to quickly fetch the114 * actual value at runtime using a simple BPF-side code.115 *116 * With basics out of the way, let's go over less immediately obvious aspects117 * of supporting USDTs.118 *119 * First, there is no special USDT BPF program type. It is actually just120 * a uprobe BPF program (which for kernel, at least currently, is just a kprobe121 * program, so BPF_PROG_TYPE_KPROBE program type). With the only difference122 * that uprobe is usually attached at the function entry, while USDT will123 * normally will be somewhere inside the function. But it should always be124 * pointing to NOP instruction, which makes such uprobes the fastest uprobe125 * kind.126 *127 * Second, it's important to realize that such STAP_PROBEn(provider, name, ...)128 * macro invocations can end up being inlined many-many times, depending on129 * specifics of each individual user application. So single conceptual USDT130 * (identified by provider:name pair of identifiers) is, generally speaking,131 * multiple uprobe locations (USDT call sites) in different places in user132 * application. Further, again due to inlining, each USDT call site might end133 * up having the same argument #N be located in a different place. In one call134 * site it could be a constant, in another will end up in a register, and in135 * yet another could be some other register or even somewhere on the stack.136 *137 * As such, "attaching to USDT" means (in general case) attaching the same138 * uprobe BPF program to multiple target locations in user application, each139 * potentially having a completely different USDT spec associated with it.140 * To wire all this up together libbpf allocates a unique integer spec ID for141 * each unique USDT spec. Spec IDs are allocated as sequential small integers142 * so that they can be used as keys in array BPF map (for performance reasons).143 * Spec ID allocation and accounting is big part of what usdt_manager is144 * about. This state has to be maintained per-BPF object and coordinate145 * between different USDT attachments within the same BPF object.146 *147 * Spec ID is the key in spec BPF map, value is the actual USDT spec layed out148 * as struct usdt_spec. Each invocation of BPF program at runtime needs to149 * know its associated spec ID. It gets it either through BPF cookie, which150 * libbpf sets to spec ID during attach time, or, if kernel is too old to151 * support BPF cookie, through IP-to-spec-ID map that libbpf maintains in such152 * case. The latter means that some modes of operation can't be supported153 * without BPF cookie. Such mode is attaching to shared library "generically",154 * without specifying target process. In such case, it's impossible to155 * calculate absolute IP addresses for IP-to-spec-ID map, and thus such mode156 * is not supported without BPF cookie support.157 *158 * Note that libbpf is using BPF cookie functionality for its own internal159 * needs, so user itself can't rely on BPF cookie feature. To that end, libbpf160 * provides conceptually equivalent USDT cookie support. It's still u64161 * user-provided value that can be associated with USDT attachment. Note that162 * this will be the same value for all USDT call sites within the same single163 * *logical* USDT attachment. This makes sense because to user attaching to164 * USDT is a single BPF program triggered for singular USDT probe. The fact165 * that this is done at multiple actual locations is a mostly hidden166 * implementation details. This USDT cookie value can be fetched with167 * bpf_usdt_cookie(ctx) API provided by usdt.bpf.h168 *169 * Lastly, while single USDT can have tons of USDT call sites, it doesn't170 * necessarily have that many different USDT specs. It very well might be171 * that 1000 USDT call sites only need 5 different USDT specs, because all the172 * arguments are typically contained in a small set of registers or stack173 * locations. As such, it's wasteful to allocate as many USDT spec IDs as174 * there are USDT call sites. So libbpf tries to be frugal and performs175 * on-the-fly deduplication during a single USDT attachment to only allocate176 * the minimal required amount of unique USDT specs (and thus spec IDs). This177 * is trivially achieved by using USDT spec string (Arguments string from USDT178 * note) as a lookup key in a hashmap. USDT spec string uniquely defines179 * everything about how to fetch USDT arguments, so two USDT call sites180 * sharing USDT spec string can safely share the same USDT spec and spec ID.181 * Note, this spec string deduplication is happening only during the same USDT182 * attachment, so each USDT spec shares the same USDT cookie value. This is183 * not generally true for other USDT attachments within the same BPF object,184 * as even if USDT spec string is the same, USDT cookie value can be185 * different. It was deemed excessive to try to deduplicate across independent186 * USDT attachments by taking into account USDT spec string *and* USDT cookie187 * value, which would complicated spec ID accounting significantly for little188 * gain.189 */190 191#define USDT_BASE_SEC ".stapsdt.base"192#define USDT_SEMA_SEC ".probes"193#define USDT_NOTE_SEC ".note.stapsdt"194#define USDT_NOTE_TYPE 3195#define USDT_NOTE_NAME "stapsdt"196 197/* should match exactly enum __bpf_usdt_arg_type from usdt.bpf.h */198enum usdt_arg_type {199 USDT_ARG_CONST,200 USDT_ARG_REG,201 USDT_ARG_REG_DEREF,202};203 204/* should match exactly struct __bpf_usdt_arg_spec from usdt.bpf.h */205struct usdt_arg_spec {206 __u64 val_off;207 enum usdt_arg_type arg_type;208 short reg_off;209 bool arg_signed;210 char arg_bitshift;211};212 213/* should match BPF_USDT_MAX_ARG_CNT in usdt.bpf.h */214#define USDT_MAX_ARG_CNT 12215 216/* should match struct __bpf_usdt_spec from usdt.bpf.h */217struct usdt_spec {218 struct usdt_arg_spec args[USDT_MAX_ARG_CNT];219 __u64 usdt_cookie;220 short arg_cnt;221};222 223struct usdt_note {224 const char *provider;225 const char *name;226 /* USDT args specification string, e.g.:227 * "-4@%esi -4@-24(%rbp) -4@%ecx 2@%ax 8@%rdx"228 */229 const char *args;230 long loc_addr;231 long base_addr;232 long sema_addr;233};234 235struct usdt_target {236 long abs_ip;237 long rel_ip;238 long sema_off;239 struct usdt_spec spec;240 const char *spec_str;241};242 243struct usdt_manager {244 struct bpf_map *specs_map;245 struct bpf_map *ip_to_spec_id_map;246 247 int *free_spec_ids;248 size_t free_spec_cnt;249 size_t next_free_spec_id;250 251 bool has_bpf_cookie;252 bool has_sema_refcnt;253 bool has_uprobe_multi;254};255 256struct usdt_manager *usdt_manager_new(struct bpf_object *obj)257{258 static const char *ref_ctr_sysfs_path = "/sys/bus/event_source/devices/uprobe/format/ref_ctr_offset";259 struct usdt_manager *man;260 struct bpf_map *specs_map, *ip_to_spec_id_map;261 262 specs_map = bpf_object__find_map_by_name(obj, "__bpf_usdt_specs");263 ip_to_spec_id_map = bpf_object__find_map_by_name(obj, "__bpf_usdt_ip_to_spec_id");264 if (!specs_map || !ip_to_spec_id_map) {265 pr_warn("usdt: failed to find USDT support BPF maps, did you forget to include bpf/usdt.bpf.h?\n");266 return ERR_PTR(-ESRCH);267 }268 269 man = calloc(1, sizeof(*man));270 if (!man)271 return ERR_PTR(-ENOMEM);272 273 man->specs_map = specs_map;274 man->ip_to_spec_id_map = ip_to_spec_id_map;275 276 /* Detect if BPF cookie is supported for kprobes.277 * We don't need IP-to-ID mapping if we can use BPF cookies.278 * Added in: 7adfc6c9b315 ("bpf: Add bpf_get_attach_cookie() BPF helper to access bpf_cookie value")279 */280 man->has_bpf_cookie = kernel_supports(obj, FEAT_BPF_COOKIE);281 282 /* Detect kernel support for automatic refcounting of USDT semaphore.283 * If this is not supported, USDTs with semaphores will not be supported.284 * Added in: a6ca88b241d5 ("trace_uprobe: support reference counter in fd-based uprobe")285 */286 man->has_sema_refcnt = faccessat(AT_FDCWD, ref_ctr_sysfs_path, F_OK, AT_EACCESS) == 0;287 288 /*289 * Detect kernel support for uprobe multi link to be used for attaching290 * usdt probes.291 */292 man->has_uprobe_multi = kernel_supports(obj, FEAT_UPROBE_MULTI_LINK);293 return man;294}295 296void usdt_manager_free(struct usdt_manager *man)297{298 if (IS_ERR_OR_NULL(man))299 return;300 301 free(man->free_spec_ids);302 free(man);303}304 305static int sanity_check_usdt_elf(Elf *elf, const char *path)306{307 GElf_Ehdr ehdr;308 int endianness;309 310 if (elf_kind(elf) != ELF_K_ELF) {311 pr_warn("usdt: unrecognized ELF kind %d for '%s'\n", elf_kind(elf), path);312 return -EBADF;313 }314 315 switch (gelf_getclass(elf)) {316 case ELFCLASS64:317 if (sizeof(void *) != 8) {318 pr_warn("usdt: attaching to 64-bit ELF binary '%s' is not supported\n", path);319 return -EBADF;320 }321 break;322 case ELFCLASS32:323 if (sizeof(void *) != 4) {324 pr_warn("usdt: attaching to 32-bit ELF binary '%s' is not supported\n", path);325 return -EBADF;326 }327 break;328 default:329 pr_warn("usdt: unsupported ELF class for '%s'\n", path);330 return -EBADF;331 }332 333 if (!gelf_getehdr(elf, &ehdr))334 return -EINVAL;335 336 if (ehdr.e_type != ET_EXEC && ehdr.e_type != ET_DYN) {337 pr_warn("usdt: unsupported type of ELF binary '%s' (%d), only ET_EXEC and ET_DYN are supported\n",338 path, ehdr.e_type);339 return -EBADF;340 }341 342#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__343 endianness = ELFDATA2LSB;344#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__345 endianness = ELFDATA2MSB;346#else347# error "Unrecognized __BYTE_ORDER__"348#endif349 if (endianness != ehdr.e_ident[EI_DATA]) {350 pr_warn("usdt: ELF endianness mismatch for '%s'\n", path);351 return -EBADF;352 }353 354 return 0;355}356 357static int find_elf_sec_by_name(Elf *elf, const char *sec_name, GElf_Shdr *shdr, Elf_Scn **scn)358{359 Elf_Scn *sec = NULL;360 size_t shstrndx;361 362 if (elf_getshdrstrndx(elf, &shstrndx))363 return -EINVAL;364 365 /* check if ELF is corrupted and avoid calling elf_strptr if yes */366 if (!elf_rawdata(elf_getscn(elf, shstrndx), NULL))367 return -EINVAL;368 369 while ((sec = elf_nextscn(elf, sec)) != NULL) {370 char *name;371 372 if (!gelf_getshdr(sec, shdr))373 return -EINVAL;374 375 name = elf_strptr(elf, shstrndx, shdr->sh_name);376 if (name && strcmp(sec_name, name) == 0) {377 *scn = sec;378 return 0;379 }380 }381 382 return -ENOENT;383}384 385struct elf_seg {386 long start;387 long end;388 long offset;389 bool is_exec;390};391 392static int cmp_elf_segs(const void *_a, const void *_b)393{394 const struct elf_seg *a = _a;395 const struct elf_seg *b = _b;396 397 return a->start < b->start ? -1 : 1;398}399 400static int parse_elf_segs(Elf *elf, const char *path, struct elf_seg **segs, size_t *seg_cnt)401{402 GElf_Phdr phdr;403 size_t n;404 int i, err;405 struct elf_seg *seg;406 void *tmp;407 408 *seg_cnt = 0;409 410 if (elf_getphdrnum(elf, &n)) {411 err = -errno;412 return err;413 }414 415 for (i = 0; i < n; i++) {416 if (!gelf_getphdr(elf, i, &phdr)) {417 err = -errno;418 return err;419 }420 421 pr_debug("usdt: discovered PHDR #%d in '%s': vaddr 0x%lx memsz 0x%lx offset 0x%lx type 0x%lx flags 0x%lx\n",422 i, path, (long)phdr.p_vaddr, (long)phdr.p_memsz, (long)phdr.p_offset,423 (long)phdr.p_type, (long)phdr.p_flags);424 if (phdr.p_type != PT_LOAD)425 continue;426 427 tmp = libbpf_reallocarray(*segs, *seg_cnt + 1, sizeof(**segs));428 if (!tmp)429 return -ENOMEM;430 431 *segs = tmp;432 seg = *segs + *seg_cnt;433 (*seg_cnt)++;434 435 seg->start = phdr.p_vaddr;436 seg->end = phdr.p_vaddr + phdr.p_memsz;437 seg->offset = phdr.p_offset;438 seg->is_exec = phdr.p_flags & PF_X;439 }440 441 if (*seg_cnt == 0) {442 pr_warn("usdt: failed to find PT_LOAD program headers in '%s'\n", path);443 return -ESRCH;444 }445 446 qsort(*segs, *seg_cnt, sizeof(**segs), cmp_elf_segs);447 return 0;448}449 450static int parse_vma_segs(int pid, const char *lib_path, struct elf_seg **segs, size_t *seg_cnt)451{452 char path[PATH_MAX], line[PATH_MAX], mode[16];453 size_t seg_start, seg_end, seg_off;454 struct elf_seg *seg;455 int tmp_pid, i, err;456 FILE *f;457 458 *seg_cnt = 0;459 460 /* Handle containerized binaries only accessible from461 * /proc/<pid>/root/<path>. They will be reported as just /<path> in462 * /proc/<pid>/maps.463 */464 if (sscanf(lib_path, "/proc/%d/root%s", &tmp_pid, path) == 2 && pid == tmp_pid)465 goto proceed;466 467 if (!realpath(lib_path, path)) {468 pr_warn("usdt: failed to get absolute path of '%s' (err %d), using path as is...\n",469 lib_path, -errno);470 libbpf_strlcpy(path, lib_path, sizeof(path));471 }472 473proceed:474 sprintf(line, "/proc/%d/maps", pid);475 f = fopen(line, "re");476 if (!f) {477 err = -errno;478 pr_warn("usdt: failed to open '%s' to get base addr of '%s': %d\n",479 line, lib_path, err);480 return err;481 }482 483 /* We need to handle lines with no path at the end:484 *485 * 7f5c6f5d1000-7f5c6f5d3000 rw-p 001c7000 08:04 21238613 /usr/lib64/libc-2.17.so486 * 7f5c6f5d3000-7f5c6f5d8000 rw-p 00000000 00:00 0487 * 7f5c6f5d8000-7f5c6f5d9000 r-xp 00000000 103:01 362990598 /data/users/andriin/linux/tools/bpf/usdt/libhello_usdt.so488 */489 while (fscanf(f, "%zx-%zx %s %zx %*s %*d%[^\n]\n",490 &seg_start, &seg_end, mode, &seg_off, line) == 5) {491 void *tmp;492 493 /* to handle no path case (see above) we need to capture line494 * without skipping any whitespaces. So we need to strip495 * leading whitespaces manually here496 */497 i = 0;498 while (isblank(line[i]))499 i++;500 if (strcmp(line + i, path) != 0)501 continue;502 503 pr_debug("usdt: discovered segment for lib '%s': addrs %zx-%zx mode %s offset %zx\n",504 path, seg_start, seg_end, mode, seg_off);505 506 /* ignore non-executable sections for shared libs */507 if (mode[2] != 'x')508 continue;509 510 tmp = libbpf_reallocarray(*segs, *seg_cnt + 1, sizeof(**segs));511 if (!tmp) {512 err = -ENOMEM;513 goto err_out;514 }515 516 *segs = tmp;517 seg = *segs + *seg_cnt;518 *seg_cnt += 1;519 520 seg->start = seg_start;521 seg->end = seg_end;522 seg->offset = seg_off;523 seg->is_exec = true;524 }525 526 if (*seg_cnt == 0) {527 pr_warn("usdt: failed to find '%s' (resolved to '%s') within PID %d memory mappings\n",528 lib_path, path, pid);529 err = -ESRCH;530 goto err_out;531 }532 533 qsort(*segs, *seg_cnt, sizeof(**segs), cmp_elf_segs);534 err = 0;535err_out:536 fclose(f);537 return err;538}539 540static struct elf_seg *find_elf_seg(struct elf_seg *segs, size_t seg_cnt, long virtaddr)541{542 struct elf_seg *seg;543 int i;544 545 /* for ELF binaries (both executables and shared libraries), we are546 * given virtual address (absolute for executables, relative for547 * libraries) which should match address range of [seg_start, seg_end)548 */549 for (i = 0, seg = segs; i < seg_cnt; i++, seg++) {550 if (seg->start <= virtaddr && virtaddr < seg->end)551 return seg;552 }553 return NULL;554}555 556static struct elf_seg *find_vma_seg(struct elf_seg *segs, size_t seg_cnt, long offset)557{558 struct elf_seg *seg;559 int i;560 561 /* for VMA segments from /proc/<pid>/maps file, provided "address" is562 * actually a file offset, so should be fall within logical563 * offset-based range of [offset_start, offset_end)564 */565 for (i = 0, seg = segs; i < seg_cnt; i++, seg++) {566 if (seg->offset <= offset && offset < seg->offset + (seg->end - seg->start))567 return seg;568 }569 return NULL;570}571 572static int parse_usdt_note(Elf *elf, const char *path, GElf_Nhdr *nhdr,573 const char *data, size_t name_off, size_t desc_off,574 struct usdt_note *usdt_note);575 576static int parse_usdt_spec(struct usdt_spec *spec, const struct usdt_note *note, __u64 usdt_cookie);577 578static int collect_usdt_targets(struct usdt_manager *man, Elf *elf, const char *path, pid_t pid,579 const char *usdt_provider, const char *usdt_name, __u64 usdt_cookie,580 struct usdt_target **out_targets, size_t *out_target_cnt)581{582 size_t off, name_off, desc_off, seg_cnt = 0, vma_seg_cnt = 0, target_cnt = 0;583 struct elf_seg *segs = NULL, *vma_segs = NULL;584 struct usdt_target *targets = NULL, *target;585 long base_addr = 0;586 Elf_Scn *notes_scn, *base_scn;587 GElf_Shdr base_shdr, notes_shdr;588 GElf_Ehdr ehdr;589 GElf_Nhdr nhdr;590 Elf_Data *data;591 int err;592 593 *out_targets = NULL;594 *out_target_cnt = 0;595 596 err = find_elf_sec_by_name(elf, USDT_NOTE_SEC, ¬es_shdr, ¬es_scn);597 if (err) {598 pr_warn("usdt: no USDT notes section (%s) found in '%s'\n", USDT_NOTE_SEC, path);599 return err;600 }601 602 if (notes_shdr.sh_type != SHT_NOTE || !gelf_getehdr(elf, &ehdr)) {603 pr_warn("usdt: invalid USDT notes section (%s) in '%s'\n", USDT_NOTE_SEC, path);604 return -EINVAL;605 }606 607 err = parse_elf_segs(elf, path, &segs, &seg_cnt);608 if (err) {609 pr_warn("usdt: failed to process ELF program segments for '%s': %d\n", path, err);610 goto err_out;611 }612 613 /* .stapsdt.base ELF section is optional, but is used for prelink614 * offset compensation (see a big comment further below)615 */616 if (find_elf_sec_by_name(elf, USDT_BASE_SEC, &base_shdr, &base_scn) == 0)617 base_addr = base_shdr.sh_addr;618 619 data = elf_getdata(notes_scn, 0);620 off = 0;621 while ((off = gelf_getnote(data, off, &nhdr, &name_off, &desc_off)) > 0) {622 long usdt_abs_ip, usdt_rel_ip, usdt_sema_off = 0;623 struct usdt_note note;624 struct elf_seg *seg = NULL;625 void *tmp;626 627 err = parse_usdt_note(elf, path, &nhdr, data->d_buf, name_off, desc_off, ¬e);628 if (err)629 goto err_out;630 631 if (strcmp(note.provider, usdt_provider) != 0 || strcmp(note.name, usdt_name) != 0)632 continue;633 634 /* We need to compensate "prelink effect". See [0] for details,635 * relevant parts quoted here:636 *637 * Each SDT probe also expands into a non-allocated ELF note. You can638 * find this by looking at SHT_NOTE sections and decoding the format;639 * see below for details. Because the note is non-allocated, it means640 * there is no runtime cost, and also preserved in both stripped files641 * and .debug files.642 *643 * However, this means that prelink won't adjust the note's contents644 * for address offsets. Instead, this is done via the .stapsdt.base645 * section. This is a special section that is added to the text. We646 * will only ever have one of these sections in a final link and it647 * will only ever be one byte long. Nothing about this section itself648 * matters, we just use it as a marker to detect prelink address649 * adjustments.650 *651 * Each probe note records the link-time address of the .stapsdt.base652 * section alongside the probe PC address. The decoder compares the653 * base address stored in the note with the .stapsdt.base section's654 * sh_addr. Initially these are the same, but the section header will655 * be adjusted by prelink. So the decoder applies the difference to656 * the probe PC address to get the correct prelinked PC address; the657 * same adjustment is applied to the semaphore address, if any.658 *659 * [0] https://sourceware.org/systemtap/wiki/UserSpaceProbeImplementation660 */661 usdt_abs_ip = note.loc_addr;662 if (base_addr)663 usdt_abs_ip += base_addr - note.base_addr;664 665 /* When attaching uprobes (which is what USDTs basically are)666 * kernel expects file offset to be specified, not a relative667 * virtual address, so we need to translate virtual address to668 * file offset, for both ET_EXEC and ET_DYN binaries.669 */670 seg = find_elf_seg(segs, seg_cnt, usdt_abs_ip);671 if (!seg) {672 err = -ESRCH;673 pr_warn("usdt: failed to find ELF program segment for '%s:%s' in '%s' at IP 0x%lx\n",674 usdt_provider, usdt_name, path, usdt_abs_ip);675 goto err_out;676 }677 if (!seg->is_exec) {678 err = -ESRCH;679 pr_warn("usdt: matched ELF binary '%s' segment [0x%lx, 0x%lx) for '%s:%s' at IP 0x%lx is not executable\n",680 path, seg->start, seg->end, usdt_provider, usdt_name,681 usdt_abs_ip);682 goto err_out;683 }684 /* translate from virtual address to file offset */685 usdt_rel_ip = usdt_abs_ip - seg->start + seg->offset;686 687 if (ehdr.e_type == ET_DYN && !man->has_bpf_cookie) {688 /* If we don't have BPF cookie support but need to689 * attach to a shared library, we'll need to know and690 * record absolute addresses of attach points due to691 * the need to lookup USDT spec by absolute IP of692 * triggered uprobe. Doing this resolution is only693 * possible when we have a specific PID of the process694 * that's using specified shared library. BPF cookie695 * removes the absolute address limitation as we don't696 * need to do this lookup (we just use BPF cookie as697 * an index of USDT spec), so for newer kernels with698 * BPF cookie support libbpf supports USDT attachment699 * to shared libraries with no PID filter.700 */701 if (pid < 0) {702 pr_warn("usdt: attaching to shared libraries without specific PID is not supported on current kernel\n");703 err = -ENOTSUP;704 goto err_out;705 }706 707 /* vma_segs are lazily initialized only if necessary */708 if (vma_seg_cnt == 0) {709 err = parse_vma_segs(pid, path, &vma_segs, &vma_seg_cnt);710 if (err) {711 pr_warn("usdt: failed to get memory segments in PID %d for shared library '%s': %d\n",712 pid, path, err);713 goto err_out;714 }715 }716 717 seg = find_vma_seg(vma_segs, vma_seg_cnt, usdt_rel_ip);718 if (!seg) {719 err = -ESRCH;720 pr_warn("usdt: failed to find shared lib memory segment for '%s:%s' in '%s' at relative IP 0x%lx\n",721 usdt_provider, usdt_name, path, usdt_rel_ip);722 goto err_out;723 }724 725 usdt_abs_ip = seg->start - seg->offset + usdt_rel_ip;726 }727 728 pr_debug("usdt: probe for '%s:%s' in %s '%s': addr 0x%lx base 0x%lx (resolved abs_ip 0x%lx rel_ip 0x%lx) args '%s' in segment [0x%lx, 0x%lx) at offset 0x%lx\n",729 usdt_provider, usdt_name, ehdr.e_type == ET_EXEC ? "exec" : "lib ", path,730 note.loc_addr, note.base_addr, usdt_abs_ip, usdt_rel_ip, note.args,731 seg ? seg->start : 0, seg ? seg->end : 0, seg ? seg->offset : 0);732 733 /* Adjust semaphore address to be a file offset */734 if (note.sema_addr) {735 if (!man->has_sema_refcnt) {736 pr_warn("usdt: kernel doesn't support USDT semaphore refcounting for '%s:%s' in '%s'\n",737 usdt_provider, usdt_name, path);738 err = -ENOTSUP;739 goto err_out;740 }741 742 seg = find_elf_seg(segs, seg_cnt, note.sema_addr);743 if (!seg) {744 err = -ESRCH;745 pr_warn("usdt: failed to find ELF loadable segment with semaphore of '%s:%s' in '%s' at 0x%lx\n",746 usdt_provider, usdt_name, path, note.sema_addr);747 goto err_out;748 }749 if (seg->is_exec) {750 err = -ESRCH;751 pr_warn("usdt: matched ELF binary '%s' segment [0x%lx, 0x%lx] for semaphore of '%s:%s' at 0x%lx is executable\n",752 path, seg->start, seg->end, usdt_provider, usdt_name,753 note.sema_addr);754 goto err_out;755 }756 757 usdt_sema_off = note.sema_addr - seg->start + seg->offset;758 759 pr_debug("usdt: sema for '%s:%s' in %s '%s': addr 0x%lx base 0x%lx (resolved 0x%lx) in segment [0x%lx, 0x%lx] at offset 0x%lx\n",760 usdt_provider, usdt_name, ehdr.e_type == ET_EXEC ? "exec" : "lib ",761 path, note.sema_addr, note.base_addr, usdt_sema_off,762 seg->start, seg->end, seg->offset);763 }764 765 /* Record adjusted addresses and offsets and parse USDT spec */766 tmp = libbpf_reallocarray(targets, target_cnt + 1, sizeof(*targets));767 if (!tmp) {768 err = -ENOMEM;769 goto err_out;770 }771 targets = tmp;772 773 target = &targets[target_cnt];774 memset(target, 0, sizeof(*target));775 776 target->abs_ip = usdt_abs_ip;777 target->rel_ip = usdt_rel_ip;778 target->sema_off = usdt_sema_off;779 780 /* notes.args references strings from ELF itself, so they can781 * be referenced safely until elf_end() call782 */783 target->spec_str = note.args;784 785 err = parse_usdt_spec(&target->spec, ¬e, usdt_cookie);786 if (err)787 goto err_out;788 789 target_cnt++;790 }791 792 *out_targets = targets;793 *out_target_cnt = target_cnt;794 err = target_cnt;795 796err_out:797 free(segs);798 free(vma_segs);799 if (err < 0)800 free(targets);801 return err;802}803 804struct bpf_link_usdt {805 struct bpf_link link;806 807 struct usdt_manager *usdt_man;808 809 size_t spec_cnt;810 int *spec_ids;811 812 size_t uprobe_cnt;813 struct {814 long abs_ip;815 struct bpf_link *link;816 } *uprobes;817 818 struct bpf_link *multi_link;819};820 821static int bpf_link_usdt_detach(struct bpf_link *link)822{823 struct bpf_link_usdt *usdt_link = container_of(link, struct bpf_link_usdt, link);824 struct usdt_manager *man = usdt_link->usdt_man;825 int i;826 827 bpf_link__destroy(usdt_link->multi_link);828 829 /* When having multi_link, uprobe_cnt is 0 */830 for (i = 0; i < usdt_link->uprobe_cnt; i++) {831 /* detach underlying uprobe link */832 bpf_link__destroy(usdt_link->uprobes[i].link);833 /* there is no need to update specs map because it will be834 * unconditionally overwritten on subsequent USDT attaches,835 * but if BPF cookies are not used we need to remove entry836 * from ip_to_spec_id map, otherwise we'll run into false837 * conflicting IP errors838 */839 if (!man->has_bpf_cookie) {840 /* not much we can do about errors here */841 (void)bpf_map_delete_elem(bpf_map__fd(man->ip_to_spec_id_map),842 &usdt_link->uprobes[i].abs_ip);843 }844 }845 846 /* try to return the list of previously used spec IDs to usdt_manager847 * for future reuse for subsequent USDT attaches848 */849 if (!man->free_spec_ids) {850 /* if there were no free spec IDs yet, just transfer our IDs */851 man->free_spec_ids = usdt_link->spec_ids;852 man->free_spec_cnt = usdt_link->spec_cnt;853 usdt_link->spec_ids = NULL;854 } else {855 /* otherwise concat IDs */856 size_t new_cnt = man->free_spec_cnt + usdt_link->spec_cnt;857 int *new_free_ids;858 859 new_free_ids = libbpf_reallocarray(man->free_spec_ids, new_cnt,860 sizeof(*new_free_ids));861 /* If we couldn't resize free_spec_ids, we'll just leak862 * a bunch of free IDs; this is very unlikely to happen and if863 * system is so exhausted on memory, it's the least of user's864 * concerns, probably.865 * So just do our best here to return those IDs to usdt_manager.866 * Another edge case when we can legitimately get NULL is when867 * new_cnt is zero, which can happen in some edge cases, so we868 * need to be careful about that.869 */870 if (new_free_ids || new_cnt == 0) {871 memcpy(new_free_ids + man->free_spec_cnt, usdt_link->spec_ids,872 usdt_link->spec_cnt * sizeof(*usdt_link->spec_ids));873 man->free_spec_ids = new_free_ids;874 man->free_spec_cnt = new_cnt;875 }876 }877 878 return 0;879}880 881static void bpf_link_usdt_dealloc(struct bpf_link *link)882{883 struct bpf_link_usdt *usdt_link = container_of(link, struct bpf_link_usdt, link);884 885 free(usdt_link->spec_ids);886 free(usdt_link->uprobes);887 free(usdt_link);888}889 890static size_t specs_hash_fn(long key, void *ctx)891{892 return str_hash((char *)key);893}894 895static bool specs_equal_fn(long key1, long key2, void *ctx)896{897 return strcmp((char *)key1, (char *)key2) == 0;898}899 900static int allocate_spec_id(struct usdt_manager *man, struct hashmap *specs_hash,901 struct bpf_link_usdt *link, struct usdt_target *target,902 int *spec_id, bool *is_new)903{904 long tmp;905 void *new_ids;906 int err;907 908 /* check if we already allocated spec ID for this spec string */909 if (hashmap__find(specs_hash, target->spec_str, &tmp)) {910 *spec_id = tmp;911 *is_new = false;912 return 0;913 }914 915 /* otherwise it's a new ID that needs to be set up in specs map and916 * returned back to usdt_manager when USDT link is detached917 */918 new_ids = libbpf_reallocarray(link->spec_ids, link->spec_cnt + 1, sizeof(*link->spec_ids));919 if (!new_ids)920 return -ENOMEM;921 link->spec_ids = new_ids;922 923 /* get next free spec ID, giving preference to free list, if not empty */924 if (man->free_spec_cnt) {925 *spec_id = man->free_spec_ids[man->free_spec_cnt - 1];926 927 /* cache spec ID for current spec string for future lookups */928 err = hashmap__add(specs_hash, target->spec_str, *spec_id);929 if (err)930 return err;931 932 man->free_spec_cnt--;933 } else {934 /* don't allocate spec ID bigger than what fits in specs map */935 if (man->next_free_spec_id >= bpf_map__max_entries(man->specs_map))936 return -E2BIG;937 938 *spec_id = man->next_free_spec_id;939 940 /* cache spec ID for current spec string for future lookups */941 err = hashmap__add(specs_hash, target->spec_str, *spec_id);942 if (err)943 return err;944 945 man->next_free_spec_id++;946 }947 948 /* remember new spec ID in the link for later return back to free list on detach */949 link->spec_ids[link->spec_cnt] = *spec_id;950 link->spec_cnt++;951 *is_new = true;952 return 0;953}954 955struct bpf_link *usdt_manager_attach_usdt(struct usdt_manager *man, const struct bpf_program *prog,956 pid_t pid, const char *path,957 const char *usdt_provider, const char *usdt_name,958 __u64 usdt_cookie)959{960 unsigned long *offsets = NULL, *ref_ctr_offsets = NULL;961 int i, err, spec_map_fd, ip_map_fd;962 LIBBPF_OPTS(bpf_uprobe_opts, opts);963 struct hashmap *specs_hash = NULL;964 struct bpf_link_usdt *link = NULL;965 struct usdt_target *targets = NULL;966 __u64 *cookies = NULL;967 struct elf_fd elf_fd;968 size_t target_cnt;969 970 spec_map_fd = bpf_map__fd(man->specs_map);971 ip_map_fd = bpf_map__fd(man->ip_to_spec_id_map);972 973 err = elf_open(path, &elf_fd);974 if (err)975 return libbpf_err_ptr(err);976 977 err = sanity_check_usdt_elf(elf_fd.elf, path);978 if (err)979 goto err_out;980 981 /* normalize PID filter */982 if (pid < 0)983 pid = -1;984 else if (pid == 0)985 pid = getpid();986 987 /* discover USDT in given binary, optionally limiting988 * activations to a given PID, if pid > 0989 */990 err = collect_usdt_targets(man, elf_fd.elf, path, pid, usdt_provider, usdt_name,991 usdt_cookie, &targets, &target_cnt);992 if (err <= 0) {993 err = (err == 0) ? -ENOENT : err;994 goto err_out;995 }996 997 specs_hash = hashmap__new(specs_hash_fn, specs_equal_fn, NULL);998 if (IS_ERR(specs_hash)) {999 err = PTR_ERR(specs_hash);1000 goto err_out;1001 }1002 1003 link = calloc(1, sizeof(*link));1004 if (!link) {1005 err = -ENOMEM;1006 goto err_out;1007 }1008 1009 link->usdt_man = man;1010 link->link.detach = &bpf_link_usdt_detach;1011 link->link.dealloc = &bpf_link_usdt_dealloc;1012 1013 if (man->has_uprobe_multi) {1014 offsets = calloc(target_cnt, sizeof(*offsets));1015 cookies = calloc(target_cnt, sizeof(*cookies));1016 ref_ctr_offsets = calloc(target_cnt, sizeof(*ref_ctr_offsets));1017 1018 if (!offsets || !ref_ctr_offsets || !cookies) {1019 err = -ENOMEM;1020 goto err_out;1021 }1022 } else {1023 link->uprobes = calloc(target_cnt, sizeof(*link->uprobes));1024 if (!link->uprobes) {1025 err = -ENOMEM;1026 goto err_out;1027 }1028 }1029 1030 for (i = 0; i < target_cnt; i++) {1031 struct usdt_target *target = &targets[i];1032 struct bpf_link *uprobe_link;1033 bool is_new;1034 int spec_id;1035 1036 /* Spec ID can be either reused or newly allocated. If it is1037 * newly allocated, we'll need to fill out spec map, otherwise1038 * entire spec should be valid and can be just used by a new1039 * uprobe. We reuse spec when USDT arg spec is identical. We1040 * also never share specs between two different USDT1041 * attachments ("links"), so all the reused specs already1042 * share USDT cookie value implicitly.1043 */1044 err = allocate_spec_id(man, specs_hash, link, target, &spec_id, &is_new);1045 if (err)1046 goto err_out;1047 1048 if (is_new && bpf_map_update_elem(spec_map_fd, &spec_id, &target->spec, BPF_ANY)) {1049 err = -errno;1050 pr_warn("usdt: failed to set USDT spec #%d for '%s:%s' in '%s': %d\n",1051 spec_id, usdt_provider, usdt_name, path, err);1052 goto err_out;1053 }1054 if (!man->has_bpf_cookie &&1055 bpf_map_update_elem(ip_map_fd, &target->abs_ip, &spec_id, BPF_NOEXIST)) {1056 err = -errno;1057 if (err == -EEXIST) {1058 pr_warn("usdt: IP collision detected for spec #%d for '%s:%s' in '%s'\n",1059 spec_id, usdt_provider, usdt_name, path);1060 } else {1061 pr_warn("usdt: failed to map IP 0x%lx to spec #%d for '%s:%s' in '%s': %d\n",1062 target->abs_ip, spec_id, usdt_provider, usdt_name,1063 path, err);1064 }1065 goto err_out;1066 }1067 1068 if (man->has_uprobe_multi) {1069 offsets[i] = target->rel_ip;1070 ref_ctr_offsets[i] = target->sema_off;1071 cookies[i] = spec_id;1072 } else {1073 opts.ref_ctr_offset = target->sema_off;1074 opts.bpf_cookie = man->has_bpf_cookie ? spec_id : 0;1075 uprobe_link = bpf_program__attach_uprobe_opts(prog, pid, path,1076 target->rel_ip, &opts);1077 err = libbpf_get_error(uprobe_link);1078 if (err) {1079 pr_warn("usdt: failed to attach uprobe #%d for '%s:%s' in '%s': %d\n",1080 i, usdt_provider, usdt_name, path, err);1081 goto err_out;1082 }1083 1084 link->uprobes[i].link = uprobe_link;1085 link->uprobes[i].abs_ip = target->abs_ip;1086 link->uprobe_cnt++;1087 }1088 }1089 1090 if (man->has_uprobe_multi) {1091 LIBBPF_OPTS(bpf_uprobe_multi_opts, opts_multi,1092 .ref_ctr_offsets = ref_ctr_offsets,1093 .offsets = offsets,1094 .cookies = cookies,1095 .cnt = target_cnt,1096 );1097 1098 link->multi_link = bpf_program__attach_uprobe_multi(prog, pid, path,1099 NULL, &opts_multi);1100 if (!link->multi_link) {1101 err = -errno;1102 pr_warn("usdt: failed to attach uprobe multi for '%s:%s' in '%s': %d\n",1103 usdt_provider, usdt_name, path, err);1104 goto err_out;1105 }1106 1107 free(offsets);1108 free(ref_ctr_offsets);1109 free(cookies);1110 }1111 1112 free(targets);1113 hashmap__free(specs_hash);1114 elf_close(&elf_fd);1115 return &link->link;1116 1117err_out:1118 free(offsets);1119 free(ref_ctr_offsets);1120 free(cookies);1121 1122 if (link)1123 bpf_link__destroy(&link->link);1124 free(targets);1125 hashmap__free(specs_hash);1126 elf_close(&elf_fd);1127 return libbpf_err_ptr(err);1128}1129 1130/* Parse out USDT ELF note from '.note.stapsdt' section.1131 * Logic inspired by perf's code.1132 */1133static int parse_usdt_note(Elf *elf, const char *path, GElf_Nhdr *nhdr,1134 const char *data, size_t name_off, size_t desc_off,1135 struct usdt_note *note)1136{1137 const char *provider, *name, *args;1138 long addrs[3];1139 size_t len;1140 1141 /* sanity check USDT note name and type first */1142 if (strncmp(data + name_off, USDT_NOTE_NAME, nhdr->n_namesz) != 0)1143 return -EINVAL;1144 if (nhdr->n_type != USDT_NOTE_TYPE)1145 return -EINVAL;1146 1147 /* sanity check USDT note contents ("description" in ELF terminology) */1148 len = nhdr->n_descsz;1149 data = data + desc_off;1150 1151 /* +3 is the very minimum required to store three empty strings */1152 if (len < sizeof(addrs) + 3)1153 return -EINVAL;1154 1155 /* get location, base, and semaphore addrs */1156 memcpy(&addrs, data, sizeof(addrs));1157 1158 /* parse string fields: provider, name, args */1159 provider = data + sizeof(addrs);1160 1161 name = (const char *)memchr(provider, '\0', data + len - provider);1162 if (!name) /* non-zero-terminated provider */1163 return -EINVAL;1164 name++;1165 if (name >= data + len || *name == '\0') /* missing or empty name */1166 return -EINVAL;1167 1168 args = memchr(name, '\0', data + len - name);1169 if (!args) /* non-zero-terminated name */1170 return -EINVAL;1171 ++args;1172 if (args >= data + len) /* missing arguments spec */1173 return -EINVAL;1174 1175 note->provider = provider;1176 note->name = name;1177 if (*args == '\0' || *args == ':')1178 note->args = "";1179 else1180 note->args = args;1181 note->loc_addr = addrs[0];1182 note->base_addr = addrs[1];1183 note->sema_addr = addrs[2];1184 1185 return 0;1186}1187 1188static int parse_usdt_arg(const char *arg_str, int arg_num, struct usdt_arg_spec *arg, int *arg_sz);1189 1190static int parse_usdt_spec(struct usdt_spec *spec, const struct usdt_note *note, __u64 usdt_cookie)1191{1192 struct usdt_arg_spec *arg;1193 const char *s;1194 int arg_sz, len;1195 1196 spec->usdt_cookie = usdt_cookie;1197 spec->arg_cnt = 0;1198 1199 s = note->args;1200 while (s[0]) {1201 if (spec->arg_cnt >= USDT_MAX_ARG_CNT) {1202 pr_warn("usdt: too many USDT arguments (> %d) for '%s:%s' with args spec '%s'\n",1203 USDT_MAX_ARG_CNT, note->provider, note->name, note->args);1204 return -E2BIG;1205 }1206 1207 arg = &spec->args[spec->arg_cnt];1208 len = parse_usdt_arg(s, spec->arg_cnt, arg, &arg_sz);1209 if (len < 0)1210 return len;1211 1212 arg->arg_signed = arg_sz < 0;1213 if (arg_sz < 0)1214 arg_sz = -arg_sz;1215 1216 switch (arg_sz) {1217 case 1: case 2: case 4: case 8:1218 arg->arg_bitshift = 64 - arg_sz * 8;1219 break;1220 default:1221 pr_warn("usdt: unsupported arg #%d (spec '%s') size: %d\n",1222 spec->arg_cnt, s, arg_sz);1223 return -EINVAL;1224 }1225 1226 s += len;1227 spec->arg_cnt++;1228 }1229 1230 return 0;1231}1232 1233/* Architecture-specific logic for parsing USDT argument location specs */1234 1235#if defined(__x86_64__) || defined(__i386__)1236 1237static int calc_pt_regs_off(const char *reg_name)1238{1239 static struct {1240 const char *names[4];1241 size_t pt_regs_off;1242 } reg_map[] = {1243#ifdef __x86_64__1244#define reg_off(reg64, reg32) offsetof(struct pt_regs, reg64)1245#else1246#define reg_off(reg64, reg32) offsetof(struct pt_regs, reg32)1247#endif1248 { {"rip", "eip", "", ""}, reg_off(rip, eip) },1249 { {"rax", "eax", "ax", "al"}, reg_off(rax, eax) },1250 { {"rbx", "ebx", "bx", "bl"}, reg_off(rbx, ebx) },1251 { {"rcx", "ecx", "cx", "cl"}, reg_off(rcx, ecx) },1252 { {"rdx", "edx", "dx", "dl"}, reg_off(rdx, edx) },1253 { {"rsi", "esi", "si", "sil"}, reg_off(rsi, esi) },1254 { {"rdi", "edi", "di", "dil"}, reg_off(rdi, edi) },1255 { {"rbp", "ebp", "bp", "bpl"}, reg_off(rbp, ebp) },1256 { {"rsp", "esp", "sp", "spl"}, reg_off(rsp, esp) },1257#undef reg_off1258#ifdef __x86_64__1259 { {"r8", "r8d", "r8w", "r8b"}, offsetof(struct pt_regs, r8) },1260 { {"r9", "r9d", "r9w", "r9b"}, offsetof(struct pt_regs, r9) },1261 { {"r10", "r10d", "r10w", "r10b"}, offsetof(struct pt_regs, r10) },1262 { {"r11", "r11d", "r11w", "r11b"}, offsetof(struct pt_regs, r11) },1263 { {"r12", "r12d", "r12w", "r12b"}, offsetof(struct pt_regs, r12) },1264 { {"r13", "r13d", "r13w", "r13b"}, offsetof(struct pt_regs, r13) },1265 { {"r14", "r14d", "r14w", "r14b"}, offsetof(struct pt_regs, r14) },1266 { {"r15", "r15d", "r15w", "r15b"}, offsetof(struct pt_regs, r15) },1267#endif1268 };1269 int i, j;1270 1271 for (i = 0; i < ARRAY_SIZE(reg_map); i++) {1272 for (j = 0; j < ARRAY_SIZE(reg_map[i].names); j++) {1273 if (strcmp(reg_name, reg_map[i].names[j]) == 0)1274 return reg_map[i].pt_regs_off;1275 }1276 }1277 1278 pr_warn("usdt: unrecognized register '%s'\n", reg_name);1279 return -ENOENT;1280}1281 1282static int parse_usdt_arg(const char *arg_str, int arg_num, struct usdt_arg_spec *arg, int *arg_sz)1283{1284 char reg_name[16];1285 int len, reg_off;1286 long off;1287 1288 if (sscanf(arg_str, " %d @ %ld ( %%%15[^)] ) %n", arg_sz, &off, reg_name, &len) == 3) {1289 /* Memory dereference case, e.g., -4@-20(%rbp) */1290 arg->arg_type = USDT_ARG_REG_DEREF;1291 arg->val_off = off;1292 reg_off = calc_pt_regs_off(reg_name);1293 if (reg_off < 0)1294 return reg_off;1295 arg->reg_off = reg_off;1296 } else if (sscanf(arg_str, " %d @ ( %%%15[^)] ) %n", arg_sz, reg_name, &len) == 2) {1297 /* Memory dereference case without offset, e.g., 8@(%rsp) */1298 arg->arg_type = USDT_ARG_REG_DEREF;1299 arg->val_off = 0;1300 reg_off = calc_pt_regs_off(reg_name);1301 if (reg_off < 0)1302 return reg_off;1303 arg->reg_off = reg_off;1304 } else if (sscanf(arg_str, " %d @ %%%15s %n", arg_sz, reg_name, &len) == 2) {1305 /* Register read case, e.g., -4@%eax */1306 arg->arg_type = USDT_ARG_REG;1307 arg->val_off = 0;1308 1309 reg_off = calc_pt_regs_off(reg_name);1310 if (reg_off < 0)1311 return reg_off;1312 arg->reg_off = reg_off;1313 } else if (sscanf(arg_str, " %d @ $%ld %n", arg_sz, &off, &len) == 2) {1314 /* Constant value case, e.g., 4@$71 */1315 arg->arg_type = USDT_ARG_CONST;1316 arg->val_off = off;1317 arg->reg_off = 0;1318 } else {1319 pr_warn("usdt: unrecognized arg #%d spec '%s'\n", arg_num, arg_str);1320 return -EINVAL;1321 }1322 1323 return len;1324}1325 1326#elif defined(__s390x__)1327 1328/* Do not support __s390__ for now, since user_pt_regs is broken with -m31. */1329 1330static int parse_usdt_arg(const char *arg_str, int arg_num, struct usdt_arg_spec *arg, int *arg_sz)1331{1332 unsigned int reg;1333 int len;1334 long off;1335 1336 if (sscanf(arg_str, " %d @ %ld ( %%r%u ) %n", arg_sz, &off, ®, &len) == 3) {1337 /* Memory dereference case, e.g., -2@-28(%r15) */1338 arg->arg_type = USDT_ARG_REG_DEREF;1339 arg->val_off = off;1340 if (reg > 15) {1341 pr_warn("usdt: unrecognized register '%%r%u'\n", reg);1342 return -EINVAL;1343 }1344 arg->reg_off = offsetof(user_pt_regs, gprs[reg]);1345 } else if (sscanf(arg_str, " %d @ %%r%u %n", arg_sz, ®, &len) == 2) {1346 /* Register read case, e.g., -8@%r0 */1347 arg->arg_type = USDT_ARG_REG;1348 arg->val_off = 0;1349 if (reg > 15) {1350 pr_warn("usdt: unrecognized register '%%r%u'\n", reg);1351 return -EINVAL;1352 }1353 arg->reg_off = offsetof(user_pt_regs, gprs[reg]);1354 } else if (sscanf(arg_str, " %d @ %ld %n", arg_sz, &off, &len) == 2) {1355 /* Constant value case, e.g., 4@71 */1356 arg->arg_type = USDT_ARG_CONST;1357 arg->val_off = off;1358 arg->reg_off = 0;1359 } else {1360 pr_warn("usdt: unrecognized arg #%d spec '%s'\n", arg_num, arg_str);1361 return -EINVAL;1362 }1363 1364 return len;1365}1366 1367#elif defined(__aarch64__)1368 1369static int calc_pt_regs_off(const char *reg_name)1370{1371 int reg_num;1372 1373 if (sscanf(reg_name, "x%d", ®_num) == 1) {1374 if (reg_num >= 0 && reg_num < 31)1375 return offsetof(struct user_pt_regs, regs[reg_num]);1376 } else if (strcmp(reg_name, "sp") == 0) {1377 return offsetof(struct user_pt_regs, sp);1378 }1379 pr_warn("usdt: unrecognized register '%s'\n", reg_name);1380 return -ENOENT;1381}1382 1383static int parse_usdt_arg(const char *arg_str, int arg_num, struct usdt_arg_spec *arg, int *arg_sz)1384{1385 char reg_name[16];1386 int len, reg_off;1387 long off;1388 1389 if (sscanf(arg_str, " %d @ \[ %15[a-z0-9] , %ld ] %n", arg_sz, reg_name, &off, &len) == 3) {1390 /* Memory dereference case, e.g., -4@[sp, 96] */1391 arg->arg_type = USDT_ARG_REG_DEREF;1392 arg->val_off = off;1393 reg_off = calc_pt_regs_off(reg_name);1394 if (reg_off < 0)1395 return reg_off;1396 arg->reg_off = reg_off;1397 } else if (sscanf(arg_str, " %d @ \[ %15[a-z0-9] ] %n", arg_sz, reg_name, &len) == 2) {1398 /* Memory dereference case, e.g., -4@[sp] */1399 arg->arg_type = USDT_ARG_REG_DEREF;1400 arg->val_off = 0;1401 reg_off = calc_pt_regs_off(reg_name);1402 if (reg_off < 0)1403 return reg_off;1404 arg->reg_off = reg_off;1405 } else if (sscanf(arg_str, " %d @ %ld %n", arg_sz, &off, &len) == 2) {1406 /* Constant value case, e.g., 4@5 */1407 arg->arg_type = USDT_ARG_CONST;1408 arg->val_off = off;1409 arg->reg_off = 0;1410 } else if (sscanf(arg_str, " %d @ %15[a-z0-9] %n", arg_sz, reg_name, &len) == 2) {1411 /* Register read case, e.g., -8@x4 */1412 arg->arg_type = USDT_ARG_REG;1413 arg->val_off = 0;1414 reg_off = calc_pt_regs_off(reg_name);1415 if (reg_off < 0)1416 return reg_off;1417 arg->reg_off = reg_off;1418 } else {1419 pr_warn("usdt: unrecognized arg #%d spec '%s'\n", arg_num, arg_str);1420 return -EINVAL;1421 }1422 1423 return len;1424}1425 1426#elif defined(__riscv)1427 1428static int calc_pt_regs_off(const char *reg_name)1429{1430 static struct {1431 const char *name;1432 size_t pt_regs_off;1433 } reg_map[] = {1434 { "ra", offsetof(struct user_regs_struct, ra) },1435 { "sp", offsetof(struct user_regs_struct, sp) },1436 { "gp", offsetof(struct user_regs_struct, gp) },1437 { "tp", offsetof(struct user_regs_struct, tp) },1438 { "a0", offsetof(struct user_regs_struct, a0) },1439 { "a1", offsetof(struct user_regs_struct, a1) },1440 { "a2", offsetof(struct user_regs_struct, a2) },1441 { "a3", offsetof(struct user_regs_struct, a3) },1442 { "a4", offsetof(struct user_regs_struct, a4) },1443 { "a5", offsetof(struct user_regs_struct, a5) },1444 { "a6", offsetof(struct user_regs_struct, a6) },1445 { "a7", offsetof(struct user_regs_struct, a7) },1446 { "s0", offsetof(struct user_regs_struct, s0) },1447 { "s1", offsetof(struct user_regs_struct, s1) },1448 { "s2", offsetof(struct user_regs_struct, s2) },1449 { "s3", offsetof(struct user_regs_struct, s3) },1450 { "s4", offsetof(struct user_regs_struct, s4) },1451 { "s5", offsetof(struct user_regs_struct, s5) },1452 { "s6", offsetof(struct user_regs_struct, s6) },1453 { "s7", offsetof(struct user_regs_struct, s7) },1454 { "s8", offsetof(struct user_regs_struct, rv_s8) },1455 { "s9", offsetof(struct user_regs_struct, s9) },1456 { "s10", offsetof(struct user_regs_struct, s10) },1457 { "s11", offsetof(struct user_regs_struct, s11) },1458 { "t0", offsetof(struct user_regs_struct, t0) },1459 { "t1", offsetof(struct user_regs_struct, t1) },1460 { "t2", offsetof(struct user_regs_struct, t2) },1461 { "t3", offsetof(struct user_regs_struct, t3) },1462 { "t4", offsetof(struct user_regs_struct, t4) },1463 { "t5", offsetof(struct user_regs_struct, t5) },1464 { "t6", offsetof(struct user_regs_struct, t6) },1465 };1466 int i;1467 1468 for (i = 0; i < ARRAY_SIZE(reg_map); i++) {1469 if (strcmp(reg_name, reg_map[i].name) == 0)1470 return reg_map[i].pt_regs_off;1471 }1472 1473 pr_warn("usdt: unrecognized register '%s'\n", reg_name);1474 return -ENOENT;1475}1476 1477static int parse_usdt_arg(const char *arg_str, int arg_num, struct usdt_arg_spec *arg, int *arg_sz)1478{1479 char reg_name[16];1480 int len, reg_off;1481 long off;1482 1483 if (sscanf(arg_str, " %d @ %ld ( %15[a-z0-9] ) %n", arg_sz, &off, reg_name, &len) == 3) {1484 /* Memory dereference case, e.g., -8@-88(s0) */1485 arg->arg_type = USDT_ARG_REG_DEREF;1486 arg->val_off = off;1487 reg_off = calc_pt_regs_off(reg_name);1488 if (reg_off < 0)1489 return reg_off;1490 arg->reg_off = reg_off;1491 } else if (sscanf(arg_str, " %d @ %ld %n", arg_sz, &off, &len) == 2) {1492 /* Constant value case, e.g., 4@5 */1493 arg->arg_type = USDT_ARG_CONST;1494 arg->val_off = off;1495 arg->reg_off = 0;1496 } else if (sscanf(arg_str, " %d @ %15[a-z0-9] %n", arg_sz, reg_name, &len) == 2) {1497 /* Register read case, e.g., -8@a1 */1498 arg->arg_type = USDT_ARG_REG;1499 arg->val_off = 0;1500 reg_off = calc_pt_regs_off(reg_name);1501 if (reg_off < 0)1502 return reg_off;1503 arg->reg_off = reg_off;1504 } else {1505 pr_warn("usdt: unrecognized arg #%d spec '%s'\n", arg_num, arg_str);1506 return -EINVAL;1507 }1508 1509 return len;1510}1511 1512#elif defined(__arm__)1513 1514static int calc_pt_regs_off(const char *reg_name)1515{1516 static struct {1517 const char *name;1518 size_t pt_regs_off;1519 } reg_map[] = {1520 { "r0", offsetof(struct pt_regs, uregs[0]) },1521 { "r1", offsetof(struct pt_regs, uregs[1]) },1522 { "r2", offsetof(struct pt_regs, uregs[2]) },1523 { "r3", offsetof(struct pt_regs, uregs[3]) },1524 { "r4", offsetof(struct pt_regs, uregs[4]) },1525 { "r5", offsetof(struct pt_regs, uregs[5]) },1526 { "r6", offsetof(struct pt_regs, uregs[6]) },1527 { "r7", offsetof(struct pt_regs, uregs[7]) },1528 { "r8", offsetof(struct pt_regs, uregs[8]) },1529 { "r9", offsetof(struct pt_regs, uregs[9]) },1530 { "r10", offsetof(struct pt_regs, uregs[10]) },1531 { "fp", offsetof(struct pt_regs, uregs[11]) },1532 { "ip", offsetof(struct pt_regs, uregs[12]) },1533 { "sp", offsetof(struct pt_regs, uregs[13]) },1534 { "lr", offsetof(struct pt_regs, uregs[14]) },1535 { "pc", offsetof(struct pt_regs, uregs[15]) },1536 };1537 int i;1538 1539 for (i = 0; i < ARRAY_SIZE(reg_map); i++) {1540 if (strcmp(reg_name, reg_map[i].name) == 0)1541 return reg_map[i].pt_regs_off;1542 }1543 1544 pr_warn("usdt: unrecognized register '%s'\n", reg_name);1545 return -ENOENT;1546}1547 1548static int parse_usdt_arg(const char *arg_str, int arg_num, struct usdt_arg_spec *arg, int *arg_sz)1549{1550 char reg_name[16];1551 int len, reg_off;1552 long off;1553 1554 if (sscanf(arg_str, " %d @ \[ %15[a-z0-9] , #%ld ] %n",1555 arg_sz, reg_name, &off, &len) == 3) {1556 /* Memory dereference case, e.g., -4@[fp, #96] */1557 arg->arg_type = USDT_ARG_REG_DEREF;1558 arg->val_off = off;1559 reg_off = calc_pt_regs_off(reg_name);1560 if (reg_off < 0)1561 return reg_off;1562 arg->reg_off = reg_off;1563 } else if (sscanf(arg_str, " %d @ \[ %15[a-z0-9] ] %n", arg_sz, reg_name, &len) == 2) {1564 /* Memory dereference case, e.g., -4@[sp] */1565 arg->arg_type = USDT_ARG_REG_DEREF;1566 arg->val_off = 0;1567 reg_off = calc_pt_regs_off(reg_name);1568 if (reg_off < 0)1569 return reg_off;1570 arg->reg_off = reg_off;1571 } else if (sscanf(arg_str, " %d @ #%ld %n", arg_sz, &off, &len) == 2) {1572 /* Constant value case, e.g., 4@#5 */1573 arg->arg_type = USDT_ARG_CONST;1574 arg->val_off = off;1575 arg->reg_off = 0;1576 } else if (sscanf(arg_str, " %d @ %15[a-z0-9] %n", arg_sz, reg_name, &len) == 2) {1577 /* Register read case, e.g., -8@r4 */1578 arg->arg_type = USDT_ARG_REG;1579 arg->val_off = 0;1580 reg_off = calc_pt_regs_off(reg_name);1581 if (reg_off < 0)1582 return reg_off;1583 arg->reg_off = reg_off;1584 } else {1585 pr_warn("usdt: unrecognized arg #%d spec '%s'\n", arg_num, arg_str);1586 return -EINVAL;1587 }1588 1589 return len;1590}1591 1592#else1593 1594static int parse_usdt_arg(const char *arg_str, int arg_num, struct usdt_arg_spec *arg, int *arg_sz)1595{1596 pr_warn("usdt: libbpf doesn't support USDTs on current architecture\n");1597 return -ENOTSUP;1598}1599 1600#endif1601