636 lines · c
1// SPDX-License-Identifier: GPL-2.02// Copyright (c) 2019 Facebook3 4#include <stdint.h>5#include <stddef.h>6#include <stdbool.h>7#include <linux/bpf.h>8#include <linux/ptrace.h>9#include <linux/sched.h>10#include <linux/types.h>11#include <bpf/bpf_helpers.h>12 13#include "bpf_compiler.h"14 15typedef uint32_t pid_t;16struct task_struct {};17 18#define TASK_COMM_LEN 1619#define PERF_MAX_STACK_DEPTH 12720 21#define STROBE_TYPE_INVALID 022#define STROBE_TYPE_INT 123#define STROBE_TYPE_STR 224#define STROBE_TYPE_MAP 325 26#define STACK_TABLE_EPOCH_SHIFT 2027#define STROBE_MAX_STR_LEN 128#define STROBE_MAX_CFGS 3229#define READ_MAP_VAR_PAYLOAD_CAP \30 ((1 + STROBE_MAX_MAP_ENTRIES * 2) * STROBE_MAX_STR_LEN)31#define STROBE_MAX_PAYLOAD \32 (STROBE_MAX_STRS * STROBE_MAX_STR_LEN + \33 STROBE_MAX_MAPS * READ_MAP_VAR_PAYLOAD_CAP)34 35struct strobe_value_header {36 /*37 * meaning depends on type:38 * 1. int: 0, if value not set, 1 otherwise39 * 2. str: 1 always, whether value is set or not is determined by ptr40 * 3. map: 1 always, pointer points to additional struct with number41 * of entries (up to STROBE_MAX_MAP_ENTRIES)42 */43 uint16_t len;44 /*45 * _reserved might be used for some future fields/flags, but we always46 * want to keep strobe_value_header to be 8 bytes, so BPF can read 1647 * bytes in one go and get both header and value48 */49 uint8_t _reserved[6];50};51 52/*53 * strobe_value_generic is used from BPF probe only, but needs to be a union54 * of strobe_value_int/strobe_value_str/strobe_value_map55 */56struct strobe_value_generic {57 struct strobe_value_header header;58 union {59 int64_t val;60 void *ptr;61 };62};63 64struct strobe_value_int {65 struct strobe_value_header header;66 int64_t value;67};68 69struct strobe_value_str {70 struct strobe_value_header header;71 const char* value;72};73 74struct strobe_value_map {75 struct strobe_value_header header;76 const struct strobe_map_raw* value;77};78 79struct strobe_map_entry {80 const char* key;81 const char* val;82};83 84/*85 * Map of C-string key/value pairs with fixed maximum capacity. Each map has86 * corresponding int64 ID, which application can use (or ignore) in whatever87 * way appropriate. Map is "write-only", there is no way to get data out of88 * map. Map is intended to be used to provide metadata for profilers and is89 * not to be used for internal in-app communication. All methods are90 * thread-safe.91 */92struct strobe_map_raw {93 /*94 * general purpose unique ID that's up to application to decide95 * whether and how to use; for request metadata use case id is unique96 * request ID that's used to match metadata with stack traces on97 * Strobelight backend side98 */99 int64_t id;100 /* number of used entries in map */101 int64_t cnt;102 /*103 * having volatile doesn't change anything on BPF side, but clang104 * emits warnings for passing `volatile const char *` into105 * bpf_probe_read_user_str that expects just `const char *`106 */107 const char* tag;108 /*109 * key/value entries, each consisting of 2 pointers to key and value110 * C strings111 */112 struct strobe_map_entry entries[STROBE_MAX_MAP_ENTRIES];113};114 115/* Following values define supported values of TLS mode */116#define TLS_NOT_SET -1117#define TLS_LOCAL_EXEC 0118#define TLS_IMM_EXEC 1119#define TLS_GENERAL_DYN 2120 121/*122 * structure that universally represents TLS location (both for static123 * executables and shared libraries)124 */125struct strobe_value_loc {126 /*127 * tls_mode defines what TLS mode was used for particular metavariable:128 * - -1 (TLS_NOT_SET) - no metavariable;129 * - 0 (TLS_LOCAL_EXEC) - Local Executable mode;130 * - 1 (TLS_IMM_EXEC) - Immediate Executable mode;131 * - 2 (TLS_GENERAL_DYN) - General Dynamic mode;132 * Local Dynamic mode is not yet supported, because never seen in133 * practice. Mode defines how offset field is interpreted. See134 * calc_location() in below for details.135 */136 int64_t tls_mode;137 /*138 * TLS_LOCAL_EXEC: offset from thread pointer (fs:0 for x86-64,139 * tpidr_el0 for aarch64).140 * TLS_IMM_EXEC: absolute address of GOT entry containing offset141 * from thread pointer;142 * TLS_GENERAL_DYN: absolute address of double GOT entry143 * containing tls_index_t struct;144 */145 int64_t offset;146};147 148struct strobemeta_cfg {149 int64_t req_meta_idx;150 struct strobe_value_loc int_locs[STROBE_MAX_INTS];151 struct strobe_value_loc str_locs[STROBE_MAX_STRS];152 struct strobe_value_loc map_locs[STROBE_MAX_MAPS];153};154 155struct strobe_map_descr {156 uint64_t id;157 int16_t tag_len;158 /*159 * cnt <0 - map value isn't set;160 * 0 - map has id set, but no key/value entries161 */162 int16_t cnt;163 /*164 * both key_lens[i] and val_lens[i] should be >0 for present key/value165 * entry166 */167 uint16_t key_lens[STROBE_MAX_MAP_ENTRIES];168 uint16_t val_lens[STROBE_MAX_MAP_ENTRIES];169};170 171struct strobemeta_payload {172 /* req_id has valid request ID, if req_meta_valid == 1 */173 int64_t req_id;174 uint8_t req_meta_valid;175 /*176 * mask has Nth bit set to 1, if Nth metavar was present and177 * successfully read178 */179 uint64_t int_vals_set_mask;180 int64_t int_vals[STROBE_MAX_INTS];181 /* len is >0 for present values */182 uint16_t str_lens[STROBE_MAX_STRS];183 /* if map_descrs[i].cnt == -1, metavar is not present/set */184 struct strobe_map_descr map_descrs[STROBE_MAX_MAPS];185 /*186 * payload has compactly packed values of str and map variables in the187 * form: strval1\0strval2\0map1key1\0map1val1\0map2key1\0map2val1\0188 * (and so on); str_lens[i], key_lens[i] and val_lens[i] determines189 * value length190 */191 char payload[STROBE_MAX_PAYLOAD];192};193 194struct strobelight_bpf_sample {195 uint64_t ktime;196 char comm[TASK_COMM_LEN];197 pid_t pid;198 int user_stack_id;199 int kernel_stack_id;200 int has_meta;201 struct strobemeta_payload metadata;202 /*203 * makes it possible to pass (<real payload size> + 1) as data size to204 * perf_submit() to avoid perf_submit's paranoia about passing zero as205 * size, as it deduces that <real payload size> might be206 * **theoretically** zero207 */208 char dummy_safeguard;209};210 211struct {212 __uint(type, BPF_MAP_TYPE_PERF_EVENT_ARRAY);213 __uint(max_entries, 32);214 __uint(key_size, sizeof(int));215 __uint(value_size, sizeof(int));216} samples SEC(".maps");217 218struct {219 __uint(type, BPF_MAP_TYPE_STACK_TRACE);220 __uint(max_entries, 16);221 __uint(key_size, sizeof(uint32_t));222 __uint(value_size, sizeof(uint64_t) * PERF_MAX_STACK_DEPTH);223} stacks_0 SEC(".maps");224 225struct {226 __uint(type, BPF_MAP_TYPE_STACK_TRACE);227 __uint(max_entries, 16);228 __uint(key_size, sizeof(uint32_t));229 __uint(value_size, sizeof(uint64_t) * PERF_MAX_STACK_DEPTH);230} stacks_1 SEC(".maps");231 232struct {233 __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY);234 __uint(max_entries, 1);235 __type(key, uint32_t);236 __type(value, struct strobelight_bpf_sample);237} sample_heap SEC(".maps");238 239struct {240 __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY);241 __uint(max_entries, STROBE_MAX_CFGS);242 __type(key, pid_t);243 __type(value, struct strobemeta_cfg);244} strobemeta_cfgs SEC(".maps");245 246/* Type for the dtv. */247/* https://github.com/lattera/glibc/blob/master/nptl/sysdeps/x86_64/tls.h#L34 */248typedef union dtv {249 size_t counter;250 struct {251 void* val;252 bool is_static;253 } pointer;254} dtv_t;255 256/* Partial definition for tcbhead_t */257/* https://github.com/bminor/glibc/blob/master/sysdeps/x86_64/nptl/tls.h#L42 */258struct tcbhead {259 void* tcb;260 dtv_t* dtv;261};262 263/*264 * TLS module/offset information for shared library case.265 * For x86-64, this is mapped onto two entries in GOT.266 * For aarch64, this is pointed to by second GOT entry.267 */268struct tls_index {269 uint64_t module;270 uint64_t offset;271};272 273#ifdef SUBPROGS274__noinline275#else276__always_inline277#endif278static void *calc_location(struct strobe_value_loc *loc, void *tls_base)279{280 /*281 * tls_mode value is:282 * - -1 (TLS_NOT_SET), if no metavar is present;283 * - 0 (TLS_LOCAL_EXEC), if metavar uses Local Executable mode of TLS284 * (offset from fs:0 for x86-64 or tpidr_el0 for aarch64);285 * - 1 (TLS_IMM_EXEC), if metavar uses Immediate Executable mode of TLS;286 * - 2 (TLS_GENERAL_DYN), if metavar uses General Dynamic mode of TLS;287 * This schema allows to use something like:288 * (tls_mode + 1) * (tls_base + offset)289 * to get NULL for "no metavar" location, or correct pointer for local290 * executable mode without doing extra ifs.291 */292 if (loc->tls_mode <= TLS_LOCAL_EXEC) {293 /* static executable is simple, we just have offset from294 * tls_base */295 void *addr = tls_base + loc->offset;296 /* multiply by (tls_mode + 1) to get NULL, if we have no297 * metavar in this slot */298 return (void *)((loc->tls_mode + 1) * (int64_t)addr);299 }300 /*301 * Other modes are more complicated, we need to jump through few hoops.302 *303 * For immediate executable mode (currently supported only for aarch64):304 * - loc->offset is pointing to a GOT entry containing fixed offset305 * relative to tls_base;306 *307 * For general dynamic mode:308 * - loc->offset is pointing to a beginning of double GOT entries;309 * - (for aarch64 only) second entry points to tls_index_t struct;310 * - (for x86-64 only) two GOT entries are already tls_index_t;311 * - tls_index_t->module is used to find start of TLS section in312 * which variable resides;313 * - tls_index_t->offset provides offset within that TLS section,314 * pointing to value of variable.315 */316 struct tls_index tls_index;317 dtv_t *dtv;318 void *tls_ptr;319 320 bpf_probe_read_user(&tls_index, sizeof(struct tls_index),321 (void *)loc->offset);322 /* valid module index is always positive */323 if (tls_index.module > 0) {324 /* dtv = ((struct tcbhead *)tls_base)->dtv[tls_index.module] */325 bpf_probe_read_user(&dtv, sizeof(dtv),326 &((struct tcbhead *)tls_base)->dtv);327 dtv += tls_index.module;328 } else {329 dtv = NULL;330 }331 bpf_probe_read_user(&tls_ptr, sizeof(void *), dtv);332 /* if pointer has (void *)-1 value, then TLS wasn't initialized yet */333 return tls_ptr && tls_ptr != (void *)-1334 ? tls_ptr + tls_index.offset335 : NULL;336}337 338#ifdef SUBPROGS339__noinline340#else341__always_inline342#endif343static void read_int_var(struct strobemeta_cfg *cfg,344 size_t idx, void *tls_base,345 struct strobe_value_generic *value,346 struct strobemeta_payload *data)347{348 void *location = calc_location(&cfg->int_locs[idx], tls_base);349 if (!location)350 return;351 352 bpf_probe_read_user(value, sizeof(struct strobe_value_generic), location);353 data->int_vals[idx] = value->val;354 if (value->header.len)355 data->int_vals_set_mask |= (1 << idx);356}357 358static __always_inline uint64_t read_str_var(struct strobemeta_cfg *cfg,359 size_t idx, void *tls_base,360 struct strobe_value_generic *value,361 struct strobemeta_payload *data,362 size_t off)363{364 void *location;365 uint64_t len;366 367 data->str_lens[idx] = 0;368 location = calc_location(&cfg->str_locs[idx], tls_base);369 if (!location)370 return 0;371 372 bpf_probe_read_user(value, sizeof(struct strobe_value_generic), location);373 len = bpf_probe_read_user_str(&data->payload[off], STROBE_MAX_STR_LEN, value->ptr);374 /*375 * if bpf_probe_read_user_str returns error (<0), due to casting to376 * unsigned int, it will become big number, so next check is377 * sufficient to check for errors AND prove to BPF verifier, that378 * bpf_probe_read_user_str won't return anything bigger than379 * STROBE_MAX_STR_LEN380 */381 if (len > STROBE_MAX_STR_LEN)382 return 0;383 384 data->str_lens[idx] = len;385 return off + len;386}387 388static __always_inline uint64_t read_map_var(struct strobemeta_cfg *cfg,389 size_t idx, void *tls_base,390 struct strobe_value_generic *value,391 struct strobemeta_payload *data,392 size_t off)393{394 struct strobe_map_descr* descr = &data->map_descrs[idx];395 struct strobe_map_raw map;396 void *location;397 uint64_t len;398 399 descr->tag_len = 0; /* presume no tag is set */400 descr->cnt = -1; /* presume no value is set */401 402 location = calc_location(&cfg->map_locs[idx], tls_base);403 if (!location)404 return off;405 406 bpf_probe_read_user(value, sizeof(struct strobe_value_generic), location);407 if (bpf_probe_read_user(&map, sizeof(struct strobe_map_raw), value->ptr))408 return off;409 410 descr->id = map.id;411 descr->cnt = map.cnt;412 if (cfg->req_meta_idx == idx) {413 data->req_id = map.id;414 data->req_meta_valid = 1;415 }416 417 len = bpf_probe_read_user_str(&data->payload[off], STROBE_MAX_STR_LEN, map.tag);418 if (len <= STROBE_MAX_STR_LEN) {419 descr->tag_len = len;420 off += len;421 }422 423#ifdef NO_UNROLL424 __pragma_loop_no_unroll425#else426 __pragma_loop_unroll427#endif428 for (int i = 0; i < STROBE_MAX_MAP_ENTRIES; ++i) {429 if (i >= map.cnt)430 break;431 432 descr->key_lens[i] = 0;433 len = bpf_probe_read_user_str(&data->payload[off], STROBE_MAX_STR_LEN,434 map.entries[i].key);435 if (len <= STROBE_MAX_STR_LEN) {436 descr->key_lens[i] = len;437 off += len;438 }439 descr->val_lens[i] = 0;440 len = bpf_probe_read_user_str(&data->payload[off], STROBE_MAX_STR_LEN,441 map.entries[i].val);442 if (len <= STROBE_MAX_STR_LEN) {443 descr->val_lens[i] = len;444 off += len;445 }446 }447 448 return off;449}450 451#ifdef USE_BPF_LOOP452enum read_type {453 READ_INT_VAR,454 READ_MAP_VAR,455 READ_STR_VAR,456};457 458struct read_var_ctx {459 struct strobemeta_payload *data;460 void *tls_base;461 struct strobemeta_cfg *cfg;462 size_t payload_off;463 /* value gets mutated */464 struct strobe_value_generic *value;465 enum read_type type;466};467 468static int read_var_callback(__u64 index, struct read_var_ctx *ctx)469{470 /* lose precision info for ctx->payload_off, verifier won't track471 * double xor, barrier_var() is needed to force clang keep both xors.472 */473 ctx->payload_off ^= index;474 barrier_var(ctx->payload_off);475 ctx->payload_off ^= index;476 switch (ctx->type) {477 case READ_INT_VAR:478 if (index >= STROBE_MAX_INTS)479 return 1;480 read_int_var(ctx->cfg, index, ctx->tls_base, ctx->value, ctx->data);481 break;482 case READ_MAP_VAR:483 if (index >= STROBE_MAX_MAPS)484 return 1;485 if (ctx->payload_off > sizeof(ctx->data->payload) - READ_MAP_VAR_PAYLOAD_CAP)486 return 1;487 ctx->payload_off = read_map_var(ctx->cfg, index, ctx->tls_base,488 ctx->value, ctx->data, ctx->payload_off);489 break;490 case READ_STR_VAR:491 if (index >= STROBE_MAX_STRS)492 return 1;493 if (ctx->payload_off > sizeof(ctx->data->payload) - STROBE_MAX_STR_LEN)494 return 1;495 ctx->payload_off = read_str_var(ctx->cfg, index, ctx->tls_base,496 ctx->value, ctx->data, ctx->payload_off);497 break;498 }499 return 0;500}501#endif /* USE_BPF_LOOP */502 503/*504 * read_strobe_meta returns NULL, if no metadata was read; otherwise returns505 * pointer to *right after* payload ends506 */507#ifdef SUBPROGS508__noinline509#else510__always_inline511#endif512static void *read_strobe_meta(struct task_struct *task,513 struct strobemeta_payload *data)514{515 pid_t pid = bpf_get_current_pid_tgid() >> 32;516 struct strobe_value_generic value = {0};517 struct strobemeta_cfg *cfg;518 size_t payload_off;519 void *tls_base;520 521 cfg = bpf_map_lookup_elem(&strobemeta_cfgs, &pid);522 if (!cfg)523 return NULL;524 525 data->int_vals_set_mask = 0;526 data->req_meta_valid = 0;527 payload_off = 0;528 /*529 * we don't have struct task_struct definition, it should be:530 * tls_base = (void *)task->thread.fsbase;531 */532 tls_base = (void *)task;533 534#ifdef USE_BPF_LOOP535 struct read_var_ctx ctx = {536 .cfg = cfg,537 .tls_base = tls_base,538 .value = &value,539 .data = data,540 .payload_off = 0,541 };542 int err;543 544 ctx.type = READ_INT_VAR;545 err = bpf_loop(STROBE_MAX_INTS, read_var_callback, &ctx, 0);546 if (err != STROBE_MAX_INTS)547 return NULL;548 549 ctx.type = READ_STR_VAR;550 err = bpf_loop(STROBE_MAX_STRS, read_var_callback, &ctx, 0);551 if (err != STROBE_MAX_STRS)552 return NULL;553 554 ctx.type = READ_MAP_VAR;555 err = bpf_loop(STROBE_MAX_MAPS, read_var_callback, &ctx, 0);556 if (err != STROBE_MAX_MAPS)557 return NULL;558 559 payload_off = ctx.payload_off;560 /* this should not really happen, here only to satisfy verifier */561 if (payload_off > sizeof(data->payload))562 payload_off = sizeof(data->payload);563#else564#ifdef NO_UNROLL565 __pragma_loop_no_unroll566#else567 __pragma_loop_unroll568#endif /* NO_UNROLL */569 for (int i = 0; i < STROBE_MAX_INTS; ++i) {570 read_int_var(cfg, i, tls_base, &value, data);571 }572#ifdef NO_UNROLL573 __pragma_loop_no_unroll574#else575 __pragma_loop_unroll576#endif /* NO_UNROLL */577 for (int i = 0; i < STROBE_MAX_STRS; ++i) {578 payload_off = read_str_var(cfg, i, tls_base, &value, data, payload_off);579 }580#ifdef NO_UNROLL581 __pragma_loop_no_unroll582#else583 __pragma_loop_unroll584#endif /* NO_UNROLL */585 for (int i = 0; i < STROBE_MAX_MAPS; ++i) {586 payload_off = read_map_var(cfg, i, tls_base, &value, data, payload_off);587 }588#endif /* USE_BPF_LOOP */589 590 /*591 * return pointer right after end of payload, so it's possible to592 * calculate exact amount of useful data that needs to be sent593 */594 return &data->payload[payload_off];595}596 597SEC("raw_tracepoint/kfree_skb")598int on_event(struct pt_regs *ctx) {599 pid_t pid = bpf_get_current_pid_tgid() >> 32;600 struct strobelight_bpf_sample* sample;601 struct task_struct *task;602 uint32_t zero = 0;603 uint64_t ktime_ns;604 void *sample_end;605 606 sample = bpf_map_lookup_elem(&sample_heap, &zero);607 if (!sample)608 return 0; /* this will never happen */609 610 sample->pid = pid;611 bpf_get_current_comm(&sample->comm, TASK_COMM_LEN);612 ktime_ns = bpf_ktime_get_ns();613 sample->ktime = ktime_ns;614 615 task = (struct task_struct *)bpf_get_current_task();616 sample_end = read_strobe_meta(task, &sample->metadata);617 sample->has_meta = sample_end != NULL;618 sample_end = sample_end ? : &sample->metadata;619 620 if ((ktime_ns >> STACK_TABLE_EPOCH_SHIFT) & 1) {621 sample->kernel_stack_id = bpf_get_stackid(ctx, &stacks_1, 0);622 sample->user_stack_id = bpf_get_stackid(ctx, &stacks_1, BPF_F_USER_STACK);623 } else {624 sample->kernel_stack_id = bpf_get_stackid(ctx, &stacks_0, 0);625 sample->user_stack_id = bpf_get_stackid(ctx, &stacks_0, BPF_F_USER_STACK);626 }627 628 uint64_t sample_size = sample_end - (void *)sample;629 /* should always be true */630 if (sample_size < sizeof(struct strobelight_bpf_sample))631 bpf_perf_event_output(ctx, &samples, 0, sample, 1 + sample_size);632 return 0;633}634 635char _license[] SEC("license") = "GPL";636