567 lines · c
1// SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)2// Copyright (c) 2022 Google3#include "vmlinux.h"4#include <bpf/bpf_helpers.h>5#include <bpf/bpf_tracing.h>6#include <bpf/bpf_core_read.h>7#include <asm-generic/errno-base.h>8 9#include "lock_data.h"10 11/* for collect_lock_syms(). 4096 was rejected by the verifier */12#define MAX_CPUS 102413 14/* lock contention flags from include/trace/events/lock.h */15#define LCB_F_SPIN (1U << 0)16#define LCB_F_READ (1U << 1)17#define LCB_F_WRITE (1U << 2)18#define LCB_F_RT (1U << 3)19#define LCB_F_PERCPU (1U << 4)20#define LCB_F_MUTEX (1U << 5)21 22/* callstack storage */23struct {24 __uint(type, BPF_MAP_TYPE_STACK_TRACE);25 __uint(key_size, sizeof(__u32));26 __uint(value_size, sizeof(__u64));27 __uint(max_entries, MAX_ENTRIES);28} stacks SEC(".maps");29 30/* maintain timestamp at the beginning of contention */31struct {32 __uint(type, BPF_MAP_TYPE_HASH);33 __type(key, int);34 __type(value, struct tstamp_data);35 __uint(max_entries, MAX_ENTRIES);36} tstamp SEC(".maps");37 38/* maintain per-CPU timestamp at the beginning of contention */39struct {40 __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY);41 __uint(key_size, sizeof(__u32));42 __uint(value_size, sizeof(struct tstamp_data));43 __uint(max_entries, 1);44} tstamp_cpu SEC(".maps");45 46/* actual lock contention statistics */47struct {48 __uint(type, BPF_MAP_TYPE_HASH);49 __uint(key_size, sizeof(struct contention_key));50 __uint(value_size, sizeof(struct contention_data));51 __uint(max_entries, MAX_ENTRIES);52} lock_stat SEC(".maps");53 54struct {55 __uint(type, BPF_MAP_TYPE_HASH);56 __uint(key_size, sizeof(__u32));57 __uint(value_size, sizeof(struct contention_task_data));58 __uint(max_entries, MAX_ENTRIES);59} task_data SEC(".maps");60 61struct {62 __uint(type, BPF_MAP_TYPE_HASH);63 __uint(key_size, sizeof(__u64));64 __uint(value_size, sizeof(__u32));65 __uint(max_entries, MAX_ENTRIES);66} lock_syms SEC(".maps");67 68struct {69 __uint(type, BPF_MAP_TYPE_HASH);70 __uint(key_size, sizeof(__u32));71 __uint(value_size, sizeof(__u8));72 __uint(max_entries, 1);73} cpu_filter SEC(".maps");74 75struct {76 __uint(type, BPF_MAP_TYPE_HASH);77 __uint(key_size, sizeof(__u32));78 __uint(value_size, sizeof(__u8));79 __uint(max_entries, 1);80} task_filter SEC(".maps");81 82struct {83 __uint(type, BPF_MAP_TYPE_HASH);84 __uint(key_size, sizeof(__u32));85 __uint(value_size, sizeof(__u8));86 __uint(max_entries, 1);87} type_filter SEC(".maps");88 89struct {90 __uint(type, BPF_MAP_TYPE_HASH);91 __uint(key_size, sizeof(__u64));92 __uint(value_size, sizeof(__u8));93 __uint(max_entries, 1);94} addr_filter SEC(".maps");95 96struct {97 __uint(type, BPF_MAP_TYPE_HASH);98 __uint(key_size, sizeof(__u64));99 __uint(value_size, sizeof(__u8));100 __uint(max_entries, 1);101} cgroup_filter SEC(".maps");102 103struct rw_semaphore___old {104 struct task_struct *owner;105} __attribute__((preserve_access_index));106 107struct rw_semaphore___new {108 atomic_long_t owner;109} __attribute__((preserve_access_index));110 111struct mm_struct___old {112 struct rw_semaphore mmap_sem;113} __attribute__((preserve_access_index));114 115struct mm_struct___new {116 struct rw_semaphore mmap_lock;117} __attribute__((preserve_access_index));118 119/* control flags */120const volatile int has_cpu;121const volatile int has_task;122const volatile int has_type;123const volatile int has_addr;124const volatile int has_cgroup;125const volatile int needs_callstack;126const volatile int stack_skip;127const volatile int lock_owner;128const volatile int use_cgroup_v2;129 130/* determine the key of lock stat */131const volatile int aggr_mode;132 133int enabled;134 135int perf_subsys_id = -1;136 137__u64 end_ts;138 139/* error stat */140int task_fail;141int stack_fail;142int time_fail;143int data_fail;144 145int task_map_full;146int data_map_full;147 148static inline __u64 get_current_cgroup_id(void)149{150 struct task_struct *task;151 struct cgroup *cgrp;152 153 if (use_cgroup_v2)154 return bpf_get_current_cgroup_id();155 156 task = bpf_get_current_task_btf();157 158 if (perf_subsys_id == -1) {159#if __has_builtin(__builtin_preserve_enum_value)160 perf_subsys_id = bpf_core_enum_value(enum cgroup_subsys_id,161 perf_event_cgrp_id);162#else163 perf_subsys_id = perf_event_cgrp_id;164#endif165 }166 167 cgrp = BPF_CORE_READ(task, cgroups, subsys[perf_subsys_id], cgroup);168 return BPF_CORE_READ(cgrp, kn, id);169}170 171static inline int can_record(u64 *ctx)172{173 if (has_cpu) {174 __u32 cpu = bpf_get_smp_processor_id();175 __u8 *ok;176 177 ok = bpf_map_lookup_elem(&cpu_filter, &cpu);178 if (!ok)179 return 0;180 }181 182 if (has_task) {183 __u8 *ok;184 __u32 pid = bpf_get_current_pid_tgid();185 186 ok = bpf_map_lookup_elem(&task_filter, &pid);187 if (!ok)188 return 0;189 }190 191 if (has_type) {192 __u8 *ok;193 __u32 flags = (__u32)ctx[1];194 195 ok = bpf_map_lookup_elem(&type_filter, &flags);196 if (!ok)197 return 0;198 }199 200 if (has_addr) {201 __u8 *ok;202 __u64 addr = ctx[0];203 204 ok = bpf_map_lookup_elem(&addr_filter, &addr);205 if (!ok)206 return 0;207 }208 209 if (has_cgroup) {210 __u8 *ok;211 __u64 cgrp = get_current_cgroup_id();212 213 ok = bpf_map_lookup_elem(&cgroup_filter, &cgrp);214 if (!ok)215 return 0;216 }217 218 return 1;219}220 221static inline int update_task_data(struct task_struct *task)222{223 struct contention_task_data *p;224 int pid, err;225 226 err = bpf_core_read(&pid, sizeof(pid), &task->pid);227 if (err)228 return -1;229 230 p = bpf_map_lookup_elem(&task_data, &pid);231 if (p == NULL && !task_map_full) {232 struct contention_task_data data = {};233 234 BPF_CORE_READ_STR_INTO(&data.comm, task, comm);235 if (bpf_map_update_elem(&task_data, &pid, &data, BPF_NOEXIST) == -E2BIG)236 task_map_full = 1;237 }238 239 return 0;240}241 242#ifndef __has_builtin243# define __has_builtin(x) 0244#endif245 246static inline struct task_struct *get_lock_owner(__u64 lock, __u32 flags)247{248 struct task_struct *task;249 __u64 owner = 0;250 251 if (flags & LCB_F_MUTEX) {252 struct mutex *mutex = (void *)lock;253 owner = BPF_CORE_READ(mutex, owner.counter);254 } else if (flags == LCB_F_READ || flags == LCB_F_WRITE) {255 /*256 * Support for the BPF_TYPE_MATCHES argument to the257 * __builtin_preserve_type_info builtin was added at some point during258 * development of clang 15 and it's what is needed for259 * bpf_core_type_matches.260 */261#if __has_builtin(__builtin_preserve_type_info) && __clang_major__ >= 15262 if (bpf_core_type_matches(struct rw_semaphore___old)) {263 struct rw_semaphore___old *rwsem = (void *)lock;264 owner = (unsigned long)BPF_CORE_READ(rwsem, owner);265 } else if (bpf_core_type_matches(struct rw_semaphore___new)) {266 struct rw_semaphore___new *rwsem = (void *)lock;267 owner = BPF_CORE_READ(rwsem, owner.counter);268 }269#else270 /* assume new struct */271 struct rw_semaphore *rwsem = (void *)lock;272 owner = BPF_CORE_READ(rwsem, owner.counter);273#endif274 }275 276 if (!owner)277 return NULL;278 279 task = (void *)(owner & ~7UL);280 return task;281}282 283static inline __u32 check_lock_type(__u64 lock, __u32 flags)284{285 struct task_struct *curr;286 struct mm_struct___old *mm_old;287 struct mm_struct___new *mm_new;288 struct sighand_struct *sighand;289 290 switch (flags) {291 case LCB_F_READ: /* rwsem */292 case LCB_F_WRITE:293 curr = bpf_get_current_task_btf();294 if (curr->mm == NULL)295 break;296 mm_new = (void *)curr->mm;297 if (bpf_core_field_exists(mm_new->mmap_lock)) {298 if (&mm_new->mmap_lock == (void *)lock)299 return LCD_F_MMAP_LOCK;300 break;301 }302 mm_old = (void *)curr->mm;303 if (bpf_core_field_exists(mm_old->mmap_sem)) {304 if (&mm_old->mmap_sem == (void *)lock)305 return LCD_F_MMAP_LOCK;306 }307 break;308 case LCB_F_SPIN: /* spinlock */309 curr = bpf_get_current_task_btf();310 sighand = curr->sighand;311 312 if (sighand && &sighand->siglock == (void *)lock)313 return LCD_F_SIGHAND_LOCK;314 break;315 default:316 break;317 }318 return 0;319}320 321static inline struct tstamp_data *get_tstamp_elem(__u32 flags)322{323 __u32 pid;324 struct tstamp_data *pelem;325 326 /* Use per-cpu array map for spinlock and rwlock */327 if ((flags & (LCB_F_SPIN | LCB_F_MUTEX)) == LCB_F_SPIN) {328 __u32 idx = 0;329 330 pelem = bpf_map_lookup_elem(&tstamp_cpu, &idx);331 /* Do not update the element for nested locks */332 if (pelem && pelem->lock)333 pelem = NULL;334 return pelem;335 }336 337 pid = bpf_get_current_pid_tgid();338 pelem = bpf_map_lookup_elem(&tstamp, &pid);339 /* Do not update the element for nested locks */340 if (pelem && pelem->lock)341 return NULL;342 343 if (pelem == NULL) {344 struct tstamp_data zero = {};345 346 if (bpf_map_update_elem(&tstamp, &pid, &zero, BPF_NOEXIST) < 0) {347 __sync_fetch_and_add(&task_fail, 1);348 return NULL;349 }350 351 pelem = bpf_map_lookup_elem(&tstamp, &pid);352 if (pelem == NULL) {353 __sync_fetch_and_add(&task_fail, 1);354 return NULL;355 }356 }357 return pelem;358}359 360SEC("tp_btf/contention_begin")361int contention_begin(u64 *ctx)362{363 struct tstamp_data *pelem;364 365 if (!enabled || !can_record(ctx))366 return 0;367 368 pelem = get_tstamp_elem(ctx[1]);369 if (pelem == NULL)370 return 0;371 372 pelem->timestamp = bpf_ktime_get_ns();373 pelem->lock = (__u64)ctx[0];374 pelem->flags = (__u32)ctx[1];375 376 if (needs_callstack) {377 pelem->stack_id = bpf_get_stackid(ctx, &stacks,378 BPF_F_FAST_STACK_CMP | stack_skip);379 if (pelem->stack_id < 0)380 __sync_fetch_and_add(&stack_fail, 1);381 } else if (aggr_mode == LOCK_AGGR_TASK) {382 struct task_struct *task;383 384 if (lock_owner) {385 task = get_lock_owner(pelem->lock, pelem->flags);386 387 /* The flags is not used anymore. Pass the owner pid. */388 if (task)389 pelem->flags = BPF_CORE_READ(task, pid);390 else391 pelem->flags = -1U;392 393 } else {394 task = bpf_get_current_task_btf();395 }396 397 if (task) {398 if (update_task_data(task) < 0 && lock_owner)399 pelem->flags = -1U;400 }401 }402 403 return 0;404}405 406SEC("tp_btf/contention_end")407int contention_end(u64 *ctx)408{409 __u32 pid = 0, idx = 0;410 struct tstamp_data *pelem;411 struct contention_key key = {};412 struct contention_data *data;413 __u64 duration;414 bool need_delete = false;415 416 if (!enabled)417 return 0;418 419 /*420 * For spinlock and rwlock, it needs to get the timestamp for the421 * per-cpu map. However, contention_end does not have the flags422 * so it cannot know whether it reads percpu or hash map.423 *424 * Try per-cpu map first and check if there's active contention.425 * If it is, do not read hash map because it cannot go to sleeping426 * locks before releasing the spinning locks.427 */428 pelem = bpf_map_lookup_elem(&tstamp_cpu, &idx);429 if (pelem && pelem->lock) {430 if (pelem->lock != ctx[0])431 return 0;432 } else {433 pid = bpf_get_current_pid_tgid();434 pelem = bpf_map_lookup_elem(&tstamp, &pid);435 if (!pelem || pelem->lock != ctx[0])436 return 0;437 need_delete = true;438 }439 440 duration = bpf_ktime_get_ns() - pelem->timestamp;441 if ((__s64)duration < 0) {442 __sync_fetch_and_add(&time_fail, 1);443 goto out;444 }445 446 switch (aggr_mode) {447 case LOCK_AGGR_CALLER:448 key.stack_id = pelem->stack_id;449 break;450 case LOCK_AGGR_TASK:451 if (lock_owner)452 key.pid = pelem->flags;453 else {454 if (!need_delete)455 pid = bpf_get_current_pid_tgid();456 key.pid = pid;457 }458 if (needs_callstack)459 key.stack_id = pelem->stack_id;460 break;461 case LOCK_AGGR_ADDR:462 key.lock_addr_or_cgroup = pelem->lock;463 if (needs_callstack)464 key.stack_id = pelem->stack_id;465 break;466 case LOCK_AGGR_CGROUP:467 key.lock_addr_or_cgroup = get_current_cgroup_id();468 break;469 default:470 /* should not happen */471 return 0;472 }473 474 data = bpf_map_lookup_elem(&lock_stat, &key);475 if (!data) {476 if (data_map_full) {477 __sync_fetch_and_add(&data_fail, 1);478 goto out;479 }480 481 struct contention_data first = {482 .total_time = duration,483 .max_time = duration,484 .min_time = duration,485 .count = 1,486 .flags = pelem->flags,487 };488 int err;489 490 if (aggr_mode == LOCK_AGGR_ADDR)491 first.flags |= check_lock_type(pelem->lock, pelem->flags);492 493 err = bpf_map_update_elem(&lock_stat, &key, &first, BPF_NOEXIST);494 if (err < 0) {495 if (err == -EEXIST) {496 /* it lost the race, try to get it again */497 data = bpf_map_lookup_elem(&lock_stat, &key);498 if (data != NULL)499 goto found;500 }501 if (err == -E2BIG)502 data_map_full = 1;503 __sync_fetch_and_add(&data_fail, 1);504 }505 goto out;506 }507 508found:509 __sync_fetch_and_add(&data->total_time, duration);510 __sync_fetch_and_add(&data->count, 1);511 512 /* FIXME: need atomic operations */513 if (data->max_time < duration)514 data->max_time = duration;515 if (data->min_time > duration)516 data->min_time = duration;517 518out:519 pelem->lock = 0;520 if (need_delete)521 bpf_map_delete_elem(&tstamp, &pid);522 return 0;523}524 525extern struct rq runqueues __ksym;526 527struct rq___old {528 raw_spinlock_t lock;529} __attribute__((preserve_access_index));530 531struct rq___new {532 raw_spinlock_t __lock;533} __attribute__((preserve_access_index));534 535SEC("raw_tp/bpf_test_finish")536int BPF_PROG(collect_lock_syms)537{538 __u64 lock_addr, lock_off;539 __u32 lock_flag;540 541 if (bpf_core_field_exists(struct rq___new, __lock))542 lock_off = offsetof(struct rq___new, __lock);543 else544 lock_off = offsetof(struct rq___old, lock);545 546 for (int i = 0; i < MAX_CPUS; i++) {547 struct rq *rq = bpf_per_cpu_ptr(&runqueues, i);548 549 if (rq == NULL)550 break;551 552 lock_addr = (__u64)(void *)rq + lock_off;553 lock_flag = LOCK_CLASS_RQLOCK;554 bpf_map_update_elem(&lock_syms, &lock_addr, &lock_flag, BPF_ANY);555 }556 return 0;557}558 559SEC("raw_tp/bpf_test_finish")560int BPF_PROG(end_timestamp)561{562 end_ts = bpf_ktime_get_ns();563 return 0;564}565 566char LICENSE[] SEC("license") = "Dual BSD/GPL";567