930 lines · c
1/* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */2#ifndef __BPF_TRACING_H__3#define __BPF_TRACING_H__4 5#include "bpf_helpers.h"6 7/* Scan the ARCH passed in from ARCH env variable (see Makefile) */8#if defined(__TARGET_ARCH_x86)9 #define bpf_target_x8610 #define bpf_target_defined11#elif defined(__TARGET_ARCH_s390)12 #define bpf_target_s39013 #define bpf_target_defined14#elif defined(__TARGET_ARCH_arm)15 #define bpf_target_arm16 #define bpf_target_defined17#elif defined(__TARGET_ARCH_arm64)18 #define bpf_target_arm6419 #define bpf_target_defined20#elif defined(__TARGET_ARCH_mips)21 #define bpf_target_mips22 #define bpf_target_defined23#elif defined(__TARGET_ARCH_powerpc)24 #define bpf_target_powerpc25 #define bpf_target_defined26#elif defined(__TARGET_ARCH_sparc)27 #define bpf_target_sparc28 #define bpf_target_defined29#elif defined(__TARGET_ARCH_riscv)30 #define bpf_target_riscv31 #define bpf_target_defined32#elif defined(__TARGET_ARCH_arc)33 #define bpf_target_arc34 #define bpf_target_defined35#elif defined(__TARGET_ARCH_loongarch)36 #define bpf_target_loongarch37 #define bpf_target_defined38#else39 40/* Fall back to what the compiler says */41#if defined(__x86_64__)42 #define bpf_target_x8643 #define bpf_target_defined44#elif defined(__s390__)45 #define bpf_target_s39046 #define bpf_target_defined47#elif defined(__arm__)48 #define bpf_target_arm49 #define bpf_target_defined50#elif defined(__aarch64__)51 #define bpf_target_arm6452 #define bpf_target_defined53#elif defined(__mips__)54 #define bpf_target_mips55 #define bpf_target_defined56#elif defined(__powerpc__)57 #define bpf_target_powerpc58 #define bpf_target_defined59#elif defined(__sparc__)60 #define bpf_target_sparc61 #define bpf_target_defined62#elif defined(__riscv) && __riscv_xlen == 6463 #define bpf_target_riscv64 #define bpf_target_defined65#elif defined(__arc__)66 #define bpf_target_arc67 #define bpf_target_defined68#elif defined(__loongarch__)69 #define bpf_target_loongarch70 #define bpf_target_defined71#endif /* no compiler target */72 73#endif74 75#ifndef __BPF_TARGET_MISSING76#define __BPF_TARGET_MISSING "GCC error \"Must specify a BPF target arch via __TARGET_ARCH_xxx\""77#endif78 79#if defined(bpf_target_x86)80 81/*82 * https://en.wikipedia.org/wiki/X86_calling_conventions#System_V_AMD64_ABI83 */84 85#if defined(__KERNEL__) || defined(__VMLINUX_H__)86 87#define __PT_PARM1_REG di88#define __PT_PARM2_REG si89#define __PT_PARM3_REG dx90#define __PT_PARM4_REG cx91#define __PT_PARM5_REG r892#define __PT_PARM6_REG r993/*94 * Syscall uses r10 for PARM4. See arch/x86/entry/entry_64.S:entry_SYSCALL_6495 * comments in Linux sources. And refer to syscall(2) manpage.96 */97#define __PT_PARM1_SYSCALL_REG __PT_PARM1_REG98#define __PT_PARM2_SYSCALL_REG __PT_PARM2_REG99#define __PT_PARM3_SYSCALL_REG __PT_PARM3_REG100#define __PT_PARM4_SYSCALL_REG r10101#define __PT_PARM5_SYSCALL_REG __PT_PARM5_REG102#define __PT_PARM6_SYSCALL_REG __PT_PARM6_REG103 104#define __PT_RET_REG sp105#define __PT_FP_REG bp106#define __PT_RC_REG ax107#define __PT_SP_REG sp108#define __PT_IP_REG ip109 110#else111 112#ifdef __i386__113 114/* i386 kernel is built with -mregparm=3 */115#define __PT_PARM1_REG eax116#define __PT_PARM2_REG edx117#define __PT_PARM3_REG ecx118/* i386 syscall ABI is very different, refer to syscall(2) manpage */119#define __PT_PARM1_SYSCALL_REG ebx120#define __PT_PARM2_SYSCALL_REG ecx121#define __PT_PARM3_SYSCALL_REG edx122#define __PT_PARM4_SYSCALL_REG esi123#define __PT_PARM5_SYSCALL_REG edi124#define __PT_PARM6_SYSCALL_REG ebp125 126#define __PT_RET_REG esp127#define __PT_FP_REG ebp128#define __PT_RC_REG eax129#define __PT_SP_REG esp130#define __PT_IP_REG eip131 132#else /* __i386__ */133 134#define __PT_PARM1_REG rdi135#define __PT_PARM2_REG rsi136#define __PT_PARM3_REG rdx137#define __PT_PARM4_REG rcx138#define __PT_PARM5_REG r8139#define __PT_PARM6_REG r9140 141#define __PT_PARM1_SYSCALL_REG __PT_PARM1_REG142#define __PT_PARM2_SYSCALL_REG __PT_PARM2_REG143#define __PT_PARM3_SYSCALL_REG __PT_PARM3_REG144#define __PT_PARM4_SYSCALL_REG r10145#define __PT_PARM5_SYSCALL_REG __PT_PARM5_REG146#define __PT_PARM6_SYSCALL_REG __PT_PARM6_REG147 148#define __PT_RET_REG rsp149#define __PT_FP_REG rbp150#define __PT_RC_REG rax151#define __PT_SP_REG rsp152#define __PT_IP_REG rip153 154#endif /* __i386__ */155 156#endif /* __KERNEL__ || __VMLINUX_H__ */157 158#elif defined(bpf_target_s390)159 160/*161 * https://github.com/IBM/s390x-abi/releases/download/v1.6/lzsabi_s390x.pdf162 */163 164struct pt_regs___s390 {165 unsigned long orig_gpr2;166} __attribute__((preserve_access_index));167 168/* s390 provides user_pt_regs instead of struct pt_regs to userspace */169#define __PT_REGS_CAST(x) ((const user_pt_regs *)(x))170#define __PT_PARM1_REG gprs[2]171#define __PT_PARM2_REG gprs[3]172#define __PT_PARM3_REG gprs[4]173#define __PT_PARM4_REG gprs[5]174#define __PT_PARM5_REG gprs[6]175 176#define __PT_PARM1_SYSCALL_REG orig_gpr2177#define __PT_PARM2_SYSCALL_REG __PT_PARM2_REG178#define __PT_PARM3_SYSCALL_REG __PT_PARM3_REG179#define __PT_PARM4_SYSCALL_REG __PT_PARM4_REG180#define __PT_PARM5_SYSCALL_REG __PT_PARM5_REG181#define __PT_PARM6_SYSCALL_REG gprs[7]182#define PT_REGS_PARM1_SYSCALL(x) (((const struct pt_regs___s390 *)(x))->__PT_PARM1_SYSCALL_REG)183#define PT_REGS_PARM1_CORE_SYSCALL(x) \184 BPF_CORE_READ((const struct pt_regs___s390 *)(x), __PT_PARM1_SYSCALL_REG)185 186#define __PT_RET_REG gprs[14]187#define __PT_FP_REG gprs[11] /* Works only with CONFIG_FRAME_POINTER */188#define __PT_RC_REG gprs[2]189#define __PT_SP_REG gprs[15]190#define __PT_IP_REG psw.addr191 192#elif defined(bpf_target_arm)193 194/*195 * https://github.com/ARM-software/abi-aa/blob/main/aapcs32/aapcs32.rst#machine-registers196 */197 198#define __PT_PARM1_REG uregs[0]199#define __PT_PARM2_REG uregs[1]200#define __PT_PARM3_REG uregs[2]201#define __PT_PARM4_REG uregs[3]202 203#define __PT_PARM1_SYSCALL_REG __PT_PARM1_REG204#define __PT_PARM2_SYSCALL_REG __PT_PARM2_REG205#define __PT_PARM3_SYSCALL_REG __PT_PARM3_REG206#define __PT_PARM4_SYSCALL_REG __PT_PARM4_REG207#define __PT_PARM5_SYSCALL_REG uregs[4]208#define __PT_PARM6_SYSCALL_REG uregs[5]209#define __PT_PARM7_SYSCALL_REG uregs[6]210 211#define __PT_RET_REG uregs[14]212#define __PT_FP_REG uregs[11] /* Works only with CONFIG_FRAME_POINTER */213#define __PT_RC_REG uregs[0]214#define __PT_SP_REG uregs[13]215#define __PT_IP_REG uregs[12]216 217#elif defined(bpf_target_arm64)218 219/*220 * https://github.com/ARM-software/abi-aa/blob/main/aapcs64/aapcs64.rst#machine-registers221 */222 223struct pt_regs___arm64 {224 unsigned long orig_x0;225} __attribute__((preserve_access_index));226 227/* arm64 provides struct user_pt_regs instead of struct pt_regs to userspace */228#define __PT_REGS_CAST(x) ((const struct user_pt_regs *)(x))229#define __PT_PARM1_REG regs[0]230#define __PT_PARM2_REG regs[1]231#define __PT_PARM3_REG regs[2]232#define __PT_PARM4_REG regs[3]233#define __PT_PARM5_REG regs[4]234#define __PT_PARM6_REG regs[5]235#define __PT_PARM7_REG regs[6]236#define __PT_PARM8_REG regs[7]237 238#define __PT_PARM1_SYSCALL_REG orig_x0239#define __PT_PARM2_SYSCALL_REG __PT_PARM2_REG240#define __PT_PARM3_SYSCALL_REG __PT_PARM3_REG241#define __PT_PARM4_SYSCALL_REG __PT_PARM4_REG242#define __PT_PARM5_SYSCALL_REG __PT_PARM5_REG243#define __PT_PARM6_SYSCALL_REG __PT_PARM6_REG244#define PT_REGS_PARM1_SYSCALL(x) (((const struct pt_regs___arm64 *)(x))->__PT_PARM1_SYSCALL_REG)245#define PT_REGS_PARM1_CORE_SYSCALL(x) \246 BPF_CORE_READ((const struct pt_regs___arm64 *)(x), __PT_PARM1_SYSCALL_REG)247 248#define __PT_RET_REG regs[30]249#define __PT_FP_REG regs[29] /* Works only with CONFIG_FRAME_POINTER */250#define __PT_RC_REG regs[0]251#define __PT_SP_REG sp252#define __PT_IP_REG pc253 254#elif defined(bpf_target_mips)255 256/*257 * N64 ABI is assumed right now.258 * https://en.wikipedia.org/wiki/MIPS_architecture#Calling_conventions259 */260 261#define __PT_PARM1_REG regs[4]262#define __PT_PARM2_REG regs[5]263#define __PT_PARM3_REG regs[6]264#define __PT_PARM4_REG regs[7]265#define __PT_PARM5_REG regs[8]266#define __PT_PARM6_REG regs[9]267#define __PT_PARM7_REG regs[10]268#define __PT_PARM8_REG regs[11]269 270#define __PT_PARM1_SYSCALL_REG __PT_PARM1_REG271#define __PT_PARM2_SYSCALL_REG __PT_PARM2_REG272#define __PT_PARM3_SYSCALL_REG __PT_PARM3_REG273#define __PT_PARM4_SYSCALL_REG __PT_PARM4_REG274#define __PT_PARM5_SYSCALL_REG __PT_PARM5_REG /* only N32/N64 */275#define __PT_PARM6_SYSCALL_REG __PT_PARM6_REG /* only N32/N64 */276 277#define __PT_RET_REG regs[31]278#define __PT_FP_REG regs[30] /* Works only with CONFIG_FRAME_POINTER */279#define __PT_RC_REG regs[2]280#define __PT_SP_REG regs[29]281#define __PT_IP_REG cp0_epc282 283#elif defined(bpf_target_powerpc)284 285/*286 * http://refspecs.linux-foundation.org/elf/elfspec_ppc.pdf (page 3-14,287 * section "Function Calling Sequence")288 */289 290#define __PT_PARM1_REG gpr[3]291#define __PT_PARM2_REG gpr[4]292#define __PT_PARM3_REG gpr[5]293#define __PT_PARM4_REG gpr[6]294#define __PT_PARM5_REG gpr[7]295#define __PT_PARM6_REG gpr[8]296#define __PT_PARM7_REG gpr[9]297#define __PT_PARM8_REG gpr[10]298 299/* powerpc does not select ARCH_HAS_SYSCALL_WRAPPER. */300#define PT_REGS_SYSCALL_REGS(ctx) ctx301#define __PT_PARM1_SYSCALL_REG orig_gpr3302#define __PT_PARM2_SYSCALL_REG __PT_PARM2_REG303#define __PT_PARM3_SYSCALL_REG __PT_PARM3_REG304#define __PT_PARM4_SYSCALL_REG __PT_PARM4_REG305#define __PT_PARM5_SYSCALL_REG __PT_PARM5_REG306#define __PT_PARM6_SYSCALL_REG __PT_PARM6_REG307#if !defined(__arch64__)308#define __PT_PARM7_SYSCALL_REG __PT_PARM7_REG /* only powerpc (not powerpc64) */309#endif310 311#define __PT_RET_REG regs[31]312#define __PT_FP_REG __unsupported__313#define __PT_RC_REG gpr[3]314#define __PT_SP_REG sp315#define __PT_IP_REG nip316 317#elif defined(bpf_target_sparc)318 319/*320 * https://en.wikipedia.org/wiki/Calling_convention#SPARC321 */322 323#define __PT_PARM1_REG u_regs[UREG_I0]324#define __PT_PARM2_REG u_regs[UREG_I1]325#define __PT_PARM3_REG u_regs[UREG_I2]326#define __PT_PARM4_REG u_regs[UREG_I3]327#define __PT_PARM5_REG u_regs[UREG_I4]328#define __PT_PARM6_REG u_regs[UREG_I5]329 330#define __PT_PARM1_SYSCALL_REG __PT_PARM1_REG331#define __PT_PARM2_SYSCALL_REG __PT_PARM2_REG332#define __PT_PARM3_SYSCALL_REG __PT_PARM3_REG333#define __PT_PARM4_SYSCALL_REG __PT_PARM4_REG334#define __PT_PARM5_SYSCALL_REG __PT_PARM5_REG335#define __PT_PARM6_SYSCALL_REG __PT_PARM6_REG336 337#define __PT_RET_REG u_regs[UREG_I7]338#define __PT_FP_REG __unsupported__339#define __PT_RC_REG u_regs[UREG_I0]340#define __PT_SP_REG u_regs[UREG_FP]341/* Should this also be a bpf_target check for the sparc case? */342#if defined(__arch64__)343#define __PT_IP_REG tpc344#else345#define __PT_IP_REG pc346#endif347 348#elif defined(bpf_target_riscv)349 350/*351 * https://github.com/riscv-non-isa/riscv-elf-psabi-doc/blob/master/riscv-cc.adoc#risc-v-calling-conventions352 */353 354struct pt_regs___riscv {355 unsigned long orig_a0;356} __attribute__((preserve_access_index));357 358/* riscv provides struct user_regs_struct instead of struct pt_regs to userspace */359#define __PT_REGS_CAST(x) ((const struct user_regs_struct *)(x))360#define __PT_PARM1_REG a0361#define __PT_PARM2_REG a1362#define __PT_PARM3_REG a2363#define __PT_PARM4_REG a3364#define __PT_PARM5_REG a4365#define __PT_PARM6_REG a5366#define __PT_PARM7_REG a6367#define __PT_PARM8_REG a7368 369#define __PT_PARM1_SYSCALL_REG orig_a0370#define __PT_PARM2_SYSCALL_REG __PT_PARM2_REG371#define __PT_PARM3_SYSCALL_REG __PT_PARM3_REG372#define __PT_PARM4_SYSCALL_REG __PT_PARM4_REG373#define __PT_PARM5_SYSCALL_REG __PT_PARM5_REG374#define __PT_PARM6_SYSCALL_REG __PT_PARM6_REG375#define PT_REGS_PARM1_SYSCALL(x) (((const struct pt_regs___riscv *)(x))->__PT_PARM1_SYSCALL_REG)376#define PT_REGS_PARM1_CORE_SYSCALL(x) \377 BPF_CORE_READ((const struct pt_regs___riscv *)(x), __PT_PARM1_SYSCALL_REG)378 379#define __PT_RET_REG ra380#define __PT_FP_REG s0381#define __PT_RC_REG a0382#define __PT_SP_REG sp383#define __PT_IP_REG pc384 385#elif defined(bpf_target_arc)386 387/*388 * Section "Function Calling Sequence" (page 24):389 * https://raw.githubusercontent.com/wiki/foss-for-synopsys-dwc-arc-processors/toolchain/files/ARCv2_ABI.pdf390 */391 392/* arc provides struct user_regs_struct instead of struct pt_regs to userspace */393#define __PT_REGS_CAST(x) ((const struct user_regs_struct *)(x))394#define __PT_PARM1_REG scratch.r0395#define __PT_PARM2_REG scratch.r1396#define __PT_PARM3_REG scratch.r2397#define __PT_PARM4_REG scratch.r3398#define __PT_PARM5_REG scratch.r4399#define __PT_PARM6_REG scratch.r5400#define __PT_PARM7_REG scratch.r6401#define __PT_PARM8_REG scratch.r7402 403/* arc does not select ARCH_HAS_SYSCALL_WRAPPER. */404#define PT_REGS_SYSCALL_REGS(ctx) ctx405#define __PT_PARM1_SYSCALL_REG __PT_PARM1_REG406#define __PT_PARM2_SYSCALL_REG __PT_PARM2_REG407#define __PT_PARM3_SYSCALL_REG __PT_PARM3_REG408#define __PT_PARM4_SYSCALL_REG __PT_PARM4_REG409#define __PT_PARM5_SYSCALL_REG __PT_PARM5_REG410#define __PT_PARM6_SYSCALL_REG __PT_PARM6_REG411 412#define __PT_RET_REG scratch.blink413#define __PT_FP_REG scratch.fp414#define __PT_RC_REG scratch.r0415#define __PT_SP_REG scratch.sp416#define __PT_IP_REG scratch.ret417 418#elif defined(bpf_target_loongarch)419 420/*421 * https://docs.kernel.org/loongarch/introduction.html422 * https://loongson.github.io/LoongArch-Documentation/LoongArch-ELF-ABI-EN.html423 */424 425/* loongarch provides struct user_pt_regs instead of struct pt_regs to userspace */426#define __PT_REGS_CAST(x) ((const struct user_pt_regs *)(x))427#define __PT_PARM1_REG regs[4]428#define __PT_PARM2_REG regs[5]429#define __PT_PARM3_REG regs[6]430#define __PT_PARM4_REG regs[7]431#define __PT_PARM5_REG regs[8]432#define __PT_PARM6_REG regs[9]433#define __PT_PARM7_REG regs[10]434#define __PT_PARM8_REG regs[11]435 436/* loongarch does not select ARCH_HAS_SYSCALL_WRAPPER. */437#define PT_REGS_SYSCALL_REGS(ctx) ctx438#define __PT_PARM1_SYSCALL_REG __PT_PARM1_REG439#define __PT_PARM2_SYSCALL_REG __PT_PARM2_REG440#define __PT_PARM3_SYSCALL_REG __PT_PARM3_REG441#define __PT_PARM4_SYSCALL_REG __PT_PARM4_REG442#define __PT_PARM5_SYSCALL_REG __PT_PARM5_REG443#define __PT_PARM6_SYSCALL_REG __PT_PARM6_REG444 445#define __PT_RET_REG regs[1]446#define __PT_FP_REG regs[22]447#define __PT_RC_REG regs[4]448#define __PT_SP_REG regs[3]449#define __PT_IP_REG csr_era450 451#endif452 453#if defined(bpf_target_defined)454 455struct pt_regs;456 457/* allow some architectures to override `struct pt_regs` */458#ifndef __PT_REGS_CAST459#define __PT_REGS_CAST(x) (x)460#endif461 462/*463 * Different architectures support different number of arguments passed464 * through registers. i386 supports just 3, some arches support up to 8.465 */466#ifndef __PT_PARM4_REG467#define __PT_PARM4_REG __unsupported__468#endif469#ifndef __PT_PARM5_REG470#define __PT_PARM5_REG __unsupported__471#endif472#ifndef __PT_PARM6_REG473#define __PT_PARM6_REG __unsupported__474#endif475#ifndef __PT_PARM7_REG476#define __PT_PARM7_REG __unsupported__477#endif478#ifndef __PT_PARM8_REG479#define __PT_PARM8_REG __unsupported__480#endif481/*482 * Similarly, syscall-specific conventions might differ between function call483 * conventions within each architecture. All supported architectures pass484 * either 6 or 7 syscall arguments in registers.485 *486 * See syscall(2) manpage for succinct table with information on each arch.487 */488#ifndef __PT_PARM7_SYSCALL_REG489#define __PT_PARM7_SYSCALL_REG __unsupported__490#endif491 492#define PT_REGS_PARM1(x) (__PT_REGS_CAST(x)->__PT_PARM1_REG)493#define PT_REGS_PARM2(x) (__PT_REGS_CAST(x)->__PT_PARM2_REG)494#define PT_REGS_PARM3(x) (__PT_REGS_CAST(x)->__PT_PARM3_REG)495#define PT_REGS_PARM4(x) (__PT_REGS_CAST(x)->__PT_PARM4_REG)496#define PT_REGS_PARM5(x) (__PT_REGS_CAST(x)->__PT_PARM5_REG)497#define PT_REGS_PARM6(x) (__PT_REGS_CAST(x)->__PT_PARM6_REG)498#define PT_REGS_PARM7(x) (__PT_REGS_CAST(x)->__PT_PARM7_REG)499#define PT_REGS_PARM8(x) (__PT_REGS_CAST(x)->__PT_PARM8_REG)500#define PT_REGS_RET(x) (__PT_REGS_CAST(x)->__PT_RET_REG)501#define PT_REGS_FP(x) (__PT_REGS_CAST(x)->__PT_FP_REG)502#define PT_REGS_RC(x) (__PT_REGS_CAST(x)->__PT_RC_REG)503#define PT_REGS_SP(x) (__PT_REGS_CAST(x)->__PT_SP_REG)504#define PT_REGS_IP(x) (__PT_REGS_CAST(x)->__PT_IP_REG)505 506#define PT_REGS_PARM1_CORE(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_PARM1_REG)507#define PT_REGS_PARM2_CORE(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_PARM2_REG)508#define PT_REGS_PARM3_CORE(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_PARM3_REG)509#define PT_REGS_PARM4_CORE(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_PARM4_REG)510#define PT_REGS_PARM5_CORE(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_PARM5_REG)511#define PT_REGS_PARM6_CORE(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_PARM6_REG)512#define PT_REGS_PARM7_CORE(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_PARM7_REG)513#define PT_REGS_PARM8_CORE(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_PARM8_REG)514#define PT_REGS_RET_CORE(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_RET_REG)515#define PT_REGS_FP_CORE(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_FP_REG)516#define PT_REGS_RC_CORE(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_RC_REG)517#define PT_REGS_SP_CORE(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_SP_REG)518#define PT_REGS_IP_CORE(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_IP_REG)519 520#if defined(bpf_target_powerpc)521 522#define BPF_KPROBE_READ_RET_IP(ip, ctx) ({ (ip) = (ctx)->link; })523#define BPF_KRETPROBE_READ_RET_IP BPF_KPROBE_READ_RET_IP524 525#elif defined(bpf_target_sparc) || defined(bpf_target_arm64)526 527#define BPF_KPROBE_READ_RET_IP(ip, ctx) ({ (ip) = PT_REGS_RET(ctx); })528#define BPF_KRETPROBE_READ_RET_IP BPF_KPROBE_READ_RET_IP529 530#else531 532#define BPF_KPROBE_READ_RET_IP(ip, ctx) \533 ({ bpf_probe_read_kernel(&(ip), sizeof(ip), (void *)PT_REGS_RET(ctx)); })534#define BPF_KRETPROBE_READ_RET_IP(ip, ctx) \535 ({ bpf_probe_read_kernel(&(ip), sizeof(ip), (void *)(PT_REGS_FP(ctx) + sizeof(ip))); })536 537#endif538 539#ifndef PT_REGS_PARM1_SYSCALL540#define PT_REGS_PARM1_SYSCALL(x) (__PT_REGS_CAST(x)->__PT_PARM1_SYSCALL_REG)541#define PT_REGS_PARM1_CORE_SYSCALL(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_PARM1_SYSCALL_REG)542#endif543#ifndef PT_REGS_PARM2_SYSCALL544#define PT_REGS_PARM2_SYSCALL(x) (__PT_REGS_CAST(x)->__PT_PARM2_SYSCALL_REG)545#define PT_REGS_PARM2_CORE_SYSCALL(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_PARM2_SYSCALL_REG)546#endif547#ifndef PT_REGS_PARM3_SYSCALL548#define PT_REGS_PARM3_SYSCALL(x) (__PT_REGS_CAST(x)->__PT_PARM3_SYSCALL_REG)549#define PT_REGS_PARM3_CORE_SYSCALL(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_PARM3_SYSCALL_REG)550#endif551#ifndef PT_REGS_PARM4_SYSCALL552#define PT_REGS_PARM4_SYSCALL(x) (__PT_REGS_CAST(x)->__PT_PARM4_SYSCALL_REG)553#define PT_REGS_PARM4_CORE_SYSCALL(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_PARM4_SYSCALL_REG)554#endif555#ifndef PT_REGS_PARM5_SYSCALL556#define PT_REGS_PARM5_SYSCALL(x) (__PT_REGS_CAST(x)->__PT_PARM5_SYSCALL_REG)557#define PT_REGS_PARM5_CORE_SYSCALL(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_PARM5_SYSCALL_REG)558#endif559#ifndef PT_REGS_PARM6_SYSCALL560#define PT_REGS_PARM6_SYSCALL(x) (__PT_REGS_CAST(x)->__PT_PARM6_SYSCALL_REG)561#define PT_REGS_PARM6_CORE_SYSCALL(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_PARM6_SYSCALL_REG)562#endif563#ifndef PT_REGS_PARM7_SYSCALL564#define PT_REGS_PARM7_SYSCALL(x) (__PT_REGS_CAST(x)->__PT_PARM7_SYSCALL_REG)565#define PT_REGS_PARM7_CORE_SYSCALL(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_PARM7_SYSCALL_REG)566#endif567 568#else /* defined(bpf_target_defined) */569 570#define PT_REGS_PARM1(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })571#define PT_REGS_PARM2(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })572#define PT_REGS_PARM3(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })573#define PT_REGS_PARM4(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })574#define PT_REGS_PARM5(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })575#define PT_REGS_PARM6(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })576#define PT_REGS_PARM7(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })577#define PT_REGS_PARM8(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })578#define PT_REGS_RET(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })579#define PT_REGS_FP(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })580#define PT_REGS_RC(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })581#define PT_REGS_SP(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })582#define PT_REGS_IP(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })583 584#define PT_REGS_PARM1_CORE(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })585#define PT_REGS_PARM2_CORE(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })586#define PT_REGS_PARM3_CORE(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })587#define PT_REGS_PARM4_CORE(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })588#define PT_REGS_PARM5_CORE(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })589#define PT_REGS_PARM6_CORE(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })590#define PT_REGS_PARM7_CORE(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })591#define PT_REGS_PARM8_CORE(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })592#define PT_REGS_RET_CORE(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })593#define PT_REGS_FP_CORE(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })594#define PT_REGS_RC_CORE(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })595#define PT_REGS_SP_CORE(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })596#define PT_REGS_IP_CORE(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })597 598#define BPF_KPROBE_READ_RET_IP(ip, ctx) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })599#define BPF_KRETPROBE_READ_RET_IP(ip, ctx) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })600 601#define PT_REGS_PARM1_SYSCALL(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })602#define PT_REGS_PARM2_SYSCALL(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })603#define PT_REGS_PARM3_SYSCALL(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })604#define PT_REGS_PARM4_SYSCALL(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })605#define PT_REGS_PARM5_SYSCALL(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })606#define PT_REGS_PARM6_SYSCALL(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })607#define PT_REGS_PARM7_SYSCALL(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })608 609#define PT_REGS_PARM1_CORE_SYSCALL(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })610#define PT_REGS_PARM2_CORE_SYSCALL(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })611#define PT_REGS_PARM3_CORE_SYSCALL(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })612#define PT_REGS_PARM4_CORE_SYSCALL(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })613#define PT_REGS_PARM5_CORE_SYSCALL(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })614#define PT_REGS_PARM6_CORE_SYSCALL(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })615#define PT_REGS_PARM7_CORE_SYSCALL(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })616 617#endif /* defined(bpf_target_defined) */618 619/*620 * When invoked from a syscall handler kprobe, returns a pointer to a621 * struct pt_regs containing syscall arguments and suitable for passing to622 * PT_REGS_PARMn_SYSCALL() and PT_REGS_PARMn_CORE_SYSCALL().623 */624#ifndef PT_REGS_SYSCALL_REGS625/* By default, assume that the arch selects ARCH_HAS_SYSCALL_WRAPPER. */626#define PT_REGS_SYSCALL_REGS(ctx) ((struct pt_regs *)PT_REGS_PARM1(ctx))627#endif628 629#ifndef ___bpf_concat630#define ___bpf_concat(a, b) a ## b631#endif632#ifndef ___bpf_apply633#define ___bpf_apply(fn, n) ___bpf_concat(fn, n)634#endif635#ifndef ___bpf_nth636#define ___bpf_nth(_, _1, _2, _3, _4, _5, _6, _7, _8, _9, _a, _b, _c, N, ...) N637#endif638#ifndef ___bpf_narg639#define ___bpf_narg(...) ___bpf_nth(_, ##__VA_ARGS__, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0)640#endif641 642#define ___bpf_ctx_cast0() ctx643#define ___bpf_ctx_cast1(x) ___bpf_ctx_cast0(), ctx[0]644#define ___bpf_ctx_cast2(x, args...) ___bpf_ctx_cast1(args), ctx[1]645#define ___bpf_ctx_cast3(x, args...) ___bpf_ctx_cast2(args), ctx[2]646#define ___bpf_ctx_cast4(x, args...) ___bpf_ctx_cast3(args), ctx[3]647#define ___bpf_ctx_cast5(x, args...) ___bpf_ctx_cast4(args), ctx[4]648#define ___bpf_ctx_cast6(x, args...) ___bpf_ctx_cast5(args), ctx[5]649#define ___bpf_ctx_cast7(x, args...) ___bpf_ctx_cast6(args), ctx[6]650#define ___bpf_ctx_cast8(x, args...) ___bpf_ctx_cast7(args), ctx[7]651#define ___bpf_ctx_cast9(x, args...) ___bpf_ctx_cast8(args), ctx[8]652#define ___bpf_ctx_cast10(x, args...) ___bpf_ctx_cast9(args), ctx[9]653#define ___bpf_ctx_cast11(x, args...) ___bpf_ctx_cast10(args), ctx[10]654#define ___bpf_ctx_cast12(x, args...) ___bpf_ctx_cast11(args), ctx[11]655#define ___bpf_ctx_cast(args...) ___bpf_apply(___bpf_ctx_cast, ___bpf_narg(args))(args)656 657/*658 * BPF_PROG is a convenience wrapper for generic tp_btf/fentry/fexit and659 * similar kinds of BPF programs, that accept input arguments as a single660 * pointer to untyped u64 array, where each u64 can actually be a typed661 * pointer or integer of different size. Instead of requiring user to write662 * manual casts and work with array elements by index, BPF_PROG macro663 * allows user to declare a list of named and typed input arguments in the664 * same syntax as for normal C function. All the casting is hidden and665 * performed transparently, while user code can just assume working with666 * function arguments of specified type and name.667 *668 * Original raw context argument is preserved as well as 'ctx' argument.669 * This is useful when using BPF helpers that expect original context670 * as one of the parameters (e.g., for bpf_perf_event_output()).671 */672#define BPF_PROG(name, args...) \673name(unsigned long long *ctx); \674static __always_inline typeof(name(0)) \675____##name(unsigned long long *ctx, ##args); \676typeof(name(0)) name(unsigned long long *ctx) \677{ \678 _Pragma("GCC diagnostic push") \679 _Pragma("GCC diagnostic ignored \"-Wint-conversion\"") \680 return ____##name(___bpf_ctx_cast(args)); \681 _Pragma("GCC diagnostic pop") \682} \683static __always_inline typeof(name(0)) \684____##name(unsigned long long *ctx, ##args)685 686#ifndef ___bpf_nth2687#define ___bpf_nth2(_, _1, _2, _3, _4, _5, _6, _7, _8, _9, _10, _11, _12, _13, \688 _14, _15, _16, _17, _18, _19, _20, _21, _22, _23, _24, N, ...) N689#endif690#ifndef ___bpf_narg2691#define ___bpf_narg2(...) \692 ___bpf_nth2(_, ##__VA_ARGS__, 12, 12, 11, 11, 10, 10, 9, 9, 8, 8, 7, 7, \693 6, 6, 5, 5, 4, 4, 3, 3, 2, 2, 1, 1, 0)694#endif695 696#define ___bpf_treg_cnt(t) \697 __builtin_choose_expr(sizeof(t) == 1, 1, \698 __builtin_choose_expr(sizeof(t) == 2, 1, \699 __builtin_choose_expr(sizeof(t) == 4, 1, \700 __builtin_choose_expr(sizeof(t) == 8, 1, \701 __builtin_choose_expr(sizeof(t) == 16, 2, \702 (void)0)))))703 704#define ___bpf_reg_cnt0() (0)705#define ___bpf_reg_cnt1(t, x) (___bpf_reg_cnt0() + ___bpf_treg_cnt(t))706#define ___bpf_reg_cnt2(t, x, args...) (___bpf_reg_cnt1(args) + ___bpf_treg_cnt(t))707#define ___bpf_reg_cnt3(t, x, args...) (___bpf_reg_cnt2(args) + ___bpf_treg_cnt(t))708#define ___bpf_reg_cnt4(t, x, args...) (___bpf_reg_cnt3(args) + ___bpf_treg_cnt(t))709#define ___bpf_reg_cnt5(t, x, args...) (___bpf_reg_cnt4(args) + ___bpf_treg_cnt(t))710#define ___bpf_reg_cnt6(t, x, args...) (___bpf_reg_cnt5(args) + ___bpf_treg_cnt(t))711#define ___bpf_reg_cnt7(t, x, args...) (___bpf_reg_cnt6(args) + ___bpf_treg_cnt(t))712#define ___bpf_reg_cnt8(t, x, args...) (___bpf_reg_cnt7(args) + ___bpf_treg_cnt(t))713#define ___bpf_reg_cnt9(t, x, args...) (___bpf_reg_cnt8(args) + ___bpf_treg_cnt(t))714#define ___bpf_reg_cnt10(t, x, args...) (___bpf_reg_cnt9(args) + ___bpf_treg_cnt(t))715#define ___bpf_reg_cnt11(t, x, args...) (___bpf_reg_cnt10(args) + ___bpf_treg_cnt(t))716#define ___bpf_reg_cnt12(t, x, args...) (___bpf_reg_cnt11(args) + ___bpf_treg_cnt(t))717#define ___bpf_reg_cnt(args...) ___bpf_apply(___bpf_reg_cnt, ___bpf_narg2(args))(args)718 719#define ___bpf_union_arg(t, x, n) \720 __builtin_choose_expr(sizeof(t) == 1, ({ union { __u8 z[1]; t x; } ___t = { .z = {ctx[n]}}; ___t.x; }), \721 __builtin_choose_expr(sizeof(t) == 2, ({ union { __u16 z[1]; t x; } ___t = { .z = {ctx[n]} }; ___t.x; }), \722 __builtin_choose_expr(sizeof(t) == 4, ({ union { __u32 z[1]; t x; } ___t = { .z = {ctx[n]} }; ___t.x; }), \723 __builtin_choose_expr(sizeof(t) == 8, ({ union { __u64 z[1]; t x; } ___t = {.z = {ctx[n]} }; ___t.x; }), \724 __builtin_choose_expr(sizeof(t) == 16, ({ union { __u64 z[2]; t x; } ___t = {.z = {ctx[n], ctx[n + 1]} }; ___t.x; }), \725 (void)0)))))726 727#define ___bpf_ctx_arg0(n, args...)728#define ___bpf_ctx_arg1(n, t, x) , ___bpf_union_arg(t, x, n - ___bpf_reg_cnt1(t, x))729#define ___bpf_ctx_arg2(n, t, x, args...) , ___bpf_union_arg(t, x, n - ___bpf_reg_cnt2(t, x, args)) ___bpf_ctx_arg1(n, args)730#define ___bpf_ctx_arg3(n, t, x, args...) , ___bpf_union_arg(t, x, n - ___bpf_reg_cnt3(t, x, args)) ___bpf_ctx_arg2(n, args)731#define ___bpf_ctx_arg4(n, t, x, args...) , ___bpf_union_arg(t, x, n - ___bpf_reg_cnt4(t, x, args)) ___bpf_ctx_arg3(n, args)732#define ___bpf_ctx_arg5(n, t, x, args...) , ___bpf_union_arg(t, x, n - ___bpf_reg_cnt5(t, x, args)) ___bpf_ctx_arg4(n, args)733#define ___bpf_ctx_arg6(n, t, x, args...) , ___bpf_union_arg(t, x, n - ___bpf_reg_cnt6(t, x, args)) ___bpf_ctx_arg5(n, args)734#define ___bpf_ctx_arg7(n, t, x, args...) , ___bpf_union_arg(t, x, n - ___bpf_reg_cnt7(t, x, args)) ___bpf_ctx_arg6(n, args)735#define ___bpf_ctx_arg8(n, t, x, args...) , ___bpf_union_arg(t, x, n - ___bpf_reg_cnt8(t, x, args)) ___bpf_ctx_arg7(n, args)736#define ___bpf_ctx_arg9(n, t, x, args...) , ___bpf_union_arg(t, x, n - ___bpf_reg_cnt9(t, x, args)) ___bpf_ctx_arg8(n, args)737#define ___bpf_ctx_arg10(n, t, x, args...) , ___bpf_union_arg(t, x, n - ___bpf_reg_cnt10(t, x, args)) ___bpf_ctx_arg9(n, args)738#define ___bpf_ctx_arg11(n, t, x, args...) , ___bpf_union_arg(t, x, n - ___bpf_reg_cnt11(t, x, args)) ___bpf_ctx_arg10(n, args)739#define ___bpf_ctx_arg12(n, t, x, args...) , ___bpf_union_arg(t, x, n - ___bpf_reg_cnt12(t, x, args)) ___bpf_ctx_arg11(n, args)740#define ___bpf_ctx_arg(args...) ___bpf_apply(___bpf_ctx_arg, ___bpf_narg2(args))(___bpf_reg_cnt(args), args)741 742#define ___bpf_ctx_decl0()743#define ___bpf_ctx_decl1(t, x) , t x744#define ___bpf_ctx_decl2(t, x, args...) , t x ___bpf_ctx_decl1(args)745#define ___bpf_ctx_decl3(t, x, args...) , t x ___bpf_ctx_decl2(args)746#define ___bpf_ctx_decl4(t, x, args...) , t x ___bpf_ctx_decl3(args)747#define ___bpf_ctx_decl5(t, x, args...) , t x ___bpf_ctx_decl4(args)748#define ___bpf_ctx_decl6(t, x, args...) , t x ___bpf_ctx_decl5(args)749#define ___bpf_ctx_decl7(t, x, args...) , t x ___bpf_ctx_decl6(args)750#define ___bpf_ctx_decl8(t, x, args...) , t x ___bpf_ctx_decl7(args)751#define ___bpf_ctx_decl9(t, x, args...) , t x ___bpf_ctx_decl8(args)752#define ___bpf_ctx_decl10(t, x, args...) , t x ___bpf_ctx_decl9(args)753#define ___bpf_ctx_decl11(t, x, args...) , t x ___bpf_ctx_decl10(args)754#define ___bpf_ctx_decl12(t, x, args...) , t x ___bpf_ctx_decl11(args)755#define ___bpf_ctx_decl(args...) ___bpf_apply(___bpf_ctx_decl, ___bpf_narg2(args))(args)756 757/*758 * BPF_PROG2 is an enhanced version of BPF_PROG in order to handle struct759 * arguments. Since each struct argument might take one or two u64 values760 * in the trampoline stack, argument type size is needed to place proper number761 * of u64 values for each argument. Therefore, BPF_PROG2 has different762 * syntax from BPF_PROG. For example, for the following BPF_PROG syntax:763 *764 * int BPF_PROG(test2, int a, int b) { ... }765 *766 * the corresponding BPF_PROG2 syntax is:767 *768 * int BPF_PROG2(test2, int, a, int, b) { ... }769 *770 * where type and the corresponding argument name are separated by comma.771 *772 * Use BPF_PROG2 macro if one of the arguments might be a struct/union larger773 * than 8 bytes:774 *775 * int BPF_PROG2(test_struct_arg, struct bpf_testmod_struct_arg_1, a, int, b,776 * int, c, int, d, struct bpf_testmod_struct_arg_2, e, int, ret)777 * {778 * // access a, b, c, d, e, and ret directly779 * ...780 * }781 */782#define BPF_PROG2(name, args...) \783name(unsigned long long *ctx); \784static __always_inline typeof(name(0)) \785____##name(unsigned long long *ctx ___bpf_ctx_decl(args)); \786typeof(name(0)) name(unsigned long long *ctx) \787{ \788 return ____##name(ctx ___bpf_ctx_arg(args)); \789} \790static __always_inline typeof(name(0)) \791____##name(unsigned long long *ctx ___bpf_ctx_decl(args))792 793struct pt_regs;794 795#define ___bpf_kprobe_args0() ctx796#define ___bpf_kprobe_args1(x) ___bpf_kprobe_args0(), (unsigned long long)PT_REGS_PARM1(ctx)797#define ___bpf_kprobe_args2(x, args...) ___bpf_kprobe_args1(args), (unsigned long long)PT_REGS_PARM2(ctx)798#define ___bpf_kprobe_args3(x, args...) ___bpf_kprobe_args2(args), (unsigned long long)PT_REGS_PARM3(ctx)799#define ___bpf_kprobe_args4(x, args...) ___bpf_kprobe_args3(args), (unsigned long long)PT_REGS_PARM4(ctx)800#define ___bpf_kprobe_args5(x, args...) ___bpf_kprobe_args4(args), (unsigned long long)PT_REGS_PARM5(ctx)801#define ___bpf_kprobe_args6(x, args...) ___bpf_kprobe_args5(args), (unsigned long long)PT_REGS_PARM6(ctx)802#define ___bpf_kprobe_args7(x, args...) ___bpf_kprobe_args6(args), (unsigned long long)PT_REGS_PARM7(ctx)803#define ___bpf_kprobe_args8(x, args...) ___bpf_kprobe_args7(args), (unsigned long long)PT_REGS_PARM8(ctx)804#define ___bpf_kprobe_args(args...) ___bpf_apply(___bpf_kprobe_args, ___bpf_narg(args))(args)805 806/*807 * BPF_KPROBE serves the same purpose for kprobes as BPF_PROG for808 * tp_btf/fentry/fexit BPF programs. It hides the underlying platform-specific809 * low-level way of getting kprobe input arguments from struct pt_regs, and810 * provides a familiar typed and named function arguments syntax and811 * semantics of accessing kprobe input parameters.812 *813 * Original struct pt_regs* context is preserved as 'ctx' argument. This might814 * be necessary when using BPF helpers like bpf_perf_event_output().815 */816#define BPF_KPROBE(name, args...) \817name(struct pt_regs *ctx); \818static __always_inline typeof(name(0)) \819____##name(struct pt_regs *ctx, ##args); \820typeof(name(0)) name(struct pt_regs *ctx) \821{ \822 _Pragma("GCC diagnostic push") \823 _Pragma("GCC diagnostic ignored \"-Wint-conversion\"") \824 return ____##name(___bpf_kprobe_args(args)); \825 _Pragma("GCC diagnostic pop") \826} \827static __always_inline typeof(name(0)) \828____##name(struct pt_regs *ctx, ##args)829 830#define ___bpf_kretprobe_args0() ctx831#define ___bpf_kretprobe_args1(x) ___bpf_kretprobe_args0(), (unsigned long long)PT_REGS_RC(ctx)832#define ___bpf_kretprobe_args(args...) ___bpf_apply(___bpf_kretprobe_args, ___bpf_narg(args))(args)833 834/*835 * BPF_KRETPROBE is similar to BPF_KPROBE, except, it only provides optional836 * return value (in addition to `struct pt_regs *ctx`), but no input837 * arguments, because they will be clobbered by the time probed function838 * returns.839 */840#define BPF_KRETPROBE(name, args...) \841name(struct pt_regs *ctx); \842static __always_inline typeof(name(0)) \843____##name(struct pt_regs *ctx, ##args); \844typeof(name(0)) name(struct pt_regs *ctx) \845{ \846 _Pragma("GCC diagnostic push") \847 _Pragma("GCC diagnostic ignored \"-Wint-conversion\"") \848 return ____##name(___bpf_kretprobe_args(args)); \849 _Pragma("GCC diagnostic pop") \850} \851static __always_inline typeof(name(0)) ____##name(struct pt_regs *ctx, ##args)852 853/* If kernel has CONFIG_ARCH_HAS_SYSCALL_WRAPPER, read pt_regs directly */854#define ___bpf_syscall_args0() ctx855#define ___bpf_syscall_args1(x) ___bpf_syscall_args0(), (unsigned long long)PT_REGS_PARM1_SYSCALL(regs)856#define ___bpf_syscall_args2(x, args...) ___bpf_syscall_args1(args), (unsigned long long)PT_REGS_PARM2_SYSCALL(regs)857#define ___bpf_syscall_args3(x, args...) ___bpf_syscall_args2(args), (unsigned long long)PT_REGS_PARM3_SYSCALL(regs)858#define ___bpf_syscall_args4(x, args...) ___bpf_syscall_args3(args), (unsigned long long)PT_REGS_PARM4_SYSCALL(regs)859#define ___bpf_syscall_args5(x, args...) ___bpf_syscall_args4(args), (unsigned long long)PT_REGS_PARM5_SYSCALL(regs)860#define ___bpf_syscall_args6(x, args...) ___bpf_syscall_args5(args), (unsigned long long)PT_REGS_PARM6_SYSCALL(regs)861#define ___bpf_syscall_args7(x, args...) ___bpf_syscall_args6(args), (unsigned long long)PT_REGS_PARM7_SYSCALL(regs)862#define ___bpf_syscall_args(args...) ___bpf_apply(___bpf_syscall_args, ___bpf_narg(args))(args)863 864/* If kernel doesn't have CONFIG_ARCH_HAS_SYSCALL_WRAPPER, we have to BPF_CORE_READ from pt_regs */865#define ___bpf_syswrap_args0() ctx866#define ___bpf_syswrap_args1(x) ___bpf_syswrap_args0(), (unsigned long long)PT_REGS_PARM1_CORE_SYSCALL(regs)867#define ___bpf_syswrap_args2(x, args...) ___bpf_syswrap_args1(args), (unsigned long long)PT_REGS_PARM2_CORE_SYSCALL(regs)868#define ___bpf_syswrap_args3(x, args...) ___bpf_syswrap_args2(args), (unsigned long long)PT_REGS_PARM3_CORE_SYSCALL(regs)869#define ___bpf_syswrap_args4(x, args...) ___bpf_syswrap_args3(args), (unsigned long long)PT_REGS_PARM4_CORE_SYSCALL(regs)870#define ___bpf_syswrap_args5(x, args...) ___bpf_syswrap_args4(args), (unsigned long long)PT_REGS_PARM5_CORE_SYSCALL(regs)871#define ___bpf_syswrap_args6(x, args...) ___bpf_syswrap_args5(args), (unsigned long long)PT_REGS_PARM6_CORE_SYSCALL(regs)872#define ___bpf_syswrap_args7(x, args...) ___bpf_syswrap_args6(args), (unsigned long long)PT_REGS_PARM7_CORE_SYSCALL(regs)873#define ___bpf_syswrap_args(args...) ___bpf_apply(___bpf_syswrap_args, ___bpf_narg(args))(args)874 875/*876 * BPF_KSYSCALL is a variant of BPF_KPROBE, which is intended for877 * tracing syscall functions, like __x64_sys_close. It hides the underlying878 * platform-specific low-level way of getting syscall input arguments from879 * struct pt_regs, and provides a familiar typed and named function arguments880 * syntax and semantics of accessing syscall input parameters.881 *882 * Original struct pt_regs * context is preserved as 'ctx' argument. This might883 * be necessary when using BPF helpers like bpf_perf_event_output().884 *885 * At the moment BPF_KSYSCALL does not transparently handle all the calling886 * convention quirks for the following syscalls:887 *888 * - mmap(): __ARCH_WANT_SYS_OLD_MMAP.889 * - clone(): CONFIG_CLONE_BACKWARDS, CONFIG_CLONE_BACKWARDS2 and890 * CONFIG_CLONE_BACKWARDS3.891 * - socket-related syscalls: __ARCH_WANT_SYS_SOCKETCALL.892 * - compat syscalls.893 *894 * This may or may not change in the future. User needs to take extra measures895 * to handle such quirks explicitly, if necessary.896 *897 * This macro relies on BPF CO-RE support and virtual __kconfig externs.898 */899#define BPF_KSYSCALL(name, args...) \900name(struct pt_regs *ctx); \901extern _Bool LINUX_HAS_SYSCALL_WRAPPER __kconfig; \902static __always_inline typeof(name(0)) \903____##name(struct pt_regs *ctx, ##args); \904typeof(name(0)) name(struct pt_regs *ctx) \905{ \906 struct pt_regs *regs = LINUX_HAS_SYSCALL_WRAPPER \907 ? (struct pt_regs *)PT_REGS_PARM1(ctx) \908 : ctx; \909 _Pragma("GCC diagnostic push") \910 _Pragma("GCC diagnostic ignored \"-Wint-conversion\"") \911 if (LINUX_HAS_SYSCALL_WRAPPER) \912 return ____##name(___bpf_syswrap_args(args)); \913 else \914 return ____##name(___bpf_syscall_args(args)); \915 _Pragma("GCC diagnostic pop") \916} \917static __always_inline typeof(name(0)) \918____##name(struct pt_regs *ctx, ##args)919 920#define BPF_KPROBE_SYSCALL BPF_KSYSCALL921 922/* BPF_UPROBE and BPF_URETPROBE are identical to BPF_KPROBE and BPF_KRETPROBE,923 * but are named way less confusingly for SEC("uprobe") and SEC("uretprobe")924 * use cases.925 */926#define BPF_UPROBE(name, args...) BPF_KPROBE(name, ##args)927#define BPF_URETPROBE(name, args...) BPF_KRETPROBE(name, ##args)928 929#endif930