brintos / linux-shallow public Read only

1// SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause)2 3/*4 * Common eBPF ELF object loading operations.5 *6 * Copyright (C) 2013-2015 Alexei Starovoitov <ast@kernel.org>7 * Copyright (C) 2015 Wang Nan <wangnan0@huawei.com>8 * Copyright (C) 2015 Huawei Inc.9 * Copyright (C) 2017 Nicira, Inc.10 * Copyright (C) 2019 Isovalent, Inc.11 */12 13#ifndef _GNU_SOURCE14#define _GNU_SOURCE15#endif16#include <stdlib.h>17#include <stdio.h>18#include <stdarg.h>19#include <libgen.h>20#include <inttypes.h>21#include <limits.h>22#include <string.h>23#include <unistd.h>24#include <endian.h>25#include <fcntl.h>26#include <errno.h>27#include <ctype.h>28#include <asm/unistd.h>29#include <linux/err.h>30#include <linux/kernel.h>31#include <linux/bpf.h>32#include <linux/btf.h>33#include <linux/filter.h>34#include <linux/limits.h>35#include <linux/perf_event.h>36#include <linux/bpf_perf_event.h>37#include <linux/ring_buffer.h>38#include <sys/epoll.h>39#include <sys/ioctl.h>40#include <sys/mman.h>41#include <sys/stat.h>42#include <sys/types.h>43#include <sys/vfs.h>44#include <sys/utsname.h>45#include <sys/resource.h>46#include <libelf.h>47#include <gelf.h>48#include <zlib.h>49 50#include "libbpf.h"51#include "bpf.h"52#include "btf.h"53#include "str_error.h"54#include "libbpf_internal.h"55#include "hashmap.h"56#include "bpf_gen_internal.h"57#include "zip.h"58 59#ifndef BPF_FS_MAGIC60#define BPF_FS_MAGIC		0xcafe4a1161#endif62 63#define BPF_FS_DEFAULT_PATH "/sys/fs/bpf"64 65#define BPF_INSN_SZ (sizeof(struct bpf_insn))66 67/* vsprintf() in __base_pr() uses nonliteral format string. It may break68 * compilation if user enables corresponding warning. Disable it explicitly.69 */70#pragma GCC diagnostic ignored "-Wformat-nonliteral"71 72#define __printf(a, b)	__attribute__((format(printf, a, b)))73 74static struct bpf_map *bpf_object__add_map(struct bpf_object *obj);75static bool prog_is_subprog(const struct bpf_object *obj, const struct bpf_program *prog);76static int map_set_def_max_entries(struct bpf_map *map);77 78static const char * const attach_type_name[] = {79	[BPF_CGROUP_INET_INGRESS]	= "cgroup_inet_ingress",80	[BPF_CGROUP_INET_EGRESS]	= "cgroup_inet_egress",81	[BPF_CGROUP_INET_SOCK_CREATE]	= "cgroup_inet_sock_create",82	[BPF_CGROUP_INET_SOCK_RELEASE]	= "cgroup_inet_sock_release",83	[BPF_CGROUP_SOCK_OPS]		= "cgroup_sock_ops",84	[BPF_CGROUP_DEVICE]		= "cgroup_device",85	[BPF_CGROUP_INET4_BIND]		= "cgroup_inet4_bind",86	[BPF_CGROUP_INET6_BIND]		= "cgroup_inet6_bind",87	[BPF_CGROUP_INET4_CONNECT]	= "cgroup_inet4_connect",88	[BPF_CGROUP_INET6_CONNECT]	= "cgroup_inet6_connect",89	[BPF_CGROUP_UNIX_CONNECT]       = "cgroup_unix_connect",90	[BPF_CGROUP_INET4_POST_BIND]	= "cgroup_inet4_post_bind",91	[BPF_CGROUP_INET6_POST_BIND]	= "cgroup_inet6_post_bind",92	[BPF_CGROUP_INET4_GETPEERNAME]	= "cgroup_inet4_getpeername",93	[BPF_CGROUP_INET6_GETPEERNAME]	= "cgroup_inet6_getpeername",94	[BPF_CGROUP_UNIX_GETPEERNAME]	= "cgroup_unix_getpeername",95	[BPF_CGROUP_INET4_GETSOCKNAME]	= "cgroup_inet4_getsockname",96	[BPF_CGROUP_INET6_GETSOCKNAME]	= "cgroup_inet6_getsockname",97	[BPF_CGROUP_UNIX_GETSOCKNAME]	= "cgroup_unix_getsockname",98	[BPF_CGROUP_UDP4_SENDMSG]	= "cgroup_udp4_sendmsg",99	[BPF_CGROUP_UDP6_SENDMSG]	= "cgroup_udp6_sendmsg",100	[BPF_CGROUP_UNIX_SENDMSG]	= "cgroup_unix_sendmsg",101	[BPF_CGROUP_SYSCTL]		= "cgroup_sysctl",102	[BPF_CGROUP_UDP4_RECVMSG]	= "cgroup_udp4_recvmsg",103	[BPF_CGROUP_UDP6_RECVMSG]	= "cgroup_udp6_recvmsg",104	[BPF_CGROUP_UNIX_RECVMSG]	= "cgroup_unix_recvmsg",105	[BPF_CGROUP_GETSOCKOPT]		= "cgroup_getsockopt",106	[BPF_CGROUP_SETSOCKOPT]		= "cgroup_setsockopt",107	[BPF_SK_SKB_STREAM_PARSER]	= "sk_skb_stream_parser",108	[BPF_SK_SKB_STREAM_VERDICT]	= "sk_skb_stream_verdict",109	[BPF_SK_SKB_VERDICT]		= "sk_skb_verdict",110	[BPF_SK_MSG_VERDICT]		= "sk_msg_verdict",111	[BPF_LIRC_MODE2]		= "lirc_mode2",112	[BPF_FLOW_DISSECTOR]		= "flow_dissector",113	[BPF_TRACE_RAW_TP]		= "trace_raw_tp",114	[BPF_TRACE_FENTRY]		= "trace_fentry",115	[BPF_TRACE_FEXIT]		= "trace_fexit",116	[BPF_MODIFY_RETURN]		= "modify_return",117	[BPF_LSM_MAC]			= "lsm_mac",118	[BPF_LSM_CGROUP]		= "lsm_cgroup",119	[BPF_SK_LOOKUP]			= "sk_lookup",120	[BPF_TRACE_ITER]		= "trace_iter",121	[BPF_XDP_DEVMAP]		= "xdp_devmap",122	[BPF_XDP_CPUMAP]		= "xdp_cpumap",123	[BPF_XDP]			= "xdp",124	[BPF_SK_REUSEPORT_SELECT]	= "sk_reuseport_select",125	[BPF_SK_REUSEPORT_SELECT_OR_MIGRATE]	= "sk_reuseport_select_or_migrate",126	[BPF_PERF_EVENT]		= "perf_event",127	[BPF_TRACE_KPROBE_MULTI]	= "trace_kprobe_multi",128	[BPF_STRUCT_OPS]		= "struct_ops",129	[BPF_NETFILTER]			= "netfilter",130	[BPF_TCX_INGRESS]		= "tcx_ingress",131	[BPF_TCX_EGRESS]		= "tcx_egress",132	[BPF_TRACE_UPROBE_MULTI]	= "trace_uprobe_multi",133	[BPF_NETKIT_PRIMARY]		= "netkit_primary",134	[BPF_NETKIT_PEER]		= "netkit_peer",135	[BPF_TRACE_KPROBE_SESSION]	= "trace_kprobe_session",136};137 138static const char * const link_type_name[] = {139	[BPF_LINK_TYPE_UNSPEC]			= "unspec",140	[BPF_LINK_TYPE_RAW_TRACEPOINT]		= "raw_tracepoint",141	[BPF_LINK_TYPE_TRACING]			= "tracing",142	[BPF_LINK_TYPE_CGROUP]			= "cgroup",143	[BPF_LINK_TYPE_ITER]			= "iter",144	[BPF_LINK_TYPE_NETNS]			= "netns",145	[BPF_LINK_TYPE_XDP]			= "xdp",146	[BPF_LINK_TYPE_PERF_EVENT]		= "perf_event",147	[BPF_LINK_TYPE_KPROBE_MULTI]		= "kprobe_multi",148	[BPF_LINK_TYPE_STRUCT_OPS]		= "struct_ops",149	[BPF_LINK_TYPE_NETFILTER]		= "netfilter",150	[BPF_LINK_TYPE_TCX]			= "tcx",151	[BPF_LINK_TYPE_UPROBE_MULTI]		= "uprobe_multi",152	[BPF_LINK_TYPE_NETKIT]			= "netkit",153	[BPF_LINK_TYPE_SOCKMAP]			= "sockmap",154};155 156static const char * const map_type_name[] = {157	[BPF_MAP_TYPE_UNSPEC]			= "unspec",158	[BPF_MAP_TYPE_HASH]			= "hash",159	[BPF_MAP_TYPE_ARRAY]			= "array",160	[BPF_MAP_TYPE_PROG_ARRAY]		= "prog_array",161	[BPF_MAP_TYPE_PERF_EVENT_ARRAY]		= "perf_event_array",162	[BPF_MAP_TYPE_PERCPU_HASH]		= "percpu_hash",163	[BPF_MAP_TYPE_PERCPU_ARRAY]		= "percpu_array",164	[BPF_MAP_TYPE_STACK_TRACE]		= "stack_trace",165	[BPF_MAP_TYPE_CGROUP_ARRAY]		= "cgroup_array",166	[BPF_MAP_TYPE_LRU_HASH]			= "lru_hash",167	[BPF_MAP_TYPE_LRU_PERCPU_HASH]		= "lru_percpu_hash",168	[BPF_MAP_TYPE_LPM_TRIE]			= "lpm_trie",169	[BPF_MAP_TYPE_ARRAY_OF_MAPS]		= "array_of_maps",170	[BPF_MAP_TYPE_HASH_OF_MAPS]		= "hash_of_maps",171	[BPF_MAP_TYPE_DEVMAP]			= "devmap",172	[BPF_MAP_TYPE_DEVMAP_HASH]		= "devmap_hash",173	[BPF_MAP_TYPE_SOCKMAP]			= "sockmap",174	[BPF_MAP_TYPE_CPUMAP]			= "cpumap",175	[BPF_MAP_TYPE_XSKMAP]			= "xskmap",176	[BPF_MAP_TYPE_SOCKHASH]			= "sockhash",177	[BPF_MAP_TYPE_CGROUP_STORAGE]		= "cgroup_storage",178	[BPF_MAP_TYPE_REUSEPORT_SOCKARRAY]	= "reuseport_sockarray",179	[BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE]	= "percpu_cgroup_storage",180	[BPF_MAP_TYPE_QUEUE]			= "queue",181	[BPF_MAP_TYPE_STACK]			= "stack",182	[BPF_MAP_TYPE_SK_STORAGE]		= "sk_storage",183	[BPF_MAP_TYPE_STRUCT_OPS]		= "struct_ops",184	[BPF_MAP_TYPE_RINGBUF]			= "ringbuf",185	[BPF_MAP_TYPE_INODE_STORAGE]		= "inode_storage",186	[BPF_MAP_TYPE_TASK_STORAGE]		= "task_storage",187	[BPF_MAP_TYPE_BLOOM_FILTER]		= "bloom_filter",188	[BPF_MAP_TYPE_USER_RINGBUF]             = "user_ringbuf",189	[BPF_MAP_TYPE_CGRP_STORAGE]		= "cgrp_storage",190	[BPF_MAP_TYPE_ARENA]			= "arena",191};192 193static const char * const prog_type_name[] = {194	[BPF_PROG_TYPE_UNSPEC]			= "unspec",195	[BPF_PROG_TYPE_SOCKET_FILTER]		= "socket_filter",196	[BPF_PROG_TYPE_KPROBE]			= "kprobe",197	[BPF_PROG_TYPE_SCHED_CLS]		= "sched_cls",198	[BPF_PROG_TYPE_SCHED_ACT]		= "sched_act",199	[BPF_PROG_TYPE_TRACEPOINT]		= "tracepoint",200	[BPF_PROG_TYPE_XDP]			= "xdp",201	[BPF_PROG_TYPE_PERF_EVENT]		= "perf_event",202	[BPF_PROG_TYPE_CGROUP_SKB]		= "cgroup_skb",203	[BPF_PROG_TYPE_CGROUP_SOCK]		= "cgroup_sock",204	[BPF_PROG_TYPE_LWT_IN]			= "lwt_in",205	[BPF_PROG_TYPE_LWT_OUT]			= "lwt_out",206	[BPF_PROG_TYPE_LWT_XMIT]		= "lwt_xmit",207	[BPF_PROG_TYPE_SOCK_OPS]		= "sock_ops",208	[BPF_PROG_TYPE_SK_SKB]			= "sk_skb",209	[BPF_PROG_TYPE_CGROUP_DEVICE]		= "cgroup_device",210	[BPF_PROG_TYPE_SK_MSG]			= "sk_msg",211	[BPF_PROG_TYPE_RAW_TRACEPOINT]		= "raw_tracepoint",212	[BPF_PROG_TYPE_CGROUP_SOCK_ADDR]	= "cgroup_sock_addr",213	[BPF_PROG_TYPE_LWT_SEG6LOCAL]		= "lwt_seg6local",214	[BPF_PROG_TYPE_LIRC_MODE2]		= "lirc_mode2",215	[BPF_PROG_TYPE_SK_REUSEPORT]		= "sk_reuseport",216	[BPF_PROG_TYPE_FLOW_DISSECTOR]		= "flow_dissector",217	[BPF_PROG_TYPE_CGROUP_SYSCTL]		= "cgroup_sysctl",218	[BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE]	= "raw_tracepoint_writable",219	[BPF_PROG_TYPE_CGROUP_SOCKOPT]		= "cgroup_sockopt",220	[BPF_PROG_TYPE_TRACING]			= "tracing",221	[BPF_PROG_TYPE_STRUCT_OPS]		= "struct_ops",222	[BPF_PROG_TYPE_EXT]			= "ext",223	[BPF_PROG_TYPE_LSM]			= "lsm",224	[BPF_PROG_TYPE_SK_LOOKUP]		= "sk_lookup",225	[BPF_PROG_TYPE_SYSCALL]			= "syscall",226	[BPF_PROG_TYPE_NETFILTER]		= "netfilter",227};228 229static int __base_pr(enum libbpf_print_level level, const char *format,230		     va_list args)231{232	const char *env_var = "LIBBPF_LOG_LEVEL";233	static enum libbpf_print_level min_level = LIBBPF_INFO;234	static bool initialized;235 236	if (!initialized) {237		char *verbosity;238 239		initialized = true;240		verbosity = getenv(env_var);241		if (verbosity) {242			if (strcasecmp(verbosity, "warn") == 0)243				min_level = LIBBPF_WARN;244			else if (strcasecmp(verbosity, "debug") == 0)245				min_level = LIBBPF_DEBUG;246			else if (strcasecmp(verbosity, "info") == 0)247				min_level = LIBBPF_INFO;248			else249				fprintf(stderr, "libbpf: unrecognized '%s' envvar value: '%s', should be one of 'warn', 'debug', or 'info'.\n",250					env_var, verbosity);251		}252	}253 254	/* if too verbose, skip logging  */255	if (level > min_level)256		return 0;257 258	return vfprintf(stderr, format, args);259}260 261static libbpf_print_fn_t __libbpf_pr = __base_pr;262 263libbpf_print_fn_t libbpf_set_print(libbpf_print_fn_t fn)264{265	libbpf_print_fn_t old_print_fn;266 267	old_print_fn = __atomic_exchange_n(&__libbpf_pr, fn, __ATOMIC_RELAXED);268 269	return old_print_fn;270}271 272__printf(2, 3)273void libbpf_print(enum libbpf_print_level level, const char *format, ...)274{275	va_list args;276	int old_errno;277	libbpf_print_fn_t print_fn;278 279	print_fn = __atomic_load_n(&__libbpf_pr, __ATOMIC_RELAXED);280	if (!print_fn)281		return;282 283	old_errno = errno;284 285	va_start(args, format);286	__libbpf_pr(level, format, args);287	va_end(args);288 289	errno = old_errno;290}291 292static void pr_perm_msg(int err)293{294	struct rlimit limit;295	char buf[100];296 297	if (err != -EPERM || geteuid() != 0)298		return;299 300	err = getrlimit(RLIMIT_MEMLOCK, &limit);301	if (err)302		return;303 304	if (limit.rlim_cur == RLIM_INFINITY)305		return;306 307	if (limit.rlim_cur < 1024)308		snprintf(buf, sizeof(buf), "%zu bytes", (size_t)limit.rlim_cur);309	else if (limit.rlim_cur < 1024*1024)310		snprintf(buf, sizeof(buf), "%.1f KiB", (double)limit.rlim_cur / 1024);311	else312		snprintf(buf, sizeof(buf), "%.1f MiB", (double)limit.rlim_cur / (1024*1024));313 314	pr_warn("permission error while running as root; try raising 'ulimit -l'? current value: %s\n",315		buf);316}317 318#define STRERR_BUFSIZE  128319 320/* Copied from tools/perf/util/util.h */321#ifndef zfree322# define zfree(ptr) ({ free(*ptr); *ptr = NULL; })323#endif324 325#ifndef zclose326# define zclose(fd) ({			\327	int ___err = 0;			\328	if ((fd) >= 0)			\329		___err = close((fd));	\330	fd = -1;			\331	___err; })332#endif333 334static inline __u64 ptr_to_u64(const void *ptr)335{336	return (__u64) (unsigned long) ptr;337}338 339int libbpf_set_strict_mode(enum libbpf_strict_mode mode)340{341	/* as of v1.0 libbpf_set_strict_mode() is a no-op */342	return 0;343}344 345__u32 libbpf_major_version(void)346{347	return LIBBPF_MAJOR_VERSION;348}349 350__u32 libbpf_minor_version(void)351{352	return LIBBPF_MINOR_VERSION;353}354 355const char *libbpf_version_string(void)356{357#define __S(X) #X358#define _S(X) __S(X)359	return  "v" _S(LIBBPF_MAJOR_VERSION) "." _S(LIBBPF_MINOR_VERSION);360#undef _S361#undef __S362}363 364enum reloc_type {365	RELO_LD64,366	RELO_CALL,367	RELO_DATA,368	RELO_EXTERN_LD64,369	RELO_EXTERN_CALL,370	RELO_SUBPROG_ADDR,371	RELO_CORE,372};373 374struct reloc_desc {375	enum reloc_type type;376	int insn_idx;377	union {378		const struct bpf_core_relo *core_relo; /* used when type == RELO_CORE */379		struct {380			int map_idx;381			int sym_off;382			int ext_idx;383		};384	};385};386 387/* stored as sec_def->cookie for all libbpf-supported SEC()s */388enum sec_def_flags {389	SEC_NONE = 0,390	/* expected_attach_type is optional, if kernel doesn't support that */391	SEC_EXP_ATTACH_OPT = 1,392	/* legacy, only used by libbpf_get_type_names() and393	 * libbpf_attach_type_by_name(), not used by libbpf itself at all.394	 * This used to be associated with cgroup (and few other) BPF programs395	 * that were attachable through BPF_PROG_ATTACH command. Pretty396	 * meaningless nowadays, though.397	 */398	SEC_ATTACHABLE = 2,399	SEC_ATTACHABLE_OPT = SEC_ATTACHABLE | SEC_EXP_ATTACH_OPT,400	/* attachment target is specified through BTF ID in either kernel or401	 * other BPF program's BTF object402	 */403	SEC_ATTACH_BTF = 4,404	/* BPF program type allows sleeping/blocking in kernel */405	SEC_SLEEPABLE = 8,406	/* BPF program support non-linear XDP buffer */407	SEC_XDP_FRAGS = 16,408	/* Setup proper attach type for usdt probes. */409	SEC_USDT = 32,410};411 412struct bpf_sec_def {413	char *sec;414	enum bpf_prog_type prog_type;415	enum bpf_attach_type expected_attach_type;416	long cookie;417	int handler_id;418 419	libbpf_prog_setup_fn_t prog_setup_fn;420	libbpf_prog_prepare_load_fn_t prog_prepare_load_fn;421	libbpf_prog_attach_fn_t prog_attach_fn;422};423 424/*425 * bpf_prog should be a better name but it has been used in426 * linux/filter.h.427 */428struct bpf_program {429	char *name;430	char *sec_name;431	size_t sec_idx;432	const struct bpf_sec_def *sec_def;433	/* this program's instruction offset (in number of instructions)434	 * within its containing ELF section435	 */436	size_t sec_insn_off;437	/* number of original instructions in ELF section belonging to this438	 * program, not taking into account subprogram instructions possible439	 * appended later during relocation440	 */441	size_t sec_insn_cnt;442	/* Offset (in number of instructions) of the start of instruction443	 * belonging to this BPF program  within its containing main BPF444	 * program. For the entry-point (main) BPF program, this is always445	 * zero. For a sub-program, this gets reset before each of main BPF446	 * programs are processed and relocated and is used to determined447	 * whether sub-program was already appended to the main program, and448	 * if yes, at which instruction offset.449	 */450	size_t sub_insn_off;451 452	/* instructions that belong to BPF program; insns[0] is located at453	 * sec_insn_off instruction within its ELF section in ELF file, so454	 * when mapping ELF file instruction index to the local instruction,455	 * one needs to subtract sec_insn_off; and vice versa.456	 */457	struct bpf_insn *insns;458	/* actual number of instruction in this BPF program's image; for459	 * entry-point BPF programs this includes the size of main program460	 * itself plus all the used sub-programs, appended at the end461	 */462	size_t insns_cnt;463 464	struct reloc_desc *reloc_desc;465	int nr_reloc;466 467	/* BPF verifier log settings */468	char *log_buf;469	size_t log_size;470	__u32 log_level;471 472	struct bpf_object *obj;473 474	int fd;475	bool autoload;476	bool autoattach;477	bool sym_global;478	bool mark_btf_static;479	enum bpf_prog_type type;480	enum bpf_attach_type expected_attach_type;481	int exception_cb_idx;482 483	int prog_ifindex;484	__u32 attach_btf_obj_fd;485	__u32 attach_btf_id;486	__u32 attach_prog_fd;487 488	void *func_info;489	__u32 func_info_rec_size;490	__u32 func_info_cnt;491 492	void *line_info;493	__u32 line_info_rec_size;494	__u32 line_info_cnt;495	__u32 prog_flags;496};497 498struct bpf_struct_ops {499	struct bpf_program **progs;500	__u32 *kern_func_off;501	/* e.g. struct tcp_congestion_ops in bpf_prog's btf format */502	void *data;503	/* e.g. struct bpf_struct_ops_tcp_congestion_ops in504	 *      btf_vmlinux's format.505	 * struct bpf_struct_ops_tcp_congestion_ops {506	 *	[... some other kernel fields ...]507	 *	struct tcp_congestion_ops data;508	 * }509	 * kern_vdata-size == sizeof(struct bpf_struct_ops_tcp_congestion_ops)510	 * bpf_map__init_kern_struct_ops() will populate the "kern_vdata"511	 * from "data".512	 */513	void *kern_vdata;514	__u32 type_id;515};516 517#define DATA_SEC ".data"518#define BSS_SEC ".bss"519#define RODATA_SEC ".rodata"520#define KCONFIG_SEC ".kconfig"521#define KSYMS_SEC ".ksyms"522#define STRUCT_OPS_SEC ".struct_ops"523#define STRUCT_OPS_LINK_SEC ".struct_ops.link"524#define ARENA_SEC ".addr_space.1"525 526enum libbpf_map_type {527	LIBBPF_MAP_UNSPEC,528	LIBBPF_MAP_DATA,529	LIBBPF_MAP_BSS,530	LIBBPF_MAP_RODATA,531	LIBBPF_MAP_KCONFIG,532};533 534struct bpf_map_def {535	unsigned int type;536	unsigned int key_size;537	unsigned int value_size;538	unsigned int max_entries;539	unsigned int map_flags;540};541 542struct bpf_map {543	struct bpf_object *obj;544	char *name;545	/* real_name is defined for special internal maps (.rodata*,546	 * .data*, .bss, .kconfig) and preserves their original ELF section547	 * name. This is important to be able to find corresponding BTF548	 * DATASEC information.549	 */550	char *real_name;551	int fd;552	int sec_idx;553	size_t sec_offset;554	int map_ifindex;555	int inner_map_fd;556	struct bpf_map_def def;557	__u32 numa_node;558	__u32 btf_var_idx;559	int mod_btf_fd;560	__u32 btf_key_type_id;561	__u32 btf_value_type_id;562	__u32 btf_vmlinux_value_type_id;563	enum libbpf_map_type libbpf_type;564	void *mmaped;565	struct bpf_struct_ops *st_ops;566	struct bpf_map *inner_map;567	void **init_slots;568	int init_slots_sz;569	char *pin_path;570	bool pinned;571	bool reused;572	bool autocreate;573	bool autoattach;574	__u64 map_extra;575};576 577enum extern_type {578	EXT_UNKNOWN,579	EXT_KCFG,580	EXT_KSYM,581};582 583enum kcfg_type {584	KCFG_UNKNOWN,585	KCFG_CHAR,586	KCFG_BOOL,587	KCFG_INT,588	KCFG_TRISTATE,589	KCFG_CHAR_ARR,590};591 592struct extern_desc {593	enum extern_type type;594	int sym_idx;595	int btf_id;596	int sec_btf_id;597	const char *name;598	char *essent_name;599	bool is_set;600	bool is_weak;601	union {602		struct {603			enum kcfg_type type;604			int sz;605			int align;606			int data_off;607			bool is_signed;608		} kcfg;609		struct {610			unsigned long long addr;611 612			/* target btf_id of the corresponding kernel var. */613			int kernel_btf_obj_fd;614			int kernel_btf_id;615 616			/* local btf_id of the ksym extern's type. */617			__u32 type_id;618			/* BTF fd index to be patched in for insn->off, this is619			 * 0 for vmlinux BTF, index in obj->fd_array for module620			 * BTF621			 */622			__s16 btf_fd_idx;623		} ksym;624	};625};626 627struct module_btf {628	struct btf *btf;629	char *name;630	__u32 id;631	int fd;632	int fd_array_idx;633};634 635enum sec_type {636	SEC_UNUSED = 0,637	SEC_RELO,638	SEC_BSS,639	SEC_DATA,640	SEC_RODATA,641	SEC_ST_OPS,642};643 644struct elf_sec_desc {645	enum sec_type sec_type;646	Elf64_Shdr *shdr;647	Elf_Data *data;648};649 650struct elf_state {651	int fd;652	const void *obj_buf;653	size_t obj_buf_sz;654	Elf *elf;655	Elf64_Ehdr *ehdr;656	Elf_Data *symbols;657	Elf_Data *arena_data;658	size_t shstrndx; /* section index for section name strings */659	size_t strtabidx;660	struct elf_sec_desc *secs;661	size_t sec_cnt;662	int btf_maps_shndx;663	__u32 btf_maps_sec_btf_id;664	int text_shndx;665	int symbols_shndx;666	bool has_st_ops;667	int arena_data_shndx;668};669 670struct usdt_manager;671 672struct bpf_object {673	char name[BPF_OBJ_NAME_LEN];674	char license[64];675	__u32 kern_version;676 677	struct bpf_program *programs;678	size_t nr_programs;679	struct bpf_map *maps;680	size_t nr_maps;681	size_t maps_cap;682 683	char *kconfig;684	struct extern_desc *externs;685	int nr_extern;686	int kconfig_map_idx;687 688	bool loaded;689	bool has_subcalls;690	bool has_rodata;691 692	struct bpf_gen *gen_loader;693 694	/* Information when doing ELF related work. Only valid if efile.elf is not NULL */695	struct elf_state efile;696 697	struct btf *btf;698	struct btf_ext *btf_ext;699 700	/* Parse and load BTF vmlinux if any of the programs in the object need701	 * it at load time.702	 */703	struct btf *btf_vmlinux;704	/* Path to the custom BTF to be used for BPF CO-RE relocations as an705	 * override for vmlinux BTF.706	 */707	char *btf_custom_path;708	/* vmlinux BTF override for CO-RE relocations */709	struct btf *btf_vmlinux_override;710	/* Lazily initialized kernel module BTFs */711	struct module_btf *btf_modules;712	bool btf_modules_loaded;713	size_t btf_module_cnt;714	size_t btf_module_cap;715 716	/* optional log settings passed to BPF_BTF_LOAD and BPF_PROG_LOAD commands */717	char *log_buf;718	size_t log_size;719	__u32 log_level;720 721	int *fd_array;722	size_t fd_array_cap;723	size_t fd_array_cnt;724 725	struct usdt_manager *usdt_man;726 727	struct bpf_map *arena_map;728	void *arena_data;729	size_t arena_data_sz;730 731	struct kern_feature_cache *feat_cache;732	char *token_path;733	int token_fd;734 735	char path[];736};737 738static const char *elf_sym_str(const struct bpf_object *obj, size_t off);739static const char *elf_sec_str(const struct bpf_object *obj, size_t off);740static Elf_Scn *elf_sec_by_idx(const struct bpf_object *obj, size_t idx);741static Elf_Scn *elf_sec_by_name(const struct bpf_object *obj, const char *name);742static Elf64_Shdr *elf_sec_hdr(const struct bpf_object *obj, Elf_Scn *scn);743static const char *elf_sec_name(const struct bpf_object *obj, Elf_Scn *scn);744static Elf_Data *elf_sec_data(const struct bpf_object *obj, Elf_Scn *scn);745static Elf64_Sym *elf_sym_by_idx(const struct bpf_object *obj, size_t idx);746static Elf64_Rel *elf_rel_by_idx(Elf_Data *data, size_t idx);747 748void bpf_program__unload(struct bpf_program *prog)749{750	if (!prog)751		return;752 753	zclose(prog->fd);754 755	zfree(&prog->func_info);756	zfree(&prog->line_info);757}758 759static void bpf_program__exit(struct bpf_program *prog)760{761	if (!prog)762		return;763 764	bpf_program__unload(prog);765	zfree(&prog->name);766	zfree(&prog->sec_name);767	zfree(&prog->insns);768	zfree(&prog->reloc_desc);769 770	prog->nr_reloc = 0;771	prog->insns_cnt = 0;772	prog->sec_idx = -1;773}774 775static bool insn_is_subprog_call(const struct bpf_insn *insn)776{777	return BPF_CLASS(insn->code) == BPF_JMP &&778	       BPF_OP(insn->code) == BPF_CALL &&779	       BPF_SRC(insn->code) == BPF_K &&780	       insn->src_reg == BPF_PSEUDO_CALL &&781	       insn->dst_reg == 0 &&782	       insn->off == 0;783}784 785static bool is_call_insn(const struct bpf_insn *insn)786{787	return insn->code == (BPF_JMP | BPF_CALL);788}789 790static bool insn_is_pseudo_func(struct bpf_insn *insn)791{792	return is_ldimm64_insn(insn) && insn->src_reg == BPF_PSEUDO_FUNC;793}794 795static int796bpf_object__init_prog(struct bpf_object *obj, struct bpf_program *prog,797		      const char *name, size_t sec_idx, const char *sec_name,798		      size_t sec_off, void *insn_data, size_t insn_data_sz)799{800	if (insn_data_sz == 0 || insn_data_sz % BPF_INSN_SZ || sec_off % BPF_INSN_SZ) {801		pr_warn("sec '%s': corrupted program '%s', offset %zu, size %zu\n",802			sec_name, name, sec_off, insn_data_sz);803		return -EINVAL;804	}805 806	memset(prog, 0, sizeof(*prog));807	prog->obj = obj;808 809	prog->sec_idx = sec_idx;810	prog->sec_insn_off = sec_off / BPF_INSN_SZ;811	prog->sec_insn_cnt = insn_data_sz / BPF_INSN_SZ;812	/* insns_cnt can later be increased by appending used subprograms */813	prog->insns_cnt = prog->sec_insn_cnt;814 815	prog->type = BPF_PROG_TYPE_UNSPEC;816	prog->fd = -1;817	prog->exception_cb_idx = -1;818 819	/* libbpf's convention for SEC("?abc...") is that it's just like820	 * SEC("abc...") but the corresponding bpf_program starts out with821	 * autoload set to false.822	 */823	if (sec_name[0] == '?') {824		prog->autoload = false;825		/* from now on forget there was ? in section name */826		sec_name++;827	} else {828		prog->autoload = true;829	}830 831	prog->autoattach = true;832 833	/* inherit object's log_level */834	prog->log_level = obj->log_level;835 836	prog->sec_name = strdup(sec_name);837	if (!prog->sec_name)838		goto errout;839 840	prog->name = strdup(name);841	if (!prog->name)842		goto errout;843 844	prog->insns = malloc(insn_data_sz);845	if (!prog->insns)846		goto errout;847	memcpy(prog->insns, insn_data, insn_data_sz);848 849	return 0;850errout:851	pr_warn("sec '%s': failed to allocate memory for prog '%s'\n", sec_name, name);852	bpf_program__exit(prog);853	return -ENOMEM;854}855 856static int857bpf_object__add_programs(struct bpf_object *obj, Elf_Data *sec_data,858			 const char *sec_name, int sec_idx)859{860	Elf_Data *symbols = obj->efile.symbols;861	struct bpf_program *prog, *progs;862	void *data = sec_data->d_buf;863	size_t sec_sz = sec_data->d_size, sec_off, prog_sz, nr_syms;864	int nr_progs, err, i;865	const char *name;866	Elf64_Sym *sym;867 868	progs = obj->programs;869	nr_progs = obj->nr_programs;870	nr_syms = symbols->d_size / sizeof(Elf64_Sym);871 872	for (i = 0; i < nr_syms; i++) {873		sym = elf_sym_by_idx(obj, i);874 875		if (sym->st_shndx != sec_idx)876			continue;877		if (ELF64_ST_TYPE(sym->st_info) != STT_FUNC)878			continue;879 880		prog_sz = sym->st_size;881		sec_off = sym->st_value;882 883		name = elf_sym_str(obj, sym->st_name);884		if (!name) {885			pr_warn("sec '%s': failed to get symbol name for offset %zu\n",886				sec_name, sec_off);887			return -LIBBPF_ERRNO__FORMAT;888		}889 890		if (sec_off + prog_sz > sec_sz) {891			pr_warn("sec '%s': program at offset %zu crosses section boundary\n",892				sec_name, sec_off);893			return -LIBBPF_ERRNO__FORMAT;894		}895 896		if (sec_idx != obj->efile.text_shndx && ELF64_ST_BIND(sym->st_info) == STB_LOCAL) {897			pr_warn("sec '%s': program '%s' is static and not supported\n", sec_name, name);898			return -ENOTSUP;899		}900 901		pr_debug("sec '%s': found program '%s' at insn offset %zu (%zu bytes), code size %zu insns (%zu bytes)\n",902			 sec_name, name, sec_off / BPF_INSN_SZ, sec_off, prog_sz / BPF_INSN_SZ, prog_sz);903 904		progs = libbpf_reallocarray(progs, nr_progs + 1, sizeof(*progs));905		if (!progs) {906			/*907			 * In this case the original obj->programs908			 * is still valid, so don't need special treat for909			 * bpf_close_object().910			 */911			pr_warn("sec '%s': failed to alloc memory for new program '%s'\n",912				sec_name, name);913			return -ENOMEM;914		}915		obj->programs = progs;916 917		prog = &progs[nr_progs];918 919		err = bpf_object__init_prog(obj, prog, name, sec_idx, sec_name,920					    sec_off, data + sec_off, prog_sz);921		if (err)922			return err;923 924		if (ELF64_ST_BIND(sym->st_info) != STB_LOCAL)925			prog->sym_global = true;926 927		/* if function is a global/weak symbol, but has restricted928		 * (STV_HIDDEN or STV_INTERNAL) visibility, mark its BTF FUNC929		 * as static to enable more permissive BPF verification mode930		 * with more outside context available to BPF verifier931		 */932		if (prog->sym_global && (ELF64_ST_VISIBILITY(sym->st_other) == STV_HIDDEN933		    || ELF64_ST_VISIBILITY(sym->st_other) == STV_INTERNAL))934			prog->mark_btf_static = true;935 936		nr_progs++;937		obj->nr_programs = nr_progs;938	}939 940	return 0;941}942 943static const struct btf_member *944find_member_by_offset(const struct btf_type *t, __u32 bit_offset)945{946	struct btf_member *m;947	int i;948 949	for (i = 0, m = btf_members(t); i < btf_vlen(t); i++, m++) {950		if (btf_member_bit_offset(t, i) == bit_offset)951			return m;952	}953 954	return NULL;955}956 957static const struct btf_member *958find_member_by_name(const struct btf *btf, const struct btf_type *t,959		    const char *name)960{961	struct btf_member *m;962	int i;963 964	for (i = 0, m = btf_members(t); i < btf_vlen(t); i++, m++) {965		if (!strcmp(btf__name_by_offset(btf, m->name_off), name))966			return m;967	}968 969	return NULL;970}971 972static int find_ksym_btf_id(struct bpf_object *obj, const char *ksym_name,973			    __u16 kind, struct btf **res_btf,974			    struct module_btf **res_mod_btf);975 976#define STRUCT_OPS_VALUE_PREFIX "bpf_struct_ops_"977static int find_btf_by_prefix_kind(const struct btf *btf, const char *prefix,978				   const char *name, __u32 kind);979 980static int981find_struct_ops_kern_types(struct bpf_object *obj, const char *tname_raw,982			   struct module_btf **mod_btf,983			   const struct btf_type **type, __u32 *type_id,984			   const struct btf_type **vtype, __u32 *vtype_id,985			   const struct btf_member **data_member)986{987	const struct btf_type *kern_type, *kern_vtype;988	const struct btf_member *kern_data_member;989	struct btf *btf = NULL;990	__s32 kern_vtype_id, kern_type_id;991	char tname[256];992	__u32 i;993 994	snprintf(tname, sizeof(tname), "%.*s",995		 (int)bpf_core_essential_name_len(tname_raw), tname_raw);996 997	kern_type_id = find_ksym_btf_id(obj, tname, BTF_KIND_STRUCT,998					&btf, mod_btf);999	if (kern_type_id < 0) {1000		pr_warn("struct_ops init_kern: struct %s is not found in kernel BTF\n",1001			tname);1002		return kern_type_id;1003	}1004	kern_type = btf__type_by_id(btf, kern_type_id);1005 1006	/* Find the corresponding "map_value" type that will be used1007	 * in map_update(BPF_MAP_TYPE_STRUCT_OPS).  For example,1008	 * find "struct bpf_struct_ops_tcp_congestion_ops" from the1009	 * btf_vmlinux.1010	 */1011	kern_vtype_id = find_btf_by_prefix_kind(btf, STRUCT_OPS_VALUE_PREFIX,1012						tname, BTF_KIND_STRUCT);1013	if (kern_vtype_id < 0) {1014		pr_warn("struct_ops init_kern: struct %s%s is not found in kernel BTF\n",1015			STRUCT_OPS_VALUE_PREFIX, tname);1016		return kern_vtype_id;1017	}1018	kern_vtype = btf__type_by_id(btf, kern_vtype_id);1019 1020	/* Find "struct tcp_congestion_ops" from1021	 * struct bpf_struct_ops_tcp_congestion_ops {1022	 *	[ ... ]1023	 *	struct tcp_congestion_ops data;1024	 * }1025	 */1026	kern_data_member = btf_members(kern_vtype);1027	for (i = 0; i < btf_vlen(kern_vtype); i++, kern_data_member++) {1028		if (kern_data_member->type == kern_type_id)1029			break;1030	}1031	if (i == btf_vlen(kern_vtype)) {1032		pr_warn("struct_ops init_kern: struct %s data is not found in struct %s%s\n",1033			tname, STRUCT_OPS_VALUE_PREFIX, tname);1034		return -EINVAL;1035	}1036 1037	*type = kern_type;1038	*type_id = kern_type_id;1039	*vtype = kern_vtype;1040	*vtype_id = kern_vtype_id;1041	*data_member = kern_data_member;1042 1043	return 0;1044}1045 1046static bool bpf_map__is_struct_ops(const struct bpf_map *map)1047{1048	return map->def.type == BPF_MAP_TYPE_STRUCT_OPS;1049}1050 1051static bool is_valid_st_ops_program(struct bpf_object *obj,1052				    const struct bpf_program *prog)1053{1054	int i;1055 1056	for (i = 0; i < obj->nr_programs; i++) {1057		if (&obj->programs[i] == prog)1058			return prog->type == BPF_PROG_TYPE_STRUCT_OPS;1059	}1060 1061	return false;1062}1063 1064/* For each struct_ops program P, referenced from some struct_ops map M,1065 * enable P.autoload if there are Ms for which M.autocreate is true,1066 * disable P.autoload if for all Ms M.autocreate is false.1067 * Don't change P.autoload for programs that are not referenced from any maps.1068 */1069static int bpf_object_adjust_struct_ops_autoload(struct bpf_object *obj)1070{1071	struct bpf_program *prog, *slot_prog;1072	struct bpf_map *map;1073	int i, j, k, vlen;1074 1075	for (i = 0; i < obj->nr_programs; ++i) {1076		int should_load = false;1077		int use_cnt = 0;1078 1079		prog = &obj->programs[i];1080		if (prog->type != BPF_PROG_TYPE_STRUCT_OPS)1081			continue;1082 1083		for (j = 0; j < obj->nr_maps; ++j) {1084			const struct btf_type *type;1085 1086			map = &obj->maps[j];1087			if (!bpf_map__is_struct_ops(map))1088				continue;1089 1090			type = btf__type_by_id(obj->btf, map->st_ops->type_id);1091			vlen = btf_vlen(type);1092			for (k = 0; k < vlen; ++k) {1093				slot_prog = map->st_ops->progs[k];1094				if (prog != slot_prog)1095					continue;1096 1097				use_cnt++;1098				if (map->autocreate)1099					should_load = true;1100			}1101		}1102		if (use_cnt)1103			prog->autoload = should_load;1104	}1105 1106	return 0;1107}1108 1109/* Init the map's fields that depend on kern_btf */1110static int bpf_map__init_kern_struct_ops(struct bpf_map *map)1111{1112	const struct btf_member *member, *kern_member, *kern_data_member;1113	const struct btf_type *type, *kern_type, *kern_vtype;1114	__u32 i, kern_type_id, kern_vtype_id, kern_data_off;1115	struct bpf_object *obj = map->obj;1116	const struct btf *btf = obj->btf;1117	struct bpf_struct_ops *st_ops;1118	const struct btf *kern_btf;1119	struct module_btf *mod_btf = NULL;1120	void *data, *kern_data;1121	const char *tname;1122	int err;1123 1124	st_ops = map->st_ops;1125	type = btf__type_by_id(btf, st_ops->type_id);1126	tname = btf__name_by_offset(btf, type->name_off);1127	err = find_struct_ops_kern_types(obj, tname, &mod_btf,1128					 &kern_type, &kern_type_id,1129					 &kern_vtype, &kern_vtype_id,1130					 &kern_data_member);1131	if (err)1132		return err;1133 1134	kern_btf = mod_btf ? mod_btf->btf : obj->btf_vmlinux;1135 1136	pr_debug("struct_ops init_kern %s: type_id:%u kern_type_id:%u kern_vtype_id:%u\n",1137		 map->name, st_ops->type_id, kern_type_id, kern_vtype_id);1138 1139	map->mod_btf_fd = mod_btf ? mod_btf->fd : -1;1140	map->def.value_size = kern_vtype->size;1141	map->btf_vmlinux_value_type_id = kern_vtype_id;1142 1143	st_ops->kern_vdata = calloc(1, kern_vtype->size);1144	if (!st_ops->kern_vdata)1145		return -ENOMEM;1146 1147	data = st_ops->data;1148	kern_data_off = kern_data_member->offset / 8;1149	kern_data = st_ops->kern_vdata + kern_data_off;1150 1151	member = btf_members(type);1152	for (i = 0; i < btf_vlen(type); i++, member++) {1153		const struct btf_type *mtype, *kern_mtype;1154		__u32 mtype_id, kern_mtype_id;1155		void *mdata, *kern_mdata;1156		struct bpf_program *prog;1157		__s64 msize, kern_msize;1158		__u32 moff, kern_moff;1159		__u32 kern_member_idx;1160		const char *mname;1161 1162		mname = btf__name_by_offset(btf, member->name_off);1163		moff = member->offset / 8;1164		mdata = data + moff;1165		msize = btf__resolve_size(btf, member->type);1166		if (msize < 0) {1167			pr_warn("struct_ops init_kern %s: failed to resolve the size of member %s\n",1168				map->name, mname);1169			return msize;1170		}1171 1172		kern_member = find_member_by_name(kern_btf, kern_type, mname);1173		if (!kern_member) {1174			if (!libbpf_is_mem_zeroed(mdata, msize)) {1175				pr_warn("struct_ops init_kern %s: Cannot find member %s in kernel BTF\n",1176					map->name, mname);1177				return -ENOTSUP;1178			}1179 1180			if (st_ops->progs[i]) {1181				/* If we had declaratively set struct_ops callback, we need to1182				 * force its autoload to false, because it doesn't have1183				 * a chance of succeeding from POV of the current struct_ops map.1184				 * If this program is still referenced somewhere else, though,1185				 * then bpf_object_adjust_struct_ops_autoload() will update its1186				 * autoload accordingly.1187				 */1188				st_ops->progs[i]->autoload = false;1189				st_ops->progs[i] = NULL;1190			}1191 1192			/* Skip all-zero/NULL fields if they are not present in the kernel BTF */1193			pr_info("struct_ops %s: member %s not found in kernel, skipping it as it's set to zero\n",1194				map->name, mname);1195			continue;1196		}1197 1198		kern_member_idx = kern_member - btf_members(kern_type);1199		if (btf_member_bitfield_size(type, i) ||1200		    btf_member_bitfield_size(kern_type, kern_member_idx)) {1201			pr_warn("struct_ops init_kern %s: bitfield %s is not supported\n",1202				map->name, mname);1203			return -ENOTSUP;1204		}1205 1206		kern_moff = kern_member->offset / 8;1207		kern_mdata = kern_data + kern_moff;1208 1209		mtype = skip_mods_and_typedefs(btf, member->type, &mtype_id);1210		kern_mtype = skip_mods_and_typedefs(kern_btf, kern_member->type,1211						    &kern_mtype_id);1212		if (BTF_INFO_KIND(mtype->info) !=1213		    BTF_INFO_KIND(kern_mtype->info)) {1214			pr_warn("struct_ops init_kern %s: Unmatched member type %s %u != %u(kernel)\n",1215				map->name, mname, BTF_INFO_KIND(mtype->info),1216				BTF_INFO_KIND(kern_mtype->info));1217			return -ENOTSUP;1218		}1219 1220		if (btf_is_ptr(mtype)) {1221			prog = *(void **)mdata;1222			/* just like for !kern_member case above, reset declaratively1223			 * set (at compile time) program's autload to false,1224			 * if user replaced it with another program or NULL1225			 */1226			if (st_ops->progs[i] && st_ops->progs[i] != prog)1227				st_ops->progs[i]->autoload = false;1228 1229			/* Update the value from the shadow type */1230			st_ops->progs[i] = prog;1231			if (!prog)1232				continue;1233 1234			if (!is_valid_st_ops_program(obj, prog)) {1235				pr_warn("struct_ops init_kern %s: member %s is not a struct_ops program\n",1236					map->name, mname);1237				return -ENOTSUP;1238			}1239 1240			kern_mtype = skip_mods_and_typedefs(kern_btf,1241							    kern_mtype->type,1242							    &kern_mtype_id);1243 1244			/* mtype->type must be a func_proto which was1245			 * guaranteed in bpf_object__collect_st_ops_relos(),1246			 * so only check kern_mtype for func_proto here.1247			 */1248			if (!btf_is_func_proto(kern_mtype)) {1249				pr_warn("struct_ops init_kern %s: kernel member %s is not a func ptr\n",1250					map->name, mname);1251				return -ENOTSUP;1252			}1253 1254			if (mod_btf)1255				prog->attach_btf_obj_fd = mod_btf->fd;1256 1257			/* if we haven't yet processed this BPF program, record proper1258			 * attach_btf_id and member_idx1259			 */1260			if (!prog->attach_btf_id) {1261				prog->attach_btf_id = kern_type_id;1262				prog->expected_attach_type = kern_member_idx;1263			}1264 1265			/* struct_ops BPF prog can be re-used between multiple1266			 * .struct_ops & .struct_ops.link as long as it's the1267			 * same struct_ops struct definition and the same1268			 * function pointer field1269			 */1270			if (prog->attach_btf_id != kern_type_id) {1271				pr_warn("struct_ops init_kern %s func ptr %s: invalid reuse of prog %s in sec %s with type %u: attach_btf_id %u != kern_type_id %u\n",1272					map->name, mname, prog->name, prog->sec_name, prog->type,1273					prog->attach_btf_id, kern_type_id);1274				return -EINVAL;1275			}1276			if (prog->expected_attach_type != kern_member_idx) {1277				pr_warn("struct_ops init_kern %s func ptr %s: invalid reuse of prog %s in sec %s with type %u: expected_attach_type %u != kern_member_idx %u\n",1278					map->name, mname, prog->name, prog->sec_name, prog->type,1279					prog->expected_attach_type, kern_member_idx);1280				return -EINVAL;1281			}1282 1283			st_ops->kern_func_off[i] = kern_data_off + kern_moff;1284 1285			pr_debug("struct_ops init_kern %s: func ptr %s is set to prog %s from data(+%u) to kern_data(+%u)\n",1286				 map->name, mname, prog->name, moff,1287				 kern_moff);1288 1289			continue;1290		}1291 1292		kern_msize = btf__resolve_size(kern_btf, kern_mtype_id);1293		if (kern_msize < 0 || msize != kern_msize) {1294			pr_warn("struct_ops init_kern %s: Error in size of member %s: %zd != %zd(kernel)\n",1295				map->name, mname, (ssize_t)msize,1296				(ssize_t)kern_msize);1297			return -ENOTSUP;1298		}1299 1300		pr_debug("struct_ops init_kern %s: copy %s %u bytes from data(+%u) to kern_data(+%u)\n",1301			 map->name, mname, (unsigned int)msize,1302			 moff, kern_moff);1303		memcpy(kern_mdata, mdata, msize);1304	}1305 1306	return 0;1307}1308 1309static int bpf_object__init_kern_struct_ops_maps(struct bpf_object *obj)1310{1311	struct bpf_map *map;1312	size_t i;1313	int err;1314 1315	for (i = 0; i < obj->nr_maps; i++) {1316		map = &obj->maps[i];1317 1318		if (!bpf_map__is_struct_ops(map))1319			continue;1320 1321		if (!map->autocreate)1322			continue;1323 1324		err = bpf_map__init_kern_struct_ops(map);1325		if (err)1326			return err;1327	}1328 1329	return 0;1330}1331 1332static int init_struct_ops_maps(struct bpf_object *obj, const char *sec_name,1333				int shndx, Elf_Data *data)1334{1335	const struct btf_type *type, *datasec;1336	const struct btf_var_secinfo *vsi;1337	struct bpf_struct_ops *st_ops;1338	const char *tname, *var_name;1339	__s32 type_id, datasec_id;1340	const struct btf *btf;1341	struct bpf_map *map;1342	__u32 i;1343 1344	if (shndx == -1)1345		return 0;1346 1347	btf = obj->btf;1348	datasec_id = btf__find_by_name_kind(btf, sec_name,1349					    BTF_KIND_DATASEC);1350	if (datasec_id < 0) {1351		pr_warn("struct_ops init: DATASEC %s not found\n",1352			sec_name);1353		return -EINVAL;1354	}1355 1356	datasec = btf__type_by_id(btf, datasec_id);1357	vsi = btf_var_secinfos(datasec);1358	for (i = 0; i < btf_vlen(datasec); i++, vsi++) {1359		type = btf__type_by_id(obj->btf, vsi->type);1360		var_name = btf__name_by_offset(obj->btf, type->name_off);1361 1362		type_id = btf__resolve_type(obj->btf, vsi->type);1363		if (type_id < 0) {1364			pr_warn("struct_ops init: Cannot resolve var type_id %u in DATASEC %s\n",1365				vsi->type, sec_name);1366			return -EINVAL;1367		}1368 1369		type = btf__type_by_id(obj->btf, type_id);1370		tname = btf__name_by_offset(obj->btf, type->name_off);1371		if (!tname[0]) {1372			pr_warn("struct_ops init: anonymous type is not supported\n");1373			return -ENOTSUP;1374		}1375		if (!btf_is_struct(type)) {1376			pr_warn("struct_ops init: %s is not a struct\n", tname);1377			return -EINVAL;1378		}1379 1380		map = bpf_object__add_map(obj);1381		if (IS_ERR(map))1382			return PTR_ERR(map);1383 1384		map->sec_idx = shndx;1385		map->sec_offset = vsi->offset;1386		map->name = strdup(var_name);1387		if (!map->name)1388			return -ENOMEM;1389		map->btf_value_type_id = type_id;1390 1391		/* Follow same convention as for programs autoload:1392		 * SEC("?.struct_ops") means map is not created by default.1393		 */1394		if (sec_name[0] == '?') {1395			map->autocreate = false;1396			/* from now on forget there was ? in section name */1397			sec_name++;1398		}1399 1400		map->def.type = BPF_MAP_TYPE_STRUCT_OPS;1401		map->def.key_size = sizeof(int);1402		map->def.value_size = type->size;1403		map->def.max_entries = 1;1404		map->def.map_flags = strcmp(sec_name, STRUCT_OPS_LINK_SEC) == 0 ? BPF_F_LINK : 0;1405		map->autoattach = true;1406 1407		map->st_ops = calloc(1, sizeof(*map->st_ops));1408		if (!map->st_ops)1409			return -ENOMEM;1410		st_ops = map->st_ops;1411		st_ops->data = malloc(type->size);1412		st_ops->progs = calloc(btf_vlen(type), sizeof(*st_ops->progs));1413		st_ops->kern_func_off = malloc(btf_vlen(type) *1414					       sizeof(*st_ops->kern_func_off));1415		if (!st_ops->data || !st_ops->progs || !st_ops->kern_func_off)1416			return -ENOMEM;1417 1418		if (vsi->offset + type->size > data->d_size) {1419			pr_warn("struct_ops init: var %s is beyond the end of DATASEC %s\n",1420				var_name, sec_name);1421			return -EINVAL;1422		}1423 1424		memcpy(st_ops->data,1425		       data->d_buf + vsi->offset,1426		       type->size);1427		st_ops->type_id = type_id;1428 1429		pr_debug("struct_ops init: struct %s(type_id=%u) %s found at offset %u\n",1430			 tname, type_id, var_name, vsi->offset);1431	}1432 1433	return 0;1434}1435 1436static int bpf_object_init_struct_ops(struct bpf_object *obj)1437{1438	const char *sec_name;1439	int sec_idx, err;1440 1441	for (sec_idx = 0; sec_idx < obj->efile.sec_cnt; ++sec_idx) {1442		struct elf_sec_desc *desc = &obj->efile.secs[sec_idx];1443 1444		if (desc->sec_type != SEC_ST_OPS)1445			continue;1446 1447		sec_name = elf_sec_name(obj, elf_sec_by_idx(obj, sec_idx));1448		if (!sec_name)1449			return -LIBBPF_ERRNO__FORMAT;1450 1451		err = init_struct_ops_maps(obj, sec_name, sec_idx, desc->data);1452		if (err)1453			return err;1454	}1455 1456	return 0;1457}1458 1459static struct bpf_object *bpf_object__new(const char *path,1460					  const void *obj_buf,1461					  size_t obj_buf_sz,1462					  const char *obj_name)1463{1464	struct bpf_object *obj;1465	char *end;1466 1467	obj = calloc(1, sizeof(struct bpf_object) + strlen(path) + 1);1468	if (!obj) {1469		pr_warn("alloc memory failed for %s\n", path);1470		return ERR_PTR(-ENOMEM);1471	}1472 1473	strcpy(obj->path, path);1474	if (obj_name) {1475		libbpf_strlcpy(obj->name, obj_name, sizeof(obj->name));1476	} else {1477		/* Using basename() GNU version which doesn't modify arg. */1478		libbpf_strlcpy(obj->name, basename((void *)path), sizeof(obj->name));1479		end = strchr(obj->name, '.');1480		if (end)1481			*end = 0;1482	}1483 1484	obj->efile.fd = -1;1485	/*1486	 * Caller of this function should also call1487	 * bpf_object__elf_finish() after data collection to return1488	 * obj_buf to user. If not, we should duplicate the buffer to1489	 * avoid user freeing them before elf finish.1490	 */1491	obj->efile.obj_buf = obj_buf;1492	obj->efile.obj_buf_sz = obj_buf_sz;1493	obj->efile.btf_maps_shndx = -1;1494	obj->kconfig_map_idx = -1;1495 1496	obj->kern_version = get_kernel_version();1497	obj->loaded = false;1498 1499	return obj;1500}1501 1502static void bpf_object__elf_finish(struct bpf_object *obj)1503{1504	if (!obj->efile.elf)1505		return;1506 1507	elf_end(obj->efile.elf);1508	obj->efile.elf = NULL;1509	obj->efile.symbols = NULL;1510	obj->efile.arena_data = NULL;1511 1512	zfree(&obj->efile.secs);1513	obj->efile.sec_cnt = 0;1514	zclose(obj->efile.fd);1515	obj->efile.obj_buf = NULL;1516	obj->efile.obj_buf_sz = 0;1517}1518 1519static int bpf_object__elf_init(struct bpf_object *obj)1520{1521	Elf64_Ehdr *ehdr;1522	int err = 0;1523	Elf *elf;1524 1525	if (obj->efile.elf) {1526		pr_warn("elf: init internal error\n");1527		return -LIBBPF_ERRNO__LIBELF;1528	}1529 1530	if (obj->efile.obj_buf_sz > 0) {1531		/* obj_buf should have been validated by bpf_object__open_mem(). */1532		elf = elf_memory((char *)obj->efile.obj_buf, obj->efile.obj_buf_sz);1533	} else {1534		obj->efile.fd = open(obj->path, O_RDONLY | O_CLOEXEC);1535		if (obj->efile.fd < 0) {1536			char errmsg[STRERR_BUFSIZE], *cp;1537 1538			err = -errno;1539			cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg));1540			pr_warn("elf: failed to open %s: %s\n", obj->path, cp);1541			return err;1542		}1543 1544		elf = elf_begin(obj->efile.fd, ELF_C_READ_MMAP, NULL);1545	}1546 1547	if (!elf) {1548		pr_warn("elf: failed to open %s as ELF file: %s\n", obj->path, elf_errmsg(-1));1549		err = -LIBBPF_ERRNO__LIBELF;1550		goto errout;1551	}1552 1553	obj->efile.elf = elf;1554 1555	if (elf_kind(elf) != ELF_K_ELF) {1556		err = -LIBBPF_ERRNO__FORMAT;1557		pr_warn("elf: '%s' is not a proper ELF object\n", obj->path);1558		goto errout;1559	}1560 1561	if (gelf_getclass(elf) != ELFCLASS64) {1562		err = -LIBBPF_ERRNO__FORMAT;1563		pr_warn("elf: '%s' is not a 64-bit ELF object\n", obj->path);1564		goto errout;1565	}1566 1567	obj->efile.ehdr = ehdr = elf64_getehdr(elf);1568	if (!obj->efile.ehdr) {1569		pr_warn("elf: failed to get ELF header from %s: %s\n", obj->path, elf_errmsg(-1));1570		err = -LIBBPF_ERRNO__FORMAT;1571		goto errout;1572	}1573 1574	if (elf_getshdrstrndx(elf, &obj->efile.shstrndx)) {1575		pr_warn("elf: failed to get section names section index for %s: %s\n",1576			obj->path, elf_errmsg(-1));1577		err = -LIBBPF_ERRNO__FORMAT;1578		goto errout;1579	}1580 1581	/* ELF is corrupted/truncated, avoid calling elf_strptr. */1582	if (!elf_rawdata(elf_getscn(elf, obj->efile.shstrndx), NULL)) {1583		pr_warn("elf: failed to get section names strings from %s: %s\n",1584			obj->path, elf_errmsg(-1));1585		err = -LIBBPF_ERRNO__FORMAT;1586		goto errout;1587	}1588 1589	/* Old LLVM set e_machine to EM_NONE */1590	if (ehdr->e_type != ET_REL || (ehdr->e_machine && ehdr->e_machine != EM_BPF)) {1591		pr_warn("elf: %s is not a valid eBPF object file\n", obj->path);1592		err = -LIBBPF_ERRNO__FORMAT;1593		goto errout;1594	}1595 1596	return 0;1597errout:1598	bpf_object__elf_finish(obj);1599	return err;1600}1601 1602static int bpf_object__check_endianness(struct bpf_object *obj)1603{1604#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__1605	if (obj->efile.ehdr->e_ident[EI_DATA] == ELFDATA2LSB)1606		return 0;1607#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__1608	if (obj->efile.ehdr->e_ident[EI_DATA] == ELFDATA2MSB)1609		return 0;1610#else1611# error "Unrecognized __BYTE_ORDER__"1612#endif1613	pr_warn("elf: endianness mismatch in %s.\n", obj->path);1614	return -LIBBPF_ERRNO__ENDIAN;1615}1616 1617static int1618bpf_object__init_license(struct bpf_object *obj, void *data, size_t size)1619{1620	if (!data) {1621		pr_warn("invalid license section in %s\n", obj->path);1622		return -LIBBPF_ERRNO__FORMAT;1623	}1624	/* libbpf_strlcpy() only copies first N - 1 bytes, so size + 1 won't1625	 * go over allowed ELF data section buffer1626	 */1627	libbpf_strlcpy(obj->license, data, min(size + 1, sizeof(obj->license)));1628	pr_debug("license of %s is %s\n", obj->path, obj->license);1629	return 0;1630}1631 1632static int1633bpf_object__init_kversion(struct bpf_object *obj, void *data, size_t size)1634{1635	__u32 kver;1636 1637	if (!data || size != sizeof(kver)) {1638		pr_warn("invalid kver section in %s\n", obj->path);1639		return -LIBBPF_ERRNO__FORMAT;1640	}1641	memcpy(&kver, data, sizeof(kver));1642	obj->kern_version = kver;1643	pr_debug("kernel version of %s is %x\n", obj->path, obj->kern_version);1644	return 0;1645}1646 1647static bool bpf_map_type__is_map_in_map(enum bpf_map_type type)1648{1649	if (type == BPF_MAP_TYPE_ARRAY_OF_MAPS ||1650	    type == BPF_MAP_TYPE_HASH_OF_MAPS)1651		return true;1652	return false;1653}1654 1655static int find_elf_sec_sz(const struct bpf_object *obj, const char *name, __u32 *size)1656{1657	Elf_Data *data;1658	Elf_Scn *scn;1659 1660	if (!name)1661		return -EINVAL;1662 1663	scn = elf_sec_by_name(obj, name);1664	data = elf_sec_data(obj, scn);1665	if (data) {1666		*size = data->d_size;1667		return 0; /* found it */1668	}1669 1670	return -ENOENT;1671}1672 1673static Elf64_Sym *find_elf_var_sym(const struct bpf_object *obj, const char *name)1674{1675	Elf_Data *symbols = obj->efile.symbols;1676	const char *sname;1677	size_t si;1678 1679	for (si = 0; si < symbols->d_size / sizeof(Elf64_Sym); si++) {1680		Elf64_Sym *sym = elf_sym_by_idx(obj, si);1681 1682		if (ELF64_ST_TYPE(sym->st_info) != STT_OBJECT)1683			continue;1684 1685		if (ELF64_ST_BIND(sym->st_info) != STB_GLOBAL &&1686		    ELF64_ST_BIND(sym->st_info) != STB_WEAK)1687			continue;1688 1689		sname = elf_sym_str(obj, sym->st_name);1690		if (!sname) {1691			pr_warn("failed to get sym name string for var %s\n", name);1692			return ERR_PTR(-EIO);1693		}1694		if (strcmp(name, sname) == 0)1695			return sym;1696	}1697 1698	return ERR_PTR(-ENOENT);1699}1700 1701/* Some versions of Android don't provide memfd_create() in their libc1702 * implementation, so avoid complications and just go straight to Linux1703 * syscall.1704 */1705static int sys_memfd_create(const char *name, unsigned flags)1706{1707	return syscall(__NR_memfd_create, name, flags);1708}1709 1710#ifndef MFD_CLOEXEC1711#define MFD_CLOEXEC 0x0001U1712#endif1713 1714static int create_placeholder_fd(void)1715{1716	int fd;1717 1718	fd = ensure_good_fd(sys_memfd_create("libbpf-placeholder-fd", MFD_CLOEXEC));1719	if (fd < 0)1720		return -errno;1721	return fd;1722}1723 1724static struct bpf_map *bpf_object__add_map(struct bpf_object *obj)1725{1726	struct bpf_map *map;1727	int err;1728 1729	err = libbpf_ensure_mem((void **)&obj->maps, &obj->maps_cap,1730				sizeof(*obj->maps), obj->nr_maps + 1);1731	if (err)1732		return ERR_PTR(err);1733 1734	map = &obj->maps[obj->nr_maps++];1735	map->obj = obj;1736	/* Preallocate map FD without actually creating BPF map just yet.1737	 * These map FD "placeholders" will be reused later without changing1738	 * FD value when map is actually created in the kernel.1739	 *1740	 * This is useful to be able to perform BPF program relocations1741	 * without having to create BPF maps before that step. This allows us1742	 * to finalize and load BTF very late in BPF object's loading phase,1743	 * right before BPF maps have to be created and BPF programs have to1744	 * be loaded. By having these map FD placeholders we can perform all1745	 * the sanitizations, relocations, and any other adjustments before we1746	 * start creating actual BPF kernel objects (BTF, maps, progs).1747	 */1748	map->fd = create_placeholder_fd();1749	if (map->fd < 0)1750		return ERR_PTR(map->fd);1751	map->inner_map_fd = -1;1752	map->autocreate = true;1753 1754	return map;1755}1756 1757static size_t array_map_mmap_sz(unsigned int value_sz, unsigned int max_entries)1758{1759	const long page_sz = sysconf(_SC_PAGE_SIZE);1760	size_t map_sz;1761 1762	map_sz = (size_t)roundup(value_sz, 8) * max_entries;1763	map_sz = roundup(map_sz, page_sz);1764	return map_sz;1765}1766 1767static size_t bpf_map_mmap_sz(const struct bpf_map *map)1768{1769	const long page_sz = sysconf(_SC_PAGE_SIZE);1770 1771	switch (map->def.type) {1772	case BPF_MAP_TYPE_ARRAY:1773		return array_map_mmap_sz(map->def.value_size, map->def.max_entries);1774	case BPF_MAP_TYPE_ARENA:1775		return page_sz * map->def.max_entries;1776	default:1777		return 0; /* not supported */1778	}1779}1780 1781static int bpf_map_mmap_resize(struct bpf_map *map, size_t old_sz, size_t new_sz)1782{1783	void *mmaped;1784 1785	if (!map->mmaped)1786		return -EINVAL;1787 1788	if (old_sz == new_sz)1789		return 0;1790 1791	mmaped = mmap(NULL, new_sz, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANONYMOUS, -1, 0);1792	if (mmaped == MAP_FAILED)1793		return -errno;1794 1795	memcpy(mmaped, map->mmaped, min(old_sz, new_sz));1796	munmap(map->mmaped, old_sz);1797	map->mmaped = mmaped;1798	return 0;1799}1800 1801static char *internal_map_name(struct bpf_object *obj, const char *real_name)1802{1803	char map_name[BPF_OBJ_NAME_LEN], *p;1804	int pfx_len, sfx_len = max((size_t)7, strlen(real_name));1805 1806	/* This is one of the more confusing parts of libbpf for various1807	 * reasons, some of which are historical. The original idea for naming1808	 * internal names was to include as much of BPF object name prefix as1809	 * possible, so that it can be distinguished from similar internal1810	 * maps of a different BPF object.1811	 * As an example, let's say we have bpf_object named 'my_object_name'1812	 * and internal map corresponding to '.rodata' ELF section. The final1813	 * map name advertised to user and to the kernel will be1814	 * 'my_objec.rodata', taking first 8 characters of object name and1815	 * entire 7 characters of '.rodata'.1816	 * Somewhat confusingly, if internal map ELF section name is shorter1817	 * than 7 characters, e.g., '.bss', we still reserve 7 characters1818	 * for the suffix, even though we only have 4 actual characters, and1819	 * resulting map will be called 'my_objec.bss', not even using all 151820	 * characters allowed by the kernel. Oh well, at least the truncated1821	 * object name is somewhat consistent in this case. But if the map1822	 * name is '.kconfig', we'll still have entirety of '.kconfig' added1823	 * (8 chars) and thus will be left with only first 7 characters of the1824	 * object name ('my_obje'). Happy guessing, user, that the final map1825	 * name will be "my_obje.kconfig".1826	 * Now, with libbpf starting to support arbitrarily named .rodata.*1827	 * and .data.* data sections, it's possible that ELF section name is1828	 * longer than allowed 15 chars, so we now need to be careful to take1829	 * only up to 15 first characters of ELF name, taking no BPF object1830	 * name characters at all. So '.rodata.abracadabra' will result in1831	 * '.rodata.abracad' kernel and user-visible name.1832	 * We need to keep this convoluted logic intact for .data, .bss and1833	 * .rodata maps, but for new custom .data.custom and .rodata.custom1834	 * maps we use their ELF names as is, not prepending bpf_object name1835	 * in front. We still need to truncate them to 15 characters for the1836	 * kernel. Full name can be recovered for such maps by using DATASEC1837	 * BTF type associated with such map's value type, though.1838	 */1839	if (sfx_len >= BPF_OBJ_NAME_LEN)1840		sfx_len = BPF_OBJ_NAME_LEN - 1;1841 1842	/* if there are two or more dots in map name, it's a custom dot map */1843	if (strchr(real_name + 1, '.') != NULL)1844		pfx_len = 0;1845	else1846		pfx_len = min((size_t)BPF_OBJ_NAME_LEN - sfx_len - 1, strlen(obj->name));1847 1848	snprintf(map_name, sizeof(map_name), "%.*s%.*s", pfx_len, obj->name,1849		 sfx_len, real_name);1850 1851	/* sanities map name to characters allowed by kernel */1852	for (p = map_name; *p && p < map_name + sizeof(map_name); p++)1853		if (!isalnum(*p) && *p != '_' && *p != '.')1854			*p = '_';1855 1856	return strdup(map_name);1857}1858 1859static int1860map_fill_btf_type_info(struct bpf_object *obj, struct bpf_map *map);1861 1862/* Internal BPF map is mmap()'able only if at least one of corresponding1863 * DATASEC's VARs are to be exposed through BPF skeleton. I.e., it's a GLOBAL1864 * variable and it's not marked as __hidden (which turns it into, effectively,1865 * a STATIC variable).1866 */1867static bool map_is_mmapable(struct bpf_object *obj, struct bpf_map *map)1868{1869	const struct btf_type *t, *vt;1870	struct btf_var_secinfo *vsi;1871	int i, n;1872 1873	if (!map->btf_value_type_id)1874		return false;1875 1876	t = btf__type_by_id(obj->btf, map->btf_value_type_id);1877	if (!btf_is_datasec(t))1878		return false;1879 1880	vsi = btf_var_secinfos(t);1881	for (i = 0, n = btf_vlen(t); i < n; i++, vsi++) {1882		vt = btf__type_by_id(obj->btf, vsi->type);1883		if (!btf_is_var(vt))1884			continue;1885 1886		if (btf_var(vt)->linkage != BTF_VAR_STATIC)1887			return true;1888	}1889 1890	return false;1891}1892 1893static int1894bpf_object__init_internal_map(struct bpf_object *obj, enum libbpf_map_type type,1895			      const char *real_name, int sec_idx, void *data, size_t data_sz)1896{1897	struct bpf_map_def *def;1898	struct bpf_map *map;1899	size_t mmap_sz;1900	int err;1901 1902	map = bpf_object__add_map(obj);1903	if (IS_ERR(map))1904		return PTR_ERR(map);1905 1906	map->libbpf_type = type;1907	map->sec_idx = sec_idx;1908	map->sec_offset = 0;1909	map->real_name = strdup(real_name);1910	map->name = internal_map_name(obj, real_name);1911	if (!map->real_name || !map->name) {1912		zfree(&map->real_name);1913		zfree(&map->name);1914		return -ENOMEM;1915	}1916 1917	def = &map->def;1918	def->type = BPF_MAP_TYPE_ARRAY;1919	def->key_size = sizeof(int);1920	def->value_size = data_sz;1921	def->max_entries = 1;1922	def->map_flags = type == LIBBPF_MAP_RODATA || type == LIBBPF_MAP_KCONFIG1923		? BPF_F_RDONLY_PROG : 0;1924 1925	/* failures are fine because of maps like .rodata.str1.1 */1926	(void) map_fill_btf_type_info(obj, map);1927 1928	if (map_is_mmapable(obj, map))1929		def->map_flags |= BPF_F_MMAPABLE;1930 1931	pr_debug("map '%s' (global data): at sec_idx %d, offset %zu, flags %x.\n",1932		 map->name, map->sec_idx, map->sec_offset, def->map_flags);1933 1934	mmap_sz = bpf_map_mmap_sz(map);1935	map->mmaped = mmap(NULL, mmap_sz, PROT_READ | PROT_WRITE,1936			   MAP_SHARED | MAP_ANONYMOUS, -1, 0);1937	if (map->mmaped == MAP_FAILED) {1938		err = -errno;1939		map->mmaped = NULL;1940		pr_warn("failed to alloc map '%s' content buffer: %d\n",1941			map->name, err);1942		zfree(&map->real_name);1943		zfree(&map->name);1944		return err;1945	}1946 1947	if (data)1948		memcpy(map->mmaped, data, data_sz);1949 1950	pr_debug("map %td is \"%s\"\n", map - obj->maps, map->name);1951	return 0;1952}1953 1954static int bpf_object__init_global_data_maps(struct bpf_object *obj)1955{1956	struct elf_sec_desc *sec_desc;1957	const char *sec_name;1958	int err = 0, sec_idx;1959 1960	/*1961	 * Populate obj->maps with libbpf internal maps.1962	 */1963	for (sec_idx = 1; sec_idx < obj->efile.sec_cnt; sec_idx++) {1964		sec_desc = &obj->efile.secs[sec_idx];1965 1966		/* Skip recognized sections with size 0. */1967		if (!sec_desc->data || sec_desc->data->d_size == 0)1968			continue;1969 1970		switch (sec_desc->sec_type) {1971		case SEC_DATA:1972			sec_name = elf_sec_name(obj, elf_sec_by_idx(obj, sec_idx));1973			err = bpf_object__init_internal_map(obj, LIBBPF_MAP_DATA,1974							    sec_name, sec_idx,1975							    sec_desc->data->d_buf,1976							    sec_desc->data->d_size);1977			break;1978		case SEC_RODATA:1979			obj->has_rodata = true;1980			sec_name = elf_sec_name(obj, elf_sec_by_idx(obj, sec_idx));1981			err = bpf_object__init_internal_map(obj, LIBBPF_MAP_RODATA,1982							    sec_name, sec_idx,1983							    sec_desc->data->d_buf,1984							    sec_desc->data->d_size);1985			break;1986		case SEC_BSS:1987			sec_name = elf_sec_name(obj, elf_sec_by_idx(obj, sec_idx));1988			err = bpf_object__init_internal_map(obj, LIBBPF_MAP_BSS,1989							    sec_name, sec_idx,1990							    NULL,1991							    sec_desc->data->d_size);1992			break;1993		default:1994			/* skip */1995			break;1996		}1997		if (err)1998			return err;1999	}2000	return 0;2001}2002 2003 2004static struct extern_desc *find_extern_by_name(const struct bpf_object *obj,2005					       const void *name)2006{2007	int i;2008 2009	for (i = 0; i < obj->nr_extern; i++) {2010		if (strcmp(obj->externs[i].name, name) == 0)2011			return &obj->externs[i];2012	}2013	return NULL;2014}2015 2016static struct extern_desc *find_extern_by_name_with_len(const struct bpf_object *obj,2017							const void *name, int len)2018{2019	const char *ext_name;2020	int i;2021 2022	for (i = 0; i < obj->nr_extern; i++) {2023		ext_name = obj->externs[i].name;2024		if (strlen(ext_name) == len && strncmp(ext_name, name, len) == 0)2025			return &obj->externs[i];2026	}2027	return NULL;2028}2029 2030static int set_kcfg_value_tri(struct extern_desc *ext, void *ext_val,2031			      char value)2032{2033	switch (ext->kcfg.type) {2034	case KCFG_BOOL:2035		if (value == 'm') {2036			pr_warn("extern (kcfg) '%s': value '%c' implies tristate or char type\n",2037				ext->name, value);2038			return -EINVAL;2039		}2040		*(bool *)ext_val = value == 'y' ? true : false;2041		break;2042	case KCFG_TRISTATE:2043		if (value == 'y')2044			*(enum libbpf_tristate *)ext_val = TRI_YES;2045		else if (value == 'm')2046			*(enum libbpf_tristate *)ext_val = TRI_MODULE;2047		else /* value == 'n' */2048			*(enum libbpf_tristate *)ext_val = TRI_NO;2049		break;2050	case KCFG_CHAR:2051		*(char *)ext_val = value;2052		break;2053	case KCFG_UNKNOWN:2054	case KCFG_INT:2055	case KCFG_CHAR_ARR:2056	default:2057		pr_warn("extern (kcfg) '%s': value '%c' implies bool, tristate, or char type\n",2058			ext->name, value);2059		return -EINVAL;2060	}2061	ext->is_set = true;2062	return 0;2063}2064 2065static int set_kcfg_value_str(struct extern_desc *ext, char *ext_val,2066			      const char *value)2067{2068	size_t len;2069 2070	if (ext->kcfg.type != KCFG_CHAR_ARR) {2071		pr_warn("extern (kcfg) '%s': value '%s' implies char array type\n",2072			ext->name, value);2073		return -EINVAL;2074	}2075 2076	len = strlen(value);2077	if (value[len - 1] != '"') {2078		pr_warn("extern (kcfg) '%s': invalid string config '%s'\n",2079			ext->name, value);2080		return -EINVAL;2081	}2082 2083	/* strip quotes */2084	len -= 2;2085	if (len >= ext->kcfg.sz) {2086		pr_warn("extern (kcfg) '%s': long string '%s' of (%zu bytes) truncated to %d bytes\n",2087			ext->name, value, len, ext->kcfg.sz - 1);2088		len = ext->kcfg.sz - 1;2089	}2090	memcpy(ext_val, value + 1, len);2091	ext_val[len] = '\0';2092	ext->is_set = true;2093	return 0;2094}2095 2096static int parse_u64(const char *value, __u64 *res)2097{2098	char *value_end;2099	int err;2100 2101	errno = 0;2102	*res = strtoull(value, &value_end, 0);2103	if (errno) {2104		err = -errno;2105		pr_warn("failed to parse '%s' as integer: %d\n", value, err);2106		return err;2107	}2108	if (*value_end) {2109		pr_warn("failed to parse '%s' as integer completely\n", value);2110		return -EINVAL;2111	}2112	return 0;2113}2114 2115static bool is_kcfg_value_in_range(const struct extern_desc *ext, __u64 v)2116{2117	int bit_sz = ext->kcfg.sz * 8;2118 2119	if (ext->kcfg.sz == 8)2120		return true;2121 2122	/* Validate that value stored in u64 fits in integer of `ext->sz`2123	 * bytes size without any loss of information. If the target integer2124	 * is signed, we rely on the following limits of integer type of2125	 * Y bits and subsequent transformation:2126	 *2127	 *     -2^(Y-1) <= X           <= 2^(Y-1) - 12128	 *            0 <= X + 2^(Y-1) <= 2^Y - 12129	 *            0 <= X + 2^(Y-1) <  2^Y2130	 *2131	 *  For unsigned target integer, check that all the (64 - Y) bits are2132	 *  zero.2133	 */2134	if (ext->kcfg.is_signed)2135		return v + (1ULL << (bit_sz - 1)) < (1ULL << bit_sz);2136	else2137		return (v >> bit_sz) == 0;2138}2139 2140static int set_kcfg_value_num(struct extern_desc *ext, void *ext_val,2141			      __u64 value)2142{2143	if (ext->kcfg.type != KCFG_INT && ext->kcfg.type != KCFG_CHAR &&2144	    ext->kcfg.type != KCFG_BOOL) {2145		pr_warn("extern (kcfg) '%s': value '%llu' implies integer, char, or boolean type\n",2146			ext->name, (unsigned long long)value);2147		return -EINVAL;2148	}2149	if (ext->kcfg.type == KCFG_BOOL && value > 1) {2150		pr_warn("extern (kcfg) '%s': value '%llu' isn't boolean compatible\n",2151			ext->name, (unsigned long long)value);2152		return -EINVAL;2153 2154	}2155	if (!is_kcfg_value_in_range(ext, value)) {2156		pr_warn("extern (kcfg) '%s': value '%llu' doesn't fit in %d bytes\n",2157			ext->name, (unsigned long long)value, ext->kcfg.sz);2158		return -ERANGE;2159	}2160	switch (ext->kcfg.sz) {2161	case 1:2162		*(__u8 *)ext_val = value;2163		break;2164	case 2:2165		*(__u16 *)ext_val = value;2166		break;2167	case 4:2168		*(__u32 *)ext_val = value;2169		break;2170	case 8:2171		*(__u64 *)ext_val = value;2172		break;2173	default:2174		return -EINVAL;2175	}2176	ext->is_set = true;2177	return 0;2178}2179 2180static int bpf_object__process_kconfig_line(struct bpf_object *obj,2181					    char *buf, void *data)2182{2183	struct extern_desc *ext;2184	char *sep, *value;2185	int len, err = 0;2186	void *ext_val;2187	__u64 num;2188 2189	if (!str_has_pfx(buf, "CONFIG_"))2190		return 0;2191 2192	sep = strchr(buf, '=');2193	if (!sep) {2194		pr_warn("failed to parse '%s': no separator\n", buf);2195		return -EINVAL;2196	}2197 2198	/* Trim ending '\n' */2199	len = strlen(buf);2200	if (buf[len - 1] == '\n')2201		buf[len - 1] = '\0';2202	/* Split on '=' and ensure that a value is present. */2203	*sep = '\0';2204	if (!sep[1]) {2205		*sep = '=';2206		pr_warn("failed to parse '%s': no value\n", buf);2207		return -EINVAL;2208	}2209 2210	ext = find_extern_by_name(obj, buf);2211	if (!ext || ext->is_set)2212		return 0;2213 2214	ext_val = data + ext->kcfg.data_off;2215	value = sep + 1;2216 2217	switch (*value) {2218	case 'y': case 'n': case 'm':2219		err = set_kcfg_value_tri(ext, ext_val, *value);2220		break;2221	case '"':2222		err = set_kcfg_value_str(ext, ext_val, value);2223		break;2224	default:2225		/* assume integer */2226		err = parse_u64(value, &num);2227		if (err) {2228			pr_warn("extern (kcfg) '%s': value '%s' isn't a valid integer\n", ext->name, value);2229			return err;2230		}2231		if (ext->kcfg.type != KCFG_INT && ext->kcfg.type != KCFG_CHAR) {2232			pr_warn("extern (kcfg) '%s': value '%s' implies integer type\n", ext->name, value);2233			return -EINVAL;2234		}2235		err = set_kcfg_value_num(ext, ext_val, num);2236		break;2237	}2238	if (err)2239		return err;2240	pr_debug("extern (kcfg) '%s': set to %s\n", ext->name, value);2241	return 0;2242}2243 2244static int bpf_object__read_kconfig_file(struct bpf_object *obj, void *data)2245{2246	char buf[PATH_MAX];2247	struct utsname uts;2248	int len, err = 0;2249	gzFile file;2250 2251	uname(&uts);2252	len = snprintf(buf, PATH_MAX, "/boot/config-%s", uts.release);2253	if (len < 0)2254		return -EINVAL;2255	else if (len >= PATH_MAX)2256		return -ENAMETOOLONG;2257 2258	/* gzopen also accepts uncompressed files. */2259	file = gzopen(buf, "re");2260	if (!file)2261		file = gzopen("/proc/config.gz", "re");2262 2263	if (!file) {2264		pr_warn("failed to open system Kconfig\n");2265		return -ENOENT;2266	}2267 2268	while (gzgets(file, buf, sizeof(buf))) {2269		err = bpf_object__process_kconfig_line(obj, buf, data);2270		if (err) {2271			pr_warn("error parsing system Kconfig line '%s': %d\n",2272				buf, err);2273			goto out;2274		}2275	}2276 2277out:2278	gzclose(file);2279	return err;2280}2281 2282static int bpf_object__read_kconfig_mem(struct bpf_object *obj,2283					const char *config, void *data)2284{2285	char buf[PATH_MAX];2286	int err = 0;2287	FILE *file;2288 2289	file = fmemopen((void *)config, strlen(config), "r");2290	if (!file) {2291		err = -errno;2292		pr_warn("failed to open in-memory Kconfig: %d\n", err);2293		return err;2294	}2295 2296	while (fgets(buf, sizeof(buf), file)) {2297		err = bpf_object__process_kconfig_line(obj, buf, data);2298		if (err) {2299			pr_warn("error parsing in-memory Kconfig line '%s': %d\n",2300				buf, err);2301			break;2302		}2303	}2304 2305	fclose(file);2306	return err;2307}2308 2309static int bpf_object__init_kconfig_map(struct bpf_object *obj)2310{2311	struct extern_desc *last_ext = NULL, *ext;2312	size_t map_sz;2313	int i, err;2314 2315	for (i = 0; i < obj->nr_extern; i++) {2316		ext = &obj->externs[i];2317		if (ext->type == EXT_KCFG)2318			last_ext = ext;2319	}2320 2321	if (!last_ext)2322		return 0;2323 2324	map_sz = last_ext->kcfg.data_off + last_ext->kcfg.sz;2325	err = bpf_object__init_internal_map(obj, LIBBPF_MAP_KCONFIG,2326					    ".kconfig", obj->efile.symbols_shndx,2327					    NULL, map_sz);2328	if (err)2329		return err;2330 2331	obj->kconfig_map_idx = obj->nr_maps - 1;2332 2333	return 0;2334}2335 2336const struct btf_type *2337skip_mods_and_typedefs(const struct btf *btf, __u32 id, __u32 *res_id)2338{2339	const struct btf_type *t = btf__type_by_id(btf, id);2340 2341	if (res_id)2342		*res_id = id;2343 2344	while (btf_is_mod(t) || btf_is_typedef(t)) {2345		if (res_id)2346			*res_id = t->type;2347		t = btf__type_by_id(btf, t->type);2348	}2349 2350	return t;2351}2352 2353static const struct btf_type *2354resolve_func_ptr(const struct btf *btf, __u32 id, __u32 *res_id)2355{2356	const struct btf_type *t;2357 2358	t = skip_mods_and_typedefs(btf, id, NULL);2359	if (!btf_is_ptr(t))2360		return NULL;2361 2362	t = skip_mods_and_typedefs(btf, t->type, res_id);2363 2364	return btf_is_func_proto(t) ? t : NULL;2365}2366 2367static const char *__btf_kind_str(__u16 kind)2368{2369	switch (kind) {2370	case BTF_KIND_UNKN: return "void";2371	case BTF_KIND_INT: return "int";2372	case BTF_KIND_PTR: return "ptr";2373	case BTF_KIND_ARRAY: return "array";2374	case BTF_KIND_STRUCT: return "struct";2375	case BTF_KIND_UNION: return "union";2376	case BTF_KIND_ENUM: return "enum";2377	case BTF_KIND_FWD: return "fwd";2378	case BTF_KIND_TYPEDEF: return "typedef";2379	case BTF_KIND_VOLATILE: return "volatile";2380	case BTF_KIND_CONST: return "const";2381	case BTF_KIND_RESTRICT: return "restrict";2382	case BTF_KIND_FUNC: return "func";2383	case BTF_KIND_FUNC_PROTO: return "func_proto";2384	case BTF_KIND_VAR: return "var";2385	case BTF_KIND_DATASEC: return "datasec";2386	case BTF_KIND_FLOAT: return "float";2387	case BTF_KIND_DECL_TAG: return "decl_tag";2388	case BTF_KIND_TYPE_TAG: return "type_tag";2389	case BTF_KIND_ENUM64: return "enum64";2390	default: return "unknown";2391	}2392}2393 2394const char *btf_kind_str(const struct btf_type *t)2395{2396	return __btf_kind_str(btf_kind(t));2397}2398 2399/*2400 * Fetch integer attribute of BTF map definition. Such attributes are2401 * represented using a pointer to an array, in which dimensionality of array2402 * encodes specified integer value. E.g., int (*type)[BPF_MAP_TYPE_ARRAY];2403 * encodes `type => BPF_MAP_TYPE_ARRAY` key/value pair completely using BTF2404 * type definition, while using only sizeof(void *) space in ELF data section.2405 */2406static bool get_map_field_int(const char *map_name, const struct btf *btf,2407			      const struct btf_member *m, __u32 *res)2408{2409	const struct btf_type *t = skip_mods_and_typedefs(btf, m->type, NULL);2410	const char *name = btf__name_by_offset(btf, m->name_off);2411	const struct btf_array *arr_info;2412	const struct btf_type *arr_t;2413 2414	if (!btf_is_ptr(t)) {2415		pr_warn("map '%s': attr '%s': expected PTR, got %s.\n",2416			map_name, name, btf_kind_str(t));2417		return false;2418	}2419 2420	arr_t = btf__type_by_id(btf, t->type);2421	if (!arr_t) {2422		pr_warn("map '%s': attr '%s': type [%u] not found.\n",2423			map_name, name, t->type);2424		return false;2425	}2426	if (!btf_is_array(arr_t)) {2427		pr_warn("map '%s': attr '%s': expected ARRAY, got %s.\n",2428			map_name, name, btf_kind_str(arr_t));2429		return false;2430	}2431	arr_info = btf_array(arr_t);2432	*res = arr_info->nelems;2433	return true;2434}2435 2436static bool get_map_field_long(const char *map_name, const struct btf *btf,2437			       const struct btf_member *m, __u64 *res)2438{2439	const struct btf_type *t = skip_mods_and_typedefs(btf, m->type, NULL);2440	const char *name = btf__name_by_offset(btf, m->name_off);2441 2442	if (btf_is_ptr(t)) {2443		__u32 res32;2444		bool ret;2445 2446		ret = get_map_field_int(map_name, btf, m, &res32);2447		if (ret)2448			*res = (__u64)res32;2449		return ret;2450	}2451 2452	if (!btf_is_enum(t) && !btf_is_enum64(t)) {2453		pr_warn("map '%s': attr '%s': expected ENUM or ENUM64, got %s.\n",2454			map_name, name, btf_kind_str(t));2455		return false;2456	}2457 2458	if (btf_vlen(t) != 1) {2459		pr_warn("map '%s': attr '%s': invalid __ulong\n",2460			map_name, name);2461		return false;2462	}2463 2464	if (btf_is_enum(t)) {2465		const struct btf_enum *e = btf_enum(t);2466 2467		*res = e->val;2468	} else {2469		const struct btf_enum64 *e = btf_enum64(t);2470 2471		*res = btf_enum64_value(e);2472	}2473	return true;2474}2475 2476static int pathname_concat(char *buf, size_t buf_sz, const char *path, const char *name)2477{2478	int len;2479 2480	len = snprintf(buf, buf_sz, "%s/%s", path, name);2481	if (len < 0)2482		return -EINVAL;2483	if (len >= buf_sz)2484		return -ENAMETOOLONG;2485 2486	return 0;2487}2488 2489static int build_map_pin_path(struct bpf_map *map, const char *path)2490{2491	char buf[PATH_MAX];2492	int err;2493 2494	if (!path)2495		path = BPF_FS_DEFAULT_PATH;2496 2497	err = pathname_concat(buf, sizeof(buf), path, bpf_map__name(map));2498	if (err)2499		return err;2500 2501	return bpf_map__set_pin_path(map, buf);2502}2503 2504/* should match definition in bpf_helpers.h */2505enum libbpf_pin_type {2506	LIBBPF_PIN_NONE,2507	/* PIN_BY_NAME: pin maps by name (in /sys/fs/bpf by default) */2508	LIBBPF_PIN_BY_NAME,2509};2510 2511int parse_btf_map_def(const char *map_name, struct btf *btf,2512		      const struct btf_type *def_t, bool strict,2513		      struct btf_map_def *map_def, struct btf_map_def *inner_def)2514{2515	const struct btf_type *t;2516	const struct btf_member *m;2517	bool is_inner = inner_def == NULL;2518	int vlen, i;2519 2520	vlen = btf_vlen(def_t);2521	m = btf_members(def_t);2522	for (i = 0; i < vlen; i++, m++) {2523		const char *name = btf__name_by_offset(btf, m->name_off);2524 2525		if (!name) {2526			pr_warn("map '%s': invalid field #%d.\n", map_name, i);2527			return -EINVAL;2528		}2529		if (strcmp(name, "type") == 0) {2530			if (!get_map_field_int(map_name, btf, m, &map_def->map_type))2531				return -EINVAL;2532			map_def->parts |= MAP_DEF_MAP_TYPE;2533		} else if (strcmp(name, "max_entries") == 0) {2534			if (!get_map_field_int(map_name, btf, m, &map_def->max_entries))2535				return -EINVAL;2536			map_def->parts |= MAP_DEF_MAX_ENTRIES;2537		} else if (strcmp(name, "map_flags") == 0) {2538			if (!get_map_field_int(map_name, btf, m, &map_def->map_flags))2539				return -EINVAL;2540			map_def->parts |= MAP_DEF_MAP_FLAGS;2541		} else if (strcmp(name, "numa_node") == 0) {2542			if (!get_map_field_int(map_name, btf, m, &map_def->numa_node))2543				return -EINVAL;2544			map_def->parts |= MAP_DEF_NUMA_NODE;2545		} else if (strcmp(name, "key_size") == 0) {2546			__u32 sz;2547 2548			if (!get_map_field_int(map_name, btf, m, &sz))2549				return -EINVAL;2550			if (map_def->key_size && map_def->key_size != sz) {2551				pr_warn("map '%s': conflicting key size %u != %u.\n",2552					map_name, map_def->key_size, sz);2553				return -EINVAL;2554			}2555			map_def->key_size = sz;2556			map_def->parts |= MAP_DEF_KEY_SIZE;2557		} else if (strcmp(name, "key") == 0) {2558			__s64 sz;2559 2560			t = btf__type_by_id(btf, m->type);2561			if (!t) {2562				pr_warn("map '%s': key type [%d] not found.\n",2563					map_name, m->type);2564				return -EINVAL;2565			}2566			if (!btf_is_ptr(t)) {2567				pr_warn("map '%s': key spec is not PTR: %s.\n",2568					map_name, btf_kind_str(t));2569				return -EINVAL;2570			}2571			sz = btf__resolve_size(btf, t->type);2572			if (sz < 0) {2573				pr_warn("map '%s': can't determine key size for type [%u]: %zd.\n",2574					map_name, t->type, (ssize_t)sz);2575				return sz;2576			}2577			if (map_def->key_size && map_def->key_size != sz) {2578				pr_warn("map '%s': conflicting key size %u != %zd.\n",2579					map_name, map_def->key_size, (ssize_t)sz);2580				return -EINVAL;2581			}2582			map_def->key_size = sz;2583			map_def->key_type_id = t->type;2584			map_def->parts |= MAP_DEF_KEY_SIZE | MAP_DEF_KEY_TYPE;2585		} else if (strcmp(name, "value_size") == 0) {2586			__u32 sz;2587 2588			if (!get_map_field_int(map_name, btf, m, &sz))2589				return -EINVAL;2590			if (map_def->value_size && map_def->value_size != sz) {2591				pr_warn("map '%s': conflicting value size %u != %u.\n",2592					map_name, map_def->value_size, sz);2593				return -EINVAL;2594			}2595			map_def->value_size = sz;2596			map_def->parts |= MAP_DEF_VALUE_SIZE;2597		} else if (strcmp(name, "value") == 0) {2598			__s64 sz;2599 2600			t = btf__type_by_id(btf, m->type);2601			if (!t) {2602				pr_warn("map '%s': value type [%d] not found.\n",2603					map_name, m->type);2604				return -EINVAL;2605			}2606			if (!btf_is_ptr(t)) {2607				pr_warn("map '%s': value spec is not PTR: %s.\n",2608					map_name, btf_kind_str(t));2609				return -EINVAL;2610			}2611			sz = btf__resolve_size(btf, t->type);2612			if (sz < 0) {2613				pr_warn("map '%s': can't determine value size for type [%u]: %zd.\n",2614					map_name, t->type, (ssize_t)sz);2615				return sz;2616			}2617			if (map_def->value_size && map_def->value_size != sz) {2618				pr_warn("map '%s': conflicting value size %u != %zd.\n",2619					map_name, map_def->value_size, (ssize_t)sz);2620				return -EINVAL;2621			}2622			map_def->value_size = sz;2623			map_def->value_type_id = t->type;2624			map_def->parts |= MAP_DEF_VALUE_SIZE | MAP_DEF_VALUE_TYPE;2625		}2626		else if (strcmp(name, "values") == 0) {2627			bool is_map_in_map = bpf_map_type__is_map_in_map(map_def->map_type);2628			bool is_prog_array = map_def->map_type == BPF_MAP_TYPE_PROG_ARRAY;2629			const char *desc = is_map_in_map ? "map-in-map inner" : "prog-array value";2630			char inner_map_name[128];2631			int err;2632 2633			if (is_inner) {2634				pr_warn("map '%s': multi-level inner maps not supported.\n",2635					map_name);2636				return -ENOTSUP;2637			}2638			if (i != vlen - 1) {2639				pr_warn("map '%s': '%s' member should be last.\n",2640					map_name, name);2641				return -EINVAL;2642			}2643			if (!is_map_in_map && !is_prog_array) {2644				pr_warn("map '%s': should be map-in-map or prog-array.\n",2645					map_name);2646				return -ENOTSUP;2647			}2648			if (map_def->value_size && map_def->value_size != 4) {2649				pr_warn("map '%s': conflicting value size %u != 4.\n",2650					map_name, map_def->value_size);2651				return -EINVAL;2652			}2653			map_def->value_size = 4;2654			t = btf__type_by_id(btf, m->type);2655			if (!t) {2656				pr_warn("map '%s': %s type [%d] not found.\n",2657					map_name, desc, m->type);2658				return -EINVAL;2659			}2660			if (!btf_is_array(t) || btf_array(t)->nelems) {2661				pr_warn("map '%s': %s spec is not a zero-sized array.\n",2662					map_name, desc);2663				return -EINVAL;2664			}2665			t = skip_mods_and_typedefs(btf, btf_array(t)->type, NULL);2666			if (!btf_is_ptr(t)) {2667				pr_warn("map '%s': %s def is of unexpected kind %s.\n",2668					map_name, desc, btf_kind_str(t));2669				return -EINVAL;2670			}2671			t = skip_mods_and_typedefs(btf, t->type, NULL);2672			if (is_prog_array) {2673				if (!btf_is_func_proto(t)) {2674					pr_warn("map '%s': prog-array value def is of unexpected kind %s.\n",2675						map_name, btf_kind_str(t));2676					return -EINVAL;2677				}2678				continue;2679			}2680			if (!btf_is_struct(t)) {2681				pr_warn("map '%s': map-in-map inner def is of unexpected kind %s.\n",2682					map_name, btf_kind_str(t));2683				return -EINVAL;2684			}2685 2686			snprintf(inner_map_name, sizeof(inner_map_name), "%s.inner", map_name);2687			err = parse_btf_map_def(inner_map_name, btf, t, strict, inner_def, NULL);2688			if (err)2689				return err;2690 2691			map_def->parts |= MAP_DEF_INNER_MAP;2692		} else if (strcmp(name, "pinning") == 0) {2693			__u32 val;2694 2695			if (is_inner) {2696				pr_warn("map '%s': inner def can't be pinned.\n", map_name);2697				return -EINVAL;2698			}2699			if (!get_map_field_int(map_name, btf, m, &val))2700				return -EINVAL;2701			if (val != LIBBPF_PIN_NONE && val != LIBBPF_PIN_BY_NAME) {2702				pr_warn("map '%s': invalid pinning value %u.\n",2703					map_name, val);2704				return -EINVAL;2705			}2706			map_def->pinning = val;2707			map_def->parts |= MAP_DEF_PINNING;2708		} else if (strcmp(name, "map_extra") == 0) {2709			__u64 map_extra;2710 2711			if (!get_map_field_long(map_name, btf, m, &map_extra))2712				return -EINVAL;2713			map_def->map_extra = map_extra;2714			map_def->parts |= MAP_DEF_MAP_EXTRA;2715		} else {2716			if (strict) {2717				pr_warn("map '%s': unknown field '%s'.\n", map_name, name);2718				return -ENOTSUP;2719			}2720			pr_debug("map '%s': ignoring unknown field '%s'.\n", map_name, name);2721		}2722	}2723 2724	if (map_def->map_type == BPF_MAP_TYPE_UNSPEC) {2725		pr_warn("map '%s': map type isn't specified.\n", map_name);2726		return -EINVAL;2727	}2728 2729	return 0;2730}2731 2732static size_t adjust_ringbuf_sz(size_t sz)2733{2734	__u32 page_sz = sysconf(_SC_PAGE_SIZE);2735	__u32 mul;2736 2737	/* if user forgot to set any size, make sure they see error */2738	if (sz == 0)2739		return 0;2740	/* Kernel expects BPF_MAP_TYPE_RINGBUF's max_entries to be2741	 * a power-of-2 multiple of kernel's page size. If user diligently2742	 * satisified these conditions, pass the size through.2743	 */2744	if ((sz % page_sz) == 0 && is_pow_of_2(sz / page_sz))2745		return sz;2746 2747	/* Otherwise find closest (page_sz * power_of_2) product bigger than2748	 * user-set size to satisfy both user size request and kernel2749	 * requirements and substitute correct max_entries for map creation.2750	 */2751	for (mul = 1; mul <= UINT_MAX / page_sz; mul <<= 1) {2752		if (mul * page_sz > sz)2753			return mul * page_sz;2754	}2755 2756	/* if it's impossible to satisfy the conditions (i.e., user size is2757	 * very close to UINT_MAX but is not a power-of-2 multiple of2758	 * page_size) then just return original size and let kernel reject it2759	 */2760	return sz;2761}2762 2763static bool map_is_ringbuf(const struct bpf_map *map)2764{2765	return map->def.type == BPF_MAP_TYPE_RINGBUF ||2766	       map->def.type == BPF_MAP_TYPE_USER_RINGBUF;2767}2768 2769static void fill_map_from_def(struct bpf_map *map, const struct btf_map_def *def)2770{2771	map->def.type = def->map_type;2772	map->def.key_size = def->key_size;2773	map->def.value_size = def->value_size;2774	map->def.max_entries = def->max_entries;2775	map->def.map_flags = def->map_flags;2776	map->map_extra = def->map_extra;2777 2778	map->numa_node = def->numa_node;2779	map->btf_key_type_id = def->key_type_id;2780	map->btf_value_type_id = def->value_type_id;2781 2782	/* auto-adjust BPF ringbuf map max_entries to be a multiple of page size */2783	if (map_is_ringbuf(map))2784		map->def.max_entries = adjust_ringbuf_sz(map->def.max_entries);2785 2786	if (def->parts & MAP_DEF_MAP_TYPE)2787		pr_debug("map '%s': found type = %u.\n", map->name, def->map_type);2788 2789	if (def->parts & MAP_DEF_KEY_TYPE)2790		pr_debug("map '%s': found key [%u], sz = %u.\n",2791			 map->name, def->key_type_id, def->key_size);2792	else if (def->parts & MAP_DEF_KEY_SIZE)2793		pr_debug("map '%s': found key_size = %u.\n", map->name, def->key_size);2794 2795	if (def->parts & MAP_DEF_VALUE_TYPE)2796		pr_debug("map '%s': found value [%u], sz = %u.\n",2797			 map->name, def->value_type_id, def->value_size);2798	else if (def->parts & MAP_DEF_VALUE_SIZE)2799		pr_debug("map '%s': found value_size = %u.\n", map->name, def->value_size);2800 2801	if (def->parts & MAP_DEF_MAX_ENTRIES)2802		pr_debug("map '%s': found max_entries = %u.\n", map->name, def->max_entries);2803	if (def->parts & MAP_DEF_MAP_FLAGS)2804		pr_debug("map '%s': found map_flags = 0x%x.\n", map->name, def->map_flags);2805	if (def->parts & MAP_DEF_MAP_EXTRA)2806		pr_debug("map '%s': found map_extra = 0x%llx.\n", map->name,2807			 (unsigned long long)def->map_extra);2808	if (def->parts & MAP_DEF_PINNING)2809		pr_debug("map '%s': found pinning = %u.\n", map->name, def->pinning);2810	if (def->parts & MAP_DEF_NUMA_NODE)2811		pr_debug("map '%s': found numa_node = %u.\n", map->name, def->numa_node);2812 2813	if (def->parts & MAP_DEF_INNER_MAP)2814		pr_debug("map '%s': found inner map definition.\n", map->name);2815}2816 2817static const char *btf_var_linkage_str(__u32 linkage)2818{2819	switch (linkage) {2820	case BTF_VAR_STATIC: return "static";2821	case BTF_VAR_GLOBAL_ALLOCATED: return "global";2822	case BTF_VAR_GLOBAL_EXTERN: return "extern";2823	default: return "unknown";2824	}2825}2826 2827static int bpf_object__init_user_btf_map(struct bpf_object *obj,2828					 const struct btf_type *sec,2829					 int var_idx, int sec_idx,2830					 const Elf_Data *data, bool strict,2831					 const char *pin_root_path)2832{2833	struct btf_map_def map_def = {}, inner_def = {};2834	const struct btf_type *var, *def;2835	const struct btf_var_secinfo *vi;2836	const struct btf_var *var_extra;2837	const char *map_name;2838	struct bpf_map *map;2839	int err;2840 2841	vi = btf_var_secinfos(sec) + var_idx;2842	var = btf__type_by_id(obj->btf, vi->type);2843	var_extra = btf_var(var);2844	map_name = btf__name_by_offset(obj->btf, var->name_off);2845 2846	if (map_name == NULL || map_name[0] == '\0') {2847		pr_warn("map #%d: empty name.\n", var_idx);2848		return -EINVAL;2849	}2850	if ((__u64)vi->offset + vi->size > data->d_size) {2851		pr_warn("map '%s' BTF data is corrupted.\n", map_name);2852		return -EINVAL;2853	}2854	if (!btf_is_var(var)) {2855		pr_warn("map '%s': unexpected var kind %s.\n",2856			map_name, btf_kind_str(var));2857		return -EINVAL;2858	}2859	if (var_extra->linkage != BTF_VAR_GLOBAL_ALLOCATED) {2860		pr_warn("map '%s': unsupported map linkage %s.\n",2861			map_name, btf_var_linkage_str(var_extra->linkage));2862		return -EOPNOTSUPP;2863	}2864 2865	def = skip_mods_and_typedefs(obj->btf, var->type, NULL);2866	if (!btf_is_struct(def)) {2867		pr_warn("map '%s': unexpected def kind %s.\n",2868			map_name, btf_kind_str(var));2869		return -EINVAL;2870	}2871	if (def->size > vi->size) {2872		pr_warn("map '%s': invalid def size.\n", map_name);2873		return -EINVAL;2874	}2875 2876	map = bpf_object__add_map(obj);2877	if (IS_ERR(map))2878		return PTR_ERR(map);2879	map->name = strdup(map_name);2880	if (!map->name) {2881		pr_warn("map '%s': failed to alloc map name.\n", map_name);2882		return -ENOMEM;2883	}2884	map->libbpf_type = LIBBPF_MAP_UNSPEC;2885	map->def.type = BPF_MAP_TYPE_UNSPEC;2886	map->sec_idx = sec_idx;2887	map->sec_offset = vi->offset;2888	map->btf_var_idx = var_idx;2889	pr_debug("map '%s': at sec_idx %d, offset %zu.\n",2890		 map_name, map->sec_idx, map->sec_offset);2891 2892	err = parse_btf_map_def(map->name, obj->btf, def, strict, &map_def, &inner_def);2893	if (err)2894		return err;2895 2896	fill_map_from_def(map, &map_def);2897 2898	if (map_def.pinning == LIBBPF_PIN_BY_NAME) {2899		err = build_map_pin_path(map, pin_root_path);2900		if (err) {2901			pr_warn("map '%s': couldn't build pin path.\n", map->name);2902			return err;2903		}2904	}2905 2906	if (map_def.parts & MAP_DEF_INNER_MAP) {2907		map->inner_map = calloc(1, sizeof(*map->inner_map));2908		if (!map->inner_map)2909			return -ENOMEM;2910		map->inner_map->fd = create_placeholder_fd();2911		if (map->inner_map->fd < 0)2912			return map->inner_map->fd;2913		map->inner_map->sec_idx = sec_idx;2914		map->inner_map->name = malloc(strlen(map_name) + sizeof(".inner") + 1);2915		if (!map->inner_map->name)2916			return -ENOMEM;2917		sprintf(map->inner_map->name, "%s.inner", map_name);2918 2919		fill_map_from_def(map->inner_map, &inner_def);2920	}2921 2922	err = map_fill_btf_type_info(obj, map);2923	if (err)2924		return err;2925 2926	return 0;2927}2928 2929static int init_arena_map_data(struct bpf_object *obj, struct bpf_map *map,2930			       const char *sec_name, int sec_idx,2931			       void *data, size_t data_sz)2932{2933	const long page_sz = sysconf(_SC_PAGE_SIZE);2934	size_t mmap_sz;2935 2936	mmap_sz = bpf_map_mmap_sz(obj->arena_map);2937	if (roundup(data_sz, page_sz) > mmap_sz) {2938		pr_warn("elf: sec '%s': declared ARENA map size (%zu) is too small to hold global __arena variables of size %zu\n",2939			sec_name, mmap_sz, data_sz);2940		return -E2BIG;2941	}2942 2943	obj->arena_data = malloc(data_sz);2944	if (!obj->arena_data)2945		return -ENOMEM;2946	memcpy(obj->arena_data, data, data_sz);2947	obj->arena_data_sz = data_sz;2948 2949	/* make bpf_map__init_value() work for ARENA maps */2950	map->mmaped = obj->arena_data;2951 2952	return 0;2953}2954 2955static int bpf_object__init_user_btf_maps(struct bpf_object *obj, bool strict,2956					  const char *pin_root_path)2957{2958	const struct btf_type *sec = NULL;2959	int nr_types, i, vlen, err;2960	const struct btf_type *t;2961	const char *name;2962	Elf_Data *data;2963	Elf_Scn *scn;2964 2965	if (obj->efile.btf_maps_shndx < 0)2966		return 0;2967 2968	scn = elf_sec_by_idx(obj, obj->efile.btf_maps_shndx);2969	data = elf_sec_data(obj, scn);2970	if (!scn || !data) {2971		pr_warn("elf: failed to get %s map definitions for %s\n",2972			MAPS_ELF_SEC, obj->path);2973		return -EINVAL;2974	}2975 2976	nr_types = btf__type_cnt(obj->btf);2977	for (i = 1; i < nr_types; i++) {2978		t = btf__type_by_id(obj->btf, i);2979		if (!btf_is_datasec(t))2980			continue;2981		name = btf__name_by_offset(obj->btf, t->name_off);2982		if (strcmp(name, MAPS_ELF_SEC) == 0) {2983			sec = t;2984			obj->efile.btf_maps_sec_btf_id = i;2985			break;2986		}2987	}2988 2989	if (!sec) {2990		pr_warn("DATASEC '%s' not found.\n", MAPS_ELF_SEC);2991		return -ENOENT;2992	}2993 2994	vlen = btf_vlen(sec);2995	for (i = 0; i < vlen; i++) {2996		err = bpf_object__init_user_btf_map(obj, sec, i,2997						    obj->efile.btf_maps_shndx,2998						    data, strict,2999						    pin_root_path);3000		if (err)3001			return err;3002	}3003 3004	for (i = 0; i < obj->nr_maps; i++) {3005		struct bpf_map *map = &obj->maps[i];3006 3007		if (map->def.type != BPF_MAP_TYPE_ARENA)3008			continue;3009 3010		if (obj->arena_map) {3011			pr_warn("map '%s': only single ARENA map is supported (map '%s' is also ARENA)\n",3012				map->name, obj->arena_map->name);3013			return -EINVAL;3014		}3015		obj->arena_map = map;3016 3017		if (obj->efile.arena_data) {3018			err = init_arena_map_data(obj, map, ARENA_SEC, obj->efile.arena_data_shndx,3019						  obj->efile.arena_data->d_buf,3020						  obj->efile.arena_data->d_size);3021			if (err)3022				return err;3023		}3024	}3025	if (obj->efile.arena_data && !obj->arena_map) {3026		pr_warn("elf: sec '%s': to use global __arena variables the ARENA map should be explicitly declared in SEC(\".maps\")\n",3027			ARENA_SEC);3028		return -ENOENT;3029	}3030 3031	return 0;3032}3033 3034static int bpf_object__init_maps(struct bpf_object *obj,3035				 const struct bpf_object_open_opts *opts)3036{3037	const char *pin_root_path;3038	bool strict;3039	int err = 0;3040 3041	strict = !OPTS_GET(opts, relaxed_maps, false);3042	pin_root_path = OPTS_GET(opts, pin_root_path, NULL);3043 3044	err = bpf_object__init_user_btf_maps(obj, strict, pin_root_path);3045	err = err ?: bpf_object__init_global_data_maps(obj);3046	err = err ?: bpf_object__init_kconfig_map(obj);3047	err = err ?: bpf_object_init_struct_ops(obj);3048 3049	return err;3050}3051 3052static bool section_have_execinstr(struct bpf_object *obj, int idx)3053{3054	Elf64_Shdr *sh;3055 3056	sh = elf_sec_hdr(obj, elf_sec_by_idx(obj, idx));3057	if (!sh)3058		return false;3059 3060	return sh->sh_flags & SHF_EXECINSTR;3061}3062 3063static bool starts_with_qmark(const char *s)3064{3065	return s && s[0] == '?';3066}3067 3068static bool btf_needs_sanitization(struct bpf_object *obj)3069{3070	bool has_func_global = kernel_supports(obj, FEAT_BTF_GLOBAL_FUNC);3071	bool has_datasec = kernel_supports(obj, FEAT_BTF_DATASEC);3072	bool has_float = kernel_supports(obj, FEAT_BTF_FLOAT);3073	bool has_func = kernel_supports(obj, FEAT_BTF_FUNC);3074	bool has_decl_tag = kernel_supports(obj, FEAT_BTF_DECL_TAG);3075	bool has_type_tag = kernel_supports(obj, FEAT_BTF_TYPE_TAG);3076	bool has_enum64 = kernel_supports(obj, FEAT_BTF_ENUM64);3077	bool has_qmark_datasec = kernel_supports(obj, FEAT_BTF_QMARK_DATASEC);3078 3079	return !has_func || !has_datasec || !has_func_global || !has_float ||3080	       !has_decl_tag || !has_type_tag || !has_enum64 || !has_qmark_datasec;3081}3082 3083static int bpf_object__sanitize_btf(struct bpf_object *obj, struct btf *btf)3084{3085	bool has_func_global = kernel_supports(obj, FEAT_BTF_GLOBAL_FUNC);3086	bool has_datasec = kernel_supports(obj, FEAT_BTF_DATASEC);3087	bool has_float = kernel_supports(obj, FEAT_BTF_FLOAT);3088	bool has_func = kernel_supports(obj, FEAT_BTF_FUNC);3089	bool has_decl_tag = kernel_supports(obj, FEAT_BTF_DECL_TAG);3090	bool has_type_tag = kernel_supports(obj, FEAT_BTF_TYPE_TAG);3091	bool has_enum64 = kernel_supports(obj, FEAT_BTF_ENUM64);3092	bool has_qmark_datasec = kernel_supports(obj, FEAT_BTF_QMARK_DATASEC);3093	int enum64_placeholder_id = 0;3094	struct btf_type *t;3095	int i, j, vlen;3096 3097	for (i = 1; i < btf__type_cnt(btf); i++) {3098		t = (struct btf_type *)btf__type_by_id(btf, i);3099 3100		if ((!has_datasec && btf_is_var(t)) || (!has_decl_tag && btf_is_decl_tag(t))) {3101			/* replace VAR/DECL_TAG with INT */3102			t->info = BTF_INFO_ENC(BTF_KIND_INT, 0, 0);3103			/*3104			 * using size = 1 is the safest choice, 4 will be too3105			 * big and cause kernel BTF validation failure if3106			 * original variable took less than 4 bytes3107			 */3108			t->size = 1;3109			*(int *)(t + 1) = BTF_INT_ENC(0, 0, 8);3110		} else if (!has_datasec && btf_is_datasec(t)) {3111			/* replace DATASEC with STRUCT */3112			const struct btf_var_secinfo *v = btf_var_secinfos(t);3113			struct btf_member *m = btf_members(t);3114			struct btf_type *vt;3115			char *name;3116 3117			name = (char *)btf__name_by_offset(btf, t->name_off);3118			while (*name) {3119				if (*name == '.' || *name == '?')3120					*name = '_';3121				name++;3122			}3123 3124			vlen = btf_vlen(t);3125			t->info = BTF_INFO_ENC(BTF_KIND_STRUCT, 0, vlen);3126			for (j = 0; j < vlen; j++, v++, m++) {3127				/* order of field assignments is important */3128				m->offset = v->offset * 8;3129				m->type = v->type;3130				/* preserve variable name as member name */3131				vt = (void *)btf__type_by_id(btf, v->type);3132				m->name_off = vt->name_off;3133			}3134		} else if (!has_qmark_datasec && btf_is_datasec(t) &&3135			   starts_with_qmark(btf__name_by_offset(btf, t->name_off))) {3136			/* replace '?' prefix with '_' for DATASEC names */3137			char *name;3138 3139			name = (char *)btf__name_by_offset(btf, t->name_off);3140			if (name[0] == '?')3141				name[0] = '_';3142		} else if (!has_func && btf_is_func_proto(t)) {3143			/* replace FUNC_PROTO with ENUM */3144			vlen = btf_vlen(t);3145			t->info = BTF_INFO_ENC(BTF_KIND_ENUM, 0, vlen);3146			t->size = sizeof(__u32); /* kernel enforced */3147		} else if (!has_func && btf_is_func(t)) {3148			/* replace FUNC with TYPEDEF */3149			t->info = BTF_INFO_ENC(BTF_KIND_TYPEDEF, 0, 0);3150		} else if (!has_func_global && btf_is_func(t)) {3151			/* replace BTF_FUNC_GLOBAL with BTF_FUNC_STATIC */3152			t->info = BTF_INFO_ENC(BTF_KIND_FUNC, 0, 0);3153		} else if (!has_float && btf_is_float(t)) {3154			/* replace FLOAT with an equally-sized empty STRUCT;3155			 * since C compilers do not accept e.g. "float" as a3156			 * valid struct name, make it anonymous3157			 */3158			t->name_off = 0;3159			t->info = BTF_INFO_ENC(BTF_KIND_STRUCT, 0, 0);3160		} else if (!has_type_tag && btf_is_type_tag(t)) {3161			/* replace TYPE_TAG with a CONST */3162			t->name_off = 0;3163			t->info = BTF_INFO_ENC(BTF_KIND_CONST, 0, 0);3164		} else if (!has_enum64 && btf_is_enum(t)) {3165			/* clear the kflag */3166			t->info = btf_type_info(btf_kind(t), btf_vlen(t), false);3167		} else if (!has_enum64 && btf_is_enum64(t)) {3168			/* replace ENUM64 with a union */3169			struct btf_member *m;3170 3171			if (enum64_placeholder_id == 0) {3172				enum64_placeholder_id = btf__add_int(btf, "enum64_placeholder", 1, 0);3173				if (enum64_placeholder_id < 0)3174					return enum64_placeholder_id;3175 3176				t = (struct btf_type *)btf__type_by_id(btf, i);3177			}3178 3179			m = btf_members(t);3180			vlen = btf_vlen(t);3181			t->info = BTF_INFO_ENC(BTF_KIND_UNION, 0, vlen);3182			for (j = 0; j < vlen; j++, m++) {3183				m->type = enum64_placeholder_id;3184				m->offset = 0;3185			}3186		}3187	}3188 3189	return 0;3190}3191 3192static bool libbpf_needs_btf(const struct bpf_object *obj)3193{3194	return obj->efile.btf_maps_shndx >= 0 ||3195	       obj->efile.has_st_ops ||3196	       obj->nr_extern > 0;3197}3198 3199static bool kernel_needs_btf(const struct bpf_object *obj)3200{3201	return obj->efile.has_st_ops;3202}3203 3204static int bpf_object__init_btf(struct bpf_object *obj,3205				Elf_Data *btf_data,3206				Elf_Data *btf_ext_data)3207{3208	int err = -ENOENT;3209 3210	if (btf_data) {3211		obj->btf = btf__new(btf_data->d_buf, btf_data->d_size);3212		err = libbpf_get_error(obj->btf);3213		if (err) {3214			obj->btf = NULL;3215			pr_warn("Error loading ELF section %s: %d.\n", BTF_ELF_SEC, err);3216			goto out;3217		}3218		/* enforce 8-byte pointers for BPF-targeted BTFs */3219		btf__set_pointer_size(obj->btf, 8);3220	}3221	if (btf_ext_data) {3222		struct btf_ext_info *ext_segs[3];3223		int seg_num, sec_num;3224 3225		if (!obj->btf) {3226			pr_debug("Ignore ELF section %s because its depending ELF section %s is not found.\n",3227				 BTF_EXT_ELF_SEC, BTF_ELF_SEC);3228			goto out;3229		}3230		obj->btf_ext = btf_ext__new(btf_ext_data->d_buf, btf_ext_data->d_size);3231		err = libbpf_get_error(obj->btf_ext);3232		if (err) {3233			pr_warn("Error loading ELF section %s: %d. Ignored and continue.\n",3234				BTF_EXT_ELF_SEC, err);3235			obj->btf_ext = NULL;3236			goto out;3237		}3238 3239		/* setup .BTF.ext to ELF section mapping */3240		ext_segs[0] = &obj->btf_ext->func_info;3241		ext_segs[1] = &obj->btf_ext->line_info;3242		ext_segs[2] = &obj->btf_ext->core_relo_info;3243		for (seg_num = 0; seg_num < ARRAY_SIZE(ext_segs); seg_num++) {3244			struct btf_ext_info *seg = ext_segs[seg_num];3245			const struct btf_ext_info_sec *sec;3246			const char *sec_name;3247			Elf_Scn *scn;3248 3249			if (seg->sec_cnt == 0)3250				continue;3251 3252			seg->sec_idxs = calloc(seg->sec_cnt, sizeof(*seg->sec_idxs));3253			if (!seg->sec_idxs) {3254				err = -ENOMEM;3255				goto out;3256			}3257 3258			sec_num = 0;3259			for_each_btf_ext_sec(seg, sec) {3260				/* preventively increment index to avoid doing3261				 * this before every continue below3262				 */3263				sec_num++;3264 3265				sec_name = btf__name_by_offset(obj->btf, sec->sec_name_off);3266				if (str_is_empty(sec_name))3267					continue;3268				scn = elf_sec_by_name(obj, sec_name);3269				if (!scn)3270					continue;3271 3272				seg->sec_idxs[sec_num - 1] = elf_ndxscn(scn);3273			}3274		}3275	}3276out:3277	if (err && libbpf_needs_btf(obj)) {3278		pr_warn("BTF is required, but is missing or corrupted.\n");3279		return err;3280	}3281	return 0;3282}3283 3284static int compare_vsi_off(const void *_a, const void *_b)3285{3286	const struct btf_var_secinfo *a = _a;3287	const struct btf_var_secinfo *b = _b;3288 3289	return a->offset - b->offset;3290}3291 3292static int btf_fixup_datasec(struct bpf_object *obj, struct btf *btf,3293			     struct btf_type *t)3294{3295	__u32 size = 0, i, vars = btf_vlen(t);3296	const char *sec_name = btf__name_by_offset(btf, t->name_off);3297	struct btf_var_secinfo *vsi;3298	bool fixup_offsets = false;3299	int err;3300 3301	if (!sec_name) {3302		pr_debug("No name found in string section for DATASEC kind.\n");3303		return -ENOENT;3304	}3305 3306	/* Extern-backing datasecs (.ksyms, .kconfig) have their size and3307	 * variable offsets set at the previous step. Further, not every3308	 * extern BTF VAR has corresponding ELF symbol preserved, so we skip3309	 * all fixups altogether for such sections and go straight to sorting3310	 * VARs within their DATASEC.3311	 */3312	if (strcmp(sec_name, KCONFIG_SEC) == 0 || strcmp(sec_name, KSYMS_SEC) == 0)3313		goto sort_vars;3314 3315	/* Clang leaves DATASEC size and VAR offsets as zeroes, so we need to3316	 * fix this up. But BPF static linker already fixes this up and fills3317	 * all the sizes and offsets during static linking. So this step has3318	 * to be optional. But the STV_HIDDEN handling is non-optional for any3319	 * non-extern DATASEC, so the variable fixup loop below handles both3320	 * functions at the same time, paying the cost of BTF VAR <-> ELF3321	 * symbol matching just once.3322	 */3323	if (t->size == 0) {3324		err = find_elf_sec_sz(obj, sec_name, &size);3325		if (err || !size) {3326			pr_debug("sec '%s': failed to determine size from ELF: size %u, err %d\n",3327				 sec_name, size, err);3328			return -ENOENT;3329		}3330 3331		t->size = size;3332		fixup_offsets = true;3333	}3334 3335	for (i = 0, vsi = btf_var_secinfos(t); i < vars; i++, vsi++) {3336		const struct btf_type *t_var;3337		struct btf_var *var;3338		const char *var_name;3339		Elf64_Sym *sym;3340 3341		t_var = btf__type_by_id(btf, vsi->type);3342		if (!t_var || !btf_is_var(t_var)) {3343			pr_debug("sec '%s': unexpected non-VAR type found\n", sec_name);3344			return -EINVAL;3345		}3346 3347		var = btf_var(t_var);3348		if (var->linkage == BTF_VAR_STATIC || var->linkage == BTF_VAR_GLOBAL_EXTERN)3349			continue;3350 3351		var_name = btf__name_by_offset(btf, t_var->name_off);3352		if (!var_name) {3353			pr_debug("sec '%s': failed to find name of DATASEC's member #%d\n",3354				 sec_name, i);3355			return -ENOENT;3356		}3357 3358		sym = find_elf_var_sym(obj, var_name);3359		if (IS_ERR(sym)) {3360			pr_debug("sec '%s': failed to find ELF symbol for VAR '%s'\n",3361				 sec_name, var_name);3362			return -ENOENT;3363		}3364 3365		if (fixup_offsets)3366			vsi->offset = sym->st_value;3367 3368		/* if variable is a global/weak symbol, but has restricted3369		 * (STV_HIDDEN or STV_INTERNAL) visibility, mark its BTF VAR3370		 * as static. This follows similar logic for functions (BPF3371		 * subprogs) and influences libbpf's further decisions about3372		 * whether to make global data BPF array maps as3373		 * BPF_F_MMAPABLE.3374		 */3375		if (ELF64_ST_VISIBILITY(sym->st_other) == STV_HIDDEN3376		    || ELF64_ST_VISIBILITY(sym->st_other) == STV_INTERNAL)3377			var->linkage = BTF_VAR_STATIC;3378	}3379 3380sort_vars:3381	qsort(btf_var_secinfos(t), vars, sizeof(*vsi), compare_vsi_off);3382	return 0;3383}3384 3385static int bpf_object_fixup_btf(struct bpf_object *obj)3386{3387	int i, n, err = 0;3388 3389	if (!obj->btf)3390		return 0;3391 3392	n = btf__type_cnt(obj->btf);3393	for (i = 1; i < n; i++) {3394		struct btf_type *t = btf_type_by_id(obj->btf, i);3395 3396		/* Loader needs to fix up some of the things compiler3397		 * couldn't get its hands on while emitting BTF. This3398		 * is section size and global variable offset. We use3399		 * the info from the ELF itself for this purpose.3400		 */3401		if (btf_is_datasec(t)) {3402			err = btf_fixup_datasec(obj, obj->btf, t);3403			if (err)3404				return err;3405		}3406	}3407 3408	return 0;3409}3410 3411static bool prog_needs_vmlinux_btf(struct bpf_program *prog)3412{3413	if (prog->type == BPF_PROG_TYPE_STRUCT_OPS ||3414	    prog->type == BPF_PROG_TYPE_LSM)3415		return true;3416 3417	/* BPF_PROG_TYPE_TRACING programs which do not attach to other programs3418	 * also need vmlinux BTF3419	 */3420	if (prog->type == BPF_PROG_TYPE_TRACING && !prog->attach_prog_fd)3421		return true;3422 3423	return false;3424}3425 3426static bool map_needs_vmlinux_btf(struct bpf_map *map)3427{3428	return bpf_map__is_struct_ops(map);3429}3430 3431static bool obj_needs_vmlinux_btf(const struct bpf_object *obj)3432{3433	struct bpf_program *prog;3434	struct bpf_map *map;3435	int i;3436 3437	/* CO-RE relocations need kernel BTF, only when btf_custom_path3438	 * is not specified3439	 */3440	if (obj->btf_ext && obj->btf_ext->core_relo_info.len && !obj->btf_custom_path)3441		return true;3442 3443	/* Support for typed ksyms needs kernel BTF */3444	for (i = 0; i < obj->nr_extern; i++) {3445		const struct extern_desc *ext;3446 3447		ext = &obj->externs[i];3448		if (ext->type == EXT_KSYM && ext->ksym.type_id)3449			return true;3450	}3451 3452	bpf_object__for_each_program(prog, obj) {3453		if (!prog->autoload)3454			continue;3455		if (prog_needs_vmlinux_btf(prog))3456			return true;3457	}3458 3459	bpf_object__for_each_map(map, obj) {3460		if (map_needs_vmlinux_btf(map))3461			return true;3462	}3463 3464	return false;3465}3466 3467static int bpf_object__load_vmlinux_btf(struct bpf_object *obj, bool force)3468{3469	int err;3470 3471	/* btf_vmlinux could be loaded earlier */3472	if (obj->btf_vmlinux || obj->gen_loader)3473		return 0;3474 3475	if (!force && !obj_needs_vmlinux_btf(obj))3476		return 0;3477 3478	obj->btf_vmlinux = btf__load_vmlinux_btf();3479	err = libbpf_get_error(obj->btf_vmlinux);3480	if (err) {3481		pr_warn("Error loading vmlinux BTF: %d\n", err);3482		obj->btf_vmlinux = NULL;3483		return err;3484	}3485	return 0;3486}3487 3488static int bpf_object__sanitize_and_load_btf(struct bpf_object *obj)3489{3490	struct btf *kern_btf = obj->btf;3491	bool btf_mandatory, sanitize;3492	int i, err = 0;3493 3494	if (!obj->btf)3495		return 0;3496 3497	if (!kernel_supports(obj, FEAT_BTF)) {3498		if (kernel_needs_btf(obj)) {3499			err = -EOPNOTSUPP;3500			goto report;3501		}3502		pr_debug("Kernel doesn't support BTF, skipping uploading it.\n");3503		return 0;3504	}3505 3506	/* Even though some subprogs are global/weak, user might prefer more3507	 * permissive BPF verification process that BPF verifier performs for3508	 * static functions, taking into account more context from the caller3509	 * functions. In such case, they need to mark such subprogs with3510	 * __attribute__((visibility("hidden"))) and libbpf will adjust3511	 * corresponding FUNC BTF type to be marked as static and trigger more3512	 * involved BPF verification process.3513	 */3514	for (i = 0; i < obj->nr_programs; i++) {3515		struct bpf_program *prog = &obj->programs[i];3516		struct btf_type *t;3517		const char *name;3518		int j, n;3519 3520		if (!prog->mark_btf_static || !prog_is_subprog(obj, prog))3521			continue;3522 3523		n = btf__type_cnt(obj->btf);3524		for (j = 1; j < n; j++) {3525			t = btf_type_by_id(obj->btf, j);3526			if (!btf_is_func(t) || btf_func_linkage(t) != BTF_FUNC_GLOBAL)3527				continue;3528 3529			name = btf__str_by_offset(obj->btf, t->name_off);3530			if (strcmp(name, prog->name) != 0)3531				continue;3532 3533			t->info = btf_type_info(BTF_KIND_FUNC, BTF_FUNC_STATIC, 0);3534			break;3535		}3536	}3537 3538	sanitize = btf_needs_sanitization(obj);3539	if (sanitize) {3540		const void *raw_data;3541		__u32 sz;3542 3543		/* clone BTF to sanitize a copy and leave the original intact */3544		raw_data = btf__raw_data(obj->btf, &sz);3545		kern_btf = btf__new(raw_data, sz);3546		err = libbpf_get_error(kern_btf);3547		if (err)3548			return err;3549 3550		/* enforce 8-byte pointers for BPF-targeted BTFs */3551		btf__set_pointer_size(obj->btf, 8);3552		err = bpf_object__sanitize_btf(obj, kern_btf);3553		if (err)3554			return err;3555	}3556 3557	if (obj->gen_loader) {3558		__u32 raw_size = 0;3559		const void *raw_data = btf__raw_data(kern_btf, &raw_size);3560 3561		if (!raw_data)3562			return -ENOMEM;3563		bpf_gen__load_btf(obj->gen_loader, raw_data, raw_size);3564		/* Pretend to have valid FD to pass various fd >= 0 checks.3565		 * This fd == 0 will not be used with any syscall and will be reset to -1 eventually.3566		 */3567		btf__set_fd(kern_btf, 0);3568	} else {3569		/* currently BPF_BTF_LOAD only supports log_level 1 */3570		err = btf_load_into_kernel(kern_btf, obj->log_buf, obj->log_size,3571					   obj->log_level ? 1 : 0, obj->token_fd);3572	}3573	if (sanitize) {3574		if (!err) {3575			/* move fd to libbpf's BTF */3576			btf__set_fd(obj->btf, btf__fd(kern_btf));3577			btf__set_fd(kern_btf, -1);3578		}3579		btf__free(kern_btf);3580	}3581report:3582	if (err) {3583		btf_mandatory = kernel_needs_btf(obj);3584		pr_warn("Error loading .BTF into kernel: %d. %s\n", err,3585			btf_mandatory ? "BTF is mandatory, can't proceed."3586				      : "BTF is optional, ignoring.");3587		if (!btf_mandatory)3588			err = 0;3589	}3590	return err;3591}3592 3593static const char *elf_sym_str(const struct bpf_object *obj, size_t off)3594{3595	const char *name;3596 3597	name = elf_strptr(obj->efile.elf, obj->efile.strtabidx, off);3598	if (!name) {3599		pr_warn("elf: failed to get section name string at offset %zu from %s: %s\n",3600			off, obj->path, elf_errmsg(-1));3601		return NULL;3602	}3603 3604	return name;3605}3606 3607static const char *elf_sec_str(const struct bpf_object *obj, size_t off)3608{3609	const char *name;3610 3611	name = elf_strptr(obj->efile.elf, obj->efile.shstrndx, off);3612	if (!name) {3613		pr_warn("elf: failed to get section name string at offset %zu from %s: %s\n",3614			off, obj->path, elf_errmsg(-1));3615		return NULL;3616	}3617 3618	return name;3619}3620 3621static Elf_Scn *elf_sec_by_idx(const struct bpf_object *obj, size_t idx)3622{3623	Elf_Scn *scn;3624 3625	scn = elf_getscn(obj->efile.elf, idx);3626	if (!scn) {3627		pr_warn("elf: failed to get section(%zu) from %s: %s\n",3628			idx, obj->path, elf_errmsg(-1));3629		return NULL;3630	}3631	return scn;3632}3633 3634static Elf_Scn *elf_sec_by_name(const struct bpf_object *obj, const char *name)3635{3636	Elf_Scn *scn = NULL;3637	Elf *elf = obj->efile.elf;3638	const char *sec_name;3639 3640	while ((scn = elf_nextscn(elf, scn)) != NULL) {3641		sec_name = elf_sec_name(obj, scn);3642		if (!sec_name)3643			return NULL;3644 3645		if (strcmp(sec_name, name) != 0)3646			continue;3647 3648		return scn;3649	}3650	return NULL;3651}3652 3653static Elf64_Shdr *elf_sec_hdr(const struct bpf_object *obj, Elf_Scn *scn)3654{3655	Elf64_Shdr *shdr;3656 3657	if (!scn)3658		return NULL;3659 3660	shdr = elf64_getshdr(scn);3661	if (!shdr) {3662		pr_warn("elf: failed to get section(%zu) header from %s: %s\n",3663			elf_ndxscn(scn), obj->path, elf_errmsg(-1));3664		return NULL;3665	}3666 3667	return shdr;3668}3669 3670static const char *elf_sec_name(const struct bpf_object *obj, Elf_Scn *scn)3671{3672	const char *name;3673	Elf64_Shdr *sh;3674 3675	if (!scn)3676		return NULL;3677 3678	sh = elf_sec_hdr(obj, scn);3679	if (!sh)3680		return NULL;3681 3682	name = elf_sec_str(obj, sh->sh_name);3683	if (!name) {3684		pr_warn("elf: failed to get section(%zu) name from %s: %s\n",3685			elf_ndxscn(scn), obj->path, elf_errmsg(-1));3686		return NULL;3687	}3688 3689	return name;3690}3691 3692static Elf_Data *elf_sec_data(const struct bpf_object *obj, Elf_Scn *scn)3693{3694	Elf_Data *data;3695 3696	if (!scn)3697		return NULL;3698 3699	data = elf_getdata(scn, 0);3700	if (!data) {3701		pr_warn("elf: failed to get section(%zu) %s data from %s: %s\n",3702			elf_ndxscn(scn), elf_sec_name(obj, scn) ?: "<?>",3703			obj->path, elf_errmsg(-1));3704		return NULL;3705	}3706 3707	return data;3708}3709 3710static Elf64_Sym *elf_sym_by_idx(const struct bpf_object *obj, size_t idx)3711{3712	if (idx >= obj->efile.symbols->d_size / sizeof(Elf64_Sym))3713		return NULL;3714 3715	return (Elf64_Sym *)obj->efile.symbols->d_buf + idx;3716}3717 3718static Elf64_Rel *elf_rel_by_idx(Elf_Data *data, size_t idx)3719{3720	if (idx >= data->d_size / sizeof(Elf64_Rel))3721		return NULL;3722 3723	return (Elf64_Rel *)data->d_buf + idx;3724}3725 3726static bool is_sec_name_dwarf(const char *name)3727{3728	/* approximation, but the actual list is too long */3729	return str_has_pfx(name, ".debug_");3730}3731 3732static bool ignore_elf_section(Elf64_Shdr *hdr, const char *name)3733{3734	/* no special handling of .strtab */3735	if (hdr->sh_type == SHT_STRTAB)3736		return true;3737 3738	/* ignore .llvm_addrsig section as well */3739	if (hdr->sh_type == SHT_LLVM_ADDRSIG)3740		return true;3741 3742	/* no subprograms will lead to an empty .text section, ignore it */3743	if (hdr->sh_type == SHT_PROGBITS && hdr->sh_size == 0 &&3744	    strcmp(name, ".text") == 0)3745		return true;3746 3747	/* DWARF sections */3748	if (is_sec_name_dwarf(name))3749		return true;3750 3751	if (str_has_pfx(name, ".rel")) {3752		name += sizeof(".rel") - 1;3753		/* DWARF section relocations */3754		if (is_sec_name_dwarf(name))3755			return true;3756 3757		/* .BTF and .BTF.ext don't need relocations */3758		if (strcmp(name, BTF_ELF_SEC) == 0 ||3759		    strcmp(name, BTF_EXT_ELF_SEC) == 0)3760			return true;3761	}3762 3763	return false;3764}3765 3766static int cmp_progs(const void *_a, const void *_b)3767{3768	const struct bpf_program *a = _a;3769	const struct bpf_program *b = _b;3770 3771	if (a->sec_idx != b->sec_idx)3772		return a->sec_idx < b->sec_idx ? -1 : 1;3773 3774	/* sec_insn_off can't be the same within the section */3775	return a->sec_insn_off < b->sec_insn_off ? -1 : 1;3776}3777 3778static int bpf_object__elf_collect(struct bpf_object *obj)3779{3780	struct elf_sec_desc *sec_desc;3781	Elf *elf = obj->efile.elf;3782	Elf_Data *btf_ext_data = NULL;3783	Elf_Data *btf_data = NULL;3784	int idx = 0, err = 0;3785	const char *name;3786	Elf_Data *data;3787	Elf_Scn *scn;3788	Elf64_Shdr *sh;3789 3790	/* ELF section indices are 0-based, but sec #0 is special "invalid"3791	 * section. Since section count retrieved by elf_getshdrnum() does3792	 * include sec #0, it is already the necessary size of an array to keep3793	 * all the sections.3794	 */3795	if (elf_getshdrnum(obj->efile.elf, &obj->efile.sec_cnt)) {3796		pr_warn("elf: failed to get the number of sections for %s: %s\n",3797			obj->path, elf_errmsg(-1));3798		return -LIBBPF_ERRNO__FORMAT;3799	}3800	obj->efile.secs = calloc(obj->efile.sec_cnt, sizeof(*obj->efile.secs));3801	if (!obj->efile.secs)3802		return -ENOMEM;3803 3804	/* a bunch of ELF parsing functionality depends on processing symbols,3805	 * so do the first pass and find the symbol table3806	 */3807	scn = NULL;3808	while ((scn = elf_nextscn(elf, scn)) != NULL) {3809		sh = elf_sec_hdr(obj, scn);3810		if (!sh)3811			return -LIBBPF_ERRNO__FORMAT;3812 3813		if (sh->sh_type == SHT_SYMTAB) {3814			if (obj->efile.symbols) {3815				pr_warn("elf: multiple symbol tables in %s\n", obj->path);3816				return -LIBBPF_ERRNO__FORMAT;3817			}3818 3819			data = elf_sec_data(obj, scn);3820			if (!data)3821				return -LIBBPF_ERRNO__FORMAT;3822 3823			idx = elf_ndxscn(scn);3824 3825			obj->efile.symbols = data;3826			obj->efile.symbols_shndx = idx;3827			obj->efile.strtabidx = sh->sh_link;3828		}3829	}3830 3831	if (!obj->efile.symbols) {3832		pr_warn("elf: couldn't find symbol table in %s, stripped object file?\n",3833			obj->path);3834		return -ENOENT;3835	}3836 3837	scn = NULL;3838	while ((scn = elf_nextscn(elf, scn)) != NULL) {3839		idx = elf_ndxscn(scn);3840		sec_desc = &obj->efile.secs[idx];3841 3842		sh = elf_sec_hdr(obj, scn);3843		if (!sh)3844			return -LIBBPF_ERRNO__FORMAT;3845 3846		name = elf_sec_str(obj, sh->sh_name);3847		if (!name)3848			return -LIBBPF_ERRNO__FORMAT;3849 3850		if (ignore_elf_section(sh, name))3851			continue;3852 3853		data = elf_sec_data(obj, scn);3854		if (!data)3855			return -LIBBPF_ERRNO__FORMAT;3856 3857		pr_debug("elf: section(%d) %s, size %ld, link %d, flags %lx, type=%d\n",3858			 idx, name, (unsigned long)data->d_size,3859			 (int)sh->sh_link, (unsigned long)sh->sh_flags,3860			 (int)sh->sh_type);3861 3862		if (strcmp(name, "license") == 0) {3863			err = bpf_object__init_license(obj, data->d_buf, data->d_size);3864			if (err)3865				return err;3866		} else if (strcmp(name, "version") == 0) {3867			err = bpf_object__init_kversion(obj, data->d_buf, data->d_size);3868			if (err)3869				return err;3870		} else if (strcmp(name, "maps") == 0) {3871			pr_warn("elf: legacy map definitions in 'maps' section are not supported by libbpf v1.0+\n");3872			return -ENOTSUP;3873		} else if (strcmp(name, MAPS_ELF_SEC) == 0) {3874			obj->efile.btf_maps_shndx = idx;3875		} else if (strcmp(name, BTF_ELF_SEC) == 0) {3876			if (sh->sh_type != SHT_PROGBITS)3877				return -LIBBPF_ERRNO__FORMAT;3878			btf_data = data;3879		} else if (strcmp(name, BTF_EXT_ELF_SEC) == 0) {3880			if (sh->sh_type != SHT_PROGBITS)3881				return -LIBBPF_ERRNO__FORMAT;3882			btf_ext_data = data;3883		} else if (sh->sh_type == SHT_SYMTAB) {3884			/* already processed during the first pass above */3885		} else if (sh->sh_type == SHT_PROGBITS && data->d_size > 0) {3886			if (sh->sh_flags & SHF_EXECINSTR) {3887				if (strcmp(name, ".text") == 0)3888					obj->efile.text_shndx = idx;3889				err = bpf_object__add_programs(obj, data, name, idx);3890				if (err)3891					return err;3892			} else if (strcmp(name, DATA_SEC) == 0 ||3893				   str_has_pfx(name, DATA_SEC ".")) {3894				sec_desc->sec_type = SEC_DATA;3895				sec_desc->shdr = sh;3896				sec_desc->data = data;3897			} else if (strcmp(name, RODATA_SEC) == 0 ||3898				   str_has_pfx(name, RODATA_SEC ".")) {3899				sec_desc->sec_type = SEC_RODATA;3900				sec_desc->shdr = sh;3901				sec_desc->data = data;3902			} else if (strcmp(name, STRUCT_OPS_SEC) == 0 ||3903				   strcmp(name, STRUCT_OPS_LINK_SEC) == 0 ||3904				   strcmp(name, "?" STRUCT_OPS_SEC) == 0 ||3905				   strcmp(name, "?" STRUCT_OPS_LINK_SEC) == 0) {3906				sec_desc->sec_type = SEC_ST_OPS;3907				sec_desc->shdr = sh;3908				sec_desc->data = data;3909				obj->efile.has_st_ops = true;3910			} else if (strcmp(name, ARENA_SEC) == 0) {3911				obj->efile.arena_data = data;3912				obj->efile.arena_data_shndx = idx;3913			} else {3914				pr_info("elf: skipping unrecognized data section(%d) %s\n",3915					idx, name);3916			}3917		} else if (sh->sh_type == SHT_REL) {3918			int targ_sec_idx = sh->sh_info; /* points to other section */3919 3920			if (sh->sh_entsize != sizeof(Elf64_Rel) ||3921			    targ_sec_idx >= obj->efile.sec_cnt)3922				return -LIBBPF_ERRNO__FORMAT;3923 3924			/* Only do relo for section with exec instructions */3925			if (!section_have_execinstr(obj, targ_sec_idx) &&3926			    strcmp(name, ".rel" STRUCT_OPS_SEC) &&3927			    strcmp(name, ".rel" STRUCT_OPS_LINK_SEC) &&3928			    strcmp(name, ".rel?" STRUCT_OPS_SEC) &&3929			    strcmp(name, ".rel?" STRUCT_OPS_LINK_SEC) &&3930			    strcmp(name, ".rel" MAPS_ELF_SEC)) {3931				pr_info("elf: skipping relo section(%d) %s for section(%d) %s\n",3932					idx, name, targ_sec_idx,3933					elf_sec_name(obj, elf_sec_by_idx(obj, targ_sec_idx)) ?: "<?>");3934				continue;3935			}3936 3937			sec_desc->sec_type = SEC_RELO;3938			sec_desc->shdr = sh;3939			sec_desc->data = data;3940		} else if (sh->sh_type == SHT_NOBITS && (strcmp(name, BSS_SEC) == 0 ||3941							 str_has_pfx(name, BSS_SEC "."))) {3942			sec_desc->sec_type = SEC_BSS;3943			sec_desc->shdr = sh;3944			sec_desc->data = data;3945		} else {3946			pr_info("elf: skipping section(%d) %s (size %zu)\n", idx, name,3947				(size_t)sh->sh_size);3948		}3949	}3950 3951	if (!obj->efile.strtabidx || obj->efile.strtabidx > idx) {3952		pr_warn("elf: symbol strings section missing or invalid in %s\n", obj->path);3953		return -LIBBPF_ERRNO__FORMAT;3954	}3955 3956	/* sort BPF programs by section name and in-section instruction offset3957	 * for faster search3958	 */3959	if (obj->nr_programs)3960		qsort(obj->programs, obj->nr_programs, sizeof(*obj->programs), cmp_progs);3961 3962	return bpf_object__init_btf(obj, btf_data, btf_ext_data);3963}3964 3965static bool sym_is_extern(const Elf64_Sym *sym)3966{3967	int bind = ELF64_ST_BIND(sym->st_info);3968	/* externs are symbols w/ type=NOTYPE, bind=GLOBAL|WEAK, section=UND */3969	return sym->st_shndx == SHN_UNDEF &&3970	       (bind == STB_GLOBAL || bind == STB_WEAK) &&3971	       ELF64_ST_TYPE(sym->st_info) == STT_NOTYPE;3972}3973 3974static bool sym_is_subprog(const Elf64_Sym *sym, int text_shndx)3975{3976	int bind = ELF64_ST_BIND(sym->st_info);3977	int type = ELF64_ST_TYPE(sym->st_info);3978 3979	/* in .text section */3980	if (sym->st_shndx != text_shndx)3981		return false;3982 3983	/* local function */3984	if (bind == STB_LOCAL && type == STT_SECTION)3985		return true;3986 3987	/* global function */3988	return bind == STB_GLOBAL && type == STT_FUNC;3989}3990 3991static int find_extern_btf_id(const struct btf *btf, const char *ext_name)3992{3993	const struct btf_type *t;3994	const char *tname;3995	int i, n;3996 3997	if (!btf)3998		return -ESRCH;3999 4000	n = btf__type_cnt(btf);4001	for (i = 1; i < n; i++) {4002		t = btf__type_by_id(btf, i);4003 4004		if (!btf_is_var(t) && !btf_is_func(t))4005			continue;4006 4007		tname = btf__name_by_offset(btf, t->name_off);4008		if (strcmp(tname, ext_name))4009			continue;4010 4011		if (btf_is_var(t) &&4012		    btf_var(t)->linkage != BTF_VAR_GLOBAL_EXTERN)4013			return -EINVAL;4014 4015		if (btf_is_func(t) && btf_func_linkage(t) != BTF_FUNC_EXTERN)4016			return -EINVAL;4017 4018		return i;4019	}4020 4021	return -ENOENT;4022}4023 4024static int find_extern_sec_btf_id(struct btf *btf, int ext_btf_id) {4025	const struct btf_var_secinfo *vs;4026	const struct btf_type *t;4027	int i, j, n;4028 4029	if (!btf)4030		return -ESRCH;4031 4032	n = btf__type_cnt(btf);4033	for (i = 1; i < n; i++) {4034		t = btf__type_by_id(btf, i);4035 4036		if (!btf_is_datasec(t))4037			continue;4038 4039		vs = btf_var_secinfos(t);4040		for (j = 0; j < btf_vlen(t); j++, vs++) {4041			if (vs->type == ext_btf_id)4042				return i;4043		}4044	}4045 4046	return -ENOENT;4047}4048 4049static enum kcfg_type find_kcfg_type(const struct btf *btf, int id,4050				     bool *is_signed)4051{4052	const struct btf_type *t;4053	const char *name;4054 4055	t = skip_mods_and_typedefs(btf, id, NULL);4056	name = btf__name_by_offset(btf, t->name_off);4057 4058	if (is_signed)4059		*is_signed = false;4060	switch (btf_kind(t)) {4061	case BTF_KIND_INT: {4062		int enc = btf_int_encoding(t);4063 4064		if (enc & BTF_INT_BOOL)4065			return t->size == 1 ? KCFG_BOOL : KCFG_UNKNOWN;4066		if (is_signed)4067			*is_signed = enc & BTF_INT_SIGNED;4068		if (t->size == 1)4069			return KCFG_CHAR;4070		if (t->size < 1 || t->size > 8 || (t->size & (t->size - 1)))4071			return KCFG_UNKNOWN;4072		return KCFG_INT;4073	}4074	case BTF_KIND_ENUM:4075		if (t->size != 4)4076			return KCFG_UNKNOWN;4077		if (strcmp(name, "libbpf_tristate"))4078			return KCFG_UNKNOWN;4079		return KCFG_TRISTATE;4080	case BTF_KIND_ENUM64:4081		if (strcmp(name, "libbpf_tristate"))4082			return KCFG_UNKNOWN;4083		return KCFG_TRISTATE;4084	case BTF_KIND_ARRAY:4085		if (btf_array(t)->nelems == 0)4086			return KCFG_UNKNOWN;4087		if (find_kcfg_type(btf, btf_array(t)->type, NULL) != KCFG_CHAR)4088			return KCFG_UNKNOWN;4089		return KCFG_CHAR_ARR;4090	default:4091		return KCFG_UNKNOWN;4092	}4093}4094 4095static int cmp_externs(const void *_a, const void *_b)4096{4097	const struct extern_desc *a = _a;4098	const struct extern_desc *b = _b;4099 4100	if (a->type != b->type)4101		return a->type < b->type ? -1 : 1;4102 4103	if (a->type == EXT_KCFG) {4104		/* descending order by alignment requirements */4105		if (a->kcfg.align != b->kcfg.align)4106			return a->kcfg.align > b->kcfg.align ? -1 : 1;4107		/* ascending order by size, within same alignment class */4108		if (a->kcfg.sz != b->kcfg.sz)4109			return a->kcfg.sz < b->kcfg.sz ? -1 : 1;4110	}4111 4112	/* resolve ties by name */4113	return strcmp(a->name, b->name);4114}4115 4116static int find_int_btf_id(const struct btf *btf)4117{4118	const struct btf_type *t;4119	int i, n;4120 4121	n = btf__type_cnt(btf);4122	for (i = 1; i < n; i++) {4123		t = btf__type_by_id(btf, i);4124 4125		if (btf_is_int(t) && btf_int_bits(t) == 32)4126			return i;4127	}4128 4129	return 0;4130}4131 4132static int add_dummy_ksym_var(struct btf *btf)4133{4134	int i, int_btf_id, sec_btf_id, dummy_var_btf_id;4135	const struct btf_var_secinfo *vs;4136	const struct btf_type *sec;4137 4138	if (!btf)4139		return 0;4140 4141	sec_btf_id = btf__find_by_name_kind(btf, KSYMS_SEC,4142					    BTF_KIND_DATASEC);4143	if (sec_btf_id < 0)4144		return 0;4145 4146	sec = btf__type_by_id(btf, sec_btf_id);4147	vs = btf_var_secinfos(sec);4148	for (i = 0; i < btf_vlen(sec); i++, vs++) {4149		const struct btf_type *vt;4150 4151		vt = btf__type_by_id(btf, vs->type);4152		if (btf_is_func(vt))4153			break;4154	}4155 4156	/* No func in ksyms sec.  No need to add dummy var. */4157	if (i == btf_vlen(sec))4158		return 0;4159 4160	int_btf_id = find_int_btf_id(btf);4161	dummy_var_btf_id = btf__add_var(btf,4162					"dummy_ksym",4163					BTF_VAR_GLOBAL_ALLOCATED,4164					int_btf_id);4165	if (dummy_var_btf_id < 0)4166		pr_warn("cannot create a dummy_ksym var\n");4167 4168	return dummy_var_btf_id;4169}4170 4171static int bpf_object__collect_externs(struct bpf_object *obj)4172{4173	struct btf_type *sec, *kcfg_sec = NULL, *ksym_sec = NULL;4174	const struct btf_type *t;4175	struct extern_desc *ext;4176	int i, n, off, dummy_var_btf_id;4177	const char *ext_name, *sec_name;4178	size_t ext_essent_len;4179	Elf_Scn *scn;4180	Elf64_Shdr *sh;4181 4182	if (!obj->efile.symbols)4183		return 0;4184 4185	scn = elf_sec_by_idx(obj, obj->efile.symbols_shndx);4186	sh = elf_sec_hdr(obj, scn);4187	if (!sh || sh->sh_entsize != sizeof(Elf64_Sym))4188		return -LIBBPF_ERRNO__FORMAT;4189 4190	dummy_var_btf_id = add_dummy_ksym_var(obj->btf);4191	if (dummy_var_btf_id < 0)4192		return dummy_var_btf_id;4193 4194	n = sh->sh_size / sh->sh_entsize;4195	pr_debug("looking for externs among %d symbols...\n", n);4196 4197	for (i = 0; i < n; i++) {4198		Elf64_Sym *sym = elf_sym_by_idx(obj, i);4199 4200		if (!sym)4201			return -LIBBPF_ERRNO__FORMAT;4202		if (!sym_is_extern(sym))4203			continue;4204		ext_name = elf_sym_str(obj, sym->st_name);4205		if (!ext_name || !ext_name[0])4206			continue;4207 4208		ext = obj->externs;4209		ext = libbpf_reallocarray(ext, obj->nr_extern + 1, sizeof(*ext));4210		if (!ext)4211			return -ENOMEM;4212		obj->externs = ext;4213		ext = &ext[obj->nr_extern];4214		memset(ext, 0, sizeof(*ext));4215		obj->nr_extern++;4216 4217		ext->btf_id = find_extern_btf_id(obj->btf, ext_name);4218		if (ext->btf_id <= 0) {4219			pr_warn("failed to find BTF for extern '%s': %d\n",4220				ext_name, ext->btf_id);4221			return ext->btf_id;4222		}4223		t = btf__type_by_id(obj->btf, ext->btf_id);4224		ext->name = btf__name_by_offset(obj->btf, t->name_off);4225		ext->sym_idx = i;4226		ext->is_weak = ELF64_ST_BIND(sym->st_info) == STB_WEAK;4227 4228		ext_essent_len = bpf_core_essential_name_len(ext->name);4229		ext->essent_name = NULL;4230		if (ext_essent_len != strlen(ext->name)) {4231			ext->essent_name = strndup(ext->name, ext_essent_len);4232			if (!ext->essent_name)4233				return -ENOMEM;4234		}4235 4236		ext->sec_btf_id = find_extern_sec_btf_id(obj->btf, ext->btf_id);4237		if (ext->sec_btf_id <= 0) {4238			pr_warn("failed to find BTF for extern '%s' [%d] section: %d\n",4239				ext_name, ext->btf_id, ext->sec_btf_id);4240			return ext->sec_btf_id;4241		}4242		sec = (void *)btf__type_by_id(obj->btf, ext->sec_btf_id);4243		sec_name = btf__name_by_offset(obj->btf, sec->name_off);4244 4245		if (strcmp(sec_name, KCONFIG_SEC) == 0) {4246			if (btf_is_func(t)) {4247				pr_warn("extern function %s is unsupported under %s section\n",4248					ext->name, KCONFIG_SEC);4249				return -ENOTSUP;4250			}4251			kcfg_sec = sec;4252			ext->type = EXT_KCFG;4253			ext->kcfg.sz = btf__resolve_size(obj->btf, t->type);4254			if (ext->kcfg.sz <= 0) {4255				pr_warn("failed to resolve size of extern (kcfg) '%s': %d\n",4256					ext_name, ext->kcfg.sz);4257				return ext->kcfg.sz;4258			}4259			ext->kcfg.align = btf__align_of(obj->btf, t->type);4260			if (ext->kcfg.align <= 0) {4261				pr_warn("failed to determine alignment of extern (kcfg) '%s': %d\n",4262					ext_name, ext->kcfg.align);4263				return -EINVAL;4264			}4265			ext->kcfg.type = find_kcfg_type(obj->btf, t->type,4266							&ext->kcfg.is_signed);4267			if (ext->kcfg.type == KCFG_UNKNOWN) {4268				pr_warn("extern (kcfg) '%s': type is unsupported\n", ext_name);4269				return -ENOTSUP;4270			}4271		} else if (strcmp(sec_name, KSYMS_SEC) == 0) {4272			ksym_sec = sec;4273			ext->type = EXT_KSYM;4274			skip_mods_and_typedefs(obj->btf, t->type,4275					       &ext->ksym.type_id);4276		} else {4277			pr_warn("unrecognized extern section '%s'\n", sec_name);4278			return -ENOTSUP;4279		}4280	}4281	pr_debug("collected %d externs total\n", obj->nr_extern);4282 4283	if (!obj->nr_extern)4284		return 0;4285 4286	/* sort externs by type, for kcfg ones also by (align, size, name) */4287	qsort(obj->externs, obj->nr_extern, sizeof(*ext), cmp_externs);4288 4289	/* for .ksyms section, we need to turn all externs into allocated4290	 * variables in BTF to pass kernel verification; we do this by4291	 * pretending that each extern is a 8-byte variable4292	 */4293	if (ksym_sec) {4294		/* find existing 4-byte integer type in BTF to use for fake4295		 * extern variables in DATASEC4296		 */4297		int int_btf_id = find_int_btf_id(obj->btf);4298		/* For extern function, a dummy_var added earlier4299		 * will be used to replace the vs->type and4300		 * its name string will be used to refill4301		 * the missing param's name.4302		 */4303		const struct btf_type *dummy_var;4304 4305		dummy_var = btf__type_by_id(obj->btf, dummy_var_btf_id);4306		for (i = 0; i < obj->nr_extern; i++) {4307			ext = &obj->externs[i];4308			if (ext->type != EXT_KSYM)4309				continue;4310			pr_debug("extern (ksym) #%d: symbol %d, name %s\n",4311				 i, ext->sym_idx, ext->name);4312		}4313 4314		sec = ksym_sec;4315		n = btf_vlen(sec);4316		for (i = 0, off = 0; i < n; i++, off += sizeof(int)) {4317			struct btf_var_secinfo *vs = btf_var_secinfos(sec) + i;4318			struct btf_type *vt;4319 4320			vt = (void *)btf__type_by_id(obj->btf, vs->type);4321			ext_name = btf__name_by_offset(obj->btf, vt->name_off);4322			ext = find_extern_by_name(obj, ext_name);4323			if (!ext) {4324				pr_warn("failed to find extern definition for BTF %s '%s'\n",4325					btf_kind_str(vt), ext_name);4326				return -ESRCH;4327			}4328			if (btf_is_func(vt)) {4329				const struct btf_type *func_proto;4330				struct btf_param *param;4331				int j;4332 4333				func_proto = btf__type_by_id(obj->btf,4334							     vt->type);4335				param = btf_params(func_proto);4336				/* Reuse the dummy_var string if the4337				 * func proto does not have param name.4338				 */4339				for (j = 0; j < btf_vlen(func_proto); j++)4340					if (param[j].type && !param[j].name_off)4341						param[j].name_off =4342							dummy_var->name_off;4343				vs->type = dummy_var_btf_id;4344				vt->info &= ~0xffff;4345				vt->info |= BTF_FUNC_GLOBAL;4346			} else {4347				btf_var(vt)->linkage = BTF_VAR_GLOBAL_ALLOCATED;4348				vt->type = int_btf_id;4349			}4350			vs->offset = off;4351			vs->size = sizeof(int);4352		}4353		sec->size = off;4354	}4355 4356	if (kcfg_sec) {4357		sec = kcfg_sec;4358		/* for kcfg externs calculate their offsets within a .kconfig map */4359		off = 0;4360		for (i = 0; i < obj->nr_extern; i++) {4361			ext = &obj->externs[i];4362			if (ext->type != EXT_KCFG)4363				continue;4364 4365			ext->kcfg.data_off = roundup(off, ext->kcfg.align);4366			off = ext->kcfg.data_off + ext->kcfg.sz;4367			pr_debug("extern (kcfg) #%d: symbol %d, off %u, name %s\n",4368				 i, ext->sym_idx, ext->kcfg.data_off, ext->name);4369		}4370		sec->size = off;4371		n = btf_vlen(sec);4372		for (i = 0; i < n; i++) {4373			struct btf_var_secinfo *vs = btf_var_secinfos(sec) + i;4374 4375			t = btf__type_by_id(obj->btf, vs->type);4376			ext_name = btf__name_by_offset(obj->btf, t->name_off);4377			ext = find_extern_by_name(obj, ext_name);4378			if (!ext) {4379				pr_warn("failed to find extern definition for BTF var '%s'\n",4380					ext_name);4381				return -ESRCH;4382			}4383			btf_var(t)->linkage = BTF_VAR_GLOBAL_ALLOCATED;4384			vs->offset = ext->kcfg.data_off;4385		}4386	}4387	return 0;4388}4389 4390static bool prog_is_subprog(const struct bpf_object *obj, const struct bpf_program *prog)4391{4392	return prog->sec_idx == obj->efile.text_shndx && obj->nr_programs > 1;4393}4394 4395struct bpf_program *4396bpf_object__find_program_by_name(const struct bpf_object *obj,4397				 const char *name)4398{4399	struct bpf_program *prog;4400 4401	bpf_object__for_each_program(prog, obj) {4402		if (prog_is_subprog(obj, prog))4403			continue;4404		if (!strcmp(prog->name, name))4405			return prog;4406	}4407	return errno = ENOENT, NULL;4408}4409 4410static bool bpf_object__shndx_is_data(const struct bpf_object *obj,4411				      int shndx)4412{4413	switch (obj->efile.secs[shndx].sec_type) {4414	case SEC_BSS:4415	case SEC_DATA:4416	case SEC_RODATA:4417		return true;4418	default:4419		return false;4420	}4421}4422 4423static bool bpf_object__shndx_is_maps(const struct bpf_object *obj,4424				      int shndx)4425{4426	return shndx == obj->efile.btf_maps_shndx;4427}4428 4429static enum libbpf_map_type4430bpf_object__section_to_libbpf_map_type(const struct bpf_object *obj, int shndx)4431{4432	if (shndx == obj->efile.symbols_shndx)4433		return LIBBPF_MAP_KCONFIG;4434 4435	switch (obj->efile.secs[shndx].sec_type) {4436	case SEC_BSS:4437		return LIBBPF_MAP_BSS;4438	case SEC_DATA:4439		return LIBBPF_MAP_DATA;4440	case SEC_RODATA:4441		return LIBBPF_MAP_RODATA;4442	default:4443		return LIBBPF_MAP_UNSPEC;4444	}4445}4446 4447static int bpf_program__record_reloc(struct bpf_program *prog,4448				     struct reloc_desc *reloc_desc,4449				     __u32 insn_idx, const char *sym_name,4450				     const Elf64_Sym *sym, const Elf64_Rel *rel)4451{4452	struct bpf_insn *insn = &prog->insns[insn_idx];4453	size_t map_idx, nr_maps = prog->obj->nr_maps;4454	struct bpf_object *obj = prog->obj;4455	__u32 shdr_idx = sym->st_shndx;4456	enum libbpf_map_type type;4457	const char *sym_sec_name;4458	struct bpf_map *map;4459 4460	if (!is_call_insn(insn) && !is_ldimm64_insn(insn)) {4461		pr_warn("prog '%s': invalid relo against '%s' for insns[%d].code 0x%x\n",4462			prog->name, sym_name, insn_idx, insn->code);4463		return -LIBBPF_ERRNO__RELOC;4464	}4465 4466	if (sym_is_extern(sym)) {4467		int sym_idx = ELF64_R_SYM(rel->r_info);4468		int i, n = obj->nr_extern;4469		struct extern_desc *ext;4470 4471		for (i = 0; i < n; i++) {4472			ext = &obj->externs[i];4473			if (ext->sym_idx == sym_idx)4474				break;4475		}4476		if (i >= n) {4477			pr_warn("prog '%s': extern relo failed to find extern for '%s' (%d)\n",4478				prog->name, sym_name, sym_idx);4479			return -LIBBPF_ERRNO__RELOC;4480		}4481		pr_debug("prog '%s': found extern #%d '%s' (sym %d) for insn #%u\n",4482			 prog->name, i, ext->name, ext->sym_idx, insn_idx);4483		if (insn->code == (BPF_JMP | BPF_CALL))4484			reloc_desc->type = RELO_EXTERN_CALL;4485		else4486			reloc_desc->type = RELO_EXTERN_LD64;4487		reloc_desc->insn_idx = insn_idx;4488		reloc_desc->ext_idx = i;4489		return 0;4490	}4491 4492	/* sub-program call relocation */4493	if (is_call_insn(insn)) {4494		if (insn->src_reg != BPF_PSEUDO_CALL) {4495			pr_warn("prog '%s': incorrect bpf_call opcode\n", prog->name);4496			return -LIBBPF_ERRNO__RELOC;4497		}4498		/* text_shndx can be 0, if no default "main" program exists */4499		if (!shdr_idx || shdr_idx != obj->efile.text_shndx) {4500			sym_sec_name = elf_sec_name(obj, elf_sec_by_idx(obj, shdr_idx));4501			pr_warn("prog '%s': bad call relo against '%s' in section '%s'\n",4502				prog->name, sym_name, sym_sec_name);4503			return -LIBBPF_ERRNO__RELOC;4504		}4505		if (sym->st_value % BPF_INSN_SZ) {4506			pr_warn("prog '%s': bad call relo against '%s' at offset %zu\n",4507				prog->name, sym_name, (size_t)sym->st_value);4508			return -LIBBPF_ERRNO__RELOC;4509		}4510		reloc_desc->type = RELO_CALL;4511		reloc_desc->insn_idx = insn_idx;4512		reloc_desc->sym_off = sym->st_value;4513		return 0;4514	}4515 4516	if (!shdr_idx || shdr_idx >= SHN_LORESERVE) {4517		pr_warn("prog '%s': invalid relo against '%s' in special section 0x%x; forgot to initialize global var?..\n",4518			prog->name, sym_name, shdr_idx);4519		return -LIBBPF_ERRNO__RELOC;4520	}4521 4522	/* loading subprog addresses */4523	if (sym_is_subprog(sym, obj->efile.text_shndx)) {4524		/* global_func: sym->st_value = offset in the section, insn->imm = 0.4525		 * local_func: sym->st_value = 0, insn->imm = offset in the section.4526		 */4527		if ((sym->st_value % BPF_INSN_SZ) || (insn->imm % BPF_INSN_SZ)) {4528			pr_warn("prog '%s': bad subprog addr relo against '%s' at offset %zu+%d\n",4529				prog->name, sym_name, (size_t)sym->st_value, insn->imm);4530			return -LIBBPF_ERRNO__RELOC;4531		}4532 4533		reloc_desc->type = RELO_SUBPROG_ADDR;4534		reloc_desc->insn_idx = insn_idx;4535		reloc_desc->sym_off = sym->st_value;4536		return 0;4537	}4538 4539	type = bpf_object__section_to_libbpf_map_type(obj, shdr_idx);4540	sym_sec_name = elf_sec_name(obj, elf_sec_by_idx(obj, shdr_idx));4541 4542	/* arena data relocation */4543	if (shdr_idx == obj->efile.arena_data_shndx) {4544		reloc_desc->type = RELO_DATA;4545		reloc_desc->insn_idx = insn_idx;4546		reloc_desc->map_idx = obj->arena_map - obj->maps;4547		reloc_desc->sym_off = sym->st_value;4548		return 0;4549	}4550 4551	/* generic map reference relocation */4552	if (type == LIBBPF_MAP_UNSPEC) {4553		if (!bpf_object__shndx_is_maps(obj, shdr_idx)) {4554			pr_warn("prog '%s': bad map relo against '%s' in section '%s'\n",4555				prog->name, sym_name, sym_sec_name);4556			return -LIBBPF_ERRNO__RELOC;4557		}4558		for (map_idx = 0; map_idx < nr_maps; map_idx++) {4559			map = &obj->maps[map_idx];4560			if (map->libbpf_type != type ||4561			    map->sec_idx != sym->st_shndx ||4562			    map->sec_offset != sym->st_value)4563				continue;4564			pr_debug("prog '%s': found map %zd (%s, sec %d, off %zu) for insn #%u\n",4565				 prog->name, map_idx, map->name, map->sec_idx,4566				 map->sec_offset, insn_idx);4567			break;4568		}4569		if (map_idx >= nr_maps) {4570			pr_warn("prog '%s': map relo failed to find map for section '%s', off %zu\n",4571				prog->name, sym_sec_name, (size_t)sym->st_value);4572			return -LIBBPF_ERRNO__RELOC;4573		}4574		reloc_desc->type = RELO_LD64;4575		reloc_desc->insn_idx = insn_idx;4576		reloc_desc->map_idx = map_idx;4577		reloc_desc->sym_off = 0; /* sym->st_value determines map_idx */4578		return 0;4579	}4580 4581	/* global data map relocation */4582	if (!bpf_object__shndx_is_data(obj, shdr_idx)) {4583		pr_warn("prog '%s': bad data relo against section '%s'\n",4584			prog->name, sym_sec_name);4585		return -LIBBPF_ERRNO__RELOC;4586	}4587	for (map_idx = 0; map_idx < nr_maps; map_idx++) {4588		map = &obj->maps[map_idx];4589		if (map->libbpf_type != type || map->sec_idx != sym->st_shndx)4590			continue;4591		pr_debug("prog '%s': found data map %zd (%s, sec %d, off %zu) for insn %u\n",4592			 prog->name, map_idx, map->name, map->sec_idx,4593			 map->sec_offset, insn_idx);4594		break;4595	}4596	if (map_idx >= nr_maps) {4597		pr_warn("prog '%s': data relo failed to find map for section '%s'\n",4598			prog->name, sym_sec_name);4599		return -LIBBPF_ERRNO__RELOC;4600	}4601 4602	reloc_desc->type = RELO_DATA;4603	reloc_desc->insn_idx = insn_idx;4604	reloc_desc->map_idx = map_idx;4605	reloc_desc->sym_off = sym->st_value;4606	return 0;4607}4608 4609static bool prog_contains_insn(const struct bpf_program *prog, size_t insn_idx)4610{4611	return insn_idx >= prog->sec_insn_off &&4612	       insn_idx < prog->sec_insn_off + prog->sec_insn_cnt;4613}4614 4615static struct bpf_program *find_prog_by_sec_insn(const struct bpf_object *obj,4616						 size_t sec_idx, size_t insn_idx)4617{4618	int l = 0, r = obj->nr_programs - 1, m;4619	struct bpf_program *prog;4620 4621	if (!obj->nr_programs)4622		return NULL;4623 4624	while (l < r) {4625		m = l + (r - l + 1) / 2;4626		prog = &obj->programs[m];4627 4628		if (prog->sec_idx < sec_idx ||4629		    (prog->sec_idx == sec_idx && prog->sec_insn_off <= insn_idx))4630			l = m;4631		else4632			r = m - 1;4633	}4634	/* matching program could be at index l, but it still might be the4635	 * wrong one, so we need to double check conditions for the last time4636	 */4637	prog = &obj->programs[l];4638	if (prog->sec_idx == sec_idx && prog_contains_insn(prog, insn_idx))4639		return prog;4640	return NULL;4641}4642 4643static int4644bpf_object__collect_prog_relos(struct bpf_object *obj, Elf64_Shdr *shdr, Elf_Data *data)4645{4646	const char *relo_sec_name, *sec_name;4647	size_t sec_idx = shdr->sh_info, sym_idx;4648	struct bpf_program *prog;4649	struct reloc_desc *relos;4650	int err, i, nrels;4651	const char *sym_name;4652	__u32 insn_idx;4653	Elf_Scn *scn;4654	Elf_Data *scn_data;4655	Elf64_Sym *sym;4656	Elf64_Rel *rel;4657 4658	if (sec_idx >= obj->efile.sec_cnt)4659		return -EINVAL;4660 4661	scn = elf_sec_by_idx(obj, sec_idx);4662	scn_data = elf_sec_data(obj, scn);4663	if (!scn_data)4664		return -LIBBPF_ERRNO__FORMAT;4665 4666	relo_sec_name = elf_sec_str(obj, shdr->sh_name);4667	sec_name = elf_sec_name(obj, scn);4668	if (!relo_sec_name || !sec_name)4669		return -EINVAL;4670 4671	pr_debug("sec '%s': collecting relocation for section(%zu) '%s'\n",4672		 relo_sec_name, sec_idx, sec_name);4673	nrels = shdr->sh_size / shdr->sh_entsize;4674 4675	for (i = 0; i < nrels; i++) {4676		rel = elf_rel_by_idx(data, i);4677		if (!rel) {4678			pr_warn("sec '%s': failed to get relo #%d\n", relo_sec_name, i);4679			return -LIBBPF_ERRNO__FORMAT;4680		}4681 4682		sym_idx = ELF64_R_SYM(rel->r_info);4683		sym = elf_sym_by_idx(obj, sym_idx);4684		if (!sym) {4685			pr_warn("sec '%s': symbol #%zu not found for relo #%d\n",4686				relo_sec_name, sym_idx, i);4687			return -LIBBPF_ERRNO__FORMAT;4688		}4689 4690		if (sym->st_shndx >= obj->efile.sec_cnt) {4691			pr_warn("sec '%s': corrupted symbol #%zu pointing to invalid section #%zu for relo #%d\n",4692				relo_sec_name, sym_idx, (size_t)sym->st_shndx, i);4693			return -LIBBPF_ERRNO__FORMAT;4694		}4695 4696		if (rel->r_offset % BPF_INSN_SZ || rel->r_offset >= scn_data->d_size) {4697			pr_warn("sec '%s': invalid offset 0x%zx for relo #%d\n",4698				relo_sec_name, (size_t)rel->r_offset, i);4699			return -LIBBPF_ERRNO__FORMAT;4700		}4701 4702		insn_idx = rel->r_offset / BPF_INSN_SZ;4703		/* relocations against static functions are recorded as4704		 * relocations against the section that contains a function;4705		 * in such case, symbol will be STT_SECTION and sym.st_name4706		 * will point to empty string (0), so fetch section name4707		 * instead4708		 */4709		if (ELF64_ST_TYPE(sym->st_info) == STT_SECTION && sym->st_name == 0)4710			sym_name = elf_sec_name(obj, elf_sec_by_idx(obj, sym->st_shndx));4711		else4712			sym_name = elf_sym_str(obj, sym->st_name);4713		sym_name = sym_name ?: "<?";4714 4715		pr_debug("sec '%s': relo #%d: insn #%u against '%s'\n",4716			 relo_sec_name, i, insn_idx, sym_name);4717 4718		prog = find_prog_by_sec_insn(obj, sec_idx, insn_idx);4719		if (!prog) {4720			pr_debug("sec '%s': relo #%d: couldn't find program in section '%s' for insn #%u, probably overridden weak function, skipping...\n",4721				relo_sec_name, i, sec_name, insn_idx);4722			continue;4723		}4724 4725		relos = libbpf_reallocarray(prog->reloc_desc,4726					    prog->nr_reloc + 1, sizeof(*relos));4727		if (!relos)4728			return -ENOMEM;4729		prog->reloc_desc = relos;4730 4731		/* adjust insn_idx to local BPF program frame of reference */4732		insn_idx -= prog->sec_insn_off;4733		err = bpf_program__record_reloc(prog, &relos[prog->nr_reloc],4734						insn_idx, sym_name, sym, rel);4735		if (err)4736			return err;4737 4738		prog->nr_reloc++;4739	}4740	return 0;4741}4742 4743static int map_fill_btf_type_info(struct bpf_object *obj, struct bpf_map *map)4744{4745	int id;4746 4747	if (!obj->btf)4748		return -ENOENT;4749 4750	/* if it's BTF-defined map, we don't need to search for type IDs.4751	 * For struct_ops map, it does not need btf_key_type_id and4752	 * btf_value_type_id.4753	 */4754	if (map->sec_idx == obj->efile.btf_maps_shndx || bpf_map__is_struct_ops(map))4755		return 0;4756 4757	/*4758	 * LLVM annotates global data differently in BTF, that is,4759	 * only as '.data', '.bss' or '.rodata'.4760	 */4761	if (!bpf_map__is_internal(map))4762		return -ENOENT;4763 4764	id = btf__find_by_name(obj->btf, map->real_name);4765	if (id < 0)4766		return id;4767 4768	map->btf_key_type_id = 0;4769	map->btf_value_type_id = id;4770	return 0;4771}4772 4773static int bpf_get_map_info_from_fdinfo(int fd, struct bpf_map_info *info)4774{4775	char file[PATH_MAX], buff[4096];4776	FILE *fp;4777	__u32 val;4778	int err;4779 4780	snprintf(file, sizeof(file), "/proc/%d/fdinfo/%d", getpid(), fd);4781	memset(info, 0, sizeof(*info));4782 4783	fp = fopen(file, "re");4784	if (!fp) {4785		err = -errno;4786		pr_warn("failed to open %s: %d. No procfs support?\n", file,4787			err);4788		return err;4789	}4790 4791	while (fgets(buff, sizeof(buff), fp)) {4792		if (sscanf(buff, "map_type:\t%u", &val) == 1)4793			info->type = val;4794		else if (sscanf(buff, "key_size:\t%u", &val) == 1)4795			info->key_size = val;4796		else if (sscanf(buff, "value_size:\t%u", &val) == 1)4797			info->value_size = val;4798		else if (sscanf(buff, "max_entries:\t%u", &val) == 1)4799			info->max_entries = val;4800		else if (sscanf(buff, "map_flags:\t%i", &val) == 1)4801			info->map_flags = val;4802	}4803 4804	fclose(fp);4805 4806	return 0;4807}4808 4809bool bpf_map__autocreate(const struct bpf_map *map)4810{4811	return map->autocreate;4812}4813 4814int bpf_map__set_autocreate(struct bpf_map *map, bool autocreate)4815{4816	if (map->obj->loaded)4817		return libbpf_err(-EBUSY);4818 4819	map->autocreate = autocreate;4820	return 0;4821}4822 4823int bpf_map__set_autoattach(struct bpf_map *map, bool autoattach)4824{4825	if (!bpf_map__is_struct_ops(map))4826		return libbpf_err(-EINVAL);4827 4828	map->autoattach = autoattach;4829	return 0;4830}4831 4832bool bpf_map__autoattach(const struct bpf_map *map)4833{4834	return map->autoattach;4835}4836 4837int bpf_map__reuse_fd(struct bpf_map *map, int fd)4838{4839	struct bpf_map_info info;4840	__u32 len = sizeof(info), name_len;4841	int new_fd, err;4842	char *new_name;4843 4844	memset(&info, 0, len);4845	err = bpf_map_get_info_by_fd(fd, &info, &len);4846	if (err && errno == EINVAL)4847		err = bpf_get_map_info_from_fdinfo(fd, &info);4848	if (err)4849		return libbpf_err(err);4850 4851	name_len = strlen(info.name);4852	if (name_len == BPF_OBJ_NAME_LEN - 1 && strncmp(map->name, info.name, name_len) == 0)4853		new_name = strdup(map->name);4854	else4855		new_name = strdup(info.name);4856 4857	if (!new_name)4858		return libbpf_err(-errno);4859 4860	/*4861	 * Like dup(), but make sure new FD is >= 3 and has O_CLOEXEC set.4862	 * This is similar to what we do in ensure_good_fd(), but without4863	 * closing original FD.4864	 */4865	new_fd = fcntl(fd, F_DUPFD_CLOEXEC, 3);4866	if (new_fd < 0) {4867		err = -errno;4868		goto err_free_new_name;4869	}4870 4871	err = reuse_fd(map->fd, new_fd);4872	if (err)4873		goto err_free_new_name;4874 4875	free(map->name);4876 4877	map->name = new_name;4878	map->def.type = info.type;4879	map->def.key_size = info.key_size;4880	map->def.value_size = info.value_size;4881	map->def.max_entries = info.max_entries;4882	map->def.map_flags = info.map_flags;4883	map->btf_key_type_id = info.btf_key_type_id;4884	map->btf_value_type_id = info.btf_value_type_id;4885	map->reused = true;4886	map->map_extra = info.map_extra;4887 4888	return 0;4889 4890err_free_new_name:4891	free(new_name);4892	return libbpf_err(err);4893}4894 4895__u32 bpf_map__max_entries(const struct bpf_map *map)4896{4897	return map->def.max_entries;4898}4899 4900struct bpf_map *bpf_map__inner_map(struct bpf_map *map)4901{4902	if (!bpf_map_type__is_map_in_map(map->def.type))4903		return errno = EINVAL, NULL;4904 4905	return map->inner_map;4906}4907 4908int bpf_map__set_max_entries(struct bpf_map *map, __u32 max_entries)4909{4910	if (map->obj->loaded)4911		return libbpf_err(-EBUSY);4912 4913	map->def.max_entries = max_entries;4914 4915	/* auto-adjust BPF ringbuf map max_entries to be a multiple of page size */4916	if (map_is_ringbuf(map))4917		map->def.max_entries = adjust_ringbuf_sz(map->def.max_entries);4918 4919	return 0;4920}4921 4922static int bpf_object_prepare_token(struct bpf_object *obj)4923{4924	const char *bpffs_path;4925	int bpffs_fd = -1, token_fd, err;4926	bool mandatory;4927	enum libbpf_print_level level;4928 4929	/* token is explicitly prevented */4930	if (obj->token_path && obj->token_path[0] == '\0') {4931		pr_debug("object '%s': token is prevented, skipping...\n", obj->name);4932		return 0;4933	}4934 4935	mandatory = obj->token_path != NULL;4936	level = mandatory ? LIBBPF_WARN : LIBBPF_DEBUG;4937 4938	bpffs_path = obj->token_path ?: BPF_FS_DEFAULT_PATH;4939	bpffs_fd = open(bpffs_path, O_DIRECTORY, O_RDWR);4940	if (bpffs_fd < 0) {4941		err = -errno;4942		__pr(level, "object '%s': failed (%d) to open BPF FS mount at '%s'%s\n",4943		     obj->name, err, bpffs_path,4944		     mandatory ? "" : ", skipping optional step...");4945		return mandatory ? err : 0;4946	}4947 4948	token_fd = bpf_token_create(bpffs_fd, 0);4949	close(bpffs_fd);4950	if (token_fd < 0) {4951		if (!mandatory && token_fd == -ENOENT) {4952			pr_debug("object '%s': BPF FS at '%s' doesn't have BPF token delegation set up, skipping...\n",4953				 obj->name, bpffs_path);4954			return 0;4955		}4956		__pr(level, "object '%s': failed (%d) to create BPF token from '%s'%s\n",4957		     obj->name, token_fd, bpffs_path,4958		     mandatory ? "" : ", skipping optional step...");4959		return mandatory ? token_fd : 0;4960	}4961 4962	obj->feat_cache = calloc(1, sizeof(*obj->feat_cache));4963	if (!obj->feat_cache) {4964		close(token_fd);4965		return -ENOMEM;4966	}4967 4968	obj->token_fd = token_fd;4969	obj->feat_cache->token_fd = token_fd;4970 4971	return 0;4972}4973 4974static int4975bpf_object__probe_loading(struct bpf_object *obj)4976{4977	char *cp, errmsg[STRERR_BUFSIZE];4978	struct bpf_insn insns[] = {4979		BPF_MOV64_IMM(BPF_REG_0, 0),4980		BPF_EXIT_INSN(),4981	};4982	int ret, insn_cnt = ARRAY_SIZE(insns);4983	LIBBPF_OPTS(bpf_prog_load_opts, opts,4984		.token_fd = obj->token_fd,4985		.prog_flags = obj->token_fd ? BPF_F_TOKEN_FD : 0,4986	);4987 4988	if (obj->gen_loader)4989		return 0;4990 4991	ret = bump_rlimit_memlock();4992	if (ret)4993		pr_warn("Failed to bump RLIMIT_MEMLOCK (err = %d), you might need to do it explicitly!\n", ret);4994 4995	/* make sure basic loading works */4996	ret = bpf_prog_load(BPF_PROG_TYPE_SOCKET_FILTER, NULL, "GPL", insns, insn_cnt, &opts);4997	if (ret < 0)4998		ret = bpf_prog_load(BPF_PROG_TYPE_TRACEPOINT, NULL, "GPL", insns, insn_cnt, &opts);4999	if (ret < 0) {5000		ret = errno;5001		cp = libbpf_strerror_r(ret, errmsg, sizeof(errmsg));5002		pr_warn("Error in %s():%s(%d). Couldn't load trivial BPF "5003			"program. Make sure your kernel supports BPF "5004			"(CONFIG_BPF_SYSCALL=y) and/or that RLIMIT_MEMLOCK is "5005			"set to big enough value.\n", __func__, cp, ret);5006		return -ret;5007	}5008	close(ret);5009 5010	return 0;5011}5012 5013bool kernel_supports(const struct bpf_object *obj, enum kern_feature_id feat_id)5014{5015	if (obj->gen_loader)5016		/* To generate loader program assume the latest kernel5017		 * to avoid doing extra prog_load, map_create syscalls.5018		 */5019		return true;5020 5021	if (obj->token_fd)5022		return feat_supported(obj->feat_cache, feat_id);5023 5024	return feat_supported(NULL, feat_id);5025}5026 5027static bool map_is_reuse_compat(const struct bpf_map *map, int map_fd)5028{5029	struct bpf_map_info map_info;5030	char msg[STRERR_BUFSIZE];5031	__u32 map_info_len = sizeof(map_info);5032	int err;5033 5034	memset(&map_info, 0, map_info_len);5035	err = bpf_map_get_info_by_fd(map_fd, &map_info, &map_info_len);5036	if (err && errno == EINVAL)5037		err = bpf_get_map_info_from_fdinfo(map_fd, &map_info);5038	if (err) {5039		pr_warn("failed to get map info for map FD %d: %s\n", map_fd,5040			libbpf_strerror_r(errno, msg, sizeof(msg)));5041		return false;5042	}5043 5044	return (map_info.type == map->def.type &&5045		map_info.key_size == map->def.key_size &&5046		map_info.value_size == map->def.value_size &&5047		map_info.max_entries == map->def.max_entries &&5048		map_info.map_flags == map->def.map_flags &&5049		map_info.map_extra == map->map_extra);5050}5051 5052static int5053bpf_object__reuse_map(struct bpf_map *map)5054{5055	char *cp, errmsg[STRERR_BUFSIZE];5056	int err, pin_fd;5057 5058	pin_fd = bpf_obj_get(map->pin_path);5059	if (pin_fd < 0) {5060		err = -errno;5061		if (err == -ENOENT) {5062			pr_debug("found no pinned map to reuse at '%s'\n",5063				 map->pin_path);5064			return 0;5065		}5066 5067		cp = libbpf_strerror_r(-err, errmsg, sizeof(errmsg));5068		pr_warn("couldn't retrieve pinned map '%s': %s\n",5069			map->pin_path, cp);5070		return err;5071	}5072 5073	if (!map_is_reuse_compat(map, pin_fd)) {5074		pr_warn("couldn't reuse pinned map at '%s': parameter mismatch\n",5075			map->pin_path);5076		close(pin_fd);5077		return -EINVAL;5078	}5079 5080	err = bpf_map__reuse_fd(map, pin_fd);5081	close(pin_fd);5082	if (err)5083		return err;5084 5085	map->pinned = true;5086	pr_debug("reused pinned map at '%s'\n", map->pin_path);5087 5088	return 0;5089}5090 5091static int5092bpf_object__populate_internal_map(struct bpf_object *obj, struct bpf_map *map)5093{5094	enum libbpf_map_type map_type = map->libbpf_type;5095	char *cp, errmsg[STRERR_BUFSIZE];5096	int err, zero = 0;5097 5098	if (obj->gen_loader) {5099		bpf_gen__map_update_elem(obj->gen_loader, map - obj->maps,5100					 map->mmaped, map->def.value_size);5101		if (map_type == LIBBPF_MAP_RODATA || map_type == LIBBPF_MAP_KCONFIG)5102			bpf_gen__map_freeze(obj->gen_loader, map - obj->maps);5103		return 0;5104	}5105 5106	err = bpf_map_update_elem(map->fd, &zero, map->mmaped, 0);5107	if (err) {5108		err = -errno;5109		cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg));5110		pr_warn("Error setting initial map(%s) contents: %s\n",5111			map->name, cp);5112		return err;5113	}5114 5115	/* Freeze .rodata and .kconfig map as read-only from syscall side. */5116	if (map_type == LIBBPF_MAP_RODATA || map_type == LIBBPF_MAP_KCONFIG) {5117		err = bpf_map_freeze(map->fd);5118		if (err) {5119			err = -errno;5120			cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg));5121			pr_warn("Error freezing map(%s) as read-only: %s\n",5122				map->name, cp);5123			return err;5124		}5125	}5126	return 0;5127}5128 5129static void bpf_map__destroy(struct bpf_map *map);5130 5131static bool map_is_created(const struct bpf_map *map)5132{5133	return map->obj->loaded || map->reused;5134}5135 5136static int bpf_object__create_map(struct bpf_object *obj, struct bpf_map *map, bool is_inner)5137{5138	LIBBPF_OPTS(bpf_map_create_opts, create_attr);5139	struct bpf_map_def *def = &map->def;5140	const char *map_name = NULL;5141	int err = 0, map_fd;5142 5143	if (kernel_supports(obj, FEAT_PROG_NAME))5144		map_name = map->name;5145	create_attr.map_ifindex = map->map_ifindex;5146	create_attr.map_flags = def->map_flags;5147	create_attr.numa_node = map->numa_node;5148	create_attr.map_extra = map->map_extra;5149	create_attr.token_fd = obj->token_fd;5150	if (obj->token_fd)5151		create_attr.map_flags |= BPF_F_TOKEN_FD;5152 5153	if (bpf_map__is_struct_ops(map)) {5154		create_attr.btf_vmlinux_value_type_id = map->btf_vmlinux_value_type_id;5155		if (map->mod_btf_fd >= 0) {5156			create_attr.value_type_btf_obj_fd = map->mod_btf_fd;5157			create_attr.map_flags |= BPF_F_VTYPE_BTF_OBJ_FD;5158		}5159	}5160 5161	if (obj->btf && btf__fd(obj->btf) >= 0) {5162		create_attr.btf_fd = btf__fd(obj->btf);5163		create_attr.btf_key_type_id = map->btf_key_type_id;5164		create_attr.btf_value_type_id = map->btf_value_type_id;5165	}5166 5167	if (bpf_map_type__is_map_in_map(def->type)) {5168		if (map->inner_map) {5169			err = map_set_def_max_entries(map->inner_map);5170			if (err)5171				return err;5172			err = bpf_object__create_map(obj, map->inner_map, true);5173			if (err) {5174				pr_warn("map '%s': failed to create inner map: %d\n",5175					map->name, err);5176				return err;5177			}5178			map->inner_map_fd = map->inner_map->fd;5179		}5180		if (map->inner_map_fd >= 0)5181			create_attr.inner_map_fd = map->inner_map_fd;5182	}5183 5184	switch (def->type) {5185	case BPF_MAP_TYPE_PERF_EVENT_ARRAY:5186	case BPF_MAP_TYPE_CGROUP_ARRAY:5187	case BPF_MAP_TYPE_STACK_TRACE:5188	case BPF_MAP_TYPE_ARRAY_OF_MAPS:5189	case BPF_MAP_TYPE_HASH_OF_MAPS:5190	case BPF_MAP_TYPE_DEVMAP:5191	case BPF_MAP_TYPE_DEVMAP_HASH:5192	case BPF_MAP_TYPE_CPUMAP:5193	case BPF_MAP_TYPE_XSKMAP:5194	case BPF_MAP_TYPE_SOCKMAP:5195	case BPF_MAP_TYPE_SOCKHASH:5196	case BPF_MAP_TYPE_QUEUE:5197	case BPF_MAP_TYPE_STACK:5198	case BPF_MAP_TYPE_ARENA:5199		create_attr.btf_fd = 0;5200		create_attr.btf_key_type_id = 0;5201		create_attr.btf_value_type_id = 0;5202		map->btf_key_type_id = 0;5203		map->btf_value_type_id = 0;5204		break;5205	case BPF_MAP_TYPE_STRUCT_OPS:5206		create_attr.btf_value_type_id = 0;5207		break;5208	default:5209		break;5210	}5211 5212	if (obj->gen_loader) {5213		bpf_gen__map_create(obj->gen_loader, def->type, map_name,5214				    def->key_size, def->value_size, def->max_entries,5215				    &create_attr, is_inner ? -1 : map - obj->maps);5216		/* We keep pretenting we have valid FD to pass various fd >= 05217		 * checks by just keeping original placeholder FDs in place.5218		 * See bpf_object__add_map() comment.5219		 * This placeholder fd will not be used with any syscall and5220		 * will be reset to -1 eventually.5221		 */5222		map_fd = map->fd;5223	} else {5224		map_fd = bpf_map_create(def->type, map_name,5225					def->key_size, def->value_size,5226					def->max_entries, &create_attr);5227	}5228	if (map_fd < 0 && (create_attr.btf_key_type_id || create_attr.btf_value_type_id)) {5229		char *cp, errmsg[STRERR_BUFSIZE];5230 5231		err = -errno;5232		cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg));5233		pr_warn("Error in bpf_create_map_xattr(%s):%s(%d). Retrying without BTF.\n",5234			map->name, cp, err);5235		create_attr.btf_fd = 0;5236		create_attr.btf_key_type_id = 0;5237		create_attr.btf_value_type_id = 0;5238		map->btf_key_type_id = 0;5239		map->btf_value_type_id = 0;5240		map_fd = bpf_map_create(def->type, map_name,5241					def->key_size, def->value_size,5242					def->max_entries, &create_attr);5243	}5244 5245	if (bpf_map_type__is_map_in_map(def->type) && map->inner_map) {5246		if (obj->gen_loader)5247			map->inner_map->fd = -1;5248		bpf_map__destroy(map->inner_map);5249		zfree(&map->inner_map);5250	}5251 5252	if (map_fd < 0)5253		return map_fd;5254 5255	/* obj->gen_loader case, prevent reuse_fd() from closing map_fd */5256	if (map->fd == map_fd)5257		return 0;5258 5259	/* Keep placeholder FD value but now point it to the BPF map object.5260	 * This way everything that relied on this map's FD (e.g., relocated5261	 * ldimm64 instructions) will stay valid and won't need adjustments.5262	 * map->fd stays valid but now point to what map_fd points to.5263	 */5264	return reuse_fd(map->fd, map_fd);5265}5266 5267static int init_map_in_map_slots(struct bpf_object *obj, struct bpf_map *map)5268{5269	const struct bpf_map *targ_map;5270	unsigned int i;5271	int fd, err = 0;5272 5273	for (i = 0; i < map->init_slots_sz; i++) {5274		if (!map->init_slots[i])5275			continue;5276 5277		targ_map = map->init_slots[i];5278		fd = targ_map->fd;5279 5280		if (obj->gen_loader) {5281			bpf_gen__populate_outer_map(obj->gen_loader,5282						    map - obj->maps, i,5283						    targ_map - obj->maps);5284		} else {5285			err = bpf_map_update_elem(map->fd, &i, &fd, 0);5286		}5287		if (err) {5288			err = -errno;5289			pr_warn("map '%s': failed to initialize slot [%d] to map '%s' fd=%d: %d\n",5290				map->name, i, targ_map->name, fd, err);5291			return err;5292		}5293		pr_debug("map '%s': slot [%d] set to map '%s' fd=%d\n",5294			 map->name, i, targ_map->name, fd);5295	}5296 5297	zfree(&map->init_slots);5298	map->init_slots_sz = 0;5299 5300	return 0;5301}5302 5303static int init_prog_array_slots(struct bpf_object *obj, struct bpf_map *map)5304{5305	const struct bpf_program *targ_prog;5306	unsigned int i;5307	int fd, err;5308 5309	if (obj->gen_loader)5310		return -ENOTSUP;5311 5312	for (i = 0; i < map->init_slots_sz; i++) {5313		if (!map->init_slots[i])5314			continue;5315 5316		targ_prog = map->init_slots[i];5317		fd = bpf_program__fd(targ_prog);5318 5319		err = bpf_map_update_elem(map->fd, &i, &fd, 0);5320		if (err) {5321			err = -errno;5322			pr_warn("map '%s': failed to initialize slot [%d] to prog '%s' fd=%d: %d\n",5323				map->name, i, targ_prog->name, fd, err);5324			return err;5325		}5326		pr_debug("map '%s': slot [%d] set to prog '%s' fd=%d\n",5327			 map->name, i, targ_prog->name, fd);5328	}5329 5330	zfree(&map->init_slots);5331	map->init_slots_sz = 0;5332 5333	return 0;5334}5335 5336static int bpf_object_init_prog_arrays(struct bpf_object *obj)5337{5338	struct bpf_map *map;5339	int i, err;5340 5341	for (i = 0; i < obj->nr_maps; i++) {5342		map = &obj->maps[i];5343 5344		if (!map->init_slots_sz || map->def.type != BPF_MAP_TYPE_PROG_ARRAY)5345			continue;5346 5347		err = init_prog_array_slots(obj, map);5348		if (err < 0)5349			return err;5350	}5351	return 0;5352}5353 5354static int map_set_def_max_entries(struct bpf_map *map)5355{5356	if (map->def.type == BPF_MAP_TYPE_PERF_EVENT_ARRAY && !map->def.max_entries) {5357		int nr_cpus;5358 5359		nr_cpus = libbpf_num_possible_cpus();5360		if (nr_cpus < 0) {5361			pr_warn("map '%s': failed to determine number of system CPUs: %d\n",5362				map->name, nr_cpus);5363			return nr_cpus;5364		}5365		pr_debug("map '%s': setting size to %d\n", map->name, nr_cpus);5366		map->def.max_entries = nr_cpus;5367	}5368 5369	return 0;5370}5371 5372static int5373bpf_object__create_maps(struct bpf_object *obj)5374{5375	struct bpf_map *map;5376	char *cp, errmsg[STRERR_BUFSIZE];5377	unsigned int i, j;5378	int err;5379	bool retried;5380 5381	for (i = 0; i < obj->nr_maps; i++) {5382		map = &obj->maps[i];5383 5384		/* To support old kernels, we skip creating global data maps5385		 * (.rodata, .data, .kconfig, etc); later on, during program5386		 * loading, if we detect that at least one of the to-be-loaded5387		 * programs is referencing any global data map, we'll error5388		 * out with program name and relocation index logged.5389		 * This approach allows to accommodate Clang emitting5390		 * unnecessary .rodata.str1.1 sections for string literals,5391		 * but also it allows to have CO-RE applications that use5392		 * global variables in some of BPF programs, but not others.5393		 * If those global variable-using programs are not loaded at5394		 * runtime due to bpf_program__set_autoload(prog, false),5395		 * bpf_object loading will succeed just fine even on old5396		 * kernels.5397		 */5398		if (bpf_map__is_internal(map) && !kernel_supports(obj, FEAT_GLOBAL_DATA))5399			map->autocreate = false;5400 5401		if (!map->autocreate) {5402			pr_debug("map '%s': skipped auto-creating...\n", map->name);5403			continue;5404		}5405 5406		err = map_set_def_max_entries(map);5407		if (err)5408			goto err_out;5409 5410		retried = false;5411retry:5412		if (map->pin_path) {5413			err = bpf_object__reuse_map(map);5414			if (err) {5415				pr_warn("map '%s': error reusing pinned map\n",5416					map->name);5417				goto err_out;5418			}5419			if (retried && map->fd < 0) {5420				pr_warn("map '%s': cannot find pinned map\n",5421					map->name);5422				err = -ENOENT;5423				goto err_out;5424			}5425		}5426 5427		if (map->reused) {5428			pr_debug("map '%s': skipping creation (preset fd=%d)\n",5429				 map->name, map->fd);5430		} else {5431			err = bpf_object__create_map(obj, map, false);5432			if (err)5433				goto err_out;5434 5435			pr_debug("map '%s': created successfully, fd=%d\n",5436				 map->name, map->fd);5437 5438			if (bpf_map__is_internal(map)) {5439				err = bpf_object__populate_internal_map(obj, map);5440				if (err < 0)5441					goto err_out;5442			}5443			if (map->def.type == BPF_MAP_TYPE_ARENA) {5444				map->mmaped = mmap((void *)(long)map->map_extra,5445						   bpf_map_mmap_sz(map), PROT_READ | PROT_WRITE,5446						   map->map_extra ? MAP_SHARED | MAP_FIXED : MAP_SHARED,5447						   map->fd, 0);5448				if (map->mmaped == MAP_FAILED) {5449					err = -errno;5450					map->mmaped = NULL;5451					pr_warn("map '%s': failed to mmap arena: %d\n",5452						map->name, err);5453					return err;5454				}5455				if (obj->arena_data) {5456					memcpy(map->mmaped, obj->arena_data, obj->arena_data_sz);5457					zfree(&obj->arena_data);5458				}5459			}5460			if (map->init_slots_sz && map->def.type != BPF_MAP_TYPE_PROG_ARRAY) {5461				err = init_map_in_map_slots(obj, map);5462				if (err < 0)5463					goto err_out;5464			}5465		}5466 5467		if (map->pin_path && !map->pinned) {5468			err = bpf_map__pin(map, NULL);5469			if (err) {5470				if (!retried && err == -EEXIST) {5471					retried = true;5472					goto retry;5473				}5474				pr_warn("map '%s': failed to auto-pin at '%s': %d\n",5475					map->name, map->pin_path, err);5476				goto err_out;5477			}5478		}5479	}5480 5481	return 0;5482 5483err_out:5484	cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg));5485	pr_warn("map '%s': failed to create: %s(%d)\n", map->name, cp, err);5486	pr_perm_msg(err);5487	for (j = 0; j < i; j++)5488		zclose(obj->maps[j].fd);5489	return err;5490}5491 5492static bool bpf_core_is_flavor_sep(const char *s)5493{5494	/* check X___Y name pattern, where X and Y are not underscores */5495	return s[0] != '_' &&				      /* X */5496	       s[1] == '_' && s[2] == '_' && s[3] == '_' &&   /* ___ */5497	       s[4] != '_';				      /* Y */5498}5499 5500/* Given 'some_struct_name___with_flavor' return the length of a name prefix5501 * before last triple underscore. Struct name part after last triple5502 * underscore is ignored by BPF CO-RE relocation during relocation matching.5503 */5504size_t bpf_core_essential_name_len(const char *name)5505{5506	size_t n = strlen(name);5507	int i;5508 5509	for (i = n - 5; i >= 0; i--) {5510		if (bpf_core_is_flavor_sep(name + i))5511			return i + 1;5512	}5513	return n;5514}5515 5516void bpf_core_free_cands(struct bpf_core_cand_list *cands)5517{5518	if (!cands)5519		return;5520 5521	free(cands->cands);5522	free(cands);5523}5524 5525int bpf_core_add_cands(struct bpf_core_cand *local_cand,5526		       size_t local_essent_len,5527		       const struct btf *targ_btf,5528		       const char *targ_btf_name,5529		       int targ_start_id,5530		       struct bpf_core_cand_list *cands)5531{5532	struct bpf_core_cand *new_cands, *cand;5533	const struct btf_type *t, *local_t;5534	const char *targ_name, *local_name;5535	size_t targ_essent_len;5536	int n, i;5537 5538	local_t = btf__type_by_id(local_cand->btf, local_cand->id);5539	local_name = btf__str_by_offset(local_cand->btf, local_t->name_off);5540 5541	n = btf__type_cnt(targ_btf);5542	for (i = targ_start_id; i < n; i++) {5543		t = btf__type_by_id(targ_btf, i);5544		if (!btf_kind_core_compat(t, local_t))5545			continue;5546 5547		targ_name = btf__name_by_offset(targ_btf, t->name_off);5548		if (str_is_empty(targ_name))5549			continue;5550 5551		targ_essent_len = bpf_core_essential_name_len(targ_name);5552		if (targ_essent_len != local_essent_len)5553			continue;5554 5555		if (strncmp(local_name, targ_name, local_essent_len) != 0)5556			continue;5557 5558		pr_debug("CO-RE relocating [%d] %s %s: found target candidate [%d] %s %s in [%s]\n",5559			 local_cand->id, btf_kind_str(local_t),5560			 local_name, i, btf_kind_str(t), targ_name,5561			 targ_btf_name);5562		new_cands = libbpf_reallocarray(cands->cands, cands->len + 1,5563					      sizeof(*cands->cands));5564		if (!new_cands)5565			return -ENOMEM;5566 5567		cand = &new_cands[cands->len];5568		cand->btf = targ_btf;5569		cand->id = i;5570 5571		cands->cands = new_cands;5572		cands->len++;5573	}5574	return 0;5575}5576 5577static int load_module_btfs(struct bpf_object *obj)5578{5579	struct bpf_btf_info info;5580	struct module_btf *mod_btf;5581	struct btf *btf;5582	char name[64];5583	__u32 id = 0, len;5584	int err, fd;5585 5586	if (obj->btf_modules_loaded)5587		return 0;5588 5589	if (obj->gen_loader)5590		return 0;5591 5592	/* don't do this again, even if we find no module BTFs */5593	obj->btf_modules_loaded = true;5594 5595	/* kernel too old to support module BTFs */5596	if (!kernel_supports(obj, FEAT_MODULE_BTF))5597		return 0;5598 5599	while (true) {5600		err = bpf_btf_get_next_id(id, &id);5601		if (err && errno == ENOENT)5602			return 0;5603		if (err && errno == EPERM) {5604			pr_debug("skipping module BTFs loading, missing privileges\n");5605			return 0;5606		}5607		if (err) {5608			err = -errno;5609			pr_warn("failed to iterate BTF objects: %d\n", err);5610			return err;5611		}5612 5613		fd = bpf_btf_get_fd_by_id(id);5614		if (fd < 0) {5615			if (errno == ENOENT)5616				continue; /* expected race: BTF was unloaded */5617			err = -errno;5618			pr_warn("failed to get BTF object #%d FD: %d\n", id, err);5619			return err;5620		}5621 5622		len = sizeof(info);5623		memset(&info, 0, sizeof(info));5624		info.name = ptr_to_u64(name);5625		info.name_len = sizeof(name);5626 5627		err = bpf_btf_get_info_by_fd(fd, &info, &len);5628		if (err) {5629			err = -errno;5630			pr_warn("failed to get BTF object #%d info: %d\n", id, err);5631			goto err_out;5632		}5633 5634		/* ignore non-module BTFs */5635		if (!info.kernel_btf || strcmp(name, "vmlinux") == 0) {5636			close(fd);5637			continue;5638		}5639 5640		btf = btf_get_from_fd(fd, obj->btf_vmlinux);5641		err = libbpf_get_error(btf);5642		if (err) {5643			pr_warn("failed to load module [%s]'s BTF object #%d: %d\n",5644				name, id, err);5645			goto err_out;5646		}5647 5648		err = libbpf_ensure_mem((void **)&obj->btf_modules, &obj->btf_module_cap,5649					sizeof(*obj->btf_modules), obj->btf_module_cnt + 1);5650		if (err)5651			goto err_out;5652 5653		mod_btf = &obj->btf_modules[obj->btf_module_cnt++];5654 5655		mod_btf->btf = btf;5656		mod_btf->id = id;5657		mod_btf->fd = fd;5658		mod_btf->name = strdup(name);5659		if (!mod_btf->name) {5660			err = -ENOMEM;5661			goto err_out;5662		}5663		continue;5664 5665err_out:5666		close(fd);5667		return err;5668	}5669 5670	return 0;5671}5672 5673static struct bpf_core_cand_list *5674bpf_core_find_cands(struct bpf_object *obj, const struct btf *local_btf, __u32 local_type_id)5675{5676	struct bpf_core_cand local_cand = {};5677	struct bpf_core_cand_list *cands;5678	const struct btf *main_btf;5679	const struct btf_type *local_t;5680	const char *local_name;5681	size_t local_essent_len;5682	int err, i;5683 5684	local_cand.btf = local_btf;5685	local_cand.id = local_type_id;5686	local_t = btf__type_by_id(local_btf, local_type_id);5687	if (!local_t)5688		return ERR_PTR(-EINVAL);5689 5690	local_name = btf__name_by_offset(local_btf, local_t->name_off);5691	if (str_is_empty(local_name))5692		return ERR_PTR(-EINVAL);5693	local_essent_len = bpf_core_essential_name_len(local_name);5694 5695	cands = calloc(1, sizeof(*cands));5696	if (!cands)5697		return ERR_PTR(-ENOMEM);5698 5699	/* Attempt to find target candidates in vmlinux BTF first */5700	main_btf = obj->btf_vmlinux_override ?: obj->btf_vmlinux;5701	err = bpf_core_add_cands(&local_cand, local_essent_len, main_btf, "vmlinux", 1, cands);5702	if (err)5703		goto err_out;5704 5705	/* if vmlinux BTF has any candidate, don't got for module BTFs */5706	if (cands->len)5707		return cands;5708 5709	/* if vmlinux BTF was overridden, don't attempt to load module BTFs */5710	if (obj->btf_vmlinux_override)5711		return cands;5712 5713	/* now look through module BTFs, trying to still find candidates */5714	err = load_module_btfs(obj);5715	if (err)5716		goto err_out;5717 5718	for (i = 0; i < obj->btf_module_cnt; i++) {5719		err = bpf_core_add_cands(&local_cand, local_essent_len,5720					 obj->btf_modules[i].btf,5721					 obj->btf_modules[i].name,5722					 btf__type_cnt(obj->btf_vmlinux),5723					 cands);5724		if (err)5725			goto err_out;5726	}5727 5728	return cands;5729err_out:5730	bpf_core_free_cands(cands);5731	return ERR_PTR(err);5732}5733 5734/* Check local and target types for compatibility. This check is used for5735 * type-based CO-RE relocations and follow slightly different rules than5736 * field-based relocations. This function assumes that root types were already5737 * checked for name match. Beyond that initial root-level name check, names5738 * are completely ignored. Compatibility rules are as follows:5739 *   - any two STRUCTs/UNIONs/FWDs/ENUMs/INTs are considered compatible, but5740 *     kind should match for local and target types (i.e., STRUCT is not5741 *     compatible with UNION);5742 *   - for ENUMs, the size is ignored;5743 *   - for INT, size and signedness are ignored;5744 *   - for ARRAY, dimensionality is ignored, element types are checked for5745 *     compatibility recursively;5746 *   - CONST/VOLATILE/RESTRICT modifiers are ignored;5747 *   - TYPEDEFs/PTRs are compatible if types they pointing to are compatible;5748 *   - FUNC_PROTOs are compatible if they have compatible signature: same5749 *     number of input args and compatible return and argument types.5750 * These rules are not set in stone and probably will be adjusted as we get5751 * more experience with using BPF CO-RE relocations.5752 */5753int bpf_core_types_are_compat(const struct btf *local_btf, __u32 local_id,5754			      const struct btf *targ_btf, __u32 targ_id)5755{5756	return __bpf_core_types_are_compat(local_btf, local_id, targ_btf, targ_id, 32);5757}5758 5759int bpf_core_types_match(const struct btf *local_btf, __u32 local_id,5760			 const struct btf *targ_btf, __u32 targ_id)5761{5762	return __bpf_core_types_match(local_btf, local_id, targ_btf, targ_id, false, 32);5763}5764 5765static size_t bpf_core_hash_fn(const long key, void *ctx)5766{5767	return key;5768}5769 5770static bool bpf_core_equal_fn(const long k1, const long k2, void *ctx)5771{5772	return k1 == k2;5773}5774 5775static int record_relo_core(struct bpf_program *prog,5776			    const struct bpf_core_relo *core_relo, int insn_idx)5777{5778	struct reloc_desc *relos, *relo;5779 5780	relos = libbpf_reallocarray(prog->reloc_desc,5781				    prog->nr_reloc + 1, sizeof(*relos));5782	if (!relos)5783		return -ENOMEM;5784	relo = &relos[prog->nr_reloc];5785	relo->type = RELO_CORE;5786	relo->insn_idx = insn_idx;5787	relo->core_relo = core_relo;5788	prog->reloc_desc = relos;5789	prog->nr_reloc++;5790	return 0;5791}5792 5793static const struct bpf_core_relo *find_relo_core(struct bpf_program *prog, int insn_idx)5794{5795	struct reloc_desc *relo;5796	int i;5797 5798	for (i = 0; i < prog->nr_reloc; i++) {5799		relo = &prog->reloc_desc[i];5800		if (relo->type != RELO_CORE || relo->insn_idx != insn_idx)5801			continue;5802 5803		return relo->core_relo;5804	}5805 5806	return NULL;5807}5808 5809static int bpf_core_resolve_relo(struct bpf_program *prog,5810				 const struct bpf_core_relo *relo,5811				 int relo_idx,5812				 const struct btf *local_btf,5813				 struct hashmap *cand_cache,5814				 struct bpf_core_relo_res *targ_res)5815{5816	struct bpf_core_spec specs_scratch[3] = {};5817	struct bpf_core_cand_list *cands = NULL;5818	const char *prog_name = prog->name;5819	const struct btf_type *local_type;5820	const char *local_name;5821	__u32 local_id = relo->type_id;5822	int err;5823 5824	local_type = btf__type_by_id(local_btf, local_id);5825	if (!local_type)5826		return -EINVAL;5827 5828	local_name = btf__name_by_offset(local_btf, local_type->name_off);5829	if (!local_name)5830		return -EINVAL;5831 5832	if (relo->kind != BPF_CORE_TYPE_ID_LOCAL &&5833	    !hashmap__find(cand_cache, local_id, &cands)) {5834		cands = bpf_core_find_cands(prog->obj, local_btf, local_id);5835		if (IS_ERR(cands)) {5836			pr_warn("prog '%s': relo #%d: target candidate search failed for [%d] %s %s: %ld\n",5837				prog_name, relo_idx, local_id, btf_kind_str(local_type),5838				local_name, PTR_ERR(cands));5839			return PTR_ERR(cands);5840		}5841		err = hashmap__set(cand_cache, local_id, cands, NULL, NULL);5842		if (err) {5843			bpf_core_free_cands(cands);5844			return err;5845		}5846	}5847 5848	return bpf_core_calc_relo_insn(prog_name, relo, relo_idx, local_btf, cands, specs_scratch,5849				       targ_res);5850}5851 5852static int5853bpf_object__relocate_core(struct bpf_object *obj, const char *targ_btf_path)5854{5855	const struct btf_ext_info_sec *sec;5856	struct bpf_core_relo_res targ_res;5857	const struct bpf_core_relo *rec;5858	const struct btf_ext_info *seg;5859	struct hashmap_entry *entry;5860	struct hashmap *cand_cache = NULL;5861	struct bpf_program *prog;5862	struct bpf_insn *insn;5863	const char *sec_name;5864	int i, err = 0, insn_idx, sec_idx, sec_num;5865 5866	if (obj->btf_ext->core_relo_info.len == 0)5867		return 0;5868 5869	if (targ_btf_path) {5870		obj->btf_vmlinux_override = btf__parse(targ_btf_path, NULL);5871		err = libbpf_get_error(obj->btf_vmlinux_override);5872		if (err) {5873			pr_warn("failed to parse target BTF: %d\n", err);5874			return err;5875		}5876	}5877 5878	cand_cache = hashmap__new(bpf_core_hash_fn, bpf_core_equal_fn, NULL);5879	if (IS_ERR(cand_cache)) {5880		err = PTR_ERR(cand_cache);5881		goto out;5882	}5883 5884	seg = &obj->btf_ext->core_relo_info;5885	sec_num = 0;5886	for_each_btf_ext_sec(seg, sec) {5887		sec_idx = seg->sec_idxs[sec_num];5888		sec_num++;5889 5890		sec_name = btf__name_by_offset(obj->btf, sec->sec_name_off);5891		if (str_is_empty(sec_name)) {5892			err = -EINVAL;5893			goto out;5894		}5895 5896		pr_debug("sec '%s': found %d CO-RE relocations\n", sec_name, sec->num_info);5897 5898		for_each_btf_ext_rec(seg, sec, i, rec) {5899			if (rec->insn_off % BPF_INSN_SZ)5900				return -EINVAL;5901			insn_idx = rec->insn_off / BPF_INSN_SZ;5902			prog = find_prog_by_sec_insn(obj, sec_idx, insn_idx);5903			if (!prog) {5904				/* When __weak subprog is "overridden" by another instance5905				 * of the subprog from a different object file, linker still5906				 * appends all the .BTF.ext info that used to belong to that5907				 * eliminated subprogram.5908				 * This is similar to what x86-64 linker does for relocations.5909				 * So just ignore such relocations just like we ignore5910				 * subprog instructions when discovering subprograms.5911				 */5912				pr_debug("sec '%s': skipping CO-RE relocation #%d for insn #%d belonging to eliminated weak subprogram\n",5913					 sec_name, i, insn_idx);5914				continue;5915			}5916			/* no need to apply CO-RE relocation if the program is5917			 * not going to be loaded5918			 */5919			if (!prog->autoload)5920				continue;5921 5922			/* adjust insn_idx from section frame of reference to the local5923			 * program's frame of reference; (sub-)program code is not yet5924			 * relocated, so it's enough to just subtract in-section offset5925			 */5926			insn_idx = insn_idx - prog->sec_insn_off;5927			if (insn_idx >= prog->insns_cnt)5928				return -EINVAL;5929			insn = &prog->insns[insn_idx];5930 5931			err = record_relo_core(prog, rec, insn_idx);5932			if (err) {5933				pr_warn("prog '%s': relo #%d: failed to record relocation: %d\n",5934					prog->name, i, err);5935				goto out;5936			}5937 5938			if (prog->obj->gen_loader)5939				continue;5940 5941			err = bpf_core_resolve_relo(prog, rec, i, obj->btf, cand_cache, &targ_res);5942			if (err) {5943				pr_warn("prog '%s': relo #%d: failed to relocate: %d\n",5944					prog->name, i, err);5945				goto out;5946			}5947 5948			err = bpf_core_patch_insn(prog->name, insn, insn_idx, rec, i, &targ_res);5949			if (err) {5950				pr_warn("prog '%s': relo #%d: failed to patch insn #%u: %d\n",5951					prog->name, i, insn_idx, err);5952				goto out;5953			}5954		}5955	}5956 5957out:5958	/* obj->btf_vmlinux and module BTFs are freed after object load */5959	btf__free(obj->btf_vmlinux_override);5960	obj->btf_vmlinux_override = NULL;5961 5962	if (!IS_ERR_OR_NULL(cand_cache)) {5963		hashmap__for_each_entry(cand_cache, entry, i) {5964			bpf_core_free_cands(entry->pvalue);5965		}5966		hashmap__free(cand_cache);5967	}5968	return err;5969}5970 5971/* base map load ldimm64 special constant, used also for log fixup logic */5972#define POISON_LDIMM64_MAP_BASE 20010000005973#define POISON_LDIMM64_MAP_PFX "200100"5974 5975static void poison_map_ldimm64(struct bpf_program *prog, int relo_idx,5976			       int insn_idx, struct bpf_insn *insn,5977			       int map_idx, const struct bpf_map *map)5978{5979	int i;5980 5981	pr_debug("prog '%s': relo #%d: poisoning insn #%d that loads map #%d '%s'\n",5982		 prog->name, relo_idx, insn_idx, map_idx, map->name);5983 5984	/* we turn single ldimm64 into two identical invalid calls */5985	for (i = 0; i < 2; i++) {5986		insn->code = BPF_JMP | BPF_CALL;5987		insn->dst_reg = 0;5988		insn->src_reg = 0;5989		insn->off = 0;5990		/* if this instruction is reachable (not a dead code),5991		 * verifier will complain with something like:5992		 * invalid func unknown#20010001235993		 * where lower 123 is map index into obj->maps[] array5994		 */5995		insn->imm = POISON_LDIMM64_MAP_BASE + map_idx;5996 5997		insn++;5998	}5999}6000 6001/* unresolved kfunc call special constant, used also for log fixup logic */6002#define POISON_CALL_KFUNC_BASE 20020000006003#define POISON_CALL_KFUNC_PFX "2002"6004 6005static void poison_kfunc_call(struct bpf_program *prog, int relo_idx,6006			      int insn_idx, struct bpf_insn *insn,6007			      int ext_idx, const struct extern_desc *ext)6008{6009	pr_debug("prog '%s': relo #%d: poisoning insn #%d that calls kfunc '%s'\n",6010		 prog->name, relo_idx, insn_idx, ext->name);6011 6012	/* we turn kfunc call into invalid helper call with identifiable constant */6013	insn->code = BPF_JMP | BPF_CALL;6014	insn->dst_reg = 0;6015	insn->src_reg = 0;6016	insn->off = 0;6017	/* if this instruction is reachable (not a dead code),6018	 * verifier will complain with something like:6019	 * invalid func unknown#20010001236020	 * where lower 123 is extern index into obj->externs[] array6021	 */6022	insn->imm = POISON_CALL_KFUNC_BASE + ext_idx;6023}6024 6025/* Relocate data references within program code:6026 *  - map references;6027 *  - global variable references;6028 *  - extern references.6029 */6030static int6031bpf_object__relocate_data(struct bpf_object *obj, struct bpf_program *prog)6032{6033	int i;6034 6035	for (i = 0; i < prog->nr_reloc; i++) {6036		struct reloc_desc *relo = &prog->reloc_desc[i];6037		struct bpf_insn *insn = &prog->insns[relo->insn_idx];6038		const struct bpf_map *map;6039		struct extern_desc *ext;6040 6041		switch (relo->type) {6042		case RELO_LD64:6043			map = &obj->maps[relo->map_idx];6044			if (obj->gen_loader) {6045				insn[0].src_reg = BPF_PSEUDO_MAP_IDX;6046				insn[0].imm = relo->map_idx;6047			} else if (map->autocreate) {6048				insn[0].src_reg = BPF_PSEUDO_MAP_FD;6049				insn[0].imm = map->fd;6050			} else {6051				poison_map_ldimm64(prog, i, relo->insn_idx, insn,6052						   relo->map_idx, map);6053			}6054			break;6055		case RELO_DATA:6056			map = &obj->maps[relo->map_idx];6057			insn[1].imm = insn[0].imm + relo->sym_off;6058			if (obj->gen_loader) {6059				insn[0].src_reg = BPF_PSEUDO_MAP_IDX_VALUE;6060				insn[0].imm = relo->map_idx;6061			} else if (map->autocreate) {6062				insn[0].src_reg = BPF_PSEUDO_MAP_VALUE;6063				insn[0].imm = map->fd;6064			} else {6065				poison_map_ldimm64(prog, i, relo->insn_idx, insn,6066						   relo->map_idx, map);6067			}6068			break;6069		case RELO_EXTERN_LD64:6070			ext = &obj->externs[relo->ext_idx];6071			if (ext->type == EXT_KCFG) {6072				if (obj->gen_loader) {6073					insn[0].src_reg = BPF_PSEUDO_MAP_IDX_VALUE;6074					insn[0].imm = obj->kconfig_map_idx;6075				} else {6076					insn[0].src_reg = BPF_PSEUDO_MAP_VALUE;6077					insn[0].imm = obj->maps[obj->kconfig_map_idx].fd;6078				}6079				insn[1].imm = ext->kcfg.data_off;6080			} else /* EXT_KSYM */ {6081				if (ext->ksym.type_id && ext->is_set) { /* typed ksyms */6082					insn[0].src_reg = BPF_PSEUDO_BTF_ID;6083					insn[0].imm = ext->ksym.kernel_btf_id;6084					insn[1].imm = ext->ksym.kernel_btf_obj_fd;6085				} else { /* typeless ksyms or unresolved typed ksyms */6086					insn[0].imm = (__u32)ext->ksym.addr;6087					insn[1].imm = ext->ksym.addr >> 32;6088				}6089			}6090			break;6091		case RELO_EXTERN_CALL:6092			ext = &obj->externs[relo->ext_idx];6093			insn[0].src_reg = BPF_PSEUDO_KFUNC_CALL;6094			if (ext->is_set) {6095				insn[0].imm = ext->ksym.kernel_btf_id;6096				insn[0].off = ext->ksym.btf_fd_idx;6097			} else { /* unresolved weak kfunc call */6098				poison_kfunc_call(prog, i, relo->insn_idx, insn,6099						  relo->ext_idx, ext);6100			}6101			break;6102		case RELO_SUBPROG_ADDR:6103			if (insn[0].src_reg != BPF_PSEUDO_FUNC) {6104				pr_warn("prog '%s': relo #%d: bad insn\n",6105					prog->name, i);6106				return -EINVAL;6107			}6108			/* handled already */6109			break;6110		case RELO_CALL:6111			/* handled already */6112			break;6113		case RELO_CORE:6114			/* will be handled by bpf_program_record_relos() */6115			break;6116		default:6117			pr_warn("prog '%s': relo #%d: bad relo type %d\n",6118				prog->name, i, relo->type);6119			return -EINVAL;6120		}6121	}6122 6123	return 0;6124}6125 6126static int adjust_prog_btf_ext_info(const struct bpf_object *obj,6127				    const struct bpf_program *prog,6128				    const struct btf_ext_info *ext_info,6129				    void **prog_info, __u32 *prog_rec_cnt,6130				    __u32 *prog_rec_sz)6131{6132	void *copy_start = NULL, *copy_end = NULL;6133	void *rec, *rec_end, *new_prog_info;6134	const struct btf_ext_info_sec *sec;6135	size_t old_sz, new_sz;6136	int i, sec_num, sec_idx, off_adj;6137 6138	sec_num = 0;6139	for_each_btf_ext_sec(ext_info, sec) {6140		sec_idx = ext_info->sec_idxs[sec_num];6141		sec_num++;6142		if (prog->sec_idx != sec_idx)6143			continue;6144 6145		for_each_btf_ext_rec(ext_info, sec, i, rec) {6146			__u32 insn_off = *(__u32 *)rec / BPF_INSN_SZ;6147 6148			if (insn_off < prog->sec_insn_off)6149				continue;6150			if (insn_off >= prog->sec_insn_off + prog->sec_insn_cnt)6151				break;6152 6153			if (!copy_start)6154				copy_start = rec;6155			copy_end = rec + ext_info->rec_size;6156		}6157 6158		if (!copy_start)6159			return -ENOENT;6160 6161		/* append func/line info of a given (sub-)program to the main6162		 * program func/line info6163		 */6164		old_sz = (size_t)(*prog_rec_cnt) * ext_info->rec_size;6165		new_sz = old_sz + (copy_end - copy_start);6166		new_prog_info = realloc(*prog_info, new_sz);6167		if (!new_prog_info)6168			return -ENOMEM;6169		*prog_info = new_prog_info;6170		*prog_rec_cnt = new_sz / ext_info->rec_size;6171		memcpy(new_prog_info + old_sz, copy_start, copy_end - copy_start);6172 6173		/* Kernel instruction offsets are in units of 8-byte6174		 * instructions, while .BTF.ext instruction offsets generated6175		 * by Clang are in units of bytes. So convert Clang offsets6176		 * into kernel offsets and adjust offset according to program6177		 * relocated position.6178		 */6179		off_adj = prog->sub_insn_off - prog->sec_insn_off;6180		rec = new_prog_info + old_sz;6181		rec_end = new_prog_info + new_sz;6182		for (; rec < rec_end; rec += ext_info->rec_size) {6183			__u32 *insn_off = rec;6184 6185			*insn_off = *insn_off / BPF_INSN_SZ + off_adj;6186		}6187		*prog_rec_sz = ext_info->rec_size;6188		return 0;6189	}6190 6191	return -ENOENT;6192}6193 6194static int6195reloc_prog_func_and_line_info(const struct bpf_object *obj,6196			      struct bpf_program *main_prog,6197			      const struct bpf_program *prog)6198{6199	int err;6200 6201	/* no .BTF.ext relocation if .BTF.ext is missing or kernel doesn't6202	 * support func/line info6203	 */6204	if (!obj->btf_ext || !kernel_supports(obj, FEAT_BTF_FUNC))6205		return 0;6206 6207	/* only attempt func info relocation if main program's func_info6208	 * relocation was successful6209	 */6210	if (main_prog != prog && !main_prog->func_info)6211		goto line_info;6212 6213	err = adjust_prog_btf_ext_info(obj, prog, &obj->btf_ext->func_info,6214				       &main_prog->func_info,6215				       &main_prog->func_info_cnt,6216				       &main_prog->func_info_rec_size);6217	if (err) {6218		if (err != -ENOENT) {6219			pr_warn("prog '%s': error relocating .BTF.ext function info: %d\n",6220				prog->name, err);6221			return err;6222		}6223		if (main_prog->func_info) {6224			/*6225			 * Some info has already been found but has problem6226			 * in the last btf_ext reloc. Must have to error out.6227			 */6228			pr_warn("prog '%s': missing .BTF.ext function info.\n", prog->name);6229			return err;6230		}6231		/* Have problem loading the very first info. Ignore the rest. */6232		pr_warn("prog '%s': missing .BTF.ext function info for the main program, skipping all of .BTF.ext func info.\n",6233			prog->name);6234	}6235 6236line_info:6237	/* don't relocate line info if main program's relocation failed */6238	if (main_prog != prog && !main_prog->line_info)6239		return 0;6240 6241	err = adjust_prog_btf_ext_info(obj, prog, &obj->btf_ext->line_info,6242				       &main_prog->line_info,6243				       &main_prog->line_info_cnt,6244				       &main_prog->line_info_rec_size);6245	if (err) {6246		if (err != -ENOENT) {6247			pr_warn("prog '%s': error relocating .BTF.ext line info: %d\n",6248				prog->name, err);6249			return err;6250		}6251		if (main_prog->line_info) {6252			/*6253			 * Some info has already been found but has problem6254			 * in the last btf_ext reloc. Must have to error out.6255			 */6256			pr_warn("prog '%s': missing .BTF.ext line info.\n", prog->name);6257			return err;6258		}6259		/* Have problem loading the very first info. Ignore the rest. */6260		pr_warn("prog '%s': missing .BTF.ext line info for the main program, skipping all of .BTF.ext line info.\n",6261			prog->name);6262	}6263	return 0;6264}6265 6266static int cmp_relo_by_insn_idx(const void *key, const void *elem)6267{6268	size_t insn_idx = *(const size_t *)key;6269	const struct reloc_desc *relo = elem;6270 6271	if (insn_idx == relo->insn_idx)6272		return 0;6273	return insn_idx < relo->insn_idx ? -1 : 1;6274}6275 6276static struct reloc_desc *find_prog_insn_relo(const struct bpf_program *prog, size_t insn_idx)6277{6278	if (!prog->nr_reloc)6279		return NULL;6280	return bsearch(&insn_idx, prog->reloc_desc, prog->nr_reloc,6281		       sizeof(*prog->reloc_desc), cmp_relo_by_insn_idx);6282}6283 6284static int append_subprog_relos(struct bpf_program *main_prog, struct bpf_program *subprog)6285{6286	int new_cnt = main_prog->nr_reloc + subprog->nr_reloc;6287	struct reloc_desc *relos;6288	int i;6289 6290	if (main_prog == subprog)6291		return 0;6292	relos = libbpf_reallocarray(main_prog->reloc_desc, new_cnt, sizeof(*relos));6293	/* if new count is zero, reallocarray can return a valid NULL result;6294	 * in this case the previous pointer will be freed, so we *have to*6295	 * reassign old pointer to the new value (even if it's NULL)6296	 */6297	if (!relos && new_cnt)6298		return -ENOMEM;6299	if (subprog->nr_reloc)6300		memcpy(relos + main_prog->nr_reloc, subprog->reloc_desc,6301		       sizeof(*relos) * subprog->nr_reloc);6302 6303	for (i = main_prog->nr_reloc; i < new_cnt; i++)6304		relos[i].insn_idx += subprog->sub_insn_off;6305	/* After insn_idx adjustment the 'relos' array is still sorted6306	 * by insn_idx and doesn't break bsearch.6307	 */6308	main_prog->reloc_desc = relos;6309	main_prog->nr_reloc = new_cnt;6310	return 0;6311}6312 6313static int6314bpf_object__append_subprog_code(struct bpf_object *obj, struct bpf_program *main_prog,6315				struct bpf_program *subprog)6316{6317       struct bpf_insn *insns;6318       size_t new_cnt;6319       int err;6320 6321       subprog->sub_insn_off = main_prog->insns_cnt;6322 6323       new_cnt = main_prog->insns_cnt + subprog->insns_cnt;6324       insns = libbpf_reallocarray(main_prog->insns, new_cnt, sizeof(*insns));6325       if (!insns) {6326               pr_warn("prog '%s': failed to realloc prog code\n", main_prog->name);6327               return -ENOMEM;6328       }6329       main_prog->insns = insns;6330       main_prog->insns_cnt = new_cnt;6331 6332       memcpy(main_prog->insns + subprog->sub_insn_off, subprog->insns,6333              subprog->insns_cnt * sizeof(*insns));6334 6335       pr_debug("prog '%s': added %zu insns from sub-prog '%s'\n",6336                main_prog->name, subprog->insns_cnt, subprog->name);6337 6338       /* The subprog insns are now appended. Append its relos too. */6339       err = append_subprog_relos(main_prog, subprog);6340       if (err)6341               return err;6342       return 0;6343}6344 6345static int6346bpf_object__reloc_code(struct bpf_object *obj, struct bpf_program *main_prog,6347		       struct bpf_program *prog)6348{6349	size_t sub_insn_idx, insn_idx;6350	struct bpf_program *subprog;6351	struct reloc_desc *relo;6352	struct bpf_insn *insn;6353	int err;6354 6355	err = reloc_prog_func_and_line_info(obj, main_prog, prog);6356	if (err)6357		return err;6358 6359	for (insn_idx = 0; insn_idx < prog->sec_insn_cnt; insn_idx++) {6360		insn = &main_prog->insns[prog->sub_insn_off + insn_idx];6361		if (!insn_is_subprog_call(insn) && !insn_is_pseudo_func(insn))6362			continue;6363 6364		relo = find_prog_insn_relo(prog, insn_idx);6365		if (relo && relo->type == RELO_EXTERN_CALL)6366			/* kfunc relocations will be handled later6367			 * in bpf_object__relocate_data()6368			 */6369			continue;6370		if (relo && relo->type != RELO_CALL && relo->type != RELO_SUBPROG_ADDR) {6371			pr_warn("prog '%s': unexpected relo for insn #%zu, type %d\n",6372				prog->name, insn_idx, relo->type);6373			return -LIBBPF_ERRNO__RELOC;6374		}6375		if (relo) {6376			/* sub-program instruction index is a combination of6377			 * an offset of a symbol pointed to by relocation and6378			 * call instruction's imm field; for global functions,6379			 * call always has imm = -1, but for static functions6380			 * relocation is against STT_SECTION and insn->imm6381			 * points to a start of a static function6382			 *6383			 * for subprog addr relocation, the relo->sym_off + insn->imm is6384			 * the byte offset in the corresponding section.6385			 */6386			if (relo->type == RELO_CALL)6387				sub_insn_idx = relo->sym_off / BPF_INSN_SZ + insn->imm + 1;6388			else6389				sub_insn_idx = (relo->sym_off + insn->imm) / BPF_INSN_SZ;6390		} else if (insn_is_pseudo_func(insn)) {6391			/*6392			 * RELO_SUBPROG_ADDR relo is always emitted even if both6393			 * functions are in the same section, so it shouldn't reach here.6394			 */6395			pr_warn("prog '%s': missing subprog addr relo for insn #%zu\n",6396				prog->name, insn_idx);6397			return -LIBBPF_ERRNO__RELOC;6398		} else {6399			/* if subprogram call is to a static function within6400			 * the same ELF section, there won't be any relocation6401			 * emitted, but it also means there is no additional6402			 * offset necessary, insns->imm is relative to6403			 * instruction's original position within the section6404			 */6405			sub_insn_idx = prog->sec_insn_off + insn_idx + insn->imm + 1;6406		}6407 6408		/* we enforce that sub-programs should be in .text section */6409		subprog = find_prog_by_sec_insn(obj, obj->efile.text_shndx, sub_insn_idx);6410		if (!subprog) {6411			pr_warn("prog '%s': no .text section found yet sub-program call exists\n",6412				prog->name);6413			return -LIBBPF_ERRNO__RELOC;6414		}6415 6416		/* if it's the first call instruction calling into this6417		 * subprogram (meaning this subprog hasn't been processed6418		 * yet) within the context of current main program:6419		 *   - append it at the end of main program's instructions blog;6420		 *   - process is recursively, while current program is put on hold;6421		 *   - if that subprogram calls some other not yet processes6422		 *   subprogram, same thing will happen recursively until6423		 *   there are no more unprocesses subprograms left to append6424		 *   and relocate.6425		 */6426		if (subprog->sub_insn_off == 0) {6427			err = bpf_object__append_subprog_code(obj, main_prog, subprog);6428			if (err)6429				return err;6430			err = bpf_object__reloc_code(obj, main_prog, subprog);6431			if (err)6432				return err;6433		}6434 6435		/* main_prog->insns memory could have been re-allocated, so6436		 * calculate pointer again6437		 */6438		insn = &main_prog->insns[prog->sub_insn_off + insn_idx];6439		/* calculate correct instruction position within current main6440		 * prog; each main prog can have a different set of6441		 * subprograms appended (potentially in different order as6442		 * well), so position of any subprog can be different for6443		 * different main programs6444		 */6445		insn->imm = subprog->sub_insn_off - (prog->sub_insn_off + insn_idx) - 1;6446 6447		pr_debug("prog '%s': insn #%zu relocated, imm %d points to subprog '%s' (now at %zu offset)\n",6448			 prog->name, insn_idx, insn->imm, subprog->name, subprog->sub_insn_off);6449	}6450 6451	return 0;6452}6453 6454/*6455 * Relocate sub-program calls.6456 *6457 * Algorithm operates as follows. Each entry-point BPF program (referred to as6458 * main prog) is processed separately. For each subprog (non-entry functions,6459 * that can be called from either entry progs or other subprogs) gets their6460 * sub_insn_off reset to zero. This serves as indicator that this subprogram6461 * hasn't been yet appended and relocated within current main prog. Once its6462 * relocated, sub_insn_off will point at the position within current main prog6463 * where given subprog was appended. This will further be used to relocate all6464 * the call instructions jumping into this subprog.6465 *6466 * We start with main program and process all call instructions. If the call6467 * is into a subprog that hasn't been processed (i.e., subprog->sub_insn_off6468 * is zero), subprog instructions are appended at the end of main program's6469 * instruction array. Then main program is "put on hold" while we recursively6470 * process newly appended subprogram. If that subprogram calls into another6471 * subprogram that hasn't been appended, new subprogram is appended again to6472 * the *main* prog's instructions (subprog's instructions are always left6473 * untouched, as they need to be in unmodified state for subsequent main progs6474 * and subprog instructions are always sent only as part of a main prog) and6475 * the process continues recursively. Once all the subprogs called from a main6476 * prog or any of its subprogs are appended (and relocated), all their6477 * positions within finalized instructions array are known, so it's easy to6478 * rewrite call instructions with correct relative offsets, corresponding to6479 * desired target subprog.6480 *6481 * Its important to realize that some subprogs might not be called from some6482 * main prog and any of its called/used subprogs. Those will keep their6483 * subprog->sub_insn_off as zero at all times and won't be appended to current6484 * main prog and won't be relocated within the context of current main prog.6485 * They might still be used from other main progs later.6486 *6487 * Visually this process can be shown as below. Suppose we have two main6488 * programs mainA and mainB and BPF object contains three subprogs: subA,6489 * subB, and subC. mainA calls only subA, mainB calls only subC, but subA and6490 * subC both call subB:6491 *6492 *        +--------+ +-------+6493 *        |        v v       |6494 *     +--+---+ +--+-+-+ +---+--+6495 *     | subA | | subB | | subC |6496 *     +--+---+ +------+ +---+--+6497 *        ^                  ^6498 *        |                  |6499 *    +---+-------+   +------+----+6500 *    |   mainA   |   |   mainB   |6501 *    +-----------+   +-----------+6502 *6503 * We'll start relocating mainA, will find subA, append it and start6504 * processing sub A recursively:6505 *6506 *    +-----------+------+6507 *    |   mainA   | subA |6508 *    +-----------+------+6509 *6510 * At this point we notice that subB is used from subA, so we append it and6511 * relocate (there are no further subcalls from subB):6512 *6513 *    +-----------+------+------+6514 *    |   mainA   | subA | subB |6515 *    +-----------+------+------+6516 *6517 * At this point, we relocate subA calls, then go one level up and finish with6518 * relocatin mainA calls. mainA is done.6519 *6520 * For mainB process is similar but results in different order. We start with6521 * mainB and skip subA and subB, as mainB never calls them (at least6522 * directly), but we see subC is needed, so we append and start processing it:6523 *6524 *    +-----------+------+6525 *    |   mainB   | subC |6526 *    +-----------+------+6527 * Now we see subC needs subB, so we go back to it, append and relocate it:6528 *6529 *    +-----------+------+------+6530 *    |   mainB   | subC | subB |6531 *    +-----------+------+------+6532 *6533 * At this point we unwind recursion, relocate calls in subC, then in mainB.6534 */6535static int6536bpf_object__relocate_calls(struct bpf_object *obj, struct bpf_program *prog)6537{6538	struct bpf_program *subprog;6539	int i, err;6540 6541	/* mark all subprogs as not relocated (yet) within the context of6542	 * current main program6543	 */6544	for (i = 0; i < obj->nr_programs; i++) {6545		subprog = &obj->programs[i];6546		if (!prog_is_subprog(obj, subprog))6547			continue;6548 6549		subprog->sub_insn_off = 0;6550	}6551 6552	err = bpf_object__reloc_code(obj, prog, prog);6553	if (err)6554		return err;6555 6556	return 0;6557}6558 6559static void6560bpf_object__free_relocs(struct bpf_object *obj)6561{6562	struct bpf_program *prog;6563	int i;6564 6565	/* free up relocation descriptors */6566	for (i = 0; i < obj->nr_programs; i++) {6567		prog = &obj->programs[i];6568		zfree(&prog->reloc_desc);6569		prog->nr_reloc = 0;6570	}6571}6572 6573static int cmp_relocs(const void *_a, const void *_b)6574{6575	const struct reloc_desc *a = _a;6576	const struct reloc_desc *b = _b;6577 6578	if (a->insn_idx != b->insn_idx)6579		return a->insn_idx < b->insn_idx ? -1 : 1;6580 6581	/* no two relocations should have the same insn_idx, but ... */6582	if (a->type != b->type)6583		return a->type < b->type ? -1 : 1;6584 6585	return 0;6586}6587 6588static void bpf_object__sort_relos(struct bpf_object *obj)6589{6590	int i;6591 6592	for (i = 0; i < obj->nr_programs; i++) {6593		struct bpf_program *p = &obj->programs[i];6594 6595		if (!p->nr_reloc)6596			continue;6597 6598		qsort(p->reloc_desc, p->nr_reloc, sizeof(*p->reloc_desc), cmp_relocs);6599	}6600}6601 6602static int bpf_prog_assign_exc_cb(struct bpf_object *obj, struct bpf_program *prog)6603{6604	const char *str = "exception_callback:";6605	size_t pfx_len = strlen(str);6606	int i, j, n;6607 6608	if (!obj->btf || !kernel_supports(obj, FEAT_BTF_DECL_TAG))6609		return 0;6610 6611	n = btf__type_cnt(obj->btf);6612	for (i = 1; i < n; i++) {6613		const char *name;6614		struct btf_type *t;6615 6616		t = btf_type_by_id(obj->btf, i);6617		if (!btf_is_decl_tag(t) || btf_decl_tag(t)->component_idx != -1)6618			continue;6619 6620		name = btf__str_by_offset(obj->btf, t->name_off);6621		if (strncmp(name, str, pfx_len) != 0)6622			continue;6623 6624		t = btf_type_by_id(obj->btf, t->type);6625		if (!btf_is_func(t) || btf_func_linkage(t) != BTF_FUNC_GLOBAL) {6626			pr_warn("prog '%s': exception_callback:<value> decl tag not applied to the main program\n",6627				prog->name);6628			return -EINVAL;6629		}6630		if (strcmp(prog->name, btf__str_by_offset(obj->btf, t->name_off)) != 0)6631			continue;6632		/* Multiple callbacks are specified for the same prog,6633		 * the verifier will eventually return an error for this6634		 * case, hence simply skip appending a subprog.6635		 */6636		if (prog->exception_cb_idx >= 0) {6637			prog->exception_cb_idx = -1;6638			break;6639		}6640 6641		name += pfx_len;6642		if (str_is_empty(name)) {6643			pr_warn("prog '%s': exception_callback:<value> decl tag contains empty value\n",6644				prog->name);6645			return -EINVAL;6646		}6647 6648		for (j = 0; j < obj->nr_programs; j++) {6649			struct bpf_program *subprog = &obj->programs[j];6650 6651			if (!prog_is_subprog(obj, subprog))6652				continue;6653			if (strcmp(name, subprog->name) != 0)6654				continue;6655			/* Enforce non-hidden, as from verifier point of6656			 * view it expects global functions, whereas the6657			 * mark_btf_static fixes up linkage as static.6658			 */6659			if (!subprog->sym_global || subprog->mark_btf_static) {6660				pr_warn("prog '%s': exception callback %s must be a global non-hidden function\n",6661					prog->name, subprog->name);6662				return -EINVAL;6663			}6664			/* Let's see if we already saw a static exception callback with the same name */6665			if (prog->exception_cb_idx >= 0) {6666				pr_warn("prog '%s': multiple subprogs with same name as exception callback '%s'\n",6667					prog->name, subprog->name);6668				return -EINVAL;6669			}6670			prog->exception_cb_idx = j;6671			break;6672		}6673 6674		if (prog->exception_cb_idx >= 0)6675			continue;6676 6677		pr_warn("prog '%s': cannot find exception callback '%s'\n", prog->name, name);6678		return -ENOENT;6679	}6680 6681	return 0;6682}6683 6684static struct {6685	enum bpf_prog_type prog_type;6686	const char *ctx_name;6687} global_ctx_map[] = {6688	{ BPF_PROG_TYPE_CGROUP_DEVICE,           "bpf_cgroup_dev_ctx" },6689	{ BPF_PROG_TYPE_CGROUP_SKB,              "__sk_buff" },6690	{ BPF_PROG_TYPE_CGROUP_SOCK,             "bpf_sock" },6691	{ BPF_PROG_TYPE_CGROUP_SOCK_ADDR,        "bpf_sock_addr" },6692	{ BPF_PROG_TYPE_CGROUP_SOCKOPT,          "bpf_sockopt" },6693	{ BPF_PROG_TYPE_CGROUP_SYSCTL,           "bpf_sysctl" },6694	{ BPF_PROG_TYPE_FLOW_DISSECTOR,          "__sk_buff" },6695	{ BPF_PROG_TYPE_KPROBE,                  "bpf_user_pt_regs_t" },6696	{ BPF_PROG_TYPE_LWT_IN,                  "__sk_buff" },6697	{ BPF_PROG_TYPE_LWT_OUT,                 "__sk_buff" },6698	{ BPF_PROG_TYPE_LWT_SEG6LOCAL,           "__sk_buff" },6699	{ BPF_PROG_TYPE_LWT_XMIT,                "__sk_buff" },6700	{ BPF_PROG_TYPE_NETFILTER,               "bpf_nf_ctx" },6701	{ BPF_PROG_TYPE_PERF_EVENT,              "bpf_perf_event_data" },6702	{ BPF_PROG_TYPE_RAW_TRACEPOINT,          "bpf_raw_tracepoint_args" },6703	{ BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE, "bpf_raw_tracepoint_args" },6704	{ BPF_PROG_TYPE_SCHED_ACT,               "__sk_buff" },6705	{ BPF_PROG_TYPE_SCHED_CLS,               "__sk_buff" },6706	{ BPF_PROG_TYPE_SK_LOOKUP,               "bpf_sk_lookup" },6707	{ BPF_PROG_TYPE_SK_MSG,                  "sk_msg_md" },6708	{ BPF_PROG_TYPE_SK_REUSEPORT,            "sk_reuseport_md" },6709	{ BPF_PROG_TYPE_SK_SKB,                  "__sk_buff" },6710	{ BPF_PROG_TYPE_SOCK_OPS,                "bpf_sock_ops" },6711	{ BPF_PROG_TYPE_SOCKET_FILTER,           "__sk_buff" },6712	{ BPF_PROG_TYPE_XDP,                     "xdp_md" },6713	/* all other program types don't have "named" context structs */6714};6715 6716/* forward declarations for arch-specific underlying types of bpf_user_pt_regs_t typedef,6717 * for below __builtin_types_compatible_p() checks;6718 * with this approach we don't need any extra arch-specific #ifdef guards6719 */6720struct pt_regs;6721struct user_pt_regs;6722struct user_regs_struct;6723 6724static bool need_func_arg_type_fixup(const struct btf *btf, const struct bpf_program *prog,6725				     const char *subprog_name, int arg_idx,6726				     int arg_type_id, const char *ctx_name)6727{6728	const struct btf_type *t;6729	const char *tname;6730 6731	/* check if existing parameter already matches verifier expectations */6732	t = skip_mods_and_typedefs(btf, arg_type_id, NULL);6733	if (!btf_is_ptr(t))6734		goto out_warn;6735 6736	/* typedef bpf_user_pt_regs_t is a special PITA case, valid for kprobe6737	 * and perf_event programs, so check this case early on and forget6738	 * about it for subsequent checks6739	 */6740	while (btf_is_mod(t))6741		t = btf__type_by_id(btf, t->type);6742	if (btf_is_typedef(t) &&6743	    (prog->type == BPF_PROG_TYPE_KPROBE || prog->type == BPF_PROG_TYPE_PERF_EVENT)) {6744		tname = btf__str_by_offset(btf, t->name_off) ?: "<anon>";6745		if (strcmp(tname, "bpf_user_pt_regs_t") == 0)6746			return false; /* canonical type for kprobe/perf_event */6747	}6748 6749	/* now we can ignore typedefs moving forward */6750	t = skip_mods_and_typedefs(btf, t->type, NULL);6751 6752	/* if it's `void *`, definitely fix up BTF info */6753	if (btf_is_void(t))6754		return true;6755 6756	/* if it's already proper canonical type, no need to fix up */6757	tname = btf__str_by_offset(btf, t->name_off) ?: "<anon>";6758	if (btf_is_struct(t) && strcmp(tname, ctx_name) == 0)6759		return false;6760 6761	/* special cases */6762	switch (prog->type) {6763	case BPF_PROG_TYPE_KPROBE:6764		/* `struct pt_regs *` is expected, but we need to fix up */6765		if (btf_is_struct(t) && strcmp(tname, "pt_regs") == 0)6766			return true;6767		break;6768	case BPF_PROG_TYPE_PERF_EVENT:6769		if (__builtin_types_compatible_p(bpf_user_pt_regs_t, struct pt_regs) &&6770		    btf_is_struct(t) && strcmp(tname, "pt_regs") == 0)6771			return true;6772		if (__builtin_types_compatible_p(bpf_user_pt_regs_t, struct user_pt_regs) &&6773		    btf_is_struct(t) && strcmp(tname, "user_pt_regs") == 0)6774			return true;6775		if (__builtin_types_compatible_p(bpf_user_pt_regs_t, struct user_regs_struct) &&6776		    btf_is_struct(t) && strcmp(tname, "user_regs_struct") == 0)6777			return true;6778		break;6779	case BPF_PROG_TYPE_RAW_TRACEPOINT:6780	case BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE:6781		/* allow u64* as ctx */6782		if (btf_is_int(t) && t->size == 8)6783			return true;6784		break;6785	default:6786		break;6787	}6788 6789out_warn:6790	pr_warn("prog '%s': subprog '%s' arg#%d is expected to be of `struct %s *` type\n",6791		prog->name, subprog_name, arg_idx, ctx_name);6792	return false;6793}6794 6795static int clone_func_btf_info(struct btf *btf, int orig_fn_id, struct bpf_program *prog)6796{6797	int fn_id, fn_proto_id, ret_type_id, orig_proto_id;6798	int i, err, arg_cnt, fn_name_off, linkage;6799	struct btf_type *fn_t, *fn_proto_t, *t;6800	struct btf_param *p;6801 6802	/* caller already validated FUNC -> FUNC_PROTO validity */6803	fn_t = btf_type_by_id(btf, orig_fn_id);6804	fn_proto_t = btf_type_by_id(btf, fn_t->type);6805 6806	/* Note that each btf__add_xxx() operation invalidates6807	 * all btf_type and string pointers, so we need to be6808	 * very careful when cloning BTF types. BTF type6809	 * pointers have to be always refetched. And to avoid6810	 * problems with invalidated string pointers, we6811	 * add empty strings initially, then just fix up6812	 * name_off offsets in place. Offsets are stable for6813	 * existing strings, so that works out.6814	 */6815	fn_name_off = fn_t->name_off; /* we are about to invalidate fn_t */6816	linkage = btf_func_linkage(fn_t);6817	orig_proto_id = fn_t->type; /* original FUNC_PROTO ID */6818	ret_type_id = fn_proto_t->type; /* fn_proto_t will be invalidated */6819	arg_cnt = btf_vlen(fn_proto_t);6820 6821	/* clone FUNC_PROTO and its params */6822	fn_proto_id = btf__add_func_proto(btf, ret_type_id);6823	if (fn_proto_id < 0)6824		return -EINVAL;6825 6826	for (i = 0; i < arg_cnt; i++) {6827		int name_off;6828 6829		/* copy original parameter data */6830		t = btf_type_by_id(btf, orig_proto_id);6831		p = &btf_params(t)[i];6832		name_off = p->name_off;6833 6834		err = btf__add_func_param(btf, "", p->type);6835		if (err)6836			return err;6837 6838		fn_proto_t = btf_type_by_id(btf, fn_proto_id);6839		p = &btf_params(fn_proto_t)[i];6840		p->name_off = name_off; /* use remembered str offset */6841	}6842 6843	/* clone FUNC now, btf__add_func() enforces non-empty name, so use6844	 * entry program's name as a placeholder, which we replace immediately6845	 * with original name_off6846	 */6847	fn_id = btf__add_func(btf, prog->name, linkage, fn_proto_id);6848	if (fn_id < 0)6849		return -EINVAL;6850 6851	fn_t = btf_type_by_id(btf, fn_id);6852	fn_t->name_off = fn_name_off; /* reuse original string */6853 6854	return fn_id;6855}6856 6857/* Check if main program or global subprog's function prototype has `arg:ctx`6858 * argument tags, and, if necessary, substitute correct type to match what BPF6859 * verifier would expect, taking into account specific program type. This6860 * allows to support __arg_ctx tag transparently on old kernels that don't yet6861 * have a native support for it in the verifier, making user's life much6862 * easier.6863 */6864static int bpf_program_fixup_func_info(struct bpf_object *obj, struct bpf_program *prog)6865{6866	const char *ctx_name = NULL, *ctx_tag = "arg:ctx", *fn_name;6867	struct bpf_func_info_min *func_rec;6868	struct btf_type *fn_t, *fn_proto_t;6869	struct btf *btf = obj->btf;6870	const struct btf_type *t;6871	struct btf_param *p;6872	int ptr_id = 0, struct_id, tag_id, orig_fn_id;6873	int i, n, arg_idx, arg_cnt, err, rec_idx;6874	int *orig_ids;6875 6876	/* no .BTF.ext, no problem */6877	if (!obj->btf_ext || !prog->func_info)6878		return 0;6879 6880	/* don't do any fix ups if kernel natively supports __arg_ctx */6881	if (kernel_supports(obj, FEAT_ARG_CTX_TAG))6882		return 0;6883 6884	/* some BPF program types just don't have named context structs, so6885	 * this fallback mechanism doesn't work for them6886	 */6887	for (i = 0; i < ARRAY_SIZE(global_ctx_map); i++) {6888		if (global_ctx_map[i].prog_type != prog->type)6889			continue;6890		ctx_name = global_ctx_map[i].ctx_name;6891		break;6892	}6893	if (!ctx_name)6894		return 0;6895 6896	/* remember original func BTF IDs to detect if we already cloned them */6897	orig_ids = calloc(prog->func_info_cnt, sizeof(*orig_ids));6898	if (!orig_ids)6899		return -ENOMEM;6900	for (i = 0; i < prog->func_info_cnt; i++) {6901		func_rec = prog->func_info + prog->func_info_rec_size * i;6902		orig_ids[i] = func_rec->type_id;6903	}6904 6905	/* go through each DECL_TAG with "arg:ctx" and see if it points to one6906	 * of our subprogs; if yes and subprog is global and needs adjustment,6907	 * clone and adjust FUNC -> FUNC_PROTO combo6908	 */6909	for (i = 1, n = btf__type_cnt(btf); i < n; i++) {6910		/* only DECL_TAG with "arg:ctx" value are interesting */6911		t = btf__type_by_id(btf, i);6912		if (!btf_is_decl_tag(t))6913			continue;6914		if (strcmp(btf__str_by_offset(btf, t->name_off), ctx_tag) != 0)6915			continue;6916 6917		/* only global funcs need adjustment, if at all */6918		orig_fn_id = t->type;6919		fn_t = btf_type_by_id(btf, orig_fn_id);6920		if (!btf_is_func(fn_t) || btf_func_linkage(fn_t) != BTF_FUNC_GLOBAL)6921			continue;6922 6923		/* sanity check FUNC -> FUNC_PROTO chain, just in case */6924		fn_proto_t = btf_type_by_id(btf, fn_t->type);6925		if (!fn_proto_t || !btf_is_func_proto(fn_proto_t))6926			continue;6927 6928		/* find corresponding func_info record */6929		func_rec = NULL;6930		for (rec_idx = 0; rec_idx < prog->func_info_cnt; rec_idx++) {6931			if (orig_ids[rec_idx] == t->type) {6932				func_rec = prog->func_info + prog->func_info_rec_size * rec_idx;6933				break;6934			}6935		}6936		/* current main program doesn't call into this subprog */6937		if (!func_rec)6938			continue;6939 6940		/* some more sanity checking of DECL_TAG */6941		arg_cnt = btf_vlen(fn_proto_t);6942		arg_idx = btf_decl_tag(t)->component_idx;6943		if (arg_idx < 0 || arg_idx >= arg_cnt)6944			continue;6945 6946		/* check if we should fix up argument type */6947		p = &btf_params(fn_proto_t)[arg_idx];6948		fn_name = btf__str_by_offset(btf, fn_t->name_off) ?: "<anon>";6949		if (!need_func_arg_type_fixup(btf, prog, fn_name, arg_idx, p->type, ctx_name))6950			continue;6951 6952		/* clone fn/fn_proto, unless we already did it for another arg */6953		if (func_rec->type_id == orig_fn_id) {6954			int fn_id;6955 6956			fn_id = clone_func_btf_info(btf, orig_fn_id, prog);6957			if (fn_id < 0) {6958				err = fn_id;6959				goto err_out;6960			}6961 6962			/* point func_info record to a cloned FUNC type */6963			func_rec->type_id = fn_id;6964		}6965 6966		/* create PTR -> STRUCT type chain to mark PTR_TO_CTX argument;6967		 * we do it just once per main BPF program, as all global6968		 * funcs share the same program type, so need only PTR ->6969		 * STRUCT type chain6970		 */6971		if (ptr_id == 0) {6972			struct_id = btf__add_struct(btf, ctx_name, 0);6973			ptr_id = btf__add_ptr(btf, struct_id);6974			if (ptr_id < 0 || struct_id < 0) {6975				err = -EINVAL;6976				goto err_out;6977			}6978		}6979 6980		/* for completeness, clone DECL_TAG and point it to cloned param */6981		tag_id = btf__add_decl_tag(btf, ctx_tag, func_rec->type_id, arg_idx);6982		if (tag_id < 0) {6983			err = -EINVAL;6984			goto err_out;6985		}6986 6987		/* all the BTF manipulations invalidated pointers, refetch them */6988		fn_t = btf_type_by_id(btf, func_rec->type_id);6989		fn_proto_t = btf_type_by_id(btf, fn_t->type);6990 6991		/* fix up type ID pointed to by param */6992		p = &btf_params(fn_proto_t)[arg_idx];6993		p->type = ptr_id;6994	}6995 6996	free(orig_ids);6997	return 0;6998err_out:6999	free(orig_ids);7000	return err;7001}7002 7003static int bpf_object__relocate(struct bpf_object *obj, const char *targ_btf_path)7004{7005	struct bpf_program *prog;7006	size_t i, j;7007	int err;7008 7009	if (obj->btf_ext) {7010		err = bpf_object__relocate_core(obj, targ_btf_path);7011		if (err) {7012			pr_warn("failed to perform CO-RE relocations: %d\n",7013				err);7014			return err;7015		}7016		bpf_object__sort_relos(obj);7017	}7018 7019	/* Before relocating calls pre-process relocations and mark7020	 * few ld_imm64 instructions that points to subprogs.7021	 * Otherwise bpf_object__reloc_code() later would have to consider7022	 * all ld_imm64 insns as relocation candidates. That would7023	 * reduce relocation speed, since amount of find_prog_insn_relo()7024	 * would increase and most of them will fail to find a relo.7025	 */7026	for (i = 0; i < obj->nr_programs; i++) {7027		prog = &obj->programs[i];7028		for (j = 0; j < prog->nr_reloc; j++) {7029			struct reloc_desc *relo = &prog->reloc_desc[j];7030			struct bpf_insn *insn = &prog->insns[relo->insn_idx];7031 7032			/* mark the insn, so it's recognized by insn_is_pseudo_func() */7033			if (relo->type == RELO_SUBPROG_ADDR)7034				insn[0].src_reg = BPF_PSEUDO_FUNC;7035		}7036	}7037 7038	/* relocate subprogram calls and append used subprograms to main7039	 * programs; each copy of subprogram code needs to be relocated7040	 * differently for each main program, because its code location might7041	 * have changed.7042	 * Append subprog relos to main programs to allow data relos to be7043	 * processed after text is completely relocated.7044	 */7045	for (i = 0; i < obj->nr_programs; i++) {7046		prog = &obj->programs[i];7047		/* sub-program's sub-calls are relocated within the context of7048		 * its main program only7049		 */7050		if (prog_is_subprog(obj, prog))7051			continue;7052		if (!prog->autoload)7053			continue;7054 7055		err = bpf_object__relocate_calls(obj, prog);7056		if (err) {7057			pr_warn("prog '%s': failed to relocate calls: %d\n",7058				prog->name, err);7059			return err;7060		}7061 7062		err = bpf_prog_assign_exc_cb(obj, prog);7063		if (err)7064			return err;7065		/* Now, also append exception callback if it has not been done already. */7066		if (prog->exception_cb_idx >= 0) {7067			struct bpf_program *subprog = &obj->programs[prog->exception_cb_idx];7068 7069			/* Calling exception callback directly is disallowed, which the7070			 * verifier will reject later. In case it was processed already,7071			 * we can skip this step, otherwise for all other valid cases we7072			 * have to append exception callback now.7073			 */7074			if (subprog->sub_insn_off == 0) {7075				err = bpf_object__append_subprog_code(obj, prog, subprog);7076				if (err)7077					return err;7078				err = bpf_object__reloc_code(obj, prog, subprog);7079				if (err)7080					return err;7081			}7082		}7083	}7084	for (i = 0; i < obj->nr_programs; i++) {7085		prog = &obj->programs[i];7086		if (prog_is_subprog(obj, prog))7087			continue;7088		if (!prog->autoload)7089			continue;7090 7091		/* Process data relos for main programs */7092		err = bpf_object__relocate_data(obj, prog);7093		if (err) {7094			pr_warn("prog '%s': failed to relocate data references: %d\n",7095				prog->name, err);7096			return err;7097		}7098 7099		/* Fix up .BTF.ext information, if necessary */7100		err = bpf_program_fixup_func_info(obj, prog);7101		if (err) {7102			pr_warn("prog '%s': failed to perform .BTF.ext fix ups: %d\n",7103				prog->name, err);7104			return err;7105		}7106	}7107 7108	return 0;7109}7110 7111static int bpf_object__collect_st_ops_relos(struct bpf_object *obj,7112					    Elf64_Shdr *shdr, Elf_Data *data);7113 7114static int bpf_object__collect_map_relos(struct bpf_object *obj,7115					 Elf64_Shdr *shdr, Elf_Data *data)7116{7117	const int bpf_ptr_sz = 8, host_ptr_sz = sizeof(void *);7118	int i, j, nrels, new_sz;7119	const struct btf_var_secinfo *vi = NULL;7120	const struct btf_type *sec, *var, *def;7121	struct bpf_map *map = NULL, *targ_map = NULL;7122	struct bpf_program *targ_prog = NULL;7123	bool is_prog_array, is_map_in_map;7124	const struct btf_member *member;7125	const char *name, *mname, *type;7126	unsigned int moff;7127	Elf64_Sym *sym;7128	Elf64_Rel *rel;7129	void *tmp;7130 7131	if (!obj->efile.btf_maps_sec_btf_id || !obj->btf)7132		return -EINVAL;7133	sec = btf__type_by_id(obj->btf, obj->efile.btf_maps_sec_btf_id);7134	if (!sec)7135		return -EINVAL;7136 7137	nrels = shdr->sh_size / shdr->sh_entsize;7138	for (i = 0; i < nrels; i++) {7139		rel = elf_rel_by_idx(data, i);7140		if (!rel) {7141			pr_warn(".maps relo #%d: failed to get ELF relo\n", i);7142			return -LIBBPF_ERRNO__FORMAT;7143		}7144 7145		sym = elf_sym_by_idx(obj, ELF64_R_SYM(rel->r_info));7146		if (!sym) {7147			pr_warn(".maps relo #%d: symbol %zx not found\n",7148				i, (size_t)ELF64_R_SYM(rel->r_info));7149			return -LIBBPF_ERRNO__FORMAT;7150		}7151		name = elf_sym_str(obj, sym->st_name) ?: "<?>";7152 7153		pr_debug(".maps relo #%d: for %zd value %zd rel->r_offset %zu name %d ('%s')\n",7154			 i, (ssize_t)(rel->r_info >> 32), (size_t)sym->st_value,7155			 (size_t)rel->r_offset, sym->st_name, name);7156 7157		for (j = 0; j < obj->nr_maps; j++) {7158			map = &obj->maps[j];7159			if (map->sec_idx != obj->efile.btf_maps_shndx)7160				continue;7161 7162			vi = btf_var_secinfos(sec) + map->btf_var_idx;7163			if (vi->offset <= rel->r_offset &&7164			    rel->r_offset + bpf_ptr_sz <= vi->offset + vi->size)7165				break;7166		}7167		if (j == obj->nr_maps) {7168			pr_warn(".maps relo #%d: cannot find map '%s' at rel->r_offset %zu\n",7169				i, name, (size_t)rel->r_offset);7170			return -EINVAL;7171		}7172 7173		is_map_in_map = bpf_map_type__is_map_in_map(map->def.type);7174		is_prog_array = map->def.type == BPF_MAP_TYPE_PROG_ARRAY;7175		type = is_map_in_map ? "map" : "prog";7176		if (is_map_in_map) {7177			if (sym->st_shndx != obj->efile.btf_maps_shndx) {7178				pr_warn(".maps relo #%d: '%s' isn't a BTF-defined map\n",7179					i, name);7180				return -LIBBPF_ERRNO__RELOC;7181			}7182			if (map->def.type == BPF_MAP_TYPE_HASH_OF_MAPS &&7183			    map->def.key_size != sizeof(int)) {7184				pr_warn(".maps relo #%d: hash-of-maps '%s' should have key size %zu.\n",7185					i, map->name, sizeof(int));7186				return -EINVAL;7187			}7188			targ_map = bpf_object__find_map_by_name(obj, name);7189			if (!targ_map) {7190				pr_warn(".maps relo #%d: '%s' isn't a valid map reference\n",7191					i, name);7192				return -ESRCH;7193			}7194		} else if (is_prog_array) {7195			targ_prog = bpf_object__find_program_by_name(obj, name);7196			if (!targ_prog) {7197				pr_warn(".maps relo #%d: '%s' isn't a valid program reference\n",7198					i, name);7199				return -ESRCH;7200			}7201			if (targ_prog->sec_idx != sym->st_shndx ||7202			    targ_prog->sec_insn_off * 8 != sym->st_value ||7203			    prog_is_subprog(obj, targ_prog)) {7204				pr_warn(".maps relo #%d: '%s' isn't an entry-point program\n",7205					i, name);7206				return -LIBBPF_ERRNO__RELOC;7207			}7208		} else {7209			return -EINVAL;7210		}7211 7212		var = btf__type_by_id(obj->btf, vi->type);7213		def = skip_mods_and_typedefs(obj->btf, var->type, NULL);7214		if (btf_vlen(def) == 0)7215			return -EINVAL;7216		member = btf_members(def) + btf_vlen(def) - 1;7217		mname = btf__name_by_offset(obj->btf, member->name_off);7218		if (strcmp(mname, "values"))7219			return -EINVAL;7220 7221		moff = btf_member_bit_offset(def, btf_vlen(def) - 1) / 8;7222		if (rel->r_offset - vi->offset < moff)7223			return -EINVAL;7224 7225		moff = rel->r_offset - vi->offset - moff;7226		/* here we use BPF pointer size, which is always 64 bit, as we7227		 * are parsing ELF that was built for BPF target7228		 */7229		if (moff % bpf_ptr_sz)7230			return -EINVAL;7231		moff /= bpf_ptr_sz;7232		if (moff >= map->init_slots_sz) {7233			new_sz = moff + 1;7234			tmp = libbpf_reallocarray(map->init_slots, new_sz, host_ptr_sz);7235			if (!tmp)7236				return -ENOMEM;7237			map->init_slots = tmp;7238			memset(map->init_slots + map->init_slots_sz, 0,7239			       (new_sz - map->init_slots_sz) * host_ptr_sz);7240			map->init_slots_sz = new_sz;7241		}7242		map->init_slots[moff] = is_map_in_map ? (void *)targ_map : (void *)targ_prog;7243 7244		pr_debug(".maps relo #%d: map '%s' slot [%d] points to %s '%s'\n",7245			 i, map->name, moff, type, name);7246	}7247 7248	return 0;7249}7250 7251static int bpf_object__collect_relos(struct bpf_object *obj)7252{7253	int i, err;7254 7255	for (i = 0; i < obj->efile.sec_cnt; i++) {7256		struct elf_sec_desc *sec_desc = &obj->efile.secs[i];7257		Elf64_Shdr *shdr;7258		Elf_Data *data;7259		int idx;7260 7261		if (sec_desc->sec_type != SEC_RELO)7262			continue;7263 7264		shdr = sec_desc->shdr;7265		data = sec_desc->data;7266		idx = shdr->sh_info;7267 7268		if (shdr->sh_type != SHT_REL || idx < 0 || idx >= obj->efile.sec_cnt) {7269			pr_warn("internal error at %d\n", __LINE__);7270			return -LIBBPF_ERRNO__INTERNAL;7271		}7272 7273		if (obj->efile.secs[idx].sec_type == SEC_ST_OPS)7274			err = bpf_object__collect_st_ops_relos(obj, shdr, data);7275		else if (idx == obj->efile.btf_maps_shndx)7276			err = bpf_object__collect_map_relos(obj, shdr, data);7277		else7278			err = bpf_object__collect_prog_relos(obj, shdr, data);7279		if (err)7280			return err;7281	}7282 7283	bpf_object__sort_relos(obj);7284	return 0;7285}7286 7287static bool insn_is_helper_call(struct bpf_insn *insn, enum bpf_func_id *func_id)7288{7289	if (BPF_CLASS(insn->code) == BPF_JMP &&7290	    BPF_OP(insn->code) == BPF_CALL &&7291	    BPF_SRC(insn->code) == BPF_K &&7292	    insn->src_reg == 0 &&7293	    insn->dst_reg == 0) {7294		    *func_id = insn->imm;7295		    return true;7296	}7297	return false;7298}7299 7300static int bpf_object__sanitize_prog(struct bpf_object *obj, struct bpf_program *prog)7301{7302	struct bpf_insn *insn = prog->insns;7303	enum bpf_func_id func_id;7304	int i;7305 7306	if (obj->gen_loader)7307		return 0;7308 7309	for (i = 0; i < prog->insns_cnt; i++, insn++) {7310		if (!insn_is_helper_call(insn, &func_id))7311			continue;7312 7313		/* on kernels that don't yet support7314		 * bpf_probe_read_{kernel,user}[_str] helpers, fall back7315		 * to bpf_probe_read() which works well for old kernels7316		 */7317		switch (func_id) {7318		case BPF_FUNC_probe_read_kernel:7319		case BPF_FUNC_probe_read_user:7320			if (!kernel_supports(obj, FEAT_PROBE_READ_KERN))7321				insn->imm = BPF_FUNC_probe_read;7322			break;7323		case BPF_FUNC_probe_read_kernel_str:7324		case BPF_FUNC_probe_read_user_str:7325			if (!kernel_supports(obj, FEAT_PROBE_READ_KERN))7326				insn->imm = BPF_FUNC_probe_read_str;7327			break;7328		default:7329			break;7330		}7331	}7332	return 0;7333}7334 7335static int libbpf_find_attach_btf_id(struct bpf_program *prog, const char *attach_name,7336				     int *btf_obj_fd, int *btf_type_id);7337 7338/* this is called as prog->sec_def->prog_prepare_load_fn for libbpf-supported sec_defs */7339static int libbpf_prepare_prog_load(struct bpf_program *prog,7340				    struct bpf_prog_load_opts *opts, long cookie)7341{7342	enum sec_def_flags def = cookie;7343 7344	/* old kernels might not support specifying expected_attach_type */7345	if ((def & SEC_EXP_ATTACH_OPT) && !kernel_supports(prog->obj, FEAT_EXP_ATTACH_TYPE))7346		opts->expected_attach_type = 0;7347 7348	if (def & SEC_SLEEPABLE)7349		opts->prog_flags |= BPF_F_SLEEPABLE;7350 7351	if (prog->type == BPF_PROG_TYPE_XDP && (def & SEC_XDP_FRAGS))7352		opts->prog_flags |= BPF_F_XDP_HAS_FRAGS;7353 7354	/* special check for usdt to use uprobe_multi link */7355	if ((def & SEC_USDT) && kernel_supports(prog->obj, FEAT_UPROBE_MULTI_LINK))7356		prog->expected_attach_type = BPF_TRACE_UPROBE_MULTI;7357 7358	if ((def & SEC_ATTACH_BTF) && !prog->attach_btf_id) {7359		int btf_obj_fd = 0, btf_type_id = 0, err;7360		const char *attach_name;7361 7362		attach_name = strchr(prog->sec_name, '/');7363		if (!attach_name) {7364			/* if BPF program is annotated with just SEC("fentry")7365			 * (or similar) without declaratively specifying7366			 * target, then it is expected that target will be7367			 * specified with bpf_program__set_attach_target() at7368			 * runtime before BPF object load step. If not, then7369			 * there is nothing to load into the kernel as BPF7370			 * verifier won't be able to validate BPF program7371			 * correctness anyways.7372			 */7373			pr_warn("prog '%s': no BTF-based attach target is specified, use bpf_program__set_attach_target()\n",7374				prog->name);7375			return -EINVAL;7376		}7377		attach_name++; /* skip over / */7378 7379		err = libbpf_find_attach_btf_id(prog, attach_name, &btf_obj_fd, &btf_type_id);7380		if (err)7381			return err;7382 7383		/* cache resolved BTF FD and BTF type ID in the prog */7384		prog->attach_btf_obj_fd = btf_obj_fd;7385		prog->attach_btf_id = btf_type_id;7386 7387		/* but by now libbpf common logic is not utilizing7388		 * prog->atach_btf_obj_fd/prog->attach_btf_id anymore because7389		 * this callback is called after opts were populated by7390		 * libbpf, so this callback has to update opts explicitly here7391		 */7392		opts->attach_btf_obj_fd = btf_obj_fd;7393		opts->attach_btf_id = btf_type_id;7394	}7395	return 0;7396}7397 7398static void fixup_verifier_log(struct bpf_program *prog, char *buf, size_t buf_sz);7399 7400static int bpf_object_load_prog(struct bpf_object *obj, struct bpf_program *prog,7401				struct bpf_insn *insns, int insns_cnt,7402				const char *license, __u32 kern_version, int *prog_fd)7403{7404	LIBBPF_OPTS(bpf_prog_load_opts, load_attr);7405	const char *prog_name = NULL;7406	char *cp, errmsg[STRERR_BUFSIZE];7407	size_t log_buf_size = 0;7408	char *log_buf = NULL, *tmp;7409	bool own_log_buf = true;7410	__u32 log_level = prog->log_level;7411	int ret, err;7412 7413	/* Be more helpful by rejecting programs that can't be validated early7414	 * with more meaningful and actionable error message.7415	 */7416	switch (prog->type) {7417	case BPF_PROG_TYPE_UNSPEC:7418		/*7419		 * The program type must be set.  Most likely we couldn't find a proper7420		 * section definition at load time, and thus we didn't infer the type.7421		 */7422		pr_warn("prog '%s': missing BPF prog type, check ELF section name '%s'\n",7423			prog->name, prog->sec_name);7424		return -EINVAL;7425	case BPF_PROG_TYPE_STRUCT_OPS:7426		if (prog->attach_btf_id == 0) {7427			pr_warn("prog '%s': SEC(\"struct_ops\") program isn't referenced anywhere, did you forget to use it?\n",7428				prog->name);7429			return -EINVAL;7430		}7431		break;7432	default:7433		break;7434	}7435 7436	if (!insns || !insns_cnt)7437		return -EINVAL;7438 7439	if (kernel_supports(obj, FEAT_PROG_NAME))7440		prog_name = prog->name;7441	load_attr.attach_prog_fd = prog->attach_prog_fd;7442	load_attr.attach_btf_obj_fd = prog->attach_btf_obj_fd;7443	load_attr.attach_btf_id = prog->attach_btf_id;7444	load_attr.kern_version = kern_version;7445	load_attr.prog_ifindex = prog->prog_ifindex;7446 7447	/* specify func_info/line_info only if kernel supports them */7448	if (obj->btf && btf__fd(obj->btf) >= 0 && kernel_supports(obj, FEAT_BTF_FUNC)) {7449		load_attr.prog_btf_fd = btf__fd(obj->btf);7450		load_attr.func_info = prog->func_info;7451		load_attr.func_info_rec_size = prog->func_info_rec_size;7452		load_attr.func_info_cnt = prog->func_info_cnt;7453		load_attr.line_info = prog->line_info;7454		load_attr.line_info_rec_size = prog->line_info_rec_size;7455		load_attr.line_info_cnt = prog->line_info_cnt;7456	}7457	load_attr.log_level = log_level;7458	load_attr.prog_flags = prog->prog_flags;7459	load_attr.fd_array = obj->fd_array;7460 7461	load_attr.token_fd = obj->token_fd;7462	if (obj->token_fd)7463		load_attr.prog_flags |= BPF_F_TOKEN_FD;7464 7465	/* adjust load_attr if sec_def provides custom preload callback */7466	if (prog->sec_def && prog->sec_def->prog_prepare_load_fn) {7467		err = prog->sec_def->prog_prepare_load_fn(prog, &load_attr, prog->sec_def->cookie);7468		if (err < 0) {7469			pr_warn("prog '%s': failed to prepare load attributes: %d\n",7470				prog->name, err);7471			return err;7472		}7473		insns = prog->insns;7474		insns_cnt = prog->insns_cnt;7475	}7476 7477	/* allow prog_prepare_load_fn to change expected_attach_type */7478	load_attr.expected_attach_type = prog->expected_attach_type;7479 7480	if (obj->gen_loader) {7481		bpf_gen__prog_load(obj->gen_loader, prog->type, prog->name,7482				   license, insns, insns_cnt, &load_attr,7483				   prog - obj->programs);7484		*prog_fd = -1;7485		return 0;7486	}7487 7488retry_load:7489	/* if log_level is zero, we don't request logs initially even if7490	 * custom log_buf is specified; if the program load fails, then we'll7491	 * bump log_level to 1 and use either custom log_buf or we'll allocate7492	 * our own and retry the load to get details on what failed7493	 */7494	if (log_level) {7495		if (prog->log_buf) {7496			log_buf = prog->log_buf;7497			log_buf_size = prog->log_size;7498			own_log_buf = false;7499		} else if (obj->log_buf) {7500			log_buf = obj->log_buf;7501			log_buf_size = obj->log_size;7502			own_log_buf = false;7503		} else {7504			log_buf_size = max((size_t)BPF_LOG_BUF_SIZE, log_buf_size * 2);7505			tmp = realloc(log_buf, log_buf_size);7506			if (!tmp) {7507				ret = -ENOMEM;7508				goto out;7509			}7510			log_buf = tmp;7511			log_buf[0] = '\0';7512			own_log_buf = true;7513		}7514	}7515 7516	load_attr.log_buf = log_buf;7517	load_attr.log_size = log_buf_size;7518	load_attr.log_level = log_level;7519 7520	ret = bpf_prog_load(prog->type, prog_name, license, insns, insns_cnt, &load_attr);7521	if (ret >= 0) {7522		if (log_level && own_log_buf) {7523			pr_debug("prog '%s': -- BEGIN PROG LOAD LOG --\n%s-- END PROG LOAD LOG --\n",7524				 prog->name, log_buf);7525		}7526 7527		if (obj->has_rodata && kernel_supports(obj, FEAT_PROG_BIND_MAP)) {7528			struct bpf_map *map;7529			int i;7530 7531			for (i = 0; i < obj->nr_maps; i++) {7532				map = &prog->obj->maps[i];7533				if (map->libbpf_type != LIBBPF_MAP_RODATA)7534					continue;7535 7536				if (bpf_prog_bind_map(ret, map->fd, NULL)) {7537					cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg));7538					pr_warn("prog '%s': failed to bind map '%s': %s\n",7539						prog->name, map->real_name, cp);7540					/* Don't fail hard if can't bind rodata. */7541				}7542			}7543		}7544 7545		*prog_fd = ret;7546		ret = 0;7547		goto out;7548	}7549 7550	if (log_level == 0) {7551		log_level = 1;7552		goto retry_load;7553	}7554	/* On ENOSPC, increase log buffer size and retry, unless custom7555	 * log_buf is specified.7556	 * Be careful to not overflow u32, though. Kernel's log buf size limit7557	 * isn't part of UAPI so it can always be bumped to full 4GB. So don't7558	 * multiply by 2 unless we are sure we'll fit within 32 bits.7559	 * Currently, we'll get -EINVAL when we reach (UINT_MAX >> 2).7560	 */7561	if (own_log_buf && errno == ENOSPC && log_buf_size <= UINT_MAX / 2)7562		goto retry_load;7563 7564	ret = -errno;7565 7566	/* post-process verifier log to improve error descriptions */7567	fixup_verifier_log(prog, log_buf, log_buf_size);7568 7569	cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg));7570	pr_warn("prog '%s': BPF program load failed: %s\n", prog->name, cp);7571	pr_perm_msg(ret);7572 7573	if (own_log_buf && log_buf && log_buf[0] != '\0') {7574		pr_warn("prog '%s': -- BEGIN PROG LOAD LOG --\n%s-- END PROG LOAD LOG --\n",7575			prog->name, log_buf);7576	}7577 7578out:7579	if (own_log_buf)7580		free(log_buf);7581	return ret;7582}7583 7584static char *find_prev_line(char *buf, char *cur)7585{7586	char *p;7587 7588	if (cur == buf) /* end of a log buf */7589		return NULL;7590 7591	p = cur - 1;7592	while (p - 1 >= buf && *(p - 1) != '\n')7593		p--;7594 7595	return p;7596}7597 7598static void patch_log(char *buf, size_t buf_sz, size_t log_sz,7599		      char *orig, size_t orig_sz, const char *patch)7600{7601	/* size of the remaining log content to the right from the to-be-replaced part */7602	size_t rem_sz = (buf + log_sz) - (orig + orig_sz);7603	size_t patch_sz = strlen(patch);7604 7605	if (patch_sz != orig_sz) {7606		/* If patch line(s) are longer than original piece of verifier log,7607		 * shift log contents by (patch_sz - orig_sz) bytes to the right7608		 * starting from after to-be-replaced part of the log.7609		 *7610		 * If patch line(s) are shorter than original piece of verifier log,7611		 * shift log contents by (orig_sz - patch_sz) bytes to the left7612		 * starting from after to-be-replaced part of the log7613		 *7614		 * We need to be careful about not overflowing available7615		 * buf_sz capacity. If that's the case, we'll truncate the end7616		 * of the original log, as necessary.7617		 */7618		if (patch_sz > orig_sz) {7619			if (orig + patch_sz >= buf + buf_sz) {7620				/* patch is big enough to cover remaining space completely */7621				patch_sz -= (orig + patch_sz) - (buf + buf_sz) + 1;7622				rem_sz = 0;7623			} else if (patch_sz - orig_sz > buf_sz - log_sz) {7624				/* patch causes part of remaining log to be truncated */7625				rem_sz -= (patch_sz - orig_sz) - (buf_sz - log_sz);7626			}7627		}7628		/* shift remaining log to the right by calculated amount */7629		memmove(orig + patch_sz, orig + orig_sz, rem_sz);7630	}7631 7632	memcpy(orig, patch, patch_sz);7633}7634 7635static void fixup_log_failed_core_relo(struct bpf_program *prog,7636				       char *buf, size_t buf_sz, size_t log_sz,7637				       char *line1, char *line2, char *line3)7638{7639	/* Expected log for failed and not properly guarded CO-RE relocation:7640	 * line1 -> 123: (85) call unknown#1958960807641	 * line2 -> invalid func unknown#1958960807642	 * line3 -> <anything else or end of buffer>7643	 *7644	 * "123" is the index of the instruction that was poisoned. We extract7645	 * instruction index to find corresponding CO-RE relocation and7646	 * replace this part of the log with more relevant information about7647	 * failed CO-RE relocation.7648	 */7649	const struct bpf_core_relo *relo;7650	struct bpf_core_spec spec;7651	char patch[512], spec_buf[256];7652	int insn_idx, err, spec_len;7653 7654	if (sscanf(line1, "%d: (%*d) call unknown#195896080\n", &insn_idx) != 1)7655		return;7656 7657	relo = find_relo_core(prog, insn_idx);7658	if (!relo)7659		return;7660 7661	err = bpf_core_parse_spec(prog->name, prog->obj->btf, relo, &spec);7662	if (err)7663		return;7664 7665	spec_len = bpf_core_format_spec(spec_buf, sizeof(spec_buf), &spec);7666	snprintf(patch, sizeof(patch),7667		 "%d: <invalid CO-RE relocation>\n"7668		 "failed to resolve CO-RE relocation %s%s\n",7669		 insn_idx, spec_buf, spec_len >= sizeof(spec_buf) ? "..." : "");7670 7671	patch_log(buf, buf_sz, log_sz, line1, line3 - line1, patch);7672}7673 7674static void fixup_log_missing_map_load(struct bpf_program *prog,7675				       char *buf, size_t buf_sz, size_t log_sz,7676				       char *line1, char *line2, char *line3)7677{7678	/* Expected log for failed and not properly guarded map reference:7679	 * line1 -> 123: (85) call unknown#20010003457680	 * line2 -> invalid func unknown#20010003457681	 * line3 -> <anything else or end of buffer>7682	 *7683	 * "123" is the index of the instruction that was poisoned.7684	 * "345" in "2001000345" is a map index in obj->maps to fetch map name.7685	 */7686	struct bpf_object *obj = prog->obj;7687	const struct bpf_map *map;7688	int insn_idx, map_idx;7689	char patch[128];7690 7691	if (sscanf(line1, "%d: (%*d) call unknown#%d\n", &insn_idx, &map_idx) != 2)7692		return;7693 7694	map_idx -= POISON_LDIMM64_MAP_BASE;7695	if (map_idx < 0 || map_idx >= obj->nr_maps)7696		return;7697	map = &obj->maps[map_idx];7698 7699	snprintf(patch, sizeof(patch),7700		 "%d: <invalid BPF map reference>\n"7701		 "BPF map '%s' is referenced but wasn't created\n",7702		 insn_idx, map->name);7703 7704	patch_log(buf, buf_sz, log_sz, line1, line3 - line1, patch);7705}7706 7707static void fixup_log_missing_kfunc_call(struct bpf_program *prog,7708					 char *buf, size_t buf_sz, size_t log_sz,7709					 char *line1, char *line2, char *line3)7710{7711	/* Expected log for failed and not properly guarded kfunc call:7712	 * line1 -> 123: (85) call unknown#20020003457713	 * line2 -> invalid func unknown#20020003457714	 * line3 -> <anything else or end of buffer>7715	 *7716	 * "123" is the index of the instruction that was poisoned.7717	 * "345" in "2002000345" is an extern index in obj->externs to fetch kfunc name.7718	 */7719	struct bpf_object *obj = prog->obj;7720	const struct extern_desc *ext;7721	int insn_idx, ext_idx;7722	char patch[128];7723 7724	if (sscanf(line1, "%d: (%*d) call unknown#%d\n", &insn_idx, &ext_idx) != 2)7725		return;7726 7727	ext_idx -= POISON_CALL_KFUNC_BASE;7728	if (ext_idx < 0 || ext_idx >= obj->nr_extern)7729		return;7730	ext = &obj->externs[ext_idx];7731 7732	snprintf(patch, sizeof(patch),7733		 "%d: <invalid kfunc call>\n"7734		 "kfunc '%s' is referenced but wasn't resolved\n",7735		 insn_idx, ext->name);7736 7737	patch_log(buf, buf_sz, log_sz, line1, line3 - line1, patch);7738}7739 7740static void fixup_verifier_log(struct bpf_program *prog, char *buf, size_t buf_sz)7741{7742	/* look for familiar error patterns in last N lines of the log */7743	const size_t max_last_line_cnt = 10;7744	char *prev_line, *cur_line, *next_line;7745	size_t log_sz;7746	int i;7747 7748	if (!buf)7749		return;7750 7751	log_sz = strlen(buf) + 1;7752	next_line = buf + log_sz - 1;7753 7754	for (i = 0; i < max_last_line_cnt; i++, next_line = cur_line) {7755		cur_line = find_prev_line(buf, next_line);7756		if (!cur_line)7757			return;7758 7759		if (str_has_pfx(cur_line, "invalid func unknown#195896080\n")) {7760			prev_line = find_prev_line(buf, cur_line);7761			if (!prev_line)7762				continue;7763 7764			/* failed CO-RE relocation case */7765			fixup_log_failed_core_relo(prog, buf, buf_sz, log_sz,7766						   prev_line, cur_line, next_line);7767			return;7768		} else if (str_has_pfx(cur_line, "invalid func unknown#"POISON_LDIMM64_MAP_PFX)) {7769			prev_line = find_prev_line(buf, cur_line);7770			if (!prev_line)7771				continue;7772 7773			/* reference to uncreated BPF map */7774			fixup_log_missing_map_load(prog, buf, buf_sz, log_sz,7775						   prev_line, cur_line, next_line);7776			return;7777		} else if (str_has_pfx(cur_line, "invalid func unknown#"POISON_CALL_KFUNC_PFX)) {7778			prev_line = find_prev_line(buf, cur_line);7779			if (!prev_line)7780				continue;7781 7782			/* reference to unresolved kfunc */7783			fixup_log_missing_kfunc_call(prog, buf, buf_sz, log_sz,7784						     prev_line, cur_line, next_line);7785			return;7786		}7787	}7788}7789 7790static int bpf_program_record_relos(struct bpf_program *prog)7791{7792	struct bpf_object *obj = prog->obj;7793	int i;7794 7795	for (i = 0; i < prog->nr_reloc; i++) {7796		struct reloc_desc *relo = &prog->reloc_desc[i];7797		struct extern_desc *ext = &obj->externs[relo->ext_idx];7798		int kind;7799 7800		switch (relo->type) {7801		case RELO_EXTERN_LD64:7802			if (ext->type != EXT_KSYM)7803				continue;7804			kind = btf_is_var(btf__type_by_id(obj->btf, ext->btf_id)) ?7805				BTF_KIND_VAR : BTF_KIND_FUNC;7806			bpf_gen__record_extern(obj->gen_loader, ext->name,7807					       ext->is_weak, !ext->ksym.type_id,7808					       true, kind, relo->insn_idx);7809			break;7810		case RELO_EXTERN_CALL:7811			bpf_gen__record_extern(obj->gen_loader, ext->name,7812					       ext->is_weak, false, false, BTF_KIND_FUNC,7813					       relo->insn_idx);7814			break;7815		case RELO_CORE: {7816			struct bpf_core_relo cr = {7817				.insn_off = relo->insn_idx * 8,7818				.type_id = relo->core_relo->type_id,7819				.access_str_off = relo->core_relo->access_str_off,7820				.kind = relo->core_relo->kind,7821			};7822 7823			bpf_gen__record_relo_core(obj->gen_loader, &cr);7824			break;7825		}7826		default:7827			continue;7828		}7829	}7830	return 0;7831}7832 7833static int7834bpf_object__load_progs(struct bpf_object *obj, int log_level)7835{7836	struct bpf_program *prog;7837	size_t i;7838	int err;7839 7840	for (i = 0; i < obj->nr_programs; i++) {7841		prog = &obj->programs[i];7842		err = bpf_object__sanitize_prog(obj, prog);7843		if (err)7844			return err;7845	}7846 7847	for (i = 0; i < obj->nr_programs; i++) {7848		prog = &obj->programs[i];7849		if (prog_is_subprog(obj, prog))7850			continue;7851		if (!prog->autoload) {7852			pr_debug("prog '%s': skipped loading\n", prog->name);7853			continue;7854		}7855		prog->log_level |= log_level;7856 7857		if (obj->gen_loader)7858			bpf_program_record_relos(prog);7859 7860		err = bpf_object_load_prog(obj, prog, prog->insns, prog->insns_cnt,7861					   obj->license, obj->kern_version, &prog->fd);7862		if (err) {7863			pr_warn("prog '%s': failed to load: %d\n", prog->name, err);7864			return err;7865		}7866	}7867 7868	bpf_object__free_relocs(obj);7869	return 0;7870}7871 7872static const struct bpf_sec_def *find_sec_def(const char *sec_name);7873 7874static int bpf_object_init_progs(struct bpf_object *obj, const struct bpf_object_open_opts *opts)7875{7876	struct bpf_program *prog;7877	int err;7878 7879	bpf_object__for_each_program(prog, obj) {7880		prog->sec_def = find_sec_def(prog->sec_name);7881		if (!prog->sec_def) {7882			/* couldn't guess, but user might manually specify */7883			pr_debug("prog '%s': unrecognized ELF section name '%s'\n",7884				prog->name, prog->sec_name);7885			continue;7886		}7887 7888		prog->type = prog->sec_def->prog_type;7889		prog->expected_attach_type = prog->sec_def->expected_attach_type;7890 7891		/* sec_def can have custom callback which should be called7892		 * after bpf_program is initialized to adjust its properties7893		 */7894		if (prog->sec_def->prog_setup_fn) {7895			err = prog->sec_def->prog_setup_fn(prog, prog->sec_def->cookie);7896			if (err < 0) {7897				pr_warn("prog '%s': failed to initialize: %d\n",7898					prog->name, err);7899				return err;7900			}7901		}7902	}7903 7904	return 0;7905}7906 7907static struct bpf_object *bpf_object_open(const char *path, const void *obj_buf, size_t obj_buf_sz,7908					  const char *obj_name,7909					  const struct bpf_object_open_opts *opts)7910{7911	const char *kconfig, *btf_tmp_path, *token_path;7912	struct bpf_object *obj;7913	int err;7914	char *log_buf;7915	size_t log_size;7916	__u32 log_level;7917 7918	if (obj_buf && !obj_name)7919		return ERR_PTR(-EINVAL);7920 7921	if (elf_version(EV_CURRENT) == EV_NONE) {7922		pr_warn("failed to init libelf for %s\n",7923			path ? : "(mem buf)");7924		return ERR_PTR(-LIBBPF_ERRNO__LIBELF);7925	}7926 7927	if (!OPTS_VALID(opts, bpf_object_open_opts))7928		return ERR_PTR(-EINVAL);7929 7930	obj_name = OPTS_GET(opts, object_name, NULL) ?: obj_name;7931	if (obj_buf) {7932		path = obj_name;7933		pr_debug("loading object '%s' from buffer\n", obj_name);7934	} else {7935		pr_debug("loading object from %s\n", path);7936	}7937 7938	log_buf = OPTS_GET(opts, kernel_log_buf, NULL);7939	log_size = OPTS_GET(opts, kernel_log_size, 0);7940	log_level = OPTS_GET(opts, kernel_log_level, 0);7941	if (log_size > UINT_MAX)7942		return ERR_PTR(-EINVAL);7943	if (log_size && !log_buf)7944		return ERR_PTR(-EINVAL);7945 7946	token_path = OPTS_GET(opts, bpf_token_path, NULL);7947	/* if user didn't specify bpf_token_path explicitly, check if7948	 * LIBBPF_BPF_TOKEN_PATH envvar was set and treat it as bpf_token_path7949	 * option7950	 */7951	if (!token_path)7952		token_path = getenv("LIBBPF_BPF_TOKEN_PATH");7953	if (token_path && strlen(token_path) >= PATH_MAX)7954		return ERR_PTR(-ENAMETOOLONG);7955 7956	obj = bpf_object__new(path, obj_buf, obj_buf_sz, obj_name);7957	if (IS_ERR(obj))7958		return obj;7959 7960	obj->log_buf = log_buf;7961	obj->log_size = log_size;7962	obj->log_level = log_level;7963 7964	if (token_path) {7965		obj->token_path = strdup(token_path);7966		if (!obj->token_path) {7967			err = -ENOMEM;7968			goto out;7969		}7970	}7971 7972	btf_tmp_path = OPTS_GET(opts, btf_custom_path, NULL);7973	if (btf_tmp_path) {7974		if (strlen(btf_tmp_path) >= PATH_MAX) {7975			err = -ENAMETOOLONG;7976			goto out;7977		}7978		obj->btf_custom_path = strdup(btf_tmp_path);7979		if (!obj->btf_custom_path) {7980			err = -ENOMEM;7981			goto out;7982		}7983	}7984 7985	kconfig = OPTS_GET(opts, kconfig, NULL);7986	if (kconfig) {7987		obj->kconfig = strdup(kconfig);7988		if (!obj->kconfig) {7989			err = -ENOMEM;7990			goto out;7991		}7992	}7993 7994	err = bpf_object__elf_init(obj);7995	err = err ? : bpf_object__check_endianness(obj);7996	err = err ? : bpf_object__elf_collect(obj);7997	err = err ? : bpf_object__collect_externs(obj);7998	err = err ? : bpf_object_fixup_btf(obj);7999	err = err ? : bpf_object__init_maps(obj, opts);8000	err = err ? : bpf_object_init_progs(obj, opts);8001	err = err ? : bpf_object__collect_relos(obj);8002	if (err)8003		goto out;8004 8005	bpf_object__elf_finish(obj);8006 8007	return obj;8008out:8009	bpf_object__close(obj);8010	return ERR_PTR(err);8011}8012 8013struct bpf_object *8014bpf_object__open_file(const char *path, const struct bpf_object_open_opts *opts)8015{8016	if (!path)8017		return libbpf_err_ptr(-EINVAL);8018 8019	return libbpf_ptr(bpf_object_open(path, NULL, 0, NULL, opts));8020}8021 8022struct bpf_object *bpf_object__open(const char *path)8023{8024	return bpf_object__open_file(path, NULL);8025}8026 8027struct bpf_object *8028bpf_object__open_mem(const void *obj_buf, size_t obj_buf_sz,8029		     const struct bpf_object_open_opts *opts)8030{8031	char tmp_name[64];8032 8033	if (!obj_buf || obj_buf_sz == 0)8034		return libbpf_err_ptr(-EINVAL);8035 8036	/* create a (quite useless) default "name" for this memory buffer object */8037	snprintf(tmp_name, sizeof(tmp_name), "%lx-%zx", (unsigned long)obj_buf, obj_buf_sz);8038 8039	return libbpf_ptr(bpf_object_open(NULL, obj_buf, obj_buf_sz, tmp_name, opts));8040}8041 8042static int bpf_object_unload(struct bpf_object *obj)8043{8044	size_t i;8045 8046	if (!obj)8047		return libbpf_err(-EINVAL);8048 8049	for (i = 0; i < obj->nr_maps; i++) {8050		zclose(obj->maps[i].fd);8051		if (obj->maps[i].st_ops)8052			zfree(&obj->maps[i].st_ops->kern_vdata);8053	}8054 8055	for (i = 0; i < obj->nr_programs; i++)8056		bpf_program__unload(&obj->programs[i]);8057 8058	return 0;8059}8060 8061static int bpf_object__sanitize_maps(struct bpf_object *obj)8062{8063	struct bpf_map *m;8064 8065	bpf_object__for_each_map(m, obj) {8066		if (!bpf_map__is_internal(m))8067			continue;8068		if (!kernel_supports(obj, FEAT_ARRAY_MMAP))8069			m->def.map_flags &= ~BPF_F_MMAPABLE;8070	}8071 8072	return 0;8073}8074 8075typedef int (*kallsyms_cb_t)(unsigned long long sym_addr, char sym_type,8076			     const char *sym_name, void *ctx);8077 8078static int libbpf_kallsyms_parse(kallsyms_cb_t cb, void *ctx)8079{8080	char sym_type, sym_name[500];8081	unsigned long long sym_addr;8082	int ret, err = 0;8083	FILE *f;8084 8085	f = fopen("/proc/kallsyms", "re");8086	if (!f) {8087		err = -errno;8088		pr_warn("failed to open /proc/kallsyms: %d\n", err);8089		return err;8090	}8091 8092	while (true) {8093		ret = fscanf(f, "%llx %c %499s%*[^\n]\n",8094			     &sym_addr, &sym_type, sym_name);8095		if (ret == EOF && feof(f))8096			break;8097		if (ret != 3) {8098			pr_warn("failed to read kallsyms entry: %d\n", ret);8099			err = -EINVAL;8100			break;8101		}8102 8103		err = cb(sym_addr, sym_type, sym_name, ctx);8104		if (err)8105			break;8106	}8107 8108	fclose(f);8109	return err;8110}8111 8112static int kallsyms_cb(unsigned long long sym_addr, char sym_type,8113		       const char *sym_name, void *ctx)8114{8115	struct bpf_object *obj = ctx;8116	const struct btf_type *t;8117	struct extern_desc *ext;8118	char *res;8119 8120	res = strstr(sym_name, ".llvm.");8121	if (sym_type == 'd' && res)8122		ext = find_extern_by_name_with_len(obj, sym_name, res - sym_name);8123	else8124		ext = find_extern_by_name(obj, sym_name);8125	if (!ext || ext->type != EXT_KSYM)8126		return 0;8127 8128	t = btf__type_by_id(obj->btf, ext->btf_id);8129	if (!btf_is_var(t))8130		return 0;8131 8132	if (ext->is_set && ext->ksym.addr != sym_addr) {8133		pr_warn("extern (ksym) '%s': resolution is ambiguous: 0x%llx or 0x%llx\n",8134			sym_name, ext->ksym.addr, sym_addr);8135		return -EINVAL;8136	}8137	if (!ext->is_set) {8138		ext->is_set = true;8139		ext->ksym.addr = sym_addr;8140		pr_debug("extern (ksym) '%s': set to 0x%llx\n", sym_name, sym_addr);8141	}8142	return 0;8143}8144 8145static int bpf_object__read_kallsyms_file(struct bpf_object *obj)8146{8147	return libbpf_kallsyms_parse(kallsyms_cb, obj);8148}8149 8150static int find_ksym_btf_id(struct bpf_object *obj, const char *ksym_name,8151			    __u16 kind, struct btf **res_btf,8152			    struct module_btf **res_mod_btf)8153{8154	struct module_btf *mod_btf;8155	struct btf *btf;8156	int i, id, err;8157 8158	btf = obj->btf_vmlinux;8159	mod_btf = NULL;8160	id = btf__find_by_name_kind(btf, ksym_name, kind);8161 8162	if (id == -ENOENT) {8163		err = load_module_btfs(obj);8164		if (err)8165			return err;8166 8167		for (i = 0; i < obj->btf_module_cnt; i++) {8168			/* we assume module_btf's BTF FD is always >0 */8169			mod_btf = &obj->btf_modules[i];8170			btf = mod_btf->btf;8171			id = btf__find_by_name_kind_own(btf, ksym_name, kind);8172			if (id != -ENOENT)8173				break;8174		}8175	}8176	if (id <= 0)8177		return -ESRCH;8178 8179	*res_btf = btf;8180	*res_mod_btf = mod_btf;8181	return id;8182}8183 8184static int bpf_object__resolve_ksym_var_btf_id(struct bpf_object *obj,8185					       struct extern_desc *ext)8186{8187	const struct btf_type *targ_var, *targ_type;8188	__u32 targ_type_id, local_type_id;8189	struct module_btf *mod_btf = NULL;8190	const char *targ_var_name;8191	struct btf *btf = NULL;8192	int id, err;8193 8194	id = find_ksym_btf_id(obj, ext->name, BTF_KIND_VAR, &btf, &mod_btf);8195	if (id < 0) {8196		if (id == -ESRCH && ext->is_weak)8197			return 0;8198		pr_warn("extern (var ksym) '%s': not found in kernel BTF\n",8199			ext->name);8200		return id;8201	}8202 8203	/* find local type_id */8204	local_type_id = ext->ksym.type_id;8205 8206	/* find target type_id */8207	targ_var = btf__type_by_id(btf, id);8208	targ_var_name = btf__name_by_offset(btf, targ_var->name_off);8209	targ_type = skip_mods_and_typedefs(btf, targ_var->type, &targ_type_id);8210 8211	err = bpf_core_types_are_compat(obj->btf, local_type_id,8212					btf, targ_type_id);8213	if (err <= 0) {8214		const struct btf_type *local_type;8215		const char *targ_name, *local_name;8216 8217		local_type = btf__type_by_id(obj->btf, local_type_id);8218		local_name = btf__name_by_offset(obj->btf, local_type->name_off);8219		targ_name = btf__name_by_offset(btf, targ_type->name_off);8220 8221		pr_warn("extern (var ksym) '%s': incompatible types, expected [%d] %s %s, but kernel has [%d] %s %s\n",8222			ext->name, local_type_id,8223			btf_kind_str(local_type), local_name, targ_type_id,8224			btf_kind_str(targ_type), targ_name);8225		return -EINVAL;8226	}8227 8228	ext->is_set = true;8229	ext->ksym.kernel_btf_obj_fd = mod_btf ? mod_btf->fd : 0;8230	ext->ksym.kernel_btf_id = id;8231	pr_debug("extern (var ksym) '%s': resolved to [%d] %s %s\n",8232		 ext->name, id, btf_kind_str(targ_var), targ_var_name);8233 8234	return 0;8235}8236 8237static int bpf_object__resolve_ksym_func_btf_id(struct bpf_object *obj,8238						struct extern_desc *ext)8239{8240	int local_func_proto_id, kfunc_proto_id, kfunc_id;8241	struct module_btf *mod_btf = NULL;8242	const struct btf_type *kern_func;8243	struct btf *kern_btf = NULL;8244	int ret;8245 8246	local_func_proto_id = ext->ksym.type_id;8247 8248	kfunc_id = find_ksym_btf_id(obj, ext->essent_name ?: ext->name, BTF_KIND_FUNC, &kern_btf,8249				    &mod_btf);8250	if (kfunc_id < 0) {8251		if (kfunc_id == -ESRCH && ext->is_weak)8252			return 0;8253		pr_warn("extern (func ksym) '%s': not found in kernel or module BTFs\n",8254			ext->name);8255		return kfunc_id;8256	}8257 8258	kern_func = btf__type_by_id(kern_btf, kfunc_id);8259	kfunc_proto_id = kern_func->type;8260 8261	ret = bpf_core_types_are_compat(obj->btf, local_func_proto_id,8262					kern_btf, kfunc_proto_id);8263	if (ret <= 0) {8264		if (ext->is_weak)8265			return 0;8266 8267		pr_warn("extern (func ksym) '%s': func_proto [%d] incompatible with %s [%d]\n",8268			ext->name, local_func_proto_id,8269			mod_btf ? mod_btf->name : "vmlinux", kfunc_proto_id);8270		return -EINVAL;8271	}8272 8273	/* set index for module BTF fd in fd_array, if unset */8274	if (mod_btf && !mod_btf->fd_array_idx) {8275		/* insn->off is s16 */8276		if (obj->fd_array_cnt == INT16_MAX) {8277			pr_warn("extern (func ksym) '%s': module BTF fd index %d too big to fit in bpf_insn offset\n",8278				ext->name, mod_btf->fd_array_idx);8279			return -E2BIG;8280		}8281		/* Cannot use index 0 for module BTF fd */8282		if (!obj->fd_array_cnt)8283			obj->fd_array_cnt = 1;8284 8285		ret = libbpf_ensure_mem((void **)&obj->fd_array, &obj->fd_array_cap, sizeof(int),8286					obj->fd_array_cnt + 1);8287		if (ret)8288			return ret;8289		mod_btf->fd_array_idx = obj->fd_array_cnt;8290		/* we assume module BTF FD is always >0 */8291		obj->fd_array[obj->fd_array_cnt++] = mod_btf->fd;8292	}8293 8294	ext->is_set = true;8295	ext->ksym.kernel_btf_id = kfunc_id;8296	ext->ksym.btf_fd_idx = mod_btf ? mod_btf->fd_array_idx : 0;8297	/* Also set kernel_btf_obj_fd to make sure that bpf_object__relocate_data()8298	 * populates FD into ld_imm64 insn when it's used to point to kfunc.8299	 * {kernel_btf_id, btf_fd_idx} -> fixup bpf_call.8300	 * {kernel_btf_id, kernel_btf_obj_fd} -> fixup ld_imm64.8301	 */8302	ext->ksym.kernel_btf_obj_fd = mod_btf ? mod_btf->fd : 0;8303	pr_debug("extern (func ksym) '%s': resolved to %s [%d]\n",8304		 ext->name, mod_btf ? mod_btf->name : "vmlinux", kfunc_id);8305 8306	return 0;8307}8308 8309static int bpf_object__resolve_ksyms_btf_id(struct bpf_object *obj)8310{8311	const struct btf_type *t;8312	struct extern_desc *ext;8313	int i, err;8314 8315	for (i = 0; i < obj->nr_extern; i++) {8316		ext = &obj->externs[i];8317		if (ext->type != EXT_KSYM || !ext->ksym.type_id)8318			continue;8319 8320		if (obj->gen_loader) {8321			ext->is_set = true;8322			ext->ksym.kernel_btf_obj_fd = 0;8323			ext->ksym.kernel_btf_id = 0;8324			continue;8325		}8326		t = btf__type_by_id(obj->btf, ext->btf_id);8327		if (btf_is_var(t))8328			err = bpf_object__resolve_ksym_var_btf_id(obj, ext);8329		else8330			err = bpf_object__resolve_ksym_func_btf_id(obj, ext);8331		if (err)8332			return err;8333	}8334	return 0;8335}8336 8337static int bpf_object__resolve_externs(struct bpf_object *obj,8338				       const char *extra_kconfig)8339{8340	bool need_config = false, need_kallsyms = false;8341	bool need_vmlinux_btf = false;8342	struct extern_desc *ext;8343	void *kcfg_data = NULL;8344	int err, i;8345 8346	if (obj->nr_extern == 0)8347		return 0;8348 8349	if (obj->kconfig_map_idx >= 0)8350		kcfg_data = obj->maps[obj->kconfig_map_idx].mmaped;8351 8352	for (i = 0; i < obj->nr_extern; i++) {8353		ext = &obj->externs[i];8354 8355		if (ext->type == EXT_KSYM) {8356			if (ext->ksym.type_id)8357				need_vmlinux_btf = true;8358			else8359				need_kallsyms = true;8360			continue;8361		} else if (ext->type == EXT_KCFG) {8362			void *ext_ptr = kcfg_data + ext->kcfg.data_off;8363			__u64 value = 0;8364 8365			/* Kconfig externs need actual /proc/config.gz */8366			if (str_has_pfx(ext->name, "CONFIG_")) {8367				need_config = true;8368				continue;8369			}8370 8371			/* Virtual kcfg externs are customly handled by libbpf */8372			if (strcmp(ext->name, "LINUX_KERNEL_VERSION") == 0) {8373				value = get_kernel_version();8374				if (!value) {8375					pr_warn("extern (kcfg) '%s': failed to get kernel version\n", ext->name);8376					return -EINVAL;8377				}8378			} else if (strcmp(ext->name, "LINUX_HAS_BPF_COOKIE") == 0) {8379				value = kernel_supports(obj, FEAT_BPF_COOKIE);8380			} else if (strcmp(ext->name, "LINUX_HAS_SYSCALL_WRAPPER") == 0) {8381				value = kernel_supports(obj, FEAT_SYSCALL_WRAPPER);8382			} else if (!str_has_pfx(ext->name, "LINUX_") || !ext->is_weak) {8383				/* Currently libbpf supports only CONFIG_ and LINUX_ prefixed8384				 * __kconfig externs, where LINUX_ ones are virtual and filled out8385				 * customly by libbpf (their values don't come from Kconfig).8386				 * If LINUX_xxx variable is not recognized by libbpf, but is marked8387				 * __weak, it defaults to zero value, just like for CONFIG_xxx8388				 * externs.8389				 */8390				pr_warn("extern (kcfg) '%s': unrecognized virtual extern\n", ext->name);8391				return -EINVAL;8392			}8393 8394			err = set_kcfg_value_num(ext, ext_ptr, value);8395			if (err)8396				return err;8397			pr_debug("extern (kcfg) '%s': set to 0x%llx\n",8398				 ext->name, (long long)value);8399		} else {8400			pr_warn("extern '%s': unrecognized extern kind\n", ext->name);8401			return -EINVAL;8402		}8403	}8404	if (need_config && extra_kconfig) {8405		err = bpf_object__read_kconfig_mem(obj, extra_kconfig, kcfg_data);8406		if (err)8407			return -EINVAL;8408		need_config = false;8409		for (i = 0; i < obj->nr_extern; i++) {8410			ext = &obj->externs[i];8411			if (ext->type == EXT_KCFG && !ext->is_set) {8412				need_config = true;8413				break;8414			}8415		}8416	}8417	if (need_config) {8418		err = bpf_object__read_kconfig_file(obj, kcfg_data);8419		if (err)8420			return -EINVAL;8421	}8422	if (need_kallsyms) {8423		err = bpf_object__read_kallsyms_file(obj);8424		if (err)8425			return -EINVAL;8426	}8427	if (need_vmlinux_btf) {8428		err = bpf_object__resolve_ksyms_btf_id(obj);8429		if (err)8430			return -EINVAL;8431	}8432	for (i = 0; i < obj->nr_extern; i++) {8433		ext = &obj->externs[i];8434 8435		if (!ext->is_set && !ext->is_weak) {8436			pr_warn("extern '%s' (strong): not resolved\n", ext->name);8437			return -ESRCH;8438		} else if (!ext->is_set) {8439			pr_debug("extern '%s' (weak): not resolved, defaulting to zero\n",8440				 ext->name);8441		}8442	}8443 8444	return 0;8445}8446 8447static void bpf_map_prepare_vdata(const struct bpf_map *map)8448{8449	const struct btf_type *type;8450	struct bpf_struct_ops *st_ops;8451	__u32 i;8452 8453	st_ops = map->st_ops;8454	type = btf__type_by_id(map->obj->btf, st_ops->type_id);8455	for (i = 0; i < btf_vlen(type); i++) {8456		struct bpf_program *prog = st_ops->progs[i];8457		void *kern_data;8458		int prog_fd;8459 8460		if (!prog)8461			continue;8462 8463		prog_fd = bpf_program__fd(prog);8464		kern_data = st_ops->kern_vdata + st_ops->kern_func_off[i];8465		*(unsigned long *)kern_data = prog_fd;8466	}8467}8468 8469static int bpf_object_prepare_struct_ops(struct bpf_object *obj)8470{8471	struct bpf_map *map;8472	int i;8473 8474	for (i = 0; i < obj->nr_maps; i++) {8475		map = &obj->maps[i];8476 8477		if (!bpf_map__is_struct_ops(map))8478			continue;8479 8480		if (!map->autocreate)8481			continue;8482 8483		bpf_map_prepare_vdata(map);8484	}8485 8486	return 0;8487}8488 8489static int bpf_object_load(struct bpf_object *obj, int extra_log_level, const char *target_btf_path)8490{8491	int err, i;8492 8493	if (!obj)8494		return libbpf_err(-EINVAL);8495 8496	if (obj->loaded) {8497		pr_warn("object '%s': load can't be attempted twice\n", obj->name);8498		return libbpf_err(-EINVAL);8499	}8500 8501	if (obj->gen_loader)8502		bpf_gen__init(obj->gen_loader, extra_log_level, obj->nr_programs, obj->nr_maps);8503 8504	err = bpf_object_prepare_token(obj);8505	err = err ? : bpf_object__probe_loading(obj);8506	err = err ? : bpf_object__load_vmlinux_btf(obj, false);8507	err = err ? : bpf_object__resolve_externs(obj, obj->kconfig);8508	err = err ? : bpf_object__sanitize_maps(obj);8509	err = err ? : bpf_object__init_kern_struct_ops_maps(obj);8510	err = err ? : bpf_object_adjust_struct_ops_autoload(obj);8511	err = err ? : bpf_object__relocate(obj, obj->btf_custom_path ? : target_btf_path);8512	err = err ? : bpf_object__sanitize_and_load_btf(obj);8513	err = err ? : bpf_object__create_maps(obj);8514	err = err ? : bpf_object__load_progs(obj, extra_log_level);8515	err = err ? : bpf_object_init_prog_arrays(obj);8516	err = err ? : bpf_object_prepare_struct_ops(obj);8517 8518	if (obj->gen_loader) {8519		/* reset FDs */8520		if (obj->btf)8521			btf__set_fd(obj->btf, -1);8522		if (!err)8523			err = bpf_gen__finish(obj->gen_loader, obj->nr_programs, obj->nr_maps);8524	}8525 8526	/* clean up fd_array */8527	zfree(&obj->fd_array);8528 8529	/* clean up module BTFs */8530	for (i = 0; i < obj->btf_module_cnt; i++) {8531		close(obj->btf_modules[i].fd);8532		btf__free(obj->btf_modules[i].btf);8533		free(obj->btf_modules[i].name);8534	}8535	free(obj->btf_modules);8536 8537	/* clean up vmlinux BTF */8538	btf__free(obj->btf_vmlinux);8539	obj->btf_vmlinux = NULL;8540 8541	obj->loaded = true; /* doesn't matter if successfully or not */8542 8543	if (err)8544		goto out;8545 8546	return 0;8547out:8548	/* unpin any maps that were auto-pinned during load */8549	for (i = 0; i < obj->nr_maps; i++)8550		if (obj->maps[i].pinned && !obj->maps[i].reused)8551			bpf_map__unpin(&obj->maps[i], NULL);8552 8553	bpf_object_unload(obj);8554	pr_warn("failed to load object '%s'\n", obj->path);8555	return libbpf_err(err);8556}8557 8558int bpf_object__load(struct bpf_object *obj)8559{8560	return bpf_object_load(obj, 0, NULL);8561}8562 8563static int make_parent_dir(const char *path)8564{8565	char *cp, errmsg[STRERR_BUFSIZE];8566	char *dname, *dir;8567	int err = 0;8568 8569	dname = strdup(path);8570	if (dname == NULL)8571		return -ENOMEM;8572 8573	dir = dirname(dname);8574	if (mkdir(dir, 0700) && errno != EEXIST)8575		err = -errno;8576 8577	free(dname);8578	if (err) {8579		cp = libbpf_strerror_r(-err, errmsg, sizeof(errmsg));8580		pr_warn("failed to mkdir %s: %s\n", path, cp);8581	}8582	return err;8583}8584 8585static int check_path(const char *path)8586{8587	char *cp, errmsg[STRERR_BUFSIZE];8588	struct statfs st_fs;8589	char *dname, *dir;8590	int err = 0;8591 8592	if (path == NULL)8593		return -EINVAL;8594 8595	dname = strdup(path);8596	if (dname == NULL)8597		return -ENOMEM;8598 8599	dir = dirname(dname);8600	if (statfs(dir, &st_fs)) {8601		cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg));8602		pr_warn("failed to statfs %s: %s\n", dir, cp);8603		err = -errno;8604	}8605	free(dname);8606 8607	if (!err && st_fs.f_type != BPF_FS_MAGIC) {8608		pr_warn("specified path %s is not on BPF FS\n", path);8609		err = -EINVAL;8610	}8611 8612	return err;8613}8614 8615int bpf_program__pin(struct bpf_program *prog, const char *path)8616{8617	char *cp, errmsg[STRERR_BUFSIZE];8618	int err;8619 8620	if (prog->fd < 0) {8621		pr_warn("prog '%s': can't pin program that wasn't loaded\n", prog->name);8622		return libbpf_err(-EINVAL);8623	}8624 8625	err = make_parent_dir(path);8626	if (err)8627		return libbpf_err(err);8628 8629	err = check_path(path);8630	if (err)8631		return libbpf_err(err);8632 8633	if (bpf_obj_pin(prog->fd, path)) {8634		err = -errno;8635		cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg));8636		pr_warn("prog '%s': failed to pin at '%s': %s\n", prog->name, path, cp);8637		return libbpf_err(err);8638	}8639 8640	pr_debug("prog '%s': pinned at '%s'\n", prog->name, path);8641	return 0;8642}8643 8644int bpf_program__unpin(struct bpf_program *prog, const char *path)8645{8646	int err;8647 8648	if (prog->fd < 0) {8649		pr_warn("prog '%s': can't unpin program that wasn't loaded\n", prog->name);8650		return libbpf_err(-EINVAL);8651	}8652 8653	err = check_path(path);8654	if (err)8655		return libbpf_err(err);8656 8657	err = unlink(path);8658	if (err)8659		return libbpf_err(-errno);8660 8661	pr_debug("prog '%s': unpinned from '%s'\n", prog->name, path);8662	return 0;8663}8664 8665int bpf_map__pin(struct bpf_map *map, const char *path)8666{8667	char *cp, errmsg[STRERR_BUFSIZE];8668	int err;8669 8670	if (map == NULL) {8671		pr_warn("invalid map pointer\n");8672		return libbpf_err(-EINVAL);8673	}8674 8675	if (map->fd < 0) {8676		pr_warn("map '%s': can't pin BPF map without FD (was it created?)\n", map->name);8677		return libbpf_err(-EINVAL);8678	}8679 8680	if (map->pin_path) {8681		if (path && strcmp(path, map->pin_path)) {8682			pr_warn("map '%s' already has pin path '%s' different from '%s'\n",8683				bpf_map__name(map), map->pin_path, path);8684			return libbpf_err(-EINVAL);8685		} else if (map->pinned) {8686			pr_debug("map '%s' already pinned at '%s'; not re-pinning\n",8687				 bpf_map__name(map), map->pin_path);8688			return 0;8689		}8690	} else {8691		if (!path) {8692			pr_warn("missing a path to pin map '%s' at\n",8693				bpf_map__name(map));8694			return libbpf_err(-EINVAL);8695		} else if (map->pinned) {8696			pr_warn("map '%s' already pinned\n", bpf_map__name(map));8697			return libbpf_err(-EEXIST);8698		}8699 8700		map->pin_path = strdup(path);8701		if (!map->pin_path) {8702			err = -errno;8703			goto out_err;8704		}8705	}8706 8707	err = make_parent_dir(map->pin_path);8708	if (err)8709		return libbpf_err(err);8710 8711	err = check_path(map->pin_path);8712	if (err)8713		return libbpf_err(err);8714 8715	if (bpf_obj_pin(map->fd, map->pin_path)) {8716		err = -errno;8717		goto out_err;8718	}8719 8720	map->pinned = true;8721	pr_debug("pinned map '%s'\n", map->pin_path);8722 8723	return 0;8724 8725out_err:8726	cp = libbpf_strerror_r(-err, errmsg, sizeof(errmsg));8727	pr_warn("failed to pin map: %s\n", cp);8728	return libbpf_err(err);8729}8730 8731int bpf_map__unpin(struct bpf_map *map, const char *path)8732{8733	int err;8734 8735	if (map == NULL) {8736		pr_warn("invalid map pointer\n");8737		return libbpf_err(-EINVAL);8738	}8739 8740	if (map->pin_path) {8741		if (path && strcmp(path, map->pin_path)) {8742			pr_warn("map '%s' already has pin path '%s' different from '%s'\n",8743				bpf_map__name(map), map->pin_path, path);8744			return libbpf_err(-EINVAL);8745		}8746		path = map->pin_path;8747	} else if (!path) {8748		pr_warn("no path to unpin map '%s' from\n",8749			bpf_map__name(map));8750		return libbpf_err(-EINVAL);8751	}8752 8753	err = check_path(path);8754	if (err)8755		return libbpf_err(err);8756 8757	err = unlink(path);8758	if (err != 0)8759		return libbpf_err(-errno);8760 8761	map->pinned = false;8762	pr_debug("unpinned map '%s' from '%s'\n", bpf_map__name(map), path);8763 8764	return 0;8765}8766 8767int bpf_map__set_pin_path(struct bpf_map *map, const char *path)8768{8769	char *new = NULL;8770 8771	if (path) {8772		new = strdup(path);8773		if (!new)8774			return libbpf_err(-errno);8775	}8776 8777	free(map->pin_path);8778	map->pin_path = new;8779	return 0;8780}8781 8782__alias(bpf_map__pin_path)8783const char *bpf_map__get_pin_path(const struct bpf_map *map);8784 8785const char *bpf_map__pin_path(const struct bpf_map *map)8786{8787	return map->pin_path;8788}8789 8790bool bpf_map__is_pinned(const struct bpf_map *map)8791{8792	return map->pinned;8793}8794 8795static void sanitize_pin_path(char *s)8796{8797	/* bpffs disallows periods in path names */8798	while (*s) {8799		if (*s == '.')8800			*s = '_';8801		s++;8802	}8803}8804 8805int bpf_object__pin_maps(struct bpf_object *obj, const char *path)8806{8807	struct bpf_map *map;8808	int err;8809 8810	if (!obj)8811		return libbpf_err(-ENOENT);8812 8813	if (!obj->loaded) {8814		pr_warn("object not yet loaded; load it first\n");8815		return libbpf_err(-ENOENT);8816	}8817 8818	bpf_object__for_each_map(map, obj) {8819		char *pin_path = NULL;8820		char buf[PATH_MAX];8821 8822		if (!map->autocreate)8823			continue;8824 8825		if (path) {8826			err = pathname_concat(buf, sizeof(buf), path, bpf_map__name(map));8827			if (err)8828				goto err_unpin_maps;8829			sanitize_pin_path(buf);8830			pin_path = buf;8831		} else if (!map->pin_path) {8832			continue;8833		}8834 8835		err = bpf_map__pin(map, pin_path);8836		if (err)8837			goto err_unpin_maps;8838	}8839 8840	return 0;8841 8842err_unpin_maps:8843	while ((map = bpf_object__prev_map(obj, map))) {8844		if (!map->pin_path)8845			continue;8846 8847		bpf_map__unpin(map, NULL);8848	}8849 8850	return libbpf_err(err);8851}8852 8853int bpf_object__unpin_maps(struct bpf_object *obj, const char *path)8854{8855	struct bpf_map *map;8856	int err;8857 8858	if (!obj)8859		return libbpf_err(-ENOENT);8860 8861	bpf_object__for_each_map(map, obj) {8862		char *pin_path = NULL;8863		char buf[PATH_MAX];8864 8865		if (path) {8866			err = pathname_concat(buf, sizeof(buf), path, bpf_map__name(map));8867			if (err)8868				return libbpf_err(err);8869			sanitize_pin_path(buf);8870			pin_path = buf;8871		} else if (!map->pin_path) {8872			continue;8873		}8874 8875		err = bpf_map__unpin(map, pin_path);8876		if (err)8877			return libbpf_err(err);8878	}8879 8880	return 0;8881}8882 8883int bpf_object__pin_programs(struct bpf_object *obj, const char *path)8884{8885	struct bpf_program *prog;8886	char buf[PATH_MAX];8887	int err;8888 8889	if (!obj)8890		return libbpf_err(-ENOENT);8891 8892	if (!obj->loaded) {8893		pr_warn("object not yet loaded; load it first\n");8894		return libbpf_err(-ENOENT);8895	}8896 8897	bpf_object__for_each_program(prog, obj) {8898		err = pathname_concat(buf, sizeof(buf), path, prog->name);8899		if (err)8900			goto err_unpin_programs;8901 8902		err = bpf_program__pin(prog, buf);8903		if (err)8904			goto err_unpin_programs;8905	}8906 8907	return 0;8908 8909err_unpin_programs:8910	while ((prog = bpf_object__prev_program(obj, prog))) {8911		if (pathname_concat(buf, sizeof(buf), path, prog->name))8912			continue;8913 8914		bpf_program__unpin(prog, buf);8915	}8916 8917	return libbpf_err(err);8918}8919 8920int bpf_object__unpin_programs(struct bpf_object *obj, const char *path)8921{8922	struct bpf_program *prog;8923	int err;8924 8925	if (!obj)8926		return libbpf_err(-ENOENT);8927 8928	bpf_object__for_each_program(prog, obj) {8929		char buf[PATH_MAX];8930 8931		err = pathname_concat(buf, sizeof(buf), path, prog->name);8932		if (err)8933			return libbpf_err(err);8934 8935		err = bpf_program__unpin(prog, buf);8936		if (err)8937			return libbpf_err(err);8938	}8939 8940	return 0;8941}8942 8943int bpf_object__pin(struct bpf_object *obj, const char *path)8944{8945	int err;8946 8947	err = bpf_object__pin_maps(obj, path);8948	if (err)8949		return libbpf_err(err);8950 8951	err = bpf_object__pin_programs(obj, path);8952	if (err) {8953		bpf_object__unpin_maps(obj, path);8954		return libbpf_err(err);8955	}8956 8957	return 0;8958}8959 8960int bpf_object__unpin(struct bpf_object *obj, const char *path)8961{8962	int err;8963 8964	err = bpf_object__unpin_programs(obj, path);8965	if (err)8966		return libbpf_err(err);8967 8968	err = bpf_object__unpin_maps(obj, path);8969	if (err)8970		return libbpf_err(err);8971 8972	return 0;8973}8974 8975static void bpf_map__destroy(struct bpf_map *map)8976{8977	if (map->inner_map) {8978		bpf_map__destroy(map->inner_map);8979		zfree(&map->inner_map);8980	}8981 8982	zfree(&map->init_slots);8983	map->init_slots_sz = 0;8984 8985	if (map->mmaped && map->mmaped != map->obj->arena_data)8986		munmap(map->mmaped, bpf_map_mmap_sz(map));8987	map->mmaped = NULL;8988 8989	if (map->st_ops) {8990		zfree(&map->st_ops->data);8991		zfree(&map->st_ops->progs);8992		zfree(&map->st_ops->kern_func_off);8993		zfree(&map->st_ops);8994	}8995 8996	zfree(&map->name);8997	zfree(&map->real_name);8998	zfree(&map->pin_path);8999 9000	if (map->fd >= 0)9001		zclose(map->fd);9002}9003 9004void bpf_object__close(struct bpf_object *obj)9005{9006	size_t i;9007 9008	if (IS_ERR_OR_NULL(obj))9009		return;9010 9011	usdt_manager_free(obj->usdt_man);9012	obj->usdt_man = NULL;9013 9014	bpf_gen__free(obj->gen_loader);9015	bpf_object__elf_finish(obj);9016	bpf_object_unload(obj);9017	btf__free(obj->btf);9018	btf__free(obj->btf_vmlinux);9019	btf_ext__free(obj->btf_ext);9020 9021	for (i = 0; i < obj->nr_maps; i++)9022		bpf_map__destroy(&obj->maps[i]);9023 9024	zfree(&obj->btf_custom_path);9025	zfree(&obj->kconfig);9026 9027	for (i = 0; i < obj->nr_extern; i++)9028		zfree(&obj->externs[i].essent_name);9029 9030	zfree(&obj->externs);9031	obj->nr_extern = 0;9032 9033	zfree(&obj->maps);9034	obj->nr_maps = 0;9035 9036	if (obj->programs && obj->nr_programs) {9037		for (i = 0; i < obj->nr_programs; i++)9038			bpf_program__exit(&obj->programs[i]);9039	}9040	zfree(&obj->programs);9041 9042	zfree(&obj->feat_cache);9043	zfree(&obj->token_path);9044	if (obj->token_fd > 0)9045		close(obj->token_fd);9046 9047	zfree(&obj->arena_data);9048 9049	free(obj);9050}9051 9052const char *bpf_object__name(const struct bpf_object *obj)9053{9054	return obj ? obj->name : libbpf_err_ptr(-EINVAL);9055}9056 9057unsigned int bpf_object__kversion(const struct bpf_object *obj)9058{9059	return obj ? obj->kern_version : 0;9060}9061 9062int bpf_object__token_fd(const struct bpf_object *obj)9063{9064	return obj->token_fd ?: -1;9065}9066 9067struct btf *bpf_object__btf(const struct bpf_object *obj)9068{9069	return obj ? obj->btf : NULL;9070}9071 9072int bpf_object__btf_fd(const struct bpf_object *obj)9073{9074	return obj->btf ? btf__fd(obj->btf) : -1;9075}9076 9077int bpf_object__set_kversion(struct bpf_object *obj, __u32 kern_version)9078{9079	if (obj->loaded)9080		return libbpf_err(-EINVAL);9081 9082	obj->kern_version = kern_version;9083 9084	return 0;9085}9086 9087int bpf_object__gen_loader(struct bpf_object *obj, struct gen_loader_opts *opts)9088{9089	struct bpf_gen *gen;9090 9091	if (!opts)9092		return -EFAULT;9093	if (!OPTS_VALID(opts, gen_loader_opts))9094		return -EINVAL;9095	gen = calloc(sizeof(*gen), 1);9096	if (!gen)9097		return -ENOMEM;9098	gen->opts = opts;9099	obj->gen_loader = gen;9100	return 0;9101}9102 9103static struct bpf_program *9104__bpf_program__iter(const struct bpf_program *p, const struct bpf_object *obj,9105		    bool forward)9106{9107	size_t nr_programs = obj->nr_programs;9108	ssize_t idx;9109 9110	if (!nr_programs)9111		return NULL;9112 9113	if (!p)9114		/* Iter from the beginning */9115		return forward ? &obj->programs[0] :9116			&obj->programs[nr_programs - 1];9117 9118	if (p->obj != obj) {9119		pr_warn("error: program handler doesn't match object\n");9120		return errno = EINVAL, NULL;9121	}9122 9123	idx = (p - obj->programs) + (forward ? 1 : -1);9124	if (idx >= obj->nr_programs || idx < 0)9125		return NULL;9126	return &obj->programs[idx];9127}9128 9129struct bpf_program *9130bpf_object__next_program(const struct bpf_object *obj, struct bpf_program *prev)9131{9132	struct bpf_program *prog = prev;9133 9134	do {9135		prog = __bpf_program__iter(prog, obj, true);9136	} while (prog && prog_is_subprog(obj, prog));9137 9138	return prog;9139}9140 9141struct bpf_program *9142bpf_object__prev_program(const struct bpf_object *obj, struct bpf_program *next)9143{9144	struct bpf_program *prog = next;9145 9146	do {9147		prog = __bpf_program__iter(prog, obj, false);9148	} while (prog && prog_is_subprog(obj, prog));9149 9150	return prog;9151}9152 9153void bpf_program__set_ifindex(struct bpf_program *prog, __u32 ifindex)9154{9155	prog->prog_ifindex = ifindex;9156}9157 9158const char *bpf_program__name(const struct bpf_program *prog)9159{9160	return prog->name;9161}9162 9163const char *bpf_program__section_name(const struct bpf_program *prog)9164{9165	return prog->sec_name;9166}9167 9168bool bpf_program__autoload(const struct bpf_program *prog)9169{9170	return prog->autoload;9171}9172 9173int bpf_program__set_autoload(struct bpf_program *prog, bool autoload)9174{9175	if (prog->obj->loaded)9176		return libbpf_err(-EINVAL);9177 9178	prog->autoload = autoload;9179	return 0;9180}9181 9182bool bpf_program__autoattach(const struct bpf_program *prog)9183{9184	return prog->autoattach;9185}9186 9187void bpf_program__set_autoattach(struct bpf_program *prog, bool autoattach)9188{9189	prog->autoattach = autoattach;9190}9191 9192const struct bpf_insn *bpf_program__insns(const struct bpf_program *prog)9193{9194	return prog->insns;9195}9196 9197size_t bpf_program__insn_cnt(const struct bpf_program *prog)9198{9199	return prog->insns_cnt;9200}9201 9202int bpf_program__set_insns(struct bpf_program *prog,9203			   struct bpf_insn *new_insns, size_t new_insn_cnt)9204{9205	struct bpf_insn *insns;9206 9207	if (prog->obj->loaded)9208		return -EBUSY;9209 9210	insns = libbpf_reallocarray(prog->insns, new_insn_cnt, sizeof(*insns));9211	/* NULL is a valid return from reallocarray if the new count is zero */9212	if (!insns && new_insn_cnt) {9213		pr_warn("prog '%s': failed to realloc prog code\n", prog->name);9214		return -ENOMEM;9215	}9216	memcpy(insns, new_insns, new_insn_cnt * sizeof(*insns));9217 9218	prog->insns = insns;9219	prog->insns_cnt = new_insn_cnt;9220	return 0;9221}9222 9223int bpf_program__fd(const struct bpf_program *prog)9224{9225	if (!prog)9226		return libbpf_err(-EINVAL);9227 9228	if (prog->fd < 0)9229		return libbpf_err(-ENOENT);9230 9231	return prog->fd;9232}9233 9234__alias(bpf_program__type)9235enum bpf_prog_type bpf_program__get_type(const struct bpf_program *prog);9236 9237enum bpf_prog_type bpf_program__type(const struct bpf_program *prog)9238{9239	return prog->type;9240}9241 9242static size_t custom_sec_def_cnt;9243static struct bpf_sec_def *custom_sec_defs;9244static struct bpf_sec_def custom_fallback_def;9245static bool has_custom_fallback_def;9246static int last_custom_sec_def_handler_id;9247 9248int bpf_program__set_type(struct bpf_program *prog, enum bpf_prog_type type)9249{9250	if (prog->obj->loaded)9251		return libbpf_err(-EBUSY);9252 9253	/* if type is not changed, do nothing */9254	if (prog->type == type)9255		return 0;9256 9257	prog->type = type;9258 9259	/* If a program type was changed, we need to reset associated SEC()9260	 * handler, as it will be invalid now. The only exception is a generic9261	 * fallback handler, which by definition is program type-agnostic and9262	 * is a catch-all custom handler, optionally set by the application,9263	 * so should be able to handle any type of BPF program.9264	 */9265	if (prog->sec_def != &custom_fallback_def)9266		prog->sec_def = NULL;9267	return 0;9268}9269 9270__alias(bpf_program__expected_attach_type)9271enum bpf_attach_type bpf_program__get_expected_attach_type(const struct bpf_program *prog);9272 9273enum bpf_attach_type bpf_program__expected_attach_type(const struct bpf_program *prog)9274{9275	return prog->expected_attach_type;9276}9277 9278int bpf_program__set_expected_attach_type(struct bpf_program *prog,9279					   enum bpf_attach_type type)9280{9281	if (prog->obj->loaded)9282		return libbpf_err(-EBUSY);9283 9284	prog->expected_attach_type = type;9285	return 0;9286}9287 9288__u32 bpf_program__flags(const struct bpf_program *prog)9289{9290	return prog->prog_flags;9291}9292 9293int bpf_program__set_flags(struct bpf_program *prog, __u32 flags)9294{9295	if (prog->obj->loaded)9296		return libbpf_err(-EBUSY);9297 9298	prog->prog_flags = flags;9299	return 0;9300}9301 9302__u32 bpf_program__log_level(const struct bpf_program *prog)9303{9304	return prog->log_level;9305}9306 9307int bpf_program__set_log_level(struct bpf_program *prog, __u32 log_level)9308{9309	if (prog->obj->loaded)9310		return libbpf_err(-EBUSY);9311 9312	prog->log_level = log_level;9313	return 0;9314}9315 9316const char *bpf_program__log_buf(const struct bpf_program *prog, size_t *log_size)9317{9318	*log_size = prog->log_size;9319	return prog->log_buf;9320}9321 9322int bpf_program__set_log_buf(struct bpf_program *prog, char *log_buf, size_t log_size)9323{9324	if (log_size && !log_buf)9325		return -EINVAL;9326	if (prog->log_size > UINT_MAX)9327		return -EINVAL;9328	if (prog->obj->loaded)9329		return -EBUSY;9330 9331	prog->log_buf = log_buf;9332	prog->log_size = log_size;9333	return 0;9334}9335 9336#define SEC_DEF(sec_pfx, ptype, atype, flags, ...) {			    \9337	.sec = (char *)sec_pfx,						    \9338	.prog_type = BPF_PROG_TYPE_##ptype,				    \9339	.expected_attach_type = atype,					    \9340	.cookie = (long)(flags),					    \9341	.prog_prepare_load_fn = libbpf_prepare_prog_load,		    \9342	__VA_ARGS__							    \9343}9344 9345static int attach_kprobe(const struct bpf_program *prog, long cookie, struct bpf_link **link);9346static int attach_uprobe(const struct bpf_program *prog, long cookie, struct bpf_link **link);9347static int attach_ksyscall(const struct bpf_program *prog, long cookie, struct bpf_link **link);9348static int attach_usdt(const struct bpf_program *prog, long cookie, struct bpf_link **link);9349static int attach_tp(const struct bpf_program *prog, long cookie, struct bpf_link **link);9350static int attach_raw_tp(const struct bpf_program *prog, long cookie, struct bpf_link **link);9351static int attach_trace(const struct bpf_program *prog, long cookie, struct bpf_link **link);9352static int attach_kprobe_multi(const struct bpf_program *prog, long cookie, struct bpf_link **link);9353static int attach_kprobe_session(const struct bpf_program *prog, long cookie, struct bpf_link **link);9354static int attach_uprobe_multi(const struct bpf_program *prog, long cookie, struct bpf_link **link);9355static int attach_lsm(const struct bpf_program *prog, long cookie, struct bpf_link **link);9356static int attach_iter(const struct bpf_program *prog, long cookie, struct bpf_link **link);9357 9358static const struct bpf_sec_def section_defs[] = {9359	SEC_DEF("socket",		SOCKET_FILTER, 0, SEC_NONE),9360	SEC_DEF("sk_reuseport/migrate",	SK_REUSEPORT, BPF_SK_REUSEPORT_SELECT_OR_MIGRATE, SEC_ATTACHABLE),9361	SEC_DEF("sk_reuseport",		SK_REUSEPORT, BPF_SK_REUSEPORT_SELECT, SEC_ATTACHABLE),9362	SEC_DEF("kprobe+",		KPROBE,	0, SEC_NONE, attach_kprobe),9363	SEC_DEF("uprobe+",		KPROBE,	0, SEC_NONE, attach_uprobe),9364	SEC_DEF("uprobe.s+",		KPROBE,	0, SEC_SLEEPABLE, attach_uprobe),9365	SEC_DEF("kretprobe+",		KPROBE, 0, SEC_NONE, attach_kprobe),9366	SEC_DEF("uretprobe+",		KPROBE, 0, SEC_NONE, attach_uprobe),9367	SEC_DEF("uretprobe.s+",		KPROBE, 0, SEC_SLEEPABLE, attach_uprobe),9368	SEC_DEF("kprobe.multi+",	KPROBE,	BPF_TRACE_KPROBE_MULTI, SEC_NONE, attach_kprobe_multi),9369	SEC_DEF("kretprobe.multi+",	KPROBE,	BPF_TRACE_KPROBE_MULTI, SEC_NONE, attach_kprobe_multi),9370	SEC_DEF("kprobe.session+",	KPROBE,	BPF_TRACE_KPROBE_SESSION, SEC_NONE, attach_kprobe_session),9371	SEC_DEF("uprobe.multi+",	KPROBE,	BPF_TRACE_UPROBE_MULTI, SEC_NONE, attach_uprobe_multi),9372	SEC_DEF("uretprobe.multi+",	KPROBE,	BPF_TRACE_UPROBE_MULTI, SEC_NONE, attach_uprobe_multi),9373	SEC_DEF("uprobe.multi.s+",	KPROBE,	BPF_TRACE_UPROBE_MULTI, SEC_SLEEPABLE, attach_uprobe_multi),9374	SEC_DEF("uretprobe.multi.s+",	KPROBE,	BPF_TRACE_UPROBE_MULTI, SEC_SLEEPABLE, attach_uprobe_multi),9375	SEC_DEF("ksyscall+",		KPROBE,	0, SEC_NONE, attach_ksyscall),9376	SEC_DEF("kretsyscall+",		KPROBE, 0, SEC_NONE, attach_ksyscall),9377	SEC_DEF("usdt+",		KPROBE,	0, SEC_USDT, attach_usdt),9378	SEC_DEF("usdt.s+",		KPROBE,	0, SEC_USDT | SEC_SLEEPABLE, attach_usdt),9379	SEC_DEF("tc/ingress",		SCHED_CLS, BPF_TCX_INGRESS, SEC_NONE), /* alias for tcx */9380	SEC_DEF("tc/egress",		SCHED_CLS, BPF_TCX_EGRESS, SEC_NONE),  /* alias for tcx */9381	SEC_DEF("tcx/ingress",		SCHED_CLS, BPF_TCX_INGRESS, SEC_NONE),9382	SEC_DEF("tcx/egress",		SCHED_CLS, BPF_TCX_EGRESS, SEC_NONE),9383	SEC_DEF("tc",			SCHED_CLS, 0, SEC_NONE), /* deprecated / legacy, use tcx */9384	SEC_DEF("classifier",		SCHED_CLS, 0, SEC_NONE), /* deprecated / legacy, use tcx */9385	SEC_DEF("action",		SCHED_ACT, 0, SEC_NONE), /* deprecated / legacy, use tcx */9386	SEC_DEF("netkit/primary",	SCHED_CLS, BPF_NETKIT_PRIMARY, SEC_NONE),9387	SEC_DEF("netkit/peer",		SCHED_CLS, BPF_NETKIT_PEER, SEC_NONE),9388	SEC_DEF("tracepoint+",		TRACEPOINT, 0, SEC_NONE, attach_tp),9389	SEC_DEF("tp+",			TRACEPOINT, 0, SEC_NONE, attach_tp),9390	SEC_DEF("raw_tracepoint+",	RAW_TRACEPOINT, 0, SEC_NONE, attach_raw_tp),9391	SEC_DEF("raw_tp+",		RAW_TRACEPOINT, 0, SEC_NONE, attach_raw_tp),9392	SEC_DEF("raw_tracepoint.w+",	RAW_TRACEPOINT_WRITABLE, 0, SEC_NONE, attach_raw_tp),9393	SEC_DEF("raw_tp.w+",		RAW_TRACEPOINT_WRITABLE, 0, SEC_NONE, attach_raw_tp),9394	SEC_DEF("tp_btf+",		TRACING, BPF_TRACE_RAW_TP, SEC_ATTACH_BTF, attach_trace),9395	SEC_DEF("fentry+",		TRACING, BPF_TRACE_FENTRY, SEC_ATTACH_BTF, attach_trace),9396	SEC_DEF("fmod_ret+",		TRACING, BPF_MODIFY_RETURN, SEC_ATTACH_BTF, attach_trace),9397	SEC_DEF("fexit+",		TRACING, BPF_TRACE_FEXIT, SEC_ATTACH_BTF, attach_trace),9398	SEC_DEF("fentry.s+",		TRACING, BPF_TRACE_FENTRY, SEC_ATTACH_BTF | SEC_SLEEPABLE, attach_trace),9399	SEC_DEF("fmod_ret.s+",		TRACING, BPF_MODIFY_RETURN, SEC_ATTACH_BTF | SEC_SLEEPABLE, attach_trace),9400	SEC_DEF("fexit.s+",		TRACING, BPF_TRACE_FEXIT, SEC_ATTACH_BTF | SEC_SLEEPABLE, attach_trace),9401	SEC_DEF("freplace+",		EXT, 0, SEC_ATTACH_BTF, attach_trace),9402	SEC_DEF("lsm+",			LSM, BPF_LSM_MAC, SEC_ATTACH_BTF, attach_lsm),9403	SEC_DEF("lsm.s+",		LSM, BPF_LSM_MAC, SEC_ATTACH_BTF | SEC_SLEEPABLE, attach_lsm),9404	SEC_DEF("lsm_cgroup+",		LSM, BPF_LSM_CGROUP, SEC_ATTACH_BTF),9405	SEC_DEF("iter+",		TRACING, BPF_TRACE_ITER, SEC_ATTACH_BTF, attach_iter),9406	SEC_DEF("iter.s+",		TRACING, BPF_TRACE_ITER, SEC_ATTACH_BTF | SEC_SLEEPABLE, attach_iter),9407	SEC_DEF("syscall",		SYSCALL, 0, SEC_SLEEPABLE),9408	SEC_DEF("xdp.frags/devmap",	XDP, BPF_XDP_DEVMAP, SEC_XDP_FRAGS),9409	SEC_DEF("xdp/devmap",		XDP, BPF_XDP_DEVMAP, SEC_ATTACHABLE),9410	SEC_DEF("xdp.frags/cpumap",	XDP, BPF_XDP_CPUMAP, SEC_XDP_FRAGS),9411	SEC_DEF("xdp/cpumap",		XDP, BPF_XDP_CPUMAP, SEC_ATTACHABLE),9412	SEC_DEF("xdp.frags",		XDP, BPF_XDP, SEC_XDP_FRAGS),9413	SEC_DEF("xdp",			XDP, BPF_XDP, SEC_ATTACHABLE_OPT),9414	SEC_DEF("perf_event",		PERF_EVENT, 0, SEC_NONE),9415	SEC_DEF("lwt_in",		LWT_IN, 0, SEC_NONE),9416	SEC_DEF("lwt_out",		LWT_OUT, 0, SEC_NONE),9417	SEC_DEF("lwt_xmit",		LWT_XMIT, 0, SEC_NONE),9418	SEC_DEF("lwt_seg6local",	LWT_SEG6LOCAL, 0, SEC_NONE),9419	SEC_DEF("sockops",		SOCK_OPS, BPF_CGROUP_SOCK_OPS, SEC_ATTACHABLE_OPT),9420	SEC_DEF("sk_skb/stream_parser",	SK_SKB, BPF_SK_SKB_STREAM_PARSER, SEC_ATTACHABLE_OPT),9421	SEC_DEF("sk_skb/stream_verdict",SK_SKB, BPF_SK_SKB_STREAM_VERDICT, SEC_ATTACHABLE_OPT),9422	SEC_DEF("sk_skb/verdict",	SK_SKB, BPF_SK_SKB_VERDICT, SEC_ATTACHABLE_OPT),9423	SEC_DEF("sk_skb",		SK_SKB, 0, SEC_NONE),9424	SEC_DEF("sk_msg",		SK_MSG, BPF_SK_MSG_VERDICT, SEC_ATTACHABLE_OPT),9425	SEC_DEF("lirc_mode2",		LIRC_MODE2, BPF_LIRC_MODE2, SEC_ATTACHABLE_OPT),9426	SEC_DEF("flow_dissector",	FLOW_DISSECTOR, BPF_FLOW_DISSECTOR, SEC_ATTACHABLE_OPT),9427	SEC_DEF("cgroup_skb/ingress",	CGROUP_SKB, BPF_CGROUP_INET_INGRESS, SEC_ATTACHABLE_OPT),9428	SEC_DEF("cgroup_skb/egress",	CGROUP_SKB, BPF_CGROUP_INET_EGRESS, SEC_ATTACHABLE_OPT),9429	SEC_DEF("cgroup/skb",		CGROUP_SKB, 0, SEC_NONE),9430	SEC_DEF("cgroup/sock_create",	CGROUP_SOCK, BPF_CGROUP_INET_SOCK_CREATE, SEC_ATTACHABLE),9431	SEC_DEF("cgroup/sock_release",	CGROUP_SOCK, BPF_CGROUP_INET_SOCK_RELEASE, SEC_ATTACHABLE),9432	SEC_DEF("cgroup/sock",		CGROUP_SOCK, BPF_CGROUP_INET_SOCK_CREATE, SEC_ATTACHABLE_OPT),9433	SEC_DEF("cgroup/post_bind4",	CGROUP_SOCK, BPF_CGROUP_INET4_POST_BIND, SEC_ATTACHABLE),9434	SEC_DEF("cgroup/post_bind6",	CGROUP_SOCK, BPF_CGROUP_INET6_POST_BIND, SEC_ATTACHABLE),9435	SEC_DEF("cgroup/bind4",		CGROUP_SOCK_ADDR, BPF_CGROUP_INET4_BIND, SEC_ATTACHABLE),9436	SEC_DEF("cgroup/bind6",		CGROUP_SOCK_ADDR, BPF_CGROUP_INET6_BIND, SEC_ATTACHABLE),9437	SEC_DEF("cgroup/connect4",	CGROUP_SOCK_ADDR, BPF_CGROUP_INET4_CONNECT, SEC_ATTACHABLE),9438	SEC_DEF("cgroup/connect6",	CGROUP_SOCK_ADDR, BPF_CGROUP_INET6_CONNECT, SEC_ATTACHABLE),9439	SEC_DEF("cgroup/connect_unix",	CGROUP_SOCK_ADDR, BPF_CGROUP_UNIX_CONNECT, SEC_ATTACHABLE),9440	SEC_DEF("cgroup/sendmsg4",	CGROUP_SOCK_ADDR, BPF_CGROUP_UDP4_SENDMSG, SEC_ATTACHABLE),9441	SEC_DEF("cgroup/sendmsg6",	CGROUP_SOCK_ADDR, BPF_CGROUP_UDP6_SENDMSG, SEC_ATTACHABLE),9442	SEC_DEF("cgroup/sendmsg_unix",	CGROUP_SOCK_ADDR, BPF_CGROUP_UNIX_SENDMSG, SEC_ATTACHABLE),9443	SEC_DEF("cgroup/recvmsg4",	CGROUP_SOCK_ADDR, BPF_CGROUP_UDP4_RECVMSG, SEC_ATTACHABLE),9444	SEC_DEF("cgroup/recvmsg6",	CGROUP_SOCK_ADDR, BPF_CGROUP_UDP6_RECVMSG, SEC_ATTACHABLE),9445	SEC_DEF("cgroup/recvmsg_unix",	CGROUP_SOCK_ADDR, BPF_CGROUP_UNIX_RECVMSG, SEC_ATTACHABLE),9446	SEC_DEF("cgroup/getpeername4",	CGROUP_SOCK_ADDR, BPF_CGROUP_INET4_GETPEERNAME, SEC_ATTACHABLE),9447	SEC_DEF("cgroup/getpeername6",	CGROUP_SOCK_ADDR, BPF_CGROUP_INET6_GETPEERNAME, SEC_ATTACHABLE),9448	SEC_DEF("cgroup/getpeername_unix", CGROUP_SOCK_ADDR, BPF_CGROUP_UNIX_GETPEERNAME, SEC_ATTACHABLE),9449	SEC_DEF("cgroup/getsockname4",	CGROUP_SOCK_ADDR, BPF_CGROUP_INET4_GETSOCKNAME, SEC_ATTACHABLE),9450	SEC_DEF("cgroup/getsockname6",	CGROUP_SOCK_ADDR, BPF_CGROUP_INET6_GETSOCKNAME, SEC_ATTACHABLE),9451	SEC_DEF("cgroup/getsockname_unix", CGROUP_SOCK_ADDR, BPF_CGROUP_UNIX_GETSOCKNAME, SEC_ATTACHABLE),9452	SEC_DEF("cgroup/sysctl",	CGROUP_SYSCTL, BPF_CGROUP_SYSCTL, SEC_ATTACHABLE),9453	SEC_DEF("cgroup/getsockopt",	CGROUP_SOCKOPT, BPF_CGROUP_GETSOCKOPT, SEC_ATTACHABLE),9454	SEC_DEF("cgroup/setsockopt",	CGROUP_SOCKOPT, BPF_CGROUP_SETSOCKOPT, SEC_ATTACHABLE),9455	SEC_DEF("cgroup/dev",		CGROUP_DEVICE, BPF_CGROUP_DEVICE, SEC_ATTACHABLE_OPT),9456	SEC_DEF("struct_ops+",		STRUCT_OPS, 0, SEC_NONE),9457	SEC_DEF("struct_ops.s+",	STRUCT_OPS, 0, SEC_SLEEPABLE),9458	SEC_DEF("sk_lookup",		SK_LOOKUP, BPF_SK_LOOKUP, SEC_ATTACHABLE),9459	SEC_DEF("netfilter",		NETFILTER, BPF_NETFILTER, SEC_NONE),9460};9461 9462int libbpf_register_prog_handler(const char *sec,9463				 enum bpf_prog_type prog_type,9464				 enum bpf_attach_type exp_attach_type,9465				 const struct libbpf_prog_handler_opts *opts)9466{9467	struct bpf_sec_def *sec_def;9468 9469	if (!OPTS_VALID(opts, libbpf_prog_handler_opts))9470		return libbpf_err(-EINVAL);9471 9472	if (last_custom_sec_def_handler_id == INT_MAX) /* prevent overflow */9473		return libbpf_err(-E2BIG);9474 9475	if (sec) {9476		sec_def = libbpf_reallocarray(custom_sec_defs, custom_sec_def_cnt + 1,9477					      sizeof(*sec_def));9478		if (!sec_def)9479			return libbpf_err(-ENOMEM);9480 9481		custom_sec_defs = sec_def;9482		sec_def = &custom_sec_defs[custom_sec_def_cnt];9483	} else {9484		if (has_custom_fallback_def)9485			return libbpf_err(-EBUSY);9486 9487		sec_def = &custom_fallback_def;9488	}9489 9490	sec_def->sec = sec ? strdup(sec) : NULL;9491	if (sec && !sec_def->sec)9492		return libbpf_err(-ENOMEM);9493 9494	sec_def->prog_type = prog_type;9495	sec_def->expected_attach_type = exp_attach_type;9496	sec_def->cookie = OPTS_GET(opts, cookie, 0);9497 9498	sec_def->prog_setup_fn = OPTS_GET(opts, prog_setup_fn, NULL);9499	sec_def->prog_prepare_load_fn = OPTS_GET(opts, prog_prepare_load_fn, NULL);9500	sec_def->prog_attach_fn = OPTS_GET(opts, prog_attach_fn, NULL);9501 9502	sec_def->handler_id = ++last_custom_sec_def_handler_id;9503 9504	if (sec)9505		custom_sec_def_cnt++;9506	else9507		has_custom_fallback_def = true;9508 9509	return sec_def->handler_id;9510}9511 9512int libbpf_unregister_prog_handler(int handler_id)9513{9514	struct bpf_sec_def *sec_defs;9515	int i;9516 9517	if (handler_id <= 0)9518		return libbpf_err(-EINVAL);9519 9520	if (has_custom_fallback_def && custom_fallback_def.handler_id == handler_id) {9521		memset(&custom_fallback_def, 0, sizeof(custom_fallback_def));9522		has_custom_fallback_def = false;9523		return 0;9524	}9525 9526	for (i = 0; i < custom_sec_def_cnt; i++) {9527		if (custom_sec_defs[i].handler_id == handler_id)9528			break;9529	}9530 9531	if (i == custom_sec_def_cnt)9532		return libbpf_err(-ENOENT);9533 9534	free(custom_sec_defs[i].sec);9535	for (i = i + 1; i < custom_sec_def_cnt; i++)9536		custom_sec_defs[i - 1] = custom_sec_defs[i];9537	custom_sec_def_cnt--;9538 9539	/* try to shrink the array, but it's ok if we couldn't */9540	sec_defs = libbpf_reallocarray(custom_sec_defs, custom_sec_def_cnt, sizeof(*sec_defs));9541	/* if new count is zero, reallocarray can return a valid NULL result;9542	 * in this case the previous pointer will be freed, so we *have to*9543	 * reassign old pointer to the new value (even if it's NULL)9544	 */9545	if (sec_defs || custom_sec_def_cnt == 0)9546		custom_sec_defs = sec_defs;9547 9548	return 0;9549}9550 9551static bool sec_def_matches(const struct bpf_sec_def *sec_def, const char *sec_name)9552{9553	size_t len = strlen(sec_def->sec);9554 9555	/* "type/" always has to have proper SEC("type/extras") form */9556	if (sec_def->sec[len - 1] == '/') {9557		if (str_has_pfx(sec_name, sec_def->sec))9558			return true;9559		return false;9560	}9561 9562	/* "type+" means it can be either exact SEC("type") or9563	 * well-formed SEC("type/extras") with proper '/' separator9564	 */9565	if (sec_def->sec[len - 1] == '+') {9566		len--;9567		/* not even a prefix */9568		if (strncmp(sec_name, sec_def->sec, len) != 0)9569			return false;9570		/* exact match or has '/' separator */9571		if (sec_name[len] == '\0' || sec_name[len] == '/')9572			return true;9573		return false;9574	}9575 9576	return strcmp(sec_name, sec_def->sec) == 0;9577}9578 9579static const struct bpf_sec_def *find_sec_def(const char *sec_name)9580{9581	const struct bpf_sec_def *sec_def;9582	int i, n;9583 9584	n = custom_sec_def_cnt;9585	for (i = 0; i < n; i++) {9586		sec_def = &custom_sec_defs[i];9587		if (sec_def_matches(sec_def, sec_name))9588			return sec_def;9589	}9590 9591	n = ARRAY_SIZE(section_defs);9592	for (i = 0; i < n; i++) {9593		sec_def = &section_defs[i];9594		if (sec_def_matches(sec_def, sec_name))9595			return sec_def;9596	}9597 9598	if (has_custom_fallback_def)9599		return &custom_fallback_def;9600 9601	return NULL;9602}9603 9604#define MAX_TYPE_NAME_SIZE 329605 9606static char *libbpf_get_type_names(bool attach_type)9607{9608	int i, len = ARRAY_SIZE(section_defs) * MAX_TYPE_NAME_SIZE;9609	char *buf;9610 9611	buf = malloc(len);9612	if (!buf)9613		return NULL;9614 9615	buf[0] = '\0';9616	/* Forge string buf with all available names */9617	for (i = 0; i < ARRAY_SIZE(section_defs); i++) {9618		const struct bpf_sec_def *sec_def = &section_defs[i];9619 9620		if (attach_type) {9621			if (sec_def->prog_prepare_load_fn != libbpf_prepare_prog_load)9622				continue;9623 9624			if (!(sec_def->cookie & SEC_ATTACHABLE))9625				continue;9626		}9627 9628		if (strlen(buf) + strlen(section_defs[i].sec) + 2 > len) {9629			free(buf);9630			return NULL;9631		}9632		strcat(buf, " ");9633		strcat(buf, section_defs[i].sec);9634	}9635 9636	return buf;9637}9638 9639int libbpf_prog_type_by_name(const char *name, enum bpf_prog_type *prog_type,9640			     enum bpf_attach_type *expected_attach_type)9641{9642	const struct bpf_sec_def *sec_def;9643	char *type_names;9644 9645	if (!name)9646		return libbpf_err(-EINVAL);9647 9648	sec_def = find_sec_def(name);9649	if (sec_def) {9650		*prog_type = sec_def->prog_type;9651		*expected_attach_type = sec_def->expected_attach_type;9652		return 0;9653	}9654 9655	pr_debug("failed to guess program type from ELF section '%s'\n", name);9656	type_names = libbpf_get_type_names(false);9657	if (type_names != NULL) {9658		pr_debug("supported section(type) names are:%s\n", type_names);9659		free(type_names);9660	}9661 9662	return libbpf_err(-ESRCH);9663}9664 9665const char *libbpf_bpf_attach_type_str(enum bpf_attach_type t)9666{9667	if (t < 0 || t >= ARRAY_SIZE(attach_type_name))9668		return NULL;9669 9670	return attach_type_name[t];9671}9672 9673const char *libbpf_bpf_link_type_str(enum bpf_link_type t)9674{9675	if (t < 0 || t >= ARRAY_SIZE(link_type_name))9676		return NULL;9677 9678	return link_type_name[t];9679}9680 9681const char *libbpf_bpf_map_type_str(enum bpf_map_type t)9682{9683	if (t < 0 || t >= ARRAY_SIZE(map_type_name))9684		return NULL;9685 9686	return map_type_name[t];9687}9688 9689const char *libbpf_bpf_prog_type_str(enum bpf_prog_type t)9690{9691	if (t < 0 || t >= ARRAY_SIZE(prog_type_name))9692		return NULL;9693 9694	return prog_type_name[t];9695}9696 9697static struct bpf_map *find_struct_ops_map_by_offset(struct bpf_object *obj,9698						     int sec_idx,9699						     size_t offset)9700{9701	struct bpf_map *map;9702	size_t i;9703 9704	for (i = 0; i < obj->nr_maps; i++) {9705		map = &obj->maps[i];9706		if (!bpf_map__is_struct_ops(map))9707			continue;9708		if (map->sec_idx == sec_idx &&9709		    map->sec_offset <= offset &&9710		    offset - map->sec_offset < map->def.value_size)9711			return map;9712	}9713 9714	return NULL;9715}9716 9717/* Collect the reloc from ELF, populate the st_ops->progs[], and update9718 * st_ops->data for shadow type.9719 */9720static int bpf_object__collect_st_ops_relos(struct bpf_object *obj,9721					    Elf64_Shdr *shdr, Elf_Data *data)9722{9723	const struct btf_type *type;9724	const struct btf_member *member;9725	struct bpf_struct_ops *st_ops;9726	struct bpf_program *prog;9727	unsigned int shdr_idx;9728	const struct btf *btf;9729	struct bpf_map *map;9730	unsigned int moff, insn_idx;9731	const char *name;9732	__u32 member_idx;9733	Elf64_Sym *sym;9734	Elf64_Rel *rel;9735	int i, nrels;9736 9737	btf = obj->btf;9738	nrels = shdr->sh_size / shdr->sh_entsize;9739	for (i = 0; i < nrels; i++) {9740		rel = elf_rel_by_idx(data, i);9741		if (!rel) {9742			pr_warn("struct_ops reloc: failed to get %d reloc\n", i);9743			return -LIBBPF_ERRNO__FORMAT;9744		}9745 9746		sym = elf_sym_by_idx(obj, ELF64_R_SYM(rel->r_info));9747		if (!sym) {9748			pr_warn("struct_ops reloc: symbol %zx not found\n",9749				(size_t)ELF64_R_SYM(rel->r_info));9750			return -LIBBPF_ERRNO__FORMAT;9751		}9752 9753		name = elf_sym_str(obj, sym->st_name) ?: "<?>";9754		map = find_struct_ops_map_by_offset(obj, shdr->sh_info, rel->r_offset);9755		if (!map) {9756			pr_warn("struct_ops reloc: cannot find map at rel->r_offset %zu\n",9757				(size_t)rel->r_offset);9758			return -EINVAL;9759		}9760 9761		moff = rel->r_offset - map->sec_offset;9762		shdr_idx = sym->st_shndx;9763		st_ops = map->st_ops;9764		pr_debug("struct_ops reloc %s: for %lld value %lld shdr_idx %u rel->r_offset %zu map->sec_offset %zu name %d (\'%s\')\n",9765			 map->name,9766			 (long long)(rel->r_info >> 32),9767			 (long long)sym->st_value,9768			 shdr_idx, (size_t)rel->r_offset,9769			 map->sec_offset, sym->st_name, name);9770 9771		if (shdr_idx >= SHN_LORESERVE) {9772			pr_warn("struct_ops reloc %s: rel->r_offset %zu shdr_idx %u unsupported non-static function\n",9773				map->name, (size_t)rel->r_offset, shdr_idx);9774			return -LIBBPF_ERRNO__RELOC;9775		}9776		if (sym->st_value % BPF_INSN_SZ) {9777			pr_warn("struct_ops reloc %s: invalid target program offset %llu\n",9778				map->name, (unsigned long long)sym->st_value);9779			return -LIBBPF_ERRNO__FORMAT;9780		}9781		insn_idx = sym->st_value / BPF_INSN_SZ;9782 9783		type = btf__type_by_id(btf, st_ops->type_id);9784		member = find_member_by_offset(type, moff * 8);9785		if (!member) {9786			pr_warn("struct_ops reloc %s: cannot find member at moff %u\n",9787				map->name, moff);9788			return -EINVAL;9789		}9790		member_idx = member - btf_members(type);9791		name = btf__name_by_offset(btf, member->name_off);9792 9793		if (!resolve_func_ptr(btf, member->type, NULL)) {9794			pr_warn("struct_ops reloc %s: cannot relocate non func ptr %s\n",9795				map->name, name);9796			return -EINVAL;9797		}9798 9799		prog = find_prog_by_sec_insn(obj, shdr_idx, insn_idx);9800		if (!prog) {9801			pr_warn("struct_ops reloc %s: cannot find prog at shdr_idx %u to relocate func ptr %s\n",9802				map->name, shdr_idx, name);9803			return -EINVAL;9804		}9805 9806		/* prevent the use of BPF prog with invalid type */9807		if (prog->type != BPF_PROG_TYPE_STRUCT_OPS) {9808			pr_warn("struct_ops reloc %s: prog %s is not struct_ops BPF program\n",9809				map->name, prog->name);9810			return -EINVAL;9811		}9812 9813		st_ops->progs[member_idx] = prog;9814 9815		/* st_ops->data will be exposed to users, being returned by9816		 * bpf_map__initial_value() as a pointer to the shadow9817		 * type. All function pointers in the original struct type9818		 * should be converted to a pointer to struct bpf_program9819		 * in the shadow type.9820		 */9821		*((struct bpf_program **)(st_ops->data + moff)) = prog;9822	}9823 9824	return 0;9825}9826 9827#define BTF_TRACE_PREFIX "btf_trace_"9828#define BTF_LSM_PREFIX "bpf_lsm_"9829#define BTF_ITER_PREFIX "bpf_iter_"9830#define BTF_MAX_NAME_SIZE 1289831 9832void btf_get_kernel_prefix_kind(enum bpf_attach_type attach_type,9833				const char **prefix, int *kind)9834{9835	switch (attach_type) {9836	case BPF_TRACE_RAW_TP:9837		*prefix = BTF_TRACE_PREFIX;9838		*kind = BTF_KIND_TYPEDEF;9839		break;9840	case BPF_LSM_MAC:9841	case BPF_LSM_CGROUP:9842		*prefix = BTF_LSM_PREFIX;9843		*kind = BTF_KIND_FUNC;9844		break;9845	case BPF_TRACE_ITER:9846		*prefix = BTF_ITER_PREFIX;9847		*kind = BTF_KIND_FUNC;9848		break;9849	default:9850		*prefix = "";9851		*kind = BTF_KIND_FUNC;9852	}9853}9854 9855static int find_btf_by_prefix_kind(const struct btf *btf, const char *prefix,9856				   const char *name, __u32 kind)9857{9858	char btf_type_name[BTF_MAX_NAME_SIZE];9859	int ret;9860 9861	ret = snprintf(btf_type_name, sizeof(btf_type_name),9862		       "%s%s", prefix, name);9863	/* snprintf returns the number of characters written excluding the9864	 * terminating null. So, if >= BTF_MAX_NAME_SIZE are written, it9865	 * indicates truncation.9866	 */9867	if (ret < 0 || ret >= sizeof(btf_type_name))9868		return -ENAMETOOLONG;9869	return btf__find_by_name_kind(btf, btf_type_name, kind);9870}9871 9872static inline int find_attach_btf_id(struct btf *btf, const char *name,9873				     enum bpf_attach_type attach_type)9874{9875	const char *prefix;9876	int kind;9877 9878	btf_get_kernel_prefix_kind(attach_type, &prefix, &kind);9879	return find_btf_by_prefix_kind(btf, prefix, name, kind);9880}9881 9882int libbpf_find_vmlinux_btf_id(const char *name,9883			       enum bpf_attach_type attach_type)9884{9885	struct btf *btf;9886	int err;9887 9888	btf = btf__load_vmlinux_btf();9889	err = libbpf_get_error(btf);9890	if (err) {9891		pr_warn("vmlinux BTF is not found\n");9892		return libbpf_err(err);9893	}9894 9895	err = find_attach_btf_id(btf, name, attach_type);9896	if (err <= 0)9897		pr_warn("%s is not found in vmlinux BTF\n", name);9898 9899	btf__free(btf);9900	return libbpf_err(err);9901}9902 9903static int libbpf_find_prog_btf_id(const char *name, __u32 attach_prog_fd)9904{9905	struct bpf_prog_info info;9906	__u32 info_len = sizeof(info);9907	struct btf *btf;9908	int err;9909 9910	memset(&info, 0, info_len);9911	err = bpf_prog_get_info_by_fd(attach_prog_fd, &info, &info_len);9912	if (err) {9913		pr_warn("failed bpf_prog_get_info_by_fd for FD %d: %d\n",9914			attach_prog_fd, err);9915		return err;9916	}9917 9918	err = -EINVAL;9919	if (!info.btf_id) {9920		pr_warn("The target program doesn't have BTF\n");9921		goto out;9922	}9923	btf = btf__load_from_kernel_by_id(info.btf_id);9924	err = libbpf_get_error(btf);9925	if (err) {9926		pr_warn("Failed to get BTF %d of the program: %d\n", info.btf_id, err);9927		goto out;9928	}9929	err = btf__find_by_name_kind(btf, name, BTF_KIND_FUNC);9930	btf__free(btf);9931	if (err <= 0) {9932		pr_warn("%s is not found in prog's BTF\n", name);9933		goto out;9934	}9935out:9936	return err;9937}9938 9939static int find_kernel_btf_id(struct bpf_object *obj, const char *attach_name,9940			      enum bpf_attach_type attach_type,9941			      int *btf_obj_fd, int *btf_type_id)9942{9943	int ret, i, mod_len;9944	const char *fn_name, *mod_name = NULL;9945 9946	fn_name = strchr(attach_name, ':');9947	if (fn_name) {9948		mod_name = attach_name;9949		mod_len = fn_name - mod_name;9950		fn_name++;9951	}9952 9953	if (!mod_name || strncmp(mod_name, "vmlinux", mod_len) == 0) {9954		ret = find_attach_btf_id(obj->btf_vmlinux,9955					 mod_name ? fn_name : attach_name,9956					 attach_type);9957		if (ret > 0) {9958			*btf_obj_fd = 0; /* vmlinux BTF */9959			*btf_type_id = ret;9960			return 0;9961		}9962		if (ret != -ENOENT)9963			return ret;9964	}9965 9966	ret = load_module_btfs(obj);9967	if (ret)9968		return ret;9969 9970	for (i = 0; i < obj->btf_module_cnt; i++) {9971		const struct module_btf *mod = &obj->btf_modules[i];9972 9973		if (mod_name && strncmp(mod->name, mod_name, mod_len) != 0)9974			continue;9975 9976		ret = find_attach_btf_id(mod->btf,9977					 mod_name ? fn_name : attach_name,9978					 attach_type);9979		if (ret > 0) {9980			*btf_obj_fd = mod->fd;9981			*btf_type_id = ret;9982			return 0;9983		}9984		if (ret == -ENOENT)9985			continue;9986 9987		return ret;9988	}9989 9990	return -ESRCH;9991}9992 9993static int libbpf_find_attach_btf_id(struct bpf_program *prog, const char *attach_name,9994				     int *btf_obj_fd, int *btf_type_id)9995{9996	enum bpf_attach_type attach_type = prog->expected_attach_type;9997	__u32 attach_prog_fd = prog->attach_prog_fd;9998	int err = 0;9999 10000	/* BPF program's BTF ID */10001	if (prog->type == BPF_PROG_TYPE_EXT || attach_prog_fd) {10002		if (!attach_prog_fd) {10003			pr_warn("prog '%s': attach program FD is not set\n", prog->name);10004			return -EINVAL;10005		}10006		err = libbpf_find_prog_btf_id(attach_name, attach_prog_fd);10007		if (err < 0) {10008			pr_warn("prog '%s': failed to find BPF program (FD %d) BTF ID for '%s': %d\n",10009				 prog->name, attach_prog_fd, attach_name, err);10010			return err;10011		}10012		*btf_obj_fd = 0;10013		*btf_type_id = err;10014		return 0;10015	}10016 10017	/* kernel/module BTF ID */10018	if (prog->obj->gen_loader) {10019		bpf_gen__record_attach_target(prog->obj->gen_loader, attach_name, attach_type);10020		*btf_obj_fd = 0;10021		*btf_type_id = 1;10022	} else {10023		err = find_kernel_btf_id(prog->obj, attach_name,10024					 attach_type, btf_obj_fd,10025					 btf_type_id);10026	}10027	if (err) {10028		pr_warn("prog '%s': failed to find kernel BTF type ID of '%s': %d\n",10029			prog->name, attach_name, err);10030		return err;10031	}10032	return 0;10033}10034 10035int libbpf_attach_type_by_name(const char *name,10036			       enum bpf_attach_type *attach_type)10037{10038	char *type_names;10039	const struct bpf_sec_def *sec_def;10040 10041	if (!name)10042		return libbpf_err(-EINVAL);10043 10044	sec_def = find_sec_def(name);10045	if (!sec_def) {10046		pr_debug("failed to guess attach type based on ELF section name '%s'\n", name);10047		type_names = libbpf_get_type_names(true);10048		if (type_names != NULL) {10049			pr_debug("attachable section(type) names are:%s\n", type_names);10050			free(type_names);10051		}10052 10053		return libbpf_err(-EINVAL);10054	}10055 10056	if (sec_def->prog_prepare_load_fn != libbpf_prepare_prog_load)10057		return libbpf_err(-EINVAL);10058	if (!(sec_def->cookie & SEC_ATTACHABLE))10059		return libbpf_err(-EINVAL);10060 10061	*attach_type = sec_def->expected_attach_type;10062	return 0;10063}10064 10065int bpf_map__fd(const struct bpf_map *map)10066{10067	if (!map)10068		return libbpf_err(-EINVAL);10069	if (!map_is_created(map))10070		return -1;10071	return map->fd;10072}10073 10074static bool map_uses_real_name(const struct bpf_map *map)10075{10076	/* Since libbpf started to support custom .data.* and .rodata.* maps,10077	 * their user-visible name differs from kernel-visible name. Users see10078	 * such map's corresponding ELF section name as a map name.10079	 * This check distinguishes .data/.rodata from .data.* and .rodata.*10080	 * maps to know which name has to be returned to the user.10081	 */10082	if (map->libbpf_type == LIBBPF_MAP_DATA && strcmp(map->real_name, DATA_SEC) != 0)10083		return true;10084	if (map->libbpf_type == LIBBPF_MAP_RODATA && strcmp(map->real_name, RODATA_SEC) != 0)10085		return true;10086	return false;10087}10088 10089const char *bpf_map__name(const struct bpf_map *map)10090{10091	if (!map)10092		return NULL;10093 10094	if (map_uses_real_name(map))10095		return map->real_name;10096 10097	return map->name;10098}10099 10100enum bpf_map_type bpf_map__type(const struct bpf_map *map)10101{10102	return map->def.type;10103}10104 10105int bpf_map__set_type(struct bpf_map *map, enum bpf_map_type type)10106{10107	if (map_is_created(map))10108		return libbpf_err(-EBUSY);10109	map->def.type = type;10110	return 0;10111}10112 10113__u32 bpf_map__map_flags(const struct bpf_map *map)10114{10115	return map->def.map_flags;10116}10117 10118int bpf_map__set_map_flags(struct bpf_map *map, __u32 flags)10119{10120	if (map_is_created(map))10121		return libbpf_err(-EBUSY);10122	map->def.map_flags = flags;10123	return 0;10124}10125 10126__u64 bpf_map__map_extra(const struct bpf_map *map)10127{10128	return map->map_extra;10129}10130 10131int bpf_map__set_map_extra(struct bpf_map *map, __u64 map_extra)10132{10133	if (map_is_created(map))10134		return libbpf_err(-EBUSY);10135	map->map_extra = map_extra;10136	return 0;10137}10138 10139__u32 bpf_map__numa_node(const struct bpf_map *map)10140{10141	return map->numa_node;10142}10143 10144int bpf_map__set_numa_node(struct bpf_map *map, __u32 numa_node)10145{10146	if (map_is_created(map))10147		return libbpf_err(-EBUSY);10148	map->numa_node = numa_node;10149	return 0;10150}10151 10152__u32 bpf_map__key_size(const struct bpf_map *map)10153{10154	return map->def.key_size;10155}10156 10157int bpf_map__set_key_size(struct bpf_map *map, __u32 size)10158{10159	if (map_is_created(map))10160		return libbpf_err(-EBUSY);10161	map->def.key_size = size;10162	return 0;10163}10164 10165__u32 bpf_map__value_size(const struct bpf_map *map)10166{10167	return map->def.value_size;10168}10169 10170static int map_btf_datasec_resize(struct bpf_map *map, __u32 size)10171{10172	struct btf *btf;10173	struct btf_type *datasec_type, *var_type;10174	struct btf_var_secinfo *var;10175	const struct btf_type *array_type;10176	const struct btf_array *array;10177	int vlen, element_sz, new_array_id;10178	__u32 nr_elements;10179 10180	/* check btf existence */10181	btf = bpf_object__btf(map->obj);10182	if (!btf)10183		return -ENOENT;10184 10185	/* verify map is datasec */10186	datasec_type = btf_type_by_id(btf, bpf_map__btf_value_type_id(map));10187	if (!btf_is_datasec(datasec_type)) {10188		pr_warn("map '%s': cannot be resized, map value type is not a datasec\n",10189			bpf_map__name(map));10190		return -EINVAL;10191	}10192 10193	/* verify datasec has at least one var */10194	vlen = btf_vlen(datasec_type);10195	if (vlen == 0) {10196		pr_warn("map '%s': cannot be resized, map value datasec is empty\n",10197			bpf_map__name(map));10198		return -EINVAL;10199	}10200 10201	/* verify last var in the datasec is an array */10202	var = &btf_var_secinfos(datasec_type)[vlen - 1];10203	var_type = btf_type_by_id(btf, var->type);10204	array_type = skip_mods_and_typedefs(btf, var_type->type, NULL);10205	if (!btf_is_array(array_type)) {10206		pr_warn("map '%s': cannot be resized, last var must be an array\n",10207			bpf_map__name(map));10208		return -EINVAL;10209	}10210 10211	/* verify request size aligns with array */10212	array = btf_array(array_type);10213	element_sz = btf__resolve_size(btf, array->type);10214	if (element_sz <= 0 || (size - var->offset) % element_sz != 0) {10215		pr_warn("map '%s': cannot be resized, element size (%d) doesn't align with new total size (%u)\n",10216			bpf_map__name(map), element_sz, size);10217		return -EINVAL;10218	}10219 10220	/* create a new array based on the existing array, but with new length */10221	nr_elements = (size - var->offset) / element_sz;10222	new_array_id = btf__add_array(btf, array->index_type, array->type, nr_elements);10223	if (new_array_id < 0)10224		return new_array_id;10225 10226	/* adding a new btf type invalidates existing pointers to btf objects,10227	 * so refresh pointers before proceeding10228	 */10229	datasec_type = btf_type_by_id(btf, map->btf_value_type_id);10230	var = &btf_var_secinfos(datasec_type)[vlen - 1];10231	var_type = btf_type_by_id(btf, var->type);10232 10233	/* finally update btf info */10234	datasec_type->size = size;10235	var->size = size - var->offset;10236	var_type->type = new_array_id;10237 10238	return 0;10239}10240 10241int bpf_map__set_value_size(struct bpf_map *map, __u32 size)10242{10243	if (map->obj->loaded || map->reused)10244		return libbpf_err(-EBUSY);10245 10246	if (map->mmaped) {10247		size_t mmap_old_sz, mmap_new_sz;10248		int err;10249 10250		if (map->def.type != BPF_MAP_TYPE_ARRAY)10251			return -EOPNOTSUPP;10252 10253		mmap_old_sz = bpf_map_mmap_sz(map);10254		mmap_new_sz = array_map_mmap_sz(size, map->def.max_entries);10255		err = bpf_map_mmap_resize(map, mmap_old_sz, mmap_new_sz);10256		if (err) {10257			pr_warn("map '%s': failed to resize memory-mapped region: %d\n",10258				bpf_map__name(map), err);10259			return err;10260		}10261		err = map_btf_datasec_resize(map, size);10262		if (err && err != -ENOENT) {10263			pr_warn("map '%s': failed to adjust resized BTF, clearing BTF key/value info: %d\n",10264				bpf_map__name(map), err);10265			map->btf_value_type_id = 0;10266			map->btf_key_type_id = 0;10267		}10268	}10269 10270	map->def.value_size = size;10271	return 0;10272}10273 10274__u32 bpf_map__btf_key_type_id(const struct bpf_map *map)10275{10276	return map ? map->btf_key_type_id : 0;10277}10278 10279__u32 bpf_map__btf_value_type_id(const struct bpf_map *map)10280{10281	return map ? map->btf_value_type_id : 0;10282}10283 10284int bpf_map__set_initial_value(struct bpf_map *map,10285			       const void *data, size_t size)10286{10287	size_t actual_sz;10288 10289	if (map->obj->loaded || map->reused)10290		return libbpf_err(-EBUSY);10291 10292	if (!map->mmaped || map->libbpf_type == LIBBPF_MAP_KCONFIG)10293		return libbpf_err(-EINVAL);10294 10295	if (map->def.type == BPF_MAP_TYPE_ARENA)10296		actual_sz = map->obj->arena_data_sz;10297	else10298		actual_sz = map->def.value_size;10299	if (size != actual_sz)10300		return libbpf_err(-EINVAL);10301 10302	memcpy(map->mmaped, data, size);10303	return 0;10304}10305 10306void *bpf_map__initial_value(const struct bpf_map *map, size_t *psize)10307{10308	if (bpf_map__is_struct_ops(map)) {10309		if (psize)10310			*psize = map->def.value_size;10311		return map->st_ops->data;10312	}10313 10314	if (!map->mmaped)10315		return NULL;10316 10317	if (map->def.type == BPF_MAP_TYPE_ARENA)10318		*psize = map->obj->arena_data_sz;10319	else10320		*psize = map->def.value_size;10321 10322	return map->mmaped;10323}10324 10325bool bpf_map__is_internal(const struct bpf_map *map)10326{10327	return map->libbpf_type != LIBBPF_MAP_UNSPEC;10328}10329 10330__u32 bpf_map__ifindex(const struct bpf_map *map)10331{10332	return map->map_ifindex;10333}10334 10335int bpf_map__set_ifindex(struct bpf_map *map, __u32 ifindex)10336{10337	if (map_is_created(map))10338		return libbpf_err(-EBUSY);10339	map->map_ifindex = ifindex;10340	return 0;10341}10342 10343int bpf_map__set_inner_map_fd(struct bpf_map *map, int fd)10344{10345	if (!bpf_map_type__is_map_in_map(map->def.type)) {10346		pr_warn("error: unsupported map type\n");10347		return libbpf_err(-EINVAL);10348	}10349	if (map->inner_map_fd != -1) {10350		pr_warn("error: inner_map_fd already specified\n");10351		return libbpf_err(-EINVAL);10352	}10353	if (map->inner_map) {10354		bpf_map__destroy(map->inner_map);10355		zfree(&map->inner_map);10356	}10357	map->inner_map_fd = fd;10358	return 0;10359}10360 10361static struct bpf_map *10362__bpf_map__iter(const struct bpf_map *m, const struct bpf_object *obj, int i)10363{10364	ssize_t idx;10365	struct bpf_map *s, *e;10366 10367	if (!obj || !obj->maps)10368		return errno = EINVAL, NULL;10369 10370	s = obj->maps;10371	e = obj->maps + obj->nr_maps;10372 10373	if ((m < s) || (m >= e)) {10374		pr_warn("error in %s: map handler doesn't belong to object\n",10375			 __func__);10376		return errno = EINVAL, NULL;10377	}10378 10379	idx = (m - obj->maps) + i;10380	if (idx >= obj->nr_maps || idx < 0)10381		return NULL;10382	return &obj->maps[idx];10383}10384 10385struct bpf_map *10386bpf_object__next_map(const struct bpf_object *obj, const struct bpf_map *prev)10387{10388	if (prev == NULL && obj != NULL)10389		return obj->maps;10390 10391	return __bpf_map__iter(prev, obj, 1);10392}10393 10394struct bpf_map *10395bpf_object__prev_map(const struct bpf_object *obj, const struct bpf_map *next)10396{10397	if (next == NULL && obj != NULL) {10398		if (!obj->nr_maps)10399			return NULL;10400		return obj->maps + obj->nr_maps - 1;10401	}10402 10403	return __bpf_map__iter(next, obj, -1);10404}10405 10406struct bpf_map *10407bpf_object__find_map_by_name(const struct bpf_object *obj, const char *name)10408{10409	struct bpf_map *pos;10410 10411	bpf_object__for_each_map(pos, obj) {10412		/* if it's a special internal map name (which always starts10413		 * with dot) then check if that special name matches the10414		 * real map name (ELF section name)10415		 */10416		if (name[0] == '.') {10417			if (pos->real_name && strcmp(pos->real_name, name) == 0)10418				return pos;10419			continue;10420		}10421		/* otherwise map name has to be an exact match */10422		if (map_uses_real_name(pos)) {10423			if (strcmp(pos->real_name, name) == 0)10424				return pos;10425			continue;10426		}10427		if (strcmp(pos->name, name) == 0)10428			return pos;10429	}10430	return errno = ENOENT, NULL;10431}10432 10433int10434bpf_object__find_map_fd_by_name(const struct bpf_object *obj, const char *name)10435{10436	return bpf_map__fd(bpf_object__find_map_by_name(obj, name));10437}10438 10439static int validate_map_op(const struct bpf_map *map, size_t key_sz,10440			   size_t value_sz, bool check_value_sz)10441{10442	if (!map_is_created(map)) /* map is not yet created */10443		return -ENOENT;10444 10445	if (map->def.key_size != key_sz) {10446		pr_warn("map '%s': unexpected key size %zu provided, expected %u\n",10447			map->name, key_sz, map->def.key_size);10448		return -EINVAL;10449	}10450 10451	if (map->fd < 0) {10452		pr_warn("map '%s': can't use BPF map without FD (was it created?)\n", map->name);10453		return -EINVAL;10454	}10455 10456	if (!check_value_sz)10457		return 0;10458 10459	switch (map->def.type) {10460	case BPF_MAP_TYPE_PERCPU_ARRAY:10461	case BPF_MAP_TYPE_PERCPU_HASH:10462	case BPF_MAP_TYPE_LRU_PERCPU_HASH:10463	case BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE: {10464		int num_cpu = libbpf_num_possible_cpus();10465		size_t elem_sz = roundup(map->def.value_size, 8);10466 10467		if (value_sz != num_cpu * elem_sz) {10468			pr_warn("map '%s': unexpected value size %zu provided for per-CPU map, expected %d * %zu = %zd\n",10469				map->name, value_sz, num_cpu, elem_sz, num_cpu * elem_sz);10470			return -EINVAL;10471		}10472		break;10473	}10474	default:10475		if (map->def.value_size != value_sz) {10476			pr_warn("map '%s': unexpected value size %zu provided, expected %u\n",10477				map->name, value_sz, map->def.value_size);10478			return -EINVAL;10479		}10480		break;10481	}10482	return 0;10483}10484 10485int bpf_map__lookup_elem(const struct bpf_map *map,10486			 const void *key, size_t key_sz,10487			 void *value, size_t value_sz, __u64 flags)10488{10489	int err;10490 10491	err = validate_map_op(map, key_sz, value_sz, true);10492	if (err)10493		return libbpf_err(err);10494 10495	return bpf_map_lookup_elem_flags(map->fd, key, value, flags);10496}10497 10498int bpf_map__update_elem(const struct bpf_map *map,10499			 const void *key, size_t key_sz,10500			 const void *value, size_t value_sz, __u64 flags)10501{10502	int err;10503 10504	err = validate_map_op(map, key_sz, value_sz, true);10505	if (err)10506		return libbpf_err(err);10507 10508	return bpf_map_update_elem(map->fd, key, value, flags);10509}10510 10511int bpf_map__delete_elem(const struct bpf_map *map,10512			 const void *key, size_t key_sz, __u64 flags)10513{10514	int err;10515 10516	err = validate_map_op(map, key_sz, 0, false /* check_value_sz */);10517	if (err)10518		return libbpf_err(err);10519 10520	return bpf_map_delete_elem_flags(map->fd, key, flags);10521}10522 10523int bpf_map__lookup_and_delete_elem(const struct bpf_map *map,10524				    const void *key, size_t key_sz,10525				    void *value, size_t value_sz, __u64 flags)10526{10527	int err;10528 10529	err = validate_map_op(map, key_sz, value_sz, true);10530	if (err)10531		return libbpf_err(err);10532 10533	return bpf_map_lookup_and_delete_elem_flags(map->fd, key, value, flags);10534}10535 10536int bpf_map__get_next_key(const struct bpf_map *map,10537			  const void *cur_key, void *next_key, size_t key_sz)10538{10539	int err;10540 10541	err = validate_map_op(map, key_sz, 0, false /* check_value_sz */);10542	if (err)10543		return libbpf_err(err);10544 10545	return bpf_map_get_next_key(map->fd, cur_key, next_key);10546}10547 10548long libbpf_get_error(const void *ptr)10549{10550	if (!IS_ERR_OR_NULL(ptr))10551		return 0;10552 10553	if (IS_ERR(ptr))10554		errno = -PTR_ERR(ptr);10555 10556	/* If ptr == NULL, then errno should be already set by the failing10557	 * API, because libbpf never returns NULL on success and it now always10558	 * sets errno on error. So no extra errno handling for ptr == NULL10559	 * case.10560	 */10561	return -errno;10562}10563 10564/* Replace link's underlying BPF program with the new one */10565int bpf_link__update_program(struct bpf_link *link, struct bpf_program *prog)10566{10567	int ret;10568	int prog_fd = bpf_program__fd(prog);10569 10570	if (prog_fd < 0) {10571		pr_warn("prog '%s': can't use BPF program without FD (was it loaded?)\n",10572			prog->name);10573		return libbpf_err(-EINVAL);10574	}10575 10576	ret = bpf_link_update(bpf_link__fd(link), prog_fd, NULL);10577	return libbpf_err_errno(ret);10578}10579 10580/* Release "ownership" of underlying BPF resource (typically, BPF program10581 * attached to some BPF hook, e.g., tracepoint, kprobe, etc). Disconnected10582 * link, when destructed through bpf_link__destroy() call won't attempt to10583 * detach/unregisted that BPF resource. This is useful in situations where,10584 * say, attached BPF program has to outlive userspace program that attached it10585 * in the system. Depending on type of BPF program, though, there might be10586 * additional steps (like pinning BPF program in BPF FS) necessary to ensure10587 * exit of userspace program doesn't trigger automatic detachment and clean up10588 * inside the kernel.10589 */10590void bpf_link__disconnect(struct bpf_link *link)10591{10592	link->disconnected = true;10593}10594 10595int bpf_link__destroy(struct bpf_link *link)10596{10597	int err = 0;10598 10599	if (IS_ERR_OR_NULL(link))10600		return 0;10601 10602	if (!link->disconnected && link->detach)10603		err = link->detach(link);10604	if (link->pin_path)10605		free(link->pin_path);10606	if (link->dealloc)10607		link->dealloc(link);10608	else10609		free(link);10610 10611	return libbpf_err(err);10612}10613 10614int bpf_link__fd(const struct bpf_link *link)10615{10616	return link->fd;10617}10618 10619const char *bpf_link__pin_path(const struct bpf_link *link)10620{10621	return link->pin_path;10622}10623 10624static int bpf_link__detach_fd(struct bpf_link *link)10625{10626	return libbpf_err_errno(close(link->fd));10627}10628 10629struct bpf_link *bpf_link__open(const char *path)10630{10631	struct bpf_link *link;10632	int fd;10633 10634	fd = bpf_obj_get(path);10635	if (fd < 0) {10636		fd = -errno;10637		pr_warn("failed to open link at %s: %d\n", path, fd);10638		return libbpf_err_ptr(fd);10639	}10640 10641	link = calloc(1, sizeof(*link));10642	if (!link) {10643		close(fd);10644		return libbpf_err_ptr(-ENOMEM);10645	}10646	link->detach = &bpf_link__detach_fd;10647	link->fd = fd;10648 10649	link->pin_path = strdup(path);10650	if (!link->pin_path) {10651		bpf_link__destroy(link);10652		return libbpf_err_ptr(-ENOMEM);10653	}10654 10655	return link;10656}10657 10658int bpf_link__detach(struct bpf_link *link)10659{10660	return bpf_link_detach(link->fd) ? -errno : 0;10661}10662 10663int bpf_link__pin(struct bpf_link *link, const char *path)10664{10665	int err;10666 10667	if (link->pin_path)10668		return libbpf_err(-EBUSY);10669	err = make_parent_dir(path);10670	if (err)10671		return libbpf_err(err);10672	err = check_path(path);10673	if (err)10674		return libbpf_err(err);10675 10676	link->pin_path = strdup(path);10677	if (!link->pin_path)10678		return libbpf_err(-ENOMEM);10679 10680	if (bpf_obj_pin(link->fd, link->pin_path)) {10681		err = -errno;10682		zfree(&link->pin_path);10683		return libbpf_err(err);10684	}10685 10686	pr_debug("link fd=%d: pinned at %s\n", link->fd, link->pin_path);10687	return 0;10688}10689 10690int bpf_link__unpin(struct bpf_link *link)10691{10692	int err;10693 10694	if (!link->pin_path)10695		return libbpf_err(-EINVAL);10696 10697	err = unlink(link->pin_path);10698	if (err != 0)10699		return -errno;10700 10701	pr_debug("link fd=%d: unpinned from %s\n", link->fd, link->pin_path);10702	zfree(&link->pin_path);10703	return 0;10704}10705 10706struct bpf_link_perf {10707	struct bpf_link link;10708	int perf_event_fd;10709	/* legacy kprobe support: keep track of probe identifier and type */10710	char *legacy_probe_name;10711	bool legacy_is_kprobe;10712	bool legacy_is_retprobe;10713};10714 10715static int remove_kprobe_event_legacy(const char *probe_name, bool retprobe);10716static int remove_uprobe_event_legacy(const char *probe_name, bool retprobe);10717 10718static int bpf_link_perf_detach(struct bpf_link *link)10719{10720	struct bpf_link_perf *perf_link = container_of(link, struct bpf_link_perf, link);10721	int err = 0;10722 10723	if (ioctl(perf_link->perf_event_fd, PERF_EVENT_IOC_DISABLE, 0) < 0)10724		err = -errno;10725 10726	if (perf_link->perf_event_fd != link->fd)10727		close(perf_link->perf_event_fd);10728	close(link->fd);10729 10730	/* legacy uprobe/kprobe needs to be removed after perf event fd closure */10731	if (perf_link->legacy_probe_name) {10732		if (perf_link->legacy_is_kprobe) {10733			err = remove_kprobe_event_legacy(perf_link->legacy_probe_name,10734							 perf_link->legacy_is_retprobe);10735		} else {10736			err = remove_uprobe_event_legacy(perf_link->legacy_probe_name,10737							 perf_link->legacy_is_retprobe);10738		}10739	}10740 10741	return err;10742}10743 10744static void bpf_link_perf_dealloc(struct bpf_link *link)10745{10746	struct bpf_link_perf *perf_link = container_of(link, struct bpf_link_perf, link);10747 10748	free(perf_link->legacy_probe_name);10749	free(perf_link);10750}10751 10752struct bpf_link *bpf_program__attach_perf_event_opts(const struct bpf_program *prog, int pfd,10753						     const struct bpf_perf_event_opts *opts)10754{10755	char errmsg[STRERR_BUFSIZE];10756	struct bpf_link_perf *link;10757	int prog_fd, link_fd = -1, err;10758	bool force_ioctl_attach;10759 10760	if (!OPTS_VALID(opts, bpf_perf_event_opts))10761		return libbpf_err_ptr(-EINVAL);10762 10763	if (pfd < 0) {10764		pr_warn("prog '%s': invalid perf event FD %d\n",10765			prog->name, pfd);10766		return libbpf_err_ptr(-EINVAL);10767	}10768	prog_fd = bpf_program__fd(prog);10769	if (prog_fd < 0) {10770		pr_warn("prog '%s': can't attach BPF program without FD (was it loaded?)\n",10771			prog->name);10772		return libbpf_err_ptr(-EINVAL);10773	}10774 10775	link = calloc(1, sizeof(*link));10776	if (!link)10777		return libbpf_err_ptr(-ENOMEM);10778	link->link.detach = &bpf_link_perf_detach;10779	link->link.dealloc = &bpf_link_perf_dealloc;10780	link->perf_event_fd = pfd;10781 10782	force_ioctl_attach = OPTS_GET(opts, force_ioctl_attach, false);10783	if (kernel_supports(prog->obj, FEAT_PERF_LINK) && !force_ioctl_attach) {10784		DECLARE_LIBBPF_OPTS(bpf_link_create_opts, link_opts,10785			.perf_event.bpf_cookie = OPTS_GET(opts, bpf_cookie, 0));10786 10787		link_fd = bpf_link_create(prog_fd, pfd, BPF_PERF_EVENT, &link_opts);10788		if (link_fd < 0) {10789			err = -errno;10790			pr_warn("prog '%s': failed to create BPF link for perf_event FD %d: %d (%s)\n",10791				prog->name, pfd,10792				err, libbpf_strerror_r(err, errmsg, sizeof(errmsg)));10793			goto err_out;10794		}10795		link->link.fd = link_fd;10796	} else {10797		if (OPTS_GET(opts, bpf_cookie, 0)) {10798			pr_warn("prog '%s': user context value is not supported\n", prog->name);10799			err = -EOPNOTSUPP;10800			goto err_out;10801		}10802 10803		if (ioctl(pfd, PERF_EVENT_IOC_SET_BPF, prog_fd) < 0) {10804			err = -errno;10805			pr_warn("prog '%s': failed to attach to perf_event FD %d: %s\n",10806				prog->name, pfd, libbpf_strerror_r(err, errmsg, sizeof(errmsg)));10807			if (err == -EPROTO)10808				pr_warn("prog '%s': try add PERF_SAMPLE_CALLCHAIN to or remove exclude_callchain_[kernel|user] from pfd %d\n",10809					prog->name, pfd);10810			goto err_out;10811		}10812		link->link.fd = pfd;10813	}10814	if (ioctl(pfd, PERF_EVENT_IOC_ENABLE, 0) < 0) {10815		err = -errno;10816		pr_warn("prog '%s': failed to enable perf_event FD %d: %s\n",10817			prog->name, pfd, libbpf_strerror_r(err, errmsg, sizeof(errmsg)));10818		goto err_out;10819	}10820 10821	return &link->link;10822err_out:10823	if (link_fd >= 0)10824		close(link_fd);10825	free(link);10826	return libbpf_err_ptr(err);10827}10828 10829struct bpf_link *bpf_program__attach_perf_event(const struct bpf_program *prog, int pfd)10830{10831	return bpf_program__attach_perf_event_opts(prog, pfd, NULL);10832}10833 10834/*10835 * this function is expected to parse integer in the range of [0, 2^31-1] from10836 * given file using scanf format string fmt. If actual parsed value is10837 * negative, the result might be indistinguishable from error10838 */10839static int parse_uint_from_file(const char *file, const char *fmt)10840{10841	char buf[STRERR_BUFSIZE];10842	int err, ret;10843	FILE *f;10844 10845	f = fopen(file, "re");10846	if (!f) {10847		err = -errno;10848		pr_debug("failed to open '%s': %s\n", file,10849			 libbpf_strerror_r(err, buf, sizeof(buf)));10850		return err;10851	}10852	err = fscanf(f, fmt, &ret);10853	if (err != 1) {10854		err = err == EOF ? -EIO : -errno;10855		pr_debug("failed to parse '%s': %s\n", file,10856			libbpf_strerror_r(err, buf, sizeof(buf)));10857		fclose(f);10858		return err;10859	}10860	fclose(f);10861	return ret;10862}10863 10864static int determine_kprobe_perf_type(void)10865{10866	const char *file = "/sys/bus/event_source/devices/kprobe/type";10867 10868	return parse_uint_from_file(file, "%d\n");10869}10870 10871static int determine_uprobe_perf_type(void)10872{10873	const char *file = "/sys/bus/event_source/devices/uprobe/type";10874 10875	return parse_uint_from_file(file, "%d\n");10876}10877 10878static int determine_kprobe_retprobe_bit(void)10879{10880	const char *file = "/sys/bus/event_source/devices/kprobe/format/retprobe";10881 10882	return parse_uint_from_file(file, "config:%d\n");10883}10884 10885static int determine_uprobe_retprobe_bit(void)10886{10887	const char *file = "/sys/bus/event_source/devices/uprobe/format/retprobe";10888 10889	return parse_uint_from_file(file, "config:%d\n");10890}10891 10892#define PERF_UPROBE_REF_CTR_OFFSET_BITS 3210893#define PERF_UPROBE_REF_CTR_OFFSET_SHIFT 3210894 10895static int perf_event_open_probe(bool uprobe, bool retprobe, const char *name,10896				 uint64_t offset, int pid, size_t ref_ctr_off)10897{10898	const size_t attr_sz = sizeof(struct perf_event_attr);10899	struct perf_event_attr attr;10900	char errmsg[STRERR_BUFSIZE];10901	int type, pfd;10902 10903	if ((__u64)ref_ctr_off >= (1ULL << PERF_UPROBE_REF_CTR_OFFSET_BITS))10904		return -EINVAL;10905 10906	memset(&attr, 0, attr_sz);10907 10908	type = uprobe ? determine_uprobe_perf_type()10909		      : determine_kprobe_perf_type();10910	if (type < 0) {10911		pr_warn("failed to determine %s perf type: %s\n",10912			uprobe ? "uprobe" : "kprobe",10913			libbpf_strerror_r(type, errmsg, sizeof(errmsg)));10914		return type;10915	}10916	if (retprobe) {10917		int bit = uprobe ? determine_uprobe_retprobe_bit()10918				 : determine_kprobe_retprobe_bit();10919 10920		if (bit < 0) {10921			pr_warn("failed to determine %s retprobe bit: %s\n",10922				uprobe ? "uprobe" : "kprobe",10923				libbpf_strerror_r(bit, errmsg, sizeof(errmsg)));10924			return bit;10925		}10926		attr.config |= 1 << bit;10927	}10928	attr.size = attr_sz;10929	attr.type = type;10930	attr.config |= (__u64)ref_ctr_off << PERF_UPROBE_REF_CTR_OFFSET_SHIFT;10931	attr.config1 = ptr_to_u64(name); /* kprobe_func or uprobe_path */10932	attr.config2 = offset;		 /* kprobe_addr or probe_offset */10933 10934	/* pid filter is meaningful only for uprobes */10935	pfd = syscall(__NR_perf_event_open, &attr,10936		      pid < 0 ? -1 : pid /* pid */,10937		      pid == -1 ? 0 : -1 /* cpu */,10938		      -1 /* group_fd */, PERF_FLAG_FD_CLOEXEC);10939	return pfd >= 0 ? pfd : -errno;10940}10941 10942static int append_to_file(const char *file, const char *fmt, ...)10943{10944	int fd, n, err = 0;10945	va_list ap;10946	char buf[1024];10947 10948	va_start(ap, fmt);10949	n = vsnprintf(buf, sizeof(buf), fmt, ap);10950	va_end(ap);10951 10952	if (n < 0 || n >= sizeof(buf))10953		return -EINVAL;10954 10955	fd = open(file, O_WRONLY | O_APPEND | O_CLOEXEC, 0);10956	if (fd < 0)10957		return -errno;10958 10959	if (write(fd, buf, n) < 0)10960		err = -errno;10961 10962	close(fd);10963	return err;10964}10965 10966#define DEBUGFS "/sys/kernel/debug/tracing"10967#define TRACEFS "/sys/kernel/tracing"10968 10969static bool use_debugfs(void)10970{10971	static int has_debugfs = -1;10972 10973	if (has_debugfs < 0)10974		has_debugfs = faccessat(AT_FDCWD, DEBUGFS, F_OK, AT_EACCESS) == 0;10975 10976	return has_debugfs == 1;10977}10978 10979static const char *tracefs_path(void)10980{10981	return use_debugfs() ? DEBUGFS : TRACEFS;10982}10983 10984static const char *tracefs_kprobe_events(void)10985{10986	return use_debugfs() ? DEBUGFS"/kprobe_events" : TRACEFS"/kprobe_events";10987}10988 10989static const char *tracefs_uprobe_events(void)10990{10991	return use_debugfs() ? DEBUGFS"/uprobe_events" : TRACEFS"/uprobe_events";10992}10993 10994static const char *tracefs_available_filter_functions(void)10995{10996	return use_debugfs() ? DEBUGFS"/available_filter_functions"10997			     : TRACEFS"/available_filter_functions";10998}10999 11000static const char *tracefs_available_filter_functions_addrs(void)11001{11002	return use_debugfs() ? DEBUGFS"/available_filter_functions_addrs"11003			     : TRACEFS"/available_filter_functions_addrs";11004}11005 11006static void gen_kprobe_legacy_event_name(char *buf, size_t buf_sz,11007					 const char *kfunc_name, size_t offset)11008{11009	static int index = 0;11010	int i;11011 11012	snprintf(buf, buf_sz, "libbpf_%u_%s_0x%zx_%d", getpid(), kfunc_name, offset,11013		 __sync_fetch_and_add(&index, 1));11014 11015	/* sanitize binary_path in the probe name */11016	for (i = 0; buf[i]; i++) {11017		if (!isalnum(buf[i]))11018			buf[i] = '_';11019	}11020}11021 11022static int add_kprobe_event_legacy(const char *probe_name, bool retprobe,11023				   const char *kfunc_name, size_t offset)11024{11025	return append_to_file(tracefs_kprobe_events(), "%c:%s/%s %s+0x%zx",11026			      retprobe ? 'r' : 'p',11027			      retprobe ? "kretprobes" : "kprobes",11028			      probe_name, kfunc_name, offset);11029}11030 11031static int remove_kprobe_event_legacy(const char *probe_name, bool retprobe)11032{11033	return append_to_file(tracefs_kprobe_events(), "-:%s/%s",11034			      retprobe ? "kretprobes" : "kprobes", probe_name);11035}11036 11037static int determine_kprobe_perf_type_legacy(const char *probe_name, bool retprobe)11038{11039	char file[256];11040 11041	snprintf(file, sizeof(file), "%s/events/%s/%s/id",11042		 tracefs_path(), retprobe ? "kretprobes" : "kprobes", probe_name);11043 11044	return parse_uint_from_file(file, "%d\n");11045}11046 11047static int perf_event_kprobe_open_legacy(const char *probe_name, bool retprobe,11048					 const char *kfunc_name, size_t offset, int pid)11049{11050	const size_t attr_sz = sizeof(struct perf_event_attr);11051	struct perf_event_attr attr;11052	char errmsg[STRERR_BUFSIZE];11053	int type, pfd, err;11054 11055	err = add_kprobe_event_legacy(probe_name, retprobe, kfunc_name, offset);11056	if (err < 0) {11057		pr_warn("failed to add legacy kprobe event for '%s+0x%zx': %s\n",11058			kfunc_name, offset,11059			libbpf_strerror_r(err, errmsg, sizeof(errmsg)));11060		return err;11061	}11062	type = determine_kprobe_perf_type_legacy(probe_name, retprobe);11063	if (type < 0) {11064		err = type;11065		pr_warn("failed to determine legacy kprobe event id for '%s+0x%zx': %s\n",11066			kfunc_name, offset,11067			libbpf_strerror_r(err, errmsg, sizeof(errmsg)));11068		goto err_clean_legacy;11069	}11070 11071	memset(&attr, 0, attr_sz);11072	attr.size = attr_sz;11073	attr.config = type;11074	attr.type = PERF_TYPE_TRACEPOINT;11075 11076	pfd = syscall(__NR_perf_event_open, &attr,11077		      pid < 0 ? -1 : pid, /* pid */11078		      pid == -1 ? 0 : -1, /* cpu */11079		      -1 /* group_fd */,  PERF_FLAG_FD_CLOEXEC);11080	if (pfd < 0) {11081		err = -errno;11082		pr_warn("legacy kprobe perf_event_open() failed: %s\n",11083			libbpf_strerror_r(err, errmsg, sizeof(errmsg)));11084		goto err_clean_legacy;11085	}11086	return pfd;11087 11088err_clean_legacy:11089	/* Clear the newly added legacy kprobe_event */11090	remove_kprobe_event_legacy(probe_name, retprobe);11091	return err;11092}11093 11094static const char *arch_specific_syscall_pfx(void)11095{11096#if defined(__x86_64__)11097	return "x64";11098#elif defined(__i386__)11099	return "ia32";11100#elif defined(__s390x__)11101	return "s390x";11102#elif defined(__s390__)11103	return "s390";11104#elif defined(__arm__)11105	return "arm";11106#elif defined(__aarch64__)11107	return "arm64";11108#elif defined(__mips__)11109	return "mips";11110#elif defined(__riscv)11111	return "riscv";11112#elif defined(__powerpc__)11113	return "powerpc";11114#elif defined(__powerpc64__)11115	return "powerpc64";11116#else11117	return NULL;11118#endif11119}11120 11121int probe_kern_syscall_wrapper(int token_fd)11122{11123	char syscall_name[64];11124	const char *ksys_pfx;11125 11126	ksys_pfx = arch_specific_syscall_pfx();11127	if (!ksys_pfx)11128		return 0;11129 11130	snprintf(syscall_name, sizeof(syscall_name), "__%s_sys_bpf", ksys_pfx);11131 11132	if (determine_kprobe_perf_type() >= 0) {11133		int pfd;11134 11135		pfd = perf_event_open_probe(false, false, syscall_name, 0, getpid(), 0);11136		if (pfd >= 0)11137			close(pfd);11138 11139		return pfd >= 0 ? 1 : 0;11140	} else { /* legacy mode */11141		char probe_name[128];11142 11143		gen_kprobe_legacy_event_name(probe_name, sizeof(probe_name), syscall_name, 0);11144		if (add_kprobe_event_legacy(probe_name, false, syscall_name, 0) < 0)11145			return 0;11146 11147		(void)remove_kprobe_event_legacy(probe_name, false);11148		return 1;11149	}11150}11151 11152struct bpf_link *11153bpf_program__attach_kprobe_opts(const struct bpf_program *prog,11154				const char *func_name,11155				const struct bpf_kprobe_opts *opts)11156{11157	DECLARE_LIBBPF_OPTS(bpf_perf_event_opts, pe_opts);11158	enum probe_attach_mode attach_mode;11159	char errmsg[STRERR_BUFSIZE];11160	char *legacy_probe = NULL;11161	struct bpf_link *link;11162	size_t offset;11163	bool retprobe, legacy;11164	int pfd, err;11165 11166	if (!OPTS_VALID(opts, bpf_kprobe_opts))11167		return libbpf_err_ptr(-EINVAL);11168 11169	attach_mode = OPTS_GET(opts, attach_mode, PROBE_ATTACH_MODE_DEFAULT);11170	retprobe = OPTS_GET(opts, retprobe, false);11171	offset = OPTS_GET(opts, offset, 0);11172	pe_opts.bpf_cookie = OPTS_GET(opts, bpf_cookie, 0);11173 11174	legacy = determine_kprobe_perf_type() < 0;11175	switch (attach_mode) {11176	case PROBE_ATTACH_MODE_LEGACY:11177		legacy = true;11178		pe_opts.force_ioctl_attach = true;11179		break;11180	case PROBE_ATTACH_MODE_PERF:11181		if (legacy)11182			return libbpf_err_ptr(-ENOTSUP);11183		pe_opts.force_ioctl_attach = true;11184		break;11185	case PROBE_ATTACH_MODE_LINK:11186		if (legacy || !kernel_supports(prog->obj, FEAT_PERF_LINK))11187			return libbpf_err_ptr(-ENOTSUP);11188		break;11189	case PROBE_ATTACH_MODE_DEFAULT:11190		break;11191	default:11192		return libbpf_err_ptr(-EINVAL);11193	}11194 11195	if (!legacy) {11196		pfd = perf_event_open_probe(false /* uprobe */, retprobe,11197					    func_name, offset,11198					    -1 /* pid */, 0 /* ref_ctr_off */);11199	} else {11200		char probe_name[256];11201 11202		gen_kprobe_legacy_event_name(probe_name, sizeof(probe_name),11203					     func_name, offset);11204 11205		legacy_probe = strdup(probe_name);11206		if (!legacy_probe)11207			return libbpf_err_ptr(-ENOMEM);11208 11209		pfd = perf_event_kprobe_open_legacy(legacy_probe, retprobe, func_name,11210						    offset, -1 /* pid */);11211	}11212	if (pfd < 0) {11213		err = -errno;11214		pr_warn("prog '%s': failed to create %s '%s+0x%zx' perf event: %s\n",11215			prog->name, retprobe ? "kretprobe" : "kprobe",11216			func_name, offset,11217			libbpf_strerror_r(err, errmsg, sizeof(errmsg)));11218		goto err_out;11219	}11220	link = bpf_program__attach_perf_event_opts(prog, pfd, &pe_opts);11221	err = libbpf_get_error(link);11222	if (err) {11223		close(pfd);11224		pr_warn("prog '%s': failed to attach to %s '%s+0x%zx': %s\n",11225			prog->name, retprobe ? "kretprobe" : "kprobe",11226			func_name, offset,11227			libbpf_strerror_r(err, errmsg, sizeof(errmsg)));11228		goto err_clean_legacy;11229	}11230	if (legacy) {11231		struct bpf_link_perf *perf_link = container_of(link, struct bpf_link_perf, link);11232 11233		perf_link->legacy_probe_name = legacy_probe;11234		perf_link->legacy_is_kprobe = true;11235		perf_link->legacy_is_retprobe = retprobe;11236	}11237 11238	return link;11239 11240err_clean_legacy:11241	if (legacy)11242		remove_kprobe_event_legacy(legacy_probe, retprobe);11243err_out:11244	free(legacy_probe);11245	return libbpf_err_ptr(err);11246}11247 11248struct bpf_link *bpf_program__attach_kprobe(const struct bpf_program *prog,11249					    bool retprobe,11250					    const char *func_name)11251{11252	DECLARE_LIBBPF_OPTS(bpf_kprobe_opts, opts,11253		.retprobe = retprobe,11254	);11255 11256	return bpf_program__attach_kprobe_opts(prog, func_name, &opts);11257}11258 11259struct bpf_link *bpf_program__attach_ksyscall(const struct bpf_program *prog,11260					      const char *syscall_name,11261					      const struct bpf_ksyscall_opts *opts)11262{11263	LIBBPF_OPTS(bpf_kprobe_opts, kprobe_opts);11264	char func_name[128];11265 11266	if (!OPTS_VALID(opts, bpf_ksyscall_opts))11267		return libbpf_err_ptr(-EINVAL);11268 11269	if (kernel_supports(prog->obj, FEAT_SYSCALL_WRAPPER)) {11270		/* arch_specific_syscall_pfx() should never return NULL here11271		 * because it is guarded by kernel_supports(). However, since11272		 * compiler does not know that we have an explicit conditional11273		 * as well.11274		 */11275		snprintf(func_name, sizeof(func_name), "__%s_sys_%s",11276			 arch_specific_syscall_pfx() ? : "", syscall_name);11277	} else {11278		snprintf(func_name, sizeof(func_name), "__se_sys_%s", syscall_name);11279	}11280 11281	kprobe_opts.retprobe = OPTS_GET(opts, retprobe, false);11282	kprobe_opts.bpf_cookie = OPTS_GET(opts, bpf_cookie, 0);11283 11284	return bpf_program__attach_kprobe_opts(prog, func_name, &kprobe_opts);11285}11286 11287/* Adapted from perf/util/string.c */11288bool glob_match(const char *str, const char *pat)11289{11290	while (*str && *pat && *pat != '*') {11291		if (*pat == '?') {      /* Matches any single character */11292			str++;11293			pat++;11294			continue;11295		}11296		if (*str != *pat)11297			return false;11298		str++;11299		pat++;11300	}11301	/* Check wild card */11302	if (*pat == '*') {11303		while (*pat == '*')11304			pat++;11305		if (!*pat) /* Tail wild card matches all */11306			return true;11307		while (*str)11308			if (glob_match(str++, pat))11309				return true;11310	}11311	return !*str && !*pat;11312}11313 11314struct kprobe_multi_resolve {11315	const char *pattern;11316	unsigned long *addrs;11317	size_t cap;11318	size_t cnt;11319};11320 11321struct avail_kallsyms_data {11322	char **syms;11323	size_t cnt;11324	struct kprobe_multi_resolve *res;11325};11326 11327static int avail_func_cmp(const void *a, const void *b)11328{11329	return strcmp(*(const char **)a, *(const char **)b);11330}11331 11332static int avail_kallsyms_cb(unsigned long long sym_addr, char sym_type,11333			     const char *sym_name, void *ctx)11334{11335	struct avail_kallsyms_data *data = ctx;11336	struct kprobe_multi_resolve *res = data->res;11337	int err;11338 11339	if (!bsearch(&sym_name, data->syms, data->cnt, sizeof(*data->syms), avail_func_cmp))11340		return 0;11341 11342	err = libbpf_ensure_mem((void **)&res->addrs, &res->cap, sizeof(*res->addrs), res->cnt + 1);11343	if (err)11344		return err;11345 11346	res->addrs[res->cnt++] = (unsigned long)sym_addr;11347	return 0;11348}11349 11350static int libbpf_available_kallsyms_parse(struct kprobe_multi_resolve *res)11351{11352	const char *available_functions_file = tracefs_available_filter_functions();11353	struct avail_kallsyms_data data;11354	char sym_name[500];11355	FILE *f;11356	int err = 0, ret, i;11357	char **syms = NULL;11358	size_t cap = 0, cnt = 0;11359 11360	f = fopen(available_functions_file, "re");11361	if (!f) {11362		err = -errno;11363		pr_warn("failed to open %s: %d\n", available_functions_file, err);11364		return err;11365	}11366 11367	while (true) {11368		char *name;11369 11370		ret = fscanf(f, "%499s%*[^\n]\n", sym_name);11371		if (ret == EOF && feof(f))11372			break;11373 11374		if (ret != 1) {11375			pr_warn("failed to parse available_filter_functions entry: %d\n", ret);11376			err = -EINVAL;11377			goto cleanup;11378		}11379 11380		if (!glob_match(sym_name, res->pattern))11381			continue;11382 11383		err = libbpf_ensure_mem((void **)&syms, &cap, sizeof(*syms), cnt + 1);11384		if (err)11385			goto cleanup;11386 11387		name = strdup(sym_name);11388		if (!name) {11389			err = -errno;11390			goto cleanup;11391		}11392 11393		syms[cnt++] = name;11394	}11395 11396	/* no entries found, bail out */11397	if (cnt == 0) {11398		err = -ENOENT;11399		goto cleanup;11400	}11401 11402	/* sort available functions */11403	qsort(syms, cnt, sizeof(*syms), avail_func_cmp);11404 11405	data.syms = syms;11406	data.res = res;11407	data.cnt = cnt;11408	libbpf_kallsyms_parse(avail_kallsyms_cb, &data);11409 11410	if (res->cnt == 0)11411		err = -ENOENT;11412 11413cleanup:11414	for (i = 0; i < cnt; i++)11415		free((char *)syms[i]);11416	free(syms);11417 11418	fclose(f);11419	return err;11420}11421 11422static bool has_available_filter_functions_addrs(void)11423{11424	return access(tracefs_available_filter_functions_addrs(), R_OK) != -1;11425}11426 11427static int libbpf_available_kprobes_parse(struct kprobe_multi_resolve *res)11428{11429	const char *available_path = tracefs_available_filter_functions_addrs();11430	char sym_name[500];11431	FILE *f;11432	int ret, err = 0;11433	unsigned long long sym_addr;11434 11435	f = fopen(available_path, "re");11436	if (!f) {11437		err = -errno;11438		pr_warn("failed to open %s: %d\n", available_path, err);11439		return err;11440	}11441 11442	while (true) {11443		ret = fscanf(f, "%llx %499s%*[^\n]\n", &sym_addr, sym_name);11444		if (ret == EOF && feof(f))11445			break;11446 11447		if (ret != 2) {11448			pr_warn("failed to parse available_filter_functions_addrs entry: %d\n",11449				ret);11450			err = -EINVAL;11451			goto cleanup;11452		}11453 11454		if (!glob_match(sym_name, res->pattern))11455			continue;11456 11457		err = libbpf_ensure_mem((void **)&res->addrs, &res->cap,11458					sizeof(*res->addrs), res->cnt + 1);11459		if (err)11460			goto cleanup;11461 11462		res->addrs[res->cnt++] = (unsigned long)sym_addr;11463	}11464 11465	if (res->cnt == 0)11466		err = -ENOENT;11467 11468cleanup:11469	fclose(f);11470	return err;11471}11472 11473struct bpf_link *11474bpf_program__attach_kprobe_multi_opts(const struct bpf_program *prog,11475				      const char *pattern,11476				      const struct bpf_kprobe_multi_opts *opts)11477{11478	LIBBPF_OPTS(bpf_link_create_opts, lopts);11479	struct kprobe_multi_resolve res = {11480		.pattern = pattern,11481	};11482	enum bpf_attach_type attach_type;11483	struct bpf_link *link = NULL;11484	char errmsg[STRERR_BUFSIZE];11485	const unsigned long *addrs;11486	int err, link_fd, prog_fd;11487	bool retprobe, session;11488	const __u64 *cookies;11489	const char **syms;11490	size_t cnt;11491 11492	if (!OPTS_VALID(opts, bpf_kprobe_multi_opts))11493		return libbpf_err_ptr(-EINVAL);11494 11495	prog_fd = bpf_program__fd(prog);11496	if (prog_fd < 0) {11497		pr_warn("prog '%s': can't attach BPF program without FD (was it loaded?)\n",11498			prog->name);11499		return libbpf_err_ptr(-EINVAL);11500	}11501 11502	syms    = OPTS_GET(opts, syms, false);11503	addrs   = OPTS_GET(opts, addrs, false);11504	cnt     = OPTS_GET(opts, cnt, false);11505	cookies = OPTS_GET(opts, cookies, false);11506 11507	if (!pattern && !addrs && !syms)11508		return libbpf_err_ptr(-EINVAL);11509	if (pattern && (addrs || syms || cookies || cnt))11510		return libbpf_err_ptr(-EINVAL);11511	if (!pattern && !cnt)11512		return libbpf_err_ptr(-EINVAL);11513	if (addrs && syms)11514		return libbpf_err_ptr(-EINVAL);11515 11516	if (pattern) {11517		if (has_available_filter_functions_addrs())11518			err = libbpf_available_kprobes_parse(&res);11519		else11520			err = libbpf_available_kallsyms_parse(&res);11521		if (err)11522			goto error;11523		addrs = res.addrs;11524		cnt = res.cnt;11525	}11526 11527	retprobe = OPTS_GET(opts, retprobe, false);11528	session  = OPTS_GET(opts, session, false);11529 11530	if (retprobe && session)11531		return libbpf_err_ptr(-EINVAL);11532 11533	attach_type = session ? BPF_TRACE_KPROBE_SESSION : BPF_TRACE_KPROBE_MULTI;11534 11535	lopts.kprobe_multi.syms = syms;11536	lopts.kprobe_multi.addrs = addrs;11537	lopts.kprobe_multi.cookies = cookies;11538	lopts.kprobe_multi.cnt = cnt;11539	lopts.kprobe_multi.flags = retprobe ? BPF_F_KPROBE_MULTI_RETURN : 0;11540 11541	link = calloc(1, sizeof(*link));11542	if (!link) {11543		err = -ENOMEM;11544		goto error;11545	}11546	link->detach = &bpf_link__detach_fd;11547 11548	link_fd = bpf_link_create(prog_fd, 0, attach_type, &lopts);11549	if (link_fd < 0) {11550		err = -errno;11551		pr_warn("prog '%s': failed to attach: %s\n",11552			prog->name, libbpf_strerror_r(err, errmsg, sizeof(errmsg)));11553		goto error;11554	}11555	link->fd = link_fd;11556	free(res.addrs);11557	return link;11558 11559error:11560	free(link);11561	free(res.addrs);11562	return libbpf_err_ptr(err);11563}11564 11565static int attach_kprobe(const struct bpf_program *prog, long cookie, struct bpf_link **link)11566{11567	DECLARE_LIBBPF_OPTS(bpf_kprobe_opts, opts);11568	unsigned long offset = 0;11569	const char *func_name;11570	char *func;11571	int n;11572 11573	*link = NULL;11574 11575	/* no auto-attach for SEC("kprobe") and SEC("kretprobe") */11576	if (strcmp(prog->sec_name, "kprobe") == 0 || strcmp(prog->sec_name, "kretprobe") == 0)11577		return 0;11578 11579	opts.retprobe = str_has_pfx(prog->sec_name, "kretprobe/");11580	if (opts.retprobe)11581		func_name = prog->sec_name + sizeof("kretprobe/") - 1;11582	else11583		func_name = prog->sec_name + sizeof("kprobe/") - 1;11584 11585	n = sscanf(func_name, "%m[a-zA-Z0-9_.]+%li", &func, &offset);11586	if (n < 1) {11587		pr_warn("kprobe name is invalid: %s\n", func_name);11588		return -EINVAL;11589	}11590	if (opts.retprobe && offset != 0) {11591		free(func);11592		pr_warn("kretprobes do not support offset specification\n");11593		return -EINVAL;11594	}11595 11596	opts.offset = offset;11597	*link = bpf_program__attach_kprobe_opts(prog, func, &opts);11598	free(func);11599	return libbpf_get_error(*link);11600}11601 11602static int attach_ksyscall(const struct bpf_program *prog, long cookie, struct bpf_link **link)11603{11604	LIBBPF_OPTS(bpf_ksyscall_opts, opts);11605	const char *syscall_name;11606 11607	*link = NULL;11608 11609	/* no auto-attach for SEC("ksyscall") and SEC("kretsyscall") */11610	if (strcmp(prog->sec_name, "ksyscall") == 0 || strcmp(prog->sec_name, "kretsyscall") == 0)11611		return 0;11612 11613	opts.retprobe = str_has_pfx(prog->sec_name, "kretsyscall/");11614	if (opts.retprobe)11615		syscall_name = prog->sec_name + sizeof("kretsyscall/") - 1;11616	else11617		syscall_name = prog->sec_name + sizeof("ksyscall/") - 1;11618 11619	*link = bpf_program__attach_ksyscall(prog, syscall_name, &opts);11620	return *link ? 0 : -errno;11621}11622 11623static int attach_kprobe_multi(const struct bpf_program *prog, long cookie, struct bpf_link **link)11624{11625	LIBBPF_OPTS(bpf_kprobe_multi_opts, opts);11626	const char *spec;11627	char *pattern;11628	int n;11629 11630	*link = NULL;11631 11632	/* no auto-attach for SEC("kprobe.multi") and SEC("kretprobe.multi") */11633	if (strcmp(prog->sec_name, "kprobe.multi") == 0 ||11634	    strcmp(prog->sec_name, "kretprobe.multi") == 0)11635		return 0;11636 11637	opts.retprobe = str_has_pfx(prog->sec_name, "kretprobe.multi/");11638	if (opts.retprobe)11639		spec = prog->sec_name + sizeof("kretprobe.multi/") - 1;11640	else11641		spec = prog->sec_name + sizeof("kprobe.multi/") - 1;11642 11643	n = sscanf(spec, "%m[a-zA-Z0-9_.*?]", &pattern);11644	if (n < 1) {11645		pr_warn("kprobe multi pattern is invalid: %s\n", spec);11646		return -EINVAL;11647	}11648 11649	*link = bpf_program__attach_kprobe_multi_opts(prog, pattern, &opts);11650	free(pattern);11651	return libbpf_get_error(*link);11652}11653 11654static int attach_kprobe_session(const struct bpf_program *prog, long cookie,11655				 struct bpf_link **link)11656{11657	LIBBPF_OPTS(bpf_kprobe_multi_opts, opts, .session = true);11658	const char *spec;11659	char *pattern;11660	int n;11661 11662	*link = NULL;11663 11664	/* no auto-attach for SEC("kprobe.session") */11665	if (strcmp(prog->sec_name, "kprobe.session") == 0)11666		return 0;11667 11668	spec = prog->sec_name + sizeof("kprobe.session/") - 1;11669	n = sscanf(spec, "%m[a-zA-Z0-9_.*?]", &pattern);11670	if (n < 1) {11671		pr_warn("kprobe session pattern is invalid: %s\n", spec);11672		return -EINVAL;11673	}11674 11675	*link = bpf_program__attach_kprobe_multi_opts(prog, pattern, &opts);11676	free(pattern);11677	return *link ? 0 : -errno;11678}11679 11680static int attach_uprobe_multi(const struct bpf_program *prog, long cookie, struct bpf_link **link)11681{11682	char *probe_type = NULL, *binary_path = NULL, *func_name = NULL;11683	LIBBPF_OPTS(bpf_uprobe_multi_opts, opts);11684	int n, ret = -EINVAL;11685 11686	*link = NULL;11687 11688	n = sscanf(prog->sec_name, "%m[^/]/%m[^:]:%m[^\n]",11689		   &probe_type, &binary_path, &func_name);11690	switch (n) {11691	case 1:11692		/* handle SEC("u[ret]probe") - format is valid, but auto-attach is impossible. */11693		ret = 0;11694		break;11695	case 3:11696		opts.retprobe = str_has_pfx(probe_type, "uretprobe.multi");11697		*link = bpf_program__attach_uprobe_multi(prog, -1, binary_path, func_name, &opts);11698		ret = libbpf_get_error(*link);11699		break;11700	default:11701		pr_warn("prog '%s': invalid format of section definition '%s'\n", prog->name,11702			prog->sec_name);11703		break;11704	}11705	free(probe_type);11706	free(binary_path);11707	free(func_name);11708	return ret;11709}11710 11711static void gen_uprobe_legacy_event_name(char *buf, size_t buf_sz,11712					 const char *binary_path, uint64_t offset)11713{11714	int i;11715 11716	snprintf(buf, buf_sz, "libbpf_%u_%s_0x%zx", getpid(), binary_path, (size_t)offset);11717 11718	/* sanitize binary_path in the probe name */11719	for (i = 0; buf[i]; i++) {11720		if (!isalnum(buf[i]))11721			buf[i] = '_';11722	}11723}11724 11725static inline int add_uprobe_event_legacy(const char *probe_name, bool retprobe,11726					  const char *binary_path, size_t offset)11727{11728	return append_to_file(tracefs_uprobe_events(), "%c:%s/%s %s:0x%zx",11729			      retprobe ? 'r' : 'p',11730			      retprobe ? "uretprobes" : "uprobes",11731			      probe_name, binary_path, offset);11732}11733 11734static inline int remove_uprobe_event_legacy(const char *probe_name, bool retprobe)11735{11736	return append_to_file(tracefs_uprobe_events(), "-:%s/%s",11737			      retprobe ? "uretprobes" : "uprobes", probe_name);11738}11739 11740static int determine_uprobe_perf_type_legacy(const char *probe_name, bool retprobe)11741{11742	char file[512];11743 11744	snprintf(file, sizeof(file), "%s/events/%s/%s/id",11745		 tracefs_path(), retprobe ? "uretprobes" : "uprobes", probe_name);11746 11747	return parse_uint_from_file(file, "%d\n");11748}11749 11750static int perf_event_uprobe_open_legacy(const char *probe_name, bool retprobe,11751					 const char *binary_path, size_t offset, int pid)11752{11753	const size_t attr_sz = sizeof(struct perf_event_attr);11754	struct perf_event_attr attr;11755	int type, pfd, err;11756 11757	err = add_uprobe_event_legacy(probe_name, retprobe, binary_path, offset);11758	if (err < 0) {11759		pr_warn("failed to add legacy uprobe event for %s:0x%zx: %d\n",11760			binary_path, (size_t)offset, err);11761		return err;11762	}11763	type = determine_uprobe_perf_type_legacy(probe_name, retprobe);11764	if (type < 0) {11765		err = type;11766		pr_warn("failed to determine legacy uprobe event id for %s:0x%zx: %d\n",11767			binary_path, offset, err);11768		goto err_clean_legacy;11769	}11770 11771	memset(&attr, 0, attr_sz);11772	attr.size = attr_sz;11773	attr.config = type;11774	attr.type = PERF_TYPE_TRACEPOINT;11775 11776	pfd = syscall(__NR_perf_event_open, &attr,11777		      pid < 0 ? -1 : pid, /* pid */11778		      pid == -1 ? 0 : -1, /* cpu */11779		      -1 /* group_fd */,  PERF_FLAG_FD_CLOEXEC);11780	if (pfd < 0) {11781		err = -errno;11782		pr_warn("legacy uprobe perf_event_open() failed: %d\n", err);11783		goto err_clean_legacy;11784	}11785	return pfd;11786 11787err_clean_legacy:11788	/* Clear the newly added legacy uprobe_event */11789	remove_uprobe_event_legacy(probe_name, retprobe);11790	return err;11791}11792 11793/* Find offset of function name in archive specified by path. Currently11794 * supported are .zip files that do not compress their contents, as used on11795 * Android in the form of APKs, for example. "file_name" is the name of the ELF11796 * file inside the archive. "func_name" matches symbol name or name@@LIB for11797 * library functions.11798 *11799 * An overview of the APK format specifically provided here:11800 * https://en.wikipedia.org/w/index.php?title=Apk_(file_format)&oldid=1139099120#Package_contents11801 */11802static long elf_find_func_offset_from_archive(const char *archive_path, const char *file_name,11803					      const char *func_name)11804{11805	struct zip_archive *archive;11806	struct zip_entry entry;11807	long ret;11808	Elf *elf;11809 11810	archive = zip_archive_open(archive_path);11811	if (IS_ERR(archive)) {11812		ret = PTR_ERR(archive);11813		pr_warn("zip: failed to open %s: %ld\n", archive_path, ret);11814		return ret;11815	}11816 11817	ret = zip_archive_find_entry(archive, file_name, &entry);11818	if (ret) {11819		pr_warn("zip: could not find archive member %s in %s: %ld\n", file_name,11820			archive_path, ret);11821		goto out;11822	}11823	pr_debug("zip: found entry for %s in %s at 0x%lx\n", file_name, archive_path,11824		 (unsigned long)entry.data_offset);11825 11826	if (entry.compression) {11827		pr_warn("zip: entry %s of %s is compressed and cannot be handled\n", file_name,11828			archive_path);11829		ret = -LIBBPF_ERRNO__FORMAT;11830		goto out;11831	}11832 11833	elf = elf_memory((void *)entry.data, entry.data_length);11834	if (!elf) {11835		pr_warn("elf: could not read elf file %s from %s: %s\n", file_name, archive_path,11836			elf_errmsg(-1));11837		ret = -LIBBPF_ERRNO__LIBELF;11838		goto out;11839	}11840 11841	ret = elf_find_func_offset(elf, file_name, func_name);11842	if (ret > 0) {11843		pr_debug("elf: symbol address match for %s of %s in %s: 0x%x + 0x%lx = 0x%lx\n",11844			 func_name, file_name, archive_path, entry.data_offset, ret,11845			 ret + entry.data_offset);11846		ret += entry.data_offset;11847	}11848	elf_end(elf);11849 11850out:11851	zip_archive_close(archive);11852	return ret;11853}11854 11855static const char *arch_specific_lib_paths(void)11856{11857	/*11858	 * Based on https://packages.debian.org/sid/libc6.11859	 *11860	 * Assume that the traced program is built for the same architecture11861	 * as libbpf, which should cover the vast majority of cases.11862	 */11863#if defined(__x86_64__)11864	return "/lib/x86_64-linux-gnu";11865#elif defined(__i386__)11866	return "/lib/i386-linux-gnu";11867#elif defined(__s390x__)11868	return "/lib/s390x-linux-gnu";11869#elif defined(__s390__)11870	return "/lib/s390-linux-gnu";11871#elif defined(__arm__) && defined(__SOFTFP__)11872	return "/lib/arm-linux-gnueabi";11873#elif defined(__arm__) && !defined(__SOFTFP__)11874	return "/lib/arm-linux-gnueabihf";11875#elif defined(__aarch64__)11876	return "/lib/aarch64-linux-gnu";11877#elif defined(__mips__) && defined(__MIPSEL__) && _MIPS_SZLONG == 6411878	return "/lib/mips64el-linux-gnuabi64";11879#elif defined(__mips__) && defined(__MIPSEL__) && _MIPS_SZLONG == 3211880	return "/lib/mipsel-linux-gnu";11881#elif defined(__powerpc64__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__11882	return "/lib/powerpc64le-linux-gnu";11883#elif defined(__sparc__) && defined(__arch64__)11884	return "/lib/sparc64-linux-gnu";11885#elif defined(__riscv) && __riscv_xlen == 6411886	return "/lib/riscv64-linux-gnu";11887#else11888	return NULL;11889#endif11890}11891 11892/* Get full path to program/shared library. */11893static int resolve_full_path(const char *file, char *result, size_t result_sz)11894{11895	const char *search_paths[3] = {};11896	int i, perm;11897 11898	if (str_has_sfx(file, ".so") || strstr(file, ".so.")) {11899		search_paths[0] = getenv("LD_LIBRARY_PATH");11900		search_paths[1] = "/usr/lib64:/usr/lib";11901		search_paths[2] = arch_specific_lib_paths();11902		perm = R_OK;11903	} else {11904		search_paths[0] = getenv("PATH");11905		search_paths[1] = "/usr/bin:/usr/sbin";11906		perm = R_OK | X_OK;11907	}11908 11909	for (i = 0; i < ARRAY_SIZE(search_paths); i++) {11910		const char *s;11911 11912		if (!search_paths[i])11913			continue;11914		for (s = search_paths[i]; s != NULL; s = strchr(s, ':')) {11915			char *next_path;11916			int seg_len;11917 11918			if (s[0] == ':')11919				s++;11920			next_path = strchr(s, ':');11921			seg_len = next_path ? next_path - s : strlen(s);11922			if (!seg_len)11923				continue;11924			snprintf(result, result_sz, "%.*s/%s", seg_len, s, file);11925			/* ensure it has required permissions */11926			if (faccessat(AT_FDCWD, result, perm, AT_EACCESS) < 0)11927				continue;11928			pr_debug("resolved '%s' to '%s'\n", file, result);11929			return 0;11930		}11931	}11932	return -ENOENT;11933}11934 11935struct bpf_link *11936bpf_program__attach_uprobe_multi(const struct bpf_program *prog,11937				 pid_t pid,11938				 const char *path,11939				 const char *func_pattern,11940				 const struct bpf_uprobe_multi_opts *opts)11941{11942	const unsigned long *ref_ctr_offsets = NULL, *offsets = NULL;11943	LIBBPF_OPTS(bpf_link_create_opts, lopts);11944	unsigned long *resolved_offsets = NULL;11945	int err = 0, link_fd, prog_fd;11946	struct bpf_link *link = NULL;11947	char errmsg[STRERR_BUFSIZE];11948	char full_path[PATH_MAX];11949	const __u64 *cookies;11950	const char **syms;11951	size_t cnt;11952 11953	if (!OPTS_VALID(opts, bpf_uprobe_multi_opts))11954		return libbpf_err_ptr(-EINVAL);11955 11956	prog_fd = bpf_program__fd(prog);11957	if (prog_fd < 0) {11958		pr_warn("prog '%s': can't attach BPF program without FD (was it loaded?)\n",11959			prog->name);11960		return libbpf_err_ptr(-EINVAL);11961	}11962 11963	syms = OPTS_GET(opts, syms, NULL);11964	offsets = OPTS_GET(opts, offsets, NULL);11965	ref_ctr_offsets = OPTS_GET(opts, ref_ctr_offsets, NULL);11966	cookies = OPTS_GET(opts, cookies, NULL);11967	cnt = OPTS_GET(opts, cnt, 0);11968 11969	/*11970	 * User can specify 2 mutually exclusive set of inputs:11971	 *11972	 * 1) use only path/func_pattern/pid arguments11973	 *11974	 * 2) use path/pid with allowed combinations of:11975	 *    syms/offsets/ref_ctr_offsets/cookies/cnt11976	 *11977	 *    - syms and offsets are mutually exclusive11978	 *    - ref_ctr_offsets and cookies are optional11979	 *11980	 * Any other usage results in error.11981	 */11982 11983	if (!path)11984		return libbpf_err_ptr(-EINVAL);11985	if (!func_pattern && cnt == 0)11986		return libbpf_err_ptr(-EINVAL);11987 11988	if (func_pattern) {11989		if (syms || offsets || ref_ctr_offsets || cookies || cnt)11990			return libbpf_err_ptr(-EINVAL);11991	} else {11992		if (!!syms == !!offsets)11993			return libbpf_err_ptr(-EINVAL);11994	}11995 11996	if (func_pattern) {11997		if (!strchr(path, '/')) {11998			err = resolve_full_path(path, full_path, sizeof(full_path));11999			if (err) {12000				pr_warn("prog '%s': failed to resolve full path for '%s': %d\n",12001					prog->name, path, err);12002				return libbpf_err_ptr(err);12003			}12004			path = full_path;12005		}12006 12007		err = elf_resolve_pattern_offsets(path, func_pattern,12008						  &resolved_offsets, &cnt);12009		if (err < 0)12010			return libbpf_err_ptr(err);12011		offsets = resolved_offsets;12012	} else if (syms) {12013		err = elf_resolve_syms_offsets(path, cnt, syms, &resolved_offsets, STT_FUNC);12014		if (err < 0)12015			return libbpf_err_ptr(err);12016		offsets = resolved_offsets;12017	}12018 12019	lopts.uprobe_multi.path = path;12020	lopts.uprobe_multi.offsets = offsets;12021	lopts.uprobe_multi.ref_ctr_offsets = ref_ctr_offsets;12022	lopts.uprobe_multi.cookies = cookies;12023	lopts.uprobe_multi.cnt = cnt;12024	lopts.uprobe_multi.flags = OPTS_GET(opts, retprobe, false) ? BPF_F_UPROBE_MULTI_RETURN : 0;12025 12026	if (pid == 0)12027		pid = getpid();12028	if (pid > 0)12029		lopts.uprobe_multi.pid = pid;12030 12031	link = calloc(1, sizeof(*link));12032	if (!link) {12033		err = -ENOMEM;12034		goto error;12035	}12036	link->detach = &bpf_link__detach_fd;12037 12038	link_fd = bpf_link_create(prog_fd, 0, BPF_TRACE_UPROBE_MULTI, &lopts);12039	if (link_fd < 0) {12040		err = -errno;12041		pr_warn("prog '%s': failed to attach multi-uprobe: %s\n",12042			prog->name, libbpf_strerror_r(err, errmsg, sizeof(errmsg)));12043		goto error;12044	}12045	link->fd = link_fd;12046	free(resolved_offsets);12047	return link;12048 12049error:12050	free(resolved_offsets);12051	free(link);12052	return libbpf_err_ptr(err);12053}12054 12055LIBBPF_API struct bpf_link *12056bpf_program__attach_uprobe_opts(const struct bpf_program *prog, pid_t pid,12057				const char *binary_path, size_t func_offset,12058				const struct bpf_uprobe_opts *opts)12059{12060	const char *archive_path = NULL, *archive_sep = NULL;12061	char errmsg[STRERR_BUFSIZE], *legacy_probe = NULL;12062	DECLARE_LIBBPF_OPTS(bpf_perf_event_opts, pe_opts);12063	enum probe_attach_mode attach_mode;12064	char full_path[PATH_MAX];12065	struct bpf_link *link;12066	size_t ref_ctr_off;12067	int pfd, err;12068	bool retprobe, legacy;12069	const char *func_name;12070 12071	if (!OPTS_VALID(opts, bpf_uprobe_opts))12072		return libbpf_err_ptr(-EINVAL);12073 12074	attach_mode = OPTS_GET(opts, attach_mode, PROBE_ATTACH_MODE_DEFAULT);12075	retprobe = OPTS_GET(opts, retprobe, false);12076	ref_ctr_off = OPTS_GET(opts, ref_ctr_offset, 0);12077	pe_opts.bpf_cookie = OPTS_GET(opts, bpf_cookie, 0);12078 12079	if (!binary_path)12080		return libbpf_err_ptr(-EINVAL);12081 12082	/* Check if "binary_path" refers to an archive. */12083	archive_sep = strstr(binary_path, "!/");12084	if (archive_sep) {12085		full_path[0] = '\0';12086		libbpf_strlcpy(full_path, binary_path,12087			       min(sizeof(full_path), (size_t)(archive_sep - binary_path + 1)));12088		archive_path = full_path;12089		binary_path = archive_sep + 2;12090	} else if (!strchr(binary_path, '/')) {12091		err = resolve_full_path(binary_path, full_path, sizeof(full_path));12092		if (err) {12093			pr_warn("prog '%s': failed to resolve full path for '%s': %d\n",12094				prog->name, binary_path, err);12095			return libbpf_err_ptr(err);12096		}12097		binary_path = full_path;12098	}12099	func_name = OPTS_GET(opts, func_name, NULL);12100	if (func_name) {12101		long sym_off;12102 12103		if (archive_path) {12104			sym_off = elf_find_func_offset_from_archive(archive_path, binary_path,12105								    func_name);12106			binary_path = archive_path;12107		} else {12108			sym_off = elf_find_func_offset_from_file(binary_path, func_name);12109		}12110		if (sym_off < 0)12111			return libbpf_err_ptr(sym_off);12112		func_offset += sym_off;12113	}12114 12115	legacy = determine_uprobe_perf_type() < 0;12116	switch (attach_mode) {12117	case PROBE_ATTACH_MODE_LEGACY:12118		legacy = true;12119		pe_opts.force_ioctl_attach = true;12120		break;12121	case PROBE_ATTACH_MODE_PERF:12122		if (legacy)12123			return libbpf_err_ptr(-ENOTSUP);12124		pe_opts.force_ioctl_attach = true;12125		break;12126	case PROBE_ATTACH_MODE_LINK:12127		if (legacy || !kernel_supports(prog->obj, FEAT_PERF_LINK))12128			return libbpf_err_ptr(-ENOTSUP);12129		break;12130	case PROBE_ATTACH_MODE_DEFAULT:12131		break;12132	default:12133		return libbpf_err_ptr(-EINVAL);12134	}12135 12136	if (!legacy) {12137		pfd = perf_event_open_probe(true /* uprobe */, retprobe, binary_path,12138					    func_offset, pid, ref_ctr_off);12139	} else {12140		char probe_name[PATH_MAX + 64];12141 12142		if (ref_ctr_off)12143			return libbpf_err_ptr(-EINVAL);12144 12145		gen_uprobe_legacy_event_name(probe_name, sizeof(probe_name),12146					     binary_path, func_offset);12147 12148		legacy_probe = strdup(probe_name);12149		if (!legacy_probe)12150			return libbpf_err_ptr(-ENOMEM);12151 12152		pfd = perf_event_uprobe_open_legacy(legacy_probe, retprobe,12153						    binary_path, func_offset, pid);12154	}12155	if (pfd < 0) {12156		err = -errno;12157		pr_warn("prog '%s': failed to create %s '%s:0x%zx' perf event: %s\n",12158			prog->name, retprobe ? "uretprobe" : "uprobe",12159			binary_path, func_offset,12160			libbpf_strerror_r(err, errmsg, sizeof(errmsg)));12161		goto err_out;12162	}12163 12164	link = bpf_program__attach_perf_event_opts(prog, pfd, &pe_opts);12165	err = libbpf_get_error(link);12166	if (err) {12167		close(pfd);12168		pr_warn("prog '%s': failed to attach to %s '%s:0x%zx': %s\n",12169			prog->name, retprobe ? "uretprobe" : "uprobe",12170			binary_path, func_offset,12171			libbpf_strerror_r(err, errmsg, sizeof(errmsg)));12172		goto err_clean_legacy;12173	}12174	if (legacy) {12175		struct bpf_link_perf *perf_link = container_of(link, struct bpf_link_perf, link);12176 12177		perf_link->legacy_probe_name = legacy_probe;12178		perf_link->legacy_is_kprobe = false;12179		perf_link->legacy_is_retprobe = retprobe;12180	}12181	return link;12182 12183err_clean_legacy:12184	if (legacy)12185		remove_uprobe_event_legacy(legacy_probe, retprobe);12186err_out:12187	free(legacy_probe);12188	return libbpf_err_ptr(err);12189}12190 12191/* Format of u[ret]probe section definition supporting auto-attach:12192 * u[ret]probe/binary:function[+offset]12193 *12194 * binary can be an absolute/relative path or a filename; the latter is resolved to a12195 * full binary path via bpf_program__attach_uprobe_opts.12196 *12197 * Specifying uprobe+ ensures we carry out strict matching; either "uprobe" must be12198 * specified (and auto-attach is not possible) or the above format is specified for12199 * auto-attach.12200 */12201static int attach_uprobe(const struct bpf_program *prog, long cookie, struct bpf_link **link)12202{12203	DECLARE_LIBBPF_OPTS(bpf_uprobe_opts, opts);12204	char *probe_type = NULL, *binary_path = NULL, *func_name = NULL, *func_off;12205	int n, c, ret = -EINVAL;12206	long offset = 0;12207 12208	*link = NULL;12209 12210	n = sscanf(prog->sec_name, "%m[^/]/%m[^:]:%m[^\n]",12211		   &probe_type, &binary_path, &func_name);12212	switch (n) {12213	case 1:12214		/* handle SEC("u[ret]probe") - format is valid, but auto-attach is impossible. */12215		ret = 0;12216		break;12217	case 2:12218		pr_warn("prog '%s': section '%s' missing ':function[+offset]' specification\n",12219			prog->name, prog->sec_name);12220		break;12221	case 3:12222		/* check if user specifies `+offset`, if yes, this should be12223		 * the last part of the string, make sure sscanf read to EOL12224		 */12225		func_off = strrchr(func_name, '+');12226		if (func_off) {12227			n = sscanf(func_off, "+%li%n", &offset, &c);12228			if (n == 1 && *(func_off + c) == '\0')12229				func_off[0] = '\0';12230			else12231				offset = 0;12232		}12233		opts.retprobe = strcmp(probe_type, "uretprobe") == 0 ||12234				strcmp(probe_type, "uretprobe.s") == 0;12235		if (opts.retprobe && offset != 0) {12236			pr_warn("prog '%s': uretprobes do not support offset specification\n",12237				prog->name);12238			break;12239		}12240		opts.func_name = func_name;12241		*link = bpf_program__attach_uprobe_opts(prog, -1, binary_path, offset, &opts);12242		ret = libbpf_get_error(*link);12243		break;12244	default:12245		pr_warn("prog '%s': invalid format of section definition '%s'\n", prog->name,12246			prog->sec_name);12247		break;12248	}12249	free(probe_type);12250	free(binary_path);12251	free(func_name);12252 12253	return ret;12254}12255 12256struct bpf_link *bpf_program__attach_uprobe(const struct bpf_program *prog,12257					    bool retprobe, pid_t pid,12258					    const char *binary_path,12259					    size_t func_offset)12260{12261	DECLARE_LIBBPF_OPTS(bpf_uprobe_opts, opts, .retprobe = retprobe);12262 12263	return bpf_program__attach_uprobe_opts(prog, pid, binary_path, func_offset, &opts);12264}12265 12266struct bpf_link *bpf_program__attach_usdt(const struct bpf_program *prog,12267					  pid_t pid, const char *binary_path,12268					  const char *usdt_provider, const char *usdt_name,12269					  const struct bpf_usdt_opts *opts)12270{12271	char resolved_path[512];12272	struct bpf_object *obj = prog->obj;12273	struct bpf_link *link;12274	__u64 usdt_cookie;12275	int err;12276 12277	if (!OPTS_VALID(opts, bpf_uprobe_opts))12278		return libbpf_err_ptr(-EINVAL);12279 12280	if (bpf_program__fd(prog) < 0) {12281		pr_warn("prog '%s': can't attach BPF program without FD (was it loaded?)\n",12282			prog->name);12283		return libbpf_err_ptr(-EINVAL);12284	}12285 12286	if (!binary_path)12287		return libbpf_err_ptr(-EINVAL);12288 12289	if (!strchr(binary_path, '/')) {12290		err = resolve_full_path(binary_path, resolved_path, sizeof(resolved_path));12291		if (err) {12292			pr_warn("prog '%s': failed to resolve full path for '%s': %d\n",12293				prog->name, binary_path, err);12294			return libbpf_err_ptr(err);12295		}12296		binary_path = resolved_path;12297	}12298 12299	/* USDT manager is instantiated lazily on first USDT attach. It will12300	 * be destroyed together with BPF object in bpf_object__close().12301	 */12302	if (IS_ERR(obj->usdt_man))12303		return libbpf_ptr(obj->usdt_man);12304	if (!obj->usdt_man) {12305		obj->usdt_man = usdt_manager_new(obj);12306		if (IS_ERR(obj->usdt_man))12307			return libbpf_ptr(obj->usdt_man);12308	}12309 12310	usdt_cookie = OPTS_GET(opts, usdt_cookie, 0);12311	link = usdt_manager_attach_usdt(obj->usdt_man, prog, pid, binary_path,12312					usdt_provider, usdt_name, usdt_cookie);12313	err = libbpf_get_error(link);12314	if (err)12315		return libbpf_err_ptr(err);12316	return link;12317}12318 12319static int attach_usdt(const struct bpf_program *prog, long cookie, struct bpf_link **link)12320{12321	char *path = NULL, *provider = NULL, *name = NULL;12322	const char *sec_name;12323	int n, err;12324 12325	sec_name = bpf_program__section_name(prog);12326	if (strcmp(sec_name, "usdt") == 0) {12327		/* no auto-attach for just SEC("usdt") */12328		*link = NULL;12329		return 0;12330	}12331 12332	n = sscanf(sec_name, "usdt/%m[^:]:%m[^:]:%m[^:]", &path, &provider, &name);12333	if (n != 3) {12334		pr_warn("invalid section '%s', expected SEC(\"usdt/<path>:<provider>:<name>\")\n",12335			sec_name);12336		err = -EINVAL;12337	} else {12338		*link = bpf_program__attach_usdt(prog, -1 /* any process */, path,12339						 provider, name, NULL);12340		err = libbpf_get_error(*link);12341	}12342	free(path);12343	free(provider);12344	free(name);12345	return err;12346}12347 12348static int determine_tracepoint_id(const char *tp_category,12349				   const char *tp_name)12350{12351	char file[PATH_MAX];12352	int ret;12353 12354	ret = snprintf(file, sizeof(file), "%s/events/%s/%s/id",12355		       tracefs_path(), tp_category, tp_name);12356	if (ret < 0)12357		return -errno;12358	if (ret >= sizeof(file)) {12359		pr_debug("tracepoint %s/%s path is too long\n",12360			 tp_category, tp_name);12361		return -E2BIG;12362	}12363	return parse_uint_from_file(file, "%d\n");12364}12365 12366static int perf_event_open_tracepoint(const char *tp_category,12367				      const char *tp_name)12368{12369	const size_t attr_sz = sizeof(struct perf_event_attr);12370	struct perf_event_attr attr;12371	char errmsg[STRERR_BUFSIZE];12372	int tp_id, pfd, err;12373 12374	tp_id = determine_tracepoint_id(tp_category, tp_name);12375	if (tp_id < 0) {12376		pr_warn("failed to determine tracepoint '%s/%s' perf event ID: %s\n",12377			tp_category, tp_name,12378			libbpf_strerror_r(tp_id, errmsg, sizeof(errmsg)));12379		return tp_id;12380	}12381 12382	memset(&attr, 0, attr_sz);12383	attr.type = PERF_TYPE_TRACEPOINT;12384	attr.size = attr_sz;12385	attr.config = tp_id;12386 12387	pfd = syscall(__NR_perf_event_open, &attr, -1 /* pid */, 0 /* cpu */,12388		      -1 /* group_fd */, PERF_FLAG_FD_CLOEXEC);12389	if (pfd < 0) {12390		err = -errno;12391		pr_warn("tracepoint '%s/%s' perf_event_open() failed: %s\n",12392			tp_category, tp_name,12393			libbpf_strerror_r(err, errmsg, sizeof(errmsg)));12394		return err;12395	}12396	return pfd;12397}12398 12399struct bpf_link *bpf_program__attach_tracepoint_opts(const struct bpf_program *prog,12400						     const char *tp_category,12401						     const char *tp_name,12402						     const struct bpf_tracepoint_opts *opts)12403{12404	DECLARE_LIBBPF_OPTS(bpf_perf_event_opts, pe_opts);12405	char errmsg[STRERR_BUFSIZE];12406	struct bpf_link *link;12407	int pfd, err;12408 12409	if (!OPTS_VALID(opts, bpf_tracepoint_opts))12410		return libbpf_err_ptr(-EINVAL);12411 12412	pe_opts.bpf_cookie = OPTS_GET(opts, bpf_cookie, 0);12413 12414	pfd = perf_event_open_tracepoint(tp_category, tp_name);12415	if (pfd < 0) {12416		pr_warn("prog '%s': failed to create tracepoint '%s/%s' perf event: %s\n",12417			prog->name, tp_category, tp_name,12418			libbpf_strerror_r(pfd, errmsg, sizeof(errmsg)));12419		return libbpf_err_ptr(pfd);12420	}12421	link = bpf_program__attach_perf_event_opts(prog, pfd, &pe_opts);12422	err = libbpf_get_error(link);12423	if (err) {12424		close(pfd);12425		pr_warn("prog '%s': failed to attach to tracepoint '%s/%s': %s\n",12426			prog->name, tp_category, tp_name,12427			libbpf_strerror_r(err, errmsg, sizeof(errmsg)));12428		return libbpf_err_ptr(err);12429	}12430	return link;12431}12432 12433struct bpf_link *bpf_program__attach_tracepoint(const struct bpf_program *prog,12434						const char *tp_category,12435						const char *tp_name)12436{12437	return bpf_program__attach_tracepoint_opts(prog, tp_category, tp_name, NULL);12438}12439 12440static int attach_tp(const struct bpf_program *prog, long cookie, struct bpf_link **link)12441{12442	char *sec_name, *tp_cat, *tp_name;12443 12444	*link = NULL;12445 12446	/* no auto-attach for SEC("tp") or SEC("tracepoint") */12447	if (strcmp(prog->sec_name, "tp") == 0 || strcmp(prog->sec_name, "tracepoint") == 0)12448		return 0;12449 12450	sec_name = strdup(prog->sec_name);12451	if (!sec_name)12452		return -ENOMEM;12453 12454	/* extract "tp/<category>/<name>" or "tracepoint/<category>/<name>" */12455	if (str_has_pfx(prog->sec_name, "tp/"))12456		tp_cat = sec_name + sizeof("tp/") - 1;12457	else12458		tp_cat = sec_name + sizeof("tracepoint/") - 1;12459	tp_name = strchr(tp_cat, '/');12460	if (!tp_name) {12461		free(sec_name);12462		return -EINVAL;12463	}12464	*tp_name = '\0';12465	tp_name++;12466 12467	*link = bpf_program__attach_tracepoint(prog, tp_cat, tp_name);12468	free(sec_name);12469	return libbpf_get_error(*link);12470}12471 12472struct bpf_link *12473bpf_program__attach_raw_tracepoint_opts(const struct bpf_program *prog,12474					const char *tp_name,12475					struct bpf_raw_tracepoint_opts *opts)12476{12477	LIBBPF_OPTS(bpf_raw_tp_opts, raw_opts);12478	char errmsg[STRERR_BUFSIZE];12479	struct bpf_link *link;12480	int prog_fd, pfd;12481 12482	if (!OPTS_VALID(opts, bpf_raw_tracepoint_opts))12483		return libbpf_err_ptr(-EINVAL);12484 12485	prog_fd = bpf_program__fd(prog);12486	if (prog_fd < 0) {12487		pr_warn("prog '%s': can't attach before loaded\n", prog->name);12488		return libbpf_err_ptr(-EINVAL);12489	}12490 12491	link = calloc(1, sizeof(*link));12492	if (!link)12493		return libbpf_err_ptr(-ENOMEM);12494	link->detach = &bpf_link__detach_fd;12495 12496	raw_opts.tp_name = tp_name;12497	raw_opts.cookie = OPTS_GET(opts, cookie, 0);12498	pfd = bpf_raw_tracepoint_open_opts(prog_fd, &raw_opts);12499	if (pfd < 0) {12500		pfd = -errno;12501		free(link);12502		pr_warn("prog '%s': failed to attach to raw tracepoint '%s': %s\n",12503			prog->name, tp_name, libbpf_strerror_r(pfd, errmsg, sizeof(errmsg)));12504		return libbpf_err_ptr(pfd);12505	}12506	link->fd = pfd;12507	return link;12508}12509 12510struct bpf_link *bpf_program__attach_raw_tracepoint(const struct bpf_program *prog,12511						    const char *tp_name)12512{12513	return bpf_program__attach_raw_tracepoint_opts(prog, tp_name, NULL);12514}12515 12516static int attach_raw_tp(const struct bpf_program *prog, long cookie, struct bpf_link **link)12517{12518	static const char *const prefixes[] = {12519		"raw_tp",12520		"raw_tracepoint",12521		"raw_tp.w",12522		"raw_tracepoint.w",12523	};12524	size_t i;12525	const char *tp_name = NULL;12526 12527	*link = NULL;12528 12529	for (i = 0; i < ARRAY_SIZE(prefixes); i++) {12530		size_t pfx_len;12531 12532		if (!str_has_pfx(prog->sec_name, prefixes[i]))12533			continue;12534 12535		pfx_len = strlen(prefixes[i]);12536		/* no auto-attach case of, e.g., SEC("raw_tp") */12537		if (prog->sec_name[pfx_len] == '\0')12538			return 0;12539 12540		if (prog->sec_name[pfx_len] != '/')12541			continue;12542 12543		tp_name = prog->sec_name + pfx_len + 1;12544		break;12545	}12546 12547	if (!tp_name) {12548		pr_warn("prog '%s': invalid section name '%s'\n",12549			prog->name, prog->sec_name);12550		return -EINVAL;12551	}12552 12553	*link = bpf_program__attach_raw_tracepoint(prog, tp_name);12554	return libbpf_get_error(*link);12555}12556 12557/* Common logic for all BPF program types that attach to a btf_id */12558static struct bpf_link *bpf_program__attach_btf_id(const struct bpf_program *prog,12559						   const struct bpf_trace_opts *opts)12560{12561	LIBBPF_OPTS(bpf_link_create_opts, link_opts);12562	char errmsg[STRERR_BUFSIZE];12563	struct bpf_link *link;12564	int prog_fd, pfd;12565 12566	if (!OPTS_VALID(opts, bpf_trace_opts))12567		return libbpf_err_ptr(-EINVAL);12568 12569	prog_fd = bpf_program__fd(prog);12570	if (prog_fd < 0) {12571		pr_warn("prog '%s': can't attach before loaded\n", prog->name);12572		return libbpf_err_ptr(-EINVAL);12573	}12574 12575	link = calloc(1, sizeof(*link));12576	if (!link)12577		return libbpf_err_ptr(-ENOMEM);12578	link->detach = &bpf_link__detach_fd;12579 12580	/* libbpf is smart enough to redirect to BPF_RAW_TRACEPOINT_OPEN on old kernels */12581	link_opts.tracing.cookie = OPTS_GET(opts, cookie, 0);12582	pfd = bpf_link_create(prog_fd, 0, bpf_program__expected_attach_type(prog), &link_opts);12583	if (pfd < 0) {12584		pfd = -errno;12585		free(link);12586		pr_warn("prog '%s': failed to attach: %s\n",12587			prog->name, libbpf_strerror_r(pfd, errmsg, sizeof(errmsg)));12588		return libbpf_err_ptr(pfd);12589	}12590	link->fd = pfd;12591	return link;12592}12593 12594struct bpf_link *bpf_program__attach_trace(const struct bpf_program *prog)12595{12596	return bpf_program__attach_btf_id(prog, NULL);12597}12598 12599struct bpf_link *bpf_program__attach_trace_opts(const struct bpf_program *prog,12600						const struct bpf_trace_opts *opts)12601{12602	return bpf_program__attach_btf_id(prog, opts);12603}12604 12605struct bpf_link *bpf_program__attach_lsm(const struct bpf_program *prog)12606{12607	return bpf_program__attach_btf_id(prog, NULL);12608}12609 12610static int attach_trace(const struct bpf_program *prog, long cookie, struct bpf_link **link)12611{12612	*link = bpf_program__attach_trace(prog);12613	return libbpf_get_error(*link);12614}12615 12616static int attach_lsm(const struct bpf_program *prog, long cookie, struct bpf_link **link)12617{12618	*link = bpf_program__attach_lsm(prog);12619	return libbpf_get_error(*link);12620}12621 12622static struct bpf_link *12623bpf_program_attach_fd(const struct bpf_program *prog,12624		      int target_fd, const char *target_name,12625		      const struct bpf_link_create_opts *opts)12626{12627	enum bpf_attach_type attach_type;12628	char errmsg[STRERR_BUFSIZE];12629	struct bpf_link *link;12630	int prog_fd, link_fd;12631 12632	prog_fd = bpf_program__fd(prog);12633	if (prog_fd < 0) {12634		pr_warn("prog '%s': can't attach before loaded\n", prog->name);12635		return libbpf_err_ptr(-EINVAL);12636	}12637 12638	link = calloc(1, sizeof(*link));12639	if (!link)12640		return libbpf_err_ptr(-ENOMEM);12641	link->detach = &bpf_link__detach_fd;12642 12643	attach_type = bpf_program__expected_attach_type(prog);12644	link_fd = bpf_link_create(prog_fd, target_fd, attach_type, opts);12645	if (link_fd < 0) {12646		link_fd = -errno;12647		free(link);12648		pr_warn("prog '%s': failed to attach to %s: %s\n",12649			prog->name, target_name,12650			libbpf_strerror_r(link_fd, errmsg, sizeof(errmsg)));12651		return libbpf_err_ptr(link_fd);12652	}12653	link->fd = link_fd;12654	return link;12655}12656 12657struct bpf_link *12658bpf_program__attach_cgroup(const struct bpf_program *prog, int cgroup_fd)12659{12660	return bpf_program_attach_fd(prog, cgroup_fd, "cgroup", NULL);12661}12662 12663struct bpf_link *12664bpf_program__attach_netns(const struct bpf_program *prog, int netns_fd)12665{12666	return bpf_program_attach_fd(prog, netns_fd, "netns", NULL);12667}12668 12669struct bpf_link *12670bpf_program__attach_sockmap(const struct bpf_program *prog, int map_fd)12671{12672	return bpf_program_attach_fd(prog, map_fd, "sockmap", NULL);12673}12674 12675struct bpf_link *bpf_program__attach_xdp(const struct bpf_program *prog, int ifindex)12676{12677	/* target_fd/target_ifindex use the same field in LINK_CREATE */12678	return bpf_program_attach_fd(prog, ifindex, "xdp", NULL);12679}12680 12681struct bpf_link *12682bpf_program__attach_tcx(const struct bpf_program *prog, int ifindex,12683			const struct bpf_tcx_opts *opts)12684{12685	LIBBPF_OPTS(bpf_link_create_opts, link_create_opts);12686	__u32 relative_id;12687	int relative_fd;12688 12689	if (!OPTS_VALID(opts, bpf_tcx_opts))12690		return libbpf_err_ptr(-EINVAL);12691 12692	relative_id = OPTS_GET(opts, relative_id, 0);12693	relative_fd = OPTS_GET(opts, relative_fd, 0);12694 12695	/* validate we don't have unexpected combinations of non-zero fields */12696	if (!ifindex) {12697		pr_warn("prog '%s': target netdevice ifindex cannot be zero\n",12698			prog->name);12699		return libbpf_err_ptr(-EINVAL);12700	}12701	if (relative_fd && relative_id) {12702		pr_warn("prog '%s': relative_fd and relative_id cannot be set at the same time\n",12703			prog->name);12704		return libbpf_err_ptr(-EINVAL);12705	}12706 12707	link_create_opts.tcx.expected_revision = OPTS_GET(opts, expected_revision, 0);12708	link_create_opts.tcx.relative_fd = relative_fd;12709	link_create_opts.tcx.relative_id = relative_id;12710	link_create_opts.flags = OPTS_GET(opts, flags, 0);12711 12712	/* target_fd/target_ifindex use the same field in LINK_CREATE */12713	return bpf_program_attach_fd(prog, ifindex, "tcx", &link_create_opts);12714}12715 12716struct bpf_link *12717bpf_program__attach_netkit(const struct bpf_program *prog, int ifindex,12718			   const struct bpf_netkit_opts *opts)12719{12720	LIBBPF_OPTS(bpf_link_create_opts, link_create_opts);12721	__u32 relative_id;12722	int relative_fd;12723 12724	if (!OPTS_VALID(opts, bpf_netkit_opts))12725		return libbpf_err_ptr(-EINVAL);12726 12727	relative_id = OPTS_GET(opts, relative_id, 0);12728	relative_fd = OPTS_GET(opts, relative_fd, 0);12729 12730	/* validate we don't have unexpected combinations of non-zero fields */12731	if (!ifindex) {12732		pr_warn("prog '%s': target netdevice ifindex cannot be zero\n",12733			prog->name);12734		return libbpf_err_ptr(-EINVAL);12735	}12736	if (relative_fd && relative_id) {12737		pr_warn("prog '%s': relative_fd and relative_id cannot be set at the same time\n",12738			prog->name);12739		return libbpf_err_ptr(-EINVAL);12740	}12741 12742	link_create_opts.netkit.expected_revision = OPTS_GET(opts, expected_revision, 0);12743	link_create_opts.netkit.relative_fd = relative_fd;12744	link_create_opts.netkit.relative_id = relative_id;12745	link_create_opts.flags = OPTS_GET(opts, flags, 0);12746 12747	return bpf_program_attach_fd(prog, ifindex, "netkit", &link_create_opts);12748}12749 12750struct bpf_link *bpf_program__attach_freplace(const struct bpf_program *prog,12751					      int target_fd,12752					      const char *attach_func_name)12753{12754	int btf_id;12755 12756	if (!!target_fd != !!attach_func_name) {12757		pr_warn("prog '%s': supply none or both of target_fd and attach_func_name\n",12758			prog->name);12759		return libbpf_err_ptr(-EINVAL);12760	}12761 12762	if (prog->type != BPF_PROG_TYPE_EXT) {12763		pr_warn("prog '%s': only BPF_PROG_TYPE_EXT can attach as freplace",12764			prog->name);12765		return libbpf_err_ptr(-EINVAL);12766	}12767 12768	if (target_fd) {12769		LIBBPF_OPTS(bpf_link_create_opts, target_opts);12770 12771		btf_id = libbpf_find_prog_btf_id(attach_func_name, target_fd);12772		if (btf_id < 0)12773			return libbpf_err_ptr(btf_id);12774 12775		target_opts.target_btf_id = btf_id;12776 12777		return bpf_program_attach_fd(prog, target_fd, "freplace",12778					     &target_opts);12779	} else {12780		/* no target, so use raw_tracepoint_open for compatibility12781		 * with old kernels12782		 */12783		return bpf_program__attach_trace(prog);12784	}12785}12786 12787struct bpf_link *12788bpf_program__attach_iter(const struct bpf_program *prog,12789			 const struct bpf_iter_attach_opts *opts)12790{12791	DECLARE_LIBBPF_OPTS(bpf_link_create_opts, link_create_opts);12792	char errmsg[STRERR_BUFSIZE];12793	struct bpf_link *link;12794	int prog_fd, link_fd;12795	__u32 target_fd = 0;12796 12797	if (!OPTS_VALID(opts, bpf_iter_attach_opts))12798		return libbpf_err_ptr(-EINVAL);12799 12800	link_create_opts.iter_info = OPTS_GET(opts, link_info, (void *)0);12801	link_create_opts.iter_info_len = OPTS_GET(opts, link_info_len, 0);12802 12803	prog_fd = bpf_program__fd(prog);12804	if (prog_fd < 0) {12805		pr_warn("prog '%s': can't attach before loaded\n", prog->name);12806		return libbpf_err_ptr(-EINVAL);12807	}12808 12809	link = calloc(1, sizeof(*link));12810	if (!link)12811		return libbpf_err_ptr(-ENOMEM);12812	link->detach = &bpf_link__detach_fd;12813 12814	link_fd = bpf_link_create(prog_fd, target_fd, BPF_TRACE_ITER,12815				  &link_create_opts);12816	if (link_fd < 0) {12817		link_fd = -errno;12818		free(link);12819		pr_warn("prog '%s': failed to attach to iterator: %s\n",12820			prog->name, libbpf_strerror_r(link_fd, errmsg, sizeof(errmsg)));12821		return libbpf_err_ptr(link_fd);12822	}12823	link->fd = link_fd;12824	return link;12825}12826 12827static int attach_iter(const struct bpf_program *prog, long cookie, struct bpf_link **link)12828{12829	*link = bpf_program__attach_iter(prog, NULL);12830	return libbpf_get_error(*link);12831}12832 12833struct bpf_link *bpf_program__attach_netfilter(const struct bpf_program *prog,12834					       const struct bpf_netfilter_opts *opts)12835{12836	LIBBPF_OPTS(bpf_link_create_opts, lopts);12837	struct bpf_link *link;12838	int prog_fd, link_fd;12839 12840	if (!OPTS_VALID(opts, bpf_netfilter_opts))12841		return libbpf_err_ptr(-EINVAL);12842 12843	prog_fd = bpf_program__fd(prog);12844	if (prog_fd < 0) {12845		pr_warn("prog '%s': can't attach before loaded\n", prog->name);12846		return libbpf_err_ptr(-EINVAL);12847	}12848 12849	link = calloc(1, sizeof(*link));12850	if (!link)12851		return libbpf_err_ptr(-ENOMEM);12852 12853	link->detach = &bpf_link__detach_fd;12854 12855	lopts.netfilter.pf = OPTS_GET(opts, pf, 0);12856	lopts.netfilter.hooknum = OPTS_GET(opts, hooknum, 0);12857	lopts.netfilter.priority = OPTS_GET(opts, priority, 0);12858	lopts.netfilter.flags = OPTS_GET(opts, flags, 0);12859 12860	link_fd = bpf_link_create(prog_fd, 0, BPF_NETFILTER, &lopts);12861	if (link_fd < 0) {12862		char errmsg[STRERR_BUFSIZE];12863 12864		link_fd = -errno;12865		free(link);12866		pr_warn("prog '%s': failed to attach to netfilter: %s\n",12867			prog->name, libbpf_strerror_r(link_fd, errmsg, sizeof(errmsg)));12868		return libbpf_err_ptr(link_fd);12869	}12870	link->fd = link_fd;12871 12872	return link;12873}12874 12875struct bpf_link *bpf_program__attach(const struct bpf_program *prog)12876{12877	struct bpf_link *link = NULL;12878	int err;12879 12880	if (!prog->sec_def || !prog->sec_def->prog_attach_fn)12881		return libbpf_err_ptr(-EOPNOTSUPP);12882 12883	if (bpf_program__fd(prog) < 0) {12884		pr_warn("prog '%s': can't attach BPF program without FD (was it loaded?)\n",12885			prog->name);12886		return libbpf_err_ptr(-EINVAL);12887	}12888 12889	err = prog->sec_def->prog_attach_fn(prog, prog->sec_def->cookie, &link);12890	if (err)12891		return libbpf_err_ptr(err);12892 12893	/* When calling bpf_program__attach() explicitly, auto-attach support12894	 * is expected to work, so NULL returned link is considered an error.12895	 * This is different for skeleton's attach, see comment in12896	 * bpf_object__attach_skeleton().12897	 */12898	if (!link)12899		return libbpf_err_ptr(-EOPNOTSUPP);12900 12901	return link;12902}12903 12904struct bpf_link_struct_ops {12905	struct bpf_link link;12906	int map_fd;12907};12908 12909static int bpf_link__detach_struct_ops(struct bpf_link *link)12910{12911	struct bpf_link_struct_ops *st_link;12912	__u32 zero = 0;12913 12914	st_link = container_of(link, struct bpf_link_struct_ops, link);12915 12916	if (st_link->map_fd < 0)12917		/* w/o a real link */12918		return bpf_map_delete_elem(link->fd, &zero);12919 12920	return close(link->fd);12921}12922 12923struct bpf_link *bpf_map__attach_struct_ops(const struct bpf_map *map)12924{12925	struct bpf_link_struct_ops *link;12926	__u32 zero = 0;12927	int err, fd;12928 12929	if (!bpf_map__is_struct_ops(map)) {12930		pr_warn("map '%s': can't attach non-struct_ops map\n", map->name);12931		return libbpf_err_ptr(-EINVAL);12932	}12933 12934	if (map->fd < 0) {12935		pr_warn("map '%s': can't attach BPF map without FD (was it created?)\n", map->name);12936		return libbpf_err_ptr(-EINVAL);12937	}12938 12939	link = calloc(1, sizeof(*link));12940	if (!link)12941		return libbpf_err_ptr(-EINVAL);12942 12943	/* kern_vdata should be prepared during the loading phase. */12944	err = bpf_map_update_elem(map->fd, &zero, map->st_ops->kern_vdata, 0);12945	/* It can be EBUSY if the map has been used to create or12946	 * update a link before.  We don't allow updating the value of12947	 * a struct_ops once it is set.  That ensures that the value12948	 * never changed.  So, it is safe to skip EBUSY.12949	 */12950	if (err && (!(map->def.map_flags & BPF_F_LINK) || err != -EBUSY)) {12951		free(link);12952		return libbpf_err_ptr(err);12953	}12954 12955	link->link.detach = bpf_link__detach_struct_ops;12956 12957	if (!(map->def.map_flags & BPF_F_LINK)) {12958		/* w/o a real link */12959		link->link.fd = map->fd;12960		link->map_fd = -1;12961		return &link->link;12962	}12963 12964	fd = bpf_link_create(map->fd, 0, BPF_STRUCT_OPS, NULL);12965	if (fd < 0) {12966		free(link);12967		return libbpf_err_ptr(fd);12968	}12969 12970	link->link.fd = fd;12971	link->map_fd = map->fd;12972 12973	return &link->link;12974}12975 12976/*12977 * Swap the back struct_ops of a link with a new struct_ops map.12978 */12979int bpf_link__update_map(struct bpf_link *link, const struct bpf_map *map)12980{12981	struct bpf_link_struct_ops *st_ops_link;12982	__u32 zero = 0;12983	int err;12984 12985	if (!bpf_map__is_struct_ops(map))12986		return -EINVAL;12987 12988	if (map->fd < 0) {12989		pr_warn("map '%s': can't use BPF map without FD (was it created?)\n", map->name);12990		return -EINVAL;12991	}12992 12993	st_ops_link = container_of(link, struct bpf_link_struct_ops, link);12994	/* Ensure the type of a link is correct */12995	if (st_ops_link->map_fd < 0)12996		return -EINVAL;12997 12998	err = bpf_map_update_elem(map->fd, &zero, map->st_ops->kern_vdata, 0);12999	/* It can be EBUSY if the map has been used to create or13000	 * update a link before.  We don't allow updating the value of13001	 * a struct_ops once it is set.  That ensures that the value13002	 * never changed.  So, it is safe to skip EBUSY.13003	 */13004	if (err && err != -EBUSY)13005		return err;13006 13007	err = bpf_link_update(link->fd, map->fd, NULL);13008	if (err < 0)13009		return err;13010 13011	st_ops_link->map_fd = map->fd;13012 13013	return 0;13014}13015 13016typedef enum bpf_perf_event_ret (*bpf_perf_event_print_t)(struct perf_event_header *hdr,13017							  void *private_data);13018 13019static enum bpf_perf_event_ret13020perf_event_read_simple(void *mmap_mem, size_t mmap_size, size_t page_size,13021		       void **copy_mem, size_t *copy_size,13022		       bpf_perf_event_print_t fn, void *private_data)13023{13024	struct perf_event_mmap_page *header = mmap_mem;13025	__u64 data_head = ring_buffer_read_head(header);13026	__u64 data_tail = header->data_tail;13027	void *base = ((__u8 *)header) + page_size;13028	int ret = LIBBPF_PERF_EVENT_CONT;13029	struct perf_event_header *ehdr;13030	size_t ehdr_size;13031 13032	while (data_head != data_tail) {13033		ehdr = base + (data_tail & (mmap_size - 1));13034		ehdr_size = ehdr->size;13035 13036		if (((void *)ehdr) + ehdr_size > base + mmap_size) {13037			void *copy_start = ehdr;13038			size_t len_first = base + mmap_size - copy_start;13039			size_t len_secnd = ehdr_size - len_first;13040 13041			if (*copy_size < ehdr_size) {13042				free(*copy_mem);13043				*copy_mem = malloc(ehdr_size);13044				if (!*copy_mem) {13045					*copy_size = 0;13046					ret = LIBBPF_PERF_EVENT_ERROR;13047					break;13048				}13049				*copy_size = ehdr_size;13050			}13051 13052			memcpy(*copy_mem, copy_start, len_first);13053			memcpy(*copy_mem + len_first, base, len_secnd);13054			ehdr = *copy_mem;13055		}13056 13057		ret = fn(ehdr, private_data);13058		data_tail += ehdr_size;13059		if (ret != LIBBPF_PERF_EVENT_CONT)13060			break;13061	}13062 13063	ring_buffer_write_tail(header, data_tail);13064	return libbpf_err(ret);13065}13066 13067struct perf_buffer;13068 13069struct perf_buffer_params {13070	struct perf_event_attr *attr;13071	/* if event_cb is specified, it takes precendence */13072	perf_buffer_event_fn event_cb;13073	/* sample_cb and lost_cb are higher-level common-case callbacks */13074	perf_buffer_sample_fn sample_cb;13075	perf_buffer_lost_fn lost_cb;13076	void *ctx;13077	int cpu_cnt;13078	int *cpus;13079	int *map_keys;13080};13081 13082struct perf_cpu_buf {13083	struct perf_buffer *pb;13084	void *base; /* mmap()'ed memory */13085	void *buf; /* for reconstructing segmented data */13086	size_t buf_size;13087	int fd;13088	int cpu;13089	int map_key;13090};13091 13092struct perf_buffer {13093	perf_buffer_event_fn event_cb;13094	perf_buffer_sample_fn sample_cb;13095	perf_buffer_lost_fn lost_cb;13096	void *ctx; /* passed into callbacks */13097 13098	size_t page_size;13099	size_t mmap_size;13100	struct perf_cpu_buf **cpu_bufs;13101	struct epoll_event *events;13102	int cpu_cnt; /* number of allocated CPU buffers */13103	int epoll_fd; /* perf event FD */13104	int map_fd; /* BPF_MAP_TYPE_PERF_EVENT_ARRAY BPF map FD */13105};13106 13107static void perf_buffer__free_cpu_buf(struct perf_buffer *pb,13108				      struct perf_cpu_buf *cpu_buf)13109{13110	if (!cpu_buf)13111		return;13112	if (cpu_buf->base &&13113	    munmap(cpu_buf->base, pb->mmap_size + pb->page_size))13114		pr_warn("failed to munmap cpu_buf #%d\n", cpu_buf->cpu);13115	if (cpu_buf->fd >= 0) {13116		ioctl(cpu_buf->fd, PERF_EVENT_IOC_DISABLE, 0);13117		close(cpu_buf->fd);13118	}13119	free(cpu_buf->buf);13120	free(cpu_buf);13121}13122 13123void perf_buffer__free(struct perf_buffer *pb)13124{13125	int i;13126 13127	if (IS_ERR_OR_NULL(pb))13128		return;13129	if (pb->cpu_bufs) {13130		for (i = 0; i < pb->cpu_cnt; i++) {13131			struct perf_cpu_buf *cpu_buf = pb->cpu_bufs[i];13132 13133			if (!cpu_buf)13134				continue;13135 13136			bpf_map_delete_elem(pb->map_fd, &cpu_buf->map_key);13137			perf_buffer__free_cpu_buf(pb, cpu_buf);13138		}13139		free(pb->cpu_bufs);13140	}13141	if (pb->epoll_fd >= 0)13142		close(pb->epoll_fd);13143	free(pb->events);13144	free(pb);13145}13146 13147static struct perf_cpu_buf *13148perf_buffer__open_cpu_buf(struct perf_buffer *pb, struct perf_event_attr *attr,13149			  int cpu, int map_key)13150{13151	struct perf_cpu_buf *cpu_buf;13152	char msg[STRERR_BUFSIZE];13153	int err;13154 13155	cpu_buf = calloc(1, sizeof(*cpu_buf));13156	if (!cpu_buf)13157		return ERR_PTR(-ENOMEM);13158 13159	cpu_buf->pb = pb;13160	cpu_buf->cpu = cpu;13161	cpu_buf->map_key = map_key;13162 13163	cpu_buf->fd = syscall(__NR_perf_event_open, attr, -1 /* pid */, cpu,13164			      -1, PERF_FLAG_FD_CLOEXEC);13165	if (cpu_buf->fd < 0) {13166		err = -errno;13167		pr_warn("failed to open perf buffer event on cpu #%d: %s\n",13168			cpu, libbpf_strerror_r(err, msg, sizeof(msg)));13169		goto error;13170	}13171 13172	cpu_buf->base = mmap(NULL, pb->mmap_size + pb->page_size,13173			     PROT_READ | PROT_WRITE, MAP_SHARED,13174			     cpu_buf->fd, 0);13175	if (cpu_buf->base == MAP_FAILED) {13176		cpu_buf->base = NULL;13177		err = -errno;13178		pr_warn("failed to mmap perf buffer on cpu #%d: %s\n",13179			cpu, libbpf_strerror_r(err, msg, sizeof(msg)));13180		goto error;13181	}13182 13183	if (ioctl(cpu_buf->fd, PERF_EVENT_IOC_ENABLE, 0) < 0) {13184		err = -errno;13185		pr_warn("failed to enable perf buffer event on cpu #%d: %s\n",13186			cpu, libbpf_strerror_r(err, msg, sizeof(msg)));13187		goto error;13188	}13189 13190	return cpu_buf;13191 13192error:13193	perf_buffer__free_cpu_buf(pb, cpu_buf);13194	return (struct perf_cpu_buf *)ERR_PTR(err);13195}13196 13197static struct perf_buffer *__perf_buffer__new(int map_fd, size_t page_cnt,13198					      struct perf_buffer_params *p);13199 13200struct perf_buffer *perf_buffer__new(int map_fd, size_t page_cnt,13201				     perf_buffer_sample_fn sample_cb,13202				     perf_buffer_lost_fn lost_cb,13203				     void *ctx,13204				     const struct perf_buffer_opts *opts)13205{13206	const size_t attr_sz = sizeof(struct perf_event_attr);13207	struct perf_buffer_params p = {};13208	struct perf_event_attr attr;13209	__u32 sample_period;13210 13211	if (!OPTS_VALID(opts, perf_buffer_opts))13212		return libbpf_err_ptr(-EINVAL);13213 13214	sample_period = OPTS_GET(opts, sample_period, 1);13215	if (!sample_period)13216		sample_period = 1;13217 13218	memset(&attr, 0, attr_sz);13219	attr.size = attr_sz;13220	attr.config = PERF_COUNT_SW_BPF_OUTPUT;13221	attr.type = PERF_TYPE_SOFTWARE;13222	attr.sample_type = PERF_SAMPLE_RAW;13223	attr.sample_period = sample_period;13224	attr.wakeup_events = sample_period;13225 13226	p.attr = &attr;13227	p.sample_cb = sample_cb;13228	p.lost_cb = lost_cb;13229	p.ctx = ctx;13230 13231	return libbpf_ptr(__perf_buffer__new(map_fd, page_cnt, &p));13232}13233 13234struct perf_buffer *perf_buffer__new_raw(int map_fd, size_t page_cnt,13235					 struct perf_event_attr *attr,13236					 perf_buffer_event_fn event_cb, void *ctx,13237					 const struct perf_buffer_raw_opts *opts)13238{13239	struct perf_buffer_params p = {};13240 13241	if (!attr)13242		return libbpf_err_ptr(-EINVAL);13243 13244	if (!OPTS_VALID(opts, perf_buffer_raw_opts))13245		return libbpf_err_ptr(-EINVAL);13246 13247	p.attr = attr;13248	p.event_cb = event_cb;13249	p.ctx = ctx;13250	p.cpu_cnt = OPTS_GET(opts, cpu_cnt, 0);13251	p.cpus = OPTS_GET(opts, cpus, NULL);13252	p.map_keys = OPTS_GET(opts, map_keys, NULL);13253 13254	return libbpf_ptr(__perf_buffer__new(map_fd, page_cnt, &p));13255}13256 13257static struct perf_buffer *__perf_buffer__new(int map_fd, size_t page_cnt,13258					      struct perf_buffer_params *p)13259{13260	const char *online_cpus_file = "/sys/devices/system/cpu/online";13261	struct bpf_map_info map;13262	char msg[STRERR_BUFSIZE];13263	struct perf_buffer *pb;13264	bool *online = NULL;13265	__u32 map_info_len;13266	int err, i, j, n;13267 13268	if (page_cnt == 0 || (page_cnt & (page_cnt - 1))) {13269		pr_warn("page count should be power of two, but is %zu\n",13270			page_cnt);13271		return ERR_PTR(-EINVAL);13272	}13273 13274	/* best-effort sanity checks */13275	memset(&map, 0, sizeof(map));13276	map_info_len = sizeof(map);13277	err = bpf_map_get_info_by_fd(map_fd, &map, &map_info_len);13278	if (err) {13279		err = -errno;13280		/* if BPF_OBJ_GET_INFO_BY_FD is supported, will return13281		 * -EBADFD, -EFAULT, or -E2BIG on real error13282		 */13283		if (err != -EINVAL) {13284			pr_warn("failed to get map info for map FD %d: %s\n",13285				map_fd, libbpf_strerror_r(err, msg, sizeof(msg)));13286			return ERR_PTR(err);13287		}13288		pr_debug("failed to get map info for FD %d; API not supported? Ignoring...\n",13289			 map_fd);13290	} else {13291		if (map.type != BPF_MAP_TYPE_PERF_EVENT_ARRAY) {13292			pr_warn("map '%s' should be BPF_MAP_TYPE_PERF_EVENT_ARRAY\n",13293				map.name);13294			return ERR_PTR(-EINVAL);13295		}13296	}13297 13298	pb = calloc(1, sizeof(*pb));13299	if (!pb)13300		return ERR_PTR(-ENOMEM);13301 13302	pb->event_cb = p->event_cb;13303	pb->sample_cb = p->sample_cb;13304	pb->lost_cb = p->lost_cb;13305	pb->ctx = p->ctx;13306 13307	pb->page_size = getpagesize();13308	pb->mmap_size = pb->page_size * page_cnt;13309	pb->map_fd = map_fd;13310 13311	pb->epoll_fd = epoll_create1(EPOLL_CLOEXEC);13312	if (pb->epoll_fd < 0) {13313		err = -errno;13314		pr_warn("failed to create epoll instance: %s\n",13315			libbpf_strerror_r(err, msg, sizeof(msg)));13316		goto error;13317	}13318 13319	if (p->cpu_cnt > 0) {13320		pb->cpu_cnt = p->cpu_cnt;13321	} else {13322		pb->cpu_cnt = libbpf_num_possible_cpus();13323		if (pb->cpu_cnt < 0) {13324			err = pb->cpu_cnt;13325			goto error;13326		}13327		if (map.max_entries && map.max_entries < pb->cpu_cnt)13328			pb->cpu_cnt = map.max_entries;13329	}13330 13331	pb->events = calloc(pb->cpu_cnt, sizeof(*pb->events));13332	if (!pb->events) {13333		err = -ENOMEM;13334		pr_warn("failed to allocate events: out of memory\n");13335		goto error;13336	}13337	pb->cpu_bufs = calloc(pb->cpu_cnt, sizeof(*pb->cpu_bufs));13338	if (!pb->cpu_bufs) {13339		err = -ENOMEM;13340		pr_warn("failed to allocate buffers: out of memory\n");13341		goto error;13342	}13343 13344	err = parse_cpu_mask_file(online_cpus_file, &online, &n);13345	if (err) {13346		pr_warn("failed to get online CPU mask: %d\n", err);13347		goto error;13348	}13349 13350	for (i = 0, j = 0; i < pb->cpu_cnt; i++) {13351		struct perf_cpu_buf *cpu_buf;13352		int cpu, map_key;13353 13354		cpu = p->cpu_cnt > 0 ? p->cpus[i] : i;13355		map_key = p->cpu_cnt > 0 ? p->map_keys[i] : i;13356 13357		/* in case user didn't explicitly requested particular CPUs to13358		 * be attached to, skip offline/not present CPUs13359		 */13360		if (p->cpu_cnt <= 0 && (cpu >= n || !online[cpu]))13361			continue;13362 13363		cpu_buf = perf_buffer__open_cpu_buf(pb, p->attr, cpu, map_key);13364		if (IS_ERR(cpu_buf)) {13365			err = PTR_ERR(cpu_buf);13366			goto error;13367		}13368 13369		pb->cpu_bufs[j] = cpu_buf;13370 13371		err = bpf_map_update_elem(pb->map_fd, &map_key,13372					  &cpu_buf->fd, 0);13373		if (err) {13374			err = -errno;13375			pr_warn("failed to set cpu #%d, key %d -> perf FD %d: %s\n",13376				cpu, map_key, cpu_buf->fd,13377				libbpf_strerror_r(err, msg, sizeof(msg)));13378			goto error;13379		}13380 13381		pb->events[j].events = EPOLLIN;13382		pb->events[j].data.ptr = cpu_buf;13383		if (epoll_ctl(pb->epoll_fd, EPOLL_CTL_ADD, cpu_buf->fd,13384			      &pb->events[j]) < 0) {13385			err = -errno;13386			pr_warn("failed to epoll_ctl cpu #%d perf FD %d: %s\n",13387				cpu, cpu_buf->fd,13388				libbpf_strerror_r(err, msg, sizeof(msg)));13389			goto error;13390		}13391		j++;13392	}13393	pb->cpu_cnt = j;13394	free(online);13395 13396	return pb;13397 13398error:13399	free(online);13400	if (pb)13401		perf_buffer__free(pb);13402	return ERR_PTR(err);13403}13404 13405struct perf_sample_raw {13406	struct perf_event_header header;13407	uint32_t size;13408	char data[];13409};13410 13411struct perf_sample_lost {13412	struct perf_event_header header;13413	uint64_t id;13414	uint64_t lost;13415	uint64_t sample_id;13416};13417 13418static enum bpf_perf_event_ret13419perf_buffer__process_record(struct perf_event_header *e, void *ctx)13420{13421	struct perf_cpu_buf *cpu_buf = ctx;13422	struct perf_buffer *pb = cpu_buf->pb;13423	void *data = e;13424 13425	/* user wants full control over parsing perf event */13426	if (pb->event_cb)13427		return pb->event_cb(pb->ctx, cpu_buf->cpu, e);13428 13429	switch (e->type) {13430	case PERF_RECORD_SAMPLE: {13431		struct perf_sample_raw *s = data;13432 13433		if (pb->sample_cb)13434			pb->sample_cb(pb->ctx, cpu_buf->cpu, s->data, s->size);13435		break;13436	}13437	case PERF_RECORD_LOST: {13438		struct perf_sample_lost *s = data;13439 13440		if (pb->lost_cb)13441			pb->lost_cb(pb->ctx, cpu_buf->cpu, s->lost);13442		break;13443	}13444	default:13445		pr_warn("unknown perf sample type %d\n", e->type);13446		return LIBBPF_PERF_EVENT_ERROR;13447	}13448	return LIBBPF_PERF_EVENT_CONT;13449}13450 13451static int perf_buffer__process_records(struct perf_buffer *pb,13452					struct perf_cpu_buf *cpu_buf)13453{13454	enum bpf_perf_event_ret ret;13455 13456	ret = perf_event_read_simple(cpu_buf->base, pb->mmap_size,13457				     pb->page_size, &cpu_buf->buf,13458				     &cpu_buf->buf_size,13459				     perf_buffer__process_record, cpu_buf);13460	if (ret != LIBBPF_PERF_EVENT_CONT)13461		return ret;13462	return 0;13463}13464 13465int perf_buffer__epoll_fd(const struct perf_buffer *pb)13466{13467	return pb->epoll_fd;13468}13469 13470int perf_buffer__poll(struct perf_buffer *pb, int timeout_ms)13471{13472	int i, cnt, err;13473 13474	cnt = epoll_wait(pb->epoll_fd, pb->events, pb->cpu_cnt, timeout_ms);13475	if (cnt < 0)13476		return -errno;13477 13478	for (i = 0; i < cnt; i++) {13479		struct perf_cpu_buf *cpu_buf = pb->events[i].data.ptr;13480 13481		err = perf_buffer__process_records(pb, cpu_buf);13482		if (err) {13483			pr_warn("error while processing records: %d\n", err);13484			return libbpf_err(err);13485		}13486	}13487	return cnt;13488}13489 13490/* Return number of PERF_EVENT_ARRAY map slots set up by this perf_buffer13491 * manager.13492 */13493size_t perf_buffer__buffer_cnt(const struct perf_buffer *pb)13494{13495	return pb->cpu_cnt;13496}13497 13498/*13499 * Return perf_event FD of a ring buffer in *buf_idx* slot of13500 * PERF_EVENT_ARRAY BPF map. This FD can be polled for new data using13501 * select()/poll()/epoll() Linux syscalls.13502 */13503int perf_buffer__buffer_fd(const struct perf_buffer *pb, size_t buf_idx)13504{13505	struct perf_cpu_buf *cpu_buf;13506 13507	if (buf_idx >= pb->cpu_cnt)13508		return libbpf_err(-EINVAL);13509 13510	cpu_buf = pb->cpu_bufs[buf_idx];13511	if (!cpu_buf)13512		return libbpf_err(-ENOENT);13513 13514	return cpu_buf->fd;13515}13516 13517int perf_buffer__buffer(struct perf_buffer *pb, int buf_idx, void **buf, size_t *buf_size)13518{13519	struct perf_cpu_buf *cpu_buf;13520 13521	if (buf_idx >= pb->cpu_cnt)13522		return libbpf_err(-EINVAL);13523 13524	cpu_buf = pb->cpu_bufs[buf_idx];13525	if (!cpu_buf)13526		return libbpf_err(-ENOENT);13527 13528	*buf = cpu_buf->base;13529	*buf_size = pb->mmap_size;13530	return 0;13531}13532 13533/*13534 * Consume data from perf ring buffer corresponding to slot *buf_idx* in13535 * PERF_EVENT_ARRAY BPF map without waiting/polling. If there is no data to13536 * consume, do nothing and return success.13537 * Returns:13538 *   - 0 on success;13539 *   - <0 on failure.13540 */13541int perf_buffer__consume_buffer(struct perf_buffer *pb, size_t buf_idx)13542{13543	struct perf_cpu_buf *cpu_buf;13544 13545	if (buf_idx >= pb->cpu_cnt)13546		return libbpf_err(-EINVAL);13547 13548	cpu_buf = pb->cpu_bufs[buf_idx];13549	if (!cpu_buf)13550		return libbpf_err(-ENOENT);13551 13552	return perf_buffer__process_records(pb, cpu_buf);13553}13554 13555int perf_buffer__consume(struct perf_buffer *pb)13556{13557	int i, err;13558 13559	for (i = 0; i < pb->cpu_cnt; i++) {13560		struct perf_cpu_buf *cpu_buf = pb->cpu_bufs[i];13561 13562		if (!cpu_buf)13563			continue;13564 13565		err = perf_buffer__process_records(pb, cpu_buf);13566		if (err) {13567			pr_warn("perf_buffer: failed to process records in buffer #%d: %d\n", i, err);13568			return libbpf_err(err);13569		}13570	}13571	return 0;13572}13573 13574int bpf_program__set_attach_target(struct bpf_program *prog,13575				   int attach_prog_fd,13576				   const char *attach_func_name)13577{13578	int btf_obj_fd = 0, btf_id = 0, err;13579 13580	if (!prog || attach_prog_fd < 0)13581		return libbpf_err(-EINVAL);13582 13583	if (prog->obj->loaded)13584		return libbpf_err(-EINVAL);13585 13586	if (attach_prog_fd && !attach_func_name) {13587		/* remember attach_prog_fd and let bpf_program__load() find13588		 * BTF ID during the program load13589		 */13590		prog->attach_prog_fd = attach_prog_fd;13591		return 0;13592	}13593 13594	if (attach_prog_fd) {13595		btf_id = libbpf_find_prog_btf_id(attach_func_name,13596						 attach_prog_fd);13597		if (btf_id < 0)13598			return libbpf_err(btf_id);13599	} else {13600		if (!attach_func_name)13601			return libbpf_err(-EINVAL);13602 13603		/* load btf_vmlinux, if not yet */13604		err = bpf_object__load_vmlinux_btf(prog->obj, true);13605		if (err)13606			return libbpf_err(err);13607		err = find_kernel_btf_id(prog->obj, attach_func_name,13608					 prog->expected_attach_type,13609					 &btf_obj_fd, &btf_id);13610		if (err)13611			return libbpf_err(err);13612	}13613 13614	prog->attach_btf_id = btf_id;13615	prog->attach_btf_obj_fd = btf_obj_fd;13616	prog->attach_prog_fd = attach_prog_fd;13617	return 0;13618}13619 13620int parse_cpu_mask_str(const char *s, bool **mask, int *mask_sz)13621{13622	int err = 0, n, len, start, end = -1;13623	bool *tmp;13624 13625	*mask = NULL;13626	*mask_sz = 0;13627 13628	/* Each sub string separated by ',' has format \d+-\d+ or \d+ */13629	while (*s) {13630		if (*s == ',' || *s == '\n') {13631			s++;13632			continue;13633		}13634		n = sscanf(s, "%d%n-%d%n", &start, &len, &end, &len);13635		if (n <= 0 || n > 2) {13636			pr_warn("Failed to get CPU range %s: %d\n", s, n);13637			err = -EINVAL;13638			goto cleanup;13639		} else if (n == 1) {13640			end = start;13641		}13642		if (start < 0 || start > end) {13643			pr_warn("Invalid CPU range [%d,%d] in %s\n",13644				start, end, s);13645			err = -EINVAL;13646			goto cleanup;13647		}13648		tmp = realloc(*mask, end + 1);13649		if (!tmp) {13650			err = -ENOMEM;13651			goto cleanup;13652		}13653		*mask = tmp;13654		memset(tmp + *mask_sz, 0, start - *mask_sz);13655		memset(tmp + start, 1, end - start + 1);13656		*mask_sz = end + 1;13657		s += len;13658	}13659	if (!*mask_sz) {13660		pr_warn("Empty CPU range\n");13661		return -EINVAL;13662	}13663	return 0;13664cleanup:13665	free(*mask);13666	*mask = NULL;13667	return err;13668}13669 13670int parse_cpu_mask_file(const char *fcpu, bool **mask, int *mask_sz)13671{13672	int fd, err = 0, len;13673	char buf[128];13674 13675	fd = open(fcpu, O_RDONLY | O_CLOEXEC);13676	if (fd < 0) {13677		err = -errno;13678		pr_warn("Failed to open cpu mask file %s: %d\n", fcpu, err);13679		return err;13680	}13681	len = read(fd, buf, sizeof(buf));13682	close(fd);13683	if (len <= 0) {13684		err = len ? -errno : -EINVAL;13685		pr_warn("Failed to read cpu mask from %s: %d\n", fcpu, err);13686		return err;13687	}13688	if (len >= sizeof(buf)) {13689		pr_warn("CPU mask is too big in file %s\n", fcpu);13690		return -E2BIG;13691	}13692	buf[len] = '\0';13693 13694	return parse_cpu_mask_str(buf, mask, mask_sz);13695}13696 13697int libbpf_num_possible_cpus(void)13698{13699	static const char *fcpu = "/sys/devices/system/cpu/possible";13700	static int cpus;13701	int err, n, i, tmp_cpus;13702	bool *mask;13703 13704	tmp_cpus = READ_ONCE(cpus);13705	if (tmp_cpus > 0)13706		return tmp_cpus;13707 13708	err = parse_cpu_mask_file(fcpu, &mask, &n);13709	if (err)13710		return libbpf_err(err);13711 13712	tmp_cpus = 0;13713	for (i = 0; i < n; i++) {13714		if (mask[i])13715			tmp_cpus++;13716	}13717	free(mask);13718 13719	WRITE_ONCE(cpus, tmp_cpus);13720	return tmp_cpus;13721}13722 13723static int populate_skeleton_maps(const struct bpf_object *obj,13724				  struct bpf_map_skeleton *maps,13725				  size_t map_cnt, size_t map_skel_sz)13726{13727	int i;13728 13729	for (i = 0; i < map_cnt; i++) {13730		struct bpf_map_skeleton *map_skel = (void *)maps + i * map_skel_sz;13731		struct bpf_map **map = map_skel->map;13732		const char *name = map_skel->name;13733		void **mmaped = map_skel->mmaped;13734 13735		*map = bpf_object__find_map_by_name(obj, name);13736		if (!*map) {13737			pr_warn("failed to find skeleton map '%s'\n", name);13738			return -ESRCH;13739		}13740 13741		/* externs shouldn't be pre-setup from user code */13742		if (mmaped && (*map)->libbpf_type != LIBBPF_MAP_KCONFIG)13743			*mmaped = (*map)->mmaped;13744	}13745	return 0;13746}13747 13748static int populate_skeleton_progs(const struct bpf_object *obj,13749				   struct bpf_prog_skeleton *progs,13750				   size_t prog_cnt, size_t prog_skel_sz)13751{13752	int i;13753 13754	for (i = 0; i < prog_cnt; i++) {13755		struct bpf_prog_skeleton *prog_skel = (void *)progs + i * prog_skel_sz;13756		struct bpf_program **prog = prog_skel->prog;13757		const char *name = prog_skel->name;13758 13759		*prog = bpf_object__find_program_by_name(obj, name);13760		if (!*prog) {13761			pr_warn("failed to find skeleton program '%s'\n", name);13762			return -ESRCH;13763		}13764	}13765	return 0;13766}13767 13768int bpf_object__open_skeleton(struct bpf_object_skeleton *s,13769			      const struct bpf_object_open_opts *opts)13770{13771	struct bpf_object *obj;13772	int err;13773 13774	obj = bpf_object_open(NULL, s->data, s->data_sz, s->name, opts);13775	if (IS_ERR(obj)) {13776		err = PTR_ERR(obj);13777		pr_warn("failed to initialize skeleton BPF object '%s': %d\n", s->name, err);13778		return libbpf_err(err);13779	}13780 13781	*s->obj = obj;13782	err = populate_skeleton_maps(obj, s->maps, s->map_cnt, s->map_skel_sz);13783	if (err) {13784		pr_warn("failed to populate skeleton maps for '%s': %d\n", s->name, err);13785		return libbpf_err(err);13786	}13787 13788	err = populate_skeleton_progs(obj, s->progs, s->prog_cnt, s->prog_skel_sz);13789	if (err) {13790		pr_warn("failed to populate skeleton progs for '%s': %d\n", s->name, err);13791		return libbpf_err(err);13792	}13793 13794	return 0;13795}13796 13797int bpf_object__open_subskeleton(struct bpf_object_subskeleton *s)13798{13799	int err, len, var_idx, i;13800	const char *var_name;13801	const struct bpf_map *map;13802	struct btf *btf;13803	__u32 map_type_id;13804	const struct btf_type *map_type, *var_type;13805	const struct bpf_var_skeleton *var_skel;13806	struct btf_var_secinfo *var;13807 13808	if (!s->obj)13809		return libbpf_err(-EINVAL);13810 13811	btf = bpf_object__btf(s->obj);13812	if (!btf) {13813		pr_warn("subskeletons require BTF at runtime (object %s)\n",13814			bpf_object__name(s->obj));13815		return libbpf_err(-errno);13816	}13817 13818	err = populate_skeleton_maps(s->obj, s->maps, s->map_cnt, s->map_skel_sz);13819	if (err) {13820		pr_warn("failed to populate subskeleton maps: %d\n", err);13821		return libbpf_err(err);13822	}13823 13824	err = populate_skeleton_progs(s->obj, s->progs, s->prog_cnt, s->prog_skel_sz);13825	if (err) {13826		pr_warn("failed to populate subskeleton maps: %d\n", err);13827		return libbpf_err(err);13828	}13829 13830	for (var_idx = 0; var_idx < s->var_cnt; var_idx++) {13831		var_skel = (void *)s->vars + var_idx * s->var_skel_sz;13832		map = *var_skel->map;13833		map_type_id = bpf_map__btf_value_type_id(map);13834		map_type = btf__type_by_id(btf, map_type_id);13835 13836		if (!btf_is_datasec(map_type)) {13837			pr_warn("type for map '%1$s' is not a datasec: %2$s",13838				bpf_map__name(map),13839				__btf_kind_str(btf_kind(map_type)));13840			return libbpf_err(-EINVAL);13841		}13842 13843		len = btf_vlen(map_type);13844		var = btf_var_secinfos(map_type);13845		for (i = 0; i < len; i++, var++) {13846			var_type = btf__type_by_id(btf, var->type);13847			var_name = btf__name_by_offset(btf, var_type->name_off);13848			if (strcmp(var_name, var_skel->name) == 0) {13849				*var_skel->addr = map->mmaped + var->offset;13850				break;13851			}13852		}13853	}13854	return 0;13855}13856 13857void bpf_object__destroy_subskeleton(struct bpf_object_subskeleton *s)13858{13859	if (!s)13860		return;13861	free(s->maps);13862	free(s->progs);13863	free(s->vars);13864	free(s);13865}13866 13867int bpf_object__load_skeleton(struct bpf_object_skeleton *s)13868{13869	int i, err;13870 13871	err = bpf_object__load(*s->obj);13872	if (err) {13873		pr_warn("failed to load BPF skeleton '%s': %d\n", s->name, err);13874		return libbpf_err(err);13875	}13876 13877	for (i = 0; i < s->map_cnt; i++) {13878		struct bpf_map_skeleton *map_skel = (void *)s->maps + i * s->map_skel_sz;13879		struct bpf_map *map = *map_skel->map;13880		size_t mmap_sz = bpf_map_mmap_sz(map);13881		int prot, map_fd = map->fd;13882		void **mmaped = map_skel->mmaped;13883 13884		if (!mmaped)13885			continue;13886 13887		if (!(map->def.map_flags & BPF_F_MMAPABLE)) {13888			*mmaped = NULL;13889			continue;13890		}13891 13892		if (map->def.type == BPF_MAP_TYPE_ARENA) {13893			*mmaped = map->mmaped;13894			continue;13895		}13896 13897		if (map->def.map_flags & BPF_F_RDONLY_PROG)13898			prot = PROT_READ;13899		else13900			prot = PROT_READ | PROT_WRITE;13901 13902		/* Remap anonymous mmap()-ed "map initialization image" as13903		 * a BPF map-backed mmap()-ed memory, but preserving the same13904		 * memory address. This will cause kernel to change process'13905		 * page table to point to a different piece of kernel memory,13906		 * but from userspace point of view memory address (and its13907		 * contents, being identical at this point) will stay the13908		 * same. This mapping will be released by bpf_object__close()13909		 * as per normal clean up procedure, so we don't need to worry13910		 * about it from skeleton's clean up perspective.13911		 */13912		*mmaped = mmap(map->mmaped, mmap_sz, prot, MAP_SHARED | MAP_FIXED, map_fd, 0);13913		if (*mmaped == MAP_FAILED) {13914			err = -errno;13915			*mmaped = NULL;13916			pr_warn("failed to re-mmap() map '%s': %d\n",13917				 bpf_map__name(map), err);13918			return libbpf_err(err);13919		}13920	}13921 13922	return 0;13923}13924 13925int bpf_object__attach_skeleton(struct bpf_object_skeleton *s)13926{13927	int i, err;13928 13929	for (i = 0; i < s->prog_cnt; i++) {13930		struct bpf_prog_skeleton *prog_skel = (void *)s->progs + i * s->prog_skel_sz;13931		struct bpf_program *prog = *prog_skel->prog;13932		struct bpf_link **link = prog_skel->link;13933 13934		if (!prog->autoload || !prog->autoattach)13935			continue;13936 13937		/* auto-attaching not supported for this program */13938		if (!prog->sec_def || !prog->sec_def->prog_attach_fn)13939			continue;13940 13941		/* if user already set the link manually, don't attempt auto-attach */13942		if (*link)13943			continue;13944 13945		err = prog->sec_def->prog_attach_fn(prog, prog->sec_def->cookie, link);13946		if (err) {13947			pr_warn("prog '%s': failed to auto-attach: %d\n",13948				bpf_program__name(prog), err);13949			return libbpf_err(err);13950		}13951 13952		/* It's possible that for some SEC() definitions auto-attach13953		 * is supported in some cases (e.g., if definition completely13954		 * specifies target information), but is not in other cases.13955		 * SEC("uprobe") is one such case. If user specified target13956		 * binary and function name, such BPF program can be13957		 * auto-attached. But if not, it shouldn't trigger skeleton's13958		 * attach to fail. It should just be skipped.13959		 * attach_fn signals such case with returning 0 (no error) and13960		 * setting link to NULL.13961		 */13962	}13963 13964 13965	for (i = 0; i < s->map_cnt; i++) {13966		struct bpf_map_skeleton *map_skel = (void *)s->maps + i * s->map_skel_sz;13967		struct bpf_map *map = *map_skel->map;13968		struct bpf_link **link;13969 13970		if (!map->autocreate || !map->autoattach)13971			continue;13972 13973		/* only struct_ops maps can be attached */13974		if (!bpf_map__is_struct_ops(map))13975			continue;13976 13977		/* skeleton is created with earlier version of bpftool, notify user */13978		if (s->map_skel_sz < offsetofend(struct bpf_map_skeleton, link)) {13979			pr_warn("map '%s': BPF skeleton version is old, skipping map auto-attachment...\n",13980				bpf_map__name(map));13981			continue;13982		}13983 13984		link = map_skel->link;13985		if (*link)13986			continue;13987 13988		*link = bpf_map__attach_struct_ops(map);13989		if (!*link) {13990			err = -errno;13991			pr_warn("map '%s': failed to auto-attach: %d\n", bpf_map__name(map), err);13992			return libbpf_err(err);13993		}13994	}13995 13996	return 0;13997}13998 13999void bpf_object__detach_skeleton(struct bpf_object_skeleton *s)14000{14001	int i;14002 14003	for (i = 0; i < s->prog_cnt; i++) {14004		struct bpf_prog_skeleton *prog_skel = (void *)s->progs + i * s->prog_skel_sz;14005		struct bpf_link **link = prog_skel->link;14006 14007		bpf_link__destroy(*link);14008		*link = NULL;14009	}14010 14011	if (s->map_skel_sz < sizeof(struct bpf_map_skeleton))14012		return;14013 14014	for (i = 0; i < s->map_cnt; i++) {14015		struct bpf_map_skeleton *map_skel = (void *)s->maps + i * s->map_skel_sz;14016		struct bpf_link **link = map_skel->link;14017 14018		if (link) {14019			bpf_link__destroy(*link);14020			*link = NULL;14021		}14022	}14023}14024 14025void bpf_object__destroy_skeleton(struct bpf_object_skeleton *s)14026{14027	if (!s)14028		return;14029 14030	bpf_object__detach_skeleton(s);14031	if (s->obj)14032		bpf_object__close(*s->obj);14033	free(s->maps);14034	free(s->progs);14035	free(s);14036}14037