brintos / linux-shallow public Read only

1/* SPDX-License-Identifier: GPL-2.0-only */2/* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com3 */4#ifndef _LINUX_BPF_H5#define _LINUX_BPF_H 16 7#include <uapi/linux/bpf.h>8#include <uapi/linux/filter.h>9 10#include <linux/workqueue.h>11#include <linux/file.h>12#include <linux/percpu.h>13#include <linux/err.h>14#include <linux/rbtree_latch.h>15#include <linux/numa.h>16#include <linux/mm_types.h>17#include <linux/wait.h>18#include <linux/refcount.h>19#include <linux/mutex.h>20#include <linux/module.h>21#include <linux/kallsyms.h>22#include <linux/capability.h>23#include <linux/sched/mm.h>24#include <linux/slab.h>25#include <linux/percpu-refcount.h>26#include <linux/stddef.h>27#include <linux/bpfptr.h>28#include <linux/btf.h>29#include <linux/rcupdate_trace.h>30#include <linux/static_call.h>31#include <linux/memcontrol.h>32#include <linux/cfi.h>33 34struct bpf_verifier_env;35struct bpf_verifier_log;36struct perf_event;37struct bpf_prog;38struct bpf_prog_aux;39struct bpf_map;40struct bpf_arena;41struct sock;42struct seq_file;43struct btf;44struct btf_type;45struct exception_table_entry;46struct seq_operations;47struct bpf_iter_aux_info;48struct bpf_local_storage;49struct bpf_local_storage_map;50struct kobject;51struct mem_cgroup;52struct module;53struct bpf_func_state;54struct ftrace_ops;55struct cgroup;56struct bpf_token;57struct user_namespace;58struct super_block;59struct inode;60 61extern struct idr btf_idr;62extern spinlock_t btf_idr_lock;63extern struct kobject *btf_kobj;64extern struct bpf_mem_alloc bpf_global_ma, bpf_global_percpu_ma;65extern bool bpf_global_ma_set;66 67typedef u64 (*bpf_callback_t)(u64, u64, u64, u64, u64);68typedef int (*bpf_iter_init_seq_priv_t)(void *private_data,69					struct bpf_iter_aux_info *aux);70typedef void (*bpf_iter_fini_seq_priv_t)(void *private_data);71typedef unsigned int (*bpf_func_t)(const void *,72				   const struct bpf_insn *);73struct bpf_iter_seq_info {74	const struct seq_operations *seq_ops;75	bpf_iter_init_seq_priv_t init_seq_private;76	bpf_iter_fini_seq_priv_t fini_seq_private;77	u32 seq_priv_size;78};79 80/* map is generic key/value storage optionally accessible by eBPF programs */81struct bpf_map_ops {82	/* funcs callable from userspace (via syscall) */83	int (*map_alloc_check)(union bpf_attr *attr);84	struct bpf_map *(*map_alloc)(union bpf_attr *attr);85	void (*map_release)(struct bpf_map *map, struct file *map_file);86	void (*map_free)(struct bpf_map *map);87	int (*map_get_next_key)(struct bpf_map *map, void *key, void *next_key);88	void (*map_release_uref)(struct bpf_map *map);89	void *(*map_lookup_elem_sys_only)(struct bpf_map *map, void *key);90	int (*map_lookup_batch)(struct bpf_map *map, const union bpf_attr *attr,91				union bpf_attr __user *uattr);92	int (*map_lookup_and_delete_elem)(struct bpf_map *map, void *key,93					  void *value, u64 flags);94	int (*map_lookup_and_delete_batch)(struct bpf_map *map,95					   const union bpf_attr *attr,96					   union bpf_attr __user *uattr);97	int (*map_update_batch)(struct bpf_map *map, struct file *map_file,98				const union bpf_attr *attr,99				union bpf_attr __user *uattr);100	int (*map_delete_batch)(struct bpf_map *map, const union bpf_attr *attr,101				union bpf_attr __user *uattr);102 103	/* funcs callable from userspace and from eBPF programs */104	void *(*map_lookup_elem)(struct bpf_map *map, void *key);105	long (*map_update_elem)(struct bpf_map *map, void *key, void *value, u64 flags);106	long (*map_delete_elem)(struct bpf_map *map, void *key);107	long (*map_push_elem)(struct bpf_map *map, void *value, u64 flags);108	long (*map_pop_elem)(struct bpf_map *map, void *value);109	long (*map_peek_elem)(struct bpf_map *map, void *value);110	void *(*map_lookup_percpu_elem)(struct bpf_map *map, void *key, u32 cpu);111 112	/* funcs called by prog_array and perf_event_array map */113	void *(*map_fd_get_ptr)(struct bpf_map *map, struct file *map_file,114				int fd);115	/* If need_defer is true, the implementation should guarantee that116	 * the to-be-put element is still alive before the bpf program, which117	 * may manipulate it, exists.118	 */119	void (*map_fd_put_ptr)(struct bpf_map *map, void *ptr, bool need_defer);120	int (*map_gen_lookup)(struct bpf_map *map, struct bpf_insn *insn_buf);121	u32 (*map_fd_sys_lookup_elem)(void *ptr);122	void (*map_seq_show_elem)(struct bpf_map *map, void *key,123				  struct seq_file *m);124	int (*map_check_btf)(const struct bpf_map *map,125			     const struct btf *btf,126			     const struct btf_type *key_type,127			     const struct btf_type *value_type);128 129	/* Prog poke tracking helpers. */130	int (*map_poke_track)(struct bpf_map *map, struct bpf_prog_aux *aux);131	void (*map_poke_untrack)(struct bpf_map *map, struct bpf_prog_aux *aux);132	void (*map_poke_run)(struct bpf_map *map, u32 key, struct bpf_prog *old,133			     struct bpf_prog *new);134 135	/* Direct value access helpers. */136	int (*map_direct_value_addr)(const struct bpf_map *map,137				     u64 *imm, u32 off);138	int (*map_direct_value_meta)(const struct bpf_map *map,139				     u64 imm, u32 *off);140	int (*map_mmap)(struct bpf_map *map, struct vm_area_struct *vma);141	__poll_t (*map_poll)(struct bpf_map *map, struct file *filp,142			     struct poll_table_struct *pts);143	unsigned long (*map_get_unmapped_area)(struct file *filep, unsigned long addr,144					       unsigned long len, unsigned long pgoff,145					       unsigned long flags);146 147	/* Functions called by bpf_local_storage maps */148	int (*map_local_storage_charge)(struct bpf_local_storage_map *smap,149					void *owner, u32 size);150	void (*map_local_storage_uncharge)(struct bpf_local_storage_map *smap,151					   void *owner, u32 size);152	struct bpf_local_storage __rcu ** (*map_owner_storage_ptr)(void *owner);153 154	/* Misc helpers.*/155	long (*map_redirect)(struct bpf_map *map, u64 key, u64 flags);156 157	/* map_meta_equal must be implemented for maps that can be158	 * used as an inner map.  It is a runtime check to ensure159	 * an inner map can be inserted to an outer map.160	 *161	 * Some properties of the inner map has been used during the162	 * verification time.  When inserting an inner map at the runtime,163	 * map_meta_equal has to ensure the inserting map has the same164	 * properties that the verifier has used earlier.165	 */166	bool (*map_meta_equal)(const struct bpf_map *meta0,167			       const struct bpf_map *meta1);168 169 170	int (*map_set_for_each_callback_args)(struct bpf_verifier_env *env,171					      struct bpf_func_state *caller,172					      struct bpf_func_state *callee);173	long (*map_for_each_callback)(struct bpf_map *map,174				     bpf_callback_t callback_fn,175				     void *callback_ctx, u64 flags);176 177	u64 (*map_mem_usage)(const struct bpf_map *map);178 179	/* BTF id of struct allocated by map_alloc */180	int *map_btf_id;181 182	/* bpf_iter info used to open a seq_file */183	const struct bpf_iter_seq_info *iter_seq_info;184};185 186enum {187	/* Support at most 11 fields in a BTF type */188	BTF_FIELDS_MAX	   = 11,189};190 191enum btf_field_type {192	BPF_SPIN_LOCK  = (1 << 0),193	BPF_TIMER      = (1 << 1),194	BPF_KPTR_UNREF = (1 << 2),195	BPF_KPTR_REF   = (1 << 3),196	BPF_KPTR_PERCPU = (1 << 4),197	BPF_KPTR       = BPF_KPTR_UNREF | BPF_KPTR_REF | BPF_KPTR_PERCPU,198	BPF_LIST_HEAD  = (1 << 5),199	BPF_LIST_NODE  = (1 << 6),200	BPF_RB_ROOT    = (1 << 7),201	BPF_RB_NODE    = (1 << 8),202	BPF_GRAPH_NODE = BPF_RB_NODE | BPF_LIST_NODE,203	BPF_GRAPH_ROOT = BPF_RB_ROOT | BPF_LIST_HEAD,204	BPF_REFCOUNT   = (1 << 9),205	BPF_WORKQUEUE  = (1 << 10),206};207 208typedef void (*btf_dtor_kfunc_t)(void *);209 210struct btf_field_kptr {211	struct btf *btf;212	struct module *module;213	/* dtor used if btf_is_kernel(btf), otherwise the type is214	 * program-allocated, dtor is NULL,  and __bpf_obj_drop_impl is used215	 */216	btf_dtor_kfunc_t dtor;217	u32 btf_id;218};219 220struct btf_field_graph_root {221	struct btf *btf;222	u32 value_btf_id;223	u32 node_offset;224	struct btf_record *value_rec;225};226 227struct btf_field {228	u32 offset;229	u32 size;230	enum btf_field_type type;231	union {232		struct btf_field_kptr kptr;233		struct btf_field_graph_root graph_root;234	};235};236 237struct btf_record {238	u32 cnt;239	u32 field_mask;240	int spin_lock_off;241	int timer_off;242	int wq_off;243	int refcount_off;244	struct btf_field fields[];245};246 247/* Non-opaque version of bpf_rb_node in uapi/linux/bpf.h */248struct bpf_rb_node_kern {249	struct rb_node rb_node;250	void *owner;251} __attribute__((aligned(8)));252 253/* Non-opaque version of bpf_list_node in uapi/linux/bpf.h */254struct bpf_list_node_kern {255	struct list_head list_head;256	void *owner;257} __attribute__((aligned(8)));258 259struct bpf_map {260	const struct bpf_map_ops *ops;261	struct bpf_map *inner_map_meta;262#ifdef CONFIG_SECURITY263	void *security;264#endif265	enum bpf_map_type map_type;266	u32 key_size;267	u32 value_size;268	u32 max_entries;269	u64 map_extra; /* any per-map-type extra fields */270	u32 map_flags;271	u32 id;272	struct btf_record *record;273	int numa_node;274	u32 btf_key_type_id;275	u32 btf_value_type_id;276	u32 btf_vmlinux_value_type_id;277	struct btf *btf;278#ifdef CONFIG_MEMCG279	struct obj_cgroup *objcg;280#endif281	char name[BPF_OBJ_NAME_LEN];282	struct mutex freeze_mutex;283	atomic64_t refcnt;284	atomic64_t usercnt;285	/* rcu is used before freeing and work is only used during freeing */286	union {287		struct work_struct work;288		struct rcu_head rcu;289	};290	atomic64_t writecnt;291	/* 'Ownership' of program-containing map is claimed by the first program292	 * that is going to use this map or by the first program which FD is293	 * stored in the map to make sure that all callers and callees have the294	 * same prog type, JITed flag and xdp_has_frags flag.295	 */296	struct {297		const struct btf_type *attach_func_proto;298		spinlock_t lock;299		enum bpf_prog_type type;300		bool jited;301		bool xdp_has_frags;302	} owner;303	bool bypass_spec_v1;304	bool frozen; /* write-once; write-protected by freeze_mutex */305	bool free_after_mult_rcu_gp;306	bool free_after_rcu_gp;307	atomic64_t sleepable_refcnt;308	s64 __percpu *elem_count;309};310 311static inline const char *btf_field_type_name(enum btf_field_type type)312{313	switch (type) {314	case BPF_SPIN_LOCK:315		return "bpf_spin_lock";316	case BPF_TIMER:317		return "bpf_timer";318	case BPF_WORKQUEUE:319		return "bpf_wq";320	case BPF_KPTR_UNREF:321	case BPF_KPTR_REF:322		return "kptr";323	case BPF_KPTR_PERCPU:324		return "percpu_kptr";325	case BPF_LIST_HEAD:326		return "bpf_list_head";327	case BPF_LIST_NODE:328		return "bpf_list_node";329	case BPF_RB_ROOT:330		return "bpf_rb_root";331	case BPF_RB_NODE:332		return "bpf_rb_node";333	case BPF_REFCOUNT:334		return "bpf_refcount";335	default:336		WARN_ON_ONCE(1);337		return "unknown";338	}339}340 341static inline u32 btf_field_type_size(enum btf_field_type type)342{343	switch (type) {344	case BPF_SPIN_LOCK:345		return sizeof(struct bpf_spin_lock);346	case BPF_TIMER:347		return sizeof(struct bpf_timer);348	case BPF_WORKQUEUE:349		return sizeof(struct bpf_wq);350	case BPF_KPTR_UNREF:351	case BPF_KPTR_REF:352	case BPF_KPTR_PERCPU:353		return sizeof(u64);354	case BPF_LIST_HEAD:355		return sizeof(struct bpf_list_head);356	case BPF_LIST_NODE:357		return sizeof(struct bpf_list_node);358	case BPF_RB_ROOT:359		return sizeof(struct bpf_rb_root);360	case BPF_RB_NODE:361		return sizeof(struct bpf_rb_node);362	case BPF_REFCOUNT:363		return sizeof(struct bpf_refcount);364	default:365		WARN_ON_ONCE(1);366		return 0;367	}368}369 370static inline u32 btf_field_type_align(enum btf_field_type type)371{372	switch (type) {373	case BPF_SPIN_LOCK:374		return __alignof__(struct bpf_spin_lock);375	case BPF_TIMER:376		return __alignof__(struct bpf_timer);377	case BPF_WORKQUEUE:378		return __alignof__(struct bpf_wq);379	case BPF_KPTR_UNREF:380	case BPF_KPTR_REF:381	case BPF_KPTR_PERCPU:382		return __alignof__(u64);383	case BPF_LIST_HEAD:384		return __alignof__(struct bpf_list_head);385	case BPF_LIST_NODE:386		return __alignof__(struct bpf_list_node);387	case BPF_RB_ROOT:388		return __alignof__(struct bpf_rb_root);389	case BPF_RB_NODE:390		return __alignof__(struct bpf_rb_node);391	case BPF_REFCOUNT:392		return __alignof__(struct bpf_refcount);393	default:394		WARN_ON_ONCE(1);395		return 0;396	}397}398 399static inline void bpf_obj_init_field(const struct btf_field *field, void *addr)400{401	memset(addr, 0, field->size);402 403	switch (field->type) {404	case BPF_REFCOUNT:405		refcount_set((refcount_t *)addr, 1);406		break;407	case BPF_RB_NODE:408		RB_CLEAR_NODE((struct rb_node *)addr);409		break;410	case BPF_LIST_HEAD:411	case BPF_LIST_NODE:412		INIT_LIST_HEAD((struct list_head *)addr);413		break;414	case BPF_RB_ROOT:415		/* RB_ROOT_CACHED 0-inits, no need to do anything after memset */416	case BPF_SPIN_LOCK:417	case BPF_TIMER:418	case BPF_WORKQUEUE:419	case BPF_KPTR_UNREF:420	case BPF_KPTR_REF:421	case BPF_KPTR_PERCPU:422		break;423	default:424		WARN_ON_ONCE(1);425		return;426	}427}428 429static inline bool btf_record_has_field(const struct btf_record *rec, enum btf_field_type type)430{431	if (IS_ERR_OR_NULL(rec))432		return false;433	return rec->field_mask & type;434}435 436static inline void bpf_obj_init(const struct btf_record *rec, void *obj)437{438	int i;439 440	if (IS_ERR_OR_NULL(rec))441		return;442	for (i = 0; i < rec->cnt; i++)443		bpf_obj_init_field(&rec->fields[i], obj + rec->fields[i].offset);444}445 446/* 'dst' must be a temporary buffer and should not point to memory that is being447 * used in parallel by a bpf program or bpf syscall, otherwise the access from448 * the bpf program or bpf syscall may be corrupted by the reinitialization,449 * leading to weird problems. Even 'dst' is newly-allocated from bpf memory450 * allocator, it is still possible for 'dst' to be used in parallel by a bpf451 * program or bpf syscall.452 */453static inline void check_and_init_map_value(struct bpf_map *map, void *dst)454{455	bpf_obj_init(map->record, dst);456}457 458/* memcpy that is used with 8-byte aligned pointers, power-of-8 size and459 * forced to use 'long' read/writes to try to atomically copy long counters.460 * Best-effort only.  No barriers here, since it _will_ race with concurrent461 * updates from BPF programs. Called from bpf syscall and mostly used with462 * size 8 or 16 bytes, so ask compiler to inline it.463 */464static inline void bpf_long_memcpy(void *dst, const void *src, u32 size)465{466	const long *lsrc = src;467	long *ldst = dst;468 469	size /= sizeof(long);470	while (size--)471		data_race(*ldst++ = *lsrc++);472}473 474/* copy everything but bpf_spin_lock, bpf_timer, and kptrs. There could be one of each. */475static inline void bpf_obj_memcpy(struct btf_record *rec,476				  void *dst, void *src, u32 size,477				  bool long_memcpy)478{479	u32 curr_off = 0;480	int i;481 482	if (IS_ERR_OR_NULL(rec)) {483		if (long_memcpy)484			bpf_long_memcpy(dst, src, round_up(size, 8));485		else486			memcpy(dst, src, size);487		return;488	}489 490	for (i = 0; i < rec->cnt; i++) {491		u32 next_off = rec->fields[i].offset;492		u32 sz = next_off - curr_off;493 494		memcpy(dst + curr_off, src + curr_off, sz);495		curr_off += rec->fields[i].size + sz;496	}497	memcpy(dst + curr_off, src + curr_off, size - curr_off);498}499 500static inline void copy_map_value(struct bpf_map *map, void *dst, void *src)501{502	bpf_obj_memcpy(map->record, dst, src, map->value_size, false);503}504 505static inline void copy_map_value_long(struct bpf_map *map, void *dst, void *src)506{507	bpf_obj_memcpy(map->record, dst, src, map->value_size, true);508}509 510static inline void bpf_obj_memzero(struct btf_record *rec, void *dst, u32 size)511{512	u32 curr_off = 0;513	int i;514 515	if (IS_ERR_OR_NULL(rec)) {516		memset(dst, 0, size);517		return;518	}519 520	for (i = 0; i < rec->cnt; i++) {521		u32 next_off = rec->fields[i].offset;522		u32 sz = next_off - curr_off;523 524		memset(dst + curr_off, 0, sz);525		curr_off += rec->fields[i].size + sz;526	}527	memset(dst + curr_off, 0, size - curr_off);528}529 530static inline void zero_map_value(struct bpf_map *map, void *dst)531{532	bpf_obj_memzero(map->record, dst, map->value_size);533}534 535void copy_map_value_locked(struct bpf_map *map, void *dst, void *src,536			   bool lock_src);537void bpf_timer_cancel_and_free(void *timer);538void bpf_wq_cancel_and_free(void *timer);539void bpf_list_head_free(const struct btf_field *field, void *list_head,540			struct bpf_spin_lock *spin_lock);541void bpf_rb_root_free(const struct btf_field *field, void *rb_root,542		      struct bpf_spin_lock *spin_lock);543u64 bpf_arena_get_kern_vm_start(struct bpf_arena *arena);544u64 bpf_arena_get_user_vm_start(struct bpf_arena *arena);545int bpf_obj_name_cpy(char *dst, const char *src, unsigned int size);546 547struct bpf_offload_dev;548struct bpf_offloaded_map;549 550struct bpf_map_dev_ops {551	int (*map_get_next_key)(struct bpf_offloaded_map *map,552				void *key, void *next_key);553	int (*map_lookup_elem)(struct bpf_offloaded_map *map,554			       void *key, void *value);555	int (*map_update_elem)(struct bpf_offloaded_map *map,556			       void *key, void *value, u64 flags);557	int (*map_delete_elem)(struct bpf_offloaded_map *map, void *key);558};559 560struct bpf_offloaded_map {561	struct bpf_map map;562	struct net_device *netdev;563	const struct bpf_map_dev_ops *dev_ops;564	void *dev_priv;565	struct list_head offloads;566};567 568static inline struct bpf_offloaded_map *map_to_offmap(struct bpf_map *map)569{570	return container_of(map, struct bpf_offloaded_map, map);571}572 573static inline bool bpf_map_offload_neutral(const struct bpf_map *map)574{575	return map->map_type == BPF_MAP_TYPE_PERF_EVENT_ARRAY;576}577 578static inline bool bpf_map_support_seq_show(const struct bpf_map *map)579{580	return (map->btf_value_type_id || map->btf_vmlinux_value_type_id) &&581		map->ops->map_seq_show_elem;582}583 584int map_check_no_btf(const struct bpf_map *map,585		     const struct btf *btf,586		     const struct btf_type *key_type,587		     const struct btf_type *value_type);588 589bool bpf_map_meta_equal(const struct bpf_map *meta0,590			const struct bpf_map *meta1);591 592extern const struct bpf_map_ops bpf_map_offload_ops;593 594/* bpf_type_flag contains a set of flags that are applicable to the values of595 * arg_type, ret_type and reg_type. For example, a pointer value may be null,596 * or a memory is read-only. We classify types into two categories: base types597 * and extended types. Extended types are base types combined with a type flag.598 *599 * Currently there are no more than 32 base types in arg_type, ret_type and600 * reg_types.601 */602#define BPF_BASE_TYPE_BITS	8603 604enum bpf_type_flag {605	/* PTR may be NULL. */606	PTR_MAYBE_NULL		= BIT(0 + BPF_BASE_TYPE_BITS),607 608	/* MEM is read-only. When applied on bpf_arg, it indicates the arg is609	 * compatible with both mutable and immutable memory.610	 */611	MEM_RDONLY		= BIT(1 + BPF_BASE_TYPE_BITS),612 613	/* MEM points to BPF ring buffer reservation. */614	MEM_RINGBUF		= BIT(2 + BPF_BASE_TYPE_BITS),615 616	/* MEM is in user address space. */617	MEM_USER		= BIT(3 + BPF_BASE_TYPE_BITS),618 619	/* MEM is a percpu memory. MEM_PERCPU tags PTR_TO_BTF_ID. When tagged620	 * with MEM_PERCPU, PTR_TO_BTF_ID _cannot_ be directly accessed. In621	 * order to drop this tag, it must be passed into bpf_per_cpu_ptr()622	 * or bpf_this_cpu_ptr(), which will return the pointer corresponding623	 * to the specified cpu.624	 */625	MEM_PERCPU		= BIT(4 + BPF_BASE_TYPE_BITS),626 627	/* Indicates that the argument will be released. */628	OBJ_RELEASE		= BIT(5 + BPF_BASE_TYPE_BITS),629 630	/* PTR is not trusted. This is only used with PTR_TO_BTF_ID, to mark631	 * unreferenced and referenced kptr loaded from map value using a load632	 * instruction, so that they can only be dereferenced but not escape the633	 * BPF program into the kernel (i.e. cannot be passed as arguments to634	 * kfunc or bpf helpers).635	 */636	PTR_UNTRUSTED		= BIT(6 + BPF_BASE_TYPE_BITS),637 638	/* MEM can be uninitialized. */639	MEM_UNINIT		= BIT(7 + BPF_BASE_TYPE_BITS),640 641	/* DYNPTR points to memory local to the bpf program. */642	DYNPTR_TYPE_LOCAL	= BIT(8 + BPF_BASE_TYPE_BITS),643 644	/* DYNPTR points to a kernel-produced ringbuf record. */645	DYNPTR_TYPE_RINGBUF	= BIT(9 + BPF_BASE_TYPE_BITS),646 647	/* Size is known at compile time. */648	MEM_FIXED_SIZE		= BIT(10 + BPF_BASE_TYPE_BITS),649 650	/* MEM is of an allocated object of type in program BTF. This is used to651	 * tag PTR_TO_BTF_ID allocated using bpf_obj_new.652	 */653	MEM_ALLOC		= BIT(11 + BPF_BASE_TYPE_BITS),654 655	/* PTR was passed from the kernel in a trusted context, and may be656	 * passed to KF_TRUSTED_ARGS kfuncs or BPF helper functions.657	 * Confusingly, this is _not_ the opposite of PTR_UNTRUSTED above.658	 * PTR_UNTRUSTED refers to a kptr that was read directly from a map659	 * without invoking bpf_kptr_xchg(). What we really need to know is660	 * whether a pointer is safe to pass to a kfunc or BPF helper function.661	 * While PTR_UNTRUSTED pointers are unsafe to pass to kfuncs and BPF662	 * helpers, they do not cover all possible instances of unsafe663	 * pointers. For example, a pointer that was obtained from walking a664	 * struct will _not_ get the PTR_UNTRUSTED type modifier, despite the665	 * fact that it may be NULL, invalid, etc. This is due to backwards666	 * compatibility requirements, as this was the behavior that was first667	 * introduced when kptrs were added. The behavior is now considered668	 * deprecated, and PTR_UNTRUSTED will eventually be removed.669	 *670	 * PTR_TRUSTED, on the other hand, is a pointer that the kernel671	 * guarantees to be valid and safe to pass to kfuncs and BPF helpers.672	 * For example, pointers passed to tracepoint arguments are considered673	 * PTR_TRUSTED, as are pointers that are passed to struct_ops674	 * callbacks. As alluded to above, pointers that are obtained from675	 * walking PTR_TRUSTED pointers are _not_ trusted. For example, if a676	 * struct task_struct *task is PTR_TRUSTED, then accessing677	 * task->last_wakee will lose the PTR_TRUSTED modifier when it's stored678	 * in a BPF register. Similarly, pointers passed to certain programs679	 * types such as kretprobes are not guaranteed to be valid, as they may680	 * for example contain an object that was recently freed.681	 */682	PTR_TRUSTED		= BIT(12 + BPF_BASE_TYPE_BITS),683 684	/* MEM is tagged with rcu and memory access needs rcu_read_lock protection. */685	MEM_RCU			= BIT(13 + BPF_BASE_TYPE_BITS),686 687	/* Used to tag PTR_TO_BTF_ID | MEM_ALLOC references which are non-owning.688	 * Currently only valid for linked-list and rbtree nodes. If the nodes689	 * have a bpf_refcount_field, they must be tagged MEM_RCU as well.690	 */691	NON_OWN_REF		= BIT(14 + BPF_BASE_TYPE_BITS),692 693	/* DYNPTR points to sk_buff */694	DYNPTR_TYPE_SKB		= BIT(15 + BPF_BASE_TYPE_BITS),695 696	/* DYNPTR points to xdp_buff */697	DYNPTR_TYPE_XDP		= BIT(16 + BPF_BASE_TYPE_BITS),698 699	/* Memory must be aligned on some architectures, used in combination with700	 * MEM_FIXED_SIZE.701	 */702	MEM_ALIGNED		= BIT(17 + BPF_BASE_TYPE_BITS),703 704	/* MEM is being written to, often combined with MEM_UNINIT. Non-presence705	 * of MEM_WRITE means that MEM is only being read. MEM_WRITE without the706	 * MEM_UNINIT means that memory needs to be initialized since it is also707	 * read.708	 */709	MEM_WRITE		= BIT(18 + BPF_BASE_TYPE_BITS),710 711	__BPF_TYPE_FLAG_MAX,712	__BPF_TYPE_LAST_FLAG	= __BPF_TYPE_FLAG_MAX - 1,713};714 715#define DYNPTR_TYPE_FLAG_MASK	(DYNPTR_TYPE_LOCAL | DYNPTR_TYPE_RINGBUF | DYNPTR_TYPE_SKB \716				 | DYNPTR_TYPE_XDP)717 718/* Max number of base types. */719#define BPF_BASE_TYPE_LIMIT	(1UL << BPF_BASE_TYPE_BITS)720 721/* Max number of all types. */722#define BPF_TYPE_LIMIT		(__BPF_TYPE_LAST_FLAG | (__BPF_TYPE_LAST_FLAG - 1))723 724/* function argument constraints */725enum bpf_arg_type {726	ARG_DONTCARE = 0,	/* unused argument in helper function */727 728	/* the following constraints used to prototype729	 * bpf_map_lookup/update/delete_elem() functions730	 */731	ARG_CONST_MAP_PTR,	/* const argument used as pointer to bpf_map */732	ARG_PTR_TO_MAP_KEY,	/* pointer to stack used as map key */733	ARG_PTR_TO_MAP_VALUE,	/* pointer to stack used as map value */734 735	/* Used to prototype bpf_memcmp() and other functions that access data736	 * on eBPF program stack737	 */738	ARG_PTR_TO_MEM,		/* pointer to valid memory (stack, packet, map value) */739	ARG_PTR_TO_ARENA,740 741	ARG_CONST_SIZE,		/* number of bytes accessed from memory */742	ARG_CONST_SIZE_OR_ZERO,	/* number of bytes accessed from memory or 0 */743 744	ARG_PTR_TO_CTX,		/* pointer to context */745	ARG_ANYTHING,		/* any (initialized) argument is ok */746	ARG_PTR_TO_SPIN_LOCK,	/* pointer to bpf_spin_lock */747	ARG_PTR_TO_SOCK_COMMON,	/* pointer to sock_common */748	ARG_PTR_TO_SOCKET,	/* pointer to bpf_sock (fullsock) */749	ARG_PTR_TO_BTF_ID,	/* pointer to in-kernel struct */750	ARG_PTR_TO_RINGBUF_MEM,	/* pointer to dynamically reserved ringbuf memory */751	ARG_CONST_ALLOC_SIZE_OR_ZERO,	/* number of allocated bytes requested */752	ARG_PTR_TO_BTF_ID_SOCK_COMMON,	/* pointer to in-kernel sock_common or bpf-mirrored bpf_sock */753	ARG_PTR_TO_PERCPU_BTF_ID,	/* pointer to in-kernel percpu type */754	ARG_PTR_TO_FUNC,	/* pointer to a bpf program function */755	ARG_PTR_TO_STACK,	/* pointer to stack */756	ARG_PTR_TO_CONST_STR,	/* pointer to a null terminated read-only string */757	ARG_PTR_TO_TIMER,	/* pointer to bpf_timer */758	ARG_KPTR_XCHG_DEST,	/* pointer to destination that kptrs are bpf_kptr_xchg'd into */759	ARG_PTR_TO_DYNPTR,      /* pointer to bpf_dynptr. See bpf_type_flag for dynptr type */760	__BPF_ARG_TYPE_MAX,761 762	/* Extended arg_types. */763	ARG_PTR_TO_MAP_VALUE_OR_NULL	= PTR_MAYBE_NULL | ARG_PTR_TO_MAP_VALUE,764	ARG_PTR_TO_MEM_OR_NULL		= PTR_MAYBE_NULL | ARG_PTR_TO_MEM,765	ARG_PTR_TO_CTX_OR_NULL		= PTR_MAYBE_NULL | ARG_PTR_TO_CTX,766	ARG_PTR_TO_SOCKET_OR_NULL	= PTR_MAYBE_NULL | ARG_PTR_TO_SOCKET,767	ARG_PTR_TO_STACK_OR_NULL	= PTR_MAYBE_NULL | ARG_PTR_TO_STACK,768	ARG_PTR_TO_BTF_ID_OR_NULL	= PTR_MAYBE_NULL | ARG_PTR_TO_BTF_ID,769	/* Pointer to memory does not need to be initialized, since helper function770	 * fills all bytes or clears them in error case.771	 */772	ARG_PTR_TO_UNINIT_MEM		= MEM_UNINIT | MEM_WRITE | ARG_PTR_TO_MEM,773	/* Pointer to valid memory of size known at compile time. */774	ARG_PTR_TO_FIXED_SIZE_MEM	= MEM_FIXED_SIZE | ARG_PTR_TO_MEM,775 776	/* This must be the last entry. Its purpose is to ensure the enum is777	 * wide enough to hold the higher bits reserved for bpf_type_flag.778	 */779	__BPF_ARG_TYPE_LIMIT	= BPF_TYPE_LIMIT,780};781static_assert(__BPF_ARG_TYPE_MAX <= BPF_BASE_TYPE_LIMIT);782 783/* type of values returned from helper functions */784enum bpf_return_type {785	RET_INTEGER,			/* function returns integer */786	RET_VOID,			/* function doesn't return anything */787	RET_PTR_TO_MAP_VALUE,		/* returns a pointer to map elem value */788	RET_PTR_TO_SOCKET,		/* returns a pointer to a socket */789	RET_PTR_TO_TCP_SOCK,		/* returns a pointer to a tcp_sock */790	RET_PTR_TO_SOCK_COMMON,		/* returns a pointer to a sock_common */791	RET_PTR_TO_MEM,			/* returns a pointer to memory */792	RET_PTR_TO_MEM_OR_BTF_ID,	/* returns a pointer to a valid memory or a btf_id */793	RET_PTR_TO_BTF_ID,		/* returns a pointer to a btf_id */794	__BPF_RET_TYPE_MAX,795 796	/* Extended ret_types. */797	RET_PTR_TO_MAP_VALUE_OR_NULL	= PTR_MAYBE_NULL | RET_PTR_TO_MAP_VALUE,798	RET_PTR_TO_SOCKET_OR_NULL	= PTR_MAYBE_NULL | RET_PTR_TO_SOCKET,799	RET_PTR_TO_TCP_SOCK_OR_NULL	= PTR_MAYBE_NULL | RET_PTR_TO_TCP_SOCK,800	RET_PTR_TO_SOCK_COMMON_OR_NULL	= PTR_MAYBE_NULL | RET_PTR_TO_SOCK_COMMON,801	RET_PTR_TO_RINGBUF_MEM_OR_NULL	= PTR_MAYBE_NULL | MEM_RINGBUF | RET_PTR_TO_MEM,802	RET_PTR_TO_DYNPTR_MEM_OR_NULL	= PTR_MAYBE_NULL | RET_PTR_TO_MEM,803	RET_PTR_TO_BTF_ID_OR_NULL	= PTR_MAYBE_NULL | RET_PTR_TO_BTF_ID,804	RET_PTR_TO_BTF_ID_TRUSTED	= PTR_TRUSTED	 | RET_PTR_TO_BTF_ID,805 806	/* This must be the last entry. Its purpose is to ensure the enum is807	 * wide enough to hold the higher bits reserved for bpf_type_flag.808	 */809	__BPF_RET_TYPE_LIMIT	= BPF_TYPE_LIMIT,810};811static_assert(__BPF_RET_TYPE_MAX <= BPF_BASE_TYPE_LIMIT);812 813/* eBPF function prototype used by verifier to allow BPF_CALLs from eBPF programs814 * to in-kernel helper functions and for adjusting imm32 field in BPF_CALL815 * instructions after verifying816 */817struct bpf_func_proto {818	u64 (*func)(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5);819	bool gpl_only;820	bool pkt_access;821	bool might_sleep;822	/* set to true if helper follows contract for llvm823	 * attribute bpf_fastcall:824	 * - void functions do not scratch r0825	 * - functions taking N arguments scratch only registers r1-rN826	 */827	bool allow_fastcall;828	enum bpf_return_type ret_type;829	union {830		struct {831			enum bpf_arg_type arg1_type;832			enum bpf_arg_type arg2_type;833			enum bpf_arg_type arg3_type;834			enum bpf_arg_type arg4_type;835			enum bpf_arg_type arg5_type;836		};837		enum bpf_arg_type arg_type[5];838	};839	union {840		struct {841			u32 *arg1_btf_id;842			u32 *arg2_btf_id;843			u32 *arg3_btf_id;844			u32 *arg4_btf_id;845			u32 *arg5_btf_id;846		};847		u32 *arg_btf_id[5];848		struct {849			size_t arg1_size;850			size_t arg2_size;851			size_t arg3_size;852			size_t arg4_size;853			size_t arg5_size;854		};855		size_t arg_size[5];856	};857	int *ret_btf_id; /* return value btf_id */858	bool (*allowed)(const struct bpf_prog *prog);859};860 861/* bpf_context is intentionally undefined structure. Pointer to bpf_context is862 * the first argument to eBPF programs.863 * For socket filters: 'struct bpf_context *' == 'struct sk_buff *'864 */865struct bpf_context;866 867enum bpf_access_type {868	BPF_READ = 1,869	BPF_WRITE = 2870};871 872/* types of values stored in eBPF registers */873/* Pointer types represent:874 * pointer875 * pointer + imm876 * pointer + (u16) var877 * pointer + (u16) var + imm878 * if (range > 0) then [ptr, ptr + range - off) is safe to access879 * if (id > 0) means that some 'var' was added880 * if (off > 0) means that 'imm' was added881 */882enum bpf_reg_type {883	NOT_INIT = 0,		 /* nothing was written into register */884	SCALAR_VALUE,		 /* reg doesn't contain a valid pointer */885	PTR_TO_CTX,		 /* reg points to bpf_context */886	CONST_PTR_TO_MAP,	 /* reg points to struct bpf_map */887	PTR_TO_MAP_VALUE,	 /* reg points to map element value */888	PTR_TO_MAP_KEY,		 /* reg points to a map element key */889	PTR_TO_STACK,		 /* reg == frame_pointer + offset */890	PTR_TO_PACKET_META,	 /* skb->data - meta_len */891	PTR_TO_PACKET,		 /* reg points to skb->data */892	PTR_TO_PACKET_END,	 /* skb->data + headlen */893	PTR_TO_FLOW_KEYS,	 /* reg points to bpf_flow_keys */894	PTR_TO_SOCKET,		 /* reg points to struct bpf_sock */895	PTR_TO_SOCK_COMMON,	 /* reg points to sock_common */896	PTR_TO_TCP_SOCK,	 /* reg points to struct tcp_sock */897	PTR_TO_TP_BUFFER,	 /* reg points to a writable raw tp's buffer */898	PTR_TO_XDP_SOCK,	 /* reg points to struct xdp_sock */899	/* PTR_TO_BTF_ID points to a kernel struct that does not need900	 * to be null checked by the BPF program. This does not imply the901	 * pointer is _not_ null and in practice this can easily be a null902	 * pointer when reading pointer chains. The assumption is program903	 * context will handle null pointer dereference typically via fault904	 * handling. The verifier must keep this in mind and can make no905	 * assumptions about null or non-null when doing branch analysis.906	 * Further, when passed into helpers the helpers can not, without907	 * additional context, assume the value is non-null.908	 */909	PTR_TO_BTF_ID,910	/* PTR_TO_BTF_ID_OR_NULL points to a kernel struct that has not911	 * been checked for null. Used primarily to inform the verifier912	 * an explicit null check is required for this struct.913	 */914	PTR_TO_MEM,		 /* reg points to valid memory region */915	PTR_TO_ARENA,916	PTR_TO_BUF,		 /* reg points to a read/write buffer */917	PTR_TO_FUNC,		 /* reg points to a bpf program function */918	CONST_PTR_TO_DYNPTR,	 /* reg points to a const struct bpf_dynptr */919	__BPF_REG_TYPE_MAX,920 921	/* Extended reg_types. */922	PTR_TO_MAP_VALUE_OR_NULL	= PTR_MAYBE_NULL | PTR_TO_MAP_VALUE,923	PTR_TO_SOCKET_OR_NULL		= PTR_MAYBE_NULL | PTR_TO_SOCKET,924	PTR_TO_SOCK_COMMON_OR_NULL	= PTR_MAYBE_NULL | PTR_TO_SOCK_COMMON,925	PTR_TO_TCP_SOCK_OR_NULL		= PTR_MAYBE_NULL | PTR_TO_TCP_SOCK,926	PTR_TO_BTF_ID_OR_NULL		= PTR_MAYBE_NULL | PTR_TO_BTF_ID,927 928	/* This must be the last entry. Its purpose is to ensure the enum is929	 * wide enough to hold the higher bits reserved for bpf_type_flag.930	 */931	__BPF_REG_TYPE_LIMIT	= BPF_TYPE_LIMIT,932};933static_assert(__BPF_REG_TYPE_MAX <= BPF_BASE_TYPE_LIMIT);934 935/* The information passed from prog-specific *_is_valid_access936 * back to the verifier.937 */938struct bpf_insn_access_aux {939	enum bpf_reg_type reg_type;940	bool is_ldsx;941	union {942		int ctx_field_size;943		struct {944			struct btf *btf;945			u32 btf_id;946		};947	};948	struct bpf_verifier_log *log; /* for verbose logs */949	bool is_retval; /* is accessing function return value ? */950};951 952static inline void953bpf_ctx_record_field_size(struct bpf_insn_access_aux *aux, u32 size)954{955	aux->ctx_field_size = size;956}957 958static bool bpf_is_ldimm64(const struct bpf_insn *insn)959{960	return insn->code == (BPF_LD | BPF_IMM | BPF_DW);961}962 963static inline bool bpf_pseudo_func(const struct bpf_insn *insn)964{965	return bpf_is_ldimm64(insn) && insn->src_reg == BPF_PSEUDO_FUNC;966}967 968struct bpf_prog_ops {969	int (*test_run)(struct bpf_prog *prog, const union bpf_attr *kattr,970			union bpf_attr __user *uattr);971};972 973struct bpf_reg_state;974struct bpf_verifier_ops {975	/* return eBPF function prototype for verification */976	const struct bpf_func_proto *977	(*get_func_proto)(enum bpf_func_id func_id,978			  const struct bpf_prog *prog);979 980	/* return true if 'size' wide access at offset 'off' within bpf_context981	 * with 'type' (read or write) is allowed982	 */983	bool (*is_valid_access)(int off, int size, enum bpf_access_type type,984				const struct bpf_prog *prog,985				struct bpf_insn_access_aux *info);986	int (*gen_prologue)(struct bpf_insn *insn, bool direct_write,987			    const struct bpf_prog *prog);988	int (*gen_epilogue)(struct bpf_insn *insn, const struct bpf_prog *prog,989			    s16 ctx_stack_off);990	int (*gen_ld_abs)(const struct bpf_insn *orig,991			  struct bpf_insn *insn_buf);992	u32 (*convert_ctx_access)(enum bpf_access_type type,993				  const struct bpf_insn *src,994				  struct bpf_insn *dst,995				  struct bpf_prog *prog, u32 *target_size);996	int (*btf_struct_access)(struct bpf_verifier_log *log,997				 const struct bpf_reg_state *reg,998				 int off, int size);999};1000 1001struct bpf_prog_offload_ops {1002	/* verifier basic callbacks */1003	int (*insn_hook)(struct bpf_verifier_env *env,1004			 int insn_idx, int prev_insn_idx);1005	int (*finalize)(struct bpf_verifier_env *env);1006	/* verifier optimization callbacks (called after .finalize) */1007	int (*replace_insn)(struct bpf_verifier_env *env, u32 off,1008			    struct bpf_insn *insn);1009	int (*remove_insns)(struct bpf_verifier_env *env, u32 off, u32 cnt);1010	/* program management callbacks */1011	int (*prepare)(struct bpf_prog *prog);1012	int (*translate)(struct bpf_prog *prog);1013	void (*destroy)(struct bpf_prog *prog);1014};1015 1016struct bpf_prog_offload {1017	struct bpf_prog		*prog;1018	struct net_device	*netdev;1019	struct bpf_offload_dev	*offdev;1020	void			*dev_priv;1021	struct list_head	offloads;1022	bool			dev_state;1023	bool			opt_failed;1024	void			*jited_image;1025	u32			jited_len;1026};1027 1028enum bpf_cgroup_storage_type {1029	BPF_CGROUP_STORAGE_SHARED,1030	BPF_CGROUP_STORAGE_PERCPU,1031	__BPF_CGROUP_STORAGE_MAX1032};1033 1034#define MAX_BPF_CGROUP_STORAGE_TYPE __BPF_CGROUP_STORAGE_MAX1035 1036/* The longest tracepoint has 12 args.1037 * See include/trace/bpf_probe.h1038 */1039#define MAX_BPF_FUNC_ARGS 121040 1041/* The maximum number of arguments passed through registers1042 * a single function may have.1043 */1044#define MAX_BPF_FUNC_REG_ARGS 51045 1046/* The argument is a structure. */1047#define BTF_FMODEL_STRUCT_ARG		BIT(0)1048 1049/* The argument is signed. */1050#define BTF_FMODEL_SIGNED_ARG		BIT(1)1051 1052struct btf_func_model {1053	u8 ret_size;1054	u8 ret_flags;1055	u8 nr_args;1056	u8 arg_size[MAX_BPF_FUNC_ARGS];1057	u8 arg_flags[MAX_BPF_FUNC_ARGS];1058};1059 1060/* Restore arguments before returning from trampoline to let original function1061 * continue executing. This flag is used for fentry progs when there are no1062 * fexit progs.1063 */1064#define BPF_TRAMP_F_RESTORE_REGS	BIT(0)1065/* Call original function after fentry progs, but before fexit progs.1066 * Makes sense for fentry/fexit, normal calls and indirect calls.1067 */1068#define BPF_TRAMP_F_CALL_ORIG		BIT(1)1069/* Skip current frame and return to parent.  Makes sense for fentry/fexit1070 * programs only. Should not be used with normal calls and indirect calls.1071 */1072#define BPF_TRAMP_F_SKIP_FRAME		BIT(2)1073/* Store IP address of the caller on the trampoline stack,1074 * so it's available for trampoline's programs.1075 */1076#define BPF_TRAMP_F_IP_ARG		BIT(3)1077/* Return the return value of fentry prog. Only used by bpf_struct_ops. */1078#define BPF_TRAMP_F_RET_FENTRY_RET	BIT(4)1079 1080/* Get original function from stack instead of from provided direct address.1081 * Makes sense for trampolines with fexit or fmod_ret programs.1082 */1083#define BPF_TRAMP_F_ORIG_STACK		BIT(5)1084 1085/* This trampoline is on a function with another ftrace_ops with IPMODIFY,1086 * e.g., a live patch. This flag is set and cleared by ftrace call backs,1087 */1088#define BPF_TRAMP_F_SHARE_IPMODIFY	BIT(6)1089 1090/* Indicate that current trampoline is in a tail call context. Then, it has to1091 * cache and restore tail_call_cnt to avoid infinite tail call loop.1092 */1093#define BPF_TRAMP_F_TAIL_CALL_CTX	BIT(7)1094 1095/*1096 * Indicate the trampoline should be suitable to receive indirect calls;1097 * without this indirectly calling the generated code can result in #UD/#CP,1098 * depending on the CFI options.1099 *1100 * Used by bpf_struct_ops.1101 *1102 * Incompatible with FENTRY usage, overloads @func_addr argument.1103 */1104#define BPF_TRAMP_F_INDIRECT		BIT(8)1105 1106/* Each call __bpf_prog_enter + call bpf_func + call __bpf_prog_exit is ~501107 * bytes on x86.1108 */1109enum {1110#if defined(__s390x__)1111	BPF_MAX_TRAMP_LINKS = 27,1112#else1113	BPF_MAX_TRAMP_LINKS = 38,1114#endif1115};1116 1117struct bpf_tramp_links {1118	struct bpf_tramp_link *links[BPF_MAX_TRAMP_LINKS];1119	int nr_links;1120};1121 1122struct bpf_tramp_run_ctx;1123 1124/* Different use cases for BPF trampoline:1125 * 1. replace nop at the function entry (kprobe equivalent)1126 *    flags = BPF_TRAMP_F_RESTORE_REGS1127 *    fentry = a set of programs to run before returning from trampoline1128 *1129 * 2. replace nop at the function entry (kprobe + kretprobe equivalent)1130 *    flags = BPF_TRAMP_F_CALL_ORIG | BPF_TRAMP_F_SKIP_FRAME1131 *    orig_call = fentry_ip + MCOUNT_INSN_SIZE1132 *    fentry = a set of program to run before calling original function1133 *    fexit = a set of program to run after original function1134 *1135 * 3. replace direct call instruction anywhere in the function body1136 *    or assign a function pointer for indirect call (like tcp_congestion_ops->cong_avoid)1137 *    With flags = 01138 *      fentry = a set of programs to run before returning from trampoline1139 *    With flags = BPF_TRAMP_F_CALL_ORIG1140 *      orig_call = original callback addr or direct function addr1141 *      fentry = a set of program to run before calling original function1142 *      fexit = a set of program to run after original function1143 */1144struct bpf_tramp_image;1145int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *image_end,1146				const struct btf_func_model *m, u32 flags,1147				struct bpf_tramp_links *tlinks,1148				void *func_addr);1149void *arch_alloc_bpf_trampoline(unsigned int size);1150void arch_free_bpf_trampoline(void *image, unsigned int size);1151int __must_check arch_protect_bpf_trampoline(void *image, unsigned int size);1152int arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags,1153			     struct bpf_tramp_links *tlinks, void *func_addr);1154 1155u64 notrace __bpf_prog_enter_sleepable_recur(struct bpf_prog *prog,1156					     struct bpf_tramp_run_ctx *run_ctx);1157void notrace __bpf_prog_exit_sleepable_recur(struct bpf_prog *prog, u64 start,1158					     struct bpf_tramp_run_ctx *run_ctx);1159void notrace __bpf_tramp_enter(struct bpf_tramp_image *tr);1160void notrace __bpf_tramp_exit(struct bpf_tramp_image *tr);1161typedef u64 (*bpf_trampoline_enter_t)(struct bpf_prog *prog,1162				      struct bpf_tramp_run_ctx *run_ctx);1163typedef void (*bpf_trampoline_exit_t)(struct bpf_prog *prog, u64 start,1164				      struct bpf_tramp_run_ctx *run_ctx);1165bpf_trampoline_enter_t bpf_trampoline_enter(const struct bpf_prog *prog);1166bpf_trampoline_exit_t bpf_trampoline_exit(const struct bpf_prog *prog);1167 1168struct bpf_ksym {1169	unsigned long		 start;1170	unsigned long		 end;1171	char			 name[KSYM_NAME_LEN];1172	struct list_head	 lnode;1173	struct latch_tree_node	 tnode;1174	bool			 prog;1175};1176 1177enum bpf_tramp_prog_type {1178	BPF_TRAMP_FENTRY,1179	BPF_TRAMP_FEXIT,1180	BPF_TRAMP_MODIFY_RETURN,1181	BPF_TRAMP_MAX,1182	BPF_TRAMP_REPLACE, /* more than MAX */1183};1184 1185struct bpf_tramp_image {1186	void *image;1187	int size;1188	struct bpf_ksym ksym;1189	struct percpu_ref pcref;1190	void *ip_after_call;1191	void *ip_epilogue;1192	union {1193		struct rcu_head rcu;1194		struct work_struct work;1195	};1196};1197 1198struct bpf_trampoline {1199	/* hlist for trampoline_table */1200	struct hlist_node hlist;1201	struct ftrace_ops *fops;1202	/* serializes access to fields of this trampoline */1203	struct mutex mutex;1204	refcount_t refcnt;1205	u32 flags;1206	u64 key;1207	struct {1208		struct btf_func_model model;1209		void *addr;1210		bool ftrace_managed;1211	} func;1212	/* if !NULL this is BPF_PROG_TYPE_EXT program that extends another BPF1213	 * program by replacing one of its functions. func.addr is the address1214	 * of the function it replaced.1215	 */1216	struct bpf_prog *extension_prog;1217	/* list of BPF programs using this trampoline */1218	struct hlist_head progs_hlist[BPF_TRAMP_MAX];1219	/* Number of attached programs. A counter per kind. */1220	int progs_cnt[BPF_TRAMP_MAX];1221	/* Executable image of trampoline */1222	struct bpf_tramp_image *cur_image;1223};1224 1225struct bpf_attach_target_info {1226	struct btf_func_model fmodel;1227	long tgt_addr;1228	struct module *tgt_mod;1229	const char *tgt_name;1230	const struct btf_type *tgt_type;1231};1232 1233#define BPF_DISPATCHER_MAX 48 /* Fits in 2048B */1234 1235struct bpf_dispatcher_prog {1236	struct bpf_prog *prog;1237	refcount_t users;1238};1239 1240struct bpf_dispatcher {1241	/* dispatcher mutex */1242	struct mutex mutex;1243	void *func;1244	struct bpf_dispatcher_prog progs[BPF_DISPATCHER_MAX];1245	int num_progs;1246	void *image;1247	void *rw_image;1248	u32 image_off;1249	struct bpf_ksym ksym;1250#ifdef CONFIG_HAVE_STATIC_CALL1251	struct static_call_key *sc_key;1252	void *sc_tramp;1253#endif1254};1255 1256#ifndef __bpfcall1257#define __bpfcall __nocfi1258#endif1259 1260static __always_inline __bpfcall unsigned int bpf_dispatcher_nop_func(1261	const void *ctx,1262	const struct bpf_insn *insnsi,1263	bpf_func_t bpf_func)1264{1265	return bpf_func(ctx, insnsi);1266}1267 1268/* the implementation of the opaque uapi struct bpf_dynptr */1269struct bpf_dynptr_kern {1270	void *data;1271	/* Size represents the number of usable bytes of dynptr data.1272	 * If for example the offset is at 4 for a local dynptr whose data is1273	 * of type u64, the number of usable bytes is 4.1274	 *1275	 * The upper 8 bits are reserved. It is as follows:1276	 * Bits 0 - 23 = size1277	 * Bits 24 - 30 = dynptr type1278	 * Bit 31 = whether dynptr is read-only1279	 */1280	u32 size;1281	u32 offset;1282} __aligned(8);1283 1284enum bpf_dynptr_type {1285	BPF_DYNPTR_TYPE_INVALID,1286	/* Points to memory that is local to the bpf program */1287	BPF_DYNPTR_TYPE_LOCAL,1288	/* Underlying data is a ringbuf record */1289	BPF_DYNPTR_TYPE_RINGBUF,1290	/* Underlying data is a sk_buff */1291	BPF_DYNPTR_TYPE_SKB,1292	/* Underlying data is a xdp_buff */1293	BPF_DYNPTR_TYPE_XDP,1294};1295 1296int bpf_dynptr_check_size(u32 size);1297u32 __bpf_dynptr_size(const struct bpf_dynptr_kern *ptr);1298const void *__bpf_dynptr_data(const struct bpf_dynptr_kern *ptr, u32 len);1299void *__bpf_dynptr_data_rw(const struct bpf_dynptr_kern *ptr, u32 len);1300bool __bpf_dynptr_is_rdonly(const struct bpf_dynptr_kern *ptr);1301 1302#ifdef CONFIG_BPF_JIT1303int bpf_trampoline_link_prog(struct bpf_tramp_link *link, struct bpf_trampoline *tr);1304int bpf_trampoline_unlink_prog(struct bpf_tramp_link *link, struct bpf_trampoline *tr);1305struct bpf_trampoline *bpf_trampoline_get(u64 key,1306					  struct bpf_attach_target_info *tgt_info);1307void bpf_trampoline_put(struct bpf_trampoline *tr);1308int arch_prepare_bpf_dispatcher(void *image, void *buf, s64 *funcs, int num_funcs);1309 1310/*1311 * When the architecture supports STATIC_CALL replace the bpf_dispatcher_fn1312 * indirection with a direct call to the bpf program. If the architecture does1313 * not have STATIC_CALL, avoid a double-indirection.1314 */1315#ifdef CONFIG_HAVE_STATIC_CALL1316 1317#define __BPF_DISPATCHER_SC_INIT(_name)				\1318	.sc_key = &STATIC_CALL_KEY(_name),			\1319	.sc_tramp = STATIC_CALL_TRAMP_ADDR(_name),1320 1321#define __BPF_DISPATCHER_SC(name)				\1322	DEFINE_STATIC_CALL(bpf_dispatcher_##name##_call, bpf_dispatcher_nop_func)1323 1324#define __BPF_DISPATCHER_CALL(name)				\1325	static_call(bpf_dispatcher_##name##_call)(ctx, insnsi, bpf_func)1326 1327#define __BPF_DISPATCHER_UPDATE(_d, _new)			\1328	__static_call_update((_d)->sc_key, (_d)->sc_tramp, (_new))1329 1330#else1331#define __BPF_DISPATCHER_SC_INIT(name)1332#define __BPF_DISPATCHER_SC(name)1333#define __BPF_DISPATCHER_CALL(name)		bpf_func(ctx, insnsi)1334#define __BPF_DISPATCHER_UPDATE(_d, _new)1335#endif1336 1337#define BPF_DISPATCHER_INIT(_name) {				\1338	.mutex = __MUTEX_INITIALIZER(_name.mutex),		\1339	.func = &_name##_func,					\1340	.progs = {},						\1341	.num_progs = 0,						\1342	.image = NULL,						\1343	.image_off = 0,						\1344	.ksym = {						\1345		.name  = #_name,				\1346		.lnode = LIST_HEAD_INIT(_name.ksym.lnode),	\1347	},							\1348	__BPF_DISPATCHER_SC_INIT(_name##_call)			\1349}1350 1351#define DEFINE_BPF_DISPATCHER(name)					\1352	__BPF_DISPATCHER_SC(name);					\1353	noinline __bpfcall unsigned int bpf_dispatcher_##name##_func(	\1354		const void *ctx,					\1355		const struct bpf_insn *insnsi,				\1356		bpf_func_t bpf_func)					\1357	{								\1358		return __BPF_DISPATCHER_CALL(name);			\1359	}								\1360	EXPORT_SYMBOL(bpf_dispatcher_##name##_func);			\1361	struct bpf_dispatcher bpf_dispatcher_##name =			\1362		BPF_DISPATCHER_INIT(bpf_dispatcher_##name);1363 1364#define DECLARE_BPF_DISPATCHER(name)					\1365	unsigned int bpf_dispatcher_##name##_func(			\1366		const void *ctx,					\1367		const struct bpf_insn *insnsi,				\1368		bpf_func_t bpf_func);					\1369	extern struct bpf_dispatcher bpf_dispatcher_##name;1370 1371#define BPF_DISPATCHER_FUNC(name) bpf_dispatcher_##name##_func1372#define BPF_DISPATCHER_PTR(name) (&bpf_dispatcher_##name)1373void bpf_dispatcher_change_prog(struct bpf_dispatcher *d, struct bpf_prog *from,1374				struct bpf_prog *to);1375/* Called only from JIT-enabled code, so there's no need for stubs. */1376void bpf_image_ksym_add(void *data, unsigned int size, struct bpf_ksym *ksym);1377void bpf_image_ksym_del(struct bpf_ksym *ksym);1378void bpf_ksym_add(struct bpf_ksym *ksym);1379void bpf_ksym_del(struct bpf_ksym *ksym);1380int bpf_jit_charge_modmem(u32 size);1381void bpf_jit_uncharge_modmem(u32 size);1382bool bpf_prog_has_trampoline(const struct bpf_prog *prog);1383#else1384static inline int bpf_trampoline_link_prog(struct bpf_tramp_link *link,1385					   struct bpf_trampoline *tr)1386{1387	return -ENOTSUPP;1388}1389static inline int bpf_trampoline_unlink_prog(struct bpf_tramp_link *link,1390					     struct bpf_trampoline *tr)1391{1392	return -ENOTSUPP;1393}1394static inline struct bpf_trampoline *bpf_trampoline_get(u64 key,1395							struct bpf_attach_target_info *tgt_info)1396{1397	return NULL;1398}1399static inline void bpf_trampoline_put(struct bpf_trampoline *tr) {}1400#define DEFINE_BPF_DISPATCHER(name)1401#define DECLARE_BPF_DISPATCHER(name)1402#define BPF_DISPATCHER_FUNC(name) bpf_dispatcher_nop_func1403#define BPF_DISPATCHER_PTR(name) NULL1404static inline void bpf_dispatcher_change_prog(struct bpf_dispatcher *d,1405					      struct bpf_prog *from,1406					      struct bpf_prog *to) {}1407static inline bool is_bpf_image_address(unsigned long address)1408{1409	return false;1410}1411static inline bool bpf_prog_has_trampoline(const struct bpf_prog *prog)1412{1413	return false;1414}1415#endif1416 1417struct bpf_func_info_aux {1418	u16 linkage;1419	bool unreliable;1420	bool called : 1;1421	bool verified : 1;1422};1423 1424enum bpf_jit_poke_reason {1425	BPF_POKE_REASON_TAIL_CALL,1426};1427 1428/* Descriptor of pokes pointing /into/ the JITed image. */1429struct bpf_jit_poke_descriptor {1430	void *tailcall_target;1431	void *tailcall_bypass;1432	void *bypass_addr;1433	void *aux;1434	union {1435		struct {1436			struct bpf_map *map;1437			u32 key;1438		} tail_call;1439	};1440	bool tailcall_target_stable;1441	u8 adj_off;1442	u16 reason;1443	u32 insn_idx;1444};1445 1446/* reg_type info for ctx arguments */1447struct bpf_ctx_arg_aux {1448	u32 offset;1449	enum bpf_reg_type reg_type;1450	struct btf *btf;1451	u32 btf_id;1452};1453 1454struct btf_mod_pair {1455	struct btf *btf;1456	struct module *module;1457};1458 1459struct bpf_kfunc_desc_tab;1460 1461struct bpf_prog_aux {1462	atomic64_t refcnt;1463	u32 used_map_cnt;1464	u32 used_btf_cnt;1465	u32 max_ctx_offset;1466	u32 max_pkt_offset;1467	u32 max_tp_access;1468	u32 stack_depth;1469	u32 id;1470	u32 func_cnt; /* used by non-func prog as the number of func progs */1471	u32 real_func_cnt; /* includes hidden progs, only used for JIT and freeing progs */1472	u32 func_idx; /* 0 for non-func prog, the index in func array for func prog */1473	u32 attach_btf_id; /* in-kernel BTF type id to attach to */1474	u32 ctx_arg_info_size;1475	u32 max_rdonly_access;1476	u32 max_rdwr_access;1477	struct btf *attach_btf;1478	const struct bpf_ctx_arg_aux *ctx_arg_info;1479	struct mutex dst_mutex; /* protects dst_* pointers below, *after* prog becomes visible */1480	struct bpf_prog *dst_prog;1481	struct bpf_trampoline *dst_trampoline;1482	enum bpf_prog_type saved_dst_prog_type;1483	enum bpf_attach_type saved_dst_attach_type;1484	bool verifier_zext; /* Zero extensions has been inserted by verifier. */1485	bool dev_bound; /* Program is bound to the netdev. */1486	bool offload_requested; /* Program is bound and offloaded to the netdev. */1487	bool attach_btf_trace; /* true if attaching to BTF-enabled raw tp */1488	bool attach_tracing_prog; /* true if tracing another tracing program */1489	bool func_proto_unreliable;1490	bool tail_call_reachable;1491	bool xdp_has_frags;1492	bool exception_cb;1493	bool exception_boundary;1494	struct bpf_arena *arena;1495	/* BTF_KIND_FUNC_PROTO for valid attach_btf_id */1496	const struct btf_type *attach_func_proto;1497	/* function name for valid attach_btf_id */1498	const char *attach_func_name;1499	struct bpf_prog **func;1500	void *jit_data; /* JIT specific data. arch dependent */1501	struct bpf_jit_poke_descriptor *poke_tab;1502	struct bpf_kfunc_desc_tab *kfunc_tab;1503	struct bpf_kfunc_btf_tab *kfunc_btf_tab;1504	u32 size_poke_tab;1505#ifdef CONFIG_FINEIBT1506	struct bpf_ksym ksym_prefix;1507#endif1508	struct bpf_ksym ksym;1509	const struct bpf_prog_ops *ops;1510	struct bpf_map **used_maps;1511	struct mutex used_maps_mutex; /* mutex for used_maps and used_map_cnt */1512	struct btf_mod_pair *used_btfs;1513	struct bpf_prog *prog;1514	struct user_struct *user;1515	u64 load_time; /* ns since boottime */1516	u32 verified_insns;1517	int cgroup_atype; /* enum cgroup_bpf_attach_type */1518	struct bpf_map *cgroup_storage[MAX_BPF_CGROUP_STORAGE_TYPE];1519	char name[BPF_OBJ_NAME_LEN];1520	u64 (*bpf_exception_cb)(u64 cookie, u64 sp, u64 bp, u64, u64);1521#ifdef CONFIG_SECURITY1522	void *security;1523#endif1524	struct bpf_token *token;1525	struct bpf_prog_offload *offload;1526	struct btf *btf;1527	struct bpf_func_info *func_info;1528	struct bpf_func_info_aux *func_info_aux;1529	/* bpf_line_info loaded from userspace.  linfo->insn_off1530	 * has the xlated insn offset.1531	 * Both the main and sub prog share the same linfo.1532	 * The subprog can access its first linfo by1533	 * using the linfo_idx.1534	 */1535	struct bpf_line_info *linfo;1536	/* jited_linfo is the jited addr of the linfo.  It has a1537	 * one to one mapping to linfo:1538	 * jited_linfo[i] is the jited addr for the linfo[i]->insn_off.1539	 * Both the main and sub prog share the same jited_linfo.1540	 * The subprog can access its first jited_linfo by1541	 * using the linfo_idx.1542	 */1543	void **jited_linfo;1544	u32 func_info_cnt;1545	u32 nr_linfo;1546	/* subprog can use linfo_idx to access its first linfo and1547	 * jited_linfo.1548	 * main prog always has linfo_idx == 01549	 */1550	u32 linfo_idx;1551	struct module *mod;1552	u32 num_exentries;1553	struct exception_table_entry *extable;1554	union {1555		struct work_struct work;1556		struct rcu_head	rcu;1557	};1558};1559 1560struct bpf_prog {1561	u16			pages;		/* Number of allocated pages */1562	u16			jited:1,	/* Is our filter JIT'ed? */1563				jit_requested:1,/* archs need to JIT the prog */1564				gpl_compatible:1, /* Is filter GPL compatible? */1565				cb_access:1,	/* Is control block accessed? */1566				dst_needed:1,	/* Do we need dst entry? */1567				blinding_requested:1, /* needs constant blinding */1568				blinded:1,	/* Was blinded */1569				is_func:1,	/* program is a bpf function */1570				kprobe_override:1, /* Do we override a kprobe? */1571				has_callchain_buf:1, /* callchain buffer allocated? */1572				enforce_expected_attach_type:1, /* Enforce expected_attach_type checking at attach time */1573				call_get_stack:1, /* Do we call bpf_get_stack() or bpf_get_stackid() */1574				call_get_func_ip:1, /* Do we call get_func_ip() */1575				tstamp_type_access:1, /* Accessed __sk_buff->tstamp_type */1576				sleepable:1;	/* BPF program is sleepable */1577	enum bpf_prog_type	type;		/* Type of BPF program */1578	enum bpf_attach_type	expected_attach_type; /* For some prog types */1579	u32			len;		/* Number of filter blocks */1580	u32			jited_len;	/* Size of jited insns in bytes */1581	u8			tag[BPF_TAG_SIZE];1582	struct bpf_prog_stats __percpu *stats;1583	int __percpu		*active;1584	unsigned int		(*bpf_func)(const void *ctx,1585					    const struct bpf_insn *insn);1586	struct bpf_prog_aux	*aux;		/* Auxiliary fields */1587	struct sock_fprog_kern	*orig_prog;	/* Original BPF program */1588	/* Instructions for interpreter */1589	union {1590		DECLARE_FLEX_ARRAY(struct sock_filter, insns);1591		DECLARE_FLEX_ARRAY(struct bpf_insn, insnsi);1592	};1593};1594 1595struct bpf_array_aux {1596	/* Programs with direct jumps into programs part of this array. */1597	struct list_head poke_progs;1598	struct bpf_map *map;1599	struct mutex poke_mutex;1600	struct work_struct work;1601};1602 1603struct bpf_link {1604	atomic64_t refcnt;1605	u32 id;1606	enum bpf_link_type type;1607	const struct bpf_link_ops *ops;1608	struct bpf_prog *prog;1609	/* rcu is used before freeing, work can be used to schedule that1610	 * RCU-based freeing before that, so they never overlap1611	 */1612	union {1613		struct rcu_head rcu;1614		struct work_struct work;1615	};1616};1617 1618struct bpf_link_ops {1619	void (*release)(struct bpf_link *link);1620	/* deallocate link resources callback, called without RCU grace period1621	 * waiting1622	 */1623	void (*dealloc)(struct bpf_link *link);1624	/* deallocate link resources callback, called after RCU grace period;1625	 * if underlying BPF program is sleepable we go through tasks trace1626	 * RCU GP and then "classic" RCU GP1627	 */1628	void (*dealloc_deferred)(struct bpf_link *link);1629	int (*detach)(struct bpf_link *link);1630	int (*update_prog)(struct bpf_link *link, struct bpf_prog *new_prog,1631			   struct bpf_prog *old_prog);1632	void (*show_fdinfo)(const struct bpf_link *link, struct seq_file *seq);1633	int (*fill_link_info)(const struct bpf_link *link,1634			      struct bpf_link_info *info);1635	int (*update_map)(struct bpf_link *link, struct bpf_map *new_map,1636			  struct bpf_map *old_map);1637	__poll_t (*poll)(struct file *file, struct poll_table_struct *pts);1638};1639 1640struct bpf_tramp_link {1641	struct bpf_link link;1642	struct hlist_node tramp_hlist;1643	u64 cookie;1644};1645 1646struct bpf_shim_tramp_link {1647	struct bpf_tramp_link link;1648	struct bpf_trampoline *trampoline;1649};1650 1651struct bpf_tracing_link {1652	struct bpf_tramp_link link;1653	enum bpf_attach_type attach_type;1654	struct bpf_trampoline *trampoline;1655	struct bpf_prog *tgt_prog;1656};1657 1658struct bpf_raw_tp_link {1659	struct bpf_link link;1660	struct bpf_raw_event_map *btp;1661	u64 cookie;1662};1663 1664struct bpf_link_primer {1665	struct bpf_link *link;1666	struct file *file;1667	int fd;1668	u32 id;1669};1670 1671struct bpf_mount_opts {1672	kuid_t uid;1673	kgid_t gid;1674	umode_t mode;1675 1676	/* BPF token-related delegation options */1677	u64 delegate_cmds;1678	u64 delegate_maps;1679	u64 delegate_progs;1680	u64 delegate_attachs;1681};1682 1683struct bpf_token {1684	struct work_struct work;1685	atomic64_t refcnt;1686	struct user_namespace *userns;1687	u64 allowed_cmds;1688	u64 allowed_maps;1689	u64 allowed_progs;1690	u64 allowed_attachs;1691#ifdef CONFIG_SECURITY1692	void *security;1693#endif1694};1695 1696struct bpf_struct_ops_value;1697struct btf_member;1698 1699#define BPF_STRUCT_OPS_MAX_NR_MEMBERS 641700/**1701 * struct bpf_struct_ops - A structure of callbacks allowing a subsystem to1702 *			   define a BPF_MAP_TYPE_STRUCT_OPS map type composed1703 *			   of BPF_PROG_TYPE_STRUCT_OPS progs.1704 * @verifier_ops: A structure of callbacks that are invoked by the verifier1705 *		  when determining whether the struct_ops progs in the1706 *		  struct_ops map are valid.1707 * @init: A callback that is invoked a single time, and before any other1708 *	  callback, to initialize the structure. A nonzero return value means1709 *	  the subsystem could not be initialized.1710 * @check_member: When defined, a callback invoked by the verifier to allow1711 *		  the subsystem to determine if an entry in the struct_ops map1712 *		  is valid. A nonzero return value means that the map is1713 *		  invalid and should be rejected by the verifier.1714 * @init_member: A callback that is invoked for each member of the struct_ops1715 *		 map to allow the subsystem to initialize the member. A nonzero1716 *		 value means the member could not be initialized. This callback1717 *		 is exclusive with the @type, @type_id, @value_type, and1718 *		 @value_id fields.1719 * @reg: A callback that is invoked when the struct_ops map has been1720 *	 initialized and is being attached to. Zero means the struct_ops map1721 *	 has been successfully registered and is live. A nonzero return value1722 *	 means the struct_ops map could not be registered.1723 * @unreg: A callback that is invoked when the struct_ops map should be1724 *	   unregistered.1725 * @update: A callback that is invoked when the live struct_ops map is being1726 *	    updated to contain new values. This callback is only invoked when1727 *	    the struct_ops map is loaded with BPF_F_LINK. If not defined, the1728 *	    it is assumed that the struct_ops map cannot be updated.1729 * @validate: A callback that is invoked after all of the members have been1730 *	      initialized. This callback should perform static checks on the1731 *	      map, meaning that it should either fail or succeed1732 *	      deterministically. A struct_ops map that has been validated may1733 *	      not necessarily succeed in being registered if the call to @reg1734 *	      fails. For example, a valid struct_ops map may be loaded, but1735 *	      then fail to be registered due to there being another active1736 *	      struct_ops map on the system in the subsystem already. For this1737 *	      reason, if this callback is not defined, the check is skipped as1738 *	      the struct_ops map will have final verification performed in1739 *	      @reg.1740 * @type: BTF type.1741 * @value_type: Value type.1742 * @name: The name of the struct bpf_struct_ops object.1743 * @func_models: Func models1744 * @type_id: BTF type id.1745 * @value_id: BTF value id.1746 */1747struct bpf_struct_ops {1748	const struct bpf_verifier_ops *verifier_ops;1749	int (*init)(struct btf *btf);1750	int (*check_member)(const struct btf_type *t,1751			    const struct btf_member *member,1752			    const struct bpf_prog *prog);1753	int (*init_member)(const struct btf_type *t,1754			   const struct btf_member *member,1755			   void *kdata, const void *udata);1756	int (*reg)(void *kdata, struct bpf_link *link);1757	void (*unreg)(void *kdata, struct bpf_link *link);1758	int (*update)(void *kdata, void *old_kdata, struct bpf_link *link);1759	int (*validate)(void *kdata);1760	void *cfi_stubs;1761	struct module *owner;1762	const char *name;1763	struct btf_func_model func_models[BPF_STRUCT_OPS_MAX_NR_MEMBERS];1764};1765 1766/* Every member of a struct_ops type has an instance even a member is not1767 * an operator (function pointer). The "info" field will be assigned to1768 * prog->aux->ctx_arg_info of BPF struct_ops programs to provide the1769 * argument information required by the verifier to verify the program.1770 *1771 * btf_ctx_access() will lookup prog->aux->ctx_arg_info to find the1772 * corresponding entry for an given argument.1773 */1774struct bpf_struct_ops_arg_info {1775	struct bpf_ctx_arg_aux *info;1776	u32 cnt;1777};1778 1779struct bpf_struct_ops_desc {1780	struct bpf_struct_ops *st_ops;1781 1782	const struct btf_type *type;1783	const struct btf_type *value_type;1784	u32 type_id;1785	u32 value_id;1786 1787	/* Collection of argument information for each member */1788	struct bpf_struct_ops_arg_info *arg_info;1789};1790 1791enum bpf_struct_ops_state {1792	BPF_STRUCT_OPS_STATE_INIT,1793	BPF_STRUCT_OPS_STATE_INUSE,1794	BPF_STRUCT_OPS_STATE_TOBEFREE,1795	BPF_STRUCT_OPS_STATE_READY,1796};1797 1798struct bpf_struct_ops_common_value {1799	refcount_t refcnt;1800	enum bpf_struct_ops_state state;1801};1802 1803#if defined(CONFIG_BPF_JIT) && defined(CONFIG_BPF_SYSCALL)1804/* This macro helps developer to register a struct_ops type and generate1805 * type information correctly. Developers should use this macro to register1806 * a struct_ops type instead of calling __register_bpf_struct_ops() directly.1807 */1808#define register_bpf_struct_ops(st_ops, type)				\1809	({								\1810		struct bpf_struct_ops_##type {				\1811			struct bpf_struct_ops_common_value common;	\1812			struct type data ____cacheline_aligned_in_smp;	\1813		};							\1814		BTF_TYPE_EMIT(struct bpf_struct_ops_##type);		\1815		__register_bpf_struct_ops(st_ops);			\1816	})1817#define BPF_MODULE_OWNER ((void *)((0xeB9FUL << 2) + POISON_POINTER_DELTA))1818bool bpf_struct_ops_get(const void *kdata);1819void bpf_struct_ops_put(const void *kdata);1820int bpf_struct_ops_supported(const struct bpf_struct_ops *st_ops, u32 moff);1821int bpf_struct_ops_map_sys_lookup_elem(struct bpf_map *map, void *key,1822				       void *value);1823int bpf_struct_ops_prepare_trampoline(struct bpf_tramp_links *tlinks,1824				      struct bpf_tramp_link *link,1825				      const struct btf_func_model *model,1826				      void *stub_func,1827				      void **image, u32 *image_off,1828				      bool allow_alloc);1829void bpf_struct_ops_image_free(void *image);1830static inline bool bpf_try_module_get(const void *data, struct module *owner)1831{1832	if (owner == BPF_MODULE_OWNER)1833		return bpf_struct_ops_get(data);1834	else1835		return try_module_get(owner);1836}1837static inline void bpf_module_put(const void *data, struct module *owner)1838{1839	if (owner == BPF_MODULE_OWNER)1840		bpf_struct_ops_put(data);1841	else1842		module_put(owner);1843}1844int bpf_struct_ops_link_create(union bpf_attr *attr);1845 1846#ifdef CONFIG_NET1847/* Define it here to avoid the use of forward declaration */1848struct bpf_dummy_ops_state {1849	int val;1850};1851 1852struct bpf_dummy_ops {1853	int (*test_1)(struct bpf_dummy_ops_state *cb);1854	int (*test_2)(struct bpf_dummy_ops_state *cb, int a1, unsigned short a2,1855		      char a3, unsigned long a4);1856	int (*test_sleepable)(struct bpf_dummy_ops_state *cb);1857};1858 1859int bpf_struct_ops_test_run(struct bpf_prog *prog, const union bpf_attr *kattr,1860			    union bpf_attr __user *uattr);1861#endif1862int bpf_struct_ops_desc_init(struct bpf_struct_ops_desc *st_ops_desc,1863			     struct btf *btf,1864			     struct bpf_verifier_log *log);1865void bpf_map_struct_ops_info_fill(struct bpf_map_info *info, struct bpf_map *map);1866void bpf_struct_ops_desc_release(struct bpf_struct_ops_desc *st_ops_desc);1867#else1868#define register_bpf_struct_ops(st_ops, type) ({ (void *)(st_ops); 0; })1869static inline bool bpf_try_module_get(const void *data, struct module *owner)1870{1871	return try_module_get(owner);1872}1873static inline void bpf_module_put(const void *data, struct module *owner)1874{1875	module_put(owner);1876}1877static inline int bpf_struct_ops_supported(const struct bpf_struct_ops *st_ops, u32 moff)1878{1879	return -ENOTSUPP;1880}1881static inline int bpf_struct_ops_map_sys_lookup_elem(struct bpf_map *map,1882						     void *key,1883						     void *value)1884{1885	return -EINVAL;1886}1887static inline int bpf_struct_ops_link_create(union bpf_attr *attr)1888{1889	return -EOPNOTSUPP;1890}1891static inline void bpf_map_struct_ops_info_fill(struct bpf_map_info *info, struct bpf_map *map)1892{1893}1894 1895static inline void bpf_struct_ops_desc_release(struct bpf_struct_ops_desc *st_ops_desc)1896{1897}1898 1899#endif1900 1901#if defined(CONFIG_CGROUP_BPF) && defined(CONFIG_BPF_LSM)1902int bpf_trampoline_link_cgroup_shim(struct bpf_prog *prog,1903				    int cgroup_atype);1904void bpf_trampoline_unlink_cgroup_shim(struct bpf_prog *prog);1905#else1906static inline int bpf_trampoline_link_cgroup_shim(struct bpf_prog *prog,1907						  int cgroup_atype)1908{1909	return -EOPNOTSUPP;1910}1911static inline void bpf_trampoline_unlink_cgroup_shim(struct bpf_prog *prog)1912{1913}1914#endif1915 1916struct bpf_array {1917	struct bpf_map map;1918	u32 elem_size;1919	u32 index_mask;1920	struct bpf_array_aux *aux;1921	union {1922		DECLARE_FLEX_ARRAY(char, value) __aligned(8);1923		DECLARE_FLEX_ARRAY(void *, ptrs) __aligned(8);1924		DECLARE_FLEX_ARRAY(void __percpu *, pptrs) __aligned(8);1925	};1926};1927 1928#define BPF_COMPLEXITY_LIMIT_INSNS      1000000 /* yes. 1M insns */1929#define MAX_TAIL_CALL_CNT 331930 1931/* Maximum number of loops for bpf_loop and bpf_iter_num.1932 * It's enum to expose it (and thus make it discoverable) through BTF.1933 */1934enum {1935	BPF_MAX_LOOPS = 8 * 1024 * 1024,1936};1937 1938#define BPF_F_ACCESS_MASK	(BPF_F_RDONLY |		\1939				 BPF_F_RDONLY_PROG |	\1940				 BPF_F_WRONLY |		\1941				 BPF_F_WRONLY_PROG)1942 1943#define BPF_MAP_CAN_READ	BIT(0)1944#define BPF_MAP_CAN_WRITE	BIT(1)1945 1946/* Maximum number of user-producer ring buffer samples that can be drained in1947 * a call to bpf_user_ringbuf_drain().1948 */1949#define BPF_MAX_USER_RINGBUF_SAMPLES (128 * 1024)1950 1951static inline u32 bpf_map_flags_to_cap(struct bpf_map *map)1952{1953	u32 access_flags = map->map_flags & (BPF_F_RDONLY_PROG | BPF_F_WRONLY_PROG);1954 1955	/* Combination of BPF_F_RDONLY_PROG | BPF_F_WRONLY_PROG is1956	 * not possible.1957	 */1958	if (access_flags & BPF_F_RDONLY_PROG)1959		return BPF_MAP_CAN_READ;1960	else if (access_flags & BPF_F_WRONLY_PROG)1961		return BPF_MAP_CAN_WRITE;1962	else1963		return BPF_MAP_CAN_READ | BPF_MAP_CAN_WRITE;1964}1965 1966static inline bool bpf_map_flags_access_ok(u32 access_flags)1967{1968	return (access_flags & (BPF_F_RDONLY_PROG | BPF_F_WRONLY_PROG)) !=1969	       (BPF_F_RDONLY_PROG | BPF_F_WRONLY_PROG);1970}1971 1972struct bpf_event_entry {1973	struct perf_event *event;1974	struct file *perf_file;1975	struct file *map_file;1976	struct rcu_head rcu;1977};1978 1979static inline bool map_type_contains_progs(struct bpf_map *map)1980{1981	return map->map_type == BPF_MAP_TYPE_PROG_ARRAY ||1982	       map->map_type == BPF_MAP_TYPE_DEVMAP ||1983	       map->map_type == BPF_MAP_TYPE_CPUMAP;1984}1985 1986bool bpf_prog_map_compatible(struct bpf_map *map, const struct bpf_prog *fp);1987int bpf_prog_calc_tag(struct bpf_prog *fp);1988 1989const struct bpf_func_proto *bpf_get_trace_printk_proto(void);1990const struct bpf_func_proto *bpf_get_trace_vprintk_proto(void);1991 1992typedef unsigned long (*bpf_ctx_copy_t)(void *dst, const void *src,1993					unsigned long off, unsigned long len);1994typedef u32 (*bpf_convert_ctx_access_t)(enum bpf_access_type type,1995					const struct bpf_insn *src,1996					struct bpf_insn *dst,1997					struct bpf_prog *prog,1998					u32 *target_size);1999 2000u64 bpf_event_output(struct bpf_map *map, u64 flags, void *meta, u64 meta_size,2001		     void *ctx, u64 ctx_size, bpf_ctx_copy_t ctx_copy);2002 2003/* an array of programs to be executed under rcu_lock.2004 *2005 * Typical usage:2006 * ret = bpf_prog_run_array(rcu_dereference(&bpf_prog_array), ctx, bpf_prog_run);2007 *2008 * the structure returned by bpf_prog_array_alloc() should be populated2009 * with program pointers and the last pointer must be NULL.2010 * The user has to keep refcnt on the program and make sure the program2011 * is removed from the array before bpf_prog_put().2012 * The 'struct bpf_prog_array *' should only be replaced with xchg()2013 * since other cpus are walking the array of pointers in parallel.2014 */2015struct bpf_prog_array_item {2016	struct bpf_prog *prog;2017	union {2018		struct bpf_cgroup_storage *cgroup_storage[MAX_BPF_CGROUP_STORAGE_TYPE];2019		u64 bpf_cookie;2020	};2021};2022 2023struct bpf_prog_array {2024	struct rcu_head rcu;2025	struct bpf_prog_array_item items[];2026};2027 2028struct bpf_empty_prog_array {2029	struct bpf_prog_array hdr;2030	struct bpf_prog *null_prog;2031};2032 2033/* to avoid allocating empty bpf_prog_array for cgroups that2034 * don't have bpf program attached use one global 'bpf_empty_prog_array'2035 * It will not be modified the caller of bpf_prog_array_alloc()2036 * (since caller requested prog_cnt == 0)2037 * that pointer should be 'freed' by bpf_prog_array_free()2038 */2039extern struct bpf_empty_prog_array bpf_empty_prog_array;2040 2041struct bpf_prog_array *bpf_prog_array_alloc(u32 prog_cnt, gfp_t flags);2042void bpf_prog_array_free(struct bpf_prog_array *progs);2043/* Use when traversal over the bpf_prog_array uses tasks_trace rcu */2044void bpf_prog_array_free_sleepable(struct bpf_prog_array *progs);2045int bpf_prog_array_length(struct bpf_prog_array *progs);2046bool bpf_prog_array_is_empty(struct bpf_prog_array *array);2047int bpf_prog_array_copy_to_user(struct bpf_prog_array *progs,2048				__u32 __user *prog_ids, u32 cnt);2049 2050void bpf_prog_array_delete_safe(struct bpf_prog_array *progs,2051				struct bpf_prog *old_prog);2052int bpf_prog_array_delete_safe_at(struct bpf_prog_array *array, int index);2053int bpf_prog_array_update_at(struct bpf_prog_array *array, int index,2054			     struct bpf_prog *prog);2055int bpf_prog_array_copy_info(struct bpf_prog_array *array,2056			     u32 *prog_ids, u32 request_cnt,2057			     u32 *prog_cnt);2058int bpf_prog_array_copy(struct bpf_prog_array *old_array,2059			struct bpf_prog *exclude_prog,2060			struct bpf_prog *include_prog,2061			u64 bpf_cookie,2062			struct bpf_prog_array **new_array);2063 2064struct bpf_run_ctx {};2065 2066struct bpf_cg_run_ctx {2067	struct bpf_run_ctx run_ctx;2068	const struct bpf_prog_array_item *prog_item;2069	int retval;2070};2071 2072struct bpf_trace_run_ctx {2073	struct bpf_run_ctx run_ctx;2074	u64 bpf_cookie;2075	bool is_uprobe;2076};2077 2078struct bpf_tramp_run_ctx {2079	struct bpf_run_ctx run_ctx;2080	u64 bpf_cookie;2081	struct bpf_run_ctx *saved_run_ctx;2082};2083 2084static inline struct bpf_run_ctx *bpf_set_run_ctx(struct bpf_run_ctx *new_ctx)2085{2086	struct bpf_run_ctx *old_ctx = NULL;2087 2088#ifdef CONFIG_BPF_SYSCALL2089	old_ctx = current->bpf_ctx;2090	current->bpf_ctx = new_ctx;2091#endif2092	return old_ctx;2093}2094 2095static inline void bpf_reset_run_ctx(struct bpf_run_ctx *old_ctx)2096{2097#ifdef CONFIG_BPF_SYSCALL2098	current->bpf_ctx = old_ctx;2099#endif2100}2101 2102/* BPF program asks to bypass CAP_NET_BIND_SERVICE in bind. */2103#define BPF_RET_BIND_NO_CAP_NET_BIND_SERVICE			(1 << 0)2104/* BPF program asks to set CN on the packet. */2105#define BPF_RET_SET_CN						(1 << 0)2106 2107typedef u32 (*bpf_prog_run_fn)(const struct bpf_prog *prog, const void *ctx);2108 2109static __always_inline u322110bpf_prog_run_array(const struct bpf_prog_array *array,2111		   const void *ctx, bpf_prog_run_fn run_prog)2112{2113	const struct bpf_prog_array_item *item;2114	const struct bpf_prog *prog;2115	struct bpf_run_ctx *old_run_ctx;2116	struct bpf_trace_run_ctx run_ctx;2117	u32 ret = 1;2118 2119	RCU_LOCKDEP_WARN(!rcu_read_lock_held(), "no rcu lock held");2120 2121	if (unlikely(!array))2122		return ret;2123 2124	run_ctx.is_uprobe = false;2125 2126	migrate_disable();2127	old_run_ctx = bpf_set_run_ctx(&run_ctx.run_ctx);2128	item = &array->items[0];2129	while ((prog = READ_ONCE(item->prog))) {2130		run_ctx.bpf_cookie = item->bpf_cookie;2131		ret &= run_prog(prog, ctx);2132		item++;2133	}2134	bpf_reset_run_ctx(old_run_ctx);2135	migrate_enable();2136	return ret;2137}2138 2139/* Notes on RCU design for bpf_prog_arrays containing sleepable programs:2140 *2141 * We use the tasks_trace rcu flavor read section to protect the bpf_prog_array2142 * overall. As a result, we must use the bpf_prog_array_free_sleepable2143 * in order to use the tasks_trace rcu grace period.2144 *2145 * When a non-sleepable program is inside the array, we take the rcu read2146 * section and disable preemption for that program alone, so it can access2147 * rcu-protected dynamically sized maps.2148 */2149static __always_inline u322150bpf_prog_run_array_uprobe(const struct bpf_prog_array __rcu *array_rcu,2151			  const void *ctx, bpf_prog_run_fn run_prog)2152{2153	const struct bpf_prog_array_item *item;2154	const struct bpf_prog *prog;2155	const struct bpf_prog_array *array;2156	struct bpf_run_ctx *old_run_ctx;2157	struct bpf_trace_run_ctx run_ctx;2158	u32 ret = 1;2159 2160	might_fault();2161 2162	rcu_read_lock_trace();2163	migrate_disable();2164 2165	run_ctx.is_uprobe = true;2166 2167	array = rcu_dereference_check(array_rcu, rcu_read_lock_trace_held());2168	if (unlikely(!array))2169		goto out;2170	old_run_ctx = bpf_set_run_ctx(&run_ctx.run_ctx);2171	item = &array->items[0];2172	while ((prog = READ_ONCE(item->prog))) {2173		if (!prog->sleepable)2174			rcu_read_lock();2175 2176		run_ctx.bpf_cookie = item->bpf_cookie;2177		ret &= run_prog(prog, ctx);2178		item++;2179 2180		if (!prog->sleepable)2181			rcu_read_unlock();2182	}2183	bpf_reset_run_ctx(old_run_ctx);2184out:2185	migrate_enable();2186	rcu_read_unlock_trace();2187	return ret;2188}2189 2190#ifdef CONFIG_BPF_SYSCALL2191DECLARE_PER_CPU(int, bpf_prog_active);2192extern struct mutex bpf_stats_enabled_mutex;2193 2194/*2195 * Block execution of BPF programs attached to instrumentation (perf,2196 * kprobes, tracepoints) to prevent deadlocks on map operations as any of2197 * these events can happen inside a region which holds a map bucket lock2198 * and can deadlock on it.2199 */2200static inline void bpf_disable_instrumentation(void)2201{2202	migrate_disable();2203	this_cpu_inc(bpf_prog_active);2204}2205 2206static inline void bpf_enable_instrumentation(void)2207{2208	this_cpu_dec(bpf_prog_active);2209	migrate_enable();2210}2211 2212extern const struct super_operations bpf_super_ops;2213extern const struct file_operations bpf_map_fops;2214extern const struct file_operations bpf_prog_fops;2215extern const struct file_operations bpf_iter_fops;2216 2217#define BPF_PROG_TYPE(_id, _name, prog_ctx_type, kern_ctx_type) \2218	extern const struct bpf_prog_ops _name ## _prog_ops; \2219	extern const struct bpf_verifier_ops _name ## _verifier_ops;2220#define BPF_MAP_TYPE(_id, _ops) \2221	extern const struct bpf_map_ops _ops;2222#define BPF_LINK_TYPE(_id, _name)2223#include <linux/bpf_types.h>2224#undef BPF_PROG_TYPE2225#undef BPF_MAP_TYPE2226#undef BPF_LINK_TYPE2227 2228extern const struct bpf_prog_ops bpf_offload_prog_ops;2229extern const struct bpf_verifier_ops tc_cls_act_analyzer_ops;2230extern const struct bpf_verifier_ops xdp_analyzer_ops;2231 2232struct bpf_prog *bpf_prog_get(u32 ufd);2233struct bpf_prog *bpf_prog_get_type_dev(u32 ufd, enum bpf_prog_type type,2234				       bool attach_drv);2235void bpf_prog_add(struct bpf_prog *prog, int i);2236void bpf_prog_sub(struct bpf_prog *prog, int i);2237void bpf_prog_inc(struct bpf_prog *prog);2238struct bpf_prog * __must_check bpf_prog_inc_not_zero(struct bpf_prog *prog);2239void bpf_prog_put(struct bpf_prog *prog);2240 2241void bpf_prog_free_id(struct bpf_prog *prog);2242void bpf_map_free_id(struct bpf_map *map);2243 2244struct btf_field *btf_record_find(const struct btf_record *rec,2245				  u32 offset, u32 field_mask);2246void btf_record_free(struct btf_record *rec);2247void bpf_map_free_record(struct bpf_map *map);2248struct btf_record *btf_record_dup(const struct btf_record *rec);2249bool btf_record_equal(const struct btf_record *rec_a, const struct btf_record *rec_b);2250void bpf_obj_free_timer(const struct btf_record *rec, void *obj);2251void bpf_obj_free_workqueue(const struct btf_record *rec, void *obj);2252void bpf_obj_free_fields(const struct btf_record *rec, void *obj);2253void __bpf_obj_drop_impl(void *p, const struct btf_record *rec, bool percpu);2254 2255struct bpf_map *bpf_map_get(u32 ufd);2256struct bpf_map *bpf_map_get_with_uref(u32 ufd);2257 2258static inline struct bpf_map *__bpf_map_get(struct fd f)2259{2260	if (fd_empty(f))2261		return ERR_PTR(-EBADF);2262	if (unlikely(fd_file(f)->f_op != &bpf_map_fops))2263		return ERR_PTR(-EINVAL);2264	return fd_file(f)->private_data;2265}2266 2267void bpf_map_inc(struct bpf_map *map);2268void bpf_map_inc_with_uref(struct bpf_map *map);2269struct bpf_map *__bpf_map_inc_not_zero(struct bpf_map *map, bool uref);2270struct bpf_map * __must_check bpf_map_inc_not_zero(struct bpf_map *map);2271void bpf_map_put_with_uref(struct bpf_map *map);2272void bpf_map_put(struct bpf_map *map);2273void *bpf_map_area_alloc(u64 size, int numa_node);2274void *bpf_map_area_mmapable_alloc(u64 size, int numa_node);2275void bpf_map_area_free(void *base);2276bool bpf_map_write_active(const struct bpf_map *map);2277void bpf_map_init_from_attr(struct bpf_map *map, union bpf_attr *attr);2278int  generic_map_lookup_batch(struct bpf_map *map,2279			      const union bpf_attr *attr,2280			      union bpf_attr __user *uattr);2281int  generic_map_update_batch(struct bpf_map *map, struct file *map_file,2282			      const union bpf_attr *attr,2283			      union bpf_attr __user *uattr);2284int  generic_map_delete_batch(struct bpf_map *map,2285			      const union bpf_attr *attr,2286			      union bpf_attr __user *uattr);2287struct bpf_map *bpf_map_get_curr_or_next(u32 *id);2288struct bpf_prog *bpf_prog_get_curr_or_next(u32 *id);2289 2290int bpf_map_alloc_pages(const struct bpf_map *map, gfp_t gfp, int nid,2291			unsigned long nr_pages, struct page **page_array);2292#ifdef CONFIG_MEMCG2293void *bpf_map_kmalloc_node(const struct bpf_map *map, size_t size, gfp_t flags,2294			   int node);2295void *bpf_map_kzalloc(const struct bpf_map *map, size_t size, gfp_t flags);2296void *bpf_map_kvcalloc(struct bpf_map *map, size_t n, size_t size,2297		       gfp_t flags);2298void __percpu *bpf_map_alloc_percpu(const struct bpf_map *map, size_t size,2299				    size_t align, gfp_t flags);2300#else2301/*2302 * These specialized allocators have to be macros for their allocations to be2303 * accounted separately (to have separate alloc_tag).2304 */2305#define bpf_map_kmalloc_node(_map, _size, _flags, _node)	\2306		kmalloc_node(_size, _flags, _node)2307#define bpf_map_kzalloc(_map, _size, _flags)			\2308		kzalloc(_size, _flags)2309#define bpf_map_kvcalloc(_map, _n, _size, _flags)		\2310		kvcalloc(_n, _size, _flags)2311#define bpf_map_alloc_percpu(_map, _size, _align, _flags)	\2312		__alloc_percpu_gfp(_size, _align, _flags)2313#endif2314 2315static inline int2316bpf_map_init_elem_count(struct bpf_map *map)2317{2318	size_t size = sizeof(*map->elem_count), align = size;2319	gfp_t flags = GFP_USER | __GFP_NOWARN;2320 2321	map->elem_count = bpf_map_alloc_percpu(map, size, align, flags);2322	if (!map->elem_count)2323		return -ENOMEM;2324 2325	return 0;2326}2327 2328static inline void2329bpf_map_free_elem_count(struct bpf_map *map)2330{2331	free_percpu(map->elem_count);2332}2333 2334static inline void bpf_map_inc_elem_count(struct bpf_map *map)2335{2336	this_cpu_inc(*map->elem_count);2337}2338 2339static inline void bpf_map_dec_elem_count(struct bpf_map *map)2340{2341	this_cpu_dec(*map->elem_count);2342}2343 2344extern int sysctl_unprivileged_bpf_disabled;2345 2346bool bpf_token_capable(const struct bpf_token *token, int cap);2347 2348static inline bool bpf_allow_ptr_leaks(const struct bpf_token *token)2349{2350	return bpf_token_capable(token, CAP_PERFMON);2351}2352 2353static inline bool bpf_allow_uninit_stack(const struct bpf_token *token)2354{2355	return bpf_token_capable(token, CAP_PERFMON);2356}2357 2358static inline bool bpf_bypass_spec_v1(const struct bpf_token *token)2359{2360	return cpu_mitigations_off() || bpf_token_capable(token, CAP_PERFMON);2361}2362 2363static inline bool bpf_bypass_spec_v4(const struct bpf_token *token)2364{2365	return cpu_mitigations_off() || bpf_token_capable(token, CAP_PERFMON);2366}2367 2368int bpf_map_new_fd(struct bpf_map *map, int flags);2369int bpf_prog_new_fd(struct bpf_prog *prog);2370 2371void bpf_link_init(struct bpf_link *link, enum bpf_link_type type,2372		   const struct bpf_link_ops *ops, struct bpf_prog *prog);2373int bpf_link_prime(struct bpf_link *link, struct bpf_link_primer *primer);2374int bpf_link_settle(struct bpf_link_primer *primer);2375void bpf_link_cleanup(struct bpf_link_primer *primer);2376void bpf_link_inc(struct bpf_link *link);2377struct bpf_link *bpf_link_inc_not_zero(struct bpf_link *link);2378void bpf_link_put(struct bpf_link *link);2379int bpf_link_new_fd(struct bpf_link *link);2380struct bpf_link *bpf_link_get_from_fd(u32 ufd);2381struct bpf_link *bpf_link_get_curr_or_next(u32 *id);2382 2383void bpf_token_inc(struct bpf_token *token);2384void bpf_token_put(struct bpf_token *token);2385int bpf_token_create(union bpf_attr *attr);2386struct bpf_token *bpf_token_get_from_fd(u32 ufd);2387 2388bool bpf_token_allow_cmd(const struct bpf_token *token, enum bpf_cmd cmd);2389bool bpf_token_allow_map_type(const struct bpf_token *token, enum bpf_map_type type);2390bool bpf_token_allow_prog_type(const struct bpf_token *token,2391			       enum bpf_prog_type prog_type,2392			       enum bpf_attach_type attach_type);2393 2394int bpf_obj_pin_user(u32 ufd, int path_fd, const char __user *pathname);2395int bpf_obj_get_user(int path_fd, const char __user *pathname, int flags);2396struct inode *bpf_get_inode(struct super_block *sb, const struct inode *dir,2397			    umode_t mode);2398 2399#define BPF_ITER_FUNC_PREFIX "bpf_iter_"2400#define DEFINE_BPF_ITER_FUNC(target, args...)			\2401	extern int bpf_iter_ ## target(args);			\2402	int __init bpf_iter_ ## target(args) { return 0; }2403 2404/*2405 * The task type of iterators.2406 *2407 * For BPF task iterators, they can be parameterized with various2408 * parameters to visit only some of tasks.2409 *2410 * BPF_TASK_ITER_ALL (default)2411 *	Iterate over resources of every task.2412 *2413 * BPF_TASK_ITER_TID2414 *	Iterate over resources of a task/tid.2415 *2416 * BPF_TASK_ITER_TGID2417 *	Iterate over resources of every task of a process / task group.2418 */2419enum bpf_iter_task_type {2420	BPF_TASK_ITER_ALL = 0,2421	BPF_TASK_ITER_TID,2422	BPF_TASK_ITER_TGID,2423};2424 2425struct bpf_iter_aux_info {2426	/* for map_elem iter */2427	struct bpf_map *map;2428 2429	/* for cgroup iter */2430	struct {2431		struct cgroup *start; /* starting cgroup */2432		enum bpf_cgroup_iter_order order;2433	} cgroup;2434	struct {2435		enum bpf_iter_task_type	type;2436		u32 pid;2437	} task;2438};2439 2440typedef int (*bpf_iter_attach_target_t)(struct bpf_prog *prog,2441					union bpf_iter_link_info *linfo,2442					struct bpf_iter_aux_info *aux);2443typedef void (*bpf_iter_detach_target_t)(struct bpf_iter_aux_info *aux);2444typedef void (*bpf_iter_show_fdinfo_t) (const struct bpf_iter_aux_info *aux,2445					struct seq_file *seq);2446typedef int (*bpf_iter_fill_link_info_t)(const struct bpf_iter_aux_info *aux,2447					 struct bpf_link_info *info);2448typedef const struct bpf_func_proto *2449(*bpf_iter_get_func_proto_t)(enum bpf_func_id func_id,2450			     const struct bpf_prog *prog);2451 2452enum bpf_iter_feature {2453	BPF_ITER_RESCHED	= BIT(0),2454};2455 2456#define BPF_ITER_CTX_ARG_MAX 22457struct bpf_iter_reg {2458	const char *target;2459	bpf_iter_attach_target_t attach_target;2460	bpf_iter_detach_target_t detach_target;2461	bpf_iter_show_fdinfo_t show_fdinfo;2462	bpf_iter_fill_link_info_t fill_link_info;2463	bpf_iter_get_func_proto_t get_func_proto;2464	u32 ctx_arg_info_size;2465	u32 feature;2466	struct bpf_ctx_arg_aux ctx_arg_info[BPF_ITER_CTX_ARG_MAX];2467	const struct bpf_iter_seq_info *seq_info;2468};2469 2470struct bpf_iter_meta {2471	__bpf_md_ptr(struct seq_file *, seq);2472	u64 session_id;2473	u64 seq_num;2474};2475 2476struct bpf_iter__bpf_map_elem {2477	__bpf_md_ptr(struct bpf_iter_meta *, meta);2478	__bpf_md_ptr(struct bpf_map *, map);2479	__bpf_md_ptr(void *, key);2480	__bpf_md_ptr(void *, value);2481};2482 2483int bpf_iter_reg_target(const struct bpf_iter_reg *reg_info);2484void bpf_iter_unreg_target(const struct bpf_iter_reg *reg_info);2485bool bpf_iter_prog_supported(struct bpf_prog *prog);2486const struct bpf_func_proto *2487bpf_iter_get_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog);2488int bpf_iter_link_attach(const union bpf_attr *attr, bpfptr_t uattr, struct bpf_prog *prog);2489int bpf_iter_new_fd(struct bpf_link *link);2490bool bpf_link_is_iter(struct bpf_link *link);2491struct bpf_prog *bpf_iter_get_info(struct bpf_iter_meta *meta, bool in_stop);2492int bpf_iter_run_prog(struct bpf_prog *prog, void *ctx);2493void bpf_iter_map_show_fdinfo(const struct bpf_iter_aux_info *aux,2494			      struct seq_file *seq);2495int bpf_iter_map_fill_link_info(const struct bpf_iter_aux_info *aux,2496				struct bpf_link_info *info);2497 2498int map_set_for_each_callback_args(struct bpf_verifier_env *env,2499				   struct bpf_func_state *caller,2500				   struct bpf_func_state *callee);2501 2502int bpf_percpu_hash_copy(struct bpf_map *map, void *key, void *value);2503int bpf_percpu_array_copy(struct bpf_map *map, void *key, void *value);2504int bpf_percpu_hash_update(struct bpf_map *map, void *key, void *value,2505			   u64 flags);2506int bpf_percpu_array_update(struct bpf_map *map, void *key, void *value,2507			    u64 flags);2508 2509int bpf_stackmap_copy(struct bpf_map *map, void *key, void *value);2510 2511int bpf_fd_array_map_update_elem(struct bpf_map *map, struct file *map_file,2512				 void *key, void *value, u64 map_flags);2513int bpf_fd_array_map_lookup_elem(struct bpf_map *map, void *key, u32 *value);2514int bpf_fd_htab_map_update_elem(struct bpf_map *map, struct file *map_file,2515				void *key, void *value, u64 map_flags);2516int bpf_fd_htab_map_lookup_elem(struct bpf_map *map, void *key, u32 *value);2517 2518int bpf_get_file_flag(int flags);2519int bpf_check_uarg_tail_zero(bpfptr_t uaddr, size_t expected_size,2520			     size_t actual_size);2521 2522/* verify correctness of eBPF program */2523int bpf_check(struct bpf_prog **fp, union bpf_attr *attr, bpfptr_t uattr, u32 uattr_size);2524 2525#ifndef CONFIG_BPF_JIT_ALWAYS_ON2526void bpf_patch_call_args(struct bpf_insn *insn, u32 stack_depth);2527#endif2528 2529struct btf *bpf_get_btf_vmlinux(void);2530 2531/* Map specifics */2532struct xdp_frame;2533struct sk_buff;2534struct bpf_dtab_netdev;2535struct bpf_cpu_map_entry;2536 2537void __dev_flush(struct list_head *flush_list);2538int dev_xdp_enqueue(struct net_device *dev, struct xdp_frame *xdpf,2539		    struct net_device *dev_rx);2540int dev_map_enqueue(struct bpf_dtab_netdev *dst, struct xdp_frame *xdpf,2541		    struct net_device *dev_rx);2542int dev_map_enqueue_multi(struct xdp_frame *xdpf, struct net_device *dev_rx,2543			  struct bpf_map *map, bool exclude_ingress);2544int dev_map_generic_redirect(struct bpf_dtab_netdev *dst, struct sk_buff *skb,2545			     struct bpf_prog *xdp_prog);2546int dev_map_redirect_multi(struct net_device *dev, struct sk_buff *skb,2547			   struct bpf_prog *xdp_prog, struct bpf_map *map,2548			   bool exclude_ingress);2549 2550void __cpu_map_flush(struct list_head *flush_list);2551int cpu_map_enqueue(struct bpf_cpu_map_entry *rcpu, struct xdp_frame *xdpf,2552		    struct net_device *dev_rx);2553int cpu_map_generic_redirect(struct bpf_cpu_map_entry *rcpu,2554			     struct sk_buff *skb);2555 2556/* Return map's numa specified by userspace */2557static inline int bpf_map_attr_numa_node(const union bpf_attr *attr)2558{2559	return (attr->map_flags & BPF_F_NUMA_NODE) ?2560		attr->numa_node : NUMA_NO_NODE;2561}2562 2563struct bpf_prog *bpf_prog_get_type_path(const char *name, enum bpf_prog_type type);2564int array_map_alloc_check(union bpf_attr *attr);2565 2566int bpf_prog_test_run_xdp(struct bpf_prog *prog, const union bpf_attr *kattr,2567			  union bpf_attr __user *uattr);2568int bpf_prog_test_run_skb(struct bpf_prog *prog, const union bpf_attr *kattr,2569			  union bpf_attr __user *uattr);2570int bpf_prog_test_run_tracing(struct bpf_prog *prog,2571			      const union bpf_attr *kattr,2572			      union bpf_attr __user *uattr);2573int bpf_prog_test_run_flow_dissector(struct bpf_prog *prog,2574				     const union bpf_attr *kattr,2575				     union bpf_attr __user *uattr);2576int bpf_prog_test_run_raw_tp(struct bpf_prog *prog,2577			     const union bpf_attr *kattr,2578			     union bpf_attr __user *uattr);2579int bpf_prog_test_run_sk_lookup(struct bpf_prog *prog,2580				const union bpf_attr *kattr,2581				union bpf_attr __user *uattr);2582int bpf_prog_test_run_nf(struct bpf_prog *prog,2583			 const union bpf_attr *kattr,2584			 union bpf_attr __user *uattr);2585bool btf_ctx_access(int off, int size, enum bpf_access_type type,2586		    const struct bpf_prog *prog,2587		    struct bpf_insn_access_aux *info);2588 2589static inline bool bpf_tracing_ctx_access(int off, int size,2590					  enum bpf_access_type type)2591{2592	if (off < 0 || off >= sizeof(__u64) * MAX_BPF_FUNC_ARGS)2593		return false;2594	if (type != BPF_READ)2595		return false;2596	if (off % size != 0)2597		return false;2598	return true;2599}2600 2601static inline bool bpf_tracing_btf_ctx_access(int off, int size,2602					      enum bpf_access_type type,2603					      const struct bpf_prog *prog,2604					      struct bpf_insn_access_aux *info)2605{2606	if (!bpf_tracing_ctx_access(off, size, type))2607		return false;2608	return btf_ctx_access(off, size, type, prog, info);2609}2610 2611int btf_struct_access(struct bpf_verifier_log *log,2612		      const struct bpf_reg_state *reg,2613		      int off, int size, enum bpf_access_type atype,2614		      u32 *next_btf_id, enum bpf_type_flag *flag, const char **field_name);2615bool btf_struct_ids_match(struct bpf_verifier_log *log,2616			  const struct btf *btf, u32 id, int off,2617			  const struct btf *need_btf, u32 need_type_id,2618			  bool strict);2619 2620int btf_distill_func_proto(struct bpf_verifier_log *log,2621			   struct btf *btf,2622			   const struct btf_type *func_proto,2623			   const char *func_name,2624			   struct btf_func_model *m);2625 2626struct bpf_reg_state;2627int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog);2628int btf_check_type_match(struct bpf_verifier_log *log, const struct bpf_prog *prog,2629			 struct btf *btf, const struct btf_type *t);2630const char *btf_find_decl_tag_value(const struct btf *btf, const struct btf_type *pt,2631				    int comp_idx, const char *tag_key);2632int btf_find_next_decl_tag(const struct btf *btf, const struct btf_type *pt,2633			   int comp_idx, const char *tag_key, int last_id);2634 2635struct bpf_prog *bpf_prog_by_id(u32 id);2636struct bpf_link *bpf_link_by_id(u32 id);2637 2638const struct bpf_func_proto *bpf_base_func_proto(enum bpf_func_id func_id,2639						 const struct bpf_prog *prog);2640void bpf_task_storage_free(struct task_struct *task);2641void bpf_cgrp_storage_free(struct cgroup *cgroup);2642bool bpf_prog_has_kfunc_call(const struct bpf_prog *prog);2643const struct btf_func_model *2644bpf_jit_find_kfunc_model(const struct bpf_prog *prog,2645			 const struct bpf_insn *insn);2646int bpf_get_kfunc_addr(const struct bpf_prog *prog, u32 func_id,2647		       u16 btf_fd_idx, u8 **func_addr);2648 2649struct bpf_core_ctx {2650	struct bpf_verifier_log *log;2651	const struct btf *btf;2652};2653 2654bool btf_nested_type_is_trusted(struct bpf_verifier_log *log,2655				const struct bpf_reg_state *reg,2656				const char *field_name, u32 btf_id, const char *suffix);2657 2658bool btf_type_ids_nocast_alias(struct bpf_verifier_log *log,2659			       const struct btf *reg_btf, u32 reg_id,2660			       const struct btf *arg_btf, u32 arg_id);2661 2662int bpf_core_apply(struct bpf_core_ctx *ctx, const struct bpf_core_relo *relo,2663		   int relo_idx, void *insn);2664 2665static inline bool unprivileged_ebpf_enabled(void)2666{2667	return !sysctl_unprivileged_bpf_disabled;2668}2669 2670/* Not all bpf prog type has the bpf_ctx.2671 * For the bpf prog type that has initialized the bpf_ctx,2672 * this function can be used to decide if a kernel function2673 * is called by a bpf program.2674 */2675static inline bool has_current_bpf_ctx(void)2676{2677	return !!current->bpf_ctx;2678}2679 2680void notrace bpf_prog_inc_misses_counter(struct bpf_prog *prog);2681 2682void bpf_dynptr_init(struct bpf_dynptr_kern *ptr, void *data,2683		     enum bpf_dynptr_type type, u32 offset, u32 size);2684void bpf_dynptr_set_null(struct bpf_dynptr_kern *ptr);2685void bpf_dynptr_set_rdonly(struct bpf_dynptr_kern *ptr);2686 2687#else /* !CONFIG_BPF_SYSCALL */2688static inline struct bpf_prog *bpf_prog_get(u32 ufd)2689{2690	return ERR_PTR(-EOPNOTSUPP);2691}2692 2693static inline struct bpf_prog *bpf_prog_get_type_dev(u32 ufd,2694						     enum bpf_prog_type type,2695						     bool attach_drv)2696{2697	return ERR_PTR(-EOPNOTSUPP);2698}2699 2700static inline void bpf_prog_add(struct bpf_prog *prog, int i)2701{2702}2703 2704static inline void bpf_prog_sub(struct bpf_prog *prog, int i)2705{2706}2707 2708static inline void bpf_prog_put(struct bpf_prog *prog)2709{2710}2711 2712static inline void bpf_prog_inc(struct bpf_prog *prog)2713{2714}2715 2716static inline struct bpf_prog *__must_check2717bpf_prog_inc_not_zero(struct bpf_prog *prog)2718{2719	return ERR_PTR(-EOPNOTSUPP);2720}2721 2722static inline void bpf_link_init(struct bpf_link *link, enum bpf_link_type type,2723				 const struct bpf_link_ops *ops,2724				 struct bpf_prog *prog)2725{2726}2727 2728static inline int bpf_link_prime(struct bpf_link *link,2729				 struct bpf_link_primer *primer)2730{2731	return -EOPNOTSUPP;2732}2733 2734static inline int bpf_link_settle(struct bpf_link_primer *primer)2735{2736	return -EOPNOTSUPP;2737}2738 2739static inline void bpf_link_cleanup(struct bpf_link_primer *primer)2740{2741}2742 2743static inline void bpf_link_inc(struct bpf_link *link)2744{2745}2746 2747static inline struct bpf_link *bpf_link_inc_not_zero(struct bpf_link *link)2748{2749	return NULL;2750}2751 2752static inline void bpf_link_put(struct bpf_link *link)2753{2754}2755 2756static inline int bpf_obj_get_user(const char __user *pathname, int flags)2757{2758	return -EOPNOTSUPP;2759}2760 2761static inline bool bpf_token_capable(const struct bpf_token *token, int cap)2762{2763	return capable(cap) || (cap != CAP_SYS_ADMIN && capable(CAP_SYS_ADMIN));2764}2765 2766static inline void bpf_token_inc(struct bpf_token *token)2767{2768}2769 2770static inline void bpf_token_put(struct bpf_token *token)2771{2772}2773 2774static inline struct bpf_token *bpf_token_get_from_fd(u32 ufd)2775{2776	return ERR_PTR(-EOPNOTSUPP);2777}2778 2779static inline void __dev_flush(struct list_head *flush_list)2780{2781}2782 2783struct xdp_frame;2784struct bpf_dtab_netdev;2785struct bpf_cpu_map_entry;2786 2787static inline2788int dev_xdp_enqueue(struct net_device *dev, struct xdp_frame *xdpf,2789		    struct net_device *dev_rx)2790{2791	return 0;2792}2793 2794static inline2795int dev_map_enqueue(struct bpf_dtab_netdev *dst, struct xdp_frame *xdpf,2796		    struct net_device *dev_rx)2797{2798	return 0;2799}2800 2801static inline2802int dev_map_enqueue_multi(struct xdp_frame *xdpf, struct net_device *dev_rx,2803			  struct bpf_map *map, bool exclude_ingress)2804{2805	return 0;2806}2807 2808struct sk_buff;2809 2810static inline int dev_map_generic_redirect(struct bpf_dtab_netdev *dst,2811					   struct sk_buff *skb,2812					   struct bpf_prog *xdp_prog)2813{2814	return 0;2815}2816 2817static inline2818int dev_map_redirect_multi(struct net_device *dev, struct sk_buff *skb,2819			   struct bpf_prog *xdp_prog, struct bpf_map *map,2820			   bool exclude_ingress)2821{2822	return 0;2823}2824 2825static inline void __cpu_map_flush(struct list_head *flush_list)2826{2827}2828 2829static inline int cpu_map_enqueue(struct bpf_cpu_map_entry *rcpu,2830				  struct xdp_frame *xdpf,2831				  struct net_device *dev_rx)2832{2833	return 0;2834}2835 2836static inline int cpu_map_generic_redirect(struct bpf_cpu_map_entry *rcpu,2837					   struct sk_buff *skb)2838{2839	return -EOPNOTSUPP;2840}2841 2842static inline struct bpf_prog *bpf_prog_get_type_path(const char *name,2843				enum bpf_prog_type type)2844{2845	return ERR_PTR(-EOPNOTSUPP);2846}2847 2848static inline int bpf_prog_test_run_xdp(struct bpf_prog *prog,2849					const union bpf_attr *kattr,2850					union bpf_attr __user *uattr)2851{2852	return -ENOTSUPP;2853}2854 2855static inline int bpf_prog_test_run_skb(struct bpf_prog *prog,2856					const union bpf_attr *kattr,2857					union bpf_attr __user *uattr)2858{2859	return -ENOTSUPP;2860}2861 2862static inline int bpf_prog_test_run_tracing(struct bpf_prog *prog,2863					    const union bpf_attr *kattr,2864					    union bpf_attr __user *uattr)2865{2866	return -ENOTSUPP;2867}2868 2869static inline int bpf_prog_test_run_flow_dissector(struct bpf_prog *prog,2870						   const union bpf_attr *kattr,2871						   union bpf_attr __user *uattr)2872{2873	return -ENOTSUPP;2874}2875 2876static inline int bpf_prog_test_run_sk_lookup(struct bpf_prog *prog,2877					      const union bpf_attr *kattr,2878					      union bpf_attr __user *uattr)2879{2880	return -ENOTSUPP;2881}2882 2883static inline void bpf_map_put(struct bpf_map *map)2884{2885}2886 2887static inline struct bpf_prog *bpf_prog_by_id(u32 id)2888{2889	return ERR_PTR(-ENOTSUPP);2890}2891 2892static inline int btf_struct_access(struct bpf_verifier_log *log,2893				    const struct bpf_reg_state *reg,2894				    int off, int size, enum bpf_access_type atype,2895				    u32 *next_btf_id, enum bpf_type_flag *flag,2896				    const char **field_name)2897{2898	return -EACCES;2899}2900 2901static inline const struct bpf_func_proto *2902bpf_base_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)2903{2904	return NULL;2905}2906 2907static inline void bpf_task_storage_free(struct task_struct *task)2908{2909}2910 2911static inline bool bpf_prog_has_kfunc_call(const struct bpf_prog *prog)2912{2913	return false;2914}2915 2916static inline const struct btf_func_model *2917bpf_jit_find_kfunc_model(const struct bpf_prog *prog,2918			 const struct bpf_insn *insn)2919{2920	return NULL;2921}2922 2923static inline int2924bpf_get_kfunc_addr(const struct bpf_prog *prog, u32 func_id,2925		   u16 btf_fd_idx, u8 **func_addr)2926{2927	return -ENOTSUPP;2928}2929 2930static inline bool unprivileged_ebpf_enabled(void)2931{2932	return false;2933}2934 2935static inline bool has_current_bpf_ctx(void)2936{2937	return false;2938}2939 2940static inline void bpf_prog_inc_misses_counter(struct bpf_prog *prog)2941{2942}2943 2944static inline void bpf_cgrp_storage_free(struct cgroup *cgroup)2945{2946}2947 2948static inline void bpf_dynptr_init(struct bpf_dynptr_kern *ptr, void *data,2949				   enum bpf_dynptr_type type, u32 offset, u32 size)2950{2951}2952 2953static inline void bpf_dynptr_set_null(struct bpf_dynptr_kern *ptr)2954{2955}2956 2957static inline void bpf_dynptr_set_rdonly(struct bpf_dynptr_kern *ptr)2958{2959}2960#endif /* CONFIG_BPF_SYSCALL */2961 2962static __always_inline int2963bpf_probe_read_kernel_common(void *dst, u32 size, const void *unsafe_ptr)2964{2965	int ret = -EFAULT;2966 2967	if (IS_ENABLED(CONFIG_BPF_EVENTS))2968		ret = copy_from_kernel_nofault(dst, unsafe_ptr, size);2969	if (unlikely(ret < 0))2970		memset(dst, 0, size);2971	return ret;2972}2973 2974void __bpf_free_used_btfs(struct btf_mod_pair *used_btfs, u32 len);2975 2976static inline struct bpf_prog *bpf_prog_get_type(u32 ufd,2977						 enum bpf_prog_type type)2978{2979	return bpf_prog_get_type_dev(ufd, type, false);2980}2981 2982void __bpf_free_used_maps(struct bpf_prog_aux *aux,2983			  struct bpf_map **used_maps, u32 len);2984 2985bool bpf_prog_get_ok(struct bpf_prog *, enum bpf_prog_type *, bool);2986 2987int bpf_prog_offload_compile(struct bpf_prog *prog);2988void bpf_prog_dev_bound_destroy(struct bpf_prog *prog);2989int bpf_prog_offload_info_fill(struct bpf_prog_info *info,2990			       struct bpf_prog *prog);2991 2992int bpf_map_offload_info_fill(struct bpf_map_info *info, struct bpf_map *map);2993 2994int bpf_map_offload_lookup_elem(struct bpf_map *map, void *key, void *value);2995int bpf_map_offload_update_elem(struct bpf_map *map,2996				void *key, void *value, u64 flags);2997int bpf_map_offload_delete_elem(struct bpf_map *map, void *key);2998int bpf_map_offload_get_next_key(struct bpf_map *map,2999				 void *key, void *next_key);3000 3001bool bpf_offload_prog_map_match(struct bpf_prog *prog, struct bpf_map *map);3002 3003struct bpf_offload_dev *3004bpf_offload_dev_create(const struct bpf_prog_offload_ops *ops, void *priv);3005void bpf_offload_dev_destroy(struct bpf_offload_dev *offdev);3006void *bpf_offload_dev_priv(struct bpf_offload_dev *offdev);3007int bpf_offload_dev_netdev_register(struct bpf_offload_dev *offdev,3008				    struct net_device *netdev);3009void bpf_offload_dev_netdev_unregister(struct bpf_offload_dev *offdev,3010				       struct net_device *netdev);3011bool bpf_offload_dev_match(struct bpf_prog *prog, struct net_device *netdev);3012 3013void unpriv_ebpf_notify(int new_state);3014 3015#if defined(CONFIG_NET) && defined(CONFIG_BPF_SYSCALL)3016int bpf_dev_bound_kfunc_check(struct bpf_verifier_log *log,3017			      struct bpf_prog_aux *prog_aux);3018void *bpf_dev_bound_resolve_kfunc(struct bpf_prog *prog, u32 func_id);3019int bpf_prog_dev_bound_init(struct bpf_prog *prog, union bpf_attr *attr);3020int bpf_prog_dev_bound_inherit(struct bpf_prog *new_prog, struct bpf_prog *old_prog);3021void bpf_dev_bound_netdev_unregister(struct net_device *dev);3022 3023static inline bool bpf_prog_is_dev_bound(const struct bpf_prog_aux *aux)3024{3025	return aux->dev_bound;3026}3027 3028static inline bool bpf_prog_is_offloaded(const struct bpf_prog_aux *aux)3029{3030	return aux->offload_requested;3031}3032 3033bool bpf_prog_dev_bound_match(const struct bpf_prog *lhs, const struct bpf_prog *rhs);3034 3035static inline bool bpf_map_is_offloaded(struct bpf_map *map)3036{3037	return unlikely(map->ops == &bpf_map_offload_ops);3038}3039 3040struct bpf_map *bpf_map_offload_map_alloc(union bpf_attr *attr);3041void bpf_map_offload_map_free(struct bpf_map *map);3042u64 bpf_map_offload_map_mem_usage(const struct bpf_map *map);3043int bpf_prog_test_run_syscall(struct bpf_prog *prog,3044			      const union bpf_attr *kattr,3045			      union bpf_attr __user *uattr);3046 3047int sock_map_get_from_fd(const union bpf_attr *attr, struct bpf_prog *prog);3048int sock_map_prog_detach(const union bpf_attr *attr, enum bpf_prog_type ptype);3049int sock_map_update_elem_sys(struct bpf_map *map, void *key, void *value, u64 flags);3050int sock_map_bpf_prog_query(const union bpf_attr *attr,3051			    union bpf_attr __user *uattr);3052int sock_map_link_create(const union bpf_attr *attr, struct bpf_prog *prog);3053 3054void sock_map_unhash(struct sock *sk);3055void sock_map_destroy(struct sock *sk);3056void sock_map_close(struct sock *sk, long timeout);3057#else3058static inline int bpf_dev_bound_kfunc_check(struct bpf_verifier_log *log,3059					    struct bpf_prog_aux *prog_aux)3060{3061	return -EOPNOTSUPP;3062}3063 3064static inline void *bpf_dev_bound_resolve_kfunc(struct bpf_prog *prog,3065						u32 func_id)3066{3067	return NULL;3068}3069 3070static inline int bpf_prog_dev_bound_init(struct bpf_prog *prog,3071					  union bpf_attr *attr)3072{3073	return -EOPNOTSUPP;3074}3075 3076static inline int bpf_prog_dev_bound_inherit(struct bpf_prog *new_prog,3077					     struct bpf_prog *old_prog)3078{3079	return -EOPNOTSUPP;3080}3081 3082static inline void bpf_dev_bound_netdev_unregister(struct net_device *dev)3083{3084}3085 3086static inline bool bpf_prog_is_dev_bound(const struct bpf_prog_aux *aux)3087{3088	return false;3089}3090 3091static inline bool bpf_prog_is_offloaded(struct bpf_prog_aux *aux)3092{3093	return false;3094}3095 3096static inline bool bpf_prog_dev_bound_match(const struct bpf_prog *lhs, const struct bpf_prog *rhs)3097{3098	return false;3099}3100 3101static inline bool bpf_map_is_offloaded(struct bpf_map *map)3102{3103	return false;3104}3105 3106static inline struct bpf_map *bpf_map_offload_map_alloc(union bpf_attr *attr)3107{3108	return ERR_PTR(-EOPNOTSUPP);3109}3110 3111static inline void bpf_map_offload_map_free(struct bpf_map *map)3112{3113}3114 3115static inline u64 bpf_map_offload_map_mem_usage(const struct bpf_map *map)3116{3117	return 0;3118}3119 3120static inline int bpf_prog_test_run_syscall(struct bpf_prog *prog,3121					    const union bpf_attr *kattr,3122					    union bpf_attr __user *uattr)3123{3124	return -ENOTSUPP;3125}3126 3127#ifdef CONFIG_BPF_SYSCALL3128static inline int sock_map_get_from_fd(const union bpf_attr *attr,3129				       struct bpf_prog *prog)3130{3131	return -EINVAL;3132}3133 3134static inline int sock_map_prog_detach(const union bpf_attr *attr,3135				       enum bpf_prog_type ptype)3136{3137	return -EOPNOTSUPP;3138}3139 3140static inline int sock_map_update_elem_sys(struct bpf_map *map, void *key, void *value,3141					   u64 flags)3142{3143	return -EOPNOTSUPP;3144}3145 3146static inline int sock_map_bpf_prog_query(const union bpf_attr *attr,3147					  union bpf_attr __user *uattr)3148{3149	return -EINVAL;3150}3151 3152static inline int sock_map_link_create(const union bpf_attr *attr, struct bpf_prog *prog)3153{3154	return -EOPNOTSUPP;3155}3156#endif /* CONFIG_BPF_SYSCALL */3157#endif /* CONFIG_NET && CONFIG_BPF_SYSCALL */3158 3159static __always_inline void3160bpf_prog_inc_misses_counters(const struct bpf_prog_array *array)3161{3162	const struct bpf_prog_array_item *item;3163	struct bpf_prog *prog;3164 3165	if (unlikely(!array))3166		return;3167 3168	item = &array->items[0];3169	while ((prog = READ_ONCE(item->prog))) {3170		bpf_prog_inc_misses_counter(prog);3171		item++;3172	}3173}3174 3175#if defined(CONFIG_INET) && defined(CONFIG_BPF_SYSCALL)3176void bpf_sk_reuseport_detach(struct sock *sk);3177int bpf_fd_reuseport_array_lookup_elem(struct bpf_map *map, void *key,3178				       void *value);3179int bpf_fd_reuseport_array_update_elem(struct bpf_map *map, void *key,3180				       void *value, u64 map_flags);3181#else3182static inline void bpf_sk_reuseport_detach(struct sock *sk)3183{3184}3185 3186#ifdef CONFIG_BPF_SYSCALL3187static inline int bpf_fd_reuseport_array_lookup_elem(struct bpf_map *map,3188						     void *key, void *value)3189{3190	return -EOPNOTSUPP;3191}3192 3193static inline int bpf_fd_reuseport_array_update_elem(struct bpf_map *map,3194						     void *key, void *value,3195						     u64 map_flags)3196{3197	return -EOPNOTSUPP;3198}3199#endif /* CONFIG_BPF_SYSCALL */3200#endif /* defined(CONFIG_INET) && defined(CONFIG_BPF_SYSCALL) */3201 3202/* verifier prototypes for helper functions called from eBPF programs */3203extern const struct bpf_func_proto bpf_map_lookup_elem_proto;3204extern const struct bpf_func_proto bpf_map_update_elem_proto;3205extern const struct bpf_func_proto bpf_map_delete_elem_proto;3206extern const struct bpf_func_proto bpf_map_push_elem_proto;3207extern const struct bpf_func_proto bpf_map_pop_elem_proto;3208extern const struct bpf_func_proto bpf_map_peek_elem_proto;3209extern const struct bpf_func_proto bpf_map_lookup_percpu_elem_proto;3210 3211extern const struct bpf_func_proto bpf_get_prandom_u32_proto;3212extern const struct bpf_func_proto bpf_get_smp_processor_id_proto;3213extern const struct bpf_func_proto bpf_get_numa_node_id_proto;3214extern const struct bpf_func_proto bpf_tail_call_proto;3215extern const struct bpf_func_proto bpf_ktime_get_ns_proto;3216extern const struct bpf_func_proto bpf_ktime_get_boot_ns_proto;3217extern const struct bpf_func_proto bpf_ktime_get_tai_ns_proto;3218extern const struct bpf_func_proto bpf_get_current_pid_tgid_proto;3219extern const struct bpf_func_proto bpf_get_current_uid_gid_proto;3220extern const struct bpf_func_proto bpf_get_current_comm_proto;3221extern const struct bpf_func_proto bpf_get_stackid_proto;3222extern const struct bpf_func_proto bpf_get_stack_proto;3223extern const struct bpf_func_proto bpf_get_stack_sleepable_proto;3224extern const struct bpf_func_proto bpf_get_task_stack_proto;3225extern const struct bpf_func_proto bpf_get_task_stack_sleepable_proto;3226extern const struct bpf_func_proto bpf_get_stackid_proto_pe;3227extern const struct bpf_func_proto bpf_get_stack_proto_pe;3228extern const struct bpf_func_proto bpf_sock_map_update_proto;3229extern const struct bpf_func_proto bpf_sock_hash_update_proto;3230extern const struct bpf_func_proto bpf_get_current_cgroup_id_proto;3231extern const struct bpf_func_proto bpf_get_current_ancestor_cgroup_id_proto;3232extern const struct bpf_func_proto bpf_get_cgroup_classid_curr_proto;3233extern const struct bpf_func_proto bpf_current_task_under_cgroup_proto;3234extern const struct bpf_func_proto bpf_msg_redirect_hash_proto;3235extern const struct bpf_func_proto bpf_msg_redirect_map_proto;3236extern const struct bpf_func_proto bpf_sk_redirect_hash_proto;3237extern const struct bpf_func_proto bpf_sk_redirect_map_proto;3238extern const struct bpf_func_proto bpf_spin_lock_proto;3239extern const struct bpf_func_proto bpf_spin_unlock_proto;3240extern const struct bpf_func_proto bpf_get_local_storage_proto;3241extern const struct bpf_func_proto bpf_strtol_proto;3242extern const struct bpf_func_proto bpf_strtoul_proto;3243extern const struct bpf_func_proto bpf_tcp_sock_proto;3244extern const struct bpf_func_proto bpf_jiffies64_proto;3245extern const struct bpf_func_proto bpf_get_ns_current_pid_tgid_proto;3246extern const struct bpf_func_proto bpf_event_output_data_proto;3247extern const struct bpf_func_proto bpf_ringbuf_output_proto;3248extern const struct bpf_func_proto bpf_ringbuf_reserve_proto;3249extern const struct bpf_func_proto bpf_ringbuf_submit_proto;3250extern const struct bpf_func_proto bpf_ringbuf_discard_proto;3251extern const struct bpf_func_proto bpf_ringbuf_query_proto;3252extern const struct bpf_func_proto bpf_ringbuf_reserve_dynptr_proto;3253extern const struct bpf_func_proto bpf_ringbuf_submit_dynptr_proto;3254extern const struct bpf_func_proto bpf_ringbuf_discard_dynptr_proto;3255extern const struct bpf_func_proto bpf_skc_to_tcp6_sock_proto;3256extern const struct bpf_func_proto bpf_skc_to_tcp_sock_proto;3257extern const struct bpf_func_proto bpf_skc_to_tcp_timewait_sock_proto;3258extern const struct bpf_func_proto bpf_skc_to_tcp_request_sock_proto;3259extern const struct bpf_func_proto bpf_skc_to_udp6_sock_proto;3260extern const struct bpf_func_proto bpf_skc_to_unix_sock_proto;3261extern const struct bpf_func_proto bpf_skc_to_mptcp_sock_proto;3262extern const struct bpf_func_proto bpf_copy_from_user_proto;3263extern const struct bpf_func_proto bpf_snprintf_btf_proto;3264extern const struct bpf_func_proto bpf_snprintf_proto;3265extern const struct bpf_func_proto bpf_per_cpu_ptr_proto;3266extern const struct bpf_func_proto bpf_this_cpu_ptr_proto;3267extern const struct bpf_func_proto bpf_ktime_get_coarse_ns_proto;3268extern const struct bpf_func_proto bpf_sock_from_file_proto;3269extern const struct bpf_func_proto bpf_get_socket_ptr_cookie_proto;3270extern const struct bpf_func_proto bpf_task_storage_get_recur_proto;3271extern const struct bpf_func_proto bpf_task_storage_get_proto;3272extern const struct bpf_func_proto bpf_task_storage_delete_recur_proto;3273extern const struct bpf_func_proto bpf_task_storage_delete_proto;3274extern const struct bpf_func_proto bpf_for_each_map_elem_proto;3275extern const struct bpf_func_proto bpf_btf_find_by_name_kind_proto;3276extern const struct bpf_func_proto bpf_sk_setsockopt_proto;3277extern const struct bpf_func_proto bpf_sk_getsockopt_proto;3278extern const struct bpf_func_proto bpf_unlocked_sk_setsockopt_proto;3279extern const struct bpf_func_proto bpf_unlocked_sk_getsockopt_proto;3280extern const struct bpf_func_proto bpf_find_vma_proto;3281extern const struct bpf_func_proto bpf_loop_proto;3282extern const struct bpf_func_proto bpf_copy_from_user_task_proto;3283extern const struct bpf_func_proto bpf_set_retval_proto;3284extern const struct bpf_func_proto bpf_get_retval_proto;3285extern const struct bpf_func_proto bpf_user_ringbuf_drain_proto;3286extern const struct bpf_func_proto bpf_cgrp_storage_get_proto;3287extern const struct bpf_func_proto bpf_cgrp_storage_delete_proto;3288 3289const struct bpf_func_proto *tracing_prog_func_proto(3290  enum bpf_func_id func_id, const struct bpf_prog *prog);3291 3292/* Shared helpers among cBPF and eBPF. */3293void bpf_user_rnd_init_once(void);3294u64 bpf_user_rnd_u32(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5);3295u64 bpf_get_raw_cpu_id(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5);3296 3297#if defined(CONFIG_NET)3298bool bpf_sock_common_is_valid_access(int off, int size,3299				     enum bpf_access_type type,3300				     struct bpf_insn_access_aux *info);3301bool bpf_sock_is_valid_access(int off, int size, enum bpf_access_type type,3302			      struct bpf_insn_access_aux *info);3303u32 bpf_sock_convert_ctx_access(enum bpf_access_type type,3304				const struct bpf_insn *si,3305				struct bpf_insn *insn_buf,3306				struct bpf_prog *prog,3307				u32 *target_size);3308int bpf_dynptr_from_skb_rdonly(struct __sk_buff *skb, u64 flags,3309			       struct bpf_dynptr *ptr);3310#else3311static inline bool bpf_sock_common_is_valid_access(int off, int size,3312						   enum bpf_access_type type,3313						   struct bpf_insn_access_aux *info)3314{3315	return false;3316}3317static inline bool bpf_sock_is_valid_access(int off, int size,3318					    enum bpf_access_type type,3319					    struct bpf_insn_access_aux *info)3320{3321	return false;3322}3323static inline u32 bpf_sock_convert_ctx_access(enum bpf_access_type type,3324					      const struct bpf_insn *si,3325					      struct bpf_insn *insn_buf,3326					      struct bpf_prog *prog,3327					      u32 *target_size)3328{3329	return 0;3330}3331static inline int bpf_dynptr_from_skb_rdonly(struct __sk_buff *skb, u64 flags,3332					     struct bpf_dynptr *ptr)3333{3334	return -EOPNOTSUPP;3335}3336#endif3337 3338#ifdef CONFIG_INET3339struct sk_reuseport_kern {3340	struct sk_buff *skb;3341	struct sock *sk;3342	struct sock *selected_sk;3343	struct sock *migrating_sk;3344	void *data_end;3345	u32 hash;3346	u32 reuseport_id;3347	bool bind_inany;3348};3349bool bpf_tcp_sock_is_valid_access(int off, int size, enum bpf_access_type type,3350				  struct bpf_insn_access_aux *info);3351 3352u32 bpf_tcp_sock_convert_ctx_access(enum bpf_access_type type,3353				    const struct bpf_insn *si,3354				    struct bpf_insn *insn_buf,3355				    struct bpf_prog *prog,3356				    u32 *target_size);3357 3358bool bpf_xdp_sock_is_valid_access(int off, int size, enum bpf_access_type type,3359				  struct bpf_insn_access_aux *info);3360 3361u32 bpf_xdp_sock_convert_ctx_access(enum bpf_access_type type,3362				    const struct bpf_insn *si,3363				    struct bpf_insn *insn_buf,3364				    struct bpf_prog *prog,3365				    u32 *target_size);3366#else3367static inline bool bpf_tcp_sock_is_valid_access(int off, int size,3368						enum bpf_access_type type,3369						struct bpf_insn_access_aux *info)3370{3371	return false;3372}3373 3374static inline u32 bpf_tcp_sock_convert_ctx_access(enum bpf_access_type type,3375						  const struct bpf_insn *si,3376						  struct bpf_insn *insn_buf,3377						  struct bpf_prog *prog,3378						  u32 *target_size)3379{3380	return 0;3381}3382static inline bool bpf_xdp_sock_is_valid_access(int off, int size,3383						enum bpf_access_type type,3384						struct bpf_insn_access_aux *info)3385{3386	return false;3387}3388 3389static inline u32 bpf_xdp_sock_convert_ctx_access(enum bpf_access_type type,3390						  const struct bpf_insn *si,3391						  struct bpf_insn *insn_buf,3392						  struct bpf_prog *prog,3393						  u32 *target_size)3394{3395	return 0;3396}3397#endif /* CONFIG_INET */3398 3399enum bpf_text_poke_type {3400	BPF_MOD_CALL,3401	BPF_MOD_JUMP,3402};3403 3404int bpf_arch_text_poke(void *ip, enum bpf_text_poke_type t,3405		       void *addr1, void *addr2);3406 3407void bpf_arch_poke_desc_update(struct bpf_jit_poke_descriptor *poke,3408			       struct bpf_prog *new, struct bpf_prog *old);3409 3410void *bpf_arch_text_copy(void *dst, void *src, size_t len);3411int bpf_arch_text_invalidate(void *dst, size_t len);3412 3413struct btf_id_set;3414bool btf_id_set_contains(const struct btf_id_set *set, u32 id);3415 3416#define MAX_BPRINTF_VARARGS		123417#define MAX_BPRINTF_BUF			10243418 3419struct bpf_bprintf_data {3420	u32 *bin_args;3421	char *buf;3422	bool get_bin_args;3423	bool get_buf;3424};3425 3426int bpf_bprintf_prepare(char *fmt, u32 fmt_size, const u64 *raw_args,3427			u32 num_args, struct bpf_bprintf_data *data);3428void bpf_bprintf_cleanup(struct bpf_bprintf_data *data);3429 3430#ifdef CONFIG_BPF_LSM3431void bpf_cgroup_atype_get(u32 attach_btf_id, int cgroup_atype);3432void bpf_cgroup_atype_put(int cgroup_atype);3433#else3434static inline void bpf_cgroup_atype_get(u32 attach_btf_id, int cgroup_atype) {}3435static inline void bpf_cgroup_atype_put(int cgroup_atype) {}3436#endif /* CONFIG_BPF_LSM */3437 3438struct key;3439 3440#ifdef CONFIG_KEYS3441struct bpf_key {3442	struct key *key;3443	bool has_ref;3444};3445#endif /* CONFIG_KEYS */3446 3447static inline bool type_is_alloc(u32 type)3448{3449	return type & MEM_ALLOC;3450}3451 3452static inline gfp_t bpf_memcg_flags(gfp_t flags)3453{3454	if (memcg_bpf_enabled())3455		return flags | __GFP_ACCOUNT;3456	return flags;3457}3458 3459static inline bool bpf_is_subprog(const struct bpf_prog *prog)3460{3461	return prog->aux->func_idx != 0;3462}3463 3464#endif /* _LINUX_BPF_H */3465