brintos / linux-shallow public Read only

1// SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause)2/*3 * BPF static linker4 *5 * Copyright (c) 2021 Facebook6 */7#include <stdbool.h>8#include <stddef.h>9#include <stdio.h>10#include <stdlib.h>11#include <string.h>12#include <unistd.h>13#include <errno.h>14#include <linux/err.h>15#include <linux/btf.h>16#include <elf.h>17#include <libelf.h>18#include <fcntl.h>19#include "libbpf.h"20#include "btf.h"21#include "libbpf_internal.h"22#include "strset.h"23 24#define BTF_EXTERN_SEC ".extern"25 26struct src_sec {27	const char *sec_name;28	/* positional (not necessarily ELF) index in an array of sections */29	int id;30	/* positional (not necessarily ELF) index of a matching section in a final object file */31	int dst_id;32	/* section data offset in a matching output section */33	int dst_off;34	/* whether section is omitted from the final ELF file */35	bool skipped;36	/* whether section is an ephemeral section, not mapped to an ELF section */37	bool ephemeral;38 39	/* ELF info */40	size_t sec_idx;41	Elf_Scn *scn;42	Elf64_Shdr *shdr;43	Elf_Data *data;44 45	/* corresponding BTF DATASEC type ID */46	int sec_type_id;47};48 49struct src_obj {50	const char *filename;51	int fd;52	Elf *elf;53	/* Section header strings section index */54	size_t shstrs_sec_idx;55	/* SYMTAB section index */56	size_t symtab_sec_idx;57 58	struct btf *btf;59	struct btf_ext *btf_ext;60 61	/* List of sections (including ephemeral). Slot zero is unused. */62	struct src_sec *secs;63	int sec_cnt;64 65	/* mapping of symbol indices from src to dst ELF */66	int *sym_map;67	/* mapping from the src BTF type IDs to dst ones */68	int *btf_type_map;69};70 71/* single .BTF.ext data section */72struct btf_ext_sec_data {73	size_t rec_cnt;74	__u32 rec_sz;75	void *recs;76};77 78struct glob_sym {79	/* ELF symbol index */80	int sym_idx;81	/* associated section id for .ksyms, .kconfig, etc, but not .extern */82	int sec_id;83	/* extern name offset in STRTAB */84	int name_off;85	/* optional associated BTF type ID */86	int btf_id;87	/* BTF type ID to which VAR/FUNC type is pointing to; used for88	 * rewriting types when extern VAR/FUNC is resolved to a concrete89	 * definition90	 */91	int underlying_btf_id;92	/* sec_var index in the corresponding dst_sec, if exists */93	int var_idx;94 95	/* extern or resolved/global symbol */96	bool is_extern;97	/* weak or strong symbol, never goes back from strong to weak */98	bool is_weak;99};100 101struct dst_sec {102	char *sec_name;103	/* positional (not necessarily ELF) index in an array of sections */104	int id;105 106	bool ephemeral;107 108	/* ELF info */109	size_t sec_idx;110	Elf_Scn *scn;111	Elf64_Shdr *shdr;112	Elf_Data *data;113 114	/* final output section size */115	int sec_sz;116	/* final output contents of the section */117	void *raw_data;118 119	/* corresponding STT_SECTION symbol index in SYMTAB */120	int sec_sym_idx;121 122	/* section's DATASEC variable info, emitted on BTF finalization */123	bool has_btf;124	int sec_var_cnt;125	struct btf_var_secinfo *sec_vars;126 127	/* section's .BTF.ext data */128	struct btf_ext_sec_data func_info;129	struct btf_ext_sec_data line_info;130	struct btf_ext_sec_data core_relo_info;131};132 133struct bpf_linker {134	char *filename;135	int fd;136	Elf *elf;137	Elf64_Ehdr *elf_hdr;138 139	/* Output sections metadata */140	struct dst_sec *secs;141	int sec_cnt;142 143	struct strset *strtab_strs; /* STRTAB unique strings */144	size_t strtab_sec_idx; /* STRTAB section index */145	size_t symtab_sec_idx; /* SYMTAB section index */146 147	struct btf *btf;148	struct btf_ext *btf_ext;149 150	/* global (including extern) ELF symbols */151	int glob_sym_cnt;152	struct glob_sym *glob_syms;153};154 155#define pr_warn_elf(fmt, ...)									\156	libbpf_print(LIBBPF_WARN, "libbpf: " fmt ": %s\n", ##__VA_ARGS__, elf_errmsg(-1))157 158static int init_output_elf(struct bpf_linker *linker, const char *file);159 160static int linker_load_obj_file(struct bpf_linker *linker, const char *filename,161				const struct bpf_linker_file_opts *opts,162				struct src_obj *obj);163static int linker_sanity_check_elf(struct src_obj *obj);164static int linker_sanity_check_elf_symtab(struct src_obj *obj, struct src_sec *sec);165static int linker_sanity_check_elf_relos(struct src_obj *obj, struct src_sec *sec);166static int linker_sanity_check_btf(struct src_obj *obj);167static int linker_sanity_check_btf_ext(struct src_obj *obj);168static int linker_fixup_btf(struct src_obj *obj);169static int linker_append_sec_data(struct bpf_linker *linker, struct src_obj *obj);170static int linker_append_elf_syms(struct bpf_linker *linker, struct src_obj *obj);171static int linker_append_elf_sym(struct bpf_linker *linker, struct src_obj *obj,172				 Elf64_Sym *sym, const char *sym_name, int src_sym_idx);173static int linker_append_elf_relos(struct bpf_linker *linker, struct src_obj *obj);174static int linker_append_btf(struct bpf_linker *linker, struct src_obj *obj);175static int linker_append_btf_ext(struct bpf_linker *linker, struct src_obj *obj);176 177static int finalize_btf(struct bpf_linker *linker);178static int finalize_btf_ext(struct bpf_linker *linker);179 180void bpf_linker__free(struct bpf_linker *linker)181{182	int i;183 184	if (!linker)185		return;186 187	free(linker->filename);188 189	if (linker->elf)190		elf_end(linker->elf);191 192	if (linker->fd >= 0)193		close(linker->fd);194 195	strset__free(linker->strtab_strs);196 197	btf__free(linker->btf);198	btf_ext__free(linker->btf_ext);199 200	for (i = 1; i < linker->sec_cnt; i++) {201		struct dst_sec *sec = &linker->secs[i];202 203		free(sec->sec_name);204		free(sec->raw_data);205		free(sec->sec_vars);206 207		free(sec->func_info.recs);208		free(sec->line_info.recs);209		free(sec->core_relo_info.recs);210	}211	free(linker->secs);212 213	free(linker->glob_syms);214	free(linker);215}216 217struct bpf_linker *bpf_linker__new(const char *filename, struct bpf_linker_opts *opts)218{219	struct bpf_linker *linker;220	int err;221 222	if (!OPTS_VALID(opts, bpf_linker_opts))223		return errno = EINVAL, NULL;224 225	if (elf_version(EV_CURRENT) == EV_NONE) {226		pr_warn_elf("libelf initialization failed");227		return errno = EINVAL, NULL;228	}229 230	linker = calloc(1, sizeof(*linker));231	if (!linker)232		return errno = ENOMEM, NULL;233 234	linker->fd = -1;235 236	err = init_output_elf(linker, filename);237	if (err)238		goto err_out;239 240	return linker;241 242err_out:243	bpf_linker__free(linker);244	return errno = -err, NULL;245}246 247static struct dst_sec *add_dst_sec(struct bpf_linker *linker, const char *sec_name)248{249	struct dst_sec *secs = linker->secs, *sec;250	size_t new_cnt = linker->sec_cnt ? linker->sec_cnt + 1 : 2;251 252	secs = libbpf_reallocarray(secs, new_cnt, sizeof(*secs));253	if (!secs)254		return NULL;255 256	/* zero out newly allocated memory */257	memset(secs + linker->sec_cnt, 0, (new_cnt - linker->sec_cnt) * sizeof(*secs));258 259	linker->secs = secs;260	linker->sec_cnt = new_cnt;261 262	sec = &linker->secs[new_cnt - 1];263	sec->id = new_cnt - 1;264	sec->sec_name = strdup(sec_name);265	if (!sec->sec_name)266		return NULL;267 268	return sec;269}270 271static Elf64_Sym *add_new_sym(struct bpf_linker *linker, size_t *sym_idx)272{273	struct dst_sec *symtab = &linker->secs[linker->symtab_sec_idx];274	Elf64_Sym *syms, *sym;275	size_t sym_cnt = symtab->sec_sz / sizeof(*sym);276 277	syms = libbpf_reallocarray(symtab->raw_data, sym_cnt + 1, sizeof(*sym));278	if (!syms)279		return NULL;280 281	sym = &syms[sym_cnt];282	memset(sym, 0, sizeof(*sym));283 284	symtab->raw_data = syms;285	symtab->sec_sz += sizeof(*sym);286	symtab->shdr->sh_size += sizeof(*sym);287	symtab->data->d_size += sizeof(*sym);288 289	if (sym_idx)290		*sym_idx = sym_cnt;291 292	return sym;293}294 295static int init_output_elf(struct bpf_linker *linker, const char *file)296{297	int err, str_off;298	Elf64_Sym *init_sym;299	struct dst_sec *sec;300 301	linker->filename = strdup(file);302	if (!linker->filename)303		return -ENOMEM;304 305	linker->fd = open(file, O_WRONLY | O_CREAT | O_TRUNC | O_CLOEXEC, 0644);306	if (linker->fd < 0) {307		err = -errno;308		pr_warn("failed to create '%s': %d\n", file, err);309		return err;310	}311 312	linker->elf = elf_begin(linker->fd, ELF_C_WRITE, NULL);313	if (!linker->elf) {314		pr_warn_elf("failed to create ELF object");315		return -EINVAL;316	}317 318	/* ELF header */319	linker->elf_hdr = elf64_newehdr(linker->elf);320	if (!linker->elf_hdr) {321		pr_warn_elf("failed to create ELF header");322		return -EINVAL;323	}324 325	linker->elf_hdr->e_machine = EM_BPF;326	linker->elf_hdr->e_type = ET_REL;327#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__328	linker->elf_hdr->e_ident[EI_DATA] = ELFDATA2LSB;329#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__330	linker->elf_hdr->e_ident[EI_DATA] = ELFDATA2MSB;331#else332#error "Unknown __BYTE_ORDER__"333#endif334 335	/* STRTAB */336	/* initialize strset with an empty string to conform to ELF */337	linker->strtab_strs = strset__new(INT_MAX, "", sizeof(""));338	if (libbpf_get_error(linker->strtab_strs))339		return libbpf_get_error(linker->strtab_strs);340 341	sec = add_dst_sec(linker, ".strtab");342	if (!sec)343		return -ENOMEM;344 345	sec->scn = elf_newscn(linker->elf);346	if (!sec->scn) {347		pr_warn_elf("failed to create STRTAB section");348		return -EINVAL;349	}350 351	sec->shdr = elf64_getshdr(sec->scn);352	if (!sec->shdr)353		return -EINVAL;354 355	sec->data = elf_newdata(sec->scn);356	if (!sec->data) {357		pr_warn_elf("failed to create STRTAB data");358		return -EINVAL;359	}360 361	str_off = strset__add_str(linker->strtab_strs, sec->sec_name);362	if (str_off < 0)363		return str_off;364 365	sec->sec_idx = elf_ndxscn(sec->scn);366	linker->elf_hdr->e_shstrndx = sec->sec_idx;367	linker->strtab_sec_idx = sec->sec_idx;368 369	sec->shdr->sh_name = str_off;370	sec->shdr->sh_type = SHT_STRTAB;371	sec->shdr->sh_flags = SHF_STRINGS;372	sec->shdr->sh_offset = 0;373	sec->shdr->sh_link = 0;374	sec->shdr->sh_info = 0;375	sec->shdr->sh_addralign = 1;376	sec->shdr->sh_size = sec->sec_sz = 0;377	sec->shdr->sh_entsize = 0;378 379	/* SYMTAB */380	sec = add_dst_sec(linker, ".symtab");381	if (!sec)382		return -ENOMEM;383 384	sec->scn = elf_newscn(linker->elf);385	if (!sec->scn) {386		pr_warn_elf("failed to create SYMTAB section");387		return -EINVAL;388	}389 390	sec->shdr = elf64_getshdr(sec->scn);391	if (!sec->shdr)392		return -EINVAL;393 394	sec->data = elf_newdata(sec->scn);395	if (!sec->data) {396		pr_warn_elf("failed to create SYMTAB data");397		return -EINVAL;398	}399 400	str_off = strset__add_str(linker->strtab_strs, sec->sec_name);401	if (str_off < 0)402		return str_off;403 404	sec->sec_idx = elf_ndxscn(sec->scn);405	linker->symtab_sec_idx = sec->sec_idx;406 407	sec->shdr->sh_name = str_off;408	sec->shdr->sh_type = SHT_SYMTAB;409	sec->shdr->sh_flags = 0;410	sec->shdr->sh_offset = 0;411	sec->shdr->sh_link = linker->strtab_sec_idx;412	/* sh_info should be one greater than the index of the last local413	 * symbol (i.e., binding is STB_LOCAL). But why and who cares?414	 */415	sec->shdr->sh_info = 0;416	sec->shdr->sh_addralign = 8;417	sec->shdr->sh_entsize = sizeof(Elf64_Sym);418 419	/* .BTF */420	linker->btf = btf__new_empty();421	err = libbpf_get_error(linker->btf);422	if (err)423		return err;424 425	/* add the special all-zero symbol */426	init_sym = add_new_sym(linker, NULL);427	if (!init_sym)428		return -EINVAL;429 430	init_sym->st_name = 0;431	init_sym->st_info = 0;432	init_sym->st_other = 0;433	init_sym->st_shndx = SHN_UNDEF;434	init_sym->st_value = 0;435	init_sym->st_size = 0;436 437	return 0;438}439 440int bpf_linker__add_file(struct bpf_linker *linker, const char *filename,441			 const struct bpf_linker_file_opts *opts)442{443	struct src_obj obj = {};444	int err = 0;445 446	if (!OPTS_VALID(opts, bpf_linker_file_opts))447		return libbpf_err(-EINVAL);448 449	if (!linker->elf)450		return libbpf_err(-EINVAL);451 452	err = err ?: linker_load_obj_file(linker, filename, opts, &obj);453	err = err ?: linker_append_sec_data(linker, &obj);454	err = err ?: linker_append_elf_syms(linker, &obj);455	err = err ?: linker_append_elf_relos(linker, &obj);456	err = err ?: linker_append_btf(linker, &obj);457	err = err ?: linker_append_btf_ext(linker, &obj);458 459	/* free up src_obj resources */460	free(obj.btf_type_map);461	btf__free(obj.btf);462	btf_ext__free(obj.btf_ext);463	free(obj.secs);464	free(obj.sym_map);465	if (obj.elf)466		elf_end(obj.elf);467	if (obj.fd >= 0)468		close(obj.fd);469 470	return libbpf_err(err);471}472 473static bool is_dwarf_sec_name(const char *name)474{475	/* approximation, but the actual list is too long */476	return strncmp(name, ".debug_", sizeof(".debug_") - 1) == 0;477}478 479static bool is_ignored_sec(struct src_sec *sec)480{481	Elf64_Shdr *shdr = sec->shdr;482	const char *name = sec->sec_name;483 484	/* no special handling of .strtab */485	if (shdr->sh_type == SHT_STRTAB)486		return true;487 488	/* ignore .llvm_addrsig section as well */489	if (shdr->sh_type == SHT_LLVM_ADDRSIG)490		return true;491 492	/* no subprograms will lead to an empty .text section, ignore it */493	if (shdr->sh_type == SHT_PROGBITS && shdr->sh_size == 0 &&494	    strcmp(sec->sec_name, ".text") == 0)495		return true;496 497	/* DWARF sections */498	if (is_dwarf_sec_name(sec->sec_name))499		return true;500 501	if (strncmp(name, ".rel", sizeof(".rel") - 1) == 0) {502		name += sizeof(".rel") - 1;503		/* DWARF section relocations */504		if (is_dwarf_sec_name(name))505			return true;506 507		/* .BTF and .BTF.ext don't need relocations */508		if (strcmp(name, BTF_ELF_SEC) == 0 ||509		    strcmp(name, BTF_EXT_ELF_SEC) == 0)510			return true;511	}512 513	return false;514}515 516static struct src_sec *add_src_sec(struct src_obj *obj, const char *sec_name)517{518	struct src_sec *secs = obj->secs, *sec;519	size_t new_cnt = obj->sec_cnt ? obj->sec_cnt + 1 : 2;520 521	secs = libbpf_reallocarray(secs, new_cnt, sizeof(*secs));522	if (!secs)523		return NULL;524 525	/* zero out newly allocated memory */526	memset(secs + obj->sec_cnt, 0, (new_cnt - obj->sec_cnt) * sizeof(*secs));527 528	obj->secs = secs;529	obj->sec_cnt = new_cnt;530 531	sec = &obj->secs[new_cnt - 1];532	sec->id = new_cnt - 1;533	sec->sec_name = sec_name;534 535	return sec;536}537 538static int linker_load_obj_file(struct bpf_linker *linker, const char *filename,539				const struct bpf_linker_file_opts *opts,540				struct src_obj *obj)541{542#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__543	const int host_endianness = ELFDATA2LSB;544#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__545	const int host_endianness = ELFDATA2MSB;546#else547#error "Unknown __BYTE_ORDER__"548#endif549	int err = 0;550	Elf_Scn *scn;551	Elf_Data *data;552	Elf64_Ehdr *ehdr;553	Elf64_Shdr *shdr;554	struct src_sec *sec;555 556	pr_debug("linker: adding object file '%s'...\n", filename);557 558	obj->filename = filename;559 560	obj->fd = open(filename, O_RDONLY | O_CLOEXEC);561	if (obj->fd < 0) {562		err = -errno;563		pr_warn("failed to open file '%s': %d\n", filename, err);564		return err;565	}566	obj->elf = elf_begin(obj->fd, ELF_C_READ_MMAP, NULL);567	if (!obj->elf) {568		err = -errno;569		pr_warn_elf("failed to parse ELF file '%s'", filename);570		return err;571	}572 573	/* Sanity check ELF file high-level properties */574	ehdr = elf64_getehdr(obj->elf);575	if (!ehdr) {576		err = -errno;577		pr_warn_elf("failed to get ELF header for %s", filename);578		return err;579	}580	if (ehdr->e_ident[EI_DATA] != host_endianness) {581		err = -EOPNOTSUPP;582		pr_warn_elf("unsupported byte order of ELF file %s", filename);583		return err;584	}585	if (ehdr->e_type != ET_REL586	    || ehdr->e_machine != EM_BPF587	    || ehdr->e_ident[EI_CLASS] != ELFCLASS64) {588		err = -EOPNOTSUPP;589		pr_warn_elf("unsupported kind of ELF file %s", filename);590		return err;591	}592 593	if (elf_getshdrstrndx(obj->elf, &obj->shstrs_sec_idx)) {594		err = -errno;595		pr_warn_elf("failed to get SHSTRTAB section index for %s", filename);596		return err;597	}598 599	scn = NULL;600	while ((scn = elf_nextscn(obj->elf, scn)) != NULL) {601		size_t sec_idx = elf_ndxscn(scn);602		const char *sec_name;603 604		shdr = elf64_getshdr(scn);605		if (!shdr) {606			err = -errno;607			pr_warn_elf("failed to get section #%zu header for %s",608				    sec_idx, filename);609			return err;610		}611 612		sec_name = elf_strptr(obj->elf, obj->shstrs_sec_idx, shdr->sh_name);613		if (!sec_name) {614			err = -errno;615			pr_warn_elf("failed to get section #%zu name for %s",616				    sec_idx, filename);617			return err;618		}619 620		data = elf_getdata(scn, 0);621		if (!data) {622			err = -errno;623			pr_warn_elf("failed to get section #%zu (%s) data from %s",624				    sec_idx, sec_name, filename);625			return err;626		}627 628		sec = add_src_sec(obj, sec_name);629		if (!sec)630			return -ENOMEM;631 632		sec->scn = scn;633		sec->shdr = shdr;634		sec->data = data;635		sec->sec_idx = elf_ndxscn(scn);636 637		if (is_ignored_sec(sec)) {638			sec->skipped = true;639			continue;640		}641 642		switch (shdr->sh_type) {643		case SHT_SYMTAB:644			if (obj->symtab_sec_idx) {645				err = -EOPNOTSUPP;646				pr_warn("multiple SYMTAB sections found, not supported\n");647				return err;648			}649			obj->symtab_sec_idx = sec_idx;650			break;651		case SHT_STRTAB:652			/* we'll construct our own string table */653			break;654		case SHT_PROGBITS:655			if (strcmp(sec_name, BTF_ELF_SEC) == 0) {656				obj->btf = btf__new(data->d_buf, shdr->sh_size);657				err = libbpf_get_error(obj->btf);658				if (err) {659					pr_warn("failed to parse .BTF from %s: %d\n", filename, err);660					return err;661				}662				sec->skipped = true;663				continue;664			}665			if (strcmp(sec_name, BTF_EXT_ELF_SEC) == 0) {666				obj->btf_ext = btf_ext__new(data->d_buf, shdr->sh_size);667				err = libbpf_get_error(obj->btf_ext);668				if (err) {669					pr_warn("failed to parse .BTF.ext from '%s': %d\n", filename, err);670					return err;671				}672				sec->skipped = true;673				continue;674			}675 676			/* data & code */677			break;678		case SHT_NOBITS:679			/* BSS */680			break;681		case SHT_REL:682			/* relocations */683			break;684		default:685			pr_warn("unrecognized section #%zu (%s) in %s\n",686				sec_idx, sec_name, filename);687			err = -EINVAL;688			return err;689		}690	}691 692	err = err ?: linker_sanity_check_elf(obj);693	err = err ?: linker_sanity_check_btf(obj);694	err = err ?: linker_sanity_check_btf_ext(obj);695	err = err ?: linker_fixup_btf(obj);696 697	return err;698}699 700static int linker_sanity_check_elf(struct src_obj *obj)701{702	struct src_sec *sec;703	int i, err;704 705	if (!obj->symtab_sec_idx) {706		pr_warn("ELF is missing SYMTAB section in %s\n", obj->filename);707		return -EINVAL;708	}709	if (!obj->shstrs_sec_idx) {710		pr_warn("ELF is missing section headers STRTAB section in %s\n", obj->filename);711		return -EINVAL;712	}713 714	for (i = 1; i < obj->sec_cnt; i++) {715		sec = &obj->secs[i];716 717		if (sec->sec_name[0] == '\0') {718			pr_warn("ELF section #%zu has empty name in %s\n", sec->sec_idx, obj->filename);719			return -EINVAL;720		}721 722		if (is_dwarf_sec_name(sec->sec_name))723			continue;724 725		if (sec->shdr->sh_addralign && !is_pow_of_2(sec->shdr->sh_addralign)) {726			pr_warn("ELF section #%zu alignment %llu is non pow-of-2 alignment in %s\n",727				sec->sec_idx, (long long unsigned)sec->shdr->sh_addralign,728				obj->filename);729			return -EINVAL;730		}731		if (sec->shdr->sh_addralign != sec->data->d_align) {732			pr_warn("ELF section #%zu has inconsistent alignment addr=%llu != d=%llu in %s\n",733				sec->sec_idx, (long long unsigned)sec->shdr->sh_addralign,734				(long long unsigned)sec->data->d_align, obj->filename);735			return -EINVAL;736		}737 738		if (sec->shdr->sh_size != sec->data->d_size) {739			pr_warn("ELF section #%zu has inconsistent section size sh=%llu != d=%llu in %s\n",740				sec->sec_idx, (long long unsigned)sec->shdr->sh_size,741				(long long unsigned)sec->data->d_size, obj->filename);742			return -EINVAL;743		}744 745		switch (sec->shdr->sh_type) {746		case SHT_SYMTAB:747			err = linker_sanity_check_elf_symtab(obj, sec);748			if (err)749				return err;750			break;751		case SHT_STRTAB:752			break;753		case SHT_PROGBITS:754			if (sec->shdr->sh_flags & SHF_EXECINSTR) {755				if (sec->shdr->sh_size % sizeof(struct bpf_insn) != 0) {756					pr_warn("ELF section #%zu has unexpected size alignment %llu in %s\n",757						sec->sec_idx, (long long unsigned)sec->shdr->sh_size,758						obj->filename);759					return -EINVAL;760				}761			}762			break;763		case SHT_NOBITS:764			break;765		case SHT_REL:766			err = linker_sanity_check_elf_relos(obj, sec);767			if (err)768				return err;769			break;770		case SHT_LLVM_ADDRSIG:771			break;772		default:773			pr_warn("ELF section #%zu (%s) has unrecognized type %zu in %s\n",774				sec->sec_idx, sec->sec_name, (size_t)sec->shdr->sh_type, obj->filename);775			return -EINVAL;776		}777	}778 779	return 0;780}781 782static int linker_sanity_check_elf_symtab(struct src_obj *obj, struct src_sec *sec)783{784	struct src_sec *link_sec;785	Elf64_Sym *sym;786	int i, n;787 788	if (sec->shdr->sh_entsize != sizeof(Elf64_Sym))789		return -EINVAL;790	if (sec->shdr->sh_size % sec->shdr->sh_entsize != 0)791		return -EINVAL;792 793	if (!sec->shdr->sh_link || sec->shdr->sh_link >= obj->sec_cnt) {794		pr_warn("ELF SYMTAB section #%zu points to missing STRTAB section #%zu in %s\n",795			sec->sec_idx, (size_t)sec->shdr->sh_link, obj->filename);796		return -EINVAL;797	}798	link_sec = &obj->secs[sec->shdr->sh_link];799	if (link_sec->shdr->sh_type != SHT_STRTAB) {800		pr_warn("ELF SYMTAB section #%zu points to invalid STRTAB section #%zu in %s\n",801			sec->sec_idx, (size_t)sec->shdr->sh_link, obj->filename);802		return -EINVAL;803	}804 805	n = sec->shdr->sh_size / sec->shdr->sh_entsize;806	sym = sec->data->d_buf;807	for (i = 0; i < n; i++, sym++) {808		int sym_type = ELF64_ST_TYPE(sym->st_info);809		int sym_bind = ELF64_ST_BIND(sym->st_info);810		int sym_vis = ELF64_ST_VISIBILITY(sym->st_other);811 812		if (i == 0) {813			if (sym->st_name != 0 || sym->st_info != 0814			    || sym->st_other != 0 || sym->st_shndx != 0815			    || sym->st_value != 0 || sym->st_size != 0) {816				pr_warn("ELF sym #0 is invalid in %s\n", obj->filename);817				return -EINVAL;818			}819			continue;820		}821		if (sym_bind != STB_LOCAL && sym_bind != STB_GLOBAL && sym_bind != STB_WEAK) {822			pr_warn("ELF sym #%d in section #%zu has unsupported symbol binding %d\n",823				i, sec->sec_idx, sym_bind);824			return -EINVAL;825		}826		if (sym_vis != STV_DEFAULT && sym_vis != STV_HIDDEN) {827			pr_warn("ELF sym #%d in section #%zu has unsupported symbol visibility %d\n",828				i, sec->sec_idx, sym_vis);829			return -EINVAL;830		}831		if (sym->st_shndx == 0) {832			if (sym_type != STT_NOTYPE || sym_bind == STB_LOCAL833			    || sym->st_value != 0 || sym->st_size != 0) {834				pr_warn("ELF sym #%d is invalid extern symbol in %s\n",835					i, obj->filename);836 837				return -EINVAL;838			}839			continue;840		}841		if (sym->st_shndx < SHN_LORESERVE && sym->st_shndx >= obj->sec_cnt) {842			pr_warn("ELF sym #%d in section #%zu points to missing section #%zu in %s\n",843				i, sec->sec_idx, (size_t)sym->st_shndx, obj->filename);844			return -EINVAL;845		}846		if (sym_type == STT_SECTION) {847			if (sym->st_value != 0)848				return -EINVAL;849			continue;850		}851	}852 853	return 0;854}855 856static int linker_sanity_check_elf_relos(struct src_obj *obj, struct src_sec *sec)857{858	struct src_sec *link_sec, *sym_sec;859	Elf64_Rel *relo;860	int i, n;861 862	if (sec->shdr->sh_entsize != sizeof(Elf64_Rel))863		return -EINVAL;864	if (sec->shdr->sh_size % sec->shdr->sh_entsize != 0)865		return -EINVAL;866 867	/* SHT_REL's sh_link should point to SYMTAB */868	if (sec->shdr->sh_link != obj->symtab_sec_idx) {869		pr_warn("ELF relo section #%zu points to invalid SYMTAB section #%zu in %s\n",870			sec->sec_idx, (size_t)sec->shdr->sh_link, obj->filename);871		return -EINVAL;872	}873 874	/* SHT_REL's sh_info points to relocated section */875	if (!sec->shdr->sh_info || sec->shdr->sh_info >= obj->sec_cnt) {876		pr_warn("ELF relo section #%zu points to missing section #%zu in %s\n",877			sec->sec_idx, (size_t)sec->shdr->sh_info, obj->filename);878		return -EINVAL;879	}880	link_sec = &obj->secs[sec->shdr->sh_info];881 882	/* .rel<secname> -> <secname> pattern is followed */883	if (strncmp(sec->sec_name, ".rel", sizeof(".rel") - 1) != 0884	    || strcmp(sec->sec_name + sizeof(".rel") - 1, link_sec->sec_name) != 0) {885		pr_warn("ELF relo section #%zu name has invalid name in %s\n",886			sec->sec_idx, obj->filename);887		return -EINVAL;888	}889 890	/* don't further validate relocations for ignored sections */891	if (link_sec->skipped)892		return 0;893 894	/* relocatable section is data or instructions */895	if (link_sec->shdr->sh_type != SHT_PROGBITS && link_sec->shdr->sh_type != SHT_NOBITS) {896		pr_warn("ELF relo section #%zu points to invalid section #%zu in %s\n",897			sec->sec_idx, (size_t)sec->shdr->sh_info, obj->filename);898		return -EINVAL;899	}900 901	/* check sanity of each relocation */902	n = sec->shdr->sh_size / sec->shdr->sh_entsize;903	relo = sec->data->d_buf;904	sym_sec = &obj->secs[obj->symtab_sec_idx];905	for (i = 0; i < n; i++, relo++) {906		size_t sym_idx = ELF64_R_SYM(relo->r_info);907		size_t sym_type = ELF64_R_TYPE(relo->r_info);908 909		if (sym_type != R_BPF_64_64 && sym_type != R_BPF_64_32 &&910		    sym_type != R_BPF_64_ABS64 && sym_type != R_BPF_64_ABS32) {911			pr_warn("ELF relo #%d in section #%zu has unexpected type %zu in %s\n",912				i, sec->sec_idx, sym_type, obj->filename);913			return -EINVAL;914		}915 916		if (!sym_idx || sym_idx * sizeof(Elf64_Sym) >= sym_sec->shdr->sh_size) {917			pr_warn("ELF relo #%d in section #%zu points to invalid symbol #%zu in %s\n",918				i, sec->sec_idx, sym_idx, obj->filename);919			return -EINVAL;920		}921 922		if (link_sec->shdr->sh_flags & SHF_EXECINSTR) {923			if (relo->r_offset % sizeof(struct bpf_insn) != 0) {924				pr_warn("ELF relo #%d in section #%zu points to missing symbol #%zu in %s\n",925					i, sec->sec_idx, sym_idx, obj->filename);926				return -EINVAL;927			}928		}929	}930 931	return 0;932}933 934static int check_btf_type_id(__u32 *type_id, void *ctx)935{936	struct btf *btf = ctx;937 938	if (*type_id >= btf__type_cnt(btf))939		return -EINVAL;940 941	return 0;942}943 944static int check_btf_str_off(__u32 *str_off, void *ctx)945{946	struct btf *btf = ctx;947	const char *s;948 949	s = btf__str_by_offset(btf, *str_off);950 951	if (!s)952		return -EINVAL;953 954	return 0;955}956 957static int linker_sanity_check_btf(struct src_obj *obj)958{959	struct btf_type *t;960	int i, n, err;961 962	if (!obj->btf)963		return 0;964 965	n = btf__type_cnt(obj->btf);966	for (i = 1; i < n; i++) {967		struct btf_field_iter it;968		__u32 *type_id, *str_off;969 970		t = btf_type_by_id(obj->btf, i);971 972		err = btf_field_iter_init(&it, t, BTF_FIELD_ITER_IDS);973		if (err)974			return err;975		while ((type_id = btf_field_iter_next(&it))) {976			if (*type_id >= n)977				return -EINVAL;978		}979 980		err = btf_field_iter_init(&it, t, BTF_FIELD_ITER_STRS);981		if (err)982			return err;983		while ((str_off = btf_field_iter_next(&it))) {984			if (!btf__str_by_offset(obj->btf, *str_off))985				return -EINVAL;986		}987	}988 989	return 0;990}991 992static int linker_sanity_check_btf_ext(struct src_obj *obj)993{994	int err = 0;995 996	if (!obj->btf_ext)997		return 0;998 999	/* can't use .BTF.ext without .BTF */1000	if (!obj->btf)1001		return -EINVAL;1002 1003	err = err ?: btf_ext_visit_type_ids(obj->btf_ext, check_btf_type_id, obj->btf);1004	err = err ?: btf_ext_visit_str_offs(obj->btf_ext, check_btf_str_off, obj->btf);1005	if (err)1006		return err;1007 1008	return 0;1009}1010 1011static int init_sec(struct bpf_linker *linker, struct dst_sec *dst_sec, struct src_sec *src_sec)1012{1013	Elf_Scn *scn;1014	Elf_Data *data;1015	Elf64_Shdr *shdr;1016	int name_off;1017 1018	dst_sec->sec_sz = 0;1019	dst_sec->sec_idx = 0;1020	dst_sec->ephemeral = src_sec->ephemeral;1021 1022	/* ephemeral sections are just thin section shells lacking most parts */1023	if (src_sec->ephemeral)1024		return 0;1025 1026	scn = elf_newscn(linker->elf);1027	if (!scn)1028		return -ENOMEM;1029	data = elf_newdata(scn);1030	if (!data)1031		return -ENOMEM;1032	shdr = elf64_getshdr(scn);1033	if (!shdr)1034		return -ENOMEM;1035 1036	dst_sec->scn = scn;1037	dst_sec->shdr = shdr;1038	dst_sec->data = data;1039	dst_sec->sec_idx = elf_ndxscn(scn);1040 1041	name_off = strset__add_str(linker->strtab_strs, src_sec->sec_name);1042	if (name_off < 0)1043		return name_off;1044 1045	shdr->sh_name = name_off;1046	shdr->sh_type = src_sec->shdr->sh_type;1047	shdr->sh_flags = src_sec->shdr->sh_flags;1048	shdr->sh_size = 0;1049	/* sh_link and sh_info have different meaning for different types of1050	 * sections, so we leave it up to the caller code to fill them in, if1051	 * necessary1052	 */1053	shdr->sh_link = 0;1054	shdr->sh_info = 0;1055	shdr->sh_addralign = src_sec->shdr->sh_addralign;1056	shdr->sh_entsize = src_sec->shdr->sh_entsize;1057 1058	data->d_type = src_sec->data->d_type;1059	data->d_size = 0;1060	data->d_buf = NULL;1061	data->d_align = src_sec->data->d_align;1062	data->d_off = 0;1063 1064	return 0;1065}1066 1067static struct dst_sec *find_dst_sec_by_name(struct bpf_linker *linker, const char *sec_name)1068{1069	struct dst_sec *sec;1070	int i;1071 1072	for (i = 1; i < linker->sec_cnt; i++) {1073		sec = &linker->secs[i];1074 1075		if (strcmp(sec->sec_name, sec_name) == 0)1076			return sec;1077	}1078 1079	return NULL;1080}1081 1082static bool secs_match(struct dst_sec *dst, struct src_sec *src)1083{1084	if (dst->ephemeral || src->ephemeral)1085		return true;1086 1087	if (dst->shdr->sh_type != src->shdr->sh_type) {1088		pr_warn("sec %s types mismatch\n", dst->sec_name);1089		return false;1090	}1091	if (dst->shdr->sh_flags != src->shdr->sh_flags) {1092		pr_warn("sec %s flags mismatch\n", dst->sec_name);1093		return false;1094	}1095	if (dst->shdr->sh_entsize != src->shdr->sh_entsize) {1096		pr_warn("sec %s entsize mismatch\n", dst->sec_name);1097		return false;1098	}1099 1100	return true;1101}1102 1103static bool sec_content_is_same(struct dst_sec *dst_sec, struct src_sec *src_sec)1104{1105	if (dst_sec->sec_sz != src_sec->shdr->sh_size)1106		return false;1107	if (memcmp(dst_sec->raw_data, src_sec->data->d_buf, dst_sec->sec_sz) != 0)1108		return false;1109	return true;1110}1111 1112static int extend_sec(struct bpf_linker *linker, struct dst_sec *dst, struct src_sec *src)1113{1114	void *tmp;1115	size_t dst_align, src_align;1116	size_t dst_align_sz, dst_final_sz;1117	int err;1118 1119	/* Ephemeral source section doesn't contribute anything to ELF1120	 * section data.1121	 */1122	if (src->ephemeral)1123		return 0;1124 1125	/* Some sections (like .maps) can contain both externs (and thus be1126	 * ephemeral) and non-externs (map definitions). So it's possible that1127	 * it has to be "upgraded" from ephemeral to non-ephemeral when the1128	 * first non-ephemeral entity appears. In such case, we add ELF1129	 * section, data, etc.1130	 */1131	if (dst->ephemeral) {1132		err = init_sec(linker, dst, src);1133		if (err)1134			return err;1135	}1136 1137	dst_align = dst->shdr->sh_addralign;1138	src_align = src->shdr->sh_addralign;1139	if (dst_align == 0)1140		dst_align = 1;1141	if (dst_align < src_align)1142		dst_align = src_align;1143 1144	dst_align_sz = (dst->sec_sz + dst_align - 1) / dst_align * dst_align;1145 1146	/* no need to re-align final size */1147	dst_final_sz = dst_align_sz + src->shdr->sh_size;1148 1149	if (src->shdr->sh_type != SHT_NOBITS) {1150		tmp = realloc(dst->raw_data, dst_final_sz);1151		/* If dst_align_sz == 0, realloc() behaves in a special way:1152		 * 1. When dst->raw_data is NULL it returns:1153		 *    "either NULL or a pointer suitable to be passed to free()" [1].1154		 * 2. When dst->raw_data is not-NULL it frees dst->raw_data and returns NULL,1155		 *    thus invalidating any "pointer suitable to be passed to free()" obtained1156		 *    at step (1).1157		 *1158		 * The dst_align_sz > 0 check avoids error exit after (2), otherwise1159		 * dst->raw_data would be freed again in bpf_linker__free().1160		 *1161		 * [1] man 3 realloc1162		 */1163		if (!tmp && dst_align_sz > 0)1164			return -ENOMEM;1165		dst->raw_data = tmp;1166 1167		/* pad dst section, if it's alignment forced size increase */1168		memset(dst->raw_data + dst->sec_sz, 0, dst_align_sz - dst->sec_sz);1169		/* now copy src data at a properly aligned offset */1170		memcpy(dst->raw_data + dst_align_sz, src->data->d_buf, src->shdr->sh_size);1171	}1172 1173	dst->sec_sz = dst_final_sz;1174	dst->shdr->sh_size = dst_final_sz;1175	dst->data->d_size = dst_final_sz;1176 1177	dst->shdr->sh_addralign = dst_align;1178	dst->data->d_align = dst_align;1179 1180	src->dst_off = dst_align_sz;1181 1182	return 0;1183}1184 1185static bool is_data_sec(struct src_sec *sec)1186{1187	if (!sec || sec->skipped)1188		return false;1189	/* ephemeral sections are data sections, e.g., .kconfig, .ksyms */1190	if (sec->ephemeral)1191		return true;1192	return sec->shdr->sh_type == SHT_PROGBITS || sec->shdr->sh_type == SHT_NOBITS;1193}1194 1195static bool is_relo_sec(struct src_sec *sec)1196{1197	if (!sec || sec->skipped || sec->ephemeral)1198		return false;1199	return sec->shdr->sh_type == SHT_REL;1200}1201 1202static int linker_append_sec_data(struct bpf_linker *linker, struct src_obj *obj)1203{1204	int i, err;1205 1206	for (i = 1; i < obj->sec_cnt; i++) {1207		struct src_sec *src_sec;1208		struct dst_sec *dst_sec;1209 1210		src_sec = &obj->secs[i];1211		if (!is_data_sec(src_sec))1212			continue;1213 1214		dst_sec = find_dst_sec_by_name(linker, src_sec->sec_name);1215		if (!dst_sec) {1216			dst_sec = add_dst_sec(linker, src_sec->sec_name);1217			if (!dst_sec)1218				return -ENOMEM;1219			err = init_sec(linker, dst_sec, src_sec);1220			if (err) {1221				pr_warn("failed to init section '%s'\n", src_sec->sec_name);1222				return err;1223			}1224		} else {1225			if (!secs_match(dst_sec, src_sec)) {1226				pr_warn("ELF sections %s are incompatible\n", src_sec->sec_name);1227				return -1;1228			}1229 1230			/* "license" and "version" sections are deduped */1231			if (strcmp(src_sec->sec_name, "license") == 01232			    || strcmp(src_sec->sec_name, "version") == 0) {1233				if (!sec_content_is_same(dst_sec, src_sec)) {1234					pr_warn("non-identical contents of section '%s' are not supported\n", src_sec->sec_name);1235					return -EINVAL;1236				}1237				src_sec->skipped = true;1238				src_sec->dst_id = dst_sec->id;1239				continue;1240			}1241		}1242 1243		/* record mapped section index */1244		src_sec->dst_id = dst_sec->id;1245 1246		err = extend_sec(linker, dst_sec, src_sec);1247		if (err)1248			return err;1249	}1250 1251	return 0;1252}1253 1254static int linker_append_elf_syms(struct bpf_linker *linker, struct src_obj *obj)1255{1256	struct src_sec *symtab = &obj->secs[obj->symtab_sec_idx];1257	Elf64_Sym *sym = symtab->data->d_buf;1258	int i, n = symtab->shdr->sh_size / symtab->shdr->sh_entsize, err;1259	int str_sec_idx = symtab->shdr->sh_link;1260	const char *sym_name;1261 1262	obj->sym_map = calloc(n + 1, sizeof(*obj->sym_map));1263	if (!obj->sym_map)1264		return -ENOMEM;1265 1266	for (i = 0; i < n; i++, sym++) {1267		/* We already validated all-zero symbol #0 and we already1268		 * appended it preventively to the final SYMTAB, so skip it.1269		 */1270		if (i == 0)1271			continue;1272 1273		sym_name = elf_strptr(obj->elf, str_sec_idx, sym->st_name);1274		if (!sym_name) {1275			pr_warn("can't fetch symbol name for symbol #%d in '%s'\n", i, obj->filename);1276			return -EINVAL;1277		}1278 1279		err = linker_append_elf_sym(linker, obj, sym, sym_name, i);1280		if (err)1281			return err;1282	}1283 1284	return 0;1285}1286 1287static Elf64_Sym *get_sym_by_idx(struct bpf_linker *linker, size_t sym_idx)1288{1289	struct dst_sec *symtab = &linker->secs[linker->symtab_sec_idx];1290	Elf64_Sym *syms = symtab->raw_data;1291 1292	return &syms[sym_idx];1293}1294 1295static struct glob_sym *find_glob_sym(struct bpf_linker *linker, const char *sym_name)1296{1297	struct glob_sym *glob_sym;1298	const char *name;1299	int i;1300 1301	for (i = 0; i < linker->glob_sym_cnt; i++) {1302		glob_sym = &linker->glob_syms[i];1303		name = strset__data(linker->strtab_strs) + glob_sym->name_off;1304 1305		if (strcmp(name, sym_name) == 0)1306			return glob_sym;1307	}1308 1309	return NULL;1310}1311 1312static struct glob_sym *add_glob_sym(struct bpf_linker *linker)1313{1314	struct glob_sym *syms, *sym;1315 1316	syms = libbpf_reallocarray(linker->glob_syms, linker->glob_sym_cnt + 1,1317				   sizeof(*linker->glob_syms));1318	if (!syms)1319		return NULL;1320 1321	sym = &syms[linker->glob_sym_cnt];1322	memset(sym, 0, sizeof(*sym));1323	sym->var_idx = -1;1324 1325	linker->glob_syms = syms;1326	linker->glob_sym_cnt++;1327 1328	return sym;1329}1330 1331static bool glob_sym_btf_matches(const char *sym_name, bool exact,1332				 const struct btf *btf1, __u32 id1,1333				 const struct btf *btf2, __u32 id2)1334{1335	const struct btf_type *t1, *t2;1336	bool is_static1, is_static2;1337	const char *n1, *n2;1338	int i, n;1339 1340recur:1341	n1 = n2 = NULL;1342	t1 = skip_mods_and_typedefs(btf1, id1, &id1);1343	t2 = skip_mods_and_typedefs(btf2, id2, &id2);1344 1345	/* check if only one side is FWD, otherwise handle with common logic */1346	if (!exact && btf_is_fwd(t1) != btf_is_fwd(t2)) {1347		n1 = btf__str_by_offset(btf1, t1->name_off);1348		n2 = btf__str_by_offset(btf2, t2->name_off);1349		if (strcmp(n1, n2) != 0) {1350			pr_warn("global '%s': incompatible forward declaration names '%s' and '%s'\n",1351				sym_name, n1, n2);1352			return false;1353		}1354		/* validate if FWD kind matches concrete kind */1355		if (btf_is_fwd(t1)) {1356			if (btf_kflag(t1) && btf_is_union(t2))1357				return true;1358			if (!btf_kflag(t1) && btf_is_struct(t2))1359				return true;1360			pr_warn("global '%s': incompatible %s forward declaration and concrete kind %s\n",1361				sym_name, btf_kflag(t1) ? "union" : "struct", btf_kind_str(t2));1362		} else {1363			if (btf_kflag(t2) && btf_is_union(t1))1364				return true;1365			if (!btf_kflag(t2) && btf_is_struct(t1))1366				return true;1367			pr_warn("global '%s': incompatible %s forward declaration and concrete kind %s\n",1368				sym_name, btf_kflag(t2) ? "union" : "struct", btf_kind_str(t1));1369		}1370		return false;1371	}1372 1373	if (btf_kind(t1) != btf_kind(t2)) {1374		pr_warn("global '%s': incompatible BTF kinds %s and %s\n",1375			sym_name, btf_kind_str(t1), btf_kind_str(t2));1376		return false;1377	}1378 1379	switch (btf_kind(t1)) {1380	case BTF_KIND_STRUCT:1381	case BTF_KIND_UNION:1382	case BTF_KIND_ENUM:1383	case BTF_KIND_ENUM64:1384	case BTF_KIND_FWD:1385	case BTF_KIND_FUNC:1386	case BTF_KIND_VAR:1387		n1 = btf__str_by_offset(btf1, t1->name_off);1388		n2 = btf__str_by_offset(btf2, t2->name_off);1389		if (strcmp(n1, n2) != 0) {1390			pr_warn("global '%s': incompatible %s names '%s' and '%s'\n",1391				sym_name, btf_kind_str(t1), n1, n2);1392			return false;1393		}1394		break;1395	default:1396		break;1397	}1398 1399	switch (btf_kind(t1)) {1400	case BTF_KIND_UNKN: /* void */1401	case BTF_KIND_FWD:1402		return true;1403	case BTF_KIND_INT:1404	case BTF_KIND_FLOAT:1405	case BTF_KIND_ENUM:1406	case BTF_KIND_ENUM64:1407		/* ignore encoding for int and enum values for enum */1408		if (t1->size != t2->size) {1409			pr_warn("global '%s': incompatible %s '%s' size %u and %u\n",1410				sym_name, btf_kind_str(t1), n1, t1->size, t2->size);1411			return false;1412		}1413		return true;1414	case BTF_KIND_PTR:1415		/* just validate overall shape of the referenced type, so no1416		 * contents comparison for struct/union, and allowed fwd vs1417		 * struct/union1418		 */1419		exact = false;1420		id1 = t1->type;1421		id2 = t2->type;1422		goto recur;1423	case BTF_KIND_ARRAY:1424		/* ignore index type and array size */1425		id1 = btf_array(t1)->type;1426		id2 = btf_array(t2)->type;1427		goto recur;1428	case BTF_KIND_FUNC:1429		/* extern and global linkages are compatible */1430		is_static1 = btf_func_linkage(t1) == BTF_FUNC_STATIC;1431		is_static2 = btf_func_linkage(t2) == BTF_FUNC_STATIC;1432		if (is_static1 != is_static2) {1433			pr_warn("global '%s': incompatible func '%s' linkage\n", sym_name, n1);1434			return false;1435		}1436 1437		id1 = t1->type;1438		id2 = t2->type;1439		goto recur;1440	case BTF_KIND_VAR:1441		/* extern and global linkages are compatible */1442		is_static1 = btf_var(t1)->linkage == BTF_VAR_STATIC;1443		is_static2 = btf_var(t2)->linkage == BTF_VAR_STATIC;1444		if (is_static1 != is_static2) {1445			pr_warn("global '%s': incompatible var '%s' linkage\n", sym_name, n1);1446			return false;1447		}1448 1449		id1 = t1->type;1450		id2 = t2->type;1451		goto recur;1452	case BTF_KIND_STRUCT:1453	case BTF_KIND_UNION: {1454		const struct btf_member *m1, *m2;1455 1456		if (!exact)1457			return true;1458 1459		if (btf_vlen(t1) != btf_vlen(t2)) {1460			pr_warn("global '%s': incompatible number of %s fields %u and %u\n",1461				sym_name, btf_kind_str(t1), btf_vlen(t1), btf_vlen(t2));1462			return false;1463		}1464 1465		n = btf_vlen(t1);1466		m1 = btf_members(t1);1467		m2 = btf_members(t2);1468		for (i = 0; i < n; i++, m1++, m2++) {1469			n1 = btf__str_by_offset(btf1, m1->name_off);1470			n2 = btf__str_by_offset(btf2, m2->name_off);1471			if (strcmp(n1, n2) != 0) {1472				pr_warn("global '%s': incompatible field #%d names '%s' and '%s'\n",1473					sym_name, i, n1, n2);1474				return false;1475			}1476			if (m1->offset != m2->offset) {1477				pr_warn("global '%s': incompatible field #%d ('%s') offsets\n",1478					sym_name, i, n1);1479				return false;1480			}1481			if (!glob_sym_btf_matches(sym_name, exact, btf1, m1->type, btf2, m2->type))1482				return false;1483		}1484 1485		return true;1486	}1487	case BTF_KIND_FUNC_PROTO: {1488		const struct btf_param *m1, *m2;1489 1490		if (btf_vlen(t1) != btf_vlen(t2)) {1491			pr_warn("global '%s': incompatible number of %s params %u and %u\n",1492				sym_name, btf_kind_str(t1), btf_vlen(t1), btf_vlen(t2));1493			return false;1494		}1495 1496		n = btf_vlen(t1);1497		m1 = btf_params(t1);1498		m2 = btf_params(t2);1499		for (i = 0; i < n; i++, m1++, m2++) {1500			/* ignore func arg names */1501			if (!glob_sym_btf_matches(sym_name, exact, btf1, m1->type, btf2, m2->type))1502				return false;1503		}1504 1505		/* now check return type as well */1506		id1 = t1->type;1507		id2 = t2->type;1508		goto recur;1509	}1510 1511	/* skip_mods_and_typedefs() make this impossible */1512	case BTF_KIND_TYPEDEF:1513	case BTF_KIND_VOLATILE:1514	case BTF_KIND_CONST:1515	case BTF_KIND_RESTRICT:1516	/* DATASECs are never compared with each other */1517	case BTF_KIND_DATASEC:1518	default:1519		pr_warn("global '%s': unsupported BTF kind %s\n",1520			sym_name, btf_kind_str(t1));1521		return false;1522	}1523}1524 1525static bool map_defs_match(const char *sym_name,1526			   const struct btf *main_btf,1527			   const struct btf_map_def *main_def,1528			   const struct btf_map_def *main_inner_def,1529			   const struct btf *extra_btf,1530			   const struct btf_map_def *extra_def,1531			   const struct btf_map_def *extra_inner_def)1532{1533	const char *reason;1534 1535	if (main_def->map_type != extra_def->map_type) {1536		reason = "type";1537		goto mismatch;1538	}1539 1540	/* check key type/size match */1541	if (main_def->key_size != extra_def->key_size) {1542		reason = "key_size";1543		goto mismatch;1544	}1545	if (!!main_def->key_type_id != !!extra_def->key_type_id) {1546		reason = "key type";1547		goto mismatch;1548	}1549	if ((main_def->parts & MAP_DEF_KEY_TYPE)1550	     && !glob_sym_btf_matches(sym_name, true /*exact*/,1551				      main_btf, main_def->key_type_id,1552				      extra_btf, extra_def->key_type_id)) {1553		reason = "key type";1554		goto mismatch;1555	}1556 1557	/* validate value type/size match */1558	if (main_def->value_size != extra_def->value_size) {1559		reason = "value_size";1560		goto mismatch;1561	}1562	if (!!main_def->value_type_id != !!extra_def->value_type_id) {1563		reason = "value type";1564		goto mismatch;1565	}1566	if ((main_def->parts & MAP_DEF_VALUE_TYPE)1567	     && !glob_sym_btf_matches(sym_name, true /*exact*/,1568				      main_btf, main_def->value_type_id,1569				      extra_btf, extra_def->value_type_id)) {1570		reason = "key type";1571		goto mismatch;1572	}1573 1574	if (main_def->max_entries != extra_def->max_entries) {1575		reason = "max_entries";1576		goto mismatch;1577	}1578	if (main_def->map_flags != extra_def->map_flags) {1579		reason = "map_flags";1580		goto mismatch;1581	}1582	if (main_def->numa_node != extra_def->numa_node) {1583		reason = "numa_node";1584		goto mismatch;1585	}1586	if (main_def->pinning != extra_def->pinning) {1587		reason = "pinning";1588		goto mismatch;1589	}1590 1591	if ((main_def->parts & MAP_DEF_INNER_MAP) != (extra_def->parts & MAP_DEF_INNER_MAP)) {1592		reason = "inner map";1593		goto mismatch;1594	}1595 1596	if (main_def->parts & MAP_DEF_INNER_MAP) {1597		char inner_map_name[128];1598 1599		snprintf(inner_map_name, sizeof(inner_map_name), "%s.inner", sym_name);1600 1601		return map_defs_match(inner_map_name,1602				      main_btf, main_inner_def, NULL,1603				      extra_btf, extra_inner_def, NULL);1604	}1605 1606	return true;1607 1608mismatch:1609	pr_warn("global '%s': map %s mismatch\n", sym_name, reason);1610	return false;1611}1612 1613static bool glob_map_defs_match(const char *sym_name,1614				struct bpf_linker *linker, struct glob_sym *glob_sym,1615				struct src_obj *obj, Elf64_Sym *sym, int btf_id)1616{1617	struct btf_map_def dst_def = {}, dst_inner_def = {};1618	struct btf_map_def src_def = {}, src_inner_def = {};1619	const struct btf_type *t;1620	int err;1621 1622	t = btf__type_by_id(obj->btf, btf_id);1623	if (!btf_is_var(t)) {1624		pr_warn("global '%s': invalid map definition type [%d]\n", sym_name, btf_id);1625		return false;1626	}1627	t = skip_mods_and_typedefs(obj->btf, t->type, NULL);1628 1629	err = parse_btf_map_def(sym_name, obj->btf, t, true /*strict*/, &src_def, &src_inner_def);1630	if (err) {1631		pr_warn("global '%s': invalid map definition\n", sym_name);1632		return false;1633	}1634 1635	/* re-parse existing map definition */1636	t = btf__type_by_id(linker->btf, glob_sym->btf_id);1637	t = skip_mods_and_typedefs(linker->btf, t->type, NULL);1638	err = parse_btf_map_def(sym_name, linker->btf, t, true /*strict*/, &dst_def, &dst_inner_def);1639	if (err) {1640		/* this should not happen, because we already validated it */1641		pr_warn("global '%s': invalid dst map definition\n", sym_name);1642		return false;1643	}1644 1645	/* Currently extern map definition has to be complete and match1646	 * concrete map definition exactly. This restriction might be lifted1647	 * in the future.1648	 */1649	return map_defs_match(sym_name, linker->btf, &dst_def, &dst_inner_def,1650			      obj->btf, &src_def, &src_inner_def);1651}1652 1653static bool glob_syms_match(const char *sym_name,1654			    struct bpf_linker *linker, struct glob_sym *glob_sym,1655			    struct src_obj *obj, Elf64_Sym *sym, size_t sym_idx, int btf_id)1656{1657	const struct btf_type *src_t;1658 1659	/* if we are dealing with externs, BTF types describing both global1660	 * and extern VARs/FUNCs should be completely present in all files1661	 */1662	if (!glob_sym->btf_id || !btf_id) {1663		pr_warn("BTF info is missing for global symbol '%s'\n", sym_name);1664		return false;1665	}1666 1667	src_t = btf__type_by_id(obj->btf, btf_id);1668	if (!btf_is_var(src_t) && !btf_is_func(src_t)) {1669		pr_warn("only extern variables and functions are supported, but got '%s' for '%s'\n",1670			btf_kind_str(src_t), sym_name);1671		return false;1672	}1673 1674	/* deal with .maps definitions specially */1675	if (glob_sym->sec_id && strcmp(linker->secs[glob_sym->sec_id].sec_name, MAPS_ELF_SEC) == 0)1676		return glob_map_defs_match(sym_name, linker, glob_sym, obj, sym, btf_id);1677 1678	if (!glob_sym_btf_matches(sym_name, true /*exact*/,1679				  linker->btf, glob_sym->btf_id, obj->btf, btf_id))1680		return false;1681 1682	return true;1683}1684 1685static bool btf_is_non_static(const struct btf_type *t)1686{1687	return (btf_is_var(t) && btf_var(t)->linkage != BTF_VAR_STATIC)1688	       || (btf_is_func(t) && btf_func_linkage(t) != BTF_FUNC_STATIC);1689}1690 1691static int find_glob_sym_btf(struct src_obj *obj, Elf64_Sym *sym, const char *sym_name,1692			     int *out_btf_sec_id, int *out_btf_id)1693{1694	int i, j, n, m, btf_id = 0;1695	const struct btf_type *t;1696	const struct btf_var_secinfo *vi;1697	const char *name;1698 1699	if (!obj->btf) {1700		pr_warn("failed to find BTF info for object '%s'\n", obj->filename);1701		return -EINVAL;1702	}1703 1704	n = btf__type_cnt(obj->btf);1705	for (i = 1; i < n; i++) {1706		t = btf__type_by_id(obj->btf, i);1707 1708		/* some global and extern FUNCs and VARs might not be associated with any1709		 * DATASEC, so try to detect them in the same pass1710		 */1711		if (btf_is_non_static(t)) {1712			name = btf__str_by_offset(obj->btf, t->name_off);1713			if (strcmp(name, sym_name) != 0)1714				continue;1715 1716			/* remember and still try to find DATASEC */1717			btf_id = i;1718			continue;1719		}1720 1721		if (!btf_is_datasec(t))1722			continue;1723 1724		vi = btf_var_secinfos(t);1725		for (j = 0, m = btf_vlen(t); j < m; j++, vi++) {1726			t = btf__type_by_id(obj->btf, vi->type);1727			name = btf__str_by_offset(obj->btf, t->name_off);1728 1729			if (strcmp(name, sym_name) != 0)1730				continue;1731			if (btf_is_var(t) && btf_var(t)->linkage == BTF_VAR_STATIC)1732				continue;1733			if (btf_is_func(t) && btf_func_linkage(t) == BTF_FUNC_STATIC)1734				continue;1735 1736			if (btf_id && btf_id != vi->type) {1737				pr_warn("global/extern '%s' BTF is ambiguous: both types #%d and #%u match\n",1738					sym_name, btf_id, vi->type);1739				return -EINVAL;1740			}1741 1742			*out_btf_sec_id = i;1743			*out_btf_id = vi->type;1744 1745			return 0;1746		}1747	}1748 1749	/* free-floating extern or global FUNC */1750	if (btf_id) {1751		*out_btf_sec_id = 0;1752		*out_btf_id = btf_id;1753		return 0;1754	}1755 1756	pr_warn("failed to find BTF info for global/extern symbol '%s'\n", sym_name);1757	return -ENOENT;1758}1759 1760static struct src_sec *find_src_sec_by_name(struct src_obj *obj, const char *sec_name)1761{1762	struct src_sec *sec;1763	int i;1764 1765	for (i = 1; i < obj->sec_cnt; i++) {1766		sec = &obj->secs[i];1767 1768		if (strcmp(sec->sec_name, sec_name) == 0)1769			return sec;1770	}1771 1772	return NULL;1773}1774 1775static int complete_extern_btf_info(struct btf *dst_btf, int dst_id,1776				    struct btf *src_btf, int src_id)1777{1778	struct btf_type *dst_t = btf_type_by_id(dst_btf, dst_id);1779	struct btf_type *src_t = btf_type_by_id(src_btf, src_id);1780	struct btf_param *src_p, *dst_p;1781	const char *s;1782	int i, n, off;1783 1784	/* We already made sure that source and destination types (FUNC or1785	 * VAR) match in terms of types and argument names.1786	 */1787	if (btf_is_var(dst_t)) {1788		btf_var(dst_t)->linkage = BTF_VAR_GLOBAL_ALLOCATED;1789		return 0;1790	}1791 1792	dst_t->info = btf_type_info(BTF_KIND_FUNC, BTF_FUNC_GLOBAL, 0);1793 1794	/* now onto FUNC_PROTO types */1795	src_t = btf_type_by_id(src_btf, src_t->type);1796	dst_t = btf_type_by_id(dst_btf, dst_t->type);1797 1798	/* Fill in all the argument names, which for extern FUNCs are missing.1799	 * We'll end up with two copies of FUNCs/VARs for externs, but that1800	 * will be taken care of by BTF dedup at the very end.1801	 * It might be that BTF types for extern in one file has less/more BTF1802	 * information (e.g., FWD instead of full STRUCT/UNION information),1803	 * but that should be (in most cases, subject to BTF dedup rules)1804	 * handled and resolved by BTF dedup algorithm as well, so we won't1805	 * worry about it. Our only job is to make sure that argument names1806	 * are populated on both sides, otherwise BTF dedup will pedantically1807	 * consider them different.1808	 */1809	src_p = btf_params(src_t);1810	dst_p = btf_params(dst_t);1811	for (i = 0, n = btf_vlen(dst_t); i < n; i++, src_p++, dst_p++) {1812		if (!src_p->name_off)1813			continue;1814 1815		/* src_btf has more complete info, so add name to dst_btf */1816		s = btf__str_by_offset(src_btf, src_p->name_off);1817		off = btf__add_str(dst_btf, s);1818		if (off < 0)1819			return off;1820		dst_p->name_off = off;1821	}1822	return 0;1823}1824 1825static void sym_update_bind(Elf64_Sym *sym, int sym_bind)1826{1827	sym->st_info = ELF64_ST_INFO(sym_bind, ELF64_ST_TYPE(sym->st_info));1828}1829 1830static void sym_update_type(Elf64_Sym *sym, int sym_type)1831{1832	sym->st_info = ELF64_ST_INFO(ELF64_ST_BIND(sym->st_info), sym_type);1833}1834 1835static void sym_update_visibility(Elf64_Sym *sym, int sym_vis)1836{1837	/* libelf doesn't provide setters for ST_VISIBILITY,1838	 * but it is stored in the lower 2 bits of st_other1839	 */1840	sym->st_other &= ~0x03;1841	sym->st_other |= sym_vis;1842}1843 1844static int linker_append_elf_sym(struct bpf_linker *linker, struct src_obj *obj,1845				 Elf64_Sym *sym, const char *sym_name, int src_sym_idx)1846{1847	struct src_sec *src_sec = NULL;1848	struct dst_sec *dst_sec = NULL;1849	struct glob_sym *glob_sym = NULL;1850	int name_off, sym_type, sym_bind, sym_vis, err;1851	int btf_sec_id = 0, btf_id = 0;1852	size_t dst_sym_idx;1853	Elf64_Sym *dst_sym;1854	bool sym_is_extern;1855 1856	sym_type = ELF64_ST_TYPE(sym->st_info);1857	sym_bind = ELF64_ST_BIND(sym->st_info);1858	sym_vis = ELF64_ST_VISIBILITY(sym->st_other);1859	sym_is_extern = sym->st_shndx == SHN_UNDEF;1860 1861	if (sym_is_extern) {1862		if (!obj->btf) {1863			pr_warn("externs without BTF info are not supported\n");1864			return -ENOTSUP;1865		}1866	} else if (sym->st_shndx < SHN_LORESERVE) {1867		src_sec = &obj->secs[sym->st_shndx];1868		if (src_sec->skipped)1869			return 0;1870		dst_sec = &linker->secs[src_sec->dst_id];1871 1872		/* allow only one STT_SECTION symbol per section */1873		if (sym_type == STT_SECTION && dst_sec->sec_sym_idx) {1874			obj->sym_map[src_sym_idx] = dst_sec->sec_sym_idx;1875			return 0;1876		}1877	}1878 1879	if (sym_bind == STB_LOCAL)1880		goto add_sym;1881 1882	/* find matching BTF info */1883	err = find_glob_sym_btf(obj, sym, sym_name, &btf_sec_id, &btf_id);1884	if (err)1885		return err;1886 1887	if (sym_is_extern && btf_sec_id) {1888		const char *sec_name = NULL;1889		const struct btf_type *t;1890 1891		t = btf__type_by_id(obj->btf, btf_sec_id);1892		sec_name = btf__str_by_offset(obj->btf, t->name_off);1893 1894		/* Clang puts unannotated extern vars into1895		 * '.extern' BTF DATASEC. Treat them the same1896		 * as unannotated extern funcs (which are1897		 * currently not put into any DATASECs).1898		 * Those don't have associated src_sec/dst_sec.1899		 */1900		if (strcmp(sec_name, BTF_EXTERN_SEC) != 0) {1901			src_sec = find_src_sec_by_name(obj, sec_name);1902			if (!src_sec) {1903				pr_warn("failed to find matching ELF sec '%s'\n", sec_name);1904				return -ENOENT;1905			}1906			dst_sec = &linker->secs[src_sec->dst_id];1907		}1908	}1909 1910	glob_sym = find_glob_sym(linker, sym_name);1911	if (glob_sym) {1912		/* Preventively resolve to existing symbol. This is1913		 * needed for further relocation symbol remapping in1914		 * the next step of linking.1915		 */1916		obj->sym_map[src_sym_idx] = glob_sym->sym_idx;1917 1918		/* If both symbols are non-externs, at least one of1919		 * them has to be STB_WEAK, otherwise they are in1920		 * a conflict with each other.1921		 */1922		if (!sym_is_extern && !glob_sym->is_extern1923		    && !glob_sym->is_weak && sym_bind != STB_WEAK) {1924			pr_warn("conflicting non-weak symbol #%d (%s) definition in '%s'\n",1925				src_sym_idx, sym_name, obj->filename);1926			return -EINVAL;1927		}1928 1929		if (!glob_syms_match(sym_name, linker, glob_sym, obj, sym, src_sym_idx, btf_id))1930			return -EINVAL;1931 1932		dst_sym = get_sym_by_idx(linker, glob_sym->sym_idx);1933 1934		/* If new symbol is strong, then force dst_sym to be strong as1935		 * well; this way a mix of weak and non-weak extern1936		 * definitions will end up being strong.1937		 */1938		if (sym_bind == STB_GLOBAL) {1939			/* We still need to preserve type (NOTYPE or1940			 * OBJECT/FUNC, depending on whether the symbol is1941			 * extern or not)1942			 */1943			sym_update_bind(dst_sym, STB_GLOBAL);1944			glob_sym->is_weak = false;1945		}1946 1947		/* Non-default visibility is "contaminating", with stricter1948		 * visibility overwriting more permissive ones, even if more1949		 * permissive visibility comes from just an extern definition.1950		 * Currently only STV_DEFAULT and STV_HIDDEN are allowed and1951		 * ensured by ELF symbol sanity checks above.1952		 */1953		if (sym_vis > ELF64_ST_VISIBILITY(dst_sym->st_other))1954			sym_update_visibility(dst_sym, sym_vis);1955 1956		/* If the new symbol is extern, then regardless if1957		 * existing symbol is extern or resolved global, just1958		 * keep the existing one untouched.1959		 */1960		if (sym_is_extern)1961			return 0;1962 1963		/* If existing symbol is a strong resolved symbol, bail out,1964		 * because we lost resolution battle have nothing to1965		 * contribute. We already checked above that there is no1966		 * strong-strong conflict. We also already tightened binding1967		 * and visibility, so nothing else to contribute at that point.1968		 */1969		if (!glob_sym->is_extern && sym_bind == STB_WEAK)1970			return 0;1971 1972		/* At this point, new symbol is strong non-extern,1973		 * so overwrite glob_sym with new symbol information.1974		 * Preserve binding and visibility.1975		 */1976		sym_update_type(dst_sym, sym_type);1977		dst_sym->st_shndx = dst_sec->sec_idx;1978		dst_sym->st_value = src_sec->dst_off + sym->st_value;1979		dst_sym->st_size = sym->st_size;1980 1981		/* see comment below about dst_sec->id vs dst_sec->sec_idx */1982		glob_sym->sec_id = dst_sec->id;1983		glob_sym->is_extern = false;1984 1985		if (complete_extern_btf_info(linker->btf, glob_sym->btf_id,1986					     obj->btf, btf_id))1987			return -EINVAL;1988 1989		/* request updating VAR's/FUNC's underlying BTF type when appending BTF type */1990		glob_sym->underlying_btf_id = 0;1991 1992		obj->sym_map[src_sym_idx] = glob_sym->sym_idx;1993		return 0;1994	}1995 1996add_sym:1997	name_off = strset__add_str(linker->strtab_strs, sym_name);1998	if (name_off < 0)1999		return name_off;2000 2001	dst_sym = add_new_sym(linker, &dst_sym_idx);2002	if (!dst_sym)2003		return -ENOMEM;2004 2005	dst_sym->st_name = name_off;2006	dst_sym->st_info = sym->st_info;2007	dst_sym->st_other = sym->st_other;2008	dst_sym->st_shndx = dst_sec ? dst_sec->sec_idx : sym->st_shndx;2009	dst_sym->st_value = (src_sec ? src_sec->dst_off : 0) + sym->st_value;2010	dst_sym->st_size = sym->st_size;2011 2012	obj->sym_map[src_sym_idx] = dst_sym_idx;2013 2014	if (sym_type == STT_SECTION && dst_sym) {2015		dst_sec->sec_sym_idx = dst_sym_idx;2016		dst_sym->st_value = 0;2017	}2018 2019	if (sym_bind != STB_LOCAL) {2020		glob_sym = add_glob_sym(linker);2021		if (!glob_sym)2022			return -ENOMEM;2023 2024		glob_sym->sym_idx = dst_sym_idx;2025		/* we use dst_sec->id (and not dst_sec->sec_idx), because2026		 * ephemeral sections (.kconfig, .ksyms, etc) don't have2027		 * sec_idx (as they don't have corresponding ELF section), but2028		 * still have id. .extern doesn't have even ephemeral section2029		 * associated with it, so dst_sec->id == dst_sec->sec_idx == 0.2030		 */2031		glob_sym->sec_id = dst_sec ? dst_sec->id : 0;2032		glob_sym->name_off = name_off;2033		/* we will fill btf_id in during BTF merging step */2034		glob_sym->btf_id = 0;2035		glob_sym->is_extern = sym_is_extern;2036		glob_sym->is_weak = sym_bind == STB_WEAK;2037	}2038 2039	return 0;2040}2041 2042static int linker_append_elf_relos(struct bpf_linker *linker, struct src_obj *obj)2043{2044	struct src_sec *src_symtab = &obj->secs[obj->symtab_sec_idx];2045	int i, err;2046 2047	for (i = 1; i < obj->sec_cnt; i++) {2048		struct src_sec *src_sec, *src_linked_sec;2049		struct dst_sec *dst_sec, *dst_linked_sec;2050		Elf64_Rel *src_rel, *dst_rel;2051		int j, n;2052 2053		src_sec = &obj->secs[i];2054		if (!is_relo_sec(src_sec))2055			continue;2056 2057		/* shdr->sh_info points to relocatable section */2058		src_linked_sec = &obj->secs[src_sec->shdr->sh_info];2059		if (src_linked_sec->skipped)2060			continue;2061 2062		dst_sec = find_dst_sec_by_name(linker, src_sec->sec_name);2063		if (!dst_sec) {2064			dst_sec = add_dst_sec(linker, src_sec->sec_name);2065			if (!dst_sec)2066				return -ENOMEM;2067			err = init_sec(linker, dst_sec, src_sec);2068			if (err) {2069				pr_warn("failed to init section '%s'\n", src_sec->sec_name);2070				return err;2071			}2072		} else if (!secs_match(dst_sec, src_sec)) {2073			pr_warn("sections %s are not compatible\n", src_sec->sec_name);2074			return -1;2075		}2076 2077		/* shdr->sh_link points to SYMTAB */2078		dst_sec->shdr->sh_link = linker->symtab_sec_idx;2079 2080		/* shdr->sh_info points to relocated section */2081		dst_linked_sec = &linker->secs[src_linked_sec->dst_id];2082		dst_sec->shdr->sh_info = dst_linked_sec->sec_idx;2083 2084		src_sec->dst_id = dst_sec->id;2085		err = extend_sec(linker, dst_sec, src_sec);2086		if (err)2087			return err;2088 2089		src_rel = src_sec->data->d_buf;2090		dst_rel = dst_sec->raw_data + src_sec->dst_off;2091		n = src_sec->shdr->sh_size / src_sec->shdr->sh_entsize;2092		for (j = 0; j < n; j++, src_rel++, dst_rel++) {2093			size_t src_sym_idx, dst_sym_idx, sym_type;2094			Elf64_Sym *src_sym;2095 2096			src_sym_idx = ELF64_R_SYM(src_rel->r_info);2097			src_sym = src_symtab->data->d_buf + sizeof(*src_sym) * src_sym_idx;2098 2099			dst_sym_idx = obj->sym_map[src_sym_idx];2100			dst_rel->r_offset += src_linked_sec->dst_off;2101			sym_type = ELF64_R_TYPE(src_rel->r_info);2102			dst_rel->r_info = ELF64_R_INFO(dst_sym_idx, sym_type);2103 2104			if (ELF64_ST_TYPE(src_sym->st_info) == STT_SECTION) {2105				struct src_sec *sec = &obj->secs[src_sym->st_shndx];2106				struct bpf_insn *insn;2107 2108				if (src_linked_sec->shdr->sh_flags & SHF_EXECINSTR) {2109					/* calls to the very first static function inside2110					 * .text section at offset 0 will2111					 * reference section symbol, not the2112					 * function symbol. Fix that up,2113					 * otherwise it won't be possible to2114					 * relocate calls to two different2115					 * static functions with the same name2116					 * (rom two different object files)2117					 */2118					insn = dst_linked_sec->raw_data + dst_rel->r_offset;2119					if (insn->code == (BPF_JMP | BPF_CALL))2120						insn->imm += sec->dst_off / sizeof(struct bpf_insn);2121					else2122						insn->imm += sec->dst_off;2123				} else {2124					pr_warn("relocation against STT_SECTION in non-exec section is not supported!\n");2125					return -EINVAL;2126				}2127			}2128 2129		}2130	}2131 2132	return 0;2133}2134 2135static Elf64_Sym *find_sym_by_name(struct src_obj *obj, size_t sec_idx,2136				   int sym_type, const char *sym_name)2137{2138	struct src_sec *symtab = &obj->secs[obj->symtab_sec_idx];2139	Elf64_Sym *sym = symtab->data->d_buf;2140	int i, n = symtab->shdr->sh_size / symtab->shdr->sh_entsize;2141	int str_sec_idx = symtab->shdr->sh_link;2142	const char *name;2143 2144	for (i = 0; i < n; i++, sym++) {2145		if (sym->st_shndx != sec_idx)2146			continue;2147		if (ELF64_ST_TYPE(sym->st_info) != sym_type)2148			continue;2149 2150		name = elf_strptr(obj->elf, str_sec_idx, sym->st_name);2151		if (!name)2152			return NULL;2153 2154		if (strcmp(sym_name, name) != 0)2155			continue;2156 2157		return sym;2158	}2159 2160	return NULL;2161}2162 2163static int linker_fixup_btf(struct src_obj *obj)2164{2165	const char *sec_name;2166	struct src_sec *sec;2167	int i, j, n, m;2168 2169	if (!obj->btf)2170		return 0;2171 2172	n = btf__type_cnt(obj->btf);2173	for (i = 1; i < n; i++) {2174		struct btf_var_secinfo *vi;2175		struct btf_type *t;2176 2177		t = btf_type_by_id(obj->btf, i);2178		if (btf_kind(t) != BTF_KIND_DATASEC)2179			continue;2180 2181		sec_name = btf__str_by_offset(obj->btf, t->name_off);2182		sec = find_src_sec_by_name(obj, sec_name);2183		if (sec) {2184			/* record actual section size, unless ephemeral */2185			if (sec->shdr)2186				t->size = sec->shdr->sh_size;2187		} else {2188			/* BTF can have some sections that are not represented2189			 * in ELF, e.g., .kconfig, .ksyms, .extern, which are used2190			 * for special extern variables.2191			 *2192			 * For all but one such special (ephemeral)2193			 * sections, we pre-create "section shells" to be able2194			 * to keep track of extra per-section metadata later2195			 * (e.g., those BTF extern variables).2196			 *2197			 * .extern is even more special, though, because it2198			 * contains extern variables that need to be resolved2199			 * by static linker, not libbpf and kernel. When such2200			 * externs are resolved, we are going to remove them2201			 * from .extern BTF section and might end up not2202			 * needing it at all. Each resolved extern should have2203			 * matching non-extern VAR/FUNC in other sections.2204			 *2205			 * We do support leaving some of the externs2206			 * unresolved, though, to support cases of building2207			 * libraries, which will later be linked against final2208			 * BPF applications. So if at finalization we still2209			 * see unresolved externs, we'll create .extern2210			 * section on our own.2211			 */2212			if (strcmp(sec_name, BTF_EXTERN_SEC) == 0)2213				continue;2214 2215			sec = add_src_sec(obj, sec_name);2216			if (!sec)2217				return -ENOMEM;2218 2219			sec->ephemeral = true;2220			sec->sec_idx = 0; /* will match UNDEF shndx in ELF */2221		}2222 2223		/* remember ELF section and its BTF type ID match */2224		sec->sec_type_id = i;2225 2226		/* fix up variable offsets */2227		vi = btf_var_secinfos(t);2228		for (j = 0, m = btf_vlen(t); j < m; j++, vi++) {2229			const struct btf_type *vt = btf__type_by_id(obj->btf, vi->type);2230			const char *var_name;2231			int var_linkage;2232			Elf64_Sym *sym;2233 2234			/* could be a variable or function */2235			if (!btf_is_var(vt))2236				continue;2237 2238			var_name = btf__str_by_offset(obj->btf, vt->name_off);2239			var_linkage = btf_var(vt)->linkage;2240 2241			/* no need to patch up static or extern vars */2242			if (var_linkage != BTF_VAR_GLOBAL_ALLOCATED)2243				continue;2244 2245			sym = find_sym_by_name(obj, sec->sec_idx, STT_OBJECT, var_name);2246			if (!sym) {2247				pr_warn("failed to find symbol for variable '%s' in section '%s'\n", var_name, sec_name);2248				return -ENOENT;2249			}2250 2251			vi->offset = sym->st_value;2252		}2253	}2254 2255	return 0;2256}2257 2258static int linker_append_btf(struct bpf_linker *linker, struct src_obj *obj)2259{2260	const struct btf_type *t;2261	int i, j, n, start_id, id, err;2262	const char *name;2263 2264	if (!obj->btf)2265		return 0;2266 2267	start_id = btf__type_cnt(linker->btf);2268	n = btf__type_cnt(obj->btf);2269 2270	obj->btf_type_map = calloc(n + 1, sizeof(int));2271	if (!obj->btf_type_map)2272		return -ENOMEM;2273 2274	for (i = 1; i < n; i++) {2275		struct glob_sym *glob_sym = NULL;2276 2277		t = btf__type_by_id(obj->btf, i);2278 2279		/* DATASECs are handled specially below */2280		if (btf_kind(t) == BTF_KIND_DATASEC)2281			continue;2282 2283		if (btf_is_non_static(t)) {2284			/* there should be glob_sym already */2285			name = btf__str_by_offset(obj->btf, t->name_off);2286			glob_sym = find_glob_sym(linker, name);2287 2288			/* VARs without corresponding glob_sym are those that2289			 * belong to skipped/deduplicated sections (i.e.,2290			 * license and version), so just skip them2291			 */2292			if (!glob_sym)2293				continue;2294 2295			/* linker_append_elf_sym() might have requested2296			 * updating underlying type ID, if extern was resolved2297			 * to strong symbol or weak got upgraded to non-weak2298			 */2299			if (glob_sym->underlying_btf_id == 0)2300				glob_sym->underlying_btf_id = -t->type;2301 2302			/* globals from previous object files that match our2303			 * VAR/FUNC already have a corresponding associated2304			 * BTF type, so just make sure to use it2305			 */2306			if (glob_sym->btf_id) {2307				/* reuse existing BTF type for global var/func */2308				obj->btf_type_map[i] = glob_sym->btf_id;2309				continue;2310			}2311		}2312 2313		id = btf__add_type(linker->btf, obj->btf, t);2314		if (id < 0) {2315			pr_warn("failed to append BTF type #%d from file '%s'\n", i, obj->filename);2316			return id;2317		}2318 2319		obj->btf_type_map[i] = id;2320 2321		/* record just appended BTF type for var/func */2322		if (glob_sym) {2323			glob_sym->btf_id = id;2324			glob_sym->underlying_btf_id = -t->type;2325		}2326	}2327 2328	/* remap all the types except DATASECs */2329	n = btf__type_cnt(linker->btf);2330	for (i = start_id; i < n; i++) {2331		struct btf_type *dst_t = btf_type_by_id(linker->btf, i);2332		struct btf_field_iter it;2333		__u32 *type_id;2334 2335		err = btf_field_iter_init(&it, dst_t, BTF_FIELD_ITER_IDS);2336		if (err)2337			return err;2338 2339		while ((type_id = btf_field_iter_next(&it))) {2340			int new_id = obj->btf_type_map[*type_id];2341 2342			/* Error out if the type wasn't remapped. Ignore VOID which stays VOID. */2343			if (new_id == 0 && *type_id != 0) {2344				pr_warn("failed to find new ID mapping for original BTF type ID %u\n",2345					*type_id);2346				return -EINVAL;2347			}2348 2349			*type_id = obj->btf_type_map[*type_id];2350		}2351	}2352 2353	/* Rewrite VAR/FUNC underlying types (i.e., FUNC's FUNC_PROTO and VAR's2354	 * actual type), if necessary2355	 */2356	for (i = 0; i < linker->glob_sym_cnt; i++) {2357		struct glob_sym *glob_sym = &linker->glob_syms[i];2358		struct btf_type *glob_t;2359 2360		if (glob_sym->underlying_btf_id >= 0)2361			continue;2362 2363		glob_sym->underlying_btf_id = obj->btf_type_map[-glob_sym->underlying_btf_id];2364 2365		glob_t = btf_type_by_id(linker->btf, glob_sym->btf_id);2366		glob_t->type = glob_sym->underlying_btf_id;2367	}2368 2369	/* append DATASEC info */2370	for (i = 1; i < obj->sec_cnt; i++) {2371		struct src_sec *src_sec;2372		struct dst_sec *dst_sec;2373		const struct btf_var_secinfo *src_var;2374		struct btf_var_secinfo *dst_var;2375 2376		src_sec = &obj->secs[i];2377		if (!src_sec->sec_type_id || src_sec->skipped)2378			continue;2379		dst_sec = &linker->secs[src_sec->dst_id];2380 2381		/* Mark section as having BTF regardless of the presence of2382		 * variables. In some cases compiler might generate empty BTF2383		 * with no variables information. E.g., when promoting local2384		 * array/structure variable initial values and BPF object2385		 * file otherwise has no read-only static variables in2386		 * .rodata. We need to preserve such empty BTF and just set2387		 * correct section size.2388		 */2389		dst_sec->has_btf = true;2390 2391		t = btf__type_by_id(obj->btf, src_sec->sec_type_id);2392		src_var = btf_var_secinfos(t);2393		n = btf_vlen(t);2394		for (j = 0; j < n; j++, src_var++) {2395			void *sec_vars = dst_sec->sec_vars;2396			int new_id = obj->btf_type_map[src_var->type];2397			struct glob_sym *glob_sym = NULL;2398 2399			t = btf_type_by_id(linker->btf, new_id);2400			if (btf_is_non_static(t)) {2401				name = btf__str_by_offset(linker->btf, t->name_off);2402				glob_sym = find_glob_sym(linker, name);2403				if (glob_sym->sec_id != dst_sec->id) {2404					pr_warn("global '%s': section mismatch %d vs %d\n",2405						name, glob_sym->sec_id, dst_sec->id);2406					return -EINVAL;2407				}2408			}2409 2410			/* If there is already a member (VAR or FUNC) mapped2411			 * to the same type, don't add a duplicate entry.2412			 * This will happen when multiple object files define2413			 * the same extern VARs/FUNCs.2414			 */2415			if (glob_sym && glob_sym->var_idx >= 0) {2416				__s64 sz;2417 2418				dst_var = &dst_sec->sec_vars[glob_sym->var_idx];2419				/* Because underlying BTF type might have2420				 * changed, so might its size have changed, so2421				 * re-calculate and update it in sec_var.2422				 */2423				sz = btf__resolve_size(linker->btf, glob_sym->underlying_btf_id);2424				if (sz < 0) {2425					pr_warn("global '%s': failed to resolve size of underlying type: %d\n",2426						name, (int)sz);2427					return -EINVAL;2428				}2429				dst_var->size = sz;2430				continue;2431			}2432 2433			sec_vars = libbpf_reallocarray(sec_vars,2434						       dst_sec->sec_var_cnt + 1,2435						       sizeof(*dst_sec->sec_vars));2436			if (!sec_vars)2437				return -ENOMEM;2438 2439			dst_sec->sec_vars = sec_vars;2440			dst_sec->sec_var_cnt++;2441 2442			dst_var = &dst_sec->sec_vars[dst_sec->sec_var_cnt - 1];2443			dst_var->type = obj->btf_type_map[src_var->type];2444			dst_var->size = src_var->size;2445			dst_var->offset = src_sec->dst_off + src_var->offset;2446 2447			if (glob_sym)2448				glob_sym->var_idx = dst_sec->sec_var_cnt - 1;2449		}2450	}2451 2452	return 0;2453}2454 2455static void *add_btf_ext_rec(struct btf_ext_sec_data *ext_data, const void *src_rec)2456{2457	void *tmp;2458 2459	tmp = libbpf_reallocarray(ext_data->recs, ext_data->rec_cnt + 1, ext_data->rec_sz);2460	if (!tmp)2461		return NULL;2462	ext_data->recs = tmp;2463 2464	tmp += ext_data->rec_cnt * ext_data->rec_sz;2465	memcpy(tmp, src_rec, ext_data->rec_sz);2466 2467	ext_data->rec_cnt++;2468 2469	return tmp;2470}2471 2472static int linker_append_btf_ext(struct bpf_linker *linker, struct src_obj *obj)2473{2474	const struct btf_ext_info_sec *ext_sec;2475	const char *sec_name, *s;2476	struct src_sec *src_sec;2477	struct dst_sec *dst_sec;2478	int rec_sz, str_off, i;2479 2480	if (!obj->btf_ext)2481		return 0;2482 2483	rec_sz = obj->btf_ext->func_info.rec_size;2484	for_each_btf_ext_sec(&obj->btf_ext->func_info, ext_sec) {2485		struct bpf_func_info_min *src_rec, *dst_rec;2486 2487		sec_name = btf__name_by_offset(obj->btf, ext_sec->sec_name_off);2488		src_sec = find_src_sec_by_name(obj, sec_name);2489		if (!src_sec) {2490			pr_warn("can't find section '%s' referenced from .BTF.ext\n", sec_name);2491			return -EINVAL;2492		}2493		dst_sec = &linker->secs[src_sec->dst_id];2494 2495		if (dst_sec->func_info.rec_sz == 0)2496			dst_sec->func_info.rec_sz = rec_sz;2497		if (dst_sec->func_info.rec_sz != rec_sz) {2498			pr_warn("incompatible .BTF.ext record sizes for section '%s'\n", sec_name);2499			return -EINVAL;2500		}2501 2502		for_each_btf_ext_rec(&obj->btf_ext->func_info, ext_sec, i, src_rec) {2503			dst_rec = add_btf_ext_rec(&dst_sec->func_info, src_rec);2504			if (!dst_rec)2505				return -ENOMEM;2506 2507			dst_rec->insn_off += src_sec->dst_off;2508			dst_rec->type_id = obj->btf_type_map[dst_rec->type_id];2509		}2510	}2511 2512	rec_sz = obj->btf_ext->line_info.rec_size;2513	for_each_btf_ext_sec(&obj->btf_ext->line_info, ext_sec) {2514		struct bpf_line_info_min *src_rec, *dst_rec;2515 2516		sec_name = btf__name_by_offset(obj->btf, ext_sec->sec_name_off);2517		src_sec = find_src_sec_by_name(obj, sec_name);2518		if (!src_sec) {2519			pr_warn("can't find section '%s' referenced from .BTF.ext\n", sec_name);2520			return -EINVAL;2521		}2522		dst_sec = &linker->secs[src_sec->dst_id];2523 2524		if (dst_sec->line_info.rec_sz == 0)2525			dst_sec->line_info.rec_sz = rec_sz;2526		if (dst_sec->line_info.rec_sz != rec_sz) {2527			pr_warn("incompatible .BTF.ext record sizes for section '%s'\n", sec_name);2528			return -EINVAL;2529		}2530 2531		for_each_btf_ext_rec(&obj->btf_ext->line_info, ext_sec, i, src_rec) {2532			dst_rec = add_btf_ext_rec(&dst_sec->line_info, src_rec);2533			if (!dst_rec)2534				return -ENOMEM;2535 2536			dst_rec->insn_off += src_sec->dst_off;2537 2538			s = btf__str_by_offset(obj->btf, src_rec->file_name_off);2539			str_off = btf__add_str(linker->btf, s);2540			if (str_off < 0)2541				return -ENOMEM;2542			dst_rec->file_name_off = str_off;2543 2544			s = btf__str_by_offset(obj->btf, src_rec->line_off);2545			str_off = btf__add_str(linker->btf, s);2546			if (str_off < 0)2547				return -ENOMEM;2548			dst_rec->line_off = str_off;2549 2550			/* dst_rec->line_col is fine */2551		}2552	}2553 2554	rec_sz = obj->btf_ext->core_relo_info.rec_size;2555	for_each_btf_ext_sec(&obj->btf_ext->core_relo_info, ext_sec) {2556		struct bpf_core_relo *src_rec, *dst_rec;2557 2558		sec_name = btf__name_by_offset(obj->btf, ext_sec->sec_name_off);2559		src_sec = find_src_sec_by_name(obj, sec_name);2560		if (!src_sec) {2561			pr_warn("can't find section '%s' referenced from .BTF.ext\n", sec_name);2562			return -EINVAL;2563		}2564		dst_sec = &linker->secs[src_sec->dst_id];2565 2566		if (dst_sec->core_relo_info.rec_sz == 0)2567			dst_sec->core_relo_info.rec_sz = rec_sz;2568		if (dst_sec->core_relo_info.rec_sz != rec_sz) {2569			pr_warn("incompatible .BTF.ext record sizes for section '%s'\n", sec_name);2570			return -EINVAL;2571		}2572 2573		for_each_btf_ext_rec(&obj->btf_ext->core_relo_info, ext_sec, i, src_rec) {2574			dst_rec = add_btf_ext_rec(&dst_sec->core_relo_info, src_rec);2575			if (!dst_rec)2576				return -ENOMEM;2577 2578			dst_rec->insn_off += src_sec->dst_off;2579			dst_rec->type_id = obj->btf_type_map[dst_rec->type_id];2580 2581			s = btf__str_by_offset(obj->btf, src_rec->access_str_off);2582			str_off = btf__add_str(linker->btf, s);2583			if (str_off < 0)2584				return -ENOMEM;2585			dst_rec->access_str_off = str_off;2586 2587			/* dst_rec->kind is fine */2588		}2589	}2590 2591	return 0;2592}2593 2594int bpf_linker__finalize(struct bpf_linker *linker)2595{2596	struct dst_sec *sec;2597	size_t strs_sz;2598	const void *strs;2599	int err, i;2600 2601	if (!linker->elf)2602		return libbpf_err(-EINVAL);2603 2604	err = finalize_btf(linker);2605	if (err)2606		return libbpf_err(err);2607 2608	/* Finalize strings */2609	strs_sz = strset__data_size(linker->strtab_strs);2610	strs = strset__data(linker->strtab_strs);2611 2612	sec = &linker->secs[linker->strtab_sec_idx];2613	sec->data->d_align = 1;2614	sec->data->d_off = 0LL;2615	sec->data->d_buf = (void *)strs;2616	sec->data->d_type = ELF_T_BYTE;2617	sec->data->d_size = strs_sz;2618	sec->shdr->sh_size = strs_sz;2619 2620	for (i = 1; i < linker->sec_cnt; i++) {2621		sec = &linker->secs[i];2622 2623		/* STRTAB is handled specially above */2624		if (sec->sec_idx == linker->strtab_sec_idx)2625			continue;2626 2627		/* special ephemeral sections (.ksyms, .kconfig, etc) */2628		if (!sec->scn)2629			continue;2630 2631		sec->data->d_buf = sec->raw_data;2632	}2633 2634	/* Finalize ELF layout */2635	if (elf_update(linker->elf, ELF_C_NULL) < 0) {2636		err = -errno;2637		pr_warn_elf("failed to finalize ELF layout");2638		return libbpf_err(err);2639	}2640 2641	/* Write out final ELF contents */2642	if (elf_update(linker->elf, ELF_C_WRITE) < 0) {2643		err = -errno;2644		pr_warn_elf("failed to write ELF contents");2645		return libbpf_err(err);2646	}2647 2648	elf_end(linker->elf);2649	close(linker->fd);2650 2651	linker->elf = NULL;2652	linker->fd = -1;2653 2654	return 0;2655}2656 2657static int emit_elf_data_sec(struct bpf_linker *linker, const char *sec_name,2658			     size_t align, const void *raw_data, size_t raw_sz)2659{2660	Elf_Scn *scn;2661	Elf_Data *data;2662	Elf64_Shdr *shdr;2663	int name_off;2664 2665	name_off = strset__add_str(linker->strtab_strs, sec_name);2666	if (name_off < 0)2667		return name_off;2668 2669	scn = elf_newscn(linker->elf);2670	if (!scn)2671		return -ENOMEM;2672	data = elf_newdata(scn);2673	if (!data)2674		return -ENOMEM;2675	shdr = elf64_getshdr(scn);2676	if (!shdr)2677		return -EINVAL;2678 2679	shdr->sh_name = name_off;2680	shdr->sh_type = SHT_PROGBITS;2681	shdr->sh_flags = 0;2682	shdr->sh_size = raw_sz;2683	shdr->sh_link = 0;2684	shdr->sh_info = 0;2685	shdr->sh_addralign = align;2686	shdr->sh_entsize = 0;2687 2688	data->d_type = ELF_T_BYTE;2689	data->d_size = raw_sz;2690	data->d_buf = (void *)raw_data;2691	data->d_align = align;2692	data->d_off = 0;2693 2694	return 0;2695}2696 2697static int finalize_btf(struct bpf_linker *linker)2698{2699	LIBBPF_OPTS(btf_dedup_opts, opts);2700	struct btf *btf = linker->btf;2701	const void *raw_data;2702	int i, j, id, err;2703	__u32 raw_sz;2704 2705	/* bail out if no BTF data was produced */2706	if (btf__type_cnt(linker->btf) == 1)2707		return 0;2708 2709	for (i = 1; i < linker->sec_cnt; i++) {2710		struct dst_sec *sec = &linker->secs[i];2711 2712		if (!sec->has_btf)2713			continue;2714 2715		id = btf__add_datasec(btf, sec->sec_name, sec->sec_sz);2716		if (id < 0) {2717			pr_warn("failed to add consolidated BTF type for datasec '%s': %d\n",2718				sec->sec_name, id);2719			return id;2720		}2721 2722		for (j = 0; j < sec->sec_var_cnt; j++) {2723			struct btf_var_secinfo *vi = &sec->sec_vars[j];2724 2725			if (btf__add_datasec_var_info(btf, vi->type, vi->offset, vi->size))2726				return -EINVAL;2727		}2728	}2729 2730	err = finalize_btf_ext(linker);2731	if (err) {2732		pr_warn(".BTF.ext generation failed: %d\n", err);2733		return err;2734	}2735 2736	opts.btf_ext = linker->btf_ext;2737	err = btf__dedup(linker->btf, &opts);2738	if (err) {2739		pr_warn("BTF dedup failed: %d\n", err);2740		return err;2741	}2742 2743	/* Emit .BTF section */2744	raw_data = btf__raw_data(linker->btf, &raw_sz);2745	if (!raw_data)2746		return -ENOMEM;2747 2748	err = emit_elf_data_sec(linker, BTF_ELF_SEC, 8, raw_data, raw_sz);2749	if (err) {2750		pr_warn("failed to write out .BTF ELF section: %d\n", err);2751		return err;2752	}2753 2754	/* Emit .BTF.ext section */2755	if (linker->btf_ext) {2756		raw_data = btf_ext__raw_data(linker->btf_ext, &raw_sz);2757		if (!raw_data)2758			return -ENOMEM;2759 2760		err = emit_elf_data_sec(linker, BTF_EXT_ELF_SEC, 8, raw_data, raw_sz);2761		if (err) {2762			pr_warn("failed to write out .BTF.ext ELF section: %d\n", err);2763			return err;2764		}2765	}2766 2767	return 0;2768}2769 2770static int emit_btf_ext_data(struct bpf_linker *linker, void *output,2771			     const char *sec_name, struct btf_ext_sec_data *sec_data)2772{2773	struct btf_ext_info_sec *sec_info;2774	void *cur = output;2775	int str_off;2776	size_t sz;2777 2778	if (!sec_data->rec_cnt)2779		return 0;2780 2781	str_off = btf__add_str(linker->btf, sec_name);2782	if (str_off < 0)2783		return -ENOMEM;2784 2785	sec_info = cur;2786	sec_info->sec_name_off = str_off;2787	sec_info->num_info = sec_data->rec_cnt;2788	cur += sizeof(struct btf_ext_info_sec);2789 2790	sz = sec_data->rec_cnt * sec_data->rec_sz;2791	memcpy(cur, sec_data->recs, sz);2792	cur += sz;2793 2794	return cur - output;2795}2796 2797static int finalize_btf_ext(struct bpf_linker *linker)2798{2799	size_t funcs_sz = 0, lines_sz = 0, core_relos_sz = 0, total_sz = 0;2800	size_t func_rec_sz = 0, line_rec_sz = 0, core_relo_rec_sz = 0;2801	struct btf_ext_header *hdr;2802	void *data, *cur;2803	int i, err, sz;2804 2805	/* validate that all sections have the same .BTF.ext record sizes2806	 * and calculate total data size for each type of data (func info,2807	 * line info, core relos)2808	 */2809	for (i = 1; i < linker->sec_cnt; i++) {2810		struct dst_sec *sec = &linker->secs[i];2811 2812		if (sec->func_info.rec_cnt) {2813			if (func_rec_sz == 0)2814				func_rec_sz = sec->func_info.rec_sz;2815			if (func_rec_sz != sec->func_info.rec_sz) {2816				pr_warn("mismatch in func_info record size %zu != %u\n",2817					func_rec_sz, sec->func_info.rec_sz);2818				return -EINVAL;2819			}2820 2821			funcs_sz += sizeof(struct btf_ext_info_sec) + func_rec_sz * sec->func_info.rec_cnt;2822		}2823		if (sec->line_info.rec_cnt) {2824			if (line_rec_sz == 0)2825				line_rec_sz = sec->line_info.rec_sz;2826			if (line_rec_sz != sec->line_info.rec_sz) {2827				pr_warn("mismatch in line_info record size %zu != %u\n",2828					line_rec_sz, sec->line_info.rec_sz);2829				return -EINVAL;2830			}2831 2832			lines_sz += sizeof(struct btf_ext_info_sec) + line_rec_sz * sec->line_info.rec_cnt;2833		}2834		if (sec->core_relo_info.rec_cnt) {2835			if (core_relo_rec_sz == 0)2836				core_relo_rec_sz = sec->core_relo_info.rec_sz;2837			if (core_relo_rec_sz != sec->core_relo_info.rec_sz) {2838				pr_warn("mismatch in core_relo_info record size %zu != %u\n",2839					core_relo_rec_sz, sec->core_relo_info.rec_sz);2840				return -EINVAL;2841			}2842 2843			core_relos_sz += sizeof(struct btf_ext_info_sec) + core_relo_rec_sz * sec->core_relo_info.rec_cnt;2844		}2845	}2846 2847	if (!funcs_sz && !lines_sz && !core_relos_sz)2848		return 0;2849 2850	total_sz += sizeof(struct btf_ext_header);2851	if (funcs_sz) {2852		funcs_sz += sizeof(__u32); /* record size prefix */2853		total_sz += funcs_sz;2854	}2855	if (lines_sz) {2856		lines_sz += sizeof(__u32); /* record size prefix */2857		total_sz += lines_sz;2858	}2859	if (core_relos_sz) {2860		core_relos_sz += sizeof(__u32); /* record size prefix */2861		total_sz += core_relos_sz;2862	}2863 2864	cur = data = calloc(1, total_sz);2865	if (!data)2866		return -ENOMEM;2867 2868	hdr = cur;2869	hdr->magic = BTF_MAGIC;2870	hdr->version = BTF_VERSION;2871	hdr->flags = 0;2872	hdr->hdr_len = sizeof(struct btf_ext_header);2873	cur += sizeof(struct btf_ext_header);2874 2875	/* All offsets are in bytes relative to the end of this header */2876	hdr->func_info_off = 0;2877	hdr->func_info_len = funcs_sz;2878	hdr->line_info_off = funcs_sz;2879	hdr->line_info_len = lines_sz;2880	hdr->core_relo_off = funcs_sz + lines_sz;2881	hdr->core_relo_len = core_relos_sz;2882 2883	if (funcs_sz) {2884		*(__u32 *)cur = func_rec_sz;2885		cur += sizeof(__u32);2886 2887		for (i = 1; i < linker->sec_cnt; i++) {2888			struct dst_sec *sec = &linker->secs[i];2889 2890			sz = emit_btf_ext_data(linker, cur, sec->sec_name, &sec->func_info);2891			if (sz < 0) {2892				err = sz;2893				goto out;2894			}2895 2896			cur += sz;2897		}2898	}2899 2900	if (lines_sz) {2901		*(__u32 *)cur = line_rec_sz;2902		cur += sizeof(__u32);2903 2904		for (i = 1; i < linker->sec_cnt; i++) {2905			struct dst_sec *sec = &linker->secs[i];2906 2907			sz = emit_btf_ext_data(linker, cur, sec->sec_name, &sec->line_info);2908			if (sz < 0) {2909				err = sz;2910				goto out;2911			}2912 2913			cur += sz;2914		}2915	}2916 2917	if (core_relos_sz) {2918		*(__u32 *)cur = core_relo_rec_sz;2919		cur += sizeof(__u32);2920 2921		for (i = 1; i < linker->sec_cnt; i++) {2922			struct dst_sec *sec = &linker->secs[i];2923 2924			sz = emit_btf_ext_data(linker, cur, sec->sec_name, &sec->core_relo_info);2925			if (sz < 0) {2926				err = sz;2927				goto out;2928			}2929 2930			cur += sz;2931		}2932	}2933 2934	linker->btf_ext = btf_ext__new(data, total_sz);2935	err = libbpf_get_error(linker->btf_ext);2936	if (err) {2937		linker->btf_ext = NULL;2938		pr_warn("failed to parse final .BTF.ext data: %d\n", err);2939		goto out;2940	}2941 2942out:2943	free(data);2944	return err;2945}2946