# shellcheck shell=bash # Answers about the toolchain's own files that do not depend on the program. # # A link asks LLVM the same questions about the same files every time: which # names a start file leaves undefined, which of its functions hold a loop, # whether an archive of the toolchain's is all native, what libc.so imports # through GOT.mem and exports, the digest its manifest publishes. Each answer # is a property of the file's bytes. The toolchain asks them once, when it is # built (sysroot/write-link-facts.sh), and writes them to .facts beside # the file. A link that finds such a record and the file's bytes and the # questions it was asked by are the ones the record names, reads the answer; # anything else asks LLVM as it always did, and says why. # # What is held, and why each: # - the file's sha256: the answer is about these bytes and no others; # - the sha256 of this file: the questions are the functions below, and a # record written by other questions answers other questions. # Both are taken in one sha256sum per link, with everything else the link # hashes up front (link_facts_prime). LLVM's version is not held: an nm # listing or a disassembly of the same bytes is the same names, and the # tools are the ones the record's own questions name. # # Sourced by brintld and by sysroot/write-link-facts.sh, so that the answer a # link computes and the answer the record holds come from the same lines. # # THE RECORD, .facts, one line each: # brintium-link-facts 1 # questions sha256=<64 hex> this file, when the record was written # input sha256=<64 hex> the file the record is about # answered ... the questions it answers, even with nothing # one per line of the answer, in its order # # THE QUESTIONS. undefined: the names llvm-nm lists undefined, one a line, in # its order. loops: the functions llvm-objdump -d shows a wasm `loop` in, # sorted bytewise. archive: `native` for an archive with no bitcode member, # which a link then need not unpack. got_mem, exports, abi_digest: libc.so's # GOT.mem imports, its exports as ` `, and field 9 of its hwjs.abi # manifest. LINK_FACTS_LIB="${BASH_SOURCE[0]}" LINK_FACTS_QUESTIONS="undefined loops archive got_mem exports abi_digest" # ── the questions, asked of LLVM, node or brint-tool ──────────────────────── # Each prints its answer on stdout. HWJS_NM, HWJS_OBJDUMP, HWJS_AR, BRINT_TOOL # and BRINT_TOOL_INSTALLED are the caller's. link_facts_ask_undefined() { "$HWJS_NM" --undefined-only "$1" 2>/dev/null | awk '{print $NF}' } link_facts_ask_loops() { "$HWJS_OBJDUMP" -d "$1" 2>/dev/null \ | awk '/^[0-9a-f]+ <.*>:/ { fn = $2; gsub(/[<>:]/, "", fn) } /[[:space:]]loop[[:space:]]/ { if (fn != "") print fn }' \ | LC_ALL=C sort -u } # Unpacks into a directory of its own and classes each member by magic, as a # link's own unpacking does. Nothing is printed for an archive with a bitcode # member, or one that cannot be read: a link unpacks those. link_facts_ask_archive() { local _lfa_dir _lfa_m _lfa_bc=0 _lfa_n=0 _lfa_abs _lfa_abs="$(cd "$(dirname "$1")" && pwd)/$(basename "$1")" _lfa_dir="$(mktemp -d "${TMPDIR:-/tmp}/link-facts-ar.XXXXXX")" || return 0 # Same-named members overwrite each other when unpacked, so an archive with # any is left to a link, which unpacks each one. [[ -z "$("$HWJS_AR" t "$_lfa_abs" 2>/dev/null | sort | uniq -d)" ]] || { rm -rf "$_lfa_dir"; return 0; } if (cd "$_lfa_dir" && "$HWJS_AR" x "$_lfa_abs") 2>/dev/null; then while IFS= read -r -d '' _lfa_m; do _lfa_n=$((_lfa_n + 1)) link_facts_kind_v "$_lfa_m" [[ "$LINK_FACTS_KIND" == bc ]] && _lfa_bc=1 done < <(find "$_lfa_dir" -type f -print0) [[ $_lfa_n -gt 0 && $_lfa_bc == 0 ]] && echo native fi rm -rf "$_lfa_dir" } link_facts_ask_got_mem() { if [[ "${BRINT_TOOL_INSTALLED:-0}" == 1 ]]; then "$BRINT_TOOL" got-mem-imports "$1" else node -e ' const fs = require("fs"); const m = new WebAssembly.Module(fs.readFileSync(process.argv[1])); for (const i of WebAssembly.Module.imports(m)) if (i.module === "GOT.mem") console.log(i.name); ' "$1" fi } link_facts_ask_exports() { if [[ "${BRINT_TOOL_INSTALLED:-0}" == 1 ]]; then "$BRINT_TOOL" exports "$1" --with-kind else node -e ' const fs = require("fs"); const m = new WebAssembly.Module(fs.readFileSync(process.argv[1])); for (const e of WebAssembly.Module.exports(m)) console.log(e.kind + " " + e.name); ' "$1" fi } link_facts_ask_abi_digest() { if [[ "${BRINT_TOOL_INSTALLED:-0}" == 1 ]]; then "$BRINT_TOOL" abi "$1" digest else node -e ' const fs = require("fs"); const b = fs.readFileSync(process.argv[1]); let p = 8, payload = null; const leb = () => { let r = 0, s = 0; for (;;) { const x = b[p++]; r |= (x & 127) << s; if (!(x & 128)) return r >>> 0; s += 7; } }; while (p < b.length) { const id = b[p++]; const sz = leb(); const end = p + sz; if (id === 0) { const nl = leb(); const name = b.slice(p, p + nl).toString(); p += nl; if (name === "hwjs.abi" && !payload) payload = b.slice(p, end); } p = end; } if (!payload) { console.error("no hwjs.abi manifest in libc.so"); process.exit(1); } let q = 0; const fields = []; while (q < payload.length) { let r = 0n, s = 0n; for (;;) { if (q >= payload.length) { console.error("truncated LEB in hwjs.abi"); process.exit(1); } const x = BigInt(payload[q++]); r |= (x & 127n) << s; if (!(x & 128n)) break; s += 7n; } fields.push(r); } if (fields.length !== 10 || fields[0] !== 2n) { console.error(`libc.so manifest is not v2/10-field (version=${fields[0] ?? "?"} nfields=${fields.length}) — not an libc.so producer artifact`); process.exit(1); } console.log(fields[8].toString(16).padStart(16, "0")); ' "$1" fi } # bc | wasm | archive | thin | other in $LINK_FACTS_KIND, by magic. The same # reading as brintld's file_kind_v. link_facts_kind_v() { local _lfk_a='' _lfk_b='' LINK_FACTS_KIND=other { IFS= read -r -d '' -n 8 _lfk_a; IFS= read -r -d '' -n 3 _lfk_b; } 2>/dev/null < "$1" || true case "$_lfk_a" in BC$'\xc0\xde'*) LINK_FACTS_KIND=bc ;; '!'$'\n') LINK_FACTS_KIND=archive ;; '!'$'\n') LINK_FACTS_KIND=thin ;; '') if [[ "$_lfk_b" == asm ]]; then LINK_FACTS_KIND=wasm; fi ;; esac } # ── writing, when the toolchain is built ──────────────────────────────────── # link_facts_write ...: asks each question of and # writes .facts. A question that fails writes no record at all. link_facts_write() { local _lfw_f="$1" _lfw_q _lfw_ans _lfw_tmp _lfw_qsha _lfw_isha shift _lfw_qsha="$(sha256sum < "$LINK_FACTS_LIB" | cut -c1-64)" || return 1 _lfw_isha="$(sha256sum < "$_lfw_f" | cut -c1-64)" || return 1 _lfw_tmp="$_lfw_f.facts.tmp.$$" { echo "brintium-link-facts 1" echo "questions sha256=$_lfw_qsha" echo "input sha256=$_lfw_isha" echo "answered $*" } > "$_lfw_tmp" || return 1 for _lfw_q in "$@"; do case " $LINK_FACTS_QUESTIONS " in *" $_lfw_q "*) ;; *) rm -f "$_lfw_tmp"; echo "link-facts: no question '$_lfw_q'" >&2; return 1 ;; esac # A link reads libc.so's answers with the tool's stderr in them, and a # warning printed here would be replayed in every link: such an answer is # not recorded. case "$_lfw_q" in got_mem|exports|abi_digest) _lfw_ans="$("link_facts_ask_$_lfw_q" "$_lfw_f" 2>"$_lfw_tmp.err")" \ && [[ ! -s "$_lfw_tmp.err" ]] \ || { cat "$_lfw_tmp.err" >&2; false; } ;; *) _lfw_ans="$("link_facts_ask_$_lfw_q" "$_lfw_f")" ;; esac || { rm -f "$_lfw_tmp" "$_lfw_tmp.err"; echo "link-facts: $_lfw_q of $_lfw_f failed, or printed to stderr" >&2; return 1; } rm -f "$_lfw_tmp.err" [[ -z "$_lfw_ans" ]] || sed "s/^/$_lfw_q /" <<<"$_lfw_ans" >> "$_lfw_tmp" done # The input again: a record is about the bytes it was asked of. [[ "$(sha256sum < "$_lfw_f" | cut -c1-64)" == "$_lfw_isha" ]] \ || { rm -f "$_lfw_tmp"; echo "link-facts: $_lfw_f changed while it was asked" >&2; return 1; } mv -f "$_lfw_tmp" "$_lfw_f.facts" } # ── reading, at a link ────────────────────────────────────────────────────── # link_facts_prime =...: one sha256sum over this file and every # , then each is read and held to the two digests. A file with # no record beside it is not given. The digests are kept in LINK_FACTS_SHA # for the link's other uses of the same bytes; the answers in LINK_FACTS, # keyed ":"; a record that cannot be used is in # LINK_FACTS_WHY with the reason. is the record of the bytes in # : a snapshot of libc.so is read with the installed file's record. declare -A LINK_FACTS=() LINK_FACTS_SHA=() LINK_FACTS_WHY=() LINK_FACTS_ANSWERED=() LINK_FACTS_TAKEN=() link_facts_prime() { local _lfp_pair _lfp_f _lfp_r _lfp_sha _lfp_qsha _lfp_n local -a _lfp_files=() _lfp_recs=() _lfp_shas=() _lfp_read=() for _lfp_pair in "$@"; do _lfp_files+=( "${_lfp_pair%%=*}" ); _lfp_recs+=( "${_lfp_pair#*=}" ) done [[ ${#_lfp_files[@]} -gt 0 ]] || return 0 # One line per file, in order; a file sha256sum cannot read gives none, so # the count says whether the lines can be matched to the files. mapfile -t _lfp_shas < <(sha256sum -- "$LINK_FACTS_LIB" "${_lfp_files[@]}" 2>/dev/null | cut -c1-64) if [[ ${#_lfp_shas[@]} != $(( ${#_lfp_files[@]} + 1 )) ]]; then for _lfp_f in "${_lfp_files[@]}"; do LINK_FACTS_WHY[$_lfp_f]="sha256sum could not read every file given to it"; done return 0 fi _lfp_qsha="${_lfp_shas[0]}" for ((_lfp_n = 0; _lfp_n < ${#_lfp_files[@]}; _lfp_n++)); do _lfp_f="${_lfp_files[$_lfp_n]}"; _lfp_r="${_lfp_recs[$_lfp_n]}"; _lfp_sha="${_lfp_shas[$((_lfp_n + 1))]}" LINK_FACTS_SHA[$_lfp_f]="$_lfp_sha" [[ -n "$_lfp_r" ]] || continue _lfp_read+=( "$_lfp_f" "$_lfp_r" "$_lfp_sha" ) done link_facts_read "$_lfp_qsha" ${_lfp_read[@]+"${_lfp_read[@]}"} } # link_facts_read ( )...: every # record in one awk, which hands each answer back whole. A line at a time in # the shell costs a guest link seconds on libc.so's record: a turn of a bash # loop is most of a millisecond there, and each `read` an ioctl on the file. # A line is read as `IFS=' ' read -r ` reads it. The awk # writes link-facts.answers in LINK_FACTS_DIR, a directory of the caller's # own, when it is set: mapfile reads a pipe a byte per read(2), which is # seconds again in a guest. LINK_FACTS_AWK=' BEGIN { head[1] = "brintium-link-facts"; head[2] = "questions"; head[3] = "input"; head[4] = "answered" nul = sprintf("%c", 0) for (i = 1; i < ARGC; i++) { f = ARGV[i]; n = 0; nk = 0; bad = ""; h[1] = h[2] = h[3] = h[4] = "" split("", cnt); split("", kq) while ((got = (getline line < f)) > 0) { n++ if (index(line, nul)) { bad = "nul " n; break } sub(/^ +/, "", line) p = index(line, " ") if (p) { k = substr(line, 1, p - 1); v = substr(line, p + 1); sub(/^ +/, "", v); sub(/ +$/, "", v) } else { k = line; v = "" } if (n <= 4) { if (k != head[n]) { bad = "head"; break } h[n] = v; continue } if (k == "") { bad = "blank " n; break } if (!(k in cnt)) { cnt[k] = 0; kq[++nk] = k } ans[k, ++cnt[k]] = v } if (got < 0) bad = "unreadable" close(f) if (bad == "" && n < 4) bad = "head" if (bad != "") nk = 0 printf "%s%c%s%c%s%c%s%c%s%c%d%c", bad, 0, h[1], 0, h[2], 0, h[3], 0, h[4], 0, nk, 0 for (j = 1; j <= nk; j++) { k = kq[j] printf "%s%c", k, 0 for (m = 1; m <= cnt[k]; m++) { printf "%s\n", ans[k, m]; delete ans[k, m] } printf "%c", 0 } } printf "end%c", 0 }' link_facts_read() { local _lfr_qsha="$1" _lfr_f _lfr_r _lfr_sha _lfr_i _lfr_o=0 _lfr_k _lfr_q _lfr_nk local _lfr_head _lfr_qs _lfr_is _lfr_answered _lfr_bad local -a _lfr_files=() _lfr_recs=() _lfr_shas=() _lfr_args=() _lfr_out=() local -A _lfr_ans=() shift while [[ $# -ge 3 ]]; do if [[ -r "$2" && ! -d "$2" ]]; then _lfr_files+=( "$1" ); _lfr_recs+=( "$2" ); _lfr_shas+=( "$3" ) # getline < "-" reads awk's standard input case "$2" in /*) _lfr_args+=( "$2" ) ;; *) _lfr_args+=( "./$2" ) ;; esac else LINK_FACTS_WHY[$1]="$2 cannot be read" fi shift 3 done [[ ${#_lfr_files[@]} -gt 0 ]] || return 0 if [[ -n "${LINK_FACTS_DIR:-}" ]]; then if LC_ALL=C awk "$LINK_FACTS_AWK" "${_lfr_args[@]}" 2>/dev/null > "$LINK_FACTS_DIR/link-facts.answers"; then mapfile -d '' -t _lfr_out < "$LINK_FACTS_DIR/link-facts.answers" 2>/dev/null || true fi else mapfile -d '' -t _lfr_out < <(LC_ALL=C awk "$LINK_FACTS_AWK" "${_lfr_args[@]}" 2>/dev/null) fi # Every record's fields, then "end" and nothing after it, or no answer is # taken: an awk cut short leaves them short. for ((_lfr_i = 0; _lfr_i < ${#_lfr_files[@]}; _lfr_i++)); do _lfr_nk="${_lfr_out[$((_lfr_o + 5))]:-}" [[ "$_lfr_nk" =~ ^[0-9]{1,6}$ ]] || break _lfr_o=$(( _lfr_o + 6 + 2 * 10#$_lfr_nk )) done if [[ $_lfr_i != "${#_lfr_files[@]}" || "${_lfr_out[$_lfr_o]:-}" != end || $(( _lfr_o + 1 )) != "${#_lfr_out[@]}" ]]; then for ((_lfr_i = 0; _lfr_i < ${#_lfr_files[@]}; _lfr_i++)); do LINK_FACTS_WHY[${_lfr_files[$_lfr_i]}]="awk did not read ${_lfr_recs[$_lfr_i]} through" done return 0 fi _lfr_o=0 for ((_lfr_i = 0; _lfr_i < ${#_lfr_files[@]}; _lfr_i++)); do _lfr_f="${_lfr_files[$_lfr_i]}"; _lfr_r="${_lfr_recs[$_lfr_i]}"; _lfr_sha="${_lfr_shas[$_lfr_i]}" _lfr_nk="${_lfr_out[$((_lfr_o + 5))]}" _lfr_bad="${_lfr_out[$_lfr_o]}"; _lfr_head="${_lfr_out[$((_lfr_o + 1))]}"; _lfr_qs="${_lfr_out[$((_lfr_o + 2))]}" _lfr_is="${_lfr_out[$((_lfr_o + 3))]}"; _lfr_answered="${_lfr_out[$((_lfr_o + 4))]}" _lfr_ans=() for ((_lfr_o += 6, _lfr_nk = 10#$_lfr_nk; _lfr_nk > 0; _lfr_nk--, _lfr_o += 2)); do _lfr_ans[${_lfr_out[$_lfr_o]}]="${_lfr_out[$((_lfr_o + 1))]}" done case "$_lfr_bad" in '') ;; unreadable) LINK_FACTS_WHY[$_lfr_f]="$_lfr_r cannot be read"; continue ;; blank*) LINK_FACTS_WHY[$_lfr_f]="$_lfr_r has a line with no question in it (line ${_lfr_bad#blank })"; continue ;; nul*) LINK_FACTS_WHY[$_lfr_f]="$_lfr_r has a NUL byte in it (line ${_lfr_bad#nul })"; continue ;; *) LINK_FACTS_WHY[$_lfr_f]="$_lfr_r is not a version 1 record"; continue ;; esac if [[ "$_lfr_head" != 1 || -z "$_lfr_qs" || -z "$_lfr_is" ]]; then LINK_FACTS_WHY[$_lfr_f]="$_lfr_r is not a version 1 record"; continue fi if [[ "$_lfr_is" != "sha256=$_lfr_sha" ]]; then LINK_FACTS_WHY[$_lfr_f]="$_lfr_r was written for other bytes ($_lfr_is) than $_lfr_f (sha256=$_lfr_sha)"; continue fi if [[ "$_lfr_qs" != "sha256=$_lfr_qsha" ]]; then LINK_FACTS_WHY[$_lfr_f]="$_lfr_r was written by other questions ($_lfr_qs) than this link's $LINK_FACTS_LIB (sha256=$_lfr_qsha)"; continue fi for _lfr_q in $_lfr_answered; do case " $LINK_FACTS_QUESTIONS " in *" $_lfr_q "*) ;; *) LINK_FACTS_WHY[$_lfr_f]="$_lfr_r answers [$_lfr_q], which is no question of this link"; continue 2 ;; esac done for _lfr_k in "${!_lfr_ans[@]}"; do case " $_lfr_answered " in *" $_lfr_k "*) ;; *) LINK_FACTS_WHY[$_lfr_f]="$_lfr_r has an answer to [$_lfr_k], which it does not say it answers"; continue 2 ;; esac done for _lfr_q in $_lfr_answered; do LINK_FACTS["$_lfr_q:$_lfr_f"]="${_lfr_ans[$_lfr_q]:-}" LINK_FACTS_ANSWERED["$_lfr_q:$_lfr_f"]=1 done done } # link_facts_v : 0 and the answer in $LINK_FACTS_V, with no # trailing newline, when a record holds it; else 1. link_facts_v() { [[ -n "${LINK_FACTS_ANSWERED["$1:$2"]:-}" ]] || return 1 LINK_FACTS_V="${LINK_FACTS["$1:$2"]%$'\n'}" LINK_FACTS_TAKEN+=( "$1:$2" ) return 0 } # The answer from the record, or asked now: for a caller that pipes it on. link_facts_answer() { if link_facts_v "$1" "$2"; then [[ -z "$LINK_FACTS_V" ]] || printf '%s\n' "$LINK_FACTS_V" else "link_facts_ask_$1" "$2" fi } # ── the rt-import publication seed ────────────────────────────────────────── # The seed is ten lines of C that a dynamic main's link compiles each time. # The bitcode depends on the compiler too, so besides the text it is held to # the compile's flags and to the brintium commit, whose llvm-project is the # LLVM that compiled it. An installed toolchain names its commit in # libexec/hwjs-installed.stamp; a checkout would need git, so a link from one # compiles the seed. # # The stamp beside , written by sysroot/rt-import-seed.sh: # brintium-rt-import-seed 1 # source sha256= # flags # brintium commit=<40 hex> dirty=0 # seed sha256= # # rt_import_seed_take ...: 0 when the # installed seed is copied to ; else 1 with the reason in # $RT_IMPORT_SEED_WHY. is the text this link would compile. rt_import_seed_flags_v() { local _rsf_skip=0 _rsf_a RT_IMPORT_SEED_FLAGS="" for _rsf_a in "$@"; do if [[ $_rsf_skip == 1 ]]; then _rsf_skip=0; continue; fi [[ "$_rsf_a" == -isystem ]] && { _rsf_skip=1; continue; } RT_IMPORT_SEED_FLAGS+="${RT_IMPORT_SEED_FLAGS:+ }$_rsf_a" done } rt_import_seed_take() { local _rst_c="$1" _rst_bc="$2" _rst_out="$3" _rst_key _rst_rest _rst_installed="" local _rst_head="" _rst_src="" _rst_flags="" _rst_commit="" _rst_seed="" local -a _rst_have=() shift 3 RT_IMPORT_SEED_WHY="" if [[ ! -f "$_rst_bc" || ! -r "$_rst_bc.stamp" ]]; then RT_IMPORT_SEED_WHY="there is no $_rst_bc with its stamp"; return 1 fi if [[ ! -r "$HWJS_HOME/libexec/hwjs-installed.stamp" ]]; then RT_IMPORT_SEED_WHY="this toolchain names no commit in libexec/hwjs-installed.stamp to hold the seed's compiler to"; return 1 fi IFS= read -r _rst_installed < "$HWJS_HOME/libexec/hwjs-installed.stamp" || true while IFS=' ' read -r _rst_key _rst_rest || [[ -n "$_rst_key" ]]; do case "$_rst_key" in brintium-rt-import-seed) _rst_head="$_rst_rest" ;; source) _rst_src="$_rst_rest" ;; flags) _rst_flags="$_rst_rest" ;; brintium) _rst_commit="$_rst_rest" ;; seed) _rst_seed="$_rst_rest" ;; esac done < "$_rst_bc.stamp" if [[ "$_rst_head" != 1 ]]; then RT_IMPORT_SEED_WHY="$_rst_bc.stamp is not a version 1 record"; return 1 fi if [[ ! "$_rst_installed" =~ ^commit=[0-9a-f]{40}$ || "$_rst_commit" != "$_rst_installed dirty=0" ]]; then RT_IMPORT_SEED_WHY="$_rst_bc.stamp records [brintium $_rst_commit] and this toolchain is installed at [$_rst_installed]"; return 1 fi rt_import_seed_flags_v "$@" if [[ "$_rst_flags" != "$RT_IMPORT_SEED_FLAGS" ]]; then RT_IMPORT_SEED_WHY="$_rst_bc.stamp was compiled with [$_rst_flags] and this link compiles with [$RT_IMPORT_SEED_FLAGS]"; return 1 fi mapfile -t _rst_have < <(sha256sum -- "$_rst_c" "$_rst_bc" 2>/dev/null | cut -c1-64) if [[ ${#_rst_have[@]} != 2 ]]; then RT_IMPORT_SEED_WHY="sha256sum could not read $_rst_c and $_rst_bc"; return 1 fi if [[ "$_rst_src" != "sha256=${_rst_have[0]}" ]]; then RT_IMPORT_SEED_WHY="$_rst_bc.stamp was compiled from other text ($_rst_src) than this link's seed (sha256=${_rst_have[0]})"; return 1 fi if [[ "$_rst_seed" != "sha256=${_rst_have[1]}" ]]; then RT_IMPORT_SEED_WHY="$_rst_bc is not the bitcode its stamp was written for"; return 1 fi cp -f "$_rst_bc" "$_rst_out" || { RT_IMPORT_SEED_WHY="$_rst_bc could not be copied to $_rst_out"; return 1; } }