#!/bin/sh
# brintos-selftest — run the on-target brintOS package self-tests and emit a
# machine-parseable report. POSIX sh (busybox ash); the target image has no bash.
#
# Usage: brintos-selftest [suite ...]
#   With no args, runs every tests/*.test. An arg ending in `.test` selects that
#   file exactly; any other arg is a substring that selects every matching test
#   file (e.g. `brintos-selftest tcl net` runs *tcl*.test + *net*.test).
#
#   BST_SHARD=<i>/<n> runs only the i-th of n contiguous slices of whatever the
#   selection resolved to (1-based). A whole-image sweep in ONE boot starves the
#   guest — memory is not returned on process exit, so from roughly the 65th
#   test file on, suites fail to fork and report nothing — and the guest's RAM
#   ceiling is fixed by the kernel module's declared kernel_memory maximum, so
#   it cannot simply be raised. Sharding gives each slice a fresh guest, which
#   is the only way to cover the whole image today. The selector is a slice
#   rather than a file list because the runner command is typed onto the guest's
#   tty, whose line discipline truncates around 1.1 kB.
#
# Output protocol (consumed by brintos_browser_verify --runner):
#   ==BST-BEGIN==
#   ==BST-PRIME==            (one per suite; a sacrificial line, see below)
#   BST ok     <suite>/<case>
#   BST not ok <suite>/<case> : <detail>
#   BST skip   <suite>/<case> : <reason>
#   ==BST-IDS <suite> <id> <id> ...==   (one or more; the ids the suite tallied)
#   ==BST-SUITE <suite> elapsed=<N>s bound=<N>s cases=<N>==
#   ==BST-SUMMARY pass=<N> fail=<N> skip=<N> total=<N>==
#   ==BST-END==
# ==BST-PRIME== is not a result and is in no tally: it is a sacrificial write
# that exists to be swallowed by brintos/linux-in-the-browser#177, and a parser
# should ignore it exactly as it ignores any other unrecognised line. If it ever
# starts ARRIVING on target, #177 is fixed and this line can go.
# The BEGIN/END sentinels bracket the run so a truncated or hung run is
# detectable. Exit status is 0 iff fail==0. Each suite runs under a per-suite
# wall-clock bound (BST_SUITE_TIMEOUT, default 60s) so one hanging suite cannot
# starve the rest — a timed-out suite is reported as a <suite>/suite-timeout red
# carrying the count of cases it had already emitted, which is what separates a
# suite that hung at case 0 from one that was merely slow. A suite too heavy for
# the default bound raises its own with a `# BST-TIMEOUT: <seconds>` marker; the
# ==BST-SUITE== line reports what every suite actually spent, so that number is
# read off a run rather than guessed.
#
# Every test file that runs reports SOMETHING. A suite that produces no case at
# all is reported as a <suite>/no-cases red, and a selector that matches no file
# as a runner/no-such-suite red: an empty report is indistinguishable from
# coverage in the summary, and it is the failure that hides best — a whole-image
# sweep lost 54 of 196 files to a starved guest while still reading as a
# 150-suite run, and a mistyped --suite selector ran nothing and exited 0.

BST_DIR=${BST_DIR:-/usr/lib/brintos-tests}
BST_TESTS="$BST_DIR/tests"
export BST_DIR
export BST_DATA="${BST_DATA:-$BST_DIR/data}"
export BST_LIB="${BST_LIB:-$BST_DIR/lib/bst.sh}"

# This arm ships no locale data at all, so perl writes a five-line preamble to
# stderr on EVERY invocation, with LANGUAGE, LC_ALL and LANG all unset — it is
# complaining that it cannot honour an empty environment, and falls back to "C"
# regardless. A suite that captures with 2>&1 (most of them do, deliberately, so
# a failed module load is named rather than reported as empty output) then gets
# those five lines in front of the value it means to compare, and _bst_1line's
# 100-character cut leaves nothing else in the detail. The red then names a
# locale problem whatever actually went wrong: 18 of one batch's 35 reds were
# this, and NINE more reds were misattributed to it, because an empty stdout and
# a displaced one render identically once truncated. Those nine were a missing
# perl module in a port; the count of genuinely locale-caused reds was stated as
# 26, then 21, before this export made it readable off the capture.
#
# PERL_BADLANG=0 suppresses the preamble and nothing else. It is exported rather
# than prefixed per command because a `LC_ALL=C cmd' prefix was measured NOT to
# take on this arm while an exported value did, and because the alternative is
# getting that detail right independently in 900 suites.
#
# BRINTOS-WORKAROUND: linux-in-the-browser#179 remove — perl's no-locale
# preamble; retire this export when #179 (or its root, #160, which is the
# missing locale data itself) lands and perl stops warning.
export PERL_BADLANG=0

# Per-suite wall-clock bound. A suite that hangs (a tool that blocks on the
# multiplexer or waits for input) would otherwise stall the sequential run and
# starve every later suite until the harness' own cap trips, truncating the whole
# capture on one bad suite.
#
# We do not use timeout(1) or alarm. On a kernel without the brintos/linux#77
# fix their timer never fires, so a timeout-wrapped suite adds a second hang
# rather than bounding the first — measured on the batch-1826 image, `timeout 2
# sleep 10' returned 0 after 10.5s (BRINTOS-1868). The loop below bounds a suite on
# either kernel. `sleep' is NOT in that class and does
# work: 0.96s per `sleep 1' standing alone, and a replica of the loop below ran
# 120 iterations over a 120s child. `read -t' fires correctly too (rc=1 at
# 2.06s), but nothing here has a descriptor whose readability means "the suite
# finished", so it is not the bound either. The wall clock (/proc/uptime) does
# advance and explicit signals do reap a syscall-blocked process — so bound
# each suite by running it in the background behind a completion sentinel,
# polling /proc/uptime with `sleep 1' plus shell builtins, and kill -9 on
# overrun. The sentinel file (not kill -0) signals completion: a fast suite's
# zombie is never reaped inside the tight poll, so kill -0 would read it as
# forever-alive.
#
# NOTHING IN THE LOOP BODY MAY REAP A FOREGROUND CHILD — no command
# substitution, no pipeline. A relative sleep placed directly after one returns
# instantly (brintos/linux-in-the-browser#120): a `sleep 3' following a
# `$(...)' measured 0.04s against 0.96s standing alone. That is why the body is
# read/[/kill and a single `sleep', and it is a CORRECTNESS constraint, not a
# cost one — the one `sleep' per second costs nothing measurable (BRINTOS-1868
# priced the poll against a no-poll control: 961/969 MiB peak host RSS versus
# 966/1014 MiB).
BST_SUITE_TIMEOUT=${BST_SUITE_TIMEOUT:-60}
# Ceiling on what a suite may declare for itself (below). The bound exists to
# protect the ==BST-END== sentinel from a hung suite, and a marker carrying one
# zero too many would hand that protection back without anyone noticing. Keep it
# clear of the largest bound any suite actually declares (1800, sqlite3) — a
# ceiling sitting exactly on a real value silently becomes the bound the day that
# suite grows a case. Raised 2400 -> 3600 when 80-libtomcrypt measured 1705s
# and declared 2700: at 2400 the ceiling was no longer clear of the largest
# declared bound, it WAS it, and this variable would have silently become
# libtomcrypt's bound instead of the suite's own marker.
BST_SUITE_TIMEOUT_MAX=${BST_SUITE_TIMEOUT_MAX:-3600}
# The clock's path is a variable so the runner's WATCHDOG BRANCH can be exercised
# off-target. It is not a knob: the default is the only value the target ever
# uses. Without it every test of this runner runs the no-clock branch, because a
# build host that is not Linux has no /proc/uptime -- and the branch that ships
# is the other one. A `died` red for a suite that ended early is read out of the
# watchdog's own exit-status sentinel, so it lives entirely on the untested side.
BST_UPTIME=${BST_UPTIME:-/proc/uptime}
if [ -r "$BST_UPTIME" ]; then _have_clock=1; else _have_clock=0; fi

pass=0; fail=0; skip=0

# One bound for every suite is wrong in both directions: tight enough to catch a
# hang quickly is not enough for a suite that legitimately emits ~200 cases, and
# on a loaded host such a suite is killed and reported as a hang — a red that is
# neither true nor actionable. So a suite raises its own bound, in its first
# lines, where the knowledge is and where it stays reviewed beside the cases it
# covers:
#
#   # BST-TIMEOUT: 900
#
# It is a marker in the file rather than an env override because the runner is
# typed onto the guest's tty, whose line discipline truncates around 1.1 kB — one
# override per heavy suite does not fit, and would have to be re-typed by every
# caller. A malformed marker is reported (<suite>/bad-timeout), never silently
# ignored: a bound nobody applied reads exactly like a bound that held.
#
# Sets _bound and _bound_src. A function, so the read costs no fork and the
# `set --` word-split is the function's own — not the caller's file list.
_suite_bound() {   # $1 = path to a .test file
  _bf=$1
  _bound=$BST_SUITE_TIMEOUT
  _bound_src=default
  _bn=0
  while [ "$_bn" -lt 20 ] && IFS= read -r _bl; do
    _bn=$((_bn + 1))
    case "$_bl" in
      "#"*BST-TIMEOUT:*) ;;
      *) continue ;;
    esac
    set -- ${_bl#*BST-TIMEOUT:}
    if [ $# -ge 1 ] && [ "$1" -ge 1 ] 2>/dev/null; then
      _bound=$1
      _bound_src=suite
      if [ "$_bound" -gt "$BST_SUITE_TIMEOUT_MAX" ]; then
        _bound=$BST_SUITE_TIMEOUT_MAX
        _bound_src=suite-clamped
      fi
    else
      _bound_src=bad
    fi
    break
  done < "$_bf"
}

# BRINTOS-WORKAROUND: brintos/linux#73 revisit -- the kernel leaked EINTR (and,
# on some paths, the private -ERESTARTRESCHED sentinel, 517) into userspace, so a
# /proc/uptime read failed transiently. Root-caused and fixed on our branch as
# kernel patch 0006; the guard stays until the PINNED kernel carries it. Sets
# _up_s to whole seconds; returns 1 if the clock stayed unreadable. No fork: a
# command substitution here would cost one spawn per second, which is the very
# thing the watchdog below is written to avoid.
#
# ONE READ, NOT A RETRY LOOP, and no `2>/dev/null` on it. Both were in the
# first cut of this function and both are removed, because that cut made the
# guest wedge on EVERY scoped run instead of occasionally. Measured on the
# merged image, scoped 2-suite runs, only this function differing:
#
#   first cut (5-attempt loop + 2>/dev/null)   0/7 reached ==BST-END==
#   loop kept, 2>/dev/null removed             0/4
#   THIS shape (one read, no redirect)         2/4
#   this function absent entirely (unpatched)  5/7
#
# Pooled by the only variable that separates them -- retry loop present or
# absent -- that is 0/11 against 7/11 (Fisher exact p ~ 0.001). The 2/4 above is
# four runs and is not, on its own, a rate to quote; what it establishes is that
# this shape is no worse than having no watchdog guard at all, which is the bar.
#
# So the redirect was not the mechanism -- the extra reads are. Each attempt is
# an interruptible open on this port, and hammering five of them back to back
# with no delay, inside the loop that already runs once a second for the life
# of every suite, multiplies the exposure that brintos/linux#73 needs. Retrying
# a fault whose probability rises with the number of attempts is not
# resilience; it is the fault's amplifier.
#
# The `2>/dev/null` went with it for a separate reason worth recording so it is
# not re-added as a tidy-up: redirections are applied left to right, so
# `< /proc/uptime` is attempted BEFORE `2>` is installed, and the message it
# existed to hide was printed anyway. It never worked, and it cost one more
# interruptible open (`can't create /dev/null: Interrupted system call`) per
# attempt.
# NO `IFS=` ON THE READ, and the reason is the whole bug this line once had.
# `IFS= read -r _uu _` disables field splitting, so `_uu` gets the ENTIRE line
# -- "1234.56 5678.90", both of /proc/uptime's fields -- and `${_uu%.*}` then
# yields "1234.56 5678", which `$(( ))` rejects: `line 192: arithmetic syntax
# error`, fatal to the whole run. The default IFS is what makes `_uu` the first
# field and `_` the rest, which is what the two names were written for.
#
# It went unseen because it was UNREACHABLE. While brintos/linux#73 was open
# this read essentially always failed, so `_s0` stayed empty and every caller
# took the `_ticks` fallback; the success path ran for the first time on the
# kernel that fixed #73, and died on the first suite of the first sweep. A
# platform fix makes a previously-dead branch live, and that branch had never
# been executed.
#
# The `case` is the second half: a numeric check, so a short read or a garbled
# line degrades to the tick fallback instead of reaching the arithmetic at all.
# It is a builtin -- no fork, which is the constraint the comment below explains.
_uptime_s() {
  _up_s=
  if read -r _uu _ < "$BST_UPTIME"; then
    _up_s=${_uu%.*}
    case $_up_s in
      ''|*[!0-9]*) _up_s=; return 1 ;;
    esac
    return 0
  fi
  return 1
}

run_one() {   # $1 = path to a .test file
  _name=$(basename "$1" .test | sed 's/^[0-9]*-//')
  _killed=0
  _srrc=0
  _elapsed=
  _suite_bound "$1"
  if [ "$_have_clock" = 1 ]; then
    _rcf="/tmp/.bst.$$.rc"; _outf="/tmp/.bst.$$.out"; rm -f "$_rcf" "$_outf"
    { sh "$1"; echo $? > "$_rcf"; } > "$_outf" 2>&1 &
    _sp=$!
    if _uptime_s; then _s0=$_up_s; else _s0=; fi
    _ticks=0
    # SLEEP, DO NOT SPIN. Without the sleep this loop polls /proc/uptime as
    # fast as the guest will schedule it for the entire life of the suite. On a
    # short suite that is invisible; on a long one the spinning shell is the
    # dominant cost on this runtime and it grows the HOST guest process without
    # bound -- measured on BRINTOS-563 at 12.7 GiB and climbing 2.5 GiB/min on a
    # ~30-minute suite, which the harness then kills as a runaway. The suite
    # being blamed is a bystander: the same work run without this watchdog peaks
    # at 1.0 GiB. brintos/hardwarejs#20 is the platform side of the same shape
    # (a spinning process grows the host without bound).
    #
    # One `sleep` per second costs one spawn per second -- on a runtime where
    # spawns are the scarce resource that is still four orders of magnitude
    # cheaper than the spin, and it only coarsens the bound's resolution to 1 s,
    # which is meaningless against bounds measured in minutes.
    while [ ! -e "$_rcf" ]; do
      if _uptime_s && [ -n "$_s0" ]; then _delta=$((_up_s - _s0)); else _delta=$_ticks; fi
      if [ "$_delta" -ge "$_bound" ]; then
        kill -9 "$_sp" 2>/dev/null; _killed=1; break
      fi
      sleep 1
      _ticks=$((_ticks + 1))
    done
    if _uptime_s && [ -n "$_s0" ]; then _elapsed=$((_up_s - _s0)); else _elapsed=$_ticks; fi
    # The sentinel carries the suite's own exit status, and until BRINTOS-1360
    # this line deleted it unread. That status is the ONLY signal a suite which
    # ended early produces: the tally below counts the lines the suite emitted,
    # so `cases=` and the console agree exactly and the host parser's
    # declared-vs-reached deficit is zero. A suite that died at case 19 of 100
    # is otherwise indistinguishable from one that declares 19.
    _srrc=0
    [ -s "$_rcf" ] && read -r _srrc < "$_rcf" 2>/dev/null
    case "$_srrc" in ''|*[!0-9]*) _srrc=0 ;; esac
    rm -f "$_rcf"
  else
    _outf="/tmp/.bst.$$.out"; sh "$1" > "$_outf" 2>&1; _srrc=$?
  fi
  # Emit ONE LINE PER write(), never the whole suite in a single one. A large
  # multi-line console write intermittently fails with `sh: write error:
  # Invalid argument`, truncating the suite's results AND swallowing the next
  # line -- so a truncated run reads as a merely shorter one. It is not a fixed
  # size cap (14 kB has failed where 18 kB passed) and a same-size payload with
  # no newlines does not reproduce it: brintos/linux-in-the-browser#138.
  # read/printf are shell builtins, so this loop forks nothing per line.
  # `|| [ -n "$_l" ]` so a suite whose last line lacks a trailing newline is
  # still emitted -- read returns non-zero on that line, and dropping it would
  # be the same silent truncation this loop exists to avoid.
  #
  # Tally in the SAME pass, with `case` rather than three greps: a suite emits
  # nothing precisely when the guest is out of memory, which is also when a
  # forked grep returns an empty count and the arithmetic that reads it fails.
  # Counting with builtins keeps the tally correct exactly when it matters, and
  # spares three spawns per suite on a runtime where spawns are the scarce
  # resource.
  #
  # The ids tallied are echoed as ==BST-IDS== so the host can name a result line
  # the console lost, including in a suite whose ids are composed at runtime and
  # so cannot be read out of its file. Chunked, because one line per case would
  # double the console traffic and one line per suite can run to kilobytes.
  # BRINTOS-WORKAROUND: brintos/linux-in-the-browser#177 remove -- the guest
  # console drops exactly one line after the writer waits on a FOREGROUND child,
  # and `_name=$(basename ... | sed ...)` above is one. The drop is latched, not
  # counted, so a single sacrificial write absorbs it and every real result line
  # survives. Measured on core-image-brintos-emcraft: stock runner 2 of 3 cases
  # for 25-findutils, this line present 3 of 3. Without it EVERY suite loses its
  # FIRST result line -- 1137 of 1686 suites across the transcripts on builder,
  # a `not ok` as readily as an `ok`, while the run still exits 0.
  printf '==BST-PRIME==\n'
  _p=0; _f=0; _s=0; _why=; _ids=
  while IFS= read -r _l || [ -n "$_l" ]; do
    printf '%s\n' "$_l"
    case "$_l" in
      "BST ok "*)     _p=$((_p + 1)); _r=${_l#BST ok } ;;
      "BST not ok "*) _f=$((_f + 1)); _r=${_l#BST not ok } ;;
      "BST skip "*)   _s=$((_s + 1)); _r=${_l#BST skip } ;;
      *)              _r= ;;
    esac
    if [ -n "$_r" ]; then
      _r=${_r#"${_r%%[! ]*}"}
      _ids="$_ids ${_r%% *}"
      if [ ${#_ids} -gt 700 ]; then
        printf '==BST-IDS %s%s==\n' "$_name" "$_ids"; _ids=
      fi
    fi
    if [ -z "$_why" ]; then
      case "$_l" in *[!\ ]*) _why=$_l ;; esac
    fi
  done < "$_outf"
  # Before the rm, not after: rm is a foreground child, and the first line
  # written after one is the line brintos/linux-in-the-browser#177 drops.
  # Measured: flushed after the rm, the last chunk of every suite was lost (2/2).
  [ -z "$_ids" ] || printf '==BST-IDS %s%s==\n' "$_name" "$_ids"
  rm -f "$_outf"
  # Frozen before the runner's own synthetic reds are added, so every message
  # below quotes what the SUITE emitted, not what the runner made of it.
  _cases=$((_p + _f + _s))
  # A suite that reported no case at all vanishes from the summary and reads as
  # coverage — the worst of the three outcomes, since nothing downstream can tell
  # it from a suite that was never meant to run. Carry the first line it did
  # print, which is where "sh: Cannot allocate memory" shows up.
  # A suite whose very first line is the shell refusing to stage an exec image
  # did not run at all, so it carries the same marker bst.sh puts on an unrun
  # case — the two are one population and are counted together downstream.
  if [ "$_killed" = 0 ] && [ "$_cases" -eq 0 ]; then
    [ -n "$_why" ] || _why="no output at all"
    case "$_why" in
      *": "*": Cannot allocate memory"*|"sh: Cannot allocate memory"*)
        _tag="UNRUN-EXEC-ENOMEM (brintos/linux#44): " ;;
      *) _tag= ;;
    esac
    echo "BST not ok $_name/no-cases : ${_tag}suite reported no cases; first output: $(printf %.100s "$_why")"
    _f=$((_f + 1))
  fi
  # A suite whose shell exited non-zero did not reach its end: ash exits on a
  # top-level stdin redirect from the virtiofs root (dup2(2) returns 513,
  # brintos/linux#73), and any other fatal shell error ends it the same way. The
  # cases it had already emitted are still reported -- they ran -- but the run no
  # longer reads as a clean, merely shorter one.
  #
  # Only for a suite that emitted cases and THEN stopped, which is the shape
  # nothing else can see. A suite that died before its first case already has
  # `no-cases`, whose detail carries the shell's own diagnostic, and a suite the
  # watchdog killed already has `suite-timeout`: reporting either of those twice
  # would inflate the fail count for one event.
  if [ "$_killed" = 0 ] && [ "$_cases" -gt 0 ] && [ "$_srrc" -ne 0 ]; then
    echo "BST not ok $_name/died : suite shell exited $_srrc after $_cases case(s); it did not reach the end of the file"
    _f=$((_f + 1))
  fi
  # A marker the runner could not read is a bound nobody applied, and the suite
  # then runs under the default while its file says otherwise.
  if [ "$_bound_src" = bad ]; then
    echo "BST not ok $_name/bad-timeout : unreadable '# BST-TIMEOUT:' marker; ran under the ${_bound}s default"
    _f=$((_f + 1))
  fi
  pass=$((pass + _p)); fail=$((fail + _f)); skip=$((skip + _s))
  # Surface the hang as an attributed red instead of silently dropping the
  # suite's remaining cases; partial BST lines above still count. The case count
  # is what tells a reader whether the suite hung at case 0 or was merely slow,
  # and the bound's provenance whether raising its marker is the fix.
  if [ "$_killed" = 1 ]; then
    echo "BST not ok $_name/suite-timeout : exceeded ${_bound}s wall-clock (runner-bounded, $_bound_src) after $_cases case(s)"
    fail=$((fail + 1))
  fi
  # What the suite actually spent, every run — the only way a per-suite bound
  # gets set from measurement instead of from a guess, and an early warning for
  # a suite drifting up toward its own.
  if [ -n "$_elapsed" ]; then
    echo "==BST-SUITE $_name elapsed=${_elapsed}s bound=${_bound}s cases=$_cases=="
  fi
}

echo "==BST-BEGIN=="

# Resolve the selection first, then run it: a shard has to be cut out of the
# whole list, so the list has to exist before anything runs.
_files=
if [ $# -eq 0 ]; then
  for f in "$BST_TESTS"/*.test; do
    [ -e "$f" ] || continue
    _files="$_files $f"
  done
  if [ -z "$_files" ]; then
    echo "BST not ok runner/no-tests : no test files under $BST_TESTS"
    fail=$((fail + 1))
  fi
else
  for pat in "$@"; do
    # A selector naming a file (`61-bc.test`) is matched exactly; anything else
    # stays a substring. Exactness is what lets a caller run one suite of a
    # family without dragging in its siblings — `94-e2fsprogs` as a substring
    # also pulls -fsck and -resize. Unquoted so both forms glob.
    case "$pat" in
      *.test) _glob="$BST_TESTS/$pat" ;;
      *)      _glob="$BST_TESTS/*$pat*.test" ;;
    esac
    _hit=0
    for f in $_glob; do
      [ -e "$f" ] || continue
      _hit=1
      _files="$_files $f"
    done
    # A selector that matched nothing used to run zero suites and still exit 0,
    # so a typo in a gate's --suite reported PASS on an empty run.
    if [ "$_hit" = 0 ]; then
      echo "BST not ok runner/no-such-suite : selector '$pat' matched no test file under $BST_TESTS"
      fail=$((fail + 1))
    fi
  done
fi

# Word-split is the intent here: test paths carry no spaces.
set -- $_files
_lo=0; _hi=$#
if [ -n "${BST_SHARD:-}" ]; then
  _si=${BST_SHARD%%/*}; _sn=${BST_SHARD##*/}
  if [ "$_si" -ge 1 ] 2>/dev/null && [ "$_sn" -ge 1 ] 2>/dev/null && [ "$_si" -le "$_sn" ]; then
    _lo=$(( (_si - 1) * $# / _sn ))
    _hi=$(( _si * $# / _sn ))
    echo "==BST-SHARD $_si/$_sn files=$((_hi - _lo)) of $#=="
  else
    echo "BST not ok runner/bad-shard : BST_SHARD='$BST_SHARD' is not <i>/<n> with 1<=i<=n"
    fail=$((fail + 1)); _hi=0
  fi
fi
_i=0
for f in "$@"; do
  if [ "$_i" -ge "$_lo" ] && [ "$_i" -lt "$_hi" ]; then
    run_one "$f"
  fi
  _i=$((_i + 1))
done
echo "==BST-SUMMARY pass=$pass fail=$fail skip=$skip total=$((pass + fail + skip))=="
echo "==BST-END=="

[ "$fail" -eq 0 ]