. "$BST_LIB" # bash (GNU Bash 5.2.21) -- and, like 77-cunit.test, a suite whose evidence is # UPSTREAM'S OWN TEST SUITE rather than one we wrote. # # WHY THAT MATTERS MORE HERE THAN ANYWHERE ELSE ON THIS IMAGE. bash is the # largest single program that ships, and the one least amenable to sampling by # hand: its observable behaviour is a grammar plus a dozen expansion phases, and # a port that gets 95% of it right looks identical to one that gets all of it # right until the day a script hits the 5%. Chet Ramey ships 83 runnable run-* # drivers (85 files, two of which run-all skips: itself and run-minimal), # each running a .tests script under the built shell and DIFFING its output # against a .right file recorded from the reference implementation. That is a # byte-exact oracle for word splitting, quote removal, parameter and arithmetic # expansion, redirection, here-documents, process substitution, traps, aliases, # arrays, printf, extended globs and the builtins -- none of which we could have # asserted at that resolution ourselves. # # THE ORACLE IS THE .right FILES, WHICH SHIP IN THE PTEST. There is no second # control to build and nothing to keep in step: a wrong answer on this arm is a # diff against a file the release itself carries. What a native x86_64 control of # the identical 5.2.21 source IS for is the other question -- which reds are ours # and which are the suite's own environment sensitivity -- and the design doc # records that comparison rather than this file. # # HOW THE SUITE IS DRIVEN, AND WHY IT IS NOT run-all DIRECTLY. This ticket closes # on the suite run UNDER ptest-runner, so the full path execs the harness once and # asserts its discovery (walk /usr/lib/*/ptest), its fork+execv+openpty child # plumbing and its tally alongside bash's own verdicts. An image with no # ptest-runner skips those cases and stops -- there is no run-ptest-directly # fallback, because the harness IS half of what this ticket closes on and a # silently different driver would make the record mean something else. # # THE SUITE RUNS AS ROOT, AND THAT IS A STATED COST, NOT AN OVERSIGHT. Upstream's # run-ptest creates a "bashtest" user, chowns tests/ to it and re-execs under # setpriv. This image has no shadow (glibc 2.43 moved crypt() to libxcrypt and # nothing here provides it, so shadow's configure dies) and no util-linux-setpriv, # and a browser guest is a single-user root system with no account database to # drop into. bash_%.bbappend therefore ships a run-ptest that runs in place. Any # red whose subject IS an unprivileged uid -- a mode-0444 file that must refuse a # write, a mode-0644 file that must refuse an exec -- is that cost and not a # defect in the port; the design doc names each one. # # SPAWN BUDGET. exec on this arm is a fresh wasm module instantiation, ~2-3 s # apiece, and the suite itself is already thousands of them. So this file spends # five spawns of its own on direct bash evidence (version, arithmetic, arrays, # [[ ]], and the /bin/sh identity check), runs the harness ONCE, and does the # whole join in a single awk pass that emits a pre-computed " " # triple per case; the shell loop that reports them forks nothing. Presence # checks use check_rc with `test`, an ash builtin. require bash bash/present ptestdir=/usr/lib/bash/ptest runner=/usr/bin/ptest-runner tmp=${TMPDIR:-/tmp}/bst-bash.$$ mkdir -p "$tmp" 2>/dev/null || tmp=/tmp # ----------------------------------------------------------- what shipped ---- # The alternatives link and its target. /bin/bash is what THIS_SH=/bin/bash in # run-bash-ptests resolves, and /bin/bash.bash is the real binary that # update-alternatives renamed -- a missing link here means the suite would run # under whatever else answered to that name. check_rc bash/bin-bash 0 test -x /bin/bash check_rc bash/bin-bash-bash 0 test -x /bin/bash.bash # The ptest package: the harness entry point, the driver it execs, the copied # build files run-bash-ptests needs to drive make on target, and the tests tree. check_rc bash/ptest-dir 0 test -d "$ptestdir" check_rc bash/ptest-runptest 0 test -x "$ptestdir/run-ptest" check_rc bash/ptest-driver 0 test -x "$ptestdir/run-bash-ptests" check_rc bash/ptest-makefile 0 test -s "$ptestdir/Makefile" check_rc bash/ptest-configh 0 test -s "$ptestdir/config.h" check_rc bash/ptest-versionh 0 test -s "$ptestdir/version.h" check_rc bash/ptest-tests-dir 0 test -d "$ptestdir/tests" # The four support programs the suite's own scripts exec. build-tests.patch # switches them from CC_FOR_BUILD to CC precisely so they are TARGET binaries; # a host x86_64 recho here would mean the patch did not take and every test that # echoes through it would diff against a .right it can never match. for p in recho zecho printenv xcase; do check_rc "bash/ptest-support-$p" 0 test -x "$ptestdir/tests/$p" done # bash.pc and /usr/lib/bash/Makefile.inc are NOT asserted here, and their absence # is correct rather than a gap: both are packaged into bash-dev, which this image # does not install -- bash ships as a TOOL, and there is no compiler on this # image for a loadable-builtin consumer to exist. (bash_%.bbappend still installs # them, because a -dev package that silently loses its two files would be the # harder thing to notice.) Measured, not assumed: the first run asserted them # present and both reddened. # ------------------------------------------------- bash runs, on its own ------ # Five spawns, deliberately spent: they are evidence INDEPENDENT of the suite, so # a suite that cannot start still leaves a statement about whether bash executes. # The three constructs are ones no POSIX shell on this image provides, so each # also proves it is bash answering and not a fallback. check_out bash/version "5.2.21(1)-release" bash --version check_eq bash/arith 81 "$(bash -c 'echo $((3**4))' 2>&1)" check_eq bash/array c "$(bash -c 'a=(a b c); echo ${a[2]}' 2>&1)" check_eq bash/dbracket M "$(bash -c 'x=abc; [[ $x == a* ]] && echo M' 2>&1)" # THE DIVERGENCE THIS PORT DELIBERATELY MADE, ASSERTED RATHER THAN TRUSTED. # Upstream registers TWO alternatives at priority 100, "bash" and "sh"; the # second would hand /bin/sh to bash for the whole image and silently re-point # every #!/bin/sh script, the login shell, and this runner. bash_%.bbappend drops # it. If that ever regresses, $BASH_VERSION is set in /bin/sh and this reds. check_eq bash/sh-not-bash none "$(sh -c 'echo ${BASH_VERSION:-none}' 2>&1)" # ------------------------------------------------------- run it, once --------- # ptest-runner resolves /usr/lib itself, discovers bash/ptest, chdirs in and execs # run-ptest under a pty; naming the ptest confines the run to this one suite. # # -t IS SET HIGH ENOUGH THAT IT CANNOT BITE, DELIBERATELY. This suite takes ~105 # minutes on this arm -- 83 drivers and several thousand execs, each of which is # a fresh wasm module instantiation rather than a fork -- measured at ~1.25 # min/driver by streaming a run to the console. The bound that actually holds the # run is the brintos-selftest runner's BST_SUITE_TIMEOUT, which busy-waits on # /proc/uptime and therefore works here; ptest-runner -t rides the relative timer # path, which on this arm never fires at all (brintos/linux#48), so a low -t # would be an assertion about nothing. Whoever runs this suite must raise # BST_SUITE_TIMEOUT well past its 60 s default -- 9000 or more. That is stated in # the test plan, because a default sweep will otherwise report only # bash/suite-timeout. # # AND WHY THE FULL RUN IS BEHIND AN ENV VAR RATHER THAN ON BY DEFAULT. The suite # does not currently complete on this arm: it WEDGES at driver 5 of 83 # (run-array), permanently, on a platform defect this file reproduces in four # lines below. A default-on full run would therefore cost every whole-image sweep # 30+ minutes to arrive at the same red, so the default path asserts the wedge # directly -- as a RED, not a skip, because bash's suite is present and blocked # rather than legitimately absent -- and `BST_BASH_PTEST=1 brintos-selftest # 15-bash.test` (with BST_SUITE_TIMEOUT raised past 9000) still runs the real # thing. Turn the default back on in the same change that closes the defect. out=$tmp/runner.out harness=0 if [ ! -x "$runner" ]; then bst_skip bash/ptest-runner-present "this image ships no ptest-runner" for c in ptest-runner-rc runner-begin runner-end runner-tally runner-no-error \ suite-drivers-run ptest-suite; do bst_skip "bash/$c" "no ptest-runner to drive the suite" done rm -rf "$tmp" return 2>/dev/null || exit 0 fi bst_ok bash/ptest-runner-present if [ -z "${BST_BASH_PTEST:-}" ]; then # THE WEDGE IS ASSERTED, NOT RE-RUN, AND THAT IS DELIBERATE. The minimal # reproducer is two execs -- one whose argv block fits the guest's # __wasm32_args_buf and one that does not -- and the second NEVER RETURNS: the # kernel prints "argv block (NNNN B) exceeds __wasm32_args_buf (8192B)", the # child is lost, and the parent waits forever. Running it here would hang this # suite exactly the way it hangs bash's, which is the cost this branch exists # to avoid; the reproducer lives in the design doc instead. This case flips # green the day the defect is fixed, by being deleted along with this branch. # BRINTOS-WORKAROUND: hardwarejs#15 remove -- run the real suite by default # again once an oversized-argv execve fails with E2BIG instead of losing the # child; see the design doc for BRINTOS-506. bst_notok bash/ptest-suite \ "upstream suite not run by default: it wedges at run-array (driver 5 of 83) on brintos/hardwarejs#15 -- an execve whose argv block exceeds __wasm32_args_buf (8192B) never returns. Run it with BST_BASH_PTEST=1 BST_SUITE_TIMEOUT=9000." for c in ptest-runner-rc runner-begin runner-end runner-tally runner-no-error \ suite-drivers-run; do bst_skip "bash/$c" "full ptest not run (BST_BASH_PTEST unset); see bash/ptest-suite" done rm -rf "$tmp" return 2>/dev/null || exit 0 fi harness=1 "$runner" -t 20000 bash > "$out" 2>&1 rc=$? check_eq bash/ptest-runner-rc 0 "$rc" # ------------------------------------------------------- the single join ------ # One awk pass over the transcript, emitting "\t\t" per case. # run-all (with oe's test-output.patch) prints the driver name, then any output # the driver produced, then "PASS: " or "FAIL: " -- so a driver # that produced ANY output is a FAIL by upstream's own definition, and the diff # against the .right file is that output. awk -v OFS='\t' ' function emit(id, want, got) { print id, want, got } $0 ~ /^PASS: run-/ { name = $2; verdict[name] = "PASS"; order[++n] = name; next } $0 ~ /^FAIL: run-/ { name = $2; verdict[name] = "FAIL"; order[++n] = name; next } $0 ~ /^BEGIN: / { begin_line = $2; next } $0 ~ /^END: / { end_line = $2; next } $0 ~ /^TOTAL: / { tally = $0; next } $0 ~ /^TIMEOUT: / { runner_error = runner_error $0 " "; next } $0 ~ /^ERROR: / { runner_error = runner_error $0 " "; next } END { # --- the upstream suite, one case per driver, in the order it ran. --- for (i = 1; i <= n; i++) emit("bash/suite-" order[i], "PASS", verdict[order[i]]) # A run that stopped a third of the way through leaves every driver it DID # reach green, so the count is asserted separately from the verdicts. 83 is # the 85 run-* files bash 5.2.21 ships minus the two run-all skips itself: # itself, and run-minimal (the reduced suite for a stripped build). emit("bash/suite-drivers-run", DRIVERS, n "") # --- the harness: discovery, the child plumbing, and the tally. --- if (HARNESS == "1") { emit("bash/runner-begin", PTESTDIR, begin_line "") emit("bash/runner-end", PTESTDIR, end_line "") # ptest-runner counts PTESTS, not cases: the run-ptest script exits on the # status of `make -k ... runtest`, and run-all always exits 0, so this is # "the harness completed one ptest" and NOT the suite verdict. The # bash/suite-* cases above are the verdict. emit("bash/runner-tally", "TOTAL: 1 FAIL: 0", tally "") # ptest-runner reports its own fork/execv/openpty/pipe2 failures, and its # -t expiry, as ERROR/TIMEOUT lines while still exiting 0 for some of them, # so absence is asserted directly. emit("bash/runner-no-error", "", runner_error "") } } ' DRIVERS=83 PTESTDIR="$ptestdir" HARNESS="$harness" \ "$out" > "$tmp/verdicts" 2>/dev/null # ------------------------------------------------------ report, fork-free ----- n=0 while IFS=' ' read -r id want got; do [ -n "$id" ] || continue check_eq "$id" "$want" "$got" n=$((n + 1)) done < "$tmp/verdicts" # A join that emitted nothing at all is itself a failure, and one that emitted # half its cases would otherwise read as a short but green suite. check_rc bash/join-nonempty 0 test "$n" -gt 1 rm -rf "$tmp"