. "$BST_LIB" # lzlib 1.14 -- Antonio Diaz Diaz's LZMA compression library in the lzip format # (poky recipes-extended/lzip/lzlib_1.14.bb), plus upstream's own minilzip(1). # # --------------------------------------------------------------------------- # WHY THIS SUITE IS SHAPED THE WAY IT IS # # lzlib is not a syscall library. Unlike lockdev or libutempter next door it # opens no file, forks nothing and touches no VFS -- it is LZMA range coding # over malloc'd buffers. So there is no platform primitive to interrogate and # a "does it run" test would prove almost nothing: an encoder with a broken # shift still produces a member its own decoder reads back perfectly. # # What this port is really asking is whether the ARITHMETIC is right on # wasm32-hwjs, and the sharpest available question is BYTE IDENTITY. The # compressed stream is a pure function of the input bytes and the encoder # parameters, so one carry lost in the range coder's renormalisation, one # sign-extension in the match finder's 32-bit position arithmetic, or one # wrong entry in the probability price table yields a stream that is still # valid, still round-trips, and is a different sequence of bytes from every # other lzip implementation on earth. Only a comparison against a native build # can see that -- which is why most cases below are a compressed SIZE or a # compressed CRC rather than a success flag. # # EXPECTATIONS COME FROM A NATIVE x86_64 CONTROL, never from reading the # source. data/lzlib-native.txt is the verbatim output of the same probe built # against a native build of the identical unpatched 1.14 tarball. The probe # prints no pointer, no size_t and no timing, and generates its payload # in-process from a fixed xorshift32 seed rather than reading a file, so the # two runs cannot diverge on WHAT they compressed. Every case is scored by # comparing the target's value for a tag against the control's value for the # SAME tag: a red always means "wasm32-hwjs answered differently from x86_64" # and never "the test author guessed". # # WHAT THE PROBE COVERS, and why each part earns its place: # # both encoders LZ_compress_open dispatches to the FAST encoder # (fast_encoder.c -- its own match finder and price model) # if and only if dictionary_size == 65535 AND # match_len_limit == 16. p0 is the only preset that # reaches it; the other four drive LZ_encoder. One preset # would leave an entire encoder unexecuted. # the wire format the 6-byte header and the 20-byte trailer are dumped raw. # The trailer is a CRC32 plus TWO little-endian 64-bit # integers, and a round-trip cannot see their byte order -- # our own decoder would read back a big-endian trailer it # had itself written just as happily. # two CRCs the probe computes its own CRC-32 over the payload and # also prints the decoder's LZ_decompress_data_crc(). They # must agree with each other AND with native, which # verifies the library's integrity check rather than merely # showing it to be self-consistent. # chunking measured, and the answer is asymmetric: chunking the # WRITES changes the compressed bytes (encoding starts # before the whole input has arrived, so the match finder # parses against what it can see), chunking the READS does # not. Both are pinned -- see Z34/Z35. # multi-member the encoder does not roll over by itself; the caller must # call LZ_compress_restart_member after member_finished. # damage a flipped bit in the body, a flipped byte in the stored # CRC, and a truncated member must each produce a specific # LZ_Errno and never silent wrong output. That is the # property the whole format exists for. # recovery sync_to_member must bound the loss of a corrupt # multi-member stream to ONE member, which is the entire # reason lzip splits large files into members. # # minilzip(1) IS A SECOND, INDEPENDENT EVIDENCE CHANNEL. It is upstream's own # program, built by the default `all` target and shipped by this port, so the # suite can also drive the library end to end through a real CLI on real files # -- and minilzip-cksum below compresses a file that is byte-identical on both # machines (the control itself) and compares the result against the native # byte count and checksum, which is the same byte-identity assertion made # without the probe. # --------------------------------------------------------------------------- ARCHIVE=/usr/lib/liblz.a HDR=/usr/include/lzlib.h MAN=/usr/share/man/man1/minilzip.1 MINILZIP=/usr/bin/minilzip PROBE=$BST_DATA/lzlib-probe NATIVE=$BST_DATA/lzlib-native.txt if [ ! -f "$ARCHIVE" ] && [ ! -f "$HDR" ]; then bst_skip lzlib/present "lzlib not installed on this image" exit 0 fi bst_ok lzlib/present # --- the package is whole ---------------------------------------------------- # FORK BUDGET, and it is the binding constraint on this file. The runner bounds # each suite by wall clock (BST_SUITE_TIMEOUT, default 60 s) and on this runtime # the forking is the cost rather than the work: MEASURED in-guest at ~200 ms per # /bin/true and ~650 ms per awk. The first draft of this suite spent 111 s in one # place -- an awk + sed pair per line of the 102-line control below -- and the # whole file came back as a single suite-timeout red with 145 results attached to # it. So every lookup here is a shell builtin over text read ONCE, and the only # forks left are the ones that do actual work (minilzip, cmp, the probe). if [ -f "$HDR" ]; then bst_ok lzlib/header; else bst_notok lzlib/header "absent: $HDR"; fi if [ -f "$ARCHIVE" ]; then bst_ok lzlib/staticlib; else bst_notok lzlib/staticlib "absent: $ARCHIVE"; fi # ${PN} carries minilzip and nothing else: upstream's only other output is a # shared object, and shared objects are refused distro-wide. A .so appearing # here would mean the bbappend's CONFIGUREOPTS:remove stopped taking effect. if [ -e /usr/lib/liblz.so ] || [ -e /usr/lib/liblz.so.1 ] \ || [ -e /usr/lib/liblz.so.1.14 ]; then bst_notok lzlib/no-shared "a shared liblz is installed on a static-only distro" else bst_ok lzlib/no-shared fi # minilzip.1 goes to lzlib-doc and this image installs no -doc packages at all, # so its absence is an image policy, not a packaging fault of this port. if [ -s "$MAN" ]; then bst_ok lzlib/man else bst_skip lzlib/man "lzlib-doc is not installed on this image" fi # --- the header declares what the manual promises ---------------------------- # has -- ok iff contains , in-process. has() { case "$3" in *"$2"*) bst_ok "$1" ;; *) bst_notok "$1" "not found: $2" ;; esac; } hdr=$(cat "$HDR" 2>/dev/null) for fn in LZ_api_version LZ_version LZ_strerror \ LZ_min_dictionary_size LZ_max_dictionary_size \ LZ_compress_open LZ_compress_close LZ_compress_read \ LZ_compress_write LZ_compress_finish LZ_compress_sync_flush \ LZ_compress_restart_member LZ_compress_errno \ LZ_decompress_open LZ_decompress_close LZ_decompress_read \ LZ_decompress_write LZ_decompress_finish LZ_decompress_reset \ LZ_decompress_sync_to_member LZ_decompress_data_crc; do case "$hdr" in *"$fn"*) bst_ok "lzlib/hdr-$fn" ;; *) bst_notok "lzlib/hdr-$fn" "$fn missing from $HDR" ;; esac done # LZ_API_VERSION is the number a consumer #if's on; it is not derivable from # the version string and is what tells a caller whether restart_member exists. case "$hdr" in *'#define LZ_API_VERSION 1014'*) bst_ok lzlib/hdr-api-version-macro ;; *) bst_notok lzlib/hdr-api-version-macro "LZ_API_VERSION 1014 missing from $HDR" ;; esac # --- the archive carries the symbols ----------------------------------------- # On this arm the archive is ThinLTO bitcode, so symbol NAMES survive as plain # text while string constants do not -- the limitation libutempter recorded. if command -v strings >/dev/null 2>&1; then syms=$(strings "$ARCHIVE" 2>/dev/null) has lzlib/sym-compress-open "LZ_compress_open" "$syms" has lzlib/sym-compress-read "LZ_compress_read" "$syms" has lzlib/sym-decompress-open "LZ_decompress_open" "$syms" has lzlib/sym-decompress-read "LZ_decompress_read" "$syms" has lzlib/sym-strerror "LZ_strerror" "$syms" else for s in compress-open compress-read decompress-open decompress-read strerror; do bst_skip "lzlib/sym-$s" "strings not installed on this image" done fi # --- minilzip(1): the library end to end, through a real program ------------- tmp=${TMPDIR:-/tmp}/bst-lzlib.$$ mkdir -p "$tmp" 2>/dev/null if [ ! -x "$MINILZIP" ]; then bst_skip lzlib/minilzip-present "minilzip is not installed on this image" else bst_ok lzlib/minilzip-present check_out lzlib/minilzip-version "1.14" "$MINILZIP" --version # A file that is byte-identical on this machine and on the control host: the # native control itself. That makes the compressed output comparable, which # is the byte-identity assertion again -- reached through the CLI rather than # through the probe, so a defect that somehow spared the library API would # still be caught here. if [ -f "$NATIVE" ]; then cp "$NATIVE" "$tmp/in" 2>/dev/null if "$MINILZIP" -6 -k -c "$tmp/in" > "$tmp/in.lz" 2>/dev/null && [ -s "$tmp/in.lz" ]; then bst_ok lzlib/minilzip-compress check lzlib/minilzip-test "$MINILZIP" -t "$tmp/in.lz" if "$MINILZIP" -d -c "$tmp/in.lz" > "$tmp/out" 2>/dev/null; then check lzlib/minilzip-roundtrip cmp -s "$tmp/in" "$tmp/out" else bst_notok lzlib/minilzip-roundtrip "minilzip -d failed" fi # The compressed size and checksum are a pure function of the input and # the level. Both were MEASURED on a native x86_64 minilzip 1.14 built # from the identical tarball, compressing this same file: # # $ minilzip -6 -k -c data/lzlib-native.txt | wc -c -> 999 # $ minilzip -6 -k -c data/lzlib-native.txt | cksum -> 3692058210 # # The input size is asserted FIRST and separately, so that regenerating # data/lzlib-native.txt without refreshing these two numbers shows up as # "the control file changed" rather than as a fabricated port failure. check_eq lzlib/minilzip-input-size 2519 \ "$(wc -c < "$tmp/in" 2>/dev/null | tr -d ' ')" check_eq lzlib/minilzip-csize 999 \ "$(wc -c < "$tmp/in.lz" 2>/dev/null | tr -d ' ')" if command -v cksum >/dev/null 2>&1; then check_eq lzlib/minilzip-cksum 3692058210 \ "$(cksum < "$tmp/in.lz" | awk '{print $1}')" else bst_skip lzlib/minilzip-cksum "cksum not installed on this image" fi # a single flipped byte must be refused, not decoded if command -v dd >/dev/null 2>&1; then cp "$tmp/in.lz" "$tmp/bad.lz" printf 'X' | dd of="$tmp/bad.lz" bs=1 seek=40 conv=notrunc 2>/dev/null if "$MINILZIP" -t "$tmp/bad.lz" >/dev/null 2>&1; then bst_notok lzlib/minilzip-detects-corruption \ "minilzip -t accepted a stream with a flipped byte" else bst_ok lzlib/minilzip-detects-corruption fi else bst_skip lzlib/minilzip-detects-corruption "dd not installed on this image" fi # THE LEVEL SWEEP, and the one hard platform limit this port ran into. # # -0 is the fast encoder and -9 the largest dictionary; on any ordinary # machine both round-trip. -8 AND -9 DO NOT RUN HERE, and the red is the # platform's rather than lzlib's: every process on wasm32-hwjs is capped at # 256 MiB of user address space (hardwarejs PhysRam userMaxBytes, a # WebAssembly.Memory `maximum` reserved per process) NO MATTER how much RAM # the machine has -- this run advertised 916 MiB in /proc/meminfo. lzlib's # LZ_encoder needs about ten times its dictionary, so the ceiling lands # between a 20 MiB dictionary (runs) and 21 MiB (refused), which is between # level 7 and level 8. Measured, not reasoned: the sweep below brackets it, # so the red carries the real boundary instead of a claim about it. # # BRINTOS-WORKAROUND: linux-in-the-browser#143 remove -- a process cannot # reserve more than 256 MiB of user address space, so -9's dictionary does # not fit and lzlib declines it. The level cases are therefore scored three # ways rather than two: a level that round-trips is ok, a level the library # DECLINES CLEANLY under that cap is an attributed skip, and anything else # is not ok. When #143 closes, -9 round-trips and reports ok on its own -- # drop the `refused` branch below and this marker, and nothing else changes. # # The skip is deliberately narrow. It is reached only when lzlib refused # with its own out-of-memory diagnostic AND wrote nothing; a truncated # stream, a wrong stream, or a refusal for any other reason still lands in # `broken` and is still not ok. The case therefore keeps every bit of its # regression value -- what it stops reporting nightly is the one outcome # already known, measured, and tracked by an open issue of its own. # # The invariant asserted on every platform is level-refusals-are-clean: a # level either round-trips EXACTLY, or lzlib declines it with its own # out-of-memory diagnostic and writes nothing. A truncated or wrong stream # would be a genuine defect; a clean refusal is the library behaving # correctly under a limit it was told about. ran= ; refused= ; broken= for lvl in 0 6 7 8 9; do if "$MINILZIP" "-$lvl" -k -c "$tmp/in" > "$tmp/l.lz" 2>"$tmp/l.err"; then if "$MINILZIP" -d -c "$tmp/l.lz" > "$tmp/l.out" 2>/dev/null && cmp -s "$tmp/in" "$tmp/l.out"; then ran="$ran$lvl" else broken="$broken$lvl"; fi elif [ -s "$tmp/l.lz" ]; then broken="$broken$lvl" # refused, yet left output behind else case $(cat "$tmp/l.err" 2>/dev/null) in *"Not enough memory"*) refused="$refused$lvl" ;; *) broken="$broken$lvl" ;; esac fi [ "$lvl" = 0 ] || [ "$lvl" = 9 ] || continue case $ran in *"$lvl"*) bst_ok "lzlib/minilzip-level$lvl" ;; *) case $refused in *"$lvl"*) bst_skip "lzlib/minilzip-level$lvl" \ "-$lvl's dictionary does not fit the 256 MiB per-process address space (brintos/linux-in-the-browser#143); lzlib declined it and wrote nothing: $(_bst_1line "$(cat "$tmp/l.err" 2>/dev/null)")" ;; *) bst_notok "lzlib/minilzip-level$lvl" \ "-$lvl did not round-trip: $(_bst_1line "$(cat "$tmp/l.err" 2>/dev/null)")" ;; esac ;; esac done if [ -z "$broken" ]; then bst_ok lzlib/level-refusals-are-clean else bst_notok lzlib/level-refusals-are-clean \ "level(s) '$broken' neither round-tripped nor declined cleanly (ran:'$ran' refused:'$refused')" fi # stdin -> stdout, the pipeline shape everything actually uses if "$MINILZIP" -c < "$tmp/in" 2>/dev/null | "$MINILZIP" -d -c > "$tmp/p.out" 2>/dev/null && cmp -s "$tmp/in" "$tmp/p.out"; then bst_ok lzlib/minilzip-pipe else bst_notok lzlib/minilzip-pipe "compress|decompress through a pipe did not round-trip" fi else bst_notok lzlib/minilzip-compress "minilzip -6 produced no output" fi else bst_skip lzlib/minilzip-compress "data/lzlib-native.txt absent; no comparable input" fi fi # --- the probe: the library API actually running ----------------------------- if [ ! -f "$PROBE" ]; then bst_skip lzlib/probe-runs "data/lzlib-probe absent from the image" rm -rf "$tmp"; exit 0 fi if [ ! -f "$NATIVE" ]; then bst_skip lzlib/probe-runs "data/lzlib-native.txt absent from the image" rm -rf "$tmp"; exit 0 fi # data/* installs 0644, so stage a copy and make it executable. probe=$tmp/lzlib-probe pout=$tmp/probe.out if ! cp "$PROBE" "$probe" 2>/dev/null || ! chmod +x "$probe" 2>/dev/null; then bst_notok lzlib/probe-runs "cannot stage the probe to $probe" rm -rf "$tmp"; exit 0 fi perr=$("$probe" 2>&1 >"$pout"); prc=$? if [ "$prc" -eq 0 ]; then bst_ok lzlib/probe-runs else bst_notok lzlib/probe-runs "probe exited rc=$prc $(_bst_1line "$perr")"; fi # The probe prints a terminator, so a run that died halfway is distinguishable # from one that merely disagreed -- the truncated-output failure mode again. if grep -q '^LZLIB-PROBE-END$' "$pout" 2>/dev/null; then bst_ok lzlib/probe-complete else bst_notok lzlib/probe-complete "probe output has no LZLIB-PROBE-END terminator" fi # Aggregate: every line, compared IN THE GUEST with cmp(1) -- the BRINTOS-260 # lesson that a console dump truncates long lines, so the comparison has to # happen on the target and not in the transcript. if cmp -s "$pout" "$NATIVE"; then bst_ok lzlib/matches-native else bst_notok lzlib/matches-native \ "probe output differs from the native x86_64 control; see the per-case results below" fi # ...and then case by case, so a red names itself. The expected value for each # tag is READ OUT OF THE NATIVE CONTROL, never written here. # # Both sides are read with the shell's own `read`, and the lookup is parameter # expansion over one string -- no awk and no sed. That is the fork budget at the # top of this file: the obvious spelling (an awk per tag to find the value, a sed # per tag to trim the tag) is 204 forks over 102 lines and took 111 s in-guest on # its own, more than the runner's whole per-suite budget. nl=' ' # Normalise every probe line to "TAG VALUE" with the separator always present, so # a tag whose value is empty is still found rather than reported . lookup= while read -r t v; do [ -n "$t" ] || continue lookup="$lookup$nl$t $v" done < "$pout" while read -r tag want; do [ -n "$tag" ] || continue case "$tag" in LZLIB-PROBE-END) continue ;; esac case "$lookup" in *"$nl$tag "*) rest=${lookup#*"$nl$tag "}; got=${rest%%"$nl"*} ;; *) got='' ;; esac # every tag is Z-, so this is the sed that used to be here id=lzlib/${tag#*-} if [ "$want" = "$got" ]; then bst_ok "$id" else bst_notok "$id" "want:'$want' got:'$got'" fi done < "$NATIVE" rm -rf "$tmp"