. "$BST_LIB" # libdeflate is a whole-buffer DEFLATE/zlib/gzip codec: no streaming API, no # window state carried between calls, just "here is a buffer, here is a buffer to # put the result in". That shape is why it is fast, and it is also why it exercises # a lot of this arm in a small package -- the whole body is bit-level pointer # arithmetic over loop-carried cursors (the 64-bit bitbuffer refill in # deflate_decompress.c, the three sliding-window matchfinders), which is the class # hwjs-cc#24 broke for oniguruma. # # The port ships three things, and this suite has one leg for each: # # /usr/bin/libdeflate-gzip the gzip-compatible CLI (and libdeflate-gunzip, # /usr/bin/libdeflate-gunzip the same binary under a second name) # /usr/lib/libdeflate.a the archive a consumer links (-staticdev) # /usr/include/libdeflate.h the API (-dev) # # --------------------------------------------------------------------------- # WHY THERE IS A PROBE AS WELL AS A CLI # # The CLI is real evidence -- unlike jansson or oniguruma, this port ships a # consumer that runs on the image -- but it is gzip end to end. A fully green CLI # leg says nothing about raw DEFLATE, nothing about the zlib container, nothing # about adler32, and nothing about the four-valued libdeflate_result contract that # every consumer of libdeflate.a branches on. Those are exactly the surfaces # -dev/-staticdev exist to ship, so data/libdeflate-probe is cross-linked with # wasmcc-glibc against THE VERY ARCHIVE THAT SHIPS and run below. Its source is # src/libdeflate_probe.c, and every value it asserts is the value a native x86_64 # build of the identical release produces (75/75, exit 0). # # THE CLI LEG IS ALSO THE mmap PROBE, which is the one platform hazard the hazard # sweep did find in this package. programs/prog_util.c map_file_contents() mmaps # every input file (MAP_SHARED, PROT_READ) and falls back to a plain read() ONLY # for ENODEV and EINVAL -- any other errno is a hard error. So a target where # file-backed mmap answers differently takes every file-argument case below with # it, while the stdin cases and the probe stay green. That split is the diagnosis: # CLI file cases red + stdin/probe green means mmap, not libdeflate. # # FORK BUDGET: the runner bounds each suite by wall clock (BST_SUITE_TIMEOUT), and # on this runtime the forking is the cost rather than the work -- the cjson suite # was killed at 156 check_out cases. So the payload leg uses shell builtins only, # the two file reads are one `cat` each, the input files are built by string # concatenation rather than a loop of appends, and the probe runs ONCE with its # exit status taken from that same run. # --------------------------------------------------------------------------- require libdeflate-gzip libdeflate/present bin=/usr/bin incd=/usr/include lib=/usr/lib/libdeflate.a pc=/usr/lib/pkgconfig/libdeflate.pc cm=/usr/lib/cmake/libdeflate nl=' ' # hasfile -- ok iff is a non-empty regular file. # hasexe -- ok iff is executable. # has -- ok iff contains the literal . # hasline -- ok iff carries as a WHOLE line, so # libdeflate_crc32 cannot satisfy a lookup for # libdeflate_crc and _compress_bound cannot # satisfy one for _compress. hasfile() { if [ -s "$2" ]; then bst_ok "$1"; else bst_notok "$1" "absent or empty: $2"; fi; } hasexe() { if [ -x "$2" ]; then bst_ok "$1"; else bst_notok "$1" "not executable: $2"; fi; } has() { case "$3" in *"$2"*) bst_ok "$1" ;; *) bst_notok "$1" "not found: $2" ;; esac; } hasline() { case "$nl$3$nl" in *"$nl$2$nl"*) bst_ok "$1" ;; *) bst_notok "$1" "not in archive: $2" ;; esac; } # out_rc -- ok iff exits AND says # . bst.sh has check_out (needle, but rc MUST be 0) and check_rc (rc, but # no output), and every diagnostic below is a DELIBERATE non-zero exit whose # message is the thing being checked -- so neither helper fits and check_out would # report a correct tool as broken. The native control caught exactly that on five # cases before this helper existed. out_rc() { _id=$1; _want=$2; _needle=$3; shift 3 _o=$("$@" 2>&1); _r=$? case "$_o" in *"$_needle"*) _h=1 ;; *) _h=0 ;; esac if [ "$_r" -eq "$_want" ] && [ "$_h" -eq 1 ]; then bst_ok "$_id" else bst_notok "$_id" "rc=$_r want=$_want needle:'$_needle' got:'$(_bst_1line "$_o")'"; fi } # --- the shipped payload ------------------------------------------------------ hasexe libdeflate/bin-gzip "$bin/libdeflate-gzip" hasexe libdeflate/bin-gunzip "$bin/libdeflate-gunzip" hasfile libdeflate/header "$incd/libdeflate.h" hasfile libdeflate/archive "$lib" hasfile libdeflate/pkgconfig "$pc" hasfile libdeflate/cmake-config "$cm/libdeflate-config.cmake" hasfile libdeflate/cmake-version "$cm/libdeflate-config-version.cmake" hasfile libdeflate/cmake-targets "$cm/libdeflate-targets.cmake" pctext=$(cat "$pc" 2>/dev/null) has libdeflate/pc-name "Name: libdeflate" "$pctext" has libdeflate/pc-version "Version: 1.20" "$pctext" has libdeflate/pc-libs "-ldeflate" "$pctext" has libdeflate/pc-cflags "-I\${includedir}" "$pctext" hdrtext=$(cat "$incd/libdeflate.h" 2>/dev/null) has libdeflate/hdr-version "LIBDEFLATE_VERSION_STRING" "$hdrtext" has libdeflate/hdr-1-20 "\"1.20\"" "$hdrtext" has libdeflate/hdr-minor-20 "LIBDEFLATE_VERSION_MINOR 20" "$hdrtext" # --- symbol completeness, read out of the INSTALLED archive ------------------- # The library has one flat public API and no plugin surface, so the whole export # list can simply be named. A partial compile, a wrong LIBDEFLATE_* feature # switch (the recipe could have turned the zlib or gzip wrapper off) or a # header/library skew each fail one of these rather than passing silently. arcsyms=$(strings -a "$lib" 2>/dev/null | grep -o 'libdeflate_[a-z0-9_]*' | sort -u) [ -n "$arcsyms" ] || arcsyms=$(grep -ao 'libdeflate_[a-z0-9_]*' "$lib" 2>/dev/null | sort -u) hasline libdeflate/sym-alloc-compressor libdeflate_alloc_compressor "$arcsyms" hasline libdeflate/sym-alloc-compressor-ex libdeflate_alloc_compressor_ex "$arcsyms" hasline libdeflate/sym-alloc-decompressor libdeflate_alloc_decompressor "$arcsyms" hasline libdeflate/sym-alloc-decompressor-ex libdeflate_alloc_decompressor_ex "$arcsyms" hasline libdeflate/sym-deflate-compress libdeflate_deflate_compress "$arcsyms" hasline libdeflate/sym-deflate-bound libdeflate_deflate_compress_bound "$arcsyms" hasline libdeflate/sym-deflate-decompress libdeflate_deflate_decompress "$arcsyms" hasline libdeflate/sym-deflate-decompress-ex libdeflate_deflate_decompress_ex "$arcsyms" hasline libdeflate/sym-zlib-compress libdeflate_zlib_compress "$arcsyms" hasline libdeflate/sym-zlib-decompress libdeflate_zlib_decompress "$arcsyms" hasline libdeflate/sym-gzip-compress libdeflate_gzip_compress "$arcsyms" hasline libdeflate/sym-gzip-decompress libdeflate_gzip_decompress "$arcsyms" hasline libdeflate/sym-crc32 libdeflate_crc32 "$arcsyms" hasline libdeflate/sym-adler32 libdeflate_adler32 "$arcsyms" hasline libdeflate/sym-set-memory-allocator libdeflate_set_memory_allocator "$arcsyms" hasline libdeflate/sym-free-compressor libdeflate_free_compressor "$arcsyms" hasline libdeflate/sym-free-decompressor libdeflate_free_decompressor "$arcsyms" # --- the CLI ------------------------------------------------------------------ w=${TMPDIR:-/tmp}/bst-libdeflate.$$ mkdir -p "$w" 2>/dev/null cd "$w" 2>/dev/null || { bst_notok libdeflate/workdir "cannot create $w"; exit 0; } check_out libdeflate/cli-version "gzip compression program v1.20" libdeflate-gzip -V check_out libdeflate/cli-usage "Usage: libdeflate-gzip" libdeflate-gzip -h # Inputs built by string concatenation: 54 bytes -> 6912 -> 55296, all in the # shell, so building a 54 KiB file costs one redirect instead of 64 appends. line='the quick brown fox jumps over the lazy dog 0123456789' blk="$line$line$line$line$line$line$line$line" s="$blk$blk$blk$blk$blk$blk$blk$blk" s="$s$s$s$s$s$s$s$s" printf '%s' "$s" > big.txt printf 'abcabcabcabcabc\n' > small.txt : > empty.txt # rt -- compress big.txt with , decompress, and require the # bytes back byte-identical. One result line, four forks. rt() { _id=$1; shift rm -f rt.txt rt.txt.gz cp big.txt rt.txt if ! libdeflate-gzip "$@" rt.txt >rt.err 2>&1; then bst_notok "$_id" "compress failed: $(_bst_1line "$(cat rt.err)")"; return fi if ! libdeflate-gunzip rt.txt.gz >rt.err 2>&1; then bst_notok "$_id" "decompress failed: $(_bst_1line "$(cat rt.err)")"; return fi if cmp -s rt.txt big.txt; then bst_ok "$_id" else bst_notok "$_id" "round trip differs"; fi } rt libdeflate/cli-roundtrip rt libdeflate/cli-level1 -1 rt libdeflate/cli-level6 -6 rt libdeflate/cli-level12 -12 # -c leaves the input in place and writes the stream to stdout; the whole # pipeline is the round trip, so this is the one case that never touches a .gz # file on disk. libdeflate-gzip -c big.txt > c1.gz 2>/dev/null check libdeflate/cli-stdout-keeps-input test -f big.txt libdeflate-gunzip -c c1.gz > c1.out 2>/dev/null check libdeflate/cli-stdout cmp -s c1.out big.txt # stdin/stdout only -- no file argument, so no mmap (see the header note). printf 'stdin-payload-stdin-payload\n' | libdeflate-gzip -c 2>/dev/null > sin.gz libdeflate-gunzip -c sin.gz > sin.out 2>/dev/null check_out libdeflate/cli-stdin "stdin-payload-stdin-payload" cat sin.out # -k keeps the input; without it the input is removed. cp small.txt k.txt libdeflate-gzip -k k.txt 2>/dev/null check libdeflate/cli-keep-input test -f k.txt check libdeflate/cli-keep-output test -f k.txt.gz cp small.txt d.txt libdeflate-gzip d.txt 2>/dev/null check libdeflate/cli-removes-input test '!' -f d.txt # libdeflate-gunzip decompresses from its NAME alone -- no -d. That is the # argv[0] dispatch in is_gunzip(), and it is how the tool is normally invoked. libdeflate-gunzip d.txt.gz 2>/dev/null check libdeflate/cli-gunzip-argv0 cmp -s d.txt small.txt # The empty file is its own case at both layers: mmap is not supported on a # zero-length file, so map_file_contents() takes its read_full_contents() # fallback, and the encoder must still emit a valid empty gzip member. cp empty.txt e.txt libdeflate-gzip -k e.txt 2>/dev/null check libdeflate/cli-empty-compress test -s e.txt.gz libdeflate-gunzip -c e.txt.gz > e.out 2>/dev/null check libdeflate/cli-empty-roundtrip test '!' -s e.out # -t verifies integrity without writing anything, and the gzip CRC-32 is what # makes that verification mean something -- so flip one byte in the middle of the # deflate stream and require -t to catch it. libdeflate-gzip -c big.txt > t.gz 2>/dev/null check libdeflate/cli-test-ok libdeflate-gzip -t t.gz { head -c 30 t.gz; printf 'X'; tail -c +32 t.gz; } > bad.gz 2>/dev/null check_rc libdeflate/cli-test-corrupt 1 libdeflate-gzip -t bad.gz out_rc libdeflate/cli-corrupt-msg 1 "file corrupt or not in gzip format" \ libdeflate-gunzip -c bad.gz # Truncation is caught too, and by the same message rather than a short read. head -c 40 t.gz > trunc.gz 2>/dev/null check_rc libdeflate/cli-truncated 1 libdeflate-gunzip -c trunc.gz # -S replaces the .gz suffix on both halves. cp small.txt s.txt libdeflate-gzip -S .dfl s.txt 2>/dev/null check libdeflate/cli-suffix-compress test -f s.txt.dfl libdeflate-gunzip -S .dfl s.txt.dfl 2>/dev/null check libdeflate/cli-suffix-decompress cmp -s s.txt small.txt # Two files in one invocation, each getting its own member. cp small.txt m1.txt; cp big.txt m2.txt libdeflate-gzip m1.txt m2.txt 2>/dev/null check libdeflate/cli-multi-compress test -f m1.txt.gz -a -f m2.txt.gz libdeflate-gunzip m1.txt.gz m2.txt.gz 2>/dev/null check libdeflate/cli-multi-decompress cmp -s m2.txt big.txt # The five documented refusals, asserted by EXIT CODE and not only by message -- # 1 for a real failure, 2 for "skipping this file". A tool that reported them all # as 1, or all as 0, would still print the right words. out_rc libdeflate/cli-not-gzip 1 "not in gzip format" libdeflate-gunzip -c small.txt out_rc libdeflate/cli-no-suffix 2 "does not end with the .gz suffix" \ libdeflate-gunzip small.txt # "already has .gz suffix -- skipping" is a WARNING, not a failure: the tool # leaves the file alone and still exits 0, which is what gzip(1) does too. check_out libdeflate/cli-already-gz "already has .gz suffix" libdeflate-gzip -k t.gz cp small.txt o.txt; libdeflate-gzip -k o.txt 2>/dev/null out_rc libdeflate/cli-overwrite-refused 2 "already exists; use -f to overwrite" \ libdeflate-gzip -k o.txt check libdeflate/cli-overwrite-forced libdeflate-gzip -k -f o.txt out_rc libdeflate/cli-missing-file 1 "Can't open" libdeflate-gzip -c ./no-such-file out_rc libdeflate/cli-bad-level 1 "Must be an integer in the range [0, 12]" \ libdeflate-gzip -13 -c small.txt out_rc libdeflate/cli-bad-option 1 "invalid option" libdeflate-gzip -Z -c small.txt # -f -c on non-gzipped input passes the bytes through unchanged, which is the one # case where gunzip is asked NOT to decode. libdeflate-gunzip -c -f small.txt > pt.out 2>/dev/null check libdeflate/cli-passthrough cmp -s pt.out small.txt # --- interoperability with the image's GNU gzip ------------------------------- # This is the strongest single statement the CLI leg makes: the bitstream this # arm produces is not merely self-consistent, it is DEFLATE. A wrong Huffman # table or a mis-emitted distance code round-trips fine through libdeflate's own # decoder and fails here. if command -v gzip >/dev/null 2>&1; then libdeflate-gzip -c big.txt > x1.gz 2>/dev/null check libdeflate/interop-gnu-reads gzip -t x1.gz gzip -c -d x1.gz > x1.out 2>/dev/null check libdeflate/interop-gnu-decode cmp -s x1.out big.txt gzip -c -9 big.txt > x2.gz 2>/dev/null libdeflate-gunzip -c x2.gz > x2.out 2>/dev/null check libdeflate/interop-reads-gnu cmp -s x2.out big.txt # A GNU stream at level 1 uses different block types than at 9, so decode both. gzip -c -1 big.txt > x3.gz 2>/dev/null libdeflate-gunzip -c x3.gz > x3.out 2>/dev/null check libdeflate/interop-reads-gnu-l1 cmp -s x3.out big.txt else bst_skip libdeflate/interop-gnu-reads "gzip not installed on this image" bst_skip libdeflate/interop-gnu-decode "gzip not installed on this image" bst_skip libdeflate/interop-reads-gnu "gzip not installed on this image" bst_skip libdeflate/interop-reads-gnu-l1 "gzip not installed on this image" fi # --- the library RUNNING, via the cross-linked probe -------------------------- probe=$w/libdeflate-probe if cp "$BST_DATA/libdeflate-probe" "$probe" 2>/dev/null && chmod +x "$probe" 2>/dev/null; then out=$("$probe" 2>&1); prc=$? # identity, straight out of the archive's own header has libdeflate/run-version-macros "LD-version-macros OK" "$out" has libdeflate/run-version-1-20 "LD-version-1-20 OK" "$out" # crc32 and adler32 against PUBLISHED vectors -- see the probe's header has libdeflate/run-crc32-empty "LD-crc32-empty OK" "$out" has libdeflate/run-crc32-null "LD-crc32-null OK" "$out" has libdeflate/run-crc32-digits "LD-crc32-digits OK" "$out" has libdeflate/run-crc32-fox "LD-crc32-fox OK" "$out" has libdeflate/run-crc32-a "LD-crc32-a OK" "$out" has libdeflate/run-crc32-abc "LD-crc32-abc OK" "$out" has libdeflate/run-crc32-incremental "LD-crc32-incremental OK" "$out" has libdeflate/run-crc32-256k "LD-crc32-256k OK" "$out" has libdeflate/run-adler32-init "LD-adler32-init OK" "$out" has libdeflate/run-adler32-empty "LD-adler32-empty OK" "$out" has libdeflate/run-adler32-digits "LD-adler32-digits OK" "$out" has libdeflate/run-adler32-wiki "LD-adler32-wiki OK" "$out" has libdeflate/run-adler32-a "LD-adler32-a OK" "$out" has libdeflate/run-adler32-abc "LD-adler32-abc OK" "$out" has libdeflate/run-adler32-256k "LD-adler32-256k OK" "$out" # raw DEFLATE at every level libdeflate has: 0 is the stored path and 1..12 # span all three matchfinders (ht_, hc_, bt_) has libdeflate/run-alloc-decompressor "LD-deflate-alloc-decompressor OK" "$out" has libdeflate/run-level0 "LD-deflate-level0 OK" "$out" has libdeflate/run-level1 "LD-deflate-level1 OK" "$out" has libdeflate/run-level2 "LD-deflate-level2 OK" "$out" has libdeflate/run-level3 "LD-deflate-level3 OK" "$out" has libdeflate/run-level4 "LD-deflate-level4 OK" "$out" has libdeflate/run-level5 "LD-deflate-level5 OK" "$out" has libdeflate/run-level6 "LD-deflate-level6 OK" "$out" has libdeflate/run-level7 "LD-deflate-level7 OK" "$out" has libdeflate/run-level8 "LD-deflate-level8 OK" "$out" has libdeflate/run-level9 "LD-deflate-level9 OK" "$out" has libdeflate/run-level10 "LD-deflate-level10 OK" "$out" has libdeflate/run-level11 "LD-deflate-level11 OK" "$out" has libdeflate/run-level12 "LD-deflate-level12 OK" "$out" has libdeflate/run-store "LD-deflate-store OK" "$out" has libdeflate/run-order "LD-deflate-order OK" "$out" # the three DEFLATE block types, each a different bitbuffer refill path has libdeflate/run-bitbuf-stored "LD-bitbuf-stored OK" "$out" has libdeflate/run-bitbuf-static "LD-bitbuf-static OK" "$out" has libdeflate/run-bitbuf-dynamic "LD-bitbuf-dynamic OK" "$out" has libdeflate/run-bitbuf-empty "LD-bitbuf-empty OK" "$out" # the zlib and gzip containers, their fixed headers and their trailers has libdeflate/run-zlib-compress "LD-zlib-compress OK" "$out" has libdeflate/run-gzip-compress "LD-gzip-compress OK" "$out" has libdeflate/run-zlib-header "LD-zlib-header OK" "$out" has libdeflate/run-zlib-fcheck "LD-zlib-fcheck OK" "$out" has libdeflate/run-gzip-magic "LD-gzip-magic OK" "$out" has libdeflate/run-zlib-trailer "LD-zlib-trailer-adler32 OK" "$out" has libdeflate/run-gzip-trailer-crc "LD-gzip-trailer-crc32 OK" "$out" has libdeflate/run-gzip-trailer-len "LD-gzip-trailer-isize OK" "$out" has libdeflate/run-zlib-roundtrip "LD-zlib-roundtrip OK" "$out" has libdeflate/run-gzip-roundtrip "LD-gzip-roundtrip OK" "$out" has libdeflate/run-zlib-rejects-gzip "LD-zlib-rejects-gzip OK" "$out" has libdeflate/run-gzip-rejects-zlib "LD-gzip-rejects-zlib OK" "$out" # compress_bound: an upper bound, and ordered raw <= zlib <= gzip has libdeflate/run-bound-zlib "LD-bound-zlib-ge OK" "$out" has libdeflate/run-bound-gzip "LD-bound-gzip-ge OK" "$out" has libdeflate/run-bound-order "LD-bound-order OK" "$out" has libdeflate/run-bound-null "LD-bound-null-ok OK" "$out" # all four libdeflate_result values, each reached deliberately has libdeflate/run-rc-success "LD-rc-success OK" "$out" has libdeflate/run-rc-short-output "LD-rc-short-output OK" "$out" has libdeflate/run-rc-insufficient "LD-rc-insufficient-space OK" "$out" has libdeflate/run-rc-bad-data "LD-rc-bad-data OK" "$out" has libdeflate/run-rc-bad-zlib "LD-rc-bad-data-zlib OK" "$out" has libdeflate/run-rc-truncated "LD-rc-truncated OK" "$out" has libdeflate/run-rc-bad-crc "LD-rc-gzip-bad-crc OK" "$out" has libdeflate/run-rc-nofit "LD-rc-compress-nofit OK" "$out" has libdeflate/run-ex-first "LD-rc-ex-first OK" "$out" has libdeflate/run-ex-in-used "LD-rc-ex-in-used OK" "$out" has libdeflate/run-ex-second "LD-rc-ex-second OK" "$out" # 256 KiB through the matchfinders, and an explicit 45000-byte match distance has libdeflate/run-big-level1 "LD-big-level1 OK" "$out" has libdeflate/run-big-level6 "LD-big-level6 OK" "$out" has libdeflate/run-big-level12 "LD-big-level12 OK" "$out" has libdeflate/run-match-far "LD-match-far OK" "$out" # both allocator hooks, which the library reaches through function pointers has libdeflate/run-alloc-global-called "LD-alloc-global-called OK" "$out" has libdeflate/run-alloc-global-works "LD-alloc-global-works OK" "$out" has libdeflate/run-alloc-global-balanced "LD-alloc-global-balanced OK" "$out" has libdeflate/run-alloc-object-called "LD-alloc-object-called OK" "$out" has libdeflate/run-alloc-object-works "LD-alloc-object-works OK" "$out" has libdeflate/run-alloc-object-balanced "LD-alloc-object-balanced OK" "$out" has libdeflate/run-alloc-level-reject "LD-alloc-level-reject OK" "$out" # The probe's own tally and its exit status, both from the SAME run: 75/75 is # what this source scores against a host build of the identical release. has libdeflate/run-summary "LD-SUMMARY 75/75" "$out" check_eq libdeflate/run-exit0 0 "$prc" else bst_notok libdeflate/run-stage "cannot stage $BST_DATA/libdeflate-probe" fi cd / rm -rf "$w"