. "$BST_LIB" # BST-TIMEOUT: 300 # 131 cases, measured 91 s on wasm32; the 60 s default trips it # zchunk is a container format for files that are DELTA-FRIENDLY BY CONSTRUCTION. # Instead of compressing a file as one stream, it splits the input into chunks on # content-defined boundaries (a buzhash rolling window, or an explicit separator # string with -s), compresses each chunk independently with zstd, and writes an # index of per-chunk checksums into a header. A client holding an older copy can # then hash its own chunks, diff the two indexes, and fetch only the chunks whose # checksums it does not already have -- which is what dnf/librepo use it for. # # So the property that matters here is not "it compresses", it is that the # CHUNKING AND THE INDEX ARE REAL: chunk boundaries respond to the input, the # per-chunk checksums verify, a corrupted chunk is caught rather than decoded, # and two related files report a partial match. A round trip alone would pass on # a one-chunk file that had no index worth the name, so it is asserted here as # the floor rather than the finding. # # TWO REGISTERS, as the sibling compression suites use: the shipped payload # (header, pkg-config, static archive) and the codec RUNNING through the five # installed tools. The payload half matters more than usual for this port, # because the library ships a download API whose front-end binary does not -- # see the zckdl block near the end. # # PERFORMANCE NOTE. The runner bounds each suite (60 s default, 300 s at the # ticket gate), and on wasm32 a fork+exec is dear enough that ~80 of them spend # the budget on their own. So the archive/header scans below do a SMALL number of # passes that each build a newline-delimited token set, then test membership with # the shell's `case` builtin, which forks nothing. Keep that shape when adding # cases: a per-name `grep` here costs a process, not a string compare. inc=/usr/include/zck.h pc=/usr/lib/pkgconfig/zck.pc lib=/usr/lib/libzck.a NL=' ' # _have — true when is a whole line of $toks. Anchoring on the # newlines keeps it exact: a plain substring test would let zck_get_chunk_count # satisfy zck_get_chunk, and dl_range.c.o satisfy range.c.o. _have() { case "$toks" in *"$NL$1$NL"*) return 0 ;; esac; return 1; } # ---- the header ----------------------------------------------------------- check zchunk/header-present test -f "$inc" check zchunk/header-nonempty test -s "$inc" check zchunk/header-guard grep -q "define ZCK_H" "$inc" # zck.h is GENERATED: include/meson.build runs configure_file over zck.h.in and # substitutes @version@ from the meson project version. Asserting the literal # here therefore proves the generation step ran, not merely that a header was # copied -- an unsubstituted header would carry the bare @version@ token. check_out zchunk/header-version '#define ZCK_VERSION "1.4.0"' cat "$inc" check zchunk/header-no-unsubstituted sh -c "! grep -q '@version@' '$inc'" # The two public enums a caller switches on. check zchunk/type-hash-enum grep -q "typedef enum zck_hash" "$inc" check zchunk/type-comp-enum grep -q "typedef enum zck_comp" "$inc" toks="$NL$(grep -oE '[A-Za-z_][A-Za-z0-9_]*' "$inc" | sort -u)$NL" # The hash tiers zchunk can be asked for. SHA512_128 is the default for chunk # checksums (128 bits of SHA-512 -- short enough that an index over thousands of # chunks stays small) and is asserted running, below. for e in ZCK_HASH_SHA1 ZCK_HASH_SHA256 ZCK_HASH_SHA512 ZCK_HASH_SHA512_128 \ ZCK_COMP_NONE ZCK_COMP_ZSTD; do if _have "$e"; then bst_ok "zchunk/enum-$e"; else bst_notok "zchunk/enum-$e" "absent from zck.h"; fi done # The declared API, sampled across every axis the format has: lifecycle, the # write path, the read path, the chunk index, the validation entry points the # format exists for, and the error reporting every call routes through. for f in zck_create zck_free zck_init_read zck_init_write zck_close \ zck_set_ioption zck_set_soption \ zck_read zck_write zck_end_chunk \ zck_get_chunk_count zck_get_chunk_digest zck_get_chunk_size \ zck_get_chunk_start zck_get_chunk_valid \ zck_get_header_length zck_get_data_length \ zck_get_chunk_hash_type zck_hash_name_from_type \ zck_validate_checksums zck_validate_data_checksum \ zck_find_matching_chunks zck_failed_chunks \ zck_get_error zck_is_error zck_clear_error; do if _have "$f"; then bst_ok "zchunk/api-$f"; else bst_notok "zchunk/api-$f" "not declared in zck.h"; fi done # ---- pkg-config ----------------------------------------------------------- # What a future recipe DEPENDS-ing on zchunk actually consumes. filebase is # 'zck' while name is 'libzck', so the file is zck.pc -- assert both, because a # recipe writing `dependency('zchunk')` would find neither. check zchunk/pc-present test -f "$pc" check_out zchunk/pc-name "Name: libzck" cat "$pc" check_out zchunk/pc-version "Version: 1.4.0" cat "$pc" check_out zchunk/pc-libs "-lzck" cat "$pc" check_out zchunk/pc-cflags "Cflags: -I" cat "$pc" check_out zchunk/pc-libdir "libdir=" cat "$pc" # Requires names both real dependencies. This is the pkg-config-level statement # of the same fact the archive half proves by symbol, and it is what makes a # downstream static link resolve -- libzck.a alone does not carry libzstd's or # libcrypto's objects. check_out zchunk/pc-requires-zstd "libzstd" cat "$pc" check_out zchunk/pc-requires-openssl "openssl" cat "$pc" # ---- the archive ---------------------------------------------------------- check zchunk/lib-present test -f "$lib" check zchunk/lib-nonempty test -s "$lib" check_out zchunk/lib-archive "!" head -c 8 "$lib" # One scan builds the token set for both blocks below. Flatten first: GNU grep # suppresses matches inside data it classifies as binary, and busybox tr does # not handle [:print:] under -c, so the octal range is the spelling both greps # and both trs agree on. flat=${TMPDIR:-/tmp}/bst-zchunk-flat.$$ tr -c '\40-\176\n' ' ' < "$lib" > "$flat" 2>/dev/null toks="$NL$(grep -oE '[A-Za-z_][A-Za-z0-9_]*\.c\.o|zck_[A-Za-z0-9_]*' "$flat" | sort -u)$NL" rm -f "$flat" # Members sampled across every subdirectory src/lib/meson.build pulls in. meson # mangles the source path into the object name (comp/zstd/zstd.c -> # comp_zstd_zstd.c.o), so these names also witness WHICH backend was compiled in # rather than only that the file list was complete. # # comp_zstd_zstd.c.o and hash_openssl_openssl.c.o are the load-bearing pair: # they are the two PACKAGECONFIG-selected backends, and their presence is what # proves -Dwith-zstd and -Dwith-openssl actually reached the build instead of # quietly resolving to the 'auto' no-op. A zchunk with neither still links and # still runs -- and cannot compress or checksum anything. for m in zck.c.o header.c.o io.c.o compint.c.o error.c.o log.c.o \ buzhash_buzhash.c.o \ comp_comp.c.o comp_nocomp_nocomp.c.o comp_zstd_zstd.c.o \ hash_hash.c.o hash_openssl_openssl.c.o \ index_index_common.c.o index_index_create.c.o index_index_read.c.o \ dl_dl.c.o dl_range.c.o dl_multipart.c.o; do if _have "$m"; then bst_ok "zchunk/member-$m"; else bst_notok "zchunk/member-$m" "absent from libzck.a"; fi done # Exported symbols, so the archive's index is evidence the wasm32 compile really # emitted each entry point rather than only that a header declares it. for s in zck_create zck_free zck_init_read zck_init_write zck_close \ zck_get_chunk_count zck_get_chunk_digest zck_validate_checksums \ zck_find_matching_chunks zck_hash_name_from_type zck_get_error; do if _have "$s"; then bst_ok "zchunk/sym-$s"; else bst_notok "zchunk/sym-$s" "absent from libzck.a"; fi done # THE DOWNLOAD API IS IN THE LIBRARY EVEN THOUGH zckdl IS NOT INSTALLED, and # that split is deliberate rather than a packaging accident. src/lib/dl holds # range/multipart bookkeeping over byte ranges and includes no transport at all; # only the zckdl front end links libcurl. So a consumer can still build ranged # fetching against libzck.a with its own transport, and these symbols are what # say so. If a future curl port re-enables zckdl, this block keeps passing -- # it is the binary's absence below that would flip. for s in zck_dl_init zck_dl_free zck_dl_get_range zck_dl_set_range \ zck_dl_get_bytes_downloaded; do if _have "$s"; then bst_ok "zchunk/sym-$s"; else bst_notok "zchunk/sym-$s" "absent from libzck.a"; fi done # Static-only is distro policy and meson does not read the blanket # EXTRA_OECONF --disable-shared, so the recipe restates it as # -Ddefault_library=static. A .so here would mean that override was dropped. check zchunk/no-shared sh -c '! ls /usr/lib/libzck.so* >/dev/null 2>&1' check zchunk/no-libtool sh -c '! ls /usr/lib/libzck.la >/dev/null 2>&1' # --------------------------------------------------------------------------- # Runtime register: the format executing on wasm32, through the shipped tools. # --------------------------------------------------------------------------- require zck zchunk/cli-present # Five of upstream's six tools install here. Named individually so a packaging # change that drops one is a red against that name rather than a vague miss. check zchunk/cli-unzck test -x /usr/bin/unzck check zchunk/cli-zck-read-header test -x /usr/bin/zck_read_header check zchunk/cli-zck-delta-size test -x /usr/bin/zck_delta_size check zchunk/cli-zck-gen-zdict test -x /usr/bin/zck_gen_zdict # THE SIXTH IS DELIBERATELY ABSENT. zckdl is the only tool that links libcurl, # curl does not build on this arm yet, and the recipe drops the zckdl # PACKAGECONFIG; upstream's `disabler: true` on the libcurl dependency then makes # the executable() evaluate away on its own. Asserted so that re-enabling curl # without revisiting the recipe is a red here rather than a silent change -- # and note it would still have nothing to talk to, outbound TCP being NaaS-gated. check zchunk/no-zckdl sh -c '! command -v zckdl >/dev/null 2>&1' check_out zchunk/version "zchunk 1.4.0" zck -V check_out zchunk/copyright "Jonathan Dieter" zck -V check_out zchunk/help "Create a new zchunk file" sh -c 'zck --help 2>&1' d=${TMPDIR:-/tmp}/bst-zchunk.$$ mkdir -p "$d" # Two vectors, both generated from tools already on the image so the suite stays # a single file with no payload to keep in sync. The small one carries the # framing and round-trip assertions; seq gives the chunker something with real # internal structure to index and delta against. printf 'brintOS zchunk round-trip vector 0123456789\n' > "$d/small.txt" seq 1 500 > "$d/a.txt" (seq 1 400; seq 460 560) > "$d/b.txt" # a.zck is reused three ways below -- as the delta baseline, as the zstd-default # size against which --compression-format none is compared, and as the donor for # the corruption vector -- so it is compressed here and asserted, rather than # left to a silent redirect whose failure would surface as an unrelated red. check zchunk/compress-a test -s "$d/a.txt" zck -o "$d/a.zck" "$d/a.txt" 2>/dev/null zck -o "$d/b.zck" "$d/b.txt" 2>/dev/null check zchunk/compress-a-zck test -s "$d/a.zck" check zchunk/compress-b-zck test -s "$d/b.zck" check zchunk/compress-file zck -o "$d/small.zck" "$d/small.txt" check zchunk/compressed-exists test -s "$d/small.zck" # The zchunk lead: a NUL followed by "ZCK1". This is what makes the output a # self-describing container rather than a bare zstd frame, and it is the byte # sequence header.c checks before reading anything else. check_out zchunk/lead-magic "00 5a 43 4b 31" sh -c "od -An -tx1 -N5 '$d/small.zck' | tr -s ' '" # ---- the header the format exists for ------------------------------------- check_out zchunk/header-reads "zchunk file" zck_read_header "$d/small.zck" check_out zchunk/header-overall "Overall checksum type: SHA-256" zck_read_header "$d/small.zck" check_out zchunk/header-chunk-hash "Chunk checksum type: SHA-512/128" zck_read_header "$d/small.zck" check_out zchunk/header-size "Header size:" zck_read_header "$d/small.zck" check_out zchunk/header-datasize "Data size:" zck_read_header "$d/small.zck" check_out zchunk/header-nodict "No dictionary" zck_read_header "$d/small.zck" # -c lists the per-chunk index; -f verifies every chunk against it. Together # they are the closest thing the CLI has to "this file is internally consistent", # and the exit status is the machine-readable half of that answer. check zchunk/verify-good zck_read_header -c -f "$d/small.zck" check_out zchunk/chunk-table "Chunk Checksum" sh -c "zck_read_header -c '$d/small.zck'" # ---- round trip ----------------------------------------------------------- # unzck writes to basename-minus-.zck in the CURRENT directory and has no -o, so # the file-output path runs in a subshell that cd's first. Both routes are worth # having: -c is the pipeline path, the file route is what a user types. unzck -c "$d/small.zck" > "$d/small.stdout" 2>/dev/null check zchunk/roundtrip-stdout cmp -s "$d/small.txt" "$d/small.stdout" (cd "$d" && cp small.zck rt.zck && unzck rt.zck >/dev/null 2>&1) check zchunk/roundtrip-file test -f "$d/rt" check zchunk/roundtrip-identical cmp -s "$d/small.txt" "$d/rt" # ---- chunking is real ------------------------------------------------------ # -s splits at every occurrence of a separator, so the chunk count must respond # to the content. Compare against the same input chunked automatically: if both # came back equal, the index would be decorative and every delta claim above it # meaningless. Measured on this vector: 2 automatic, 4 split. printf 'AAAA\nSPLITME\nBBBB\nSPLITME\nCCCC\n' > "$d/sp.txt" zck -o "$d/sp0.zck" "$d/sp.txt" 2>/dev/null zck -s SPLITME -o "$d/sp1.zck" "$d/sp.txt" 2>/dev/null n0=$(zck_read_header "$d/sp0.zck" 2>/dev/null | awk '/^Chunk count:/{print $3}') n1=$(zck_read_header "$d/sp1.zck" 2>/dev/null | awk '/^Chunk count:/{print $3}') check_eq zchunk/chunks-auto 2 "$n0" check_eq zchunk/chunks-split 4 "$n1" check zchunk/chunks-respond-to-content test "$n1" -gt "$n0" # Both chunkings must still decode to the original -- a splitter that produced # more chunks by losing data would satisfy the count assertions above. unzck -c "$d/sp1.zck" > "$d/sp1.out" 2>/dev/null check zchunk/split-roundtrip cmp -s "$d/sp.txt" "$d/sp1.out" # ---- the delta claim ------------------------------------------------------- # zck_delta_size reports what a client holding a.zck would have to fetch to # reconstruct b.zck. The two inputs share their first 400 lines, so a partial # match is the correct answer; "Matched 0" would mean the chunk index is not # doing the one job it exists for. check_out zchunk/delta-reports "Would download in total" zck_delta_size "$d/a.zck" "$d/b.zck" check_out zchunk/delta-matched "Matched" zck_delta_size "$d/a.zck" "$d/b.zck" # ---- compression backends -------------------------------------------------- # --compression-format none stores chunks uncompressed. Asserting the zstd # default is SMALLER than the none form is what proves the zstd backend is # actually compressing rather than being selected and no-op'd. zck --compression-format none -o "$d/none.zck" "$d/a.txt" 2>/dev/null check zchunk/comp-none-exists test -s "$d/none.zck" unzck -c "$d/none.zck" > "$d/none.out" 2>/dev/null check zchunk/comp-none-roundtrip cmp -s "$d/a.txt" "$d/none.out" szs=$(wc -c < "$d/a.zck") szn=$(wc -c < "$d/none.zck") check zchunk/zstd-smaller-than-none test "$szs" -lt "$szn" check zchunk/zstd-smaller-than-input test "$szs" -lt "$(wc -c < "$d/a.txt")" # A non-default chunk hash, proving the tier is selectable and recorded in the # header rather than hardcoded. zck -h sha512 -o "$d/sha512.zck" "$d/small.txt" 2>/dev/null check_out zchunk/hash-sha512 "Chunk checksum type: SHA-512" zck_read_header "$d/sha512.zck" unzck -c "$d/sha512.zck" > "$d/sha512.out" 2>/dev/null check zchunk/hash-sha512-roundtrip cmp -s "$d/small.txt" "$d/sha512.out" # ---- corruption must be CAUGHT --------------------------------------------- # This is the assertion the whole per-chunk index buys, and it is what separates # verifying from echoing. Measured on 1.4.0: zck_read_header -c -f exits 2 and # prints "Some checksums failed", and unzck refuses with "Data checksum failed # verification" rather than emitting the damaged bytes. cp "$d/a.zck" "$d/corrupt.zck" printf 'ZZZZZZZZ' | dd of="$d/corrupt.zck" bs=1 seek=200 conv=notrunc 2>/dev/null check_rc zchunk/corrupt-rc 2 zck_read_header -c -f "$d/corrupt.zck" check_out zchunk/corrupt-detected "Some checksums failed" sh -c "zck_read_header -c -f '$d/corrupt.zck' 2>&1 || true" check_rc zchunk/corrupt-unzck-rc 1 sh -c "unzck -c '$d/corrupt.zck' >/dev/null 2>&1" check_out zchunk/corrupt-unzck-msg "checksum failed" sh -c "unzck -c '$d/corrupt.zck' 2>&1 >/dev/null || true" # ---- error paths ----------------------------------------------------------- # Neither must be mistaken for success, and the lead check must reject a file # that is simply not a zchunk container. check_rc zchunk/err-not-zchunk-rc 1 sh -c "unzck -c '$d/small.txt' >/dev/null 2>&1" check_out zchunk/err-not-zchunk-msg "not a zck file" sh -c "unzck -c '$d/small.txt' 2>&1 >/dev/null || true" check_rc zchunk/err-missing-rc 1 sh -c "unzck -c '$d/nosuch.zck' >/dev/null 2>&1" check_rc zchunk/err-header-missing-rc 1 sh -c "zck_read_header '$d/nosuch.zck' >/dev/null 2>&1" rm -rf "$d"