. "$BST_LIB" # libstemmer is Snowball 2.2.0: 29 stemming algorithms behind FIVE entry points, # plus two real executables -- stemwords(1), upstream's CLI linked against the # very archive that ships, and snowball(1), the Snowball-to-C compiler. # # --------------------------------------------------------------------------- # WHY THIS SUITE RUNS A PROBE INSTEAD OF STOPPING AT PAYLOAD SHAPE # # BRINTOS-217 is why: oniguruma COMPILED and PACKAGED perfectly on this arm and # then returned silently wrong offsets to every caller, so a payload-only suite # would have gone green on the exact defect the port existed to find. # # libstemmer is the first library on this Epic whose subject is THE HEAP. Where # http-parser allocated nothing and kept no state outside the caller's struct, # every stemmer here is a malloc'd graph that lives across calls: # # run-new-* run-multi-* sb_stemmer_new mallocs a struct holding THREE function # pointers (create/close/stem) and calls the first # immediately; every later stem is an indirect call # through the second -- one dispatch table per # algorithm, and run-multi-* holds all 29 live at once. # run-stem-* inside, find_among() calls w->function(z) out of a # `const struct among` table, per token per rule. That is # a SECOND, deeper indirect level than any sibling port # has exercised, and the corpus drives it in 29 # languages and three character encodings. # run-grow-* the runtime keeps a block's size and capacity in two # ints placed BEFORE the pointer it hands out -- # ((int*)p)[-1] and ((int*)p)[-2], payload at malloc'd + # HEAD. increase_size() reallocs and re-lays that out. # Negative-offset addressing into a malloc'd block is # exactly the arithmetic a new arm gets wrong, so the # probe grows one buffer to 60 000 bytes and back down. # run-churn-* 5000 stems on one live stemmer, 500 grow/shrink # alternations, and 2000 create/destroy cycles, because # a leak or a double-free shows up as a wrong answer or # a trap rather than as a message. # run-utf8-* run-enc-* skip_utf8 / skip_b_utf8 walk characters forwards AND # backwards over multi-byte input; the identical # algorithms are also compiled for ISO_8859_1 and # KOI8_R, where a byte IS a character. Both ship. # # AND THIS PORT HAS SOMETHING NO SIBLING LIBRARY PORT ON THIS EPIC HAD: A REAL # ON-TARGET CONSUMER. cli-* drives stemwords(1), which links the shipped archive # and reaches it across a process boundary -- so the library is proven through # the image's own binary, not only through a probe this suite brought with it. # # EVERY EXPECTED VALUE IS THE VALUE A NATIVE x86_64 BUILD OF THE IDENTICAL # RELEASE PRODUCES. The probe compares each value internally and prints it on # both the OK and the FAIL path, so a target run can be diffed against the native # control line by line rather than compared by tally -- which # native-byte-identical below does, on target, with cmp(1). # # THE SUITE FORKS AS LITTLE AS IT CAN. On this runtime a multi-MB wasm module's # start-up dominates, and the cjson suite next door was killed by the runner's # per-suite bound at 156 cases written as one fork per case. So stemwords is run # SEVEN times in total with batched input, its output read once into a variable, # and every case below is a shell builtin `case` match against it. require stemwords libstemmer/present # has -- builtin substring match, no fork has() { case "$3" in *"$2"*) bst_ok "$1" ;; *) bst_notok "$1" "missing: $2" ;; esac } # hasnt hasnt() { case "$3" in *"$2"*) bst_notok "$1" "unexpectedly present: $2" ;; *) bst_ok "$1" ;; esac } hdr=/usr/include/libstemmer.h arc=/usr/lib/libstemmer.a # --- payload ----------------------------------------------------------------- check libstemmer/payload-header test -s "$hdr" check libstemmer/payload-archive test -s "$arc" check libstemmer/payload-stemwords test -x /usr/bin/stemwords check libstemmer/payload-snowball test -x /usr/bin/snowball # The whole point of the bbappend: meta-oe's recipe installs ONLY a shared object # and this arm can link none, so a .so appearing here means the stock recipe # somehow ran and the port is not what it claims to be. if [ -e /usr/lib/libstemmer.so ] || [ -e /usr/lib/libstemmer.so.0 ] || \ [ -e /usr/lib/libstemmer.so.0.0.0 ]; then bst_notok libstemmer/no-shared-object "a libstemmer .so is present on a static-only image" else bst_ok libstemmer/no-shared-object fi # --- header ABI -------------------------------------------------------------- # Every runtime value below depends on these, so a header change should fail # loudly here rather than shift every result underneath it. hdrtext=$(cat "$hdr" 2>/dev/null) has libstemmer/hdr-list "sb_stemmer_list(void)" "$hdrtext" has libstemmer/hdr-new "sb_stemmer_new(const char * algorithm" "$hdrtext" has libstemmer/hdr-delete "sb_stemmer_delete(struct sb_stemmer" "$hdrtext" has libstemmer/hdr-stem "sb_stemmer_stem(struct sb_stemmer" "$hdrtext" has libstemmer/hdr-length "sb_stemmer_length(struct sb_stemmer" "$hdrtext" has libstemmer/hdr-symbol-t "typedef unsigned char sb_symbol" "$hdrtext" has libstemmer/hdr-opaque "struct sb_stemmer;" "$hdrtext" has libstemmer/hdr-cplusplus "extern \"C\"" "$hdrtext" # sb_stemmer_stem returns a pointer INTO the stemmer's own buffer, and the # header says so; a caller that freed it would corrupt the heap. has libstemmer/hdr-stem-const "const sb_symbol * sb_stemmer_stem" "$hdrtext" has libstemmer/hdr-list-const "const char ** sb_stemmer_list" "$hdrtext" # --- symbol completeness ----------------------------------------------------- # Derived ON TARGET from the installed header rather than written into the test, # so it survives a version bump; the count is pinned and each name is also # spot-checked, so an empty derivation cannot make the loop vacuously green. hdrsyms=$(grep -o 'sb_stemmer_[a-z_]*' "$hdr" 2>/dev/null | sort -u) arcsyms=$(grep -ao 'sb_stemmer_[a-z_]*' "$arc" 2>/dev/null | sort -u) nsym=0; nmiss=0; missing= for s in $hdrsyms; do nsym=$((nsym + 1)) case "$arcsyms" in *"$s"*) ;; *) nmiss=$((nmiss + 1)); missing="$missing $s" ;; esac done if [ "$nmiss" -eq 0 ] && [ "$nsym" -gt 0 ]; then bst_ok libstemmer/symbols-complete else bst_notok libstemmer/symbols-complete "$nmiss missing:$missing" fi check_eq libstemmer/symbols-count 5 "$nsym" has libstemmer/sym-list "sb_stemmer_list" "$arcsyms" has libstemmer/sym-new "sb_stemmer_new" "$arcsyms" has libstemmer/sym-delete "sb_stemmer_delete" "$arcsyms" has libstemmer/sym-stem "sb_stemmer_stem" "$arcsyms" has libstemmer/sym-length "sb_stemmer_length" "$arcsyms" # --- stemwords(1): the library reached through the image's OWN binary --------- # ONE fork, a batch of words, output read once. -p prints "word -> stem", which # is unambiguous to substring-match (the plain format prints stems only, so a # wrong pairing would be invisible). cli=$(printf '%s\n' running runs ran hopping hoping tanned falling hissing \ failing filing motoring sing plastered bled agreed feed happy happiness \ sky flies dies cries generalization oscillators national relational \ conditional rational digitizer differently radically analogously \ predication operator feudalism decisiveness hopefulness callousness \ sensitivity sensibility consolingly activate creative news proceed \ herring dying lying tying idly gently ugly early only singly \ | stemwords -l english -p 2>&1) has libstemmer/cli-running "running -> run" "$cli" has libstemmer/cli-runs "runs -> run" "$cli" has libstemmer/cli-ran "ran -> ran" "$cli" has libstemmer/cli-hopping "hopping -> hop" "$cli" has libstemmer/cli-hoping "hoping -> hope" "$cli" has libstemmer/cli-tanned "tanned -> tan" "$cli" has libstemmer/cli-falling "falling -> fall" "$cli" has libstemmer/cli-hissing "hissing -> hiss" "$cli" has libstemmer/cli-failing "failing -> fail" "$cli" has libstemmer/cli-filing "filing -> file" "$cli" has libstemmer/cli-motoring "motoring -> motor" "$cli" has libstemmer/cli-sing "sing -> sing" "$cli" has libstemmer/cli-plastered "plastered -> plaster" "$cli" has libstemmer/cli-bled "bled -> bled" "$cli" has libstemmer/cli-agreed "agreed -> agre" "$cli" has libstemmer/cli-feed "feed -> feed" "$cli" has libstemmer/cli-happy "happy -> happi" "$cli" has libstemmer/cli-happiness "happiness -> happi" "$cli" has libstemmer/cli-sky "sky -> sky" "$cli" has libstemmer/cli-flies "flies -> fli" "$cli" has libstemmer/cli-dies "dies -> die" "$cli" has libstemmer/cli-cries "cries -> cri" "$cli" has libstemmer/cli-generalization "generalization -> general" "$cli" has libstemmer/cli-oscillators "oscillators -> oscil" "$cli" has libstemmer/cli-national "national -> nation" "$cli" has libstemmer/cli-relational "relational -> relat" "$cli" has libstemmer/cli-conditional "conditional -> condit" "$cli" has libstemmer/cli-rational "rational -> ration" "$cli" has libstemmer/cli-digitizer "digitizer -> digit" "$cli" has libstemmer/cli-differently "differently -> differ" "$cli" has libstemmer/cli-radically "radically -> radic" "$cli" has libstemmer/cli-analogously "analogously -> analog" "$cli" has libstemmer/cli-predication "predication -> predic" "$cli" has libstemmer/cli-operator "operator -> oper" "$cli" has libstemmer/cli-feudalism "feudalism -> feudal" "$cli" has libstemmer/cli-decisiveness "decisiveness -> decis" "$cli" has libstemmer/cli-hopefulness "hopefulness -> hope" "$cli" has libstemmer/cli-callousness "callousness -> callous" "$cli" has libstemmer/cli-sensitivity "sensitivity -> sensit" "$cli" has libstemmer/cli-sensibility "sensibility -> sensibl" "$cli" has libstemmer/cli-consolingly "consolingly -> consol" "$cli" has libstemmer/cli-activate "activate -> activ" "$cli" has libstemmer/cli-creative "creative -> creativ" "$cli" # the exception lists, which are table lookups rather than rules has libstemmer/cli-news "news -> news" "$cli" has libstemmer/cli-proceed "proceed -> proceed" "$cli" has libstemmer/cli-herring "herring -> herring" "$cli" has libstemmer/cli-dying "dying -> die" "$cli" has libstemmer/cli-lying "lying -> lie" "$cli" has libstemmer/cli-tying "tying -> tie" "$cli" has libstemmer/cli-idly "idly -> idl" "$cli" has libstemmer/cli-gently "gently -> gentl" "$cli" has libstemmer/cli-ugly "ugly -> ugli" "$cli" has libstemmer/cli-early "early -> earli" "$cli" has libstemmer/cli-only "only -> onli" "$cli" has libstemmer/cli-singly "singly -> singl" "$cli" # the plain (default) format prints stems only, and -l defaults to english cli2=$(printf 'generalization\nhappiness\n' | stemwords 2>&1) has libstemmer/cli-plain-general "general" "$cli2" has libstemmer/cli-plain-happi "happi" "$cli2" hasnt libstemmer/cli-plain-no-arrow "->" "$cli2" # -p2 is a two-column layout: the word, padded, then the stem cli3=$(printf 'flies\n' | stemwords -l english -p2 2>&1) has libstemmer/cli-p2-word "flies " "$cli3" has libstemmer/cli-p2-stem "fli" "$cli3" # file in / file out, which is a different path from the stdin/stdout one tmp=${TMPDIR:-/tmp}/bst-stem.$$ printf 'generalization\noscillators\n' > "$tmp.in" stemwords -l english -i "$tmp.in" -o "$tmp.out" 2>/dev/null cli4=$(cat "$tmp.out" 2>&1) has libstemmer/cli-file-general "general" "$cli4" has libstemmer/cli-file-oscil "oscil" "$cli4" rm -f "$tmp.in" "$tmp.out" # a UTF-8 language through the CLI: the stemmer walks multi-byte characters cli5=$(printf '\320\272\321\200\320\260\321\201\320\270\320\262\320\260\321\217\n' \ | stemwords -l russian -c UTF_8 2>&1) has libstemmer/cli-russian-utf8 \ "$(printf '\320\272\321\200\320\260\321\201\320\270\320\262')" "$cli5" # the same algorithm compiled for a SINGLE-BYTE encoding, driven through the CLI cli6=$(printf 'g\351n\351ralement\n' | stemwords -l french -c ISO_8859_1 2>&1) has libstemmer/cli-french-latin1 "$(printf 'g\351n\351ral')" "$cli6" # An unknown language is diagnosed on stderr and exits 1 (stemwords.c:191), so # the CLI's error path is asserted on BOTH the message and the status -- the # status because a caller scripting stemwords keys on it, and the message # because exit(1) alone would not distinguish this from a write failure. cli7=$(printf 'x\n' | stemwords -l klingon 2>&1); clirc=$? has libstemmer/cli-bad-lang-msg "not available for stemming" "$cli7" check_eq libstemmer/cli-bad-lang-rc 1 "$clirc" # --- snowball(1): a parser and code generator, a third code path ------------- sbl=$BST_DATA/stemmer-tiny.sbl outb=${TMPDIR:-/tmp}/bst-sn.$$ if [ -f "$sbl" ]; then sn=$(snowball "$sbl" -o "$outb" -n tiny -u 2>&1); snrc=$? check_eq libstemmer/sn-rc 0 "$snrc" check libstemmer/sn-emits-c test -s "$outb.c" check libstemmer/sn-emits-h test -s "$outb.h" gen=$(cat "$outb.c" "$outb.h" 2>/dev/null) has libstemmer/sn-banner "Generated by Snowball 2.2.0" "$gen" has libstemmer/sn-create "create_env(void)" "$gen" has libstemmer/sn-stem "int stem(struct SN_env" "$gen" rm -f "$outb.c" "$outb.h" # -syntax parses and dumps the tree WITHOUT generating code sn2=$(snowball "$sbl" -syntax 2>&1) has libstemmer/sn-syntax "define stem" "$sn2" else bst_skip libstemmer/sn-rc "payload stemmer-tiny.sbl absent" bst_skip libstemmer/sn-emits-c "payload stemmer-tiny.sbl absent" bst_skip libstemmer/sn-emits-h "payload stemmer-tiny.sbl absent" bst_skip libstemmer/sn-banner "payload stemmer-tiny.sbl absent" bst_skip libstemmer/sn-create "payload stemmer-tiny.sbl absent" bst_skip libstemmer/sn-stem "payload stemmer-tiny.sbl absent" bst_skip libstemmer/sn-syntax "payload stemmer-tiny.sbl absent" fi # --- the library RUNNING, via the cross-linked probe ------------------------- # data/stemmer-probe is src/stemmer_probe.c cross-linked with wasmcc-glibc # against THE VERY ARCHIVE THAT SHIPS (/usr/lib/libstemmer.a), staged to $TMPDIR # and run here. It is run ONCE and its exit status kept, rather than re-run for # the rc case: on this runtime a multi-MB wasm module's start-up dominates. probe=${TMPDIR:-/tmp}/bst-stemmer-probe.$$ if cp "$BST_DATA/stemmer-probe" "$probe" 2>/dev/null && chmod +x "$probe" 2>/dev/null; then out=$("$probe" 2>&1); prc=$? # sb_stemmer_list: a static table of string pointers, read through an index has libstemmer/run-list-nonnull "SB-list-nonnull OK" "$out" has libstemmer/run-list-count "SB-list-count OK" "$out" has libstemmer/run-list-first "SB-list-first OK" "$out" has libstemmer/run-list-last "SB-list-last OK" "$out" has libstemmer/run-list-terminated "SB-list-terminated OK" "$out" has libstemmer/run-list-english "SB-list-english OK" "$out" has libstemmer/run-list-porter "SB-list-porter OK" "$out" has libstemmer/run-list-russian "SB-list-russian OK" "$out" has libstemmer/run-list-stable "SB-list-stable OK" "$out" # construction: the malloc + the create() indirect call, and every rejection has libstemmer/run-new-default-enc "SB-new-default-enc OK" "$out" has libstemmer/run-new-utf8 "SB-new-utf8 OK" "$out" has libstemmer/run-new-latin1 "SB-new-latin1 OK" "$out" has libstemmer/run-new-koi8r "SB-new-koi8r OK" "$out" has libstemmer/run-new-bad-algo "SB-new-bad-algo OK" "$out" has libstemmer/run-new-case-significant "SB-new-case-significant OK" "$out" has libstemmer/run-new-bad-enc-spelling "SB-new-bad-enc-spelling OK" "$out" has libstemmer/run-new-enc-not-available "SB-new-enc-not-available OK" "$out" has libstemmer/run-new-russian-latin1 "SB-new-russian-latin1 OK" "$out" has libstemmer/run-new-empty-algo "SB-new-empty-algo OK" "$out" has libstemmer/run-new-all-listed "SB-new-all-listed OK" "$out" has libstemmer/run-delete-null-safe "SB-delete-null-safe OK" "$out" # the English (Porter2) corpus: 75 stems and 75 lengths i=0 while [ $i -lt 75 ]; do n=$(printf '%02d' $i) has "libstemmer/run-stem-en-$n" "SB-stem-en-$n OK" "$out" has "libstemmer/run-len-en-$n" "SB-len-en-$n OK" "$out" i=$((i + 1)) done # one word per remaining algorithm, UTF-8 -- 27 stems, 27 lengths for l in arabic armenian basque catalan danish dutch finnish french german \ greek hindi hungarian indonesian irish italian lithuanian nepali \ norwegian portuguese romanian russian serbian spanish swedish tamil \ turkish yiddish; do has "libstemmer/run-stem-utf8-$l" "SB-stem-utf8-$l OK" "$out" has "libstemmer/run-len-utf8-$l" "SB-len-utf8-$l OK" "$out" done # and every one of them must actually have CHANGED its input, so a substrate # that quietly returned the word untouched cannot pass the 27 rows above has libstemmer/run-stem-utf8-all-changed "SB-stem-utf8-all-changed OK" "$out" # the single-byte encodings, where a byte IS a character i=0 while [ $i -lt 8 ]; do n=$(printf '%02d' $i) has "libstemmer/run-stem-enc-$n" "SB-stem-enc-$n OK" "$out" i=$((i + 1)) done has libstemmer/run-stem-enc-all-changed "SB-stem-enc-all-changed OK" "$out" # input edges: size (not NUL) delimits, and the buffer is reused across calls has libstemmer/run-edge-empty "SB-edge-empty OK" "$out" has libstemmer/run-edge-empty-len "SB-edge-empty-len OK" "$out" has libstemmer/run-edge-one-char "SB-edge-one-char OK" "$out" has libstemmer/run-edge-size-delimits "SB-edge-size-delimits OK" "$out" has libstemmer/run-edge-reuse-1 "SB-edge-reuse-1 OK" "$out" has libstemmer/run-edge-reuse-2 "SB-edge-reuse-2 OK" "$out" has libstemmer/run-edge-reuse-len "SB-edge-reuse-len OK" "$out" has libstemmer/run-edge-punct "SB-edge-punct OK" "$out" has libstemmer/run-edge-embedded-nul-len "SB-edge-embedded-nul-len OK" "$out" # THE HEAP GROUP. ((int*)p)[-1] / [-2] under repeated realloc. has libstemmer/run-grow-60k-len "SB-grow-60k-len OK" "$out" has libstemmer/run-grow-60k-intact "SB-grow-60k-intact OK" "$out" has libstemmer/run-grow-then-short "SB-grow-then-short OK" "$out" has libstemmer/run-grow-then-short-len "SB-grow-then-short-len OK" "$out" has libstemmer/run-grow-again-len "SB-grow-again-len OK" "$out" has libstemmer/run-grow-suffix-len "SB-grow-suffix-len OK" "$out" has libstemmer/run-grow-suffix-head "SB-grow-suffix-head OK" "$out" has libstemmer/run-churn-5000-stable "SB-churn-5000-stable OK" "$out" has libstemmer/run-churn-5000-value "SB-churn-5000-value OK" "$out" has libstemmer/run-churn-alternating "SB-churn-alternating OK" "$out" has libstemmer/run-churn-alternating-ok "SB-churn-alternating-ok OK" "$out" has libstemmer/run-churn-newdelete-2000 "SB-churn-newdelete-2000 OK" "$out" # 29 dispatch tables live at once has libstemmer/run-multi-alive "SB-multi-alive OK" "$out" has libstemmer/run-multi-all-answer "SB-multi-all-answer OK" "$out" has libstemmer/run-multi-english-isolated "SB-multi-english-isolated OK" "$out" has libstemmer/run-multi-deleted "SB-multi-deleted OK" "$out" has libstemmer/run-fixpoint-english "SB-fixpoint-english OK" "$out" # the arm banner, held OUT of the byte comparison below has libstemmer/run-probe-banner "SB-PROBE libstemmer probe, 32-bit pointers" "$out" # THE LOAD-BEARING ONE. data/stemmer-native.txt is the verbatim output of this # same probe source built NATIVELY against a host build of the identical # release, with the SB-PROBE banner removed. Comparing it inside the guest with # cmp(1) is deliberate and is the accumulated lesson of two sibling tickets: # the console dump truncates long lines (BRINTOS-260) and the expect-backed # host SSH is pty-based and rewrites LF to CRLF (BRINTOS-263). ctl=$BST_DATA/stemmer-native.txt gotf=${TMPDIR:-/tmp}/bst-stemmer-got.$$ printf '%s\n' "$out" | grep -v '^SB-PROBE ' > "$gotf" 2>/dev/null if [ ! -s "$ctl" ]; then bst_notok libstemmer/native-byte-identical "control absent or empty: $ctl" elif cmp -s "$ctl" "$gotf"; then bst_ok libstemmer/native-byte-identical else bst_notok libstemmer/native-byte-identical \ "differs from the native control: $(_bst_1line "$(cmp "$ctl" "$gotf" 2>&1)")" fi rm -f "$gotf" has libstemmer/run-summary "SB-SUMMARY 261/261" "$out" check_eq libstemmer/run-rc 0 "$prc" rm -f "$probe" else bst_skip libstemmer/run-probe "data/stemmer-probe absent from the image" fi