# BST-TIMEOUT: 300   # measured 63s on target (one full accounting pass over a
#                    # 1.9 MB UnicodeData.txt, 34,626 records, plus four companion
#                    # tables and a perl cross-check). The default of 60 is not close.
. "$BST_LIB"

# unicode-ucd is data only -- the Unicode Character Database 14.0.0, the
# normative tables, at /usr/share/unicode/ucd. There is no tool to `require`:
# what ships is the TABLES, so this suite PARSES them by their own grammars
# rather than stat-ing them.
#
# Every count asserted below was derived from the artifact THIS image built --
# a state-aware accounting pass that classifies each line of each table into a
# named production and reports the residual. The residual is zero on all five
# tables, which is what makes these numbers a measurement of the data rather
# than of the regex: a line the grammar does not recognise is not silently
# skipped, it is counted.
#
# The load-bearing cases are the ones a truncated payload survives: `test -s`
# passes on a table with 500 lines cut out of its middle, and only the exact
# counts and the ascending-order check fail.

UCD=/usr/share/unicode/ucd
UD=$UCD/UnicodeData.txt

if [ ! -d "$UCD" ]; then
  bst_skip unicode-ucd/present "no $UCD on this image"
  exit 0
fi
bst_ok unicode-ucd/present

# --- the tables arrived ------------------------------------------------------
check unicode-ucd/file-unicodedata test -s "$UD"
check unicode-ucd/file-blocks      test -s "$UCD/Blocks.txt"
check unicode-ucd/file-scripts     test -s "$UCD/Scripts.txt"
check unicode-ucd/file-proplist    test -s "$UCD/PropList.txt"
check unicode-ucd/file-casefolding test -s "$UCD/CaseFolding.txt"
check unicode-ucd/file-readme      test -s "$UCD/ReadMe.txt"

# The package is one directory tree and nothing else installs into it, so the
# file count is an exact assertion rather than a floor -- a partially unpacked
# UCD.zip fails here and nowhere else.
check_eq unicode-ucd/file-count 69 "$(find "$UCD" -type f 2>/dev/null | wc -l | tr -d ' ')"
check_eq unicode-ucd/unicodedata-size 1897793 "$(wc -c < "$UD" 2>/dev/null | tr -d ' ')"

# The three subdirectories the zip carries. Their absence is the shape a
# `cp -rf` that lost its recursion would take.
check unicode-ucd/dir-auxiliary test -d "$UCD/auxiliary"
check unicode-ucd/dir-extracted test -d "$UCD/extracted"
check unicode-ucd/dir-emoji     test -d "$UCD/emoji"

# --- which release is installed ---------------------------------------------
# Pinned, not floored: the counts below are 14.0.0's, so a version bump must
# fail here first and be read, rather than silently invalidating forty
# assertions further down.
check_out unicode-ucd/version "Unicode14.0.0" grep -m1 Unicode14 "$UCD/ReadMe.txt"
check_out unicode-ucd/date    "2021-09-10"    grep -m1 "^# Date:" "$UCD/ReadMe.txt"

# --- UnicodeData.txt: the 15-field grammar, in one pass ----------------------
# Both files are ascending, so block membership is a merge with a pointer that
# only advances -- 34,626 + 320 steps rather than 34,626 x 320 comparisons,
# which matters on this target.
UDK=$(awk '
function hex(s,   i,c,v,d) {
  v = 0
  for (i = 1; i <= length(s); i++) {
    c = substr(s, i, 1)
    d = index("0123456789ABCDEF", c) - 1
    if (d < 0) return -1
    v = v * 16 + d
  }
  return v
}
BEGIN { FS = ";"; prev = -1; bi = 1; pend = "" }
FILENAME ~ /Blocks\.txt$/ {
  line = $0
  sub(/#.*$/, "", line)
  if (line ~ /^[ \t]*$/) next
  if (match(line, /^[0-9A-F]+\.\.[0-9A-F]+;/)) {
    split(line, p, /[.;]+/)
    nb++; bs[nb] = hex(p[1]); be[nb] = hex(p[2])
  }
  next
}
{
  n++
  if ($0 ~ /^[ \t]*$/) { blank++; next }
  if (NF != 15) { badf++; next }
  cp = hex($1)
  if (cp <= prev) nonasc++
  prev = cp
  if (!($3 in gcs)) { gcs[$3] = 1; ngc++ }
  if ($4 != "0") cccnz++
  if ($2 ~ /^<.*, First>$/) {
    if (pend != "") unpaired++
    nm = $2; sub(/^</, "", nm); sub(/, First>$/, "", nm)
    pend = nm; pstart = cp
  } else if ($2 ~ /^<.*, Last>$/) {
    nm = $2; sub(/^</, "", nm); sub(/, Last>$/, "", nm)
    if (pend != nm) { unpaired++ }
    else {
      pairs++; span = cp - pstart + 1; rangecp += span
      if (nm ~ /Surrogate/) surcp += span
      else if (nm ~ /Private Use/) puacp += span
    }
    pend = ""
  } else singles++
  while (bi <= nb && cp > be[bi]) bi++
  if (bi > nb || cp < bs[bi]) noblock++
}
END {
  if (pend != "") unpaired++
  printf "K lines %d\n",      n
  printf "K blank %d\n",      blank + 0
  printf "K badfields %d\n",  badf + 0
  printf "K nonasc %d\n",     nonasc + 0
  printf "K pairs %d\n",      pairs + 0
  printf "K unpaired %d\n",   unpaired + 0
  printf "K rangecp %d\n",    rangecp + 0
  printf "K surcp %d\n",      surcp + 0
  printf "K puacp %d\n",      puacp + 0
  printf "K singles %d\n",    singles + 0
  printf "K assigned %d\n",   singles + rangecp
  printf "K graphic %d\n",    singles + rangecp - surcp - puacp
  printf "K ngc %d\n",        ngc + 0
  printf "K cccnz %d\n",      cccnz + 0
  printf "K noblock %d\n",    noblock + 0
  printf "K residual %d\n",   n - blank - badf - (2 * pairs) - singles - unpaired
}' "$UCD/Blocks.txt" "$UD" 2>&1)

k() { printf '%s\n' "$UDK" | awk -v want="$1" '$1 == "K" && $2 == want { print $3; exit }'; }

check_eq unicode-ucd/ud-lines      34626 "$(k lines)"
check_eq unicode-ucd/ud-fields     0     "$(k badfields)"
check_eq unicode-ucd/ud-ascending  0     "$(k nonasc)"
check_eq unicode-ucd/ud-no-blank   0     "$(k blank)"
check_eq unicode-ucd/ud-residual   0     "$(k residual)"
check_eq unicode-ucd/ud-pairs      17    "$(k pairs)"
check_eq unicode-ucd/ud-pairs-balanced 0 "$(k unpaired)"
check_eq unicode-ucd/ud-range-codepoints 249686 "$(k rangecp)"
check_eq unicode-ucd/ud-singles    34592 "$(k singles)"
check_eq unicode-ucd/ud-assigned   284278 "$(k assigned)"
check_eq unicode-ucd/ud-categories 29    "$(k ngc)"
check_eq unicode-ucd/ud-ccc-nonzero 912  "$(k cccnz)"

# Every record falls inside a declared block. This is the one assertion that
# reads Blocks.txt and UnicodeData.txt AGAINST each other rather than each on
# its own terms.
check_eq unicode-ucd/blocks-cover-ud 0 "$(k noblock)"

# --- specific codepoints, by field ------------------------------------------
# A deliberate spread: ASCII letter and digit, a Latin-1 letter with a case
# mapping and a canonical decomposition, a combining mark with a non-zero
# class, an ideograph that exists only INSIDE a First/Last pair, and two
# astral-plane characters -- the last being what a 32-bit arm is most likely
# to get wrong.
fld() { grep -m1 "^$1;" "$UD" 2>/dev/null | cut -d';' -f"$2"; }

check_eq unicode-ucd/cp-0041-name   "LATIN CAPITAL LETTER A" "$(fld 0041 2)"
check_eq unicode-ucd/cp-0041-gc     Lu    "$(fld 0041 3)"
check_eq unicode-ucd/cp-0041-ccc    0     "$(fld 0041 4)"
check_eq unicode-ucd/cp-0041-lower  0061  "$(fld 0041 14)"
check_eq unicode-ucd/cp-0039-gc     Nd    "$(fld 0039 3)"
check_eq unicode-ucd/cp-0039-numeric 9    "$(fld 0039 9)"
check_eq unicode-ucd/cp-00C9-gc     Lu    "$(fld 00C9 3)"
check_eq unicode-ucd/cp-00C9-decomp "0045 0301" "$(fld 00C9 6)"
check_eq unicode-ucd/cp-00C9-lower  00E9  "$(fld 00C9 14)"
check_eq unicode-ucd/cp-0301-gc     Mn    "$(fld 0301 3)"
check_eq unicode-ucd/cp-0301-ccc    230   "$(fld 0301 4)"
check_eq unicode-ucd/cp-4E00-first  "<CJK Ideograph, First>" "$(fld 4E00 2)"
check_eq unicode-ucd/cp-9FFF-last   "<CJK Ideograph, Last>"  "$(fld 9FFF 2)"
check_eq unicode-ucd/cp-4E00-gc     Lo    "$(fld 4E00 3)"
check_eq unicode-ucd/cp-10400-gc    Lu    "$(fld 10400 3)"
check_eq unicode-ucd/cp-10400-lower 10428 "$(fld 10400 14)"
check_eq unicode-ucd/cp-1D11E-gc    So    "$(fld 1D11E 3)"

# U+5000 is a CJK ideograph with NO record of its own -- it exists only by
# virtue of sitting inside the 4E00..9FFF pair. A parser that treated the
# First/Last convention as ordinary records would find it here.
check_eq unicode-ucd/cp-5000-in-range 0 "$(grep -c "^5000;" "$UD" 2>/dev/null | tr -d ' ')"

# --- the companion tables, each under its own grammar ------------------------
acct() {
  awk -v pat="$2" '
  { tot++
    line = $0; sub(/#.*$/, "", line)
    if ($0 ~ /^[ \t]*$/) { blank++; next }
    if (line ~ /^[ \t]*$/) { com++; next }
    sub(/[ \t]+$/, "", line)
    if (line ~ pat) ent++; else bad++
  }
  END { printf "K total %d\nK comments %d\nK blank %d\nK entries %d\nK bad %d\nK residual %d\n",
        tot, com + 0, blank + 0, ent + 0, bad + 0, tot - com - blank - ent - bad }' "$1" 2>&1
}

RNG='^[0-9A-F]+(\.\.[0-9A-F]+)?[ \t]*;[ \t]*[^ \t]+$'

BLK=$(acct "$UCD/Blocks.txt" '^[0-9A-F]+\.\.[0-9A-F]+;[ \t]*.+$')
SCR=$(acct "$UCD/Scripts.txt" "$RNG")
PRP=$(acct "$UCD/PropList.txt" "$RNG")
CFD=$(acct "$UCD/CaseFolding.txt" '^[0-9A-F]+;[ \t]*[CFST];[ \t]*[0-9A-F ]+;$')

kk() { printf '%s\n' "$1" | awk -v want="$2" '$1 == "K" && $2 == want { print $3; exit }'; }

check_eq unicode-ucd/blocks-lines        356  "$(kk "$BLK" total)"
check_eq unicode-ucd/blocks-entries      320  "$(kk "$BLK" entries)"
check_eq unicode-ucd/blocks-residual     0    "$(kk "$BLK" residual)"
check_eq unicode-ucd/blocks-unparsed     0    "$(kk "$BLK" bad)"
check_eq unicode-ucd/scripts-lines       2991 "$(kk "$SCR" total)"
check_eq unicode-ucd/scripts-entries     2161 "$(kk "$SCR" entries)"
check_eq unicode-ucd/scripts-residual    0    "$(kk "$SCR" residual)"
check_eq unicode-ucd/scripts-unparsed    0    "$(kk "$SCR" bad)"
check_eq unicode-ucd/proplist-lines      1743 "$(kk "$PRP" total)"
check_eq unicode-ucd/proplist-entries    1563 "$(kk "$PRP" entries)"
check_eq unicode-ucd/proplist-residual   0    "$(kk "$PRP" residual)"
check_eq unicode-ucd/proplist-unparsed   0    "$(kk "$PRP" bad)"
check_eq unicode-ucd/casefolding-lines   1624 "$(kk "$CFD" total)"
check_eq unicode-ucd/casefolding-entries 1560 "$(kk "$CFD" entries)"
check_eq unicode-ucd/casefolding-residual 0   "$(kk "$CFD" residual)"
check_eq unicode-ucd/casefolding-unparsed 0   "$(kk "$CFD" bad)"

# Blocks are declared in ascending order and must not overlap -- the property
# the merge above depends on, asserted rather than assumed.
BLKORD=$(awk '
function hex(s,   i,c,v,d) { v=0
  for (i=1; i<=length(s); i++) { c=substr(s,i,1); d=index("0123456789ABCDEF",c)-1
    if (d<0) return -1; v=v*16+d }
  return v }
{ line=$0; sub(/#.*$/,"",line)
  if (line !~ /^[0-9A-F]+\.\.[0-9A-F]+;/) next
  split(line, p, /[.;]+/); s=hex(p[1]); e=hex(p[2])
  n++; cov += e - s + 1
  if (e < s) mal++
  if (n > 1 && s <= pe) ov++
  pe = e }
END { printf "K n %d\nK overlaps %d\nK malformed %d\nK coverage %d\n", n, ov+0, mal+0, cov }' \
  "$UCD/Blocks.txt" 2>&1)
check_eq unicode-ucd/blocks-ordered   0      "$(kk "$BLKORD" overlaps)"
check_eq unicode-ucd/blocks-wellformed 0     "$(kk "$BLKORD" malformed)"
check_eq unicode-ucd/blocks-coverage  288512 "$(kk "$BLKORD" coverage)"
check_out unicode-ucd/blocks-first "0000..007F; Basic Latin" \
  grep -m1 "Basic Latin" "$UCD/Blocks.txt"
check_out unicode-ucd/blocks-last "100000..10FFFF" \
  grep -m1 "Supplementary Private Use Area-B" "$UCD/Blocks.txt"

# Scripts.txt covers exactly the graphic-and-format characters: every assigned
# codepoint except the surrogates and the private-use areas. Two files that
# nothing links, agreeing to the character -- 144,762 both ways.
SCRCOV=$(awk '
function hex(s,   i,c,v,d) { gsub(/[ \t]/, "", s); v=0
  if (s == "") return -1
  for (i=1; i<=length(s); i++) { c=substr(s,i,1); d=index("0123456789ABCDEF",c)-1
    if (d<0) return -1; v=v*16+d }
  return v }
{ line=$0; sub(/#.*$/,"",line); sub(/[ \t]+$/,"",line)
  if (line !~ /^[0-9A-F]+([ \t]*\.\.[ \t]*[0-9A-F]+)?[ \t]*;/) next
  split(line, p, /\.\.|;/)
  s = hex(p[1]); e = (index(line, "..") > 0) ? hex(p[2]) : s
  if (s < 0 || e < 0 || e < s) { mal++; next }
  cov += e - s + 1; n++
  v = line; sub(/^[^;]*;[ \t]*/, "", v)
  if (!(v in seen)) { seen[v]=1; nv++ } }
END { printf "K n %d\nK coverage %d\nK distinct %d\nK malformed %d\n", n, cov, nv, mal+0 }' \
  "$UCD/Scripts.txt" 2>&1)
check_eq unicode-ucd/scripts-coverage  144762 "$(kk "$SCRCOV" coverage)"
check_eq unicode-ucd/scripts-distinct  161    "$(kk "$SCRCOV" distinct)"
check_eq unicode-ucd/scripts-wellformed 0     "$(kk "$SCRCOV" malformed)"
check_eq unicode-ucd/scripts-arithmetic "$(k graphic)" "$(kk "$SCRCOV" coverage)"

check_eq unicode-ucd/casefolding-status-c 1426 \
  "$(awk -F';' '$2 ~ /^ *C *$/ { n++ } END { print n+0 }' "$UCD/CaseFolding.txt")"
check_eq unicode-ucd/casefolding-status-f 104 \
  "$(awk -F';' '$2 ~ /^ *F *$/ { n++ } END { print n+0 }' "$UCD/CaseFolding.txt")"
check_eq unicode-ucd/casefolding-status-s 28 \
  "$(awk -F';' '$2 ~ /^ *S *$/ { n++ } END { print n+0 }' "$UCD/CaseFolding.txt")"
check_eq unicode-ucd/casefolding-status-t 2 \
  "$(awk -F';' '$2 ~ /^ *T *$/ { n++ } END { print n+0 }' "$UCD/CaseFolding.txt")"

# --- the exemplars, located in the block and script tables -------------------
inrange() {
  awk -v target="$1" '
  function hex(s,   i,c,v,d) { gsub(/[ \t]/, "", s); v=0
    if (s == "") return -1
    for (i=1; i<=length(s); i++) { c=substr(s,i,1); d=index("0123456789ABCDEF",c)-1
      if (d<0) return -1; v=v*16+d }
    return v }
  BEGIN { t = hex(target) }
  { line=$0; sub(/#.*$/,"",line); sub(/[ \t]+$/,"",line)
    if (line !~ /^[0-9A-F]+([ \t]*\.\.[ \t]*[0-9A-F]+)?[ \t]*;/) next
    split(line, p, /\.\.|;/)
    s = hex(p[1]); e = (index(line, "..") > 0) ? hex(p[2]) : s
    if (s < 0 || e < 0) next
    if (t >= s && t <= e) { v = line; sub(/^[^;]*;[ \t]*/, "", v); print v; exit } }' "$2" 2>&1
}

check_eq unicode-ucd/cp-4E00-block  "CJK Unified Ideographs" "$(inrange 4E00 "$UCD/Blocks.txt")"
check_eq unicode-ucd/cp-10400-block "Deseret"                "$(inrange 10400 "$UCD/Blocks.txt")"
check_eq unicode-ucd/cp-1D11E-block "Musical Symbols"        "$(inrange 1D11E "$UCD/Blocks.txt")"
check_eq unicode-ucd/cp-4E00-script  Han       "$(inrange 4E00 "$UCD/Scripts.txt")"
check_eq unicode-ucd/cp-10400-script Deseret   "$(inrange 10400 "$UCD/Scripts.txt")"
check_eq unicode-ucd/cp-0301-script  Inherited "$(inrange 0301 "$UCD/Scripts.txt")"

# --- cross-check against this image's own perl -------------------------------
# perl carries its own case-mapping tables, compiled into the interpreter, so
# this compares two INDEPENDENT sources rather than a table against itself.
# What it can compare is bounded by linux-in-the-browser#154 -- perl cannot
# compile a non-trivial body of code on this arm, so every module load aborts
# it in glibc. lc and uc need no module and exit 0; everything richer is a
# skip below, naming the issue.
if ! command -v perl >/dev/null 2>&1; then
  bst_skip unicode-ucd/perl "no perl on this image; nothing here can cross-check the tables"
  exit 0
fi

PLOUT=$(perl "$BST_DATA/unicode-ucd-crosscheck.pl" 2>/dev/null); PLRC=$?
case "$PLOUT" in
  *ALL-PL-OK*) bst_ok unicode-ucd/perl-payload ;;
  *) bst_notok unicode-ucd/perl-payload "rc=$PLRC $(_bst_1line "$PLOUT")" ;;
esac

if [ "$PLRC" -ne 0 ]; then
  bst_skip unicode-ucd/perl-crosscheck "payload did not run; nothing to compare"
  exit 0
fi

pl() { printf '%s\n' "$PLOUT" | awk -v want="$1" '$1 == want { print $2; exit }'; }

# Pinned, like the UCD version above: perl's case tables are perl's own, so a
# perl bump is a change of the OTHER side of every comparison below and must
# be read rather than absorbed.
check_eq unicode-ucd/perl-version 5.38.4 "$(pl R-perlversion)"

# UnicodeData.txt leaves the mapping fields EMPTY where a codepoint maps to
# itself; the payload writes "-" for the same case, so the two are made to
# agree on one spelling before they are compared. U+4E00 is here precisely
# because it has no case mapping: it checks the empty case rather than
# skipping it, which is where a parser that mis-indexes the 15 fields lands.
ucdmap() { m=$(fld "$1" "$2"); [ -n "$m" ] && printf '%s\n' "$m" || printf -- '-\n'; }

check_eq unicode-ucd/perl-lower-0041  "$(ucdmap 0041 14)"  "$(pl R-lower-0041)"
check_eq unicode-ucd/perl-lower-00C9  "$(ucdmap 00C9 14)"  "$(pl R-lower-00C9)"
check_eq unicode-ucd/perl-lower-10400 "$(ucdmap 10400 14)" "$(pl R-lower-10400)"
check_eq unicode-ucd/perl-lower-4E00  "$(ucdmap 4E00 14)"  "$(pl R-lower-4E00)"

# The same mappings read from the other end. U+10428 -> U+10400 is the astral
# one, and it is the assertion this suite exists for: a 32-bit arm that
# truncated a supplementary-plane codepoint would answer 0400 here.
check_eq unicode-ucd/perl-upper-0061  "$(ucdmap 0061 13)"  "$(pl R-upper-0061)"
check_eq unicode-ucd/perl-upper-00E9  "$(ucdmap 00E9 13)"  "$(pl R-upper-00E9)"
check_eq unicode-ucd/perl-upper-10428 "$(ucdmap 10428 13)" "$(pl R-upper-10428)"

# The two halves of the cross-check this image cannot make, each naming what
# would unblock it. Both are SKIPs rather than reds because the modules are
# legitimately absent -- they are not installed, and installing them was tried
# and measured to change nothing while #154 stands.
bst_skip unicode-ucd/perl-general-category \
  "gc needs Unicode::UCD or \\p{...}; the module aborts on load and \\p{Han} answers wrong then aborts -- linux-in-the-browser#154"
bst_skip unicode-ucd/perl-case-folding \
  "fc() computes the right fold then aborts the process in glibc -- linux-in-the-browser#154; the C+F rows above are asserted against the table only"