# cfengine-masterfiles-verify.awk -- the whole of 44-cfengine-masterfiles's
# judgement. The .test file execs eight times and hands the results here; every
# BST line the suite emits is printed below.
#
# TWO MODES, ONE SCANNER, AND THAT IS THE POINT.
#
#   mode=""      (default) the suite's verifier.
#   mode="refs"  the reference census, printing `refs_distinct N' and
#                `refs_total M' and nothing else.
#
# src/cfengine_masterfiles_mkmanifest.sh calls the second to write the two
# `#=' counts the first then asserts as an equality. One implementation, so the
# reference and the assertion cannot drift apart by one of them being fixed --
# which they would, because the rule is subtle: see THE STRING SCANNER below.
#
# INPUT ORDER (the .test builds them; ARGIND is what selects the pass):
#   1  the manifest, data/cfengine-masterfiles-manifest.txt
#   2  find(1) inventory of the installed tree: %p %y %s %m
#   3  sha256sum over every installed file
#   4  the package's stanza from /var/lib/dpkg/status
#   5  grep -H . over /var/lib/dpkg/info/cfengine-masterfiles.list
#   6+ every installed *.cf file, in manifest order
#
# ============================================================ THE STRING SCANNER
#
# The policy files are read with a character-level scanner that tracks quoting
# ACROSS LINES, and that is not fastidiousness. 583 lines in 38 of the 80 files
# end inside an open string, and two of those strings contain the words
#
#     promise to terminate misbehaving daemons        (limit_robot_agents.cf:73)
#     promise logs to derive delta reports.           (lib/cfe_internal.cf:35)
#
# A line-oriented reader sees two `promise' declarations of types `to' and
# `logs', neither of which exists, and reports a correct policy tree as
# malformed. The same scanner is what keeps a `#' inside a string from
# truncating the line, and what stops a brace inside a string from unbalancing
# the file.
#
# It jumps between the six characters that can change state -- # " ' ` { } --
# rather than stepping one character at a time: 44,387 of those in 797,857
# bytes, so the scan is bounded by the tokens and not by the corpus.
#
# ================================================= THE GRAMMAR TABLES ARE CITED
#
# Every recognised name below is read off cfengine 3.21.0's own source, the
# version this package is cut from, and not from documentation:
#
#   bundle types   libpromises/policy.c:3324 BundleTypeCheck() -- CF_AGENTTYPES
#                  (libpromises/constants.c:65, expanded through cf3.defs.h:390)
#                  plus knowledge, edit_line and edit_xml.
#   body types     libpromises/syntax.c:173 BodySyntaxGet() -- every
#                  ConstraintSyntaxNewBody() lval across mod_*.c, plus
#                  CONTROL_BODIES (mod_common.c:462).
#   promise types  every PromiseTypeSyntaxNew(<bundle>, <promise>) across
#                  mod_*.c, plus CF_COMMON_PROMISE_TYPES's "*" bundle
#                  (mod_common.c:528), which is what makes vars, classes,
#                  reports, meta and defaults legal in every bundle type.
#
# ONE OF THESE IS DELIBERATELY STRICTER THAN CFENGINE. BodySyntaxGet returns
# CUSTOM_BODY_BLOCK_SYNTAX rather than NULL for a name it does not know
# (syntax.c:213), so cf-promises accepts an unknown body type as a custom body
# and would NOT reject one. This tree defines no custom bodies -- all 37 body
# types it uses are built in -- so the strict reading costs nothing today and
# makes a new one visible instead of silent. A red here is therefore a CHANGE,
# not necessarily a defect, and its message says so.
#
# ===================================================== WHAT IS NOT ASSERTED
#
# THAT THE POLICY IS CORRECT. Nothing here evaluates it: cf-agent is not on this
# image (cfengine is on BRINTOS_HWJS81_EXCLUDED), which is the whole reason this
# package is ported on its own. 44-cfengine.test is where an agent-driven
# assertion lives, and it runs when cfengine returns to the image.
#
# THAT A VARIABLE-EXPANDED INPUT RESOLVES. `$(sys.local_libdir)/stdlib.cf' names
# a path only cf-agent can compute. Those literals are skipped, by the rule
# below, and the count of what WAS resolved is asserted so the skipping cannot
# quietly grow.

function rel(p) {
    sub(/^\.\//, "", p)
    if (p == ROOT) return "."
    sub("^" ROOT "/", "", p)
    return p
}

# Backslashes immediately preceding position `pos' -- an odd count means the
# character there is escaped and does not close the string.
function nbs(s, pos,    n, k) {
    n = 0; k = pos - 1
    while (k >= 1 && substr(s, k, 1) == "\\") { n++; k-- }
    return n
}

function find_close(s,    i, p, pos) {
    i = 1
    while (1) {
        p = index(substr(s, i), Q)
        if (p == 0) return 0
        pos = i + p - 1
        if (Q == "\"" && nbs(s, pos) % 2 == 1) { i = pos + 1; continue }
        return pos
    }
}

# A double-quoted literal is a policy reference iff it is a relative path of
# path characters ending in .cf or .mustache AND contains a '/'. The '/' is
# load-bearing: lib/paths.cf:670 tags variables with the literal "paths.cf",
# which is metadata and not a file, and it is the only bare-name literal in the
# tree that would otherwise be resolved and reported missing.
function note_literal(f, s) {
    if (s !~ /^[A-Za-z0-9._\/-]+\.(cf|mustache)$/) return
    if (index(s, "/") == 0) return
    reftotal++
    if (!(s in refseen)) { refseen[s] = 1; refdistinct++; reforder[refdistinct] = s }
    # Distinct referrers, bounded: a red names where the reference came from,
    # and promises.cf alone makes the same one nine times.
    if (!((s SUBSEP f) in refpair)) {
        refpair[s SUBSEP f] = 1
        if (refn[s] < 4) reffrom[s] = reffrom[s] " " f
        else if (refn[s] == 4) reffrom[s] = reffrom[s] " ..."
        refn[s]++
    }
}

function scan_line(f, line,    c, pos) {
    REST = line
    while (1) {
        if (Q != "") {
            pos = find_close(REST)
            if (pos == 0) { STRBUF = STRBUF REST; return }
            STRBUF = STRBUF substr(REST, 1, pos - 1)
            REST = substr(REST, pos + 1)
            if (Q == "\"") note_literal(f, STRBUF)
            Q = ""; STRBUF = ""
            continue
        }
        if (match(REST, /[#"'`{}]/) == 0) return
        c = substr(REST, RSTART, 1)
        REST = substr(REST, RSTART + 1)
        if (c == "#") return
        if (c == "{") { DEPTH++; continue }
        if (c == "}") { DEPTH--; if (DEPTH < MIND) MIND = DEPTH; continue }
        Q = c; STRBUF = ""
    }
}

function bad(f, why) {
    if (index(cfbad[f], why) == 0) cfbad[f] = cfbad[f] (cfbad[f] == "" ? "" : "; ") why
}

function ok(id)          { printf "BST ok %s\n", id }
function notok(id, why)  { printf "BST not ok %s : %s\n", id, why }
function verdict(id, why) { if (why == "") ok(id); else notok(id, why) }

BEGIN {
    S = "cfengine-masterfiles"
    # The installed prefix. Overridable with -v root=<dir> for ONE purpose: to
    # run this verifier natively against an extracted .deb before it ships, which
    # is how the fault-injection matrix in the design doc was measured. The suite
    # never sets it.
    ROOT = (root != "" ? root : "/usr/share/cfengine")
    FIXED = 5
    if (mode == "refs") FIXED = 0

    split("common agent server monitor executor runagent keygenerator hub " \
          "knowledge edit_line edit_xml", t, " ")
    for (i in t) BUNDLETYPE[t[i]] = 1

    split("acl action association changes classes contain copy_from " \
          "database_server delete delete_select depth_search edit_defaults " \
          "edit_field environment_interface environment_resources file_select " \
          "insert_select link_from location match_value mount package_method " \
          "package_module password perms printfile process_count " \
          "process_select rename replace_with report_data_select " \
          "select_region service_method volume " \
          "common agent server monitor runagent executor hub file " \
          "reporter knowledge", t, " ")
    for (i in t) BODYTYPE[t[i]] = 1

    split("classes defaults meta reports vars", t, " ")
    for (i in t) UNIVERSAL[t[i]] = 1

    PT["agent"]     = " commands databases files guest_environments methods" \
                      " outputs packages processes reports services storage users "
    PT["edit_line"] = " delete_lines field_edits insert_lines replace_patterns "
    PT["edit_xml"]  = " build_xpath delete_attribute delete_text delete_tree" \
                      " insert_text insert_tree set_attribute set_text "
    PT["knowledge"] = " inferences occurrences things topics "
    PT["monitor"]   = " measurements "
    PT["server"]    = " access roles "
    PT["common"]    = " "
    PT["executor"] = PT["runagent"] = PT["keygenerator"] = PT["hub"] = " "
}

# ------------------------------------------------------------------- 1: manifest
ARGIND == 1 && mode != "refs" {
    if ($1 == "#=") { meta[$2] = substr($0, index($0, $3)); next }
    if (/^#/ || NF == 0) next
    if ($1 == "F") {
        p = rel($2)
        mf_size[p] = $3; mf_mode[p] = $4; mf_sha[p] = $5
        nmf++; mforder[nmf] = p
        if (p ~ /\.cf$/) { ncf_expect++; cf_expect[p] = 1 }
        next
    }
    if ($1 == "D") { p = rel($2); md_mode[p] = $3; nmd++; mdorder[nmd] = p }
    next
}

# ---------------------------------------------------- 2: find(1) inventory
ARGIND == 2 && mode != "refs" {
    p = rel($1)
    inv_type[p] = $2; inv_size[p] = $3; inv_mode[p] = $4
    if ($2 == "f") { ninv_f++; inv_bytes += $3 } else if ($2 == "d") ninv_d++
    else { nother++; other = other " " p "(" $2 ")" }
    next
}

# ------------------------------------------------------------------ 3: sha256sum
ARGIND == 3 && mode != "refs" { got_sha[rel($2)] = $1; next }

# --------------------------------------------------------------- 4: dpkg status
ARGIND == 4 && mode != "refs" {
    if ($1 == "Package:")      st_pkg  = $2
    else if ($1 == "Status:")  st_stat = substr($0, index($0, $2))
    else if ($1 == "Version:") st_ver  = $2
    else if ($1 == "Architecture:") st_arch = $2
    else if ($1 == "Depends:") st_dep = substr($0, index($0, $2))
    next
}

# ----------------------------------------------------------------- 5: .list file
ARGIND == 5 && mode != "refs" {
    p = $0
    sub(/^[^:]*:/, "", p)                       # grep -H's "file:" prefix
    if (index(p, ROOT) != 1) next
    q = rel(p)
    if (q == ".") next
    listed[q] = 1; nlisted++
    next
}

# ------------------------------------------------------------- 6+: policy files
ARGIND > FIXED {
    if (FNR == 1) {
        CF = rel(FILENAME)
        cfseen[CF] = 1; ncf_seen++
        DEPTH = 0; MIND = 0; Q = ""; STRBUF = ""
    }
    if (Q == "" && DEPTH == 0 &&
        match($0, /^[ \t]*(bundle|body|promise)[ \t]+[A-Za-z_][A-Za-z0-9_]*[ \t]/)) {
        line = $0
        sub(/^[ \t]+/, "", line)
        split(line, d, /[ \t]+/)
        ndecl++
        if (d[1] == "bundle") {
            CURB = d[2]; used_bundle[d[2]] = 1
            if (!(d[2] in BUNDLETYPE)) bad(CF, "unknown bundle type '" d[2] "' at line " FNR)
        } else if (d[1] == "body") {
            CURB = ""; used_body[d[2]] = 1
            if (!(d[2] in BODYTYPE))
                bad(CF, "body type '" d[2] "' at line " FNR " is not built in to cfengine 3.21.0 -- a custom body, which cf-promises accepts and this tree has never used")
        } else {
            CURB = ""
            if (d[2] != "agent") bad(CF, "custom promise block '" d[2] "' at line " FNR " -- only `promise agent' exists")
        }
    }
    else if (Q == "" && DEPTH == 1 && CURB != "" &&
             match($0, /^[ \t]*[a-z_]+:[ \t]*(#.*)?$/)) {
        pt = $0
        sub(/^[ \t]*/, "", pt); sub(/:.*$/, "", pt)
        npt++
        used_pt[CURB SUBSEP pt] = 1
        if (!(pt in UNIVERSAL) && index(PT[CURB], " " pt " ") == 0)
            bad(CF, "promise type '" pt "' at line " FNR " is not admitted by a `" CURB "' bundle")
    }
    # The tree's own two version declarations, matched on the line rather than
    # through the scanner: both are single-line and both are asserted below
    # against the version the recipe pinned.
    if (CF == "masterfiles/promises.cf" &&
        match($0, /version[ \t]*=>[ \t]*"CFEngine Promises\.cf [0-9][0-9.]*"/)) {
        tree_ver_promises = substr($0, RSTART, RLENGTH)
        sub(/^.*Promises\.cf /, "", tree_ver_promises); sub(/"$/, "", tree_ver_promises)
    }
    if (CF == "masterfiles/controls/update_def.cf" &&
        match($0, /"current_version"[ \t]*string[ \t]*=>[ \t]*"[0-9][0-9.]*"/)) {
        tree_ver_update = substr($0, RSTART, RLENGTH)
        sub(/^.*=>[ \t]*"/, "", tree_ver_update); sub(/"$/, "", tree_ver_update)
    }

    scan_line(CF, $0)
    next
}

ENDFILE {
    if (ARGIND > FIXED && CF != "") {
        if (Q != "")   bad(CF, "file ends inside an unterminated " (Q == "\"" ? "double" : (Q == "'" ? "single" : "back")) "-quoted string")
        if (DEPTH != 0) bad(CF, "braces unbalanced: depth " DEPTH " at end of file")
        if (MIND < 0)   bad(CF, "a '}' closed more than was open (depth reached " MIND ")")
        CF = ""
    }
}

END {
    if (mode == "refs") {
        printf "refs_distinct %d\n", refdistinct + 0
        printf "refs_total %d\n", reftotal + 0
        exit 0
    }

    # ---------------------------------------------------------- the manifest
    hdr = ""
    split("version arch files dirs cf_files bytes refs_distinct refs_total " \
          "bundle_types body_types", k, " ")
    for (i in k) if (!(k[i] in meta)) hdr = hdr " " k[i]
    verdict(S "/manifest-header", hdr == "" ? "" : "the manifest declares no" hdr)

    verdict(S "/tree-root", inv_type["."] == "d" ? "" \
            : "the package's own directory " ROOT " is " (inv_type["."] == "" ? "absent" : "a '" inv_type["."] "', not a directory"))

    # ------------------------------------------------------ set equalities
    miss = ""; nmiss = 0
    for (i = 1; i <= nmf; i++) if (!(mforder[i] in inv_type)) { nmiss++; if (nmiss <= 5) miss = miss " " mforder[i] }
    extra = ""; nextra = 0
    for (p in inv_type) if (inv_type[p] == "f" && !(p in mf_size)) { nextra++; if (nextra <= 5) extra = extra " " p }
    verdict(S "/file-set",
            (nmiss == 0 && nextra == 0) ? "" \
            : nmiss " manifest file(s) absent[" miss " ], " nextra " unexpected file(s)[" extra " ]")

    miss = ""; nmiss = 0
    for (i = 1; i <= nmd; i++) if (!(mdorder[i] in inv_type)) { nmiss++; if (nmiss <= 5) miss = miss " " mdorder[i] }
    extra = ""; nextra = 0
    for (p in inv_type) if (inv_type[p] == "d" && !(p in md_mode)) { nextra++; if (nextra <= 5) extra = extra " " p }
    verdict(S "/dir-set",
            (nmiss == 0 && nextra == 0) ? "" \
            : nmiss " manifest directory(s) absent[" miss " ], " nextra " unexpected[" extra " ]")

    verdict(S "/no-other-entries", nother == 0 ? "" \
            : nother " entry(s) under " ROOT " are neither a regular file nor a directory:" other)

    verdict(S "/file-count", ninv_f == meta["files"] + 0 ? "" \
            : "installed " ninv_f " files, the manifest names " meta["files"])
    verdict(S "/dir-count", ninv_d == meta["dirs"] + 0 ? "" \
            : "installed " ninv_d " directories, the manifest names " meta["dirs"])
    verdict(S "/total-bytes", inv_bytes == meta["bytes"] + 0 ? "" \
            : "installed tree is " inv_bytes " bytes, the manifest says " meta["bytes"])

    # -------------------------------------------------- per-file and per-dir
    nmode_f = 0; nmode_d = 0; gw = ""; ww = ""
    for (i = 1; i <= nmf; i++) {
        p = mforder[i]; why = ""
        if (!(p in inv_type))            why = "absent from the installed tree"
        else if (inv_type[p] != "f")     why = "installed as a '" inv_type[p] "', not a regular file"
        else {
            if (inv_size[p] != mf_size[p]) why = "size " inv_size[p] " want " mf_size[p]
            if (inv_mode[p] != mf_mode[p]) { why = why (why == "" ? "" : "; ") "mode " inv_mode[p] " want " mf_mode[p]; nmode_f++ }
        }
        verdict(S "/file/" p, why)
        if (p in inv_type) {
            if (inv_mode[p] ~ /^[0-9][2367][0-9]$/) gw = gw " " p "(" inv_mode[p] ")"
            if (inv_mode[p] ~ /^[0-9][0-9][2367]$/) ww = ww " " p "(" inv_mode[p] ")"
        }
        why = ""
        if (!(p in got_sha))            why = "no digest was produced for it"
        else if (got_sha[p] != mf_sha[p]) why = "sha256 " substr(got_sha[p], 1, 16) "... want " substr(mf_sha[p], 1, 16) "..."
        verdict(S "/sha/" p, why)
    }
    for (i = 1; i <= nmd; i++) {
        p = mdorder[i]; why = ""
        if (!(p in inv_type))         why = "absent from the installed tree"
        else if (inv_type[p] != "d")  why = "installed as a '" inv_type[p] "', not a directory"
        else if (inv_mode[p] != md_mode[p]) { why = "mode " inv_mode[p] " want " md_mode[p]; nmode_d++ }
        verdict(S "/dir/" p, why)
        if (p in inv_type) {
            if (inv_mode[p] ~ /^[0-9][2367][0-9]$/) gw = gw " " p "(" inv_mode[p] ")"
            if (inv_mode[p] ~ /^[0-9][0-9][2367]$/) ww = ww " " p "(" inv_mode[p] ")"
        }
    }

    # cf-agent refuses to load a policy tree that anyone but its owner can
    # write -- these two are the reason the per-entry mode above is asserted at
    # all, stated separately so the refusal has a case of its own.
    verdict(S "/not-group-writable", gw == "" ? "" : "group-writable, which cf-agent refuses to load:" gw)
    verdict(S "/not-world-writable", ww == "" ? "" : "world-writable, which cf-agent refuses to load:" ww)

    # -------------------------------------------------------- the entry points
    entry["promises.cf"]              = "masterfiles/promises.cf"
    entry["update.cf"]                = "masterfiles/update.cf"
    entry["stdlib"]                   = "masterfiles/lib/stdlib.cf"
    entry["controls-def"]             = "masterfiles/controls/def.cf"
    entry["controls-def-inputs"]      = "masterfiles/controls/def_inputs.cf"
    entry["services-main"]            = "masterfiles/services/main.cf"
    entry["standalone-self-upgrade"]  = "masterfiles/standalone_self_upgrade.cf"
    for (e in entry)
        verdict(S "/entry/" e, (entry[e] in inv_type && inv_type[entry[e]] == "f") ? "" \
                : "the tree's " e " entry point is not an installed regular file: " entry[e])

    # --------------------------------------------------------- the references
    unres = ""; nunres = 0; esc = ""; nesc = 0
    for (i = 1; i <= refdistinct; i++) {
        r = reforder[i]; why = ""
        if (r ~ /(^|\/)\.\.(\/|$)/ || substr(r, 1, 1) == "/") {
            why = "escapes the policy tree; referenced by" reffrom[r]; nesc++; esc = esc " " r
        } else if (!("masterfiles/" r in inv_type)) {
            why = "dangling: no masterfiles/" r " in the installed tree; referenced by" reffrom[r]
            nunres++; if (nunres <= 5) unres = unres " " r
        }
        verdict(S "/ref/" r, why)
    }
    verdict(S "/refs-resolve", nunres == 0 ? "" \
            : nunres " of " refdistinct " policy references do not resolve:" unres)
    verdict(S "/refs-no-escape", nesc == 0 ? "" \
            : nesc " reference(s) name a path outside the tree:" esc)
    verdict(S "/refs-distinct-count", refdistinct == meta["refs_distinct"] + 0 ? "" \
            : "found " refdistinct " distinct references, the manifest recorded " meta["refs_distinct"])
    verdict(S "/refs-total-count", reftotal == meta["refs_total"] + 0 ? "" \
            : "found " reftotal " reference occurrences, the manifest recorded " meta["refs_total"])

    # ---------------------------------------------------------- the .cf corpus
    verdict(S "/cf-count", (ncf_seen == ncf_expect && ncf_seen == meta["cf_files"] + 0) ? "" \
            : "scanned " ncf_seen " policy files; the manifest names " ncf_expect " and declares " meta["cf_files"])
    for (p in cf_expect)
        verdict(S "/cf/" p, (p in cfseen) ? cfbad[p] : "was not scanned -- it is not an installed readable file")
    verdict(S "/declarations", ndecl > 0 ? "" : "not one top-level bundle or body declaration was found in " ncf_seen " policy files")
    verdict(S "/promise-types", npt > 0 ? "" : "not one promise type was found in " ncf_seen " policy files")

    # The two type sets, as equalities against the manifest: every name is
    # already known-good above, so what these add is that the SET has not moved.
    n = 0; got = ""
    for (b in used_bundle) sorted[++n] = b
    asort_simple(sorted, n); for (i = 1; i <= n; i++) got = got (i > 1 ? " " : "") sorted[i]
    verdict(S "/bundle-type-set", got == meta["bundle_types"] ? "" \
            : "bundle types used are '" got "', the manifest recorded '" meta["bundle_types"] "'")
    delete sorted
    n = 0; got = ""
    for (b in used_body) sorted[++n] = b
    asort_simple(sorted, n); for (i = 1; i <= n; i++) got = got (i > 1 ? " " : "") sorted[i]
    verdict(S "/body-type-set", got == meta["body_types"] ? "" \
            : "body types used are '" got "', the manifest recorded '" meta["body_types"] "'")

    # ---------------------------------------------------------------- versions
    #
    # THREE INDEPENDENT DECLARATIONS OF THE SAME NUMBER, and they are worth
    # asserting separately because they come from different places: two are
    # written into the policy by upstream and travel with the tarball, and the
    # third is what the recipe pinned and dpkg recorded. A tarball swapped under
    # an unchanged PV moves the first two and not the third.
    verdict(S "/version-tree-promises", tree_ver_promises == meta["version"] ? "" \
            : "masterfiles/promises.cf declares version '" tree_ver_promises "', the recipe pins " meta["version"])
    verdict(S "/version-tree-update-def", tree_ver_update == meta["version"] ? "" \
            : "masterfiles/controls/update_def.cf declares current_version '" tree_ver_update "', the recipe pins " meta["version"])
    verdict(S "/version-package", st_ver ~ ("^" meta["version"] "-r[0-9]+$") ? "" \
            : "dpkg records version '" st_ver "', the recipe pins " meta["version"])
    verdict(S "/arch-field", st_arch == meta["arch"] ? "" \
            : "dpkg records architecture '" st_arch "', the build produced " meta["arch"])
    verdict(S "/dpkg-installed", (st_pkg == S && st_stat ~ /install ok installed/) ? "" \
            : "dpkg's status for " S " is '" st_stat "'")
    verdict(S "/dpkg-no-depends", st_dep == "" ? "" \
            : "the package declares Depends: " st_dep " -- the bbappend removes the python3-core RDEPENDS and nothing should replace it")

    # dpkg's own file list against what is on disk: what the package OWNS, as
    # opposed to what merely sits at the right path.
    miss = ""; nmiss = 0
    for (i = 1; i <= nmf; i++) if (!(mforder[i] in listed)) { nmiss++; if (nmiss <= 5) miss = miss " " mforder[i] }
    verdict(S "/dpkg-owns-every-file", nmiss == 0 ? "" \
            : nmiss " installed file(s) are not in the package's dpkg file list:" miss)
}

# gawk has asort(), but this suite runs wherever the image's awk runs, and an
# insertion sort over at most 40 type names is not worth a dependency.
function asort_simple(a, n,    i, j, v) {
    for (i = 2; i <= n; i++) {
        v = a[i]; j = i - 1
        while (j >= 1 && a[j] > v) { a[j + 1] = a[j]; j-- }
        a[j + 1] = v
    }
}