# -*- coding: utf-8 -*-
"""One python3 start, every measurement charset-normalizer 3.3.2 can be asked for.

Emitted as `key=value' lines on stdout, one per line, for
tests/99-python3-charset-normalizer.test to score with check_eq. An interpreter
start on this arm costs tens of seconds and the work below costs milliseconds,
so everything that needs a live charset_normalizer happens here rather than in a
shell loop of one-liners.

Every fixture is written by this file into its own directory under $TMPDIR and
removed again; nothing on the rootfs is read as input and nothing survives.

A measurement that raises is emitted as `<key>=EXC:<ExceptionClass>' rather than
killing the probe, so one broken case costs one red instead of every case after
it.

THE FIXTURE TEXTS ARE NOT INTERCHANGEABLE, and choosing them was most of the
work. Detection here is a heuristic over a chaos ratio and a per-language letter
histogram, not a validity check, so WHICH text is encoded decides whether the
answer is determined at all:

  DE  German  -- the determined Unicode subject. utf_8, utf_8 with a BOM, and
                 all four UTF-16 framings answer the right label exactly.
  RU  Russian -- the determined single-byte subject. koi8_r and cp1251 map the
                 same letters to different bytes, so the histogram separates
                 them and the label can be asserted.
  FR  French, heavily accented -- the AMBIGUOUS subject, kept deliberately. Its
                 latin_1 encoding is one of a dozen equally valid single-byte
                 readings, and its UTF-8 encoding is REJECTED as utf_8 (see
                 below). Both are asserted as measured, which is the honest
                 record of what this library does rather than of what it is
                 assumed to do.
"""
import os
import shutil
import sys
import tempfile

OUT = []


def emit(key, value):
    OUT.append("%s=%s" % (key, value))


def yn(b):
    return "yes" if b else "no"


def measure(key, fn):
    """Run fn and emit its value, or the exception class name it raised."""
    try:
        emit(key, fn())
    except Exception as exc:                       # noqa: BLE001 -- the point
        emit(key, "EXC:%s" % type(exc).__name__)


def raises(key, fn):
    """Emit the exception class fn raised, or `no-raise' with what it returned."""
    try:
        got = fn()
    except Exception as exc:                       # noqa: BLE001 -- the point
        emit(key, type(exc).__name__)
    else:
        emit(key, "no-raise:%r" % (got,))


# ---------------------------------------------------------------- fixtures --
DE = (
    "Die Straße war nass und die Kinder spielten trotzdem draußen im Regen. "
    "Über dem Fluss hing ein grauer Nebel, der erst am Nachmittag verschwand. "
    "Wir grüßen alle Gäste herzlich und wünschen einen schönen Aufenthalt. "
) * 3
RU = (
    "Съешь же ещё этих мягких французских булок да выпей чаю. "
    "Широкая электрификация южных губерний даст мощный толчок подъёму. "
) * 3
FR = (
    "Le café était déjà prêt, naturellement. "
    "L'élève a répondu à la question posée par le "
    "maître d'école. Où êtes-vous allés hier soir "
    "après la représentation théâtrale ? Ça coûte "
    "cher, mais la qualité est là, sans aucun doute possible. "
) * 3
EN = (
    "The “smart quotes” cost €5 — really. He said: "
    "‘it’s fine’, and the manager™ agreed… mostly. "
) * 4
ASCII = (
    "The quick brown fox jumps over the lazy dog. Pack my box with five dozen "
    "liquor jugs. How vexingly quick daft zebras jump! 0123456789. "
) * 3
FR_DECL = "# -*- coding: latin-1 -*-\n" + FR
EN_DECL = '<meta charset="cp1252">\n' + EN

emit("py_version", "%d.%d" % sys.version_info[:2])

W = tempfile.mkdtemp(prefix="bst-cn.")
FIX = {}
try:
    FIX["ascii"] = ASCII.encode("ascii")
    FIX["utf8"] = DE.encode("utf-8")
    FIX["utf8_bom"] = DE.encode("utf-8-sig")
    FIX["utf16"] = DE.encode("utf-16")                        # BOM, LE payload
    FIX["utf16_be"] = b"\xfe\xff" + DE.encode("utf-16-be")    # BOM, BE payload
    FIX["utf16_le"] = DE.encode("utf-16-le")                  # no BOM
    FIX["utf16_be_nobom"] = DE.encode("utf-16-be")            # no BOM
    FIX["koi8r"] = RU.encode("koi8-r")
    FIX["cp1251"] = RU.encode("cp1251")
    FIX["latin1"] = FR.encode("latin-1")
    FIX["cp1252"] = EN.encode("cp1252")
    FIX["fr_utf8"] = FR.encode("utf-8")
    FIX["latin1_decl"] = FR_DECL.encode("latin-1")
    FIX["cp1252_decl"] = EN_DECL.encode("cp1252")
    FIX["empty"] = b""
    for name, data in FIX.items():
        with open(os.path.join(W, name + ".bin"), "wb") as fh:
            fh.write(data)
    emit("fixtures_written", "yes")
    emit("fixture_count", str(len(FIX)))
except Exception as exc:                           # noqa: BLE001
    emit("fixtures_written", "EXC:%s" % type(exc).__name__)

# ------------------------------------------------------------------ import --
try:
    import charset_normalizer as cn
    from charset_normalizer import (
        CharsetMatch,
        CharsetMatches,
        detect,
        from_bytes,
        from_fp,
        from_path,
        is_binary,
    )
    emit("import_ok", "yes")
except Exception as exc:                           # noqa: BLE001
    emit("import_ok", "EXC:%s" % type(exc).__name__)
    emit("probe_done", "yes")
    emit("emitted_count", str(len(OUT) + 1))
    print("\n".join(OUT))
    shutil.rmtree(W, ignore_errors=True)
    sys.exit(0)

# ---------------------------------------------------------------- identity --
emit("version", cn.__version__)
emit("version_tuple", ",".join(cn.VERSION))
emit("module_file", cn.__file__)
emit("all_names", ",".join(cn.__all__))
emit("public_count", str(len(cn.__all__)))

# The import closure. Every name here is a SEPARATE python3 sub-package on this
# distro, so a missing one is an ImportError at `import charset_normalizer' with
# nothing else to see. hashlib is the one the recipe's own RDEPENDS does not
# name upstream -- python3-charset-normalizer_%.bbappend adds it, and
# `pkg-depends-crypt' in the .test asserts the bbappend took.
for mod in ("logging", "json", "codecs", "encodings.aliases", "unicodedata",
            "_multibytecodec", "hashlib", "importlib", "platform", "argparse",
            "typing", "re", "functools", "collections", "warnings"):
    key = "reachable_" + mod.replace(".", "_")
    measure(key, (lambda m: (lambda: yn(bool(__import__(m)))))(mod))

# `unicodedata' and `_multibytecodec' are lib-dynload extension modules
# upstream, and this image's lib-dynload is EMPTY BY CONSTRUCTION -- they
# resolve only because every stdlib extension is linked into the one static
# interpreter (BRINTOS-1476). Assert that directly, so the day it changes is a
# named red rather than a bare ImportError with no explanation attached.
measure("unicodedata_builtin",
        lambda: yn("unicodedata" in sys.builtin_module_names))
measure("multibytecodec_builtin",
        lambda: yn("_multibytecodec" in sys.builtin_module_names))
measure("unidata_version", lambda: __import__("unicodedata").unidata_version)


# --------------------------------------------------------------- detection --
def best_of(data, **kw):
    return from_bytes(data, **kw).best()


def label(name, data, **kw):
    """encoding | none, for the best match of `data'."""
    def go():
        m = best_of(data, **kw)
        return m.encoding if m is not None else "none"
    measure(name, go)


def decodes_to(name, data, text, **kw):
    """yes when the best match's str() is exactly `text'."""
    def go():
        m = best_of(data, **kw)
        if m is None:
            return "none"
        return yn(str(m) == text)
    measure(name, go)


def bom_of(name, data):
    def go():
        m = best_of(data)
        if m is None:
            return "none"
        return yn(m.bom)
    measure(name, go)


def count(name, data, **kw):
    measure(name, lambda: str(len(from_bytes(data, **kw))))


# ---- determined: the LABEL is the assertion, and the decoded text with it ----
label("best_ascii", FIX["ascii"])
label("best_utf8", FIX["utf8"])
label("best_utf8_bom", FIX["utf8_bom"])
label("best_utf16", FIX["utf16"])
label("best_utf16_be", FIX["utf16_be"])
label("best_utf16_le", FIX["utf16_le"])
label("best_utf16_be_nobom", FIX["utf16_be_nobom"])
label("best_koi8r", FIX["koi8r"])
label("best_cp1251", FIX["cp1251"])

decodes_to("decode_ascii", FIX["ascii"], ASCII)
decodes_to("decode_utf8", FIX["utf8"], DE)
decodes_to("decode_utf8_bom", FIX["utf8_bom"], DE)
decodes_to("decode_utf16", FIX["utf16"], DE)
decodes_to("decode_utf16_be", FIX["utf16_be"], DE)
decodes_to("decode_utf16_le", FIX["utf16_le"], DE)
decodes_to("decode_utf16_be_nobom", FIX["utf16_be_nobom"], DE)
decodes_to("decode_koi8r", FIX["koi8r"], RU)
decodes_to("decode_cp1251", FIX["cp1251"], RU)

count("count_utf8", FIX["utf8"])
count("count_utf16", FIX["utf16"])
count("count_ascii", FIX["ascii"])

# KOI8-R and CP1251 carry the SAME letters in DIFFERENT bytes, so this pair is
# the whole claim that the detector is reading the content rather than guessing
# a default: the two fixtures decode to one identical string through two
# different code pages.
measure("cyrillic_pair_differs", lambda: yn(FIX["koi8r"] != FIX["cp1251"]))
measure("cyrillic_pair_same_text",
        lambda: yn(str(best_of(FIX["koi8r"])) == str(best_of(FIX["cp1251"]))))

# ---- the BOM is REPORTED and STRIPPED ----
bom_of("bom_ascii", FIX["ascii"])
bom_of("bom_utf8", FIX["utf8"])
bom_of("bom_utf8_bom", FIX["utf8_bom"])
bom_of("bom_utf16", FIX["utf16"])
bom_of("bom_utf16_be", FIX["utf16_be"])
bom_of("bom_utf16_le", FIX["utf16_le"])
bom_of("bom_utf16_be_nobom", FIX["utf16_be_nobom"])
measure("bom_is_byte_order_mark",
        lambda: yn(best_of(FIX["utf16"]).bom is best_of(FIX["utf16"]).byte_order_mark))
# The decoded text asserted above already carries no U+FEFF, but only by
# implication; these two say it outright, which is what "strips it" means.
measure("no_feff_utf8_bom", lambda: yn("" not in str(best_of(FIX["utf8_bom"]))))
measure("no_feff_utf16", lambda: yn("" not in str(best_of(FIX["utf16"]))))
measure("no_feff_utf16_be", lambda: yn("" not in str(best_of(FIX["utf16_be"]))))
# Both BOM'd UTF-16 fixtures answer `utf_16': the codec consumes the mark and
# the endianness is not in the label. So the DECODED TEXT is the only thing that
# separates LE from BE here, and it is asserted above. Without the BOM the
# label does carry the endianness -- `best_utf16_le' against
# `best_utf16_be_nobom' -- which is the contrast worth having.
measure("utf16_bom_labels_agree",
        lambda: yn(best_of(FIX["utf16"]).encoding == best_of(FIX["utf16_be"]).encoding))
measure("utf16_nobom_labels_differ",
        lambda: yn(best_of(FIX["utf16_le"]).encoding
                   != best_of(FIX["utf16_be_nobom"]).encoding))

# ---- NOT determined, and this is where the ticket's premise does not hold ----
#
# Two separate findings, both measured and both reproducible.
#
# (1) A single-byte Latin payload is one of a dozen equally valid readings. The
#     free run picks a WRONG code page and decodes to mojibake, and the original
#     encoding is not even among the candidates.
label("free_latin1", FIX["latin1"])
label("free_cp1252", FIX["cp1252"])
decodes_to("free_latin1_exact", FIX["latin1"], FR)
decodes_to("free_cp1252_exact", FIX["cp1252"], EN)
measure("free_latin1_has_latin1",
        lambda: yn("latin_1" in [m.encoding for m in from_bytes(FIX["latin1"])]))
measure("free_cp1252_has_cp1252",
        lambda: yn("cp1252" in [m.encoding for m in from_bytes(FIX["cp1252"])]))
# Isolating the two candidates does not rescue latin_1 either, and the reason is
# structural rather than a tuning accident: api.py only ever keeps latin_1 as a
# FALLBACK, and reaches for a fallback solely when nothing else survived chaos
# probing or when the payload DECLARED that encoding. cp1252 is an ordinary
# candidate and does survive isolation, so the pair separates the two rules.
count("iso_latin1_count", FIX["latin1"], cp_isolation=["latin_1", "cp1252"])
label("iso_latin1", FIX["latin1"], cp_isolation=["latin_1", "cp1252"])
count("iso_cp1252_count", FIX["cp1252"], cp_isolation=["latin_1", "cp1252"])
label("iso_cp1252", FIX["cp1252"], cp_isolation=["latin_1", "cp1252"])
decodes_to("iso_cp1252_exact", FIX["cp1252"], EN,
           cp_isolation=["latin_1", "cp1252"])
# cp_exclusion is the same filter from the other end.
label("excl_koi8r", FIX["koi8r"], cp_exclusion=["koi8_r"])

# A coding declaration IS the documented way to make a single-byte payload
# determined, and `any_specified_encoding' reads both the Python
# `# -*- coding: -*-' form and an HTML `<meta charset=>'. It works for latin_1,
# which is what promotes the fallback into a real match. It does NOT rescue the
# cp1252 sample -- the declaration is honoured but the sample still loses on
# chaos -- so the pair is a statement about the DECLARATION rather than a claim
# that declaring an encoding always wins.
label("decl_latin1", FIX["latin1_decl"])
label("decl_cp1252", FIX["cp1252_decl"])
decodes_to("decl_latin1_exact", FIX["latin1_decl"], FR_DECL)
decodes_to("decl_cp1252_exact", FIX["cp1252_decl"], EN_DECL)
count("decl_latin1_count", FIX["latin1_decl"])

# (2) THE SAME FRENCH TEXT IN UTF-8 IS REJECTED AS utf_8. Heavy accent density
#     puts the mess ratio at 22.3%, over the 0.2 threshold, so utf_8 is dropped
#     during chaos probing -- while cp949, a Korean double-byte code page in
#     which those same bytes are valid and quiet, passes at 0.0% and wins. The
#     German fixture above, ordinary prose with ordinary accent density, is
#     detected as utf_8 correctly. So this is a property of the CONTENT, not of
#     the port, and the two fixtures together are what say so.
label("fr_utf8_best", FIX["fr_utf8"])
decodes_to("fr_utf8_exact", FIX["fr_utf8"], FR)
measure("fr_utf8_has_utf8",
        lambda: yn("utf_8" in [m.encoding for m in from_bytes(FIX["fr_utf8"])]))
# Raising the threshold above the measured 22.3% brings utf_8 back, which is the
# evidence that chaos probing is what dropped it rather than anything else.
label("fr_utf8_threshold", FIX["fr_utf8"], threshold=0.3)
decodes_to("fr_utf8_threshold_exact", FIX["fr_utf8"], FR, threshold=0.3)

# ------------------------------------------------------- the empty input case --
# The ticket expects "the documented empty result rather than raising". It does
# not raise -- but it does not answer empty either: 3.3.2 returns ONE match,
# utf_8, whose str() is the empty string. Measured, not predicted.
count("empty_count", FIX["empty"])
label("empty_label", FIX["empty"])
measure("empty_str", lambda: repr(str(best_of(FIX["empty"]))))
measure("empty_bom", lambda: yn(best_of(FIX["empty"]).bom))
measure("empty_truthy", lambda: yn(bool(from_bytes(FIX["empty"]))))
measure("empty_no_raise", lambda: yn(from_bytes(b"") is not None))
measure("empty_raw_len", lambda: str(len(best_of(FIX["empty"]).raw)))

# ------------------------------------------------------------- the ordering --
# `CharsetMatches' sorts on construction and re-sorts on every append, so
# `best()' is the first element BY CONSTRUCTION. Assert that invariant rather
# than a fixed candidate list: the order is the library's own confidence, and
# pinning it would assert the heuristic instead of the container.
def ordered():
    xs = list(from_bytes(FIX["koi8r"]))
    for a, b in zip(xs, xs[1:]):
        if b < a:                                  # a strictly better follower
            return "no"
    return "yes"


count("koi8r_count", FIX["koi8r"])
measure("koi8r_ordered", ordered)
measure("best_is_index0",
        lambda: yn(from_bytes(FIX["koi8r"]).best().encoding
                   == from_bytes(FIX["koi8r"])[0].encoding))
measure("first_is_best",
        lambda: yn(from_bytes(FIX["koi8r"]).first().encoding
                   == from_bytes(FIX["koi8r"]).best().encoding))
measure("iter_len_agrees",
        lambda: yn(len(list(from_bytes(FIX["koi8r"])))
                   == len(from_bytes(FIX["koi8r"]))))
# __getitem__ takes an encoding NAME as well as an index, and resolves aliases
# through iana_name -- `koi8-r' with a hyphen is not the label the match holds.
measure("getitem_by_name", lambda: from_bytes(FIX["koi8r"])["koi8-r"].encoding)
raises("getitem_absent_raises", lambda: from_bytes(FIX["koi8r"])["shift_jis"])
measure("empty_matches_len", lambda: str(len(CharsetMatches([]))))
measure("empty_matches_best", lambda: repr(CharsetMatches([]).best()))
measure("empty_matches_first", lambda: repr(CharsetMatches([]).first()))
measure("empty_matches_falsy", lambda: yn(not CharsetMatches([])))
raises("append_wrong_type_raises", lambda: CharsetMatches([]).append("not a match"))

# --------------------------------------------------------- match attributes --
KM = best_of(FIX["koi8r"])
measure("koi8r_language", lambda: KM.language)
measure("koi8r_alphabets", lambda: ",".join(KM.alphabets))
measure("koi8r_chaos", lambda: "%.4f" % KM.chaos)
measure("koi8r_percent_chaos", lambda: "%.2f" % KM.percent_chaos)
measure("koi8r_multi_byte", lambda: "%.4f" % KM.multi_byte_usage)
measure("koi8r_raw_len", lambda: str(len(KM.raw)))
measure("koi8r_aliases", lambda: ",".join(sorted(KM.encoding_aliases)))
measure("koi8r_could_be", lambda: ",".join(KM.could_be_from_charset[:3]))
# `output()' re-encodes -- to UTF-8 by default, or to whatever is asked for,
# which round-trips back to the fixture's own bytes.
measure("koi8r_output_is_utf8", lambda: yn(KM.output() == RU.encode("utf-8")))
measure("koi8r_output_roundtrip", lambda: yn(KM.output("koi8-r") == FIX["koi8r"]))
# The fingerprint is sha256 OF THE UTF-8 OUTPUT, not of the raw bytes -- which
# is why two matches of the same text through different code pages compare
# equal on it, and why this package needs hashlib at all.
measure("koi8r_fingerprint_is_sha256",
        lambda: yn(len(KM.fingerprint) == 64
                   and all(c in "0123456789abcdef" for c in KM.fingerprint)))
measure("koi8r_fingerprint_of_output",
        lambda: yn(KM.fingerprint
                   == __import__("hashlib").sha256(KM.output()).hexdigest()))
measure("fingerprint_matches_across_codepages",
        lambda: yn(best_of(FIX["koi8r"]).fingerprint
                   == best_of(FIX["cp1251"]).fingerprint))
measure("koi8r_has_submatch", lambda: yn(KM.has_submatch))
measure("koi8r_repr_has_encoding", lambda: yn("koi8_r" in repr(KM)))
# `__eq__' against a str RAISES rather than answering False. `match == "utf_8"'
# reads as the obvious thing to write and is a TypeError, so it is asserted
# rather than left to be discovered by a caller.
raises("eq_str_raises", lambda: KM == "koi8_r")
measure("eq_self", lambda: yn(best_of(FIX["koi8r"]) == best_of(FIX["koi8r"])))
measure("eq_across_codepages",
        lambda: yn(best_of(FIX["koi8r"]) == best_of(FIX["cp1251"])))

# ------------------------------------------------------- from_path, from_fp --
def path_of(name):
    return os.path.join(W, name + ".bin")


measure("from_path_koi8r", lambda: from_path(path_of("koi8r")).best().encoding)
measure("from_path_utf8_bom", lambda: from_path(path_of("utf8_bom")).best().encoding)
measure("from_path_utf16", lambda: from_path(path_of("utf16")).best().encoding)
measure("from_path_decodes", lambda: yn(str(from_path(path_of("koi8r")).best()) == RU))
measure("from_path_bom_stripped",
        lambda: yn(str(from_path(path_of("utf8_bom")).best()) == DE))
measure("from_path_agrees_with_bytes",
        lambda: yn(from_path(path_of("koi8r")).best().encoding
                   == from_bytes(FIX["koi8r"]).best().encoding))
raises("from_path_missing_raises", lambda: from_path(os.path.join(W, "nope.bin")))


def fp_read():
    with open(path_of("koi8r"), "rb") as fh:
        return from_fp(fh).best().encoding


measure("from_fp_koi8r", fp_read)

# ------------------------------------------------------------------ detect --
# The chardet-compatibility shim. Its label is the CHARDET spelling and is NOT
# what from_bytes answers -- `KOI8-R' against `koi8_r' -- which is exactly the
# difference a caller migrating off chardet is relying on.
measure("detect_keys", lambda: ",".join(sorted(detect(FIX["koi8r"]).keys())))
measure("detect_koi8r_encoding", lambda: str(detect(FIX["koi8r"])["encoding"]))
measure("detect_koi8r_language", lambda: str(detect(FIX["koi8r"])["language"]))
measure("detect_koi8r_confidence", lambda: "%.2f" % detect(FIX["koi8r"])["confidence"])
measure("detect_label_differs",
        lambda: yn(detect(FIX["koi8r"])["encoding"]
                   != from_bytes(FIX["koi8r"]).best().encoding))
measure("detect_ascii_encoding", lambda: str(detect(FIX["ascii"])["encoding"]))
measure("detect_utf8_encoding", lambda: str(detect(FIX["utf8"])["encoding"]))
measure("detect_utf16_encoding", lambda: str(detect(FIX["utf16"])["encoding"]))
measure("detect_empty_encoding", lambda: str(detect(FIX["empty"])["encoding"]))
measure("detect_empty_language", lambda: repr(detect(FIX["empty"])["language"]))
measure("detect_confidence_is_float",
        lambda: type(detect(FIX["koi8r"])["confidence"]).__name__)

# --------------------------------------------------------------- is_binary --
measure("is_binary_png", lambda: yn(is_binary(b"\x89PNG\r\n\x1a\n" + bytes(256))))
measure("is_binary_utf8", lambda: yn(is_binary(FIX["utf8"])))
measure("is_binary_koi8r", lambda: yn(is_binary(FIX["koi8r"])))
measure("is_binary_utf16", lambda: yn(is_binary(FIX["utf16"])))
measure("is_binary_path", lambda: yn(is_binary(path_of("koi8r"))))

# --------------------------------------------- what the .test needs from here --
# The CLI is run as a real process by the .test, against two files, and its
# answer is compared with the library's. Emit the library's side here so the
# comparison is between two measurements rather than against a constant -- and
# hand the .test the same BYTES, by copying two fixtures into the directory it
# named on argv[1]. It owns that directory and removes it; the probe's own
# workdir below is removed here regardless.
measure("cli_subject_koi8r", lambda: from_path(path_of("koi8r")).best().encoding)
measure("cli_subject_utf8", lambda: from_path(path_of("utf8")).best().encoding)

if len(sys.argv) > 1:
    def hand_over():
        dst = sys.argv[1]
        os.makedirs(dst, exist_ok=True)
        for name in ("koi8r", "utf8"):
            with open(os.path.join(dst, "cli-%s.bin" % name), "wb") as fh:
                fh.write(FIX[name])
        return yn(all(os.path.exists(os.path.join(dst, "cli-%s.bin" % n))
                      for n in ("koi8r", "utf8")))
    measure("cli_fixtures_handed_over", hand_over)
else:
    emit("cli_fixtures_handed_over", "not-asked")

# ------------------------------------------------------------------ hygiene --
try:
    shutil.rmtree(W)
    emit("workdir_removed", yn(not os.path.exists(W)))
except Exception as exc:                           # noqa: BLE001
    emit("workdir_removed", "EXC:%s" % type(exc).__name__)

emit("probe_done", "yes")
emit("emitted_count", str(len(OUT) + 1))
print("\n".join(OUT))