"""Drive wcwidth over a table of code points and strings; print one key=value per line. ONE INTERPRETER START DOES THE WHOLE OF IT. Every python3 start on this arm is a fresh load of one large static image (99-python3 records the cost), so the suite that drives this asserts on the OUTPUT rather than calling python3 once per case. Every line is `key=value'; a key is emitted at most once. The suite reads the keys it names and ignores the rest, so adding a fact here cannot break it. Failure is reported as a key too -- `*_error' -- rather than only as an exit status, because the case that reds should say which stage stopped and why. WHAT THIS PACKAGE ACTUALLY IS, AND WHY THE CASES LOOK LIKE THIS. wcwidth answers one question -- how many terminal COLUMNS does this character occupy -- and the answer is neither its byte count nor its code-point count. Three classes make that true at once: a CJK ideograph occupies TWO cells, a combining mark occupies ZERO, and a C0 control character occupies an indeterminate number, for which the POSIX signature returns -1. So the cases are a TABLE over those classes rather than a narrative, and the -1 cases are the ones naive width code gets wrong. THE PACKAGE IS ITS DATA. wcwidth.py is 14 KB of binary search over 467 KB of GENERATED Unicode tables -- table_zero.py alone is 359 KB, and it carries a separate range list for each of the nineteen Unicode version levels. So the port is not "does the module import", it is "did the tables land, whole, for every level". `zero_ranges_*' / `wide_ranges_*' pin their sizes, `table_monotonic' walks every range of every level checking the invariant `_bisearch' relies on -- a binary search over an unsorted table returns a WRONG ANSWER rather than an error -- and the version-scoped width cases prove the older levels are real data rather than aliases of the latest. THE CONTROL IS THE STDLIB, NOT A TRANSCRIBED LITERAL. `ctl_eaw_bad' cross-checks wcwidth against `unicodedata.east_asian_width' and `ctl_comb_bad' against `unicodedata.combining' -- an INDEPENDENT reading of the same question out of the interpreter's own UCD, computed on target rather than typed here. Both are scoped to long-stable code points so the comparison survives the UCD version skew between the control interpreter and the image (see `unidata_version'), and a genuine disagreement is what they exist to surface. EVERY CONTROL CHARACTER BELOW IS WRITTEN AS AN ESCAPE, never as a literal byte. A literal ESC or NUL in this file would be invisible in a diff and would be mangled by anything that transports the suite; the escapes say exactly which code point is meant. """ import os import sys _seen = set() def emit(key, value): if key in _seen: return _seen.add(key) # One line per fact: the suite greps by key, and an embedded newline would # land as a second, unkeyed line. print("%s=%s" % (key, str(value).replace("\n", " ").replace("\r", " "))) def fail(key, exc): emit(key, "%s: %s" % (type(exc).__name__, exc)) raise SystemExit(1) # ------------------------------------------------------------------ the import try: import wcwidth from wcwidth import wcwidth as ww, wcswidth as wsw, list_versions except Exception as exc: # noqa: BLE001 - reported, not raised fail("import_error", exc) emit("module_file", getattr(wcwidth, "__file__", "")) # IT IS A PACKAGE, NOT A SINGLE MODULE. setup.py declares `packages=['wcwidth']', # so what lands in site-packages is a DIRECTORY of six modules -- the opposite # shape from python3-jstyleson's one file, and the reason the tables can be # separate modules at all. A release that collapsed it would still import and # still pass every behavioural case below while the shape this image ships had # changed. emit("is_package", "yes" if hasattr(wcwidth, "__path__") else "no") # THE VERSION, TWICE, FROM TWO INDEPENDENT PLACES. Unlike python3-jstyleson this # module DOES declare `__version__' (setup.py's comment says it is hand-managed # there), so it can be cross-checked against the installed dist-info. A module # that landed while its dist-info did not -- or a dist-info describing some other # release -- is what the pair catches. emit("version_attr", getattr(wcwidth, "__version__", "")) try: from importlib.metadata import version as _dist_version emit("dist_version", _dist_version("wcwidth")) except Exception as exc: # noqa: BLE001 emit("dist_version_error", "%s: %s" % (type(exc).__name__, exc)) # THE PUBLIC API, as the module itself declares it. This module DOES define # `__all__', so it is asserted directly rather than reconstructed from dir(): # `__all__' is the package's own statement of what it exports, and a release that # widened or narrowed it says so here. emit("all_names", ",".join(wcwidth.__all__)) for _n in ("wcwidth", "wcswidth", "list_versions"): emit("has_" + _n, "yes" if hasattr(wcwidth, _n) else "no") # `wcwidth' is lru_cache-decorated (maxsize=1000) and that is load-bearing rather # than incidental: wcswidth calls it once per character, so on an arm where the # interpreter is this expensive a release that dropped the cache would be a # measurable regression in every table formatter downstream. cache_info is the # decorator's own footprint. emit("has_cache_info", "yes" if hasattr(ww, "cache_info") else "no") # ============================================================================ # THE VERSION LEVELS. # ============================================================================ # # wcwidth ships a separate table for each level and lets the caller pick one, so # the level list is part of the payload rather than metadata about it. _v = list_versions() emit("versions_count", len(_v)) emit("versions_first", _v[0]) emit("versions_last", _v[-1]) emit("versions_joined", ",".join(_v)) # The nearest-match rule, which is what an arbitrary caller-supplied version goes # through. Each case is a different branch of it: a value BETWEEN two supported # levels rounds DOWN, a short dotted form matches on its leading pieces, a value # below every level clamps to the earliest, and an unparseable one falls back to # the latest with a warning. These are upstream's own documented examples. try: emit("match_499", wcwidth._wcmatch_version("4.9.9")) emit("match_80", wcwidth._wcmatch_version("8.0")) emit("match_1", wcwidth._wcmatch_version("1")) emit("match_latest", wcwidth._wcmatch_version("latest")) emit("version_value_1510", ",".join(str(x) for x in wcwidth._wcversion_value("15.1.0"))) except Exception as exc: # noqa: BLE001 emit("match_error", "%s: %s" % (type(exc).__name__, exc)) # An unparseable level WARNS rather than raising, and still answers. A caller who # set UNICODE_VERSION to something wrong gets the latest tables and a warning on # stderr -- not a traceback out of a width calculation -- and that is a promise # worth pinning, because the alternative failure mode is a crash inside somebody # else's terminal renderer. try: import warnings with warnings.catch_warnings(record=True) as _w: warnings.simplefilter("always") emit("match_bogus", wcwidth._wcmatch_version("bogus")) emit("match_bogus_warned", "yes" if _w else "no") except Exception as exc: # noqa: BLE001 emit("match_bogus_error", "%s: %s" % (type(exc).__name__, exc)) # ============================================================================ # THE TABLES: THE ACTUAL PAYLOAD. # ============================================================================ # # Their SIZES, at the latest level and at the earliest, because equal sizes at # both would be the signature of a build that shipped one table nineteen times. try: emit("zero_keys", len(wcwidth.ZERO_WIDTH)) emit("wide_keys", len(wcwidth.WIDE_EASTASIAN)) emit("vs16_keys", ",".join(sorted(wcwidth.VS16_NARROW_TO_WIDE))) emit("zero_ranges_latest", len(wcwidth.ZERO_WIDTH["15.1.0"])) emit("wide_ranges_latest", len(wcwidth.WIDE_EASTASIAN["15.1.0"])) emit("zero_ranges_410", len(wcwidth.ZERO_WIDTH["4.1.0"])) emit("wide_ranges_410", len(wcwidth.WIDE_EASTASIAN["4.1.0"])) emit("vs16_ranges", len(wcwidth.VS16_NARROW_TO_WIDE["9.0.0"])) except Exception as exc: # noqa: BLE001 emit("tables_error", "%s: %s" % (type(exc).__name__, exc)) # ...AND THEIR STRUCTURAL INVARIANT, over every range of every level. # # `_bisearch' is a binary search, so it does not merely need the data present -- # it needs each table sorted and non-overlapping. Handed a table that is not, it # returns a WRONG ANSWER rather than an error, which is the one failure mode a # behavioural case sampling thirty code points would miss on hundreds of ranges. # This is also the part of upstream's own tests/test_table_integrity.py that can # be run without pytest, and it is the reason this suite is not an import check. try: _bad = [] for _name, _tbl in (("zero", wcwidth.ZERO_WIDTH), ("wide", wcwidth.WIDE_EASTASIAN), ("vs16", wcwidth.VS16_NARROW_TO_WIDE)): for _lvl, _ranges in _tbl.items(): _prev = -1 for _r in _ranges: if len(_r) != 2 or _r[0] > _r[1] or _r[0] <= _prev: _bad.append("%s/%s" % (_name, _lvl)) break _prev = _r[1] emit("table_monotonic", "ok" if not _bad else ",".join(sorted(set(_bad)))) except Exception as exc: # noqa: BLE001 emit("table_monotonic_error", "%s: %s" % (type(exc).__name__, exc)) # ============================================================================ # wcwidth(): ONE CHARACTER, THE THREE CLASSES AND THE -1. # ============================================================================ # # Named rather than looped over a list of ordinals, so a red says WHICH class # broke. The grouping is the whole point: 1 is the uninteresting case, 2 says the # east-asian table answered, 0 says the zero-width table answered, and -1 is the # one that is not a width at all. CHARS = ( # printable, one cell ("space", " "), ("letter_a", "a"), ("tilde", "~"), ("nbsp", " "), ("eacute", "é"), ("box", "─"), ("greek_alpha", "α"), ("snowman", "☃"), ("halfwidth_ka", "カ"), # two cells: the east-asian table ("cjk", "一"), ("hiragana", "あ"), ("fullwidth_a", "A"), ("hangul", "가"), ("emoji_grin", "\U0001f600"), ("cjk_extb", "\U00020000"), # zero cells: the zero-width table ("comb_acute", "́"), ("zwsp", "​"), ("zwj", "‍"), ("vs16", "️"), ("soft_hyphen", "­"), ("virama", "्"), # not a width: C0/C1 controls and DEL ("soh", "\x01"), ("bel", "\x07"), ("tab", "\t"), ("lf", "\n"), ("esc", "\x1b"), ("del", "\x7f"), ("c1_80", "\x80"), ("c1_9f", "\x9f"), # the two that are neither: NUL has no printable effect, and the empty # string is not a character at all. Both answer 0, by separate paths inside # the function, and both are the arguments a caller reaches by accident. ("nul", "\x00"), ("empty", ""), ) for _name, _c in CHARS: try: emit("w_" + _name, ww(_c)) except Exception as exc: # noqa: BLE001 emit("w_" + _name, "%s: %s" % (type(exc).__name__, exc)) # ============================================================================ # wcswidth(): THE STRING, WHERE THE CLASSES COMBINE. # ============================================================================ # # `combining' is the case the package exists for: "e" + COMBINING ACUTE is TWO # code points and ONE column, so any caller measuring len() is already wrong. # The C0 rule is the POSIX contract that surprises people -- ONE control anywhere # in the string makes the whole answer -1, not the width of the rest -- and it is # asserted in three positions because an implementation that checked only the # first character would pass one of them. STRINGS = ( ("ascii", "hello"), ("empty", ""), ("cjk", "一二三"), ("mixed", "a一b"), ("combining", "é"), ("wide_comb", "一́"), ("zwsp_mid", "a​b"), ("nul_mid", "a\x00b"), ("c0_first", "\x01ab"), ("c0_mid", "a\x01b"), ("c0_last", "ab\x01"), # ZWJ: the joiner and THE CHARACTER AFTER IT are both skipped, so a # man+ZWJ+woman sequence measures as the man alone. That is upstream's rule # for emoji sequences a terminal renders as one glyph. ("zwj_emoji", "\U0001f468‍\U0001f469"), # VS16 after a narrow character that the VS16 table names PROMOTES it to two # cells; VS16 with nothing before it adds nothing. ("vs16_narrow", "⚠️"), ("vs16_alone", "️"), ) for _name, _s in STRINGS: try: emit("s_" + _name, wsw(_s)) except Exception as exc: # noqa: BLE001 emit("s_" + _name, "%s: %s" % (type(exc).__name__, exc)) # THE `n' ARGUMENT, which exists only for compatibility with the C signature: # measure the first n characters. Over "ab" the running total steps # 0,1,3,4,6 -- the odd steps are the ASCII characters and the even ones the # ideographs -- so the sequence states BOTH that n is honoured and that it counts # CHARACTERS while the answer counts COLUMNS. Getting those two confused is the # bug this argument invites. NSTR = "a一b一" for _n in range(0, 5): try: emit("n_%d" % _n, wsw(NSTR, _n)) except Exception as exc: # noqa: BLE001 emit("n_%d" % _n, "%s: %s" % (type(exc).__name__, exc)) try: emit("n_neg", wsw("abc", -1)) except Exception as exc: # noqa: BLE001 emit("n_neg", "%s: %s" % (type(exc).__name__, exc)) # AN UPSTREAM LIMITATION, ASSERTED AS ONE. POSIX wcswidth() bounds n by the # string; 0.2.13 indexes the string directly, so n GREATER THAN len() raises # IndexError out of a width calculation. That is upstream's behaviour on x86-64 # and it is not this platform's. It is asserted so the port ships a statement # about what it does rather than a silence, and so a release that fixed it # announces itself here rather than somewhere a caller would find it. try: emit("n_beyond", wsw("ab", 10)) except Exception as exc: # noqa: BLE001 emit("n_beyond", type(exc).__name__) # ============================================================================ # THE VERSION-SCOPED WIDTHS: PROOF THE OLDER TABLES ARE REAL. # ============================================================================ # # U+1F600 GRINNING FACE became WIDE in Unicode 9.0.0. Asking for its width at # 4.1.0 and at 8.0.0 must answer 1 and at 9.0.0 and later 2 -- so this pair says # the nineteen shipped tables are nineteen DIFFERENT tables rather than nineteen # references to the latest, which no single-level case could distinguish. The # CJK ideograph is the control: it has been wide since before 4.1.0 and must # answer 2 at every level. for _lvl in ("4.1.0", "8.0.0", "9.0.0", "13.0.0", "15.1.0"): _k = _lvl.replace(".", "") try: emit("v%s_emoji" % _k, ww("\U0001f600", _lvl)) emit("v%s_cjk" % _k, ww("一", _lvl)) emit("v%s_vs16_str" % _k, wsw("⚠️", unicode_version=_lvl)) except Exception as exc: # noqa: BLE001 emit("v%s_error" % _k, "%s: %s" % (type(exc).__name__, exc)) # ============================================================================ # THE STDLIB AS AN INDEPENDENT CONTROL. # ============================================================================ # # Every case above compares wcwidth against a value written down here. These two # compare it against the INTERPRETER'S OWN Unicode database, read on target -- # `east_asian_width' for the two-cell claim and `combining' for the zero-cell # claim -- so they are the cases that would still be meaningful if every literal # above had been transcribed wrong. # # `unicodedata' is a BUILTIN on this image, not a lib-dynload .so: python3 here # is one static binary with an empty lib-dynload (BRINTOS-1476), and 99-python3's # `builtin-modules' case asserts unicodedata is among the modules linked in. So # this control costs nothing and cannot fail for want of an extension module. # # The samples are long-stable code points on purpose. The control interpreter # carries a different UCD level from the image's (see `unidata_version'), and a # character whose class changed between them would red on a version skew rather # than on a defect. Nothing sampled here has moved since Unicode 4.1. try: import unicodedata emit("unidata_version", unicodedata.unidata_version) EAW = (0x41, 0x7e, 0xa0, 0xe9, 0x2500, 0x3b1, 0x4e00, 0x3042, 0xff21, 0xff76, 0xac00, 0x9fa5, 0x30a2, 0x2588, 0x263a, 0x5b57) _bad = [] for _cp in EAW: _c = chr(_cp) _eaw = unicodedata.east_asian_width(_c) _want = 2 if _eaw in ("W", "F") else 1 if ww(_c) != _want: _bad.append("U+%04X:eaw=%s,want=%d,got=%d" % (_cp, _eaw, _want, ww(_c))) emit("ctl_eaw_count", len(EAW)) emit("ctl_eaw_bad", ",".join(_bad)) COMB = (0x300, 0x301, 0x302, 0x64b, 0x94d, 0x5b0) _bad = [] for _cp in COMB: _c = chr(_cp) if unicodedata.combining(_c) == 0: _bad.append("U+%04X:not-combining" % _cp) elif ww(_c) != 0: _bad.append("U+%04X:got=%d" % (_cp, ww(_c))) emit("ctl_comb_count", len(COMB)) emit("ctl_comb_bad", ",".join(_bad)) except Exception as exc: # noqa: BLE001 emit("ctl_error", "%s: %s" % (type(exc).__name__, exc)) # The environment the probe itself ran under, so a red in a version-scoped case # above can be told apart from an inherited UNICODE_VERSION. `auto' reads this # variable, and the suite runs the probe with it unset. emit("env_unicode_version", os.environ.get("UNICODE_VERSION", "")) emit("py_version", "%d.%d" % sys.version_info[:2]) emit("probe_done", "yes")