"""Drive u-msgpack-python's MessagePack encoder and decoder and print one key=value per line. ONE INTERPRETER START DOES THE WHOLE OF IT. Every python3 start on this arm is a fresh load of one large static image (99-python3 records the cost), so the suite that drives this asserts on the OUTPUT rather than calling python3 once per case. Every line is `key=value`; a key is emitted at most once. The suite reads the keys it names and ignores the rest, so adding a fact here cannot break it. Failure is reported as a key too -- `*_error` -- rather than only as an exit status, because the case that reds should say which stage stopped and why. WHAT THIS PROBE IS FOR, AND WHY THE EXPECTATIONS ARE SPEC LITERALS. u-msgpack-python is pure Python with no accelerator and no optional C half, so there is nothing between the source and the bytes. The MessagePack specification fixes those bytes exactly -- big-endian lengths, one format byte per form, the boundary at which each form gives way to the next -- so the expectation for `packb` is not "whatever this library produced on a control host", it is what the SPECIFICATION says. That is why the encodings below are asserted as literal hex by the suite rather than as a digest of a control run: a wrong answer here is wrong against msgpack/msgpack spec.md, not merely different from x86-64. THE VECTORS SIT ON THE ENCODING BOUNDARIES, BOTH SIDES. A serialiser that got a `<` where the spec says `<=` produces correct bytes for almost every value and the wrong form for exactly one, so each form is driven at its first value and at its last: 127/128 for fixint->uint8, 65535/65536 for uint16->uint32, 31/32 for fixstr->str8, 15/16 for fixarray->array16, and so on through every form the format has. EACH VECTOR IS ALSO ROUND-TRIPPED, and the two halves answer different questions: `enc` says the encoder agrees with the spec, `rt` says the decoder reads its own output back to the value that went in. A library whose encoder and decoder were wrong in the same direction passes `rt` and fails `enc`, which is the whole reason the literal bytes are asserted rather than only the round trip. THE THREE 32-BIT FORMS ARE ASSERTED BY PREFIX, LENGTH AND DIGEST rather than by their whole encoding: str32, bin32, array32, map32 and ext32 all need >= 65536 elements or bytes, and a 65 kB hex literal in a shell suite would be unreadable and unmaintainable. The prefix is the part the spec fixes -- the format byte and the big-endian 32-bit length -- and the length and digest pin the rest. WHAT IS DELIBERATELY NOT MEASURED: the `huge string' / `huge binary string' / `huge array' / `huge map' guards. Each needs an object of 2**32 elements or bytes -- 4 GiB -- which this image cannot allocate, so the probe cannot reach them and does not pretend to. The integer guards ARE reachable (2**64 is a small object) and are driven below. """ import datetime import hashlib import sys _seen = set() def emit(key, value): if key in _seen: return _seen.add(key) # One line per fact: the suite greps by key, and an embedded newline would # land as a second, unkeyed line. print("%s=%s" % (key, str(value).replace("\n", " ").replace("\r", " "))) def fail(key, exc): emit(key, "%s: %s" % (type(exc).__name__, exc)) raise SystemExit(1) def sha16(data): return hashlib.sha256(data).hexdigest()[:16] # ------------------------------------------------------------------ the import try: import umsgpack except Exception as exc: # noqa: BLE001 - reported, not raised fail("import_error", exc) emit("module_file", umsgpack.__file__) emit("package_dir", umsgpack.__file__.rsplit("/", 1)[0]) # THE VERSION FROM THREE SIDES. The module carries it twice by design -- a string # for humans and a tuple for comparisons, both set at module scope -- and the # installed metadata carries it a third time. Two of the three disagreeing means a # mixed install: a dist-info from one version beside a package from another. emit("version", getattr(umsgpack, "__version__", "")) emit("version_tuple", ".".join(str(n) for n in getattr(umsgpack, "version", ()))) try: from importlib.metadata import version as _dist_version emit("dist_version", _dist_version("u-msgpack-python")) except Exception as exc: # noqa: BLE001 emit("dist_version_error", "%s: %s" % (type(exc).__name__, exc)) # THE PUBLIC SURFACE. umsgpack declares no `__all__', so `dir()' minus the # underscored names is the only statement of surface it has. It is not a static # list: `__init()' runs at import and BINDS `pack'/`packb'/`unpack'/`unpackb', # their `dump'/`dumps'/`load'/`loads' aliases, `compatibility' and `xrange' as # module globals, choosing the Python 2 or Python 3 implementation. So this key is # also the statement that the module's own initialiser RAN -- a package that # imported without it would have the classes and none of the functions. emit("api_surface", ",".join(sorted(n for n in dir(umsgpack) if not n.startswith("_")))) # WHICH IMPLEMENTATION `__init()' PICKED. On Python 3 `packb' must be `_packb3'; # the Python 2 arm returns `str' and would fail every encoding case below with a # type error rather than with a wrong byte. emit("packb_impl", umsgpack.packb.__name__) emit("unpackb_impl", umsgpack.unpackb.__name__) # THE AUTO-DETECTED FLOAT PRECISION, WHICH IS A FACT ABOUT THIS ARM'S INTERPRETER # AND NOT ABOUT THE LIBRARY. `__init()' reads `sys.float_info.mant_dig' and packs # a Python float as float64 when it is 53 and as float32 otherwise -- so an # interpreter whose double were not IEEE-754 binary64 would silently change the # WIRE FORMAT of every float this image writes. It is emitted, and the suite # asserts it, because that is the one platform property this port depends on. emit("float_precision", umsgpack._float_precision) emit("mant_dig", sys.float_info.mant_dig) # The old-spec toggle is a module global, default off. Every vector below is # written against the CURRENT specification, so a build that shipped it on would # encode str as the pre-2013 raw form and red across the string cases. emit("compatibility_default", umsgpack.compatibility) # ------------------------------------------------------------------- the vectors # (name, value, expected-encoding-is-asserted-by-the-suite). The probe emits the # hex of `packb(value)' and whether `unpackb' returned an equal value; the suite # carries the spec literal each must equal. VECTORS = [ # positive fixint: 0x00 - 0x7f, the value IS the byte ("fixint-min", 0), ("fixint-max", 127), # uint 8/16/32/64: 0xcc 0xcd 0xce 0xcf, big-endian payload ("uint8-min", 128), ("uint8-max", 255), ("uint16-min", 256), ("uint16-max", 65535), ("uint32-min", 65536), ("uint32-max", 4294967295), ("uint64-min", 4294967296), ("uint64-max", 2 ** 64 - 1), # negative fixint: 0xe0 - 0xff, -32 .. -1 ("negfixint-max", -1), ("negfixint-min", -32), # int 8/16/32/64: 0xd0 0xd1 0xd2 0xd3, two's complement big-endian ("int8-max", -33), ("int8-min", -128), ("int16-max", -129), ("int16-min", -32768), ("int32-max", -32769), ("int32-min", -2 ** 31), ("int64-max", -2 ** 31 - 1), ("int64-min", -2 ** 63), # the singletons ("nil", None), ("false", False), ("true", True), # float 64 -- the arm's auto-detected precision, asserted above ("float64-one", 1.0), ("float64-half", 0.5), ("float64-neg", -2.0), # str: fixstr 0xa0|len, then 0xd9 0xda 0xdb with 8/16/32-bit lengths ("fixstr-empty", ""), ("fixstr-max", "a" * 31), ("str8-min", "a" * 32), ("str8-max", "a" * 255), ("str16-min", "a" * 256), # ...and UTF-8 is measured in BYTES, not in characters: this one is three # characters and six bytes, so a length taken from len(str) would emit 0xa3. ("fixstr-utf8", "éèê"), # bin: 0xc4 0xc5 0xc6. There is no "fixbin" -- the shortest form still # carries an explicit 8-bit length, which is what separates bin from str. ("bin8-empty", b""), ("bin8-max", b"\x01" * 255), ("bin16-min", b"\x01" * 256), # array: fixarray 0x90|len, then 0xdc 0xdd ("fixarray-empty", []), ("fixarray-max", [None] * 15), ("array16-min", [None] * 16), ("array-nested", [1, [2, 3], "a"]), # map: fixmap 0x80|len, then 0xde 0xdf. Insertion order is what `packb' # walks, so these literals are stable for a Python 3.7+ dict. ("fixmap-empty", {}), ("fixmap-one", {"a": 1}), ("map16-min", dict((i, None) for i in range(16))), # ext: fixext1/2/4/8/16 are 0xd4-0xd8 (LENGTH IS IMPLIED BY THE FORM), then # ext8/16/32 are 0xc7-0xc9 with an explicit length. A 3-byte payload has no # fixext form and must fall to ext8 -- the boundary a naive implementation # gets wrong. ("fixext1", umsgpack.Ext(5, b"\x01")), ("fixext2", umsgpack.Ext(5, b"\x01\x02")), ("fixext4", umsgpack.Ext(5, b"\x01" * 4)), ("fixext8", umsgpack.Ext(5, b"\x01" * 8)), ("fixext16", umsgpack.Ext(5, b"\x01" * 16)), ("ext8-three", umsgpack.Ext(5, b"\x01" * 3)), ("ext8-max", umsgpack.Ext(5, b"\x01" * 255)), ("ext16-min", umsgpack.Ext(5, b"\x01" * 256)), # ...and a NEGATIVE ext type is written as one signed byte: type -2 is 0xfe. ("fixext1-negtype", umsgpack.Ext(-2, b"\x01")), ] for name, value in VECTORS: key = name.replace("-", "_") try: enc = umsgpack.packb(value) except Exception as exc: # noqa: BLE001 emit("enc_%s_error" % key, "%s: %s" % (type(exc).__name__, exc)) continue emit("enc_%s" % key, enc.hex()) # ...and the same encoding as prefix, length and digest. The vectors whose # payload is a long constant run -- str8-max, str16-min, bin8-max, bin16-min, # ext8-max, ext16-min -- would otherwise need a 500-character hex literal in # the suite to say what the format byte and the big-endian length already # say. Those cases assert these three keys instead; the short vectors assert # `enc_' whole and ignore them. emit("pfx_%s" % key, enc.hex()[:12]) emit("len_%s" % key, len(enc)) emit("sha_%s" % key, sha16(enc)) try: back = umsgpack.unpackb(enc) except Exception as exc: # noqa: BLE001 emit("rt_%s" % key, "%s: %s" % (type(exc).__name__, exc)) continue # `==' and not `is': the point is that the VALUE survived. The type is # asserted separately where it matters (`use_tuple', `use_ordered_dict', and # the bool/int distinction below). emit("rt_%s" % key, "yes" if back == value else "no: %r" % (back,)) # THE BOOL/INT DISTINCTION, WHICH `==' CANNOT SEE. In Python `True == 1', so # `rt_true' would pass on a decoder that returned the integer 1 for 0xc3. This is # the case that says the type came back too -- and it is not pedantry: `_pack3' # tests `isinstance(obj, bool)' BEFORE `isinstance(obj, int)' precisely because # the reverse order would encode True as fixint 1. emit("rt_true_type", type(umsgpack.unpackb(umsgpack.packb(True))).__name__) emit("rt_zero_type", type(umsgpack.unpackb(umsgpack.packb(0))).__name__) # ...and the same distinction on the way out: True must not encode as 0x01. emit("enc_true_not_int", "yes" if umsgpack.packb(True) == b"\xc3" else "no") # THE FORCED SINGLE PRECISION. `force_float_precision' is the one packing option # that changes the format byte, so it is driven here rather than left to the # auto-detected default: 1.0 as float32 is 0xca 3f800000. emit("enc_float32_one", umsgpack.packb(1.0, force_float_precision="single").hex()) emit("enc_float32_forced_double", umsgpack.packb(1.0, force_float_precision="double").hex()) # ------------------------------------------------ the 32-bit forms, by prefix # See the header: these need >= 65536 elements or bytes, so the whole encoding is # not a readable literal. The PREFIX is the part the spec fixes -- format byte # plus big-endian 32-bit length -- and the length and digest pin the rest. BIG = [ ("str32", "a" * 65536, 10), ("bin32", b"\x01" * 65536, 10), ("array32", [None] * 65536, 10), ("map32", dict((i, None) for i in range(65536)), 10), ("ext32", umsgpack.Ext(5, b"\x01" * 65536), 12), ] for name, value, prefix_hexlen in BIG: try: enc = umsgpack.packb(value) except Exception as exc: # noqa: BLE001 emit("big_%s_error" % name, "%s: %s" % (type(exc).__name__, exc)) continue emit("big_%s_prefix" % name, enc.hex()[:prefix_hexlen]) emit("big_%s_len" % name, len(enc)) emit("big_%s_sha" % name, sha16(enc)) try: back = umsgpack.unpackb(enc) except Exception as exc: # noqa: BLE001 emit("big_%s_rt" % name, "%s: %s" % (type(exc).__name__, exc)) continue emit("big_%s_rt" % name, "yes" if back == value else "no") # ---------------------------------------------------- the timestamp extension # WHY THIS RECIPE RDEPENDS ON python3-datetime, DEMONSTRATED. umsgpack imports # `datetime' at MODULE scope and packs a `datetime.datetime' as the reserved ext # type -1, so on an image whose python3 lacked the `datetime' split the package # would not import at all. The spec fixes the 32-bit form as 0xd6 0xff followed # by a big-endian uint32 of seconds since the epoch: 2020-01-01T00:00:00Z is # 1577836800 = 0x5e0be100. _utc = datetime.timezone.utc _ts32 = datetime.datetime(2020, 1, 1, 0, 0, 0, tzinfo=_utc) emit("enc_timestamp32", umsgpack.packb(_ts32).hex()) emit("rt_timestamp32", "yes" if umsgpack.unpackb(umsgpack.packb(_ts32)) == _ts32 else "no") # ...and the 64-bit form the moment microseconds are non-zero: 0xd7 0xff with the # nanoseconds in the top 30 bits and the seconds in the low 34. _ts64 = datetime.datetime(2020, 1, 1, 0, 0, 0, 500000, tzinfo=_utc) emit("enc_timestamp64", umsgpack.packb(_ts64).hex()) emit("rt_timestamp64", "yes" if umsgpack.unpackb(umsgpack.packb(_ts64)) == _ts64 else "no") # The decoded object is TIMEZONE-AWARE UTC, not naive -- a decoder that dropped # the tzinfo would compare unequal above only by luck of the local zone. emit("timestamp_tzinfo", str(umsgpack.unpackb(umsgpack.packb(_ts32)).tzinfo)) # --------------------------------------------------------------- the Ext type # Equality is by (type, data) and NOT by identity, and the hash agrees with it -- # which is what lets an Ext be a dict key. `__ne__' is defined separately in this # module rather than derived, so it is asserted separately too. _e1 = umsgpack.Ext(5, b"\x01\x02") _e2 = umsgpack.Ext(5, b"\x01\x02") _e3 = umsgpack.Ext(6, b"\x01\x02") emit("ext_eq", "yes" if _e1 == _e2 else "no") emit("ext_not_identical", "yes" if _e1 is not _e2 else "no") emit("ext_ne_type", "yes" if _e1 != _e3 else "no") emit("ext_ne_data", "yes" if _e1 != umsgpack.Ext(5, b"\x01\x03") else "no") emit("ext_ne_other", "yes" if _e1 != "not an ext" else "no") emit("ext_hash_eq", "yes" if hash(_e1) == hash(_e2) else "no") emit("ext_hash_usable", "yes" if len({_e1: 1, _e2: 2}) == 1 else "no") emit("ext_str", str(_e1)) emit("ext_type_attr", _e1.type) emit("ext_data_attr", _e1.data.hex()) # The constructor's own guards, which are what keep an out-of-range type from # being silently truncated into the one signed byte the wire format has. for label, args in (("type-not-int", (1.5, b"\x01")), ("type-too-big", (128, b"\x01")), ("type-too-small", (-129, b"\x01")), ("data-not-bytes", (5, "not bytes"))): try: umsgpack.Ext(*args) emit("ext_reject_%s" % label.replace("-", "_"), "no: accepted") except Exception as exc: # noqa: BLE001 emit("ext_reject_%s" % label.replace("-", "_"), type(exc).__name__) # ------------------------------------------------------------- ext_handlers # The documented route for a type the FORMAT does not define. A handler maps a # Python class to an Ext on the way out and an ext type code back to the class on # the way in; nothing else in the library knows about the type. class Point(object): def __init__(self, x, y): self.x, self.y = x, y def __eq__(self, other): return isinstance(other, Point) and (self.x, self.y) == (other.x, other.y) try: _pt = Point(3, 4) _enc = umsgpack.packb(_pt, ext_handlers={ Point: lambda obj: umsgpack.Ext(0x10, umsgpack.packb([obj.x, obj.y])), }) emit("ext_handler_enc", _enc.hex()) _back = umsgpack.unpackb(_enc, ext_handlers={ 0x10: lambda ext: Point(*umsgpack.unpackb(ext.data)), }) emit("ext_handler_rt", "yes" if _back == _pt else "no") emit("ext_handler_rt_type", type(_back).__name__) # WITHOUT the unpack handler the same bytes decode to a plain Ext rather than # failing -- the ext payload is opaque to the decoder, which is the property # that makes an unknown extension forward-compatible rather than fatal. _raw = umsgpack.unpackb(_enc) emit("ext_handler_absent_type", type(_raw).__name__) emit("ext_handler_absent_code", _raw.type) except Exception as exc: # noqa: BLE001 emit("ext_handler_error", "%s: %s" % (type(exc).__name__, exc)) # ...and WITHOUT a pack handler the same object is refused rather than encoded to # something lossy. try: umsgpack.packb(Point(3, 4)) emit("ext_handler_required", "no: accepted") except Exception as exc: # noqa: BLE001 emit("ext_handler_required", type(exc).__name__) # ------------------------------------------------ use_ordered_dict / use_tuple # Both change the TYPE the decoder builds, not the value, so each is asserted by # type name -- an `==' comparison cannot see either (an OrderedDict equals the # dict with the same items, and a tuple does not equal a list, which is why the # tuple case also states the value). _m = umsgpack.packb({"a": 1, "b": 2}) emit("ordered_default_type", type(umsgpack.unpackb(_m)).__name__) emit("ordered_opt_type", type(umsgpack.unpackb(_m, use_ordered_dict=True)).__name__) emit("ordered_opt_keys", ",".join(umsgpack.unpackb(_m, use_ordered_dict=True).keys())) _a = umsgpack.packb([1, [2, 3]]) emit("tuple_default_type", type(umsgpack.unpackb(_a)).__name__) emit("tuple_opt_type", type(umsgpack.unpackb(_a, use_tuple=True)).__name__) # ...and it is DEEP: the nested array is a tuple too. emit("tuple_opt_nested_type", type(umsgpack.unpackb(_a, use_tuple=True)[1]).__name__) emit("tuple_opt_value", str(umsgpack.unpackb(_a, use_tuple=True))) # --------------------------------------------------------- the documented errors # Each of these is behaviour the module DOCUMENTS, so each is asserted as the # named exception rather than tolerated. A decoder that returned a partial value # for any of them would be a silent data-corruption bug in a serialiser. ERRORS = [ # a truncated stream: 0xcd declares a 16-bit length and one byte follows ("truncated", lambda: umsgpack.unpackb(b"\xcd\x01")), # ...and truncated at the very first byte: an empty buffer ("truncated-empty", lambda: umsgpack.unpackb(b"")), # 0xc1 is reserved by the specification and must never appear on the wire ("reserved-code", lambda: umsgpack.unpackb(b"\xc1")), # a map whose KEY is itself a map: dict is unhashable, so the object cannot # be built at all. 0x81 = fixmap(1), 0x80 = fixmap(0) as the key, 0xc0 = nil ("unhashable-key", lambda: umsgpack.unpackb(b"\x81\x80\xc0")), # ...and a map with the SAME key twice, which a dict would silently collapse # to one entry: 0x82 fixmap(2), "a"->1, "a"->2 ("duplicate-key", lambda: umsgpack.unpackb(b"\x82\xa1a\x01\xa1a\x02")), # an integer past what the format can carry, both directions ("huge-uint", lambda: umsgpack.packb(2 ** 64)), ("huge-int", lambda: umsgpack.packb(-2 ** 63 - 1)), # a str whose bytes are not valid UTF-8: 0xa1 fixstr(1) followed by 0xff ("invalid-utf8", lambda: umsgpack.unpackb(b"\xa1\xff")), # an unsupported timestamp: ext type -1 with a 2-byte payload, which is # neither the 32-, 64- nor 96-bit form. 0xd5 = fixext2, 0xff = type -1 ("bad-timestamp", lambda: umsgpack.unpackb(b"\xd5\xff\x00\x00")), # the argument type guard: unpackb takes bytes, not str ("unpackb-str", lambda: umsgpack.unpackb("not bytes")), ] for label, fn in ERRORS: key = "err_" + label.replace("-", "_") try: fn() emit(key, "no: returned without raising") except Exception as exc: # noqa: BLE001 emit(key, type(exc).__name__) # THE EXCEPTION HIERARCHY, WHICH IS THE PART A CALLER WRITES `except' AGAINST. # Catching `UnpackException' has to catch every decode failure above; the module # would still pass every case above with a flat hierarchy and break every caller # that relied on the documented one. emit("exc_pack_base", "yes" if issubclass(umsgpack.UnsupportedTypeException, umsgpack.PackException) else "no") emit("exc_unpack_base", "yes" if all(issubclass(c, umsgpack.UnpackException) for c in ( umsgpack.InsufficientDataException, umsgpack.InvalidStringException, umsgpack.UnsupportedTimestampException, umsgpack.ReservedCodeException, umsgpack.UnhashableKeyException, umsgpack.DuplicateKeyException)) else "no") # `allow_invalid_utf8' is the documented escape from the InvalidStringException # above: the bytes come back wrapped in InvalidString, which is a bytes subclass, # so a caller can inspect what arrived instead of losing it. try: _inv = umsgpack.unpackb(b"\xa1\xff", allow_invalid_utf8=True) emit("invalid_utf8_type", type(_inv).__name__) emit("invalid_utf8_is_bytes", "yes" if isinstance(_inv, bytes) else "no") emit("invalid_utf8_value", _inv.hex()) except Exception as exc: # noqa: BLE001 emit("invalid_utf8_error", "%s: %s" % (type(exc).__name__, exc)) # ------------------------------------------------------- the file-like interface # `pack'/`unpack' are the same codecs over a stream rather than over a buffer, and # they are what `dump'/`load' alias. Driven over io.BytesIO so the assertion is # about the API rather than about the filesystem. try: import io _fp = io.BytesIO() umsgpack.pack({"a": [1, 2]}, _fp) emit("stream_enc", _fp.getvalue().hex()) _fp.seek(0) emit("stream_rt", "yes" if umsgpack.unpack(_fp) == {"a": [1, 2]} else "no") # The four aliases are the SAME objects, not re-implementations. emit("alias_dumps", "yes" if umsgpack.dumps is umsgpack.packb else "no") emit("alias_loads", "yes" if umsgpack.loads is umsgpack.unpackb else "no") emit("alias_dump", "yes" if umsgpack.dump is umsgpack.pack else "no") emit("alias_load", "yes" if umsgpack.load is umsgpack.unpack else "no") except Exception as exc: # noqa: BLE001 emit("stream_error", "%s: %s" % (type(exc).__name__, exc)) # ------------------------------------------------------------ the module closure # WHAT THIS PACKAGE PULLS IN, MEASURED RATHER THAN READ OFF THE RECIPE. This # image's python3 is split into sub-packages (BRINTOS-1476), so a module reached # at import scope that lives outside python3-core is a RUNTIME DEPENDENCY the # recipe has to declare -- `datetime' is exactly that, and is why this recipe # carries `RDEPENDS += python3-datetime'. Every name here is scored against # `sys.stdlib_module_names' so a third-party import could not hide in the list. # The list is umsgpack's OWN module-scope imports, not this probe's -- `hashlib' # is imported above for the digests and is deliberately absent here, or the key # would report the probe's closure as the package's. _closure = sorted(n for n in ("struct", "collections", "datetime", "sys", "io") if n in sys.modules) emit("closure_present", ",".join(_closure)) emit("closure_all_stdlib", "yes" if all(n.split(".")[0] in sys.stdlib_module_names for n in _closure) else "no") emit("datetime_imported", "yes" if "datetime" in sys.modules else "no") emit("done", "yes")