"""Drive jsonref over documents held in a scratch tree, and print one key=value per line. ONE INTERPRETER START DOES THE WHOLE OF IT. Every python3 start on this arm is a fresh load of one large static image (99-python3 records the cost), so the suite that drives this asserts on the OUTPUT rather than calling python3 once per case. Every line is `key=value`; a key is emitted at most once. The suite reads the keys it names and ignores the rest, so adding a fact here cannot break it. Failure is reported as a key too -- `*_error` -- rather than only as an exit status, because the case that reds should say which stage stopped and why. THE STAGES ARE INDEPENDENT, AND THAT IS THE POINT. The `file:' stage is the one this port expects to be able to fail on this platform (it is the only stage that leaves the interpreter -- see the suite header), so every stage is wrapped separately: a red there must not take the local-reference, chain, recursion, laziness and error stages down with it, because those are the ones that say the LIBRARY works. VALUES ARE EMITTED AS json.dumps WITH SORTED KEYS, NEVER AS A repr. A resolved reference can be a dict or a list, and a Python repr of either carries quoting that is Python's rather than JSON's -- and for a whole-document reference it would carry DICT ORDER, which is a property of the parse rather than of the reference. sort_keys makes those cases assertions about CONTENT. NOTHING HERE REACHES THE NETWORK. The remote-reference stages use a loader of this probe's own -- a plain callable returning a dict -- and the one stage that does I/O reads a file this probe just wrote under TMPDIR. jsonref's own default loader would go to urlopen(); it is never invoked with an http URI. """ import hashlib import json import os import shutil import sys import tempfile _seen = set() def emit(key, value): if key in _seen: return _seen.add(key) # One line per fact: the suite greps by key, and an embedded newline would # land as a second, unkeyed line. print("%s=%s" % (key, str(value).replace("\n", " ").replace("\r", " "))) def fail(key, exc): emit(key, "%s: %s" % (type(exc).__name__, exc)) raise SystemExit(1) def jout(value): """Canonical JSON: sorted keys, no incidental whitespace. A JsonRef proxy is not a dict, so json.dumps would reach the encoder's default() and raise; `dumps' from jsonref would re-serialise it back to its `$ref' form, which is the opposite of what a value case wants. The proxy is therefore collapsed to the data it stands for first.""" if hasattr(value, "__subject__"): value = value.__subject__ return json.dumps(value, sort_keys=True, separators=(",", ":")) def digest(text): return hashlib.sha256(text.encode("utf-8")).hexdigest()[:16] # ------------------------------------------------------------------ the import try: import jsonref from jsonref import (JsonRef, JsonRefError, replace_refs, load, loads, load_uri, dump, dumps, jsonloader, URIDict) except Exception as exc: # noqa: BLE001 - reported, not raised fail("import_error", exc) emit("version", jsonref.__version__) emit("module_file", getattr(jsonref, "__file__", "")) # THE SECOND MODULE, WHICH IS THE PACKAGING RISK. jsonref.py opens with a # top-level absolute `from proxytypes import LazyProxy' -- so `proxytypes' is a # separate importable this package must also install, under a name generic # enough to collide. The import above already proves it resolved; this says out # of WHICH file, so a proxytypes belonging to something else is distinguishable. try: import proxytypes emit("proxytypes_file", getattr(proxytypes, "__file__", "")) emit("proxytypes_lazyproxy", "yes" if hasattr(proxytypes, "LazyProxy") else "no") # The identity that says jsonref got its proxy base from THIS module rather # than carrying a vendored copy of its own. emit("proxytypes_is_base", "yes" if JsonRef.__mro__[1] is proxytypes.LazyProxy else "no") except Exception as exc: # noqa: BLE001 emit("proxytypes_error", "%s: %s" % (type(exc).__name__, exc)) # The installed metadata, read independently of the module's own __version__. try: from importlib.metadata import version as _dist_version emit("dist_version", _dist_version("jsonref")) except Exception as exc: # noqa: BLE001 emit("dist_version_error", "%s: %s" % (type(exc).__name__, exc)) # The public surface, one key each. This module declares NO __all__, so there is # no whole-surface string to assert -- dir() would report its own imports # (`json', `functools', `urlparse', `warnings', the `requests' probe) and a case # over it would be pinning implementation detail. for _name in ("replace_refs", "load", "loads", "load_uri", "dump", "dumps", "jsonloader", "JsonRef", "JsonRefError", "URIDict"): emit("has_" + _name, "yes" if hasattr(jsonref, _name) else "no") # The exception's BASE, which is what a caller's `except' clause names. A release # re-basing it on ValueError would leave every case below passing while breaking # every caller that catches it -- including the miss-* cases here, which catch # the class by name and would not notice. emit("exc_base", JsonRefError.__bases__[0].__name__) # WHICH FETCH PATH THE DEFAULT LOADER WOULD TAKE. jsonref tries `import requests' # at module scope and falls back to urllib, recording the answer in a module # global. This image installs no requests, so the fallback is the live path -- # and that is a fact worth pinning, because a future image that grew requests # would change how every remote reference is fetched without changing one line # of this package. emit("requests_backend", "none" if jsonref.requests is None else "requests") # ============================================================================ # STAGE 1 -- A LOCAL REFERENCE, AND THE ROUND TRIP BACK TO `$ref'. # ============================================================================ # # The fixture is inline and pinned by digest below: a shipped .json could drift # by an editor's trailing newline and turn a correct library into a red. LOCAL_DOC = { "definitions": {"widget": {"kind": "widget", "size": 3}}, "here": {"$ref": "#/definitions/widget"}, } LOCAL_TEXT = json.dumps(LOCAL_DOC, sort_keys=True, separators=(",", ":")) emit("local_fixture_len", len(LOCAL_TEXT)) emit("local_fixture_sha", digest(LOCAL_TEXT)) try: _r = replace_refs(LOCAL_DOC) # It is a PROXY, not the value copied into place. The distinction is the # whole package: a copy would serialise as the value and would not stay in # step with a document that is still being edited. emit("local_type", type(_r["here"]).__name__) # ...and the proxy COMPARES EQUAL to what it points at, which is what makes # it usable without the caller knowing a reference was ever there. emit("local_equal", "yes" if _r["here"] == LOCAL_DOC["definitions"]["widget"] else "no") # Subscripting through the proxy reaches the referent's members. emit("local_kind", _r["here"]["kind"]) emit("local_size", jout(_r["here"]["size"])) emit("local_value", jout(_r["here"])) # THE ROUND TRIP. jsonref.dumps restores every proxy to the `$ref' object it # was built from, so a document read, walked and written back is unchanged. # This is the property that lets a config file survive a tool that only meant # to read it -- and json.dumps CANNOT do it (the next key says so). emit("roundtrip", dumps(_r, sort_keys=True, separators=(",", ":"))) emit("roundtrip_identical", "yes" if dumps(_r, sort_keys=True, separators=(",", ":")) == LOCAL_TEXT else "no") except Exception as exc: # noqa: BLE001 emit("local_error", "%s: %s" % (type(exc).__name__, exc)) # The stdlib encoder has no idea what a proxy is, so it raises. This is the case # that says the round trip above is jsonref's doing rather than something that # would have happened anyway. try: json.dumps(replace_refs(LOCAL_DOC)) emit("plain_dumps", "encoded") except Exception as exc: # noqa: BLE001 emit("plain_dumps", type(exc).__name__) # `loads' is the drop-in for json.loads: parse and replace in one call. Asserted # because it is the entry point a caller actually uses, and it takes a different # path into _replace_refs than the replace_refs() above. try: _l = loads(LOCAL_TEXT) emit("loads_kind", _l["here"]["kind"]) emit("loads_type", type(_l["here"]).__name__) except Exception as exc: # noqa: BLE001 emit("loads_error", "%s: %s" % (type(exc).__name__, exc)) # ============================================================================ # STAGE 2 -- A CHAIN OF THREE REFERENCES. # ============================================================================ # # `a' points at `b', which points at `c', which points at the value. Resolving # `a' has to follow all three, and each hop is a separate proxy resolving during # the previous one's resolution. A library that dereferenced one level would # return a `$ref' object here rather than the string. CHAIN_DOC = {"a": {"$ref": "#/b"}, "b": {"$ref": "#/c"}, "c": {"$ref": "#/end"}, "end": "terminal"} try: _c = replace_refs(CHAIN_DOC) emit("chain_value", jout(_c["a"])) emit("chain_equal", "yes" if _c["a"] == "terminal" else "no") # The middle hops resolve to the same terminal, which says the chain is # followed from wherever it is entered rather than only from its head. emit("chain_mid", jout(_c["b"])) except Exception as exc: # noqa: BLE001 emit("chain_error", "%s: %s" % (type(exc).__name__, exc)) # ============================================================================ # STAGE 3 -- A `file:' REFERENCE INTO A SCRATCH TREE. # ============================================================================ # # THE ONLY STAGE THAT LEAVES THE INTERPRETER. Everything above resolves inside # one parsed document; this one goes through jsonref's default loader, which is # urlopen() over a `file:' URI -- so it exercises urllib's FileHandler, and with # it this platform's open/read on the scratch filesystem. It is wrapped on its # own because a platform gap here says nothing about the stages above. # # base_uri is the scratch DIRECTORY with a trailing slash: urljoin resolves the # relative `other.json' against it, and without the slash the last path segment # would be replaced rather than appended. try: _d = tempfile.mkdtemp(prefix="jsonref-probe.") try: _other = {"colour": "green", "nested": {"deep": [10, 20, 30]}} _p = os.path.join(_d, "other.json") with open(_p, "w") as _f: json.dump(_other, _f) emit("file_fixture_written", "yes" if os.path.isfile(_p) else "no") _base = "file://" + _d + "/" emit("file_base_uri_shape", "dir-with-slash" if _base.startswith("file://") and _base.endswith("/") else "other") _main = { "colour": {"$ref": "other.json#/colour"}, "deep": {"$ref": "other.json#/nested/deep/1"}, "whole": {"$ref": "other.json"}, } _fr = replace_refs(_main, base_uri=_base) # A fragment into the second file: the reference names both a document # and a pointer within it, and both halves have to work. emit("file_colour", jout(_fr["colour"])) # ...and an ARRAY INDEX inside that fragment, which is the token type a # pointer resolver treats differently from a mapping key. emit("file_deep", jout(_fr["deep"])) # A reference with NO fragment is the whole of the other document. emit("file_whole", jout(_fr["whole"])) # The round trip holds across files too: the written form is the # relative reference, not the file's contents inlined. emit("file_roundtrip", dumps(_fr, sort_keys=True, separators=(",", ":"))) finally: shutil.rmtree(_d, ignore_errors=True) except Exception as exc: # noqa: BLE001 emit("file_error", "%s: %s" % (type(exc).__name__, exc)) # ============================================================================ # STAGE 4 -- THE CASE THE PACKAGE EXISTS FOR: A REFERENCE TO ITS OWN ANCESTOR. # ============================================================================ # # `#' addresses the whole document, and it appears INSIDE that document. Resolved # eagerly this is an infinite regress -- the resolver would substitute the # document into itself until the stack ends. Lazily it is a finite object that # can be walked to any depth on demand, which is exactly how a recursive JSON # Schema (a node whose children are nodes) is represented. # # So the assertions are: it LOADS at all, and then it can be walked -- one level, # and a second level, each reaching the same name. The second level is the one # that says the proxy is re-entrant rather than merely surviving one hop. RECUR_DOC = {"name": "node", "children": {"type": "array", "items": {"$ref": "#"}}} try: _rr = replace_refs(RECUR_DOC) emit("recur_loaded", "yes") emit("recur_top_name", jout(_rr["name"])) emit("recur_one_level", jout(_rr["children"]["items"]["name"])) emit("recur_two_levels", jout(_rr["children"]["items"]["children"]["items"]["name"])) # ...and it still writes back as `{"$ref": "#"}' rather than as whatever # finite prefix happened to be walked, which is the property that makes a # recursive schema editable. emit("recur_roundtrip", dumps(_rr, sort_keys=True, separators=(",", ":"))) except RecursionError as exc: # named: this is THE failure mode emit("recur_error", "RecursionError: %s" % exc) except Exception as exc: # noqa: BLE001 emit("recur_error", "%s: %s" % (type(exc).__name__, exc)) # ============================================================================ # STAGE 5 -- LAZINESS, MEASURED RATHER THAN ASSERTED. # ============================================================================ # # Stage 4 shows a recursive document surviving, which is evidence that resolution # is deferred but not a measurement of it. A COUNTING LOADER is: it records every # call, so the number of loads before and after the first access is the whole # statement, and it needs neither network nor filesystem. # # Four facts, and the last two are what make the first two mean something: # after replace_refs -- 0, nothing was fetched at load time # after first access -- 1, the fetch happened when the value was asked for # after a SECOND $ref to the same URI -- still 1, the store is shared # after re-access -- still 1, LazyProxy caches on first use _calls = [] def counting_loader(uri, **_kwargs): _calls.append(uri) return {"payload": "fetched"} LAZY_DOC = {"a": {"$ref": "remote.json"}, "b": {"$ref": "remote.json"}} try: _lz = replace_refs(LAZY_DOC, base_uri="file:///base/", loader=counting_loader) emit("lazy_calls_after_load", len(_calls)) emit("lazy_value", jout(_lz["a"]["payload"])) emit("lazy_calls_after_access", len(_calls)) emit("lazy_uri_requested", _calls[0] if _calls else "") _ = _lz["b"]["payload"] emit("lazy_calls_shared_store", len(_calls)) _ = _lz["a"]["payload"] emit("lazy_calls_reaccess", len(_calls)) except Exception as exc: # noqa: BLE001 emit("lazy_error", "%s: %s" % (type(exc).__name__, exc)) # THE CONTRAST, which is what says laziness is a CHOICE this library makes and # not an accident of how the fixture was built. Same document, same loader, both # opt-outs -- and each fetches immediately. try: _calls2 = [] def _loader2(uri, **_kwargs): _calls2.append(uri) return {"payload": "fetched"} replace_refs(LAZY_DOC, base_uri="file:///base/", loader=_loader2, lazy_load=False) emit("eager_calls_after_load", len(_calls2)) except Exception as exc: # noqa: BLE001 emit("eager_error", "%s: %s" % (type(exc).__name__, exc)) # proxies=False replaces each reference with the DATA rather than a proxy, so the # result carries no proxy at all -- the type case is the one that says so, and it # is also the shape a caller wants when the document is about to be handed to # something that type-checks. try: _calls3 = [] def _loader3(uri, **_kwargs): _calls3.append(uri) return {"payload": "fetched"} _np = replace_refs(LAZY_DOC, base_uri="file:///base/", loader=_loader3, proxies=False) emit("noproxy_calls_after_load", len(_calls3)) emit("noproxy_type", type(_np["a"]).__name__) except Exception as exc: # noqa: BLE001 emit("noproxy_error", "%s: %s" % (type(exc).__name__, exc)) # ============================================================================ # STAGE 6 -- A REFERENCE THAT NAMES NOTHING, WHICH RAISES AT ACCESS TIME. # ============================================================================ # # WHERE it raises is as much the contract as THAT it raises. Loading a document # whose reference is unresolvable must succeed -- otherwise a schema with one bad # `$ref' in a branch nobody touches could not be opened at all -- and the error # has to arrive when the bad branch is actually read. MISS_DOC = {"good": {"kind": "fine"}, "oops": {"$ref": "#/definitions/nope"}} try: _mr = replace_refs(MISS_DOC) emit("miss_load", "ok") emit("miss_type_before_access", type(_mr["oops"]).__name__) # The rest of the document is still usable, which is the practical half of # deferring the error. emit("miss_sibling_readable", jout(_mr["good"]["kind"])) try: _mr["oops"]["anything"] emit("miss_raised", "no") except JsonRefError as exc: emit("miss_raised", "yes") emit("miss_class", type(exc).__name__) # The message carries the full reference URI and the pointer that failed. # It interpolates NO document repr, so asserting it does not pin dict # ordering or the fixture's shape into a case about a lookup. emit("miss_message", str(exc)) # The original reference object is kept on the exception, which is what # lets a caller report WHICH `$ref' in its input was wrong. emit("miss_reference", jout(exc.reference)) # ...and the underlying lookup failure is chained, not swallowed. emit("miss_cause", type(exc.cause).__name__) except Exception as exc: # noqa: BLE001 emit("miss_raised", "wrong-class") emit("miss_class", type(exc).__name__) except Exception as exc: # noqa: BLE001 emit("miss_error", "%s: %s" % (type(exc).__name__, exc)) emit("probe_complete", "yes")