"""One interpreter start for the whole of 99-python3-pickleshare.

Emits `key=value' lines, one per line, in file order; the suite reads them back
with grep.  A python3 start on this arm is tens of seconds, so taking one start
for every measurement here is the difference between a one-minute run and a
half-hour one.

WHAT THIS PACKAGE IS, AND THEREFORE WHAT IS MEASURED.  `PickleShareDB' is a
MutableMapping whose backing store is A DIRECTORY OF PICKLE FILES: one file per
key, written with `pickle.dump(..., protocol=2)'.  That is the whole design, and
it is why the package exists -- because the values live in separate files, two
processes can share a store without a lock, which `shelve' cannot do.  So the
assertions here come in two halves that have to agree: what the MAPPING API
returns, and what actually LANDS ON DISK.  A suite that checked only the first
would pass on an implementation that kept everything in memory.

NO PEER, NO DEVICE, NO TERMINAL.  The "concurrency" this package advertises is
not a second process -- it is a second `PickleShareDB' OBJECT over the same
directory, plus the mtime check that invalidates a stale cache.  Both are
reachable from one interpreter, so the concurrency claim is measured here rather
than assumed, and the suite still needs nothing but a writable $TMPDIR.

THE FIXTURES ARE BUILT HERE, NOT SHIPPED.  Every store this probe reads is one
it wrote a moment earlier under $TMPDIR, which is what lets the cases state
their expectations as literals.  The tree is removed at the end and
`workdir_gone' says so.

TWO RULES THE VALUES OBEY.  The runner parses its input line by line, so no
value may contain a newline -- every value below is a single line, and raw bytes
are emitted as hex.  And no value may depend on the encoding stdout happens to
be in: every one is ASCII.

WHAT IS DELIBERATELY NOT ASSERTED.  `gethashfile()' is built on `hash()' of a
str, which CPython RANDOMISES PER PROCESS unless PYTHONHASHSEED is set.  So the
NAME of the file an `hset' lands in is not reproducible across interpreter
starts and nothing here asserts it -- only its SHAPE (two lowercase hex digits)
and the round trip through it, both of which hold whatever the seed is.  For the
same reason `hget' is never called for a MISSING key without an explicit
default: whether that raises KeyError or returns the module's private sentinel
depends on whether the missing key happens to collide into an existing bucket.
"""

import errno
import os
import pickle
import shutil
import stat
import sys
import tempfile
import time

import pickleshare
from pickleshare import PickleShareDB, PickleShareLink, gethashfile

emitted = []


def emit(key, value):
    emitted.append(key)
    print('%s=%s' % (key, value))


def call(fn, *a, **kw):
    """Return fn(*a) as a string, or `!ExceptionName' if it raised.

    Asserted by exception NAME rather than by message: the message is CPython's
    or the library's and has moved between releases, while the name is the
    contract a caller has to code against."""
    try:
        return fn(*a, **kw)
    except Exception as exc:
        return '!' + type(exc).__name__


def yn(b):
    return 'yes' if b else 'no'


class Thing(object):
    """A custom object, to prove the store round-trips more than builtins."""

    def __init__(self, a, b):
        self.a = a
        self.b = b

    def __eq__(self, other):
        return isinstance(other, Thing) and (self.a, self.b) == (other.a, other.b)

    def __repr__(self):
        return 'Thing(%r,%r)' % (self.a, self.b)


_saved_cwd = os.getcwd()
W = tempfile.mkdtemp(prefix='bst-pickleshare.')


def unW(value):
    """Replace the random workdir with `$W', so a value is a LITERAL."""
    return str(value).replace(W, '$W')


def listing(root):
    """Every FILE under root, as store-relative slash-joined names, sorted."""
    out = []
    for dirpath, _dirnames, filenames in os.walk(root):
        for name in filenames:
            full = os.path.join(dirpath, name)
            out.append(os.path.relpath(full, root).replace(os.sep, '/'))
    out.sort()
    return out


def dirs_under(root):
    out = []
    for dirpath, dirnames, _files in os.walk(root):
        for name in dirnames:
            full = os.path.join(dirpath, name)
            out.append(os.path.relpath(full, root).replace(os.sep, '/'))
    out.sort()
    return out


# ================================================================ identity ===
# WHICH COPY ANSWERED. Every behavioural case below is a statement about the
# module named here.
emit('module_file', pickleshare.__file__)
emit('is_package', yn(hasattr(pickleshare, '__path__')))
emit('version_attr', pickleshare.__version__)

# NO `__all__'. The module never defines one, so `from pickleshare import *'
# leaks every public name it happens to hold -- including the stdlib modules it
# imported at the top (`os', `stat', `time', `pickle', `errno', `sys').  That is
# a real property of this module and the README's own example uses star-import.
emit('has_dunder_all', yn(hasattr(pickleshare, '__all__')))
emit('star_leaks_os', yn(getattr(pickleshare, 'os', None) is os))
emit('star_leaks_pickle', yn(getattr(pickleshare, 'pickle', None) is pickle))

# `PickleShareDB' IS A MutableMapping, which is where `clear', `items',
# `values', `get', `pop', `popitem', `update' and `setdefault' come from --
# every one of them built out of the five methods the class defines itself.
import collections.abc as _abc
emit('is_mutablemapping', yn(issubclass(PickleShareDB, _abc.MutableMapping)))
emit('own_methods', ','.join(sorted(
    k for k in vars(PickleShareDB) if not k.startswith('_') or k in (
        '__getitem__', '__setitem__', '__delitem__', '__iter__', '__len__'))))

# ============================================================== root setup ===
# `__init__' EXPANDS AND ABSOLUTISES, and creates the directory if it is absent.
store = os.path.join(W, 'store')
db = PickleShareDB(store)
emit('root_created', yn(os.path.isdir(store)))
emit('root_is_path', type(db.root).__name__)
emit('root_value', unW(db.root))
emit('repr_db', unW(repr(db)))
emit('cache_empty_at_start', yn(db.cache == {}))

# A RELATIVE root is made absolute, and `~' is expanded -- both by `__init__',
# so a caller's cwd cannot move a store.
rel = PickleShareDB(os.path.relpath(os.path.join(W, 'relstore'), os.getcwd()))
emit('relative_root_absolute', yn(os.path.isabs(str(rel.root))))
# `~' is expanded by __init__ via expanduser.  HOME is pointed at the workdir
# for the measurement: the assertion is about the expansion, and a probe has no
# business creating a directory in the target's real home -- still less finding
# out the hard way that it is read-only, which would kill every case below.
_home = os.environ.get('HOME')
os.environ['HOME'] = W
try:
    emit('tilde_expanded', unW(PickleShareDB('~/tildestore').root))
finally:
    if _home is None:
        os.environ.pop('HOME', None)
    else:
        os.environ['HOME'] = _home

# A non-str root is accepted: `__init__' calls str() on anything that is not a
# string, which is how a pathlib.Path reaches it.
from pathlib import Path as _P
emit('pathlib_root_ok', unW(PickleShareDB(_P(W) / 'pstore').root))

# NEW DB ON AN EXISTING DIRECTORY DOES NOT WIPE IT -- `__init__' only mkdirs.
emit('mkdir_of_existing_ok', call(lambda: yn(PickleShareDB(store) is not None)))

# ============================================================= round trip ===
# THE TICKET'S FIRST CLAIM: every picklable shape survives a close-and-reopen.
# There is no `close()' on this class -- the store IS the directory, so
# "reopening" is a second PickleShareDB over the same path, which is exactly
# what a second process would have.  Deleting the object first makes the point
# that nothing is flushed on the way out.
values = {
    'an_int': 42,
    'a_str': 'hello',
    'a_list': [1, 2, 313],
    'a_dict': {'x': 1, 'y': [2, 3]},
    'a_nested': {'a': [1, (5, 46), {'b': ('c', 7)}]},
    'an_obj': Thing(1, 'two'),
    'a_none': None,
    'a_bool': True,
    'a_float': 1.5,
    'a_bytes': b'\x00\x01\xfe',
    'a_tuple': (1, 2),
}
for k, v in values.items():
    db[k] = v
del db

db2 = PickleShareDB(store)
for k, v in values.items():
    emit('rt_' + k, yn(db2[k] == v))
# The custom object comes back as its own class, not as a dict of its attrs.
emit('rt_obj_type', type(db2['an_obj']).__name__)
emit('rt_obj_attr', db2['an_obj'].a)
# ...and as a DISTINCT object: unpickling reconstructs, it does not share.
emit('rt_obj_not_identical', yn(db2['an_obj'] is not values['an_obj']))
# A tuple stays a tuple and bytes stay bytes -- protocol 2 is not JSON.
emit('rt_tuple_type', type(db2['a_tuple']).__name__)
emit('rt_bytes_type', type(db2['a_bytes']).__name__)

# ========================================================== the file layout ===
# THE TICKET'S SECOND CLAIM, AND THE ONE THAT MATTERS: a key IS a file.
emit('file_per_key', yn(os.path.isfile(os.path.join(store, 'an_int'))))
emit('file_count', len(listing(store)))
emit('len_matches_files', yn(len(db2) == len(listing(store))))

# THE FILE IS A PICKLE AT PROTOCOL 2, which the module hardcodes so a store can
# be read by a Python 2.  Protocol 2 opens with the PROTO opcode \x80\x02, and
# asserting those two bytes is the most direct statement of that promise there
# is -- a release that quietly moved to protocol 5 reds here.
with open(os.path.join(store, 'an_int'), 'rb') as f:
    head = f.read(2)
emit('pickle_proto_magic', head.hex())
emit('file_is_loadable_pickle',
     yn(pickle.loads(open(os.path.join(store, 'an_int'), 'rb').read()) == 42))

# A HIERARCHICAL KEY BECOMES A SUBDIRECTORY -- `__setitem__' mkdirs the parent.
db2['paths/are/ok/key'] = [1, (5, 46)]
emit('nested_is_subdir', yn(os.path.isdir(os.path.join(store, 'paths', 'are', 'ok'))))
emit('nested_is_file', yn(os.path.isfile(os.path.join(store, 'paths', 'are', 'ok', 'key'))))
emit('nested_roundtrip', yn(db2['paths/are/ok/key'] == [1, (5, 46)]))
# ...and it reads back through the SAME slash-joined name, on every platform:
# `_normalized' rewrites the separator.
emit('nested_key_in_keys', yn('paths/are/ok/key' in db2.keys()))

# `keys()' IS A LIST, NOT A VIEW -- the class overrides it, so it does not
# behave like dict.keys().  `hdict' depends on that (it calls .sort() on the
# result), and so does any caller who indexes it.
emit('keys_type', type(db2.keys()).__name__)
emit('items_type', type(db2.items()).__name__)
emit('values_type', type(db2.values()).__name__)
emit('keys_matches_disk', yn(sorted(db2.keys()) == listing(store)))
emit('items_len', len(list(db2.items())))
emit('iter_matches_keys', yn(sorted(iter(db2)) == sorted(db2.keys())))
emit('contains_hit', yn('an_int' in db2))
emit('contains_miss', yn('nope' in db2))

# DIRECTORIES ARE NOT KEYS. `keys()' filters on is_file(), so the intermediate
# `paths/are' is on disk and absent from the mapping.
emit('dir_not_a_key', yn('paths/are' not in db2.keys()))
emit('dirs_on_disk', ','.join(dirs_under(store)))

# THE GLOB FORMS. With no argument `keys()' is rglob('*') -- RECURSIVE. With one
# it is glob(pat), which is NOT recursive unless the pattern says so.
emit('glob_nested_star', ','.join(sorted(db2.keys('paths/are/ok/*'))))
emit('glob_top_star_excludes_nested',
     yn('paths/are/ok/key' not in db2.keys('*')))
emit('glob_recursive_star', yn('paths/are/ok/key' in db2.keys('**/*')))
emit('glob_prefix', ','.join(sorted(db2.keys('a_*'))))
emit('glob_no_match', ','.join(sorted(db2.keys('zzz*'))))

# ================================================================ deletion ===
db2['doomed'] = 1
emit('del_file_existed', yn(os.path.isfile(os.path.join(store, 'doomed'))))
del db2['doomed']
emit('del_removes_file', yn(not os.path.exists(os.path.join(store, 'doomed'))))
emit('del_removes_key', yn('doomed' not in db2.keys()))

# DELETING A MISSING KEY DOES NOT RAISE. `__delitem__' swallows OSError on
# purpose -- "we lost, the other process wins the conflict" -- so this mapping
# DIVERGES FROM dict, where `del d['nope']' is a KeyError.  A caller porting
# code from dict to PickleShareDB loses that error.
emit('del_missing', call(lambda: yn(db2.__delitem__('nope') is None)))
# `pop' DOES raise, because MutableMapping.pop goes through __getitem__ first.
emit('pop_missing', call(lambda: db2.pop('nope')))
emit('pop_missing_default', call(lambda: db2.pop('nope', 'dflt')))

# `clear()' EMPTIES THE KEYS AND LEAVES THE DIRECTORIES. It comes from
# MutableMapping -- popitem() in a loop -- and popitem only ever unlinks FILES.
# So the store is empty as a mapping while `paths/are/ok' is still on disk.
db2.clear()
emit('clear_keys_empty', yn(db2.keys() == []))
emit('clear_len_zero', len(db2))
emit('clear_files_gone', yn(listing(store) == []))
emit('clear_leaves_dirs', ','.join(dirs_under(store)))
emit('clear_root_still_there', yn(os.path.isdir(store)))

# ============================================================= concurrency ===
# THE CLAIM THE PACKAGE IS FOR: two PickleShareDB objects over ONE directory,
# with no lock between them.  This is what a second process has, minus the
# process.
cstore = os.path.join(W, 'concur')
a = PickleShareDB(cstore)
b = PickleShareDB(cstore)
a['shared'] = 'from-a'
emit('peer_sees_write', b['shared'])
b['shared'] = 'from-b'
emit('origin_sees_peer_write', a['shared'])
emit('peer_sees_new_key_in_keys', yn('shared' in b.keys()))

# A WRITE BEHIND THE LIBRARY'S BACK IS RE-READ, and the mechanism is the mtime
# in the cache -- not a lock and not a notification.  `a' has `shared' cached
# from the read above; the file is rewritten by pickle directly and its mtime
# pushed forward, and the next read has to notice.
#
# The mtime is moved EXPLICITLY rather than by writing quickly and hoping: on a
# filesystem with coarse mtime granularity a rewrite inside the same tick keeps
# the old mtime, the cache stays valid, and the library returns the stale value
# -- which is a real property of this design, but not one a suite can assert
# without controlling the clock.
emit('cached_before_external', a['shared'])
sfile = os.path.join(cstore, 'shared')
with open(sfile, 'wb') as f:
    pickle.dump('from-outside', f, protocol=2)
st = os.stat(sfile)
os.utime(sfile, (st.st_atime + 10, st.st_mtime + 10))
emit('external_rewrite_reread', a['shared'])

# ...and the converse, which is the cache actually working: with the mtime held
# still, a rewritten file is NOT re-read.  This is the stale window the design
# accepts, stated as a fact rather than left to be discovered.
with open(sfile, 'wb') as f:
    pickle.dump('third-value', f, protocol=2)
os.utime(sfile, (st.st_atime + 10, st.st_mtime + 10))
emit('unchanged_mtime_serves_cache', a['shared'])

# `uncache()' IS THE ESCAPE HATCH for exactly that window.
a.uncache()
emit('uncache_empties_cache', yn(a.cache == {}))
emit('uncache_forces_reread', a['shared'])
emit('uncache_refills_cache', len(a.cache))

# THE CACHE COMPARES AN int TO A float, AND THAT IS WHY A WRITE IS RE-READ.
# `__setitem__' stores `fil.stat().st_mtime' -- a float -- while `__getitem__'
# compares against `fil.stat()[stat.ST_MTIME]' -- an int.  They are equal only
# when the file's mtime happens to have no fractional part, so on a filesystem
# with sub-second timestamps the cache MISSES on the value just written and the
# object comes back re-unpickled rather than identical.
#
# Neither answer is wrong on its own -- it depends on the filesystem's mtime
# granularity, which is a property of the target and not of this package.  So
# what is asserted is that THE TWO AGREE: identity holds exactly when the mtime
# is integral.  That is a statement about the code path, and it is true on any
# filesystem.
istore = os.path.join(W, 'ident')
c = PickleShareDB(istore)
marker = ['a', 'list', 'object']
c['m'] = marker
mt = os.stat(os.path.join(istore, 'm')).st_mtime
integral = (mt == int(mt))
identical = c['m'] is marker
emit('mtime_is_integral', yn(integral))
emit('write_then_read_identical', yn(identical))
emit('write_then_read_equal', yn(c['m'] == marker))
emit('cache_identity_tracks_mtime', yn(identical == integral))
# Either way the cache holds exactly one entry after a write, keyed by the Path.
emit('cache_size_after_write', len(c.cache))
emit('cache_key_type', type(list(c.cache)[0]).__name__)

# =========================================================== error paths ===
# A MISSING KEY IS A KeyError -- the one place this mapping does behave like a
# dict, and it is `stat()' failing that produces it, not a lookup.
emit('missing_key', call(lambda: db2['no-such-key']))
# The KeyError carries the KEY, not the path -- so a caller's error message
# names what it asked for rather than where the store happened to be.
def _keyerr_arg():
    try:
        db2['no-such-key']
    except KeyError as exc:
        return exc.args[0]
    return 'did-not-raise'
emit('missing_key_arg', _keyerr_arg())
estore = os.path.join(W, 'err')
e = PickleShareDB(estore)

# AN UNPICKLABLE VALUE RAISES -- AND LEAVES A FILE BEHIND. `__setitem__' opens
# the file for writing BEFORE it pickles, so `open('wb')' has already truncated
# (or created) it by the time pickle refuses.  The key is therefore VISIBLE in
# keys() and UNREADABLE through __getitem__, which is the worst of both: the
# store is left in a state the mapping API cannot describe.
#
# This is the one place the ticket's own description is wrong -- it says the
# write raises "rather than leaving a truncated file", and a truncated file is
# exactly what it leaves.  Measured, not predicted.
unpicklable = lambda x: x  # a lambda has no importable qualified name
emit('unpicklable_raises', call(lambda: e.__setitem__('bad', unpicklable)))
emit('unpicklable_left_file', yn(os.path.exists(os.path.join(estore, 'bad'))))
emit('unpicklable_file_size', os.path.getsize(os.path.join(estore, 'bad'))
     if os.path.exists(os.path.join(estore, 'bad')) else 'absent')
emit('unpicklable_key_listed', yn('bad' in e.keys()))
emit('unpicklable_key_unreadable', call(lambda: e['bad']))
emit('unpicklable_not_in_contains', yn('bad' not in e))

# A CORRUPTED STORE FILE IS REPORTED, NOT RETURNED -- but it is reported as
# KeyError, because `__getitem__' wraps the unpickle in a BARE `except:'.  So a
# caller cannot tell "absent" from "corrupt", and `in' answers False for a file
# that is plainly there.
with open(os.path.join(estore, 'corrupt'), 'wb') as f:
    f.write(b'this is not a pickle at all')
emit('corrupt_file_exists', yn(os.path.isfile(os.path.join(estore, 'corrupt'))))
emit('corrupt_read', call(lambda: e['corrupt']))
emit('corrupt_in_keys', yn('corrupt' in e.keys()))
emit('corrupt_not_in_contains', yn('corrupt' not in e))
emit('corrupt_len_counts_it', yn(len(e) == len(listing(estore))))

# ...AND A CORRUPT FILE STOPS `clear()' DEAD. clear() is popitem() in a loop,
# popitem() reads the value before deleting it, and the KeyError that read
# raises is the same KeyError clear() uses as its "store is empty" signal.  So
# the loop exits early and the corrupt file -- the one a caller most wants gone
# -- is the one clear() cannot remove.
before = len(listing(estore))
emit('clear_with_corrupt_raises', call(lambda: yn(e.clear() is None)))
emit('clear_with_corrupt_left', yn(os.path.isfile(os.path.join(estore, 'corrupt'))))
emit('clear_with_corrupt_shrank', yn(len(listing(estore)) < before))

# AN ABSOLUTE KEY ESCAPES THE STORE. `self.root / key' is pathlib's `/', and
# joining an ABSOLUTE path onto a Path REPLACES it -- so a key that starts with
# a slash writes wherever it points, outside the directory that is supposed to
# govern every file.  The written file is then invisible to keys(), which only
# rglobs under the root.
outside = os.path.join(W, 'outside-the-store')
e[outside] = 'escaped'
emit('absolute_key_escapes', yn(os.path.isfile(outside)))
emit('absolute_key_invisible', yn(not any('outside' in k for k in e.keys())))
emit('absolute_key_reads_back', e[outside])

# ============================================================ waitget ===
# `waitget' POLLS for a key to appear and gives up with KeyError.  Present, it
# returns at once and costs nothing; absent, it sleeps through its own schedule
# (0.2s three times, 0.5s twice, then 1s) and the FIRST sleep happens before the
# timeout is ever compared -- so even maxwaittime=0 costs 0.2s, not 0.
w = PickleShareDB(os.path.join(W, 'wait'))
#
# ONLY A LOWER BOUND IS ASSERTED ON THE TIMING. This suite runs inside a batch
# verify where several images' worth of work shares one machine, so an UPPER
# bound on elapsed time is a red waiting for a busy afternoon.  `slept' is a
# lower bound and load can only make it truer; `bounded' is deliberately loose
# enough (30s against a schedule that owes 0.2s) that only a hang trips it.
w['present'] = 'here'
emit('waitget_present', w.waitget('present'))
t0 = time.time()
emit('waitget_missing', call(lambda: w.waitget('gone', 0)))
elapsed = time.time() - t0
emit('waitget_missing_slept', yn(elapsed >= 0.2))
emit('waitget_missing_bounded', yn(elapsed < 30))

# ============================================================= hashed set ===
# `hset'/`hget' BUCKET MANY KEYS INTO FEW FILES, for a store that would
# otherwise grow a file per key.  The bucket name comes from hash(), so it is
# not reproducible across processes -- only its shape and the round trip are.
h = PickleShareDB(os.path.join(W, 'hash'))
h.hset('cat', 'k1', 'v1')
h.hset('cat', 'k2', 'v2')
emit('hset_roundtrip_k1', h.hget('cat', 'k1'))
emit('hset_roundtrip_k2', h.hget('cat', 'k2'))
emit('hget_missing_default', h.hget('cat', 'nope', 'dflt'))
emit('hashfile_shape', yn(len(gethashfile('k1')) == 2 and
                          all(ch in '0123456789abcdef' for ch in gethashfile('k1'))))
emit('hashfile_stable_in_process', yn(gethashfile('k1') == gethashfile('k1')))
emit('hset_made_files', yn(len(h.keys('cat/*')) >= 1))
emit('hdict_has_both', ','.join(sorted(h.hdict('cat').keys())))

# `hcompress' FOLDS A CATEGORY INTO ONE `xx' FILE, after which `hget' in its
# default fast_only mode can no longer find a key -- which the docstring says
# outright.  The slow path still can.
h.hcompress('cat')
emit('hcompress_single_file', ','.join(sorted(h.keys('cat/*'))))
emit('hcompress_slow_get', h.hget('cat', 'k1', None, False))
emit('hcompress_hdict_intact', ','.join(sorted(h.hdict('cat').keys())))

# =============================================================== getlink ===
# `getlink' RETURNS AN ATTRIBUTE FACADE over a subdirectory: `lnk.foo = 2' is
# `db['dir/foo'] = 2'.
g = PickleShareDB(os.path.join(W, 'link'))
lnk = g.getlink('objs')
lnk.foo = 2
emit('link_type', type(lnk).__name__)
emit('link_set_reaches_db', g['objs/foo'])
emit('link_get', lnk.foo)
emit('link_missing', call(lambda: lnk.nope))

# `repr()' OF A LINK IS BROKEN IN THIS RELEASE, and it is broken by a stdlib
# rename: it calls `Path(k).basename()', and pathlib.Path has NO `basename' --
# that is os.path's spelling.  The line has never worked on any Python 3.
# Asserted rather than skipped, because it is the kind of defect a later release
# fixes silently and a suite should notice.
emit('link_repr', call(lambda: repr(lnk)))
# `__init__' does `self.__dict__.update(locals())', which stores `self' in its
# own __dict__ -- a reference cycle, and one more name than the class documents.
emit('link_dict_keys', ','.join(sorted(lnk.__dict__)))

# ================================================================= cleanup ===
os.chdir(_saved_cwd)
shutil.rmtree(W, ignore_errors=True)
emit('workdir_gone', yn(not os.path.exists(W)))
emit('cwd_restored', yn(os.getcwd() == _saved_cwd))

emit('emitted_count', len(emitted))
print('probe_done=yes')