106 lines · plain
1#!/usr/bin/env python32 3import argparse4import logging5import os6import pathlib7import subprocess8import sys9import tempfile10 11PARENT_DIR = pathlib.Path(os.path.dirname(os.path.abspath(__file__)))12 13def directory_path(string):14 if os.path.isdir(string):15 return pathlib.Path(string)16 else:17 raise NotADirectoryError(string)18 19def whitespace_separated(stream):20 """21 Iterate over a stream, yielding whitespace-delimited elements.22 """23 for line in stream:24 for element in line.split():25 yield element26 27def resolve_commit(git_repo, commit):28 """29 Resolve the full commit SHA from any tree-ish.30 """31 return subprocess.check_output(['git', '-C', git_repo, 'rev-parse', commit], text=True).strip()32 33 34def main(argv):35 parser = argparse.ArgumentParser(36 prog='benchmark-historical',37 description='Run the libc++ benchmarks against the commits provided on standard input and store the results in '38 'LNT format in a directory. This makes it easy to generate historical benchmark results of libc++ '39 'for analysis purposes. This script\'s usage is optimized to be run on a set of commits and then '40 're-run on a potentially-overlapping set of commits, such as after pulling new commits with Git.')41 parser.add_argument('--output', '-o', type=pathlib.Path, required=True,42 help='Path to the directory where the resulting .lnt files are stored.')43 parser.add_argument('--commit-list', type=argparse.FileType('r'), default=sys.stdin,44 help='Path to a file containing a whitespace separated list of commits to test. '45 'By default, this is read from standard input.')46 parser.add_argument('--existing', type=str, choices=['skip', 'overwrite', 'append'], default='skip',47 help='This option instructs what to do when data for a commit already exists in the output directory. '48 'Selecting "skip" instructs the tool to skip generating data for a commit that already has data, '49 '"overwrite" will overwrite the existing data with the newly-generated one, and "append" will '50 'append the new data to the existing one. By default, the tool uses "skip".')51 parser.add_argument('lit_options', nargs=argparse.REMAINDER,52 help='Optional arguments passed to lit when running the tests. Should be provided last and '53 'separated from other arguments with a `--`.')54 parser.add_argument('--git-repo', type=directory_path, default=pathlib.Path(os.getcwd()),55 help='Optional path to the Git repository to use. By default, the current working directory is used.')56 parser.add_argument('--dry-run', action='store_true',57 help='Do not actually run anything, just print what would be done.')58 args = parser.parse_args(argv)59 60 logging.getLogger().setLevel(logging.INFO)61 62 # Gather lit options63 lit_options = []64 if args.lit_options:65 if args.lit_options[0] != '--':66 raise ArgumentError('For clarity, Lit options must be separated from other options by --')67 lit_options = args.lit_options[1:]68 69 # Process commits one by one. Commits just need to be whitespace separated: we also handle70 # the case where there is more than one commit per line.71 for commit in whitespace_separated(args.commit_list):72 commit = resolve_commit(args.git_repo, commit) # resolve e.g. HEAD to a real SHA73 74 output_file = args.output / (commit + '.lnt')75 if output_file.exists() and args.existing == 'skip':76 logging.info(f'Skipping {commit} which already has data in {output_file}')77 continue78 else:79 logging.info(f'Benchmarking {commit}')80 81 with tempfile.TemporaryDirectory() as build_dir:82 test_cmd = [PARENT_DIR / 'test-at-commit', '--git-repo', args.git_repo,83 '--build', build_dir,84 '--commit', commit]85 test_cmd += ['--'] + lit_options86 87 if args.dry_run:88 pretty = ' '.join(str(a) for a in test_cmd)89 logging.info(f'Running {pretty}')90 continue91 92 subprocess.call(test_cmd)93 output_file.parent.mkdir(parents=True, exist_ok=True)94 mode = 'a' if args.existing == 'append' else 'w'95 if output_file.exists() and args.existing == 'append':96 logging.info(f'Appending to existing data for {commit}')97 elif output_file.exists() and args.existing == 'overwrite':98 logging.info(f'Overwriting existing data for {commit}')99 else:100 logging.info(f'Writing data for {commit}')101 with open(output_file, mode) as out:102 subprocess.check_call([(PARENT_DIR / 'consolidate-benchmarks'), build_dir], stdout=out)103 104if __name__ == '__main__':105 main(sys.argv[1:])106