152 lines · python
1#!/usr/bin/env python2#3# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4# See https://llvm.org/LICENSE.txt for license information.5# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6#7# ==------------------------------------------------------------------------==#8 9import os10import glob11import re12import subprocess13import json14import datetime15import argparse16from urllib.parse import urlencode17from urllib.request import urlopen, Request18 19 20parser = argparse.ArgumentParser()21parser.add_argument("benchmark_directory")22parser.add_argument("--runs", type=int, default=10)23parser.add_argument("--wrapper", default="")24parser.add_argument("--machine", required=True)25parser.add_argument("--revision", required=True)26parser.add_argument("--threads", action="store_true")27parser.add_argument(28 "--url",29 help="The lnt server url to send the results to",30 default="http://localhost:8000/db_default/v4/link/submitRun",31)32args = parser.parse_args()33 34 35class Bench:36 def __init__(self, directory, variant):37 self.directory = directory38 self.variant = variant39 40 def __str__(self):41 if not self.variant:42 return self.directory43 return "%s-%s" % (self.directory, self.variant)44 45 46def getBenchmarks():47 ret = []48 for i in glob.glob("*/response*.txt"):49 m = re.match(r"response-(.*)\.txt", os.path.basename(i))50 variant = m.groups()[0] if m else None51 ret.append(Bench(os.path.dirname(i), variant))52 return ret53 54 55def parsePerfNum(num):56 num = num.replace(b",", b"")57 try:58 return int(num)59 except ValueError:60 return float(num)61 62 63def parsePerfLine(line):64 ret = {}65 line = line.split(b"#")[0].strip()66 if len(line) != 0:67 p = line.split()68 ret[p[1].strip().decode("ascii")] = parsePerfNum(p[0])69 return ret70 71 72def parsePerf(output):73 ret = {}74 lines = [x.strip() for x in output.split(b"\n")]75 76 seconds = [x for x in lines if b"seconds time elapsed" in x][0]77 seconds = seconds.strip().split()[0].strip()78 ret["seconds-elapsed"] = parsePerfNum(seconds)79 80 measurement_lines = [x for x in lines if b"#" in x]81 for l in measurement_lines:82 ret.update(parsePerfLine(l))83 return ret84 85 86def run(cmd):87 try:88 return subprocess.check_output(cmd, stderr=subprocess.STDOUT)89 except subprocess.CalledProcessError as e:90 print(e.output)91 raise e92 93 94def combinePerfRun(acc, d):95 for k, v in d.items():96 a = acc.get(k, [])97 a.append(v)98 acc[k] = a99 100 101def perf(cmd):102 # Discard the first run to warm up any system cache.103 run(cmd)104 105 ret = {}106 wrapper_args = [x for x in args.wrapper.split(",") if x]107 for i in range(args.runs):108 os.unlink("t")109 out = run(wrapper_args + ["perf", "stat"] + cmd)110 r = parsePerf(out)111 combinePerfRun(ret, r)112 os.unlink("t")113 return ret114 115 116def runBench(bench):117 thread_arg = [] if args.threads else ["--no-threads"]118 os.chdir(bench.directory)119 suffix = "-%s" % bench.variant if bench.variant else ""120 response = "response" + suffix + ".txt"121 ret = perf(["../ld.lld", "@" + response, "-o", "t"] + thread_arg)122 ret["name"] = str(bench)123 os.chdir("..")124 return ret125 126 127def buildLntJson(benchmarks):128 start = datetime.datetime.utcnow().isoformat()129 tests = [runBench(b) for b in benchmarks]130 end = datetime.datetime.utcnow().isoformat()131 ret = {132 "format_version": 2,133 "machine": {"name": args.machine},134 "run": {135 "end_time": start,136 "start_time": end,137 "llvm_project_revision": args.revision,138 },139 "tests": tests,140 }141 return json.dumps(ret, sort_keys=True, indent=4)142 143 144def submitToServer(data):145 data2 = urlencode({"input_data": data}).encode("ascii")146 urlopen(Request(args.url, data2))147 148 149os.chdir(args.benchmark_directory)150data = buildLntJson(getBenchmarks())151submitToServer(data)152