455 lines · python
1#!/usr/bin/env python32import argparse3import subprocess4from typing import *5import tempfile6import copy7import os8import shutil9import sys10import re11import configparser12from types import SimpleNamespace13from textwrap import dedent14 15# USAGE:16# 0. Prepare two BOLT build versions: base and compare.17# 1. Create the config by invoking this script with required options.18# Save the config as `llvm-bolt-wrapper.ini` next to the script or19# in the testing directory.20# In the base BOLT build directory:21# 2. Rename `llvm-bolt` to `llvm-bolt.real`22# 3. Create a symlink from this script to `llvm-bolt`23# 4. Create `llvm-bolt-wrapper.ini` and fill it using the example below.24#25# This script will compare binaries produced by base and compare BOLT, and26# report elapsed processing time and max RSS.27 28# read options from config file llvm-bolt-wrapper.ini in script CWD29#30# [config]31# # mandatory32# base_bolt = /full/path/to/llvm-bolt.real33# cmp_bolt = /full/path/to/other/llvm-bolt34# # optional, default to False35# verbose36# keep_tmp37# no_minimize38# run_sequentially39# compare_output40# skip_binary_cmp41# # optional, defaults to timing.log in CWD42# timing_file = timing1.log43 44 45def read_cfg():46 src_dir = os.path.dirname(os.path.abspath(__file__))47 cfg = configparser.ConfigParser(allow_no_value=True)48 cfgs = cfg.read("llvm-bolt-wrapper.ini")49 if not cfgs:50 cfgs = cfg.read(os.path.join(src_dir, "llvm-bolt-wrapper.ini"))51 assert cfgs, f"llvm-bolt-wrapper.ini is not found in {os.getcwd()}"52 53 def get_cfg(key):54 # if key is not present in config, assume False55 if key not in cfg["config"]:56 return False57 # if key is present, but has no value, assume True58 if not cfg["config"][key]:59 return True60 # if key has associated value, interpret the value61 return cfg["config"].getboolean(key)62 63 d = {64 # BOLT binary locations65 "BASE_BOLT": cfg["config"]["base_bolt"],66 "CMP_BOLT": cfg["config"]["cmp_bolt"],67 # optional68 "VERBOSE": get_cfg("verbose"),69 "KEEP_TMP": get_cfg("keep_tmp"),70 "NO_MINIMIZE": get_cfg("no_minimize"),71 "RUN_SEQUENTIALLY": get_cfg("run_sequentially"),72 "COMPARE_OUTPUT": get_cfg("compare_output"),73 "SKIP_BINARY_CMP": get_cfg("skip_binary_cmp"),74 "TIMING_FILE": cfg["config"].get("timing_file", "timing.log"),75 }76 if d["VERBOSE"]:77 print(f"Using config {os.path.abspath(cfgs[0])}")78 return SimpleNamespace(**d)79 80 81# perf2bolt mode82PERF2BOLT_MODE = ["-aggregate-only", "-ignore-build-id", "-show-density"]83 84# boltdiff mode85BOLTDIFF_MODE = ["-diff-only", "-o", "/dev/null"]86 87# options to suppress binary differences as much as possible88MINIMIZE_DIFFS = ["-bolt-info=0"]89 90# bolt output options that need to be intercepted91BOLT_OUTPUT_OPTS = {92 "-o": "BOLT output binary",93 "-w": "BOLT recorded profile",94}95 96# regex patterns to exclude the line from log comparison97SKIP_MATCH = [98 "BOLT-INFO: BOLT version",99 r"^Args: ",100 r"^BOLT-DEBUG:",101 r"BOLT-INFO:.*data.*output data",102 "WARNING: reading perf data directly",103]104 105 106def run_cmd(cmd, out_f, cfg):107 if cfg.VERBOSE:108 print(" ".join(cmd))109 return subprocess.Popen(cmd, stdout=out_f, stderr=subprocess.STDOUT)110 111 112def run_bolt(bolt_path, bolt_args, out_f, cfg):113 p2b = os.path.basename(sys.argv[0]) == "perf2bolt" # perf2bolt mode114 bd = os.path.basename(sys.argv[0]) == "llvm-boltdiff" # boltdiff mode115 cmd = ["/usr/bin/time", "-f", "%e %M", bolt_path] + bolt_args116 if p2b:117 # -ignore-build-id can occur at most once, hence remove it from cmd118 if "-ignore-build-id" in cmd:119 cmd.remove("-ignore-build-id")120 cmd += PERF2BOLT_MODE121 elif bd:122 cmd += BOLTDIFF_MODE123 elif not cfg.NO_MINIMIZE:124 cmd += MINIMIZE_DIFFS125 return run_cmd(cmd, out_f, cfg)126 127 128def prepend_dash(args: Mapping[AnyStr, AnyStr]) -> Sequence[AnyStr]:129 """130 Accepts parsed arguments and returns flat list with dash prepended to131 the option.132 Example: Namespace(o='test.tmp') -> ['-o', 'test.tmp']133 """134 dashed = [("-" + key, value) for (key, value) in args.items()]135 flattened = list(sum(dashed, ()))136 return flattened137 138 139def replace_cmp_path(tmp: AnyStr, args: Mapping[AnyStr, AnyStr]) -> Sequence[AnyStr]:140 """141 Keeps file names, but replaces the path to a temp folder.142 Example: Namespace(o='abc/test.tmp') -> Namespace(o='/tmp/tmpf9un/test.tmp')143 Except preserve /dev/null.144 """145 replace_path = (146 lambda x: os.path.join(tmp, os.path.basename(x))147 if x != "/dev/null"148 else "/dev/null"149 )150 new_args = {key: replace_path(value) for key, value in args.items()}151 return prepend_dash(new_args)152 153 154def preprocess_args(args: argparse.Namespace) -> Mapping[AnyStr, AnyStr]:155 """156 Drop options that weren't parsed (e.g. -w), convert to a dict157 """158 return {key: value for key, value in vars(args).items() if value}159 160 161def write_to(txt, filename, mode="w"):162 with open(filename, mode) as f:163 f.write(txt)164 165 166def wait(proc, fdesc):167 proc.wait()168 fdesc.close()169 return open(fdesc.name)170 171 172def compare_logs(main, cmp, skip_begin=0, skip_end=0, str_input=True):173 """174 Compares logs but allows for certain lines to be excluded from comparison.175 If str_input is True (default), the input it assumed to be a string,176 which is split into lines. Otherwise the input is assumed to be a file.177 Returns None on success, mismatch otherwise.178 """179 main_inp = main.splitlines() if str_input else main.readlines()180 cmp_inp = cmp.splitlines() if str_input else cmp.readlines()181 # rewind logs after consumption182 if not str_input:183 main.seek(0)184 cmp.seek(0)185 for lhs, rhs in list(zip(main_inp, cmp_inp))[skip_begin : -skip_end or None]:186 if lhs != rhs:187 # check skip patterns188 for skip in SKIP_MATCH:189 # both lines must contain the pattern190 if re.search(skip, lhs) and re.search(skip, rhs):191 break192 # otherwise return mismatching lines193 else:194 return (lhs, rhs)195 return None196 197 198def fmt_cmp(cmp_tuple):199 if not cmp_tuple:200 return ""201 return f"main:\n{cmp_tuple[0]}\ncmp:\n{cmp_tuple[1]}\n"202 203 204def compare_with(lhs, rhs, cmd, skip_begin=0, skip_end=0):205 """206 Runs cmd on both lhs and rhs and compares stdout.207 Returns tuple (mismatch, lhs_stdout):208 - if stdout matches between two files, mismatch is None,209 - otherwise mismatch is a tuple of mismatching lines.210 """211 run = lambda binary: subprocess.run(212 cmd.split() + [binary], text=True, check=True, capture_output=True213 ).stdout214 run_lhs = run(lhs)215 run_rhs = run(rhs)216 cmp = compare_logs(run_lhs, run_rhs, skip_begin, skip_end)217 return cmp, run_lhs218 219 220def parse_cmp_offset(cmp_out):221 """222 Extracts byte number from cmp output:223 file1 file2 differ: byte X, line Y224 """225 # NOTE: cmp counts bytes starting from 1!226 return int(re.search(r"byte (\d+),", cmp_out).groups()[0]) - 1227 228 229def report_real_time(binary, main_err, cmp_err, cfg):230 """231 Extracts real time from stderr and appends it to TIMING FILE it as csv:232 "output binary; base bolt; cmp bolt"233 """234 235 def get_real_from_stderr(logline):236 return "; ".join(logline.split())237 238 for line in main_err:239 pass240 main = get_real_from_stderr(line)241 for line in cmp_err:242 pass243 cmp = get_real_from_stderr(line)244 write_to(f"{binary}; {main}; {cmp}\n", cfg.TIMING_FILE, "a")245 # rewind logs after consumption246 main_err.seek(0)247 cmp_err.seek(0)248 249 250def clean_exit(tmp, out, exitcode, cfg):251 # temp files are only cleaned on success252 if not cfg.KEEP_TMP:253 shutil.rmtree(tmp)254 255 # report stdout and stderr from the main process256 shutil.copyfileobj(out, sys.stdout)257 sys.exit(exitcode)258 259 260def find_section(offset, readelf_hdr):261 hdr = readelf_hdr.split("\n")262 section = None263 # extract sections table (parse objdump -hw output)264 for line in hdr[5:-1]:265 cols = line.strip().split()266 # extract section offset267 file_offset = int(cols[5], 16)268 # section size269 size = int(cols[2], 16)270 if offset >= file_offset and offset < file_offset + size:271 if sys.stdout.isatty(): # terminal supports colors272 print(f"\033[1m{line}\033[0m")273 else:274 print(f">{line}")275 section = cols[1]276 else:277 print(line)278 return section279 280 281def main_config_generator():282 parser = argparse.ArgumentParser()283 parser.add_argument("base_bolt", help="Full path to base llvm-bolt binary")284 parser.add_argument("cmp_bolt", help="Full path to cmp llvm-bolt binary")285 parser.add_argument(286 "--verbose",287 action="store_true",288 help="Print subprocess invocation cmdline (default False)",289 )290 parser.add_argument(291 "--keep_tmp",292 action="store_true",293 help="Preserve tmp folder on a clean exit "294 "(tmp directory is preserved on crash by default)",295 )296 parser.add_argument(297 "--no_minimize",298 action="store_true",299 help=f"Do not add `{MINIMIZE_DIFFS}` that is used "300 "by default to reduce binary differences",301 )302 parser.add_argument(303 "--run_sequentially",304 action="store_true",305 help="Run both binaries sequentially (default "306 "in parallel). Use for timing comparison",307 )308 parser.add_argument(309 "--compare_output",310 action="store_true",311 help="Compare bolt stdout/stderr (disabled by default)",312 )313 parser.add_argument(314 "--skip_binary_cmp", action="store_true", help="Disable output comparison"315 )316 parser.add_argument(317 "--timing_file",318 help="Override path to timing log " "file (default `timing.log` in CWD)",319 )320 args = parser.parse_args()321 322 print(323 dedent(324 f"""\325 [config]326 # mandatory327 base_bolt = {args.base_bolt}328 cmp_bolt = {args.cmp_bolt}"""329 )330 )331 del args.base_bolt332 del args.cmp_bolt333 d = vars(args)334 if any(d.values()):335 print("# optional")336 for key, value in d.items():337 if value:338 print(key)339 340 341def main():342 cfg = read_cfg()343 # intercept output arguments344 parser = argparse.ArgumentParser(add_help=False)345 for option, help in BOLT_OUTPUT_OPTS.items():346 parser.add_argument(option, help=help)347 args, unknownargs = parser.parse_known_args()348 args = preprocess_args(args)349 cmp_args = copy.deepcopy(args)350 tmp = tempfile.mkdtemp()351 cmp_args = replace_cmp_path(tmp, cmp_args)352 353 # reconstruct output arguments: prepend dash354 args = prepend_dash(args)355 356 # run both BOLT binaries357 main_f = open(os.path.join(tmp, "main_bolt.stdout"), "w")358 cmp_f = open(os.path.join(tmp, "cmp_bolt.stdout"), "w")359 main_bolt = run_bolt(cfg.BASE_BOLT, unknownargs + args, main_f, cfg)360 if cfg.RUN_SEQUENTIALLY:361 main_out = wait(main_bolt, main_f)362 cmp_bolt = run_bolt(cfg.CMP_BOLT, unknownargs + cmp_args, cmp_f, cfg)363 else:364 cmp_bolt = run_bolt(cfg.CMP_BOLT, unknownargs + cmp_args, cmp_f, cfg)365 main_out = wait(main_bolt, main_f)366 cmp_out = wait(cmp_bolt, cmp_f)367 368 # check exit code369 if main_bolt.returncode != cmp_bolt.returncode:370 print(tmp)371 exit("exitcode mismatch")372 373 # don't compare output upon unsuccessful exit374 if main_bolt.returncode != 0:375 cfg.SKIP_BINARY_CMP = True376 377 # compare logs, skip_end=1 skips the line with time378 out = (379 compare_logs(main_out, cmp_out, skip_end=1, str_input=False)380 if cfg.COMPARE_OUTPUT381 else None382 )383 if out:384 print(tmp)385 print(fmt_cmp(out))386 write_to(fmt_cmp(out), os.path.join(tmp, "summary.txt"))387 exit("logs mismatch")388 389 if os.path.basename(sys.argv[0]) == "llvm-boltdiff": # boltdiff mode390 # no output binary to compare, so just exit391 clean_exit(tmp, main_out, main_bolt.returncode, cfg)392 393 # compare binaries (using cmp)394 main_binary = args[args.index("-o") + 1]395 cmp_binary = cmp_args[cmp_args.index("-o") + 1]396 if main_binary == "/dev/null":397 assert cmp_binary == "/dev/null"398 cfg.SKIP_BINARY_CMP = True399 400 # report binary timing as csv: output binary; base bolt real; cmp bolt real401 report_real_time(main_binary, main_out, cmp_out, cfg)402 403 if not cfg.SKIP_BINARY_CMP:404 # check if files exist405 main_exists = os.path.exists(main_binary)406 cmp_exists = os.path.exists(cmp_binary)407 if main_exists and cmp_exists:408 # proceed to comparison409 pass410 elif not main_exists and not cmp_exists:411 # both don't exist, assume it's intended, skip comparison412 clean_exit(tmp, main_out, main_bolt.returncode, cfg)413 elif main_exists:414 assert not cmp_exists415 exit(f"{cmp_binary} doesn't exist")416 else:417 assert not main_exists418 exit(f"{main_binary} doesn't exist")419 420 cmp_proc = subprocess.run(421 ["cmp", "-b", main_binary, cmp_binary], capture_output=True, text=True422 )423 if cmp_proc.returncode:424 # check if output is an ELF file (magic bytes)425 with open(main_binary, "rb") as f:426 magic = f.read(4)427 if magic != b"\x7fELF":428 exit("output mismatch")429 # check if ELF headers match430 mismatch, _ = compare_with(main_binary, cmp_binary, "readelf -We")431 if mismatch:432 print(fmt_cmp(mismatch))433 write_to(fmt_cmp(mismatch), os.path.join(tmp, "headers.txt"))434 exit("headers mismatch")435 # if headers match, compare sections (skip line with filename)436 mismatch, hdr = compare_with(437 main_binary, cmp_binary, "objdump -hw", skip_begin=2438 )439 assert not mismatch440 # check which section has the first mismatch441 mismatch_offset = parse_cmp_offset(cmp_proc.stdout)442 section = find_section(mismatch_offset, hdr)443 exit(f"binary mismatch @{hex(mismatch_offset)} ({section})")444 445 clean_exit(tmp, main_out, main_bolt.returncode, cfg)446 447 448if __name__ == "__main__":449 # config generator mode if the script is launched as is450 if os.path.basename(__file__) == "llvm-bolt-wrapper.py":451 main_config_generator()452 else:453 # llvm-bolt interceptor mode otherwise454 main()455