152 lines · python
1#!/usr/bin/env python32#3# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4# See https://llvm.org/LICENSE.txt for license information.5# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6#7# ==------------------------------------------------------------------------==#8 9import argparse10import os11import shutil12import subprocess13import tempfile14 15# The purpose of this script is to measure the performance effect16# of an lld change in a statistically sound way, automating all the17# tedious parts of doing so. It copies the test case into /tmp as well as18# running the test binaries from /tmp to reduce the influence on the test19# machine's storage medium on the results. It accounts for measurement20# bias caused by binary layout (using the --randomize-section-padding21# flag to link the test binaries) and by environment variable size22# (implemented by hyperfine [1]). Runs of the base and test case are23# interleaved to account for environmental factors which may influence24# the result due to the passage of time. The results of running hyperfine25# are collected into a results.csv file in the output directory and may26# be analyzed by the user with a tool such as ministat.27#28# Requirements: Linux host, hyperfine [2] in $PATH, run from a build directory29# configured to use ninja and a recent version of lld that supports30# --randomize-section-padding, /tmp is tmpfs.31#32# [1] https://github.com/sharkdp/hyperfine/blob/3cedcc38d0c430cbf38b4364b441c43a938d2bf3/src/util/randomized_environment_offset.rs#L133# [2] https://github.com/sharkdp/hyperfine34#35# Example invocation for comparing the performance of the current commit36# against the previous commit which is treated as the baseline, without37# linking debug info:38#39# lld/utils/run_benchmark.py \40# --base-commit HEAD^ \41# --test-commit HEAD \42# --test-case lld/utils/speed-test-reproducers/result/firefox-x64/response.txt \43# --num-iterations 512 \44# --num-binary-variants 16 \45# --output-dir outdir \46# --ldflags=-S47#48# Then this bash command will compare the real time of the base and test cases.49#50# ministat -A \51# <(grep lld-base outdir/results.csv | cut -d, -f2) \52# <(grep lld-test outdir/results.csv | cut -d, -f2)53 54# We don't want to copy stat() information when we copy the reproducer55# to the temporary directory. Files in the Nix store are read-only so this will56# cause trouble when the linker writes the output file and when we want to clean57# up the temporary directory. Python doesn't provide a way to disable copying58# stat() information in shutil.copytree so we just monkeypatch shutil.copystat59# to do nothing.60shutil.copystat = lambda *args, **kwargs: 061 62parser = argparse.ArgumentParser(prog="benchmark_change.py")63parser.add_argument("--base-commit", required=True)64parser.add_argument("--test-commit", required=True)65parser.add_argument("--test-case", required=True)66parser.add_argument("--num-iterations", type=int, required=True)67parser.add_argument("--num-binary-variants", type=int, required=True)68parser.add_argument("--output-dir", required=True)69parser.add_argument("--ldflags", required=False)70args = parser.parse_args()71 72test_dir = tempfile.mkdtemp()73print(f"Using {test_dir} as temporary directory")74 75os.makedirs(args.output_dir)76print(f"Using {args.output_dir} as output directory")77 78 79def extract_link_command(target):80 # We assume that the last command printed by "ninja -t commands" containing a81 # "-o" flag is the link command (we need to check for -o because subsequent82 # commands create symlinks for ld.lld and so on). This is true for CMake and83 # gn.84 link_command = None85 for line in subprocess.Popen(86 ["ninja", "-t", "commands", target], stdout=subprocess.PIPE87 ).stdout.readlines():88 commands = line.decode("utf-8").split("&&")89 for command in commands:90 if " -o " in command:91 link_command = command.strip()92 return link_command93 94 95def generate_binary_variants(case_name):96 subprocess.run(["ninja", "lld"])97 link_command = extract_link_command("lld")98 99 for i in range(0, args.num_binary_variants):100 print(f"Generating binary variant {i} for {case_name} case")101 command = f"{link_command} -o {test_dir}/lld-{case_name}{i} -Wl,--randomize-section-padding={i}"102 subprocess.run(command, check=True, shell=True)103 104 105# Make sure that there are no local changes.106subprocess.run(["git", "diff", "--exit-code", "HEAD"], check=True)107 108# Resolve the base and test commit, since if they are relative to HEAD we will109# check out the wrong commit below.110resolved_base_commit = subprocess.check_output(111 ["git", "rev-parse", args.base_commit]112).strip()113resolved_test_commit = subprocess.check_output(114 ["git", "rev-parse", args.test_commit]115).strip()116 117test_case_dir = os.path.dirname(args.test_case)118test_case_respfile = os.path.basename(args.test_case)119 120test_dir_test_case_dir = f"{test_dir}/testcase"121shutil.copytree(test_case_dir, test_dir_test_case_dir)122 123subprocess.run(["git", "checkout", resolved_base_commit], check=True)124generate_binary_variants("base")125 126subprocess.run(["git", "checkout", resolved_test_commit], check=True)127generate_binary_variants("test")128 129 130def hyperfine_link_command(case_name):131 return f'../lld-{case_name}$(({{iter}}%{args.num_binary_variants})) -flavor ld.lld @{test_case_respfile} {args.ldflags or ""}'132 133 134results_csv = f"{args.output_dir}/results.csv"135subprocess.run(136 [137 "hyperfine",138 "--export-csv",139 os.path.abspath(results_csv),140 "-P",141 "iter",142 "0",143 str(args.num_iterations - 1),144 hyperfine_link_command("base"),145 hyperfine_link_command("test"),146 ],147 check=True,148 cwd=test_dir_test_case_dir,149)150 151shutil.rmtree(test_dir)152