215 lines · python
1#!/usr/bin/env python32# ===----------------------------------------------------------------------===##3#4# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.5# See https://llvm.org/LICENSE.txt for license information.6# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception7#8# ===----------------------------------------------------------------------===##9"""Script to bisect over files in an rsp file.10 11This is mostly used for detecting which file contains a miscompile between two12compiler revisions. It does this by bisecting over an rsp file. Between two13build directories, this script will make the rsp file reference the current14build directory's version of some set of the rsp's object files/libraries, and15reference the other build directory's version of the same files for the16remaining set of object files/libraries.17 18Build the target in two separate directories with the two compiler revisions,19keeping the rsp file around since ninja by default deletes the rsp file after20building.21$ ninja -d keeprsp mytarget22 23Create a script to build the target and run an interesting test. Get the24command to build the target via25$ ninja -t commands | grep mytarget26The command to build the target should reference the rsp file.27This script doesn't care if the test script returns 0 or 1 for specifically the28successful or failing test, just that the test script returns a different29return code for success vs failure.30Since the command that `ninja -t commands` is run from the build directory,31usually the test script cd's to the build directory.32 33$ rsp_bisect.py --test=path/to/test_script --rsp=path/to/build/target.rsp34 --other_rel_path=../Other35where --other_rel_path is the relative path from the first build directory to36the other build directory. This is prepended to files in the rsp.37 38 39For a full example, if the foo target is suspected to contain a miscompile in40some file, have two different build directories, buildgood/ and buildbad/ and41run42$ ninja -d keeprsp foo43in both so we have two versions of all relevant object files that may contain a44miscompile, one built by a good compiler and one by a bad compiler.45 46In buildgood/, run47$ ninja -t commands | grep '-o .*foo'48to get the command to link the files together. It may look something like49 clang -o foo @foo.rsp50 51Now create a test script that runs the link step and whatever test reproduces a52miscompile and returns a non-zero exit code when there is a miscompile. For53example54```55 #!/bin/bash56 # immediately bail out of script if any command returns a non-zero return code57 set -e58 clang -o foo @foo.rsp59 ./foo60```61 62With buildgood/ as the working directory, run63$ path/to/llvm-project/llvm/utils/rsp_bisect.py \64 --test=path/to/test_script --rsp=./foo.rsp --other_rel_path=../buildbad/65If rsp_bisect is successful, it will print the first file in the rsp file that66when using the bad build directory's version causes the test script to return a67different return code. foo.rsp.0 and foo.rsp.1 will also be written. foo.rsp.068will be a copy of foo.rsp with the relevant file using the version in69buildgood/, and foo.rsp.1 will be a copy of foo.rsp with the relevant file70using the version in buildbad/.71 72"""73 74import argparse75import os76import subprocess77import sys78 79 80def is_path(s):81 return "/" in s82 83 84def run_test(test):85 """Runs the test and returns whether it was successful or not."""86 return subprocess.run([test], capture_output=True).returncode == 087 88 89def modify_rsp(rsp_entries, other_rel_path, modify_after_num):90 """Create a modified rsp file for use in bisection.91 92 Returns a new list from rsp.93 For each file in rsp after the first modify_after_num files, prepend94 other_rel_path.95 """96 ret = []97 for r in rsp_entries:98 if is_path(r):99 if modify_after_num == 0:100 r = os.path.join(other_rel_path, r)101 else:102 modify_after_num -= 1103 ret.append(r)104 assert modify_after_num == 0105 return ret106 107 108def test_modified_rsp(test, modified_rsp_entries, rsp_path):109 """Write the rsp file to disk and run the test."""110 with open(rsp_path, "w") as f:111 f.write(" ".join(modified_rsp_entries))112 return run_test(test)113 114 115def bisect(test, zero_result, rsp_entries, num_files_in_rsp, other_rel_path, rsp_path):116 """Bisect over rsp entries.117 118 Args:119 zero_result: the test result when modify_after_num is 0.120 121 Returns:122 The index of the file in the rsp file where the test result changes.123 """124 lower = 0125 upper = num_files_in_rsp126 while lower != upper - 1:127 assert lower < upper - 1128 mid = int((lower + upper) / 2)129 assert lower != mid and mid != upper130 print("Trying {} ({}-{})".format(mid, lower, upper))131 result = test_modified_rsp(132 test, modify_rsp(rsp_entries, other_rel_path, mid), rsp_path133 )134 if zero_result == result:135 lower = mid136 else:137 upper = mid138 return upper139 140 141def main():142 parser = argparse.ArgumentParser()143 parser.add_argument(144 "--test", help="Binary to test if current setup is good or bad", required=True145 )146 parser.add_argument("--rsp", help="rsp file", required=True)147 parser.add_argument(148 "--other-rel-path",149 help="Relative path from current build directory to other build "150 + 'directory, e.g. from "out/Default" to "out/Other" specify "../Other"',151 required=True,152 )153 args = parser.parse_args()154 155 with open(args.rsp, "r") as f:156 rsp_entries = f.read()157 rsp_entries = rsp_entries.split()158 num_files_in_rsp = sum(1 for a in rsp_entries if is_path(a))159 if num_files_in_rsp == 0:160 print("No files in rsp?")161 return 1162 print("{} files in rsp".format(num_files_in_rsp))163 164 try:165 print("Initial testing")166 test0 = test_modified_rsp(167 args.test, modify_rsp(rsp_entries, args.other_rel_path, 0), args.rsp168 )169 test_all = test_modified_rsp(170 args.test,171 modify_rsp(rsp_entries, args.other_rel_path, num_files_in_rsp),172 args.rsp,173 )174 175 if test0 == test_all:176 print("Test returned same exit code for both build directories")177 return 1178 179 print("First build directory returned " + ("0" if test_all else "1"))180 181 result = bisect(182 args.test,183 test0,184 rsp_entries,185 num_files_in_rsp,186 args.other_rel_path,187 args.rsp,188 )189 print(190 "First file change: {} ({})".format(191 list(filter(is_path, rsp_entries))[result - 1], result192 )193 )194 195 rsp_out_0 = args.rsp + ".0"196 rsp_out_1 = args.rsp + ".1"197 with open(rsp_out_0, "w") as f:198 f.write(" ".join(modify_rsp(rsp_entries, args.other_rel_path, result - 1)))199 with open(rsp_out_1, "w") as f:200 f.write(" ".join(modify_rsp(rsp_entries, args.other_rel_path, result)))201 print(202 "Bisection point rsp files written to {} and {}".format(203 rsp_out_0, rsp_out_1204 )205 )206 finally:207 # Always make sure to write the original rsp file contents back so it's208 # less of a pain to rerun this script.209 with open(args.rsp, "w") as f:210 f.write(" ".join(rsp_entries))211 212 213if __name__ == "__main__":214 sys.exit(main())215