444 lines · python
1#!/usr/bin/env python2 3"""Check CFC - Check Compile Flow Consistency4 5This is a compiler wrapper for testing that code generation is consistent with6different compilation processes. It checks that code is not unduly affected by7compiler options or other changes which should not have side effects.8 9To use:10-Ensure that the compiler under test (i.e. clang, clang++) is on the PATH11-On Linux copy this script to the name of the compiler12 e.g. cp check_cfc.py clang && cp check_cfc.py clang++13-On Windows use setup.py to generate check_cfc.exe and copy that to clang.exe14 and clang++.exe15-Enable the desired checks in check_cfc.cfg (in the same directory as the16 wrapper)17 e.g.18[Checks]19dash_g_no_change = true20dash_s_no_change = false21 22-The wrapper can be run using its absolute path or added to PATH before the23 compiler under test24 e.g. export PATH=<path to check_cfc>:$PATH25-Compile as normal. The wrapper intercepts normal -c compiles and will return26 non-zero if the check fails.27 e.g.28$ clang -c test.cpp29Code difference detected with -g30--- /tmp/tmp5nv893.o31+++ /tmp/tmp6Vwjnc.o32@@ -1 +1 @@33- 0: 48 8b 05 51 0b 20 00 mov 0x200b51(%rip),%rax34+ 0: 48 39 3d 51 0b 20 00 cmp %rdi,0x200b51(%rip)35 36-To run LNT with Check CFC specify the absolute path to the wrapper to the --cc37 and --cxx options38 e.g.39 lnt runtest nt --cc <path to check_cfc>/clang \\40 --cxx <path to check_cfc>/clang++ ...41 42To add a new check:43-Create a new subclass of WrapperCheck44-Implement the perform_check() method. This should perform the alternate compile45 and do the comparison.46-Add the new check to check_cfc.cfg. The check has the same name as the47 subclass.48"""49 50from __future__ import absolute_import, division, print_function51 52import imp53import os54import platform55import shutil56import subprocess57import sys58import tempfile59import configparser60import io61 62import obj_diff63 64 65def is_windows():66 """Returns True if running on Windows."""67 return platform.system() == "Windows"68 69 70class WrapperStepException(Exception):71 """Exception type to be used when a step other than the original compile72 fails."""73 74 def __init__(self, msg, stdout, stderr):75 self.msg = msg76 self.stdout = stdout77 self.stderr = stderr78 79 80class WrapperCheckException(Exception):81 """Exception type to be used when a comparison check fails."""82 83 def __init__(self, msg):84 self.msg = msg85 86 87def main_is_frozen():88 """Returns True when running as a py2exe executable."""89 return (90 hasattr(sys, "frozen")91 or hasattr(sys, "importers") # new py2exe92 or imp.is_frozen("__main__") # old py2exe93 ) # tools/freeze94 95 96def get_main_dir():97 """Get the directory that the script or executable is located in."""98 if main_is_frozen():99 return os.path.dirname(sys.executable)100 return os.path.dirname(sys.argv[0])101 102 103def remove_dir_from_path(path_var, directory):104 """Remove the specified directory from path_var, a string representing105 PATH"""106 pathlist = path_var.split(os.pathsep)107 norm_directory = os.path.normpath(os.path.normcase(directory))108 pathlist = [109 x for x in pathlist if os.path.normpath(os.path.normcase(x)) != norm_directory110 ]111 return os.pathsep.join(pathlist)112 113 114def path_without_wrapper():115 """Returns the PATH variable modified to remove the path to this program."""116 scriptdir = get_main_dir()117 path = os.environ["PATH"]118 return remove_dir_from_path(path, scriptdir)119 120 121def flip_dash_g(args):122 """Search for -g in args. If it exists then return args without. If not then123 add it."""124 if "-g" in args:125 # Return args without any -g126 return [x for x in args if x != "-g"]127 else:128 # No -g, add one129 return args + ["-g"]130 131 132def derive_output_file(args):133 """Derive output file from the input file (if just one) or None134 otherwise."""135 infile = get_input_file(args)136 if infile is None:137 return None138 else:139 return "{}.o".format(os.path.splitext(infile)[0])140 141 142def get_output_file(args):143 """Return the output file specified by this command or None if not144 specified."""145 grabnext = False146 for arg in args:147 if grabnext:148 return arg149 if arg == "-o":150 # Specified as a separate arg151 grabnext = True152 elif arg.startswith("-o"):153 # Specified conjoined with -o154 return arg[2:]155 assert not grabnext156 157 return None158 159 160def is_output_specified(args):161 """Return true is output file is specified in args."""162 return get_output_file(args) is not None163 164 165def replace_output_file(args, new_name):166 """Replaces the specified name of an output file with the specified name.167 Assumes that the output file name is specified in the command line args."""168 replaceidx = None169 attached = False170 for idx, val in enumerate(args):171 if val == "-o":172 replaceidx = idx + 1173 attached = False174 elif val.startswith("-o"):175 replaceidx = idx176 attached = True177 178 if replaceidx is None:179 raise Exception180 replacement = new_name181 if attached:182 replacement = "-o" + new_name183 args[replaceidx] = replacement184 return args185 186 187def add_output_file(args, output_file):188 """Append an output file to args, presuming not already specified."""189 return args + ["-o", output_file]190 191 192def set_output_file(args, output_file):193 """Set the output file within the arguments. Appends or replaces as194 appropriate."""195 if is_output_specified(args):196 args = replace_output_file(args, output_file)197 else:198 args = add_output_file(args, output_file)199 return args200 201 202gSrcFileSuffixes = (".c", ".cpp", ".cxx", ".c++", ".cp", ".cc")203 204 205def get_input_file(args):206 """Return the input file string if it can be found (and there is only207 one)."""208 inputFiles = list()209 for arg in args:210 testarg = arg211 quotes = ('"', "'")212 while testarg.endswith(quotes):213 testarg = testarg[:-1]214 testarg = os.path.normcase(testarg)215 216 # Test if it is a source file217 if testarg.endswith(gSrcFileSuffixes):218 inputFiles.append(arg)219 if len(inputFiles) == 1:220 return inputFiles[0]221 else:222 return None223 224 225def set_input_file(args, input_file):226 """Replaces the input file with that specified."""227 infile = get_input_file(args)228 if infile:229 infile_idx = args.index(infile)230 args[infile_idx] = input_file231 return args232 else:233 # Could not find input file234 assert False235 236 237def is_normal_compile(args):238 """Check if this is a normal compile which will output an object file rather239 than a preprocess or link. args is a list of command line arguments."""240 compile_step = "-c" in args241 # Bitcode cannot be disassembled in the same way242 bitcode = "-flto" in args or "-emit-llvm" in args243 # Version and help are queries of the compiler and override -c if specified244 query = "--version" in args or "--help" in args245 # Options to output dependency files for make246 dependency = "-M" in args or "-MM" in args247 # Check if the input is recognised as a source file (this may be too248 # strong a restriction)249 input_is_valid = bool(get_input_file(args))250 return (251 compile_step and not bitcode and not query and not dependency and input_is_valid252 )253 254 255def run_step(command, my_env, error_on_failure):256 """Runs a step of the compilation. Reports failure as exception."""257 # Need to use shell=True on Windows as Popen won't use PATH otherwise.258 p = subprocess.Popen(259 command,260 stdout=subprocess.PIPE,261 stderr=subprocess.PIPE,262 env=my_env,263 shell=is_windows(),264 )265 (stdout, stderr) = p.communicate()266 if p.returncode != 0:267 raise WrapperStepException(error_on_failure, stdout, stderr)268 269 270def get_temp_file_name(suffix):271 """Get a temporary file name with a particular suffix. Let the caller be272 responsible for deleting it."""273 tf = tempfile.NamedTemporaryFile(suffix=suffix, delete=False)274 tf.close()275 return tf.name276 277 278class WrapperCheck(object):279 """Base class for a check. Subclass this to add a check."""280 281 def __init__(self, output_file_a):282 """Record the base output file that will be compared against."""283 self._output_file_a = output_file_a284 285 def perform_check(self, arguments, my_env):286 """Override this to perform the modified compilation and required287 checks."""288 raise NotImplementedError("Please Implement this method")289 290 291class dash_g_no_change(WrapperCheck):292 def perform_check(self, arguments, my_env):293 """Check if different code is generated with/without the -g flag."""294 output_file_b = get_temp_file_name(".o")295 296 alternate_command = list(arguments)297 alternate_command = flip_dash_g(alternate_command)298 alternate_command = set_output_file(alternate_command, output_file_b)299 run_step(alternate_command, my_env, "Error compiling with -g")300 301 # Compare disassembly (returns first diff if differs)302 difference = obj_diff.compare_object_files(self._output_file_a, output_file_b)303 if difference:304 raise WrapperCheckException(305 "Code difference detected with -g\n{}".format(difference)306 )307 308 # Clean up temp file if comparison okay309 os.remove(output_file_b)310 311 312class dash_s_no_change(WrapperCheck):313 def perform_check(self, arguments, my_env):314 """Check if compiling to asm then assembling in separate steps results315 in different code than compiling to object directly."""316 output_file_b = get_temp_file_name(".o")317 318 alternate_command = arguments + ["-via-file-asm"]319 alternate_command = set_output_file(alternate_command, output_file_b)320 run_step(alternate_command, my_env, "Error compiling with -via-file-asm")321 322 # Compare if object files are exactly the same323 exactly_equal = obj_diff.compare_exact(self._output_file_a, output_file_b)324 if not exactly_equal:325 # Compare disassembly (returns first diff if differs)326 difference = obj_diff.compare_object_files(327 self._output_file_a, output_file_b328 )329 if difference:330 raise WrapperCheckException(331 "Code difference detected with -S\n{}".format(difference)332 )333 334 # Code is identical, compare debug info335 dbgdifference = obj_diff.compare_debug_info(336 self._output_file_a, output_file_b337 )338 if dbgdifference:339 raise WrapperCheckException(340 "Debug info difference detected with -S\n{}".format(dbgdifference)341 )342 343 raise WrapperCheckException("Object files not identical with -S\n")344 345 # Clean up temp file if comparison okay346 os.remove(output_file_b)347 348 349if __name__ == "__main__":350 # Create configuration defaults from list of checks351 default_config = """352[Checks]353"""354 355 # Find all subclasses of WrapperCheck356 checks = [cls.__name__ for cls in vars()["WrapperCheck"].__subclasses__()]357 358 for c in checks:359 default_config += "{} = false\n".format(c)360 361 config = configparser.RawConfigParser()362 config.readfp(io.BytesIO(default_config))363 scriptdir = get_main_dir()364 config_path = os.path.join(scriptdir, "check_cfc.cfg")365 try:366 config.read(os.path.join(config_path))367 except:368 print("Could not read config from {}, " "using defaults.".format(config_path))369 370 my_env = os.environ.copy()371 my_env["PATH"] = path_without_wrapper()372 373 arguments_a = list(sys.argv)374 375 # Prevent infinite loop if called with absolute path.376 arguments_a[0] = os.path.basename(arguments_a[0])377 378 # Basic correctness check379 enabled_checks = [380 check_name for check_name in checks if config.getboolean("Checks", check_name)381 ]382 checks_comma_separated = ", ".join(enabled_checks)383 print("Check CFC, checking: {}".format(checks_comma_separated))384 385 # A - original compilation386 output_file_orig = get_output_file(arguments_a)387 if output_file_orig is None:388 output_file_orig = derive_output_file(arguments_a)389 390 p = subprocess.Popen(arguments_a, env=my_env, shell=is_windows())391 p.communicate()392 if p.returncode != 0:393 sys.exit(p.returncode)394 395 if not is_normal_compile(arguments_a) or output_file_orig is None:396 # Bail out here if we can't apply checks in this case.397 # Does not indicate an error.398 # Maybe not straight compilation (e.g. -S or --version or -flto)399 # or maybe > 1 input files.400 sys.exit(0)401 402 # Sometimes we generate files which have very long names which can't be403 # read/disassembled. This will exit early if we can't find the file we404 # expected to be output.405 if not os.path.isfile(output_file_orig):406 sys.exit(0)407 408 # Copy output file to a temp file409 temp_output_file_orig = get_temp_file_name(".o")410 shutil.copyfile(output_file_orig, temp_output_file_orig)411 412 # Run checks, if they are enabled in config and if they are appropriate for413 # this command line.414 current_module = sys.modules[__name__]415 for check_name in checks:416 if config.getboolean("Checks", check_name):417 class_ = getattr(current_module, check_name)418 checker = class_(temp_output_file_orig)419 try:420 checker.perform_check(arguments_a, my_env)421 except WrapperCheckException as e:422 # Check failure423 print(424 "{} {}".format(get_input_file(arguments_a), e.msg), file=sys.stderr425 )426 427 # Remove file to comply with build system expectations (no428 # output file if failed)429 os.remove(output_file_orig)430 sys.exit(1)431 432 except WrapperStepException as e:433 # Compile step failure434 print(e.msg, file=sys.stderr)435 print("*** stdout ***", file=sys.stderr)436 print(e.stdout, file=sys.stderr)437 print("*** stderr ***", file=sys.stderr)438 print(e.stderr, file=sys.stderr)439 440 # Remove file to comply with build system expectations (no441 # output file if failed)442 os.remove(output_file_orig)443 sys.exit(1)444