280 lines · python
1# DExTer : Debugging Experience Tester2# ~~~~~~ ~ ~~ ~ ~~3#4# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.5# See https://llvm.org/LICENSE.txt for license information.6# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception7"""Test tool."""8 9import math10import os11import csv12import pickle13import shutil14import platform15 16from dex.command.ParseCommand import get_command_infos17from dex.debugger.Debuggers import run_debugger_subprocess18from dex.debugger.DebuggerControllers.DefaultController import DefaultController19from dex.debugger.DebuggerControllers.ConditionalController import ConditionalController20from dex.dextIR.DextIR import DextIR21from dex.heuristic import Heuristic22from dex.tools import TestToolBase23from dex.utils.Exceptions import DebuggerException24from dex.utils.Exceptions import BuildScriptException, HeuristicException25from dex.utils.PrettyOutputBase import Stream26from dex.utils.ReturnCode import ReturnCode27 28 29class TestCase(object):30 def __init__(self, context, name, heuristic, error):31 self.context = context32 self.name = name33 self.heuristic = heuristic34 self.error = error35 36 @property37 def penalty(self):38 try:39 return self.heuristic.penalty40 except AttributeError:41 return float("nan")42 43 @property44 def max_penalty(self):45 try:46 return self.heuristic.max_penalty47 except AttributeError:48 return float("nan")49 50 @property51 def score(self):52 try:53 return self.heuristic.score54 except AttributeError:55 return float("nan")56 57 def __str__(self):58 if self.error and self.context.options.verbose:59 verbose_error = str(self.error)60 else:61 verbose_error = ""62 63 if self.error:64 script_error = (65 " : {}".format(self.error.script_error.splitlines()[0])66 if getattr(self.error, "script_error", None)67 else ""68 )69 70 error = " [{}{}]".format(str(self.error).splitlines()[0], script_error)71 else:72 error = ""73 74 try:75 summary = self.heuristic.summary_string76 except AttributeError:77 summary = "<r>nan/nan (nan)</>"78 return "{}: {}{}\n{}".format(self.name, summary, error, verbose_error)79 80 81class Tool(TestToolBase):82 """Run the specified DExTer test(s) with the specified compiler and linker83 options and produce a dextIR file as well as printing out the debugging84 experience score calculated by the DExTer heuristic.85 """86 87 def __init__(self, *args, **kwargs):88 super(Tool, self).__init__(*args, **kwargs)89 self._test_cases = []90 91 @property92 def name(self):93 return "DExTer test"94 95 def add_tool_arguments(self, parser, defaults):96 parser.add_argument(97 "--fail-lt",98 type=float,99 default=0.0, # By default TEST always succeeds.100 help="exit with status FAIL(2) if the test result"101 " is less than this value.",102 metavar="<float>",103 )104 parser.add_argument(105 "--calculate-average",106 action="store_true",107 help="calculate the average score of every test run",108 )109 super(Tool, self).add_tool_arguments(parser, defaults)110 111 def _init_debugger_controller(self):112 step_collection = DextIR(113 executable_path=self.context.options.executable,114 source_paths=self.context.options.source_files,115 dexter_version=self.context.version,116 )117 118 step_collection.commands, new_source_files = get_command_infos(119 self.context.options.test_files, self.context.options.source_root_dir120 )121 122 self.context.options.source_files.extend(list(new_source_files))123 124 cond_controller_cmds = ["DexLimitSteps", "DexStepFunction", "DexContinue"]125 if any(c in step_collection.commands for c in cond_controller_cmds):126 debugger_controller = ConditionalController(self.context, step_collection)127 else:128 debugger_controller = DefaultController(self.context, step_collection)129 130 return debugger_controller131 132 def _get_steps(self):133 """Generate a list of debugger steps from a test case."""134 debugger_controller = self._init_debugger_controller()135 debugger_controller = run_debugger_subprocess(136 debugger_controller, self.context.working_directory.path137 )138 steps = debugger_controller.step_collection139 return steps140 141 def _get_results_basename(self, test_name):142 def splitall(x):143 while len(x) > 0:144 x, y = os.path.split(x)145 yield y146 147 all_components = reversed([x for x in splitall(test_name)])148 return "_".join(all_components)149 150 def _get_results_path(self, test_name):151 """Returns the path to the test results directory for the test denoted152 by test_name.153 """154 assert self.context.options.results_directory is not None155 return os.path.join(156 self.context.options.results_directory,157 self._get_results_basename(test_name),158 )159 160 def _get_results_text_path(self, test_name):161 """Returns path results .txt file for test denoted by test_name."""162 test_results_path = self._get_results_path(test_name)163 return "{}.txt".format(test_results_path)164 165 def _get_results_pickle_path(self, test_name):166 """Returns path results .dextIR file for test denoted by test_name."""167 test_results_path = self._get_results_path(test_name)168 return "{}.dextIR".format(test_results_path)169 170 def _record_steps(self, test_name, steps):171 """Write out the set of steps out to the test's .txt and .json172 results file if a results directory has been specified.173 """174 if self.context.options.results_directory:175 output_text_path = self._get_results_text_path(test_name)176 with open(output_text_path, "w") as fp:177 self.context.o.auto(str(steps), stream=Stream(fp))178 179 output_dextIR_path = self._get_results_pickle_path(test_name)180 with open(output_dextIR_path, "wb") as fp:181 pickle.dump(steps, fp, protocol=pickle.HIGHEST_PROTOCOL)182 183 def _record_score(self, test_name, heuristic):184 """Write out the test's heuristic score to the results .txt file185 if a results directory has been specified.186 """187 if self.context.options.results_directory:188 output_text_path = self._get_results_text_path(test_name)189 with open(output_text_path, "a") as fp:190 self.context.o.auto(heuristic.verbose_output, stream=Stream(fp))191 192 def _record_test_and_display(self, test_case):193 """Output test case to o stream and record test case internally for194 handling later.195 """196 self.context.o.auto(test_case)197 self._test_cases.append(test_case)198 199 def _record_failed_test(self, test_name, exception):200 """Instantiate a failed test case with failure exception and201 store internally.202 """203 test_case = TestCase(self.context, test_name, None, exception)204 self._record_test_and_display(test_case)205 206 def _record_successful_test(self, test_name, steps, heuristic):207 """Instantiate a successful test run, store test for handling later.208 Display verbose output for test case if required.209 """210 test_case = TestCase(self.context, test_name, heuristic, None)211 self._record_test_and_display(test_case)212 if self.context.options.verbose:213 self.context.o.auto("\n{}\n".format(steps))214 self.context.o.auto(heuristic.verbose_output)215 216 def _run_test(self, test_name):217 """Attempt to run test files specified in options.source_files. Store218 result internally in self._test_cases.219 """220 try:221 if self.context.options.binary:222 if platform.system() == 'Darwin' and os.path.exists(self.context.options.binary + '.dSYM'):223 # On Darwin, the debug info is in the .dSYM which might not be found by lldb, copy it into the tmp working directory224 shutil.copytree(self.context.options.binary + '.dSYM', self.context.options.executable + '.dSYM')225 # Copy user's binary into the tmp working directory.226 shutil.copy(227 self.context.options.binary, self.context.options.executable228 )229 steps = self._get_steps()230 self._record_steps(test_name, steps)231 heuristic_score = Heuristic(self.context, steps)232 self._record_score(test_name, heuristic_score)233 except (BuildScriptException, DebuggerException, HeuristicException) as e:234 self._record_failed_test(test_name, e)235 return236 237 self._record_successful_test(test_name, steps, heuristic_score)238 return239 240 def _handle_results(self) -> ReturnCode:241 return_code = ReturnCode.OK242 options = self.context.options243 244 if not options.verbose:245 self.context.o.auto("\n")246 247 if options.calculate_average:248 # Calculate and print the average score249 score_sum = 0.0250 num_tests = 0251 for test_case in self._test_cases:252 score = test_case.score253 if not test_case.error and not math.isnan(score):254 score_sum += test_case.score255 num_tests += 1256 257 if num_tests != 0:258 print("@avg: ({:.4f})".format(score_sum / num_tests))259 260 has_failed = lambda test: test.score < options.fail_lt or test.error261 if any(map(has_failed, self._test_cases)):262 return_code = ReturnCode.FAIL263 264 if options.results_directory:265 summary_path = os.path.join(options.results_directory, "summary.csv")266 with open(summary_path, mode="w", newline="") as fp:267 writer = csv.writer(fp, delimiter=",")268 writer.writerow(["Test Case", "Score", "Error"])269 270 for test_case in self._test_cases:271 writer.writerow(272 [273 test_case.name,274 "{:.4f}".format(test_case.score),275 test_case.error,276 ]277 )278 279 return return_code280