299 lines · python
1"""2Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.3See https://llvm.org/LICENSE.txt for license information.4SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception5 6Provides the LLDBTestResult class, which holds information about progress7and results of a single test run.8"""9 10# System modules11import os12import traceback13 14# Third-party modules15import unittest16 17# LLDB Modules18from . import configuration19from lldbsuite.test_event import build_exception20 21 22class LLDBTestResult(unittest.TextTestResult):23 """24 Enforce a singleton pattern to allow introspection of test progress.25 26 Overwrite addError(), addFailure(), and addExpectedFailure() methods27 to enable each test instance to track its failure/error status. It28 is used in the LLDB test framework to emit detailed trace messages29 to a log file for easier human inspection of test failures/errors.30 """31 32 __singleton__ = None33 __ignore_singleton__ = False34 35 @staticmethod36 def getTerminalSize():37 import os38 39 env = os.environ40 41 def ioctl_GWINSZ(fd):42 try:43 import fcntl44 import termios45 import struct46 47 cr = struct.unpack("hh", fcntl.ioctl(fd, termios.TIOCGWINSZ, "1234"))48 except:49 return50 return cr51 52 cr = ioctl_GWINSZ(0) or ioctl_GWINSZ(1) or ioctl_GWINSZ(2)53 if not cr:54 try:55 fd = os.open(os.ctermid(), os.O_RDONLY)56 cr = ioctl_GWINSZ(fd)57 os.close(fd)58 except:59 pass60 if not cr:61 cr = (env.get("LINES", 25), env.get("COLUMNS", 80))62 return int(cr[1]), int(cr[0])63 64 def __init__(self, *args):65 if not LLDBTestResult.__ignore_singleton__ and LLDBTestResult.__singleton__:66 raise Exception("LLDBTestResult instantiated more than once")67 super(LLDBTestResult, self).__init__(*args)68 LLDBTestResult.__singleton__ = self69 # Now put this singleton into the lldb module namespace.70 configuration.test_result = self71 # Computes the format string for displaying the counter.72 counterWidth = len(str(configuration.suite.countTestCases()))73 self.fmt = "%" + str(counterWidth) + "d: "74 self.indentation = " " * (counterWidth + 2)75 # This counts from 1 .. suite.countTestCases().76 self.counter = 077 (width, height) = LLDBTestResult.getTerminalSize()78 79 def _config_string(self, test):80 compiler = getattr(test, "getCompiler", None)81 arch = getattr(test, "getArchitecture", None)82 return "%s-%s" % (compiler() if compiler else "", arch() if arch else "")83 84 def _exc_info_to_string(self, err, test):85 """Overrides superclass TestResult's method in order to append86 our test config info string to the exception info string."""87 if hasattr(test, "getArchitecture") and hasattr(test, "getCompiler"):88 return "%sConfig=%s-%s" % (89 super(LLDBTestResult, self)._exc_info_to_string(err, test),90 test.getArchitecture(),91 test.getCompiler(),92 )93 else:94 return super(LLDBTestResult, self)._exc_info_to_string(err, test)95 96 def getDescription(self, test):97 doc_first_line = test.shortDescription()98 if self.descriptions and doc_first_line:99 return "\n".join((str(test), self.indentation + doc_first_line))100 else:101 return str(test)102 103 def _getTestPath(self, test):104 # Use test.test_filename if the test was created with105 # lldbinline.MakeInlineTest().106 if test is None:107 return ""108 elif hasattr(test, "test_filename"):109 return test.test_filename110 else:111 import inspect112 113 return inspect.getsourcefile(test.__class__)114 115 def _getFileBasedCategories(self, test):116 """117 Returns the list of categories to which this test case belongs by118 collecting values of "categories" files. We start at the folder the test is in119 and traverse the hierarchy upwards until the test-suite root directory.120 """121 start_path = self._getTestPath(test)122 123 import os.path124 125 folder = os.path.dirname(start_path)126 127 from lldbsuite import lldb_test_root as test_root128 129 if test_root != os.path.commonprefix([folder, test_root]):130 raise Exception(131 "The test file %s is outside the test root directory" % start_path132 )133 134 categories = set()135 while not os.path.samefile(folder, test_root):136 categories_file_name = os.path.join(folder, "categories")137 if os.path.exists(categories_file_name):138 categories_file = open(categories_file_name, "r")139 categories_str = categories_file.readline().strip()140 categories_file.close()141 categories.update(categories_str.split(","))142 folder = os.path.dirname(folder)143 144 return list(categories)145 146 def getCategoriesForTest(self, test):147 """148 Gets all the categories for the currently running test method in test case149 """150 test_categories = []151 test_categories.extend(getattr(test, "categories", []))152 153 test_method = getattr(test, test._testMethodName)154 if test_method is not None:155 test_categories.extend(getattr(test_method, "categories", []))156 157 test_categories.extend(self._getFileBasedCategories(test))158 159 return test_categories160 161 def hardMarkAsSkipped(self, test):162 getattr(test, test._testMethodName).__func__.__unittest_skip__ = True163 getattr(164 test, test._testMethodName165 ).__func__.__unittest_skip_why__ = (166 "test case does not fall in any category of interest for this run"167 )168 169 def checkExclusion(self, exclusion_list, name):170 if exclusion_list:171 import re172 173 for item in exclusion_list:174 if re.search(item, name):175 return True176 return False177 178 def checkCategoryExclusion(self, exclusion_list, test):179 return not set(exclusion_list).isdisjoint(self.getCategoriesForTest(test))180 181 def startTest(self, test):182 if configuration.shouldSkipBecauseOfCategories(self.getCategoriesForTest(test)):183 self.hardMarkAsSkipped(test)184 if self.checkExclusion(configuration.skip_tests, test.id()):185 self.hardMarkAsSkipped(test)186 187 self.counter += 1188 test.test_number = self.counter189 if self.showAll:190 self.stream.write(self.fmt % self.counter)191 super(LLDBTestResult, self).startTest(test)192 193 def addSuccess(self, test):194 if self.checkExclusion(195 configuration.xfail_tests, test.id()196 ) or self.checkCategoryExclusion(configuration.xfail_categories, test):197 self.addUnexpectedSuccess(test, None)198 return199 200 super(LLDBTestResult, self).addSuccess(test)201 self.stream.write(202 "PASS: LLDB (%s) :: %s\n" % (self._config_string(test), str(test))203 )204 205 def _isBuildError(self, err_tuple):206 exception = err_tuple[1]207 return isinstance(exception, build_exception.BuildError)208 209 def _saveBuildErrorTuple(self, test, err):210 # Adjust the error description so it prints the build command and build error211 # rather than an uninformative Python backtrace.212 build_error = err[1]213 error_description = "{}\nTest Directory:\n{}".format(214 str(build_error), os.path.dirname(self._getTestPath(test))215 )216 self.errors.append((test, error_description))217 self._mirrorOutput = True218 219 def addError(self, test, err):220 configuration.sdir_has_content = True221 if self._isBuildError(err):222 self._saveBuildErrorTuple(test, err)223 else:224 super(LLDBTestResult, self).addError(test, err)225 226 method = getattr(test, "markError", None)227 if method:228 method()229 self.stream.write(230 "FAIL: LLDB (%s) :: %s\n" % (self._config_string(test), str(test))231 )232 233 def addCleanupError(self, test, err):234 configuration.sdir_has_content = True235 super(LLDBTestResult, self).addCleanupError(test, err)236 method = getattr(test, "markCleanupError", None)237 if method:238 method()239 self.stream.write(240 "CLEANUP ERROR: LLDB (%s) :: %s\n%s\n"241 % (self._config_string(test), str(test), traceback.format_exc())242 )243 244 def addFailure(self, test, err):245 if self.checkExclusion(246 configuration.xfail_tests, test.id()247 ) or self.checkCategoryExclusion(configuration.xfail_categories, test):248 self.addExpectedFailure(test, err)249 return250 251 configuration.sdir_has_content = True252 super(LLDBTestResult, self).addFailure(test, err)253 method = getattr(test, "markFailure", None)254 if method:255 method()256 self.stream.write(257 "FAIL: LLDB (%s) :: %s\n" % (self._config_string(test), str(test))258 )259 if configuration.use_categories:260 test_categories = self.getCategoriesForTest(test)261 for category in test_categories:262 if category in configuration.failures_per_category:263 configuration.failures_per_category[category] = (264 configuration.failures_per_category[category] + 1265 )266 else:267 configuration.failures_per_category[category] = 1268 269 def addExpectedFailure(self, test, err):270 configuration.sdir_has_content = True271 super(LLDBTestResult, self).addExpectedFailure(test, err)272 method = getattr(test, "markExpectedFailure", None)273 if method:274 method(err)275 self.stream.write(276 "XFAIL: LLDB (%s) :: %s\n" % (self._config_string(test), str(test))277 )278 279 def addSkip(self, test, reason):280 configuration.sdir_has_content = True281 super(LLDBTestResult, self).addSkip(test, reason)282 method = getattr(test, "markSkippedTest", None)283 if method:284 method()285 self.stream.write(286 "UNSUPPORTED: LLDB (%s) :: %s (%s) \n"287 % (self._config_string(test), str(test), reason)288 )289 290 def addUnexpectedSuccess(self, test):291 configuration.sdir_has_content = True292 super(LLDBTestResult, self).addUnexpectedSuccess(test)293 method = getattr(test, "markUnexpectedSuccess", None)294 if method:295 method()296 self.stream.write(297 "XPASS: LLDB (%s) :: %s\n" % (self._config_string(test), str(test))298 )299