321 lines · python
1# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.2# See https://llvm.org/LICENSE.txt for license information.3# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception4"""Library to parse JUnit XML files and return a markdown report."""5 6from typing import TypedDict, Optional7import platform8 9from junitparser import JUnitXml, Failure10 11 12# This data structure should match the definition in llvm-zorg in13# premerge/advisor/advisor_lib.py14# TODO(boomanaiden154): Drop the Optional here and switch to str | None when15# we require Python 3.10.16class FailureExplanation(TypedDict):17 name: str18 explained: bool19 reason: Optional[str]20 21 22SEE_BUILD_FILE_STR = "Download the build's log file to see the details."23UNRELATED_FAILURES_STR = (24 "If these failures are unrelated to your changes (for example "25 "tests are broken or flaky at HEAD), please open an issue at "26 "https://github.com/llvm/llvm-project/issues and add the "27 "`infrastructure` label."28)29# The maximum number of lines to pull from a ninja failure.30NINJA_LOG_SIZE_THRESHOLD = 50031 32 33def _parse_ninja_log(ninja_log: list[str]) -> list[tuple[str, str]]:34 """Parses an individual ninja log."""35 failures = []36 index = 037 while index < len(ninja_log):38 while index < len(ninja_log) and not ninja_log[index].startswith("FAILED:"):39 index += 140 if index == len(ninja_log):41 # We hit the end of the log without finding a build failure, go to42 # the next log.43 return failures44 # If we are doing a build with LLVM_ENABLE_RUNTIMES, we can have nested45 # ninja invocations. The sub-ninja will print that a subcommand failed,46 # and then the outer ninja will list the command that failed. We should47 # ignore the outer failure.48 if ninja_log[index - 1].startswith("ninja: build stopped:"):49 index += 150 continue51 # We are trying to parse cases like the following:52 #53 # [4/5] test/4.stamp54 # FAILED: touch test/4.stamp55 # touch test/4.stamp56 #57 # index will point to the line that starts with Failed:. The progress58 # indicator is sometimes the line before this ([4/5] test/4.stamp) and59 # will contain a pretty printed version of the target being built60 # (test/4.stamp) when accurate. We instead parse the failed line rather61 # than the progress indicator as the progress indicator may not be62 # aligned with the failure.63 failing_action = ninja_log[index].split("FAILED: ")[1]64 failure_log = []65 while (66 index < len(ninja_log)67 and not ninja_log[index].startswith("[")68 and not ninja_log[index].startswith("ninja: build stopped:")69 and len(failure_log) < NINJA_LOG_SIZE_THRESHOLD70 ):71 failure_log.append(ninja_log[index])72 index += 173 failures.append((failing_action, "\n".join(failure_log)))74 return failures75 76 77def find_failure_in_ninja_logs(ninja_logs: list[list[str]]) -> list[tuple[str, str]]:78 """Extracts failure messages from ninja output.79 80 This function takes stdout/stderr from ninja in the form of a list of files81 represented as a list of lines. This function then returns tuples containing82 the name of the target and the error message.83 84 Args:85 ninja_logs: A list of files in the form of a list of lines representing the log86 files captured from ninja.87 88 Returns:89 A list of tuples. The first string is the name of the target that failed. The90 second string is the error message.91 """92 failures = []93 for ninja_log in ninja_logs:94 log_failures = _parse_ninja_log(ninja_log)95 failures.extend(log_failures)96 return failures97 98 99def _format_failures(100 failures: list[tuple[str, str]], failure_explanations: dict[str, FailureExplanation]101) -> list[str]:102 """Formats failures into summary views for the report."""103 output = []104 for build_failure in failures:105 failed_action, failure_message = build_failure106 failure_explanation = None107 if failed_action in failure_explanations:108 failure_explanation = failure_explanations[failed_action]109 output.append("<details>")110 if failure_explanation:111 output.extend(112 [113 f"<summary>{failed_action} (Likely Already Failing)</summary>" "",114 failure_explanation["reason"],115 "",116 ]117 )118 else:119 output.extend([f"<summary>{failed_action}</summary>", ""])120 output.extend(121 [122 "```",123 failure_message,124 "```",125 "</details>",126 ]127 )128 return output129 130 131def get_failures(junit_objects) -> dict[str, list[tuple[str, str]]]:132 failures = {}133 for results in junit_objects:134 for testsuite in results:135 for test in testsuite:136 if (137 not test.is_passed138 and test.result139 and isinstance(test.result[0], Failure)140 ):141 if failures.get(testsuite.name) is None:142 failures[testsuite.name] = []143 failures[testsuite.name].append(144 (test.classname + "/" + test.name, test.result[0].text)145 )146 return failures147 148 149# Set size_limit to limit the byte size of the report. The default is 1MB as this150# is the most that can be put into an annotation. If the generated report exceeds151# this limit and failures are listed, it will be generated again without failures152# listed. This minimal report will always fit into an annotation.153# If include failures is False, total number of test will be reported but their names154# and output will not be.155def generate_report(156 title,157 return_code,158 junit_objects,159 ninja_logs: list[list[str]],160 size_limit=1024 * 1024,161 list_failures=True,162 failure_explanations_list: list[FailureExplanation] = [],163):164 failures = get_failures(junit_objects)165 tests_run = 0166 tests_skipped = 0167 tests_failed = 0168 169 failure_explanations: dict[str, FailureExplanation] = {}170 for failure_explanation in failure_explanations_list:171 if not failure_explanation["explained"]:172 continue173 failure_explanations[failure_explanation["name"]] = failure_explanation174 175 for results in junit_objects:176 for testsuite in results:177 tests_run += testsuite.tests178 tests_skipped += testsuite.skipped179 tests_failed += testsuite.failures180 181 report = [f"# {title}", ""]182 183 if tests_run == 0:184 if return_code == 0:185 report.extend(186 [187 "The build succeeded and no tests ran. This is expected in some "188 "build configurations."189 ]190 )191 else:192 ninja_failures = find_failure_in_ninja_logs(ninja_logs)193 if not ninja_failures:194 report.extend(195 [196 "The build failed before running any tests. Detailed "197 "information about the build failure could not be "198 "automatically obtained.",199 "",200 SEE_BUILD_FILE_STR,201 "",202 UNRELATED_FAILURES_STR,203 ]204 )205 else:206 report.extend(207 [208 "The build failed before running any tests. Click on a "209 "failure below to see the details.",210 "",211 ]212 )213 report.extend(_format_failures(ninja_failures, failure_explanations))214 report.extend(215 [216 "",217 UNRELATED_FAILURES_STR,218 ]219 )220 return "\n".join(report)221 222 tests_passed = tests_run - tests_skipped - tests_failed223 224 def plural(num_tests):225 return "test" if num_tests == 1 else "tests"226 227 if tests_passed:228 report.append(f"* {tests_passed} {plural(tests_passed)} passed")229 if tests_skipped:230 report.append(f"* {tests_skipped} {plural(tests_skipped)} skipped")231 if tests_failed:232 report.append(f"* {tests_failed} {plural(tests_failed)} failed")233 234 if not list_failures:235 report.extend(236 [237 "",238 "Failed tests and their output was too large to report. "239 + SEE_BUILD_FILE_STR,240 ]241 )242 elif failures:243 report.extend(244 ["", "## Failed Tests", "(click on a test name to see its output)"]245 )246 247 for testsuite_name, failures in failures.items():248 report.extend(["", f"### {testsuite_name}"])249 report.extend(_format_failures(failures, failure_explanations))250 elif return_code != 0:251 # No tests failed but the build was in a failed state. Bring this to the user's252 # attention.253 ninja_failures = find_failure_in_ninja_logs(ninja_logs)254 if not ninja_failures:255 report.extend(256 [257 "",258 "All tests passed but another part of the build **failed**. "259 "Information about the build failure could not be automatically "260 "obtained.",261 "",262 SEE_BUILD_FILE_STR,263 ]264 )265 else:266 report.extend(267 [268 "",269 "All tests passed but another part of the build **failed**. Click on "270 "a failure below to see the details.",271 "",272 ]273 )274 report.extend(_format_failures(ninja_failures, failure_explanations))275 276 if failures or return_code != 0:277 report.extend(["", UNRELATED_FAILURES_STR])278 279 report = "\n".join(report)280 if len(report.encode("utf-8")) > size_limit:281 return generate_report(282 title,283 return_code,284 junit_objects,285 size_limit,286 list_failures=False,287 )288 289 return report290 291 292def load_info_from_files(build_log_files):293 junit_files = [294 junit_file for junit_file in build_log_files if junit_file.endswith(".xml")295 ]296 ninja_log_files = [297 ninja_log for ninja_log in build_log_files if ninja_log.endswith(".log")298 ]299 ninja_logs = []300 for ninja_log_file in ninja_log_files:301 with open(ninja_log_file, "r") as ninja_log_file_handle:302 ninja_logs.append(303 [log_line.strip() for log_line in ninja_log_file_handle.readlines()]304 )305 return [JUnitXml.fromfile(p) for p in junit_files], ninja_logs306 307 308def generate_report_from_files(title, return_code, build_log_files):309 junit_objects, ninja_logs = load_info_from_files(build_log_files)310 return generate_report(title, return_code, junit_objects, ninja_logs)311 312 313def compute_platform_title() -> str:314 logo = ":window:" if platform.system() == "Windows" else ":penguin:"315 # On Linux the machine value is x86_64 on Windows it is AMD64.316 if platform.machine() == "x86_64" or platform.machine() == "AMD64":317 arch = "x64"318 else:319 arch = platform.machine()320 return f"{logo} {platform.system()} {arch} Test Results"321