351 lines · python
1# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.2# See https://llvm.org/LICENSE.txt for license information.3# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception4"""Computes the list of projects that need to be tested from a diff.5 6Does some things, spits out a list of projects.7"""8 9from collections.abc import Set10import pathlib11import platform12import sys13 14# This mapping lists out the dependencies for each project. These should be15# direct dependencies. The code will handle transitive dependencies. Some16# projects might have optional dependencies depending upon how they are built.17# The dependencies listed here should be the dependencies required for the18# configuration built/tested in the premerge CI.19PROJECT_DEPENDENCIES = {20 "llvm": set(),21 "clang": {"llvm"},22 "CIR": {"clang", "mlir"},23 "bolt": {"clang", "lld", "llvm"},24 "clang-tools-extra": {"clang", "llvm"},25 "compiler-rt": {"clang", "lld"},26 "libc": {"clang", "lld"},27 "openmp": {"clang", "lld"},28 "flang": {"llvm", "clang"},29 "flang-rt": {"flang"},30 "lldb": {"llvm", "clang"},31 "libclc": {"llvm", "clang"},32 "lld": {"llvm"},33 "mlir": {"llvm"},34 "polly": {"llvm"},35}36 37# This mapping describes the additional projects that should be tested when a38# specific project is touched. We enumerate them specifically rather than39# just invert the dependencies list to give more control over what exactly is40# tested.41DEPENDENTS_TO_TEST = {42 "llvm": {43 "bolt",44 "clang",45 "clang-tools-extra",46 "lld",47 "lldb",48 "mlir",49 "polly",50 "flang",51 },52 "lld": {"bolt", "cross-project-tests"},53 "clang": {"clang-tools-extra", "cross-project-tests", "lldb"},54 "mlir": {"flang"},55 # Test everything if ci scripts are changed.56 ".ci": {57 "llvm",58 "clang",59 "CIR",60 "lld",61 "lldb",62 "bolt",63 "clang-tools-extra",64 "mlir",65 "polly",66 "flang",67 "libclc",68 "openmp",69 },70}71 72# This mapping describes runtimes that should be enabled for a specific project,73# but not necessarily run for testing. The only case of this currently is lldb74# which needs some runtimes enabled for tests.75DEPENDENT_RUNTIMES_TO_BUILD = {"lldb": {"libcxx", "libcxxabi", "libunwind"}}76 77# This mapping describes runtimes that should be tested when the key project is78# touched.79DEPENDENT_RUNTIMES_TO_TEST = {80 "clang": {"compiler-rt"},81 "clang-tools-extra": {"libc"},82 "libc": {"libc"},83 "compiler-rt": {"compiler-rt"},84 "flang": {"flang-rt"},85 "flang-rt": {"flang-rt"},86 ".ci": {"compiler-rt", "libc", "flang-rt"},87}88DEPENDENT_RUNTIMES_TO_TEST_NEEDS_RECONFIG = {89 "llvm": {"libcxx", "libcxxabi", "libunwind"},90 "clang": {"libcxx", "libcxxabi", "libunwind"},91 ".ci": {"libcxx", "libcxxabi", "libunwind"},92}93 94EXCLUDE_LINUX = {95 "cross-project-tests", # TODO(issues/132796): Tests are failing.96 "openmp", # https://github.com/google/llvm-premerge-checks/issues/41097}98 99EXCLUDE_WINDOWS = {100 "cross-project-tests", # TODO(issues/132797): Tests are failing.101 "openmp", # TODO(issues/132799): Does not detect perl installation.102 "libc", # No Windows Support.103 "lldb", # TODO(issues/132800): Needs environment setup.104 "bolt", # No Windows Support.105 "libcxx",106 "libcxxabi",107 "libunwind",108 "flang-rt",109}110 111# These are projects that we should test if the project itself is changed but112# where testing is not yet stable enough for it to be enabled on changes to113# dependencies.114EXCLUDE_DEPENDENTS_WINDOWS = {115 "flang", # TODO(issues/132803): Flang is not stable.116}117 118EXCLUDE_MAC = {119 "bolt",120 "compiler-rt",121 "cross-project-tests",122 "flang",123 "libc",124 "lldb",125 "openmp",126 "polly",127 "libcxx",128 "libcxxabi",129 "libunwind",130}131 132PROJECT_CHECK_TARGETS = {133 "clang-tools-extra": "check-clang-tools",134 "compiler-rt": "check-compiler-rt",135 "cross-project-tests": "check-cross-project",136 "libcxx": "check-cxx",137 "libcxxabi": "check-cxxabi",138 "libunwind": "check-unwind",139 "lldb": "check-lldb",140 "llvm": "check-llvm",141 "clang": "check-clang",142 "CIR": "check-clang-cir",143 "bolt": "check-bolt",144 "lld": "check-lld",145 "flang": "check-flang",146 "flang-rt": "check-flang-rt",147 "libc": "check-libc",148 "lld": "check-lld",149 "lldb": "check-lldb",150 "mlir": "check-mlir",151 "openmp": "check-openmp",152 "polly": "check-polly",153 "lit": "check-lit",154}155 156RUNTIMES = {"libcxx", "libcxxabi", "libunwind", "compiler-rt", "libc", "flang-rt"}157 158# Meta projects are projects that need explicit handling but do not reside159# in their own top level folder. To add a meta project, the start of the path160# for the metaproject should be mapped to the name of the project below.161# Multiple paths can map to the same metaproject.162META_PROJECTS = {163 ("clang", "lib", "CIR"): "CIR",164 ("clang", "test", "CIR"): "CIR",165 ("clang", "include", "clang", "CIR"): "CIR",166 ("*", "docs"): "docs",167 ("llvm", "utils", "gn"): "gn",168 (".github", "workflows", "premerge.yaml"): ".ci",169 ("third-party",): ".ci",170 ("llvm", "utils", "lit"): "lit",171}172 173# Projects that should run tests but cannot be explicitly built.174SKIP_BUILD_PROJECTS = ["CIR", "lit"]175 176# Projects that should not run any tests. These need to be metaprojects.177SKIP_PROJECTS = ["docs", "gn"]178 179 180def _add_dependencies(projects: Set[str], runtimes: Set[str]) -> Set[str]:181 projects_with_dependents = set(projects)182 current_projects_count = 0183 while current_projects_count != len(projects_with_dependents):184 current_projects_count = len(projects_with_dependents)185 for project in list(projects_with_dependents):186 if project in PROJECT_DEPENDENCIES:187 projects_with_dependents.update(PROJECT_DEPENDENCIES[project])188 for runtime in runtimes:189 if runtime in PROJECT_DEPENDENCIES:190 projects_with_dependents.update(PROJECT_DEPENDENCIES[runtime])191 return projects_with_dependents192 193 194def _exclude_projects(current_projects: Set[str], platform: str) -> Set[str]:195 if platform == "Linux":196 to_exclude = EXCLUDE_LINUX197 elif platform == "Windows":198 to_exclude = EXCLUDE_WINDOWS199 elif platform == "Darwin":200 to_exclude = EXCLUDE_MAC201 else:202 raise ValueError(f"Unexpected platform: {platform}")203 return current_projects.difference(to_exclude)204 205 206def _compute_projects_to_test(modified_projects: Set[str], platform: str) -> Set[str]:207 projects_to_test = set()208 for modified_project in modified_projects:209 if modified_project in RUNTIMES:210 continue211 # Skip all projects where we cannot run tests.212 if modified_project in PROJECT_CHECK_TARGETS:213 projects_to_test.add(modified_project)214 if modified_project not in DEPENDENTS_TO_TEST:215 continue216 for dependent_project in DEPENDENTS_TO_TEST[modified_project]:217 if (218 platform == "Windows"219 and dependent_project in EXCLUDE_DEPENDENTS_WINDOWS220 ):221 continue222 projects_to_test.add(dependent_project)223 projects_to_test = _exclude_projects(projects_to_test, platform)224 return projects_to_test225 226 227def _compute_projects_to_build(228 projects_to_test: Set[str], runtimes: Set[str]229) -> Set[str]:230 return _add_dependencies(projects_to_test, runtimes)231 232 233def _compute_project_check_targets(projects_to_test: Set[str]) -> Set[str]:234 check_targets = set()235 for project_to_test in projects_to_test:236 if project_to_test in PROJECT_CHECK_TARGETS:237 check_targets.add(PROJECT_CHECK_TARGETS[project_to_test])238 return check_targets239 240 241def _compute_runtimes_to_test(modified_projects: Set[str], platform: str) -> Set[str]:242 runtimes_to_test = set()243 for modified_project in modified_projects:244 if modified_project in DEPENDENT_RUNTIMES_TO_TEST:245 runtimes_to_test.update(DEPENDENT_RUNTIMES_TO_TEST[modified_project])246 return _exclude_projects(runtimes_to_test, platform)247 248 249def _compute_runtimes_to_test_needs_reconfig(250 modified_projects: Set[str], platform: str251) -> Set[str]:252 runtimes_to_test = set()253 for modified_project in modified_projects:254 if modified_project in DEPENDENT_RUNTIMES_TO_TEST_NEEDS_RECONFIG:255 runtimes_to_test.update(256 DEPENDENT_RUNTIMES_TO_TEST_NEEDS_RECONFIG[modified_project]257 )258 return _exclude_projects(runtimes_to_test, platform)259 260 261def _compute_runtimes_to_build(262 runtimes_to_test: Set[str], modified_projects: Set[str], platform: str263) -> Set[str]:264 runtimes_to_build = set(runtimes_to_test)265 for modified_project in modified_projects:266 if modified_project in DEPENDENT_RUNTIMES_TO_BUILD:267 runtimes_to_build.update(DEPENDENT_RUNTIMES_TO_BUILD[modified_project])268 return _exclude_projects(runtimes_to_build, platform)269 270 271def _path_matches(matcher: tuple[str], file_path: tuple[str]) -> bool:272 if len(file_path) < len(matcher):273 return False274 for match_part, file_part in zip(matcher, file_path):275 if match_part == "*" or file_part == "*":276 continue277 if match_part != file_part:278 return False279 return True280 281 282def _get_modified_projects_for_file(modified_file: str) -> Set[str]:283 modified_projects = set()284 path_parts = pathlib.Path(modified_file).parts285 for meta_project_files in META_PROJECTS.keys():286 if _path_matches(meta_project_files, path_parts):287 meta_project = META_PROJECTS[meta_project_files]288 if meta_project in SKIP_PROJECTS:289 return set()290 modified_projects.add(meta_project)291 modified_projects.add(pathlib.Path(modified_file).parts[0])292 return modified_projects293 294 295def _get_modified_projects(modified_files: list[str]) -> Set[str]:296 modified_projects = set()297 for modified_file in modified_files:298 modified_projects.update(_get_modified_projects_for_file(modified_file))299 return modified_projects300 301 302def get_env_variables(modified_files: list[str], platform: str) -> Set[str]:303 modified_projects = _get_modified_projects(modified_files)304 projects_to_test = _compute_projects_to_test(modified_projects, platform)305 runtimes_to_test = _compute_runtimes_to_test(modified_projects, platform)306 runtimes_to_test_needs_reconfig = _compute_runtimes_to_test_needs_reconfig(307 modified_projects, platform308 )309 runtimes_to_build = _compute_runtimes_to_build(310 runtimes_to_test | runtimes_to_test_needs_reconfig, modified_projects, platform311 )312 projects_to_build = _compute_projects_to_build(projects_to_test, runtimes_to_build)313 projects_check_targets = _compute_project_check_targets(projects_to_test)314 runtimes_check_targets = _compute_project_check_targets(runtimes_to_test)315 runtimes_check_targets_needs_reconfig = _compute_project_check_targets(316 runtimes_to_test_needs_reconfig317 )318 319 # CIR is used as a pseudo-project in this script. It is built as part of the320 # clang build, but it requires an explicit option to enable. We set that321 # option here, and remove it from the projects_to_build list.322 enable_cir = "ON" if "CIR" in projects_to_build else "OFF"323 # Remove any metaprojects from the list of projects to build.324 for project in SKIP_BUILD_PROJECTS:325 projects_to_build.discard(project)326 327 # We use a semicolon to separate the projects/runtimes as they get passed328 # to the CMake invocation and thus we need to use the CMake list separator329 # (;). We use spaces to separate the check targets as they end up getting330 # passed to ninja.331 return {332 "projects_to_build": ";".join(sorted(projects_to_build)),333 "project_check_targets": " ".join(sorted(projects_check_targets)),334 "runtimes_to_build": ";".join(sorted(runtimes_to_build)),335 "runtimes_check_targets": " ".join(sorted(runtimes_check_targets)),336 "runtimes_check_targets_needs_reconfig": " ".join(337 sorted(runtimes_check_targets_needs_reconfig)338 ),339 "enable_cir": enable_cir,340 }341 342 343if __name__ == "__main__":344 current_platform = platform.system()345 if len(sys.argv) == 2:346 current_platform = sys.argv[1]347 changed_files = [line.strip() for line in sys.stdin.readlines()]348 env_variables = get_env_variables(changed_files, current_platform)349 for env_variable in env_variables:350 print(f"{env_variable}='{env_variables[env_variable]}'")351