561 lines · python
1#!/usr/bin/env python32#3# ====- code-format-helper, runs code formatters from the ci or in a hook --*- python -*--==#4#5# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.6# See https://llvm.org/LICENSE.txt for license information.7# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception8#9# ==--------------------------------------------------------------------------------------==#10 11import argparse12import os13import re14import shlex15import subprocess16import sys17from typing import List, Optional18 19"""20This script is run by GitHub actions to ensure that the code in PR's conform to21the coding style of LLVM. It can also be installed as a pre-commit git hook to22check the coding style before submitting it. The canonical source of this script23is in the LLVM source tree under llvm/utils/git.24 25For C/C++ code it uses clang-format and for Python code it uses darker (which26in turn invokes black).27 28You can learn more about the LLVM coding style on llvm.org:29https://llvm.org/docs/CodingStandards.html30 31You can install this script as a git hook by symlinking it to the .git/hooks32directory:33 34ln -s $(pwd)/llvm/utils/git/code-format-helper.py .git/hooks/pre-commit35 36You can control the exact path to clang-format or darker with the following37environment variables: $CLANG_FORMAT_PATH and $DARKER_FORMAT_PATH.38"""39 40 41class FormatArgs:42 start_rev: str = None43 end_rev: str = None44 repo: str = None45 changed_files: List[str] = []46 token: str = None47 verbose: bool = True48 issue_number: int = 049 write_comment_to_file: bool = False50 51 def __init__(self, args: argparse.Namespace = None) -> None:52 if not args is None:53 self.start_rev = args.start_rev54 self.end_rev = args.end_rev55 self.repo = args.repo56 self.token = args.token57 self.changed_files = args.changed_files58 self.issue_number = args.issue_number59 self.write_comment_to_file = args.write_comment_to_file60 61 62class FormatHelper:63 COMMENT_TAG = "<!--LLVM CODE FORMAT COMMENT: {fmt}-->"64 name: str65 friendly_name: str66 comment: dict = None67 68 @property69 def comment_tag(self) -> str:70 return self.COMMENT_TAG.replace("fmt", self.name)71 72 @property73 def instructions(self) -> str:74 raise NotImplementedError()75 76 def has_tool(self) -> bool:77 raise NotImplementedError()78 79 def format_run(self, changed_files: List[str], args: FormatArgs) -> Optional[str]:80 raise NotImplementedError()81 82 def pr_comment_text_for_diff(self, diff: str) -> str:83 return f"""84:warning: {self.friendly_name}, {self.name} found issues in your code. :warning:85 86<details>87<summary>88You can test this locally with the following command:89</summary>90 91``````````bash92{self.instructions}93``````````94 95:warning:96The reproduction instructions above might return results for more than one PR97in a stack if you are using a stacked PR workflow. You can limit the results by98changing `origin/main` to the base branch/commit you want to compare against.99:warning:100 101</details>102 103<details>104<summary>105View the diff from {self.name} here.106</summary>107 108``````````diff109{diff}110``````````111 112</details>113"""114 115 # TODO: any type should be replaced with the correct github type, but it requires refactoring to116 # not require the github module to be installed everywhere.117 def find_comment(self, pr: any) -> any:118 for comment in pr.as_issue().get_comments():119 if self.comment_tag in comment.body:120 return comment121 return None122 123 def update_pr(self, comment_text: str, args: FormatArgs, create_new: bool) -> None:124 import github125 from github import IssueComment, PullRequest126 127 repo = github.Github(args.token).get_repo(args.repo)128 pr = repo.get_issue(args.issue_number).as_pull_request()129 130 comment_text = self.comment_tag + "\n\n" + comment_text131 132 existing_comment = self.find_comment(pr)133 134 if args.write_comment_to_file:135 if create_new or existing_comment:136 self.comment = {"body": comment_text}137 if existing_comment:138 self.comment["id"] = existing_comment.id139 return140 141 if existing_comment:142 existing_comment.edit(comment_text)143 elif create_new:144 pr.as_issue().create_comment(comment_text)145 146 def run(self, changed_files: List[str], args: FormatArgs) -> bool:147 changed_files = [arg for arg in changed_files if "third-party" not in arg]148 diff = self.format_run(changed_files, args)149 should_update_gh = args.token is not None and args.repo is not None150 151 if diff is None:152 if should_update_gh:153 comment_text = (154 ":white_check_mark: With the latest revision "155 f"this PR passed the {self.friendly_name}."156 )157 self.update_pr(comment_text, args, create_new=False)158 return True159 elif len(diff) > 0:160 if should_update_gh:161 comment_text = self.pr_comment_text_for_diff(diff)162 self.update_pr(comment_text, args, create_new=True)163 else:164 print(165 f"Warning: {self.friendly_name}, {self.name} detected "166 "some issues with your code formatting..."167 )168 return False169 else:170 # The formatter failed but didn't output a diff (e.g. some sort of171 # infrastructure failure).172 comment_text = (173 f":warning: The {self.friendly_name} failed without printing "174 "a diff. Check the logs for stderr output. :warning:"175 )176 if should_update_gh:177 self.update_pr(comment_text, args, create_new=False)178 return False179 180 181class ClangFormatHelper(FormatHelper):182 name = "clang-format"183 friendly_name = "C/C++ code formatter"184 185 def _construct_command(self, diff_expression: list[str] | None):186 cf_cmd = [self.clang_fmt_path, "--diff"]187 188 if diff_expression:189 cf_cmd.extend(diff_expression)190 191 # Gather the extension of all modified files and pass them explicitly to git-clang-format.192 # This prevents git-clang-format from applying its own filtering rules on top of ours.193 extensions = set()194 for file in self._cpp_files:195 _, ext = os.path.splitext(file)196 extensions.add(197 ext.strip(".")198 ) # Exclude periods since git-clang-format takes extensions without them199 cf_cmd.append("--extensions")200 cf_cmd.append(",".join(extensions))201 202 cf_cmd.append("--")203 cf_cmd += self._cpp_files204 return cf_cmd205 206 @property207 def instructions(self) -> str:208 return (209 " ".join(self._construct_command(["origin/main", "HEAD"]))210 + " --diff_from_common_commit"211 )212 213 def should_include_extensionless_file(self, path: str) -> bool:214 return path.startswith("libcxx/include")215 216 def filter_changed_files(self, changed_files: List[str]) -> List[str]:217 filtered_files = []218 for path in changed_files:219 _, ext = os.path.splitext(path)220 if ext in (221 ".cpp",222 ".c",223 ".h",224 ".hpp",225 ".hxx",226 ".cxx",227 ".inc",228 ".cppm",229 ".cl",230 ):231 filtered_files.append(path)232 elif ext == "" and self.should_include_extensionless_file(path):233 filtered_files.append(path)234 return filtered_files235 236 @property237 def clang_fmt_path(self) -> str:238 if "CLANG_FORMAT_PATH" in os.environ:239 return os.environ["CLANG_FORMAT_PATH"]240 return "git-clang-format"241 242 def has_tool(self) -> bool:243 cmd = [self.clang_fmt_path, "-h"]244 proc = None245 try:246 proc = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)247 except:248 return False249 return proc.returncode == 0250 251 def format_run(self, changed_files: List[str], args: FormatArgs) -> Optional[str]:252 self._cpp_files = self.filter_changed_files(changed_files)253 if not self._cpp_files:254 return None255 256 diff_expression = []257 if args.start_rev and args.end_rev:258 diff_expression.append(args.start_rev)259 diff_expression.append(args.end_rev)260 261 cf_cmd = self._construct_command(diff_expression)262 263 if args.verbose:264 print(f"Running: {' '.join(cf_cmd)}")265 proc = subprocess.run(cf_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)266 sys.stdout.write(proc.stderr.decode("utf-8"))267 268 if proc.returncode != 0:269 # formatting needed, or the command otherwise failed270 if args.verbose:271 print(f"error: {self.name} exited with code {proc.returncode}")272 # Print the diff in the log so that it is viewable there273 print(proc.stdout.decode("utf-8"))274 return proc.stdout.decode("utf-8")275 else:276 return None277 278 279class DarkerFormatHelper(FormatHelper):280 name = "darker"281 friendly_name = "Python code formatter"282 283 def _construct_command(self, diff_expression: str | None) -> str:284 darker_cmd = [285 self.darker_fmt_path,286 "--check",287 "--diff",288 ]289 if diff_expression:290 darker_cmd += ["-r", diff_expression]291 darker_cmd += self._py_files292 return darker_cmd293 294 @property295 def instructions(self) -> str:296 return " ".join(self._construct_command("origin/main...HEAD"))297 298 def filter_changed_files(self, changed_files: List[str]) -> List[str]:299 filtered_files = []300 for path in changed_files:301 name, ext = os.path.splitext(path)302 if ext == ".py":303 filtered_files.append(path)304 305 return filtered_files306 307 @property308 def darker_fmt_path(self) -> str:309 if "DARKER_FORMAT_PATH" in os.environ:310 return os.environ["DARKER_FORMAT_PATH"]311 return "darker"312 313 def has_tool(self) -> bool:314 cmd = [self.darker_fmt_path, "--version"]315 proc = None316 try:317 proc = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)318 except:319 return False320 return proc.returncode == 0321 322 def format_run(self, changed_files: List[str], args: FormatArgs) -> Optional[str]:323 py_files = self.filter_changed_files(changed_files)324 if not py_files:325 return None326 self._py_files = py_files327 diff_expression = None328 if args.start_rev and args.end_rev:329 diff_expression = f"{args.start_rev}...{args.end_rev}"330 darker_cmd = self._construct_command(diff_expression)331 if args.verbose:332 print(f"Running: {' '.join(darker_cmd)}")333 proc = subprocess.run(334 darker_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE335 )336 if args.verbose:337 sys.stdout.write(proc.stderr.decode("utf-8"))338 339 if proc.returncode != 0:340 # formatting needed, or the command otherwise failed341 if args.verbose:342 print(f"error: {self.name} exited with code {proc.returncode}")343 # Print the diff in the log so that it is viewable there344 print(proc.stdout.decode("utf-8"))345 return proc.stdout.decode("utf-8")346 else:347 sys.stdout.write(proc.stdout.decode("utf-8"))348 return None349 350 351class UndefGetFormatHelper(FormatHelper):352 name = "undef deprecator"353 friendly_name = "undef deprecator"354 355 @property356 def instructions(self) -> str:357 return " ".join(shlex.quote(c) for c in self.cmd)358 359 def filter_changed_files(self, changed_files: List[str]) -> List[str]:360 filtered_files = []361 for path in changed_files:362 _, ext = os.path.splitext(path)363 if ext in (".cpp", ".c", ".h", ".hpp", ".hxx", ".cxx", ".inc", ".cppm", ".ll"):364 filtered_files.append(path)365 return filtered_files366 367 def has_tool(self) -> bool:368 return True369 370 def pr_comment_text_for_diff(self, diff: str) -> str:371 return f"""372:warning: {self.name} found issues in your code. :warning:373 374<details>375<summary>376You can test this locally with the following command:377</summary>378 379``````````bash380{self.instructions}381``````````382 383</details>384 385{diff}386"""387 388 def format_run(self, changed_files: List[str], args: FormatArgs) -> Optional[str]:389 files = self.filter_changed_files(changed_files)390 if not files:391 return None392 393 # Use git to find files that have had a change in the number of undefs394 regex = "([^a-zA-Z0-9#_-]undef([^a-zA-Z0-9_-]|$)|UndefValue::get)"395 cmd = ["git", "diff", "-U0", "--pickaxe-regex", "-S", regex]396 397 if args.start_rev and args.end_rev:398 cmd.append(args.start_rev)399 cmd.append(args.end_rev)400 401 cmd += files402 self.cmd = cmd403 404 if args.verbose:405 print(f"Running: {self.instructions}")406 407 proc = subprocess.run(408 cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, encoding="utf-8"409 )410 sys.stdout.write(proc.stderr)411 stdout = proc.stdout412 413 if not stdout:414 return None415 416 files = []417 418 # Split the diff so we have one array entry per file.419 # Each file is prefixed like:420 # diff --git a/file b/file421 for file in re.split("^diff --git ", stdout, 0, re.MULTILINE):422 lines = file.splitlines()423 match = re.match("a/([^ ]+)", lines[0] if lines else "")424 filename = match[1] if match else ""425 if filename.endswith(".ll"):426 undef_regex = r"(?<!%)\bundef\b"427 else:428 undef_regex = r"UndefValue::get"429 # search for additions of undef430 if re.search(r"^[+].*" + undef_regex, file, re.MULTILINE):431 files.append(filename)432 433 if not files:434 return None435 436 files = "\n".join(" - " + f for f in files)437 report = f"""438The following files introduce new uses of undef:439{files}440 441[Undef](https://llvm.org/docs/LangRef.html#undefined-values) is now deprecated and should only be used in the rare cases where no replacement is possible. For example, a load of uninitialized memory yields `undef`. You should use `poison` values for placeholders instead.442 443In tests, avoid using `undef` and having tests that trigger undefined behavior. If you need an operand with some unimportant value, you can add a new argument to the function and use that instead.444 445For example, this is considered a bad practice:446```llvm447define void @fn() {{448 ...449 br i1 undef, ...450}}451```452 453Please use the following instead:454```llvm455define void @fn(i1 %cond) {{456 ...457 br i1 %cond, ...458}}459```460 461Please refer to the [Undefined Behavior Manual](https://llvm.org/docs/UndefinedBehavior.html) for more information.462"""463 if args.verbose:464 print(f"error: {self.name} failed")465 print(report)466 return report467 468 469ALL_FORMATTERS = (DarkerFormatHelper(), ClangFormatHelper(), UndefGetFormatHelper())470 471 472def hook_main():473 # fill out args474 args = FormatArgs()475 args.verbose = os.getenv("FORMAT_HOOK_VERBOSE", False)476 477 # find the changed files478 cmd = ["git", "diff", "--cached", "--name-only", "--diff-filter=d"]479 proc = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)480 output = proc.stdout.decode("utf-8")481 for line in output.splitlines():482 args.changed_files.append(line)483 484 failed_fmts = []485 for fmt in ALL_FORMATTERS:486 if fmt.has_tool():487 if not fmt.run(args.changed_files, args):488 failed_fmts.append(fmt.name)489 else:490 print(f"Couldn't find {fmt.name}, can't check " + fmt.friendly_name.lower())491 492 if len(failed_fmts) > 0:493 print(494 "Pre-commit format hook failed, rerun with FORMAT_HOOK_VERBOSE=1 environment for verbose output"495 )496 sys.exit(1)497 498 sys.exit(0)499 500 501if __name__ == "__main__":502 script_path = os.path.abspath(__file__)503 if ".git/hooks" in script_path:504 hook_main()505 sys.exit(0)506 507 parser = argparse.ArgumentParser()508 parser.add_argument(509 "--token", type=str, required=True, help="GitHub authentication token"510 )511 parser.add_argument(512 "--repo",513 type=str,514 default=os.getenv("GITHUB_REPOSITORY", "llvm/llvm-project"),515 help="The GitHub repository that we are working with in the form of <owner>/<repo> (e.g. llvm/llvm-project)",516 )517 parser.add_argument("--issue-number", type=int, required=True)518 parser.add_argument(519 "--start-rev",520 type=str,521 required=True,522 help="Compute changes from this revision.",523 )524 parser.add_argument(525 "--end-rev", type=str, required=True, help="Compute changes to this revision"526 )527 parser.add_argument(528 "--changed-files",529 type=str,530 help="Comma separated list of files that has been changed",531 )532 parser.add_argument(533 "--write-comment-to-file",534 action="store_true",535 help="Don't post comments on the PR, instead write the comments and metadata a file called 'comment'",536 )537 538 args = FormatArgs(parser.parse_args())539 540 changed_files = []541 if args.changed_files:542 changed_files = args.changed_files.split(",")543 544 failed_formatters = []545 comments = []546 for fmt in ALL_FORMATTERS:547 if not fmt.run(changed_files, args):548 failed_formatters.append(fmt.name)549 if fmt.comment:550 comments.append(fmt.comment)551 552 if len(comments):553 with open("comments", "w") as f:554 import json555 556 json.dump(comments, f)557 558 if len(failed_formatters) > 0:559 print(f"error: some formatters failed: {' '.join(failed_formatters)}")560 sys.exit(1)561