433 lines · python
1#!/usr/bin/env python32#3# ===-----------------------------------------------------------------------===#4#5# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.6# See https://llvm.org/LICENSE.txt for license information.7# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception8#9# ===-----------------------------------------------------------------------===#10 11r"""12ClangTidy Diff Checker13======================14 15This script reads input from a unified diff, runs clang-tidy on all changed16files and outputs clang-tidy warnings in changed lines only. This is useful to17detect clang-tidy regressions in the lines touched by a specific patch.18Example usage for git/svn users:19 20 git diff -U0 HEAD^ | clang-tidy-diff.py -p121 svn diff --diff-cmd=diff -x-U0 | \22 clang-tidy-diff.py -fix -checks=-*,modernize-use-override23 24"""25 26import argparse27import glob28import json29import multiprocessing30import os31import queue32import re33import shutil34import subprocess35import sys36import tempfile37import threading38import traceback39from pathlib import Path40 41try:42 import yaml43except ImportError:44 yaml = None45 46 47def run_tidy(task_queue, lock, timeout, failed_files):48 watchdog = None49 while True:50 command = task_queue.get()51 try:52 proc = subprocess.Popen(53 command, stdout=subprocess.PIPE, stderr=subprocess.PIPE54 )55 56 if timeout is not None:57 watchdog = threading.Timer(timeout, proc.kill)58 watchdog.start()59 60 stdout, stderr = proc.communicate()61 if proc.returncode != 0:62 if proc.returncode < 0:63 msg = "Terminated by signal %d : %s\n" % (64 -proc.returncode,65 " ".join(command),66 )67 stderr += msg.encode("utf-8")68 failed_files.append(command)69 70 with lock:71 sys.stdout.write(stdout.decode("utf-8") + "\n")72 sys.stdout.flush()73 if stderr:74 sys.stderr.write(stderr.decode("utf-8") + "\n")75 sys.stderr.flush()76 except Exception as e:77 with lock:78 sys.stderr.write("Failed: " + str(e) + ": ".join(command) + "\n")79 finally:80 with lock:81 if not (timeout is None or watchdog is None):82 if not watchdog.is_alive():83 sys.stderr.write(84 "Terminated by timeout: " + " ".join(command) + "\n"85 )86 watchdog.cancel()87 task_queue.task_done()88 89 90def start_workers(max_tasks, tidy_caller, arguments):91 for _ in range(max_tasks):92 t = threading.Thread(target=tidy_caller, args=arguments)93 t.daemon = True94 t.start()95 96 97def merge_replacement_files(tmpdir, mergefile):98 """Merge all replacement files in a directory into a single file"""99 # The fixes suggested by clang-tidy >= 4.0.0 are given under100 # the top level key 'Diagnostics' in the output yaml files101 mergekey = "Diagnostics"102 merged = []103 for replacefile in glob.iglob(os.path.join(tmpdir, "*.yaml")):104 content = yaml.safe_load(open(replacefile, "r"))105 if not content:106 continue # Skip empty files.107 merged.extend(content.get(mergekey, []))108 109 if merged:110 # MainSourceFile: The key is required by the definition inside111 # include/clang/Tooling/ReplacementsYaml.h, but the value112 # is actually never used inside clang-apply-replacements,113 # so we set it to '' here.114 output = {"MainSourceFile": "", mergekey: merged}115 with open(mergefile, "w") as out:116 yaml.safe_dump(output, out)117 else:118 # Empty the file:119 open(mergefile, "w").close()120 121 122def get_compiling_files(args):123 """Read a compile_commands.json database and return a set of file paths"""124 current_dir = Path.cwd()125 compile_commands_json = (126 (current_dir / args.build_path) if args.build_path else current_dir127 )128 compile_commands_json = compile_commands_json / "compile_commands.json"129 files = set()130 with open(compile_commands_json) as db_file:131 db_json = json.load(db_file)132 for entry in db_json:133 if "file" not in entry:134 continue135 files.add(Path(entry["file"]))136 return files137 138 139def main():140 parser = argparse.ArgumentParser(141 description="Run clang-tidy against changed files, and "142 "output diagnostics only for modified "143 "lines."144 )145 parser.add_argument(146 "-clang-tidy-binary",147 metavar="PATH",148 default="clang-tidy",149 help="path to clang-tidy binary",150 )151 parser.add_argument(152 "-p",153 metavar="NUM",154 default=0,155 help="strip the smallest prefix containing P slashes",156 )157 parser.add_argument(158 "-regex",159 metavar="PATTERN",160 default=None,161 help="custom pattern selecting file paths to check "162 "(case sensitive, overrides -iregex)",163 )164 parser.add_argument(165 "-iregex",166 metavar="PATTERN",167 default=r".*\.(cpp|cc|c\+\+|cxx|c|cl|h|hpp|m|mm|inc)",168 help="custom pattern selecting file paths to check "169 "(case insensitive, overridden by -regex)",170 )171 parser.add_argument(172 "-j",173 type=int,174 default=0,175 help="number of tidy instances to be run in parallel.",176 )177 parser.add_argument(178 "-timeout", type=int, default=None, help="timeout per each file in seconds."179 )180 parser.add_argument(181 "-fix", action="store_true", default=False, help="apply suggested fixes"182 )183 parser.add_argument(184 "-checks",185 help="checks filter, when not specified, use clang-tidy " "default",186 default="",187 )188 parser.add_argument(189 "-config-file",190 dest="config_file",191 help="Specify the path of .clang-tidy or custom config file",192 default="",193 )194 parser.add_argument("-use-color", action="store_true", help="Use colors in output")195 parser.add_argument(196 "-path", dest="build_path", help="Path used to read a compile command database."197 )198 if yaml:199 parser.add_argument(200 "-export-fixes",201 metavar="FILE_OR_DIRECTORY",202 dest="export_fixes",203 help="A directory or a yaml file to store suggested fixes in, "204 "which can be applied with clang-apply-replacements. If the "205 "parameter is a directory, the fixes of each compilation unit are "206 "stored in individual yaml files in the directory.",207 )208 else:209 parser.add_argument(210 "-export-fixes",211 metavar="DIRECTORY",212 dest="export_fixes",213 help="A directory to store suggested fixes in, which can be applied "214 "with clang-apply-replacements. The fixes of each compilation unit are "215 "stored in individual yaml files in the directory.",216 )217 parser.add_argument(218 "-extra-arg",219 dest="extra_arg",220 action="append",221 default=[],222 help="Additional argument to append to the compiler " "command line.",223 )224 parser.add_argument(225 "-extra-arg-before",226 dest="extra_arg_before",227 action="append",228 default=[],229 help="Additional argument to prepend to the compiler " "command line.",230 )231 parser.add_argument(232 "-quiet",233 action="store_true",234 default=False,235 help="Run clang-tidy in quiet mode",236 )237 parser.add_argument(238 "-load",239 dest="plugins",240 action="append",241 default=[],242 help="Load the specified plugin in clang-tidy.",243 )244 parser.add_argument(245 "-allow-no-checks",246 action="store_true",247 help="Allow empty enabled checks.",248 )249 parser.add_argument(250 "-only-check-in-db",251 dest="skip_non_compiling",252 default=False,253 action="store_true",254 help="Only check files in the compilation database",255 )256 parser.add_argument(257 "-warnings-as-errors",258 help="Upgrades clang-tidy warnings to errors. Same format as '-checks'.",259 default="",260 )261 parser.add_argument(262 "-hide-progress",263 action="store_true",264 help="Hide progress",265 )266 267 clang_tidy_args = []268 argv = sys.argv[1:]269 if "--" in argv:270 clang_tidy_args.extend(argv[argv.index("--") :])271 argv = argv[: argv.index("--")]272 273 args = parser.parse_args(argv)274 275 compiling_files = get_compiling_files(args) if args.skip_non_compiling else None276 277 # Extract changed lines for each file.278 filename = None279 lines_by_file = {}280 for line in sys.stdin:281 match = re.search(r'^\+\+\+\ "?(.*?/){%s}([^ \t\n"]*)' % args.p, line)282 if match:283 filename = match.group(2)284 if filename is None:285 continue286 287 if args.regex is not None:288 if not re.match("^%s$" % args.regex, filename):289 continue290 else:291 if not re.match("^%s$" % args.iregex, filename, re.IGNORECASE):292 continue293 294 # Skip any files not in the compiling list295 if (296 compiling_files is not None297 and (Path.cwd() / filename) not in compiling_files298 ):299 continue300 301 match = re.search(r"^@@.*\+(\d+)(,(\d+))?", line)302 if match:303 start_line = int(match.group(1))304 line_count = 1305 if match.group(3):306 line_count = int(match.group(3))307 if line_count == 0:308 continue309 end_line = start_line + line_count - 1310 lines_by_file.setdefault(filename, []).append([start_line, end_line])311 312 if not any(lines_by_file):313 print("No relevant changes found.")314 sys.exit(0)315 316 max_task_count = args.j317 if max_task_count == 0:318 max_task_count = multiprocessing.cpu_count()319 max_task_count = min(len(lines_by_file), max_task_count)320 if not args.hide_progress:321 print(f"Running clang-tidy in {max_task_count} threads...")322 323 combine_fixes = False324 export_fixes_dir = None325 delete_fixes_dir = False326 if args.export_fixes is not None:327 # if a directory is given, create it if it does not exist328 if args.export_fixes.endswith(os.path.sep) and not os.path.isdir(329 args.export_fixes330 ):331 os.makedirs(args.export_fixes)332 333 if not os.path.isdir(args.export_fixes):334 if not yaml:335 raise RuntimeError(336 "Cannot combine fixes in one yaml file. Either install PyYAML or specify an output directory."337 )338 339 combine_fixes = True340 341 if os.path.isdir(args.export_fixes):342 export_fixes_dir = args.export_fixes343 344 if combine_fixes:345 export_fixes_dir = tempfile.mkdtemp()346 delete_fixes_dir = True347 348 # Tasks for clang-tidy.349 task_queue = queue.Queue(max_task_count)350 # A lock for console output.351 lock = threading.Lock()352 353 # List of files with a non-zero return code.354 failed_files = []355 356 # Run a pool of clang-tidy workers.357 start_workers(358 max_task_count, run_tidy, (task_queue, lock, args.timeout, failed_files)359 )360 361 # Form the common args list.362 common_clang_tidy_args = []363 if args.fix:364 common_clang_tidy_args.append("-fix")365 if args.checks != "":366 common_clang_tidy_args.append("-checks=" + args.checks)367 if args.config_file != "":368 common_clang_tidy_args.append("-config-file=" + args.config_file)369 if args.quiet:370 common_clang_tidy_args.append("-quiet")371 if args.build_path is not None:372 common_clang_tidy_args.append("-p=%s" % args.build_path)373 if args.use_color:374 common_clang_tidy_args.append("--use-color")375 if args.allow_no_checks:376 common_clang_tidy_args.append("--allow-no-checks")377 for arg in args.extra_arg:378 common_clang_tidy_args.append("-extra-arg=%s" % arg)379 for arg in args.extra_arg_before:380 common_clang_tidy_args.append("-extra-arg-before=%s" % arg)381 for plugin in args.plugins:382 common_clang_tidy_args.append("-load=%s" % plugin)383 if args.warnings_as_errors:384 common_clang_tidy_args.append("-warnings-as-errors=" + args.warnings_as_errors)385 386 for name in lines_by_file:387 line_filter_json = json.dumps(388 [{"name": name, "lines": lines_by_file[name]}], separators=(",", ":")389 )390 391 # Run clang-tidy on files containing changes.392 command = [args.clang_tidy_binary]393 command.append("-line-filter=" + line_filter_json)394 if args.export_fixes is not None:395 # Get a temporary file. We immediately close the handle so clang-tidy can396 # overwrite it.397 (handle, tmp_name) = tempfile.mkstemp(suffix=".yaml", dir=export_fixes_dir)398 os.close(handle)399 command.append("-export-fixes=" + tmp_name)400 command.extend(common_clang_tidy_args)401 command.append(name)402 command.extend(clang_tidy_args)403 404 task_queue.put(command)405 406 # Application return code407 return_code = 0408 409 # Wait for all threads to be done.410 task_queue.join()411 # Application return code412 return_code = 0413 if failed_files:414 return_code = 1415 416 if combine_fixes:417 if not args.hide_progress:418 print(f"Writing fixes to {args.export_fixes} ...")419 try:420 merge_replacement_files(export_fixes_dir, args.export_fixes)421 except:422 sys.stderr.write("Error exporting fixes.\n")423 traceback.print_exc()424 return_code = 1425 426 if delete_fixes_dir:427 shutil.rmtree(export_fixes_dir)428 sys.exit(return_code)429 430 431if __name__ == "__main__":432 main()433