534 lines · python
1#!/usr/bin/env python32# -*- coding: UTF-8 -*-3 4# Polly/LLVM update_check.py5# Update lit FileCheck files by replacing the 'CHECK:' lines by the actual output of the 'RUN:' command.6 7import argparse8import os9import subprocess10import shlex11import re12 13 14polly_src_dir = """@POLLY_SOURCE_DIR@"""15polly_lib_dir = """@POLLY_LIB_DIR@"""16shlibext = """@LLVM_SHLIBEXT@"""17llvm_tools_dir = """@LLVM_TOOLS_DIR@"""18llvm_polly_link_into_tools = not """@LLVM_POLLY_LINK_INTO_TOOLS@""".lower() in {19 "",20 "0",21 "n",22 "no",23 "off",24 "false",25 "notfound",26 "llvm_polly_link_into_tools-notfound",27}28 29runre = re.compile(r"\s*\;\s*RUN\s*\:(?P<tool>.*)")30filecheckre = re.compile(r"\s*(?P<tool>.*)\|\s*(?P<filecheck>FileCheck\s[^|]*)")31emptyline = re.compile(r"\s*(\;\s*)?")32commentline = re.compile(r"\s*(\;.*)?")33 34 35def ltrim_emptylines(lines, meta=None):36 while len(lines) and emptyline.fullmatch(lines[0]):37 del lines[0]38 if meta is not None:39 del meta[0]40 41 42def rtrim_emptylines(lines):43 while len(lines) and emptyline.fullmatch(lines[-1]):44 del lines[-1]45 46 47def trim_emptylines(lines):48 ltrim_emptylines(lines)49 rtrim_emptylines(lines)50 51 52def complete_exename(path, filename):53 complpath = os.path.join(path, filename)54 if os.path.isfile(complpath):55 return complpath56 elif os.path.isfile(complpath + ".exe"):57 return complpath + ".exe"58 return filename59 60 61def indention(line):62 for i, c in enumerate(line):63 if c != " " and c != "\t":64 return i65 return None66 67 68def common_indent(lines):69 indentions = (indention(line) for line in lines)70 indentions = (indent for indent in indentions if indent is not None)71 return min(indentions, default=0)72 73 74funcre = re.compile(r"^ Function: \S*$")75regionre = re.compile(r"^ Region: \S*$")76depthre = re.compile(r"^ Max Loop Depth: .*")77paramre = re.compile(r" [0-9a-z-A-Z_]+\: .*")78 79 80def classyfier1(lines):81 i = iter(lines)82 line = i.__next__()83 while True:84 if line.startswith(85 "Printing analysis 'Polly - Calculate dependences' for region: "86 ):87 yield {"PrintingDependenceInfo"}88 elif line.startswith("remark: "):89 yield {"Remark"}90 elif funcre.fullmatch(line):91 yield {"Function"}92 elif regionre.fullmatch(line):93 yield {"Region"}94 elif depthre.fullmatch(line):95 yield {"MaxLoopDepth"}96 elif line == " Invariant Accesses: {":97 while True:98 yield {"InvariantAccesses"}99 if line == " }":100 break101 line = i.__next__()102 elif line == " Context:":103 yield {"Context"}104 line = i.__next__()105 yield {"Context"}106 elif line == " Assumed Context:":107 yield {"AssumedContext"}108 line = i.__next__()109 yield {"AssumedContext"}110 elif line == " Invalid Context:":111 yield {"InvalidContext"}112 line = i.__next__()113 yield {"InvalidContext"}114 elif line == " Boundary Context:":115 yield {"BoundaryContext"}116 line = i.__next__()117 yield {"BoundaryContext"}118 line = i.__next__()119 while paramre.fullmatch(line):120 yield {"Param"}121 line = i.__next__()122 continue123 elif line == " Arrays {":124 while True:125 yield {"Arrays"}126 if line == " }":127 break128 line = i.__next__()129 elif line == " Arrays (Bounds as pw_affs) {":130 while True:131 yield {"PwAffArrays"}132 if line == " }":133 break134 line = i.__next__()135 elif line.startswith(" Alias Groups ("):136 while True:137 yield {"AliasGroups"}138 line = i.__next__()139 if not line.startswith(" "):140 break141 continue142 elif line == " Statements {":143 while True:144 yield {"Statements"}145 if line == " }":146 break147 line = i.__next__()148 elif line == " RAW dependences:":149 yield {"RAWDep", "BasicDep", "Dep", "DepInfo"}150 line = i.__next__()151 while line.startswith(" "):152 yield {"RAWDep", "BasicDep", "Dep", "DepInfo"}153 line = i.__next__()154 continue155 elif line == " WAR dependences:":156 yield {"WARDep", "BasicDep", "Dep", "DepInfo"}157 line = i.__next__()158 while line.startswith(" "):159 yield {"WARDep", "BasicDep", "Dep", "DepInfo"}160 line = i.__next__()161 continue162 elif line == " WAW dependences:":163 yield {"WAWDep", "BasicDep", "Dep", "DepInfo"}164 line = i.__next__()165 while line.startswith(" "):166 yield {"WAWDep", "BasicDep", "Dep", "DepInfo"}167 line = i.__next__()168 continue169 elif line == " Reduction dependences:":170 yield {"RedDep", "Dep", "DepInfo"}171 line = i.__next__()172 while line.startswith(" "):173 yield {"RedDep", "Dep", "DepInfo"}174 line = i.__next__()175 continue176 elif line == " Transitive closure of reduction dependences:":177 yield {"TransitiveClosureDep", "DepInfo"}178 line = i.__next__()179 while line.startswith(" "):180 yield {"TransitiveClosureDep", "DepInfo"}181 line = i.__next__()182 continue183 elif line.startswith("New access function '"):184 yield {"NewAccessFunction"}185 elif line == "Schedule before flattening {":186 while True:187 yield {"ScheduleBeforeFlattening"}188 if line == "}":189 break190 line = i.__next__()191 elif line == "Schedule after flattening {":192 while True:193 yield {"ScheduleAfterFlattening"}194 if line == "}":195 break196 line = i.__next__()197 else:198 yield set()199 line = i.__next__()200 201 202def classyfier2(lines):203 i = iter(lines)204 line = i.__next__()205 while True:206 if funcre.fullmatch(line):207 while line.startswith(" "):208 yield {"FunctionDetail"}209 line = i.__next__()210 continue211 elif line.startswith(212 "Printing analysis 'Polly - Generate an AST from the SCoP (isl)' for region: "213 ):214 yield {"PrintingIslAst"}215 line = i.__next__()216 while not line.startswith("Printing analysis"):217 yield {"AstDetail"}218 line = i.__next__()219 continue220 else:221 yield set()222 line = i.__next__()223 224 225replrepl = {226 "{{": "{{[{][{]}}",227 "}}": "{{[}][}]}}",228 "[[": r"{{\[\[}}",229 "]]": r"{{\]\]}}",230}231replre = re.compile("|".join(re.escape(k) for k in replrepl.keys()))232 233 234def main():235 parser = argparse.ArgumentParser(description="Update CHECK lines")236 parser.add_argument(237 "testfile", help="File to update (absolute or relative to --testdir)"238 )239 parser.add_argument(240 "--check-style",241 choices=["CHECK", "CHECK-NEXT"],242 default="CHECK-NEXT",243 help="What kind of checks lines to generate",244 )245 parser.add_argument(246 "--check-position",247 choices=["end", "before-content", "autodetect"],248 default="autodetect",249 help="Where to add the CHECK lines into the file; 'autodetect' searches for the first 'CHECK' line ind inserts it there",250 )251 parser.add_argument(252 "--check-include",253 action="append",254 default=[],255 help="What parts of the output lines to check; use syntax 'CHECK=include' to apply to one CHECK-prefix only (by default, everything)",256 )257 parser.add_argument(258 "--check-label-include",259 action="append",260 default=[],261 help="Use CHECK-LABEL for these includes",262 )263 parser.add_argument(264 "--check-part-newline",265 action="store_true",266 help="Add empty line between different check parts",267 )268 parser.add_argument(269 "--prefix-only",270 action="append",271 default=None,272 help="Update only these prefixes (default: all)",273 )274 parser.add_argument("--bindir", help="Location of the opt program")275 parser.add_argument("--testdir", help="Root dir for unit tests")276 parser.add_argument(277 "--inplace", "-i", action="store_true", help="Replace input file"278 )279 parser.add_argument("--output", "-o", help="Write changed input to this file")280 known = parser.parse_args()281 282 if not known.inplace and known.output is None:283 print("Must specify what to do with output (--output or --inplace)")284 exit(1)285 if known.inplace and known.output is not None:286 print("--inplace and --output are mutually exclusive")287 exit(1)288 289 outfile = known.output290 291 filecheckparser = argparse.ArgumentParser(add_help=False)292 filecheckparser.add_argument("-check-prefix", "--check-prefix", default="CHECK")293 294 filename = known.testfile295 for dir in [".", known.testdir, os.path.join(polly_src_dir, "test"), polly_src_dir]:296 if not dir:297 continue298 testfilename = os.path.join(dir, filename)299 if os.path.isfile(testfilename):300 filename = testfilename301 break302 303 if known.inplace:304 outfile = filename305 306 allchecklines = []307 checkprefixes = []308 309 with open(filename, "r") as file:310 oldlines = [line.rstrip("\r\n") for line in file.readlines()]311 312 runlines = []313 for line in oldlines:314 m = runre.match(line)315 if m:316 runlines.append(m.group("tool"))317 318 continuation = ""319 newrunlines = []320 for line in runlines:321 if line.endswith("\\"):322 continuation += line[:-2] + " "323 else:324 newrunlines.append(continuation + line)325 continuation = ""326 if continuation:327 newrunlines.append(continuation)328 329 for line in newrunlines:330 m = filecheckre.match(line)331 if not m:332 continue333 334 tool, filecheck = m.group("tool", "filecheck")335 filecheck = shlex.split(filecheck)336 tool = shlex.split(tool)337 if known.bindir is not None:338 tool[0] = complete_exename(known.bindir, tool[0])339 if os.path.isdir(llvm_tools_dir):340 tool[0] = complete_exename(llvm_tools_dir, tool[0])341 check_prefix = filecheckparser.parse_known_args(filecheck)[0].check_prefix342 if known.prefix_only is not None and not check_prefix in known.prefix_only:343 continue344 if check_prefix in checkprefixes:345 continue346 checkprefixes.append(check_prefix)347 348 newtool = []349 optstderr = None350 for toolarg in tool:351 toolarg = toolarg.replace("%s", filename)352 toolarg = toolarg.replace("%S", os.path.dirname(filename))353 if toolarg == "%loadPolly":354 if not llvm_polly_link_into_tools:355 newtool += [356 "-load",357 os.path.join(polly_lib_dir, "LLVMPolly" + shlibext),358 ]359 newtool.append("-polly-process-unprofitable")360 newtool.append("-polly-remarks-minimal")361 elif toolarg == "2>&1":362 optstderr = subprocess.STDOUT363 else:364 newtool.append(toolarg)365 tool = newtool366 367 inpfile = None368 i = 1369 while i < len(tool):370 if tool[i] == "<":371 inpfile = tool[i + 1]372 del tool[i : i + 2]373 continue374 i += 1375 if inpfile:376 with open(inpfile) as inp:377 retlines = subprocess.check_output(378 tool, universal_newlines=True, stdin=inp, stderr=optstderr379 )380 else:381 retlines = subprocess.check_output(382 tool, universal_newlines=True, stderr=optstderr383 )384 retlines = [line.replace("\t", " ") for line in retlines.splitlines()]385 check_include = []386 for checkme in known.check_include + known.check_label_include:387 parts = checkme.split("=")388 if len(parts) == 2:389 if parts[0] == check_prefix:390 check_include.append(parts[1])391 else:392 check_include.append(checkme)393 394 if check_include:395 filtered_retlines = []396 classified_retlines = []397 lastmatch = None398 for line, kind in (399 (line, class1.union(class2))400 for line, class1, class2 in zip(401 retlines, classyfier1(retlines), classyfier2(retlines)402 )403 ):404 match = kind.intersection(check_include)405 if match:406 if lastmatch != match:407 filtered_retlines.append("")408 classified_retlines.append({"Separator"})409 filtered_retlines.append(line)410 classified_retlines.append(kind)411 lastmatch = match412 413 retlines = filtered_retlines414 else:415 classified_retlines = (set() for line in retlines)416 417 rtrim_emptylines(retlines)418 ltrim_emptylines(retlines, classified_retlines)419 retlines = [420 replre.sub(lambda m: replrepl[m.group(0)], line) for line in retlines421 ]422 indent = common_indent(retlines)423 retlines = [line[indent:] for line in retlines]424 checklines = []425 previous_was_empty = True426 for line, kind in zip(retlines, classified_retlines):427 if line:428 if known.check_style == "CHECK" and known.check_label_include:429 if not kind.isdisjoint(known.check_label_include):430 checklines.append("; " + check_prefix + "-LABEL: " + line)431 else:432 checklines.append("; " + check_prefix + ": " + line)433 elif known.check_style == "CHECK":434 checklines.append("; " + check_prefix + ": " + line)435 elif known.check_label_include and known.check_label_include:436 if not kind.isdisjoint(known.check_label_include):437 checklines.append("; " + check_prefix + "-LABEL: " + line)438 elif previous_was_empty:439 checklines.append("; " + check_prefix + ": " + line)440 else:441 checklines.append("; " + check_prefix + "-NEXT: " + line)442 else:443 if previous_was_empty:444 checklines.append("; " + check_prefix + ": " + line)445 else:446 checklines.append("; " + check_prefix + "-NEXT: " + line)447 previous_was_empty = False448 else:449 if not "Separator" in kind or known.check_part_newline:450 checklines.append(";")451 previous_was_empty = True452 allchecklines.append(checklines)453 454 if not checkprefixes:455 return456 457 checkre = re.compile(458 r"^\s*\;\s*("459 + "|".join([re.escape(s) for s in checkprefixes])460 + r")(\-NEXT|\-DAG|\-NOT|\-LABEL|\-SAME)?\s*\:"461 )462 firstcheckline = None463 firstnoncommentline = None464 headerlines = []465 newlines = []466 uptonowlines = []467 emptylines = []468 lastwascheck = False469 for line in oldlines:470 if checkre.match(line):471 if firstcheckline is None:472 firstcheckline = len(newlines) + len(emptylines)473 if not lastwascheck:474 uptonowlines += emptylines475 emptylines = []476 lastwascheck = True477 elif emptyline.fullmatch(line):478 emptylines.append(line)479 else:480 newlines += uptonowlines481 newlines += emptylines482 newlines.append(line)483 emptylines = []484 uptonowlines = []485 lastwascheck = False486 487 for i, line in enumerate(newlines):488 if not commentline.fullmatch(line):489 firstnoncommentline = i490 break491 492 with open(outfile, "w", newline="") as file:493 494 def writelines(lines):495 for line in lines:496 file.write(line)497 file.write("\n")498 499 if firstcheckline is not None and known.check_position == "autodetect":500 writelines(newlines[:firstcheckline])501 writelines(uptonowlines)502 for i, checklines in enumerate(allchecklines):503 if i != 0:504 file.write("\n")505 writelines(checklines)506 writelines(newlines[firstcheckline:])507 writelines(emptylines)508 elif (509 firstnoncommentline is not None and known.check_position == "before-content"510 ):511 headerlines = newlines[:firstnoncommentline]512 rtrim_emptylines(headerlines)513 contentlines = newlines[firstnoncommentline:]514 ltrim_emptylines(contentlines)515 516 writelines(headerlines)517 for checklines in allchecklines:518 file.write("\n")519 writelines(checklines)520 file.write("\n")521 writelines(contentlines)522 writelines(uptonowlines)523 writelines(emptylines)524 else:525 writelines(newlines)526 rtrim_emptylines(newlines)527 for checklines in allchecklines:528 file.write("\n\n")529 writelines(checklines)530 531 532if __name__ == "__main__":533 main()534