531 lines · python
1#!/usr/bin/env python32"""3A test update script. This script is a utility to update LLVM 'llvm-mc' based test cases with new FileCheck patterns.4"""5 6from __future__ import print_function7 8from sys import stderr9from traceback import print_exc10import argparse11import functools12import os # Used to advertise this file's name ("autogenerated_note").13import subprocess14import re15import sys16 17from UpdateTestChecks import common18 19mc_LIKE_TOOLS = [20 "llvm-mc",21]22ERROR_RE = re.compile(r":\d+: (warning|error): .*")23ERROR_CHECK_RE = re.compile(r"# COM: .*")24OUTPUT_SKIPPED_RE = re.compile(r"(.text)")25COMMENT = {"asm": "//", "dasm": "#"}26 27SUBSTITUTIONS = [28 ("%extract-encodings", "sed -n 's/.*encoding://p'"),29]30 31 32class Error(Exception):33 def __init__(self, test_info, line_no, msg):34 super().__init__(f"{test_info.path}:{line_no}: {msg}")35 36 37def invoke_tool(exe, check_rc, cmd_args, testline, verbose=False):38 substs = SUBSTITUTIONS + [(t, exe) for t in mc_LIKE_TOOLS]39 args = [common.applySubstitutions(cmd, substs) for cmd in cmd_args.split("|")]40 41 cmd = 'echo "' + testline + '" | ' + exe + " " + " | ".join(args)42 if verbose:43 print("Command: ", cmd)44 45 out = subprocess.run(46 cmd,47 shell=True,48 check=check_rc,49 stdout=subprocess.PIPE,50 stderr=subprocess.DEVNULL,51 ).stdout52 53 # Fix line endings to unix CR style.54 return out.decode().replace("\r\n", "\n")55 56 57# create tests line-by-line, here we just filter out the check lines and comments58# and treat all others as tests59def isTestLine(input_line, mc_mode):60 line = input_line.strip()61 # Skip empty and comment lines62 if not line or line.startswith(COMMENT[mc_mode]):63 return False64 # skip any CHECK lines.65 elif common.CHECK_RE.match(input_line):66 return False67 return True68 69 70def isRunLine(l):71 return common.RUN_LINE_RE.match(l)72 73 74def hasErr(err):75 return err and ERROR_RE.search(err) is not None76 77 78def getErrString(err):79 if not err:80 return ""81 82 # take the first match83 for line in err.splitlines():84 s = ERROR_RE.search(line)85 if s:86 return s.group(0)87 return ""88 89 90def getOutputString(out):91 if not out:92 return ""93 output = ""94 95 for line in out.splitlines():96 if OUTPUT_SKIPPED_RE.search(line):97 continue98 if line.strip("\t ") == "":99 continue100 output += line.lstrip("\t ")101 return output102 103 104def should_add_line_to_output(input_line, prefix_set, mc_mode):105 # special check line106 if mc_mode == "dasm" and ERROR_CHECK_RE.search(input_line):107 return False108 else:109 return common.should_add_line_to_output(110 input_line, prefix_set, comment_marker=COMMENT[mc_mode]111 )112 113 114def getStdCheckLine(prefix, output, mc_mode):115 o = ""116 for line in output.splitlines():117 o += COMMENT[mc_mode] + " " + prefix + ": " + line + "\n"118 return o119 120 121def getErrCheckLine(prefix, output, mc_mode, line_offset=1):122 return (123 COMMENT[mc_mode]124 + " "125 + prefix126 + ": "127 + ":[[@LINE-{}]]".format(line_offset)128 + output129 + "\n"130 )131 132 133def parse_token_defs(test_info):134 tokens = {}135 current_token = None136 for line_no, line in enumerate(test_info.input_lines, start=1):137 # Remove comments.138 line = line.split("#")[0].rstrip()139 140 # Skip everything up to the instructions definition.141 if not tokens and not current_token and line != "// INSTS=":142 continue143 144 if not line.startswith("//"):145 break146 147 original_len = len(line)148 line = line[2:].lstrip(" ")149 indent = original_len - len(line)150 151 if not line:152 current_token = None153 continue154 155 # Define a new token.156 if not current_token:157 if indent != 4 or not line.endswith("="):158 raise Error(test_info, line_no, "token definition expected")159 160 current_token = line[:-1].strip()161 if current_token in tokens:162 raise Error(test_info, line_no, f"'{current_token}' redefined")163 164 tokens[current_token] = []165 continue166 167 # Add token value.168 if indent != 8:169 raise Error(test_info, line_no, "wrong indentation for token value")170 171 tokens[current_token].append(line)172 173 return tokens174 175 176def expand_insts(tokens):177 def subst(s):178 for token, values in tokens.items():179 if token in s:180 for value in values:181 yield from subst(s.replace(token, value, 1))182 return183 184 yield s185 186 yield from subst("INSTS")187 188 189def update_test(ti: common.TestInfo):190 if ti.path.endswith(".s"):191 mc_mode = "asm"192 elif ti.path.endswith(".txt"):193 mc_mode = "dasm"194 else:195 common.warn("Expected .s and .txt, Skipping file : ", ti.path)196 return197 198 triple_in_ir = None199 for l in ti.input_lines:200 m = common.TRIPLE_IR_RE.match(l)201 if m:202 triple_in_ir = m.groups()[0]203 break204 205 run_list = []206 for l in ti.run_lines:207 if "|" not in l:208 common.warn("Skipping unparsable RUN line: " + l)209 continue210 211 commands = [cmd.strip() for cmd in l.split("|")]212 assert len(commands) >= 2213 mc_cmd = " | ".join(commands[:-1])214 filecheck_cmd = commands[-1]215 216 # special handling for negating exit status217 # if not is used in runline, disable rc check, since218 # the command might or might not219 # return non-zero code on a single line run220 check_rc = True221 mc_cmd_args = mc_cmd.strip().split()222 if mc_cmd_args[0] == "not":223 check_rc = False224 mc_tool = mc_cmd_args[1]225 mc_cmd = mc_cmd[len(mc_cmd_args[0]) :].strip()226 else:227 mc_tool = mc_cmd_args[0]228 229 triple_in_cmd = None230 m = common.TRIPLE_ARG_RE.search(mc_cmd)231 if m:232 triple_in_cmd = m.groups()[0]233 234 march_in_cmd = ti.args.default_march235 m = common.MARCH_ARG_RE.search(mc_cmd)236 if m:237 march_in_cmd = m.groups()[0]238 239 common.verify_filecheck_prefixes(filecheck_cmd)240 241 mc_like_tools = mc_LIKE_TOOLS[:]242 if ti.args.tool:243 mc_like_tools.append(ti.args.tool)244 if mc_tool not in mc_like_tools:245 common.warn("Skipping non-mc RUN line: " + l)246 continue247 248 if not filecheck_cmd.startswith("FileCheck "):249 common.warn("Skipping non-FileChecked RUN line: " + l)250 continue251 252 mc_cmd_args = mc_cmd[len(mc_tool) :].strip()253 mc_cmd_args = mc_cmd_args.replace("< %s", "").replace("%s", "").strip()254 check_prefixes = common.get_check_prefixes(filecheck_cmd)255 256 run_list.append(257 (258 check_prefixes,259 mc_tool,260 check_rc,261 mc_cmd_args,262 triple_in_cmd,263 march_in_cmd,264 )265 )266 267 # find all test line from input268 testlines = [l for l in ti.input_lines if isTestLine(l, mc_mode)]269 # remove duplicated lines to save running time270 testlines = list(dict.fromkeys(testlines))271 common.debug("Valid test line found: ", len(testlines))272 273 # Where instruction templates are specified, use them instead.274 use_asm_templates = False275 if mc_mode == "asm":276 tokens = parse_token_defs(ti)277 if "INSTS" in tokens:278 testlines = list(expand_insts(tokens))279 use_asm_templates = True280 281 raw_output = []282 raw_prefixes = []283 for (284 prefixes,285 mc_tool,286 check_rc,287 mc_args,288 triple_in_cmd,289 march_in_cmd,290 ) in run_list:291 common.debug("Extracted mc cmd:", mc_tool, mc_args)292 common.debug("Extracted FileCheck prefixes:", str(prefixes))293 common.debug("Extracted triple :", str(triple_in_cmd))294 common.debug("Extracted march:", str(march_in_cmd))295 296 triple = triple_in_cmd or triple_in_ir297 if not triple:298 triple = common.get_triple_from_march(march_in_cmd)299 300 raw_output.append([])301 for line in testlines:302 # get output for each testline303 out = invoke_tool(304 ti.args.llvm_mc_binary or mc_tool,305 check_rc,306 mc_args,307 line,308 verbose=ti.args.verbose,309 )310 raw_output[-1].append(out)311 312 common.debug("Collect raw tool lines:", str(len(raw_output[-1])))313 314 raw_prefixes.append(prefixes)315 316 generated_prefixes = {}317 sort_keys = {}318 used_prefixes = set()319 prefix_set = set([prefix for p in run_list for prefix in p[0]])320 common.debug("Rewriting FileCheck prefixes:", str(prefix_set))321 322 for test_id, input_line in enumerate(testlines):323 # a {prefix : output, [runid] } dict324 # insert output to a prefix-key dict, and do a max sorting325 # to select the most-used prefix which share the same output string326 p_dict = {}327 for run_id in range(len(run_list)):328 out = raw_output[run_id][test_id]329 330 if hasErr(out):331 o = getErrString(out)332 else:333 o = getOutputString(out)334 335 for p in raw_prefixes[run_id]:336 if p not in p_dict:337 p_dict[p] = o, [run_id]338 continue339 340 if p_dict[p] == (None, []):341 continue342 343 prev_o, run_ids = p_dict[p]344 if o == prev_o:345 run_ids.append(run_id)346 p_dict[p] = o, run_ids347 else:348 # conflict, discard349 p_dict[p] = None, []350 351 # prefix is selected and generated with most shared output lines352 # each run_id can only be used once353 used_run_ids = set()354 selected_prefixes = set()355 get_num_runs = lambda item: len(item[1][1])356 p_dict_sorted = sorted(p_dict.items(), key=get_num_runs, reverse=True)357 for prefix, (o, run_ids) in p_dict_sorted:358 if run_ids and used_run_ids.isdisjoint(run_ids):359 selected_prefixes.add(prefix)360 361 used_run_ids.update(run_ids)362 363 # Use smallest outputs across RUN lines as sorting keys for364 # disassembler tests. Sort by instruction codes if no RUN line365 # produced a disassembled instruction.366 if mc_mode == "dasm":367 instr_outs = [368 o369 for prefix, (o, run_ids) in p_dict_sorted370 if o is not None and "encoding:" in o371 ]372 sort_keys[input_line] = min(instr_outs) if instr_outs else input_line373 374 # Generate check lines in alphabetical order.375 check_lines = []376 for prefix in sorted(selected_prefixes):377 o, run_ids = p_dict[prefix]378 used_prefixes.add(prefix)379 380 if hasErr(o):381 line_offset = len(check_lines) + 1382 check = getErrCheckLine(prefix, o, mc_mode, line_offset)383 else:384 check = getStdCheckLine(prefix, o, mc_mode)385 386 if check:387 check_lines.append(check.strip())388 389 generated_prefixes[input_line] = "\n".join(check_lines)390 391 # write output392 output_lines = []393 if use_asm_templates:394 # Keep all leading comments and empty lines.395 for input_info in ti.iterlines(output_lines):396 input_line = input_info.line397 if not input_line or input_line.startswith(COMMENT[mc_mode]):398 output_lines.append(input_line)399 continue400 break401 402 # Remove tail empty lines.403 while not output_lines[-1]:404 del output_lines[-1]405 406 # Emit test and check lines.407 for input_line in testlines:408 output_lines.extend(["", input_line, generated_prefixes[input_line]])409 else:410 for input_info in ti.iterlines(output_lines):411 input_line = input_info.line412 if input_line in testlines:413 output_lines.append(input_line)414 output_lines.append(generated_prefixes[input_line])415 416 elif should_add_line_to_output(input_line, prefix_set, mc_mode):417 output_lines.append(input_line)418 419 if ti.args.unique or ti.args.sort:420 # split with double newlines421 test_units = "\n".join(output_lines).split("\n\n")422 423 # select the key line for each test unit424 test_dic = {}425 for unit in test_units:426 lines = unit.split("\n")427 for l in lines:428 # if contains multiple lines, use429 # the first testline or runline as key430 if isTestLine(l, mc_mode):431 test_dic[unit] = l432 break433 if isRunLine(l):434 test_dic[unit] = l435 break436 437 # unique438 if ti.args.unique:439 new_test_units = []440 written_lines = set()441 for unit in test_units:442 # if not testline/runline, we just add it443 if unit not in test_dic:444 new_test_units.append(unit)445 else:446 if test_dic[unit] in written_lines:447 common.debug("Duplicated test skipped: ", unit)448 continue449 450 written_lines.add(test_dic[unit])451 new_test_units.append(unit)452 test_units = new_test_units453 454 # sort455 if ti.args.sort:456 457 def getkey(l):458 # find key of test unit, otherwise use first line459 if l in test_dic:460 line = test_dic[l]461 else:462 line = l.split("\n")[0]463 464 # runline placed on the top465 return (not isRunLine(line), sort_keys.get(line, line))466 467 test_units = sorted(test_units, key=getkey)468 469 # join back to be output string470 output_lines = "\n\n".join(test_units).split("\n")471 472 # output473 if ti.args.gen_unused_prefix_body:474 output_lines.extend(ti.get_checks_for_unused_prefixes(run_list, used_prefixes))475 476 common.debug("Writing %d lines to %s..." % (len(output_lines), ti.path))477 with open(ti.path, "wb") as f:478 f.writelines(["{}\n".format(l).encode("utf-8") for l in output_lines])479 480 481def main():482 parser = argparse.ArgumentParser(description=__doc__)483 parser.add_argument(484 "--llvm-mc-binary",485 default=None,486 help='The "mc" binary to use to generate the test case',487 )488 parser.add_argument(489 "--tool",490 default=None,491 help="Treat the given tool name as an mc-like tool for which check lines should be generated",492 )493 parser.add_argument(494 "--default-march",495 default=None,496 help="Set a default -march for when neither triple nor arch are found in a RUN line",497 )498 parser.add_argument(499 "--unique",500 action="store_true",501 default=False,502 help="remove duplicated test line if found",503 )504 parser.add_argument(505 "--sort",506 action="store_true",507 default=False,508 help="sort testline in alphabetic order (keep run-lines on top), this option could be dangerous as it"509 "could change the order of lines that are not expected",510 )511 parser.add_argument("tests", nargs="+")512 initial_args = common.parse_commandline_args(parser)513 514 script_name = os.path.basename(__file__)515 516 returncode = 0517 for ti in common.itertests(518 initial_args.tests, parser, script_name="utils/" + script_name519 ):520 try:521 update_test(ti)522 except Exception:523 stderr.write(f"Error: Failed to update test {ti.path}\n")524 print_exc()525 returncode = 1526 return returncode527 528 529if __name__ == "__main__":530 sys.exit(main())531