269 lines · python
1#!/usr/bin/env python32 3from __future__ import print_function4from collections import OrderedDict5from shutil import copyfile6import argparse7import json8import os9import re10import subprocess11import sys12import tempfile13 14 15def normalize(dict_var):16 for k, v in dict_var.items():17 if isinstance(v, OrderedDict):18 normalize(v)19 elif isinstance(v, list):20 for e in v:21 if isinstance(e, OrderedDict):22 normalize(e)23 elif type(v) is str:24 if v != "0x0" and re.match(r"0x[0-9A-Fa-f]+", v):25 dict_var[k] = "0x{{.*}}"26 elif os.path.isfile(v):27 dict_var[k] = "{{.*}}"28 else:29 splits = v.split(" ")30 out_splits = []31 for split in splits:32 inner_splits = split.rsplit(":", 2)33 if os.path.isfile(inner_splits[0]):34 out_splits.append(35 "{{.*}}:%s:%s" % (inner_splits[1], inner_splits[2])36 )37 continue38 out_splits.append(split)39 40 dict_var[k] = " ".join(out_splits)41 42 43def filter_json(dict_var, filters, out):44 for k, v in dict_var.items():45 if type(v) is str:46 if v in filters:47 out.append(dict_var)48 break49 elif isinstance(v, OrderedDict):50 filter_json(v, filters, out)51 elif isinstance(v, list):52 for e in v:53 if isinstance(e, OrderedDict):54 filter_json(e, filters, out)55 56 57def default_clang_path():58 guessed_clang = os.path.join(os.path.dirname(__file__), "clang")59 if os.path.isfile(guessed_clang):60 return guessed_clang61 return None62 63 64def main():65 parser = argparse.ArgumentParser()66 parser.add_argument(67 "--clang",68 help="The clang binary (could be a relative or absolute path)",69 action="store",70 default=default_clang_path(),71 )72 parser.add_argument(73 "--source",74 help="the source file(s). Without --update, the command used to generate the JSON "75 "will be of the format <clang> -cc1 -ast-dump=json <opts> <source>",76 action="store",77 nargs=argparse.ONE_OR_MORE,78 required=True,79 )80 parser.add_argument(81 "--filters",82 help="comma separated list of AST filters. Ex: --filters=TypedefDecl,BuiltinType",83 action="store",84 default="",85 )86 parser.add_argument(87 "--prefix",88 help="The FileCheck prefix",89 action="store",90 default="CHECK",91 )92 update_or_generate_group = parser.add_mutually_exclusive_group()93 update_or_generate_group.add_argument(94 "--update", help="Update the file in-place", action="store_true"95 )96 update_or_generate_group.add_argument(97 "--opts", help="other options", action="store", default="", type=str98 )99 parser.add_argument(100 "--update-manual",101 help="When using --update, also update files that do not have the "102 "autogenerated disclaimer",103 action="store_true",104 )105 args = parser.parse_args()106 107 if not args.source:108 sys.exit("Specify the source file to give to clang.")109 110 clang_binary = os.path.abspath(args.clang)111 if not os.path.isfile(clang_binary):112 sys.exit("clang binary specified not present.")113 114 for src in args.source:115 process_file(116 src,117 clang_binary,118 cmdline_filters=args.filters,119 cmdline_opts=args.opts,120 do_update=args.update,121 force_update=args.update_manual,122 prefix=args.prefix,123 )124 125 126def process_file(127 source_file,128 clang_binary,129 cmdline_filters,130 cmdline_opts,131 do_update,132 force_update,133 prefix,134):135 note_firstline = (136 "// NOTE: CHECK lines have been autogenerated by " "gen_ast_dump_json_test.py"137 )138 filters_line_prefix = "// using --filters="139 note = note_firstline140 141 cmd = [clang_binary, "-cc1"]142 if do_update:143 # When updating the first line of the test must be a RUN: line144 with open(source_file, "r") as srcf:145 first_line = srcf.readline()146 found_autogenerated_line = False147 filters_line = None148 for i, line in enumerate(srcf.readlines()):149 if found_autogenerated_line:150 # print("Filters line: '", line.rstrip(), "'", sep="")151 if line.startswith(filters_line_prefix):152 filters_line = line[len(filters_line_prefix) :].rstrip()153 break154 if line.startswith(note_firstline):155 found_autogenerated_line = True156 # print("Found autogenerated disclaimer at line", i + 1)157 if not found_autogenerated_line and not force_update:158 print(159 "Not updating",160 source_file,161 "since it is not autogenerated.",162 file=sys.stderr,163 )164 return165 if not cmdline_filters and filters_line:166 cmdline_filters = filters_line167 print("Inferred filters as '" + cmdline_filters + "'")168 169 if "RUN: %clang_cc1 " not in first_line:170 sys.exit(171 "When using --update the first line of the input file must contain RUN: %clang_cc1"172 )173 clang_start = first_line.find("%clang_cc1") + len("%clang_cc1")174 file_check_idx = first_line.rfind("| FileCheck")175 if file_check_idx:176 dump_cmd = first_line[clang_start:file_check_idx]177 else:178 dump_cmd = first_line[clang_start:]179 print("Inferred run arguments as '", dump_cmd, "'", sep="")180 options = dump_cmd.split()181 if "-ast-dump=json" not in options:182 sys.exit("ERROR: RUN: line does not contain -ast-dump=json")183 if "%s" not in options:184 sys.exit("ERROR: RUN: line does not contain %s")185 options.remove("%s")186 else:187 options = cmdline_opts.split()188 options.append("-ast-dump=json")189 cmd.extend(options)190 using_ast_dump_filter = any("ast-dump-filter" in arg for arg in cmd)191 cmd.append(source_file)192 print("Will run", cmd)193 filters = set()194 if cmdline_filters:195 note += "\n" + filters_line_prefix + cmdline_filters196 filters = set(cmdline_filters.split(","))197 print("Will use the following filters:", filters)198 199 try:200 json_str = subprocess.check_output(cmd).decode()201 except Exception as ex:202 print("The clang command failed with %s" % ex)203 return -1204 205 out_asts = []206 if using_ast_dump_filter:207 # If we're using a filter, then we might have multiple JSON objects208 # in the output. To parse each out, we use a manual JSONDecoder in209 # "raw" mode and update our location in the string based on where the210 # last document ended.211 decoder = json.JSONDecoder(object_hook=OrderedDict)212 doc_start = 0213 prev_end = 0214 while True:215 try:216 prev_end = doc_start217 (j, doc_start) = decoder.raw_decode(json_str[doc_start:])218 doc_start += prev_end + 1219 normalize(j)220 out_asts.append(j)221 except:222 break223 else:224 j = json.loads(json_str, object_pairs_hook=OrderedDict)225 normalize(j)226 227 if len(filters) == 0:228 out_asts.append(j)229 else:230 filter_json(j, filters, out_asts)231 232 with tempfile.NamedTemporaryFile("w", delete=False) as f:233 with open(source_file, "r") as srcf:234 for line in srcf.readlines():235 # copy up to the note:236 if line.rstrip() == note_firstline:237 break238 f.write(line)239 f.write(note + "\n")240 for out_ast in out_asts:241 append_str = json.dumps(out_ast, indent=1, ensure_ascii=False)242 out_str = "\n\n"243 out_str += f"// {prefix}-NOT: {{{{^}}}}Dumping\n"244 index = 0245 for append_line in append_str.splitlines()[2:]:246 if index == 0:247 out_str += f"// {prefix}: %s\n" % (append_line.rstrip())248 index += 1249 else:250 out_str += f"// {prefix}-NEXT: %s\n" % (append_line.rstrip())251 252 f.write(out_str)253 f.flush()254 f.close()255 if do_update:256 print("Updating json appended source file to %s." % source_file)257 copyfile(f.name, source_file)258 else:259 partition = source_file.rpartition(".")260 dest_path = "%s-json%s%s" % (partition[0], partition[1], partition[2])261 print("Writing json appended source file to %s." % dest_path)262 copyfile(f.name, dest_path)263 os.remove(f.name)264 return 0265 266 267if __name__ == "__main__":268 main()269