617 lines · python
1#!/usr/bin/env python32# A tool to parse ASTMatchers.h and update the documentation in3# ../LibASTMatchersReference.html automatically. Run from the4# directory in which this file is located to update the docs.5 6import collections7import re8import os9from urllib.request import urlopen10 11 12CLASS_INDEX_PAGE_URL = "https://clang.llvm.org/doxygen/classes.html"13try:14 CLASS_INDEX_PAGE = urlopen(CLASS_INDEX_PAGE_URL).read().decode("utf-8")15except Exception as e:16 CLASS_INDEX_PAGE = None17 print("Unable to get %s: %s" % (CLASS_INDEX_PAGE_URL, e))18 19CURRENT_DIR = os.path.dirname(__file__)20MATCHERS_FILE = os.path.join(21 CURRENT_DIR, "../../include/clang/ASTMatchers/ASTMatchers.h"22)23HTML_FILE = os.path.join(CURRENT_DIR, "../LibASTMatchersReference.html")24 25# Each matcher is documented in one row of the form:26# result | name | argA27# The subsequent row contains the documentation and is hidden by default,28# becoming visible via javascript when the user clicks the matcher name.29TD_TEMPLATE = """30<tr><td>%(result)s</td><td class="name" onclick="toggle('%(id)s')"><a name="%(id)sAnchor">%(name)s</a></td><td>%(args)s</td></tr>31<tr><td colspan="4" class="doc" id="%(id)s"><pre>%(comment)s</pre></td></tr>32"""33 34# We categorize the matchers into these three categories in the reference:35node_matchers = {}36narrowing_matchers = {}37traversal_matchers = {}38 39# We output multiple rows per matcher if the matcher can be used on multiple40# node types. Thus, we need a new id per row to control the documentation41# pop-up. ids[name] keeps track of those ids.42ids = collections.defaultdict(int)43 44# Cache for doxygen urls we have already verified.45doxygen_probes = {}46 47 48def esc(text):49 """Escape any html in the given text."""50 text = re.sub(r"&", "&", text)51 text = re.sub(r"<", "<", text)52 text = re.sub(r">", ">", text)53 54 def link_if_exists(m):55 """Wrap a likely AST node name in a link to its clang docs.56 57 We want to do this only if the page exists, in which case it will be58 referenced from the class index page.59 """60 name = m.group(1)61 url = "https://clang.llvm.org/doxygen/classclang_1_1%s.html" % name62 if url not in doxygen_probes:63 search_str = 'href="classclang_1_1%s.html"' % name64 if CLASS_INDEX_PAGE is not None:65 doxygen_probes[url] = search_str in CLASS_INDEX_PAGE66 else:67 doxygen_probes[url] = True68 if not doxygen_probes[url]:69 print("Did not find %s in class index page" % name)70 if doxygen_probes[url]:71 return r'Matcher<<a href="%s">%s</a>>' % (url, name)72 else:73 return m.group(0)74 75 text = re.sub(r"Matcher<([^\*&]+)>", link_if_exists, text)76 return text77 78 79def extract_result_types(comment):80 """Extracts a list of result types from the given comment.81 82 We allow annotations in the comment of the matcher to specify what83 nodes a matcher can match on. Those comments have the form:84 Usable as: Any Matcher | (Matcher<T1>[, Matcher<t2>[, ...]])85 86 Returns ['*'] in case of 'Any Matcher', or ['T1', 'T2', ...].87 Returns the empty list if no 'Usable as' specification could be88 parsed.89 """90 result_types = []91 m = re.search(r"Usable as: Any Matcher[\s\n]*$", comment, re.S)92 if m:93 return ["*"]94 while True:95 m = re.match(r"^(.*)Matcher<([^>]+)>\s*,?[\s\n]*$", comment, re.S)96 if not m:97 if re.search(r"Usable as:\s*$", comment):98 return result_types99 else:100 return None101 result_types += [m.group(2)]102 comment = m.group(1)103 104 105def strip_doxygen(comment):106 """Returns the given comment without -escaped words."""107 # If there is only a doxygen keyword in the line, delete the whole line.108 comment = re.sub(r"^\\[^\s]+\n", r"", comment, flags=re.M)109 110 # If there is a doxygen \see command, change the \see prefix into "See also:".111 # FIXME: it would be better to turn this into a link to the target instead.112 comment = re.sub(r"\\see", r"See also:", comment)113 114 # Delete the doxygen command and the following whitespace.115 comment = re.sub(r"\\[^\s]+\s+", r"", comment)116 return comment117 118 119def unify_arguments(args):120 """Gets rid of anything the user doesn't care about in the argument list."""121 args = re.sub(r"clang::ast_matchers::internal::", r"", args)122 args = re.sub(r"ast_matchers::internal::", r"", args)123 args = re.sub(r"internal::", r"", args)124 args = re.sub(r"extern const\s+(.*)&", r"\1 ", args)125 args = re.sub(r"&", r" ", args)126 args = re.sub(r"(^|\s)M\d?(\s)", r"\1Matcher<*>\2", args)127 args = re.sub(r"BindableMatcher", r"Matcher", args)128 args = re.sub(r"const Matcher", r"Matcher", args)129 return args130 131 132def unify_type(result_type):133 """Gets rid of anything the user doesn't care about in the type name."""134 result_type = re.sub(135 r"^internal::(Bindable)?Matcher<([a-zA-Z_][a-zA-Z0-9_]*)>$", r"\2", result_type136 )137 return result_type138 139 140def add_matcher(result_type, name, args, comment, is_dyncast=False):141 """Adds a matcher to one of our categories."""142 if name == "id":143 # FIXME: Figure out whether we want to support the 'id' matcher.144 return145 matcher_id = "%s%d" % (name, ids[name])146 ids[name] += 1147 args = unify_arguments(args)148 result_type = unify_type(result_type)149 150 docs_result_type = esc("Matcher<%s>" % result_type)151 152 if name == "mapAnyOf":153 args = "nodeMatcherFunction..."154 docs_result_type = "<em>unspecified</em>"155 156 matcher_html = TD_TEMPLATE % {157 "result": docs_result_type,158 "name": name,159 "args": esc(args),160 "comment": esc(strip_doxygen(comment)),161 "id": matcher_id,162 }163 if is_dyncast:164 dict = node_matchers165 lookup = result_type + name166 # Use a heuristic to figure out whether a matcher is a narrowing or167 # traversal matcher. By default, matchers that take other matchers as168 # arguments (and are not node matchers) do traversal. We specifically169 # exclude known narrowing matchers that also take other matchers as170 # arguments.171 elif "Matcher<" not in args or name in [172 "allOf",173 "anyOf",174 "anything",175 "unless",176 "mapAnyOf",177 ]:178 dict = narrowing_matchers179 lookup = result_type + name + esc(args)180 else:181 dict = traversal_matchers182 lookup = result_type + name + esc(args)183 184 if dict.get(lookup) is None or len(dict.get(lookup)) < len(matcher_html):185 dict[lookup] = matcher_html186 187 188def act_on_decl(declaration, comment, allowed_types):189 """Parse the matcher out of the given declaration and comment.190 191 If 'allowed_types' is set, it contains a list of node types the matcher192 can match on, as extracted from the static type asserts in the matcher193 definition.194 """195 if declaration.strip():196 197 if re.match(r"^\s?(#|namespace|using|template <typename NodeType> using|})", declaration):198 return199 200 # Node matchers are defined by writing:201 # VariadicDynCastAllOfMatcher<ResultType, ArgumentType> name;202 m = re.match(203 r""".*Variadic(?:DynCast)?AllOfMatcher\s*<204 \s*([^\s,]+)\s*(?:,205 \s*([^\s>]+)\s*)?>206 \s*([^\s;]+)\s*;\s*$""",207 declaration,208 flags=re.X,209 )210 if m:211 result, inner, name = m.groups()212 if not inner:213 inner = result214 add_matcher(215 result, name, "Matcher<%s>..." % inner, comment, is_dyncast=True216 )217 return218 219 # Special case of type matchers:220 # AstTypeMatcher<ArgumentType> name221 m = re.match(222 r""".*AstTypeMatcher\s*<223 \s*([^\s>]+)\s*>224 \s*([^\s;]+)\s*;\s*$""",225 declaration,226 flags=re.X,227 )228 if m:229 inner, name = m.groups()230 add_matcher(231 "Type", name, "Matcher<%s>..." % inner, comment, is_dyncast=True232 )233 # FIXME: re-enable once we have implemented casting on the TypeLoc234 # hierarchy.235 # add_matcher('TypeLoc', '%sLoc' % name, 'Matcher<%sLoc>...' % inner,236 # comment, is_dyncast=True)237 return238 239 # Parse the various matcher definition macros.240 m = re.match(241 r""".*AST_TYPE(LOC)?_TRAVERSE_MATCHER(?:_DECL)?\(242 \s*([^\s,]+\s*),243 \s*(?:[^\s,]+\s*),244 \s*AST_POLYMORPHIC_SUPPORTED_TYPES\(([^)]*)\)245 \)\s*;\s*$""",246 declaration,247 flags=re.X,248 )249 if m:250 loc, name, results = m.groups()[0:3]251 result_types = [r.strip() for r in results.split(",")]252 253 comment_result_types = extract_result_types(comment)254 if comment_result_types and sorted(result_types) != sorted(255 comment_result_types256 ):257 raise Exception("Inconsistent documentation for: %s" % name)258 for result_type in result_types:259 add_matcher(result_type, name, "Matcher<Type>", comment)260 # if loc:261 # add_matcher('%sLoc' % result_type, '%sLoc' % name, 'Matcher<TypeLoc>',262 # comment)263 return264 265 m = re.match(266 r"""^\s*AST_POLYMORPHIC_MATCHER(_P)?(.?)(?:_OVERLOAD)?\(267 \s*([^\s,]+)\s*,268 \s*AST_POLYMORPHIC_SUPPORTED_TYPES\(([^)]*)\)269 (?:,\s*([^\s,]+)\s*270 ,\s*([^\s,]+)\s*)?271 (?:,\s*([^\s,]+)\s*272 ,\s*([^\s,]+)\s*)?273 (?:,\s*\d+\s*)?274 \)\s*{\s*$""",275 declaration,276 flags=re.X,277 )278 279 if m:280 p, n, name, results = m.groups()[0:4]281 args = m.groups()[4:]282 result_types = [r.strip() for r in results.split(",")]283 if allowed_types and allowed_types != result_types:284 raise Exception("Inconsistent documentation for: %s" % name)285 if n not in ["", "2"]:286 raise Exception('Cannot parse "%s"' % declaration)287 args = ", ".join(288 "%s %s" % (args[i], args[i + 1])289 for i in range(0, len(args), 2)290 if args[i]291 )292 for result_type in result_types:293 add_matcher(result_type, name, args, comment)294 return295 296 m = re.match(297 r"""^\s*AST_POLYMORPHIC_MATCHER_REGEX(?:_OVERLOAD)?\(298 \s*([^\s,]+)\s*,299 \s*AST_POLYMORPHIC_SUPPORTED_TYPES\(([^)]*)\),300 \s*([^\s,]+)\s*301 (?:,\s*\d+\s*)?302 \)\s*{\s*$""",303 declaration,304 flags=re.X,305 )306 307 if m:308 name, results, arg_name = m.groups()[0:3]309 result_types = [r.strip() for r in results.split(",")]310 if allowed_types and allowed_types != result_types:311 raise Exception("Inconsistent documentation for: %s" % name)312 arg = "StringRef %s, Regex::RegexFlags Flags = NoFlags" % arg_name313 comment += """314If the matcher is used in clang-query, RegexFlags parameter315should be passed as a quoted string. e.g: "NoFlags".316Flags can be combined with '|' example \"IgnoreCase | BasicRegex\"317"""318 for result_type in result_types:319 add_matcher(result_type, name, arg, comment)320 return321 322 m = re.match(323 r"""^\s*AST_MATCHER_FUNCTION(_P)?(.?)(?:_OVERLOAD)?\(324 (?:\s*([^\s,]+)\s*,)?325 \s*([^\s,]+)\s*326 (?:,\s*([^\s,]+)\s*327 ,\s*([^\s,]+)\s*)?328 (?:,\s*([^\s,]+)\s*329 ,\s*([^\s,]+)\s*)?330 (?:,\s*\d+\s*)?331 \)\s*{\s*$""",332 declaration,333 flags=re.X,334 )335 if m:336 p, n, result, name = m.groups()[0:4]337 args = m.groups()[4:]338 if n not in ["", "2"]:339 raise Exception('Cannot parse "%s"' % declaration)340 args = ", ".join(341 "%s %s" % (args[i], args[i + 1])342 for i in range(0, len(args), 2)343 if args[i]344 )345 add_matcher(result, name, args, comment)346 return347 348 m = re.match(349 r"""^\s*AST_MATCHER(_P)?(.?)(?:_OVERLOAD)?\(350 (?:\s*([^\s,]+)\s*,)?351 \s*([^\s,]+)\s*352 (?:,\s*([^,]+)\s*353 ,\s*([^\s,]+)\s*)?354 (?:,\s*([^\s,]+)\s*355 ,\s*([^\s,]+)\s*)?356 (?:,\s*\d+\s*)?357 \)\s*{""",358 declaration,359 flags=re.X,360 )361 if m:362 p, n, result, name = m.groups()[0:4]363 args = m.groups()[4:]364 if not result:365 if not allowed_types:366 raise Exception("Did not find allowed result types for: %s" % name)367 result_types = allowed_types368 else:369 result_types = [result]370 if n not in ["", "2"]:371 raise Exception('Cannot parse "%s"' % declaration)372 args = ", ".join(373 "%s %s" % (args[i], args[i + 1])374 for i in range(0, len(args), 2)375 if args[i]376 )377 for result_type in result_types:378 add_matcher(result_type, name, args, comment)379 return380 381 m = re.match(382 r"""^\s*AST_MATCHER_REGEX(?:_OVERLOAD)?\(383 \s*([^\s,]+)\s*,384 \s*([^\s,]+)\s*,385 \s*([^\s,]+)\s*386 (?:,\s*\d+\s*)?387 \)\s*{""",388 declaration,389 flags=re.X,390 )391 if m:392 result, name, arg_name = m.groups()[0:3]393 if not result:394 if not allowed_types:395 raise Exception("Did not find allowed result types for: %s" % name)396 result_types = allowed_types397 else:398 result_types = [result]399 arg = "StringRef %s, Regex::RegexFlags Flags = NoFlags" % arg_name400 comment += """401If the matcher is used in clang-query, RegexFlags parameter402should be passed as a quoted string. e.g: "NoFlags".403Flags can be combined with '|' example \"IgnoreCase | BasicRegex\"404"""405 406 for result_type in result_types:407 add_matcher(result_type, name, arg, comment)408 return409 410 # Parse ArgumentAdapting matchers.411 m = re.match(412 r"""^.*ArgumentAdaptingMatcherFunc<.*>\s*413 ([a-zA-Z]*);$""",414 declaration,415 flags=re.X,416 )417 if m:418 name = m.groups()[0]419 add_matcher("*", name, "Matcher<*>", comment)420 return421 422 # Parse Variadic functions.423 m = re.match(424 r"""^.*internal::VariadicFunction\s*<\s*([^,]+),\s*([^,]+),\s*[^>]+>\s*425 ([a-zA-Z]*);$""",426 declaration,427 flags=re.X,428 )429 if m:430 result, arg, name = m.groups()[:3]431 add_matcher(result, name, "%s, ..., %s" % (arg, arg), comment)432 return433 434 m = re.match(435 r"""^.*internal::VariadicFunction\s*<\s*436 internal::PolymorphicMatcher<[\S\s]+437 AST_POLYMORPHIC_SUPPORTED_TYPES\(([^)]*)\),\s*(.*);$""",438 declaration,439 flags=re.X,440 )441 442 if m:443 results, trailing = m.groups()444 trailing, name = trailing.rsplit(">", 1)445 name = name.strip()446 trailing, _ = trailing.rsplit(",", 1)447 _, arg = trailing.rsplit(",", 1)448 arg = arg.strip()449 450 result_types = [r.strip() for r in results.split(",")]451 for result_type in result_types:452 add_matcher(result_type, name, "%s, ..., %s" % (arg, arg), comment)453 return454 455 # Parse Variadic operator matchers.456 m = re.match(457 r"""^.*VariadicOperatorMatcherFunc\s*<\s*([^,]+),\s*([^\s]+)\s*>\s*458 ([a-zA-Z]*);$""",459 declaration,460 flags=re.X,461 )462 if m:463 min_args, max_args, name = m.groups()[:3]464 if max_args == "1":465 add_matcher("*", name, "Matcher<*>", comment)466 return467 elif max_args == "std::numeric_limits<unsigned>::max()":468 add_matcher("*", name, "Matcher<*>, ..., Matcher<*>", comment)469 return470 471 m = re.match(472 r"""^.*MapAnyOfMatcher<.*>\s*473 ([a-zA-Z]*);$""",474 declaration,475 flags=re.X,476 )477 if m:478 name = m.groups()[0]479 add_matcher("*", name, "Matcher<*>...Matcher<*>", comment)480 return481 482 # Parse free standing matcher functions, like:483 # Matcher<ResultType> Name(Matcher<ArgumentType> InnerMatcher) {484 m = re.match(485 r"""^\s*(?:template\s+<\s*(?:class|typename)\s+(.+)\s*>\s+)?486 (.*)\s+487 ([^\s\(]+)\s*\(488 (.*)489 \)\s*{""",490 declaration,491 re.X,492 )493 if m:494 template_name, result, name, args = m.groups()495 if template_name:496 matcherTemplateArgs = re.findall(497 r"Matcher<\s*(%s)\s*>" % template_name, args498 )499 templateArgs = re.findall(500 r"(?:^|[\s,<])(%s)(?:$|[\s,>])" % template_name, args501 )502 if len(matcherTemplateArgs) < len(templateArgs):503 # The template name is used naked, so don't replace with `*`` later on504 template_name = None505 else:506 args = re.sub(507 r"(^|[\s,<])%s($|[\s,>])" % template_name, r"\1*\2", args508 )509 args = ", ".join(p.strip() for p in args.split(","))510 m = re.match(r"(?:^|.*\s+)internal::(?:Bindable)?Matcher<([^>]+)>$", result)511 if m:512 result_types = [m.group(1)]513 if (514 template_name515 and len(result_types) == 1516 and result_types[0] == template_name517 ):518 result_types = ["*"]519 else:520 result_types = extract_result_types(comment)521 if not result_types:522 if not comment:523 # Only overloads don't have their own doxygen comments; ignore those.524 print('Ignoring "%s"' % name)525 else:526 print('Cannot determine result type for "%s"' % name)527 else:528 for result_type in result_types:529 add_matcher(result_type, name, args, comment)530 else:531 print('*** Unparsable: "' + declaration + '" ***')532 533 534def sort_table(matcher_type, matcher_map):535 """Returns the sorted html table for the given row map."""536 table = ""537 for key in sorted(matcher_map.keys()):538 table += matcher_map[key] + "\n"539 return (540 "<!-- START_%(type)s_MATCHERS -->\n"541 + "%(table)s"542 + "<!--END_%(type)s_MATCHERS -->"543 ) % {544 "type": matcher_type,545 "table": table,546 }547 548 549# Parse the ast matchers.550# We alternate between two modes:551# body = True: We parse the definition of a matcher. We need552# to parse the full definition before adding a matcher, as the553# definition might contain static asserts that specify the result554# type.555# body = False: We parse the comments and declaration of the matcher.556comment = ""557declaration = ""558allowed_types = []559body = False560for line in open(MATCHERS_FILE).read().splitlines():561 if body:562 if line.strip() and line[0] == "}":563 if declaration:564 act_on_decl(declaration, comment, allowed_types)565 comment = ""566 declaration = ""567 allowed_types = []568 body = False569 else:570 m = re.search(r"is_base_of<([^,]+), NodeType>", line)571 if m and m.group(1):572 allowed_types += [m.group(1)]573 continue574 if line.strip() and line.lstrip()[0] == "/":575 comment += re.sub(r"^/+\s?", "", line) + "\n"576 else:577 declaration += " " + line578 if (579 (not line.strip())580 or line.rstrip()[-1] == ";"581 or (line.rstrip()[-1] == "{" and line.rstrip()[-3:] != "= {")582 ):583 if line.strip() and line.rstrip()[-1] == "{":584 body = True585 else:586 act_on_decl(declaration, comment, allowed_types)587 comment = ""588 declaration = ""589 allowed_types = []590 591node_matcher_table = sort_table("DECL", node_matchers)592narrowing_matcher_table = sort_table("NARROWING", narrowing_matchers)593traversal_matcher_table = sort_table("TRAVERSAL", traversal_matchers)594 595reference = open(HTML_FILE).read()596reference = re.sub(597 r"<!-- START_DECL_MATCHERS.*END_DECL_MATCHERS -->",598 node_matcher_table,599 reference,600 flags=re.S,601)602reference = re.sub(603 r"<!-- START_NARROWING_MATCHERS.*END_NARROWING_MATCHERS -->",604 narrowing_matcher_table,605 reference,606 flags=re.S,607)608reference = re.sub(609 r"<!-- START_TRAVERSAL_MATCHERS.*END_TRAVERSAL_MATCHERS -->",610 traversal_matcher_table,611 reference,612 flags=re.S,613)614 615with open(HTML_FILE, "w", newline="\n") as output:616 output.write(reference)617