294 lines · python
1#!/usr/bin/env python32# A tool to automatically generate documentation for the config options of the3# clang static analyzer by reading `AnalyzerOptions.def`.4 5import argparse6from collections import namedtuple7from enum import Enum, auto8import re9import sys10import textwrap11 12 13# The following code implements a trivial parser for the narrow subset of C++14# which is used in AnalyzerOptions.def. This supports the following features:15# - ignores preprocessor directives, even if they are continued with \ at EOL16# - ignores comments: both /* ... */ and // ...17# - parses string literals (even if they contain \" escapes)18# - concatenates adjacent string literals19# - parses numbers even if they contain ' as a thousands separator20# - recognizes MACRO(arg1, arg2, ..., argN) calls21 22 23class TT(Enum):24 "Token type enum."25 number = auto()26 ident = auto()27 string = auto()28 punct = auto()29 30 31TOKENS = [32 (re.compile(r"-?[0-9']+"), TT.number),33 (re.compile(r"\w+"), TT.ident),34 (re.compile(r'"([^\\"]|\\.)*"'), TT.string),35 (re.compile(r"[(),]"), TT.punct),36 (re.compile(r"/\*((?!\*/).)*\*/", re.S), None), # C-style comment37 (re.compile(r"//.*\n"), None), # C++ style oneline comment38 (re.compile(r"#.*(\\\n.*)*(?<!\\)\n"), None), # preprocessor directive39 (re.compile(r"\s+"), None), # whitespace40]41 42Token = namedtuple("Token", "kind code")43 44 45class ErrorHandler:46 def __init__(self):47 self.seen_errors = False48 49 # This script uses some heuristical tweaks to modify the documentation50 # of some analyzer options. As this code is fragile, we record the use51 # of these tweaks and report them if they become obsolete:52 self.unused_tweaks = [53 "escape star",54 "escape underline",55 "accepted values",56 "example file content",57 ]58 59 def record_use_of_tweak(self, tweak_name):60 try:61 self.unused_tweaks.remove(tweak_name)62 except ValueError:63 pass64 65 def replace_as_tweak(self, string, pattern, repl, tweak_name):66 res = string.replace(pattern, repl)67 if res != string:68 self.record_use_of_tweak(tweak_name)69 return res70 71 def report_error(self, msg):72 print("Error:", msg, file=sys.stderr)73 self.seen_errors = True74 75 def report_unexpected_char(self, s, pos):76 lines = (s[:pos] + "X").split("\n")77 lineno, col = (len(lines), len(lines[-1]))78 self.report_error(79 "unexpected character %r in AnalyzerOptions.def at line %d column %d"80 % (s[pos], lineno, col),81 )82 83 def report_unused_tweaks(self):84 if not self.unused_tweaks:85 return86 _is = " is" if len(self.unused_tweaks) == 1 else "s are"87 names = ", ".join(self.unused_tweaks)88 self.report_error(f"textual tweak{_is} unused in script: {names}")89 90 91err_handler = ErrorHandler()92 93 94def tokenize(s):95 result = []96 pos = 097 while pos < len(s):98 for regex, kind in TOKENS:99 if m := regex.match(s, pos):100 if kind is not None:101 result.append(Token(kind, m.group(0)))102 pos = m.end()103 break104 else:105 err_handler.report_unexpected_char(s, pos)106 pos += 1107 return result108 109 110def join_strings(tokens):111 result = []112 for tok in tokens:113 if tok.kind == TT.string and result and result[-1].kind == TT.string:114 # If this token is a string, and the previous non-ignored token is115 # also a string, then merge them into a single token. We need to116 # discard the closing " of the previous string and the opening " of117 # this string.118 prev = result.pop()119 result.append(Token(TT.string, prev.code[:-1] + tok.code[1:]))120 else:121 result.append(tok)122 return result123 124 125MacroCall = namedtuple("MacroCall", "name args")126 127 128class State(Enum):129 "States of the state machine used for parsing the macro calls."130 init = auto()131 after_ident = auto()132 before_arg = auto()133 after_arg = auto()134 135 136def get_calls(tokens, macro_names):137 state = State.init138 result = []139 current = None140 for tok in tokens:141 if state == State.init and tok.kind == TT.ident and tok.code in macro_names:142 current = MacroCall(tok.code, [])143 state = State.after_ident144 elif state == State.after_ident and tok == Token(TT.punct, "("):145 state = State.before_arg146 elif state == State.before_arg:147 if current is not None:148 current.args.append(tok)149 state = State.after_arg150 elif state == State.after_arg and tok.kind == TT.punct:151 if tok.code == ")":152 result.append(current)153 current = None154 state = State.init155 elif tok.code == ",":156 state = State.before_arg157 else:158 current = None159 state = State.init160 return result161 162 163# The information will be extracted from calls to these two macros:164# #define ANALYZER_OPTION(TYPE, NAME, CMDFLAG, DESC, DEFAULT_VAL)165# #define ANALYZER_OPTION_DEPENDS_ON_USER_MODE(TYPE, NAME, CMDFLAG, DESC,166# SHALLOW_VAL, DEEP_VAL)167 168MACRO_NAMES_PARAMCOUNTS = {169 "ANALYZER_OPTION": 5,170 "ANALYZER_OPTION_DEPENDS_ON_USER_MODE": 6,171}172 173 174def string_value(tok):175 if tok.kind != TT.string:176 raise ValueError(f"expected a string token, got {tok.kind.name}")177 text = tok.code[1:-1] # Remove quotes178 text = re.sub(r"\\(.)", r"\1", text) # Resolve backslash escapes179 return text180 181 182def cmdflag_to_rst_title(cmdflag_tok):183 text = string_value(cmdflag_tok)184 underline = "-" * len(text)185 ref = f".. _analyzer-option-{text}:"186 187 return f"{ref}\n\n{text}\n{underline}\n\n"188 189 190def desc_to_rst_paragraphs(tok):191 desc = string_value(tok)192 193 # Escape some characters that have special meaning in RST:194 desc = err_handler.replace_as_tweak(desc, "*", r"\*", "escape star")195 desc = err_handler.replace_as_tweak(desc, "_", r"\_", "escape underline")196 197 # Many descriptions end with "Value: <list of accepted values>", which is198 # OK for a terse command line printout, but should be prettified for web199 # documentation.200 # Moreover, the option ctu-invocation-list shows some example file content201 # which is formatted as a preformatted block.202 paragraphs = [desc]203 extra = ""204 if m := re.search(r"(^|\s)Value:", desc):205 err_handler.record_use_of_tweak("accepted values")206 paragraphs = [desc[: m.start()], "Accepted values:" + desc[m.end() :]]207 elif m := re.search(r"\s*Example file.content:", desc):208 err_handler.record_use_of_tweak("example file content")209 paragraphs = [desc[: m.start()]]210 extra = "Example file content::\n\n " + desc[m.end() :] + "\n\n"211 212 wrapped = [textwrap.fill(p, width=80) for p in paragraphs if p.strip()]213 214 return "\n\n".join(wrapped + [""]) + extra215 216 217def default_to_rst(tok):218 if tok.kind == TT.string:219 if tok.code == '""':220 return "(empty string)"221 return tok.code222 if tok.kind == TT.ident:223 return tok.code224 if tok.kind == TT.number:225 return tok.code.replace("'", "")226 raise ValueError(f"unexpected token as default value: {tok.kind.name}")227 228 229def defaults_to_rst_paragraph(defaults):230 strs = [default_to_rst(d) for d in defaults]231 232 if len(strs) == 1:233 return f"Default value: {strs[0]}\n\n"234 if len(strs) == 2:235 return (236 f"Default value: {strs[0]} (in shallow mode) / {strs[1]} (in deep mode)\n\n"237 )238 raise ValueError("unexpected count of default values: %d" % len(defaults))239 240 241def macro_call_to_rst_paragraphs(macro_call):242 try:243 arg_count = len(macro_call.args)244 param_count = MACRO_NAMES_PARAMCOUNTS[macro_call.name]245 if arg_count != param_count:246 raise ValueError(247 f"expected {param_count} arguments for {macro_call.name}, found {arg_count}"248 )249 250 _, _, cmdflag, desc, *defaults = macro_call.args251 252 return (253 cmdflag_to_rst_title(cmdflag)254 + desc_to_rst_paragraphs(desc)255 + defaults_to_rst_paragraph(defaults)256 )257 except ValueError as ve:258 err_handler.report_error(ve.args[0])259 return ""260 261 262def get_option_list(input_file):263 with open(input_file, encoding="utf-8") as f:264 contents = f.read()265 tokens = join_strings(tokenize(contents))266 macro_calls = get_calls(tokens, MACRO_NAMES_PARAMCOUNTS)267 268 result = ""269 for mc in macro_calls:270 result += macro_call_to_rst_paragraphs(mc)271 return result272 273 274p = argparse.ArgumentParser()275p.add_argument("--options-def", help="path to AnalyzerOptions.def")276p.add_argument("--template", help="template file")277p.add_argument("--out", help="output file")278opts = p.parse_args()279 280with open(opts.template, encoding="utf-8") as f:281 doc_template = f.read()282 283PLACEHOLDER = ".. OPTIONS_LIST_PLACEHOLDER\n"284 285rst_output = doc_template.replace(PLACEHOLDER, get_option_list(opts.options_def))286 287err_handler.report_unused_tweaks()288 289with open(opts.out, "w", newline="", encoding="utf-8") as f:290 f.write(rst_output)291 292if err_handler.seen_errors:293 sys.exit(1)294