499 lines · python
1#!/usr/bin/env python32# A tool to parse the FormatStyle struct from Format.h and update the3# documentation in ../ClangFormatStyleOptions.rst automatically.4# Run from the directory in which this file is located to update the docs.5 6import argparse7import inspect8import os9import re10import sys11from io import TextIOWrapper12from typing import Set13 14CLANG_DIR = os.path.join(os.path.dirname(__file__), "../..")15FORMAT_STYLE_FILE = os.path.join(CLANG_DIR, "include/clang/Format/Format.h")16INCLUDE_STYLE_FILE = os.path.join(17 CLANG_DIR, "include/clang/Tooling/Inclusions/IncludeStyle.h"18)19DOC_FILE = os.path.join(CLANG_DIR, "docs/ClangFormatStyleOptions.rst")20 21PLURALS_FILE = os.path.join(os.path.dirname(__file__), "plurals.txt")22 23plurals: Set[str] = set()24with open(PLURALS_FILE) as f:25 f.seek(0)26 plurals = set(f.read().splitlines())27 28 29def substitute(text, tag, contents):30 replacement = "\n.. START_%s\n\n%s\n\n.. END_%s\n" % (tag, contents, tag)31 pattern = r"\n\.\. START_%s\n.*\n\.\. END_%s\n" % (tag, tag)32 return re.sub(pattern, "%s", text, flags=re.S) % replacement33 34 35def register_plural(singular: str, plural: str):36 if plural not in plurals:37 if not hasattr(register_plural, "generated_new_plural"):38 print(39 "Plural generation: you can use "40 f"`git checkout -- {os.path.relpath(PLURALS_FILE)}` "41 "to reemit warnings or `git add` to include new plurals\n"42 )43 register_plural.generated_new_plural = True44 45 plurals.add(plural)46 with open(PLURALS_FILE, "a") as f:47 f.write(plural + "\n")48 cf = inspect.currentframe()49 lineno = ""50 if cf and cf.f_back:51 lineno = ":" + str(cf.f_back.f_lineno)52 print(53 f"{__file__}{lineno} check if plural of {singular} is {plural}",54 file=sys.stderr,55 )56 return plural57 58 59def pluralize(word: str):60 lword = word.lower()61 if len(lword) >= 2 and lword[-1] == "y" and lword[-2] not in "aeiou":62 return register_plural(word, word[:-1] + "ies")63 elif lword.endswith(("s", "sh", "ch", "x", "z")):64 return register_plural(word, word[:-1] + "es")65 elif lword.endswith("fe"):66 return register_plural(word, word[:-2] + "ves")67 elif lword.endswith("f") and not lword.endswith("ff"):68 return register_plural(word, word[:-1] + "ves")69 else:70 return register_plural(word, word + "s")71 72 73def to_yaml_type(typestr: str):74 if typestr == "bool":75 return "Boolean"76 elif typestr == "int":77 return "Integer"78 elif typestr == "unsigned":79 return "Unsigned"80 elif typestr == "std::string":81 return "String"82 83 match = re.match(r"std::vector<(.*)>$", typestr)84 if match:85 return "List of " + pluralize(to_yaml_type(match.group(1)))86 87 match = re.match(r"std::optional<(.*)>$", typestr)88 if match:89 return to_yaml_type(match.group(1))90 91 return typestr92 93 94def doxygen2rst(text):95 text = re.sub(r"<tt>\s*(.*?)\s*<\/tt>", r"``\1``", text)96 text = re.sub(r"\\c ([^ ,;\.]+)", r"``\1``", text)97 text = re.sub(r"\\\w+ ", "", text)98 return text99 100 101def indent(text, columns, indent_first_line=True):102 indent_str = " " * columns103 s = re.sub(r"\n([^\n])", "\n" + indent_str + "\\1", text, flags=re.S)104 if not indent_first_line or s.startswith("\n"):105 return s106 return indent_str + s107 108 109class Option(object):110 def __init__(self, name, opt_type, comment, version):111 self.name = name112 self.type = opt_type113 self.comment = comment.strip()114 self.enum = None115 self.nested_struct = None116 self.version = version117 118 def __str__(self):119 s = ".. _%s:\n\n**%s** (``%s``) " % (120 self.name,121 self.name,122 to_yaml_type(self.type),123 )124 if self.version:125 s += ":versionbadge:`clang-format %s` " % self.version126 s += ":ref:`¶ <%s>`\n%s" % (self.name, doxygen2rst(indent(self.comment, 2)))127 if self.enum and self.enum.values:128 s += indent("\n\nPossible values:\n\n%s\n" % self.enum, 2)129 if self.nested_struct:130 s += indent(131 "\n\nNested configuration flags:\n\n%s\n" % self.nested_struct, 2132 )133 s = s.replace("<option-name>", self.name)134 return s135 136 137class NestedStruct(object):138 def __init__(self, name, comment):139 self.name = name140 self.comment = comment.strip()141 self.values = []142 143 def __str__(self):144 return self.comment + "\n" + "\n".join(map(str, self.values))145 146 147class NestedField(object):148 def __init__(self, name, comment, version):149 self.name = name150 self.comment = comment.strip()151 self.version = version152 153 def __str__(self):154 if self.version:155 return "\n* ``%s`` :versionbadge:`clang-format %s`\n%s" % (156 self.name,157 self.version,158 doxygen2rst(indent(self.comment, 2, indent_first_line=False)),159 )160 return "\n* ``%s`` %s" % (161 self.name,162 doxygen2rst(indent(self.comment, 2, indent_first_line=False)),163 )164 165 166class Enum(object):167 def __init__(self, name, comment):168 self.name = name169 self.comment = comment.strip()170 self.values = []171 172 def __str__(self):173 return "\n".join(map(str, self.values))174 175 176class NestedEnum(object):177 def __init__(self, name, enumtype, comment, version, values):178 self.name = name179 self.comment = comment180 self.values = values181 self.type = enumtype182 self.version = version183 184 def __str__(self):185 s = ""186 if self.version:187 s = "\n* ``%s %s`` :versionbadge:`clang-format %s`\n\n%s" % (188 to_yaml_type(self.type),189 self.name,190 self.version,191 doxygen2rst(indent(self.comment, 2)),192 )193 else:194 s = "\n* ``%s %s``\n%s" % (195 to_yaml_type(self.type),196 self.name,197 doxygen2rst(indent(self.comment, 2)),198 )199 s += indent("\nPossible values:\n\n", 2)200 s += indent("\n".join(map(str, self.values)), 2)201 return s202 203 204class EnumValue(object):205 def __init__(self, name, comment, config):206 self.name = name207 self.comment = comment208 self.config = config209 210 def __str__(self):211 return "* ``%s`` (in configuration: ``%s``)\n%s" % (212 self.name,213 re.sub(".*_", "", self.config),214 doxygen2rst(indent(self.comment, 2)),215 )216 217 218class OptionsReader:219 def __init__(self, header: TextIOWrapper):220 self.header = header221 self.in_code_block = False222 self.code_indent = 0223 self.lineno = 0224 self.last_err_lineno = -1225 226 def __file_path(self):227 return os.path.relpath(self.header.name)228 229 def __print_line(self, line: str):230 print(f"{self.lineno:>6} | {line}", file=sys.stderr)231 232 def __warning(self, msg: str, line: str):233 print(f"{self.__file_path()}:{self.lineno}: warning: {msg}:", file=sys.stderr)234 self.__print_line(line)235 236 def __clean_comment_line(self, line: str):237 match = re.match(r"^/// (?P<indent> +)?\\code(\{.(?P<lang>\w+)\})?$", line)238 if match:239 if self.in_code_block:240 self.__warning("`\\code` in another `\\code`", line)241 self.in_code_block = True242 indent_str = match.group("indent")243 if not indent_str:244 indent_str = ""245 self.code_indent = len(indent_str)246 lang = match.group("lang")247 if not lang:248 lang = "c++"249 return f"\n{indent_str}.. code-block:: {lang}\n\n"250 251 endcode_match = re.match(r"^/// +\\endcode$", line)252 if endcode_match:253 if not self.in_code_block:254 self.__warning(255 "no correct `\\code` found before this `\\endcode`", line256 )257 self.in_code_block = False258 return ""259 260 # check code block indentation261 if (262 self.in_code_block263 and not line == "///"264 and not line.startswith("/// " + " " * self.code_indent)265 ):266 if self.last_err_lineno == self.lineno - 1:267 self.__print_line(line)268 else:269 self.__warning("code block should be indented", line)270 self.last_err_lineno = self.lineno271 272 match = re.match(r"^/// \\warning$", line)273 if match:274 return "\n.. warning::\n\n"275 276 endwarning_match = re.match(r"^/// +\\endwarning$", line)277 if endwarning_match:278 return ""279 280 match = re.match(r"^/// \\note$", line)281 if match:282 return "\n.. note::\n\n"283 284 endnote_match = re.match(r"^/// +\\endnote$", line)285 if endnote_match:286 return ""287 return line[4:] + "\n"288 289 def read_options(self):290 class State:291 (292 BeforeStruct,293 Finished,294 InStruct,295 InNestedStruct,296 InNestedFieldComment,297 InFieldComment,298 InEnum,299 InEnumMemberComment,300 InNestedEnum,301 InNestedEnumMemberComment,302 ) = range(10)303 304 state = State.BeforeStruct305 306 options = []307 enums = {}308 nested_structs = {}309 comment = ""310 enum = None311 nested_struct = None312 version = None313 deprecated = False314 315 for line in self.header:316 self.lineno += 1317 line = line.strip()318 if state == State.BeforeStruct:319 if line in ("struct FormatStyle {", "struct IncludeStyle {"):320 state = State.InStruct321 elif state == State.InStruct:322 if line.startswith("///"):323 state = State.InFieldComment324 comment = self.__clean_comment_line(line)325 elif line == "};":326 state = State.Finished327 break328 elif state == State.InFieldComment:329 if line.startswith(r"/// \version"):330 match = re.match(r"/// \\version\s*(?P<version>[0-9.]+)*", line)331 if match:332 version = match.group("version")333 elif line.startswith("/// @deprecated"):334 deprecated = True335 elif line.startswith("///"):336 comment += self.__clean_comment_line(line)337 elif line.startswith("enum"):338 state = State.InEnum339 name = re.sub(r"enum\s+(\w+)\s*(:((\s*\w+)+)\s*)?\{", "\\1", line)340 enum = Enum(name, comment)341 elif line.startswith("struct"):342 state = State.InNestedStruct343 name = re.sub(r"struct\s+(\w+)\s*\{", "\\1", line)344 nested_struct = NestedStruct(name, comment)345 elif line.endswith(";"):346 prefix = "// "347 if line.startswith(prefix):348 line = line[len(prefix) :]349 state = State.InStruct350 field_type, field_name = re.match(351 r"([<>:\w(,\s)]+)\s+(\w+);", line352 ).groups()353 if deprecated:354 field_type = "deprecated"355 deprecated = False356 357 if not version:358 self.__warning(f"missing version for {field_name}", line)359 option = Option(str(field_name), str(field_type), comment, version)360 options.append(option)361 version = None362 else:363 raise Exception(364 "Invalid format, expected comment, field or enum\n" + line365 )366 elif state == State.InNestedStruct:367 if line.startswith("///"):368 state = State.InNestedFieldComment369 comment = self.__clean_comment_line(line)370 elif line == "};":371 state = State.InStruct372 nested_structs[nested_struct.name] = nested_struct373 elif state == State.InNestedFieldComment:374 if line.startswith(r"/// \version"):375 match = re.match(r"/// \\version\s*(?P<version>[0-9.]+)*", line)376 if match:377 version = match.group("version")378 elif line.startswith("///"):379 comment += self.__clean_comment_line(line)380 elif line.startswith("enum"):381 state = State.InNestedEnum382 name = re.sub(r"enum\s+(\w+)\s*(:((\s*\w+)+)\s*)?\{", "\\1", line)383 enum = Enum(name, comment)384 else:385 state = State.InNestedStruct386 field_type, field_name = re.match(387 r"([<>:\w(,\s)]+)\s+(\w+);", line388 ).groups()389 # if not version:390 # self.__warning(f"missing version for {field_name}", line)391 if field_type in enums:392 nested_struct.values.append(393 NestedEnum(394 field_name,395 field_type,396 comment,397 version,398 enums[field_type].values,399 )400 )401 else:402 nested_struct.values.append(403 NestedField(field_type + " " + field_name, comment, version)404 )405 version = None406 elif state == State.InEnum:407 if line.startswith("///"):408 state = State.InEnumMemberComment409 comment = self.__clean_comment_line(line)410 elif line == "};":411 state = State.InStruct412 enums[enum.name] = enum413 else:414 # Enum member without documentation. Must be documented415 # where the enum is used.416 pass417 elif state == State.InNestedEnum:418 if line.startswith("///"):419 state = State.InNestedEnumMemberComment420 comment = self.__clean_comment_line(line)421 elif line == "};":422 state = State.InNestedStruct423 enums[enum.name] = enum424 else:425 # Enum member without documentation. Must be426 # documented where the enum is used.427 pass428 elif state == State.InEnumMemberComment:429 if line.startswith("///"):430 comment += self.__clean_comment_line(line)431 else:432 state = State.InEnum433 val = line.replace(",", "")434 pos = val.find(" // ")435 if pos != -1:436 config = val[pos + 4 :]437 val = val[:pos]438 else:439 config = val440 enum.values.append(EnumValue(val, comment, config))441 elif state == State.InNestedEnumMemberComment:442 if line.startswith("///"):443 comment += self.__clean_comment_line(line)444 else:445 state = State.InNestedEnum446 val = line.replace(",", "")447 pos = val.find(" // ")448 if pos != -1:449 config = val[pos + 4 :]450 val = val[:pos]451 else:452 config = val453 enum.values.append(EnumValue(val, comment, config))454 if state != State.Finished:455 raise Exception("Not finished by the end of file")456 457 for option in options:458 if option.type not in [459 "bool",460 "unsigned",461 "int",462 "std::string",463 "std::vector<std::string>",464 "std::vector<IncludeCategory>",465 "std::vector<RawStringFormat>",466 "std::optional<unsigned>",467 "deprecated",468 ]:469 if option.type in enums:470 option.enum = enums[option.type]471 elif option.type in nested_structs:472 option.nested_struct = nested_structs[option.type]473 else:474 raise Exception("Unknown type: %s" % option.type)475 return options476 477 478p = argparse.ArgumentParser()479p.add_argument("-o", "--output", help="path of output file")480args = p.parse_args()481 482with open(FORMAT_STYLE_FILE) as f:483 opts = OptionsReader(f).read_options()484with open(INCLUDE_STYLE_FILE) as f:485 opts += OptionsReader(f).read_options()486 487opts = sorted(opts, key=lambda x: x.name)488options_text = "\n\n".join(map(str, opts))489 490with open(DOC_FILE, encoding="utf-8") as f:491 contents = f.read()492 493contents = substitute(contents, "FORMAT_STYLE_OPTIONS", options_text)494 495with open(496 args.output if args.output else DOC_FILE, "w", newline="", encoding="utf-8"497) as f:498 f.write(contents)499