640 lines · cpp
1//===- OpDocGen.cpp - MLIR operation documentation generator --------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// OpDocGen uses the description of operations to generate documentation for the10// operations.11//12//===----------------------------------------------------------------------===//13 14#include "DialectGenUtilities.h"15#include "DocGenUtilities.h"16#include "OpGenHelpers.h"17#include "mlir/Support/IndentedOstream.h"18#include "mlir/TableGen/AttrOrTypeDef.h"19#include "mlir/TableGen/Attribute.h"20#include "mlir/TableGen/EnumInfo.h"21#include "mlir/TableGen/GenInfo.h"22#include "mlir/TableGen/Operator.h"23#include "llvm/ADT/DenseMap.h"24#include "llvm/ADT/SetVector.h"25#include "llvm/ADT/StringExtras.h"26#include "llvm/ADT/StringRef.h"27#include "llvm/Support/CommandLine.h"28#include "llvm/Support/FormatVariadic.h"29#include "llvm/Support/Regex.h"30#include "llvm/Support/Signals.h"31#include "llvm/TableGen/Error.h"32#include "llvm/TableGen/Record.h"33#include "llvm/TableGen/TableGenBackend.h"34 35#include <set>36#include <string>37 38using namespace llvm;39using namespace mlir;40using namespace mlir::tblgen;41using mlir::tblgen::Operator;42 43//===----------------------------------------------------------------------===//44// Commandline Options45//===----------------------------------------------------------------------===//46static cl::OptionCategory47 docCat("Options for -gen-(attrdef|typedef|enum|op|dialect)-doc");48static cl::opt<std::string>49 stripPrefix("strip-prefix",50 cl::desc("Strip prefix of the fully qualified names"),51 cl::init("::mlir::"), cl::cat(docCat));52static cl::opt<bool> allowHugoSpecificFeatures(53 "allow-hugo-specific-features",54 cl::desc("Allows using features specific to Hugo"), cl::init(false),55 cl::cat(docCat));56 57void mlir::tblgen::emitSummary(StringRef summary, raw_ostream &os) {58 if (summary.empty())59 return;60 StringRef trimmed = summary.trim();61 char first = std::toupper(trimmed.front());62 StringRef rest = trimmed.drop_front();63 os << "\n_" << first << rest << "_\n";64}65 66// Emit the description by aligning the text to the left per line (e.g.,67// removing the minimum indentation across the block).68//69// This expects that the description in the tablegen file is already formatted70// in a way the user wanted but has some additional indenting due to being71// nested in the op definition.72void mlir::tblgen::emitDescription(StringRef description, raw_ostream &os) {73 if (description.empty())74 return;75 os << "\n";76 raw_indented_ostream ros(os);77 StringRef trimmed = description.rtrim(" \t");78 ros.printReindented(trimmed);79 if (!trimmed.ends_with("\n"))80 ros << "\n";81}82 83void mlir::tblgen::emitDescriptionComment(StringRef description,84 raw_ostream &os, StringRef prefix) {85 if (description.empty())86 return;87 os << "\n";88 raw_indented_ostream ros(os);89 StringRef trimmed = description.rtrim(" \t");90 ros.printReindented(trimmed, (Twine(prefix) + "/// ").str());91 if (!trimmed.ends_with("\n"))92 ros << "\n";93}94 95/// Emit the given named constraint.96template <typename T>97static void emitNamedConstraint(const T &it, raw_ostream &os) {98 if (!it.name.empty())99 os << "| `" << it.name << "`";100 else101 os << "| «unnamed»";102 os << " | " << it.constraint.getSummary() << " |\n";103}104 105//===----------------------------------------------------------------------===//106// Operation Documentation107//===----------------------------------------------------------------------===//108 109/// Emit the assembly format of an operation.110static void emitAssemblyFormat(StringRef opName, StringRef format,111 raw_ostream &os) {112 if (format.empty())113 return;114 os << "\nSyntax:\n\n```\noperation ::= `" << opName << "` ";115 116 // Print the assembly format aligned.117 unsigned indent = strlen("operation ::= ");118 std::pair<StringRef, StringRef> split = format.split('\n');119 os << split.first.trim() << "\n";120 do {121 split = split.second.split('\n');122 StringRef formatChunk = split.first.trim();123 if (!formatChunk.empty())124 os.indent(indent) << formatChunk << "\n";125 } while (!split.second.empty());126 os << "```\n";127}128 129/// Place `text` between backticks so that the Markdown processor renders it as130/// inline code.131static std::string backticks(const std::string &text) {132 return '`' + text + '`';133}134 135static void emitOpTraitsDoc(const Operator &op, raw_ostream &os) {136 // TODO: We should link to the trait/documentation of it. That also means we137 // should add descriptions to traits that can be queried.138 // Collect using set to sort effects, interfaces & traits.139 std::set<std::string> effects, interfaces, traits;140 for (auto &trait : op.getTraits()) {141 if (isa<PredTrait>(&trait))142 continue;143 144 std::string name = trait.getDef().getName().str();145 StringRef ref = name;146 StringRef traitName = trait.getDef().getValueAsString("trait");147 traitName.consume_back("::Trait");148 traitName.consume_back("::Impl");149 if (ref.starts_with("anonymous_"))150 name = traitName.str();151 if (isa<InterfaceTrait>(&trait)) {152 if (trait.getDef().isSubClassOf("SideEffectsTraitBase")) {153 auto effectName = trait.getDef().getValueAsString("baseEffectName");154 effectName.consume_front("::");155 effectName.consume_front("mlir::");156 std::string effectStr;157 raw_string_ostream os(effectStr);158 os << effectName << "{";159 auto list = trait.getDef().getValueAsListOfDefs("effects");160 interleaveComma(list, os, [&](const Record *rec) {161 StringRef effect = rec->getValueAsString("effect");162 effect.consume_front("::");163 effect.consume_front("mlir::");164 os << effect << " on " << rec->getValueAsString("resource");165 });166 os << "}";167 effects.insert(backticks(effectStr));168 name.append(formatv(" ({0})", traitName).str());169 }170 interfaces.insert(backticks(name));171 continue;172 }173 174 traits.insert(backticks(name));175 }176 if (!traits.empty()) {177 interleaveComma(traits, os << "\nTraits: ");178 os << "\n";179 }180 if (!interfaces.empty()) {181 interleaveComma(interfaces, os << "\nInterfaces: ");182 os << "\n";183 }184 if (!effects.empty()) {185 interleaveComma(effects, os << "\nEffects: ");186 os << "\n";187 }188}189 190static StringRef resolveAttrDescription(const Attribute &attr) {191 StringRef description = attr.getDescription();192 if (description.empty())193 return attr.getBaseAttr().getDescription();194 return description;195}196 197static void emitOpDoc(const Operator &op, raw_ostream &os) {198 std::string classNameStr = op.getQualCppClassName();199 StringRef className = classNameStr;200 (void)className.consume_front(stripPrefix);201 os << formatv("\n### `{0}` ({1})\n", op.getOperationName(), className);202 203 // Emit the summary, syntax, and description if present.204 if (op.hasSummary())205 emitSummary(op.getSummary(), os);206 if (op.hasAssemblyFormat())207 emitAssemblyFormat(op.getOperationName(), op.getAssemblyFormat().trim(),208 os);209 if (op.hasDescription())210 mlir::tblgen::emitDescription(op.getDescription(), os);211 212 emitOpTraitsDoc(op, os);213 214 // Emit attributes.215 if (op.getNumAttributes() != 0) {216 os << "\n#### Attributes:\n\n";217 // Note: This table is HTML rather than markdown so the attribute's218 // description can appear in an expandable region. The description may be219 // multiple lines, which is not supported in a markdown table cell.220 os << "<table>\n";221 // Header.222 os << "<tr><th>Attribute</th><th>MLIR Type</th><th>Description</th></tr>\n";223 for (const auto &it : op.getAttributes()) {224 StringRef storageType = it.attr.getStorageType();225 // Name and storage type.226 os << "<tr>";227 os << "<td><code>" << it.name << "</code></td><td>" << storageType228 << "</td><td>";229 StringRef description = resolveAttrDescription(it.attr);230 if (allowHugoSpecificFeatures && !description.empty()) {231 // Expandable description.232 // This appears as just the summary, but when clicked shows the full233 // description.234 os << "<details>" << "<summary>" << it.attr.getSummary() << "</summary>"235 << "{{% markdown %}}" << description << "{{% /markdown %}}"236 << "</details>";237 } else {238 // Fallback: Single-line summary.239 os << it.attr.getSummary();240 }241 os << "</td></tr>\n";242 }243 os << "</table>\n";244 }245 246 // Emit each of the operands.247 if (op.getNumOperands() != 0) {248 os << "\n#### Operands:\n\n";249 os << "| Operand | Description |\n"250 << "| :-----: | ----------- |\n";251 for (const auto &it : op.getOperands())252 emitNamedConstraint(it, os);253 }254 255 // Emit results.256 if (op.getNumResults() != 0) {257 os << "\n#### Results:\n\n";258 os << "| Result | Description |\n"259 << "| :----: | ----------- |\n";260 for (const auto &it : op.getResults())261 emitNamedConstraint(it, os);262 }263 264 // Emit successors.265 if (op.getNumSuccessors() != 0) {266 os << "\n#### Successors:\n\n";267 os << "| Successor | Description |\n"268 << "| :-------: | ----------- |\n";269 for (const auto &it : op.getSuccessors())270 emitNamedConstraint(it, os);271 }272 273 os << "\n";274}275 276static void emitSourceLink(StringRef inputFilename, raw_ostream &os) {277 size_t pathBegin = inputFilename.find("mlir/include/mlir/");278 if (pathBegin == StringRef::npos)279 return;280 281 StringRef inputFromMlirInclude = inputFilename.substr(pathBegin);282 283 os << "\n[source](https://github.com/llvm/llvm-project/blob/main/"284 << inputFromMlirInclude << ")\n";285}286 287static void emitOpDoc(const RecordKeeper &records, raw_ostream &os) {288 auto opDefs = getRequestedOpDefinitions(records);289 290 os << "<!-- Autogenerated by mlir-tblgen; don't manually edit -->\n";291 emitSourceLink(records.getInputFilename(), os);292 for (const Record *opDef : opDefs)293 emitOpDoc(Operator(opDef), os);294}295 296//===----------------------------------------------------------------------===//297// Attribute Documentation298//===----------------------------------------------------------------------===//299 300static void emitAttrDoc(const Attribute &attr, raw_ostream &os) {301 os << "\n### " << attr.getSummary() << "\n";302 emitDescription(attr.getDescription(), os);303 os << "\n";304}305 306//===----------------------------------------------------------------------===//307// Type Documentation308//===----------------------------------------------------------------------===//309 310static void emitTypeDoc(const Type &type, raw_ostream &os) {311 os << "\n### " << type.getSummary() << "\n";312 emitDescription(type.getDescription(), os);313 os << "\n";314}315 316//===----------------------------------------------------------------------===//317// TypeDef Documentation318//===----------------------------------------------------------------------===//319 320static void emitAttrOrTypeDefAssemblyFormat(const AttrOrTypeDef &def,321 raw_ostream &os) {322 ArrayRef<AttrOrTypeParameter> parameters = def.getParameters();323 char prefix = isa<AttrDef>(def) ? '#' : '!';324 if (parameters.empty()) {325 os << "\nSyntax: `" << prefix << def.getDialect().getName() << "."326 << def.getMnemonic() << "`\n";327 return;328 }329 330 os << "\nSyntax:\n\n```\n"331 << prefix << def.getDialect().getName() << "." << def.getMnemonic()332 << "<\n";333 for (const auto &it : llvm::enumerate(parameters)) {334 const AttrOrTypeParameter ¶m = it.value();335 os << " " << param.getSyntax();336 if (it.index() < (parameters.size() - 1))337 os << ",";338 os << " # " << param.getName() << "\n";339 }340 os << ">\n```\n";341}342 343static void emitAttrOrTypeDefDoc(const AttrOrTypeDef &def, raw_ostream &os) {344 os << formatv("\n### {0}\n", def.getCppClassName());345 346 // Emit the summary if present.347 if (def.hasSummary())348 emitSummary(def.getSummary(), os);349 350 // Emit the syntax if present.351 if (def.getMnemonic() && !def.hasCustomAssemblyFormat())352 emitAttrOrTypeDefAssemblyFormat(def, os);353 354 // Emit the description if present.355 if (def.hasDescription()) {356 mlir::tblgen::emitDescription(def.getDescription(), os);357 }358 359 // Emit parameter documentation.360 ArrayRef<AttrOrTypeParameter> parameters = def.getParameters();361 if (!parameters.empty()) {362 os << "\n#### Parameters:\n\n";363 os << "| Parameter | C++ type | Description |\n"364 << "| :-------: | :-------: | ----------- |";365 for (const auto &it : parameters) {366 auto desc = it.getSummary();367 os << "\n| " << it.getName() << " | `" << it.getCppType() << "` | "368 << (desc ? *desc : "") << " |";369 }370 }371 372 os << "\n";373}374 375static void emitAttrOrTypeDefDoc(const RecordKeeper &records, raw_ostream &os,376 StringRef recordTypeName) {377 auto defs = records.getAllDerivedDefinitions(recordTypeName);378 379 os << "<!-- Autogenerated by mlir-tblgen; don't manually edit -->\n";380 for (const Record *def : defs)381 emitAttrOrTypeDefDoc(AttrOrTypeDef(def), os);382}383 384//===----------------------------------------------------------------------===//385// Enum Documentation386//===----------------------------------------------------------------------===//387 388static void emitEnumDoc(const EnumInfo &def, raw_ostream &os) {389 os << formatv("\n### {0}\n", def.getEnumClassName());390 391 // Emit the summary if present.392 emitSummary(def.getSummary(), os);393 394 // Emit case documentation.395 std::vector<EnumCase> cases = def.getAllCases();396 os << "\n#### Cases:\n\n";397 os << "| Symbol | Value | String |\n"398 << "| :----: | :---: | ------ |";399 for (const auto &it : cases) {400 os << "\n| " << it.getSymbol() << " | `" << it.getValue() << "` | "401 << it.getStr() << " |";402 }403 404 os << "\n";405}406 407static void emitEnumDoc(const RecordKeeper &records, raw_ostream &os) {408 os << "<!-- Autogenerated by mlir-tblgen; don't manually edit -->\n";409 for (const Record *def : records.getAllDerivedDefinitions("EnumInfo"))410 emitEnumDoc(EnumInfo(def), os);411}412 413//===----------------------------------------------------------------------===//414// Dialect Documentation415//===----------------------------------------------------------------------===//416 417struct OpDocGroup {418 const Dialect &getDialect() const { return ops.front().getDialect(); }419 420 // Returns the summary description of the section.421 std::string summary = "";422 423 // Returns the description of the section.424 StringRef description = "";425 426 // Instances inside the section.427 std::vector<Operator> ops;428};429 430static void maybeNest(bool nest, llvm::function_ref<void(raw_ostream &os)> fn,431 raw_ostream &os) {432 std::string str;433 raw_string_ostream ss(str);434 fn(ss);435 for (StringRef x : llvm::split(str, "\n")) {436 if (nest && x.starts_with("#"))437 os << "#";438 os << x << "\n";439 }440}441 442static void emitBlock(ArrayRef<Attribute> attributes, StringRef inputFilename,443 ArrayRef<AttrDef> attrDefs, ArrayRef<OpDocGroup> ops,444 ArrayRef<Type> types, ArrayRef<TypeDef> typeDefs,445 ArrayRef<EnumInfo> enums, raw_ostream &os) {446 if (!ops.empty()) {447 os << "\n## Operations\n";448 emitSourceLink(inputFilename, os);449 for (const OpDocGroup &grouping : ops) {450 bool nested = !grouping.summary.empty();451 maybeNest(452 nested,453 [&](raw_ostream &os) {454 if (nested) {455 os << "\n## " << StringRef(grouping.summary).trim() << "\n";456 emitDescription(grouping.description, os);457 os << "\n";458 }459 for (const Operator &op : grouping.ops) {460 emitOpDoc(op, os);461 }462 },463 os);464 }465 }466 467 if (!attributes.empty()) {468 os << "\n## Attribute constraints\n";469 for (const Attribute &attr : attributes)470 emitAttrDoc(attr, os);471 }472 473 if (!attrDefs.empty()) {474 os << "\n## Attributes\n";475 for (const AttrDef &def : attrDefs)476 emitAttrOrTypeDefDoc(def, os);477 }478 479 // TODO: Add link between use and def for types480 if (!types.empty()) {481 os << "\n## Type constraints\n";482 for (const Type &type : types)483 emitTypeDoc(type, os);484 }485 486 if (!typeDefs.empty()) {487 os << "\n## Types\n";488 for (const TypeDef &def : typeDefs)489 emitAttrOrTypeDefDoc(def, os);490 }491 492 if (!enums.empty()) {493 os << "\n## Enums\n";494 for (const EnumInfo &def : enums)495 emitEnumDoc(def, os);496 }497}498 499static void emitDialectDoc(const Dialect &dialect, StringRef inputFilename,500 ArrayRef<Attribute> attributes,501 ArrayRef<AttrDef> attrDefs, ArrayRef<OpDocGroup> ops,502 ArrayRef<Type> types, ArrayRef<TypeDef> typeDefs,503 ArrayRef<EnumInfo> enums, raw_ostream &os) {504 os << "\n# '" << dialect.getName() << "' Dialect\n";505 emitSummary(dialect.getSummary(), os);506 emitDescription(dialect.getDescription(), os);507 508 // Generate a TOC marker except if description already contains one.509 Regex r("^[[:space:]]*\\[TOC\\]$", Regex::RegexFlags::Newline);510 if (!r.match(dialect.getDescription()))511 os << "\n[TOC]\n";512 513 emitBlock(attributes, inputFilename, attrDefs, ops, types, typeDefs, enums,514 os);515}516 517static bool emitDialectDoc(const RecordKeeper &records, raw_ostream &os) {518 auto dialectDefs = records.getAllDerivedDefinitionsIfDefined("Dialect");519 SmallVector<Dialect> dialects(dialectDefs.begin(), dialectDefs.end());520 std::optional<Dialect> dialect = findDialectToGenerate(dialects);521 if (!dialect)522 return true;523 524 std::vector<const Record *> opDefs = getRequestedOpDefinitions(records);525 auto attrDefs = records.getAllDerivedDefinitionsIfDefined("DialectAttr");526 auto typeDefs = records.getAllDerivedDefinitionsIfDefined("DialectType");527 auto typeDefDefs = records.getAllDerivedDefinitionsIfDefined("TypeDef");528 auto attrDefDefs = records.getAllDerivedDefinitionsIfDefined("AttrDef");529 auto enumDefs = records.getAllDerivedDefinitionsIfDefined("EnumInfo");530 531 std::vector<Attribute> dialectAttrs;532 std::vector<AttrDef> dialectAttrDefs;533 std::vector<OpDocGroup> dialectOps;534 std::vector<Type> dialectTypes;535 std::vector<TypeDef> dialectTypeDefs;536 std::vector<EnumInfo> dialectEnums;537 538 SmallDenseSet<const Record *> seen;539 auto addIfNotSeen = [&](const Record *record, const auto &def, auto &vec) {540 if (seen.insert(record).second) {541 vec.push_back(def);542 return true;543 }544 return false;545 };546 auto addIfInDialect = [&](const Record *record, const auto &def, auto &vec) {547 return def.getDialect() == *dialect && addIfNotSeen(record, def, vec);548 };549 550 SmallDenseMap<const Record *, OpDocGroup> opDocGroup;551 552 for (const Record *def : attrDefDefs)553 addIfInDialect(def, AttrDef(def), dialectAttrDefs);554 for (const Record *def : attrDefs)555 addIfInDialect(def, Attribute(def), dialectAttrs);556 for (const Record *def : opDefs) {557 if (const Record *group = def->getValueAsOptionalDef("opDocGroup")) {558 OpDocGroup &op = opDocGroup[group];559 addIfInDialect(def, Operator(def), op.ops);560 } else {561 OpDocGroup op;562 op.ops.emplace_back(def);563 addIfInDialect(def, op, dialectOps);564 }565 }566 for (const Record *rec :567 records.getAllDerivedDefinitionsIfDefined("OpDocGroup")) {568 if (opDocGroup[rec].ops.empty())569 continue;570 opDocGroup[rec].summary = rec->getValueAsString("summary");571 opDocGroup[rec].description = rec->getValueAsString("description");572 dialectOps.push_back(opDocGroup[rec]);573 }574 for (const Record *def : typeDefDefs)575 addIfInDialect(def, TypeDef(def), dialectTypeDefs);576 for (const Record *def : typeDefs)577 addIfInDialect(def, Type(def), dialectTypes);578 dialectEnums.reserve(enumDefs.size());579 for (const Record *def : enumDefs)580 addIfNotSeen(def, EnumInfo(def), dialectEnums);581 582 // Sort alphabetically ignorning dialect for ops and section name for583 // sections.584 // TODO: The sorting order could be revised, currently attempting to sort of585 // keep in alphabetical order.586 llvm::sort(dialectOps, [](const OpDocGroup &lhs, const OpDocGroup &rhs) {587 auto getDesc = [](const OpDocGroup &arg) -> StringRef {588 if (!arg.summary.empty())589 return arg.summary;590 return arg.ops.front().getDef().getValueAsString("opName");591 };592 return getDesc(lhs).compare_insensitive(getDesc(rhs)) < 0;593 });594 595 os << "<!-- Autogenerated by mlir-tblgen; don't manually edit -->\n";596 emitDialectDoc(*dialect, records.getInputFilename(), dialectAttrs,597 dialectAttrDefs, dialectOps, dialectTypes, dialectTypeDefs,598 dialectEnums, os);599 return false;600}601 602//===----------------------------------------------------------------------===//603// Gen Registration604//===----------------------------------------------------------------------===//605 606static mlir::GenRegistration607 genAttrRegister("gen-attrdef-doc",608 "Generate dialect attribute documentation",609 [](const RecordKeeper &records, raw_ostream &os) {610 emitAttrOrTypeDefDoc(records, os, "AttrDef");611 return false;612 });613 614static mlir::GenRegistration615 genOpRegister("gen-op-doc", "Generate dialect documentation",616 [](const RecordKeeper &records, raw_ostream &os) {617 emitOpDoc(records, os);618 return false;619 });620 621static mlir::GenRegistration622 genTypeRegister("gen-typedef-doc", "Generate dialect type documentation",623 [](const RecordKeeper &records, raw_ostream &os) {624 emitAttrOrTypeDefDoc(records, os, "TypeDef");625 return false;626 });627 628static mlir::GenRegistration629 genEnumRegister("gen-enum-doc", "Generate dialect enum documentation",630 [](const RecordKeeper &records, raw_ostream &os) {631 emitEnumDoc(records, os);632 return false;633 });634 635static mlir::GenRegistration636 genRegister("gen-dialect-doc", "Generate dialect documentation",637 [](const RecordKeeper &records, raw_ostream &os) {638 return emitDialectDoc(records, os);639 });640