632 lines · c
1//===- FormatGen.h - Utilities for custom assembly formats ------*- C++ -*-===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// This file contains common classes for building custom assembly format parsers10// and generators.11//12//===----------------------------------------------------------------------===//13 14#ifndef MLIR_TOOLS_MLIRTBLGEN_FORMATGEN_H_15#define MLIR_TOOLS_MLIRTBLGEN_FORMATGEN_H_16 17#include "mlir/Support/LLVM.h"18#include "llvm/ADT/StringRef.h"19#include "llvm/ADT/StringSet.h"20#include "llvm/Support/Allocator.h"21#include "llvm/Support/CommandLine.h"22#include "llvm/Support/SMLoc.h"23#include <vector>24 25namespace llvm {26class SourceMgr;27} // namespace llvm28 29namespace mlir {30namespace tblgen {31 32//===----------------------------------------------------------------------===//33// FormatToken34//===----------------------------------------------------------------------===//35 36/// This class represents a specific token in the input format.37class FormatToken {38public:39 /// Basic token kinds.40 enum Kind {41 // Markers.42 eof,43 error,44 45 // Tokens with no info.46 l_paren,47 r_paren,48 caret,49 colon,50 comma,51 equal,52 less,53 greater,54 question,55 star,56 pipe,57 58 // Keywords.59 keyword_start,60 kw_attr_dict,61 kw_attr_dict_w_keyword,62 kw_prop_dict,63 kw_custom,64 kw_functional_type,65 kw_oilist,66 kw_operands,67 kw_params,68 kw_qualified,69 kw_ref,70 kw_regions,71 kw_results,72 kw_struct,73 kw_successors,74 kw_type,75 keyword_end,76 77 // String valued tokens.78 identifier,79 literal,80 variable,81 string,82 };83 84 FormatToken(Kind kind, StringRef spelling) : kind(kind), spelling(spelling) {}85 86 /// Return the bytes that make up this token.87 StringRef getSpelling() const { return spelling; }88 89 /// Return the kind of this token.90 Kind getKind() const { return kind; }91 92 /// Return a location for this token.93 SMLoc getLoc() const;94 95 /// Returns true if the token is of the given kind.96 bool is(Kind kind) { return getKind() == kind; }97 98 /// Return if this token is a keyword.99 bool isKeyword() const {100 return getKind() > Kind::keyword_start && getKind() < Kind::keyword_end;101 }102 103private:104 /// Discriminator that indicates the kind of token this is.105 Kind kind;106 107 /// A reference to the entire token contents; this is always a pointer into108 /// a memory buffer owned by the source manager.109 StringRef spelling;110};111 112//===----------------------------------------------------------------------===//113// FormatLexer114//===----------------------------------------------------------------------===//115 116/// This class implements a simple lexer for operation assembly format strings.117class FormatLexer {118public:119 FormatLexer(llvm::SourceMgr &mgr, SMLoc loc);120 121 /// Lex the next token and return it.122 FormatToken lexToken();123 124 /// Emit an error to the lexer with the given location and message.125 FormatToken emitError(SMLoc loc, const Twine &msg);126 FormatToken emitError(const char *loc, const Twine &msg);127 128 FormatToken emitErrorAndNote(SMLoc loc, const Twine &msg, const Twine ¬e);129 130private:131 /// Return the next character in the stream.132 int getNextChar();133 134 /// Lex an identifier, literal, variable, or string.135 FormatToken lexIdentifier(const char *tokStart);136 FormatToken lexLiteral(const char *tokStart);137 FormatToken lexVariable(const char *tokStart);138 FormatToken lexString(const char *tokStart);139 140 /// Create a token with the current pointer and a start pointer.141 FormatToken formToken(FormatToken::Kind kind, const char *tokStart) {142 return FormatToken(kind, StringRef(tokStart, curPtr - tokStart));143 }144 145 /// The source manager containing the format string.146 llvm::SourceMgr &mgr;147 /// Location of the format string.148 SMLoc loc;149 /// Buffer containing the format string.150 StringRef curBuffer;151 /// Current pointer in the buffer.152 const char *curPtr;153};154 155//===----------------------------------------------------------------------===//156// FormatElement157//===----------------------------------------------------------------------===//158 159/// This class represents a single format element.160///161/// If you squint and take a close look, you can see the outline of a `Format`162/// dialect.163class FormatElement {164public:165 virtual ~FormatElement();166 167 // The top-level kinds of format elements.168 enum Kind { Literal, String, Variable, Whitespace, Directive, Optional };169 170 /// Support LLVM-style RTTI.171 static bool classof(const FormatElement *el) { return true; }172 173 /// Get the element kind.174 Kind getKind() const { return kind; }175 176protected:177 /// Create a format element with the given kind.178 FormatElement(Kind kind) : kind(kind) {}179 180private:181 /// The kind of the element.182 Kind kind;183};184 185/// The base class for all format elements. This class implements common methods186/// for LLVM-style RTTI.187template <FormatElement::Kind ElementKind>188class FormatElementBase : public FormatElement {189public:190 /// Support LLVM-style RTTI.191 static bool classof(const FormatElement *el) {192 return ElementKind == el->getKind();193 }194 195protected:196 /// Create a format element with the given kind.197 FormatElementBase() : FormatElement(ElementKind) {}198};199 200/// This class represents a literal element. A literal is either one of the201/// supported punctuation characters (e.g. `(` or `,`) or a string literal (e.g.202/// `literal`).203class LiteralElement : public FormatElementBase<FormatElement::Literal> {204public:205 /// Create a literal element with the given spelling.206 explicit LiteralElement(StringRef spelling) : spelling(spelling) {}207 208 /// Get the spelling of the literal.209 StringRef getSpelling() const { return spelling; }210 211private:212 /// The spelling of the variable, i.e. the string contained within the213 /// backticks.214 StringRef spelling;215};216 217/// This class represents a raw string that can contain arbitrary C++ code.218class StringElement : public FormatElementBase<FormatElement::String> {219public:220 /// Create a string element with the given contents.221 explicit StringElement(std::string value) : value(std::move(value)) {}222 223 /// Get the value of the string element.224 StringRef getValue() const { return value; }225 226private:227 /// The contents of the string.228 std::string value;229};230 231/// This class represents a variable element. A variable refers to some part of232/// the object being parsed, e.g. an attribute or operand on an operation or a233/// parameter on an attribute.234class VariableElement : public FormatElementBase<FormatElement::Variable> {235public:236 /// These are the kinds of variables.237 enum Kind {238 Attribute,239 Operand,240 Region,241 Result,242 Successor,243 Parameter,244 Property245 };246 247 /// Get the kind of variable.248 Kind getKind() const { return kind; }249 250protected:251 /// Create a variable with a kind.252 VariableElement(Kind kind) : kind(kind) {}253 254private:255 /// The kind of variable.256 Kind kind;257};258 259/// Base class for variable elements. This class implements common methods for260/// LLVM-style RTTI.261template <VariableElement::Kind VariableKind>262class VariableElementBase : public VariableElement {263public:264 /// An element is of this class if it is a variable and has the same variable265 /// type.266 static bool classof(const FormatElement *el) {267 if (auto *varEl = dyn_cast<VariableElement>(el))268 return VariableKind == varEl->getKind();269 return false;270 }271 272protected:273 /// Create a variable element with the given variable kind.274 VariableElementBase() : VariableElement(VariableKind) {}275};276 277/// This class represents a whitespace element, e.g. a newline or space. It is a278/// literal that is printed but never parsed. When the value is empty, i.e. ``,279/// a space is elided where one would have been printed automatically.280class WhitespaceElement : public FormatElementBase<FormatElement::Whitespace> {281public:282 /// Create a whitespace element.283 explicit WhitespaceElement(StringRef value) : value(value) {}284 285 /// Get the whitespace value.286 StringRef getValue() const { return value; }287 288private:289 /// The value of the whitespace element. Can be empty.290 StringRef value;291};292 293class DirectiveElement : public FormatElementBase<FormatElement::Directive> {294public:295 /// These are the kinds of directives.296 enum Kind {297 AttrDict,298 PropDict,299 Custom,300 FunctionalType,301 OIList,302 Operands,303 Ref,304 Regions,305 Results,306 Successors,307 Type,308 Params,309 Struct310 };311 312 /// Get the directive kind.313 Kind getKind() const { return kind; }314 315protected:316 /// Create a directive element with a kind.317 DirectiveElement(Kind kind) : kind(kind) {}318 319private:320 /// The directive kind.321 Kind kind;322};323 324/// Base class for directive elements. This class implements common methods for325/// LLVM-style RTTI.326template <DirectiveElement::Kind DirectiveKind>327class DirectiveElementBase : public DirectiveElement {328public:329 /// Create a directive element with the specified kind.330 DirectiveElementBase() : DirectiveElement(DirectiveKind) {}331 332 /// A format element is of this class if it is a directive element and has the333 /// same kind.334 static bool classof(const FormatElement *el) {335 if (auto *directiveEl = dyn_cast<DirectiveElement>(el))336 return DirectiveKind == directiveEl->getKind();337 return false;338 }339};340 341/// Base class for a directive that contains references to elements of type `T`342/// in a vector.343template <DirectiveElement::Kind DirectiveKind, typename T>344class VectorDirectiveBase : public DirectiveElementBase<DirectiveKind> {345public:346 using Base = VectorDirectiveBase<DirectiveKind, T>;347 348 VectorDirectiveBase(std::vector<T> &&elems) : elems(std::move(elems)) {}349 350 /// Get the elements contained in this directive.351 ArrayRef<T> getElements() const { return elems; }352 353 /// Get the number of elements.354 unsigned getNumElements() const { return elems.size(); }355 356 /// Take all of the elements from this directive.357 std::vector<T> takeElements() { return std::move(elems); }358 359protected:360 /// The elements captured by this directive.361 std::vector<T> elems;362};363 364/// This class represents a custom format directive that is implemented by the365/// user in C++. The directive accepts a list of arguments that is passed to the366/// C++ function.367class CustomDirective368 : public VectorDirectiveBase<DirectiveElement::Custom, FormatElement *> {369public:370 using Base::Base;371 /// Create a custom directive with a name and list of arguments.372 CustomDirective(StringRef name, std::vector<FormatElement *> &&arguments)373 : Base(std::move(arguments)), name(name) {}374 375 /// Get the custom directive name.376 StringRef getName() const { return name; }377 378 template <typename T>379 FailureOr<T *> getFrontAs() const {380 if (getNumElements() != 1)381 return failure();382 if (T *elem = dyn_cast<T>(getElements()[0]))383 return elem;384 return failure();385 }386 387private:388 /// The name of the custom directive. The name is used to call two C++389 /// methods: `parse{name}` and `print{name}` with the given arguments.390 StringRef name;391};392 393/// This class represents a reference directive. This directive can be used to394/// reference but not bind a previously bound variable or format object. Its395/// current only use is to pass variables as arguments to the custom directive.396class RefDirective : public DirectiveElementBase<DirectiveElement::Ref> {397public:398 /// Create a reference directive with the single referenced child.399 RefDirective(FormatElement *arg) : arg(arg) {}400 401 /// Get the reference argument.402 FormatElement *getArg() const { return arg; }403 404private:405 /// The referenced argument.406 FormatElement *arg;407};408 409/// This class represents a group of elements that are optionally emitted based410/// on an optional variable "anchor" and a group of elements that are emitted411/// when the anchor element is not present.412class OptionalElement : public FormatElementBase<FormatElement::Optional> {413public:414 /// Create an optional group with the given child elements.415 OptionalElement(std::vector<FormatElement *> &&thenElements,416 std::vector<FormatElement *> &&elseElements,417 unsigned thenParseStart, unsigned elseParseStart,418 FormatElement *anchor, bool inverted)419 : thenElements(std::move(thenElements)),420 elseElements(std::move(elseElements)), thenParseStart(thenParseStart),421 elseParseStart(elseParseStart), anchor(anchor), inverted(inverted) {}422 423 /// Return the `then` elements of the optional group. Drops the first424 /// `thenParseStart` whitespace elements if `parseable` is true.425 ArrayRef<FormatElement *> getThenElements(bool parseable = false) const {426 return llvm::ArrayRef(thenElements)427 .drop_front(parseable ? thenParseStart : 0);428 }429 430 /// Return the `else` elements of the optional group. Drops the first431 /// `elseParseStart` whitespace elements if `parseable` is true.432 ArrayRef<FormatElement *> getElseElements(bool parseable = false) const {433 return llvm::ArrayRef(elseElements)434 .drop_front(parseable ? elseParseStart : 0);435 }436 437 /// Return the anchor of the optional group.438 FormatElement *getAnchor() const { return anchor; }439 440 /// Return true if the optional group is inverted.441 bool isInverted() const { return inverted; }442 443private:444 /// The child elements emitted when the anchor is present.445 std::vector<FormatElement *> thenElements;446 /// The child elements emitted when the anchor is not present.447 std::vector<FormatElement *> elseElements;448 /// The index of the first element that is parsed in `thenElements`. That is,449 /// the first non-whitespace element.450 unsigned thenParseStart;451 /// The index of the first element that is parsed in `elseElements`. That is,452 /// the first non-whitespace element.453 unsigned elseParseStart;454 /// The anchor element of the optional group.455 FormatElement *anchor;456 /// Whether the optional group condition is inverted and the anchor element is457 /// in the else group.458 bool inverted;459};460 461//===----------------------------------------------------------------------===//462// FormatParserBase463//===----------------------------------------------------------------------===//464 465/// Base class for a parser that implements an assembly format. This class466/// defines a common assembly format syntax and the creation of format elements.467/// Subclasses will need to implement parsing for the format elements they468/// support.469class FormatParser {470public:471 /// Vtable anchor.472 virtual ~FormatParser();473 474 /// Parse the assembly format.475 FailureOr<std::vector<FormatElement *>> parse();476 477protected:478 /// The current context of the parser when parsing an element.479 enum Context {480 /// The element is being parsed in a "top-level" context, i.e. at the top of481 /// the format or in an optional group.482 TopLevelContext,483 /// The element is being parsed as a custom directive child.484 CustomDirectiveContext,485 /// The element is being parsed as a type directive child.486 TypeDirectiveContext,487 /// The element is being parsed as a reference directive child.488 RefDirectiveContext,489 /// The element is being parsed as a struct directive child.490 StructDirectiveContext491 };492 493 /// Create a format parser with the given source manager and a location.494 explicit FormatParser(llvm::SourceMgr &mgr, llvm::SMLoc loc)495 : lexer(mgr, loc), curToken(lexer.lexToken()) {}496 497 /// Allocate and construct a format element.498 template <typename FormatElementT, typename... Args>499 FormatElementT *create(Args &&...args) {500 // FormatElementT *ptr = allocator.Allocate<FormatElementT>();501 // ::new (ptr) FormatElementT(std::forward<Args>(args)...);502 // return ptr;503 auto mem = std::make_unique<FormatElementT>(std::forward<Args>(args)...);504 FormatElementT *ptr = mem.get();505 allocator.push_back(std::move(mem));506 return ptr;507 }508 509 //===--------------------------------------------------------------------===//510 // Element Parsing511 512 /// Parse a single element of any kind.513 FailureOr<FormatElement *> parseElement(Context ctx);514 /// Parse a literal.515 FailureOr<FormatElement *> parseLiteral(Context ctx);516 /// Parse a string.517 FailureOr<FormatElement *> parseString(Context ctx);518 /// Parse a variable.519 FailureOr<FormatElement *> parseVariable(Context ctx);520 /// Parse a directive.521 FailureOr<FormatElement *> parseDirective(Context ctx);522 /// Parse an optional group.523 FailureOr<FormatElement *> parseOptionalGroup(Context ctx);524 /// Parse a custom directive.525 FailureOr<FormatElement *> parseCustomDirective(llvm::SMLoc loc, Context ctx);526 /// Parse a ref directive.527 FailureOr<FormatElement *> parseRefDirective(SMLoc loc, Context context);528 /// Parse a qualified directive.529 FailureOr<FormatElement *> parseQualifiedDirective(SMLoc loc, Context ctx);530 531 /// Parse a format-specific variable kind.532 virtual FailureOr<FormatElement *>533 parseVariableImpl(llvm::SMLoc loc, StringRef name, Context ctx) = 0;534 /// Parse a format-specific directive kind.535 virtual FailureOr<FormatElement *>536 parseDirectiveImpl(llvm::SMLoc loc, FormatToken::Kind kind, Context ctx) = 0;537 538 //===--------------------------------------------------------------------===//539 // Format Verification540 541 /// Verify that the format is well-formed.542 virtual LogicalResult verify(llvm::SMLoc loc,543 ArrayRef<FormatElement *> elements) = 0;544 /// Verify the arguments to a custom directive.545 virtual LogicalResult546 verifyCustomDirectiveArguments(llvm::SMLoc loc,547 ArrayRef<FormatElement *> arguments) = 0;548 /// Verify the elements of an optional group.549 virtual LogicalResult550 verifyOptionalGroupElements(llvm::SMLoc loc,551 ArrayRef<FormatElement *> elements,552 FormatElement *anchor) = 0;553 554 /// Mark 'element' as qualified. If 'element' cannot be qualified an error555 /// should be emitted and failure returned.556 virtual LogicalResult markQualified(llvm::SMLoc loc,557 FormatElement *element) = 0;558 559 //===--------------------------------------------------------------------===//560 // Lexer Utilities561 562 /// Emit an error at the given location.563 LogicalResult emitError(llvm::SMLoc loc, const Twine &msg) {564 lexer.emitError(loc, msg);565 return failure();566 }567 568 /// Emit an error and a note at the given notation.569 LogicalResult emitErrorAndNote(llvm::SMLoc loc, const Twine &msg,570 const Twine ¬e) {571 lexer.emitErrorAndNote(loc, msg, note);572 return failure();573 }574 575 /// Parse a single token of the expected kind.576 FailureOr<FormatToken> parseToken(FormatToken::Kind kind, const Twine &msg) {577 if (!curToken.is(kind))578 return emitError(curToken.getLoc(), msg);579 FormatToken tok = curToken;580 consumeToken();581 return tok;582 }583 584 /// Advance the lexer to the next token.585 void consumeToken() {586 assert(!curToken.is(FormatToken::eof) && !curToken.is(FormatToken::error) &&587 "shouldn't advance past EOF or errors");588 curToken = lexer.lexToken();589 }590 591 /// Get the current token.592 FormatToken peekToken() { return curToken; }593 594private:595 /// The format parser retains ownership of the format elements in a bump596 /// pointer allocator.597 // FIXME: FormatElement with `std::vector` need to be converted to use598 // trailing objects.599 // llvm::BumpPtrAllocator allocator;600 std::vector<std::unique_ptr<FormatElement>> allocator;601 /// The format lexer to use.602 FormatLexer lexer;603 /// The current token in the lexer.604 FormatToken curToken;605};606 607//===----------------------------------------------------------------------===//608// Utility Functions609//===----------------------------------------------------------------------===//610 611/// Whether a space needs to be emitted before a literal. E.g., two keywords612/// back-to-back require a space separator, but a keyword followed by '<' does613/// not require a space.614bool shouldEmitSpaceBefore(StringRef value, bool lastWasPunctuation);615 616/// Returns true if the given string can be formatted as a keyword.617bool canFormatStringAsKeyword(StringRef value,618 function_ref<void(Twine)> emitError = nullptr);619 620/// Returns true if the given string is valid format literal element.621/// If `emitError` is provided, it is invoked with the reason for the failure.622bool isValidLiteral(StringRef value,623 function_ref<void(Twine)> emitError = nullptr);624 625/// Whether a failure in parsing the assembly format should be a fatal error.626extern llvm::cl::opt<bool> formatErrorIsFatal;627 628} // namespace tblgen629} // namespace mlir630 631#endif // MLIR_TOOLS_MLIRTBLGEN_FORMATGEN_H_632