140 lines · c
1//===- Token.h - MLIR Token Interface ---------------------------*- C++ -*-===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8 9#ifndef MLIR_LIB_PARSER_TOKEN_H10#define MLIR_LIB_PARSER_TOKEN_H11 12#include "mlir/Support/LLVM.h"13#include "llvm/ADT/StringRef.h"14#include "llvm/Support/SMLoc.h"15#include <optional>16 17namespace mlir {18 19/// This represents a token in the MLIR syntax.20class Token {21public:22 enum Kind {23#define TOK_MARKER(NAME) NAME,24#define TOK_IDENTIFIER(NAME) NAME,25#define TOK_LITERAL(NAME) NAME,26#define TOK_PUNCTUATION(NAME, SPELLING) NAME,27#define TOK_KEYWORD(SPELLING) kw_##SPELLING,28#include "TokenKinds.def"29 };30 31 Token(Kind kind, StringRef spelling) : kind(kind), spelling(spelling) {}32 33 // Return the bytes that make up this token.34 StringRef getSpelling() const { return spelling; }35 36 // Token classification.37 Kind getKind() const { return kind; }38 bool is(Kind k) const { return kind == k; }39 40 bool isAny(Kind k1, Kind k2) const { return is(k1) || is(k2); }41 42 /// Return true if this token is one of the specified kinds.43 template <typename... T>44 bool isAny(Kind k1, Kind k2, Kind k3, T... others) const {45 if (is(k1))46 return true;47 return isAny(k2, k3, others...);48 }49 50 bool isNot(Kind k) const { return kind != k; }51 52 /// Return true if this token isn't one of the specified kinds.53 template <typename... T>54 bool isNot(Kind k1, Kind k2, T... others) const {55 return !isAny(k1, k2, others...);56 }57 58 /// Return true if this is one of the keyword token kinds (e.g. kw_if).59 bool isKeyword() const;60 61 /// Returns true if the current token represents a code completion.62 bool isCodeCompletion() const { return is(code_complete); }63 64 /// Returns true if the current token represents a code completion for the65 /// "normal" token type.66 bool isCodeCompletionFor(Kind kind) const;67 68 /// Returns true if the current token is the given type, or represents a code69 /// completion for that type.70 bool isOrIsCodeCompletionFor(Kind kind) const {71 return is(kind) || isCodeCompletionFor(kind);72 }73 74 // Helpers to decode specific sorts of tokens.75 76 /// For an integer token, return its value as an unsigned. If it doesn't fit,77 /// return std::nullopt.78 std::optional<unsigned> getUnsignedIntegerValue() const;79 80 /// For an integer token, return its value as an uint64_t. If it doesn't fit,81 /// return std::nullopt.82 static std::optional<uint64_t> getUInt64IntegerValue(StringRef spelling);83 std::optional<uint64_t> getUInt64IntegerValue() const {84 return getUInt64IntegerValue(getSpelling());85 }86 87 /// For a floatliteral token, return its value as a double. Returns88 /// std::nullopt in the case of underflow or overflow.89 std::optional<double> getFloatingPointValue() const;90 91 /// For an inttype token, return its bitwidth.92 std::optional<unsigned> getIntTypeBitwidth() const;93 94 /// For an inttype token, return its signedness semantics: std::nullopt means95 /// no signedness semantics; true means signed integer type; false means96 /// unsigned integer type.97 std::optional<bool> getIntTypeSignedness() const;98 99 /// Given a hash_identifier token like #123, try to parse the number out of100 /// the identifier, returning std::nullopt if it is a named identifier like #x101 /// or if the integer doesn't fit.102 std::optional<unsigned> getHashIdentifierNumber() const;103 104 /// Given a token containing a string literal, return its value, including105 /// removing the quote characters and unescaping the contents of the string.106 std::string getStringValue() const;107 108 /// Given a token containing a hex string literal, return its value or109 /// std::nullopt if the token does not contain a valid hex string. A hex110 /// string literal is a string starting with `0x` and only containing hex111 /// digits.112 std::optional<std::string> getHexStringValue() const;113 114 /// Given a token containing a symbol reference, return the unescaped string115 /// value.116 std::string getSymbolReference() const;117 118 // Location processing.119 SMLoc getLoc() const;120 SMLoc getEndLoc() const;121 SMRange getLocRange() const;122 123 /// Given a punctuation or keyword token kind, return the spelling of the124 /// token as a string. Warning: This will abort on markers, identifiers and125 /// literal tokens since they have no fixed spelling.126 static StringRef getTokenSpelling(Kind kind);127 128private:129 /// Discriminator that indicates the sort of token this is.130 Kind kind;131 132 /// A reference to the entire token contents; this is always a pointer into133 /// a memory buffer owned by the source manager.134 StringRef spelling;135};136 137} // namespace mlir138 139#endif // MLIR_LIB_PARSER_TOKEN_H140