1102 lines · cpp
1//===- TGLexer.cpp - Lexer for TableGen -----------------------------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// Implement the Lexer for TableGen.10//11//===----------------------------------------------------------------------===//12 13#include "TGLexer.h"14#include "llvm/ADT/ArrayRef.h"15#include "llvm/ADT/StringExtras.h"16#include "llvm/ADT/StringSwitch.h"17#include "llvm/ADT/Twine.h"18#include "llvm/Config/config.h" // for strtoull()/strtoll() define19#include "llvm/Support/Compiler.h"20#include "llvm/Support/MemoryBuffer.h"21#include "llvm/Support/SourceMgr.h"22#include "llvm/TableGen/Error.h"23#include <cerrno>24#include <cstdio>25#include <cstdlib>26#include <cstring>27 28using namespace llvm;29 30namespace {31// A list of supported preprocessing directives with their32// internal token kinds and names.33struct PreprocessorDir {34 tgtok::TokKind Kind;35 StringRef Word;36};37} // end anonymous namespace38 39/// Returns true if `C` is a valid character in an identifier. If `First` is40/// true, returns true if `C` is a valid first character of an identifier,41/// else returns true if `C` is a valid non-first character of an identifier.42/// Identifiers match the following regular expression:43/// [a-zA-Z_][0-9a-zA-Z_]*44static bool isValidIDChar(char C, bool First) {45 if (C == '_' || isAlpha(C))46 return true;47 return !First && isDigit(C);48}49 50constexpr PreprocessorDir PreprocessorDirs[] = {{tgtok::Ifdef, "ifdef"},51 {tgtok::Ifndef, "ifndef"},52 {tgtok::Else, "else"},53 {tgtok::Endif, "endif"},54 {tgtok::Define, "define"}};55 56// Returns a pointer past the end of a valid macro name at the start of `Str`.57// Valid macro names match the regular expression [a-zA-Z_][0-9a-zA-Z_]*.58static const char *lexMacroName(StringRef Str) {59 assert(!Str.empty());60 61 // Macro names start with [a-zA-Z_].62 const char *Next = Str.begin();63 if (!isValidIDChar(*Next, /*First=*/true))64 return Next;65 // Eat the first character of the name.66 ++Next;67 68 // Match the rest of the identifier regex: [0-9a-zA-Z_]*69 const char *End = Str.end();70 while (Next != End && isValidIDChar(*Next, /*First=*/false))71 ++Next;72 return Next;73}74 75TGLexer::TGLexer(SourceMgr &SM, ArrayRef<std::string> Macros) : SrcMgr(SM) {76 CurBuffer = SrcMgr.getMainFileID();77 CurBuf = SrcMgr.getMemoryBuffer(CurBuffer)->getBuffer();78 CurPtr = CurBuf.begin();79 TokStart = nullptr;80 81 // Pretend that we enter the "top-level" include file.82 PrepIncludeStack.emplace_back();83 84 // Add all macros defined on the command line to the DefinedMacros set.85 // Check invalid macro names and print fatal error if we find one.86 for (StringRef MacroName : Macros) {87 const char *End = lexMacroName(MacroName);88 if (End != MacroName.end())89 PrintFatalError("invalid macro name `" + MacroName +90 "` specified on command line");91 92 DefinedMacros.insert(MacroName);93 }94}95 96SMLoc TGLexer::getLoc() const { return SMLoc::getFromPointer(TokStart); }97 98SMRange TGLexer::getLocRange() const {99 return {getLoc(), SMLoc::getFromPointer(CurPtr)};100}101 102/// ReturnError - Set the error to the specified string at the specified103/// location. This is defined to always return tgtok::Error.104tgtok::TokKind TGLexer::ReturnError(SMLoc Loc, const Twine &Msg) {105 PrintError(Loc, Msg);106 return tgtok::Error;107}108 109tgtok::TokKind TGLexer::ReturnError(const char *Loc, const Twine &Msg) {110 return ReturnError(SMLoc::getFromPointer(Loc), Msg);111}112 113bool TGLexer::processEOF() {114 SMLoc ParentIncludeLoc = SrcMgr.getParentIncludeLoc(CurBuffer);115 if (ParentIncludeLoc != SMLoc()) {116 // If prepExitInclude() detects a problem with the preprocessing117 // control stack, it will return false. Pretend that we reached118 // the final EOF and stop lexing more tokens by returning false119 // to LexToken().120 if (!prepExitInclude(false))121 return false;122 123 CurBuffer = SrcMgr.FindBufferContainingLoc(ParentIncludeLoc);124 CurBuf = SrcMgr.getMemoryBuffer(CurBuffer)->getBuffer();125 CurPtr = ParentIncludeLoc.getPointer();126 // Make sure TokStart points into the parent file's buffer.127 // LexToken() assigns to it before calling getNextChar(),128 // so it is pointing into the included file now.129 TokStart = CurPtr;130 return true;131 }132 133 // Pretend that we exit the "top-level" include file.134 // Note that in case of an error (e.g. control stack imbalance)135 // the routine will issue a fatal error.136 prepExitInclude(true);137 return false;138}139 140int TGLexer::getNextChar() {141 char CurChar = *CurPtr++;142 switch (CurChar) {143 default:144 return (unsigned char)CurChar;145 146 case 0: {147 // A NUL character in the stream is either the end of the current buffer or148 // a spurious NUL in the file. Disambiguate that here.149 if (CurPtr - 1 == CurBuf.end()) {150 --CurPtr; // Arrange for another call to return EOF again.151 return EOF;152 }153 PrintError(getLoc(),154 "NUL character is invalid in source; treated as space");155 return ' ';156 }157 158 case '\n':159 case '\r':160 // Handle the newline character by ignoring it and incrementing the line161 // count. However, be careful about 'dos style' files with \n\r in them.162 // Only treat a \n\r or \r\n as a single line.163 if ((*CurPtr == '\n' || (*CurPtr == '\r')) && *CurPtr != CurChar)164 ++CurPtr; // Eat the two char newline sequence.165 return '\n';166 }167}168 169int TGLexer::peekNextChar(int Index) const { return *(CurPtr + Index); }170 171tgtok::TokKind TGLexer::LexToken(bool FileOrLineStart) {172 while (true) {173 TokStart = CurPtr;174 // This always consumes at least one character.175 int CurChar = getNextChar();176 177 switch (CurChar) {178 default:179 // Handle letters: [a-zA-Z_]180 if (isValidIDChar(CurChar, /*First=*/true))181 return LexIdentifier();182 183 // Unknown character, emit an error.184 return ReturnError(TokStart, "unexpected character");185 case EOF:186 // Lex next token, if we just left an include file.187 if (processEOF()) {188 // Leaving an include file means that the next symbol is located at the189 // end of the 'include "..."' construct.190 FileOrLineStart = false;191 break;192 }193 194 // Return EOF denoting the end of lexing.195 return tgtok::Eof;196 197 case ':':198 return tgtok::colon;199 case ';':200 return tgtok::semi;201 case ',':202 return tgtok::comma;203 case '<':204 return tgtok::less;205 case '>':206 return tgtok::greater;207 case ']':208 return tgtok::r_square;209 case '{':210 return tgtok::l_brace;211 case '}':212 return tgtok::r_brace;213 case '(':214 return tgtok::l_paren;215 case ')':216 return tgtok::r_paren;217 case '=':218 return tgtok::equal;219 case '?':220 return tgtok::question;221 case '#':222 if (FileOrLineStart) {223 tgtok::TokKind Kind = prepIsDirective();224 if (Kind != tgtok::Error)225 return lexPreprocessor(Kind);226 }227 228 return tgtok::paste;229 230 // The period is a separate case so we can recognize the "..."231 // range punctuator.232 case '.':233 if (peekNextChar(0) == '.') {234 ++CurPtr; // Eat second dot.235 if (peekNextChar(0) == '.') {236 ++CurPtr; // Eat third dot.237 return tgtok::dotdotdot;238 }239 return ReturnError(TokStart, "invalid '..' punctuation");240 }241 return tgtok::dot;242 243 case '\r':244 llvm_unreachable("getNextChar() must never return '\r'");245 246 case ' ':247 case '\t':248 // Ignore whitespace.249 break;250 case '\n':251 // Ignore whitespace, and identify the new line.252 FileOrLineStart = true;253 break;254 case '/':255 // If this is the start of a // comment, skip until the end of the line or256 // the end of the buffer.257 if (*CurPtr == '/')258 SkipBCPLComment();259 else if (*CurPtr == '*') {260 if (SkipCComment())261 return tgtok::Error;262 } else // Otherwise, this is an error.263 return ReturnError(TokStart, "unexpected character");264 break;265 case '-':266 case '+':267 case '0':268 case '1':269 case '2':270 case '3':271 case '4':272 case '5':273 case '6':274 case '7':275 case '8':276 case '9': {277 int NextChar = 0;278 if (isDigit(CurChar)) {279 // Allow identifiers to start with a number if it is followed by280 // an identifier. This can happen with paste operations like281 // foo#8i.282 int i = 0;283 do {284 NextChar = peekNextChar(i++);285 } while (isDigit(NextChar));286 287 if (NextChar == 'x' || NextChar == 'b') {288 // If this is [0-9]b[01] or [0-9]x[0-9A-fa-f] this is most289 // likely a number.290 int NextNextChar = peekNextChar(i);291 switch (NextNextChar) {292 default:293 break;294 case '0':295 case '1':296 if (NextChar == 'b')297 return LexNumber();298 [[fallthrough]];299 case '2':300 case '3':301 case '4':302 case '5':303 case '6':304 case '7':305 case '8':306 case '9':307 case 'a':308 case 'b':309 case 'c':310 case 'd':311 case 'e':312 case 'f':313 case 'A':314 case 'B':315 case 'C':316 case 'D':317 case 'E':318 case 'F':319 if (NextChar == 'x')320 return LexNumber();321 break;322 }323 }324 }325 326 if (isValidIDChar(NextChar, /*First=*/true))327 return LexIdentifier();328 329 return LexNumber();330 }331 case '"':332 return LexString();333 case '$':334 return LexVarName();335 case '[':336 return LexBracket();337 case '!':338 return LexExclaim();339 }340 }341}342 343/// LexString - Lex "[^"]*"344tgtok::TokKind TGLexer::LexString() {345 const char *StrStart = CurPtr;346 347 CurStrVal = "";348 349 while (*CurPtr != '"') {350 // If we hit the end of the buffer, report an error.351 if (*CurPtr == 0 && CurPtr == CurBuf.end())352 return ReturnError(StrStart, "end of file in string literal");353 354 if (*CurPtr == '\n' || *CurPtr == '\r')355 return ReturnError(StrStart, "end of line in string literal");356 357 if (*CurPtr != '\\') {358 CurStrVal += *CurPtr++;359 continue;360 }361 362 ++CurPtr;363 364 switch (*CurPtr) {365 case '\\':366 case '\'':367 case '"':368 // These turn into their literal character.369 CurStrVal += *CurPtr++;370 break;371 case 't':372 CurStrVal += '\t';373 ++CurPtr;374 break;375 case 'n':376 CurStrVal += '\n';377 ++CurPtr;378 break;379 380 case '\n':381 case '\r':382 return ReturnError(CurPtr, "escaped newlines not supported in tblgen");383 384 // If we hit the end of the buffer, report an error.385 case '\0':386 if (CurPtr == CurBuf.end())387 return ReturnError(StrStart, "end of file in string literal");388 [[fallthrough]];389 default:390 return ReturnError(CurPtr, "invalid escape in string literal");391 }392 }393 394 ++CurPtr;395 return tgtok::StrVal;396}397 398tgtok::TokKind TGLexer::LexVarName() {399 if (!isValidIDChar(CurPtr[0], /*First=*/true))400 return ReturnError(TokStart, "invalid variable name");401 402 // Otherwise, we're ok, consume the rest of the characters.403 const char *VarNameStart = CurPtr++;404 405 while (isValidIDChar(*CurPtr, /*First=*/false))406 ++CurPtr;407 408 CurStrVal.assign(VarNameStart, CurPtr);409 return tgtok::VarName;410}411 412tgtok::TokKind TGLexer::LexIdentifier() {413 // The first letter is [a-zA-Z_].414 const char *IdentStart = TokStart;415 416 // Match the rest of the identifier regex: [0-9a-zA-Z_]*417 while (isValidIDChar(*CurPtr, /*First=*/false))418 ++CurPtr;419 420 // Check to see if this identifier is a reserved keyword.421 StringRef Str(IdentStart, CurPtr - IdentStart);422 423 tgtok::TokKind Kind = StringSwitch<tgtok::TokKind>(Str)424 .Case("int", tgtok::Int)425 .Case("bit", tgtok::Bit)426 .Case("bits", tgtok::Bits)427 .Case("string", tgtok::String)428 .Case("list", tgtok::List)429 .Case("code", tgtok::Code)430 .Case("dag", tgtok::Dag)431 .Case("class", tgtok::Class)432 .Case("def", tgtok::Def)433 .Case("true", tgtok::TrueVal)434 .Case("false", tgtok::FalseVal)435 .Case("foreach", tgtok::Foreach)436 .Case("defm", tgtok::Defm)437 .Case("defset", tgtok::Defset)438 .Case("deftype", tgtok::Deftype)439 .Case("multiclass", tgtok::MultiClass)440 .Case("field", tgtok::Field)441 .Case("let", tgtok::Let)442 .Case("in", tgtok::In)443 .Case("defvar", tgtok::Defvar)444 .Case("include", tgtok::Include)445 .Case("if", tgtok::If)446 .Case("then", tgtok::Then)447 .Case("else", tgtok::ElseKW)448 .Case("assert", tgtok::Assert)449 .Case("dump", tgtok::Dump)450 .Default(tgtok::Id);451 452 // A couple of tokens require special processing.453 switch (Kind) {454 case tgtok::Include:455 if (LexInclude())456 return tgtok::Error;457 return Lex();458 case tgtok::Id:459 CurStrVal.assign(Str.begin(), Str.end());460 break;461 default:462 break;463 }464 465 return Kind;466}467 468/// LexInclude - We just read the "include" token. Get the string token that469/// comes next and enter the include.470bool TGLexer::LexInclude() {471 // The token after the include must be a string.472 tgtok::TokKind Tok = LexToken();473 if (Tok == tgtok::Error)474 return true;475 if (Tok != tgtok::StrVal) {476 PrintError(getLoc(), "expected filename after include");477 return true;478 }479 480 // Get the string.481 std::string Filename = CurStrVal;482 std::string IncludedFile;483 484 CurBuffer = SrcMgr.AddIncludeFile(Filename, SMLoc::getFromPointer(CurPtr),485 IncludedFile);486 if (!CurBuffer) {487 PrintError(getLoc(), "could not find include file '" + Filename + "'");488 return true;489 }490 491 Dependencies.insert(IncludedFile);492 // Save the line number and lex buffer of the includer.493 CurBuf = SrcMgr.getMemoryBuffer(CurBuffer)->getBuffer();494 CurPtr = CurBuf.begin();495 496 PrepIncludeStack.emplace_back();497 return false;498}499 500/// SkipBCPLComment - Skip over the comment by finding the next CR or LF.501/// Or we may end up at the end of the buffer.502void TGLexer::SkipBCPLComment() {503 ++CurPtr; // Skip the second slash.504 auto EOLPos = CurBuf.find_first_of("\r\n", CurPtr - CurBuf.data());505 CurPtr = (EOLPos == StringRef::npos) ? CurBuf.end() : CurBuf.data() + EOLPos;506}507 508/// SkipCComment - This skips C-style /**/ comments. The only difference from C509/// is that we allow nesting.510bool TGLexer::SkipCComment() {511 ++CurPtr; // Skip the star.512 unsigned CommentDepth = 1;513 514 while (true) {515 int CurChar = getNextChar();516 switch (CurChar) {517 case EOF:518 PrintError(TokStart, "unterminated comment");519 return true;520 case '*':521 // End of the comment?522 if (CurPtr[0] != '/')523 break;524 525 ++CurPtr; // End the */.526 if (--CommentDepth == 0)527 return false;528 break;529 case '/':530 // Start of a nested comment?531 if (CurPtr[0] != '*')532 break;533 ++CurPtr;534 ++CommentDepth;535 break;536 }537 }538}539 540/// LexNumber - Lex:541/// [-+]?[0-9]+542/// 0x[0-9a-fA-F]+543/// 0b[01]+544tgtok::TokKind TGLexer::LexNumber() {545 unsigned Base = 0;546 const char *NumStart;547 548 // Check if it's a hex or a binary value.549 if (CurPtr[-1] == '0') {550 NumStart = CurPtr + 1;551 if (CurPtr[0] == 'x') {552 Base = 16;553 do554 ++CurPtr;555 while (isHexDigit(CurPtr[0]));556 } else if (CurPtr[0] == 'b') {557 Base = 2;558 do559 ++CurPtr;560 while (CurPtr[0] == '0' || CurPtr[0] == '1');561 }562 }563 564 // For a hex or binary value, we always convert it to an unsigned value.565 bool IsMinus = false;566 567 // Check if it's a decimal value.568 if (Base == 0) {569 // Check for a sign without a digit.570 if (!isDigit(CurPtr[0])) {571 if (CurPtr[-1] == '-')572 return tgtok::minus;573 else if (CurPtr[-1] == '+')574 return tgtok::plus;575 }576 577 Base = 10;578 NumStart = TokStart;579 IsMinus = CurPtr[-1] == '-';580 581 while (isDigit(CurPtr[0]))582 ++CurPtr;583 }584 585 // Requires at least one digit.586 if (CurPtr == NumStart)587 return ReturnError(TokStart, "invalid number");588 589 errno = 0;590 if (IsMinus)591 CurIntVal = strtoll(NumStart, nullptr, Base);592 else593 CurIntVal = strtoull(NumStart, nullptr, Base);594 595 if (errno == EINVAL)596 return ReturnError(TokStart, "invalid number");597 if (errno == ERANGE)598 return ReturnError(TokStart, "number out of range");599 600 return Base == 2 ? tgtok::BinaryIntVal : tgtok::IntVal;601}602 603/// LexBracket - We just read '['. If this is a code block, return it,604/// otherwise return the bracket. Match: '[' and '[{ ( [^}]+ | }[^]] )* }]'605tgtok::TokKind TGLexer::LexBracket() {606 if (CurPtr[0] != '{')607 return tgtok::l_square;608 ++CurPtr;609 const char *CodeStart = CurPtr;610 while (true) {611 int Char = getNextChar();612 if (Char == EOF)613 break;614 615 if (Char != '}')616 continue;617 618 Char = getNextChar();619 if (Char == EOF)620 break;621 if (Char == ']') {622 CurStrVal.assign(CodeStart, CurPtr - 2);623 return tgtok::CodeFragment;624 }625 }626 627 return ReturnError(CodeStart - 2, "unterminated code block");628}629 630/// LexExclaim - Lex '!' and '![a-zA-Z]+'.631tgtok::TokKind TGLexer::LexExclaim() {632 if (!isAlpha(*CurPtr))633 return ReturnError(CurPtr - 1, "invalid \"!operator\"");634 635 const char *Start = CurPtr++;636 while (isAlpha(*CurPtr))637 ++CurPtr;638 639 // Check to see which operator this is.640 tgtok::TokKind Kind =641 StringSwitch<tgtok::TokKind>(StringRef(Start, CurPtr - Start))642 .Case("eq", tgtok::XEq)643 .Case("ne", tgtok::XNe)644 .Case("le", tgtok::XLe)645 .Case("lt", tgtok::XLt)646 .Case("ge", tgtok::XGe)647 .Case("gt", tgtok::XGt)648 .Case("if", tgtok::XIf)649 .Case("cond", tgtok::XCond)650 .Case("isa", tgtok::XIsA)651 .Case("head", tgtok::XHead)652 .Case("tail", tgtok::XTail)653 .Case("size", tgtok::XSize)654 .Case("con", tgtok::XConcat)655 .Case("dag", tgtok::XDag)656 .Case("add", tgtok::XADD)657 .Case("sub", tgtok::XSUB)658 .Case("mul", tgtok::XMUL)659 .Case("div", tgtok::XDIV)660 .Case("not", tgtok::XNOT)661 .Case("logtwo", tgtok::XLOG2)662 .Case("and", tgtok::XAND)663 .Case("or", tgtok::XOR)664 .Case("xor", tgtok::XXOR)665 .Case("shl", tgtok::XSHL)666 .Case("sra", tgtok::XSRA)667 .Case("srl", tgtok::XSRL)668 .Case("cast", tgtok::XCast)669 .Case("empty", tgtok::XEmpty)670 .Case("subst", tgtok::XSubst)671 .Case("foldl", tgtok::XFoldl)672 .Case("foreach", tgtok::XForEach)673 .Case("filter", tgtok::XFilter)674 .Case("listconcat", tgtok::XListConcat)675 .Case("listflatten", tgtok::XListFlatten)676 .Case("listsplat", tgtok::XListSplat)677 .Case("listremove", tgtok::XListRemove)678 .Case("range", tgtok::XRange)679 .Case("strconcat", tgtok::XStrConcat)680 .Case("initialized", tgtok::XInitialized)681 .Case("interleave", tgtok::XInterleave)682 .Case("instances", tgtok::XInstances)683 .Case("substr", tgtok::XSubstr)684 .Case("find", tgtok::XFind)685 .Cases({"setdagop", "setop"},686 tgtok::XSetDagOp) // !setop is deprecated.687 .Cases({"getdagop", "getop"},688 tgtok::XGetDagOp) // !getop is deprecated.689 .Case("setdagopname", tgtok::XSetDagOpName)690 .Case("getdagopname", tgtok::XGetDagOpName)691 .Case("getdagarg", tgtok::XGetDagArg)692 .Case("getdagname", tgtok::XGetDagName)693 .Case("setdagarg", tgtok::XSetDagArg)694 .Case("setdagname", tgtok::XSetDagName)695 .Case("exists", tgtok::XExists)696 .Case("tolower", tgtok::XToLower)697 .Case("toupper", tgtok::XToUpper)698 .Case("repr", tgtok::XRepr)699 .Case("match", tgtok::XMatch)700 .Default(tgtok::Error);701 702 return Kind != tgtok::Error ? Kind703 : ReturnError(Start - 1, "unknown operator");704}705 706bool TGLexer::prepExitInclude(bool IncludeStackMustBeEmpty) {707 // Report an error, if preprocessor control stack for the current708 // file is not empty.709 if (!PrepIncludeStack.back().empty()) {710 prepReportPreprocessorStackError();711 712 return false;713 }714 715 // Pop the preprocessing controls from the include stack.716 PrepIncludeStack.pop_back();717 718 if (IncludeStackMustBeEmpty) {719 assert(PrepIncludeStack.empty() &&720 "preprocessor include stack is not empty");721 } else {722 assert(!PrepIncludeStack.empty() && "preprocessor include stack is empty");723 }724 725 return true;726}727 728tgtok::TokKind TGLexer::prepIsDirective() const {729 for (const auto [Kind, Word] : PreprocessorDirs) {730 if (StringRef(CurPtr, Word.size()) != Word)731 continue;732 int NextChar = peekNextChar(Word.size());733 734 // Check for whitespace after the directive. If there is no whitespace,735 // then we do not recognize it as a preprocessing directive.736 737 // New line and EOF may follow only #else/#endif. It will be reported738 // as an error for #ifdef/#define after the call to prepLexMacroName().739 if (NextChar == ' ' || NextChar == '\t' || NextChar == EOF ||740 NextChar == '\n' ||741 // It looks like TableGen does not support '\r' as the actual742 // carriage return, e.g. getNextChar() treats a single '\r'743 // as '\n'. So we do the same here.744 NextChar == '\r')745 return Kind;746 747 // Allow comments after some directives, e.g.:748 // #else// OR #else/**/749 // #endif// OR #endif/**/750 //751 // Note that we do allow comments after #ifdef/#define here, e.g.752 // #ifdef/**/ AND #ifdef//753 // #define/**/ AND #define//754 //755 // These cases will be reported as incorrect after calling756 // prepLexMacroName(). We could have supported C-style comments757 // after #ifdef/#define, but this would complicate the code758 // for little benefit.759 if (NextChar == '/') {760 NextChar = peekNextChar(Word.size() + 1);761 762 if (NextChar == '*' || NextChar == '/')763 return Kind;764 765 // Pretend that we do not recognize the directive.766 }767 }768 769 return tgtok::Error;770}771 772void TGLexer::prepEatPreprocessorDirective(tgtok::TokKind Kind) {773 TokStart = CurPtr;774 775 for (const auto [PKind, PWord] : PreprocessorDirs) {776 if (PKind == Kind) {777 // Advance CurPtr to the end of the preprocessing word.778 CurPtr += PWord.size();779 return;780 }781 }782 783 llvm_unreachable(784 "unsupported preprocessing token in prepEatPreprocessorDirective()");785}786 787tgtok::TokKind TGLexer::lexPreprocessor(tgtok::TokKind Kind,788 bool ReturnNextLiveToken) {789 // We must be looking at a preprocessing directive. Eat it!790 prepEatPreprocessorDirective(Kind);791 792 if (Kind == tgtok::Ifdef || Kind == tgtok::Ifndef) {793 StringRef MacroName = prepLexMacroName();794 StringRef IfTokName = Kind == tgtok::Ifdef ? "#ifdef" : "#ifndef";795 if (MacroName.empty())796 return ReturnError(TokStart, "expected macro name after " + IfTokName);797 798 bool MacroIsDefined = DefinedMacros.count(MacroName) != 0;799 800 // Canonicalize ifndef's MacroIsDefined to its ifdef equivalent.801 if (Kind == tgtok::Ifndef)802 MacroIsDefined = !MacroIsDefined;803 804 // Regardless of whether we are processing tokens or not,805 // we put the #ifdef control on stack.806 // Note that MacroIsDefined has been canonicalized against ifdef.807 PrepIncludeStack.back().push_back(808 {tgtok::Ifdef, MacroIsDefined, SMLoc::getFromPointer(TokStart)});809 810 if (!prepSkipDirectiveEnd())811 return ReturnError(CurPtr, "only comments are supported after " +812 IfTokName + " NAME");813 814 // If we were not processing tokens before this #ifdef,815 // then just return back to the lines skipping code.816 if (!ReturnNextLiveToken)817 return Kind;818 819 // If we were processing tokens before this #ifdef,820 // and the macro is defined, then just return the next token.821 if (MacroIsDefined)822 return LexToken();823 824 // We were processing tokens before this #ifdef, and the macro825 // is not defined, so we have to start skipping the lines.826 // If the skipping is successful, it will return the token following827 // either #else or #endif corresponding to this #ifdef.828 if (prepSkipRegion(ReturnNextLiveToken))829 return LexToken();830 831 return tgtok::Error;832 } else if (Kind == tgtok::Else) {833 // Check if this #else is correct before calling prepSkipDirectiveEnd(),834 // which will move CurPtr away from the beginning of #else.835 if (PrepIncludeStack.back().empty())836 return ReturnError(TokStart, "#else without #ifdef or #ifndef");837 838 PreprocessorControlDesc IfdefEntry = PrepIncludeStack.back().back();839 840 if (IfdefEntry.Kind != tgtok::Ifdef) {841 PrintError(TokStart, "double #else");842 return ReturnError(IfdefEntry.SrcPos, "previous #else is here");843 }844 845 // Replace the corresponding #ifdef's control with its negation846 // on the control stack.847 PrepIncludeStack.back().back() = {Kind, !IfdefEntry.IsDefined,848 SMLoc::getFromPointer(TokStart)};849 850 if (!prepSkipDirectiveEnd())851 return ReturnError(CurPtr, "only comments are supported after #else");852 853 // If we were processing tokens before this #else,854 // we have to start skipping lines until the matching #endif.855 if (ReturnNextLiveToken) {856 if (prepSkipRegion(ReturnNextLiveToken))857 return LexToken();858 859 return tgtok::Error;860 }861 862 // Return to the lines skipping code.863 return Kind;864 } else if (Kind == tgtok::Endif) {865 // Check if this #endif is correct before calling prepSkipDirectiveEnd(),866 // which will move CurPtr away from the beginning of #endif.867 if (PrepIncludeStack.back().empty())868 return ReturnError(TokStart, "#endif without #ifdef");869 870 [[maybe_unused]] auto &IfdefOrElseEntry = PrepIncludeStack.back().back();871 872 assert((IfdefOrElseEntry.Kind == tgtok::Ifdef ||873 IfdefOrElseEntry.Kind == tgtok::Else) &&874 "invalid preprocessor control on the stack");875 876 if (!prepSkipDirectiveEnd())877 return ReturnError(CurPtr, "only comments are supported after #endif");878 879 PrepIncludeStack.back().pop_back();880 881 // If we were processing tokens before this #endif, then882 // we should continue it.883 if (ReturnNextLiveToken) {884 return LexToken();885 }886 887 // Return to the lines skipping code.888 return Kind;889 } else if (Kind == tgtok::Define) {890 StringRef MacroName = prepLexMacroName();891 if (MacroName.empty())892 return ReturnError(TokStart, "expected macro name after #define");893 894 if (!DefinedMacros.insert(MacroName).second)895 PrintWarning(getLoc(),896 "duplicate definition of macro: " + Twine(MacroName));897 898 if (!prepSkipDirectiveEnd())899 return ReturnError(CurPtr,900 "only comments are supported after #define NAME");901 902 assert(ReturnNextLiveToken &&903 "#define must be ignored during the lines skipping");904 905 return LexToken();906 }907 908 llvm_unreachable("preprocessing directive is not supported");909}910 911bool TGLexer::prepSkipRegion(bool MustNeverBeFalse) {912 assert(MustNeverBeFalse && "invalid recursion.");913 914 do {915 // Skip all symbols to the line end.916 while (*CurPtr != '\n')917 ++CurPtr;918 919 // Find the first non-whitespace symbol in the next line(s).920 if (!prepSkipLineBegin())921 return false;922 923 // If the first non-blank/comment symbol on the line is '#',924 // it may be a start of preprocessing directive.925 //926 // If it is not '#' just go to the next line.927 if (*CurPtr == '#')928 ++CurPtr;929 else930 continue;931 932 tgtok::TokKind Kind = prepIsDirective();933 934 // If we did not find a preprocessing directive or it is #define,935 // then just skip to the next line. We do not have to do anything936 // for #define in the line-skipping mode.937 if (Kind == tgtok::Error || Kind == tgtok::Define)938 continue;939 940 tgtok::TokKind ProcessedKind = lexPreprocessor(Kind, false);941 942 // If lexPreprocessor() encountered an error during lexing this943 // preprocessor idiom, then return false to the calling lexPreprocessor().944 // This will force tgtok::Error to be returned to the tokens processing.945 if (ProcessedKind == tgtok::Error)946 return false;947 948 assert(Kind == ProcessedKind && "prepIsDirective() and lexPreprocessor() "949 "returned different token kinds");950 951 // If this preprocessing directive enables tokens processing,952 // then return to the lexPreprocessor() and get to the next token.953 // We can move from line-skipping mode to processing tokens only954 // due to #else or #endif.955 if (prepIsProcessingEnabled()) {956 assert((Kind == tgtok::Else || Kind == tgtok::Endif) &&957 "tokens processing was enabled by an unexpected preprocessing "958 "directive");959 960 return true;961 }962 } while (CurPtr != CurBuf.end());963 964 // We have reached the end of the file, but never left the lines-skipping965 // mode. This means there is no matching #endif.966 prepReportPreprocessorStackError();967 return false;968}969 970StringRef TGLexer::prepLexMacroName() {971 // Skip whitespaces between the preprocessing directive and the macro name.972 while (*CurPtr == ' ' || *CurPtr == '\t')973 ++CurPtr;974 975 TokStart = CurPtr;976 CurPtr = lexMacroName(StringRef(CurPtr, CurBuf.end() - CurPtr));977 return StringRef(TokStart, CurPtr - TokStart);978}979 980bool TGLexer::prepSkipLineBegin() {981 while (CurPtr != CurBuf.end()) {982 switch (*CurPtr) {983 case ' ':984 case '\t':985 case '\n':986 case '\r':987 break;988 989 case '/': {990 int NextChar = peekNextChar(1);991 if (NextChar == '*') {992 // Skip C-style comment.993 // Note that we do not care about skipping the C++-style comments.994 // If the line contains "//", it may not contain any processable995 // preprocessing directive. Just return CurPtr pointing to996 // the first '/' in this case. We also do not care about997 // incorrect symbols after the first '/' - we are in lines-skipping998 // mode, so incorrect code is allowed to some extent.999 1000 // Set TokStart to the beginning of the comment to enable proper1001 // diagnostic printing in case of error in SkipCComment().1002 TokStart = CurPtr;1003 1004 // CurPtr must point to '*' before call to SkipCComment().1005 ++CurPtr;1006 if (SkipCComment())1007 return false;1008 } else {1009 // CurPtr points to the non-whitespace '/'.1010 return true;1011 }1012 1013 // We must not increment CurPtr after the comment was lexed.1014 continue;1015 }1016 1017 default:1018 return true;1019 }1020 1021 ++CurPtr;1022 }1023 1024 // We have reached the end of the file. Return to the lines skipping1025 // code, and allow it to handle the EOF as needed.1026 return true;1027}1028 1029bool TGLexer::prepSkipDirectiveEnd() {1030 while (CurPtr != CurBuf.end()) {1031 switch (*CurPtr) {1032 case ' ':1033 case '\t':1034 break;1035 1036 case '\n':1037 case '\r':1038 return true;1039 1040 case '/': {1041 int NextChar = peekNextChar(1);1042 if (NextChar == '/') {1043 // Skip C++-style comment.1044 // We may just return true now, but let's skip to the line/buffer end1045 // to simplify the method specification.1046 ++CurPtr;1047 SkipBCPLComment();1048 } else if (NextChar == '*') {1049 // When we are skipping C-style comment at the end of a preprocessing1050 // directive, we can skip several lines. If any meaningful TD token1051 // follows the end of the C-style comment on the same line, it will1052 // be considered as an invalid usage of TD token.1053 // For example, we want to forbid usages like this one:1054 // #define MACRO class Class {}1055 // But with C-style comments we also disallow the following:1056 // #define MACRO /* This macro is used1057 // to ... */ class Class {}1058 // One can argue that this should be allowed, but it does not seem1059 // to be worth of the complication. Moreover, this matches1060 // the C preprocessor behavior.1061 1062 // Set TokStart to the beginning of the comment to enable proper1063 // diagnostic printer in case of error in SkipCComment().1064 TokStart = CurPtr;1065 ++CurPtr;1066 if (SkipCComment())1067 return false;1068 } else {1069 TokStart = CurPtr;1070 PrintError(CurPtr, "unexpected character");1071 return false;1072 }1073 1074 // We must not increment CurPtr after the comment was lexed.1075 continue;1076 }1077 1078 default:1079 // Do not allow any non-whitespaces after the directive.1080 TokStart = CurPtr;1081 return false;1082 }1083 1084 ++CurPtr;1085 }1086 1087 return true;1088}1089 1090bool TGLexer::prepIsProcessingEnabled() {1091 return all_of(PrepIncludeStack.back(),1092 [](const PreprocessorControlDesc &I) { return I.IsDefined; });1093}1094 1095void TGLexer::prepReportPreprocessorStackError() {1096 auto &PrepControl = PrepIncludeStack.back().back();1097 PrintError(CurBuf.end(), "reached EOF without matching #endif");1098 PrintError(PrepControl.SrcPos, "the latest preprocessor control is here");1099 1100 TokStart = CurPtr;1101}1102