1128 lines · cpp
1//===- DependencyDirectivesScanner.cpp ------------------------------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8///9/// \file10/// This is the interface for scanning header and source files to get the11/// minimum necessary preprocessor directives for evaluating includes. It12/// reduces the source down to #define, #include, #import, @import, and any13/// conditional preprocessor logic that contains one of those.14///15//===----------------------------------------------------------------------===//16 17#include "clang/Lex/DependencyDirectivesScanner.h"18#include "clang/Basic/CharInfo.h"19#include "clang/Basic/Diagnostic.h"20#include "clang/Lex/LexDiagnostic.h"21#include "clang/Lex/Lexer.h"22#include "clang/Lex/Pragma.h"23#include "llvm/ADT/ScopeExit.h"24#include "llvm/ADT/SmallString.h"25#include "llvm/ADT/StringMap.h"26#include "llvm/ADT/StringSwitch.h"27#include <optional>28 29using namespace clang;30using namespace clang::dependency_directives_scan;31using namespace llvm;32 33namespace {34 35struct DirectiveWithTokens {36 DirectiveKind Kind;37 unsigned NumTokens;38 39 DirectiveWithTokens(DirectiveKind Kind, unsigned NumTokens)40 : Kind(Kind), NumTokens(NumTokens) {}41};42 43/// Does an efficient "scan" of the sources to detect the presence of44/// preprocessor (or module import) directives and collects the raw lexed tokens45/// for those directives so that the \p Lexer can "replay" them when the file is46/// included.47///48/// Note that the behavior of the raw lexer is affected by the language mode,49/// while at this point we want to do a scan and collect tokens once,50/// irrespective of the language mode that the file will get included in. To51/// compensate for that the \p Lexer, while "replaying", will adjust a token52/// where appropriate, when it could affect the preprocessor's state.53/// For example in a directive like54///55/// \code56/// #if __has_cpp_attribute(clang::fallthrough)57/// \endcode58///59/// The preprocessor needs to see '::' as 'tok::coloncolon' instead of 260/// 'tok::colon'. The \p Lexer will adjust if it sees consecutive 'tok::colon'61/// while in C++ mode.62struct Scanner {63 Scanner(StringRef Input,64 SmallVectorImpl<dependency_directives_scan::Token> &Tokens,65 DiagnosticsEngine *Diags, SourceLocation InputSourceLoc)66 : Input(Input), Tokens(Tokens), Diags(Diags),67 InputSourceLoc(InputSourceLoc), LangOpts(getLangOptsForDepScanning()),68 TheLexer(InputSourceLoc, LangOpts, Input.begin(), Input.begin(),69 Input.end()) {}70 71 static LangOptions getLangOptsForDepScanning() {72 LangOptions LangOpts;73 // Set the lexer to use 'tok::at' for '@', instead of 'tok::unknown'.74 LangOpts.ObjC = true;75 LangOpts.LineComment = true;76 LangOpts.RawStringLiterals = true;77 // FIXME: we do not enable C11 or C++11, so we are missing u/u8/U"".78 return LangOpts;79 }80 81 /// Lex the provided source and emit the directive tokens.82 ///83 /// \returns True on error.84 bool scan(SmallVectorImpl<Directive> &Directives);85 86 friend bool clang::scanInputForCXX20ModulesUsage(StringRef Source);87 88private:89 /// Lexes next token and advances \p First and the \p Lexer.90 [[nodiscard]] dependency_directives_scan::Token &91 lexToken(const char *&First, const char *const End);92 93 [[nodiscard]] dependency_directives_scan::Token &94 lexIncludeFilename(const char *&First, const char *const End);95 96 void skipLine(const char *&First, const char *const End);97 void skipDirective(StringRef Name, const char *&First, const char *const End);98 99 /// Returns the spelling of a string literal or identifier after performing100 /// any processing needed to handle \c clang::Token::NeedsCleaning.101 StringRef cleanStringIfNeeded(const dependency_directives_scan::Token &Tok);102 103 /// Lexes next token and if it is identifier returns its string, otherwise104 /// it skips the current line and returns \p std::nullopt.105 ///106 /// In any case (whatever the token kind) \p First and the \p Lexer will107 /// advance beyond the token.108 [[nodiscard]] std::optional<StringRef>109 tryLexIdentifierOrSkipLine(const char *&First, const char *const End);110 111 /// Used when it is certain that next token is an identifier.112 [[nodiscard]] StringRef lexIdentifier(const char *&First,113 const char *const End);114 115 /// Lexes next token and returns true iff it is an identifier that matches \p116 /// Id, otherwise it skips the current line and returns false.117 ///118 /// In any case (whatever the token kind) \p First and the \p Lexer will119 /// advance beyond the token.120 [[nodiscard]] bool isNextIdentifierOrSkipLine(StringRef Id,121 const char *&First,122 const char *const End);123 124 /// Lexes next token and returns true iff it matches the kind \p K.125 /// Otherwise it skips the current line and returns false.126 ///127 /// In any case (whatever the token kind) \p First and the \p Lexer will128 /// advance beyond the token.129 [[nodiscard]] bool isNextTokenOrSkipLine(tok::TokenKind K, const char *&First,130 const char *const End);131 132 /// Lexes next token and if it is string literal, returns its string.133 /// Otherwise, it skips the current line and returns \p std::nullopt.134 ///135 /// In any case (whatever the token kind) \p First and the \p Lexer will136 /// advance beyond the token.137 [[nodiscard]] std::optional<StringRef>138 tryLexStringLiteralOrSkipLine(const char *&First, const char *const End);139 140 [[nodiscard]] bool scanImpl(const char *First, const char *const End);141 [[nodiscard]] bool lexPPLine(const char *&First, const char *const End);142 [[nodiscard]] bool lexAt(const char *&First, const char *const End);143 [[nodiscard]] bool lexModule(const char *&First, const char *const End);144 [[nodiscard]] bool lexDefine(const char *HashLoc, const char *&First,145 const char *const End);146 [[nodiscard]] bool lexPragma(const char *&First, const char *const End);147 [[nodiscard]] bool lex_Pragma(const char *&First, const char *const End);148 [[nodiscard]] bool lexEndif(const char *&First, const char *const End);149 [[nodiscard]] bool lexDefault(DirectiveKind Kind, const char *&First,150 const char *const End);151 [[nodiscard]] bool lexModuleDirectiveBody(DirectiveKind Kind,152 const char *&First,153 const char *const End);154 void lexPPDirectiveBody(const char *&First, const char *const End);155 156 DirectiveWithTokens &pushDirective(DirectiveKind Kind) {157 Tokens.append(CurDirToks);158 DirsWithToks.emplace_back(Kind, CurDirToks.size());159 CurDirToks.clear();160 return DirsWithToks.back();161 }162 void popDirective() {163 Tokens.pop_back_n(DirsWithToks.pop_back_val().NumTokens);164 }165 DirectiveKind topDirective() const {166 return DirsWithToks.empty() ? pp_none : DirsWithToks.back().Kind;167 }168 169 unsigned getOffsetAt(const char *CurPtr) const {170 return CurPtr - Input.data();171 }172 173 /// Reports a diagnostic if the diagnostic engine is provided. Always returns174 /// true at the end.175 bool reportError(const char *CurPtr, unsigned Err);176 177 StringMap<char> SplitIds;178 StringRef Input;179 SmallVectorImpl<dependency_directives_scan::Token> &Tokens;180 DiagnosticsEngine *Diags;181 SourceLocation InputSourceLoc;182 183 const char *LastTokenPtr = nullptr;184 /// Keeps track of the tokens for the currently lexed directive. Once a185 /// directive is fully lexed and "committed" then the tokens get appended to186 /// \p Tokens and \p CurDirToks is cleared for the next directive.187 SmallVector<dependency_directives_scan::Token, 32> CurDirToks;188 /// The directives that were lexed along with the number of tokens that each189 /// directive contains. The tokens of all the directives are kept in \p Tokens190 /// vector, in the same order as the directives order in \p DirsWithToks.191 SmallVector<DirectiveWithTokens, 64> DirsWithToks;192 LangOptions LangOpts;193 Lexer TheLexer;194};195 196} // end anonymous namespace197 198bool Scanner::reportError(const char *CurPtr, unsigned Err) {199 if (!Diags)200 return true;201 assert(CurPtr >= Input.data() && "invalid buffer ptr");202 Diags->Report(InputSourceLoc.getLocWithOffset(getOffsetAt(CurPtr)), Err);203 return true;204}205 206static void skipOverSpaces(const char *&First, const char *const End) {207 while (First != End && isHorizontalWhitespace(*First))208 ++First;209}210 211// Move back by one character, skipping escaped newlines (backslash + \n)212static char previousChar(const char *First, const char *&Current) {213 assert(Current > First);214 --Current;215 while (Current > First && isVerticalWhitespace(*Current)) {216 // Check if the previous character is a backslash217 if (Current > First && *(Current - 1) == '\\') {218 // Use Lexer's getEscapedNewLineSize to get the size of the escaped219 // newline220 unsigned EscapeSize = Lexer::getEscapedNewLineSize(Current);221 if (EscapeSize > 0) {222 // Skip back over the entire escaped newline sequence (backslash +223 // newline)224 Current -= (1 + EscapeSize);225 } else {226 break;227 }228 } else {229 break;230 }231 }232 return *Current;233}234 235[[nodiscard]] static bool isRawStringLiteral(const char *First,236 const char *Current) {237 assert(First <= Current);238 239 // Check if we can even back up.240 if (*Current != '"' || First == Current)241 return false;242 243 // Check for an "R".244 if (previousChar(First, Current) != 'R')245 return false;246 if (First == Current ||247 !isAsciiIdentifierContinue(previousChar(First, Current)))248 return true;249 250 // Check for a prefix of "u", "U", or "L".251 if (*Current == 'u' || *Current == 'U' || *Current == 'L')252 return First == Current ||253 !isAsciiIdentifierContinue(previousChar(First, Current));254 255 // Check for a prefix of "u8".256 if (*Current != '8' || First == Current ||257 previousChar(First, Current) != 'u')258 return false;259 return First == Current ||260 !isAsciiIdentifierContinue(previousChar(First, Current));261}262 263static void skipRawString(const char *&First, const char *const End) {264 assert(First[0] == '"');265 266 const char *Last = ++First;267 while (Last != End && *Last != '(')268 ++Last;269 if (Last == End) {270 First = Last; // Hit the end... just give up.271 return;272 }273 274 StringRef Terminator(First, Last - First);275 for (;;) {276 // Move First to just past the next ")".277 First = Last;278 while (First != End && *First != ')')279 ++First;280 if (First == End)281 return;282 ++First;283 284 // Look ahead for the terminator sequence.285 Last = First;286 while (Last != End && size_t(Last - First) < Terminator.size() &&287 Terminator[Last - First] == *Last)288 ++Last;289 290 // Check if we hit it (or the end of the file).291 if (Last == End) {292 First = Last;293 return;294 }295 if (size_t(Last - First) < Terminator.size())296 continue;297 if (*Last != '"')298 continue;299 First = Last + 1;300 return;301 }302}303 304// Returns the length of EOL, either 0 (no end-of-line), 1 (\n) or 2 (\r\n)305static unsigned isEOL(const char *First, const char *const End) {306 if (First == End)307 return 0;308 if (End - First > 1 && isVerticalWhitespace(First[0]) &&309 isVerticalWhitespace(First[1]) && First[0] != First[1])310 return 2;311 return !!isVerticalWhitespace(First[0]);312}313 314static void skipString(const char *&First, const char *const End) {315 assert(*First == '\'' || *First == '"' || *First == '<');316 const char Terminator = *First == '<' ? '>' : *First;317 for (++First; First != End && *First != Terminator; ++First) {318 // String and character literals don't extend past the end of the line.319 if (isVerticalWhitespace(*First))320 return;321 if (*First != '\\')322 continue;323 // Skip past backslash to the next character. This ensures that the324 // character right after it is skipped as well, which matters if it's325 // the terminator.326 if (++First == End)327 return;328 if (!isWhitespace(*First))329 continue;330 // Whitespace after the backslash might indicate a line continuation.331 const char *FirstAfterBackslashPastSpace = First;332 skipOverSpaces(FirstAfterBackslashPastSpace, End);333 if (unsigned NLSize = isEOL(FirstAfterBackslashPastSpace, End)) {334 // Advance the character pointer to the next line for the next335 // iteration.336 First = FirstAfterBackslashPastSpace + NLSize - 1;337 }338 }339 if (First != End)340 ++First; // Finish off the string.341}342 343// Returns the length of the skipped newline344static unsigned skipNewline(const char *&First, const char *End) {345 if (First == End)346 return 0;347 assert(isVerticalWhitespace(*First));348 unsigned Len = isEOL(First, End);349 assert(Len && "expected newline");350 First += Len;351 return Len;352}353 354static void skipToNewlineRaw(const char *&First, const char *const End) {355 for (;;) {356 if (First == End)357 return;358 359 unsigned Len = isEOL(First, End);360 if (Len)361 return;362 363 char LastNonWhitespace = ' ';364 do {365 if (!isHorizontalWhitespace(*First))366 LastNonWhitespace = *First;367 if (++First == End)368 return;369 Len = isEOL(First, End);370 } while (!Len);371 372 if (LastNonWhitespace != '\\')373 return;374 375 First += Len;376 // Keep skipping lines...377 }378}379 380static void skipLineComment(const char *&First, const char *const End) {381 assert(First[0] == '/' && First[1] == '/');382 First += 2;383 skipToNewlineRaw(First, End);384}385 386static void skipBlockComment(const char *&First, const char *const End) {387 assert(First[0] == '/' && First[1] == '*');388 if (End - First < 4) {389 First = End;390 return;391 }392 for (First += 3; First != End; ++First)393 if (First[-1] == '*' && First[0] == '/') {394 ++First;395 return;396 }397}398 399/// \returns True if the current single quotation mark character is a C++14400/// digit separator.401static bool isQuoteCppDigitSeparator(const char *const Start,402 const char *const Cur,403 const char *const End) {404 assert(*Cur == '\'' && "expected quotation character");405 // skipLine called in places where we don't expect a valid number406 // body before `start` on the same line, so always return false at the start.407 if (Start == Cur)408 return false;409 // The previous character must be a valid PP number character.410 // Make sure that the L, u, U, u8 prefixes don't get marked as a411 // separator though.412 char Prev = *(Cur - 1);413 if (Prev == 'L' || Prev == 'U' || Prev == 'u')414 return false;415 if (Prev == '8' && (Cur - 1 != Start) && *(Cur - 2) == 'u')416 return false;417 if (!isPreprocessingNumberBody(Prev))418 return false;419 // The next character should be a valid identifier body character.420 return (Cur + 1) < End && isAsciiIdentifierContinue(*(Cur + 1));421}422 423void Scanner::skipLine(const char *&First, const char *const End) {424 for (;;) {425 assert(First <= End);426 if (First == End)427 return;428 429 if (isVerticalWhitespace(*First)) {430 skipNewline(First, End);431 return;432 }433 const char *Start = First;434 // Use `LastNonWhitespace`to track if a line-continuation has ever been seen435 // before a new-line character:436 char LastNonWhitespace = ' ';437 while (First != End && !isVerticalWhitespace(*First)) {438 // Iterate over strings correctly to avoid comments and newlines.439 if (*First == '"' ||440 (*First == '\'' && !isQuoteCppDigitSeparator(Start, First, End))) {441 LastTokenPtr = First;442 if (isRawStringLiteral(Start, First))443 skipRawString(First, End);444 else445 skipString(First, End);446 continue;447 }448 449 // Continue on the same line if an EOL is preceded with backslash450 if (First + 1 < End && *First == '\\') {451 if (unsigned Len = isEOL(First + 1, End)) {452 First += 1 + Len;453 continue;454 }455 }456 457 // Iterate over comments correctly.458 if (*First != '/' || End - First < 2) {459 LastTokenPtr = First;460 if (!isWhitespace(*First))461 LastNonWhitespace = *First;462 ++First;463 continue;464 }465 466 if (First[1] == '/') {467 // "//...".468 skipLineComment(First, End);469 continue;470 }471 472 if (First[1] != '*') {473 LastTokenPtr = First;474 if (!isWhitespace(*First))475 LastNonWhitespace = *First;476 ++First;477 continue;478 }479 480 // "/*...*/".481 skipBlockComment(First, End);482 }483 if (First == End)484 return;485 486 // Skip over the newline.487 skipNewline(First, End);488 489 if (LastNonWhitespace != '\\')490 break;491 }492}493 494void Scanner::skipDirective(StringRef Name, const char *&First,495 const char *const End) {496 if (llvm::StringSwitch<bool>(Name)497 .Case("warning", true)498 .Case("error", true)499 .Default(false))500 // Do not process quotes or comments.501 skipToNewlineRaw(First, End);502 else503 skipLine(First, End);504}505 506static void skipWhitespace(const char *&First, const char *const End) {507 for (;;) {508 assert(First <= End);509 skipOverSpaces(First, End);510 511 if (End - First < 2)512 return;513 514 if (*First == '\\') {515 const char *Ptr = First + 1;516 while (Ptr < End && isHorizontalWhitespace(*Ptr))517 ++Ptr;518 if (Ptr != End && isVerticalWhitespace(*Ptr)) {519 skipNewline(Ptr, End);520 First = Ptr;521 continue;522 }523 return;524 }525 526 // Check for a non-comment character.527 if (First[0] != '/')528 return;529 530 // "// ...".531 if (First[1] == '/') {532 skipLineComment(First, End);533 return;534 }535 536 // Cannot be a comment.537 if (First[1] != '*')538 return;539 540 // "/*...*/".541 skipBlockComment(First, End);542 }543}544 545bool Scanner::lexModuleDirectiveBody(DirectiveKind Kind, const char *&First,546 const char *const End) {547 const char *DirectiveLoc = Input.data() + CurDirToks.front().Offset;548 for (;;) {549 // Keep a copy of the First char incase it needs to be reset.550 const char *Previous = First;551 const dependency_directives_scan::Token &Tok = lexToken(First, End);552 if ((Tok.is(tok::hash) || Tok.is(tok::at)) &&553 (Tok.Flags & clang::Token::StartOfLine)) {554 CurDirToks.pop_back();555 First = Previous;556 return false;557 }558 if (Tok.is(tok::eof))559 return reportError(560 DirectiveLoc,561 diag::err_dep_source_scanner_missing_semi_after_at_import);562 if (Tok.is(tok::semi))563 break;564 }565 566 const auto &Tok = lexToken(First, End);567 pushDirective(Kind);568 if (Tok.is(tok::eof) || Tok.is(tok::eod))569 return false;570 return reportError(DirectiveLoc,571 diag::err_dep_source_scanner_unexpected_tokens_at_import);572}573 574dependency_directives_scan::Token &Scanner::lexToken(const char *&First,575 const char *const End) {576 clang::Token Tok;577 TheLexer.LexFromRawLexer(Tok);578 First = Input.data() + TheLexer.getCurrentBufferOffset();579 assert(First <= End);580 581 unsigned Offset = TheLexer.getCurrentBufferOffset() - Tok.getLength();582 CurDirToks.emplace_back(Offset, Tok.getLength(), Tok.getKind(),583 Tok.getFlags());584 return CurDirToks.back();585}586 587dependency_directives_scan::Token &588Scanner::lexIncludeFilename(const char *&First, const char *const End) {589 clang::Token Tok;590 TheLexer.LexIncludeFilename(Tok);591 First = Input.data() + TheLexer.getCurrentBufferOffset();592 assert(First <= End);593 594 unsigned Offset = TheLexer.getCurrentBufferOffset() - Tok.getLength();595 CurDirToks.emplace_back(Offset, Tok.getLength(), Tok.getKind(),596 Tok.getFlags());597 return CurDirToks.back();598}599 600void Scanner::lexPPDirectiveBody(const char *&First, const char *const End) {601 while (true) {602 const dependency_directives_scan::Token &Tok = lexToken(First, End);603 if (Tok.is(tok::eod) || Tok.is(tok::eof))604 break;605 }606}607 608StringRef609Scanner::cleanStringIfNeeded(const dependency_directives_scan::Token &Tok) {610 bool NeedsCleaning = Tok.Flags & clang::Token::NeedsCleaning;611 if (LLVM_LIKELY(!NeedsCleaning))612 return Input.slice(Tok.Offset, Tok.getEnd());613 614 SmallString<64> Spelling;615 Spelling.resize(Tok.Length);616 617 // FIXME: C++11 raw string literals need special handling (see getSpellingSlow618 // in the Lexer). Currently we cannot see them due to our LangOpts.619 620 unsigned SpellingLength = 0;621 const char *BufPtr = Input.begin() + Tok.Offset;622 const char *AfterIdent = Input.begin() + Tok.getEnd();623 while (BufPtr < AfterIdent) {624 auto [Char, Size] = Lexer::getCharAndSizeNoWarn(BufPtr, LangOpts);625 Spelling[SpellingLength++] = Char;626 BufPtr += Size;627 }628 629 return SplitIds.try_emplace(StringRef(Spelling.begin(), SpellingLength), 0)630 .first->first();631}632 633std::optional<StringRef>634Scanner::tryLexIdentifierOrSkipLine(const char *&First, const char *const End) {635 const dependency_directives_scan::Token &Tok = lexToken(First, End);636 if (Tok.isNot(tok::raw_identifier)) {637 if (!Tok.is(tok::eod))638 skipLine(First, End);639 return std::nullopt;640 }641 642 return cleanStringIfNeeded(Tok);643}644 645StringRef Scanner::lexIdentifier(const char *&First, const char *const End) {646 std::optional<StringRef> Id = tryLexIdentifierOrSkipLine(First, End);647 assert(Id && "expected identifier token");648 return *Id;649}650 651bool Scanner::isNextIdentifierOrSkipLine(StringRef Id, const char *&First,652 const char *const End) {653 if (std::optional<StringRef> FoundId =654 tryLexIdentifierOrSkipLine(First, End)) {655 if (*FoundId == Id)656 return true;657 skipLine(First, End);658 }659 return false;660}661 662bool Scanner::isNextTokenOrSkipLine(tok::TokenKind K, const char *&First,663 const char *const End) {664 const dependency_directives_scan::Token &Tok = lexToken(First, End);665 if (Tok.is(K))666 return true;667 skipLine(First, End);668 return false;669}670 671std::optional<StringRef>672Scanner::tryLexStringLiteralOrSkipLine(const char *&First,673 const char *const End) {674 const dependency_directives_scan::Token &Tok = lexToken(First, End);675 if (!tok::isStringLiteral(Tok.Kind)) {676 if (!Tok.is(tok::eod))677 skipLine(First, End);678 return std::nullopt;679 }680 681 return cleanStringIfNeeded(Tok);682}683 684bool Scanner::lexAt(const char *&First, const char *const End) {685 // Handle "@import".686 687 // Lex '@'.688 const dependency_directives_scan::Token &AtTok = lexToken(First, End);689 assert(AtTok.is(tok::at));690 (void)AtTok;691 692 if (!isNextIdentifierOrSkipLine("import", First, End))693 return false;694 return lexModuleDirectiveBody(decl_at_import, First, End);695}696 697bool Scanner::lexModule(const char *&First, const char *const End) {698 StringRef Id = lexIdentifier(First, End);699 bool Export = false;700 if (Id == "export") {701 Export = true;702 std::optional<StringRef> NextId = tryLexIdentifierOrSkipLine(First, End);703 if (!NextId)704 return false;705 Id = *NextId;706 }707 708 if (Id != "module" && Id != "import") {709 skipLine(First, End);710 return false;711 }712 713 skipWhitespace(First, End);714 715 // Ignore this as a module directive if the next character can't be part of716 // an import.717 718 switch (*First) {719 case ':': {720 // `module :` is never the start of a valid module declaration.721 if (Id == "module") {722 skipLine(First, End);723 return false;724 }725 // A module partition starts with exactly one ':'. If we have '::', this is726 // a scope resolution instead and shouldn't be recognized as a directive727 // per P1857R3.728 if (First + 1 != End && First[1] == ':') {729 skipLine(First, End);730 return false;731 }732 // `import:(type)name` is a valid ObjC method decl, so check one more token.733 (void)lexToken(First, End);734 if (!tryLexIdentifierOrSkipLine(First, End))735 return false;736 break;737 }738 case ';': {739 // Handle the global module fragment `module;`.740 if (Id == "module" && !Export)741 break;742 skipLine(First, End);743 return false;744 }745 case '<':746 case '"':747 break;748 default:749 if (!isAsciiIdentifierContinue(*First)) {750 skipLine(First, End);751 return false;752 }753 }754 755 TheLexer.seek(getOffsetAt(First), /*IsAtStartOfLine*/ false);756 757 DirectiveKind Kind;758 if (Id == "module")759 Kind = Export ? cxx_export_module_decl : cxx_module_decl;760 else761 Kind = Export ? cxx_export_import_decl : cxx_import_decl;762 763 return lexModuleDirectiveBody(Kind, First, End);764}765 766bool Scanner::lex_Pragma(const char *&First, const char *const End) {767 if (!isNextTokenOrSkipLine(tok::l_paren, First, End))768 return false;769 770 std::optional<StringRef> Str = tryLexStringLiteralOrSkipLine(First, End);771 772 if (!Str || !isNextTokenOrSkipLine(tok::r_paren, First, End))773 return false;774 775 SmallString<64> Buffer(*Str);776 prepare_PragmaString(Buffer);777 778 // Use a new scanner instance since the tokens will be inside the allocated779 // string. We should already have captured all the relevant tokens in the780 // current scanner.781 SmallVector<dependency_directives_scan::Token> DiscardTokens;782 const char *Begin = Buffer.c_str();783 Scanner PragmaScanner{StringRef(Begin, Buffer.size()), DiscardTokens, Diags,784 InputSourceLoc};785 786 PragmaScanner.TheLexer.setParsingPreprocessorDirective(true);787 if (PragmaScanner.lexPragma(Begin, Buffer.end()))788 return true;789 790 DirectiveKind K = PragmaScanner.topDirective();791 if (K == pp_none) {792 skipLine(First, End);793 return false;794 }795 796 assert(Begin == Buffer.end());797 pushDirective(K);798 return false;799}800 801bool Scanner::lexPragma(const char *&First, const char *const End) {802 std::optional<StringRef> FoundId = tryLexIdentifierOrSkipLine(First, End);803 if (!FoundId)804 return false;805 806 StringRef Id = *FoundId;807 auto Kind = llvm::StringSwitch<DirectiveKind>(Id)808 .Case("once", pp_pragma_once)809 .Case("push_macro", pp_pragma_push_macro)810 .Case("pop_macro", pp_pragma_pop_macro)811 .Case("include_alias", pp_pragma_include_alias)812 .Default(pp_none);813 if (Kind != pp_none) {814 lexPPDirectiveBody(First, End);815 pushDirective(Kind);816 return false;817 }818 819 if (Id != "clang") {820 skipLine(First, End);821 return false;822 }823 824 FoundId = tryLexIdentifierOrSkipLine(First, End);825 if (!FoundId)826 return false;827 Id = *FoundId;828 829 // #pragma clang system_header830 if (Id == "system_header") {831 lexPPDirectiveBody(First, End);832 pushDirective(pp_pragma_system_header);833 return false;834 }835 836 if (Id != "module") {837 skipLine(First, End);838 return false;839 }840 841 // #pragma clang module.842 if (!isNextIdentifierOrSkipLine("import", First, End))843 return false;844 845 // #pragma clang module import.846 lexPPDirectiveBody(First, End);847 pushDirective(pp_pragma_import);848 return false;849}850 851bool Scanner::lexEndif(const char *&First, const char *const End) {852 // Strip out "#else" if it's empty.853 if (topDirective() == pp_else)854 popDirective();855 856 // If "#ifdef" is empty, strip it and skip the "#endif".857 //858 // FIXME: Once/if Clang starts disallowing __has_include in macro expansions,859 // we can skip empty `#if` and `#elif` blocks as well after scanning for a860 // literal __has_include in the condition. Even without that rule we could861 // drop the tokens if we scan for identifiers in the condition and find none.862 if (topDirective() == pp_ifdef || topDirective() == pp_ifndef) {863 popDirective();864 skipLine(First, End);865 return false;866 }867 868 return lexDefault(pp_endif, First, End);869}870 871bool Scanner::lexDefault(DirectiveKind Kind, const char *&First,872 const char *const End) {873 lexPPDirectiveBody(First, End);874 pushDirective(Kind);875 return false;876}877 878static bool isStartOfRelevantLine(char First) {879 switch (First) {880 case '#':881 case '@':882 case 'i':883 case 'e':884 case 'm':885 case '_':886 return true;887 }888 return false;889}890 891bool Scanner::lexPPLine(const char *&First, const char *const End) {892 assert(First != End);893 894 skipWhitespace(First, End);895 assert(First <= End);896 if (First == End)897 return false;898 899 if (!isStartOfRelevantLine(*First)) {900 skipLine(First, End);901 assert(First <= End);902 return false;903 }904 905 LastTokenPtr = First;906 907 TheLexer.seek(getOffsetAt(First), /*IsAtStartOfLine*/ true);908 909 auto ScEx1 = make_scope_exit([&]() {910 /// Clear Scanner's CurDirToks before returning, in case we didn't push a911 /// new directive.912 CurDirToks.clear();913 });914 915 if (*First == '_') {916 if (isNextIdentifierOrSkipLine("_Pragma", First, End))917 return lex_Pragma(First, End);918 return false;919 }920 921 // Handle preprocessing directives.922 923 TheLexer.setParsingPreprocessorDirective(true);924 auto ScEx2 = make_scope_exit(925 [&]() { TheLexer.setParsingPreprocessorDirective(false); });926 927 // Handle "@import".928 if (*First == '@')929 return lexAt(First, End);930 931 // Handle module directives for C++20 modules.932 if (*First == 'i' || *First == 'e' || *First == 'm')933 return lexModule(First, End);934 935 // Lex '#'.936 const dependency_directives_scan::Token &HashTok = lexToken(First, End);937 if (HashTok.is(tok::hashhash)) {938 // A \p tok::hashhash at this location is passed by the preprocessor to the939 // parser to interpret, like any other token. So for dependency scanning940 // skip it like a normal token not affecting the preprocessor.941 skipLine(First, End);942 assert(First <= End);943 return false;944 }945 assert(HashTok.is(tok::hash));946 (void)HashTok;947 948 std::optional<StringRef> FoundId = tryLexIdentifierOrSkipLine(First, End);949 if (!FoundId)950 return false;951 952 StringRef Id = *FoundId;953 954 if (Id == "pragma")955 return lexPragma(First, End);956 957 auto Kind = llvm::StringSwitch<DirectiveKind>(Id)958 .Case("include", pp_include)959 .Case("__include_macros", pp___include_macros)960 .Case("define", pp_define)961 .Case("undef", pp_undef)962 .Case("import", pp_import)963 .Case("include_next", pp_include_next)964 .Case("if", pp_if)965 .Case("ifdef", pp_ifdef)966 .Case("ifndef", pp_ifndef)967 .Case("elif", pp_elif)968 .Case("elifdef", pp_elifdef)969 .Case("elifndef", pp_elifndef)970 .Case("else", pp_else)971 .Case("endif", pp_endif)972 .Default(pp_none);973 if (Kind == pp_none) {974 skipDirective(Id, First, End);975 return false;976 }977 978 if (Kind == pp_endif)979 return lexEndif(First, End);980 981 switch (Kind) {982 case pp_include:983 case pp___include_macros:984 case pp_include_next:985 case pp_import:986 // Ignore missing filenames in include or import directives.987 if (lexIncludeFilename(First, End).is(tok::eod)) {988 return false;989 }990 break;991 default:992 break;993 }994 995 // Everything else.996 return lexDefault(Kind, First, End);997}998 999static void skipUTF8ByteOrderMark(const char *&First, const char *const End) {1000 if ((End - First) >= 3 && First[0] == '\xef' && First[1] == '\xbb' &&1001 First[2] == '\xbf')1002 First += 3;1003}1004 1005bool Scanner::scanImpl(const char *First, const char *const End) {1006 skipUTF8ByteOrderMark(First, End);1007 while (First != End)1008 if (lexPPLine(First, End))1009 return true;1010 return false;1011}1012 1013bool Scanner::scan(SmallVectorImpl<Directive> &Directives) {1014 bool Error = scanImpl(Input.begin(), Input.end());1015 1016 if (!Error) {1017 // Add an EOF on success.1018 if (LastTokenPtr &&1019 (Tokens.empty() || LastTokenPtr > Input.begin() + Tokens.back().Offset))1020 pushDirective(tokens_present_before_eof);1021 pushDirective(pp_eof);1022 }1023 1024 ArrayRef<dependency_directives_scan::Token> RemainingTokens = Tokens;1025 for (const DirectiveWithTokens &DirWithToks : DirsWithToks) {1026 assert(RemainingTokens.size() >= DirWithToks.NumTokens);1027 Directives.emplace_back(DirWithToks.Kind,1028 RemainingTokens.take_front(DirWithToks.NumTokens));1029 RemainingTokens = RemainingTokens.drop_front(DirWithToks.NumTokens);1030 }1031 assert(RemainingTokens.empty());1032 1033 return Error;1034}1035 1036bool clang::scanSourceForDependencyDirectives(1037 StringRef Input, SmallVectorImpl<dependency_directives_scan::Token> &Tokens,1038 SmallVectorImpl<Directive> &Directives, DiagnosticsEngine *Diags,1039 SourceLocation InputSourceLoc) {1040 return Scanner(Input, Tokens, Diags, InputSourceLoc).scan(Directives);1041}1042 1043void clang::printDependencyDirectivesAsSource(1044 StringRef Source,1045 ArrayRef<dependency_directives_scan::Directive> Directives,1046 llvm::raw_ostream &OS) {1047 // Add a space separator where it is convenient for testing purposes.1048 auto needsSpaceSeparator =1049 [](tok::TokenKind Prev,1050 const dependency_directives_scan::Token &Tok) -> bool {1051 if (Prev == Tok.Kind)1052 return !Tok.isOneOf(tok::l_paren, tok::r_paren, tok::l_square,1053 tok::r_square);1054 if (Prev == tok::raw_identifier &&1055 Tok.isOneOf(tok::hash, tok::numeric_constant, tok::string_literal,1056 tok::char_constant, tok::header_name))1057 return true;1058 if (Prev == tok::r_paren &&1059 Tok.isOneOf(tok::raw_identifier, tok::hash, tok::string_literal,1060 tok::char_constant, tok::unknown))1061 return true;1062 if (Prev == tok::comma &&1063 Tok.isOneOf(tok::l_paren, tok::string_literal, tok::less))1064 return true;1065 return false;1066 };1067 1068 for (const dependency_directives_scan::Directive &Directive : Directives) {1069 if (Directive.Kind == tokens_present_before_eof)1070 OS << "<TokBeforeEOF>";1071 std::optional<tok::TokenKind> PrevTokenKind;1072 for (const dependency_directives_scan::Token &Tok : Directive.Tokens) {1073 if (PrevTokenKind && needsSpaceSeparator(*PrevTokenKind, Tok))1074 OS << ' ';1075 PrevTokenKind = Tok.Kind;1076 OS << Source.slice(Tok.Offset, Tok.getEnd());1077 }1078 }1079}1080 1081static void skipUntilMaybeCXX20ModuleDirective(const char *&First,1082 const char *const End) {1083 assert(First <= End);1084 while (First != End) {1085 if (*First == '#') {1086 ++First;1087 skipToNewlineRaw(First, End);1088 }1089 skipWhitespace(First, End);1090 if (const auto Len = isEOL(First, End)) {1091 First += Len;1092 continue;1093 }1094 break;1095 }1096}1097 1098bool clang::scanInputForCXX20ModulesUsage(StringRef Source) {1099 const char *First = Source.begin();1100 const char *const End = Source.end();1101 skipUntilMaybeCXX20ModuleDirective(First, End);1102 if (First == End)1103 return false;1104 1105 // Check if the next token can even be a module directive before creating a1106 // full lexer.1107 if (!(*First == 'i' || *First == 'e' || *First == 'm'))1108 return false;1109 1110 llvm::SmallVector<dependency_directives_scan::Token> Tokens;1111 Scanner S(StringRef(First, End - First), Tokens, nullptr, SourceLocation());1112 S.TheLexer.setParsingPreprocessorDirective(true);1113 if (S.lexModule(First, End))1114 return false;1115 auto IsCXXNamedModuleDirective = [](const DirectiveWithTokens &D) {1116 switch (D.Kind) {1117 case dependency_directives_scan::cxx_module_decl:1118 case dependency_directives_scan::cxx_import_decl:1119 case dependency_directives_scan::cxx_export_module_decl:1120 case dependency_directives_scan::cxx_export_import_decl:1121 return true;1122 default:1123 return false;1124 }1125 };1126 return llvm::any_of(S.DirsWithToks, IsCXXNamedModuleDirective);1127}1128