1152 lines · cpp
1//===- TokensTest.cpp -----------------------------------------------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8 9#include "clang/Tooling/Syntax/Tokens.h"10#include "clang/AST/ASTConsumer.h"11#include "clang/AST/Expr.h"12#include "clang/Basic/Diagnostic.h"13#include "clang/Basic/DiagnosticIDs.h"14#include "clang/Basic/DiagnosticOptions.h"15#include "clang/Basic/FileManager.h"16#include "clang/Basic/FileSystemOptions.h"17#include "clang/Basic/LLVM.h"18#include "clang/Basic/LangOptions.h"19#include "clang/Basic/SourceLocation.h"20#include "clang/Basic/SourceManager.h"21#include "clang/Basic/TokenKinds.def"22#include "clang/Basic/TokenKinds.h"23#include "clang/Driver/CreateInvocationFromArgs.h"24#include "clang/Frontend/CompilerInstance.h"25#include "clang/Frontend/FrontendAction.h"26#include "clang/Frontend/Utils.h"27#include "clang/Lex/Lexer.h"28#include "clang/Lex/PreprocessorOptions.h"29#include "clang/Lex/Token.h"30#include "clang/Tooling/Tooling.h"31#include "llvm/ADT/ArrayRef.h"32#include "llvm/ADT/IntrusiveRefCntPtr.h"33#include "llvm/ADT/STLExtras.h"34#include "llvm/ADT/StringRef.h"35#include "llvm/Support/FormatVariadic.h"36#include "llvm/Support/MemoryBuffer.h"37#include "llvm/Support/VirtualFileSystem.h"38#include "llvm/Support/raw_os_ostream.h"39#include "llvm/Support/raw_ostream.h"40#include "llvm/Testing/Annotations/Annotations.h"41#include "llvm/Testing/Support/SupportHelpers.h"42#include <cassert>43#include <cstdlib>44#include <gmock/gmock.h>45#include <gtest/gtest.h>46#include <memory>47#include <optional>48#include <ostream>49#include <string>50 51using namespace clang;52using namespace clang::syntax;53 54using llvm::ValueIs;55using ::testing::_;56using ::testing::AllOf;57using ::testing::Contains;58using ::testing::ElementsAre;59using ::testing::Field;60using ::testing::IsEmpty;61using ::testing::Matcher;62using ::testing::Not;63using ::testing::Pointee;64using ::testing::StartsWith;65 66namespace {67// Checks the passed ArrayRef<T> has the same begin() and end() iterators as the68// argument.69MATCHER_P(SameRange, A, "") {70 return A.begin() == arg.begin() && A.end() == arg.end();71}72 73Matcher<TokenBuffer::Expansion>74IsExpansion(Matcher<llvm::ArrayRef<syntax::Token>> Spelled,75 Matcher<llvm::ArrayRef<syntax::Token>> Expanded) {76 return AllOf(Field(&TokenBuffer::Expansion::Spelled, Spelled),77 Field(&TokenBuffer::Expansion::Expanded, Expanded));78}79// Matchers for syntax::Token.80MATCHER_P(Kind, K, "") { return arg.kind() == K; }81MATCHER_P2(HasText, Text, SourceMgr, "") {82 return arg.text(*SourceMgr) == Text;83}84/// Checks the start and end location of a token are equal to SourceRng.85MATCHER_P(RangeIs, SourceRng, "") {86 return arg.location() == SourceRng.first &&87 arg.endLocation() == SourceRng.second;88}89 90class TokenCollectorTest : public ::testing::Test {91public:92 /// Run the clang frontend, collect the preprocessed tokens from the frontend93 /// invocation and store them in this->Buffer.94 /// This also clears SourceManager before running the compiler.95 void recordTokens(llvm::StringRef Code) {96 class RecordTokens : public ASTFrontendAction {97 public:98 explicit RecordTokens(TokenBuffer &Result) : Result(Result) {}99 100 bool BeginSourceFileAction(CompilerInstance &CI) override {101 assert(!Collector && "expected only a single call to BeginSourceFile");102 Collector.emplace(CI.getPreprocessor());103 return true;104 }105 void EndSourceFileAction() override {106 assert(Collector && "BeginSourceFileAction was never called");107 Result = std::move(*Collector).consume();108 Result.indexExpandedTokens();109 }110 111 std::unique_ptr<ASTConsumer>112 CreateASTConsumer(CompilerInstance &CI, StringRef InFile) override {113 return std::make_unique<ASTConsumer>();114 }115 116 private:117 TokenBuffer &Result;118 std::optional<TokenCollector> Collector;119 };120 121 constexpr const char *FileName = "./input.cpp";122 FS->addFile(FileName, time_t(), llvm::MemoryBuffer::getMemBufferCopy(""));123 // Prepare to run a compiler.124 if (!Diags->getClient())125 Diags->setClient(new IgnoringDiagConsumer);126 std::vector<const char *> Args = {"tok-test", "-std=c++03", "-fsyntax-only",127 FileName};128 CreateInvocationOptions CIOpts;129 CIOpts.Diags = Diags;130 CIOpts.VFS = FS;131 auto CI = createInvocation(Args, std::move(CIOpts));132 assert(CI);133 CI->getFrontendOpts().DisableFree = false;134 CI->getPreprocessorOpts().addRemappedFile(135 FileName, llvm::MemoryBuffer::getMemBufferCopy(Code).release());136 CompilerInstance Compiler(std::move(CI));137 Compiler.setDiagnostics(Diags);138 Compiler.setVirtualFileSystem(FS);139 Compiler.setFileManager(FileMgr);140 Compiler.setSourceManager(SourceMgr);141 142 this->Buffer = TokenBuffer(*SourceMgr);143 RecordTokens Recorder(this->Buffer);144 ASSERT_TRUE(Compiler.ExecuteAction(Recorder))145 << "failed to run the frontend";146 }147 148 /// Record the tokens and return a test dump of the resulting buffer.149 std::string collectAndDump(llvm::StringRef Code) {150 recordTokens(Code);151 return Buffer.dumpForTests();152 }153 154 // Adds a file to the test VFS.155 void addFile(llvm::StringRef Path, llvm::StringRef Contents) {156 if (!FS->addFile(Path, time_t(),157 llvm::MemoryBuffer::getMemBufferCopy(Contents))) {158 ADD_FAILURE() << "could not add a file to VFS: " << Path;159 }160 }161 162 /// Add a new file, run syntax::tokenize() on the range if any, run it on the163 /// whole file otherwise and return the results.164 std::vector<syntax::Token> tokenize(llvm::StringRef Text) {165 llvm::Annotations Annot(Text);166 auto FID = SourceMgr->createFileID(167 llvm::MemoryBuffer::getMemBufferCopy(Annot.code()));168 // FIXME: pass proper LangOptions.169 if (Annot.ranges().empty())170 return syntax::tokenize(FID, *SourceMgr, LangOptions());171 return syntax::tokenize(172 syntax::FileRange(FID, Annot.range().Begin, Annot.range().End),173 *SourceMgr, LangOptions());174 }175 176 // Specialized versions of matchers that hide the SourceManager from clients.177 Matcher<syntax::Token> HasText(std::string Text) const {178 return ::HasText(Text, SourceMgr.get());179 }180 Matcher<syntax::Token> RangeIs(llvm::Annotations::Range R) const {181 std::pair<SourceLocation, SourceLocation> Ls;182 Ls.first = SourceMgr->getLocForStartOfFile(SourceMgr->getMainFileID())183 .getLocWithOffset(R.Begin);184 Ls.second = SourceMgr->getLocForStartOfFile(SourceMgr->getMainFileID())185 .getLocWithOffset(R.End);186 return ::RangeIs(Ls);187 }188 189 /// Finds a subrange in O(n * m).190 template <class T, class U, class Eq>191 llvm::ArrayRef<T> findSubrange(llvm::ArrayRef<U> Subrange,192 llvm::ArrayRef<T> Range, Eq F) {193 assert(Subrange.size() >= 1);194 if (Range.size() < Subrange.size())195 return llvm::ArrayRef(Range.end(), Range.end());196 for (auto Begin = Range.begin(), Last = Range.end() - Subrange.size();197 Begin <= Last; ++Begin) {198 auto It = Begin;199 for (auto ItSub = Subrange.begin(); ItSub != Subrange.end();200 ++ItSub, ++It) {201 if (!F(*ItSub, *It))202 goto continue_outer;203 }204 return llvm::ArrayRef(Begin, It);205 continue_outer:;206 }207 return llvm::ArrayRef(Range.end(), Range.end());208 }209 210 /// Finds a subrange in \p Tokens that match the tokens specified in \p Query.211 /// The match should be unique. \p Query is a whitespace-separated list of212 /// tokens to search for.213 llvm::ArrayRef<syntax::Token>214 findTokenRange(llvm::StringRef Query, llvm::ArrayRef<syntax::Token> Tokens) {215 llvm::SmallVector<llvm::StringRef, 8> QueryTokens;216 Query.split(QueryTokens, ' ', /*MaxSplit=*/-1, /*KeepEmpty=*/false);217 if (QueryTokens.empty()) {218 ADD_FAILURE() << "will not look for an empty list of tokens";219 std::abort();220 }221 // An equality test for search.222 auto TextMatches = [this](llvm::StringRef Q, const syntax::Token &T) {223 return Q == T.text(*SourceMgr);224 };225 // Find a match.226 auto Found = findSubrange(llvm::ArrayRef(QueryTokens), Tokens, TextMatches);227 if (Found.begin() == Tokens.end()) {228 ADD_FAILURE() << "could not find the subrange for " << Query;229 std::abort();230 }231 // Check that the match is unique.232 if (findSubrange(llvm::ArrayRef(QueryTokens),233 llvm::ArrayRef(Found.end(), Tokens.end()), TextMatches)234 .begin() != Tokens.end()) {235 ADD_FAILURE() << "match is not unique for " << Query;236 std::abort();237 }238 return Found;239 };240 241 // Specialized versions of findTokenRange for expanded and spelled tokens.242 llvm::ArrayRef<syntax::Token> findExpanded(llvm::StringRef Query) {243 return findTokenRange(Query, Buffer.expandedTokens());244 }245 llvm::ArrayRef<syntax::Token> findSpelled(llvm::StringRef Query,246 FileID File = FileID()) {247 if (!File.isValid())248 File = SourceMgr->getMainFileID();249 return findTokenRange(Query, Buffer.spelledTokens(File));250 }251 252 // Data fields.253 DiagnosticOptions DiagOpts;254 llvm::IntrusiveRefCntPtr<DiagnosticsEngine> Diags =255 llvm::makeIntrusiveRefCnt<DiagnosticsEngine>(DiagnosticIDs::create(),256 DiagOpts);257 IntrusiveRefCntPtr<llvm::vfs::InMemoryFileSystem> FS =258 llvm::makeIntrusiveRefCnt<llvm::vfs::InMemoryFileSystem>();259 llvm::IntrusiveRefCntPtr<FileManager> FileMgr =260 llvm::makeIntrusiveRefCnt<FileManager>(FileSystemOptions(), FS);261 llvm::IntrusiveRefCntPtr<SourceManager> SourceMgr =262 llvm::makeIntrusiveRefCnt<SourceManager>(*Diags, *FileMgr);263 /// Contains last result of calling recordTokens().264 TokenBuffer Buffer = TokenBuffer(*SourceMgr);265};266 267TEST_F(TokenCollectorTest, RawMode) {268 EXPECT_THAT(tokenize("int main() {}"),269 ElementsAre(Kind(tok::kw_int),270 AllOf(HasText("main"), Kind(tok::identifier)),271 Kind(tok::l_paren), Kind(tok::r_paren),272 Kind(tok::l_brace), Kind(tok::r_brace)));273 // Comments are ignored for now.274 EXPECT_THAT(tokenize("/* foo */int a; // more comments"),275 ElementsAre(Kind(tok::kw_int),276 AllOf(HasText("a"), Kind(tok::identifier)),277 Kind(tok::semi)));278 EXPECT_THAT(tokenize("int [[main() {]]}"),279 ElementsAre(AllOf(HasText("main"), Kind(tok::identifier)),280 Kind(tok::l_paren), Kind(tok::r_paren),281 Kind(tok::l_brace)));282 EXPECT_THAT(tokenize("int [[main() { ]]}"),283 ElementsAre(AllOf(HasText("main"), Kind(tok::identifier)),284 Kind(tok::l_paren), Kind(tok::r_paren),285 Kind(tok::l_brace)));286 // First token is partially parsed, last token is fully included even though287 // only a part of it is contained in the range.288 EXPECT_THAT(tokenize("int m[[ain() {ret]]urn 0;}"),289 ElementsAre(AllOf(HasText("ain"), Kind(tok::identifier)),290 Kind(tok::l_paren), Kind(tok::r_paren),291 Kind(tok::l_brace), Kind(tok::kw_return)));292}293 294TEST_F(TokenCollectorTest, Basic) {295 std::pair</*Input*/ std::string, /*Expected*/ std::string> TestCases[] = {296 {"int main() {}",297 R"(expanded tokens:298 int main ( ) { }299file './input.cpp'300 spelled tokens:301 int main ( ) { }302 no mappings.303)"},304 // All kinds of whitespace are ignored.305 {"\t\n int\t\n main\t\n (\t\n )\t\n{\t\n }\t\n",306 R"(expanded tokens:307 int main ( ) { }308file './input.cpp'309 spelled tokens:310 int main ( ) { }311 no mappings.312)"},313 // Annotation tokens are ignored.314 {R"cpp(315 #pragma GCC visibility push (public)316 #pragma GCC visibility pop317 )cpp",318 R"(expanded tokens:319 <empty>320file './input.cpp'321 spelled tokens:322 # pragma GCC visibility push ( public ) # pragma GCC visibility pop323 mappings:324 ['#'_0, '<eof>'_13) => ['<eof>'_0, '<eof>'_0)325)"},326 // Empty files should not crash.327 {R"cpp()cpp", R"(expanded tokens:328 <empty>329file './input.cpp'330 spelled tokens:331 <empty>332 no mappings.333)"},334 // Should not crash on errors inside '#define' directives. Error is that335 // stringification (#B) does not refer to a macro parameter.336 {337 R"cpp(338a339#define MACRO() A #B340)cpp",341 R"(expanded tokens:342 a343file './input.cpp'344 spelled tokens:345 a # define MACRO ( ) A # B346 mappings:347 ['#'_1, '<eof>'_9) => ['<eof>'_1, '<eof>'_1)348)"}};349 for (auto &Test : TestCases)350 EXPECT_EQ(collectAndDump(Test.first), Test.second)351 << collectAndDump(Test.first);352}353 354TEST_F(TokenCollectorTest, Locations) {355 // Check locations of the tokens.356 llvm::Annotations Code(R"cpp(357 $r1[[int]] $r2[[a]] $r3[[=]] $r4[["foo bar baz"]] $r5[[;]]358 )cpp");359 recordTokens(Code.code());360 // Check expanded tokens.361 EXPECT_THAT(362 Buffer.expandedTokens(),363 ElementsAre(AllOf(Kind(tok::kw_int), RangeIs(Code.range("r1"))),364 AllOf(Kind(tok::identifier), RangeIs(Code.range("r2"))),365 AllOf(Kind(tok::equal), RangeIs(Code.range("r3"))),366 AllOf(Kind(tok::string_literal), RangeIs(Code.range("r4"))),367 AllOf(Kind(tok::semi), RangeIs(Code.range("r5"))),368 Kind(tok::eof)));369 // Check spelled tokens.370 EXPECT_THAT(371 Buffer.spelledTokens(SourceMgr->getMainFileID()),372 ElementsAre(AllOf(Kind(tok::kw_int), RangeIs(Code.range("r1"))),373 AllOf(Kind(tok::identifier), RangeIs(Code.range("r2"))),374 AllOf(Kind(tok::equal), RangeIs(Code.range("r3"))),375 AllOf(Kind(tok::string_literal), RangeIs(Code.range("r4"))),376 AllOf(Kind(tok::semi), RangeIs(Code.range("r5")))));377 378 auto StartLoc = SourceMgr->getLocForStartOfFile(SourceMgr->getMainFileID());379 for (auto &R : Code.ranges()) {380 EXPECT_THAT(381 Buffer.spelledTokenContaining(StartLoc.getLocWithOffset(R.Begin)),382 Pointee(RangeIs(R)));383 }384}385 386TEST_F(TokenCollectorTest, LocationInMiddleOfSpelledToken) {387 llvm::Annotations Code(R"cpp(388 int foo = [[baa^aar]];389 )cpp");390 recordTokens(Code.code());391 // Check spelled tokens.392 auto StartLoc = SourceMgr->getLocForStartOfFile(SourceMgr->getMainFileID());393 EXPECT_THAT(394 Buffer.spelledTokenContaining(StartLoc.getLocWithOffset(Code.point())),395 Pointee(RangeIs(Code.range())));396}397 398TEST_F(TokenCollectorTest, MacroDirectives) {399 // Macro directives are not stored anywhere at the moment.400 std::string Code = R"cpp(401 #define FOO a402 #include "unresolved_file.h"403 #undef FOO404 #ifdef X405 #else406 #endif407 #ifndef Y408 #endif409 #if 1410 #elif 2411 #else412 #endif413 #pragma once414 #pragma something lalala415 416 int a;417 )cpp";418 std::string Expected =419 "expanded tokens:\n"420 " int a ;\n"421 "file './input.cpp'\n"422 " spelled tokens:\n"423 " # define FOO a # include \"unresolved_file.h\" # undef FOO "424 "# ifdef X # else # endif # ifndef Y # endif # if 1 # elif 2 # else "425 "# endif # pragma once # pragma something lalala int a ;\n"426 " mappings:\n"427 " ['#'_0, 'int'_39) => ['int'_0, 'int'_0)\n";428 EXPECT_EQ(collectAndDump(Code), Expected);429}430 431TEST_F(TokenCollectorTest, MacroReplacements) {432 std::pair</*Input*/ std::string, /*Expected*/ std::string> TestCases[] = {433 // A simple object-like macro.434 {R"cpp(435 #define INT int const436 INT a;437 )cpp",438 R"(expanded tokens:439 int const a ;440file './input.cpp'441 spelled tokens:442 # define INT int const INT a ;443 mappings:444 ['#'_0, 'INT'_5) => ['int'_0, 'int'_0)445 ['INT'_5, 'a'_6) => ['int'_0, 'a'_2)446)"},447 // A simple function-like macro.448 {R"cpp(449 #define INT(a) const int450 INT(10+10) a;451 )cpp",452 R"(expanded tokens:453 const int a ;454file './input.cpp'455 spelled tokens:456 # define INT ( a ) const int INT ( 10 + 10 ) a ;457 mappings:458 ['#'_0, 'INT'_8) => ['const'_0, 'const'_0)459 ['INT'_8, 'a'_14) => ['const'_0, 'a'_2)460)"},461 // Recursive macro replacements.462 {R"cpp(463 #define ID(X) X464 #define INT int const465 ID(ID(INT)) a;466 )cpp",467 R"(expanded tokens:468 int const a ;469file './input.cpp'470 spelled tokens:471 # define ID ( X ) X # define INT int const ID ( ID ( INT ) ) a ;472 mappings:473 ['#'_0, 'ID'_12) => ['int'_0, 'int'_0)474 ['ID'_12, 'a'_19) => ['int'_0, 'a'_2)475)"},476 // A little more complicated recursive macro replacements.477 {R"cpp(478 #define ADD(X, Y) X+Y479 #define MULT(X, Y) X*Y480 481 int a = ADD(MULT(1,2), MULT(3,ADD(4,5)));482 )cpp",483 "expanded tokens:\n"484 " int a = 1 * 2 + 3 * 4 + 5 ;\n"485 "file './input.cpp'\n"486 " spelled tokens:\n"487 " # define ADD ( X , Y ) X + Y # define MULT ( X , Y ) X * Y int "488 "a = ADD ( MULT ( 1 , 2 ) , MULT ( 3 , ADD ( 4 , 5 ) ) ) ;\n"489 " mappings:\n"490 " ['#'_0, 'int'_22) => ['int'_0, 'int'_0)\n"491 " ['ADD'_25, ';'_46) => ['1'_3, ';'_12)\n"},492 // Empty macro replacement.493 // FIXME: the #define directives should not be glued together.494 {R"cpp(495 #define EMPTY496 #define EMPTY_FUNC(X)497 EMPTY498 EMPTY_FUNC(1+2+3)499 )cpp",500 R"(expanded tokens:501 <empty>502file './input.cpp'503 spelled tokens:504 # define EMPTY # define EMPTY_FUNC ( X ) EMPTY EMPTY_FUNC ( 1 + 2 + 3 )505 mappings:506 ['#'_0, 'EMPTY'_9) => ['<eof>'_0, '<eof>'_0)507 ['EMPTY'_9, 'EMPTY_FUNC'_10) => ['<eof>'_0, '<eof>'_0)508 ['EMPTY_FUNC'_10, '<eof>'_18) => ['<eof>'_0, '<eof>'_0)509)"},510 // File ends with a macro replacement.511 {R"cpp(512 #define FOO 10+10;513 int a = FOO514 )cpp",515 R"(expanded tokens:516 int a = 10 + 10 ;517file './input.cpp'518 spelled tokens:519 # define FOO 10 + 10 ; int a = FOO520 mappings:521 ['#'_0, 'int'_7) => ['int'_0, 'int'_0)522 ['FOO'_10, '<eof>'_11) => ['10'_3, '<eof>'_7)523)"},524 {R"cpp(525 #define NUM 42526 #define ID(a) a527 #define M 1 + ID528 M(NUM)529 )cpp",530 R"(expanded tokens:531 1 + 42532file './input.cpp'533 spelled tokens:534 # define NUM 42 # define ID ( a ) a # define M 1 + ID M ( NUM )535 mappings:536 ['#'_0, 'M'_17) => ['1'_0, '1'_0)537 ['M'_17, '<eof>'_21) => ['1'_0, '<eof>'_3)538)"},539 };540 541 for (auto &Test : TestCases) {542 std::string Dump = collectAndDump(Test.first);543 EXPECT_EQ(Test.second, Dump) << Dump;544 }545}546 547TEST_F(TokenCollectorTest, SpecialTokens) {548 // Tokens coming from concatenations.549 recordTokens(R"cpp(550 #define CONCAT(a, b) a ## b551 int a = CONCAT(1, 2);552 )cpp");553 EXPECT_THAT(std::vector<syntax::Token>(Buffer.expandedTokens()),554 Contains(HasText("12")));555 // Multi-line tokens with slashes at the end.556 recordTokens("i\\\nn\\\nt");557 EXPECT_THAT(Buffer.expandedTokens(),558 ElementsAre(AllOf(Kind(tok::kw_int), HasText("i\\\nn\\\nt")),559 Kind(tok::eof)));560 // FIXME: test tokens with digraphs and UCN identifiers.561}562 563TEST_F(TokenCollectorTest, LateBoundTokens) {564 // The parser eventually breaks the first '>>' into two tokens ('>' and '>'),565 // but we choose to record them as a single token (for now).566 llvm::Annotations Code(R"cpp(567 template <class T>568 struct foo { int a; };569 int bar = foo<foo<int$br[[>>]]().a;570 int baz = 10 $op[[>>]] 2;571 )cpp");572 recordTokens(Code.code());573 EXPECT_THAT(std::vector<syntax::Token>(Buffer.expandedTokens()),574 AllOf(Contains(AllOf(Kind(tok::greatergreater),575 RangeIs(Code.range("br")))),576 Contains(AllOf(Kind(tok::greatergreater),577 RangeIs(Code.range("op"))))));578}579 580TEST_F(TokenCollectorTest, DelayedParsing) {581 llvm::StringLiteral Code = R"cpp(582 struct Foo {583 int method() {584 // Parser will visit method bodies and initializers multiple times, but585 // TokenBuffer should only record the first walk over the tokens;586 return 100;587 }588 int a = 10;589 590 struct Subclass {591 void foo() {592 Foo().method();593 }594 };595 };596 )cpp";597 std::string ExpectedTokens =598 "expanded tokens:\n"599 " struct Foo { int method ( ) { return 100 ; } int a = 10 ; struct "600 "Subclass { void foo ( ) { Foo ( ) . method ( ) ; } } ; } ;\n";601 EXPECT_THAT(collectAndDump(Code), StartsWith(ExpectedTokens));602}603 604TEST_F(TokenCollectorTest, MultiFile) {605 addFile("./foo.h", R"cpp(606 #define ADD(X, Y) X+Y607 int a = 100;608 #include "bar.h"609 )cpp");610 addFile("./bar.h", R"cpp(611 int b = ADD(1, 2);612 #define MULT(X, Y) X*Y613 )cpp");614 llvm::StringLiteral Code = R"cpp(615 #include "foo.h"616 int c = ADD(1, MULT(2,3));617 )cpp";618 619 std::string Expected = R"(expanded tokens:620 int a = 100 ; int b = 1 + 2 ; int c = 1 + 2 * 3 ;621file './input.cpp'622 spelled tokens:623 # include "foo.h" int c = ADD ( 1 , MULT ( 2 , 3 ) ) ;624 mappings:625 ['#'_0, 'int'_3) => ['int'_12, 'int'_12)626 ['ADD'_6, ';'_17) => ['1'_15, ';'_20)627file './foo.h'628 spelled tokens:629 # define ADD ( X , Y ) X + Y int a = 100 ; # include "bar.h"630 mappings:631 ['#'_0, 'int'_11) => ['int'_0, 'int'_0)632 ['#'_16, '<eof>'_19) => ['int'_5, 'int'_5)633file './bar.h'634 spelled tokens:635 int b = ADD ( 1 , 2 ) ; # define MULT ( X , Y ) X * Y636 mappings:637 ['ADD'_3, ';'_9) => ['1'_8, ';'_11)638 ['#'_10, '<eof>'_21) => ['int'_12, 'int'_12)639)";640 641 EXPECT_EQ(Expected, collectAndDump(Code))642 << "input: " << Code << "\nresults: " << collectAndDump(Code);643}644 645class TokenBufferTest : public TokenCollectorTest {};646 647TEST_F(TokenBufferTest, SpelledByExpanded) {648 recordTokens(R"cpp(649 a1 a2 a3 b1 b2650 )cpp");651 652 // Expanded and spelled tokens are stored separately.653 EXPECT_THAT(findExpanded("a1 a2"), Not(SameRange(findSpelled("a1 a2"))));654 // Searching for subranges of expanded tokens should give the corresponding655 // spelled ones.656 EXPECT_THAT(Buffer.spelledForExpanded(findExpanded("a1 a2 a3 b1 b2")),657 ValueIs(SameRange(findSpelled("a1 a2 a3 b1 b2"))));658 EXPECT_THAT(Buffer.spelledForExpanded(findExpanded("a1 a2 a3")),659 ValueIs(SameRange(findSpelled("a1 a2 a3"))));660 EXPECT_THAT(Buffer.spelledForExpanded(findExpanded("b1 b2")),661 ValueIs(SameRange(findSpelled("b1 b2"))));662 663 // Test search on simple macro expansions.664 recordTokens(R"cpp(665 #define A a1 a2 a3666 #define B b1 b2667 668 A split B669 )cpp");670 // Ranges going across expansion boundaries.671 EXPECT_THAT(Buffer.spelledForExpanded(findExpanded("a1 a2 a3 split b1 b2")),672 ValueIs(SameRange(findSpelled("A split B"))));673 EXPECT_THAT(Buffer.spelledForExpanded(findExpanded("a1 a2 a3")),674 ValueIs(SameRange(findSpelled("A split").drop_back())));675 EXPECT_THAT(Buffer.spelledForExpanded(findExpanded("b1 b2")),676 ValueIs(SameRange(findSpelled("split B").drop_front())));677 // Ranges not fully covering macro invocations should fail.678 EXPECT_EQ(Buffer.spelledForExpanded(findExpanded("a1 a2")), std::nullopt);679 EXPECT_EQ(Buffer.spelledForExpanded(findExpanded("b2")), std::nullopt);680 EXPECT_EQ(Buffer.spelledForExpanded(findExpanded("a2 a3 split b1 b2")),681 std::nullopt);682 683 // Recursive macro invocations.684 recordTokens(R"cpp(685 #define ID(x) x686 #define B b1 b2687 688 ID(ID(ID(a1) a2 a3)) split ID(B)689 )cpp");690 691 EXPECT_THAT(Buffer.spelledForExpanded(findExpanded("b1 b2")),692 ValueIs(SameRange(findSpelled("( B").drop_front())));693 EXPECT_THAT(Buffer.spelledForExpanded(findExpanded("a1 a2 a3 split b1 b2")),694 ValueIs(SameRange(findSpelled(695 "ID ( ID ( ID ( a1 ) a2 a3 ) ) split ID ( B )"))));696 // Mixed ranges with expanded and spelled tokens.697 EXPECT_THAT(698 Buffer.spelledForExpanded(findExpanded("a1 a2 a3 split")),699 ValueIs(SameRange(findSpelled("ID ( ID ( ID ( a1 ) a2 a3 ) ) split"))));700 EXPECT_THAT(Buffer.spelledForExpanded(findExpanded("split b1 b2")),701 ValueIs(SameRange(findSpelled("split ID ( B )"))));702 // Macro arguments703 EXPECT_THAT(Buffer.spelledForExpanded(findExpanded("a1")),704 ValueIs(SameRange(findSpelled("a1"))));705 EXPECT_THAT(Buffer.spelledForExpanded(findExpanded("a2")),706 ValueIs(SameRange(findSpelled("a2"))));707 EXPECT_THAT(Buffer.spelledForExpanded(findExpanded("a3")),708 ValueIs(SameRange(findSpelled("a3"))));709 EXPECT_THAT(Buffer.spelledForExpanded(findExpanded("a1 a2")),710 ValueIs(SameRange(findSpelled("ID ( a1 ) a2"))));711 EXPECT_THAT(Buffer.spelledForExpanded(findExpanded("a1 a2 a3")),712 ValueIs(SameRange(findSpelled("ID ( a1 ) a2 a3"))));713 714 // Empty macro expansions.715 recordTokens(R"cpp(716 #define EMPTY717 #define ID(X) X718 719 EMPTY EMPTY ID(1 2 3) EMPTY EMPTY split1720 EMPTY EMPTY ID(4 5 6) split2721 ID(7 8 9) EMPTY EMPTY722 )cpp");723 EXPECT_THAT(Buffer.spelledForExpanded(findExpanded("1 2 3")),724 ValueIs(SameRange(findSpelled("1 2 3"))));725 EXPECT_THAT(Buffer.spelledForExpanded(findExpanded("4 5 6")),726 ValueIs(SameRange(findSpelled("4 5 6"))));727 EXPECT_THAT(Buffer.spelledForExpanded(findExpanded("7 8 9")),728 ValueIs(SameRange(findSpelled("7 8 9"))));729 730 // Empty mappings coming from various directives.731 recordTokens(R"cpp(732 #define ID(X) X733 ID(1)734 #pragma lalala735 not_mapped736 )cpp");737 EXPECT_THAT(Buffer.spelledForExpanded(findExpanded("not_mapped")),738 ValueIs(SameRange(findSpelled("not_mapped"))));739 740 // Multiple macro arguments741 recordTokens(R"cpp(742 #define ID(X) X743 #define ID2(X, Y) X Y744 745 ID2(ID(a1), ID(a2) a3) ID2(a4, a5 a6 a7)746 )cpp");747 // Should fail, spans multiple arguments.748 EXPECT_EQ(Buffer.spelledForExpanded(findExpanded("a1 a2")), std::nullopt);749 EXPECT_THAT(Buffer.spelledForExpanded(findExpanded("a2 a3")),750 ValueIs(SameRange(findSpelled("ID ( a2 ) a3"))));751 EXPECT_THAT(752 Buffer.spelledForExpanded(findExpanded("a1 a2 a3")),753 ValueIs(SameRange(findSpelled("ID2 ( ID ( a1 ) , ID ( a2 ) a3 )"))));754 EXPECT_THAT(Buffer.spelledForExpanded(findExpanded("a5 a6")),755 ValueIs(SameRange(findSpelled("a5 a6"))));756 EXPECT_THAT(Buffer.spelledForExpanded(findExpanded("a4 a5 a6 a7")),757 ValueIs(SameRange(findSpelled("ID2 ( a4 , a5 a6 a7 )"))));758 // Should fail, spans multiple invocations.759 EXPECT_EQ(Buffer.spelledForExpanded(findExpanded("a1 a2 a3 a4")),760 std::nullopt);761 762 // https://github.com/clangd/clangd/issues/1289763 recordTokens(R"cpp(764 #define FOO(X) foo(X)765 #define INDIRECT FOO(y)766 INDIRECT // expands to foo(y)767 )cpp");768 EXPECT_EQ(Buffer.spelledForExpanded(findExpanded("y")), std::nullopt);769 770 recordTokens(R"cpp(771 #define FOO(X) a X b772 FOO(y)773 )cpp");774 EXPECT_THAT(Buffer.spelledForExpanded(findExpanded("y")),775 ValueIs(SameRange(findSpelled("y"))));776 777 recordTokens(R"cpp(778 #define ID(X) X779 #define BAR ID(1)780 BAR781 )cpp");782 EXPECT_THAT(Buffer.spelledForExpanded(findExpanded("1")),783 ValueIs(SameRange(findSpelled(") BAR").drop_front())));784 785 // Critical cases for mapping of Prev/Next in spelledForExpandedSlow.786 recordTokens(R"cpp(787 #define ID(X) X788 ID(prev good)789 ID(prev ID(good2))790 #define LARGE ID(prev ID(bad))791 LARGE792 )cpp");793 EXPECT_THAT(Buffer.spelledForExpanded(findExpanded("good")),794 ValueIs(SameRange(findSpelled("good"))));795 EXPECT_THAT(Buffer.spelledForExpanded(findExpanded("good2")),796 ValueIs(SameRange(findSpelled("good2"))));797 EXPECT_EQ(Buffer.spelledForExpanded(findExpanded("bad")), std::nullopt);798 799 recordTokens(R"cpp(800 #define PREV prev801 #define ID(X) X802 PREV ID(good)803 #define LARGE PREV ID(bad)804 LARGE805 )cpp");806 EXPECT_THAT(Buffer.spelledForExpanded(findExpanded("good")),807 ValueIs(SameRange(findSpelled("good"))));808 EXPECT_EQ(Buffer.spelledForExpanded(findExpanded("bad")), std::nullopt);809 810 recordTokens(R"cpp(811 #define ID(X) X812 #define ID2(X, Y) X Y813 ID2(prev, good)814 ID2(prev, ID(good2))815 #define LARGE ID2(prev, bad)816 LARGE817 )cpp");818 EXPECT_THAT(Buffer.spelledForExpanded(findExpanded("good")),819 ValueIs(SameRange(findSpelled("good"))));820 EXPECT_THAT(Buffer.spelledForExpanded(findExpanded("good2")),821 ValueIs(SameRange(findSpelled("good2"))));822 EXPECT_EQ(Buffer.spelledForExpanded(findExpanded("bad")), std::nullopt);823 824 // Prev from macro body.825 recordTokens(R"cpp(826 #define ID(X) X827 #define ID2(X, Y) X prev ID(Y)828 ID2(not_prev, good)829 )cpp");830 EXPECT_THAT(Buffer.spelledForExpanded(findExpanded("good")),831 ValueIs(SameRange(findSpelled("good"))));832 EXPECT_EQ(Buffer.spelledForExpanded(findExpanded("prev good")), std::nullopt);833}834 835TEST_F(TokenBufferTest, NoCrashForEofToken) {836 recordTokens(R"cpp(837 int main() {838 )cpp");839 ASSERT_TRUE(!Buffer.expandedTokens().empty());840 ASSERT_EQ(Buffer.expandedTokens().back().kind(), tok::eof);841 // Expanded range including `eof` is handled gracefully (`eof` is ignored).842 EXPECT_THAT(843 Buffer.spelledForExpanded(Buffer.expandedTokens()),844 ValueIs(SameRange(Buffer.spelledTokens(SourceMgr->getMainFileID()))));845}846 847TEST_F(TokenBufferTest, ExpandedTokensForRange) {848 recordTokens(R"cpp(849 #define SIGN(X) X##_washere850 A SIGN(B) C SIGN(D) E SIGN(F) G851 )cpp");852 853 SourceRange R(findExpanded("C").front().location(),854 findExpanded("F_washere").front().location());855 // Expanded and spelled tokens are stored separately.856 EXPECT_THAT(Buffer.expandedTokens(R),857 SameRange(findExpanded("C D_washere E F_washere")));858 EXPECT_THAT(Buffer.expandedTokens(SourceRange()), testing::IsEmpty());859}860 861TEST_F(TokenBufferTest, ExpansionsOverlapping) {862 // Object-like macro expansions.863 recordTokens(R"cpp(864 #define FOO 3+4865 int a = FOO 1;866 int b = FOO 2;867 )cpp");868 869 llvm::ArrayRef<syntax::Token> Foo1 = findSpelled("FOO 1");870 EXPECT_THAT(871 Buffer.expansionStartingAt(Foo1.data()),872 ValueIs(IsExpansion(SameRange(Foo1.drop_back()),873 SameRange(findExpanded("3 + 4 1").drop_back()))));874 EXPECT_THAT(875 Buffer.expansionsOverlapping(Foo1),876 ElementsAre(IsExpansion(SameRange(Foo1.drop_back()),877 SameRange(findExpanded("3 + 4 1").drop_back()))));878 879 llvm::ArrayRef<syntax::Token> Foo2 = findSpelled("FOO 2");880 EXPECT_THAT(881 Buffer.expansionStartingAt(Foo2.data()),882 ValueIs(IsExpansion(SameRange(Foo2.drop_back()),883 SameRange(findExpanded("3 + 4 2").drop_back()))));884 EXPECT_THAT(885 Buffer.expansionsOverlapping(llvm::ArrayRef(Foo1.begin(), Foo2.end())),886 ElementsAre(IsExpansion(SameRange(Foo1.drop_back()), _),887 IsExpansion(SameRange(Foo2.drop_back()), _)));888 889 // Function-like macro expansions.890 recordTokens(R"cpp(891 #define ID(X) X892 int a = ID(1+2+3);893 int b = ID(ID(2+3+4));894 )cpp");895 896 llvm::ArrayRef<syntax::Token> ID1 = findSpelled("ID ( 1 + 2 + 3 )");897 EXPECT_THAT(Buffer.expansionStartingAt(&ID1.front()),898 ValueIs(IsExpansion(SameRange(ID1),899 SameRange(findExpanded("1 + 2 + 3")))));900 // Only the first spelled token should be found.901 for (const auto &T : ID1.drop_front())902 EXPECT_EQ(Buffer.expansionStartingAt(&T), std::nullopt);903 904 llvm::ArrayRef<syntax::Token> ID2 = findSpelled("ID ( ID ( 2 + 3 + 4 ) )");905 EXPECT_THAT(Buffer.expansionStartingAt(&ID2.front()),906 ValueIs(IsExpansion(SameRange(ID2),907 SameRange(findExpanded("2 + 3 + 4")))));908 // Only the first spelled token should be found.909 for (const auto &T : ID2.drop_front())910 EXPECT_EQ(Buffer.expansionStartingAt(&T), std::nullopt);911 912 EXPECT_THAT(Buffer.expansionsOverlapping(llvm::ArrayRef(913 findSpelled("1 + 2").data(), findSpelled("4").data())),914 ElementsAre(IsExpansion(SameRange(ID1), _),915 IsExpansion(SameRange(ID2), _)));916 917 // PP directives.918 recordTokens(R"cpp(919#define FOO 1920int a = FOO;921#pragma once922int b = 1;923 )cpp");924 925 llvm::ArrayRef<syntax::Token> DefineFoo = findSpelled("# define FOO 1");926 EXPECT_THAT(927 Buffer.expansionStartingAt(&DefineFoo.front()),928 ValueIs(IsExpansion(SameRange(DefineFoo),929 SameRange(findExpanded("int a").take_front(0)))));930 // Only the first spelled token should be found.931 for (const auto &T : DefineFoo.drop_front())932 EXPECT_EQ(Buffer.expansionStartingAt(&T), std::nullopt);933 934 llvm::ArrayRef<syntax::Token> PragmaOnce = findSpelled("# pragma once");935 EXPECT_THAT(936 Buffer.expansionStartingAt(&PragmaOnce.front()),937 ValueIs(IsExpansion(SameRange(PragmaOnce),938 SameRange(findExpanded("int b").take_front(0)))));939 // Only the first spelled token should be found.940 for (const auto &T : PragmaOnce.drop_front())941 EXPECT_EQ(Buffer.expansionStartingAt(&T), std::nullopt);942 943 EXPECT_THAT(944 Buffer.expansionsOverlapping(findSpelled("FOO ; # pragma")),945 ElementsAre(IsExpansion(SameRange(findSpelled("FOO ;").drop_back()), _),946 IsExpansion(SameRange(PragmaOnce), _)));947}948 949TEST_F(TokenBufferTest, TokensToFileRange) {950 addFile("./foo.h", "token_from_header");951 llvm::Annotations Code(R"cpp(952 #define FOO token_from_expansion953 #include "./foo.h"954 $all[[$i[[int]] a = FOO;]]955 )cpp");956 recordTokens(Code.code());957 958 auto &SM = *SourceMgr;959 960 // Two simple examples.961 auto Int = findExpanded("int").front();962 auto Semi = findExpanded(";").front();963 EXPECT_EQ(Int.range(SM), FileRange(SM.getMainFileID(), Code.range("i").Begin,964 Code.range("i").End));965 EXPECT_EQ(syntax::Token::range(SM, Int, Semi),966 FileRange(SM.getMainFileID(), Code.range("all").Begin,967 Code.range("all").End));968 // We don't test assertion failures because death tests are slow.969}970 971TEST_F(TokenBufferTest, MacroExpansions) {972 llvm::Annotations Code(R"cpp(973 #define FOO B974 #define FOO2 BA975 #define CALL(X) int X976 #define G CALL(FOO2)977 int B;978 $macro[[FOO]];979 $macro[[CALL]](A);980 $macro[[G]];981 )cpp");982 recordTokens(Code.code());983 auto &SM = *SourceMgr;984 auto Expansions = Buffer.macroExpansions(SM.getMainFileID());985 std::vector<FileRange> ExpectedMacroRanges;986 for (auto Range : Code.ranges("macro"))987 ExpectedMacroRanges.push_back(988 FileRange(SM.getMainFileID(), Range.Begin, Range.End));989 std::vector<FileRange> ActualMacroRanges;990 for (auto Expansion : Expansions)991 ActualMacroRanges.push_back(Expansion->range(SM));992 EXPECT_EQ(ExpectedMacroRanges, ActualMacroRanges);993}994 995TEST_F(TokenBufferTest, Touching) {996 llvm::Annotations Code("^i^nt^ ^a^b^=^1;^");997 recordTokens(Code.code());998 999 auto Touching = [&](int Index) {1000 SourceLocation Loc = SourceMgr->getComposedLoc(SourceMgr->getMainFileID(),1001 Code.points()[Index]);1002 return spelledTokensTouching(Loc, Buffer);1003 };1004 auto Identifier = [&](int Index) {1005 SourceLocation Loc = SourceMgr->getComposedLoc(SourceMgr->getMainFileID(),1006 Code.points()[Index]);1007 const syntax::Token *Tok = spelledIdentifierTouching(Loc, Buffer);1008 return Tok ? Tok->text(*SourceMgr) : "";1009 };1010 1011 EXPECT_THAT(Touching(0), SameRange(findSpelled("int")));1012 EXPECT_EQ(Identifier(0), "");1013 EXPECT_THAT(Touching(1), SameRange(findSpelled("int")));1014 EXPECT_EQ(Identifier(1), "");1015 EXPECT_THAT(Touching(2), SameRange(findSpelled("int")));1016 EXPECT_EQ(Identifier(2), "");1017 1018 EXPECT_THAT(Touching(3), SameRange(findSpelled("ab")));1019 EXPECT_EQ(Identifier(3), "ab");1020 EXPECT_THAT(Touching(4), SameRange(findSpelled("ab")));1021 EXPECT_EQ(Identifier(4), "ab");1022 1023 EXPECT_THAT(Touching(5), SameRange(findSpelled("ab =")));1024 EXPECT_EQ(Identifier(5), "ab");1025 1026 EXPECT_THAT(Touching(6), SameRange(findSpelled("= 1")));1027 EXPECT_EQ(Identifier(6), "");1028 1029 EXPECT_THAT(Touching(7), SameRange(findSpelled(";")));1030 EXPECT_EQ(Identifier(7), "");1031 1032 ASSERT_EQ(Code.points().size(), 8u);1033}1034 1035TEST_F(TokenBufferTest, ExpandedBySpelled) {1036 recordTokens(R"cpp(1037 a1 a2 a3 b1 b21038 )cpp");1039 // Expanded and spelled tokens are stored separately.1040 EXPECT_THAT(findExpanded("a1 a2"), Not(SameRange(findSpelled("a1 a2"))));1041 // Searching for subranges of expanded tokens should give the corresponding1042 // spelled ones.1043 EXPECT_THAT(Buffer.expandedForSpelled(findSpelled("a1 a2 a3 b1 b2")),1044 ElementsAre(SameRange(findExpanded("a1 a2 a3 b1 b2"))));1045 EXPECT_THAT(Buffer.expandedForSpelled(findSpelled("a1 a2 a3")),1046 ElementsAre(SameRange(findExpanded("a1 a2 a3"))));1047 EXPECT_THAT(Buffer.expandedForSpelled(findSpelled("b1 b2")),1048 ElementsAre(SameRange(findExpanded("b1 b2"))));1049 1050 // Test search on simple macro expansions.1051 recordTokens(R"cpp(1052 #define A a1 a2 a31053 #define B b1 b21054 1055 A split B1056 )cpp");1057 EXPECT_THAT(Buffer.expandedForSpelled(findSpelled("A split B")),1058 ElementsAre(SameRange(findExpanded("a1 a2 a3 split b1 b2"))));1059 EXPECT_THAT(Buffer.expandedForSpelled(findSpelled("A split").drop_back()),1060 ElementsAre(SameRange(findExpanded("a1 a2 a3"))));1061 EXPECT_THAT(Buffer.expandedForSpelled(findSpelled("split B").drop_front()),1062 ElementsAre(SameRange(findExpanded("b1 b2"))));1063 1064 // Ranges not fully covering macro expansions should fail.1065 recordTokens(R"cpp(1066 #define ID(x) x1067 1068 ID(a)1069 )cpp");1070 // Spelled don't cover entire mapping (missing ID token) -> empty result1071 EXPECT_THAT(Buffer.expandedForSpelled(findSpelled("( a )")), IsEmpty());1072 // Spelled don't cover entire mapping (missing ) token) -> empty result1073 EXPECT_THAT(Buffer.expandedForSpelled(findSpelled("ID ( a")), IsEmpty());1074 1075 // Recursive macro invocations.1076 recordTokens(R"cpp(1077 #define ID(x) x1078 #define B b1 b21079 1080 ID(ID(ID(a1) a2 a3)) split ID(B)1081 )cpp");1082 1083 EXPECT_THAT(1084 Buffer.expandedForSpelled(findSpelled("ID ( ID ( ID ( a1 ) a2 a3 ) )")),1085 ElementsAre(SameRange(findExpanded("a1 a2 a3"))));1086 EXPECT_THAT(Buffer.expandedForSpelled(findSpelled("ID ( B )")),1087 ElementsAre(SameRange(findExpanded("b1 b2"))));1088 EXPECT_THAT(Buffer.expandedForSpelled(1089 findSpelled("ID ( ID ( ID ( a1 ) a2 a3 ) ) split ID ( B )")),1090 ElementsAre(SameRange(findExpanded("a1 a2 a3 split b1 b2"))));1091 // FIXME: these should succeed, but we do not support macro arguments yet.1092 EXPECT_THAT(Buffer.expandedForSpelled(findSpelled("a1")), IsEmpty());1093 EXPECT_THAT(Buffer.expandedForSpelled(findSpelled("ID ( a1 ) a2")),1094 IsEmpty());1095 1096 // Empty macro expansions.1097 recordTokens(R"cpp(1098 #define EMPTY1099 #define ID(X) X1100 1101 EMPTY EMPTY ID(1 2 3) EMPTY EMPTY split11102 EMPTY EMPTY ID(4 5 6) split21103 ID(7 8 9) EMPTY EMPTY1104 )cpp");1105 // Covered by empty expansions on one of both of the sides.1106 EXPECT_THAT(Buffer.expandedForSpelled(findSpelled("ID ( 1 2 3 )")),1107 ElementsAre(SameRange(findExpanded("1 2 3"))));1108 EXPECT_THAT(Buffer.expandedForSpelled(findSpelled("ID ( 4 5 6 )")),1109 ElementsAre(SameRange(findExpanded("4 5 6"))));1110 EXPECT_THAT(Buffer.expandedForSpelled(findSpelled("ID ( 7 8 9 )")),1111 ElementsAre(SameRange(findExpanded("7 8 9"))));1112 // Including the empty macro expansions on the side.1113 EXPECT_THAT(Buffer.expandedForSpelled(findSpelled("EMPTY ID ( 1 2 3 )")),1114 ElementsAre(SameRange(findExpanded("1 2 3"))));1115 EXPECT_THAT(Buffer.expandedForSpelled(findSpelled("ID ( 1 2 3 ) EMPTY")),1116 ElementsAre(SameRange(findExpanded("1 2 3"))));1117 EXPECT_THAT(1118 Buffer.expandedForSpelled(findSpelled("EMPTY ID ( 1 2 3 ) EMPTY")),1119 ElementsAre(SameRange(findExpanded("1 2 3"))));1120 1121 // Empty mappings coming from various directives.1122 recordTokens(R"cpp(1123 #define ID(X) X1124 ID(1)1125 #pragma lalala1126 not_mapped1127 )cpp");1128 EXPECT_THAT(Buffer.expandedForSpelled(findSpelled("# define ID ( X ) X")),1129 IsEmpty());1130 EXPECT_THAT(Buffer.expandedForSpelled(findSpelled("# pragma lalala")),1131 IsEmpty());1132 1133 // Empty macro expansion.1134 recordTokens(R"cpp(1135 #define EMPTY1136 EMPTY int a = 100;1137 )cpp");1138 EXPECT_THAT(Buffer.expandedForSpelled(findSpelled("EMPTY int").drop_back()),1139 IsEmpty());1140}1141 1142TEST_F(TokenCollectorTest, Pragmas) {1143 // Tokens coming from concatenations.1144 recordTokens(R"cpp(1145 void foo() {1146 #pragma unroll 41147 for(int i=0;i<4;++i);1148 }1149 )cpp");1150}1151} // namespace1152