820 lines · cpp
1//===- unittests/Lex/LexerTest.cpp ------ Lexer tests ---------------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8 9#include "clang/Lex/Lexer.h"10#include "clang/Basic/Diagnostic.h"11#include "clang/Basic/DiagnosticOptions.h"12#include "clang/Basic/FileManager.h"13#include "clang/Basic/LangOptions.h"14#include "clang/Basic/SourceLocation.h"15#include "clang/Basic/SourceManager.h"16#include "clang/Basic/TargetInfo.h"17#include "clang/Basic/TargetOptions.h"18#include "clang/Basic/TokenKinds.h"19#include "clang/Lex/HeaderSearch.h"20#include "clang/Lex/HeaderSearchOptions.h"21#include "clang/Lex/LiteralSupport.h"22#include "clang/Lex/MacroArgs.h"23#include "clang/Lex/MacroInfo.h"24#include "clang/Lex/ModuleLoader.h"25#include "clang/Lex/Preprocessor.h"26#include "clang/Lex/PreprocessorOptions.h"27#include "llvm/ADT/ArrayRef.h"28#include "llvm/ADT/StringRef.h"29#include "llvm/Testing/Annotations/Annotations.h"30#include "gmock/gmock.h"31#include "gtest/gtest.h"32#include <memory>33#include <string>34#include <vector>35 36namespace {37using namespace clang;38using testing::ElementsAre;39 40// The test fixture.41class LexerTest : public ::testing::Test {42protected:43 LexerTest()44 : FileMgr(FileMgrOpts),45 Diags(DiagnosticIDs::create(), DiagOpts, new IgnoringDiagConsumer()),46 SourceMgr(Diags, FileMgr), TargetOpts(new TargetOptions) {47 TargetOpts->Triple = "x86_64-apple-darwin11.1.0";48 Target = TargetInfo::CreateTargetInfo(Diags, *TargetOpts);49 }50 51 std::unique_ptr<Preprocessor> CreatePP(StringRef Source,52 TrivialModuleLoader &ModLoader) {53 std::unique_ptr<llvm::MemoryBuffer> Buf =54 llvm::MemoryBuffer::getMemBuffer(Source);55 SourceMgr.setMainFileID(SourceMgr.createFileID(std::move(Buf)));56 57 HeaderSearchOptions HSOpts;58 HeaderSearch HeaderInfo(HSOpts, SourceMgr, Diags, LangOpts, Target.get());59 PreprocessorOptions PPOpts;60 std::unique_ptr<Preprocessor> PP = std::make_unique<Preprocessor>(61 PPOpts, Diags, LangOpts, SourceMgr, HeaderInfo, ModLoader,62 /*IILookup =*/nullptr,63 /*OwnsHeaderSearch =*/false);64 if (!PreDefines.empty())65 PP->setPredefines(PreDefines);66 PP->Initialize(*Target);67 PP->EnterMainSourceFile();68 return PP;69 }70 71 std::vector<Token> Lex(StringRef Source) {72 TrivialModuleLoader ModLoader;73 PP = CreatePP(Source, ModLoader);74 75 std::vector<Token> toks;76 PP->LexTokensUntilEOF(&toks);77 78 return toks;79 }80 81 std::vector<Token> CheckLex(StringRef Source,82 ArrayRef<tok::TokenKind> ExpectedTokens) {83 auto toks = Lex(Source);84 EXPECT_EQ(ExpectedTokens.size(), toks.size());85 for (unsigned i = 0, e = ExpectedTokens.size(); i != e; ++i) {86 EXPECT_EQ(ExpectedTokens[i], toks[i].getKind());87 }88 89 return toks;90 }91 92 std::string getSourceText(Token Begin, Token End) {93 bool Invalid;94 StringRef Str =95 Lexer::getSourceText(CharSourceRange::getTokenRange(SourceRange(96 Begin.getLocation(), End.getLocation())),97 SourceMgr, LangOpts, &Invalid);98 if (Invalid)99 return "<INVALID>";100 return std::string(Str);101 }102 103 FileSystemOptions FileMgrOpts;104 FileManager FileMgr;105 DiagnosticOptions DiagOpts;106 DiagnosticsEngine Diags;107 SourceManager SourceMgr;108 LangOptions LangOpts;109 std::shared_ptr<TargetOptions> TargetOpts;110 IntrusiveRefCntPtr<TargetInfo> Target;111 std::unique_ptr<Preprocessor> PP;112 std::string PreDefines;113};114 115TEST_F(LexerTest, GetSourceTextExpandsToMaximumInMacroArgument) {116 std::vector<tok::TokenKind> ExpectedTokens;117 ExpectedTokens.push_back(tok::identifier);118 ExpectedTokens.push_back(tok::l_paren);119 ExpectedTokens.push_back(tok::identifier);120 ExpectedTokens.push_back(tok::r_paren);121 122 std::vector<Token> toks = CheckLex("#define M(x) x\n"123 "M(f(M(i)))",124 ExpectedTokens);125 126 EXPECT_EQ("M(i)", getSourceText(toks[2], toks[2]));127}128 129TEST_F(LexerTest, GetSourceTextExpandsToMaximumInMacroArgumentForEndOfMacro) {130 std::vector<tok::TokenKind> ExpectedTokens;131 ExpectedTokens.push_back(tok::identifier);132 ExpectedTokens.push_back(tok::identifier);133 134 std::vector<Token> toks = CheckLex("#define M(x) x\n"135 "M(M(i) c)",136 ExpectedTokens);137 138 EXPECT_EQ("M(i)", getSourceText(toks[0], toks[0]));139}140 141TEST_F(LexerTest, GetSourceTextExpandsInMacroArgumentForBeginOfMacro) {142 std::vector<tok::TokenKind> ExpectedTokens;143 ExpectedTokens.push_back(tok::identifier);144 ExpectedTokens.push_back(tok::identifier);145 ExpectedTokens.push_back(tok::identifier);146 147 std::vector<Token> toks = CheckLex("#define M(x) x\n"148 "M(c c M(i))",149 ExpectedTokens);150 151 EXPECT_EQ("c M(i)", getSourceText(toks[1], toks[2]));152}153 154TEST_F(LexerTest, GetSourceTextExpandsInMacroArgumentForEndOfMacro) {155 std::vector<tok::TokenKind> ExpectedTokens;156 ExpectedTokens.push_back(tok::identifier);157 ExpectedTokens.push_back(tok::identifier);158 ExpectedTokens.push_back(tok::identifier);159 160 std::vector<Token> toks = CheckLex("#define M(x) x\n"161 "M(M(i) c c)",162 ExpectedTokens);163 164 EXPECT_EQ("M(i) c", getSourceText(toks[0], toks[1]));165}166 167TEST_F(LexerTest, GetSourceTextInSeparateFnMacros) {168 std::vector<tok::TokenKind> ExpectedTokens;169 ExpectedTokens.push_back(tok::identifier);170 ExpectedTokens.push_back(tok::identifier);171 ExpectedTokens.push_back(tok::identifier);172 ExpectedTokens.push_back(tok::identifier);173 174 std::vector<Token> toks = CheckLex("#define M(x) x\n"175 "M(c M(i)) M(M(i) c)",176 ExpectedTokens);177 178 EXPECT_EQ("<INVALID>", getSourceText(toks[1], toks[2]));179}180 181TEST_F(LexerTest, GetSourceTextWorksAcrossTokenPastes) {182 std::vector<tok::TokenKind> ExpectedTokens;183 ExpectedTokens.push_back(tok::identifier);184 ExpectedTokens.push_back(tok::l_paren);185 ExpectedTokens.push_back(tok::identifier);186 ExpectedTokens.push_back(tok::r_paren);187 188 std::vector<Token> toks = CheckLex("#define M(x) x\n"189 "#define C(x) M(x##c)\n"190 "M(f(C(i)))",191 ExpectedTokens);192 193 EXPECT_EQ("C(i)", getSourceText(toks[2], toks[2]));194}195 196TEST_F(LexerTest, GetSourceTextExpandsAcrossMultipleMacroCalls) {197 std::vector<tok::TokenKind> ExpectedTokens;198 ExpectedTokens.push_back(tok::identifier);199 ExpectedTokens.push_back(tok::l_paren);200 ExpectedTokens.push_back(tok::identifier);201 ExpectedTokens.push_back(tok::r_paren);202 203 std::vector<Token> toks = CheckLex("#define M(x) x\n"204 "f(M(M(i)))",205 ExpectedTokens);206 EXPECT_EQ("M(M(i))", getSourceText(toks[2], toks[2]));207}208 209TEST_F(LexerTest, GetSourceTextInMiddleOfMacroArgument) {210 std::vector<tok::TokenKind> ExpectedTokens;211 ExpectedTokens.push_back(tok::identifier);212 ExpectedTokens.push_back(tok::l_paren);213 ExpectedTokens.push_back(tok::identifier);214 ExpectedTokens.push_back(tok::r_paren);215 216 std::vector<Token> toks = CheckLex("#define M(x) x\n"217 "M(f(i))",218 ExpectedTokens);219 EXPECT_EQ("i", getSourceText(toks[2], toks[2]));220}221 222TEST_F(LexerTest, GetSourceTextExpandsAroundDifferentMacroCalls) {223 std::vector<tok::TokenKind> ExpectedTokens;224 ExpectedTokens.push_back(tok::identifier);225 ExpectedTokens.push_back(tok::l_paren);226 ExpectedTokens.push_back(tok::identifier);227 ExpectedTokens.push_back(tok::r_paren);228 229 std::vector<Token> toks = CheckLex("#define M(x) x\n"230 "#define C(x) x\n"231 "f(C(M(i)))",232 ExpectedTokens);233 EXPECT_EQ("C(M(i))", getSourceText(toks[2], toks[2]));234}235 236TEST_F(LexerTest, GetSourceTextOnlyExpandsIfFirstTokenInMacro) {237 std::vector<tok::TokenKind> ExpectedTokens;238 ExpectedTokens.push_back(tok::identifier);239 ExpectedTokens.push_back(tok::l_paren);240 ExpectedTokens.push_back(tok::identifier);241 ExpectedTokens.push_back(tok::identifier);242 ExpectedTokens.push_back(tok::r_paren);243 244 std::vector<Token> toks = CheckLex("#define M(x) x\n"245 "#define C(x) c x\n"246 "f(C(M(i)))",247 ExpectedTokens);248 EXPECT_EQ("M(i)", getSourceText(toks[3], toks[3]));249}250 251TEST_F(LexerTest, GetSourceTextExpandsRecursively) {252 std::vector<tok::TokenKind> ExpectedTokens;253 ExpectedTokens.push_back(tok::identifier);254 ExpectedTokens.push_back(tok::identifier);255 ExpectedTokens.push_back(tok::l_paren);256 ExpectedTokens.push_back(tok::identifier);257 ExpectedTokens.push_back(tok::r_paren);258 259 std::vector<Token> toks = CheckLex("#define M(x) x\n"260 "#define C(x) c M(x)\n"261 "C(f(M(i)))",262 ExpectedTokens);263 EXPECT_EQ("M(i)", getSourceText(toks[3], toks[3]));264}265 266TEST_F(LexerTest, LexAPI) {267 std::vector<tok::TokenKind> ExpectedTokens;268 // Line 1 (after the #defines)269 ExpectedTokens.push_back(tok::l_square);270 ExpectedTokens.push_back(tok::identifier);271 ExpectedTokens.push_back(tok::r_square);272 ExpectedTokens.push_back(tok::l_square);273 ExpectedTokens.push_back(tok::identifier);274 ExpectedTokens.push_back(tok::r_square);275 // Line 2276 ExpectedTokens.push_back(tok::identifier);277 ExpectedTokens.push_back(tok::identifier);278 ExpectedTokens.push_back(tok::identifier);279 ExpectedTokens.push_back(tok::identifier);280 281 std::vector<Token> toks = CheckLex("#define M(x) [x]\n"282 "#define N(x) x\n"283 "#define INN(x) x\n"284 "#define NOF1 INN(val)\n"285 "#define NOF2 val\n"286 "M(foo) N([bar])\n"287 "N(INN(val)) N(NOF1) N(NOF2) N(val)",288 ExpectedTokens);289 290 SourceLocation lsqrLoc = toks[0].getLocation();291 SourceLocation idLoc = toks[1].getLocation();292 SourceLocation rsqrLoc = toks[2].getLocation();293 CharSourceRange macroRange = SourceMgr.getExpansionRange(lsqrLoc);294 295 SourceLocation Loc;296 EXPECT_TRUE(Lexer::isAtStartOfMacroExpansion(lsqrLoc, SourceMgr, LangOpts, &Loc));297 EXPECT_EQ(Loc, macroRange.getBegin());298 EXPECT_FALSE(Lexer::isAtStartOfMacroExpansion(idLoc, SourceMgr, LangOpts));299 EXPECT_FALSE(Lexer::isAtEndOfMacroExpansion(idLoc, SourceMgr, LangOpts));300 EXPECT_TRUE(Lexer::isAtEndOfMacroExpansion(rsqrLoc, SourceMgr, LangOpts, &Loc));301 EXPECT_EQ(Loc, macroRange.getEnd());302 EXPECT_TRUE(macroRange.isTokenRange());303 304 CharSourceRange range = Lexer::makeFileCharRange(305 CharSourceRange::getTokenRange(lsqrLoc, idLoc), SourceMgr, LangOpts);306 EXPECT_TRUE(range.isInvalid());307 range = Lexer::makeFileCharRange(CharSourceRange::getTokenRange(idLoc, rsqrLoc),308 SourceMgr, LangOpts);309 EXPECT_TRUE(range.isInvalid());310 range = Lexer::makeFileCharRange(CharSourceRange::getTokenRange(lsqrLoc, rsqrLoc),311 SourceMgr, LangOpts);312 EXPECT_TRUE(!range.isTokenRange());313 EXPECT_EQ(range.getAsRange(),314 SourceRange(macroRange.getBegin(),315 macroRange.getEnd().getLocWithOffset(1)));316 317 StringRef text = Lexer::getSourceText(318 CharSourceRange::getTokenRange(lsqrLoc, rsqrLoc),319 SourceMgr, LangOpts);320 EXPECT_EQ(text, "M(foo)");321 322 SourceLocation macroLsqrLoc = toks[3].getLocation();323 SourceLocation macroIdLoc = toks[4].getLocation();324 SourceLocation macroRsqrLoc = toks[5].getLocation();325 SourceLocation fileLsqrLoc = SourceMgr.getSpellingLoc(macroLsqrLoc);326 SourceLocation fileIdLoc = SourceMgr.getSpellingLoc(macroIdLoc);327 SourceLocation fileRsqrLoc = SourceMgr.getSpellingLoc(macroRsqrLoc);328 329 range = Lexer::makeFileCharRange(330 CharSourceRange::getTokenRange(macroLsqrLoc, macroIdLoc),331 SourceMgr, LangOpts);332 EXPECT_EQ(SourceRange(fileLsqrLoc, fileIdLoc.getLocWithOffset(3)),333 range.getAsRange());334 335 range = Lexer::makeFileCharRange(CharSourceRange::getTokenRange(macroIdLoc, macroRsqrLoc),336 SourceMgr, LangOpts);337 EXPECT_EQ(SourceRange(fileIdLoc, fileRsqrLoc.getLocWithOffset(1)),338 range.getAsRange());339 340 macroRange = SourceMgr.getExpansionRange(macroLsqrLoc);341 range = Lexer::makeFileCharRange(342 CharSourceRange::getTokenRange(macroLsqrLoc, macroRsqrLoc),343 SourceMgr, LangOpts);344 EXPECT_EQ(SourceRange(macroRange.getBegin(), macroRange.getEnd().getLocWithOffset(1)),345 range.getAsRange());346 347 text = Lexer::getSourceText(348 CharSourceRange::getTokenRange(SourceRange(macroLsqrLoc, macroIdLoc)),349 SourceMgr, LangOpts);350 EXPECT_EQ(text, "[bar");351 352 353 SourceLocation idLoc1 = toks[6].getLocation();354 SourceLocation idLoc2 = toks[7].getLocation();355 SourceLocation idLoc3 = toks[8].getLocation();356 SourceLocation idLoc4 = toks[9].getLocation();357 EXPECT_EQ("INN", Lexer::getImmediateMacroName(idLoc1, SourceMgr, LangOpts));358 EXPECT_EQ("INN", Lexer::getImmediateMacroName(idLoc2, SourceMgr, LangOpts));359 EXPECT_EQ("NOF2", Lexer::getImmediateMacroName(idLoc3, SourceMgr, LangOpts));360 EXPECT_EQ("N", Lexer::getImmediateMacroName(idLoc4, SourceMgr, LangOpts));361}362 363TEST_F(LexerTest, HandlesSplitTokens) {364 std::vector<tok::TokenKind> ExpectedTokens;365 // Line 1 (after the #defines)366 ExpectedTokens.push_back(tok::identifier);367 ExpectedTokens.push_back(tok::less);368 ExpectedTokens.push_back(tok::identifier);369 ExpectedTokens.push_back(tok::less);370 ExpectedTokens.push_back(tok::greatergreater);371 // Line 2372 ExpectedTokens.push_back(tok::identifier);373 ExpectedTokens.push_back(tok::less);374 ExpectedTokens.push_back(tok::identifier);375 ExpectedTokens.push_back(tok::less);376 ExpectedTokens.push_back(tok::greatergreater);377 378 std::vector<Token> toks = CheckLex("#define TY ty\n"379 "#define RANGLE ty<ty<>>\n"380 "TY<ty<>>\n"381 "RANGLE",382 ExpectedTokens);383 384 SourceLocation outerTyLoc = toks[0].getLocation();385 SourceLocation innerTyLoc = toks[2].getLocation();386 SourceLocation gtgtLoc = toks[4].getLocation();387 // Split the token to simulate the action of the parser and force creation of388 // an `ExpansionTokenRange`.389 SourceLocation rangleLoc = PP->SplitToken(gtgtLoc, 1);390 391 // Verify that it only captures the first greater-then and not the second one.392 CharSourceRange range = Lexer::makeFileCharRange(393 CharSourceRange::getTokenRange(innerTyLoc, rangleLoc), SourceMgr,394 LangOpts);395 EXPECT_TRUE(range.isCharRange());396 EXPECT_EQ(range.getAsRange(),397 SourceRange(innerTyLoc, gtgtLoc.getLocWithOffset(1)));398 399 // Verify case where range begins in a macro expansion.400 range = Lexer::makeFileCharRange(401 CharSourceRange::getTokenRange(outerTyLoc, rangleLoc), SourceMgr,402 LangOpts);403 EXPECT_TRUE(range.isCharRange());404 EXPECT_EQ(range.getAsRange(),405 SourceRange(SourceMgr.getExpansionLoc(outerTyLoc),406 gtgtLoc.getLocWithOffset(1)));407 408 SourceLocation macroInnerTyLoc = toks[7].getLocation();409 SourceLocation macroGtgtLoc = toks[9].getLocation();410 // Split the token to simulate the action of the parser and force creation of411 // an `ExpansionTokenRange`.412 SourceLocation macroRAngleLoc = PP->SplitToken(macroGtgtLoc, 1);413 414 // Verify that it fails (because it only captures the first greater-then and415 // not the second one, so it doesn't span the entire macro expansion).416 range = Lexer::makeFileCharRange(417 CharSourceRange::getTokenRange(macroInnerTyLoc, macroRAngleLoc),418 SourceMgr, LangOpts);419 EXPECT_TRUE(range.isInvalid());420}421 422TEST_F(LexerTest, DontMergeMacroArgsFromDifferentMacroFiles) {423 std::vector<Token> toks =424 Lex("#define helper1 0\n"425 "void helper2(const char *, ...);\n"426 "#define M1(a, ...) helper2(a, ##__VA_ARGS__)\n"427 "#define M2(a, ...) M1(a, helper1, ##__VA_ARGS__)\n"428 "void f1() { M2(\"a\", \"b\"); }");429 430 // Check the file corresponding to the "helper1" macro arg in M2.431 //432 // The lexer used to report its size as 31, meaning that the end of the433 // expansion would be on the *next line* (just past `M2("a", "b")`). Make434 // sure that we get the correct end location (the comma after "helper1").435 SourceLocation helper1ArgLoc = toks[20].getLocation();436 EXPECT_EQ(SourceMgr.getFileIDSize(SourceMgr.getFileID(helper1ArgLoc)), 8U);437}438 439TEST_F(LexerTest, DontOverallocateStringifyArgs) {440 TrivialModuleLoader ModLoader;441 auto PP = CreatePP("\"StrArg\", 5, 'C'", ModLoader);442 443 llvm::BumpPtrAllocator Allocator;444 std::array<IdentifierInfo *, 3> ParamList;445 MacroInfo *MI = PP->AllocateMacroInfo({});446 MI->setIsFunctionLike();447 MI->setParameterList(ParamList, Allocator);448 EXPECT_EQ(3u, MI->getNumParams());449 EXPECT_TRUE(MI->isFunctionLike());450 451 Token Eof;452 Eof.setKind(tok::eof);453 std::vector<Token> ArgTokens;454 while (1) {455 Token tok;456 PP->Lex(tok);457 if (tok.is(tok::eof)) {458 ArgTokens.push_back(Eof);459 break;460 }461 if (tok.is(tok::comma))462 ArgTokens.push_back(Eof);463 else464 ArgTokens.push_back(tok);465 }466 467 auto MacroArgsDeleter = [&PP](MacroArgs *M) { M->destroy(*PP); };468 std::unique_ptr<MacroArgs, decltype(MacroArgsDeleter)> MA(469 MacroArgs::create(MI, ArgTokens, false, *PP), MacroArgsDeleter);470 auto StringifyArg = [&](int ArgNo) {471 return MA->StringifyArgument(MA->getUnexpArgument(ArgNo), *PP,472 /*Charify=*/false, {}, {});473 };474 Token Result = StringifyArg(0);475 EXPECT_EQ(tok::string_literal, Result.getKind());476 EXPECT_STREQ("\"\\\"StrArg\\\"\"", Result.getLiteralData());477 Result = StringifyArg(1);478 EXPECT_EQ(tok::string_literal, Result.getKind());479 EXPECT_STREQ("\"5\"", Result.getLiteralData());480 Result = StringifyArg(2);481 EXPECT_EQ(tok::string_literal, Result.getKind());482 EXPECT_STREQ("\"'C'\"", Result.getLiteralData());483#if !defined(NDEBUG) && GTEST_HAS_DEATH_TEST484 EXPECT_DEATH(StringifyArg(3), "Invalid arg #");485#endif486}487 488TEST_F(LexerTest, IsNewLineEscapedValid) {489 auto hasNewLineEscaped = [](const char *S) {490 return Lexer::isNewLineEscaped(S, S + strlen(S) - 1);491 };492 493 EXPECT_TRUE(hasNewLineEscaped("\\\r"));494 EXPECT_TRUE(hasNewLineEscaped("\\\n"));495 EXPECT_TRUE(hasNewLineEscaped("\\\r\n"));496 EXPECT_TRUE(hasNewLineEscaped("\\\n\r"));497 EXPECT_TRUE(hasNewLineEscaped("\\ \t\v\f\r"));498 EXPECT_TRUE(hasNewLineEscaped("\\ \t\v\f\r\n"));499 500 EXPECT_FALSE(hasNewLineEscaped("\\\r\r"));501 EXPECT_FALSE(hasNewLineEscaped("\\\r\r\n"));502 EXPECT_FALSE(hasNewLineEscaped("\\\n\n"));503 EXPECT_FALSE(hasNewLineEscaped("\r"));504 EXPECT_FALSE(hasNewLineEscaped("\n"));505 EXPECT_FALSE(hasNewLineEscaped("\r\n"));506 EXPECT_FALSE(hasNewLineEscaped("\n\r"));507 EXPECT_FALSE(hasNewLineEscaped("\r\r"));508 EXPECT_FALSE(hasNewLineEscaped("\n\n"));509}510 511TEST_F(LexerTest, GetBeginningOfTokenWithEscapedNewLine) {512 // Each line should have the same length for513 // further offset calculation to be more straightforward.514 const unsigned IdentifierLength = 8;515 std::string TextToLex = "rabarbar\n"516 "foo\\\nbar\n"517 "foo\\\rbar\n"518 "fo\\\r\nbar\n"519 "foo\\\n\rba\n";520 std::vector<tok::TokenKind> ExpectedTokens{5, tok::identifier};521 std::vector<Token> LexedTokens = CheckLex(TextToLex, ExpectedTokens);522 523 for (const Token &Tok : LexedTokens) {524 FileIDAndOffset OriginalLocation =525 SourceMgr.getDecomposedLoc(Tok.getLocation());526 for (unsigned Offset = 0; Offset < IdentifierLength; ++Offset) {527 SourceLocation LookupLocation =528 Tok.getLocation().getLocWithOffset(Offset);529 530 FileIDAndOffset FoundLocation = SourceMgr.getDecomposedExpansionLoc(531 Lexer::GetBeginningOfToken(LookupLocation, SourceMgr, LangOpts));532 533 // Check that location returned by the GetBeginningOfToken534 // is the same as original token location reported by Lexer.535 EXPECT_EQ(FoundLocation.second, OriginalLocation.second);536 }537 }538}539 540TEST_F(LexerTest, AvoidPastEndOfStringDereference) {541 EXPECT_TRUE(Lex(" // \\\n").empty());542 EXPECT_TRUE(Lex("#include <\\\\").empty());543 EXPECT_TRUE(Lex("#include <\\\\\n").empty());544}545 546TEST_F(LexerTest, StringizingRasString) {547 // For "std::string Lexer::Stringify(StringRef Str, bool Charify)".548 std::string String1 = R"(foo549 {"bar":[]}550 baz)";551 // For "void Lexer::Stringify(SmallVectorImpl<char> &Str)".552 SmallString<128> String2;553 String2 += String1.c_str();554 555 // Corner cases.556 std::string String3 = R"(\557 \n558 \\n559 \\)";560 SmallString<128> String4;561 String4 += String3.c_str();562 std::string String5 = R"(a\563 564 565 \\b)";566 SmallString<128> String6;567 String6 += String5.c_str();568 569 String1 = Lexer::Stringify(StringRef(String1));570 Lexer::Stringify(String2);571 String3 = Lexer::Stringify(StringRef(String3));572 Lexer::Stringify(String4);573 String5 = Lexer::Stringify(StringRef(String5));574 Lexer::Stringify(String6);575 576 EXPECT_EQ(String1, R"(foo\n {\"bar\":[]}\n baz)");577 EXPECT_EQ(String2, R"(foo\n {\"bar\":[]}\n baz)");578 EXPECT_EQ(String3, R"(\\\n \\n\n \\\\n\n \\\\)");579 EXPECT_EQ(String4, R"(\\\n \\n\n \\\\n\n \\\\)");580 EXPECT_EQ(String5, R"(a\\\n\n\n \\\\b)");581 EXPECT_EQ(String6, R"(a\\\n\n\n \\\\b)");582}583 584TEST_F(LexerTest, CharRangeOffByOne) {585 std::vector<Token> toks = Lex(R"(#define MOO 1586 void foo() { MOO; })");587 const Token &moo = toks[5];588 589 EXPECT_EQ(getSourceText(moo, moo), "MOO");590 591 SourceRange R{moo.getLocation(), moo.getLocation()};592 593 EXPECT_TRUE(594 Lexer::isAtStartOfMacroExpansion(R.getBegin(), SourceMgr, LangOpts));595 EXPECT_TRUE(596 Lexer::isAtEndOfMacroExpansion(R.getEnd(), SourceMgr, LangOpts));597 598 CharSourceRange CR = Lexer::getAsCharRange(R, SourceMgr, LangOpts);599 600 EXPECT_EQ(Lexer::getSourceText(CR, SourceMgr, LangOpts), "MOO"); // Was "MO".601}602 603TEST_F(LexerTest, FindNextToken) {604 Lex("int abcd = 0;\n"605 "// A comment.\n"606 "int xyz = abcd;\n");607 std::vector<std::string> GeneratedByNextToken;608 SourceLocation Loc =609 SourceMgr.getLocForStartOfFile(SourceMgr.getMainFileID());610 while (true) {611 auto T = Lexer::findNextToken(Loc, SourceMgr, LangOpts);612 ASSERT_TRUE(T);613 if (T->is(tok::eof))614 break;615 GeneratedByNextToken.push_back(getSourceText(*T, *T));616 Loc = T->getLocation();617 }618 EXPECT_THAT(GeneratedByNextToken, ElementsAre("abcd", "=", "0", ";", "int",619 "xyz", "=", "abcd", ";"));620}621 622TEST_F(LexerTest, FindNextTokenIncludingComments) {623 Lex("int abcd = 0;\n"624 "// A comment.\n"625 "int xyz = abcd;\n");626 std::vector<std::string> GeneratedByNextToken;627 SourceLocation Loc =628 SourceMgr.getLocForStartOfFile(SourceMgr.getMainFileID());629 while (true) {630 auto T = Lexer::findNextToken(Loc, SourceMgr, LangOpts, true);631 ASSERT_TRUE(T);632 if (T->is(tok::eof))633 break;634 GeneratedByNextToken.push_back(getSourceText(*T, *T));635 Loc = T->getLocation();636 }637 EXPECT_THAT(GeneratedByNextToken,638 ElementsAre("abcd", "=", "0", ";", "// A comment.", "int", "xyz",639 "=", "abcd", ";"));640}641 642TEST_F(LexerTest, FindPreviousToken) {643 Lex("int abcd = 0;\n"644 "// A comment.\n"645 "int xyz = abcd;\n");646 std::vector<std::string> GeneratedByPrevToken;647 SourceLocation Loc = SourceMgr.getLocForEndOfFile(SourceMgr.getMainFileID());648 while (true) {649 auto T = Lexer::findPreviousToken(Loc, SourceMgr, LangOpts, false);650 if (!T.has_value())651 break;652 GeneratedByPrevToken.push_back(getSourceText(*T, *T));653 Loc = Lexer::GetBeginningOfToken(T->getLocation(), SourceMgr, LangOpts);654 }655 EXPECT_THAT(GeneratedByPrevToken, ElementsAre(";", "abcd", "=", "xyz", "int",656 ";", "0", "=", "abcd", "int"));657}658 659TEST_F(LexerTest, FindPreviousTokenIncludingComments) {660 Lex("int abcd = 0;\n"661 "// A comment.\n"662 "int xyz = abcd;\n");663 std::vector<std::string> GeneratedByPrevToken;664 SourceLocation Loc = SourceMgr.getLocForEndOfFile(SourceMgr.getMainFileID());665 while (true) {666 auto T = Lexer::findPreviousToken(Loc, SourceMgr, LangOpts, true);667 if (!T.has_value())668 break;669 GeneratedByPrevToken.push_back(getSourceText(*T, *T));670 Loc = Lexer::GetBeginningOfToken(T->getLocation(), SourceMgr, LangOpts);671 }672 EXPECT_THAT(GeneratedByPrevToken,673 ElementsAre(";", "abcd", "=", "xyz", "int", "// A comment.", ";",674 "0", "=", "abcd", "int"));675}676 677TEST_F(LexerTest, CreatedFIDCountForPredefinedBuffer) {678 TrivialModuleLoader ModLoader;679 auto PP = CreatePP("", ModLoader);680 PP->LexTokensUntilEOF();681 EXPECT_EQ(SourceMgr.getNumCreatedFIDsForFileID(PP->getPredefinesFileID()),682 1U);683}684 685TEST_F(LexerTest, RawAndNormalLexSameForLineComments) {686 const llvm::StringLiteral Source = R"cpp(687 // First line comment.688 //* Second line comment which is ambigious.689 ; // Have a non-comment token to make sure something is lexed.690 )cpp";691 LangOpts.LineComment = false;692 auto Toks = Lex(Source);693 auto &SM = PP->getSourceManager();694 auto SrcBuffer = SM.getBufferData(SM.getMainFileID());695 Lexer L(SM.getLocForStartOfFile(SM.getMainFileID()), PP->getLangOpts(),696 SrcBuffer.data(), SrcBuffer.data(),697 SrcBuffer.data() + SrcBuffer.size());698 699 auto ToksView = llvm::ArrayRef(Toks);700 clang::Token T;701 EXPECT_FALSE(ToksView.empty());702 while (!L.LexFromRawLexer(T)) {703 ASSERT_TRUE(!ToksView.empty());704 EXPECT_EQ(T.getKind(), ToksView.front().getKind());705 ToksView = ToksView.drop_front();706 }707 EXPECT_TRUE(ToksView.empty());708}709 710TEST_F(LexerTest, GetRawTokenOnEscapedNewLineChecksWhitespace) {711 const llvm::StringLiteral Source = R"cc(712 #define ONE \713 1714 715 int i = ONE;716 )cc";717 std::vector<Token> Toks =718 CheckLex(Source, {tok::kw_int, tok::identifier, tok::equal,719 tok::numeric_constant, tok::semi});720 721 // Set up by getting the raw token for the `1` in the macro definition.722 const Token &OneExpanded = Toks[3];723 Token Tok;724 ASSERT_FALSE(725 Lexer::getRawToken(OneExpanded.getLocation(), Tok, SourceMgr, LangOpts));726 // The `ONE`.727 ASSERT_EQ(Tok.getKind(), tok::raw_identifier);728 ASSERT_FALSE(729 Lexer::getRawToken(SourceMgr.getSpellingLoc(OneExpanded.getLocation()),730 Tok, SourceMgr, LangOpts));731 // The `1` in the macro definition.732 ASSERT_EQ(Tok.getKind(), tok::numeric_constant);733 734 // Go back 4 characters: two spaces, one newline, and the backslash.735 SourceLocation EscapedNewLineLoc = Tok.getLocation().getLocWithOffset(-4);736 // Expect true (=failure) because the whitespace immediately after the737 // escaped newline is not ignored.738 EXPECT_TRUE(Lexer::getRawToken(EscapedNewLineLoc, Tok, SourceMgr, LangOpts,739 /*IgnoreWhiteSpace=*/false));740}741 742TEST(LexerPreambleTest, PreambleBounds) {743 std::vector<std::string> Cases = {744 R"cc([[745 #include <foo>746 ]]int bar;747 )cc",748 R"cc([[749 #include <foo>750 ]])cc",751 R"cc([[752 // leading comment753 #include <foo>754 ]]// trailing comment755 int bar;756 )cc",757 R"cc([[758 module;759 #include <foo>760 ]]module bar;761 int x;762 )cc",763 };764 for (const auto& Case : Cases) {765 llvm::Annotations A(Case);766 clang::LangOptions LangOpts;767 LangOpts.CPlusPlusModules = true;768 auto Bounds = Lexer::ComputePreamble(A.code(), LangOpts);769 EXPECT_EQ(Bounds.Size, A.range().End) << Case;770 }771}772 773TEST_F(LexerTest, CheckFirstPPToken) {774 LangOpts.CPlusPlusModules = true;775 {776 TrivialModuleLoader ModLoader;777 auto PP = CreatePP("// This is a comment\n"778 "int a;",779 ModLoader);780 Token Tok;781 PP->Lex(Tok);782 EXPECT_TRUE(Tok.is(tok::kw_int));783 EXPECT_TRUE(PP->getMainFileFirstPPTokenLoc().isValid());784 EXPECT_EQ(PP->getMainFileFirstPPTokenLoc(), Tok.getLocation());785 }786 {787 TrivialModuleLoader ModLoader;788 auto PP = CreatePP("// This is a comment\n"789 "#define FOO int\n"790 "FOO a;",791 ModLoader);792 Token Tok;793 PP->Lex(Tok);794 EXPECT_TRUE(Tok.is(tok::kw_int));795 EXPECT_FALSE(Lexer::getRawToken(PP->getMainFileFirstPPTokenLoc(), Tok,796 PP->getSourceManager(), PP->getLangOpts(),797 /*IgnoreWhiteSpace=*/false));798 EXPECT_TRUE(PP->getMainFileFirstPPTokenLoc() == Tok.getLocation());799 EXPECT_TRUE(Tok.is(tok::hash));800 }801 802 {803 PreDefines = "#define FOO int\n";804 TrivialModuleLoader ModLoader;805 auto PP = CreatePP("// This is a comment\n"806 "FOO a;",807 ModLoader);808 Token Tok;809 PP->Lex(Tok);810 EXPECT_TRUE(Tok.is(tok::kw_int));811 EXPECT_FALSE(Lexer::getRawToken(PP->getMainFileFirstPPTokenLoc(), Tok,812 PP->getSourceManager(), PP->getLangOpts(),813 /*IgnoreWhiteSpace=*/false));814 EXPECT_TRUE(PP->getMainFileFirstPPTokenLoc() == Tok.getLocation());815 EXPECT_TRUE(Tok.is(tok::raw_identifier));816 EXPECT_TRUE(Tok.getRawIdentifier() == "FOO");817 }818}819} // anonymous namespace820