1104 lines · cpp
1//===- TokenLexer.cpp - Lex from a token stream ---------------------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// This file implements the TokenLexer interface.10//11//===----------------------------------------------------------------------===//12 13#include "clang/Lex/TokenLexer.h"14#include "clang/Basic/Diagnostic.h"15#include "clang/Basic/IdentifierTable.h"16#include "clang/Basic/LangOptions.h"17#include "clang/Basic/SourceLocation.h"18#include "clang/Basic/SourceManager.h"19#include "clang/Basic/TokenKinds.h"20#include "clang/Lex/LexDiagnostic.h"21#include "clang/Lex/Lexer.h"22#include "clang/Lex/MacroArgs.h"23#include "clang/Lex/MacroInfo.h"24#include "clang/Lex/Preprocessor.h"25#include "clang/Lex/Token.h"26#include "clang/Lex/VariadicMacroSupport.h"27#include "llvm/ADT/ArrayRef.h"28#include "llvm/ADT/STLExtras.h"29#include "llvm/ADT/SmallVector.h"30#include "llvm/ADT/iterator_range.h"31#include <cassert>32#include <cstring>33#include <optional>34 35using namespace clang;36 37/// Create a TokenLexer for the specified macro with the specified actual38/// arguments. Note that this ctor takes ownership of the ActualArgs pointer.39void TokenLexer::Init(Token &Tok, SourceLocation ELEnd, MacroInfo *MI,40 MacroArgs *Actuals) {41 // If the client is reusing a TokenLexer, make sure to free any memory42 // associated with it.43 destroy();44 45 Macro = MI;46 ActualArgs = Actuals;47 CurTokenIdx = 0;48 49 ExpandLocStart = Tok.getLocation();50 ExpandLocEnd = ELEnd;51 AtStartOfLine = Tok.isAtStartOfLine();52 HasLeadingSpace = Tok.hasLeadingSpace();53 NextTokGetsSpace = false;54 Tokens = &*Macro->tokens_begin();55 OwnsTokens = false;56 DisableMacroExpansion = false;57 IsReinject = false;58 NumTokens = Macro->tokens_end()-Macro->tokens_begin();59 MacroExpansionStart = SourceLocation();60 61 SourceManager &SM = PP.getSourceManager();62 MacroStartSLocOffset = SM.getNextLocalOffset();63 64 if (NumTokens > 0) {65 assert(Tokens[0].getLocation().isValid());66 assert((Tokens[0].getLocation().isFileID() || Tokens[0].is(tok::comment)) &&67 "Macro defined in macro?");68 assert(ExpandLocStart.isValid());69 70 // Reserve a source location entry chunk for the length of the macro71 // definition. Tokens that get lexed directly from the definition will72 // have their locations pointing inside this chunk. This is to avoid73 // creating separate source location entries for each token.74 MacroDefStart = SM.getExpansionLoc(Tokens[0].getLocation());75 MacroDefLength = Macro->getDefinitionLength(SM);76 MacroExpansionStart = SM.createExpansionLoc(MacroDefStart,77 ExpandLocStart,78 ExpandLocEnd,79 MacroDefLength);80 }81 82 // If this is a function-like macro, expand the arguments and change83 // Tokens to point to the expanded tokens.84 if (Macro->isFunctionLike() && Macro->getNumParams())85 ExpandFunctionArguments();86 87 // Mark the macro as currently disabled, so that it is not recursively88 // expanded. The macro must be disabled only after argument pre-expansion of89 // function-like macro arguments occurs.90 Macro->DisableMacro();91}92 93/// Create a TokenLexer for the specified token stream. This does not94/// take ownership of the specified token vector.95void TokenLexer::Init(const Token *TokArray, unsigned NumToks,96 bool disableMacroExpansion, bool ownsTokens,97 bool isReinject) {98 assert(!isReinject || disableMacroExpansion);99 // If the client is reusing a TokenLexer, make sure to free any memory100 // associated with it.101 destroy();102 103 Macro = nullptr;104 ActualArgs = nullptr;105 Tokens = TokArray;106 OwnsTokens = ownsTokens;107 DisableMacroExpansion = disableMacroExpansion;108 IsReinject = isReinject;109 NumTokens = NumToks;110 CurTokenIdx = 0;111 ExpandLocStart = ExpandLocEnd = SourceLocation();112 AtStartOfLine = false;113 HasLeadingSpace = false;114 NextTokGetsSpace = false;115 MacroExpansionStart = SourceLocation();116 117 // Set HasLeadingSpace/AtStartOfLine so that the first token will be118 // returned unmodified.119 if (NumToks != 0) {120 AtStartOfLine = TokArray[0].isAtStartOfLine();121 HasLeadingSpace = TokArray[0].hasLeadingSpace();122 }123}124 125void TokenLexer::destroy() {126 // If this was a function-like macro that actually uses its arguments, delete127 // the expanded tokens.128 if (OwnsTokens) {129 delete [] Tokens;130 Tokens = nullptr;131 OwnsTokens = false;132 }133 134 // TokenLexer owns its formal arguments.135 if (ActualArgs) ActualArgs->destroy(PP);136}137 138bool TokenLexer::MaybeRemoveCommaBeforeVaArgs(139 SmallVectorImpl<Token> &ResultToks, bool HasPasteOperator, MacroInfo *Macro,140 unsigned MacroArgNo, Preprocessor &PP) {141 // Is the macro argument __VA_ARGS__?142 if (!Macro->isVariadic() || MacroArgNo != Macro->getNumParams()-1)143 return false;144 145 // In Microsoft-compatibility mode, a comma is removed in the expansion146 // of " ... , __VA_ARGS__ " if __VA_ARGS__ is empty. This extension is147 // not supported by gcc.148 if (!HasPasteOperator && !PP.getLangOpts().MSVCCompat)149 return false;150 151 // GCC removes the comma in the expansion of " ... , ## __VA_ARGS__ " if152 // __VA_ARGS__ is empty, but not in strict C99 mode where there are no153 // named arguments, where it remains. In all other modes, including C99154 // with GNU extensions, it is removed regardless of named arguments.155 // Microsoft also appears to support this extension, unofficially.156 if (PP.getLangOpts().C99 && !PP.getLangOpts().GNUMode157 && Macro->getNumParams() < 2)158 return false;159 160 // Is a comma available to be removed?161 if (ResultToks.empty() || !ResultToks.back().is(tok::comma))162 return false;163 164 // Issue an extension diagnostic for the paste operator.165 if (HasPasteOperator)166 PP.Diag(ResultToks.back().getLocation(), diag::ext_paste_comma);167 168 // Remove the comma.169 ResultToks.pop_back();170 171 if (!ResultToks.empty()) {172 // If the comma was right after another paste (e.g. "X##,##__VA_ARGS__"),173 // then removal of the comma should produce a placemarker token (in C99174 // terms) which we model by popping off the previous ##, giving us a plain175 // "X" when __VA_ARGS__ is empty.176 if (ResultToks.back().is(tok::hashhash))177 ResultToks.pop_back();178 179 // Remember that this comma was elided.180 ResultToks.back().setFlag(Token::CommaAfterElided);181 }182 183 // Never add a space, even if the comma, ##, or arg had a space.184 NextTokGetsSpace = false;185 return true;186}187 188void TokenLexer::stringifyVAOPTContents(189 SmallVectorImpl<Token> &ResultToks, const VAOptExpansionContext &VCtx,190 const SourceLocation VAOPTClosingParenLoc) {191 const int NumToksPriorToVAOpt = VCtx.getNumberOfTokensPriorToVAOpt();192 const unsigned int NumVAOptTokens = ResultToks.size() - NumToksPriorToVAOpt;193 Token *const VAOPTTokens =194 NumVAOptTokens ? &ResultToks[NumToksPriorToVAOpt] : nullptr;195 196 SmallVector<Token, 64> ConcatenatedVAOPTResultToks;197 // FIXME: Should we keep track within VCtx that we did or didnot198 // encounter pasting - and only then perform this loop.199 200 // Perform token pasting (concatenation) prior to stringization.201 for (unsigned int CurTokenIdx = 0; CurTokenIdx != NumVAOptTokens;202 ++CurTokenIdx) {203 if (VAOPTTokens[CurTokenIdx].is(tok::hashhash)) {204 assert(CurTokenIdx != 0 &&205 "Can not have __VAOPT__ contents begin with a ##");206 Token &LHS = VAOPTTokens[CurTokenIdx - 1];207 pasteTokens(LHS, llvm::ArrayRef(VAOPTTokens, NumVAOptTokens),208 CurTokenIdx);209 // Replace the token prior to the first ## in this iteration.210 ConcatenatedVAOPTResultToks.back() = LHS;211 if (CurTokenIdx == NumVAOptTokens)212 break;213 }214 ConcatenatedVAOPTResultToks.push_back(VAOPTTokens[CurTokenIdx]);215 }216 217 ConcatenatedVAOPTResultToks.push_back(VCtx.getEOFTok());218 // Get the SourceLocation that represents the start location within219 // the macro definition that marks where this string is substituted220 // into: i.e. the __VA_OPT__ and the ')' within the spelling of the221 // macro definition, and use it to indicate that the stringified token222 // was generated from that location.223 const SourceLocation ExpansionLocStartWithinMacro =224 getExpansionLocForMacroDefLoc(VCtx.getVAOptLoc());225 const SourceLocation ExpansionLocEndWithinMacro =226 getExpansionLocForMacroDefLoc(VAOPTClosingParenLoc);227 228 Token StringifiedVAOPT = MacroArgs::StringifyArgument(229 &ConcatenatedVAOPTResultToks[0], PP, VCtx.hasCharifyBefore() /*Charify*/,230 ExpansionLocStartWithinMacro, ExpansionLocEndWithinMacro);231 232 if (VCtx.getLeadingSpaceForStringifiedToken())233 StringifiedVAOPT.setFlag(Token::LeadingSpace);234 235 StringifiedVAOPT.setFlag(Token::StringifiedInMacro);236 // Resize (shrink) the token stream to just capture this stringified token.237 ResultToks.resize(NumToksPriorToVAOpt + 1);238 ResultToks.back() = StringifiedVAOPT;239}240 241/// Expand the arguments of a function-like macro so that we can quickly242/// return preexpanded tokens from Tokens.243void TokenLexer::ExpandFunctionArguments() {244 SmallVector<Token, 128> ResultToks;245 246 // Loop through 'Tokens', expanding them into ResultToks. Keep247 // track of whether we change anything. If not, no need to keep them. If so,248 // we install the newly expanded sequence as the new 'Tokens' list.249 bool MadeChange = false;250 251 std::optional<bool> CalledWithVariadicArguments;252 253 VAOptExpansionContext VCtx(PP);254 255 for (unsigned I = 0, E = NumTokens; I != E; ++I) {256 const Token &CurTok = Tokens[I];257 // We don't want a space for the next token after a paste258 // operator. In valid code, the token will get smooshed onto the259 // preceding one anyway. In assembler-with-cpp mode, invalid260 // pastes are allowed through: in this case, we do not want the261 // extra whitespace to be added. For example, we want ". ## foo"262 // -> ".foo" not ". foo".263 if (I != 0 && !Tokens[I-1].is(tok::hashhash) && CurTok.hasLeadingSpace())264 NextTokGetsSpace = true;265 266 if (VCtx.isVAOptToken(CurTok)) {267 MadeChange = true;268 assert(Tokens[I + 1].is(tok::l_paren) &&269 "__VA_OPT__ must be followed by '('");270 271 ++I; // Skip the l_paren272 VCtx.sawVAOptFollowedByOpeningParens(CurTok.getLocation(),273 ResultToks.size());274 275 continue;276 }277 278 // We have entered into the __VA_OPT__ context, so handle tokens279 // appropriately.280 if (VCtx.isInVAOpt()) {281 // If we are about to process a token that is either an argument to282 // __VA_OPT__ or its closing rparen, then:283 // 1) If the token is the closing rparen that exits us out of __VA_OPT__,284 // perform any necessary stringification or placemarker processing,285 // and/or skip to the next token.286 // 2) else if macro was invoked without variadic arguments skip this287 // token.288 // 3) else (macro was invoked with variadic arguments) process the token289 // normally.290 291 if (Tokens[I].is(tok::l_paren))292 VCtx.sawOpeningParen(Tokens[I].getLocation());293 // Continue skipping tokens within __VA_OPT__ if the macro was not294 // called with variadic arguments, else let the rest of the loop handle295 // this token. Note sawClosingParen() returns true only if the r_paren matches296 // the closing r_paren of the __VA_OPT__.297 if (!Tokens[I].is(tok::r_paren) || !VCtx.sawClosingParen()) {298 // Lazily expand __VA_ARGS__ when we see the first __VA_OPT__.299 if (!CalledWithVariadicArguments) {300 CalledWithVariadicArguments =301 ActualArgs->invokedWithVariadicArgument(Macro, PP);302 }303 if (!*CalledWithVariadicArguments) {304 // Skip this token.305 continue;306 }307 // ... else the macro was called with variadic arguments, and we do not308 // have a closing rparen - so process this token normally.309 } else {310 // Current token is the closing r_paren which marks the end of the311 // __VA_OPT__ invocation, so handle any place-marker pasting (if312 // empty) by removing hashhash either before (if exists) or after. And313 // also stringify the entire contents if VAOPT was preceded by a hash,314 // but do so only after any token concatenation that needs to occur315 // within the contents of VAOPT.316 317 if (VCtx.hasStringifyOrCharifyBefore()) {318 // Replace all the tokens just added from within VAOPT into a single319 // stringified token. This requires token-pasting to eagerly occur320 // within these tokens. If either the contents of VAOPT were empty321 // or the macro wasn't called with any variadic arguments, the result322 // is a token that represents an empty string.323 stringifyVAOPTContents(ResultToks, VCtx,324 /*ClosingParenLoc*/ Tokens[I].getLocation());325 326 } else if (/*No tokens within VAOPT*/327 ResultToks.size() == VCtx.getNumberOfTokensPriorToVAOpt()) {328 // Treat VAOPT as a placemarker token. Eat either the '##' before the329 // RHS/VAOPT (if one exists, suggesting that the LHS (if any) to that330 // hashhash was not a placemarker) or the '##'331 // after VAOPT, but not both.332 333 if (ResultToks.size() && ResultToks.back().is(tok::hashhash)) {334 ResultToks.pop_back();335 } else if ((I + 1 != E) && Tokens[I + 1].is(tok::hashhash)) {336 ++I; // Skip the following hashhash.337 }338 } else {339 // If there's a ## before the __VA_OPT__, we might have discovered340 // that the __VA_OPT__ begins with a placeholder. We delay action on341 // that to now to avoid messing up our stashed count of tokens before342 // __VA_OPT__.343 if (VCtx.beginsWithPlaceholder()) {344 assert(VCtx.getNumberOfTokensPriorToVAOpt() > 0 &&345 ResultToks.size() >= VCtx.getNumberOfTokensPriorToVAOpt() &&346 ResultToks[VCtx.getNumberOfTokensPriorToVAOpt() - 1].is(347 tok::hashhash) &&348 "no token paste before __VA_OPT__");349 ResultToks.erase(ResultToks.begin() +350 VCtx.getNumberOfTokensPriorToVAOpt() - 1);351 }352 // If the expansion of __VA_OPT__ ends with a placeholder, eat any353 // following '##' token.354 if (VCtx.endsWithPlaceholder() && I + 1 != E &&355 Tokens[I + 1].is(tok::hashhash)) {356 ++I;357 }358 }359 VCtx.reset();360 // We processed __VA_OPT__'s closing paren (and the exit out of361 // __VA_OPT__), so skip to the next token.362 continue;363 }364 }365 366 // If we found the stringify operator, get the argument stringified. The367 // preprocessor already verified that the following token is a macro368 // parameter or __VA_OPT__ when the #define was lexed.369 370 if (CurTok.isOneOf(tok::hash, tok::hashat)) {371 int ArgNo = Macro->getParameterNum(Tokens[I+1].getIdentifierInfo());372 assert((ArgNo != -1 || VCtx.isVAOptToken(Tokens[I + 1])) &&373 "Token following # is not an argument or __VA_OPT__!");374 375 if (ArgNo == -1) {376 // Handle the __VA_OPT__ case.377 VCtx.sawHashOrHashAtBefore(NextTokGetsSpace,378 CurTok.is(tok::hashat));379 continue;380 }381 // Else handle the simple argument case.382 SourceLocation ExpansionLocStart =383 getExpansionLocForMacroDefLoc(CurTok.getLocation());384 SourceLocation ExpansionLocEnd =385 getExpansionLocForMacroDefLoc(Tokens[I+1].getLocation());386 387 bool Charify = CurTok.is(tok::hashat);388 const Token *UnexpArg = ActualArgs->getUnexpArgument(ArgNo);389 Token Res = MacroArgs::StringifyArgument(390 UnexpArg, PP, Charify, ExpansionLocStart, ExpansionLocEnd);391 Res.setFlag(Token::StringifiedInMacro);392 393 // The stringified/charified string leading space flag gets set to match394 // the #/#@ operator.395 if (NextTokGetsSpace)396 Res.setFlag(Token::LeadingSpace);397 398 ResultToks.push_back(Res);399 MadeChange = true;400 ++I; // Skip arg name.401 NextTokGetsSpace = false;402 continue;403 }404 405 // Find out if there is a paste (##) operator before or after the token.406 bool NonEmptyPasteBefore =407 !ResultToks.empty() && ResultToks.back().is(tok::hashhash);408 bool PasteBefore = I != 0 && Tokens[I-1].is(tok::hashhash);409 bool PasteAfter = I+1 != E && Tokens[I+1].is(tok::hashhash);410 bool RParenAfter = I+1 != E && Tokens[I+1].is(tok::r_paren);411 412 assert((!NonEmptyPasteBefore || PasteBefore || VCtx.isInVAOpt()) &&413 "unexpected ## in ResultToks");414 415 // Otherwise, if this is not an argument token, just add the token to the416 // output buffer.417 IdentifierInfo *II = CurTok.getIdentifierInfo();418 int ArgNo = II ? Macro->getParameterNum(II) : -1;419 if (ArgNo == -1) {420 // This isn't an argument, just add it.421 ResultToks.push_back(CurTok);422 423 if (NextTokGetsSpace) {424 ResultToks.back().setFlag(Token::LeadingSpace);425 NextTokGetsSpace = false;426 } else if (PasteBefore && !NonEmptyPasteBefore)427 ResultToks.back().clearFlag(Token::LeadingSpace);428 429 continue;430 }431 432 // An argument is expanded somehow, the result is different than the433 // input.434 MadeChange = true;435 436 // Otherwise, this is a use of the argument.437 438 // In Microsoft mode, remove the comma before __VA_ARGS__ to ensure there439 // are no trailing commas if __VA_ARGS__ is empty.440 if (!PasteBefore && ActualArgs->isVarargsElidedUse() &&441 MaybeRemoveCommaBeforeVaArgs(ResultToks,442 /*HasPasteOperator=*/false,443 Macro, ArgNo, PP))444 continue;445 446 // If it is not the LHS/RHS of a ## operator, we must pre-expand the447 // argument and substitute the expanded tokens into the result. This is448 // C99 6.10.3.1p1.449 if (!PasteBefore && !PasteAfter) {450 const Token *ResultArgToks;451 452 // Only preexpand the argument if it could possibly need it. This453 // avoids some work in common cases.454 const Token *ArgTok = ActualArgs->getUnexpArgument(ArgNo);455 if (ActualArgs->ArgNeedsPreexpansion(ArgTok, PP))456 ResultArgToks = &ActualArgs->getPreExpArgument(ArgNo, PP)[0];457 else458 ResultArgToks = ArgTok; // Use non-preexpanded tokens.459 460 // If the arg token expanded into anything, append it.461 if (ResultArgToks->isNot(tok::eof)) {462 size_t FirstResult = ResultToks.size();463 unsigned NumToks = MacroArgs::getArgLength(ResultArgToks);464 ResultToks.append(ResultArgToks, ResultArgToks+NumToks);465 466 // In Microsoft-compatibility mode, we follow MSVC's preprocessing467 // behavior by not considering single commas from nested macro468 // expansions as argument separators. Set a flag on the token so we can469 // test for this later when the macro expansion is processed.470 if (PP.getLangOpts().MSVCCompat && NumToks == 1 &&471 ResultToks.back().is(tok::comma))472 ResultToks.back().setFlag(Token::IgnoredComma);473 474 // If the '##' came from expanding an argument, turn it into 'unknown'475 // to avoid pasting.476 for (Token &Tok : llvm::drop_begin(ResultToks, FirstResult))477 if (Tok.is(tok::hashhash))478 Tok.setKind(tok::unknown);479 480 if(ExpandLocStart.isValid()) {481 updateLocForMacroArgTokens(CurTok.getLocation(),482 ResultToks.begin()+FirstResult,483 ResultToks.end());484 }485 486 // If any tokens were substituted from the argument, the whitespace487 // before the first token should match the whitespace of the arg488 // identifier.489 ResultToks[FirstResult].setFlagValue(Token::LeadingSpace,490 NextTokGetsSpace);491 ResultToks[FirstResult].setFlagValue(Token::StartOfLine, false);492 NextTokGetsSpace = false;493 } else {494 // We're creating a placeholder token. Usually this doesn't matter,495 // but it can affect paste behavior when at the start or end of a496 // __VA_OPT__.497 if (NonEmptyPasteBefore) {498 // We're imagining a placeholder token is inserted here. If this is499 // the first token in a __VA_OPT__ after a ##, delete the ##.500 assert(VCtx.isInVAOpt() && "should only happen inside a __VA_OPT__");501 VCtx.hasPlaceholderAfterHashhashAtStart();502 } else if (RParenAfter)503 VCtx.hasPlaceholderBeforeRParen();504 }505 continue;506 }507 508 // Okay, we have a token that is either the LHS or RHS of a paste (##)509 // argument. It gets substituted as its non-pre-expanded tokens.510 const Token *ArgToks = ActualArgs->getUnexpArgument(ArgNo);511 unsigned NumToks = MacroArgs::getArgLength(ArgToks);512 if (NumToks) { // Not an empty argument?513 bool VaArgsPseudoPaste = false;514 // If this is the GNU ", ## __VA_ARGS__" extension, and we just learned515 // that __VA_ARGS__ expands to multiple tokens, avoid a pasting error when516 // the expander tries to paste ',' with the first token of the __VA_ARGS__517 // expansion.518 if (NonEmptyPasteBefore && ResultToks.size() >= 2 &&519 ResultToks[ResultToks.size()-2].is(tok::comma) &&520 (unsigned)ArgNo == Macro->getNumParams()-1 &&521 Macro->isVariadic()) {522 VaArgsPseudoPaste = true;523 // Remove the paste operator, report use of the extension.524 PP.Diag(ResultToks.pop_back_val().getLocation(), diag::ext_paste_comma);525 }526 527 ResultToks.append(ArgToks, ArgToks+NumToks);528 529 // If the '##' came from expanding an argument, turn it into 'unknown'530 // to avoid pasting.531 for (Token &Tok : llvm::make_range(ResultToks.end() - NumToks,532 ResultToks.end())) {533 if (Tok.is(tok::hashhash))534 Tok.setKind(tok::unknown);535 }536 537 if (ExpandLocStart.isValid()) {538 updateLocForMacroArgTokens(CurTok.getLocation(),539 ResultToks.end()-NumToks, ResultToks.end());540 }541 542 // Transfer the leading whitespace information from the token543 // (the macro argument) onto the first token of the544 // expansion. Note that we don't do this for the GNU545 // pseudo-paste extension ", ## __VA_ARGS__".546 if (!VaArgsPseudoPaste) {547 ResultToks[ResultToks.size() - NumToks].setFlagValue(Token::StartOfLine,548 false);549 ResultToks[ResultToks.size() - NumToks].setFlagValue(550 Token::LeadingSpace, NextTokGetsSpace);551 }552 553 NextTokGetsSpace = false;554 continue;555 }556 557 // If an empty argument is on the LHS or RHS of a paste, the standard (C99558 // 6.10.3.3p2,3) calls for a bunch of placemarker stuff to occur. We559 // implement this by eating ## operators when a LHS or RHS expands to560 // empty.561 if (PasteAfter) {562 // Discard the argument token and skip (don't copy to the expansion563 // buffer) the paste operator after it.564 ++I;565 continue;566 }567 568 if (RParenAfter && !NonEmptyPasteBefore)569 VCtx.hasPlaceholderBeforeRParen();570 571 // If this is on the RHS of a paste operator, we've already copied the572 // paste operator to the ResultToks list, unless the LHS was empty too.573 // Remove it.574 assert(PasteBefore);575 if (NonEmptyPasteBefore) {576 assert(ResultToks.back().is(tok::hashhash));577 // Do not remove the paste operator if it is the one before __VA_OPT__578 // (and we are still processing tokens within VA_OPT). We handle the case579 // of removing the paste operator if __VA_OPT__ reduces to the notional580 // placemarker above when we encounter the closing paren of VA_OPT.581 if (!VCtx.isInVAOpt() ||582 ResultToks.size() > VCtx.getNumberOfTokensPriorToVAOpt())583 ResultToks.pop_back();584 else585 VCtx.hasPlaceholderAfterHashhashAtStart();586 }587 588 // If this is the __VA_ARGS__ token, and if the argument wasn't provided,589 // and if the macro had at least one real argument, and if the token before590 // the ## was a comma, remove the comma. This is a GCC extension which is591 // disabled when using -std=c99.592 if (ActualArgs->isVarargsElidedUse())593 MaybeRemoveCommaBeforeVaArgs(ResultToks,594 /*HasPasteOperator=*/true,595 Macro, ArgNo, PP);596 }597 598 // If anything changed, install this as the new Tokens list.599 if (MadeChange) {600 assert(!OwnsTokens && "This would leak if we already own the token list");601 // This is deleted in the dtor.602 NumTokens = ResultToks.size();603 // The tokens will be added to Preprocessor's cache and will be removed604 // when this TokenLexer finishes lexing them.605 Tokens = PP.cacheMacroExpandedTokens(this, ResultToks);606 607 // The preprocessor cache of macro expanded tokens owns these tokens,not us.608 OwnsTokens = false;609 }610}611 612/// Checks if two tokens form wide string literal.613static bool isWideStringLiteralFromMacro(const Token &FirstTok,614 const Token &SecondTok) {615 return FirstTok.is(tok::identifier) &&616 FirstTok.getIdentifierInfo()->isStr("L") && SecondTok.isLiteral() &&617 SecondTok.stringifiedInMacro();618}619 620/// Lex - Lex and return a token from this macro stream.621bool TokenLexer::Lex(Token &Tok) {622 // Lexing off the end of the macro, pop this macro off the expansion stack.623 if (isAtEnd()) {624 // If this is a macro (not a token stream), mark the macro enabled now625 // that it is no longer being expanded.626 if (Macro) Macro->EnableMacro();627 628 Tok.startToken();629 Tok.setFlagValue(Token::StartOfLine , AtStartOfLine);630 Tok.setFlagValue(Token::LeadingSpace, HasLeadingSpace || NextTokGetsSpace);631 if (CurTokenIdx == 0)632 Tok.setFlag(Token::LeadingEmptyMacro);633 return PP.HandleEndOfTokenLexer(Tok);634 }635 636 SourceManager &SM = PP.getSourceManager();637 638 // If this is the first token of the expanded result, we inherit spacing639 // properties later.640 bool isFirstToken = CurTokenIdx == 0;641 642 // Get the next token to return.643 Tok = Tokens[CurTokenIdx++];644 if (IsReinject)645 Tok.setFlag(Token::IsReinjected);646 647 bool TokenIsFromPaste = false;648 649 // If this token is followed by a token paste (##) operator, paste the tokens!650 // Note that ## is a normal token when not expanding a macro.651 if (!isAtEnd() && Macro &&652 (Tokens[CurTokenIdx].is(tok::hashhash) ||653 // Special processing of L#x macros in -fms-compatibility mode.654 // Microsoft compiler is able to form a wide string literal from655 // 'L#macro_arg' construct in a function-like macro.656 (PP.getLangOpts().MSVCCompat &&657 isWideStringLiteralFromMacro(Tok, Tokens[CurTokenIdx])))) {658 // When handling the microsoft /##/ extension, the final token is659 // returned by pasteTokens, not the pasted token.660 if (pasteTokens(Tok))661 return true;662 663 TokenIsFromPaste = true;664 }665 666 // The token's current location indicate where the token was lexed from. We667 // need this information to compute the spelling of the token, but any668 // diagnostics for the expanded token should appear as if they came from669 // ExpansionLoc. Pull this information together into a new SourceLocation670 // that captures all of this.671 if (ExpandLocStart.isValid() && // Don't do this for token streams.672 // Check that the token's location was not already set properly.673 SM.isBeforeInSLocAddrSpace(Tok.getLocation(), MacroStartSLocOffset)) {674 SourceLocation instLoc;675 if (Tok.is(tok::comment)) {676 instLoc = SM.createExpansionLoc(Tok.getLocation(),677 ExpandLocStart,678 ExpandLocEnd,679 Tok.getLength());680 } else {681 instLoc = getExpansionLocForMacroDefLoc(Tok.getLocation());682 }683 684 Tok.setLocation(instLoc);685 }686 687 // If this is the first token, set the lexical properties of the token to688 // match the lexical properties of the macro identifier.689 if (isFirstToken) {690 Tok.setFlagValue(Token::StartOfLine , AtStartOfLine);691 Tok.setFlagValue(Token::LeadingSpace, HasLeadingSpace);692 } else {693 // If this is not the first token, we may still need to pass through694 // leading whitespace if we've expanded a macro.695 if (AtStartOfLine) Tok.setFlag(Token::StartOfLine);696 if (HasLeadingSpace) Tok.setFlag(Token::LeadingSpace);697 }698 AtStartOfLine = false;699 HasLeadingSpace = false;700 701 // Handle recursive expansion!702 if (!Tok.isAnnotation() && Tok.getIdentifierInfo() != nullptr) {703 // Change the kind of this identifier to the appropriate token kind, e.g.704 // turning "for" into a keyword.705 IdentifierInfo *II = Tok.getIdentifierInfo();706 Tok.setKind(II->getTokenID());707 708 // If this identifier was poisoned and from a paste, emit an error. This709 // won't be handled by Preprocessor::HandleIdentifier because this is coming710 // from a macro expansion.711 if (II->isPoisoned() && TokenIsFromPaste) {712 PP.HandlePoisonedIdentifier(Tok);713 }714 715 if (!DisableMacroExpansion && II->isHandleIdentifierCase())716 return PP.HandleIdentifier(Tok);717 }718 719 // Otherwise, return a normal token.720 return true;721}722 723bool TokenLexer::pasteTokens(Token &Tok) {724 return pasteTokens(Tok, llvm::ArrayRef(Tokens, NumTokens), CurTokenIdx);725}726 727/// LHSTok is the LHS of a ## operator, and CurTokenIdx is the ##728/// operator. Read the ## and RHS, and paste the LHS/RHS together. If there729/// are more ## after it, chomp them iteratively. Return the result as LHSTok.730/// If this returns true, the caller should immediately return the token.731bool TokenLexer::pasteTokens(Token &LHSTok, ArrayRef<Token> TokenStream,732 unsigned int &CurIdx) {733 assert(CurIdx > 0 && "## can not be the first token within tokens");734 assert((TokenStream[CurIdx].is(tok::hashhash) ||735 (PP.getLangOpts().MSVCCompat &&736 isWideStringLiteralFromMacro(LHSTok, TokenStream[CurIdx]))) &&737 "Token at this Index must be ## or part of the MSVC 'L "738 "#macro-arg' pasting pair");739 740 // MSVC: If previous token was pasted, this must be a recovery from an invalid741 // paste operation. Ignore spaces before this token to mimic MSVC output.742 // Required for generating valid UUID strings in some MS headers.743 if (PP.getLangOpts().MicrosoftExt && (CurIdx >= 2) &&744 TokenStream[CurIdx - 2].is(tok::hashhash))745 LHSTok.clearFlag(Token::LeadingSpace);746 747 SmallString<128> Buffer;748 const char *ResultTokStrPtr = nullptr;749 SourceLocation StartLoc = LHSTok.getLocation();750 SourceLocation PasteOpLoc;751 bool HasUCNs = false;752 753 auto IsAtEnd = [&TokenStream, &CurIdx] {754 return TokenStream.size() == CurIdx;755 };756 757 do {758 // Consume the ## operator if any.759 PasteOpLoc = TokenStream[CurIdx].getLocation();760 if (TokenStream[CurIdx].is(tok::hashhash))761 ++CurIdx;762 assert(!IsAtEnd() && "No token on the RHS of a paste operator!");763 764 // Get the RHS token.765 const Token &RHS = TokenStream[CurIdx];766 767 // Allocate space for the result token. This is guaranteed to be enough for768 // the two tokens.769 Buffer.resize(LHSTok.getLength() + RHS.getLength());770 771 // Get the spelling of the LHS token in Buffer.772 const char *BufPtr = &Buffer[0];773 bool Invalid = false;774 unsigned LHSLen = PP.getSpelling(LHSTok, BufPtr, &Invalid);775 if (BufPtr != &Buffer[0]) // Really, we want the chars in Buffer!776 memcpy(&Buffer[0], BufPtr, LHSLen);777 if (Invalid)778 return true;779 780 BufPtr = Buffer.data() + LHSLen;781 unsigned RHSLen = PP.getSpelling(RHS, BufPtr, &Invalid);782 if (Invalid)783 return true;784 if (RHSLen && BufPtr != &Buffer[LHSLen])785 // Really, we want the chars in Buffer!786 memcpy(&Buffer[LHSLen], BufPtr, RHSLen);787 788 // Trim excess space.789 Buffer.resize(LHSLen+RHSLen);790 791 // Plop the pasted result (including the trailing newline and null) into a792 // scratch buffer where we can lex it.793 Token ResultTokTmp;794 ResultTokTmp.startToken();795 796 // Claim that the tmp token is a string_literal so that we can get the797 // character pointer back from CreateString in getLiteralData().798 ResultTokTmp.setKind(tok::string_literal);799 PP.CreateString(Buffer, ResultTokTmp);800 SourceLocation ResultTokLoc = ResultTokTmp.getLocation();801 ResultTokStrPtr = ResultTokTmp.getLiteralData();802 803 // Lex the resultant pasted token into Result.804 Token Result;805 806 if (LHSTok.isAnyIdentifier() && RHS.isAnyIdentifier()) {807 // Common paste case: identifier+identifier = identifier. Avoid creating808 // a lexer and other overhead.809 PP.IncrementPasteCounter(true);810 Result.startToken();811 Result.setKind(tok::raw_identifier);812 Result.setRawIdentifierData(ResultTokStrPtr);813 Result.setLocation(ResultTokLoc);814 Result.setLength(LHSLen+RHSLen);815 } else {816 PP.IncrementPasteCounter(false);817 818 assert(ResultTokLoc.isFileID() &&819 "Should be a raw location into scratch buffer");820 SourceManager &SourceMgr = PP.getSourceManager();821 FileID LocFileID = SourceMgr.getFileID(ResultTokLoc);822 823 bool Invalid = false;824 const char *ScratchBufStart825 = SourceMgr.getBufferData(LocFileID, &Invalid).data();826 if (Invalid)827 return false;828 829 // Make a lexer to lex this string from. Lex just this one token.830 // Make a lexer object so that we lex and expand the paste result.831 Lexer TL(SourceMgr.getLocForStartOfFile(LocFileID),832 PP.getLangOpts(), ScratchBufStart,833 ResultTokStrPtr, ResultTokStrPtr+LHSLen+RHSLen);834 835 // Lex a token in raw mode. This way it won't look up identifiers836 // automatically, lexing off the end will return an eof token, and837 // warnings are disabled. This returns true if the result token is the838 // entire buffer.839 bool isInvalid = !TL.LexFromRawLexer(Result);840 841 // If we got an EOF token, we didn't form even ONE token. For example, we842 // did "/ ## /" to get "//".843 isInvalid |= Result.is(tok::eof);844 845 // If pasting the two tokens didn't form a full new token, this is an846 // error. This occurs with "x ## +" and other stuff. Return with LHSTok847 // unmodified and with RHS as the next token to lex.848 if (isInvalid) {849 // Explicitly convert the token location to have proper expansion850 // information so that the user knows where it came from.851 SourceManager &SM = PP.getSourceManager();852 SourceLocation Loc =853 SM.createExpansionLoc(PasteOpLoc, ExpandLocStart, ExpandLocEnd, 2);854 855 // Test for the Microsoft extension of /##/ turning into // here on the856 // error path.857 if (PP.getLangOpts().MicrosoftExt && LHSTok.is(tok::slash) &&858 RHS.is(tok::slash)) {859 HandleMicrosoftCommentPaste(LHSTok, Loc);860 return true;861 }862 863 // Do not emit the error when preprocessing assembler code.864 if (!PP.getLangOpts().AsmPreprocessor) {865 // If we're in microsoft extensions mode, downgrade this from a hard866 // error to an extension that defaults to an error. This allows867 // disabling it.868 PP.Diag(Loc, PP.getLangOpts().MicrosoftExt ? diag::ext_pp_bad_paste_ms869 : diag::err_pp_bad_paste)870 << Buffer;871 }872 873 // An error has occurred so exit loop.874 break;875 }876 877 // Turn ## into 'unknown' to avoid # ## # from looking like a paste878 // operator.879 if (Result.is(tok::hashhash))880 Result.setKind(tok::unknown);881 }882 883 // Transfer properties of the LHS over the Result.884 Result.setFlagValue(Token::StartOfLine , LHSTok.isAtStartOfLine());885 Result.setFlagValue(Token::LeadingSpace, LHSTok.hasLeadingSpace());886 887 // Finally, replace LHS with the result, consume the RHS, and iterate.888 ++CurIdx;889 890 // Set Token::HasUCN flag if LHS or RHS contains any UCNs.891 HasUCNs = LHSTok.hasUCN() || RHS.hasUCN() || HasUCNs;892 LHSTok = Result;893 } while (!IsAtEnd() && TokenStream[CurIdx].is(tok::hashhash));894 895 SourceLocation EndLoc = TokenStream[CurIdx - 1].getLocation();896 897 // The token's current location indicate where the token was lexed from. We898 // need this information to compute the spelling of the token, but any899 // diagnostics for the expanded token should appear as if the token was900 // expanded from the full ## expression. Pull this information together into901 // a new SourceLocation that captures all of this.902 SourceManager &SM = PP.getSourceManager();903 if (StartLoc.isFileID())904 StartLoc = getExpansionLocForMacroDefLoc(StartLoc);905 if (EndLoc.isFileID())906 EndLoc = getExpansionLocForMacroDefLoc(EndLoc);907 FileID MacroFID = SM.getFileID(MacroExpansionStart);908 while (SM.getFileID(StartLoc) != MacroFID)909 StartLoc = SM.getImmediateExpansionRange(StartLoc).getBegin();910 while (SM.getFileID(EndLoc) != MacroFID)911 EndLoc = SM.getImmediateExpansionRange(EndLoc).getEnd();912 913 LHSTok.setLocation(SM.createExpansionLoc(LHSTok.getLocation(), StartLoc, EndLoc,914 LHSTok.getLength()));915 916 // Now that we got the result token, it will be subject to expansion. Since917 // token pasting re-lexes the result token in raw mode, identifier information918 // isn't looked up. As such, if the result is an identifier, look up id info.919 if (LHSTok.is(tok::raw_identifier)) {920 921 // If there has any UNCs in concated token, we should mark this token922 // with Token::HasUCN flag, then LookUpIdentifierInfo will expand UCNs in923 // token.924 if (HasUCNs)925 LHSTok.setFlag(Token::HasUCN);926 927 // Look up the identifier info for the token. We disabled identifier lookup928 // by saying we're skipping contents, so we need to do this manually.929 PP.LookUpIdentifierInfo(LHSTok);930 }931 return false;932}933 934/// isNextTokenLParen - If the next token lexed will pop this macro off the935/// expansion stack, return std::nullopt, otherwise return the next unexpanded936/// token.937std::optional<Token> TokenLexer::peekNextPPToken() const {938 // Out of tokens?939 if (isAtEnd())940 return std::nullopt;941 return Tokens[CurTokenIdx];942}943 944/// isParsingPreprocessorDirective - Return true if we are in the middle of a945/// preprocessor directive.946bool TokenLexer::isParsingPreprocessorDirective() const {947 return Tokens[NumTokens-1].is(tok::eod) && !isAtEnd();948}949 950/// HandleMicrosoftCommentPaste - In microsoft compatibility mode, /##/ pastes951/// together to form a comment that comments out everything in the current952/// macro, other active macros, and anything left on the current physical953/// source line of the expanded buffer. Handle this by returning the954/// first token on the next line.955void TokenLexer::HandleMicrosoftCommentPaste(Token &Tok, SourceLocation OpLoc) {956 PP.Diag(OpLoc, diag::ext_comment_paste_microsoft);957 958 // We 'comment out' the rest of this macro by just ignoring the rest of the959 // tokens that have not been lexed yet, if any.960 961 // Since this must be a macro, mark the macro enabled now that it is no longer962 // being expanded.963 assert(Macro && "Token streams can't paste comments");964 Macro->EnableMacro();965 966 PP.HandleMicrosoftCommentPaste(Tok);967}968 969/// If \arg loc is a file ID and points inside the current macro970/// definition, returns the appropriate source location pointing at the971/// macro expansion source location entry, otherwise it returns an invalid972/// SourceLocation.973SourceLocation974TokenLexer::getExpansionLocForMacroDefLoc(SourceLocation loc) const {975 assert(ExpandLocStart.isValid() && MacroExpansionStart.isValid() &&976 "Not appropriate for token streams");977 assert(loc.isValid() && loc.isFileID());978 979 SourceManager &SM = PP.getSourceManager();980 assert(SM.isInSLocAddrSpace(loc, MacroDefStart, MacroDefLength) &&981 "Expected loc to come from the macro definition");982 983 SourceLocation::UIntTy relativeOffset = 0;984 SM.isInSLocAddrSpace(loc, MacroDefStart, MacroDefLength, &relativeOffset);985 return MacroExpansionStart.getLocWithOffset(relativeOffset);986}987 988/// Finds the tokens that are consecutive (from the same FileID)989/// creates a single SLocEntry, and assigns SourceLocations to each token that990/// point to that SLocEntry. e.g for991/// assert(foo == bar);992/// There will be a single SLocEntry for the "foo == bar" chunk and locations993/// for the 'foo', '==', 'bar' tokens will point inside that chunk.994///995/// \arg begin_tokens will be updated to a position past all the found996/// consecutive tokens.997static void updateConsecutiveMacroArgTokens(SourceManager &SM,998 SourceLocation ExpandLoc,999 Token *&begin_tokens,1000 Token * end_tokens) {1001 assert(begin_tokens + 1 < end_tokens);1002 SourceLocation BeginLoc = begin_tokens->getLocation();1003 llvm::MutableArrayRef<Token> All(begin_tokens, end_tokens);1004 llvm::MutableArrayRef<Token> Partition;1005 1006 auto NearLast = [&, Last = BeginLoc](SourceLocation Loc) mutable {1007 // The maximum distance between two consecutive tokens in a partition.1008 // This is an important trick to avoid using too much SourceLocation address1009 // space!1010 static constexpr SourceLocation::IntTy MaxDistance = 50;1011 auto Distance = Loc.getRawEncoding() - Last.getRawEncoding();1012 Last = Loc;1013 return Distance <= MaxDistance;1014 };1015 1016 // Partition the tokens by their FileID.1017 // This is a hot function, and calling getFileID can be expensive, the1018 // implementation is optimized by reducing the number of getFileID.1019 if (BeginLoc.isFileID()) {1020 // Consecutive tokens not written in macros must be from the same file.1021 // (Neither #include nor eof can occur inside a macro argument.)1022 Partition = All.take_while([&](const Token &T) {1023 return T.getLocation().isFileID() && NearLast(T.getLocation());1024 });1025 } else {1026 // Call getFileID once to calculate the bounds, and use the cheaper1027 // sourcelocation-against-bounds comparison.1028 FileID BeginFID = SM.getFileID(BeginLoc);1029 SourceLocation Limit =1030 SM.getComposedLoc(BeginFID, SM.getFileIDSize(BeginFID));1031 Partition = All.take_while([&](const Token &T) {1032 // NOTE: the Limit is included! The lexer recovery only ever inserts a1033 // single token past the end of the FileID, specifically the ) when a1034 // macro-arg containing a comma should be guarded by parentheses.1035 //1036 // It is safe to include the Limit here because SourceManager allocates1037 // FileSize + 1 for each SLocEntry.1038 //1039 // See https://github.com/llvm/llvm-project/issues/60722.1040 return T.getLocation() >= BeginLoc && T.getLocation() <= Limit1041 && NearLast(T.getLocation());1042 });1043 }1044 assert(!Partition.empty());1045 1046 // For the consecutive tokens, find the length of the SLocEntry to contain1047 // all of them.1048 SourceLocation::UIntTy FullLength =1049 Partition.back().getEndLoc().getRawEncoding() -1050 Partition.front().getLocation().getRawEncoding();1051 // Create a macro expansion SLocEntry that will "contain" all of the tokens.1052 SourceLocation Expansion =1053 SM.createMacroArgExpansionLoc(BeginLoc, ExpandLoc, FullLength);1054 1055#ifdef EXPENSIVE_CHECKS1056 assert(llvm::all_of(Partition.drop_front(),1057 [&SM, ID = SM.getFileID(Partition.front().getLocation())](1058 const Token &T) {1059 return ID == SM.getFileID(T.getLocation());1060 }) &&1061 "Must have the same FIleID!");1062#endif1063 // Change the location of the tokens from the spelling location to the new1064 // expanded location.1065 for (Token& T : Partition) {1066 SourceLocation::IntTy RelativeOffset =1067 T.getLocation().getRawEncoding() - BeginLoc.getRawEncoding();1068 T.setLocation(Expansion.getLocWithOffset(RelativeOffset));1069 }1070 begin_tokens = &Partition.back() + 1;1071}1072 1073/// Creates SLocEntries and updates the locations of macro argument1074/// tokens to their new expanded locations.1075///1076/// \param ArgIdSpellLoc the location of the macro argument id inside the macro1077/// definition.1078void TokenLexer::updateLocForMacroArgTokens(SourceLocation ArgIdSpellLoc,1079 Token *begin_tokens,1080 Token *end_tokens) {1081 SourceManager &SM = PP.getSourceManager();1082 1083 SourceLocation ExpandLoc =1084 getExpansionLocForMacroDefLoc(ArgIdSpellLoc);1085 1086 while (begin_tokens < end_tokens) {1087 // If there's only one token just create a SLocEntry for it.1088 if (end_tokens - begin_tokens == 1) {1089 Token &Tok = *begin_tokens;1090 Tok.setLocation(SM.createMacroArgExpansionLoc(Tok.getLocation(),1091 ExpandLoc,1092 Tok.getLength()));1093 return;1094 }1095 1096 updateConsecutiveMacroArgTokens(SM, ExpandLoc, begin_tokens, end_tokens);1097 }1098}1099 1100void TokenLexer::PropagateLineStartLeadingSpaceInfo(Token &Result) {1101 AtStartOfLine = Result.isAtStartOfLine();1102 HasLeadingSpace = Result.hasLeadingSpace();1103}1104