510 lines · cpp
1//===--- SourceCode.cpp - Source code manipulation routines -----*- C++ -*-===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// This file provides functions that simplify extraction of source code.10//11//===----------------------------------------------------------------------===//12#include "clang/Tooling/Transformer/SourceCode.h"13#include "clang/AST/ASTContext.h"14#include "clang/AST/Attr.h"15#include "clang/AST/Comment.h"16#include "clang/AST/Decl.h"17#include "clang/AST/DeclCXX.h"18#include "clang/AST/DeclTemplate.h"19#include "clang/AST/Expr.h"20#include "clang/Basic/SourceManager.h"21#include "clang/Lex/Lexer.h"22#include "llvm/Support/Errc.h"23#include "llvm/Support/Error.h"24#include <set>25 26using namespace clang;27 28using llvm::errc;29using llvm::StringError;30 31StringRef clang::tooling::getText(CharSourceRange Range,32 const ASTContext &Context) {33 return Lexer::getSourceText(Range, Context.getSourceManager(),34 Context.getLangOpts());35}36 37CharSourceRange clang::tooling::maybeExtendRange(CharSourceRange Range,38 tok::TokenKind Next,39 ASTContext &Context) {40 CharSourceRange R = Lexer::getAsCharRange(Range, Context.getSourceManager(),41 Context.getLangOpts());42 if (R.isInvalid())43 return Range;44 Token Tok;45 bool Err =46 Lexer::getRawToken(R.getEnd(), Tok, Context.getSourceManager(),47 Context.getLangOpts(), /*IgnoreWhiteSpace=*/true);48 if (Err || !Tok.is(Next))49 return Range;50 return CharSourceRange::getTokenRange(Range.getBegin(), Tok.getLocation());51}52 53llvm::Error clang::tooling::validateRange(const CharSourceRange &Range,54 const SourceManager &SM,55 bool AllowSystemHeaders) {56 if (Range.isInvalid())57 return llvm::make_error<StringError>(errc::invalid_argument,58 "Invalid range");59 60 if (Range.getBegin().isMacroID() || Range.getEnd().isMacroID())61 return llvm::make_error<StringError>(62 errc::invalid_argument, "Range starts or ends in a macro expansion");63 64 if (!AllowSystemHeaders) {65 if (SM.isInSystemHeader(Range.getBegin()) ||66 SM.isInSystemHeader(Range.getEnd()))67 return llvm::make_error<StringError>(errc::invalid_argument,68 "Range is in system header");69 }70 71 FileIDAndOffset BeginInfo = SM.getDecomposedLoc(Range.getBegin());72 FileIDAndOffset EndInfo = SM.getDecomposedLoc(Range.getEnd());73 if (BeginInfo.first != EndInfo.first)74 return llvm::make_error<StringError>(75 errc::invalid_argument, "Range begins and ends in different files");76 77 if (BeginInfo.second > EndInfo.second)78 return llvm::make_error<StringError>(errc::invalid_argument,79 "Range's begin is past its end");80 81 return llvm::Error::success();82}83 84llvm::Error clang::tooling::validateEditRange(const CharSourceRange &Range,85 const SourceManager &SM) {86 return validateRange(Range, SM, /*AllowSystemHeaders=*/false);87}88 89// Returns the location of the top-level macro argument that is the spelling for90// the expansion `Loc` is from. If `Loc` is spelled in the macro definition,91// returns an invalid `SourceLocation`.92static SourceLocation getMacroArgumentSpellingLoc(SourceLocation Loc,93 const SourceManager &SM) {94 assert(Loc.isMacroID() && "Location must be in a macro");95 while (Loc.isMacroID()) {96 const auto &Expansion = SM.getSLocEntry(SM.getFileID(Loc)).getExpansion();97 if (Expansion.isMacroArgExpansion()) {98 // Check the spelling location of the macro arg, in case the arg itself is99 // in a macro expansion.100 Loc = Expansion.getSpellingLoc();101 } else {102 return {};103 }104 }105 return Loc;106}107 108static bool spelledInMacroDefinition(CharSourceRange Range,109 const SourceManager &SM) {110 if (Range.getBegin().isMacroID() && Range.getEnd().isMacroID()) {111 // Check whether the range is entirely within a single macro argument.112 auto B = getMacroArgumentSpellingLoc(Range.getBegin(), SM);113 auto E = getMacroArgumentSpellingLoc(Range.getEnd(), SM);114 return B.isInvalid() || B != E;115 }116 117 return Range.getBegin().isMacroID() || Range.getEnd().isMacroID();118}119 120// Returns the expansion char-range of `Loc` if `Loc` is a split token. For121// example, `>>` in nested templates needs the first `>` to be split, otherwise122// the `SourceLocation` of the token would lex as `>>` instead of `>`.123static std::optional<CharSourceRange>124getExpansionForSplitToken(SourceLocation Loc, const SourceManager &SM,125 const LangOptions &LangOpts) {126 if (Loc.isMacroID()) {127 bool Invalid = false;128 auto &SLoc = SM.getSLocEntry(SM.getFileID(Loc), &Invalid);129 if (Invalid)130 return std::nullopt;131 if (auto &Expansion = SLoc.getExpansion();132 !Expansion.isExpansionTokenRange()) {133 // A char-range expansion is only used where a token-range would be134 // incorrect, and so identifies this as a split token (and importantly,135 // not as a macro).136 return Expansion.getExpansionLocRange();137 }138 }139 return std::nullopt;140}141 142// If `Range` covers a split token, returns the expansion range, otherwise143// returns `Range`.144static CharSourceRange getRangeForSplitTokens(CharSourceRange Range,145 const SourceManager &SM,146 const LangOptions &LangOpts) {147 if (Range.isTokenRange()) {148 auto BeginToken = getExpansionForSplitToken(Range.getBegin(), SM, LangOpts);149 auto EndToken = getExpansionForSplitToken(Range.getEnd(), SM, LangOpts);150 if (EndToken) {151 SourceLocation BeginLoc =152 BeginToken ? BeginToken->getBegin() : Range.getBegin();153 // We can't use the expansion location with a token-range, because that154 // will incorrectly lex the end token, so use a char-range that ends at155 // the split.156 return CharSourceRange::getCharRange(BeginLoc, EndToken->getEnd());157 } else if (BeginToken) {158 // Since the end token is not split, the whole range covers the split, so159 // the only adjustment we make is to use the expansion location of the160 // begin token.161 return CharSourceRange::getTokenRange(BeginToken->getBegin(),162 Range.getEnd());163 }164 }165 return Range;166}167 168static CharSourceRange getRange(const CharSourceRange &EditRange,169 const SourceManager &SM,170 const LangOptions &LangOpts,171 bool IncludeMacroExpansion) {172 CharSourceRange Range;173 if (IncludeMacroExpansion) {174 Range = Lexer::makeFileCharRange(EditRange, SM, LangOpts);175 } else {176 auto AdjustedRange = getRangeForSplitTokens(EditRange, SM, LangOpts);177 if (spelledInMacroDefinition(AdjustedRange, SM))178 return {};179 180 auto B = SM.getSpellingLoc(AdjustedRange.getBegin());181 auto E = SM.getSpellingLoc(AdjustedRange.getEnd());182 if (AdjustedRange.isTokenRange())183 E = Lexer::getLocForEndOfToken(E, 0, SM, LangOpts);184 Range = CharSourceRange::getCharRange(B, E);185 }186 return Range;187}188 189std::optional<CharSourceRange> clang::tooling::getFileRangeForEdit(190 const CharSourceRange &EditRange, const SourceManager &SM,191 const LangOptions &LangOpts, bool IncludeMacroExpansion) {192 CharSourceRange Range =193 getRange(EditRange, SM, LangOpts, IncludeMacroExpansion);194 bool IsInvalid = llvm::errorToBool(validateEditRange(Range, SM));195 if (IsInvalid)196 return std::nullopt;197 return Range;198}199 200std::optional<CharSourceRange> clang::tooling::getFileRange(201 const CharSourceRange &EditRange, const SourceManager &SM,202 const LangOptions &LangOpts, bool IncludeMacroExpansion) {203 CharSourceRange Range =204 getRange(EditRange, SM, LangOpts, IncludeMacroExpansion);205 bool IsInvalid =206 llvm::errorToBool(validateRange(Range, SM, /*AllowSystemHeaders=*/true));207 if (IsInvalid)208 return std::nullopt;209 return Range;210}211 212static bool startsWithNewline(const SourceManager &SM, const Token &Tok) {213 return isVerticalWhitespace(SM.getCharacterData(Tok.getLocation())[0]);214}215 216static bool contains(const std::set<tok::TokenKind> &Terminators,217 const Token &Tok) {218 return Terminators.count(Tok.getKind()) > 0;219}220 221// Returns the exclusive, *file* end location of the entity whose last token is222// at location 'EntityLast'. That is, it returns the location one past the last223// relevant character.224//225// Associated tokens include comments, horizontal whitespace and 'Terminators'226// -- optional tokens, which, if any are found, will be included; if227// 'Terminators' is empty, we will not include any extra tokens beyond comments228// and horizontal whitespace.229static SourceLocation230getEntityEndLoc(const SourceManager &SM, SourceLocation EntityLast,231 const std::set<tok::TokenKind> &Terminators,232 const LangOptions &LangOpts) {233 assert(EntityLast.isValid() && "Invalid end location found.");234 235 // We remember the last location of a non-horizontal-whitespace token we have236 // lexed; this is the location up to which we will want to delete.237 // FIXME: Support using the spelling loc here for cases where we want to238 // analyze the macro text.239 240 CharSourceRange ExpansionRange = SM.getExpansionRange(EntityLast);241 // FIXME: Should check isTokenRange(), for the (rare) case that242 // `ExpansionRange` is a character range.243 std::unique_ptr<Lexer> Lexer = [&]() {244 bool Invalid = false;245 auto FileOffset = SM.getDecomposedLoc(ExpansionRange.getEnd());246 llvm::StringRef File = SM.getBufferData(FileOffset.first, &Invalid);247 assert(!Invalid && "Cannot get file/offset");248 return std::make_unique<clang::Lexer>(249 SM.getLocForStartOfFile(FileOffset.first), LangOpts, File.begin(),250 File.data() + FileOffset.second, File.end());251 }();252 253 // Tell Lexer to return whitespace as pseudo-tokens (kind is tok::unknown).254 Lexer->SetKeepWhitespaceMode(true);255 256 // Generally, the code we want to include looks like this ([] are optional),257 // If Terminators is empty:258 // [ <comment> ] [ <newline> ]259 // Otherwise:260 // ... <terminator> [ <comment> ] [ <newline> ]261 262 Token Tok;263 bool Terminated = false;264 265 // First, lex to the current token (which is the last token of the range that266 // is definitely associated with the decl). Then, we process the first token267 // separately from the rest based on conditions that hold specifically for268 // that first token.269 //270 // We do not search for a terminator if none is required or we've already271 // encountered it. Otherwise, if the original `EntityLast` location was in a272 // macro expansion, we don't have visibility into the text, so we assume we've273 // already terminated. However, we note this assumption with274 // `TerminatedByMacro`, because we'll want to handle it somewhat differently275 // for the terminators semicolon and comma. These terminators can be safely276 // associated with the entity when they appear after the macro -- extra277 // semicolons have no effect on the program and a well-formed program won't278 // have multiple commas in a row, so we're guaranteed that there is only one.279 //280 // FIXME: This handling of macros is more conservative than necessary. When281 // the end of the expansion coincides with the end of the node, we can still282 // safely analyze the code. But, it is more complicated, because we need to283 // start by lexing the spelling loc for the first token and then switch to the284 // expansion loc.285 bool TerminatedByMacro = false;286 Lexer->LexFromRawLexer(Tok);287 if (Terminators.empty() || contains(Terminators, Tok))288 Terminated = true;289 else if (EntityLast.isMacroID()) {290 Terminated = true;291 TerminatedByMacro = true;292 }293 294 // We save the most recent candidate for the exclusive end location.295 SourceLocation End = Tok.getEndLoc();296 297 while (!Terminated) {298 // Lex the next token we want to possibly expand the range with.299 Lexer->LexFromRawLexer(Tok);300 301 switch (Tok.getKind()) {302 case tok::eof:303 // Unexpected separators.304 case tok::l_brace:305 case tok::r_brace:306 case tok::comma:307 return End;308 // Whitespace pseudo-tokens.309 case tok::unknown:310 if (startsWithNewline(SM, Tok))311 // Include at least until the end of the line.312 End = Tok.getEndLoc();313 break;314 default:315 if (contains(Terminators, Tok))316 Terminated = true;317 End = Tok.getEndLoc();318 break;319 }320 }321 322 do {323 // Lex the next token we want to possibly expand the range with.324 Lexer->LexFromRawLexer(Tok);325 326 switch (Tok.getKind()) {327 case tok::unknown:328 if (startsWithNewline(SM, Tok))329 // We're done, but include this newline.330 return Tok.getEndLoc();331 break;332 case tok::comment:333 // Include any comments we find on the way.334 End = Tok.getEndLoc();335 break;336 case tok::semi:337 case tok::comma:338 if (TerminatedByMacro && contains(Terminators, Tok)) {339 End = Tok.getEndLoc();340 // We've found a real terminator.341 TerminatedByMacro = false;342 break;343 }344 // Found an unrelated token; stop and don't include it.345 return End;346 default:347 // Found an unrelated token; stop and don't include it.348 return End;349 }350 } while (true);351}352 353// Returns the expected terminator tokens for the given declaration.354//355// If we do not know the correct terminator token, returns an empty set.356//357// There are cases where we have more than one possible terminator (for example,358// we find either a comma or a semicolon after a VarDecl).359static std::set<tok::TokenKind> getTerminators(const Decl &D) {360 if (llvm::isa<RecordDecl>(D) || llvm::isa<UsingDecl>(D))361 return {tok::semi};362 363 if (llvm::isa<FunctionDecl>(D) || llvm::isa<LinkageSpecDecl>(D))364 return {tok::r_brace, tok::semi};365 366 if (llvm::isa<VarDecl>(D) || llvm::isa<FieldDecl>(D))367 return {tok::comma, tok::semi};368 369 return {};370}371 372// Starting from `Loc`, skips whitespace up to, and including, a single373// newline. Returns the (exclusive) end of any skipped whitespace (that is, the374// location immediately after the whitespace).375static SourceLocation skipWhitespaceAndNewline(const SourceManager &SM,376 SourceLocation Loc,377 const LangOptions &LangOpts) {378 const char *LocChars = SM.getCharacterData(Loc);379 int i = 0;380 while (isHorizontalWhitespace(LocChars[i]))381 ++i;382 if (isVerticalWhitespace(LocChars[i]))383 ++i;384 return Loc.getLocWithOffset(i);385}386 387// Is `Loc` separated from any following decl by something meaningful (e.g. an388// empty line, a comment), ignoring horizontal whitespace? Since this is a389// heuristic, we return false when in doubt. `Loc` cannot be the first location390// in the file.391static bool atOrBeforeSeparation(const SourceManager &SM, SourceLocation Loc,392 const LangOptions &LangOpts) {393 // If the preceding character is a newline, we'll check for an empty line as a394 // separator. However, we can't identify an empty line using tokens, so we395 // analyse the characters. If we try to use tokens, we'll just end up with a396 // whitespace token, whose characters we'd have to analyse anyhow.397 bool Invalid = false;398 const char *LocChars =399 SM.getCharacterData(Loc.getLocWithOffset(-1), &Invalid);400 assert(!Invalid &&401 "Loc must be a valid character and not the first of the source file.");402 if (isVerticalWhitespace(LocChars[0])) {403 for (int i = 1; isWhitespace(LocChars[i]); ++i)404 if (isVerticalWhitespace(LocChars[i]))405 return true;406 }407 // We didn't find an empty line, so lex the next token, skipping past any408 // whitespace we just scanned.409 Token Tok;410 bool Failed = Lexer::getRawToken(Loc, Tok, SM, LangOpts,411 /*IgnoreWhiteSpace=*/true);412 if (Failed)413 // Any text that confuses the lexer seems fair to consider a separation.414 return true;415 416 switch (Tok.getKind()) {417 case tok::comment:418 case tok::l_brace:419 case tok::r_brace:420 case tok::eof:421 return true;422 default:423 return false;424 }425}426 427CharSourceRange tooling::getAssociatedRange(const Decl &Decl,428 ASTContext &Context) {429 const SourceManager &SM = Context.getSourceManager();430 const LangOptions &LangOpts = Context.getLangOpts();431 CharSourceRange Range = CharSourceRange::getTokenRange(Decl.getSourceRange());432 433 // First, expand to the start of the template<> declaration if necessary.434 if (const auto *Record = llvm::dyn_cast<CXXRecordDecl>(&Decl)) {435 if (const auto *T = Record->getDescribedClassTemplate())436 if (SM.isBeforeInTranslationUnit(T->getBeginLoc(), Range.getBegin()))437 Range.setBegin(T->getBeginLoc());438 } else if (const auto *F = llvm::dyn_cast<FunctionDecl>(&Decl)) {439 if (const auto *T = F->getDescribedFunctionTemplate())440 if (SM.isBeforeInTranslationUnit(T->getBeginLoc(), Range.getBegin()))441 Range.setBegin(T->getBeginLoc());442 }443 444 // Next, expand the end location past trailing comments to include a potential445 // newline at the end of the decl's line.446 Range.setEnd(447 getEntityEndLoc(SM, Decl.getEndLoc(), getTerminators(Decl), LangOpts));448 Range.setTokenRange(false);449 450 // Expand to include preceeding associated comments. We ignore any comments451 // that are not preceeding the decl, since we've already skipped trailing452 // comments with getEntityEndLoc.453 if (const RawComment *Comment =454 Decl.getASTContext().getRawCommentForDeclNoCache(&Decl))455 // Only include a preceding comment if:456 // * it is *not* separate from the declaration (not including any newline457 // that immediately follows the comment),458 // * the decl *is* separate from any following entity (so, there are no459 // other entities the comment could refer to), and460 // * it is not a IfThisThenThat lint check.461 if (SM.isBeforeInTranslationUnit(Comment->getBeginLoc(),462 Range.getBegin()) &&463 !atOrBeforeSeparation(464 SM, skipWhitespaceAndNewline(SM, Comment->getEndLoc(), LangOpts),465 LangOpts) &&466 atOrBeforeSeparation(SM, Range.getEnd(), LangOpts)) {467 const StringRef CommentText = Comment->getRawText(SM);468 if (!CommentText.contains("LINT.IfChange") &&469 !CommentText.contains("LINT.ThenChange"))470 Range.setBegin(Comment->getBeginLoc());471 }472 // Add leading attributes.473 for (auto *Attr : Decl.attrs()) {474 if (Attr->getLocation().isInvalid() ||475 !SM.isBeforeInTranslationUnit(Attr->getLocation(), Range.getBegin()))476 continue;477 Range.setBegin(Attr->getLocation());478 479 // Extend to the left '[[' or '__attribute((' if we saw the attribute,480 // unless it is not a valid location.481 bool Invalid;482 StringRef Source =483 SM.getBufferData(SM.getFileID(Range.getBegin()), &Invalid);484 if (Invalid)485 continue;486 llvm::StringRef BeforeAttr =487 Source.substr(0, SM.getFileOffset(Range.getBegin()));488 llvm::StringRef BeforeAttrStripped = BeforeAttr.rtrim();489 490 for (llvm::StringRef Prefix : {"[[", "__attribute__(("}) {491 // Handle whitespace between attribute prefix and attribute value.492 if (BeforeAttrStripped.ends_with(Prefix)) {493 // Move start to start position of prefix, which is494 // length(BeforeAttr) - length(BeforeAttrStripped) + length(Prefix)495 // positions to the left.496 Range.setBegin(Range.getBegin().getLocWithOffset(static_cast<int>(497 -BeforeAttr.size() + BeforeAttrStripped.size() - Prefix.size())));498 break;499 // If we didn't see '[[' or '__attribute' it's probably coming from a500 // macro expansion which is already handled by makeFileCharRange(),501 // below.502 }503 }504 }505 506 // Range.getEnd() is already fully un-expanded by getEntityEndLoc. But,507 // Range.getBegin() may be inside an expansion.508 return Lexer::makeFileCharRange(Range, SM, LangOpts);509}510