1158 lines · cpp
1//===--- PrintPreprocessedOutput.cpp - Implement the -E mode --------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// This code simply runs the preprocessor on the input file and prints out the10// result. This is the traditional behavior of the -E option.11//12//===----------------------------------------------------------------------===//13 14#include "clang/Basic/CharInfo.h"15#include "clang/Basic/Diagnostic.h"16#include "clang/Basic/SourceManager.h"17#include "clang/Frontend/PreprocessorOutputOptions.h"18#include "clang/Frontend/Utils.h"19#include "clang/Lex/MacroInfo.h"20#include "clang/Lex/PPCallbacks.h"21#include "clang/Lex/Pragma.h"22#include "clang/Lex/Preprocessor.h"23#include "clang/Lex/TokenConcatenation.h"24#include "llvm/ADT/STLExtras.h"25#include "llvm/ADT/StringRef.h"26#include "llvm/Support/ErrorHandling.h"27#include "llvm/Support/raw_ostream.h"28#include <cstdio>29using namespace clang;30 31/// PrintMacroDefinition - Print a macro definition in a form that will be32/// properly accepted back as a definition.33static void PrintMacroDefinition(const IdentifierInfo &II, const MacroInfo &MI,34 Preprocessor &PP, raw_ostream *OS) {35 *OS << "#define " << II.getName();36 37 if (MI.isFunctionLike()) {38 *OS << '(';39 if (!MI.param_empty()) {40 MacroInfo::param_iterator AI = MI.param_begin(), E = MI.param_end();41 for (; AI+1 != E; ++AI) {42 *OS << (*AI)->getName();43 *OS << ',';44 }45 46 // Last argument.47 if ((*AI)->getName() == "__VA_ARGS__")48 *OS << "...";49 else50 *OS << (*AI)->getName();51 }52 53 if (MI.isGNUVarargs())54 *OS << "..."; // #define foo(x...)55 56 *OS << ')';57 }58 59 // GCC always emits a space, even if the macro body is empty. However, do not60 // want to emit two spaces if the first token has a leading space.61 if (MI.tokens_empty() || !MI.tokens_begin()->hasLeadingSpace())62 *OS << ' ';63 64 SmallString<128> SpellingBuffer;65 for (const auto &T : MI.tokens()) {66 if (T.hasLeadingSpace())67 *OS << ' ';68 69 *OS << PP.getSpelling(T, SpellingBuffer);70 }71}72 73//===----------------------------------------------------------------------===//74// Preprocessed token printer75//===----------------------------------------------------------------------===//76 77namespace {78class PrintPPOutputPPCallbacks : public PPCallbacks {79 Preprocessor &PP;80 SourceManager &SM;81 TokenConcatenation ConcatInfo;82public:83 raw_ostream *OS;84private:85 unsigned CurLine;86 87 bool EmittedTokensOnThisLine;88 bool EmittedDirectiveOnThisLine;89 SrcMgr::CharacteristicKind FileType;90 SmallString<512> CurFilename;91 bool Initialized;92 bool DisableLineMarkers;93 bool DumpDefines;94 bool DumpIncludeDirectives;95 bool DumpEmbedDirectives;96 bool UseLineDirectives;97 bool IsFirstFileEntered;98 bool MinimizeWhitespace;99 bool DirectivesOnly;100 bool KeepSystemIncludes;101 raw_ostream *OrigOS;102 std::unique_ptr<llvm::raw_null_ostream> NullOS;103 unsigned NumToksToSkip;104 105 Token PrevTok;106 Token PrevPrevTok;107 108public:109 PrintPPOutputPPCallbacks(Preprocessor &pp, raw_ostream *os, bool lineMarkers,110 bool defines, bool DumpIncludeDirectives,111 bool DumpEmbedDirectives, bool UseLineDirectives,112 bool MinimizeWhitespace, bool DirectivesOnly,113 bool KeepSystemIncludes)114 : PP(pp), SM(PP.getSourceManager()), ConcatInfo(PP), OS(os),115 DisableLineMarkers(lineMarkers), DumpDefines(defines),116 DumpIncludeDirectives(DumpIncludeDirectives),117 DumpEmbedDirectives(DumpEmbedDirectives),118 UseLineDirectives(UseLineDirectives),119 MinimizeWhitespace(MinimizeWhitespace), DirectivesOnly(DirectivesOnly),120 KeepSystemIncludes(KeepSystemIncludes), OrigOS(os), NumToksToSkip(0) {121 CurLine = 0;122 CurFilename += "<uninit>";123 EmittedTokensOnThisLine = false;124 EmittedDirectiveOnThisLine = false;125 FileType = SrcMgr::C_User;126 Initialized = false;127 IsFirstFileEntered = false;128 if (KeepSystemIncludes)129 NullOS = std::make_unique<llvm::raw_null_ostream>();130 131 PrevTok.startToken();132 PrevPrevTok.startToken();133 }134 135 /// Returns true if #embed directives should be expanded into a comma-136 /// delimited list of integer constants or not.137 bool expandEmbedContents() const { return !DumpEmbedDirectives; }138 139 bool isMinimizeWhitespace() const { return MinimizeWhitespace; }140 141 void setEmittedTokensOnThisLine() { EmittedTokensOnThisLine = true; }142 bool hasEmittedTokensOnThisLine() const { return EmittedTokensOnThisLine; }143 144 void setEmittedDirectiveOnThisLine() { EmittedDirectiveOnThisLine = true; }145 bool hasEmittedDirectiveOnThisLine() const {146 return EmittedDirectiveOnThisLine;147 }148 149 /// Ensure that the output stream position is at the beginning of a new line150 /// and inserts one if it does not. It is intended to ensure that directives151 /// inserted by the directives not from the input source (such as #line) are152 /// in the first column. To insert newlines that represent the input, use153 /// MoveToLine(/*...*/, /*RequireStartOfLine=*/true).154 void startNewLineIfNeeded();155 156 void FileChanged(SourceLocation Loc, FileChangeReason Reason,157 SrcMgr::CharacteristicKind FileType,158 FileID PrevFID) override;159 void EmbedDirective(SourceLocation HashLoc, StringRef FileName, bool IsAngled,160 OptionalFileEntryRef File,161 const LexEmbedParametersResult &Params) override;162 void InclusionDirective(SourceLocation HashLoc, const Token &IncludeTok,163 StringRef FileName, bool IsAngled,164 CharSourceRange FilenameRange,165 OptionalFileEntryRef File, StringRef SearchPath,166 StringRef RelativePath, const Module *SuggestedModule,167 bool ModuleImported,168 SrcMgr::CharacteristicKind FileType) override;169 void Ident(SourceLocation Loc, StringRef str) override;170 void PragmaMessage(SourceLocation Loc, StringRef Namespace,171 PragmaMessageKind Kind, StringRef Str) override;172 void PragmaDebug(SourceLocation Loc, StringRef DebugType) override;173 void PragmaDiagnosticPush(SourceLocation Loc, StringRef Namespace) override;174 void PragmaDiagnosticPop(SourceLocation Loc, StringRef Namespace) override;175 void PragmaDiagnostic(SourceLocation Loc, StringRef Namespace,176 diag::Severity Map, StringRef Str) override;177 void PragmaWarning(SourceLocation Loc, PragmaWarningSpecifier WarningSpec,178 ArrayRef<int> Ids) override;179 void PragmaWarningPush(SourceLocation Loc, int Level) override;180 void PragmaWarningPop(SourceLocation Loc) override;181 void PragmaExecCharsetPush(SourceLocation Loc, StringRef Str) override;182 void PragmaExecCharsetPop(SourceLocation Loc) override;183 void PragmaAssumeNonNullBegin(SourceLocation Loc) override;184 void PragmaAssumeNonNullEnd(SourceLocation Loc) override;185 186 /// Insert whitespace before emitting the next token.187 ///188 /// @param Tok Next token to be emitted.189 /// @param RequireSpace Ensure at least one whitespace is emitted. Useful190 /// if non-tokens have been emitted to the stream.191 /// @param RequireSameLine Never emit newlines. Useful when semantics depend192 /// on being on the same line, such as directives.193 void HandleWhitespaceBeforeTok(const Token &Tok, bool RequireSpace,194 bool RequireSameLine);195 196 /// Move to the line of the provided source location. This will197 /// return true if a newline was inserted or if198 /// the requested location is the first token on the first line.199 /// In these cases the next output will be the first column on the line and200 /// make it possible to insert indention. The newline was inserted201 /// implicitly when at the beginning of the file.202 ///203 /// @param Tok Token where to move to.204 /// @param RequireStartOfLine Whether the next line depends on being in the205 /// first column, such as a directive.206 ///207 /// @return Whether column adjustments are necessary.208 bool MoveToLine(const Token &Tok, bool RequireStartOfLine) {209 PresumedLoc PLoc = SM.getPresumedLoc(Tok.getLocation());210 unsigned TargetLine = PLoc.isValid() ? PLoc.getLine() : CurLine;211 bool IsFirstInFile =212 Tok.isAtStartOfLine() && PLoc.isValid() && PLoc.getLine() == 1;213 return MoveToLine(TargetLine, RequireStartOfLine) || IsFirstInFile;214 }215 216 /// Move to the line of the provided source location. Returns true if a new217 /// line was inserted.218 bool MoveToLine(SourceLocation Loc, bool RequireStartOfLine) {219 PresumedLoc PLoc = SM.getPresumedLoc(Loc);220 unsigned TargetLine = PLoc.isValid() ? PLoc.getLine() : CurLine;221 return MoveToLine(TargetLine, RequireStartOfLine);222 }223 bool MoveToLine(unsigned LineNo, bool RequireStartOfLine);224 225 bool AvoidConcat(const Token &PrevPrevTok, const Token &PrevTok,226 const Token &Tok) {227 return ConcatInfo.AvoidConcat(PrevPrevTok, PrevTok, Tok);228 }229 void WriteLineInfo(unsigned LineNo, const char *Extra=nullptr,230 unsigned ExtraLen=0);231 bool LineMarkersAreDisabled() const { return DisableLineMarkers; }232 void HandleNewlinesInToken(const char *TokStr, unsigned Len);233 234 /// MacroDefined - This hook is called whenever a macro definition is seen.235 void MacroDefined(const Token &MacroNameTok,236 const MacroDirective *MD) override;237 238 /// MacroUndefined - This hook is called whenever a macro #undef is seen.239 void MacroUndefined(const Token &MacroNameTok,240 const MacroDefinition &MD,241 const MacroDirective *Undef) override;242 243 void BeginModule(const Module *M);244 void EndModule(const Module *M);245 246 unsigned GetNumToksToSkip() const { return NumToksToSkip; }247 void ResetSkipToks() { NumToksToSkip = 0; }248};249} // end anonymous namespace250 251void PrintPPOutputPPCallbacks::WriteLineInfo(unsigned LineNo,252 const char *Extra,253 unsigned ExtraLen) {254 startNewLineIfNeeded();255 256 // Emit #line directives or GNU line markers depending on what mode we're in.257 if (UseLineDirectives) {258 *OS << "#line" << ' ' << LineNo << ' ' << '"';259 OS->write_escaped(CurFilename);260 *OS << '"';261 } else {262 *OS << '#' << ' ' << LineNo << ' ' << '"';263 OS->write_escaped(CurFilename);264 *OS << '"';265 266 if (ExtraLen)267 OS->write(Extra, ExtraLen);268 269 if (FileType == SrcMgr::C_System)270 OS->write(" 3", 2);271 else if (FileType == SrcMgr::C_ExternCSystem)272 OS->write(" 3 4", 4);273 }274 *OS << '\n';275}276 277/// MoveToLine - Move the output to the source line specified by the location278/// object. We can do this by emitting some number of \n's, or be emitting a279/// #line directive. This returns false if already at the specified line, true280/// if some newlines were emitted.281bool PrintPPOutputPPCallbacks::MoveToLine(unsigned LineNo,282 bool RequireStartOfLine) {283 // If it is required to start a new line or finish the current, insert284 // vertical whitespace now and take it into account when moving to the285 // expected line.286 bool StartedNewLine = false;287 if ((RequireStartOfLine && EmittedTokensOnThisLine) ||288 EmittedDirectiveOnThisLine) {289 *OS << '\n';290 StartedNewLine = true;291 CurLine += 1;292 EmittedTokensOnThisLine = false;293 EmittedDirectiveOnThisLine = false;294 }295 296 // If this line is "close enough" to the original line, just print newlines,297 // otherwise print a #line directive.298 if (CurLine == LineNo) {299 // Nothing to do if we are already on the correct line.300 } else if (MinimizeWhitespace && DisableLineMarkers) {301 // With -E -P -fminimize-whitespace, don't emit anything if not necessary.302 } else if (!StartedNewLine && LineNo - CurLine == 1) {303 // Printing a single line has priority over printing a #line directive, even304 // when minimizing whitespace which otherwise would print #line directives305 // for every single line.306 *OS << '\n';307 StartedNewLine = true;308 } else if (!DisableLineMarkers) {309 if (LineNo - CurLine <= 8) {310 const char *NewLines = "\n\n\n\n\n\n\n\n";311 OS->write(NewLines, LineNo - CurLine);312 } else {313 // Emit a #line or line marker.314 WriteLineInfo(LineNo, nullptr, 0);315 }316 StartedNewLine = true;317 } else if (EmittedTokensOnThisLine) {318 // If we are not on the correct line and don't need to be line-correct,319 // at least ensure we start on a new line.320 *OS << '\n';321 StartedNewLine = true;322 }323 324 if (StartedNewLine) {325 EmittedTokensOnThisLine = false;326 EmittedDirectiveOnThisLine = false;327 }328 329 CurLine = LineNo;330 return StartedNewLine;331}332 333void PrintPPOutputPPCallbacks::startNewLineIfNeeded() {334 if (EmittedTokensOnThisLine || EmittedDirectiveOnThisLine) {335 *OS << '\n';336 EmittedTokensOnThisLine = false;337 EmittedDirectiveOnThisLine = false;338 }339}340 341/// FileChanged - Whenever the preprocessor enters or exits a #include file342/// it invokes this handler. Update our conception of the current source343/// position.344void PrintPPOutputPPCallbacks::FileChanged(SourceLocation Loc,345 FileChangeReason Reason,346 SrcMgr::CharacteristicKind NewFileType,347 FileID PrevFID) {348 // Unless we are exiting a #include, make sure to skip ahead to the line the349 // #include directive was at.350 SourceManager &SourceMgr = SM;351 352 PresumedLoc UserLoc = SourceMgr.getPresumedLoc(Loc);353 if (UserLoc.isInvalid())354 return;355 356 unsigned NewLine = UserLoc.getLine();357 358 if (Reason == PPCallbacks::EnterFile) {359 SourceLocation IncludeLoc = UserLoc.getIncludeLoc();360 if (IncludeLoc.isValid())361 MoveToLine(IncludeLoc, /*RequireStartOfLine=*/false);362 } else if (Reason == PPCallbacks::SystemHeaderPragma) {363 // GCC emits the # directive for this directive on the line AFTER the364 // directive and emits a bunch of spaces that aren't needed. This is because365 // otherwise we will emit a line marker for THIS line, which requires an366 // extra blank line after the directive to avoid making all following lines367 // off by one. We can do better by simply incrementing NewLine here.368 NewLine += 1;369 }370 371 CurLine = NewLine;372 373 // In KeepSystemIncludes mode, redirect OS as needed.374 if (KeepSystemIncludes && (isSystem(FileType) != isSystem(NewFileType)))375 OS = isSystem(FileType) ? OrigOS : NullOS.get();376 377 CurFilename.clear();378 CurFilename += UserLoc.getFilename();379 FileType = NewFileType;380 381 if (DisableLineMarkers) {382 if (!MinimizeWhitespace)383 startNewLineIfNeeded();384 return;385 }386 387 if (!Initialized) {388 WriteLineInfo(CurLine);389 Initialized = true;390 }391 392 // Do not emit an enter marker for the main file (which we expect is the first393 // entered file). This matches gcc, and improves compatibility with some tools394 // which track the # line markers as a way to determine when the preprocessed395 // output is in the context of the main file.396 if (Reason == PPCallbacks::EnterFile && !IsFirstFileEntered) {397 IsFirstFileEntered = true;398 return;399 }400 401 switch (Reason) {402 case PPCallbacks::EnterFile:403 WriteLineInfo(CurLine, " 1", 2);404 break;405 case PPCallbacks::ExitFile:406 WriteLineInfo(CurLine, " 2", 2);407 break;408 case PPCallbacks::SystemHeaderPragma:409 case PPCallbacks::RenameFile:410 WriteLineInfo(CurLine);411 break;412 }413}414 415void PrintPPOutputPPCallbacks::EmbedDirective(416 SourceLocation HashLoc, StringRef FileName, bool IsAngled,417 OptionalFileEntryRef File, const LexEmbedParametersResult &Params) {418 if (!DumpEmbedDirectives)419 return;420 421 // The EmbedDirective() callback is called before we produce the annotation422 // token stream for the directive. We skip printing the annotation tokens423 // within PrintPreprocessedTokens(), but we also need to skip the prefix,424 // suffix, and if_empty tokens as those are inserted directly into the token425 // stream and would otherwise be printed immediately after printing the426 // #embed directive.427 //428 // FIXME: counting tokens to skip is a kludge but we have no way to know429 // which tokens were inserted as part of the embed and which ones were430 // explicitly written by the user.431 MoveToLine(HashLoc, /*RequireStartOfLine=*/true);432 *OS << "#embed " << (IsAngled ? '<' : '"') << FileName433 << (IsAngled ? '>' : '"');434 435 auto PrintToks = [&](llvm::ArrayRef<Token> Toks) {436 SmallString<128> SpellingBuffer;437 for (const Token &T : Toks) {438 if (T.hasLeadingSpace())439 *OS << " ";440 *OS << PP.getSpelling(T, SpellingBuffer);441 }442 };443 bool SkipAnnotToks = true;444 if (Params.MaybeIfEmptyParam) {445 *OS << " if_empty(";446 PrintToks(Params.MaybeIfEmptyParam->Tokens);447 *OS << ")";448 // If the file is empty, we can skip those tokens. If the file is not449 // empty, we skip the annotation tokens.450 if (File && !File->getSize()) {451 NumToksToSkip += Params.MaybeIfEmptyParam->Tokens.size();452 SkipAnnotToks = false;453 }454 }455 456 if (Params.MaybeLimitParam) {457 *OS << " limit(" << Params.MaybeLimitParam->Limit << ")";458 }459 if (Params.MaybeOffsetParam) {460 *OS << " clang::offset(" << Params.MaybeOffsetParam->Offset << ")";461 }462 if (Params.MaybePrefixParam) {463 *OS << " prefix(";464 PrintToks(Params.MaybePrefixParam->Tokens);465 *OS << ")";466 NumToksToSkip += Params.MaybePrefixParam->Tokens.size();467 }468 if (Params.MaybeSuffixParam) {469 *OS << " suffix(";470 PrintToks(Params.MaybeSuffixParam->Tokens);471 *OS << ")";472 NumToksToSkip += Params.MaybeSuffixParam->Tokens.size();473 }474 475 // We may need to skip the annotation token.476 if (SkipAnnotToks)477 NumToksToSkip++;478 479 *OS << " /* clang -E -dE */";480 setEmittedDirectiveOnThisLine();481}482 483void PrintPPOutputPPCallbacks::InclusionDirective(484 SourceLocation HashLoc, const Token &IncludeTok, StringRef FileName,485 bool IsAngled, CharSourceRange FilenameRange, OptionalFileEntryRef File,486 StringRef SearchPath, StringRef RelativePath, const Module *SuggestedModule,487 bool ModuleImported, SrcMgr::CharacteristicKind FileType) {488 // In -dI mode, dump #include directives prior to dumping their content or489 // interpretation. Similar for -fkeep-system-includes.490 if (DumpIncludeDirectives || (KeepSystemIncludes && isSystem(FileType))) {491 MoveToLine(HashLoc, /*RequireStartOfLine=*/true);492 const std::string TokenText = PP.getSpelling(IncludeTok);493 assert(!TokenText.empty());494 *OS << "#" << TokenText << " "495 << (IsAngled ? '<' : '"') << FileName << (IsAngled ? '>' : '"')496 << " /* clang -E "497 << (DumpIncludeDirectives ? "-dI" : "-fkeep-system-includes")498 << " */";499 setEmittedDirectiveOnThisLine();500 }501 502 // When preprocessing, turn implicit imports into module import pragmas.503 if (ModuleImported) {504 switch (IncludeTok.getIdentifierInfo()->getPPKeywordID()) {505 case tok::pp_include:506 case tok::pp_import:507 case tok::pp_include_next:508 MoveToLine(HashLoc, /*RequireStartOfLine=*/true);509 *OS << "#pragma clang module import "510 << SuggestedModule->getFullModuleName(true)511 << " /* clang -E: implicit import for "512 << "#" << PP.getSpelling(IncludeTok) << " "513 << (IsAngled ? '<' : '"') << FileName << (IsAngled ? '>' : '"')514 << " */";515 setEmittedDirectiveOnThisLine();516 break;517 518 case tok::pp___include_macros:519 // #__include_macros has no effect on a user of a preprocessed source520 // file; the only effect is on preprocessing.521 //522 // FIXME: That's not *quite* true: it causes the module in question to523 // be loaded, which can affect downstream diagnostics.524 break;525 526 default:527 llvm_unreachable("unknown include directive kind");528 break;529 }530 }531}532 533/// Handle entering the scope of a module during a module compilation.534void PrintPPOutputPPCallbacks::BeginModule(const Module *M) {535 startNewLineIfNeeded();536 *OS << "#pragma clang module begin " << M->getFullModuleName(true);537 setEmittedDirectiveOnThisLine();538}539 540/// Handle leaving the scope of a module during a module compilation.541void PrintPPOutputPPCallbacks::EndModule(const Module *M) {542 startNewLineIfNeeded();543 *OS << "#pragma clang module end /*" << M->getFullModuleName(true) << "*/";544 setEmittedDirectiveOnThisLine();545}546 547/// Ident - Handle #ident directives when read by the preprocessor.548///549void PrintPPOutputPPCallbacks::Ident(SourceLocation Loc, StringRef S) {550 MoveToLine(Loc, /*RequireStartOfLine=*/true);551 552 OS->write("#ident ", strlen("#ident "));553 OS->write(S.begin(), S.size());554 setEmittedTokensOnThisLine();555}556 557/// MacroDefined - This hook is called whenever a macro definition is seen.558void PrintPPOutputPPCallbacks::MacroDefined(const Token &MacroNameTok,559 const MacroDirective *MD) {560 const MacroInfo *MI = MD->getMacroInfo();561 // Print out macro definitions in -dD mode and when we have -fdirectives-only562 // for C++20 header units.563 if ((!DumpDefines && !DirectivesOnly) ||564 // Ignore __FILE__ etc.565 MI->isBuiltinMacro())566 return;567 568 SourceLocation DefLoc = MI->getDefinitionLoc();569 if (DirectivesOnly && !MI->isUsed()) {570 SourceManager &SM = PP.getSourceManager();571 if (SM.isInPredefinedFile(DefLoc))572 return;573 }574 MoveToLine(DefLoc, /*RequireStartOfLine=*/true);575 PrintMacroDefinition(*MacroNameTok.getIdentifierInfo(), *MI, PP, OS);576 setEmittedDirectiveOnThisLine();577}578 579void PrintPPOutputPPCallbacks::MacroUndefined(const Token &MacroNameTok,580 const MacroDefinition &MD,581 const MacroDirective *Undef) {582 // Print out macro definitions in -dD mode and when we have -fdirectives-only583 // for C++20 header units.584 if (!DumpDefines && !DirectivesOnly)585 return;586 587 MoveToLine(MacroNameTok.getLocation(), /*RequireStartOfLine=*/true);588 *OS << "#undef " << MacroNameTok.getIdentifierInfo()->getName();589 setEmittedDirectiveOnThisLine();590}591 592static void outputPrintable(raw_ostream *OS, StringRef Str) {593 for (unsigned char Char : Str) {594 if (isPrintable(Char) && Char != '\\' && Char != '"')595 *OS << (char)Char;596 else // Output anything hard as an octal escape.597 *OS << '\\'598 << (char)('0' + ((Char >> 6) & 7))599 << (char)('0' + ((Char >> 3) & 7))600 << (char)('0' + ((Char >> 0) & 7));601 }602}603 604void PrintPPOutputPPCallbacks::PragmaMessage(SourceLocation Loc,605 StringRef Namespace,606 PragmaMessageKind Kind,607 StringRef Str) {608 MoveToLine(Loc, /*RequireStartOfLine=*/true);609 *OS << "#pragma ";610 if (!Namespace.empty())611 *OS << Namespace << ' ';612 switch (Kind) {613 case PMK_Message:614 *OS << "message(\"";615 break;616 case PMK_Warning:617 *OS << "warning \"";618 break;619 case PMK_Error:620 *OS << "error \"";621 break;622 }623 624 outputPrintable(OS, Str);625 *OS << '"';626 if (Kind == PMK_Message)627 *OS << ')';628 setEmittedDirectiveOnThisLine();629}630 631void PrintPPOutputPPCallbacks::PragmaDebug(SourceLocation Loc,632 StringRef DebugType) {633 MoveToLine(Loc, /*RequireStartOfLine=*/true);634 635 *OS << "#pragma clang __debug ";636 *OS << DebugType;637 638 setEmittedDirectiveOnThisLine();639}640 641void PrintPPOutputPPCallbacks::642PragmaDiagnosticPush(SourceLocation Loc, StringRef Namespace) {643 MoveToLine(Loc, /*RequireStartOfLine=*/true);644 *OS << "#pragma " << Namespace << " diagnostic push";645 setEmittedDirectiveOnThisLine();646}647 648void PrintPPOutputPPCallbacks::649PragmaDiagnosticPop(SourceLocation Loc, StringRef Namespace) {650 MoveToLine(Loc, /*RequireStartOfLine=*/true);651 *OS << "#pragma " << Namespace << " diagnostic pop";652 setEmittedDirectiveOnThisLine();653}654 655void PrintPPOutputPPCallbacks::PragmaDiagnostic(SourceLocation Loc,656 StringRef Namespace,657 diag::Severity Map,658 StringRef Str) {659 MoveToLine(Loc, /*RequireStartOfLine=*/true);660 *OS << "#pragma " << Namespace << " diagnostic ";661 switch (Map) {662 case diag::Severity::Remark:663 *OS << "remark";664 break;665 case diag::Severity::Warning:666 *OS << "warning";667 break;668 case diag::Severity::Error:669 *OS << "error";670 break;671 case diag::Severity::Ignored:672 *OS << "ignored";673 break;674 case diag::Severity::Fatal:675 *OS << "fatal";676 break;677 }678 *OS << " \"" << Str << '"';679 setEmittedDirectiveOnThisLine();680}681 682void PrintPPOutputPPCallbacks::PragmaWarning(SourceLocation Loc,683 PragmaWarningSpecifier WarningSpec,684 ArrayRef<int> Ids) {685 MoveToLine(Loc, /*RequireStartOfLine=*/true);686 687 *OS << "#pragma warning(";688 switch(WarningSpec) {689 case PWS_Default: *OS << "default"; break;690 case PWS_Disable: *OS << "disable"; break;691 case PWS_Error: *OS << "error"; break;692 case PWS_Once: *OS << "once"; break;693 case PWS_Suppress: *OS << "suppress"; break;694 case PWS_Level1: *OS << '1'; break;695 case PWS_Level2: *OS << '2'; break;696 case PWS_Level3: *OS << '3'; break;697 case PWS_Level4: *OS << '4'; break;698 }699 *OS << ':';700 701 for (ArrayRef<int>::iterator I = Ids.begin(), E = Ids.end(); I != E; ++I)702 *OS << ' ' << *I;703 *OS << ')';704 setEmittedDirectiveOnThisLine();705}706 707void PrintPPOutputPPCallbacks::PragmaWarningPush(SourceLocation Loc,708 int Level) {709 MoveToLine(Loc, /*RequireStartOfLine=*/true);710 *OS << "#pragma warning(push";711 if (Level >= 0)712 *OS << ", " << Level;713 *OS << ')';714 setEmittedDirectiveOnThisLine();715}716 717void PrintPPOutputPPCallbacks::PragmaWarningPop(SourceLocation Loc) {718 MoveToLine(Loc, /*RequireStartOfLine=*/true);719 *OS << "#pragma warning(pop)";720 setEmittedDirectiveOnThisLine();721}722 723void PrintPPOutputPPCallbacks::PragmaExecCharsetPush(SourceLocation Loc,724 StringRef Str) {725 MoveToLine(Loc, /*RequireStartOfLine=*/true);726 *OS << "#pragma character_execution_set(push";727 if (!Str.empty())728 *OS << ", " << Str;729 *OS << ')';730 setEmittedDirectiveOnThisLine();731}732 733void PrintPPOutputPPCallbacks::PragmaExecCharsetPop(SourceLocation Loc) {734 MoveToLine(Loc, /*RequireStartOfLine=*/true);735 *OS << "#pragma character_execution_set(pop)";736 setEmittedDirectiveOnThisLine();737}738 739void PrintPPOutputPPCallbacks::740PragmaAssumeNonNullBegin(SourceLocation Loc) {741 MoveToLine(Loc, /*RequireStartOfLine=*/true);742 *OS << "#pragma clang assume_nonnull begin";743 setEmittedDirectiveOnThisLine();744}745 746void PrintPPOutputPPCallbacks::747PragmaAssumeNonNullEnd(SourceLocation Loc) {748 MoveToLine(Loc, /*RequireStartOfLine=*/true);749 *OS << "#pragma clang assume_nonnull end";750 setEmittedDirectiveOnThisLine();751}752 753void PrintPPOutputPPCallbacks::HandleWhitespaceBeforeTok(const Token &Tok,754 bool RequireSpace,755 bool RequireSameLine) {756 // These tokens are not expanded to anything and don't need whitespace before757 // them.758 if (Tok.is(tok::eof) ||759 (Tok.isAnnotation() && !Tok.is(tok::annot_header_unit) &&760 !Tok.is(tok::annot_module_begin) && !Tok.is(tok::annot_module_end) &&761 !Tok.is(tok::annot_repl_input_end) && !Tok.is(tok::annot_embed)))762 return;763 764 // EmittedDirectiveOnThisLine takes priority over RequireSameLine.765 if ((!RequireSameLine || EmittedDirectiveOnThisLine) &&766 MoveToLine(Tok, /*RequireStartOfLine=*/EmittedDirectiveOnThisLine)) {767 if (MinimizeWhitespace) {768 // Avoid interpreting hash as a directive under -fpreprocessed.769 if (Tok.is(tok::hash))770 *OS << ' ';771 } else {772 // Print out space characters so that the first token on a line is773 // indented for easy reading.774 unsigned ColNo = SM.getExpansionColumnNumber(Tok.getLocation());775 776 // The first token on a line can have a column number of 1, yet still777 // expect leading white space, if a macro expansion in column 1 starts778 // with an empty macro argument, or an empty nested macro expansion. In779 // this case, move the token to column 2.780 if (ColNo == 1 && Tok.hasLeadingSpace())781 ColNo = 2;782 783 // This hack prevents stuff like:784 // #define HASH #785 // HASH define foo bar786 // From having the # character end up at column 1, which makes it so it787 // is not handled as a #define next time through the preprocessor if in788 // -fpreprocessed mode.789 if (ColNo <= 1 && Tok.is(tok::hash))790 *OS << ' ';791 792 // Otherwise, indent the appropriate number of spaces.793 for (; ColNo > 1; --ColNo)794 *OS << ' ';795 }796 } else {797 // Insert whitespace between the previous and next token if either798 // - The caller requires it799 // - The input had whitespace between them and we are not in800 // whitespace-minimization mode801 // - The whitespace is necessary to keep the tokens apart and there is not802 // already a newline between them803 if (RequireSpace || (!MinimizeWhitespace && Tok.hasLeadingSpace()) ||804 ((EmittedTokensOnThisLine || EmittedDirectiveOnThisLine) &&805 AvoidConcat(PrevPrevTok, PrevTok, Tok)))806 *OS << ' ';807 }808 809 PrevPrevTok = PrevTok;810 PrevTok = Tok;811}812 813void PrintPPOutputPPCallbacks::HandleNewlinesInToken(const char *TokStr,814 unsigned Len) {815 unsigned NumNewlines = 0;816 for (; Len; --Len, ++TokStr) {817 if (*TokStr != '\n' &&818 *TokStr != '\r')819 continue;820 821 ++NumNewlines;822 823 // If we have \n\r or \r\n, skip both and count as one line.824 if (Len != 1 &&825 (TokStr[1] == '\n' || TokStr[1] == '\r') &&826 TokStr[0] != TokStr[1]) {827 ++TokStr;828 --Len;829 }830 }831 832 if (NumNewlines == 0) return;833 834 CurLine += NumNewlines;835}836 837 838namespace {839struct UnknownPragmaHandler : public PragmaHandler {840 const char *Prefix;841 PrintPPOutputPPCallbacks *Callbacks;842 843 // Set to true if tokens should be expanded844 bool ShouldExpandTokens;845 846 UnknownPragmaHandler(const char *prefix, PrintPPOutputPPCallbacks *callbacks,847 bool RequireTokenExpansion)848 : Prefix(prefix), Callbacks(callbacks),849 ShouldExpandTokens(RequireTokenExpansion) {}850 void HandlePragma(Preprocessor &PP, PragmaIntroducer Introducer,851 Token &PragmaTok) override {852 // Figure out what line we went to and insert the appropriate number of853 // newline characters.854 Callbacks->MoveToLine(PragmaTok.getLocation(), /*RequireStartOfLine=*/true);855 Callbacks->OS->write(Prefix, strlen(Prefix));856 Callbacks->setEmittedTokensOnThisLine();857 858 if (ShouldExpandTokens) {859 // The first token does not have expanded macros. Expand them, if860 // required.861 auto Toks = std::make_unique<Token[]>(1);862 Toks[0] = PragmaTok;863 PP.EnterTokenStream(std::move(Toks), /*NumToks=*/1,864 /*DisableMacroExpansion=*/false,865 /*IsReinject=*/false);866 PP.Lex(PragmaTok);867 }868 869 // Read and print all of the pragma tokens.870 bool IsFirst = true;871 while (PragmaTok.isNot(tok::eod)) {872 Callbacks->HandleWhitespaceBeforeTok(PragmaTok, /*RequireSpace=*/IsFirst,873 /*RequireSameLine=*/true);874 IsFirst = false;875 std::string TokSpell = PP.getSpelling(PragmaTok);876 Callbacks->OS->write(&TokSpell[0], TokSpell.size());877 Callbacks->setEmittedTokensOnThisLine();878 879 if (ShouldExpandTokens)880 PP.Lex(PragmaTok);881 else882 PP.LexUnexpandedToken(PragmaTok);883 }884 Callbacks->setEmittedDirectiveOnThisLine();885 }886};887} // end anonymous namespace888 889 890static void PrintPreprocessedTokens(Preprocessor &PP, Token &Tok,891 PrintPPOutputPPCallbacks *Callbacks) {892 bool DropComments = PP.getLangOpts().TraditionalCPP &&893 !PP.getCommentRetentionState();894 895 bool IsStartOfLine = false;896 char Buffer[256];897 while (true) {898 // Two lines joined with line continuation ('\' as last character on the899 // line) must be emitted as one line even though Tok.getLine() returns two900 // different values. In this situation Tok.isAtStartOfLine() is false even901 // though it may be the first token on the lexical line. When902 // dropping/skipping a token that is at the start of a line, propagate the903 // start-of-line-ness to the next token to not append it to the previous904 // line.905 IsStartOfLine = IsStartOfLine || Tok.isAtStartOfLine();906 907 Callbacks->HandleWhitespaceBeforeTok(Tok, /*RequireSpace=*/false,908 /*RequireSameLine=*/!IsStartOfLine);909 910 if (DropComments && Tok.is(tok::comment)) {911 // Skip comments. Normally the preprocessor does not generate912 // tok::comment nodes at all when not keeping comments, but under913 // -traditional-cpp the lexer keeps /all/ whitespace, including comments.914 PP.Lex(Tok);915 continue;916 } else if (Tok.is(tok::annot_repl_input_end)) {917 // Fall through to exit the loop.918 } else if (Tok.is(tok::eod)) {919 // Don't print end of directive tokens, since they are typically newlines920 // that mess up our line tracking. These come from unknown pre-processor921 // directives or hash-prefixed comments in standalone assembly files.922 PP.Lex(Tok);923 // FIXME: The token on the next line after #include should have924 // Tok.isAtStartOfLine() set.925 IsStartOfLine = true;926 continue;927 } else if (Tok.is(tok::annot_module_include)) {928 // PrintPPOutputPPCallbacks::InclusionDirective handles producing929 // appropriate output here. Ignore this token entirely.930 PP.Lex(Tok);931 IsStartOfLine = true;932 continue;933 } else if (Tok.is(tok::annot_module_begin)) {934 // FIXME: We retrieve this token after the FileChanged callback, and935 // retrieve the module_end token before the FileChanged callback, so936 // we render this within the file and render the module end outside the937 // file, but this is backwards from the token locations: the module_begin938 // token is at the include location (outside the file) and the module_end939 // token is at the EOF location (within the file).940 Callbacks->BeginModule(941 reinterpret_cast<Module *>(Tok.getAnnotationValue()));942 PP.Lex(Tok);943 IsStartOfLine = true;944 continue;945 } else if (Tok.is(tok::annot_module_end)) {946 Callbacks->EndModule(947 reinterpret_cast<Module *>(Tok.getAnnotationValue()));948 PP.Lex(Tok);949 IsStartOfLine = true;950 continue;951 } else if (Tok.is(tok::annot_header_unit)) {952 // This is a header-name that has been (effectively) converted into a953 // module-name, print them inside quote.954 // FIXME: The module name could contain non-identifier module name955 // components and OS specific file paths components. We don't have a good956 // way to round-trip those.957 Module *M = reinterpret_cast<Module *>(Tok.getAnnotationValue());958 std::string Name = M->getFullModuleName();959 *Callbacks->OS << '"';960 Callbacks->OS->write_escaped(Name);961 *Callbacks->OS << '"';962 } else if (Tok.is(tok::annot_embed)) {963 // Manually explode the binary data out to a stream of comma-delimited964 // integer values. If the user passed -dE, that is handled by the965 // EmbedDirective() callback. We should only get here if the user did not966 // pass -dE.967 assert(Callbacks->expandEmbedContents() &&968 "did not expect an embed annotation");969 auto *Data =970 reinterpret_cast<EmbedAnnotationData *>(Tok.getAnnotationValue());971 972 // Loop over the contents and print them as a comma-delimited list of973 // values.974 bool PrintComma = false;975 for (unsigned char Byte : Data->BinaryData.bytes()) {976 if (PrintComma)977 *Callbacks->OS << ", ";978 *Callbacks->OS << static_cast<int>(Byte);979 PrintComma = true;980 }981 } else if (Tok.isAnnotation()) {982 // Ignore annotation tokens created by pragmas - the pragmas themselves983 // will be reproduced in the preprocessed output.984 PP.Lex(Tok);985 continue;986 } else if (IdentifierInfo *II = Tok.getIdentifierInfo()) {987 *Callbacks->OS << II->getName();988 } else if (Tok.isLiteral() && !Tok.needsCleaning() &&989 Tok.getLiteralData()) {990 Callbacks->OS->write(Tok.getLiteralData(), Tok.getLength());991 } else if (Tok.getLength() < std::size(Buffer)) {992 const char *TokPtr = Buffer;993 unsigned Len = PP.getSpelling(Tok, TokPtr);994 Callbacks->OS->write(TokPtr, Len);995 996 // Tokens that can contain embedded newlines need to adjust our current997 // line number.998 // FIXME: The token may end with a newline in which case999 // setEmittedDirectiveOnThisLine/setEmittedTokensOnThisLine afterwards is1000 // wrong.1001 if (Tok.getKind() == tok::comment || Tok.getKind() == tok::unknown)1002 Callbacks->HandleNewlinesInToken(TokPtr, Len);1003 if (Tok.is(tok::comment) && Len >= 2 && TokPtr[0] == '/' &&1004 TokPtr[1] == '/') {1005 // It's a line comment;1006 // Ensure that we don't concatenate anything behind it.1007 Callbacks->setEmittedDirectiveOnThisLine();1008 }1009 } else {1010 std::string S = PP.getSpelling(Tok);1011 Callbacks->OS->write(S.data(), S.size());1012 1013 // Tokens that can contain embedded newlines need to adjust our current1014 // line number.1015 if (Tok.getKind() == tok::comment || Tok.getKind() == tok::unknown)1016 Callbacks->HandleNewlinesInToken(S.data(), S.size());1017 if (Tok.is(tok::comment) && S.size() >= 2 && S[0] == '/' && S[1] == '/') {1018 // It's a line comment;1019 // Ensure that we don't concatenate anything behind it.1020 Callbacks->setEmittedDirectiveOnThisLine();1021 }1022 }1023 Callbacks->setEmittedTokensOnThisLine();1024 IsStartOfLine = false;1025 1026 if (Tok.is(tok::eof) || Tok.is(tok::annot_repl_input_end))1027 break;1028 1029 PP.Lex(Tok);1030 // If lexing that token causes us to need to skip future tokens, do so now.1031 for (unsigned I = 0, Skip = Callbacks->GetNumToksToSkip(); I < Skip; ++I)1032 PP.Lex(Tok);1033 Callbacks->ResetSkipToks();1034 }1035}1036 1037typedef std::pair<const IdentifierInfo *, MacroInfo *> id_macro_pair;1038static int MacroIDCompare(const id_macro_pair *LHS, const id_macro_pair *RHS) {1039 return LHS->first->getName().compare(RHS->first->getName());1040}1041 1042static void DoPrintMacros(Preprocessor &PP, raw_ostream *OS) {1043 // Ignore unknown pragmas.1044 PP.IgnorePragmas();1045 1046 // -dM mode just scans and ignores all tokens in the files, then dumps out1047 // the macro table at the end.1048 PP.EnterMainSourceFile();1049 1050 PP.LexTokensUntilEOF();1051 1052 SmallVector<id_macro_pair, 128> MacrosByID;1053 for (Preprocessor::macro_iterator I = PP.macro_begin(), E = PP.macro_end();1054 I != E; ++I) {1055 auto *MD = I->second.getLatest();1056 if (MD && MD->isDefined())1057 MacrosByID.push_back(id_macro_pair(I->first, MD->getMacroInfo()));1058 }1059 llvm::array_pod_sort(MacrosByID.begin(), MacrosByID.end(), MacroIDCompare);1060 1061 for (unsigned i = 0, e = MacrosByID.size(); i != e; ++i) {1062 MacroInfo &MI = *MacrosByID[i].second;1063 // Ignore computed macros like __LINE__ and friends.1064 if (MI.isBuiltinMacro()) continue;1065 1066 PrintMacroDefinition(*MacrosByID[i].first, MI, PP, OS);1067 *OS << '\n';1068 }1069}1070 1071/// DoPrintPreprocessedInput - This implements -E mode.1072///1073void clang::DoPrintPreprocessedInput(Preprocessor &PP, raw_ostream *OS,1074 const PreprocessorOutputOptions &Opts) {1075 // Show macros with no output is handled specially.1076 if (!Opts.ShowCPP) {1077 assert(Opts.ShowMacros && "Not yet implemented!");1078 DoPrintMacros(PP, OS);1079 return;1080 }1081 1082 // Inform the preprocessor whether we want it to retain comments or not, due1083 // to -C or -CC.1084 PP.SetCommentRetentionState(Opts.ShowComments, Opts.ShowMacroComments);1085 1086 PrintPPOutputPPCallbacks *Callbacks = new PrintPPOutputPPCallbacks(1087 PP, OS, !Opts.ShowLineMarkers, Opts.ShowMacros,1088 Opts.ShowIncludeDirectives, Opts.ShowEmbedDirectives,1089 Opts.UseLineDirectives, Opts.MinimizeWhitespace, Opts.DirectivesOnly,1090 Opts.KeepSystemIncludes);1091 1092 // Expand macros in pragmas with -fms-extensions. The assumption is that1093 // the majority of pragmas in such a file will be Microsoft pragmas.1094 // Remember the handlers we will add so that we can remove them later.1095 std::unique_ptr<UnknownPragmaHandler> MicrosoftExtHandler(1096 new UnknownPragmaHandler(1097 "#pragma", Callbacks,1098 /*RequireTokenExpansion=*/PP.getLangOpts().MicrosoftExt));1099 1100 std::unique_ptr<UnknownPragmaHandler> GCCHandler(new UnknownPragmaHandler(1101 "#pragma GCC", Callbacks,1102 /*RequireTokenExpansion=*/PP.getLangOpts().MicrosoftExt));1103 1104 std::unique_ptr<UnknownPragmaHandler> ClangHandler(new UnknownPragmaHandler(1105 "#pragma clang", Callbacks,1106 /*RequireTokenExpansion=*/PP.getLangOpts().MicrosoftExt));1107 1108 PP.AddPragmaHandler(MicrosoftExtHandler.get());1109 PP.AddPragmaHandler("GCC", GCCHandler.get());1110 PP.AddPragmaHandler("clang", ClangHandler.get());1111 1112 // The tokens after pragma omp need to be expanded.1113 //1114 // OpenMP [2.1, Directive format]1115 // Preprocessing tokens following the #pragma omp are subject to macro1116 // replacement.1117 std::unique_ptr<UnknownPragmaHandler> OpenMPHandler(1118 new UnknownPragmaHandler("#pragma omp", Callbacks,1119 /*RequireTokenExpansion=*/true));1120 PP.AddPragmaHandler("omp", OpenMPHandler.get());1121 1122 PP.addPPCallbacks(std::unique_ptr<PPCallbacks>(Callbacks));1123 1124 // After we have configured the preprocessor, enter the main file.1125 PP.EnterMainSourceFile();1126 if (Opts.DirectivesOnly)1127 PP.SetMacroExpansionOnlyInDirectives();1128 1129 // Consume all of the tokens that come from the predefines buffer. Those1130 // should not be emitted into the output and are guaranteed to be at the1131 // start.1132 const SourceManager &SourceMgr = PP.getSourceManager();1133 Token Tok;1134 do {1135 PP.Lex(Tok);1136 if (Tok.is(tok::eof) || !Tok.getLocation().isFileID())1137 break;1138 1139 PresumedLoc PLoc = SourceMgr.getPresumedLoc(Tok.getLocation());1140 if (PLoc.isInvalid())1141 break;1142 1143 if (strcmp(PLoc.getFilename(), "<built-in>"))1144 break;1145 } while (true);1146 1147 // Read all the preprocessed tokens, printing them out to the stream.1148 PrintPreprocessedTokens(PP, Tok, Callbacks);1149 *OS << '\n';1150 1151 // Remove the handlers we just added to leave the preprocessor in a sane state1152 // so that it can be reused (for example by a clang::Parser instance).1153 PP.RemovePragmaHandler(MicrosoftExtHandler.get());1154 PP.RemovePragmaHandler("GCC", GCCHandler.get());1155 PP.RemovePragmaHandler("clang", ClangHandler.get());1156 PP.RemovePragmaHandler("omp", OpenMPHandler.get());1157}1158