557 lines · cpp
1//===- ExtractAPI/ExtractAPIConsumer.cpp ------------------------*- C++ -*-===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8///9/// \file10/// This file implements the ExtractAPIAction, and ASTConsumer to collect API11/// information.12///13//===----------------------------------------------------------------------===//14 15#include "clang/AST/ASTConcept.h"16#include "clang/AST/ASTConsumer.h"17#include "clang/AST/ASTContext.h"18#include "clang/AST/DeclObjC.h"19#include "clang/Basic/DiagnosticFrontend.h"20#include "clang/Basic/FileEntry.h"21#include "clang/Basic/SourceLocation.h"22#include "clang/Basic/SourceManager.h"23#include "clang/Basic/TargetInfo.h"24#include "clang/ExtractAPI/API.h"25#include "clang/ExtractAPI/APIIgnoresList.h"26#include "clang/ExtractAPI/ExtractAPIVisitor.h"27#include "clang/ExtractAPI/FrontendActions.h"28#include "clang/ExtractAPI/Serialization/SymbolGraphSerializer.h"29#include "clang/Frontend/ASTConsumers.h"30#include "clang/Frontend/CompilerInstance.h"31#include "clang/Frontend/FrontendOptions.h"32#include "clang/Frontend/MultiplexConsumer.h"33#include "clang/Index/USRGeneration.h"34#include "clang/InstallAPI/HeaderFile.h"35#include "clang/Lex/MacroInfo.h"36#include "clang/Lex/PPCallbacks.h"37#include "clang/Lex/Preprocessor.h"38#include "clang/Lex/PreprocessorOptions.h"39#include "llvm/ADT/DenseSet.h"40#include "llvm/ADT/STLExtras.h"41#include "llvm/ADT/SmallString.h"42#include "llvm/ADT/SmallVector.h"43#include "llvm/ADT/StringRef.h"44#include "llvm/Support/Casting.h"45#include "llvm/Support/Error.h"46#include "llvm/Support/MemoryBuffer.h"47#include "llvm/Support/Path.h"48#include "llvm/Support/Regex.h"49#include "llvm/Support/raw_ostream.h"50#include <memory>51#include <optional>52#include <utility>53 54using namespace clang;55using namespace extractapi;56 57namespace {58 59std::optional<std::string> getRelativeIncludeName(const CompilerInstance &CI,60 StringRef File,61 bool *IsQuoted = nullptr) {62 assert(CI.hasFileManager() &&63 "CompilerInstance does not have a FileNamager!");64 65 using namespace llvm::sys;66 const auto &FS = CI.getVirtualFileSystem();67 68 SmallString<128> FilePath(File.begin(), File.end());69 FS.makeAbsolute(FilePath);70 path::remove_dots(FilePath, true);71 FilePath = path::convert_to_slash(FilePath);72 File = FilePath;73 74 // Checks whether `Dir` is a strict path prefix of `File`. If so returns75 // the prefix length. Otherwise return 0.76 auto CheckDir = [&](llvm::StringRef Dir) -> unsigned {77 llvm::SmallString<32> DirPath(Dir.begin(), Dir.end());78 FS.makeAbsolute(DirPath);79 path::remove_dots(DirPath, true);80 Dir = DirPath;81 for (auto NI = path::begin(File), NE = path::end(File),82 DI = path::begin(Dir), DE = path::end(Dir);83 /*termination condition in loop*/; ++NI, ++DI) {84 // '.' components in File are ignored.85 while (NI != NE && *NI == ".")86 ++NI;87 if (NI == NE)88 break;89 90 // '.' components in Dir are ignored.91 while (DI != DE && *DI == ".")92 ++DI;93 94 // Dir is a prefix of File, up to '.' components and choice of path95 // separators.96 if (DI == DE)97 return NI - path::begin(File);98 99 // Consider all path separators equal.100 if (NI->size() == 1 && DI->size() == 1 &&101 path::is_separator(NI->front()) && path::is_separator(DI->front()))102 continue;103 104 // Special case Apple .sdk folders since the search path is typically a105 // symlink like `iPhoneSimulator14.5.sdk` while the file is instead106 // located in `iPhoneSimulator.sdk` (the real folder).107 if (NI->ends_with(".sdk") && DI->ends_with(".sdk")) {108 StringRef NBasename = path::stem(*NI);109 StringRef DBasename = path::stem(*DI);110 if (DBasename.starts_with(NBasename))111 continue;112 }113 114 if (*NI != *DI)115 break;116 }117 return 0;118 };119 120 unsigned PrefixLength = 0;121 122 // Go through the search paths and find the first one that is a prefix of123 // the header.124 for (const auto &Entry : CI.getHeaderSearchOpts().UserEntries) {125 // Note whether the match is found in a quoted entry.126 if (IsQuoted)127 *IsQuoted = Entry.Group == frontend::Quoted;128 129 if (auto EntryFile = CI.getFileManager().getOptionalFileRef(Entry.Path)) {130 if (auto HMap = HeaderMap::Create(*EntryFile, CI.getFileManager())) {131 // If this is a headermap entry, try to reverse lookup the full path132 // for a spelled name before mapping.133 StringRef SpelledFilename = HMap->reverseLookupFilename(File);134 if (!SpelledFilename.empty())135 return SpelledFilename.str();136 137 // No matching mapping in this headermap, try next search entry.138 continue;139 }140 }141 142 // Entry is a directory search entry, try to check if it's a prefix of File.143 PrefixLength = CheckDir(Entry.Path);144 if (PrefixLength > 0) {145 // The header is found in a framework path, construct the framework-style146 // include name `<Framework/Header.h>`147 if (Entry.IsFramework) {148 SmallVector<StringRef, 4> Matches;149 clang::installapi::HeaderFile::getFrameworkIncludeRule().match(150 File, &Matches);151 // Returned matches are always in stable order.152 if (Matches.size() != 4)153 return std::nullopt;154 155 return path::convert_to_slash(156 (Matches[1].drop_front(Matches[1].rfind('/') + 1) + "/" +157 Matches[3])158 .str());159 }160 161 // The header is found in a normal search path, strip the search path162 // prefix to get an include name.163 return path::convert_to_slash(File.drop_front(PrefixLength));164 }165 }166 167 // Couldn't determine a include name, use full path instead.168 return std::nullopt;169}170 171std::optional<std::string> getRelativeIncludeName(const CompilerInstance &CI,172 FileEntryRef FE,173 bool *IsQuoted = nullptr) {174 return getRelativeIncludeName(CI, FE.getNameAsRequested(), IsQuoted);175}176 177struct LocationFileChecker {178 bool operator()(SourceLocation Loc) {179 // If the loc refers to a macro expansion we need to first get the file180 // location of the expansion.181 auto &SM = CI.getSourceManager();182 auto FileLoc = SM.getFileLoc(Loc);183 FileID FID = SM.getFileID(FileLoc);184 if (FID.isInvalid())185 return false;186 187 OptionalFileEntryRef File = SM.getFileEntryRefForID(FID);188 if (!File)189 return false;190 191 if (KnownFileEntries.count(*File))192 return true;193 194 if (ExternalFileEntries.count(*File))195 return false;196 197 // Try to reduce the include name the same way we tried to include it.198 bool IsQuoted = false;199 if (auto IncludeName = getRelativeIncludeName(CI, *File, &IsQuoted))200 if (llvm::any_of(KnownFiles,201 [&IsQuoted, &IncludeName](const auto &KnownFile) {202 return KnownFile.first.equals(*IncludeName) &&203 KnownFile.second == IsQuoted;204 })) {205 KnownFileEntries.insert(*File);206 return true;207 }208 209 // Record that the file was not found to avoid future reverse lookup for210 // the same file.211 ExternalFileEntries.insert(*File);212 return false;213 }214 215 LocationFileChecker(const CompilerInstance &CI,216 SmallVector<std::pair<SmallString<32>, bool>> &KnownFiles)217 : CI(CI), KnownFiles(KnownFiles), ExternalFileEntries() {218 for (const auto &KnownFile : KnownFiles)219 if (auto FE = CI.getFileManager().getOptionalFileRef(KnownFile.first))220 KnownFileEntries.insert(*FE);221 }222 223private:224 const CompilerInstance &CI;225 SmallVector<std::pair<SmallString<32>, bool>> &KnownFiles;226 llvm::DenseSet<const FileEntry *> KnownFileEntries;227 llvm::DenseSet<const FileEntry *> ExternalFileEntries;228};229 230struct BatchExtractAPIVisitor : ExtractAPIVisitor<BatchExtractAPIVisitor> {231 bool shouldDeclBeIncluded(const Decl *D) const {232 bool ShouldBeIncluded = true;233 // Check that we have the definition for redeclarable types.234 if (auto *TD = llvm::dyn_cast<TagDecl>(D))235 ShouldBeIncluded = TD->isThisDeclarationADefinition();236 else if (auto *Interface = llvm::dyn_cast<ObjCInterfaceDecl>(D))237 ShouldBeIncluded = Interface->isThisDeclarationADefinition();238 else if (auto *Protocol = llvm::dyn_cast<ObjCProtocolDecl>(D))239 ShouldBeIncluded = Protocol->isThisDeclarationADefinition();240 241 ShouldBeIncluded = ShouldBeIncluded && LCF(D->getLocation());242 return ShouldBeIncluded;243 }244 245 BatchExtractAPIVisitor(LocationFileChecker &LCF, ASTContext &Context,246 APISet &API)247 : ExtractAPIVisitor<BatchExtractAPIVisitor>(Context, API), LCF(LCF) {}248 249private:250 LocationFileChecker &LCF;251};252 253class WrappingExtractAPIConsumer : public ASTConsumer {254public:255 WrappingExtractAPIConsumer(ASTContext &Context, APISet &API)256 : Visitor(Context, API) {}257 258 void HandleTranslationUnit(ASTContext &Context) override {259 // Use ExtractAPIVisitor to traverse symbol declarations in the context.260 Visitor.TraverseDecl(Context.getTranslationUnitDecl());261 }262 263private:264 ExtractAPIVisitor<> Visitor;265};266 267class ExtractAPIConsumer : public ASTConsumer {268public:269 ExtractAPIConsumer(ASTContext &Context,270 std::unique_ptr<LocationFileChecker> LCF, APISet &API)271 : Visitor(*LCF, Context, API), LCF(std::move(LCF)) {}272 273 void HandleTranslationUnit(ASTContext &Context) override {274 // Use ExtractAPIVisitor to traverse symbol declarations in the context.275 Visitor.TraverseDecl(Context.getTranslationUnitDecl());276 }277 278private:279 BatchExtractAPIVisitor Visitor;280 std::unique_ptr<LocationFileChecker> LCF;281};282 283class MacroCallback : public PPCallbacks {284public:285 MacroCallback(const SourceManager &SM, APISet &API, Preprocessor &PP)286 : SM(SM), API(API), PP(PP) {}287 288 void EndOfMainFile() override {289 for (const auto &M : PP.macros()) {290 auto *II = M.getFirst();291 auto MD = PP.getMacroDefinition(II);292 auto *MI = MD.getMacroInfo();293 294 if (!MI)295 continue;296 297 // Ignore header guard macros298 if (MI->isUsedForHeaderGuard())299 continue;300 301 // Ignore builtin macros and ones defined via the command line.302 if (MI->isBuiltinMacro())303 continue;304 305 auto DefLoc = MI->getDefinitionLoc();306 307 if (SM.isInPredefinedFile(DefLoc))308 continue;309 310 auto AssociatedModuleMacros = MD.getModuleMacros();311 StringRef OwningModuleName;312 if (!AssociatedModuleMacros.empty())313 OwningModuleName = AssociatedModuleMacros.back()314 ->getOwningModule()315 ->getTopLevelModuleName();316 317 if (!shouldMacroBeIncluded(DefLoc, OwningModuleName))318 continue;319 320 StringRef Name = II->getName();321 PresumedLoc Loc = SM.getPresumedLoc(DefLoc);322 SmallString<128> USR;323 index::generateUSRForMacro(Name, DefLoc, SM, USR);324 API.createRecord<extractapi::MacroDefinitionRecord>(325 USR, Name, SymbolReference(), Loc,326 DeclarationFragmentsBuilder::getFragmentsForMacro(Name, MI),327 DeclarationFragmentsBuilder::getSubHeadingForMacro(Name),328 SM.isInSystemHeader(DefLoc));329 }330 }331 332 virtual bool shouldMacroBeIncluded(const SourceLocation &MacroLoc,333 StringRef ModuleName) {334 return true;335 }336 337 const SourceManager &SM;338 APISet &API;339 Preprocessor &PP;340};341 342class APIMacroCallback : public MacroCallback {343public:344 APIMacroCallback(const SourceManager &SM, APISet &API, Preprocessor &PP,345 LocationFileChecker &LCF)346 : MacroCallback(SM, API, PP), LCF(LCF) {}347 348 bool shouldMacroBeIncluded(const SourceLocation &MacroLoc,349 StringRef ModuleName) override {350 // Do not include macros from external files351 return LCF(MacroLoc);352 }353 354private:355 LocationFileChecker &LCF;356};357 358std::unique_ptr<llvm::raw_pwrite_stream>359createAdditionalSymbolGraphFile(CompilerInstance &CI, Twine BaseName) {360 auto OutputDirectory = CI.getFrontendOpts().SymbolGraphOutputDir;361 362 SmallString<256> FileName;363 llvm::sys::path::append(FileName, OutputDirectory,364 BaseName + ".symbols.json");365 return CI.createOutputFile(366 FileName, /*Binary*/ false, /*RemoveFileOnSignal*/ false,367 /*UseTemporary*/ true, /*CreateMissingDirectories*/ true);368}369 370} // namespace371 372void ExtractAPIActionBase::ImplEndSourceFileAction(CompilerInstance &CI) {373 SymbolGraphSerializerOption SerializationOptions;374 SerializationOptions.Compact = !CI.getFrontendOpts().EmitPrettySymbolGraphs;375 SerializationOptions.EmitSymbolLabelsForTesting =376 CI.getFrontendOpts().EmitSymbolGraphSymbolLabelsForTesting;377 378 if (CI.getFrontendOpts().EmitExtensionSymbolGraphs) {379 auto ConstructOutputFile = [&CI](Twine BaseName) {380 return createAdditionalSymbolGraphFile(CI, BaseName);381 };382 383 SymbolGraphSerializer::serializeWithExtensionGraphs(384 *OS, *API, IgnoresList, ConstructOutputFile, SerializationOptions);385 } else {386 SymbolGraphSerializer::serializeMainSymbolGraph(*OS, *API, IgnoresList,387 SerializationOptions);388 }389 390 // Flush the stream and close the main output stream.391 OS.reset();392}393 394std::unique_ptr<ASTConsumer>395ExtractAPIAction::CreateASTConsumer(CompilerInstance &CI, StringRef InFile) {396 auto ProductName = CI.getFrontendOpts().ProductName;397 398 if (CI.getFrontendOpts().SymbolGraphOutputDir.empty())399 OS = CI.createDefaultOutputFile(/*Binary*/ false, InFile,400 /*Extension*/ "symbols.json",401 /*RemoveFileOnSignal*/ false,402 /*CreateMissingDirectories*/ true);403 else404 OS = createAdditionalSymbolGraphFile(CI, ProductName);405 406 if (!OS)407 return nullptr;408 409 // Now that we have enough information about the language options and the410 // target triple, let's create the APISet before anyone uses it.411 API = std::make_unique<APISet>(412 CI.getTarget().getTriple(),413 CI.getFrontendOpts().Inputs.back().getKind().getLanguage(), ProductName);414 415 auto LCF = std::make_unique<LocationFileChecker>(CI, KnownInputFiles);416 417 CI.getPreprocessor().addPPCallbacks(std::make_unique<APIMacroCallback>(418 CI.getSourceManager(), *API, CI.getPreprocessor(), *LCF));419 420 // Do not include location in anonymous decls.421 PrintingPolicy Policy = CI.getASTContext().getPrintingPolicy();422 Policy.AnonymousTagLocations = false;423 CI.getASTContext().setPrintingPolicy(Policy);424 425 if (!CI.getFrontendOpts().ExtractAPIIgnoresFileList.empty()) {426 llvm::handleAllErrors(427 APIIgnoresList::create(CI.getFrontendOpts().ExtractAPIIgnoresFileList,428 CI.getFileManager())429 .moveInto(IgnoresList),430 [&CI](const IgnoresFileNotFound &Err) {431 CI.getDiagnostics().Report(432 diag::err_extract_api_ignores_file_not_found)433 << Err.Path;434 });435 }436 437 return std::make_unique<ExtractAPIConsumer>(CI.getASTContext(),438 std::move(LCF), *API);439}440 441bool ExtractAPIAction::PrepareToExecuteAction(CompilerInstance &CI) {442 auto &Inputs = CI.getFrontendOpts().Inputs;443 if (Inputs.empty())444 return true;445 446 if (!CI.hasFileManager())447 CI.createFileManager();448 449 auto Kind = Inputs[0].getKind();450 451 // Convert the header file inputs into a single input buffer.452 SmallString<256> HeaderContents;453 bool IsQuoted = false;454 for (const FrontendInputFile &FIF : Inputs) {455 if (Kind.isObjectiveC())456 HeaderContents += "#import";457 else458 HeaderContents += "#include";459 460 StringRef FilePath = FIF.getFile();461 if (auto RelativeName = getRelativeIncludeName(CI, FilePath, &IsQuoted)) {462 if (IsQuoted)463 HeaderContents += " \"";464 else465 HeaderContents += " <";466 467 HeaderContents += *RelativeName;468 469 if (IsQuoted)470 HeaderContents += "\"\n";471 else472 HeaderContents += ">\n";473 KnownInputFiles.emplace_back(static_cast<SmallString<32>>(*RelativeName),474 IsQuoted);475 } else {476 HeaderContents += " \"";477 HeaderContents += FilePath;478 HeaderContents += "\"\n";479 KnownInputFiles.emplace_back(FilePath, true);480 }481 }482 483 if (CI.getHeaderSearchOpts().Verbose)484 CI.getVerboseOutputStream() << getInputBufferName() << ":\n"485 << HeaderContents << "\n";486 487 Buffer = llvm::MemoryBuffer::getMemBufferCopy(HeaderContents,488 getInputBufferName());489 490 // Set that buffer up as our "real" input in the CompilerInstance.491 Inputs.clear();492 Inputs.emplace_back(Buffer->getMemBufferRef(), Kind, /*IsSystem*/ false);493 494 return true;495}496 497void ExtractAPIAction::EndSourceFileAction() {498 ImplEndSourceFileAction(getCompilerInstance());499}500 501std::unique_ptr<ASTConsumer>502WrappingExtractAPIAction::CreateASTConsumer(CompilerInstance &CI,503 StringRef InFile) {504 auto OtherConsumer = WrapperFrontendAction::CreateASTConsumer(CI, InFile);505 if (!OtherConsumer)506 return nullptr;507 508 CreatedASTConsumer = true;509 510 ProductName = CI.getFrontendOpts().ProductName;511 auto InputFilename = llvm::sys::path::filename(InFile);512 OS = createAdditionalSymbolGraphFile(CI, InputFilename);513 514 // Now that we have enough information about the language options and the515 // target triple, let's create the APISet before anyone uses it.516 API = std::make_unique<APISet>(517 CI.getTarget().getTriple(),518 CI.getFrontendOpts().Inputs.back().getKind().getLanguage(), ProductName);519 520 CI.getPreprocessor().addPPCallbacks(std::make_unique<MacroCallback>(521 CI.getSourceManager(), *API, CI.getPreprocessor()));522 523 // Do not include location in anonymous decls.524 PrintingPolicy Policy = CI.getASTContext().getPrintingPolicy();525 Policy.AnonymousTagLocations = false;526 CI.getASTContext().setPrintingPolicy(Policy);527 528 if (!CI.getFrontendOpts().ExtractAPIIgnoresFileList.empty()) {529 llvm::handleAllErrors(530 APIIgnoresList::create(CI.getFrontendOpts().ExtractAPIIgnoresFileList,531 CI.getFileManager())532 .moveInto(IgnoresList),533 [&CI](const IgnoresFileNotFound &Err) {534 CI.getDiagnostics().Report(535 diag::err_extract_api_ignores_file_not_found)536 << Err.Path;537 });538 }539 540 auto WrappingConsumer =541 std::make_unique<WrappingExtractAPIConsumer>(CI.getASTContext(), *API);542 std::vector<std::unique_ptr<ASTConsumer>> Consumers;543 Consumers.push_back(std::move(OtherConsumer));544 Consumers.push_back(std::move(WrappingConsumer));545 546 return std::make_unique<MultiplexConsumer>(std::move(Consumers));547}548 549void WrappingExtractAPIAction::EndSourceFileAction() {550 // Invoke wrapped action's method.551 WrapperFrontendAction::EndSourceFileAction();552 553 if (CreatedASTConsumer) {554 ImplEndSourceFileAction(getCompilerInstance());555 }556}557