881 lines · cpp
1//===- DependencyScannerImpl.cpp - Implements module dependency scanning --===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8 9#include "DependencyScannerImpl.h"10#include "clang/Basic/DiagnosticFrontend.h"11#include "clang/Basic/DiagnosticSerialization.h"12#include "clang/Driver/Driver.h"13#include "clang/Frontend/FrontendActions.h"14#include "clang/Tooling/DependencyScanning/DependencyScanningWorker.h"15#include "llvm/ADT/ScopeExit.h"16#include "llvm/TargetParser/Host.h"17 18using namespace clang;19using namespace tooling;20using namespace dependencies;21 22namespace {23/// Forwards the gatherered dependencies to the consumer.24class DependencyConsumerForwarder : public DependencyFileGenerator {25public:26 DependencyConsumerForwarder(std::unique_ptr<DependencyOutputOptions> Opts,27 StringRef WorkingDirectory, DependencyConsumer &C)28 : DependencyFileGenerator(*Opts), WorkingDirectory(WorkingDirectory),29 Opts(std::move(Opts)), C(C) {}30 31 void finishedMainFile(DiagnosticsEngine &Diags) override {32 C.handleDependencyOutputOpts(*Opts);33 llvm::SmallString<256> CanonPath;34 for (const auto &File : getDependencies()) {35 CanonPath = File;36 llvm::sys::path::remove_dots(CanonPath, /*remove_dot_dot=*/true);37 llvm::sys::path::make_absolute(WorkingDirectory, CanonPath);38 C.handleFileDependency(CanonPath);39 }40 }41 42private:43 StringRef WorkingDirectory;44 std::unique_ptr<DependencyOutputOptions> Opts;45 DependencyConsumer &C;46};47 48static bool checkHeaderSearchPaths(const HeaderSearchOptions &HSOpts,49 const HeaderSearchOptions &ExistingHSOpts,50 DiagnosticsEngine *Diags,51 const LangOptions &LangOpts) {52 if (LangOpts.Modules) {53 if (HSOpts.VFSOverlayFiles != ExistingHSOpts.VFSOverlayFiles) {54 if (Diags) {55 Diags->Report(diag::warn_pch_vfsoverlay_mismatch);56 auto VFSNote = [&](int Type, ArrayRef<std::string> VFSOverlays) {57 if (VFSOverlays.empty()) {58 Diags->Report(diag::note_pch_vfsoverlay_empty) << Type;59 } else {60 std::string Files = llvm::join(VFSOverlays, "\n");61 Diags->Report(diag::note_pch_vfsoverlay_files) << Type << Files;62 }63 };64 VFSNote(0, HSOpts.VFSOverlayFiles);65 VFSNote(1, ExistingHSOpts.VFSOverlayFiles);66 }67 }68 }69 return false;70}71 72using PrebuiltModuleFilesT = decltype(HeaderSearchOptions::PrebuiltModuleFiles);73 74/// A listener that collects the imported modules and the input75/// files. While visiting, collect vfsoverlays and file inputs that determine76/// whether prebuilt modules fully resolve in stable directories.77class PrebuiltModuleListener : public ASTReaderListener {78public:79 PrebuiltModuleListener(PrebuiltModuleFilesT &PrebuiltModuleFiles,80 llvm::SmallVector<std::string> &NewModuleFiles,81 PrebuiltModulesAttrsMap &PrebuiltModulesASTMap,82 const HeaderSearchOptions &HSOpts,83 const LangOptions &LangOpts, DiagnosticsEngine &Diags,84 const ArrayRef<StringRef> StableDirs)85 : PrebuiltModuleFiles(PrebuiltModuleFiles),86 NewModuleFiles(NewModuleFiles),87 PrebuiltModulesASTMap(PrebuiltModulesASTMap), ExistingHSOpts(HSOpts),88 ExistingLangOpts(LangOpts), Diags(Diags), StableDirs(StableDirs) {}89 90 bool needsImportVisitation() const override { return true; }91 bool needsInputFileVisitation() override { return true; }92 bool needsSystemInputFileVisitation() override { return true; }93 94 /// Accumulate the modules are transitively depended on by the initial95 /// prebuilt module.96 void visitImport(StringRef ModuleName, StringRef Filename) override {97 if (PrebuiltModuleFiles.insert({ModuleName.str(), Filename.str()}).second)98 NewModuleFiles.push_back(Filename.str());99 100 auto PrebuiltMapEntry = PrebuiltModulesASTMap.try_emplace(Filename);101 PrebuiltModuleASTAttrs &PrebuiltModule = PrebuiltMapEntry.first->second;102 if (PrebuiltMapEntry.second)103 PrebuiltModule.setInStableDir(!StableDirs.empty());104 105 if (auto It = PrebuiltModulesASTMap.find(CurrentFile);106 It != PrebuiltModulesASTMap.end() && CurrentFile != Filename)107 PrebuiltModule.addDependent(It->getKey());108 }109 110 /// For each input file discovered, check whether it's external path is in a111 /// stable directory. Traversal is stopped if the current module is not112 /// considered stable.113 bool visitInputFileAsRequested(StringRef FilenameAsRequested,114 StringRef Filename, bool isSystem,115 bool isOverridden,116 bool isExplicitModule) override {117 if (StableDirs.empty())118 return false;119 auto PrebuiltEntryIt = PrebuiltModulesASTMap.find(CurrentFile);120 if ((PrebuiltEntryIt == PrebuiltModulesASTMap.end()) ||121 (!PrebuiltEntryIt->second.isInStableDir()))122 return false;123 124 PrebuiltEntryIt->second.setInStableDir(125 isPathInStableDir(StableDirs, Filename));126 return PrebuiltEntryIt->second.isInStableDir();127 }128 129 /// Update which module that is being actively traversed.130 void visitModuleFile(StringRef Filename,131 serialization::ModuleKind Kind) override {132 // If the CurrentFile is not133 // considered stable, update any of it's transitive dependents.134 auto PrebuiltEntryIt = PrebuiltModulesASTMap.find(CurrentFile);135 if ((PrebuiltEntryIt != PrebuiltModulesASTMap.end()) &&136 !PrebuiltEntryIt->second.isInStableDir())137 PrebuiltEntryIt->second.updateDependentsNotInStableDirs(138 PrebuiltModulesASTMap);139 CurrentFile = Filename;140 }141 142 /// Check the header search options for a given module when considering143 /// if the module comes from stable directories.144 bool ReadHeaderSearchOptions(const HeaderSearchOptions &HSOpts,145 StringRef ModuleFilename,146 StringRef SpecificModuleCachePath,147 bool Complain) override {148 149 auto PrebuiltMapEntry = PrebuiltModulesASTMap.try_emplace(CurrentFile);150 PrebuiltModuleASTAttrs &PrebuiltModule = PrebuiltMapEntry.first->second;151 if (PrebuiltMapEntry.second)152 PrebuiltModule.setInStableDir(!StableDirs.empty());153 154 if (PrebuiltModule.isInStableDir())155 PrebuiltModule.setInStableDir(areOptionsInStableDir(StableDirs, HSOpts));156 157 return false;158 }159 160 /// Accumulate vfsoverlays used to build these prebuilt modules.161 bool ReadHeaderSearchPaths(const HeaderSearchOptions &HSOpts,162 bool Complain) override {163 164 auto PrebuiltMapEntry = PrebuiltModulesASTMap.try_emplace(CurrentFile);165 PrebuiltModuleASTAttrs &PrebuiltModule = PrebuiltMapEntry.first->second;166 if (PrebuiltMapEntry.second)167 PrebuiltModule.setInStableDir(!StableDirs.empty());168 169 PrebuiltModule.setVFS(170 llvm::StringSet<>(llvm::from_range, HSOpts.VFSOverlayFiles));171 172 return checkHeaderSearchPaths(173 HSOpts, ExistingHSOpts, Complain ? &Diags : nullptr, ExistingLangOpts);174 }175 176private:177 PrebuiltModuleFilesT &PrebuiltModuleFiles;178 llvm::SmallVector<std::string> &NewModuleFiles;179 PrebuiltModulesAttrsMap &PrebuiltModulesASTMap;180 const HeaderSearchOptions &ExistingHSOpts;181 const LangOptions &ExistingLangOpts;182 DiagnosticsEngine &Diags;183 std::string CurrentFile;184 const ArrayRef<StringRef> StableDirs;185};186 187/// Visit the given prebuilt module and collect all of the modules it188/// transitively imports and contributing input files.189static bool visitPrebuiltModule(StringRef PrebuiltModuleFilename,190 CompilerInstance &CI,191 PrebuiltModuleFilesT &ModuleFiles,192 PrebuiltModulesAttrsMap &PrebuiltModulesASTMap,193 DiagnosticsEngine &Diags,194 const ArrayRef<StringRef> StableDirs) {195 // List of module files to be processed.196 llvm::SmallVector<std::string> Worklist;197 198 PrebuiltModuleListener Listener(ModuleFiles, Worklist, PrebuiltModulesASTMap,199 CI.getHeaderSearchOpts(), CI.getLangOpts(),200 Diags, StableDirs);201 202 Listener.visitModuleFile(PrebuiltModuleFilename,203 serialization::MK_ExplicitModule);204 if (ASTReader::readASTFileControlBlock(205 PrebuiltModuleFilename, CI.getFileManager(), CI.getModuleCache(),206 CI.getPCHContainerReader(),207 /*FindModuleFileExtensions=*/false, Listener,208 /*ValidateDiagnosticOptions=*/false, ASTReader::ARR_OutOfDate))209 return true;210 211 while (!Worklist.empty()) {212 Listener.visitModuleFile(Worklist.back(), serialization::MK_ExplicitModule);213 if (ASTReader::readASTFileControlBlock(214 Worklist.pop_back_val(), CI.getFileManager(), CI.getModuleCache(),215 CI.getPCHContainerReader(),216 /*FindModuleFileExtensions=*/false, Listener,217 /*ValidateDiagnosticOptions=*/false))218 return true;219 }220 return false;221}222 223/// Transform arbitrary file name into an object-like file name.224static std::string makeObjFileName(StringRef FileName) {225 SmallString<128> ObjFileName(FileName);226 llvm::sys::path::replace_extension(ObjFileName, "o");227 return std::string(ObjFileName);228}229 230/// Deduce the dependency target based on the output file and input files.231static std::string232deduceDepTarget(const std::string &OutputFile,233 const SmallVectorImpl<FrontendInputFile> &InputFiles) {234 if (OutputFile != "-")235 return OutputFile;236 237 if (InputFiles.empty() || !InputFiles.front().isFile())238 return "clang-scan-deps\\ dependency";239 240 return makeObjFileName(InputFiles.front().getFile());241}242 243// Clang implements -D and -U by splatting text into a predefines buffer. This244// allows constructs such as `-DFඞ=3 "-D F\u{0D9E} 4 3 2”` to be accepted and245// define the same macro, or adding C++ style comments before the macro name.246//247// This function checks that the first non-space characters in the macro248// obviously form an identifier that can be uniqued on without lexing. Failing249// to do this could lead to changing the final definition of a macro.250//251// We could set up a preprocessor and actually lex the name, but that's very252// heavyweight for a situation that will almost never happen in practice.253static std::optional<StringRef> getSimpleMacroName(StringRef Macro) {254 StringRef Name = Macro.split("=").first.ltrim(" \t");255 std::size_t I = 0;256 257 auto FinishName = [&]() -> std::optional<StringRef> {258 StringRef SimpleName = Name.slice(0, I);259 if (SimpleName.empty())260 return std::nullopt;261 return SimpleName;262 };263 264 for (; I != Name.size(); ++I) {265 switch (Name[I]) {266 case '(': // Start of macro parameter list267 case ' ': // End of macro name268 case '\t':269 return FinishName();270 case '_':271 continue;272 default:273 if (llvm::isAlnum(Name[I]))274 continue;275 return std::nullopt;276 }277 }278 return FinishName();279}280 281static void canonicalizeDefines(PreprocessorOptions &PPOpts) {282 using MacroOpt = std::pair<StringRef, std::size_t>;283 std::vector<MacroOpt> SimpleNames;284 SimpleNames.reserve(PPOpts.Macros.size());285 std::size_t Index = 0;286 for (const auto &M : PPOpts.Macros) {287 auto SName = getSimpleMacroName(M.first);288 // Skip optimizing if we can't guarantee we can preserve relative order.289 if (!SName)290 return;291 SimpleNames.emplace_back(*SName, Index);292 ++Index;293 }294 295 llvm::stable_sort(SimpleNames, llvm::less_first());296 // Keep the last instance of each macro name by going in reverse297 auto NewEnd = std::unique(298 SimpleNames.rbegin(), SimpleNames.rend(),299 [](const MacroOpt &A, const MacroOpt &B) { return A.first == B.first; });300 SimpleNames.erase(SimpleNames.begin(), NewEnd.base());301 302 // Apply permutation.303 decltype(PPOpts.Macros) NewMacros;304 NewMacros.reserve(SimpleNames.size());305 for (std::size_t I = 0, E = SimpleNames.size(); I != E; ++I) {306 std::size_t OriginalIndex = SimpleNames[I].second;307 // We still emit undefines here as they may be undefining a predefined macro308 NewMacros.push_back(std::move(PPOpts.Macros[OriginalIndex]));309 }310 std::swap(PPOpts.Macros, NewMacros);311}312 313class ScanningDependencyDirectivesGetter : public DependencyDirectivesGetter {314 DependencyScanningWorkerFilesystem *DepFS;315 316public:317 ScanningDependencyDirectivesGetter(FileManager &FileMgr) : DepFS(nullptr) {318 FileMgr.getVirtualFileSystem().visit([&](llvm::vfs::FileSystem &FS) {319 auto *DFS = llvm::dyn_cast<DependencyScanningWorkerFilesystem>(&FS);320 if (DFS) {321 assert(!DepFS && "Found multiple scanning VFSs");322 DepFS = DFS;323 }324 });325 assert(DepFS && "Did not find scanning VFS");326 }327 328 std::unique_ptr<DependencyDirectivesGetter>329 cloneFor(FileManager &FileMgr) override {330 return std::make_unique<ScanningDependencyDirectivesGetter>(FileMgr);331 }332 333 std::optional<ArrayRef<dependency_directives_scan::Directive>>334 operator()(FileEntryRef File) override {335 return DepFS->getDirectiveTokens(File.getName());336 }337};338 339/// Sanitize diagnostic options for dependency scan.340void sanitizeDiagOpts(DiagnosticOptions &DiagOpts) {341 // Don't print 'X warnings and Y errors generated'.342 DiagOpts.ShowCarets = false;343 // Don't write out diagnostic file.344 DiagOpts.DiagnosticSerializationFile.clear();345 // Don't emit warnings except for scanning specific warnings.346 // TODO: It would be useful to add a more principled way to ignore all347 // warnings that come from source code. The issue is that we need to348 // ignore warnings that could be surpressed by349 // `#pragma clang diagnostic`, while still allowing some scanning350 // warnings for things we're not ready to turn into errors yet.351 // See `test/ClangScanDeps/diagnostic-pragmas.c` for an example.352 llvm::erase_if(DiagOpts.Warnings, [](StringRef Warning) {353 return llvm::StringSwitch<bool>(Warning)354 .Cases({"pch-vfs-diff", "error=pch-vfs-diff"}, false)355 .StartsWith("no-error=", false)356 .Default(true);357 });358}359} // namespace360 361std::unique_ptr<DiagnosticOptions>362dependencies::createDiagOptions(ArrayRef<std::string> CommandLine) {363 std::vector<const char *> CLI;364 for (const std::string &Arg : CommandLine)365 CLI.push_back(Arg.c_str());366 auto DiagOpts = CreateAndPopulateDiagOpts(CLI);367 sanitizeDiagOpts(*DiagOpts);368 return DiagOpts;369}370 371DignosticsEngineWithDiagOpts::DignosticsEngineWithDiagOpts(372 ArrayRef<std::string> CommandLine,373 IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS, DiagnosticConsumer &DC) {374 std::vector<const char *> CCommandLine(CommandLine.size(), nullptr);375 llvm::transform(CommandLine, CCommandLine.begin(),376 [](const std::string &Str) { return Str.c_str(); });377 DiagOpts = CreateAndPopulateDiagOpts(CCommandLine);378 sanitizeDiagOpts(*DiagOpts);379 DiagEngine = CompilerInstance::createDiagnostics(*FS, *DiagOpts, &DC,380 /*ShouldOwnClient=*/false);381}382 383std::pair<std::unique_ptr<driver::Driver>, std::unique_ptr<driver::Compilation>>384dependencies::buildCompilation(ArrayRef<std::string> ArgStrs,385 DiagnosticsEngine &Diags,386 IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS,387 llvm::BumpPtrAllocator &Alloc) {388 SmallVector<const char *, 256> Argv;389 Argv.reserve(ArgStrs.size());390 for (const std::string &Arg : ArgStrs)391 Argv.push_back(Arg.c_str());392 393 std::unique_ptr<driver::Driver> Driver = std::make_unique<driver::Driver>(394 Argv[0], llvm::sys::getDefaultTargetTriple(), Diags,395 "clang LLVM compiler", FS);396 Driver->setTitle("clang_based_tool");397 398 bool CLMode = driver::IsClangCL(399 driver::getDriverMode(Argv[0], ArrayRef(Argv).slice(1)));400 401 if (llvm::Error E =402 driver::expandResponseFiles(Argv, CLMode, Alloc, FS.get())) {403 Diags.Report(diag::err_drv_expand_response_file)404 << llvm::toString(std::move(E));405 return std::make_pair(nullptr, nullptr);406 }407 408 std::unique_ptr<driver::Compilation> Compilation(409 Driver->BuildCompilation(Argv));410 if (!Compilation)411 return std::make_pair(nullptr, nullptr);412 413 if (Compilation->containsError())414 return std::make_pair(nullptr, nullptr);415 416 return std::make_pair(std::move(Driver), std::move(Compilation));417}418 419std::unique_ptr<CompilerInvocation>420dependencies::createCompilerInvocation(ArrayRef<std::string> CommandLine,421 DiagnosticsEngine &Diags) {422 llvm::opt::ArgStringList Argv;423 for (const std::string &Str : ArrayRef(CommandLine).drop_front())424 Argv.push_back(Str.c_str());425 426 auto Invocation = std::make_unique<CompilerInvocation>();427 if (!CompilerInvocation::CreateFromArgs(*Invocation, Argv, Diags)) {428 // FIXME: Should we just go on like cc1_main does?429 return nullptr;430 }431 return Invocation;432}433 434std::pair<IntrusiveRefCntPtr<llvm::vfs::FileSystem>, std::vector<std::string>>435dependencies::initVFSForTUBufferScanning(436 IntrusiveRefCntPtr<llvm::vfs::FileSystem> BaseFS,437 ArrayRef<std::string> CommandLine, StringRef WorkingDirectory,438 llvm::MemoryBufferRef TUBuffer) {439 // Reset what might have been modified in the previous worker invocation.440 BaseFS->setCurrentWorkingDirectory(WorkingDirectory);441 442 IntrusiveRefCntPtr<llvm::vfs::FileSystem> ModifiedFS;443 auto OverlayFS =444 llvm::makeIntrusiveRefCnt<llvm::vfs::OverlayFileSystem>(BaseFS);445 auto InMemoryFS = llvm::makeIntrusiveRefCnt<llvm::vfs::InMemoryFileSystem>();446 InMemoryFS->setCurrentWorkingDirectory(WorkingDirectory);447 auto InputPath = TUBuffer.getBufferIdentifier();448 InMemoryFS->addFile(449 InputPath, 0, llvm::MemoryBuffer::getMemBufferCopy(TUBuffer.getBuffer()));450 IntrusiveRefCntPtr<llvm::vfs::FileSystem> InMemoryOverlay = InMemoryFS;451 452 OverlayFS->pushOverlay(InMemoryOverlay);453 ModifiedFS = OverlayFS;454 std::vector<std::string> ModifiedCommandLine(CommandLine);455 ModifiedCommandLine.emplace_back(InputPath);456 457 return std::make_pair(ModifiedFS, ModifiedCommandLine);458}459 460std::pair<IntrusiveRefCntPtr<llvm::vfs::OverlayFileSystem>,461 std::vector<std::string>>462dependencies::initVFSForByNameScanning(463 IntrusiveRefCntPtr<llvm::vfs::FileSystem> BaseFS,464 ArrayRef<std::string> CommandLine, StringRef WorkingDirectory,465 StringRef ModuleName) {466 // Reset what might have been modified in the previous worker invocation.467 BaseFS->setCurrentWorkingDirectory(WorkingDirectory);468 469 // If we're scanning based on a module name alone, we don't expect the client470 // to provide us with an input file. However, the driver really wants to have471 // one. Let's just make it up to make the driver happy.472 auto OverlayFS =473 llvm::makeIntrusiveRefCnt<llvm::vfs::OverlayFileSystem>(BaseFS);474 auto InMemoryFS = llvm::makeIntrusiveRefCnt<llvm::vfs::InMemoryFileSystem>();475 InMemoryFS->setCurrentWorkingDirectory(WorkingDirectory);476 SmallString<128> FakeInputPath;477 // TODO: We should retry the creation if the path already exists.478 llvm::sys::fs::createUniquePath(ModuleName + "-%%%%%%%%.input", FakeInputPath,479 /*MakeAbsolute=*/false);480 InMemoryFS->addFile(FakeInputPath, 0, llvm::MemoryBuffer::getMemBuffer(""));481 IntrusiveRefCntPtr<llvm::vfs::FileSystem> InMemoryOverlay = InMemoryFS;482 OverlayFS->pushOverlay(InMemoryOverlay);483 484 std::vector<std::string> ModifiedCommandLine(CommandLine);485 ModifiedCommandLine.emplace_back(FakeInputPath);486 487 return std::make_pair(OverlayFS, ModifiedCommandLine);488}489 490bool dependencies::initializeScanCompilerInstance(491 CompilerInstance &ScanInstance,492 IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS,493 DiagnosticConsumer *DiagConsumer, DependencyScanningService &Service,494 IntrusiveRefCntPtr<DependencyScanningWorkerFilesystem> DepFS) {495 ScanInstance.setBuildingModule(false);496 497 ScanInstance.createVirtualFileSystem(FS, DiagConsumer);498 499 // Create the compiler's actual diagnostics engine.500 sanitizeDiagOpts(ScanInstance.getDiagnosticOpts());501 ScanInstance.createDiagnostics(DiagConsumer, /*ShouldOwnClient=*/false);502 if (!ScanInstance.hasDiagnostics())503 return false;504 505 ScanInstance.getPreprocessorOpts().AllowPCHWithDifferentModulesCachePath =506 true;507 508 if (ScanInstance.getHeaderSearchOpts().ModulesValidateOncePerBuildSession)509 ScanInstance.getHeaderSearchOpts().BuildSessionTimestamp =510 Service.getBuildSessionTimestamp();511 512 ScanInstance.getFrontendOpts().DisableFree = false;513 ScanInstance.getFrontendOpts().GenerateGlobalModuleIndex = false;514 ScanInstance.getFrontendOpts().UseGlobalModuleIndex = false;515 ScanInstance.getFrontendOpts().GenReducedBMI = false;516 ScanInstance.getFrontendOpts().ModuleOutputPath.clear();517 // This will prevent us compiling individual modules asynchronously since518 // FileManager is not thread-safe, but it does improve performance for now.519 ScanInstance.getFrontendOpts().ModulesShareFileManager = true;520 ScanInstance.getHeaderSearchOpts().ModuleFormat = "raw";521 ScanInstance.getHeaderSearchOpts().ModulesIncludeVFSUsage =522 any(Service.getOptimizeArgs() & ScanningOptimizations::VFS);523 524 // Create a new FileManager to match the invocation's FileSystemOptions.525 ScanInstance.createFileManager();526 527 // Use the dependency scanning optimized file system if requested to do so.528 if (DepFS) {529 DepFS->resetBypassedPathPrefix();530 SmallString<256> ModulesCachePath;531 normalizeModuleCachePath(ScanInstance.getFileManager(),532 ScanInstance.getHeaderSearchOpts().ModuleCachePath,533 ModulesCachePath);534 if (!ModulesCachePath.empty())535 DepFS->setBypassedPathPrefix(ModulesCachePath);536 537 ScanInstance.setDependencyDirectivesGetter(538 std::make_unique<ScanningDependencyDirectivesGetter>(539 ScanInstance.getFileManager()));540 }541 542 ScanInstance.createSourceManager();543 544 // Consider different header search and diagnostic options to create545 // different modules. This avoids the unsound aliasing of module PCMs.546 //547 // TODO: Implement diagnostic bucketing to reduce the impact of strict548 // context hashing.549 ScanInstance.getHeaderSearchOpts().ModulesStrictContextHash = true;550 ScanInstance.getHeaderSearchOpts().ModulesSerializeOnlyPreprocessor = true;551 ScanInstance.getHeaderSearchOpts().ModulesSkipDiagnosticOptions = true;552 ScanInstance.getHeaderSearchOpts().ModulesSkipHeaderSearchPaths = true;553 ScanInstance.getHeaderSearchOpts().ModulesSkipPragmaDiagnosticMappings = true;554 ScanInstance.getHeaderSearchOpts().ModulesForceValidateUserHeaders = false;555 556 // Avoid some checks and module map parsing when loading PCM files.557 ScanInstance.getPreprocessorOpts().ModulesCheckRelocated = false;558 559 return true;560}561 562llvm::SmallVector<StringRef>563dependencies::getInitialStableDirs(const CompilerInstance &ScanInstance) {564 // Create a collection of stable directories derived from the ScanInstance565 // for determining whether module dependencies would fully resolve from566 // those directories.567 llvm::SmallVector<StringRef> StableDirs;568 const StringRef Sysroot = ScanInstance.getHeaderSearchOpts().Sysroot;569 if (!Sysroot.empty() && (llvm::sys::path::root_directory(Sysroot) != Sysroot))570 StableDirs = {Sysroot, ScanInstance.getHeaderSearchOpts().ResourceDir};571 return StableDirs;572}573 574std::optional<PrebuiltModulesAttrsMap>575dependencies::computePrebuiltModulesASTMap(576 CompilerInstance &ScanInstance, llvm::SmallVector<StringRef> &StableDirs) {577 // Store a mapping of prebuilt module files and their properties like header578 // search options. This will prevent the implicit build to create duplicate579 // modules and will force reuse of the existing prebuilt module files580 // instead.581 PrebuiltModulesAttrsMap PrebuiltModulesASTMap;582 583 if (!ScanInstance.getPreprocessorOpts().ImplicitPCHInclude.empty())584 if (visitPrebuiltModule(585 ScanInstance.getPreprocessorOpts().ImplicitPCHInclude, ScanInstance,586 ScanInstance.getHeaderSearchOpts().PrebuiltModuleFiles,587 PrebuiltModulesASTMap, ScanInstance.getDiagnostics(), StableDirs))588 return {};589 590 return PrebuiltModulesASTMap;591}592 593std::unique_ptr<DependencyOutputOptions>594dependencies::takeAndUpdateDependencyOutputOptionsFrom(595 CompilerInstance &ScanInstance) {596 // This function moves the existing dependency output options from the597 // invocation to the collector. The options in the invocation are reset,598 // which ensures that the compiler won't create new dependency collectors,599 // and thus won't write out the extra '.d' files to disk.600 auto Opts = std::make_unique<DependencyOutputOptions>();601 std::swap(*Opts, ScanInstance.getInvocation().getDependencyOutputOpts());602 // We need at least one -MT equivalent for the generator of make dependency603 // files to work.604 if (Opts->Targets.empty())605 Opts->Targets = {deduceDepTarget(ScanInstance.getFrontendOpts().OutputFile,606 ScanInstance.getFrontendOpts().Inputs)};607 Opts->IncludeSystemHeaders = true;608 609 return Opts;610}611 612std::shared_ptr<ModuleDepCollector>613dependencies::initializeScanInstanceDependencyCollector(614 CompilerInstance &ScanInstance,615 std::unique_ptr<DependencyOutputOptions> DepOutputOpts,616 StringRef WorkingDirectory, DependencyConsumer &Consumer,617 DependencyScanningService &Service, CompilerInvocation &Inv,618 DependencyActionController &Controller,619 PrebuiltModulesAttrsMap PrebuiltModulesASTMap,620 llvm::SmallVector<StringRef> &StableDirs) {621 std::shared_ptr<ModuleDepCollector> MDC;622 switch (Service.getFormat()) {623 case ScanningOutputFormat::Make:624 ScanInstance.addDependencyCollector(625 std::make_shared<DependencyConsumerForwarder>(626 std::move(DepOutputOpts), WorkingDirectory, Consumer));627 break;628 case ScanningOutputFormat::P1689:629 case ScanningOutputFormat::Full:630 MDC = std::make_shared<ModuleDepCollector>(631 Service, std::move(DepOutputOpts), ScanInstance, Consumer, Controller,632 Inv, std::move(PrebuiltModulesASTMap), StableDirs);633 ScanInstance.addDependencyCollector(MDC);634 break;635 }636 637 return MDC;638}639 640bool DependencyScanningAction::runInvocation(641 std::unique_ptr<CompilerInvocation> Invocation,642 IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS,643 std::shared_ptr<PCHContainerOperations> PCHContainerOps,644 DiagnosticConsumer *DiagConsumer) {645 // Making sure that we canonicalize the defines before we create the deep646 // copy to avoid unnecessary variants in the scanner and in the resulting647 // explicit command lines.648 if (any(Service.getOptimizeArgs() & ScanningOptimizations::Macros))649 canonicalizeDefines(Invocation->getPreprocessorOpts());650 651 // Make a deep copy of the original Clang invocation.652 CompilerInvocation OriginalInvocation(*Invocation);653 654 if (Scanned) {655 // Scanning runs once for the first -cc1 invocation in a chain of driver656 // jobs. For any dependent jobs, reuse the scanning result and just657 // update the LastCC1Arguments to correspond to the new invocation.658 // FIXME: to support multi-arch builds, each arch requires a separate scan659 setLastCC1Arguments(std::move(OriginalInvocation));660 return true;661 }662 663 Scanned = true;664 665 // Create a compiler instance to handle the actual work.666 auto ModCache = makeInProcessModuleCache(Service.getModuleCacheEntries());667 ScanInstanceStorage.emplace(std::move(Invocation), std::move(PCHContainerOps),668 ModCache.get());669 CompilerInstance &ScanInstance = *ScanInstanceStorage;670 671 assert(!DiagConsumerFinished && "attempt to reuse finished consumer");672 if (!initializeScanCompilerInstance(ScanInstance, FS, DiagConsumer, Service,673 DepFS))674 return false;675 676 llvm::SmallVector<StringRef> StableDirs = getInitialStableDirs(ScanInstance);677 auto MaybePrebuiltModulesASTMap =678 computePrebuiltModulesASTMap(ScanInstance, StableDirs);679 if (!MaybePrebuiltModulesASTMap)680 return false;681 682 auto DepOutputOpts = takeAndUpdateDependencyOutputOptionsFrom(ScanInstance);683 684 MDC = initializeScanInstanceDependencyCollector(685 ScanInstance, std::move(DepOutputOpts), WorkingDirectory, Consumer,686 Service, OriginalInvocation, Controller, *MaybePrebuiltModulesASTMap,687 StableDirs);688 689 std::unique_ptr<FrontendAction> Action;690 691 if (Service.getFormat() == ScanningOutputFormat::P1689)692 Action = std::make_unique<PreprocessOnlyAction>();693 else694 Action = std::make_unique<ReadPCHAndPreprocessAction>();695 696 if (ScanInstance.getDiagnostics().hasErrorOccurred())697 return false;698 699 const bool Result = ScanInstance.ExecuteAction(*Action);700 701 // ExecuteAction is responsible for calling finish.702 DiagConsumerFinished = true;703 704 if (Result)705 setLastCC1Arguments(std::move(OriginalInvocation));706 707 return Result;708}709 710bool CompilerInstanceWithContext::initialize(DiagnosticConsumer *DC) {711 if (DC) {712 DiagConsumer = DC;713 } else {714 DiagPrinterWithOS =715 std::make_unique<TextDiagnosticsPrinterWithOutput>(CommandLine);716 DiagConsumer = &DiagPrinterWithOS->DiagPrinter;717 }718 719 std::tie(OverlayFS, CommandLine) = initVFSForByNameScanning(720 Worker.BaseFS, CommandLine, CWD, "ScanningByName");721 722 DiagEngineWithCmdAndOpts = std::make_unique<DignosticsEngineWithDiagOpts>(723 CommandLine, OverlayFS, *DiagConsumer);724 725 std::tie(Driver, Compilation) = buildCompilation(726 CommandLine, *DiagEngineWithCmdAndOpts->DiagEngine, OverlayFS, Alloc);727 728 if (!Compilation)729 return false;730 731 assert(Compilation->getJobs().size() &&732 "Must have a job list of non-zero size");733 const driver::Command &Command = *(Compilation->getJobs().begin());734 const auto &CommandArgs = Command.getArguments();735 assert(!CommandArgs.empty() && "Cannot have a command with 0 args");736 assert(StringRef(CommandArgs[0]) == "-cc1" && "Requires a cc1 job.");737 OriginalInvocation = std::make_unique<CompilerInvocation>();738 739 if (!CompilerInvocation::CreateFromArgs(*OriginalInvocation, CommandArgs,740 *DiagEngineWithCmdAndOpts->DiagEngine,741 Command.getExecutable())) {742 DiagEngineWithCmdAndOpts->DiagEngine->Report(743 diag::err_fe_expected_compiler_job)744 << llvm::join(CommandLine, " ");745 return false;746 }747 748 if (any(Worker.Service.getOptimizeArgs() & ScanningOptimizations::Macros))749 canonicalizeDefines(OriginalInvocation->getPreprocessorOpts());750 751 // Create the CompilerInstance.752 IntrusiveRefCntPtr<ModuleCache> ModCache =753 makeInProcessModuleCache(Worker.Service.getModuleCacheEntries());754 CIPtr = std::make_unique<CompilerInstance>(755 std::make_shared<CompilerInvocation>(*OriginalInvocation),756 Worker.PCHContainerOps, ModCache.get());757 auto &CI = *CIPtr;758 759 if (!initializeScanCompilerInstance(760 CI, OverlayFS, DiagEngineWithCmdAndOpts->DiagEngine->getClient(),761 Worker.Service, Worker.DepFS))762 return false;763 764 StableDirs = getInitialStableDirs(CI);765 auto MaybePrebuiltModulesASTMap =766 computePrebuiltModulesASTMap(CI, StableDirs);767 if (!MaybePrebuiltModulesASTMap)768 return false;769 770 PrebuiltModuleASTMap = std::move(*MaybePrebuiltModulesASTMap);771 OutputOpts = takeAndUpdateDependencyOutputOptionsFrom(CI);772 773 // We do not create the target in initializeScanCompilerInstance because774 // setting it here is unique for by-name lookups. We create the target only775 // once here, and the information is reused for all computeDependencies calls.776 // We do not need to call createTarget explicitly if we go through777 // CompilerInstance::ExecuteAction to perform scanning.778 CI.createTarget();779 780 return true;781}782 783bool CompilerInstanceWithContext::computeDependencies(784 StringRef ModuleName, DependencyConsumer &Consumer,785 DependencyActionController &Controller) {786 assert(CIPtr && "CIPtr must be initialized before calling this method");787 auto &CI = *CIPtr;788 789 // We create this cleanup object because computeDependencies may exit790 // early with errors.791 auto CleanUp = llvm::make_scope_exit([&]() {792 CI.clearDependencyCollectors();793 // The preprocessor may not be created at the entry of this method,794 // but it must have been created when this method returns, whether795 // there are errors during scanning or not.796 CI.getPreprocessor().removePPCallbacks();797 });798 799 auto MDC = initializeScanInstanceDependencyCollector(800 CI, std::make_unique<DependencyOutputOptions>(*OutputOpts), CWD, Consumer,801 Worker.Service,802 /* The MDC's constructor makes a copy of the OriginalInvocation, so803 we can pass it in without worrying that it might be changed across804 invocations of computeDependencies. */805 *OriginalInvocation, Controller, PrebuiltModuleASTMap, StableDirs);806 807 if (!SrcLocOffset) {808 // When SrcLocOffset is zero, we are at the beginning of the fake source809 // file. In this case, we call BeginSourceFile to initialize.810 std::unique_ptr<FrontendAction> Action =811 std::make_unique<PreprocessOnlyAction>();812 auto InputFile = CI.getFrontendOpts().Inputs.begin();813 bool ActionBeginSucceeded = Action->BeginSourceFile(CI, *InputFile);814 assert(ActionBeginSucceeded && "Action BeginSourceFile must succeed");815 (void)ActionBeginSucceeded;816 }817 818 Preprocessor &PP = CI.getPreprocessor();819 SourceManager &SM = PP.getSourceManager();820 FileID MainFileID = SM.getMainFileID();821 SourceLocation FileStart = SM.getLocForStartOfFile(MainFileID);822 SourceLocation IDLocation = FileStart.getLocWithOffset(SrcLocOffset);823 PPCallbacks *CB = nullptr;824 if (!SrcLocOffset) {825 // We need to call EnterSourceFile when SrcLocOffset is zero to initialize826 // the preprocessor.827 bool PPFailed = PP.EnterSourceFile(MainFileID, nullptr, SourceLocation());828 assert(!PPFailed && "Preprocess must be able to enter the main file.");829 (void)PPFailed;830 CB = MDC->getPPCallbacks();831 } else {832 // When SrcLocOffset is non-zero, the preprocessor has already been833 // initialized through a previous call of computeDependencies. We want to834 // preserve the PP's state, hence we do not call EnterSourceFile again.835 MDC->attachToPreprocessor(PP);836 CB = MDC->getPPCallbacks();837 838 FileID PrevFID;839 SrcMgr::CharacteristicKind FileType = SM.getFileCharacteristic(IDLocation);840 CB->LexedFileChanged(MainFileID,841 PPChainedCallbacks::LexedFileChangeReason::EnterFile,842 FileType, PrevFID, IDLocation);843 }844 845 SrcLocOffset++;846 SmallVector<IdentifierLoc, 2> Path;847 IdentifierInfo *ModuleID = PP.getIdentifierInfo(ModuleName);848 Path.emplace_back(IDLocation, ModuleID);849 auto ModResult = CI.loadModule(IDLocation, Path, Module::Hidden, false);850 851 assert(CB && "Must have PPCallbacks after module loading");852 CB->moduleImport(SourceLocation(), Path, ModResult);853 // Note that we are calling the CB's EndOfMainFile function, which854 // forwards the results to the dependency consumer.855 // It does not indicate the end of processing the fake file.856 CB->EndOfMainFile();857 858 if (!ModResult)859 return false;860 861 CompilerInvocation ModuleInvocation(*OriginalInvocation);862 MDC->applyDiscoveredDependencies(ModuleInvocation);863 Consumer.handleBuildCommand(864 {CommandLine[0], ModuleInvocation.getCC1CommandLine()});865 866 return true;867}868 869bool CompilerInstanceWithContext::finalize() {870 DiagConsumer->finish();871 return true;872}873 874llvm::Error CompilerInstanceWithContext::handleReturnStatus(bool Success) {875 assert(DiagPrinterWithOS && "Must use the default DiagnosticConsumer.");876 return Success ? llvm::Error::success()877 : llvm::make_error<llvm::StringError>(878 DiagPrinterWithOS->DiagnosticsOS.str(),879 llvm::inconvertibleErrorCode());880}881