978 lines · cpp
1//===- ModuleDepCollector.cpp - Callbacks to collect deps -------*- C++ -*-===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8 9#include "clang/Tooling/DependencyScanning/ModuleDepCollector.h"10 11#include "clang/Basic/MakeSupport.h"12#include "clang/Frontend/CompilerInstance.h"13#include "clang/Lex/Preprocessor.h"14#include "clang/Tooling/DependencyScanning/DependencyScanningWorker.h"15#include "llvm/ADT/STLExtras.h"16#include "llvm/Support/BLAKE3.h"17#include <optional>18 19using namespace clang;20using namespace tooling;21using namespace dependencies;22 23void ModuleDeps::forEachFileDep(llvm::function_ref<void(StringRef)> Cb) const {24 SmallString<0> PathBuf;25 PathBuf.reserve(256);26 for (StringRef FileDep : FileDeps) {27 auto ResolvedFileDep =28 ASTReader::ResolveImportedPath(PathBuf, FileDep, FileDepsBaseDir);29 Cb(*ResolvedFileDep);30 }31}32 33const std::vector<std::string> &ModuleDeps::getBuildArguments() const {34 // FIXME: this operation is not thread safe and is expected to be called35 // on a single thread. Otherwise it should be protected with a lock.36 assert(!std::holds_alternative<std::monostate>(BuildInfo) &&37 "Using uninitialized ModuleDeps");38 if (const auto *CI = std::get_if<CowCompilerInvocation>(&BuildInfo))39 BuildInfo = CI->getCC1CommandLine();40 return std::get<std::vector<std::string>>(BuildInfo);41}42 43void PrebuiltModuleASTAttrs::updateDependentsNotInStableDirs(44 PrebuiltModulesAttrsMap &PrebuiltModulesMap) {45 setInStableDir();46 for (const auto Dep : ModuleFileDependents) {47 if (!PrebuiltModulesMap[Dep].isInStableDir())48 return;49 PrebuiltModulesMap[Dep].updateDependentsNotInStableDirs(PrebuiltModulesMap);50 }51}52 53static void54optimizeHeaderSearchOpts(HeaderSearchOptions &Opts, ASTReader &Reader,55 const serialization::ModuleFile &MF,56 const PrebuiltModulesAttrsMap &PrebuiltModulesASTMap,57 ScanningOptimizations OptimizeArgs) {58 if (any(OptimizeArgs & ScanningOptimizations::HeaderSearch)) {59 // Only preserve search paths that were used during the dependency scan.60 std::vector<HeaderSearchOptions::Entry> Entries;61 std::swap(Opts.UserEntries, Entries);62 63 llvm::BitVector SearchPathUsage(Entries.size());64 llvm::DenseSet<const serialization::ModuleFile *> Visited;65 std::function<void(const serialization::ModuleFile *)> VisitMF =66 [&](const serialization::ModuleFile *MF) {67 SearchPathUsage |= MF->SearchPathUsage;68 Visited.insert(MF);69 for (const serialization::ModuleFile *Import : MF->Imports)70 if (!Visited.contains(Import))71 VisitMF(Import);72 };73 VisitMF(&MF);74 75 if (SearchPathUsage.size() != Entries.size())76 llvm::report_fatal_error(77 "Inconsistent search path options between modules detected");78 79 for (auto Idx : SearchPathUsage.set_bits())80 Opts.UserEntries.push_back(std::move(Entries[Idx]));81 }82 if (any(OptimizeArgs & ScanningOptimizations::VFS)) {83 std::vector<std::string> VFSOverlayFiles;84 std::swap(Opts.VFSOverlayFiles, VFSOverlayFiles);85 86 llvm::BitVector VFSUsage(VFSOverlayFiles.size());87 llvm::DenseSet<const serialization::ModuleFile *> Visited;88 std::function<void(const serialization::ModuleFile *)> VisitMF =89 [&](const serialization::ModuleFile *MF) {90 Visited.insert(MF);91 if (MF->Kind == serialization::MK_ImplicitModule) {92 VFSUsage |= MF->VFSUsage;93 // We only need to recurse into implicit modules. Other module types94 // will have the correct set of VFSs for anything they depend on.95 for (const serialization::ModuleFile *Import : MF->Imports)96 if (!Visited.contains(Import))97 VisitMF(Import);98 } else {99 // This is not an implicitly built module, so it may have different100 // VFS options. Fall back to a string comparison instead.101 auto PrebuiltModulePropIt =102 PrebuiltModulesASTMap.find(MF->FileName);103 if (PrebuiltModulePropIt == PrebuiltModulesASTMap.end())104 return;105 for (std::size_t I = 0, E = VFSOverlayFiles.size(); I != E; ++I) {106 if (PrebuiltModulePropIt->second.getVFS().contains(107 VFSOverlayFiles[I]))108 VFSUsage[I] = true;109 }110 }111 };112 VisitMF(&MF);113 114 if (VFSUsage.size() != VFSOverlayFiles.size())115 llvm::report_fatal_error(116 "Inconsistent -ivfsoverlay options between modules detected");117 118 for (auto Idx : VFSUsage.set_bits())119 Opts.VFSOverlayFiles.push_back(std::move(VFSOverlayFiles[Idx]));120 }121}122 123static void optimizeDiagnosticOpts(DiagnosticOptions &Opts,124 bool IsSystemModule) {125 // If this is not a system module or -Wsystem-headers was passed, don't126 // optimize.127 if (!IsSystemModule)128 return;129 bool Wsystem_headers = false;130 for (StringRef Opt : Opts.Warnings) {131 bool isPositive = !Opt.consume_front("no-");132 if (Opt == "system-headers")133 Wsystem_headers = isPositive;134 }135 if (Wsystem_headers)136 return;137 138 // Remove all warning flags. System modules suppress most, but not all,139 // warnings.140 Opts.Warnings.clear();141 Opts.UndefPrefixes.clear();142 Opts.Remarks.clear();143}144 145static void optimizeCWD(CowCompilerInvocation &BuildInvocation, StringRef CWD) {146 BuildInvocation.getMutFileSystemOpts().WorkingDir.clear();147 BuildInvocation.getMutCodeGenOpts().DebugCompilationDir.clear();148 BuildInvocation.getMutCodeGenOpts().CoverageCompilationDir.clear();149}150 151static std::vector<std::string> splitString(std::string S, char Separator) {152 SmallVector<StringRef> Segments;153 StringRef(S).split(Segments, Separator, /*MaxSplit=*/-1, /*KeepEmpty=*/false);154 std::vector<std::string> Result;155 Result.reserve(Segments.size());156 for (StringRef Segment : Segments)157 Result.push_back(Segment.str());158 return Result;159}160 161void ModuleDepCollector::addOutputPaths(CowCompilerInvocation &CI,162 ModuleDeps &Deps) {163 CI.getMutFrontendOpts().OutputFile =164 Controller.lookupModuleOutput(Deps, ModuleOutputKind::ModuleFile);165 if (!CI.getDiagnosticOpts().DiagnosticSerializationFile.empty())166 CI.getMutDiagnosticOpts().DiagnosticSerializationFile =167 Controller.lookupModuleOutput(168 Deps, ModuleOutputKind::DiagnosticSerializationFile);169 if (!CI.getDependencyOutputOpts().OutputFile.empty()) {170 CI.getMutDependencyOutputOpts().OutputFile =171 Controller.lookupModuleOutput(Deps, ModuleOutputKind::DependencyFile);172 CI.getMutDependencyOutputOpts().Targets =173 splitString(Controller.lookupModuleOutput(174 Deps, ModuleOutputKind::DependencyTargets),175 '\0');176 if (!CI.getDependencyOutputOpts().OutputFile.empty() &&177 CI.getDependencyOutputOpts().Targets.empty()) {178 // Fallback to -o as dependency target, as in the driver.179 SmallString<128> Target;180 quoteMakeTarget(CI.getFrontendOpts().OutputFile, Target);181 CI.getMutDependencyOutputOpts().Targets.push_back(std::string(Target));182 }183 }184}185 186void dependencies::resetBenignCodeGenOptions(frontend::ActionKind ProgramAction,187 const LangOptions &LangOpts,188 CodeGenOptions &CGOpts) {189 // TODO: Figure out better way to set options to their default value.190 if (ProgramAction == frontend::GenerateModule) {191 CGOpts.MainFileName.clear();192 CGOpts.DwarfDebugFlags.clear();193 }194 if (ProgramAction == frontend::GeneratePCH ||195 (ProgramAction == frontend::GenerateModule && !LangOpts.ModulesCodegen)) {196 CGOpts.DebugCompilationDir.clear();197 CGOpts.CoverageCompilationDir.clear();198 CGOpts.CoverageDataFile.clear();199 CGOpts.CoverageNotesFile.clear();200 CGOpts.ProfileInstrumentUsePath.clear();201 CGOpts.SampleProfileFile.clear();202 CGOpts.ProfileRemappingFile.clear();203 }204}205 206bool dependencies::isPathInStableDir(const ArrayRef<StringRef> Directories,207 const StringRef Input) {208 using namespace llvm::sys;209 210 if (!path::is_absolute(Input))211 return false;212 213 auto PathStartsWith = [](StringRef Prefix, StringRef Path) {214 auto PrefixIt = path::begin(Prefix), PrefixEnd = path::end(Prefix);215 for (auto PathIt = path::begin(Path), PathEnd = path::end(Path);216 PrefixIt != PrefixEnd && PathIt != PathEnd; ++PrefixIt, ++PathIt) {217 if (*PrefixIt != *PathIt)218 return false;219 }220 return PrefixIt == PrefixEnd;221 };222 223 return any_of(Directories, [&](StringRef Dir) {224 return !Dir.empty() && PathStartsWith(Dir, Input);225 });226}227 228bool dependencies::areOptionsInStableDir(const ArrayRef<StringRef> Directories,229 const HeaderSearchOptions &HSOpts) {230 assert(isPathInStableDir(Directories, HSOpts.Sysroot) &&231 "Sysroots differ between module dependencies and current TU");232 233 assert(isPathInStableDir(Directories, HSOpts.ResourceDir) &&234 "ResourceDirs differ between module dependencies and current TU");235 236 for (const auto &Entry : HSOpts.UserEntries) {237 if (!Entry.IgnoreSysRoot)238 continue;239 if (!isPathInStableDir(Directories, Entry.Path))240 return false;241 }242 243 for (const auto &SysPrefix : HSOpts.SystemHeaderPrefixes) {244 if (!isPathInStableDir(Directories, SysPrefix.Prefix))245 return false;246 }247 248 return true;249}250 251static CowCompilerInvocation252makeCommonInvocationForModuleBuild(CompilerInvocation CI) {253 CI.resetNonModularOptions();254 CI.clearImplicitModuleBuildOptions();255 256 // The scanner takes care to avoid passing non-affecting module maps to the257 // explicit compiles. No need to do extra work just to find out there are no258 // module map files to prune.259 CI.getHeaderSearchOpts().ModulesPruneNonAffectingModuleMaps = false;260 261 // Remove options incompatible with explicit module build or are likely to262 // differ between identical modules discovered from different translation263 // units.264 CI.getFrontendOpts().Inputs.clear();265 CI.getFrontendOpts().OutputFile.clear();266 CI.getFrontendOpts().GenReducedBMI = false;267 CI.getFrontendOpts().ModuleOutputPath.clear();268 CI.getHeaderSearchOpts().ModulesSkipHeaderSearchPaths = false;269 CI.getHeaderSearchOpts().ModulesSkipDiagnosticOptions = false;270 // LLVM options are not going to affect the AST271 CI.getFrontendOpts().LLVMArgs.clear();272 273 resetBenignCodeGenOptions(frontend::GenerateModule, CI.getLangOpts(),274 CI.getCodeGenOpts());275 276 // Map output paths that affect behaviour to "-" so their existence is in the277 // context hash. The final path will be computed in addOutputPaths.278 if (!CI.getDiagnosticOpts().DiagnosticSerializationFile.empty())279 CI.getDiagnosticOpts().DiagnosticSerializationFile = "-";280 if (!CI.getDependencyOutputOpts().OutputFile.empty())281 CI.getDependencyOutputOpts().OutputFile = "-";282 CI.getDependencyOutputOpts().Targets.clear();283 284 CI.getFrontendOpts().ProgramAction = frontend::GenerateModule;285 CI.getLangOpts().ModuleName.clear();286 287 // Remove any macro definitions that are explicitly ignored.288 if (!CI.getHeaderSearchOpts().ModulesIgnoreMacros.empty()) {289 llvm::erase_if(290 CI.getPreprocessorOpts().Macros,291 [&CI](const std::pair<std::string, bool> &Def) {292 StringRef MacroDef = Def.first;293 return CI.getHeaderSearchOpts().ModulesIgnoreMacros.contains(294 llvm::CachedHashString(MacroDef.split('=').first));295 });296 // Remove the now unused option.297 CI.getHeaderSearchOpts().ModulesIgnoreMacros.clear();298 }299 300 return CI;301}302 303CowCompilerInvocation304ModuleDepCollector::getInvocationAdjustedForModuleBuildWithoutOutputs(305 const ModuleDeps &Deps,306 llvm::function_ref<void(CowCompilerInvocation &)> Optimize) const {307 CowCompilerInvocation CI = CommonInvocation;308 309 CI.getMutLangOpts().ModuleName = Deps.ID.ModuleName;310 CI.getMutFrontendOpts().IsSystemModule = Deps.IsSystem;311 312 // Inputs313 InputKind ModuleMapInputKind(CI.getFrontendOpts().DashX.getLanguage(),314 InputKind::Format::ModuleMap);315 CI.getMutFrontendOpts().Inputs.emplace_back(Deps.ClangModuleMapFile,316 ModuleMapInputKind);317 318 auto CurrentModuleMapEntry =319 ScanInstance.getFileManager().getOptionalFileRef(Deps.ClangModuleMapFile);320 assert(CurrentModuleMapEntry && "module map file entry not found");321 322 // Remove directly passed modulemap files. They will get added back if they323 // were actually used.324 CI.getMutFrontendOpts().ModuleMapFiles.clear();325 326 auto DepModuleMapFiles = collectModuleMapFiles(Deps.ClangModuleDeps);327 for (StringRef ModuleMapFile : Deps.ModuleMapFileDeps) {328 // TODO: Track these as `FileEntryRef` to simplify the equality check below.329 auto ModuleMapEntry =330 ScanInstance.getFileManager().getOptionalFileRef(ModuleMapFile);331 assert(ModuleMapEntry && "module map file entry not found");332 333 // Don't report module maps describing eagerly-loaded dependency. This334 // information will be deserialized from the PCM.335 // TODO: Verify this works fine when modulemap for module A is eagerly336 // loaded from A.pcm, and module map passed on the command line contains337 // definition of a submodule: "explicit module A.Private { ... }".338 if (Service.shouldEagerLoadModules() &&339 DepModuleMapFiles.contains(*ModuleMapEntry))340 continue;341 342 // Don't report module map file of the current module unless it also343 // describes a dependency (for symmetry).344 if (*ModuleMapEntry == *CurrentModuleMapEntry &&345 !DepModuleMapFiles.contains(*ModuleMapEntry))346 continue;347 348 CI.getMutFrontendOpts().ModuleMapFiles.emplace_back(ModuleMapFile);349 }350 351 // Report the prebuilt modules this module uses.352 for (const auto &PrebuiltModule : Deps.PrebuiltModuleDeps)353 CI.getMutFrontendOpts().ModuleFiles.push_back(PrebuiltModule.PCMFile);354 355 // Add module file inputs from dependencies.356 addModuleFiles(CI, Deps.ClangModuleDeps);357 358 if (!CI.getDiagnosticOpts().SystemHeaderWarningsModules.empty()) {359 // Apply -Wsystem-headers-in-module for the current module.360 if (llvm::is_contained(CI.getDiagnosticOpts().SystemHeaderWarningsModules,361 Deps.ID.ModuleName))362 CI.getMutDiagnosticOpts().Warnings.push_back("system-headers");363 // Remove the now unused option(s).364 CI.getMutDiagnosticOpts().SystemHeaderWarningsModules.clear();365 }366 367 Optimize(CI);368 369 return CI;370}371 372llvm::DenseSet<const FileEntry *> ModuleDepCollector::collectModuleMapFiles(373 ArrayRef<ModuleID> ClangModuleDeps) const {374 llvm::DenseSet<const FileEntry *> ModuleMapFiles;375 for (const ModuleID &MID : ClangModuleDeps) {376 ModuleDeps *MD = ModuleDepsByID.lookup(MID);377 assert(MD && "Inconsistent dependency info");378 // TODO: Track ClangModuleMapFile as `FileEntryRef`.379 auto FE = ScanInstance.getFileManager().getOptionalFileRef(380 MD->ClangModuleMapFile);381 assert(FE && "Missing module map file that was previously found");382 ModuleMapFiles.insert(*FE);383 }384 return ModuleMapFiles;385}386 387void ModuleDepCollector::addModuleMapFiles(388 CompilerInvocation &CI, ArrayRef<ModuleID> ClangModuleDeps) const {389 if (Service.shouldEagerLoadModules())390 return; // Only pcm is needed for eager load.391 392 for (const ModuleID &MID : ClangModuleDeps) {393 ModuleDeps *MD = ModuleDepsByID.lookup(MID);394 assert(MD && "Inconsistent dependency info");395 CI.getFrontendOpts().ModuleMapFiles.push_back(MD->ClangModuleMapFile);396 }397}398 399void ModuleDepCollector::addModuleFiles(400 CompilerInvocation &CI, ArrayRef<ModuleID> ClangModuleDeps) const {401 for (const ModuleID &MID : ClangModuleDeps) {402 ModuleDeps *MD = ModuleDepsByID.lookup(MID);403 std::string PCMPath =404 Controller.lookupModuleOutput(*MD, ModuleOutputKind::ModuleFile);405 406 if (Service.shouldEagerLoadModules())407 CI.getFrontendOpts().ModuleFiles.push_back(std::move(PCMPath));408 else409 CI.getHeaderSearchOpts().PrebuiltModuleFiles.insert(410 {MID.ModuleName, std::move(PCMPath)});411 }412}413 414void ModuleDepCollector::addModuleFiles(415 CowCompilerInvocation &CI, ArrayRef<ModuleID> ClangModuleDeps) const {416 for (const ModuleID &MID : ClangModuleDeps) {417 ModuleDeps *MD = ModuleDepsByID.lookup(MID);418 std::string PCMPath =419 Controller.lookupModuleOutput(*MD, ModuleOutputKind::ModuleFile);420 421 if (Service.shouldEagerLoadModules())422 CI.getMutFrontendOpts().ModuleFiles.push_back(std::move(PCMPath));423 else424 CI.getMutHeaderSearchOpts().PrebuiltModuleFiles.insert(425 {MID.ModuleName, std::move(PCMPath)});426 }427}428 429static bool needsModules(FrontendInputFile FIF) {430 switch (FIF.getKind().getLanguage()) {431 case Language::Unknown:432 case Language::Asm:433 case Language::LLVM_IR:434 return false;435 default:436 return true;437 }438}439 440void ModuleDepCollector::applyDiscoveredDependencies(CompilerInvocation &CI) {441 CI.clearImplicitModuleBuildOptions();442 resetBenignCodeGenOptions(CI.getFrontendOpts().ProgramAction,443 CI.getLangOpts(), CI.getCodeGenOpts());444 445 if (llvm::any_of(CI.getFrontendOpts().Inputs, needsModules)) {446 Preprocessor &PP = ScanInstance.getPreprocessor();447 if (Module *CurrentModule = PP.getCurrentModuleImplementation())448 if (OptionalFileEntryRef CurrentModuleMap =449 PP.getHeaderSearchInfo()450 .getModuleMap()451 .getModuleMapFileForUniquing(CurrentModule))452 CI.getFrontendOpts().ModuleMapFiles.emplace_back(453 CurrentModuleMap->getNameAsRequested());454 455 SmallVector<ModuleID> DirectDeps;456 for (const auto &KV : ModularDeps)457 if (DirectModularDeps.contains(KV.first))458 DirectDeps.push_back(KV.second->ID);459 460 // TODO: Report module maps the same way it's done for modular dependencies.461 addModuleMapFiles(CI, DirectDeps);462 463 addModuleFiles(CI, DirectDeps);464 465 for (const auto &KV : DirectPrebuiltModularDeps)466 CI.getFrontendOpts().ModuleFiles.push_back(KV.second.PCMFile);467 }468}469 470static bool isSafeToIgnoreCWD(const CowCompilerInvocation &CI) {471 // Check if the command line input uses relative paths.472 // It is not safe to ignore the current working directory if any of the473 // command line inputs use relative paths.474 bool AnyRelative = false;475 CI.visitPaths([&](StringRef Path) {476 assert(!AnyRelative && "Continuing path visitation despite returning true");477 AnyRelative |= !Path.empty() && !llvm::sys::path::is_absolute(Path);478 return AnyRelative;479 });480 return !AnyRelative;481}482 483static std::string getModuleContextHash(const ModuleDeps &MD,484 const CowCompilerInvocation &CI,485 bool EagerLoadModules, bool IgnoreCWD,486 llvm::vfs::FileSystem &VFS) {487 llvm::HashBuilder<llvm::TruncatedBLAKE3<16>, llvm::endianness::native>488 HashBuilder;489 490 // Hash the compiler version and serialization version to ensure the module491 // will be readable.492 HashBuilder.add(getClangFullRepositoryVersion());493 HashBuilder.add(serialization::VERSION_MAJOR, serialization::VERSION_MINOR);494 llvm::ErrorOr<std::string> CWD = VFS.getCurrentWorkingDirectory();495 if (CWD && !IgnoreCWD)496 HashBuilder.add(*CWD);497 498 // Hash the BuildInvocation without any input files.499 SmallString<0> ArgVec;500 ArgVec.reserve(4096);501 CI.generateCC1CommandLine([&](const Twine &Arg) {502 Arg.toVector(ArgVec);503 ArgVec.push_back('\0');504 });505 HashBuilder.add(ArgVec);506 507 // Hash the module dependencies. These paths may differ even if the invocation508 // is identical if they depend on the contents of the files in the TU -- for509 // example, case-insensitive paths to modulemap files. Usually such a case510 // would indicate a missed optimization to canonicalize, but it may be511 // difficult to canonicalize all cases when there is a VFS.512 for (const auto &ID : MD.ClangModuleDeps) {513 HashBuilder.add(ID.ModuleName);514 HashBuilder.add(ID.ContextHash);515 }516 517 HashBuilder.add(EagerLoadModules);518 519 llvm::BLAKE3Result<16> Hash = HashBuilder.final();520 std::array<uint64_t, 2> Words;521 static_assert(sizeof(Hash) == sizeof(Words), "Hash must match Words");522 std::memcpy(Words.data(), Hash.data(), sizeof(Hash));523 return toString(llvm::APInt(sizeof(Words) * 8, Words), 36, /*Signed=*/false);524}525 526void ModuleDepCollector::associateWithContextHash(527 const CowCompilerInvocation &CI, bool IgnoreCWD, ModuleDeps &Deps) {528 Deps.ID.ContextHash =529 getModuleContextHash(Deps, CI, Service.shouldEagerLoadModules(),530 IgnoreCWD, ScanInstance.getVirtualFileSystem());531 bool Inserted = ModuleDepsByID.insert({Deps.ID, &Deps}).second;532 (void)Inserted;533 assert(Inserted && "duplicate module mapping");534}535 536void ModuleDepCollectorPP::LexedFileChanged(FileID FID,537 LexedFileChangeReason Reason,538 SrcMgr::CharacteristicKind FileType,539 FileID PrevFID,540 SourceLocation Loc) {541 if (Reason != LexedFileChangeReason::EnterFile)542 return;543 544 SourceManager &SM = MDC.ScanInstance.getSourceManager();545 546 // Dependency generation really does want to go all the way to the547 // file entry for a source location to find out what is depended on.548 // We do not want #line markers to affect dependency generation!549 if (std::optional<StringRef> Filename = SM.getNonBuiltinFilenameForID(FID))550 MDC.addFileDep(llvm::sys::path::remove_leading_dotslash(*Filename));551}552 553void ModuleDepCollectorPP::InclusionDirective(554 SourceLocation HashLoc, const Token &IncludeTok, StringRef FileName,555 bool IsAngled, CharSourceRange FilenameRange, OptionalFileEntryRef File,556 StringRef SearchPath, StringRef RelativePath, const Module *SuggestedModule,557 bool ModuleImported, SrcMgr::CharacteristicKind FileType) {558 if (!File && !ModuleImported) {559 // This is a non-modular include that HeaderSearch failed to find. Add it560 // here as `FileChanged` will never see it.561 MDC.addFileDep(FileName);562 }563 handleImport(SuggestedModule);564}565 566void ModuleDepCollectorPP::moduleImport(SourceLocation ImportLoc,567 ModuleIdPath Path,568 const Module *Imported) {569 if (MDC.ScanInstance.getPreprocessor().isInImportingCXXNamedModules()) {570 P1689ModuleInfo RequiredModule;571 RequiredModule.ModuleName = Path[0].getIdentifierInfo()->getName().str();572 RequiredModule.Type = P1689ModuleInfo::ModuleType::NamedCXXModule;573 MDC.RequiredStdCXXModules.push_back(std::move(RequiredModule));574 return;575 }576 577 handleImport(Imported);578}579 580void ModuleDepCollectorPP::handleImport(const Module *Imported) {581 if (!Imported)582 return;583 584 const Module *TopLevelModule = Imported->getTopLevelModule();585 586 if (MDC.isPrebuiltModule(TopLevelModule))587 MDC.DirectPrebuiltModularDeps.insert(588 {TopLevelModule, PrebuiltModuleDep{TopLevelModule}});589 else {590 MDC.DirectModularDeps.insert(TopLevelModule);591 MDC.DirectImports.insert(Imported);592 }593}594 595void ModuleDepCollectorPP::EndOfMainFile() {596 FileID MainFileID = MDC.ScanInstance.getSourceManager().getMainFileID();597 MDC.MainFile = std::string(MDC.ScanInstance.getSourceManager()598 .getFileEntryRefForID(MainFileID)599 ->getName());600 601 auto &PP = MDC.ScanInstance.getPreprocessor();602 if (PP.isInNamedModule()) {603 P1689ModuleInfo ProvidedModule;604 ProvidedModule.ModuleName = PP.getNamedModuleName();605 ProvidedModule.Type = P1689ModuleInfo::ModuleType::NamedCXXModule;606 ProvidedModule.IsStdCXXModuleInterface = PP.isInNamedInterfaceUnit();607 // Don't put implementation (non partition) unit as Provide.608 // Put the module as required instead. Since the implementation609 // unit will import the primary module implicitly.610 if (PP.isInImplementationUnit())611 MDC.RequiredStdCXXModules.push_back(ProvidedModule);612 else613 MDC.ProvidedStdCXXModule = ProvidedModule;614 }615 616 if (!MDC.ScanInstance.getPreprocessorOpts().ImplicitPCHInclude.empty())617 MDC.addFileDep(MDC.ScanInstance.getPreprocessorOpts().ImplicitPCHInclude);618 619 for (StringRef VFS : MDC.ScanInstance.getHeaderSearchOpts().VFSOverlayFiles)620 MDC.addFileDep(VFS);621 622 if (Module *CurrentModule = PP.getCurrentModuleImplementation()) {623 if (OptionalFileEntryRef CurrentModuleMap =624 PP.getHeaderSearchInfo().getModuleMap().getModuleMapFileForUniquing(625 CurrentModule))626 MDC.addFileDep(CurrentModuleMap->getName());627 }628 629 for (const Module *M :630 MDC.ScanInstance.getPreprocessor().getAffectingClangModules())631 if (!MDC.isPrebuiltModule(M))632 MDC.DirectModularDeps.insert(M);633 634 MDC.addVisibleModules();635 636 for (const Module *M : MDC.DirectModularDeps)637 handleTopLevelModule(M);638 639 MDC.Consumer.handleContextHash(640 MDC.ScanInstance.getInvocation().getModuleHash());641 642 MDC.Consumer.handleDependencyOutputOpts(*MDC.Opts);643 644 MDC.Consumer.handleProvidedAndRequiredStdCXXModules(645 MDC.ProvidedStdCXXModule, MDC.RequiredStdCXXModules);646 647 for (auto &&I : MDC.ModularDeps)648 MDC.Consumer.handleModuleDependency(*I.second);649 650 for (const Module *M : MDC.DirectModularDeps) {651 auto It = MDC.ModularDeps.find(M);652 // Only report direct dependencies that were successfully handled.653 if (It != MDC.ModularDeps.end())654 MDC.Consumer.handleDirectModuleDependency(It->second->ID);655 }656 657 for (auto &&I : MDC.VisibleModules)658 MDC.Consumer.handleVisibleModule(std::string(I.getKey()));659 660 for (auto &&I : MDC.FileDeps)661 MDC.Consumer.handleFileDependency(I);662 663 for (auto &&I : MDC.DirectPrebuiltModularDeps)664 MDC.Consumer.handlePrebuiltModuleDependency(I.second);665}666 667std::optional<ModuleID>668ModuleDepCollectorPP::handleTopLevelModule(const Module *M) {669 assert(M == M->getTopLevelModule() && "Expected top level module!");670 671 // A top-level module might not be actually imported as a module when672 // -fmodule-name is used to compile a translation unit that imports this673 // module. In that case it can be skipped. The appropriate header674 // dependencies will still be reported as expected.675 if (!M->getASTFile())676 return {};677 678 // If this module has been handled already, just return its ID.679 if (auto ModI = MDC.ModularDeps.find(M); ModI != MDC.ModularDeps.end())680 return ModI->second->ID;681 682 auto OwnedMD = std::make_unique<ModuleDeps>();683 ModuleDeps &MD = *OwnedMD;684 685 MD.ID.ModuleName = M->getFullModuleName();686 MD.IsSystem = M->IsSystem;687 688 // Start off with the assumption that this module is shareable when there689 // are stable directories. As more dependencies are discovered, check if those690 // come from the provided directories.691 MD.IsInStableDirectories = !MDC.StableDirs.empty();692 693 // For modules which use export_as link name, the linked product that of the694 // corresponding export_as-named module.695 if (!M->UseExportAsModuleLinkName)696 MD.LinkLibraries = M->LinkLibraries;697 698 ModuleMap &ModMapInfo =699 MDC.ScanInstance.getPreprocessor().getHeaderSearchInfo().getModuleMap();700 701 if (auto ModuleMap = ModMapInfo.getModuleMapFileForUniquing(M)) {702 SmallString<128> Path = ModuleMap->getNameAsRequested();703 ModMapInfo.canonicalizeModuleMapPath(Path);704 MD.ClangModuleMapFile = std::string(Path);705 }706 707 serialization::ModuleFile *MF =708 MDC.ScanInstance.getASTReader()->getModuleManager().lookup(709 *M->getASTFile());710 MD.FileDepsBaseDir = MF->BaseDirectory;711 MDC.ScanInstance.getASTReader()->visitInputFileInfos(712 *MF, /*IncludeSystem=*/true,713 [&](const serialization::InputFileInfo &IFI, bool IsSystem) {714 // The __inferred_module.map file is an insignificant implementation715 // detail of implicitly-built modules. The PCM will also report the716 // actual on-disk module map file that allowed inferring the module,717 // which is what we need for building the module explicitly718 // Let's ignore this file.719 if (IFI.UnresolvedImportedFilename.ends_with("__inferred_module.map"))720 return;721 MDC.addFileDep(MD, IFI.UnresolvedImportedFilename);722 });723 724 llvm::DenseSet<const Module *> SeenDeps;725 addAllSubmodulePrebuiltDeps(M, MD, SeenDeps);726 addAllSubmoduleDeps(M, MD, SeenDeps);727 addAllAffectingClangModules(M, MD, SeenDeps);728 729 SmallString<0> PathBuf;730 PathBuf.reserve(256);731 MDC.ScanInstance.getASTReader()->visitInputFileInfos(732 *MF, /*IncludeSystem=*/true,733 [&](const serialization::InputFileInfo &IFI, bool IsSystem) {734 if (MD.IsInStableDirectories) {735 auto FullFilePath = ASTReader::ResolveImportedPath(736 PathBuf, IFI.UnresolvedImportedFilename, MF->BaseDirectory);737 MD.IsInStableDirectories =738 isPathInStableDir(MDC.StableDirs, *FullFilePath);739 }740 if (!(IFI.TopLevel && IFI.ModuleMap))741 return;742 if (IFI.UnresolvedImportedFilenameAsRequested.ends_with(743 "__inferred_module.map"))744 return;745 auto ResolvedFilenameAsRequested = ASTReader::ResolveImportedPath(746 PathBuf, IFI.UnresolvedImportedFilenameAsRequested,747 MF->BaseDirectory);748 MD.ModuleMapFileDeps.emplace_back(*ResolvedFilenameAsRequested);749 });750 751 bool IgnoreCWD = false;752 CowCompilerInvocation CI =753 MDC.getInvocationAdjustedForModuleBuildWithoutOutputs(754 MD, [&](CowCompilerInvocation &BuildInvocation) {755 if (any(MDC.Service.getOptimizeArgs() &756 (ScanningOptimizations::HeaderSearch |757 ScanningOptimizations::VFS)))758 optimizeHeaderSearchOpts(BuildInvocation.getMutHeaderSearchOpts(),759 *MDC.ScanInstance.getASTReader(), *MF,760 MDC.PrebuiltModulesASTMap,761 MDC.Service.getOptimizeArgs());762 763 if (any(MDC.Service.getOptimizeArgs() &764 ScanningOptimizations::SystemWarnings))765 optimizeDiagnosticOpts(766 BuildInvocation.getMutDiagnosticOpts(),767 BuildInvocation.getFrontendOpts().IsSystemModule);768 769 IgnoreCWD = any(MDC.Service.getOptimizeArgs() &770 ScanningOptimizations::IgnoreCWD) &&771 isSafeToIgnoreCWD(BuildInvocation);772 if (IgnoreCWD) {773 llvm::ErrorOr<std::string> CWD =774 MDC.ScanInstance.getVirtualFileSystem()775 .getCurrentWorkingDirectory();776 if (CWD)777 optimizeCWD(BuildInvocation, *CWD);778 }779 });780 781 // Check provided input paths from the invocation for determining782 // IsInStableDirectories.783 if (MD.IsInStableDirectories)784 MD.IsInStableDirectories =785 areOptionsInStableDir(MDC.StableDirs, CI.getHeaderSearchOpts());786 787 MDC.associateWithContextHash(CI, IgnoreCWD, MD);788 789 // Finish the compiler invocation. Requires dependencies and the context hash.790 MDC.addOutputPaths(CI, MD);791 792 MD.BuildInfo = std::move(CI);793 794 MDC.ModularDeps.insert({M, std::move(OwnedMD)});795 796 return MD.ID;797}798 799static void forEachSubmoduleSorted(const Module *M,800 llvm::function_ref<void(const Module *)> F) {801 // Submodule order depends on order of header includes for inferred submodules802 // we don't care about the exact order, so sort so that it's consistent across803 // TUs to improve sharing.804 SmallVector<const Module *> Submodules(M->submodules());805 llvm::stable_sort(Submodules, [](const Module *A, const Module *B) {806 return A->Name < B->Name;807 });808 for (const Module *SubM : Submodules)809 F(SubM);810}811 812void ModuleDepCollectorPP::addAllSubmodulePrebuiltDeps(813 const Module *M, ModuleDeps &MD,814 llvm::DenseSet<const Module *> &SeenSubmodules) {815 addModulePrebuiltDeps(M, MD, SeenSubmodules);816 817 forEachSubmoduleSorted(M, [&](const Module *SubM) {818 addAllSubmodulePrebuiltDeps(SubM, MD, SeenSubmodules);819 });820}821 822void ModuleDepCollectorPP::addModulePrebuiltDeps(823 const Module *M, ModuleDeps &MD,824 llvm::DenseSet<const Module *> &SeenSubmodules) {825 for (const Module *Import : M->Imports)826 if (Import->getTopLevelModule() != M->getTopLevelModule())827 if (MDC.isPrebuiltModule(Import->getTopLevelModule()))828 if (SeenSubmodules.insert(Import->getTopLevelModule()).second) {829 MD.PrebuiltModuleDeps.emplace_back(Import->getTopLevelModule());830 if (MD.IsInStableDirectories) {831 auto PrebuiltModulePropIt = MDC.PrebuiltModulesASTMap.find(832 MD.PrebuiltModuleDeps.back().PCMFile);833 MD.IsInStableDirectories =834 (PrebuiltModulePropIt != MDC.PrebuiltModulesASTMap.end()) &&835 PrebuiltModulePropIt->second.isInStableDir();836 }837 }838}839 840void ModuleDepCollectorPP::addAllSubmoduleDeps(841 const Module *M, ModuleDeps &MD,842 llvm::DenseSet<const Module *> &AddedModules) {843 addModuleDep(M, MD, AddedModules);844 845 forEachSubmoduleSorted(M, [&](const Module *SubM) {846 addAllSubmoduleDeps(SubM, MD, AddedModules);847 });848}849 850void ModuleDepCollectorPP::addOneModuleDep(const Module *M, const ModuleID ID,851 ModuleDeps &MD) {852 MD.ClangModuleDeps.push_back(std::move(ID));853 if (MD.IsInStableDirectories)854 MD.IsInStableDirectories = MDC.ModularDeps[M]->IsInStableDirectories;855}856 857void ModuleDepCollectorPP::addModuleDep(858 const Module *M, ModuleDeps &MD,859 llvm::DenseSet<const Module *> &AddedModules) {860 for (const Module *Import : M->Imports) {861 if (Import->getTopLevelModule() != M->getTopLevelModule() &&862 !MDC.isPrebuiltModule(Import)) {863 if (auto ImportID = handleTopLevelModule(Import->getTopLevelModule()))864 if (AddedModules.insert(Import->getTopLevelModule()).second)865 addOneModuleDep(Import->getTopLevelModule(), *ImportID, MD);866 }867 }868}869 870void ModuleDepCollectorPP::addAllAffectingClangModules(871 const Module *M, ModuleDeps &MD,872 llvm::DenseSet<const Module *> &AddedModules) {873 addAffectingClangModule(M, MD, AddedModules);874 875 for (const Module *SubM : M->submodules())876 addAllAffectingClangModules(SubM, MD, AddedModules);877}878 879void ModuleDepCollectorPP::addAffectingClangModule(880 const Module *M, ModuleDeps &MD,881 llvm::DenseSet<const Module *> &AddedModules) {882 for (const Module *Affecting : M->AffectingClangModules) {883 assert(Affecting == Affecting->getTopLevelModule() &&884 "Not quite import not top-level module");885 if (Affecting != M->getTopLevelModule() &&886 !MDC.isPrebuiltModule(Affecting)) {887 if (auto ImportID = handleTopLevelModule(Affecting))888 if (AddedModules.insert(Affecting).second)889 addOneModuleDep(Affecting, *ImportID, MD);890 }891 }892}893 894ModuleDepCollector::ModuleDepCollector(895 DependencyScanningService &Service,896 std::unique_ptr<DependencyOutputOptions> Opts,897 CompilerInstance &ScanInstance, DependencyConsumer &C,898 DependencyActionController &Controller, CompilerInvocation OriginalCI,899 const PrebuiltModulesAttrsMap PrebuiltModulesASTMap,900 const ArrayRef<StringRef> StableDirs)901 : Service(Service), ScanInstance(ScanInstance), Consumer(C),902 Controller(Controller),903 PrebuiltModulesASTMap(std::move(PrebuiltModulesASTMap)),904 StableDirs(StableDirs), Opts(std::move(Opts)),905 CommonInvocation(906 makeCommonInvocationForModuleBuild(std::move(OriginalCI))) {}907 908void ModuleDepCollector::attachToPreprocessor(Preprocessor &PP) {909 auto CollectorPP = std::make_unique<ModuleDepCollectorPP>(*this);910 CollectorPPPtr = CollectorPP.get();911 PP.addPPCallbacks(std::move(CollectorPP));912}913 914void ModuleDepCollector::attachToASTReader(ASTReader &R) {}915 916bool ModuleDepCollector::isPrebuiltModule(const Module *M) {917 std::string Name(M->getTopLevelModuleName());918 const auto &PrebuiltModuleFiles =919 ScanInstance.getHeaderSearchOpts().PrebuiltModuleFiles;920 auto PrebuiltModuleFileIt = PrebuiltModuleFiles.find(Name);921 if (PrebuiltModuleFileIt == PrebuiltModuleFiles.end())922 return false;923 assert("Prebuilt module came from the expected AST file" &&924 PrebuiltModuleFileIt->second == M->getASTFile()->getName());925 return true;926}927 928void ModuleDepCollector::addVisibleModules() {929 llvm::DenseSet<const Module *> ImportedModules;930 auto InsertVisibleModules = [&](const Module *M) {931 if (ImportedModules.contains(M))932 return;933 934 VisibleModules.insert(M->getTopLevelModuleName());935 SmallVector<Module *> Stack;936 M->getExportedModules(Stack);937 while (!Stack.empty()) {938 const Module *CurrModule = Stack.pop_back_val();939 if (ImportedModules.contains(CurrModule))940 continue;941 ImportedModules.insert(CurrModule);942 VisibleModules.insert(CurrModule->getTopLevelModuleName());943 CurrModule->getExportedModules(Stack);944 }945 };946 947 for (const Module *Import : DirectImports)948 InsertVisibleModules(Import);949}950 951static StringRef makeAbsoluteAndPreferred(CompilerInstance &CI, StringRef Path,952 SmallVectorImpl<char> &Storage) {953 if (llvm::sys::path::is_absolute(Path) &&954 !llvm::sys::path::is_style_windows(llvm::sys::path::Style::native))955 return Path;956 Storage.assign(Path.begin(), Path.end());957 CI.getFileManager().makeAbsolutePath(Storage);958 llvm::sys::path::make_preferred(Storage);959 return StringRef(Storage.data(), Storage.size());960}961 962void ModuleDepCollector::addFileDep(StringRef Path) {963 if (Service.getFormat() == ScanningOutputFormat::P1689) {964 // Within P1689 format, we don't want all the paths to be absolute path965 // since it may violate the traditional make style dependencies info.966 FileDeps.emplace_back(Path);967 return;968 }969 970 llvm::SmallString<256> Storage;971 Path = makeAbsoluteAndPreferred(ScanInstance, Path, Storage);972 FileDeps.emplace_back(Path);973}974 975void ModuleDepCollector::addFileDep(ModuleDeps &MD, StringRef Path) {976 MD.FileDeps.emplace_back(Path);977}978