504 lines · cpp
1//===- ModuleManager.cpp - Module Manager ---------------------------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// This file defines the ModuleManager class, which manages a set of loaded10// modules for the ASTReader.11//12//===----------------------------------------------------------------------===//13 14#include "clang/Serialization/ModuleManager.h"15#include "clang/Basic/FileManager.h"16#include "clang/Basic/LLVM.h"17#include "clang/Lex/HeaderSearch.h"18#include "clang/Lex/ModuleMap.h"19#include "clang/Serialization/GlobalModuleIndex.h"20#include "clang/Serialization/InMemoryModuleCache.h"21#include "clang/Serialization/ModuleCache.h"22#include "clang/Serialization/ModuleFile.h"23#include "clang/Serialization/PCHContainerOperations.h"24#include "llvm/ADT/STLExtras.h"25#include "llvm/ADT/SetVector.h"26#include "llvm/ADT/SmallPtrSet.h"27#include "llvm/ADT/SmallVector.h"28#include "llvm/ADT/StringRef.h"29#include "llvm/ADT/iterator.h"30#include "llvm/Support/DOTGraphTraits.h"31#include "llvm/Support/ErrorOr.h"32#include "llvm/Support/GraphWriter.h"33#include "llvm/Support/MemoryBuffer.h"34#include "llvm/Support/VirtualFileSystem.h"35#include <cassert>36#include <memory>37#include <string>38#include <system_error>39 40using namespace clang;41using namespace serialization;42 43ModuleFile *ModuleManager::lookupByFileName(StringRef Name) const {44 auto Entry = FileMgr.getOptionalFileRef(Name, /*OpenFile=*/false,45 /*CacheFailure=*/false);46 if (Entry)47 return lookup(*Entry);48 49 return nullptr;50}51 52ModuleFile *ModuleManager::lookupByModuleName(StringRef Name) const {53 if (const Module *Mod = HeaderSearchInfo.getModuleMap().findModule(Name))54 if (OptionalFileEntryRef File = Mod->getASTFile())55 return lookup(*File);56 57 return nullptr;58}59 60ModuleFile *ModuleManager::lookup(const FileEntry *File) const {61 return Modules.lookup(File);62}63 64std::unique_ptr<llvm::MemoryBuffer>65ModuleManager::lookupBuffer(StringRef Name) {66 auto Entry = FileMgr.getOptionalFileRef(Name, /*OpenFile=*/false,67 /*CacheFailure=*/false);68 if (!Entry)69 return nullptr;70 return std::move(InMemoryBuffers[*Entry]);71}72 73static bool checkSignature(ASTFileSignature Signature,74 ASTFileSignature ExpectedSignature,75 std::string &ErrorStr) {76 if (!ExpectedSignature || Signature == ExpectedSignature)77 return false;78 79 ErrorStr =80 Signature ? "signature mismatch" : "could not read module signature";81 return true;82}83 84static void updateModuleImports(ModuleFile &MF, ModuleFile *ImportedBy,85 SourceLocation ImportLoc) {86 if (ImportedBy) {87 MF.ImportedBy.insert(ImportedBy);88 ImportedBy->Imports.insert(&MF);89 } else {90 if (!MF.DirectlyImported)91 MF.ImportLoc = ImportLoc;92 93 MF.DirectlyImported = true;94 }95}96 97ModuleManager::AddModuleResult98ModuleManager::addModule(StringRef FileName, ModuleKind Type,99 SourceLocation ImportLoc, ModuleFile *ImportedBy,100 unsigned Generation,101 off_t ExpectedSize, time_t ExpectedModTime,102 ASTFileSignature ExpectedSignature,103 ASTFileSignatureReader ReadSignature,104 ModuleFile *&Module,105 std::string &ErrorStr) {106 Module = nullptr;107 108 // Look for the file entry. This only fails if the expected size or109 // modification time differ.110 OptionalFileEntryRef Entry;111 bool IgnoreModTime = Type == MK_ExplicitModule || Type == MK_PrebuiltModule;112 if (ImportedBy)113 IgnoreModTime &= ImportedBy->Kind == MK_ExplicitModule ||114 ImportedBy->Kind == MK_PrebuiltModule;115 if (IgnoreModTime) {116 // If neither this file nor the importer are in the module cache, this file117 // might have a different mtime due to being moved across filesystems in118 // a distributed build. The size must still match, though. (As must the119 // contents, but we can't check that.)120 ExpectedModTime = 0;121 }122 // Note: ExpectedSize and ExpectedModTime will be 0 for MK_ImplicitModule123 // when using an ASTFileSignature.124 if (lookupModuleFile(FileName, ExpectedSize, ExpectedModTime, Entry)) {125 ErrorStr = IgnoreModTime ? "module file has a different size than expected"126 : "module file has a different size or "127 "modification time than expected";128 return OutOfDate;129 }130 131 if (!Entry) {132 ErrorStr = "module file not found";133 return Missing;134 }135 136 // The ModuleManager's use of FileEntry nodes as the keys for its map of137 // loaded modules is less than ideal. Uniqueness for FileEntry nodes is138 // maintained by FileManager, which in turn uses inode numbers on hosts139 // that support that. When coupled with the module cache's proclivity for140 // turning over and deleting stale PCMs, this means entries for different141 // module files can wind up reusing the same underlying inode. When this142 // happens, subsequent accesses to the Modules map will disagree on the143 // ModuleFile associated with a given file. In general, it is not sufficient144 // to resolve this conundrum with a type like FileEntryRef that stores the145 // name of the FileEntry node on first access because of path canonicalization146 // issues. However, the paths constructed for implicit module builds are147 // fully under Clang's control. We *can*, therefore, rely on their structure148 // being consistent across operating systems and across subsequent accesses149 // to the Modules map.150 auto implicitModuleNamesMatch = [](ModuleKind Kind, const ModuleFile *MF,151 FileEntryRef Entry) -> bool {152 if (Kind != MK_ImplicitModule)153 return true;154 return Entry.getName() == MF->FileName;155 };156 157 // Check whether we already loaded this module, before158 if (ModuleFile *ModuleEntry = Modules.lookup(*Entry)) {159 if (implicitModuleNamesMatch(Type, ModuleEntry, *Entry)) {160 // Check the stored signature.161 if (checkSignature(ModuleEntry->Signature, ExpectedSignature, ErrorStr))162 return OutOfDate;163 164 Module = ModuleEntry;165 updateModuleImports(*ModuleEntry, ImportedBy, ImportLoc);166 return AlreadyLoaded;167 }168 }169 170 // Allocate a new module.171 auto NewModule = std::make_unique<ModuleFile>(Type, *Entry, Generation);172 NewModule->Index = Chain.size();173 NewModule->FileName = FileName.str();174 NewModule->ImportLoc = ImportLoc;175 NewModule->InputFilesValidationTimestamp = 0;176 177 if (NewModule->Kind == MK_ImplicitModule)178 NewModule->InputFilesValidationTimestamp =179 ModCache->getModuleTimestamp(NewModule->FileName);180 181 // Load the contents of the module182 if (std::unique_ptr<llvm::MemoryBuffer> Buffer = lookupBuffer(FileName)) {183 // The buffer was already provided for us.184 NewModule->Buffer = &getModuleCache().getInMemoryModuleCache().addBuiltPCM(185 FileName, std::move(Buffer));186 // Since the cached buffer is reused, it is safe to close the file187 // descriptor that was opened while stat()ing the PCM in188 // lookupModuleFile() above, it won't be needed any longer.189 Entry->closeFile();190 } else if (llvm::MemoryBuffer *Buffer =191 getModuleCache().getInMemoryModuleCache().lookupPCM(192 FileName)) {193 NewModule->Buffer = Buffer;194 // As above, the file descriptor is no longer needed.195 Entry->closeFile();196 } else if (getModuleCache().getInMemoryModuleCache().shouldBuildPCM(197 FileName)) {198 // Report that the module is out of date, since we tried (and failed) to199 // import it earlier.200 Entry->closeFile();201 return OutOfDate;202 } else {203 // Get a buffer of the file and close the file descriptor when done.204 // The file is volatile because in a parallel build we expect multiple205 // compiler processes to use the same module file rebuilding it if needed.206 //207 // RequiresNullTerminator is false because module files don't need it, and208 // this allows the file to still be mmapped.209 auto Buf = FileMgr.getBufferForFile(NewModule->File,210 /*IsVolatile=*/true,211 /*RequiresNullTerminator=*/false);212 213 if (!Buf) {214 ErrorStr = Buf.getError().message();215 return Missing;216 }217 218 NewModule->Buffer = &getModuleCache().getInMemoryModuleCache().addPCM(219 FileName, std::move(*Buf));220 }221 222 // Initialize the stream.223 NewModule->Data = PCHContainerRdr.ExtractPCH(*NewModule->Buffer);224 225 // Read the signature eagerly now so that we can check it. Avoid calling226 // ReadSignature unless there's something to check though.227 if (ExpectedSignature && checkSignature(ReadSignature(NewModule->Data),228 ExpectedSignature, ErrorStr))229 return OutOfDate;230 231 // We're keeping this module. Store it everywhere.232 Module = Modules[*Entry] = NewModule.get();233 234 updateModuleImports(*NewModule, ImportedBy, ImportLoc);235 236 if (!NewModule->isModule())237 PCHChain.push_back(NewModule.get());238 if (!ImportedBy)239 Roots.push_back(NewModule.get());240 241 Chain.push_back(std::move(NewModule));242 return NewlyLoaded;243}244 245void ModuleManager::removeModules(ModuleIterator First) {246 auto Last = end();247 if (First == Last)248 return;249 250 // Explicitly clear VisitOrder since we might not notice it is stale.251 VisitOrder.clear();252 253 // Collect the set of module file pointers that we'll be removing.254 llvm::SmallPtrSet<ModuleFile *, 4> victimSet(255 (llvm::pointer_iterator<ModuleIterator>(First)),256 (llvm::pointer_iterator<ModuleIterator>(Last)));257 258 auto IsVictim = [&](ModuleFile *MF) {259 return victimSet.count(MF);260 };261 // Remove any references to the now-destroyed modules.262 for (auto I = begin(); I != First; ++I) {263 I->Imports.remove_if(IsVictim);264 I->ImportedBy.remove_if(IsVictim);265 }266 llvm::erase_if(Roots, IsVictim);267 268 // Remove the modules from the PCH chain.269 for (auto I = First; I != Last; ++I) {270 if (!I->isModule()) {271 PCHChain.erase(llvm::find(PCHChain, &*I), PCHChain.end());272 break;273 }274 }275 276 // Delete the modules.277 for (ModuleIterator victim = First; victim != Last; ++victim)278 Modules.erase(victim->File);279 280 Chain.erase(Chain.begin() + (First - begin()), Chain.end());281}282 283void284ModuleManager::addInMemoryBuffer(StringRef FileName,285 std::unique_ptr<llvm::MemoryBuffer> Buffer) {286 FileEntryRef Entry =287 FileMgr.getVirtualFileRef(FileName, Buffer->getBufferSize(), 0);288 InMemoryBuffers[Entry] = std::move(Buffer);289}290 291std::unique_ptr<ModuleManager::VisitState> ModuleManager::allocateVisitState() {292 // Fast path: if we have a cached state, use it.293 if (FirstVisitState) {294 auto Result = std::move(FirstVisitState);295 FirstVisitState = std::move(Result->NextState);296 return Result;297 }298 299 // Allocate and return a new state.300 return std::make_unique<VisitState>(size());301}302 303void ModuleManager::returnVisitState(std::unique_ptr<VisitState> State) {304 assert(State->NextState == nullptr && "Visited state is in list?");305 State->NextState = std::move(FirstVisitState);306 FirstVisitState = std::move(State);307}308 309void ModuleManager::setGlobalIndex(GlobalModuleIndex *Index) {310 GlobalIndex = Index;311 if (!GlobalIndex) {312 ModulesInCommonWithGlobalIndex.clear();313 return;314 }315 316 // Notify the global module index about all of the modules we've already317 // loaded.318 for (ModuleFile &M : *this)319 if (!GlobalIndex->loadedModuleFile(&M))320 ModulesInCommonWithGlobalIndex.push_back(&M);321}322 323void ModuleManager::moduleFileAccepted(ModuleFile *MF) {324 if (!GlobalIndex || GlobalIndex->loadedModuleFile(MF))325 return;326 327 ModulesInCommonWithGlobalIndex.push_back(MF);328}329 330ModuleManager::ModuleManager(FileManager &FileMgr, ModuleCache &ModCache,331 const PCHContainerReader &PCHContainerRdr,332 const HeaderSearch &HeaderSearchInfo)333 : FileMgr(FileMgr), ModCache(&ModCache), PCHContainerRdr(PCHContainerRdr),334 HeaderSearchInfo(HeaderSearchInfo) {}335 336void ModuleManager::visit(llvm::function_ref<bool(ModuleFile &M)> Visitor,337 llvm::SmallPtrSetImpl<ModuleFile *> *ModuleFilesHit) {338 // If the visitation order vector is the wrong size, recompute the order.339 if (VisitOrder.size() != Chain.size()) {340 unsigned N = size();341 VisitOrder.clear();342 VisitOrder.reserve(N);343 344 // Record the number of incoming edges for each module. When we345 // encounter a module with no incoming edges, push it into the queue346 // to seed the queue.347 SmallVector<ModuleFile *, 4> Queue;348 Queue.reserve(N);349 llvm::SmallVector<unsigned, 4> UnusedIncomingEdges;350 UnusedIncomingEdges.resize(size());351 for (ModuleFile &M : llvm::reverse(*this)) {352 unsigned Size = M.ImportedBy.size();353 UnusedIncomingEdges[M.Index] = Size;354 if (!Size)355 Queue.push_back(&M);356 }357 358 // Traverse the graph, making sure to visit a module before visiting any359 // of its dependencies.360 while (!Queue.empty()) {361 ModuleFile *CurrentModule = Queue.pop_back_val();362 VisitOrder.push_back(CurrentModule);363 364 // For any module that this module depends on, push it on the365 // stack (if it hasn't already been marked as visited).366 for (ModuleFile *M : llvm::reverse(CurrentModule->Imports)) {367 // Remove our current module as an impediment to visiting the368 // module we depend on. If we were the last unvisited module369 // that depends on this particular module, push it into the370 // queue to be visited.371 unsigned &NumUnusedEdges = UnusedIncomingEdges[M->Index];372 if (NumUnusedEdges && (--NumUnusedEdges == 0))373 Queue.push_back(M);374 }375 }376 377 assert(VisitOrder.size() == N && "Visitation order is wrong?");378 379 FirstVisitState = nullptr;380 }381 382 auto State = allocateVisitState();383 unsigned VisitNumber = State->NextVisitNumber++;384 385 // If the caller has provided us with a hit-set that came from the global386 // module index, mark every module file in common with the global module387 // index that is *not* in that set as 'visited'.388 if (ModuleFilesHit && !ModulesInCommonWithGlobalIndex.empty()) {389 for (unsigned I = 0, N = ModulesInCommonWithGlobalIndex.size(); I != N; ++I)390 {391 ModuleFile *M = ModulesInCommonWithGlobalIndex[I];392 if (!ModuleFilesHit->count(M))393 State->VisitNumber[M->Index] = VisitNumber;394 }395 }396 397 for (unsigned I = 0, N = VisitOrder.size(); I != N; ++I) {398 ModuleFile *CurrentModule = VisitOrder[I];399 // Should we skip this module file?400 if (State->VisitNumber[CurrentModule->Index] == VisitNumber)401 continue;402 403 // Visit the module.404 assert(State->VisitNumber[CurrentModule->Index] == VisitNumber - 1);405 State->VisitNumber[CurrentModule->Index] = VisitNumber;406 if (!Visitor(*CurrentModule))407 continue;408 409 // The visitor has requested that cut off visitation of any410 // module that the current module depends on. To indicate this411 // behavior, we mark all of the reachable modules as having been visited.412 ModuleFile *NextModule = CurrentModule;413 do {414 // For any module that this module depends on, push it on the415 // stack (if it hasn't already been marked as visited).416 for (llvm::SetVector<ModuleFile *>::iterator417 M = NextModule->Imports.begin(),418 MEnd = NextModule->Imports.end();419 M != MEnd; ++M) {420 if (State->VisitNumber[(*M)->Index] != VisitNumber) {421 State->Stack.push_back(*M);422 State->VisitNumber[(*M)->Index] = VisitNumber;423 }424 }425 426 if (State->Stack.empty())427 break;428 429 // Pop the next module off the stack.430 NextModule = State->Stack.pop_back_val();431 } while (true);432 }433 434 returnVisitState(std::move(State));435}436 437bool ModuleManager::lookupModuleFile(StringRef FileName, off_t ExpectedSize,438 time_t ExpectedModTime,439 OptionalFileEntryRef &File) {440 if (FileName == "-") {441 File = expectedToOptional(FileMgr.getSTDIN());442 return false;443 }444 445 // Open the file immediately to ensure there is no race between stat'ing and446 // opening the file.447 File = FileMgr.getOptionalFileRef(FileName, /*OpenFile=*/true,448 /*CacheFailure=*/false);449 450 if (File &&451 ((ExpectedSize && ExpectedSize != File->getSize()) ||452 (ExpectedModTime && ExpectedModTime != File->getModificationTime())))453 // Do not destroy File, as it may be referenced. If we need to rebuild it,454 // it will be destroyed by removeModules.455 return true;456 457 return false;458}459 460#ifndef NDEBUG461namespace llvm {462 463 template<>464 struct GraphTraits<ModuleManager> {465 using NodeRef = ModuleFile *;466 using ChildIteratorType = llvm::SetVector<ModuleFile *>::const_iterator;467 using nodes_iterator = pointer_iterator<ModuleManager::ModuleConstIterator>;468 469 static ChildIteratorType child_begin(NodeRef Node) {470 return Node->Imports.begin();471 }472 473 static ChildIteratorType child_end(NodeRef Node) {474 return Node->Imports.end();475 }476 477 static nodes_iterator nodes_begin(const ModuleManager &Manager) {478 return nodes_iterator(Manager.begin());479 }480 481 static nodes_iterator nodes_end(const ModuleManager &Manager) {482 return nodes_iterator(Manager.end());483 }484 };485 486 template<>487 struct DOTGraphTraits<ModuleManager> : public DefaultDOTGraphTraits {488 explicit DOTGraphTraits(bool IsSimple = false)489 : DefaultDOTGraphTraits(IsSimple) {}490 491 static bool renderGraphFromBottomUp() { return true; }492 493 std::string getNodeLabel(ModuleFile *M, const ModuleManager&) {494 return M->ModuleName;495 }496 };497 498} // namespace llvm499 500void ModuleManager::viewGraph() {501 llvm::ViewGraph(*this, "Modules");502}503#endif504