1229 lines · cpp
1//===- ClangScanDeps.cpp - Implementation of clang-scan-deps --------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8 9#include "clang/Driver/Compilation.h"10#include "clang/Driver/Driver.h"11#include "clang/Frontend/CompilerInstance.h"12#include "clang/Frontend/TextDiagnosticPrinter.h"13#include "clang/Tooling/CommonOptionsParser.h"14#include "clang/Tooling/DependencyScanning/DependencyScanningService.h"15#include "clang/Tooling/DependencyScanning/DependencyScanningTool.h"16#include "clang/Tooling/DependencyScanning/DependencyScanningWorker.h"17#include "clang/Tooling/JSONCompilationDatabase.h"18#include "clang/Tooling/Tooling.h"19#include "llvm/ADT/STLExtras.h"20#include "llvm/ADT/Twine.h"21#include "llvm/Support/CommandLine.h"22#include "llvm/Support/FileUtilities.h"23#include "llvm/Support/Format.h"24#include "llvm/Support/JSON.h"25#include "llvm/Support/LLVMDriver.h"26#include "llvm/Support/MemoryBuffer.h"27#include "llvm/Support/Program.h"28#include "llvm/Support/Signals.h"29#include "llvm/Support/TargetSelect.h"30#include "llvm/Support/ThreadPool.h"31#include "llvm/Support/Threading.h"32#include "llvm/Support/Timer.h"33#include "llvm/Support/VirtualFileSystem.h"34#include "llvm/TargetParser/Host.h"35#include <memory>36#include <mutex>37#include <optional>38#include <thread>39 40#include "Opts.inc"41 42using namespace clang;43using namespace tooling::dependencies;44 45namespace {46 47using namespace llvm::opt;48enum ID {49 OPT_INVALID = 0, // This is not an option ID.50#define OPTION(...) LLVM_MAKE_OPT_ID(__VA_ARGS__),51#include "Opts.inc"52#undef OPTION53};54 55#define OPTTABLE_STR_TABLE_CODE56#include "Opts.inc"57#undef OPTTABLE_STR_TABLE_CODE58 59#define OPTTABLE_PREFIXES_TABLE_CODE60#include "Opts.inc"61#undef OPTTABLE_PREFIXES_TABLE_CODE62 63const llvm::opt::OptTable::Info InfoTable[] = {64#define OPTION(...) LLVM_CONSTRUCT_OPT_INFO(__VA_ARGS__),65#include "Opts.inc"66#undef OPTION67};68 69class ScanDepsOptTable : public llvm::opt::GenericOptTable {70public:71 ScanDepsOptTable()72 : GenericOptTable(OptionStrTable, OptionPrefixesTable, InfoTable) {73 setGroupedShortOptions(true);74 }75};76 77enum ResourceDirRecipeKind {78 RDRK_ModifyCompilerPath,79 RDRK_InvokeCompiler,80};81 82static std::string OutputFileName = "-";83static ScanningMode ScanMode = ScanningMode::DependencyDirectivesScan;84static ScanningOutputFormat Format = ScanningOutputFormat::Make;85static ScanningOptimizations OptimizeArgs;86static std::string ModuleFilesDir;87static bool EagerLoadModules;88static unsigned NumThreads = 0;89static std::string CompilationDB;90static std::optional<std::string> ModuleNames;91static std::vector<std::string> ModuleDepTargets;92static std::string TranslationUnitFile;93static bool DeprecatedDriverCommand;94static ResourceDirRecipeKind ResourceDirRecipe;95static bool Verbose;96static bool PrintTiming;97static bool EmitVisibleModules;98static llvm::BumpPtrAllocator Alloc;99static llvm::StringSaver Saver{Alloc};100static std::vector<const char *> CommandLine;101 102#ifndef NDEBUG103static constexpr bool DoRoundTripDefault = true;104#else105static constexpr bool DoRoundTripDefault = false;106#endif107 108static bool RoundTripArgs = DoRoundTripDefault;109static bool VerbatimArgs = false;110 111static void ParseArgs(int argc, char **argv) {112 ScanDepsOptTable Tbl;113 llvm::StringRef ToolName = argv[0];114 llvm::opt::InputArgList Args =115 Tbl.parseArgs(argc, argv, OPT_UNKNOWN, Saver, [&](StringRef Msg) {116 llvm::errs() << Msg << '\n';117 std::exit(1);118 });119 120 if (Args.hasArg(OPT_help)) {121 Tbl.printHelp(llvm::outs(), "clang-scan-deps [options]", "clang-scan-deps");122 std::exit(0);123 }124 if (Args.hasArg(OPT_version)) {125 llvm::outs() << ToolName << '\n';126 llvm::cl::PrintVersionMessage();127 std::exit(0);128 }129 if (const llvm::opt::Arg *A = Args.getLastArg(OPT_mode_EQ)) {130 auto ModeType =131 llvm::StringSwitch<std::optional<ScanningMode>>(A->getValue())132 .Case("preprocess-dependency-directives",133 ScanningMode::DependencyDirectivesScan)134 .Case("preprocess", ScanningMode::CanonicalPreprocessing)135 .Default(std::nullopt);136 if (!ModeType) {137 llvm::errs() << ToolName138 << ": for the --mode option: Cannot find option named '"139 << A->getValue() << "'\n";140 std::exit(1);141 }142 ScanMode = *ModeType;143 }144 145 if (const llvm::opt::Arg *A = Args.getLastArg(OPT_format_EQ)) {146 auto FormatType =147 llvm::StringSwitch<std::optional<ScanningOutputFormat>>(A->getValue())148 .Case("make", ScanningOutputFormat::Make)149 .Case("p1689", ScanningOutputFormat::P1689)150 .Case("experimental-full", ScanningOutputFormat::Full)151 .Default(std::nullopt);152 if (!FormatType) {153 llvm::errs() << ToolName154 << ": for the --format option: Cannot find option named '"155 << A->getValue() << "'\n";156 std::exit(1);157 }158 Format = *FormatType;159 }160 161 std::vector<std::string> OptimizationFlags =162 Args.getAllArgValues(OPT_optimize_args_EQ);163 OptimizeArgs = ScanningOptimizations::None;164 for (const auto &Arg : OptimizationFlags) {165 auto Optimization =166 llvm::StringSwitch<std::optional<ScanningOptimizations>>(Arg)167 .Case("none", ScanningOptimizations::None)168 .Case("header-search", ScanningOptimizations::HeaderSearch)169 .Case("system-warnings", ScanningOptimizations::SystemWarnings)170 .Case("vfs", ScanningOptimizations::VFS)171 .Case("canonicalize-macros", ScanningOptimizations::Macros)172 .Case("ignore-current-working-dir",173 ScanningOptimizations::IgnoreCWD)174 .Case("all", ScanningOptimizations::All)175 .Default(std::nullopt);176 if (!Optimization) {177 llvm::errs()178 << ToolName179 << ": for the --optimize-args option: Cannot find option named '"180 << Arg << "'\n";181 std::exit(1);182 }183 OptimizeArgs |= *Optimization;184 }185 if (OptimizationFlags.empty())186 OptimizeArgs = ScanningOptimizations::Default;187 188 if (const llvm::opt::Arg *A = Args.getLastArg(OPT_module_files_dir_EQ))189 ModuleFilesDir = A->getValue();190 191 if (const llvm::opt::Arg *A = Args.getLastArg(OPT_o))192 OutputFileName = A->getValue();193 194 EagerLoadModules = Args.hasArg(OPT_eager_load_pcm);195 196 if (const llvm::opt::Arg *A = Args.getLastArg(OPT_j)) {197 StringRef S{A->getValue()};198 if (!llvm::to_integer(S, NumThreads, 0)) {199 llvm::errs() << ToolName << ": for the -j option: '" << S200 << "' value invalid for uint argument!\n";201 std::exit(1);202 }203 }204 205 if (const llvm::opt::Arg *A = Args.getLastArg(OPT_compilation_database_EQ))206 CompilationDB = A->getValue();207 208 if (const llvm::opt::Arg *A = Args.getLastArg(OPT_module_names_EQ))209 ModuleNames = A->getValue();210 211 for (const llvm::opt::Arg *A : Args.filtered(OPT_dependency_target_EQ))212 ModuleDepTargets.emplace_back(A->getValue());213 214 if (const llvm::opt::Arg *A = Args.getLastArg(OPT_tu_buffer_path_EQ))215 TranslationUnitFile = A->getValue();216 217 DeprecatedDriverCommand = Args.hasArg(OPT_deprecated_driver_command);218 219 if (const llvm::opt::Arg *A = Args.getLastArg(OPT_resource_dir_recipe_EQ)) {220 auto Kind =221 llvm::StringSwitch<std::optional<ResourceDirRecipeKind>>(A->getValue())222 .Case("modify-compiler-path", RDRK_ModifyCompilerPath)223 .Case("invoke-compiler", RDRK_InvokeCompiler)224 .Default(std::nullopt);225 if (!Kind) {226 llvm::errs() << ToolName227 << ": for the --resource-dir-recipe option: Cannot find "228 "option named '"229 << A->getValue() << "'\n";230 std::exit(1);231 }232 ResourceDirRecipe = *Kind;233 }234 235 PrintTiming = Args.hasArg(OPT_print_timing);236 237 EmitVisibleModules = Args.hasArg(OPT_emit_visible_modules);238 239 Verbose = Args.hasArg(OPT_verbose);240 241 RoundTripArgs = Args.hasArg(OPT_round_trip_args);242 243 VerbatimArgs = Args.hasArg(OPT_verbatim_args);244 245 if (const llvm::opt::Arg *A = Args.getLastArgNoClaim(OPT_DASH_DASH))246 CommandLine.assign(A->getValues().begin(), A->getValues().end());247}248 249class SharedStream {250public:251 SharedStream(raw_ostream &OS) : OS(OS) {}252 void applyLocked(llvm::function_ref<void(raw_ostream &OS)> Fn) {253 std::unique_lock<std::mutex> LockGuard(Lock);254 Fn(OS);255 OS.flush();256 }257 258private:259 std::mutex Lock;260 raw_ostream &OS;261};262 263class ResourceDirectoryCache {264public:265 /// findResourceDir finds the resource directory relative to the clang266 /// compiler being used in Args, by running it with "-print-resource-dir"267 /// option and cache the results for reuse. \returns resource directory path268 /// associated with the given invocation command or empty string if the269 /// compiler path is NOT an absolute path.270 StringRef findResourceDir(const tooling::CommandLineArguments &Args,271 bool ClangCLMode) {272 if (Args.size() < 1)273 return "";274 275 const std::string &ClangBinaryPath = Args[0];276 if (!llvm::sys::path::is_absolute(ClangBinaryPath))277 return "";278 279 const std::string &ClangBinaryName =280 std::string(llvm::sys::path::filename(ClangBinaryPath));281 282 std::unique_lock<std::mutex> LockGuard(CacheLock);283 const auto &CachedResourceDir = Cache.find(ClangBinaryPath);284 if (CachedResourceDir != Cache.end())285 return CachedResourceDir->second;286 287 std::vector<StringRef> PrintResourceDirArgs{ClangBinaryName};288 if (ClangCLMode)289 PrintResourceDirArgs.push_back("/clang:-print-resource-dir");290 else291 PrintResourceDirArgs.push_back("-print-resource-dir");292 293 llvm::SmallString<64> OutputFile, ErrorFile;294 llvm::sys::fs::createTemporaryFile("print-resource-dir-output",295 "" /*no-suffix*/, OutputFile);296 llvm::sys::fs::createTemporaryFile("print-resource-dir-error",297 "" /*no-suffix*/, ErrorFile);298 llvm::FileRemover OutputRemover(OutputFile.c_str());299 llvm::FileRemover ErrorRemover(ErrorFile.c_str());300 std::optional<StringRef> Redirects[] = {301 {""}, // Stdin302 OutputFile.str(),303 ErrorFile.str(),304 };305 if (llvm::sys::ExecuteAndWait(ClangBinaryPath, PrintResourceDirArgs, {},306 Redirects)) {307 auto ErrorBuf =308 llvm::MemoryBuffer::getFile(ErrorFile.c_str(), /*IsText=*/true);309 llvm::errs() << ErrorBuf.get()->getBuffer();310 return "";311 }312 313 auto OutputBuf =314 llvm::MemoryBuffer::getFile(OutputFile.c_str(), /*IsText=*/true);315 if (!OutputBuf)316 return "";317 StringRef Output = OutputBuf.get()->getBuffer().rtrim('\n');318 319 return Cache[ClangBinaryPath] = Output.str();320 }321 322private:323 std::map<std::string, std::string> Cache;324 std::mutex CacheLock;325};326 327} // end anonymous namespace328 329/// Takes the result of a dependency scan and prints error / dependency files330/// based on the result.331///332/// \returns True on error.333static bool334handleMakeDependencyToolResult(const std::string &Input,335 llvm::Expected<std::string> &MaybeFile,336 SharedStream &OS, SharedStream &Errs) {337 if (!MaybeFile) {338 llvm::handleAllErrors(339 MaybeFile.takeError(), [&Input, &Errs](llvm::StringError &Err) {340 Errs.applyLocked([&](raw_ostream &OS) {341 OS << "Error while scanning dependencies for " << Input << ":\n";342 OS << Err.getMessage();343 });344 });345 return true;346 }347 OS.applyLocked([&](raw_ostream &OS) { OS << *MaybeFile; });348 return false;349}350 351template <typename Container>352static auto toJSONStrings(llvm::json::OStream &JOS, Container &&Strings) {353 return [&JOS, Strings = std::forward<Container>(Strings)] {354 for (StringRef Str : Strings)355 // Not reporting SDKSettings.json so that test checks can remain (mostly)356 // platform-agnostic.357 if (!Str.ends_with("SDKSettings.json"))358 JOS.value(Str);359 };360}361 362// Technically, we don't need to sort the dependency list to get determinism.363// Leaving these be will simply preserve the import order.364static auto toJSONSorted(llvm::json::OStream &JOS, std::vector<ModuleID> V) {365 llvm::sort(V);366 return [&JOS, V = std::move(V)] {367 for (const ModuleID &MID : V)368 JOS.object([&] {369 JOS.attribute("context-hash", StringRef(MID.ContextHash));370 JOS.attribute("module-name", StringRef(MID.ModuleName));371 });372 };373}374 375static auto toJSONSorted(llvm::json::OStream &JOS,376 SmallVector<Module::LinkLibrary, 2> LinkLibs) {377 llvm::sort(LinkLibs, [](const auto &LHS, const auto &RHS) {378 return LHS.Library < RHS.Library;379 });380 return [&JOS, LinkLibs = std::move(LinkLibs)] {381 for (const auto &LL : LinkLibs)382 JOS.object([&] {383 JOS.attribute("isFramework", LL.IsFramework);384 JOS.attribute("link-name", StringRef(LL.Library));385 });386 };387}388 389static auto toJSONSorted(llvm::json::OStream &JOS, std::vector<std::string> V) {390 llvm::sort(V);391 return [&JOS, V = std::move(V)] {392 for (const StringRef Entry : V)393 JOS.value(Entry);394 };395}396 397// Thread safe.398class FullDeps {399public:400 FullDeps(size_t NumInputs) : Inputs(NumInputs) {}401 402 void mergeDeps(StringRef Input, TranslationUnitDeps TUDeps,403 size_t InputIndex) {404 mergeDeps(std::move(TUDeps.ModuleGraph), InputIndex);405 406 InputDeps ID;407 ID.FileName = std::string(Input);408 ID.ContextHash = std::move(TUDeps.ID.ContextHash);409 ID.FileDeps = std::move(TUDeps.FileDeps);410 ID.NamedModule = std::move(TUDeps.ID.ModuleName);411 ID.NamedModuleDeps = std::move(TUDeps.NamedModuleDeps);412 ID.ClangModuleDeps = std::move(TUDeps.ClangModuleDeps);413 ID.VisibleModules = std::move(TUDeps.VisibleModules);414 ID.DriverCommandLine = std::move(TUDeps.DriverCommandLine);415 ID.Commands = std::move(TUDeps.Commands);416 417 assert(InputIndex < Inputs.size() && "Input index out of bounds");418 assert(Inputs[InputIndex].FileName.empty() && "Result already populated");419 Inputs[InputIndex] = std::move(ID);420 }421 422 void mergeDeps(ModuleDepsGraph Graph, size_t InputIndex) {423 std::vector<ModuleDeps *> NewMDs;424 {425 std::unique_lock<std::mutex> ul(Lock);426 for (ModuleDeps &MD : Graph) {427 auto I = Modules.find({MD.ID, 0});428 if (I != Modules.end()) {429 I->first.InputIndex = std::min(I->first.InputIndex, InputIndex);430 continue;431 }432 auto Res = Modules.insert(I, {{MD.ID, InputIndex}, std::move(MD)});433 NewMDs.push_back(&Res->second);434 }435 }436 // First call to \c getBuildArguments is somewhat expensive. Let's call it437 // on the current thread (instead of the main one), and outside the438 // critical section.439 for (ModuleDeps *MD : NewMDs)440 (void)MD->getBuildArguments();441 }442 443 bool roundTripCommand(ArrayRef<std::string> ArgStrs,444 DiagnosticsEngine &Diags) {445 if (ArgStrs.empty() || ArgStrs[0] != "-cc1")446 return false;447 SmallVector<const char *> Args;448 for (const std::string &Arg : ArgStrs)449 Args.push_back(Arg.c_str());450 return !CompilerInvocation::checkCC1RoundTrip(Args, Diags);451 }452 453 // Returns \c true if any command lines fail to round-trip. We expect454 // commands already be canonical when output by the scanner.455 bool roundTripCommands(raw_ostream &ErrOS) {456 DiagnosticOptions DiagOpts;457 TextDiagnosticPrinter DiagConsumer(ErrOS, DiagOpts);458 IntrusiveRefCntPtr<DiagnosticsEngine> Diags =459 CompilerInstance::createDiagnostics(*llvm::vfs::getRealFileSystem(),460 DiagOpts, &DiagConsumer,461 /*ShouldOwnClient=*/false);462 463 for (auto &&M : Modules)464 if (roundTripCommand(M.second.getBuildArguments(), *Diags))465 return true;466 467 for (auto &&I : Inputs)468 for (const auto &Cmd : I.Commands)469 if (roundTripCommand(Cmd.Arguments, *Diags))470 return true;471 472 return false;473 }474 475 void printFullOutput(raw_ostream &OS) {476 // Skip sorting modules and constructing the JSON object if the output477 // cannot be observed anyway. This makes timings less noisy.478 if (&OS == &llvm::nulls())479 return;480 481 // Sort the modules by name to get a deterministic order.482 std::vector<IndexedModuleID> ModuleIDs;483 for (auto &&M : Modules)484 ModuleIDs.push_back(M.first);485 llvm::sort(ModuleIDs);486 487 llvm::json::OStream JOS(OS, /*IndentSize=*/2);488 489 JOS.object([&] {490 JOS.attributeArray("modules", [&] {491 for (auto &&ModID : ModuleIDs) {492 auto &MD = Modules[ModID];493 JOS.object([&] {494 if (MD.IsInStableDirectories)495 JOS.attribute("is-in-stable-directories",496 MD.IsInStableDirectories);497 JOS.attributeArray("clang-module-deps",498 toJSONSorted(JOS, MD.ClangModuleDeps));499 JOS.attribute("clang-modulemap-file",500 StringRef(MD.ClangModuleMapFile));501 JOS.attributeArray("command-line",502 toJSONStrings(JOS, MD.getBuildArguments()));503 JOS.attribute("context-hash", StringRef(MD.ID.ContextHash));504 JOS.attributeArray("file-deps", [&] {505 MD.forEachFileDep([&](StringRef FileDep) {506 // Not reporting SDKSettings.json so that test checks can remain507 // (mostly) platform-agnostic.508 if (!FileDep.ends_with("SDKSettings.json"))509 JOS.value(FileDep);510 });511 });512 JOS.attributeArray("link-libraries",513 toJSONSorted(JOS, MD.LinkLibraries));514 JOS.attribute("name", StringRef(MD.ID.ModuleName));515 });516 }517 });518 519 JOS.attributeArray("translation-units", [&] {520 for (auto &&I : Inputs) {521 JOS.object([&] {522 JOS.attributeArray("commands", [&] {523 if (I.DriverCommandLine.empty()) {524 for (const auto &Cmd : I.Commands) {525 JOS.object([&] {526 JOS.attribute("clang-context-hash",527 StringRef(I.ContextHash));528 if (!I.NamedModule.empty())529 JOS.attribute("named-module", (I.NamedModule));530 if (!I.NamedModuleDeps.empty())531 JOS.attributeArray("named-module-deps", [&] {532 for (const auto &Dep : I.NamedModuleDeps)533 JOS.value(Dep);534 });535 JOS.attributeArray("clang-module-deps",536 toJSONSorted(JOS, I.ClangModuleDeps));537 JOS.attributeArray("command-line",538 toJSONStrings(JOS, Cmd.Arguments));539 JOS.attribute("executable", StringRef(Cmd.Executable));540 JOS.attributeArray("file-deps",541 toJSONStrings(JOS, I.FileDeps));542 JOS.attribute("input-file", StringRef(I.FileName));543 if (EmitVisibleModules)544 JOS.attributeArray("visible-clang-modules",545 toJSONSorted(JOS, I.VisibleModules));546 });547 }548 } else {549 JOS.object([&] {550 JOS.attribute("clang-context-hash", StringRef(I.ContextHash));551 if (!I.NamedModule.empty())552 JOS.attribute("named-module", (I.NamedModule));553 if (!I.NamedModuleDeps.empty())554 JOS.attributeArray("named-module-deps", [&] {555 for (const auto &Dep : I.NamedModuleDeps)556 JOS.value(Dep);557 });558 JOS.attributeArray("clang-module-deps",559 toJSONSorted(JOS, I.ClangModuleDeps));560 JOS.attributeArray("command-line",561 toJSONStrings(JOS, I.DriverCommandLine));562 JOS.attribute("executable", "clang");563 JOS.attributeArray("file-deps",564 toJSONStrings(JOS, I.FileDeps));565 JOS.attribute("input-file", StringRef(I.FileName));566 if (EmitVisibleModules)567 JOS.attributeArray("visible-clang-modules",568 toJSONSorted(JOS, I.VisibleModules));569 });570 }571 });572 });573 }574 });575 });576 }577 578private:579 struct IndexedModuleID {580 ModuleID ID;581 582 // FIXME: This is mutable so that it can still be updated after insertion583 // into an unordered associative container. This is "fine", since this584 // field doesn't contribute to the hash, but it's a brittle hack.585 mutable size_t InputIndex;586 587 bool operator==(const IndexedModuleID &Other) const {588 return ID == Other.ID;589 }590 591 bool operator<(const IndexedModuleID &Other) const {592 /// We need the output of clang-scan-deps to be deterministic. However,593 /// the dependency graph may contain two modules with the same name. How594 /// do we decide which one to print first? If we made that decision based595 /// on the context hash, the ordering would be deterministic, but596 /// different across machines. This can happen for example when the inputs597 /// or the SDKs (which both contribute to the "context" hash) live in598 /// different absolute locations. We solve that by tracking the index of599 /// the first input TU that (transitively) imports the dependency, which600 /// is always the same for the same input, resulting in deterministic601 /// sorting that's also reproducible across machines.602 return std::tie(ID.ModuleName, InputIndex) <603 std::tie(Other.ID.ModuleName, Other.InputIndex);604 }605 606 struct Hasher {607 std::size_t operator()(const IndexedModuleID &IMID) const {608 return llvm::hash_value(IMID.ID);609 }610 };611 };612 613 struct InputDeps {614 std::string FileName;615 std::string ContextHash;616 std::vector<std::string> FileDeps;617 std::string NamedModule;618 std::vector<std::string> NamedModuleDeps;619 std::vector<ModuleID> ClangModuleDeps;620 std::vector<std::string> VisibleModules;621 std::vector<std::string> DriverCommandLine;622 std::vector<Command> Commands;623 };624 625 std::mutex Lock;626 std::unordered_map<IndexedModuleID, ModuleDeps, IndexedModuleID::Hasher>627 Modules;628 std::vector<InputDeps> Inputs;629};630 631static bool handleTranslationUnitResult(632 StringRef Input, llvm::Expected<TranslationUnitDeps> &MaybeTUDeps,633 FullDeps &FD, size_t InputIndex, SharedStream &OS, SharedStream &Errs) {634 if (!MaybeTUDeps) {635 llvm::handleAllErrors(636 MaybeTUDeps.takeError(), [&Input, &Errs](llvm::StringError &Err) {637 Errs.applyLocked([&](raw_ostream &OS) {638 OS << "Error while scanning dependencies for " << Input << ":\n";639 OS << Err.getMessage();640 });641 });642 return true;643 }644 FD.mergeDeps(Input, std::move(*MaybeTUDeps), InputIndex);645 return false;646}647 648static bool handleModuleResult(StringRef ModuleName,649 llvm::Expected<TranslationUnitDeps> &MaybeTUDeps,650 FullDeps &FD, size_t InputIndex,651 SharedStream &OS, SharedStream &Errs) {652 if (!MaybeTUDeps) {653 llvm::handleAllErrors(MaybeTUDeps.takeError(),654 [&ModuleName, &Errs](llvm::StringError &Err) {655 Errs.applyLocked([&](raw_ostream &OS) {656 OS << "Error while scanning dependencies for "657 << ModuleName << ":\n";658 OS << Err.getMessage();659 });660 });661 return true;662 }663 FD.mergeDeps(std::move(MaybeTUDeps->ModuleGraph), InputIndex);664 return false;665}666 667static void handleErrorWithInfoString(StringRef Info, llvm::Error E,668 SharedStream &OS, SharedStream &Errs) {669 llvm::handleAllErrors(std::move(E), [&Info, &Errs](llvm::StringError &Err) {670 Errs.applyLocked([&](raw_ostream &OS) {671 OS << "Error: " << Info << ":\n";672 OS << Err.getMessage();673 });674 });675}676 677class P1689Deps {678public:679 void printDependencies(raw_ostream &OS) {680 addSourcePathsToRequires();681 // Sort the modules by name to get a deterministic order.682 llvm::sort(Rules, [](const P1689Rule &A, const P1689Rule &B) {683 return A.PrimaryOutput < B.PrimaryOutput;684 });685 686 using namespace llvm::json;687 Array OutputRules;688 for (const P1689Rule &R : Rules) {689 Object O{{"primary-output", R.PrimaryOutput}};690 691 if (R.Provides) {692 Array Provides;693 Object Provided{{"logical-name", R.Provides->ModuleName},694 {"source-path", R.Provides->SourcePath},695 {"is-interface", R.Provides->IsStdCXXModuleInterface}};696 Provides.push_back(std::move(Provided));697 O.insert({"provides", std::move(Provides)});698 }699 700 Array Requires;701 for (const P1689ModuleInfo &Info : R.Requires) {702 Object RequiredInfo{{"logical-name", Info.ModuleName}};703 if (!Info.SourcePath.empty())704 RequiredInfo.insert({"source-path", Info.SourcePath});705 Requires.push_back(std::move(RequiredInfo));706 }707 708 if (!Requires.empty())709 O.insert({"requires", std::move(Requires)});710 711 OutputRules.push_back(std::move(O));712 }713 714 Object Output{715 {"version", 1}, {"revision", 0}, {"rules", std::move(OutputRules)}};716 717 OS << llvm::formatv("{0:2}\n", Value(std::move(Output)));718 }719 720 void addRules(P1689Rule &Rule) {721 std::unique_lock<std::mutex> LockGuard(Lock);722 Rules.push_back(Rule);723 }724 725private:726 void addSourcePathsToRequires() {727 llvm::DenseMap<StringRef, StringRef> ModuleSourceMapper;728 for (const P1689Rule &R : Rules)729 if (R.Provides && !R.Provides->SourcePath.empty())730 ModuleSourceMapper[R.Provides->ModuleName] = R.Provides->SourcePath;731 732 for (P1689Rule &R : Rules) {733 for (P1689ModuleInfo &Info : R.Requires) {734 auto Iter = ModuleSourceMapper.find(Info.ModuleName);735 if (Iter != ModuleSourceMapper.end())736 Info.SourcePath = Iter->second;737 }738 }739 }740 741 std::mutex Lock;742 std::vector<P1689Rule> Rules;743};744 745static bool746handleP1689DependencyToolResult(const std::string &Input,747 llvm::Expected<P1689Rule> &MaybeRule,748 P1689Deps &PD, SharedStream &Errs) {749 if (!MaybeRule) {750 llvm::handleAllErrors(751 MaybeRule.takeError(), [&Input, &Errs](llvm::StringError &Err) {752 Errs.applyLocked([&](raw_ostream &OS) {753 OS << "Error while scanning dependencies for " << Input << ":\n";754 OS << Err.getMessage();755 });756 });757 return true;758 }759 PD.addRules(*MaybeRule);760 return false;761}762 763/// Construct a path for the explicitly built PCM.764static std::string constructPCMPath(ModuleID MID, StringRef OutputDir) {765 SmallString<256> ExplicitPCMPath(OutputDir);766 llvm::sys::path::append(ExplicitPCMPath, MID.ContextHash,767 MID.ModuleName + "-" + MID.ContextHash + ".pcm");768 return std::string(ExplicitPCMPath);769}770 771static std::string lookupModuleOutput(const ModuleDeps &MD,772 ModuleOutputKind MOK,773 StringRef OutputDir) {774 std::string PCMPath = constructPCMPath(MD.ID, OutputDir);775 switch (MOK) {776 case ModuleOutputKind::ModuleFile:777 return PCMPath;778 case ModuleOutputKind::DependencyFile:779 return PCMPath + ".d";780 case ModuleOutputKind::DependencyTargets:781 // Null-separate the list of targets.782 return join(ModuleDepTargets, StringRef("\0", 1));783 case ModuleOutputKind::DiagnosticSerializationFile:784 return PCMPath + ".diag";785 }786 llvm_unreachable("Fully covered switch above!");787}788 789static std::string getModuleCachePath(ArrayRef<std::string> Args) {790 for (StringRef Arg : llvm::reverse(Args)) {791 Arg.consume_front("/clang:");792 if (Arg.consume_front("-fmodules-cache-path="))793 return std::string(Arg);794 }795 SmallString<128> Path;796 driver::Driver::getDefaultModuleCachePath(Path);797 return std::string(Path);798}799 800/// Attempts to construct the compilation database from '-compilation-database'801/// or from the arguments following the positional '--'.802static std::unique_ptr<tooling::CompilationDatabase>803getCompilationDatabase(int argc, char **argv, std::string &ErrorMessage) {804 ParseArgs(argc, argv);805 806 if (!(CommandLine.empty() ^ CompilationDB.empty())) {807 llvm::errs() << "The compilation command line must be provided either via "808 "'-compilation-database' or after '--'.";809 return nullptr;810 }811 812 if (!CompilationDB.empty())813 return tooling::JSONCompilationDatabase::loadFromFile(814 CompilationDB, ErrorMessage,815 tooling::JSONCommandLineSyntax::AutoDetect);816 817 DiagnosticOptions DiagOpts;818 llvm::IntrusiveRefCntPtr<DiagnosticsEngine> Diags =819 CompilerInstance::createDiagnostics(*llvm::vfs::getRealFileSystem(),820 DiagOpts);821 driver::Driver TheDriver(CommandLine[0], llvm::sys::getDefaultTargetTriple(),822 *Diags);823 TheDriver.setCheckInputsExist(false);824 std::unique_ptr<driver::Compilation> C(825 TheDriver.BuildCompilation(CommandLine));826 if (!C || C->getJobs().empty())827 return nullptr;828 829 auto Cmd = C->getJobs().begin();830 auto CI = std::make_unique<CompilerInvocation>();831 CompilerInvocation::CreateFromArgs(*CI, Cmd->getArguments(), *Diags,832 CommandLine[0]);833 if (!CI)834 return nullptr;835 836 FrontendOptions &FEOpts = CI->getFrontendOpts();837 if (FEOpts.Inputs.size() != 1) {838 llvm::errs()839 << "Exactly one input file is required in the per-file mode ('--').\n";840 return nullptr;841 }842 843 // There might be multiple jobs for a compilation. Extract the specified844 // output filename from the last job.845 auto LastCmd = C->getJobs().end();846 LastCmd--;847 if (LastCmd->getOutputFilenames().size() != 1) {848 llvm::errs()849 << "Exactly one output file is required in the per-file mode ('--').\n";850 return nullptr;851 }852 StringRef OutputFile = LastCmd->getOutputFilenames().front();853 854 class InplaceCompilationDatabase : public tooling::CompilationDatabase {855 public:856 InplaceCompilationDatabase(StringRef InputFile, StringRef OutputFile,857 ArrayRef<const char *> CommandLine)858 : Command(".", InputFile, {}, OutputFile) {859 for (auto *C : CommandLine)860 Command.CommandLine.push_back(C);861 }862 863 std::vector<tooling::CompileCommand>864 getCompileCommands(StringRef FilePath) const override {865 if (FilePath != Command.Filename)866 return {};867 return {Command};868 }869 870 std::vector<std::string> getAllFiles() const override {871 return {Command.Filename};872 }873 874 std::vector<tooling::CompileCommand>875 getAllCompileCommands() const override {876 return {Command};877 }878 879 private:880 tooling::CompileCommand Command;881 };882 883 return std::make_unique<InplaceCompilationDatabase>(884 FEOpts.Inputs[0].getFile(), OutputFile, CommandLine);885}886 887int clang_scan_deps_main(int argc, char **argv, const llvm::ToolContext &) {888 llvm::InitializeAllTargetInfos();889 std::string ErrorMessage;890 std::unique_ptr<tooling::CompilationDatabase> Compilations =891 getCompilationDatabase(argc, argv, ErrorMessage);892 if (!Compilations) {893 llvm::errs() << ErrorMessage << "\n";894 return 1;895 }896 897 llvm::cl::PrintOptionValues();898 899 if (!VerbatimArgs) {900 // Expand response files in advance, so that we can "see" all the arguments901 // when adjusting below.902 Compilations = expandResponseFiles(std::move(Compilations),903 llvm::vfs::getRealFileSystem());904 905 Compilations = inferTargetAndDriverMode(std::move(Compilations));906 907 Compilations = inferToolLocation(std::move(Compilations));908 }909 910 // The command options are rewritten to run Clang in preprocessor only mode.911 auto AdjustingCompilations =912 std::make_unique<tooling::ArgumentsAdjustingCompilations>(913 std::move(Compilations));914 ResourceDirectoryCache ResourceDirCache;915 916 auto ArgsAdjuster =917 [&ResourceDirCache](const tooling::CommandLineArguments &Args,918 StringRef FileName) {919 std::string LastO;920 bool HasResourceDir = false;921 bool ClangCLMode = false;922 auto FlagsEnd = llvm::find(Args, "--");923 if (FlagsEnd != Args.begin()) {924 ClangCLMode =925 llvm::sys::path::stem(Args[0]).contains_insensitive("clang-cl") ||926 llvm::is_contained(Args, "--driver-mode=cl");927 928 // Reverse scan, starting at the end or at the element before "--".929 auto R = std::make_reverse_iterator(FlagsEnd);930 auto E = Args.rend();931 // Don't include Args[0] in the iteration; that's the executable, not932 // an option.933 if (E != R)934 E--;935 for (auto I = R; I != E; ++I) {936 StringRef Arg = *I;937 if (ClangCLMode) {938 // Ignore arguments that are preceded by "-Xclang".939 if ((I + 1) != E && I[1] == "-Xclang")940 continue;941 if (LastO.empty()) {942 // With clang-cl, the output obj file can be specified with943 // "/opath", "/o path", "/Fopath", and the dash counterparts.944 // Also, clang-cl adds ".obj" extension if none is found.945 if ((Arg == "-o" || Arg == "/o") && I != R)946 LastO = I[-1]; // Next argument (reverse iterator)947 else if (Arg.starts_with("/Fo") || Arg.starts_with("-Fo"))948 LastO = Arg.drop_front(3).str();949 else if (Arg.starts_with("/o") || Arg.starts_with("-o"))950 LastO = Arg.drop_front(2).str();951 952 if (!LastO.empty() && !llvm::sys::path::has_extension(LastO))953 LastO.append(".obj");954 }955 }956 if (Arg == "-resource-dir")957 HasResourceDir = true;958 }959 }960 tooling::CommandLineArguments AdjustedArgs(Args.begin(), FlagsEnd);961 // The clang-cl driver passes "-o -" to the frontend. Inject the real962 // file here to ensure "-MT" can be deduced if need be.963 if (ClangCLMode && !LastO.empty()) {964 AdjustedArgs.push_back("/clang:-o");965 AdjustedArgs.push_back("/clang:" + LastO);966 }967 968 if (!HasResourceDir && ResourceDirRecipe == RDRK_InvokeCompiler) {969 StringRef ResourceDir =970 ResourceDirCache.findResourceDir(Args, ClangCLMode);971 if (!ResourceDir.empty()) {972 AdjustedArgs.push_back("-resource-dir");973 AdjustedArgs.push_back(std::string(ResourceDir));974 }975 }976 AdjustedArgs.insert(AdjustedArgs.end(), FlagsEnd, Args.end());977 return AdjustedArgs;978 };979 980 if (!VerbatimArgs)981 AdjustingCompilations->appendArgumentsAdjuster(ArgsAdjuster);982 983 SharedStream Errs(llvm::errs());984 985 std::optional<llvm::raw_fd_ostream> FileOS;986 llvm::raw_ostream &ThreadUnsafeDependencyOS = [&]() -> llvm::raw_ostream & {987 if (OutputFileName == "-")988 return llvm::outs();989 990 if (OutputFileName == "/dev/null")991 return llvm::nulls();992 993 std::error_code EC;994 FileOS.emplace(OutputFileName, EC, llvm::sys::fs::OF_Text);995 if (EC) {996 llvm::errs() << "Failed to open output file '" << OutputFileName997 << "': " << EC.message() << '\n';998 std::exit(1);999 }1000 return *FileOS;1001 }();1002 SharedStream DependencyOS(ThreadUnsafeDependencyOS);1003 1004 std::vector<tooling::CompileCommand> Inputs =1005 AdjustingCompilations->getAllCompileCommands();1006 1007 std::atomic<bool> HadErrors(false);1008 std::optional<FullDeps> FD;1009 P1689Deps PD;1010 1011 std::mutex Lock;1012 size_t Index = 0;1013 auto GetNextInputIndex = [&]() -> std::optional<size_t> {1014 std::unique_lock<std::mutex> LockGuard(Lock);1015 if (Index < Inputs.size())1016 return Index++;1017 return {};1018 };1019 1020 if (Format == ScanningOutputFormat::Full)1021 FD.emplace(!ModuleNames ? Inputs.size() : 0);1022 1023 std::atomic<size_t> NumStatusCalls = 0;1024 std::atomic<size_t> NumOpenFileForReadCalls = 0;1025 std::atomic<size_t> NumDirBeginCalls = 0;1026 std::atomic<size_t> NumGetRealPathCalls = 0;1027 std::atomic<size_t> NumExistsCalls = 0;1028 std::atomic<size_t> NumIsLocalCalls = 0;1029 1030 auto ScanningTask = [&](DependencyScanningService &Service) {1031 DependencyScanningTool WorkerTool(Service);1032 1033 llvm::DenseSet<ModuleID> AlreadySeenModules;1034 while (auto MaybeInputIndex = GetNextInputIndex()) {1035 size_t LocalIndex = *MaybeInputIndex;1036 const tooling::CompileCommand *Input = &Inputs[LocalIndex];1037 std::string Filename = std::move(Input->Filename);1038 std::string CWD = std::move(Input->Directory);1039 1040 std::string OutputDir(ModuleFilesDir);1041 if (OutputDir.empty())1042 OutputDir = getModuleCachePath(Input->CommandLine);1043 auto LookupOutput = [&](const ModuleDeps &MD, ModuleOutputKind MOK) {1044 return ::lookupModuleOutput(MD, MOK, OutputDir);1045 };1046 1047 // Run the tool on it.1048 if (Format == ScanningOutputFormat::Make) {1049 auto MaybeFile = WorkerTool.getDependencyFile(Input->CommandLine, CWD);1050 if (handleMakeDependencyToolResult(Filename, MaybeFile, DependencyOS,1051 Errs))1052 HadErrors = true;1053 } else if (Format == ScanningOutputFormat::P1689) {1054 // It is useful to generate the make-format dependency output during1055 // the scanning for P1689. Otherwise the users need to scan again for1056 // it. We will generate the make-format dependency output if we find1057 // `-MF` in the command lines.1058 std::string MakeformatOutputPath;1059 std::string MakeformatOutput;1060 1061 auto MaybeRule = WorkerTool.getP1689ModuleDependencyFile(1062 *Input, CWD, MakeformatOutput, MakeformatOutputPath);1063 1064 if (handleP1689DependencyToolResult(Filename, MaybeRule, PD, Errs))1065 HadErrors = true;1066 1067 if (!MakeformatOutputPath.empty() && !MakeformatOutput.empty() &&1068 !HadErrors) {1069 static std::mutex Lock;1070 // With compilation database, we may open different files1071 // concurrently or we may write the same file concurrently. So we1072 // use a map here to allow multiple compile commands to write to the1073 // same file. Also we need a lock here to avoid data race.1074 static llvm::StringMap<llvm::raw_fd_ostream> OSs;1075 std::unique_lock<std::mutex> LockGuard(Lock);1076 1077 auto OSIter = OSs.find(MakeformatOutputPath);1078 if (OSIter == OSs.end()) {1079 std::error_code EC;1080 OSIter = OSs.try_emplace(MakeformatOutputPath, MakeformatOutputPath,1081 EC, llvm::sys::fs::OF_Text)1082 .first;1083 if (EC)1084 llvm::errs() << "Failed to open P1689 make format output file \""1085 << MakeformatOutputPath << "\" for " << EC.message()1086 << "\n";1087 }1088 1089 SharedStream MakeformatOS(OSIter->second);1090 llvm::Expected<std::string> MaybeOutput(MakeformatOutput);1091 if (handleMakeDependencyToolResult(Filename, MaybeOutput,1092 MakeformatOS, Errs))1093 HadErrors = true;1094 }1095 } else if (ModuleNames) {1096 StringRef ModuleNameRef(*ModuleNames);1097 SmallVector<StringRef> Names;1098 ModuleNameRef.split(Names, ',');1099 1100 if (Names.size() == 1) {1101 auto MaybeModuleDepsGraph = WorkerTool.getModuleDependencies(1102 Names[0], Input->CommandLine, CWD, AlreadySeenModules,1103 LookupOutput);1104 if (handleModuleResult(Names[0], MaybeModuleDepsGraph, *FD,1105 LocalIndex, DependencyOS, Errs))1106 HadErrors = true;1107 } else {1108 if (llvm::Error Err =1109 WorkerTool.initializeCompilerInstanceWithContext(1110 CWD, Input->CommandLine)) {1111 handleErrorWithInfoString(1112 "Compiler instance with context setup error", std::move(Err),1113 DependencyOS, Errs);1114 HadErrors = true;1115 continue;1116 }1117 1118 for (auto N : Names) {1119 auto MaybeModuleDepsGraph =1120 WorkerTool.computeDependenciesByNameWithContext(1121 N, AlreadySeenModules, LookupOutput);1122 if (handleModuleResult(N, MaybeModuleDepsGraph, *FD, LocalIndex,1123 DependencyOS, Errs)) {1124 HadErrors = true;1125 break;1126 }1127 }1128 1129 if (llvm::Error Err =1130 WorkerTool.finalizeCompilerInstanceWithContext()) {1131 handleErrorWithInfoString(1132 "Compiler instance with context finialization error",1133 std::move(Err), DependencyOS, Errs);1134 HadErrors = true;1135 }1136 }1137 } else {1138 std::unique_ptr<llvm::MemoryBuffer> TU;1139 std::optional<llvm::MemoryBufferRef> TUBuffer;1140 if (!TranslationUnitFile.empty()) {1141 auto MaybeTU =1142 llvm::MemoryBuffer::getFile(TranslationUnitFile, /*IsText=*/true);1143 if (!MaybeTU) {1144 llvm::errs() << "cannot open input translation unit: "1145 << MaybeTU.getError().message() << "\n";1146 HadErrors = true;1147 continue;1148 }1149 TU = std::move(*MaybeTU);1150 TUBuffer = TU->getMemBufferRef();1151 Filename = TU->getBufferIdentifier();1152 }1153 auto MaybeTUDeps = WorkerTool.getTranslationUnitDependencies(1154 Input->CommandLine, CWD, AlreadySeenModules, LookupOutput,1155 TUBuffer);1156 if (handleTranslationUnitResult(Filename, MaybeTUDeps, *FD, LocalIndex,1157 DependencyOS, Errs))1158 HadErrors = true;1159 }1160 }1161 1162 WorkerTool.getWorkerVFS().visit([&](llvm::vfs::FileSystem &VFS) {1163 if (auto *T = dyn_cast_or_null<llvm::vfs::TracingFileSystem>(&VFS)) {1164 NumStatusCalls += T->NumStatusCalls;1165 NumOpenFileForReadCalls += T->NumOpenFileForReadCalls;1166 NumDirBeginCalls += T->NumDirBeginCalls;1167 NumGetRealPathCalls += T->NumGetRealPathCalls;1168 NumExistsCalls += T->NumExistsCalls;1169 NumIsLocalCalls += T->NumIsLocalCalls;1170 }1171 });1172 };1173 1174 DependencyScanningService Service(ScanMode, Format, OptimizeArgs,1175 EagerLoadModules, /*TraceVFS=*/Verbose);1176 1177 llvm::Timer T;1178 T.startTimer();1179 1180 if (Inputs.size() == 1) {1181 ScanningTask(Service);1182 } else {1183 llvm::DefaultThreadPool Pool(llvm::hardware_concurrency(NumThreads));1184 1185 if (Verbose) {1186 llvm::outs() << "Running clang-scan-deps on " << Inputs.size()1187 << " files using " << Pool.getMaxConcurrency()1188 << " workers\n";1189 }1190 1191 for (unsigned I = 0; I < Pool.getMaxConcurrency(); ++I)1192 Pool.async([ScanningTask, &Service]() { ScanningTask(Service); });1193 1194 Pool.wait();1195 }1196 1197 T.stopTimer();1198 1199 if (Verbose)1200 llvm::errs() << "\n*** Virtual File System Stats:\n"1201 << NumStatusCalls << " status() calls\n"1202 << NumOpenFileForReadCalls << " openFileForRead() calls\n"1203 << NumDirBeginCalls << " dir_begin() calls\n"1204 << NumGetRealPathCalls << " getRealPath() calls\n"1205 << NumExistsCalls << " exists() calls\n"1206 << NumIsLocalCalls << " isLocal() calls\n";1207 1208 if (PrintTiming) {1209 llvm::errs() << "wall time [s]\t"1210 << "process time [s]\t"1211 << "instruction count\n";1212 const llvm::TimeRecord &R = T.getTotalTime();1213 llvm::errs() << llvm::format("%0.4f", R.getWallTime()) << "\t"1214 << llvm::format("%0.4f", R.getProcessTime()) << "\t"1215 << llvm::format("%llu", R.getInstructionsExecuted()) << "\n";1216 }1217 1218 if (RoundTripArgs)1219 if (FD && FD->roundTripCommands(llvm::errs()))1220 HadErrors = true;1221 1222 if (Format == ScanningOutputFormat::Full)1223 FD->printFullOutput(ThreadUnsafeDependencyOS);1224 else if (Format == ScanningOutputFormat::P1689)1225 PD.printDependencies(ThreadUnsafeDependencyOS);1226 1227 return HadErrors;1228}1229