457 lines · cpp
1//===- Job.cpp - Command to Execute ---------------------------------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8 9#include "clang/Driver/Job.h"10#include "clang/Basic/LLVM.h"11#include "clang/Driver/Driver.h"12#include "clang/Driver/InputInfo.h"13#include "clang/Driver/Tool.h"14#include "clang/Driver/ToolChain.h"15#include "llvm/ADT/ArrayRef.h"16#include "llvm/ADT/SmallString.h"17#include "llvm/ADT/SmallVector.h"18#include "llvm/ADT/StringExtras.h"19#include "llvm/ADT/StringRef.h"20#include "llvm/ADT/StringSet.h"21#include "llvm/ADT/StringSwitch.h"22#include "llvm/Support/CrashRecoveryContext.h"23#include "llvm/Support/FileSystem.h"24#include "llvm/Support/Path.h"25#include "llvm/Support/PrettyStackTrace.h"26#include "llvm/Support/Program.h"27#include "llvm/Support/raw_ostream.h"28#include <cassert>29#include <cstddef>30#include <string>31#include <system_error>32#include <utility>33 34using namespace clang;35using namespace driver;36 37Command::Command(const Action &Source, const Tool &Creator,38 ResponseFileSupport ResponseSupport, const char *Executable,39 const llvm::opt::ArgStringList &Arguments,40 ArrayRef<InputInfo> Inputs, ArrayRef<InputInfo> Outputs,41 const char *PrependArg)42 : Source(Source), Creator(Creator), ResponseSupport(ResponseSupport),43 Executable(Executable), PrependArg(PrependArg), Arguments(Arguments) {44 for (const auto &II : Inputs)45 if (II.isFilename())46 InputInfoList.push_back(II);47 for (const auto &II : Outputs)48 if (II.isFilename())49 OutputFilenames.push_back(II.getFilename());50}51 52/// Check if the compiler flag in question should be skipped when53/// emitting a reproducer. Also track how many arguments it has and if the54/// option is some kind of include path.55static bool skipArgs(const char *Flag, bool HaveCrashVFS, int &SkipNum,56 bool &IsInclude) {57 SkipNum = 2;58 // These flags are all of the form -Flag <Arg> and are treated as two59 // arguments. Therefore, we need to skip the flag and the next argument.60 bool ShouldSkip =61 llvm::StringSwitch<bool>(Flag)62 .Cases({"-MF", "-MT", "-MQ", "-serialize-diagnostic-file"}, true)63 .Cases({"-o", "-dependency-file"}, true)64 .Cases({"-fdebug-compilation-dir", "-diagnostic-log-file"}, true)65 .Cases({"-dwarf-debug-flags", "-ivfsoverlay"}, true)66 .Default(false);67 if (ShouldSkip)68 return true;69 70 // Some include flags shouldn't be skipped if we have a crash VFS71 IsInclude =72 llvm::StringSwitch<bool>(Flag)73 .Cases({"-include", "-header-include-file"}, true)74 .Cases({"-idirafter", "-internal-isystem", "-iwithprefix"}, true)75 .Cases({"-internal-externc-isystem", "-iprefix"}, true)76 .Cases({"-iwithprefixbefore", "-isystem", "-iquote"}, true)77 .Cases({"-isysroot", "-I", "-F", "-resource-dir"}, true)78 .Cases({"-internal-iframework", "-iframework", "-include-pch"}, true)79 .Default(false);80 if (IsInclude)81 return !HaveCrashVFS;82 83 // The remaining flags are treated as a single argument.84 85 // These flags are all of the form -Flag and have no second argument.86 ShouldSkip = llvm::StringSwitch<bool>(Flag)87 .Cases({"-M", "-MM", "-MG", "-MP", "-MD"}, true)88 .Case("-MMD", true)89 .Default(false);90 91 // Match found.92 SkipNum = 1;93 if (ShouldSkip)94 return true;95 96 // These flags are treated as a single argument (e.g., -F<Dir>).97 StringRef FlagRef(Flag);98 IsInclude = FlagRef.starts_with("-F") || FlagRef.starts_with("-I");99 if (IsInclude)100 return !HaveCrashVFS;101 if (FlagRef.starts_with("-fmodules-cache-path="))102 return true;103 104 SkipNum = 0;105 return false;106}107 108void Command::writeResponseFile(raw_ostream &OS) const {109 // In a file list, we only write the set of inputs to the response file110 if (ResponseSupport.ResponseKind == ResponseFileSupport::RF_FileList) {111 for (const auto *Arg : InputFileList) {112 OS << Arg << '\n';113 }114 return;115 }116 117 // In regular response files, we send all arguments to the response file.118 // Wrapping all arguments in double quotes ensures that both Unix tools and119 // Windows tools understand the response file.120 for (const auto *Arg : Arguments) {121 OS << '"';122 123 for (; *Arg != '\0'; Arg++) {124 if (*Arg == '\"' || *Arg == '\\') {125 OS << '\\';126 }127 OS << *Arg;128 }129 130 OS << "\" ";131 }132}133 134void Command::buildArgvForResponseFile(135 llvm::SmallVectorImpl<const char *> &Out) const {136 // When not a file list, all arguments are sent to the response file.137 // This leaves us to set the argv to a single parameter, requesting the tool138 // to read the response file.139 if (ResponseSupport.ResponseKind != ResponseFileSupport::RF_FileList) {140 Out.push_back(Executable);141 Out.push_back(ResponseFileFlag.c_str());142 return;143 }144 145 llvm::StringSet<> Inputs(llvm::from_range, InputFileList);146 Out.push_back(Executable);147 148 if (PrependArg)149 Out.push_back(PrependArg);150 151 // In a file list, build args vector ignoring parameters that will go in the152 // response file (elements of the InputFileList vector)153 bool FirstInput = true;154 for (const auto *Arg : Arguments) {155 if (Inputs.count(Arg) == 0) {156 Out.push_back(Arg);157 } else if (FirstInput) {158 FirstInput = false;159 Out.push_back(ResponseSupport.ResponseFlag);160 Out.push_back(ResponseFile);161 }162 }163}164 165/// Rewrite relative include-like flag paths to absolute ones.166static void167rewriteIncludes(const llvm::ArrayRef<const char *> &Args, size_t Idx,168 size_t NumArgs,169 llvm::SmallVectorImpl<llvm::SmallString<128>> &IncFlags) {170 using namespace llvm;171 using namespace sys;172 173 auto getAbsPath = [](StringRef InInc, SmallVectorImpl<char> &OutInc) -> bool {174 if (path::is_absolute(InInc)) // Nothing to do here...175 return false;176 std::error_code EC = fs::current_path(OutInc);177 if (EC)178 return false;179 path::append(OutInc, InInc);180 return true;181 };182 183 SmallString<128> NewInc;184 if (NumArgs == 1) {185 StringRef FlagRef(Args[Idx + NumArgs - 1]);186 assert((FlagRef.starts_with("-F") || FlagRef.starts_with("-I")) &&187 "Expecting -I or -F");188 StringRef Inc = FlagRef.substr(2);189 if (getAbsPath(Inc, NewInc)) {190 SmallString<128> NewArg(FlagRef.slice(0, 2));191 NewArg += NewInc;192 IncFlags.push_back(std::move(NewArg));193 }194 return;195 }196 197 assert(NumArgs == 2 && "Not expecting more than two arguments");198 StringRef Inc(Args[Idx + NumArgs - 1]);199 if (!getAbsPath(Inc, NewInc))200 return;201 IncFlags.push_back(SmallString<128>(Args[Idx]));202 IncFlags.push_back(std::move(NewInc));203}204 205void Command::Print(raw_ostream &OS, const char *Terminator, bool Quote,206 CrashReportInfo *CrashInfo) const {207 // Always quote the exe.208 OS << ' ';209 llvm::sys::printArg(OS, Executable, /*Quote=*/true);210 211 ArrayRef<const char *> Args = Arguments;212 SmallVector<const char *, 128> ArgsRespFile;213 if (ResponseFile != nullptr) {214 buildArgvForResponseFile(ArgsRespFile);215 Args = ArrayRef<const char *>(ArgsRespFile).slice(1); // no executable name216 } else if (PrependArg) {217 OS << ' ';218 llvm::sys::printArg(OS, PrependArg, /*Quote=*/true);219 }220 221 bool HaveCrashVFS = CrashInfo && !CrashInfo->VFSPath.empty();222 for (size_t i = 0, e = Args.size(); i < e; ++i) {223 const char *const Arg = Args[i];224 225 if (CrashInfo) {226 int NumArgs = 0;227 bool IsInclude = false;228 if (skipArgs(Arg, HaveCrashVFS, NumArgs, IsInclude)) {229 i += NumArgs - 1;230 continue;231 }232 233 // Relative includes need to be expanded to absolute paths.234 if (HaveCrashVFS && IsInclude) {235 SmallVector<SmallString<128>, 2> NewIncFlags;236 rewriteIncludes(Args, i, NumArgs, NewIncFlags);237 if (!NewIncFlags.empty()) {238 for (auto &F : NewIncFlags) {239 OS << ' ';240 llvm::sys::printArg(OS, F.c_str(), Quote);241 }242 i += NumArgs - 1;243 continue;244 }245 }246 247 auto Found = llvm::find_if(InputInfoList, [&Arg](const InputInfo &II) {248 return II.getFilename() == Arg;249 });250 if (Found != InputInfoList.end() &&251 (i == 0 || StringRef(Args[i - 1]) != "-main-file-name")) {252 // Replace the input file name with the crashinfo's file name.253 OS << ' ';254 StringRef ShortName = llvm::sys::path::filename(CrashInfo->Filename);255 llvm::sys::printArg(OS, ShortName.str(), Quote);256 continue;257 }258 }259 260 OS << ' ';261 llvm::sys::printArg(OS, Arg, Quote);262 }263 264 if (CrashInfo && HaveCrashVFS) {265 OS << ' ';266 llvm::sys::printArg(OS, "-ivfsoverlay", Quote);267 OS << ' ';268 llvm::sys::printArg(OS, CrashInfo->VFSPath.str(), Quote);269 270 // The leftover modules from the crash are stored in271 // <name>.cache/vfs/modules272 // Leave it untouched for pcm inspection and provide a clean/empty dir273 // path to contain the future generated module cache:274 // <name>.cache/vfs/repro-modules275 SmallString<128> RelModCacheDir = llvm::sys::path::parent_path(276 llvm::sys::path::parent_path(CrashInfo->VFSPath));277 llvm::sys::path::append(RelModCacheDir, "repro-modules");278 279 std::string ModCachePath = "-fmodules-cache-path=";280 ModCachePath.append(RelModCacheDir.c_str());281 282 OS << ' ';283 llvm::sys::printArg(OS, ModCachePath, Quote);284 }285 286 if (ResponseFile != nullptr) {287 OS << "\n Arguments passed via response file:\n";288 writeResponseFile(OS);289 // Avoiding duplicated newline terminator, since FileLists are290 // newline-separated.291 if (ResponseSupport.ResponseKind != ResponseFileSupport::RF_FileList)292 OS << "\n";293 OS << " (end of response file)";294 }295 296 OS << Terminator;297}298 299void Command::setResponseFile(const char *FileName) {300 ResponseFile = FileName;301 ResponseFileFlag = ResponseSupport.ResponseFlag;302 ResponseFileFlag += FileName;303}304 305void Command::setEnvironment(llvm::ArrayRef<const char *> NewEnvironment) {306 Environment.reserve(NewEnvironment.size() + 1);307 Environment.assign(NewEnvironment.begin(), NewEnvironment.end());308 Environment.push_back(nullptr);309}310 311void Command::setRedirectFiles(312 const std::vector<std::optional<std::string>> &Redirects) {313 RedirectFiles = Redirects;314}315 316void Command::PrintFileNames() const {317 if (PrintInputFilenames) {318 for (const auto &Arg : InputInfoList)319 llvm::outs() << llvm::sys::path::filename(Arg.getFilename()) << "\n";320 llvm::outs().flush();321 }322}323 324int Command::Execute(ArrayRef<std::optional<StringRef>> Redirects,325 std::string *ErrMsg, bool *ExecutionFailed) const {326 PrintFileNames();327 328 SmallVector<const char *, 128> Argv;329 if (ResponseFile == nullptr) {330 Argv.push_back(Executable);331 if (PrependArg)332 Argv.push_back(PrependArg);333 Argv.append(Arguments.begin(), Arguments.end());334 Argv.push_back(nullptr);335 } else {336 // If the command is too large, we need to put arguments in a response file.337 std::string RespContents;338 llvm::raw_string_ostream SS(RespContents);339 340 // Write file contents and build the Argv vector341 writeResponseFile(SS);342 buildArgvForResponseFile(Argv);343 Argv.push_back(nullptr);344 345 // Save the response file in the appropriate encoding346 if (std::error_code EC = writeFileWithEncoding(347 ResponseFile, RespContents, ResponseSupport.ResponseEncoding)) {348 if (ErrMsg)349 *ErrMsg = EC.message();350 if (ExecutionFailed)351 *ExecutionFailed = true;352 // Return -1 by convention (see llvm/include/llvm/Support/Program.h) to353 // indicate the requested executable cannot be started.354 return -1;355 }356 }357 358 std::optional<ArrayRef<StringRef>> Env;359 std::vector<StringRef> ArgvVectorStorage;360 if (!Environment.empty()) {361 assert(Environment.back() == nullptr &&362 "Environment vector should be null-terminated by now");363 ArgvVectorStorage = llvm::toStringRefArray(Environment.data());364 Env = ArrayRef(ArgvVectorStorage);365 }366 367 auto Args = llvm::toStringRefArray(Argv.data());368 369 // Use Job-specific redirect files if they are present.370 if (!RedirectFiles.empty()) {371 std::vector<std::optional<StringRef>> RedirectFilesOptional;372 for (const auto &Ele : RedirectFiles)373 if (Ele)374 RedirectFilesOptional.push_back(std::optional<StringRef>(*Ele));375 else376 RedirectFilesOptional.push_back(std::nullopt);377 378 return llvm::sys::ExecuteAndWait(Executable, Args, Env,379 ArrayRef(RedirectFilesOptional),380 /*secondsToWait=*/0, /*memoryLimit=*/0,381 ErrMsg, ExecutionFailed, &ProcStat);382 }383 384 return llvm::sys::ExecuteAndWait(Executable, Args, Env, Redirects,385 /*secondsToWait*/ 0, /*memoryLimit*/ 0,386 ErrMsg, ExecutionFailed, &ProcStat);387}388 389CC1Command::CC1Command(const Action &Source, const Tool &Creator,390 ResponseFileSupport ResponseSupport,391 const char *Executable,392 const llvm::opt::ArgStringList &Arguments,393 ArrayRef<InputInfo> Inputs, ArrayRef<InputInfo> Outputs,394 const char *PrependArg)395 : Command(Source, Creator, ResponseSupport, Executable, Arguments, Inputs,396 Outputs, PrependArg) {397 InProcess = true;398}399 400void CC1Command::Print(raw_ostream &OS, const char *Terminator, bool Quote,401 CrashReportInfo *CrashInfo) const {402 if (InProcess)403 OS << " (in-process)\n";404 Command::Print(OS, Terminator, Quote, CrashInfo);405}406 407int CC1Command::Execute(ArrayRef<std::optional<StringRef>> Redirects,408 std::string *ErrMsg, bool *ExecutionFailed) const {409 // FIXME: Currently, if there're more than one job, we disable410 // -fintegrate-cc1. If we're no longer a integrated-cc1 job, fallback to411 // out-of-process execution. See discussion in https://reviews.llvm.org/D74447412 if (!InProcess)413 return Command::Execute(Redirects, ErrMsg, ExecutionFailed);414 415 PrintFileNames();416 417 SmallVector<const char *, 128> Argv;418 Argv.push_back(getExecutable());419 Argv.append(getArguments().begin(), getArguments().end());420 Argv.push_back(nullptr);421 Argv.pop_back(); // The terminating null element shall not be part of the422 // slice (main() behavior).423 424 // This flag simply indicates that the program couldn't start, which isn't425 // applicable here.426 if (ExecutionFailed)427 *ExecutionFailed = false;428 429 llvm::CrashRecoveryContext CRC;430 CRC.DumpStackAndCleanupOnFailure = true;431 432 const void *PrettyState = llvm::SavePrettyStackState();433 const Driver &D = getCreator().getToolChain().getDriver();434 435 int R = 0;436 // Enter ExecuteCC1Tool() instead of starting up a new process437 if (!CRC.RunSafely([&]() { R = D.CC1Main(Argv); })) {438 llvm::RestorePrettyStackState(PrettyState);439 return CRC.RetCode;440 }441 return R;442}443 444void CC1Command::setEnvironment(llvm::ArrayRef<const char *> NewEnvironment) {445 // We don't support set a new environment when calling into ExecuteCC1Tool()446 llvm_unreachable(447 "The CC1Command doesn't support changing the environment vars!");448}449 450void JobList::Print(raw_ostream &OS, const char *Terminator, bool Quote,451 CrashReportInfo *CrashInfo) const {452 for (const auto &Job : *this)453 Job.Print(OS, Terminator, Quote, CrashInfo);454}455 456void JobList::clear() { Jobs.clear(); }457