550 lines · cpp
1//===--- SystemIncludeExtractor.cpp ------------------------------*- C++-*-===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8// Some compiler drivers have implicit search mechanism for system headers.9// This compilation database implementation tries to extract that information by10// executing the driver in verbose mode. gcc-compatible drivers print something11// like:12// ....13// ....14// #include <...> search starts here:15// /usr/lib/gcc/x86_64-linux-gnu/7/include16// /usr/local/include17// /usr/lib/gcc/x86_64-linux-gnu/7/include-fixed18// /usr/include/x86_64-linux-gnu19// /usr/include20// End of search list.21// ....22// ....23// This component parses that output and adds each path to command line args24// provided by Base, after prepending them with -isystem. Therefore current25// implementation would not work with a driver that is not gcc-compatible.26//27// First argument of the command line received from underlying compilation28// database is used as compiler driver path. Due to this arbitrary binary29// execution, this mechanism is not used by default and only executes binaries30// in the paths that are explicitly included by the user.31 32#include "CompileCommands.h"33#include "Config.h"34#include "GlobalCompilationDatabase.h"35#include "support/Logger.h"36#include "support/Threading.h"37#include "support/Trace.h"38#include "clang/Basic/Diagnostic.h"39#include "clang/Basic/DiagnosticIDs.h"40#include "clang/Basic/DiagnosticOptions.h"41#include "clang/Basic/TargetInfo.h"42#include "clang/Basic/TargetOptions.h"43#include "clang/Driver/Types.h"44#include "clang/Tooling/CompilationDatabase.h"45#include "llvm/ADT/ArrayRef.h"46#include "llvm/ADT/DenseMap.h"47#include "llvm/ADT/Hashing.h"48#include "llvm/ADT/IntrusiveRefCntPtr.h"49#include "llvm/ADT/STLExtras.h"50#include "llvm/ADT/ScopeExit.h"51#include "llvm/ADT/SmallString.h"52#include "llvm/ADT/SmallVector.h"53#include "llvm/ADT/StringExtras.h"54#include "llvm/ADT/StringRef.h"55#include "llvm/Support/ErrorHandling.h"56#include "llvm/Support/FileSystem.h"57#include "llvm/Support/MemoryBuffer.h"58#include "llvm/Support/Path.h"59#include "llvm/Support/Program.h"60#include "llvm/Support/Regex.h"61#include "llvm/Support/ScopedPrinter.h"62#include "llvm/Support/raw_ostream.h"63#include <cassert>64#include <cstddef>65#include <iterator>66#include <memory>67#include <optional>68#include <string>69#include <tuple>70#include <utility>71#include <vector>72 73namespace clang::clangd {74namespace {75 76struct DriverInfo {77 std::vector<std::string> SystemIncludes;78 std::string Target;79};80 81struct DriverArgs {82 // Name of the driver program to execute or absolute path to it.83 std::string Driver;84 // Whether certain includes should be part of query.85 bool StandardIncludes = true;86 bool StandardCXXIncludes = true;87 // Language to use while querying.88 std::string Lang;89 std::string Sysroot;90 std::string ISysroot;91 std::string Target;92 std::string Stdlib;93 llvm::SmallVector<std::string> Specs;94 95 bool operator==(const DriverArgs &RHS) const {96 return std::tie(Driver, StandardIncludes, StandardCXXIncludes, Lang,97 Sysroot, ISysroot, Target, Stdlib, Specs) ==98 std::tie(RHS.Driver, RHS.StandardIncludes, RHS.StandardCXXIncludes,99 RHS.Lang, RHS.Sysroot, RHS.ISysroot, RHS.Target, RHS.Stdlib,100 RHS.Specs);101 }102 103 DriverArgs(const tooling::CompileCommand &Cmd, llvm::StringRef File) {104 llvm::SmallString<128> Driver(Cmd.CommandLine.front());105 // Driver is a not a single executable name but instead a path (either106 // relative or absolute).107 if (llvm::any_of(Driver,108 [](char C) { return llvm::sys::path::is_separator(C); })) {109 llvm::sys::path::make_absolute(Cmd.Directory, Driver);110 }111 this->Driver = Driver.str().str();112 for (size_t I = 0, E = Cmd.CommandLine.size(); I < E; ++I) {113 llvm::StringRef Arg = Cmd.CommandLine[I];114 115 // Look for Language related flags.116 if (Arg.consume_front("-x")) {117 if (Arg.empty() && I + 1 < E)118 Lang = Cmd.CommandLine[I + 1];119 else120 Lang = Arg.str();121 }122 // Look for standard/builtin includes.123 else if (Arg == "-nostdinc" || Arg == "--no-standard-includes")124 StandardIncludes = false;125 else if (Arg == "-nostdinc++")126 StandardCXXIncludes = false;127 // Figure out sysroot128 else if (Arg.consume_front("--sysroot")) {129 if (Arg.consume_front("="))130 Sysroot = Arg.str();131 else if (Arg.empty() && I + 1 < E)132 Sysroot = Cmd.CommandLine[I + 1];133 } else if (Arg.consume_front("-isysroot")) {134 if (Arg.empty() && I + 1 < E)135 ISysroot = Cmd.CommandLine[I + 1];136 else137 ISysroot = Arg.str();138 } else if (Arg.consume_front("--target=")) {139 Target = Arg.str();140 } else if (Arg.consume_front("-target")) {141 if (Arg.empty() && I + 1 < E)142 Target = Cmd.CommandLine[I + 1];143 } else if (Arg.consume_front("--stdlib")) {144 if (Arg.consume_front("="))145 Stdlib = Arg.str();146 else if (Arg.empty() && I + 1 < E)147 Stdlib = Cmd.CommandLine[I + 1];148 } else if (Arg.consume_front("-stdlib=")) {149 Stdlib = Arg.str();150 } else if (Arg.starts_with("-specs=")) {151 // clang requires a single token like `-specs=file` or `--specs=file`,152 // but gcc will accept two tokens like `--specs file`. Since the153 // compilation database is presumably correct, we just forward the flags154 // as-is.155 Specs.push_back(Arg.str());156 } else if (Arg.starts_with("--specs=")) {157 Specs.push_back(Arg.str());158 } else if (Arg == "--specs" && I + 1 < E) {159 Specs.push_back(Arg.str());160 Specs.push_back(Cmd.CommandLine[I + 1]);161 }162 }163 164 // Downgrade objective-c++-header (used in clangd's fallback flags for .h165 // files) to c++-header, as some drivers may fail to run the extraction166 // command if it contains `-xobjective-c++-header` and objective-c++ support167 // is not installed.168 // In practice, we don't see different include paths for the two on169 // clang+mac, which is the most common objectve-c compiler.170 if (Lang == "objective-c++-header") {171 Lang = "c++-header";172 }173 174 // If language is not explicit in the flags, infer from the file.175 // This is important as we want to cache each language separately.176 if (Lang.empty()) {177 llvm::StringRef Ext = llvm::sys::path::extension(File).trim('.');178 auto Type = driver::types::lookupTypeForExtension(Ext);179 if (Type == driver::types::TY_INVALID) {180 elog("System include extraction: invalid file type for {0}", Ext);181 } else {182 Lang = driver::types::getTypeName(Type);183 }184 }185 }186 llvm::SmallVector<llvm::StringRef> render() const {187 // FIXME: Don't treat lang specially?188 assert(!Lang.empty());189 llvm::SmallVector<llvm::StringRef> Args = {"-x", Lang};190 if (!StandardIncludes)191 Args.push_back("-nostdinc");192 if (!StandardCXXIncludes)193 Args.push_back("-nostdinc++");194 if (!Sysroot.empty())195 Args.append({"--sysroot", Sysroot});196 if (!ISysroot.empty())197 Args.append({"-isysroot", ISysroot});198 if (!Target.empty())199 Args.append({"-target", Target});200 if (!Stdlib.empty())201 Args.append({"--stdlib", Stdlib});202 203 for (llvm::StringRef Spec : Specs) {204 Args.push_back(Spec);205 }206 207 return Args;208 }209 210 static DriverArgs getEmpty() { return {}; }211 212private:213 DriverArgs() = default;214};215} // namespace216} // namespace clang::clangd217namespace llvm {218using DriverArgs = clang::clangd::DriverArgs;219template <> struct DenseMapInfo<DriverArgs> {220 static DriverArgs getEmptyKey() {221 auto Driver = DriverArgs::getEmpty();222 Driver.Driver = "EMPTY_KEY";223 return Driver;224 }225 static DriverArgs getTombstoneKey() {226 auto Driver = DriverArgs::getEmpty();227 Driver.Driver = "TOMBSTONE_KEY";228 return Driver;229 }230 static unsigned getHashValue(const DriverArgs &Val) {231 unsigned FixedFieldsHash = llvm::hash_value(std::tuple{232 Val.Driver,233 Val.StandardIncludes,234 Val.StandardCXXIncludes,235 Val.Lang,236 Val.Sysroot,237 Val.ISysroot,238 Val.Target,239 Val.Stdlib,240 });241 242 unsigned SpecsHash = llvm::hash_combine_range(Val.Specs);243 244 return llvm::hash_combine(FixedFieldsHash, SpecsHash);245 }246 static bool isEqual(const DriverArgs &LHS, const DriverArgs &RHS) {247 return LHS == RHS;248 }249};250} // namespace llvm251namespace clang::clangd {252namespace {253bool isValidTarget(llvm::StringRef Triple) {254 std::shared_ptr<TargetOptions> TargetOpts(new TargetOptions);255 TargetOpts->Triple = Triple.str();256 DiagnosticOptions DiagOpts;257 DiagnosticsEngine Diags(DiagnosticIDs::create(), DiagOpts,258 new IgnoringDiagConsumer);259 llvm::IntrusiveRefCntPtr<TargetInfo> Target =260 TargetInfo::CreateTargetInfo(Diags, *TargetOpts);261 return bool(Target);262}263 264std::optional<DriverInfo> parseDriverOutput(llvm::StringRef Output) {265 DriverInfo Info;266 const char SIS[] = "#include <...> search starts here:";267 const char SIE[] = "End of search list.";268 const char TS[] = "Target: ";269 llvm::SmallVector<llvm::StringRef> Lines;270 Output.split(Lines, '\n', /*MaxSplit=*/-1, /*KeepEmpty=*/false);271 272 enum {273 Initial, // Initial state: searching for target or includes list.274 IncludesExtracting, // Includes extracting.275 Done // Includes and target extraction done.276 } State = Initial;277 bool SeenIncludes = false;278 bool SeenTarget = false;279 for (auto *It = Lines.begin(); State != Done && It != Lines.end(); ++It) {280 auto Line = *It;281 switch (State) {282 case Initial:283 if (!SeenIncludes && Line.trim() == SIS) {284 SeenIncludes = true;285 State = IncludesExtracting;286 } else if (!SeenTarget && Line.trim().starts_with(TS)) {287 SeenTarget = true;288 llvm::StringRef TargetLine = Line.trim();289 TargetLine.consume_front(TS);290 // Only detect targets that clang understands291 if (!isValidTarget(TargetLine)) {292 elog("System include extraction: invalid target \"{0}\", ignoring",293 TargetLine);294 } else {295 Info.Target = TargetLine.str();296 vlog("System include extraction: target extracted: \"{0}\"",297 TargetLine);298 }299 }300 break;301 case IncludesExtracting:302 if (Line.trim() == SIE) {303 State = SeenTarget ? Done : Initial;304 } else {305 Info.SystemIncludes.push_back(Line.trim().str());306 vlog("System include extraction: adding {0}", Line);307 }308 break;309 default:310 llvm_unreachable("Impossible state of the driver output parser");311 break;312 }313 }314 if (!SeenIncludes) {315 elog("System include extraction: start marker not found: {0}", Output);316 return std::nullopt;317 }318 if (State == IncludesExtracting) {319 elog("System include extraction: end marker missing: {0}", Output);320 return std::nullopt;321 }322 return std::move(Info);323}324 325std::optional<std::string> run(llvm::ArrayRef<llvm::StringRef> Argv,326 bool OutputIsStderr) {327 llvm::SmallString<128> OutputPath;328 if (auto EC = llvm::sys::fs::createTemporaryFile("system-includes", "clangd",329 OutputPath)) {330 elog("System include extraction: failed to create temporary file with "331 "error {0}",332 EC.message());333 return std::nullopt;334 }335 auto CleanUp = llvm::make_scope_exit(336 [&OutputPath]() { llvm::sys::fs::remove(OutputPath); });337 338 std::optional<llvm::StringRef> Redirects[] = {{""}, {""}, {""}};339 Redirects[OutputIsStderr ? 2 : 1] = OutputPath.str();340 341 std::string ErrMsg;342 if (int RC =343 llvm::sys::ExecuteAndWait(Argv.front(), Argv, /*Env=*/std::nullopt,344 Redirects, /*SecondsToWait=*/0,345 /*MemoryLimit=*/0, &ErrMsg)) {346 elog("System include extraction: driver execution failed with return code: "347 "{0} - '{1}'. Args: [{2}]",348 llvm::to_string(RC), ErrMsg, printArgv(Argv));349 return std::nullopt;350 }351 352 auto BufOrError = llvm::MemoryBuffer::getFile(OutputPath);353 if (!BufOrError) {354 elog("System include extraction: failed to read {0} with error {1}",355 OutputPath, BufOrError.getError().message());356 return std::nullopt;357 }358 return BufOrError.get().get()->getBuffer().str();359}360 361std::optional<DriverInfo>362extractSystemIncludesAndTarget(const DriverArgs &InputArgs,363 const llvm::Regex &QueryDriverRegex) {364 trace::Span Tracer("Extract system includes and target");365 366 std::string Driver = InputArgs.Driver;367 if (!llvm::sys::path::is_absolute(Driver)) {368 auto DriverProgram = llvm::sys::findProgramByName(Driver);369 if (DriverProgram) {370 vlog("System include extraction: driver {0} expanded to {1}", Driver,371 *DriverProgram);372 Driver = *DriverProgram;373 } else {374 elog("System include extraction: driver {0} not found in PATH", Driver);375 return std::nullopt;376 }377 }378 379 SPAN_ATTACH(Tracer, "driver", Driver);380 SPAN_ATTACH(Tracer, "lang", InputArgs.Lang);381 382 // If driver was "../foo" then having to allowlist "/path/a/../foo" rather383 // than "/path/foo" is absurd.384 // Allow either to match the allowlist, then proceed with "/path/a/../foo".385 // This was our historical behavior, and it *could* resolve to something else.386 llvm::SmallString<256> NoDots(Driver);387 llvm::sys::path::remove_dots(NoDots, /*remove_dot_dot=*/true);388 if (!QueryDriverRegex.match(Driver) && !QueryDriverRegex.match(NoDots)) {389 vlog("System include extraction: not allowed driver {0}", Driver);390 return std::nullopt;391 }392 393 llvm::SmallVector<llvm::StringRef> Args = {Driver, "-E", "-v"};394 Args.append(InputArgs.render());395 // Input needs to go after Lang flags.396 Args.push_back("-");397 auto Output = run(Args, /*OutputIsStderr=*/true);398 if (!Output)399 return std::nullopt;400 401 std::optional<DriverInfo> Info = parseDriverOutput(*Output);402 if (!Info)403 return std::nullopt;404 405 switch (Config::current().CompileFlags.BuiltinHeaders) {406 case Config::BuiltinHeaderPolicy::Clangd: {407 // The built-in headers are tightly coupled to parser builtins.408 // (These are clang's "resource dir", GCC's GCC_INCLUDE_DIR.)409 // We should keep using clangd's versions, so exclude the queried410 // builtins. They're not specially marked in the -v output, but we can411 // get the path with `$DRIVER -print-file-name=include`.412 if (auto BuiltinHeaders = run({Driver, "-print-file-name=include"},413 /*OutputIsStderr=*/false)) {414 auto Path = llvm::StringRef(*BuiltinHeaders).trim();415 if (!Path.empty() && llvm::sys::path::is_absolute(Path)) {416 auto Size = Info->SystemIncludes.size();417 llvm::erase(Info->SystemIncludes, Path);418 vlog("System includes extractor: builtin headers {0} {1}", Path,419 (Info->SystemIncludes.size() != Size)420 ? "excluded"421 : "not found in driver's response");422 }423 }424 break;425 }426 case Config::BuiltinHeaderPolicy::QueryDriver:427 vlog("System includes extractor: Using builtin headers from query driver.");428 break;429 }430 431 log("System includes extractor: successfully executed {0}\n\tgot includes: "432 "\"{1}\"\n\tgot target: \"{2}\"",433 Driver, llvm::join(Info->SystemIncludes, ", "), Info->Target);434 return Info;435}436 437tooling::CompileCommand &438addSystemIncludes(tooling::CompileCommand &Cmd,439 llvm::ArrayRef<std::string> SystemIncludes) {440 std::vector<std::string> ToAppend;441 for (llvm::StringRef Include : SystemIncludes) {442 // FIXME(kadircet): This doesn't work when we have "--driver-mode=cl"443 ToAppend.push_back("-isystem");444 ToAppend.push_back(Include.str());445 }446 if (!ToAppend.empty()) {447 // Just append when `--` isn't present.448 auto InsertAt = llvm::find(Cmd.CommandLine, "--");449 Cmd.CommandLine.insert(InsertAt, std::make_move_iterator(ToAppend.begin()),450 std::make_move_iterator(ToAppend.end()));451 }452 return Cmd;453}454 455tooling::CompileCommand &setTarget(tooling::CompileCommand &Cmd,456 const std::string &Target) {457 if (!Target.empty()) {458 // We do not want to override existing target with extracted one.459 for (llvm::StringRef Arg : Cmd.CommandLine) {460 if (Arg == "-target" || Arg.starts_with("--target="))461 return Cmd;462 }463 // Just append when `--` isn't present.464 auto InsertAt = llvm::find(Cmd.CommandLine, "--");465 Cmd.CommandLine.insert(InsertAt, "--target=" + Target);466 }467 return Cmd;468}469 470/// Converts a glob containing only ** or * into a regex.471std::string convertGlobToRegex(llvm::StringRef Glob) {472 std::string RegText;473 llvm::raw_string_ostream RegStream(RegText);474 RegStream << '^';475 for (size_t I = 0, E = Glob.size(); I < E; ++I) {476 if (Glob[I] == '*') {477 if (I + 1 < E && Glob[I + 1] == '*') {478 // Double star, accept any sequence.479 RegStream << ".*";480 // Also skip the second star.481 ++I;482 } else {483 // Single star, accept any sequence without a slash.484 RegStream << "[^/]*";485 }486 } else if (llvm::sys::path::is_separator(Glob[I]) &&487 llvm::sys::path::is_separator('/') &&488 llvm::sys::path::is_separator('\\')) {489 RegStream << R"([/\\])"; // Accept either slash on windows.490 } else {491 RegStream << llvm::Regex::escape(Glob.substr(I, 1));492 }493 }494 RegStream << '$';495 return RegText;496}497 498/// Converts a glob containing only ** or * into a regex.499llvm::Regex convertGlobsToRegex(llvm::ArrayRef<std::string> Globs) {500 assert(!Globs.empty() && "Globs cannot be empty!");501 std::vector<std::string> RegTexts;502 RegTexts.reserve(Globs.size());503 for (llvm::StringRef Glob : Globs)504 RegTexts.push_back(convertGlobToRegex(Glob));505 506 // Tempting to pass IgnoreCase, but we don't know the FS sensitivity.507 llvm::Regex Reg(llvm::join(RegTexts, "|"));508 assert(Reg.isValid(RegTexts.front()) &&509 "Created an invalid regex from globs");510 return Reg;511}512 513/// Extracts system includes from a trusted driver by parsing the output of514/// include search path and appends them to the commands coming from underlying515/// compilation database.516class SystemIncludeExtractor {517public:518 SystemIncludeExtractor(llvm::ArrayRef<std::string> QueryDriverGlobs)519 : QueryDriverRegex(convertGlobsToRegex(QueryDriverGlobs)) {}520 521 void operator()(tooling::CompileCommand &Cmd, llvm::StringRef File) const {522 if (Cmd.CommandLine.empty())523 return;524 525 DriverArgs Args(Cmd, File);526 if (Args.Lang.empty())527 return;528 if (auto Info = QueriedDrivers.get(Args, [&] {529 return extractSystemIncludesAndTarget(Args, QueryDriverRegex);530 })) {531 setTarget(addSystemIncludes(Cmd, Info->SystemIncludes), Info->Target);532 }533 }534 535private:536 // Caches includes extracted from a driver. Key is driver:lang.537 Memoize<llvm::DenseMap<DriverArgs, std::optional<DriverInfo>>> QueriedDrivers;538 llvm::Regex QueryDriverRegex;539};540} // namespace541 542SystemIncludeExtractorFn543getSystemIncludeExtractor(llvm::ArrayRef<std::string> QueryDriverGlobs) {544 if (QueryDriverGlobs.empty())545 return nullptr;546 return SystemIncludeExtractor(QueryDriverGlobs);547}548 549} // namespace clang::clangd550