1051 lines · cpp
1//===--- Cuda.cpp - Cuda Tool and ToolChain Implementations -----*- C++ -*-===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8 9#include "Cuda.h"10#include "clang/Basic/Cuda.h"11#include "clang/Config/config.h"12#include "clang/Driver/CommonArgs.h"13#include "clang/Driver/Compilation.h"14#include "clang/Driver/Distro.h"15#include "clang/Driver/Driver.h"16#include "clang/Driver/InputInfo.h"17#include "clang/Options/Options.h"18#include "llvm/ADT/StringExtras.h"19#include "llvm/Config/llvm-config.h" // for LLVM_HOST_TRIPLE20#include "llvm/Option/ArgList.h"21#include "llvm/Support/FileSystem.h"22#include "llvm/Support/Path.h"23#include "llvm/Support/Process.h"24#include "llvm/Support/Program.h"25#include "llvm/Support/VirtualFileSystem.h"26#include "llvm/TargetParser/Host.h"27#include "llvm/TargetParser/TargetParser.h"28#include <system_error>29 30using namespace clang::driver;31using namespace clang::driver::toolchains;32using namespace clang::driver::tools;33using namespace clang;34using namespace llvm::opt;35 36namespace {37 38CudaVersion getCudaVersion(uint32_t raw_version) {39 if (raw_version < 7050)40 return CudaVersion::CUDA_70;41 if (raw_version < 8000)42 return CudaVersion::CUDA_75;43 if (raw_version < 9000)44 return CudaVersion::CUDA_80;45 if (raw_version < 9010)46 return CudaVersion::CUDA_90;47 if (raw_version < 9020)48 return CudaVersion::CUDA_91;49 if (raw_version < 10000)50 return CudaVersion::CUDA_92;51 if (raw_version < 10010)52 return CudaVersion::CUDA_100;53 if (raw_version < 10020)54 return CudaVersion::CUDA_101;55 if (raw_version < 11000)56 return CudaVersion::CUDA_102;57 if (raw_version < 11010)58 return CudaVersion::CUDA_110;59 if (raw_version < 11020)60 return CudaVersion::CUDA_111;61 if (raw_version < 11030)62 return CudaVersion::CUDA_112;63 if (raw_version < 11040)64 return CudaVersion::CUDA_113;65 if (raw_version < 11050)66 return CudaVersion::CUDA_114;67 if (raw_version < 11060)68 return CudaVersion::CUDA_115;69 if (raw_version < 11070)70 return CudaVersion::CUDA_116;71 if (raw_version < 11080)72 return CudaVersion::CUDA_117;73 if (raw_version < 11090)74 return CudaVersion::CUDA_118;75 if (raw_version < 12010)76 return CudaVersion::CUDA_120;77 if (raw_version < 12020)78 return CudaVersion::CUDA_121;79 if (raw_version < 12030)80 return CudaVersion::CUDA_122;81 if (raw_version < 12040)82 return CudaVersion::CUDA_123;83 if (raw_version < 12050)84 return CudaVersion::CUDA_124;85 if (raw_version < 12060)86 return CudaVersion::CUDA_125;87 if (raw_version < 12070)88 return CudaVersion::CUDA_126;89 if (raw_version < 12090)90 return CudaVersion::CUDA_128;91 if (raw_version < 13000)92 return CudaVersion::CUDA_129;93 return CudaVersion::NEW;94}95 96CudaVersion parseCudaHFile(llvm::StringRef Input) {97 // Helper lambda which skips the words if the line starts with them or returns98 // std::nullopt otherwise.99 auto StartsWithWords =100 [](llvm::StringRef Line,101 const SmallVector<StringRef, 3> words) -> std::optional<StringRef> {102 for (StringRef word : words) {103 if (!Line.consume_front(word))104 return {};105 Line = Line.ltrim();106 }107 return Line;108 };109 110 Input = Input.ltrim();111 while (!Input.empty()) {112 if (auto Line =113 StartsWithWords(Input.ltrim(), {"#", "define", "CUDA_VERSION"})) {114 uint32_t RawVersion;115 Line->consumeInteger(10, RawVersion);116 return getCudaVersion(RawVersion);117 }118 // Find next non-empty line.119 Input = Input.drop_front(Input.find_first_of("\n\r")).ltrim();120 }121 return CudaVersion::UNKNOWN;122}123} // namespace124 125void CudaInstallationDetector::WarnIfUnsupportedVersion() const {126 if (Version > CudaVersion::PARTIALLY_SUPPORTED) {127 std::string VersionString = CudaVersionToString(Version);128 if (!VersionString.empty())129 VersionString.insert(0, " ");130 D.Diag(diag::warn_drv_new_cuda_version)131 << VersionString132 << (CudaVersion::PARTIALLY_SUPPORTED != CudaVersion::FULLY_SUPPORTED)133 << CudaVersionToString(CudaVersion::PARTIALLY_SUPPORTED);134 } else if (Version > CudaVersion::FULLY_SUPPORTED)135 D.Diag(diag::warn_drv_partially_supported_cuda_version)136 << CudaVersionToString(Version);137}138 139CudaInstallationDetector::CudaInstallationDetector(140 const Driver &D, const llvm::Triple &HostTriple,141 const llvm::opt::ArgList &Args)142 : D(D) {143 struct Candidate {144 std::string Path;145 bool StrictChecking;146 147 Candidate(std::string Path, bool StrictChecking = false)148 : Path(Path), StrictChecking(StrictChecking) {}149 };150 SmallVector<Candidate, 4> Candidates;151 152 // In decreasing order so we prefer newer versions to older versions.153 std::initializer_list<const char *> Versions = {"8.0", "7.5", "7.0"};154 auto &FS = D.getVFS();155 156 if (Args.hasArg(options::OPT_cuda_path_EQ)) {157 Candidates.emplace_back(158 Args.getLastArgValue(options::OPT_cuda_path_EQ).str());159 } else if (HostTriple.isOSWindows()) {160 for (const char *Ver : Versions)161 Candidates.emplace_back(162 D.SysRoot + "/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v" +163 Ver);164 } else {165 if (!Args.hasArg(options::OPT_cuda_path_ignore_env)) {166 // Try to find ptxas binary. If the executable is located in a directory167 // called 'bin/', its parent directory might be a good guess for a valid168 // CUDA installation.169 // However, some distributions might installs 'ptxas' to /usr/bin. In that170 // case the candidate would be '/usr' which passes the following checks171 // because '/usr/include' exists as well. To avoid this case, we always172 // check for the directory potentially containing files for libdevice,173 // even if the user passes -nocudalib.174 if (llvm::ErrorOr<std::string> ptxas =175 llvm::sys::findProgramByName("ptxas")) {176 SmallString<256> ptxasAbsolutePath;177 llvm::sys::fs::real_path(*ptxas, ptxasAbsolutePath);178 179 StringRef ptxasDir = llvm::sys::path::parent_path(ptxasAbsolutePath);180 if (llvm::sys::path::filename(ptxasDir) == "bin")181 Candidates.emplace_back(182 std::string(llvm::sys::path::parent_path(ptxasDir)),183 /*StrictChecking=*/true);184 }185 }186 187 Candidates.emplace_back(D.SysRoot + "/usr/local/cuda");188 for (const char *Ver : Versions)189 Candidates.emplace_back(D.SysRoot + "/usr/local/cuda-" + Ver);190 191 Distro Dist(FS, llvm::Triple(llvm::sys::getProcessTriple()));192 if (Dist.IsDebian() || Dist.IsUbuntu())193 // Special case for Debian to have nvidia-cuda-toolkit work194 // out of the box. More info on http://bugs.debian.org/882505195 Candidates.emplace_back(D.SysRoot + "/usr/lib/cuda");196 }197 198 bool NoCudaLib =199 !Args.hasFlag(options::OPT_offloadlib, options::OPT_no_offloadlib, true);200 201 for (const auto &Candidate : Candidates) {202 InstallPath = Candidate.Path;203 if (InstallPath.empty() || !FS.exists(InstallPath))204 continue;205 206 BinPath = InstallPath + "/bin";207 IncludePath = InstallPath + "/include";208 LibDevicePath = InstallPath + "/nvvm/libdevice";209 210 if (!(FS.exists(IncludePath) && FS.exists(BinPath)))211 continue;212 bool CheckLibDevice = (!NoCudaLib || Candidate.StrictChecking);213 if (CheckLibDevice && !FS.exists(LibDevicePath))214 continue;215 216 Version = CudaVersion::UNKNOWN;217 if (auto CudaHFile = FS.getBufferForFile(InstallPath + "/include/cuda.h"))218 Version = parseCudaHFile((*CudaHFile)->getBuffer());219 // As the last resort, make an educated guess between CUDA-7.0, which had220 // old-style libdevice bitcode, and an unknown recent CUDA version.221 if (Version == CudaVersion::UNKNOWN) {222 Version = FS.exists(LibDevicePath + "/libdevice.10.bc")223 ? CudaVersion::NEW224 : CudaVersion::CUDA_70;225 }226 227 if (Version >= CudaVersion::CUDA_90) {228 // CUDA-9+ uses single libdevice file for all GPU variants.229 std::string FilePath = LibDevicePath + "/libdevice.10.bc";230 if (FS.exists(FilePath)) {231 for (int Arch = (int)OffloadArch::SM_30, E = (int)OffloadArch::LAST;232 Arch < E; ++Arch) {233 OffloadArch OA = static_cast<OffloadArch>(Arch);234 if (!IsNVIDIAOffloadArch(OA))235 continue;236 std::string OffloadArchName(OffloadArchToString(OA));237 LibDeviceMap[OffloadArchName] = FilePath;238 }239 }240 } else {241 std::error_code EC;242 for (llvm::vfs::directory_iterator LI = FS.dir_begin(LibDevicePath, EC),243 LE;244 !EC && LI != LE; LI = LI.increment(EC)) {245 StringRef FilePath = LI->path();246 StringRef FileName = llvm::sys::path::filename(FilePath);247 // Process all bitcode filenames that look like248 // libdevice.compute_XX.YY.bc249 const StringRef LibDeviceName = "libdevice.";250 if (!(FileName.starts_with(LibDeviceName) && FileName.ends_with(".bc")))251 continue;252 StringRef GpuArch = FileName.slice(253 LibDeviceName.size(), FileName.find('.', LibDeviceName.size()));254 LibDeviceMap[GpuArch] = FilePath.str();255 // Insert map entries for specific devices with this compute256 // capability. NVCC's choice of the libdevice library version is257 // rather peculiar and depends on the CUDA version.258 if (GpuArch == "compute_20") {259 LibDeviceMap["sm_20"] = std::string(FilePath);260 LibDeviceMap["sm_21"] = std::string(FilePath);261 LibDeviceMap["sm_32"] = std::string(FilePath);262 } else if (GpuArch == "compute_30") {263 LibDeviceMap["sm_30"] = std::string(FilePath);264 if (Version < CudaVersion::CUDA_80) {265 LibDeviceMap["sm_50"] = std::string(FilePath);266 LibDeviceMap["sm_52"] = std::string(FilePath);267 LibDeviceMap["sm_53"] = std::string(FilePath);268 }269 LibDeviceMap["sm_60"] = std::string(FilePath);270 LibDeviceMap["sm_61"] = std::string(FilePath);271 LibDeviceMap["sm_62"] = std::string(FilePath);272 } else if (GpuArch == "compute_35") {273 LibDeviceMap["sm_35"] = std::string(FilePath);274 LibDeviceMap["sm_37"] = std::string(FilePath);275 } else if (GpuArch == "compute_50") {276 if (Version >= CudaVersion::CUDA_80) {277 LibDeviceMap["sm_50"] = std::string(FilePath);278 LibDeviceMap["sm_52"] = std::string(FilePath);279 LibDeviceMap["sm_53"] = std::string(FilePath);280 }281 }282 }283 }284 285 // Check that we have found at least one libdevice that we can link in if286 // -nocudalib hasn't been specified.287 if (LibDeviceMap.empty() && !NoCudaLib)288 continue;289 290 IsValid = true;291 break;292 }293}294 295void CudaInstallationDetector::AddCudaIncludeArgs(296 const ArgList &DriverArgs, ArgStringList &CC1Args) const {297 if (!DriverArgs.hasArg(options::OPT_nobuiltininc)) {298 // Add cuda_wrappers/* to our system include path. This lets us wrap299 // standard library headers.300 SmallString<128> P(D.ResourceDir);301 llvm::sys::path::append(P, "include");302 llvm::sys::path::append(P, "cuda_wrappers");303 CC1Args.push_back("-internal-isystem");304 CC1Args.push_back(DriverArgs.MakeArgString(P));305 }306 307 if (!DriverArgs.hasFlag(options::OPT_offload_inc, options::OPT_no_offload_inc,308 true))309 return;310 311 if (!isValid()) {312 D.Diag(diag::err_drv_no_cuda_installation);313 return;314 }315 316 CC1Args.push_back("-include");317 CC1Args.push_back("__clang_cuda_runtime_wrapper.h");318}319 320void CudaInstallationDetector::CheckCudaVersionSupportsArch(321 OffloadArch Arch) const {322 if (Arch == OffloadArch::UNKNOWN || Version == CudaVersion::UNKNOWN ||323 ArchsWithBadVersion[(int)Arch])324 return;325 326 auto MinVersion = MinVersionForOffloadArch(Arch);327 auto MaxVersion = MaxVersionForOffloadArch(Arch);328 if (Version < MinVersion || Version > MaxVersion) {329 ArchsWithBadVersion[(int)Arch] = true;330 D.Diag(diag::err_drv_cuda_version_unsupported)331 << OffloadArchToString(Arch) << CudaVersionToString(MinVersion)332 << CudaVersionToString(MaxVersion) << InstallPath333 << CudaVersionToString(Version);334 }335}336 337void CudaInstallationDetector::print(raw_ostream &OS) const {338 if (isValid())339 OS << "Found CUDA installation: " << InstallPath << ", version "340 << CudaVersionToString(Version) << "\n";341}342 343namespace {344/// Debug info level for the NVPTX devices. We may need to emit different debug345/// info level for the host and for the device itselfi. This type controls346/// emission of the debug info for the devices. It either prohibits disable info347/// emission completely, or emits debug directives only, or emits same debug348/// info as for the host.349enum DeviceDebugInfoLevel {350 DisableDebugInfo, /// Do not emit debug info for the devices.351 DebugDirectivesOnly, /// Emit only debug directives.352 EmitSameDebugInfoAsHost, /// Use the same debug info level just like for the353 /// host.354};355} // anonymous namespace356 357/// Define debug info level for the NVPTX devices. If the debug info for both358/// the host and device are disabled (-g0/-ggdb0 or no debug options at all). If359/// only debug directives are requested for the both host and device360/// (-gline-directvies-only), or the debug info only for the device is disabled361/// (optimization is on and --cuda-noopt-device-debug was not specified), the362/// debug directves only must be emitted for the device. Otherwise, use the same363/// debug info level just like for the host (with the limitations of only364/// supported DWARF2 standard).365static DeviceDebugInfoLevel mustEmitDebugInfo(const ArgList &Args) {366 const Arg *A = Args.getLastArg(options::OPT_O_Group);367 bool IsDebugEnabled = !A || A->getOption().matches(options::OPT_O0) ||368 Args.hasFlag(options::OPT_cuda_noopt_device_debug,369 options::OPT_no_cuda_noopt_device_debug,370 /*Default=*/false);371 if (const Arg *A = Args.getLastArg(options::OPT_g_Group)) {372 const Option &Opt = A->getOption();373 if (Opt.matches(options::OPT_gN_Group)) {374 if (Opt.matches(options::OPT_g0) || Opt.matches(options::OPT_ggdb0))375 return DisableDebugInfo;376 if (Opt.matches(options::OPT_gline_directives_only))377 return DebugDirectivesOnly;378 }379 return IsDebugEnabled ? EmitSameDebugInfoAsHost : DebugDirectivesOnly;380 }381 return willEmitRemarks(Args) ? DebugDirectivesOnly : DisableDebugInfo;382}383 384void NVPTX::Assembler::ConstructJob(Compilation &C, const JobAction &JA,385 const InputInfo &Output,386 const InputInfoList &Inputs,387 const ArgList &Args,388 const char *LinkingOutput) const {389 const auto &TC =390 static_cast<const toolchains::NVPTXToolChain &>(getToolChain());391 assert(TC.getTriple().isNVPTX() && "Wrong platform");392 393 StringRef GPUArchName;394 // If this is a CUDA action we need to extract the device architecture395 // from the Job's associated architecture, otherwise use the -march=arch396 // option. This option may come from -Xopenmp-target flag or the default397 // value.398 if (JA.isDeviceOffloading(Action::OFK_Cuda)) {399 GPUArchName = JA.getOffloadingArch();400 } else {401 GPUArchName = Args.getLastArgValue(options::OPT_march_EQ);402 if (GPUArchName.empty()) {403 C.getDriver().Diag(diag::err_drv_offload_missing_gpu_arch)404 << getToolChain().getArchName() << getShortName();405 return;406 }407 }408 409 // Obtain architecture from the action.410 OffloadArch gpu_arch = StringToOffloadArch(GPUArchName);411 assert(gpu_arch != OffloadArch::UNKNOWN &&412 "Device action expected to have an architecture.");413 414 // Check that our installation's ptxas supports gpu_arch.415 if (!Args.hasArg(options::OPT_no_cuda_version_check)) {416 TC.CudaInstallation.CheckCudaVersionSupportsArch(gpu_arch);417 }418 419 ArgStringList CmdArgs;420 CmdArgs.push_back(TC.getTriple().isArch64Bit() ? "-m64" : "-m32");421 DeviceDebugInfoLevel DIKind = mustEmitDebugInfo(Args);422 if (DIKind == EmitSameDebugInfoAsHost) {423 // ptxas does not accept -g option if optimization is enabled, so424 // we ignore the compiler's -O* options if we want debug info.425 CmdArgs.push_back("-g");426 CmdArgs.push_back("--dont-merge-basicblocks");427 CmdArgs.push_back("--return-at-end");428 } else if (Arg *A = Args.getLastArg(options::OPT_O_Group)) {429 // Map the -O we received to -O{0,1,2,3}.430 //431 // TODO: Perhaps we should map host -O2 to ptxas -O3. -O3 is ptxas's432 // default, so it may correspond more closely to the spirit of clang -O2.433 434 // -O3 seems like the least-bad option when -Osomething is specified to435 // clang but it isn't handled below.436 StringRef OOpt = "3";437 if (A->getOption().matches(options::OPT_O4) ||438 A->getOption().matches(options::OPT_Ofast))439 OOpt = "3";440 else if (A->getOption().matches(options::OPT_O0))441 OOpt = "0";442 else if (A->getOption().matches(options::OPT_O)) {443 // -Os, -Oz, and -O(anything else) map to -O2, for lack of better options.444 OOpt = llvm::StringSwitch<const char *>(A->getValue())445 .Case("1", "1")446 .Case("2", "2")447 .Case("3", "3")448 .Case("s", "2")449 .Case("z", "2")450 .Default("2");451 }452 CmdArgs.push_back(Args.MakeArgString(llvm::Twine("-O") + OOpt));453 } else {454 // If no -O was passed, pass -O0 to ptxas -- no opt flag should correspond455 // to no optimizations, but ptxas's default is -O3.456 CmdArgs.push_back("-O0");457 }458 if (DIKind == DebugDirectivesOnly)459 CmdArgs.push_back("-lineinfo");460 461 // Pass -v to ptxas if it was passed to the driver.462 if (Args.hasArg(options::OPT_v))463 CmdArgs.push_back("-v");464 465 CmdArgs.push_back("--gpu-name");466 CmdArgs.push_back(Args.MakeArgString(OffloadArchToString(gpu_arch)));467 CmdArgs.push_back("--output-file");468 std::string OutputFileName = TC.getInputFilename(Output);469 470 if (Output.isFilename() && OutputFileName != Output.getFilename())471 C.addTempFile(Args.MakeArgString(OutputFileName));472 473 CmdArgs.push_back(Args.MakeArgString(OutputFileName));474 for (const auto &II : Inputs)475 CmdArgs.push_back(Args.MakeArgString(II.getFilename()));476 477 for (const auto &A : Args.getAllArgValues(options::OPT_Xcuda_ptxas))478 CmdArgs.push_back(Args.MakeArgString(A));479 480 bool Relocatable;481 if (JA.isOffloading(Action::OFK_OpenMP))482 // In OpenMP we need to generate relocatable code.483 Relocatable = Args.hasFlag(options::OPT_fopenmp_relocatable_target,484 options::OPT_fnoopenmp_relocatable_target,485 /*Default=*/true);486 else if (JA.isOffloading(Action::OFK_Cuda))487 // In CUDA we generate relocatable code by default.488 Relocatable = Args.hasFlag(options::OPT_fgpu_rdc, options::OPT_fno_gpu_rdc,489 /*Default=*/false);490 else491 // Otherwise, we are compiling directly and should create linkable output.492 Relocatable = true;493 494 if (Relocatable)495 CmdArgs.push_back("-c");496 497 const char *Exec;498 if (Arg *A = Args.getLastArg(options::OPT_ptxas_path_EQ))499 Exec = A->getValue();500 else501 Exec = Args.MakeArgString(TC.GetProgramPath("ptxas"));502 C.addCommand(std::make_unique<Command>(503 JA, *this,504 ResponseFileSupport{ResponseFileSupport::RF_Full, llvm::sys::WEM_UTF8,505 "--options-file"},506 Exec, CmdArgs, Inputs, Output));507}508 509static bool shouldIncludePTX(const ArgList &Args, StringRef InputArch) {510 // The new driver does not include PTX by default to avoid overhead.511 bool includePTX = !Args.hasFlag(options::OPT_offload_new_driver,512 options::OPT_no_offload_new_driver, true);513 for (Arg *A : Args.filtered(options::OPT_cuda_include_ptx_EQ,514 options::OPT_no_cuda_include_ptx_EQ)) {515 A->claim();516 const StringRef ArchStr = A->getValue();517 if (A->getOption().matches(options::OPT_cuda_include_ptx_EQ) &&518 (ArchStr == "all" || ArchStr == InputArch))519 includePTX = true;520 else if (A->getOption().matches(options::OPT_no_cuda_include_ptx_EQ) &&521 (ArchStr == "all" || ArchStr == InputArch))522 includePTX = false;523 }524 return includePTX;525}526 527// All inputs to this linker must be from CudaDeviceActions, as we need to look528// at the Inputs' Actions in order to figure out which GPU architecture they529// correspond to.530void NVPTX::FatBinary::ConstructJob(Compilation &C, const JobAction &JA,531 const InputInfo &Output,532 const InputInfoList &Inputs,533 const ArgList &Args,534 const char *LinkingOutput) const {535 const auto &TC =536 static_cast<const toolchains::CudaToolChain &>(getToolChain());537 assert(TC.getTriple().isNVPTX() && "Wrong platform");538 539 ArgStringList CmdArgs;540 if (TC.CudaInstallation.version() <= CudaVersion::CUDA_100)541 CmdArgs.push_back("--cuda");542 CmdArgs.push_back(TC.getTriple().isArch64Bit() ? "-64" : "-32");543 CmdArgs.push_back(Args.MakeArgString("--create"));544 CmdArgs.push_back(Args.MakeArgString(Output.getFilename()));545 if (mustEmitDebugInfo(Args) == EmitSameDebugInfoAsHost)546 CmdArgs.push_back("-g");547 548 for (const auto &II : Inputs) {549 auto *A = II.getAction();550 assert(A->getInputs().size() == 1 &&551 "Device offload action is expected to have a single input");552 StringRef GpuArch = A->getOffloadingArch();553 assert(!GpuArch.empty() &&554 "Device action expected to have associated a GPU architecture!");555 556 if (II.getType() == types::TY_PP_Asm && !shouldIncludePTX(Args, GpuArch))557 continue;558 StringRef Kind = (II.getType() == types::TY_PP_Asm) ? "ptx" : "elf";559 CmdArgs.push_back(Args.MakeArgString(560 "--image3=kind=" + Kind + ",sm=" + GpuArch.drop_front(3) +561 ",file=" + getToolChain().getInputFilename(II)));562 }563 564 for (const auto &A : Args.getAllArgValues(options::OPT_Xcuda_fatbinary))565 CmdArgs.push_back(Args.MakeArgString(A));566 567 const char *Exec = Args.MakeArgString(TC.GetProgramPath("fatbinary"));568 C.addCommand(std::make_unique<Command>(569 JA, *this,570 ResponseFileSupport{ResponseFileSupport::RF_Full, llvm::sys::WEM_UTF8,571 "--options-file"},572 Exec, CmdArgs, Inputs, Output));573}574 575void NVPTX::Linker::ConstructJob(Compilation &C, const JobAction &JA,576 const InputInfo &Output,577 const InputInfoList &Inputs,578 const ArgList &Args,579 const char *LinkingOutput) const {580 const auto &TC =581 static_cast<const toolchains::NVPTXToolChain &>(getToolChain());582 ArgStringList CmdArgs;583 584 assert(TC.getTriple().isNVPTX() && "Wrong platform");585 586 assert((Output.isFilename() || Output.isNothing()) && "Invalid output.");587 if (Output.isFilename()) {588 CmdArgs.push_back("-o");589 CmdArgs.push_back(Output.getFilename());590 }591 592 if (mustEmitDebugInfo(Args) == EmitSameDebugInfoAsHost)593 CmdArgs.push_back("-g");594 595 if (Args.hasArg(options::OPT_v))596 CmdArgs.push_back("-v");597 598 StringRef GPUArch = Args.getLastArgValue(options::OPT_march_EQ);599 if (GPUArch.empty() && !C.getDriver().isUsingLTO()) {600 C.getDriver().Diag(diag::err_drv_offload_missing_gpu_arch)601 << getToolChain().getArchName() << getShortName();602 return;603 }604 605 if (!GPUArch.empty()) {606 CmdArgs.push_back("-arch");607 CmdArgs.push_back(Args.MakeArgString(GPUArch));608 }609 610 if (Args.hasArg(options::OPT_ptxas_path_EQ))611 CmdArgs.push_back(Args.MakeArgString(612 "--pxtas-path=" + Args.getLastArgValue(options::OPT_ptxas_path_EQ)));613 614 if (Args.hasArg(options::OPT_cuda_path_EQ) || TC.CudaInstallation.isValid()) {615 StringRef CudaPath = Args.getLastArgValue(616 options::OPT_cuda_path_EQ,617 llvm::sys::path::parent_path(TC.CudaInstallation.getBinPath()));618 CmdArgs.push_back(Args.MakeArgString("--cuda-path=" + CudaPath));619 }620 621 // Add paths specified in LIBRARY_PATH environment variable as -L options.622 addDirectoryList(Args, CmdArgs, "-L", "LIBRARY_PATH");623 624 // Add standard library search paths passed on the command line.625 Args.AddAllArgs(CmdArgs, options::OPT_L);626 getToolChain().AddFilePathLibArgs(Args, CmdArgs);627 AddLinkerInputs(getToolChain(), Inputs, Args, CmdArgs, JA);628 629 if (C.getDriver().isUsingLTO())630 addLTOOptions(getToolChain(), Args, CmdArgs, Output, Inputs,631 C.getDriver().getLTOMode() == LTOK_Thin);632 633 // Forward the PTX features if the nvlink-wrapper needs it.634 std::vector<StringRef> Features;635 getNVPTXTargetFeatures(C.getDriver(), getToolChain().getTriple(), Args,636 Features);637 CmdArgs.push_back(638 Args.MakeArgString("--plugin-opt=-mattr=" + llvm::join(Features, ",")));639 640 // Add paths for the default clang library path.641 SmallString<256> DefaultLibPath =642 llvm::sys::path::parent_path(TC.getDriver().Dir);643 llvm::sys::path::append(DefaultLibPath, CLANG_INSTALL_LIBDIR_BASENAME);644 CmdArgs.push_back(Args.MakeArgString(Twine("-L") + DefaultLibPath));645 646 if (Args.hasArg(options::OPT_stdlib))647 CmdArgs.append({"-lc", "-lm"});648 if (Args.hasArg(options::OPT_startfiles)) {649 std::optional<std::string> IncludePath = getToolChain().getStdlibPath();650 if (!IncludePath)651 IncludePath = "/lib";652 SmallString<128> P(*IncludePath);653 llvm::sys::path::append(P, "crt1.o");654 CmdArgs.push_back(Args.MakeArgString(P));655 }656 657 C.addCommand(std::make_unique<Command>(658 JA, *this,659 ResponseFileSupport{ResponseFileSupport::RF_Full, llvm::sys::WEM_UTF8,660 "--options-file"},661 Args.MakeArgString(getToolChain().GetProgramPath("clang-nvlink-wrapper")),662 CmdArgs, Inputs, Output));663}664 665void NVPTX::getNVPTXTargetFeatures(const Driver &D, const llvm::Triple &Triple,666 const llvm::opt::ArgList &Args,667 std::vector<StringRef> &Features) {668 if (Args.hasArg(options::OPT_cuda_feature_EQ)) {669 StringRef PtxFeature =670 Args.getLastArgValue(options::OPT_cuda_feature_EQ, "+ptx42");671 Features.push_back(Args.MakeArgString(PtxFeature));672 return;673 }674 CudaInstallationDetector CudaInstallation(D, Triple, Args);675 676 // New CUDA versions often introduce new instructions that are only supported677 // by new PTX version, so we need to raise PTX level to enable them in NVPTX678 // back-end.679 const char *PtxFeature = nullptr;680 switch (CudaInstallation.version()) {681#define CASE_CUDA_VERSION(CUDA_VER, PTX_VER) \682 case CudaVersion::CUDA_##CUDA_VER: \683 PtxFeature = "+ptx" #PTX_VER; \684 break;685 CASE_CUDA_VERSION(129, 88);686 CASE_CUDA_VERSION(128, 87);687 CASE_CUDA_VERSION(126, 85);688 CASE_CUDA_VERSION(125, 85);689 CASE_CUDA_VERSION(124, 84);690 CASE_CUDA_VERSION(123, 83);691 CASE_CUDA_VERSION(122, 82);692 CASE_CUDA_VERSION(121, 81);693 CASE_CUDA_VERSION(120, 80);694 CASE_CUDA_VERSION(118, 78);695 CASE_CUDA_VERSION(117, 77);696 CASE_CUDA_VERSION(116, 76);697 CASE_CUDA_VERSION(115, 75);698 CASE_CUDA_VERSION(114, 74);699 CASE_CUDA_VERSION(113, 73);700 CASE_CUDA_VERSION(112, 72);701 CASE_CUDA_VERSION(111, 71);702 CASE_CUDA_VERSION(110, 70);703 CASE_CUDA_VERSION(102, 65);704 CASE_CUDA_VERSION(101, 64);705 CASE_CUDA_VERSION(100, 63);706 CASE_CUDA_VERSION(92, 61);707 CASE_CUDA_VERSION(91, 61);708 CASE_CUDA_VERSION(90, 60);709#undef CASE_CUDA_VERSION710 // TODO: Use specific CUDA version once it's public.711 case clang::CudaVersion::NEW:712 PtxFeature = "+ptx86";713 break;714 default:715 PtxFeature = "+ptx42";716 }717 Features.push_back(PtxFeature);718}719 720/// NVPTX toolchain. Our assembler is ptxas, and our linker is nvlink. This721/// operates as a stand-alone version of the NVPTX tools without the host722/// toolchain.723NVPTXToolChain::NVPTXToolChain(const Driver &D, const llvm::Triple &Triple,724 const llvm::Triple &HostTriple,725 const ArgList &Args)726 : ToolChain(D, Triple, Args), CudaInstallation(D, HostTriple, Args) {727 if (CudaInstallation.isValid())728 getProgramPaths().push_back(std::string(CudaInstallation.getBinPath()));729 // Lookup binaries into the driver directory, this is used to730 // discover the 'nvptx-arch' executable.731 getProgramPaths().push_back(getDriver().Dir);732}733 734/// We only need the host triple to locate the CUDA binary utilities, use the735/// system's default triple if not provided.736NVPTXToolChain::NVPTXToolChain(const Driver &D, const llvm::Triple &Triple,737 const ArgList &Args)738 : NVPTXToolChain(D, Triple, llvm::Triple(LLVM_HOST_TRIPLE), Args) {}739 740llvm::opt::DerivedArgList *741NVPTXToolChain::TranslateArgs(const llvm::opt::DerivedArgList &Args,742 StringRef BoundArch,743 Action::OffloadKind OffloadKind) const {744 DerivedArgList *DAL = ToolChain::TranslateArgs(Args, BoundArch, OffloadKind);745 if (!DAL)746 DAL = new DerivedArgList(Args.getBaseArgs());747 748 const OptTable &Opts = getDriver().getOpts();749 750 for (Arg *A : Args)751 if (!llvm::is_contained(*DAL, A))752 DAL->append(A);753 754 if (!DAL->hasArg(options::OPT_march_EQ) && OffloadKind != Action::OFK_None) {755 DAL->AddJoinedArg(nullptr, Opts.getOption(options::OPT_march_EQ),756 OffloadArchToString(OffloadArch::CudaDefault));757 } else if (DAL->getLastArgValue(options::OPT_march_EQ) == "generic" &&758 OffloadKind == Action::OFK_None) {759 DAL->eraseArg(options::OPT_march_EQ);760 } else if (DAL->getLastArgValue(options::OPT_march_EQ) == "native") {761 auto GPUsOrErr = getSystemGPUArchs(Args);762 if (!GPUsOrErr) {763 getDriver().Diag(diag::err_drv_undetermined_gpu_arch)764 << getArchName() << llvm::toString(GPUsOrErr.takeError()) << "-march";765 } else {766 if (GPUsOrErr->size() > 1)767 getDriver().Diag(diag::warn_drv_multi_gpu_arch)768 << getArchName() << llvm::join(*GPUsOrErr, ", ") << "-march";769 DAL->AddJoinedArg(nullptr, Opts.getOption(options::OPT_march_EQ),770 Args.MakeArgString(GPUsOrErr->front()));771 }772 }773 774 return DAL;775}776 777void NVPTXToolChain::addClangTargetOptions(778 const llvm::opt::ArgList &DriverArgs, llvm::opt::ArgStringList &CC1Args,779 Action::OffloadKind DeviceOffloadingKind) const {}780 781void NVPTXToolChain::AddClangSystemIncludeArgs(const ArgList &DriverArgs,782 ArgStringList &CC1Args) const {783 if (DriverArgs.hasArg(options::OPT_nostdinc) ||784 DriverArgs.hasArg(options::OPT_nostdlibinc))785 return;786 787 if (std::optional<std::string> Path = getStdlibIncludePath())788 addSystemInclude(DriverArgs, CC1Args, *Path);789}790 791bool NVPTXToolChain::supportsDebugInfoOption(const llvm::opt::Arg *A) const {792 const Option &O = A->getOption();793 return (O.matches(options::OPT_gN_Group) &&794 !O.matches(options::OPT_gmodules)) ||795 O.matches(options::OPT_g_Flag) ||796 O.matches(options::OPT_ggdbN_Group) || O.matches(options::OPT_ggdb) ||797 O.matches(options::OPT_gdwarf) || O.matches(options::OPT_gdwarf_2) ||798 O.matches(options::OPT_gdwarf_3) || O.matches(options::OPT_gdwarf_4) ||799 O.matches(options::OPT_gdwarf_5) ||800 O.matches(options::OPT_gcolumn_info);801}802 803void NVPTXToolChain::adjustDebugInfoKind(804 llvm::codegenoptions::DebugInfoKind &DebugInfoKind,805 const ArgList &Args) const {806 switch (mustEmitDebugInfo(Args)) {807 case DisableDebugInfo:808 DebugInfoKind = llvm::codegenoptions::NoDebugInfo;809 break;810 case DebugDirectivesOnly:811 DebugInfoKind = llvm::codegenoptions::DebugDirectivesOnly;812 break;813 case EmitSameDebugInfoAsHost:814 // Use same debug info level as the host.815 break;816 }817}818 819Expected<SmallVector<std::string>>820NVPTXToolChain::getSystemGPUArchs(const ArgList &Args) const {821 // Detect NVIDIA GPUs availible on the system.822 std::string Program;823 if (Arg *A = Args.getLastArg(options::OPT_offload_arch_tool_EQ))824 Program = A->getValue();825 else826 Program = GetProgramPath("nvptx-arch");827 828 auto StdoutOrErr = getDriver().executeProgram({Program});829 if (!StdoutOrErr)830 return StdoutOrErr.takeError();831 832 SmallVector<std::string, 1> GPUArchs;833 for (StringRef Arch : llvm::split((*StdoutOrErr)->getBuffer(), "\n"))834 if (!Arch.empty())835 GPUArchs.push_back(Arch.str());836 837 if (GPUArchs.empty())838 return llvm::createStringError(std::error_code(),839 "No NVIDIA GPU detected in the system");840 841 return std::move(GPUArchs);842}843 844/// CUDA toolchain. Our assembler is ptxas, and our "linker" is fatbinary,845/// which isn't properly a linker but nonetheless performs the step of stitching846/// together object files from the assembler into a single blob.847 848CudaToolChain::CudaToolChain(const Driver &D, const llvm::Triple &Triple,849 const ToolChain &HostTC, const ArgList &Args)850 : NVPTXToolChain(D, Triple, HostTC.getTriple(), Args), HostTC(HostTC) {}851 852void CudaToolChain::addClangTargetOptions(853 const llvm::opt::ArgList &DriverArgs, llvm::opt::ArgStringList &CC1Args,854 Action::OffloadKind DeviceOffloadingKind) const {855 HostTC.addClangTargetOptions(DriverArgs, CC1Args, DeviceOffloadingKind);856 857 StringRef GpuArch = DriverArgs.getLastArgValue(options::OPT_march_EQ);858 assert((DeviceOffloadingKind == Action::OFK_OpenMP ||859 DeviceOffloadingKind == Action::OFK_Cuda) &&860 "Only OpenMP or CUDA offloading kinds are supported for NVIDIA GPUs.");861 862 CC1Args.append({"-fcuda-is-device", "-mllvm",863 "-enable-memcpyopt-without-libcalls",864 "-fno-threadsafe-statics"});865 866 // Unsized function arguments used for variadics were introduced in CUDA-9.0867 // We still do not support generating code that actually uses variadic868 // arguments yet, but we do need to allow parsing them as recent CUDA869 // headers rely on that. https://github.com/llvm/llvm-project/issues/58410870 if (CudaInstallation.version() >= CudaVersion::CUDA_90)871 CC1Args.push_back("-fcuda-allow-variadic-functions");872 873 if (DriverArgs.hasFlag(options::OPT_fcuda_short_ptr,874 options::OPT_fno_cuda_short_ptr, false))875 CC1Args.append({"-mllvm", "--nvptx-short-ptr"});876 877 if (!DriverArgs.hasFlag(options::OPT_offloadlib, options::OPT_no_offloadlib,878 true))879 return;880 881 if (DeviceOffloadingKind == Action::OFK_OpenMP &&882 DriverArgs.hasArg(options::OPT_S))883 return;884 885 std::string LibDeviceFile = CudaInstallation.getLibDeviceFile(GpuArch);886 if (LibDeviceFile.empty()) {887 getDriver().Diag(diag::err_drv_no_cuda_libdevice) << GpuArch;888 return;889 }890 891 CC1Args.push_back("-mlink-builtin-bitcode");892 CC1Args.push_back(DriverArgs.MakeArgString(LibDeviceFile));893 894 // For now, we don't use any Offload/OpenMP device runtime when we offload895 // CUDA via LLVM/Offload. We should split the Offload/OpenMP device runtime896 // and include the "generic" (or CUDA-specific) parts.897 if (DriverArgs.hasFlag(options::OPT_foffload_via_llvm,898 options::OPT_fno_offload_via_llvm, false))899 return;900 901 clang::CudaVersion CudaInstallationVersion = CudaInstallation.version();902 903 if (CudaInstallationVersion >= CudaVersion::UNKNOWN)904 CC1Args.push_back(905 DriverArgs.MakeArgString(Twine("-target-sdk-version=") +906 CudaVersionToString(CudaInstallationVersion)));907 908 if (DeviceOffloadingKind == Action::OFK_OpenMP) {909 if (CudaInstallationVersion < CudaVersion::CUDA_92) {910 getDriver().Diag(911 diag::err_drv_omp_offload_target_cuda_version_not_support)912 << CudaVersionToString(CudaInstallationVersion);913 return;914 }915 916 // Link the bitcode library late if we're using device LTO.917 if (getDriver().isUsingOffloadLTO())918 return;919 920 addOpenMPDeviceRTL(getDriver(), DriverArgs, CC1Args, GpuArch.str(),921 getTriple(), HostTC);922 }923}924 925llvm::DenormalMode CudaToolChain::getDefaultDenormalModeForType(926 const llvm::opt::ArgList &DriverArgs, const JobAction &JA,927 const llvm::fltSemantics *FPType) const {928 if (JA.getOffloadingDeviceKind() == Action::OFK_Cuda) {929 if (FPType && FPType == &llvm::APFloat::IEEEsingle() &&930 DriverArgs.hasFlag(options::OPT_fgpu_flush_denormals_to_zero,931 options::OPT_fno_gpu_flush_denormals_to_zero, false))932 return llvm::DenormalMode::getPreserveSign();933 }934 935 assert(JA.getOffloadingDeviceKind() != Action::OFK_Host);936 return llvm::DenormalMode::getIEEE();937}938 939void CudaToolChain::AddCudaIncludeArgs(const ArgList &DriverArgs,940 ArgStringList &CC1Args) const {941 // Check our CUDA version if we're going to include the CUDA headers.942 if (DriverArgs.hasFlag(options::OPT_offload_inc, options::OPT_no_offload_inc,943 true) &&944 !DriverArgs.hasArg(options::OPT_no_cuda_version_check)) {945 StringRef Arch = DriverArgs.getLastArgValue(options::OPT_march_EQ);946 assert(!Arch.empty() && "Must have an explicit GPU arch.");947 CudaInstallation.CheckCudaVersionSupportsArch(StringToOffloadArch(Arch));948 }949 CudaInstallation.AddCudaIncludeArgs(DriverArgs, CC1Args);950}951 952std::string CudaToolChain::getInputFilename(const InputInfo &Input) const {953 // Only object files are changed, for example assembly files keep their .s954 // extensions. If the user requested device-only compilation don't change it.955 if (Input.getType() != types::TY_Object || getDriver().offloadDeviceOnly())956 return ToolChain::getInputFilename(Input);957 958 return ToolChain::getInputFilename(Input);959}960 961llvm::opt::DerivedArgList *962CudaToolChain::TranslateArgs(const llvm::opt::DerivedArgList &Args,963 StringRef BoundArch,964 Action::OffloadKind DeviceOffloadKind) const {965 DerivedArgList *DAL =966 HostTC.TranslateArgs(Args, BoundArch, DeviceOffloadKind);967 if (!DAL)968 DAL = new DerivedArgList(Args.getBaseArgs());969 970 const OptTable &Opts = getDriver().getOpts();971 972 for (Arg *A : Args) {973 // Make sure flags are not duplicated.974 if (!llvm::is_contained(*DAL, A)) {975 DAL->append(A);976 }977 }978 979 if (!BoundArch.empty()) {980 DAL->eraseArg(options::OPT_march_EQ);981 DAL->AddJoinedArg(nullptr, Opts.getOption(options::OPT_march_EQ),982 BoundArch);983 }984 return DAL;985}986 987Tool *NVPTXToolChain::buildAssembler() const {988 return new tools::NVPTX::Assembler(*this);989}990 991Tool *NVPTXToolChain::buildLinker() const {992 return new tools::NVPTX::Linker(*this);993}994 995Tool *CudaToolChain::buildAssembler() const {996 return new tools::NVPTX::Assembler(*this);997}998 999Tool *CudaToolChain::buildLinker() const {1000 return new tools::NVPTX::FatBinary(*this);1001}1002 1003void CudaToolChain::addClangWarningOptions(ArgStringList &CC1Args) const {1004 HostTC.addClangWarningOptions(CC1Args);1005}1006 1007ToolChain::CXXStdlibType1008CudaToolChain::GetCXXStdlibType(const ArgList &Args) const {1009 return HostTC.GetCXXStdlibType(Args);1010}1011 1012void CudaToolChain::AddClangSystemIncludeArgs(const ArgList &DriverArgs,1013 ArgStringList &CC1Args) const {1014 HostTC.AddClangSystemIncludeArgs(DriverArgs, CC1Args);1015 1016 if (DriverArgs.hasFlag(options::OPT_offload_inc, options::OPT_no_offload_inc,1017 true) &&1018 CudaInstallation.isValid())1019 CC1Args.append(1020 {"-internal-isystem",1021 DriverArgs.MakeArgString(CudaInstallation.getIncludePath())});1022}1023 1024void CudaToolChain::AddClangCXXStdlibIncludeArgs(const ArgList &Args,1025 ArgStringList &CC1Args) const {1026 HostTC.AddClangCXXStdlibIncludeArgs(Args, CC1Args);1027}1028 1029void CudaToolChain::AddIAMCUIncludeArgs(const ArgList &Args,1030 ArgStringList &CC1Args) const {1031 HostTC.AddIAMCUIncludeArgs(Args, CC1Args);1032}1033 1034SanitizerMask CudaToolChain::getSupportedSanitizers() const {1035 // The CudaToolChain only supports sanitizers in the sense that it allows1036 // sanitizer arguments on the command line if they are supported by the host1037 // toolchain. The CudaToolChain will actually ignore any command line1038 // arguments for any of these "supported" sanitizers. That means that no1039 // sanitization of device code is actually supported at this time.1040 //1041 // This behavior is necessary because the host and device toolchains1042 // invocations often share the command line, so the device toolchain must1043 // tolerate flags meant only for the host toolchain.1044 return HostTC.getSupportedSanitizers();1045}1046 1047VersionTuple CudaToolChain::computeMSVCVersion(const Driver *D,1048 const ArgList &Args) const {1049 return HostTC.computeMSVCVersion(D, Args);1050}1051