608 lines · cpp
1//===----------------------------------------------------------------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8///9/// \file10/// This file implements the VirtualOutputBackend types, including:11/// * NullOutputBackend: Outputs to NullOutputBackend are discarded.12/// * FilteringOutputBackend: Filter paths from output.13/// * MirroringOutputBackend: Mirror the output into two different backend.14/// * OnDiskOutputBackend: Write output files to disk.15///16//===----------------------------------------------------------------------===//17 18#include "llvm/Support/VirtualOutputBackends.h"19#include "llvm/ADT/ScopeExit.h"20#include "llvm/Support/FileSystem.h"21#include "llvm/Support/LockFileManager.h"22#include "llvm/Support/MemoryBuffer.h"23#include "llvm/Support/Path.h"24#include "llvm/Support/Process.h"25#include "llvm/Support/Signals.h"26#include "llvm/Support/VirtualOutputConfig.h"27#include "llvm/Support/VirtualOutputError.h"28 29using namespace llvm;30using namespace llvm::vfs;31 32void ProxyOutputBackend::anchor() {}33void OnDiskOutputBackend::anchor() {}34 35IntrusiveRefCntPtr<OutputBackend> vfs::makeNullOutputBackend() {36 struct NullOutputBackend : public OutputBackend {37 IntrusiveRefCntPtr<OutputBackend> cloneImpl() const override {38 return const_cast<NullOutputBackend *>(this);39 }40 Expected<std::unique_ptr<OutputFileImpl>>41 createFileImpl(StringRef Path, std::optional<OutputConfig>) override {42 return std::make_unique<NullOutputFileImpl>();43 }44 };45 46 return makeIntrusiveRefCnt<NullOutputBackend>();47}48 49IntrusiveRefCntPtr<OutputBackend> vfs::makeFilteringOutputBackend(50 IntrusiveRefCntPtr<OutputBackend> UnderlyingBackend,51 std::function<bool(StringRef, std::optional<OutputConfig>)> Filter) {52 struct FilteringOutputBackend : public ProxyOutputBackend {53 Expected<std::unique_ptr<OutputFileImpl>>54 createFileImpl(StringRef Path,55 std::optional<OutputConfig> Config) override {56 if (Filter(Path, Config))57 return ProxyOutputBackend::createFileImpl(Path, Config);58 return std::make_unique<NullOutputFileImpl>();59 }60 61 IntrusiveRefCntPtr<OutputBackend> cloneImpl() const override {62 return makeIntrusiveRefCnt<FilteringOutputBackend>(63 getUnderlyingBackend().clone(), Filter);64 }65 66 FilteringOutputBackend(67 IntrusiveRefCntPtr<OutputBackend> UnderlyingBackend,68 std::function<bool(StringRef, std::optional<OutputConfig>)> Filter)69 : ProxyOutputBackend(std::move(UnderlyingBackend)),70 Filter(std::move(Filter)) {71 assert(this->Filter && "Expected a non-null function");72 }73 std::function<bool(StringRef, std::optional<OutputConfig>)> Filter;74 };75 76 return makeIntrusiveRefCnt<FilteringOutputBackend>(77 std::move(UnderlyingBackend), std::move(Filter));78}79 80IntrusiveRefCntPtr<OutputBackend>81vfs::makeMirroringOutputBackend(IntrusiveRefCntPtr<OutputBackend> Backend1,82 IntrusiveRefCntPtr<OutputBackend> Backend2) {83 struct ProxyOutputBackend1 : public ProxyOutputBackend {84 using ProxyOutputBackend::ProxyOutputBackend;85 };86 struct ProxyOutputBackend2 : public ProxyOutputBackend {87 using ProxyOutputBackend::ProxyOutputBackend;88 };89 struct MirroringOutput final : public OutputFileImpl, raw_pwrite_stream {90 Error keep() final {91 flush();92 return joinErrors(F1->keep(), F2->keep());93 }94 Error discard() final {95 flush();96 return joinErrors(F1->discard(), F2->discard());97 }98 raw_pwrite_stream &getOS() final { return *this; }99 100 void write_impl(const char *Ptr, size_t Size) override {101 F1->getOS().write(Ptr, Size);102 F2->getOS().write(Ptr, Size);103 }104 void pwrite_impl(const char *Ptr, size_t Size, uint64_t Offset) override {105 this->flush();106 F1->getOS().pwrite(Ptr, Size, Offset);107 F2->getOS().pwrite(Ptr, Size, Offset);108 }109 uint64_t current_pos() const override { return F1->getOS().tell(); }110 size_t preferred_buffer_size() const override {111 return PreferredBufferSize;112 }113 void reserveExtraSpace(uint64_t ExtraSize) override {114 F1->getOS().reserveExtraSpace(ExtraSize);115 F2->getOS().reserveExtraSpace(ExtraSize);116 }117 bool is_displayed() const override {118 return F1->getOS().is_displayed() && F2->getOS().is_displayed();119 }120 bool has_colors() const override {121 return F1->getOS().has_colors() && F2->getOS().has_colors();122 }123 void enable_colors(bool enable) override {124 raw_pwrite_stream::enable_colors(enable);125 F1->getOS().enable_colors(enable);126 F2->getOS().enable_colors(enable);127 }128 129 MirroringOutput(std::unique_ptr<OutputFileImpl> F1,130 std::unique_ptr<OutputFileImpl> F2)131 : PreferredBufferSize(std::max(F1->getOS().GetBufferSize(),132 F1->getOS().GetBufferSize())),133 F1(std::move(F1)), F2(std::move(F2)) {134 // Don't double buffer.135 this->F1->getOS().SetUnbuffered();136 this->F2->getOS().SetUnbuffered();137 }138 size_t PreferredBufferSize;139 std::unique_ptr<OutputFileImpl> F1;140 std::unique_ptr<OutputFileImpl> F2;141 };142 struct MirroringOutputBackend : public ProxyOutputBackend1,143 public ProxyOutputBackend2 {144 Expected<std::unique_ptr<OutputFileImpl>>145 createFileImpl(StringRef Path,146 std::optional<OutputConfig> Config) override {147 std::unique_ptr<OutputFileImpl> File1;148 std::unique_ptr<OutputFileImpl> File2;149 if (Error E =150 ProxyOutputBackend1::createFileImpl(Path, Config).moveInto(File1))151 return std::move(E);152 if (Error E =153 ProxyOutputBackend2::createFileImpl(Path, Config).moveInto(File2))154 return joinErrors(std::move(E), File1->discard());155 156 // Skip the extra indirection if one of these is a null output.157 if (isa<NullOutputFileImpl>(*File1)) {158 consumeError(File1->discard());159 return std::move(File2);160 }161 if (isa<NullOutputFileImpl>(*File2)) {162 consumeError(File2->discard());163 return std::move(File1);164 }165 return std::make_unique<MirroringOutput>(std::move(File1),166 std::move(File2));167 }168 169 IntrusiveRefCntPtr<OutputBackend> cloneImpl() const override {170 return IntrusiveRefCntPtr<ProxyOutputBackend1>(171 makeIntrusiveRefCnt<MirroringOutputBackend>(172 ProxyOutputBackend1::getUnderlyingBackend().clone(),173 ProxyOutputBackend2::getUnderlyingBackend().clone()));174 }175 void Retain() const { ProxyOutputBackend1::Retain(); }176 void Release() const { ProxyOutputBackend1::Release(); }177 178 MirroringOutputBackend(IntrusiveRefCntPtr<OutputBackend> Backend1,179 IntrusiveRefCntPtr<OutputBackend> Backend2)180 : ProxyOutputBackend1(std::move(Backend1)),181 ProxyOutputBackend2(std::move(Backend2)) {}182 };183 184 assert(Backend1 && "Expected actual backend");185 assert(Backend2 && "Expected actual backend");186 return IntrusiveRefCntPtr<ProxyOutputBackend1>(187 makeIntrusiveRefCnt<MirroringOutputBackend>(std::move(Backend1),188 std::move(Backend2)));189}190 191static OutputConfig192applySettings(std::optional<OutputConfig> &&Config,193 const OnDiskOutputBackend::OutputSettings &Settings) {194 if (!Config)195 Config = Settings.DefaultConfig;196 if (!Settings.UseTemporaries)197 Config->setNoAtomicWrite();198 if (!Settings.RemoveOnSignal)199 Config->setNoDiscardOnSignal();200 return *Config;201}202 203namespace {204class OnDiskOutputFile final : public OutputFileImpl {205public:206 Error keep() override;207 Error discard() override;208 raw_pwrite_stream &getOS() override {209 assert(FileOS && "Expected valid file");210 if (BufferOS)211 return *BufferOS;212 return *FileOS;213 }214 215 /// Attempt to open a temporary file for \p OutputPath.216 ///217 /// This tries to open a uniquely-named temporary file for \p OutputPath,218 /// possibly also creating any missing directories if \a219 /// OnDiskOutputConfig::UseTemporaryCreateMissingDirectories is set in \a220 /// Config.221 ///222 /// \post FD and \a TempPath are initialized if this is successful.223 Error tryToCreateTemporary(std::optional<int> &FD);224 225 Error initializeFile(std::optional<int> &FD);226 Error initializeStream();227 Error reset();228 229 OnDiskOutputFile(StringRef OutputPath, std::optional<OutputConfig> Config,230 const OnDiskOutputBackend::OutputSettings &Settings)231 : Config(applySettings(std::move(Config), Settings)),232 OutputPath(OutputPath.str()) {}233 234 OutputConfig Config;235 const std::string OutputPath;236 std::optional<std::string> TempPath;237 std::optional<raw_fd_ostream> FileOS;238 std::optional<buffer_ostream> BufferOS;239};240} // end namespace241 242static Error createDirectoriesOnDemand(StringRef OutputPath,243 OutputConfig Config,244 llvm::function_ref<Error()> CreateFile) {245 return handleErrors(CreateFile(), [&](std::unique_ptr<ECError> EC) {246 if (EC->convertToErrorCode() != std::errc::no_such_file_or_directory ||247 Config.getNoImplyCreateDirectories())248 return Error(std::move(EC));249 250 StringRef ParentPath = sys::path::parent_path(OutputPath);251 if (std::error_code EC = sys::fs::create_directories(ParentPath))252 return make_error<OutputError>(ParentPath, EC);253 return CreateFile();254 });255}256 257static sys::fs::OpenFlags generateFlagsFromConfig(OutputConfig Config) {258 sys::fs::OpenFlags OF = sys::fs::OF_None;259 if (Config.getTextWithCRLF())260 OF |= sys::fs::OF_TextWithCRLF;261 else if (Config.getText())262 OF |= sys::fs::OF_Text;263 // Don't pass OF_Append if writting to temporary since OF_Append is264 // not Atomic Append265 if (Config.getAppend() && !Config.getAtomicWrite())266 OF |= sys::fs::OF_Append;267 268 return OF;269}270 271Error OnDiskOutputFile::tryToCreateTemporary(std::optional<int> &FD) {272 // Create a temporary file.273 // Insert -%%%%%%%% before the extension (if any), and because some tools274 // (noticeable, clang's own GlobalModuleIndex.cpp) glob for build275 // artifacts, also append .tmp.276 StringRef OutputExtension = sys::path::extension(OutputPath);277 SmallString<128> ModelPath =278 StringRef(OutputPath).drop_back(OutputExtension.size());279 ModelPath += "-%%%%%%%%";280 ModelPath += OutputExtension;281 ModelPath += ".tmp";282 283 return createDirectoriesOnDemand(OutputPath, Config, [&]() -> Error {284 int NewFD;285 SmallString<128> UniquePath;286 sys::fs::OpenFlags OF = generateFlagsFromConfig(Config);287 if (std::error_code EC =288 sys::fs::createUniqueFile(ModelPath, NewFD, UniquePath, OF))289 return make_error<TempFileOutputError>(ModelPath, OutputPath, EC);290 291 if (Config.getDiscardOnSignal())292 sys::RemoveFileOnSignal(UniquePath);293 294 TempPath = UniquePath.str().str();295 FD.emplace(NewFD);296 return Error::success();297 });298}299 300Error OnDiskOutputFile::initializeFile(std::optional<int> &FD) {301 assert(OutputPath != "-" && "Unexpected request for FD of stdout");302 303 // Disable temporary file for other non-regular files, and if we get a status304 // object, also check if we can write and disable write-through buffers if305 // appropriate.306 if (Config.getAtomicWrite()) {307 sys::fs::file_status Status;308 sys::fs::status(OutputPath, Status);309 if (sys::fs::exists(Status)) {310 if (!sys::fs::is_regular_file(Status))311 Config.setNoAtomicWrite();312 313 // Fail now if we can't write to the final destination.314 if (!sys::fs::can_write(OutputPath))315 return make_error<OutputError>(316 OutputPath,317 std::make_error_code(std::errc::operation_not_permitted));318 }319 }320 321 // If (still) using a temporary file, try to create it (and return success if322 // that works).323 if (Config.getAtomicWrite())324 if (!errorToBool(tryToCreateTemporary(FD)))325 return Error::success();326 327 // Not using a temporary file. Open the final output file.328 return createDirectoriesOnDemand(OutputPath, Config, [&]() -> Error {329 int NewFD;330 sys::fs::OpenFlags OF = generateFlagsFromConfig(Config);331 if (std::error_code EC = sys::fs::openFileForWrite(332 OutputPath, NewFD, sys::fs::CD_CreateAlways, OF))333 return convertToOutputError(OutputPath, EC);334 FD.emplace(NewFD);335 336 if (Config.getDiscardOnSignal())337 sys::RemoveFileOnSignal(OutputPath);338 return Error::success();339 });340}341 342Error OnDiskOutputFile::initializeStream() {343 // Open the file stream.344 if (OutputPath == "-") {345 std::error_code EC;346 FileOS.emplace(OutputPath, EC);347 if (EC)348 return make_error<OutputError>(OutputPath, EC);349 } else {350 std::optional<int> FD;351 if (Error E = initializeFile(FD))352 return E;353 FileOS.emplace(*FD, /*shouldClose=*/true);354 }355 356 // Buffer the stream if necessary.357 if (!FileOS->supportsSeeking() && !Config.getText())358 BufferOS.emplace(*FileOS);359 360 return Error::success();361}362 363namespace {364class OpenFileRAII {365 static const int InvalidFd = -1;366 367public:368 int Fd = InvalidFd;369 370 ~OpenFileRAII() {371 if (Fd != InvalidFd)372 llvm::sys::Process::SafelyCloseFileDescriptor(Fd);373 }374};375 376enum class FileDifference : uint8_t {377 /// The source and destination paths refer to the exact same file.378 IdenticalFile,379 /// The source and destination paths refer to separate files with identical380 /// contents.381 SameContents,382 /// The source and destination paths refer to separate files with different383 /// contents.384 DifferentContents385};386} // end anonymous namespace387 388static Expected<FileDifference>389areFilesDifferent(const llvm::Twine &Source, const llvm::Twine &Destination) {390 if (sys::fs::equivalent(Source, Destination))391 return FileDifference::IdenticalFile;392 393 OpenFileRAII SourceFile;394 sys::fs::file_status SourceStatus;395 // If we can't open the source file, fail.396 if (std::error_code EC = sys::fs::openFileForRead(Source, SourceFile.Fd))397 return convertToOutputError(Source, EC);398 399 // If we can't stat the source file, fail.400 if (std::error_code EC = sys::fs::status(SourceFile.Fd, SourceStatus))401 return convertToOutputError(Source, EC);402 403 OpenFileRAII DestFile;404 sys::fs::file_status DestStatus;405 // If we can't open the destination file, report different.406 if (std::error_code Error =407 sys::fs::openFileForRead(Destination, DestFile.Fd))408 return FileDifference::DifferentContents;409 410 // If we can't open the destination file, report different.411 if (std::error_code Error = sys::fs::status(DestFile.Fd, DestStatus))412 return FileDifference::DifferentContents;413 414 // If the files are different sizes, they must be different.415 uint64_t Size = SourceStatus.getSize();416 if (Size != DestStatus.getSize())417 return FileDifference::DifferentContents;418 419 // If both files are zero size, they must be the same.420 if (Size == 0)421 return FileDifference::SameContents;422 423 // The two files match in size, so we have to compare the bytes to determine424 // if they're the same.425 std::error_code SourceRegionErr;426 sys::fs::mapped_file_region SourceRegion(427 sys::fs::convertFDToNativeFile(SourceFile.Fd),428 sys::fs::mapped_file_region::readonly, Size, 0, SourceRegionErr);429 if (SourceRegionErr)430 return convertToOutputError(Source, SourceRegionErr);431 432 std::error_code DestRegionErr;433 sys::fs::mapped_file_region DestRegion(434 sys::fs::convertFDToNativeFile(DestFile.Fd),435 sys::fs::mapped_file_region::readonly, Size, 0, DestRegionErr);436 437 if (DestRegionErr)438 return FileDifference::DifferentContents;439 440 if (memcmp(SourceRegion.const_data(), DestRegion.const_data(), Size) != 0)441 return FileDifference::DifferentContents;442 443 return FileDifference::SameContents;444}445 446Error OnDiskOutputFile::reset() {447 // Destroy the streams to flush them.448 BufferOS.reset();449 if (!FileOS)450 return Error::success();451 452 // Remember the error in raw_fd_ostream to be reported later.453 std::error_code EC = FileOS->error();454 // Clear the error to avoid fatal error when reset.455 FileOS->clear_error();456 FileOS.reset();457 return errorCodeToError(EC);458}459 460Error OnDiskOutputFile::keep() {461 if (auto E = reset())462 return E;463 464 // Close the file descriptor and remove crash cleanup before exit.465 auto RemoveDiscardOnSignal = make_scope_exit([&]() {466 if (Config.getDiscardOnSignal())467 sys::DontRemoveFileOnSignal(TempPath ? *TempPath : OutputPath);468 });469 470 if (!TempPath)471 return Error::success();472 473 // See if we should append instead of move.474 if (Config.getAppend() && OutputPath != "-") {475 // Read TempFile for the content to append.476 auto Content = MemoryBuffer::getFile(*TempPath);477 if (!Content)478 return convertToTempFileOutputError(*TempPath, OutputPath,479 Content.getError());480 while (1) {481 // Attempt to lock the output file.482 // Only one process is allowed to append to this file at a time.483 llvm::LockFileManager Lock(OutputPath);484 bool Owned;485 if (Error Err = Lock.tryLock().moveInto(Owned)) {486 // If we error acquiring a lock, we cannot ensure appends487 // to the trace file are atomic - cannot ensure output correctness.488 Lock.unsafeMaybeUnlock();489 return convertToOutputError(490 OutputPath, std::make_error_code(std::errc::no_lock_available));491 }492 if (Owned) {493 // Lock acquired, perform the write and release the lock.494 std::error_code EC;495 llvm::raw_fd_ostream Out(OutputPath, EC, llvm::sys::fs::OF_Append);496 if (EC)497 return convertToOutputError(OutputPath, EC);498 Out << (*Content)->getBuffer();499 Out.close();500 Lock.unsafeMaybeUnlock();501 if (Out.has_error())502 return convertToOutputError(OutputPath, Out.error());503 // Remove temp file and done.504 (void)sys::fs::remove(*TempPath);505 return Error::success();506 }507 // Someone else owns the lock on this file, wait.508 switch (Lock.waitForUnlockFor(std::chrono::seconds(256))) {509 case WaitForUnlockResult::Success:510 [[fallthrough]];511 case WaitForUnlockResult::OwnerDied: {512 continue; // try again to get the lock.513 }514 case WaitForUnlockResult::Timeout: {515 // We could error on timeout to avoid potentially hanging forever, but516 // it may be more likely that an interrupted process failed to clear517 // the lock, causing other waiting processes to time-out. Let's clear518 // the lock and try again right away. If we do start seeing compiler519 // hangs in this location, we will need to re-consider.520 Lock.unsafeMaybeUnlock();521 continue;522 }523 }524 break;525 }526 }527 528 if (Config.getOnlyIfDifferent()) {529 auto Result = areFilesDifferent(*TempPath, OutputPath);530 if (!Result)531 return Result.takeError();532 switch (*Result) {533 case FileDifference::IdenticalFile:534 // Do nothing for a self-move.535 return Error::success();536 537 case FileDifference::SameContents:538 // Files are identical; remove the source file.539 (void)sys::fs::remove(*TempPath);540 return Error::success();541 542 case FileDifference::DifferentContents:543 break; // Rename the file.544 }545 }546 547 // Move temporary to the final output path and remove it if that fails.548 std::error_code RenameEC = sys::fs::rename(*TempPath, OutputPath);549 if (!RenameEC)550 return Error::success();551 552 // FIXME: TempPath should be in the same directory as OutputPath but try to553 // copy the output to see if makes any difference. If this path is used,554 // investigate why we need to copy.555 RenameEC = sys::fs::copy_file(*TempPath, OutputPath);556 (void)sys::fs::remove(*TempPath);557 558 if (!RenameEC)559 return Error::success();560 561 return make_error<TempFileOutputError>(*TempPath, OutputPath, RenameEC);562}563 564Error OnDiskOutputFile::discard() {565 // Destroy the streams to flush them.566 if (auto E = reset())567 return E;568 569 // Nothing on the filesystem to remove for stdout.570 if (OutputPath == "-")571 return Error::success();572 573 auto discardPath = [&](StringRef Path) {574 std::error_code EC = sys::fs::remove(Path);575 sys::DontRemoveFileOnSignal(Path);576 return EC;577 };578 579 // Clean up the file that's in-progress.580 if (!TempPath)581 return convertToOutputError(OutputPath, discardPath(OutputPath));582 return convertToTempFileOutputError(*TempPath, OutputPath,583 discardPath(*TempPath));584}585 586Error OnDiskOutputBackend::makeAbsolute(SmallVectorImpl<char> &Path) const {587 return convertToOutputError(StringRef(Path.data(), Path.size()),588 sys::fs::make_absolute(Path));589}590 591Expected<std::unique_ptr<OutputFileImpl>>592OnDiskOutputBackend::createFileImpl(StringRef Path,593 std::optional<OutputConfig> Config) {594 SmallString<256> AbsPath;595 if (Path != "-") {596 AbsPath = Path;597 if (Error E = makeAbsolute(AbsPath))598 return std::move(E);599 Path = AbsPath;600 }601 602 auto File = std::make_unique<OnDiskOutputFile>(Path, Config, Settings);603 if (Error E = File->initializeStream())604 return std::move(E);605 606 return std::move(File);607}608