947 lines · cpp
1//===----------------------------------------------------------------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8 9#include "llvm/Support/VirtualOutputBackends.h"10#include "llvm/ADT/StringMap.h"11#include "llvm/Support/BLAKE3.h"12#include "llvm/Support/HashingOutputBackend.h"13#include "llvm/Support/MemoryBuffer.h"14#include "llvm/Support/ThreadPool.h"15#include "llvm/Support/raw_ostream.h"16#include "llvm/Testing/Support/Error.h"17#include "gtest/gtest.h"18 19using namespace llvm;20using namespace llvm::vfs;21 22namespace {23 24class OutputBackendProvider {25public:26 virtual bool rejectsMissingDirectories() = 0;27 28 virtual IntrusiveRefCntPtr<OutputBackend> createBackend() = 0;29 virtual std::string getFilePathToCreate() = 0;30 virtual std::string getFilePathToCreateUnder(StringRef Parent1,31 StringRef Parent2 = "") = 0;32 virtual Error checkCreated(StringRef FilePath,33 OutputConfig Config = OutputConfig()) = 0;34 virtual Error checkWrote(StringRef FilePath, StringRef Data) = 0;35 virtual Error checkFlushed(StringRef FilePath, StringRef Data) = 0;36 virtual Error checkKept(StringRef FilePath, StringRef Data) = 0;37 virtual Error checkDiscarded(StringRef FilePath) = 0;38 39 virtual ~OutputBackendProvider() = default;40 41 struct Generator {42 std::string Name;43 std::function<std::unique_ptr<OutputBackendProvider>()> Generate;44 45 std::unique_ptr<OutputBackendProvider> operator()() const {46 return Generate();47 }48 };49};50 51struct BackendTest52 : public ::testing::TestWithParam<OutputBackendProvider::Generator> {53 std::unique_ptr<OutputBackendProvider> Provider;54 55 void SetUp() override { Provider = GetParam()(); }56 void TearDown() override { Provider = nullptr; }57 58 IntrusiveRefCntPtr<OutputBackend> createBackend() {59 return Provider->createBackend();60 }61};62 63TEST_P(BackendTest, Discard) {64 auto Backend = createBackend();65 std::string FilePath = Provider->getFilePathToCreate();66 StringRef Data = "some data";67 OutputFile O;68 EXPECT_THAT_ERROR(Backend->createFile(FilePath).moveInto(O), Succeeded());69 consumeDiscardOnDestroy(O);70 ASSERT_THAT_ERROR(Provider->checkCreated(FilePath), Succeeded());71 72 O << Data;73 EXPECT_THAT_ERROR(O.discard(), Succeeded());74 EXPECT_THAT_ERROR(Provider->checkDiscarded(FilePath), Succeeded());75 EXPECT_FALSE(O.isOpen());76}77 78TEST_P(BackendTest, DiscardNoAtomicWrite) {79 auto Backend = createBackend();80 std::string FilePath = Provider->getFilePathToCreate();81 StringRef Data = "some data";82 OutputConfig Config = OutputConfig().setNoAtomicWrite();83 84 OutputFile O;85 EXPECT_THAT_ERROR(Backend->createFile(FilePath, Config).moveInto(O),86 Succeeded());87 consumeDiscardOnDestroy(O);88 ASSERT_THAT_ERROR(Provider->checkCreated(FilePath, Config), Succeeded());89 90 O << Data;91 EXPECT_THAT_ERROR(O.discard(), Succeeded());92 EXPECT_THAT_ERROR(Provider->checkDiscarded(FilePath), Succeeded());93 EXPECT_FALSE(O.isOpen());94}95 96TEST_P(BackendTest, Keep) {97 auto Backend = createBackend();98 std::string FilePath = Provider->getFilePathToCreate();99 StringRef Data = "some data";100 101 OutputFile O;102 EXPECT_THAT_ERROR(Backend->createFile(FilePath).moveInto(O), Succeeded());103 consumeDiscardOnDestroy(O);104 ASSERT_THAT_ERROR(Provider->checkCreated(FilePath), Succeeded());105 ASSERT_TRUE(O.isOpen());106 107 O << Data;108 EXPECT_THAT_ERROR(Provider->checkWrote(FilePath, Data), Succeeded());109 110 EXPECT_THAT_ERROR(O.keep(), Succeeded());111 EXPECT_THAT_ERROR(Provider->checkKept(FilePath, Data), Succeeded());112 EXPECT_FALSE(O.isOpen());113}114 115TEST_P(BackendTest, KeepFlush) {116 auto Backend = createBackend();117 std::string FilePath = Provider->getFilePathToCreate();118 StringRef Data = "some data";119 OutputFile O;120 EXPECT_THAT_ERROR(Backend->createFile(FilePath).moveInto(O), Succeeded());121 consumeDiscardOnDestroy(O);122 ASSERT_THAT_ERROR(Provider->checkCreated(FilePath), Succeeded());123 124 O << Data;125 EXPECT_THAT_ERROR(Provider->checkWrote(FilePath, Data), Succeeded());126 127 O.getOS().flush();128 EXPECT_THAT_ERROR(Provider->checkFlushed(FilePath, Data), Succeeded());129 130 EXPECT_THAT_ERROR(O.keep(), Succeeded());131 EXPECT_THAT_ERROR(Provider->checkKept(FilePath, Data), Succeeded());132}133 134TEST_P(BackendTest, KeepFlushProxy) {135 auto Backend = createBackend();136 std::string FilePath = Provider->getFilePathToCreate();137 StringRef Data = "some data";138 OutputFile O;139 EXPECT_THAT_ERROR(Backend->createFile(FilePath).moveInto(O), Succeeded());140 consumeDiscardOnDestroy(O);141 ASSERT_THAT_ERROR(Provider->checkCreated(FilePath), Succeeded());142 {143 std::unique_ptr<raw_pwrite_stream> Proxy;144 EXPECT_THAT_ERROR(O.createProxy().moveInto(Proxy), Succeeded());145 *Proxy << Data;146 EXPECT_THAT_ERROR(Provider->checkWrote(FilePath, Data), Succeeded());147 148 Proxy->flush();149 EXPECT_THAT_ERROR(Provider->checkFlushed(FilePath, Data), Succeeded());150 }151 EXPECT_THAT_ERROR(O.keep(), Succeeded());152 EXPECT_THAT_ERROR(Provider->checkKept(FilePath, Data), Succeeded());153}154 155TEST_P(BackendTest, KeepEmpty) {156 auto Backend = createBackend();157 std::string FilePath = Provider->getFilePathToCreate();158 OutputFile O;159 EXPECT_THAT_ERROR(Backend->createFile(FilePath).moveInto(O), Succeeded());160 consumeDiscardOnDestroy(O);161 ASSERT_THAT_ERROR(Provider->checkCreated(FilePath), Succeeded());162 EXPECT_THAT_ERROR(O.keep(), Succeeded());163 EXPECT_THAT_ERROR(Provider->checkKept(FilePath, ""), Succeeded());164}165 166TEST_P(BackendTest, KeepMissingDirectory) {167 auto Backend = createBackend();168 std::string FilePath = Provider->getFilePathToCreateUnder("missing");169 StringRef Data = "some data";170 171 OutputFile O;172 EXPECT_THAT_ERROR(Backend->createFile(FilePath).moveInto(O), Succeeded());173 consumeDiscardOnDestroy(O);174 ASSERT_THAT_ERROR(Provider->checkCreated(FilePath), Succeeded());175 176 O << Data;177 EXPECT_THAT_ERROR(O.keep(), Succeeded());178 EXPECT_THAT_ERROR(Provider->checkKept(FilePath, Data), Succeeded());179}180 181TEST_P(BackendTest, KeepMissingDirectoryNested) {182 auto Backend = createBackend();183 std::string FilePath =184 Provider->getFilePathToCreateUnder("missing", "nested");185 StringRef Data = "some data";186 187 OutputFile O;188 EXPECT_THAT_ERROR(Backend->createFile(FilePath).moveInto(O), Succeeded());189 consumeDiscardOnDestroy(O);190 ASSERT_THAT_ERROR(Provider->checkCreated(FilePath), Succeeded());191 192 O << Data;193 EXPECT_THAT_ERROR(O.keep(), Succeeded());194 EXPECT_THAT_ERROR(Provider->checkKept(FilePath, Data), Succeeded());195}196 197TEST_P(BackendTest, KeepNoAtomicWrite) {198 auto Backend = createBackend();199 std::string FilePath = Provider->getFilePathToCreate();200 StringRef Data = "some data";201 OutputConfig Config = OutputConfig().setNoAtomicWrite();202 203 OutputFile O;204 EXPECT_THAT_ERROR(Backend->createFile(FilePath, Config).moveInto(O),205 Succeeded());206 consumeDiscardOnDestroy(O);207 ASSERT_THAT_ERROR(Provider->checkCreated(FilePath, Config), Succeeded());208 O << Data;209 EXPECT_THAT_ERROR(Provider->checkWrote(FilePath, Data), Succeeded());210 211 EXPECT_THAT_ERROR(O.keep(), Succeeded());212 EXPECT_THAT_ERROR(Provider->checkKept(FilePath, Data), Succeeded());213 EXPECT_FALSE(O.isOpen());214}215 216TEST_P(BackendTest, KeepNoAtomicWriteMissingDirectory) {217 auto Backend = createBackend();218 std::string FilePath = Provider->getFilePathToCreate();219 StringRef Data = "some data";220 OutputConfig Config = OutputConfig().setNoAtomicWrite();221 222 OutputFile O;223 EXPECT_THAT_ERROR(Backend->createFile(FilePath, Config).moveInto(O),224 Succeeded());225 consumeDiscardOnDestroy(O);226 ASSERT_THAT_ERROR(Provider->checkCreated(FilePath, Config), Succeeded());227 228 O << Data;229 EXPECT_THAT_ERROR(Provider->checkWrote(FilePath, Data), Succeeded());230 231 EXPECT_THAT_ERROR(O.keep(), Succeeded());232 EXPECT_THAT_ERROR(Provider->checkKept(FilePath, Data), Succeeded());233 EXPECT_FALSE(O.isOpen());234}235 236TEST_P(BackendTest, KeepMissingDirectoryNoImply) {237 // Skip this test if the backend doesn't have a concept of missing238 // directories.239 if (!Provider->rejectsMissingDirectories())240 return;241 242 auto Backend = createBackend();243 std::string FilePath = Provider->getFilePathToCreateUnder("missing");244 std::error_code EC = errorToErrorCode(245 consumeDiscardOnDestroy(246 Backend->createFile(FilePath,247 OutputConfig().setNoImplyCreateDirectories()))248 .takeError());249 EXPECT_EQ(int(std::errc::no_such_file_or_directory), EC.value());250}251 252class NullOutputBackendProvider : public OutputBackendProvider {253public:254 bool rejectsMissingDirectories() override { return false; }255 256 IntrusiveRefCntPtr<OutputBackend> createBackend() override {257 return makeNullOutputBackend();258 }259 std::string getFilePathToCreate() override { return "ignored.data"; }260 std::string getFilePathToCreateUnder(StringRef Parent1,261 StringRef Parent2) override {262 SmallString<128> Path;263 sys::path::append(Path, Parent1, Parent2, getFilePathToCreate());264 return Path.str().str();265 }266 Error checkCreated(StringRef, OutputConfig) override {267 return Error::success();268 }269 Error checkWrote(StringRef, StringRef) override { return Error::success(); }270 Error checkFlushed(StringRef, StringRef) override { return Error::success(); }271 Error checkKept(StringRef, StringRef) override { return Error::success(); }272 Error checkDiscarded(StringRef) override { return Error::success(); }273};274 275struct OnDiskFile {276 const unittest::TempDir &D;277 SmallString<128> Path;278 StringRef ParentPath;279 StringRef Filename;280 StringRef Stem;281 StringRef Extension;282 std::unique_ptr<MemoryBuffer> LastBuffer;283 284 OnDiskFile(const unittest::TempDir &D, const Twine &InputPath) : D(D) {285 if (sys::path::is_absolute(InputPath))286 InputPath.toVector(Path);287 else288 sys::path::append(Path, D.path(), InputPath);289 ParentPath = sys::path::parent_path(Path);290 Filename = sys::path::filename(Path);291 Stem = sys::path::stem(Filename);292 Extension = sys::path::extension(Filename);293 }294 295 std::optional<OnDiskFile> findTemp() const;296 297 std::optional<sys::fs::UniqueID> getCurrentUniqueID();298 299 bool hasUniqueID(sys::fs::UniqueID ID) {300 auto CurrentID = getCurrentUniqueID();301 if (!CurrentID)302 return false;303 return *CurrentID == ID;304 }305 306 std::optional<StringRef> getCurrentContent() {307 auto OnDiskOrErr = MemoryBuffer::getFile(Path);308 if (!OnDiskOrErr)309 return std::nullopt;310 LastBuffer = std::move(*OnDiskOrErr);311 return LastBuffer->getBuffer();312 }313 314 bool equalsCurrentContent(StringRef Data) {315 auto CurrentContent = getCurrentContent();316 if (!CurrentContent)317 return false;318 return *CurrentContent == Data;319 }320 321 bool equalsCurrentContent(std::nullopt_t) {322 return getCurrentContent() == std::nullopt;323 }324};325 326class OnDiskOutputBackendProvider : public OutputBackendProvider {327public:328 bool rejectsMissingDirectories() override { return true; }329 330 std::optional<unittest::TempDir> D;331 332 IntrusiveRefCntPtr<OutputBackend> createBackend() override {333 auto Backend = makeIntrusiveRefCnt<OnDiskOutputBackend>();334 Backend->Settings = Settings;335 return Backend;336 }337 void init() {338 if (!D)339 D.emplace("OutputBackendTest.d", /*Unique=*/true);340 }341 std::string getFilePathToCreate() override {342 init();343 return OnDiskFile(*D, "file.data").Path.str().str();344 }345 std::string getFilePathToCreateUnder(StringRef Parent1,346 StringRef Parent2) override {347 init();348 SmallString<128> Path;349 sys::path::append(Path, D->path(), Parent1, Parent2, "file.data");350 return Path.str().str();351 }352 353 Error checkCreated(StringRef FilePath, OutputConfig Config) override;354 Error checkWrote(StringRef FilePath, StringRef Data) override;355 Error checkFlushed(StringRef FilePath, StringRef Data) override;356 Error checkKept(StringRef FilePath, StringRef Data) override;357 Error checkDiscarded(StringRef FilePath) override;358 359 struct FileInfo {360 OutputConfig Config;361 std::optional<OnDiskFile> F;362 std::optional<OnDiskFile> Temp;363 std::optional<sys::fs::UniqueID> UID;364 std::optional<sys::fs::UniqueID> TempUID;365 };366 Error checkOpen(FileInfo &Info);367 bool shouldUseTemporaries(const FileInfo &Info) const;368 369 OnDiskOutputBackendProvider() = default;370 explicit OnDiskOutputBackendProvider(371 const OnDiskOutputBackend::OutputSettings &Settings)372 : Settings(Settings) {}373 OnDiskOutputBackend::OutputSettings Settings;374 375 StringMap<FileInfo> Files;376 Error lookupFileInfo(StringRef FilePath, FileInfo *&Info);377};378 379bool OnDiskOutputBackendProvider::shouldUseTemporaries(380 const FileInfo &Info) const {381 return Info.Config.getAtomicWrite() && Settings.UseTemporaries;382}383 384struct ProviderGeneratorList {385 std::vector<OutputBackendProvider::Generator> Generators;386 ProviderGeneratorList(387 std::initializer_list<OutputBackendProvider::Generator> IL)388 : Generators(IL) {}389 390 std::string operator()(391 const ::testing::TestParamInfo<OutputBackendProvider::Generator> &Info) {392 return Info.param.Name;393 }394};395 396ProviderGeneratorList BackendGenerators = {397 {"Null", []() { return std::make_unique<NullOutputBackendProvider>(); }},398 {"OnDisk",399 []() { return std::make_unique<OnDiskOutputBackendProvider>(); }},400 {"OnDisk_DisableRemoveOnSignal",401 []() {402 OnDiskOutputBackend::OutputSettings Settings;403 Settings.RemoveOnSignal = false;404 return std::make_unique<OnDiskOutputBackendProvider>(Settings);405 }},406 {"OnDisk_DisableTemporaries",407 []() {408 OnDiskOutputBackend::OutputSettings Settings;409 Settings.UseTemporaries = false;410 return std::make_unique<OnDiskOutputBackendProvider>(Settings);411 }},412};413 414INSTANTIATE_TEST_SUITE_P(VirtualOutput, BackendTest,415 ::testing::ValuesIn(BackendGenerators.Generators),416 BackendGenerators);417 418std::optional<sys::fs::UniqueID> OnDiskFile::getCurrentUniqueID() {419 sys::fs::file_status Status;420 sys::fs::status(Path, Status, /*follow=*/false);421 if (!sys::fs::is_regular_file(Status))422 return std::nullopt;423 return Status.getUniqueID();424}425 426std::optional<OnDiskFile> OnDiskFile::findTemp() const {427 std::error_code EC;428 for (sys::fs::directory_iterator I(ParentPath, EC), E; !EC && I != E;429 I.increment(EC)) {430 StringRef TempPath = I->path();431 if (!TempPath.starts_with(D.path()))432 continue;433 434 // Look for "<stem>-*.<extension>.tmp".435 if (sys::path::extension(TempPath) != ".tmp")436 continue;437 438 // Drop the ".tmp" and check the extension and stem.439 StringRef TempStem = sys::path::stem(TempPath);440 if (sys::path::extension(TempStem) != Extension)441 continue;442 StringRef OriginalStem = sys::path::stem(TempStem);443 if (!OriginalStem.starts_with(Stem))444 continue;445 if (!OriginalStem.drop_front(Stem.size()).starts_with("-"))446 continue;447 448 // Found it.449 return OnDiskFile(D, TempPath.drop_front(D.path().size() + 1));450 }451 return std::nullopt;452}453 454Error OnDiskOutputBackendProvider::lookupFileInfo(StringRef FilePath,455 FileInfo *&Info) {456 auto I = Files.find(FilePath);457 if (Files.find(FilePath) == Files.end())458 return createStringError(inconvertibleErrorCode(),459 "Missing call to checkCreated()");460 Info = &I->second;461 assert(Info->F && "Expected OnDiskFile to be initialized");462 return Error::success();463}464 465Error OnDiskOutputBackendProvider::checkOpen(FileInfo &Info) {466 // Collect info about filesystem state.467 assert(Info.F);468 std::optional<sys::fs::UniqueID> UID = Info.F->getCurrentUniqueID();469 std::optional<OnDiskFile> Temp = Info.F->findTemp();470 std::optional<sys::fs::UniqueID> TempUID;471 if (Temp)472 TempUID = Temp->getCurrentUniqueID();473 474 // Check if it's correct.475 if (shouldUseTemporaries(Info)) {476 if (!Temp)477 return createStringError(inconvertibleErrorCode(),478 "Missing temporary file");479 if (!TempUID)480 return createStringError(inconvertibleErrorCode(),481 "Missing UID for temporary");482 if (UID)483 return createStringError(484 inconvertibleErrorCode(),485 "Unexpected final UID when temporaries should be used");486 487 // Check previous data.488 if (Info.Temp)489 if (Temp->Path != Info.Temp->Path)490 return createStringError(inconvertibleErrorCode(),491 "Temporary path changed");492 if (Info.TempUID)493 if (*TempUID != *Info.TempUID)494 return createStringError(inconvertibleErrorCode(),495 "Temporary UID changed");496 } else {497 if (Temp)498 return createStringError(inconvertibleErrorCode(),499 "Unexpected temporary file");500 if (!UID)501 return createStringError(inconvertibleErrorCode(),502 "Missing UID for temporary");503 504 // Check previous data.505 if (Info.UID)506 if (*UID != *Info.UID)507 return createStringError(inconvertibleErrorCode(), "UID changed");508 }509 510 Info.UID = UID;511 if (Temp)512 Info.Temp.emplace(*D, Temp->Path);513 else514 Info.Temp.reset();515 Info.TempUID = TempUID;516 return Error::success();517}518 519Error OnDiskOutputBackendProvider::checkCreated(StringRef FilePath,520 OutputConfig Config) {521 auto &Info = Files[FilePath];522 if (Info.F) {523 assert(OnDiskFile(*D, FilePath).Path == Info.F->Path);524 Info.UID = std::nullopt;525 Info.Temp.reset();526 Info.TempUID = std::nullopt;527 } else {528 Info.F.emplace(*D, FilePath);529 }530 Info.Config = Config;531 return checkOpen(Info);532}533 534Error OnDiskOutputBackendProvider::checkWrote(StringRef FilePath,535 StringRef Data) {536 FileInfo *Info = nullptr;537 if (Error E = lookupFileInfo(FilePath, Info))538 return E;539 return checkOpen(*Info);540}541 542Error OnDiskOutputBackendProvider::checkFlushed(StringRef FilePath,543 StringRef Data) {544 FileInfo *Info = nullptr;545 if (Error E = lookupFileInfo(FilePath, Info))546 return E;547 if (Error E = checkOpen(*Info))548 return E;549 550 OnDiskFile &F = shouldUseTemporaries(*Info) ? *Info->Temp : *Info->F;551 if (!F.equalsCurrentContent(Data))552 return createStringError(inconvertibleErrorCode(), "content not flushed");553 return Error::success();554}555 556Error OnDiskOutputBackendProvider::checkKept(StringRef FilePath,557 StringRef Data) {558 FileInfo *Info = nullptr;559 if (Error E = lookupFileInfo(FilePath, Info))560 return E;561 562#ifndef _WIN32563 sys::fs::UniqueID UID =564 shouldUseTemporaries(*Info) ? *Info->TempUID : *Info->UID;565 if (!Info->F->hasUniqueID(UID))566 return createStringError(inconvertibleErrorCode(),567 "File not created by keep or changed UID");568#else569 // On Windows, the UID changes in a rename that happens in keep()570 // because it's based on hash of file paths. Instead check that the571 // file contents are the same.572 if (!Info->F->equalsCurrentContent(Data))573 return createStringError(inconvertibleErrorCode(),574 "File not created by keep");575#endif576 577 if (std::optional<OnDiskFile> Temp = Info->F->findTemp())578 return createStringError(inconvertibleErrorCode(),579 "Temporary not removed by keep");580 581 return Error::success();582}583 584Error OnDiskOutputBackendProvider::checkDiscarded(StringRef FilePath) {585 FileInfo *Info = nullptr;586 if (Error E = lookupFileInfo(FilePath, Info))587 return E;588 589 if (std::optional<sys::fs::UniqueID> UID = Info->F->getCurrentUniqueID())590 return createStringError(inconvertibleErrorCode(),591 "File not removed by discard");592 593 if (std::optional<OnDiskFile> Temp = Info->F->findTemp())594 return createStringError(inconvertibleErrorCode(),595 "Temporary not removed by discard");596 597 return Error::success();598}599 600TEST(VirtualOutputBackendAdaptors, makeFilteringOutputBackend) {601 bool ShouldCreate = false;602 auto Backend = makeFilteringOutputBackend(603 makeIntrusiveRefCnt<OnDiskOutputBackend>(),604 [&ShouldCreate](StringRef, std::optional<OutputConfig>) {605 return ShouldCreate;606 });607 608 int Count = 0;609 unittest::TempDir D("FilteringOutputBackendTest.d", /*Unique=*/true);610 for (bool ShouldCreateVal : {false, true, true, false}) {611 ShouldCreate = ShouldCreateVal;612 OnDiskFile OnDisk(D, "file." + Twine(Count++) + "." + Twine(ShouldCreate));613 OutputFile Output;614 ASSERT_THAT_ERROR(consumeDiscardOnDestroy(Backend->createFile(OnDisk.Path))615 .moveInto(Output),616 Succeeded());617 EXPECT_NE(ShouldCreate, Output.isNull());618 Output << "content";619 EXPECT_THAT_ERROR(Output.keep(), Succeeded());620 621 if (ShouldCreate) {622 EXPECT_EQ(StringRef("content"), OnDisk.getCurrentContent());623 } else {624 EXPECT_FALSE(OnDisk.getCurrentUniqueID());625 }626 }627 SmallString<128> Path;628}629 630class AbsolutePathBackend : public ProxyOutputBackend {631 IntrusiveRefCntPtr<OutputBackend> cloneImpl() const override {632 llvm_unreachable("unimplemented");633 }634 635 Expected<std::unique_ptr<OutputFileImpl>>636 createFileImpl(StringRef Path, std::optional<OutputConfig> Config) override {637 assert(!sys::path::is_absolute(Path) &&638 "Expected tests to pass all relative paths");639 SmallString<256> AbsPath;640 sys::path::append(AbsPath, CWD, Path);641 return ProxyOutputBackend::createFileImpl(AbsPath, Config);642 }643 644public:645 AbsolutePathBackend(const Twine &CWD,646 IntrusiveRefCntPtr<OutputBackend> Backend)647 : ProxyOutputBackend(std::move(Backend)), CWD(CWD.str()) {648 assert(sys::path::is_absolute(this->CWD) &&649 "Expected tests to pass a relative path");650 }651 652private:653 std::string CWD;654};655 656TEST(VirtualOutputBackendAdaptors, makeMirroringOutputBackend) {657 unittest::TempDir D1("MirroringOutputBackendTest.1.d", /*Unique=*/true);658 unittest::TempDir D2("MirroringOutputBackendTest.2.d", /*Unique=*/true);659 660 IntrusiveRefCntPtr<OutputBackend> Backend;661 {662 auto OnDisk = makeIntrusiveRefCnt<OnDiskOutputBackend>();663 Backend = makeMirroringOutputBackend(664 makeIntrusiveRefCnt<AbsolutePathBackend>(D1.path(), OnDisk),665 makeIntrusiveRefCnt<AbsolutePathBackend>(D2.path(), OnDisk));666 }667 668 OnDiskFile OnDisk1(D1, "file");669 OnDiskFile OnDisk2(D2, "file");670 OutputFile Output;671 ASSERT_THAT_ERROR(672 consumeDiscardOnDestroy(Backend->createFile("file")).moveInto(Output),673 Succeeded());674 EXPECT_TRUE(OnDisk1.findTemp());675 EXPECT_TRUE(OnDisk2.findTemp());676 677 Output << "content";678 Output.getOS().pwrite("ON", /*Size=*/2, /*Offset=*/1);679 EXPECT_THAT_ERROR(Output.keep(), Succeeded());680 EXPECT_EQ(StringRef("cONtent"), OnDisk1.getCurrentContent());681 EXPECT_EQ(StringRef("cONtent"), OnDisk2.getCurrentContent());682 EXPECT_NE(OnDisk1.getCurrentUniqueID(), OnDisk2.getCurrentUniqueID());683}684 685/// Behaves like NullOutputFileImpl, but doesn't match the RTTI (so OutputFile686/// cannot tell).687class LikeNullOutputFile final : public OutputFileImpl {688 Error keep() final { return Error::success(); }689 Error discard() final { return Error::success(); }690 raw_pwrite_stream &getOS() final { return OS; }691 692public:693 LikeNullOutputFile(raw_null_ostream &OS) : OS(OS) {}694 raw_null_ostream &OS;695};696class LikeNullOutputBackend final : public OutputBackend {697 IntrusiveRefCntPtr<OutputBackend> cloneImpl() const override {698 llvm_unreachable("not implemented");699 }700 701 Expected<std::unique_ptr<OutputFileImpl>>702 createFileImpl(StringRef Path, std::optional<OutputConfig> Config) override {703 return std::make_unique<LikeNullOutputFile>(OS);704 }705 706public:707 raw_null_ostream OS;708};709 710TEST(VirtualOutputBackendAdaptors, makeMirroringOutputBackendNull) {711 // Check that null outputs are skipped by seeing that LikeNull->OS is passed712 // through directly (without a mirroring proxy stream) to Output.713 auto LikeNull = makeIntrusiveRefCnt<LikeNullOutputBackend>();714 auto Null1 = makeNullOutputBackend();715 auto Mirror = makeMirroringOutputBackend(Null1, LikeNull);716 OutputFile Output;717 ASSERT_THAT_ERROR(718 consumeDiscardOnDestroy(Mirror->createFile("file")).moveInto(Output),719 Succeeded());720 EXPECT_TRUE(!Output.isNull());721 EXPECT_EQ(&Output.getOS(), &LikeNull->OS);722 723 // Check the other direction.724 Mirror = makeMirroringOutputBackend(LikeNull, Null1);725 ASSERT_THAT_ERROR(726 consumeDiscardOnDestroy(Mirror->createFile("file")).moveInto(Output),727 Succeeded());728 EXPECT_TRUE(!Output.isNull());729 EXPECT_EQ(&Output.getOS(), &LikeNull->OS);730 731 // Same null backend, twice.732 Mirror = makeMirroringOutputBackend(Null1, Null1);733 ASSERT_THAT_ERROR(734 consumeDiscardOnDestroy(Mirror->createFile("file")).moveInto(Output),735 Succeeded());736 EXPECT_TRUE(Output.isNull());737 738 // Two null backends.739 auto Null2 = makeNullOutputBackend();740 Mirror = makeMirroringOutputBackend(Null1, Null2);741 ASSERT_THAT_ERROR(742 consumeDiscardOnDestroy(Mirror->createFile("file")).moveInto(Output),743 Succeeded());744 EXPECT_TRUE(Output.isNull());745}746 747class StringErrorBackend final : public OutputBackend {748 IntrusiveRefCntPtr<OutputBackend> cloneImpl() const override {749 llvm_unreachable("not implemented");750 }751 752 Expected<std::unique_ptr<OutputFileImpl>>753 createFileImpl(StringRef Path, std::optional<OutputConfig> Config) override {754 return createStringError(inconvertibleErrorCode(), Msg);755 }756 757public:758 StringErrorBackend(const Twine &Msg) : Msg(Msg.str()) {}759 std::string Msg;760};761 762TEST(VirtualOutputBackendAdaptors, makeMirroringOutputBackendCreateError) {763 auto Error1 = makeIntrusiveRefCnt<StringErrorBackend>("error-backend-1");764 auto Null = makeNullOutputBackend();765 766 auto Mirror = makeMirroringOutputBackend(Null, Error1);767 EXPECT_THAT_ERROR(768 consumeDiscardOnDestroy(Mirror->createFile("file")).takeError(),769 FailedWithMessage(Error1->Msg));770 771 Mirror = makeMirroringOutputBackend(Error1, Null);772 EXPECT_THAT_ERROR(773 consumeDiscardOnDestroy(Mirror->createFile("file")).takeError(),774 FailedWithMessage(Error1->Msg));775 776 auto Error2 = makeIntrusiveRefCnt<StringErrorBackend>("error-backend-2");777 Mirror = makeMirroringOutputBackend(Error1, Error2);778 EXPECT_THAT_ERROR(779 consumeDiscardOnDestroy(Mirror->createFile("file")).takeError(),780 FailedWithMessage(Error1->Msg));781}782 783TEST(OnDiskBackendTest, OnlyIfDifferent) {784 OnDiskOutputBackendProvider Provider;785 auto Backend = Provider.createBackend();786 std::string FilePath = Provider.getFilePathToCreate();787 StringRef Data = "some data";788 OutputConfig Config = OutputConfig().setOnlyIfDifferent();789 790 OutputFile O1, O2, O3;791 sys::fs::file_status Status1, Status2, Status3;792 // Write first file.793 EXPECT_THAT_ERROR(Backend->createFile(FilePath, Config).moveInto(O1),794 Succeeded());795 O1 << Data;796 EXPECT_THAT_ERROR(O1.keep(), Succeeded());797 EXPECT_FALSE(O1.isOpen());798 EXPECT_FALSE(sys::fs::status(FilePath, Status1, /*follow=*/false));799 800 // Write second with same content.801 EXPECT_THAT_ERROR(Backend->createFile(FilePath, Config).moveInto(O2),802 Succeeded());803 O2 << Data;804 EXPECT_THAT_ERROR(O2.keep(), Succeeded());805 EXPECT_FALSE(O2.isOpen());806 EXPECT_FALSE(sys::fs::status(FilePath, Status2, /*follow=*/false));807 808#ifndef _WIN32809 // Make sure the output path file is not modified with same content.810 EXPECT_EQ(Status1.getUniqueID(), Status2.getUniqueID());811#else812 // On Windows, UniqueIDs are currently hash of file paths and don't813 // change on overwrites or depend on the file content. Check that814 // the file content is what is expected instead.815 auto EqualsCurrentContent = [](StringRef FilePath, StringRef Data) -> bool {816 auto BufOrErr = MemoryBuffer::getFile(FilePath);817 if (!BufOrErr)818 return false;819 return (*BufOrErr)->getBuffer() == Data;820 };821 EXPECT_TRUE(EqualsCurrentContent(FilePath, Data));822#endif823 824 // Write third with different content.825 EXPECT_THAT_ERROR(Backend->createFile(FilePath, Config).moveInto(O3),826 Succeeded());827 O3 << Data << "\n";828 EXPECT_THAT_ERROR(O3.keep(), Succeeded());829 EXPECT_FALSE(O3.isOpen());830 EXPECT_FALSE(sys::fs::status(FilePath, Status3, /*follow=*/false));831 832#ifndef _WIN32833 // This should overwrite the file and create a different UniqueID.834 EXPECT_NE(Status1.getUniqueID(), Status3.getUniqueID());835#else836 // On Windows, UniqueIDs are currently hash of file paths and don't837 // change on overwrites or depend on the file content. Check that838 // the file content is what is expected instead.839 EXPECT_TRUE(EqualsCurrentContent(FilePath, (Data + "\n").str()));840#endif841}842 843TEST(OnDiskBackendTest, Append) {844 OnDiskOutputBackendProvider Provider;845 auto Backend = Provider.createBackend();846 std::string FilePath = Provider.getFilePathToCreate();847 OutputConfig Config = OutputConfig().setAppend();848 849 OutputFile O1, O2, O3;850 // Write first file.851 EXPECT_THAT_ERROR(Backend->createFile(FilePath, Config).moveInto(O1),852 Succeeded());853 O1 << "some data\n";854 EXPECT_THAT_ERROR(O1.keep(), Succeeded());855 EXPECT_FALSE(O1.isOpen());856 857 OnDiskFile File1(*Provider.D, FilePath);858 EXPECT_TRUE(File1.equalsCurrentContent("some data\n"));859 860 // Append same data.861 EXPECT_THAT_ERROR(Backend->createFile(FilePath, Config).moveInto(O2),862 Succeeded());863 O2 << "more data\n";864 EXPECT_THAT_ERROR(O2.keep(), Succeeded());865 EXPECT_FALSE(O2.isOpen());866 867 // Check data is appended.868 OnDiskFile File2(*Provider.D, FilePath);869 EXPECT_TRUE(File2.equalsCurrentContent("some data\nmore data\n"));870 871 // Non atomic append.872 EXPECT_THAT_ERROR(873 Backend->createFile(FilePath, Config.setNoAtomicWrite()).moveInto(O3),874 Succeeded());875 O3 << "more more\n";876 EXPECT_THAT_ERROR(O3.keep(), Succeeded());877 EXPECT_FALSE(O3.isOpen());878 879 // Check data is appended.880 OnDiskFile File3(*Provider.D, FilePath);881 EXPECT_TRUE(File3.equalsCurrentContent("some data\nmore data\nmore more\n"));882}883 884TEST(HashingBackendTest, HashOutput) {885 HashingOutputBackend<BLAKE3> Backend;886 OutputFile O1, O2, O3, O4, O5;887 EXPECT_THAT_ERROR(Backend.createFile("file1").moveInto(O1), Succeeded());888 O1 << "some data";889 EXPECT_THAT_ERROR(O1.keep(), Succeeded());890 EXPECT_THAT_ERROR(Backend.createFile("file2").moveInto(O2), Succeeded());891 O2 << "some data";892 EXPECT_THAT_ERROR(O2.keep(), Succeeded());893 EXPECT_EQ(Backend.getHashValueForFile("file1"),894 Backend.getHashValueForFile("file2"));895 896 EXPECT_THAT_ERROR(Backend.createFile("file3").moveInto(O3), Succeeded());897 O3 << "some ";898 O3 << "data";899 EXPECT_THAT_ERROR(O3.keep(), Succeeded());900 EXPECT_EQ(Backend.getHashValueForFile("file1"),901 Backend.getHashValueForFile("file3"));902 903 EXPECT_THAT_ERROR(Backend.createFile("file4").moveInto(O4), Succeeded());904 O4 << "same data";905 O4.getOS().pwrite("o", 1, 1);906 EXPECT_THAT_ERROR(O4.keep(), Succeeded());907 EXPECT_EQ(Backend.getHashValueForFile("file1"),908 Backend.getHashValueForFile("file4"));909 910 EXPECT_THAT_ERROR(Backend.createFile("file5").moveInto(O5), Succeeded());911 O5 << "different data";912 EXPECT_THAT_ERROR(O5.keep(), Succeeded());913 EXPECT_NE(Backend.getHashValueForFile("file1"),914 Backend.getHashValueForFile("file5"));915}916 917TEST(HashingBackendTest, ParallelHashOutput) {918 const unsigned NumFile = 20;919 DefaultThreadPool Pool;920 HashingOutputBackend<BLAKE3> Backend;921 auto getFileName = [](unsigned Idx) -> std::string {922 std::string Name;923 raw_string_ostream OS(Name);924 OS << "file" << Idx;925 return Name;926 };927 for (unsigned I = 0; I < NumFile; ++I) {928 Pool.async(929 [&](unsigned Idx) {930 OutputFile O;931 auto Name = getFileName(Idx);932 EXPECT_THAT_ERROR(Backend.createFile(Name).moveInto(O), Succeeded());933 O << "some data" << Idx;934 EXPECT_THAT_ERROR(O.keep(), Succeeded());935 },936 I);937 }938 Pool.wait();939 940 for (unsigned I = 0; I < NumFile; ++I) {941 auto Name = getFileName(I);942 auto Hash = Backend.getHashValueForFile(Name);943 EXPECT_TRUE(Hash);944 }945}946} // end namespace947