199 lines · cpp
1//===----------------------------------------------------------------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8 9#include "llvm/CAS/UnifiedOnDiskCache.h"10#include "CASTestConfig.h"11#include "OnDiskCommonUtils.h"12#include "llvm/Testing/Support/Error.h"13#include "llvm/Testing/Support/SupportHelpers.h"14#include "gtest/gtest.h"15 16using namespace llvm;17using namespace llvm::cas;18using namespace llvm::cas::ondisk;19using namespace llvm::unittest::cas;20 21/// Visits all the files of a directory recursively and returns the sum of their22/// sizes.23static Expected<size_t> countFileSizes(StringRef Path) {24 size_t TotalSize = 0;25 std::error_code EC;26 for (sys::fs::directory_iterator DirI(Path, EC), DirE; !EC && DirI != DirE;27 DirI.increment(EC)) {28 if (DirI->type() == sys::fs::file_type::directory_file) {29 Expected<size_t> Subsize = countFileSizes(DirI->path());30 if (!Subsize)31 return Subsize.takeError();32 TotalSize += *Subsize;33 continue;34 }35 ErrorOr<sys::fs::basic_file_status> Stat = DirI->status();36 if (!Stat)37 return createFileError(DirI->path(), Stat.getError());38 TotalSize += Stat->getSize();39 }40 if (EC)41 return createFileError(Path, EC);42 return TotalSize;43}44 45TEST_F(OnDiskCASTest, UnifiedOnDiskCacheTest) {46 unittest::TempDir Temp("ondisk-unified", /*Unique=*/true);47 std::unique_ptr<UnifiedOnDiskCache> UniDB;48 49 const uint64_t SizeLimit = 1024ull * 64;50 auto reopenDB = [&]() {51 UniDB.reset();52 ASSERT_THAT_ERROR(UnifiedOnDiskCache::open(Temp.path(), SizeLimit, "blake3",53 sizeof(HashType))54 .moveInto(UniDB),55 Succeeded());56 };57 58 reopenDB();59 60 HashType RootHash;61 HashType OtherHash;62 HashType Key1Hash;63 HashType Key2Hash;64 {65 OnDiskGraphDB &DB = UniDB->getGraphDB();66 std::optional<ObjectID> ID1;67 ASSERT_THAT_ERROR(store(DB, "1", {}).moveInto(ID1), Succeeded());68 std::optional<ObjectID> ID2;69 ASSERT_THAT_ERROR(store(DB, "2", {}).moveInto(ID2), Succeeded());70 std::optional<ObjectID> IDRoot;71 ASSERT_THAT_ERROR(store(DB, "root", {*ID1, *ID2}).moveInto(IDRoot),72 Succeeded());73 ArrayRef<uint8_t> Digest = DB.getDigest(*IDRoot);74 ASSERT_EQ(Digest.size(), RootHash.size());75 llvm::copy(Digest, RootHash.data());76 77 std::optional<ObjectID> IDOther;78 ASSERT_THAT_ERROR(store(DB, "other", {}).moveInto(IDOther), Succeeded());79 Digest = DB.getDigest(*IDOther);80 ASSERT_EQ(Digest.size(), OtherHash.size());81 llvm::copy(Digest, OtherHash.data());82 83 Key1Hash = digest("key1");84 std::optional<ObjectID> Val;85 ASSERT_THAT_ERROR(86 cachePut(UniDB->getKeyValueDB(), Key1Hash, *IDRoot).moveInto(Val),87 Succeeded());88 EXPECT_EQ(IDRoot, Val);89 90 Key2Hash = digest("key2");91 std::optional<ObjectID> KeyID;92 ASSERT_THAT_ERROR(DB.getReference(Key2Hash).moveInto(KeyID), Succeeded());93 ASSERT_THAT_ERROR(cachePut(UniDB->getKeyValueDB(),94 UniDB->getGraphDB().getDigest(*KeyID), *ID1)95 .moveInto(Val),96 Succeeded());97 }98 99 auto checkTree = [&](const HashType &Digest, StringRef ExpectedTree) {100 OnDiskGraphDB &DB = UniDB->getGraphDB();101 std::optional<ObjectID> ID;102 ASSERT_THAT_ERROR(DB.getReference(Digest).moveInto(ID), Succeeded());103 std::string PrintedTree;104 raw_string_ostream OS(PrintedTree);105 ASSERT_THAT_ERROR(printTree(DB, *ID, OS), Succeeded());106 EXPECT_EQ(PrintedTree, ExpectedTree);107 };108 auto checkRootTree = [&]() {109 return checkTree(RootHash, "root\n 1\n 2\n");110 };111 112 auto checkKey = [&](const HashType &Key, StringRef ExpectedData) {113 OnDiskGraphDB &DB = UniDB->getGraphDB();114 std::optional<ObjectID> Val;115 ASSERT_THAT_ERROR(cacheGet(UniDB->getKeyValueDB(), Key).moveInto(Val),116 Succeeded());117 118 ASSERT_TRUE(Val.has_value());119 std::optional<ondisk::ObjectHandle> Obj;120 ASSERT_THAT_ERROR(DB.load(*Val).moveInto(Obj), Succeeded());121 EXPECT_EQ(toStringRef(DB.getObjectData(*Obj)), ExpectedData);122 };123 124 checkRootTree();125 checkTree(OtherHash, "other\n");126 checkKey(Key1Hash, "root");127 checkKey(Key2Hash, "1");128 129 auto storeBigObject = [&](unsigned Index) {130 SmallString<1000> Buf;131 Buf.append(970, 'a');132 raw_svector_ostream(Buf) << Index;133 std::optional<ObjectID> ID;134 ASSERT_THAT_ERROR(store(UniDB->getGraphDB(), Buf, {}).moveInto(ID),135 Succeeded());136 };137 138 uint64_t PrevStoreSize = UniDB->getStorageSize();139 unsigned Index = 0;140 while (!UniDB->hasExceededSizeLimit()) {141 storeBigObject(Index++);142 }143 EXPECT_GT(UniDB->getStorageSize(), PrevStoreSize);144 UniDB->setSizeLimit(SizeLimit * 2);145 EXPECT_FALSE(UniDB->hasExceededSizeLimit());146 UniDB->setSizeLimit(SizeLimit);147 EXPECT_TRUE(UniDB->hasExceededSizeLimit());148 149 reopenDB();150 151 EXPECT_FALSE(UniDB->hasExceededSizeLimit());152 EXPECT_FALSE(UniDB->needsGarbageCollection());153 154 checkRootTree();155 checkKey(Key1Hash, "root");156 157 while (!UniDB->hasExceededSizeLimit()) {158 storeBigObject(Index++);159 }160 PrevStoreSize = UniDB->getStorageSize();161 ASSERT_THAT_ERROR(UniDB->close(), Succeeded());162 EXPECT_TRUE(UniDB->needsGarbageCollection());163 164 reopenDB();165 EXPECT_TRUE(UniDB->needsGarbageCollection());166 167 std::optional<size_t> DirSizeBefore;168 ASSERT_THAT_ERROR(countFileSizes(Temp.path()).moveInto(DirSizeBefore),169 Succeeded());170 171 ASSERT_THAT_ERROR(UnifiedOnDiskCache::collectGarbage(Temp.path()),172 Succeeded());173 174 std::optional<size_t> DirSizeAfter;175 ASSERT_THAT_ERROR(countFileSizes(Temp.path()).moveInto(DirSizeAfter),176 Succeeded());177 EXPECT_LT(*DirSizeAfter, *DirSizeBefore);178 179 reopenDB();180 EXPECT_FALSE(UniDB->needsGarbageCollection());181 182 checkRootTree();183 checkKey(Key1Hash, "root");184 185 EXPECT_LT(UniDB->getStorageSize(), PrevStoreSize);186 187 // 'Other' tree and 'Key2' got garbage-collected.188 {189 OnDiskGraphDB &DB = UniDB->getGraphDB();190 std::optional<ObjectID> ID;191 ASSERT_THAT_ERROR(DB.getReference(OtherHash).moveInto(ID), Succeeded());192 EXPECT_FALSE(DB.containsObject(*ID));193 std::optional<ObjectID> Val;194 ASSERT_THAT_ERROR(cacheGet(UniDB->getKeyValueDB(), Key2Hash).moveInto(Val),195 Succeeded());196 EXPECT_FALSE(Val.has_value());197 }198}199