311 lines · cpp
1//===----------------------------------------------------------------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8 9#include "CASTestConfig.h"10#include "OnDiskCommonUtils.h"11#include "llvm/Testing/Support/Error.h"12#include "llvm/Testing/Support/SupportHelpers.h"13#include "gtest/gtest.h"14 15using namespace llvm;16using namespace llvm::cas;17using namespace llvm::cas::ondisk;18using namespace llvm::unittest::cas;19 20TEST_F(OnDiskCASTest, OnDiskGraphDBTest) {21 unittest::TempDir Temp("ondiskcas", /*Unique=*/true);22 std::unique_ptr<OnDiskGraphDB> DB;23 ASSERT_THAT_ERROR(24 OnDiskGraphDB::open(Temp.path(), "blake3", sizeof(HashType)).moveInto(DB),25 Succeeded());26 27 auto digest = [&DB](StringRef Data, ArrayRef<ObjectID> Refs) -> ObjectID {28 return ::digest(*DB, Data, Refs);29 };30 31 auto store = [&](StringRef Data,32 ArrayRef<ObjectID> Refs) -> Expected<ObjectID> {33 return ::store(*DB, Data, Refs);34 };35 36 std::optional<ObjectID> ID1;37 ASSERT_THAT_ERROR(store("hello", {}).moveInto(ID1), Succeeded());38 39 std::optional<ondisk::ObjectHandle> Obj1;40 ASSERT_THAT_ERROR(DB->load(*ID1).moveInto(Obj1), Succeeded());41 ASSERT_TRUE(Obj1.has_value());42 EXPECT_EQ(toStringRef(DB->getObjectData(*Obj1)), "hello");43 44 ArrayRef<uint8_t> Digest1 = DB->getDigest(*ID1);45 std::optional<ObjectID> ID2;46 ASSERT_THAT_ERROR(DB->getReference(Digest1).moveInto(ID2), Succeeded());47 EXPECT_EQ(ID1, ID2);48 49 ObjectID ID3 = digest("world", {});50 EXPECT_FALSE(DB->containsObject(ID3));51 std::optional<ondisk::ObjectHandle> Obj2;52 ASSERT_THAT_ERROR(DB->load(ID3).moveInto(Obj2), Succeeded());53 EXPECT_FALSE(Obj2.has_value());54 55 ASSERT_THAT_ERROR(DB->store(ID3, {}, arrayRefFromStringRef<char>("world")),56 Succeeded());57 EXPECT_TRUE(DB->containsObject(ID3));58 ASSERT_THAT_ERROR(DB->load(ID3).moveInto(Obj2), Succeeded());59 ASSERT_TRUE(Obj2.has_value());60 EXPECT_EQ(toStringRef(DB->getObjectData(*Obj2)), "world");61 62 size_t LargeDataSize = 256LL * 1024LL; // 256K.63 // The precise size number is not important, we mainly check that the large64 // object will be properly accounted for.65 EXPECT_TRUE(DB->getStorageSize() > 10 &&66 DB->getStorageSize() < LargeDataSize);67 68 SmallString<16> Buffer;69 Buffer.resize(LargeDataSize);70 ASSERT_THAT_ERROR(store(Buffer, {}).moveInto(ID1), Succeeded());71 size_t StorageSize = DB->getStorageSize();72 EXPECT_TRUE(StorageSize > LargeDataSize);73 74 // Close & re-open the DB and check that it reports the same storage size.75 DB.reset();76 ASSERT_THAT_ERROR(77 OnDiskGraphDB::open(Temp.path(), "blake3", sizeof(HashType)).moveInto(DB),78 Succeeded());79 EXPECT_EQ(DB->getStorageSize(), StorageSize);80}81 82TEST_F(OnDiskCASTest, OnDiskGraphDBFaultInSingleNode) {83 unittest::TempDir TempUpstream("ondiskcas-upstream", /*Unique=*/true);84 std::unique_ptr<OnDiskGraphDB> UpstreamDB;85 ASSERT_THAT_ERROR(86 OnDiskGraphDB::open(TempUpstream.path(), "blake3", sizeof(HashType))87 .moveInto(UpstreamDB),88 Succeeded());89 {90 std::optional<ObjectID> ID1;91 ASSERT_THAT_ERROR(store(*UpstreamDB, "hello", {}).moveInto(ID1),92 Succeeded());93 std::optional<ObjectID> ID2;94 ASSERT_THAT_ERROR(store(*UpstreamDB, "another", {}).moveInto(ID2),95 Succeeded());96 std::optional<ObjectID> ID3;97 ASSERT_THAT_ERROR(store(*UpstreamDB, "world", {*ID1, *ID2}).moveInto(ID3),98 Succeeded());99 }100 101 unittest::TempDir Temp("ondiskcas", /*Unique=*/true);102 std::unique_ptr<OnDiskGraphDB> DB;103 ASSERT_THAT_ERROR(104 OnDiskGraphDB::open(Temp.path(), "blake3", sizeof(HashType),105 UpstreamDB.get(),106 OnDiskGraphDB::FaultInPolicy::SingleNode)107 .moveInto(DB),108 Succeeded());109 110 ObjectID ID1 = digest(*DB, "hello", {});111 ObjectID ID2 = digest(*DB, "another", {});112 ObjectID ID3 = digest(*DB, "world", {ID1, ID2});113 ObjectID ID4 = digest(*DB, "world", {});114 115 EXPECT_TRUE(DB->containsObject(ID1));116 EXPECT_TRUE(DB->containsObject(ID2));117 EXPECT_TRUE(DB->containsObject(ID3));118 EXPECT_FALSE(DB->containsObject(ID4));119 120 EXPECT_TRUE(DB->getExistingReference(digest("hello", {})).has_value());121 EXPECT_TRUE(DB->getExistingReference(DB->getDigest(ID3)).has_value());122 EXPECT_FALSE(DB->getExistingReference(digest("world", {})).has_value());123 124 {125 std::optional<ondisk::ObjectHandle> Obj;126 ASSERT_THAT_ERROR(DB->load(ID1).moveInto(Obj), Succeeded());127 ASSERT_TRUE(Obj.has_value());128 EXPECT_EQ(toStringRef(DB->getObjectData(*Obj)), "hello");129 auto Refs = DB->getObjectRefs(*Obj);130 EXPECT_TRUE(Refs.empty());131 }132 {133 std::optional<ondisk::ObjectHandle> Obj;134 ASSERT_THAT_ERROR(DB->load(ID3).moveInto(Obj), Succeeded());135 ASSERT_TRUE(Obj.has_value());136 EXPECT_EQ(toStringRef(DB->getObjectData(*Obj)), "world");137 auto Refs = DB->getObjectRefs(*Obj);138 ASSERT_EQ(std::distance(Refs.begin(), Refs.end()), 2);139 EXPECT_EQ(Refs.begin()[0], ID1);140 EXPECT_EQ(Refs.begin()[1], ID2);141 }142 {143 std::optional<ondisk::ObjectHandle> Obj;144 ASSERT_THAT_ERROR(DB->load(ID4).moveInto(Obj), Succeeded());145 EXPECT_FALSE(Obj.has_value());146 }147 148 // Re-open the primary without chaining, to verify the data were copied from149 // the upstream.150 ASSERT_THAT_ERROR(151 OnDiskGraphDB::open(Temp.path(), "blake3", sizeof(HashType),152 /*UpstreamDB=*/nullptr,153 OnDiskGraphDB::FaultInPolicy::SingleNode)154 .moveInto(DB),155 Succeeded());156 ID1 = digest(*DB, "hello", {});157 ID2 = digest(*DB, "another", {});158 ID3 = digest(*DB, "world", {ID1, ID2});159 EXPECT_TRUE(DB->containsObject(ID1));160 EXPECT_FALSE(DB->containsObject(ID2));161 EXPECT_TRUE(DB->containsObject(ID3));162 {163 std::optional<ondisk::ObjectHandle> Obj;164 ASSERT_THAT_ERROR(DB->load(ID1).moveInto(Obj), Succeeded());165 ASSERT_TRUE(Obj.has_value());166 EXPECT_EQ(toStringRef(DB->getObjectData(*Obj)), "hello");167 auto Refs = DB->getObjectRefs(*Obj);168 EXPECT_TRUE(Refs.empty());169 }170}171 172TEST_F(OnDiskCASTest, OnDiskGraphDBFaultInFullTree) {173 unittest::TempDir TempUpstream("ondiskcas-upstream", /*Unique=*/true);174 std::unique_ptr<OnDiskGraphDB> UpstreamDB;175 ASSERT_THAT_ERROR(176 OnDiskGraphDB::open(TempUpstream.path(), "blake3", sizeof(HashType))177 .moveInto(UpstreamDB),178 Succeeded());179 HashType RootHash;180 {181 std::optional<ObjectID> ID11;182 ASSERT_THAT_ERROR(store(*UpstreamDB, "11", {}).moveInto(ID11), Succeeded());183 std::optional<ObjectID> ID121;184 ASSERT_THAT_ERROR(store(*UpstreamDB, "121", {}).moveInto(ID121),185 Succeeded());186 std::optional<ObjectID> ID12;187 ASSERT_THAT_ERROR(store(*UpstreamDB, "12", {*ID121}).moveInto(ID12),188 Succeeded());189 std::optional<ObjectID> ID1;190 ASSERT_THAT_ERROR(store(*UpstreamDB, "1", {*ID11, *ID12}).moveInto(ID1),191 Succeeded());192 std::optional<ObjectID> ID21;193 ASSERT_THAT_ERROR(store(*UpstreamDB, "21", {}).moveInto(ID21), Succeeded());194 std::optional<ObjectID> ID22;195 ASSERT_THAT_ERROR(store(*UpstreamDB, "22", {}).moveInto(ID22), Succeeded());196 std::optional<ObjectID> ID2;197 ASSERT_THAT_ERROR(198 store(*UpstreamDB, "2", {*ID12, *ID21, *ID22}).moveInto(ID2),199 Succeeded());200 std::optional<ObjectID> IDRoot;201 ASSERT_THAT_ERROR(store(*UpstreamDB, "root", {*ID1, *ID2}).moveInto(IDRoot),202 Succeeded());203 ArrayRef<uint8_t> Digest = UpstreamDB->getDigest(*IDRoot);204 ASSERT_EQ(Digest.size(), RootHash.size());205 llvm::copy(Digest, RootHash.data());206 }207 208 unittest::TempDir Temp("ondiskcas", /*Unique=*/true);209 std::unique_ptr<OnDiskGraphDB> DB;210 ASSERT_THAT_ERROR(OnDiskGraphDB::open(Temp.path(), "blake3", sizeof(HashType),211 UpstreamDB.get(),212 OnDiskGraphDB::FaultInPolicy::FullTree)213 .moveInto(DB),214 Succeeded());215 216 {217 std::optional<ObjectID> IDRoot;218 ASSERT_THAT_ERROR(DB->getReference(RootHash).moveInto(IDRoot), Succeeded());219 std::optional<ondisk::ObjectHandle> Obj;220 ASSERT_THAT_ERROR(DB->load(*IDRoot).moveInto(Obj), Succeeded());221 ASSERT_TRUE(Obj.has_value());222 EXPECT_EQ(toStringRef(DB->getObjectData(*Obj)), "root");223 auto Refs = DB->getObjectRefs(*Obj);224 ASSERT_EQ(std::distance(Refs.begin(), Refs.end()), 2);225 }226 227 // Re-open the primary without chaining, to verify the data were copied from228 // the upstream.229 ASSERT_THAT_ERROR(OnDiskGraphDB::open(Temp.path(), "blake3", sizeof(HashType),230 /*UpstreamDB=*/nullptr,231 OnDiskGraphDB::FaultInPolicy::FullTree)232 .moveInto(DB),233 Succeeded());234 235 std::optional<ObjectID> IDRoot;236 ASSERT_THAT_ERROR(DB->getReference(RootHash).moveInto(IDRoot), Succeeded());237 std::string PrintedTree;238 raw_string_ostream OS(PrintedTree);239 ASSERT_THAT_ERROR(printTree(*DB, *IDRoot, OS), Succeeded());240 StringRef Expected = R"(root241 1242 11243 12244 121245 2246 12247 121248 21249 22250)";251 EXPECT_EQ(PrintedTree, Expected);252}253 254TEST_F(OnDiskCASTest, OnDiskGraphDBFaultInPolicyConflict) {255 auto tryFaultInPolicyConflict = [](OnDiskGraphDB::FaultInPolicy Policy1,256 OnDiskGraphDB::FaultInPolicy Policy2) {257 unittest::TempDir TempUpstream("ondiskcas-upstream", /*Unique=*/true);258 std::unique_ptr<OnDiskGraphDB> UpstreamDB;259 ASSERT_THAT_ERROR(260 OnDiskGraphDB::open(TempUpstream.path(), "blake3", sizeof(HashType))261 .moveInto(UpstreamDB),262 Succeeded());263 264 unittest::TempDir Temp("ondiskcas", /*Unique=*/true);265 std::unique_ptr<OnDiskGraphDB> DB;266 ASSERT_THAT_ERROR(OnDiskGraphDB::open(Temp.path(), "blake3",267 sizeof(HashType), UpstreamDB.get(),268 Policy1)269 .moveInto(DB),270 Succeeded());271 DB.reset();272 ASSERT_THAT_ERROR(OnDiskGraphDB::open(Temp.path(), "blake3",273 sizeof(HashType), UpstreamDB.get(),274 Policy2)275 .moveInto(DB),276 Failed());277 };278 // Open as 'single', then as 'full'.279 tryFaultInPolicyConflict(OnDiskGraphDB::FaultInPolicy::SingleNode,280 OnDiskGraphDB::FaultInPolicy::FullTree);281 // Open as 'full', then as 'single'.282 tryFaultInPolicyConflict(OnDiskGraphDB::FaultInPolicy::FullTree,283 OnDiskGraphDB::FaultInPolicy::SingleNode);284}285 286#if defined(EXPENSIVE_CHECKS) && !defined(_WIN32)287TEST_F(OnDiskCASTest, OnDiskGraphDBSpaceLimit) {288 setMaxOnDiskCASMappingSize();289 unittest::TempDir Temp("ondiskcas", /*Unique=*/true);290 std::unique_ptr<OnDiskGraphDB> DB;291 ASSERT_THAT_ERROR(292 OnDiskGraphDB::open(Temp.path(), "blake3", sizeof(HashType)).moveInto(DB),293 Succeeded());294 295 std::optional<ObjectID> ID;296 std::string Data(500, '0');297 auto storeSmallObject = [&]() {298 SmallVector<ObjectID, 1> Refs;299 if (ID)300 Refs.push_back(*ID);301 ASSERT_THAT_ERROR(store(*DB, Data, Refs).moveInto(ID), Succeeded());302 };303 304 // Insert enough small elements to overflow the data pool.305 for (unsigned I = 0; I < 1024 * 256; ++I)306 storeSmallObject();307 308 EXPECT_GE(DB->getHardStorageLimitUtilization(), 99U);309}310#endif311