212 lines · cpp
1//===----------------------------------------------------------------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8 9#include "BuiltinCAS.h"10#include "llvm/CAS/BuiltinCASContext.h"11#include "llvm/CAS/BuiltinObjectHasher.h"12#include "llvm/CAS/OnDiskGraphDB.h"13#include "llvm/CAS/UnifiedOnDiskCache.h"14#include "llvm/Support/Compiler.h"15#include "llvm/Support/Error.h"16 17using namespace llvm;18using namespace llvm::cas;19using namespace llvm::cas::builtin;20 21namespace {22 23class OnDiskCAS : public BuiltinCAS {24public:25 Expected<ObjectRef> storeImpl(ArrayRef<uint8_t> ComputedHash,26 ArrayRef<ObjectRef> Refs,27 ArrayRef<char> Data) final;28 29 Expected<std::optional<ObjectHandle>> loadIfExists(ObjectRef Ref) final;30 31 CASID getID(ObjectRef Ref) const final;32 33 std::optional<ObjectRef> getReference(const CASID &ID) const final;34 35 Expected<bool> isMaterialized(ObjectRef Ref) const final;36 37 ArrayRef<char> getDataConst(ObjectHandle Node) const final;38 39 void print(raw_ostream &OS) const final;40 Error validate(bool CheckHash) const final;41 42 static Expected<std::unique_ptr<OnDiskCAS>> open(StringRef Path);43 44 OnDiskCAS(std::shared_ptr<ondisk::UnifiedOnDiskCache> UniDB)45 : UnifiedDB(std::move(UniDB)), DB(&UnifiedDB->getGraphDB()) {}46 47private:48 ObjectHandle convertHandle(ondisk::ObjectHandle Node) const {49 return makeObjectHandle(Node.getOpaqueData());50 }51 52 ondisk::ObjectHandle convertHandle(ObjectHandle Node) const {53 return ondisk::ObjectHandle(Node.getInternalRef(*this));54 }55 56 ObjectRef convertRef(ondisk::ObjectID Ref) const {57 return makeObjectRef(Ref.getOpaqueData());58 }59 60 ondisk::ObjectID convertRef(ObjectRef Ref) const {61 return ondisk::ObjectID::fromOpaqueData(Ref.getInternalRef(*this));62 }63 64 size_t getNumRefs(ObjectHandle Node) const final {65 auto RefsRange = DB->getObjectRefs(convertHandle(Node));66 return llvm::size(RefsRange);67 }68 69 ObjectRef readRef(ObjectHandle Node, size_t I) const final {70 auto RefsRange = DB->getObjectRefs(convertHandle(Node));71 return convertRef(RefsRange.begin()[I]);72 }73 74 Error forEachRef(ObjectHandle Node,75 function_ref<Error(ObjectRef)> Callback) const final;76 77 Error setSizeLimit(std::optional<uint64_t> SizeLimit) final;78 Expected<std::optional<uint64_t>> getStorageSize() const final;79 Error pruneStorageData() final;80 81 OnDiskCAS(std::unique_ptr<ondisk::OnDiskGraphDB> GraphDB)82 : OwnedDB(std::move(GraphDB)), DB(OwnedDB.get()) {}83 84 std::unique_ptr<ondisk::OnDiskGraphDB> OwnedDB;85 std::shared_ptr<ondisk::UnifiedOnDiskCache> UnifiedDB;86 ondisk::OnDiskGraphDB *DB;87};88 89} // end anonymous namespace90 91void OnDiskCAS::print(raw_ostream &OS) const { DB->print(OS); }92Error OnDiskCAS::validate(bool CheckHash) const {93 auto Hasher = [](ArrayRef<ArrayRef<uint8_t>> Refs, ArrayRef<char> Data,94 SmallVectorImpl<uint8_t> &Result) {95 auto Hash = BuiltinObjectHasher<llvm::cas::builtin::HasherT>::hashObject(96 Refs, Data);97 Result.assign(Hash.begin(), Hash.end());98 };99 100 if (auto E = DB->validate(CheckHash, Hasher))101 return E;102 103 return Error::success();104}105 106CASID OnDiskCAS::getID(ObjectRef Ref) const {107 ArrayRef<uint8_t> Hash = DB->getDigest(convertRef(Ref));108 return CASID::create(&getContext(), toStringRef(Hash));109}110 111std::optional<ObjectRef> OnDiskCAS::getReference(const CASID &ID) const {112 std::optional<ondisk::ObjectID> ObjID =113 DB->getExistingReference(ID.getHash());114 if (!ObjID)115 return std::nullopt;116 return convertRef(*ObjID);117}118 119Expected<bool> OnDiskCAS::isMaterialized(ObjectRef ExternalRef) const {120 return DB->isMaterialized(convertRef(ExternalRef));121}122 123ArrayRef<char> OnDiskCAS::getDataConst(ObjectHandle Node) const {124 return DB->getObjectData(convertHandle(Node));125}126 127Expected<std::optional<ObjectHandle>>128OnDiskCAS::loadIfExists(ObjectRef ExternalRef) {129 Expected<std::optional<ondisk::ObjectHandle>> ObjHnd =130 DB->load(convertRef(ExternalRef));131 if (!ObjHnd)132 return ObjHnd.takeError();133 if (!*ObjHnd)134 return std::nullopt;135 return convertHandle(**ObjHnd);136}137 138Expected<ObjectRef> OnDiskCAS::storeImpl(ArrayRef<uint8_t> ComputedHash,139 ArrayRef<ObjectRef> Refs,140 ArrayRef<char> Data) {141 SmallVector<ondisk::ObjectID, 64> IDs;142 IDs.reserve(Refs.size());143 for (ObjectRef Ref : Refs) {144 IDs.push_back(convertRef(Ref));145 }146 147 auto StoredID = DB->getReference(ComputedHash);148 if (LLVM_UNLIKELY(!StoredID))149 return StoredID.takeError();150 if (Error E = DB->store(*StoredID, IDs, Data))151 return std::move(E);152 return convertRef(*StoredID);153}154 155Error OnDiskCAS::forEachRef(ObjectHandle Node,156 function_ref<Error(ObjectRef)> Callback) const {157 auto RefsRange = DB->getObjectRefs(convertHandle(Node));158 for (ondisk::ObjectID Ref : RefsRange) {159 if (Error E = Callback(convertRef(Ref)))160 return E;161 }162 return Error::success();163}164 165Error OnDiskCAS::setSizeLimit(std::optional<uint64_t> SizeLimit) {166 UnifiedDB->setSizeLimit(SizeLimit);167 return Error::success();168}169 170Expected<std::optional<uint64_t>> OnDiskCAS::getStorageSize() const {171 return UnifiedDB->getStorageSize();172}173 174Error OnDiskCAS::pruneStorageData() { return UnifiedDB->collectGarbage(); }175 176Expected<std::unique_ptr<OnDiskCAS>> OnDiskCAS::open(StringRef AbsPath) {177 Expected<std::unique_ptr<ondisk::OnDiskGraphDB>> DB =178 ondisk::OnDiskGraphDB::open(AbsPath, BuiltinCASContext::getHashName(),179 sizeof(HashType));180 if (!DB)181 return DB.takeError();182 return std::unique_ptr<OnDiskCAS>(new OnDiskCAS(std::move(*DB)));183}184 185bool cas::isOnDiskCASEnabled() {186#if LLVM_ENABLE_ONDISK_CAS187 return true;188#else189 return false;190#endif191}192 193Expected<std::unique_ptr<ObjectStore>> cas::createOnDiskCAS(const Twine &Path) {194#if LLVM_ENABLE_ONDISK_CAS195 // FIXME: An absolute path isn't really good enough. Should open a directory196 // and use openat() for files underneath.197 SmallString<256> AbsPath;198 Path.toVector(AbsPath);199 sys::fs::make_absolute(AbsPath);200 201 return OnDiskCAS::open(AbsPath);202#else203 return createStringError(inconvertibleErrorCode(), "OnDiskCAS is disabled");204#endif /* LLVM_ENABLE_ONDISK_CAS */205}206 207std::unique_ptr<ObjectStore>208cas::builtin::createObjectStoreFromUnifiedOnDiskCache(209 std::shared_ptr<ondisk::UnifiedOnDiskCache> UniDB) {210 return std::make_unique<OnDiskCAS>(std::move(UniDB));211}212