406 lines · cpp
1//===----------------------------------------------------------------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8///9/// \file A utility for operating on LLVM CAS.10///11//===----------------------------------------------------------------------===//12 13#include "llvm/CAS/ActionCache.h"14#include "llvm/CAS/BuiltinUnifiedCASDatabases.h"15#include "llvm/CAS/ObjectStore.h"16#include "llvm/Option/Arg.h"17#include "llvm/Option/ArgList.h"18#include "llvm/Option/Option.h"19#include "llvm/Support/CommandLine.h"20#include "llvm/Support/Error.h"21#include "llvm/Support/InitLLVM.h"22#include "llvm/Support/MemoryBuffer.h"23#include "llvm/Support/raw_ostream.h"24 25using namespace llvm;26using namespace llvm::cas;27 28namespace {29enum ID {30 OPT_INVALID = 0, // This is not an option ID.31#define OPTION(...) LLVM_MAKE_OPT_ID(__VA_ARGS__),32#include "Options.inc"33#undef OPTION34};35 36#define OPTTABLE_STR_TABLE_CODE37#include "Options.inc"38#undef OPTTABLE_STR_TABLE_CODE39 40#define OPTTABLE_PREFIXES_TABLE_CODE41#include "Options.inc"42#undef OPTTABLE_PREFIXES_TABLE_CODE43 44using namespace llvm::opt;45static constexpr opt::OptTable::Info InfoTable[] = {46#define OPTION(...) LLVM_CONSTRUCT_OPT_INFO(__VA_ARGS__),47#include "Options.inc"48#undef OPTION49};50 51class LLVMCASOptTable : public opt::GenericOptTable {52public:53 LLVMCASOptTable()54 : opt::GenericOptTable(OptionStrTable, OptionPrefixesTable, InfoTable) {}55};56 57enum class CommandKind {58 Invalid,59 Dump,60 CatNodeData,61 MakeBlob,62 MakeNode,63 ListObjectReferences,64 Import,65 PutCacheKey,66 GetCacheResult,67 Validate,68 ValidateObject,69 ValidateIfNeeded,70 Prune,71};72 73struct CommandOptions {74 CommandKind Command = CommandKind::Invalid;75 std::vector<std::string> Inputs;76 std::string CASPath;77 std::string UpstreamCASPath;78 std::string DataPath;79 bool CheckHash;80 bool AllowRecovery;81 bool Force;82 bool InProcess;83 84 static CommandKind getCommandKind(opt::Arg &A) {85 switch (A.getOption().getID()) {86 case OPT_cas_dump:87 return CommandKind::Dump;88 case OPT_cat_node_data:89 return CommandKind::CatNodeData;90 case OPT_make_blob:91 return CommandKind::MakeBlob;92 case OPT_make_node:93 return CommandKind::MakeNode;94 case OPT_ls_node_refs:95 return CommandKind::ListObjectReferences;96 case OPT_import:97 return CommandKind::Import;98 case OPT_put_cache_key:99 return CommandKind::PutCacheKey;100 case OPT_get_cache_result:101 return CommandKind::GetCacheResult;102 case OPT_validate:103 return CommandKind::Validate;104 case OPT_validate_object:105 return CommandKind::ValidateObject;106 case OPT_validate_if_needed:107 return CommandKind::ValidateIfNeeded;108 case OPT_prune:109 return CommandKind::Prune;110 }111 return CommandKind::Invalid;112 }113 114 // Command requires input.115 static bool requiresInput(CommandKind Kind) {116 return Kind != CommandKind::ValidateIfNeeded &&117 Kind != CommandKind::Validate && Kind != CommandKind::MakeBlob &&118 Kind != CommandKind::MakeNode && Kind != CommandKind::Dump &&119 Kind != CommandKind::Prune;120 }121};122} // namespace123 124static int dump(ObjectStore &CAS);125static int listObjectReferences(ObjectStore &CAS, const CASID &ID);126static int catNodeData(ObjectStore &CAS, const CASID &ID);127static int makeBlob(ObjectStore &CAS, StringRef DataPath);128static int makeNode(ObjectStore &CAS, ArrayRef<std::string> References,129 StringRef DataPath);130static int import(ObjectStore &FromCAS, ObjectStore &ToCAS,131 ArrayRef<std::string> Objects);132static int putCacheKey(ObjectStore &CAS, ActionCache &AC,133 ArrayRef<std::string> Objects);134static int getCacheResult(ObjectStore &CAS, ActionCache &AC, const CASID &ID);135static int validateObject(ObjectStore &CAS, const CASID &ID);136static int validate(ObjectStore &CAS, ActionCache &AC, bool CheckHash);137static int validateIfNeeded(StringRef Path, bool CheckHash, bool Force,138 bool AllowRecovery, bool InProcess,139 const char *Argv0);140static int prune(cas::ObjectStore &CAS);141 142static Expected<CommandOptions> parseOptions(int Argc, char **Argv) {143 BumpPtrAllocator Alloc;144 StringSaver Saver(Alloc);145 SmallVector<const char *> ExpanedArgs;146 if (!cl::expandResponseFiles(Argc, Argv, nullptr, Saver, ExpanedArgs))147 return createStringError("cannot expand response file");148 149 LLVMCASOptTable T;150 unsigned MI, MC;151 opt::InputArgList Args = T.ParseArgs(ExpanedArgs, MI, MC);152 153 for (auto *Arg : Args.filtered(OPT_UNKNOWN)) {154 llvm::errs() << "ignoring unknown option: " << Arg->getSpelling() << '\n';155 }156 157 if (Args.hasArg(OPT_help)) {158 T.printHelp(159 outs(),160 (std::string(Argv[0]) + " [action] [options] <input files>").c_str(),161 "llvm-cas tool that performs CAS actions.", false);162 exit(0);163 }164 165 CommandOptions Opts;166 for (auto *A : Args.filtered(OPT_grp_action))167 Opts.Command = CommandOptions::getCommandKind(*A);168 169 if (Opts.Command == CommandKind::Invalid)170 return createStringError("no command action is specified");171 172 for (auto *File : Args.filtered(OPT_INPUT))173 Opts.Inputs.push_back(File->getValue());174 Opts.CASPath = Args.getLastArgValue(OPT_cas_path);175 Opts.UpstreamCASPath = Args.getLastArgValue(OPT_upstream_cas);176 Opts.DataPath = Args.getLastArgValue(OPT_data);177 Opts.CheckHash = Args.hasArg(OPT_check_hash);178 Opts.AllowRecovery = Args.hasArg(OPT_allow_recovery);179 Opts.Force = Args.hasArg(OPT_force);180 Opts.InProcess = Args.hasArg(OPT_in_process);181 182 // Validate options.183 if (Opts.CASPath.empty())184 return createStringError("missing --cas <path>");185 186 if (Opts.Inputs.empty() && CommandOptions::requiresInput(Opts.Command))187 return createStringError("missing <input> to operate on");188 189 return Opts;190}191 192int main(int Argc, char **Argv) {193 InitLLVM X(Argc, Argv);194 195 ExitOnError ExitOnErr;196 auto Opts = ExitOnErr(parseOptions(Argc, Argv));197 198 if (Opts.Command == CommandKind::ValidateIfNeeded)199 return validateIfNeeded(Opts.CASPath, Opts.CheckHash, Opts.Force,200 Opts.AllowRecovery, Opts.InProcess, Argv[0]);201 202 auto [CAS, AC] = ExitOnErr(createOnDiskUnifiedCASDatabases(Opts.CASPath));203 assert(CAS);204 205 if (Opts.Command == CommandKind::Dump)206 return dump(*CAS);207 208 if (Opts.Command == CommandKind::Validate)209 return validate(*CAS, *AC, Opts.CheckHash);210 211 if (Opts.Command == CommandKind::MakeBlob)212 return makeBlob(*CAS, Opts.DataPath);213 214 if (Opts.Command == CommandKind::MakeNode)215 return makeNode(*CAS, Opts.Inputs, Opts.DataPath);216 217 if (Opts.Command == CommandKind::Prune)218 return prune(*CAS);219 220 if (Opts.Command == CommandKind::Import) {221 if (Opts.UpstreamCASPath.empty())222 ExitOnErr(createStringError("missing '-upstream-cas'"));223 224 auto [UpstreamCAS, _] =225 ExitOnErr(createOnDiskUnifiedCASDatabases(Opts.UpstreamCASPath));226 return import(*UpstreamCAS, *CAS, Opts.Inputs);227 }228 229 if (Opts.Command == CommandKind::PutCacheKey ||230 Opts.Command == CommandKind::GetCacheResult) {231 if (!AC)232 ExitOnErr(createStringError("no action-cache available"));233 }234 235 if (Opts.Command == CommandKind::PutCacheKey)236 return putCacheKey(*CAS, *AC, Opts.Inputs);237 238 // Remaining commands need exactly one CAS object.239 if (Opts.Inputs.size() > 1)240 ExitOnErr(createStringError("too many <object>s, expected 1"));241 CASID ID = ExitOnErr(CAS->parseID(Opts.Inputs.front()));242 243 if (Opts.Command == CommandKind::GetCacheResult)244 return getCacheResult(*CAS, *AC, ID);245 246 if (Opts.Command == CommandKind::ListObjectReferences)247 return listObjectReferences(*CAS, ID);248 249 if (Opts.Command == CommandKind::CatNodeData)250 return catNodeData(*CAS, ID);251 252 assert(Opts.Command == CommandKind::ValidateObject);253 return validateObject(*CAS, ID);254}255 256static Expected<std::unique_ptr<MemoryBuffer>> openBuffer(StringRef DataPath) {257 if (DataPath.empty())258 return createStringError("--data missing");259 return errorOrToExpected(DataPath == "-"260 ? llvm::MemoryBuffer::getSTDIN()261 : llvm::MemoryBuffer::getFile(DataPath));262}263 264int dump(ObjectStore &CAS) {265 ExitOnError ExitOnErr("llvm-cas: dump: ");266 CAS.print(llvm::outs());267 return 0;268}269 270int makeBlob(ObjectStore &CAS, StringRef DataPath) {271 ExitOnError ExitOnErr("llvm-cas: make-blob: ");272 std::unique_ptr<MemoryBuffer> Buffer = ExitOnErr(openBuffer(DataPath));273 274 ObjectProxy Blob = ExitOnErr(CAS.createProxy({}, Buffer->getBuffer()));275 llvm::outs() << Blob.getID() << "\n";276 return 0;277}278 279int catNodeData(ObjectStore &CAS, const CASID &ID) {280 ExitOnError ExitOnErr("llvm-cas: cat-node-data: ");281 llvm::outs() << ExitOnErr(CAS.getProxy(ID)).getData();282 return 0;283}284 285int listObjectReferences(ObjectStore &CAS, const CASID &ID) {286 ExitOnError ExitOnErr("llvm-cas: ls-node-refs: ");287 288 ObjectProxy Object = ExitOnErr(CAS.getProxy(ID));289 ExitOnErr(Object.forEachReference([&](ObjectRef Ref) -> Error {290 llvm::outs() << CAS.getID(Ref) << "\n";291 return Error::success();292 }));293 294 return 0;295}296 297static int makeNode(ObjectStore &CAS, ArrayRef<std::string> Objects,298 StringRef DataPath) {299 std::unique_ptr<MemoryBuffer> Data =300 ExitOnError("llvm-cas: make-node: data: ")(openBuffer(DataPath));301 302 SmallVector<ObjectRef> IDs;303 for (StringRef Object : Objects) {304 ExitOnError ObjectErr("llvm-cas: make-node: ref: ");305 std::optional<ObjectRef> ID =306 CAS.getReference(ObjectErr(CAS.parseID(Object)));307 if (!ID)308 ObjectErr(createStringError("unknown object '" + Object + "'"));309 IDs.push_back(*ID);310 }311 312 ExitOnError ExitOnErr("llvm-cas: make-node: ");313 ObjectProxy Object = ExitOnErr(CAS.createProxy(IDs, Data->getBuffer()));314 llvm::outs() << Object.getID() << "\n";315 return 0;316}317 318static int import(ObjectStore &FromCAS, ObjectStore &ToCAS,319 ArrayRef<std::string> Objects) {320 ExitOnError ExitOnErr("llvm-cas: import: ");321 322 for (StringRef Object : Objects) {323 CASID ID = ExitOnErr(FromCAS.parseID(Object));324 auto Ref = FromCAS.getReference(ID);325 if (!Ref)326 ExitOnErr(createStringError("input not found: " + ID.toString()));327 328 auto Imported = ExitOnErr(ToCAS.importObject(FromCAS, *Ref));329 llvm::outs() << ToCAS.getID(Imported).toString() << "\n";330 }331 return 0;332}333 334static int putCacheKey(ObjectStore &CAS, ActionCache &AC,335 ArrayRef<std::string> Objects) {336 ExitOnError ExitOnErr("llvm-cas: put-cache-key: ");337 338 if (Objects.size() % 2 != 0)339 ExitOnErr(createStringError("expected pairs of inputs"));340 while (!Objects.empty()) {341 CASID Key = ExitOnErr(CAS.parseID(Objects[0]));342 CASID Result = ExitOnErr(CAS.parseID(Objects[1]));343 Objects = Objects.drop_front(2);344 ExitOnErr(AC.put(Key, Result));345 }346 return 0;347}348 349static int getCacheResult(ObjectStore &CAS, ActionCache &AC, const CASID &ID) {350 ExitOnError ExitOnErr("llvm-cas: get-cache-result: ");351 352 auto Result = ExitOnErr(AC.get(ID));353 if (!Result) {354 outs() << "result not found\n";355 return 1;356 }357 outs() << *Result << "\n";358 return 0;359}360 361int validateObject(ObjectStore &CAS, const CASID &ID) {362 ExitOnError ExitOnErr("llvm-cas: validate-object: ");363 ExitOnErr(CAS.validateObject(ID));364 outs() << ID << ": validated successfully\n";365 return 0;366}367 368int validate(ObjectStore &CAS, ActionCache &AC, bool CheckHash) {369 ExitOnError ExitOnErr("llvm-cas: validate: ");370 ExitOnErr(CAS.validate(CheckHash));371 ExitOnErr(AC.validate());372 outs() << "validated successfully\n";373 return 0;374}375 376int validateIfNeeded(StringRef Path, bool CheckHash, bool Force,377 bool AllowRecovery, bool InProcess, const char *Argv0) {378 ExitOnError ExitOnErr("llvm-cas: validate-if-needed: ");379 std::string ExecStorage;380 std::optional<StringRef> Exec;381 if (!InProcess) {382 ExecStorage = sys::fs::getMainExecutable(Argv0, (void *)validateIfNeeded);383 Exec = ExecStorage;384 }385 ValidationResult Result = ExitOnErr(validateOnDiskUnifiedCASDatabasesIfNeeded(386 Path, CheckHash, AllowRecovery, Force, Exec));387 switch (Result) {388 case ValidationResult::Valid:389 outs() << "validated successfully\n";390 break;391 case ValidationResult::Recovered:392 outs() << "recovered from invalid data\n";393 break;394 case ValidationResult::Skipped:395 outs() << "validation skipped\n";396 break;397 }398 return 0;399}400 401static int prune(cas::ObjectStore &CAS) {402 ExitOnError ExitOnErr("llvm-cas: prune: ");403 ExitOnErr(CAS.pruneStorageData());404 return 0;405}406