570 lines · cpp
1//===-- YAMLSerialization.cpp ------------------------------------*- C++-*-===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// A YAML index file is a sequence of tagged entries.10// Each entry either encodes a Symbol or the list of references to a symbol11// (a "ref bundle").12//13//===----------------------------------------------------------------------===//14 15#include "Headers.h"16#include "index/Ref.h"17#include "index/Relation.h"18#include "index/Serialization.h"19#include "index/Symbol.h"20#include "index/SymbolLocation.h"21#include "index/SymbolOrigin.h"22#include "clang/Tooling/CompilationDatabase.h"23#include "llvm/ADT/StringRef.h"24#include "llvm/Support/Allocator.h"25#include "llvm/Support/StringSaver.h"26#include "llvm/Support/YAMLTraits.h"27#include "llvm/Support/raw_ostream.h"28#include <cstdint>29#include <optional>30 31namespace {32struct YIncludeHeaderWithReferences;33}34 35LLVM_YAML_IS_SEQUENCE_VECTOR(clang::clangd::Symbol::IncludeHeaderWithReferences)36LLVM_YAML_IS_SEQUENCE_VECTOR(clang::clangd::Ref)37LLVM_YAML_IS_SEQUENCE_VECTOR(YIncludeHeaderWithReferences)38 39namespace {40using RefBundle =41 std::pair<clang::clangd::SymbolID, std::vector<clang::clangd::Ref>>;42// This is a pale imitation of std::variant<Symbol, RefBundle, Relation>43struct VariantEntry {44 std::optional<clang::clangd::Symbol> Symbol;45 std::optional<RefBundle> Refs;46 std::optional<clang::clangd::Relation> Relation;47 std::optional<clang::clangd::IncludeGraphNode> Source;48 std::optional<clang::tooling::CompileCommand> Cmd;49};50// A class helps YAML to serialize the 32-bit encoded position (Line&Column),51// as YAMLIO can't directly map bitfields.52struct YPosition {53 uint32_t Line;54 uint32_t Column;55};56// A class helps YAML to serialize the IncludeHeaderWithReferences as YAMLIO57// can't directly map bitfields.58struct YIncludeHeaderWithReferences {59 llvm::StringRef IncludeHeader;60 uint32_t References;61 clang::clangd::Symbol::IncludeDirective SupportedDirectives;62 63 YIncludeHeaderWithReferences() = default;64 65 YIncludeHeaderWithReferences(66 llvm::StringRef IncludeHeader, uint32_t References,67 clang::clangd::Symbol::IncludeDirective SupportedDirectives)68 : IncludeHeader(IncludeHeader), References(References),69 SupportedDirectives(SupportedDirectives) {}70};71 72// avoid ODR violation of specialization for non-owned CompileCommand73struct CompileCommandYAML : clang::tooling::CompileCommand {};74 75} // namespace76namespace llvm {77namespace yaml {78 79using clang::clangd::FileDigest;80using clang::clangd::IncludeGraph;81using clang::clangd::IncludeGraphNode;82using clang::clangd::Ref;83using clang::clangd::RefKind;84using clang::clangd::Relation;85using clang::clangd::RelationKind;86using clang::clangd::Symbol;87using clang::clangd::SymbolID;88using clang::clangd::SymbolLocation;89using clang::index::SymbolInfo;90using clang::index::SymbolKind;91using clang::index::SymbolLanguage;92using clang::tooling::CompileCommand;93 94// Helper to (de)serialize the SymbolID. We serialize it as a hex string.95struct NormalizedSymbolID {96 NormalizedSymbolID(IO &) {}97 NormalizedSymbolID(IO &, const SymbolID &ID) {98 llvm::raw_string_ostream OS(HexString);99 OS << ID;100 }101 102 SymbolID denormalize(IO &I) {103 auto ID = SymbolID::fromStr(HexString);104 if (!ID) {105 I.setError(llvm::toString(ID.takeError()));106 return SymbolID();107 }108 return *ID;109 }110 111 std::string HexString;112};113 114struct NormalizedSymbolFlag {115 NormalizedSymbolFlag(IO &) {}116 NormalizedSymbolFlag(IO &, Symbol::SymbolFlag F) {117 Flag = static_cast<uint8_t>(F);118 }119 120 Symbol::SymbolFlag denormalize(IO &) {121 return static_cast<Symbol::SymbolFlag>(Flag);122 }123 124 uint8_t Flag = 0;125};126 127template <> struct MappingTraits<YPosition> {128 static void mapping(IO &IO, YPosition &Value) {129 IO.mapRequired("Line", Value.Line);130 IO.mapRequired("Column", Value.Column);131 }132};133 134struct NormalizedPosition {135 using Position = clang::clangd::SymbolLocation::Position;136 NormalizedPosition(IO &) {}137 NormalizedPosition(IO &, const Position &Pos) {138 P.Line = Pos.line();139 P.Column = Pos.column();140 }141 142 Position denormalize(IO &) {143 Position Pos;144 Pos.setLine(P.Line);145 Pos.setColumn(P.Column);146 return Pos;147 }148 YPosition P;149};150 151struct NormalizedFileURI {152 NormalizedFileURI(IO &) {}153 NormalizedFileURI(IO &, const char *FileURI) { URI = FileURI; }154 155 const char *denormalize(IO &IO) {156 assert(IO.getContext() &&157 "Expecting an UniqueStringSaver to allocate data");158 return static_cast<llvm::UniqueStringSaver *>(IO.getContext())159 ->save(URI)160 .data();161 }162 163 std::string URI;164};165 166template <> struct MappingTraits<SymbolLocation> {167 static void mapping(IO &IO, SymbolLocation &Value) {168 MappingNormalization<NormalizedFileURI, const char *> NFile(IO,169 Value.FileURI);170 IO.mapRequired("FileURI", NFile->URI);171 MappingNormalization<NormalizedPosition, SymbolLocation::Position> NStart(172 IO, Value.Start);173 IO.mapRequired("Start", NStart->P);174 MappingNormalization<NormalizedPosition, SymbolLocation::Position> NEnd(175 IO, Value.End);176 IO.mapRequired("End", NEnd->P);177 }178};179 180template <> struct MappingTraits<SymbolInfo> {181 static void mapping(IO &IO, SymbolInfo &SymInfo) {182 // FIXME: expose other fields?183 IO.mapRequired("Kind", SymInfo.Kind);184 IO.mapRequired("Lang", SymInfo.Lang);185 }186};187 188template <> struct ScalarBitSetTraits<clang::clangd::Symbol::IncludeDirective> {189 static void bitset(IO &IO, clang::clangd::Symbol::IncludeDirective &Value) {190 IO.bitSetCase(Value, "Include", clang::clangd::Symbol::Include);191 IO.bitSetCase(Value, "Import", clang::clangd::Symbol::Import);192 }193};194 195template <> struct MappingTraits<YIncludeHeaderWithReferences> {196 static void mapping(IO &IO, YIncludeHeaderWithReferences &Inc) {197 IO.mapRequired("Header", Inc.IncludeHeader);198 IO.mapRequired("References", Inc.References);199 IO.mapOptional("Directives", Inc.SupportedDirectives,200 clang::clangd::Symbol::Include);201 }202};203 204struct NormalizedIncludeHeaders {205 using IncludeHeader = clang::clangd::Symbol::IncludeHeaderWithReferences;206 NormalizedIncludeHeaders(IO &) {}207 NormalizedIncludeHeaders(208 IO &, const llvm::SmallVector<IncludeHeader, 1> &IncludeHeaders) {209 for (auto &I : IncludeHeaders) {210 Headers.emplace_back(I.IncludeHeader, I.References,211 I.supportedDirectives());212 }213 }214 215 llvm::SmallVector<IncludeHeader, 1> denormalize(IO &) {216 llvm::SmallVector<IncludeHeader, 1> Result;217 for (auto &H : Headers)218 Result.emplace_back(H.IncludeHeader, H.References, H.SupportedDirectives);219 return Result;220 }221 llvm::SmallVector<YIncludeHeaderWithReferences, 1> Headers;222};223 224template <> struct MappingTraits<Symbol> {225 static void mapping(IO &IO, Symbol &Sym) {226 MappingNormalization<NormalizedSymbolID, SymbolID> NSymbolID(IO, Sym.ID);227 MappingNormalization<NormalizedSymbolFlag, Symbol::SymbolFlag> NSymbolFlag(228 IO, Sym.Flags);229 MappingNormalization<230 NormalizedIncludeHeaders,231 llvm::SmallVector<Symbol::IncludeHeaderWithReferences, 1>>232 NIncludeHeaders(IO, Sym.IncludeHeaders);233 IO.mapRequired("ID", NSymbolID->HexString);234 IO.mapRequired("Name", Sym.Name);235 IO.mapRequired("Scope", Sym.Scope);236 IO.mapRequired("SymInfo", Sym.SymInfo);237 IO.mapOptional("CanonicalDeclaration", Sym.CanonicalDeclaration,238 SymbolLocation());239 IO.mapOptional("Definition", Sym.Definition, SymbolLocation());240 IO.mapOptional("References", Sym.References, 0u);241 IO.mapOptional("Flags", NSymbolFlag->Flag);242 IO.mapOptional("Signature", Sym.Signature);243 IO.mapOptional("TemplateSpecializationArgs",244 Sym.TemplateSpecializationArgs);245 IO.mapOptional("CompletionSnippetSuffix", Sym.CompletionSnippetSuffix);246 IO.mapOptional("Documentation", Sym.Documentation);247 IO.mapOptional("ReturnType", Sym.ReturnType);248 IO.mapOptional("Type", Sym.Type);249 IO.mapOptional("IncludeHeaders", NIncludeHeaders->Headers);250 }251};252 253template <> struct ScalarEnumerationTraits<SymbolLanguage> {254 static void enumeration(IO &IO, SymbolLanguage &Value) {255 IO.enumCase(Value, "C", SymbolLanguage::C);256 IO.enumCase(Value, "Cpp", SymbolLanguage::CXX);257 IO.enumCase(Value, "ObjC", SymbolLanguage::ObjC);258 IO.enumCase(Value, "Swift", SymbolLanguage::Swift);259 }260};261 262template <> struct ScalarEnumerationTraits<SymbolKind> {263 static void enumeration(IO &IO, SymbolKind &Value) {264#define DEFINE_ENUM(name) IO.enumCase(Value, #name, SymbolKind::name)265 266 DEFINE_ENUM(Unknown);267 DEFINE_ENUM(Function);268 DEFINE_ENUM(Module);269 DEFINE_ENUM(Namespace);270 DEFINE_ENUM(NamespaceAlias);271 DEFINE_ENUM(Macro);272 DEFINE_ENUM(Enum);273 DEFINE_ENUM(Struct);274 DEFINE_ENUM(Class);275 DEFINE_ENUM(Protocol);276 DEFINE_ENUM(Extension);277 DEFINE_ENUM(Union);278 DEFINE_ENUM(TypeAlias);279 DEFINE_ENUM(Function);280 DEFINE_ENUM(Variable);281 DEFINE_ENUM(Field);282 DEFINE_ENUM(EnumConstant);283 DEFINE_ENUM(InstanceMethod);284 DEFINE_ENUM(ClassMethod);285 DEFINE_ENUM(StaticMethod);286 DEFINE_ENUM(InstanceProperty);287 DEFINE_ENUM(ClassProperty);288 DEFINE_ENUM(StaticProperty);289 DEFINE_ENUM(Constructor);290 DEFINE_ENUM(Destructor);291 DEFINE_ENUM(ConversionFunction);292 DEFINE_ENUM(Parameter);293 DEFINE_ENUM(Using);294 295#undef DEFINE_ENUM296 }297};298 299template <> struct MappingTraits<RefBundle> {300 static void mapping(IO &IO, RefBundle &Refs) {301 MappingNormalization<NormalizedSymbolID, SymbolID> NSymbolID(IO,302 Refs.first);303 IO.mapRequired("ID", NSymbolID->HexString);304 IO.mapRequired("References", Refs.second);305 }306};307 308struct NormalizedRefKind {309 NormalizedRefKind(IO &) {}310 NormalizedRefKind(IO &, RefKind O) { Kind = static_cast<uint8_t>(O); }311 312 RefKind denormalize(IO &) { return static_cast<RefKind>(Kind); }313 314 uint8_t Kind = 0;315};316 317template <> struct MappingTraits<Ref> {318 static void mapping(IO &IO, Ref &R) {319 MappingNormalization<NormalizedRefKind, RefKind> NKind(IO, R.Kind);320 IO.mapRequired("Kind", NKind->Kind);321 IO.mapRequired("Location", R.Location);322 IO.mapOptional("Container", R.Container);323 }324};325 326struct NormalizedSymbolRole {327 NormalizedSymbolRole(IO &) {}328 NormalizedSymbolRole(IO &IO, RelationKind R) {329 Kind = static_cast<uint8_t>(R);330 }331 332 RelationKind denormalize(IO &IO) { return static_cast<RelationKind>(Kind); }333 334 uint8_t Kind = 0;335};336 337template <> struct MappingTraits<SymbolID> {338 static void mapping(IO &IO, SymbolID &ID) {339 MappingNormalization<NormalizedSymbolID, SymbolID> NSymbolID(IO, ID);340 IO.mapRequired("ID", NSymbolID->HexString);341 }342};343 344template <> struct MappingTraits<Relation> {345 static void mapping(IO &IO, Relation &Relation) {346 MappingNormalization<NormalizedSymbolRole, RelationKind> NRole(347 IO, Relation.Predicate);348 IO.mapRequired("Subject", Relation.Subject);349 IO.mapRequired("Predicate", NRole->Kind);350 IO.mapRequired("Object", Relation.Object);351 }352};353 354struct NormalizedSourceFlag {355 NormalizedSourceFlag(IO &) {}356 NormalizedSourceFlag(IO &, IncludeGraphNode::SourceFlag O) {357 Flag = static_cast<uint8_t>(O);358 }359 360 IncludeGraphNode::SourceFlag denormalize(IO &) {361 return static_cast<IncludeGraphNode::SourceFlag>(Flag);362 }363 364 uint8_t Flag = 0;365};366 367struct NormalizedFileDigest {368 NormalizedFileDigest(IO &) {}369 NormalizedFileDigest(IO &, const FileDigest &Digest) {370 HexString = llvm::toHex(Digest);371 }372 373 FileDigest denormalize(IO &I) {374 FileDigest Digest;375 if (HexString.size() == Digest.size() * 2 &&376 llvm::all_of(HexString, llvm::isHexDigit)) {377 memcpy(Digest.data(), llvm::fromHex(HexString).data(), Digest.size());378 } else {379 I.setError(std::string("Bad hex file digest: ") + HexString);380 }381 return Digest;382 }383 384 std::string HexString;385};386 387template <> struct MappingTraits<IncludeGraphNode> {388 static void mapping(IO &IO, IncludeGraphNode &Node) {389 IO.mapRequired("URI", Node.URI);390 MappingNormalization<NormalizedSourceFlag, IncludeGraphNode::SourceFlag>391 NSourceFlag(IO, Node.Flags);392 IO.mapRequired("Flags", NSourceFlag->Flag);393 MappingNormalization<NormalizedFileDigest, FileDigest> NDigest(IO,394 Node.Digest);395 IO.mapRequired("Digest", NDigest->HexString);396 IO.mapRequired("DirectIncludes", Node.DirectIncludes);397 }398};399 400template <> struct MappingTraits<CompileCommandYAML> {401 static void mapping(IO &IO, CompileCommandYAML &Cmd) {402 IO.mapRequired("Directory", Cmd.Directory);403 IO.mapRequired("CommandLine", Cmd.CommandLine);404 }405};406 407template <> struct MappingTraits<VariantEntry> {408 static void mapping(IO &IO, VariantEntry &Variant) {409 if (IO.mapTag("!Symbol", Variant.Symbol.has_value())) {410 if (!IO.outputting())411 Variant.Symbol.emplace();412 MappingTraits<Symbol>::mapping(IO, *Variant.Symbol);413 } else if (IO.mapTag("!Refs", Variant.Refs.has_value())) {414 if (!IO.outputting())415 Variant.Refs.emplace();416 MappingTraits<RefBundle>::mapping(IO, *Variant.Refs);417 } else if (IO.mapTag("!Relations", Variant.Relation.has_value())) {418 if (!IO.outputting())419 Variant.Relation.emplace();420 MappingTraits<Relation>::mapping(IO, *Variant.Relation);421 } else if (IO.mapTag("!Source", Variant.Source.has_value())) {422 if (!IO.outputting())423 Variant.Source.emplace();424 MappingTraits<IncludeGraphNode>::mapping(IO, *Variant.Source);425 } else if (IO.mapTag("!Cmd", Variant.Cmd.has_value())) {426 if (!IO.outputting())427 Variant.Cmd.emplace();428 MappingTraits<CompileCommandYAML>::mapping(429 IO, static_cast<CompileCommandYAML &>(*Variant.Cmd));430 }431 }432};433 434} // namespace yaml435} // namespace llvm436 437namespace clang {438namespace clangd {439 440void writeYAML(const IndexFileOut &O, llvm::raw_ostream &OS) {441 llvm::yaml::Output Yout(OS);442 for (const auto &Sym : *O.Symbols) {443 VariantEntry Entry;444 Entry.Symbol = Sym;445 Yout << Entry;446 }447 if (O.Refs)448 for (auto &Sym : *O.Refs) {449 VariantEntry Entry;450 Entry.Refs = Sym;451 Yout << Entry;452 }453 if (O.Relations)454 for (auto &R : *O.Relations) {455 VariantEntry Entry;456 Entry.Relation = R;457 Yout << Entry;458 }459 if (O.Sources) {460 for (const auto &Source : *O.Sources) {461 VariantEntry Entry;462 Entry.Source = Source.getValue();463 Yout << Entry;464 }465 }466 if (O.Cmd) {467 VariantEntry Entry;468 Entry.Cmd = *O.Cmd;469 Yout << Entry;470 }471}472 473llvm::Expected<IndexFileIn> readYAML(llvm::StringRef Data,474 SymbolOrigin Origin) {475 SymbolSlab::Builder Symbols;476 RefSlab::Builder Refs;477 RelationSlab::Builder Relations;478 llvm::BumpPtrAllocator479 Arena; // store the underlying data of Position::FileURI.480 llvm::UniqueStringSaver Strings(Arena);481 llvm::yaml::Input Yin(Data, &Strings);482 IncludeGraph Sources;483 std::optional<tooling::CompileCommand> Cmd;484 while (Yin.setCurrentDocument()) {485 llvm::yaml::EmptyContext Ctx;486 VariantEntry Variant;487 yamlize(Yin, Variant, true, Ctx);488 if (Yin.error())489 return llvm::errorCodeToError(Yin.error());490 491 if (Variant.Symbol) {492 Variant.Symbol->Origin = Origin;493 Symbols.insert(*Variant.Symbol);494 }495 if (Variant.Refs)496 for (const auto &Ref : Variant.Refs->second)497 Refs.insert(Variant.Refs->first, Ref);498 if (Variant.Relation)499 Relations.insert(*Variant.Relation);500 if (Variant.Source) {501 auto &IGN = *Variant.Source;502 auto Entry = Sources.try_emplace(IGN.URI).first;503 Entry->getValue() = std::move(IGN);504 // Fixup refs to refer to map keys which will live on505 Entry->getValue().URI = Entry->getKey();506 for (auto &Include : Entry->getValue().DirectIncludes)507 Include = Sources.try_emplace(Include).first->getKey();508 }509 if (Variant.Cmd)510 Cmd = *Variant.Cmd;511 Yin.nextDocument();512 }513 514 IndexFileIn Result;515 Result.Symbols.emplace(std::move(Symbols).build());516 Result.Refs.emplace(std::move(Refs).build());517 Result.Relations.emplace(std::move(Relations).build());518 if (Sources.size())519 Result.Sources = std::move(Sources);520 Result.Cmd = std::move(Cmd);521 return std::move(Result);522}523 524std::string toYAML(const Symbol &S) {525 std::string Buf;526 {527 llvm::raw_string_ostream OS(Buf);528 llvm::yaml::Output Yout(OS);529 Symbol Sym = S; // copy: Yout<< requires mutability.530 Yout << Sym;531 }532 return Buf;533}534 535std::string toYAML(const std::pair<SymbolID, llvm::ArrayRef<Ref>> &Data) {536 RefBundle Refs = {Data.first, Data.second};537 std::string Buf;538 {539 llvm::raw_string_ostream OS(Buf);540 llvm::yaml::Output Yout(OS);541 Yout << Refs;542 }543 return Buf;544}545 546std::string toYAML(const Relation &R) {547 std::string Buf;548 {549 llvm::raw_string_ostream OS(Buf);550 llvm::yaml::Output Yout(OS);551 Relation Rel = R; // copy: Yout<< requires mutability.552 Yout << Rel;553 }554 return Buf;555}556 557std::string toYAML(const Ref &R) {558 std::string Buf;559 {560 llvm::raw_string_ostream OS(Buf);561 llvm::yaml::Output Yout(OS);562 Ref Reference = R; // copy: Yout<< requires mutability.563 Yout << Reference;564 }565 return Buf;566}567 568} // namespace clangd569} // namespace clang570