181 lines · c
1//===--- Dex.h - Dex Symbol Index Implementation ----------------*- C++ -*-===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8///9/// \file10/// This defines Dex - a symbol index implementation based on query iterators11/// over symbol tokens, such as fuzzy matching trigrams, scopes, types, etc.12/// While consuming more memory and having longer build stage due to13/// preprocessing, Dex will have substantially lower latency. It will also allow14/// efficient symbol searching which is crucial for operations like code15/// completion, and can be very important for a number of different code16/// transformations which will be eventually supported by Clangd.17///18//===----------------------------------------------------------------------===//19 20#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANGD_INDEX_DEX_DEX_H21#define LLVM_CLANG_TOOLS_EXTRA_CLANGD_INDEX_DEX_DEX_H22 23#include "index/dex/Iterator.h"24#include "index/Index.h"25#include "index/Relation.h"26#include "index/dex/PostingList.h"27#include "index/dex/Token.h"28#include "llvm/ADT/StringSet.h"29 30namespace clang {31namespace clangd {32namespace dex {33 34/// In-memory Dex trigram-based index implementation.35class Dex : public SymbolIndex {36public:37 // All data must outlive this index.38 template <typename SymbolRange, typename RefsRange, typename RelationsRange>39 Dex(SymbolRange &&Symbols, RefsRange &&Refs, RelationsRange &&Relations,40 bool SupportContainedRefs)41 : Corpus(0) {42 for (auto &&Sym : Symbols)43 this->Symbols.push_back(&Sym);44 for (auto &&Ref : Refs)45 this->Refs.try_emplace(Ref.first, Ref.second);46 for (auto &&Rel : Relations) {47 this->Relations[std::make_pair(Rel.Subject,48 static_cast<uint8_t>(Rel.Predicate))]49 .push_back(Rel.Object);50 if (Rel.Predicate == RelationKind::OverriddenBy) {51 this->ReverseRelations[std::make_pair(Rel.Object, static_cast<uint8_t>(52 Rel.Predicate))]53 .push_back(Rel.Subject);54 }55 }56 buildIndex(SupportContainedRefs);57 }58 // Symbols and Refs are owned by BackingData, Index takes ownership.59 template <typename SymbolRange, typename RefsRange, typename RelationsRange,60 typename Payload>61 Dex(SymbolRange &&Symbols, RefsRange &&Refs, RelationsRange &&Relations,62 Payload &&BackingData, size_t BackingDataSize, bool SupportContainedRefs)63 : Dex(std::forward<SymbolRange>(Symbols), std::forward<RefsRange>(Refs),64 std::forward<RelationsRange>(Relations), SupportContainedRefs) {65 KeepAlive = std::shared_ptr<void>(66 std::make_shared<Payload>(std::move(BackingData)), nullptr);67 this->BackingDataSize = BackingDataSize;68 }69 70 template <typename SymbolRange, typename RefsRange, typename RelationsRange,71 typename FileRange, typename Payload>72 Dex(SymbolRange &&Symbols, RefsRange &&Refs, RelationsRange &&Relations,73 FileRange &&Files, IndexContents IdxContents, Payload &&BackingData,74 size_t BackingDataSize, bool SupportContainedRefs)75 : Dex(std::forward<SymbolRange>(Symbols), std::forward<RefsRange>(Refs),76 std::forward<RelationsRange>(Relations),77 std::forward<Payload>(BackingData), BackingDataSize,78 SupportContainedRefs) {79 this->Files = std::forward<FileRange>(Files);80 this->IdxContents = IdxContents;81 }82 83 /// Builds an index from slabs. The index takes ownership of the slab.84 static std::unique_ptr<SymbolIndex> build(SymbolSlab, RefSlab, RelationSlab,85 bool SupportContainedRefs);86 87 bool88 fuzzyFind(const FuzzyFindRequest &Req,89 llvm::function_ref<void(const Symbol &)> Callback) const override;90 91 void lookup(const LookupRequest &Req,92 llvm::function_ref<void(const Symbol &)> Callback) const override;93 94 bool refs(const RefsRequest &Req,95 llvm::function_ref<void(const Ref &)> Callback) const override;96 97 bool containedRefs(const ContainedRefsRequest &Req,98 llvm::function_ref<void(const ContainedRefsResult &)>99 Callback) const override;100 101 void relations(const RelationsRequest &Req,102 llvm::function_ref<void(const SymbolID &, const Symbol &)>103 Callback) const override;104 105 void106 reverseRelations(const RelationsRequest &,107 llvm::function_ref<void(const SymbolID &, const Symbol &)>)108 const override;109 110 llvm::unique_function<IndexContents(llvm::StringRef) const>111 indexedFiles() const override;112 113 size_t estimateMemoryUsage() const override;114 115private:116 class RevRef {117 const Ref *Reference;118 SymbolID Target;119 120 public:121 RevRef(const Ref &Reference, SymbolID Target)122 : Reference(&Reference), Target(Target) {}123 const Ref &ref() const { return *Reference; }124 ContainedRefsResult containedRefsResult() const {125 return {ref().Location, ref().Kind, Target};126 }127 };128 129 void buildIndex(bool EnableOutgoingCalls);130 llvm::iterator_range<std::vector<RevRef>::const_iterator>131 lookupRevRefs(const SymbolID &Container) const;132 std::unique_ptr<Iterator> iterator(const Token &Tok) const;133 std::unique_ptr<Iterator>134 createFileProximityIterator(llvm::ArrayRef<std::string> ProximityPaths) const;135 std::unique_ptr<Iterator>136 createTypeBoostingIterator(llvm::ArrayRef<std::string> Types) const;137 138 /// Stores symbols sorted in the descending order of symbol quality..139 std::vector<const Symbol *> Symbols;140 /// SymbolQuality[I] is the quality of Symbols[I].141 std::vector<float> SymbolQuality;142 llvm::DenseMap<SymbolID, const Symbol *> LookupTable;143 /// Inverted index is a mapping from the search token to the posting list,144 /// which contains all items which can be characterized by such search token.145 /// For example, if the search token is scope "std::", the corresponding146 /// posting list would contain all indices of symbols defined in namespace147 /// std. Inverted index is used to retrieve posting lists which are processed148 /// during the fuzzyFind process.149 llvm::DenseMap<Token, PostingList> InvertedIndex;150 dex::Corpus Corpus;151 llvm::DenseMap<SymbolID, llvm::ArrayRef<Ref>> Refs;152 std::vector<RevRef> RevRefs; // sorted by container ID153 static_assert(sizeof(RelationKind) == sizeof(uint8_t),154 "RelationKind should be of same size as a uint8_t");155 llvm::DenseMap<std::pair<SymbolID, uint8_t>, std::vector<SymbolID>> Relations;156 // Reverse relations, currently only for OverriddenBy157 llvm::DenseMap<std::pair<SymbolID, uint8_t>, std::vector<SymbolID>>158 ReverseRelations;159 std::shared_ptr<void> KeepAlive; // poor man's move-only std::any160 // Set of files which were used during this index build.161 llvm::StringSet<> Files;162 // Contents of the index (symbols, references, etc.)163 // This is only populated if `Files` is, which applies to some but not all164 // consumers of this class.165 IndexContents IdxContents = IndexContents::None;166 // Size of memory retained by KeepAlive.167 size_t BackingDataSize = 0;168};169 170/// Returns Search Token for a number of parent directories of given Path.171/// Should be used within the index build process.172///173/// This function is exposed for testing only.174llvm::SmallVector<llvm::StringRef, 5> generateProximityURIs(llvm::StringRef);175 176} // namespace dex177} // namespace clangd178} // namespace clang179 180#endif // LLVM_CLANG_TOOLS_EXTRA_CLANGD_INDEX_DEX_DEX_H181