1567 lines · cpp
1//===--- SemanticHighlighting.cpp - ------------------------- ---*- C++ -*-===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8 9#include "SemanticHighlighting.h"10#include "Config.h"11#include "FindTarget.h"12#include "ParsedAST.h"13#include "Protocol.h"14#include "SourceCode.h"15#include "support/Logger.h"16#include "clang/AST/ASTContext.h"17#include "clang/AST/Decl.h"18#include "clang/AST/DeclCXX.h"19#include "clang/AST/DeclObjC.h"20#include "clang/AST/DeclTemplate.h"21#include "clang/AST/DeclarationName.h"22#include "clang/AST/ExprCXX.h"23#include "clang/AST/RecursiveASTVisitor.h"24#include "clang/AST/Type.h"25#include "clang/AST/TypeLoc.h"26#include "clang/Basic/LangOptions.h"27#include "clang/Basic/SourceLocation.h"28#include "clang/Basic/SourceManager.h"29#include "clang/Sema/HeuristicResolver.h"30#include "clang/Tooling/Syntax/Tokens.h"31#include "llvm/ADT/STLExtras.h"32#include "llvm/ADT/StringRef.h"33#include "llvm/Support/Casting.h"34#include "llvm/Support/Error.h"35#include <algorithm>36#include <optional>37 38namespace clang {39namespace clangd {40namespace {41 42/// Get the last Position on a given line.43llvm::Expected<Position> endOfLine(llvm::StringRef Code, int Line) {44 auto StartOfLine = positionToOffset(Code, Position{Line, 0});45 if (!StartOfLine)46 return StartOfLine.takeError();47 StringRef LineText = Code.drop_front(*StartOfLine).take_until([](char C) {48 return C == '\n';49 });50 return Position{Line, static_cast<int>(lspLength(LineText))};51}52 53/// Some names are not written in the source code and cannot be highlighted,54/// e.g. anonymous classes. This function detects those cases.55bool canHighlightName(DeclarationName Name) {56 switch (Name.getNameKind()) {57 case DeclarationName::Identifier: {58 auto *II = Name.getAsIdentifierInfo();59 return II && !II->getName().empty();60 }61 case DeclarationName::CXXConstructorName:62 case DeclarationName::CXXDestructorName:63 return true;64 case DeclarationName::ObjCZeroArgSelector:65 case DeclarationName::ObjCOneArgSelector:66 case DeclarationName::ObjCMultiArgSelector:67 // Multi-arg selectors need special handling, and we handle 0/1 arg68 // selectors there too.69 return false;70 case DeclarationName::CXXConversionFunctionName:71 case DeclarationName::CXXOperatorName:72 case DeclarationName::CXXDeductionGuideName:73 case DeclarationName::CXXLiteralOperatorName:74 case DeclarationName::CXXUsingDirective:75 return false;76 }77 llvm_unreachable("invalid name kind");78}79 80bool isUniqueDefinition(const NamedDecl *Decl) {81 if (auto *Func = dyn_cast<FunctionDecl>(Decl))82 return Func->isThisDeclarationADefinition();83 if (auto *Klass = dyn_cast<CXXRecordDecl>(Decl))84 return Klass->isThisDeclarationADefinition();85 if (auto *Iface = dyn_cast<ObjCInterfaceDecl>(Decl))86 return Iface->isThisDeclarationADefinition();87 if (auto *Proto = dyn_cast<ObjCProtocolDecl>(Decl))88 return Proto->isThisDeclarationADefinition();89 if (auto *Var = dyn_cast<VarDecl>(Decl))90 return Var->isThisDeclarationADefinition();91 return isa<TemplateTypeParmDecl>(Decl) ||92 isa<NonTypeTemplateParmDecl>(Decl) ||93 isa<TemplateTemplateParmDecl>(Decl) || isa<ObjCCategoryDecl>(Decl) ||94 isa<ObjCImplDecl>(Decl);95}96 97std::optional<HighlightingKind> kindForType(const Type *TP,98 const HeuristicResolver *Resolver);99std::optional<HighlightingKind> kindForDecl(const NamedDecl *D,100 const HeuristicResolver *Resolver) {101 if (auto *USD = dyn_cast<UsingShadowDecl>(D)) {102 if (auto *Target = USD->getTargetDecl())103 D = Target;104 }105 if (auto *TD = dyn_cast<TemplateDecl>(D)) {106 if (auto *Templated = TD->getTemplatedDecl())107 D = Templated;108 }109 if (auto *TD = dyn_cast<TypedefNameDecl>(D)) {110 // We try to highlight typedefs as their underlying type.111 if (auto K =112 kindForType(TD->getUnderlyingType().getTypePtrOrNull(), Resolver))113 return K;114 // And fallback to a generic kind if this fails.115 return HighlightingKind::Typedef;116 }117 // We highlight class decls, constructor decls and destructor decls as118 // `Class` type. The destructor decls are handled in `VisitTagTypeLoc` (we119 // will visit a TypeLoc where the underlying Type is a CXXRecordDecl).120 if (auto *RD = llvm::dyn_cast<RecordDecl>(D)) {121 // We don't want to highlight lambdas like classes.122 if (RD->isLambda())123 return std::nullopt;124 return HighlightingKind::Class;125 }126 if (isa<ClassTemplateDecl, RecordDecl, CXXConstructorDecl, ObjCInterfaceDecl,127 ObjCImplementationDecl>(D))128 return HighlightingKind::Class;129 if (isa<ObjCProtocolDecl>(D))130 return HighlightingKind::Interface;131 if (isa<ObjCCategoryDecl, ObjCCategoryImplDecl>(D))132 return HighlightingKind::Namespace;133 if (auto *MD = dyn_cast<CXXMethodDecl>(D))134 return MD->isStatic() ? HighlightingKind::StaticMethod135 : HighlightingKind::Method;136 if (auto *OMD = dyn_cast<ObjCMethodDecl>(D))137 return OMD->isClassMethod() ? HighlightingKind::StaticMethod138 : HighlightingKind::Method;139 if (isa<FieldDecl, IndirectFieldDecl, ObjCPropertyDecl>(D))140 return HighlightingKind::Field;141 if (isa<EnumDecl>(D))142 return HighlightingKind::Enum;143 if (isa<EnumConstantDecl>(D))144 return HighlightingKind::EnumConstant;145 if (isa<ParmVarDecl>(D))146 return HighlightingKind::Parameter;147 if (auto *VD = dyn_cast<VarDecl>(D)) {148 if (isa<ImplicitParamDecl>(VD)) // e.g. ObjC Self149 return std::nullopt;150 return VD->isStaticDataMember()151 ? HighlightingKind::StaticField152 : VD->isLocalVarDecl() ? HighlightingKind::LocalVariable153 : HighlightingKind::Variable;154 }155 if (const auto *BD = dyn_cast<BindingDecl>(D))156 return BD->getDeclContext()->isFunctionOrMethod()157 ? HighlightingKind::LocalVariable158 : HighlightingKind::Variable;159 if (isa<FunctionDecl>(D))160 return HighlightingKind::Function;161 if (isa<NamespaceDecl>(D) || isa<NamespaceAliasDecl>(D) ||162 isa<UsingDirectiveDecl>(D))163 return HighlightingKind::Namespace;164 if (isa<TemplateTemplateParmDecl>(D) || isa<TemplateTypeParmDecl>(D) ||165 isa<NonTypeTemplateParmDecl>(D))166 return HighlightingKind::TemplateParameter;167 if (isa<ConceptDecl>(D))168 return HighlightingKind::Concept;169 if (isa<LabelDecl>(D))170 return HighlightingKind::Label;171 if (const auto *UUVD = dyn_cast<UnresolvedUsingValueDecl>(D)) {172 auto Targets = Resolver->resolveUsingValueDecl(UUVD);173 if (!Targets.empty() && Targets[0] != UUVD) {174 return kindForDecl(Targets[0], Resolver);175 }176 return HighlightingKind::Unknown;177 }178 return std::nullopt;179}180std::optional<HighlightingKind> kindForType(const Type *TP,181 const HeuristicResolver *Resolver) {182 if (!TP)183 return std::nullopt;184 if (TP->isBuiltinType()) // Builtins are special, they do not have decls.185 return HighlightingKind::Primitive;186 if (auto *TD = dyn_cast<TemplateTypeParmType>(TP))187 return kindForDecl(TD->getDecl(), Resolver);188 if (isa<ObjCObjectPointerType>(TP))189 return HighlightingKind::Class;190 if (auto *TD = TP->getAsTagDecl())191 return kindForDecl(TD, Resolver);192 return std::nullopt;193}194 195// Whether T is const in a loose sense - is a variable with this type readonly?196bool isConst(QualType T) {197 if (T.isNull())198 return false;199 T = T.getNonReferenceType();200 if (T.isConstQualified())201 return true;202 if (const auto *AT = T->getAsArrayTypeUnsafe())203 return isConst(AT->getElementType());204 if (isConst(T->getPointeeType()))205 return true;206 return false;207}208 209// Whether D is const in a loose sense (should it be highlighted as such?)210// FIXME: This is separate from whether *a particular usage* can mutate D.211// We may want V in V.size() to be readonly even if V is mutable.212bool isConst(const Decl *D) {213 if (llvm::isa<EnumConstantDecl>(D) || llvm::isa<NonTypeTemplateParmDecl>(D))214 return true;215 if (llvm::isa<FieldDecl>(D) || llvm::isa<VarDecl>(D) ||216 llvm::isa<MSPropertyDecl>(D) || llvm::isa<BindingDecl>(D)) {217 if (isConst(llvm::cast<ValueDecl>(D)->getType()))218 return true;219 }220 if (const auto *OCPD = llvm::dyn_cast<ObjCPropertyDecl>(D)) {221 if (OCPD->isReadOnly())222 return true;223 }224 if (const auto *MPD = llvm::dyn_cast<MSPropertyDecl>(D)) {225 if (!MPD->hasSetter())226 return true;227 }228 if (const auto *CMD = llvm::dyn_cast<CXXMethodDecl>(D)) {229 if (CMD->isConst())230 return true;231 }232 return false;233}234 235// "Static" means many things in C++, only some get the "static" modifier.236//237// Meanings that do:238// - Members associated with the class rather than the instance.239// This is what 'static' most often means across languages.240// - static local variables241// These are similarly "detached from their context" by the static keyword.242// In practice, these are rarely used inside classes, reducing confusion.243//244// Meanings that don't:245// - Namespace-scoped variables, which have static storage class.246// This is implicit, so the keyword "static" isn't so strongly associated.247// If we want a modifier for these, "global scope" is probably the concept.248// - Namespace-scoped variables/functions explicitly marked "static".249// There the keyword changes *linkage* , which is a totally different concept.250// If we want to model this, "file scope" would be a nice modifier.251//252// This is confusing, and maybe we should use another name, but because "static"253// is a standard LSP modifier, having one with that name has advantages.254bool isStatic(const Decl *D) {255 if (const auto *CMD = llvm::dyn_cast<CXXMethodDecl>(D))256 return CMD->isStatic();257 if (const VarDecl *VD = llvm::dyn_cast<VarDecl>(D))258 return VD->isStaticDataMember() || VD->isStaticLocal();259 if (const auto *OPD = llvm::dyn_cast<ObjCPropertyDecl>(D))260 return OPD->isClassProperty();261 if (const auto *OMD = llvm::dyn_cast<ObjCMethodDecl>(D))262 return OMD->isClassMethod();263 return false;264}265 266bool isAbstract(const Decl *D) {267 if (const auto *CMD = llvm::dyn_cast<CXXMethodDecl>(D))268 return CMD->isPureVirtual();269 if (const auto *CRD = llvm::dyn_cast<CXXRecordDecl>(D))270 return CRD->hasDefinition() && CRD->isAbstract();271 return false;272}273 274bool isVirtual(const Decl *D) {275 if (const auto *CMD = llvm::dyn_cast<CXXMethodDecl>(D))276 return CMD->isVirtual();277 return false;278}279 280bool isDependent(const Decl *D) {281 if (isa<UnresolvedUsingValueDecl>(D))282 return true;283 return false;284}285 286/// Returns true if `Decl` is considered to be from a default/system library.287/// This currently checks the systemness of the file by include type, although288/// different heuristics may be used in the future (e.g. sysroot paths).289bool isDefaultLibrary(const Decl *D) {290 SourceLocation Loc = D->getLocation();291 if (!Loc.isValid())292 return false;293 return D->getASTContext().getSourceManager().isInSystemHeader(Loc);294}295 296bool isDefaultLibrary(const Type *T) {297 if (!T)298 return false;299 const Type *Underlying = T->getPointeeOrArrayElementType();300 if (Underlying->isBuiltinType())301 return true;302 if (auto *TD = dyn_cast<TemplateTypeParmType>(Underlying))303 return isDefaultLibrary(TD->getDecl());304 if (auto *TD = Underlying->getAsTagDecl())305 return isDefaultLibrary(TD);306 return false;307}308 309// For a macro usage `DUMP(foo)`, we want:310// - DUMP --> "macro"311// - foo --> "variable".312SourceLocation getHighlightableSpellingToken(SourceLocation L,313 const SourceManager &SM) {314 if (L.isFileID())315 return SM.isWrittenInMainFile(L) ? L : SourceLocation{};316 // Tokens expanded from the macro body contribute no highlightings.317 if (!SM.isMacroArgExpansion(L))318 return {};319 // Tokens expanded from macro args are potentially highlightable.320 return getHighlightableSpellingToken(SM.getImmediateSpellingLoc(L), SM);321}322 323unsigned evaluateHighlightPriority(const HighlightingToken &Tok) {324 enum HighlightPriority { Dependent = 0, Resolved = 1 };325 return (Tok.Modifiers & (1 << uint32_t(HighlightingModifier::DependentName)))326 ? Dependent327 : Resolved;328}329 330// Sometimes we get multiple tokens at the same location:331//332// - findExplicitReferences() returns a heuristic result for a dependent name333// (e.g. Method) and CollectExtraHighlighting returning a fallback dependent334// highlighting (e.g. Unknown+Dependent).335// - macro arguments are expanded multiple times and have different roles336// - broken code recovery produces several AST nodes at the same location337//338// We should either resolve these to a single token, or drop them all.339// Our heuristics are:340//341// - token kinds that come with "dependent-name" modifiers are less reliable342// (these tend to be vague, like Type or Unknown)343// - if we have multiple equally reliable kinds, drop token rather than guess344// - take the union of modifiers from all tokens345//346// In particular, heuristically resolved dependent names get their heuristic347// kind, plus the dependent modifier.348std::optional<HighlightingToken> resolveConflict(const HighlightingToken &A,349 const HighlightingToken &B) {350 unsigned Priority1 = evaluateHighlightPriority(A);351 unsigned Priority2 = evaluateHighlightPriority(B);352 if (Priority1 == Priority2 && A.Kind != B.Kind)353 return std::nullopt;354 auto Result = Priority1 > Priority2 ? A : B;355 Result.Modifiers = A.Modifiers | B.Modifiers;356 return Result;357}358std::optional<HighlightingToken>359resolveConflict(ArrayRef<HighlightingToken> Tokens) {360 if (Tokens.size() == 1)361 return Tokens[0];362 363 assert(Tokens.size() >= 2);364 std::optional<HighlightingToken> Winner =365 resolveConflict(Tokens[0], Tokens[1]);366 for (size_t I = 2; Winner && I < Tokens.size(); ++I)367 Winner = resolveConflict(*Winner, Tokens[I]);368 return Winner;369}370 371/// Filter to remove particular kinds of highlighting tokens and modifiers from372/// the output.373class HighlightingFilter {374public:375 HighlightingFilter() {376 for (auto &Active : ActiveKindLookup)377 Active = true;378 379 ActiveModifiersMask = ~0;380 }381 382 void disableKind(HighlightingKind Kind) {383 ActiveKindLookup[static_cast<size_t>(Kind)] = false;384 }385 386 void disableModifier(HighlightingModifier Modifier) {387 ActiveModifiersMask &= ~(1 << static_cast<uint32_t>(Modifier));388 }389 390 bool isHighlightKindActive(HighlightingKind Kind) const {391 return ActiveKindLookup[static_cast<size_t>(Kind)];392 }393 394 uint32_t maskModifiers(uint32_t Modifiers) const {395 return Modifiers & ActiveModifiersMask;396 }397 398 static HighlightingFilter fromCurrentConfig() {399 const Config &C = Config::current();400 HighlightingFilter Filter;401 for (const auto &Kind : C.SemanticTokens.DisabledKinds)402 if (auto K = highlightingKindFromString(Kind))403 Filter.disableKind(*K);404 for (const auto &Modifier : C.SemanticTokens.DisabledModifiers)405 if (auto M = highlightingModifierFromString(Modifier))406 Filter.disableModifier(*M);407 408 return Filter;409 }410 411private:412 bool ActiveKindLookup[static_cast<size_t>(HighlightingKind::LastKind) + 1];413 uint32_t ActiveModifiersMask;414};415 416/// Consumes source locations and maps them to text ranges for highlightings.417class HighlightingsBuilder {418public:419 HighlightingsBuilder(const ParsedAST &AST, const HighlightingFilter &Filter)420 : TB(AST.getTokens()), SourceMgr(AST.getSourceManager()),421 LangOpts(AST.getLangOpts()), Filter(Filter),422 Resolver(AST.getHeuristicResolver()) {}423 424 HighlightingToken &addToken(SourceLocation Loc, HighlightingKind Kind) {425 auto Range = getRangeForSourceLocation(Loc);426 if (!Range)427 return InvalidHighlightingToken;428 429 return addToken(*Range, Kind);430 }431 432 // Most of this function works around433 // https://github.com/clangd/clangd/issues/871.434 void addAngleBracketTokens(SourceLocation LLoc, SourceLocation RLoc) {435 if (!LLoc.isValid() || !RLoc.isValid())436 return;437 438 auto LRange = getRangeForSourceLocation(LLoc);439 if (!LRange)440 return;441 442 // RLoc might be pointing at a virtual buffer when it's part of a `>>`443 // token.444 RLoc = SourceMgr.getFileLoc(RLoc);445 // Make sure token is part of the main file.446 RLoc = getHighlightableSpellingToken(RLoc, SourceMgr);447 if (!RLoc.isValid())448 return;449 450 const auto *RTok = TB.spelledTokenContaining(RLoc);451 // Handle `>>`. RLoc is either part of `>>` or a spelled token on its own452 // `>`. If it's the former, slice to have length of 1, if latter use the453 // token as-is.454 if (!RTok || RTok->kind() == tok::greatergreater) {455 Position Begin = sourceLocToPosition(SourceMgr, RLoc);456 Position End = sourceLocToPosition(SourceMgr, RLoc.getLocWithOffset(1));457 addToken(*LRange, HighlightingKind::Bracket);458 addToken({Begin, End}, HighlightingKind::Bracket);459 return;460 }461 462 // Easy case, we have the `>` token directly available.463 if (RTok->kind() == tok::greater) {464 if (auto RRange = getRangeForSourceLocation(RLoc)) {465 addToken(*LRange, HighlightingKind::Bracket);466 addToken(*RRange, HighlightingKind::Bracket);467 }468 return;469 }470 }471 472 HighlightingToken &addToken(Range R, HighlightingKind Kind) {473 if (!Filter.isHighlightKindActive(Kind))474 return InvalidHighlightingToken;475 476 HighlightingToken HT;477 HT.R = std::move(R);478 HT.Kind = Kind;479 Tokens.push_back(std::move(HT));480 return Tokens.back();481 }482 483 void addExtraModifier(SourceLocation Loc, HighlightingModifier Modifier) {484 if (auto Range = getRangeForSourceLocation(Loc))485 ExtraModifiers[*Range].push_back(Modifier);486 }487 488 std::vector<HighlightingToken> collect(ParsedAST &AST) && {489 // Initializer lists can give duplicates of tokens, therefore all tokens490 // must be deduplicated.491 llvm::sort(Tokens);492 auto Last = llvm::unique(Tokens);493 Tokens.erase(Last, Tokens.end());494 495 // Macros can give tokens that have the same source range but conflicting496 // kinds. In this case all tokens sharing this source range should be497 // removed.498 std::vector<HighlightingToken> NonConflicting;499 NonConflicting.reserve(Tokens.size());500 for (ArrayRef<HighlightingToken> TokRef = Tokens; !TokRef.empty();) {501 ArrayRef<HighlightingToken> Conflicting =502 TokRef.take_while([&](const HighlightingToken &T) {503 // TokRef is guaranteed at least one element here because otherwise504 // this predicate would never fire.505 return T.R == TokRef.front().R;506 });507 if (auto Resolved = resolveConflict(Conflicting)) {508 // Apply extra collected highlighting modifiers509 auto Modifiers = ExtraModifiers.find(Resolved->R);510 if (Modifiers != ExtraModifiers.end()) {511 for (HighlightingModifier Mod : Modifiers->second) {512 Resolved->addModifier(Mod);513 }514 }515 516 Resolved->Modifiers = Filter.maskModifiers(Resolved->Modifiers);517 NonConflicting.push_back(*Resolved);518 }519 // TokRef[Conflicting.size()] is the next token with a different range (or520 // the end of the Tokens).521 TokRef = TokRef.drop_front(Conflicting.size());522 }523 524 if (!Filter.isHighlightKindActive(HighlightingKind::InactiveCode))525 return NonConflicting;526 527 const auto &SM = AST.getSourceManager();528 StringRef MainCode = SM.getBufferOrFake(SM.getMainFileID()).getBuffer();529 530 // Merge token stream with "inactive line" markers.531 std::vector<HighlightingToken> WithInactiveLines;532 auto SortedInactiveRegions = getInactiveRegions(AST);533 llvm::sort(SortedInactiveRegions);534 auto It = NonConflicting.begin();535 for (const Range &R : SortedInactiveRegions) {536 // Create one token for each line in the inactive range, so it works537 // with line-based diffing.538 assert(R.start.line <= R.end.line);539 for (int Line = R.start.line; Line <= R.end.line; ++Line) {540 // Copy tokens before the inactive line541 for (; It != NonConflicting.end() && It->R.start.line < Line; ++It)542 WithInactiveLines.push_back(std::move(*It));543 // Add a token for the inactive line itself.544 auto EndOfLine = endOfLine(MainCode, Line);545 if (EndOfLine) {546 HighlightingToken HT;547 WithInactiveLines.emplace_back();548 WithInactiveLines.back().Kind = HighlightingKind::InactiveCode;549 WithInactiveLines.back().R.start.line = Line;550 WithInactiveLines.back().R.end = *EndOfLine;551 } else {552 elog("Failed to determine end of line: {0}", EndOfLine.takeError());553 }554 555 // Skip any other tokens on the inactive line. e.g.556 // `#ifndef Foo` is considered as part of an inactive region when Foo is557 // defined, and there is a Foo macro token.558 // FIXME: we should reduce the scope of the inactive region to not559 // include the directive itself.560 while (It != NonConflicting.end() && It->R.start.line == Line)561 ++It;562 }563 }564 // Copy tokens after the last inactive line565 for (; It != NonConflicting.end(); ++It)566 WithInactiveLines.push_back(std::move(*It));567 return WithInactiveLines;568 }569 570 const HeuristicResolver *getResolver() const { return Resolver; }571 572private:573 std::optional<Range> getRangeForSourceLocation(SourceLocation Loc) {574 Loc = getHighlightableSpellingToken(Loc, SourceMgr);575 if (Loc.isInvalid())576 return std::nullopt;577 // We might have offsets in the main file that don't correspond to any578 // spelled tokens.579 const auto *Tok = TB.spelledTokenContaining(Loc);580 if (!Tok)581 return std::nullopt;582 return halfOpenToRange(SourceMgr,583 Tok->range(SourceMgr).toCharRange(SourceMgr));584 }585 586 const syntax::TokenBuffer &TB;587 const SourceManager &SourceMgr;588 const LangOptions &LangOpts;589 HighlightingFilter Filter;590 std::vector<HighlightingToken> Tokens;591 std::map<Range, llvm::SmallVector<HighlightingModifier, 1>> ExtraModifiers;592 const HeuristicResolver *Resolver;593 // returned from addToken(InvalidLoc)594 HighlightingToken InvalidHighlightingToken;595};596 597std::optional<HighlightingModifier> scopeModifier(const NamedDecl *D) {598 const DeclContext *DC = D->getDeclContext();599 // Injected "Foo" within the class "Foo" has file scope, not class scope.600 if (auto *R = dyn_cast_or_null<CXXRecordDecl>(D))601 if (R->isInjectedClassName())602 DC = DC->getParent();603 // Lambda captures are considered function scope, not class scope.604 if (llvm::isa<FieldDecl>(D))605 if (const auto *RD = llvm::dyn_cast<RecordDecl>(DC))606 if (RD->isLambda())607 return HighlightingModifier::FunctionScope;608 // Walk up the DeclContext hierarchy until we find something interesting.609 for (; !DC->isFileContext(); DC = DC->getParent()) {610 if (DC->isFunctionOrMethod())611 return HighlightingModifier::FunctionScope;612 if (DC->isRecord())613 return HighlightingModifier::ClassScope;614 }615 // Some template parameters (e.g. those for variable templates) don't have616 // meaningful DeclContexts. That doesn't mean they're global!617 if (DC->isTranslationUnit() && D->isTemplateParameter())618 return std::nullopt;619 // ExternalLinkage threshold could be tweaked, e.g. module-visible as global.620 if (llvm::to_underlying(D->getLinkageInternal()) <621 llvm::to_underlying(Linkage::External))622 return HighlightingModifier::FileScope;623 return HighlightingModifier::GlobalScope;624}625 626std::optional<HighlightingModifier> scopeModifier(const Type *T) {627 if (!T)628 return std::nullopt;629 if (T->isBuiltinType())630 return HighlightingModifier::GlobalScope;631 if (auto *TD = dyn_cast<TemplateTypeParmType>(T))632 return scopeModifier(TD->getDecl());633 if (auto *TD = T->getAsTagDecl())634 return scopeModifier(TD);635 return std::nullopt;636}637 638/// Produces highlightings, which are not captured by findExplicitReferences,639/// e.g. highlights dependent names and 'auto' as the underlying type.640class CollectExtraHighlightings641 : public RecursiveASTVisitor<CollectExtraHighlightings> {642 using Base = RecursiveASTVisitor<CollectExtraHighlightings>;643 644public:645 CollectExtraHighlightings(HighlightingsBuilder &H) : H(H) {}646 647 bool VisitCXXConstructExpr(CXXConstructExpr *E) {648 highlightMutableReferenceArguments(E->getConstructor(),649 {E->getArgs(), E->getNumArgs()});650 651 return true;652 }653 654 bool TraverseConstructorInitializer(CXXCtorInitializer *Init) {655 if (Init->isMemberInitializer())656 if (auto *Member = Init->getMember())657 highlightMutableReferenceArgument(Member->getType(), Init->getInit());658 return Base::TraverseConstructorInitializer(Init);659 }660 661 bool TraverseTypeConstraint(const TypeConstraint *C) {662 if (auto *Args = C->getTemplateArgsAsWritten())663 H.addAngleBracketTokens(Args->getLAngleLoc(), Args->getRAngleLoc());664 return Base::TraverseTypeConstraint(C);665 }666 667 bool VisitPredefinedExpr(PredefinedExpr *E) {668 H.addToken(E->getLocation(), HighlightingKind::LocalVariable)669 .addModifier(HighlightingModifier::Static)670 .addModifier(HighlightingModifier::Readonly)671 .addModifier(HighlightingModifier::FunctionScope);672 return true;673 }674 675 bool VisitConceptSpecializationExpr(ConceptSpecializationExpr *E) {676 if (auto *Args = E->getTemplateArgsAsWritten())677 H.addAngleBracketTokens(Args->getLAngleLoc(), Args->getRAngleLoc());678 return true;679 }680 681 bool VisitTemplateDecl(TemplateDecl *D) {682 if (auto *TPL = D->getTemplateParameters())683 H.addAngleBracketTokens(TPL->getLAngleLoc(), TPL->getRAngleLoc());684 return true;685 }686 687 bool VisitTagDecl(TagDecl *D) {688 for (unsigned i = 0; i < D->getNumTemplateParameterLists(); ++i) {689 if (auto *TPL = D->getTemplateParameterList(i))690 H.addAngleBracketTokens(TPL->getLAngleLoc(), TPL->getRAngleLoc());691 }692 return true;693 }694 695 bool696 VisitClassTemplateSpecializationDecl(ClassTemplateSpecializationDecl *D) {697 if (auto *Args = D->getTemplateArgsAsWritten())698 H.addAngleBracketTokens(Args->getLAngleLoc(), Args->getRAngleLoc());699 return true;700 }701 702 bool VisitClassTemplatePartialSpecializationDecl(703 ClassTemplatePartialSpecializationDecl *D) {704 if (auto *TPL = D->getTemplateParameters())705 H.addAngleBracketTokens(TPL->getLAngleLoc(), TPL->getRAngleLoc());706 return true;707 }708 709 bool VisitVarTemplateSpecializationDecl(VarTemplateSpecializationDecl *D) {710 if (auto *Args = D->getTemplateArgsAsWritten())711 H.addAngleBracketTokens(Args->getLAngleLoc(), Args->getRAngleLoc());712 return true;713 }714 715 bool VisitVarTemplatePartialSpecializationDecl(716 VarTemplatePartialSpecializationDecl *D) {717 if (auto *TPL = D->getTemplateParameters())718 H.addAngleBracketTokens(TPL->getLAngleLoc(), TPL->getRAngleLoc());719 return true;720 }721 722 bool VisitDeclRefExpr(DeclRefExpr *E) {723 H.addAngleBracketTokens(E->getLAngleLoc(), E->getRAngleLoc());724 return true;725 }726 bool VisitMemberExpr(MemberExpr *E) {727 H.addAngleBracketTokens(E->getLAngleLoc(), E->getRAngleLoc());728 return true;729 }730 731 bool VisitFunctionDecl(FunctionDecl *D) {732 if (D->isOverloadedOperator()) {733 const auto AddOpDeclToken = [&](SourceLocation Loc) {734 auto &Token = H.addToken(Loc, HighlightingKind::Operator)735 .addModifier(HighlightingModifier::Declaration);736 if (D->isThisDeclarationADefinition())737 Token.addModifier(HighlightingModifier::Definition);738 };739 const auto Range = D->getNameInfo().getCXXOperatorNameRange();740 AddOpDeclToken(Range.getBegin());741 const auto Kind = D->getOverloadedOperator();742 if (Kind == OO_Call || Kind == OO_Subscript)743 AddOpDeclToken(Range.getEnd());744 }745 if (auto *Args = D->getTemplateSpecializationArgsAsWritten())746 H.addAngleBracketTokens(Args->getLAngleLoc(), Args->getRAngleLoc());747 return true;748 }749 750 bool VisitCXXOperatorCallExpr(CXXOperatorCallExpr *E) {751 const auto AddOpToken = [&](SourceLocation Loc) {752 H.addToken(Loc, HighlightingKind::Operator)753 .addModifier(HighlightingModifier::UserDefined);754 };755 AddOpToken(E->getOperatorLoc());756 const auto Kind = E->getOperator();757 if (Kind == OO_Call || Kind == OO_Subscript) {758 if (auto *Callee = E->getCallee())759 AddOpToken(Callee->getBeginLoc());760 }761 return true;762 }763 764 bool VisitUnaryOperator(UnaryOperator *Op) {765 auto &Token = H.addToken(Op->getOperatorLoc(), HighlightingKind::Operator);766 if (Op->getSubExpr()->isTypeDependent())767 Token.addModifier(HighlightingModifier::UserDefined);768 return true;769 }770 771 bool VisitBinaryOperator(BinaryOperator *Op) {772 auto &Token = H.addToken(Op->getOperatorLoc(), HighlightingKind::Operator);773 if (Op->getLHS()->isTypeDependent() || Op->getRHS()->isTypeDependent())774 Token.addModifier(HighlightingModifier::UserDefined);775 return true;776 }777 778 bool VisitConditionalOperator(ConditionalOperator *Op) {779 H.addToken(Op->getQuestionLoc(), HighlightingKind::Operator);780 H.addToken(Op->getColonLoc(), HighlightingKind::Operator);781 return true;782 }783 784 bool VisitCXXNewExpr(CXXNewExpr *E) {785 auto &Token = H.addToken(E->getBeginLoc(), HighlightingKind::Operator);786 if (isa_and_present<CXXMethodDecl>(E->getOperatorNew()))787 Token.addModifier(HighlightingModifier::UserDefined);788 return true;789 }790 791 bool VisitCXXDeleteExpr(CXXDeleteExpr *E) {792 auto &Token = H.addToken(E->getBeginLoc(), HighlightingKind::Operator);793 if (isa_and_present<CXXMethodDecl>(E->getOperatorDelete()))794 Token.addModifier(HighlightingModifier::UserDefined);795 return true;796 }797 798 bool VisitCXXNamedCastExpr(CXXNamedCastExpr *E) {799 const auto &B = E->getAngleBrackets();800 H.addAngleBracketTokens(B.getBegin(), B.getEnd());801 return true;802 }803 804 bool VisitCallExpr(CallExpr *E) {805 // Highlighting parameters passed by non-const reference does not really806 // make sense for literals...807 if (isa<UserDefinedLiteral>(E))808 return true;809 810 // FIXME: consider highlighting parameters of some other overloaded811 // operators as well812 llvm::ArrayRef<const Expr *> Args = {E->getArgs(), E->getNumArgs()};813 if (auto *CallOp = dyn_cast<CXXOperatorCallExpr>(E)) {814 switch (CallOp->getOperator()) {815 case OO_Call:816 case OO_Subscript:817 Args = Args.drop_front(); // Drop object parameter818 break;819 default:820 return true;821 }822 }823 824 highlightMutableReferenceArguments(825 dyn_cast_or_null<FunctionDecl>(E->getCalleeDecl()), Args);826 827 return true;828 }829 830 void highlightMutableReferenceArgument(QualType T, const Expr *Arg) {831 if (!Arg)832 return;833 834 // Is this parameter passed by non-const pointer or reference?835 // FIXME The condition T->idDependentType() could be relaxed a bit,836 // e.g. std::vector<T>& is dependent but we would want to highlight it837 bool IsRef = T->isLValueReferenceType();838 bool IsPtr = T->isPointerType();839 if ((!IsRef && !IsPtr) || T->getPointeeType().isConstQualified() ||840 T->isDependentType()) {841 return;842 }843 844 std::optional<SourceLocation> Location;845 846 // FIXME Add "unwrapping" for ArraySubscriptExpr,847 // e.g. highlight `a` in `a[i]`848 // FIXME Handle dependent expression types849 if (auto *IC = dyn_cast<ImplicitCastExpr>(Arg))850 Arg = IC->getSubExprAsWritten();851 if (auto *UO = dyn_cast<UnaryOperator>(Arg)) {852 if (UO->getOpcode() == UO_AddrOf)853 Arg = UO->getSubExpr();854 }855 if (auto *DR = dyn_cast<DeclRefExpr>(Arg))856 Location = DR->getLocation();857 else if (auto *M = dyn_cast<MemberExpr>(Arg))858 Location = M->getMemberLoc();859 860 if (Location)861 H.addExtraModifier(*Location,862 IsRef ? HighlightingModifier::UsedAsMutableReference863 : HighlightingModifier::UsedAsMutablePointer);864 }865 866 void867 highlightMutableReferenceArguments(const FunctionDecl *FD,868 llvm::ArrayRef<const Expr *const> Args) {869 if (!FD)870 return;871 872 if (auto *ProtoType = FD->getType()->getAs<FunctionProtoType>()) {873 // Iterate over the types of the function parameters.874 // If any of them are non-const reference paramteres, add it as a875 // highlighting modifier to the corresponding expression876 for (size_t I = 0;877 I < std::min(size_t(ProtoType->getNumParams()), Args.size()); ++I) {878 highlightMutableReferenceArgument(ProtoType->getParamType(I), Args[I]);879 }880 }881 }882 883 bool VisitDecltypeTypeLoc(DecltypeTypeLoc L) {884 if (auto K = kindForType(L.getTypePtr(), H.getResolver())) {885 auto &Tok = H.addToken(L.getBeginLoc(), *K)886 .addModifier(HighlightingModifier::Deduced);887 if (auto Mod = scopeModifier(L.getTypePtr()))888 Tok.addModifier(*Mod);889 if (isDefaultLibrary(L.getTypePtr()))890 Tok.addModifier(HighlightingModifier::DefaultLibrary);891 }892 return true;893 }894 895 bool VisitCXXDestructorDecl(CXXDestructorDecl *D) {896 if (auto *TI = D->getNameInfo().getNamedTypeInfo()) {897 SourceLocation Loc = TI->getTypeLoc().getBeginLoc();898 H.addExtraModifier(Loc, HighlightingModifier::ConstructorOrDestructor);899 H.addExtraModifier(Loc, HighlightingModifier::Declaration);900 if (D->isThisDeclarationADefinition())901 H.addExtraModifier(Loc, HighlightingModifier::Definition);902 }903 return true;904 }905 906 bool VisitCXXMemberCallExpr(CXXMemberCallExpr *CE) {907 // getMethodDecl can return nullptr with member pointers, e.g.908 // `(foo.*pointer_to_member_fun)(arg);`909 if (auto *D = CE->getMethodDecl()) {910 if (isa<CXXDestructorDecl>(D)) {911 if (auto *ME = dyn_cast<MemberExpr>(CE->getCallee())) {912 if (auto *TI = ME->getMemberNameInfo().getNamedTypeInfo()) {913 H.addExtraModifier(TI->getTypeLoc().getBeginLoc(),914 HighlightingModifier::ConstructorOrDestructor);915 }916 }917 } else if (D->isOverloadedOperator()) {918 if (auto *ME = dyn_cast<MemberExpr>(CE->getCallee()))919 H.addToken(920 ME->getMemberNameInfo().getCXXOperatorNameRange().getBegin(),921 HighlightingKind::Operator)922 .addModifier(HighlightingModifier::UserDefined);923 }924 }925 return true;926 }927 928 bool VisitDeclaratorDecl(DeclaratorDecl *D) {929 for (unsigned i = 0; i < D->getNumTemplateParameterLists(); ++i) {930 if (auto *TPL = D->getTemplateParameterList(i))931 H.addAngleBracketTokens(TPL->getLAngleLoc(), TPL->getRAngleLoc());932 }933 auto *AT = D->getType()->getContainedAutoType();934 if (!AT)935 return true;936 auto K =937 kindForType(AT->getDeducedType().getTypePtrOrNull(), H.getResolver());938 if (!K)939 return true;940 auto *TSI = D->getTypeSourceInfo();941 if (!TSI)942 return true;943 SourceLocation StartLoc =944 TSI->getTypeLoc().getContainedAutoTypeLoc().getNameLoc();945 // The AutoType may not have a corresponding token, e.g. in the case of946 // init-captures. In this case, StartLoc overlaps with the location947 // of the decl itself, and producing a token for the type here would result948 // in both it and the token for the decl being dropped due to conflict.949 if (StartLoc == D->getLocation())950 return true;951 952 auto &Tok =953 H.addToken(StartLoc, *K).addModifier(HighlightingModifier::Deduced);954 const Type *Deduced = AT->getDeducedType().getTypePtrOrNull();955 if (auto Mod = scopeModifier(Deduced))956 Tok.addModifier(*Mod);957 if (isDefaultLibrary(Deduced))958 Tok.addModifier(HighlightingModifier::DefaultLibrary);959 return true;960 }961 962 // We handle objective-C selectors specially, because one reference can963 // cover several non-contiguous tokens.964 void highlightObjCSelector(const ArrayRef<SourceLocation> &Locs, bool Decl,965 bool Def, bool Class, bool DefaultLibrary) {966 HighlightingKind Kind =967 Class ? HighlightingKind::StaticMethod : HighlightingKind::Method;968 for (SourceLocation Part : Locs) {969 auto &Tok =970 H.addToken(Part, Kind).addModifier(HighlightingModifier::ClassScope);971 if (Decl)972 Tok.addModifier(HighlightingModifier::Declaration);973 if (Def)974 Tok.addModifier(HighlightingModifier::Definition);975 if (Class)976 Tok.addModifier(HighlightingModifier::Static);977 if (DefaultLibrary)978 Tok.addModifier(HighlightingModifier::DefaultLibrary);979 }980 }981 982 bool VisitObjCMethodDecl(ObjCMethodDecl *OMD) {983 llvm::SmallVector<SourceLocation> Locs;984 OMD->getSelectorLocs(Locs);985 highlightObjCSelector(Locs, /*Decl=*/true,986 OMD->isThisDeclarationADefinition(),987 OMD->isClassMethod(), isDefaultLibrary(OMD));988 return true;989 }990 991 bool VisitObjCMessageExpr(ObjCMessageExpr *OME) {992 llvm::SmallVector<SourceLocation> Locs;993 OME->getSelectorLocs(Locs);994 bool DefaultLibrary = false;995 if (ObjCMethodDecl *OMD = OME->getMethodDecl())996 DefaultLibrary = isDefaultLibrary(OMD);997 highlightObjCSelector(Locs, /*Decl=*/false, /*Def=*/false,998 OME->isClassMessage(), DefaultLibrary);999 return true;1000 }1001 1002 // Objective-C allows you to use property syntax `self.prop` as sugar for1003 // `[self prop]` and `[self setProp:]` when there's no explicit `@property`1004 // for `prop` as well as for class properties. We treat this like a property1005 // even though semantically it's equivalent to a method expression.1006 void highlightObjCImplicitPropertyRef(const ObjCMethodDecl *OMD,1007 SourceLocation Loc) {1008 auto &Tok = H.addToken(Loc, HighlightingKind::Field)1009 .addModifier(HighlightingModifier::ClassScope);1010 if (OMD->isClassMethod())1011 Tok.addModifier(HighlightingModifier::Static);1012 if (isDefaultLibrary(OMD))1013 Tok.addModifier(HighlightingModifier::DefaultLibrary);1014 }1015 1016 bool VisitObjCPropertyRefExpr(ObjCPropertyRefExpr *OPRE) {1017 // We need to handle implicit properties here since they will appear to1018 // reference `ObjCMethodDecl` via an implicit `ObjCMessageExpr`, so normal1019 // highlighting will not work.1020 if (!OPRE->isImplicitProperty())1021 return true;1022 // A single property expr can reference both a getter and setter, but we can1023 // only provide a single semantic token, so prefer the getter. In most cases1024 // the end result should be the same, although it's technically possible1025 // that the user defines a setter for a system SDK.1026 if (OPRE->isMessagingGetter()) {1027 highlightObjCImplicitPropertyRef(OPRE->getImplicitPropertyGetter(),1028 OPRE->getLocation());1029 return true;1030 }1031 if (OPRE->isMessagingSetter()) {1032 highlightObjCImplicitPropertyRef(OPRE->getImplicitPropertySetter(),1033 OPRE->getLocation());1034 }1035 return true;1036 }1037 1038 bool VisitOverloadExpr(OverloadExpr *E) {1039 H.addAngleBracketTokens(E->getLAngleLoc(), E->getRAngleLoc());1040 if (!E->decls().empty())1041 return true; // handled by findExplicitReferences.1042 auto &Tok = H.addToken(E->getNameLoc(), HighlightingKind::Unknown)1043 .addModifier(HighlightingModifier::DependentName);1044 if (llvm::isa<UnresolvedMemberExpr>(E))1045 Tok.addModifier(HighlightingModifier::ClassScope);1046 // other case is UnresolvedLookupExpr, scope is unknown.1047 return true;1048 }1049 1050 bool VisitCXXDependentScopeMemberExpr(CXXDependentScopeMemberExpr *E) {1051 H.addToken(E->getMemberNameInfo().getLoc(), HighlightingKind::Unknown)1052 .addModifier(HighlightingModifier::DependentName)1053 .addModifier(HighlightingModifier::ClassScope);1054 H.addAngleBracketTokens(E->getLAngleLoc(), E->getRAngleLoc());1055 return true;1056 }1057 1058 bool VisitDependentScopeDeclRefExpr(DependentScopeDeclRefExpr *E) {1059 H.addToken(E->getNameInfo().getLoc(), HighlightingKind::Unknown)1060 .addModifier(HighlightingModifier::DependentName)1061 .addModifier(HighlightingModifier::ClassScope);1062 H.addAngleBracketTokens(E->getLAngleLoc(), E->getRAngleLoc());1063 return true;1064 }1065 1066 bool VisitAttr(Attr *A) {1067 switch (A->getKind()) {1068 case attr::Override:1069 case attr::Final:1070 H.addToken(A->getLocation(), HighlightingKind::Modifier);1071 break;1072 default:1073 break;1074 }1075 return true;1076 }1077 1078 bool VisitDependentNameTypeLoc(DependentNameTypeLoc L) {1079 H.addToken(L.getNameLoc(), HighlightingKind::Type)1080 .addModifier(HighlightingModifier::DependentName)1081 .addModifier(HighlightingModifier::ClassScope);1082 return true;1083 }1084 1085 bool VisitTemplateSpecializationTypeLoc(TemplateSpecializationTypeLoc L) {1086 if (!L.getTypePtr()->getTemplateName().getAsTemplateDecl(1087 /*IgnoreDeduced=*/true))1088 H.addToken(L.getTemplateNameLoc(), HighlightingKind::Type)1089 .addModifier(HighlightingModifier::DependentName)1090 .addModifier(HighlightingModifier::ClassScope);1091 H.addAngleBracketTokens(L.getLAngleLoc(), L.getRAngleLoc());1092 return true;1093 }1094 1095 bool TraverseTemplateArgumentLoc(TemplateArgumentLoc L) {1096 // Handle template template arguments only (other arguments are handled by1097 // their Expr, TypeLoc etc values).1098 if (L.getArgument().getKind() != TemplateArgument::Template &&1099 L.getArgument().getKind() != TemplateArgument::TemplateExpansion)1100 return RecursiveASTVisitor::TraverseTemplateArgumentLoc(L);1101 1102 TemplateName N = L.getArgument().getAsTemplateOrTemplatePattern();1103 switch (N.getKind()) {1104 case TemplateName::OverloadedTemplate:1105 // Template template params must always be class templates.1106 // Don't bother to try to work out the scope here.1107 H.addToken(L.getTemplateNameLoc(), HighlightingKind::Class);1108 break;1109 case TemplateName::DependentTemplate:1110 case TemplateName::AssumedTemplate:1111 H.addToken(L.getTemplateNameLoc(), HighlightingKind::Class)1112 .addModifier(HighlightingModifier::DependentName);1113 break;1114 case TemplateName::Template:1115 case TemplateName::QualifiedTemplate:1116 case TemplateName::SubstTemplateTemplateParm:1117 case TemplateName::SubstTemplateTemplateParmPack:1118 case TemplateName::UsingTemplate:1119 case TemplateName::DeducedTemplate:1120 // Names that could be resolved to a TemplateDecl are handled elsewhere.1121 break;1122 }1123 return RecursiveASTVisitor::TraverseTemplateArgumentLoc(L);1124 }1125 1126private:1127 HighlightingsBuilder &H;1128};1129} // namespace1130 1131std::vector<HighlightingToken>1132getSemanticHighlightings(ParsedAST &AST, bool IncludeInactiveRegionTokens) {1133 auto &C = AST.getASTContext();1134 HighlightingFilter Filter = HighlightingFilter::fromCurrentConfig();1135 if (!IncludeInactiveRegionTokens)1136 Filter.disableKind(HighlightingKind::InactiveCode);1137 // Add highlightings for AST nodes.1138 HighlightingsBuilder Builder(AST, Filter);1139 // Highlight 'decltype' and 'auto' as their underlying types.1140 CollectExtraHighlightings(Builder).TraverseAST(C);1141 // Highlight all decls and references coming from the AST.1142 findExplicitReferences(1143 C,1144 [&](ReferenceLoc R) {1145 for (const NamedDecl *Decl : R.Targets) {1146 if (!canHighlightName(Decl->getDeclName()))1147 continue;1148 auto Kind = kindForDecl(Decl, AST.getHeuristicResolver());1149 if (!Kind)1150 continue;1151 auto &Tok = Builder.addToken(R.NameLoc, *Kind);1152 1153 // The attribute tests don't want to look at the template.1154 if (auto *TD = dyn_cast<TemplateDecl>(Decl)) {1155 if (auto *Templated = TD->getTemplatedDecl())1156 Decl = Templated;1157 }1158 if (auto Mod = scopeModifier(Decl))1159 Tok.addModifier(*Mod);1160 if (isConst(Decl))1161 Tok.addModifier(HighlightingModifier::Readonly);1162 if (isStatic(Decl))1163 Tok.addModifier(HighlightingModifier::Static);1164 if (isAbstract(Decl))1165 Tok.addModifier(HighlightingModifier::Abstract);1166 if (isVirtual(Decl))1167 Tok.addModifier(HighlightingModifier::Virtual);1168 if (isDependent(Decl))1169 Tok.addModifier(HighlightingModifier::DependentName);1170 if (isDefaultLibrary(Decl))1171 Tok.addModifier(HighlightingModifier::DefaultLibrary);1172 if (Decl->isDeprecated())1173 Tok.addModifier(HighlightingModifier::Deprecated);1174 if (isa<CXXConstructorDecl>(Decl))1175 Tok.addModifier(HighlightingModifier::ConstructorOrDestructor);1176 if (R.IsDecl) {1177 // Do not treat an UnresolvedUsingValueDecl as a declaration.1178 // It's more common to think of it as a reference to the1179 // underlying declaration.1180 if (!isa<UnresolvedUsingValueDecl>(Decl))1181 Tok.addModifier(HighlightingModifier::Declaration);1182 if (isUniqueDefinition(Decl))1183 Tok.addModifier(HighlightingModifier::Definition);1184 }1185 }1186 },1187 AST.getHeuristicResolver());1188 // Add highlightings for macro references.1189 auto AddMacro = [&](const MacroOccurrence &M) {1190 auto &T = Builder.addToken(M.toRange(C.getSourceManager()),1191 HighlightingKind::Macro);1192 T.addModifier(HighlightingModifier::GlobalScope);1193 if (M.IsDefinition)1194 T.addModifier(HighlightingModifier::Declaration);1195 };1196 for (const auto &SIDToRefs : AST.getMacros().MacroRefs)1197 for (const auto &M : SIDToRefs.second)1198 AddMacro(M);1199 for (const auto &M : AST.getMacros().UnknownMacros)1200 AddMacro(M);1201 1202 return std::move(Builder).collect(AST);1203}1204 1205llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, HighlightingKind K) {1206 switch (K) {1207 case HighlightingKind::Variable:1208 return OS << "Variable";1209 case HighlightingKind::LocalVariable:1210 return OS << "LocalVariable";1211 case HighlightingKind::Parameter:1212 return OS << "Parameter";1213 case HighlightingKind::Function:1214 return OS << "Function";1215 case HighlightingKind::Method:1216 return OS << "Method";1217 case HighlightingKind::StaticMethod:1218 return OS << "StaticMethod";1219 case HighlightingKind::Field:1220 return OS << "Field";1221 case HighlightingKind::StaticField:1222 return OS << "StaticField";1223 case HighlightingKind::Class:1224 return OS << "Class";1225 case HighlightingKind::Interface:1226 return OS << "Interface";1227 case HighlightingKind::Enum:1228 return OS << "Enum";1229 case HighlightingKind::EnumConstant:1230 return OS << "EnumConstant";1231 case HighlightingKind::Typedef:1232 return OS << "Typedef";1233 case HighlightingKind::Type:1234 return OS << "Type";1235 case HighlightingKind::Unknown:1236 return OS << "Unknown";1237 case HighlightingKind::Namespace:1238 return OS << "Namespace";1239 case HighlightingKind::TemplateParameter:1240 return OS << "TemplateParameter";1241 case HighlightingKind::Concept:1242 return OS << "Concept";1243 case HighlightingKind::Primitive:1244 return OS << "Primitive";1245 case HighlightingKind::Macro:1246 return OS << "Macro";1247 case HighlightingKind::Modifier:1248 return OS << "Modifier";1249 case HighlightingKind::Operator:1250 return OS << "Operator";1251 case HighlightingKind::Bracket:1252 return OS << "Bracket";1253 case HighlightingKind::Label:1254 return OS << "Label";1255 case HighlightingKind::InactiveCode:1256 return OS << "InactiveCode";1257 }1258 llvm_unreachable("invalid HighlightingKind");1259}1260std::optional<HighlightingKind>1261highlightingKindFromString(llvm::StringRef Name) {1262 static llvm::StringMap<HighlightingKind> Lookup = {1263 {"Variable", HighlightingKind::Variable},1264 {"LocalVariable", HighlightingKind::LocalVariable},1265 {"Parameter", HighlightingKind::Parameter},1266 {"Function", HighlightingKind::Function},1267 {"Method", HighlightingKind::Method},1268 {"StaticMethod", HighlightingKind::StaticMethod},1269 {"Field", HighlightingKind::Field},1270 {"StaticField", HighlightingKind::StaticField},1271 {"Class", HighlightingKind::Class},1272 {"Interface", HighlightingKind::Interface},1273 {"Enum", HighlightingKind::Enum},1274 {"EnumConstant", HighlightingKind::EnumConstant},1275 {"Typedef", HighlightingKind::Typedef},1276 {"Type", HighlightingKind::Type},1277 {"Unknown", HighlightingKind::Unknown},1278 {"Namespace", HighlightingKind::Namespace},1279 {"TemplateParameter", HighlightingKind::TemplateParameter},1280 {"Concept", HighlightingKind::Concept},1281 {"Primitive", HighlightingKind::Primitive},1282 {"Macro", HighlightingKind::Macro},1283 {"Modifier", HighlightingKind::Modifier},1284 {"Operator", HighlightingKind::Operator},1285 {"Bracket", HighlightingKind::Bracket},1286 {"InactiveCode", HighlightingKind::InactiveCode},1287 };1288 1289 auto It = Lookup.find(Name);1290 return It != Lookup.end() ? std::make_optional(It->getValue()) : std::nullopt;1291}1292llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, HighlightingModifier K) {1293 switch (K) {1294 case HighlightingModifier::Declaration:1295 return OS << "decl"; // abbreviation for common case1296 case HighlightingModifier::Definition:1297 return OS << "def"; // abbrevation for common case1298 case HighlightingModifier::ConstructorOrDestructor:1299 return OS << "constrDestr";1300 default:1301 return OS << toSemanticTokenModifier(K);1302 }1303}1304std::optional<HighlightingModifier>1305highlightingModifierFromString(llvm::StringRef Name) {1306 static llvm::StringMap<HighlightingModifier> Lookup = {1307 {"Declaration", HighlightingModifier::Declaration},1308 {"Definition", HighlightingModifier::Definition},1309 {"Deprecated", HighlightingModifier::Deprecated},1310 {"Deduced", HighlightingModifier::Deduced},1311 {"Readonly", HighlightingModifier::Readonly},1312 {"Static", HighlightingModifier::Static},1313 {"Abstract", HighlightingModifier::Abstract},1314 {"Virtual", HighlightingModifier::Virtual},1315 {"DependentName", HighlightingModifier::DependentName},1316 {"DefaultLibrary", HighlightingModifier::DefaultLibrary},1317 {"UsedAsMutableReference", HighlightingModifier::UsedAsMutableReference},1318 {"UsedAsMutablePointer", HighlightingModifier::UsedAsMutablePointer},1319 {"ConstructorOrDestructor",1320 HighlightingModifier::ConstructorOrDestructor},1321 {"UserDefined", HighlightingModifier::UserDefined},1322 {"FunctionScope", HighlightingModifier::FunctionScope},1323 {"ClassScope", HighlightingModifier::ClassScope},1324 {"FileScope", HighlightingModifier::FileScope},1325 {"GlobalScope", HighlightingModifier::GlobalScope},1326 };1327 1328 auto It = Lookup.find(Name);1329 return It != Lookup.end() ? std::make_optional(It->getValue()) : std::nullopt;1330}1331 1332bool operator==(const HighlightingToken &L, const HighlightingToken &R) {1333 return std::tie(L.R, L.Kind, L.Modifiers) ==1334 std::tie(R.R, R.Kind, R.Modifiers);1335}1336bool operator<(const HighlightingToken &L, const HighlightingToken &R) {1337 return std::tie(L.R, L.Kind, L.Modifiers) <1338 std::tie(R.R, R.Kind, R.Modifiers);1339}1340 1341std::vector<SemanticToken>1342toSemanticTokens(llvm::ArrayRef<HighlightingToken> Tokens,1343 llvm::StringRef Code) {1344 assert(llvm::is_sorted(Tokens));1345 std::vector<SemanticToken> Result;1346 // In case we split a HighlightingToken into multiple tokens (e.g. because it1347 // was spanning multiple lines), this tracks the last one. This prevents1348 // having a copy all the time.1349 HighlightingToken Scratch;1350 const HighlightingToken *Last = nullptr;1351 for (const HighlightingToken &Tok : Tokens) {1352 Result.emplace_back();1353 SemanticToken *Out = &Result.back();1354 // deltaStart/deltaLine are relative if possible.1355 if (Last) {1356 assert(Tok.R.start.line >= Last->R.end.line);1357 Out->deltaLine = Tok.R.start.line - Last->R.end.line;1358 if (Out->deltaLine == 0) {1359 assert(Tok.R.start.character >= Last->R.start.character);1360 Out->deltaStart = Tok.R.start.character - Last->R.start.character;1361 } else {1362 Out->deltaStart = Tok.R.start.character;1363 }1364 } else {1365 Out->deltaLine = Tok.R.start.line;1366 Out->deltaStart = Tok.R.start.character;1367 }1368 Out->tokenType = static_cast<unsigned>(Tok.Kind);1369 Out->tokenModifiers = Tok.Modifiers;1370 Last = &Tok;1371 1372 if (Tok.R.end.line == Tok.R.start.line) {1373 Out->length = Tok.R.end.character - Tok.R.start.character;1374 } else {1375 // If the token spans a line break, split it into multiple pieces for each1376 // line.1377 // This is slow, but multiline tokens are rare.1378 // FIXME: There's a client capability for supporting multiline tokens,1379 // respect that.1380 auto TokStartOffset = llvm::cantFail(positionToOffset(Code, Tok.R.start));1381 // Note that the loop doesn't cover the last line, which has a special1382 // length.1383 for (int I = Tok.R.start.line; I < Tok.R.end.line; ++I) {1384 auto LineEnd = Code.find('\n', TokStartOffset);1385 assert(LineEnd != Code.npos);1386 Out->length = LineEnd - TokStartOffset;1387 // Token continues on next line, right after the line break.1388 TokStartOffset = LineEnd + 1;1389 Result.emplace_back();1390 Out = &Result.back();1391 *Out = Result[Result.size() - 2];1392 // New token starts at the first column of the next line.1393 Out->deltaLine = 1;1394 Out->deltaStart = 0;1395 }1396 // This is the token on last line.1397 Out->length = Tok.R.end.character;1398 // Update the start location for last token, as that's used in the1399 // relative delta calculation for following tokens.1400 Scratch = *Last;1401 Scratch.R.start.line = Tok.R.end.line;1402 Scratch.R.start.character = 0;1403 Last = &Scratch;1404 }1405 }1406 return Result;1407}1408llvm::StringRef toSemanticTokenType(HighlightingKind Kind) {1409 switch (Kind) {1410 case HighlightingKind::Variable:1411 case HighlightingKind::LocalVariable:1412 case HighlightingKind::StaticField:1413 return "variable";1414 case HighlightingKind::Parameter:1415 return "parameter";1416 case HighlightingKind::Function:1417 return "function";1418 case HighlightingKind::Method:1419 return "method";1420 case HighlightingKind::StaticMethod:1421 // FIXME: better method with static modifier?1422 return "function";1423 case HighlightingKind::Field:1424 return "property";1425 case HighlightingKind::Class:1426 return "class";1427 case HighlightingKind::Interface:1428 return "interface";1429 case HighlightingKind::Enum:1430 return "enum";1431 case HighlightingKind::EnumConstant:1432 return "enumMember";1433 case HighlightingKind::Typedef:1434 case HighlightingKind::Type:1435 return "type";1436 case HighlightingKind::Unknown:1437 return "unknown"; // nonstandard1438 case HighlightingKind::Namespace:1439 return "namespace";1440 case HighlightingKind::TemplateParameter:1441 return "typeParameter";1442 case HighlightingKind::Concept:1443 return "concept"; // nonstandard1444 case HighlightingKind::Primitive:1445 return "type";1446 case HighlightingKind::Macro:1447 return "macro";1448 case HighlightingKind::Modifier:1449 return "modifier";1450 case HighlightingKind::Operator:1451 return "operator";1452 case HighlightingKind::Bracket:1453 return "bracket";1454 case HighlightingKind::Label:1455 return "label";1456 case HighlightingKind::InactiveCode:1457 return "comment";1458 }1459 llvm_unreachable("unhandled HighlightingKind");1460}1461 1462llvm::StringRef toSemanticTokenModifier(HighlightingModifier Modifier) {1463 switch (Modifier) {1464 case HighlightingModifier::Declaration:1465 return "declaration";1466 case HighlightingModifier::Definition:1467 return "definition";1468 case HighlightingModifier::Deprecated:1469 return "deprecated";1470 case HighlightingModifier::Readonly:1471 return "readonly";1472 case HighlightingModifier::Static:1473 return "static";1474 case HighlightingModifier::Deduced:1475 return "deduced"; // nonstandard1476 case HighlightingModifier::Abstract:1477 return "abstract";1478 case HighlightingModifier::Virtual:1479 return "virtual";1480 case HighlightingModifier::DependentName:1481 return "dependentName"; // nonstandard1482 case HighlightingModifier::DefaultLibrary:1483 return "defaultLibrary";1484 case HighlightingModifier::UsedAsMutableReference:1485 return "usedAsMutableReference"; // nonstandard1486 case HighlightingModifier::UsedAsMutablePointer:1487 return "usedAsMutablePointer"; // nonstandard1488 case HighlightingModifier::ConstructorOrDestructor:1489 return "constructorOrDestructor"; // nonstandard1490 case HighlightingModifier::UserDefined:1491 return "userDefined"; // nonstandard1492 case HighlightingModifier::FunctionScope:1493 return "functionScope"; // nonstandard1494 case HighlightingModifier::ClassScope:1495 return "classScope"; // nonstandard1496 case HighlightingModifier::FileScope:1497 return "fileScope"; // nonstandard1498 case HighlightingModifier::GlobalScope:1499 return "globalScope"; // nonstandard1500 }1501 llvm_unreachable("unhandled HighlightingModifier");1502}1503 1504std::vector<SemanticTokensEdit>1505diffTokens(llvm::ArrayRef<SemanticToken> Old,1506 llvm::ArrayRef<SemanticToken> New) {1507 // For now, just replace everything from the first-last modification.1508 // FIXME: use a real diff instead, this is bad with include-insertion.1509 1510 unsigned Offset = 0;1511 while (!Old.empty() && !New.empty() && Old.front() == New.front()) {1512 ++Offset;1513 Old = Old.drop_front();1514 New = New.drop_front();1515 }1516 while (!Old.empty() && !New.empty() && Old.back() == New.back()) {1517 Old = Old.drop_back();1518 New = New.drop_back();1519 }1520 1521 if (Old.empty() && New.empty())1522 return {};1523 SemanticTokensEdit Edit;1524 Edit.startToken = Offset;1525 Edit.deleteTokens = Old.size();1526 Edit.tokens = New;1527 return {std::move(Edit)};1528}1529 1530std::vector<Range> getInactiveRegions(ParsedAST &AST) {1531 std::vector<Range> SkippedRanges(std::move(AST.getMacros().SkippedRanges));1532 const auto &SM = AST.getSourceManager();1533 StringRef MainCode = SM.getBufferOrFake(SM.getMainFileID()).getBuffer();1534 std::vector<Range> InactiveRegions;1535 for (const Range &Skipped : SkippedRanges) {1536 Range Inactive = Skipped;1537 // Sometimes, SkippedRanges contains a range ending at position 01538 // of a line. Clients that apply whole-line styles will treat that1539 // line as inactive which is not desirable, so adjust the ending1540 // position to be the end of the previous line.1541 if (Inactive.end.character == 0 && Inactive.end.line > 0) {1542 --Inactive.end.line;1543 }1544 // Exclude the directive lines themselves from the range.1545 if (Inactive.end.line >= Inactive.start.line + 2) {1546 ++Inactive.start.line;1547 --Inactive.end.line;1548 } else {1549 // range would be empty, e.g. #endif on next line after #ifdef1550 continue;1551 }1552 // Since we've adjusted the ending line, we need to recompute the1553 // column to reflect the end of that line.1554 if (auto EndOfLine = endOfLine(MainCode, Inactive.end.line)) {1555 Inactive.end = *EndOfLine;1556 } else {1557 elog("Failed to determine end of line: {0}", EndOfLine.takeError());1558 continue;1559 }1560 InactiveRegions.push_back(Inactive);1561 }1562 return InactiveRegions;1563}1564 1565} // namespace clangd1566} // namespace clang1567