562 lines · cpp
1//===--- SymbolDocumentation.cpp ==-------------------------------*- C++-*-===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8 9#include "SymbolDocumentation.h"10 11#include "support/Markup.h"12#include "clang/AST/Comment.h"13#include "clang/AST/CommentCommandTraits.h"14#include "clang/AST/CommentVisitor.h"15#include "llvm/ADT/DenseMap.h"16#include "llvm/ADT/StringExtras.h"17#include "llvm/ADT/StringRef.h"18 19namespace clang {20namespace clangd {21namespace {22 23std::string commandMarkerAsString(comments::CommandMarkerKind CommandMarker) {24 switch (CommandMarker) {25 case comments::CommandMarkerKind::CMK_At:26 return "@";27 case comments::CommandMarkerKind::CMK_Backslash:28 return "\\";29 }30 llvm_unreachable("Unknown command marker kind");31}32 33void commandToMarkup(markup::Paragraph &Out, StringRef Command,34 comments::CommandMarkerKind CommandMarker,35 StringRef Args) {36 Out.appendBoldText(commandMarkerAsString(CommandMarker) + Command.str());37 Out.appendSpace();38 if (!Args.empty())39 Out.appendCode(Args.str());40}41 42template <typename T> std::string getArgText(const T *Command) {43 std::string ArgText;44 for (unsigned I = 0; I < Command->getNumArgs(); ++I) {45 if (!ArgText.empty())46 ArgText += " ";47 ArgText += Command->getArgText(I);48 }49 return ArgText;50}51 52} // namespace53 54class ParagraphToMarkupDocument55 : public comments::ConstCommentVisitor<ParagraphToMarkupDocument> {56public:57 ParagraphToMarkupDocument(markup::Paragraph &Out,58 const comments::CommandTraits &Traits)59 : Out(Out), Traits(Traits) {}60 61 void visitParagraphComment(const comments::ParagraphComment *C) {62 if (!C)63 return;64 65 for (const auto *Child = C->child_begin(); Child != C->child_end();66 ++Child) {67 visit(*Child);68 }69 }70 71 void visitTextComment(const comments::TextComment *C) {72 // Always trim leading space after a newline.73 StringRef Text = C->getText();74 if (LastChunkEndsWithNewline && C->getText().starts_with(' '))75 Text = Text.drop_front();76 77 LastChunkEndsWithNewline = C->hasTrailingNewline();78 Out.appendText(Text.str() + (LastChunkEndsWithNewline ? "\n" : ""));79 }80 81 void visitInlineCommandComment(const comments::InlineCommandComment *C) {82 83 if (C->getNumArgs() > 0) {84 std::string ArgText = getArgText(C);85 86 switch (C->getRenderKind()) {87 case comments::InlineCommandRenderKind::Monospaced:88 Out.appendCode(ArgText);89 break;90 case comments::InlineCommandRenderKind::Bold:91 Out.appendBoldText(ArgText);92 break;93 case comments::InlineCommandRenderKind::Emphasized:94 Out.appendEmphasizedText(ArgText);95 break;96 default:97 commandToMarkup(Out, C->getCommandName(Traits), C->getCommandMarker(),98 ArgText);99 break;100 }101 } else {102 if (C->getCommandName(Traits) == "n") {103 // \n is a special case, it is used to create a new line.104 Out.appendText(" \n");105 LastChunkEndsWithNewline = true;106 return;107 }108 109 commandToMarkup(Out, C->getCommandName(Traits), C->getCommandMarker(),110 "");111 }112 }113 114 void visitHTMLStartTagComment(const comments::HTMLStartTagComment *STC) {115 std::string TagText = "<" + STC->getTagName().str();116 117 for (unsigned I = 0; I < STC->getNumAttrs(); ++I) {118 const comments::HTMLStartTagComment::Attribute &Attr = STC->getAttr(I);119 TagText += " " + Attr.Name.str() + "=\"" + Attr.Value.str() + "\"";120 }121 122 if (STC->isSelfClosing())123 TagText += " /";124 TagText += ">";125 126 LastChunkEndsWithNewline = STC->hasTrailingNewline();127 Out.appendText(TagText + (LastChunkEndsWithNewline ? "\n" : ""));128 }129 130 void visitHTMLEndTagComment(const comments::HTMLEndTagComment *ETC) {131 LastChunkEndsWithNewline = ETC->hasTrailingNewline();132 Out.appendText("</" + ETC->getTagName().str() + ">" +133 (LastChunkEndsWithNewline ? "\n" : ""));134 }135 136private:137 markup::Paragraph &Out;138 const comments::CommandTraits &Traits;139 140 /// If true, the next leading space after a new line is trimmed.141 /// Initially set it to true, to always trim the first text line.142 bool LastChunkEndsWithNewline = true;143};144 145class ParagraphToString146 : public comments::ConstCommentVisitor<ParagraphToString> {147public:148 ParagraphToString(llvm::raw_string_ostream &Out,149 const comments::CommandTraits &Traits)150 : Out(Out), Traits(Traits) {}151 152 void visitParagraphComment(const comments::ParagraphComment *C) {153 if (!C)154 return;155 156 for (const auto *Child = C->child_begin(); Child != C->child_end();157 ++Child) {158 visit(*Child);159 }160 }161 162 void visitTextComment(const comments::TextComment *C) { Out << C->getText(); }163 164 void visitInlineCommandComment(const comments::InlineCommandComment *C) {165 Out << commandMarkerAsString(C->getCommandMarker());166 Out << C->getCommandName(Traits);167 std::string ArgText = getArgText(C);168 if (!ArgText.empty())169 Out << " " << ArgText;170 Out << " ";171 }172 173 void visitHTMLStartTagComment(const comments::HTMLStartTagComment *STC) {174 Out << "<" << STC->getTagName().str();175 176 for (unsigned I = 0; I < STC->getNumAttrs(); ++I) {177 const comments::HTMLStartTagComment::Attribute &Attr = STC->getAttr(I);178 Out << " " << Attr.Name.str();179 if (!Attr.Value.str().empty())180 Out << "=\"" << Attr.Value.str() << "\"";181 }182 183 if (STC->isSelfClosing())184 Out << " /";185 Out << ">";186 187 Out << (STC->hasTrailingNewline() ? "\n" : "");188 }189 190 void visitHTMLEndTagComment(const comments::HTMLEndTagComment *ETC) {191 Out << "</" << ETC->getTagName().str() << ">"192 << (ETC->hasTrailingNewline() ? "\n" : "");193 }194 195private:196 llvm::raw_string_ostream &Out;197 const comments::CommandTraits &Traits;198};199 200class BlockCommentToMarkupDocument201 : public comments::ConstCommentVisitor<BlockCommentToMarkupDocument> {202public:203 BlockCommentToMarkupDocument(markup::Document &Out,204 const comments::CommandTraits &Traits)205 : Out(Out), Traits(Traits) {}206 207 void visitBlockCommandComment(const comments::BlockCommandComment *B) {208 209 switch (B->getCommandID()) {210 case comments::CommandTraits::KCI_arg:211 case comments::CommandTraits::KCI_li:212 // \li and \arg are special cases, they are used to create a list item.213 // In markdown it is a bullet list.214 ParagraphToMarkupDocument(Out.addBulletList().addItem().addParagraph(),215 Traits)216 .visit(B->getParagraph());217 break;218 case comments::CommandTraits::KCI_note:219 case comments::CommandTraits::KCI_warning:220 commandToHeadedParagraph(B);221 break;222 case comments::CommandTraits::KCI_retval: {223 // The \retval command describes the return value given as its single224 // argument in the corresponding paragraph.225 // Note: We know that we have exactly one argument but not if it has an226 // associated paragraph.227 auto &P = Out.addParagraph().appendCode(getArgText(B));228 if (B->getParagraph() && !B->getParagraph()->isWhitespace()) {229 P.appendText(" - ");230 ParagraphToMarkupDocument(P, Traits).visit(B->getParagraph());231 }232 return;233 }234 case comments::CommandTraits::KCI_details: {235 // The \details command is just used to separate the brief from the236 // detailed description. This separation is already done in the237 // SymbolDocCommentVisitor. Therefore we can omit the command itself238 // here and just process the paragraph.239 if (B->getParagraph() && !B->getParagraph()->isWhitespace()) {240 ParagraphToMarkupDocument(Out.addParagraph(), Traits)241 .visit(B->getParagraph());242 }243 return;244 }245 default: {246 // Some commands have arguments, like \throws.247 // The arguments are not part of the paragraph.248 // We need reconstruct them here.249 std::string ArgText = getArgText(B);250 auto &P = Out.addParagraph();251 commandToMarkup(P, B->getCommandName(Traits), B->getCommandMarker(),252 ArgText);253 if (B->getParagraph() && !B->getParagraph()->isWhitespace()) {254 // For commands with arguments, the paragraph starts after the first255 // space. Therefore we need to append a space manually in this case.256 if (!ArgText.empty())257 P.appendSpace();258 ParagraphToMarkupDocument(P, Traits).visit(B->getParagraph());259 }260 }261 }262 }263 264 void visitCodeCommand(const comments::VerbatimBlockComment *VB) {265 std::string CodeLang = "";266 auto *FirstLine = VB->child_begin();267 // The \\code command has an optional language argument.268 // This argument is currently not parsed by the clang doxygen parser.269 // Therefore we try to extract it from the first line of the verbatim270 // block.271 if (VB->getNumLines() > 0) {272 if (const auto *Line =273 cast<comments::VerbatimBlockLineComment>(*FirstLine)) {274 llvm::StringRef Text = Line->getText();275 // Language is a single word enclosed in {}.276 if (llvm::none_of(Text, llvm::isSpace) && Text.consume_front("{") &&277 Text.consume_back("}")) {278 // drop a potential . since this is not supported in Markdown279 // fenced code blocks.280 Text.consume_front(".");281 // Language is alphanumeric or '+'.282 CodeLang = Text.take_while([](char C) {283 return llvm::isAlnum(C) || C == '+';284 })285 .str();286 // Skip the first line for the verbatim text.287 ++FirstLine;288 }289 }290 }291 292 std::string CodeBlockText;293 294 for (const auto *LI = FirstLine; LI != VB->child_end(); ++LI) {295 if (const auto *Line = cast<comments::VerbatimBlockLineComment>(*LI)) {296 CodeBlockText += Line->getText().str() + "\n";297 }298 }299 300 Out.addCodeBlock(CodeBlockText, CodeLang);301 }302 303 void visitVerbatimBlockComment(const comments::VerbatimBlockComment *VB) {304 // The \\code command is a special verbatim block command which we handle305 // separately.306 if (VB->getCommandID() == comments::CommandTraits::KCI_code) {307 visitCodeCommand(VB);308 return;309 }310 311 commandToMarkup(Out.addParagraph(), VB->getCommandName(Traits),312 VB->getCommandMarker(), "");313 314 std::string VerbatimText;315 316 for (const auto *LI = VB->child_begin(); LI != VB->child_end(); ++LI) {317 if (const auto *Line = cast<comments::VerbatimBlockLineComment>(*LI)) {318 VerbatimText += Line->getText().str() + "\n";319 }320 }321 322 Out.addCodeBlock(VerbatimText, "");323 324 commandToMarkup(Out.addParagraph(), VB->getCloseName(),325 VB->getCommandMarker(), "");326 }327 328 void visitVerbatimLineComment(const comments::VerbatimLineComment *VL) {329 auto &P = Out.addParagraph();330 commandToMarkup(P, VL->getCommandName(Traits), VL->getCommandMarker(), "");331 P.appendSpace().appendCode(VL->getText().str(), true).appendSpace();332 }333 334private:335 markup::Document &Out;336 const comments::CommandTraits &Traits;337 StringRef CommentEscapeMarker;338 339 /// Emphasize the given command in a paragraph.340 /// Uses the command name with the first letter capitalized as the heading.341 void commandToHeadedParagraph(const comments::BlockCommandComment *B) {342 auto &P = Out.addParagraph();343 std::string Heading = B->getCommandName(Traits).slice(0, 1).upper() +344 B->getCommandName(Traits).drop_front().str();345 P.appendBoldText(Heading + ":");346 P.appendText(" \n");347 ParagraphToMarkupDocument(P, Traits).visit(B->getParagraph());348 }349};350 351void SymbolDocCommentVisitor::preprocessDocumentation(StringRef Doc) {352 enum State {353 Normal,354 FencedCodeblock,355 } State = Normal;356 std::string CodeFence;357 358 llvm::raw_string_ostream OS(CommentWithMarkers);359 360 // The documentation string is processed line by line.361 // The raw documentation string does not contain the comment markers362 // (e.g. /// or /** */).363 // But the comment lexer expects doxygen markers, so add them back.364 // We need to use the /// style doxygen markers because the comment could365 // contain the closing tag "*/" of a C Style "/** */" comment366 // which would break the parsing if we would just enclose the comment text367 // with "/** */".368 369 // Escape doxygen commands inside markdown inline code spans.370 // This is required to not let the doxygen parser interpret them as371 // commands.372 // Note: This is a heuristic which may fail in some cases.373 bool InCodeSpan = false;374 375 llvm::StringRef Line, Rest;376 for (std::tie(Line, Rest) = Doc.split('\n'); !(Line.empty() && Rest.empty());377 std::tie(Line, Rest) = Rest.split('\n')) {378 379 // Detect code fence (``` or ~~~)380 if (State == Normal) {381 llvm::StringRef Trimmed = Line.ltrim();382 if (Trimmed.starts_with("```") || Trimmed.starts_with("~~~")) {383 // https://www.doxygen.nl/manual/markdown.html#md_fenced384 CodeFence =385 Trimmed.take_while([](char C) { return C == '`' || C == '~'; })386 .str();387 // Try to detect language: first word after fence. Could also be388 // enclosed in {}389 llvm::StringRef AfterFence =390 Trimmed.drop_front(CodeFence.size()).ltrim();391 // ignore '{' at the beginning of the language name to not duplicate it392 // for the doxygen command393 AfterFence.consume_front("{");394 // The name is alphanumeric or '.' or '+'395 StringRef CodeLang = AfterFence.take_while(396 [](char C) { return llvm::isAlnum(C) || C == '.' || C == '+'; });397 398 OS << "///@code";399 400 if (!CodeLang.empty())401 OS << "{" << CodeLang.str() << "}";402 403 OS << "\n";404 405 State = FencedCodeblock;406 continue;407 }408 409 // FIXME: handle indented code blocks too?410 // In doxygen, the indentation which triggers a code block depends on the411 // indentation of the previous paragraph.412 // https://www.doxygen.nl/manual/markdown.html#mddox_code_blocks413 } else if (State == FencedCodeblock) {414 // End of code fence415 if (Line.ltrim().starts_with(CodeFence)) {416 OS << "///@endcode\n";417 State = Normal;418 continue;419 }420 OS << "///" << Line << "\n";421 continue;422 }423 424 // Normal line preprocessing (add doxygen markers, handle escaping)425 OS << "///";426 427 if (Line.empty() || Line.trim().empty()) {428 OS << "\n";429 // Empty lines reset the InCodeSpan state.430 InCodeSpan = false;431 continue;432 }433 434 if (Line.starts_with("<"))435 // A comment line starting with '///<' is treated as a doxygen436 // command. To avoid this, we add a space before the '<'.437 OS << ' ';438 439 for (char C : Line) {440 if (C == '`')441 InCodeSpan = !InCodeSpan;442 else if (InCodeSpan && (C == '@' || C == '\\'))443 OS << '\\';444 OS << C;445 }446 447 OS << "\n";448 }449 450 // Close any unclosed code block451 if (State == FencedCodeblock)452 OS << "///@endcode\n";453}454 455void SymbolDocCommentVisitor::visitBlockCommandComment(456 const comments::BlockCommandComment *B) {457 switch (B->getCommandID()) {458 case comments::CommandTraits::KCI_brief: {459 if (!BriefParagraph) {460 BriefParagraph = B->getParagraph();461 return;462 }463 break;464 }465 case comments::CommandTraits::KCI_return:466 case comments::CommandTraits::KCI_returns:467 if (!ReturnParagraph) {468 ReturnParagraph = B->getParagraph();469 return;470 }471 break;472 case comments::CommandTraits::KCI_retval:473 // Only consider retval commands having an argument.474 // The argument contains the described return value which is needed to475 // convert it to markup.476 if (B->getNumArgs() == 1)477 RetvalCommands.push_back(B);478 return;479 default:480 break;481 }482 483 // For all other commands, we store them in the BlockCommands map.484 // This allows us to keep the order of the comments.485 BlockCommands[CommentPartIndex] = B;486 CommentPartIndex++;487}488 489void SymbolDocCommentVisitor::briefToMarkup(markup::Paragraph &Out) const {490 if (!BriefParagraph)491 return;492 ParagraphToMarkupDocument(Out, Traits).visit(BriefParagraph);493}494 495void SymbolDocCommentVisitor::returnToMarkup(markup::Paragraph &Out) const {496 if (!ReturnParagraph)497 return;498 ParagraphToMarkupDocument(Out, Traits).visit(ReturnParagraph);499}500 501void SymbolDocCommentVisitor::parameterDocToMarkup(502 StringRef ParamName, markup::Paragraph &Out) const {503 if (ParamName.empty())504 return;505 506 if (const auto *P = Parameters.lookup(ParamName)) {507 ParagraphToMarkupDocument(Out, Traits).visit(P->getParagraph());508 }509}510 511void SymbolDocCommentVisitor::parameterDocToString(512 StringRef ParamName, llvm::raw_string_ostream &Out) const {513 if (ParamName.empty())514 return;515 516 if (const auto *P = Parameters.lookup(ParamName)) {517 ParagraphToString(Out, Traits).visit(P->getParagraph());518 }519}520 521void SymbolDocCommentVisitor::detailedDocToMarkup(markup::Document &Out) const {522 for (unsigned I = 0; I < CommentPartIndex; ++I) {523 if (const auto *BC = BlockCommands.lookup(I)) {524 BlockCommentToMarkupDocument(Out, Traits).visit(BC);525 } else if (const auto *P = FreeParagraphs.lookup(I)) {526 ParagraphToMarkupDocument(Out.addParagraph(), Traits).visit(P);527 }528 }529}530 531void SymbolDocCommentVisitor::templateTypeParmDocToMarkup(532 StringRef TemplateParamName, markup::Paragraph &Out) const {533 if (TemplateParamName.empty())534 return;535 536 if (const auto *TP = TemplateParameters.lookup(TemplateParamName)) {537 ParagraphToMarkupDocument(Out, Traits).visit(TP->getParagraph());538 }539}540 541void SymbolDocCommentVisitor::templateTypeParmDocToString(542 StringRef TemplateParamName, llvm::raw_string_ostream &Out) const {543 if (TemplateParamName.empty())544 return;545 546 if (const auto *P = TemplateParameters.lookup(TemplateParamName)) {547 ParagraphToString(Out, Traits).visit(P->getParagraph());548 }549}550 551void SymbolDocCommentVisitor::retvalsToMarkup(markup::Document &Out) const {552 if (RetvalCommands.empty())553 return;554 markup::BulletList &BL = Out.addBulletList();555 for (const auto *P : RetvalCommands) {556 BlockCommentToMarkupDocument(BL.addItem(), Traits).visit(P);557 }558}559 560} // namespace clangd561} // namespace clang562