542 lines · c
1//===--- BreakableToken.h - Format C++ code ---------------------*- C++ -*-===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8///9/// \file10/// Declares BreakableToken, BreakableStringLiteral, BreakableComment,11/// BreakableBlockComment and BreakableLineCommentSection classes, that contain12/// token type-specific logic to break long lines in tokens and reflow content13/// between tokens.14///15//===----------------------------------------------------------------------===//16 17#ifndef LLVM_CLANG_LIB_FORMAT_BREAKABLETOKEN_H18#define LLVM_CLANG_LIB_FORMAT_BREAKABLETOKEN_H19 20#include "Encoding.h"21#include "WhitespaceManager.h"22#include "llvm/ADT/StringSet.h"23 24namespace clang {25namespace format {26 27/// Checks if \p Token switches formatting, like /* clang-format off */.28/// \p Token must be a comment.29bool switchesFormatting(const FormatToken &Token);30 31struct FormatStyle;32 33/// Base class for tokens / ranges of tokens that can allow breaking34/// within the tokens - for example, to avoid whitespace beyond the column35/// limit, or to reflow text.36///37/// Generally, a breakable token consists of logical lines, addressed by a line38/// index. For example, in a sequence of line comments, each line comment is its39/// own logical line; similarly, for a block comment, each line in the block40/// comment is on its own logical line.41///42/// There are two methods to compute the layout of the token:43/// - getRangeLength measures the number of columns needed for a range of text44/// within a logical line, and45/// - getContentStartColumn returns the start column at which we want the46/// content of a logical line to start (potentially after introducing a line47/// break).48///49/// The mechanism to adapt the layout of the breakable token is organised50/// around the concept of a \c Split, which is a whitespace range that signifies51/// a position of the content of a token where a reformatting might be done.52///53/// Operating with splits is divided into two operations:54/// - getSplit, for finding a split starting at a position,55/// - insertBreak, for executing the split using a whitespace manager.56///57/// There is a pair of operations that are used to compress a long whitespace58/// range with a single space if that will bring the line length under the59/// column limit:60/// - getLineLengthAfterCompression, for calculating the size in columns of the61/// line after a whitespace range has been compressed, and62/// - compressWhitespace, for executing the whitespace compression using a63/// whitespace manager; note that the compressed whitespace may be in the64/// middle of the original line and of the reformatted line.65///66/// For tokens where the whitespace before each line needs to be also67/// reformatted, for example for tokens supporting reflow, there are analogous68/// operations that might be executed before the main line breaking occurs:69/// - getReflowSplit, for finding a split such that the content preceding it70/// needs to be specially reflown,71/// - reflow, for executing the split using a whitespace manager,72/// - introducesBreakBefore, for checking if reformatting the beginning73/// of the content introduces a line break before it,74/// - adaptStartOfLine, for executing the reflow using a whitespace75/// manager.76///77/// For tokens that require the whitespace after the last line to be78/// reformatted, for example in multiline jsdoc comments that require the79/// trailing '*/' to be on a line of itself, there are analogous operations80/// that might be executed after the last line has been reformatted:81/// - getSplitAfterLastLine, for finding a split after the last line that needs82/// to be reflown,83/// - replaceWhitespaceAfterLastLine, for executing the reflow using a84/// whitespace manager.85///86class BreakableToken {87public:88 /// Contains starting character index and length of split.89 typedef std::pair<StringRef::size_type, unsigned> Split;90 91 virtual ~BreakableToken() {}92 93 /// Returns the number of lines in this token in the original code.94 virtual unsigned getLineCount() const = 0;95 96 /// Returns the number of columns required to format the text in the97 /// byte range [\p Offset, \p Offset \c + \p Length).98 ///99 /// \p Offset is the byte offset from the start of the content of the line100 /// at \p LineIndex.101 ///102 /// \p StartColumn is the column at which the text starts in the formatted103 /// file, needed to compute tab stops correctly.104 virtual unsigned getRangeLength(unsigned LineIndex, unsigned Offset,105 StringRef::size_type Length,106 unsigned StartColumn) const = 0;107 108 /// Returns the number of columns required to format the text following109 /// the byte \p Offset in the line \p LineIndex, including potentially110 /// unbreakable sequences of tokens following after the end of the token.111 ///112 /// \p Offset is the byte offset from the start of the content of the line113 /// at \p LineIndex.114 ///115 /// \p StartColumn is the column at which the text starts in the formatted116 /// file, needed to compute tab stops correctly.117 ///118 /// For breakable tokens that never use extra space at the end of a line, this119 /// is equivalent to getRangeLength with a Length of StringRef::npos.120 virtual unsigned getRemainingLength(unsigned LineIndex, unsigned Offset,121 unsigned StartColumn) const {122 return getRangeLength(LineIndex, Offset, StringRef::npos, StartColumn);123 }124 125 /// Returns the column at which content in line \p LineIndex starts,126 /// assuming no reflow.127 ///128 /// If \p Break is true, returns the column at which the line should start129 /// after the line break.130 /// If \p Break is false, returns the column at which the line itself will131 /// start.132 virtual unsigned getContentStartColumn(unsigned LineIndex,133 bool Break) const = 0;134 135 /// Returns additional content indent required for the second line after the136 /// content at line \p LineIndex is broken.137 ///138 // (Next lines do not start with `///` since otherwise -Wdocumentation picks139 // up the example annotations and generates warnings for them)140 // For example, Javadoc @param annotations require and indent of 4 spaces and141 // in this example getContentIndex(1) returns 4.142 // /**143 // * @param loooooooooooooong line144 // * continuation145 // */146 virtual unsigned getContentIndent(unsigned LineIndex) const { return 0; }147 148 /// Returns a range (offset, length) at which to break the line at149 /// \p LineIndex, if previously broken at \p TailOffset. If possible, do not150 /// violate \p ColumnLimit, assuming the text starting at \p TailOffset in151 /// the token is formatted starting at ContentStartColumn in the reformatted152 /// file.153 virtual Split getSplit(unsigned LineIndex, unsigned TailOffset,154 unsigned ColumnLimit, unsigned ContentStartColumn,155 const llvm::Regex &CommentPragmasRegex) const = 0;156 157 /// Emits the previously retrieved \p Split via \p Whitespaces.158 virtual void insertBreak(unsigned LineIndex, unsigned TailOffset, Split Split,159 unsigned ContentIndent,160 WhitespaceManager &Whitespaces) const = 0;161 162 /// Returns the number of columns needed to format163 /// \p RemainingTokenColumns, assuming that Split is within the range measured164 /// by \p RemainingTokenColumns, and that the whitespace in Split is reduced165 /// to a single space.166 unsigned getLengthAfterCompression(unsigned RemainingTokenColumns,167 Split Split) const;168 169 /// Replaces the whitespace range described by \p Split with a single170 /// space.171 virtual void compressWhitespace(unsigned LineIndex, unsigned TailOffset,172 Split Split,173 WhitespaceManager &Whitespaces) const = 0;174 175 /// Returns whether the token supports reflowing text.176 virtual bool supportsReflow() const { return false; }177 178 /// Returns a whitespace range (offset, length) of the content at \p179 /// LineIndex such that the content of that line is reflown to the end of the180 /// previous one.181 ///182 /// Returning (StringRef::npos, 0) indicates reflowing is not possible.183 ///184 /// The range will include any whitespace preceding the specified line's185 /// content.186 ///187 /// If the split is not contained within one token, for example when reflowing188 /// line comments, returns (0, <length>).189 virtual Split getReflowSplit(unsigned LineIndex,190 const llvm::Regex &CommentPragmasRegex) const {191 return Split(StringRef::npos, 0);192 }193 194 /// Reflows the current line into the end of the previous one.195 virtual void reflow(unsigned LineIndex,196 WhitespaceManager &Whitespaces) const {}197 198 /// Returns whether there will be a line break at the start of the199 /// token.200 virtual bool introducesBreakBeforeToken() const { return false; }201 202 /// Replaces the whitespace between \p LineIndex-1 and \p LineIndex.203 virtual void adaptStartOfLine(unsigned LineIndex,204 WhitespaceManager &Whitespaces) const {}205 206 /// Returns a whitespace range (offset, length) of the content at207 /// the last line that needs to be reformatted after the last line has been208 /// reformatted.209 ///210 /// A result having offset == StringRef::npos means that no reformat is211 /// necessary.212 virtual Split getSplitAfterLastLine(unsigned TailOffset) const {213 return Split(StringRef::npos, 0);214 }215 216 /// Replaces the whitespace from \p SplitAfterLastLine on the last line217 /// after the last line has been formatted by performing a reformatting.218 void replaceWhitespaceAfterLastLine(unsigned TailOffset,219 Split SplitAfterLastLine,220 WhitespaceManager &Whitespaces) const {221 insertBreak(getLineCount() - 1, TailOffset, SplitAfterLastLine,222 /*ContentIndent=*/0, Whitespaces);223 }224 225 /// Updates the next token of \p State to the next token after this226 /// one. This can be used when this token manages a set of underlying tokens227 /// as a unit and is responsible for the formatting of the them.228 virtual void updateNextToken(LineState &State) const {}229 230 /// Adds replacements that are needed when the token is broken. Such as231 /// wrapping a JavaScript string in parentheses after it gets broken with plus232 /// signs.233 virtual void updateAfterBroken(WhitespaceManager &Whitespaces) const {}234 235protected:236 BreakableToken(const FormatToken &Tok, bool InPPDirective,237 encoding::Encoding Encoding, const FormatStyle &Style)238 : Tok(Tok), InPPDirective(InPPDirective), Encoding(Encoding),239 Style(Style) {}240 241 const FormatToken &Tok;242 const bool InPPDirective;243 const encoding::Encoding Encoding;244 const FormatStyle &Style;245};246 247class BreakableStringLiteral : public BreakableToken {248public:249 /// Creates a breakable token for a single line string literal.250 ///251 /// \p StartColumn specifies the column in which the token will start252 /// after formatting.253 BreakableStringLiteral(const FormatToken &Tok, unsigned StartColumn,254 StringRef Prefix, StringRef Postfix,255 unsigned UnbreakableTailLength, bool InPPDirective,256 encoding::Encoding Encoding, const FormatStyle &Style);257 258 Split getSplit(unsigned LineIndex, unsigned TailOffset, unsigned ColumnLimit,259 unsigned ContentStartColumn,260 const llvm::Regex &CommentPragmasRegex) const override;261 void insertBreak(unsigned LineIndex, unsigned TailOffset, Split Split,262 unsigned ContentIndent,263 WhitespaceManager &Whitespaces) const override;264 void compressWhitespace(unsigned LineIndex, unsigned TailOffset, Split Split,265 WhitespaceManager &Whitespaces) const override {}266 unsigned getLineCount() const override;267 unsigned getRangeLength(unsigned LineIndex, unsigned Offset,268 StringRef::size_type Length,269 unsigned StartColumn) const override;270 unsigned getRemainingLength(unsigned LineIndex, unsigned Offset,271 unsigned StartColumn) const override;272 unsigned getContentStartColumn(unsigned LineIndex, bool Break) const override;273 274protected:275 // The column in which the token starts.276 unsigned StartColumn;277 // The prefix a line needs after a break in the token.278 StringRef Prefix;279 // The postfix a line needs before introducing a break.280 StringRef Postfix;281 // The token text excluding the prefix and postfix.282 StringRef Line;283 // Length of the sequence of tokens after this string literal that cannot284 // contain line breaks.285 unsigned UnbreakableTailLength;286};287 288class BreakableStringLiteralUsingOperators : public BreakableStringLiteral {289public:290 enum QuoteStyleType {291 DoubleQuotes, // The string is quoted with double quotes.292 SingleQuotes, // The JavaScript string is quoted with single quotes.293 AtDoubleQuotes, // The C# verbatim string is quoted with the at sign and294 // double quotes.295 };296 /// Creates a breakable token for a single line string literal for C#, Java,297 /// JavaScript, or Verilog.298 ///299 /// \p StartColumn specifies the column in which the token will start300 /// after formatting.301 BreakableStringLiteralUsingOperators(302 const FormatToken &Tok, QuoteStyleType QuoteStyle, bool UnindentPlus,303 unsigned StartColumn, unsigned UnbreakableTailLength, bool InPPDirective,304 encoding::Encoding Encoding, const FormatStyle &Style);305 unsigned getRemainingLength(unsigned LineIndex, unsigned Offset,306 unsigned StartColumn) const override;307 unsigned getContentStartColumn(unsigned LineIndex, bool Break) const override;308 void insertBreak(unsigned LineIndex, unsigned TailOffset, Split Split,309 unsigned ContentIndent,310 WhitespaceManager &Whitespaces) const override;311 void updateAfterBroken(WhitespaceManager &Whitespaces) const override;312 313protected:314 // Whether braces or parentheses should be inserted around the string to form315 // a concatenation.316 bool BracesNeeded;317 QuoteStyleType QuoteStyle;318 // The braces or parentheses along with the first character which they319 // replace, either a quote or at sign.320 StringRef LeftBraceQuote;321 StringRef RightBraceQuote;322 // Width added to the left due to the added brace or parenthesis. Does not323 // apply to the first line.324 int ContinuationIndent;325};326 327class BreakableComment : public BreakableToken {328protected:329 /// Creates a breakable token for a comment.330 ///331 /// \p StartColumn specifies the column in which the comment will start after332 /// formatting.333 BreakableComment(const FormatToken &Token, unsigned StartColumn,334 bool InPPDirective, encoding::Encoding Encoding,335 const FormatStyle &Style);336 337public:338 bool supportsReflow() const override { return true; }339 unsigned getLineCount() const override;340 Split getSplit(unsigned LineIndex, unsigned TailOffset, unsigned ColumnLimit,341 unsigned ContentStartColumn,342 const llvm::Regex &CommentPragmasRegex) const override;343 void compressWhitespace(unsigned LineIndex, unsigned TailOffset, Split Split,344 WhitespaceManager &Whitespaces) const override;345 346protected:347 // Returns the token containing the line at LineIndex.348 const FormatToken &tokenAt(unsigned LineIndex) const;349 350 // Checks if the content of line LineIndex may be reflown with the previous351 // line.352 virtual bool mayReflow(unsigned LineIndex,353 const llvm::Regex &CommentPragmasRegex) const = 0;354 355 // Contains the original text of the lines of the block comment.356 //357 // In case of a block comments, excludes the leading /* in the first line and358 // trailing */ in the last line. In case of line comments, excludes the359 // leading // and spaces.360 SmallVector<StringRef, 16> Lines;361 362 // Contains the text of the lines excluding all leading and trailing363 // whitespace between the lines. Note that the decoration (if present) is also364 // not considered part of the text.365 SmallVector<StringRef, 16> Content;366 367 // Tokens[i] contains a reference to the token containing Lines[i] if the368 // whitespace range before that token is managed by this block.369 // Otherwise, Tokens[i] is a null pointer.370 SmallVector<FormatToken *, 16> Tokens;371 372 // ContentColumn[i] is the target column at which Content[i] should be.373 // Note that this excludes a leading "* " or "*" in case of block comments374 // where all lines have a "*" prefix, or the leading "// " or "//" in case of375 // line comments.376 //377 // In block comments, the first line's target column is always positive. The378 // remaining lines' target columns are relative to the first line to allow379 // correct indentation of comments in \c WhitespaceManager. Thus they can be380 // negative as well (in case the first line needs to be unindented more than381 // there's actual whitespace in another line).382 SmallVector<int, 16> ContentColumn;383 384 // The intended start column of the first line of text from this section.385 unsigned StartColumn;386 387 const bool AlwaysReflow = Style.ReflowComments == FormatStyle::RCS_Always;388 389 // The prefix to use in front a line that has been reflown up.390 // For example, when reflowing the second line after the first here:391 // // comment 1392 // // comment 2393 // we expect:394 // // comment 1 comment 2395 // and not:396 // // comment 1comment 2397 StringRef ReflowPrefix = " ";398};399 400class BreakableBlockComment : public BreakableComment {401public:402 BreakableBlockComment(const FormatToken &Token, unsigned StartColumn,403 unsigned OriginalStartColumn, bool FirstInLine,404 bool InPPDirective, encoding::Encoding Encoding,405 const FormatStyle &Style, bool UseCRLF);406 407 Split getSplit(unsigned LineIndex, unsigned TailOffset, unsigned ColumnLimit,408 unsigned ContentStartColumn,409 const llvm::Regex &CommentPragmasRegex) const override;410 unsigned getRangeLength(unsigned LineIndex, unsigned Offset,411 StringRef::size_type Length,412 unsigned StartColumn) const override;413 unsigned getRemainingLength(unsigned LineIndex, unsigned Offset,414 unsigned StartColumn) const override;415 unsigned getContentStartColumn(unsigned LineIndex, bool Break) const override;416 unsigned getContentIndent(unsigned LineIndex) const override;417 void insertBreak(unsigned LineIndex, unsigned TailOffset, Split Split,418 unsigned ContentIndent,419 WhitespaceManager &Whitespaces) const override;420 Split getReflowSplit(unsigned LineIndex,421 const llvm::Regex &CommentPragmasRegex) const override;422 void reflow(unsigned LineIndex,423 WhitespaceManager &Whitespaces) const override;424 bool introducesBreakBeforeToken() const override;425 void adaptStartOfLine(unsigned LineIndex,426 WhitespaceManager &Whitespaces) const override;427 Split getSplitAfterLastLine(unsigned TailOffset) const override;428 429 bool mayReflow(unsigned LineIndex,430 const llvm::Regex &CommentPragmasRegex) const override;431 432 // Contains Javadoc annotations that require additional indent when continued433 // on multiple lines.434 static const llvm::StringSet<> ContentIndentingJavadocAnnotations;435 436private:437 // Rearranges the whitespace between Lines[LineIndex-1] and Lines[LineIndex].438 //439 // Updates Content[LineIndex-1] and Content[LineIndex] by stripping off440 // leading and trailing whitespace.441 //442 // Sets ContentColumn to the intended column in which the text at443 // Lines[LineIndex] starts (note that the decoration, if present, is not444 // considered part of the text).445 void adjustWhitespace(unsigned LineIndex, int IndentDelta);446 447 // The column at which the text of a broken line should start.448 // Note that an optional decoration would go before that column.449 // IndentAtLineBreak is a uniform position for all lines in a block comment,450 // regardless of their relative position.451 // FIXME: Revisit the decision to do this; the main reason was to support452 // patterns like453 // /**************//**454 // * Comment455 // We could also support such patterns by special casing the first line456 // instead.457 unsigned IndentAtLineBreak;458 459 // This is to distinguish between the case when the last line was empty and460 // the case when it started with a decoration ("*" or "* ").461 bool LastLineNeedsDecoration;462 463 // Either "* " if all lines begin with a "*", or empty.464 StringRef Decoration;465 466 // If this block comment has decorations, this is the column of the start of467 // the decorations.468 unsigned DecorationColumn;469 470 // If true, make sure that the opening '/**' and the closing '*/' ends on a471 // line of itself. Styles like jsdoc require this for multiline comments.472 bool DelimitersOnNewline;473 474 // Length of the sequence of tokens after this string literal that cannot475 // contain line breaks.476 unsigned UnbreakableTailLength;477};478 479class BreakableLineCommentSection : public BreakableComment {480public:481 BreakableLineCommentSection(const FormatToken &Token, unsigned StartColumn,482 bool InPPDirective, encoding::Encoding Encoding,483 const FormatStyle &Style);484 485 unsigned getRangeLength(unsigned LineIndex, unsigned Offset,486 StringRef::size_type Length,487 unsigned StartColumn) const override;488 unsigned getContentStartColumn(unsigned LineIndex, bool Break) const override;489 void insertBreak(unsigned LineIndex, unsigned TailOffset, Split Split,490 unsigned ContentIndent,491 WhitespaceManager &Whitespaces) const override;492 Split getReflowSplit(unsigned LineIndex,493 const llvm::Regex &CommentPragmasRegex) const override;494 void reflow(unsigned LineIndex,495 WhitespaceManager &Whitespaces) const override;496 void adaptStartOfLine(unsigned LineIndex,497 WhitespaceManager &Whitespaces) const override;498 void updateNextToken(LineState &State) const override;499 bool mayReflow(unsigned LineIndex,500 const llvm::Regex &CommentPragmasRegex) const override;501 502private:503 // OriginalPrefix[i] contains the original prefix of line i, including504 // trailing whitespace before the start of the content. The indentation505 // preceding the prefix is not included.506 // For example, if the line is:507 // // content508 // then the original prefix is "// ".509 SmallVector<StringRef, 16> OriginalPrefix;510 511 /// Prefix[i] + SpacesToAdd[i] contains the intended leading "//" with512 /// trailing spaces to account for the indentation of content within the513 /// comment at line i after formatting. It can be different than the original514 /// prefix.515 /// When the original line starts like this:516 /// //content517 /// Then the OriginalPrefix[i] is "//", but the Prefix[i] is "// " in the LLVM518 /// style.519 /// When the line starts like:520 /// // content521 /// And we want to remove the spaces the OriginalPrefix[i] is "// " and522 /// Prefix[i] is "//".523 SmallVector<std::string, 16> Prefix;524 525 /// How many spaces are added or removed from the OriginalPrefix to form526 /// Prefix.527 SmallVector<int, 16> PrefixSpaceChange;528 529 /// The token to which the last line of this breakable token belongs530 /// to; nullptr if that token is the initial token.531 ///532 /// The distinction is because if the token of the last line of this breakable533 /// token is distinct from the initial token, this breakable token owns the534 /// whitespace before the token of the last line, and the whitespace manager535 /// must be able to modify it.536 FormatToken *LastLineTok = nullptr;537};538} // namespace format539} // namespace clang540 541#endif542