brintos / llvm-project-archived public Read only

1//===- unittest/Format/TokenAnnotatorTest.cpp - Formatting unit tests -----===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8 9#include "clang/Format/Format.h"10 11#include "FormatTestUtils.h"12#include "TestLexer.h"13#include "gtest/gtest.h"14 15namespace clang {16namespace format {17 18// Not really the equality, but everything we need.19static bool operator==(const FormatToken &LHS,20                       const FormatToken &RHS) noexcept {21  return LHS.Tok.getKind() == RHS.Tok.getKind() &&22         LHS.getType() == RHS.getType();23}24 25namespace {26 27class TokenAnnotatorTest : public testing::Test {28protected:29  TokenList annotate(StringRef Code,30                     const FormatStyle &Style = getLLVMStyle()) {31    return TestLexer(Allocator, Buffers, Style).annotate(Code);32  }33  llvm::SpecificBumpPtrAllocator<FormatToken> Allocator;34  std::vector<std::unique_ptr<llvm::MemoryBuffer>> Buffers;35};36 37#define EXPECT_TOKEN_KIND(FormatTok, Kind)                                     \38  EXPECT_EQ((FormatTok)->Tok.getKind(), Kind) << *(FormatTok)39#define EXPECT_TOKEN_TYPE(FormatTok, Type)                                     \40  EXPECT_EQ((FormatTok)->getType(), Type) << *(FormatTok)41#define EXPECT_TOKEN_PRECEDENCE(FormatTok, Prec)                               \42  EXPECT_EQ((FormatTok)->getPrecedence(), Prec) << *(FormatTok)43#define EXPECT_BRACE_KIND(FormatTok, Kind)                                     \44  EXPECT_EQ(FormatTok->getBlockKind(), Kind) << *(FormatTok)45#define EXPECT_SPLIT_PENALTY(FormatTok, Penalty)                               \46  EXPECT_EQ(FormatTok->SplitPenalty, Penalty) << *(FormatTok)47#define EXPECT_TOKEN(FormatTok, Kind, Type)                                    \48  do {                                                                         \49    EXPECT_TOKEN_KIND(FormatTok, Kind);                                        \50    EXPECT_TOKEN_TYPE(FormatTok, Type);                                        \51  } while (false)52 53TEST_F(TokenAnnotatorTest, UnderstandsUsesOfStarAndAmp) {54  auto Tokens = annotate("auto x = [](const decltype(x) &ptr) {};");55  ASSERT_EQ(Tokens.size(), 18u) << Tokens;56  EXPECT_TOKEN(Tokens[7], tok::kw_decltype, TT_Unknown);57  EXPECT_TOKEN(Tokens[8], tok::l_paren, TT_TypeDeclarationParen);58  EXPECT_TOKEN(Tokens[9], tok::identifier, TT_Unknown);59  EXPECT_TOKEN(Tokens[10], tok::r_paren, TT_TypeDeclarationParen);60  EXPECT_TOKEN(Tokens[11], tok::amp, TT_PointerOrReference);61 62  Tokens = annotate("auto x = [](const decltype(x) *ptr) {};");63  ASSERT_EQ(Tokens.size(), 18u) << Tokens;64  EXPECT_TOKEN(Tokens[10], tok::r_paren, TT_TypeDeclarationParen);65  EXPECT_TOKEN(Tokens[11], tok::star, TT_PointerOrReference);66 67  Tokens = annotate("#define lambda [](const decltype(x) &ptr) {}");68  ASSERT_EQ(Tokens.size(), 17u) << Tokens;69  EXPECT_TOKEN(Tokens[7], tok::kw_decltype, TT_Unknown);70  EXPECT_TOKEN(Tokens[8], tok::l_paren, TT_TypeDeclarationParen);71  EXPECT_TOKEN(Tokens[9], tok::identifier, TT_Unknown);72  EXPECT_TOKEN(Tokens[10], tok::r_paren, TT_TypeDeclarationParen);73  EXPECT_TOKEN(Tokens[11], tok::amp, TT_PointerOrReference);74 75  Tokens = annotate("#define lambda [](const decltype(x) *ptr) {}");76  ASSERT_EQ(Tokens.size(), 17u) << Tokens;77  EXPECT_TOKEN(Tokens[10], tok::r_paren, TT_TypeDeclarationParen);78  EXPECT_TOKEN(Tokens[11], tok::star, TT_PointerOrReference);79 80  Tokens = annotate("void f() {\n"81                    "  while (p < a && *p == 'a')\n"82                    "    p++;\n"83                    "}");84  ASSERT_EQ(Tokens.size(), 21u) << Tokens;85  EXPECT_TOKEN(Tokens[10], tok::ampamp, TT_BinaryOperator);86  EXPECT_TOKEN(Tokens[11], tok::star, TT_UnaryOperator);87 88  Tokens = annotate("case *x:");89  ASSERT_EQ(Tokens.size(), 5u) << Tokens;90  EXPECT_TOKEN(Tokens[1], tok::star, TT_UnaryOperator);91  Tokens = annotate("case &x:");92  ASSERT_EQ(Tokens.size(), 5u) << Tokens;93  EXPECT_TOKEN(Tokens[1], tok::amp, TT_UnaryOperator);94 95  Tokens = annotate("bool b = 3 == int{3} && true;");96  ASSERT_EQ(Tokens.size(), 13u) << Tokens;97  EXPECT_TOKEN(Tokens[9], tok::ampamp, TT_BinaryOperator);98 99  Tokens = annotate("struct {\n"100                    "} *ptr;");101  ASSERT_EQ(Tokens.size(), 7u) << Tokens;102  EXPECT_TOKEN(Tokens[3], tok::star, TT_PointerOrReference);103  Tokens = annotate("union {\n"104                    "} *ptr;");105  ASSERT_EQ(Tokens.size(), 7u) << Tokens;106  EXPECT_TOKEN(Tokens[3], tok::star, TT_PointerOrReference);107  Tokens = annotate("class {\n"108                    "} *ptr;");109  ASSERT_EQ(Tokens.size(), 7u) << Tokens;110  EXPECT_TOKEN(Tokens[3], tok::star, TT_PointerOrReference);111 112  Tokens = annotate("struct {\n"113                    "} &&ptr = {};");114  ASSERT_EQ(Tokens.size(), 10u) << Tokens;115  EXPECT_TOKEN(Tokens[3], tok::ampamp, TT_PointerOrReference);116  Tokens = annotate("union {\n"117                    "} &&ptr = {};");118  ASSERT_EQ(Tokens.size(), 10u) << Tokens;119  EXPECT_TOKEN(Tokens[3], tok::ampamp, TT_PointerOrReference);120  Tokens = annotate("class {\n"121                    "} &&ptr = {};");122  ASSERT_EQ(Tokens.size(), 10u) << Tokens;123  EXPECT_TOKEN(Tokens[3], tok::ampamp, TT_PointerOrReference);124  Tokens = annotate("int i = int{42} * 2;");125  ASSERT_EQ(Tokens.size(), 11u) << Tokens;126  EXPECT_TOKEN(Tokens[7], tok::star, TT_BinaryOperator);127 128  Tokens = annotate("delete[] *ptr;");129  ASSERT_EQ(Tokens.size(), 7u) << Tokens;130  EXPECT_TOKEN(Tokens[3], tok::star, TT_UnaryOperator);131  Tokens = annotate("delete[] **ptr;");132  ASSERT_EQ(Tokens.size(), 8u) << Tokens;133  EXPECT_TOKEN(Tokens[3], tok::star, TT_UnaryOperator);134  EXPECT_TOKEN(Tokens[4], tok::star, TT_UnaryOperator);135  Tokens = annotate("delete[] *(ptr);");136  ASSERT_EQ(Tokens.size(), 9u) << Tokens;137  EXPECT_TOKEN(Tokens[3], tok::star, TT_UnaryOperator);138 139  Tokens = annotate("void f() { void (*fnptr)(char* foo); }");140  ASSERT_EQ(Tokens.size(), 18u) << Tokens;141  EXPECT_TOKEN(Tokens[6], tok::l_paren, TT_FunctionTypeLParen);142  // FIXME: The star of a function pointer probably makes more sense as143  // TT_PointerOrReference.144  EXPECT_TOKEN(Tokens[7], tok::star, TT_UnaryOperator);145  EXPECT_TOKEN(Tokens[12], tok::star, TT_PointerOrReference);146 147  Tokens = annotate("void f() { void (*fnptr)(t* foo); }");148  ASSERT_EQ(Tokens.size(), 18u) << Tokens;149  EXPECT_TOKEN(Tokens[6], tok::l_paren, TT_FunctionTypeLParen);150  EXPECT_TOKEN(Tokens[7], tok::star, TT_UnaryOperator);151  EXPECT_TOKEN(Tokens[12], tok::star, TT_PointerOrReference);152 153  Tokens = annotate("int f3() { return sizeof(Foo&); }");154  ASSERT_EQ(Tokens.size(), 14u) << Tokens;155  EXPECT_TOKEN(Tokens[9], tok::amp, TT_PointerOrReference);156 157  Tokens = annotate("int f4() { return sizeof(Foo&&); }");158  ASSERT_EQ(Tokens.size(), 14u) << Tokens;159  EXPECT_TOKEN(Tokens[9], tok::ampamp, TT_PointerOrReference);160 161  Tokens = annotate("void f5() { int f6(Foo&, Bar&); }");162  ASSERT_EQ(Tokens.size(), 17u) << Tokens;163  EXPECT_TOKEN(Tokens[9], tok::amp, TT_PointerOrReference);164  EXPECT_TOKEN(Tokens[12], tok::amp, TT_PointerOrReference);165 166  Tokens = annotate("void f7() { int f8(Foo&&, Bar&&); }");167  ASSERT_EQ(Tokens.size(), 17u) << Tokens;168  EXPECT_TOKEN(Tokens[9], tok::ampamp, TT_PointerOrReference);169  EXPECT_TOKEN(Tokens[12], tok::ampamp, TT_PointerOrReference);170 171  Tokens = annotate("Type1 &val1 = val2;");172  ASSERT_EQ(Tokens.size(), 7u) << Tokens;173  EXPECT_TOKEN(Tokens[1], tok::amp, TT_PointerOrReference);174 175  Tokens = annotate("Type1 *val1 = &val2;");176  ASSERT_EQ(Tokens.size(), 8u) << Tokens;177  EXPECT_TOKEN(Tokens[1], tok::star, TT_PointerOrReference);178  EXPECT_TOKEN(Tokens[4], tok::amp, TT_UnaryOperator);179 180  Tokens = annotate("val1 & val2;");181  ASSERT_EQ(Tokens.size(), 5u) << Tokens;182  EXPECT_TOKEN(Tokens[1], tok::amp, TT_BinaryOperator);183 184  Tokens = annotate("val1 & val2.member;");185  ASSERT_EQ(Tokens.size(), 7u) << Tokens;186  EXPECT_TOKEN(Tokens[1], tok::amp, TT_BinaryOperator);187 188  Tokens = annotate("val1 & val2.*member;");189  ASSERT_EQ(Tokens.size(), 7u) << Tokens;190  EXPECT_TOKEN(Tokens[1], tok::amp, TT_BinaryOperator);191 192  Tokens = annotate("val1.*member & val2;");193  ASSERT_EQ(Tokens.size(), 7u) << Tokens;194  EXPECT_TOKEN(Tokens[3], tok::amp, TT_BinaryOperator);195 196  Tokens = annotate("val1 & val2->*member;");197  ASSERT_EQ(Tokens.size(), 7u) << Tokens;198  EXPECT_TOKEN(Tokens[1], tok::amp, TT_BinaryOperator);199 200  Tokens = annotate("val1->member & val2;");201  ASSERT_EQ(Tokens.size(), 7u) << Tokens;202  EXPECT_TOKEN(Tokens[3], tok::amp, TT_BinaryOperator);203 204  Tokens = annotate("val1 & val2 & val3;");205  ASSERT_EQ(Tokens.size(), 7u) << Tokens;206  EXPECT_TOKEN(Tokens[1], tok::amp, TT_BinaryOperator);207  EXPECT_TOKEN(Tokens[3], tok::amp, TT_BinaryOperator);208 209  Tokens = annotate("val1 & val2 // comment\n"210                    "     & val3;");211  ASSERT_EQ(Tokens.size(), 8u) << Tokens;212  EXPECT_TOKEN(Tokens[1], tok::amp, TT_BinaryOperator);213  EXPECT_TOKEN(Tokens[4], tok::amp, TT_BinaryOperator);214 215  Tokens =216      annotate("val1 & val2.member & val3.member() & val4 & val5->member;");217  ASSERT_EQ(Tokens.size(), 19u) << Tokens;218  EXPECT_TOKEN(Tokens[1], tok::amp, TT_BinaryOperator);219  EXPECT_TOKEN(Tokens[5], tok::amp, TT_BinaryOperator);220  EXPECT_TOKEN(Tokens[11], tok::amp, TT_BinaryOperator);221  EXPECT_TOKEN(Tokens[13], tok::amp, TT_BinaryOperator);222 223  Tokens = annotate("class c {\n"224                    "  void func(type &a) { a & member; }\n"225                    "  anotherType &member;\n"226                    "}");227  ASSERT_EQ(Tokens.size(), 22u) << Tokens;228  EXPECT_TOKEN(Tokens[7], tok::amp, TT_PointerOrReference);229  EXPECT_TOKEN(Tokens[12], tok::amp, TT_BinaryOperator);230  EXPECT_TOKEN(Tokens[17], tok::amp, TT_PointerOrReference);231 232  Tokens = annotate("struct S {\n"233                    "  auto Mem = C & D;\n"234                    "}");235  ASSERT_EQ(Tokens.size(), 12u) << Tokens;236  EXPECT_TOKEN(Tokens[7], tok::amp, TT_BinaryOperator);237 238  Tokens =239      annotate("template <typename T> void swap() noexcept(Bar<T> && Foo<T>);");240  ASSERT_EQ(Tokens.size(), 23u) << Tokens;241  EXPECT_TOKEN(Tokens[15], tok::ampamp, TT_BinaryOperator);242 243  Tokens = annotate("template <typename T> struct S {\n"244                    "  explicit(Bar<T> && Foo<T>) S(const S &);\n"245                    "};");246  ASSERT_EQ(Tokens.size(), 30u) << Tokens;247  EXPECT_TOKEN(Tokens[14], tok::ampamp, TT_BinaryOperator);248 249  Tokens = annotate("template <bool B = C && D> struct S {};");250  ASSERT_EQ(Tokens.size(), 15u) << Tokens;251  EXPECT_TOKEN(Tokens[6], tok::ampamp, TT_BinaryOperator);252 253  Tokens = annotate("template <typename T, bool B = C && D> struct S {};");254  ASSERT_EQ(Tokens.size(), 18u) << Tokens;255  EXPECT_TOKEN(Tokens[9], tok::ampamp, TT_BinaryOperator);256 257  Tokens = annotate("template <typename T, typename U = T&&> struct S {};");258  ASSERT_EQ(Tokens.size(), 17u) << Tokens;259  EXPECT_TOKEN(Tokens[9], tok::ampamp, TT_PointerOrReference);260 261  Tokens = annotate("template <typename T = int (*)(int)> struct S {};");262  ASSERT_EQ(Tokens.size(), 19u) << Tokens;263  EXPECT_TOKEN(Tokens[6], tok::l_paren, TT_FunctionTypeLParen);264  EXPECT_TOKEN(Tokens[7], tok::star, TT_PointerOrReference);265 266  Tokens = annotate("Foo<A && B> a = {};");267  ASSERT_EQ(Tokens.size(), 12u) << Tokens;268  EXPECT_TOKEN(Tokens[3], tok::ampamp, TT_BinaryOperator);269 270  Tokens = annotate("Foo<A &&> a = {};");271  ASSERT_EQ(Tokens.size(), 11u) << Tokens;272  EXPECT_TOKEN(Tokens[3], tok::ampamp, TT_PointerOrReference);273 274  Tokens = annotate("template <typename T>\n"275                    "enable_if_t<is_integral_v<T>, bool> // comment\n"276                    "operator~(T &a);");277  ASSERT_EQ(Tokens.size(), 24u) << Tokens;278  EXPECT_TOKEN(Tokens[19], tok::amp, TT_PointerOrReference);279 280  Tokens = annotate("template <enable_if_t<foo && !bar>* = nullptr> void f();");281  ASSERT_EQ(Tokens.size(), 19u) << Tokens;282  EXPECT_TOKEN(Tokens[5], tok::ampamp, TT_BinaryOperator);283 284  Tokens =285      annotate("auto foo() noexcept(noexcept(bar()) && "286               "trait<std::decay_t<decltype(bar())>> && noexcept(baz())) {}");287  ASSERT_EQ(Tokens.size(), 38u) << Tokens;288  EXPECT_TOKEN(Tokens[12], tok::ampamp, TT_BinaryOperator);289  EXPECT_TOKEN(Tokens[27], tok::ampamp, TT_BinaryOperator);290 291  Tokens = annotate("foo = *i < *j && *j > *k;");292  ASSERT_EQ(Tokens.size(), 15u) << Tokens;293  EXPECT_TOKEN(Tokens[4], tok::less, TT_BinaryOperator);294  EXPECT_TOKEN(Tokens[7], tok::ampamp, TT_BinaryOperator);295  EXPECT_TOKEN(Tokens[10], tok::greater, TT_BinaryOperator);296 297  Tokens = annotate("if (Foo *foo; bar)");298  ASSERT_EQ(Tokens.size(), 9u) << Tokens;299  EXPECT_TOKEN(Tokens[3], tok::star, TT_PointerOrReference);300 301  Tokens = annotate("if (Foo **foo(); bar)");302  ASSERT_EQ(Tokens.size(), 12u) << Tokens;303  EXPECT_TOKEN(Tokens[3], tok::star, TT_PointerOrReference);304  EXPECT_TOKEN(Tokens[4], tok::star, TT_PointerOrReference);305 306  Tokens = annotate("if (Foo *&foo{a}; bar)");307  ASSERT_EQ(Tokens.size(), 13u) << Tokens;308  EXPECT_TOKEN(Tokens[3], tok::star, TT_PointerOrReference);309  EXPECT_TOKEN(Tokens[4], tok::amp, TT_PointerOrReference);310 311  FormatStyle Style = getLLVMStyle();312  Style.TypeNames.push_back("MYI");313  Tokens = annotate("if (MYI *p{nullptr})", Style);314  ASSERT_EQ(Tokens.size(), 10u) << Tokens;315  EXPECT_TOKEN(Tokens[2], tok::identifier, TT_TypeName);316  EXPECT_TOKEN(Tokens[3], tok::star, TT_PointerOrReference);317 318  Style.TypeNames.push_back("Class");319  Tokens = annotate("if (Class *obj {getObj()})", Style);320  ASSERT_EQ(Tokens.size(), 12u) << Tokens;321  EXPECT_TOKEN(Tokens[2], tok::identifier, TT_TypeName);322  EXPECT_TOKEN(Tokens[3], tok::star, TT_PointerOrReference);323 324  Tokens = annotate("class Foo {\n"325                    "  void operator<() {}\n"326                    "  Foo &f;\n"327                    "};");328  ASSERT_EQ(Tokens.size(), 17u) << Tokens;329  EXPECT_TOKEN(Tokens[4], tok::kw_operator, TT_FunctionDeclarationName);330  EXPECT_TOKEN(Tokens[5], tok::less, TT_OverloadedOperator);331  EXPECT_TOKEN(Tokens[6], tok::l_paren, TT_OverloadedOperatorLParen);332  EXPECT_TOKEN(Tokens[8], tok::l_brace, TT_FunctionLBrace);333  EXPECT_TOKEN(Tokens[11], tok::amp, TT_PointerOrReference);334 335  Tokens = annotate("if (new && num) {\n"336                    "  new = 1;\n"337                    "}\n"338                    "if (!delete && num) {\n"339                    "  delete = 1;\n"340                    "}");341  ASSERT_EQ(Tokens.size(), 26u) << Tokens;342  EXPECT_TOKEN(Tokens[3], tok::ampamp, TT_BinaryOperator);343  EXPECT_TOKEN(Tokens[16], tok::ampamp, TT_BinaryOperator);344 345  Tokens = annotate("#define FOO \\\n"346                    "  void foo() { f(a * b); }");347  ASSERT_EQ(Tokens.size(), 17u) << Tokens;348  EXPECT_TOKEN(Tokens[11], tok::star, TT_BinaryOperator);349 350  Tokens = annotate("for (int i; Foo *&foo : foos)");351  ASSERT_EQ(Tokens.size(), 13u) << Tokens;352  EXPECT_TOKEN(Tokens[6], tok::star, TT_PointerOrReference);353  EXPECT_TOKEN(Tokens[7], tok::amp, TT_PointerOrReference);354  EXPECT_TOKEN(Tokens[9], tok::colon, TT_RangeBasedForLoopColon);355 356  Tokens = annotate("#define FOO auto Foo = [] { f(a * b); };");357  ASSERT_EQ(Tokens.size(), 19u) << Tokens;358  EXPECT_TOKEN(Tokens[12], tok::star, TT_BinaryOperator);359 360  Tokens = annotate("namespace {\n"361                    "#define FOO(x) void foo(a##x *b);\n"362                    "}");363  ASSERT_EQ(Tokens.size(), 20u) << Tokens;364  EXPECT_TOKEN(Tokens[14], tok::star, TT_PointerOrReference);365 366  Tokens = annotate("#define foo(x) _Generic(x, bar *: 1, default: 0)");367  ASSERT_EQ(Tokens.size(), 20u) << Tokens;368  EXPECT_TOKEN(Tokens[11], tok::star, TT_PointerOrReference);369 370  Tokens = annotate("Thingy kConfig = {\n"371                    "    1,\n"372                    "    (uint16_t)(kScale * height_pixels),\n"373                    "};");374  ASSERT_EQ(Tokens.size(), 18u) << Tokens;375  EXPECT_TOKEN(Tokens[8], tok::r_paren, TT_CastRParen);376  EXPECT_TOKEN(Tokens[11], tok::star, TT_BinaryOperator);377 378  Tokens = annotate("template <typename T>\n"379                    "concept C = requires(T a, T b) { a && b; };");380  ASSERT_EQ(Tokens.size(), 24u) << Tokens;381  EXPECT_TOKEN(Tokens[16], tok::l_brace, TT_RequiresExpressionLBrace);382  EXPECT_TOKEN(Tokens[18], tok::ampamp, TT_BinaryOperator);383 384  Tokens = annotate("template <typename T, typename V>\n"385                    "concept CheckMultiplicableBy = requires(T a, V b) {\n"386                    "  { a * b } -> std::same_as<T>;\n"387                    "};");388  ASSERT_EQ(Tokens.size(), 36u) << Tokens;389  EXPECT_TOKEN(Tokens[19], tok::l_brace, TT_RequiresExpressionLBrace);390  EXPECT_TOKEN(Tokens[20], tok::l_brace, TT_CompoundRequirementLBrace);391  EXPECT_TOKEN(Tokens[22], tok::star, TT_BinaryOperator);392 393  Tokens = annotate("bool foo = requires { static_cast<Foo &&>(1); };");394  ASSERT_EQ(Tokens.size(), 17u) << Tokens;395  EXPECT_TOKEN(Tokens[8], tok::ampamp, TT_PointerOrReference);396 397  Tokens = annotate("return s.operator int *();");398  ASSERT_EQ(Tokens.size(), 10u) << Tokens;399  // Not TT_FunctionDeclarationName.400  EXPECT_TOKEN(Tokens[3], tok::kw_operator, TT_Unknown);401  EXPECT_TOKEN(Tokens[5], tok::star, TT_PointerOrReference);402  // Not TT_FunctionDeclarationLParen.403  EXPECT_TOKEN(Tokens[6], tok::l_paren, TT_Unknown);404 405  Tokens = annotate("B &b = x.operator B &();");406  ASSERT_EQ(Tokens.size(), 13u) << Tokens;407  EXPECT_TOKEN(Tokens[8], tok::amp, TT_PointerOrReference);408 409  Tokens = annotate("int8_t *a = MacroCall(int8_t, width * height * length);");410  ASSERT_EQ(Tokens.size(), 16u) << Tokens;411  EXPECT_TOKEN(Tokens[9], tok::star, TT_BinaryOperator);412  EXPECT_TOKEN(Tokens[11], tok::star, TT_BinaryOperator);413 414  Tokens = annotate("float foo[3] = {M * H * H, H * M * H, H * H * M};");415  ASSERT_EQ(Tokens.size(), 27u) << Tokens;416  EXPECT_TOKEN(Tokens[16], tok::star, TT_BinaryOperator);417  EXPECT_TOKEN(Tokens[22], tok::star, TT_BinaryOperator);418 419  Tokens = annotate("NSError *__autoreleasing *foo;",420                    getLLVMStyle(FormatStyle::LK_ObjC));421  ASSERT_EQ(Tokens.size(), 7u) << Tokens;422  EXPECT_TOKEN(Tokens[1], tok::star, TT_PointerOrReference);423  EXPECT_TOKEN(Tokens[3], tok::star, TT_PointerOrReference);424}425 426TEST_F(TokenAnnotatorTest, UnderstandsUsesOfPlusAndMinus) {427  auto Tokens = annotate("x - 0");428  ASSERT_EQ(Tokens.size(), 4u) << Tokens;429  EXPECT_TOKEN(Tokens[1], tok::minus, TT_BinaryOperator);430  Tokens = annotate("0 + 0");431  ASSERT_EQ(Tokens.size(), 4u) << Tokens;432  EXPECT_TOKEN(Tokens[1], tok::plus, TT_BinaryOperator);433  Tokens = annotate("x + +0");434  ASSERT_EQ(Tokens.size(), 5u) << Tokens;435  EXPECT_TOKEN(Tokens[2], tok::plus, TT_UnaryOperator);436  Tokens = annotate("x ? -0 : +0");437  ASSERT_EQ(Tokens.size(), 8u) << Tokens;438  EXPECT_TOKEN(Tokens[2], tok::minus, TT_UnaryOperator);439  EXPECT_TOKEN(Tokens[5], tok::plus, TT_UnaryOperator);440  Tokens = annotate("(-0)");441  ASSERT_EQ(Tokens.size(), 5u) << Tokens;442  EXPECT_TOKEN(Tokens[1], tok::minus, TT_UnaryOperator);443  Tokens = annotate("0, -0");444  ASSERT_EQ(Tokens.size(), 5u) << Tokens;445  EXPECT_TOKEN(Tokens[2], tok::minus, TT_UnaryOperator);446  Tokens = annotate("for (; -1;) {\n}");447  ASSERT_EQ(Tokens.size(), 10u) << Tokens;448  EXPECT_TOKEN(Tokens[3], tok::minus, TT_UnaryOperator);449  Tokens = annotate("x = -1;");450  ASSERT_EQ(Tokens.size(), 6u) << Tokens;451  EXPECT_TOKEN(Tokens[2], tok::minus, TT_UnaryOperator);452  Tokens = annotate("x[-1]");453  ASSERT_EQ(Tokens.size(), 6u) << Tokens;454  EXPECT_TOKEN(Tokens[2], tok::minus, TT_UnaryOperator);455  Tokens = annotate("x = {-1};");456  ASSERT_EQ(Tokens.size(), 8u) << Tokens;457  EXPECT_TOKEN(Tokens[3], tok::minus, TT_UnaryOperator);458  Tokens = annotate("case -x:");459  ASSERT_EQ(Tokens.size(), 5u) << Tokens;460  EXPECT_TOKEN(Tokens[1], tok::minus, TT_UnaryOperator);461  Tokens = annotate("co_await -x;");462  ASSERT_EQ(Tokens.size(), 5u) << Tokens;463  EXPECT_TOKEN(Tokens[1], tok::minus, TT_UnaryOperator);464  Tokens = annotate("co_return -x;");465  ASSERT_EQ(Tokens.size(), 5u) << Tokens;466  EXPECT_TOKEN(Tokens[1], tok::minus, TT_UnaryOperator);467  Tokens = annotate("co_yield -x;");468  ASSERT_EQ(Tokens.size(), 5u) << Tokens;469  EXPECT_TOKEN(Tokens[1], tok::minus, TT_UnaryOperator);470  Tokens = annotate("delete -x;");471  ASSERT_EQ(Tokens.size(), 5u) << Tokens;472  EXPECT_TOKEN(Tokens[1], tok::minus, TT_UnaryOperator);473  Tokens = annotate("return -x;");474  ASSERT_EQ(Tokens.size(), 5u) << Tokens;475  EXPECT_TOKEN(Tokens[1], tok::minus, TT_UnaryOperator);476  Tokens = annotate("throw -x;");477  ASSERT_EQ(Tokens.size(), 5u) << Tokens;478  EXPECT_TOKEN(Tokens[1], tok::minus, TT_UnaryOperator);479  Tokens = annotate("sizeof -x");480  ASSERT_EQ(Tokens.size(), 4u) << Tokens;481  EXPECT_TOKEN(Tokens[1], tok::minus, TT_UnaryOperator);482  Tokens = annotate("co_await +x;");483  ASSERT_EQ(Tokens.size(), 5u) << Tokens;484  EXPECT_TOKEN(Tokens[1], tok::plus, TT_UnaryOperator);485  Tokens = annotate("co_return +x;");486  ASSERT_EQ(Tokens.size(), 5u) << Tokens;487  EXPECT_TOKEN(Tokens[1], tok::plus, TT_UnaryOperator);488  Tokens = annotate("co_yield +x;");489  ASSERT_EQ(Tokens.size(), 5u) << Tokens;490  EXPECT_TOKEN(Tokens[1], tok::plus, TT_UnaryOperator);491  Tokens = annotate("delete +x;");492  ASSERT_EQ(Tokens.size(), 5u) << Tokens;493  EXPECT_TOKEN(Tokens[1], tok::plus, TT_UnaryOperator);494  Tokens = annotate("return +x;");495  ASSERT_EQ(Tokens.size(), 5u) << Tokens;496  EXPECT_TOKEN(Tokens[1], tok::plus, TT_UnaryOperator);497  Tokens = annotate("throw +x;");498  ASSERT_EQ(Tokens.size(), 5u) << Tokens;499  EXPECT_TOKEN(Tokens[1], tok::plus, TT_UnaryOperator);500  Tokens = annotate("sizeof +x");501  ASSERT_EQ(Tokens.size(), 4u) << Tokens;502  EXPECT_TOKEN(Tokens[1], tok::plus, TT_UnaryOperator);503  Tokens = annotate("(int)-x");504  ASSERT_EQ(Tokens.size(), 6u) << Tokens;505  EXPECT_TOKEN(Tokens[3], tok::minus, TT_UnaryOperator);506  Tokens = annotate("(-x)");507  ASSERT_EQ(Tokens.size(), 5u) << Tokens;508  EXPECT_TOKEN(Tokens[1], tok::minus, TT_UnaryOperator);509  Tokens = annotate("!+x");510  ASSERT_EQ(Tokens.size(), 4u) << Tokens;511  EXPECT_TOKEN(Tokens[0], tok::exclaim, TT_UnaryOperator);512  EXPECT_TOKEN(Tokens[1], tok::plus, TT_UnaryOperator);513}514 515TEST_F(TokenAnnotatorTest, UnderstandsClasses) {516  auto Tokens = annotate("class C {};");517  ASSERT_EQ(Tokens.size(), 6u) << Tokens;518  EXPECT_TOKEN(Tokens[1], tok::identifier, TT_ClassHeadName);519  EXPECT_TOKEN(Tokens[2], tok::l_brace, TT_ClassLBrace);520  EXPECT_TOKEN(Tokens[3], tok::r_brace, TT_ClassRBrace);521 522  Tokens = annotate("const class C {} c;");523  ASSERT_EQ(Tokens.size(), 8u) << Tokens;524  EXPECT_TOKEN(Tokens[2], tok::identifier, TT_ClassHeadName);525  EXPECT_TOKEN(Tokens[3], tok::l_brace, TT_ClassLBrace);526  EXPECT_TOKEN(Tokens[4], tok::r_brace, TT_ClassRBrace);527 528  Tokens = annotate("const class {} c;");529  ASSERT_EQ(Tokens.size(), 7u) << Tokens;530  EXPECT_TOKEN(Tokens[2], tok::l_brace, TT_ClassLBrace);531  EXPECT_TOKEN(Tokens[3], tok::r_brace, TT_ClassRBrace);532 533  Tokens = annotate("class [[deprecated(\"\")]] C { int i; };");534  ASSERT_EQ(Tokens.size(), 17u) << Tokens;535  EXPECT_TOKEN(Tokens[9], tok::identifier, TT_ClassHeadName);536  EXPECT_TOKEN(Tokens[10], tok::l_brace, TT_ClassLBrace);537  EXPECT_TOKEN(Tokens[14], tok::r_brace, TT_ClassRBrace);538}539 540TEST_F(TokenAnnotatorTest, UnderstandsStructs) {541  auto Tokens = annotate("struct S {};");542  ASSERT_EQ(Tokens.size(), 6u) << Tokens;543  EXPECT_TOKEN(Tokens[1], tok::identifier, TT_ClassHeadName);544  EXPECT_TOKEN(Tokens[2], tok::l_brace, TT_StructLBrace);545  EXPECT_TOKEN(Tokens[3], tok::r_brace, TT_StructRBrace);546 547  Tokens = annotate("struct macro(a) S {};");548  ASSERT_EQ(Tokens.size(), 10u) << Tokens;549  EXPECT_TOKEN(Tokens[5], tok::identifier, TT_ClassHeadName);550  EXPECT_TOKEN(Tokens[6], tok::l_brace, TT_StructLBrace);551  EXPECT_TOKEN(Tokens[7], tok::r_brace, TT_StructRBrace);552 553  Tokens = annotate("struct EXPORT_MACRO [[nodiscard]] C { int i; };");554  ASSERT_EQ(Tokens.size(), 15u) << Tokens;555  EXPECT_TOKEN(Tokens[7], tok::identifier, TT_ClassHeadName);556  EXPECT_TOKEN(Tokens[8], tok::l_brace, TT_StructLBrace);557  EXPECT_TOKEN(Tokens[12], tok::r_brace, TT_StructRBrace);558 559  Tokens = annotate("struct [[deprecated]] [[nodiscard]] C { int i; };");560  ASSERT_EQ(Tokens.size(), 19u) << Tokens;561  EXPECT_TOKEN(Tokens[11], tok::identifier, TT_ClassHeadName);562  EXPECT_TOKEN(Tokens[12], tok::l_brace, TT_StructLBrace);563  EXPECT_TOKEN(Tokens[16], tok::r_brace, TT_StructRBrace);564 565  Tokens = annotate("struct macro(a) S {\n"566                    "  void f(T &t);\n"567                    "};");568  ASSERT_EQ(Tokens.size(), 18u) << Tokens;569  EXPECT_TOKEN(Tokens[5], tok::identifier, TT_ClassHeadName);570  EXPECT_TOKEN(Tokens[6], tok::l_brace, TT_StructLBrace);571  EXPECT_TOKEN(Tokens[11], tok::amp, TT_PointerOrReference);572  EXPECT_TOKEN(Tokens[15], tok::r_brace, TT_StructRBrace);573 574  Tokens = annotate("template <typename T> struct S<const T[N]> {};");575  ASSERT_EQ(Tokens.size(), 18u) << Tokens;576  EXPECT_TOKEN(Tokens[6], tok::identifier, TT_ClassHeadName);577  EXPECT_TOKEN(Tokens[7], tok::less, TT_TemplateOpener);578  EXPECT_TOKEN(Tokens[10], tok::l_square, TT_ArraySubscriptLSquare);579  EXPECT_TOKEN(Tokens[13], tok::greater, TT_TemplateCloser);580  EXPECT_TOKEN(Tokens[14], tok::l_brace, TT_StructLBrace);581  EXPECT_TOKEN(Tokens[15], tok::r_brace, TT_StructRBrace);582 583  Tokens = annotate("template <typename T> struct S<T const[N]> {};");584  ASSERT_EQ(Tokens.size(), 18u) << Tokens;585  EXPECT_TOKEN(Tokens[6], tok::identifier, TT_ClassHeadName);586  EXPECT_TOKEN(Tokens[7], tok::less, TT_TemplateOpener);587  EXPECT_TOKEN(Tokens[10], tok::l_square, TT_ArraySubscriptLSquare);588  EXPECT_TOKEN(Tokens[13], tok::greater, TT_TemplateCloser);589  EXPECT_TOKEN(Tokens[14], tok::l_brace, TT_StructLBrace);590  EXPECT_TOKEN(Tokens[15], tok::r_brace, TT_StructRBrace);591 592  Tokens = annotate("template <typename T, unsigned n> struct S<T const[n]> {\n"593                    "  void f(T const (&a)[n]);\n"594                    "};");595  ASSERT_EQ(Tokens.size(), 35u) << Tokens;596  EXPECT_TOKEN(Tokens[9], tok::identifier, TT_ClassHeadName);597  EXPECT_TOKEN(Tokens[10], tok::less, TT_TemplateOpener);598  EXPECT_TOKEN(Tokens[13], tok::l_square, TT_ArraySubscriptLSquare);599  EXPECT_TOKEN(Tokens[16], tok::greater, TT_TemplateCloser);600  EXPECT_TOKEN(Tokens[17], tok::l_brace, TT_StructLBrace);601  EXPECT_TOKEN(Tokens[23], tok::l_paren, TT_FunctionTypeLParen);602  EXPECT_TOKEN(Tokens[24], tok::amp, TT_UnaryOperator);603  EXPECT_TOKEN(Tokens[27], tok::l_square, TT_ArraySubscriptLSquare);604  EXPECT_TOKEN(Tokens[32], tok::r_brace, TT_StructRBrace);605 606  Tokens = annotate("template <typename T, enum E e> struct S {};");607  ASSERT_EQ(Tokens.size(), 15u) << Tokens;608  EXPECT_TOKEN(Tokens[10], tok::identifier, TT_ClassHeadName);609  EXPECT_TOKEN(Tokens[11], tok::l_brace, TT_StructLBrace);610 611  Tokens = annotate(612      "template <> struct __declspec(foo) Op<Bar *> : OpImpl<Bar *> {};");613  ASSERT_EQ(Tokens.size(), 23u) << Tokens;614  EXPECT_TOKEN(Tokens[5], tok::l_paren, TT_AttributeLParen);615  EXPECT_TOKEN(Tokens[7], tok::r_paren, TT_AttributeRParen);616  EXPECT_TOKEN(Tokens[8], tok::identifier, TT_ClassHeadName);617  EXPECT_TOKEN(Tokens[13], tok::colon, TT_InheritanceColon);618  EXPECT_TOKEN(Tokens[19], tok::l_brace, TT_StructLBrace);619  EXPECT_TOKEN(Tokens[20], tok::r_brace, TT_StructRBrace);620 621  Tokens = annotate("class Outer {\n"622                    "  struct Inner final : Base {};\n"623                    "};");624  ASSERT_EQ(Tokens.size(), 14u) << Tokens;625  EXPECT_TOKEN(Tokens[5], tok::identifier, TT_Unknown); // Not TT_StartOfName626  EXPECT_TOKEN(Tokens[6], tok::colon, TT_InheritanceColon);627 628  constexpr StringRef Code("struct EXPORT StructName {};");629 630  Tokens = annotate(Code);631  ASSERT_EQ(Tokens.size(), 7u) << Tokens;632  EXPECT_TOKEN(Tokens[2], tok::identifier, TT_ClassHeadName);633  EXPECT_TOKEN(Tokens[3], tok::l_brace, TT_StructLBrace);634  EXPECT_TOKEN(Tokens[4], tok::r_brace, TT_StructRBrace);635 636  auto Style = getLLVMStyle();637  Style.AttributeMacros.push_back("EXPORT");638  Tokens = annotate(Code, Style);639  ASSERT_EQ(Tokens.size(), 7u) << Tokens;640  EXPECT_TOKEN(Tokens[1], tok::identifier, TT_AttributeMacro);641  EXPECT_TOKEN(Tokens[2], tok::identifier, TT_ClassHeadName);642  EXPECT_TOKEN(Tokens[3], tok::l_brace, TT_StructLBrace);643  EXPECT_TOKEN(Tokens[4], tok::r_brace, TT_StructRBrace);644}645 646TEST_F(TokenAnnotatorTest, UnderstandsUnions) {647  auto Tokens = annotate("union U {};");648  ASSERT_EQ(Tokens.size(), 6u) << Tokens;649  EXPECT_TOKEN(Tokens[1], tok::identifier, TT_ClassHeadName);650  EXPECT_TOKEN(Tokens[2], tok::l_brace, TT_UnionLBrace);651  EXPECT_TOKEN(Tokens[3], tok::r_brace, TT_UnionRBrace);652 653  Tokens = annotate("union U { void f() { return; } };");654  ASSERT_EQ(Tokens.size(), 14u) << Tokens;655  EXPECT_TOKEN(Tokens[1], tok::identifier, TT_ClassHeadName);656  EXPECT_TOKEN(Tokens[2], tok::l_brace, TT_UnionLBrace);657  EXPECT_TOKEN(Tokens[7], tok::l_brace, TT_FunctionLBrace);658  EXPECT_TOKEN(Tokens[11], tok::r_brace, TT_UnionRBrace);659}660 661TEST_F(TokenAnnotatorTest, UnderstandsEnums) {662  auto Tokens = annotate("enum E {};");663  ASSERT_EQ(Tokens.size(), 6u) << Tokens;664  EXPECT_TOKEN(Tokens[2], tok::l_brace, TT_EnumLBrace);665  EXPECT_TOKEN(Tokens[3], tok::r_brace, TT_EnumRBrace);666}667 668TEST_F(TokenAnnotatorTest, UnderstandsDefaultedAndDeletedFunctions) {669  auto Tokens = annotate("auto operator<=>(const T &) const & = default;");670  ASSERT_EQ(Tokens.size(), 14u) << Tokens;671  EXPECT_TOKEN(Tokens[9], tok::amp, TT_PointerOrReference);672 673  Tokens = annotate("template <typename T> void F(T) && = delete;");674  ASSERT_EQ(Tokens.size(), 15u) << Tokens;675  EXPECT_TOKEN(Tokens[10], tok::ampamp, TT_PointerOrReference);676}677 678TEST_F(TokenAnnotatorTest, UnderstandsVariables) {679  auto Tokens =680      annotate("inline bool var = is_integral_v<int> && is_signed_v<int>;");681  ASSERT_EQ(Tokens.size(), 15u) << Tokens;682  EXPECT_TOKEN(Tokens[8], tok::ampamp, TT_BinaryOperator);683}684 685TEST_F(TokenAnnotatorTest, UnderstandsVariableTemplates) {686  auto Tokens =687      annotate("template <typename T> "688               "inline bool var = is_integral_v<int> && is_signed_v<int>;");689  ASSERT_EQ(Tokens.size(), 20u) << Tokens;690  EXPECT_TOKEN(Tokens[13], tok::ampamp, TT_BinaryOperator);691}692 693TEST_F(TokenAnnotatorTest, UnderstandsTemplatesInMacros) {694  auto Tokens =695      annotate("#define FOO(typeName) \\\n"696               "  { #typeName, foo<FooType>(new foo<realClass>(#typeName)) }");697  ASSERT_EQ(Tokens.size(), 27u) << Tokens;698  EXPECT_TOKEN(Tokens[11], tok::less, TT_TemplateOpener);699  EXPECT_TOKEN(Tokens[13], tok::greater, TT_TemplateCloser);700  EXPECT_TOKEN(Tokens[17], tok::less, TT_TemplateOpener);701  EXPECT_TOKEN(Tokens[19], tok::greater, TT_TemplateCloser);702}703 704TEST_F(TokenAnnotatorTest, UnderstandsGreaterAfterTemplateCloser) {705  auto Tokens = annotate("if (std::tuple_size_v<T> > 0)");706  ASSERT_EQ(Tokens.size(), 12u) << Tokens;707  EXPECT_TOKEN(Tokens[5], tok::less, TT_TemplateOpener);708  EXPECT_TOKEN(Tokens[7], tok::greater, TT_TemplateCloser);709  EXPECT_TOKEN(Tokens[8], tok::greater, TT_BinaryOperator);710}711 712TEST_F(TokenAnnotatorTest, UnderstandsTernaryInTemplate) {713  // IsExpression = false714  auto Tokens = annotate("foo<true ? 1 : 2>();");715  ASSERT_EQ(Tokens.size(), 12u) << Tokens;716  EXPECT_TOKEN(Tokens[1], tok::less, TT_TemplateOpener);717  EXPECT_TOKEN(Tokens[3], tok::question, TT_ConditionalExpr);718  EXPECT_TOKEN(Tokens[5], tok::colon, TT_ConditionalExpr);719  EXPECT_TOKEN(Tokens[7], tok::greater, TT_TemplateCloser);720 721  // IsExpression = true722 723  Tokens = annotate("return foo<true ? 1 : 2>();");724  ASSERT_EQ(Tokens.size(), 13u) << Tokens;725  EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);726  EXPECT_TOKEN(Tokens[4], tok::question, TT_ConditionalExpr);727  EXPECT_TOKEN(Tokens[6], tok::colon, TT_ConditionalExpr);728  EXPECT_TOKEN(Tokens[8], tok::greater, TT_TemplateCloser);729 730  Tokens = annotate("return foo<true ? 1 : 2>{};");731  ASSERT_EQ(Tokens.size(), 13u) << Tokens;732  EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);733  EXPECT_TOKEN(Tokens[4], tok::question, TT_ConditionalExpr);734  EXPECT_TOKEN(Tokens[6], tok::colon, TT_ConditionalExpr);735  EXPECT_TOKEN(Tokens[8], tok::greater, TT_TemplateCloser);736}737 738TEST_F(TokenAnnotatorTest, UnderstandsNonTemplateAngleBrackets) {739  auto Tokens = annotate("return a < b && c > d;");740  ASSERT_EQ(Tokens.size(), 10u) << Tokens;741  EXPECT_TOKEN(Tokens[2], tok::less, TT_BinaryOperator);742  EXPECT_TOKEN(Tokens[6], tok::greater, TT_BinaryOperator);743 744  Tokens = annotate("a < 0 ? b : a > 0 ? c : d;");745  ASSERT_EQ(Tokens.size(), 15u) << Tokens;746  EXPECT_TOKEN(Tokens[1], tok::less, TT_BinaryOperator);747  EXPECT_TOKEN(Tokens[7], tok::greater, TT_BinaryOperator);748 749  Tokens = annotate("return A < B ? true : A > B;");750  ASSERT_EQ(Tokens.size(), 12u) << Tokens;751  EXPECT_TOKEN(Tokens[2], tok::less, TT_BinaryOperator);752  EXPECT_TOKEN(Tokens[8], tok::greater, TT_BinaryOperator);753 754  Tokens = annotate("return A < B ? true : A > B ? false : false;");755  ASSERT_EQ(Tokens.size(), 16u) << Tokens;756  EXPECT_TOKEN(Tokens[2], tok::less, TT_BinaryOperator);757  EXPECT_TOKEN(Tokens[8], tok::greater, TT_BinaryOperator);758 759  Tokens = annotate("return checklower ? a < b : a > b;");760  ASSERT_EQ(Tokens.size(), 12u) << Tokens;761  EXPECT_TOKEN(Tokens[4], tok::less, TT_BinaryOperator);762  EXPECT_TOKEN(Tokens[8], tok::greater, TT_BinaryOperator);763 764  Tokens = annotate("return A < B != A > B;");765  ASSERT_EQ(Tokens.size(), 10u) << Tokens;766  // FIXME:767  // EXPECT_TOKEN(Tokens[2], tok::less, TT_BinaryOperator);768  // EXPECT_TOKEN(Tokens[6], tok::greater, TT_BinaryOperator);769 770  Tokens = annotate("ratio{-1, 2} < ratio{-1, 3} == -1 / 3 > -1 / 2;");771  ASSERT_EQ(Tokens.size(), 27u) << Tokens;772  EXPECT_TOKEN(Tokens[7], tok::less, TT_BinaryOperator);773  EXPECT_TOKEN(Tokens[20], tok::greater, TT_BinaryOperator);774 775  Tokens = annotate("bool foo = a < b && (c * d) > e;");776  ASSERT_EQ(Tokens.size(), 16u) << Tokens;777  EXPECT_TOKEN(Tokens[4], tok::less, TT_BinaryOperator);778  EXPECT_TOKEN(Tokens[6], tok::ampamp, TT_BinaryOperator);779  EXPECT_TOKEN(Tokens[9], tok::star, TT_BinaryOperator);780  EXPECT_TOKEN(Tokens[12], tok::greater, TT_BinaryOperator);781}782 783TEST_F(TokenAnnotatorTest, UnderstandsTemplateTemplateParameters) {784  auto Tokens = annotate("template <template <typename...> typename X,\n"785                         "          template <typename...> class Y,\n"786                         "          typename... T>\n"787                         "class A {};");788  ASSERT_EQ(Tokens.size(), 28u) << Tokens;789  EXPECT_TOKEN(Tokens[1], tok::less, TT_TemplateOpener);790  EXPECT_TOKEN(Tokens[3], tok::less, TT_TemplateOpener);791  EXPECT_TOKEN(Tokens[6], tok::greater, TT_TemplateCloser);792  EXPECT_FALSE(Tokens[6]->ClosesTemplateDeclaration);793  EXPECT_TOKEN(Tokens[11], tok::less, TT_TemplateOpener);794  EXPECT_TOKEN(Tokens[14], tok::greater, TT_TemplateCloser);795  EXPECT_FALSE(Tokens[14]->ClosesTemplateDeclaration);796  EXPECT_TOKEN(Tokens[16], tok::identifier, TT_Unknown);797  EXPECT_TOKEN(Tokens[21], tok::greater, TT_TemplateCloser);798  EXPECT_TRUE(Tokens[21]->ClosesTemplateDeclaration);799  EXPECT_TOKEN(Tokens[23], tok::identifier, TT_ClassHeadName);800}801 802TEST_F(TokenAnnotatorTest, UnderstandsCommonCppTemplates) {803  auto Tokens =804      annotate("static_assert(std::conditional_t<A || B, C, D>::value);");805  ASSERT_EQ(Tokens.size(), 19u) << Tokens;806  EXPECT_TOKEN(Tokens[5], tok::less, TT_TemplateOpener);807  EXPECT_TOKEN(Tokens[13], tok::greater, TT_TemplateCloser);808 809  Tokens =810      annotate("static_assert(std::conditional<A || B, C, D>::type::value);");811  ASSERT_EQ(Tokens.size(), 21u) << Tokens;812  EXPECT_TOKEN(Tokens[5], tok::less, TT_TemplateOpener);813  EXPECT_TOKEN(Tokens[13], tok::greater, TT_TemplateCloser);814 815  Tokens = annotate("static_assert(fancy_v<A || B>);");816  ASSERT_EQ(Tokens.size(), 11u) << Tokens;817  EXPECT_TOKEN(Tokens[3], tok::less, TT_TemplateOpener);818  EXPECT_TOKEN(Tokens[7], tok::greater, TT_TemplateCloser);819 820  Tokens = annotate("static_assert(fancy<A || B>::value);");821  ASSERT_EQ(Tokens.size(), 13u) << Tokens;822  EXPECT_TOKEN(Tokens[3], tok::less, TT_TemplateOpener);823  EXPECT_TOKEN(Tokens[7], tok::greater, TT_TemplateCloser);824}825 826TEST_F(TokenAnnotatorTest, UnderstandsWhitespaceSensitiveMacros) {827  FormatStyle Style = getLLVMStyle();828  Style.WhitespaceSensitiveMacros.push_back("FOO");829 830  auto Tokens = annotate("FOO(1+2 )", Style);831  ASSERT_EQ(Tokens.size(), 7u) << Tokens;832  EXPECT_TOKEN(Tokens[0], tok::identifier, TT_UntouchableMacroFunc);833 834  Tokens = annotate("FOO(a:b:c)", Style);835  ASSERT_EQ(Tokens.size(), 9u) << Tokens;836  EXPECT_TOKEN(Tokens[0], tok::identifier, TT_UntouchableMacroFunc);837}838 839TEST_F(TokenAnnotatorTest, UnderstandsDelete) {840  auto Tokens = annotate("delete (void *)p;");841  ASSERT_EQ(Tokens.size(), 8u) << Tokens;842  EXPECT_TOKEN(Tokens[4], tok::r_paren, TT_CastRParen);843 844  Tokens = annotate("delete[] (void *)p;");845  ASSERT_EQ(Tokens.size(), 10u) << Tokens;846  EXPECT_TOKEN(Tokens[6], tok::r_paren, TT_CastRParen);847 848  Tokens = annotate("delete[] /*comment*/ (void *)p;");849  ASSERT_EQ(Tokens.size(), 11u) << Tokens;850  EXPECT_TOKEN(Tokens[7], tok::r_paren, TT_CastRParen);851 852  Tokens = annotate("delete[/*comment*/] (void *)p;");853  ASSERT_EQ(Tokens.size(), 11u) << Tokens;854  EXPECT_TOKEN(Tokens[7], tok::r_paren, TT_CastRParen);855 856  Tokens = annotate("delete/*comment*/[] (void *)p;");857  ASSERT_EQ(Tokens.size(), 11u) << Tokens;858  EXPECT_TOKEN(Tokens[7], tok::r_paren, TT_CastRParen);859}860 861TEST_F(TokenAnnotatorTest, UnderstandsCasts) {862  auto Tokens = annotate("(void)p;");863  ASSERT_EQ(Tokens.size(), 6u) << Tokens;864  EXPECT_TOKEN(Tokens[2], tok::r_paren, TT_CastRParen);865 866  Tokens = annotate("(uint32_t)&&label;");867  ASSERT_EQ(Tokens.size(), 7u) << Tokens;868  EXPECT_TOKEN(Tokens[2], tok::r_paren, TT_CastRParen);869  EXPECT_TOKEN(Tokens[3], tok::ampamp, TT_UnaryOperator);870  EXPECT_TOKEN(Tokens[4], tok::identifier, TT_Unknown);871 872  Tokens = annotate("auto x = (Foo)p;");873  ASSERT_EQ(Tokens.size(), 9u) << Tokens;874  EXPECT_TOKEN(Tokens[5], tok::r_paren, TT_CastRParen);875 876  Tokens = annotate("(std::vector<int>)p;");877  ASSERT_EQ(Tokens.size(), 11u) << Tokens;878  EXPECT_TOKEN(Tokens[7], tok::r_paren, TT_CastRParen);879 880  Tokens = annotate("return (Foo)p;");881  ASSERT_EQ(Tokens.size(), 7u) << Tokens;882  EXPECT_TOKEN(Tokens[3], tok::r_paren, TT_CastRParen);883 884  Tokens = annotate("throw (Foo)p;");885  ASSERT_EQ(Tokens.size(), 7u) << Tokens;886  EXPECT_TOKEN(Tokens[3], tok::r_paren, TT_CastRParen);887 888  Tokens = annotate("#define FOO(x) (((uint64_t)(x) * BAR) / 100)");889  ASSERT_EQ(Tokens.size(), 21u) << Tokens;890  EXPECT_TOKEN(Tokens[10], tok::r_paren, TT_CastRParen);891  EXPECT_TOKEN(Tokens[13], tok::r_paren, TT_Unknown);892  EXPECT_TOKEN(Tokens[14], tok::star, TT_BinaryOperator);893 894  Tokens = annotate("#define foo(i) ((i) - bar)");895  ASSERT_EQ(Tokens.size(), 14u) << Tokens;896  EXPECT_TOKEN(Tokens[9], tok::r_paren, TT_Unknown);897  EXPECT_TOKEN(Tokens[10], tok::minus, TT_BinaryOperator);898 899  Tokens = annotate("return (Foo) & 10;");900  ASSERT_EQ(Tokens.size(), 8u) << Tokens;901  EXPECT_TOKEN(Tokens[3], tok::r_paren, TT_Unknown);902  EXPECT_TOKEN(Tokens[4], tok::amp, TT_BinaryOperator);903 904  Tokens = annotate("return (struct foo){};");905  ASSERT_EQ(Tokens.size(), 9u) << Tokens;906  EXPECT_TOKEN(Tokens[3], tok::identifier, TT_Unknown);907  EXPECT_TOKEN(Tokens[4], tok::r_paren, TT_CastRParen);908 909  Tokens = annotate("#define FOO(bar) foo((uint64_t)&bar)");910  ASSERT_EQ(Tokens.size(), 15u) << Tokens;911  EXPECT_TOKEN(Tokens[10], tok::r_paren, TT_CastRParen);912  EXPECT_TOKEN(Tokens[11], tok::amp, TT_UnaryOperator);913 914  Tokens = annotate("#define FOO(bar) foo((Foo) & bar)");915  ASSERT_EQ(Tokens.size(), 15u) << Tokens;916  EXPECT_TOKEN(Tokens[10], tok::r_paren, TT_Unknown);917  EXPECT_TOKEN(Tokens[11], tok::amp, TT_BinaryOperator);918 919  Tokens = annotate("func((void (*)())&a);");920  ASSERT_EQ(Tokens.size(), 15u) << Tokens;921  EXPECT_TOKEN(Tokens[4], tok::l_paren, TT_FunctionTypeLParen);922  EXPECT_TOKEN(Tokens[5], tok::star, TT_PointerOrReference);923  EXPECT_TOKEN(Tokens[9], tok::r_paren, TT_CastRParen);924  EXPECT_TOKEN(Tokens[10], tok::amp, TT_UnaryOperator);925 926  Tokens = annotate("int result = ((int)a) - b;");927  ASSERT_EQ(Tokens.size(), 13u) << Tokens;928  EXPECT_TOKEN(Tokens[6], tok::r_paren, TT_CastRParen);929  EXPECT_TOKEN(Tokens[8], tok::r_paren, TT_Unknown);930  EXPECT_TOKEN(Tokens[9], tok::minus, TT_BinaryOperator);931 932  Tokens = annotate("return (double)(foo(30)) - 15;");933  ASSERT_EQ(Tokens.size(), 14u) << Tokens;934  EXPECT_TOKEN(Tokens[3], tok::r_paren, TT_CastRParen);935  EXPECT_TOKEN(Tokens[9], tok::r_paren, TT_Unknown);936  EXPECT_TOKEN(Tokens[10], tok::minus, TT_BinaryOperator);937 938  Tokens = annotate("return (::Type)(1 + 2);");939  ASSERT_EQ(Tokens.size(), 12u) << Tokens;940  EXPECT_TOKEN(Tokens[4], tok::r_paren, TT_CastRParen);941 942  Tokens = annotate("return (Namespace::Class)(1 + 2);");943  ASSERT_EQ(Tokens.size(), 13u) << Tokens;944  EXPECT_TOKEN(Tokens[5], tok::r_paren, TT_CastRParen);945 946  Tokens = annotate("return (Foo (*)(void *, Bar, ...))&foo;");947  ASSERT_EQ(Tokens.size(), 19u) << Tokens;948  EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_FunctionTypeLParen);949  EXPECT_TOKEN(Tokens[14], tok::r_paren, TT_CastRParen);950  EXPECT_TOKEN(Tokens[15], tok::amp, TT_UnaryOperator);951 952  Tokens = annotate("return (Foo (Bar::*)())&Bar::foo;");953  ASSERT_EQ(Tokens.size(), 17u) << Tokens;954  EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_FunctionTypeLParen);955  EXPECT_TOKEN(Tokens[10], tok::r_paren, TT_CastRParen);956  EXPECT_TOKEN(Tokens[11], tok::amp, TT_UnaryOperator);957 958  auto Style = getLLVMStyle();959  Style.TypeNames.push_back("Foo");960  Tokens = annotate("#define FOO(bar) foo((Foo)&bar)", Style);961  ASSERT_EQ(Tokens.size(), 15u) << Tokens;962  EXPECT_TOKEN(Tokens[9], tok::identifier, TT_TypeName);963  EXPECT_TOKEN(Tokens[10], tok::r_paren, TT_CastRParen);964  EXPECT_TOKEN(Tokens[11], tok::amp, TT_UnaryOperator);965}966 967TEST_F(TokenAnnotatorTest, UnderstandsDynamicExceptionSpecifier) {968  auto Tokens = annotate("void f() throw(int);");969  ASSERT_EQ(Tokens.size(), 10u) << Tokens;970  EXPECT_TOKEN(Tokens[4], tok::kw_throw, TT_Unknown);971}972 973TEST_F(TokenAnnotatorTest, UnderstandsFunctionRefQualifiers) {974  auto Tokens = annotate("void f() &;");975  ASSERT_EQ(Tokens.size(), 7u) << Tokens;976  EXPECT_TOKEN(Tokens[4], tok::amp, TT_PointerOrReference);977 978  Tokens = annotate("void operator=(T) &&;");979  ASSERT_EQ(Tokens.size(), 9u) << Tokens;980  EXPECT_TOKEN(Tokens[6], tok::ampamp, TT_PointerOrReference);981 982  Tokens = annotate("template <typename T> void f() &;");983  ASSERT_EQ(Tokens.size(), 12u) << Tokens;984  EXPECT_TOKEN(Tokens[9], tok::amp, TT_PointerOrReference);985 986  Tokens = annotate("template <typename T> void operator=(T) &;");987  ASSERT_EQ(Tokens.size(), 14u) << Tokens;988  EXPECT_TOKEN(Tokens[11], tok::amp, TT_PointerOrReference);989}990 991TEST_F(TokenAnnotatorTest, UnderstandsOverloadedOperators) {992  auto Tokens = annotate("x.operator+()");993  ASSERT_EQ(Tokens.size(), 7u) << Tokens;994  EXPECT_TOKEN(Tokens[2], tok::kw_operator, TT_Unknown);995  EXPECT_TOKEN(Tokens[3], tok::plus, TT_OverloadedOperator);996  EXPECT_TOKEN(Tokens[4], tok::l_paren, TT_OverloadedOperatorLParen);997  Tokens = annotate("x.operator=()");998  ASSERT_EQ(Tokens.size(), 7u) << Tokens;999  EXPECT_TOKEN(Tokens[2], tok::kw_operator, TT_Unknown);1000  EXPECT_TOKEN(Tokens[3], tok::equal, TT_OverloadedOperator);1001  EXPECT_TOKEN(Tokens[4], tok::l_paren, TT_OverloadedOperatorLParen);1002  Tokens = annotate("x.operator+=()");1003  ASSERT_EQ(Tokens.size(), 7u) << Tokens;1004  EXPECT_TOKEN(Tokens[2], tok::kw_operator, TT_Unknown);1005  EXPECT_TOKEN(Tokens[3], tok::plusequal, TT_OverloadedOperator);1006  EXPECT_TOKEN(Tokens[4], tok::l_paren, TT_OverloadedOperatorLParen);1007  Tokens = annotate("x.operator,()");1008  ASSERT_EQ(Tokens.size(), 7u) << Tokens;1009  EXPECT_TOKEN(Tokens[2], tok::kw_operator, TT_Unknown);1010  EXPECT_TOKEN(Tokens[3], tok::comma, TT_OverloadedOperator);1011  EXPECT_TOKEN(Tokens[4], tok::l_paren, TT_OverloadedOperatorLParen);1012  Tokens = annotate("x.operator()()");1013  ASSERT_EQ(Tokens.size(), 8u) << Tokens;1014  EXPECT_TOKEN(Tokens[2], tok::kw_operator, TT_Unknown);1015  EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_OverloadedOperator);1016  EXPECT_TOKEN(Tokens[4], tok::r_paren, TT_OverloadedOperator);1017  EXPECT_TOKEN(Tokens[5], tok::l_paren, TT_OverloadedOperatorLParen);1018  Tokens = annotate("x.operator[]()");1019  ASSERT_EQ(Tokens.size(), 8u) << Tokens;1020  EXPECT_TOKEN(Tokens[2], tok::kw_operator, TT_Unknown);1021  // EXPECT_TOKEN(Tokens[3], tok::l_square, TT_OverloadedOperator);1022  // EXPECT_TOKEN(Tokens[4], tok::r_square, TT_OverloadedOperator);1023  EXPECT_TOKEN(Tokens[5], tok::l_paren, TT_OverloadedOperatorLParen);1024  Tokens = annotate("x.operator\"\"_a()");1025  ASSERT_EQ(Tokens.size(), 7u) << Tokens;1026  EXPECT_TOKEN(Tokens[2], tok::kw_operator, TT_Unknown);1027  EXPECT_TOKEN(Tokens[3], tok::string_literal, TT_OverloadedOperator);1028  EXPECT_TOKEN(Tokens[4], tok::l_paren, TT_OverloadedOperatorLParen);1029  Tokens = annotate("x.operator\"\" _a()");1030  ASSERT_EQ(Tokens.size(), 8u) << Tokens;1031  EXPECT_TOKEN(Tokens[2], tok::kw_operator, TT_Unknown);1032  EXPECT_TOKEN(Tokens[3], tok::string_literal, TT_OverloadedOperator);1033  EXPECT_TOKEN(Tokens[5], tok::l_paren, TT_OverloadedOperatorLParen);1034  Tokens = annotate("x.operator\"\"if()");1035  ASSERT_EQ(Tokens.size(), 7u) << Tokens;1036  EXPECT_TOKEN(Tokens[2], tok::kw_operator, TT_Unknown);1037  EXPECT_TOKEN(Tokens[3], tok::string_literal, TT_OverloadedOperator);1038  EXPECT_TOKEN(Tokens[4], tok::l_paren, TT_OverloadedOperatorLParen);1039  Tokens = annotate("x.operator\"\"s()");1040  ASSERT_EQ(Tokens.size(), 7u) << Tokens;1041  EXPECT_TOKEN(Tokens[2], tok::kw_operator, TT_Unknown);1042  EXPECT_TOKEN(Tokens[3], tok::string_literal, TT_OverloadedOperator);1043  EXPECT_TOKEN(Tokens[4], tok::l_paren, TT_OverloadedOperatorLParen);1044  Tokens = annotate("x.operator\"\" s()");1045  ASSERT_EQ(Tokens.size(), 8u) << Tokens;1046  EXPECT_TOKEN(Tokens[2], tok::kw_operator, TT_Unknown);1047  EXPECT_TOKEN(Tokens[3], tok::string_literal, TT_OverloadedOperator);1048  EXPECT_TOKEN(Tokens[5], tok::l_paren, TT_OverloadedOperatorLParen);1049 1050  Tokens = annotate("int operator+(int);");1051  ASSERT_EQ(Tokens.size(), 8u) << Tokens;1052  EXPECT_TOKEN(Tokens[1], tok::kw_operator, TT_FunctionDeclarationName);1053  EXPECT_TOKEN(Tokens[2], tok::plus, TT_OverloadedOperator);1054  EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_OverloadedOperatorLParen);1055  Tokens = annotate("auto operator=(T&) {}");1056  ASSERT_EQ(Tokens.size(), 10u) << Tokens;1057  EXPECT_TOKEN(Tokens[1], tok::kw_operator, TT_FunctionDeclarationName);1058  EXPECT_TOKEN(Tokens[2], tok::equal, TT_OverloadedOperator);1059  EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_OverloadedOperatorLParen);1060  Tokens = annotate("auto operator()() {}");1061  ASSERT_EQ(Tokens.size(), 9u) << Tokens;1062  EXPECT_TOKEN(Tokens[1], tok::kw_operator, TT_FunctionDeclarationName);1063  EXPECT_TOKEN(Tokens[2], tok::l_paren, TT_OverloadedOperator);1064  EXPECT_TOKEN(Tokens[3], tok::r_paren, TT_OverloadedOperator);1065  EXPECT_TOKEN(Tokens[4], tok::l_paren, TT_OverloadedOperatorLParen);1066 1067  Tokens = annotate("class Foo {\n"1068                    "  int operator+(a* b);\n"1069                    "}");1070  ASSERT_EQ(Tokens.size(), 14u) << Tokens;1071  EXPECT_TOKEN(Tokens[4], tok::kw_operator, TT_FunctionDeclarationName);1072  EXPECT_TOKEN(Tokens[5], tok::plus, TT_OverloadedOperator);1073  EXPECT_TOKEN(Tokens[6], tok::l_paren, TT_OverloadedOperatorLParen);1074  EXPECT_TOKEN(Tokens[8], tok::star, TT_PointerOrReference);1075 1076  Tokens = annotate("class Foo {\n"1077                    "  int c = operator+(a * b);\n"1078                    "}");1079  ASSERT_EQ(Tokens.size(), 16u) << Tokens;1080  EXPECT_TOKEN(Tokens[6], tok::kw_operator, TT_Unknown);1081  EXPECT_TOKEN(Tokens[7], tok::plus, TT_OverloadedOperator);1082  EXPECT_TOKEN(Tokens[8], tok::l_paren, TT_OverloadedOperatorLParen);1083  EXPECT_TOKEN(Tokens[10], tok::star, TT_BinaryOperator);1084 1085  Tokens = annotate("void foo() {\n"1086                    "  operator+(a * b);\n"1087                    "}");1088  ASSERT_EQ(Tokens.size(), 15u) << Tokens;1089  EXPECT_TOKEN(Tokens[5], tok::kw_operator, TT_Unknown);1090  EXPECT_TOKEN(Tokens[6], tok::plus, TT_OverloadedOperator);1091  EXPECT_TOKEN(Tokens[7], tok::l_paren, TT_OverloadedOperatorLParen);1092  EXPECT_TOKEN(Tokens[9], tok::star, TT_BinaryOperator);1093 1094  Tokens = annotate("return operator+(a * b, c & d) + operator+(a && b && c);");1095  ASSERT_EQ(Tokens.size(), 24u) << Tokens;1096  EXPECT_TOKEN(Tokens[1], tok::kw_operator, TT_Unknown);1097  EXPECT_TOKEN(Tokens[2], tok::plus, TT_OverloadedOperator);1098  EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_OverloadedOperatorLParen);1099  EXPECT_TOKEN(Tokens[5], tok::star, TT_BinaryOperator);1100  EXPECT_TOKEN(Tokens[9], tok::amp, TT_BinaryOperator);1101  EXPECT_TOKEN(Tokens[13], tok::kw_operator, TT_Unknown);1102  EXPECT_TOKEN(Tokens[14], tok::plus, TT_OverloadedOperator);1103  EXPECT_TOKEN(Tokens[15], tok::l_paren, TT_OverloadedOperatorLParen);1104  EXPECT_TOKEN(Tokens[17], tok::ampamp, TT_BinaryOperator);1105  EXPECT_TOKEN(Tokens[19], tok::ampamp, TT_BinaryOperator);1106 1107  Tokens = annotate("class Foo {\n"1108                    "  void foo() {\n"1109                    "    operator+(a * b);\n"1110                    "  }\n"1111                    "}");1112  ASSERT_EQ(Tokens.size(), 19u) << Tokens;1113  EXPECT_TOKEN(Tokens[8], tok::kw_operator, TT_Unknown);1114  EXPECT_TOKEN(Tokens[9], tok::plus, TT_OverloadedOperator);1115  EXPECT_TOKEN(Tokens[10], tok::l_paren, TT_OverloadedOperatorLParen);1116  EXPECT_TOKEN(Tokens[12], tok::star, TT_BinaryOperator);1117 1118  Tokens = annotate("std::vector<Foo> operator()(Foo &foo);");1119  ASSERT_EQ(Tokens.size(), 16u) << Tokens;1120  EXPECT_TOKEN(Tokens[3], tok::less, TT_TemplateOpener);1121  EXPECT_TOKEN(Tokens[5], tok::greater, TT_TemplateCloser);1122  EXPECT_TOKEN(Tokens[6], tok::kw_operator, TT_FunctionDeclarationName);1123  EXPECT_TOKEN(Tokens[7], tok::l_paren, TT_OverloadedOperator);1124  EXPECT_TOKEN(Tokens[8], tok::r_paren, TT_OverloadedOperator);1125  EXPECT_TOKEN(Tokens[9], tok::l_paren, TT_OverloadedOperatorLParen);1126  EXPECT_TOKEN(Tokens[11], tok::amp, TT_PointerOrReference);1127 1128  Tokens = annotate("decltype(auto) operator()(T &x);");1129  ASSERT_EQ(Tokens.size(), 14u) << Tokens;1130  EXPECT_TOKEN(Tokens[1], tok::l_paren, TT_TypeDeclarationParen);1131  EXPECT_TOKEN(Tokens[3], tok::r_paren, TT_TypeDeclarationParen);1132  EXPECT_TOKEN(Tokens[4], tok::kw_operator, TT_FunctionDeclarationName);1133  EXPECT_TOKEN(Tokens[5], tok::l_paren, TT_OverloadedOperator);1134  EXPECT_TOKEN(Tokens[6], tok::r_paren, TT_OverloadedOperator);1135  EXPECT_TOKEN(Tokens[7], tok::l_paren, TT_OverloadedOperatorLParen);1136  EXPECT_TOKEN(Tokens[9], tok::amp, TT_PointerOrReference);1137 1138  Tokens = annotate("friend ostream& ::operator<<(ostream& lhs, foo& rhs);");1139  ASSERT_EQ(Tokens.size(), 17u) << Tokens;1140  EXPECT_TOKEN(Tokens[4], tok::kw_operator, TT_FunctionDeclarationName);1141  EXPECT_TOKEN(Tokens[5], tok::lessless, TT_OverloadedOperator);1142  EXPECT_TOKEN(Tokens[6], tok::l_paren, TT_OverloadedOperatorLParen);1143  EXPECT_TOKEN(Tokens[8], tok::amp, TT_PointerOrReference);1144  EXPECT_TOKEN(Tokens[12], tok::amp, TT_PointerOrReference);1145 1146  Tokens = annotate("::foo::bar& ::foo::bar::operator=(::foo::bar& other);");1147  ASSERT_EQ(Tokens.size(), 22u) << Tokens;1148  EXPECT_TOKEN(Tokens[6], tok::identifier, TT_FunctionDeclarationName);1149  EXPECT_TOKEN(Tokens[17], tok::amp, TT_PointerOrReference);1150 1151  Tokens = annotate("SomeLoooooooooooooooooType::Awaitable\n"1152                    "SomeLoooooooooooooooooType::operator co_await();");1153  ASSERT_EQ(Tokens.size(), 11u) << Tokens;1154  EXPECT_TOKEN(Tokens[3], tok::identifier, TT_FunctionDeclarationName);1155  EXPECT_TOKEN(Tokens[7], tok::l_paren, TT_OverloadedOperatorLParen);1156 1157  Tokens = annotate("using std::operator==;");1158  ASSERT_EQ(Tokens.size(), 7u) << Tokens;1159  // Not TT_FunctionDeclarationName.1160  EXPECT_TOKEN(Tokens[3], tok::kw_operator, TT_Unknown);1161 1162  Tokens = annotate("SomeAPI::operator()();");1163  ASSERT_EQ(Tokens.size(), 9u) << Tokens;1164  // Not TT_FunctionDeclarationName.1165  EXPECT_TOKEN(Tokens[2], tok::kw_operator, TT_Unknown);1166}1167 1168TEST_F(TokenAnnotatorTest, OverloadedOperatorInTemplate) {1169  struct {1170    const char *Text;1171    tok::TokenKind Kind;1172  } Operators[] = {{"+", tok::plus},1173                   {"-", tok::minus},1174                   // FIXME:1175                   // {"*", tok::star},1176                   {"/", tok::slash},1177                   {"%", tok::percent},1178                   {"^", tok::caret},1179                   // FIXME:1180                   // {"&", tok::amp},1181                   {"|", tok::pipe},1182                   {"~", tok::tilde},1183                   {"!", tok::exclaim},1184                   {"=", tok::equal},1185                   // FIXME:1186                   // {"<", tok::less},1187                   {">", tok::greater},1188                   {"+=", tok::plusequal},1189                   {"-=", tok::minusequal},1190                   {"*=", tok::starequal},1191                   {"/=", tok::slashequal},1192                   {"%=", tok::percentequal},1193                   {"^=", tok::caretequal},1194                   {"&=", tok::ampequal},1195                   {"|=", tok::pipeequal},1196                   {"<<", tok::lessless},1197                   {">>", tok::greatergreater},1198                   {">>=", tok::greatergreaterequal},1199                   {"<<=", tok::lesslessequal},1200                   {"==", tok::equalequal},1201                   {"!=", tok::exclaimequal},1202                   {"<=", tok::lessequal},1203                   {">=", tok::greaterequal},1204                   {"<=>", tok::spaceship},1205                   {"&&", tok::ampamp},1206                   {"||", tok::pipepipe},1207                   {"++", tok::plusplus},1208                   {"--", tok::minusminus},1209                   {",", tok::comma},1210                   {"->*", tok::arrowstar},1211                   {"->", tok::arrow}};1212 1213  for (const auto &Operator : Operators) {1214    std::string Input("C<&operator");1215    Input += Operator.Text;1216    Input += " > a;";1217    auto Tokens = annotate(std::string(Input));1218    ASSERT_EQ(Tokens.size(), 9u) << Tokens;1219    EXPECT_TOKEN(Tokens[1], tok::less, TT_TemplateOpener);1220    EXPECT_TOKEN(Tokens[4], Operator.Kind, TT_OverloadedOperator);1221    EXPECT_TOKEN(Tokens[5], tok::greater, TT_TemplateCloser);1222  }1223 1224  auto Tokens = annotate("C<&operator< <X>> lt;");1225  ASSERT_EQ(Tokens.size(), 12u) << Tokens;1226  EXPECT_TOKEN(Tokens[1], tok::less, TT_TemplateOpener);1227  EXPECT_TOKEN(Tokens[4], tok::less, TT_OverloadedOperator);1228  EXPECT_TOKEN(Tokens[5], tok::less, TT_TemplateOpener);1229  EXPECT_TOKEN(Tokens[7], tok::greater, TT_TemplateCloser);1230  EXPECT_TOKEN(Tokens[8], tok::greater, TT_TemplateCloser);1231}1232 1233TEST_F(TokenAnnotatorTest, UnderstandsRequiresClausesAndConcepts) {1234  auto Tokens = annotate("template <typename T>\n"1235                         "concept C = (Foo && Bar) && (Bar && Baz);");1236 1237  ASSERT_EQ(Tokens.size(), 21u) << Tokens;1238  EXPECT_TOKEN(Tokens[10], tok::ampamp, TT_BinaryOperator);1239  EXPECT_TOKEN(Tokens[13], tok::ampamp, TT_BinaryOperator);1240  EXPECT_TOKEN(Tokens[16], tok::ampamp, TT_BinaryOperator);1241 1242  Tokens = annotate("template <typename T>\n"1243                    "concept C = Foo && !Bar;");1244 1245  ASSERT_EQ(Tokens.size(), 14u) << Tokens;1246  EXPECT_TOKEN(Tokens[9], tok::ampamp, TT_BinaryOperator);1247  EXPECT_TOKEN(Tokens[10], tok::exclaim, TT_UnaryOperator);1248 1249  Tokens = annotate("template <typename T>\n"1250                    "concept C = requires(T t) {\n"1251                    "  { t.foo() };\n"1252                    "} && Bar<T> && Baz<T>;");1253  ASSERT_EQ(Tokens.size(), 35u) << Tokens;1254  EXPECT_TOKEN(Tokens[8], tok::kw_requires, TT_RequiresExpression);1255  EXPECT_TOKEN(Tokens[9], tok::l_paren, TT_RequiresExpressionLParen);1256  EXPECT_TOKEN(Tokens[13], tok::l_brace, TT_RequiresExpressionLBrace);1257  EXPECT_TOKEN(Tokens[23], tok::ampamp, TT_BinaryOperator);1258  EXPECT_TOKEN(Tokens[28], tok::ampamp, TT_BinaryOperator);1259 1260  Tokens = annotate("template<typename T>\n"1261                    "requires C1<T> && (C21<T> || C22<T> && C2e<T>) && C3<T>\n"1262                    "struct Foo;");1263  ASSERT_EQ(Tokens.size(), 36u) << Tokens;1264  EXPECT_TOKEN(Tokens[5], tok::kw_requires, TT_RequiresClause);1265  EXPECT_TOKEN(Tokens[6], tok::identifier, TT_Unknown);1266  EXPECT_EQ(Tokens[6]->FakeLParens.size(), 1u);1267  EXPECT_TOKEN(Tokens[10], tok::ampamp, TT_BinaryOperator);1268  EXPECT_TOKEN(Tokens[16], tok::pipepipe, TT_BinaryOperator);1269  EXPECT_TOKEN(Tokens[21], tok::ampamp, TT_BinaryOperator);1270  EXPECT_TOKEN(Tokens[27], tok::ampamp, TT_BinaryOperator);1271  // Not TT_TrailingAnnotation.1272  EXPECT_TOKEN(Tokens[28], tok::identifier, TT_Unknown);1273  EXPECT_TOKEN(Tokens[31], tok::greater, TT_TemplateCloser);1274  EXPECT_EQ(Tokens[31]->FakeRParens, 1u);1275  EXPECT_TRUE(Tokens[31]->ClosesRequiresClause);1276  // Not TT_TrailingAnnotation.1277  EXPECT_TOKEN(Tokens[33], tok::identifier, TT_Unknown);1278 1279  Tokens =1280      annotate("template <typename T>\n"1281               "  requires(C1<T> && (C21<T> || C22<T> && C2e<T>) && C3<T>)\n"1282               "struct Foo;");1283  ASSERT_EQ(Tokens.size(), 38u) << Tokens;1284  EXPECT_TOKEN(Tokens[5], tok::kw_requires, TT_RequiresClause);1285  EXPECT_TOKEN(Tokens[7], tok::identifier, TT_Unknown);1286  EXPECT_EQ(Tokens[7]->FakeLParens.size(), 1u);1287  EXPECT_TOKEN(Tokens[11], tok::ampamp, TT_BinaryOperator);1288  EXPECT_TOKEN(Tokens[17], tok::pipepipe, TT_BinaryOperator);1289  EXPECT_TOKEN(Tokens[22], tok::ampamp, TT_BinaryOperator);1290  EXPECT_TOKEN(Tokens[28], tok::ampamp, TT_BinaryOperator);1291  EXPECT_TOKEN(Tokens[32], tok::greater, TT_TemplateCloser);1292  EXPECT_EQ(Tokens[32]->FakeRParens, 1u);1293  EXPECT_TOKEN(Tokens[33], tok::r_paren, TT_Unknown);1294  EXPECT_TRUE(Tokens[33]->ClosesRequiresClause);1295  EXPECT_TOKEN(Tokens[35], tok::identifier, TT_Unknown);1296 1297  Tokens = annotate("template <typename T>\n"1298                    "void foo(T) noexcept requires Bar<T>;");1299  ASSERT_EQ(Tokens.size(), 18u) << Tokens;1300  EXPECT_TOKEN(Tokens[11], tok::kw_requires, TT_RequiresClause);1301 1302  Tokens = annotate("template <typename T>\n"1303                    "requires Bar<T> || Baz<T>\n"1304                    "auto foo(T) -> int;");1305  ASSERT_EQ(Tokens.size(), 24u) << Tokens;1306  EXPECT_TOKEN(Tokens[5], tok::kw_requires, TT_RequiresClause);1307  EXPECT_EQ(Tokens[11]->FakeLParens.size(), 0u);1308  EXPECT_TRUE(Tokens[14]->ClosesRequiresClause);1309  EXPECT_TOKEN(Tokens[20], tok::arrow, TT_TrailingReturnArrow);1310 1311  Tokens = annotate("template <typename T>\n"1312                    "requires Bar<T>\n"1313                    "bool foo(T) { return false; }");1314  ASSERT_EQ(Tokens.size(), 21u) << Tokens;1315  EXPECT_TOKEN(Tokens[5], tok::kw_requires, TT_RequiresClause);1316  EXPECT_TRUE(Tokens[9]->ClosesRequiresClause);1317  EXPECT_TOKEN(Tokens[11], tok::identifier, TT_FunctionDeclarationName);1318  EXPECT_TOKEN(Tokens[12], tok::l_paren, TT_FunctionDeclarationLParen);1319 1320  Tokens = annotate("template <typename T>\n"1321                    "requires Bar<T>\n"1322                    "decltype(auto) foo(T) { return false; }");1323  ASSERT_EQ(Tokens.size(), 24u) << Tokens;1324  EXPECT_TOKEN(Tokens[5], tok::kw_requires, TT_RequiresClause);1325  EXPECT_TRUE(Tokens[9]->ClosesRequiresClause);1326  EXPECT_TOKEN(Tokens[14], tok::identifier, TT_FunctionDeclarationName);1327  EXPECT_TOKEN(Tokens[15], tok::l_paren, TT_FunctionDeclarationLParen);1328 1329  Tokens = annotate("template <typename T>\n"1330                    "struct S {\n"1331                    "  void foo() const requires Bar<T>;\n"1332                    "  void bar() const & requires Baz<T>;\n"1333                    "  void bar() && requires Baz2<T>;\n"1334                    "  void baz() const & noexcept requires Baz<T>;\n"1335                    "  void baz() && noexcept requires Baz2<T>;\n"1336                    "};\n"1337                    "\n"1338                    "void S::bar() const & requires Baz<T> { }");1339  ASSERT_EQ(Tokens.size(), 85u) << Tokens;1340  EXPECT_TOKEN(Tokens[13], tok::kw_requires, TT_RequiresClause);1341  EXPECT_TOKEN(Tokens[24], tok::amp, TT_PointerOrReference);1342  EXPECT_TOKEN(Tokens[25], tok::kw_requires, TT_RequiresClause);1343  EXPECT_TOKEN(Tokens[35], tok::ampamp, TT_PointerOrReference);1344  EXPECT_TOKEN(Tokens[36], tok::kw_requires, TT_RequiresClause);1345  EXPECT_TOKEN(Tokens[47], tok::amp, TT_PointerOrReference);1346  EXPECT_TOKEN(Tokens[49], tok::kw_requires, TT_RequiresClause);1347  EXPECT_TOKEN(Tokens[59], tok::ampamp, TT_PointerOrReference);1348  EXPECT_TOKEN(Tokens[61], tok::kw_requires, TT_RequiresClause);1349  EXPECT_TOKEN(Tokens[76], tok::amp, TT_PointerOrReference);1350  EXPECT_TOKEN(Tokens[77], tok::kw_requires, TT_RequiresClause);1351 1352  Tokens = annotate("void Class::member() && requires(Constant) {}");1353  ASSERT_EQ(Tokens.size(), 14u) << Tokens;1354  EXPECT_TOKEN(Tokens[7], tok::kw_requires, TT_RequiresClause);1355 1356  Tokens = annotate("void Class::member() && requires(Constant<T>) {}");1357  ASSERT_EQ(Tokens.size(), 17u) << Tokens;1358  EXPECT_TOKEN(Tokens[7], tok::kw_requires, TT_RequiresClause);1359 1360  Tokens =1361      annotate("void Class::member() && requires(Namespace::Constant<T>) {}");1362  ASSERT_EQ(Tokens.size(), 19u) << Tokens;1363  EXPECT_TOKEN(Tokens[7], tok::kw_requires, TT_RequiresClause);1364 1365  Tokens = annotate("void Class::member() && requires(typename "1366                    "Namespace::Outer<T>::Inner::Constant) {}");1367  ASSERT_EQ(Tokens.size(), 24u) << Tokens;1368  EXPECT_TOKEN(Tokens[7], tok::kw_requires, TT_RequiresClause);1369 1370  Tokens = annotate("struct [[nodiscard]] zero_t {\n"1371                    "  template<class T>\n"1372                    "    requires requires { number_zero_v<T>; }\n"1373                    "  [[nodiscard]] constexpr operator T() const { "1374                    "return number_zero_v<T>; }\n"1375                    "};");1376  ASSERT_EQ(Tokens.size(), 44u) << Tokens;1377  EXPECT_TOKEN(Tokens[6], tok::identifier, TT_ClassHeadName);1378  EXPECT_TOKEN(Tokens[11], tok::identifier, TT_Unknown);1379  EXPECT_TOKEN(Tokens[13], tok::kw_requires, TT_RequiresClause);1380  EXPECT_TOKEN(Tokens[14], tok::kw_requires, TT_RequiresExpression);1381  EXPECT_TOKEN(Tokens[15], tok::l_brace, TT_RequiresExpressionLBrace);1382  EXPECT_TOKEN(Tokens[21], tok::r_brace, TT_Unknown);1383  EXPECT_EQ(Tokens[21]->MatchingParen, Tokens[15]);1384  EXPECT_TRUE(Tokens[21]->ClosesRequiresClause);1385 1386  Tokens = annotate("template <typename Foo>\n"1387                    "void Fun(const Foo &F)\n"1388                    "  requires requires(Foo F) {\n"1389                    "    { F.Bar() } -> std::same_as<int>;\n"1390                    "  };");1391  ASSERT_EQ(Tokens.size(), 38u) << Tokens;1392  EXPECT_TOKEN(Tokens[19], tok::l_brace, TT_RequiresExpressionLBrace);1393 1394  Tokens =1395      annotate("template <typename... Ts>\n"1396               "  requires requires {\n"1397               "    requires std::same_as<int, SomeTemplate<void(Ts &&...)>>;\n"1398               "  }\n"1399               "void Foo();");1400  ASSERT_EQ(Tokens.size(), 34u) << Tokens;1401  EXPECT_TOKEN(Tokens[21], tok::ampamp, TT_PointerOrReference);1402 1403  Tokens =1404      annotate("template <class A, class B> concept C ="1405               "std::same_as<std::iter_value_t<A>, std::iter_value_t<B>>;");1406  ASSERT_EQ(Tokens.size(), 31u) << Tokens;1407  EXPECT_TOKEN(Tokens[3], tok::identifier, TT_Unknown);1408  EXPECT_TOKEN(Tokens[6], tok::identifier, TT_Unknown);1409  EXPECT_TOKEN(Tokens[8], tok::kw_concept, TT_Unknown);1410  EXPECT_TOKEN(Tokens[14], tok::less, TT_TemplateOpener);1411  EXPECT_TOKEN(Tokens[18], tok::less, TT_TemplateOpener);1412  EXPECT_TOKEN(Tokens[20], tok::greater, TT_TemplateCloser);1413  EXPECT_TOKEN(Tokens[25], tok::less, TT_TemplateOpener);1414  EXPECT_TOKEN(Tokens[27], tok::greater, TT_TemplateCloser);1415  EXPECT_TOKEN(Tokens[28], tok::greater, TT_TemplateCloser);1416 1417  Tokens = annotate("auto bar() -> int requires(is_integral_v<T>) {}");1418  ASSERT_EQ(Tokens.size(), 16u) << Tokens;1419  EXPECT_TOKEN(Tokens[6], tok::kw_requires, TT_RequiresClause);1420 1421  Tokens = annotate("auto bar() -> void requires(is_integral_v<T>) {}");1422  ASSERT_EQ(Tokens.size(), 16u) << Tokens;1423  EXPECT_TOKEN(Tokens[6], tok::kw_requires, TT_RequiresClause);1424 1425  Tokens = annotate("auto bar() -> MyType requires(is_integral_v<T>) {}");1426  ASSERT_EQ(Tokens.size(), 16u) << Tokens;1427  EXPECT_TOKEN(Tokens[6], tok::kw_requires, TT_RequiresClause);1428 1429  Tokens =1430      annotate("auto bar() -> SOME_MACRO_TYPE requires(is_integral_v<T>) {}");1431  ASSERT_EQ(Tokens.size(), 16u) << Tokens;1432  EXPECT_TOKEN(Tokens[6], tok::kw_requires, TT_RequiresClause);1433 1434  Tokens =1435      annotate("auto bar() -> qualified::type requires(is_integral_v<T>) {}");1436  ASSERT_EQ(Tokens.size(), 18u) << Tokens;1437  EXPECT_TOKEN(Tokens[8], tok::kw_requires, TT_RequiresClause);1438 1439  Tokens =1440      annotate("auto bar() -> Template<type> requires(is_integral_v<T>) {}");1441  ASSERT_EQ(Tokens.size(), 19u) << Tokens;1442  EXPECT_TOKEN(Tokens[9], tok::kw_requires, TT_RequiresClause);1443 1444  Tokens = annotate("void foo() requires((A<T>) && C) {}");1445  ASSERT_EQ(Tokens.size(), 18u) << Tokens;1446  EXPECT_TOKEN(Tokens[4], tok::kw_requires, TT_RequiresClause);1447  EXPECT_TOKEN(Tokens[12], tok::ampamp, TT_BinaryOperator);1448 1449  Tokens = annotate("void foo() requires(((A<T>) && C)) {}");1450  ASSERT_EQ(Tokens.size(), 20u) << Tokens;1451  EXPECT_TOKEN(Tokens[4], tok::kw_requires, TT_RequiresClause);1452  EXPECT_TOKEN(Tokens[13], tok::ampamp, TT_BinaryOperator);1453 1454  Tokens = annotate("void foo() requires([](T&&){}(t)) {}");1455  ASSERT_EQ(Tokens.size(), 21u) << Tokens;1456  EXPECT_TOKEN(Tokens[4], tok::kw_requires, TT_RequiresClause);1457  EXPECT_TOKEN(Tokens[10], tok::ampamp, TT_PointerOrReference);1458 1459  Tokens = annotate("void foo() requires([](T&& u){}(t)) {}");1460  ASSERT_EQ(Tokens.size(), 22u) << Tokens;1461  EXPECT_TOKEN(Tokens[4], tok::kw_requires, TT_RequiresClause);1462  EXPECT_TOKEN(Tokens[10], tok::ampamp, TT_PointerOrReference);1463 1464  Tokens = annotate("void f() & requires(true) {}");1465  ASSERT_EQ(Tokens.size(), 12u) << Tokens;1466  EXPECT_TOKEN(Tokens[4], tok::amp, TT_PointerOrReference);1467  EXPECT_TOKEN(Tokens[5], tok::kw_requires, TT_RequiresClause);1468 1469  Tokens = annotate("void f() & requires(C<true, true>) {}");1470  ASSERT_EQ(Tokens.size(), 17u) << Tokens;1471  EXPECT_TOKEN(Tokens[4], tok::amp, TT_PointerOrReference);1472  EXPECT_TOKEN(Tokens[5], tok::kw_requires, TT_RequiresClause);1473 1474  Tokens = annotate("template <typename T>\n"1475                    "concept C = (!Foo<T>) && Bar;");1476  ASSERT_EQ(Tokens.size(), 19u) << Tokens;1477  EXPECT_TOKEN(Tokens[15], tok::ampamp, TT_BinaryOperator);1478 1479  Tokens = annotate("void f() & requires(C<decltype(x)>) {}");1480  ASSERT_EQ(Tokens.size(), 18u) << Tokens;1481  EXPECT_TOKEN(Tokens[4], tok::amp, TT_PointerOrReference);1482  EXPECT_TOKEN(Tokens[5], tok::kw_requires, TT_RequiresClause);1483 1484  Tokens = annotate("auto f() -> int& requires(C<decltype(x)>) {}");1485  ASSERT_EQ(Tokens.size(), 20u) << Tokens;1486  EXPECT_TOKEN(Tokens[6], tok::amp, TT_PointerOrReference);1487  EXPECT_TOKEN(Tokens[7], tok::kw_requires, TT_RequiresClause);1488 1489  Tokens = annotate("bool x = t && requires(decltype(t) x) { x.foo(); };");1490  ASSERT_EQ(Tokens.size(), 23u) << Tokens;1491  EXPECT_TOKEN(Tokens[5], tok::kw_requires, TT_RequiresExpression);1492 1493  Tokens = annotate("bool x = t && requires(Foo<decltype(t)> x) { x.foo(); };");1494  ASSERT_EQ(Tokens.size(), 26u) << Tokens;1495  EXPECT_TOKEN(Tokens[5], tok::kw_requires, TT_RequiresExpression);1496 1497  Tokens = annotate("bool x = t && requires(Foo<C1 || C2> x) { x.foo(); };");1498  ASSERT_EQ(Tokens.size(), 25u) << Tokens;1499  EXPECT_TOKEN(Tokens[5], tok::kw_requires, TT_RequiresExpression);1500 1501  // Second function definition is required due to lookahead1502  Tokens = annotate("void f() &\n"1503                    "  requires(n == 1)\n"1504                    "{}\n"1505                    "void g();");1506  ASSERT_EQ(Tokens.size(), 19u) << Tokens;1507  EXPECT_TOKEN(Tokens[4], tok::amp, TT_PointerOrReference);1508  EXPECT_TOKEN(Tokens[5], tok::kw_requires, TT_RequiresClause);1509 1510  Tokens = annotate("void foo() &&\n"1511                    "  requires(!bar)\n"1512                    "{\n"1513                    "  baz();\n"1514                    "}");1515  ASSERT_EQ(Tokens.size(), 17u) << Tokens;1516  EXPECT_TOKEN(Tokens[4], tok::ampamp, TT_PointerOrReference);1517  EXPECT_TOKEN(Tokens[5], tok::kw_requires, TT_RequiresClause);1518 1519  Tokens = annotate("auto foo() -> auto *\n"1520                    "  requires(not bar)\n"1521                    "{\n"1522                    "  return baz;\n"1523                    "}");1524  ASSERT_EQ(Tokens.size(), 18u) << Tokens;1525  EXPECT_TOKEN(Tokens[7], tok::kw_requires, TT_RequiresClause);1526  EXPECT_TOKEN(Tokens[12], tok::l_brace, TT_FunctionLBrace);1527}1528 1529TEST_F(TokenAnnotatorTest, UnderstandsRequiresExpressions) {1530  auto Tokens = annotate("bool b = requires(int i) { i + 5; };");1531  ASSERT_EQ(Tokens.size(), 16u) << Tokens;1532  EXPECT_TOKEN(Tokens[3], tok::kw_requires, TT_RequiresExpression);1533  EXPECT_TOKEN(Tokens[4], tok::l_paren, TT_RequiresExpressionLParen);1534  EXPECT_TOKEN(Tokens[8], tok::l_brace, TT_RequiresExpressionLBrace);1535 1536  Tokens = annotate("bool foo{requires { 0; }};");1537  ASSERT_EQ(Tokens.size(), 11u) << Tokens;1538  EXPECT_TOKEN(Tokens[3], tok::kw_requires, TT_RequiresExpression);1539  EXPECT_TOKEN(Tokens[4], tok::l_brace, TT_RequiresExpressionLBrace);1540 1541  Tokens = annotate("if (requires(int i) { i + 5; }) return;");1542  ASSERT_EQ(Tokens.size(), 17u) << Tokens;1543  EXPECT_TOKEN(Tokens[2], tok::kw_requires, TT_RequiresExpression);1544  EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_RequiresExpressionLParen);1545  EXPECT_TOKEN(Tokens[7], tok::l_brace, TT_RequiresExpressionLBrace);1546 1547  Tokens = annotate("if (func() && requires(int i) { i + 5; }) return;");1548  ASSERT_EQ(Tokens.size(), 21u) << Tokens;1549  EXPECT_TOKEN(Tokens[6], tok::kw_requires, TT_RequiresExpression);1550  EXPECT_TOKEN(Tokens[7], tok::l_paren, TT_RequiresExpressionLParen);1551  EXPECT_TOKEN(Tokens[11], tok::l_brace, TT_RequiresExpressionLBrace);1552 1553  Tokens = annotate("foo(requires(const T t) {});");1554  ASSERT_EQ(Tokens.size(), 13u) << Tokens;1555  EXPECT_TOKEN(Tokens[2], tok::kw_requires, TT_RequiresExpression);1556  EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_RequiresExpressionLParen);1557  EXPECT_TOKEN(Tokens[8], tok::l_brace, TT_RequiresExpressionLBrace);1558 1559  Tokens = annotate("foo(requires(const int t) {});");1560  ASSERT_EQ(Tokens.size(), 13u) << Tokens;1561  EXPECT_TOKEN(Tokens[2], tok::kw_requires, TT_RequiresExpression);1562  EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_RequiresExpressionLParen);1563  EXPECT_TOKEN(Tokens[8], tok::l_brace, TT_RequiresExpressionLBrace);1564 1565  Tokens = annotate("foo(requires(const T t) {});");1566  ASSERT_EQ(Tokens.size(), 13u) << Tokens;1567  EXPECT_TOKEN(Tokens[2], tok::kw_requires, TT_RequiresExpression);1568  EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_RequiresExpressionLParen);1569  EXPECT_TOKEN(Tokens[8], tok::l_brace, TT_RequiresExpressionLBrace);1570 1571  Tokens = annotate("foo(requires(int const* volatile t) {});");1572  ASSERT_EQ(Tokens.size(), 15u) << Tokens;1573  EXPECT_TOKEN(Tokens[2], tok::kw_requires, TT_RequiresExpression);1574  EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_RequiresExpressionLParen);1575  EXPECT_TOKEN(Tokens[6], tok::star, TT_PointerOrReference);1576  EXPECT_TOKEN(Tokens[10], tok::l_brace, TT_RequiresExpressionLBrace);1577 1578  Tokens = annotate("foo(requires(T const* volatile t) {});");1579  ASSERT_EQ(Tokens.size(), 15u) << Tokens;1580  EXPECT_TOKEN(Tokens[2], tok::kw_requires, TT_RequiresExpression);1581  EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_RequiresExpressionLParen);1582  EXPECT_TOKEN(Tokens[6], tok::star, TT_PointerOrReference);1583  EXPECT_TOKEN(Tokens[10], tok::l_brace, TT_RequiresExpressionLBrace);1584 1585  Tokens = annotate("foo(requires(T& t) {});");1586  ASSERT_EQ(Tokens.size(), 13u) << Tokens;1587  EXPECT_TOKEN(Tokens[2], tok::kw_requires, TT_RequiresExpression);1588  EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_RequiresExpressionLParen);1589  EXPECT_TOKEN(Tokens[5], tok::amp, TT_PointerOrReference);1590  EXPECT_TOKEN(Tokens[8], tok::l_brace, TT_RequiresExpressionLBrace);1591 1592  Tokens = annotate("foo(requires(T&& t) {});");1593  ASSERT_EQ(Tokens.size(), 13u) << Tokens;1594  EXPECT_TOKEN(Tokens[2], tok::kw_requires, TT_RequiresExpression);1595  EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_RequiresExpressionLParen);1596  EXPECT_TOKEN(Tokens[5], tok::ampamp, TT_PointerOrReference);1597  EXPECT_TOKEN(Tokens[8], tok::l_brace, TT_RequiresExpressionLBrace);1598 1599  Tokens = annotate("bool foo = requires(T& t) {};");1600  ASSERT_EQ(Tokens.size(), 13u) << Tokens;1601  EXPECT_TOKEN(Tokens[3], tok::kw_requires, TT_RequiresExpression);1602  EXPECT_TOKEN(Tokens[4], tok::l_paren, TT_RequiresExpressionLParen);1603  EXPECT_TOKEN(Tokens[6], tok::amp, TT_PointerOrReference);1604  EXPECT_TOKEN(Tokens[9], tok::l_brace, TT_RequiresExpressionLBrace);1605 1606  Tokens = annotate("bool foo = requires(T&& t) {};");1607  ASSERT_EQ(Tokens.size(), 13u) << Tokens;1608  EXPECT_TOKEN(Tokens[3], tok::kw_requires, TT_RequiresExpression);1609  EXPECT_TOKEN(Tokens[4], tok::l_paren, TT_RequiresExpressionLParen);1610  EXPECT_TOKEN(Tokens[6], tok::ampamp, TT_PointerOrReference);1611  EXPECT_TOKEN(Tokens[9], tok::l_brace, TT_RequiresExpressionLBrace);1612 1613  Tokens =1614      annotate("foo(requires(const typename Outer<T>::Inner * const t) {});");1615  ASSERT_EQ(Tokens.size(), 21u) << Tokens;1616  EXPECT_TOKEN(Tokens[2], tok::kw_requires, TT_RequiresExpression);1617  EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_RequiresExpressionLParen);1618  EXPECT_TOKEN(Tokens[12], tok::star, TT_PointerOrReference);1619  EXPECT_TOKEN(Tokens[16], tok::l_brace, TT_RequiresExpressionLBrace);1620 1621  Tokens = annotate("template <typename T>\n"1622                    "concept C = requires(T T) {\n"1623                    "  requires Bar<T> && Foo<T>;\n"1624                    "};");1625  ASSERT_EQ(Tokens.size(), 28u) << Tokens;1626  EXPECT_TOKEN(Tokens[8], tok::kw_requires, TT_RequiresExpression);1627  EXPECT_TOKEN(Tokens[9], tok::l_paren, TT_RequiresExpressionLParen);1628  EXPECT_TOKEN(Tokens[13], tok::l_brace, TT_RequiresExpressionLBrace);1629  EXPECT_TOKEN(Tokens[14], tok::kw_requires,1630               TT_RequiresClauseInARequiresExpression);1631 1632  Tokens = annotate("template <typename T>\n"1633                    "concept C = requires(T T) {\n"1634                    "  { t.func() } -> std::same_as<int>;"1635                    "  requires Bar<T> && Foo<T>;\n"1636                    "};");1637  ASSERT_EQ(Tokens.size(), 43u) << Tokens;1638  EXPECT_TOKEN(Tokens[8], tok::kw_requires, TT_RequiresExpression);1639  EXPECT_TOKEN(Tokens[9], tok::l_paren, TT_RequiresExpressionLParen);1640  EXPECT_TOKEN(Tokens[13], tok::l_brace, TT_RequiresExpressionLBrace);1641  EXPECT_TOKEN(Tokens[29], tok::kw_requires,1642               TT_RequiresClauseInARequiresExpression);1643 1644  // Invalid Code, but we don't want to crash. See http://llvm.org/PR54350.1645  Tokens = annotate("bool r10 = requires (struct new_struct { int x; } s) { "1646                    "requires true; };");1647  ASSERT_EQ(Tokens.size(), 21u) << Tokens;1648  EXPECT_TOKEN(Tokens[3], tok::kw_requires, TT_RequiresExpression);1649  EXPECT_TOKEN(Tokens[4], tok::l_paren, TT_RequiresExpressionLParen);1650  EXPECT_TOKEN(Tokens[14], tok::l_brace, TT_RequiresExpressionLBrace);1651 1652  Tokens = annotate("bool foo = requires(C<true, true> c) {\n"1653                    "  { c.foo(); }\n"1654                    "};");1655  ASSERT_EQ(Tokens.size(), 25u) << Tokens;1656  EXPECT_TOKEN(Tokens[3], tok::kw_requires, TT_RequiresExpression);1657  EXPECT_TOKEN(Tokens[4], tok::l_paren, TT_RequiresExpressionLParen);1658  EXPECT_TOKEN(Tokens[13], tok::l_brace, TT_RequiresExpressionLBrace);1659}1660 1661TEST_F(TokenAnnotatorTest, UnderstandsPragmaRegion) {1662  // Everything after #pragma region should be ImplicitStringLiteral1663  auto Tokens = annotate("#pragma region Foo(Bar: Hello)");1664  ASSERT_EQ(Tokens.size(), 10u) << Tokens;1665  EXPECT_TOKEN(Tokens[5], tok::identifier, TT_ImplicitStringLiteral);1666  EXPECT_TOKEN(Tokens[6], tok::colon, TT_ImplicitStringLiteral);1667  EXPECT_TOKEN(Tokens[7], tok::identifier, TT_ImplicitStringLiteral);1668 1669  // Make sure it's annotated correctly inside a function as well1670  Tokens = annotate("void test(){\n#pragma region Foo(Bar: Hello)\n}");1671  ASSERT_EQ(Tokens.size(), 16u) << Tokens;1672  EXPECT_TOKEN(Tokens[10], tok::identifier, TT_ImplicitStringLiteral);1673  EXPECT_TOKEN(Tokens[11], tok::colon, TT_ImplicitStringLiteral);1674  EXPECT_TOKEN(Tokens[12], tok::identifier, TT_ImplicitStringLiteral);1675}1676 1677TEST_F(TokenAnnotatorTest, RequiresDoesNotChangeParsingOfTheRest) {1678  const char *BaseCode = nullptr;1679  const char *ConstrainedCode = nullptr;1680  auto BaseTokenCount = 0u;1681  auto RequiresTokenCount = 0u;1682  auto PrefixTokenCount = 0u;1683 1684  auto TestRequires = [&](int Line) {1685    const auto BaseTokens = annotate(BaseCode);1686    const auto ConstrainedTokens = annotate(ConstrainedCode);1687 1688#define LINE " (Line " << Line << ')'1689 1690    ASSERT_EQ(BaseTokens.size(), BaseTokenCount) << BaseTokens << LINE;1691    ASSERT_EQ(ConstrainedTokens.size(), BaseTokenCount + RequiresTokenCount)1692        << LINE;1693 1694    for (auto I = 0u; I < BaseTokenCount; ++I) {1695      EXPECT_EQ(1696          *BaseTokens[I],1697          *ConstrainedTokens[I < PrefixTokenCount ? I : I + RequiresTokenCount])1698          << I << LINE;1699    }1700 1701#undef LINE1702  };1703 1704  BaseCode = "template<typename T>\n"1705             "T Pi = 3.14;";1706  ConstrainedCode = "template<typename T>\n"1707                    "  requires Foo<T>\n"1708                    "T Pi = 3.14;";1709  BaseTokenCount = 11;1710  RequiresTokenCount = 5;1711  PrefixTokenCount = 5;1712  TestRequires(__LINE__);1713 1714  BaseCode = "template<typename T>\n"1715             "struct Bar;";1716  ConstrainedCode = "template<typename T>\n"1717                    "  requires Foo<T>\n"1718                    "struct Bar;";1719  BaseTokenCount = 9;1720  TestRequires(__LINE__);1721 1722  BaseCode = "template<typename T>\n"1723             "struct Bar {\n"1724             "  T foo();\n"1725             "  T bar();\n"1726             "};";1727  ConstrainedCode = "template<typename T>\n"1728                    "  requires Foo<T>\n"1729                    "struct Bar {\n"1730                    "  T foo();\n"1731                    "  T bar();\n"1732                    "};";1733  BaseTokenCount = 21;1734  TestRequires(__LINE__);1735 1736  BaseCode = "template<typename T>\n"1737             "Bar(T) -> Bar<T>;";1738  ConstrainedCode = "template<typename T>\n"1739                    "  requires Foo<T>\n"1740                    "Bar(T) -> Bar<T>;";1741  BaseTokenCount = 16;1742  TestRequires(__LINE__);1743 1744  BaseCode = "template<typename T>\n"1745             "T foo();";1746  ConstrainedCode = "template<typename T>\n"1747                    "  requires Foo<T>\n"1748                    "T foo();";1749  BaseTokenCount = 11;1750  TestRequires(__LINE__);1751 1752  BaseCode = "template<typename T>\n"1753             "T foo() {\n"1754             "  auto bar = baz();\n"1755             "  return bar + T{};\n"1756             "}";1757  ConstrainedCode = "template<typename T>\n"1758                    "  requires Foo<T>\n"1759                    "T foo() {\n"1760                    "  auto bar = baz();\n"1761                    "  return bar + T{};\n"1762                    "}";1763  BaseTokenCount = 26;1764  TestRequires(__LINE__);1765 1766  BaseCode = "template<typename T>\n"1767             "T foo();";1768  ConstrainedCode = "template<typename T>\n"1769                    "T foo() requires Foo<T>;";1770  BaseTokenCount = 11;1771  PrefixTokenCount = 9;1772  TestRequires(__LINE__);1773 1774  BaseCode = "template<typename T>\n"1775             "T foo() {\n"1776             "  auto bar = baz();\n"1777             "  return bar + T{};\n"1778             "}";1779  ConstrainedCode = "template<typename T>\n"1780                    "T foo() requires Foo<T> {\n"1781                    "  auto bar = baz();\n"1782                    "  return bar + T{};\n"1783                    "}";1784  BaseTokenCount = 26;1785  TestRequires(__LINE__);1786 1787  BaseCode = "template<typename T>\n"1788             "T foo();";1789  ConstrainedCode = "template<typename T>\n"1790                    "  requires(Foo<T>)\n"1791                    "T foo();";1792  BaseTokenCount = 11;1793  RequiresTokenCount = 7;1794  PrefixTokenCount = 5;1795  TestRequires(__LINE__);1796 1797  BaseCode = "template<typename T>\n"1798             "Bar(T) -> Bar<typename T::I>;";1799  ConstrainedCode = "template<typename T>\n"1800                    "  requires requires(T &&t) {\n"1801                    "             typename T::I;\n"1802                    "           }\n"1803                    "Bar(T) -> Bar<typename T::I>;";1804  BaseTokenCount = 19;1805  RequiresTokenCount = 14;1806  PrefixTokenCount = 5;1807  TestRequires(__LINE__);1808 1809  BaseCode = "struct [[nodiscard]] zero_t {\n"1810             "  template<class T>\n"1811             "  [[nodiscard]] constexpr operator T() const { return v<T>; }\n"1812             "};";1813  ConstrainedCode =1814      "struct [[nodiscard]] zero_t {\n"1815      "  template<class T>\n"1816      "    requires requires { v<T>; }\n"1817      "  [[nodiscard]] constexpr operator T() const { return v<T>; }\n"1818      "};";1819  BaseTokenCount = 35;1820  RequiresTokenCount = 9;1821  PrefixTokenCount = 13;1822  TestRequires(__LINE__);1823 1824  BaseCode = "constexpr Foo(Foo const &other)\n"1825             "    : value{other.value} {\n"1826             "  do_magic();\n"1827             "  do_more_magic();\n"1828             "}";1829  ConstrainedCode = "constexpr Foo(Foo const &other)\n"1830                    "  requires std::is_copy_constructible<T>\n"1831                    "    : value{other.value} {\n"1832                    "  do_magic();\n"1833                    "  do_more_magic();\n"1834                    "}";1835  BaseTokenCount = 26;1836  RequiresTokenCount = 7;1837  PrefixTokenCount = 8;1838  TestRequires(__LINE__);1839 1840  BaseCode = "constexpr Foo(Foo const &other)\n"1841             "    : value{other.value} {\n"1842             "  do_magic();\n"1843             "  do_more_magic();\n"1844             "}";1845  ConstrainedCode = "constexpr Foo(Foo const &other)\n"1846                    "  requires (std::is_copy_constructible<T>)\n"1847                    "    : value{other.value} {\n"1848                    "  do_magic();\n"1849                    "  do_more_magic();\n"1850                    "}";1851  RequiresTokenCount = 9;1852  TestRequires(__LINE__);1853 1854  BaseCode = "template<typename T>\n"1855             "ANNOTATE(\"S\"\n"1856             "         \"S\")\n"1857             "void foo();";1858  ConstrainedCode = "template<typename T>\n"1859                    "  requires(true)\n"1860                    "ANNOTATE(\"S\"\n"1861                    "         \"S\")\n"1862                    "void foo();";1863  BaseTokenCount = 16;1864  RequiresTokenCount = 4;1865  PrefixTokenCount = 5;1866  TestRequires(__LINE__);1867}1868 1869TEST_F(TokenAnnotatorTest, UnderstandsAsm) {1870  auto Tokens = annotate("__asm{\n"1871                         "\"a\":\n"1872                         ": x\n"1873                         ":};");1874  ASSERT_EQ(Tokens.size(), 10u) << Tokens;1875  EXPECT_TOKEN(Tokens[0], tok::kw_asm, TT_Unknown);1876  EXPECT_TOKEN(Tokens[1], tok::l_brace, TT_InlineASMBrace);1877  EXPECT_TOKEN(Tokens[3], tok::colon, TT_InlineASMColon);1878  EXPECT_TOKEN(Tokens[4], tok::colon, TT_InlineASMColon);1879  EXPECT_TOKEN(Tokens[6], tok::colon, TT_InlineASMColon);1880  EXPECT_TOKEN(Tokens[7], tok::r_brace, TT_InlineASMBrace);1881 1882  Tokens = annotate("__asm(\n"1883                    "\"a\":\n"1884                    ": x\n"1885                    ":);");1886  ASSERT_EQ(Tokens.size(), 10u) << Tokens;1887  EXPECT_TOKEN(Tokens[0], tok::kw_asm, TT_Unknown);1888  EXPECT_TOKEN(Tokens[3], tok::colon, TT_InlineASMColon);1889  EXPECT_TOKEN(Tokens[4], tok::colon, TT_InlineASMColon);1890  EXPECT_TOKEN(Tokens[6], tok::colon, TT_InlineASMColon);1891 1892  Tokens = annotate("asm volatile (\n"1893                    "\"a_label:\"\n"1894                    ":\n"1895                    ": x\n"1896                    ":);");1897  ASSERT_EQ(Tokens.size(), 11u) << Tokens;1898  EXPECT_TOKEN(Tokens[0], tok::kw_asm, TT_Unknown);1899  EXPECT_TOKEN(Tokens[4], tok::colon, TT_InlineASMColon);1900  EXPECT_TOKEN(Tokens[5], tok::colon, TT_InlineASMColon);1901  EXPECT_TOKEN(Tokens[7], tok::colon, TT_InlineASMColon);1902 1903  Tokens = annotate("__asm__(\n"1904                    "\"a_label:\"\n"1905                    ": x\n"1906                    ":\n"1907                    ": y);");1908  ASSERT_EQ(Tokens.size(), 11u) << Tokens;1909  EXPECT_TOKEN(Tokens[0], tok::kw_asm, TT_Unknown);1910  EXPECT_TOKEN(Tokens[3], tok::colon, TT_InlineASMColon);1911  EXPECT_TOKEN(Tokens[5], tok::colon, TT_InlineASMColon);1912  EXPECT_TOKEN(Tokens[6], tok::colon, TT_InlineASMColon);1913 1914  Tokens = annotate("__asm volatile (\n"1915                    "\"a_label:\"\n"1916                    "\"a b c(%%x)\"\n"1917                    ":\n"1918                    ": x\n"1919                    ":);");1920  ASSERT_EQ(Tokens.size(), 12u) << Tokens;1921  EXPECT_TOKEN(Tokens[0], tok::kw_asm, TT_Unknown);1922  EXPECT_TOKEN(Tokens[5], tok::colon, TT_InlineASMColon);1923  EXPECT_TOKEN(Tokens[6], tok::colon, TT_InlineASMColon);1924  EXPECT_TOKEN(Tokens[8], tok::colon, TT_InlineASMColon);1925 1926  Tokens = annotate("asm(\n"1927                    "\"insn\"\n"1928                    ": \"=r\" (var1), \"=&r\" (value)\n"1929                    ":\n"1930                    ": \"memory\");");1931  ASSERT_EQ(Tokens.size(), 19u) << Tokens;1932  EXPECT_TOKEN(Tokens[0], tok::kw_asm, TT_Unknown);1933  EXPECT_TOKEN(Tokens[3], tok::colon, TT_InlineASMColon);1934  EXPECT_TOKEN(Tokens[13], tok::colon, TT_InlineASMColon);1935  EXPECT_TOKEN(Tokens[14], tok::colon, TT_InlineASMColon);1936 1937  Tokens = annotate("__asm__ volatile (\n"1938                    "\"ldr r1, [r0, %%[sym]]\"\n"1939                    ":\n"1940                    ": [sym] \"J\" (aaaaa(aaaa, aaaa))\n"1941                    ");");1942  ASSERT_EQ(Tokens.size(), 21u) << Tokens;1943  EXPECT_TOKEN(Tokens[0], tok::kw_asm, TT_Unknown);1944  EXPECT_TOKEN(Tokens[4], tok::colon, TT_InlineASMColon);1945  EXPECT_TOKEN(Tokens[5], tok::colon, TT_InlineASMColon);1946  EXPECT_TOKEN(Tokens[6], tok::l_square, TT_InlineASMSymbolicNameLSquare);1947}1948 1949TEST_F(TokenAnnotatorTest, UnderstandsObjCBlock) {1950  auto Tokens = annotate("int (^)() = ^ ()\n"1951                         "  external_source_symbol() { //\n"1952                         "  return 1;\n"1953                         "};");1954  ASSERT_EQ(Tokens.size(), 21u) << Tokens;1955  EXPECT_TOKEN(Tokens[1], tok::l_paren, TT_ObjCBlockLParen);1956  EXPECT_TOKEN(Tokens[13], tok::l_brace, TT_ObjCBlockLBrace);1957 1958  Tokens = annotate("int *p = ^int*(){ //\n"1959                    "  return nullptr;\n"1960                    "}();");1961  ASSERT_EQ(Tokens.size(), 19u) << Tokens;1962  EXPECT_TOKEN(Tokens[9], tok::l_brace, TT_ObjCBlockLBrace);1963 1964  Tokens = annotate("id (^block)(Foo *a) = ^id _Nullable(Foo *_Nullable a) {\n"1965                    "  return a;\n"1966                    "};");1967  ASSERT_EQ(Tokens.size(), 27u) << Tokens;1968  EXPECT_TOKEN(Tokens[0], tok::identifier, TT_Unknown); // Not CtorDtorDeclName.1969  EXPECT_TOKEN(Tokens[1], tok::l_paren, TT_ObjCBlockLParen);1970}1971 1972TEST_F(TokenAnnotatorTest, UnderstandsObjCMethodExpr) {1973  auto Tokens = annotate("void f() {\n"1974                         "  //\n"1975                         "  BOOL a = [b.c n] > 1;\n"1976                         "}");1977  ASSERT_EQ(Tokens.size(), 20u) << Tokens;1978  EXPECT_TOKEN(Tokens[9], tok::l_square, TT_ObjCMethodExpr);1979  EXPECT_TOKEN(Tokens[15], tok::greater, TT_BinaryOperator);1980 1981  Tokens = annotate("a = @selector(name:);");1982  ASSERT_EQ(Tokens.size(), 10u) << Tokens;1983  EXPECT_TOKEN(Tokens[4], tok::l_paren, TT_ObjCSelector);1984  EXPECT_TOKEN(Tokens[6], tok::colon, TT_ObjCSelector);1985  EXPECT_TOKEN(Tokens[7], tok::r_paren, TT_ObjCSelector);1986 1987  Tokens =1988      annotate("[object respondsToSelector:@selector(name:param1:param2:)\n"1989               "        respondsToSelector:@selector(name:param1:param2:)];");1990  ASSERT_EQ(Tokens.size(), 29u) << Tokens;1991  EXPECT_TOKEN(Tokens[0], tok::l_square, TT_ObjCMethodExpr);1992  EXPECT_TOKEN(Tokens[3], tok::colon, TT_ObjCMethodExpr);1993  EXPECT_TOKEN(Tokens[6], tok::l_paren, TT_ObjCSelector);1994  EXPECT_TOKEN(Tokens[8], tok::colon, TT_ObjCSelector);1995  EXPECT_TOKEN(Tokens[10], tok::colon, TT_ObjCSelector);1996  EXPECT_TOKEN(Tokens[12], tok::colon, TT_ObjCSelector);1997  EXPECT_TOKEN(Tokens[13], tok::r_paren, TT_ObjCSelector);1998  EXPECT_TOKEN(Tokens[15], tok::colon, TT_ObjCMethodExpr);1999  EXPECT_TOKEN(Tokens[18], tok::l_paren, TT_ObjCSelector);2000  EXPECT_TOKEN(Tokens[20], tok::colon, TT_ObjCSelector);2001  EXPECT_TOKEN(Tokens[22], tok::colon, TT_ObjCSelector);2002  EXPECT_TOKEN(Tokens[24], tok::colon, TT_ObjCSelector);2003  EXPECT_TOKEN(Tokens[25], tok::r_paren, TT_ObjCSelector);2004  EXPECT_TOKEN(Tokens[26], tok::r_square, TT_ObjCMethodExpr);2005 2006  Tokens = annotate("[a b:c];");2007  ASSERT_EQ(Tokens.size(), 8u) << Tokens;2008  EXPECT_TOKEN(Tokens[0], tok::l_square, TT_ObjCMethodExpr);2009  EXPECT_TOKEN(Tokens[3], tok::colon, TT_ObjCMethodExpr);2010  EXPECT_TOKEN(Tokens[5], tok::r_square, TT_ObjCMethodExpr);2011}2012 2013TEST_F(TokenAnnotatorTest, UnderstandsObjCMethodDecl) {2014  auto Tokens = annotate("/**/ - (void)foo;");2015  ASSERT_EQ(Tokens.size(), 8u) << Tokens;2016  EXPECT_TOKEN(Tokens[1], tok::minus, TT_ObjCMethodSpecifier);2017  EXPECT_TOKEN(Tokens[5], tok::identifier, TT_SelectorName);2018}2019 2020TEST_F(TokenAnnotatorTest, UnderstandsLambdas) {2021  auto Tokens = annotate("[]() constexpr {}");2022  ASSERT_EQ(Tokens.size(), 8u) << Tokens;2023  EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);2024  EXPECT_TOKEN(Tokens[2], tok::l_paren, TT_LambdaDefinitionLParen);2025  EXPECT_TOKEN(Tokens[5], tok::l_brace, TT_LambdaLBrace);2026 2027  Tokens = annotate("[]() consteval {}");2028  ASSERT_EQ(Tokens.size(), 8u) << Tokens;2029  EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);2030  EXPECT_TOKEN(Tokens[2], tok::l_paren, TT_LambdaDefinitionLParen);2031  EXPECT_TOKEN(Tokens[5], tok::l_brace, TT_LambdaLBrace);2032 2033  Tokens = annotate("[]() mutable {}");2034  ASSERT_EQ(Tokens.size(), 8u) << Tokens;2035  EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);2036  EXPECT_TOKEN(Tokens[2], tok::l_paren, TT_LambdaDefinitionLParen);2037  EXPECT_TOKEN(Tokens[5], tok::l_brace, TT_LambdaLBrace);2038 2039  Tokens = annotate("[]() static {}");2040  ASSERT_EQ(Tokens.size(), 8u) << Tokens;2041  EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);2042  EXPECT_TOKEN(Tokens[2], tok::l_paren, TT_LambdaDefinitionLParen);2043  EXPECT_TOKEN(Tokens[5], tok::l_brace, TT_LambdaLBrace);2044 2045  Tokens = annotate("[]() -> auto {}");2046  ASSERT_EQ(Tokens.size(), 9u) << Tokens;2047  EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);2048  EXPECT_TOKEN(Tokens[2], tok::l_paren, TT_LambdaDefinitionLParen);2049  EXPECT_TOKEN(Tokens[4], tok::arrow, TT_LambdaArrow);2050  EXPECT_TOKEN(Tokens[6], tok::l_brace, TT_LambdaLBrace);2051 2052  Tokens = annotate("[]() -> auto & {}");2053  ASSERT_EQ(Tokens.size(), 10u) << Tokens;2054  EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);2055  EXPECT_TOKEN(Tokens[2], tok::l_paren, TT_LambdaDefinitionLParen);2056  EXPECT_TOKEN(Tokens[4], tok::arrow, TT_LambdaArrow);2057  EXPECT_TOKEN(Tokens[7], tok::l_brace, TT_LambdaLBrace);2058 2059  Tokens = annotate("[]() -> auto * {}");2060  ASSERT_EQ(Tokens.size(), 10u) << Tokens;2061  EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);2062  EXPECT_TOKEN(Tokens[2], tok::l_paren, TT_LambdaDefinitionLParen);2063  EXPECT_TOKEN(Tokens[4], tok::arrow, TT_LambdaArrow);2064  EXPECT_TOKEN(Tokens[7], tok::l_brace, TT_LambdaLBrace);2065 2066  Tokens = annotate("[] {}");2067  ASSERT_EQ(Tokens.size(), 5u) << Tokens;2068  EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);2069  EXPECT_TOKEN(Tokens[2], tok::l_brace, TT_LambdaLBrace);2070 2071  Tokens = annotate("[] noexcept {}");2072  ASSERT_EQ(Tokens.size(), 6u) << Tokens;2073  EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);2074  EXPECT_TOKEN(Tokens[3], tok::l_brace, TT_LambdaLBrace);2075 2076  Tokens = annotate("[] -> auto {}");2077  ASSERT_EQ(Tokens.size(), 7u) << Tokens;2078  EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);2079  EXPECT_TOKEN(Tokens[2], tok::arrow, TT_LambdaArrow);2080  EXPECT_TOKEN(Tokens[4], tok::l_brace, TT_LambdaLBrace);2081 2082  Tokens = annotate("[] -> struct S { return {}; }");2083  ASSERT_EQ(Tokens.size(), 12u) << Tokens;2084  EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);2085  EXPECT_TOKEN(Tokens[2], tok::arrow, TT_LambdaArrow);2086  EXPECT_TOKEN(Tokens[4], tok::identifier, TT_Unknown);2087  EXPECT_TOKEN(Tokens[5], tok::l_brace, TT_LambdaLBrace);2088 2089  Tokens = annotate("foo([&](u32 bar) __attribute__((attr)) -> void {});");2090  ASSERT_EQ(Tokens.size(), 22u) << Tokens;2091  EXPECT_TOKEN(Tokens[2], tok::l_square, TT_LambdaLSquare);2092  EXPECT_TOKEN(Tokens[5], tok::l_paren, TT_LambdaDefinitionLParen);2093  EXPECT_TOKEN(Tokens[15], tok::arrow, TT_LambdaArrow);2094  EXPECT_TOKEN(Tokens[17], tok::l_brace, TT_LambdaLBrace);2095 2096  Tokens = annotate("[] <typename T> () {}");2097  ASSERT_EQ(Tokens.size(), 11u) << Tokens;2098  EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);2099  EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);2100  EXPECT_TOKEN(Tokens[6], tok::l_paren, TT_LambdaDefinitionLParen);2101  EXPECT_TOKEN(Tokens[8], tok::l_brace, TT_LambdaLBrace);2102 2103  Tokens = annotate("[] <typename T> {}");2104  ASSERT_EQ(Tokens.size(), 9u) << Tokens;2105  EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);2106  EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);2107  EXPECT_TOKEN(Tokens[6], tok::l_brace, TT_LambdaLBrace);2108 2109  Tokens = annotate("[] <typename... T> () {}");2110  ASSERT_EQ(Tokens.size(), 12u) << Tokens;2111  EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);2112  EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);2113  EXPECT_TOKEN(Tokens[7], tok::l_paren, TT_LambdaDefinitionLParen);2114  EXPECT_TOKEN(Tokens[9], tok::l_brace, TT_LambdaLBrace);2115 2116  Tokens = annotate("[] <typename... T> {}");2117  ASSERT_EQ(Tokens.size(), 10u) << Tokens;2118  EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);2119  EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);2120  EXPECT_TOKEN(Tokens[7], tok::l_brace, TT_LambdaLBrace);2121 2122  Tokens = annotate("[] <int... T> () {}");2123  ASSERT_EQ(Tokens.size(), 12u) << Tokens;2124  EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);2125  EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);2126  EXPECT_TOKEN(Tokens[7], tok::l_paren, TT_LambdaDefinitionLParen);2127  EXPECT_TOKEN(Tokens[9], tok::l_brace, TT_LambdaLBrace);2128 2129  Tokens = annotate("[] <int... T> {}");2130  ASSERT_EQ(Tokens.size(), 10u) << Tokens;2131  EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);2132  EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);2133  EXPECT_TOKEN(Tokens[7], tok::l_brace, TT_LambdaLBrace);2134 2135  Tokens = annotate("[] <Foo... T> () {}");2136  ASSERT_EQ(Tokens.size(), 12u) << Tokens;2137  EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);2138  EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);2139  EXPECT_TOKEN(Tokens[7], tok::l_paren, TT_LambdaDefinitionLParen);2140  EXPECT_TOKEN(Tokens[9], tok::l_brace, TT_LambdaLBrace);2141 2142  Tokens = annotate("[] <Foo... T> {}");2143  ASSERT_EQ(Tokens.size(), 10u) << Tokens;2144  EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);2145  EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);2146  EXPECT_TOKEN(Tokens[7], tok::l_brace, TT_LambdaLBrace);2147 2148  // Lambdas with a requires-clause2149  Tokens = annotate("[] <typename T> (T t) requires Bar<T> {}");2150  ASSERT_EQ(Tokens.size(), 18u) << Tokens;2151  EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);2152  EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);2153  EXPECT_TOKEN(Tokens[6], tok::l_paren, TT_LambdaDefinitionLParen);2154  EXPECT_TOKEN(Tokens[10], tok::kw_requires, TT_RequiresClause);2155  EXPECT_TRUE(Tokens[14]->ClosesRequiresClause);2156  EXPECT_TOKEN(Tokens[15], tok::l_brace, TT_LambdaLBrace);2157 2158  Tokens = annotate("[] <typename T> (T &&t) requires Bar<T> {}");2159  ASSERT_EQ(Tokens.size(), 19u) << Tokens;2160  EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);2161  EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);2162  EXPECT_TOKEN(Tokens[6], tok::l_paren, TT_LambdaDefinitionLParen);2163  EXPECT_TOKEN(Tokens[8], tok::ampamp, TT_PointerOrReference);2164  EXPECT_TOKEN(Tokens[11], tok::kw_requires, TT_RequiresClause);2165  EXPECT_TRUE(Tokens[15]->ClosesRequiresClause);2166  EXPECT_TOKEN(Tokens[16], tok::l_brace, TT_LambdaLBrace);2167 2168  Tokens = annotate("[] <typename T> (T t) requires Foo<T> || Bar<T> {}");2169  ASSERT_EQ(Tokens.size(), 23u) << Tokens;2170  EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);2171  EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);2172  EXPECT_TOKEN(Tokens[6], tok::l_paren, TT_LambdaDefinitionLParen);2173  EXPECT_TOKEN(Tokens[10], tok::kw_requires, TT_RequiresClause);2174  EXPECT_TRUE(Tokens[19]->ClosesRequiresClause);2175  EXPECT_TOKEN(Tokens[20], tok::l_brace, TT_LambdaLBrace);2176 2177  Tokens = annotate("[] <typename T> (T t) -> T requires Bar<T> {}");2178  ASSERT_EQ(Tokens.size(), 20u) << Tokens;2179  EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);2180  EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);2181  EXPECT_TOKEN(Tokens[6], tok::l_paren, TT_LambdaDefinitionLParen);2182  EXPECT_TOKEN(Tokens[10], tok::arrow, TT_LambdaArrow);2183  EXPECT_TOKEN(Tokens[12], tok::kw_requires, TT_RequiresClause);2184  EXPECT_TRUE(Tokens[16]->ClosesRequiresClause);2185  EXPECT_TOKEN(Tokens[17], tok::l_brace, TT_LambdaLBrace);2186 2187  Tokens = annotate("[] <typename T> requires Bar<T> (T t) {}");2188  ASSERT_EQ(Tokens.size(), 18u) << Tokens;2189  EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);2190  EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);2191  EXPECT_TOKEN(Tokens[6], tok::kw_requires, TT_RequiresClause);2192  EXPECT_TRUE(Tokens[10]->ClosesRequiresClause);2193  // FIXME:2194  // EXPECT_TOKEN(Tokens[11], tok::l_paren, TT_LambdaDefinitionLParen);2195  EXPECT_TOKEN(Tokens[15], tok::l_brace, TT_LambdaLBrace);2196 2197  Tokens = annotate("[] <typename T> requires Bar<T> (T &&t) {}");2198  ASSERT_EQ(Tokens.size(), 19u) << Tokens;2199  EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);2200  EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);2201  EXPECT_TOKEN(Tokens[6], tok::kw_requires, TT_RequiresClause);2202  EXPECT_TRUE(Tokens[10]->ClosesRequiresClause);2203  // FIXME:2204  // EXPECT_TOKEN(Tokens[11], tok::l_paren, TT_LambdaDefinitionLParen);2205  EXPECT_TOKEN(Tokens[13], tok::ampamp, TT_PointerOrReference);2206  EXPECT_TOKEN(Tokens[16], tok::l_brace, TT_LambdaLBrace);2207 2208  Tokens = annotate("[] <typename T> requires Foo<T> || Bar<T> (T t) {}");2209  ASSERT_EQ(Tokens.size(), 23u) << Tokens;2210  EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);2211  EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);2212  EXPECT_TOKEN(Tokens[6], tok::kw_requires, TT_RequiresClause);2213  EXPECT_TRUE(Tokens[15]->ClosesRequiresClause);2214  // FIXME:2215  // EXPECT_TOKEN(Tokens[16], tok::l_paren, TT_LambdaDefinitionLParen);2216  EXPECT_TOKEN(Tokens[20], tok::l_brace, TT_LambdaLBrace);2217 2218  Tokens = annotate("[] <typename T> requires true (T&& t) {}");2219  ASSERT_EQ(Tokens.size(), 16u) << Tokens;2220  EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);2221  EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);2222  EXPECT_TOKEN(Tokens[6], tok::kw_requires, TT_RequiresClause);2223  EXPECT_TRUE(Tokens[7]->ClosesRequiresClause);2224  // FIXME:2225  // EXPECT_TOKEN(Tokens[8], tok::l_paren, TT_LambdaDefinitionLParen);2226  EXPECT_TOKEN(Tokens[10], tok::ampamp, TT_PointerOrReference);2227  EXPECT_TOKEN(Tokens[13], tok::l_brace, TT_LambdaLBrace);2228 2229  Tokens = annotate("[] <typename T> requires Bar<T> {}");2230  ASSERT_EQ(Tokens.size(), 14u) << Tokens;2231  EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);2232  EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);2233  EXPECT_TOKEN(Tokens[6], tok::kw_requires, TT_RequiresClause);2234  EXPECT_TRUE(Tokens[10]->ClosesRequiresClause);2235  EXPECT_TOKEN(Tokens[11], tok::l_brace, TT_LambdaLBrace);2236 2237  Tokens = annotate("[] <typename T> requires Bar<T> noexcept {}");2238  ASSERT_EQ(Tokens.size(), 15u) << Tokens;2239  EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);2240  EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);2241  EXPECT_TOKEN(Tokens[6], tok::kw_requires, TT_RequiresClause);2242  EXPECT_TRUE(Tokens[10]->ClosesRequiresClause);2243  EXPECT_TOKEN(Tokens[12], tok::l_brace, TT_LambdaLBrace);2244 2245  Tokens = annotate("[] <typename T> requires Bar<T> -> T {}");2246  ASSERT_EQ(Tokens.size(), 16u) << Tokens;2247  EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);2248  EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);2249  EXPECT_TOKEN(Tokens[6], tok::kw_requires, TT_RequiresClause);2250  EXPECT_TRUE(Tokens[10]->ClosesRequiresClause);2251  EXPECT_TOKEN(Tokens[11], tok::arrow, TT_LambdaArrow);2252  EXPECT_TOKEN(Tokens[13], tok::l_brace, TT_LambdaLBrace);2253 2254  Tokens = annotate("[] <typename T> requires Foo<T> (T t) requires Bar<T> {}");2255  ASSERT_EQ(Tokens.size(), 23u) << Tokens;2256  EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);2257  EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);2258  EXPECT_TOKEN(Tokens[6], tok::kw_requires, TT_RequiresClause);2259  EXPECT_TRUE(Tokens[10]->ClosesRequiresClause);2260  // FIXME:2261  // EXPECT_TOKEN(Tokens[11], tok::l_paren, TT_LambdaDefinitionLParen);2262  EXPECT_TOKEN(Tokens[15], tok::kw_requires, TT_RequiresClause);2263  EXPECT_TRUE(Tokens[19]->ClosesRequiresClause);2264  EXPECT_TOKEN(Tokens[20], tok::l_brace, TT_LambdaLBrace);2265 2266  Tokens = annotate("[] <typename T = int> (T t) {}");2267  ASSERT_EQ(Tokens.size(), 15u) << Tokens;2268  EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);2269  EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);2270  EXPECT_TOKEN(Tokens[7], tok::greater, TT_TemplateCloser);2271  EXPECT_TOKEN(Tokens[8], tok::l_paren, TT_LambdaDefinitionLParen);2272  EXPECT_TOKEN(Tokens[12], tok::l_brace, TT_LambdaLBrace);2273 2274  Tokens = annotate("[] <int I = 0> (T t) {}");2275  ASSERT_EQ(Tokens.size(), 15u) << Tokens;2276  EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);2277  EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);2278  EXPECT_TOKEN(Tokens[7], tok::greater, TT_TemplateCloser);2279  EXPECT_TOKEN(Tokens[8], tok::l_paren, TT_LambdaDefinitionLParen);2280  EXPECT_TOKEN(Tokens[12], tok::l_brace, TT_LambdaLBrace);2281 2282  Tokens = annotate("[] <bool b = false> (T t) {}");2283  ASSERT_EQ(Tokens.size(), 15u) << Tokens;2284  EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);2285  EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);2286  EXPECT_TOKEN(Tokens[7], tok::greater, TT_TemplateCloser);2287  EXPECT_TOKEN(Tokens[8], tok::l_paren, TT_LambdaDefinitionLParen);2288  EXPECT_TOKEN(Tokens[12], tok::l_brace, TT_LambdaLBrace);2289 2290  Tokens = annotate("[] <bool b = true && false> (T&& t) {}");2291  ASSERT_EQ(Tokens.size(), 18u) << Tokens;2292  EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);2293  EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);2294  EXPECT_TOKEN(Tokens[7], tok::ampamp, TT_BinaryOperator);2295  EXPECT_TOKEN(Tokens[9], tok::greater, TT_TemplateCloser);2296  EXPECT_TOKEN(Tokens[10], tok::l_paren, TT_LambdaDefinitionLParen);2297  EXPECT_TOKEN(Tokens[12], tok::ampamp, TT_PointerOrReference);2298  EXPECT_TOKEN(Tokens[15], tok::l_brace, TT_LambdaLBrace);2299 2300  Tokens = annotate("[] <typename T = int> requires Foo<T> (T t) {}");2301  ASSERT_EQ(Tokens.size(), 20u) << Tokens;2302  EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);2303  EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);2304  EXPECT_TOKEN(Tokens[7], tok::greater, TT_TemplateCloser);2305  EXPECT_TOKEN(Tokens[8], tok::kw_requires, TT_RequiresClause);2306  // FIXME:2307  // EXPECT_TOKEN(Tokens[13], tok::l_paren, TT_LambdaDefinitionLParen);2308  EXPECT_TOKEN(Tokens[17], tok::l_brace, TT_LambdaLBrace);2309 2310  Tokens = annotate("auto foo{(std::function<int()>)[] { return 0; }};");2311  ASSERT_EQ(Tokens.size(), 23u) << Tokens;2312  EXPECT_TOKEN(Tokens[13], tok::l_square, TT_LambdaLSquare);2313  EXPECT_TOKEN(Tokens[15], tok::l_brace, TT_LambdaLBrace);2314 2315  Tokens = annotate("auto foo{(int (*)())[] { return 0; }};");2316  ASSERT_EQ(Tokens.size(), 21u) << Tokens;2317  EXPECT_TOKEN(Tokens[11], tok::l_square, TT_LambdaLSquare);2318  EXPECT_TOKEN(Tokens[13], tok::l_brace, TT_LambdaLBrace);2319 2320  Tokens = annotate("SomeFunction({[]() -> int *[] { return {}; }});");2321  ASSERT_EQ(Tokens.size(), 22u) << Tokens;2322  EXPECT_TOKEN(Tokens[3], tok::l_square, TT_LambdaLSquare);2323  EXPECT_TOKEN(Tokens[5], tok::l_paren, TT_LambdaDefinitionLParen);2324  EXPECT_TOKEN(Tokens[10], tok::l_square, TT_ArraySubscriptLSquare);2325}2326 2327TEST_F(TokenAnnotatorTest, UnderstandsFunctionAnnotations) {2328  auto Tokens = annotate("template <typename T>\n"2329                         "DEPRECATED(\"Use NewClass::NewFunction instead.\")\n"2330                         "string OldFunction(const string &parameter) {}");2331  ASSERT_EQ(Tokens.size(), 20u) << Tokens;2332  EXPECT_TOKEN(Tokens[8], tok::r_paren, TT_FunctionAnnotationRParen);2333 2334  Tokens = annotate("template <typename T>\n"2335                    "A(T) noexcept;");2336  ASSERT_EQ(Tokens.size(), 12u) << Tokens;2337  EXPECT_TOKEN(Tokens[8], tok::r_paren, TT_Unknown);2338 2339  Tokens = annotate("FOO(bar)();");2340  ASSERT_EQ(Tokens.size(), 8u) << Tokens;2341  EXPECT_TOKEN(Tokens[3], tok::r_paren, TT_Unknown);2342}2343 2344TEST_F(TokenAnnotatorTest, UnderstandsFunctionDeclarationNames) {2345  auto Tokens = annotate("void f [[noreturn]] ();");2346  ASSERT_EQ(Tokens.size(), 11u) << Tokens;2347  EXPECT_TOKEN(Tokens[1], tok::identifier, TT_FunctionDeclarationName);2348  EXPECT_TOKEN(Tokens[7], tok::l_paren, TT_FunctionDeclarationLParen);2349 2350  Tokens = annotate("void f [[noreturn]] () {}");2351  ASSERT_EQ(Tokens.size(), 12u) << Tokens;2352  EXPECT_TOKEN(Tokens[1], tok::identifier, TT_FunctionDeclarationName);2353  EXPECT_TOKEN(Tokens[7], tok::l_paren, TT_FunctionDeclarationLParen);2354 2355  Tokens = annotate("#define FOO Foo::\n"2356                    "FOO Foo();");2357  ASSERT_EQ(Tokens.size(), 11u) << Tokens;2358  EXPECT_TOKEN(Tokens[6], tok::identifier, TT_FunctionDeclarationName);2359  EXPECT_TOKEN(Tokens[7], tok::l_paren, TT_FunctionDeclarationLParen);2360 2361  Tokens = annotate("struct Foo {\n"2362                    "  Bar (*func)();\n"2363                    "};");2364  ASSERT_EQ(Tokens.size(), 14u) << Tokens;2365  EXPECT_TOKEN(Tokens[3], tok::identifier, TT_Unknown);2366  EXPECT_TOKEN(Tokens[4], tok::l_paren, TT_FunctionTypeLParen);2367 2368  Tokens = annotate("void instanceof();");2369  ASSERT_EQ(Tokens.size(), 6u) << Tokens;2370  EXPECT_TOKEN(Tokens[1], tok::identifier, TT_FunctionDeclarationName);2371  EXPECT_TOKEN(Tokens[2], tok::l_paren, TT_FunctionDeclarationLParen);2372 2373  Tokens = annotate("#define FUNC(foo, bar, baz) \\\n"2374                    "  auto foo##bar##baz() -> Type {}");2375  ASSERT_EQ(Tokens.size(), 23u) << Tokens;2376  EXPECT_TOKEN(Tokens[11], tok::identifier, TT_FunctionDeclarationName);2377  EXPECT_TOKEN(Tokens[16], tok::l_paren, TT_FunctionDeclarationLParen);2378  EXPECT_TOKEN(Tokens[18], tok::arrow, TT_TrailingReturnArrow);2379 2380  Tokens = annotate("void __stdcall f();");2381  ASSERT_EQ(Tokens.size(), 7u) << Tokens;2382  EXPECT_TOKEN(Tokens[2], tok::identifier, TT_FunctionDeclarationName);2383  EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_FunctionDeclarationLParen);2384 2385  Tokens = annotate("int iso_time(time_t);");2386  ASSERT_EQ(Tokens.size(), 7u) << Tokens;2387  EXPECT_TOKEN(Tokens[1], tok::identifier, TT_FunctionDeclarationName);2388  EXPECT_TOKEN(Tokens[2], tok::l_paren, TT_FunctionDeclarationLParen);2389 2390  auto Style = getLLVMStyle();2391  Style.TypeNames.push_back("MyType");2392  Tokens = annotate("int iso_time(MyType);", Style);2393  ASSERT_EQ(Tokens.size(), 7u) << Tokens;2394  EXPECT_TOKEN(Tokens[1], tok::identifier, TT_FunctionDeclarationName);2395  EXPECT_TOKEN(Tokens[2], tok::l_paren, TT_FunctionDeclarationLParen);2396  EXPECT_TOKEN(Tokens[3], tok::identifier, TT_TypeName);2397}2398 2399TEST_F(TokenAnnotatorTest, UnderstandsCtorAndDtorDeclNames) {2400  auto Tokens = annotate("class Foo { public: Foo(); };");2401  ASSERT_EQ(Tokens.size(), 12u) << Tokens;2402  EXPECT_TOKEN(Tokens[5], tok::identifier, TT_CtorDtorDeclName);2403  EXPECT_TOKEN(Tokens[6], tok::l_paren, TT_FunctionDeclarationLParen);2404 2405  Tokens = annotate("class Foo { public: ~Foo(); };");2406  ASSERT_EQ(Tokens.size(), 13u) << Tokens;2407  EXPECT_TOKEN(Tokens[6], tok::identifier, TT_CtorDtorDeclName);2408  EXPECT_TOKEN(Tokens[7], tok::l_paren, TT_FunctionDeclarationLParen);2409 2410  Tokens = annotate("struct Foo { [[deprecated]] Foo() {} };");2411  ASSERT_EQ(Tokens.size(), 16u) << Tokens;2412  EXPECT_TOKEN(Tokens[8], tok::identifier, TT_CtorDtorDeclName);2413  EXPECT_TOKEN(Tokens[9], tok::l_paren, TT_FunctionDeclarationLParen);2414  EXPECT_TOKEN(Tokens[11], tok::l_brace, TT_FunctionLBrace);2415 2416  Tokens = annotate("struct Foo { [[deprecated]] ~Foo() {} };");2417  ASSERT_EQ(Tokens.size(), 17u) << Tokens;2418  EXPECT_TOKEN(Tokens[9], tok::identifier, TT_CtorDtorDeclName);2419  EXPECT_TOKEN(Tokens[10], tok::l_paren, TT_FunctionDeclarationLParen);2420  EXPECT_TOKEN(Tokens[12], tok::l_brace, TT_FunctionLBrace);2421 2422  Tokens = annotate("struct Foo { Foo() [[deprecated]] {} };");2423  ASSERT_EQ(Tokens.size(), 16u) << Tokens;2424  EXPECT_TOKEN(Tokens[3], tok::identifier, TT_CtorDtorDeclName);2425  EXPECT_TOKEN(Tokens[4], tok::l_paren, TT_FunctionDeclarationLParen);2426  EXPECT_TOKEN(Tokens[11], tok::l_brace, TT_FunctionLBrace);2427 2428  Tokens = annotate("struct Foo { ~Foo() [[deprecated]] {} };");2429  ASSERT_EQ(Tokens.size(), 17u) << Tokens;2430  EXPECT_TOKEN(Tokens[4], tok::identifier, TT_CtorDtorDeclName);2431  EXPECT_TOKEN(Tokens[5], tok::l_paren, TT_FunctionDeclarationLParen);2432  EXPECT_TOKEN(Tokens[12], tok::l_brace, TT_FunctionLBrace);2433 2434  Tokens = annotate("struct Foo { [[deprecated]] explicit Foo() {} };");2435  ASSERT_EQ(Tokens.size(), 17u) << Tokens;2436  EXPECT_TOKEN(Tokens[9], tok::identifier, TT_CtorDtorDeclName);2437  EXPECT_TOKEN(Tokens[10], tok::l_paren, TT_FunctionDeclarationLParen);2438  EXPECT_TOKEN(Tokens[12], tok::l_brace, TT_FunctionLBrace);2439 2440  Tokens = annotate("struct Foo { virtual [[deprecated]] ~Foo() {} };");2441  ASSERT_EQ(Tokens.size(), 18u) << Tokens;2442  EXPECT_TOKEN(Tokens[10], tok::identifier, TT_CtorDtorDeclName);2443  EXPECT_TOKEN(Tokens[11], tok::l_paren, TT_FunctionDeclarationLParen);2444  EXPECT_TOKEN(Tokens[13], tok::l_brace, TT_FunctionLBrace);2445 2446  Tokens = annotate("Foo::Foo() {}");2447  ASSERT_EQ(Tokens.size(), 8u) << Tokens;2448  EXPECT_TOKEN(Tokens[2], tok::identifier, TT_CtorDtorDeclName);2449  EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_FunctionDeclarationLParen);2450  EXPECT_TOKEN(Tokens[5], tok::l_brace, TT_FunctionLBrace);2451 2452  Tokens = annotate("Foo::~Foo() {}");2453  ASSERT_EQ(Tokens.size(), 9u) << Tokens;2454  EXPECT_TOKEN(Tokens[3], tok::identifier, TT_CtorDtorDeclName);2455  EXPECT_TOKEN(Tokens[4], tok::l_paren, TT_FunctionDeclarationLParen);2456  EXPECT_TOKEN(Tokens[6], tok::l_brace, TT_FunctionLBrace);2457 2458  Tokens = annotate("struct Test {\n"2459                    "  Test()\n"2460                    "      : l([] {\n"2461                    "          Short::foo();\n"2462                    "        }) {}\n"2463                    "};");2464  ASSERT_EQ(Tokens.size(), 25u) << Tokens;2465  EXPECT_TOKEN(Tokens[3], tok::identifier, TT_CtorDtorDeclName);2466  EXPECT_TOKEN(Tokens[4], tok::l_paren, TT_FunctionDeclarationLParen);2467  EXPECT_TOKEN(Tokens[14], tok::identifier, TT_Unknown);2468}2469 2470TEST_F(TokenAnnotatorTest, UnderstandsC11GenericSelection) {2471  auto Tokens = annotate("_Generic(x, int: 1, default: 0)");2472  ASSERT_EQ(Tokens.size(), 13u) << Tokens;2473  EXPECT_TOKEN(Tokens[0], tok::kw__Generic, TT_Unknown);2474  EXPECT_TOKEN(Tokens[5], tok::colon, TT_GenericSelectionColon);2475  EXPECT_TOKEN(Tokens[9], tok::colon, TT_GenericSelectionColon);2476}2477 2478TEST_F(TokenAnnotatorTest, UnderstandsTrailingReturnArrow) {2479  auto Tokens = annotate("auto f() -> int;");2480  ASSERT_EQ(Tokens.size(), 8u) << Tokens;2481  EXPECT_TOKEN(Tokens[4], tok::arrow, TT_TrailingReturnArrow);2482 2483  Tokens = annotate("auto operator->() -> int;");2484  ASSERT_EQ(Tokens.size(), 9u) << Tokens;2485  EXPECT_TOKEN(Tokens[2], tok::arrow, TT_OverloadedOperator);2486  EXPECT_TOKEN(Tokens[5], tok::arrow, TT_TrailingReturnArrow);2487 2488  Tokens = annotate("auto operator++(int) -> int;");2489  ASSERT_EQ(Tokens.size(), 10u) << Tokens;2490  EXPECT_TOKEN(Tokens[6], tok::arrow, TT_TrailingReturnArrow);2491 2492  Tokens = annotate("auto operator=() -> int;");2493  ASSERT_EQ(Tokens.size(), 9u) << Tokens;2494  EXPECT_TOKEN(Tokens[5], tok::arrow, TT_TrailingReturnArrow);2495 2496  Tokens = annotate("auto operator=(int) -> int;");2497  ASSERT_EQ(Tokens.size(), 10u) << Tokens;2498  EXPECT_TOKEN(Tokens[6], tok::arrow, TT_TrailingReturnArrow);2499 2500  Tokens = annotate("auto foo() -> auto { return Val; }");2501  ASSERT_EQ(Tokens.size(), 12u) << Tokens;2502  EXPECT_TOKEN(Tokens[4], tok::arrow, TT_TrailingReturnArrow);2503 2504  Tokens = annotate("struct S { auto bar() const -> int; };");2505  ASSERT_EQ(Tokens.size(), 14u) << Tokens;2506  EXPECT_TOKEN(Tokens[8], tok::arrow, TT_TrailingReturnArrow);2507 2508  // Not trailing return arrows2509  Tokens = annotate("auto a = b->c;");2510  ASSERT_EQ(Tokens.size(), 8u) << Tokens;2511  EXPECT_TOKEN(Tokens[4], tok::arrow, TT_Unknown);2512 2513  Tokens = annotate("auto a = (b)->c;");2514  ASSERT_EQ(Tokens.size(), 10u) << Tokens;2515  EXPECT_TOKEN(Tokens[6], tok::arrow, TT_Unknown);2516 2517  Tokens = annotate("auto a = b()->c;");2518  ASSERT_EQ(Tokens.size(), 10u) << Tokens;2519  EXPECT_TOKEN(Tokens[6], tok::arrow, TT_Unknown);2520 2521  Tokens = annotate("auto a = b->c();");2522  ASSERT_EQ(Tokens.size(), 10u) << Tokens;2523  EXPECT_TOKEN(Tokens[4], tok::arrow, TT_Unknown);2524 2525  Tokens = annotate("decltype(auto) a = b()->c;");2526  ASSERT_EQ(Tokens.size(), 13u) << Tokens;2527  EXPECT_TOKEN(Tokens[9], tok::arrow, TT_Unknown);2528 2529  Tokens = annotate("void f() { auto a = b->c(); }");2530  ASSERT_EQ(Tokens.size(), 16u) << Tokens;2531  EXPECT_TOKEN(Tokens[9], tok::arrow, TT_Unknown);2532 2533  Tokens = annotate("void f() { auto a = b()->c; }");2534  ASSERT_EQ(Tokens.size(), 16u) << Tokens;2535  EXPECT_TOKEN(Tokens[11], tok::arrow, TT_Unknown);2536 2537  Tokens = annotate("#define P(ptr) auto p = (ptr)->p");2538  ASSERT_EQ(Tokens.size(), 15u) << Tokens;2539  EXPECT_TOKEN(Tokens[12], tok::arrow, TT_Unknown);2540 2541  Tokens = annotate("void f() FOO(foo->bar);");2542  ASSERT_EQ(Tokens.size(), 12u) << Tokens;2543  EXPECT_TOKEN(Tokens[7], tok::arrow, TT_Unknown);2544 2545  Tokens = annotate("__attribute__((cold)) C() : Base(obj->func()) {}");2546  ASSERT_EQ(Tokens.size(), 21u) << Tokens;2547  EXPECT_TOKEN(Tokens[13], tok::arrow, TT_Unknown);2548 2549  Tokens = annotate("Class<Type>{foo}->func(arg);");2550  ASSERT_EQ(Tokens.size(), 14u) << Tokens;2551  EXPECT_TOKEN(Tokens[4], tok::l_brace, TT_Unknown); // Not FunctionLBrace2552  EXPECT_BRACE_KIND(Tokens[4], BK_BracedInit);2553  EXPECT_BRACE_KIND(Tokens[6], BK_BracedInit);2554  EXPECT_TOKEN(Tokens[7], tok::arrow, TT_Unknown);2555 2556  auto Style = getLLVMStyle();2557  Style.StatementAttributeLikeMacros.push_back("emit");2558  Tokens = annotate("emit foo()->bar;", Style);2559  ASSERT_EQ(Tokens.size(), 8u) << Tokens;2560  EXPECT_TOKEN(Tokens[0], tok::identifier, TT_StatementAttributeLikeMacro);2561  EXPECT_TOKEN(Tokens[4], tok::arrow, TT_Unknown);2562 2563  // Mixed2564  Tokens = annotate("auto f() -> int { auto a = b()->c; }");2565  ASSERT_EQ(Tokens.size(), 18u) << Tokens;2566  EXPECT_TOKEN(Tokens[4], tok::arrow, TT_TrailingReturnArrow);2567  EXPECT_TOKEN(Tokens[13], tok::arrow, TT_Unknown);2568}2569 2570TEST_F(TokenAnnotatorTest, UnderstandHashInMacro) {2571  auto Tokens = annotate("#define Foo(Bar) \\\n"2572                         "  { \\\n"2573                         "    #Bar \\\n"2574                         "  }");2575  ASSERT_EQ(Tokens.size(), 11u) << Tokens;2576  EXPECT_BRACE_KIND(Tokens[6], BK_BracedInit);2577  EXPECT_BRACE_KIND(Tokens[9], BK_BracedInit);2578 2579  Tokens = annotate("#define Foo(Bar) \\\n"2580                    "  { #Bar }");2581  ASSERT_EQ(Tokens.size(), 11u) << Tokens;2582  EXPECT_BRACE_KIND(Tokens[6], BK_BracedInit);2583  EXPECT_BRACE_KIND(Tokens[9], BK_BracedInit);2584 2585  Tokens = annotate("#define FOO(typeName, realClass) \\\n"2586                    "  {#typeName, foo<Foo>(new foo<realClass>(#typeName))}");2587  ASSERT_EQ(Tokens.size(), 29u) << Tokens;2588  EXPECT_BRACE_KIND(Tokens[8], BK_BracedInit);2589  EXPECT_BRACE_KIND(Tokens[27], BK_BracedInit);2590}2591 2592TEST_F(TokenAnnotatorTest, UnderstandsAttributeMacros) {2593  // '__attribute__' has special handling.2594  auto Tokens = annotate("__attribute__((X)) void Foo(void);");2595  ASSERT_EQ(Tokens.size(), 13u) << Tokens;2596  EXPECT_TOKEN(Tokens[0], tok::kw___attribute, TT_Unknown);2597  EXPECT_TOKEN(Tokens[1], tok::l_paren, TT_AttributeLParen);2598  EXPECT_TOKEN(Tokens[5], tok::r_paren, TT_AttributeRParen);2599 2600  // Generic macro has no special handling in this location.2601  Tokens = annotate("A(X) void Foo(void);");2602  ASSERT_EQ(Tokens.size(), 11u) << Tokens;2603  EXPECT_TOKEN(Tokens[0], tok::identifier, TT_Unknown);2604  EXPECT_TOKEN(Tokens[1], tok::l_paren, TT_Unknown);2605 2606  // Add a custom AttributeMacro. Test that it has the same behavior.2607  FormatStyle Style = getLLVMStyle();2608  Style.AttributeMacros.push_back("A");2609 2610  // An "AttributeMacro" gets annotated like '__attribute__'.2611  Tokens = annotate("A(X) void Foo(void);", Style);2612  ASSERT_EQ(Tokens.size(), 11u) << Tokens;2613  EXPECT_TOKEN(Tokens[0], tok::identifier, TT_AttributeMacro);2614  EXPECT_TOKEN(Tokens[1], tok::l_paren, TT_AttributeLParen);2615  EXPECT_TOKEN(Tokens[3], tok::r_paren, TT_AttributeRParen);2616}2617 2618TEST_F(TokenAnnotatorTest, UnderstandsAttributeMacrosOnObjCDecl) {2619  // '__attribute__' has special handling.2620  auto Tokens = annotate("__attribute__((X)) @interface Foo");2621  ASSERT_EQ(Tokens.size(), 10u) << Tokens;2622  EXPECT_TOKEN(Tokens[0], tok::kw___attribute, TT_Unknown);2623  EXPECT_TOKEN(Tokens[1], tok::l_paren, TT_AttributeLParen);2624  EXPECT_TOKEN(Tokens[5], tok::r_paren, TT_AttributeRParen);2625 2626  // Generic macro has no special handling in this location.2627  Tokens = annotate("A(X) @interface Foo");2628  ASSERT_EQ(Tokens.size(), 8u) << Tokens;2629  // Note: Don't check token-type as a random token in this position is hard to2630  // reason about.2631  EXPECT_TOKEN_KIND(Tokens[0], tok::identifier);2632  EXPECT_TOKEN_KIND(Tokens[1], tok::l_paren);2633 2634  // Add a custom AttributeMacro. Test that it has the same behavior.2635  FormatStyle Style = getLLVMStyle();2636  Style.AttributeMacros.push_back("A");2637 2638  // An "AttributeMacro" gets annotated like '__attribute__'.2639  Tokens = annotate("A(X) @interface Foo", Style);2640  ASSERT_EQ(Tokens.size(), 8u) << Tokens;2641  EXPECT_TOKEN(Tokens[0], tok::identifier, TT_AttributeMacro);2642  EXPECT_TOKEN(Tokens[1], tok::l_paren, TT_AttributeLParen);2643  EXPECT_TOKEN(Tokens[3], tok::r_paren, TT_AttributeRParen);2644}2645 2646TEST_F(TokenAnnotatorTest, UnderstandsAttributeMacrosOnObjCMethodDecl) {2647  // '__attribute__' has special handling.2648  auto Tokens = annotate("- (id)init __attribute__((X));");2649  ASSERT_EQ(Tokens.size(), 13u) << Tokens;2650  EXPECT_TOKEN(Tokens[5], tok::kw___attribute, TT_Unknown);2651  EXPECT_TOKEN(Tokens[6], tok::l_paren, TT_AttributeLParen);2652  EXPECT_TOKEN(Tokens[10], tok::r_paren, TT_AttributeRParen);2653 2654  // Generic macro has no special handling in this location.2655  Tokens = annotate("- (id)init A(X);");2656  ASSERT_EQ(Tokens.size(), 11u) << Tokens;2657  // Note: Don't check token-type as a random token in this position is hard to2658  // reason about.2659  EXPECT_TOKEN_KIND(Tokens[5], tok::identifier);2660  EXPECT_TOKEN_KIND(Tokens[6], tok::l_paren);2661 2662  // Add a custom AttributeMacro. Test that it has the same behavior.2663  FormatStyle Style = getLLVMStyle();2664  Style.AttributeMacros.push_back("A");2665 2666  // An "AttributeMacro" gets annotated like '__attribute__'.2667  Tokens = annotate("- (id)init A(X);", Style);2668  ASSERT_EQ(Tokens.size(), 11u) << Tokens;2669  EXPECT_TOKEN(Tokens[5], tok::identifier, TT_AttributeMacro);2670  EXPECT_TOKEN(Tokens[6], tok::l_paren, TT_AttributeLParen);2671  EXPECT_TOKEN(Tokens[8], tok::r_paren, TT_AttributeRParen);2672}2673 2674TEST_F(TokenAnnotatorTest, UnderstandsAttributeMacrosOnObjCProperty) {2675  // '__attribute__' has special handling.2676  auto Tokens = annotate("@property(weak) id delegate __attribute__((X));");2677  ASSERT_EQ(Tokens.size(), 15u) << Tokens;2678  EXPECT_TOKEN(Tokens[7], tok::kw___attribute, TT_Unknown);2679  EXPECT_TOKEN(Tokens[8], tok::l_paren, TT_AttributeLParen);2680  EXPECT_TOKEN(Tokens[12], tok::r_paren, TT_AttributeRParen);2681 2682  // Generic macro has no special handling in this location.2683  Tokens = annotate("@property(weak) id delegate A(X);");2684  ASSERT_EQ(Tokens.size(), 13u) << Tokens;2685  // Note: Don't check token-type as a random token in this position is hard to2686  // reason about.2687  EXPECT_TOKEN_KIND(Tokens[7], tok::identifier);2688  EXPECT_TOKEN_KIND(Tokens[8], tok::l_paren);2689 2690  // Add a custom AttributeMacro. Test that it has the same behavior.2691  FormatStyle Style = getLLVMStyle();2692  Style.AttributeMacros.push_back("A");2693 2694  // An "AttributeMacro" gets annotated like '__attribute__'.2695  Tokens = annotate("@property(weak) id delegate A(X);", Style);2696  ASSERT_EQ(Tokens.size(), 13u) << Tokens;2697  EXPECT_TOKEN(Tokens[7], tok::identifier, TT_AttributeMacro);2698  EXPECT_TOKEN(Tokens[8], tok::l_paren, TT_AttributeLParen);2699  EXPECT_TOKEN(Tokens[10], tok::r_paren, TT_AttributeRParen);2700}2701 2702TEST_F(TokenAnnotatorTest, UnderstandsVerilogOperators) {2703  auto Annotate = [this](StringRef Code) {2704    return annotate(Code, getLLVMStyle(FormatStyle::LK_Verilog));2705  };2706  // Test that unary operators get labeled as such and that operators like '++'2707  // don't get split.2708  tok::TokenKind Unary[] = {tok::plus,  tok::minus,    tok::exclaim,2709                            tok::tilde, tok::amp,      tok::pipe,2710                            tok::caret, tok::plusplus, tok::minusminus};2711  for (auto Kind : Unary) {2712    auto Tokens =2713        Annotate(std::string("x = ") + tok::getPunctuatorSpelling(Kind) + "x;");2714    ASSERT_EQ(Tokens.size(), 6u) << Tokens;2715    EXPECT_TOKEN(Tokens[2], Kind, TT_UnaryOperator);2716  }2717  // Operators formed by joining two operators like '^~'. For some of these2718  // joined operators, we don't have a separate type, so we only test for their2719  // precedence.2720  std::pair<prec::Level, std::string> JoinedBinary[] = {2721      {prec::Comma, "->"},        {prec::Comma, "<->"},2722      {prec::Comma, "#-#"},       {prec::Comma, "#=#"},2723      {prec::Assignment, "+="},   {prec::Assignment, "-="},2724      {prec::Assignment, "*="},   {prec::Assignment, "/="},2725      {prec::Assignment, "%="},   {prec::Assignment, "&="},2726      {prec::Assignment, "^="},   {prec::Assignment, "<<="},2727      {prec::Assignment, ">>="},  {prec::Assignment, "<<<="},2728      {prec::Assignment, ">>>="}, {prec::LogicalOr, "||"},2729      {prec::LogicalAnd, "&&"},   {prec::Equality, "=="},2730      {prec::Equality, "!="},     {prec::Equality, "==="},2731      {prec::Equality, "!=="},    {prec::Equality, "==?"},2732      {prec::Equality, "!=?"},    {prec::ExclusiveOr, "~^"},2733      {prec::ExclusiveOr, "^~"},2734  };2735  for (auto Operator : JoinedBinary) {2736    auto Tokens = Annotate(std::string("x = x ") + Operator.second + " x;");2737    ASSERT_EQ(Tokens.size(), 7u) << Tokens;2738    EXPECT_TOKEN_TYPE(Tokens[3], TT_BinaryOperator);2739    EXPECT_TOKEN_PRECEDENCE(Tokens[3], Operator.first);2740  }2741  // '~^' and '^~' can be unary as well as binary operators.2742  auto Tokens = Annotate("x = ~^x;");2743  ASSERT_EQ(Tokens.size(), 6u) << Tokens;2744  EXPECT_TOKEN_TYPE(Tokens[2], TT_UnaryOperator);2745  Tokens = Annotate("x = ^~x;");2746  ASSERT_EQ(Tokens.size(), 6u) << Tokens;2747  EXPECT_TOKEN_TYPE(Tokens[2], TT_UnaryOperator);2748  // The unary operators '~&' and '~|' can only be unary operators. The current2749  // implementation treats each of them as separate unary '~' and '&' or '|'2750  // operators, which is enough for formatting purposes. In FormatTestVerilog,2751  // there is a test that there is no space in between. And even if a new line2752  // is inserted between the '~' and '|', the semantic meaning is the same as2753  // the joined operator, so the CanBreakBefore property doesn't need to be2754  // false for the second operator.2755  Tokens = Annotate("x = ~&x;");2756  ASSERT_EQ(Tokens.size(), 7u) << Tokens;2757  EXPECT_TOKEN(Tokens[2], tok::tilde, TT_UnaryOperator);2758  EXPECT_TOKEN(Tokens[3], tok::amp, TT_UnaryOperator);2759  Tokens = Annotate("x = ~|x;");2760  ASSERT_EQ(Tokens.size(), 7u) << Tokens;2761  EXPECT_TOKEN(Tokens[2], tok::tilde, TT_UnaryOperator);2762  EXPECT_TOKEN(Tokens[3], tok::pipe, TT_UnaryOperator);2763  // Test for block label colons.2764  Tokens = Annotate("begin : x\n"2765                    "end : x");2766  ASSERT_EQ(Tokens.size(), 7u) << Tokens;2767  EXPECT_TOKEN(Tokens[1], tok::colon, TT_VerilogBlockLabelColon);2768  EXPECT_TOKEN(Tokens[4], tok::colon, TT_VerilogBlockLabelColon);2769  // Test that the dimension colon is annotated correctly.2770  Tokens = Annotate("var [1 : 0] x;");2771  ASSERT_EQ(Tokens.size(), 9u) << Tokens;2772  EXPECT_TOKEN(Tokens[3], tok::colon, TT_BitFieldColon);2773  Tokens = Annotate("extern function [1 : 0] x;");2774  ASSERT_EQ(Tokens.size(), 10u) << Tokens;2775  EXPECT_TOKEN(Tokens[4], tok::colon, TT_BitFieldColon);2776  Tokens = Annotate("module test\n"2777                    "    (input wire [7 : 0] a[7 : 0]);\n"2778                    "endmodule");2779  ASSERT_EQ(Tokens.size(), 20u) << Tokens;2780  EXPECT_TOKEN(Tokens[4], tok::identifier, TT_VerilogDimensionedTypeName);2781  EXPECT_TOKEN(Tokens[7], tok::colon, TT_BitFieldColon);2782  EXPECT_TOKEN(Tokens[13], tok::colon, TT_BitFieldColon);2783  // Test case labels and ternary operators.2784  Tokens = Annotate("case (x)\n"2785                    "  x:\n"2786                    "    x;\n"2787                    "endcase");2788  ASSERT_EQ(Tokens.size(), 10u) << Tokens;2789  EXPECT_TOKEN(Tokens[5], tok::colon, TT_CaseLabelColon);2790  Tokens = Annotate("case (x)\n"2791                    "  x ? x : x:\n"2792                    "    x;\n"2793                    "endcase");2794  ASSERT_EQ(Tokens.size(), 14u) << Tokens;2795  EXPECT_TOKEN(Tokens[5], tok::question, TT_ConditionalExpr);2796  EXPECT_TOKEN(Tokens[7], tok::colon, TT_ConditionalExpr);2797  EXPECT_TOKEN(Tokens[9], tok::colon, TT_CaseLabelColon);2798  // Non-blocking assignments.2799  Tokens = Annotate("a <= b;");2800  ASSERT_EQ(Tokens.size(), 5u) << Tokens;2801  EXPECT_TOKEN(Tokens[1], tok::lessequal, TT_BinaryOperator);2802  EXPECT_TOKEN_PRECEDENCE(Tokens[1], prec::Assignment);2803  Tokens = Annotate("if (a <= b) break;");2804  ASSERT_EQ(Tokens.size(), 9u) << Tokens;2805  EXPECT_TOKEN(Tokens[3], tok::lessequal, TT_BinaryOperator);2806  EXPECT_TOKEN_PRECEDENCE(Tokens[3], prec::Relational);2807  Tokens = Annotate("a <= b <= a;");2808  ASSERT_EQ(Tokens.size(), 7u) << Tokens;2809  EXPECT_TOKEN(Tokens[1], tok::lessequal, TT_BinaryOperator);2810  EXPECT_TOKEN_PRECEDENCE(Tokens[1], prec::Assignment);2811  EXPECT_TOKEN(Tokens[3], tok::lessequal, TT_BinaryOperator);2812  EXPECT_TOKEN_PRECEDENCE(Tokens[3], prec::Relational);2813 2814  // Port lists in module instantiation.2815  Tokens = Annotate("module_x instance_1(port_1), instance_2(port_2);");2816  ASSERT_EQ(Tokens.size(), 12u) << Tokens;2817  EXPECT_TOKEN(Tokens[2], tok::l_paren, TT_VerilogInstancePortLParen);2818  EXPECT_TOKEN(Tokens[7], tok::l_paren, TT_VerilogInstancePortLParen);2819  Tokens = Annotate("module_x #(parameter) instance_1(port_1), "2820                    "instance_2(port_2);");2821  ASSERT_EQ(Tokens.size(), 16u) << Tokens;2822  EXPECT_TOKEN(Tokens[2], tok::l_paren, TT_VerilogInstancePortLParen);2823  EXPECT_TOKEN(Tokens[6], tok::l_paren, TT_VerilogInstancePortLParen);2824  EXPECT_TOKEN(Tokens[11], tok::l_paren, TT_VerilogInstancePortLParen);2825 2826  // Condition parentheses.2827  Tokens = Annotate("assert (x);");2828  ASSERT_EQ(Tokens.size(), 6u) << Tokens;2829  EXPECT_TOKEN(Tokens[1], tok::l_paren, TT_ConditionLParen);2830  Tokens = Annotate("assert #0 (x);");2831  ASSERT_EQ(Tokens.size(), 8u) << Tokens;2832  EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_ConditionLParen);2833  Tokens = Annotate("assert final (x);");2834  ASSERT_EQ(Tokens.size(), 7u) << Tokens;2835  EXPECT_TOKEN(Tokens[2], tok::l_paren, TT_ConditionLParen);2836  Tokens = Annotate("foreach (x[x]) continue;");2837  ASSERT_EQ(Tokens.size(), 10u) << Tokens;2838  EXPECT_TOKEN(Tokens[1], tok::l_paren, TT_ConditionLParen);2839  Tokens = Annotate("repeat (x[x]) continue;");2840  ASSERT_EQ(Tokens.size(), 10u) << Tokens;2841  EXPECT_TOKEN(Tokens[1], tok::l_paren, TT_ConditionLParen);2842  Tokens = Annotate("case (x) endcase;");2843  ASSERT_EQ(Tokens.size(), 7u) << Tokens;2844  EXPECT_TOKEN(Tokens[1], tok::l_paren, TT_ConditionLParen);2845 2846  // Sensitivity list. The TT_Unknown type is clearly not binding for the2847  // future, please adapt if those tokens get annotated.  This test is only here2848  // to prevent the comma from being annotated as TT_VerilogInstancePortComma.2849  Tokens = Annotate("always @(posedge x, posedge y);");2850  ASSERT_EQ(Tokens.size(), 11u) << Tokens;2851  EXPECT_TOKEN(Tokens[2], tok::l_paren, TT_Unknown);2852  EXPECT_TOKEN(Tokens[5], tok::comma, TT_Unknown);2853  EXPECT_TOKEN(Tokens[8], tok::r_paren, TT_Unknown);2854 2855  // String literals in concatenation.2856  Tokens = Annotate("x = {\"\"};");2857  ASSERT_EQ(Tokens.size(), 7u) << Tokens;2858  EXPECT_TOKEN(Tokens[3], tok::string_literal, TT_StringInConcatenation);2859  Tokens = Annotate("x = {\"\", \"\"};");2860  ASSERT_EQ(Tokens.size(), 9u) << Tokens;2861  EXPECT_TOKEN(Tokens[3], tok::string_literal, TT_StringInConcatenation);2862  EXPECT_TOKEN(Tokens[5], tok::string_literal, TT_StringInConcatenation);2863  Tokens = Annotate("x = '{{\"\"}};");2864  ASSERT_EQ(Tokens.size(), 10u) << Tokens;2865  EXPECT_TOKEN(Tokens[5], tok::string_literal, TT_StringInConcatenation);2866  // Cases where the string should not be annotated that type.  Fix the2867  // `TT_Unknown` if needed in the future.2868  Tokens = Annotate("x = {\"\" == \"\"};");2869  ASSERT_EQ(Tokens.size(), 9u) << Tokens;2870  EXPECT_TOKEN(Tokens[3], tok::string_literal, TT_Unknown);2871  EXPECT_TOKEN(Tokens[5], tok::string_literal, TT_Unknown);2872  Tokens = Annotate("x = {(\"\")};");2873  ASSERT_EQ(Tokens.size(), 9u) << Tokens;2874  EXPECT_TOKEN(Tokens[4], tok::string_literal, TT_Unknown);2875  Tokens = Annotate("x = '{\"\"};");2876  ASSERT_EQ(Tokens.size(), 8u) << Tokens;2877  EXPECT_TOKEN(Tokens[4], tok::string_literal, TT_Unknown);2878 2879  // Module headers.2880  Tokens = Annotate("module x();\n"2881                    "endmodule");2882  ASSERT_EQ(Tokens.size(), 7u) << Tokens;2883  EXPECT_TOKEN(Tokens[2], tok::l_paren, TT_VerilogMultiLineListLParen);2884  Tokens = Annotate("function automatic `x x();\n"2885                    "endmodule");2886  ASSERT_EQ(Tokens.size(), 10u) << Tokens;2887  EXPECT_TOKEN(Tokens[5], tok::l_paren, TT_VerilogMultiLineListLParen);2888  Tokens = Annotate("function automatic x``x x();\n"2889                    "endmodule");2890  ASSERT_EQ(Tokens.size(), 11u) << Tokens;2891  EXPECT_TOKEN(Tokens[6], tok::l_paren, TT_VerilogMultiLineListLParen);2892  Tokens = Annotate("function automatic x::x x();\n"2893                    "endmodule");2894  ASSERT_EQ(Tokens.size(), 11u) << Tokens;2895  EXPECT_TOKEN(Tokens[6], tok::l_paren, TT_VerilogMultiLineListLParen);2896 2897  // Escaped identifiers.2898  Tokens = Annotate(R"(\busa+index)");2899  ASSERT_EQ(Tokens.size(), 2u) << Tokens;2900  EXPECT_TOKEN(Tokens[0], tok::identifier, TT_Unknown);2901  Tokens = Annotate(R"(\busa+index ;)");2902  ASSERT_EQ(Tokens.size(), 3u) << Tokens;2903  EXPECT_TOKEN(Tokens[0], tok::identifier, TT_Unknown);2904  EXPECT_EQ(Tokens[0]->TokenText, R"(\busa+index)");2905  EXPECT_TOKEN(Tokens[1], tok::semi, TT_Unknown);2906  Tokens = Annotate(R"(\busa+index2907;)");2908  ASSERT_EQ(Tokens.size(), 3u) << Tokens;2909  EXPECT_TOKEN(Tokens[0], tok::identifier, TT_Unknown);2910  EXPECT_TOKEN(Tokens[1], tok::semi, TT_Unknown);2911  // The escaped identifier can be broken by an escaped newline. The result is2912  // still 1 identifier.2913  Tokens = Annotate(R"(\busa+index\2914+2915;)");2916  ASSERT_EQ(Tokens.size(), 3u) << Tokens;2917  EXPECT_TOKEN(Tokens[0], tok::identifier, TT_Unknown);2918  EXPECT_EQ(Tokens[0]->TokenText, R"(\busa+index\2919+)");2920  EXPECT_TOKEN(Tokens[1], tok::semi, TT_Unknown);2921  // An escaped newline should not be treated as an escaped identifier.2922  Tokens = Annotate("\\\n");2923  ASSERT_EQ(Tokens.size(), 1u) << Tokens;2924  EXPECT_TOKEN(Tokens[0], tok::eof, TT_Unknown);2925  // Macros.2926  Tokens = Annotate("`define x x``x");2927  ASSERT_EQ(Tokens.size(), 7u) << Tokens;2928  EXPECT_TOKEN(Tokens[0], tok::hash, TT_Unknown);2929  EXPECT_TOKEN(Tokens[4], tok::hashhash, TT_Unknown);2930}2931 2932TEST_F(TokenAnnotatorTest, UnderstandTableGenTokens) {2933  auto Style = getLLVMStyle(FormatStyle::LK_TableGen);2934  ASSERT_TRUE(Style.isTableGen());2935 2936  TestLexer Lexer(Allocator, Buffers, Style);2937  AdditionalKeywords Keywords(Lexer.IdentTable);2938  auto Annotate = [&Lexer](StringRef Code) { return Lexer.annotate(Code); };2939 2940  // Additional keywords representation test.2941  auto Tokens = Annotate("def foo : Bar<1>;");2942  ASSERT_TRUE(Keywords.isTableGenKeyword(*Tokens[0]));2943  ASSERT_TRUE(Keywords.isTableGenDefinition(*Tokens[0]));2944  ASSERT_TRUE(Tokens[0]->is(Keywords.kw_def));2945  ASSERT_TRUE(Tokens[1]->is(TT_StartOfName));2946 2947  // Code, the multiline string token.2948  Tokens = Annotate("[{ code is multiline string }]");2949  ASSERT_EQ(Tokens.size(), 2u) << Tokens;2950  EXPECT_TOKEN(Tokens[0], tok::string_literal, TT_TableGenMultiLineString);2951  EXPECT_FALSE(Tokens[0]->IsMultiline);2952  // Case with multiple lines.2953  Tokens = Annotate("[{ It can break\n"2954                    "   across lines and the line breaks\n"2955                    "   are retained in \n"2956                    "   the string. }]");2957  ASSERT_EQ(Tokens.size(), 2u) << Tokens;2958  EXPECT_TOKEN(Tokens[0], tok::string_literal, TT_TableGenMultiLineString);2959  EXPECT_EQ(Tokens[0]->ColumnWidth, sizeof("[{ It can break\n") - 1);2960  EXPECT_TRUE(Tokens[0]->IsMultiline);2961  EXPECT_EQ(Tokens[0]->LastLineColumnWidth, sizeof("   the string. }]") - 1);2962 2963  // Numeric literals.2964  Tokens = Annotate("1234");2965  EXPECT_TOKEN(Tokens[0], tok::numeric_constant, TT_Unknown);2966  Tokens = Annotate("-1");2967  EXPECT_TOKEN(Tokens[0], tok::numeric_constant, TT_Unknown);2968  Tokens = Annotate("+1234");2969  EXPECT_TOKEN(Tokens[0], tok::numeric_constant, TT_Unknown);2970  Tokens = Annotate("0b0110");2971  EXPECT_TOKEN(Tokens[0], tok::numeric_constant, TT_Unknown);2972  Tokens = Annotate("0x1abC");2973  EXPECT_TOKEN(Tokens[0], tok::numeric_constant, TT_Unknown);2974 2975  // Identifier tokens. In TableGen, identifiers can begin with a number.2976  // In ambiguous cases, the lexer tries to lex it as a number.2977  // Even if the try fails, it does not fall back to identifier lexing and2978  // regard as an error.2979  // The ambiguity is not documented. The result of those tests are based on the2980  // implementation of llvm::TGLexer::LexToken.2981  // This is invalid syntax of number, but not an identifier.2982  Tokens = Annotate("0x1234x");2983  EXPECT_TOKEN(Tokens[0], tok::numeric_constant, TT_Unknown);2984  Tokens = Annotate("identifier");2985  EXPECT_TOKEN(Tokens[0], tok::identifier, TT_Unknown);2986  // Identifier beginning with a number.2987  Tokens = Annotate("0x");2988  EXPECT_TOKEN(Tokens[0], tok::identifier, TT_Unknown);2989  Tokens = Annotate("2dVector");2990  EXPECT_TOKEN(Tokens[0], tok::identifier, TT_Unknown);2991  Tokens = Annotate("01234Vector");2992  EXPECT_TOKEN(Tokens[0], tok::identifier, TT_Unknown);2993 2994  // Structured statements.2995  Tokens = Annotate("class Foo {}");2996  EXPECT_TOKEN(Tokens[1], tok::identifier, TT_StartOfName);2997  EXPECT_TOKEN(Tokens[2], tok::l_brace, TT_FunctionLBrace);2998  Tokens = Annotate("def Def: Foo {}");2999  EXPECT_TOKEN(Tokens[2], tok::colon, TT_InheritanceColon);3000  EXPECT_TOKEN(Tokens[4], tok::l_brace, TT_FunctionLBrace);3001  Tokens = Annotate("if cond then {} else {}");3002  EXPECT_TOKEN(Tokens[3], tok::l_brace, TT_ControlStatementLBrace);3003  EXPECT_TOKEN(Tokens[6], tok::l_brace, TT_ElseLBrace);3004  Tokens = Annotate("defset Foo Def2 = {}");3005  EXPECT_TOKEN(Tokens[4], tok::l_brace, TT_FunctionLBrace);3006 3007  // Bang Operators.3008  Tokens = Annotate("!foreach");3009  EXPECT_TOKEN(Tokens[0], tok::identifier, TT_TableGenBangOperator);3010  Tokens = Annotate("!if");3011  EXPECT_TOKEN(Tokens[0], tok::identifier, TT_TableGenBangOperator);3012  Tokens = Annotate("!cond");3013  EXPECT_TOKEN(Tokens[0], tok::identifier, TT_TableGenCondOperator);3014 3015  // The paste operator should not be treated as a preprocessor directive even3016  // if it is on a separate line.3017  Tokens = Annotate("def x\n"3018                    "#embed {}");3019  ASSERT_EQ(Tokens.size(), 7u) << Tokens;3020  EXPECT_TOKEN(Tokens[1], tok::identifier, TT_StartOfName);3021  EXPECT_EQ(Tokens[1]->Next, Tokens[2]);3022  Tokens = Annotate("def x\n"3023                    "#define x\n"3024                    "#embed {}");3025  ASSERT_EQ(Tokens.size(), 10u) << Tokens;3026  EXPECT_TOKEN(Tokens[1], tok::identifier, TT_StartOfName);3027  EXPECT_EQ(Tokens[1]->Next, Tokens[5]);3028  Tokens = Annotate("def x\n"3029                    "#ifdef x\n"3030                    "#else\n"3031                    "#endif\n"3032                    "#embed {}");3033  ASSERT_EQ(Tokens.size(), 14u) << Tokens;3034  EXPECT_TOKEN(Tokens[1], tok::identifier, TT_StartOfName);3035  EXPECT_EQ(Tokens[1]->Next, Tokens[9]);3036 3037  auto AnnotateValue = [this, &Style](StringRef Code) {3038    // Values are annotated only in specific context.3039    auto Result = annotate(("def X { let V = " + Code + "; }").str(), Style);3040    return decltype(Result){Result.begin() + 6, Result.end() - 3};3041  };3042  // Both of bang/cond operators.3043  Tokens = AnnotateValue("!cond(!eq(x, 0): 1, true: x)");3044  ASSERT_EQ(Tokens.size(), 15u) << Tokens;3045  EXPECT_TOKEN(Tokens[0], tok::identifier, TT_TableGenCondOperator);3046  EXPECT_TOKEN(Tokens[2], tok::identifier, TT_TableGenBangOperator);3047  EXPECT_TOKEN(Tokens[8], tok::colon, TT_TableGenCondOperatorColon);3048  EXPECT_TOKEN(Tokens[10], tok::comma, TT_TableGenCondOperatorComma);3049  EXPECT_TOKEN(Tokens[12], tok::colon, TT_TableGenCondOperatorColon);3050  // DAGArg values with operator identifier3051  Tokens = AnnotateValue("(ins type1:$src1, type2:$src2)");3052  ASSERT_EQ(Tokens.size(), 10u) << Tokens;3053  EXPECT_TOKEN(Tokens[0], tok::l_paren, TT_TableGenDAGArgOpener);3054  EXPECT_TOKEN(Tokens[3], tok::colon, TT_TableGenDAGArgListColon);3055  EXPECT_TOKEN(Tokens[4], tok::identifier, TT_Unknown); // $src13056  EXPECT_TOKEN(Tokens[5], tok::comma, TT_TableGenDAGArgListComma);3057  EXPECT_TOKEN(Tokens[7], tok::colon, TT_TableGenDAGArgListColon);3058  EXPECT_TOKEN(Tokens[9], tok::r_paren, TT_TableGenDAGArgCloser);3059  // List literal3060  Tokens = AnnotateValue("[1, 2, 3]");3061  ASSERT_EQ(Tokens.size(), 7u) << Tokens;3062  EXPECT_TOKEN(Tokens[0], tok::l_square, TT_TableGenListOpener);3063  EXPECT_TOKEN(Tokens[6], tok::r_square, TT_TableGenListCloser);3064  // Suffixes of values3065  Tokens = AnnotateValue("valid.field");3066  ASSERT_EQ(Tokens.size(), 3u) << Tokens;3067  EXPECT_TOKEN(Tokens[1], tok::period, TT_TableGenValueSuffix);3068  // Code3069  Tokens = AnnotateValue("[{ code is multiline string }]");3070  ASSERT_EQ(Tokens.size(), 1u) << Tokens;3071  EXPECT_TOKEN(Tokens[0], tok::string_literal, TT_TableGenMultiLineString);3072 3073  // The definition3074  Tokens = annotate("def Def : Parent<Child> {}", Style);3075  ASSERT_EQ(Tokens.size(), 10u) << Tokens; // This contains eof.3076  // We use inheritance colon and function brace. They are enough.3077  EXPECT_TOKEN(Tokens[2], tok::colon, TT_InheritanceColon);3078  EXPECT_TOKEN(Tokens[4], tok::less, TT_TemplateOpener);3079  EXPECT_TOKEN(Tokens[6], tok::greater, TT_TemplateCloser);3080  EXPECT_TOKEN(Tokens[7], tok::l_brace, TT_FunctionLBrace);3081 3082  // DAGArg breaking options. They use different token types depending on what3083  // is specified.3084  Style.TableGenBreakInsideDAGArg = FormatStyle::DAS_BreakElements;3085 3086  // When TableGenBreakInsideDAGArg is DAS_BreakElements and3087  // TableGenBreakingDAGArgOperators is not specified, it makes all the DAGArg3088  // elements to have line break.3089  Tokens = AnnotateValue("(ins type1:$src1, type2:$src2)");3090  ASSERT_EQ(Tokens.size(), 10u) << Tokens;3091  EXPECT_TOKEN(Tokens[0], tok::l_paren, TT_TableGenDAGArgOpenerToBreak);3092  EXPECT_TOKEN(Tokens[1], tok::identifier,3093               TT_TableGenDAGArgOperatorID); // ins3094  EXPECT_TOKEN(Tokens[5], tok::comma, TT_TableGenDAGArgListCommaToBreak);3095  EXPECT_TOKEN(Tokens[9], tok::r_paren, TT_TableGenDAGArgCloser);3096 3097  Tokens = AnnotateValue("(other type1:$src1, type2:$src2)");3098  ASSERT_EQ(Tokens.size(), 10u) << Tokens;3099  EXPECT_TOKEN(Tokens[0], tok::l_paren, TT_TableGenDAGArgOpenerToBreak);3100  EXPECT_TOKEN(Tokens[1], tok::identifier,3101               TT_TableGenDAGArgOperatorID); // other3102  EXPECT_TOKEN(Tokens[5], tok::comma, TT_TableGenDAGArgListCommaToBreak);3103  EXPECT_TOKEN(Tokens[9], tok::r_paren, TT_TableGenDAGArgCloser);3104 3105  // For non-identifier operators, breaks after the operator.3106  Tokens = AnnotateValue("(!cast<Type>(\"Name\") type1:$src1, type2:$src2)");3107  ASSERT_EQ(Tokens.size(), 16u) << Tokens;3108  EXPECT_TOKEN(Tokens[0], tok::l_paren, TT_TableGenDAGArgOpenerToBreak);3109  EXPECT_TOKEN(Tokens[7], tok::r_paren, TT_TableGenDAGArgOperatorToBreak);3110  EXPECT_TOKEN(Tokens[11], tok::comma, TT_TableGenDAGArgListCommaToBreak);3111  EXPECT_TOKEN(Tokens[15], tok::r_paren, TT_TableGenDAGArgCloser);3112 3113  Style.TableGenBreakInsideDAGArg = FormatStyle::DAS_BreakAll;3114 3115  // When TableGenBreakInsideDAGArg is DAS_BreakAll and3116  // TableGenBreakingDAGArgOperators is not specified, it makes all the DAGArg3117  // to have line break inside it.3118  Tokens = AnnotateValue("(ins type1:$src1, type2:$src2)");3119  ASSERT_EQ(Tokens.size(), 10u) << Tokens;3120  EXPECT_TOKEN(Tokens[0], tok::l_paren, TT_TableGenDAGArgOpenerToBreak);3121  EXPECT_TOKEN(Tokens[1], tok::identifier,3122               TT_TableGenDAGArgOperatorToBreak); // ins3123  EXPECT_TOKEN(Tokens[5], tok::comma, TT_TableGenDAGArgListCommaToBreak);3124  EXPECT_TOKEN(Tokens[9], tok::r_paren, TT_TableGenDAGArgCloser);3125 3126  Tokens = AnnotateValue("(other type1:$src1, type2:$src2)");3127  ASSERT_EQ(Tokens.size(), 10u) << Tokens;3128  EXPECT_TOKEN(Tokens[0], tok::l_paren, TT_TableGenDAGArgOpenerToBreak);3129  EXPECT_TOKEN(Tokens[1], tok::identifier,3130               TT_TableGenDAGArgOperatorToBreak); // other3131  EXPECT_TOKEN(Tokens[5], tok::comma, TT_TableGenDAGArgListCommaToBreak);3132  EXPECT_TOKEN(Tokens[9], tok::r_paren, TT_TableGenDAGArgCloser);3133 3134  // If TableGenBreakingDAGArgOperators is specified, it is limited to the3135  // specified operators.3136  Style.TableGenBreakingDAGArgOperators = {"ins", "outs"};3137  Tokens = AnnotateValue("(ins type1:$src1, type2:$src2)");3138  ASSERT_EQ(Tokens.size(), 10u) << Tokens;3139  EXPECT_TOKEN(Tokens[0], tok::l_paren, TT_TableGenDAGArgOpenerToBreak);3140  EXPECT_TOKEN(Tokens[1], tok::identifier,3141               TT_TableGenDAGArgOperatorToBreak); // ins3142  EXPECT_TOKEN(Tokens[5], tok::comma, TT_TableGenDAGArgListCommaToBreak);3143  EXPECT_TOKEN(Tokens[9], tok::r_paren, TT_TableGenDAGArgCloser);3144 3145  Tokens = AnnotateValue("(other type1:$src1, type2:$src2)");3146  ASSERT_EQ(Tokens.size(), 10u) << Tokens;3147  EXPECT_TOKEN(Tokens[0], tok::l_paren, TT_TableGenDAGArgOpener);3148  EXPECT_TOKEN(Tokens[1], tok::identifier, TT_Unknown); // other3149  EXPECT_TOKEN(Tokens[5], tok::comma, TT_TableGenDAGArgListComma);3150  EXPECT_TOKEN(Tokens[9], tok::r_paren, TT_TableGenDAGArgCloser);3151 3152  // If TableGenBreakingDAGArgOperators is enabled, it uses3153  // TT_TableGenDAGArgListColonToAlign to annotate the colon to align.3154  Style.AlignConsecutiveTableGenBreakingDAGArgColons.Enabled = true;3155  Tokens = AnnotateValue("(ins type1:$src1, type2:$src2)");3156  ASSERT_EQ(Tokens.size(), 10u) << Tokens;3157  EXPECT_TOKEN(Tokens[1], tok::identifier,3158               TT_TableGenDAGArgOperatorToBreak); // ins3159  EXPECT_TOKEN(Tokens[3], tok::colon, TT_TableGenDAGArgListColonToAlign);3160  EXPECT_TOKEN(Tokens[7], tok::colon, TT_TableGenDAGArgListColonToAlign);3161 3162  Tokens = AnnotateValue("(other type1:$src1, type2:$src2)");3163  ASSERT_EQ(Tokens.size(), 10u) << Tokens;3164  EXPECT_TOKEN(Tokens[1], tok::identifier, TT_Unknown); // other3165  EXPECT_TOKEN(Tokens[3], tok::colon, TT_TableGenDAGArgListColon);3166  EXPECT_TOKEN(Tokens[7], tok::colon, TT_TableGenDAGArgListColon);3167}3168 3169TEST_F(TokenAnnotatorTest, UnderstandConstructors) {3170  auto Tokens = annotate("Class::Class() : BaseClass(), Member() {}");3171 3172  // The TT_Unknown is clearly not binding for the future, please adapt if those3173  // tokens get annotated.3174  ASSERT_EQ(Tokens.size(), 16u) << Tokens;3175  EXPECT_TOKEN(Tokens[5], tok::colon, TT_CtorInitializerColon);3176  EXPECT_TOKEN(Tokens[6], tok::identifier, TT_Unknown);3177  EXPECT_TOKEN(Tokens[7], tok::l_paren, TT_Unknown);3178  EXPECT_TOKEN(Tokens[8], tok::r_paren, TT_Unknown);3179  EXPECT_TOKEN(Tokens[9], tok::comma, TT_CtorInitializerComma);3180  EXPECT_TOKEN(Tokens[10], tok::identifier, TT_Unknown);3181  EXPECT_TOKEN(Tokens[11], tok::l_paren, TT_Unknown);3182  EXPECT_TOKEN(Tokens[12], tok::r_paren, TT_Unknown);3183  EXPECT_TOKEN(Tokens[13], tok::l_brace, TT_FunctionLBrace);3184  EXPECT_BRACE_KIND(Tokens[13], BK_Block);3185 3186  Tokens = annotate("Class::Class() : BaseClass{}, Member{} {}");3187  ASSERT_EQ(Tokens.size(), 16u) << Tokens;3188  EXPECT_TOKEN(Tokens[5], tok::colon, TT_CtorInitializerColon);3189  EXPECT_TOKEN(Tokens[6], tok::identifier, TT_Unknown);3190  EXPECT_TOKEN(Tokens[7], tok::l_brace, TT_Unknown);3191  EXPECT_TOKEN(Tokens[8], tok::r_brace, TT_Unknown);3192  EXPECT_TOKEN(Tokens[9], tok::comma, TT_CtorInitializerComma);3193  EXPECT_TOKEN(Tokens[10], tok::identifier, TT_Unknown);3194  EXPECT_TOKEN(Tokens[11], tok::l_brace, TT_Unknown);3195  EXPECT_TOKEN(Tokens[12], tok::r_brace, TT_Unknown);3196  EXPECT_TOKEN(Tokens[13], tok::l_brace, TT_FunctionLBrace);3197  EXPECT_BRACE_KIND(Tokens[13], BK_Block);3198 3199  Tokens = annotate("class Class {\n"3200                    "  Class() : BaseClass() {\n"3201                    "#if 0\n"3202                    "    // comment\n"3203                    "#endif\n"3204                    "  }\n"3205                    "  Class f();\n"3206                    "}");3207  ASSERT_EQ(Tokens.size(), 25u) << Tokens;3208  EXPECT_TOKEN(Tokens[6], tok::colon, TT_CtorInitializerColon);3209  EXPECT_TOKEN(Tokens[10], tok::l_brace, TT_FunctionLBrace);3210  EXPECT_BRACE_KIND(Tokens[10], BK_Block);3211}3212 3213TEST_F(TokenAnnotatorTest, UnderstandsConditionParens) {3214  auto Tokens = annotate("if (x) {}");3215  ASSERT_EQ(Tokens.size(), 7u) << Tokens;3216  EXPECT_TOKEN(Tokens[1], tok::l_paren, TT_ConditionLParen);3217  Tokens = annotate("if constexpr (x) {}");3218  ASSERT_EQ(Tokens.size(), 8u) << Tokens;3219  EXPECT_TOKEN(Tokens[2], tok::l_paren, TT_ConditionLParen);3220  Tokens = annotate("if CONSTEXPR (x) {}");3221  ASSERT_EQ(Tokens.size(), 8u) << Tokens;3222  EXPECT_TOKEN(Tokens[2], tok::l_paren, TT_ConditionLParen);3223  Tokens = annotate("if (x) {} else if (x) {}");3224  ASSERT_EQ(Tokens.size(), 14u) << Tokens;3225  EXPECT_TOKEN(Tokens[1], tok::l_paren, TT_ConditionLParen);3226  EXPECT_TOKEN(Tokens[8], tok::l_paren, TT_ConditionLParen);3227}3228 3229TEST_F(TokenAnnotatorTest, CSharpNullableTypes) {3230  FormatStyle Style = getGoogleStyle(FormatStyle::LK_CSharp);3231 3232  auto Tokens = annotate("int? a;", Style);3233  ASSERT_EQ(Tokens.size(), 5u) << Tokens;3234  EXPECT_TOKEN(Tokens[1], tok::question, TT_CSharpNullable);3235 3236  Tokens = annotate("int? a = 1;", Style);3237  ASSERT_EQ(Tokens.size(), 7u) << Tokens;3238  EXPECT_TOKEN(Tokens[1], tok::question, TT_CSharpNullable);3239 3240  Tokens = annotate("int?)", Style);3241  ASSERT_EQ(Tokens.size(), 4u) << Tokens;3242  EXPECT_TOKEN(Tokens[1], tok::question, TT_CSharpNullable);3243 3244  Tokens = annotate("int?>", Style);3245  ASSERT_EQ(Tokens.size(), 4u) << Tokens;3246  EXPECT_TOKEN(Tokens[1], tok::question, TT_CSharpNullable);3247 3248  Tokens = annotate("{\n"3249                    "  int? a;\n"3250                    "  if (b is int?)\n"3251                    "    f();\n"3252                    "  var foo = A<Foo?>();\n"3253                    "}",3254                    Style);3255  ASSERT_EQ(Tokens.size(), 29u) << Tokens;3256  EXPECT_TOKEN(Tokens[2], tok::question, TT_CSharpNullable);3257  EXPECT_TOKEN(Tokens[10], tok::question, TT_CSharpNullable);3258  EXPECT_TOKEN(Tokens[20], tok::less, TT_TemplateOpener);3259  EXPECT_TOKEN(Tokens[22], tok::question, TT_CSharpNullable);3260 3261  Tokens = annotate("cond? id : id2", Style);3262  ASSERT_EQ(Tokens.size(), 6u) << Tokens;3263  EXPECT_TOKEN(Tokens[1], tok::question, TT_ConditionalExpr);3264 3265  Tokens = annotate("cond ? cond2 ? : id1 : id2", Style);3266  ASSERT_EQ(Tokens.size(), 9u) << Tokens;3267  EXPECT_TOKEN(Tokens[1], tok::question, TT_ConditionalExpr);3268}3269 3270TEST_F(TokenAnnotatorTest, CSharpGenericTypeConstraint) {3271  auto Tokens = annotate("namespace A {\n"3272                         "  delegate T MyDelegate<T>()\n"3273                         "      where T : new();\n"3274                         "}",3275                         getGoogleStyle(FormatStyle::LK_CSharp));3276  ASSERT_EQ(Tokens.size(), 20u) << Tokens;3277  EXPECT_TOKEN(Tokens[18], tok::r_brace, TT_NamespaceRBrace);3278}3279 3280TEST_F(TokenAnnotatorTest, CSharpNewModifier) {3281  auto Tokens = annotate("new public class NestedC {\n"3282                         "  public int x = 100;\n"3283                         "}",3284                         getGoogleStyle(FormatStyle::LK_CSharp));3285  ASSERT_EQ(Tokens.size(), 13u) << Tokens;3286  EXPECT_TOKEN(Tokens[3], tok::identifier, TT_ClassHeadName);3287  EXPECT_TOKEN(Tokens[4], tok::l_brace, TT_ClassLBrace);3288  EXPECT_TOKEN(Tokens[11], tok::r_brace, TT_ClassRBrace);3289}3290 3291TEST_F(TokenAnnotatorTest, UnderstandsLabels) {3292  auto Tokens = annotate("{ x: break; }");3293  ASSERT_EQ(Tokens.size(), 7u) << Tokens;3294  EXPECT_TOKEN(Tokens[2], tok::colon, TT_GotoLabelColon);3295 3296  Tokens = annotate("{ case x: break; }");3297  ASSERT_EQ(Tokens.size(), 8u) << Tokens;3298  EXPECT_TOKEN(Tokens[3], tok::colon, TT_CaseLabelColon);3299 3300  Tokens = annotate("{ x: { break; } }");3301  ASSERT_EQ(Tokens.size(), 9u) << Tokens;3302  EXPECT_TOKEN(Tokens[2], tok::colon, TT_GotoLabelColon);3303 3304  Tokens = annotate("{ case x: { break; } }");3305  ASSERT_EQ(Tokens.size(), 10u) << Tokens;3306  EXPECT_TOKEN(Tokens[3], tok::colon, TT_CaseLabelColon);3307 3308  Tokens = annotate("#define FOO label:");3309  ASSERT_EQ(Tokens.size(), 6u) << Tokens;3310  EXPECT_TOKEN(Tokens[4], tok::colon, TT_GotoLabelColon);3311 3312  Tokens = annotate("#define FOO \\\n"3313                    "label: \\\n"3314                    "  break;");3315  ASSERT_EQ(Tokens.size(), 8u) << Tokens;3316  EXPECT_TOKEN(Tokens[4], tok::colon, TT_GotoLabelColon);3317}3318 3319TEST_F(TokenAnnotatorTest, UnderstandsNestedBlocks) {3320  // The closing braces are not annotated. It doesn't seem to cause a problem.3321  // So we only test for the opening braces.3322  auto Tokens = annotate("{\n"3323                         "  {\n"3324                         "    { int a = 0; }\n"3325                         "  }\n"3326                         "  {}\n"3327                         "}");3328  ASSERT_EQ(Tokens.size(), 14u) << Tokens;3329  EXPECT_BRACE_KIND(Tokens[0], BK_Block);3330  EXPECT_BRACE_KIND(Tokens[1], BK_Block);3331  EXPECT_BRACE_KIND(Tokens[2], BK_Block);3332  EXPECT_BRACE_KIND(Tokens[10], BK_Block);3333}3334 3335TEST_F(TokenAnnotatorTest, UnderstandDesignatedInitializers) {3336  auto Tokens = annotate("SomeStruct { .a = 1 };");3337  ASSERT_EQ(Tokens.size(), 9u) << Tokens;3338  EXPECT_BRACE_KIND(Tokens[1], BK_BracedInit);3339  EXPECT_TOKEN(Tokens[2], tok::period, TT_DesignatedInitializerPeriod);3340 3341  Tokens = annotate("SomeStruct { .a = 1, .b = 2 };");3342  ASSERT_EQ(Tokens.size(), 14u) << Tokens;3343  EXPECT_BRACE_KIND(Tokens[1], BK_BracedInit);3344  EXPECT_TOKEN(Tokens[2], tok::period, TT_DesignatedInitializerPeriod);3345  EXPECT_TOKEN(Tokens[7], tok::period, TT_DesignatedInitializerPeriod);3346 3347  Tokens = annotate("SomeStruct {\n"3348                    "#ifdef FOO\n"3349                    "  .a = 1,\n"3350                    "#endif\n"3351                    "  .b = 2\n"3352                    "};");3353  ASSERT_EQ(Tokens.size(), 19u) << Tokens;3354  EXPECT_BRACE_KIND(Tokens[1], BK_BracedInit);3355  EXPECT_TOKEN(Tokens[5], tok::period, TT_DesignatedInitializerPeriod);3356  EXPECT_TOKEN(Tokens[12], tok::period, TT_DesignatedInitializerPeriod);3357 3358  Tokens = annotate("SomeStruct {\n"3359                    "#if defined FOO\n"3360                    "  .a = 1,\n"3361                    "#endif\n"3362                    "  .b = 2\n"3363                    "};");3364  ASSERT_EQ(Tokens.size(), 20u) << Tokens;3365  EXPECT_BRACE_KIND(Tokens[1], BK_BracedInit);3366  EXPECT_TOKEN(Tokens[6], tok::period, TT_DesignatedInitializerPeriod);3367  EXPECT_TOKEN(Tokens[13], tok::period, TT_DesignatedInitializerPeriod);3368 3369  Tokens = annotate("Foo foo[] = {[0]{}};");3370  ASSERT_EQ(Tokens.size(), 14u) << Tokens;3371  EXPECT_TOKEN(Tokens[6], tok::l_square, TT_DesignatedInitializerLSquare);3372  EXPECT_BRACE_KIND(Tokens[9], BK_BracedInit);3373 3374  Tokens = annotate("Foo foo[] = {[0] = 1, [1] = 2};");3375  ASSERT_EQ(Tokens.size(), 20u) << Tokens;3376  EXPECT_TOKEN(Tokens[6], tok::l_square, TT_DesignatedInitializerLSquare);3377  EXPECT_TOKEN(Tokens[12], tok::l_square, TT_DesignatedInitializerLSquare);3378}3379 3380TEST_F(TokenAnnotatorTest, UnderstandsJavaScript) {3381  auto Annotate = [this](StringRef Code) {3382    return annotate(Code, getLLVMStyle(FormatStyle::LK_JavaScript));3383  };3384 3385  // Dictionary.3386  auto Tokens = Annotate("var x = {'x' : 1, 'y' : 2};");3387  ASSERT_EQ(Tokens.size(), 14u) << Tokens;3388  EXPECT_TOKEN(Tokens[3], tok::l_brace, TT_DictLiteral);3389  EXPECT_TOKEN(Tokens[4], tok::string_literal, TT_SelectorName);3390  EXPECT_TOKEN(Tokens[5], tok::colon, TT_DictLiteral);3391  EXPECT_TOKEN(Tokens[8], tok::string_literal, TT_SelectorName);3392  EXPECT_TOKEN(Tokens[9], tok::colon, TT_DictLiteral);3393  // Change when we need to annotate these.3394  EXPECT_BRACE_KIND(Tokens[3], BK_Unknown);3395  EXPECT_BRACE_KIND(Tokens[11], BK_Unknown);3396  EXPECT_TOKEN(Tokens[11], tok::r_brace, TT_Unknown);3397}3398 3399TEST_F(TokenAnnotatorTest, UnderstandsAttributes) {3400  auto Tokens = annotate("bool foo __attribute__((unused));");3401  ASSERT_EQ(Tokens.size(), 10u) << Tokens;3402  EXPECT_TOKEN(Tokens[1], tok::identifier, TT_StartOfName);3403  EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_AttributeLParen);3404  EXPECT_TOKEN(Tokens[4], tok::l_paren, TT_Unknown);3405  EXPECT_TOKEN(Tokens[6], tok::r_paren, TT_Unknown);3406  EXPECT_TOKEN(Tokens[7], tok::r_paren, TT_AttributeRParen);3407 3408  Tokens = annotate("bool foo __declspec(dllimport);");3409  ASSERT_EQ(Tokens.size(), 8u) << Tokens;3410  EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_AttributeLParen);3411  EXPECT_TOKEN(Tokens[5], tok::r_paren, TT_AttributeRParen);3412 3413  Tokens = annotate("bool __attribute__((unused)) foo;");3414  ASSERT_EQ(Tokens.size(), 10u) << Tokens;3415  EXPECT_TOKEN(Tokens[2], tok::l_paren, TT_AttributeLParen);3416  EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_Unknown);3417  EXPECT_TOKEN(Tokens[5], tok::r_paren, TT_Unknown);3418  EXPECT_TOKEN(Tokens[6], tok::r_paren, TT_AttributeRParen);3419  EXPECT_TOKEN(Tokens[7], tok::identifier, TT_StartOfName);3420 3421  Tokens = annotate("void __attribute__((x)) Foo();");3422  ASSERT_EQ(Tokens.size(), 12u) << Tokens;3423  EXPECT_TOKEN(Tokens[2], tok::l_paren, TT_AttributeLParen);3424  EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_Unknown);3425  EXPECT_TOKEN(Tokens[5], tok::r_paren, TT_Unknown);3426  EXPECT_TOKEN(Tokens[6], tok::r_paren, TT_AttributeRParen);3427  EXPECT_TOKEN(Tokens[7], tok::identifier, TT_FunctionDeclarationName);3428  EXPECT_TOKEN(Tokens[8], tok::l_paren, TT_FunctionDeclarationLParen);3429 3430  FormatStyle Style = getLLVMStyle();3431  Style.AttributeMacros.push_back("FOO");3432  Tokens = annotate("bool foo FOO(unused);", Style);3433  ASSERT_EQ(Tokens.size(), 8u) << Tokens;3434  EXPECT_TOKEN(Tokens[2], tok::identifier, TT_AttributeMacro);3435  EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_AttributeLParen);3436  EXPECT_TOKEN(Tokens[5], tok::r_paren, TT_AttributeRParen);3437}3438 3439TEST_F(TokenAnnotatorTest, UnderstandsNullabilityAttributeMacros) {3440  // Under Google style, handles the Abseil macro aliases for the Clang3441  // nullability annotations.3442  auto Style = getGoogleStyle(FormatStyle::LK_Cpp);3443  auto Tokens = annotate("x = (foo* absl_nullable)*v;", Style);3444  ASSERT_EQ(Tokens.size(), 11u) << Tokens;3445  EXPECT_TOKEN(Tokens[4], tok::star, TT_PointerOrReference);3446  EXPECT_TOKEN(Tokens[5], tok::identifier, TT_AttributeMacro);3447  EXPECT_TOKEN(Tokens[7], tok::star, TT_UnaryOperator);3448 3449  Tokens = annotate("x = (foo* absl_nonnull)*v;", Style);3450  ASSERT_EQ(Tokens.size(), 11u) << Tokens;3451  EXPECT_TOKEN(Tokens[4], tok::star, TT_PointerOrReference);3452  EXPECT_TOKEN(Tokens[5], tok::identifier, TT_AttributeMacro);3453  EXPECT_TOKEN(Tokens[7], tok::star, TT_UnaryOperator);3454 3455  Tokens = annotate("x = (foo* absl_nullability_unknown)*v;", Style);3456  ASSERT_EQ(Tokens.size(), 11u) << Tokens;3457  EXPECT_TOKEN(Tokens[4], tok::star, TT_PointerOrReference);3458  EXPECT_TOKEN(Tokens[5], tok::identifier, TT_AttributeMacro);3459  EXPECT_TOKEN(Tokens[7], tok::star, TT_UnaryOperator);3460}3461 3462TEST_F(TokenAnnotatorTest, UnderstandsControlStatements) {3463  auto Tokens = annotate("while (true) {}");3464  ASSERT_EQ(Tokens.size(), 7u) << Tokens;3465  EXPECT_TOKEN(Tokens[4], tok::l_brace, TT_ControlStatementLBrace);3466  EXPECT_TOKEN(Tokens[5], tok::r_brace, TT_ControlStatementRBrace);3467 3468  Tokens = annotate("for (;;) {}");3469  ASSERT_EQ(Tokens.size(), 8u) << Tokens;3470  EXPECT_TOKEN(Tokens[5], tok::l_brace, TT_ControlStatementLBrace);3471  EXPECT_TOKEN(Tokens[6], tok::r_brace, TT_ControlStatementRBrace);3472 3473  Tokens = annotate("do {} while (true);");3474  ASSERT_EQ(Tokens.size(), 9u) << Tokens;3475  EXPECT_TOKEN(Tokens[1], tok::l_brace, TT_ControlStatementLBrace);3476  EXPECT_TOKEN(Tokens[2], tok::r_brace, TT_ControlStatementRBrace);3477 3478  Tokens = annotate("if (true) {} else if (false) {} else {}");3479  ASSERT_EQ(Tokens.size(), 17u) << Tokens;3480  EXPECT_TOKEN(Tokens[4], tok::l_brace, TT_ControlStatementLBrace);3481  EXPECT_TOKEN(Tokens[5], tok::r_brace, TT_ControlStatementRBrace);3482  EXPECT_TOKEN(Tokens[11], tok::l_brace, TT_ControlStatementLBrace);3483  EXPECT_TOKEN(Tokens[12], tok::r_brace, TT_ControlStatementRBrace);3484  EXPECT_TOKEN(Tokens[14], tok::l_brace, TT_ElseLBrace);3485  EXPECT_TOKEN(Tokens[15], tok::r_brace, TT_ElseRBrace);3486 3487  Tokens = annotate("switch (foo) {}");3488  ASSERT_EQ(Tokens.size(), 7u) << Tokens;3489  EXPECT_TOKEN(Tokens[4], tok::l_brace, TT_ControlStatementLBrace);3490  EXPECT_TOKEN(Tokens[5], tok::r_brace, TT_ControlStatementRBrace);3491}3492 3493TEST_F(TokenAnnotatorTest, UnderstandsDoWhile) {3494  auto Tokens = annotate("do { ++i; } while ( i > 5 );");3495  ASSERT_EQ(Tokens.size(), 14u) << Tokens;3496  EXPECT_TOKEN(Tokens[6], tok::kw_while, TT_DoWhile);3497 3498  Tokens = annotate("do ++i; while ( i > 5 );");3499  ASSERT_EQ(Tokens.size(), 12u) << Tokens;3500  EXPECT_TOKEN(Tokens[4], tok::kw_while, TT_DoWhile);3501}3502 3503TEST_F(TokenAnnotatorTest, StartOfName) {3504  auto Tokens = annotate("#pragma clang diagnostic push");3505  ASSERT_EQ(Tokens.size(), 6u) << Tokens;3506  EXPECT_TOKEN(Tokens[2], tok::identifier, TT_Unknown);3507  EXPECT_TOKEN(Tokens[3], tok::identifier, TT_Unknown);3508  EXPECT_TOKEN(Tokens[4], tok::identifier, TT_Unknown);3509 3510  Tokens = annotate("#pragma clang diagnostic ignored \"-Wzero-length-array\"");3511  ASSERT_EQ(Tokens.size(), 7u) << Tokens;3512  EXPECT_TOKEN(Tokens[2], tok::identifier, TT_Unknown);3513  EXPECT_TOKEN(Tokens[3], tok::identifier, TT_Unknown);3514  EXPECT_TOKEN(Tokens[4], tok::identifier, TT_Unknown);3515 3516  Tokens = annotate("#define FOO Foo foo");3517  ASSERT_EQ(Tokens.size(), 6u) << Tokens;3518  EXPECT_TOKEN(Tokens[2], tok::identifier, TT_Unknown);3519  EXPECT_TOKEN(Tokens[3], tok::identifier, TT_Unknown);3520  EXPECT_TOKEN(Tokens[4], tok::identifier, TT_StartOfName);3521 3522  Tokens = annotate("@interface NSCoder (TestCoder)");3523  ASSERT_EQ(Tokens.size(), 7u) << Tokens;3524  EXPECT_TOKEN(Tokens[0], tok::at, TT_ObjCDecl);3525  EXPECT_TOKEN(Tokens[2], tok::identifier, TT_StartOfName);3526 3527  Tokens = annotate("class FOO BAR C {};");3528  ASSERT_EQ(Tokens.size(), 8u) << Tokens;3529  EXPECT_TOKEN(Tokens[2], tok::identifier, TT_Unknown); // Not StartOfName3530 3531  Tokens = annotate("int* ::foo::bar;");3532  ASSERT_EQ(Tokens.size(), 8u) << Tokens;3533  EXPECT_TOKEN(Tokens[3], tok::identifier, TT_StartOfName);3534 3535  auto Style = getLLVMStyle();3536  Style.StatementAttributeLikeMacros.push_back("emit");3537  Tokens = annotate("emit foo = 0;", Style);3538  ASSERT_EQ(Tokens.size(), 6u) << Tokens;3539  EXPECT_TOKEN(Tokens[0], tok::identifier, TT_StatementAttributeLikeMacro);3540  EXPECT_TOKEN(Tokens[1], tok::identifier, TT_Unknown);3541}3542 3543TEST_F(TokenAnnotatorTest, BraceKind) {3544  auto Tokens = annotate("void f() {};");3545  ASSERT_EQ(Tokens.size(), 8u) << Tokens;3546  EXPECT_TOKEN(Tokens[1], tok::identifier, TT_FunctionDeclarationName);3547  EXPECT_TOKEN(Tokens[2], tok::l_paren, TT_FunctionDeclarationLParen);3548  EXPECT_TOKEN(Tokens[4], tok::l_brace, TT_FunctionLBrace);3549  EXPECT_BRACE_KIND(Tokens[4], BK_Block);3550  EXPECT_BRACE_KIND(Tokens[5], BK_Block);3551 3552  Tokens = annotate("class Foo<int> f() {}");3553  ASSERT_EQ(Tokens.size(), 11u) << Tokens;3554  EXPECT_TOKEN(Tokens[1], tok::identifier, TT_Unknown);3555  EXPECT_TOKEN(Tokens[5], tok::identifier, TT_FunctionDeclarationName);3556  EXPECT_TOKEN(Tokens[6], tok::l_paren, TT_FunctionDeclarationLParen);3557  EXPECT_TOKEN(Tokens[8], tok::l_brace, TT_FunctionLBrace);3558  EXPECT_BRACE_KIND(Tokens[8], BK_Block);3559  EXPECT_BRACE_KIND(Tokens[9], BK_Block);3560 3561  Tokens = annotate("template <typename T> class Foo<T> f() {}");3562  ASSERT_EQ(Tokens.size(), 16u) << Tokens;3563  EXPECT_TOKEN(Tokens[6], tok::identifier, TT_Unknown);3564  EXPECT_TOKEN(Tokens[10], tok::identifier, TT_FunctionDeclarationName);3565  EXPECT_TOKEN(Tokens[11], tok::l_paren, TT_FunctionDeclarationLParen);3566  EXPECT_TOKEN(Tokens[13], tok::l_brace, TT_FunctionLBrace);3567  EXPECT_BRACE_KIND(Tokens[13], BK_Block);3568  EXPECT_BRACE_KIND(Tokens[14], BK_Block);3569 3570  Tokens = annotate("void f() override {};");3571  ASSERT_EQ(Tokens.size(), 9u) << Tokens;3572  EXPECT_TOKEN(Tokens[1], tok::identifier, TT_FunctionDeclarationName);3573  EXPECT_TOKEN(Tokens[2], tok::l_paren, TT_FunctionDeclarationLParen);3574  EXPECT_TOKEN(Tokens[5], tok::l_brace, TT_FunctionLBrace);3575  EXPECT_BRACE_KIND(Tokens[5], BK_Block);3576  EXPECT_BRACE_KIND(Tokens[6], BK_Block);3577 3578  Tokens = annotate("void f() noexcept(false) {};");3579  ASSERT_EQ(Tokens.size(), 12u) << Tokens;3580  EXPECT_TOKEN(Tokens[1], tok::identifier, TT_FunctionDeclarationName);3581  EXPECT_TOKEN(Tokens[2], tok::l_paren, TT_FunctionDeclarationLParen);3582  EXPECT_TOKEN(Tokens[8], tok::l_brace, TT_FunctionLBrace);3583  EXPECT_BRACE_KIND(Tokens[8], BK_Block);3584  EXPECT_BRACE_KIND(Tokens[9], BK_Block);3585 3586  Tokens = annotate("auto f() -> void {};");3587  ASSERT_EQ(Tokens.size(), 10u) << Tokens;3588  EXPECT_TOKEN(Tokens[1], tok::identifier, TT_FunctionDeclarationName);3589  EXPECT_TOKEN(Tokens[2], tok::l_paren, TT_FunctionDeclarationLParen);3590  EXPECT_TOKEN(Tokens[6], tok::l_brace, TT_FunctionLBrace);3591  EXPECT_BRACE_KIND(Tokens[6], BK_Block);3592  EXPECT_BRACE_KIND(Tokens[7], BK_Block);3593 3594  Tokens = annotate("void f() { /**/ };");3595  ASSERT_EQ(Tokens.size(), 9u) << Tokens;3596  EXPECT_TOKEN(Tokens[1], tok::identifier, TT_FunctionDeclarationName);3597  EXPECT_TOKEN(Tokens[2], tok::l_paren, TT_FunctionDeclarationLParen);3598  EXPECT_TOKEN(Tokens[4], tok::l_brace, TT_FunctionLBrace);3599  EXPECT_BRACE_KIND(Tokens[4], BK_Block);3600  EXPECT_BRACE_KIND(Tokens[6], BK_Block);3601 3602  Tokens = annotate("void f() { //\n"3603                    "};");3604  ASSERT_EQ(Tokens.size(), 9u) << Tokens;3605  EXPECT_TOKEN(Tokens[1], tok::identifier, TT_FunctionDeclarationName);3606  EXPECT_TOKEN(Tokens[2], tok::l_paren, TT_FunctionDeclarationLParen);3607  EXPECT_TOKEN(Tokens[4], tok::l_brace, TT_FunctionLBrace);3608  EXPECT_BRACE_KIND(Tokens[4], BK_Block);3609  EXPECT_BRACE_KIND(Tokens[6], BK_Block);3610 3611  Tokens = annotate("void f() {\n"3612                    "  //\n"3613                    "};");3614  ASSERT_EQ(Tokens.size(), 9u) << Tokens;3615  EXPECT_TOKEN(Tokens[1], tok::identifier, TT_FunctionDeclarationName);3616  EXPECT_TOKEN(Tokens[2], tok::l_paren, TT_FunctionDeclarationLParen);3617  EXPECT_TOKEN(Tokens[4], tok::l_brace, TT_FunctionLBrace);3618  EXPECT_BRACE_KIND(Tokens[4], BK_Block);3619  EXPECT_BRACE_KIND(Tokens[6], BK_Block);3620 3621  Tokens = annotate("struct Foo {\n"3622                    "  Foo() {};\n"3623                    "  ~Foo() {};\n"3624                    "};");3625  ASSERT_EQ(Tokens.size(), 19u) << Tokens;3626  EXPECT_TOKEN(Tokens[3], tok::identifier, TT_CtorDtorDeclName);3627  EXPECT_TOKEN(Tokens[4], tok::l_paren, TT_FunctionDeclarationLParen);3628  EXPECT_TOKEN(Tokens[6], tok::l_brace, TT_FunctionLBrace);3629  EXPECT_BRACE_KIND(Tokens[6], BK_Block);3630  EXPECT_BRACE_KIND(Tokens[7], BK_Block);3631  EXPECT_TOKEN(Tokens[10], tok::identifier, TT_CtorDtorDeclName);3632  EXPECT_TOKEN(Tokens[11], tok::l_paren, TT_FunctionDeclarationLParen);3633  EXPECT_TOKEN(Tokens[13], tok::l_brace, TT_FunctionLBrace);3634  EXPECT_BRACE_KIND(Tokens[13], BK_Block);3635  EXPECT_BRACE_KIND(Tokens[14], BK_Block);3636 3637  Tokens = annotate("{\n"3638                    "  char *a[] = {\n"3639                    "      /* abc */ \"abc\",\n"3640                    "#if FOO\n"3641                    "      /* xyz */ \"xyz\",\n"3642                    "#endif\n"3643                    "      /* last */ \"last\"};\n"3644                    "}");3645  ASSERT_EQ(Tokens.size(), 25u) << Tokens;3646  EXPECT_BRACE_KIND(Tokens[0], BK_Block);3647  EXPECT_BRACE_KIND(Tokens[7], BK_BracedInit);3648  EXPECT_BRACE_KIND(Tokens[21], BK_BracedInit);3649 3650  Tokens =3651      annotate("#define SCOP_STAT(NAME, DESC) \\\n"3652               "  {\"polly\", #NAME, \"Number of rejected regions: \" DESC}");3653  ASSERT_EQ(Tokens.size(), 18u) << Tokens;3654  EXPECT_BRACE_KIND(Tokens[8], BK_BracedInit);3655  EXPECT_BRACE_KIND(Tokens[16], BK_BracedInit);3656 3657  Tokens = annotate("struct {};");3658  ASSERT_EQ(Tokens.size(), 5u) << Tokens;3659  EXPECT_BRACE_KIND(Tokens[1], BK_Block);3660  EXPECT_BRACE_KIND(Tokens[2], BK_Block);3661 3662  Tokens = annotate("struct : Base {};");3663  ASSERT_EQ(Tokens.size(), 7u) << Tokens;3664  EXPECT_BRACE_KIND(Tokens[3], BK_Block);3665  EXPECT_BRACE_KIND(Tokens[4], BK_Block);3666 3667  Tokens = annotate("struct Foo {};");3668  ASSERT_EQ(Tokens.size(), 6u) << Tokens;3669  EXPECT_BRACE_KIND(Tokens[2], BK_Block);3670  EXPECT_BRACE_KIND(Tokens[3], BK_Block);3671 3672  Tokens = annotate("struct ::Foo {};");3673  ASSERT_EQ(Tokens.size(), 7u) << Tokens;3674  EXPECT_TOKEN(Tokens[2], tok::identifier, TT_ClassHeadName);3675  EXPECT_BRACE_KIND(Tokens[3], BK_Block);3676  EXPECT_BRACE_KIND(Tokens[4], BK_Block);3677 3678  Tokens = annotate("struct NS::Foo {};");3679  ASSERT_EQ(Tokens.size(), 8u) << Tokens;3680  EXPECT_TOKEN(Tokens[1], tok::identifier, TT_Unknown);3681  EXPECT_TOKEN(Tokens[3], tok::identifier, TT_ClassHeadName);3682  EXPECT_BRACE_KIND(Tokens[4], BK_Block);3683  EXPECT_BRACE_KIND(Tokens[5], BK_Block);3684 3685  Tokens = annotate("struct Foo<int> {};");3686  ASSERT_EQ(Tokens.size(), 9u) << Tokens;3687  EXPECT_TOKEN(Tokens[1], tok::identifier, TT_ClassHeadName);3688  EXPECT_BRACE_KIND(Tokens[5], BK_Block);3689  EXPECT_BRACE_KIND(Tokens[6], BK_Block);3690 3691  Tokens = annotate("struct Foo<int>::Bar {};");3692  ASSERT_EQ(Tokens.size(), 11u) << Tokens;3693  EXPECT_TOKEN(Tokens[1], tok::identifier, TT_Unknown);3694  EXPECT_TOKEN(Tokens[6], tok::identifier, TT_ClassHeadName);3695  EXPECT_BRACE_KIND(Tokens[7], BK_Block);3696  EXPECT_BRACE_KIND(Tokens[8], BK_Block);3697 3698  Tokens = annotate("struct Foo<int> : Base {};");3699  ASSERT_EQ(Tokens.size(), 11u) << Tokens;3700  EXPECT_TOKEN(Tokens[1], tok::identifier, TT_ClassHeadName);3701  EXPECT_BRACE_KIND(Tokens[7], BK_Block);3702  EXPECT_BRACE_KIND(Tokens[8], BK_Block);3703 3704  Tokens = annotate("struct Foo<int> : Base::Bar {};");3705  ASSERT_EQ(Tokens.size(), 13u) << Tokens;3706  EXPECT_TOKEN(Tokens[1], tok::identifier, TT_ClassHeadName);3707  EXPECT_TOKEN(Tokens[8], tok::identifier, TT_Unknown); // Not TT_ClassHeadName.3708  EXPECT_BRACE_KIND(Tokens[9], BK_Block);3709  EXPECT_BRACE_KIND(Tokens[10], BK_Block);3710 3711  Tokens = annotate("struct Foo final {};");3712  ASSERT_EQ(Tokens.size(), 7u) << Tokens;3713  EXPECT_TOKEN(Tokens[1], tok::identifier, TT_ClassHeadName);3714  EXPECT_BRACE_KIND(Tokens[3], BK_Block);3715  EXPECT_BRACE_KIND(Tokens[4], BK_Block);3716 3717  Tokens = annotate("struct [[foo]] [[bar]] Foo final : Base1, Base2 {};");3718  ASSERT_EQ(Tokens.size(), 21u) << Tokens;3719  EXPECT_TOKEN(Tokens[11], tok::identifier, TT_ClassHeadName);3720  EXPECT_BRACE_KIND(Tokens[17], BK_Block);3721  EXPECT_BRACE_KIND(Tokens[18], BK_Block);3722 3723  Tokens = annotate("struct Foo x{};");3724  ASSERT_EQ(Tokens.size(), 7u) << Tokens;3725  EXPECT_BRACE_KIND(Tokens[3], BK_BracedInit);3726  EXPECT_BRACE_KIND(Tokens[4], BK_BracedInit);3727 3728  Tokens = annotate("struct ::Foo x{};");3729  ASSERT_EQ(Tokens.size(), 8u) << Tokens;3730  EXPECT_BRACE_KIND(Tokens[4], BK_BracedInit);3731  EXPECT_BRACE_KIND(Tokens[5], BK_BracedInit);3732 3733  Tokens = annotate("struct NS::Foo x{};");3734  ASSERT_EQ(Tokens.size(), 9u) << Tokens;3735  EXPECT_BRACE_KIND(Tokens[5], BK_BracedInit);3736  EXPECT_BRACE_KIND(Tokens[6], BK_BracedInit);3737 3738  Tokens = annotate("struct Foo<int> x{};");3739  ASSERT_EQ(Tokens.size(), 10u) << Tokens;3740  EXPECT_BRACE_KIND(Tokens[6], BK_BracedInit);3741  EXPECT_BRACE_KIND(Tokens[7], BK_BracedInit);3742 3743  Tokens = annotate("#ifdef DEBUG_ENABLED\n"3744                    "#else\n"3745                    "#endif\n"3746                    "class RenderingServer : Object {\n"3747                    "#ifndef DISABLE_DEPRECATED\n"3748                    "  enum Features {\n"3749                    "    FEATURE_SHADERS,\n"3750                    "    FEATURE_MULTITHREADED,\n"3751                    "  };\n"3752                    "#endif\n"3753                    "};");3754  ASSERT_EQ(Tokens.size(), 29u) << Tokens;3755  EXPECT_TOKEN(Tokens[8], tok::identifier, TT_ClassHeadName);3756  EXPECT_BRACE_KIND(Tokens[11], BK_Block);3757  EXPECT_BRACE_KIND(Tokens[17], BK_Block);3758  EXPECT_BRACE_KIND(Tokens[22], BK_Block);3759  EXPECT_BRACE_KIND(Tokens[26], BK_Block);3760 3761  Tokens = annotate("{\n"3762                    "#define M(x) \\\n"3763                    "  return {#x};\n"3764                    "}");3765  ASSERT_EQ(Tokens.size(), 15u) << Tokens;3766  EXPECT_TOKEN(Tokens[0], tok::l_brace, TT_BlockLBrace);3767  EXPECT_BRACE_KIND(Tokens[0], BK_Block);3768  EXPECT_BRACE_KIND(Tokens[8], BK_BracedInit);3769  EXPECT_BRACE_KIND(Tokens[11], BK_BracedInit);3770  EXPECT_BRACE_KIND(Tokens[13], BK_Block);3771 3772  Tokens = annotate("{\n"3773                    "  {\n"3774                    "#define GEN_ID(_x) char *_x{#_x}\n"3775                    "    GEN_ID(one);\n"3776                    "  }\n"3777                    "}");3778  ASSERT_EQ(Tokens.size(), 23u) << Tokens;3779  EXPECT_TOKEN(Tokens[0], tok::l_brace, TT_BlockLBrace);3780  EXPECT_BRACE_KIND(Tokens[0], BK_Block);3781  EXPECT_TOKEN(Tokens[1], tok::l_brace, TT_BlockLBrace);3782  EXPECT_BRACE_KIND(Tokens[1], BK_Block);3783  EXPECT_BRACE_KIND(Tokens[11], BK_BracedInit);3784  EXPECT_BRACE_KIND(Tokens[14], BK_BracedInit);3785  EXPECT_BRACE_KIND(Tokens[20], BK_Block);3786  EXPECT_BRACE_KIND(Tokens[21], BK_Block);3787 3788  Tokens = annotate("{\n"3789                    "#define FOO \\\n"3790                    "  { \\\n"3791                    "    case bar: { \\\n"3792                    "      break; \\\n"3793                    "    } \\\n"3794                    "  }\n"3795                    "}");3796  ASSERT_EQ(Tokens.size(), 15u) << Tokens;3797  EXPECT_TOKEN(Tokens[4], tok::l_brace, TT_BlockLBrace);3798  EXPECT_BRACE_KIND(Tokens[4], BK_Block);3799  EXPECT_TOKEN(Tokens[7], tok::colon, TT_CaseLabelColon);3800  EXPECT_BRACE_KIND(Tokens[8], BK_Block);3801  EXPECT_BRACE_KIND(Tokens[11], BK_Block);3802  EXPECT_BRACE_KIND(Tokens[12], BK_Block);3803 3804  Tokens = annotate("class foo {\n"3805                    "  foo() {}\n"3806                    "#if defined(_MSC_VER__clang____GNUC__FOO_) || \\\n"3807                    "    (defined(__GNUC__) && defined(FOO))\n"3808                    "  foo() {}\n"3809                    "#endif\n"3810                    "};");3811  ASSERT_EQ(Tokens.size(), 36u) << Tokens;3812  EXPECT_TOKEN(Tokens[6], tok::l_brace, TT_FunctionLBrace);3813  EXPECT_BRACE_KIND(Tokens[7], BK_Block);3814  EXPECT_TOKEN(Tokens[26], tok::identifier, TT_CtorDtorDeclName);3815  EXPECT_TOKEN(Tokens[27], tok::l_paren, TT_FunctionDeclarationLParen);3816 3817  Tokens = annotate("a = class extends goog.a {};",3818                    getGoogleStyle(FormatStyle::LK_JavaScript));3819  ASSERT_EQ(Tokens.size(), 11u) << Tokens;3820  EXPECT_TOKEN(Tokens[3], tok::identifier, TT_Unknown);3821  EXPECT_TOKEN(Tokens[7], tok::l_brace, TT_ClassLBrace);3822  EXPECT_BRACE_KIND(Tokens[7], BK_Block);3823  EXPECT_TOKEN(Tokens[8], tok::r_brace, TT_ClassRBrace);3824  EXPECT_BRACE_KIND(Tokens[8], BK_Block);3825 3826  Tokens = annotate("a = class Foo extends goog.a {};",3827                    getGoogleStyle(FormatStyle::LK_JavaScript));3828  ASSERT_EQ(Tokens.size(), 12u) << Tokens;3829  EXPECT_TOKEN(Tokens[3], tok::identifier, TT_ClassHeadName);3830  EXPECT_TOKEN(Tokens[4], tok::identifier, TT_Unknown); // Not TT_StartOfName3831  EXPECT_TOKEN(Tokens[8], tok::l_brace, TT_ClassLBrace);3832  EXPECT_BRACE_KIND(Tokens[8], BK_Block);3833  EXPECT_TOKEN(Tokens[9], tok::r_brace, TT_ClassRBrace);3834  EXPECT_BRACE_KIND(Tokens[9], BK_Block);3835 3836  Tokens = annotate("#define FOO(X) \\\n"3837                    "  struct X##_tag_ {};");3838  ASSERT_EQ(Tokens.size(), 14u) << Tokens;3839  EXPECT_TOKEN(Tokens[7], tok::identifier, TT_Unknown);3840  EXPECT_TOKEN(Tokens[9], tok::identifier, TT_ClassHeadName);3841  EXPECT_TOKEN(Tokens[10], tok::l_brace, TT_StructLBrace);3842  EXPECT_BRACE_KIND(Tokens[10], BK_Block);3843  EXPECT_TOKEN(Tokens[11], tok::r_brace, TT_StructRBrace);3844  EXPECT_BRACE_KIND(Tokens[11], BK_Block);3845 3846  Tokens = annotate("#define MACRO            \\\n"3847                    "  struct hash<type> {    \\\n"3848                    "    void f() { return; } \\\n"3849                    "  };");3850  ASSERT_EQ(Tokens.size(), 20u) << Tokens;3851  EXPECT_TOKEN(Tokens[4], tok::identifier, TT_ClassHeadName);3852  EXPECT_TOKEN(Tokens[8], tok::l_brace, TT_StructLBrace);3853  EXPECT_BRACE_KIND(Tokens[8], BK_Block);3854  EXPECT_TOKEN(Tokens[10], tok::identifier, TT_FunctionDeclarationName);3855  EXPECT_TOKEN(Tokens[11], tok::l_paren, TT_FunctionDeclarationLParen);3856  EXPECT_TOKEN(Tokens[13], tok::l_brace, TT_FunctionLBrace);3857  EXPECT_BRACE_KIND(Tokens[13], BK_Block);3858  EXPECT_BRACE_KIND(Tokens[16], BK_Block);3859  EXPECT_TOKEN(Tokens[17], tok::r_brace, TT_StructRBrace);3860  EXPECT_BRACE_KIND(Tokens[17], BK_Block);3861 3862  Tokens = annotate("#define MEMBER(NAME) NAME{\"\"}");3863  ASSERT_EQ(Tokens.size(), 11u) << Tokens;3864  EXPECT_BRACE_KIND(Tokens[7], BK_BracedInit);3865  EXPECT_BRACE_KIND(Tokens[9], BK_BracedInit);3866 3867  Tokens = annotate("return lhs ^ Byte{rhs};");3868  ASSERT_EQ(Tokens.size(), 9u) << Tokens;3869  EXPECT_BRACE_KIND(Tokens[4], BK_BracedInit);3870  EXPECT_BRACE_KIND(Tokens[6], BK_BracedInit);3871 3872  Tokens = annotate("auto f1{&T::operator()};");3873  ASSERT_EQ(Tokens.size(), 12u) << Tokens;3874  EXPECT_BRACE_KIND(Tokens[2], BK_BracedInit);3875  // Not TT_FunctionDeclarationName.3876  EXPECT_TOKEN(Tokens[6], tok::kw_operator, TT_Unknown);3877  EXPECT_BRACE_KIND(Tokens[9], BK_BracedInit);3878}3879 3880TEST_F(TokenAnnotatorTest, UnderstandsElaboratedTypeSpecifier) {3881  auto Tokens = annotate("auto foo() -> enum En {}");3882  ASSERT_EQ(Tokens.size(), 10u) << Tokens;3883  EXPECT_TOKEN(Tokens[7], tok::l_brace, TT_FunctionLBrace);3884}3885 3886TEST_F(TokenAnnotatorTest, BlockLBrace) {3887  auto Tokens = annotate("{\n"3888                         "  {\n"3889                         "    foo();\n"3890                         "  }\n"3891                         "}");3892  ASSERT_EQ(Tokens.size(), 9u) << Tokens;3893  EXPECT_TOKEN(Tokens[0], tok::l_brace, TT_BlockLBrace);3894  EXPECT_BRACE_KIND(Tokens[0], BK_Block);3895  EXPECT_TOKEN(Tokens[1], tok::l_brace, TT_BlockLBrace);3896  EXPECT_BRACE_KIND(Tokens[1], BK_Block);3897 3898  Tokens = annotate("void bar() {\n"3899                    "  {\n"3900                    "    foo();\n"3901                    "  }\n"3902                    "}");3903  ASSERT_EQ(Tokens.size(), 13u) << Tokens;3904  EXPECT_TOKEN(Tokens[4], tok::l_brace, TT_FunctionLBrace);3905  EXPECT_BRACE_KIND(Tokens[4], BK_Block);3906  EXPECT_TOKEN(Tokens[5], tok::l_brace, TT_BlockLBrace);3907  EXPECT_BRACE_KIND(Tokens[5], BK_Block);3908 3909  Tokens = annotate("[foo bar:{{0, 1}} baz:baz];",3910                    getLLVMStyle(FormatStyle::LK_ObjC));3911  ASSERT_EQ(Tokens.size(), 17u) << Tokens;3912  EXPECT_TOKEN(Tokens[4], tok::l_brace, TT_Unknown); // Not TT_BlockLBrace.3913  EXPECT_BRACE_KIND(Tokens[4], BK_Unknown);          // Not BK_Block.3914  EXPECT_BRACE_KIND(Tokens[5], BK_BracedInit);3915  EXPECT_BRACE_KIND(Tokens[9], BK_Unknown);  // Not BK_Block.3916  EXPECT_BRACE_KIND(Tokens[10], BK_Unknown); // Not BK_Block.3917}3918 3919TEST_F(TokenAnnotatorTest, SwitchExpression) {3920  auto Style = getLLVMStyle(FormatStyle::LK_Java);3921  auto Tokens = annotate("i = switch (day) {\n"3922                         "  case THURSDAY, SATURDAY -> 8;\n"3923                         "  case WEDNESDAY -> 9;\n"3924                         "  default -> 1;\n"3925                         "};",3926                         Style);3927  ASSERT_EQ(Tokens.size(), 26u) << Tokens;3928  EXPECT_TOKEN(Tokens[6], tok::l_brace, TT_SwitchExpressionLBrace);3929  EXPECT_TOKEN(Tokens[7], tok::kw_case, TT_SwitchExpressionLabel);3930  EXPECT_TOKEN(Tokens[11], tok::arrow, TT_CaseLabelArrow);3931  EXPECT_TOKEN(Tokens[14], tok::kw_case, TT_SwitchExpressionLabel);3932  EXPECT_TOKEN(Tokens[16], tok::arrow, TT_CaseLabelArrow);3933  EXPECT_TOKEN(Tokens[19], tok::kw_default, TT_SwitchExpressionLabel);3934  EXPECT_TOKEN(Tokens[20], tok::arrow, TT_CaseLabelArrow);3935}3936 3937TEST_F(TokenAnnotatorTest, JavaRecord) {3938  auto Tokens = annotate("public record MyRecord() {}",3939                         getLLVMStyle(FormatStyle::LK_Java));3940  ASSERT_EQ(Tokens.size(), 8u) << Tokens;3941  EXPECT_TOKEN(Tokens[2], tok::identifier, TT_ClassHeadName);3942  // Not TT_FunctionDeclarationLParen.3943  EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_Unknown);3944  EXPECT_TOKEN(Tokens[5], tok::l_brace, TT_RecordLBrace);3945  EXPECT_TOKEN(Tokens[6], tok::r_brace, TT_RecordRBrace);3946}3947 3948TEST_F(TokenAnnotatorTest, CppAltOperatorKeywords) {3949  auto Tokens = annotate("a = b and c;");3950  ASSERT_EQ(Tokens.size(), 7u) << Tokens;3951  EXPECT_TOKEN(Tokens[3], tok::ampamp, TT_BinaryOperator);3952 3953  Tokens = annotate("a = b and_eq c;");3954  ASSERT_EQ(Tokens.size(), 7u) << Tokens;3955  EXPECT_TOKEN(Tokens[3], tok::ampequal, TT_BinaryOperator);3956 3957  Tokens = annotate("a = b bitand c;");3958  ASSERT_EQ(Tokens.size(), 7u) << Tokens;3959  EXPECT_TOKEN(Tokens[3], tok::amp, TT_BinaryOperator);3960 3961  Tokens = annotate("a = b bitor c;");3962  ASSERT_EQ(Tokens.size(), 7u) << Tokens;3963  EXPECT_TOKEN(Tokens[3], tok::pipe, TT_BinaryOperator);3964 3965  Tokens = annotate("a = b compl c;");3966  ASSERT_EQ(Tokens.size(), 7u) << Tokens;3967  EXPECT_TOKEN(Tokens[3], tok::tilde, TT_UnaryOperator);3968 3969  Tokens = annotate("a = b not c;");3970  ASSERT_EQ(Tokens.size(), 7u) << Tokens;3971  EXPECT_TOKEN(Tokens[3], tok::exclaim, TT_UnaryOperator);3972 3973  Tokens = annotate("a = b not_eq c;");3974  ASSERT_EQ(Tokens.size(), 7u) << Tokens;3975  EXPECT_TOKEN(Tokens[3], tok::exclaimequal, TT_BinaryOperator);3976 3977  Tokens = annotate("a = b or c;");3978  ASSERT_EQ(Tokens.size(), 7u) << Tokens;3979  EXPECT_TOKEN(Tokens[3], tok::pipepipe, TT_BinaryOperator);3980 3981  Tokens = annotate("return segment < *this or *this < segment;");3982  ASSERT_EQ(Tokens.size(), 12u) << Tokens;3983  EXPECT_TOKEN(Tokens[5], tok::pipepipe, TT_BinaryOperator);3984  EXPECT_TOKEN(Tokens[6], tok::star, TT_UnaryOperator);3985 3986  Tokens = annotate("a = b or_eq c;");3987  ASSERT_EQ(Tokens.size(), 7u) << Tokens;3988  EXPECT_TOKEN(Tokens[3], tok::pipeequal, TT_BinaryOperator);3989 3990  Tokens = annotate("a = b xor c;");3991  ASSERT_EQ(Tokens.size(), 7u) << Tokens;3992  EXPECT_TOKEN(Tokens[3], tok::caret, TT_BinaryOperator);3993 3994  Tokens = annotate("a = b xor_eq c;");3995  ASSERT_EQ(Tokens.size(), 7u) << Tokens;3996  EXPECT_TOKEN(Tokens[3], tok::caretequal, TT_BinaryOperator);3997 3998  const auto StyleC = getLLVMStyle(FormatStyle::LK_C);3999 4000  Tokens = annotate("xor = foo;", StyleC);4001  ASSERT_EQ(Tokens.size(), 5u) << Tokens;4002  EXPECT_TOKEN(Tokens[0], tok::identifier, TT_Unknown);4003 4004  Tokens = annotate("int xor = foo;", StyleC);4005  ASSERT_EQ(Tokens.size(), 6u) << Tokens;4006  EXPECT_TOKEN(Tokens[1], tok::identifier, TT_StartOfName);4007}4008 4009TEST_F(TokenAnnotatorTest, CppOnlyKeywordInC) {4010  auto Tokens = annotate("int maximized = new & STATE_MAXIMIZED;",4011                         getLLVMStyle(FormatStyle::LK_C));4012  ASSERT_EQ(Tokens.size(), 8u) << Tokens;4013  EXPECT_TOKEN(Tokens[3], tok::identifier, TT_Unknown); // Not tok::kw_new4014  EXPECT_TOKEN(Tokens[4], tok::amp, TT_BinaryOperator);4015  EXPECT_TOKEN(Tokens[3], tok::identifier, TT_Unknown); // Not TT_StartOfName4016}4017 4018TEST_F(TokenAnnotatorTest, FunctionTryBlock) {4019  auto Tokens =4020      annotate("Foo::Foo(int x, int y) try\n"4021               "    : foo{[] -> std::string { return {}; }(), x}, bar{y} {\n"4022               "} catch (...) {\n"4023               "}");4024  ASSERT_EQ(Tokens.size(), 45u) << Tokens;4025  EXPECT_TOKEN(Tokens[2], tok::identifier, TT_CtorDtorDeclName);4026  EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_FunctionDeclarationLParen);4027  EXPECT_TOKEN(Tokens[11], tok::colon, TT_CtorInitializerColon);4028  EXPECT_TOKEN(Tokens[14], tok::l_square, TT_LambdaLSquare);4029  EXPECT_TOKEN(Tokens[16], tok::arrow, TT_LambdaArrow);4030  EXPECT_TOKEN(Tokens[20], tok::l_brace, TT_LambdaLBrace);4031  EXPECT_TOKEN(Tokens[31], tok::comma, TT_CtorInitializerComma);4032  EXPECT_TOKEN(Tokens[36], tok::l_brace, TT_FunctionLBrace);4033}4034 4035TEST_F(TokenAnnotatorTest, TypenameMacro) {4036  auto Style = getLLVMStyle();4037  Style.TypenameMacros.push_back("STRUCT");4038 4039  auto Tokens = annotate("STRUCT(T, B) { int i; };", Style);4040  ASSERT_EQ(Tokens.size(), 13u) << Tokens;4041  EXPECT_TOKEN(Tokens[0], tok::identifier, TT_TypenameMacro);4042  EXPECT_TOKEN(Tokens[1], tok::l_paren, TT_TypeDeclarationParen);4043  EXPECT_TOKEN(Tokens[5], tok::r_paren, TT_TypeDeclarationParen);4044  EXPECT_TOKEN(Tokens[6], tok::l_brace, TT_Unknown);4045}4046 4047TEST_F(TokenAnnotatorTest, GNULanguageStandard) {4048  auto Style = getGNUStyle();4049  EXPECT_EQ(Style.Standard, FormatStyle::LS_Latest);4050 4051  auto Tokens = annotate("return 1 <=> 2;", Style);4052  ASSERT_EQ(Tokens.size(), 6u) << Tokens;4053  EXPECT_TOKEN(Tokens[2], tok::spaceship, TT_BinaryOperator);4054}4055 4056TEST_F(TokenAnnotatorTest, SplitPenalty) {4057  auto Style = getLLVMStyle();4058  Style.ColumnLimit = 20;4059 4060  auto Tokens = annotate("class foo {\n"4061                         "  auto bar()\n"4062                         "      -> bool;\n"4063                         "};",4064                         Style);4065  ASSERT_EQ(Tokens.size(), 13u) << Tokens;4066  EXPECT_TOKEN(Tokens[7], tok::arrow, TT_TrailingReturnArrow);4067  EXPECT_SPLIT_PENALTY(Tokens[7], 23u);4068}4069 4070TEST_F(TokenAnnotatorTest, TemplateName) {4071  constexpr StringRef Code("return Foo < A || B > (C ^ D);");4072 4073  auto Tokens = annotate(Code);4074  ASSERT_EQ(Tokens.size(), 14u) << Tokens;4075  EXPECT_TOKEN(Tokens[2], tok::less, TT_BinaryOperator);4076  EXPECT_TOKEN(Tokens[6], tok::greater, TT_BinaryOperator);4077 4078  auto Style = getLLVMStyle();4079  Style.TemplateNames.push_back("Foo");4080 4081  Tokens = annotate(Code, Style);4082  EXPECT_TOKEN(Tokens[1], tok::identifier, TT_TemplateName);4083  EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);4084  EXPECT_TOKEN(Tokens[6], tok::greater, TT_TemplateCloser);4085}4086 4087TEST_F(TokenAnnotatorTest, TemplateInstantiation) {4088  auto Tokens = annotate("return FixedInt<N | M>();");4089  ASSERT_EQ(Tokens.size(), 11u) << Tokens;4090  EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);4091  EXPECT_TOKEN(Tokens[6], tok::greater, TT_TemplateCloser);4092 4093  Tokens = annotate("return FixedInt<N | M>(foo);");4094  ASSERT_EQ(Tokens.size(), 12u) << Tokens;4095  EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);4096  EXPECT_TOKEN(Tokens[6], tok::greater, TT_TemplateCloser);4097 4098  Tokens = annotate("return std::conditional_t<T::value == U::value, T, U>{};");4099  ASSERT_EQ(Tokens.size(), 21u) << Tokens;4100  EXPECT_TOKEN(Tokens[4], tok::less, TT_TemplateOpener);4101  EXPECT_TOKEN(Tokens[16], tok::greater, TT_TemplateCloser);4102 4103  Tokens =4104      annotate("auto x{std::conditional_t<T::value == U::value, T, U>{}};");4105  ASSERT_EQ(Tokens.size(), 24u) << Tokens;4106  EXPECT_TOKEN(Tokens[6], tok::less, TT_TemplateOpener);4107  EXPECT_TOKEN(Tokens[18], tok::greater, TT_TemplateCloser);4108 4109  Tokens = annotate(4110      "std::uint16_t kMTU = std::conditional<\n"4111      "    kTypeKind == KindA,\n"4112      "    std::integral_constant<std::uint16_t, kIoSockMtu>>::type::value;");4113  ASSERT_EQ(Tokens.size(), 30u) << Tokens;4114  EXPECT_TOKEN(Tokens[8], tok::less, TT_TemplateOpener);4115  EXPECT_TOKEN(Tokens[16], tok::less, TT_TemplateOpener);4116  EXPECT_TOKEN(Tokens[22], tok::greater, TT_TemplateCloser);4117  EXPECT_TOKEN(Tokens[23], tok::greater, TT_TemplateCloser);4118}4119 4120TEST_F(TokenAnnotatorTest, VariableTemplate) {4121  auto Style = getLLVMStyle();4122  Style.VariableTemplates.push_back("a");4123 4124  auto Tokens = annotate("auto t3 = (a<int>) + b;", Style);4125  ASSERT_EQ(Tokens.size(), 13u) << Tokens;4126  EXPECT_TOKEN(Tokens[4], tok::identifier, TT_VariableTemplate);4127  EXPECT_TOKEN(Tokens[5], tok::less, TT_TemplateOpener);4128  EXPECT_TOKEN(Tokens[7], tok::greater, TT_TemplateCloser);4129  EXPECT_TOKEN(Tokens[8], tok::r_paren, TT_Unknown); // Not TT_CastRParen4130  EXPECT_TOKEN(Tokens[9], tok::plus, TT_BinaryOperator);4131}4132 4133TEST_F(TokenAnnotatorTest, SwitchInMacroArgument) {4134  auto Tokens = annotate("FOOBAR(switch);\n"4135                         "void f() {}");4136  ASSERT_EQ(Tokens.size(), 12u) << Tokens;4137  EXPECT_TOKEN(Tokens[9], tok::l_brace, TT_FunctionLBrace);4138}4139 4140TEST_F(TokenAnnotatorTest, AfterPPDirective) {4141  auto Tokens = annotate("#error -- My error message");4142 4143  ASSERT_EQ(Tokens.size(), 7u) << Tokens;4144  EXPECT_TOKEN(Tokens[2], tok::minusminus, TT_AfterPPDirective);4145}4146 4147TEST_F(TokenAnnotatorTest, UserDefinedConversionFunction) {4148  auto Tokens = annotate("operator int(void);");4149  ASSERT_EQ(Tokens.size(), 7u) << Tokens;4150  EXPECT_TOKEN(Tokens[0], tok::kw_operator, TT_FunctionDeclarationName);4151  EXPECT_TOKEN(Tokens[2], tok::l_paren, TT_FunctionDeclarationLParen);4152 4153  Tokens = annotate("explicit operator int *();");4154  ASSERT_EQ(Tokens.size(), 8u) << Tokens;4155  EXPECT_TOKEN(Tokens[1], tok::kw_operator, TT_FunctionDeclarationName);4156  EXPECT_TOKEN(Tokens[3], tok::star, TT_PointerOrReference);4157  EXPECT_TOKEN(Tokens[4], tok::l_paren, TT_FunctionDeclarationLParen);4158 4159  Tokens = annotate("operator int &();");4160  ASSERT_EQ(Tokens.size(), 7u) << Tokens;4161  EXPECT_TOKEN(Tokens[0], tok::kw_operator, TT_FunctionDeclarationName);4162  EXPECT_TOKEN(Tokens[2], tok::amp, TT_PointerOrReference);4163  EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_FunctionDeclarationLParen);4164 4165  Tokens = annotate("operator auto() const { return 2; }");4166  ASSERT_EQ(Tokens.size(), 11u) << Tokens;4167  EXPECT_TOKEN(Tokens[0], tok::kw_operator, TT_FunctionDeclarationName);4168  EXPECT_TOKEN(Tokens[2], tok::l_paren, TT_FunctionDeclarationLParen);4169  EXPECT_TOKEN(Tokens[4], tok::kw_const, TT_TrailingAnnotation);4170  EXPECT_TOKEN(Tokens[5], tok::l_brace, TT_FunctionLBrace);4171 4172  Tokens = annotate("operator decltype(auto)() const;");4173  ASSERT_EQ(Tokens.size(), 10u) << Tokens;4174  EXPECT_TOKEN(Tokens[0], tok::kw_operator, TT_FunctionDeclarationName);4175  EXPECT_TOKEN(Tokens[2], tok::l_paren, TT_TypeDeclarationParen);4176  EXPECT_TOKEN(Tokens[4], tok::r_paren, TT_TypeDeclarationParen);4177  EXPECT_TOKEN(Tokens[5], tok::l_paren, TT_FunctionDeclarationLParen);4178  EXPECT_TOKEN(Tokens[7], tok::kw_const, TT_TrailingAnnotation);4179 4180  Tokens = annotate("virtual operator Foo() = 0;");4181  ASSERT_EQ(Tokens.size(), 9u) << Tokens;4182  EXPECT_TOKEN(Tokens[1], tok::kw_operator, TT_FunctionDeclarationName);4183  EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_FunctionDeclarationLParen);4184 4185  Tokens = annotate("operator Foo() override { return Foo(); }");4186  ASSERT_EQ(Tokens.size(), 13u) << Tokens;4187  EXPECT_TOKEN(Tokens[0], tok::kw_operator, TT_FunctionDeclarationName);4188  EXPECT_TOKEN(Tokens[2], tok::l_paren, TT_FunctionDeclarationLParen);4189  EXPECT_TOKEN(Tokens[5], tok::l_brace, TT_FunctionLBrace);4190 4191  Tokens = annotate("friend Bar::operator Foo();");4192  ASSERT_EQ(Tokens.size(), 9u) << Tokens;4193  EXPECT_TOKEN(Tokens[3], tok::kw_operator, TT_FunctionDeclarationName);4194  EXPECT_TOKEN(Tokens[5], tok::l_paren, TT_FunctionDeclarationLParen);4195}4196 4197TEST_F(TokenAnnotatorTest, UTF8StringLiteral) {4198  auto Tokens = annotate("return u8\"foo\";", getLLVMStyle(FormatStyle::LK_C));4199  ASSERT_EQ(Tokens.size(), 4u) << Tokens;4200  EXPECT_TOKEN(Tokens[1], tok::utf8_string_literal, TT_Unknown);4201}4202 4203TEST_F(TokenAnnotatorTest, C23DigitSeparator) {4204  auto Tokens = annotate("return 1'000;", getLLVMStyle(FormatStyle::LK_C));4205  ASSERT_EQ(Tokens.size(), 4u) << Tokens;4206  EXPECT_EQ(Tokens[1]->TokenText, "1'000");4207  EXPECT_TOKEN(Tokens[2], tok::semi, TT_Unknown);4208}4209 4210TEST_F(TokenAnnotatorTest, IdentifierPackage) {4211  auto Tokens = annotate("auto package;");4212  ASSERT_EQ(Tokens.size(), 4u) << Tokens;4213  EXPECT_FALSE(Tokens[0]->isObjCAccessSpecifier());4214}4215 4216TEST_F(TokenAnnotatorTest, UserDefinedLiteral) {4217  auto Tokens = annotate("auto dollars = 2_$;");4218  ASSERT_EQ(Tokens.size(), 6u) << Tokens;4219  EXPECT_EQ(Tokens[3]->TokenText, "2_$");4220}4221 4222TEST_F(TokenAnnotatorTest, EnumColonInTypedef) {4223  auto Tokens = annotate("typedef enum : int {} foo;");4224  ASSERT_EQ(Tokens.size(), 9u) << Tokens;4225  EXPECT_TOKEN(Tokens[2], tok::colon, TT_Unknown); // Not TT_InheritanceColon.4226}4227 4228TEST_F(TokenAnnotatorTest, BitFieldColon) {4229  auto Tokens = annotate("class C {\n"4230                         "  int f : SIZE;\n"4231                         "};");4232  ASSERT_EQ(Tokens.size(), 11u) << Tokens;4233  EXPECT_TOKEN(Tokens[5], tok::colon, TT_BitFieldColon);4234}4235 4236TEST_F(TokenAnnotatorTest, JsonCodeInRawString) {4237  auto Tokens = annotate("{\n"4238                         "  \"foo\": \"bar\",\n"4239                         "  \"str\": \"test\"\n"4240                         "}",4241                         getLLVMStyle(FormatStyle::LK_Json));4242  ASSERT_EQ(Tokens.size(), 10u) << Tokens;4243  EXPECT_TOKEN(Tokens[0], tok::l_brace, TT_DictLiteral);4244  EXPECT_TOKEN(Tokens[1], tok::string_literal, TT_SelectorName);4245  EXPECT_TOKEN(Tokens[2], tok::colon, TT_DictLiteral);4246  EXPECT_TOKEN(Tokens[5], tok::string_literal, TT_SelectorName);4247  EXPECT_TOKEN(Tokens[6], tok::colon, TT_DictLiteral);4248}4249 4250TEST_F(TokenAnnotatorTest, LineCommentTrailingBackslash) {4251  auto Tokens = annotate("// a \\\n"4252                         "// b");4253  ASSERT_EQ(Tokens.size(), 3u) << Tokens;4254  EXPECT_TOKEN(Tokens[1], tok::comment, TT_LineComment);4255}4256 4257TEST_F(TokenAnnotatorTest, ArrowAfterSubscript) {4258  auto Tokens =4259      annotate("return (getStructType()->getElements())[eIdx]->getName();");4260  ASSERT_EQ(Tokens.size(), 19u) << Tokens;4261  // Not TT_LambdaArrow.4262  EXPECT_TOKEN(Tokens[13], tok::arrow, TT_Unknown);4263}4264 4265TEST_F(TokenAnnotatorTest, QtProperty) {4266  auto Style = getLLVMStyle();4267  Style.AllowBreakBeforeQtProperty = true;4268 4269  auto Tokens = annotate(4270      "Q_PROPERTY(int value READ value WRITE setValue NOTIFY valueChanged)",4271      Style);4272  ASSERT_EQ(Tokens.size(), 12u) << Tokens;4273  EXPECT_TOKEN(Tokens[4], tok::identifier, TT_QtProperty);4274  EXPECT_TOKEN(Tokens[6], tok::identifier, TT_QtProperty);4275  EXPECT_TOKEN(Tokens[8], tok::identifier, TT_QtProperty);4276 4277  Tokens = annotate(4278      "struct S {\n"4279      "  Q_OBJECT Q_PROPERTY(int value READ value WRITE setValue NOTIFY foo)\n"4280      "};",4281      Style);4282  ASSERT_EQ(Tokens.size(), 18u) << Tokens;4283  EXPECT_TOKEN(Tokens[8], tok::identifier, TT_QtProperty);4284  EXPECT_TOKEN(Tokens[10], tok::identifier, TT_QtProperty);4285  EXPECT_TOKEN(Tokens[12], tok::identifier, TT_QtProperty);4286}4287 4288TEST_F(TokenAnnotatorTest, AttributeSquares) {4289  auto Tokens = annotate("[[maybe_unused]] const int i;");4290  ASSERT_EQ(Tokens.size(), 10u) << Tokens;4291  EXPECT_TOKEN(Tokens[0], tok::l_square, TT_AttributeLSquare);4292  EXPECT_TOKEN(Tokens[1], tok::l_square, TT_Unknown);4293  EXPECT_TOKEN(Tokens[3], tok::r_square, TT_Unknown);4294  EXPECT_TOKEN(Tokens[4], tok::r_square, TT_AttributeRSquare);4295  EXPECT_TRUE(Tokens[4]->EndsCppAttributeGroup);4296 4297  Tokens = annotate("[[foo([[]])]] [[maybe_unused]] int j;");4298  ASSERT_EQ(Tokens.size(), 20u) << Tokens;4299  EXPECT_TOKEN(Tokens[0], tok::l_square, TT_AttributeLSquare);4300  EXPECT_TOKEN(Tokens[1], tok::l_square, TT_Unknown);4301  EXPECT_TOKEN(Tokens[9], tok::r_square, TT_Unknown);4302  EXPECT_TOKEN(Tokens[10], tok::r_square, TT_AttributeRSquare);4303  EXPECT_FALSE(Tokens[10]->EndsCppAttributeGroup);4304  EXPECT_TOKEN(Tokens[11], tok::l_square, TT_AttributeLSquare);4305  EXPECT_TOKEN(Tokens[12], tok::l_square, TT_Unknown);4306  EXPECT_TOKEN(Tokens[14], tok::r_square, TT_Unknown);4307  EXPECT_TOKEN(Tokens[15], tok::r_square, TT_AttributeRSquare);4308  EXPECT_TRUE(Tokens[15]->EndsCppAttributeGroup);4309}4310 4311} // namespace4312} // namespace format4313} // namespace clang4314