2008 lines · cpp
1//===- unittests/AST/CommentLexer.cpp ------ Comment lexer tests ----------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8 9#include "clang/AST/CommentLexer.h"10#include "clang/AST/CommentCommandTraits.h"11#include "clang/Basic/CommentOptions.h"12#include "clang/Basic/Diagnostic.h"13#include "clang/Basic/DiagnosticOptions.h"14#include "clang/Basic/FileManager.h"15#include "clang/Basic/SourceManager.h"16#include "llvm/ADT/STLExtras.h"17#include "gtest/gtest.h"18#include <vector>19 20using namespace llvm;21using namespace clang;22 23namespace clang {24namespace comments {25 26namespace {27class CommentLexerTest : public ::testing::Test {28protected:29 CommentLexerTest()30 : FileMgr(FileMgrOpts),31 Diags(DiagnosticIDs::create(), DiagOpts, new IgnoringDiagConsumer()),32 SourceMgr(Diags, FileMgr), Traits(Allocator, CommentOptions()) {}33 34 FileSystemOptions FileMgrOpts;35 FileManager FileMgr;36 DiagnosticOptions DiagOpts;37 DiagnosticsEngine Diags;38 SourceManager SourceMgr;39 llvm::BumpPtrAllocator Allocator;40 CommandTraits Traits;41 42 void lexString(const char *Source, std::vector<Token> &Toks);43 44 StringRef getCommandName(const Token &Tok) {45 return Traits.getCommandInfo(Tok.getCommandID())->Name;46 }47 48 StringRef getVerbatimBlockName(const Token &Tok) {49 return Traits.getCommandInfo(Tok.getVerbatimBlockID())->Name;50 }51 52 StringRef getVerbatimLineName(const Token &Tok) {53 return Traits.getCommandInfo(Tok.getVerbatimLineID())->Name;54 }55};56 57void CommentLexerTest::lexString(const char *Source,58 std::vector<Token> &Toks) {59 std::unique_ptr<MemoryBuffer> Buf = MemoryBuffer::getMemBuffer(Source);60 FileID File = SourceMgr.createFileID(std::move(Buf));61 SourceLocation Begin = SourceMgr.getLocForStartOfFile(File);62 63 Lexer L(Allocator, Diags, Traits, Begin, Source, Source + strlen(Source));64 65 while (1) {66 Token Tok;67 L.lex(Tok);68 if (Tok.is(tok::eof))69 break;70 Toks.push_back(Tok);71 }72}73 74} // unnamed namespace75 76// Empty source range should be handled.77TEST_F(CommentLexerTest, Basic1) {78 const char *Source = "";79 std::vector<Token> Toks;80 81 lexString(Source, Toks);82 83 ASSERT_EQ(0U, Toks.size());84}85 86// Empty comments should be handled.87TEST_F(CommentLexerTest, Basic2) {88 const char *Sources[] = {89 "//", "///", "//!", "///<", "//!<"90 };91 for (size_t i = 0, e = std::size(Sources); i != e; i++) {92 std::vector<Token> Toks;93 94 lexString(Sources[i], Toks);95 96 ASSERT_EQ(1U, Toks.size());97 98 ASSERT_EQ(tok::newline, Toks[0].getKind());99 }100}101 102// Empty comments should be handled.103TEST_F(CommentLexerTest, Basic3) {104 const char *Sources[] = {105 "/**/", "/***/", "/*!*/", "/**<*/", "/*!<*/"106 };107 for (size_t i = 0, e = std::size(Sources); i != e; i++) {108 std::vector<Token> Toks;109 110 lexString(Sources[i], Toks);111 112 ASSERT_EQ(2U, Toks.size());113 114 ASSERT_EQ(tok::newline, Toks[0].getKind());115 ASSERT_EQ(tok::newline, Toks[1].getKind());116 }117}118 119// Single comment with plain text.120TEST_F(CommentLexerTest, Basic4) {121 const char *Sources[] = {122 "// Meow", "/// Meow", "//! Meow",123 "// Meow\n", "// Meow\r\n", "//! Meow\r",124 };125 126 for (size_t i = 0, e = std::size(Sources); i != e; i++) {127 std::vector<Token> Toks;128 129 lexString(Sources[i], Toks);130 131 ASSERT_EQ(2U, Toks.size());132 133 ASSERT_EQ(tok::text, Toks[0].getKind());134 ASSERT_EQ(StringRef(" Meow"), Toks[0].getText());135 136 ASSERT_EQ(tok::newline, Toks[1].getKind());137 }138}139 140// Single comment with plain text.141TEST_F(CommentLexerTest, Basic5) {142 const char *Sources[] = {143 "/* Meow*/", "/** Meow*/", "/*! Meow*/"144 };145 146 for (size_t i = 0, e = std::size(Sources); i != e; i++) {147 std::vector<Token> Toks;148 149 lexString(Sources[i], Toks);150 151 ASSERT_EQ(3U, Toks.size());152 153 ASSERT_EQ(tok::text, Toks[0].getKind());154 ASSERT_EQ(StringRef(" Meow"), Toks[0].getText());155 156 ASSERT_EQ(tok::newline, Toks[1].getKind());157 ASSERT_EQ(tok::newline, Toks[2].getKind());158 }159}160 161// Test newline escaping.162TEST_F(CommentLexerTest, Basic6) {163 const char *Sources[] = {164 "// Aaa\\\n" " Bbb\\ \n" " Ccc?" "?/\n",165 "// Aaa\\\r\n" " Bbb\\ \r\n" " Ccc?" "?/\r\n",166 "// Aaa\\\r" " Bbb\\ \r" " Ccc?" "?/\r"167 };168 169 for (size_t i = 0, e = std::size(Sources); i != e; i++) {170 std::vector<Token> Toks;171 172 lexString(Sources[i], Toks);173 174 ASSERT_EQ(10U, Toks.size());175 176 ASSERT_EQ(tok::text, Toks[0].getKind());177 ASSERT_EQ(StringRef(" Aaa"), Toks[0].getText());178 ASSERT_EQ(tok::text, Toks[1].getKind());179 ASSERT_EQ(StringRef("\\"), Toks[1].getText());180 ASSERT_EQ(tok::newline, Toks[2].getKind());181 182 ASSERT_EQ(tok::text, Toks[3].getKind());183 ASSERT_EQ(StringRef(" Bbb"), Toks[3].getText());184 ASSERT_EQ(tok::text, Toks[4].getKind());185 ASSERT_EQ(StringRef("\\"), Toks[4].getText());186 ASSERT_EQ(tok::text, Toks[5].getKind());187 ASSERT_EQ(StringRef(" "), Toks[5].getText());188 ASSERT_EQ(tok::newline, Toks[6].getKind());189 190 ASSERT_EQ(tok::text, Toks[7].getKind());191 ASSERT_EQ(StringRef(" Ccc?" "?/"), Toks[7].getText());192 ASSERT_EQ(tok::newline, Toks[8].getKind());193 194 ASSERT_EQ(tok::newline, Toks[9].getKind());195 }196}197 198// Check that we skip C-style aligned stars correctly.199TEST_F(CommentLexerTest, Basic7) {200 const char *Source =201 "/* Aaa\n"202 " * Bbb\r\n"203 "\t* Ccc\n"204 " ! Ddd\n"205 " * Eee\n"206 " ** Fff\n"207 " */";208 std::vector<Token> Toks;209 210 lexString(Source, Toks);211 212 ASSERT_EQ(15U, Toks.size());213 214 ASSERT_EQ(tok::text, Toks[0].getKind());215 ASSERT_EQ(StringRef(" Aaa"), Toks[0].getText());216 ASSERT_EQ(tok::newline, Toks[1].getKind());217 218 ASSERT_EQ(tok::text, Toks[2].getKind());219 ASSERT_EQ(StringRef(" Bbb"), Toks[2].getText());220 ASSERT_EQ(tok::newline, Toks[3].getKind());221 222 ASSERT_EQ(tok::text, Toks[4].getKind());223 ASSERT_EQ(StringRef(" Ccc"), Toks[4].getText());224 ASSERT_EQ(tok::newline, Toks[5].getKind());225 226 ASSERT_EQ(tok::text, Toks[6].getKind());227 ASSERT_EQ(StringRef(" ! Ddd"), Toks[6].getText());228 ASSERT_EQ(tok::newline, Toks[7].getKind());229 230 ASSERT_EQ(tok::text, Toks[8].getKind());231 ASSERT_EQ(StringRef(" Eee"), Toks[8].getText());232 ASSERT_EQ(tok::newline, Toks[9].getKind());233 234 ASSERT_EQ(tok::text, Toks[10].getKind());235 ASSERT_EQ(StringRef("* Fff"), Toks[10].getText());236 ASSERT_EQ(tok::newline, Toks[11].getKind());237 238 ASSERT_EQ(tok::text, Toks[12].getKind());239 ASSERT_EQ(StringRef(" "), Toks[12].getText());240 241 ASSERT_EQ(tok::newline, Toks[13].getKind());242 ASSERT_EQ(tok::newline, Toks[14].getKind());243}244 245// A command marker followed by comment end.246TEST_F(CommentLexerTest, DoxygenCommand1) {247 const char *Sources[] = { "//@", "///@", "//!@" };248 for (size_t i = 0, e = std::size(Sources); i != e; i++) {249 std::vector<Token> Toks;250 251 lexString(Sources[i], Toks);252 253 ASSERT_EQ(2U, Toks.size());254 255 ASSERT_EQ(tok::text, Toks[0].getKind());256 ASSERT_EQ(StringRef("@"), Toks[0].getText());257 258 ASSERT_EQ(tok::newline, Toks[1].getKind());259 }260}261 262// A command marker followed by comment end.263TEST_F(CommentLexerTest, DoxygenCommand2) {264 const char *Sources[] = { "/*@*/", "/**@*/", "/*!@*/"};265 for (size_t i = 0, e = std::size(Sources); i != e; i++) {266 std::vector<Token> Toks;267 268 lexString(Sources[i], Toks);269 270 ASSERT_EQ(3U, Toks.size());271 272 ASSERT_EQ(tok::text, Toks[0].getKind());273 ASSERT_EQ(StringRef("@"), Toks[0].getText());274 275 ASSERT_EQ(tok::newline, Toks[1].getKind());276 ASSERT_EQ(tok::newline, Toks[2].getKind());277 }278}279 280// A command marker followed by comment end.281TEST_F(CommentLexerTest, DoxygenCommand3) {282 const char *Sources[] = { "/*\\*/", "/**\\*/" };283 for (size_t i = 0, e = std::size(Sources); i != e; i++) {284 std::vector<Token> Toks;285 286 lexString(Sources[i], Toks);287 288 ASSERT_EQ(3U, Toks.size());289 290 ASSERT_EQ(tok::text, Toks[0].getKind());291 ASSERT_EQ(StringRef("\\"), Toks[0].getText());292 293 ASSERT_EQ(tok::newline, Toks[1].getKind());294 ASSERT_EQ(tok::newline, Toks[2].getKind());295 }296}297 298// Doxygen escape sequences.299TEST_F(CommentLexerTest, DoxygenCommand4) {300 const char *Sources[] = {301 "/// \\\\ \\@ \\& \\$ \\# \\< \\> \\% \\\" \\. \\::",302 "/// @\\ @@ @& @$ @# @< @> @% @\" @. @::"303 };304 const char *Text[] = {305 " ",306 "\\", " ", "@", " ", "&", " ", "$", " ", "#", " ",307 "<", " ", ">", " ", "%", " ", "\"", " ", ".", " ",308 "::", ""309 };310 311 for (size_t i = 0, e = std::size(Sources); i != e; i++) {312 std::vector<Token> Toks;313 314 lexString(Sources[i], Toks);315 316 ASSERT_EQ(std::size(Text), Toks.size());317 318 for (size_t j = 0, e = Toks.size(); j != e; j++) {319 if(Toks[j].is(tok::text)) {320 ASSERT_EQ(StringRef(Text[j]), Toks[j].getText())321 << "index " << i;322 }323 }324 }325}326 327// A command marker followed by a non-letter that is not a part of an escape328// sequence.329TEST_F(CommentLexerTest, DoxygenCommand5) {330 const char *Source = "/// \\^ \\0";331 std::vector<Token> Toks;332 333 lexString(Source, Toks);334 335 ASSERT_EQ(6U, Toks.size());336 337 ASSERT_EQ(tok::text, Toks[0].getKind());338 ASSERT_EQ(StringRef(" "), Toks[0].getText());339 340 ASSERT_EQ(tok::text, Toks[1].getKind());341 ASSERT_EQ(StringRef("\\"), Toks[1].getText());342 343 ASSERT_EQ(tok::text, Toks[2].getKind());344 ASSERT_EQ(StringRef("^ "), Toks[2].getText());345 346 ASSERT_EQ(tok::text, Toks[3].getKind());347 ASSERT_EQ(StringRef("\\"), Toks[3].getText());348 349 ASSERT_EQ(tok::text, Toks[4].getKind());350 ASSERT_EQ(StringRef("0"), Toks[4].getText());351 352 ASSERT_EQ(tok::newline, Toks[5].getKind());353}354 355TEST_F(CommentLexerTest, DoxygenCommand6) {356 const char *Source = "/// \\brief Aaa.";357 std::vector<Token> Toks;358 359 lexString(Source, Toks);360 361 ASSERT_EQ(4U, Toks.size());362 363 ASSERT_EQ(tok::text, Toks[0].getKind());364 ASSERT_EQ(StringRef(" "), Toks[0].getText());365 366 ASSERT_EQ(tok::backslash_command, Toks[1].getKind());367 ASSERT_EQ(StringRef("brief"), getCommandName(Toks[1]));368 369 ASSERT_EQ(tok::text, Toks[2].getKind());370 ASSERT_EQ(StringRef(" Aaa."), Toks[2].getText());371 372 ASSERT_EQ(tok::newline, Toks[3].getKind());373}374 375TEST_F(CommentLexerTest, DoxygenCommand7) {376 const char *Source = "/// \\em\\em \\em\t\\em\n";377 std::vector<Token> Toks;378 379 lexString(Source, Toks);380 381 ASSERT_EQ(8U, Toks.size());382 383 ASSERT_EQ(tok::text, Toks[0].getKind());384 ASSERT_EQ(StringRef(" "), Toks[0].getText());385 386 ASSERT_EQ(tok::backslash_command, Toks[1].getKind());387 ASSERT_EQ(StringRef("em"), getCommandName(Toks[1]));388 389 ASSERT_EQ(tok::backslash_command, Toks[2].getKind());390 ASSERT_EQ(StringRef("em"), getCommandName(Toks[2]));391 392 ASSERT_EQ(tok::text, Toks[3].getKind());393 ASSERT_EQ(StringRef(" "), Toks[3].getText());394 395 ASSERT_EQ(tok::backslash_command, Toks[4].getKind());396 ASSERT_EQ(StringRef("em"), getCommandName(Toks[4]));397 398 ASSERT_EQ(tok::text, Toks[5].getKind());399 ASSERT_EQ(StringRef("\t"), Toks[5].getText());400 401 ASSERT_EQ(tok::backslash_command, Toks[6].getKind());402 ASSERT_EQ(StringRef("em"), getCommandName(Toks[6]));403 404 ASSERT_EQ(tok::newline, Toks[7].getKind());405}406 407TEST_F(CommentLexerTest, DoxygenCommand8) {408 const char *Source = "/// @em@em @em\t@em\n";409 std::vector<Token> Toks;410 411 lexString(Source, Toks);412 413 ASSERT_EQ(8U, Toks.size());414 415 ASSERT_EQ(tok::text, Toks[0].getKind());416 ASSERT_EQ(StringRef(" "), Toks[0].getText());417 418 ASSERT_EQ(tok::at_command, Toks[1].getKind());419 ASSERT_EQ(StringRef("em"), getCommandName(Toks[1]));420 421 ASSERT_EQ(tok::at_command, Toks[2].getKind());422 ASSERT_EQ(StringRef("em"), getCommandName(Toks[2]));423 424 ASSERT_EQ(tok::text, Toks[3].getKind());425 ASSERT_EQ(StringRef(" "), Toks[3].getText());426 427 ASSERT_EQ(tok::at_command, Toks[4].getKind());428 ASSERT_EQ(StringRef("em"), getCommandName(Toks[4]));429 430 ASSERT_EQ(tok::text, Toks[5].getKind());431 ASSERT_EQ(StringRef("\t"), Toks[5].getText());432 433 ASSERT_EQ(tok::at_command, Toks[6].getKind());434 ASSERT_EQ(StringRef("em"), getCommandName(Toks[6]));435 436 ASSERT_EQ(tok::newline, Toks[7].getKind());437}438 439TEST_F(CommentLexerTest, DoxygenCommand9) {440 const char *Source = "/// \\aaa\\bbb \\ccc\t\\ddd\n";441 std::vector<Token> Toks;442 443 lexString(Source, Toks);444 445 ASSERT_EQ(8U, Toks.size());446 447 ASSERT_EQ(tok::text, Toks[0].getKind());448 ASSERT_EQ(StringRef(" "), Toks[0].getText());449 450 ASSERT_EQ(tok::unknown_command, Toks[1].getKind());451 ASSERT_EQ(StringRef("aaa"), Toks[1].getUnknownCommandName());452 453 ASSERT_EQ(tok::unknown_command, Toks[2].getKind());454 ASSERT_EQ(StringRef("bbb"), Toks[2].getUnknownCommandName());455 456 ASSERT_EQ(tok::text, Toks[3].getKind());457 ASSERT_EQ(StringRef(" "), Toks[3].getText());458 459 ASSERT_EQ(tok::unknown_command, Toks[4].getKind());460 ASSERT_EQ(StringRef("ccc"), Toks[4].getUnknownCommandName());461 462 ASSERT_EQ(tok::text, Toks[5].getKind());463 ASSERT_EQ(StringRef("\t"), Toks[5].getText());464 465 ASSERT_EQ(tok::unknown_command, Toks[6].getKind());466 ASSERT_EQ(StringRef("ddd"), Toks[6].getUnknownCommandName());467 468 ASSERT_EQ(tok::newline, Toks[7].getKind());469}470 471TEST_F(CommentLexerTest, DoxygenCommand10) {472 const char *Source = "// \\c\n";473 std::vector<Token> Toks;474 475 lexString(Source, Toks);476 477 ASSERT_EQ(3U, Toks.size());478 479 ASSERT_EQ(tok::text, Toks[0].getKind());480 ASSERT_EQ(StringRef(" "), Toks[0].getText());481 482 ASSERT_EQ(tok::backslash_command, Toks[1].getKind());483 ASSERT_EQ(StringRef("c"), getCommandName(Toks[1]));484 485 ASSERT_EQ(tok::newline, Toks[2].getKind());486}487 488TEST_F(CommentLexerTest, RegisterCustomBlockCommand) {489 const char *Source =490 "/// \\NewBlockCommand Aaa.\n"491 "/// @NewBlockCommand Aaa.\n";492 493 Traits.registerBlockCommand(StringRef("NewBlockCommand"));494 495 std::vector<Token> Toks;496 497 lexString(Source, Toks);498 499 ASSERT_EQ(8U, Toks.size());500 501 ASSERT_EQ(tok::text, Toks[0].getKind());502 ASSERT_EQ(StringRef(" "), Toks[0].getText());503 504 ASSERT_EQ(tok::backslash_command, Toks[1].getKind());505 ASSERT_EQ(StringRef("NewBlockCommand"), getCommandName(Toks[1]));506 507 ASSERT_EQ(tok::text, Toks[2].getKind());508 ASSERT_EQ(StringRef(" Aaa."), Toks[2].getText());509 510 ASSERT_EQ(tok::newline, Toks[3].getKind());511 512 ASSERT_EQ(tok::text, Toks[4].getKind());513 ASSERT_EQ(StringRef(" "), Toks[4].getText());514 515 ASSERT_EQ(tok::at_command, Toks[5].getKind());516 ASSERT_EQ(StringRef("NewBlockCommand"), getCommandName(Toks[5]));517 518 ASSERT_EQ(tok::text, Toks[6].getKind());519 ASSERT_EQ(StringRef(" Aaa."), Toks[6].getText());520 521 ASSERT_EQ(tok::newline, Toks[7].getKind());522}523 524TEST_F(CommentLexerTest, RegisterMultipleBlockCommands) {525 const char *Source =526 "/// \\Foo\n"527 "/// \\Bar Baz\n"528 "/// \\Blech quux=corge\n";529 530 Traits.registerBlockCommand(StringRef("Foo"));531 Traits.registerBlockCommand(StringRef("Bar"));532 Traits.registerBlockCommand(StringRef("Blech"));533 534 std::vector<Token> Toks;535 536 lexString(Source, Toks);537 538 ASSERT_EQ(11U, Toks.size());539 540 ASSERT_EQ(tok::text, Toks[0].getKind());541 ASSERT_EQ(StringRef(" "), Toks[0].getText());542 543 ASSERT_EQ(tok::backslash_command, Toks[1].getKind());544 ASSERT_EQ(StringRef("Foo"), getCommandName(Toks[1]));545 546 ASSERT_EQ(tok::newline, Toks[2].getKind());547 548 ASSERT_EQ(tok::text, Toks[3].getKind());549 ASSERT_EQ(StringRef(" "), Toks[3].getText());550 551 ASSERT_EQ(tok::backslash_command, Toks[4].getKind());552 ASSERT_EQ(StringRef("Bar"), getCommandName(Toks[4]));553 554 ASSERT_EQ(tok::text, Toks[5].getKind());555 ASSERT_EQ(StringRef(" Baz"), Toks[5].getText());556 557 ASSERT_EQ(tok::newline, Toks[6].getKind());558 559 ASSERT_EQ(tok::text, Toks[7].getKind());560 ASSERT_EQ(StringRef(" "), Toks[7].getText());561 562 ASSERT_EQ(tok::backslash_command, Toks[8].getKind());563 ASSERT_EQ(StringRef("Blech"), getCommandName(Toks[8]));564 565 ASSERT_EQ(tok::text, Toks[9].getKind());566 ASSERT_EQ(StringRef(" quux=corge"), Toks[9].getText());567 568 ASSERT_EQ(tok::newline, Toks[10].getKind());569}570 571// Empty verbatim block.572TEST_F(CommentLexerTest, VerbatimBlock1) {573 const char *Sources[] = {574 "/// \\verbatim\\endverbatim\n//",575 "/** \\verbatim\\endverbatim*/"576 };577 578 for (size_t i = 0, e = std::size(Sources); i != e; i++) {579 std::vector<Token> Toks;580 581 lexString(Sources[i], Toks);582 583 ASSERT_EQ(5U, Toks.size());584 585 ASSERT_EQ(tok::text, Toks[0].getKind());586 ASSERT_EQ(StringRef(" "), Toks[0].getText());587 588 ASSERT_EQ(tok::verbatim_block_begin, Toks[1].getKind());589 ASSERT_EQ(StringRef("verbatim"), getVerbatimBlockName(Toks[1]));590 591 ASSERT_EQ(tok::verbatim_block_end, Toks[2].getKind());592 ASSERT_EQ(StringRef("endverbatim"), getVerbatimBlockName(Toks[2]));593 594 ASSERT_EQ(tok::newline, Toks[3].getKind());595 ASSERT_EQ(tok::newline, Toks[4].getKind());596 }597}598 599// Empty verbatim block without an end command.600TEST_F(CommentLexerTest, VerbatimBlock2) {601 const char *Source = "/// \\verbatim";602 603 std::vector<Token> Toks;604 605 lexString(Source, Toks);606 607 ASSERT_EQ(3U, Toks.size());608 609 ASSERT_EQ(tok::text, Toks[0].getKind());610 ASSERT_EQ(StringRef(" "), Toks[0].getText());611 612 ASSERT_EQ(tok::verbatim_block_begin, Toks[1].getKind());613 ASSERT_EQ(StringRef("verbatim"), getVerbatimBlockName(Toks[1]));614 615 ASSERT_EQ(tok::newline, Toks[2].getKind());616}617 618// Empty verbatim block without an end command.619TEST_F(CommentLexerTest, VerbatimBlock3) {620 const char *Source = "/** \\verbatim*/";621 622 std::vector<Token> Toks;623 624 lexString(Source, Toks);625 626 ASSERT_EQ(4U, Toks.size());627 628 ASSERT_EQ(tok::text, Toks[0].getKind());629 ASSERT_EQ(StringRef(" "), Toks[0].getText());630 631 ASSERT_EQ(tok::verbatim_block_begin, Toks[1].getKind());632 ASSERT_EQ(StringRef("verbatim"), getVerbatimBlockName(Toks[1]));633 634 ASSERT_EQ(tok::newline, Toks[2].getKind());635 ASSERT_EQ(tok::newline, Toks[3].getKind());636}637 638// Single-line verbatim block.639TEST_F(CommentLexerTest, VerbatimBlock4) {640 const char *Sources[] = {641 "/// Meow \\verbatim aaa \\endverbatim\n//",642 "/** Meow \\verbatim aaa \\endverbatim*/"643 };644 645 for (size_t i = 0, e = std::size(Sources); i != e; i++) {646 std::vector<Token> Toks;647 648 lexString(Sources[i], Toks);649 650 ASSERT_EQ(6U, Toks.size());651 652 ASSERT_EQ(tok::text, Toks[0].getKind());653 ASSERT_EQ(StringRef(" Meow "), Toks[0].getText());654 655 ASSERT_EQ(tok::verbatim_block_begin, Toks[1].getKind());656 ASSERT_EQ(StringRef("verbatim"), getVerbatimBlockName(Toks[1]));657 658 ASSERT_EQ(tok::verbatim_block_line, Toks[2].getKind());659 ASSERT_EQ(StringRef(" aaa "), Toks[2].getVerbatimBlockText());660 661 ASSERT_EQ(tok::verbatim_block_end, Toks[3].getKind());662 ASSERT_EQ(StringRef("endverbatim"), getVerbatimBlockName(Toks[3]));663 664 ASSERT_EQ(tok::newline, Toks[4].getKind());665 ASSERT_EQ(tok::newline, Toks[5].getKind());666 }667}668 669// Single-line verbatim block without an end command.670TEST_F(CommentLexerTest, VerbatimBlock5) {671 const char *Sources[] = {672 "/// Meow \\verbatim aaa \n//",673 "/** Meow \\verbatim aaa */"674 };675 676 for (size_t i = 0, e = std::size(Sources); i != e; i++) {677 std::vector<Token> Toks;678 679 lexString(Sources[i], Toks);680 681 ASSERT_EQ(5U, Toks.size());682 683 ASSERT_EQ(tok::text, Toks[0].getKind());684 ASSERT_EQ(StringRef(" Meow "), Toks[0].getText());685 686 ASSERT_EQ(tok::verbatim_block_begin, Toks[1].getKind());687 ASSERT_EQ(StringRef("verbatim"), getVerbatimBlockName(Toks[1]));688 689 ASSERT_EQ(tok::verbatim_block_line, Toks[2].getKind());690 ASSERT_EQ(StringRef(" aaa "), Toks[2].getVerbatimBlockText());691 692 ASSERT_EQ(tok::newline, Toks[3].getKind());693 ASSERT_EQ(tok::newline, Toks[4].getKind());694 }695}696 697TEST_F(CommentLexerTest, VerbatimBlock6) {698 const char *Source =699 "// \\verbatim\n"700 "// Aaa\n"701 "//\n"702 "// Bbb\n"703 "// \\endverbatim\n";704 705 std::vector<Token> Toks;706 707 lexString(Source, Toks);708 709 ASSERT_EQ(10U, Toks.size());710 711 ASSERT_EQ(tok::text, Toks[0].getKind());712 ASSERT_EQ(StringRef(" "), Toks[0].getText());713 714 ASSERT_EQ(tok::verbatim_block_begin, Toks[1].getKind());715 ASSERT_EQ(StringRef("verbatim"), getVerbatimBlockName(Toks[1]));716 717 ASSERT_EQ(tok::newline, Toks[2].getKind());718 719 ASSERT_EQ(tok::verbatim_block_line, Toks[3].getKind());720 ASSERT_EQ(StringRef(" Aaa"), Toks[3].getVerbatimBlockText());721 722 ASSERT_EQ(tok::newline, Toks[4].getKind());723 724 ASSERT_EQ(tok::newline, Toks[5].getKind());725 726 ASSERT_EQ(tok::verbatim_block_line, Toks[6].getKind());727 ASSERT_EQ(StringRef(" Bbb"), Toks[6].getVerbatimBlockText());728 729 ASSERT_EQ(tok::newline, Toks[7].getKind());730 731 ASSERT_EQ(tok::verbatim_block_end, Toks[8].getKind());732 ASSERT_EQ(StringRef("endverbatim"), getVerbatimBlockName(Toks[8]));733 734 ASSERT_EQ(tok::newline, Toks[9].getKind());735}736 737TEST_F(CommentLexerTest, VerbatimBlock7) {738 const char *Source =739 "/* \\verbatim\n"740 " * Aaa\n"741 " *\n"742 " * Bbb\n"743 " * \\endverbatim\n"744 " */";745 746 std::vector<Token> Toks;747 748 lexString(Source, Toks);749 750 ASSERT_EQ(10U, Toks.size());751 752 ASSERT_EQ(tok::text, Toks[0].getKind());753 ASSERT_EQ(StringRef(" "), Toks[0].getText());754 755 ASSERT_EQ(tok::verbatim_block_begin, Toks[1].getKind());756 ASSERT_EQ(StringRef("verbatim"), getVerbatimBlockName(Toks[1]));757 758 ASSERT_EQ(tok::verbatim_block_line, Toks[2].getKind());759 ASSERT_EQ(StringRef(" Aaa"), Toks[2].getVerbatimBlockText());760 761 ASSERT_EQ(tok::verbatim_block_line, Toks[3].getKind());762 ASSERT_EQ(StringRef(""), Toks[3].getVerbatimBlockText());763 764 ASSERT_EQ(tok::verbatim_block_line, Toks[4].getKind());765 ASSERT_EQ(StringRef(" Bbb"), Toks[4].getVerbatimBlockText());766 767 ASSERT_EQ(tok::verbatim_block_end, Toks[5].getKind());768 ASSERT_EQ(StringRef("endverbatim"), getVerbatimBlockName(Toks[5]));769 770 ASSERT_EQ(tok::newline, Toks[6].getKind());771 772 ASSERT_EQ(tok::text, Toks[7].getKind());773 ASSERT_EQ(StringRef(" "), Toks[7].getText());774 775 ASSERT_EQ(tok::newline, Toks[8].getKind());776 ASSERT_EQ(tok::newline, Toks[9].getKind());777}778 779// Complex test for verbatim blocks.780TEST_F(CommentLexerTest, VerbatimBlock8) {781 const char *Source =782 "/* Meow \\verbatim aaa\\$\\@\n"783 "bbb \\endverbati\r"784 "ccc\r\n"785 "ddd \\endverbatim Blah \\verbatim eee\n"786 "\\endverbatim BlahBlah*/";787 std::vector<Token> Toks;788 789 lexString(Source, Toks);790 791 ASSERT_EQ(14U, Toks.size());792 793 ASSERT_EQ(tok::text, Toks[0].getKind());794 ASSERT_EQ(StringRef(" Meow "), Toks[0].getText());795 796 ASSERT_EQ(tok::verbatim_block_begin, Toks[1].getKind());797 ASSERT_EQ(StringRef("verbatim"), getVerbatimBlockName(Toks[1]));798 799 ASSERT_EQ(tok::verbatim_block_line, Toks[2].getKind());800 ASSERT_EQ(StringRef(" aaa\\$\\@"), Toks[2].getVerbatimBlockText());801 802 ASSERT_EQ(tok::verbatim_block_line, Toks[3].getKind());803 ASSERT_EQ(StringRef("bbb \\endverbati"), Toks[3].getVerbatimBlockText());804 805 ASSERT_EQ(tok::verbatim_block_line, Toks[4].getKind());806 ASSERT_EQ(StringRef("ccc"), Toks[4].getVerbatimBlockText());807 808 ASSERT_EQ(tok::verbatim_block_line, Toks[5].getKind());809 ASSERT_EQ(StringRef("ddd "), Toks[5].getVerbatimBlockText());810 811 ASSERT_EQ(tok::verbatim_block_end, Toks[6].getKind());812 ASSERT_EQ(StringRef("endverbatim"), getVerbatimBlockName(Toks[6]));813 814 ASSERT_EQ(tok::text, Toks[7].getKind());815 ASSERT_EQ(StringRef(" Blah "), Toks[7].getText());816 817 ASSERT_EQ(tok::verbatim_block_begin, Toks[8].getKind());818 ASSERT_EQ(StringRef("verbatim"), getVerbatimBlockName(Toks[8]));819 820 ASSERT_EQ(tok::verbatim_block_line, Toks[9].getKind());821 ASSERT_EQ(StringRef(" eee"), Toks[9].getVerbatimBlockText());822 823 ASSERT_EQ(tok::verbatim_block_end, Toks[10].getKind());824 ASSERT_EQ(StringRef("endverbatim"), getVerbatimBlockName(Toks[10]));825 826 ASSERT_EQ(tok::text, Toks[11].getKind());827 ASSERT_EQ(StringRef(" BlahBlah"), Toks[11].getText());828 829 ASSERT_EQ(tok::newline, Toks[12].getKind());830 ASSERT_EQ(tok::newline, Toks[13].getKind());831}832 833// LaTeX verbatim blocks.834TEST_F(CommentLexerTest, VerbatimBlock9) {835 const char *Source =836 "/// \\f$ Aaa \\f$ \\f[ Bbb \\f] \\f{ Ccc \\f} \\f( Ddd \\f)";837 std::vector<Token> Toks;838 839 lexString(Source, Toks);840 841 ASSERT_EQ(17U, Toks.size());842 843 ASSERT_EQ(tok::text, Toks[0].getKind());844 ASSERT_EQ(StringRef(" "), Toks[0].getText());845 846 ASSERT_EQ(tok::verbatim_block_begin, Toks[1].getKind());847 ASSERT_EQ(StringRef("f$"), getVerbatimBlockName(Toks[1]));848 849 ASSERT_EQ(tok::verbatim_block_line, Toks[2].getKind());850 ASSERT_EQ(StringRef(" Aaa "), Toks[2].getVerbatimBlockText());851 852 ASSERT_EQ(tok::verbatim_block_end, Toks[3].getKind());853 ASSERT_EQ(StringRef("f$"), getVerbatimBlockName(Toks[3]));854 855 ASSERT_EQ(tok::text, Toks[4].getKind());856 ASSERT_EQ(StringRef(" "), Toks[4].getText());857 858 ASSERT_EQ(tok::verbatim_block_begin, Toks[5].getKind());859 ASSERT_EQ(StringRef("f["), getVerbatimBlockName(Toks[5]));860 861 ASSERT_EQ(tok::verbatim_block_line, Toks[6].getKind());862 ASSERT_EQ(StringRef(" Bbb "), Toks[6].getVerbatimBlockText());863 864 ASSERT_EQ(tok::verbatim_block_end, Toks[7].getKind());865 ASSERT_EQ(StringRef("f]"), getVerbatimBlockName(Toks[7]));866 867 ASSERT_EQ(tok::text, Toks[8].getKind());868 ASSERT_EQ(StringRef(" "), Toks[8].getText());869 870 ASSERT_EQ(tok::verbatim_block_begin, Toks[9].getKind());871 ASSERT_EQ(StringRef("f{"), getVerbatimBlockName(Toks[9]));872 873 ASSERT_EQ(tok::verbatim_block_line, Toks[10].getKind());874 ASSERT_EQ(StringRef(" Ccc "), Toks[10].getVerbatimBlockText());875 876 ASSERT_EQ(tok::verbatim_block_end, Toks[11].getKind());877 ASSERT_EQ(StringRef("f}"), getVerbatimBlockName(Toks[11]));878 879 ASSERT_EQ(tok::text, Toks[12].getKind());880 ASSERT_EQ(StringRef(" "), Toks[12].getText());881 882 ASSERT_EQ(tok::verbatim_block_begin, Toks[13].getKind());883 ASSERT_EQ(StringRef("f("), getVerbatimBlockName(Toks[13]));884 885 ASSERT_EQ(tok::verbatim_block_line, Toks[14].getKind());886 ASSERT_EQ(StringRef(" Ddd "), Toks[14].getVerbatimBlockText());887 888 ASSERT_EQ(tok::verbatim_block_end, Toks[15].getKind());889 ASSERT_EQ(StringRef("f)"), getVerbatimBlockName(Toks[15]));890 891 ASSERT_EQ(tok::newline, Toks[16].getKind());892}893 894// Empty verbatim line.895TEST_F(CommentLexerTest, VerbatimLine1) {896 const char *Sources[] = {897 "/// \\fn\n//",898 "/** \\fn*/"899 };900 901 for (size_t i = 0, e = std::size(Sources); i != e; i++) {902 std::vector<Token> Toks;903 904 lexString(Sources[i], Toks);905 906 ASSERT_EQ(4U, Toks.size());907 908 ASSERT_EQ(tok::text, Toks[0].getKind());909 ASSERT_EQ(StringRef(" "), Toks[0].getText());910 911 ASSERT_EQ(tok::verbatim_line_name, Toks[1].getKind());912 ASSERT_EQ(StringRef("fn"), getVerbatimLineName(Toks[1]));913 914 ASSERT_EQ(tok::newline, Toks[2].getKind());915 ASSERT_EQ(tok::newline, Toks[3].getKind());916 }917}918 919// Verbatim line with Doxygen escape sequences, which should not be expanded.920TEST_F(CommentLexerTest, VerbatimLine2) {921 const char *Sources[] = {922 "/// \\fn void *foo(const char *zzz = \"\\$\");\n//",923 "/** \\fn void *foo(const char *zzz = \"\\$\");*/"924 };925 926 for (size_t i = 0, e = std::size(Sources); i != e; i++) {927 std::vector<Token> Toks;928 929 lexString(Sources[i], Toks);930 931 ASSERT_EQ(5U, Toks.size());932 933 ASSERT_EQ(tok::text, Toks[0].getKind());934 ASSERT_EQ(StringRef(" "), Toks[0].getText());935 936 ASSERT_EQ(tok::verbatim_line_name, Toks[1].getKind());937 ASSERT_EQ(StringRef("fn"), getVerbatimLineName(Toks[1]));938 939 ASSERT_EQ(tok::verbatim_line_text, Toks[2].getKind());940 ASSERT_EQ(StringRef(" void *foo(const char *zzz = \"\\$\");"),941 Toks[2].getVerbatimLineText());942 943 ASSERT_EQ(tok::newline, Toks[3].getKind());944 ASSERT_EQ(tok::newline, Toks[4].getKind());945 }946}947 948// Verbatim line should not eat anything from next source line.949TEST_F(CommentLexerTest, VerbatimLine3) {950 const char *Source =951 "/** \\fn void *foo(const char *zzz = \"\\$\");\n"952 " * Meow\n"953 " */";954 955 std::vector<Token> Toks;956 957 lexString(Source, Toks);958 959 ASSERT_EQ(9U, Toks.size());960 961 ASSERT_EQ(tok::text, Toks[0].getKind());962 ASSERT_EQ(StringRef(" "), Toks[0].getText());963 964 ASSERT_EQ(tok::verbatim_line_name, Toks[1].getKind());965 ASSERT_EQ(StringRef("fn"), getVerbatimLineName(Toks[1]));966 967 ASSERT_EQ(tok::verbatim_line_text, Toks[2].getKind());968 ASSERT_EQ(StringRef(" void *foo(const char *zzz = \"\\$\");"),969 Toks[2].getVerbatimLineText());970 ASSERT_EQ(tok::newline, Toks[3].getKind());971 972 ASSERT_EQ(tok::text, Toks[4].getKind());973 ASSERT_EQ(StringRef(" Meow"), Toks[4].getText());974 ASSERT_EQ(tok::newline, Toks[5].getKind());975 976 ASSERT_EQ(tok::text, Toks[6].getKind());977 ASSERT_EQ(StringRef(" "), Toks[6].getText());978 979 ASSERT_EQ(tok::newline, Toks[7].getKind());980 ASSERT_EQ(tok::newline, Toks[8].getKind());981}982 983TEST_F(CommentLexerTest, HTML1) {984 const char *Source =985 "// <";986 987 std::vector<Token> Toks;988 989 lexString(Source, Toks);990 991 ASSERT_EQ(3U, Toks.size());992 993 ASSERT_EQ(tok::text, Toks[0].getKind());994 ASSERT_EQ(StringRef(" "), Toks[0].getText());995 996 ASSERT_EQ(tok::text, Toks[1].getKind());997 ASSERT_EQ(StringRef("<"), Toks[1].getText());998 999 ASSERT_EQ(tok::newline, Toks[2].getKind());1000}1001 1002TEST_F(CommentLexerTest, HTML2) {1003 const char *Source =1004 "// a<2";1005 1006 std::vector<Token> Toks;1007 1008 lexString(Source, Toks);1009 1010 ASSERT_EQ(4U, Toks.size());1011 1012 ASSERT_EQ(tok::text, Toks[0].getKind());1013 ASSERT_EQ(StringRef(" a"), Toks[0].getText());1014 1015 ASSERT_EQ(tok::text, Toks[1].getKind());1016 ASSERT_EQ(StringRef("<"), Toks[1].getText());1017 1018 ASSERT_EQ(tok::text, Toks[2].getKind());1019 ASSERT_EQ(StringRef("2"), Toks[2].getText());1020 1021 ASSERT_EQ(tok::newline, Toks[3].getKind());1022}1023 1024TEST_F(CommentLexerTest, HTML3) {1025 const char *Source =1026 "// < img";1027 1028 std::vector<Token> Toks;1029 1030 lexString(Source, Toks);1031 1032 ASSERT_EQ(4U, Toks.size());1033 1034 ASSERT_EQ(tok::text, Toks[0].getKind());1035 ASSERT_EQ(StringRef(" "), Toks[0].getText());1036 1037 ASSERT_EQ(tok::text, Toks[1].getKind());1038 ASSERT_EQ(StringRef("<"), Toks[1].getText());1039 1040 ASSERT_EQ(tok::text, Toks[2].getKind());1041 ASSERT_EQ(StringRef(" img"), Toks[2].getText());1042 1043 ASSERT_EQ(tok::newline, Toks[3].getKind());1044}1045 1046TEST_F(CommentLexerTest, HTML4) {1047 const char *Sources[] = {1048 "// <img",1049 "// <img "1050 };1051 1052 for (size_t i = 0, e = std::size(Sources); i != e; i++) {1053 std::vector<Token> Toks;1054 1055 lexString(Sources[i], Toks);1056 1057 ASSERT_EQ(3U, Toks.size());1058 1059 ASSERT_EQ(tok::text, Toks[0].getKind());1060 ASSERT_EQ(StringRef(" "), Toks[0].getText());1061 1062 ASSERT_EQ(tok::html_start_tag, Toks[1].getKind());1063 ASSERT_EQ(StringRef("img"), Toks[1].getHTMLTagStartName());1064 1065 ASSERT_EQ(tok::newline, Toks[2].getKind());1066 }1067}1068 1069TEST_F(CommentLexerTest, HTML5) {1070 const char *Source =1071 "// <img 42";1072 1073 std::vector<Token> Toks;1074 1075 lexString(Source, Toks);1076 1077 ASSERT_EQ(4U, Toks.size());1078 1079 ASSERT_EQ(tok::text, Toks[0].getKind());1080 ASSERT_EQ(StringRef(" "), Toks[0].getText());1081 1082 ASSERT_EQ(tok::html_start_tag, Toks[1].getKind());1083 ASSERT_EQ(StringRef("img"), Toks[1].getHTMLTagStartName());1084 1085 ASSERT_EQ(tok::text, Toks[2].getKind());1086 ASSERT_EQ(StringRef("42"), Toks[2].getText());1087 1088 ASSERT_EQ(tok::newline, Toks[3].getKind());1089}1090 1091TEST_F(CommentLexerTest, HTML6) {1092 const char *Source = "// <img> Meow";1093 1094 std::vector<Token> Toks;1095 1096 lexString(Source, Toks);1097 1098 ASSERT_EQ(5U, Toks.size());1099 1100 ASSERT_EQ(tok::text, Toks[0].getKind());1101 ASSERT_EQ(StringRef(" "), Toks[0].getText());1102 1103 ASSERT_EQ(tok::html_start_tag, Toks[1].getKind());1104 ASSERT_EQ(StringRef("img"), Toks[1].getHTMLTagStartName());1105 1106 ASSERT_EQ(tok::html_greater, Toks[2].getKind());1107 1108 ASSERT_EQ(tok::text, Toks[3].getKind());1109 ASSERT_EQ(StringRef(" Meow"), Toks[3].getText());1110 1111 ASSERT_EQ(tok::newline, Toks[4].getKind());1112}1113 1114TEST_F(CommentLexerTest, HTML7) {1115 const char *Source = "// <img=";1116 1117 std::vector<Token> Toks;1118 1119 lexString(Source, Toks);1120 1121 ASSERT_EQ(4U, Toks.size());1122 1123 ASSERT_EQ(tok::text, Toks[0].getKind());1124 ASSERT_EQ(StringRef(" "), Toks[0].getText());1125 1126 ASSERT_EQ(tok::html_start_tag, Toks[1].getKind());1127 ASSERT_EQ(StringRef("img"), Toks[1].getHTMLTagStartName());1128 1129 ASSERT_EQ(tok::text, Toks[2].getKind());1130 ASSERT_EQ(StringRef("="), Toks[2].getText());1131 1132 ASSERT_EQ(tok::newline, Toks[3].getKind());1133}1134 1135TEST_F(CommentLexerTest, HTML8) {1136 const char *Source = "// <img src=> Meow";1137 1138 std::vector<Token> Toks;1139 1140 lexString(Source, Toks);1141 1142 ASSERT_EQ(7U, Toks.size());1143 1144 ASSERT_EQ(tok::text, Toks[0].getKind());1145 ASSERT_EQ(StringRef(" "), Toks[0].getText());1146 1147 ASSERT_EQ(tok::html_start_tag, Toks[1].getKind());1148 ASSERT_EQ(StringRef("img"), Toks[1].getHTMLTagStartName());1149 1150 ASSERT_EQ(tok::html_ident, Toks[2].getKind());1151 ASSERT_EQ(StringRef("src"), Toks[2].getHTMLIdent());1152 1153 ASSERT_EQ(tok::html_equals, Toks[3].getKind());1154 1155 ASSERT_EQ(tok::html_greater, Toks[4].getKind());1156 1157 ASSERT_EQ(tok::text, Toks[5].getKind());1158 ASSERT_EQ(StringRef(" Meow"), Toks[5].getText());1159 1160 ASSERT_EQ(tok::newline, Toks[6].getKind());1161}1162 1163TEST_F(CommentLexerTest, HTML9) {1164 const char *Sources[] = {1165 "// <img src",1166 "// <img src "1167 };1168 1169 for (size_t i = 0, e = std::size(Sources); i != e; i++) {1170 std::vector<Token> Toks;1171 1172 lexString(Sources[i], Toks);1173 1174 ASSERT_EQ(4U, Toks.size());1175 1176 ASSERT_EQ(tok::text, Toks[0].getKind());1177 ASSERT_EQ(StringRef(" "), Toks[0].getText());1178 1179 ASSERT_EQ(tok::html_start_tag, Toks[1].getKind());1180 ASSERT_EQ(StringRef("img"), Toks[1].getHTMLTagStartName());1181 1182 ASSERT_EQ(tok::html_ident, Toks[2].getKind());1183 ASSERT_EQ(StringRef("src"), Toks[2].getHTMLIdent());1184 1185 ASSERT_EQ(tok::newline, Toks[3].getKind());1186 }1187}1188 1189TEST_F(CommentLexerTest, HTML10) {1190 const char *Sources[] = {1191 "// <img src=",1192 "// <img src ="1193 };1194 1195 for (size_t i = 0, e = std::size(Sources); i != e; i++) {1196 std::vector<Token> Toks;1197 1198 lexString(Sources[i], Toks);1199 1200 ASSERT_EQ(5U, Toks.size());1201 1202 ASSERT_EQ(tok::text, Toks[0].getKind());1203 ASSERT_EQ(StringRef(" "), Toks[0].getText());1204 1205 ASSERT_EQ(tok::html_start_tag, Toks[1].getKind());1206 ASSERT_EQ(StringRef("img"), Toks[1].getHTMLTagStartName());1207 1208 ASSERT_EQ(tok::html_ident, Toks[2].getKind());1209 ASSERT_EQ(StringRef("src"), Toks[2].getHTMLIdent());1210 1211 ASSERT_EQ(tok::html_equals, Toks[3].getKind());1212 1213 ASSERT_EQ(tok::newline, Toks[4].getKind());1214 }1215}1216 1217TEST_F(CommentLexerTest, HTML11) {1218 const char *Sources[] = {1219 "// <img src=\"",1220 "// <img src = \"",1221 "// <img src=\'",1222 "// <img src = \'"1223 };1224 1225 for (size_t i = 0, e = std::size(Sources); i != e; i++) {1226 std::vector<Token> Toks;1227 1228 lexString(Sources[i], Toks);1229 1230 ASSERT_EQ(6U, Toks.size());1231 1232 ASSERT_EQ(tok::text, Toks[0].getKind());1233 ASSERT_EQ(StringRef(" "), Toks[0].getText());1234 1235 ASSERT_EQ(tok::html_start_tag, Toks[1].getKind());1236 ASSERT_EQ(StringRef("img"), Toks[1].getHTMLTagStartName());1237 1238 ASSERT_EQ(tok::html_ident, Toks[2].getKind());1239 ASSERT_EQ(StringRef("src"), Toks[2].getHTMLIdent());1240 1241 ASSERT_EQ(tok::html_equals, Toks[3].getKind());1242 1243 ASSERT_EQ(tok::html_quoted_string, Toks[4].getKind());1244 ASSERT_EQ(StringRef(""), Toks[4].getHTMLQuotedString());1245 1246 ASSERT_EQ(tok::newline, Toks[5].getKind());1247 }1248}1249 1250TEST_F(CommentLexerTest, HTML12) {1251 const char *Source = "// <img src=@";1252 1253 std::vector<Token> Toks;1254 1255 lexString(Source, Toks);1256 1257 ASSERT_EQ(6U, Toks.size());1258 1259 ASSERT_EQ(tok::text, Toks[0].getKind());1260 ASSERT_EQ(StringRef(" "), Toks[0].getText());1261 1262 ASSERT_EQ(tok::html_start_tag, Toks[1].getKind());1263 ASSERT_EQ(StringRef("img"), Toks[1].getHTMLTagStartName());1264 1265 ASSERT_EQ(tok::html_ident, Toks[2].getKind());1266 ASSERT_EQ(StringRef("src"), Toks[2].getHTMLIdent());1267 1268 ASSERT_EQ(tok::html_equals, Toks[3].getKind());1269 1270 ASSERT_EQ(tok::text, Toks[4].getKind());1271 ASSERT_EQ(StringRef("@"), Toks[4].getText());1272 1273 ASSERT_EQ(tok::newline, Toks[5].getKind());1274}1275 1276TEST_F(CommentLexerTest, HTML13) {1277 const char *Sources[] = {1278 "// <img src=\"val\\\"\\'val",1279 "// <img src=\"val\\\"\\'val\"",1280 "// <img src=\'val\\\"\\'val",1281 "// <img src=\'val\\\"\\'val\'"1282 };1283 1284 for (size_t i = 0, e = std::size(Sources); i != e; i++) {1285 std::vector<Token> Toks;1286 1287 lexString(Sources[i], Toks);1288 1289 ASSERT_EQ(6U, Toks.size());1290 1291 ASSERT_EQ(tok::text, Toks[0].getKind());1292 ASSERT_EQ(StringRef(" "), Toks[0].getText());1293 1294 ASSERT_EQ(tok::html_start_tag, Toks[1].getKind());1295 ASSERT_EQ(StringRef("img"), Toks[1].getHTMLTagStartName());1296 1297 ASSERT_EQ(tok::html_ident, Toks[2].getKind());1298 ASSERT_EQ(StringRef("src"), Toks[2].getHTMLIdent());1299 1300 ASSERT_EQ(tok::html_equals, Toks[3].getKind());1301 1302 ASSERT_EQ(tok::html_quoted_string, Toks[4].getKind());1303 ASSERT_EQ(StringRef("val\\\"\\'val"), Toks[4].getHTMLQuotedString());1304 1305 ASSERT_EQ(tok::newline, Toks[5].getKind());1306 }1307}1308 1309TEST_F(CommentLexerTest, HTML14) {1310 const char *Sources[] = {1311 "// <img src=\"val\\\"\\'val\">",1312 "// <img src=\'val\\\"\\'val\'>"1313 };1314 1315 for (size_t i = 0, e = std::size(Sources); i != e; i++) {1316 std::vector<Token> Toks;1317 1318 lexString(Sources[i], Toks);1319 1320 ASSERT_EQ(7U, Toks.size());1321 1322 ASSERT_EQ(tok::text, Toks[0].getKind());1323 ASSERT_EQ(StringRef(" "), Toks[0].getText());1324 1325 ASSERT_EQ(tok::html_start_tag, Toks[1].getKind());1326 ASSERT_EQ(StringRef("img"), Toks[1].getHTMLTagStartName());1327 1328 ASSERT_EQ(tok::html_ident, Toks[2].getKind());1329 ASSERT_EQ(StringRef("src"), Toks[2].getHTMLIdent());1330 1331 ASSERT_EQ(tok::html_equals, Toks[3].getKind());1332 1333 ASSERT_EQ(tok::html_quoted_string, Toks[4].getKind());1334 ASSERT_EQ(StringRef("val\\\"\\'val"), Toks[4].getHTMLQuotedString());1335 1336 ASSERT_EQ(tok::html_greater, Toks[5].getKind());1337 1338 ASSERT_EQ(tok::newline, Toks[6].getKind());1339 }1340}1341 1342TEST_F(CommentLexerTest, HTML15) {1343 const char *Sources[] = {1344 "// <img/>",1345 "// <img />"1346 };1347 1348 for (size_t i = 0, e = std::size(Sources); i != e; i++) {1349 std::vector<Token> Toks;1350 1351 lexString(Sources[i], Toks);1352 1353 ASSERT_EQ(4U, Toks.size());1354 1355 ASSERT_EQ(tok::text, Toks[0].getKind());1356 ASSERT_EQ(StringRef(" "), Toks[0].getText());1357 1358 ASSERT_EQ(tok::html_start_tag, Toks[1].getKind());1359 ASSERT_EQ(StringRef("img"), Toks[1].getHTMLTagStartName());1360 1361 ASSERT_EQ(tok::html_slash_greater, Toks[2].getKind());1362 1363 ASSERT_EQ(tok::newline, Toks[3].getKind());1364 }1365}1366 1367TEST_F(CommentLexerTest, HTML16) {1368 const char *Sources[] = {1369 "// <img/ Aaa",1370 "// <img / Aaa"1371 };1372 1373 for (size_t i = 0, e = std::size(Sources); i != e; i++) {1374 std::vector<Token> Toks;1375 1376 lexString(Sources[i], Toks);1377 1378 ASSERT_EQ(5U, Toks.size());1379 1380 ASSERT_EQ(tok::text, Toks[0].getKind());1381 ASSERT_EQ(StringRef(" "), Toks[0].getText());1382 1383 ASSERT_EQ(tok::html_start_tag, Toks[1].getKind());1384 ASSERT_EQ(StringRef("img"), Toks[1].getHTMLTagStartName());1385 1386 ASSERT_EQ(tok::text, Toks[2].getKind());1387 ASSERT_EQ(StringRef("/"), Toks[2].getText());1388 1389 ASSERT_EQ(tok::text, Toks[3].getKind());1390 ASSERT_EQ(StringRef(" Aaa"), Toks[3].getText());1391 1392 ASSERT_EQ(tok::newline, Toks[4].getKind());1393 }1394}1395 1396TEST_F(CommentLexerTest, HTML17) {1397 const char *Source = "// </";1398 1399 std::vector<Token> Toks;1400 1401 lexString(Source, Toks);1402 1403 ASSERT_EQ(3U, Toks.size());1404 1405 ASSERT_EQ(tok::text, Toks[0].getKind());1406 ASSERT_EQ(StringRef(" "), Toks[0].getText());1407 1408 ASSERT_EQ(tok::text, Toks[1].getKind());1409 ASSERT_EQ(StringRef("</"), Toks[1].getText());1410 1411 ASSERT_EQ(tok::newline, Toks[2].getKind());1412}1413 1414TEST_F(CommentLexerTest, HTML18) {1415 const char *Source = "// </@";1416 1417 std::vector<Token> Toks;1418 1419 lexString(Source, Toks);1420 1421 ASSERT_EQ(4U, Toks.size());1422 1423 ASSERT_EQ(tok::text, Toks[0].getKind());1424 ASSERT_EQ(StringRef(" "), Toks[0].getText());1425 1426 ASSERT_EQ(tok::text, Toks[1].getKind());1427 ASSERT_EQ(StringRef("</"), Toks[1].getText());1428 1429 ASSERT_EQ(tok::text, Toks[2].getKind());1430 ASSERT_EQ(StringRef("@"), Toks[2].getText());1431 1432 ASSERT_EQ(tok::newline, Toks[3].getKind());1433}1434 1435TEST_F(CommentLexerTest, HTML19) {1436 const char *Source = "// </img";1437 1438 std::vector<Token> Toks;1439 1440 lexString(Source, Toks);1441 1442 ASSERT_EQ(3U, Toks.size());1443 1444 ASSERT_EQ(tok::text, Toks[0].getKind());1445 ASSERT_EQ(StringRef(" "), Toks[0].getText());1446 1447 ASSERT_EQ(tok::html_end_tag, Toks[1].getKind());1448 ASSERT_EQ(StringRef("img"), Toks[1].getHTMLTagEndName());1449 1450 ASSERT_EQ(tok::newline, Toks[2].getKind());1451}1452 1453TEST_F(CommentLexerTest, HTML20) {1454 const char *Source = "// <a\n"1455 "// \n"1456 "// href=\"foo\"\n"1457 "// \n"1458 "// bar>text</a>";1459 1460 std::vector<Token> Toks;1461 1462 lexString(Source, Toks);1463 1464 ASSERT_EQ(11U, Toks.size());1465 1466 ASSERT_EQ(tok::text, Toks[0].getKind());1467 ASSERT_EQ(StringRef(" "), Toks[0].getText());1468 1469 ASSERT_EQ(tok::html_start_tag, Toks[1].getKind());1470 ASSERT_EQ(StringRef("a"), Toks[1].getHTMLTagStartName());1471 1472 ASSERT_EQ(tok::html_ident, Toks[2].getKind());1473 ASSERT_EQ(StringRef("href"), Toks[2].getHTMLIdent());1474 1475 ASSERT_EQ(tok::html_equals, Toks[3].getKind());1476 1477 ASSERT_EQ(tok::html_quoted_string, Toks[4].getKind());1478 ASSERT_EQ(StringRef("foo"), Toks[4].getHTMLQuotedString());1479 1480 ASSERT_EQ(tok::html_ident, Toks[5].getKind());1481 ASSERT_EQ(StringRef("bar"), Toks[5].getHTMLIdent());1482 1483 ASSERT_EQ(tok::html_greater, Toks[6].getKind());1484 1485 ASSERT_EQ(tok::text, Toks[7].getKind());1486 ASSERT_EQ(StringRef("text"), Toks[7].getText());1487 1488 ASSERT_EQ(tok::html_end_tag, Toks[8].getKind());1489 ASSERT_EQ(StringRef("a"), Toks[8].getHTMLTagEndName());1490 1491 ASSERT_EQ(tok::html_greater, Toks[9].getKind());1492 1493 ASSERT_EQ(tok::newline, Toks[10].getKind());1494}1495 1496TEST_F(CommentLexerTest, HTML21) {1497 const char *Source = "/**\n"1498 " * <a\n"1499 " * \n"1500 " * href=\"foo\"\n"1501 " * \n"1502 " * bar>text</a>\n"1503 " */";1504 1505 std::vector<Token> Toks;1506 1507 lexString(Source, Toks);1508 1509 ASSERT_EQ(15U, Toks.size());1510 1511 ASSERT_EQ(tok::newline, Toks[0].getKind());1512 1513 ASSERT_EQ(tok::text, Toks[1].getKind());1514 ASSERT_EQ(StringRef(" "), Toks[1].getText());1515 1516 ASSERT_EQ(tok::html_start_tag, Toks[2].getKind());1517 ASSERT_EQ(StringRef("a"), Toks[2].getHTMLTagStartName());1518 1519 ASSERT_EQ(tok::html_ident, Toks[3].getKind());1520 ASSERT_EQ(StringRef("href"), Toks[3].getHTMLIdent());1521 1522 ASSERT_EQ(tok::html_equals, Toks[4].getKind());1523 1524 ASSERT_EQ(tok::html_quoted_string, Toks[5].getKind());1525 ASSERT_EQ(StringRef("foo"), Toks[5].getHTMLQuotedString());1526 1527 ASSERT_EQ(tok::html_ident, Toks[6].getKind());1528 ASSERT_EQ(StringRef("bar"), Toks[6].getHTMLIdent());1529 1530 ASSERT_EQ(tok::html_greater, Toks[7].getKind());1531 1532 ASSERT_EQ(tok::text, Toks[8].getKind());1533 ASSERT_EQ(StringRef("text"), Toks[8].getText());1534 1535 ASSERT_EQ(tok::html_end_tag, Toks[9].getKind());1536 ASSERT_EQ(StringRef("a"), Toks[9].getHTMLTagEndName());1537 1538 ASSERT_EQ(tok::html_greater, Toks[10].getKind());1539 1540 ASSERT_EQ(tok::newline, Toks[11].getKind());1541 1542 ASSERT_EQ(tok::text, Toks[12].getKind());1543 ASSERT_EQ(StringRef(" "), Toks[12].getText());1544 1545 ASSERT_EQ(tok::newline, Toks[13].getKind());1546 1547 ASSERT_EQ(tok::newline, Toks[14].getKind());1548}1549 1550TEST_F(CommentLexerTest, HTML22) {1551 const char *Source = "/**\n"1552 " * <a\n"1553 " */";1554 1555 std::vector<Token> Toks;1556 1557 lexString(Source, Toks);1558 1559 ASSERT_EQ(6U, Toks.size());1560 1561 ASSERT_EQ(tok::newline, Toks[0].getKind());1562 1563 ASSERT_EQ(tok::text, Toks[1].getKind());1564 ASSERT_EQ(StringRef(" "), Toks[1].getText());1565 1566 ASSERT_EQ(tok::html_start_tag, Toks[2].getKind());1567 ASSERT_EQ(StringRef("a"), Toks[2].getHTMLTagStartName());1568 1569 ASSERT_EQ(tok::newline, Toks[3].getKind());1570 1571 ASSERT_EQ(tok::newline, Toks[4].getKind());1572 1573 ASSERT_EQ(tok::newline, Toks[5].getKind());1574}1575 1576TEST_F(CommentLexerTest, HTML23) {1577 // NOTE: "//<" is considered a comment start1578 const char *Source = "// <\n"1579 "// a\n"1580 "// >";1581 1582 std::vector<Token> Toks;1583 1584 lexString(Source, Toks);1585 1586 ASSERT_EQ(7U, Toks.size());1587 1588 ASSERT_EQ(tok::text, Toks[0].getKind());1589 ASSERT_EQ(StringRef(" "), Toks[0].getText());1590 1591 ASSERT_EQ(tok::text, Toks[1].getKind());1592 ASSERT_EQ(StringRef("<"), Toks[1].getText());1593 1594 ASSERT_EQ(tok::newline, Toks[2].getKind());1595 1596 ASSERT_EQ(tok::text, Toks[3].getKind());1597 ASSERT_EQ(StringRef(" a"), Toks[3].getText());1598 1599 ASSERT_EQ(tok::newline, Toks[4].getKind());1600 1601 ASSERT_EQ(tok::text, Toks[5].getKind());1602 ASSERT_EQ(StringRef(" >"), Toks[5].getText());1603 1604 ASSERT_EQ(tok::newline, Toks[6].getKind());1605}1606 1607TEST_F(CommentLexerTest, NotAKnownHTMLTag1) {1608 const char *Source = "// <tag>";1609 1610 std::vector<Token> Toks;1611 1612 lexString(Source, Toks);1613 1614 ASSERT_EQ(4U, Toks.size());1615 1616 ASSERT_EQ(tok::text, Toks[0].getKind());1617 ASSERT_EQ(StringRef(" "), Toks[0].getText());1618 1619 ASSERT_EQ(tok::text, Toks[1].getKind());1620 ASSERT_EQ(StringRef("<tag"), Toks[1].getText());1621 1622 ASSERT_EQ(tok::text, Toks[2].getKind());1623 ASSERT_EQ(StringRef(">"), Toks[2].getText());1624 1625 ASSERT_EQ(tok::newline, Toks[3].getKind());1626}1627 1628TEST_F(CommentLexerTest, NotAKnownHTMLTag2) {1629 const char *Source = "// </tag>";1630 1631 std::vector<Token> Toks;1632 1633 lexString(Source, Toks);1634 1635 ASSERT_EQ(4U, Toks.size());1636 1637 ASSERT_EQ(tok::text, Toks[0].getKind());1638 ASSERT_EQ(StringRef(" "), Toks[0].getText());1639 1640 ASSERT_EQ(tok::text, Toks[1].getKind());1641 ASSERT_EQ(StringRef("</tag"), Toks[1].getText());1642 1643 ASSERT_EQ(tok::text, Toks[2].getKind());1644 ASSERT_EQ(StringRef(">"), Toks[2].getText());1645 1646 ASSERT_EQ(tok::newline, Toks[3].getKind());1647}1648 1649TEST_F(CommentLexerTest, HTMLCharacterReferences1) {1650 const char *Source = "// &";1651 1652 std::vector<Token> Toks;1653 1654 lexString(Source, Toks);1655 1656 ASSERT_EQ(3U, Toks.size());1657 1658 ASSERT_EQ(tok::text, Toks[0].getKind());1659 ASSERT_EQ(StringRef(" "), Toks[0].getText());1660 1661 ASSERT_EQ(tok::text, Toks[1].getKind());1662 ASSERT_EQ(StringRef("&"), Toks[1].getText());1663 1664 ASSERT_EQ(tok::newline, Toks[2].getKind());1665}1666 1667TEST_F(CommentLexerTest, HTMLCharacterReferences2) {1668 const char *Source = "// &!";1669 1670 std::vector<Token> Toks;1671 1672 lexString(Source, Toks);1673 1674 ASSERT_EQ(4U, Toks.size());1675 1676 ASSERT_EQ(tok::text, Toks[0].getKind());1677 ASSERT_EQ(StringRef(" "), Toks[0].getText());1678 1679 ASSERT_EQ(tok::text, Toks[1].getKind());1680 ASSERT_EQ(StringRef("&"), Toks[1].getText());1681 1682 ASSERT_EQ(tok::text, Toks[2].getKind());1683 ASSERT_EQ(StringRef("!"), Toks[2].getText());1684 1685 ASSERT_EQ(tok::newline, Toks[3].getKind());1686}1687 1688TEST_F(CommentLexerTest, HTMLCharacterReferences3) {1689 const char *Source = "// &";1690 1691 std::vector<Token> Toks;1692 1693 lexString(Source, Toks);1694 1695 ASSERT_EQ(3U, Toks.size());1696 1697 ASSERT_EQ(tok::text, Toks[0].getKind());1698 ASSERT_EQ(StringRef(" "), Toks[0].getText());1699 1700 ASSERT_EQ(tok::text, Toks[1].getKind());1701 ASSERT_EQ(StringRef("&"), Toks[1].getText());1702 1703 ASSERT_EQ(tok::newline, Toks[2].getKind());1704}1705 1706TEST_F(CommentLexerTest, HTMLCharacterReferences4) {1707 const char *Source = "// &!";1708 1709 std::vector<Token> Toks;1710 1711 lexString(Source, Toks);1712 1713 ASSERT_EQ(4U, Toks.size());1714 1715 ASSERT_EQ(tok::text, Toks[0].getKind());1716 ASSERT_EQ(StringRef(" "), Toks[0].getText());1717 1718 ASSERT_EQ(tok::text, Toks[1].getKind());1719 ASSERT_EQ(StringRef("&"), Toks[1].getText());1720 1721 ASSERT_EQ(tok::text, Toks[2].getKind());1722 ASSERT_EQ(StringRef("!"), Toks[2].getText());1723 1724 ASSERT_EQ(tok::newline, Toks[3].getKind());1725}1726 1727TEST_F(CommentLexerTest, HTMLCharacterReferences5) {1728 const char *Source = "// &#";1729 1730 std::vector<Token> Toks;1731 1732 lexString(Source, Toks);1733 1734 ASSERT_EQ(3U, Toks.size());1735 1736 ASSERT_EQ(tok::text, Toks[0].getKind());1737 ASSERT_EQ(StringRef(" "), Toks[0].getText());1738 1739 ASSERT_EQ(tok::text, Toks[1].getKind());1740 ASSERT_EQ(StringRef("&#"), Toks[1].getText());1741 1742 ASSERT_EQ(tok::newline, Toks[2].getKind());1743}1744 1745TEST_F(CommentLexerTest, HTMLCharacterReferences6) {1746 const char *Source = "// &#a";1747 1748 std::vector<Token> Toks;1749 1750 lexString(Source, Toks);1751 1752 ASSERT_EQ(4U, Toks.size());1753 1754 ASSERT_EQ(tok::text, Toks[0].getKind());1755 ASSERT_EQ(StringRef(" "), Toks[0].getText());1756 1757 ASSERT_EQ(tok::text, Toks[1].getKind());1758 ASSERT_EQ(StringRef("&#"), Toks[1].getText());1759 1760 ASSERT_EQ(tok::text, Toks[2].getKind());1761 ASSERT_EQ(StringRef("a"), Toks[2].getText());1762 1763 ASSERT_EQ(tok::newline, Toks[3].getKind());1764}1765 1766TEST_F(CommentLexerTest, HTMLCharacterReferences7) {1767 const char *Source = "// *";1768 1769 std::vector<Token> Toks;1770 1771 lexString(Source, Toks);1772 1773 ASSERT_EQ(3U, Toks.size());1774 1775 ASSERT_EQ(tok::text, Toks[0].getKind());1776 ASSERT_EQ(StringRef(" "), Toks[0].getText());1777 1778 ASSERT_EQ(tok::text, Toks[1].getKind());1779 ASSERT_EQ(StringRef("*"), Toks[1].getText());1780 1781 ASSERT_EQ(tok::newline, Toks[2].getKind());1782}1783 1784TEST_F(CommentLexerTest, HTMLCharacterReferences8) {1785 const char *Source = "// *a";1786 1787 std::vector<Token> Toks;1788 1789 lexString(Source, Toks);1790 1791 ASSERT_EQ(4U, Toks.size());1792 1793 ASSERT_EQ(tok::text, Toks[0].getKind());1794 ASSERT_EQ(StringRef(" "), Toks[0].getText());1795 1796 ASSERT_EQ(tok::text, Toks[1].getKind());1797 ASSERT_EQ(StringRef("*"), Toks[1].getText());1798 1799 ASSERT_EQ(tok::text, Toks[2].getKind());1800 ASSERT_EQ(StringRef("a"), Toks[2].getText());1801 1802 ASSERT_EQ(tok::newline, Toks[3].getKind());1803}1804 1805TEST_F(CommentLexerTest, HTMLCharacterReferences9) {1806 const char *Source = "// &#x";1807 1808 std::vector<Token> Toks;1809 1810 lexString(Source, Toks);1811 1812 ASSERT_EQ(3U, Toks.size());1813 1814 ASSERT_EQ(tok::text, Toks[0].getKind());1815 ASSERT_EQ(StringRef(" "), Toks[0].getText());1816 1817 ASSERT_EQ(tok::text, Toks[1].getKind());1818 ASSERT_EQ(StringRef("&#x"), Toks[1].getText());1819 1820 ASSERT_EQ(tok::newline, Toks[2].getKind());1821}1822 1823TEST_F(CommentLexerTest, HTMLCharacterReferences10) {1824 const char *Source = "// &#xz";1825 1826 std::vector<Token> Toks;1827 1828 lexString(Source, Toks);1829 1830 ASSERT_EQ(4U, Toks.size());1831 1832 ASSERT_EQ(tok::text, Toks[0].getKind());1833 ASSERT_EQ(StringRef(" "), Toks[0].getText());1834 1835 ASSERT_EQ(tok::text, Toks[1].getKind());1836 ASSERT_EQ(StringRef("&#x"), Toks[1].getText());1837 1838 ASSERT_EQ(tok::text, Toks[2].getKind());1839 ASSERT_EQ(StringRef("z"), Toks[2].getText());1840 1841 ASSERT_EQ(tok::newline, Toks[3].getKind());1842}1843 1844TEST_F(CommentLexerTest, HTMLCharacterReferences11) {1845 const char *Source = "// «";1846 1847 std::vector<Token> Toks;1848 1849 lexString(Source, Toks);1850 1851 ASSERT_EQ(3U, Toks.size());1852 1853 ASSERT_EQ(tok::text, Toks[0].getKind());1854 ASSERT_EQ(StringRef(" "), Toks[0].getText());1855 1856 ASSERT_EQ(tok::text, Toks[1].getKind());1857 ASSERT_EQ(StringRef("«"), Toks[1].getText());1858 1859 ASSERT_EQ(tok::newline, Toks[2].getKind());1860}1861 1862TEST_F(CommentLexerTest, HTMLCharacterReferences12) {1863 const char *Source = "// «z";1864 1865 std::vector<Token> Toks;1866 1867 lexString(Source, Toks);1868 1869 ASSERT_EQ(4U, Toks.size());1870 1871 ASSERT_EQ(tok::text, Toks[0].getKind());1872 ASSERT_EQ(StringRef(" "), Toks[0].getText());1873 1874 ASSERT_EQ(tok::text, Toks[1].getKind());1875 ASSERT_EQ(StringRef("«"), Toks[1].getText());1876 1877 ASSERT_EQ(tok::text, Toks[2].getKind());1878 ASSERT_EQ(StringRef("z"), Toks[2].getText());1879 1880 ASSERT_EQ(tok::newline, Toks[3].getKind());1881}1882 1883TEST_F(CommentLexerTest, HTMLCharacterReferences13) {1884 const char *Source = "// &";1885 1886 std::vector<Token> Toks;1887 1888 lexString(Source, Toks);1889 1890 ASSERT_EQ(3U, Toks.size());1891 1892 ASSERT_EQ(tok::text, Toks[0].getKind());1893 ASSERT_EQ(StringRef(" "), Toks[0].getText());1894 1895 ASSERT_EQ(tok::text, Toks[1].getKind());1896 ASSERT_EQ(StringRef("&"), Toks[1].getText());1897 1898 ASSERT_EQ(tok::newline, Toks[2].getKind());1899}1900 1901TEST_F(CommentLexerTest, HTMLCharacterReferences14) {1902 const char *Source = "// &<";1903 1904 std::vector<Token> Toks;1905 1906 lexString(Source, Toks);1907 1908 ASSERT_EQ(4U, Toks.size());1909 1910 ASSERT_EQ(tok::text, Toks[0].getKind());1911 ASSERT_EQ(StringRef(" "), Toks[0].getText());1912 1913 ASSERT_EQ(tok::text, Toks[1].getKind());1914 ASSERT_EQ(StringRef("&"), Toks[1].getText());1915 1916 ASSERT_EQ(tok::text, Toks[2].getKind());1917 ASSERT_EQ(StringRef("<"), Toks[2].getText());1918 1919 ASSERT_EQ(tok::newline, Toks[3].getKind());1920}1921 1922TEST_F(CommentLexerTest, HTMLCharacterReferences15) {1923 const char *Source = "// & meow";1924 1925 std::vector<Token> Toks;1926 1927 lexString(Source, Toks);1928 1929 ASSERT_EQ(4U, Toks.size());1930 1931 ASSERT_EQ(tok::text, Toks[0].getKind());1932 ASSERT_EQ(StringRef(" "), Toks[0].getText());1933 1934 ASSERT_EQ(tok::text, Toks[1].getKind());1935 ASSERT_EQ(StringRef("&"), Toks[1].getText());1936 1937 ASSERT_EQ(tok::text, Toks[2].getKind());1938 ASSERT_EQ(StringRef(" meow"), Toks[2].getText());1939 1940 ASSERT_EQ(tok::newline, Toks[3].getKind());1941}1942 1943TEST_F(CommentLexerTest, HTMLCharacterReferences16) {1944 const char *Sources[] = {1945 "// =",1946 "// =",1947 "// =",1948 "// ="1949 };1950 1951 for (size_t i = 0, e = std::size(Sources); i != e; i++) {1952 std::vector<Token> Toks;1953 1954 lexString(Sources[i], Toks);1955 1956 ASSERT_EQ(3U, Toks.size());1957 1958 ASSERT_EQ(tok::text, Toks[0].getKind());1959 ASSERT_EQ(StringRef(" "), Toks[0].getText());1960 1961 ASSERT_EQ(tok::text, Toks[1].getKind());1962 ASSERT_EQ(StringRef("="), Toks[1].getText());1963 1964 ASSERT_EQ(tok::newline, Toks[2].getKind());1965 }1966}1967 1968TEST_F(CommentLexerTest, MultipleComments) {1969 const char *Source =1970 "// Aaa\n"1971 "/// Bbb\n"1972 "/* Ccc\n"1973 " * Ddd*/\n"1974 "/** Eee*/";1975 1976 std::vector<Token> Toks;1977 1978 lexString(Source, Toks);1979 1980 ASSERT_EQ(12U, Toks.size());1981 1982 ASSERT_EQ(tok::text, Toks[0].getKind());1983 ASSERT_EQ(StringRef(" Aaa"), Toks[0].getText());1984 ASSERT_EQ(tok::newline, Toks[1].getKind());1985 1986 ASSERT_EQ(tok::text, Toks[2].getKind());1987 ASSERT_EQ(StringRef(" Bbb"), Toks[2].getText());1988 ASSERT_EQ(tok::newline, Toks[3].getKind());1989 1990 ASSERT_EQ(tok::text, Toks[4].getKind());1991 ASSERT_EQ(StringRef(" Ccc"), Toks[4].getText());1992 ASSERT_EQ(tok::newline, Toks[5].getKind());1993 1994 ASSERT_EQ(tok::text, Toks[6].getKind());1995 ASSERT_EQ(StringRef(" Ddd"), Toks[6].getText());1996 ASSERT_EQ(tok::newline, Toks[7].getKind());1997 ASSERT_EQ(tok::newline, Toks[8].getKind());1998 1999 ASSERT_EQ(tok::text, Toks[9].getKind());2000 ASSERT_EQ(StringRef(" Eee"), Toks[9].getText());2001 2002 ASSERT_EQ(tok::newline, Toks[10].getKind());2003 ASSERT_EQ(tok::newline, Toks[11].getKind());2004}2005 2006} // end namespace comments2007} // end namespace clang2008