212 lines · cpp
1//===-- DILLexerTests.cpp --------------------------------------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8 9#include "lldb/ValueObject/DILLexer.h"10#include "llvm/ADT/StringRef.h"11#include "llvm/Testing/Support/Error.h"12#include "gtest/gtest.h"13#include <string>14 15using llvm::StringRef;16 17using namespace lldb_private::dil;18 19llvm::Expected<std::vector<std::pair<Token::Kind, std::string>>>20ExtractTokenData(llvm::StringRef input_expr) {21 22 llvm::Expected<DILLexer> maybe_lexer = DILLexer::Create(input_expr);23 if (!maybe_lexer)24 return maybe_lexer.takeError();25 DILLexer lexer(*maybe_lexer);26 27 std::vector<std::pair<Token::Kind, std::string>> data;28 do {29 Token tok = lexer.GetCurrentToken();30 data.push_back(std::make_pair(tok.GetKind(), tok.GetSpelling()));31 lexer.Advance();32 } while (data.back().first != Token::eof);33 // Don't return the eof token.34 data.pop_back();35 return data;36}37 38TEST(DILLexerTests, SimpleTest) {39 StringRef input_expr("simple_var");40 llvm::Expected<DILLexer> maybe_lexer = DILLexer::Create(input_expr);41 ASSERT_THAT_EXPECTED(maybe_lexer, llvm::Succeeded());42 DILLexer lexer(*maybe_lexer);43 Token token = lexer.GetCurrentToken();44 45 EXPECT_EQ(token.GetKind(), Token::identifier);46 EXPECT_EQ(token.GetSpelling(), "simple_var");47 lexer.Advance();48 token = lexer.GetCurrentToken();49 EXPECT_EQ(token.GetKind(), Token::eof);50}51 52TEST(DILLexerTests, TokenKindTest) {53 Token token = Token(Token::identifier, "ident", 0);54 55 EXPECT_TRUE(token.Is(Token::identifier));56 EXPECT_FALSE(token.Is(Token::l_paren));57 EXPECT_TRUE(token.IsOneOf({Token::eof, Token::identifier}));58 EXPECT_FALSE(token.IsOneOf(59 {Token::l_paren, Token::r_paren, Token::coloncolon, Token::eof}));60}61 62TEST(DILLexerTests, LookAheadTest) {63 StringRef input_expr("(anonymous namespace)::some_var");64 llvm::Expected<DILLexer> maybe_lexer = DILLexer::Create(input_expr);65 ASSERT_THAT_EXPECTED(maybe_lexer, llvm::Succeeded());66 DILLexer lexer(*maybe_lexer);67 Token token = lexer.GetCurrentToken();68 69 // Current token is '('; check the next 4 tokens, to make70 // sure they are the identifier 'anonymous', the identifier 'namespace'71 // ')' and '::', in that order.72 EXPECT_EQ(token.GetKind(), Token::l_paren);73 EXPECT_EQ(lexer.LookAhead(1).GetKind(), Token::identifier);74 EXPECT_EQ(lexer.LookAhead(1).GetSpelling(), "anonymous");75 EXPECT_EQ(lexer.LookAhead(2).GetKind(), Token::identifier);76 EXPECT_EQ(lexer.LookAhead(2).GetSpelling(), "namespace");77 EXPECT_EQ(lexer.LookAhead(3).GetKind(), Token::r_paren);78 EXPECT_EQ(lexer.LookAhead(4).GetKind(), Token::coloncolon);79 80 // Our current index should still be 0, as we only looked ahead; we are still81 // officially on the '('.82 EXPECT_EQ(lexer.GetCurrentTokenIdx(), 0u);83 84 // Accept the 'lookahead', so our current token is '::', which has the index85 // 4 in our vector of tokens (which starts at zero).86 lexer.Advance(4);87 token = lexer.GetCurrentToken();88 EXPECT_EQ(token.GetKind(), Token::coloncolon);89 EXPECT_EQ(lexer.GetCurrentTokenIdx(), 4u);90 91 lexer.Advance();92 token = lexer.GetCurrentToken();93 EXPECT_EQ(token.GetKind(), Token::identifier);94 EXPECT_EQ(token.GetSpelling(), "some_var");95 EXPECT_EQ(lexer.GetCurrentTokenIdx(), 5u);96 EXPECT_EQ(token.GetLocation(), strlen("(anonymous namespace)::"));97 98 lexer.Advance();99 token = lexer.GetCurrentToken();100 EXPECT_EQ(token.GetKind(), Token::eof);101}102 103TEST(DILLexerTests, MultiTokenLexTest) {104 EXPECT_THAT_EXPECTED(105 ExtractTokenData("This string has (several ) ::identifiers"),106 llvm::HasValue(testing::ElementsAre(107 testing::Pair(Token::identifier, "This"),108 testing::Pair(Token::identifier, "string"),109 testing::Pair(Token::identifier, "has"),110 testing::Pair(Token::l_paren, "("),111 testing::Pair(Token::identifier, "several"),112 testing::Pair(Token::r_paren, ")"),113 testing::Pair(Token::coloncolon, "::"),114 testing::Pair(Token::identifier, "identifiers"))));115}116 117TEST(DILLexerTests, IdentifiersTest) {118 // These strings should lex into identifier tokens.119 std::vector<std::string> valid_identifiers = {120 "$My_name1", "$pc", "abcd", "_", "_a", "_a_", "$",121 "a_b", "this", "self", "a", "MyName", "namespace"};122 123 // The lexer can lex these strings, but they should not be identifiers.124 std::vector<std::string> invalid_identifiers = {"", "::", "(", ")", "0abc"};125 126 // The lexer is expected to fail attempting to lex these strings (it cannot127 // create valid tokens out of them).128 std::vector<std::string> invalid_tok_strings = {"#include", "a@a"};129 130 // Verify that all of the valid identifiers come out as identifier tokens.131 for (auto &str : valid_identifiers) {132 SCOPED_TRACE(str);133 EXPECT_THAT_EXPECTED(ExtractTokenData(str),134 llvm::HasValue(testing::ElementsAre(135 testing::Pair(Token::identifier, str))));136 }137 138 // Verify that the lexer fails on invalid token strings.139 for (auto &str : invalid_tok_strings) {140 SCOPED_TRACE(str);141 auto maybe_lexer = DILLexer::Create(str);142 EXPECT_THAT_EXPECTED(maybe_lexer, llvm::Failed());143 }144 145 // Verify that none of the invalid identifiers come out as identifier tokens.146 for (auto &str : invalid_identifiers) {147 SCOPED_TRACE(str);148 llvm::Expected<DILLexer> maybe_lexer = DILLexer::Create(str);149 EXPECT_THAT_EXPECTED(maybe_lexer, llvm::Succeeded());150 DILLexer lexer(*maybe_lexer);151 Token token = lexer.GetCurrentToken();152 EXPECT_TRUE(token.IsNot(Token::identifier));153 EXPECT_TRUE(token.IsOneOf({Token::eof, Token::coloncolon, Token::l_paren,154 Token::r_paren, Token::integer_constant}));155 }156}157 158TEST(DILLexerTests, NumbersTest) {159 // These strings should lex into number tokens.160 std::vector<std::string> valid_integers = {"123", "0x123", "0123", "0b101"};161 std::vector<std::string> valid_floats = {162 "1.2", ".2", "2.f", "0x1.2", "0x.2", ".2e1f",163 "2.e+1f", "0x1.f", "0x1.2P1", "0x1.p-1f", "0x1.2P+3f", "1E1",164 "1E+1", "0x1p1", "0x1p+1", "0xf.fp1f"};165 166 // The lexer can lex these strings, but they should not be numbers.167 std::vector<std::string> invalid_numbers = {"", "x123", "b123", "a.b"};168 169 for (auto &str : valid_integers) {170 SCOPED_TRACE(str);171 EXPECT_THAT_EXPECTED(ExtractTokenData(str),172 llvm::HasValue(testing::ElementsAre(173 testing::Pair(Token::integer_constant, str))));174 }175 for (auto &str : valid_floats) {176 SCOPED_TRACE(str);177 EXPECT_THAT_EXPECTED(ExtractTokenData(str),178 llvm::HasValue(testing::ElementsAre(179 testing::Pair(Token::float_constant, str))));180 }181 // Verify that none of the invalid numbers come out as numeric tokens.182 for (auto &str : invalid_numbers) {183 SCOPED_TRACE(str);184 llvm::Expected<DILLexer> maybe_lexer = DILLexer::Create(str);185 EXPECT_THAT_EXPECTED(maybe_lexer, llvm::Succeeded());186 DILLexer lexer(*maybe_lexer);187 Token token = lexer.GetCurrentToken();188 EXPECT_TRUE(token.IsNot(Token::integer_constant));189 EXPECT_TRUE(token.IsOneOf({Token::eof, Token::identifier}));190 }191 192 // Verify that '-' and '+' are not lexed if they're not part of a number193 std::vector<std::string> expressions = {"1+e", "0x1+p", "1.1+e",194 "1.1e1+e", "0x1.1p-1-p", "1e-1+e",195 "1e1+e", "0x1p-1-p", "0xe+e"};196 for (auto &str : expressions) {197 SCOPED_TRACE(str);198 llvm::Expected<DILLexer> maybe_lexer = DILLexer::Create(str);199 EXPECT_THAT_EXPECTED(maybe_lexer, llvm::Succeeded());200 DILLexer lexer(*maybe_lexer);201 Token token = lexer.GetCurrentToken();202 EXPECT_TRUE(203 token.IsOneOf({Token::integer_constant, Token::float_constant}));204 lexer.Advance();205 token = lexer.GetCurrentToken();206 EXPECT_TRUE(token.IsOneOf({Token::plus, Token::minus}));207 lexer.Advance();208 token = lexer.GetCurrentToken();209 EXPECT_TRUE(token.Is(Token::identifier));210 }211}212