131 lines · cpp
1//===--- Token.cpp - Tokens and token streams in the pseudoparser ---------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8 9#include "Token.h"10#include "clang/Basic/LangOptions.h"11#include "llvm/ADT/StringExtras.h"12#include "llvm/Support/Format.h"13#include "llvm/Support/FormatVariadic.h"14 15namespace clang {16namespace clangd {17 18llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, const Token &T) {19 OS << llvm::formatv("{0} {1}:{2} ", clang::tok::getTokenName(T.Kind), T.Line,20 T.Indent);21 OS << '"';22 llvm::printEscapedString(T.text(), OS);23 OS << '"';24 if (T.Flags)25 OS << llvm::format(" flags=%x", T.Flags);26 return OS;27}28 29llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, const TokenStream &TS) {30 OS << "Index Kind Line Text\n";31 for (const auto &T : TS.tokens()) {32 OS << llvm::format("%5d: %16s %4d:%-2d ", TS.index(T),33 clang::tok::getTokenName(T.Kind), T.Line, T.Indent);34 OS << '"';35 llvm::printEscapedString(T.text(), OS);36 OS << '"';37 if (T.Flags)38 OS << llvm::format(" flags=%x", T.Flags);39 OS << '\n';40 }41 return OS;42}43 44llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, const Token::Range &R) {45 OS << llvm::formatv("[{0},{1})", R.Begin, R.End);46 return OS;47}48 49TokenStream::TokenStream(std::shared_ptr<void> Payload)50 : Payload(std::move(Payload)) {51 Storage.emplace_back();52 Storage.back().Kind = clang::tok::eof;53}54 55void TokenStream::finalize() {56 assert(!isFinalized());57 unsigned LastLine = Storage.back().Line;58 Storage.emplace_back();59 Storage.back().Kind = tok::eof;60 Storage.back().Line = LastLine + 1;61 62 Tokens = Storage;63 Tokens = Tokens.drop_front().drop_back();64}65 66bool TokenStream::isFinalized() const {67 assert(!Storage.empty() && Storage.front().Kind == tok::eof);68 if (Storage.size() == 1)69 return false;70 return Storage.back().Kind == tok::eof;71}72 73void TokenStream::print(llvm::raw_ostream &OS) const {74 bool FirstToken = true;75 unsigned LastLine = -1;76 StringRef LastText;77 for (const auto &T : tokens()) {78 StringRef Text = T.text();79 if (FirstToken) {80 FirstToken = false;81 } else if (T.Line == LastLine) {82 if (LastText.data() + LastText.size() != Text.data())83 OS << ' ';84 } else {85 OS << '\n';86 OS.indent(T.Indent);87 }88 OS << Text;89 LastLine = T.Line;90 LastText = Text;91 }92 if (!FirstToken)93 OS << '\n';94}95 96clang::LangOptions genericLangOpts(clang::Language Lang,97 clang::LangStandard::Kind Standard) {98 clang::LangOptions Opts;99 std::vector<std::string> UnusedIncludes;100 LangOptions::setLangDefaults(Opts, Lang, llvm::Triple(), UnusedIncludes,101 Standard);102 103 // Some options are "on by default", but e.g. at the driver level.104 if (Opts.CPlusPlus)105 Opts.CXXOperatorNames = true;106 if (Opts.CPlusPlus20)107 Opts.Coroutines = true;108 109 // Some options are off by default, but define keywords we want to tolerate.110 if (Opts.CPlusPlus)111 Opts.MicrosoftExt = true; // kw__try, kw__finally112 Opts.DeclSpecKeyword = true; // __declspec113 Opts.WChar = true;114 115 return Opts;116}117 118TokenStream stripComments(const TokenStream &Input) {119 TokenStream Out(Input.getPayload());120 for (const Token &T : Input.tokens()) {121 if (T.Kind == tok::comment)122 continue;123 Out.push(T);124 }125 Out.finalize();126 return Out;127}128 129} // namespace clangd130} // namespace clang131