162 lines · cpp
1//===--- TokenAnalyzer.cpp - Analyze Token Streams --------------*- C++ -*-===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8///9/// \file10/// This file implements an abstract TokenAnalyzer and associated helper11/// classes. TokenAnalyzer can be extended to generate replacements based on12/// an annotated and pre-processed token stream.13///14//===----------------------------------------------------------------------===//15 16#include "TokenAnalyzer.h"17#include "AffectedRangeManager.h"18#include "Encoding.h"19#include "FormatToken.h"20#include "FormatTokenLexer.h"21#include "TokenAnnotator.h"22#include "UnwrappedLineParser.h"23#include "clang/Basic/Diagnostic.h"24#include "clang/Basic/DiagnosticOptions.h"25#include "clang/Basic/SourceManager.h"26#include "clang/Format/Format.h"27#include "llvm/ADT/SmallVector.h"28#include "llvm/Support/Debug.h"29 30#define DEBUG_TYPE "format-formatter"31 32namespace clang {33namespace format {34 35// FIXME: Instead of printing the diagnostic we should store it and have a36// better way to return errors through the format APIs.37class FatalDiagnosticConsumer : public DiagnosticConsumer {38public:39 void HandleDiagnostic(DiagnosticsEngine::Level DiagLevel,40 const Diagnostic &Info) override {41 if (DiagLevel == DiagnosticsEngine::Fatal) {42 Fatal = true;43 llvm::SmallVector<char, 128> Message;44 Info.FormatDiagnostic(Message);45 llvm::errs() << Message << "\n";46 }47 }48 49 bool fatalError() const { return Fatal; }50 51private:52 bool Fatal = false;53};54 55std::unique_ptr<Environment>56Environment::make(StringRef Code, StringRef FileName,57 ArrayRef<tooling::Range> Ranges, unsigned FirstStartColumn,58 unsigned NextStartColumn, unsigned LastStartColumn) {59 auto Env = std::make_unique<Environment>(Code, FileName, FirstStartColumn,60 NextStartColumn, LastStartColumn);61 FatalDiagnosticConsumer Diags;62 Env->SM.getDiagnostics().setClient(&Diags, /*ShouldOwnClient=*/false);63 SourceLocation StartOfFile = Env->SM.getLocForStartOfFile(Env->ID);64 for (const tooling::Range &Range : Ranges) {65 SourceLocation Start = StartOfFile.getLocWithOffset(Range.getOffset());66 SourceLocation End = Start.getLocWithOffset(Range.getLength());67 Env->CharRanges.push_back(CharSourceRange::getCharRange(Start, End));68 }69 // Validate that we can get the buffer data without a fatal error.70 Env->SM.getBufferData(Env->ID);71 if (Diags.fatalError())72 return nullptr;73 return Env;74}75 76Environment::Environment(StringRef Code, StringRef FileName,77 unsigned FirstStartColumn, unsigned NextStartColumn,78 unsigned LastStartColumn)79 : VirtualSM(new SourceManagerForFile(FileName, Code)), SM(VirtualSM->get()),80 ID(VirtualSM->get().getMainFileID()), FirstStartColumn(FirstStartColumn),81 NextStartColumn(NextStartColumn), LastStartColumn(LastStartColumn) {}82 83TokenAnalyzer::TokenAnalyzer(const Environment &Env, const FormatStyle &Style)84 : Style(Style), LangOpts(getFormattingLangOpts(Style)), Env(Env),85 AffectedRangeMgr(Env.getSourceManager(), Env.getCharRanges()),86 UnwrappedLines(1),87 Encoding(encoding::detectEncoding(88 Env.getSourceManager().getBufferData(Env.getFileID()))) {89 LLVM_DEBUG(90 llvm::dbgs() << "File encoding: "91 << (Encoding == encoding::Encoding_UTF8 ? "UTF8" : "unknown")92 << "\n");93 LLVM_DEBUG(llvm::dbgs() << "Language: " << getLanguageName(Style.Language)94 << "\n");95}96 97std::pair<tooling::Replacements, unsigned>98TokenAnalyzer::process(bool SkipAnnotation) {99 tooling::Replacements Result;100 llvm::SpecificBumpPtrAllocator<FormatToken> Allocator;101 IdentifierTable IdentTable(LangOpts);102 FormatTokenLexer Lex(Env.getSourceManager(), Env.getFileID(),103 Env.getFirstStartColumn(), Style, Encoding, Allocator,104 IdentTable);105 ArrayRef<FormatToken *> Toks(Lex.lex());106 SmallVector<FormatToken *, 10> Tokens(Toks);107 UnwrappedLineParser Parser(Env.getSourceManager(), Style, Lex.getKeywords(),108 Env.getFirstStartColumn(), Tokens, *this,109 Allocator, IdentTable);110 Parser.parse();111 assert(UnwrappedLines.back().empty());112 unsigned Penalty = 0;113 for (unsigned Run = 0, RunE = UnwrappedLines.size(); Run + 1 != RunE; ++Run) {114 const auto &Lines = UnwrappedLines[Run];115 LLVM_DEBUG(llvm::dbgs() << "Run " << Run << "...\n");116 SmallVector<AnnotatedLine *, 16> AnnotatedLines;117 AnnotatedLines.reserve(Lines.size());118 119 TokenAnnotator Annotator(Style, Lex.getKeywords());120 for (const UnwrappedLine &Line : Lines) {121 AnnotatedLines.push_back(new AnnotatedLine(Line));122 if (!SkipAnnotation)123 Annotator.annotate(*AnnotatedLines.back());124 }125 126 std::pair<tooling::Replacements, unsigned> RunResult =127 analyze(Annotator, AnnotatedLines, Lex);128 129 LLVM_DEBUG({130 llvm::dbgs() << "Replacements for run " << Run << ":\n";131 for (const tooling::Replacement &Fix : RunResult.first)132 llvm::dbgs() << Fix.toString() << "\n";133 });134 for (AnnotatedLine *Line : AnnotatedLines)135 delete Line;136 137 Penalty += RunResult.second;138 for (const auto &R : RunResult.first) {139 auto Err = Result.add(R);140 // FIXME: better error handling here. For now, simply return an empty141 // Replacements to indicate failure.142 if (Err) {143 llvm::errs() << llvm::toString(std::move(Err)) << "\n";144 return {tooling::Replacements(), 0};145 }146 }147 }148 return {Result, Penalty};149}150 151void TokenAnalyzer::consumeUnwrappedLine(const UnwrappedLine &TheLine) {152 assert(!UnwrappedLines.empty());153 UnwrappedLines.back().push_back(TheLine);154}155 156void TokenAnalyzer::finishRun() {157 UnwrappedLines.push_back(SmallVector<UnwrappedLine, 16>());158}159 160} // end namespace format161} // end namespace clang162