314 lines · c
1//===-- lib/Parser/prescan.h ------------------------------------*- C++ -*-===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8 9#ifndef FORTRAN_PARSER_PRESCAN_H_10#define FORTRAN_PARSER_PRESCAN_H_11 12// Defines a fast Fortran source prescanning phase that implements some13// character-level features of the language that can be inefficient to14// support directly in a backtracking parser. This phase handles Fortran15// line continuation, comment removal, card image margins, padding out16// fixed form character literals on truncated card images, file17// inclusion, and driving the Fortran source preprocessor.18 19#include "flang/Parser/characters.h"20#include "flang/Parser/message.h"21#include "flang/Parser/provenance.h"22#include "flang/Parser/token-sequence.h"23#include "flang/Support/Fortran-features.h"24#include <bitset>25#include <optional>26#include <string>27#include <unordered_set>28 29namespace Fortran::parser {30 31class Messages;32class Preprocessor;33 34class Prescanner {35public:36 Prescanner(Messages &, CookedSource &, Preprocessor &,37 common::LanguageFeatureControl);38 Prescanner(39 const Prescanner &, Preprocessor &, bool isNestedInIncludeDirective);40 Prescanner(const Prescanner &) = delete;41 Prescanner(Prescanner &&) = delete;42 43 const AllSources &allSources() const { return allSources_; }44 AllSources &allSources() { return allSources_; }45 const Messages &messages() const { return messages_; }46 Messages &messages() { return messages_; }47 const Preprocessor &preprocessor() const { return preprocessor_; }48 Preprocessor &preprocessor() { return preprocessor_; }49 common::LanguageFeatureControl &features() { return features_; }50 51 Prescanner &set_preprocessingOnly(bool yes) {52 preprocessingOnly_ = yes;53 return *this;54 }55 Prescanner &set_expandIncludeLines(bool yes) {56 expandIncludeLines_ = yes;57 return *this;58 }59 Prescanner &set_fixedForm(bool yes) {60 inFixedForm_ = yes;61 return *this;62 }63 Prescanner &set_encoding(Encoding code) {64 encoding_ = code;65 return *this;66 }67 Prescanner &set_fixedFormColumnLimit(int limit) {68 fixedFormColumnLimit_ = limit;69 return *this;70 }71 72 Prescanner &AddCompilerDirectiveSentinel(const std::string &);73 74 void Prescan(ProvenanceRange);75 void Statement();76 void NextLine();77 78 // Callbacks for use by Preprocessor.79 bool IsAtEnd() const { return nextLine_ >= limit_; }80 bool IsNextLinePreprocessorDirective() const;81 TokenSequence TokenizePreprocessorDirective();82 Provenance GetCurrentProvenance() const { return GetProvenance(at_); }83 84 const char *IsCompilerDirectiveSentinel(const char *, std::size_t) const;85 const char *IsCompilerDirectiveSentinel(CharBlock) const;86 // 'first' is the sentinel, 'second' is beginning of payload87 std::optional<std::pair<const char *, const char *>>88 IsCompilerDirectiveSentinel(const char *p) const;89 90 template <typename... A> Message &Say(A &&...a) {91 return messages_.Say(std::forward<A>(a)...);92 }93 94 template <typename... A>95 Message *Warn(common::UsageWarning warning, A &&...a) {96 return messages_.Warn(false, features_, warning, std::forward<A>(a)...);97 }98 template <typename... A>99 Message *Warn(common::LanguageFeature feature, A &&...a) {100 return messages_.Warn(false, features_, feature, std::forward<A>(a)...);101 }102 103private:104 struct LineClassification {105 enum class Kind {106 Comment,107 ConditionalCompilationDirective,108 IncludeDirective, // #include109 DefinitionDirective, // #define & #undef110 PreprocessorDirective,111 IncludeLine, // Fortran INCLUDE112 CompilerDirective,113 Source114 };115 LineClassification(Kind k, std::size_t po = 0, const char *s = nullptr)116 : kind{k}, payloadOffset{po}, sentinel{s} {}117 LineClassification(LineClassification &&) = default;118 LineClassification &operator=(LineClassification &&) = default;119 Kind kind;120 std::size_t payloadOffset; // byte offset of content121 const char *sentinel; // if it's a compiler directive122 };123 124 void BeginSourceLine(const char *at) {125 at_ = at;126 column_ = 1;127 tabInCurrentLine_ = false;128 }129 130 void BeginSourceLineAndAdvance() {131 BeginSourceLine(nextLine_);132 NextLine();133 }134 135 void BeginStatementAndAdvance() {136 BeginSourceLineAndAdvance();137 slashInCurrentStatement_ = false;138 preventHollerith_ = false;139 parenthesisNesting_ = 0;140 continuationLines_ = 0;141 isPossibleMacroCall_ = false;142 disableSourceContinuation_ = false;143 }144 145 Provenance GetProvenance(const char *sourceChar) const {146 return startProvenance_ + (sourceChar - start_);147 }148 149 ProvenanceRange GetProvenanceRange(150 const char *first, const char *afterLast) const {151 std::size_t bytes = afterLast - first;152 return {startProvenance_ + (first - start_), bytes};153 }154 155 void EmitChar(TokenSequence &tokens, char ch) {156 tokens.PutNextTokenChar(ch, GetCurrentProvenance());157 }158 159 void EmitInsertedChar(TokenSequence &tokens, char ch) {160 Provenance provenance{allSources_.CompilerInsertionProvenance(ch)};161 tokens.PutNextTokenChar(ch, provenance);162 }163 164 char EmitCharAndAdvance(TokenSequence &tokens, char ch) {165 EmitChar(tokens, ch);166 NextChar();167 return *at_;168 }169 170 bool InCompilerDirective() const { return directiveSentinel_ != nullptr; }171 bool InOpenMPConditionalLine() const {172 return directiveSentinel_ && directiveSentinel_[0] == '$' &&173 !directiveSentinel_[1];174 }175 bool InOpenACCOrCUDAConditionalLine() const {176 return directiveSentinel_ && directiveSentinel_[0] == '@' &&177 ((directiveSentinel_[1] == 'a' && directiveSentinel_[2] == 'c' &&178 directiveSentinel_[3] == 'c') ||179 (directiveSentinel_[1] == 'c' && directiveSentinel_[2] == 'u' &&180 directiveSentinel_[3] == 'f')) &&181 directiveSentinel_[4] == '\0';182 }183 bool InConditionalLine() const {184 return InOpenMPConditionalLine() || InOpenACCOrCUDAConditionalLine();185 }186 bool IsOpenMPDirective() const {187 return directiveSentinel_ && std::strcmp(directiveSentinel_, "$omp") == 0;188 }189 bool InFixedFormSource() const {190 return inFixedForm_ && !inPreprocessorDirective_ && !InCompilerDirective();191 }192 193 bool IsCComment(const char *p) const {194 return p[0] == '/' && p[1] == '*' &&195 (inPreprocessorDirective_ ||196 (!inCharLiteral_ &&197 features_.IsEnabled(198 common::LanguageFeature::ClassicCComments)));199 }200 201 void CheckAndEmitLine(TokenSequence &, Provenance newlineProvenance);202 void LabelField(TokenSequence &);203 void EnforceStupidEndStatementRules(const TokenSequence &);204 void SkipToEndOfLine();205 bool MustSkipToEndOfLine() const;206 void NextChar();207 // True when input flowed to a continuation line208 bool SkipToNextSignificantCharacter();209 void SkipCComments();210 void SkipSpaces();211 static const char *SkipWhiteSpace(const char *);212 const char *SkipWhiteSpaceIncludingEmptyMacros(const char *) const;213 const char *SkipWhiteSpaceAndCComments(const char *) const;214 const char *SkipCComment(const char *) const;215 bool NextToken(TokenSequence &);216 bool HandleExponent(TokenSequence &);217 bool HandleKindSuffix(TokenSequence &);218 bool HandleExponentAndOrKindSuffix(TokenSequence &);219 void QuotedCharacterLiteral(TokenSequence &, const char *start);220 void Hollerith(TokenSequence &, int count, const char *start);221 bool PadOutCharacterLiteral(TokenSequence &);222 bool SkipCommentLine(bool afterAmpersand);223 bool IsFixedFormCommentLine(const char *) const;224 const char *IsFreeFormComment(const char *) const;225 std::optional<std::size_t> IsIncludeLine(const char *) const;226 void FortranInclude(const char *quote);227 const char *IsPreprocessorDirectiveLine(const char *) const;228 const char *FixedFormContinuationLine(bool atNewline);229 const char *FreeFormContinuationLine(bool ampersand);230 bool IsImplicitContinuation() const;231 bool FixedFormContinuation(bool atNewline);232 bool FreeFormContinuation();233 bool Continuation(bool mightNeedFixedFormSpace);234 std::optional<LineClassification> IsFixedFormCompilerDirectiveLine(235 const char *) const;236 std::optional<LineClassification> IsFreeFormCompilerDirectiveLine(237 const char *) const;238 LineClassification ClassifyLine(const char *) const;239 LineClassification ClassifyLine(240 TokenSequence &, Provenance newlineProvenance) const;241 bool SourceFormChange(std::string &&);242 bool CompilerDirectiveContinuation(TokenSequence &, const char *sentinel);243 bool SourceLineContinuation(TokenSequence &);244 245 Messages &messages_;246 CookedSource &cooked_;247 Preprocessor &preprocessor_;248 AllSources &allSources_;249 common::LanguageFeatureControl features_;250 bool preprocessingOnly_{false};251 bool expandIncludeLines_{true};252 bool isNestedInIncludeDirective_{false};253 bool backslashFreeFormContinuation_{false};254 bool inFixedForm_{false};255 int fixedFormColumnLimit_{72};256 Encoding encoding_{Encoding::UTF_8};257 int parenthesisNesting_{0};258 int prescannerNesting_{0};259 int continuationLines_{0};260 bool isPossibleMacroCall_{false};261 bool afterPreprocessingDirective_{false};262 bool disableSourceContinuation_{false};263 264 Provenance startProvenance_;265 const char *start_{nullptr}; // beginning of current source file content266 const char *limit_{nullptr}; // first address after end of current source267 const char *nextLine_{nullptr}; // next line to process; <= limit_268 const char *directiveSentinel_{nullptr}; // current compiler directive269 270 // These data members are state for processing the source line containing271 // "at_", which goes to up to the newline character before "nextLine_".272 const char *at_{nullptr}; // next character to process; < nextLine_273 int column_{1}; // card image column position of next character274 bool tabInCurrentLine_{false};275 bool slashInCurrentStatement_{false};276 bool preventHollerith_{false}; // CHARACTER*4HIMOM not Hollerith277 bool inCharLiteral_{false};278 bool continuationInCharLiteral_{false};279 bool inPreprocessorDirective_{false};280 281 // True after processing a continuation that can't be allowed282 // to appear in the middle of an identifier token, but is fixed form,283 // or is free form and doesn't have a space character handy to use as284 // a separator when:285 // a) (standard) doesn't begin with a leading '&' on the continuation286 // line, but has a non-blank in column 1, or287 // b) (extension) does have a leading '&', but didn't have one288 // on the continued line.289 bool brokenToken_{false};290 291 // When a free form continuation marker (&) appears at the end of a line292 // before a INCLUDE or #include, we delete it and omit the newline, so293 // that the first line of the included file is truly a continuation of294 // the line before. Also used when the & appears at the end of the last295 // line in an include file.296 bool omitNewline_{false};297 bool skipLeadingAmpersand_{false};298 299 const std::size_t firstCookedCharacterOffset_{cooked_.BufferedBytes()};300 301 const Provenance spaceProvenance_{302 allSources_.CompilerInsertionProvenance(' ')};303 const Provenance backslashProvenance_{304 allSources_.CompilerInsertionProvenance('\\')};305 306 // To avoid probing the set of active compiler directive sentinel strings307 // on every comment line, they're checked first with a cheap Bloom filter.308 static const int prime1{1019}, prime2{1021};309 std::bitset<prime2> compilerDirectiveBloomFilter_; // 128 bytes310 std::unordered_set<std::string> compilerDirectiveSentinels_;311};312} // namespace Fortran::parser313#endif // FORTRAN_PARSER_PRESCAN_H_314