1929 lines · cpp
1//===-- lib/Parser/prescan.cpp --------------------------------------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8 9#include "prescan.h"10#include "flang/Common/idioms.h"11#include "flang/Parser/characters.h"12#include "flang/Parser/message.h"13#include "flang/Parser/preprocessor.h"14#include "flang/Parser/source.h"15#include "flang/Parser/token-sequence.h"16#include "llvm/Support/raw_ostream.h"17#include <cstddef>18#include <cstring>19#include <utility>20#include <vector>21 22namespace Fortran::parser {23 24using common::LanguageFeature;25 26static constexpr int maxPrescannerNesting{100};27 28Prescanner::Prescanner(Messages &messages, CookedSource &cooked,29 Preprocessor &preprocessor, common::LanguageFeatureControl lfc)30 : messages_{messages}, cooked_{cooked}, preprocessor_{preprocessor},31 allSources_{preprocessor_.allSources()}, features_{lfc},32 backslashFreeFormContinuation_{preprocessor.AnyDefinitions()},33 encoding_{allSources_.encoding()} {}34 35Prescanner::Prescanner(const Prescanner &that, Preprocessor &prepro,36 bool isNestedInIncludeDirective)37 : messages_{that.messages_}, cooked_{that.cooked_}, preprocessor_{prepro},38 allSources_{that.allSources_}, features_{that.features_},39 preprocessingOnly_{that.preprocessingOnly_},40 expandIncludeLines_{that.expandIncludeLines_},41 isNestedInIncludeDirective_{isNestedInIncludeDirective},42 backslashFreeFormContinuation_{that.backslashFreeFormContinuation_},43 inFixedForm_{that.inFixedForm_},44 fixedFormColumnLimit_{that.fixedFormColumnLimit_},45 encoding_{that.encoding_},46 prescannerNesting_{that.prescannerNesting_ + 1},47 skipLeadingAmpersand_{that.skipLeadingAmpersand_},48 compilerDirectiveBloomFilter_{that.compilerDirectiveBloomFilter_},49 compilerDirectiveSentinels_{that.compilerDirectiveSentinels_} {}50 51// Returns number of bytes to skip52static inline int IsSpace(const char *p) {53 if (*p == ' ') {54 return 1;55 } else if (*p == '\xa0') { // LATIN-1 NBSP non-breaking space56 return 1;57 } else if (p[0] == '\xc2' && p[1] == '\xa0') { // UTF-8 NBSP58 return 2;59 } else {60 return 0;61 }62}63 64static inline int IsSpaceOrTab(const char *p) {65 return *p == '\t' ? 1 : IsSpace(p);66}67 68static inline constexpr bool IsFixedFormCommentChar(char ch) {69 return ch == '!' || ch == '*' || ch == 'C' || ch == 'c';70}71 72static void NormalizeCompilerDirectiveCommentMarker(TokenSequence &dir) {73 char *p{dir.GetMutableCharData()};74 char *limit{p + dir.SizeInChars()};75 for (; p < limit; ++p) {76 if (*p != ' ') {77 CHECK(IsFixedFormCommentChar(*p));78 *p = '!';79 return;80 }81 }82 DIE("compiler directive all blank");83}84 85void Prescanner::Prescan(ProvenanceRange range) {86 startProvenance_ = range.start();87 start_ = allSources_.GetSource(range);88 CHECK(start_);89 limit_ = start_ + range.size();90 nextLine_ = start_;91 const bool beganInFixedForm{inFixedForm_};92 if (prescannerNesting_ > maxPrescannerNesting) {93 Say(GetProvenance(start_),94 "too many nested INCLUDE/#include files, possibly circular"_err_en_US);95 return;96 }97 while (!IsAtEnd()) {98 Statement();99 }100 inFixedForm_ = beganInFixedForm;101}102 103void Prescanner::Statement() {104 TokenSequence tokens;105 const char *statementStart{nextLine_};106 LineClassification line{ClassifyLine(statementStart)};107 switch (line.kind) {108 case LineClassification::Kind::Comment:109 nextLine_ += line.payloadOffset; // advance to '!' or newline110 NextLine();111 return;112 case LineClassification::Kind::IncludeLine:113 FortranInclude(nextLine_ + line.payloadOffset);114 NextLine();115 return;116 case LineClassification::Kind::ConditionalCompilationDirective:117 case LineClassification::Kind::IncludeDirective:118 preprocessor_.Directive(TokenizePreprocessorDirective(), *this);119 afterPreprocessingDirective_ = true;120 skipLeadingAmpersand_ |= !inFixedForm_;121 return;122 case LineClassification::Kind::PreprocessorDirective:123 preprocessor_.Directive(TokenizePreprocessorDirective(), *this);124 afterPreprocessingDirective_ = true;125 // Don't set skipLeadingAmpersand_126 return;127 case LineClassification::Kind::DefinitionDirective:128 preprocessor_.Directive(TokenizePreprocessorDirective(), *this);129 // Don't set afterPreprocessingDirective_ or skipLeadingAmpersand_130 return;131 case LineClassification::Kind::CompilerDirective: {132 directiveSentinel_ = line.sentinel;133 CHECK(InCompilerDirective());134 BeginStatementAndAdvance();135 if (inFixedForm_) {136 CHECK(IsFixedFormCommentChar(*at_));137 } else {138 at_ += line.payloadOffset;139 column_ += line.payloadOffset;140 CHECK(*at_ == '!');141 }142 std::optional<int> condOffset;143 if (InOpenMPConditionalLine()) { // !$144 condOffset = 2;145 } else if (InOpenACCOrCUDAConditionalLine()) { // !@acc or !@cuf146 condOffset = 5;147 }148 if (condOffset && !preprocessingOnly_) {149 at_ += *condOffset, column_ += *condOffset;150 if (auto payload{IsIncludeLine(at_)}) {151 FortranInclude(at_ + *payload);152 return;153 }154 if (inFixedForm_) {155 LabelField(tokens);156 }157 SkipSpaces();158 } else {159 // Compiler directive. Emit normalized sentinel, squash following spaces.160 // Conditional compilation lines (!$) take this path in -E mode too161 // so that -fopenmp only has to appear on the later compilation162 // (ditto for !@cuf and !@acc).163 EmitChar(tokens, '!');164 ++at_, ++column_;165 for (const char *sp{directiveSentinel_}; *sp != '\0';166 ++sp, ++at_, ++column_) {167 EmitChar(tokens, *sp);168 }169 if (inFixedForm_) {170 // We need to add the whitespace after the sentinel because otherwise171 // the line cannot be re-categorised as a compiler directive.172 while (column_ <= 6) {173 if (*at_ == '\t') {174 tabInCurrentLine_ = true;175 ++at_;176 for (; column_ < 7; ++column_) {177 EmitChar(tokens, ' ');178 }179 } else if (int spaceBytes{IsSpace(at_)}) {180 EmitChar(tokens, ' ');181 at_ += spaceBytes;182 ++column_;183 } else {184 if (InOpenMPConditionalLine() && column_ == 3 &&185 IsDecimalDigit(*at_)) {186 // subtle: !$ in -E mode can't be immediately followed by a digit187 EmitChar(tokens, ' ');188 }189 break;190 }191 }192 } else if (int spaceBytes{IsSpaceOrTab(at_)}) {193 EmitChar(tokens, ' ');194 at_ += spaceBytes, ++column_;195 }196 tokens.CloseToken();197 SkipSpaces();198 if (InConditionalLine() && inFixedForm_ && !tabInCurrentLine_ &&199 column_ == 6 && *at_ != '\n') {200 // !$ 0 - turn '0' into a space201 // !$ 1 - turn '1' into '&'202 if (int n{IsSpace(at_)}; n || *at_ == '0') {203 at_ += n ? n : 1;204 } else {205 ++at_;206 EmitChar(tokens, '&');207 tokens.CloseToken();208 }209 ++column_;210 SkipSpaces();211 }212 }213 break;214 }215 case LineClassification::Kind::Source: {216 BeginStatementAndAdvance();217 bool checkLabelField{false};218 if (inFixedForm_) {219 if (features_.IsEnabled(LanguageFeature::OldDebugLines) &&220 (*at_ == 'D' || *at_ == 'd')) {221 NextChar();222 }223 checkLabelField = true;224 } else {225 if (skipLeadingAmpersand_) {226 skipLeadingAmpersand_ = false;227 const char *p{SkipWhiteSpace(at_)};228 if (p < limit_ && *p == '&') {229 column_ += ++p - at_;230 at_ = p;231 }232 } else {233 SkipSpaces();234 }235 }236 // Check for a leading identifier that might be a keyword macro237 // that will expand to anything indicating a non-source line, like238 // a comment marker or directive sentinel. If so, disable line239 // continuation, so that NextToken() won't consume anything from240 // following lines.241 if (IsLegalIdentifierStart(*at_)) {242 // TODO: Only bother with these cases when any keyword macro has243 // been defined with replacement text that could begin a comment244 // or directive sentinel.245 const char *p{at_};246 while (IsLegalInIdentifier(*++p)) {247 }248 CharBlock id{at_, static_cast<std::size_t>(p - at_)};249 if (preprocessor_.IsNameDefined(id) &&250 !preprocessor_.IsFunctionLikeDefinition(id)) {251 checkLabelField = false;252 TokenSequence toks;253 toks.Put(id, GetProvenance(at_));254 if (auto replaced{preprocessor_.MacroReplacement(toks, *this)}) {255 auto newLineClass{ClassifyLine(*replaced, GetCurrentProvenance())};256 if (newLineClass.kind ==257 LineClassification::Kind::CompilerDirective) {258 directiveSentinel_ = newLineClass.sentinel;259 disableSourceContinuation_ = false;260 } else {261 disableSourceContinuation_ = !replaced->empty() &&262 newLineClass.kind != LineClassification::Kind::Source;263 }264 }265 }266 }267 if (checkLabelField) {268 LabelField(tokens);269 }270 } break;271 }272 273 while (NextToken(tokens)) {274 }275 if (continuationLines_ > 255) {276 if (features_.ShouldWarn(common::LanguageFeature::MiscSourceExtensions)) {277 Say(common::LanguageFeature::MiscSourceExtensions,278 GetProvenance(statementStart),279 "%d continuation lines is more than the Fortran standard allows"_port_en_US,280 continuationLines_);281 }282 }283 284 Provenance newlineProvenance{GetCurrentProvenance()};285 if (std::optional<TokenSequence> preprocessed{286 preprocessor_.MacroReplacement(tokens, *this)}) {287 // Reprocess the preprocessed line.288 LineClassification ppl{ClassifyLine(*preprocessed, newlineProvenance)};289 switch (ppl.kind) {290 case LineClassification::Kind::Comment:291 break;292 case LineClassification::Kind::IncludeLine:293 FortranInclude(preprocessed->TokenAt(0).begin() + ppl.payloadOffset);294 break;295 case LineClassification::Kind::ConditionalCompilationDirective:296 case LineClassification::Kind::IncludeDirective:297 case LineClassification::Kind::DefinitionDirective:298 case LineClassification::Kind::PreprocessorDirective:299 if (features_.ShouldWarn(common::UsageWarning::Preprocessing)) {300 Say(common::UsageWarning::Preprocessing,301 preprocessed->GetProvenanceRange(),302 "Preprocessed line resembles a preprocessor directive"_warn_en_US);303 }304 CheckAndEmitLine(preprocessed->ToLowerCase(), newlineProvenance);305 break;306 case LineClassification::Kind::CompilerDirective:307 if (preprocessed->HasRedundantBlanks()) {308 preprocessed->RemoveRedundantBlanks();309 }310 while (CompilerDirectiveContinuation(*preprocessed, ppl.sentinel)) {311 newlineProvenance = GetCurrentProvenance();312 }313 NormalizeCompilerDirectiveCommentMarker(*preprocessed);314 preprocessed->ToLowerCase();315 if (!SourceFormChange(preprocessed->ToString())) {316 CheckAndEmitLine(317 preprocessed->ClipComment(*this, true /* skip first ! */),318 newlineProvenance);319 }320 break;321 case LineClassification::Kind::Source:322 if (inFixedForm_) {323 if (!preprocessingOnly_ && preprocessed->HasBlanks()) {324 preprocessed->RemoveBlanks();325 }326 } else {327 while (SourceLineContinuation(*preprocessed)) {328 newlineProvenance = GetCurrentProvenance();329 }330 if (preprocessed->HasRedundantBlanks()) {331 preprocessed->RemoveRedundantBlanks();332 }333 }334 CheckAndEmitLine(335 preprocessed->ToLowerCase().ClipComment(*this), newlineProvenance);336 break;337 }338 } else { // no macro replacement339 if (line.kind == LineClassification::Kind::CompilerDirective) {340 while (CompilerDirectiveContinuation(tokens, line.sentinel)) {341 newlineProvenance = GetCurrentProvenance();342 }343 if (preprocessingOnly_ && inFixedForm_ && InConditionalLine() &&344 nextLine_ < limit_) {345 // In -E mode, when the line after !$ conditional compilation is a346 // regular fixed form continuation line, append a '&' to the line.347 const char *p{nextLine_};348 int col{1};349 while (int n{IsSpace(p)}) {350 if (*p == '\t') {351 break;352 }353 p += n;354 ++col;355 }356 if (col == 6 && *p != '0' && *p != '\t' && *p != '\n') {357 EmitChar(tokens, '&');358 tokens.CloseToken();359 }360 }361 tokens.ToLowerCase();362 if (!SourceFormChange(tokens.ToString())) {363 CheckAndEmitLine(tokens, newlineProvenance);364 }365 } else { // Kind::Source366 tokens.ToLowerCase();367 if (inFixedForm_) {368 EnforceStupidEndStatementRules(tokens);369 }370 CheckAndEmitLine(tokens, newlineProvenance);371 }372 }373 directiveSentinel_ = nullptr;374}375 376void Prescanner::CheckAndEmitLine(377 TokenSequence &tokens, Provenance newlineProvenance) {378 tokens.CheckBadFortranCharacters(379 messages_, *this, disableSourceContinuation_ || preprocessingOnly_);380 // Parenthesis nesting check does not apply while any #include is381 // active, nor on the lines before and after a top-level #include,382 // nor before or after conditional source.383 // Applications play shenanigans with line continuation before and384 // after #include'd subprogram argument lists and conditional source.385 if (!preprocessingOnly_ && !isNestedInIncludeDirective_ && !omitNewline_ &&386 !afterPreprocessingDirective_ && tokens.BadlyNestedParentheses() &&387 !preprocessor_.InConditional()) {388 if (nextLine_ < limit_ && IsPreprocessorDirectiveLine(nextLine_)) {389 // don't complain390 } else {391 tokens.CheckBadParentheses(messages_);392 }393 }394 tokens.Emit(cooked_);395 if (omitNewline_) {396 omitNewline_ = false;397 } else {398 cooked_.Put('\n', newlineProvenance);399 afterPreprocessingDirective_ = false;400 }401}402 403TokenSequence Prescanner::TokenizePreprocessorDirective() {404 CHECK(!IsAtEnd() && !inPreprocessorDirective_);405 inPreprocessorDirective_ = true;406 BeginStatementAndAdvance();407 TokenSequence tokens;408 while (NextToken(tokens)) {409 }410 inPreprocessorDirective_ = false;411 return tokens;412}413 414void Prescanner::NextLine() {415 void *vstart{static_cast<void *>(const_cast<char *>(nextLine_))};416 void *v{std::memchr(vstart, '\n', limit_ - nextLine_)};417 if (!v) {418 nextLine_ = limit_;419 } else {420 const char *nl{const_cast<const char *>(static_cast<char *>(v))};421 nextLine_ = nl + 1;422 }423}424 425void Prescanner::LabelField(TokenSequence &token) {426 int outCol{1};427 const char *start{at_};428 std::optional<int> badColumn;429 for (; *at_ != '\n' && column_ <= 6; ++at_) {430 if (*at_ == '\t') {431 ++at_;432 column_ = 7;433 break;434 }435 if (int n{IsSpace(at_)}; n == 0 &&436 !(*at_ == '0' && column_ == 6)) { // '0' in column 6 becomes space437 EmitChar(token, *at_);438 ++outCol;439 if (!badColumn && (column_ == 6 || !IsDecimalDigit(*at_))) {440 badColumn = column_;441 }442 }443 ++column_;444 }445 if (badColumn && !preprocessor_.IsNameDefined(token.CurrentOpenToken())) {446 if ((prescannerNesting_ > 0 && *badColumn == 6 &&447 cooked_.BufferedBytes() == firstCookedCharacterOffset_) ||448 afterPreprocessingDirective_) {449 // This is the first source line in #include'd text or conditional450 // code under #if, or the first source line after such.451 // If it turns out that the preprocessed text begins with a452 // fixed form continuation line, the newline at the end453 // of the latest source line beforehand will be deleted in454 // CookedSource::Marshal().455 cooked_.MarkPossibleFixedFormContinuation();456 } else if (features_.ShouldWarn(common::UsageWarning::Scanning)) {457 Say(common::UsageWarning::Scanning, GetProvenance(start + *badColumn - 1),458 *badColumn == 6459 ? "Statement should not begin with a continuation line"_warn_en_US460 : "Character in fixed-form label field must be a digit"_warn_en_US);461 }462 token.clear();463 if (*badColumn < 6) {464 at_ = start;465 column_ = 1;466 return;467 }468 outCol = 1;469 }470 if (outCol == 1) { // empty label field471 // Emit a space so that, if the line is rescanned after preprocessing,472 // a leading 'C' or 'D' won't be left-justified and then accidentally473 // misinterpreted as a comment card.474 EmitChar(token, ' ');475 ++outCol;476 }477 token.CloseToken();478 SkipToNextSignificantCharacter();479 if (IsDecimalDigit(*at_)) {480 if (features_.ShouldWarn(common::LanguageFeature::MiscSourceExtensions)) {481 Say(common::LanguageFeature::MiscSourceExtensions, GetCurrentProvenance(),482 "Label digit is not in fixed-form label field"_port_en_US);483 }484 }485}486 487// 6.3.3.5: A program unit END statement, or any other statement whose488// initial line resembles an END statement, shall not be continued in489// fixed form source.490void Prescanner::EnforceStupidEndStatementRules(const TokenSequence &tokens) {491 CharBlock cBlock{tokens.ToCharBlock()};492 const char *str{cBlock.begin()};493 std::size_t n{cBlock.size()};494 if (n < 3) {495 return;496 }497 std::size_t j{0};498 for (; j < n && (str[j] == ' ' || (str[j] >= '0' && str[j] <= '9')); ++j) {499 }500 if (j + 3 > n || std::memcmp(str + j, "end", 3) != 0) {501 return;502 }503 // It starts with END, possibly after a label.504 auto start{allSources_.GetSourcePosition(tokens.GetCharProvenance(j))};505 auto end{allSources_.GetSourcePosition(tokens.GetCharProvenance(n - 1))};506 if (!start || !end) {507 return;508 }509 if (&*start->sourceFile == &*end->sourceFile && start->line == end->line) {510 return; // no continuation511 }512 j += 3;513 static const char *const prefixes[]{"program", "subroutine", "function",514 "blockdata", "module", "submodule", nullptr};515 bool isPrefix{j == n || !IsLegalInIdentifier(str[j])}; // prefix is END516 std::size_t endOfPrefix{j - 1};517 for (const char *const *p{prefixes}; *p; ++p) {518 std::size_t pLen{std::strlen(*p)};519 if (j + pLen <= n && std::memcmp(str + j, *p, pLen) == 0) {520 isPrefix = true; // END thing as prefix521 j += pLen;522 endOfPrefix = j - 1;523 for (; j < n && IsLegalInIdentifier(str[j]); ++j) {524 }525 break;526 }527 }528 if (isPrefix) {529 auto range{tokens.GetTokenProvenanceRange(1)};530 if (j == n) { // END or END thing [name]531 Say(range,532 "Program unit END statement may not be continued in fixed form source"_err_en_US);533 } else {534 auto endOfPrefixPos{535 allSources_.GetSourcePosition(tokens.GetCharProvenance(endOfPrefix))};536 auto next{allSources_.GetSourcePosition(tokens.GetCharProvenance(j))};537 if (endOfPrefixPos && next &&538 &*endOfPrefixPos->sourceFile == &*start->sourceFile &&539 endOfPrefixPos->line == start->line &&540 (&*next->sourceFile != &*start->sourceFile ||541 next->line != start->line)) {542 Say(range,543 "Initial line of continued statement must not appear to be a program unit END in fixed form source"_err_en_US);544 }545 }546 }547}548 549void Prescanner::SkipToEndOfLine() {550 while (*at_ != '\n') {551 ++at_, ++column_;552 }553}554 555bool Prescanner::MustSkipToEndOfLine() const {556 if (inFixedForm_ && column_ > fixedFormColumnLimit_ && !tabInCurrentLine_) {557 return true; // skip over ignored columns in right margin (73:80)558 } else if (*at_ == '!' && !inCharLiteral_ &&559 (!inFixedForm_ || tabInCurrentLine_ || column_ != 6)) {560 return InCompilerDirective() || !IsCompilerDirectiveSentinel(at_ + 1);561 } else {562 return false;563 }564}565 566void Prescanner::NextChar() {567 CHECK(*at_ != '\n');568 int n{IsSpace(at_)};569 at_ += n ? n : 1;570 ++column_;571 while (at_[0] == '\xef' && at_[1] == '\xbb' && at_[2] == '\xbf') {572 // UTF-8 byte order mark - treat this file as UTF-8573 at_ += 3;574 encoding_ = Encoding::UTF_8;575 }576 SkipToNextSignificantCharacter();577}578 579// Skip everything that should be ignored until the next significant580// character is reached; handles C-style comments in preprocessing581// directives, Fortran ! comments, stuff after the right margin in582// fixed form, and all forms of line continuation.583bool Prescanner::SkipToNextSignificantCharacter() {584 if (inPreprocessorDirective_) {585 SkipCComments();586 return false;587 } else {588 auto anyContinuationLine{false};589 bool atNewline{false};590 if (MustSkipToEndOfLine()) {591 SkipToEndOfLine();592 } else {593 atNewline = *at_ == '\n';594 }595 for (; Continuation(atNewline); atNewline = false) {596 anyContinuationLine = true;597 ++continuationLines_;598 if (MustSkipToEndOfLine()) {599 SkipToEndOfLine();600 }601 }602 if (*at_ == '\t') {603 tabInCurrentLine_ = true;604 }605 return anyContinuationLine;606 }607}608 609void Prescanner::SkipCComments() {610 while (true) {611 if (IsCComment(at_)) {612 if (const char *after{SkipCComment(at_)}) {613 column_ += after - at_;614 // May have skipped over one or more newlines; relocate the start of615 // the next line.616 nextLine_ = at_ = after;617 NextLine();618 } else {619 // Don't emit any messages about unclosed C-style comments, because620 // the sequence /* can appear legally in a FORMAT statement. There's621 // no ambiguity, since the sequence */ cannot appear legally.622 break;623 }624 } else if (inPreprocessorDirective_ && at_[0] == '\\' && at_ + 2 < limit_ &&625 at_[1] == '\n' && !IsAtEnd()) {626 BeginSourceLineAndAdvance();627 } else {628 break;629 }630 }631}632 633void Prescanner::SkipSpaces() {634 while (IsSpaceOrTab(at_)) {635 NextChar();636 }637 brokenToken_ = false;638}639 640const char *Prescanner::SkipWhiteSpace(const char *p) {641 while (int n{IsSpaceOrTab(p)}) {642 p += n;643 }644 return p;645}646 647const char *Prescanner::SkipWhiteSpaceIncludingEmptyMacros(648 const char *p) const {649 while (true) {650 if (int n{IsSpaceOrTab(p)}) {651 p += n;652 } else if (preprocessor_.AnyDefinitions() && IsLegalIdentifierStart(*p)) {653 // Skip keyword macros with empty definitions654 const char *q{p + 1};655 while (IsLegalInIdentifier(*q)) {656 ++q;657 }658 if (preprocessor_.IsNameDefinedEmpty(659 CharBlock{p, static_cast<std::size_t>(q - p)})) {660 p = q;661 } else {662 break;663 }664 } else {665 break;666 }667 }668 return p;669}670 671const char *Prescanner::SkipWhiteSpaceAndCComments(const char *p) const {672 while (true) {673 if (int n{IsSpaceOrTab(p)}) {674 p += n;675 } else if (IsCComment(p)) {676 if (const char *after{SkipCComment(p)}) {677 p = after;678 } else {679 break;680 }681 } else {682 break;683 }684 }685 return p;686}687 688const char *Prescanner::SkipCComment(const char *p) const {689 char star{' '}, slash{' '};690 p += 2;691 while (star != '*' || slash != '/') {692 if (p >= limit_) {693 return nullptr; // signifies an unterminated comment694 }695 star = slash;696 slash = *p++;697 }698 return p;699}700 701bool Prescanner::NextToken(TokenSequence &tokens) {702 CHECK(at_ >= start_ && at_ < limit_);703 if (InFixedFormSource() && !preprocessingOnly_) {704 SkipSpaces();705 } else {706 if (*at_ == '/' && IsCComment(at_)) {707 // Recognize and skip over classic C style /*comments*/ when708 // outside a character literal.709 if (features_.ShouldWarn(LanguageFeature::ClassicCComments)) {710 Say(LanguageFeature::ClassicCComments, GetCurrentProvenance(),711 "nonstandard usage: C-style comment"_port_en_US);712 }713 SkipCComments();714 }715 if (IsSpaceOrTab(at_)) {716 // Compress free-form white space into a single space character.717 const auto theSpace{at_};718 char previous{at_ <= start_ ? ' ' : at_[-1]};719 NextChar();720 SkipSpaces();721 if (*at_ == '\n' && !omitNewline_) {722 // Discard white space at the end of a line.723 } else if (!inPreprocessorDirective_ &&724 (previous == '(' || *at_ == '(' || *at_ == ')')) {725 // Discard white space before/after '(' and before ')', unless in a726 // preprocessor directive. This helps yield space-free contiguous727 // names for generic interfaces like OPERATOR( + ) and728 // READ ( UNFORMATTED ), without misinterpreting #define f (notAnArg).729 // This has the effect of silently ignoring the illegal spaces in730 // the array constructor ( /1,2/ ) but that seems benign; it's731 // hard to avoid that while still removing spaces from OPERATOR( / )732 // and OPERATOR( // ).733 } else {734 // Preserve the squashed white space as a single space character.735 tokens.PutNextTokenChar(' ', GetProvenance(theSpace));736 tokens.CloseToken();737 return true;738 }739 }740 }741 brokenToken_ = false;742 if (*at_ == '\n') {743 return false;744 }745 const char *start{at_};746 if (*at_ == '\'' || *at_ == '"') {747 QuotedCharacterLiteral(tokens, start);748 preventHollerith_ = false;749 } else if (IsDecimalDigit(*at_)) {750 int n{0}, digits{0};751 static constexpr int maxHollerith{256 /*lines*/ * (132 - 6 /*columns*/)};752 do {753 if (n < maxHollerith) {754 n = 10 * n + DecimalDigitValue(*at_);755 }756 EmitCharAndAdvance(tokens, *at_);757 ++digits;758 if (InFixedFormSource()) {759 SkipSpaces();760 }761 } while (IsDecimalDigit(*at_));762 if ((*at_ == 'h' || *at_ == 'H') && n > 0 && n < maxHollerith &&763 !preventHollerith_) {764 Hollerith(tokens, n, start);765 } else if (*at_ == '.') {766 while (IsDecimalDigit(EmitCharAndAdvance(tokens, *at_))) {767 }768 HandleExponentAndOrKindSuffix(tokens);769 } else if (HandleExponentAndOrKindSuffix(tokens)) {770 } else if (digits == 1 && n == 0 && (*at_ == 'x' || *at_ == 'X') &&771 inPreprocessorDirective_) {772 do {773 EmitCharAndAdvance(tokens, *at_);774 } while (IsHexadecimalDigit(*at_));775 } else if (at_[0] == '_' && (at_[1] == '\'' || at_[1] == '"')) { // 4_"..."776 EmitCharAndAdvance(tokens, *at_);777 QuotedCharacterLiteral(tokens, start);778 } else if (IsLetter(*at_) && !preventHollerith_ &&779 parenthesisNesting_ > 0 &&780 !preprocessor_.IsNameDefined(CharBlock{at_, 1})) {781 // Handles FORMAT(3I9HHOLLERITH) by skipping over the first I so that782 // we don't misrecognize I9HHOLLERITH as an identifier in the next case.783 EmitCharAndAdvance(tokens, *at_);784 }785 preventHollerith_ = false;786 } else if (*at_ == '.') {787 char nch{EmitCharAndAdvance(tokens, '.')};788 if (!inPreprocessorDirective_ && IsDecimalDigit(nch)) {789 while (IsDecimalDigit(EmitCharAndAdvance(tokens, *at_))) {790 }791 HandleExponentAndOrKindSuffix(tokens);792 } else if (nch == '.' && EmitCharAndAdvance(tokens, '.') == '.') {793 EmitCharAndAdvance(tokens, '.'); // variadic macro definition ellipsis794 }795 preventHollerith_ = false;796 } else if (IsLegalInIdentifier(*at_)) {797 std::size_t parts{1};798 bool anyDefined{false};799 bool hadContinuation{false};800 // Subtlety: When an identifier is split across continuation lines,801 // its parts are kept as distinct pp-tokens if macro replacement802 // should operate on them independently. This trick accommodates the803 // historic practice of using line continuation for token pasting after804 // replacement.805 // In free form, the macro to be replaced must have been preceded806 // by '&' and followed by either '&' or, if last, the end of a line.807 // call & call foo& call foo&808 // &MACRO& OR &MACRO& OR &MACRO809 // &foo(...) &(...)810 do {811 EmitChar(tokens, *at_);812 ++at_, ++column_;813 hadContinuation = SkipToNextSignificantCharacter();814 if (hadContinuation && IsLegalIdentifierStart(*at_)) {815 if (brokenToken_) {816 break;817 }818 // Continued identifier819 tokens.CloseToken();820 ++parts;821 if (!anyDefined &&822 (parts > 2 || inFixedForm_ ||823 (start > start_ && start[-1] == '&')) &&824 preprocessor_.IsNameDefined(825 tokens.TokenAt(tokens.SizeInTokens() - 1))) {826 anyDefined = true;827 }828 }829 } while (IsLegalInIdentifier(*at_));830 if (!anyDefined && parts > 1) {831 tokens.CloseToken();832 char after{*SkipWhiteSpace(at_)};833 anyDefined = (hadContinuation || after == '\n' || after == '&') &&834 preprocessor_.IsNameDefined(835 tokens.TokenAt(tokens.SizeInTokens() - 1));836 tokens.ReopenLastToken();837 }838 if (!anyDefined) {839 // If no part was a defined macro, combine the parts into one so that840 // the combination itself can be subject to macro replacement.841 while (parts-- > 1) {842 tokens.ReopenLastToken();843 }844 }845 if (InFixedFormSource()) {846 SkipSpaces();847 }848 if (inFixedForm_ && (IsOpenMPDirective() && parenthesisNesting_ > 0)) {849 SkipSpaces();850 }851 if ((*at_ == '\'' || *at_ == '"') &&852 tokens.CharAt(tokens.SizeInChars() - 1) == '_') { // kind_"..."853 QuotedCharacterLiteral(tokens, start);854 preventHollerith_ = false;855 } else {856 preventHollerith_ = true; // DO 10 H = ...857 }858 } else if (*at_ == '*') {859 if (EmitCharAndAdvance(tokens, '*') == '*') {860 EmitCharAndAdvance(tokens, '*');861 } else {862 // Subtle ambiguity:863 // CHARACTER*2H declares H because *2 is a kind specifier864 // DATAC/N*2H / is repeated Hollerith865 preventHollerith_ = !slashInCurrentStatement_;866 }867 } else {868 char ch{*at_};869 if (ch == '(') {870 if (parenthesisNesting_++ == 0) {871 isPossibleMacroCall_ = tokens.SizeInTokens() > 0 &&872 preprocessor_.IsFunctionLikeDefinition(873 tokens.TokenAt(tokens.SizeInTokens() - 1));874 }875 } else if (ch == ')' && parenthesisNesting_ > 0) {876 --parenthesisNesting_;877 }878 char nch{EmitCharAndAdvance(tokens, ch)};879 preventHollerith_ = false;880 if ((nch == '=' &&881 (ch == '<' || ch == '>' || ch == '/' || ch == '=' || ch == '!')) ||882 (ch == nch &&883 (ch == '/' || ch == ':' || ch == '*' || ch == '#' || ch == '&' ||884 ch == '|' || ch == '<' || ch == '>')) ||885 (ch == '=' && nch == '>')) {886 // token comprises two characters887 EmitCharAndAdvance(tokens, nch);888 } else if (ch == '/') {889 slashInCurrentStatement_ = true;890 } else if (ch == ';' && InFixedFormSource()) {891 SkipSpaces();892 if (IsDecimalDigit(*at_)) {893 if (features_.ShouldWarn(894 common::LanguageFeature::MiscSourceExtensions)) {895 Say(common::LanguageFeature::MiscSourceExtensions,896 GetProvenanceRange(at_, at_ + 1),897 "Label should be in the label field"_port_en_US);898 }899 }900 }901 }902 tokens.CloseToken();903 return true;904}905 906bool Prescanner::HandleExponent(TokenSequence &tokens) {907 if (char ed{ToLowerCaseLetter(*at_)}; ed == 'e' || ed == 'd') {908 // Do some look-ahead to ensure that this 'e'/'d' is an exponent,909 // not the start of an identifier that could be a macro.910 const char *startAt{at_};911 int startColumn{column_};912 TokenSequence possible;913 EmitCharAndAdvance(possible, *at_);914 if (InFixedFormSource()) {915 SkipSpaces();916 }917 if (*at_ == '+' || *at_ == '-') {918 EmitCharAndAdvance(possible, *at_);919 if (InFixedFormSource()) {920 SkipSpaces();921 }922 }923 if (IsDecimalDigit(*at_)) { // it's an exponent; scan it924 while (IsDecimalDigit(*at_)) {925 EmitCharAndAdvance(possible, *at_);926 if (InFixedFormSource()) {927 SkipSpaces();928 }929 }930 possible.CloseToken();931 tokens.AppendRange(possible, 0); // appends to current token932 return true;933 }934 // Not an exponent; backtrack935 at_ = startAt;936 column_ = startColumn;937 }938 return false;939}940 941bool Prescanner::HandleKindSuffix(TokenSequence &tokens) {942 if (*at_ != '_') {943 return false;944 }945 TokenSequence withUnderscore, separate;946 EmitChar(withUnderscore, '_');947 EmitCharAndAdvance(separate, '_');948 if (InFixedFormSource()) {949 SkipSpaces();950 }951 if (IsLegalInIdentifier(*at_)) {952 separate.CloseToken();953 EmitChar(withUnderscore, *at_);954 EmitCharAndAdvance(separate, *at_);955 if (InFixedFormSource()) {956 SkipSpaces();957 }958 while (IsLegalInIdentifier(*at_)) {959 EmitChar(withUnderscore, *at_);960 EmitCharAndAdvance(separate, *at_);961 if (InFixedFormSource()) {962 SkipSpaces();963 }964 }965 }966 withUnderscore.CloseToken();967 separate.CloseToken();968 tokens.CloseToken();969 if (separate.SizeInTokens() == 2 &&970 preprocessor_.IsNameDefined(separate.TokenAt(1)) &&971 !preprocessor_.IsNameDefined(withUnderscore.ToCharBlock())) {972 // "_foo" is not defined, but "foo" is973 tokens.CopyAll(separate); // '_' "foo"974 } else {975 tokens.CopyAll(withUnderscore); // "_foo"976 }977 return true;978}979 980bool Prescanner::HandleExponentAndOrKindSuffix(TokenSequence &tokens) {981 bool hadExponent{HandleExponent(tokens)};982 if (HandleKindSuffix(tokens)) {983 return true;984 } else {985 return hadExponent;986 }987}988 989void Prescanner::QuotedCharacterLiteral(990 TokenSequence &tokens, const char *start) {991 char quote{*at_};992 const char *end{at_ + 1};993 inCharLiteral_ = true;994 continuationInCharLiteral_ = true;995 const auto emit{[&](char ch) { EmitChar(tokens, ch); }};996 const auto insert{[&](char ch) { EmitInsertedChar(tokens, ch); }};997 bool isEscaped{false};998 bool escapesEnabled{features_.IsEnabled(LanguageFeature::BackslashEscapes)};999 while (true) {1000 if (*at_ == '\\') {1001 if (escapesEnabled) {1002 isEscaped = !isEscaped;1003 } else {1004 // The parser always processes escape sequences, so don't confuse it1005 // when escapes are disabled.1006 insert('\\');1007 }1008 } else {1009 isEscaped = false;1010 }1011 if (*at_ == '\n') {1012 if (inPreprocessorDirective_) {1013 EmitQuotedChar(static_cast<unsigned char>(*at_), emit, insert, false,1014 Encoding::LATIN_1);1015 } else if (InCompilerDirective() && preprocessingOnly_) {1016 // don't complain about -E output of !$, do it in later compilation1017 } else {1018 Say(GetProvenanceRange(start, end),1019 "Incomplete character literal"_err_en_US);1020 }1021 break;1022 }1023 EmitQuotedChar(static_cast<unsigned char>(*at_), emit, insert, false,1024 Encoding::LATIN_1);1025 while (PadOutCharacterLiteral(tokens)) {1026 }1027 // Here's a weird edge case. When there's a two or more following1028 // continuation lines at this point, and the entire significant part of1029 // the next continuation line is the name of a keyword macro, replace1030 // it in the character literal with its definition. Example:1031 // #define FOO foo1032 // subroutine subr() bind(c, name="my_&1033 // &FOO&1034 // &_bar") ...1035 // produces a binding name of "my_foo_bar".1036 while (at_[1] == '&' && nextLine_ < limit_ && !InFixedFormSource()) {1037 const char *idStart{nextLine_};1038 if (const char *amper{SkipWhiteSpace(nextLine_)}; *amper == '&') {1039 idStart = amper + 1;1040 }1041 if (IsLegalIdentifierStart(*idStart)) {1042 std::size_t idLen{1};1043 for (; IsLegalInIdentifier(idStart[idLen]); ++idLen) {1044 }1045 if (idStart[idLen] == '&') {1046 CharBlock id{idStart, idLen};1047 if (preprocessor_.IsNameDefined(id)) {1048 TokenSequence ppTokens;1049 ppTokens.Put(id, GetProvenance(idStart));1050 if (auto replaced{1051 preprocessor_.MacroReplacement(ppTokens, *this)}) {1052 tokens.CopyAll(*replaced);1053 at_ = &idStart[idLen - 1];1054 NextLine();1055 continue; // try again on the next line1056 }1057 }1058 }1059 }1060 break;1061 }1062 end = at_ + 1;1063 NextChar();1064 if (*at_ == quote && !isEscaped) {1065 // A doubled unescaped quote mark becomes a single instance of that1066 // quote character in the literal (later). There can be spaces between1067 // the quotes in fixed form source.1068 EmitChar(tokens, quote);1069 inCharLiteral_ = false; // for cases like print *, '...'!comment1070 NextChar();1071 if (InFixedFormSource()) {1072 SkipSpaces();1073 }1074 if (*at_ != quote) {1075 break;1076 }1077 inCharLiteral_ = true;1078 }1079 }1080 continuationInCharLiteral_ = false;1081 inCharLiteral_ = false;1082}1083 1084void Prescanner::Hollerith(1085 TokenSequence &tokens, int count, const char *start) {1086 inCharLiteral_ = true;1087 CHECK(*at_ == 'h' || *at_ == 'H');1088 EmitChar(tokens, 'H');1089 while (count-- > 0) {1090 if (PadOutCharacterLiteral(tokens)) {1091 } else if (*at_ == '\n') {1092 if (features_.ShouldWarn(common::UsageWarning::Scanning)) {1093 Say(common::UsageWarning::Scanning, GetProvenanceRange(start, at_),1094 "Possible truncated Hollerith literal"_warn_en_US);1095 }1096 break;1097 } else {1098 NextChar();1099 // Each multi-byte character encoding counts as a single character.1100 // No escape sequences are recognized.1101 // Hollerith is always emitted to the cooked character1102 // stream in UTF-8.1103 DecodedCharacter decoded{DecodeCharacter(1104 encoding_, at_, static_cast<std::size_t>(limit_ - at_), false)};1105 if (decoded.bytes > 0) {1106 EncodedCharacter utf8{1107 EncodeCharacter<Encoding::UTF_8>(decoded.codepoint)};1108 for (int j{0}; j < utf8.bytes; ++j) {1109 EmitChar(tokens, utf8.buffer[j]);1110 }1111 at_ += decoded.bytes - 1;1112 } else {1113 Say(GetProvenanceRange(start, at_),1114 "Bad character in Hollerith literal"_err_en_US);1115 break;1116 }1117 }1118 }1119 if (*at_ != '\n') {1120 NextChar();1121 }1122 inCharLiteral_ = false;1123}1124 1125// In fixed form, source card images must be processed as if they were at1126// least 72 columns wide, at least in character literal contexts.1127bool Prescanner::PadOutCharacterLiteral(TokenSequence &tokens) {1128 while (inFixedForm_ && !tabInCurrentLine_ && at_[1] == '\n') {1129 if (column_ < fixedFormColumnLimit_) {1130 tokens.PutNextTokenChar(' ', spaceProvenance_);1131 ++column_;1132 return true;1133 }1134 if (!FixedFormContinuation(false /*no need to insert space*/) ||1135 tabInCurrentLine_) {1136 return false;1137 }1138 CHECK(column_ == 7);1139 --at_; // point to column 6 of continuation line1140 column_ = 6;1141 }1142 return false;1143}1144 1145static bool IsAtProcess(const char *p) {1146 static const char pAtProc[]{"process"};1147 for (std::size_t i{0}; i < sizeof pAtProc - 1; ++i) {1148 if (ToLowerCaseLetter(*++p) != pAtProc[i])1149 return false;1150 }1151 return true;1152}1153 1154bool Prescanner::IsFixedFormCommentLine(const char *start) const {1155 const char *p{start};1156 // The @process directive must start in column 1.1157 if (*p == '@' && IsAtProcess(p)) {1158 return true;1159 }1160 if (IsFixedFormCommentChar(*p) || *p == '%' || // VAX %list, %eject, &c.1161 ((*p == 'D' || *p == 'd') &&1162 !features_.IsEnabled(LanguageFeature::OldDebugLines))) {1163 return true;1164 }1165 bool anyTabs{false};1166 while (true) {1167 if (int n{IsSpace(p)}) {1168 p += n;1169 } else if (*p == '\t') {1170 anyTabs = true;1171 ++p;1172 } else if (*p == '0' && !anyTabs && p == start + 5) {1173 ++p; // 0 in column 6 must treated as a space1174 } else {1175 break;1176 }1177 }1178 if (!anyTabs && p >= start + fixedFormColumnLimit_) {1179 return true;1180 }1181 if (*p == '!' && !inCharLiteral_ && (anyTabs || p != start + 5)) {1182 return true;1183 }1184 return *p == '\n';1185}1186 1187const char *Prescanner::IsFreeFormComment(const char *p) const {1188 p = SkipWhiteSpaceAndCComments(p);1189 if (*p == '!' || *p == '\n') {1190 return p;1191 } else if (*p == '@') {1192 return IsAtProcess(p) ? p : nullptr;1193 } else {1194 return nullptr;1195 }1196}1197 1198std::optional<std::size_t> Prescanner::IsIncludeLine(const char *start) const {1199 if (!expandIncludeLines_) {1200 return std::nullopt;1201 }1202 const char *p{SkipWhiteSpace(start)};1203 if (*p == '0' && inFixedForm_ && p == start + 5) {1204 // Accept " 0INCLUDE" in fixed form.1205 p = SkipWhiteSpace(p + 1);1206 }1207 for (const char *q{"include"}; *q; ++q) {1208 if (ToLowerCaseLetter(*p) != *q) {1209 return std::nullopt;1210 }1211 p = SkipWhiteSpace(p + 1);1212 }1213 if (IsDecimalDigit(*p)) { // accept & ignore a numeric kind prefix1214 for (p = SkipWhiteSpace(p + 1); IsDecimalDigit(*p);1215 p = SkipWhiteSpace(p + 1)) {1216 }1217 if (*p != '_') {1218 return std::nullopt;1219 }1220 p = SkipWhiteSpace(p + 1);1221 }1222 if (*p == '"' || *p == '\'') {1223 return {p - start};1224 }1225 return std::nullopt;1226}1227 1228void Prescanner::FortranInclude(const char *firstQuote) {1229 const char *p{firstQuote};1230 while (*p != '"' && *p != '\'') {1231 ++p;1232 }1233 char quote{*p};1234 std::string path;1235 for (++p; *p != '\n'; ++p) {1236 if (*p == quote) {1237 if (p[1] != quote) {1238 break;1239 }1240 ++p;1241 }1242 path += *p;1243 }1244 if (*p != quote) {1245 Say(GetProvenanceRange(firstQuote, p),1246 "malformed path name string"_err_en_US);1247 return;1248 }1249 p = SkipWhiteSpace(p + 1);1250 if (*p != '\n' && *p != '!') {1251 const char *garbage{p};1252 for (; *p != '\n' && *p != '!'; ++p) {1253 }1254 if (features_.ShouldWarn(common::UsageWarning::Scanning)) {1255 Say(common::UsageWarning::Scanning, GetProvenanceRange(garbage, p),1256 "excess characters after path name"_warn_en_US);1257 }1258 }1259 std::string buf;1260 llvm::raw_string_ostream error{buf};1261 Provenance provenance{GetProvenance(nextLine_)};1262 std::optional<std::string> prependPath;1263 if (const SourceFile * currentFile{allSources_.GetSourceFile(provenance)}) {1264 prependPath = DirectoryName(currentFile->path());1265 }1266 const SourceFile *included{1267 allSources_.Open(path, error, std::move(prependPath))};1268 if (!included) {1269 Say(provenance, "INCLUDE: %s"_err_en_US, buf);1270 } else if (included->bytes() > 0) {1271 ProvenanceRange includeLineRange{1272 provenance, static_cast<std::size_t>(p - nextLine_)};1273 ProvenanceRange fileRange{1274 allSources_.AddIncludedFile(*included, includeLineRange)};1275 Preprocessor cleanPrepro{allSources_};1276 if (preprocessor_.IsNameDefined("__FILE__"s)) {1277 cleanPrepro.DefineStandardMacros(); // __FILE__, __LINE__, &c.1278 }1279 if (preprocessor_.IsNameDefined("_CUDA"s)) {1280 cleanPrepro.Define("_CUDA"s, "1");1281 }1282 Prescanner{*this, cleanPrepro, /*isNestedInIncludeDirective=*/false}1283 .set_encoding(included->encoding())1284 .Prescan(fileRange);1285 }1286}1287 1288const char *Prescanner::IsPreprocessorDirectiveLine(const char *start) const {1289 const char *p{start};1290 while (int n{IsSpace(p)}) {1291 p += n;1292 }1293 if (*p == '#') {1294 if (inFixedForm_ && p == start + 5) {1295 return nullptr;1296 }1297 } else {1298 p = SkipWhiteSpace(p);1299 if (*p != '#') {1300 return nullptr;1301 }1302 }1303 return SkipWhiteSpace(p + 1);1304}1305 1306bool Prescanner::IsNextLinePreprocessorDirective() const {1307 return IsPreprocessorDirectiveLine(nextLine_) != nullptr;1308}1309 1310bool Prescanner::SkipCommentLine(bool afterAmpersand) {1311 if (IsAtEnd()) {1312 if (afterAmpersand && prescannerNesting_ > 0) {1313 // A continuation marker at the end of the last line in an1314 // include file inhibits the newline for that line.1315 SkipToEndOfLine();1316 omitNewline_ = true;1317 }1318 } else if (inPreprocessorDirective_) {1319 } else {1320 auto lineClass{ClassifyLine(nextLine_)};1321 if (lineClass.kind == LineClassification::Kind::Comment) {1322 NextLine();1323 return true;1324 } else if (lineClass.kind ==1325 LineClassification::Kind::ConditionalCompilationDirective ||1326 lineClass.kind == LineClassification::Kind::PreprocessorDirective) {1327 // Allow conditional compilation directives (e.g., #ifdef) to affect1328 // continuation lines.1329 // Allow other preprocessor directives, too, except #include1330 // (when it does not follow '&'), #define, and #undef (because1331 // they cannot be allowed to affect preceding text on a1332 // continued line).1333 preprocessor_.Directive(TokenizePreprocessorDirective(), *this);1334 return true;1335 } else if (afterAmpersand &&1336 (lineClass.kind == LineClassification::Kind::DefinitionDirective ||1337 lineClass.kind == LineClassification::Kind::IncludeDirective ||1338 lineClass.kind == LineClassification::Kind::IncludeLine)) {1339 SkipToEndOfLine();1340 omitNewline_ = true;1341 skipLeadingAmpersand_ = true;1342 }1343 }1344 return false;1345}1346 1347const char *Prescanner::FixedFormContinuationLine(bool atNewline) {1348 if (IsAtEnd()) {1349 return nullptr;1350 }1351 tabInCurrentLine_ = false;1352 char col1{*nextLine_};1353 bool canBeNonDirectiveContinuation{1354 (col1 == ' ' ||1355 ((col1 == 'D' || col1 == 'd') &&1356 features_.IsEnabled(LanguageFeature::OldDebugLines))) &&1357 nextLine_[1] == ' ' && nextLine_[2] == ' ' && nextLine_[3] == ' ' &&1358 nextLine_[4] == ' '};1359 if (InCompilerDirective() && !(InConditionalLine() && !preprocessingOnly_)) {1360 // !$ under -E is not continued, but deferred to later compilation1361 if (IsFixedFormCommentChar(col1) &&1362 !(InConditionalLine() && preprocessingOnly_)) {1363 int j{1};1364 for (; j < 5; ++j) {1365 char ch{directiveSentinel_[j - 1]};1366 if (ch == '\0') {1367 break;1368 } else if (ch != ToLowerCaseLetter(nextLine_[j])) {1369 return nullptr;1370 }1371 }1372 for (; j < 5; ++j) {1373 if (nextLine_[j] != ' ') {1374 return nullptr;1375 }1376 }1377 const char *col6{nextLine_ + 5};1378 if (*col6 != '\n' && *col6 != '0' && !IsSpaceOrTab(col6)) {1379 if (atNewline && !IsSpace(nextLine_ + 6)) {1380 brokenToken_ = true;1381 }1382 return nextLine_ + 6;1383 }1384 }1385 } else { // Normal case: not in a compiler directive.1386 // Conditional compilation lines may be continuations when not1387 // just preprocessing.1388 if (!preprocessingOnly_ && IsFixedFormCommentChar(col1)) {1389 if ((nextLine_[1] == '$' && nextLine_[2] == ' ' && nextLine_[3] == ' ' &&1390 nextLine_[4] == ' ' &&1391 IsCompilerDirectiveSentinel(&nextLine_[1], 1)) ||1392 (nextLine_[1] == '@' &&1393 IsCompilerDirectiveSentinel(&nextLine_[1], 4))) {1394 if (const char *col6{nextLine_ + 5};1395 *col6 != '\n' && *col6 != '0' && !IsSpaceOrTab(col6)) {1396 if (atNewline && !IsSpace(nextLine_ + 6)) {1397 brokenToken_ = true;1398 }1399 return nextLine_ + 6;1400 } else {1401 return nullptr;1402 }1403 }1404 }1405 if (col1 == '&' &&1406 features_.IsEnabled(1407 LanguageFeature::FixedFormContinuationWithColumn1Ampersand)) {1408 // Extension: '&' as continuation marker1409 if (features_.ShouldWarn(1410 LanguageFeature::FixedFormContinuationWithColumn1Ampersand)) {1411 Say(LanguageFeature::FixedFormContinuationWithColumn1Ampersand,1412 GetProvenance(nextLine_), "nonstandard usage"_port_en_US);1413 }1414 return nextLine_ + 1;1415 }1416 if (col1 == '\t' && nextLine_[1] >= '1' && nextLine_[1] <= '9') {1417 tabInCurrentLine_ = true;1418 return nextLine_ + 2; // VAX extension1419 }1420 if (canBeNonDirectiveContinuation) {1421 const char *col6{nextLine_ + 5};1422 if (*col6 != '\n' && *col6 != '0' && !IsSpaceOrTab(col6)) {1423 if ((*col6 == 'i' || *col6 == 'I') && IsIncludeLine(nextLine_)) {1424 // It's an INCLUDE line, not a continuation1425 } else {1426 return nextLine_ + 6;1427 }1428 }1429 }1430 if (IsImplicitContinuation()) {1431 return nextLine_;1432 }1433 }1434 return nullptr; // not a continuation line1435}1436 1437constexpr bool IsDirective(const char *match, const char *dir) {1438 for (; *match; ++match) {1439 if (*match != ToLowerCaseLetter(*dir++)) {1440 return false;1441 }1442 }1443 return true;1444}1445 1446const char *Prescanner::FreeFormContinuationLine(bool ampersand) {1447 const char *lineStart{nextLine_};1448 const char *p{lineStart};1449 if (p >= limit_) {1450 return nullptr;1451 }1452 p = SkipWhiteSpaceIncludingEmptyMacros(p);1453 if (InCompilerDirective()) {1454 if (InConditionalLine()) {1455 if (preprocessingOnly_) {1456 // in -E mode, don't treat !$ as a continuation1457 return nullptr;1458 } else if (p[0] == '!' && (p[1] == '$' || p[1] == '@')) {1459 p += 2;1460 if (InOpenACCOrCUDAConditionalLine()) {1461 if (IsDirective("acc", p) || IsDirective("cuf", p)) {1462 p += 3;1463 } else {1464 return nullptr;1465 }1466 }1467 if (*p != '&' && !IsSpaceOrTab(p)) {1468 return nullptr;1469 }1470 }1471 } else if (*p++ == '!') {1472 for (const char *s{directiveSentinel_}; *s != '\0'; ++p, ++s) {1473 if (*s != ToLowerCaseLetter(*p)) {1474 return nullptr; // not the same directive class1475 }1476 }1477 } else {1478 return nullptr;1479 }1480 p = SkipWhiteSpace(p);1481 if (*p == '&') {1482 if (!ampersand) {1483 brokenToken_ = true;1484 }1485 return p + 1;1486 } else if (ampersand) {1487 return p;1488 } else {1489 return nullptr;1490 }1491 }1492 if (p[0] == '!' && !preprocessingOnly_) {1493 // Conditional lines can be continuations1494 if (p[1] == '$' && features_.IsEnabled(LanguageFeature::OpenMP)) {1495 p = lineStart = SkipWhiteSpace(p + 2);1496 } else if (IsDirective("@acc", p + 1) &&1497 features_.IsEnabled(LanguageFeature::OpenACC)) {1498 p = lineStart = SkipWhiteSpace(p + 5);1499 } else if (IsDirective("@cuf", p + 1) &&1500 features_.IsEnabled(LanguageFeature::CUDA)) {1501 p = lineStart = SkipWhiteSpace(p + 5);1502 }1503 }1504 if (*p == '&') {1505 return p + 1;1506 } else if (*p == '!' || *p == '\n' || *p == '#') {1507 return nullptr;1508 } else if (ampersand || IsImplicitContinuation()) {1509 if (continuationInCharLiteral_) {1510 // 'a'& -> 'a''b' == "a'b"1511 // 'b'1512 if (features_.ShouldWarn(common::LanguageFeature::MiscSourceExtensions)) {1513 Say(common::LanguageFeature::MiscSourceExtensions,1514 GetProvenanceRange(p, p + 1),1515 "Character literal continuation line should have been preceded by '&'"_port_en_US);1516 }1517 } else if (p > lineStart && IsSpaceOrTab(p - 1)) {1518 --p;1519 } else {1520 brokenToken_ = true;1521 }1522 return p;1523 } else {1524 return nullptr;1525 }1526}1527 1528bool Prescanner::FixedFormContinuation(bool atNewline) {1529 // N.B. We accept '&' as a continuation indicator in fixed form, too,1530 // but not in a character literal.1531 if (*at_ == '&' && inCharLiteral_) {1532 return false;1533 }1534 do {1535 if (const char *cont{FixedFormContinuationLine(atNewline)}) {1536 BeginSourceLine(cont);1537 column_ = 7;1538 NextLine();1539 return true;1540 }1541 } while (SkipCommentLine(false /* not after ampersand */));1542 return false;1543}1544 1545bool Prescanner::FreeFormContinuation() {1546 const char *p{at_};1547 bool ampersand{*p == '&'};1548 if (ampersand) {1549 p = SkipWhiteSpace(p + 1);1550 }1551 if (*p != '\n') {1552 if (inCharLiteral_) {1553 return false;1554 } else if (*p == '!') { // & ! comment - ok1555 } else if (ampersand && isPossibleMacroCall_ && (*p == ',' || *p == ')')) {1556 return false; // allow & at end of a macro argument1557 } else if (ampersand && preprocessingOnly_ && !parenthesisNesting_) {1558 return false; // allow & at start of line, maybe after !$1559 } else if (features_.ShouldWarn(LanguageFeature::CruftAfterAmpersand)) {1560 Say(LanguageFeature::CruftAfterAmpersand, GetProvenance(p),1561 "missing ! before comment after &"_warn_en_US);1562 }1563 }1564 do {1565 if (const char *cont{FreeFormContinuationLine(ampersand)}) {1566 BeginSourceLine(cont);1567 NextLine();1568 return true;1569 }1570 } while (SkipCommentLine(ampersand));1571 return false;1572}1573 1574// Implicit line continuation allows a preprocessor macro call with1575// arguments to span multiple lines.1576bool Prescanner::IsImplicitContinuation() const {1577 return !inPreprocessorDirective_ && !inCharLiteral_ && isPossibleMacroCall_ &&1578 parenthesisNesting_ > 0 && !IsAtEnd() &&1579 ClassifyLine(nextLine_).kind == LineClassification::Kind::Source;1580}1581 1582bool Prescanner::Continuation(bool mightNeedFixedFormSpace) {1583 if (disableSourceContinuation_) {1584 return false;1585 } else if (*at_ == '\n' || *at_ == '&') {1586 if (inFixedForm_) {1587 return FixedFormContinuation(mightNeedFixedFormSpace);1588 } else {1589 return FreeFormContinuation();1590 }1591 } else if (*at_ == '\\' && at_ + 2 == nextLine_ &&1592 backslashFreeFormContinuation_ && !inFixedForm_ && nextLine_ < limit_) {1593 // cpp-like handling of \ at end of a free form source line1594 BeginSourceLine(nextLine_);1595 NextLine();1596 return true;1597 } else {1598 return false;1599 }1600}1601 1602std::optional<Prescanner::LineClassification>1603Prescanner::IsFixedFormCompilerDirectiveLine(const char *start) const {1604 const char *p{start};1605 char col1{*p++};1606 if (!IsFixedFormCommentChar(col1)) {1607 return std::nullopt;1608 }1609 char sentinel[5], *sp{sentinel};1610 int column{2};1611 for (; column < 6; ++column) {1612 if (*p == '\n' || IsSpaceOrTab(p) || IsDecimalDigit(*p)) {1613 break;1614 }1615 *sp++ = ToLowerCaseLetter(*p++);1616 }1617 if (sp == sentinel) {1618 return std::nullopt;1619 }1620 *sp = '\0';1621 // A fixed form OpenMP conditional compilation sentinel must satisfy the1622 // following criteria, for initial lines:1623 // - Columns 3 through 5 must have only white space or numbers.1624 // - Column 6 must be space or zero.1625 bool isOpenMPConditional{sp == &sentinel[1] && sentinel[0] == '$'};1626 bool hadDigit{false};1627 if (isOpenMPConditional) {1628 for (; column < 6; ++column, ++p) {1629 if (IsDecimalDigit(*p)) {1630 hadDigit = true;1631 } else if (!IsSpaceOrTab(p)) {1632 return std::nullopt;1633 }1634 }1635 }1636 if (column == 6) {1637 if (*p == '0') {1638 ++p;1639 } else if (int n{IsSpaceOrTab(p)}) {1640 p += n;1641 } else if (isOpenMPConditional && preprocessingOnly_ && !hadDigit &&1642 *p != '\n') {1643 // In -E mode, "!$ &" is treated as a directive1644 } else {1645 // This is a Continuation line, not an initial directive line.1646 return std::nullopt;1647 }1648 ++column, ++p;1649 }1650 if (isOpenMPConditional) {1651 for (; column <= fixedFormColumnLimit_; ++column, ++p) {1652 if (IsSpaceOrTab(p)) {1653 } else if (*p == '!') {1654 return std::nullopt; // !$ ! is a comment, not a directive1655 } else {1656 break;1657 }1658 }1659 }1660 if (const char *ss{IsCompilerDirectiveSentinel(1661 sentinel, static_cast<std::size_t>(sp - sentinel))}) {1662 return {1663 LineClassification{LineClassification::Kind::CompilerDirective, 0, ss}};1664 }1665 return std::nullopt;1666}1667 1668std::optional<Prescanner::LineClassification>1669Prescanner::IsFreeFormCompilerDirectiveLine(const char *start) const {1670 if (const char *p{SkipWhiteSpaceIncludingEmptyMacros(start)};1671 p && *p++ == '!') {1672 if (auto maybePair{IsCompilerDirectiveSentinel(p)}) {1673 auto offset{static_cast<std::size_t>(p - start - 1)};1674 const char *sentinel{maybePair->first};1675 if ((sentinel[0] == '$' && sentinel[1] == '\0') || sentinel[1] == '@') {1676 if (const char *comment{IsFreeFormComment(maybePair->second)}) {1677 if (*comment == '!') {1678 // Conditional line comment - treat as comment1679 return std::nullopt;1680 }1681 }1682 }1683 return {LineClassification{1684 LineClassification::Kind::CompilerDirective, offset, sentinel}};1685 }1686 }1687 return std::nullopt;1688}1689 1690Prescanner &Prescanner::AddCompilerDirectiveSentinel(const std::string &dir) {1691 std::uint64_t packed{0};1692 for (char ch : dir) {1693 packed = (packed << 8) | (ToLowerCaseLetter(ch) & 0xff);1694 }1695 compilerDirectiveBloomFilter_.set(packed % prime1);1696 compilerDirectiveBloomFilter_.set(packed % prime2);1697 compilerDirectiveSentinels_.insert(dir);1698 return *this;1699}1700 1701const char *Prescanner::IsCompilerDirectiveSentinel(1702 const char *sentinel, std::size_t len) const {1703 std::uint64_t packed{0};1704 for (std::size_t j{0}; j < len; ++j) {1705 packed = (packed << 8) | (sentinel[j] & 0xff);1706 }1707 if (len == 0 || !compilerDirectiveBloomFilter_.test(packed % prime1) ||1708 !compilerDirectiveBloomFilter_.test(packed % prime2)) {1709 return nullptr;1710 }1711 const auto iter{compilerDirectiveSentinels_.find(std::string(sentinel, len))};1712 return iter == compilerDirectiveSentinels_.end() ? nullptr : iter->c_str();1713}1714 1715const char *Prescanner::IsCompilerDirectiveSentinel(CharBlock token) const {1716 const char *p{token.begin()};1717 const char *end{p + token.size()};1718 while (p < end && (*p == ' ' || *p == '\n')) {1719 ++p;1720 }1721 if (p < end && *p == '!') {1722 ++p;1723 }1724 while (end > p && (end[-1] == ' ' || end[-1] == '\t')) {1725 --end;1726 }1727 return end > p && IsCompilerDirectiveSentinel(p, end - p) ? p : nullptr;1728}1729 1730std::optional<std::pair<const char *, const char *>>1731Prescanner::IsCompilerDirectiveSentinel(const char *p) const {1732 char sentinel[8];1733 for (std::size_t j{0}; j + 1 < sizeof sentinel; ++p, ++j) {1734 if (int n{IsSpaceOrTab(p)};1735 n || !(IsLetter(*p) || *p == '$' || *p == '@')) {1736 if (j > 0) {1737 if (j == 1 && sentinel[0] == '$' && n == 0 && *p != '&' && *p != '\n') {1738 // Free form OpenMP conditional compilation line sentinels have to1739 // be immediately followed by a space or &, not a digit1740 // or anything else. A newline also works for an initial line.1741 break;1742 }1743 sentinel[j] = '\0';1744 if (*p != '!') {1745 if (const char *sp{IsCompilerDirectiveSentinel(sentinel, j)}) {1746 return std::make_pair(sp, p);1747 }1748 }1749 }1750 break;1751 } else {1752 sentinel[j] = ToLowerCaseLetter(*p);1753 }1754 }1755 return std::nullopt;1756}1757 1758Prescanner::LineClassification Prescanner::ClassifyLine(1759 const char *start) const {1760 if (inFixedForm_) {1761 if (std::optional<LineClassification> lc{1762 IsFixedFormCompilerDirectiveLine(start)}) {1763 return std::move(*lc);1764 }1765 if (IsFixedFormCommentLine(start)) {1766 return {LineClassification::Kind::Comment};1767 }1768 } else {1769 if (std::optional<LineClassification> lc{1770 IsFreeFormCompilerDirectiveLine(start)}) {1771 return std::move(*lc);1772 }1773 if (const char *bang{IsFreeFormComment(start)}) {1774 return {LineClassification::Kind::Comment,1775 static_cast<std::size_t>(bang - start)};1776 }1777 }1778 if (std::optional<std::size_t> quoteOffset{IsIncludeLine(start)}) {1779 return {LineClassification::Kind::IncludeLine, *quoteOffset};1780 }1781 if (const char *dir{IsPreprocessorDirectiveLine(start)}) {1782 if (IsDirective("if", dir) || IsDirective("elif", dir) ||1783 IsDirective("else", dir) || IsDirective("endif", dir)) {1784 return {LineClassification::Kind::ConditionalCompilationDirective};1785 } else if (IsDirective("include", dir)) {1786 return {LineClassification::Kind::IncludeDirective};1787 } else if (IsDirective("define", dir) || IsDirective("undef", dir)) {1788 return {LineClassification::Kind::DefinitionDirective};1789 } else {1790 return {LineClassification::Kind::PreprocessorDirective};1791 }1792 }1793 return {LineClassification::Kind::Source};1794}1795 1796Prescanner::LineClassification Prescanner::ClassifyLine(1797 TokenSequence &tokens, Provenance newlineProvenance) const {1798 // Append a newline temporarily.1799 tokens.PutNextTokenChar('\n', newlineProvenance);1800 tokens.CloseToken();1801 const char *ppd{tokens.ToCharBlock().begin()};1802 LineClassification classification{ClassifyLine(ppd)};1803 tokens.pop_back(); // remove the newline1804 return classification;1805}1806 1807bool Prescanner::SourceFormChange(std::string &&dir) {1808 if (dir == "!dir$ free") {1809 inFixedForm_ = false;1810 return true;1811 } else if (dir == "!dir$ fixed") {1812 inFixedForm_ = true;1813 return true;1814 } else {1815 return false;1816 }1817}1818 1819// Acquire and append compiler directive continuation lines to1820// the tokens that constitute a compiler directive, even when those1821// directive continuation lines are the result of macro expansion.1822// (Not used when neither the original compiler directive line nor1823// the directive continuation line result from preprocessing; regular1824// line continuation during tokenization handles that normal case.)1825bool Prescanner::CompilerDirectiveContinuation(1826 TokenSequence &tokens, const char *origSentinel) {1827 if (inFixedForm_ || tokens.empty() ||1828 tokens.TokenAt(tokens.SizeInTokens() - 1) != "&" ||1829 (preprocessingOnly_ && !parenthesisNesting_)) {1830 return false;1831 }1832 LineClassification followingLine{ClassifyLine(nextLine_)};1833 if (followingLine.kind == LineClassification::Kind::Comment) {1834 nextLine_ += followingLine.payloadOffset; // advance to '!' or newline1835 NextLine();1836 return true;1837 }1838 CHECK(origSentinel != nullptr);1839 directiveSentinel_ = origSentinel; // so InCompilerDirective() is true1840 const char *nextContinuation{1841 followingLine.kind == LineClassification::Kind::CompilerDirective1842 ? FreeFormContinuationLine(true)1843 : nullptr};1844 if (!nextContinuation &&1845 followingLine.kind != LineClassification::Kind::Source) {1846 return false;1847 }1848 auto origNextLine{nextLine_};1849 BeginSourceLine(nextLine_);1850 NextLine();1851 if (nextContinuation) {1852 // What follows is !DIR$ & xxx; skip over the & so that it1853 // doesn't cause a spurious continuation.1854 at_ = nextContinuation;1855 } else {1856 // What follows looks like a source line before macro expansion,1857 // but might become a directive continuation afterwards.1858 SkipSpaces();1859 }1860 TokenSequence followingTokens;1861 while (NextToken(followingTokens)) {1862 }1863 if (auto followingPrepro{1864 preprocessor_.MacroReplacement(followingTokens, *this)}) {1865 followingTokens = std::move(*followingPrepro);1866 }1867 followingTokens.RemoveRedundantBlanks();1868 std::size_t startAt{0};1869 std::size_t following{followingTokens.SizeInTokens()};1870 bool ok{false};1871 if (nextContinuation) {1872 ok = true;1873 } else {1874 startAt = 2;1875 if (startAt < following && followingTokens.TokenAt(0) == "!") {1876 CharBlock sentinel{followingTokens.TokenAt(1)};1877 if (!sentinel.empty() &&1878 std::memcmp(sentinel.begin(), origSentinel, sentinel.size()) == 0) {1879 ok = true;1880 while (1881 startAt < following && followingTokens.TokenAt(startAt).IsBlank()) {1882 ++startAt;1883 }1884 if (startAt < following && followingTokens.TokenAt(startAt) == "&") {1885 ++startAt;1886 }1887 }1888 }1889 }1890 if (ok) {1891 tokens.pop_back(); // delete original '&'1892 tokens.AppendRange(followingTokens, startAt, following - startAt);1893 tokens.RemoveRedundantBlanks();1894 } else {1895 nextLine_ = origNextLine;1896 }1897 return ok;1898}1899 1900// Similar, but for source line continuation after macro replacement.1901bool Prescanner::SourceLineContinuation(TokenSequence &tokens) {1902 if (!inFixedForm_ && !tokens.empty() &&1903 tokens.TokenAt(tokens.SizeInTokens() - 1) == "&") {1904 LineClassification followingLine{ClassifyLine(nextLine_)};1905 if (followingLine.kind == LineClassification::Kind::Comment) {1906 nextLine_ += followingLine.payloadOffset; // advance to '!' or newline1907 NextLine();1908 return true;1909 } else if (const char *nextContinuation{FreeFormContinuationLine(true)}) {1910 BeginSourceLine(nextLine_);1911 NextLine();1912 TokenSequence followingTokens;1913 at_ = nextContinuation;1914 while (NextToken(followingTokens)) {1915 }1916 if (auto followingPrepro{1917 preprocessor_.MacroReplacement(followingTokens, *this)}) {1918 followingTokens = std::move(*followingPrepro);1919 }1920 followingTokens.RemoveRedundantBlanks();1921 tokens.pop_back(); // delete original '&'1922 tokens.CopyAll(followingTokens);1923 return true;1924 }1925 }1926 return false;1927}1928} // namespace Fortran::parser1929