316 lines · cpp
1//===-- lib/Parser/characters.cpp -----------------------------------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8 9#include "flang/Parser/characters.h"10#include "flang/Common/idioms.h"11#include <algorithm>12#include <cstddef>13#include <optional>14#include <type_traits>15 16namespace Fortran::parser {17 18bool useHexadecimalEscapeSequences{false};19 20int UTF_8CharacterBytes(const char *p) {21 if ((*p & 0x80) == 0) {22 return 1;23 } else if ((*p & 0xe0) == 0xc0) {24 return 2;25 } else if ((*p & 0xf0) == 0xe0) {26 return 3;27 } else if ((*p & 0xf8) == 0xf0) {28 return 4;29 } else if ((*p & 0xfc) == 0xf8) {30 return 5;31 } else {32 return 6;33 }34}35 36template <typename STRING>37std::string QuoteCharacterLiteralHelper(38 const STRING &str, bool backslashEscapes, Encoding encoding) {39 std::string result{'"'};40 const auto emit{[&](char ch) { result += ch; }};41 for (auto ch : str) {42 using CharT = std::decay_t<decltype(ch)>;43 char32_t ch32{static_cast<std::make_unsigned_t<CharT>>(ch)};44 if (ch32 == static_cast<unsigned char>('"')) {45 emit('"'); // double the " when it appears in the text46 }47 EmitQuotedChar(ch32, emit, emit, backslashEscapes, encoding);48 }49 result += '"';50 return result;51}52 53std::string QuoteCharacterLiteral(54 const std::string &str, bool backslashEscapes, Encoding encoding) {55 return QuoteCharacterLiteralHelper(str, backslashEscapes, encoding);56}57 58std::string QuoteCharacterLiteral(59 const std::u16string &str, bool backslashEscapes, Encoding encoding) {60 return QuoteCharacterLiteralHelper(str, backslashEscapes, encoding);61}62 63std::string QuoteCharacterLiteral(64 const std::u32string &str, bool backslashEscapes, Encoding encoding) {65 return QuoteCharacterLiteralHelper(str, backslashEscapes, encoding);66}67 68template <> EncodedCharacter EncodeCharacter<Encoding::LATIN_1>(char32_t ucs) {69 CHECK(ucs <= 0xff);70 EncodedCharacter result;71 result.buffer[0] = ucs;72 result.bytes = 1;73 return result;74}75 76template <> EncodedCharacter EncodeCharacter<Encoding::UTF_8>(char32_t ucs) {77 // N.B. char32_t is unsigned78 EncodedCharacter result;79 if (ucs <= 0x7f) {80 result.buffer[0] = ucs;81 result.bytes = 1;82 } else if (ucs <= 0x7ff) {83 result.buffer[0] = 0xc0 | (ucs >> 6);84 result.buffer[1] = 0x80 | (ucs & 0x3f);85 result.bytes = 2;86 } else if (ucs <= 0xffff) {87 result.buffer[0] = 0xe0 | (ucs >> 12);88 result.buffer[1] = 0x80 | ((ucs >> 6) & 0x3f);89 result.buffer[2] = 0x80 | (ucs & 0x3f);90 result.bytes = 3;91 } else if (ucs <= 0x1fffff) {92 // UCS actually only goes up to 0x10ffff, but the93 // UTF-8 encoding can handle 32 bits.94 result.buffer[0] = 0xf0 | (ucs >> 18);95 result.buffer[1] = 0x80 | ((ucs >> 12) & 0x3f);96 result.buffer[2] = 0x80 | ((ucs >> 6) & 0x3f);97 result.buffer[3] = 0x80 | (ucs & 0x3f);98 result.bytes = 4;99 } else if (ucs <= 0x3ffffff) {100 result.buffer[0] = 0xf8 | (ucs >> 24);101 result.buffer[1] = 0x80 | ((ucs >> 18) & 0x3f);102 result.buffer[2] = 0x80 | ((ucs >> 12) & 0x3f);103 result.buffer[3] = 0x80 | ((ucs >> 6) & 0x3f);104 result.buffer[4] = 0x80 | (ucs & 0x3f);105 result.bytes = 5;106 } else {107 result.buffer[0] = 0xfc | (ucs >> 30);108 result.buffer[1] = 0x80 | ((ucs >> 24) & 0x3f);109 result.buffer[2] = 0x80 | ((ucs >> 18) & 0x3f);110 result.buffer[3] = 0x80 | ((ucs >> 12) & 0x3f);111 result.buffer[4] = 0x80 | ((ucs >> 6) & 0x3f);112 result.buffer[5] = 0x80 | (ucs & 0x3f);113 result.bytes = 6;114 }115 return result;116}117 118EncodedCharacter EncodeCharacter(Encoding encoding, char32_t ucs) {119 switch (encoding) {120 SWITCH_COVERS_ALL_CASES121 case Encoding::LATIN_1:122 return EncodeCharacter<Encoding::LATIN_1>(ucs);123 case Encoding::UTF_8:124 return EncodeCharacter<Encoding::UTF_8>(ucs);125 }126}127 128template <Encoding ENCODING, typename STRING>129std::string EncodeString(const STRING &str) {130 std::string result;131 for (auto ch : str) {132 char32_t uch{static_cast<std::make_unsigned_t<decltype(ch)>>(ch)};133 EncodedCharacter encoded{EncodeCharacter<ENCODING>(uch)};134 result.append(encoded.buffer, static_cast<std::size_t>(encoded.bytes));135 }136 return result;137}138 139template std::string EncodeString<Encoding::LATIN_1, std::string>(140 const std::string &);141template std::string EncodeString<Encoding::UTF_8, std::u16string>(142 const std::u16string &);143template std::string EncodeString<Encoding::UTF_8, std::u32string>(144 const std::u32string &);145 146template <>147DecodedCharacter DecodeRawCharacter<Encoding::LATIN_1>(148 const char *cp, std::size_t bytes) {149 if (bytes >= 1) {150 return {*reinterpret_cast<const std::uint8_t *>(cp), 1};151 } else {152 return {};153 }154}155 156template <>157DecodedCharacter DecodeRawCharacter<Encoding::UTF_8>(158 const char *cp, std::size_t bytes) {159 auto p{reinterpret_cast<const std::uint8_t *>(cp)};160 char32_t ch{*p};161 // Valid UTF-8 encodings must be minimal.162 if (ch <= 0x7f) { // 1 byte: 7 bits of payload163 return {ch, 1};164 } else if ((ch & 0xf8) == 0xf0 && bytes >= 4 &&165 ((p[1] | p[2] | p[3]) & 0xc0) == 0x80 && (ch > 0xf0 || p[1] > 0x8f)) {166 // 4 bytes: 3+6+6+6=21 bits of payload167 ch = ((ch & 7) << 6) | (p[1] & 0x3f);168 ch = (ch << 6) | (p[2] & 0x3f);169 ch = (ch << 6) | (p[3] & 0x3f);170 return {ch, 4};171 } else if ((ch & 0xf0) == 0xe0 && bytes >= 3 &&172 ((p[1] | p[2]) & 0xc0) == 0x80 && (ch > 0xe0 || p[1] > 0x9f)) {173 // 3 bytes: 4+6+6=16 bits of payload174 ch = ((ch & 0xf) << 6) | (p[1] & 0x3f);175 ch = (ch << 6) | (p[2] & 0x3f);176 return {ch, 3};177 } else if ((ch & 0xe0) == 0xc0 && bytes >= 2 && ch > 0xc0 &&178 (p[1] & 0xc0) == 0x80) { // 2 bytes: 5+6=11 bits of payload179 ch = ((ch & 0x1f) << 6) | (p[1] & 0x3f);180 return {ch, 2};181 } else {182 return {}; // not valid UTF-8183 }184}185 186static DecodedCharacter DecodeEscapedCharacter(187 const char *cp, std::size_t bytes) {188 if (cp[0] == '\\' && bytes >= 2) {189 if (std::optional<char> escChar{BackslashEscapeValue(cp[1])}) {190 return {static_cast<unsigned char>(*escChar), 2};191 } else if (IsOctalDigit(cp[1])) {192 std::size_t maxLen{std::min(std::size_t{4}, bytes)};193 char32_t code{static_cast<char32_t>(DecimalDigitValue(cp[1]))};194 std::size_t len{2}; // so far195 for (; code <= 037 && len < maxLen && IsOctalDigit(cp[len]); ++len) {196 code = 8 * code + DecimalDigitValue(cp[len]);197 }198 return {code, static_cast<int>(len)};199 } else if (bytes >= 4 && ToLowerCaseLetter(cp[1]) == 'x' &&200 IsHexadecimalDigit(cp[2]) && IsHexadecimalDigit(cp[3])) {201 return {static_cast<char32_t>(16 * HexadecimalDigitValue(cp[2]) +202 HexadecimalDigitValue(cp[3])),203 4};204 } else if (IsLetter(cp[1])) {205 // Unknown escape - ignore the '\' (PGI compatibility)206 return {static_cast<unsigned char>(cp[1]), 2};207 } else {208 // Not an escape character.209 return {'\\', 1};210 }211 }212 return {static_cast<unsigned char>(cp[0]), 1};213}214 215template <Encoding ENCODING>216static DecodedCharacter DecodeEscapedCharacters(217 const char *cp, std::size_t bytes) {218 char buffer[EncodedCharacter::maxEncodingBytes];219 int count[EncodedCharacter::maxEncodingBytes];220 std::size_t at{0}, len{0};221 for (; len < EncodedCharacter::maxEncodingBytes && at < bytes; ++len) {222 DecodedCharacter code{DecodeEscapedCharacter(cp + at, bytes - at)};223 buffer[len] = code.codepoint;224 at += code.bytes;225 count[len] = at;226 }227 DecodedCharacter code{DecodeCharacter<ENCODING>(buffer, len, false)};228 if (code.bytes > 0) {229 code.bytes = count[code.bytes - 1];230 } else {231 code.codepoint = buffer[0] & 0xff;232 code.bytes = count[0];233 }234 return code;235}236 237template <Encoding ENCODING>238DecodedCharacter DecodeCharacter(239 const char *cp, std::size_t bytes, bool backslashEscapes) {240 if (backslashEscapes && bytes >= 2 && *cp == '\\') {241 if (ENCODING == Encoding::UTF_8 && bytes >= 6 &&242 ToLowerCaseLetter(cp[1]) == 'u' && IsHexadecimalDigit(cp[2]) &&243 IsHexadecimalDigit(cp[3]) && IsHexadecimalDigit(cp[4]) &&244 IsHexadecimalDigit(cp[5])) {245 char32_t ch{246 static_cast<char32_t>(4096 * HexadecimalDigitValue(cp[2]) +247 256 * HexadecimalDigitValue(cp[3]) +248 16 * HexadecimalDigitValue(cp[4]) + HexadecimalDigitValue(cp[5])),249 };250 if (bytes >= 10 && IsHexadecimalDigit(cp[6]) &&251 IsHexadecimalDigit(cp[7]) && IsHexadecimalDigit(cp[8]) &&252 IsHexadecimalDigit(cp[9])) {253 return {(ch << 16) |254 (4096 * HexadecimalDigitValue(cp[6]) +255 256 * HexadecimalDigitValue(cp[7]) +256 16 * HexadecimalDigitValue(cp[8]) +257 HexadecimalDigitValue(cp[9])),258 10};259 } else {260 return {ch, 6};261 }262 } else {263 return DecodeEscapedCharacters<ENCODING>(cp, bytes);264 }265 } else {266 return DecodeRawCharacter<ENCODING>(cp, bytes);267 }268}269 270template DecodedCharacter DecodeCharacter<Encoding::LATIN_1>(271 const char *, std::size_t, bool);272template DecodedCharacter DecodeCharacter<Encoding::UTF_8>(273 const char *, std::size_t, bool);274 275DecodedCharacter DecodeCharacter(Encoding encoding, const char *cp,276 std::size_t bytes, bool backslashEscapes) {277 switch (encoding) {278 SWITCH_COVERS_ALL_CASES279 case Encoding::LATIN_1:280 return DecodeCharacter<Encoding::LATIN_1>(cp, bytes, backslashEscapes);281 case Encoding::UTF_8:282 return DecodeCharacter<Encoding::UTF_8>(cp, bytes, backslashEscapes);283 }284}285 286template <typename RESULT, Encoding ENCODING>287RESULT DecodeString(const std::string &s, bool backslashEscapes) {288 RESULT result;289 const char *p{s.c_str()};290 for (auto bytes{s.size()}; bytes != 0;) {291 DecodedCharacter decoded{292 DecodeCharacter<ENCODING>(p, bytes, backslashEscapes)};293 if (decoded.bytes > 0) {294 if (static_cast<std::size_t>(decoded.bytes) <= bytes) {295 result.append(296 1, static_cast<typename RESULT::value_type>(decoded.codepoint));297 bytes -= decoded.bytes;298 p += decoded.bytes;299 continue;300 }301 }302 result.append(1, static_cast<uint8_t>(*p));303 ++p;304 --bytes;305 }306 return result;307}308 309template std::string DecodeString<std::string, Encoding::LATIN_1>(310 const std::string &, bool);311template std::u16string DecodeString<std::u16string, Encoding::UTF_8>(312 const std::string &, bool);313template std::u32string DecodeString<std::u32string, Encoding::UTF_8>(314 const std::string &, bool);315} // namespace Fortran::parser316