355 lines · cpp
1//===-- TextEncoding.cpp - Text encoding conversion class ---------*- C++ -*-=//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8///9/// \file10/// This file provides utility classes to convert between different character11/// encodings.12///13//===----------------------------------------------------------------------===//14 15#include "llvm/Support/TextEncoding.h"16#include "llvm/ADT/SmallString.h"17#include "llvm/ADT/SmallVector.h"18#include "llvm/ADT/StringExtras.h"19#include "llvm/Support/ConvertEBCDIC.h"20#include <system_error>21 22#if HAVE_ICU23#include <unicode/ucnv.h>24#elif HAVE_ICONV25#include <iconv.h>26#endif27 28using namespace llvm;29 30// Normalize the charset name with the charset alias matching algorithm proposed31// in https://www.unicode.org/reports/tr22/tr22-8.html#Charset_Alias_Matching.32static void normalizeCharSetName(StringRef CSName,33 SmallVectorImpl<char> &Normalized) {34 bool PrevDigit = false;35 for (auto Ch : CSName) {36 if (isAlnum(Ch)) {37 Ch = toLower(Ch);38 if (Ch != '0' || PrevDigit) {39 PrevDigit = isDigit(Ch);40 Normalized.push_back(Ch);41 }42 }43 }44}45 46// Maps the encoding name to enum constant if possible.47static std::optional<TextEncoding> getKnownEncoding(StringRef Name) {48 SmallString<16> Normalized;49 normalizeCharSetName(Name, Normalized);50 if (Normalized.equals("utf8"))51 return TextEncoding::UTF8;52 if (Normalized.equals("ibm1047"))53 return TextEncoding::IBM1047;54 return std::nullopt;55}56 57[[maybe_unused]] static void HandleOverflow(size_t &Capacity, char *&Output,58 size_t &OutputLength,59 SmallVectorImpl<char> &Result) {60 // No space left in output buffer. Double the size of the underlying61 // memory in the SmallVectorImpl, adjust pointer and length and continue62 // the conversion.63 Capacity =64 (Capacity < Result.max_size() / 2) ? 2 * Capacity : Result.max_size();65 Result.resize(0);66 Result.resize_for_overwrite(Capacity);67 Output = static_cast<char *>(Result.data());68 OutputLength = Capacity;69}70 71namespace {72enum ConversionType {73 UTF8ToIBM1047,74 IBM1047ToUTF8,75};76 77// Support conversion between EBCDIC 1047 and UTF-8. This class uses78// built-in translation tables that allow for translation between the79// aforementioned encodings. The use of tables for conversion is only80// possible because EBCDIC 1047 is a single-byte, stateless encoding; other81// encodings are not supported.82class TextEncodingConverterTable final83 : public details::TextEncodingConverterImplBase {84 const ConversionType ConvType;85 86public:87 TextEncodingConverterTable(ConversionType ConvType) : ConvType(ConvType) {}88 89 std::error_code convertString(StringRef Source,90 SmallVectorImpl<char> &Result) override;91 92 void reset() override {}93};94 95std::error_code96TextEncodingConverterTable::convertString(StringRef Source,97 SmallVectorImpl<char> &Result) {98 switch (ConvType) {99 case IBM1047ToUTF8:100 ConverterEBCDIC::convertToUTF8(Source, Result);101 return std::error_code();102 case UTF8ToIBM1047:103 return ConverterEBCDIC::convertToEBCDIC(Source, Result);104 }105 llvm_unreachable("Invalid ConvType!");106 return std::error_code();107}108 109#if HAVE_ICU110struct UConverterDeleter {111 void operator()(UConverter *Converter) const {112 if (Converter)113 ucnv_close(Converter);114 }115};116using UConverterUniquePtr = std::unique_ptr<UConverter, UConverterDeleter>;117 118class TextEncodingConverterICU final119 : public details::TextEncodingConverterImplBase {120 UConverterUniquePtr FromConvDesc;121 UConverterUniquePtr ToConvDesc;122 123public:124 TextEncodingConverterICU(UConverterUniquePtr FromConverter,125 UConverterUniquePtr ToConverter)126 : FromConvDesc(std::move(FromConverter)),127 ToConvDesc(std::move(ToConverter)) {}128 129 std::error_code convertString(StringRef Source,130 SmallVectorImpl<char> &Result) override;131 132 void reset() override;133};134 135// TODO: The current implementation discards the partial result and restarts the136// conversion from the beginning if there is a conversion error due to137// insufficient buffer size. In the future, it would better to save the partial138// result and resume the conversion for the remaining string.139// TODO: Improve translation of ICU errors to error_code140std::error_code141TextEncodingConverterICU::convertString(StringRef Source,142 SmallVectorImpl<char> &Result) {143 // Setup the input in case it has no backing data.144 size_t InputLength = Source.size();145 const char *In = InputLength ? const_cast<char *>(Source.data()) : "";146 147 // Setup the output. We directly write into the SmallVector.148 size_t Capacity = Result.capacity();149 size_t OutputLength = Capacity;150 Result.resize_for_overwrite(Capacity);151 char *Output;152 UErrorCode EC = U_ZERO_ERROR;153 154 ucnv_setToUCallBack(&*FromConvDesc, UCNV_TO_U_CALLBACK_STOP, NULL, NULL, NULL,155 &EC);156 ucnv_setFromUCallBack(&*ToConvDesc, UCNV_FROM_U_CALLBACK_STOP, NULL, NULL,157 NULL, &EC);158 assert(U_SUCCESS(EC));159 160 do {161 EC = U_ZERO_ERROR;162 const char *Input = In;163 164 Output = static_cast<char *>(Result.data());165 ucnv_convertEx(&*ToConvDesc, &*FromConvDesc, &Output, Result.end(), &Input,166 In + InputLength, /*pivotStart=*/NULL,167 /*pivotSource=*/NULL, /*pivotTarget=*/NULL,168 /*pivotLimit=*/NULL, /*reset=*/true,169 /*flush=*/true, &EC);170 if (U_FAILURE(EC)) {171 if (EC == U_BUFFER_OVERFLOW_ERROR) {172 if (Capacity < Result.max_size()) {173 HandleOverflow(Capacity, Output, OutputLength, Result);174 continue;175 } else {176 Result.resize(Output - Result.data());177 return std::error_code(E2BIG, std::generic_category());178 }179 }180 // Some other error occured.181 Result.resize(Output - Result.data());182 return std::error_code(EILSEQ, std::generic_category());183 }184 break;185 } while (true);186 187 Result.resize(Output - Result.data());188 return std::error_code();189}190 191void TextEncodingConverterICU::reset() {192 ucnv_reset(&*FromConvDesc);193 ucnv_reset(&*ToConvDesc);194}195 196#elif HAVE_ICONV197class TextEncodingConverterIconv final198 : public details::TextEncodingConverterImplBase {199 class UniqueIconvT {200 iconv_t ConvDesc;201 202 public:203 operator iconv_t() const { return ConvDesc; }204 UniqueIconvT(iconv_t CD) : ConvDesc(CD) {}205 ~UniqueIconvT() {206 if (ConvDesc != (iconv_t)-1) {207 iconv_close(ConvDesc);208 ConvDesc = (iconv_t)-1;209 }210 }211 UniqueIconvT(UniqueIconvT &&Other) : ConvDesc(Other.ConvDesc) {212 Other.ConvDesc = (iconv_t)-1;213 }214 UniqueIconvT &operator=(UniqueIconvT &&Other) {215 if (&Other != this) {216 ConvDesc = Other.ConvDesc;217 Other.ConvDesc = (iconv_t)-1;218 }219 return *this;220 }221 };222 UniqueIconvT ConvDesc;223 224public:225 TextEncodingConverterIconv(UniqueIconvT ConvDesc)226 : ConvDesc(std::move(ConvDesc)) {}227 228 std::error_code convertString(StringRef Source,229 SmallVectorImpl<char> &Result) override;230 231 void reset() override;232};233 234// TODO: The current implementation discards the partial result and restarts the235// conversion from the beginning if there is a conversion error due to236// insufficient buffer size. In the future, it would better to save the partial237// result and resume the conversion for the remaining string.238std::error_code239TextEncodingConverterIconv::convertString(StringRef Source,240 SmallVectorImpl<char> &Result) {241 // Setup the output. We directly write into the SmallVector.242 size_t Capacity = Result.capacity();243 char *Output = static_cast<char *>(Result.data());244 size_t OutputLength = Capacity;245 Result.resize_for_overwrite(Capacity);246 247 size_t Ret;248 // Handle errors returned from iconv().249 auto HandleError = [&Capacity, &Output, &OutputLength, &Result,250 this](size_t Ret) {251 if (Ret == static_cast<size_t>(-1)) {252 // An error occured. Check if we can gracefully handle it.253 if (errno == E2BIG && Capacity < Result.max_size()) {254 HandleOverflow(Capacity, Output, OutputLength, Result);255 // Reset converter256 reset();257 return std::error_code();258 } else {259 // Some other error occured.260 Result.resize(Output - Result.data());261 return std::error_code(errno, std::generic_category());262 }263 } else {264 // A positive return value indicates that some characters were converted265 // in a nonreversible way, that is, replaced with a SUB symbol. Returning266 // an error in this case makes sure that both conversion routines behave267 // in the same way.268 return std::make_error_code(std::errc::illegal_byte_sequence);269 }270 };271 272 do {273 size_t InputLength = Source.size();274 char *Input = const_cast<char *>(Source.data());275 Ret = iconv(ConvDesc, &Input, &InputLength, &Output, &OutputLength);276 if (Ret != 0) {277 if (auto EC = HandleError(Ret))278 return EC;279 continue;280 }281 // Flush the converter282 Ret = iconv(ConvDesc, nullptr, nullptr, &Output, &OutputLength);283 if (Ret != 0) {284 if (auto EC = HandleError(Ret))285 return EC;286 continue;287 }288 break;289 } while (true);290 291 // Re-adjust size to actual size.292 Result.resize(Output - Result.data());293 return std::error_code();294}295 296inline void TextEncodingConverterIconv::reset() {297 iconv(ConvDesc, nullptr, nullptr, nullptr, nullptr);298}299 300#endif // HAVE_ICONV301} // namespace302 303ErrorOr<TextEncodingConverter>304TextEncodingConverter::create(TextEncoding CPFrom, TextEncoding CPTo) {305 306 // Text encodings should be distinct.307 if (CPFrom == CPTo)308 return std::make_error_code(std::errc::invalid_argument);309 310 ConversionType Conversion;311 if (CPFrom == TextEncoding::UTF8 && CPTo == TextEncoding::IBM1047)312 Conversion = UTF8ToIBM1047;313 else if (CPFrom == TextEncoding::IBM1047 && CPTo == TextEncoding::UTF8)314 Conversion = IBM1047ToUTF8;315 else316 return std::make_error_code(std::errc::invalid_argument);317 318 return TextEncodingConverter(319 std::make_unique<TextEncodingConverterTable>(Conversion));320}321 322ErrorOr<TextEncodingConverter> TextEncodingConverter::create(StringRef From,323 StringRef To) {324 std::optional<TextEncoding> FromEncoding = getKnownEncoding(From);325 std::optional<TextEncoding> ToEncoding = getKnownEncoding(To);326 if (FromEncoding && ToEncoding) {327 ErrorOr<TextEncodingConverter> Converter =328 create(*FromEncoding, *ToEncoding);329 if (Converter)330 return Converter;331 }332#if HAVE_ICU333 UErrorCode EC = U_ZERO_ERROR;334 UConverterUniquePtr FromConvDesc(ucnv_open(From.str().c_str(), &EC));335 if (U_FAILURE(EC))336 return std::make_error_code(std::errc::invalid_argument);337 338 UConverterUniquePtr ToConvDesc(ucnv_open(To.str().c_str(), &EC));339 if (U_FAILURE(EC))340 return std::make_error_code(std::errc::invalid_argument);341 342 auto Converter = std::make_unique<TextEncodingConverterICU>(343 std::move(FromConvDesc), std::move(ToConvDesc));344 return TextEncodingConverter(std::move(Converter));345#elif HAVE_ICONV346 iconv_t ConvDesc = iconv_open(To.str().c_str(), From.str().c_str());347 if (ConvDesc == (iconv_t)-1)348 return std::make_error_code(std::errc::invalid_argument);349 return TextEncodingConverter(350 std::make_unique<TextEncodingConverterIconv>(ConvDesc));351#else352 return std::make_error_code(std::errc::invalid_argument);353#endif354}355