300 lines · cpp
1//===----------------------------------------------------------------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8 9#include "llvm/Support/TextEncoding.h"10#include "llvm/ADT/SmallString.h"11#include "llvm/Config/config.h"12#include "gtest/gtest.h"13 14using namespace llvm;15 16namespace {17 18// String "Hello World!"19static const char HelloA[] =20 "\x48\x65\x6C\x6C\x6F\x20\x57\x6F\x72\x6C\x64\x21\x0a";21static const char HelloE[] =22 "\xC8\x85\x93\x93\x96\x40\xE6\x96\x99\x93\x84\x5A\x15";23 24// String "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"25static const char ABCStrA[] =26 "\x41\x42\x43\x44\x45\x46\x47\x48\x49\x4A\x4B\x4C\x4D\x4E\x4F\x50\x51\x52"27 "\x53\x54\x55\x56\x57\x58\x59\x5A\x61\x62\x63\x64\x65\x66\x67\x68\x69\x6A"28 "\x6B\x6C\x6D\x6E\x6F\x70\x71\x72\x73\x74\x75\x76\x77\x78\x79\x7A";29static const char ABCStrE[] =30 "\xC1\xC2\xC3\xC4\xC5\xC6\xC7\xC8\xC9\xD1\xD2\xD3\xD4\xD5\xD6\xD7\xD8\xD9"31 "\xE2\xE3\xE4\xE5\xE6\xE7\xE8\xE9\x81\x82\x83\x84\x85\x86\x87\x88\x89\x91"32 "\x92\x93\x94\x95\x96\x97\x98\x99\xA2\xA3\xA4\xA5\xA6\xA7\xA8\xA9";33 34// String "¡¢£AÄÅÆEÈÉÊaàáâãäeèéêë"35static const char AccentUTF[] =36 "\xc2\xa1\xc2\xa2\xc2\xa3\x41\xc3\x84\xc3\x85\xc3\x86\x45\xc3\x88\xc3\x89"37 "\xc3\x8a\x61\xc3\xa0\xc3\xa1\xc3\xa2\xc3\xa3\xc3\xa4\x65\xc3\xa8\xc3\xa9"38 "\xc3\xaa\xc3\xab";39static const char AccentE[] = "\xaa\x4a\xb1\xc1\x63\x67\x9e\xc5\x74\x71\x72"40 "\x81\x44\x45\x42\x46\x43\x85\x54\x51\x52\x53";41 42// String with Cyrillic character ya.43static const char CyrillicUTF[] = "\xd0\xaf";44 45// String "Earth地球".46// ISO-2022-JP: Sequence ESC $ B (\x1B\x24\x42) switches to JIS X 0208-1983, and47// sequence ESC ( B (\x1B\x28\x42) switches back to ASCII.48// IBM-939: Byte 0x0E shifts from single byte to double byte, and 0x0F shifts49// back.50static const char EarthUTF[] = "\x45\x61\x72\x74\x68\xe5\x9c\xb0\xe7\x90\x83";51static const char EarthISO2022[] =52 "\x45\x61\x72\x74\x68\x1B\x24\x42\x43\x4F\x35\x65\x1B\x28\x42";53static const char EarthIBM939[] =54 "\xc5\x81\x99\xa3\x88\x0e\x45\xc2\x48\xdb\x0f";55static const char EarthUTFExtraPartial[] =56 "\x45\x61\x72\x74\x68\xe5\x9c\xb0\xe7\x90\x83\xe5";57 58TEST(Encoding, FromUTF8) {59 // Hello string.60 StringRef Src(HelloA);61 SmallString<64> Dst;62 63 ErrorOr<TextEncodingConverter> Conv =64 TextEncodingConverter::create(TextEncoding::UTF8, TextEncoding::IBM1047);65 66 // Converter should always exist between UTF-8 and IBM-104767 EXPECT_TRUE(Conv);68 69 std::error_code EC = Conv->convert(Src, Dst);70 EXPECT_TRUE(!EC);71 EXPECT_STREQ(HelloE, static_cast<std::string>(Dst).c_str());72 Dst.clear();73 74 // ABC string.75 Src = ABCStrA;76 EC = Conv->convert(Src, Dst);77 EXPECT_TRUE(!EC);78 EXPECT_STREQ(ABCStrE, static_cast<std::string>(Dst).c_str());79 Dst.clear();80 81 // Accent string.82 Src = AccentUTF;83 EC = Conv->convert(Src, Dst);84 EXPECT_TRUE(!EC);85 EXPECT_STREQ(AccentE, static_cast<std::string>(Dst).c_str());86 Dst.clear();87 88 // Cyrillic string. Results in error because not representable in 1047.89 Src = CyrillicUTF;90 EC = Conv->convert(Src, Dst);91 EXPECT_EQ(EC, std::errc::illegal_byte_sequence);92}93 94TEST(Encoding, ToUTF8) {95 // Hello string.96 StringRef Src(HelloE);97 SmallString<64> Dst;98 99 ErrorOr<TextEncodingConverter> Conv =100 TextEncodingConverter::create(TextEncoding::IBM1047, TextEncoding::UTF8);101 102 // Converter should always exist between UTF-8 and IBM-1047103 EXPECT_TRUE(Conv);104 105 std::error_code EC = Conv->convert(Src, Dst);106 107 EXPECT_TRUE(!EC);108 EXPECT_STREQ(HelloA, static_cast<std::string>(Dst).c_str());109 Dst.clear();110 111 // ABC string.112 Src = ABCStrE;113 EC = Conv->convert(Src, Dst);114 EXPECT_TRUE(!EC);115 EXPECT_STREQ(ABCStrA, static_cast<std::string>(Dst).c_str());116 Dst.clear();117 118 // Accent string.119 Src = AccentE;120 EC = Conv->convert(Src, Dst);121 EXPECT_TRUE(!EC);122 EXPECT_STREQ(AccentUTF, static_cast<std::string>(Dst).c_str());123}124 125TEST(Encoding, RoundTrip) {126 ErrorOr<TextEncodingConverter> ConvToUTF16 =127 TextEncodingConverter::create("IBM-1047", "UTF-16");128 129#if HAVE_ICU130 EXPECT_TRUE(ConvToUTF16);131#else132 // Stop test if conversion is not supported (no underlying iconv support).133 if (!ConvToUTF16) {134 ASSERT_EQ(ConvToUTF16.getError(),135 std::make_error_code(std::errc::invalid_argument));136 return;137 }138#endif139 140 ErrorOr<TextEncodingConverter> ConvToUTF32 =141 TextEncodingConverter::create("UTF-16", "UTF-32");142 143#if HAVE_ICU144 EXPECT_TRUE(ConvToUTF32);145#else146 // Stop test if conversion is not supported (no underlying iconv support).147 if (!ConvToUTF32) {148 ASSERT_EQ(ConvToUTF32.getError(),149 std::make_error_code(std::errc::invalid_argument));150 return;151 }152#endif153 154 ErrorOr<TextEncodingConverter> ConvToEBCDIC =155 TextEncodingConverter::create("UTF-32", "IBM-1047");156 157#if HAVE_ICU158 EXPECT_TRUE(ConvToEBCDIC);159#else160 // Stop test if conversion is not supported (no underlying iconv support).161 if (!ConvToEBCDIC) {162 ASSERT_EQ(ConvToEBCDIC.getError(),163 std::make_error_code(std::errc::invalid_argument));164 return;165 }166#endif167 168 // Setup source string.169 char SrcStr[256];170 for (size_t I = 0; I < 256; ++I)171 SrcStr[I] = (I + 1) % 256;172 173 SmallString<99> Dst1Str, Dst2Str, Dst3Str;174 175 std::error_code EC = ConvToUTF16->convert(StringRef(SrcStr), Dst1Str);176 EXPECT_TRUE(!EC);177 EC = ConvToUTF32->convert(Dst1Str, Dst2Str);178 EXPECT_TRUE(!EC);179 EC = ConvToEBCDIC->convert(Dst2Str, Dst3Str);180 EXPECT_TRUE(!EC);181 EXPECT_STREQ(SrcStr, static_cast<std::string>(Dst3Str).c_str());182}183 184TEST(Encoding, ShiftState2022) {185 // Earth string.186 StringRef Src(EarthUTF);187 SmallString<8> Dst;188 189 ErrorOr<TextEncodingConverter> ConvTo2022 =190 TextEncodingConverter::create("UTF-8", "ISO-2022-JP");191 192#if HAVE_ICU193 EXPECT_TRUE(ConvTo2022);194#else195 // Stop test if conversion is not supported (no underlying iconv support).196 if (!ConvTo2022) {197 ASSERT_EQ(ConvTo2022.getError(),198 std::make_error_code(std::errc::invalid_argument));199 return;200 }201#endif202 203 // Check that the string is properly converted.204 std::error_code EC = ConvTo2022->convert(Src, Dst);205 EXPECT_TRUE(!EC);206 EXPECT_STREQ(EarthISO2022, static_cast<std::string>(Dst).c_str());207}208 209TEST(Encoding, InvalidInput) {210 // Earth string.211 StringRef Src(EarthUTFExtraPartial);212 SmallString<8> Dst;213 214 ErrorOr<TextEncodingConverter> ConvTo2022 =215 TextEncodingConverter::create("UTF-8", "ISO-2022-JP");216 217#if HAVE_ICU218 EXPECT_TRUE(ConvTo2022);219#else220 // Stop test if conversion is not supported (no underlying iconv support).221 if (!ConvTo2022) {222 ASSERT_EQ(ConvTo2022.getError(),223 std::make_error_code(std::errc::invalid_argument));224 return;225 }226#endif227 228 // Check that the string failed to convert.229 std::error_code EC = ConvTo2022->convert(Src, Dst);230 EXPECT_TRUE(EC);231}232 233TEST(Encoding, InvalidOutput) {234 // Cyrillic in UTF-16235 ErrorOr<TextEncodingConverter> ConvToUTF16 =236 TextEncodingConverter::create("UTF-8", "UTF-16");237 238#if HAVE_ICU239 EXPECT_TRUE(ConvToUTF16);240#else241 // Stop test if conversion is not supported (no underlying iconv support).242 if (!ConvToUTF16) {243 ASSERT_EQ(ConvToUTF16.getError(),244 std::make_error_code(std::errc::invalid_argument));245 return;246 }247#endif248 249 ErrorOr<TextEncodingConverter> ConvToEBCDIC =250 TextEncodingConverter::create("UTF-16", "IBM-1047");251 252#if HAVE_ICU253 EXPECT_TRUE(ConvToEBCDIC);254#else255 // Stop test if conversion is not supported (no underlying iconv support).256 if (!ConvToEBCDIC) {257 ASSERT_EQ(ConvToEBCDIC.getError(),258 std::make_error_code(std::errc::invalid_argument));259 return;260 }261#endif262 263 // Cyrillic string. Convert to UTF-16 and check if properly converted264 StringRef Src(CyrillicUTF);265 SmallString<8> Dst, Dst1;266 std::error_code EC = ConvToUTF16->convert(Src, Dst);267 EXPECT_TRUE(!EC);268 269 // Cyrillic string. Results in error because not representable in 1047.270 EC = ConvToEBCDIC->convert(Dst, Dst1);271 EXPECT_TRUE(EC);272}273 274TEST(Encoding, ShiftStateIBM939) {275 // Earth string.276 StringRef Src(EarthUTF);277 SmallString<64> Dst;278 279 ErrorOr<TextEncodingConverter> ConvToIBM939 =280 TextEncodingConverter::create("UTF-8", "IBM-939");281 282#if HAVE_ICU283 EXPECT_TRUE(ConvToIBM939);284#else285 // Stop test if conversion is not supported (no underlying iconv support).286 if (!ConvToIBM939) {287 ASSERT_EQ(ConvToIBM939.getError(),288 std::make_error_code(std::errc::invalid_argument));289 return;290 }291#endif292 293 // Check that the string is properly converted.294 std::error_code EC = ConvToIBM939->convert(Src, Dst);295 EXPECT_TRUE(!EC);296 EXPECT_STREQ(EarthIBM939, static_cast<std::string>(Dst).c_str());297}298 299} // namespace300