424 lines · cpp
1//===-- Unittests for StringConverter class -------------------------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8 9#include "hdr/errno_macros.h"10#include "hdr/types/char32_t.h"11#include "hdr/types/char8_t.h"12#include "src/__support/error_or.h"13#include "src/__support/macros/properties/os.h"14#include "src/__support/wchar/mbstate.h"15#include "src/__support/wchar/string_converter.h"16#include "test/UnitTest/Test.h"17 18// TODO: add support for 16-bit widechars to StringConverter to remove this19// macro20#ifdef LIBC_TARGET_OS_IS_WINDOWS21TEST(LlvmLibcStringConverterTest, Windows) {22 // pass on windows for now23}24 25#else26 27TEST(LlvmLibcStringConverterTest, UTF8To32) {28 // first 4 bytes are clown emoji (🤡)29 // next 3 bytes are sigma symbol (∑)30 // next 2 bytes are y with diaeresis (ÿ)31 // last byte is the letter A32 const char *src = "\xF0\x9F\xA4\xA1\xE2\x88\x91\xC3\xBF\x41";33 LIBC_NAMESPACE::internal::mbstate state;34 LIBC_NAMESPACE::internal::StringConverter<char8_t> sc(35 reinterpret_cast<const char8_t *>(src), &state, SIZE_MAX);36 37 auto res = sc.pop<char32_t>();38 ASSERT_TRUE(res.has_value());39 ASSERT_EQ(static_cast<int>(res.value()), 0x1f921);40 ASSERT_EQ(static_cast<int>(sc.getSourceIndex()), 4);41 42 res = sc.pop<char32_t>();43 ASSERT_TRUE(res.has_value());44 ASSERT_EQ(static_cast<int>(res.value()), 0x2211);45 ASSERT_EQ(static_cast<int>(sc.getSourceIndex()), 7);46 47 res = sc.pop<char32_t>();48 ASSERT_TRUE(res.has_value());49 ASSERT_EQ(static_cast<int>(res.value()), 0xff);50 ASSERT_EQ(static_cast<int>(sc.getSourceIndex()), 9);51 52 res = sc.pop<char32_t>();53 ASSERT_TRUE(res.has_value());54 ASSERT_EQ(static_cast<int>(res.value()), 0x41);55 ASSERT_EQ(static_cast<int>(sc.getSourceIndex()), 10);56 57 res = sc.pop<char32_t>();58 ASSERT_TRUE(res.has_value());59 ASSERT_EQ(static_cast<int>(res.value()), 0);60 ASSERT_EQ(static_cast<int>(sc.getSourceIndex()), 11);61 62 res = sc.pop<char32_t>();63 ASSERT_FALSE(res.has_value());64 ASSERT_EQ(res.error(), -1);65 ASSERT_EQ(static_cast<int>(sc.getSourceIndex()), 11);66}67 68TEST(LlvmLibcStringConverterTest, UTF32To8) {69 // clown emoji, sigma symbol, y with diaeresis, letter A70 const wchar_t src[] = {static_cast<wchar_t>(0x1f921),71 static_cast<wchar_t>(0x2211),72 static_cast<wchar_t>(0xff), static_cast<wchar_t>(0x41),73 static_cast<wchar_t>(0x0)};74 LIBC_NAMESPACE::internal::mbstate state;75 LIBC_NAMESPACE::internal::StringConverter<char32_t> sc(76 reinterpret_cast<const char32_t *>(src), &state, SIZE_MAX);77 78 auto res = sc.pop<char8_t>();79 ASSERT_TRUE(res.has_value());80 ASSERT_EQ(static_cast<int>(res.value()), 0xF0);81 ASSERT_EQ(static_cast<int>(sc.getSourceIndex()), 1);82 83 res = sc.pop<char8_t>();84 ASSERT_TRUE(res.has_value());85 ASSERT_EQ(static_cast<int>(res.value()), 0x9F);86 ASSERT_EQ(static_cast<int>(sc.getSourceIndex()), 1);87 88 res = sc.pop<char8_t>();89 ASSERT_TRUE(res.has_value());90 ASSERT_EQ(static_cast<int>(res.value()), 0xA4);91 ASSERT_EQ(static_cast<int>(sc.getSourceIndex()), 1);92 93 res = sc.pop<char8_t>();94 ASSERT_TRUE(res.has_value());95 ASSERT_EQ(static_cast<int>(res.value()), 0xA1);96 ASSERT_EQ(static_cast<int>(sc.getSourceIndex()), 1);97 98 // end of clown emoji, sigma symbol begins99 res = sc.pop<char8_t>();100 ASSERT_TRUE(res.has_value());101 ASSERT_EQ(static_cast<int>(res.value()), 0xE2);102 ASSERT_EQ(static_cast<int>(sc.getSourceIndex()), 2);103 104 res = sc.pop<char8_t>();105 ASSERT_TRUE(res.has_value());106 ASSERT_EQ(static_cast<int>(res.value()), 0x88);107 ASSERT_EQ(static_cast<int>(sc.getSourceIndex()), 2);108 109 res = sc.pop<char8_t>();110 ASSERT_TRUE(res.has_value());111 ASSERT_EQ(static_cast<int>(res.value()), 0x91);112 ASSERT_EQ(static_cast<int>(sc.getSourceIndex()), 2);113 114 // end of sigma symbol, y with diaeresis begins115 res = sc.pop<char8_t>();116 ASSERT_TRUE(res.has_value());117 ASSERT_EQ(static_cast<int>(res.value()), 0xC3);118 ASSERT_EQ(static_cast<int>(sc.getSourceIndex()), 3);119 120 res = sc.pop<char8_t>();121 ASSERT_TRUE(res.has_value());122 ASSERT_EQ(static_cast<int>(res.value()), 0xBF);123 ASSERT_EQ(static_cast<int>(sc.getSourceIndex()), 3);124 125 // end of y with diaeresis, letter A begins126 res = sc.pop<char8_t>();127 ASSERT_TRUE(res.has_value());128 ASSERT_EQ(static_cast<int>(res.value()), 0x41);129 ASSERT_EQ(static_cast<int>(sc.getSourceIndex()), 4);130 131 // null byte132 res = sc.pop<char8_t>();133 ASSERT_TRUE(res.has_value());134 ASSERT_EQ(static_cast<int>(res.value()), 0);135 ASSERT_EQ(static_cast<int>(sc.getSourceIndex()), 5);136 137 res = sc.pop<char8_t>();138 ASSERT_FALSE(res.has_value());139 ASSERT_EQ(res.error(), -1);140 ASSERT_EQ(static_cast<int>(sc.getSourceIndex()), 5);141}142 143TEST(LlvmLibcStringConverterTest, UTF32To8PartialRead) {144 const wchar_t src[] = {145 static_cast<wchar_t>(0x1f921), static_cast<wchar_t>(0x2211),146 static_cast<wchar_t>(0x0)}; // clown emoji, sigma symbol147 LIBC_NAMESPACE::internal::mbstate state;148 LIBC_NAMESPACE::internal::StringConverter<char32_t> sc(149 reinterpret_cast<const char32_t *>(src), &state, SIZE_MAX, 1);150 151 auto res = sc.pop<char8_t>();152 ASSERT_TRUE(res.has_value());153 ASSERT_EQ(static_cast<int>(res.value()), 0xF0);154 ASSERT_EQ(static_cast<int>(sc.getSourceIndex()), 1);155 156 res = sc.pop<char8_t>();157 ASSERT_TRUE(res.has_value());158 ASSERT_EQ(static_cast<int>(res.value()), 0x9F);159 ASSERT_EQ(static_cast<int>(sc.getSourceIndex()), 1);160 161 res = sc.pop<char8_t>();162 ASSERT_TRUE(res.has_value());163 ASSERT_EQ(static_cast<int>(res.value()), 0xA4);164 ASSERT_EQ(static_cast<int>(sc.getSourceIndex()), 1);165 166 res = sc.pop<char8_t>();167 ASSERT_TRUE(res.has_value());168 ASSERT_EQ(static_cast<int>(res.value()), 0xA1);169 ASSERT_EQ(static_cast<int>(sc.getSourceIndex()), 1);170 171 // can only read 1 character from source string, so error on next pop172 res = sc.pop<char8_t>();173 ASSERT_FALSE(res.has_value());174 ASSERT_EQ(res.error(), -1);175}176 177TEST(LlvmLibcStringConverterTest, UTF8To32PartialRead) {178 // first 4 bytes are clown emoji, then next 3 are sigma symbol179 const char *src = "\xF0\x9F\xA4\xA1\xE2\x88\x91";180 LIBC_NAMESPACE::internal::mbstate state;181 LIBC_NAMESPACE::internal::StringConverter<char8_t> sc(182 reinterpret_cast<const char8_t *>(src), &state, SIZE_MAX, 5);183 184 auto res = sc.pop<char32_t>();185 ASSERT_TRUE(res.has_value());186 ASSERT_EQ(static_cast<int>(res.value()), 0x1f921);187 ASSERT_EQ(static_cast<int>(sc.getSourceIndex()), 4);188 189 res = sc.pop<char32_t>();190 ASSERT_FALSE(res.has_value());191 ASSERT_EQ(static_cast<int>(res.error()), -1);192 ASSERT_EQ(static_cast<int>(sc.getSourceIndex()), 5);193}194 195TEST(LlvmLibcStringConverterTest, UTF32To8ErrorHandling) {196 const wchar_t src[] = {197 static_cast<wchar_t>(0x1f921), static_cast<wchar_t>(0xffffff),198 static_cast<wchar_t>(0x0)}; // clown emoji, invalid utf32199 LIBC_NAMESPACE::internal::mbstate state;200 LIBC_NAMESPACE::internal::StringConverter<char32_t> sc(201 reinterpret_cast<const char32_t *>(src), &state, SIZE_MAX);202 203 auto res = sc.pop<char8_t>();204 ASSERT_TRUE(res.has_value());205 ASSERT_EQ(static_cast<int>(res.value()), 0xF0);206 ASSERT_EQ(static_cast<int>(sc.getSourceIndex()), 1);207 208 res = sc.pop<char8_t>();209 ASSERT_TRUE(res.has_value());210 ASSERT_EQ(static_cast<int>(res.value()), 0x9F);211 ASSERT_EQ(static_cast<int>(sc.getSourceIndex()), 1);212 213 res = sc.pop<char8_t>();214 ASSERT_TRUE(res.has_value());215 ASSERT_EQ(static_cast<int>(res.value()), 0xA4);216 ASSERT_EQ(static_cast<int>(sc.getSourceIndex()), 1);217 218 res = sc.pop<char8_t>();219 ASSERT_TRUE(res.has_value());220 ASSERT_EQ(static_cast<int>(res.value()), 0xA1);221 ASSERT_EQ(static_cast<int>(sc.getSourceIndex()), 1);222 223 res = sc.pop<char8_t>();224 ASSERT_FALSE(res.has_value());225 ASSERT_EQ(static_cast<int>(res.error()), EILSEQ);226 ASSERT_EQ(static_cast<int>(sc.getSourceIndex()), 1);227}228 229TEST(LlvmLibcStringConverterTest, UTF8To32ErrorHandling) {230 // first 4 bytes are clown emoji (🤡)231 // next 3 form an invalid character232 const char *src = "\xF0\x9F\xA4\xA1\x90\x88\x30";233 LIBC_NAMESPACE::internal::mbstate state;234 LIBC_NAMESPACE::internal::StringConverter<char8_t> sc(235 reinterpret_cast<const char8_t *>(src), &state, SIZE_MAX);236 237 auto res = sc.pop<char32_t>();238 ASSERT_TRUE(res.has_value());239 ASSERT_EQ(static_cast<int>(res.value()), 0x1f921);240 ASSERT_EQ(static_cast<int>(sc.getSourceIndex()), 4);241 242 res = sc.pop<char32_t>();243 ASSERT_FALSE(res.has_value());244 ASSERT_EQ(static_cast<int>(res.error()), EILSEQ);245 ASSERT_EQ(static_cast<int>(sc.getSourceIndex()), 4);246}247 248TEST(LlvmLibcStringConverterTest, InvalidCharacterOutsideBounds) {249 // if an invalid character exists in the source string but we don't have space250 // to write it, we should return a "stop converting" error rather than an251 // invalid character error252 253 // first 4 bytes are clown emoji (🤡)254 // next 3 form an invalid character255 const char *src1 = "\xF0\x9F\xA4\xA1\x90\x88\x30";256 LIBC_NAMESPACE::internal::mbstate ps1;257 LIBC_NAMESPACE::internal::StringConverter<char8_t> sc1(258 reinterpret_cast<const char8_t *>(src1), &ps1, 1);259 260 auto res1 = sc1.pop<char32_t>();261 ASSERT_TRUE(res1.has_value());262 ASSERT_EQ(static_cast<int>(res1.value()), 0x1f921);263 ASSERT_EQ(static_cast<int>(sc1.getSourceIndex()), 4);264 265 res1 = sc1.pop<char32_t>();266 ASSERT_FALSE(res1.has_value());267 // no space to write error NOT invalid character error (EILSEQ)268 ASSERT_EQ(static_cast<int>(res1.error()), -1);269 ASSERT_EQ(static_cast<int>(sc1.getSourceIndex()), 4);270 271 const wchar_t src2[] = {272 static_cast<wchar_t>(0x1f921), static_cast<wchar_t>(0xffffff),273 static_cast<wchar_t>(0x0)}; // clown emoji, invalid utf32274 LIBC_NAMESPACE::internal::mbstate ps2;275 LIBC_NAMESPACE::internal::StringConverter<char32_t> sc2(276 reinterpret_cast<const char32_t *>(src2), &ps2, 4);277 278 auto res2 = sc2.pop<char8_t>();279 ASSERT_TRUE(res2.has_value());280 ASSERT_EQ(static_cast<int>(res2.value()), 0xF0);281 ASSERT_EQ(static_cast<int>(sc2.getSourceIndex()), 1);282 283 res2 = sc2.pop<char8_t>();284 ASSERT_TRUE(res2.has_value());285 ASSERT_EQ(static_cast<int>(res2.value()), 0x9F);286 ASSERT_EQ(static_cast<int>(sc2.getSourceIndex()), 1);287 288 res2 = sc2.pop<char8_t>();289 ASSERT_TRUE(res2.has_value());290 ASSERT_EQ(static_cast<int>(res2.value()), 0xA4);291 ASSERT_EQ(static_cast<int>(sc2.getSourceIndex()), 1);292 293 res2 = sc2.pop<char8_t>();294 ASSERT_TRUE(res2.has_value());295 ASSERT_EQ(static_cast<int>(res2.value()), 0xA1);296 ASSERT_EQ(static_cast<int>(sc2.getSourceIndex()), 1);297 298 res2 = sc2.pop<char8_t>();299 ASSERT_FALSE(res2.has_value());300 // no space to write error NOT invalid character error (EILSEQ)301 ASSERT_EQ(static_cast<int>(res2.error()), -1);302 ASSERT_EQ(static_cast<int>(sc2.getSourceIndex()), 1);303}304 305TEST(LlvmLibcStringConverterTest, MultipleStringConverters32To8) {306 /*307 We do NOT test partially popping a character and expecting the next308 StringConverter to continue where we left off. This is not expected to work309 and considered invalid.310 */311 const wchar_t src[] = {312 static_cast<wchar_t>(0x1f921), static_cast<wchar_t>(0xff),313 static_cast<wchar_t>(0x0)}; // clown emoji, y with diaeresis (ÿ)314 LIBC_NAMESPACE::internal::mbstate state;315 LIBC_NAMESPACE::internal::StringConverter<char32_t> sc1(316 reinterpret_cast<const char32_t *>(src), &state, SIZE_MAX, 1);317 318 auto res = sc1.pop<char8_t>();319 ASSERT_TRUE(res.has_value());320 ASSERT_EQ(static_cast<int>(res.value()), 0xF0);321 ASSERT_EQ(static_cast<int>(sc1.getSourceIndex()), 1);322 323 res = sc1.pop<char8_t>();324 ASSERT_TRUE(res.has_value());325 ASSERT_EQ(static_cast<int>(res.value()), 0x9F);326 ASSERT_EQ(static_cast<int>(sc1.getSourceIndex()), 1);327 328 res = sc1.pop<char8_t>();329 ASSERT_TRUE(res.has_value());330 ASSERT_EQ(static_cast<int>(res.value()), 0xA4);331 ASSERT_EQ(static_cast<int>(sc1.getSourceIndex()), 1);332 333 res = sc1.pop<char8_t>();334 ASSERT_TRUE(res.has_value());335 ASSERT_EQ(static_cast<int>(res.value()), 0xA1);336 ASSERT_EQ(static_cast<int>(sc1.getSourceIndex()), 1);337 338 // sc2 should pick up where sc1 left off and continue the conversion339 LIBC_NAMESPACE::internal::StringConverter<char32_t> sc2(340 reinterpret_cast<const char32_t *>(src) + sc1.getSourceIndex(), &state,341 SIZE_MAX, 1);342 343 res = sc2.pop<char8_t>();344 ASSERT_TRUE(res.has_value());345 ASSERT_EQ(static_cast<int>(res.value()), 0xC3);346 ASSERT_EQ(static_cast<int>(sc2.getSourceIndex()), 1);347 348 res = sc2.pop<char8_t>();349 ASSERT_TRUE(res.has_value());350 ASSERT_EQ(static_cast<int>(res.value()), 0xBF);351 ASSERT_EQ(static_cast<int>(sc2.getSourceIndex()), 1);352}353 354TEST(LlvmLibcStringConverterTest, MultipleStringConverters8To32) {355 const char *src = "\xF0\x9F\xA4\xA1"; // clown emoji356 LIBC_NAMESPACE::internal::mbstate state;357 LIBC_NAMESPACE::internal::StringConverter<char8_t> sc1(358 reinterpret_cast<const char8_t *>(src), &state, SIZE_MAX, 2);359 360 auto res = sc1.pop<char32_t>();361 ASSERT_FALSE(res.has_value());362 ASSERT_EQ(static_cast<int>(res.error()), -1);363 ASSERT_EQ(static_cast<int>(sc1.getSourceIndex()), 2);364 365 // sc2 should pick up where sc1 left off and continue the conversion366 LIBC_NAMESPACE::internal::StringConverter<char8_t> sc2(367 reinterpret_cast<const char8_t *>(src) + sc1.getSourceIndex(), &state,368 SIZE_MAX, 3);369 370 res = sc2.pop<char32_t>();371 ASSERT_TRUE(res.has_value());372 ASSERT_EQ(static_cast<int>(res.value()), 0x1f921);373 ASSERT_EQ(static_cast<int>(sc2.getSourceIndex()), 2);374 375 res = sc2.pop<char32_t>();376 ASSERT_TRUE(res.has_value());377 ASSERT_EQ(static_cast<int>(res.value()), 0);378 ASSERT_EQ(static_cast<int>(sc2.getSourceIndex()), 3);379}380 381TEST(LlvmLibcStringConverterTest, DestLimitUTF8To32) {382 const char *src = "\xF0\x9F\xA4\xA1\xF0\x9F\xA4\xA1"; // 2 clown emojis383 LIBC_NAMESPACE::internal::mbstate state;384 LIBC_NAMESPACE::internal::StringConverter<char8_t> sc(385 reinterpret_cast<const char8_t *>(src), &state, 1);386 387 auto res = sc.pop<char32_t>();388 ASSERT_TRUE(res.has_value());389 ASSERT_EQ(static_cast<int>(sc.getSourceIndex()), 4);390 391 res = sc.pop<char32_t>(); // no space to pop this into392 ASSERT_FALSE(res.has_value());393}394 395TEST(LlvmLibcStringConverterTest, DestLimitUTF32To8) {396 const wchar_t src[] = {static_cast<wchar_t>(0x1f921),397 static_cast<wchar_t>(0x1f921)}; // 2 clown emojis398 LIBC_NAMESPACE::internal::mbstate state;399 LIBC_NAMESPACE::internal::StringConverter<char32_t> sc(400 reinterpret_cast<const char32_t *>(src), &state, 5);401 402 auto res = sc.pop<char8_t>();403 ASSERT_TRUE(res.has_value());404 ASSERT_EQ(static_cast<int>(sc.getSourceIndex()), 1);405 406 res = sc.pop<char8_t>();407 ASSERT_TRUE(res.has_value());408 ASSERT_EQ(static_cast<int>(sc.getSourceIndex()), 1);409 410 res = sc.pop<char8_t>();411 ASSERT_TRUE(res.has_value());412 ASSERT_EQ(static_cast<int>(sc.getSourceIndex()), 1);413 414 res = sc.pop<char8_t>();415 ASSERT_TRUE(res.has_value());416 ASSERT_EQ(static_cast<int>(sc.getSourceIndex()), 1);417 418 res = sc.pop<char8_t>();419 ASSERT_FALSE(res.has_value());420 ASSERT_EQ(static_cast<int>(sc.getSourceIndex()), 1);421}422 423#endif424