2233 lines · cpp
1//===----------------------------------------------------------------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8 9// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_DISABLE_DEPRECATION_WARNINGS -D_LIBCPP_ENABLE_CXX26_REMOVED_CODECVT10 11// Requires the fix in 390840f.12// XFAIL: using-built-library-before-llvm-1813 14#include <algorithm>15#include <cassert>16#include <codecvt>17#include <locale>18 19#include "test_macros.h"20 21struct test_offsets_ok {22 size_t in_size;23 size_t out_size;24};25struct test_offsets_partial {26 size_t in_size;27 size_t out_size;28 size_t expected_in_next;29 size_t expected_out_next;30};31 32template <class CharT>33struct test_offsets_error {34 size_t in_size;35 size_t out_size;36 size_t expected_in_next;37 size_t expected_out_next;38 CharT replace_char;39 size_t replace_pos;40};41 42#define array_size(x) (sizeof(x) / sizeof(x)[0])43 44using std::begin;45using std::char_traits;46using std::codecvt_base;47using std::copy;48using std::end;49 50template <class InternT, class ExternT>51void utf8_to_utf32_in_ok(const std::codecvt<InternT, ExternT, mbstate_t>& cvt) {52 // UTF-8 string of 1-byte CP, 2-byte CP, 3-byte CP and 4-byte CP53 const unsigned char input[] = "b\u0448\uAAAA\U0010AAAA";54 const char32_t expected[] = {'b', 0x0448, 0xAAAA, 0x10AAAA, 0};55 static_assert(array_size(input) == 11, "");56 static_assert(array_size(expected) == 5, "");57 58 ExternT in[array_size(input)];59 InternT exp[array_size(expected)];60 copy(begin(input), end(input), begin(in));61 copy(begin(expected), end(expected), begin(exp));62 assert(char_traits<ExternT>::length(in) == 10);63 assert(char_traits<InternT>::length(exp) == 4);64 test_offsets_ok offsets[] = {{0, 0}, {1, 1}, {3, 2}, {6, 3}, {10, 4}};65 for (test_offsets_ok* it = begin(offsets); it != end(offsets); ++it) {66 test_offsets_ok t = *it;67 InternT out[array_size(exp) - 1] = {};68 assert(t.in_size <= array_size(in));69 assert(t.out_size <= array_size(out));70 mbstate_t state = {};71 const ExternT* in_next = nullptr;72 InternT* out_next = nullptr;73 codecvt_base::result res = codecvt_base::ok;74 75 res = cvt.in(state, in, in + t.in_size, in_next, out, out + t.out_size, out_next);76 assert(res == cvt.ok);77 assert(in_next == in + t.in_size);78 assert(out_next == out + t.out_size);79 assert(char_traits<InternT>::compare(out, exp, t.out_size) == 0);80 if (t.out_size < array_size(out))81 assert(out[t.out_size] == 0);82 83 state = mbstate_t();84 int len = cvt.length(state, in, in + t.in_size, t.out_size);85 assert(len >= 0);86 assert(static_cast<size_t>(len) == t.in_size);87 }88 89 for (test_offsets_ok* it = begin(offsets); it != end(offsets); ++it) {90 test_offsets_ok t = *it;91 InternT out[array_size(exp)] = {};92 assert(t.in_size <= array_size(in));93 assert(t.out_size <= array_size(out));94 mbstate_t state = {};95 const ExternT* in_next = nullptr;96 InternT* out_next = nullptr;97 codecvt_base::result res = codecvt_base::ok;98 99 res = cvt.in(state, in, in + t.in_size, in_next, out, end(out), out_next);100 assert(res == cvt.ok);101 assert(in_next == in + t.in_size);102 assert(out_next == out + t.out_size);103 assert(char_traits<InternT>::compare(out, exp, t.out_size) == 0);104 if (t.out_size < array_size(out))105 assert(out[t.out_size] == 0);106 107 state = mbstate_t();108 int len = cvt.length(state, in, in + t.in_size, array_size(out));109 assert(len >= 0);110 assert(static_cast<size_t>(len) == t.in_size);111 }112}113 114template <class InternT, class ExternT>115void utf8_to_utf32_in_partial(const std::codecvt<InternT, ExternT, mbstate_t>& cvt) {116 // UTF-8 string of 1-byte CP, 2-byte CP, 3-byte CP and 4-byte CP117 const unsigned char input[] = "b\u0448\uAAAA\U0010AAAA";118 const char32_t expected[] = {'b', 0x0448, 0xAAAA, 0x10AAAA, 0};119 static_assert(array_size(input) == 11, "");120 static_assert(array_size(expected) == 5, "");121 122 ExternT in[array_size(input)];123 InternT exp[array_size(expected)];124 copy(begin(input), end(input), begin(in));125 copy(begin(expected), end(expected), begin(exp));126 assert(char_traits<ExternT>::length(in) == 10);127 assert(char_traits<InternT>::length(exp) == 4);128 129 test_offsets_partial offsets[] = {130 {1, 0, 0, 0}, // no space for first CP131 132 {3, 1, 1, 1}, // no space for second CP133 {2, 2, 1, 1}, // incomplete second CP134 {2, 1, 1, 1}, // incomplete second CP, and no space for it135 136 {6, 2, 3, 2}, // no space for third CP137 {4, 3, 3, 2}, // incomplete third CP138 {5, 3, 3, 2}, // incomplete third CP139 {4, 2, 3, 2}, // incomplete third CP, and no space for it140 {5, 2, 3, 2}, // incomplete third CP, and no space for it141 142 {10, 3, 6, 3}, // no space for fourth CP143 {7, 4, 6, 3}, // incomplete fourth CP144 {8, 4, 6, 3}, // incomplete fourth CP145 {9, 4, 6, 3}, // incomplete fourth CP146 {7, 3, 6, 3}, // incomplete fourth CP, and no space for it147 {8, 3, 6, 3}, // incomplete fourth CP, and no space for it148 {9, 3, 6, 3}, // incomplete fourth CP, and no space for it149 };150 151 for (test_offsets_partial* it = begin(offsets); it != end(offsets); ++it) {152 test_offsets_partial t = *it;153 InternT out[array_size(exp) - 1] = {};154 assert(t.in_size <= array_size(in));155 assert(t.out_size <= array_size(out));156 assert(t.expected_in_next <= t.in_size);157 assert(t.expected_out_next <= t.out_size);158 mbstate_t state = {};159 const ExternT* in_next = nullptr;160 InternT* out_next = nullptr;161 codecvt_base::result res = codecvt_base::ok;162 163 res = cvt.in(state, in, in + t.in_size, in_next, out, out + t.out_size, out_next);164 assert(res == cvt.partial);165 assert(in_next == in + t.expected_in_next);166 assert(out_next == out + t.expected_out_next);167 assert(char_traits<InternT>::compare(out, exp, t.expected_out_next) == 0);168 if (t.expected_out_next < array_size(out))169 assert(out[t.expected_out_next] == 0);170 171 state = mbstate_t();172 int len = cvt.length(state, in, in + t.in_size, t.out_size);173 assert(len >= 0);174 assert(static_cast<size_t>(len) == t.expected_in_next);175 }176}177 178template <class InternT, class ExternT>179void utf8_to_utf32_in_error(const std::codecvt<InternT, ExternT, mbstate_t>& cvt) {180 // UTF-8 string of 1-byte CP, 2-byte CP, 3-byte CP, 4-byte CP181 const unsigned char input[] = "b\u0448\uD700\U0010AAAA";182 const char32_t expected[] = {'b', 0x0448, 0xD700, 0x10AAAA, 0};183 static_assert(array_size(input) == 11, "");184 static_assert(array_size(expected) == 5, "");185 186 ExternT in[array_size(input)];187 InternT exp[array_size(expected)];188 copy(begin(input), end(input), begin(in));189 copy(begin(expected), end(expected), begin(exp));190 assert(char_traits<ExternT>::length(in) == 10);191 assert(char_traits<InternT>::length(exp) == 4);192 193 // There are 5 classes of errors in UTF-8 decoding194 // 1. Missing leading byte195 // 2. Missing trailing byte196 // 3. Surrogate CP197 // 4. Overlong sequence198 // 5. CP out of Unicode range199 test_offsets_error<unsigned char> offsets[] = {200 201 // 1. Missing leading byte. We will replace the leading byte with202 // non-leading byte, such as a byte that is always invalid or a trailing203 // byte.204 205 // replace leading byte with invalid byte206 {1, 4, 0, 0, 0xFF, 0},207 {3, 4, 1, 1, 0xFF, 1},208 {6, 4, 3, 2, 0xFF, 3},209 {10, 4, 6, 3, 0xFF, 6},210 211 // replace leading byte with trailing byte212 {1, 4, 0, 0, 0b10101010, 0},213 {3, 4, 1, 1, 0b10101010, 1},214 {6, 4, 3, 2, 0b10101010, 3},215 {10, 4, 6, 3, 0b10101010, 6},216 217 // 2. Missing trailing byte. We will replace the trailing byte with218 // non-trailing byte, such as a byte that is always invalid or a leading219 // byte (simple ASCII byte in our case).220 221 // replace first trailing byte with ASCII byte222 {3, 4, 1, 1, 'z', 2},223 {6, 4, 3, 2, 'z', 4},224 {10, 4, 6, 3, 'z', 7},225 226 // replace first trailing byte with invalid byte227 {3, 4, 1, 1, 0xFF, 2},228 {6, 4, 3, 2, 0xFF, 4},229 {10, 4, 6, 3, 0xFF, 7},230 231 // replace second trailing byte with ASCII byte232 {6, 4, 3, 2, 'z', 5},233 {10, 4, 6, 3, 'z', 8},234 235 // replace second trailing byte with invalid byte236 {6, 4, 3, 2, 0xFF, 5},237 {10, 4, 6, 3, 0xFF, 8},238 239 // replace third trailing byte240 {10, 4, 6, 3, 'z', 9},241 {10, 4, 6, 3, 0xFF, 9},242 243 // 2.1 The following test-cases raise doubt whether error or partial should244 // be returned. For example, we have 4-byte sequence with valid leading245 // byte. If we hide the last byte we need to return partial. But, if the246 // second or third byte, which are visible to the call to codecvt, are247 // malformed then error should be returned.248 249 // replace first trailing byte with ASCII byte, also incomplete at end250 {5, 4, 3, 2, 'z', 4},251 {8, 4, 6, 3, 'z', 7},252 {9, 4, 6, 3, 'z', 7},253 254 // replace first trailing byte with invalid byte, also incomplete at end255 {5, 4, 3, 2, 0xFF, 4},256 {8, 4, 6, 3, 0xFF, 7},257 {9, 4, 6, 3, 0xFF, 7},258 259 // replace second trailing byte with ASCII byte, also incomplete at end260 {9, 4, 6, 3, 'z', 8},261 262 // replace second trailing byte with invalid byte, also incomplete at end263 {9, 4, 6, 3, 0xFF, 8},264 265 // 3. Surrogate CP. We modify the second byte (first trailing) of the 3-byte266 // CP U+D700267 {6, 4, 3, 2, 0b10100000, 4}, // turn U+D700 into U+D800268 {6, 4, 3, 2, 0b10101100, 4}, // turn U+D700 into U+DB00269 {6, 4, 3, 2, 0b10110000, 4}, // turn U+D700 into U+DC00270 {6, 4, 3, 2, 0b10111100, 4}, // turn U+D700 into U+DF00271 272 // 4. Overlong sequence. The CPs in the input are chosen such as modifying273 // just the leading byte is enough to make them overlong, i.e. for the274 // 3-byte and 4-byte CP the second byte (first trailing) has enough leading275 // zeroes.276 {3, 4, 1, 1, 0b11000000, 1}, // make the 2-byte CP overlong277 {3, 4, 1, 1, 0b11000001, 1}, // make the 2-byte CP overlong278 {6, 4, 3, 2, 0b11100000, 3}, // make the 3-byte CP overlong279 {10, 4, 6, 3, 0b11110000, 6}, // make the 4-byte CP overlong280 281 // 5. CP above range282 // turn U+10AAAA into U+14AAAA by changing its leading byte283 {10, 4, 6, 3, 0b11110101, 6},284 // turn U+10AAAA into U+11AAAA by changing its 2nd byte285 {10, 4, 6, 3, 0b10011010, 7},286 };287 for (test_offsets_error<unsigned char>* it = begin(offsets); it != end(offsets); ++it) {288 test_offsets_error<unsigned char> t = *it;289 InternT out[array_size(exp) - 1] = {};290 assert(t.in_size <= array_size(in));291 assert(t.out_size <= array_size(out));292 assert(t.expected_in_next <= t.in_size);293 assert(t.expected_out_next <= t.out_size);294 ExternT old_char = in[t.replace_pos];295 in[t.replace_pos] = t.replace_char;296 297 mbstate_t state = {};298 const ExternT* in_next = nullptr;299 InternT* out_next = nullptr;300 codecvt_base::result res = codecvt_base::ok;301 302 res = cvt.in(state, in, in + t.in_size, in_next, out, out + t.out_size, out_next);303 assert(res == cvt.error);304 assert(in_next == in + t.expected_in_next);305 assert(out_next == out + t.expected_out_next);306 assert(char_traits<InternT>::compare(out, exp, t.expected_out_next) == 0);307 if (t.expected_out_next < array_size(out))308 assert(out[t.expected_out_next] == 0);309 310 state = mbstate_t();311 int len = cvt.length(state, in, in + t.in_size, t.out_size);312 assert(len >= 0);313 assert(static_cast<size_t>(len) == t.expected_in_next);314 315 in[t.replace_pos] = old_char;316 }317}318 319template <class InternT, class ExternT>320void utf8_to_utf32_in(const std::codecvt<InternT, ExternT, mbstate_t>& cvt) {321 utf8_to_utf32_in_ok(cvt);322 utf8_to_utf32_in_partial(cvt);323 utf8_to_utf32_in_error(cvt);324}325 326template <class InternT, class ExternT>327void utf32_to_utf8_out_ok(const std::codecvt<InternT, ExternT, mbstate_t>& cvt) {328 // UTF-8 string of 1-byte CP, 2-byte CP, 3-byte CP and 4-byte CP329 const char32_t input[] = {'b', 0x0448, 0xAAAA, 0x10AAAA, 0};330 const unsigned char expected[] = "b\u0448\uAAAA\U0010AAAA";331 static_assert(array_size(input) == 5, "");332 static_assert(array_size(expected) == 11, "");333 334 InternT in[array_size(input)];335 ExternT exp[array_size(expected)];336 copy(begin(input), end(input), begin(in));337 copy(begin(expected), end(expected), begin(exp));338 assert(char_traits<InternT>::length(in) == 4);339 assert(char_traits<ExternT>::length(exp) == 10);340 341 test_offsets_ok offsets[] = {{0, 0}, {1, 1}, {2, 3}, {3, 6}, {4, 10}};342 for (test_offsets_ok* it = begin(offsets); it != end(offsets); ++it) {343 test_offsets_ok t = *it;344 ExternT out[array_size(exp) - 1] = {};345 assert(t.in_size <= array_size(in));346 assert(t.out_size <= array_size(out));347 mbstate_t state = {};348 const InternT* in_next = nullptr;349 ExternT* out_next = nullptr;350 codecvt_base::result res = codecvt_base::ok;351 352 res = cvt.out(state, in, in + t.in_size, in_next, out, out + t.out_size, out_next);353 assert(res == cvt.ok);354 assert(in_next == in + t.in_size);355 assert(out_next == out + t.out_size);356 assert(char_traits<ExternT>::compare(out, exp, t.out_size) == 0);357 if (t.out_size < array_size(out))358 assert(out[t.out_size] == 0);359 }360}361 362template <class InternT, class ExternT>363void utf32_to_utf8_out_partial(const std::codecvt<InternT, ExternT, mbstate_t>& cvt) {364 // UTF-8 string of 1-byte CP, 2-byte CP, 3-byte CP and 4-byte CP365 const char32_t input[] = {'b', 0x0448, 0xAAAA, 0x10AAAA, 0};366 const unsigned char expected[] = "b\u0448\uAAAA\U0010AAAA";367 static_assert(array_size(input) == 5, "");368 static_assert(array_size(expected) == 11, "");369 370 InternT in[array_size(input)];371 ExternT exp[array_size(expected)];372 copy(begin(input), end(input), begin(in));373 copy(begin(expected), end(expected), begin(exp));374 assert(char_traits<InternT>::length(in) == 4);375 assert(char_traits<ExternT>::length(exp) == 10);376 377 test_offsets_partial offsets[] = {378 {1, 0, 0, 0}, // no space for first CP379 380 {2, 1, 1, 1}, // no space for second CP381 {2, 2, 1, 1}, // no space for second CP382 383 {3, 3, 2, 3}, // no space for third CP384 {3, 4, 2, 3}, // no space for third CP385 {3, 5, 2, 3}, // no space for third CP386 387 {4, 6, 3, 6}, // no space for fourth CP388 {4, 7, 3, 6}, // no space for fourth CP389 {4, 8, 3, 6}, // no space for fourth CP390 {4, 9, 3, 6}, // no space for fourth CP391 };392 for (test_offsets_partial* it = begin(offsets); it != end(offsets); ++it) {393 test_offsets_partial t = *it;394 ExternT out[array_size(exp) - 1] = {};395 assert(t.in_size <= array_size(in));396 assert(t.out_size <= array_size(out));397 assert(t.expected_in_next <= t.in_size);398 assert(t.expected_out_next <= t.out_size);399 mbstate_t state = {};400 const InternT* in_next = nullptr;401 ExternT* out_next = nullptr;402 codecvt_base::result res = codecvt_base::ok;403 404 res = cvt.out(state, in, in + t.in_size, in_next, out, out + t.out_size, out_next);405 assert(res == cvt.partial);406 assert(in_next == in + t.expected_in_next);407 assert(out_next == out + t.expected_out_next);408 assert(char_traits<ExternT>::compare(out, exp, t.expected_out_next) == 0);409 if (t.expected_out_next < array_size(out))410 assert(out[t.expected_out_next] == 0);411 }412}413 414template <class InternT, class ExternT>415void utf32_to_utf8_out_error(const std::codecvt<InternT, ExternT, mbstate_t>& cvt) {416 // UTF-8 string of 1-byte CP, 2-byte CP, 3-byte CP and 4-byte CP417 const char32_t input[] = {'b', 0x0448, 0xAAAA, 0x10AAAA, 0};418 const unsigned char expected[] = "b\u0448\uAAAA\U0010AAAA";419 static_assert(array_size(input) == 5, "");420 static_assert(array_size(expected) == 11, "");421 422 InternT in[array_size(input)];423 ExternT exp[array_size(expected)];424 copy(begin(input), end(input), begin(in));425 copy(begin(expected), end(expected), begin(exp));426 assert(char_traits<InternT>::length(in) == 4);427 assert(char_traits<ExternT>::length(exp) == 10);428 429 test_offsets_error<InternT> offsets[] = {430 431 // Surrogate CP432 {4, 10, 0, 0, 0xD800, 0},433 {4, 10, 1, 1, 0xDBFF, 1},434 {4, 10, 2, 3, 0xDC00, 2},435 {4, 10, 3, 6, 0xDFFF, 3},436 437 // CP out of range438 {4, 10, 0, 0, 0x00110000, 0},439 {4, 10, 1, 1, 0x00110000, 1},440 {4, 10, 2, 3, 0x00110000, 2},441 {4, 10, 3, 6, 0x00110000, 3}};442 443 for (test_offsets_error<InternT>* it = begin(offsets); it != end(offsets); ++it) {444 test_offsets_error<InternT> t = *it;445 ExternT out[array_size(exp) - 1] = {};446 assert(t.in_size <= array_size(in));447 assert(t.out_size <= array_size(out));448 assert(t.expected_in_next <= t.in_size);449 assert(t.expected_out_next <= t.out_size);450 InternT old_char = in[t.replace_pos];451 in[t.replace_pos] = t.replace_char;452 453 mbstate_t state = {};454 const InternT* in_next = nullptr;455 ExternT* out_next = nullptr;456 codecvt_base::result res = codecvt_base::ok;457 458 res = cvt.out(state, in, in + t.in_size, in_next, out, out + t.out_size, out_next);459 assert(res == cvt.error);460 assert(in_next == in + t.expected_in_next);461 assert(out_next == out + t.expected_out_next);462 assert(char_traits<ExternT>::compare(out, exp, t.expected_out_next) == 0);463 if (t.expected_out_next < array_size(out))464 assert(out[t.expected_out_next] == 0);465 466 in[t.replace_pos] = old_char;467 }468}469 470template <class InternT, class ExternT>471void utf32_to_utf8_out(const std::codecvt<InternT, ExternT, mbstate_t>& cvt) {472 utf32_to_utf8_out_ok(cvt);473 utf32_to_utf8_out_partial(cvt);474 utf32_to_utf8_out_error(cvt);475}476 477template <class InternT, class ExternT>478void test_utf8_utf32_cvt(const std::codecvt<InternT, ExternT, mbstate_t>& cvt) {479 utf8_to_utf32_in(cvt);480 utf32_to_utf8_out(cvt);481}482 483template <class InternT, class ExternT>484void utf8_to_utf16_in_ok(const std::codecvt<InternT, ExternT, mbstate_t>& cvt) {485 // UTF-8 string of 1-byte CP, 2-byte CP, 3-byte CP and 4-byte CP486 const unsigned char input[] = "b\u0448\uAAAA\U0010AAAA";487 const InternT expected[] = {'b', 0x0448, 0xAAAA, 0xDBEA, 0xDEAA, 0};488 static_assert(array_size(input) == 11, "");489 static_assert(array_size(expected) == 6, "");490 491 ExternT in[array_size(input)];492 InternT exp[array_size(expected)];493 copy(begin(input), end(input), begin(in));494 copy(begin(expected), end(expected), begin(exp));495 assert(char_traits<ExternT>::length(in) == 10);496 assert(char_traits<InternT>::length(exp) == 5);497 498 test_offsets_ok offsets[] = {{0, 0}, {1, 1}, {3, 2}, {6, 3}, {10, 5}};499 for (test_offsets_ok* it = begin(offsets); it != end(offsets); ++it) {500 test_offsets_ok t = *it;501 InternT out[array_size(exp) - 1] = {};502 assert(t.in_size <= array_size(in));503 assert(t.out_size <= array_size(out));504 mbstate_t state = {};505 const ExternT* in_next = nullptr;506 InternT* out_next = nullptr;507 codecvt_base::result res = codecvt_base::ok;508 509 res = cvt.in(state, in, in + t.in_size, in_next, out, out + t.out_size, out_next);510 assert(res == cvt.ok);511 assert(in_next == in + t.in_size);512 assert(out_next == out + t.out_size);513 assert(char_traits<InternT>::compare(out, exp, t.out_size) == 0);514 if (t.out_size < array_size(out))515 assert(out[t.out_size] == 0);516 517 state = mbstate_t();518 int len = cvt.length(state, in, in + t.in_size, t.out_size);519 assert(len >= 0);520 assert(static_cast<size_t>(len) == t.in_size);521 }522 523 for (test_offsets_ok* it = begin(offsets); it != end(offsets); ++it) {524 test_offsets_ok t = *it;525 InternT out[array_size(exp)] = {};526 assert(t.in_size <= array_size(in));527 assert(t.out_size <= array_size(out));528 mbstate_t state = {};529 const ExternT* in_next = nullptr;530 InternT* out_next = nullptr;531 codecvt_base::result res = codecvt_base::ok;532 533 res = cvt.in(state, in, in + t.in_size, in_next, out, end(out), out_next);534 assert(res == cvt.ok);535 assert(in_next == in + t.in_size);536 assert(out_next == out + t.out_size);537 assert(char_traits<InternT>::compare(out, exp, t.out_size) == 0);538 if (t.out_size < array_size(out))539 assert(out[t.out_size] == 0);540 541 state = mbstate_t();542 int len = cvt.length(state, in, in + t.in_size, array_size(out));543 assert(len >= 0);544 assert(static_cast<size_t>(len) == t.in_size);545 }546}547 548template <class InternT, class ExternT>549void utf8_to_utf16_in_partial(const std::codecvt<InternT, ExternT, mbstate_t>& cvt) {550 // UTF-8 string of 1-byte CP, 2-byte CP, 3-byte CP and 4-byte CP551 const unsigned char input[] = "b\u0448\uAAAA\U0010AAAA";552 const InternT expected[] = {'b', 0x0448, 0xAAAA, 0xDBEA, 0xDEAA, 0};553 static_assert(array_size(input) == 11, "");554 static_assert(array_size(expected) == 6, "");555 556 ExternT in[array_size(input)];557 InternT exp[array_size(expected)];558 copy(begin(input), end(input), begin(in));559 copy(begin(expected), end(expected), begin(exp));560 assert(char_traits<ExternT>::length(in) == 10);561 assert(char_traits<InternT>::length(exp) == 5);562 563 test_offsets_partial offsets[] = {564 {1, 0, 0, 0}, // no space for first CP565 566 {3, 1, 1, 1}, // no space for second CP567 {2, 2, 1, 1}, // incomplete second CP568 {2, 1, 1, 1}, // incomplete second CP, and no space for it569 570 {6, 2, 3, 2}, // no space for third CP571 {4, 3, 3, 2}, // incomplete third CP572 {5, 3, 3, 2}, // incomplete third CP573 {4, 2, 3, 2}, // incomplete third CP, and no space for it574 {5, 2, 3, 2}, // incomplete third CP, and no space for it575 576 {10, 3, 6, 3}, // no space for fourth CP577 {10, 4, 6, 3}, // no space for fourth CP578 {7, 5, 6, 3}, // incomplete fourth CP579 {8, 5, 6, 3}, // incomplete fourth CP580 {9, 5, 6, 3}, // incomplete fourth CP581 {7, 3, 6, 3}, // incomplete fourth CP, and no space for it582 {8, 3, 6, 3}, // incomplete fourth CP, and no space for it583 {9, 3, 6, 3}, // incomplete fourth CP, and no space for it584 {7, 4, 6, 3}, // incomplete fourth CP, and no space for it585 {8, 4, 6, 3}, // incomplete fourth CP, and no space for it586 {9, 4, 6, 3}, // incomplete fourth CP, and no space for it587 588 };589 590 for (test_offsets_partial* it = begin(offsets); it != end(offsets); ++it) {591 test_offsets_partial t = *it;592 InternT out[array_size(exp) - 1] = {};593 assert(t.in_size <= array_size(in));594 assert(t.out_size <= array_size(out));595 assert(t.expected_in_next <= t.in_size);596 assert(t.expected_out_next <= t.out_size);597 mbstate_t state = {};598 const ExternT* in_next = nullptr;599 InternT* out_next = nullptr;600 codecvt_base::result res = codecvt_base::ok;601 602 res = cvt.in(state, in, in + t.in_size, in_next, out, out + t.out_size, out_next);603 assert(res == cvt.partial);604 assert(in_next == in + t.expected_in_next);605 assert(out_next == out + t.expected_out_next);606 assert(char_traits<InternT>::compare(out, exp, t.expected_out_next) == 0);607 if (t.expected_out_next < array_size(out))608 assert(out[t.expected_out_next] == 0);609 610 state = mbstate_t();611 int len = cvt.length(state, in, in + t.in_size, t.out_size);612 assert(len >= 0);613 assert(static_cast<size_t>(len) == t.expected_in_next);614 }615}616 617template <class InternT, class ExternT>618void utf8_to_utf16_in_error(const std::codecvt<InternT, ExternT, mbstate_t>& cvt) {619 // UTF-8 string of 1-byte CP, 2-byte CP, 3-byte CP, 4-byte CP620 const unsigned char input[] = "b\u0448\uD700\U0010AAAA";621 const InternT expected[] = {'b', 0x0448, 0xD700, 0xDBEA, 0xDEAA, 0};622 static_assert(array_size(input) == 11, "");623 static_assert(array_size(expected) == 6, "");624 625 ExternT in[array_size(input)];626 InternT exp[array_size(expected)];627 copy(begin(input), end(input), begin(in));628 copy(begin(expected), end(expected), begin(exp));629 assert(char_traits<ExternT>::length(in) == 10);630 assert(char_traits<InternT>::length(exp) == 5);631 632 // There are 5 classes of errors in UTF-8 decoding633 // 1. Missing leading byte634 // 2. Missing trailing byte635 // 3. Surrogate CP636 // 4. Overlong sequence637 // 5. CP out of Unicode range638 test_offsets_error<unsigned char> offsets[] = {639 640 // 1. Missing leading byte. We will replace the leading byte with641 // non-leading byte, such as a byte that is always invalid or a trailing642 // byte.643 644 // replace leading byte with invalid byte645 {1, 5, 0, 0, 0xFF, 0},646 {3, 5, 1, 1, 0xFF, 1},647 {6, 5, 3, 2, 0xFF, 3},648 {10, 5, 6, 3, 0xFF, 6},649 650 // replace leading byte with trailing byte651 {1, 5, 0, 0, 0b10101010, 0},652 {3, 5, 1, 1, 0b10101010, 1},653 {6, 5, 3, 2, 0b10101010, 3},654 {10, 5, 6, 3, 0b10101010, 6},655 656 // 2. Missing trailing byte. We will replace the trailing byte with657 // non-trailing byte, such as a byte that is always invalid or a leading658 // byte (simple ASCII byte in our case).659 660 // replace first trailing byte with ASCII byte661 {3, 5, 1, 1, 'z', 2},662 {6, 5, 3, 2, 'z', 4},663 {10, 5, 6, 3, 'z', 7},664 665 // replace first trailing byte with invalid byte666 {3, 5, 1, 1, 0xFF, 2},667 {6, 5, 3, 2, 0xFF, 4},668 {10, 5, 6, 3, 0xFF, 7},669 670 // replace second trailing byte with ASCII byte671 {6, 5, 3, 2, 'z', 5},672 {10, 5, 6, 3, 'z', 8},673 674 // replace second trailing byte with invalid byte675 {6, 5, 3, 2, 0xFF, 5},676 {10, 5, 6, 3, 0xFF, 8},677 678 // replace third trailing byte679 {10, 5, 6, 3, 'z', 9},680 {10, 5, 6, 3, 0xFF, 9},681 682 // 2.1 The following test-cases raise doubt whether error or partial should683 // be returned. For example, we have 4-byte sequence with valid leading684 // byte. If we hide the last byte we need to return partial. But, if the685 // second or third byte, which are visible to the call to codecvt, are686 // malformed then error should be returned.687 688 // replace first trailing byte with ASCII byte, also incomplete at end689 {5, 5, 3, 2, 'z', 4},690 {8, 5, 6, 3, 'z', 7},691 {9, 5, 6, 3, 'z', 7},692 693 // replace first trailing byte with invalid byte, also incomplete at end694 {5, 5, 3, 2, 0xFF, 4},695 {8, 5, 6, 3, 0xFF, 7},696 {9, 5, 6, 3, 0xFF, 7},697 698 // replace second trailing byte with ASCII byte, also incomplete at end699 {9, 5, 6, 3, 'z', 8},700 701 // replace second trailing byte with invalid byte, also incomplete at end702 {9, 5, 6, 3, 0xFF, 8},703 704 // 3. Surrogate CP. We modify the second byte (first trailing) of the 3-byte705 // CP U+D700706 {6, 5, 3, 2, 0b10100000, 4}, // turn U+D700 into U+D800707 {6, 5, 3, 2, 0b10101100, 4}, // turn U+D700 into U+DB00708 {6, 5, 3, 2, 0b10110000, 4}, // turn U+D700 into U+DC00709 {6, 5, 3, 2, 0b10111100, 4}, // turn U+D700 into U+DF00710 711 // 4. Overlong sequence. The CPs in the input are chosen such as modifying712 // just the leading byte is enough to make them overlong, i.e. for the713 // 3-byte and 4-byte CP the second byte (first trailing) has enough leading714 // zeroes.715 {3, 5, 1, 1, 0b11000000, 1}, // make the 2-byte CP overlong716 {3, 5, 1, 1, 0b11000001, 1}, // make the 2-byte CP overlong717 {6, 5, 3, 2, 0b11100000, 3}, // make the 3-byte CP overlong718 {10, 5, 6, 3, 0b11110000, 6}, // make the 4-byte CP overlong719 720 // 5. CP above range721 // turn U+10AAAA into U+14AAAA by changing its leading byte722 {10, 5, 6, 3, 0b11110101, 6},723 // turn U+10AAAA into U+11AAAA by changing its 2nd byte724 {10, 5, 6, 3, 0b10011010, 7},725 };726 for (test_offsets_error<unsigned char>* it = begin(offsets); it != end(offsets); ++it) {727 test_offsets_error<unsigned char> t = *it;728 InternT out[array_size(exp) - 1] = {};729 assert(t.in_size <= array_size(in));730 assert(t.out_size <= array_size(out));731 assert(t.expected_in_next <= t.in_size);732 assert(t.expected_out_next <= t.out_size);733 ExternT old_char = in[t.replace_pos];734 in[t.replace_pos] = t.replace_char;735 736 mbstate_t state = {};737 const ExternT* in_next = nullptr;738 InternT* out_next = nullptr;739 codecvt_base::result res = codecvt_base::ok;740 741 res = cvt.in(state, in, in + t.in_size, in_next, out, out + t.out_size, out_next);742 assert(res == cvt.error);743 assert(in_next == in + t.expected_in_next);744 assert(out_next == out + t.expected_out_next);745 assert(char_traits<InternT>::compare(out, exp, t.expected_out_next) == 0);746 if (t.expected_out_next < array_size(out))747 assert(out[t.expected_out_next] == 0);748 749 state = mbstate_t();750 int len = cvt.length(state, in, in + t.in_size, t.out_size);751 assert(len >= 0);752 assert(static_cast<size_t>(len) == t.expected_in_next);753 754 in[t.replace_pos] = old_char;755 }756}757 758template <class InternT, class ExternT>759void utf8_to_utf16_in(const std::codecvt<InternT, ExternT, mbstate_t>& cvt) {760 utf8_to_utf16_in_ok(cvt);761 utf8_to_utf16_in_partial(cvt);762 utf8_to_utf16_in_error(cvt);763}764 765template <class InternT, class ExternT>766void utf16_to_utf8_out_ok(const std::codecvt<InternT, ExternT, mbstate_t>& cvt) {767 // UTF-8 string of 1-byte CP, 2-byte CP, 3-byte CP and 4-byte CP768 const InternT input[] = {'b', 0x0448, 0xAAAA, 0xDBEA, 0xDEAA, 0};769 const unsigned char expected[] = "b\u0448\uAAAA\U0010AAAA";770 static_assert(array_size(input) == 6, "");771 static_assert(array_size(expected) == 11, "");772 773 InternT in[array_size(input)];774 ExternT exp[array_size(expected)];775 copy(begin(input), end(input), begin(in));776 copy(begin(expected), end(expected), begin(exp));777 assert(char_traits<InternT>::length(in) == 5);778 assert(char_traits<ExternT>::length(exp) == 10);779 780 test_offsets_ok offsets[] = {{0, 0}, {1, 1}, {2, 3}, {3, 6}, {5, 10}};781 for (test_offsets_ok* it = begin(offsets); it != end(offsets); ++it) {782 test_offsets_ok t = *it;783 ExternT out[array_size(exp) - 1] = {};784 assert(t.in_size <= array_size(in));785 assert(t.out_size <= array_size(out));786 mbstate_t state = {};787 const InternT* in_next = nullptr;788 ExternT* out_next = nullptr;789 codecvt_base::result res = codecvt_base::ok;790 791 res = cvt.out(state, in, in + t.in_size, in_next, out, out + t.out_size, out_next);792 assert(res == cvt.ok);793 assert(in_next == in + t.in_size);794 assert(out_next == out + t.out_size);795 assert(char_traits<ExternT>::compare(out, exp, t.out_size) == 0);796 if (t.out_size < array_size(out))797 assert(out[t.out_size] == 0);798 }799}800 801template <class InternT, class ExternT>802void utf16_to_utf8_out_partial(const std::codecvt<InternT, ExternT, mbstate_t>& cvt) {803 // UTF-8 string of 1-byte CP, 2-byte CP, 3-byte CP and 4-byte CP804 const InternT input[] = {'b', 0x0448, 0xAAAA, 0xDBEA, 0xDEAA, 0};805 const unsigned char expected[] = "b\u0448\uAAAA\U0010AAAA";806 static_assert(array_size(input) == 6, "");807 static_assert(array_size(expected) == 11, "");808 809 InternT in[array_size(input)];810 ExternT exp[array_size(expected)];811 copy(begin(input), end(input), begin(in));812 copy(begin(expected), end(expected), begin(exp));813 assert(char_traits<InternT>::length(in) == 5);814 assert(char_traits<ExternT>::length(exp) == 10);815 816 test_offsets_partial offsets[] = {817 {1, 0, 0, 0}, // no space for first CP818 819 {2, 1, 1, 1}, // no space for second CP820 {2, 2, 1, 1}, // no space for second CP821 822 {3, 3, 2, 3}, // no space for third CP823 {3, 4, 2, 3}, // no space for third CP824 {3, 5, 2, 3}, // no space for third CP825 826 {5, 6, 3, 6}, // no space for fourth CP827 {5, 7, 3, 6}, // no space for fourth CP828 {5, 8, 3, 6}, // no space for fourth CP829 {5, 9, 3, 6}, // no space for fourth CP830 831 {4, 10, 3, 6}, // incomplete fourth CP832 833 {4, 6, 3, 6}, // incomplete fourth CP, and no space for it834 {4, 7, 3, 6}, // incomplete fourth CP, and no space for it835 {4, 8, 3, 6}, // incomplete fourth CP, and no space for it836 {4, 9, 3, 6}, // incomplete fourth CP, and no space for it837 };838 for (test_offsets_partial* it = begin(offsets); it != end(offsets); ++it) {839 test_offsets_partial t = *it;840 ExternT out[array_size(exp) - 1] = {};841 assert(t.in_size <= array_size(in));842 assert(t.out_size <= array_size(out));843 assert(t.expected_in_next <= t.in_size);844 assert(t.expected_out_next <= t.out_size);845 mbstate_t state = {};846 const InternT* in_next = nullptr;847 ExternT* out_next = nullptr;848 codecvt_base::result res = codecvt_base::ok;849 850 res = cvt.out(state, in, in + t.in_size, in_next, out, out + t.out_size, out_next);851 assert(res == cvt.partial);852 assert(in_next == in + t.expected_in_next);853 assert(out_next == out + t.expected_out_next);854 assert(char_traits<ExternT>::compare(out, exp, t.expected_out_next) == 0);855 if (t.expected_out_next < array_size(out))856 assert(out[t.expected_out_next] == 0);857 }858}859 860template <class InternT, class ExternT>861void utf16_to_utf8_out_error(const std::codecvt<InternT, ExternT, mbstate_t>& cvt) {862 // UTF-8 string of 1-byte CP, 2-byte CP, 3-byte CP and 4-byte CP863 const InternT input[] = {'b', 0x0448, 0xAAAA, 0xDBEA, 0xDEAA, 0};864 const unsigned char expected[] = "b\u0448\uAAAA\U0010AAAA";865 static_assert(array_size(input) == 6, "");866 static_assert(array_size(expected) == 11, "");867 868 InternT in[array_size(input)];869 ExternT exp[array_size(expected)];870 copy(begin(input), end(input), begin(in));871 copy(begin(expected), end(expected), begin(exp));872 assert(char_traits<InternT>::length(in) == 5);873 assert(char_traits<ExternT>::length(exp) == 10);874 875 // The only possible error in UTF-16 is unpaired surrogate code units.876 // So we replace valid code points (scalar values) with lone surrogate CU.877 test_offsets_error<InternT> offsets[] = {878 {5, 10, 0, 0, 0xD800, 0},879 {5, 10, 0, 0, 0xDBFF, 0},880 {5, 10, 0, 0, 0xDC00, 0},881 {5, 10, 0, 0, 0xDFFF, 0},882 883 {5, 10, 1, 1, 0xD800, 1},884 {5, 10, 1, 1, 0xDBFF, 1},885 {5, 10, 1, 1, 0xDC00, 1},886 {5, 10, 1, 1, 0xDFFF, 1},887 888 {5, 10, 2, 3, 0xD800, 2},889 {5, 10, 2, 3, 0xDBFF, 2},890 {5, 10, 2, 3, 0xDC00, 2},891 {5, 10, 2, 3, 0xDFFF, 2},892 893 // make the leading surrogate a trailing one894 {5, 10, 3, 6, 0xDC00, 3},895 {5, 10, 3, 6, 0xDFFF, 3},896 897 // make the trailing surrogate a leading one898 {5, 10, 3, 6, 0xD800, 4},899 {5, 10, 3, 6, 0xDBFF, 4},900 901 // make the trailing surrogate a BMP char902 {5, 10, 3, 6, 'z', 4},903 };904 905 for (test_offsets_error<InternT>* it = begin(offsets); it != end(offsets); ++it) {906 test_offsets_error<InternT> t = *it;907 ExternT out[array_size(exp) - 1] = {};908 assert(t.in_size <= array_size(in));909 assert(t.out_size <= array_size(out));910 assert(t.expected_in_next <= t.in_size);911 assert(t.expected_out_next <= t.out_size);912 InternT old_char = in[t.replace_pos];913 in[t.replace_pos] = t.replace_char;914 915 mbstate_t state = {};916 const InternT* in_next = nullptr;917 ExternT* out_next = nullptr;918 codecvt_base::result res = codecvt_base::ok;919 920 res = cvt.out(state, in, in + t.in_size, in_next, out, out + t.out_size, out_next);921 assert(res == cvt.error);922 assert(in_next == in + t.expected_in_next);923 assert(out_next == out + t.expected_out_next);924 assert(char_traits<ExternT>::compare(out, exp, t.expected_out_next) == 0);925 if (t.expected_out_next < array_size(out))926 assert(out[t.expected_out_next] == 0);927 928 in[t.replace_pos] = old_char;929 }930}931 932template <class InternT, class ExternT>933void utf16_to_utf8_out(const std::codecvt<InternT, ExternT, mbstate_t>& cvt) {934 utf16_to_utf8_out_ok(cvt);935 utf16_to_utf8_out_partial(cvt);936 utf16_to_utf8_out_error(cvt);937}938 939template <class InternT, class ExternT>940void test_utf8_utf16_cvt(const std::codecvt<InternT, ExternT, mbstate_t>& cvt) {941 utf8_to_utf16_in(cvt);942 utf16_to_utf8_out(cvt);943}944 945template <class InternT, class ExternT>946void utf8_to_ucs2_in_ok(const std::codecvt<InternT, ExternT, mbstate_t>& cvt) {947 // UTF-8 string of 1-byte CP, 2-byte CP and 3-byte CP948 const unsigned char input[] = "b\u0448\uAAAA";949 const char16_t expected[] = {'b', 0x0448, 0xAAAA, 0};950 static_assert(array_size(input) == 7, "");951 static_assert(array_size(expected) == 4, "");952 953 ExternT in[array_size(input)];954 InternT exp[array_size(expected)];955 copy(begin(input), end(input), begin(in));956 copy(begin(expected), end(expected), begin(exp));957 assert(char_traits<ExternT>::length(in) == 6);958 assert(char_traits<InternT>::length(exp) == 3);959 960 test_offsets_ok offsets[] = {{0, 0}, {1, 1}, {3, 2}, {6, 3}};961 for (test_offsets_ok* it = begin(offsets); it != end(offsets); ++it) {962 test_offsets_ok t = *it;963 InternT out[array_size(exp) - 1] = {};964 assert(t.in_size <= array_size(in));965 assert(t.out_size <= array_size(out));966 mbstate_t state = {};967 const ExternT* in_next = nullptr;968 InternT* out_next = nullptr;969 codecvt_base::result res = codecvt_base::ok;970 971 res = cvt.in(state, in, in + t.in_size, in_next, out, out + t.out_size, out_next);972 assert(res == cvt.ok);973 assert(in_next == in + t.in_size);974 assert(out_next == out + t.out_size);975 assert(char_traits<InternT>::compare(out, exp, t.out_size) == 0);976 if (t.out_size < array_size(out))977 assert(out[t.out_size] == 0);978 979 state = mbstate_t();980 int len = cvt.length(state, in, in + t.in_size, t.out_size);981 assert(len >= 0);982 assert(static_cast<size_t>(len) == t.in_size);983 }984 985 for (test_offsets_ok* it = begin(offsets); it != end(offsets); ++it) {986 test_offsets_ok t = *it;987 InternT out[array_size(exp)] = {};988 assert(t.in_size <= array_size(in));989 assert(t.out_size <= array_size(out));990 mbstate_t state = {};991 const ExternT* in_next = nullptr;992 InternT* out_next = nullptr;993 codecvt_base::result res = codecvt_base::ok;994 995 res = cvt.in(state, in, in + t.in_size, in_next, out, end(out), out_next);996 assert(res == cvt.ok);997 assert(in_next == in + t.in_size);998 assert(out_next == out + t.out_size);999 assert(char_traits<InternT>::compare(out, exp, t.out_size) == 0);1000 if (t.out_size < array_size(out))1001 assert(out[t.out_size] == 0);1002 1003 state = mbstate_t();1004 int len = cvt.length(state, in, in + t.in_size, array_size(out));1005 assert(len >= 0);1006 assert(static_cast<size_t>(len) == t.in_size);1007 }1008}1009 1010template <class InternT, class ExternT>1011void utf8_to_ucs2_in_partial(const std::codecvt<InternT, ExternT, mbstate_t>& cvt) {1012 // UTF-8 string of 1-byte CP, 2-byte CP and 3-byte CP1013 const unsigned char input[] = "b\u0448\uAAAA";1014 const char16_t expected[] = {'b', 0x0448, 0xAAAA, 0};1015 static_assert(array_size(input) == 7, "");1016 static_assert(array_size(expected) == 4, "");1017 1018 ExternT in[array_size(input)];1019 InternT exp[array_size(expected)];1020 copy(begin(input), end(input), begin(in));1021 copy(begin(expected), end(expected), begin(exp));1022 assert(char_traits<ExternT>::length(in) == 6);1023 assert(char_traits<InternT>::length(exp) == 3);1024 1025 test_offsets_partial offsets[] = {1026 {1, 0, 0, 0}, // no space for first CP1027 1028 {3, 1, 1, 1}, // no space for second CP1029 {2, 2, 1, 1}, // incomplete second CP1030 {2, 1, 1, 1}, // incomplete second CP, and no space for it1031 1032 {6, 2, 3, 2}, // no space for third CP1033 {4, 3, 3, 2}, // incomplete third CP1034 {5, 3, 3, 2}, // incomplete third CP1035 {4, 2, 3, 2}, // incomplete third CP, and no space for it1036 {5, 2, 3, 2}, // incomplete third CP, and no space for it1037 };1038 1039 for (test_offsets_partial* it = begin(offsets); it != end(offsets); ++it) {1040 test_offsets_partial t = *it;1041 InternT out[array_size(exp) - 1] = {};1042 assert(t.in_size <= array_size(in));1043 assert(t.out_size <= array_size(out));1044 assert(t.expected_in_next <= t.in_size);1045 assert(t.expected_out_next <= t.out_size);1046 mbstate_t state = {};1047 const ExternT* in_next = nullptr;1048 InternT* out_next = nullptr;1049 codecvt_base::result res = codecvt_base::ok;1050 1051 res = cvt.in(state, in, in + t.in_size, in_next, out, out + t.out_size, out_next);1052 assert(res == cvt.partial);1053 assert(in_next == in + t.expected_in_next);1054 assert(out_next == out + t.expected_out_next);1055 assert(char_traits<InternT>::compare(out, exp, t.expected_out_next) == 0);1056 if (t.expected_out_next < array_size(out))1057 assert(out[t.expected_out_next] == 0);1058 1059 state = mbstate_t();1060 int len = cvt.length(state, in, in + t.in_size, t.out_size);1061 assert(len >= 0);1062 assert(static_cast<size_t>(len) == t.expected_in_next);1063 }1064}1065 1066template <class InternT, class ExternT>1067void utf8_to_ucs2_in_error(const std::codecvt<InternT, ExternT, mbstate_t>& cvt) {1068 const unsigned char input[] = "b\u0448\uD700\U0010AAAA";1069 const char16_t expected[] = {'b', 0x0448, 0xD700, 0xDBEA, 0xDEAA, 0};1070 static_assert(array_size(input) == 11, "");1071 static_assert(array_size(expected) == 6, "");1072 1073 ExternT in[array_size(input)];1074 InternT exp[array_size(expected)];1075 copy(begin(input), end(input), begin(in));1076 copy(begin(expected), end(expected), begin(exp));1077 assert(char_traits<ExternT>::length(in) == 10);1078 assert(char_traits<InternT>::length(exp) == 5);1079 1080 // There are 5 classes of errors in UTF-8 decoding1081 // 1. Missing leading byte1082 // 2. Missing trailing byte1083 // 3. Surrogate CP1084 // 4. Overlong sequence1085 // 5. CP out of Unicode range1086 test_offsets_error<unsigned char> offsets[] = {1087 1088 // 1. Missing leading byte. We will replace the leading byte with1089 // non-leading byte, such as a byte that is always invalid or a trailing1090 // byte.1091 1092 // replace leading byte with invalid byte1093 {1, 5, 0, 0, 0xFF, 0},1094 {3, 5, 1, 1, 0xFF, 1},1095 {6, 5, 3, 2, 0xFF, 3},1096 {10, 5, 6, 3, 0xFF, 6},1097 1098 // replace leading byte with trailing byte1099 {1, 5, 0, 0, 0b10101010, 0},1100 {3, 5, 1, 1, 0b10101010, 1},1101 {6, 5, 3, 2, 0b10101010, 3},1102 {10, 5, 6, 3, 0b10101010, 6},1103 1104 // 2. Missing trailing byte. We will replace the trailing byte with1105 // non-trailing byte, such as a byte that is always invalid or a leading1106 // byte (simple ASCII byte in our case).1107 1108 // replace first trailing byte with ASCII byte1109 {3, 5, 1, 1, 'z', 2},1110 {6, 5, 3, 2, 'z', 4},1111 {10, 5, 6, 3, 'z', 7},1112 1113 // replace first trailing byte with invalid byte1114 {3, 5, 1, 1, 0xFF, 2},1115 {6, 5, 3, 2, 0xFF, 4},1116 {10, 5, 6, 3, 0xFF, 7},1117 1118 // replace second trailing byte with ASCII byte1119 {6, 5, 3, 2, 'z', 5},1120 {10, 5, 6, 3, 'z', 8},1121 1122 // replace second trailing byte with invalid byte1123 {6, 5, 3, 2, 0xFF, 5},1124 {10, 5, 6, 3, 0xFF, 8},1125 1126 // replace third trailing byte1127 {10, 5, 6, 3, 'z', 9},1128 {10, 5, 6, 3, 0xFF, 9},1129 1130 // 2.1 The following test-cases raise doubt whether error or partial should1131 // be returned. For example, we have 4-byte sequence with valid leading1132 // byte. If we hide the last byte we need to return partial. But, if the1133 // second or third byte, which are visible to the call to codecvt, are1134 // malformed then error should be returned.1135 1136 // replace first trailing byte with ASCII byte, also incomplete at end1137 {5, 5, 3, 2, 'z', 4},1138 {8, 5, 6, 3, 'z', 7},1139 {9, 5, 6, 3, 'z', 7},1140 1141 // replace first trailing byte with invalid byte, also incomplete at end1142 {5, 5, 3, 2, 0xFF, 4},1143 {8, 5, 6, 3, 0xFF, 7},1144 {9, 5, 6, 3, 0xFF, 7},1145 1146 // replace second trailing byte with ASCII byte, also incomplete at end1147 {9, 5, 6, 3, 'z', 8},1148 1149 // replace second trailing byte with invalid byte, also incomplete at end1150 {9, 5, 6, 3, 0xFF, 8},1151 1152 // 3. Surrogate CP. We modify the second byte (first trailing) of the 3-byte1153 // CP U+D7001154 {6, 5, 3, 2, 0b10100000, 4}, // turn U+D700 into U+D8001155 {6, 5, 3, 2, 0b10101100, 4}, // turn U+D700 into U+DB001156 {6, 5, 3, 2, 0b10110000, 4}, // turn U+D700 into U+DC001157 {6, 5, 3, 2, 0b10111100, 4}, // turn U+D700 into U+DF001158 1159 // 4. Overlong sequence. The CPs in the input are chosen such as modifying1160 // just the leading byte is enough to make them overlong, i.e. for the1161 // 3-byte and 4-byte CP the second byte (first trailing) has enough leading1162 // zeroes.1163 {3, 5, 1, 1, 0b11000000, 1}, // make the 2-byte CP overlong1164 {3, 5, 1, 1, 0b11000001, 1}, // make the 2-byte CP overlong1165 {6, 5, 3, 2, 0b11100000, 3}, // make the 3-byte CP overlong1166 {10, 5, 6, 3, 0b11110000, 6}, // make the 4-byte CP overlong1167 1168 // 5. CP above range1169 // turn U+10AAAA into U+14AAAA by changing its leading byte1170 {10, 5, 6, 3, 0b11110101, 6},1171 // turn U+10AAAA into U+11AAAA by changing its 2nd byte1172 {10, 5, 6, 3, 0b10011010, 7},1173 // Don't replace anything, show full 4-byte CP U+10AAAA1174 {10, 4, 6, 3, 'b', 0},1175 {10, 5, 6, 3, 'b', 0},1176 // Don't replace anything, show incomplete 4-byte CP at the end. It's still1177 // out of UCS2 range just by seeing the first byte.1178 {7, 4, 6, 3, 'b', 0}, // incomplete fourth CP1179 {8, 4, 6, 3, 'b', 0}, // incomplete fourth CP1180 {9, 4, 6, 3, 'b', 0}, // incomplete fourth CP1181 {7, 5, 6, 3, 'b', 0}, // incomplete fourth CP1182 {8, 5, 6, 3, 'b', 0}, // incomplete fourth CP1183 {9, 5, 6, 3, 'b', 0}, // incomplete fourth CP1184 };1185 for (test_offsets_error<unsigned char>* it = begin(offsets); it != end(offsets); ++it) {1186 test_offsets_error<unsigned char> t = *it;1187 InternT out[array_size(exp) - 1] = {};1188 assert(t.in_size <= array_size(in));1189 assert(t.out_size <= array_size(out));1190 assert(t.expected_in_next <= t.in_size);1191 assert(t.expected_out_next <= t.out_size);1192 ExternT old_char = in[t.replace_pos];1193 in[t.replace_pos] = t.replace_char;1194 1195 mbstate_t state = {};1196 const ExternT* in_next = nullptr;1197 InternT* out_next = nullptr;1198 codecvt_base::result res = codecvt_base::ok;1199 1200 res = cvt.in(state, in, in + t.in_size, in_next, out, out + t.out_size, out_next);1201 assert(res == cvt.error);1202 assert(in_next == in + t.expected_in_next);1203 assert(out_next == out + t.expected_out_next);1204 assert(char_traits<InternT>::compare(out, exp, t.expected_out_next) == 0);1205 if (t.expected_out_next < array_size(out))1206 assert(out[t.expected_out_next] == 0);1207 1208 state = mbstate_t();1209 int len = cvt.length(state, in, in + t.in_size, t.out_size);1210 assert(len >= 0);1211 assert(static_cast<size_t>(len) == t.expected_in_next);1212 1213 in[t.replace_pos] = old_char;1214 }1215}1216 1217template <class InternT, class ExternT>1218void utf8_to_ucs2_in(const std::codecvt<InternT, ExternT, mbstate_t>& cvt) {1219 utf8_to_ucs2_in_ok(cvt);1220 utf8_to_ucs2_in_partial(cvt);1221 utf8_to_ucs2_in_error(cvt);1222}1223 1224template <class InternT, class ExternT>1225void ucs2_to_utf8_out_ok(const std::codecvt<InternT, ExternT, mbstate_t>& cvt) {1226 // UTF-8 string of 1-byte CP, 2-byte CP and 3-byte CP1227 const char16_t input[] = {'b', 0x0448, 0xAAAA, 0};1228 const unsigned char expected[] = "b\u0448\uAAAA";1229 static_assert(array_size(input) == 4, "");1230 static_assert(array_size(expected) == 7, "");1231 1232 InternT in[array_size(input)];1233 ExternT exp[array_size(expected)];1234 copy(begin(input), end(input), begin(in));1235 copy(begin(expected), end(expected), begin(exp));1236 assert(char_traits<InternT>::length(in) == 3);1237 assert(char_traits<ExternT>::length(exp) == 6);1238 1239 test_offsets_ok offsets[] = {{0, 0}, {1, 1}, {2, 3}, {3, 6}};1240 for (test_offsets_ok* it = begin(offsets); it != end(offsets); ++it) {1241 test_offsets_ok t = *it;1242 ExternT out[array_size(exp) - 1] = {};1243 assert(t.in_size <= array_size(in));1244 assert(t.out_size <= array_size(out));1245 mbstate_t state = {};1246 const InternT* in_next = nullptr;1247 ExternT* out_next = nullptr;1248 codecvt_base::result res = codecvt_base::ok;1249 1250 res = cvt.out(state, in, in + t.in_size, in_next, out, out + t.out_size, out_next);1251 assert(res == cvt.ok);1252 assert(in_next == in + t.in_size);1253 assert(out_next == out + t.out_size);1254 assert(char_traits<ExternT>::compare(out, exp, t.out_size) == 0);1255 if (t.out_size < array_size(out))1256 assert(out[t.out_size] == 0);1257 }1258}1259 1260template <class InternT, class ExternT>1261void ucs2_to_utf8_out_partial(const std::codecvt<InternT, ExternT, mbstate_t>& cvt) {1262 // UTF-8 string of 1-byte CP, 2-byte CP and 3-byte CP1263 const char16_t input[] = {'b', 0x0448, 0xAAAA, 0};1264 const unsigned char expected[] = "b\u0448\uAAAA";1265 static_assert(array_size(input) == 4, "");1266 static_assert(array_size(expected) == 7, "");1267 1268 InternT in[array_size(input)];1269 ExternT exp[array_size(expected)];1270 copy(begin(input), end(input), begin(in));1271 copy(begin(expected), end(expected), begin(exp));1272 assert(char_traits<InternT>::length(in) == 3);1273 assert(char_traits<ExternT>::length(exp) == 6);1274 1275 test_offsets_partial offsets[] = {1276 {1, 0, 0, 0}, // no space for first CP1277 1278 {2, 1, 1, 1}, // no space for second CP1279 {2, 2, 1, 1}, // no space for second CP1280 1281 {3, 3, 2, 3}, // no space for third CP1282 {3, 4, 2, 3}, // no space for third CP1283 {3, 5, 2, 3}, // no space for third CP1284 };1285 for (test_offsets_partial* it = begin(offsets); it != end(offsets); ++it) {1286 test_offsets_partial t = *it;1287 ExternT out[array_size(exp) - 1] = {};1288 assert(t.in_size <= array_size(in));1289 assert(t.out_size <= array_size(out));1290 assert(t.expected_in_next <= t.in_size);1291 assert(t.expected_out_next <= t.out_size);1292 mbstate_t state = {};1293 const InternT* in_next = nullptr;1294 ExternT* out_next = nullptr;1295 codecvt_base::result res = codecvt_base::ok;1296 1297 res = cvt.out(state, in, in + t.in_size, in_next, out, out + t.out_size, out_next);1298 assert(res == cvt.partial);1299 assert(in_next == in + t.expected_in_next);1300 assert(out_next == out + t.expected_out_next);1301 assert(char_traits<ExternT>::compare(out, exp, t.expected_out_next) == 0);1302 if (t.expected_out_next < array_size(out))1303 assert(out[t.expected_out_next] == 0);1304 }1305}1306 1307template <class InternT, class ExternT>1308void ucs2_to_utf8_out_error(const std::codecvt<InternT, ExternT, mbstate_t>& cvt) {1309 const char16_t input[] = {'b', 0x0448, 0xAAAA, 0xDBEA, 0xDEAA, 0};1310 const unsigned char expected[] = "b\u0448\uAAAA\U0010AAAA";1311 static_assert(array_size(input) == 6, "");1312 static_assert(array_size(expected) == 11, "");1313 1314 InternT in[array_size(input)];1315 ExternT exp[array_size(expected)];1316 copy(begin(input), end(input), begin(in));1317 copy(begin(expected), end(expected), begin(exp));1318 assert(char_traits<InternT>::length(in) == 5);1319 assert(char_traits<ExternT>::length(exp) == 10);1320 1321 test_offsets_error<InternT> offsets[] = {1322 {3, 6, 0, 0, 0xD800, 0},1323 {3, 6, 0, 0, 0xDBFF, 0},1324 {3, 6, 0, 0, 0xDC00, 0},1325 {3, 6, 0, 0, 0xDFFF, 0},1326 1327 {3, 6, 1, 1, 0xD800, 1},1328 {3, 6, 1, 1, 0xDBFF, 1},1329 {3, 6, 1, 1, 0xDC00, 1},1330 {3, 6, 1, 1, 0xDFFF, 1},1331 1332 {3, 6, 2, 3, 0xD800, 2},1333 {3, 6, 2, 3, 0xDBFF, 2},1334 {3, 6, 2, 3, 0xDC00, 2},1335 {3, 6, 2, 3, 0xDFFF, 2},1336 1337 // make the leading surrogate a trailing one1338 {5, 10, 3, 6, 0xDC00, 3},1339 {5, 10, 3, 6, 0xDFFF, 3},1340 1341 // make the trailing surrogate a leading one1342 {5, 10, 3, 6, 0xD800, 4},1343 {5, 10, 3, 6, 0xDBFF, 4},1344 1345 // make the trailing surrogate a BMP char1346 {5, 10, 3, 6, 'z', 4},1347 1348 // don't replace anything in the test cases bellow, just show the surrogate1349 // pair (fourth CP) fully or partially1350 {5, 10, 3, 6, 'b', 0},1351 {5, 7, 3, 6, 'b', 0}, // no space for fourth CP1352 {5, 8, 3, 6, 'b', 0}, // no space for fourth CP1353 {5, 9, 3, 6, 'b', 0}, // no space for fourth CP1354 1355 {4, 10, 3, 6, 'b', 0}, // incomplete fourth CP1356 {4, 7, 3, 6, 'b', 0}, // incomplete fourth CP, and no space for it1357 {4, 8, 3, 6, 'b', 0}, // incomplete fourth CP, and no space for it1358 {4, 9, 3, 6, 'b', 0}, // incomplete fourth CP, and no space for it1359 };1360 1361 for (test_offsets_error<InternT>* it = begin(offsets); it != end(offsets); ++it) {1362 test_offsets_error<InternT> t = *it;1363 ExternT out[array_size(exp) - 1] = {};1364 assert(t.in_size <= array_size(in));1365 assert(t.out_size <= array_size(out));1366 assert(t.expected_in_next <= t.in_size);1367 assert(t.expected_out_next <= t.out_size);1368 InternT old_char = in[t.replace_pos];1369 in[t.replace_pos] = t.replace_char;1370 1371 mbstate_t state = {};1372 const InternT* in_next = nullptr;1373 ExternT* out_next = nullptr;1374 codecvt_base::result res = codecvt_base::ok;1375 1376 res = cvt.out(state, in, in + t.in_size, in_next, out, out + t.out_size, out_next);1377 assert(res == cvt.error);1378 assert(in_next == in + t.expected_in_next);1379 assert(out_next == out + t.expected_out_next);1380 assert(char_traits<ExternT>::compare(out, exp, t.expected_out_next) == 0);1381 if (t.expected_out_next < array_size(out))1382 assert(out[t.expected_out_next] == 0);1383 1384 in[t.replace_pos] = old_char;1385 }1386}1387 1388template <class InternT, class ExternT>1389void ucs2_to_utf8_out(const std::codecvt<InternT, ExternT, mbstate_t>& cvt) {1390 ucs2_to_utf8_out_ok(cvt);1391 ucs2_to_utf8_out_partial(cvt);1392 ucs2_to_utf8_out_error(cvt);1393}1394 1395template <class InternT, class ExternT>1396void test_utf8_ucs2_cvt(const std::codecvt<InternT, ExternT, mbstate_t>& cvt) {1397 utf8_to_ucs2_in(cvt);1398 ucs2_to_utf8_out(cvt);1399}1400 1401enum utf16_endianess { utf16_big_endian, utf16_little_endian };1402 1403template <class Iter1, class Iter2>1404Iter2 utf16_to_bytes(Iter1 f, Iter1 l, Iter2 o, utf16_endianess e) {1405 if (e == utf16_big_endian)1406 for (; f != l; ++f) {1407 *o++ = (*f >> 8) & 0xFF;1408 *o++ = *f & 0xFF;1409 }1410 else1411 for (; f != l; ++f) {1412 *o++ = *f & 0xFF;1413 *o++ = (*f >> 8) & 0xFF;1414 }1415 return o;1416}1417 1418template <class InternT>1419void utf16_to_utf32_in_ok(const std::codecvt<InternT, char, mbstate_t>& cvt, utf16_endianess endianess) {1420 const char16_t input[] = {'b', 0x0448, 0xAAAA, 0xDBEA, 0xDEAA, 0};1421 const char32_t expected[] = {'b', 0x0448, 0xAAAA, 0x10AAAA, 0};1422 static_assert(array_size(input) == 6, "");1423 static_assert(array_size(expected) == 5, "");1424 1425 char in[array_size(input) * 2];1426 InternT exp[array_size(expected)];1427 utf16_to_bytes(begin(input), end(input), begin(in), endianess);1428 copy(begin(expected), end(expected), begin(exp));1429 1430 test_offsets_ok offsets[] = {{0, 0}, {2, 1}, {4, 2}, {6, 3}, {10, 4}};1431 for (test_offsets_ok* it = begin(offsets); it != end(offsets); ++it) {1432 test_offsets_ok t = *it;1433 InternT out[array_size(exp) - 1] = {};1434 assert(t.in_size <= array_size(in));1435 assert(t.out_size <= array_size(out));1436 mbstate_t state = {};1437 const char* in_next = nullptr;1438 InternT* out_next = nullptr;1439 codecvt_base::result res = codecvt_base::ok;1440 1441 res = cvt.in(state, in, in + t.in_size, in_next, out, out + t.out_size, out_next);1442 assert(res == cvt.ok);1443 assert(in_next == in + t.in_size);1444 assert(out_next == out + t.out_size);1445 assert(char_traits<InternT>::compare(out, exp, t.out_size) == 0);1446 if (t.out_size < array_size(out))1447 assert(out[t.out_size] == 0);1448 1449 state = mbstate_t();1450 int len = cvt.length(state, in, in + t.in_size, t.out_size);1451 assert(len >= 0);1452 assert(static_cast<size_t>(len) == t.in_size);1453 }1454 1455 for (test_offsets_ok* it = begin(offsets); it != end(offsets); ++it) {1456 test_offsets_ok t = *it;1457 InternT out[array_size(exp)] = {};1458 assert(t.in_size <= array_size(in));1459 assert(t.out_size <= array_size(out));1460 mbstate_t state = {};1461 const char* in_next = nullptr;1462 InternT* out_next = nullptr;1463 codecvt_base::result res = codecvt_base::ok;1464 1465 res = cvt.in(state, in, in + t.in_size, in_next, out, end(out), out_next);1466 assert(res == cvt.ok);1467 assert(in_next == in + t.in_size);1468 assert(out_next == out + t.out_size);1469 assert(char_traits<InternT>::compare(out, exp, t.out_size) == 0);1470 if (t.out_size < array_size(out))1471 assert(out[t.out_size] == 0);1472 1473 state = mbstate_t();1474 int len = cvt.length(state, in, in + t.in_size, array_size(out));1475 assert(len >= 0);1476 assert(static_cast<size_t>(len) == t.in_size);1477 }1478}1479 1480template <class InternT>1481void utf16_to_utf32_in_partial(const std::codecvt<InternT, char, mbstate_t>& cvt, utf16_endianess endianess) {1482 const char16_t input[] = {'b', 0x0448, 0xAAAA, 0xDBEA, 0xDEAA, 0};1483 const char32_t expected[] = {'b', 0x0448, 0xAAAA, 0x10AAAA, 0};1484 static_assert(array_size(input) == 6, "");1485 static_assert(array_size(expected) == 5, "");1486 1487 char in[array_size(input) * 2];1488 InternT exp[array_size(expected)];1489 utf16_to_bytes(begin(input), end(input), begin(in), endianess);1490 copy(begin(expected), end(expected), begin(exp));1491 1492 test_offsets_partial offsets[] = {1493 {2, 0, 0, 0}, // no space for first CP1494 {1, 1, 0, 0}, // incomplete first CP1495 {1, 0, 0, 0}, // incomplete first CP, and no space for it1496 1497 {4, 1, 2, 1}, // no space for second CP1498 {3, 2, 2, 1}, // incomplete second CP1499 {3, 1, 2, 1}, // incomplete second CP, and no space for it1500 1501 {6, 2, 4, 2}, // no space for third CP1502 {5, 3, 4, 2}, // incomplete third CP1503 {5, 2, 4, 2}, // incomplete third CP, and no space for it1504 1505 {10, 3, 6, 3}, // no space for fourth CP1506 {7, 4, 6, 3}, // incomplete fourth CP1507 {8, 4, 6, 3}, // incomplete fourth CP1508 {9, 4, 6, 3}, // incomplete fourth CP1509 {7, 3, 6, 3}, // incomplete fourth CP, and no space for it1510 {8, 3, 6, 3}, // incomplete fourth CP, and no space for it1511 {9, 3, 6, 3}, // incomplete fourth CP, and no space for it1512 };1513 1514 for (test_offsets_partial* it = begin(offsets); it != end(offsets); ++it) {1515 test_offsets_partial t = *it;1516 InternT out[array_size(exp) - 1] = {};1517 assert(t.in_size <= array_size(in));1518 assert(t.out_size <= array_size(out));1519 assert(t.expected_in_next <= t.in_size);1520 assert(t.expected_out_next <= t.out_size);1521 mbstate_t state = {};1522 const char* in_next = nullptr;1523 InternT* out_next = nullptr;1524 codecvt_base::result res = codecvt_base::ok;1525 1526 res = cvt.in(state, in, in + t.in_size, in_next, out, out + t.out_size, out_next);1527 assert(res == cvt.partial);1528 assert(in_next == in + t.expected_in_next);1529 assert(out_next == out + t.expected_out_next);1530 assert(char_traits<InternT>::compare(out, exp, t.expected_out_next) == 0);1531 if (t.expected_out_next < array_size(out))1532 assert(out[t.expected_out_next] == 0);1533 1534 state = mbstate_t();1535 int len = cvt.length(state, in, in + t.in_size, t.out_size);1536 assert(len >= 0);1537 assert(static_cast<size_t>(len) == t.expected_in_next);1538 }1539}1540 1541template <class InternT>1542void utf16_to_utf32_in_error(const std::codecvt<InternT, char, mbstate_t>& cvt, utf16_endianess endianess) {1543 char16_t input[] = {'b', 0x0448, 0xAAAA, 0xDBEA, 0xDEAA, 0};1544 const char32_t expected[] = {'b', 0x0448, 0xAAAA, 0x10AAAA, 0};1545 static_assert(array_size(input) == 6, "");1546 static_assert(array_size(expected) == 5, "");1547 1548 InternT exp[array_size(expected)];1549 copy(begin(expected), end(expected), begin(exp));1550 1551 // The only possible error in UTF-16 is unpaired surrogate code units.1552 // So we replace valid code points (scalar values) with lone surrogate CU.1553 test_offsets_error<char16_t> offsets[] = {1554 {10, 4, 0, 0, 0xD800, 0},1555 {10, 4, 0, 0, 0xDBFF, 0},1556 {10, 4, 0, 0, 0xDC00, 0},1557 {10, 4, 0, 0, 0xDFFF, 0},1558 1559 {10, 4, 2, 1, 0xD800, 1},1560 {10, 4, 2, 1, 0xDBFF, 1},1561 {10, 4, 2, 1, 0xDC00, 1},1562 {10, 4, 2, 1, 0xDFFF, 1},1563 1564 {10, 4, 4, 2, 0xD800, 2},1565 {10, 4, 4, 2, 0xDBFF, 2},1566 {10, 4, 4, 2, 0xDC00, 2},1567 {10, 4, 4, 2, 0xDFFF, 2},1568 1569 // make the leading surrogate a trailing one1570 {10, 4, 6, 3, 0xDC00, 3},1571 {10, 4, 6, 3, 0xDFFF, 3},1572 1573 // make the trailing surrogate a leading one1574 {10, 4, 6, 3, 0xD800, 4},1575 {10, 4, 6, 3, 0xDBFF, 4},1576 1577 // make the trailing surrogate a BMP char1578 {10, 4, 6, 3, 'z', 4},1579 };1580 1581 for (test_offsets_error<char16_t>* it = begin(offsets); it != end(offsets); ++it) {1582 test_offsets_error<char16_t> t = *it;1583 char in[array_size(input) * 2];1584 InternT out[array_size(exp) - 1] = {};1585 assert(t.in_size <= array_size(in));1586 assert(t.out_size <= array_size(out));1587 assert(t.expected_in_next <= t.in_size);1588 assert(t.expected_out_next <= t.out_size);1589 char16_t old_char = input[t.replace_pos];1590 input[t.replace_pos] = t.replace_char; // replace in input, not in in1591 utf16_to_bytes(begin(input), end(input), begin(in), endianess);1592 1593 mbstate_t state = {};1594 const char* in_next = nullptr;1595 InternT* out_next = nullptr;1596 codecvt_base::result res = codecvt_base::ok;1597 1598 res = cvt.in(state, in, in + t.in_size, in_next, out, out + t.out_size, out_next);1599 assert(res == cvt.error);1600 assert(in_next == in + t.expected_in_next);1601 assert(out_next == out + t.expected_out_next);1602 assert(char_traits<InternT>::compare(out, exp, t.expected_out_next) == 0);1603 if (t.expected_out_next < array_size(out))1604 assert(out[t.expected_out_next] == 0);1605 1606 state = mbstate_t();1607 int len = cvt.length(state, in, in + t.in_size, t.out_size);1608 assert(len >= 0);1609 assert(static_cast<size_t>(len) == t.expected_in_next);1610 1611 input[t.replace_pos] = old_char;1612 }1613}1614 1615template <class InternT>1616void utf32_to_utf16_out_ok(const std::codecvt<InternT, char, mbstate_t>& cvt, utf16_endianess endianess) {1617 const char32_t input[] = {'b', 0x0448, 0xAAAA, 0x10AAAA, 0};1618 const char16_t expected[] = {'b', 0x0448, 0xAAAA, 0xDBEA, 0xDEAA, 0};1619 static_assert(array_size(input) == 5, "");1620 static_assert(array_size(expected) == 6, "");1621 1622 InternT in[array_size(input)];1623 char exp[array_size(expected) * 2];1624 copy(begin(input), end(input), begin(in));1625 utf16_to_bytes(begin(expected), end(expected), begin(exp), endianess);1626 1627 test_offsets_ok offsets[] = {{0, 0}, {1, 2}, {2, 4}, {3, 6}, {4, 10}};1628 for (test_offsets_ok* it = begin(offsets); it != end(offsets); ++it) {1629 test_offsets_ok t = *it;1630 char out[array_size(exp) - 2] = {};1631 assert(t.in_size <= array_size(in));1632 assert(t.out_size <= array_size(out));1633 mbstate_t state = {};1634 const InternT* in_next = nullptr;1635 char* out_next = nullptr;1636 codecvt_base::result res = codecvt_base::ok;1637 1638 res = cvt.out(state, in, in + t.in_size, in_next, out, out + t.out_size, out_next);1639 assert(res == cvt.ok);1640 assert(in_next == in + t.in_size);1641 assert(out_next == out + t.out_size);1642 assert(char_traits<char>::compare(out, exp, t.out_size) == 0);1643 if (t.out_size < array_size(out))1644 assert(out[t.out_size] == 0);1645 }1646}1647 1648template <class InternT>1649void utf32_to_utf16_out_partial(const std::codecvt<InternT, char, mbstate_t>& cvt, utf16_endianess endianess) {1650 const char32_t input[] = {'b', 0x0448, 0xAAAA, 0x10AAAA, 0};1651 const char16_t expected[] = {'b', 0x0448, 0xAAAA, 0xDBEA, 0xDEAA, 0};1652 static_assert(array_size(input) == 5, "");1653 static_assert(array_size(expected) == 6, "");1654 1655 InternT in[array_size(input)];1656 char exp[array_size(expected) * 2];1657 copy(begin(input), end(input), begin(in));1658 utf16_to_bytes(begin(expected), end(expected), begin(exp), endianess);1659 1660 test_offsets_partial offsets[] = {1661 {1, 0, 0, 0}, // no space for first CP1662 {1, 1, 0, 0}, // no space for first CP1663 1664 {2, 2, 1, 2}, // no space for second CP1665 {2, 3, 1, 2}, // no space for second CP1666 1667 {3, 4, 2, 4}, // no space for third CP1668 {3, 5, 2, 4}, // no space for third CP1669 1670 {4, 6, 3, 6}, // no space for fourth CP1671 {4, 7, 3, 6}, // no space for fourth CP1672 {4, 8, 3, 6}, // no space for fourth CP1673 {4, 9, 3, 6}, // no space for fourth CP1674 };1675 for (test_offsets_partial* it = begin(offsets); it != end(offsets); ++it) {1676 test_offsets_partial t = *it;1677 char out[array_size(exp) - 2] = {};1678 assert(t.in_size <= array_size(in));1679 assert(t.out_size <= array_size(out));1680 assert(t.expected_in_next <= t.in_size);1681 assert(t.expected_out_next <= t.out_size);1682 mbstate_t state = {};1683 const InternT* in_next = nullptr;1684 char* out_next = nullptr;1685 codecvt_base::result res = codecvt_base::ok;1686 1687 res = cvt.out(state, in, in + t.in_size, in_next, out, out + t.out_size, out_next);1688 assert(res == cvt.partial);1689 assert(in_next == in + t.expected_in_next);1690 assert(out_next == out + t.expected_out_next);1691 assert(char_traits<char>::compare(out, exp, t.expected_out_next) == 0);1692 if (t.expected_out_next < array_size(out))1693 assert(out[t.expected_out_next] == 0);1694 }1695}1696 1697template <class InternT>1698void utf32_to_utf16_out_error(const std::codecvt<InternT, char, mbstate_t>& cvt, utf16_endianess endianess) {1699 const char32_t input[] = {'b', 0x0448, 0xAAAA, 0x10AAAA, 0};1700 const char16_t expected[] = {'b', 0x0448, 0xAAAA, 0xDBEA, 0xDEAA, 0};1701 static_assert(array_size(input) == 5, "");1702 static_assert(array_size(expected) == 6, "");1703 1704 InternT in[array_size(input)];1705 char exp[array_size(expected) * 2];1706 copy(begin(input), end(input), begin(in));1707 utf16_to_bytes(begin(expected), end(expected), begin(exp), endianess);1708 1709 test_offsets_error<InternT> offsets[] = {1710 1711 // Surrogate CP1712 {4, 10, 0, 0, 0xD800, 0},1713 {4, 10, 1, 2, 0xDBFF, 1},1714 {4, 10, 2, 4, 0xDC00, 2},1715 {4, 10, 3, 6, 0xDFFF, 3},1716 1717 // CP out of range1718 {4, 10, 0, 0, 0x00110000, 0},1719 {4, 10, 1, 2, 0x00110000, 1},1720 {4, 10, 2, 4, 0x00110000, 2},1721 {4, 10, 3, 6, 0x00110000, 3}};1722 1723 for (test_offsets_error<InternT>* it = begin(offsets); it != end(offsets); ++it) {1724 test_offsets_error<InternT> t = *it;1725 char out[array_size(exp) - 2] = {};1726 assert(t.in_size <= array_size(in));1727 assert(t.out_size <= array_size(out));1728 assert(t.expected_in_next <= t.in_size);1729 assert(t.expected_out_next <= t.out_size);1730 InternT old_char = in[t.replace_pos];1731 in[t.replace_pos] = t.replace_char;1732 1733 mbstate_t state = {};1734 const InternT* in_next = nullptr;1735 char* out_next = nullptr;1736 codecvt_base::result res = codecvt_base::ok;1737 1738 res = cvt.out(state, in, in + t.in_size, in_next, out, out + t.out_size, out_next);1739 assert(res == cvt.error);1740 assert(in_next == in + t.expected_in_next);1741 assert(out_next == out + t.expected_out_next);1742 assert(char_traits<char>::compare(out, exp, t.expected_out_next) == 0);1743 if (t.expected_out_next < array_size(out))1744 assert(out[t.expected_out_next] == 0);1745 1746 in[t.replace_pos] = old_char;1747 }1748}1749 1750template <class InternT>1751void test_utf16_utf32_cvt(const std::codecvt<InternT, char, mbstate_t>& cvt, utf16_endianess endianess) {1752 utf16_to_utf32_in_ok(cvt, endianess);1753 utf16_to_utf32_in_partial(cvt, endianess);1754 utf16_to_utf32_in_error(cvt, endianess);1755 utf32_to_utf16_out_ok(cvt, endianess);1756 utf32_to_utf16_out_partial(cvt, endianess);1757 utf32_to_utf16_out_error(cvt, endianess);1758}1759 1760template <class InternT>1761void utf16_to_ucs2_in_ok(const std::codecvt<InternT, char, mbstate_t>& cvt, utf16_endianess endianess) {1762 const char16_t input[] = {'b', 0x0448, 0xAAAA, 0};1763 const char16_t expected[] = {'b', 0x0448, 0xAAAA, 0};1764 static_assert(array_size(input) == 4, "");1765 static_assert(array_size(expected) == 4, "");1766 1767 char in[array_size(input) * 2];1768 InternT exp[array_size(expected)];1769 utf16_to_bytes(begin(input), end(input), begin(in), endianess);1770 copy(begin(expected), end(expected), begin(exp));1771 1772 test_offsets_ok offsets[] = {{0, 0}, {2, 1}, {4, 2}, {6, 3}};1773 for (test_offsets_ok* it = begin(offsets); it != end(offsets); ++it) {1774 test_offsets_ok t = *it;1775 InternT out[array_size(exp) - 1] = {};1776 assert(t.in_size <= array_size(in));1777 assert(t.out_size <= array_size(out));1778 mbstate_t state = {};1779 const char* in_next = nullptr;1780 InternT* out_next = nullptr;1781 codecvt_base::result res = codecvt_base::ok;1782 1783 res = cvt.in(state, in, in + t.in_size, in_next, out, out + t.out_size, out_next);1784 assert(res == cvt.ok);1785 assert(in_next == in + t.in_size);1786 assert(out_next == out + t.out_size);1787 assert(char_traits<InternT>::compare(out, exp, t.out_size) == 0);1788 if (t.out_size < array_size(out))1789 assert(out[t.out_size] == 0);1790 1791 state = mbstate_t();1792 int len = cvt.length(state, in, in + t.in_size, t.out_size);1793 assert(len >= 0);1794 assert(static_cast<size_t>(len) == t.in_size);1795 }1796 1797 for (test_offsets_ok* it = begin(offsets); it != end(offsets); ++it) {1798 test_offsets_ok t = *it;1799 InternT out[array_size(exp)] = {};1800 assert(t.in_size <= array_size(in));1801 assert(t.out_size <= array_size(out));1802 mbstate_t state = {};1803 const char* in_next = nullptr;1804 InternT* out_next = nullptr;1805 codecvt_base::result res = codecvt_base::ok;1806 1807 res = cvt.in(state, in, in + t.in_size, in_next, out, end(out), out_next);1808 assert(res == cvt.ok);1809 assert(in_next == in + t.in_size);1810 assert(out_next == out + t.out_size);1811 assert(char_traits<InternT>::compare(out, exp, t.out_size) == 0);1812 if (t.out_size < array_size(out))1813 assert(out[t.out_size] == 0);1814 1815 state = mbstate_t();1816 int len = cvt.length(state, in, in + t.in_size, array_size(out));1817 assert(len >= 0);1818 assert(static_cast<size_t>(len) == t.in_size);1819 }1820}1821 1822template <class InternT>1823void utf16_to_ucs2_in_partial(const std::codecvt<InternT, char, mbstate_t>& cvt, utf16_endianess endianess) {1824 const char16_t input[] = {'b', 0x0448, 0xAAAA, 0};1825 const char16_t expected[] = {'b', 0x0448, 0xAAAA, 0};1826 static_assert(array_size(input) == 4, "");1827 static_assert(array_size(expected) == 4, "");1828 1829 char in[array_size(input) * 2];1830 InternT exp[array_size(expected)];1831 utf16_to_bytes(begin(input), end(input), begin(in), endianess);1832 copy(begin(expected), end(expected), begin(exp));1833 1834 test_offsets_partial offsets[] = {1835 {2, 0, 0, 0}, // no space for first CP1836 {1, 1, 0, 0}, // incomplete first CP1837 {1, 0, 0, 0}, // incomplete first CP, and no space for it1838 1839 {4, 1, 2, 1}, // no space for second CP1840 {3, 2, 2, 1}, // incomplete second CP1841 {3, 1, 2, 1}, // incomplete second CP, and no space for it1842 1843 {6, 2, 4, 2}, // no space for third CP1844 {5, 3, 4, 2}, // incomplete third CP1845 {5, 2, 4, 2}, // incomplete third CP, and no space for it1846 };1847 1848 for (test_offsets_partial* it = begin(offsets); it != end(offsets); ++it) {1849 test_offsets_partial t = *it;1850 InternT out[array_size(exp) - 1] = {};1851 assert(t.in_size <= array_size(in));1852 assert(t.out_size <= array_size(out));1853 assert(t.expected_in_next <= t.in_size);1854 assert(t.expected_out_next <= t.out_size);1855 mbstate_t state = {};1856 const char* in_next = nullptr;1857 InternT* out_next = nullptr;1858 codecvt_base::result res = codecvt_base::ok;1859 1860 res = cvt.in(state, in, in + t.in_size, in_next, out, out + t.out_size, out_next);1861 assert(res == cvt.partial);1862 assert(in_next == in + t.expected_in_next);1863 assert(out_next == out + t.expected_out_next);1864 assert(char_traits<InternT>::compare(out, exp, t.expected_out_next) == 0);1865 if (t.expected_out_next < array_size(out))1866 assert(out[t.expected_out_next] == 0);1867 1868 state = mbstate_t();1869 int len = cvt.length(state, in, in + t.in_size, t.out_size);1870 assert(len >= 0);1871 assert(static_cast<size_t>(len) == t.expected_in_next);1872 }1873}1874 1875template <class InternT>1876void utf16_to_ucs2_in_error(const std::codecvt<InternT, char, mbstate_t>& cvt, utf16_endianess endianess) {1877 char16_t input[] = {'b', 0x0448, 0xAAAA, 0xDBEA, 0xDEAA, 0};1878 const char16_t expected[] = {'b', 0x0448, 0xAAAA, 0xDBEA, 0xDEAA, 0};1879 static_assert(array_size(input) == 6, "");1880 static_assert(array_size(expected) == 6, "");1881 1882 InternT exp[array_size(expected)];1883 copy(begin(expected), end(expected), begin(exp));1884 1885 // The only possible error in UTF-16 is unpaired surrogate code units.1886 // Additionally, because the target encoding is UCS-2, a proper pair of1887 // surrogates is also error. Simply, any surrogate CU is error.1888 test_offsets_error<char16_t> offsets[] = {1889 {6, 3, 0, 0, 0xD800, 0},1890 {6, 3, 0, 0, 0xDBFF, 0},1891 {6, 3, 0, 0, 0xDC00, 0},1892 {6, 3, 0, 0, 0xDFFF, 0},1893 1894 {6, 3, 2, 1, 0xD800, 1},1895 {6, 3, 2, 1, 0xDBFF, 1},1896 {6, 3, 2, 1, 0xDC00, 1},1897 {6, 3, 2, 1, 0xDFFF, 1},1898 1899 {6, 3, 4, 2, 0xD800, 2},1900 {6, 3, 4, 2, 0xDBFF, 2},1901 {6, 3, 4, 2, 0xDC00, 2},1902 {6, 3, 4, 2, 0xDFFF, 2},1903 1904 // make the leading surrogate a trailing one1905 {10, 5, 6, 3, 0xDC00, 3},1906 {10, 5, 6, 3, 0xDFFF, 3},1907 1908 // make the trailing surrogate a leading one1909 {10, 5, 6, 3, 0xD800, 4},1910 {10, 5, 6, 3, 0xDBFF, 4},1911 1912 // make the trailing surrogate a BMP char1913 {10, 5, 6, 3, 'z', 4},1914 1915 // don't replace anything in the test cases bellow, just show the surrogate1916 // pair (fourth CP) fully or partially (just the first surrogate)1917 {10, 5, 6, 3, 'b', 0},1918 {8, 5, 6, 3, 'b', 0},1919 {9, 5, 6, 3, 'b', 0},1920 1921 {10, 4, 6, 3, 'b', 0},1922 {8, 4, 6, 3, 'b', 0},1923 {9, 4, 6, 3, 'b', 0},1924 };1925 1926 for (test_offsets_error<char16_t>* it = begin(offsets); it != end(offsets); ++it) {1927 test_offsets_error<char16_t> t = *it;1928 char in[array_size(input) * 2];1929 InternT out[array_size(exp) - 1] = {};1930 assert(t.in_size <= array_size(in));1931 assert(t.out_size <= array_size(out));1932 assert(t.expected_in_next <= t.in_size);1933 assert(t.expected_out_next <= t.out_size);1934 char16_t old_char = input[t.replace_pos];1935 input[t.replace_pos] = t.replace_char; // replace in input, not in in1936 utf16_to_bytes(begin(input), end(input), begin(in), endianess);1937 1938 mbstate_t state = {};1939 const char* in_next = nullptr;1940 InternT* out_next = nullptr;1941 codecvt_base::result res = codecvt_base::ok;1942 1943 res = cvt.in(state, in, in + t.in_size, in_next, out, out + t.out_size, out_next);1944 assert(res == cvt.error);1945 assert(in_next == in + t.expected_in_next);1946 assert(out_next == out + t.expected_out_next);1947 assert(char_traits<InternT>::compare(out, exp, t.expected_out_next) == 0);1948 if (t.expected_out_next < array_size(out))1949 assert(out[t.expected_out_next] == 0);1950 1951 state = mbstate_t();1952 int len = cvt.length(state, in, in + t.in_size, t.out_size);1953 assert(len >= 0);1954 assert(static_cast<size_t>(len) == t.expected_in_next);1955 1956 input[t.replace_pos] = old_char;1957 }1958}1959 1960template <class InternT>1961void ucs2_to_utf16_out_ok(const std::codecvt<InternT, char, mbstate_t>& cvt, utf16_endianess endianess) {1962 const char16_t input[] = {'b', 0x0448, 0xAAAA, 0};1963 const char16_t expected[] = {'b', 0x0448, 0xAAAA, 0};1964 static_assert(array_size(input) == 4, "");1965 static_assert(array_size(expected) == 4, "");1966 1967 InternT in[array_size(input)];1968 char exp[array_size(expected) * 2];1969 copy(begin(input), end(input), begin(in));1970 utf16_to_bytes(begin(expected), end(expected), begin(exp), endianess);1971 1972 test_offsets_ok offsets[] = {{0, 0}, {1, 2}, {2, 4}, {3, 6}};1973 for (test_offsets_ok* it = begin(offsets); it != end(offsets); ++it) {1974 test_offsets_ok t = *it;1975 char out[array_size(exp) - 2] = {};1976 assert(t.in_size <= array_size(in));1977 assert(t.out_size <= array_size(out));1978 mbstate_t state = {};1979 const InternT* in_next = nullptr;1980 char* out_next = nullptr;1981 codecvt_base::result res = codecvt_base::ok;1982 1983 res = cvt.out(state, in, in + t.in_size, in_next, out, out + t.out_size, out_next);1984 assert(res == cvt.ok);1985 assert(in_next == in + t.in_size);1986 assert(out_next == out + t.out_size);1987 assert(char_traits<char>::compare(out, exp, t.out_size) == 0);1988 if (t.out_size < array_size(out))1989 assert(out[t.out_size] == 0);1990 }1991}1992 1993template <class InternT>1994void ucs2_to_utf16_out_partial(const std::codecvt<InternT, char, mbstate_t>& cvt, utf16_endianess endianess) {1995 const char16_t input[] = {'b', 0x0448, 0xAAAA, 0};1996 const char16_t expected[] = {'b', 0x0448, 0xAAAA, 0};1997 static_assert(array_size(input) == 4, "");1998 static_assert(array_size(expected) == 4, "");1999 2000 InternT in[array_size(input)];2001 char exp[array_size(expected) * 2];2002 copy(begin(input), end(input), begin(in));2003 utf16_to_bytes(begin(expected), end(expected), begin(exp), endianess);2004 2005 test_offsets_partial offsets[] = {2006 {1, 0, 0, 0}, // no space for first CP2007 {1, 1, 0, 0}, // no space for first CP2008 2009 {2, 2, 1, 2}, // no space for second CP2010 {2, 3, 1, 2}, // no space for second CP2011 2012 {3, 4, 2, 4}, // no space for third CP2013 {3, 5, 2, 4}, // no space for third CP2014 };2015 for (test_offsets_partial* it = begin(offsets); it != end(offsets); ++it) {2016 test_offsets_partial t = *it;2017 char out[array_size(exp) - 2] = {};2018 assert(t.in_size <= array_size(in));2019 assert(t.out_size <= array_size(out));2020 assert(t.expected_in_next <= t.in_size);2021 assert(t.expected_out_next <= t.out_size);2022 mbstate_t state = {};2023 const InternT* in_next = nullptr;2024 char* out_next = nullptr;2025 codecvt_base::result res = codecvt_base::ok;2026 2027 res = cvt.out(state, in, in + t.in_size, in_next, out, out + t.out_size, out_next);2028 assert(res == cvt.partial);2029 assert(in_next == in + t.expected_in_next);2030 assert(out_next == out + t.expected_out_next);2031 assert(char_traits<char>::compare(out, exp, t.expected_out_next) == 0);2032 if (t.expected_out_next < array_size(out))2033 assert(out[t.expected_out_next] == 0);2034 }2035}2036 2037template <class InternT>2038void ucs2_to_utf16_out_error(const std::codecvt<InternT, char, mbstate_t>& cvt, utf16_endianess endianess) {2039 const char16_t input[] = {'b', 0x0448, 0xAAAA, 0xDBEA, 0xDEAA, 0};2040 const char16_t expected[] = {'b', 0x0448, 0xAAAA, 0xDBEA, 0xDEAA, 0};2041 static_assert(array_size(input) == 6, "");2042 static_assert(array_size(expected) == 6, "");2043 2044 InternT in[array_size(input)];2045 char exp[array_size(expected) * 2];2046 copy(begin(input), end(input), begin(in));2047 utf16_to_bytes(begin(expected), end(expected), begin(exp), endianess);2048 2049 test_offsets_error<InternT> offsets[] = {2050 {3, 6, 0, 0, 0xD800, 0},2051 {3, 6, 0, 0, 0xDBFF, 0},2052 {3, 6, 0, 0, 0xDC00, 0},2053 {3, 6, 0, 0, 0xDFFF, 0},2054 2055 {3, 6, 1, 2, 0xD800, 1},2056 {3, 6, 1, 2, 0xDBFF, 1},2057 {3, 6, 1, 2, 0xDC00, 1},2058 {3, 6, 1, 2, 0xDFFF, 1},2059 2060 {3, 6, 2, 4, 0xD800, 2},2061 {3, 6, 2, 4, 0xDBFF, 2},2062 {3, 6, 2, 4, 0xDC00, 2},2063 {3, 6, 2, 4, 0xDFFF, 2},2064 2065 // make the leading surrogate a trailing one2066 {5, 10, 3, 6, 0xDC00, 3},2067 {5, 10, 3, 6, 0xDFFF, 3},2068 2069 // make the trailing surrogate a leading one2070 {5, 10, 3, 6, 0xD800, 4},2071 {5, 10, 3, 6, 0xDBFF, 4},2072 2073 // make the trailing surrogate a BMP char2074 {5, 10, 3, 6, 'z', 4},2075 2076 // don't replace anything in the test cases bellow, just show the surrogate2077 // pair (fourth CP) fully or partially (just the first surrogate)2078 {5, 10, 3, 6, 'b', 0},2079 {5, 8, 3, 6, 'b', 0},2080 {5, 9, 3, 6, 'b', 0},2081 2082 {4, 10, 3, 6, 'b', 0},2083 {4, 8, 3, 6, 'b', 0},2084 {4, 9, 3, 6, 'b', 0},2085 };2086 2087 for (test_offsets_error<InternT>* it = begin(offsets); it != end(offsets); ++it) {2088 test_offsets_error<InternT> t = *it;2089 char out[array_size(exp) - 2] = {};2090 assert(t.in_size <= array_size(in));2091 assert(t.out_size <= array_size(out));2092 assert(t.expected_in_next <= t.in_size);2093 assert(t.expected_out_next <= t.out_size);2094 InternT old_char = in[t.replace_pos];2095 in[t.replace_pos] = t.replace_char;2096 2097 mbstate_t state = {};2098 const InternT* in_next = nullptr;2099 char* out_next = nullptr;2100 codecvt_base::result res = codecvt_base::ok;2101 2102 res = cvt.out(state, in, in + t.in_size, in_next, out, out + t.out_size, out_next);2103 assert(res == cvt.error);2104 assert(in_next == in + t.expected_in_next);2105 assert(out_next == out + t.expected_out_next);2106 assert(char_traits<char>::compare(out, exp, t.expected_out_next) == 0);2107 if (t.expected_out_next < array_size(out))2108 assert(out[t.expected_out_next] == 0);2109 2110 in[t.replace_pos] = old_char;2111 }2112}2113 2114template <class InternT>2115void test_utf16_ucs2_cvt(const std::codecvt<InternT, char, mbstate_t>& cvt, utf16_endianess endianess) {2116 utf16_to_ucs2_in_ok(cvt, endianess);2117 utf16_to_ucs2_in_partial(cvt, endianess);2118 utf16_to_ucs2_in_error(cvt, endianess);2119 ucs2_to_utf16_out_ok(cvt, endianess);2120 ucs2_to_utf16_out_partial(cvt, endianess);2121 ucs2_to_utf16_out_error(cvt, endianess);2122}2123 2124using std::codecvt;2125using std::codecvt_utf16;2126using std::codecvt_utf8;2127using std::codecvt_utf8_utf16;2128using std::has_facet;2129using std::locale;2130using std::use_facet;2131 2132void test_utf8_utf32_codecvts() {2133 typedef codecvt<char32_t, char, mbstate_t> codecvt_c32;2134 const locale& loc_c = locale::classic();2135 assert(has_facet<codecvt_c32>(loc_c));2136 2137 const codecvt_c32& cvt = use_facet<codecvt_c32>(loc_c);2138 test_utf8_utf32_cvt(cvt);2139 2140 codecvt_utf8<char32_t> cvt2;2141 test_utf8_utf32_cvt(cvt2);2142 2143#if !defined(TEST_HAS_NO_WIDE_CHARACTERS) && !defined(TEST_SHORT_WCHAR)2144 codecvt_utf8<wchar_t> cvt3;2145 test_utf8_utf32_cvt(cvt3);2146#endif2147 2148#ifndef TEST_HAS_NO_CHAR8_T2149 typedef codecvt<char32_t, char8_t, mbstate_t> codecvt_c32_c8;2150 assert(has_facet<codecvt_c32_c8>(loc_c));2151 const codecvt_c32_c8& cvt4 = use_facet<codecvt_c32_c8>(loc_c);2152 test_utf8_utf32_cvt(cvt4);2153#endif2154}2155 2156void test_utf8_utf16_codecvts() {2157 typedef codecvt<char16_t, char, mbstate_t> codecvt_c16;2158 const locale& loc_c = locale::classic();2159 assert(has_facet<codecvt_c16>(loc_c));2160 2161 const codecvt_c16& cvt = use_facet<codecvt_c16>(loc_c);2162 test_utf8_utf16_cvt(cvt);2163 2164 codecvt_utf8_utf16<char16_t> cvt2;2165 test_utf8_utf16_cvt(cvt2);2166 2167 codecvt_utf8_utf16<char32_t> cvt3;2168 test_utf8_utf16_cvt(cvt3);2169 2170#ifndef TEST_HAS_NO_WIDE_CHARACTERS2171 codecvt_utf8_utf16<wchar_t> cvt4;2172 test_utf8_utf16_cvt(cvt4);2173#endif2174 2175#ifndef TEST_HAS_NO_CHAR8_T2176 typedef codecvt<char16_t, char8_t, mbstate_t> codecvt_c16_c8;2177 assert(has_facet<codecvt_c16_c8>(loc_c));2178 const codecvt_c16_c8& cvt5 = use_facet<codecvt_c16_c8>(loc_c);2179 test_utf8_utf16_cvt(cvt5);2180#endif2181}2182 2183void test_utf8_ucs2_codecvts() {2184 codecvt_utf8<char16_t> cvt;2185 test_utf8_ucs2_cvt(cvt);2186 2187#if !defined(TEST_HAS_NO_WIDE_CHARACTERS) && defined(TEST_SHORT_WCHAR)2188 codecvt_utf8<wchar_t> cvt2;2189 test_utf8_ucs2_cvt(cvt2);2190#endif2191}2192 2193void test_utf16_utf32_codecvts() {2194 codecvt_utf16<char32_t> cvt;2195 test_utf16_utf32_cvt(cvt, utf16_big_endian);2196 2197 codecvt_utf16<char32_t, 0x10FFFF, std::little_endian> cvt2;2198 test_utf16_utf32_cvt(cvt2, utf16_little_endian);2199 2200#if !defined(TEST_HAS_NO_WIDE_CHARACTERS) && !defined(TEST_SHORT_WCHAR)2201 codecvt_utf16<wchar_t> cvt3;2202 test_utf16_utf32_cvt(cvt3, utf16_big_endian);2203 2204 codecvt_utf16<wchar_t, 0x10FFFF, std::little_endian> cvt4;2205 test_utf16_utf32_cvt(cvt4, utf16_little_endian);2206#endif2207}2208 2209void test_utf16_ucs2_codecvts() {2210 codecvt_utf16<char16_t> cvt;2211 test_utf16_ucs2_cvt(cvt, utf16_big_endian);2212 2213 codecvt_utf16<char16_t, 0x10FFFF, std::little_endian> cvt2;2214 test_utf16_ucs2_cvt(cvt2, utf16_little_endian);2215 2216#if !defined(TEST_HAS_NO_WIDE_CHARACTERS) && defined(TEST_SHORT_WCHAR)2217 codecvt_utf16<wchar_t> cvt3;2218 test_utf16_ucs2_cvt(cvt3, utf16_big_endian);2219 2220 codecvt_utf16<wchar_t, 0x10FFFF, std::little_endian> cvt4;2221 test_utf16_ucs2_cvt(cvt4, utf16_little_endian);2222#endif2223}2224 2225int main(int, char**) {2226 test_utf8_utf32_codecvts();2227 test_utf8_utf16_codecvts();2228 test_utf8_ucs2_codecvts();2229 test_utf16_utf32_codecvts();2230 test_utf16_ucs2_codecvts();2231 return 0;2232}2233