375 lines · cpp
1//===-- Unittests for op_ files -------------------------------------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8 9#include "memory_check_utils.h"10#include "src/__support/macros/config.h"11#include "src/__support/macros/properties/os.h"12#include "src/__support/macros/properties/types.h" // LIBC_TYPES_HAS_INT6413#include "src/string/memory_utils/op_aarch64.h"14#include "src/string/memory_utils/op_builtin.h"15#include "src/string/memory_utils/op_generic.h"16#include "src/string/memory_utils/op_riscv.h"17#include "src/string/memory_utils/op_x86.h"18#include "test/UnitTest/Test.h"19 20namespace LIBC_NAMESPACE_DECL {21 22template <typename T> struct has_head_tail {23 template <typename C> static char sfinae(decltype(&C::head_tail));24 template <typename C> static uint16_t sfinae(...);25 static constexpr bool value = sizeof(sfinae<T>(0)) == sizeof(char);26};27 28template <typename T> struct has_loop_and_tail {29 template <typename C> static char sfinae(decltype(&C::loop_and_tail));30 template <typename C> static uint16_t sfinae(...);31 static constexpr bool value = sizeof(sfinae<T>(0)) == sizeof(char);32};33 34// Allocates two Buffer and extracts two spans out of them, one35// aligned and one misaligned. Tests are run on both spans.36struct Buffers {37 Buffers(size_t size)38 : aligned_buffer(size, Aligned::YES),39 misaligned_buffer(size, Aligned::NO) {}40 41 // Returns two spans of 'size' bytes. The first is aligned on42 // Buffer::kAlign and the second one is unaligned.43 cpp::array<cpp::span<char>, 2> spans() {44 return {aligned_buffer.span(), misaligned_buffer.span()};45 }46 47 Buffer aligned_buffer;48 Buffer misaligned_buffer;49};50 51using MemcpyImplementations = testing::TypeList<52#ifdef LLVM_LIBC_HAS_BUILTIN_MEMCPY_INLINE53 builtin::Memcpy<1>, //54 builtin::Memcpy<2>, //55 builtin::Memcpy<3>, //56 builtin::Memcpy<4>, //57 builtin::Memcpy<8>, //58 builtin::Memcpy<16>, //59 builtin::Memcpy<32>, //60 builtin::Memcpy<64>61#endif // LLVM_LIBC_HAS_BUILTIN_MEMCPY_INLINE62 >;63 64// Convenient helper to turn a span into cpp::byte *.65static inline cpp::byte *as_byte(cpp::span<char> span) {66 return reinterpret_cast<cpp::byte *>(span.data());67}68 69// Adapt CheckMemcpy signature to op implementation signatures.70template <auto FnImpl>71void CopyAdaptor(cpp::span<char> dst, cpp::span<char> src, size_t size) {72 FnImpl(as_byte(dst), as_byte(src), size);73}74template <size_t Size, auto FnImpl>75void CopyBlockAdaptor(cpp::span<char> dst, cpp::span<char> src,76 [[maybe_unused]] size_t size) {77 FnImpl(as_byte(dst), as_byte(src));78}79 80TYPED_TEST(LlvmLibcOpTest, Memcpy, MemcpyImplementations) {81 using Impl = ParamType;82 constexpr size_t kSize = Impl::SIZE;83 { // Test block operation84 static constexpr auto BlockImpl = CopyBlockAdaptor<kSize, Impl::block>;85 Buffers SrcBuffer(kSize);86 Buffers DstBuffer(kSize);87 for (auto src : SrcBuffer.spans()) {88 Randomize(src);89 for (auto dst : DstBuffer.spans()) {90 ASSERT_TRUE(CheckMemcpy<BlockImpl>(dst, src, kSize));91 }92 }93 }94 { // Test head tail operations from kSize to 2 * kSize.95 static constexpr auto HeadTailImpl = CopyAdaptor<Impl::head_tail>;96 Buffer SrcBuffer(2 * kSize);97 Buffer DstBuffer(2 * kSize);98 Randomize(SrcBuffer.span());99 for (size_t size = kSize; size < 2 * kSize; ++size) {100 auto src = SrcBuffer.span().subspan(0, size);101 auto dst = DstBuffer.span().subspan(0, size);102 ASSERT_TRUE(CheckMemcpy<HeadTailImpl>(dst, src, size));103 }104 }105 { // Test loop operations from kSize to 3 * kSize.106 if constexpr (kSize > 1) {107 static constexpr auto LoopImpl = CopyAdaptor<Impl::loop_and_tail>;108 Buffer SrcBuffer(3 * kSize);109 Buffer DstBuffer(3 * kSize);110 Randomize(SrcBuffer.span());111 for (size_t size = kSize; size < 3 * kSize; ++size) {112 auto src = SrcBuffer.span().subspan(0, size);113 auto dst = DstBuffer.span().subspan(0, size);114 ASSERT_TRUE(CheckMemcpy<LoopImpl>(dst, src, size));115 }116 }117 }118}119 120using MemsetImplementations = testing::TypeList<121#ifdef LLVM_LIBC_HAS_BUILTIN_MEMSET_INLINE122 builtin::Memset<1>, //123 builtin::Memset<2>, //124 builtin::Memset<3>, //125 builtin::Memset<4>, //126 builtin::Memset<8>, //127 builtin::Memset<16>, //128 builtin::Memset<32>, //129 builtin::Memset<64>,130#endif131#ifdef LIBC_TYPES_HAS_INT64132 generic::Memset<uint64_t>, generic::Memset<cpp::array<uint64_t, 2>>,133#endif // LIBC_TYPES_HAS_INT64134#ifdef __AVX512F__135 generic::Memset<generic_v512>, generic::Memset<cpp::array<generic_v512, 2>>,136#endif137#ifdef __AVX__138 generic::Memset<generic_v256>, generic::Memset<cpp::array<generic_v256, 2>>,139#endif140#ifdef __SSE2__141 generic::Memset<generic_v128>, generic::Memset<cpp::array<generic_v128, 2>>,142#endif143 generic::Memset<uint32_t>, generic::Memset<cpp::array<uint32_t, 2>>, //144 generic::Memset<uint16_t>, generic::Memset<cpp::array<uint16_t, 2>>, //145 generic::Memset<uint8_t>, generic::Memset<cpp::array<uint8_t, 2>>, //146 generic::MemsetSequence<uint8_t, uint8_t>, //147 generic::MemsetSequence<uint16_t, uint8_t>, //148 generic::MemsetSequence<uint32_t, uint16_t, uint8_t> //149 >;150 151// Adapt CheckMemset signature to op implementation signatures.152template <auto FnImpl>153void SetAdaptor(cpp::span<char> dst, uint8_t value, size_t size) {154 FnImpl(as_byte(dst), value, size);155}156template <size_t Size, auto FnImpl>157void SetBlockAdaptor(cpp::span<char> dst, uint8_t value,158 [[maybe_unused]] size_t size) {159 FnImpl(as_byte(dst), value);160}161 162TYPED_TEST(LlvmLibcOpTest, Memset, MemsetImplementations) {163 using Impl = ParamType;164 constexpr size_t kSize = Impl::SIZE;165 { // Test block operation166 static constexpr auto BlockImpl = SetBlockAdaptor<kSize, Impl::block>;167 Buffers DstBuffer(kSize);168 for (uint8_t value : cpp::array<uint8_t, 3>{0, 1, 255}) {169 for (auto dst : DstBuffer.spans()) {170 ASSERT_TRUE(CheckMemset<BlockImpl>(dst, value, kSize));171 }172 }173 }174 if constexpr (has_head_tail<Impl>::value) {175 // Test head tail operations from kSize to 2 * kSize.176 static constexpr auto HeadTailImpl = SetAdaptor<Impl::head_tail>;177 Buffer DstBuffer(2 * kSize);178 for (size_t size = kSize; size < 2 * kSize; ++size) {179 const uint8_t value = size % 10;180 auto dst = DstBuffer.span().subspan(0, size);181 ASSERT_TRUE(CheckMemset<HeadTailImpl>(dst, value, size));182 }183 }184 if constexpr (has_loop_and_tail<Impl>::value) {185 // Test loop operations from kSize to 3 * kSize.186 if constexpr (kSize > 1) {187 static constexpr auto LoopImpl = SetAdaptor<Impl::loop_and_tail>;188 Buffer DstBuffer(3 * kSize);189 for (size_t size = kSize; size < 3 * kSize; ++size) {190 const uint8_t value = size % 10;191 auto dst = DstBuffer.span().subspan(0, size);192 ASSERT_TRUE((CheckMemset<LoopImpl>(dst, value, size)));193 }194 }195 }196}197 198#ifdef LIBC_TARGET_ARCH_IS_X86_64199// Prevent GCC warning due to ignored __aligned__ attributes when passing x86200// SIMD types as template arguments.201#pragma GCC diagnostic push202#pragma GCC diagnostic ignored "-Wignored-attributes"203#endif // LIBC_TARGET_ARCH_IS_X86_64204 205using BcmpImplementations = testing::TypeList<206#ifdef LIBC_TARGET_ARCH_IS_X86_64207#ifdef __SSE4_1__208 generic::Bcmp<__m128i>,209#endif // __SSE4_1__210#ifdef __AVX2__211 generic::Bcmp<__m256i>,212#endif // __AVX2__213#ifdef __AVX512BW__214 generic::Bcmp<__m512i>,215#endif // __AVX512BW__216 217#endif // LIBC_TARGET_ARCH_IS_X86_64218#ifdef LIBC_TARGET_ARCH_IS_AARCH64219 aarch64::Bcmp<16>, //220 aarch64::Bcmp<32>,221#endif222#ifndef LIBC_TARGET_ARCH_IS_ARM // Removing non uint8_t types for ARM223 generic::Bcmp<uint16_t>,224 generic::Bcmp<uint32_t>, //225#ifdef LIBC_TYPES_HAS_INT64226 generic::Bcmp<uint64_t>,227#endif // LIBC_TYPES_HAS_INT64228 generic::BcmpSequence<uint16_t, uint8_t>,229 generic::BcmpSequence<uint32_t, uint8_t>, //230 generic::BcmpSequence<uint32_t, uint16_t>, //231 generic::BcmpSequence<uint32_t, uint16_t, uint8_t>,232#endif // LIBC_TARGET_ARCH_IS_ARM233 generic::BcmpSequence<uint8_t, uint8_t>,234 generic::BcmpSequence<uint8_t, uint8_t, uint8_t>, //235 generic::Bcmp<uint8_t>>;236 237#ifdef LIBC_TARGET_ARCH_IS_X86_64238#pragma GCC diagnostic pop239#endif // LIBC_TARGET_ARCH_IS_X86_64240 241// Adapt CheckBcmp signature to op implementation signatures.242template <auto FnImpl>243int CmpAdaptor(cpp::span<char> p1, cpp::span<char> p2, size_t size) {244 return (int)FnImpl(as_byte(p1), as_byte(p2), size);245}246template <size_t Size, auto FnImpl>247int CmpBlockAdaptor(cpp::span<char> p1, cpp::span<char> p2,248 [[maybe_unused]] size_t size) {249 return (int)FnImpl(as_byte(p1), as_byte(p2));250}251 252TYPED_TEST(LlvmLibcOpTest, Bcmp, BcmpImplementations) {253 using Impl = ParamType;254 constexpr size_t kSize = Impl::SIZE;255 { // Test block operation256 static constexpr auto BlockImpl = CmpBlockAdaptor<kSize, Impl::block>;257 Buffers Buffer1(kSize);258 Buffers Buffer2(kSize);259 for (auto span1 : Buffer1.spans()) {260 Randomize(span1);261 for (auto span2 : Buffer2.spans())262 ASSERT_TRUE((CheckBcmp<BlockImpl>(span1, span2, kSize)));263 }264 }265 if constexpr (has_head_tail<Impl>::value) {266 // Test head tail operations from kSize to 2 * kSize.267 static constexpr auto HeadTailImpl = CmpAdaptor<Impl::head_tail>;268 Buffer Buffer1(2 * kSize);269 Buffer Buffer2(2 * kSize);270 Randomize(Buffer1.span());271 for (size_t size = kSize; size < 2 * kSize; ++size) {272 auto span1 = Buffer1.span().subspan(0, size);273 auto span2 = Buffer2.span().subspan(0, size);274 ASSERT_TRUE((CheckBcmp<HeadTailImpl>(span1, span2, size)));275 }276 }277 if constexpr (has_loop_and_tail<Impl>::value) {278 // Test loop operations from kSize to 3 * kSize.279 if constexpr (kSize > 1) {280 static constexpr auto LoopImpl = CmpAdaptor<Impl::loop_and_tail>;281 Buffer Buffer1(3 * kSize);282 Buffer Buffer2(3 * kSize);283 Randomize(Buffer1.span());284 for (size_t size = kSize; size < 3 * kSize; ++size) {285 auto span1 = Buffer1.span().subspan(0, size);286 auto span2 = Buffer2.span().subspan(0, size);287 ASSERT_TRUE((CheckBcmp<LoopImpl>(span1, span2, size)));288 }289 }290 }291}292 293#ifdef LIBC_TARGET_ARCH_IS_X86_64294// Prevent GCC warning due to ignored __aligned__ attributes when passing x86295// SIMD types as template arguments.296#pragma GCC diagnostic push297#pragma GCC diagnostic ignored "-Wignored-attributes"298#endif // LIBC_TARGET_ARCH_IS_X86_64299 300using MemcmpImplementations = testing::TypeList<301#if defined(LIBC_TARGET_ARCH_IS_X86_64) && !defined(LIBC_TARGET_OS_IS_WINDOWS)302#ifdef __SSE2__303 generic::Memcmp<__m128i>, //304#endif305#ifdef __AVX2__306 generic::Memcmp<__m256i>, //307#endif308#ifdef __AVX512BW__309 generic::Memcmp<__m512i>, //310#endif311#endif // LIBC_TARGET_ARCH_IS_X86_64312#ifdef LIBC_TARGET_ARCH_IS_AARCH64313 generic::Memcmp<uint8x16_t>, //314 generic::Memcmp<uint8x16x2_t>,315#endif316#ifndef LIBC_TARGET_ARCH_IS_ARM // Removing non uint8_t types for ARM317 generic::Memcmp<uint16_t>,318 generic::Memcmp<uint32_t>, //319#ifdef LIBC_TYPES_HAS_INT64320 generic::Memcmp<uint64_t>,321#endif // LIBC_TYPES_HAS_INT64322 generic::MemcmpSequence<uint16_t, uint8_t>,323 generic::MemcmpSequence<uint32_t, uint16_t, uint8_t>, //324#endif // LIBC_TARGET_ARCH_IS_ARM325 generic::MemcmpSequence<uint8_t, uint8_t>,326 generic::MemcmpSequence<uint8_t, uint8_t, uint8_t>,327 generic::Memcmp<uint8_t>>;328 329#ifdef LIBC_TARGET_ARCH_IS_X86_64330#pragma GCC diagnostic pop331#endif // LIBC_TARGET_ARCH_IS_X86_64332 333TYPED_TEST(LlvmLibcOpTest, Memcmp, MemcmpImplementations) {334 using Impl = ParamType;335 constexpr size_t kSize = Impl::SIZE;336 { // Test block operation337 static constexpr auto BlockImpl = CmpBlockAdaptor<kSize, Impl::block>;338 Buffers Buffer1(kSize);339 Buffers Buffer2(kSize);340 for (auto span1 : Buffer1.spans()) {341 Randomize(span1);342 for (auto span2 : Buffer2.spans())343 ASSERT_TRUE((CheckMemcmp<BlockImpl>(span1, span2, kSize)));344 }345 }346 if constexpr (has_head_tail<Impl>::value) {347 // Test head tail operations from kSize to 2 * kSize.348 static constexpr auto HeadTailImpl = CmpAdaptor<Impl::head_tail>;349 Buffer Buffer1(2 * kSize);350 Buffer Buffer2(2 * kSize);351 Randomize(Buffer1.span());352 for (size_t size = kSize; size < 2 * kSize; ++size) {353 auto span1 = Buffer1.span().subspan(0, size);354 auto span2 = Buffer2.span().subspan(0, size);355 ASSERT_TRUE((CheckMemcmp<HeadTailImpl>(span1, span2, size)));356 }357 }358 if constexpr (has_loop_and_tail<Impl>::value) {359 // Test loop operations from kSize to 3 * kSize.360 if constexpr (kSize > 1) {361 static constexpr auto LoopImpl = CmpAdaptor<Impl::loop_and_tail>;362 Buffer Buffer1(3 * kSize);363 Buffer Buffer2(3 * kSize);364 Randomize(Buffer1.span());365 for (size_t size = kSize; size < 3 * kSize; ++size) {366 auto span1 = Buffer1.span().subspan(0, size);367 auto span2 = Buffer2.span().subspan(0, size);368 ASSERT_TRUE((CheckMemcmp<LoopImpl>(span1, span2, size)));369 }370 }371 }372}373 374} // namespace LIBC_NAMESPACE_DECL375