108 lines · c
1//===-- Strlen implementation for x86_64 ----------------------------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8#ifndef LLVM_LIBC_SRC_STRING_MEMORY_UTILS_X86_64_INLINE_STRLEN_H9#define LLVM_LIBC_SRC_STRING_MEMORY_UTILS_X86_64_INLINE_STRLEN_H10 11#include "src/__support/CPP/bit.h" // countr_zero12 13#include <immintrin.h>14#include <stddef.h> // size_t15 16namespace LIBC_NAMESPACE_DECL {17 18namespace string_length_internal {19// Return a bit-mask with the nth bit set if the nth-byte in block_ptr is zero.20template <typename Vector, typename Mask>21LIBC_NO_SANITIZE_OOB_ACCESS LIBC_INLINE static Mask22compare_and_mask(const Vector *block_ptr);23 24template <typename Vector, typename Mask,25 decltype(compare_and_mask<Vector, Mask>)>26LIBC_NO_SANITIZE_OOB_ACCESS LIBC_INLINE static size_t27string_length_vector(const char *src) {28 uintptr_t misalign_bytes = reinterpret_cast<uintptr_t>(src) % sizeof(Vector);29 30 const Vector *block_ptr =31 reinterpret_cast<const Vector *>(src - misalign_bytes);32 auto cmp = compare_and_mask<Vector, Mask>(block_ptr) >> misalign_bytes;33 if (cmp)34 return cpp::countr_zero(cmp);35 36 while (true) {37 block_ptr++;38 cmp = compare_and_mask<Vector, Mask>(block_ptr);39 if (cmp)40 return static_cast<size_t>(reinterpret_cast<uintptr_t>(block_ptr) -41 reinterpret_cast<uintptr_t>(src) +42 cpp::countr_zero(cmp));43 }44}45 46template <>47LIBC_INLINE uint32_t48compare_and_mask<__m128i, uint32_t>(const __m128i *block_ptr) {49 __m128i v = _mm_load_si128(block_ptr);50 __m128i z = _mm_setzero_si128();51 __m128i c = _mm_cmpeq_epi8(z, v);52 return _mm_movemask_epi8(c);53}54 55namespace sse2 {56[[maybe_unused]] LIBC_INLINE size_t string_length(const char *src) {57 return string_length_vector<__m128i, uint32_t,58 compare_and_mask<__m128i, uint32_t>>(src);59}60} // namespace sse261 62#if defined(__AVX2__)63template <>64LIBC_INLINE uint32_t65compare_and_mask<__m256i, uint32_t>(const __m256i *block_ptr) {66 __m256i v = _mm256_load_si256(block_ptr);67 __m256i z = _mm256_setzero_si256();68 __m256i c = _mm256_cmpeq_epi8(z, v);69 return _mm256_movemask_epi8(c);70}71 72namespace avx2 {73[[maybe_unused]] LIBC_INLINE size_t string_length(const char *src) {74 return string_length_vector<__m256i, uint32_t,75 compare_and_mask<__m256i, uint32_t>>(src);76}77} // namespace avx278#endif79 80#if defined(__AVX512F__)81template <>82LIBC_INLINE __mmask6483compare_and_mask<__m512i, __mmask64>(const __m512i *block_ptr) {84 __m512i v = _mm512_load_si512(block_ptr);85 __m512i z = _mm512_setzero_si512();86 return _mm512_cmp_epu8_mask(z, v, _MM_CMPINT_EQ);87}88namespace avx512 {89[[maybe_unused]] LIBC_INLINE size_t string_length(const char *src) {90 return string_length_vector<__m512i, __mmask64,91 compare_and_mask<__m512i, __mmask64>>(src);92}93} // namespace avx51294#endif95} // namespace string_length_internal96 97#if defined(__AVX512F__)98namespace string_length_impl = string_length_internal::avx512;99#elif defined(__AVX2__)100namespace string_length_impl = string_length_internal::avx2;101#else102namespace string_length_impl = string_length_internal::sse2;103#endif104 105} // namespace LIBC_NAMESPACE_DECL106 107#endif // LLVM_LIBC_SRC_STRING_MEMORY_UTILS_X86_64_INLINE_STRLEN_H108