107 lines · c
1//===-- Strlen implementation for aarch64 ---------------------------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8#ifndef LLVM_LIBC_SRC_STRING_MEMORY_UTILS_AARCH64_INLINE_STRLEN_H9#define LLVM_LIBC_SRC_STRING_MEMORY_UTILS_AARCH64_INLINE_STRLEN_H10 11#include "src/__support/macros/properties/cpu_features.h"12 13#if defined(__ARM_NEON)14#include "src/__support/CPP/bit.h" // countr_zero15#include <arm_neon.h>16#include <stddef.h> // size_t17namespace LIBC_NAMESPACE_DECL {18namespace neon {19[[maybe_unused]] LIBC_NO_SANITIZE_OOB_ACCESS LIBC_INLINE static size_t20string_length(const char *src) {21 using Vector __attribute__((may_alias)) = uint8x8_t;22 23 uintptr_t misalign_bytes = reinterpret_cast<uintptr_t>(src) % sizeof(Vector);24 const Vector *block_ptr =25 reinterpret_cast<const Vector *>(src - misalign_bytes);26 Vector v = *block_ptr;27 Vector vcmp = vceqz_u8(v);28 uint64x1_t cmp_mask = vreinterpret_u64_u8(vcmp);29 uint64_t cmp = vget_lane_u64(cmp_mask, 0);30 cmp = cmp >> (misalign_bytes << 3);31 if (cmp)32 return cpp::countr_zero(cmp) >> 3;33 34 while (true) {35 ++block_ptr;36 v = *block_ptr;37 vcmp = vceqz_u8(v);38 cmp_mask = vreinterpret_u64_u8(vcmp);39 cmp = vget_lane_u64(cmp_mask, 0);40 if (cmp)41 return static_cast<size_t>(reinterpret_cast<uintptr_t>(block_ptr) -42 reinterpret_cast<uintptr_t>(src) +43 (cpp::countr_zero(cmp) >> 3));44 }45}46} // namespace neon47} // namespace LIBC_NAMESPACE_DECL48#endif // __ARM_NEON49 50#ifdef LIBC_TARGET_CPU_HAS_SVE51#include "src/__support/macros/optimization.h"52#include <arm_sve.h>53namespace LIBC_NAMESPACE_DECL {54namespace sve {55[[maybe_unused]] LIBC_INLINE static size_t string_length(const char *src) {56 const uint8_t *ptr = reinterpret_cast<const uint8_t *>(src);57 // Initialize the first-fault register to all true58 svsetffr();59 const svbool_t all_true = svptrue_b8(); // all true predicate60 svbool_t cmp_zero;61 size_t len = 0;62 63 for (;;) {64 // Read a vector's worth of bytes, stopping on first fault.65 svuint8_t data = svldff1_u8(all_true, &ptr[len]);66 svbool_t fault_mask = svrdffr_z(all_true);67 bool has_no_fault = svptest_last(all_true, fault_mask);68 if (LIBC_LIKELY(has_no_fault)) {69 // First fault did not fail: the whole vector is valid.70 // Avoid depending on the contents of FFR beyond the branch.71 len += svcntb(); // speculative increment72 cmp_zero = svcmpeq_n_u8(all_true, data, 0);73 bool has_no_zero = !svptest_any(all_true, cmp_zero);74 if (LIBC_LIKELY(has_no_zero))75 continue;76 len -= svcntb(); // undo speculative increment77 break;78 } else {79 // First fault failed: only some of the vector is valid.80 // Perform the comparison only on the valid bytes.81 cmp_zero = svcmpeq_n_u8(fault_mask, data, 0);82 bool has_zero = svptest_any(fault_mask, cmp_zero);83 if (LIBC_LIKELY(has_zero))84 break;85 svsetffr();86 len += svcntp_b8(all_true, fault_mask);87 continue;88 }89 }90 // Select the bytes before the first and count them.91 svbool_t before_zero = svbrkb_z(all_true, cmp_zero);92 len += svcntp_b8(all_true, before_zero);93 return len;94}95} // namespace sve96} // namespace LIBC_NAMESPACE_DECL97#endif // LIBC_TARGET_CPU_HAS_SVE98 99namespace LIBC_NAMESPACE_DECL {100#ifdef LIBC_TARGET_CPU_HAS_SVE101namespace string_length_impl = sve;102#elif defined(__ARM_NEON)103namespace string_length_impl = neon;104#endif105} // namespace LIBC_NAMESPACE_DECL106#endif // LLVM_LIBC_SRC_STRING_MEMORY_UTILS_AARCH64_INLINE_STRLEN_H107