105 lines · c
1//===-- Memset implementation for aarch64 -----------------------*- C++ -*-===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8#ifndef LIBC_SRC_STRING_MEMORY_UTILS_AARCH64_INLINE_MEMSET_H9#define LIBC_SRC_STRING_MEMORY_UTILS_AARCH64_INLINE_MEMSET_H10 11#include "src/__support/macros/attributes.h" // LIBC_INLINE12#include "src/__support/macros/config.h" // LIBC_NAMESPACE_DECL13#include "src/string/memory_utils/op_aarch64.h"14#include "src/string/memory_utils/op_generic.h"15#include "src/string/memory_utils/utils.h" // Ptr, CPtr16 17#include <stddef.h> // size_t18 19namespace LIBC_NAMESPACE_DECL {20 21using uint128_t = generic_v128;22using uint256_t = generic_v256;23using uint512_t = generic_v512;24 25[[maybe_unused]] LIBC_INLINE static void26inline_memset_aarch64_no_fp(Ptr dst, uint8_t value, size_t count) {27 if (count == 0)28 return;29 if (count <= 3) {30 generic::Memset<uint8_t>::block(dst, value);31 if (count > 1)32 generic::Memset<uint16_t>::tail(dst, value, count);33 return;34 }35 if (count <= 8)36 return generic::Memset<uint32_t>::head_tail(dst, value, count);37 if (count <= 16)38 return generic::Memset<uint64_t>::head_tail(dst, value, count);39 if (count <= 32)40 return generic::Memset<uint128_t>::head_tail(dst, value, count);41 if (count <= (32 + 64)) {42 generic::Memset<uint256_t>::block(dst, value);43 if (count <= 64)44 return generic::Memset<uint256_t>::tail(dst, value, count);45 generic::Memset<uint256_t>::block(dst + 32, value);46 generic::Memset<uint256_t>::tail(dst, value, count);47 return;48 }49 50 generic::Memset<uint128_t>::block(dst, value);51 align_to_next_boundary<16>(dst, count);52 return generic::Memset<uint512_t>::loop_and_tail(dst, value, count);53}54 55#if defined(__ARM_NEON)56[[maybe_unused]] LIBC_INLINE static void57inline_memset_aarch64_with_fp(Ptr dst, uint8_t value, size_t count) {58 if (count == 0)59 return;60 if (count <= 3) {61 generic::Memset<uint8_t>::block(dst, value);62 if (count > 1)63 generic::Memset<uint16_t>::tail(dst, value, count);64 return;65 }66 if (count <= 8)67 return generic::Memset<uint32_t>::head_tail(dst, value, count);68 if (count <= 16)69 return generic::Memset<uint64_t>::head_tail(dst, value, count);70 if (count <= 32)71 return generic::Memset<uint128_t>::head_tail(dst, value, count);72 if (count <= (32 + 64)) {73 generic::Memset<uint256_t>::block(dst, value);74 if (count <= 64)75 return generic::Memset<uint256_t>::tail(dst, value, count);76 generic::Memset<uint256_t>::block(dst + 32, value);77 generic::Memset<uint256_t>::tail(dst, value, count);78 return;79 }80 81 if (count >= 448 && value == 0 && aarch64::neon::hasZva()) {82 generic::Memset<uint512_t>::block(dst, 0);83 align_to_next_boundary<64>(dst, count);84 return aarch64::neon::BzeroCacheLine::loop_and_tail(dst, 0, count);85 }86 87 generic::Memset<uint128_t>::block(dst, value);88 align_to_next_boundary<16>(dst, count);89 return generic::Memset<uint512_t>::loop_and_tail(dst, value, count);90}91#endif92 93[[gnu::flatten]] [[maybe_unused]] LIBC_INLINE static void94inline_memset_aarch64_dispatch(Ptr dst, uint8_t value, size_t count) {95#if defined(__ARM_NEON)96 return inline_memset_aarch64_with_fp(dst, value, count);97#else98 return inline_memset_aarch64_no_fp(dst, value, count);99#endif100}101 102} // namespace LIBC_NAMESPACE_DECL103 104#endif // LIBC_SRC_STRING_MEMORY_UTILS_AARCH64_INLINE_MEMSET_H105