278 lines · c
1//===-- aarch64 implementation of memory function building blocks ---------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// This file provides aarch64 specific building blocks to compose memory10// functions.11//12//===----------------------------------------------------------------------===//13#ifndef LLVM_LIBC_SRC_STRING_MEMORY_UTILS_OP_AARCH64_H14#define LLVM_LIBC_SRC_STRING_MEMORY_UTILS_OP_AARCH64_H15 16#include "src/__support/macros/attributes.h" // LIBC_INLINE17#include "src/__support/macros/config.h" // LIBC_NAMESPACE_DECL18#include "src/__support/macros/properties/architectures.h"19 20#if defined(LIBC_TARGET_ARCH_IS_AARCH64)21 22#include "src/__support/CPP/type_traits.h" // cpp::always_false23#include "src/__support/common.h"24#include "src/string/memory_utils/op_generic.h"25 26#ifdef __ARM_NEON27#include <arm_neon.h>28 29namespace LIBC_NAMESPACE_DECL {30namespace aarch64 {31 32LIBC_INLINE_VAR constexpr bool kNeon = LLVM_LIBC_IS_DEFINED(__ARM_NEON);33 34namespace neon {35 36struct BzeroCacheLine {37 static constexpr size_t SIZE = 64;38 39 LIBC_INLINE static void block(Ptr dst, uint8_t) {40#if __SIZEOF_POINTER__ == 441 asm("dc zva, %w[dst]" : : [dst] "r"(dst) : "memory");42#else43 asm("dc zva, %[dst]" : : [dst] "r"(dst) : "memory");44#endif45 }46 47 LIBC_INLINE static void loop_and_tail(Ptr dst, uint8_t value, size_t count) {48 size_t offset = 0;49 do {50 block(dst + offset, value);51 offset += SIZE;52 } while (offset < count - SIZE);53 // Unaligned store, we can't use 'dc zva' here.54 generic::Memset<generic_v512>::tail(dst, value, count);55 }56};57 58LIBC_INLINE bool hasZva() {59 uint64_t zva_val;60 asm("mrs %[zva_val], dczid_el0" : [zva_val] "=r"(zva_val));61 // DC ZVA is permitted if DZP, bit [4] is zero.62 // BS, bits [3:0] is log2 of the block count in words.63 // So the next line checks whether the instruction is permitted and block64 // count is 16 words (i.e. 64 bytes).65 return (zva_val & 0b11111) == 0b00100;66}67 68} // namespace neon69 70///////////////////////////////////////////////////////////////////////////////71// Bcmp72template <size_t Size> struct Bcmp {73 static constexpr size_t SIZE = Size;74 static constexpr size_t BlockSize = 32;75 76 LIBC_INLINE static const unsigned char *as_u8(CPtr ptr) {77 return reinterpret_cast<const unsigned char *>(ptr);78 }79 80 LIBC_INLINE static BcmpReturnType block(CPtr p1, CPtr p2) {81 if constexpr (Size == 16) {82 auto _p1 = as_u8(p1);83 auto _p2 = as_u8(p2);84 uint8x16_t a = vld1q_u8(_p1);85 uint8x16_t n = vld1q_u8(_p2);86 uint8x16_t an = veorq_u8(a, n);87 return vmaxvq_u32(vreinterpretq_u32_u8(an));88 } else if constexpr (Size == 32) {89 auto _p1 = as_u8(p1);90 auto _p2 = as_u8(p2);91 uint8x16_t a = vld1q_u8(_p1);92 uint8x16_t b = vld1q_u8(_p1 + 16);93 uint8x16_t n = vld1q_u8(_p2);94 uint8x16_t o = vld1q_u8(_p2 + 16);95 uint8x16_t an = veorq_u8(a, n);96 uint8x16_t bo = veorq_u8(b, o);97 // anbo = (a ^ n) | (b ^ o). At least one byte is nonzero if there is98 // a difference between the two buffers. We reduce this value down to 499 // bytes using the UMAXV instruction to compute the max across the vector.100 uint8x16_t anbo = vorrq_u8(an, bo);101 return vmaxvq_u32(vreinterpretq_u32_u8(anbo));102 } else if constexpr ((Size % BlockSize) == 0) {103 for (size_t offset = 0; offset < Size; offset += BlockSize)104 if (auto value = Bcmp<BlockSize>::block(p1 + offset, p2 + offset))105 return value;106 } else {107 static_assert(cpp::always_false<decltype(Size)>, "SIZE not implemented");108 }109 return BcmpReturnType::zero();110 }111 112 LIBC_INLINE static BcmpReturnType tail(CPtr p1, CPtr p2, size_t count) {113 return block(p1 + count - SIZE, p2 + count - SIZE);114 }115 116 LIBC_INLINE static BcmpReturnType head_tail(CPtr p1, CPtr p2, size_t count) {117 if constexpr (Size == 16) {118 auto _p1 = as_u8(p1);119 auto _p2 = as_u8(p2);120 uint8x16_t a = vld1q_u8(_p1);121 uint8x16_t b = vld1q_u8(_p1 + count - 16);122 uint8x16_t n = vld1q_u8(_p2);123 uint8x16_t o = vld1q_u8(_p2 + count - 16);124 uint8x16_t an = veorq_u8(a, n);125 uint8x16_t bo = veorq_u8(b, o);126 // anbo = (a ^ n) | (b ^ o)127 uint8x16_t anbo = vorrq_u8(an, bo);128 return vmaxvq_u32(vreinterpretq_u32_u8(anbo));129 } else if constexpr (Size == 32) {130 auto _p1 = as_u8(p1);131 auto _p2 = as_u8(p2);132 uint8x16_t a = vld1q_u8(_p1);133 uint8x16_t b = vld1q_u8(_p1 + 16);134 uint8x16_t c = vld1q_u8(_p1 + count - 16);135 uint8x16_t d = vld1q_u8(_p1 + count - 32);136 uint8x16_t n = vld1q_u8(_p2);137 uint8x16_t o = vld1q_u8(_p2 + 16);138 uint8x16_t p = vld1q_u8(_p2 + count - 16);139 uint8x16_t q = vld1q_u8(_p2 + count - 32);140 uint8x16_t an = veorq_u8(a, n);141 uint8x16_t bo = veorq_u8(b, o);142 uint8x16_t cp = veorq_u8(c, p);143 uint8x16_t dq = veorq_u8(d, q);144 uint8x16_t anbo = vorrq_u8(an, bo);145 uint8x16_t cpdq = vorrq_u8(cp, dq);146 // abnocpdq = ((a ^ n) | (b ^ o)) | ((c ^ p) | (d ^ q)). Reduce this to147 // a nonzero 32 bit value if a mismatch occurred.148 uint8x16_t abnocpdq = anbo | cpdq;149 return vmaxvq_u32(vreinterpretq_u32_u8(abnocpdq));150 } else {151 static_assert(cpp::always_false<decltype(Size)>, "SIZE not implemented");152 }153 return BcmpReturnType::zero();154 }155 156 LIBC_INLINE static BcmpReturnType loop_and_tail(CPtr p1, CPtr p2,157 size_t count) {158 static_assert(Size > 1, "a loop of size 1 does not need tail");159 size_t offset = 0;160 do {161 if (auto value = block(p1 + offset, p2 + offset))162 return value;163 offset += SIZE;164 } while (offset < count - SIZE);165 return tail(p1, p2, count);166 }167};168 169} // namespace aarch64170} // namespace LIBC_NAMESPACE_DECL171 172#endif //__ARM_NEON173 174namespace LIBC_NAMESPACE_DECL {175namespace generic {176 177///////////////////////////////////////////////////////////////////////////////178// Specializations for uint16_t179template <> struct cmp_is_expensive<uint16_t> : public cpp::false_type {};180template <> LIBC_INLINE bool eq<uint16_t>(CPtr p1, CPtr p2, size_t offset) {181 return load<uint16_t>(p1, offset) == load<uint16_t>(p2, offset);182}183template <>184LIBC_INLINE uint32_t neq<uint16_t>(CPtr p1, CPtr p2, size_t offset) {185 return load<uint16_t>(p1, offset) ^ load<uint16_t>(p2, offset);186}187template <>188LIBC_INLINE MemcmpReturnType cmp<uint16_t>(CPtr p1, CPtr p2, size_t offset) {189 return static_cast<int32_t>(load_be<uint16_t>(p1, offset)) -190 static_cast<int32_t>(load_be<uint16_t>(p2, offset));191}192 193///////////////////////////////////////////////////////////////////////////////194// Specializations for uint32_t195template <> struct cmp_is_expensive<uint32_t> : cpp::false_type {};196template <>197LIBC_INLINE uint32_t neq<uint32_t>(CPtr p1, CPtr p2, size_t offset) {198 return load<uint32_t>(p1, offset) ^ load<uint32_t>(p2, offset);199}200template <>201LIBC_INLINE MemcmpReturnType cmp<uint32_t>(CPtr p1, CPtr p2, size_t offset) {202 const auto a = load_be<uint32_t>(p1, offset);203 const auto b = load_be<uint32_t>(p2, offset);204 return a > b ? 1 : a < b ? -1 : 0;205}206 207///////////////////////////////////////////////////////////////////////////////208// Specializations for uint64_t209template <> struct cmp_is_expensive<uint64_t> : cpp::false_type {};210template <>211LIBC_INLINE uint32_t neq<uint64_t>(CPtr p1, CPtr p2, size_t offset) {212 return load<uint64_t>(p1, offset) != load<uint64_t>(p2, offset);213}214template <>215LIBC_INLINE MemcmpReturnType cmp<uint64_t>(CPtr p1, CPtr p2, size_t offset) {216 const auto a = load_be<uint64_t>(p1, offset);217 const auto b = load_be<uint64_t>(p2, offset);218 if (a != b)219 return a > b ? 1 : -1;220 return MemcmpReturnType::zero();221}222 223#if defined(__ARM_NEON)224 225///////////////////////////////////////////////////////////////////////////////226// Specializations for uint8x16_t227template <> struct is_vector<uint8x16_t> : cpp::true_type {};228template <> struct cmp_is_expensive<uint8x16_t> : cpp::false_type {};229template <>230LIBC_INLINE uint32_t neq<uint8x16_t>(CPtr p1, CPtr p2, size_t offset) {231 for (size_t i = 0; i < 2; ++i) {232 auto a = load<uint64_t>(p1, offset);233 auto b = load<uint64_t>(p2, offset);234 uint32_t cond = a != b;235 if (cond)236 return cond;237 offset += sizeof(uint64_t);238 }239 return 0;240}241template <>242LIBC_INLINE MemcmpReturnType cmp<uint8x16_t>(CPtr p1, CPtr p2, size_t offset) {243 for (size_t i = 0; i < 2; ++i) {244 auto a = load_be<uint64_t>(p1, offset);245 auto b = load_be<uint64_t>(p2, offset);246 if (a != b)247 return cmp_neq_uint64_t(a, b);248 offset += sizeof(uint64_t);249 }250 return MemcmpReturnType::zero();251}252 253///////////////////////////////////////////////////////////////////////////////254// Specializations for uint8x16x2_t255template <> struct is_vector<uint8x16x2_t> : cpp::true_type {};256template <> struct cmp_is_expensive<uint8x16x2_t> : cpp::false_type {};257template <>258LIBC_INLINE MemcmpReturnType cmp<uint8x16x2_t>(CPtr p1, CPtr p2,259 size_t offset) {260 for (size_t i = 0; i < 4; ++i) {261 auto a = load_be<uint64_t>(p1, offset);262 auto b = load_be<uint64_t>(p2, offset);263 if (a != b)264 return cmp_neq_uint64_t(a, b);265 offset += sizeof(uint64_t);266 }267 return MemcmpReturnType::zero();268}269 270#endif // __ARM_NEON271 272} // namespace generic273} // namespace LIBC_NAMESPACE_DECL274 275#endif // LIBC_TARGET_ARCH_IS_AARCH64276 277#endif // LLVM_LIBC_SRC_STRING_MEMORY_UTILS_OP_AARCH64_H278