215 lines · c
1//===-- Memcpy implementation for arm ---------------------------*- C++ -*-===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8// The functions defined in this file give approximate code size. These sizes9// assume the following configuration options:10// - LIBC_CONF_KEEP_FRAME_POINTER = false11// - LIBC_CONF_ENABLE_STRONG_STACK_PROTECTOR = false12// - LIBC_ADD_NULL_CHECKS = false13#ifndef LLVM_LIBC_SRC_STRING_MEMORY_UTILS_ARM_INLINE_MEMCPY_H14#define LLVM_LIBC_SRC_STRING_MEMORY_UTILS_ARM_INLINE_MEMCPY_H15 16#include "src/__support/CPP/type_traits.h" // always_false17#include "src/__support/macros/attributes.h" // LIBC_INLINE18#include "src/__support/macros/optimization.h" // LIBC_LOOP_NOUNROLL19#include "src/string/memory_utils/arm/common.h" // LIBC_ATTR_LIKELY, LIBC_ATTR_UNLIKELY20#include "src/string/memory_utils/utils.h" // memcpy_inline, distance_to_align21 22#include <stddef.h> // size_t23 24namespace LIBC_NAMESPACE_DECL {25 26namespace {27 28// Performs a copy of `bytes` byte from `src` to `dst`. This function has the29// semantics of `memcpy` where `src` and `dst` are `__restrict`. The compiler is30// free to use whatever instruction is best for the size and assumed access.31template <size_t bytes, AssumeAccess access>32LIBC_INLINE void copy(void *dst, const void *src) {33 if constexpr (access == AssumeAccess::kAligned) {34 constexpr size_t alignment = bytes > kWordSize ? kWordSize : bytes;35 memcpy_inline<bytes>(assume_aligned<alignment>(dst),36 assume_aligned<alignment>(src));37 } else if constexpr (access == AssumeAccess::kUnknown) {38 memcpy_inline<bytes>(dst, src);39 } else {40 static_assert(cpp::always_false<decltype(access)>, "Invalid AssumeAccess");41 }42}43 44template <size_t bytes, BlockOp block_op = BlockOp::kFull,45 AssumeAccess access = AssumeAccess::kUnknown>46LIBC_INLINE void copy_block_and_bump_pointers(Ptr &dst, CPtr &src) {47 if constexpr (block_op == BlockOp::kFull) {48 copy<bytes, access>(dst, src);49 } else if constexpr (block_op == BlockOp::kByWord) {50 // We restrict loads/stores to 4 byte to prevent the use of load/store51 // multiple (LDM, STM) and load/store double (LDRD, STRD).52 static_assert((bytes % kWordSize == 0) && (bytes >= kWordSize));53 LIBC_LOOP_UNROLL54 for (size_t offset = 0; offset < bytes; offset += kWordSize) {55 copy<kWordSize, access>(dst + offset, src + offset);56 }57 } else {58 static_assert(cpp::always_false<decltype(block_op)>, "Invalid BlockOp");59 }60 // In the 1, 2, 4 byte copy case, the compiler can fold pointer offsetting61 // into the load/store instructions.62 // e.g.,63 // ldrb r3, [r1], #164 // strb r3, [r0], #165 dst += bytes;66 src += bytes;67}68 69template <size_t bytes, BlockOp block_op, AssumeAccess access>70LIBC_INLINE void consume_by_block(Ptr &dst, CPtr &src, size_t &size) {71 LIBC_LOOP_NOUNROLL72 for (size_t i = 0; i < size / bytes; ++i)73 copy_block_and_bump_pointers<bytes, block_op, access>(dst, src);74 size %= bytes;75}76 77[[maybe_unused]] LIBC_INLINE void78copy_bytes_and_bump_pointers(Ptr &dst, CPtr &src, size_t size) {79 LIBC_LOOP_NOUNROLL80 for (size_t i = 0; i < size; ++i)81 *dst++ = *src++;82}83 84} // namespace85 86// Implementation for Cortex-M0, M0+, M1 cores that do not allow for unaligned87// loads/stores. It compiles down to 208 bytes when used through `memcpy` that88// also needs to return the `dst` ptr.89// Note:90// - When `src` and `dst` are coaligned, we start by aligning them and perform91// bulk copies. We let the compiler know the pointers are aligned so it can92// use load/store multiple (LDM, STM). This significantly increase throughput93// but it also requires more registers and push/pop instructions. This impacts94// latency for small size copies.95// - When `src` and `dst` are misaligned, we align `dst` and recompose words96// using multiple aligned loads. `load_aligned` takes care of endianness97// issues.98[[maybe_unused]] LIBC_INLINE void inline_memcpy_arm_low_end(Ptr dst, CPtr src,99 size_t size) {100 if (size >= 8) {101 if (const size_t offset = distance_to_align_up<kWordSize>(dst))102 LIBC_ATTR_UNLIKELY {103 copy_bytes_and_bump_pointers(dst, src, offset);104 size -= offset;105 }106 constexpr AssumeAccess kAligned = AssumeAccess::kAligned;107 const auto src_alignment = distance_to_align_down<kWordSize>(src);108 if (src_alignment == 0)109 LIBC_ATTR_LIKELY {110 // Both `src` and `dst` are now word-aligned.111 // We first copy by blocks of 64 bytes, the compiler will use 4112 // load/store multiple (LDM, STM), each of 4 words. This requires more113 // registers so additional push/pop are needed but the speedup is worth114 // it.115 consume_by_block<64, BlockOp::kFull, kAligned>(dst, src, size);116 // Then we use blocks of 4 word load/store.117 consume_by_block<16, BlockOp::kByWord, kAligned>(dst, src, size);118 // Then we use word by word copy.119 consume_by_block<4, BlockOp::kByWord, kAligned>(dst, src, size);120 }121 else {122 // `dst` is aligned but `src` is not.123 LIBC_LOOP_NOUNROLL124 while (size >= kWordSize) {125 // Recompose word from multiple loads depending on the alignment.126 const uint32_t value =127 src_alignment == 2128 ? load_aligned<uint32_t, uint16_t, uint16_t>(src)129 : load_aligned<uint32_t, uint8_t, uint16_t, uint8_t>(src);130 copy<kWordSize, kAligned>(dst, &value);131 dst += kWordSize;132 src += kWordSize;133 size -= kWordSize;134 }135 }136 // Up to 3 bytes may still need to be copied.137 // Handling them with the slow loop below.138 }139 copy_bytes_and_bump_pointers(dst, src, size);140}141 142// Implementation for Cortex-M3, M4, M7, M23, M33, M35P, M52 with hardware143// support for unaligned loads and stores. It compiles down to 272 bytes when144// used through `memcpy` that also needs to return the `dst` ptr.145[[maybe_unused]] LIBC_INLINE void inline_memcpy_arm_mid_end(Ptr dst, CPtr src,146 size_t size) {147 if (misaligned(bitwise_or(src, dst)))148 LIBC_ATTR_UNLIKELY {149 if (size < 8)150 LIBC_ATTR_UNLIKELY {151 if (size & 1)152 copy_block_and_bump_pointers<1>(dst, src);153 if (size & 2)154 copy_block_and_bump_pointers<2>(dst, src);155 if (size & 4)156 copy_block_and_bump_pointers<4>(dst, src);157 return;158 }159 if (misaligned(src))160 LIBC_ATTR_UNLIKELY {161 const size_t offset = distance_to_align_up<kWordSize>(dst);162 if (offset & 1)163 copy_block_and_bump_pointers<1>(dst, src);164 if (offset & 2)165 copy_block_and_bump_pointers<2>(dst, src);166 size -= offset;167 }168 }169 // `dst` and `src` are not necessarily both aligned at that point but this170 // implementation assumes hardware support for unaligned loads and stores so171 // it is still fast to perform unrolled word by word copy. Note that wider172 // accesses through the use of load/store multiple (LDM, STM) and load/store173 // double (LDRD, STRD) instructions are generally not supported and can fault.174 // By forcing decomposition of 64 bytes copy into word by word copy, the175 // compiler uses a load to prefetch the next cache line:176 // ldr r3, [r1, #64]! <- prefetch next cache line177 // str r3, [r0]178 // ldr r3, [r1, #0x4]179 // str r3, [r0, #0x4]180 // ...181 // ldr r3, [r1, #0x3c]182 // str r3, [r0, #0x3c]183 // This is a bit detrimental for sizes between 64 and 256 (less than 10%184 // penalty) but the prefetch yields better throughput for larger copies.185 constexpr AssumeAccess kUnknown = AssumeAccess::kUnknown;186 consume_by_block<64, BlockOp::kByWord, kUnknown>(dst, src, size);187 consume_by_block<16, BlockOp::kByWord, kUnknown>(dst, src, size);188 consume_by_block<4, BlockOp::kByWord, kUnknown>(dst, src, size);189 if (size & 1)190 copy_block_and_bump_pointers<1>(dst, src);191 if (size & 2)192 copy_block_and_bump_pointers<2>(dst, src);193}194 195[[maybe_unused]] LIBC_INLINE void inline_memcpy_arm(Ptr dst, CPtr src,196 size_t size) {197 // The compiler performs alias analysis and is able to prove that `dst` and198 // `src` do not alias by propagating the `__restrict` keyword from the199 // `memcpy` prototype. This allows the compiler to merge consecutive200 // load/store (LDR, STR) instructions generated in201 // `copy_block_and_bump_pointers` with `BlockOp::kByWord` into load/store202 // double (LDRD, STRD) instructions, this is is undesirable so we prevent the203 // compiler from inferring `__restrict` with the following line.204 asm volatile("" : "+r"(dst), "+r"(src));205#ifdef __ARM_FEATURE_UNALIGNED206 return inline_memcpy_arm_mid_end(dst, src, size);207#else208 return inline_memcpy_arm_low_end(dst, src, size);209#endif210}211 212} // namespace LIBC_NAMESPACE_DECL213 214#endif // LLVM_LIBC_SRC_STRING_MEMORY_UTILS_ARM_INLINE_MEMCPY_H215