157 lines · c
1//===-- Memset implementation for arm ---------------------------*- C++ -*-===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8// The functions defined in this file give approximate code size. These sizes9// assume the following configuration options:10// - LIBC_CONF_KEEP_FRAME_POINTER = false11// - LIBC_CONF_ENABLE_STRONG_STACK_PROTECTOR = false12// - LIBC_ADD_NULL_CHECKS = false13#ifndef LLVM_LIBC_SRC_STRING_MEMORY_UTILS_ARM_INLINE_MEMSET_H14#define LLVM_LIBC_SRC_STRING_MEMORY_UTILS_ARM_INLINE_MEMSET_H15 16#include "src/__support/CPP/type_traits.h" // always_false17#include "src/__support/macros/attributes.h" // LIBC_INLINE18#include "src/__support/macros/optimization.h" // LIBC_LOOP_NOUNROLL19#include "src/string/memory_utils/arm/common.h" // LIBC_ATTR_LIKELY, LIBC_ATTR_UNLIKELY20#include "src/string/memory_utils/utils.h" // memcpy_inline, distance_to_align21 22#include <stddef.h> // size_t23 24namespace LIBC_NAMESPACE_DECL {25 26namespace {27 28template <size_t bytes, AssumeAccess access>29LIBC_INLINE void set(void *dst, uint32_t value) {30 static_assert(bytes == 1 || bytes == 2 || bytes == 4);31 if constexpr (access == AssumeAccess::kAligned) {32 constexpr size_t alignment = bytes > kWordSize ? kWordSize : bytes;33 memcpy_inline<bytes>(assume_aligned<alignment>(dst), &value);34 } else if constexpr (access == AssumeAccess::kUnknown) {35 memcpy_inline<bytes>(dst, &value);36 } else {37 static_assert(cpp::always_false<decltype(access)>, "Invalid AssumeAccess");38 }39}40 41template <size_t bytes, AssumeAccess access = AssumeAccess::kUnknown>42LIBC_INLINE void set_block_and_bump_pointers(Ptr &dst, uint32_t value) {43 if constexpr (bytes <= kWordSize) {44 set<bytes, access>(dst, value);45 } else {46 static_assert(bytes % kWordSize == 0 && bytes >= kWordSize);47 LIBC_LOOP_UNROLL48 for (size_t offset = 0; offset < bytes; offset += kWordSize) {49 set<kWordSize, access>(dst + offset, value);50 }51 }52 // In the 1, 2, 4 byte set case, the compiler can fold pointer offsetting53 // into the store instructions.54 // e.g.,55 // strb r3, [r0], #156 dst += bytes;57}58 59template <size_t bytes, AssumeAccess access>60LIBC_INLINE void consume_by_block(Ptr &dst, uint32_t value, size_t &size) {61 LIBC_LOOP_NOUNROLL62 for (size_t i = 0; i < size / bytes; ++i)63 set_block_and_bump_pointers<bytes, access>(dst, value);64 size %= bytes;65}66 67[[maybe_unused]] LIBC_INLINE void68set_bytes_and_bump_pointers(Ptr &dst, uint32_t value, size_t size) {69 LIBC_LOOP_NOUNROLL70 for (size_t i = 0; i < size; ++i) {71 set<1, AssumeAccess::kUnknown>(dst++, value);72 }73}74 75} // namespace76 77// Implementation for Cortex-M0, M0+, M1. It compiles down to 140 bytes when78// used through `memset` that also needs to return the `dst` ptr. These cores do79// not allow unaligned stores so all accesses are aligned.80[[maybe_unused]] LIBC_INLINE void81inline_memset_arm_low_end(Ptr dst, uint8_t value, size_t size) {82 if (size >= 8)83 LIBC_ATTR_LIKELY {84 // Align `dst` to word boundary.85 if (const size_t offset = distance_to_align_up<kWordSize>(dst))86 LIBC_ATTR_UNLIKELY {87 set_bytes_and_bump_pointers(dst, value, offset);88 size -= offset;89 }90 const uint32_t value32 = value * 0x01010101U; // splat value in each byte91 consume_by_block<64, AssumeAccess::kAligned>(dst, value32, size);92 consume_by_block<16, AssumeAccess::kAligned>(dst, value32, size);93 consume_by_block<4, AssumeAccess::kAligned>(dst, value32, size);94 }95 set_bytes_and_bump_pointers(dst, value, size);96}97 98// Implementation for Cortex-M3, M4, M7, M23, M33, M35P, M52 with hardware99// support for unaligned loads and stores. It compiles down to 186 bytes when100// used through `memset` that also needs to return the `dst` ptr.101[[maybe_unused]] LIBC_INLINE void102inline_memset_arm_mid_end(Ptr dst, uint8_t value, size_t size) {103 const uint32_t value32 = value * 0x01010101U; // splat value in each byte104 if (misaligned(dst))105 LIBC_ATTR_UNLIKELY {106 if (size < 8)107 LIBC_ATTR_UNLIKELY {108 if (size & 1)109 set_block_and_bump_pointers<1>(dst, value32);110 if (size & 2)111 set_block_and_bump_pointers<2>(dst, value32);112 if (size & 4)113 set_block_and_bump_pointers<4>(dst, value32);114 return;115 }116 const size_t offset = distance_to_align_up<kWordSize>(dst);117 if (offset & 1)118 set_block_and_bump_pointers<1>(dst, value32);119 if (offset & 2)120 set_block_and_bump_pointers<2>(dst, value32);121 size -= offset;122 }123 // If we tell the compiler that the stores are aligned it will generate 8 x124 // STRD instructions. By not specifying alignment, the compiler conservatively125 // uses 16 x STR.W and is able to use the first one to prefetch the126 // destination in advance leading to better asymptotic performances.127 // str r12, [r3, #64]! <- prefetch next cache line128 // str.w r12, [r3, #0x4]129 // str.w r12, [r3, #0x8]130 // ...131 // str.w r12, [r3, #0x38]132 // str.w r12, [r3, #0x3c]133 consume_by_block<64, AssumeAccess::kUnknown>(dst, value32, size);134 // Prefetching does not matter anymore at this scale so using STRD yields135 // better results.136 consume_by_block<16, AssumeAccess::kAligned>(dst, value32, size);137 consume_by_block<4, AssumeAccess::kAligned>(dst, value32, size);138 if (size & 1)139 set_block_and_bump_pointers<1>(dst, value32);140 if (size & 2)141 LIBC_ATTR_UNLIKELY142 set_block_and_bump_pointers<2>(dst, value32);143}144 145[[maybe_unused]] LIBC_INLINE void146inline_memset_arm_dispatch(Ptr dst, uint8_t value, size_t size) {147#ifdef __ARM_FEATURE_UNALIGNED148 return inline_memset_arm_mid_end(dst, value, size);149#else150 return inline_memset_arm_low_end(dst, value, size);151#endif152}153 154} // namespace LIBC_NAMESPACE_DECL155 156#endif // LLVM_LIBC_SRC_STRING_MEMORY_UTILS_ARM_INLINE_MEMCPY_H157