203 lines · plain
1/*2 * memcpy - copy memory area3 *4 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.5 * See https://llvm.org/LICENSE.txt for license information.6 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception7 */8 9/* Assumptions:10 *11 * ARMv8-a, AArch64, Advanced SIMD, unaligned accesses.12 *13 */14 15#include "../asmdefs.h"16 17#define dstin x018#define src x119#define count x220#define dst x321#define srcend x422#define dstend x523#define A_l x624#define A_lw w625#define A_h x726#define B_l x827#define B_lw w828#define B_h x929#define C_lw w1030#define tmp1 x1431 32#define A_q q033#define B_q q134#define C_q q235#define D_q q336#define E_q q437#define F_q q538#define G_q q639#define H_q q740 41/* This implementation handles overlaps and supports both memcpy and memmove42 from a single entry point. It uses unaligned accesses and branchless43 sequences to keep the code small, simple and improve performance.44 45 Copies are split into 3 main cases: small copies of up to 32 bytes, medium46 copies of up to 128 bytes, and large copies. The overhead of the overlap47 check is negligible since it is only required for large copies.48 49 Large copies use a software pipelined loop processing 64 bytes per iteration.50 The source pointer is 16-byte aligned to minimize unaligned accesses.51 The loop tail is handled by always copying 64 bytes from the end.52*/53 54ENTRY (__memcpy_aarch64_simd)55ENTRY_ALIAS (__memmove_aarch64_simd)56 add srcend, src, count57 add dstend, dstin, count58 cmp count, 12859 b.hi L(copy_long)60 cmp count, 3261 b.hi L(copy32_128)62 63 /* Small copies: 0..32 bytes. */64 cmp count, 1665 b.lo L(copy16)66 ldr A_q, [src]67 ldr B_q, [srcend, -16]68 str A_q, [dstin]69 str B_q, [dstend, -16]70 ret71 72 /* Copy 8-15 bytes. */73L(copy16):74 tbz count, 3, L(copy8)75 ldr A_l, [src]76 ldr A_h, [srcend, -8]77 str A_l, [dstin]78 str A_h, [dstend, -8]79 ret80 81 .p2align 382 /* Copy 4-7 bytes. */83L(copy8):84 tbz count, 2, L(copy4)85 ldr A_lw, [src]86 ldr B_lw, [srcend, -4]87 str A_lw, [dstin]88 str B_lw, [dstend, -4]89 ret90 91 /* Copy 0..3 bytes using a branchless sequence. */92L(copy4):93 cbz count, L(copy0)94 lsr tmp1, count, 195 ldrb A_lw, [src]96 ldrb C_lw, [srcend, -1]97 ldrb B_lw, [src, tmp1]98 strb A_lw, [dstin]99 strb B_lw, [dstin, tmp1]100 strb C_lw, [dstend, -1]101L(copy0):102 ret103 104 .p2align 4105 /* Medium copies: 33..128 bytes. */106L(copy32_128):107 ldp A_q, B_q, [src]108 ldp C_q, D_q, [srcend, -32]109 cmp count, 64110 b.hi L(copy128)111 stp A_q, B_q, [dstin]112 stp C_q, D_q, [dstend, -32]113 ret114 115 .p2align 4116 /* Copy 65..128 bytes. */117L(copy128):118 ldp E_q, F_q, [src, 32]119 cmp count, 96120 b.ls L(copy96)121 ldp G_q, H_q, [srcend, -64]122 stp G_q, H_q, [dstend, -64]123L(copy96):124 stp A_q, B_q, [dstin]125 stp E_q, F_q, [dstin, 32]126 stp C_q, D_q, [dstend, -32]127 ret128 129 /* Copy more than 128 bytes. */130L(copy_long):131 /* Use backwards copy if there is an overlap. */132 sub tmp1, dstin, src133 cmp tmp1, count134 b.lo L(copy_long_backwards)135 136 /* Copy 16 bytes and then align src to 16-byte alignment. */137 ldr D_q, [src]138 and tmp1, src, 15139 bic src, src, 15140 sub dst, dstin, tmp1141 add count, count, tmp1 /* Count is now 16 too large. */142 ldp A_q, B_q, [src, 16]143 str D_q, [dstin]144 ldp C_q, D_q, [src, 48]145 subs count, count, 128 + 16 /* Test and readjust count. */146 b.ls L(copy64_from_end)147L(loop64):148 stp A_q, B_q, [dst, 16]149 ldp A_q, B_q, [src, 80]150 stp C_q, D_q, [dst, 48]151 ldp C_q, D_q, [src, 112]152 add src, src, 64153 add dst, dst, 64154 subs count, count, 64155 b.hi L(loop64)156 157 /* Write the last iteration and copy 64 bytes from the end. */158L(copy64_from_end):159 ldp E_q, F_q, [srcend, -64]160 stp A_q, B_q, [dst, 16]161 ldp A_q, B_q, [srcend, -32]162 stp C_q, D_q, [dst, 48]163 stp E_q, F_q, [dstend, -64]164 stp A_q, B_q, [dstend, -32]165 ret166 167 /* Large backwards copy for overlapping copies.168 Copy 16 bytes and then align srcend to 16-byte alignment. */169L(copy_long_backwards):170 cbz tmp1, L(copy0)171 ldr D_q, [srcend, -16]172 and tmp1, srcend, 15173 bic srcend, srcend, 15174 sub count, count, tmp1175 ldp A_q, B_q, [srcend, -32]176 str D_q, [dstend, -16]177 ldp C_q, D_q, [srcend, -64]178 sub dstend, dstend, tmp1179 subs count, count, 128180 b.ls L(copy64_from_start)181 182L(loop64_backwards):183 stp A_q, B_q, [dstend, -32]184 ldp A_q, B_q, [srcend, -96]185 stp C_q, D_q, [dstend, -64]186 ldp C_q, D_q, [srcend, -128]187 sub srcend, srcend, 64188 sub dstend, dstend, 64189 subs count, count, 64190 b.hi L(loop64_backwards)191 192 /* Write the last iteration and copy 64 bytes from the start. */193L(copy64_from_start):194 ldp E_q, F_q, [src, 32]195 stp A_q, B_q, [dstend, -32]196 ldp A_q, B_q, [src]197 stp C_q, D_q, [dstend, -64]198 stp E_q, F_q, [dstin, 32]199 stp A_q, B_q, [dstin]200 ret201 202END (__memcpy_aarch64_simd)203