241 lines · plain
1/*2 * memcpy - copy memory area3 *4 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.5 * See https://llvm.org/LICENSE.txt for license information.6 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception7 */8 9/* Assumptions:10 *11 * ARMv8-a, AArch64, unaligned accesses.12 *13 */14 15#include "../asmdefs.h"16 17#define dstin x018#define src x119#define count x220#define dst x321#define srcend x422#define dstend x523#define A_l x624#define A_lw w625#define A_h x726#define B_l x827#define B_lw w828#define B_h x929#define C_l x1030#define C_lw w1031#define C_h x1132#define D_l x1233#define D_h x1334#define E_l x1435#define E_h x1536#define F_l x1637#define F_h x1738#define G_l count39#define G_h dst40#define H_l src41#define H_h srcend42#define tmp1 x1443 44/* This implementation handles overlaps and supports both memcpy and memmove45 from a single entry point. It uses unaligned accesses and branchless46 sequences to keep the code small, simple and improve performance.47 48 Copies are split into 3 main cases: small copies of up to 32 bytes, medium49 copies of up to 128 bytes, and large copies. The overhead of the overlap50 check is negligible since it is only required for large copies.51 52 Large copies use a software pipelined loop processing 64 bytes per iteration.53 The destination pointer is 16-byte aligned to minimize unaligned accesses.54 The loop tail is handled by always copying 64 bytes from the end.55*/56 57ENTRY (__memcpy_aarch64)58ENTRY_ALIAS (__memmove_aarch64)59 add srcend, src, count60 add dstend, dstin, count61 cmp count, 12862 b.hi L(copy_long)63 cmp count, 3264 b.hi L(copy32_128)65 66 /* Small copies: 0..32 bytes. */67 cmp count, 1668 b.lo L(copy16)69 ldp A_l, A_h, [src]70 ldp D_l, D_h, [srcend, -16]71 stp A_l, A_h, [dstin]72 stp D_l, D_h, [dstend, -16]73 ret74 75 /* Copy 8-15 bytes. */76L(copy16):77 tbz count, 3, L(copy8)78 ldr A_l, [src]79 ldr A_h, [srcend, -8]80 str A_l, [dstin]81 str A_h, [dstend, -8]82 ret83 84 .p2align 385 /* Copy 4-7 bytes. */86L(copy8):87 tbz count, 2, L(copy4)88 ldr A_lw, [src]89 ldr B_lw, [srcend, -4]90 str A_lw, [dstin]91 str B_lw, [dstend, -4]92 ret93 94 /* Copy 0..3 bytes using a branchless sequence. */95L(copy4):96 cbz count, L(copy0)97 lsr tmp1, count, 198 ldrb A_lw, [src]99 ldrb C_lw, [srcend, -1]100 ldrb B_lw, [src, tmp1]101 strb A_lw, [dstin]102 strb B_lw, [dstin, tmp1]103 strb C_lw, [dstend, -1]104L(copy0):105 ret106 107 .p2align 4108 /* Medium copies: 33..128 bytes. */109L(copy32_128):110 ldp A_l, A_h, [src]111 ldp B_l, B_h, [src, 16]112 ldp C_l, C_h, [srcend, -32]113 ldp D_l, D_h, [srcend, -16]114 cmp count, 64115 b.hi L(copy128)116 stp A_l, A_h, [dstin]117 stp B_l, B_h, [dstin, 16]118 stp C_l, C_h, [dstend, -32]119 stp D_l, D_h, [dstend, -16]120 ret121 122 .p2align 4123 /* Copy 65..128 bytes. */124L(copy128):125 ldp E_l, E_h, [src, 32]126 ldp F_l, F_h, [src, 48]127 cmp count, 96128 b.ls L(copy96)129 ldp G_l, G_h, [srcend, -64]130 ldp H_l, H_h, [srcend, -48]131 stp G_l, G_h, [dstend, -64]132 stp H_l, H_h, [dstend, -48]133L(copy96):134 stp A_l, A_h, [dstin]135 stp B_l, B_h, [dstin, 16]136 stp E_l, E_h, [dstin, 32]137 stp F_l, F_h, [dstin, 48]138 stp C_l, C_h, [dstend, -32]139 stp D_l, D_h, [dstend, -16]140 ret141 142 .p2align 4143 /* Copy more than 128 bytes. */144L(copy_long):145 /* Use backwards copy if there is an overlap. */146 sub tmp1, dstin, src147 cbz tmp1, L(copy0)148 cmp tmp1, count149 b.lo L(copy_long_backwards)150 151 /* Copy 16 bytes and then align dst to 16-byte alignment. */152 153 ldp D_l, D_h, [src]154 and tmp1, dstin, 15155 bic dst, dstin, 15156 sub src, src, tmp1157 add count, count, tmp1 /* Count is now 16 too large. */158 ldp A_l, A_h, [src, 16]159 stp D_l, D_h, [dstin]160 ldp B_l, B_h, [src, 32]161 ldp C_l, C_h, [src, 48]162 ldp D_l, D_h, [src, 64]!163 subs count, count, 128 + 16 /* Test and readjust count. */164 b.ls L(copy64_from_end)165 166L(loop64):167 stp A_l, A_h, [dst, 16]168 ldp A_l, A_h, [src, 16]169 stp B_l, B_h, [dst, 32]170 ldp B_l, B_h, [src, 32]171 stp C_l, C_h, [dst, 48]172 ldp C_l, C_h, [src, 48]173 stp D_l, D_h, [dst, 64]!174 ldp D_l, D_h, [src, 64]!175 subs count, count, 64176 b.hi L(loop64)177 178 /* Write the last iteration and copy 64 bytes from the end. */179L(copy64_from_end):180 ldp E_l, E_h, [srcend, -64]181 stp A_l, A_h, [dst, 16]182 ldp A_l, A_h, [srcend, -48]183 stp B_l, B_h, [dst, 32]184 ldp B_l, B_h, [srcend, -32]185 stp C_l, C_h, [dst, 48]186 ldp C_l, C_h, [srcend, -16]187 stp D_l, D_h, [dst, 64]188 stp E_l, E_h, [dstend, -64]189 stp A_l, A_h, [dstend, -48]190 stp B_l, B_h, [dstend, -32]191 stp C_l, C_h, [dstend, -16]192 ret193 194 .p2align 4195 196 /* Large backwards copy for overlapping copies.197 Copy 16 bytes and then align dst to 16-byte alignment. */198L(copy_long_backwards):199 ldp D_l, D_h, [srcend, -16]200 and tmp1, dstend, 15201 sub srcend, srcend, tmp1202 sub count, count, tmp1203 ldp A_l, A_h, [srcend, -16]204 stp D_l, D_h, [dstend, -16]205 ldp B_l, B_h, [srcend, -32]206 ldp C_l, C_h, [srcend, -48]207 ldp D_l, D_h, [srcend, -64]!208 sub dstend, dstend, tmp1209 subs count, count, 128210 b.ls L(copy64_from_start)211 212L(loop64_backwards):213 stp A_l, A_h, [dstend, -16]214 ldp A_l, A_h, [srcend, -16]215 stp B_l, B_h, [dstend, -32]216 ldp B_l, B_h, [srcend, -32]217 stp C_l, C_h, [dstend, -48]218 ldp C_l, C_h, [srcend, -48]219 stp D_l, D_h, [dstend, -64]!220 ldp D_l, D_h, [srcend, -64]!221 subs count, count, 64222 b.hi L(loop64_backwards)223 224 /* Write the last iteration and copy 64 bytes from the start. */225L(copy64_from_start):226 ldp G_l, G_h, [src, 48]227 stp A_l, A_h, [dstend, -16]228 ldp A_l, A_h, [src, 32]229 stp B_l, B_h, [dstend, -32]230 ldp B_l, B_h, [src, 16]231 stp C_l, C_h, [dstend, -48]232 ldp C_l, C_h, [src]233 stp D_l, D_h, [dstend, -64]234 stp G_l, G_h, [dstin, 48]235 stp A_l, A_h, [dstin, 32]236 stp B_l, B_h, [dstin, 16]237 stp C_l, C_h, [dstin]238 ret239 240END (__memcpy_aarch64)241