262 lines · plain
1//===----------------------------------------------------------------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8///9/// \file10/// This file contains assembly-optimized implementations of Scalable Matrix11/// Extension (SME) compatible memset and memchr functions.12///13/// These implementations depend on unaligned access and floating-point support.14///15/// Routines taken from libc/AOR_v20.02/string/aarch64.16///17//===----------------------------------------------------------------------===//18 19#include "../assembly.h"20 21//22// __arm_sc_memset23//24 25#define dstin x026#define val x127#define valw w128#define count x229#define dst x330#define dstend2 x431#define zva_val x532 33DEFINE_COMPILERRT_FUNCTION(__arm_sc_memset)34#ifdef __ARM_FEATURE_SVE35 mov z0.b, valw36#else37 bfi valw, valw, #8, #838 bfi valw, valw, #16, #1639 bfi val, val, #32, #3240 fmov d0, val41 fmov v0.d[1], val42#endif43 add dstend2, dstin, count44 45 cmp count, 9646 b.hi 7f // set_long47 cmp count, 1648 b.hs 4f // set_medium49 mov val, v0.D[0]50 51 /* Set 0..15 bytes. */52 tbz count, 3, 1f53 str val, [dstin]54 str val, [dstend2, -8]55 ret56 nop571: tbz count, 2, 2f58 str valw, [dstin]59 str valw, [dstend2, -4]60 ret612: cbz count, 3f62 strb valw, [dstin]63 tbz count, 1, 3f64 strh valw, [dstend2, -2]653: ret66 67 /* Set 17..96 bytes. */684: // set_medium69 str q0, [dstin]70 tbnz count, 6, 6f // set9671 str q0, [dstend2, -16]72 tbz count, 5, 5f73 str q0, [dstin, 16]74 str q0, [dstend2, -32]755: ret76 77 .p2align 478 /* Set 64..96 bytes. Write 64 bytes from the start and79 32 bytes from the end. */806: // set9681 str q0, [dstin, 16]82 stp q0, q0, [dstin, 32]83 stp q0, q0, [dstend2, -32]84 ret85 86 .p2align 4877: // set_long88 and valw, valw, 25589 bic dst, dstin, 1590 str q0, [dstin]91 cmp count, 16092 ccmp valw, 0, 0, hs93 b.ne 9f // no_zva94 95#ifndef SKIP_ZVA_CHECK96 mrs zva_val, dczid_el097 and zva_val, zva_val, 3198 cmp zva_val, 4 /* ZVA size is 64 bytes. */99 b.ne 9f // no_zva100#endif101 str q0, [dst, 16]102 stp q0, q0, [dst, 32]103 bic dst, dst, 63104 sub count, dstend2, dst /* Count is now 64 too large. */105 sub count, count, 128 /* Adjust count and bias for loop. */106 107 .p2align 41088: // zva_loop109 add dst, dst, 64110 dc zva, dst111 subs count, count, 64112 b.hi 8b // zva_loop113 stp q0, q0, [dstend2, -64]114 stp q0, q0, [dstend2, -32]115 ret116 1179: // no_zva118 sub count, dstend2, dst /* Count is 16 too large. */119 sub dst, dst, 16 /* Dst is biased by -32. */120 sub count, count, 64 + 16 /* Adjust count and bias for loop. */12110: // no_zva_loop122 stp q0, q0, [dst, 32]123 stp q0, q0, [dst, 64]!124 subs count, count, 64125 b.hi 10b // no_zva_loop126 stp q0, q0, [dstend2, -64]127 stp q0, q0, [dstend2, -32]128 ret129END_COMPILERRT_FUNCTION(__arm_sc_memset)130 131//132// __arm_sc_memchr133//134 135#define srcin x0136#define chrin w1137#define cntin x2138 139#define result x0140 141#define src x3142#define tmp x4143#define wtmp2 w5144#define synd x6145#define soff x9146#define cntrem x10147 148#define vrepchr v0149#define vdata1 v1150#define vdata2 v2151#define vhas_chr1 v3152#define vhas_chr2 v4153#define vrepmask v5154#define vend v6155 156/*157 * Core algorithm:158 *159 * For each 32-byte chunk we calculate a 64-bit syndrome value, with two bits160 * per byte. For each tuple, bit 0 is set if the relevant byte matched the161 * requested character and bit 1 is not used (faster than using a 32bit162 * syndrome). Since the bits in the syndrome reflect exactly the order in which163 * things occur in the original string, counting trailing zeros allows to164 * identify exactly which byte has matched.165 */166 167DEFINE_COMPILERRT_FUNCTION(__arm_sc_memchr)168 /* Do not dereference srcin if no bytes to compare. */169 cbz cntin, 4f170 /*171 * Magic constant 0x40100401 allows us to identify which lane matches172 * the requested byte.173 */174 mov wtmp2, #0x0401175 movk wtmp2, #0x4010, lsl #16176 dup vrepchr.16b, chrin177 /* Work with aligned 32-byte chunks */178 bic src, srcin, #31179 dup vrepmask.4s, wtmp2180 ands soff, srcin, #31181 and cntrem, cntin, #31182 b.eq 0f183 184 /*185 * Input string is not 32-byte aligned. We calculate the syndrome186 * value for the aligned 32 bytes block containing the first bytes187 * and mask the irrelevant part.188 */189 190 ld1 {vdata1.16b, vdata2.16b}, [src], #32191 sub tmp, soff, #32192 adds cntin, cntin, tmp193 cmeq vhas_chr1.16b, vdata1.16b, vrepchr.16b194 cmeq vhas_chr2.16b, vdata2.16b, vrepchr.16b195 and vhas_chr1.16b, vhas_chr1.16b, vrepmask.16b196 and vhas_chr2.16b, vhas_chr2.16b, vrepmask.16b197 addp vend.16b, vhas_chr1.16b, vhas_chr2.16b /* 256->128 */198 addp vend.16b, vend.16b, vend.16b /* 128->64 */199 mov synd, vend.d[0]200 /* Clear the soff*2 lower bits */201 lsl tmp, soff, #1202 lsr synd, synd, tmp203 lsl synd, synd, tmp204 /* The first block can also be the last */205 b.ls 2f206 /* Have we found something already? */207 cbnz synd, 3f208 2090: // loop210 ld1 {vdata1.16b, vdata2.16b}, [src], #32211 subs cntin, cntin, #32212 cmeq vhas_chr1.16b, vdata1.16b, vrepchr.16b213 cmeq vhas_chr2.16b, vdata2.16b, vrepchr.16b214 /* If we're out of data we finish regardless of the result */215 b.ls 1f216 /* Use a fast check for the termination condition */217 orr vend.16b, vhas_chr1.16b, vhas_chr2.16b218 addp vend.2d, vend.2d, vend.2d219 mov synd, vend.d[0]220 /* We're not out of data, loop if we haven't found the character */221 cbz synd, 0b222 2231: // end224 /* Termination condition found, let's calculate the syndrome value */225 and vhas_chr1.16b, vhas_chr1.16b, vrepmask.16b226 and vhas_chr2.16b, vhas_chr2.16b, vrepmask.16b227 addp vend.16b, vhas_chr1.16b, vhas_chr2.16b /* 256->128 */228 addp vend.16b, vend.16b, vend.16b /* 128->64 */229 mov synd, vend.d[0]230 /* Only do the clear for the last possible block */231 b.hi 3f232 2332: // masklast234 /* Clear the (32 - ((cntrem + soff) % 32)) * 2 upper bits */235 add tmp, cntrem, soff236 and tmp, tmp, #31237 sub tmp, tmp, #32238 neg tmp, tmp, lsl #1239 lsl synd, synd, tmp240 lsr synd, synd, tmp241 2423: // tail243 /* Count the trailing zeros using bit reversing */244 rbit synd, synd245 /* Compensate the last post-increment */246 sub src, src, #32247 /* Check that we have found a character */248 cmp synd, #0249 /* And count the leading zeros */250 clz synd, synd251 /* Compute the potential result */252 add result, src, synd, lsr #1253 /* Select result or NULL */254 csel result, xzr, result, eq255 ret256 2574: // zero_length258 mov result, #0259 ret260END_COMPILERRT_FUNCTION(__arm_sc_memchr)261 262