118 lines · plain
1/*2 * strchrnul - find a character or nul in a string3 *4 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.5 * See https://llvm.org/LICENSE.txt for license information.6 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception7 */8 9/* Assumptions:10 *11 * ARMv8-a, AArch6412 * Neon Available.13 */14 15#include "../asmdefs.h"16 17/* Arguments and results. */18#define srcin x019#define chrin w120 21#define result x022 23#define src x224#define tmp1 x325#define wtmp2 w426#define tmp3 x527 28#define vrepchr v029#define vdata1 v130#define vdata2 v231#define vhas_nul1 v332#define vhas_nul2 v433#define vhas_chr1 v534#define vhas_chr2 v635#define vrepmask v736#define vend1 v1637 38/* Core algorithm.39 40 For each 32-byte hunk we calculate a 64-bit syndrome value, with41 two bits per byte (LSB is always in bits 0 and 1, for both big42 and little-endian systems). For each tuple, bit 0 is set iff43 the relevant byte matched the requested character or nul. Since the44 bits in the syndrome reflect exactly the order in which things occur45 in the original string a count_trailing_zeros() operation will46 identify exactly which byte is causing the termination. */47 48/* Locals and temporaries. */49 50ENTRY (__strchrnul_aarch64)51 /* Magic constant 0x40100401 to allow us to identify which lane52 matches the termination condition. */53 mov wtmp2, #0x040154 movk wtmp2, #0x4010, lsl #1655 dup vrepchr.16b, chrin56 bic src, srcin, #31 /* Work with aligned 32-byte hunks. */57 dup vrepmask.4s, wtmp258 ands tmp1, srcin, #3159 b.eq L(loop)60 61 /* Input string is not 32-byte aligned. Rather than forcing62 the padding bytes to a safe value, we calculate the syndrome63 for all the bytes, but then mask off those bits of the64 syndrome that are related to the padding. */65 ld1 {vdata1.16b, vdata2.16b}, [src], #3266 neg tmp1, tmp167 cmeq vhas_nul1.16b, vdata1.16b, #068 cmeq vhas_chr1.16b, vdata1.16b, vrepchr.16b69 cmeq vhas_nul2.16b, vdata2.16b, #070 cmeq vhas_chr2.16b, vdata2.16b, vrepchr.16b71 orr vhas_chr1.16b, vhas_chr1.16b, vhas_nul1.16b72 orr vhas_chr2.16b, vhas_chr2.16b, vhas_nul2.16b73 and vhas_chr1.16b, vhas_chr1.16b, vrepmask.16b74 and vhas_chr2.16b, vhas_chr2.16b, vrepmask.16b75 lsl tmp1, tmp1, #176 addp vend1.16b, vhas_chr1.16b, vhas_chr2.16b // 256->12877 mov tmp3, #~078 addp vend1.16b, vend1.16b, vend1.16b // 128->6479 lsr tmp1, tmp3, tmp180 81 mov tmp3, vend1.d[0]82 bic tmp1, tmp3, tmp1 // Mask padding bits.83 cbnz tmp1, L(tail)84 85L(loop):86 ld1 {vdata1.16b, vdata2.16b}, [src], #3287 cmeq vhas_nul1.16b, vdata1.16b, #088 cmeq vhas_chr1.16b, vdata1.16b, vrepchr.16b89 cmeq vhas_nul2.16b, vdata2.16b, #090 cmeq vhas_chr2.16b, vdata2.16b, vrepchr.16b91 /* Use a fast check for the termination condition. */92 orr vhas_chr1.16b, vhas_nul1.16b, vhas_chr1.16b93 orr vhas_chr2.16b, vhas_nul2.16b, vhas_chr2.16b94 orr vend1.16b, vhas_chr1.16b, vhas_chr2.16b95 addp vend1.2d, vend1.2d, vend1.2d96 mov tmp1, vend1.d[0]97 cbz tmp1, L(loop)98 99 /* Termination condition found. Now need to establish exactly why100 we terminated. */101 and vhas_chr1.16b, vhas_chr1.16b, vrepmask.16b102 and vhas_chr2.16b, vhas_chr2.16b, vrepmask.16b103 addp vend1.16b, vhas_chr1.16b, vhas_chr2.16b // 256->128104 addp vend1.16b, vend1.16b, vend1.16b // 128->64105 106 mov tmp1, vend1.d[0]107L(tail):108 /* Count the trailing zeros, by bit reversing... */109 rbit tmp1, tmp1110 /* Re-bias source. */111 sub src, src, #32112 clz tmp1, tmp1 /* ... and counting the leading zeros. */113 /* tmp1 is twice the offset into the fragment. */114 add result, src, tmp1, lsr #1115 ret116 117END (__strchrnul_aarch64)118