148 lines · c
1/*2 * AArch64-specific checksum implementation using NEON3 *4 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.5 * See https://llvm.org/LICENSE.txt for license information.6 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception7 */8 9#include "networking.h"10#include "../chksum_common.h"11 12#ifndef __ARM_NEON13#pragma GCC target("+simd")14#endif15 16#include <arm_neon.h>17 18always_inline19static inline uint64_t20slurp_head64(const void **pptr, uint32_t *nbytes)21{22 Assert(*nbytes >= 8);23 uint64_t sum = 0;24 uint32_t off = (uintptr_t) *pptr % 8;25 if (likely(off != 0))26 {27 /* Get rid of bytes 0..off-1 */28 const unsigned char *ptr64 = align_ptr(*pptr, 8);29 uint64_t mask = ALL_ONES << (CHAR_BIT * off);30 uint64_t val = load64(ptr64) & mask;31 /* Fold 64-bit sum to 33 bits */32 sum = val >> 32;33 sum += (uint32_t) val;34 *pptr = ptr64 + 8;35 *nbytes -= 8 - off;36 }37 return sum;38}39 40always_inline41static inline uint64_t42slurp_tail64(uint64_t sum, const void *ptr, uint32_t nbytes)43{44 Assert(nbytes < 8);45 if (likely(nbytes != 0))46 {47 /* Get rid of bytes 7..nbytes */48 uint64_t mask = ALL_ONES >> (CHAR_BIT * (8 - nbytes));49 Assert(__builtin_popcountl(mask) / CHAR_BIT == nbytes);50 uint64_t val = load64(ptr) & mask;51 sum += val >> 32;52 sum += (uint32_t) val;53 nbytes = 0;54 }55 Assert(nbytes == 0);56 return sum;57}58 59unsigned short60__chksum_aarch64_simd(const void *ptr, unsigned int nbytes)61{62 bool swap = (uintptr_t) ptr & 1;63 uint64_t sum;64 65 if (unlikely(nbytes < 50))66 {67 sum = slurp_small(ptr, nbytes);68 swap = false;69 goto fold;70 }71 72 /* 8-byte align pointer */73 Assert(nbytes >= 8);74 sum = slurp_head64(&ptr, &nbytes);75 Assert(((uintptr_t) ptr & 7) == 0);76 77 const uint32_t *may_alias ptr32 = ptr;78 79 uint64x2_t vsum0 = { 0, 0 };80 uint64x2_t vsum1 = { 0, 0 };81 uint64x2_t vsum2 = { 0, 0 };82 uint64x2_t vsum3 = { 0, 0 };83 84 /* Sum groups of 64 bytes */85 for (uint32_t i = 0; i < nbytes / 64; i++)86 {87 uint32x4_t vtmp0 = vld1q_u32(ptr32);88 uint32x4_t vtmp1 = vld1q_u32(ptr32 + 4);89 uint32x4_t vtmp2 = vld1q_u32(ptr32 + 8);90 uint32x4_t vtmp3 = vld1q_u32(ptr32 + 12);91 vsum0 = vpadalq_u32(vsum0, vtmp0);92 vsum1 = vpadalq_u32(vsum1, vtmp1);93 vsum2 = vpadalq_u32(vsum2, vtmp2);94 vsum3 = vpadalq_u32(vsum3, vtmp3);95 ptr32 += 16;96 }97 nbytes %= 64;98 99 /* Fold vsum2 and vsum3 into vsum0 and vsum1 */100 vsum0 = vpadalq_u32(vsum0, vreinterpretq_u32_u64(vsum2));101 vsum1 = vpadalq_u32(vsum1, vreinterpretq_u32_u64(vsum3));102 103 /* Add any trailing group of 32 bytes */104 if (nbytes & 32)105 {106 uint32x4_t vtmp0 = vld1q_u32(ptr32);107 uint32x4_t vtmp1 = vld1q_u32(ptr32 + 4);108 vsum0 = vpadalq_u32(vsum0, vtmp0);109 vsum1 = vpadalq_u32(vsum1, vtmp1);110 ptr32 += 8;111 nbytes -= 32;112 }113 Assert(nbytes < 32);114 115 /* Fold vsum1 into vsum0 */116 vsum0 = vpadalq_u32(vsum0, vreinterpretq_u32_u64(vsum1));117 118 /* Add any trailing group of 16 bytes */119 if (nbytes & 16)120 {121 uint32x4_t vtmp = vld1q_u32(ptr32);122 vsum0 = vpadalq_u32(vsum0, vtmp);123 ptr32 += 4;124 nbytes -= 16;125 }126 Assert(nbytes < 16);127 128 /* Add any trailing group of 8 bytes */129 if (nbytes & 8)130 {131 uint32x2_t vtmp = vld1_u32(ptr32);132 vsum0 = vaddw_u32(vsum0, vtmp);133 ptr32 += 2;134 nbytes -= 8;135 }136 Assert(nbytes < 8);137 138 uint64_t val = vaddlvq_u32(vreinterpretq_u32_u64(vsum0));139 sum += val >> 32;140 sum += (uint32_t) val;141 142 /* Handle any trailing 0..7 bytes */143 sum = slurp_tail64(sum, ptr32, nbytes);144 145fold:146 return fold_and_swap(sum, swap);147}148