411 lines · c
1//===-- Portable SIMD library similar to stdx::simd -------------*- C++ -*-===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// This file provides a generic interface into fixed-size SIMD instructions10// using the clang vector type. The API shares some similarities with the11// stdx::simd proposal, but instead chooses to use vectors as primitive types12// with several extra helper functions.13//14//===----------------------------------------------------------------------===//15 16#include "hdr/stdint_proxy.h"17#include "src/__support/CPP/algorithm.h"18#include "src/__support/CPP/limits.h"19#include "src/__support/CPP/tuple.h"20#include "src/__support/CPP/type_traits.h"21#include "src/__support/CPP/utility/integer_sequence.h"22#include "src/__support/macros/attributes.h"23#include "src/__support/macros/config.h"24 25#include <stddef.h>26 27#ifndef LLVM_LIBC_SRC___SUPPORT_CPP_SIMD_H28#define LLVM_LIBC_SRC___SUPPORT_CPP_SIMD_H29 30#if LIBC_HAS_VECTOR_TYPE31 32namespace LIBC_NAMESPACE_DECL {33namespace cpp {34 35namespace internal {36 37#if defined(LIBC_TARGET_CPU_HAS_AVX512F)38template <typename T>39LIBC_INLINE_VAR constexpr size_t native_vector_size = 64 / sizeof(T);40#elif defined(LIBC_TARGET_CPU_HAS_AVX2)41template <typename T>42LIBC_INLINE_VAR constexpr size_t native_vector_size = 32 / sizeof(T);43#elif defined(LIBC_TARGET_CPU_HAS_SSE2) || defined(LIBC_TARGET_CPU_HAS_ARM_NEON)44template <typename T>45LIBC_INLINE_VAR constexpr size_t native_vector_size = 16 / sizeof(T);46#else47template <typename T> LIBC_INLINE constexpr size_t native_vector_size = 1;48#endif49 50} // namespace internal51 52// Type aliases.53template <typename T, size_t N>54using fixed_size_simd = T [[clang::ext_vector_type(N)]];55template <typename T, size_t N = internal::native_vector_size<T>>56using simd = T [[clang::ext_vector_type(N)]];57template <typename T>58using simd_mask = simd<bool, internal::native_vector_size<T>>;59 60// Type trait helpers.61template <typename T>62struct simd_size : cpp::integral_constant<size_t, __builtin_vectorelements(T)> {63};64template <class T> constexpr size_t simd_size_v = simd_size<T>::value;65 66template <typename T> struct is_simd : cpp::integral_constant<bool, false> {};67template <typename T, unsigned N>68struct is_simd<simd<T, N>> : cpp::integral_constant<bool, true> {};69template <class T> constexpr bool is_simd_v = is_simd<T>::value;70 71template <typename T>72struct is_simd_mask : cpp::integral_constant<bool, false> {};73template <unsigned N>74struct is_simd_mask<simd<bool, N>> : cpp::integral_constant<bool, true> {};75template <class T> constexpr bool is_simd_mask_v = is_simd_mask<T>::value;76 77template <typename T> struct simd_element_type;78template <typename T, size_t N> struct simd_element_type<simd<T, N>> {79 using type = T;80};81template <typename T>82using simd_element_type_t = typename simd_element_type<T>::type;83namespace internal {84 85template <typename T>86using get_as_integer_type_t = unsigned _BitInt(sizeof(T) * CHAR_BIT);87 88template <typename T> LIBC_INLINE constexpr T poison() {89 return __builtin_nondeterministic_value(T());90}91 92template <typename T, size_t N, size_t OriginalSize, size_t... Indices>93LIBC_INLINE constexpr static cpp::simd<T, sizeof...(Indices)>94extend(cpp::simd<T, N> x, cpp::index_sequence<Indices...>) {95 return __builtin_shufflevector(96 x, x, (Indices < OriginalSize ? static_cast<int>(Indices) : -1)...);97}98 99template <typename T, size_t N, size_t TargetSize, size_t OriginalSize>100LIBC_INLINE constexpr static auto extend(cpp::simd<T, N> x) {101 // Recursively resize an input vector to the target size, increasing its size102 // by at most double the input size each step due to shufflevector limitation.103 if constexpr (N == TargetSize)104 return x;105 else if constexpr (TargetSize <= 2 * N)106 return extend<T, N, TargetSize>(x, cpp::make_index_sequence<TargetSize>{});107 else108 return extend<T, 2 * N, TargetSize, OriginalSize>(109 extend<T, N, 2 * N>(x, cpp::make_index_sequence<2 * N>{}));110}111 112template <typename T, size_t N, size_t M, size_t... Indices>113LIBC_INLINE constexpr static cpp::simd<T, N + M>114concat(cpp::simd<T, N> x, cpp::simd<T, M> y, cpp::index_sequence<Indices...>) {115 constexpr size_t Size = cpp::max(N, M);116 auto remap = [](size_t idx) -> int {117 if (idx < N)118 return static_cast<int>(idx);119 if (idx < N + M)120 return static_cast<int>((idx - N) + Size);121 return -1;122 };123 124 // Extend the input vectors until they are the same size, then use the indices125 // to shuffle in only the indices that correspond to the original values.126 auto x_ext = extend<T, N, Size, N>(x);127 auto y_ext = extend<T, M, Size, M>(y);128 return __builtin_shufflevector(x_ext, y_ext, remap(Indices)...);129}130 131template <typename T, size_t N, size_t Count, size_t Offset, size_t... Indices>132LIBC_INLINE constexpr static cpp::simd<T, Count>133slice(cpp::simd<T, N> x, cpp::index_sequence<Indices...>) {134 return __builtin_shufflevector(x, x, (Offset + Indices)...);135}136 137template <typename T, size_t N, size_t Offset, size_t Head, size_t... Tail>138LIBC_INLINE constexpr static auto split(cpp::simd<T, N> x) {139 // Recursively splits the input vector by walking the variadic template list,140 // increasing our current head each call.141 auto result = cpp::make_tuple(142 slice<T, N, Head, Offset>(x, cpp::make_index_sequence<Head>{}));143 if constexpr (sizeof...(Tail) > 0)144 return cpp::tuple_cat(result, split<T, N, Offset + Head, Tail...>(x));145 else146 return result;147}148 149// Helper trait150template <typename T>151using enable_if_integral_t = cpp::enable_if_t<cpp::is_integral_v<T>, T>;152 153template <typename T>154using enable_if_simd_t = cpp::enable_if_t<is_simd_v<T>, bool>;155 156} // namespace internal157 158// Casting.159template <typename To, typename From, size_t N>160LIBC_INLINE constexpr static simd<To, N> simd_cast(simd<From, N> v) {161 return __builtin_convertvector(v, simd<To, N>);162}163 164// SIMD mask operations.165template <typename T, size_t N, internal::enable_if_integral_t<T> = 0>166LIBC_INLINE constexpr static bool all_of(simd<T, N> v) {167 return __builtin_reduce_and(simd_cast<bool>(v));168}169template <typename T, size_t N, internal::enable_if_integral_t<T> = 0>170LIBC_INLINE constexpr static bool any_of(simd<T, N> v) {171 return __builtin_reduce_or(simd_cast<bool>(v));172}173template <typename T, size_t N, internal::enable_if_integral_t<T> = 0>174LIBC_INLINE constexpr static bool none_of(simd<T, N> v) {175 return !any_of(v);176}177template <typename T, size_t N, internal::enable_if_integral_t<T> = 0>178LIBC_INLINE constexpr static bool some_of(simd<T, N> v) {179 return any_of(v) && !all_of(v);180}181template <typename T, size_t N, internal::enable_if_integral_t<T> = 0>182LIBC_INLINE constexpr static int popcount(simd<T, N> v) {183 return __builtin_popcountg(v);184}185template <typename T, size_t N, internal::enable_if_integral_t<T> = 0>186LIBC_INLINE constexpr static int find_first_set(simd<T, N> v) {187 return __builtin_ctzg(simd_cast<bool>(v));188}189template <typename T, size_t N, internal::enable_if_integral_t<T> = 0>190LIBC_INLINE constexpr static int find_last_set(simd<T, N> v) {191 constexpr size_t size = simd_size_v<simd<T, N>>;192 return size - 1 - __builtin_clzg(simd_cast<bool>(v));193}194 195// Elementwise operations.196template <typename T, size_t N>197LIBC_INLINE constexpr static simd<T, N> min(simd<T, N> x, simd<T, N> y) {198 return __builtin_elementwise_min(x, y);199}200template <typename T, size_t N>201LIBC_INLINE constexpr static simd<T, N> max(simd<T, N> x, simd<T, N> y) {202 return __builtin_elementwise_max(x, y);203}204 205template <typename T, size_t N>206LIBC_INLINE constexpr static simd<T, N> abs(simd<T, N> x) {207 return __builtin_elementwise_abs(x);208}209template <typename T, size_t N>210LIBC_INLINE constexpr static simd<T, N> fma(simd<T, N> x, simd<T, N> y,211 simd<T, N> z) {212 return __builtin_elementwise_fma(x, y, z);213}214template <typename T, size_t N>215LIBC_INLINE constexpr static simd<T, N> ceil(simd<T, N> x) {216 return __builtin_elementwise_ceil(x);217}218template <typename T, size_t N>219LIBC_INLINE constexpr static simd<T, N> floor(simd<T, N> x) {220 return __builtin_elementwise_floor(x);221}222template <typename T, size_t N>223LIBC_INLINE constexpr static simd<T, N> roundeven(simd<T, N> x) {224 return __builtin_elementwise_roundeven(x);225}226template <typename T, size_t N>227LIBC_INLINE constexpr static simd<T, N> round(simd<T, N> x) {228 return __builtin_elementwise_round(x);229}230template <typename T, size_t N>231LIBC_INLINE constexpr static simd<T, N> trunc(simd<T, N> x) {232 return __builtin_elementwise_trunc(x);233}234template <typename T, size_t N>235LIBC_INLINE constexpr static simd<T, N> nearbyint(simd<T, N> x) {236 return __builtin_elementwise_nearbyint(x);237}238template <typename T, size_t N>239LIBC_INLINE constexpr static simd<T, N> rint(simd<T, N> x) {240 return __builtin_elementwise_rint(x);241}242template <typename T, size_t N>243LIBC_INLINE constexpr static simd<T, N> canonicalize(simd<T, N> x) {244 return __builtin_elementwise_canonicalize(x);245}246template <typename T, size_t N>247LIBC_INLINE constexpr static simd<T, N> copysign(simd<T, N> x, simd<T, N> y) {248 return __builtin_elementwise_copysign(x, y);249}250template <typename T, size_t N>251LIBC_INLINE constexpr static simd<T, N> fmod(simd<T, N> x, simd<T, N> y) {252 return __builtin_elementwise_fmod(x, y);253}254 255// Reduction operations.256template <typename T, size_t N, typename Op = cpp::plus<>>257LIBC_INLINE constexpr static T reduce(simd<T, N> v, Op op = {}) {258 return reduce(v, op);259}260template <typename T, size_t N>261LIBC_INLINE constexpr static T reduce(simd<T, N> v, cpp::plus<>) {262 return __builtin_reduce_add(v);263}264template <typename T, size_t N>265LIBC_INLINE constexpr static T reduce(simd<T, N> v, cpp::multiplies<>) {266 return __builtin_reduce_mul(v);267}268template <typename T, size_t N>269LIBC_INLINE constexpr static T reduce(simd<T, N> v, cpp::bit_and<>) {270 return __builtin_reduce_and(v);271}272template <typename T, size_t N>273LIBC_INLINE constexpr static T reduce(simd<T, N> v, cpp::bit_or<>) {274 return __builtin_reduce_or(v);275}276template <typename T, size_t N>277LIBC_INLINE constexpr static T reduce(simd<T, N> v, cpp::bit_xor<>) {278 return __builtin_reduce_xor(v);279}280template <typename T, size_t N>281LIBC_INLINE constexpr static T hmin(simd<T, N> v) {282 return __builtin_reduce_min(v);283}284template <typename T, size_t N>285LIBC_INLINE constexpr static T hmax(simd<T, N> v) {286 return __builtin_reduce_max(v);287}288 289// Accessor helpers.290template <typename T>291LIBC_INLINE T constexpr static load(const void *ptr, bool aligned = false) {292 if (aligned)293 ptr = __builtin_assume_aligned(ptr, alignof(T));294 T tmp;295 __builtin_memcpy_inline(296 &tmp, reinterpret_cast<const simd_element_type_t<T> *>(ptr), sizeof(T));297 return tmp;298}299template <typename T, internal::enable_if_simd_t<T> = 0>300LIBC_INLINE constexpr static void store(T v, void *ptr, bool aligned = false) {301 if (aligned)302 ptr = __builtin_assume_aligned(ptr, alignof(T));303 __builtin_memcpy_inline(ptr, &v, sizeof(T));304}305template <typename T, internal::enable_if_simd_t<T> = 0>306LIBC_INLINE constexpr static T307load_masked(simd<bool, simd_size_v<T>> mask, const void *ptr,308 T passthru = internal::poison<T>(), bool aligned = false) {309 if (aligned)310 ptr = __builtin_assume_aligned(ptr, alignof(T));311 return __builtin_masked_load(312 mask, reinterpret_cast<const simd_element_type_t<T> *>(ptr), passthru);313}314template <typename T, internal::enable_if_simd_t<T> = 0>315LIBC_INLINE constexpr static void store_masked(simd<bool, simd_size_v<T>> mask,316 T v, void *ptr,317 bool aligned = false) {318 if (aligned)319 ptr = __builtin_assume_aligned(ptr, alignof(T));320 __builtin_masked_store(mask, v,321 reinterpret_cast<simd_element_type_t<T> *>(ptr));322}323template <typename T, typename Idx, internal::enable_if_simd_t<T> = 0>324LIBC_INLINE constexpr static T gather(simd<bool, simd_size_v<T>> mask, Idx idx,325 const void *base, bool aligned = false) {326 if (aligned)327 base = __builtin_assume_aligned(base, alignof(T));328 return __builtin_masked_gather(329 mask, idx, reinterpret_cast<const simd_element_type_t<T> *>(base));330}331template <typename T, typename Idx, internal::enable_if_simd_t<T> = 0>332LIBC_INLINE constexpr static void scatter(simd<bool, simd_size_v<T>> mask,333 Idx idx, T v, void *base,334 bool aligned = false) {335 if (aligned)336 base = __builtin_assume_aligned(base, alignof(T));337 __builtin_masked_scatter(mask, idx, v,338 reinterpret_cast<simd_element_type_t<T> *>(base));339}340template <typename T, internal::enable_if_simd_t<T> = 0>341LIBC_INLINE constexpr static T342expand(simd<bool, simd_size_v<T>> mask, const void *ptr,343 T passthru = internal::poison<T>(), bool aligned = false) {344 if (aligned)345 ptr = __builtin_assume_aligned(ptr, alignof(T));346 return __builtin_masked_expand_load(347 mask, reinterpret_cast<const simd_element_type_t<T> *>(ptr), passthru);348}349template <typename T, internal::enable_if_simd_t<T> = 0>350LIBC_INLINE constexpr static void compress(simd<bool, simd_size_v<T>> mask, T v,351 void *ptr, bool aligned = false) {352 if (aligned)353 ptr = __builtin_assume_aligned(ptr, alignof(T));354 __builtin_masked_compress_store(355 mask, v, reinterpret_cast<simd_element_type_t<T> *>(ptr));356}357 358// Construction helpers.359template <typename T, size_t N>360LIBC_INLINE constexpr static simd<T, N> splat(T v) {361 return simd<T, N>(v);362}363template <typename T> LIBC_INLINE constexpr static simd<T> splat(T v) {364 return splat<T, simd_size_v<simd<T>>>(v);365}366template <typename T, unsigned N>367LIBC_INLINE constexpr static simd<T, N> iota(T base = T(0), T step = T(1)) {368 simd<T, N> v{};369 for (unsigned i = 0; i < N; ++i)370 v[i] = base + T(i) * step;371 return v;372}373template <typename T>374LIBC_INLINE constexpr static simd<T> iota(T base = T(0), T step = T(1)) {375 return iota<T, simd_size_v<simd<T>>>(base, step);376}377 378// Conditional helpers.379template <typename T, size_t N>380LIBC_INLINE constexpr static simd<T, N> select(simd<bool, N> m, simd<T, N> x,381 simd<T, N> y) {382 return m ? x : y;383}384 385// Shuffling helpers.386template <typename T, size_t N, size_t M>387LIBC_INLINE constexpr static auto concat(cpp::simd<T, N> x, cpp::simd<T, M> y) {388 return internal::concat(x, y, make_index_sequence<N + M>{});389}390template <typename T, size_t N, size_t M, typename... Rest>391LIBC_INLINE constexpr static auto concat(cpp::simd<T, N> x, cpp::simd<T, M> y,392 Rest... rest) {393 auto xy = concat(x, y);394 if constexpr (sizeof...(Rest))395 return concat(xy, rest...);396 else397 return xy;398}399template <size_t... Sizes, typename T, size_t N> auto split(cpp::simd<T, N> x) {400 static_assert((... + Sizes) == N, "split sizes must sum to vector size");401 return internal::split<T, N, 0, Sizes...>(x);402}403 404// TODO: where expressions, scalar overloads, ABI types.405 406} // namespace cpp407} // namespace LIBC_NAMESPACE_DECL408 409#endif // LIBC_HAS_VECTOR_TYPE410#endif411