160 lines · c
1//===----------------------------------------------------------------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8 9#ifndef _LIBCPP___CXX03___ALGORITHM_SIMD_UTILS_H10#define _LIBCPP___CXX03___ALGORITHM_SIMD_UTILS_H11 12#include <__cxx03/__algorithm/min.h>13#include <__cxx03/__bit/countl.h>14#include <__cxx03/__bit/countr.h>15#include <__cxx03/__config>16#include <__cxx03/__type_traits/is_arithmetic.h>17#include <__cxx03/__type_traits/is_same.h>18#include <__cxx03/__utility/integer_sequence.h>19#include <__cxx03/cstddef>20#include <__cxx03/cstdint>21 22#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)23# pragma GCC system_header24#endif25 26_LIBCPP_PUSH_MACROS27#include <__cxx03/__undef_macros>28 29// TODO: Find out how altivec changes things and allow vectorizations there too.30#define _LIBCPP_HAS_ALGORITHM_VECTOR_UTILS 031 32#if _LIBCPP_HAS_ALGORITHM_VECTOR_UTILS && !defined(__OPTIMIZE_SIZE__)33# define _LIBCPP_VECTORIZE_ALGORITHMS 134#else35# define _LIBCPP_VECTORIZE_ALGORITHMS 036#endif37 38#if _LIBCPP_HAS_ALGORITHM_VECTOR_UTILS39 40_LIBCPP_BEGIN_NAMESPACE_STD41 42template <class _Tp>43inline constexpr bool __can_map_to_integer_v =44 sizeof(_Tp) == alignof(_Tp) && (sizeof(_Tp) == 1 || sizeof(_Tp) == 2 || sizeof(_Tp) == 4 || sizeof(_Tp) == 8);45 46template <size_t _TypeSize>47struct __get_as_integer_type_impl;48 49template <>50struct __get_as_integer_type_impl<1> {51 using type = uint8_t;52};53 54template <>55struct __get_as_integer_type_impl<2> {56 using type = uint16_t;57};58template <>59struct __get_as_integer_type_impl<4> {60 using type = uint32_t;61};62template <>63struct __get_as_integer_type_impl<8> {64 using type = uint64_t;65};66 67template <class _Tp>68using __get_as_integer_type_t = typename __get_as_integer_type_impl<sizeof(_Tp)>::type;69 70// This isn't specialized for 64 byte vectors on purpose. They have the potential to significantly reduce performance71// in mixed simd/non-simd workloads and don't provide any performance improvement for currently vectorized algorithms72// as far as benchmarks are concerned.73# if defined(__AVX__) || defined(__MVS__)74template <class _Tp>75inline constexpr size_t __native_vector_size = 32 / sizeof(_Tp);76# elif defined(__SSE__) || defined(__ARM_NEON__)77template <class _Tp>78inline constexpr size_t __native_vector_size = 16 / sizeof(_Tp);79# elif defined(__MMX__)80template <class _Tp>81inline constexpr size_t __native_vector_size = 8 / sizeof(_Tp);82# else83template <class _Tp>84inline constexpr size_t __native_vector_size = 1;85# endif86 87template <class _ArithmeticT, size_t _Np>88using __simd_vector __attribute__((__ext_vector_type__(_Np))) = _ArithmeticT;89 90template <class _VecT>91inline constexpr size_t __simd_vector_size_v = []<bool _False = false>() -> size_t {92 static_assert(_False, "Not a vector!");93}();94 95template <class _Tp, size_t _Np>96inline constexpr size_t __simd_vector_size_v<__simd_vector<_Tp, _Np>> = _Np;97 98template <class _Tp, size_t _Np>99_LIBCPP_HIDE_FROM_ABI _Tp __simd_vector_underlying_type_impl(__simd_vector<_Tp, _Np>) {100 return _Tp{};101}102 103template <class _VecT>104using __simd_vector_underlying_type_t = decltype(std::__simd_vector_underlying_type_impl(_VecT{}));105 106// This isn't inlined without always_inline when loading chars.107template <class _VecT, class _Iter>108_LIBCPP_NODISCARD _LIBCPP_ALWAYS_INLINE _LIBCPP_HIDE_FROM_ABI _VecT __load_vector(_Iter __iter) noexcept {109 return [=]<size_t... _Indices>(index_sequence<_Indices...>) _LIBCPP_ALWAYS_INLINE noexcept {110 return _VecT{__iter[_Indices]...};111 }(make_index_sequence<__simd_vector_size_v<_VecT>>{});112}113 114template <class _Tp, size_t _Np>115_LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI bool __all_of(__simd_vector<_Tp, _Np> __vec) noexcept {116 return __builtin_reduce_and(__builtin_convertvector(__vec, __simd_vector<bool, _Np>));117}118 119template <class _Tp, size_t _Np>120_LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI size_t __find_first_set(__simd_vector<_Tp, _Np> __vec) noexcept {121 using __mask_vec = __simd_vector<bool, _Np>;122 123 // This has MSan disabled du to https://github.com/llvm/llvm-project/issues/85876124 auto __impl = [&]<class _MaskT>(_MaskT) _LIBCPP_NO_SANITIZE("memory") noexcept {125# if defined(_LIBCPP_BIG_ENDIAN)126 return std::min<size_t>(127 _Np, std::__countl_zero(__builtin_bit_cast(_MaskT, __builtin_convertvector(__vec, __mask_vec))));128# else129 return std::min<size_t>(130 _Np, std::__countr_zero(__builtin_bit_cast(_MaskT, __builtin_convertvector(__vec, __mask_vec))));131# endif132 };133 134 if constexpr (sizeof(__mask_vec) == sizeof(uint8_t)) {135 return __impl(uint8_t{});136 } else if constexpr (sizeof(__mask_vec) == sizeof(uint16_t)) {137 return __impl(uint16_t{});138 } else if constexpr (sizeof(__mask_vec) == sizeof(uint32_t)) {139 return __impl(uint32_t{});140 } else if constexpr (sizeof(__mask_vec) == sizeof(uint64_t)) {141 return __impl(uint64_t{});142 } else {143 static_assert(sizeof(__mask_vec) == 0, "unexpected required size for mask integer type");144 return 0;145 }146}147 148template <class _Tp, size_t _Np>149_LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI size_t __find_first_not_set(__simd_vector<_Tp, _Np> __vec) noexcept {150 return std::__find_first_set(~__vec);151}152 153_LIBCPP_END_NAMESPACE_STD154 155#endif // _LIBCPP_HAS_ALGORITHM_VECTOR_UTILS156 157_LIBCPP_POP_MACROS158 159#endif // _LIBCPP___CXX03___ALGORITHM_SIMD_UTILS_H160