711 lines · c
1/*===------------- avx512vlvbmi2intrin.h - VBMI2 intrinsics -----------------===2 *3 *4 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.5 * See https://llvm.org/LICENSE.txt for license information.6 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception7 *8 *===-----------------------------------------------------------------------===9 */10#ifndef __IMMINTRIN_H11#error "Never use <avx512vlvbmi2intrin.h> directly; include <immintrin.h> instead."12#endif13 14#ifndef __AVX512VLVBMI2INTRIN_H15#define __AVX512VLVBMI2INTRIN_H16 17/* Define the default attributes for the functions in this file. */18#define __DEFAULT_FN_ATTRS128 \19 __attribute__((__always_inline__, __nodebug__, \20 __target__("avx512vl,avx512vbmi2"), \21 __min_vector_width__(128)))22#define __DEFAULT_FN_ATTRS256 \23 __attribute__((__always_inline__, __nodebug__, \24 __target__("avx512vl,avx512vbmi2"), \25 __min_vector_width__(256)))26 27#if defined(__cplusplus) && (__cplusplus >= 201103L)28#define __DEFAULT_FN_ATTRS128_CONSTEXPR __DEFAULT_FN_ATTRS128 constexpr29#define __DEFAULT_FN_ATTRS256_CONSTEXPR __DEFAULT_FN_ATTRS256 constexpr30#else31#define __DEFAULT_FN_ATTRS128_CONSTEXPR __DEFAULT_FN_ATTRS12832#define __DEFAULT_FN_ATTRS256_CONSTEXPR __DEFAULT_FN_ATTRS25633#endif34 35static __inline__ __m128i __DEFAULT_FN_ATTRS12836_mm_mask_compress_epi16(__m128i __S, __mmask8 __U, __m128i __D)37{38 return (__m128i) __builtin_ia32_compresshi128_mask ((__v8hi) __D,39 (__v8hi) __S,40 __U);41}42 43static __inline__ __m128i __DEFAULT_FN_ATTRS12844_mm_maskz_compress_epi16(__mmask8 __U, __m128i __D)45{46 return (__m128i) __builtin_ia32_compresshi128_mask ((__v8hi) __D,47 (__v8hi) _mm_setzero_si128(),48 __U);49}50 51static __inline__ __m128i __DEFAULT_FN_ATTRS12852_mm_mask_compress_epi8(__m128i __S, __mmask16 __U, __m128i __D)53{54 return (__m128i) __builtin_ia32_compressqi128_mask ((__v16qi) __D,55 (__v16qi) __S,56 __U);57}58 59static __inline__ __m128i __DEFAULT_FN_ATTRS12860_mm_maskz_compress_epi8(__mmask16 __U, __m128i __D)61{62 return (__m128i) __builtin_ia32_compressqi128_mask ((__v16qi) __D,63 (__v16qi) _mm_setzero_si128(),64 __U);65}66 67static __inline__ void __DEFAULT_FN_ATTRS12868_mm_mask_compressstoreu_epi16(void *__P, __mmask8 __U, __m128i __D)69{70 __builtin_ia32_compressstorehi128_mask ((__v8hi *) __P, (__v8hi) __D,71 __U);72}73 74static __inline__ void __DEFAULT_FN_ATTRS12875_mm_mask_compressstoreu_epi8(void *__P, __mmask16 __U, __m128i __D)76{77 __builtin_ia32_compressstoreqi128_mask ((__v16qi *) __P, (__v16qi) __D,78 __U);79}80 81static __inline__ __m128i __DEFAULT_FN_ATTRS12882_mm_mask_expand_epi16(__m128i __S, __mmask8 __U, __m128i __D)83{84 return (__m128i) __builtin_ia32_expandhi128_mask ((__v8hi) __D,85 (__v8hi) __S,86 __U);87}88 89static __inline__ __m128i __DEFAULT_FN_ATTRS12890_mm_maskz_expand_epi16(__mmask8 __U, __m128i __D)91{92 return (__m128i) __builtin_ia32_expandhi128_mask ((__v8hi) __D,93 (__v8hi) _mm_setzero_si128(),94 __U);95}96 97static __inline__ __m128i __DEFAULT_FN_ATTRS12898_mm_mask_expand_epi8(__m128i __S, __mmask16 __U, __m128i __D)99{100 return (__m128i) __builtin_ia32_expandqi128_mask ((__v16qi) __D,101 (__v16qi) __S,102 __U);103}104 105static __inline__ __m128i __DEFAULT_FN_ATTRS128106_mm_maskz_expand_epi8(__mmask16 __U, __m128i __D)107{108 return (__m128i) __builtin_ia32_expandqi128_mask ((__v16qi) __D,109 (__v16qi) _mm_setzero_si128(),110 __U);111}112 113static __inline__ __m128i __DEFAULT_FN_ATTRS128114_mm_mask_expandloadu_epi16(__m128i __S, __mmask8 __U, void const *__P)115{116 return (__m128i) __builtin_ia32_expandloadhi128_mask ((const __v8hi *)__P,117 (__v8hi) __S,118 __U);119}120 121static __inline__ __m128i __DEFAULT_FN_ATTRS128122_mm_maskz_expandloadu_epi16(__mmask8 __U, void const *__P)123{124 return (__m128i) __builtin_ia32_expandloadhi128_mask ((const __v8hi *)__P,125 (__v8hi) _mm_setzero_si128(),126 __U);127}128 129static __inline__ __m128i __DEFAULT_FN_ATTRS128130_mm_mask_expandloadu_epi8(__m128i __S, __mmask16 __U, void const *__P)131{132 return (__m128i) __builtin_ia32_expandloadqi128_mask ((const __v16qi *)__P,133 (__v16qi) __S,134 __U);135}136 137static __inline__ __m128i __DEFAULT_FN_ATTRS128138_mm_maskz_expandloadu_epi8(__mmask16 __U, void const *__P)139{140 return (__m128i) __builtin_ia32_expandloadqi128_mask ((const __v16qi *)__P,141 (__v16qi) _mm_setzero_si128(),142 __U);143}144 145static __inline__ __m256i __DEFAULT_FN_ATTRS256146_mm256_mask_compress_epi16(__m256i __S, __mmask16 __U, __m256i __D)147{148 return (__m256i) __builtin_ia32_compresshi256_mask ((__v16hi) __D,149 (__v16hi) __S,150 __U);151}152 153static __inline__ __m256i __DEFAULT_FN_ATTRS256154_mm256_maskz_compress_epi16(__mmask16 __U, __m256i __D)155{156 return (__m256i) __builtin_ia32_compresshi256_mask ((__v16hi) __D,157 (__v16hi) _mm256_setzero_si256(),158 __U);159}160 161static __inline__ __m256i __DEFAULT_FN_ATTRS256162_mm256_mask_compress_epi8(__m256i __S, __mmask32 __U, __m256i __D)163{164 return (__m256i) __builtin_ia32_compressqi256_mask ((__v32qi) __D,165 (__v32qi) __S,166 __U);167}168 169static __inline__ __m256i __DEFAULT_FN_ATTRS256170_mm256_maskz_compress_epi8(__mmask32 __U, __m256i __D)171{172 return (__m256i) __builtin_ia32_compressqi256_mask ((__v32qi) __D,173 (__v32qi) _mm256_setzero_si256(),174 __U);175}176 177static __inline__ void __DEFAULT_FN_ATTRS256178_mm256_mask_compressstoreu_epi16(void *__P, __mmask16 __U, __m256i __D)179{180 __builtin_ia32_compressstorehi256_mask ((__v16hi *) __P, (__v16hi) __D,181 __U);182}183 184static __inline__ void __DEFAULT_FN_ATTRS256185_mm256_mask_compressstoreu_epi8(void *__P, __mmask32 __U, __m256i __D)186{187 __builtin_ia32_compressstoreqi256_mask ((__v32qi *) __P, (__v32qi) __D,188 __U);189}190 191static __inline__ __m256i __DEFAULT_FN_ATTRS256192_mm256_mask_expand_epi16(__m256i __S, __mmask16 __U, __m256i __D)193{194 return (__m256i) __builtin_ia32_expandhi256_mask ((__v16hi) __D,195 (__v16hi) __S,196 __U);197}198 199static __inline__ __m256i __DEFAULT_FN_ATTRS256200_mm256_maskz_expand_epi16(__mmask16 __U, __m256i __D)201{202 return (__m256i) __builtin_ia32_expandhi256_mask ((__v16hi) __D,203 (__v16hi) _mm256_setzero_si256(),204 __U);205}206 207static __inline__ __m256i __DEFAULT_FN_ATTRS256208_mm256_mask_expand_epi8(__m256i __S, __mmask32 __U, __m256i __D)209{210 return (__m256i) __builtin_ia32_expandqi256_mask ((__v32qi) __D,211 (__v32qi) __S,212 __U);213}214 215static __inline__ __m256i __DEFAULT_FN_ATTRS256216_mm256_maskz_expand_epi8(__mmask32 __U, __m256i __D)217{218 return (__m256i) __builtin_ia32_expandqi256_mask ((__v32qi) __D,219 (__v32qi) _mm256_setzero_si256(),220 __U);221}222 223static __inline__ __m256i __DEFAULT_FN_ATTRS256224_mm256_mask_expandloadu_epi16(__m256i __S, __mmask16 __U, void const *__P)225{226 return (__m256i) __builtin_ia32_expandloadhi256_mask ((const __v16hi *)__P,227 (__v16hi) __S,228 __U);229}230 231static __inline__ __m256i __DEFAULT_FN_ATTRS256232_mm256_maskz_expandloadu_epi16(__mmask16 __U, void const *__P)233{234 return (__m256i) __builtin_ia32_expandloadhi256_mask ((const __v16hi *)__P,235 (__v16hi) _mm256_setzero_si256(),236 __U);237}238 239static __inline__ __m256i __DEFAULT_FN_ATTRS256240_mm256_mask_expandloadu_epi8(__m256i __S, __mmask32 __U, void const *__P)241{242 return (__m256i) __builtin_ia32_expandloadqi256_mask ((const __v32qi *)__P,243 (__v32qi) __S,244 __U);245}246 247static __inline__ __m256i __DEFAULT_FN_ATTRS256248_mm256_maskz_expandloadu_epi8(__mmask32 __U, void const *__P)249{250 return (__m256i) __builtin_ia32_expandloadqi256_mask ((const __v32qi *)__P,251 (__v32qi) _mm256_setzero_si256(),252 __U);253}254 255#define _mm256_shldi_epi64(A, B, I) \256 ((__m256i)__builtin_ia32_vpshldq256((__v4di)(__m256i)(A), \257 (__v4di)(__m256i)(B), (int)(I)))258 259#define _mm256_mask_shldi_epi64(S, U, A, B, I) \260 ((__m256i)__builtin_ia32_selectq_256((__mmask8)(U), \261 (__v4di)_mm256_shldi_epi64((A), (B), (I)), \262 (__v4di)(__m256i)(S)))263 264#define _mm256_maskz_shldi_epi64(U, A, B, I) \265 ((__m256i)__builtin_ia32_selectq_256((__mmask8)(U), \266 (__v4di)_mm256_shldi_epi64((A), (B), (I)), \267 (__v4di)_mm256_setzero_si256()))268 269#define _mm_shldi_epi64(A, B, I) \270 ((__m128i)__builtin_ia32_vpshldq128((__v2di)(__m128i)(A), \271 (__v2di)(__m128i)(B), (int)(I)))272 273#define _mm_mask_shldi_epi64(S, U, A, B, I) \274 ((__m128i)__builtin_ia32_selectq_128((__mmask8)(U), \275 (__v2di)_mm_shldi_epi64((A), (B), (I)), \276 (__v2di)(__m128i)(S)))277 278#define _mm_maskz_shldi_epi64(U, A, B, I) \279 ((__m128i)__builtin_ia32_selectq_128((__mmask8)(U), \280 (__v2di)_mm_shldi_epi64((A), (B), (I)), \281 (__v2di)_mm_setzero_si128()))282 283#define _mm256_shldi_epi32(A, B, I) \284 ((__m256i)__builtin_ia32_vpshldd256((__v8si)(__m256i)(A), \285 (__v8si)(__m256i)(B), (int)(I)))286 287#define _mm256_mask_shldi_epi32(S, U, A, B, I) \288 ((__m256i)__builtin_ia32_selectd_256((__mmask8)(U), \289 (__v8si)_mm256_shldi_epi32((A), (B), (I)), \290 (__v8si)(__m256i)(S)))291 292#define _mm256_maskz_shldi_epi32(U, A, B, I) \293 ((__m256i)__builtin_ia32_selectd_256((__mmask8)(U), \294 (__v8si)_mm256_shldi_epi32((A), (B), (I)), \295 (__v8si)_mm256_setzero_si256()))296 297#define _mm_shldi_epi32(A, B, I) \298 ((__m128i)__builtin_ia32_vpshldd128((__v4si)(__m128i)(A), \299 (__v4si)(__m128i)(B), (int)(I)))300 301#define _mm_mask_shldi_epi32(S, U, A, B, I) \302 ((__m128i)__builtin_ia32_selectd_128((__mmask8)(U), \303 (__v4si)_mm_shldi_epi32((A), (B), (I)), \304 (__v4si)(__m128i)(S)))305 306#define _mm_maskz_shldi_epi32(U, A, B, I) \307 ((__m128i)__builtin_ia32_selectd_128((__mmask8)(U), \308 (__v4si)_mm_shldi_epi32((A), (B), (I)), \309 (__v4si)_mm_setzero_si128()))310 311#define _mm256_shldi_epi16(A, B, I) \312 ((__m256i)__builtin_ia32_vpshldw256((__v16hi)(__m256i)(A), \313 (__v16hi)(__m256i)(B), (int)(I)))314 315#define _mm256_mask_shldi_epi16(S, U, A, B, I) \316 ((__m256i)__builtin_ia32_selectw_256((__mmask16)(U), \317 (__v16hi)_mm256_shldi_epi16((A), (B), (I)), \318 (__v16hi)(__m256i)(S)))319 320#define _mm256_maskz_shldi_epi16(U, A, B, I) \321 ((__m256i)__builtin_ia32_selectw_256((__mmask16)(U), \322 (__v16hi)_mm256_shldi_epi16((A), (B), (I)), \323 (__v16hi)_mm256_setzero_si256()))324 325#define _mm_shldi_epi16(A, B, I) \326 ((__m128i)__builtin_ia32_vpshldw128((__v8hi)(__m128i)(A), \327 (__v8hi)(__m128i)(B), (int)(I)))328 329#define _mm_mask_shldi_epi16(S, U, A, B, I) \330 ((__m128i)__builtin_ia32_selectw_128((__mmask8)(U), \331 (__v8hi)_mm_shldi_epi16((A), (B), (I)), \332 (__v8hi)(__m128i)(S)))333 334#define _mm_maskz_shldi_epi16(U, A, B, I) \335 ((__m128i)__builtin_ia32_selectw_128((__mmask8)(U), \336 (__v8hi)_mm_shldi_epi16((A), (B), (I)), \337 (__v8hi)_mm_setzero_si128()))338 339#define _mm256_shrdi_epi64(A, B, I) \340 ((__m256i)__builtin_ia32_vpshrdq256((__v4di)(__m256i)(A), \341 (__v4di)(__m256i)(B), (int)(I)))342 343#define _mm256_mask_shrdi_epi64(S, U, A, B, I) \344 ((__m256i)__builtin_ia32_selectq_256((__mmask8)(U), \345 (__v4di)_mm256_shrdi_epi64((A), (B), (I)), \346 (__v4di)(__m256i)(S)))347 348#define _mm256_maskz_shrdi_epi64(U, A, B, I) \349 ((__m256i)__builtin_ia32_selectq_256((__mmask8)(U), \350 (__v4di)_mm256_shrdi_epi64((A), (B), (I)), \351 (__v4di)_mm256_setzero_si256()))352 353#define _mm_shrdi_epi64(A, B, I) \354 ((__m128i)__builtin_ia32_vpshrdq128((__v2di)(__m128i)(A), \355 (__v2di)(__m128i)(B), (int)(I)))356 357#define _mm_mask_shrdi_epi64(S, U, A, B, I) \358 ((__m128i)__builtin_ia32_selectq_128((__mmask8)(U), \359 (__v2di)_mm_shrdi_epi64((A), (B), (I)), \360 (__v2di)(__m128i)(S)))361 362#define _mm_maskz_shrdi_epi64(U, A, B, I) \363 ((__m128i)__builtin_ia32_selectq_128((__mmask8)(U), \364 (__v2di)_mm_shrdi_epi64((A), (B), (I)), \365 (__v2di)_mm_setzero_si128()))366 367#define _mm256_shrdi_epi32(A, B, I) \368 ((__m256i)__builtin_ia32_vpshrdd256((__v8si)(__m256i)(A), \369 (__v8si)(__m256i)(B), (int)(I)))370 371#define _mm256_mask_shrdi_epi32(S, U, A, B, I) \372 ((__m256i)__builtin_ia32_selectd_256((__mmask8)(U), \373 (__v8si)_mm256_shrdi_epi32((A), (B), (I)), \374 (__v8si)(__m256i)(S)))375 376#define _mm256_maskz_shrdi_epi32(U, A, B, I) \377 ((__m256i)__builtin_ia32_selectd_256((__mmask8)(U), \378 (__v8si)_mm256_shrdi_epi32((A), (B), (I)), \379 (__v8si)_mm256_setzero_si256()))380 381#define _mm_shrdi_epi32(A, B, I) \382 ((__m128i)__builtin_ia32_vpshrdd128((__v4si)(__m128i)(A), \383 (__v4si)(__m128i)(B), (int)(I)))384 385#define _mm_mask_shrdi_epi32(S, U, A, B, I) \386 ((__m128i)__builtin_ia32_selectd_128((__mmask8)(U), \387 (__v4si)_mm_shrdi_epi32((A), (B), (I)), \388 (__v4si)(__m128i)(S)))389 390#define _mm_maskz_shrdi_epi32(U, A, B, I) \391 ((__m128i)__builtin_ia32_selectd_128((__mmask8)(U), \392 (__v4si)_mm_shrdi_epi32((A), (B), (I)), \393 (__v4si)_mm_setzero_si128()))394 395#define _mm256_shrdi_epi16(A, B, I) \396 ((__m256i)__builtin_ia32_vpshrdw256((__v16hi)(__m256i)(A), \397 (__v16hi)(__m256i)(B), (int)(I)))398 399#define _mm256_mask_shrdi_epi16(S, U, A, B, I) \400 ((__m256i)__builtin_ia32_selectw_256((__mmask16)(U), \401 (__v16hi)_mm256_shrdi_epi16((A), (B), (I)), \402 (__v16hi)(__m256i)(S)))403 404#define _mm256_maskz_shrdi_epi16(U, A, B, I) \405 ((__m256i)__builtin_ia32_selectw_256((__mmask16)(U), \406 (__v16hi)_mm256_shrdi_epi16((A), (B), (I)), \407 (__v16hi)_mm256_setzero_si256()))408 409#define _mm_shrdi_epi16(A, B, I) \410 ((__m128i)__builtin_ia32_vpshrdw128((__v8hi)(__m128i)(A), \411 (__v8hi)(__m128i)(B), (int)(I)))412 413#define _mm_mask_shrdi_epi16(S, U, A, B, I) \414 ((__m128i)__builtin_ia32_selectw_128((__mmask8)(U), \415 (__v8hi)_mm_shrdi_epi16((A), (B), (I)), \416 (__v8hi)(__m128i)(S)))417 418#define _mm_maskz_shrdi_epi16(U, A, B, I) \419 ((__m128i)__builtin_ia32_selectw_128((__mmask8)(U), \420 (__v8hi)_mm_shrdi_epi16((A), (B), (I)), \421 (__v8hi)_mm_setzero_si128()))422 423static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR424_mm256_shldv_epi64(__m256i __A, __m256i __B, __m256i __C)425{426 return (__m256i)__builtin_elementwise_fshl((__v4du)__A, (__v4du)__B,427 (__v4du)__C);428}429 430static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR431_mm256_mask_shldv_epi64(__m256i __A, __mmask8 __U, __m256i __B, __m256i __C)432{433 return (__m256i)__builtin_ia32_selectq_256(__U,434 (__v4di)_mm256_shldv_epi64(__A, __B, __C),435 (__v4di)__A);436}437 438static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR439_mm256_maskz_shldv_epi64(__mmask8 __U, __m256i __A, __m256i __B, __m256i __C)440{441 return (__m256i)__builtin_ia32_selectq_256(__U,442 (__v4di)_mm256_shldv_epi64(__A, __B, __C),443 (__v4di)_mm256_setzero_si256());444}445 446static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR447_mm_shldv_epi64(__m128i __A, __m128i __B, __m128i __C)448{449 return (__m128i)__builtin_elementwise_fshl((__v2du)__A, (__v2du)__B,450 (__v2du)__C);451}452 453static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR454_mm_mask_shldv_epi64(__m128i __A, __mmask8 __U, __m128i __B, __m128i __C)455{456 return (__m128i)__builtin_ia32_selectq_128(__U,457 (__v2di)_mm_shldv_epi64(__A, __B, __C),458 (__v2di)__A);459}460 461static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR462_mm_maskz_shldv_epi64(__mmask8 __U, __m128i __A, __m128i __B, __m128i __C)463{464 return (__m128i)__builtin_ia32_selectq_128(__U,465 (__v2di)_mm_shldv_epi64(__A, __B, __C),466 (__v2di)_mm_setzero_si128());467}468 469static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR470_mm256_shldv_epi32(__m256i __A, __m256i __B, __m256i __C)471{472 return (__m256i)__builtin_elementwise_fshl((__v8su)__A, (__v8su)__B,473 (__v8su)__C);474}475 476static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR477_mm256_mask_shldv_epi32(__m256i __A, __mmask8 __U, __m256i __B, __m256i __C)478{479 return (__m256i)__builtin_ia32_selectd_256(__U,480 (__v8si)_mm256_shldv_epi32(__A, __B, __C),481 (__v8si)__A);482}483 484static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR485_mm256_maskz_shldv_epi32(__mmask8 __U, __m256i __A, __m256i __B, __m256i __C)486{487 return (__m256i)__builtin_ia32_selectd_256(__U,488 (__v8si)_mm256_shldv_epi32(__A, __B, __C),489 (__v8si)_mm256_setzero_si256());490}491 492static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR493_mm_shldv_epi32(__m128i __A, __m128i __B, __m128i __C)494{495 return (__m128i)__builtin_elementwise_fshl((__v4su)__A, (__v4su)__B,496 (__v4su)__C);497}498 499static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR500_mm_mask_shldv_epi32(__m128i __A, __mmask8 __U, __m128i __B, __m128i __C)501{502 return (__m128i)__builtin_ia32_selectd_128(__U,503 (__v4si)_mm_shldv_epi32(__A, __B, __C),504 (__v4si)__A);505}506 507static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR508_mm_maskz_shldv_epi32(__mmask8 __U, __m128i __A, __m128i __B, __m128i __C)509{510 return (__m128i)__builtin_ia32_selectd_128(__U,511 (__v4si)_mm_shldv_epi32(__A, __B, __C),512 (__v4si)_mm_setzero_si128());513}514 515static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR516_mm256_shldv_epi16(__m256i __A, __m256i __B, __m256i __C)517{518 return (__m256i)__builtin_elementwise_fshl((__v16hu)__A, (__v16hu)__B,519 (__v16hu)__C);520}521 522static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR523_mm256_mask_shldv_epi16(__m256i __A, __mmask16 __U, __m256i __B, __m256i __C)524{525 return (__m256i)__builtin_ia32_selectw_256(__U,526 (__v16hi)_mm256_shldv_epi16(__A, __B, __C),527 (__v16hi)__A);528}529 530static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR531_mm256_maskz_shldv_epi16(__mmask16 __U, __m256i __A, __m256i __B, __m256i __C)532{533 return (__m256i)__builtin_ia32_selectw_256(__U,534 (__v16hi)_mm256_shldv_epi16(__A, __B, __C),535 (__v16hi)_mm256_setzero_si256());536}537 538static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR539_mm_shldv_epi16(__m128i __A, __m128i __B, __m128i __C)540{541 return (__m128i)__builtin_elementwise_fshl((__v8hu)__A, (__v8hu)__B,542 (__v8hu)__C);543}544 545static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR546_mm_mask_shldv_epi16(__m128i __A, __mmask8 __U, __m128i __B, __m128i __C)547{548 return (__m128i)__builtin_ia32_selectw_128(__U,549 (__v8hi)_mm_shldv_epi16(__A, __B, __C),550 (__v8hi)__A);551}552 553static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR554_mm_maskz_shldv_epi16(__mmask8 __U, __m128i __A, __m128i __B, __m128i __C)555{556 return (__m128i)__builtin_ia32_selectw_128(__U,557 (__v8hi)_mm_shldv_epi16(__A, __B, __C),558 (__v8hi)_mm_setzero_si128());559}560 561static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR562_mm256_shrdv_epi64(__m256i __A, __m256i __B, __m256i __C)563{564 // Ops __A and __B are swapped.565 return (__m256i)__builtin_elementwise_fshr((__v4du)__B, (__v4du)__A,566 (__v4du)__C);567}568 569static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR570_mm256_mask_shrdv_epi64(__m256i __A, __mmask8 __U, __m256i __B, __m256i __C)571{572 return (__m256i)__builtin_ia32_selectq_256(__U,573 (__v4di)_mm256_shrdv_epi64(__A, __B, __C),574 (__v4di)__A);575}576 577static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR578_mm256_maskz_shrdv_epi64(__mmask8 __U, __m256i __A, __m256i __B, __m256i __C)579{580 return (__m256i)__builtin_ia32_selectq_256(__U,581 (__v4di)_mm256_shrdv_epi64(__A, __B, __C),582 (__v4di)_mm256_setzero_si256());583}584 585static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR586_mm_shrdv_epi64(__m128i __A, __m128i __B, __m128i __C)587{588 // Ops __A and __B are swapped.589 return (__m128i)__builtin_elementwise_fshr((__v2du)__B, (__v2du)__A,590 (__v2du)__C);591}592 593static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR594_mm_mask_shrdv_epi64(__m128i __A, __mmask8 __U, __m128i __B, __m128i __C)595{596 return (__m128i)__builtin_ia32_selectq_128(__U,597 (__v2di)_mm_shrdv_epi64(__A, __B, __C),598 (__v2di)__A);599}600 601static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR602_mm_maskz_shrdv_epi64(__mmask8 __U, __m128i __A, __m128i __B, __m128i __C)603{604 return (__m128i)__builtin_ia32_selectq_128(__U,605 (__v2di)_mm_shrdv_epi64(__A, __B, __C),606 (__v2di)_mm_setzero_si128());607}608 609static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR610_mm256_shrdv_epi32(__m256i __A, __m256i __B, __m256i __C)611{612 // Ops __A and __B are swapped.613 return (__m256i)__builtin_elementwise_fshr((__v8su)__B, (__v8su)__A,614 (__v8su)__C);615}616 617static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR618_mm256_mask_shrdv_epi32(__m256i __A, __mmask8 __U, __m256i __B, __m256i __C)619{620 return (__m256i)__builtin_ia32_selectd_256(__U,621 (__v8si)_mm256_shrdv_epi32(__A, __B, __C),622 (__v8si)__A);623}624 625static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR626_mm256_maskz_shrdv_epi32(__mmask8 __U, __m256i __A, __m256i __B, __m256i __C)627{628 return (__m256i)__builtin_ia32_selectd_256(__U,629 (__v8si)_mm256_shrdv_epi32(__A, __B, __C),630 (__v8si)_mm256_setzero_si256());631}632 633static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR634_mm_shrdv_epi32(__m128i __A, __m128i __B, __m128i __C)635{636 // Ops __A and __B are swapped.637 return (__m128i)__builtin_elementwise_fshr((__v4su)__B, (__v4su)__A,638 (__v4su)__C);639}640 641static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR642_mm_mask_shrdv_epi32(__m128i __A, __mmask8 __U, __m128i __B, __m128i __C)643{644 return (__m128i)__builtin_ia32_selectd_128(__U,645 (__v4si)_mm_shrdv_epi32(__A, __B, __C),646 (__v4si)__A);647}648 649static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR650_mm_maskz_shrdv_epi32(__mmask8 __U, __m128i __A, __m128i __B, __m128i __C)651{652 return (__m128i)__builtin_ia32_selectd_128(__U,653 (__v4si)_mm_shrdv_epi32(__A, __B, __C),654 (__v4si)_mm_setzero_si128());655}656 657static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR658_mm256_shrdv_epi16(__m256i __A, __m256i __B, __m256i __C)659{660 // Ops __A and __B are swapped.661 return (__m256i)__builtin_elementwise_fshr((__v16hu)__B, (__v16hu)__A,662 (__v16hu)__C);663}664 665static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR666_mm256_mask_shrdv_epi16(__m256i __A, __mmask16 __U, __m256i __B, __m256i __C)667{668 return (__m256i)__builtin_ia32_selectw_256(__U,669 (__v16hi)_mm256_shrdv_epi16(__A, __B, __C),670 (__v16hi)__A);671}672 673static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR674_mm256_maskz_shrdv_epi16(__mmask16 __U, __m256i __A, __m256i __B, __m256i __C)675{676 return (__m256i)__builtin_ia32_selectw_256(__U,677 (__v16hi)_mm256_shrdv_epi16(__A, __B, __C),678 (__v16hi)_mm256_setzero_si256());679}680 681static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR682_mm_shrdv_epi16(__m128i __A, __m128i __B, __m128i __C)683{684 // Ops __A and __B are swapped.685 return (__m128i)__builtin_elementwise_fshr((__v8hu)__B, (__v8hu)__A,686 (__v8hu)__C);687}688 689static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR690_mm_mask_shrdv_epi16(__m128i __A, __mmask8 __U, __m128i __B, __m128i __C)691{692 return (__m128i)__builtin_ia32_selectw_128(__U,693 (__v8hi)_mm_shrdv_epi16(__A, __B, __C),694 (__v8hi)__A);695}696 697static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR698_mm_maskz_shrdv_epi16(__mmask8 __U, __m128i __A, __m128i __B, __m128i __C)699{700 return (__m128i)__builtin_ia32_selectw_128(__U,701 (__v8hi)_mm_shrdv_epi16(__A, __B, __C),702 (__v8hi)_mm_setzero_si128());703}704 705#undef __DEFAULT_FN_ATTRS128706#undef __DEFAULT_FN_ATTRS256707#undef __DEFAULT_FN_ATTRS128_CONSTEXPR708#undef __DEFAULT_FN_ATTRS256_CONSTEXPR709 710#endif711