1353 lines · c
1/*===---- avx512dqintrin.h - AVX512DQ intrinsics ---------------------------===2 *3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4 * See https://llvm.org/LICENSE.txt for license information.5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6 *7 *===-----------------------------------------------------------------------===8 */9 10#ifndef __IMMINTRIN_H11#error "Never use <avx512dqintrin.h> directly; include <immintrin.h> instead."12#endif13 14#ifndef __AVX512DQINTRIN_H15#define __AVX512DQINTRIN_H16 17/* Define the default attributes for the functions in this file. */18#define __DEFAULT_FN_ATTRS512 \19 __attribute__((__always_inline__, __nodebug__, __target__("avx512dq"), \20 __min_vector_width__(512)))21#define __DEFAULT_FN_ATTRS \22 __attribute__((__always_inline__, __nodebug__, __target__("avx512dq")))23 24#if defined(__cplusplus) && (__cplusplus >= 201103L)25#define __DEFAULT_FN_ATTRS512_CONSTEXPR __DEFAULT_FN_ATTRS512 constexpr26#define __DEFAULT_FN_ATTRS_CONSTEXPR __DEFAULT_FN_ATTRS constexpr27#else28#define __DEFAULT_FN_ATTRS512_CONSTEXPR __DEFAULT_FN_ATTRS51229#define __DEFAULT_FN_ATTRS_CONSTEXPR __DEFAULT_FN_ATTRS30#endif31 32static __inline __mmask8 __DEFAULT_FN_ATTRS_CONSTEXPR33_knot_mask8(__mmask8 __M) {34 return __builtin_ia32_knotqi(__M);35}36 37static __inline__ __mmask8 __DEFAULT_FN_ATTRS_CONSTEXPR38_kand_mask8(__mmask8 __A, __mmask8 __B) {39 return (__mmask8)__builtin_ia32_kandqi((__mmask8)__A, (__mmask8)__B);40}41 42static __inline__ __mmask8 __DEFAULT_FN_ATTRS_CONSTEXPR43_kandn_mask8(__mmask8 __A, __mmask8 __B) {44 return (__mmask8)__builtin_ia32_kandnqi((__mmask8)__A, (__mmask8)__B);45}46 47static __inline__ __mmask8 __DEFAULT_FN_ATTRS_CONSTEXPR48_kor_mask8(__mmask8 __A, __mmask8 __B) {49 return (__mmask8)__builtin_ia32_korqi((__mmask8)__A, (__mmask8)__B);50}51 52static __inline__ __mmask8 __DEFAULT_FN_ATTRS_CONSTEXPR53_kxnor_mask8(__mmask8 __A, __mmask8 __B) {54 return (__mmask8)__builtin_ia32_kxnorqi((__mmask8)__A, (__mmask8)__B);55}56 57static __inline__ __mmask8 __DEFAULT_FN_ATTRS_CONSTEXPR58_kxor_mask8(__mmask8 __A, __mmask8 __B) {59 return (__mmask8)__builtin_ia32_kxorqi((__mmask8)__A, (__mmask8)__B);60}61 62static __inline__ unsigned char __DEFAULT_FN_ATTRS_CONSTEXPR63_kortestc_mask8_u8(__mmask8 __A, __mmask8 __B) {64 return (unsigned char)__builtin_ia32_kortestcqi(__A, __B);65}66 67static __inline__ unsigned char __DEFAULT_FN_ATTRS_CONSTEXPR68_kortestz_mask8_u8(__mmask8 __A, __mmask8 __B) {69 return (unsigned char)__builtin_ia32_kortestzqi(__A, __B);70}71 72static __inline__ unsigned char __DEFAULT_FN_ATTRS_CONSTEXPR73_kortest_mask8_u8(__mmask8 __A, __mmask8 __B, unsigned char *__C) {74 *__C = (unsigned char)__builtin_ia32_kortestcqi(__A, __B);75 return (unsigned char)__builtin_ia32_kortestzqi(__A, __B);76}77 78static __inline__ unsigned char __DEFAULT_FN_ATTRS_CONSTEXPR79_ktestc_mask8_u8(__mmask8 __A, __mmask8 __B) {80 return (unsigned char)__builtin_ia32_ktestcqi(__A, __B);81}82 83static __inline__ unsigned char __DEFAULT_FN_ATTRS_CONSTEXPR84_ktestz_mask8_u8(__mmask8 __A, __mmask8 __B) {85 return (unsigned char)__builtin_ia32_ktestzqi(__A, __B);86}87 88static __inline__ unsigned char __DEFAULT_FN_ATTRS_CONSTEXPR89_ktest_mask8_u8(__mmask8 __A, __mmask8 __B, unsigned char *__C) {90 *__C = (unsigned char)__builtin_ia32_ktestcqi(__A, __B);91 return (unsigned char)__builtin_ia32_ktestzqi(__A, __B);92}93 94static __inline__ unsigned char __DEFAULT_FN_ATTRS_CONSTEXPR95_ktestc_mask16_u8(__mmask16 __A, __mmask16 __B) {96 return (unsigned char)__builtin_ia32_ktestchi(__A, __B);97}98 99static __inline__ unsigned char __DEFAULT_FN_ATTRS_CONSTEXPR100_ktestz_mask16_u8(__mmask16 __A, __mmask16 __B) {101 return (unsigned char)__builtin_ia32_ktestzhi(__A, __B);102}103 104static __inline__ unsigned char __DEFAULT_FN_ATTRS_CONSTEXPR105_ktest_mask16_u8(__mmask16 __A, __mmask16 __B, unsigned char *__C) {106 *__C = (unsigned char)__builtin_ia32_ktestchi(__A, __B);107 return (unsigned char)__builtin_ia32_ktestzhi(__A, __B);108}109 110static __inline__ __mmask8 __DEFAULT_FN_ATTRS_CONSTEXPR111_kadd_mask8(__mmask8 __A, __mmask8 __B) {112 return (__mmask8)__builtin_ia32_kaddqi((__mmask8)__A, (__mmask8)__B);113}114 115static __inline__ __mmask16 __DEFAULT_FN_ATTRS_CONSTEXPR116_kadd_mask16(__mmask16 __A, __mmask16 __B) {117 return (__mmask16)__builtin_ia32_kaddhi((__mmask16)__A, (__mmask16)__B);118}119 120#define _kshiftli_mask8(A, I) \121 ((__mmask8)__builtin_ia32_kshiftliqi((__mmask8)(A), (unsigned int)(I)))122 123#define _kshiftri_mask8(A, I) \124 ((__mmask8)__builtin_ia32_kshiftriqi((__mmask8)(A), (unsigned int)(I)))125 126static __inline__ unsigned int127 __DEFAULT_FN_ATTRS_CONSTEXPR _cvtmask8_u32(__mmask8 __A) {128 return (unsigned int)__builtin_ia32_kmovb((__mmask8)__A);129}130 131static __inline__ __mmask8 __DEFAULT_FN_ATTRS_CONSTEXPR132_cvtu32_mask8(unsigned int __A) {133 return (__mmask8)__builtin_ia32_kmovb((__mmask8)__A);134}135 136static __inline__ __mmask8 __DEFAULT_FN_ATTRS137_load_mask8(__mmask8 *__A) {138 return (__mmask8)__builtin_ia32_kmovb(*(__mmask8 *)__A);139}140 141static __inline__ void __DEFAULT_FN_ATTRS142_store_mask8(__mmask8 *__A, __mmask8 __B) {143 *(__mmask8 *)__A = __builtin_ia32_kmovb((__mmask8)__B);144}145 146static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR147_mm512_mullo_epi64(__m512i __A, __m512i __B) {148 return (__m512i) ((__v8du) __A * (__v8du) __B);149}150 151static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR152_mm512_mask_mullo_epi64(__m512i __W, __mmask8 __U, __m512i __A, __m512i __B) {153 return (__m512i)__builtin_ia32_selectq_512((__mmask8)__U,154 (__v8di)_mm512_mullo_epi64(__A, __B),155 (__v8di)__W);156}157 158static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR159_mm512_maskz_mullo_epi64(__mmask8 __U, __m512i __A, __m512i __B) {160 return (__m512i)__builtin_ia32_selectq_512((__mmask8)__U,161 (__v8di)_mm512_mullo_epi64(__A, __B),162 (__v8di)_mm512_setzero_si512());163}164 165static __inline__ __m512d __DEFAULT_FN_ATTRS512_CONSTEXPR166_mm512_xor_pd(__m512d __A, __m512d __B) {167 return (__m512d)((__v8du)__A ^ (__v8du)__B);168}169 170static __inline__ __m512d __DEFAULT_FN_ATTRS512171_mm512_mask_xor_pd(__m512d __W, __mmask8 __U, __m512d __A, __m512d __B) {172 return (__m512d)__builtin_ia32_selectpd_512((__mmask8)__U,173 (__v8df)_mm512_xor_pd(__A, __B),174 (__v8df)__W);175}176 177static __inline__ __m512d __DEFAULT_FN_ATTRS512178_mm512_maskz_xor_pd(__mmask8 __U, __m512d __A, __m512d __B) {179 return (__m512d)__builtin_ia32_selectpd_512((__mmask8)__U,180 (__v8df)_mm512_xor_pd(__A, __B),181 (__v8df)_mm512_setzero_pd());182}183 184static __inline__ __m512 __DEFAULT_FN_ATTRS512_CONSTEXPR185_mm512_xor_ps (__m512 __A, __m512 __B) {186 return (__m512)((__v16su)__A ^ (__v16su)__B);187}188 189static __inline__ __m512 __DEFAULT_FN_ATTRS512190_mm512_mask_xor_ps(__m512 __W, __mmask16 __U, __m512 __A, __m512 __B) {191 return (__m512)__builtin_ia32_selectps_512((__mmask16)__U,192 (__v16sf)_mm512_xor_ps(__A, __B),193 (__v16sf)__W);194}195 196static __inline__ __m512 __DEFAULT_FN_ATTRS512197_mm512_maskz_xor_ps(__mmask16 __U, __m512 __A, __m512 __B) {198 return (__m512)__builtin_ia32_selectps_512((__mmask16)__U,199 (__v16sf)_mm512_xor_ps(__A, __B),200 (__v16sf)_mm512_setzero_ps());201}202 203static __inline__ __m512d __DEFAULT_FN_ATTRS512_CONSTEXPR204_mm512_or_pd(__m512d __A, __m512d __B) {205 return (__m512d)((__v8du)__A | (__v8du)__B);206}207 208static __inline__ __m512d __DEFAULT_FN_ATTRS512209_mm512_mask_or_pd(__m512d __W, __mmask8 __U, __m512d __A, __m512d __B) {210 return (__m512d)__builtin_ia32_selectpd_512((__mmask8)__U,211 (__v8df)_mm512_or_pd(__A, __B),212 (__v8df)__W);213}214 215static __inline__ __m512d __DEFAULT_FN_ATTRS512216_mm512_maskz_or_pd(__mmask8 __U, __m512d __A, __m512d __B) {217 return (__m512d)__builtin_ia32_selectpd_512((__mmask8)__U,218 (__v8df)_mm512_or_pd(__A, __B),219 (__v8df)_mm512_setzero_pd());220}221 222static __inline__ __m512 __DEFAULT_FN_ATTRS512_CONSTEXPR223_mm512_or_ps(__m512 __A, __m512 __B) {224 return (__m512)((__v16su)__A | (__v16su)__B);225}226 227static __inline__ __m512 __DEFAULT_FN_ATTRS512228_mm512_mask_or_ps(__m512 __W, __mmask16 __U, __m512 __A, __m512 __B) {229 return (__m512)__builtin_ia32_selectps_512((__mmask16)__U,230 (__v16sf)_mm512_or_ps(__A, __B),231 (__v16sf)__W);232}233 234static __inline__ __m512 __DEFAULT_FN_ATTRS512235_mm512_maskz_or_ps(__mmask16 __U, __m512 __A, __m512 __B) {236 return (__m512)__builtin_ia32_selectps_512((__mmask16)__U,237 (__v16sf)_mm512_or_ps(__A, __B),238 (__v16sf)_mm512_setzero_ps());239}240 241static __inline__ __m512d __DEFAULT_FN_ATTRS512_CONSTEXPR242_mm512_and_pd(__m512d __A, __m512d __B) {243 return (__m512d)((__v8du)__A & (__v8du)__B);244}245 246static __inline__ __m512d __DEFAULT_FN_ATTRS512247_mm512_mask_and_pd(__m512d __W, __mmask8 __U, __m512d __A, __m512d __B) {248 return (__m512d)__builtin_ia32_selectpd_512((__mmask8)__U,249 (__v8df)_mm512_and_pd(__A, __B),250 (__v8df)__W);251}252 253static __inline__ __m512d __DEFAULT_FN_ATTRS512254_mm512_maskz_and_pd(__mmask8 __U, __m512d __A, __m512d __B) {255 return (__m512d)__builtin_ia32_selectpd_512((__mmask8)__U,256 (__v8df)_mm512_and_pd(__A, __B),257 (__v8df)_mm512_setzero_pd());258}259 260static __inline__ __m512 __DEFAULT_FN_ATTRS512_CONSTEXPR261_mm512_and_ps(__m512 __A, __m512 __B) {262 return (__m512)((__v16su)__A & (__v16su)__B);263}264 265static __inline__ __m512 __DEFAULT_FN_ATTRS512266_mm512_mask_and_ps(__m512 __W, __mmask16 __U, __m512 __A, __m512 __B) {267 return (__m512)__builtin_ia32_selectps_512((__mmask16)__U,268 (__v16sf)_mm512_and_ps(__A, __B),269 (__v16sf)__W);270}271 272static __inline__ __m512 __DEFAULT_FN_ATTRS512273_mm512_maskz_and_ps(__mmask16 __U, __m512 __A, __m512 __B) {274 return (__m512)__builtin_ia32_selectps_512((__mmask16)__U,275 (__v16sf)_mm512_and_ps(__A, __B),276 (__v16sf)_mm512_setzero_ps());277}278 279static __inline__ __m512d __DEFAULT_FN_ATTRS512_CONSTEXPR280_mm512_andnot_pd(__m512d __A, __m512d __B) {281 return (__m512d)(~(__v8du)__A & (__v8du)__B);282}283 284static __inline__ __m512d __DEFAULT_FN_ATTRS512285_mm512_mask_andnot_pd(__m512d __W, __mmask8 __U, __m512d __A, __m512d __B) {286 return (__m512d)__builtin_ia32_selectpd_512((__mmask8)__U,287 (__v8df)_mm512_andnot_pd(__A, __B),288 (__v8df)__W);289}290 291static __inline__ __m512d __DEFAULT_FN_ATTRS512292_mm512_maskz_andnot_pd(__mmask8 __U, __m512d __A, __m512d __B) {293 return (__m512d)__builtin_ia32_selectpd_512((__mmask8)__U,294 (__v8df)_mm512_andnot_pd(__A, __B),295 (__v8df)_mm512_setzero_pd());296}297 298static __inline__ __m512 __DEFAULT_FN_ATTRS512_CONSTEXPR299_mm512_andnot_ps(__m512 __A, __m512 __B) {300 return (__m512)(~(__v16su)__A & (__v16su)__B);301}302 303static __inline__ __m512 __DEFAULT_FN_ATTRS512304_mm512_mask_andnot_ps(__m512 __W, __mmask16 __U, __m512 __A, __m512 __B) {305 return (__m512)__builtin_ia32_selectps_512((__mmask16)__U,306 (__v16sf)_mm512_andnot_ps(__A, __B),307 (__v16sf)__W);308}309 310static __inline__ __m512 __DEFAULT_FN_ATTRS512311_mm512_maskz_andnot_ps(__mmask16 __U, __m512 __A, __m512 __B) {312 return (__m512)__builtin_ia32_selectps_512((__mmask16)__U,313 (__v16sf)_mm512_andnot_ps(__A, __B),314 (__v16sf)_mm512_setzero_ps());315}316 317static __inline__ __m512i __DEFAULT_FN_ATTRS512318_mm512_cvtpd_epi64 (__m512d __A) {319 return (__m512i) __builtin_ia32_cvtpd2qq512_mask ((__v8df) __A,320 (__v8di) _mm512_setzero_si512(),321 (__mmask8) -1,322 _MM_FROUND_CUR_DIRECTION);323}324 325static __inline__ __m512i __DEFAULT_FN_ATTRS512326_mm512_mask_cvtpd_epi64 (__m512i __W, __mmask8 __U, __m512d __A) {327 return (__m512i) __builtin_ia32_cvtpd2qq512_mask ((__v8df) __A,328 (__v8di) __W,329 (__mmask8) __U,330 _MM_FROUND_CUR_DIRECTION);331}332 333static __inline__ __m512i __DEFAULT_FN_ATTRS512334_mm512_maskz_cvtpd_epi64 (__mmask8 __U, __m512d __A) {335 return (__m512i) __builtin_ia32_cvtpd2qq512_mask ((__v8df) __A,336 (__v8di) _mm512_setzero_si512(),337 (__mmask8) __U,338 _MM_FROUND_CUR_DIRECTION);339}340 341#define _mm512_cvt_roundpd_epi64(A, R) \342 ((__m512i)__builtin_ia32_cvtpd2qq512_mask((__v8df)(__m512d)(A), \343 (__v8di)_mm512_setzero_si512(), \344 (__mmask8)-1, (int)(R)))345 346#define _mm512_mask_cvt_roundpd_epi64(W, U, A, R) \347 ((__m512i)__builtin_ia32_cvtpd2qq512_mask((__v8df)(__m512d)(A), \348 (__v8di)(__m512i)(W), \349 (__mmask8)(U), (int)(R)))350 351#define _mm512_maskz_cvt_roundpd_epi64(U, A, R) \352 ((__m512i)__builtin_ia32_cvtpd2qq512_mask((__v8df)(__m512d)(A), \353 (__v8di)_mm512_setzero_si512(), \354 (__mmask8)(U), (int)(R)))355 356static __inline__ __m512i __DEFAULT_FN_ATTRS512357_mm512_cvtpd_epu64 (__m512d __A) {358 return (__m512i) __builtin_ia32_cvtpd2uqq512_mask ((__v8df) __A,359 (__v8di) _mm512_setzero_si512(),360 (__mmask8) -1,361 _MM_FROUND_CUR_DIRECTION);362}363 364static __inline__ __m512i __DEFAULT_FN_ATTRS512365_mm512_mask_cvtpd_epu64 (__m512i __W, __mmask8 __U, __m512d __A) {366 return (__m512i) __builtin_ia32_cvtpd2uqq512_mask ((__v8df) __A,367 (__v8di) __W,368 (__mmask8) __U,369 _MM_FROUND_CUR_DIRECTION);370}371 372static __inline__ __m512i __DEFAULT_FN_ATTRS512373_mm512_maskz_cvtpd_epu64 (__mmask8 __U, __m512d __A) {374 return (__m512i) __builtin_ia32_cvtpd2uqq512_mask ((__v8df) __A,375 (__v8di) _mm512_setzero_si512(),376 (__mmask8) __U,377 _MM_FROUND_CUR_DIRECTION);378}379 380#define _mm512_cvt_roundpd_epu64(A, R) \381 ((__m512i)__builtin_ia32_cvtpd2uqq512_mask((__v8df)(__m512d)(A), \382 (__v8di)_mm512_setzero_si512(), \383 (__mmask8)-1, (int)(R)))384 385#define _mm512_mask_cvt_roundpd_epu64(W, U, A, R) \386 ((__m512i)__builtin_ia32_cvtpd2uqq512_mask((__v8df)(__m512d)(A), \387 (__v8di)(__m512i)(W), \388 (__mmask8)(U), (int)(R)))389 390#define _mm512_maskz_cvt_roundpd_epu64(U, A, R) \391 ((__m512i)__builtin_ia32_cvtpd2uqq512_mask((__v8df)(__m512d)(A), \392 (__v8di)_mm512_setzero_si512(), \393 (__mmask8)(U), (int)(R)))394 395static __inline__ __m512i __DEFAULT_FN_ATTRS512396_mm512_cvtps_epi64 (__m256 __A) {397 return (__m512i) __builtin_ia32_cvtps2qq512_mask ((__v8sf) __A,398 (__v8di) _mm512_setzero_si512(),399 (__mmask8) -1,400 _MM_FROUND_CUR_DIRECTION);401}402 403static __inline__ __m512i __DEFAULT_FN_ATTRS512404_mm512_mask_cvtps_epi64 (__m512i __W, __mmask8 __U, __m256 __A) {405 return (__m512i) __builtin_ia32_cvtps2qq512_mask ((__v8sf) __A,406 (__v8di) __W,407 (__mmask8) __U,408 _MM_FROUND_CUR_DIRECTION);409}410 411static __inline__ __m512i __DEFAULT_FN_ATTRS512412_mm512_maskz_cvtps_epi64 (__mmask8 __U, __m256 __A) {413 return (__m512i) __builtin_ia32_cvtps2qq512_mask ((__v8sf) __A,414 (__v8di) _mm512_setzero_si512(),415 (__mmask8) __U,416 _MM_FROUND_CUR_DIRECTION);417}418 419#define _mm512_cvt_roundps_epi64(A, R) \420 ((__m512i)__builtin_ia32_cvtps2qq512_mask((__v8sf)(__m256)(A), \421 (__v8di)_mm512_setzero_si512(), \422 (__mmask8)-1, (int)(R)))423 424#define _mm512_mask_cvt_roundps_epi64(W, U, A, R) \425 ((__m512i)__builtin_ia32_cvtps2qq512_mask((__v8sf)(__m256)(A), \426 (__v8di)(__m512i)(W), \427 (__mmask8)(U), (int)(R)))428 429#define _mm512_maskz_cvt_roundps_epi64(U, A, R) \430 ((__m512i)__builtin_ia32_cvtps2qq512_mask((__v8sf)(__m256)(A), \431 (__v8di)_mm512_setzero_si512(), \432 (__mmask8)(U), (int)(R)))433 434static __inline__ __m512i __DEFAULT_FN_ATTRS512435_mm512_cvtps_epu64 (__m256 __A) {436 return (__m512i) __builtin_ia32_cvtps2uqq512_mask ((__v8sf) __A,437 (__v8di) _mm512_setzero_si512(),438 (__mmask8) -1,439 _MM_FROUND_CUR_DIRECTION);440}441 442static __inline__ __m512i __DEFAULT_FN_ATTRS512443_mm512_mask_cvtps_epu64 (__m512i __W, __mmask8 __U, __m256 __A) {444 return (__m512i) __builtin_ia32_cvtps2uqq512_mask ((__v8sf) __A,445 (__v8di) __W,446 (__mmask8) __U,447 _MM_FROUND_CUR_DIRECTION);448}449 450static __inline__ __m512i __DEFAULT_FN_ATTRS512451_mm512_maskz_cvtps_epu64 (__mmask8 __U, __m256 __A) {452 return (__m512i) __builtin_ia32_cvtps2uqq512_mask ((__v8sf) __A,453 (__v8di) _mm512_setzero_si512(),454 (__mmask8) __U,455 _MM_FROUND_CUR_DIRECTION);456}457 458#define _mm512_cvt_roundps_epu64(A, R) \459 ((__m512i)__builtin_ia32_cvtps2uqq512_mask((__v8sf)(__m256)(A), \460 (__v8di)_mm512_setzero_si512(), \461 (__mmask8)-1, (int)(R)))462 463#define _mm512_mask_cvt_roundps_epu64(W, U, A, R) \464 ((__m512i)__builtin_ia32_cvtps2uqq512_mask((__v8sf)(__m256)(A), \465 (__v8di)(__m512i)(W), \466 (__mmask8)(U), (int)(R)))467 468#define _mm512_maskz_cvt_roundps_epu64(U, A, R) \469 ((__m512i)__builtin_ia32_cvtps2uqq512_mask((__v8sf)(__m256)(A), \470 (__v8di)_mm512_setzero_si512(), \471 (__mmask8)(U), (int)(R)))472 473static __inline__ __m512d __DEFAULT_FN_ATTRS512_CONSTEXPR474_mm512_cvtepi64_pd(__m512i __A) {475 return (__m512d)__builtin_convertvector((__v8di)__A, __v8df);476}477 478static __inline__ __m512d __DEFAULT_FN_ATTRS512_CONSTEXPR479_mm512_mask_cvtepi64_pd(__m512d __W, __mmask8 __U, __m512i __A) {480 return (__m512d)__builtin_ia32_selectpd_512((__mmask8)__U,481 (__v8df)_mm512_cvtepi64_pd(__A),482 (__v8df)__W);483}484 485static __inline__ __m512d __DEFAULT_FN_ATTRS512_CONSTEXPR486_mm512_maskz_cvtepi64_pd(__mmask8 __U, __m512i __A) {487 return (__m512d)__builtin_ia32_selectpd_512((__mmask8)__U,488 (__v8df)_mm512_cvtepi64_pd(__A),489 (__v8df)_mm512_setzero_pd());490}491 492#define _mm512_cvt_roundepi64_pd(A, R) \493 ((__m512d)__builtin_ia32_cvtqq2pd512_mask((__v8di)(__m512i)(A), \494 (__v8df)_mm512_setzero_pd(), \495 (__mmask8)-1, (int)(R)))496 497#define _mm512_mask_cvt_roundepi64_pd(W, U, A, R) \498 ((__m512d)__builtin_ia32_cvtqq2pd512_mask((__v8di)(__m512i)(A), \499 (__v8df)(__m512d)(W), \500 (__mmask8)(U), (int)(R)))501 502#define _mm512_maskz_cvt_roundepi64_pd(U, A, R) \503 ((__m512d)__builtin_ia32_cvtqq2pd512_mask((__v8di)(__m512i)(A), \504 (__v8df)_mm512_setzero_pd(), \505 (__mmask8)(U), (int)(R)))506 507static __inline__ __m256 __DEFAULT_FN_ATTRS512508_mm512_cvtepi64_ps (__m512i __A) {509 return (__m256) __builtin_ia32_cvtqq2ps512_mask ((__v8di) __A,510 (__v8sf) _mm256_setzero_ps(),511 (__mmask8) -1,512 _MM_FROUND_CUR_DIRECTION);513}514 515static __inline__ __m256 __DEFAULT_FN_ATTRS512516_mm512_mask_cvtepi64_ps (__m256 __W, __mmask8 __U, __m512i __A) {517 return (__m256) __builtin_ia32_cvtqq2ps512_mask ((__v8di) __A,518 (__v8sf) __W,519 (__mmask8) __U,520 _MM_FROUND_CUR_DIRECTION);521}522 523static __inline__ __m256 __DEFAULT_FN_ATTRS512524_mm512_maskz_cvtepi64_ps (__mmask8 __U, __m512i __A) {525 return (__m256) __builtin_ia32_cvtqq2ps512_mask ((__v8di) __A,526 (__v8sf) _mm256_setzero_ps(),527 (__mmask8) __U,528 _MM_FROUND_CUR_DIRECTION);529}530 531#define _mm512_cvt_roundepi64_ps(A, R) \532 ((__m256)__builtin_ia32_cvtqq2ps512_mask((__v8di)(__m512i)(A), \533 (__v8sf)_mm256_setzero_ps(), \534 (__mmask8)-1, (int)(R)))535 536#define _mm512_mask_cvt_roundepi64_ps(W, U, A, R) \537 ((__m256)__builtin_ia32_cvtqq2ps512_mask((__v8di)(__m512i)(A), \538 (__v8sf)(__m256)(W), (__mmask8)(U), \539 (int)(R)))540 541#define _mm512_maskz_cvt_roundepi64_ps(U, A, R) \542 ((__m256)__builtin_ia32_cvtqq2ps512_mask((__v8di)(__m512i)(A), \543 (__v8sf)_mm256_setzero_ps(), \544 (__mmask8)(U), (int)(R)))545 546 547static __inline__ __m512i __DEFAULT_FN_ATTRS512548_mm512_cvttpd_epi64 (__m512d __A) {549 return (__m512i) __builtin_ia32_cvttpd2qq512_mask ((__v8df) __A,550 (__v8di) _mm512_setzero_si512(),551 (__mmask8) -1,552 _MM_FROUND_CUR_DIRECTION);553}554 555static __inline__ __m512i __DEFAULT_FN_ATTRS512556_mm512_mask_cvttpd_epi64 (__m512i __W, __mmask8 __U, __m512d __A) {557 return (__m512i) __builtin_ia32_cvttpd2qq512_mask ((__v8df) __A,558 (__v8di) __W,559 (__mmask8) __U,560 _MM_FROUND_CUR_DIRECTION);561}562 563static __inline__ __m512i __DEFAULT_FN_ATTRS512564_mm512_maskz_cvttpd_epi64 (__mmask8 __U, __m512d __A) {565 return (__m512i) __builtin_ia32_cvttpd2qq512_mask ((__v8df) __A,566 (__v8di) _mm512_setzero_si512(),567 (__mmask8) __U,568 _MM_FROUND_CUR_DIRECTION);569}570 571#define _mm512_cvtt_roundpd_epi64(A, R) \572 ((__m512i)__builtin_ia32_cvttpd2qq512_mask((__v8df)(__m512d)(A), \573 (__v8di)_mm512_setzero_si512(), \574 (__mmask8)-1, (int)(R)))575 576#define _mm512_mask_cvtt_roundpd_epi64(W, U, A, R) \577 ((__m512i)__builtin_ia32_cvttpd2qq512_mask((__v8df)(__m512d)(A), \578 (__v8di)(__m512i)(W), \579 (__mmask8)(U), (int)(R)))580 581#define _mm512_maskz_cvtt_roundpd_epi64(U, A, R) \582 ((__m512i)__builtin_ia32_cvttpd2qq512_mask((__v8df)(__m512d)(A), \583 (__v8di)_mm512_setzero_si512(), \584 (__mmask8)(U), (int)(R)))585 586static __inline__ __m512i __DEFAULT_FN_ATTRS512587_mm512_cvttpd_epu64 (__m512d __A) {588 return (__m512i) __builtin_ia32_cvttpd2uqq512_mask ((__v8df) __A,589 (__v8di) _mm512_setzero_si512(),590 (__mmask8) -1,591 _MM_FROUND_CUR_DIRECTION);592}593 594static __inline__ __m512i __DEFAULT_FN_ATTRS512595_mm512_mask_cvttpd_epu64 (__m512i __W, __mmask8 __U, __m512d __A) {596 return (__m512i) __builtin_ia32_cvttpd2uqq512_mask ((__v8df) __A,597 (__v8di) __W,598 (__mmask8) __U,599 _MM_FROUND_CUR_DIRECTION);600}601 602static __inline__ __m512i __DEFAULT_FN_ATTRS512603_mm512_maskz_cvttpd_epu64 (__mmask8 __U, __m512d __A) {604 return (__m512i) __builtin_ia32_cvttpd2uqq512_mask ((__v8df) __A,605 (__v8di) _mm512_setzero_si512(),606 (__mmask8) __U,607 _MM_FROUND_CUR_DIRECTION);608}609 610#define _mm512_cvtt_roundpd_epu64(A, R) \611 ((__m512i)__builtin_ia32_cvttpd2uqq512_mask((__v8df)(__m512d)(A), \612 (__v8di)_mm512_setzero_si512(), \613 (__mmask8)-1, (int)(R)))614 615#define _mm512_mask_cvtt_roundpd_epu64(W, U, A, R) \616 ((__m512i)__builtin_ia32_cvttpd2uqq512_mask((__v8df)(__m512d)(A), \617 (__v8di)(__m512i)(W), \618 (__mmask8)(U), (int)(R)))619 620#define _mm512_maskz_cvtt_roundpd_epu64(U, A, R) \621 ((__m512i)__builtin_ia32_cvttpd2uqq512_mask((__v8df)(__m512d)(A), \622 (__v8di)_mm512_setzero_si512(), \623 (__mmask8)(U), (int)(R)))624 625static __inline__ __m512i __DEFAULT_FN_ATTRS512626_mm512_cvttps_epi64 (__m256 __A) {627 return (__m512i) __builtin_ia32_cvttps2qq512_mask ((__v8sf) __A,628 (__v8di) _mm512_setzero_si512(),629 (__mmask8) -1,630 _MM_FROUND_CUR_DIRECTION);631}632 633static __inline__ __m512i __DEFAULT_FN_ATTRS512634_mm512_mask_cvttps_epi64 (__m512i __W, __mmask8 __U, __m256 __A) {635 return (__m512i) __builtin_ia32_cvttps2qq512_mask ((__v8sf) __A,636 (__v8di) __W,637 (__mmask8) __U,638 _MM_FROUND_CUR_DIRECTION);639}640 641static __inline__ __m512i __DEFAULT_FN_ATTRS512642_mm512_maskz_cvttps_epi64 (__mmask8 __U, __m256 __A) {643 return (__m512i) __builtin_ia32_cvttps2qq512_mask ((__v8sf) __A,644 (__v8di) _mm512_setzero_si512(),645 (__mmask8) __U,646 _MM_FROUND_CUR_DIRECTION);647}648 649#define _mm512_cvtt_roundps_epi64(A, R) \650 ((__m512i)__builtin_ia32_cvttps2qq512_mask((__v8sf)(__m256)(A), \651 (__v8di)_mm512_setzero_si512(), \652 (__mmask8)-1, (int)(R)))653 654#define _mm512_mask_cvtt_roundps_epi64(W, U, A, R) \655 ((__m512i)__builtin_ia32_cvttps2qq512_mask((__v8sf)(__m256)(A), \656 (__v8di)(__m512i)(W), \657 (__mmask8)(U), (int)(R)))658 659#define _mm512_maskz_cvtt_roundps_epi64(U, A, R) \660 ((__m512i)__builtin_ia32_cvttps2qq512_mask((__v8sf)(__m256)(A), \661 (__v8di)_mm512_setzero_si512(), \662 (__mmask8)(U), (int)(R)))663 664static __inline__ __m512i __DEFAULT_FN_ATTRS512665_mm512_cvttps_epu64 (__m256 __A) {666 return (__m512i) __builtin_ia32_cvttps2uqq512_mask ((__v8sf) __A,667 (__v8di) _mm512_setzero_si512(),668 (__mmask8) -1,669 _MM_FROUND_CUR_DIRECTION);670}671 672static __inline__ __m512i __DEFAULT_FN_ATTRS512673_mm512_mask_cvttps_epu64 (__m512i __W, __mmask8 __U, __m256 __A) {674 return (__m512i) __builtin_ia32_cvttps2uqq512_mask ((__v8sf) __A,675 (__v8di) __W,676 (__mmask8) __U,677 _MM_FROUND_CUR_DIRECTION);678}679 680static __inline__ __m512i __DEFAULT_FN_ATTRS512681_mm512_maskz_cvttps_epu64 (__mmask8 __U, __m256 __A) {682 return (__m512i) __builtin_ia32_cvttps2uqq512_mask ((__v8sf) __A,683 (__v8di) _mm512_setzero_si512(),684 (__mmask8) __U,685 _MM_FROUND_CUR_DIRECTION);686}687 688#define _mm512_cvtt_roundps_epu64(A, R) \689 ((__m512i)__builtin_ia32_cvttps2uqq512_mask((__v8sf)(__m256)(A), \690 (__v8di)_mm512_setzero_si512(), \691 (__mmask8)-1, (int)(R)))692 693#define _mm512_mask_cvtt_roundps_epu64(W, U, A, R) \694 ((__m512i)__builtin_ia32_cvttps2uqq512_mask((__v8sf)(__m256)(A), \695 (__v8di)(__m512i)(W), \696 (__mmask8)(U), (int)(R)))697 698#define _mm512_maskz_cvtt_roundps_epu64(U, A, R) \699 ((__m512i)__builtin_ia32_cvttps2uqq512_mask((__v8sf)(__m256)(A), \700 (__v8di)_mm512_setzero_si512(), \701 (__mmask8)(U), (int)(R)))702 703static __inline__ __m512d __DEFAULT_FN_ATTRS512_CONSTEXPR704_mm512_cvtepu64_pd(__m512i __A) {705 return (__m512d)__builtin_convertvector((__v8du)__A, __v8df);706}707 708static __inline__ __m512d __DEFAULT_FN_ATTRS512_CONSTEXPR709_mm512_mask_cvtepu64_pd(__m512d __W, __mmask8 __U, __m512i __A) {710 return (__m512d)__builtin_ia32_selectpd_512((__mmask8)__U,711 (__v8df)_mm512_cvtepu64_pd(__A),712 (__v8df)__W);713}714 715static __inline__ __m512d __DEFAULT_FN_ATTRS512_CONSTEXPR716_mm512_maskz_cvtepu64_pd(__mmask8 __U, __m512i __A) {717 return (__m512d)__builtin_ia32_selectpd_512((__mmask8)__U,718 (__v8df)_mm512_cvtepu64_pd(__A),719 (__v8df)_mm512_setzero_pd());720}721 722#define _mm512_cvt_roundepu64_pd(A, R) \723 ((__m512d)__builtin_ia32_cvtuqq2pd512_mask((__v8di)(__m512i)(A), \724 (__v8df)_mm512_setzero_pd(), \725 (__mmask8)-1, (int)(R)))726 727#define _mm512_mask_cvt_roundepu64_pd(W, U, A, R) \728 ((__m512d)__builtin_ia32_cvtuqq2pd512_mask((__v8di)(__m512i)(A), \729 (__v8df)(__m512d)(W), \730 (__mmask8)(U), (int)(R)))731 732 733#define _mm512_maskz_cvt_roundepu64_pd(U, A, R) \734 ((__m512d)__builtin_ia32_cvtuqq2pd512_mask((__v8di)(__m512i)(A), \735 (__v8df)_mm512_setzero_pd(), \736 (__mmask8)(U), (int)(R)))737 738 739static __inline__ __m256 __DEFAULT_FN_ATTRS512740_mm512_cvtepu64_ps (__m512i __A) {741 return (__m256) __builtin_ia32_cvtuqq2ps512_mask ((__v8di) __A,742 (__v8sf) _mm256_setzero_ps(),743 (__mmask8) -1,744 _MM_FROUND_CUR_DIRECTION);745}746 747static __inline__ __m256 __DEFAULT_FN_ATTRS512748_mm512_mask_cvtepu64_ps (__m256 __W, __mmask8 __U, __m512i __A) {749 return (__m256) __builtin_ia32_cvtuqq2ps512_mask ((__v8di) __A,750 (__v8sf) __W,751 (__mmask8) __U,752 _MM_FROUND_CUR_DIRECTION);753}754 755static __inline__ __m256 __DEFAULT_FN_ATTRS512756_mm512_maskz_cvtepu64_ps (__mmask8 __U, __m512i __A) {757 return (__m256) __builtin_ia32_cvtuqq2ps512_mask ((__v8di) __A,758 (__v8sf) _mm256_setzero_ps(),759 (__mmask8) __U,760 _MM_FROUND_CUR_DIRECTION);761}762 763#define _mm512_cvt_roundepu64_ps(A, R) \764 ((__m256)__builtin_ia32_cvtuqq2ps512_mask((__v8di)(__m512i)(A), \765 (__v8sf)_mm256_setzero_ps(), \766 (__mmask8)-1, (int)(R)))767 768#define _mm512_mask_cvt_roundepu64_ps(W, U, A, R) \769 ((__m256)__builtin_ia32_cvtuqq2ps512_mask((__v8di)(__m512i)(A), \770 (__v8sf)(__m256)(W), (__mmask8)(U), \771 (int)(R)))772 773#define _mm512_maskz_cvt_roundepu64_ps(U, A, R) \774 ((__m256)__builtin_ia32_cvtuqq2ps512_mask((__v8di)(__m512i)(A), \775 (__v8sf)_mm256_setzero_ps(), \776 (__mmask8)(U), (int)(R)))777 778#define _mm512_range_pd(A, B, C) \779 ((__m512d)__builtin_ia32_rangepd512_mask((__v8df)(__m512d)(A), \780 (__v8df)(__m512d)(B), (int)(C), \781 (__v8df)_mm512_setzero_pd(), \782 (__mmask8)-1, \783 _MM_FROUND_CUR_DIRECTION))784 785#define _mm512_mask_range_pd(W, U, A, B, C) \786 ((__m512d)__builtin_ia32_rangepd512_mask((__v8df)(__m512d)(A), \787 (__v8df)(__m512d)(B), (int)(C), \788 (__v8df)(__m512d)(W), (__mmask8)(U), \789 _MM_FROUND_CUR_DIRECTION))790 791#define _mm512_maskz_range_pd(U, A, B, C) \792 ((__m512d)__builtin_ia32_rangepd512_mask((__v8df)(__m512d)(A), \793 (__v8df)(__m512d)(B), (int)(C), \794 (__v8df)_mm512_setzero_pd(), \795 (__mmask8)(U), \796 _MM_FROUND_CUR_DIRECTION))797 798#define _mm512_range_round_pd(A, B, C, R) \799 ((__m512d)__builtin_ia32_rangepd512_mask((__v8df)(__m512d)(A), \800 (__v8df)(__m512d)(B), (int)(C), \801 (__v8df)_mm512_setzero_pd(), \802 (__mmask8)-1, (int)(R)))803 804#define _mm512_mask_range_round_pd(W, U, A, B, C, R) \805 ((__m512d)__builtin_ia32_rangepd512_mask((__v8df)(__m512d)(A), \806 (__v8df)(__m512d)(B), (int)(C), \807 (__v8df)(__m512d)(W), (__mmask8)(U), \808 (int)(R)))809 810#define _mm512_maskz_range_round_pd(U, A, B, C, R) \811 ((__m512d)__builtin_ia32_rangepd512_mask((__v8df)(__m512d)(A), \812 (__v8df)(__m512d)(B), (int)(C), \813 (__v8df)_mm512_setzero_pd(), \814 (__mmask8)(U), (int)(R)))815 816#define _mm512_range_ps(A, B, C) \817 ((__m512)__builtin_ia32_rangeps512_mask((__v16sf)(__m512)(A), \818 (__v16sf)(__m512)(B), (int)(C), \819 (__v16sf)_mm512_setzero_ps(), \820 (__mmask16)-1, \821 _MM_FROUND_CUR_DIRECTION))822 823#define _mm512_mask_range_ps(W, U, A, B, C) \824 ((__m512)__builtin_ia32_rangeps512_mask((__v16sf)(__m512)(A), \825 (__v16sf)(__m512)(B), (int)(C), \826 (__v16sf)(__m512)(W), (__mmask16)(U), \827 _MM_FROUND_CUR_DIRECTION))828 829#define _mm512_maskz_range_ps(U, A, B, C) \830 ((__m512)__builtin_ia32_rangeps512_mask((__v16sf)(__m512)(A), \831 (__v16sf)(__m512)(B), (int)(C), \832 (__v16sf)_mm512_setzero_ps(), \833 (__mmask16)(U), \834 _MM_FROUND_CUR_DIRECTION))835 836#define _mm512_range_round_ps(A, B, C, R) \837 ((__m512)__builtin_ia32_rangeps512_mask((__v16sf)(__m512)(A), \838 (__v16sf)(__m512)(B), (int)(C), \839 (__v16sf)_mm512_setzero_ps(), \840 (__mmask16)-1, (int)(R)))841 842#define _mm512_mask_range_round_ps(W, U, A, B, C, R) \843 ((__m512)__builtin_ia32_rangeps512_mask((__v16sf)(__m512)(A), \844 (__v16sf)(__m512)(B), (int)(C), \845 (__v16sf)(__m512)(W), (__mmask16)(U), \846 (int)(R)))847 848#define _mm512_maskz_range_round_ps(U, A, B, C, R) \849 ((__m512)__builtin_ia32_rangeps512_mask((__v16sf)(__m512)(A), \850 (__v16sf)(__m512)(B), (int)(C), \851 (__v16sf)_mm512_setzero_ps(), \852 (__mmask16)(U), (int)(R)))853 854#define _mm_range_round_ss(A, B, C, R) \855 ((__m128)__builtin_ia32_rangess128_round_mask((__v4sf)(__m128)(A), \856 (__v4sf)(__m128)(B), \857 (__v4sf)_mm_setzero_ps(), \858 (__mmask8) -1, (int)(C),\859 (int)(R)))860 861#define _mm_range_ss(A ,B , C) _mm_range_round_ss(A, B, C ,_MM_FROUND_CUR_DIRECTION)862 863#define _mm_mask_range_round_ss(W, U, A, B, C, R) \864 ((__m128)__builtin_ia32_rangess128_round_mask((__v4sf)(__m128)(A), \865 (__v4sf)(__m128)(B), \866 (__v4sf)(__m128)(W),\867 (__mmask8)(U), (int)(C),\868 (int)(R)))869 870#define _mm_mask_range_ss(W , U, A, B, C) _mm_mask_range_round_ss(W, U, A, B, C , _MM_FROUND_CUR_DIRECTION)871 872#define _mm_maskz_range_round_ss(U, A, B, C, R) \873 ((__m128)__builtin_ia32_rangess128_round_mask((__v4sf)(__m128)(A), \874 (__v4sf)(__m128)(B), \875 (__v4sf)_mm_setzero_ps(), \876 (__mmask8)(U), (int)(C),\877 (int)(R)))878 879#define _mm_maskz_range_ss(U, A ,B , C) _mm_maskz_range_round_ss(U, A, B, C ,_MM_FROUND_CUR_DIRECTION)880 881#define _mm_range_round_sd(A, B, C, R) \882 ((__m128d)__builtin_ia32_rangesd128_round_mask((__v2df)(__m128d)(A), \883 (__v2df)(__m128d)(B), \884 (__v2df)_mm_setzero_pd(), \885 (__mmask8) -1, (int)(C),\886 (int)(R)))887 888#define _mm_range_sd(A ,B , C) _mm_range_round_sd(A, B, C ,_MM_FROUND_CUR_DIRECTION)889 890#define _mm_mask_range_round_sd(W, U, A, B, C, R) \891 ((__m128d)__builtin_ia32_rangesd128_round_mask((__v2df)(__m128d)(A), \892 (__v2df)(__m128d)(B), \893 (__v2df)(__m128d)(W),\894 (__mmask8)(U), (int)(C),\895 (int)(R)))896 897#define _mm_mask_range_sd(W, U, A, B, C) _mm_mask_range_round_sd(W, U, A, B, C ,_MM_FROUND_CUR_DIRECTION)898 899#define _mm_maskz_range_round_sd(U, A, B, C, R) \900 ((__m128d)__builtin_ia32_rangesd128_round_mask((__v2df)(__m128d)(A), \901 (__v2df)(__m128d)(B), \902 (__v2df)_mm_setzero_pd(), \903 (__mmask8)(U), (int)(C),\904 (int)(R)))905 906#define _mm_maskz_range_sd(U, A, B, C) _mm_maskz_range_round_sd(U, A, B, C ,_MM_FROUND_CUR_DIRECTION)907 908#define _mm512_reduce_pd(A, B) \909 ((__m512d)__builtin_ia32_reducepd512_mask((__v8df)(__m512d)(A), (int)(B), \910 (__v8df)_mm512_setzero_pd(), \911 (__mmask8)-1, \912 _MM_FROUND_CUR_DIRECTION))913 914#define _mm512_mask_reduce_pd(W, U, A, B) \915 ((__m512d)__builtin_ia32_reducepd512_mask((__v8df)(__m512d)(A), (int)(B), \916 (__v8df)(__m512d)(W), \917 (__mmask8)(U), \918 _MM_FROUND_CUR_DIRECTION))919 920#define _mm512_maskz_reduce_pd(U, A, B) \921 ((__m512d)__builtin_ia32_reducepd512_mask((__v8df)(__m512d)(A), (int)(B), \922 (__v8df)_mm512_setzero_pd(), \923 (__mmask8)(U), \924 _MM_FROUND_CUR_DIRECTION))925 926#define _mm512_reduce_ps(A, B) \927 ((__m512)__builtin_ia32_reduceps512_mask((__v16sf)(__m512)(A), (int)(B), \928 (__v16sf)_mm512_setzero_ps(), \929 (__mmask16)-1, \930 _MM_FROUND_CUR_DIRECTION))931 932#define _mm512_mask_reduce_ps(W, U, A, B) \933 ((__m512)__builtin_ia32_reduceps512_mask((__v16sf)(__m512)(A), (int)(B), \934 (__v16sf)(__m512)(W), \935 (__mmask16)(U), \936 _MM_FROUND_CUR_DIRECTION))937 938#define _mm512_maskz_reduce_ps(U, A, B) \939 ((__m512)__builtin_ia32_reduceps512_mask((__v16sf)(__m512)(A), (int)(B), \940 (__v16sf)_mm512_setzero_ps(), \941 (__mmask16)(U), \942 _MM_FROUND_CUR_DIRECTION))943 944#define _mm512_reduce_round_pd(A, B, R) \945 ((__m512d)__builtin_ia32_reducepd512_mask((__v8df)(__m512d)(A), (int)(B), \946 (__v8df)_mm512_setzero_pd(), \947 (__mmask8)-1, (int)(R)))948 949#define _mm512_mask_reduce_round_pd(W, U, A, B, R) \950 ((__m512d)__builtin_ia32_reducepd512_mask((__v8df)(__m512d)(A), (int)(B), \951 (__v8df)(__m512d)(W), \952 (__mmask8)(U), (int)(R)))953 954#define _mm512_maskz_reduce_round_pd(U, A, B, R) \955 ((__m512d)__builtin_ia32_reducepd512_mask((__v8df)(__m512d)(A), (int)(B), \956 (__v8df)_mm512_setzero_pd(), \957 (__mmask8)(U), (int)(R)))958 959#define _mm512_reduce_round_ps(A, B, R) \960 ((__m512)__builtin_ia32_reduceps512_mask((__v16sf)(__m512)(A), (int)(B), \961 (__v16sf)_mm512_setzero_ps(), \962 (__mmask16)-1, (int)(R)))963 964#define _mm512_mask_reduce_round_ps(W, U, A, B, R) \965 ((__m512)__builtin_ia32_reduceps512_mask((__v16sf)(__m512)(A), (int)(B), \966 (__v16sf)(__m512)(W), \967 (__mmask16)(U), (int)(R)))968 969#define _mm512_maskz_reduce_round_ps(U, A, B, R) \970 ((__m512)__builtin_ia32_reduceps512_mask((__v16sf)(__m512)(A), (int)(B), \971 (__v16sf)_mm512_setzero_ps(), \972 (__mmask16)(U), (int)(R)))973 974#define _mm_reduce_ss(A, B, C) \975 ((__m128)__builtin_ia32_reducess_mask((__v4sf)(__m128)(A), \976 (__v4sf)(__m128)(B), \977 (__v4sf)_mm_setzero_ps(), (__mmask8)-1, \978 (int)(C), _MM_FROUND_CUR_DIRECTION))979 980#define _mm_mask_reduce_ss(W, U, A, B, C) \981 ((__m128)__builtin_ia32_reducess_mask((__v4sf)(__m128)(A), \982 (__v4sf)(__m128)(B), \983 (__v4sf)(__m128)(W), (__mmask8)(U), \984 (int)(C), _MM_FROUND_CUR_DIRECTION))985 986#define _mm_maskz_reduce_ss(U, A, B, C) \987 ((__m128)__builtin_ia32_reducess_mask((__v4sf)(__m128)(A), \988 (__v4sf)(__m128)(B), \989 (__v4sf)_mm_setzero_ps(), \990 (__mmask8)(U), (int)(C), \991 _MM_FROUND_CUR_DIRECTION))992 993#define _mm_reduce_round_ss(A, B, C, R) \994 ((__m128)__builtin_ia32_reducess_mask((__v4sf)(__m128)(A), \995 (__v4sf)(__m128)(B), \996 (__v4sf)_mm_setzero_ps(), (__mmask8)-1, \997 (int)(C), (int)(R)))998 999#define _mm_mask_reduce_round_ss(W, U, A, B, C, R) \1000 ((__m128)__builtin_ia32_reducess_mask((__v4sf)(__m128)(A), \1001 (__v4sf)(__m128)(B), \1002 (__v4sf)(__m128)(W), (__mmask8)(U), \1003 (int)(C), (int)(R)))1004 1005#define _mm_maskz_reduce_round_ss(U, A, B, C, R) \1006 ((__m128)__builtin_ia32_reducess_mask((__v4sf)(__m128)(A), \1007 (__v4sf)(__m128)(B), \1008 (__v4sf)_mm_setzero_ps(), \1009 (__mmask8)(U), (int)(C), (int)(R)))1010 1011#define _mm_reduce_sd(A, B, C) \1012 ((__m128d)__builtin_ia32_reducesd_mask((__v2df)(__m128d)(A), \1013 (__v2df)(__m128d)(B), \1014 (__v2df)_mm_setzero_pd(), \1015 (__mmask8)-1, (int)(C), \1016 _MM_FROUND_CUR_DIRECTION))1017 1018#define _mm_mask_reduce_sd(W, U, A, B, C) \1019 ((__m128d)__builtin_ia32_reducesd_mask((__v2df)(__m128d)(A), \1020 (__v2df)(__m128d)(B), \1021 (__v2df)(__m128d)(W), (__mmask8)(U), \1022 (int)(C), _MM_FROUND_CUR_DIRECTION))1023 1024#define _mm_maskz_reduce_sd(U, A, B, C) \1025 ((__m128d)__builtin_ia32_reducesd_mask((__v2df)(__m128d)(A), \1026 (__v2df)(__m128d)(B), \1027 (__v2df)_mm_setzero_pd(), \1028 (__mmask8)(U), (int)(C), \1029 _MM_FROUND_CUR_DIRECTION))1030 1031#define _mm_reduce_round_sd(A, B, C, R) \1032 ((__m128d)__builtin_ia32_reducesd_mask((__v2df)(__m128d)(A), \1033 (__v2df)(__m128d)(B), \1034 (__v2df)_mm_setzero_pd(), \1035 (__mmask8)-1, (int)(C), (int)(R)))1036 1037#define _mm_mask_reduce_round_sd(W, U, A, B, C, R) \1038 ((__m128d)__builtin_ia32_reducesd_mask((__v2df)(__m128d)(A), \1039 (__v2df)(__m128d)(B), \1040 (__v2df)(__m128d)(W), (__mmask8)(U), \1041 (int)(C), (int)(R)))1042 1043#define _mm_maskz_reduce_round_sd(U, A, B, C, R) \1044 ((__m128d)__builtin_ia32_reducesd_mask((__v2df)(__m128d)(A), \1045 (__v2df)(__m128d)(B), \1046 (__v2df)_mm_setzero_pd(), \1047 (__mmask8)(U), (int)(C), (int)(R)))1048 1049static __inline__ __mmask161050 __DEFAULT_FN_ATTRS512_CONSTEXPR _mm512_movepi32_mask(__m512i __A) {1051 return (__mmask16) __builtin_ia32_cvtd2mask512 ((__v16si) __A);1052}1053 1054static __inline__ __m512i __DEFAULT_FN_ATTRS5121055_mm512_movm_epi32 (__mmask16 __A)1056{1057 return (__m512i) __builtin_ia32_cvtmask2d512 (__A);1058}1059 1060static __inline__ __m512i __DEFAULT_FN_ATTRS5121061_mm512_movm_epi64 (__mmask8 __A)1062{1063 return (__m512i) __builtin_ia32_cvtmask2q512 (__A);1064}1065 1066static __inline__ __mmask8 __DEFAULT_FN_ATTRS512_CONSTEXPR1067_mm512_movepi64_mask(__m512i __A) {1068 return (__mmask8) __builtin_ia32_cvtq2mask512 ((__v8di) __A);1069}1070 1071static __inline__ __m512 __DEFAULT_FN_ATTRS512_CONSTEXPR1072_mm512_broadcast_f32x2(__m128 __A) {1073 return (__m512)__builtin_shufflevector((__v4sf)__A, (__v4sf)__A,1074 0, 1, 0, 1, 0, 1, 0, 1,1075 0, 1, 0, 1, 0, 1, 0, 1);1076}1077 1078static __inline__ __m512 __DEFAULT_FN_ATTRS512_CONSTEXPR1079_mm512_mask_broadcast_f32x2(__m512 __O, __mmask16 __M, __m128 __A) {1080 return (__m512)__builtin_ia32_selectps_512((__mmask16)__M,1081 (__v16sf)_mm512_broadcast_f32x2(__A),1082 (__v16sf)__O);1083}1084 1085static __inline__ __m512 __DEFAULT_FN_ATTRS512_CONSTEXPR1086_mm512_maskz_broadcast_f32x2(__mmask16 __M, __m128 __A) {1087 return (__m512)__builtin_ia32_selectps_512((__mmask16)__M,1088 (__v16sf)_mm512_broadcast_f32x2(__A),1089 (__v16sf)_mm512_setzero_ps());1090}1091 1092static __inline__ __m512 __DEFAULT_FN_ATTRS512_CONSTEXPR1093_mm512_broadcast_f32x8(__m256 __A) {1094 return (__m512)__builtin_shufflevector((__v8sf)__A, (__v8sf)__A,1095 0, 1, 2, 3, 4, 5, 6, 7,1096 0, 1, 2, 3, 4, 5, 6, 7);1097}1098 1099static __inline__ __m512 __DEFAULT_FN_ATTRS512_CONSTEXPR1100_mm512_mask_broadcast_f32x8(__m512 __O, __mmask16 __M, __m256 __A) {1101 return (__m512)__builtin_ia32_selectps_512((__mmask16)__M,1102 (__v16sf)_mm512_broadcast_f32x8(__A),1103 (__v16sf)__O);1104}1105 1106static __inline__ __m512 __DEFAULT_FN_ATTRS512_CONSTEXPR1107_mm512_maskz_broadcast_f32x8(__mmask16 __M, __m256 __A) {1108 return (__m512)__builtin_ia32_selectps_512((__mmask16)__M,1109 (__v16sf)_mm512_broadcast_f32x8(__A),1110 (__v16sf)_mm512_setzero_ps());1111}1112 1113static __inline__ __m512d __DEFAULT_FN_ATTRS512_CONSTEXPR1114_mm512_broadcast_f64x2(__m128d __A) {1115 return (__m512d)__builtin_shufflevector((__v2df)__A, (__v2df)__A,1116 0, 1, 0, 1, 0, 1, 0, 1);1117}1118 1119static __inline__ __m512d __DEFAULT_FN_ATTRS512_CONSTEXPR1120_mm512_mask_broadcast_f64x2(__m512d __O, __mmask8 __M, __m128d __A) {1121 return (__m512d)__builtin_ia32_selectpd_512((__mmask8)__M,1122 (__v8df)_mm512_broadcast_f64x2(__A),1123 (__v8df)__O);1124}1125 1126static __inline__ __m512d __DEFAULT_FN_ATTRS512_CONSTEXPR1127_mm512_maskz_broadcast_f64x2(__mmask8 __M, __m128d __A) {1128 return (__m512d)__builtin_ia32_selectpd_512((__mmask8)__M,1129 (__v8df)_mm512_broadcast_f64x2(__A),1130 (__v8df)_mm512_setzero_pd());1131}1132 1133static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR1134_mm512_broadcast_i32x2(__m128i __A) {1135 return (__m512i)__builtin_shufflevector((__v4si)__A, (__v4si)__A,1136 0, 1, 0, 1, 0, 1, 0, 1,1137 0, 1, 0, 1, 0, 1, 0, 1);1138}1139 1140static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR1141_mm512_mask_broadcast_i32x2(__m512i __O, __mmask16 __M, __m128i __A) {1142 return (__m512i)__builtin_ia32_selectd_512((__mmask16)__M,1143 (__v16si)_mm512_broadcast_i32x2(__A),1144 (__v16si)__O);1145}1146 1147static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR1148_mm512_maskz_broadcast_i32x2(__mmask16 __M, __m128i __A) {1149 return (__m512i)__builtin_ia32_selectd_512((__mmask16)__M,1150 (__v16si)_mm512_broadcast_i32x2(__A),1151 (__v16si)_mm512_setzero_si512());1152}1153 1154static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR1155_mm512_broadcast_i32x8(__m256i __A) {1156 return (__m512i)__builtin_shufflevector((__v8si)__A, (__v8si)__A,1157 0, 1, 2, 3, 4, 5, 6, 7,1158 0, 1, 2, 3, 4, 5, 6, 7);1159}1160 1161static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR1162_mm512_mask_broadcast_i32x8(__m512i __O, __mmask16 __M, __m256i __A) {1163 return (__m512i)__builtin_ia32_selectd_512((__mmask16)__M,1164 (__v16si)_mm512_broadcast_i32x8(__A),1165 (__v16si)__O);1166}1167 1168static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR1169_mm512_maskz_broadcast_i32x8(__mmask16 __M, __m256i __A) {1170 return (__m512i)__builtin_ia32_selectd_512((__mmask16)__M,1171 (__v16si)_mm512_broadcast_i32x8(__A),1172 (__v16si)_mm512_setzero_si512());1173}1174 1175static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR1176_mm512_broadcast_i64x2(__m128i __A) {1177 return (__m512i)__builtin_shufflevector((__v2di)__A, (__v2di)__A,1178 0, 1, 0, 1, 0, 1, 0, 1);1179}1180 1181static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR1182_mm512_mask_broadcast_i64x2(__m512i __O, __mmask8 __M, __m128i __A) {1183 return (__m512i)__builtin_ia32_selectq_512((__mmask8)__M,1184 (__v8di)_mm512_broadcast_i64x2(__A),1185 (__v8di)__O);1186}1187 1188static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR1189_mm512_maskz_broadcast_i64x2(__mmask8 __M, __m128i __A) {1190 return (__m512i)__builtin_ia32_selectq_512((__mmask8)__M,1191 (__v8di)_mm512_broadcast_i64x2(__A),1192 (__v8di)_mm512_setzero_si512());1193}1194 1195#define _mm512_extractf32x8_ps(A, imm) \1196 ((__m256)__builtin_ia32_extractf32x8_mask((__v16sf)(__m512)(A), (int)(imm), \1197 (__v8sf)_mm256_setzero_ps(), \1198 (__mmask8) - 1))1199 1200#define _mm512_mask_extractf32x8_ps(W, U, A, imm) \1201 ((__m256)__builtin_ia32_extractf32x8_mask((__v16sf)(__m512)(A), (int)(imm), \1202 (__v8sf)(__m256)(W), \1203 (__mmask8)(U)))1204 1205#define _mm512_maskz_extractf32x8_ps(U, A, imm) \1206 ((__m256)__builtin_ia32_extractf32x8_mask((__v16sf)(__m512)(A), (int)(imm), \1207 (__v8sf)_mm256_setzero_ps(), \1208 (__mmask8)(U)))1209 1210#define _mm512_extractf64x2_pd(A, imm) \1211 ((__m128d)__builtin_ia32_extractf64x2_512_mask( \1212 (__v8df)(__m512d)(A), (int)(imm), (__v2df)_mm_setzero_pd(), \1213 (__mmask8) - 1))1214 1215#define _mm512_mask_extractf64x2_pd(W, U, A, imm) \1216 ((__m128d)__builtin_ia32_extractf64x2_512_mask((__v8df)(__m512d)(A), \1217 (int)(imm), \1218 (__v2df)(__m128d)(W), \1219 (__mmask8)(U)))1220 1221#define _mm512_maskz_extractf64x2_pd(U, A, imm) \1222 ((__m128d)__builtin_ia32_extractf64x2_512_mask((__v8df)(__m512d)(A), \1223 (int)(imm), \1224 (__v2df)_mm_setzero_pd(), \1225 (__mmask8)(U)))1226 1227#define _mm512_extracti32x8_epi32(A, imm) \1228 ((__m256i)__builtin_ia32_extracti32x8_mask( \1229 (__v16si)(__m512i)(A), (int)(imm), (__v8si)_mm256_setzero_si256(), \1230 (__mmask8) - 1))1231 1232#define _mm512_mask_extracti32x8_epi32(W, U, A, imm) \1233 ((__m256i)__builtin_ia32_extracti32x8_mask((__v16si)(__m512i)(A), (int)(imm), \1234 (__v8si)(__m256i)(W), \1235 (__mmask8)(U)))1236 1237#define _mm512_maskz_extracti32x8_epi32(U, A, imm) \1238 ((__m256i)__builtin_ia32_extracti32x8_mask((__v16si)(__m512i)(A), (int)(imm), \1239 (__v8si)_mm256_setzero_si256(), \1240 (__mmask8)(U)))1241 1242#define _mm512_extracti64x2_epi64(A, imm) \1243 ((__m128i)__builtin_ia32_extracti64x2_512_mask( \1244 (__v8di)(__m512i)(A), (int)(imm), (__v2di)_mm_setzero_si128(), \1245 (__mmask8) - 1))1246 1247#define _mm512_mask_extracti64x2_epi64(W, U, A, imm) \1248 ((__m128i)__builtin_ia32_extracti64x2_512_mask((__v8di)(__m512i)(A), \1249 (int)(imm), \1250 (__v2di)(__m128i)(W), \1251 (__mmask8)(U)))1252 1253#define _mm512_maskz_extracti64x2_epi64(U, A, imm) \1254 ((__m128i)__builtin_ia32_extracti64x2_512_mask((__v8di)(__m512i)(A), \1255 (int)(imm), \1256 (__v2di)_mm_setzero_si128(), \1257 (__mmask8)(U)))1258 1259#define _mm512_insertf32x8(A, B, imm) \1260 ((__m512)__builtin_ia32_insertf32x8((__v16sf)(__m512)(A), \1261 (__v8sf)(__m256)(B), (int)(imm)))1262 1263#define _mm512_mask_insertf32x8(W, U, A, B, imm) \1264 ((__m512)__builtin_ia32_selectps_512((__mmask16)(U), \1265 (__v16sf)_mm512_insertf32x8((A), (B), (imm)), \1266 (__v16sf)(__m512)(W)))1267 1268#define _mm512_maskz_insertf32x8(U, A, B, imm) \1269 ((__m512)__builtin_ia32_selectps_512((__mmask16)(U), \1270 (__v16sf)_mm512_insertf32x8((A), (B), (imm)), \1271 (__v16sf)_mm512_setzero_ps()))1272 1273#define _mm512_insertf64x2(A, B, imm) \1274 ((__m512d)__builtin_ia32_insertf64x2_512((__v8df)(__m512d)(A), \1275 (__v2df)(__m128d)(B), (int)(imm)))1276 1277#define _mm512_mask_insertf64x2(W, U, A, B, imm) \1278 ((__m512d)__builtin_ia32_selectpd_512((__mmask8)(U), \1279 (__v8df)_mm512_insertf64x2((A), (B), (imm)), \1280 (__v8df)(__m512d)(W)))1281 1282#define _mm512_maskz_insertf64x2(U, A, B, imm) \1283 ((__m512d)__builtin_ia32_selectpd_512((__mmask8)(U), \1284 (__v8df)_mm512_insertf64x2((A), (B), (imm)), \1285 (__v8df)_mm512_setzero_pd()))1286 1287#define _mm512_inserti32x8(A, B, imm) \1288 ((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(A), \1289 (__v8si)(__m256i)(B), (int)(imm)))1290 1291#define _mm512_mask_inserti32x8(W, U, A, B, imm) \1292 ((__m512i)__builtin_ia32_selectd_512((__mmask16)(U), \1293 (__v16si)_mm512_inserti32x8((A), (B), (imm)), \1294 (__v16si)(__m512i)(W)))1295 1296#define _mm512_maskz_inserti32x8(U, A, B, imm) \1297 ((__m512i)__builtin_ia32_selectd_512((__mmask16)(U), \1298 (__v16si)_mm512_inserti32x8((A), (B), (imm)), \1299 (__v16si)_mm512_setzero_si512()))1300 1301#define _mm512_inserti64x2(A, B, imm) \1302 ((__m512i)__builtin_ia32_inserti64x2_512((__v8di)(__m512i)(A), \1303 (__v2di)(__m128i)(B), (int)(imm)))1304 1305#define _mm512_mask_inserti64x2(W, U, A, B, imm) \1306 ((__m512i)__builtin_ia32_selectq_512((__mmask8)(U), \1307 (__v8di)_mm512_inserti64x2((A), (B), (imm)), \1308 (__v8di)(__m512i)(W)))1309 1310#define _mm512_maskz_inserti64x2(U, A, B, imm) \1311 ((__m512i)__builtin_ia32_selectq_512((__mmask8)(U), \1312 (__v8di)_mm512_inserti64x2((A), (B), (imm)), \1313 (__v8di)_mm512_setzero_si512()))1314 1315#define _mm512_mask_fpclass_ps_mask(U, A, imm) \1316 ((__mmask16)__builtin_ia32_fpclassps512_mask((__v16sf)(__m512)(A), \1317 (int)(imm), (__mmask16)(U)))1318 1319#define _mm512_fpclass_ps_mask(A, imm) \1320 ((__mmask16)__builtin_ia32_fpclassps512_mask((__v16sf)(__m512)(A), \1321 (int)(imm), (__mmask16)-1))1322 1323#define _mm512_mask_fpclass_pd_mask(U, A, imm) \1324 ((__mmask8)__builtin_ia32_fpclasspd512_mask((__v8df)(__m512d)(A), (int)(imm), \1325 (__mmask8)(U)))1326 1327#define _mm512_fpclass_pd_mask(A, imm) \1328 ((__mmask8)__builtin_ia32_fpclasspd512_mask((__v8df)(__m512d)(A), (int)(imm), \1329 (__mmask8)-1))1330 1331#define _mm_fpclass_sd_mask(A, imm) \1332 ((__mmask8)__builtin_ia32_fpclasssd_mask((__v2df)(__m128d)(A), (int)(imm), \1333 (__mmask8)-1))1334 1335#define _mm_mask_fpclass_sd_mask(U, A, imm) \1336 ((__mmask8)__builtin_ia32_fpclasssd_mask((__v2df)(__m128d)(A), (int)(imm), \1337 (__mmask8)(U)))1338 1339#define _mm_fpclass_ss_mask(A, imm) \1340 ((__mmask8)__builtin_ia32_fpclassss_mask((__v4sf)(__m128)(A), (int)(imm), \1341 (__mmask8)-1))1342 1343#define _mm_mask_fpclass_ss_mask(U, A, imm) \1344 ((__mmask8)__builtin_ia32_fpclassss_mask((__v4sf)(__m128)(A), (int)(imm), \1345 (__mmask8)(U)))1346 1347#undef __DEFAULT_FN_ATTRS5121348#undef __DEFAULT_FN_ATTRS1349#undef __DEFAULT_FN_ATTRS512_CONSTEXPR1350#undef __DEFAULT_FN_ATTRS_CONSTEXPR1351 1352#endif1353