485 lines · c
1/*===-------------- avxneconvertintrin.h - AVXNECONVERT --------------------===2 *3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4 * See https://llvm.org/LICENSE.txt for license information.5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6 *7 *===-----------------------------------------------------------------------===8 */9 10#ifndef __IMMINTRIN_H11#error \12 "Never use <avxneconvertintrin.h> directly; include <immintrin.h> instead."13#endif // __IMMINTRIN_H14 15#ifdef __SSE2__16 17#ifndef __AVXNECONVERTINTRIN_H18#define __AVXNECONVERTINTRIN_H19 20/* Define the default attributes for the functions in this file. */21#define __DEFAULT_FN_ATTRS128 \22 __attribute__((__always_inline__, __nodebug__, __target__("avxneconvert"), \23 __min_vector_width__(128)))24#define __DEFAULT_FN_ATTRS256 \25 __attribute__((__always_inline__, __nodebug__, __target__("avxneconvert"), \26 __min_vector_width__(256)))27 28/// Convert scalar BF16 (16-bit) floating-point element29/// stored at memory locations starting at location \a __A to a30/// single-precision (32-bit) floating-point, broadcast it to packed31/// single-precision (32-bit) floating-point elements, and store the results in32/// \a dst.33///34/// \headerfile <x86intrin.h>35///36/// \code37/// _mm_bcstnebf16_ps(const void *__A);38/// \endcode39///40/// This intrinsic corresponds to the \c VBCSTNEBF162PS instruction.41///42/// \param __A43/// A pointer to a 16-bit memory location. The address of the memory44/// location does not have to be aligned.45/// \returns46/// A 128-bit vector of [4 x float].47///48/// \code{.operation}49/// b := Convert_BF16_To_FP32(MEM[__A+15:__A])50/// FOR j := 0 to 351/// m := j*3252/// dst[m+31:m] := b53/// ENDFOR54/// dst[MAX:128] := 055/// \endcode56static __inline__ __m128 __DEFAULT_FN_ATTRS12857_mm_bcstnebf16_ps(const void *__A) {58 return (__m128)__builtin_ia32_vbcstnebf162ps128((const __bf16 *)__A);59}60 61/// Convert scalar BF16 (16-bit) floating-point element62/// stored at memory locations starting at location \a __A to a63/// single-precision (32-bit) floating-point, broadcast it to packed64/// single-precision (32-bit) floating-point elements, and store the results in65/// \a dst.66///67/// \headerfile <x86intrin.h>68///69/// \code70/// _mm256_bcstnebf16_ps(const void *__A);71/// \endcode72///73/// This intrinsic corresponds to the \c VBCSTNEBF162PS instruction.74///75/// \param __A76/// A pointer to a 16-bit memory location. The address of the memory77/// location does not have to be aligned.78/// \returns79/// A 256-bit vector of [8 x float].80///81/// \code{.operation}82/// b := Convert_BF16_To_FP32(MEM[__A+15:__A])83/// FOR j := 0 to 784/// m := j*3285/// dst[m+31:m] := b86/// ENDFOR87/// dst[MAX:256] := 088/// \endcode89static __inline__ __m256 __DEFAULT_FN_ATTRS25690_mm256_bcstnebf16_ps(const void *__A) {91 return (__m256)__builtin_ia32_vbcstnebf162ps256((const __bf16 *)__A);92}93 94/// Convert scalar half-precision (16-bit) floating-point element95/// stored at memory locations starting at location \a __A to a96/// single-precision (32-bit) floating-point, broadcast it to packed97/// single-precision (32-bit) floating-point elements, and store the results in98/// \a dst.99///100/// \headerfile <x86intrin.h>101///102/// \code103/// _mm_bcstnesh_ps(const void *__A);104/// \endcode105///106/// This intrinsic corresponds to the \c VBCSTNESH2PS instruction.107///108/// \param __A109/// A pointer to a 16-bit memory location. The address of the memory110/// location does not have to be aligned.111/// \returns112/// A 128-bit vector of [4 x float].113///114/// \code{.operation}115/// b := Convert_FP16_To_FP32(MEM[__A+15:__A])116/// FOR j := 0 to 3117/// m := j*32118/// dst[m+31:m] := b119/// ENDFOR120/// dst[MAX:128] := 0121/// \endcode122static __inline__ __m128 __DEFAULT_FN_ATTRS128123_mm_bcstnesh_ps(const void *__A) {124 return (__m128)__builtin_ia32_vbcstnesh2ps128((const _Float16 *)__A);125}126 127/// Convert scalar half-precision (16-bit) floating-point element128/// stored at memory locations starting at location \a __A to a129/// single-precision (32-bit) floating-point, broadcast it to packed130/// single-precision (32-bit) floating-point elements, and store the results in131/// \a dst.132///133/// \headerfile <x86intrin.h>134///135/// \code136/// _mm256_bcstnesh_ps(const void *__A);137/// \endcode138///139/// This intrinsic corresponds to the \c VBCSTNESH2PS instruction.140///141/// \param __A142/// A pointer to a 16-bit memory location. The address of the memory143/// location does not have to be aligned.144/// \returns145/// A 256-bit vector of [8 x float].146///147/// \code{.operation}148/// b := Convert_FP16_To_FP32(MEM[__A+15:__A])149/// FOR j := 0 to 7150/// m := j*32151/// dst[m+31:m] := b152/// ENDFOR153/// dst[MAX:256] := 0154/// \endcode155static __inline__ __m256 __DEFAULT_FN_ATTRS256156_mm256_bcstnesh_ps(const void *__A) {157 return (__m256)__builtin_ia32_vbcstnesh2ps256((const _Float16 *)__A);158}159 160/// Convert packed BF16 (16-bit) floating-point even-indexed elements161/// stored at memory locations starting at location \a __A to packed162/// single-precision (32-bit) floating-point elements, and store the results in163/// \a dst.164///165/// \headerfile <x86intrin.h>166///167/// \code168/// _mm_cvtneebf16_ps(const __m128bh *__A);169/// \endcode170///171/// This intrinsic corresponds to the \c VCVTNEEBF162PS instruction.172///173/// \param __A174/// A pointer to a 128-bit memory location containing 8 consecutive175/// BF16 (16-bit) floating-point values.176/// \returns177/// A 128-bit vector of [4 x float].178///179/// \code{.operation}180/// FOR j := 0 to 3181/// k := j*2182/// i := k*16183/// m := j*32184/// dst[m+31:m] := Convert_BF16_To_FP32(MEM[__A+i+15:__A+i])185/// ENDFOR186/// dst[MAX:128] := 0187/// \endcode188static __inline__ __m128 __DEFAULT_FN_ATTRS128189_mm_cvtneebf16_ps(const __m128bh *__A) {190 return (__m128)__builtin_ia32_vcvtneebf162ps128((const __v8bf *)__A);191}192 193/// Convert packed BF16 (16-bit) floating-point even-indexed elements194/// stored at memory locations starting at location \a __A to packed195/// single-precision (32-bit) floating-point elements, and store the results in196/// \a dst.197///198/// \headerfile <x86intrin.h>199///200/// \code201/// _mm256_cvtneebf16_ps(const __m256bh *__A);202/// \endcode203///204/// This intrinsic corresponds to the \c VCVTNEEBF162PS instruction.205///206/// \param __A207/// A pointer to a 256-bit memory location containing 16 consecutive208/// BF16 (16-bit) floating-point values.209/// \returns210/// A 256-bit vector of [8 x float].211///212/// \code{.operation}213/// FOR j := 0 to 7214/// k := j*2215/// i := k*16216/// m := j*32217/// dst[m+31:m] := Convert_BF16_To_FP32(MEM[__A+i+15:__A+i])218/// ENDFOR219/// dst[MAX:256] := 0220/// \endcode221static __inline__ __m256 __DEFAULT_FN_ATTRS256222_mm256_cvtneebf16_ps(const __m256bh *__A) {223 return (__m256)__builtin_ia32_vcvtneebf162ps256((const __v16bf *)__A);224}225 226/// Convert packed half-precision (16-bit) floating-point even-indexed elements227/// stored at memory locations starting at location \a __A to packed228/// single-precision (32-bit) floating-point elements, and store the results in229/// \a dst.230///231/// \headerfile <x86intrin.h>232///233/// \code234/// _mm_cvtneeph_ps(const __m128h *__A);235/// \endcode236///237/// This intrinsic corresponds to the \c VCVTNEEPH2PS instruction.238///239/// \param __A240/// A pointer to a 128-bit memory location containing 8 consecutive241/// half-precision (16-bit) floating-point values.242/// \returns243/// A 128-bit vector of [4 x float].244///245/// \code{.operation}246/// FOR j := 0 to 3247/// k := j*2248/// i := k*16249/// m := j*32250/// dst[m+31:m] := Convert_FP16_To_FP32(MEM[__A+i+15:__A+i])251/// ENDFOR252/// dst[MAX:128] := 0253/// \endcode254static __inline__ __m128 __DEFAULT_FN_ATTRS128255_mm_cvtneeph_ps(const __m128h *__A) {256 return (__m128)__builtin_ia32_vcvtneeph2ps128((const __v8hf *)__A);257}258 259/// Convert packed half-precision (16-bit) floating-point even-indexed elements260/// stored at memory locations starting at location \a __A to packed261/// single-precision (32-bit) floating-point elements, and store the results in262/// \a dst.263///264/// \headerfile <x86intrin.h>265///266/// \code267/// _mm256_cvtneeph_ps(const __m256h *__A);268/// \endcode269///270/// This intrinsic corresponds to the \c VCVTNEEPH2PS instruction.271///272/// \param __A273/// A pointer to a 256-bit memory location containing 16 consecutive274/// half-precision (16-bit) floating-point values.275/// \returns276/// A 256-bit vector of [8 x float].277///278/// \code{.operation}279/// FOR j := 0 to 7280/// k := j*2281/// i := k*16282/// m := j*32283/// dst[m+31:m] := Convert_FP16_To_FP32(MEM[__A+i+15:__A+i])284/// ENDFOR285/// dst[MAX:256] := 0286/// \endcode287static __inline__ __m256 __DEFAULT_FN_ATTRS256288_mm256_cvtneeph_ps(const __m256h *__A) {289 return (__m256)__builtin_ia32_vcvtneeph2ps256((const __v16hf *)__A);290}291 292/// Convert packed BF16 (16-bit) floating-point odd-indexed elements293/// stored at memory locations starting at location \a __A to packed294/// single-precision (32-bit) floating-point elements, and store the results in295/// \a dst.296///297/// \headerfile <x86intrin.h>298///299/// \code300/// _mm_cvtneobf16_ps(const __m128bh *__A);301/// \endcode302///303/// This intrinsic corresponds to the \c VCVTNEOBF162PS instruction.304///305/// \param __A306/// A pointer to a 128-bit memory location containing 8 consecutive307/// BF16 (16-bit) floating-point values.308/// \returns309/// A 128-bit vector of [4 x float].310///311/// \code{.operation}312/// FOR j := 0 to 3313/// k := j*2+1314/// i := k*16315/// m := j*32316/// dst[m+31:m] := Convert_BF16_To_FP32(MEM[__A+i+15:__A+i])317/// ENDFOR318/// dst[MAX:128] := 0319/// \endcode320static __inline__ __m128 __DEFAULT_FN_ATTRS128321_mm_cvtneobf16_ps(const __m128bh *__A) {322 return (__m128)__builtin_ia32_vcvtneobf162ps128((const __v8bf *)__A);323}324 325/// Convert packed BF16 (16-bit) floating-point odd-indexed elements326/// stored at memory locations starting at location \a __A to packed327/// single-precision (32-bit) floating-point elements, and store the results in328/// \a dst.329///330/// \headerfile <x86intrin.h>331///332/// \code333/// _mm256_cvtneobf16_ps(const __m256bh *__A);334/// \endcode335///336/// This intrinsic corresponds to the \c VCVTNEOBF162PS instruction.337///338/// \param __A339/// A pointer to a 256-bit memory location containing 16 consecutive340/// BF16 (16-bit) floating-point values.341/// \returns342/// A 256-bit vector of [8 x float].343///344/// \code{.operation}345/// FOR j := 0 to 7346/// k := j*2+1347/// i := k*16348/// m := j*32349/// dst[m+31:m] := Convert_BF16_To_FP32(MEM[__A+i+15:__A+i])350/// ENDFOR351/// dst[MAX:256] := 0352/// \endcode353static __inline__ __m256 __DEFAULT_FN_ATTRS256354_mm256_cvtneobf16_ps(const __m256bh *__A) {355 return (__m256)__builtin_ia32_vcvtneobf162ps256((const __v16bf *)__A);356}357 358/// Convert packed half-precision (16-bit) floating-point odd-indexed elements359/// stored at memory locations starting at location \a __A to packed360/// single-precision (32-bit) floating-point elements, and store the results in361/// \a dst.362///363/// \headerfile <x86intrin.h>364///365/// \code366/// _mm_cvtneoph_ps(const __m128h *__A);367/// \endcode368///369/// This intrinsic corresponds to the \c VCVTNEOPH2PS instruction.370///371/// \param __A372/// A pointer to a 128-bit memory location containing 8 consecutive373/// half-precision (16-bit) floating-point values.374/// \returns375/// A 128-bit vector of [4 x float].376///377/// \code{.operation}378/// FOR j := 0 to 3379/// k := j*2+1380/// i := k*16381/// m := j*32382/// dst[m+31:m] := Convert_FP16_To_FP32(MEM[__A+i+15:__A+i])383/// ENDFOR384/// dst[MAX:128] := 0385/// \endcode386static __inline__ __m128 __DEFAULT_FN_ATTRS128387_mm_cvtneoph_ps(const __m128h *__A) {388 return (__m128)__builtin_ia32_vcvtneoph2ps128((const __v8hf *)__A);389}390 391/// Convert packed half-precision (16-bit) floating-point odd-indexed elements392/// stored at memory locations starting at location \a __A to packed393/// single-precision (32-bit) floating-point elements, and store the results in394/// \a dst.395///396/// \headerfile <x86intrin.h>397///398/// \code399/// _mm256_cvtneoph_ps(const __m256h *__A);400/// \endcode401///402/// This intrinsic corresponds to the \c VCVTNEOPH2PS instruction.403///404/// \param __A405/// A pointer to a 256-bit memory location containing 16 consecutive406/// half-precision (16-bit) floating-point values.407/// \returns408/// A 256-bit vector of [8 x float].409///410/// \code{.operation}411/// FOR j := 0 to 7412/// k := j*2+1413/// i := k*16414/// m := j*32415/// dst[m+31:m] := Convert_FP16_To_FP32(MEM[__A+i+15:__A+i])416/// ENDFOR417/// dst[MAX:256] := 0418/// \endcode419static __inline__ __m256 __DEFAULT_FN_ATTRS256420_mm256_cvtneoph_ps(const __m256h *__A) {421 return (__m256)__builtin_ia32_vcvtneoph2ps256((const __v16hf *)__A);422}423 424/// Convert packed single-precision (32-bit) floating-point elements in \a __A425/// to packed BF16 (16-bit) floating-point elements, and store the results in \a426/// dst.427///428/// \headerfile <x86intrin.h>429///430/// \code431/// _mm_cvtneps_avx_pbh(__m128 __A);432/// \endcode433///434/// This intrinsic corresponds to the \c VCVTNEPS2BF16 instruction.435///436/// \param __A437/// A 128-bit vector of [4 x float].438/// \returns439/// A 128-bit vector of [8 x bfloat].440///441/// \code{.operation}442/// FOR j := 0 to 3443/// dst.word[j] := Convert_FP32_To_BF16(__A.fp32[j])444/// ENDFOR445/// dst[MAX:128] := 0446/// \endcode447static __inline__ __m128bh __DEFAULT_FN_ATTRS128448_mm_cvtneps_avx_pbh(__m128 __A) {449 return (__m128bh)__builtin_ia32_vcvtneps2bf16128((__v4sf)__A);450}451 452/// Convert packed single-precision (32-bit) floating-point elements in \a __A453/// to packed BF16 (16-bit) floating-point elements, and store the results in \a454/// dst.455///456/// \headerfile <x86intrin.h>457///458/// \code459/// _mm256_cvtneps_avx_pbh(__m256 __A);460/// \endcode461///462/// This intrinsic corresponds to the \c VCVTNEPS2BF16 instruction.463///464/// \param __A465/// A 256-bit vector of [8 x float].466/// \returns467/// A 128-bit vector of [8 x bfloat].468///469/// \code{.operation}470/// FOR j := 0 to 7471/// dst.word[j] := Convert_FP32_To_BF16(a.fp32[j])472/// ENDFOR473/// dst[MAX:128] := 0474/// \endcode475static __inline__ __m128bh __DEFAULT_FN_ATTRS256476_mm256_cvtneps_avx_pbh(__m256 __A) {477 return (__m128bh)__builtin_ia32_vcvtneps2bf16256((__v8sf)__A);478}479 480#undef __DEFAULT_FN_ATTRS128481#undef __DEFAULT_FN_ATTRS256482 483#endif // __AVXNECONVERTINTRIN_H484#endif // __SSE2__485