293 lines · c
1/*===------------ avx512bf16intrin.h - AVX512_BF16 intrinsics --------------===2 *3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4 * See https://llvm.org/LICENSE.txt for license information.5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6 *7 *===-----------------------------------------------------------------------===8 */9#ifndef __IMMINTRIN_H10#error "Never use <avx512bf16intrin.h> directly; include <immintrin.h> instead."11#endif12 13#ifdef __SSE2__14 15#ifndef __AVX512BF16INTRIN_H16#define __AVX512BF16INTRIN_H17 18typedef __bf16 __v32bf __attribute__((__vector_size__(64), __aligned__(64)));19typedef __bf16 __m512bh __attribute__((__vector_size__(64), __aligned__(64)));20typedef __bf16 __bfloat16 __attribute__((deprecated("use __bf16 instead")));21 22#define __DEFAULT_FN_ATTRS512 \23 __attribute__((__always_inline__, __nodebug__, __target__("avx512bf16"), \24 __min_vector_width__(512)))25#define __DEFAULT_FN_ATTRS \26 __attribute__((__always_inline__, __nodebug__, __target__("avx512bf16")))27 28#if defined(__cplusplus) && (__cplusplus >= 201103L)29#define __DEFAULT_FN_ATTRS512_CONSTEXPR __DEFAULT_FN_ATTRS512 constexpr30#define __DEFAULT_FN_ATTRS_CONSTEXPR __DEFAULT_FN_ATTRS constexpr31#else32#define __DEFAULT_FN_ATTRS512_CONSTEXPR __DEFAULT_FN_ATTRS51233#define __DEFAULT_FN_ATTRS_CONSTEXPR __DEFAULT_FN_ATTRS34#endif35 36/// Convert One BF16 Data to One Single Float Data.37///38/// \headerfile <x86intrin.h>39///40/// This intrinsic does not correspond to a specific instruction.41///42/// \param __A43/// A bfloat data.44/// \returns A float data whose sign field and exponent field keep unchanged,45/// and fraction field is extended to 23 bits.46static __inline__ float __DEFAULT_FN_ATTRS_CONSTEXPR _mm_cvtsbh_ss(__bf16 __A) {47 return (float)(__A);48}49 50/// Convert Two Packed Single Data to One Packed BF16 Data.51///52/// \headerfile <x86intrin.h>53///54/// This intrinsic corresponds to the <c> VCVTNE2PS2BF16 </c> instructions.55///56/// \param __A57/// A 512-bit vector of [16 x float].58/// \param __B59/// A 512-bit vector of [16 x float].60/// \returns A 512-bit vector of [32 x bfloat] whose lower 256 bits come from61/// conversion of __B, and higher 256 bits come from conversion of __A.62static __inline__ __m512bh __DEFAULT_FN_ATTRS51263_mm512_cvtne2ps_pbh(__m512 __A, __m512 __B) {64 return (__m512bh)__builtin_ia32_cvtne2ps2bf16_512((__v16sf) __A,65 (__v16sf) __B);66}67 68/// Convert Two Packed Single Data to One Packed BF16 Data.69///70/// \headerfile <x86intrin.h>71///72/// This intrinsic corresponds to the <c> VCVTNE2PS2BF16 </c> instructions.73///74/// \param __A75/// A 512-bit vector of [16 x float].76/// \param __B77/// A 512-bit vector of [16 x float].78/// \param __W79/// A 512-bit vector of [32 x bfloat].80/// \param __U81/// A 32-bit mask value specifying what is chosen for each element.82/// A 1 means conversion of __A or __B. A 0 means element from __W.83/// \returns A 512-bit vector of [32 x bfloat] whose lower 256 bits come from84/// conversion of __B, and higher 256 bits come from conversion of __A.85static __inline__ __m512bh __DEFAULT_FN_ATTRS51286_mm512_mask_cvtne2ps_pbh(__m512bh __W, __mmask32 __U, __m512 __A, __m512 __B) {87 return (__m512bh)__builtin_ia32_selectpbf_512((__mmask32)__U,88 (__v32bf)_mm512_cvtne2ps_pbh(__A, __B),89 (__v32bf)__W);90}91 92/// Convert Two Packed Single Data to One Packed BF16 Data.93///94/// \headerfile <x86intrin.h>95///96/// This intrinsic corresponds to the <c> VCVTNE2PS2BF16 </c> instructions.97///98/// \param __A99/// A 512-bit vector of [16 x float].100/// \param __B101/// A 512-bit vector of [16 x float].102/// \param __U103/// A 32-bit mask value specifying what is chosen for each element.104/// A 1 means conversion of __A or __B. A 0 means element is zero.105/// \returns A 512-bit vector of [32 x bfloat] whose lower 256 bits come from106/// conversion of __B, and higher 256 bits come from conversion of __A.107static __inline__ __m512bh __DEFAULT_FN_ATTRS512108_mm512_maskz_cvtne2ps_pbh(__mmask32 __U, __m512 __A, __m512 __B) {109 return (__m512bh)__builtin_ia32_selectpbf_512((__mmask32)__U,110 (__v32bf)_mm512_cvtne2ps_pbh(__A, __B),111 (__v32bf)_mm512_setzero_si512());112}113 114/// Convert Packed Single Data to Packed BF16 Data.115///116/// \headerfile <x86intrin.h>117///118/// This intrinsic corresponds to the <c> VCVTNEPS2BF16 </c> instructions.119///120/// \param __A121/// A 512-bit vector of [16 x float].122/// \returns A 256-bit vector of [16 x bfloat] come from conversion of __A.123static __inline__ __m256bh __DEFAULT_FN_ATTRS512124_mm512_cvtneps_pbh(__m512 __A) {125 return (__m256bh)__builtin_ia32_cvtneps2bf16_512_mask((__v16sf)__A,126 (__v16bf)_mm256_undefined_si256(),127 (__mmask16)-1);128}129 130/// Convert Packed Single Data to Packed BF16 Data.131///132/// \headerfile <x86intrin.h>133///134/// This intrinsic corresponds to the <c> VCVTNEPS2BF16 </c> instructions.135///136/// \param __A137/// A 512-bit vector of [16 x float].138/// \param __W139/// A 256-bit vector of [16 x bfloat].140/// \param __U141/// A 16-bit mask value specifying what is chosen for each element.142/// A 1 means conversion of __A. A 0 means element from __W.143/// \returns A 256-bit vector of [16 x bfloat] come from conversion of __A.144static __inline__ __m256bh __DEFAULT_FN_ATTRS512145_mm512_mask_cvtneps_pbh(__m256bh __W, __mmask16 __U, __m512 __A) {146 return (__m256bh)__builtin_ia32_cvtneps2bf16_512_mask((__v16sf)__A,147 (__v16bf)__W,148 (__mmask16)__U);149}150 151/// Convert Packed Single Data to Packed BF16 Data.152///153/// \headerfile <x86intrin.h>154///155/// This intrinsic corresponds to the <c> VCVTNEPS2BF16 </c> instructions.156///157/// \param __A158/// A 512-bit vector of [16 x float].159/// \param __U160/// A 16-bit mask value specifying what is chosen for each element.161/// A 1 means conversion of __A. A 0 means element is zero.162/// \returns A 256-bit vector of [16 x bfloat] come from conversion of __A.163static __inline__ __m256bh __DEFAULT_FN_ATTRS512164_mm512_maskz_cvtneps_pbh(__mmask16 __U, __m512 __A) {165 return (__m256bh)__builtin_ia32_cvtneps2bf16_512_mask((__v16sf)__A,166 (__v16bf)_mm256_setzero_si256(),167 (__mmask16)__U);168}169 170/// Dot Product of BF16 Pairs Accumulated into Packed Single Precision.171///172/// \headerfile <x86intrin.h>173///174/// This intrinsic corresponds to the <c> VDPBF16PS </c> instructions.175///176/// \param __A177/// A 512-bit vector of [32 x bfloat].178/// \param __B179/// A 512-bit vector of [32 x bfloat].180/// \param __D181/// A 512-bit vector of [16 x float].182/// \returns A 512-bit vector of [16 x float] comes from Dot Product of183/// __A, __B and __D184static __inline__ __m512 __DEFAULT_FN_ATTRS512185_mm512_dpbf16_ps(__m512 __D, __m512bh __A, __m512bh __B) {186 return (__m512)__builtin_ia32_dpbf16ps_512((__v16sf) __D,187 (__v32bf) __A,188 (__v32bf) __B);189}190 191/// Dot Product of BF16 Pairs Accumulated into Packed Single Precision.192///193/// \headerfile <x86intrin.h>194///195/// This intrinsic corresponds to the <c> VDPBF16PS </c> instructions.196///197/// \param __A198/// A 512-bit vector of [32 x bfloat].199/// \param __B200/// A 512-bit vector of [32 x bfloat].201/// \param __D202/// A 512-bit vector of [16 x float].203/// \param __U204/// A 16-bit mask value specifying what is chosen for each element.205/// A 1 means __A and __B's dot product accumulated with __D. A 0 means __D.206/// \returns A 512-bit vector of [16 x float] comes from Dot Product of207/// __A, __B and __D208static __inline__ __m512 __DEFAULT_FN_ATTRS512209_mm512_mask_dpbf16_ps(__m512 __D, __mmask16 __U, __m512bh __A, __m512bh __B) {210 return (__m512)__builtin_ia32_selectps_512((__mmask16)__U,211 (__v16sf)_mm512_dpbf16_ps(__D, __A, __B),212 (__v16sf)__D);213}214 215/// Dot Product of BF16 Pairs Accumulated into Packed Single Precision.216///217/// \headerfile <x86intrin.h>218///219/// This intrinsic corresponds to the <c> VDPBF16PS </c> instructions.220///221/// \param __A222/// A 512-bit vector of [32 x bfloat].223/// \param __B224/// A 512-bit vector of [32 x bfloat].225/// \param __D226/// A 512-bit vector of [16 x float].227/// \param __U228/// A 16-bit mask value specifying what is chosen for each element.229/// A 1 means __A and __B's dot product accumulated with __D. A 0 means 0.230/// \returns A 512-bit vector of [16 x float] comes from Dot Product of231/// __A, __B and __D232static __inline__ __m512 __DEFAULT_FN_ATTRS512233_mm512_maskz_dpbf16_ps(__mmask16 __U, __m512 __D, __m512bh __A, __m512bh __B) {234 return (__m512)__builtin_ia32_selectps_512((__mmask16)__U,235 (__v16sf)_mm512_dpbf16_ps(__D, __A, __B),236 (__v16sf)_mm512_setzero_si512());237}238 239/// Convert Packed BF16 Data to Packed float Data.240///241/// \headerfile <x86intrin.h>242///243/// \param __A244/// A 256-bit vector of [16 x bfloat].245/// \returns A 512-bit vector of [16 x float] come from conversion of __A246static __inline__ __m512 __DEFAULT_FN_ATTRS512_CONSTEXPR247_mm512_cvtpbh_ps(__m256bh __A) {248 return (__m512) __builtin_convertvector(__A, __v16sf);249}250 251/// Convert Packed BF16 Data to Packed float Data using zeroing mask.252///253/// \headerfile <x86intrin.h>254///255/// \param __U256/// A 16-bit mask. Elements are zeroed out when the corresponding mask257/// bit is not set.258/// \param __A259/// A 256-bit vector of [16 x bfloat].260/// \returns A 512-bit vector of [16 x float] come from conversion of __A261static __inline__ __m512 __DEFAULT_FN_ATTRS512_CONSTEXPR262_mm512_maskz_cvtpbh_ps(__mmask16 __U, __m256bh __A) {263 return (__m512)__builtin_ia32_selectps_512((__mmask16)__U,264 (__v16sf)_mm512_cvtpbh_ps(__A),265 (__v16sf)_mm512_setzero_ps());266}267 268/// Convert Packed BF16 Data to Packed float Data using merging mask.269///270/// \headerfile <x86intrin.h>271///272/// \param __S273/// A 512-bit vector of [16 x float]. Elements are copied from __S when274/// the corresponding mask bit is not set.275/// \param __U276/// A 16-bit mask.277/// \param __A278/// A 256-bit vector of [16 x bfloat].279/// \returns A 512-bit vector of [16 x float] come from conversion of __A280static __inline__ __m512 __DEFAULT_FN_ATTRS512_CONSTEXPR281_mm512_mask_cvtpbh_ps(__m512 __S, __mmask16 __U, __m256bh __A) {282 return (__m512)__builtin_ia32_selectps_512(283 (__mmask16)__U, (__v16sf)_mm512_cvtpbh_ps(__A), (__v16sf)__S);284}285 286#undef __DEFAULT_FN_ATTRS287#undef __DEFAULT_FN_ATTRS_CONSTEXPR288#undef __DEFAULT_FN_ATTRS512289#undef __DEFAULT_FN_ATTRS512_CONSTEXPR290 291#endif292#endif293