230 lines · c
1/*===--------------- avxvnniintrin.h - VNNI intrinsics --------------------===2 *3 *4 * Permission is hereby granted, free of charge, to any person obtaining a copy5 * of this software and associated documentation files (the "Software"), to deal6 * in the Software without restriction, including without limitation the rights7 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell8 * copies of the Software, and to permit persons to whom the Software is9 * furnished to do so, subject to the following conditions:10 *11 * The above copyright notice and this permission notice shall be included in12 * all copies or substantial portions of the Software.13 *14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE17 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER18 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,19 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN20 * THE SOFTWARE.21 *22 *===-----------------------------------------------------------------------===23 */24#ifndef __IMMINTRIN_H25#error "Never use <avxvnniintrin.h> directly; include <immintrin.h> instead."26#endif27 28#ifndef __AVXVNNIINTRIN_H29#define __AVXVNNIINTRIN_H30 31/* Below intrinsics defined in avx512vlvnniintrin.h can be used for AVXVNNI */32/// \fn __m256i _mm256_dpbusd_epi32(__m256i __S, __m256i __A, __m256i __B)33/// \fn __m256i _mm256_dpbusds_epi32(__m256i __S, __m256i __A, __m256i __B)34/// \fn __m256i _mm256_dpwssd_epi32(__m256i __S, __m256i __A, __m256i __B)35/// \fn __m256i _mm256_dpwssds_epi32(__m256i __S, __m256i __A, __m256i __B)36/// \fn __m128i _mm_dpbusd_epi32(__m128i __S, __m128i __A, __m128i __B)37/// \fn __m128i _mm_dpbusds_epi32(__m128i __S, __m128i __A, __m128i __B)38/// \fn __m128i _mm_dpwssd_epi32(__m128i __S, __m128i __A, __m128i __B)39/// \fn __m128i _mm_dpwssds_epi32(__m128i __S, __m128i __A, __m128i __B)40 41/* Intrinsics with _avx_ prefix are for compatibility with msvc. */42/* Define the default attributes for the functions in this file. */43#define __DEFAULT_FN_ATTRS256 __attribute__((__always_inline__, __nodebug__, __target__("avxvnni"), __min_vector_width__(256)))44#define __DEFAULT_FN_ATTRS128 __attribute__((__always_inline__, __nodebug__, __target__("avxvnni"), __min_vector_width__(128)))45 46/// Multiply groups of 4 adjacent pairs of unsigned 8-bit integers in \a __A with47/// corresponding signed 8-bit integers in \a __B, producing 4 intermediate signed48/// 16-bit results. Sum these 4 results with the corresponding 32-bit integer49/// in \a __S, and store the packed 32-bit results in DST.50///51/// This intrinsic corresponds to the <c> VPDPBUSD </c> instructions.52///53/// \code{.operation}54/// FOR j := 0 to 755/// tmp1.word := Signed(ZeroExtend16(__A.byte[4*j]) * SignExtend16(__B.byte[4*j]))56/// tmp2.word := Signed(ZeroExtend16(__A.byte[4*j+1]) * SignExtend16(__B.byte[4*j+1]))57/// tmp3.word := Signed(ZeroExtend16(__A.byte[4*j+2]) * SignExtend16(__B.byte[4*j+2]))58/// tmp4.word := Signed(ZeroExtend16(__A.byte[4*j+3]) * SignExtend16(__B.byte[4*j+3]))59/// DST.dword[j] := __S.dword[j] + tmp1 + tmp2 + tmp3 + tmp460/// ENDFOR61/// DST[MAX:256] := 062/// \endcode63static __inline__ __m256i __DEFAULT_FN_ATTRS25664_mm256_dpbusd_avx_epi32(__m256i __S, __m256i __A, __m256i __B)65{66 return (__m256i)__builtin_ia32_vpdpbusd256((__v8si)__S, (__v32qu)__A,67 (__v32qi)__B);68}69 70/// Multiply groups of 4 adjacent pairs of unsigned 8-bit integers in \a __A with71/// corresponding signed 8-bit integers in \a __B, producing 4 intermediate signed72/// 16-bit results. Sum these 4 results with the corresponding 32-bit integer73/// in \a __S using signed saturation, and store the packed 32-bit results in DST.74///75/// This intrinsic corresponds to the <c> VPDPBUSDS </c> instructions.76///77/// \code{.operation}78/// FOR j := 0 to 779/// tmp1.word := Signed(ZeroExtend16(__A.byte[4*j]) * SignExtend16(__B.byte[4*j]))80/// tmp2.word := Signed(ZeroExtend16(__A.byte[4*j+1]) * SignExtend16(__B.byte[4*j+1]))81/// tmp3.word := Signed(ZeroExtend16(__A.byte[4*j+2]) * SignExtend16(__B.byte[4*j+2]))82/// tmp4.word := Signed(ZeroExtend16(__A.byte[4*j+3]) * SignExtend16(__B.byte[4*j+3]))83/// DST.dword[j] := Saturate32(__S.dword[j] + tmp1 + tmp2 + tmp3 + tmp4)84/// ENDFOR85/// DST[MAX:256] := 086/// \endcode87static __inline__ __m256i __DEFAULT_FN_ATTRS25688_mm256_dpbusds_avx_epi32(__m256i __S, __m256i __A, __m256i __B)89{90 return (__m256i)__builtin_ia32_vpdpbusds256((__v8si)__S, (__v32qu)__A,91 (__v32qi)__B);92}93 94/// Multiply groups of 2 adjacent pairs of signed 16-bit integers in \a __A with95/// corresponding 16-bit integers in \a __B, producing 2 intermediate signed 32-bit96/// results. Sum these 2 results with the corresponding 32-bit integer in \a __S,97/// and store the packed 32-bit results in DST.98///99/// This intrinsic corresponds to the <c> VPDPWSSD </c> instructions.100///101/// \code{.operation}102/// FOR j := 0 to 7103/// tmp1.dword := SignExtend32(__A.word[2*j]) * SignExtend32(__B.word[2*j])104/// tmp2.dword := SignExtend32(__A.word[2*j+1]) * SignExtend32(__B.word[2*j+1])105/// DST.dword[j] := __S.dword[j] + tmp1 + tmp2106/// ENDFOR107/// DST[MAX:256] := 0108/// \endcode109static __inline__ __m256i __DEFAULT_FN_ATTRS256110_mm256_dpwssd_avx_epi32(__m256i __S, __m256i __A, __m256i __B)111{112 return (__m256i)__builtin_ia32_vpdpwssd256((__v8si)__S, (__v8si)__A, (__v8si)__B);113}114 115/// Multiply groups of 2 adjacent pairs of signed 16-bit integers in \a __A with116/// corresponding 16-bit integers in \a __B, producing 2 intermediate signed 32-bit117/// results. Sum these 2 results with the corresponding 32-bit integer in \a __S118/// using signed saturation, and store the packed 32-bit results in DST.119///120/// This intrinsic corresponds to the <c> VPDPWSSDS </c> instructions.121///122/// \code{.operation}123/// FOR j := 0 to 7124/// tmp1.dword := SignExtend32(__A.word[2*j]) * SignExtend32(__B.word[2*j])125/// tmp2.dword := SignExtend32(__A.word[2*j+1]) * SignExtend32(__B.word[2*j+1])126/// DST.dword[j] := Saturate32(__S.dword[j] + tmp1 + tmp2)127/// ENDFOR128/// DST[MAX:256] := 0129/// \endcode130static __inline__ __m256i __DEFAULT_FN_ATTRS256131_mm256_dpwssds_avx_epi32(__m256i __S, __m256i __A, __m256i __B)132{133 return (__m256i)__builtin_ia32_vpdpwssds256((__v8si)__S, (__v8si)__A, (__v8si)__B);134}135 136/// Multiply groups of 4 adjacent pairs of unsigned 8-bit integers in \a __A with137/// corresponding signed 8-bit integers in \a __B, producing 4 intermediate signed138/// 16-bit results. Sum these 4 results with the corresponding 32-bit integer139/// in \a __S, and store the packed 32-bit results in DST.140///141/// This intrinsic corresponds to the <c> VPDPBUSD </c> instructions.142///143/// \code{.operation}144/// FOR j := 0 to 3145/// tmp1.word := Signed(ZeroExtend16(__A.byte[4*j]) * SignExtend16(__B.byte[4*j]))146/// tmp2.word := Signed(ZeroExtend16(__A.byte[4*j+1]) * SignExtend16(__B.byte[4*j+1]))147/// tmp3.word := Signed(ZeroExtend16(__A.byte[4*j+2]) * SignExtend16(__B.byte[4*j+2]))148/// tmp4.word := Signed(ZeroExtend16(__A.byte[4*j+3]) * SignExtend16(__B.byte[4*j+3]))149/// DST.dword[j] := __S.dword[j] + tmp1 + tmp2 + tmp3 + tmp4150/// ENDFOR151/// DST[MAX:128] := 0152/// \endcode153static __inline__ __m128i __DEFAULT_FN_ATTRS128154_mm_dpbusd_avx_epi32(__m128i __S, __m128i __A, __m128i __B)155{156 return (__m128i)__builtin_ia32_vpdpbusd128((__v4si)__S, (__v16qu)__A,157 (__v16qi)__B);158}159 160/// Multiply groups of 4 adjacent pairs of unsigned 8-bit integers in \a __A with161/// corresponding signed 8-bit integers in \a __B, producing 4 intermediate signed162/// 16-bit results. Sum these 4 results with the corresponding 32-bit integer163/// in \a __S using signed saturation, and store the packed 32-bit results in DST.164///165/// This intrinsic corresponds to the <c> VPDPBUSDS </c> instructions.166///167/// \code{.operation}168/// FOR j := 0 to 3169/// tmp1.word := Signed(ZeroExtend16(__A.byte[4*j]) * SignExtend16(__B.byte[4*j]))170/// tmp2.word := Signed(ZeroExtend16(__A.byte[4*j+1]) * SignExtend16(__B.byte[4*j+1]))171/// tmp3.word := Signed(ZeroExtend16(__A.byte[4*j+2]) * SignExtend16(__B.byte[4*j+2]))172/// tmp4.word := Signed(ZeroExtend16(__A.byte[4*j+3]) * SignExtend16(__B.byte[4*j+3]))173/// DST.dword[j] := Saturate32(__S.dword[j] + tmp1 + tmp2 + tmp3 + tmp4)174/// ENDFOR175/// DST[MAX:128] := 0176/// \endcode177static __inline__ __m128i __DEFAULT_FN_ATTRS128178_mm_dpbusds_avx_epi32(__m128i __S, __m128i __A, __m128i __B)179{180 return (__m128i)__builtin_ia32_vpdpbusds128((__v4si)__S, (__v16qu)__A,181 (__v16qi)__B);182}183 184/// Multiply groups of 2 adjacent pairs of signed 16-bit integers in \a __A with185/// corresponding 16-bit integers in \a __B, producing 2 intermediate signed 32-bit186/// results. Sum these 2 results with the corresponding 32-bit integer in \a __S,187/// and store the packed 32-bit results in DST.188///189/// This intrinsic corresponds to the <c> VPDPWSSD </c> instructions.190///191/// \code{.operation}192/// FOR j := 0 to 3193/// tmp1.dword := SignExtend32(__A.word[2*j]) * SignExtend32(__B.word[2*j])194/// tmp2.dword := SignExtend32(__A.word[2*j+1]) * SignExtend32(__B.word[2*j+1])195/// DST.dword[j] := __S.dword[j] + tmp1 + tmp2196/// ENDFOR197/// DST[MAX:128] := 0198/// \endcode199static __inline__ __m128i __DEFAULT_FN_ATTRS128200_mm_dpwssd_avx_epi32(__m128i __S, __m128i __A, __m128i __B)201{202 return (__m128i)__builtin_ia32_vpdpwssd128((__v4si)__S, (__v4si)__A, (__v4si)__B);203}204 205/// Multiply groups of 2 adjacent pairs of signed 16-bit integers in \a __A with206/// corresponding 16-bit integers in \a __B, producing 2 intermediate signed 32-bit207/// results. Sum these 2 results with the corresponding 32-bit integer in \a __S208/// using signed saturation, and store the packed 32-bit results in DST.209///210/// This intrinsic corresponds to the <c> VPDPWSSDS </c> instructions.211///212/// \code{.operation}213/// FOR j := 0 to 3214/// tmp1.dword := SignExtend32(__A.word[2*j]) * SignExtend32(__B.word[2*j])215/// tmp2.dword := SignExtend32(__A.word[2*j+1]) * SignExtend32(__B.word[2*j+1])216/// DST.dword[j] := Saturate32(__S.dword[j] + tmp1 + tmp2)217/// ENDFOR218/// DST[MAX:128] := 0219/// \endcode220static __inline__ __m128i __DEFAULT_FN_ATTRS128221_mm_dpwssds_avx_epi32(__m128i __S, __m128i __A, __m128i __B)222{223 return (__m128i)__builtin_ia32_vpdpwssds128((__v4si)__S, (__v4si)__A, (__v4si)__B);224}225 226#undef __DEFAULT_FN_ATTRS128227#undef __DEFAULT_FN_ATTRS256228 229#endif // __AVXVNNIINTRIN_H230