433 lines · c
1/*===----------- avxvnniint16intrin.h - AVXVNNIINT16 intrinsics-------------===2 *3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4 * See https://llvm.org/LICENSE.txt for license information.5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6 *7 *===-----------------------------------------------------------------------===8 */9 10#ifndef __IMMINTRIN_H11#error \12 "Never use <avxvnniint16intrin.h> directly; include <immintrin.h> instead."13#endif // __IMMINTRIN_H14 15#ifndef __AVXVNNIINT16INTRIN_H16#define __AVXVNNIINT16INTRIN_H17 18/// Multiply groups of 2 adjacent pairs of signed 16-bit integers in \a __A with19/// corresponding unsigned 16-bit integers in \a __B, producing 2 intermediate20/// signed 16-bit results. Sum these 2 results with the corresponding21/// 32-bit integer in \a __W, and store the packed 32-bit results in \a dst.22///23/// \headerfile <immintrin.h>24///25/// \code26/// __m128i _mm_dpwsud_epi32(__m128i __W, __m128i __A, __m128i __B)27/// \endcode28///29/// This intrinsic corresponds to the \c VPDPWSUD instruction.30///31/// \param __W32/// A 128-bit vector of [4 x int].33/// \param __A34/// A 128-bit vector of [8 x short].35/// \param __B36/// A 128-bit vector of [8 x unsigned short].37/// \returns38/// A 128-bit vector of [4 x int].39///40/// \code{.operation}41/// FOR j := 0 to 342/// tmp1.dword := SignExtend32(__A.word[2*j]) * ZeroExtend32(__B.word[2*j])43/// tmp2.dword := SignExtend32(__A.word[2*j+1]) * ZeroExtend32(__B.word[2*j+1])44/// dst.dword[j] := __W.dword[j] + tmp1 + tmp245/// ENDFOR46/// dst[MAX:128] := 047/// \endcode48#define _mm_dpwsud_epi32(__W, __A, __B) \49 ((__m128i)__builtin_ia32_vpdpwsud128((__v4si)(__W), (__v4si)(__A), \50 (__v4si)(__B)))51 52/// Multiply groups of 2 adjacent pairs of signed 16-bit integers in \a __A with53/// corresponding unsigned 16-bit integers in \a __B, producing 2 intermediate54/// signed 16-bit results. Sum these 2 results with the corresponding55/// 32-bit integer in \a __W, and store the packed 32-bit results in \a dst.56///57/// \headerfile <immintrin.h>58///59/// \code60/// __m256i _mm256_dpwsud_epi32(__m256i __W, __m256i __A, __m256i __B)61/// \endcode62///63/// This intrinsic corresponds to the \c VPDPWSUD instruction.64///65/// \param __W66/// A 256-bit vector of [8 x int].67/// \param __A68/// A 256-bit vector of [16 x short].69/// \param __B70/// A 256-bit vector of [16 x unsigned short].71/// \returns72/// A 256-bit vector of [8 x int].73///74/// \code{.operation}75/// FOR j := 0 to 776/// tmp1.dword := SignExtend32(__A.word[2*j]) * ZeroExtend32(__B.word[2*j])77/// tmp2.dword := SignExtend32(__A.word[2*j+1]) * ZeroExtend32(__B.word[2*j+1])78/// dst.dword[j] := __W.dword[j] + tmp1 + tmp279/// ENDFOR80/// dst[MAX:256] := 081/// \endcode82#define _mm256_dpwsud_epi32(__W, __A, __B) \83 ((__m256i)__builtin_ia32_vpdpwsud256((__v8si)(__W), (__v8si)(__A), \84 (__v8si)(__B)))85 86/// Multiply groups of 2 adjacent pairs of signed 16-bit integers in \a __A with87/// corresponding unsigned 16-bit integers in \a __B, producing 2 intermediate88/// signed 16-bit results. Sum these 2 results with the corresponding89/// 32-bit integer in \a __W with signed saturation, and store the packed90/// 32-bit results in \a dst.91///92/// \headerfile <immintrin.h>93///94/// \code95/// __m128i _mm_dpwsuds_epi32(__m128i __W, __m128i __A, __m128i __B)96/// \endcode97///98/// This intrinsic corresponds to the \c VPDPWSUDS instruction.99///100/// \param __W101/// A 128-bit vector of [4 x int].102/// \param __A103/// A 128-bit vector of [8 x short].104/// \param __B105/// A 128-bit vector of [8 x unsigned short].106/// \returns107/// A 128-bit vector of [4 x int].108///109/// \code{.operation}110/// FOR j := 0 to 3111/// tmp1.dword := SignExtend32(__A.word[2*j]) * ZeroExtend32(__B.word[2*j])112/// tmp2.dword := SignExtend32(__A.word[2*j+1]) * ZeroExtend32(__B.word[2*j+1])113/// dst.dword[j] := SIGNED_DWORD_SATURATE(__W.dword[j] + tmp1 + tmp2)114/// ENDFOR115/// dst[MAX:128] := 0116/// \endcode117#define _mm_dpwsuds_epi32(__W, __A, __B) \118 ((__m128i)__builtin_ia32_vpdpwsuds128((__v4si)(__W), (__v4si)(__A), \119 (__v4si)(__B)))120 121/// Multiply groups of 2 adjacent pairs of signed 16-bit integers in \a __A with122/// corresponding unsigned 16-bit integers in \a __B, producing 2 intermediate123/// signed 16-bit results. Sum these 2 results with the corresponding124/// 32-bit integer in \a __W with signed saturation, and store the packed125/// 32-bit results in \a dst.126///127/// \headerfile <immintrin.h>128///129/// \code130/// __m256i _mm256_dpwsuds_epi32(__m256i __W, __m256i __A, __m256i __B)131/// \endcode132///133/// This intrinsic corresponds to the \c VPDPWSUDS instruction.134///135/// \param __W136/// A 256-bit vector of [8 x int].137/// \param __A138/// A 256-bit vector of [16 x short].139/// \param __B140/// A 256-bit vector of [16 x unsigned short].141/// \returns142/// A 256-bit vector of [8 x int].143///144/// \code{.operation}145/// FOR j := 0 to 7146/// tmp1.dword := SignExtend32(__A.word[2*j]) * ZeroExtend32(__B.word[2*j])147/// tmp2.dword := SignExtend32(__A.word[2*j+1]) * ZeroExtend32(__B.word[2*j+1])148/// dst.dword[j] := SIGNED_DWORD_SATURATE(__W.dword[j] + tmp1 + tmp2)149/// ENDFOR150/// dst[MAX:256] := 0151/// \endcode152#define _mm256_dpwsuds_epi32(__W, __A, __B) \153 ((__m256i)__builtin_ia32_vpdpwsuds256((__v8si)(__W), (__v8si)(__A), \154 (__v8si)(__B)))155 156/// Multiply groups of 2 adjacent pairs of unsigned 16-bit integers in \a __A with157/// corresponding signed 16-bit integers in \a __B, producing 2 intermediate158/// signed 16-bit results. Sum these 2 results with the corresponding159/// 32-bit integer in \a __W, and store the packed 32-bit results in \a dst.160///161/// \headerfile <immintrin.h>162///163/// \code164/// __m128i _mm_dpbusd_epi32(__m128i __W, __m128i __A, __m128i __B)165/// \endcode166///167/// This intrinsic corresponds to the \c VPDPWUSD instruction.168///169/// \param __W170/// A 128-bit vector of [4 x int].171/// \param __A172/// A 128-bit vector of [8 x unsigned short].173/// \param __B174/// A 128-bit vector of [8 x short].175/// \returns176/// A 128-bit vector of [4 x int].177///178/// \code{.operation}179/// FOR j := 0 to 3180/// tmp1.dword := ZeroExtend32(__A.word[2*j]) * SignExtend32(__B.word[2*j])181/// tmp2.dword := ZeroExtend32(__A.word[2*j+1]) * SignExtend32(__B.word[2*j+1])182/// dst.dword[j] := __W.dword[j] + tmp1 + tmp2183/// ENDFOR184/// dst[MAX:128] := 0185/// \endcode186#define _mm_dpwusd_epi32(__W, __A, __B) \187 ((__m128i)__builtin_ia32_vpdpwusd128((__v4si)(__W), (__v4si)(__A), \188 (__v4si)(__B)))189 190/// Multiply groups of 2 adjacent pairs of unsigned 16-bit integers in \a __A with191/// corresponding signed 16-bit integers in \a __B, producing 2 intermediate192/// signed 16-bit results. Sum these 2 results with the corresponding193/// 32-bit integer in \a __W, and store the packed 32-bit results in \a dst.194///195/// \headerfile <immintrin.h>196///197/// \code198/// __m256i _mm256_dpwusd_epi32(__m256i __W, __m256i __A, __m256i __B)199/// \endcode200///201/// This intrinsic corresponds to the \c VPDPWUSD instruction.202///203/// \param __W204/// A 256-bit vector of [8 x int].205/// \param __A206/// A 256-bit vector of [16 x unsigned short].207/// \param __B208/// A 256-bit vector of [16 x short].209/// \returns210/// A 256-bit vector of [8 x int].211///212/// \code{.operation}213/// FOR j := 0 to 7214/// tmp1.dword := ZeroExtend32(__A.word[2*j]) * SignExtend32(__B.word[2*j])215/// tmp2.dword := ZeroExtend32(__A.word[2*j+1]) * SignExtend32(__B.word[2*j+1])216/// dst.dword[j] := __W.dword[j] + tmp1 + tmp2217/// ENDFOR218/// dst[MAX:256] := 0219/// \endcode220#define _mm256_dpwusd_epi32(__W, __A, __B) \221 ((__m256i)__builtin_ia32_vpdpwusd256((__v8si)(__W), (__v8si)(__A), \222 (__v8si)(__B)))223 224/// Multiply groups of 2 adjacent pairs of unsigned 16-bit integers in \a __A with225/// corresponding signed 16-bit integers in \a __B, producing 2 intermediate226/// signed 16-bit results. Sum these 2 results with the corresponding227/// 32-bit integer in \a __W with signed saturation, and store the packed228/// 32-bit results in \a dst.229///230/// \headerfile <immintrin.h>231///232/// \code233/// __m128i _mm_dpwusds_epi32(__m128i __W, __m128i __A, __m128i __B)234/// \endcode235///236/// This intrinsic corresponds to the \c VPDPWSUDS instruction.237///238/// \param __W239/// A 128-bit vector of [4 x int].240/// \param __A241/// A 128-bit vector of [8 x unsigned short].242/// \param __B243/// A 128-bit vector of [8 x short].244/// \returns245/// A 128-bit vector of [4 x int].246///247/// \code{.operation}248/// FOR j := 0 to 3249/// tmp1.dword := ZeroExtend32(__A.word[2*j]) * SignExtend32(__B.word[2*j])250/// tmp2.dword := ZeroExtend32(__A.word[2*j+1]) * SignExtend32(__B.word[2*j+1])251/// dst.dword[j] := SIGNED_DWORD_SATURATE(__W.dword[j] + tmp1 + tmp2)252/// ENDFOR253/// dst[MAX:128] := 0254/// \endcode255#define _mm_dpwusds_epi32(__W, __A, __B) \256 ((__m128i)__builtin_ia32_vpdpwusds128((__v4si)(__W), (__v4si)(__A), \257 (__v4si)(__B)))258 259/// Multiply groups of 2 adjacent pairs of unsigned 16-bit integers in \a __A with260/// corresponding signed 16-bit integers in \a __B, producing 2 intermediate261/// signed 16-bit results. Sum these 2 results with the corresponding262/// 32-bit integer in \a __W with signed saturation, and store the packed263/// 32-bit results in \a dst.264///265/// \headerfile <immintrin.h>266///267/// \code268/// __m256i _mm256_dpwsuds_epi32(__m256i __W, __m256i __A, __m256i __B)269/// \endcode270///271/// This intrinsic corresponds to the \c VPDPWSUDS instruction.272///273/// \param __W274/// A 256-bit vector of [8 x int].275/// \param __A276/// A 256-bit vector of [16 x unsigned short].277/// \param __B278/// A 256-bit vector of [16 x short].279/// \returns280/// A 256-bit vector of [8 x int].281///282/// \code{.operation}283/// FOR j := 0 to 7284/// tmp1.dword := ZeroExtend32(__A.word[2*j]) * SignExtend32(__B.word[2*j])285/// tmp2.dword := ZeroExtend32(__A.word[2*j+1]) * SignExtend32(__B.word[2*j+1])286/// dst.dword[j] := SIGNED_DWORD_SATURATE(__W.dword[j] + tmp1 + tmp2)287/// ENDFOR288/// dst[MAX:256] := 0289/// \endcode290#define _mm256_dpwusds_epi32(__W, __A, __B) \291 ((__m256i)__builtin_ia32_vpdpwusds256((__v8si)(__W), (__v8si)(__A), \292 (__v8si)(__B)))293 294/// Multiply groups of 2 adjacent pairs of unsigned 16-bit integers in \a __A with295/// corresponding unsigned 16-bit integers in \a __B, producing 2 intermediate296/// signed 16-bit results. Sum these 2 results with the corresponding297/// 32-bit integer in \a __W, and store the packed 32-bit results in \a dst.298///299/// \headerfile <immintrin.h>300///301/// \code302/// __m128i _mm_dpwuud_epi32(__m128i __W, __m128i __A, __m128i __B)303/// \endcode304///305/// This intrinsic corresponds to the \c VPDPWUUD instruction.306///307/// \param __W308/// A 128-bit vector of [4 x unsigned int].309/// \param __A310/// A 128-bit vector of [8 x unsigned short].311/// \param __B312/// A 128-bit vector of [8 x unsigned short].313/// \returns314/// A 128-bit vector of [4 x unsigned int].315///316/// \code{.operation}317/// FOR j := 0 to 3318/// tmp1.dword := ZeroExtend32(__A.word[2*j]) * ZeroExtend32(__B.word[2*j])319/// tmp2.dword := ZeroExtend32(__A.word[2*j+1]) * ZeroExtend32(__B.word[2*j+1])320/// dst.dword[j] := __W.dword[j] + tmp1 + tmp2321/// ENDFOR322/// dst[MAX:128] := 0323/// \endcode324#define _mm_dpwuud_epi32(__W, __A, __B) \325 ((__m128i)__builtin_ia32_vpdpwuud128((__v4si)(__W), (__v4si)(__A), \326 (__v4si)(__B)))327 328/// Multiply groups of 2 adjacent pairs of unsigned 16-bit integers in \a __A with329/// corresponding unsigned 16-bit integers in \a __B, producing 2 intermediate330/// signed 16-bit results. Sum these 2 results with the corresponding331/// 32-bit integer in \a __W, and store the packed 32-bit results in \a dst.332///333/// \headerfile <immintrin.h>334///335/// \code336/// __m256i _mm256_dpwuud_epi32(__m256i __W, __m256i __A, __m256i __B)337/// \endcode338///339/// This intrinsic corresponds to the \c VPDPWUUD instruction.340///341/// \param __W342/// A 256-bit vector of [8 x unsigned int].343/// \param __A344/// A 256-bit vector of [16 x unsigned short].345/// \param __B346/// A 256-bit vector of [16 x unsigned short].347/// \returns348/// A 256-bit vector of [8 x unsigned int].349///350/// \code{.operation}351/// FOR j := 0 to 7352/// tmp1.dword := ZeroExtend32(__A.word[2*j]) * ZeroExtend32(__B.word[2*j])353/// tmp2.dword := ZeroExtend32(__A.word[2*j+1]) * ZeroExtend32(__B.word[2*j+1])354/// dst.dword[j] := __W.dword[j] + tmp1 + tmp2355/// ENDFOR356/// dst[MAX:256] := 0357/// \endcode358#define _mm256_dpwuud_epi32(__W, __A, __B) \359 ((__m256i)__builtin_ia32_vpdpwuud256((__v8si)(__W), (__v8si)(__A), \360 (__v8si)(__B)))361 362/// Multiply groups of 2 adjacent pairs of unsigned 16-bit integers in \a __A with363/// corresponding unsigned 16-bit integers in \a __B, producing 2 intermediate364/// signed 16-bit results. Sum these 2 results with the corresponding365/// 32-bit integer in \a __W with signed saturation, and store the packed366/// 32-bit results in \a dst.367///368/// \headerfile <immintrin.h>369///370/// \code371/// __m128i _mm_dpwsuds_epi32(__m128i __W, __m128i __A, __m128i __B)372/// \endcode373///374/// This intrinsic corresponds to the \c VPDPWSUDS instruction.375///376/// \param __W377/// A 128-bit vector of [4 x unsigned int].378/// \param __A379/// A 128-bit vector of [8 x unsigned short].380/// \param __B381/// A 128-bit vector of [8 x unsigned short].382/// \returns383/// A 128-bit vector of [4 x unsigned int].384///385/// \code{.operation}386/// FOR j := 0 to 3387/// tmp1.dword := ZeroExtend32(__A.word[2*j]) * ZeroExtend32(__B.word[2*j])388/// tmp2.dword := ZeroExtend32(__A.word[2*j+1]) * ZeroExtend32(__B.word[2*j+1])389/// dst.dword[j] := UNSIGNED_DWORD_SATURATE(__W.dword[j] + tmp1 + tmp2)390/// ENDFOR391/// dst[MAX:128] := 0392/// \endcode393#define _mm_dpwuuds_epi32(__W, __A, __B) \394 ((__m128i)__builtin_ia32_vpdpwuuds128((__v4si)(__W), (__v4si)(__A), \395 (__v4si)(__B)))396 397/// Multiply groups of 2 adjacent pairs of unsigned 16-bit integers in \a __A with398/// corresponding unsigned 16-bit integers in \a __B, producing 2 intermediate399/// signed 16-bit results. Sum these 2 results with the corresponding400/// 32-bit integer in \a __W with signed saturation, and store the packed401/// 32-bit results in \a dst.402///403/// \headerfile <immintrin.h>404///405/// \code406/// __m256i _mm256_dpwuuds_epi32(__m256i __W, __m256i __A, __m256i __B)407/// \endcode408///409/// This intrinsic corresponds to the \c VPDPWSUDS instruction.410///411/// \param __W412/// A 256-bit vector of [8 x unsigned int].413/// \param __A414/// A 256-bit vector of [16 x unsigned short].415/// \param __B416/// A 256-bit vector of [16 x unsigned short].417/// \returns418/// A 256-bit vector of [8 x unsigned int].419///420/// \code{.operation}421/// FOR j := 0 to 7422/// tmp1.dword := ZeroExtend32(__A.word[2*j]) * ZeroExtend32(__B.word[2*j])423/// tmp2.dword := ZeroExtend32(__A.word[2*j+1]) * ZeroExtend32(__B.word[2*j+1])424/// dst.dword[j] := UNSIGNED_DWORD_SATURATE(__W.dword[j] + tmp1 + tmp2)425/// ENDFOR426/// dst[MAX:256] := 0427/// \endcode428#define _mm256_dpwuuds_epi32(__W, __A, __B) \429 ((__m256i)__builtin_ia32_vpdpwuuds256((__v8si)(__W), (__v8si)(__A), \430 (__v8si)(__B)))431 432#endif // __AVXVNNIINT16INTRIN_H433