231 lines · c
1/*===------------- amxfp8intrin.h - AMX intrinsics -*- C++ -*----------------===2 *3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4 * See https://llvm.org/LICENSE.txt for license information.5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6 *7 *===------------------------------------------------------------------------===8 */9 10#ifndef __IMMINTRIN_H11#error "Never use <amxfp8intrin.h> directly; include <immintrin.h> instead."12#endif /* __IMMINTRIN_H */13 14#ifndef __AMXFP8INTRIN_H15#define __AMXFP8INTRIN_H16#ifdef __x86_64__17 18#define __DEFAULT_FN_ATTRS_FP8 \19 __attribute__((__always_inline__, __nodebug__, __target__("amx-fp8")))20 21static __inline__ _tile1024i __DEFAULT_FN_ATTRS_FP822_tile_dpbf8ps_internal(unsigned short m, unsigned short n, unsigned short k,23 _tile1024i dst, _tile1024i src1, _tile1024i src2) {24 return __builtin_ia32_tdpbf8ps_internal(m, n, k, dst, src1, src2);25}26 27/// Perform the dot product of a BF8 value \a src1 by a BF8 value \a src228/// accumulating into a Single Precision (FP32) source/dest \a dst.29///30/// \headerfile <immintrin.h>31///32/// \code33/// void __tile_dpbf8ps (__tile1024i *dst, __tile1024i src1, __tile1024i src2)34/// \endcode35///36/// \code{.operation}37/// FOR m := 0 TO dst.rows - 138/// temp1[(dst.colsb / 4 - 1) : 0] = 039/// FOR k := 0 TO src1.colsb / 4 - 140/// FOR n := 0 TO dst.colsb / 4 - 141/// temp1[n] +=42/// INT64(src1.row[m].float8[4*k+0]) * INT64(src2.row[k].float8[4*n+0])43/// + INT64(src1.row[m].float8[4*k+1]) * INT64(src2.row[k].float8[4*n+1])44/// + INT64(src1.row[m].float8[4*k+2]) * INT64(src2.row[k].float8[4*n+2])45/// + INT64(src1.row[m].float8[4*k+3]) * INT64(src2.row[k].float8[4*n+3])46/// ENDFOR47/// ENDFOR48/// FOR n := 0 TO dst.colsb / 4 - 149/// tmp.row[m].fp32[n] = dst.row[m].fp32[n] + FP32(temp1[n])50/// ENDFOR51/// write_row_and_zero(dst, m, tmp, dst.colsb)52/// zero_upper_rows(dst, dst.rows)53/// zero_tileconfig_start()54/// \endcode55///56/// This intrinsic corresponds to the \c TDPBF8PS instruction.57///58/// \param dst59/// The destination tile. Max size is 1024 Bytes.60/// \param src161/// The 1st source tile. Max size is 1024 Bytes.62/// \param src263/// The 2nd source tile. Max size is 1024 Bytes.64__DEFAULT_FN_ATTRS_FP8 static void65__tile_dpbf8ps(__tile1024i *dst, __tile1024i src1, __tile1024i src2) {66 dst->tile = _tile_dpbf8ps_internal(src1.row, src2.col, src1.col, dst->tile,67 src1.tile, src2.tile);68}69 70static __inline__ _tile1024i __DEFAULT_FN_ATTRS_FP871_tile_dpbhf8ps_internal(unsigned short m, unsigned short n, unsigned short k,72 _tile1024i dst, _tile1024i src1, _tile1024i src2) {73 return __builtin_ia32_tdpbhf8ps_internal(m, n, k, dst, src1, src2);74}75 76/// Perform the dot product of a BF8 value \a src1 by an HF8 value \a src277/// accumulating into a Single Precision (FP32) source/dest \a dst.78///79/// \headerfile <immintrin.h>80///81/// \code82/// void __tile_dpbhf8ps (__tile1024i dst, __tile1024i src1, __tile1024i src2)83/// \endcode84///85/// \code{.operation}86/// FOR m := 0 TO dst.rows - 187/// temp1[(dst.colsb / 4 - 1) : 0] = 088/// FOR k := 0 TO src1.colsb / 4 - 189/// FOR n := 0 TO dst.colsb / 4 - 190/// temp1[n] +=91/// INT64(src1.row[m].float8[4*k+0]) * INT64(src2.row[k].float8[4*n+0])92/// + INT64(src1.row[m].float8[4*k+1]) * INT64(src2.row[k].float8[4*n+1])93/// + INT64(src1.row[m].float8[4*k+2]) * INT64(src2.row[k].float8[4*n+2])94/// + INT64(src1.row[m].float8[4*k+3]) * INT64(src2.row[k].float8[4*n+3])95/// ENDFOR96/// ENDFOR97/// FOR n := 0 TO dst.colsb / 4 - 198/// tmp.row[m].fp32[n] = dst.row[m].fp32[n] + FP32(temp1[n])99/// ENDFOR100/// write_row_and_zero(dst, m, tmp, dst.colsb)101/// zero_upper_rows(dst, dst.rows)102/// zero_tileconfig_start()103/// \endcode104///105/// This intrinsic corresponds to the \c TDPBHF8PS instruction.106///107/// \param dst108/// The destination tile. Max size is 1024 Bytes.109/// \param src1110/// The 1st source tile. Max size is 1024 Bytes.111/// \param src2112/// The 2nd source tile. Max size is 1024 Bytes.113__DEFAULT_FN_ATTRS_FP8 static void114__tile_dpbhf8ps(__tile1024i *dst, __tile1024i src1, __tile1024i src2) {115 dst->tile = _tile_dpbhf8ps_internal(src1.row, src2.col, src1.col, dst->tile,116 src1.tile, src2.tile);117}118 119static __inline__ _tile1024i __DEFAULT_FN_ATTRS_FP8120_tile_dphbf8ps_internal(unsigned short m, unsigned short n, unsigned short k,121 _tile1024i dst, _tile1024i src1, _tile1024i src2) {122 return __builtin_ia32_tdphbf8ps_internal(m, n, k, dst, src1, src2);123}124 125/// Perform the dot product of an HF8 value \a src1 by a BF8 value \a src2126/// accumulating into a Single Precision (FP32) source/dest \a dst.127///128/// \headerfile <immintrin.h>129///130/// \code131/// void __tile_dphbf8ps (__tile1024i dst, __tile1024i src1, __tile1024i src2)132/// \endcode133///134/// \code{.operation}135/// FOR m := 0 TO dst.rows - 1136/// temp1[(dst.colsb / 4 - 1) : 0] = 0137/// FOR k := 0 TO src1.colsb / 4 - 1138/// FOR n := 0 TO dst.colsb / 4 - 1139/// temp1[n] +=140/// INT64(src1.row[m].float8[4*k+0]) * INT64(src2.row[k].float8[4*n+0])141/// + INT64(src1.row[m].float8[4*k+1]) * INT64(src2.row[k].float8[4*n+1])142/// + INT64(src1.row[m].float8[4*k+2]) * INT64(src2.row[k].float8[4*n+2])143/// + INT64(src1.row[m].float8[4*k+3]) * INT64(src2.row[k].float8[4*n+3])144/// ENDFOR145/// ENDFOR146/// FOR n := 0 TO dst.colsb / 4 - 1147/// tmp.row[m].fp32[n] = dst.row[m].fp32[n] + FP32(temp1[n])148/// ENDFOR149/// write_row_and_zero(dst, m, tmp, dst.colsb)150/// zero_upper_rows(dst, dst.rows)151/// zero_tileconfig_start()152/// \endcode153///154/// This intrinsic corresponds to the \c TDPHBF8PS instruction.155///156/// \param dst157/// The destination tile. Max size is 1024 Bytes.158/// \param src1159/// The 1st source tile. Max size is 1024 Bytes.160/// \param src2161/// The 2nd source tile. Max size is 1024 Bytes.162 163__DEFAULT_FN_ATTRS_FP8 static void164__tile_dphbf8ps(__tile1024i *dst, __tile1024i src1, __tile1024i src2) {165 dst->tile = _tile_dphbf8ps_internal(src1.row, src2.col, src1.col, dst->tile,166 src1.tile, src2.tile);167}168 169static __inline__ _tile1024i __DEFAULT_FN_ATTRS_FP8170_tile_dphf8ps_internal(unsigned short m, unsigned short n, unsigned short k,171 _tile1024i dst, _tile1024i src1, _tile1024i src2) {172 return __builtin_ia32_tdphf8ps_internal(m, n, k, dst, src1, src2);173}174 175/// Perform the dot product of an HF8 value \a src1 by an HF8 value \a src2176/// accumulating into a Single Precision (FP32) source/dest \a dst.177///178/// \headerfile <immintrin.h>179///180/// \code181/// void __tile_dphf8ps (__tile1024i dst, __tile1024i src1, __tile1024i src2)182/// \endcode183///184/// \code{.operation}185/// FOR m := 0 TO dst.rows - 1186/// temp1[(dst.colsb / 4 - 1) : 0] = 0187/// FOR k := 0 TO src1.colsb / 4 - 1188/// FOR n := 0 TO dst.colsb / 4 - 1189/// temp1[n] +=190/// INT64(src1.row[m].float8[4*k+0]) * INT64(src2.row[k].float8[4*n+0])191/// + INT64(src1.row[m].float8[4*k+1]) * INT64(src2.row[k].float8[4*n+1])192/// + INT64(src1.row[m].float8[4*k+2]) * INT64(src2.row[k].float8[4*n+2])193/// + INT64(src1.row[m].float8[4*k+3]) * INT64(src2.row[k].float8[4*n+3])194/// ENDFOR195/// ENDFOR196/// FOR n := 0 TO dst.colsb / 4 - 1197/// tmp.row[m].fp32[n] = dst.row[m].fp32[n] + FP32(temp1[n])198/// ENDFOR199/// write_row_and_zero(dst, m, tmp, dst.colsb)200/// zero_upper_rows(dst, dst.rows)201/// zero_tileconfig_start()202/// \endcode203///204/// This intrinsic corresponds to the \c TDPHF8PS instruction.205///206/// \param dst207/// The destination tile. Max size is 1024 Bytes.208/// \param src1209/// The 1st source tile. Max size is 1024 Bytes.210/// \param src2211/// The 2nd source tile. Max size is 1024 Bytes.212__DEFAULT_FN_ATTRS_FP8 static void213__tile_dphf8ps(__tile1024i *dst, __tile1024i src1, __tile1024i src2) {214 dst->tile = _tile_dphf8ps_internal(src1.row, src2.col, src1.col, dst->tile,215 src1.tile, src2.tile);216}217 218#define _tile_dpbf8ps(dst, src1, src2) \219 __builtin_ia32_tdpbf8ps((dst), (src1), (src2))220#define _tile_dpbhf8ps(dst, src1, src2) \221 __builtin_ia32_tdpbhf8ps((dst), (src1), (src2))222#define _tile_dphbf8ps(dst, src1, src2) \223 __builtin_ia32_tdphbf8ps((dst), (src1), (src2))224#define _tile_dphf8ps(dst, src1, src2) \225 __builtin_ia32_tdphf8ps((dst), (src1), (src2))226 227#undef __DEFAULT_FN_ATTRS_FP8228 229#endif /* __x86_64__ */230#endif /* __AMXFP8INTRIN_H */231