396 lines · cpp
1//===----------------------------------------------------------------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8///9/// \file10/// This file contains the implementation of the device kernels that wrap the11/// math functions from the hip-math provider.12///13//===----------------------------------------------------------------------===//14 15#ifdef HIP_MATH_FOUND16 17#include "Conformance/device_code/DeviceAPIs.hpp"18#include "Conformance/device_code/KernelRunner.hpp"19 20#include <gpuintrin.h>21#include <stddef.h>22 23using namespace kernels;24 25//===----------------------------------------------------------------------===//26// Helpers27//===----------------------------------------------------------------------===//28 29static inline float powfRoundedExponent(float Base, float Exponent) {30 return __ocml_pow_f32(Base, __ocml_round_f32(Exponent));31}32 33static inline double sincosSin(double X) {34 double CosX;35 double SinX = __ocml_sincos_f64(X, &CosX);36 return SinX;37}38 39static inline double sincosCos(double X) {40 double CosX;41 double SinX = __ocml_sincos_f64(X, &CosX);42 return CosX;43}44 45static inline float sincosfSin(float X) {46 float CosX;47 float SinX = __ocml_sincos_f32(X, &CosX);48 return SinX;49}50 51static inline float sincosfCos(float X) {52 float CosX;53 float SinX = __ocml_sincos_f32(X, &CosX);54 return CosX;55}56 57//===----------------------------------------------------------------------===//58// Kernels59//===----------------------------------------------------------------------===//60 61extern "C" {62 63__gpu_kernel void acosKernel(const double *X, double *Out,64 size_t NumElements) noexcept {65 runKernelBody<__ocml_acos_f64>(NumElements, Out, X);66}67 68__gpu_kernel void acosfKernel(const float *X, float *Out,69 size_t NumElements) noexcept {70 runKernelBody<__ocml_acos_f32>(NumElements, Out, X);71}72 73__gpu_kernel void acosf16Kernel(const float16 *X, float16 *Out,74 size_t NumElements) noexcept {75 runKernelBody<__ocml_acos_f16>(NumElements, Out, X);76}77 78__gpu_kernel void acoshfKernel(const float *X, float *Out,79 size_t NumElements) noexcept {80 runKernelBody<__ocml_acosh_f32>(NumElements, Out, X);81}82 83__gpu_kernel void acoshf16Kernel(const float16 *X, float16 *Out,84 size_t NumElements) noexcept {85 runKernelBody<__ocml_acosh_f16>(NumElements, Out, X);86}87 88__gpu_kernel void asinKernel(const double *X, double *Out,89 size_t NumElements) noexcept {90 runKernelBody<__ocml_asin_f64>(NumElements, Out, X);91}92 93__gpu_kernel void asinfKernel(const float *X, float *Out,94 size_t NumElements) noexcept {95 runKernelBody<__ocml_asin_f32>(NumElements, Out, X);96}97 98__gpu_kernel void asinf16Kernel(const float16 *X, float16 *Out,99 size_t NumElements) noexcept {100 runKernelBody<__ocml_asin_f16>(NumElements, Out, X);101}102 103__gpu_kernel void asinhfKernel(const float *X, float *Out,104 size_t NumElements) noexcept {105 runKernelBody<__ocml_asinh_f32>(NumElements, Out, X);106}107 108__gpu_kernel void asinhf16Kernel(const float16 *X, float16 *Out,109 size_t NumElements) noexcept {110 runKernelBody<__ocml_asinh_f16>(NumElements, Out, X);111}112 113__gpu_kernel void atanfKernel(const float *X, float *Out,114 size_t NumElements) noexcept {115 runKernelBody<__ocml_atan_f32>(NumElements, Out, X);116}117 118__gpu_kernel void atanf16Kernel(const float16 *X, float16 *Out,119 size_t NumElements) noexcept {120 runKernelBody<__ocml_atan_f16>(NumElements, Out, X);121}122 123__gpu_kernel void atan2fKernel(const float *X, const float *Y, float *Out,124 size_t NumElements) noexcept {125 runKernelBody<__ocml_atan2_f32>(NumElements, Out, X, Y);126}127 128__gpu_kernel void atanhfKernel(const float *X, float *Out,129 size_t NumElements) noexcept {130 runKernelBody<__ocml_atanh_f32>(NumElements, Out, X);131}132 133__gpu_kernel void atanhf16Kernel(const float16 *X, float16 *Out,134 size_t NumElements) noexcept {135 runKernelBody<__ocml_atanh_f16>(NumElements, Out, X);136}137 138__gpu_kernel void cbrtKernel(const double *X, double *Out,139 size_t NumElements) noexcept {140 runKernelBody<__ocml_cbrt_f64>(NumElements, Out, X);141}142 143__gpu_kernel void cbrtfKernel(const float *X, float *Out,144 size_t NumElements) noexcept {145 runKernelBody<__ocml_cbrt_f32>(NumElements, Out, X);146}147 148__gpu_kernel void cosKernel(const double *X, double *Out,149 size_t NumElements) noexcept {150 runKernelBody<__ocml_cos_f64>(NumElements, Out, X);151}152 153__gpu_kernel void cosfKernel(const float *X, float *Out,154 size_t NumElements) noexcept {155 runKernelBody<__ocml_cos_f32>(NumElements, Out, X);156}157 158__gpu_kernel void cosf16Kernel(const float16 *X, float16 *Out,159 size_t NumElements) noexcept {160 runKernelBody<__ocml_cos_f16>(NumElements, Out, X);161}162 163__gpu_kernel void coshfKernel(const float *X, float *Out,164 size_t NumElements) noexcept {165 runKernelBody<__ocml_cosh_f32>(NumElements, Out, X);166}167 168__gpu_kernel void coshf16Kernel(const float16 *X, float16 *Out,169 size_t NumElements) noexcept {170 runKernelBody<__ocml_cosh_f16>(NumElements, Out, X);171}172 173__gpu_kernel void cospifKernel(const float *X, float *Out,174 size_t NumElements) noexcept {175 runKernelBody<__ocml_cospi_f32>(NumElements, Out, X);176}177 178__gpu_kernel void erffKernel(const float *X, float *Out,179 size_t NumElements) noexcept {180 runKernelBody<__ocml_erf_f32>(NumElements, Out, X);181}182 183__gpu_kernel void expKernel(const double *X, double *Out,184 size_t NumElements) noexcept {185 runKernelBody<__ocml_exp_f64>(NumElements, Out, X);186}187 188__gpu_kernel void expfKernel(const float *X, float *Out,189 size_t NumElements) noexcept {190 runKernelBody<__ocml_exp_f32>(NumElements, Out, X);191}192 193__gpu_kernel void expf16Kernel(const float16 *X, float16 *Out,194 size_t NumElements) noexcept {195 runKernelBody<__ocml_exp_f16>(NumElements, Out, X);196}197 198__gpu_kernel void exp10Kernel(const double *X, double *Out,199 size_t NumElements) noexcept {200 runKernelBody<__ocml_exp10_f64>(NumElements, Out, X);201}202 203__gpu_kernel void exp10fKernel(const float *X, float *Out,204 size_t NumElements) noexcept {205 runKernelBody<__ocml_exp10_f32>(NumElements, Out, X);206}207 208__gpu_kernel void exp10f16Kernel(const float16 *X, float16 *Out,209 size_t NumElements) noexcept {210 runKernelBody<__ocml_exp10_f16>(NumElements, Out, X);211}212 213__gpu_kernel void exp2Kernel(const double *X, double *Out,214 size_t NumElements) noexcept {215 runKernelBody<__ocml_exp2_f64>(NumElements, Out, X);216}217 218__gpu_kernel void exp2fKernel(const float *X, float *Out,219 size_t NumElements) noexcept {220 runKernelBody<__ocml_exp2_f32>(NumElements, Out, X);221}222 223__gpu_kernel void exp2f16Kernel(const float16 *X, float16 *Out,224 size_t NumElements) noexcept {225 runKernelBody<__ocml_exp2_f16>(NumElements, Out, X);226}227 228__gpu_kernel void expm1Kernel(const double *X, double *Out,229 size_t NumElements) noexcept {230 runKernelBody<__ocml_expm1_f64>(NumElements, Out, X);231}232 233__gpu_kernel void expm1fKernel(const float *X, float *Out,234 size_t NumElements) noexcept {235 runKernelBody<__ocml_expm1_f32>(NumElements, Out, X);236}237 238__gpu_kernel void expm1f16Kernel(const float16 *X, float16 *Out,239 size_t NumElements) noexcept {240 runKernelBody<__ocml_expm1_f16>(NumElements, Out, X);241}242 243__gpu_kernel void hypotKernel(const double *X, const double *Y, double *Out,244 size_t NumElements) noexcept {245 runKernelBody<__ocml_hypot_f64>(NumElements, Out, X, Y);246}247 248__gpu_kernel void hypotfKernel(const float *X, const float *Y, float *Out,249 size_t NumElements) noexcept {250 runKernelBody<__ocml_hypot_f32>(NumElements, Out, X, Y);251}252 253__gpu_kernel void logKernel(const double *X, double *Out,254 size_t NumElements) noexcept {255 runKernelBody<__ocml_log_f64>(NumElements, Out, X);256}257 258__gpu_kernel void logfKernel(const float *X, float *Out,259 size_t NumElements) noexcept {260 runKernelBody<__ocml_log_f32>(NumElements, Out, X);261}262 263__gpu_kernel void logf16Kernel(const float16 *X, float16 *Out,264 size_t NumElements) noexcept {265 runKernelBody<__ocml_log_f16>(NumElements, Out, X);266}267 268__gpu_kernel void log10Kernel(const double *X, double *Out,269 size_t NumElements) noexcept {270 runKernelBody<__ocml_log10_f64>(NumElements, Out, X);271}272 273__gpu_kernel void log10fKernel(const float *X, float *Out,274 size_t NumElements) noexcept {275 runKernelBody<__ocml_log10_f32>(NumElements, Out, X);276}277 278__gpu_kernel void log10f16Kernel(const float16 *X, float16 *Out,279 size_t NumElements) noexcept {280 runKernelBody<__ocml_log10_f16>(NumElements, Out, X);281}282 283__gpu_kernel void log1pKernel(const double *X, double *Out,284 size_t NumElements) noexcept {285 runKernelBody<__ocml_log1p_f64>(NumElements, Out, X);286}287 288__gpu_kernel void log1pfKernel(const float *X, float *Out,289 size_t NumElements) noexcept {290 runKernelBody<__ocml_log1p_f32>(NumElements, Out, X);291}292 293__gpu_kernel void log2Kernel(const double *X, double *Out,294 size_t NumElements) noexcept {295 runKernelBody<__ocml_log2_f64>(NumElements, Out, X);296}297 298__gpu_kernel void log2fKernel(const float *X, float *Out,299 size_t NumElements) noexcept {300 runKernelBody<__ocml_log2_f32>(NumElements, Out, X);301}302 303__gpu_kernel void log2f16Kernel(const float16 *X, float16 *Out,304 size_t NumElements) noexcept {305 runKernelBody<__ocml_log2_f16>(NumElements, Out, X);306}307 308__gpu_kernel void powfKernel(const float *X, float *Y, float *Out,309 size_t NumElements) noexcept {310 runKernelBody<__ocml_pow_f32>(NumElements, Out, X, Y);311}312 313__gpu_kernel void powfRoundedExponentKernel(const float *X, float *Y,314 float *Out,315 size_t NumElements) noexcept {316 runKernelBody<powfRoundedExponent>(NumElements, Out, X, Y);317}318 319__gpu_kernel void sinKernel(const double *X, double *Out,320 size_t NumElements) noexcept {321 runKernelBody<__ocml_sin_f64>(NumElements, Out, X);322}323 324__gpu_kernel void sinfKernel(const float *X, float *Out,325 size_t NumElements) noexcept {326 runKernelBody<__ocml_sin_f32>(NumElements, Out, X);327}328 329__gpu_kernel void sinf16Kernel(const float16 *X, float16 *Out,330 size_t NumElements) noexcept {331 runKernelBody<__ocml_sin_f16>(NumElements, Out, X);332}333 334__gpu_kernel void sincosSinKernel(const double *X, double *Out,335 size_t NumElements) noexcept {336 runKernelBody<sincosSin>(NumElements, Out, X);337}338 339__gpu_kernel void sincosCosKernel(const double *X, double *Out,340 size_t NumElements) noexcept {341 runKernelBody<sincosCos>(NumElements, Out, X);342}343 344__gpu_kernel void sincosfSinKernel(const float *X, float *Out,345 size_t NumElements) noexcept {346 runKernelBody<sincosfSin>(NumElements, Out, X);347}348 349__gpu_kernel void sincosfCosKernel(const float *X, float *Out,350 size_t NumElements) noexcept {351 runKernelBody<sincosfCos>(NumElements, Out, X);352}353 354__gpu_kernel void sinhfKernel(const float *X, float *Out,355 size_t NumElements) noexcept {356 runKernelBody<__ocml_sinh_f32>(NumElements, Out, X);357}358 359__gpu_kernel void sinhf16Kernel(const float16 *X, float16 *Out,360 size_t NumElements) noexcept {361 runKernelBody<__ocml_sinh_f16>(NumElements, Out, X);362}363 364__gpu_kernel void sinpifKernel(const float *X, float *Out,365 size_t NumElements) noexcept {366 runKernelBody<__ocml_sinpi_f32>(NumElements, Out, X);367}368 369__gpu_kernel void tanKernel(const double *X, double *Out,370 size_t NumElements) noexcept {371 runKernelBody<__ocml_tan_f64>(NumElements, Out, X);372}373 374__gpu_kernel void tanfKernel(const float *X, float *Out,375 size_t NumElements) noexcept {376 runKernelBody<__ocml_tan_f32>(NumElements, Out, X);377}378 379__gpu_kernel void tanf16Kernel(const float16 *X, float16 *Out,380 size_t NumElements) noexcept {381 runKernelBody<__ocml_tan_f16>(NumElements, Out, X);382}383 384__gpu_kernel void tanhfKernel(const float *X, float *Out,385 size_t NumElements) noexcept {386 runKernelBody<__ocml_tanh_f32>(NumElements, Out, X);387}388 389__gpu_kernel void tanhf16Kernel(const float16 *X, float16 *Out,390 size_t NumElements) noexcept {391 runKernelBody<__ocml_tanh_f16>(NumElements, Out, X);392}393} // extern "C"394 395#endif // HIP_MATH_FOUND396