423 lines · cpp
1//===----------------------------------------------------------------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8///9/// \file10/// This file contains the implementation of the device kernels that wrap the11/// math functions from the llvm-libm provider.12///13//===----------------------------------------------------------------------===//14 15#include "Conformance/device_code/DeviceAPIs.hpp"16#include "Conformance/device_code/KernelRunner.hpp"17 18#include <gpuintrin.h>19#include <math.h>20#include <stddef.h>21 22using namespace kernels;23 24//===----------------------------------------------------------------------===//25// Helpers26//===----------------------------------------------------------------------===//27 28static inline float powfRoundedExponent(float Base, float Exponent) {29 return powf(Base, roundf(Exponent));30}31 32static inline double sincosSin(double X) {33 double SinX, CosX;34 sincos(X, &SinX, &CosX);35 return SinX;36}37 38static inline double sincosCos(double X) {39 double SinX, CosX;40 sincos(X, &SinX, &CosX);41 return CosX;42}43 44static inline float sincosfSin(float X) {45 float SinX, CosX;46 sincosf(X, &SinX, &CosX);47 return SinX;48}49 50static inline float sincosfCos(float X) {51 float SinX, CosX;52 sincosf(X, &SinX, &CosX);53 return CosX;54}55 56//===----------------------------------------------------------------------===//57// Kernels58//===----------------------------------------------------------------------===//59 60extern "C" {61 62__gpu_kernel void acosKernel(const double *X, double *Out,63 size_t NumElements) noexcept {64 runKernelBody<acos>(NumElements, Out, X);65}66 67__gpu_kernel void acosfKernel(const float *X, float *Out,68 size_t NumElements) noexcept {69 runKernelBody<acosf>(NumElements, Out, X);70}71 72__gpu_kernel void acosf16Kernel(const float16 *X, float16 *Out,73 size_t NumElements) noexcept {74 runKernelBody<acosf16>(NumElements, Out, X);75}76 77__gpu_kernel void acoshfKernel(const float *X, float *Out,78 size_t NumElements) noexcept {79 runKernelBody<acoshf>(NumElements, Out, X);80}81 82__gpu_kernel void acoshf16Kernel(const float16 *X, float16 *Out,83 size_t NumElements) noexcept {84 runKernelBody<acoshf16>(NumElements, Out, X);85}86 87__gpu_kernel void acospif16Kernel(const float16 *X, float16 *Out,88 size_t NumElements) noexcept {89 runKernelBody<acospif16>(NumElements, Out, X);90}91 92__gpu_kernel void asinKernel(const double *X, double *Out,93 size_t NumElements) noexcept {94 runKernelBody<asin>(NumElements, Out, X);95}96 97__gpu_kernel void asinfKernel(const float *X, float *Out,98 size_t NumElements) noexcept {99 runKernelBody<asinf>(NumElements, Out, X);100}101 102__gpu_kernel void asinf16Kernel(const float16 *X, float16 *Out,103 size_t NumElements) noexcept {104 runKernelBody<asinf16>(NumElements, Out, X);105}106 107__gpu_kernel void asinhfKernel(const float *X, float *Out,108 size_t NumElements) noexcept {109 runKernelBody<asinhf>(NumElements, Out, X);110}111 112__gpu_kernel void asinhf16Kernel(const float16 *X, float16 *Out,113 size_t NumElements) noexcept {114 runKernelBody<asinhf16>(NumElements, Out, X);115}116 117__gpu_kernel void atanfKernel(const float *X, float *Out,118 size_t NumElements) noexcept {119 runKernelBody<atanf>(NumElements, Out, X);120}121 122__gpu_kernel void atanf16Kernel(const float16 *X, float16 *Out,123 size_t NumElements) noexcept {124 runKernelBody<atanf16>(NumElements, Out, X);125}126 127__gpu_kernel void atan2fKernel(const float *X, const float *Y, float *Out,128 size_t NumElements) noexcept {129 runKernelBody<atan2f>(NumElements, Out, X, Y);130}131 132__gpu_kernel void atanhfKernel(const float *X, float *Out,133 size_t NumElements) noexcept {134 runKernelBody<atanhf>(NumElements, Out, X);135}136 137__gpu_kernel void atanhf16Kernel(const float16 *X, float16 *Out,138 size_t NumElements) noexcept {139 runKernelBody<atanhf16>(NumElements, Out, X);140}141 142__gpu_kernel void cbrtKernel(const double *X, double *Out,143 size_t NumElements) noexcept {144 runKernelBody<cbrt>(NumElements, Out, X);145}146 147__gpu_kernel void cbrtfKernel(const float *X, float *Out,148 size_t NumElements) noexcept {149 runKernelBody<cbrtf>(NumElements, Out, X);150}151 152__gpu_kernel void cosKernel(const double *X, double *Out,153 size_t NumElements) noexcept {154 runKernelBody<cos>(NumElements, Out, X);155}156 157__gpu_kernel void cosfKernel(const float *X, float *Out,158 size_t NumElements) noexcept {159 runKernelBody<cosf>(NumElements, Out, X);160}161 162__gpu_kernel void cosf16Kernel(const float16 *X, float16 *Out,163 size_t NumElements) noexcept {164 runKernelBody<cosf16>(NumElements, Out, X);165}166 167__gpu_kernel void coshfKernel(const float *X, float *Out,168 size_t NumElements) noexcept {169 runKernelBody<coshf>(NumElements, Out, X);170}171 172__gpu_kernel void coshf16Kernel(const float16 *X, float16 *Out,173 size_t NumElements) noexcept {174 runKernelBody<coshf16>(NumElements, Out, X);175}176 177__gpu_kernel void cospifKernel(const float *X, float *Out,178 size_t NumElements) noexcept {179 runKernelBody<cospif>(NumElements, Out, X);180}181 182__gpu_kernel void cospif16Kernel(const float16 *X, float16 *Out,183 size_t NumElements) noexcept {184 runKernelBody<cospif16>(NumElements, Out, X);185}186 187__gpu_kernel void erffKernel(const float *X, float *Out,188 size_t NumElements) noexcept {189 runKernelBody<erff>(NumElements, Out, X);190}191 192__gpu_kernel void expKernel(const double *X, double *Out,193 size_t NumElements) noexcept {194 runKernelBody<exp>(NumElements, Out, X);195}196 197__gpu_kernel void expfKernel(const float *X, float *Out,198 size_t NumElements) noexcept {199 runKernelBody<expf>(NumElements, Out, X);200}201 202__gpu_kernel void expf16Kernel(const float16 *X, float16 *Out,203 size_t NumElements) noexcept {204 runKernelBody<expf16>(NumElements, Out, X);205}206 207__gpu_kernel void exp10Kernel(const double *X, double *Out,208 size_t NumElements) noexcept {209 runKernelBody<exp10>(NumElements, Out, X);210}211 212__gpu_kernel void exp10fKernel(const float *X, float *Out,213 size_t NumElements) noexcept {214 runKernelBody<exp10f>(NumElements, Out, X);215}216 217__gpu_kernel void exp10f16Kernel(const float16 *X, float16 *Out,218 size_t NumElements) noexcept {219 runKernelBody<exp10f16>(NumElements, Out, X);220}221 222__gpu_kernel void exp2Kernel(const double *X, double *Out,223 size_t NumElements) noexcept {224 runKernelBody<exp2>(NumElements, Out, X);225}226 227__gpu_kernel void exp2fKernel(const float *X, float *Out,228 size_t NumElements) noexcept {229 runKernelBody<exp2f>(NumElements, Out, X);230}231 232__gpu_kernel void exp2f16Kernel(const float16 *X, float16 *Out,233 size_t NumElements) noexcept {234 runKernelBody<exp2f16>(NumElements, Out, X);235}236 237__gpu_kernel void expm1Kernel(const double *X, double *Out,238 size_t NumElements) noexcept {239 runKernelBody<expm1>(NumElements, Out, X);240}241 242__gpu_kernel void expm1fKernel(const float *X, float *Out,243 size_t NumElements) noexcept {244 runKernelBody<expm1f>(NumElements, Out, X);245}246 247__gpu_kernel void expm1f16Kernel(const float16 *X, float16 *Out,248 size_t NumElements) noexcept {249 runKernelBody<expm1f16>(NumElements, Out, X);250}251 252__gpu_kernel void hypotKernel(const double *X, const double *Y, double *Out,253 size_t NumElements) noexcept {254 runKernelBody<hypot>(NumElements, Out, X, Y);255}256 257__gpu_kernel void hypotfKernel(const float *X, const float *Y, float *Out,258 size_t NumElements) noexcept {259 runKernelBody<hypotf>(NumElements, Out, X, Y);260}261 262__gpu_kernel void hypotf16Kernel(const float16 *X, const float16 *Y,263 float16 *Out, size_t NumElements) noexcept {264 runKernelBody<hypotf16>(NumElements, Out, X, Y);265}266 267__gpu_kernel void logKernel(const double *X, double *Out,268 size_t NumElements) noexcept {269 runKernelBody<log>(NumElements, Out, X);270}271 272__gpu_kernel void logfKernel(const float *X, float *Out,273 size_t NumElements) noexcept {274 runKernelBody<logf>(NumElements, Out, X);275}276 277__gpu_kernel void logf16Kernel(const float16 *X, float16 *Out,278 size_t NumElements) noexcept {279 runKernelBody<logf16>(NumElements, Out, X);280}281 282__gpu_kernel void log10Kernel(const double *X, double *Out,283 size_t NumElements) noexcept {284 runKernelBody<log10>(NumElements, Out, X);285}286 287__gpu_kernel void log10fKernel(const float *X, float *Out,288 size_t NumElements) noexcept {289 runKernelBody<log10f>(NumElements, Out, X);290}291 292__gpu_kernel void log10f16Kernel(const float16 *X, float16 *Out,293 size_t NumElements) noexcept {294 runKernelBody<log10f16>(NumElements, Out, X);295}296 297__gpu_kernel void log1pKernel(const double *X, double *Out,298 size_t NumElements) noexcept {299 runKernelBody<log1p>(NumElements, Out, X);300}301 302__gpu_kernel void log1pfKernel(const float *X, float *Out,303 size_t NumElements) noexcept {304 runKernelBody<log1pf>(NumElements, Out, X);305}306 307__gpu_kernel void log2Kernel(const double *X, double *Out,308 size_t NumElements) noexcept {309 runKernelBody<log2>(NumElements, Out, X);310}311 312__gpu_kernel void log2fKernel(const float *X, float *Out,313 size_t NumElements) noexcept {314 runKernelBody<log2f>(NumElements, Out, X);315}316 317__gpu_kernel void log2f16Kernel(const float16 *X, float16 *Out,318 size_t NumElements) noexcept {319 runKernelBody<log2f16>(NumElements, Out, X);320}321 322__gpu_kernel void powfKernel(const float *X, float *Y, float *Out,323 size_t NumElements) noexcept {324 runKernelBody<powf>(NumElements, Out, X, Y);325}326 327__gpu_kernel void powfRoundedExponentKernel(const float *X, float *Y,328 float *Out,329 size_t NumElements) noexcept {330 runKernelBody<powfRoundedExponent>(NumElements, Out, X, Y);331}332 333__gpu_kernel void sinKernel(const double *X, double *Out,334 size_t NumElements) noexcept {335 runKernelBody<sin>(NumElements, Out, X);336}337 338__gpu_kernel void sinfKernel(const float *X, float *Out,339 size_t NumElements) noexcept {340 runKernelBody<sinf>(NumElements, Out, X);341}342 343__gpu_kernel void sinf16Kernel(const float16 *X, float16 *Out,344 size_t NumElements) noexcept {345 runKernelBody<sinf16>(NumElements, Out, X);346}347 348__gpu_kernel void sincosSinKernel(const double *X, double *Out,349 size_t NumElements) noexcept {350 runKernelBody<sincosSin>(NumElements, Out, X);351}352 353__gpu_kernel void sincosCosKernel(const double *X, double *Out,354 size_t NumElements) noexcept {355 runKernelBody<sincosCos>(NumElements, Out, X);356}357 358__gpu_kernel void sincosfSinKernel(const float *X, float *Out,359 size_t NumElements) noexcept {360 runKernelBody<sincosfSin>(NumElements, Out, X);361}362 363__gpu_kernel void sincosfCosKernel(const float *X, float *Out,364 size_t NumElements) noexcept {365 runKernelBody<sincosfCos>(NumElements, Out, X);366}367 368__gpu_kernel void sinhfKernel(const float *X, float *Out,369 size_t NumElements) noexcept {370 runKernelBody<sinhf>(NumElements, Out, X);371}372 373__gpu_kernel void sinhf16Kernel(const float16 *X, float16 *Out,374 size_t NumElements) noexcept {375 runKernelBody<sinhf16>(NumElements, Out, X);376}377 378__gpu_kernel void sinpifKernel(const float *X, float *Out,379 size_t NumElements) noexcept {380 runKernelBody<sinpif>(NumElements, Out, X);381}382 383__gpu_kernel void sinpif16Kernel(const float16 *X, float16 *Out,384 size_t NumElements) noexcept {385 runKernelBody<sinpif16>(NumElements, Out, X);386}387 388__gpu_kernel void tanKernel(const double *X, double *Out,389 size_t NumElements) noexcept {390 runKernelBody<tan>(NumElements, Out, X);391}392 393__gpu_kernel void tanfKernel(const float *X, float *Out,394 size_t NumElements) noexcept {395 runKernelBody<tanf>(NumElements, Out, X);396}397 398__gpu_kernel void tanf16Kernel(const float16 *X, float16 *Out,399 size_t NumElements) noexcept {400 runKernelBody<tanf16>(NumElements, Out, X);401}402 403__gpu_kernel void tanhfKernel(const float *X, float *Out,404 size_t NumElements) noexcept {405 runKernelBody<tanhf>(NumElements, Out, X);406}407 408__gpu_kernel void tanhf16Kernel(const float16 *X, float16 *Out,409 size_t NumElements) noexcept {410 runKernelBody<tanhf16>(NumElements, Out, X);411}412 413__gpu_kernel void tanpifKernel(const float *X, float *Out,414 size_t NumElements) noexcept {415 runKernelBody<tanpif>(NumElements, Out, X);416}417 418__gpu_kernel void tanpif16Kernel(const float16 *X, float16 *Out,419 size_t NumElements) noexcept {420 runKernelBody<tanpif16>(NumElements, Out, X);421}422} // extern "C"423