512 lines · plain
1//===----------------------------------------------------------------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8 9#include <clc/internal/clc.h>10#include <clc/math/clc_exp.h>11#include <clc/math/clc_fabs.h>12#include <clc/math/clc_fma.h>13#include <clc/math/clc_mad.h>14#include <clc/math/math.h>15#include <clc/relational/clc_isnan.h>16 17/*18 * ====================================================19 * Copyright (C) 1993 by Sun Microsystems, Inc. All rights reserved.20 *21 * Developed at SunPro, a Sun Microsystems, Inc. business.22 * Permission to use, copy, modify, and distribute this23 * software is freely granted, provided that this notice24 * is preserved.25 * ====================================================26 */27 28#define erx 8.4506291151e-01f /* 0x3f58560b */29 30// Coefficients for approximation to erf on [0, 0.84375]31 32#define efx 1.2837916613e-01f /* 0x3e0375d4 */33#define efx8 1.0270333290e+00f /* 0x3f8375d4 */34 35#define pp0 1.2837916613e-01f /* 0x3e0375d4 */36#define pp1 -3.2504209876e-01f /* 0xbea66beb */37#define pp2 -2.8481749818e-02f /* 0xbce9528f */38#define pp3 -5.7702702470e-03f /* 0xbbbd1489 */39#define pp4 -2.3763017452e-05f /* 0xb7c756b1 */40#define qq1 3.9791721106e-01f /* 0x3ecbbbce */41#define qq2 6.5022252500e-02f /* 0x3d852a63 */42#define qq3 5.0813062117e-03f /* 0x3ba68116 */43#define qq4 1.3249473704e-04f /* 0x390aee49 */44#define qq5 -3.9602282413e-06f /* 0xb684e21a */45 46// Coefficients for approximation to erf in [0.84375, 1.25]47 48#define pa0 -2.3621185683e-03f /* 0xbb1acdc6 */49#define pa1 4.1485610604e-01f /* 0x3ed46805 */50#define pa2 -3.7220788002e-01f /* 0xbebe9208 */51#define pa3 3.1834661961e-01f /* 0x3ea2fe54 */52#define pa4 -1.1089469492e-01f /* 0xbde31cc2 */53#define pa5 3.5478305072e-02f /* 0x3d1151b3 */54#define pa6 -2.1663755178e-03f /* 0xbb0df9c0 */55#define qa1 1.0642088205e-01f /* 0x3dd9f331 */56#define qa2 5.4039794207e-01f /* 0x3f0a5785 */57#define qa3 7.1828655899e-02f /* 0x3d931ae7 */58#define qa4 1.2617121637e-01f /* 0x3e013307 */59#define qa5 1.3637083583e-02f /* 0x3c5f6e13 */60#define qa6 1.1984500103e-02f /* 0x3c445aa3 */61 62// Coefficients for approximation to erfc in [1.25, 1/0.35]63 64#define ra0 -9.8649440333e-03f /* 0xbc21a093 */65#define ra1 -6.9385856390e-01f /* 0xbf31a0b7 */66#define ra2 -1.0558626175e+01f /* 0xc128f022 */67#define ra3 -6.2375331879e+01f /* 0xc2798057 */68#define ra4 -1.6239666748e+02f /* 0xc322658c */69#define ra5 -1.8460508728e+02f /* 0xc3389ae7 */70#define ra6 -8.1287437439e+01f /* 0xc2a2932b */71#define ra7 -9.8143291473e+00f /* 0xc11d077e */72#define sa1 1.9651271820e+01f /* 0x419d35ce */73#define sa2 1.3765776062e+02f /* 0x4309a863 */74#define sa3 4.3456588745e+02f /* 0x43d9486f */75#define sa4 6.4538726807e+02f /* 0x442158c9 */76#define sa5 4.2900814819e+02f /* 0x43d6810b */77#define sa6 1.0863500214e+02f /* 0x42d9451f */78#define sa7 6.5702495575e+00f /* 0x40d23f7c */79#define sa8 -6.0424413532e-02f /* 0xbd777f97 */80 81// Coefficients for approximation to erfc in [1/0.35, 28]82 83#define rb0 -9.8649431020e-03f /* 0xbc21a092 */84#define rb1 -7.9928326607e-01f /* 0xbf4c9dd4 */85#define rb2 -1.7757955551e+01f /* 0xc18e104b */86#define rb3 -1.6063638306e+02f /* 0xc320a2ea */87#define rb4 -6.3756646729e+02f /* 0xc41f6441 */88#define rb5 -1.0250950928e+03f /* 0xc480230b */89#define rb6 -4.8351919556e+02f /* 0xc3f1c275 */90#define sb1 3.0338060379e+01f /* 0x41f2b459 */91#define sb2 3.2579251099e+02f /* 0x43a2e571 */92#define sb3 1.5367296143e+03f /* 0x44c01759 */93#define sb4 3.1998581543e+03f /* 0x4547fdbb */94#define sb5 2.5530502930e+03f /* 0x451f90ce */95#define sb6 4.7452853394e+02f /* 0x43ed43a7 */96#define sb7 -2.2440952301e+01f /* 0xc1b38712 */97 98_CLC_OVERLOAD _CLC_DEF float __clc_erf(float x) {99 int hx = __clc_as_uint(x);100 float absx = __clc_fabs(x);101 int ix = __clc_as_uint(absx);102 103 float x2 = absx * absx;104 float t = 1.0f / x2;105 float tt = absx - 1.0f;106 t = absx < 1.25f ? tt : t;107 t = absx < 0.84375f ? x2 : t;108 109 float u, v, tu, tv;110 111 // |x| < 6112 u = __clc_mad(113 t,114 __clc_mad(115 t,116 __clc_mad(117 t, __clc_mad(t, __clc_mad(t, __clc_mad(t, rb6, rb5), rb4), rb3),118 rb2),119 rb1),120 rb0);121 v = __clc_mad(122 t,123 __clc_mad(124 t,125 __clc_mad(126 t, __clc_mad(t, __clc_mad(t, __clc_mad(t, sb7, sb6), sb5), sb4),127 sb3),128 sb2),129 sb1);130 131 tu = __clc_mad(132 t,133 __clc_mad(134 t,135 __clc_mad(136 t,137 __clc_mad(138 t,139 __clc_mad(t, __clc_mad(t, __clc_mad(t, ra7, ra6), ra5), ra4),140 ra3),141 ra2),142 ra1),143 ra0);144 tv = __clc_mad(145 t,146 __clc_mad(147 t,148 __clc_mad(149 t,150 __clc_mad(151 t,152 __clc_mad(t, __clc_mad(t, __clc_mad(t, sa8, sa7), sa6), sa5),153 sa4),154 sa3),155 sa2),156 sa1);157 u = absx < 0x1.6db6dcp+1f ? tu : u;158 v = absx < 0x1.6db6dcp+1f ? tv : v;159 160 tu = __clc_mad(161 t,162 __clc_mad(163 t,164 __clc_mad(165 t, __clc_mad(t, __clc_mad(t, __clc_mad(t, pa6, pa5), pa4), pa3),166 pa2),167 pa1),168 pa0);169 tv = __clc_mad(170 t,171 __clc_mad(t, __clc_mad(t, __clc_mad(t, __clc_mad(t, qa6, qa5), qa4), qa3),172 qa2),173 qa1);174 u = absx < 1.25f ? tu : u;175 v = absx < 1.25f ? tv : v;176 177 tu = __clc_mad(178 t, __clc_mad(t, __clc_mad(t, __clc_mad(t, pp4, pp3), pp2), pp1), pp0);179 tv = __clc_mad(180 t, __clc_mad(t, __clc_mad(t, __clc_mad(t, qq5, qq4), qq3), qq2), qq1);181 u = absx < 0.84375f ? tu : u;182 v = absx < 0.84375f ? tv : v;183 184 v = __clc_mad(t, v, 1.0f);185 float q = MATH_DIVIDE(u, v);186 187 float ret = 1.0f;188 189 // |x| < 6190 float z = __clc_as_float(ix & 0xfffff000);191 float r = __clc_exp(-z * z) * __clc_exp(__clc_mad(z - absx, z + absx, q));192 r *= 0x1.23ba94p-1f; // exp(-0.5625)193 r = 1.0f - MATH_DIVIDE(r, absx);194 ret = absx < 6.0f ? r : ret;195 196 r = erx + q;197 ret = absx < 1.25f ? r : ret;198 199 ret = __clc_as_float((hx & 0x80000000) | __clc_as_int(ret));200 201 r = __clc_mad(x, q, x);202 ret = absx < 0.84375f ? r : ret;203 204 // Prevent underflow205 r = 0.125f * __clc_mad(8.0f, x, efx8 * x);206 ret = absx < 0x1.0p-28f ? r : ret;207 208 ret = __clc_isnan(x) ? x : ret;209 210 return ret;211}212 213#ifdef cl_khr_fp64214 215#pragma OPENCL EXTENSION cl_khr_fp64 : enable216 217/*218 * ====================================================219 * Copyright (C) 1993 by Sun Microsystems, Inc. All rights reserved.220 *221 * Developed at SunPro, a Sun Microsystems, Inc. business.222 * Permission to use, copy, modify, and distribute this223 * software is freely granted, provided that this notice224 * is preserved.225 * ====================================================226 */227 228/* double erf(double x)229 * double erfc(double x)230 * x231 * 2 |\232 * erf(x) = --------- | exp(-t*t)dt233 * sqrt(pi) \|234 * 0235 *236 * erfc(x) = 1-erf(x)237 * Note that238 * erf(-x) = -erf(x)239 * erfc(-x) = 2 - erfc(x)240 *241 * Method:242 * 1. For |x| in [0, 0.84375]243 * erf(x) = x + x*R(x^2)244 * erfc(x) = 1 - erf(x) if x in [-.84375,0.25]245 * = 0.5 + ((0.5-x)-x*R) if x in [0.25,0.84375]246 * where R = P/Q where P is an odd poly of degree 8 and247 * Q is an odd poly of degree 10.248 * -57.90249 * | R - (erf(x)-x)/x | <= 2250 *251 *252 * Remark. The formula is derived by noting253 * erf(x) = (2/sqrt(pi))*(x - x^3/3 + x^5/10 - x^7/42 + ....)254 * and that255 * 2/sqrt(pi) = 1.128379167095512573896158903121545171688256 * is close to one. The interval is chosen because the fix257 * point of erf(x) is near 0.6174 (i.e., erf(x)=x when x is258 * near 0.6174), and by some experiment, 0.84375 is chosen to259 * guarantee the error is less than one ulp for erf.260 *261 * 2. For |x| in [0.84375,1.25], let s = |x| - 1, and262 * c = 0.84506291151 rounded to single (24 bits)263 * erf(x) = sign(x) * (c + P1(s)/Q1(s))264 * erfc(x) = (1-c) - P1(s)/Q1(s) if x > 0265 * 1+(c+P1(s)/Q1(s)) if x < 0266 * |P1/Q1 - (erf(|x|)-c)| <= 2**-59.06267 * Remark: here we use the taylor series expansion at x=1.268 * erf(1+s) = erf(1) + s*Poly(s)269 * = 0.845.. + P1(s)/Q1(s)270 * That is, we use rational approximation to approximate271 * erf(1+s) - (c = (single)0.84506291151)272 * Note that |P1/Q1|< 0.078 for x in [0.84375,1.25]273 * where274 * P1(s) = degree 6 poly in s275 * Q1(s) = degree 6 poly in s276 *277 * 3. For x in [1.25,1/0.35(~2.857143)],278 * erfc(x) = (1/x)*exp(-x*x-0.5625+R1/S1)279 * erf(x) = 1 - erfc(x)280 * where281 * R1(z) = degree 7 poly in z, (z=1/x^2)282 * S1(z) = degree 8 poly in z283 *284 * 4. For x in [1/0.35,28]285 * erfc(x) = (1/x)*exp(-x*x-0.5625+R2/S2) if x > 0286 * = 2.0 - (1/x)*exp(-x*x-0.5625+R2/S2) if -6<x<0287 * = 2.0 - tiny (if x <= -6)288 * erf(x) = sign(x)*(1.0 - erfc(x)) if x < 6, else289 * erf(x) = sign(x)*(1.0 - tiny)290 * where291 * R2(z) = degree 6 poly in z, (z=1/x^2)292 * S2(z) = degree 7 poly in z293 *294 * Note1:295 * To compute exp(-x*x-0.5625+R/S), let s be a single296 * precision number and s := x; then297 * -x*x = -s*s + (s-x)*(s+x)298 * exp(-x*x-0.5626+R/S) =299 * exp(-s*s-0.5625)*exp((s-x)*(s+x)+R/S);300 * Note2:301 * Here 4 and 5 make use of the asymptotic series302 * exp(-x*x)303 * erfc(x) ~ ---------- * ( 1 + Poly(1/x^2) )304 * x*sqrt(pi)305 * We use rational approximation to approximate306 * g(s)=f(1/x^2) = log(erfc(x)*x) - x*x + 0.5625307 * Here is the error bound for R1/S1 and R2/S2308 * |R1/S1 - f(x)| < 2**(-62.57)309 * |R2/S2 - f(x)| < 2**(-61.52)310 *311 * 5. For inf > x >= 28312 * erf(x) = sign(x) *(1 - tiny) (raise inexact)313 * erfc(x) = tiny*tiny (raise underflow) if x > 0314 * = 2 - tiny if x<0315 *316 * 7. Special case:317 * erf(0) = 0, erf(inf) = 1, erf(-inf) = -1,318 * erfc(0) = 1, erfc(inf) = 0, erfc(-inf) = 2,319 * erfc/erf(NaN) is NaN320 */321 322#define AU0 -9.86494292470009928597e-03323#define AU1 -7.99283237680523006574e-01324#define AU2 -1.77579549177547519889e+01325#define AU3 -1.60636384855821916062e+02326#define AU4 -6.37566443368389627722e+02327#define AU5 -1.02509513161107724954e+03328#define AU6 -4.83519191608651397019e+02329 330#define AV1 3.03380607434824582924e+01331#define AV2 3.25792512996573918826e+02332#define AV3 1.53672958608443695994e+03333#define AV4 3.19985821950859553908e+03334#define AV5 2.55305040643316442583e+03335#define AV6 4.74528541206955367215e+02336#define AV7 -2.24409524465858183362e+01337 338#define BU0 -9.86494403484714822705e-03339#define BU1 -6.93858572707181764372e-01340#define BU2 -1.05586262253232909814e+01341#define BU3 -6.23753324503260060396e+01342#define BU4 -1.62396669462573470355e+02343#define BU5 -1.84605092906711035994e+02344#define BU6 -8.12874355063065934246e+01345#define BU7 -9.81432934416914548592e+00346 347#define BV1 1.96512716674392571292e+01348#define BV2 1.37657754143519042600e+02349#define BV3 4.34565877475229228821e+02350#define BV4 6.45387271733267880336e+02351#define BV5 4.29008140027567833386e+02352#define BV6 1.08635005541779435134e+02353#define BV7 6.57024977031928170135e+00354#define BV8 -6.04244152148580987438e-02355 356#define CU0 -2.36211856075265944077e-03357#define CU1 4.14856118683748331666e-01358#define CU2 -3.72207876035701323847e-01359#define CU3 3.18346619901161753674e-01360#define CU4 -1.10894694282396677476e-01361#define CU5 3.54783043256182359371e-02362#define CU6 -2.16637559486879084300e-03363 364#define CV1 1.06420880400844228286e-01365#define CV2 5.40397917702171048937e-01366#define CV3 7.18286544141962662868e-02367#define CV4 1.26171219808761642112e-01368#define CV5 1.36370839120290507362e-02369#define CV6 1.19844998467991074170e-02370 371#define DU0 1.28379167095512558561e-01372#define DU1 -3.25042107247001499370e-01373#define DU2 -2.84817495755985104766e-02374#define DU3 -5.77027029648944159157e-03375#define DU4 -2.37630166566501626084e-05376 377#define DV1 3.97917223959155352819e-01378#define DV2 6.50222499887672944485e-02379#define DV3 5.08130628187576562776e-03380#define DV4 1.32494738004321644526e-04381#define DV5 -3.96022827877536812320e-06382 383_CLC_OVERLOAD _CLC_DEF double __clc_erf(double y) {384 double x = __clc_fabs(y);385 double x2 = x * x;386 double xm1 = x - 1.0;387 388 // Poly variable389 double t = 1.0 / x2;390 t = x < 1.25 ? xm1 : t;391 t = x < 0.84375 ? x2 : t;392 393 double u, ut, v, vt;394 395 // Evaluate rational poly396 // XXX We need to see of we can grab 16 coefficents from a table397 // faster than evaluating 3 of the poly pairs398 // if (x < 6.0)399 u = __clc_fma(400 t,401 __clc_fma(402 t,403 __clc_fma(404 t, __clc_fma(t, __clc_fma(t, __clc_fma(t, AU6, AU5), AU4), AU3),405 AU2),406 AU1),407 AU0);408 v = __clc_fma(409 t,410 __clc_fma(411 t,412 __clc_fma(413 t, __clc_fma(t, __clc_fma(t, __clc_fma(t, AV7, AV6), AV5), AV4),414 AV3),415 AV2),416 AV1);417 418 ut = __clc_fma(419 t,420 __clc_fma(421 t,422 __clc_fma(423 t,424 __clc_fma(425 t,426 __clc_fma(t, __clc_fma(t, __clc_fma(t, BU7, BU6), BU5), BU4),427 BU3),428 BU2),429 BU1),430 BU0);431 vt = __clc_fma(432 t,433 __clc_fma(434 t,435 __clc_fma(436 t,437 __clc_fma(438 t,439 __clc_fma(t, __clc_fma(t, __clc_fma(t, BV8, BV7), BV6), BV5),440 BV4),441 BV3),442 BV2),443 BV1);444 u = x < 0x1.6db6ep+1 ? ut : u;445 v = x < 0x1.6db6ep+1 ? vt : v;446 447 ut = __clc_fma(448 t,449 __clc_fma(450 t,451 __clc_fma(452 t, __clc_fma(t, __clc_fma(t, __clc_fma(t, CU6, CU5), CU4), CU3),453 CU2),454 CU1),455 CU0);456 vt = __clc_fma(457 t,458 __clc_fma(t, __clc_fma(t, __clc_fma(t, __clc_fma(t, CV6, CV5), CV4), CV3),459 CV2),460 CV1);461 u = x < 1.25 ? ut : u;462 v = x < 1.25 ? vt : v;463 464 ut = __clc_fma(465 t, __clc_fma(t, __clc_fma(t, __clc_fma(t, DU4, DU3), DU2), DU1), DU0);466 vt = __clc_fma(467 t, __clc_fma(t, __clc_fma(t, __clc_fma(t, DV5, DV4), DV3), DV2), DV1);468 u = x < 0.84375 ? ut : u;469 v = x < 0.84375 ? vt : v;470 471 v = __clc_fma(t, v, 1.0);472 473 // Compute rational approximation474 double q = u / v;475 476 // Compute results477 double z = __clc_as_double(__clc_as_long(x) & 0xffffffff00000000L);478 double r = __clc_exp(-z * z - 0.5625) * __clc_exp((z - x) * (z + x) + q);479 r = 1.0 - r / x;480 481 double ret = x < 6.0 ? r : 1.0;482 483 r = 8.45062911510467529297e-01 + q;484 ret = x < 1.25 ? r : ret;485 486 q = x < 0x1.0p-28 ? 1.28379167095512586316e-01 : q;487 488 r = __clc_fma(x, q, x);489 ret = x < 0.84375 ? r : ret;490 491 ret = __clc_isnan(x) ? x : ret;492 493 return y < 0.0 ? -ret : ret;494}495 496#endif497 498#ifdef cl_khr_fp16499 500#pragma OPENCL EXTENSION cl_khr_fp16 : enable501 502// Forward the half version of this builtin onto the float one503_CLC_OVERLOAD _CLC_DEF half __clc_erf(half x) {504 return (half)__clc_erf((float)x);505}506 507#endif508 509#define __CLC_FUNCTION __clc_erf510#define __CLC_BODY <clc/shared/unary_def_scalarize_loop.inc>511#include <clc/math/gentype.inc>512