406 lines · plain
1//===----------------------------------------------------------------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// Computes pow using log and exp10//11// x^y = exp(y * log(x))12//13// We take care not to lose precision in the intermediate steps.14//15// When computing log, calculate it in splits:16//17// r = f * (p_invead + p_inv_tail)18// r = rh + rt19//20// Calculate log polynomial using r, in end addition, do:21//22// poly = poly + ((rh-r) + rt)23//24// lth = -r25// ltt = ((xexp * log2_t) - poly) + logT26// lt = lth + ltt27//28// lh = (xexp * log2_h) + logH29// l = lh + lt30//31// Calculate final log answer as gh and gt:32//33// gh = l & higher-half bits34// gt = (((ltt - (lt - lth)) + ((lh - l) + lt)) + (l - gh))35//36// yh = y & higher-half bits37// yt = y - yh38//39// Before entering computation of exp:40//41// vs = ((yt*gt + yt*gh) + yh*gt)42// v = vs + yh*gh43// vt = ((yh*gh - v) + vs)44//45// In calculation of exp, add vt to r that is used for poly.46//47// At the end of exp, do:48//49// ((((expT * poly) + expT) + expH*poly) + expH)50//51//===----------------------------------------------------------------------===//52 53#if __CLC_FPSIZE == 3254 55_CLC_DEF _CLC_OVERLOAD __CLC_GENTYPE __clc_rootn(__CLC_GENTYPE x,56 __CLC_INTN ny) {57 __CLC_GENTYPE y = MATH_RECIP(__CLC_CONVERT_GENTYPE(ny));58 59 __CLC_INTN ix = __CLC_AS_INTN(x);60 __CLC_INTN ax = ix & EXSIGNBIT_SP32;61 __CLC_INTN xpos = ix == ax;62 63 __CLC_INTN iy = __CLC_AS_INTN(y);64 __CLC_INTN ay = iy & EXSIGNBIT_SP32;65 __CLC_INTN ypos = iy == ay;66 67 // Extra precise log calculation68 // First handle case that x is close to 169 __CLC_GENTYPE r = 1.0f - __CLC_AS_GENTYPE(ax);70 __CLC_INTN near1 = __clc_fabs(r) < 0x1.0p-4f;71 __CLC_GENTYPE r2 = r * r;72 73 // Coefficients are just 1/3, 1/4, 1/5 and 1/674 __CLC_GENTYPE poly = __clc_mad(75 r,76 __clc_mad(r,77 __clc_mad(r, __clc_mad(r, 0x1.24924ap-3f, 0x1.555556p-3f),78 0x1.99999ap-3f),79 0x1.000000p-2f),80 0x1.555556p-2f);81 82 poly *= r2 * r;83 84 __CLC_GENTYPE lth_near1 = -r2 * 0.5f;85 __CLC_GENTYPE ltt_near1 = -poly;86 __CLC_GENTYPE lt_near1 = lth_near1 + ltt_near1;87 __CLC_GENTYPE lh_near1 = -r;88 __CLC_GENTYPE l_near1 = lh_near1 + lt_near1;89 90 // Computations for x not near 191 __CLC_INTN m = __CLC_CONVERT_INTN(ax >> EXPSHIFTBITS_SP32) - EXPBIAS_SP32;92 __CLC_GENTYPE mf = __CLC_CONVERT_GENTYPE(m);93 __CLC_INTN ixs = __CLC_AS_INTN(__CLC_AS_GENTYPE(ax | 0x3f800000) - 1.0f);94 __CLC_GENTYPE mfs = __CLC_CONVERT_GENTYPE((ixs >> EXPSHIFTBITS_SP32) - 253);95 __CLC_INTN c = m == -127;96 __CLC_INTN ixn = c ? ixs : ax;97 __CLC_GENTYPE mfn = c ? mfs : mf;98 99 __CLC_INTN indx = (ixn & 0x007f0000) + ((ixn & 0x00008000) << 1);100 101 // F - Y102 __CLC_GENTYPE f = __CLC_AS_GENTYPE(0x3f000000 | indx) -103 __CLC_AS_GENTYPE(0x3f000000 | (ixn & MANTBITS_SP32));104 105 indx = indx >> 16;106 __CLC_GENTYPE rh = f * __CLC_USE_TABLE(log_inv_tbl_ep_head, indx);107 __CLC_GENTYPE rt = f * __CLC_USE_TABLE(log_inv_tbl_ep_tail, indx);108 ;109 r = rh + rt;110 111 poly = __clc_mad(r, __clc_mad(r, 0x1.0p-2f, 0x1.555556p-2f), 0x1.0p-1f) *112 (r * r);113 poly += (rh - r) + rt;114 115 const __CLC_GENTYPE LOG2_HEAD = 0x1.62e000p-1f; // 0.693115234116 const __CLC_GENTYPE LOG2_TAIL = 0x1.0bfbe8p-15f; // 0.0000319461833117 __CLC_GENTYPE lth = -r;118 __CLC_GENTYPE ltt =119 __clc_mad(mfn, LOG2_TAIL, -poly) + __CLC_USE_TABLE(loge_tbl_hi, indx);120 __CLC_GENTYPE lt = lth + ltt;121 __CLC_GENTYPE lh =122 __clc_mad(mfn, LOG2_HEAD, __CLC_USE_TABLE(loge_tbl_lo, indx));123 __CLC_GENTYPE l = lh + lt;124 125 // Select near 1 or not126 lth = near1 ? lth_near1 : lth;127 ltt = near1 ? ltt_near1 : ltt;128 lt = near1 ? lt_near1 : lt;129 lh = near1 ? lh_near1 : lh;130 l = near1 ? l_near1 : l;131 132 __CLC_GENTYPE gh = __CLC_AS_GENTYPE(__CLC_AS_UINTN(l) & 0xfffff000);133 __CLC_GENTYPE gt = ((ltt - (lt - lth)) + ((lh - l) + lt)) + (l - gh);134 135 __CLC_GENTYPE yh = __CLC_AS_GENTYPE(__CLC_AS_UINTN(iy) & 0xfffff000);136 137 __CLC_GENTYPE fny = __CLC_CONVERT_GENTYPE(ny);138 __CLC_GENTYPE fnyh = __CLC_AS_GENTYPE(__CLC_AS_UINTN(fny) & 0xfffff000);139 __CLC_GENTYPE fnyt = __CLC_CONVERT_GENTYPE(ny - __CLC_CONVERT_INTN(fnyh));140 __CLC_GENTYPE yt =141 MATH_DIVIDE(__clc_mad(-fnyt, yh, __clc_mad(-fnyh, yh, 1.0f)), fny);142 143 __CLC_GENTYPE ylogx_s = __clc_mad(gt, yh, __clc_mad(gh, yt, yt * gt));144 __CLC_GENTYPE ylogx = __clc_mad(yh, gh, ylogx_s);145 __CLC_GENTYPE ylogx_t = __clc_mad(yh, gh, -ylogx) + ylogx_s;146 147 // Extra precise exp of ylogx148 const __CLC_GENTYPE R_64_BY_LOG2 =149 0x1.715476p+6f; // 64/log2 : 92.332482616893657150 __CLC_INTN n = __CLC_CONVERT_INTN(ylogx * R_64_BY_LOG2);151 __CLC_GENTYPE nf = __CLC_CONVERT_GENTYPE(n);152 153 __CLC_INTN j = n & 0x3f;154 m = n >> 6;155 __CLC_INTN m2 = m << EXPSHIFTBITS_SP32;156 157 // log2/64 lead: 0.0108032227158 const __CLC_GENTYPE R_LOG2_BY_64_LD = 0x1.620000p-7f;159 // log2/64 tail: 0.0000272020388160 const __CLC_GENTYPE R_LOG2_BY_64_TL = 0x1.c85fdep-16f;161 r = __clc_mad(nf, -R_LOG2_BY_64_TL, __clc_mad(nf, -R_LOG2_BY_64_LD, ylogx)) +162 ylogx_t;163 164 // Truncated Taylor series for e^r165 poly = __clc_mad(__clc_mad(__clc_mad(r, 0x1.555556p-5f, 0x1.555556p-3f), r,166 0x1.000000p-1f),167 r * r, r);168 169 __CLC_GENTYPE exph = __CLC_USE_TABLE(exp_tbl_ep_head, j);170 __CLC_GENTYPE expt = __CLC_USE_TABLE(exp_tbl_ep_tail, j);171 172 __CLC_GENTYPE expylogx =173 __clc_mad(exph, poly, __clc_mad(expt, poly, expt)) + exph;174 __CLC_GENTYPE sexpylogx =175 __clc_fp32_subnormals_supported()176 ? expylogx * __CLC_AS_GENTYPE((__CLC_INTN)0x1 << (m + 149))177 : 0.0f;178 179 __CLC_GENTYPE texpylogx = __CLC_AS_GENTYPE(__CLC_AS_INTN(expylogx) + m2);180 expylogx = m < -125 ? sexpylogx : texpylogx;181 182 // Result is +-Inf if (ylogx + ylogx_t) > 128*log2183 expylogx = ((ylogx > 0x1.62e430p+6f) |184 (ylogx == 0x1.62e430p+6f & ylogx_t > -0x1.05c610p-22f))185 ? __CLC_AS_GENTYPE((__CLC_UINTN)PINFBITPATT_SP32)186 : expylogx;187 188 // Result is 0 if ylogx < -149*log2189 expylogx = ylogx < -0x1.9d1da0p+6f ? 0.0f : expylogx;190 191 // Classify y:192 // inty = 0 means not an integer.193 // inty = 1 means odd integer.194 // inty = 2 means even integer.195 196 __CLC_INTN inty = 2 - (ny & 1);197 198 __CLC_GENTYPE signval =199 __CLC_AS_GENTYPE((__CLC_AS_UINTN(expylogx) ^ SIGNBIT_SP32));200 expylogx = ((inty == 1) & !xpos) ? signval : expylogx;201 __CLC_INTN ret = __CLC_AS_INTN(expylogx);202 203 // Corner case handling204 __CLC_BIT_INTN x_is_ninf = ix == (__CLC_INTN)NINFBITPATT_SP32;205 __CLC_BIT_INTN x_is_pinf = ix == (__CLC_INTN)PINFBITPATT_SP32;206 207 ret = (!xpos & (inty == 2)) ? __CLC_AS_INTN(__CLC_GENTYPE_NAN) : ret;208 __CLC_INTN xinf =209 xpos ? (__CLC_INTN)PINFBITPATT_SP32 : (__CLC_INTN)NINFBITPATT_SP32;210 ret = ((ax == 0) & !ypos & (inty == 1)) ? xinf : ret;211 ret = ((ax == 0) & !ypos & (inty == 2)) ? PINFBITPATT_SP32 : ret;212 ret = ((ax == 0) & ypos & (inty == 2)) ? 0 : ret;213 __CLC_INTN xzero = xpos ? 0 : (__CLC_INTN)0x80000000;214 ret = ((ax == 0) & ypos & (inty == 1)) ? xzero : ret;215 ret = (x_is_ninf & ypos & (inty == 1)) ? (__CLC_INTN)NINFBITPATT_SP32 : ret;216 ret = (x_is_ninf & !ypos & (inty == 1)) ? (__CLC_INTN)0x80000000 : ret;217 ret = (x_is_pinf & !ypos) ? 0 : ret;218 ret = (x_is_pinf & ypos) ? PINFBITPATT_SP32 : ret;219 ret = ax > PINFBITPATT_SP32 ? ix : ret;220 ret = ny == 0 ? __CLC_AS_INTN(__CLC_GENTYPE_NAN) : ret;221 222 return __CLC_AS_GENTYPE(ret);223}224 225#elif __CLC_FPSIZE == 64226 227_CLC_DEF _CLC_OVERLOAD __CLC_GENTYPE __clc_rootn(__CLC_GENTYPE x,228 __CLC_INTN ny) {229 const __CLC_GENTYPE real_log2_tail = 5.76999904754328540596e-08;230 const __CLC_GENTYPE real_log2_lead = 6.93147122859954833984e-01;231 232 __CLC_GENTYPE dny = __CLC_CONVERT_GENTYPE(ny);233 __CLC_GENTYPE y = 1.0 / dny;234 235 __CLC_LONGN ux = __CLC_AS_LONGN(x);236 __CLC_LONGN ax = __CLC_AS_LONGN(__clc_fabs(x));237 __CLC_BIT_INTN xpos = ax == ux;238 239 __CLC_LONGN uy = __CLC_AS_LONGN(y);240 __CLC_LONGN ay = __CLC_AS_LONGN(__clc_fabs(y));241 __CLC_BIT_INTN ypos = ay == uy;242 243 // Extended precision log244 __CLC_GENTYPE v, vt;245 {246 __CLC_INTN exp = __CLC_CONVERT_INTN(ax >> 52) - 1023;247 __CLC_INTN mask_exp_1023 = exp == -1023;248 __CLC_GENTYPE xexp = __CLC_CONVERT_GENTYPE(exp);249 __CLC_LONGN mantissa = ax & 0x000FFFFFFFFFFFFFL;250 251 __CLC_LONGN temp_ux =252 __CLC_AS_LONGN(__CLC_AS_GENTYPE(0x3ff0000000000000L | mantissa) - 1.0);253 exp = __CLC_CONVERT_INTN((temp_ux & 0x7FF0000000000000L) >> 52) - 2045;254 __CLC_GENTYPE xexp1 = __CLC_CONVERT_GENTYPE(exp);255 __CLC_LONGN mantissa1 = temp_ux & 0x000FFFFFFFFFFFFFL;256 257 xexp = __CLC_CONVERT_LONGN(mask_exp_1023) ? xexp1 : xexp;258 mantissa = __CLC_CONVERT_LONGN(mask_exp_1023) ? mantissa1 : mantissa;259 260 __CLC_LONGN rax = (mantissa & 0x000ff00000000000) +261 ((mantissa & 0x0000080000000000) << 1);262 __CLC_INTN index = __CLC_CONVERT_INTN(rax >> 44);263 264 __CLC_GENTYPE F = __CLC_AS_GENTYPE(rax | 0x3FE0000000000000L);265 __CLC_GENTYPE Y = __CLC_AS_GENTYPE(mantissa | 0x3FE0000000000000L);266 __CLC_GENTYPE f = F - Y;267 __CLC_GENTYPE log_h = __CLC_USE_TABLE(log_f_inv_tbl_head, index);268 __CLC_GENTYPE log_t = __CLC_USE_TABLE(log_f_inv_tbl_tail, index);269 __CLC_GENTYPE f_inv = (log_h + log_t) * f;270 __CLC_GENTYPE r1 =271 __CLC_AS_GENTYPE(__CLC_AS_ULONGN(f_inv) & 0xfffffffff8000000L);272 __CLC_GENTYPE r2 = __clc_fma(-F, r1, f) * (log_h + log_t);273 __CLC_GENTYPE r = r1 + r2;274 275 __CLC_GENTYPE poly = __clc_fma(276 r,277 __clc_fma(r,278 __clc_fma(r, __clc_fma(r, 1.0 / 7.0, 1.0 / 6.0), 1.0 / 5.0),279 1.0 / 4.0),280 1.0 / 3.0);281 poly = poly * r * r * r;282 283 __CLC_GENTYPE hr1r1 = 0.5 * r1 * r1;284 __CLC_GENTYPE poly0h = r1 + hr1r1;285 __CLC_GENTYPE poly0t = r1 - poly0h + hr1r1;286 poly = __clc_fma(r1, r2, __clc_fma(0.5 * r2, r2, poly)) + r2 + poly0t;287 288 log_h = __CLC_USE_TABLE(powlog_tbl_head, index);289 log_t = __CLC_USE_TABLE(powlog_tbl_tail, index);290 291 __CLC_GENTYPE resT_t = __clc_fma(xexp, real_log2_tail, +log_t) - poly;292 __CLC_GENTYPE resT = resT_t - poly0h;293 __CLC_GENTYPE resH = __clc_fma(xexp, real_log2_lead, log_h);294 __CLC_GENTYPE resT_h = poly0h;295 296 __CLC_GENTYPE H = resT + resH;297 __CLC_GENTYPE H_h =298 __CLC_AS_GENTYPE(__CLC_AS_ULONGN(H) & 0xfffffffff8000000L);299 __CLC_GENTYPE T =300 (resH - H + resT) + (resT_t - (resT + resT_h)) + (H - H_h);301 H = H_h;302 303 __CLC_GENTYPE y_head =304 __CLC_AS_GENTYPE(__CLC_AS_ULONGN(uy) & 0xfffffffff8000000L);305 __CLC_GENTYPE y_tail = y - y_head;306 307 __CLC_GENTYPE fnyh =308 __CLC_AS_GENTYPE(__CLC_AS_ULONGN(dny) & 0xfffffffffff00000);309 __CLC_GENTYPE fnyt = __CLC_CONVERT_GENTYPE(ny - __CLC_CONVERT_INTN(fnyh));310 y_tail = __clc_fma(-fnyt, y_head, __clc_fma(-fnyh, y_head, 1.0)) / dny;311 312 __CLC_GENTYPE temp = __clc_fma(y_tail, H, __clc_fma(y_head, T, y_tail * T));313 v = __clc_fma(y_head, H, temp);314 vt = __clc_fma(y_head, H, -v) + temp;315 }316 317 // Now calculate exp of (v,vt)318 319 __CLC_GENTYPE expv;320 {321 const __CLC_GENTYPE max_exp_arg = 709.782712893384;322 const __CLC_GENTYPE min_exp_arg = -745.1332191019411;323 const __CLC_GENTYPE sixtyfour_by_lnof2 = 92.33248261689366;324 const __CLC_GENTYPE lnof2_by_64_head = 0.010830424260348081;325 const __CLC_GENTYPE lnof2_by_64_tail = -4.359010638708991e-10;326 327 // If v is so large that we need to return INFINITY, or so small that we328 // need to return 0, set v to known values that will produce that result. Do329 // not try to continue the computation with the original v and patch it up330 // afterwards because v may be so large that temp is out of range of int, in331 // which case that conversion, and a value based on that conversion being332 // passed to __clc_ldexp, results in undefined behavior.333 v = v > max_exp_arg ? 1000.0 : v;334 v = v < min_exp_arg ? -1000.0 : v;335 336 __CLC_GENTYPE temp = v * sixtyfour_by_lnof2;337 __CLC_INTN n = __CLC_CONVERT_INTN(temp);338 __CLC_GENTYPE dn = __CLC_CONVERT_GENTYPE(n);339 __CLC_INTN j = n & 0x0000003f;340 __CLC_INTN m = n >> 6;341 342 __CLC_GENTYPE f1 = __CLC_USE_TABLE(two_to_jby64_ep_tbl_head, j);343 __CLC_GENTYPE f2 = __CLC_USE_TABLE(two_to_jby64_ep_tbl_tail, j);344 __CLC_GENTYPE f = f1 + f2;345 346 __CLC_GENTYPE r1 = __clc_fma(dn, -lnof2_by_64_head, v);347 __CLC_GENTYPE r2 = dn * lnof2_by_64_tail;348 __CLC_GENTYPE r = (r1 + r2) + vt;349 350 __CLC_GENTYPE q =351 __clc_fma(r,352 __clc_fma(r,353 __clc_fma(r,354 __clc_fma(r, 1.38889490863777199667e-03,355 8.33336798434219616221e-03),356 4.16666666662260795726e-02),357 1.66666666665260878863e-01),358 5.00000000000000008883e-01);359 q = __clc_fma(r * r, q, r);360 361 expv = __clc_fma(f, q, f2) + f1;362 expv = __clc_ldexp(expv, m);363 }364 365 // See whether y is an integer.366 // inty = 0 means not an integer.367 // inty = 1 means odd integer.368 // inty = 2 means even integer.369 370 __CLC_LONGN inty = __CLC_CONVERT_LONGN(2 - (ny & 1));371 372 expv *= ((inty == 1) & !xpos) ? -1.0 : 1.0;373 374 __CLC_LONGN ret = __CLC_AS_LONGN(expv);375 376 // Now all the edge cases377 __CLC_BIT_INTN x_is_ninf = ux == (__CLC_LONGN)NINFBITPATT_DP64;378 __CLC_BIT_INTN x_is_pinf = ux == (__CLC_LONGN)PINFBITPATT_DP64;379 ret = (!xpos & (inty == 2)) ? __CLC_AS_LONGN(__CLC_GENTYPE_NAN) : ret;380 __CLC_LONGN xinf =381 xpos ? (__CLC_LONGN)PINFBITPATT_DP64 : (__CLC_LONGN)NINFBITPATT_DP64;382 ret = ((ax == 0L) & !ypos & (inty == 1)) ? xinf : ret;383 ret =384 ((ax == 0L) & !ypos & (inty == 2)) ? (__CLC_LONGN)PINFBITPATT_DP64 : ret;385 ret = ((ax == 0L) & ypos & (inty == 2)) ? 0L : ret;386 __CLC_LONGN xzero = xpos ? 0L : (__CLC_LONGN)0x8000000000000000L;387 ret = ((ax == 0L) & ypos & (inty == 1)) ? xzero : ret;388 ret = (x_is_ninf & ypos & (inty == 1)) ? (__CLC_LONGN)NINFBITPATT_DP64 : ret;389 ret = (x_is_ninf & !ypos & (inty == 1)) ? (__CLC_LONGN)0x8000000000000000L390 : ret;391 ret = (x_is_pinf & !ypos) ? 0L : ret;392 ret = (x_is_pinf & ypos) ? (__CLC_LONGN)PINFBITPATT_DP64 : ret;393 ret = ax > (__CLC_LONGN)PINFBITPATT_DP64 ? ux : ret;394 ret = __CLC_CONVERT_LONGN(ny == 0) ? __CLC_AS_LONGN(__CLC_GENTYPE_NAN) : ret;395 return __CLC_AS_GENTYPE(ret);396}397 398#elif __CLC_FPSIZE == 16399 400_CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE __clc_rootn(__CLC_GENTYPE x,401 __CLC_INTN y) {402 return __CLC_CONVERT_GENTYPE(__clc_rootn(__CLC_CONVERT_FLOATN(x), y));403}404 405#endif406