411 lines · c
1//===-- lib/fp_lib.h - Floating-point utilities -------------------*- C -*-===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// This file is a configuration header for soft-float routines in compiler-rt.10// This file does not provide any part of the compiler-rt interface, but defines11// many useful constants and utility routines that are used in the12// implementation of the soft-float routines in compiler-rt.13//14// Assumes that float, double and long double correspond to the IEEE-75415// binary32, binary64 and binary 128 types, respectively, and that integer16// endianness matches floating point endianness on the target platform.17//18//===----------------------------------------------------------------------===//19 20#ifndef FP_LIB_HEADER21#define FP_LIB_HEADER22 23#include "int_lib.h"24#include "int_math.h"25#include "int_types.h"26#include <limits.h>27#include <stdbool.h>28#include <stdint.h>29 30#if defined SINGLE_PRECISION31 32typedef uint16_t half_rep_t;33typedef uint32_t rep_t;34typedef uint64_t twice_rep_t;35typedef int32_t srep_t;36typedef float fp_t;37#define HALF_REP_C UINT16_C38#define REP_C UINT32_C39#define significandBits 2340 41static __inline int rep_clz(rep_t a) { return clzsi(a); }42 43// 32x32 --> 64 bit multiply44static __inline void wideMultiply(rep_t a, rep_t b, rep_t *hi, rep_t *lo) {45 const uint64_t product = (uint64_t)a * b;46 *hi = (rep_t)(product >> 32);47 *lo = (rep_t)product;48}49COMPILER_RT_ABI fp_t __addsf3(fp_t a, fp_t b);50 51#elif defined DOUBLE_PRECISION52 53typedef uint32_t half_rep_t;54typedef uint64_t rep_t;55typedef int64_t srep_t;56typedef double fp_t;57#define HALF_REP_C UINT32_C58#define REP_C UINT64_C59#define significandBits 5260 61static inline int rep_clz(rep_t a) { return __builtin_clzll(a); }62 63#define loWord(a) (a & 0xffffffffU)64#define hiWord(a) (a >> 32)65 66// 64x64 -> 128 wide multiply for platforms that don't have such an operation;67// many 64-bit platforms have this operation, but they tend to have hardware68// floating-point, so we don't bother with a special case for them here.69static __inline void wideMultiply(rep_t a, rep_t b, rep_t *hi, rep_t *lo) {70 // Each of the component 32x32 -> 64 products71 const uint64_t plolo = loWord(a) * loWord(b);72 const uint64_t plohi = loWord(a) * hiWord(b);73 const uint64_t philo = hiWord(a) * loWord(b);74 const uint64_t phihi = hiWord(a) * hiWord(b);75 // Sum terms that contribute to lo in a way that allows us to get the carry76 const uint64_t r0 = loWord(plolo);77 const uint64_t r1 = hiWord(plolo) + loWord(plohi) + loWord(philo);78 *lo = r0 + (r1 << 32);79 // Sum terms contributing to hi with the carry from lo80 *hi = hiWord(plohi) + hiWord(philo) + hiWord(r1) + phihi;81}82#undef loWord83#undef hiWord84 85COMPILER_RT_ABI fp_t __adddf3(fp_t a, fp_t b);86 87#elif defined QUAD_PRECISION88#if defined(CRT_HAS_F128) && defined(CRT_HAS_128BIT)89typedef uint64_t half_rep_t;90typedef __uint128_t rep_t;91typedef __int128_t srep_t;92typedef tf_float fp_t;93#define HALF_REP_C UINT64_C94#define REP_C (__uint128_t)95#if defined(CRT_HAS_IEEE_TF)96// Note: Since there is no explicit way to tell compiler the constant is a97// 128-bit integer, we let the constant be casted to 128-bit integer98#define significandBits 11299#define TF_MANT_DIG (significandBits + 1)100 101static __inline int rep_clz(rep_t a) {102 const union {103 __uint128_t ll;104#if _YUGA_BIG_ENDIAN105 struct {106 uint64_t high, low;107 } s;108#else109 struct {110 uint64_t low, high;111 } s;112#endif113 } uu = {.ll = a};114 115 uint64_t word;116 uint64_t add;117 118 if (uu.s.high) {119 word = uu.s.high;120 add = 0;121 } else {122 word = uu.s.low;123 add = 64;124 }125 return __builtin_clzll(word) + add;126}127 128#define Word_LoMask UINT64_C(0x00000000ffffffff)129#define Word_HiMask UINT64_C(0xffffffff00000000)130#define Word_FullMask UINT64_C(0xffffffffffffffff)131#define Word_1(a) (uint64_t)((a >> 96) & Word_LoMask)132#define Word_2(a) (uint64_t)((a >> 64) & Word_LoMask)133#define Word_3(a) (uint64_t)((a >> 32) & Word_LoMask)134#define Word_4(a) (uint64_t)(a & Word_LoMask)135 136// 128x128 -> 256 wide multiply for platforms that don't have such an operation;137// many 64-bit platforms have this operation, but they tend to have hardware138// floating-point, so we don't bother with a special case for them here.139static __inline void wideMultiply(rep_t a, rep_t b, rep_t *hi, rep_t *lo) {140 141 const uint64_t product11 = Word_1(a) * Word_1(b);142 const uint64_t product12 = Word_1(a) * Word_2(b);143 const uint64_t product13 = Word_1(a) * Word_3(b);144 const uint64_t product14 = Word_1(a) * Word_4(b);145 const uint64_t product21 = Word_2(a) * Word_1(b);146 const uint64_t product22 = Word_2(a) * Word_2(b);147 const uint64_t product23 = Word_2(a) * Word_3(b);148 const uint64_t product24 = Word_2(a) * Word_4(b);149 const uint64_t product31 = Word_3(a) * Word_1(b);150 const uint64_t product32 = Word_3(a) * Word_2(b);151 const uint64_t product33 = Word_3(a) * Word_3(b);152 const uint64_t product34 = Word_3(a) * Word_4(b);153 const uint64_t product41 = Word_4(a) * Word_1(b);154 const uint64_t product42 = Word_4(a) * Word_2(b);155 const uint64_t product43 = Word_4(a) * Word_3(b);156 const uint64_t product44 = Word_4(a) * Word_4(b);157 158 const __uint128_t sum0 = (__uint128_t)product44;159 const __uint128_t sum1 = (__uint128_t)product34 + (__uint128_t)product43;160 const __uint128_t sum2 =161 (__uint128_t)product24 + (__uint128_t)product33 + (__uint128_t)product42;162 const __uint128_t sum3 = (__uint128_t)product14 + (__uint128_t)product23 +163 (__uint128_t)product32 + (__uint128_t)product41;164 const __uint128_t sum4 =165 (__uint128_t)product13 + (__uint128_t)product22 + (__uint128_t)product31;166 const __uint128_t sum5 = (__uint128_t)product12 + (__uint128_t)product21;167 const __uint128_t sum6 = (__uint128_t)product11;168 169 const __uint128_t r0 = (sum0 & Word_FullMask) + ((sum1 & Word_LoMask) << 32);170 const __uint128_t r1 = (sum0 >> 64) + ((sum1 >> 32) & Word_FullMask) +171 (sum2 & Word_FullMask) + ((sum3 << 32) & Word_HiMask);172 173 *lo = r0 + (r1 << 64);174 // The addition above can overflow, in which case `*lo` will be less than175 // `r0`. Carry any overflow into `hi`.176 const bool carry = *lo < r0;177 *hi = (r1 >> 64) + (sum1 >> 96) + (sum2 >> 64) + (sum3 >> 32) + sum4 +178 (sum5 << 32) + (sum6 << 64) + carry;179}180#undef Word_1181#undef Word_2182#undef Word_3183#undef Word_4184#undef Word_HiMask185#undef Word_LoMask186#undef Word_FullMask187#endif // defined(CRT_HAS_IEEE_TF)188#else189typedef long double fp_t;190#endif // defined(CRT_HAS_F128) && defined(CRT_HAS_128BIT)191#else192#error SINGLE_PRECISION, DOUBLE_PRECISION or QUAD_PRECISION must be defined.193#endif194 195#if defined(SINGLE_PRECISION) || defined(DOUBLE_PRECISION) || \196 (defined(QUAD_PRECISION) && defined(CRT_HAS_TF_MODE))197#define typeWidth (sizeof(rep_t) * CHAR_BIT)198 199static __inline rep_t toRep(fp_t x) {200 const union {201 fp_t f;202 rep_t i;203 } rep = {.f = x};204 return rep.i;205}206 207static __inline fp_t fromRep(rep_t x) {208 const union {209 fp_t f;210 rep_t i;211 } rep = {.i = x};212 return rep.f;213}214 215#if !defined(QUAD_PRECISION) || defined(CRT_HAS_IEEE_TF)216#define exponentBits (typeWidth - significandBits - 1)217#define maxExponent ((1 << exponentBits) - 1)218#define exponentBias (maxExponent >> 1)219 220#define implicitBit (REP_C(1) << significandBits)221#define significandMask (implicitBit - 1U)222#define signBit (REP_C(1) << (significandBits + exponentBits))223#define absMask (signBit - 1U)224#define exponentMask (absMask ^ significandMask)225#define oneRep ((rep_t)exponentBias << significandBits)226#define infRep exponentMask227#define quietBit (implicitBit >> 1)228#define qnanRep (exponentMask | quietBit)229 230static __inline int normalize(rep_t *significand) {231 const int shift = rep_clz(*significand) - rep_clz(implicitBit);232 *significand <<= shift;233 return 1 - shift;234}235 236static __inline void wideLeftShift(rep_t *hi, rep_t *lo, unsigned int count) {237 *hi = *hi << count | *lo >> (typeWidth - count);238 *lo = *lo << count;239}240 241static __inline void wideRightShiftWithSticky(rep_t *hi, rep_t *lo,242 unsigned int count) {243 if (count < typeWidth) {244 const bool sticky = (*lo << (typeWidth - count)) != 0;245 *lo = *hi << (typeWidth - count) | *lo >> count | sticky;246 *hi = *hi >> count;247 } else if (count < 2 * typeWidth) {248 const bool sticky = *hi << (2 * typeWidth - count) | *lo;249 *lo = *hi >> (count - typeWidth) | sticky;250 *hi = 0;251 } else {252 const bool sticky = *hi | *lo;253 *lo = sticky;254 *hi = 0;255 }256}257 258// Implements logb methods (logb, logbf, logbl) for IEEE-754. This avoids259// pulling in a libm dependency from compiler-rt, but is not meant to replace260// it (i.e. code calling logb() should get the one from libm, not this), hence261// the __compiler_rt prefix.262static __inline fp_t __compiler_rt_logbX(fp_t x) {263 rep_t rep = toRep(x);264 int exp = (rep & exponentMask) >> significandBits;265 266 // Abnormal cases:267 // 1) +/- inf returns +inf; NaN returns NaN268 // 2) 0.0 returns -inf269 if (exp == maxExponent) {270 if (((rep & signBit) == 0) || (x != x)) {271 return x; // NaN or +inf: return x272 } else {273 return -x; // -inf: return -x274 }275 } else if (x == 0.0) {276 // 0.0: return -inf277 return fromRep(infRep | signBit);278 }279 280 if (exp != 0) {281 // Normal number282 return exp - exponentBias; // Unbias exponent283 } else {284 // Subnormal number; normalize and repeat285 rep &= absMask;286 const int shift = 1 - normalize(&rep);287 exp = (rep & exponentMask) >> significandBits;288 return exp - exponentBias - shift; // Unbias exponent289 }290}291 292// Avoid using scalbn from libm. Unlike libc/libm scalbn, this function never293// sets errno on underflow/overflow.294static __inline fp_t __compiler_rt_scalbnX(fp_t x, int y) {295 const rep_t rep = toRep(x);296 int exp = (rep & exponentMask) >> significandBits;297 298 if (x == 0.0 || exp == maxExponent)299 return x; // +/- 0.0, NaN, or inf: return x300 301 // Normalize subnormal input.302 rep_t sig = rep & significandMask;303 if (exp == 0) {304 exp += normalize(&sig);305 sig &= ~implicitBit; // clear the implicit bit again306 }307 308 if (__builtin_sadd_overflow(exp, y, &exp)) {309 // Saturate the exponent, which will guarantee an underflow/overflow below.310 exp = (y >= 0) ? INT_MAX : INT_MIN;311 }312 313 // Return this value: [+/-] 1.sig * 2 ** (exp - exponentBias).314 const rep_t sign = rep & signBit;315 if (exp >= maxExponent) {316 // Overflow, which could produce infinity or the largest-magnitude value,317 // depending on the rounding mode.318 return fromRep(sign | ((rep_t)(maxExponent - 1) << significandBits)) * 2.0f;319 } else if (exp <= 0) {320 // Subnormal or underflow. Use floating-point multiply to handle truncation321 // correctly.322 fp_t tmp = fromRep(sign | (REP_C(1) << significandBits) | sig);323 exp += exponentBias - 1;324 if (exp < 1)325 exp = 1;326 tmp *= fromRep((rep_t)exp << significandBits);327 return tmp;328 } else329 return fromRep(sign | ((rep_t)exp << significandBits) | sig);330}331 332#endif // !defined(QUAD_PRECISION) || defined(CRT_HAS_IEEE_TF)333 334// Avoid using fmax from libm.335static __inline fp_t __compiler_rt_fmaxX(fp_t x, fp_t y) {336 // If either argument is NaN, return the other argument. If both are NaN,337 // arbitrarily return the second one. Otherwise, if both arguments are +/-0,338 // arbitrarily return the first one.339 return (crt_isnan(x) || x < y) ? y : x;340}341 342#endif343 344#if defined(SINGLE_PRECISION)345 346static __inline fp_t __compiler_rt_logbf(fp_t x) {347 return __compiler_rt_logbX(x);348}349static __inline fp_t __compiler_rt_scalbnf(fp_t x, int y) {350 return __compiler_rt_scalbnX(x, y);351}352 353#elif defined(DOUBLE_PRECISION)354 355static __inline fp_t __compiler_rt_logb(fp_t x) {356 return __compiler_rt_logbX(x);357}358static __inline fp_t __compiler_rt_scalbn(fp_t x, int y) {359 return __compiler_rt_scalbnX(x, y);360}361static __inline fp_t __compiler_rt_fmax(fp_t x, fp_t y) {362#if defined(__aarch64__) || defined(__arm64ec__)363 // Use __builtin_fmax which turns into an fmaxnm instruction on AArch64.364 return __builtin_fmax(x, y);365#else366 // __builtin_fmax frequently turns into a libm call, so inline the function.367 return __compiler_rt_fmaxX(x, y);368#endif369}370 371#elif defined(QUAD_PRECISION) && defined(CRT_HAS_TF_MODE)372// The generic implementation only works for ieee754 floating point. For other373// floating point types, continue to rely on the libm implementation for now.374#if defined(CRT_HAS_IEEE_TF)375static __inline tf_float __compiler_rt_logbtf(tf_float x) {376 return __compiler_rt_logbX(x);377}378static __inline tf_float __compiler_rt_scalbntf(tf_float x, int y) {379 return __compiler_rt_scalbnX(x, y);380}381static __inline tf_float __compiler_rt_fmaxtf(tf_float x, tf_float y) {382 return __compiler_rt_fmaxX(x, y);383}384#define __compiler_rt_logbl __compiler_rt_logbtf385#define __compiler_rt_scalbnl __compiler_rt_scalbntf386#define __compiler_rt_fmaxl __compiler_rt_fmaxtf387#define crt_fabstf crt_fabsf128388#define crt_copysigntf crt_copysignf128389#elif defined(CRT_LDBL_128BIT)390static __inline tf_float __compiler_rt_logbtf(tf_float x) {391 return crt_logbl(x);392}393static __inline tf_float __compiler_rt_scalbntf(tf_float x, int y) {394 return crt_scalbnl(x, y);395}396static __inline tf_float __compiler_rt_fmaxtf(tf_float x, tf_float y) {397 return crt_fmaxl(x, y);398}399#define __compiler_rt_logbl crt_logbl400#define __compiler_rt_scalbnl crt_scalbnl401#define __compiler_rt_fmaxl crt_fmaxl402#define crt_fabstf crt_fabsl403#define crt_copysigntf crt_copysignl404#else405#error Unsupported TF mode type406#endif407 408#endif // *_PRECISION409 410#endif // FP_LIB_HEADER411