166 lines · plain
1//===----------------------------------------------------------------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8 9#include <clc/clc_as_type.h>10#include <clc/float/definitions.h>11#include <clc/integer/clc_abs.h>12#include <clc/integer/clc_clz.h>13#include <clc/integer/definitions.h>14#include <clc/internal/clc.h>15#include <clc/math/clc_mad.h>16#include <clc/math/math.h>17#include <clc/relational/clc_isinf.h>18#include <clc/relational/clc_isnan.h>19#include <clc/shared/clc_max.h>20 21struct fp {22 ulong mantissa;23 int exponent;24 uint sign;25};26 27_CLC_DEF _CLC_OVERLOAD float __clc_sw_fma(float a, float b, float c) {28 /* special cases */29 if (__clc_isnan(a) || __clc_isnan(b) || __clc_isnan(c) || __clc_isinf(a) ||30 __clc_isinf(b)) {31 return __clc_mad(a, b, c);32 }33 34 /* If only c is inf, and both a,b are regular numbers, the result is c*/35 if (__clc_isinf(c)) {36 return c;37 }38 39 a = __clc_flush_denormal_if_not_supported(a);40 b = __clc_flush_denormal_if_not_supported(b);41 c = __clc_flush_denormal_if_not_supported(c);42 43 if (c == 0) {44 return a * b;45 }46 47 struct fp st_a, st_b, st_c;48 49 st_a.exponent = a == .0f ? 0 : ((__clc_as_uint(a) & 0x7f800000) >> 23) - 127;50 st_b.exponent = b == .0f ? 0 : ((__clc_as_uint(b) & 0x7f800000) >> 23) - 127;51 st_c.exponent = c == .0f ? 0 : ((__clc_as_uint(c) & 0x7f800000) >> 23) - 127;52 53 st_a.mantissa = a == .0f ? 0 : (__clc_as_uint(a) & 0x7fffff) | 0x800000;54 st_b.mantissa = b == .0f ? 0 : (__clc_as_uint(b) & 0x7fffff) | 0x800000;55 st_c.mantissa = c == .0f ? 0 : (__clc_as_uint(c) & 0x7fffff) | 0x800000;56 57 st_a.sign = __clc_as_uint(a) & 0x80000000;58 st_b.sign = __clc_as_uint(b) & 0x80000000;59 st_c.sign = __clc_as_uint(c) & 0x80000000;60 61 // Multiplication.62 // Move the product to the highest bits to maximize precision63 // mantissa is 24 bits => product is 48 bits, 2bits non-fraction.64 // Add one bit for future addition overflow,65 // add another bit to detect subtraction underflow66 struct fp st_mul;67 st_mul.sign = st_a.sign ^ st_b.sign;68 st_mul.mantissa = (st_a.mantissa * st_b.mantissa) << 14ul;69 st_mul.exponent = st_mul.mantissa ? st_a.exponent + st_b.exponent : 0;70 71 // FIXME: Detecting a == 0 || b == 0 above crashed GCN isel72 if (st_mul.exponent == 0 && st_mul.mantissa == 0)73 return c;74 75// Mantissa is 23 fractional bits, shift it the same way as product mantissa76#define C_ADJUST 37ul77 78 // both exponents are bias adjusted79 int exp_diff = st_mul.exponent - st_c.exponent;80 81 st_c.mantissa <<= C_ADJUST;82 ulong cutoff_bits = 0;83 ulong cutoff_mask = (1ul << __clc_abs(exp_diff)) - 1ul;84 if (exp_diff > 0) {85 cutoff_bits =86 exp_diff >= 64 ? st_c.mantissa : (st_c.mantissa & cutoff_mask);87 st_c.mantissa = exp_diff >= 64 ? 0 : (st_c.mantissa >> exp_diff);88 } else {89 cutoff_bits =90 -exp_diff >= 64 ? st_mul.mantissa : (st_mul.mantissa & cutoff_mask);91 st_mul.mantissa = -exp_diff >= 64 ? 0 : (st_mul.mantissa >> -exp_diff);92 }93 94 struct fp st_fma;95 st_fma.sign = st_mul.sign;96 st_fma.exponent = __clc_max(st_mul.exponent, st_c.exponent);97 if (st_c.sign == st_mul.sign) {98 st_fma.mantissa = st_mul.mantissa + st_c.mantissa;99 } else {100 // cutoff bits borrow one101 st_fma.mantissa =102 st_mul.mantissa - st_c.mantissa -103 (cutoff_bits && (st_mul.exponent > st_c.exponent) ? 1 : 0);104 }105 106 // underflow: st_c.sign != st_mul.sign, and magnitude switches the sign107 if (st_fma.mantissa > LONG_MAX) {108 st_fma.mantissa = 0 - st_fma.mantissa;109 st_fma.sign = st_mul.sign ^ 0x80000000;110 }111 112 // detect overflow/underflow113 int overflow_bits = 3 - __clc_clz(st_fma.mantissa);114 115 // adjust exponent116 st_fma.exponent += overflow_bits;117 118 // handle underflow119 if (overflow_bits < 0) {120 st_fma.mantissa <<= -overflow_bits;121 overflow_bits = 0;122 }123 124 // rounding125 ulong trunc_mask = (1ul << (C_ADJUST + overflow_bits)) - 1;126 ulong trunc_bits = (st_fma.mantissa & trunc_mask) | (cutoff_bits != 0);127 ulong last_bit = st_fma.mantissa & (1ul << (C_ADJUST + overflow_bits));128 ulong grs_bits = (0x4ul << (C_ADJUST - 3 + overflow_bits));129 130 // round to nearest even131 if ((trunc_bits > grs_bits) || (trunc_bits == grs_bits && last_bit != 0)) {132 st_fma.mantissa += (1ul << (C_ADJUST + overflow_bits));133 }134 135 // Shift mantissa back to bit 23136 st_fma.mantissa = (st_fma.mantissa >> (C_ADJUST + overflow_bits));137 138 // Detect rounding overflow139 if (st_fma.mantissa > 0xffffff) {140 ++st_fma.exponent;141 st_fma.mantissa >>= 1;142 }143 144 if (st_fma.mantissa == 0) {145 return .0f;146 }147 148 // Flating point range limit149 if (st_fma.exponent > 127) {150 return __clc_as_float(__clc_as_uint(INFINITY) | st_fma.sign);151 }152 153 // Flush denormals154 if (st_fma.exponent <= -127) {155 return __clc_as_float(st_fma.sign);156 }157 158 return __clc_as_float(st_fma.sign | ((st_fma.exponent + 127) << 23) |159 ((uint)st_fma.mantissa & 0x7fffff));160}161 162#define __CLC_FLOAT_ONLY163#define __CLC_FUNCTION __clc_sw_fma164#define __CLC_BODY <clc/shared/ternary_def_scalarize.inc>165#include <clc/math/gentype.inc>166