152 lines · plain
1//===----------------------------------------------------------------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8 9#include <clc/float/definitions.h>10#include <clc/opencl/as_type.h>11#include <clc/opencl/math/copysign.h>12#include <clc/opencl/math/fabs.h>13#include <clc/opencl/math/nextafter.h>14#include <clc/opencl/relational/isinf.h>15#include <clc/opencl/relational/isnan.h>16#include <clc/opencl/shared/min.h>17#include <clc/opencl/shared/vstore.h>18 19#pragma OPENCL EXTENSION cl_khr_byte_addressable_store : enable20 21#define __CLC_ROUND_VEC1(out, in, ROUNDF) out = ROUNDF(in);22#define __CLC_ROUND_VEC2(out, in, ROUNDF) \23 __CLC_ROUND_VEC1(out.lo, in.lo, ROUNDF); \24 __CLC_ROUND_VEC1(out.hi, in.hi, ROUNDF);25#define __CLC_ROUND_VEC3(out, in, ROUNDF) \26 __CLC_ROUND_VEC1(out.s0, in.s0, ROUNDF); \27 __CLC_ROUND_VEC1(out.s1, in.s1, ROUNDF); \28 __CLC_ROUND_VEC1(out.s2, in.s2, ROUNDF);29#define __CLC_ROUND_VEC4(out, in, ROUNDF) \30 __CLC_ROUND_VEC2(out.lo, in.lo, ROUNDF); \31 __CLC_ROUND_VEC2(out.hi, in.hi, ROUNDF);32#define __CLC_ROUND_VEC8(out, in, ROUNDF) \33 __CLC_ROUND_VEC4(out.lo, in.lo, ROUNDF); \34 __CLC_ROUND_VEC4(out.hi, in.hi, ROUNDF);35#define __CLC_ROUND_VEC16(out, in, ROUNDF) \36 __CLC_ROUND_VEC8(out.lo, in.lo, ROUNDF); \37 __CLC_ROUND_VEC8(out.hi, in.hi, ROUNDF);38 39#define __CLC_XFUNC_IMPL(SUFFIX, VEC_SIZE, TYPE, AS, ROUNDF) \40 void _CLC_OVERLOAD vstore_half_##VEC_SIZE(TYPE, size_t, AS half *); \41 _CLC_OVERLOAD _CLC_DEF void vstore_half##SUFFIX(TYPE vec, size_t offset, \42 AS half *mem) { \43 TYPE rounded_vec; \44 __CLC_ROUND_VEC##VEC_SIZE(rounded_vec, vec, ROUNDF); \45 vstore_half_##VEC_SIZE(rounded_vec, offset, mem); \46 } \47 void _CLC_OVERLOAD vstorea_half_##VEC_SIZE(TYPE, size_t, AS half *); \48 _CLC_OVERLOAD _CLC_DEF void vstorea_half##SUFFIX(TYPE vec, size_t offset, \49 AS half *mem) { \50 TYPE rounded_vec; \51 __CLC_ROUND_VEC##VEC_SIZE(rounded_vec, vec, ROUNDF); \52 vstorea_half_##VEC_SIZE(rounded_vec, offset, mem); \53 }54 55_CLC_DEF _CLC_OVERLOAD float __clc_rtz(float x) {56 /* Handle nan corner case */57 if (isnan(x))58 return x;59 /* RTZ does not produce Inf for large numbers */60 if (fabs(x) > 65504.0f && !isinf(x))61 return copysign(65504.0f, x);62 63 const int exp = (as_uint(x) >> 23 & 0xff) - 127;64 /* Manage range rounded to +- zero explicitely */65 if (exp < -24)66 return copysign(0.0f, x);67 68 /* Remove lower 13 bits to make sure the number is rounded down */69 int mask = 0xffffe000;70 /* Denormals cannot be flushed, and they use different bit for rounding */71 if (exp < -14)72 mask <<= min(-(exp + 14), 10);73 74 return as_float(as_uint(x) & mask);75}76 77_CLC_DEF _CLC_OVERLOAD float __clc_rti(float x) {78 /* Handle nan corner case */79 if (isnan(x))80 return x;81 82 const float inf = copysign(INFINITY, x);83 uint ux = as_uint(x);84 85 /* Manage +- infinity explicitely */86 if (as_float(ux & 0x7fffffff) > 0x1.ffcp+15f) {87 return inf;88 }89 /* Manage +- zero explicitely */90 if ((ux & 0x7fffffff) == 0) {91 return copysign(0.0f, x);92 }93 94 const int exp = (as_uint(x) >> 23 & 0xff) - 127;95 /* Manage range rounded to smallest half denormal explicitely */96 if (exp < -24) {97 return copysign(0x1.0p-24f, x);98 }99 100 /* Set lower 13 bits */101 int mask = (1 << 13) - 1;102 /* Denormals cannot be flushed, and they use different bit for rounding */103 if (exp < -14) {104 mask = (1 << (13 + min(-(exp + 14), 10))) - 1;105 }106 107 const float next = nextafter(as_float(ux | mask), inf);108 return ((ux & mask) == 0) ? as_float(ux) : next;109}110_CLC_DEF _CLC_OVERLOAD float __clc_rtn(float x) {111 return ((as_uint(x) & 0x80000000) == 0) ? __clc_rtz(x) : __clc_rti(x);112}113_CLC_DEF _CLC_OVERLOAD float __clc_rtp(float x) {114 return ((as_uint(x) & 0x80000000) == 0) ? __clc_rti(x) : __clc_rtz(x);115}116_CLC_DEF _CLC_OVERLOAD float __clc_rte(float x) {117 /* Mantisa + implicit bit */118 const uint mantissa = (as_uint(x) & 0x7fffff) | (1u << 23);119 const int exp = (as_uint(x) >> 23 & 0xff) - 127;120 int shift = 13;121 if (exp < -14) {122 /* The default assumes lower 13 bits are rounded,123 * but it might be more for denormals.124 * Shifting beyond last == 0b, and qr == 00b is not necessary */125 shift += min(-(exp + 14), 15);126 }127 int mask = (1 << shift) - 1;128 const uint grs = mantissa & mask;129 const uint last = mantissa & (1 << shift);130 /* IEEE round up rule is: grs > 101b or grs == 100b and last == 1.131 * exp > 15 should round to inf. */132 bool roundup = (grs > (1 << (shift - 1))) ||133 (grs == (1 << (shift - 1)) && last != 0) || (exp > 15);134 return roundup ? __clc_rti(x) : __clc_rtz(x);135}136 137#define __CLC_XFUNC(SUFFIX, VEC_SIZE, TYPE, AS) \138 __CLC_XFUNC_IMPL(SUFFIX, VEC_SIZE, TYPE, AS, __clc_rte) \139 __CLC_XFUNC_IMPL(SUFFIX##_rtz, VEC_SIZE, TYPE, AS, __clc_rtz) \140 __CLC_XFUNC_IMPL(SUFFIX##_rtn, VEC_SIZE, TYPE, AS, __clc_rtn) \141 __CLC_XFUNC_IMPL(SUFFIX##_rtp, VEC_SIZE, TYPE, AS, __clc_rtp) \142 __CLC_XFUNC_IMPL(SUFFIX##_rte, VEC_SIZE, TYPE, AS, __clc_rte)143 144#define __CLC_FUNC(SUFFIX, VEC_SIZE, TYPE, AS) \145 __CLC_XFUNC(SUFFIX, VEC_SIZE, TYPE, AS)146 147#define __CLC_BODY "vstore_half.inc"148#include <clc/math/gentype.inc>149#undef __CLC_FUNC150#undef __CLC_XFUNC151#undef __CLC_XFUNC_IMPL152