272 lines · plain
1//===----------------------------------------------------------------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8 9#include <clc/float/definitions.h>10#include <clc/internal/clc.h>11#include <clc/math/clc_copysign.h>12#include <clc/math/clc_fabs.h>13#include <clc/math/clc_nextafter.h>14#include <clc/relational/clc_isinf.h>15#include <clc/relational/clc_isnan.h>16#include <clc/shared/clc_min.h>17 18#pragma OPENCL EXTENSION cl_khr_byte_addressable_store : enable19 20#define __CLC_VSTORE_VECTORIZE(PRIM_TYPE, ADDR_SPACE) \21 typedef PRIM_TYPE##2 less_aligned_##ADDR_SPACE##PRIM_TYPE##2 \22 __attribute__((aligned(sizeof(PRIM_TYPE)))); \23 _CLC_OVERLOAD _CLC_DEF void __clc_vstore2(PRIM_TYPE##2 vec, size_t offset, \24 ADDR_SPACE PRIM_TYPE *mem) { \25 *((ADDR_SPACE less_aligned_##ADDR_SPACE##PRIM_TYPE##2 \26 *)(&mem[2 * offset])) = vec; \27 } \28 \29 _CLC_OVERLOAD _CLC_DEF void __clc_vstore3(PRIM_TYPE##3 vec, size_t offset, \30 ADDR_SPACE PRIM_TYPE *mem) { \31 *((ADDR_SPACE less_aligned_##ADDR_SPACE##PRIM_TYPE##2 \32 *)(&mem[3 * offset])) = (PRIM_TYPE##2)(vec.s0, vec.s1); \33 mem[3 * offset + 2] = vec.s2; \34 } \35 \36 typedef PRIM_TYPE##4 less_aligned_##ADDR_SPACE##PRIM_TYPE##4 \37 __attribute__((aligned(sizeof(PRIM_TYPE)))); \38 _CLC_OVERLOAD _CLC_DEF void __clc_vstore4(PRIM_TYPE##4 vec, size_t offset, \39 ADDR_SPACE PRIM_TYPE *mem) { \40 *((ADDR_SPACE less_aligned_##ADDR_SPACE##PRIM_TYPE##4 \41 *)(&mem[4 * offset])) = vec; \42 } \43 \44 typedef PRIM_TYPE##8 less_aligned_##ADDR_SPACE##PRIM_TYPE##8 \45 __attribute__((aligned(sizeof(PRIM_TYPE)))); \46 _CLC_OVERLOAD _CLC_DEF void __clc_vstore8(PRIM_TYPE##8 vec, size_t offset, \47 ADDR_SPACE PRIM_TYPE *mem) { \48 *((ADDR_SPACE less_aligned_##ADDR_SPACE##PRIM_TYPE##8 \49 *)(&mem[8 * offset])) = vec; \50 } \51 \52 typedef PRIM_TYPE##16 less_aligned_##ADDR_SPACE##PRIM_TYPE##16 \53 __attribute__((aligned(sizeof(PRIM_TYPE)))); \54 _CLC_OVERLOAD _CLC_DEF void __clc_vstore16(PRIM_TYPE##16 vec, size_t offset, \55 ADDR_SPACE PRIM_TYPE *mem) { \56 *((ADDR_SPACE less_aligned_##ADDR_SPACE##PRIM_TYPE##16 \57 *)(&mem[16 * offset])) = vec; \58 }59 60#if _CLC_DISTINCT_GENERIC_AS_SUPPORTED61#define __CLC_VSTORE_VECTORIZE_GENERIC __CLC_VSTORE_VECTORIZE62#else63// The generic address space isn't available, so make the macro do nothing64#define __CLC_VSTORE_VECTORIZE_GENERIC(X, Y)65#endif66 67#define __CLC_VSTORE_ADDR_SPACES(__CLC_SCALAR___CLC_GENTYPE) \68 __CLC_VSTORE_VECTORIZE(__CLC_SCALAR___CLC_GENTYPE, __private) \69 __CLC_VSTORE_VECTORIZE(__CLC_SCALAR___CLC_GENTYPE, __local) \70 __CLC_VSTORE_VECTORIZE(__CLC_SCALAR___CLC_GENTYPE, __global) \71 __CLC_VSTORE_VECTORIZE_GENERIC(__CLC_SCALAR___CLC_GENTYPE, __generic)72 73__CLC_VSTORE_ADDR_SPACES(char)74__CLC_VSTORE_ADDR_SPACES(uchar)75__CLC_VSTORE_ADDR_SPACES(short)76__CLC_VSTORE_ADDR_SPACES(ushort)77__CLC_VSTORE_ADDR_SPACES(int)78__CLC_VSTORE_ADDR_SPACES(uint)79__CLC_VSTORE_ADDR_SPACES(long)80__CLC_VSTORE_ADDR_SPACES(ulong)81__CLC_VSTORE_ADDR_SPACES(float)82 83#ifdef cl_khr_fp6484#pragma OPENCL EXTENSION cl_khr_fp64 : enable85__CLC_VSTORE_ADDR_SPACES(double)86#endif87 88#ifdef cl_khr_fp1689#pragma OPENCL EXTENSION cl_khr_fp16 : enable90__CLC_VSTORE_ADDR_SPACES(half)91#endif92 93#define __CLC_VEC_STORE1(val, ROUNDF, BUILTIN) \94 BUILTIN(ROUNDF(val), &mem[offset++]);95 96#define __CLC_VEC_STORE2(val, ROUNDF, BUILTIN) \97 __CLC_VEC_STORE1(val.lo, ROUNDF, BUILTIN) \98 __CLC_VEC_STORE1(val.hi, ROUNDF, BUILTIN)99#define __CLC_VEC_STORE3(val, ROUNDF, BUILTIN) \100 __CLC_VEC_STORE1(val.s0, ROUNDF, BUILTIN) \101 __CLC_VEC_STORE1(val.s1, ROUNDF, BUILTIN) \102 __CLC_VEC_STORE1(val.s2, ROUNDF, BUILTIN)103#define __CLC_VEC_STORE4(val, ROUNDF, BUILTIN) \104 __CLC_VEC_STORE2(val.lo, ROUNDF, BUILTIN) \105 __CLC_VEC_STORE2(val.hi, ROUNDF, BUILTIN)106#define __CLC_VEC_STORE8(val, ROUNDF, BUILTIN) \107 __CLC_VEC_STORE4(val.lo, ROUNDF, BUILTIN) \108 __CLC_VEC_STORE4(val.hi, ROUNDF, BUILTIN)109#define __CLC_VEC_STORE16(val, ROUNDF, BUILTIN) \110 __CLC_VEC_STORE8(val.lo, ROUNDF, BUILTIN) \111 __CLC_VEC_STORE8(val.hi, ROUNDF, BUILTIN)112 113#define __CLC_XFUNC_IMPL(SUFFIX, VEC_SIZE, OFFSET, TYPE, AS, ROUNDF, BUILTIN) \114 _CLC_OVERLOAD _CLC_DEF void __clc_vstore_half##SUFFIX( \115 TYPE vec, size_t offset, AS half *mem) { \116 offset *= VEC_SIZE; \117 __CLC_VEC_STORE##VEC_SIZE(vec, ROUNDF, BUILTIN) \118 } \119 _CLC_OVERLOAD _CLC_DEF void __clc_vstorea_half##SUFFIX( \120 TYPE vec, size_t offset, AS half *mem) { \121 offset *= OFFSET; \122 __CLC_VEC_STORE##VEC_SIZE(vec, ROUNDF, BUILTIN) \123 }124 125_CLC_DEF _CLC_OVERLOAD float __clc_noop(float x) { return x; }126_CLC_DEF _CLC_OVERLOAD float __clc_rtz(float x) {127 /* Remove lower 13 bits to make sure the number is rounded down */128 int mask = 0xffffe000;129 const int exp = (__clc_as_uint(x) >> 23 & 0xff) - 127;130 /* Denormals cannot be flushed, and they use different bit for rounding */131 if (exp < -14)132 mask <<= __clc_min(-(exp + 14), 10);133 /* RTZ does not produce Inf for large numbers */134 if (__clc_fabs(x) > 65504.0f && !__clc_isinf(x))135 return __clc_copysign(65504.0f, x);136 /* Handle nan corner case */137 if (__clc_isnan(x))138 return x;139 return __clc_as_float(__clc_as_uint(x) & mask);140}141_CLC_DEF _CLC_OVERLOAD float __clc_rti(float x) {142 const float inf = __clc_copysign(INFINITY, x);143 /* Set lower 13 bits */144 int mask = (1 << 13) - 1;145 const int exp = (__clc_as_uint(x) >> 23 & 0xff) - 127;146 /* Denormals cannot be flushed, and they use different bit for rounding */147 if (exp < -14)148 mask = (1 << (13 + __clc_min(-(exp + 14), 10))) - 1;149 /* Handle nan corner case */150 if (__clc_isnan(x))151 return x;152 const float next =153 __clc_nextafter(__clc_as_float(__clc_as_uint(x) | mask), inf);154 return ((__clc_as_uint(x) & mask) == 0) ? x : next;155}156_CLC_DEF _CLC_OVERLOAD float __clc_rtn(float x) {157 return ((__clc_as_uint(x) & 0x80000000) == 0) ? __clc_rtz(x) : __clc_rti(x);158}159_CLC_DEF _CLC_OVERLOAD float __clc_rtp(float x) {160 return ((__clc_as_uint(x) & 0x80000000) == 0) ? __clc_rti(x) : __clc_rtz(x);161}162_CLC_DEF _CLC_OVERLOAD float __clc_rte(float x) {163 /* Mantisa + implicit bit */164 const uint mantissa = (__clc_as_uint(x) & 0x7fffff) | (1u << 23);165 const int exp = (__clc_as_uint(x) >> 23 & 0xff) - 127;166 int shift = 13;167 if (exp < -14) {168 /* The default assumes lower 13 bits are rounded,169 * but it might be more for denormals.170 * Shifting beyond last == 0b, and qr == 00b is not necessary */171 shift += __clc_min(-(exp + 14), 15);172 }173 int mask = (1 << shift) - 1;174 const uint grs = mantissa & mask;175 const uint last = mantissa & (1 << shift);176 /* IEEE round up rule is: grs > 101b or grs == 100b and last == 1.177 * exp > 15 should round to inf. */178 bool roundup = (grs > (1 << (shift - 1))) ||179 (grs == (1 << (shift - 1)) && last != 0) || (exp > 15);180 return roundup ? __clc_rti(x) : __clc_rtz(x);181}182 183#ifdef cl_khr_fp64184_CLC_DEF _CLC_OVERLOAD double __clc_noop(double x) { return x; }185_CLC_DEF _CLC_OVERLOAD double __clc_rtz(double x) {186 /* Remove lower 42 bits to make sure the number is rounded down */187 ulong mask = 0xfffffc0000000000UL;188 const int exp = (__clc_as_ulong(x) >> 52 & 0x7ff) - 1023;189 /* Denormals cannot be flushed, and they use different bit for rounding */190 if (exp < -14)191 mask <<= __clc_min(-(exp + 14), 10);192 /* RTZ does not produce Inf for large numbers */193 if (__clc_fabs(x) > 65504.0 && !__clc_isinf(x))194 return __clc_copysign(65504.0, x);195 /* Handle nan corner case */196 if (__clc_isnan(x))197 return x;198 return __clc_as_double(__clc_as_ulong(x) & mask);199}200_CLC_DEF _CLC_OVERLOAD double __clc_rti(double x) {201 const double inf = __clc_copysign((double)INFINITY, x);202 /* Set lower 42 bits */203 long mask = (1UL << 42UL) - 1UL;204 const int exp = (__clc_as_ulong(x) >> 52 & 0x7ff) - 1023;205 /* Denormals cannot be flushed, and they use different bit for rounding */206 if (exp < -14)207 mask = (1UL << (42UL + __clc_min(-(exp + 14), 10))) - 1;208 /* Handle nan corner case */209 if (__clc_isnan(x))210 return x;211 const double next =212 __clc_nextafter(__clc_as_double(__clc_as_ulong(x) | mask), inf);213 return ((__clc_as_ulong(x) & mask) == 0) ? x : next;214}215_CLC_DEF _CLC_OVERLOAD double __clc_rtn(double x) {216 return ((__clc_as_ulong(x) & 0x8000000000000000UL) == 0) ? __clc_rtz(x)217 : __clc_rti(x);218}219_CLC_DEF _CLC_OVERLOAD double __clc_rtp(double x) {220 return ((__clc_as_ulong(x) & 0x8000000000000000UL) == 0) ? __clc_rti(x)221 : __clc_rtz(x);222}223_CLC_DEF _CLC_OVERLOAD double __clc_rte(double x) {224 /* Mantisa + implicit bit */225 const ulong mantissa = (__clc_as_ulong(x) & 0xfffffffffffff) | (1UL << 52);226 const int exp = (__clc_as_ulong(x) >> 52 & 0x7ff) - 1023;227 int shift = 42;228 if (exp < -14) {229 /* The default assumes lower 13 bits are rounded,230 * but it might be more for denormals.231 * Shifting beyond last == 0b, and qr == 00b is not necessary */232 shift += __clc_min(-(exp + 14), 15);233 }234 ulong mask = (1UL << shift) - 1UL;235 const ulong grs = mantissa & mask;236 const ulong last = mantissa & (1UL << shift);237 /* IEEE round up rule is: grs > 101b or grs == 100b and last == 1.238 * exp > 15 should round to inf. */239 bool roundup = (grs > (1UL << (shift - 1UL))) ||240 (grs == (1UL << (shift - 1UL)) && last != 0) || (exp > 15);241 return roundup ? __clc_rti(x) : __clc_rtz(x);242}243#endif244 245#define __CLC_XFUNC(SUFFIX, VEC_SIZE, OFFSET, TYPE, AS, BUILTIN) \246 __CLC_XFUNC_IMPL(SUFFIX, VEC_SIZE, OFFSET, TYPE, AS, __clc_noop, BUILTIN) \247 __CLC_XFUNC_IMPL(SUFFIX##_rtz, VEC_SIZE, OFFSET, TYPE, AS, __clc_rtz, \248 BUILTIN) \249 __CLC_XFUNC_IMPL(SUFFIX##_rtn, VEC_SIZE, OFFSET, TYPE, AS, __clc_rtn, \250 BUILTIN) \251 __CLC_XFUNC_IMPL(SUFFIX##_rtp, VEC_SIZE, OFFSET, TYPE, AS, __clc_rtp, \252 BUILTIN) \253 __CLC_XFUNC_IMPL(SUFFIX##_rte, VEC_SIZE, OFFSET, TYPE, AS, __clc_rte, BUILTIN)254 255#define __CLC_FUNC(SUFFIX, VEC_SIZE, OFFSET, TYPE, AS, BUILTIN) \256 __CLC_XFUNC(SUFFIX, VEC_SIZE, OFFSET, TYPE, AS, BUILTIN)257 258#define __CLC_BODY "clc_vstore_half.inc"259#include <clc/math/gentype.inc>260#undef __CLC_FUNC261#undef __CLC_XFUNC262#undef __CLC_XFUNC_IMPL263#undef __CLC_VEC_STORE16264#undef __CLC_VEC_STORE8265#undef __CLC_VEC_STORE4266#undef __CLC_VEC_STORE3267#undef __CLC_VEC_STORE2268#undef __CLC_VEC_STORE1269#undef __CLC_VSTORE_ADDR_SPACES270#undef __CLC_VSTORE_VECTORIZE271#undef __CLC_VSTORE_VECTORIZE_GENERIC272