226 lines · c
1//===- Synchronization.h - OpenMP synchronization utilities ------- C++ -*-===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9//10//===----------------------------------------------------------------------===//11 12#ifndef OMPTARGET_DEVICERTL_SYNCHRONIZATION_H13#define OMPTARGET_DEVICERTL_SYNCHRONIZATION_H14 15#include "DeviceTypes.h"16#include "DeviceUtils.h"17 18namespace ompx {19namespace atomic {20 21enum OrderingTy {22 relaxed = __ATOMIC_RELAXED,23 acquire = __ATOMIC_ACQUIRE,24 release = __ATOMIC_RELEASE,25 acq_rel = __ATOMIC_ACQ_REL,26 seq_cst = __ATOMIC_SEQ_CST,27};28 29enum MemScopeTy {30 system = __MEMORY_SCOPE_SYSTEM,31 device = __MEMORY_SCOPE_DEVICE,32 workgroup = __MEMORY_SCOPE_WRKGRP,33 wavefront = __MEMORY_SCOPE_WVFRNT,34 single = __MEMORY_SCOPE_SINGLE,35};36 37/// Atomically increment \p *Addr and wrap at \p V with \p Ordering semantics.38uint32_t inc(uint32_t *Addr, uint32_t V, OrderingTy Ordering,39 MemScopeTy MemScope = MemScopeTy::device);40 41/// Atomically perform <op> on \p V and \p *Addr with \p Ordering semantics. The42/// result is stored in \p *Addr;43/// {44 45template <typename Ty, typename V = utils::remove_addrspace_t<Ty>>46bool cas(Ty *Address, V ExpectedV, V DesiredV, atomic::OrderingTy OrderingSucc,47 atomic::OrderingTy OrderingFail,48 MemScopeTy MemScope = MemScopeTy::device) {49 return __scoped_atomic_compare_exchange(Address, &ExpectedV, &DesiredV, false,50 OrderingSucc, OrderingFail, MemScope);51}52 53template <typename Ty, typename V = utils::remove_addrspace_t<Ty>>54V add(Ty *Address, V Val, atomic::OrderingTy Ordering,55 MemScopeTy MemScope = MemScopeTy::device) {56 return __scoped_atomic_fetch_add(Address, Val, Ordering, MemScope);57}58 59template <typename Ty, typename V = utils::remove_addrspace_t<Ty>>60V load(Ty *Address, atomic::OrderingTy Ordering,61 MemScopeTy MemScope = MemScopeTy::device) {62#ifdef __NVPTX__63 return __scoped_atomic_fetch_add(Address, V(0), Ordering, MemScope);64#else65 return __scoped_atomic_load_n(Address, Ordering, MemScope);66#endif67}68 69template <typename Ty, typename V = utils::remove_addrspace_t<Ty>>70void store(Ty *Address, V Val, atomic::OrderingTy Ordering,71 MemScopeTy MemScope = MemScopeTy::device) {72 __scoped_atomic_store_n(Address, Val, Ordering, MemScope);73}74 75template <typename Ty, typename V = utils::remove_addrspace_t<Ty>>76V mul(Ty *Address, V Val, atomic::OrderingTy Ordering,77 MemScopeTy MemScope = MemScopeTy::device) {78 Ty TypedCurrentVal, TypedResultVal, TypedNewVal;79 bool Success;80 do {81 TypedCurrentVal = atomic::load(Address, Ordering);82 TypedNewVal = TypedCurrentVal * Val;83 Success = atomic::cas(Address, TypedCurrentVal, TypedNewVal, Ordering,84 atomic::relaxed, MemScope);85 } while (!Success);86 return TypedResultVal;87}88 89template <typename Ty, typename V = utils::remove_addrspace_t<Ty>>90utils::enable_if_t<!utils::is_floating_point_v<V>, V>91max(Ty *Address, V Val, atomic::OrderingTy Ordering,92 MemScopeTy MemScope = MemScopeTy::device) {93 return __scoped_atomic_fetch_max(Address, Val, Ordering, MemScope);94}95 96template <typename Ty, typename V = utils::remove_addrspace_t<Ty>>97utils::enable_if_t<utils::is_same_v<V, float>, V>98max(Ty *Address, V Val, atomic::OrderingTy Ordering,99 MemScopeTy MemScope = MemScopeTy::device) {100 if (Val >= 0)101 return utils::bitCast<float>(max(102 (int32_t *)Address, utils::bitCast<int32_t>(Val), Ordering, MemScope));103 return utils::bitCast<float>(min(104 (uint32_t *)Address, utils::bitCast<uint32_t>(Val), Ordering, MemScope));105}106 107template <typename Ty, typename V = utils::remove_addrspace_t<Ty>>108utils::enable_if_t<utils::is_same_v<V, double>, V>109max(Ty *Address, V Val, atomic::OrderingTy Ordering,110 MemScopeTy MemScope = MemScopeTy::device) {111 if (Val >= 0)112 return utils::bitCast<double>(max(113 (int64_t *)Address, utils::bitCast<int64_t>(Val), Ordering, MemScope));114 return utils::bitCast<double>(min(115 (uint64_t *)Address, utils::bitCast<uint64_t>(Val), Ordering, MemScope));116}117 118template <typename Ty, typename V = utils::remove_addrspace_t<Ty>>119utils::enable_if_t<!utils::is_floating_point_v<V>, V>120min(Ty *Address, V Val, atomic::OrderingTy Ordering,121 MemScopeTy MemScope = MemScopeTy::device) {122 return __scoped_atomic_fetch_min(Address, Val, Ordering, MemScope);123}124 125// TODO: Implement this with __atomic_fetch_max and remove the duplication.126template <typename Ty, typename V = utils::remove_addrspace_t<Ty>>127utils::enable_if_t<utils::is_same_v<V, float>, V>128min(Ty *Address, V Val, atomic::OrderingTy Ordering,129 MemScopeTy MemScope = MemScopeTy::device) {130 if (Val >= 0)131 return utils::bitCast<float>(min(132 (int32_t *)Address, utils::bitCast<int32_t>(Val), Ordering, MemScope));133 return utils::bitCast<float>(max(134 (uint32_t *)Address, utils::bitCast<uint32_t>(Val), Ordering, MemScope));135}136 137// TODO: Implement this with __atomic_fetch_max and remove the duplication.138template <typename Ty, typename V = utils::remove_addrspace_t<Ty>>139utils::enable_if_t<utils::is_same_v<V, double>, V>140min(Ty *Address, utils::remove_addrspace_t<Ty> Val, atomic::OrderingTy Ordering,141 MemScopeTy MemScope = MemScopeTy::device) {142 if (Val >= 0)143 return utils::bitCast<double>(min(144 (int64_t *)Address, utils::bitCast<int64_t>(Val), Ordering, MemScope));145 return utils::bitCast<double>(max(146 (uint64_t *)Address, utils::bitCast<uint64_t>(Val), Ordering, MemScope));147}148 149template <typename Ty, typename V = utils::remove_addrspace_t<Ty>>150V bit_or(Ty *Address, V Val, atomic::OrderingTy Ordering,151 MemScopeTy MemScope = MemScopeTy::device) {152 return __scoped_atomic_fetch_or(Address, Val, Ordering, MemScope);153}154 155template <typename Ty, typename V = utils::remove_addrspace_t<Ty>>156V bit_and(Ty *Address, V Val, atomic::OrderingTy Ordering,157 MemScopeTy MemScope = MemScopeTy::device) {158 return __scoped_atomic_fetch_and(Address, Val, Ordering, MemScope);159}160 161template <typename Ty, typename V = utils::remove_addrspace_t<Ty>>162V bit_xor(Ty *Address, V Val, atomic::OrderingTy Ordering,163 MemScopeTy MemScope = MemScopeTy::device) {164 return __scoped_atomic_fetch_xor(Address, Val, Ordering, MemScope);165}166 167static inline uint32_t168atomicExchange(uint32_t *Address, uint32_t Val, atomic::OrderingTy Ordering,169 MemScopeTy MemScope = MemScopeTy::device) {170 uint32_t R;171 __scoped_atomic_exchange(Address, &Val, &R, Ordering, MemScope);172 return R;173}174 175///}176 177} // namespace atomic178 179namespace synchronize {180 181/// Initialize the synchronization machinery. Must be called by all threads.182void init(bool IsSPMD);183 184/// Synchronize all threads in a warp identified by \p Mask.185void warp(LaneMaskTy Mask);186 187/// Synchronize all threads in a block and perform a fence before and after the188/// barrier according to \p Ordering. Note that the fence might be part of the189/// barrier.190void threads(atomic::OrderingTy Ordering);191 192/// Synchronizing threads is allowed even if they all hit different instances of193/// `synchronize::threads()`. However, `synchronize::threadsAligned()` is more194/// restrictive in that it requires all threads to hit the same instance. The195/// noinline is removed by the openmp-opt pass and helps to preserve the196/// information till then.197///{198 199/// Synchronize all threads in a block, they are reaching the same instruction200/// (hence all threads in the block are "aligned"). Also perform a fence before201/// and after the barrier according to \p Ordering. Note that the202/// fence might be part of the barrier if the target offers this.203[[gnu::noinline, omp::assume("ompx_aligned_barrier")]] void204threadsAligned(atomic::OrderingTy Ordering);205 206///}207 208} // namespace synchronize209 210namespace fence {211 212/// Memory fence with \p Ordering semantics for the team.213void team(atomic::OrderingTy Ordering);214 215/// Memory fence with \p Ordering semantics for the contention group.216void kernel(atomic::OrderingTy Ordering);217 218/// Memory fence with \p Ordering semantics for the system.219void system(atomic::OrderingTy Ordering);220 221} // namespace fence222 223} // namespace ompx224 225#endif226