213 lines · cpp
1//===------- Mapping.cpp - OpenMP device runtime mapping helpers -- C++ -*-===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9//10//===----------------------------------------------------------------------===//11 12#include "Mapping.h"13#include "DeviceTypes.h"14#include "DeviceUtils.h"15#include "Interface.h"16#include "State.h"17#include "gpuintrin.h"18 19using namespace ompx;20 21// FIXME: This resolves the handling for the AMDGPU workgroup size when the ABI22// is set to 'none'. We only support COV5+ but this can be removed when COV4 is23// fully deprecated.24#ifdef __AMDGPU__25extern const inline uint32_t __oclc_ABI_version = 500;26[[gnu::alias("__oclc_ABI_version")]] const uint32_t __oclc_ABI_version__;27#endif28 29static bool isInLastWarp() {30 uint32_t MainTId = (mapping::getNumberOfThreadsInBlock() - 1) &31 ~(mapping::getWarpSize() - 1);32 return mapping::getThreadIdInBlock() == MainTId;33}34 35bool mapping::isMainThreadInGenericMode(bool IsSPMD) {36 if (IsSPMD || icv::Level)37 return false;38 39 // Check if this is the last warp in the block.40 return isInLastWarp();41}42 43bool mapping::isMainThreadInGenericMode() {44 return mapping::isMainThreadInGenericMode(mapping::isSPMDMode());45}46 47bool mapping::isInitialThreadInLevel0(bool IsSPMD) {48 if (IsSPMD)49 return mapping::getThreadIdInBlock() == 0;50 return isInLastWarp();51}52 53bool mapping::isLeaderInWarp() {54 __kmpc_impl_lanemask_t Active = mapping::activemask();55 __kmpc_impl_lanemask_t LaneMaskLT = mapping::lanemaskLT();56 return utils::popc(Active & LaneMaskLT) == 0;57}58 59LaneMaskTy mapping::activemask() { return __gpu_lane_mask(); }60 61LaneMaskTy mapping::lanemaskLT() {62#ifdef __NVPTX__63 return __nvvm_read_ptx_sreg_lanemask_lt();64#else65 uint32_t Lane = mapping::getThreadIdInWarp();66 int64_t Ballot = mapping::activemask();67 uint64_t Mask = ((uint64_t)1 << Lane) - (uint64_t)1;68 return Mask & Ballot;69#endif70}71 72LaneMaskTy mapping::lanemaskGT() {73#ifdef __NVPTX__74 return __nvvm_read_ptx_sreg_lanemask_gt();75#else76 uint32_t Lane = mapping::getThreadIdInWarp();77 if (Lane == (mapping::getWarpSize() - 1))78 return 0;79 int64_t Ballot = mapping::activemask();80 uint64_t Mask = (~((uint64_t)0)) << (Lane + 1);81 return Mask & Ballot;82#endif83}84 85uint32_t mapping::getThreadIdInWarp() {86 uint32_t ThreadIdInWarp = __gpu_lane_id();87 ASSERT(ThreadIdInWarp < mapping::getWarpSize(), nullptr);88 return ThreadIdInWarp;89}90 91uint32_t mapping::getThreadIdInBlock(int32_t Dim) {92 uint32_t ThreadIdInBlock = __gpu_thread_id(Dim);93 return ThreadIdInBlock;94}95 96uint32_t mapping::getWarpSize() { return __gpu_num_lanes(); }97 98uint32_t mapping::getMaxTeamThreads(bool IsSPMD) {99 uint32_t BlockSize = mapping::getNumberOfThreadsInBlock();100 // If we are in SPMD mode, remove one warp.101 return BlockSize - (!IsSPMD * mapping::getWarpSize());102}103uint32_t mapping::getMaxTeamThreads() {104 return mapping::getMaxTeamThreads(mapping::isSPMDMode());105}106 107uint32_t mapping::getNumberOfThreadsInBlock(int32_t Dim) {108 return __gpu_num_threads(Dim);109}110 111uint32_t mapping::getNumberOfThreadsInKernel() {112 return mapping::getNumberOfThreadsInBlock(0) *113 mapping::getNumberOfBlocksInKernel(0) *114 mapping::getNumberOfThreadsInBlock(1) *115 mapping::getNumberOfBlocksInKernel(1) *116 mapping::getNumberOfThreadsInBlock(2) *117 mapping::getNumberOfBlocksInKernel(2);118}119 120uint32_t mapping::getWarpIdInBlock() {121 uint32_t WarpID =122 mapping::getThreadIdInBlock(mapping::DIM_X) / mapping::getWarpSize();123 ASSERT(WarpID < mapping::getNumberOfWarpsInBlock(), nullptr);124 return WarpID;125}126 127uint32_t mapping::getBlockIdInKernel(int32_t Dim) {128 uint32_t BlockId = __gpu_block_id(Dim);129 ASSERT(BlockId < mapping::getNumberOfBlocksInKernel(Dim), nullptr);130 return BlockId;131}132 133uint32_t mapping::getNumberOfWarpsInBlock() {134 return (mapping::getNumberOfThreadsInBlock() + mapping::getWarpSize() - 1) /135 mapping::getWarpSize();136}137 138uint32_t mapping::getNumberOfBlocksInKernel(int32_t Dim) {139 return __gpu_num_blocks(Dim);140}141 142uint32_t mapping::getNumberOfProcessorElements() {143 return static_cast<uint32_t>(config::getHardwareParallelism());144}145 146///}147 148/// Execution mode149///150///{151 152// TODO: This is a workaround for initialization coming from kernels outside of153// the TU. We will need to solve this more correctly in the future.154[[gnu::weak, clang::loader_uninitialized]] Local<int> IsSPMDMode;155 156void mapping::init(bool IsSPMD) {157 if (mapping::isInitialThreadInLevel0(IsSPMD))158 IsSPMDMode = IsSPMD;159}160 161bool mapping::isSPMDMode() { return IsSPMDMode; }162 163bool mapping::isGenericMode() { return !isSPMDMode(); }164///}165 166extern "C" {167[[gnu::noinline]] uint32_t __kmpc_get_hardware_thread_id_in_block() {168 return mapping::getThreadIdInBlock();169}170 171[[gnu::noinline]] uint32_t __kmpc_get_hardware_num_threads_in_block() {172 return mapping::getNumberOfThreadsInBlock(mapping::DIM_X);173}174 175[[gnu::noinline]] uint32_t __kmpc_get_warp_size() {176 return mapping::getWarpSize();177}178}179 180#define _TGT_KERNEL_LANGUAGE(NAME, MAPPER_NAME) \181 extern "C" int ompx_##NAME(int Dim) { return mapping::MAPPER_NAME(Dim); }182 183_TGT_KERNEL_LANGUAGE(thread_id, getThreadIdInBlock)184_TGT_KERNEL_LANGUAGE(block_id, getBlockIdInKernel)185_TGT_KERNEL_LANGUAGE(block_dim, getNumberOfThreadsInBlock)186_TGT_KERNEL_LANGUAGE(grid_dim, getNumberOfBlocksInKernel)187 188extern "C" {189uint64_t ompx_ballot_sync(uint64_t mask, int pred) {190 return utils::ballotSync(mask, pred);191}192 193int ompx_shfl_down_sync_i(uint64_t mask, int var, unsigned delta, int width) {194 return utils::shuffleDown(mask, var, delta, width);195}196 197float ompx_shfl_down_sync_f(uint64_t mask, float var, unsigned delta,198 int width) {199 return utils::bitCast<float>(200 utils::shuffleDown(mask, utils::bitCast<int32_t>(var), delta, width));201}202 203long ompx_shfl_down_sync_l(uint64_t mask, long var, unsigned delta, int width) {204 return utils::shuffleDown(mask, utils::bitCast<int64_t>(var), delta, width);205}206 207double ompx_shfl_down_sync_d(uint64_t mask, double var, unsigned delta,208 int width) {209 return utils::bitCast<double>(210 utils::shuffleDown(mask, utils::bitCast<int64_t>(var), delta, width));211}212}213