238 lines · c
1//===------------- AMDGPU implementation of timing utils --------*- C++ -*-===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8 9#ifndef LLVM_LIBC_UTILS_GPU_TIMING_AMDGPU10#define LLVM_LIBC_UTILS_GPU_TIMING_AMDGPU11 12#include "hdr/stdint_proxy.h"13#include "src/__support/CPP/algorithm.h"14#include "src/__support/CPP/array.h"15#include "src/__support/CPP/atomic.h"16#include "src/__support/CPP/type_traits.h"17#include "src/__support/GPU/utils.h"18#include "src/__support/macros/attributes.h"19#include "src/__support/macros/config.h"20 21namespace LIBC_NAMESPACE_DECL {22 23// Returns the overhead associated with calling the profiling region. This24// allows us to substract the constant-time overhead from the latency to25// obtain a true result. This can vary with system load.26[[gnu::noinline]] static LIBC_INLINE uint64_t overhead() {27 cpp::atomic_thread_fence(cpp::MemoryOrder::ACQ_REL);28 uint64_t start = gpu::processor_clock();29 uint32_t result = 0.0;30 asm("v_or_b32 %[v_reg], 0, %[v_reg]\n" ::[v_reg] "v"(result));31 asm("" ::"s"(start));32 uint64_t stop = gpu::processor_clock();33 return stop - start;34}35 36// Profile a simple function and obtain its latency in clock cycles on the37// system. This function cannot be inlined or else it will disturb the very38// delicate balance of hard-coded dependencies.39template <typename F, typename T>40[[gnu::noinline]] static LIBC_INLINE uint64_t latency(F f, T t) {41 // We need to store the input somewhere to guarantee that the compiler42 // will not constant propagate it and remove the profiling region.43 volatile T storage = t;44 T arg = storage;45 46 // The AMDGPU architecture needs to wait on pending results.47 cpp::atomic_thread_fence(cpp::MemoryOrder::ACQ_REL);48 // Get the current timestamp from the clock.49 uint64_t start = gpu::processor_clock();50 51 // This forces the compiler to load the input argument and run the clock52 // cycle counter before the profiling region.53 asm("" : "+v"(arg) : "s"(start));54 55 // Run the function under test and return its value.56 auto result = f(arg);57 58 // This inline assembly performs a no-op which forces the result to both59 // be used and prevents us from exiting this region before it's complete.60 if constexpr (cpp::is_same_v<decltype(result), char> ||61 cpp::is_same_v<decltype(result), bool>)62 // AMDGPU does not support input register constraints for i1 and i8, so we63 // cast it to a 32-bit integer. This does not add an additional assembly64 // instruction (https://godbolt.org/z/zxGqv8G91).65 asm("v_or_b32 %[v_reg], 0, %[v_reg]\n" ::[v_reg] "v"(66 static_cast<uint32_t>(result)));67 else68 asm("v_or_b32 %[v_reg], 0, %[v_reg]\n" ::[v_reg] "v"(result));69 70 // Obtain the current timestamp after running the calculation and force71 // ordering.72 uint64_t stop = gpu::processor_clock();73 asm("" ::"s"(stop));74 cpp::atomic_thread_fence(cpp::MemoryOrder::ACQ_REL);75 76 // Return the time elapsed.77 return stop - start;78}79 80template <typename F, typename T1, typename T2>81[[gnu::noinline]] static LIBC_INLINE uint64_t latency(F f, T1 t1, T2 t2) {82 volatile T1 storage1 = t1;83 volatile T2 storage2 = t2;84 T1 arg1 = storage1;85 T2 arg2 = storage2;86 87 cpp::atomic_thread_fence(cpp::MemoryOrder::ACQ_REL);88 uint64_t start = gpu::processor_clock();89 90 asm("" ::"s"(start));91 92 auto result = f(arg1, arg2);93 94 if constexpr (cpp::is_same_v<decltype(result), char> ||95 cpp::is_same_v<decltype(result), bool>)96 asm("v_or_b32 %[v_reg], 0, %[v_reg]\n" ::[v_reg] "v"(97 static_cast<uint32_t>(result)));98 else99 asm("v_or_b32 %[v_reg], 0, %[v_reg]\n" ::[v_reg] "v"(result));100 101 uint64_t stop = gpu::processor_clock();102 asm("" ::"s"(stop));103 cpp::atomic_thread_fence(cpp::MemoryOrder::ACQ_REL);104 105 return stop - start;106}107 108// Provides the *baseline* for throughput: measures loop and measurement costs109// without calling the f function110template <typename T, size_t N>111static LIBC_INLINE uint64_t112throughput_baseline(const cpp::array<T, N> &inputs) {113 asm("" ::"v"(&inputs));114 115 cpp::atomic_thread_fence(cpp::MemoryOrder::ACQ_REL);116 uint64_t start = gpu::processor_clock();117 asm("" ::"s"(start));118 119 T result{};120 121#pragma clang loop unroll(disable)122 for (auto input : inputs) {123 asm("" ::"v"(input));124 result = input;125 asm("" ::"v"(result));126 }127 128 uint64_t stop = gpu::processor_clock();129 asm("" ::"s"(stop));130 cpp::atomic_thread_fence(cpp::MemoryOrder::ACQ_REL);131 132 volatile auto output = result;133 134 return stop - start;135}136 137// Provides throughput benchmarking138template <typename F, typename T, size_t N>139static LIBC_INLINE uint64_t throughput(F f, const cpp::array<T, N> &inputs) {140 uint64_t baseline = UINT64_MAX;141 for (int i = 0; i < 5; ++i)142 baseline = cpp::min(baseline, throughput_baseline<T, N>(inputs));143 144 asm("" ::"v"(&inputs));145 146 cpp::atomic_thread_fence(cpp::MemoryOrder::ACQ_REL);147 uint64_t start = gpu::processor_clock();148 asm("" ::"s"(start));149 150 T result{};151 152#pragma clang loop unroll(disable)153 for (auto input : inputs) {154 asm("" ::"v"(input));155 result = f(input);156 asm("" ::"v"(result));157 }158 159 uint64_t stop = gpu::processor_clock();160 asm("" ::"s"(stop));161 cpp::atomic_thread_fence(cpp::MemoryOrder::ACQ_REL);162 163 volatile auto output = result;164 165 const uint64_t measured = stop - start;166 return measured > baseline ? (measured - baseline) : 0;167}168 169// Provides the *baseline* for throughput with 2 arguments: measures loop and170// measurement costs without calling the f function171template <typename T, size_t N>172static LIBC_INLINE uint64_t throughput_baseline(173 const cpp::array<T, N> &inputs1, const cpp::array<T, N> &inputs2) {174 asm("" ::"v"(&inputs1), "v"(&inputs2));175 176 cpp::atomic_thread_fence(cpp::MemoryOrder::ACQ_REL);177 uint64_t start = gpu::processor_clock();178 asm("" ::"s"(start));179 180 T result{};181 182#pragma clang loop unroll(disable)183 for (size_t i = 0; i < N; i++) {184 T x = inputs1[i];185 T y = inputs2[i];186 asm("" ::"v"(x), "v"(y));187 result = x;188 asm("" ::"v"(result));189 }190 191 uint64_t stop = gpu::processor_clock();192 asm("" ::"s"(stop));193 cpp::atomic_thread_fence(cpp::MemoryOrder::ACQ_REL);194 195 volatile auto output = result;196 197 return stop - start;198}199 200// Provides throughput benchmarking for 2 arguments (e.g. atan2())201template <typename F, typename T, size_t N>202static LIBC_INLINE uint64_t throughput(F f, const cpp::array<T, N> &inputs1,203 const cpp::array<T, N> &inputs2) {204 uint64_t baseline = UINT64_MAX;205 for (int i = 0; i < 5; ++i)206 baseline = cpp::min(baseline, throughput_baseline<T, N>(inputs1, inputs2));207 208 asm("" ::"v"(&inputs1), "v"(&inputs2));209 210 cpp::atomic_thread_fence(cpp::MemoryOrder::ACQ_REL);211 uint64_t start = gpu::processor_clock();212 asm("" ::"s"(start));213 214 T result{};215 216#pragma clang loop unroll(disable)217 for (size_t i = 0; i < N; i++) {218 T x = inputs1[i];219 T y = inputs2[i];220 asm("" ::"v"(x), "v"(y));221 result = f(x, y);222 asm("" ::"v"(result));223 }224 225 uint64_t stop = gpu::processor_clock();226 asm("" ::"s"(stop));227 cpp::atomic_thread_fence(cpp::MemoryOrder::ACQ_REL);228 229 volatile auto output = result;230 231 const uint64_t measured = stop - start;232 return measured > baseline ? (measured - baseline) : 0;233}234 235} // namespace LIBC_NAMESPACE_DECL236 237#endif // LLVM_LIBC_UTILS_GPU_TIMING_AMDGPU238