227 lines · c
1//===------------- NVPTX implementation of timing utils ---------*- C++ -*-===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8 9#ifndef LLVM_LIBC_UTILS_GPU_TIMING_NVPTX10#define LLVM_LIBC_UTILS_GPU_TIMING_NVPTX11 12#include "hdr/stdint_proxy.h"13#include "src/__support/CPP/algorithm.h"14#include "src/__support/CPP/array.h"15#include "src/__support/CPP/atomic.h"16#include "src/__support/GPU/utils.h"17#include "src/__support/macros/attributes.h"18#include "src/__support/macros/config.h"19 20namespace LIBC_NAMESPACE_DECL {21 22// Returns the overhead associated with calling the profiling region. This23// allows us to substract the constant-time overhead from the latency to24// obtain a true result. This can vary with system load.25[[gnu::noinline]] static uint64_t overhead() {26 volatile uint32_t x = 1;27 uint32_t y = x;28 uint64_t start = gpu::processor_clock();29 asm("" ::"llr"(start));30 uint32_t result = y;31 asm("or.b32 %[v_reg], %[v_reg], 0;" ::[v_reg] "r"(result));32 uint64_t stop = gpu::processor_clock();33 volatile auto storage = result;34 return stop - start;35}36 37// Stimulate a simple function and obtain its latency in clock cycles on the38// system. This function cannot be inlined or else it will disturb the very39// delicate balance of hard-coded dependencies.40template <typename F, typename T>41[[gnu::noinline]] static LIBC_INLINE uint64_t latency(F f, T t) {42 // We need to store the input somewhere to guarantee that the compiler will43 // not constant propagate it and remove the profiling region.44 volatile T storage = t;45 T arg = storage;46 47 // Get the current timestamp from the clock.48 cpp::atomic_thread_fence(cpp::MemoryOrder::ACQ_REL);49 uint64_t start = gpu::processor_clock();50 51 // This forces the compiler to load the input argument and run the clock cycle52 // counter before the profiling region.53 asm("" ::"llr"(start));54 55 // Run the function under test and return its value.56 auto result = f(arg);57 58 // This inline assembly performs a no-op which forces the result to both be59 // used and prevents us from exiting this region before it's complete.60 asm("or.b32 %[v_reg], %[v_reg], 0;" ::[v_reg] "r"(result));61 62 // Obtain the current timestamp after running the calculation and force63 // ordering.64 uint64_t stop = gpu::processor_clock();65 cpp::atomic_thread_fence(cpp::MemoryOrder::ACQ_REL);66 asm("" ::"r"(stop));67 volatile auto output = result;68 69 // Return the time elapsed.70 return stop - start;71}72 73template <typename F, typename T1, typename T2>74static LIBC_INLINE uint64_t latency(F f, T1 t1, T2 t2) {75 volatile T1 storage = t1;76 volatile T2 storage2 = t2;77 T1 arg = storage;78 T2 arg2 = storage2;79 80 cpp::atomic_thread_fence(cpp::MemoryOrder::ACQ_REL);81 uint64_t start = gpu::processor_clock();82 83 asm("" ::"llr"(start));84 85 auto result = f(arg, arg2);86 87 asm("or.b32 %[v_reg], %[v_reg], 0;" ::[v_reg] "r"(result));88 89 uint64_t stop = gpu::processor_clock();90 cpp::atomic_thread_fence(cpp::MemoryOrder::ACQ_REL);91 asm("" ::"r"(stop));92 volatile auto output = result;93 94 return stop - start;95}96 97// Provides the *baseline* for throughput: measures loop and measurement costs98// without calling the f function99template <typename T, size_t N>100static LIBC_INLINE uint64_t101throughput_baseline(const cpp::array<T, N> &inputs) {102 asm("" ::"r"(&inputs));103 104 cpp::atomic_thread_fence(cpp::MemoryOrder::ACQ_REL);105 uint64_t start = gpu::processor_clock();106 asm("" ::"llr"(start));107 108 T result{};109 110#pragma clang loop unroll(disable)111 for (auto input : inputs) {112 asm("" ::"r"(input));113 result = input;114 asm("" ::"r"(result));115 }116 117 uint64_t stop = gpu::processor_clock();118 asm("" ::"r"(stop));119 cpp::atomic_thread_fence(cpp::MemoryOrder::ACQ_REL);120 121 volatile auto output = result;122 123 return stop - start;124}125 126// Provides throughput benchmarking127template <typename F, typename T, size_t N>128static LIBC_INLINE uint64_t throughput(F f, const cpp::array<T, N> &inputs) {129 uint64_t baseline = UINT64_MAX;130 for (int i = 0; i < 5; ++i)131 baseline = cpp::min(baseline, throughput_baseline<T, N>(inputs));132 133 asm("" ::"r"(&inputs));134 135 cpp::atomic_thread_fence(cpp::MemoryOrder::ACQ_REL);136 uint64_t start = gpu::processor_clock();137 asm("" ::"llr"(start));138 139 T result{};140 141#pragma clang loop unroll(disable)142 for (auto input : inputs) {143 asm("" ::"r"(input));144 result = f(input);145 asm("" ::"r"(result));146 }147 148 uint64_t stop = gpu::processor_clock();149 asm("" ::"r"(stop));150 cpp::atomic_thread_fence(cpp::MemoryOrder::ACQ_REL);151 152 volatile auto output = result;153 154 const uint64_t measured = stop - start;155 return measured > baseline ? (measured - baseline) : 0;156}157 158// Provides the *baseline* for throughput with 2 arguments: measures loop and159// measurement costs without calling the f function160template <typename T, size_t N>161static LIBC_INLINE uint64_t throughput_baseline(162 const cpp::array<T, N> &inputs1, const cpp::array<T, N> &inputs2) {163 asm("" ::"r"(&inputs1), "r"(&inputs2));164 165 cpp::atomic_thread_fence(cpp::MemoryOrder::ACQ_REL);166 uint64_t start = gpu::processor_clock();167 asm("" ::"llr"(start));168 169 T result{};170 171#pragma clang loop unroll(disable)172 for (size_t i = 0; i < N; i++) {173 T x = inputs1[i];174 T y = inputs2[i];175 asm("" ::"r"(x), "r"(y));176 result = x;177 asm("" ::"r"(result));178 }179 180 uint64_t stop = gpu::processor_clock();181 asm("" ::"r"(stop));182 cpp::atomic_thread_fence(cpp::MemoryOrder::ACQ_REL);183 184 volatile auto output = result;185 186 return stop - start;187}188 189// Provides throughput benchmarking for 2 arguments (e.g. atan2())190template <typename F, typename T, size_t N>191static LIBC_INLINE uint64_t throughput(F f, const cpp::array<T, N> &inputs1,192 const cpp::array<T, N> &inputs2) {193 uint64_t baseline = UINT64_MAX;194 for (int i = 0; i < 5; ++i)195 baseline = cpp::min(baseline, throughput_baseline<T, N>(inputs1, inputs2));196 197 asm("" ::"r"(&inputs1), "r"(&inputs2));198 199 cpp::atomic_thread_fence(cpp::MemoryOrder::ACQ_REL);200 uint64_t start = gpu::processor_clock();201 asm("" ::"llr"(start));202 203 T result{};204 205#pragma clang loop unroll(disable)206 for (size_t i = 0; i < N; i++) {207 T x = inputs1[i];208 T y = inputs2[i];209 asm("" ::"r"(x), "r"(y));210 result = f(x, y);211 asm("" ::"r"(result));212 }213 214 uint64_t stop = gpu::processor_clock();215 asm("" ::"r"(stop));216 cpp::atomic_thread_fence(cpp::MemoryOrder::ACQ_REL);217 218 volatile auto output = result;219 220 const uint64_t measured = stop - start;221 return measured > baseline ? (measured - baseline) : 0;222}223 224} // namespace LIBC_NAMESPACE_DECL225 226#endif // LLVM_LIBC_UTILS_GPU_TIMING_NVPTX227