248 lines · c
1#ifndef LLVM_LIBC_BENCHMARKS_LIBC_GPU_BENCHMARK_H2#define LLVM_LIBC_BENCHMARKS_LIBC_GPU_BENCHMARK_H3 4#include "benchmarks/gpu/Random.h"5 6#include "benchmarks/gpu/timing/timing.h"7 8#include "hdr/stdint_proxy.h"9#include "src/__support/CPP/algorithm.h"10#include "src/__support/CPP/array.h"11#include "src/__support/CPP/string_view.h"12#include "src/__support/CPP/type_traits.h"13#include "src/__support/FPUtil/FPBits.h"14#include "src/__support/FPUtil/sqrt.h"15#include "src/__support/macros/config.h"16 17namespace LIBC_NAMESPACE_DECL {18 19namespace benchmarks {20 21struct BenchmarkOptions {22 uint32_t initial_iterations = 1;23 uint32_t min_iterations = 1;24 uint32_t max_iterations = 10000000;25 uint32_t min_samples = 4;26 uint32_t max_samples = 1000;27 int64_t min_duration = 500 * 1000; // 500 * 1000 nanoseconds = 500 us28 int64_t max_duration = 1000 * 1000 * 1000; // 1e9 nanoseconds = 1 second29 double epsilon = 0.0001;30 double scaling_factor = 1.4;31};32 33class RefinableRuntimeEstimator {34 uint32_t iterations = 0;35 uint64_t sum_of_cycles = 0;36 uint64_t sum_of_squared_cycles = 0;37 38public:39 void update(uint64_t cycles) noexcept {40 iterations += 1;41 sum_of_cycles += cycles;42 sum_of_squared_cycles += cycles * cycles;43 }44 45 void update(const RefinableRuntimeEstimator &other) noexcept {46 iterations += other.iterations;47 sum_of_cycles += other.sum_of_cycles;48 sum_of_squared_cycles += other.sum_of_squared_cycles;49 }50 51 double get_mean() const noexcept {52 if (iterations == 0)53 return 0.0;54 55 return static_cast<double>(sum_of_cycles) / iterations;56 }57 58 double get_variance() const noexcept {59 if (iterations == 0)60 return 0.0;61 62 const double num = static_cast<double>(iterations);63 const double sum_x = static_cast<double>(sum_of_cycles);64 const double sum_x2 = static_cast<double>(sum_of_squared_cycles);65 66 const double mean_of_squares = sum_x2 / num;67 const double mean = sum_x / num;68 const double mean_squared = mean * mean;69 const double variance = mean_of_squares - mean_squared;70 71 return variance < 0.0 ? 0.0 : variance;72 }73 74 double get_stddev() const noexcept {75 return fputil::sqrt<double>(get_variance());76 }77 78 uint32_t get_iterations() const noexcept { return iterations; }79};80 81// Tracks the progression of the runtime estimation82class RuntimeEstimationProgression {83 RefinableRuntimeEstimator estimator;84 double current_mean = 0.0;85 86public:87 const RefinableRuntimeEstimator &get_estimator() const noexcept {88 return estimator;89 }90 91 double92 compute_improvement(const RefinableRuntimeEstimator &sample_estimator) {93 if (sample_estimator.get_iterations() == 0)94 return 1.0;95 96 estimator.update(sample_estimator);97 98 const double new_mean = estimator.get_mean();99 if (current_mean == 0.0 || new_mean == 0.0) {100 current_mean = new_mean;101 return 1.0;102 }103 104 double ratio = (current_mean / new_mean) - 1.0;105 if (ratio < 0)106 ratio = -ratio;107 108 current_mean = new_mean;109 return ratio;110 }111};112 113struct BenchmarkResult {114 uint64_t total_iterations = 0;115 double cycles = 0;116 double standard_deviation = 0;117 uint64_t min = UINT64_MAX;118 uint64_t max = 0;119};120 121struct BenchmarkTarget {122 using IndexedFnPtr = uint64_t (*)(uint32_t);123 using IndexlessFnPtr = uint64_t (*)();124 125 enum class Kind : uint8_t { Indexed, Indexless } kind;126 union {127 IndexedFnPtr indexed_fn_ptr;128 IndexlessFnPtr indexless_fn_ptr;129 };130 131 LIBC_INLINE BenchmarkTarget(IndexedFnPtr func)132 : kind(Kind::Indexed), indexed_fn_ptr(func) {}133 LIBC_INLINE BenchmarkTarget(IndexlessFnPtr func)134 : kind(Kind::Indexless), indexless_fn_ptr(func) {}135 136 LIBC_INLINE uint64_t operator()([[maybe_unused]] uint32_t call_index) const {137 return kind == Kind::Indexed ? indexed_fn_ptr(call_index)138 : indexless_fn_ptr();139 }140};141 142BenchmarkResult benchmark(const BenchmarkOptions &options,143 const BenchmarkTarget &target);144 145class Benchmark {146 const BenchmarkTarget target;147 const cpp::string_view suite_name;148 const cpp::string_view test_name;149 const uint32_t num_threads;150 151public:152 Benchmark(uint64_t (*f)(), const char *suite, const char *test,153 uint32_t threads)154 : target(BenchmarkTarget(f)), suite_name(suite), test_name(test),155 num_threads(threads) {156 add_benchmark(this);157 }158 159 Benchmark(uint64_t (*f)(uint32_t), char const *suite_name,160 char const *test_name, uint32_t num_threads)161 : target(BenchmarkTarget(f)), suite_name(suite_name),162 test_name(test_name), num_threads(num_threads) {163 add_benchmark(this);164 }165 166 static void run_benchmarks();167 const cpp::string_view get_suite_name() const { return suite_name; }168 const cpp::string_view get_test_name() const { return test_name; }169 170protected:171 static void add_benchmark(Benchmark *benchmark);172 173private:174 BenchmarkResult run() {175 BenchmarkOptions options;176 return benchmark(options, target);177 }178};179 180template <typename T> class MathPerf {181 static LIBC_INLINE uint64_t make_seed(uint64_t base_seed, uint64_t salt) {182 const uint64_t tid = gpu::get_thread_id();183 return base_seed ^ (salt << 32) ^ (tid * 0x9E3779B97F4A7C15ULL);184 }185 186public:187 // Returns cycles-per-call (lower is better)188 template <size_t N = 1, typename Dist>189 static uint64_t run_throughput(T (*f)(T), const Dist &dist,190 uint32_t call_index) {191 cpp::array<T, N> inputs;192 193 uint64_t base_seed = static_cast<uint64_t>(call_index);194 uint64_t salt = static_cast<uint64_t>(N);195 RandomGenerator rng(make_seed(base_seed, salt));196 197 for (size_t i = 0; i < N; ++i)198 inputs[i] = dist(rng);199 200 uint64_t total_time = LIBC_NAMESPACE::throughput(f, inputs);201 202 return total_time / N;203 }204 205 // Returns cycles-per-call (lower is better)206 template <size_t N = 1, typename Dist1, typename Dist2>207 static uint64_t run_throughput(T (*f)(T, T), const Dist1 &dist1,208 const Dist2 &dist2, uint32_t call_index) {209 cpp::array<T, N> inputs1;210 cpp::array<T, N> inputs2;211 212 uint64_t base_seed = static_cast<uint64_t>(call_index);213 uint64_t salt = static_cast<uint64_t>(N);214 RandomGenerator rng(make_seed(base_seed, salt));215 216 for (size_t i = 0; i < N; ++i) {217 inputs1[i] = dist1(rng);218 inputs2[i] = dist2(rng);219 }220 221 uint64_t total_time = LIBC_NAMESPACE::throughput(f, inputs1, inputs2);222 223 return total_time / N;224 }225};226 227} // namespace benchmarks228} // namespace LIBC_NAMESPACE_DECL229 230// Passing -1 indicates the benchmark should be run with as many threads as231// allocated by the user in the benchmark's CMake.232#define BENCHMARK(SuiteName, TestName, Func) \233 LIBC_NAMESPACE::benchmarks::Benchmark SuiteName##_##TestName##_Instance( \234 Func, #SuiteName, #TestName, -1)235 236#define BENCHMARK_N_THREADS(SuiteName, TestName, Func, NumThreads) \237 LIBC_NAMESPACE::benchmarks::Benchmark SuiteName##_##TestName##_Instance( \238 Func, #SuiteName, #TestName, NumThreads)239 240#define SINGLE_THREADED_BENCHMARK(SuiteName, TestName, Func) \241 BENCHMARK_N_THREADS(SuiteName, TestName, Func, 1)242 243#define SINGLE_WAVE_BENCHMARK(SuiteName, TestName, Func) \244 BENCHMARK_N_THREADS(SuiteName, TestName, Func, \245 LIBC_NAMESPACE::gpu::get_lane_size())246 247#endif // LLVM_LIBC_BENCHMARKS_LIBC_GPU_BENCHMARK_H248