248 lines · cpp
1#include "LibcGpuBenchmark.h"2 3#include "hdr/stdint_proxy.h"4#include "src/__support/CPP/algorithm.h"5#include "src/__support/CPP/atomic.h"6#include "src/__support/CPP/string.h"7#include "src/__support/FPUtil/FPBits.h"8#include "src/__support/FPUtil/NearestIntegerOperations.h"9#include "src/__support/FPUtil/sqrt.h"10#include "src/__support/GPU/utils.h"11#include "src/__support/fixedvector.h"12#include "src/__support/macros/config.h"13#include "src/__support/time/gpu/time_utils.h"14#include "src/stdio/printf.h"15#include "src/time/clock.h"16 17namespace LIBC_NAMESPACE_DECL {18namespace benchmarks {19 20FixedVector<Benchmark *, 64> benchmarks;21 22void Benchmark::add_benchmark(Benchmark *benchmark) {23 benchmarks.push_back(benchmark);24}25 26static void atomic_add_double(cpp::Atomic<uint64_t> &atomic_bits,27 double value) {28 using FPBits = LIBC_NAMESPACE::fputil::FPBits<double>;29 30 uint64_t expected_bits = atomic_bits.load(cpp::MemoryOrder::RELAXED);31 32 while (true) {33 double current_value = FPBits(expected_bits).get_val();34 double next_value = current_value + value;35 36 uint64_t desired_bits = FPBits(next_value).uintval();37 if (atomic_bits.compare_exchange_strong(expected_bits, desired_bits,38 cpp::MemoryOrder::ACQUIRE,39 cpp::MemoryOrder::RELAXED))40 break;41 }42}43 44struct AtomicBenchmarkSums {45 cpp::Atomic<uint32_t> active_threads = 0;46 cpp::Atomic<uint64_t> iterations_sum = 0;47 cpp::Atomic<uint64_t> weighted_cycles_sum_bits = 0;48 cpp::Atomic<uint64_t> weighted_squared_cycles_sum_bits = 0;49 cpp::Atomic<uint64_t> min = UINT64_MAX;50 cpp::Atomic<uint64_t> max = 0;51 52 void reset() {53 cpp::atomic_thread_fence(cpp::MemoryOrder::RELEASE);54 active_threads.store(0, cpp::MemoryOrder::RELAXED);55 iterations_sum.store(0, cpp::MemoryOrder::RELAXED);56 weighted_cycles_sum_bits.store(0, cpp::MemoryOrder::RELAXED);57 weighted_squared_cycles_sum_bits.store(0, cpp::MemoryOrder::RELAXED);58 min.store(UINT64_MAX, cpp::MemoryOrder::RELAXED);59 max.store(0, cpp::MemoryOrder::RELAXED);60 cpp::atomic_thread_fence(cpp::MemoryOrder::RELEASE);61 }62 63 void update(const BenchmarkResult &result) {64 cpp::atomic_thread_fence(cpp::MemoryOrder::RELEASE);65 active_threads.fetch_add(1, cpp::MemoryOrder::RELAXED);66 iterations_sum.fetch_add(result.total_iterations,67 cpp::MemoryOrder::RELAXED);68 69 const double n_i = static_cast<double>(result.total_iterations);70 const double mean_i = result.cycles;71 const double stddev_i = result.standard_deviation;72 const double variance_i = stddev_i * stddev_i;73 atomic_add_double(weighted_cycles_sum_bits, n_i * mean_i);74 atomic_add_double(weighted_squared_cycles_sum_bits,75 n_i * (variance_i + mean_i * mean_i));76 77 // Perform a CAS loop to atomically update the min78 uint64_t orig_min = min.load(cpp::MemoryOrder::RELAXED);79 while (!min.compare_exchange_strong(80 orig_min, cpp::min(orig_min, result.min), cpp::MemoryOrder::ACQUIRE,81 cpp::MemoryOrder::RELAXED))82 ;83 84 // Perform a CAS loop to atomically update the max85 uint64_t orig_max = max.load(cpp::MemoryOrder::RELAXED);86 while (!max.compare_exchange_strong(87 orig_max, cpp::max(orig_max, result.max), cpp::MemoryOrder::ACQUIRE,88 cpp::MemoryOrder::RELAXED))89 ;90 91 cpp::atomic_thread_fence(cpp::MemoryOrder::RELEASE);92 }93};94 95AtomicBenchmarkSums all_results;96constexpr auto GREEN = "\033[32m";97constexpr auto RESET = "\033[0m";98 99void print_results(Benchmark *b) {100 using FPBits = LIBC_NAMESPACE::fputil::FPBits<double>;101 102 BenchmarkResult final_result;103 cpp::atomic_thread_fence(cpp::MemoryOrder::RELEASE);104 105 const uint32_t num_threads =106 all_results.active_threads.load(cpp::MemoryOrder::RELAXED);107 final_result.total_iterations =108 all_results.iterations_sum.load(cpp::MemoryOrder::RELAXED);109 110 if (final_result.total_iterations > 0) {111 const uint64_t s1_bits =112 all_results.weighted_cycles_sum_bits.load(cpp::MemoryOrder::RELAXED);113 const uint64_t s2_bits = all_results.weighted_squared_cycles_sum_bits.load(114 cpp::MemoryOrder::RELAXED);115 116 const double S1 = FPBits(s1_bits).get_val();117 const double S2 = FPBits(s2_bits).get_val();118 const double N = static_cast<double>(final_result.total_iterations);119 120 const double global_mean = S1 / N;121 const double global_mean_of_squares = S2 / N;122 const double global_variance =123 global_mean_of_squares - (global_mean * global_mean);124 125 final_result.cycles = global_mean;126 final_result.standard_deviation =127 fputil::sqrt<double>(global_variance < 0.0 ? 0.0 : global_variance);128 } else {129 final_result.cycles = 0.0;130 final_result.standard_deviation = 0.0;131 }132 133 final_result.min = all_results.min.load(cpp::MemoryOrder::RELAXED);134 final_result.max = all_results.max.load(cpp::MemoryOrder::RELAXED);135 cpp::atomic_thread_fence(cpp::MemoryOrder::RELEASE);136 137 LIBC_NAMESPACE::printf(138 "%-24s |%15.0f |%9.0f |%8llu |%8llu |%15llu |%9u |\n",139 b->get_test_name().data(), final_result.cycles,140 final_result.standard_deviation,141 static_cast<unsigned long long>(final_result.min),142 static_cast<unsigned long long>(final_result.max),143 static_cast<unsigned long long>(final_result.total_iterations),144 static_cast<unsigned>(num_threads));145}146 147void print_header() {148 LIBC_NAMESPACE::printf("%s", GREEN);149 LIBC_NAMESPACE::printf("Running Suite: %-10s\n",150 benchmarks[0]->get_suite_name().data());151 LIBC_NAMESPACE::printf("%s", RESET);152 cpp::string titles = "Benchmark | Cycles (Mean) | Stddev | "153 " Min | Max | Iterations | Threads |\n";154 LIBC_NAMESPACE::printf(titles.data());155 156 cpp::string separator(titles.size(), '-');157 separator[titles.size() - 1] = '\n';158 LIBC_NAMESPACE::printf(separator.data());159}160 161void Benchmark::run_benchmarks() {162 uint64_t id = gpu::get_thread_id();163 164 if (id == 0)165 print_header();166 167 gpu::sync_threads();168 169 for (Benchmark *b : benchmarks) {170 if (id == 0)171 all_results.reset();172 173 gpu::sync_threads();174 if (b->num_threads == static_cast<uint32_t>(-1) || id < b->num_threads) {175 auto current_result = b->run();176 all_results.update(current_result);177 }178 gpu::sync_threads();179 180 if (id == 0)181 print_results(b);182 }183 gpu::sync_threads();184}185 186BenchmarkResult benchmark(const BenchmarkOptions &options,187 const BenchmarkTarget &target) {188 BenchmarkResult result;189 RuntimeEstimationProgression rep;190 uint32_t iterations = options.initial_iterations;191 192 if (iterations < 1u)193 iterations = 1;194 195 uint32_t samples = 0;196 uint64_t total_time = 0;197 uint64_t min = UINT64_MAX;198 uint64_t max = 0;199 200 uint32_t call_index = 0;201 202 for (int64_t time_budget = options.max_duration; time_budget >= 0;) {203 RefinableRuntimeEstimator sample_estimator;204 205 const clock_t start = clock();206 while (sample_estimator.get_iterations() < iterations) {207 auto current_result = target(call_index++);208 max = cpp::max(max, current_result);209 min = cpp::min(min, current_result);210 sample_estimator.update(current_result);211 }212 const clock_t end = clock();213 214 const clock_t duration_ns =215 ((end - start) * 1000 * 1000 * 1000) / CLOCKS_PER_SEC;216 total_time += duration_ns;217 time_budget -= duration_ns;218 samples++;219 220 const double change_ratio = rep.compute_improvement(sample_estimator);221 222 if (samples >= options.max_samples || iterations >= options.max_iterations)223 break;224 225 const auto total_iterations = rep.get_estimator().get_iterations();226 227 if (total_time >= options.min_duration && samples >= options.min_samples &&228 total_iterations >= options.min_iterations &&229 change_ratio < options.epsilon)230 break;231 232 iterations = static_cast<uint32_t>(233 fputil::ceil(iterations * options.scaling_factor));234 }235 236 const auto &estimator = rep.get_estimator();237 result.total_iterations = estimator.get_iterations();238 result.cycles = estimator.get_mean();239 result.standard_deviation = estimator.get_stddev();240 result.min = min;241 result.max = max;242 243 return result;244}245 246} // namespace benchmarks247} // namespace LIBC_NAMESPACE_DECL248