172 lines · cpp
1//===----------------------------------------------------------------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8 9// UNSUPPORTED: c++03, c++11, c++14, c++1710 11// This benchmark is very expensive and we don't want to run it on a regular basis,12// only to ensure the code doesn't rot.13// REQUIRES: enable-benchmarks=dry-run14 15#include "atomic_wait_helper.h"16 17#include <atomic>18#include <cstdint>19#include <numeric>20#include <stop_token>21#include <pthread.h>22#include <sched.h>23#include <thread>24#include <chrono>25#include <array>26 27#include "benchmark/benchmark.h"28#include "make_test_thread.h"29 30using namespace std::chrono_literals;31 32template <class NotifyPolicy, class NumberOfAtomics, class NumPrioTasks>33void BM_N_atomics_N_waiter_N_notifier(benchmark::State& state) {34 [[maybe_unused]] std::array<HighPrioTask, NumPrioTasks::value> tasks{};35 const std::uint64_t total_loop_test_param = state.range(0);36 constexpr std::uint64_t num_atomics = NumberOfAtomics::value;37 std::vector<std::atomic<std::uint64_t>> atomics(num_atomics);38 39 auto notify_func = [&](std::stop_token st, size_t idx) {40 while (!st.stop_requested()) {41 NotifyPolicy::notify(atomics[idx], st);42 }43 };44 45 std::atomic<std::uint64_t> start_flag = 0;46 std::atomic<std::uint64_t> done_count = 0;47 48 auto wait_func = [&, total_loop_test_param](std::stop_token st, size_t idx) {49 auto old_start = 0;50 while (!st.stop_requested()) {51 start_flag.wait(old_start);52 old_start = start_flag.load();53 for (std::uint64_t i = 0; i < total_loop_test_param; ++i) {54 auto old = atomics[idx].load(std::memory_order_relaxed);55 atomics[idx].wait(old);56 }57 done_count.fetch_add(1);58 }59 };60 61 std::vector<std::jthread> notify_threads;62 notify_threads.reserve(num_atomics);63 64 std::vector<std::jthread> wait_threads;65 wait_threads.reserve(num_atomics);66 67 for (size_t i = 0; i < num_atomics; ++i) {68 notify_threads.emplace_back(support::make_test_jthread(notify_func, i));69 }70 71 for (size_t i = 0; i < num_atomics; ++i) {72 wait_threads.emplace_back(support::make_test_jthread(wait_func, i));73 }74 75 for (auto _ : state) {76 done_count = 0;77 start_flag.fetch_add(1);78 start_flag.notify_all();79 while (done_count < num_atomics) {80 std::this_thread::yield();81 }82 }83 for (auto& t : wait_threads) {84 t.request_stop();85 }86 start_flag.fetch_add(1);87 start_flag.notify_all();88 for (auto& t : wait_threads) {89 t.join();90 }91}92 93BENCHMARK(BM_N_atomics_N_waiter_N_notifier<KeepNotifying, NumberOfAtomics<2>, NumHighPrioTasks<0>>)94 ->RangeMultiplier(2)95 ->Range(1 << 12, 1 << 14);96BENCHMARK(BM_N_atomics_N_waiter_N_notifier<KeepNotifying, NumberOfAtomics<3>, NumHighPrioTasks<0>>)97 ->RangeMultiplier(2)98 ->Range(1 << 10, 1 << 12);99BENCHMARK(BM_N_atomics_N_waiter_N_notifier<KeepNotifying, NumberOfAtomics<5>, NumHighPrioTasks<0>>)100 ->RangeMultiplier(2)101 ->Range(1 << 10, 1 << 12);102BENCHMARK(BM_N_atomics_N_waiter_N_notifier<KeepNotifying, NumberOfAtomics<7>, NumHighPrioTasks<0>>)103 ->RangeMultiplier(2)104 ->Range(1 << 8, 1 << 10);105 106BENCHMARK(BM_N_atomics_N_waiter_N_notifier<NotifyEveryNus<50>, NumberOfAtomics<2>, NumHighPrioTasks<0>>)107 ->RangeMultiplier(2)108 ->Range(1 << 10, 1 << 12);109BENCHMARK(BM_N_atomics_N_waiter_N_notifier<NotifyEveryNus<50>, NumberOfAtomics<3>, NumHighPrioTasks<0>>)110 ->RangeMultiplier(2)111 ->Range(1 << 8, 1 << 10);112BENCHMARK(BM_N_atomics_N_waiter_N_notifier<NotifyEveryNus<50>, NumberOfAtomics<5>, NumHighPrioTasks<0>>)113 ->RangeMultiplier(2)114 ->Range(1 << 8, 1 << 10);115BENCHMARK(BM_N_atomics_N_waiter_N_notifier<NotifyEveryNus<50>, NumberOfAtomics<7>, NumHighPrioTasks<0>>)116 ->RangeMultiplier(2)117 ->Range(1 << 6, 1 << 8);118 119BENCHMARK(BM_N_atomics_N_waiter_N_notifier<NotifyEveryNus<100>, NumberOfAtomics<2>, NumHighPrioTasks<0>>)120 ->RangeMultiplier(2)121 ->Range(1 << 8, 1 << 10);122BENCHMARK(BM_N_atomics_N_waiter_N_notifier<NotifyEveryNus<100>, NumberOfAtomics<3>, NumHighPrioTasks<0>>)123 ->RangeMultiplier(2)124 ->Range(1 << 8, 1 << 10);125BENCHMARK(BM_N_atomics_N_waiter_N_notifier<NotifyEveryNus<100>, NumberOfAtomics<5>, NumHighPrioTasks<0>>)126 ->RangeMultiplier(2)127 ->Range(1 << 7, 1 << 9);128BENCHMARK(BM_N_atomics_N_waiter_N_notifier<NotifyEveryNus<100>, NumberOfAtomics<7>, NumHighPrioTasks<0>>)129 ->RangeMultiplier(2)130 ->Range(1 << 6, 1 << 8);131 132BENCHMARK(BM_N_atomics_N_waiter_N_notifier<KeepNotifying, NumberOfAtomics<2>, NumHighPrioTasks<4>>)133 ->RangeMultiplier(2)134 ->Range(1 << 7, 1 << 9);135BENCHMARK(BM_N_atomics_N_waiter_N_notifier<KeepNotifying, NumberOfAtomics<3>, NumHighPrioTasks<4>>)136 ->RangeMultiplier(2)137 ->Range(1 << 7, 1 << 9);138BENCHMARK(BM_N_atomics_N_waiter_N_notifier<KeepNotifying, NumberOfAtomics<5>, NumHighPrioTasks<4>>)139 ->RangeMultiplier(2)140 ->Range(1 << 6, 1 << 8);141BENCHMARK(BM_N_atomics_N_waiter_N_notifier<KeepNotifying, NumberOfAtomics<7>, NumHighPrioTasks<4>>)142 ->RangeMultiplier(2)143 ->Range(1 << 4, 1 << 6);144 145BENCHMARK(BM_N_atomics_N_waiter_N_notifier<NotifyEveryNus<50>, NumberOfAtomics<2>, NumHighPrioTasks<4>>)146 ->RangeMultiplier(2)147 ->Range(1 << 7, 1 << 9);148BENCHMARK(BM_N_atomics_N_waiter_N_notifier<NotifyEveryNus<50>, NumberOfAtomics<3>, NumHighPrioTasks<4>>)149 ->RangeMultiplier(2)150 ->Range(1 << 7, 1 << 9);151BENCHMARK(BM_N_atomics_N_waiter_N_notifier<NotifyEveryNus<50>, NumberOfAtomics<5>, NumHighPrioTasks<4>>)152 ->RangeMultiplier(2)153 ->Range(1 << 5, 1 << 7);154BENCHMARK(BM_N_atomics_N_waiter_N_notifier<NotifyEveryNus<50>, NumberOfAtomics<7>, NumHighPrioTasks<4>>)155 ->RangeMultiplier(2)156 ->Range(1 << 3, 1 << 5);157 158BENCHMARK(BM_N_atomics_N_waiter_N_notifier<NotifyEveryNus<100>, NumberOfAtomics<2>, NumHighPrioTasks<4>>)159 ->RangeMultiplier(2)160 ->Range(1 << 6, 1 << 8);161BENCHMARK(BM_N_atomics_N_waiter_N_notifier<NotifyEveryNus<100>, NumberOfAtomics<3>, NumHighPrioTasks<4>>)162 ->RangeMultiplier(2)163 ->Range(1 << 6, 1 << 8);164BENCHMARK(BM_N_atomics_N_waiter_N_notifier<NotifyEveryNus<100>, NumberOfAtomics<5>, NumHighPrioTasks<4>>)165 ->RangeMultiplier(2)166 ->Range(1 << 5, 1 << 7);167BENCHMARK(BM_N_atomics_N_waiter_N_notifier<NotifyEveryNus<100>, NumberOfAtomics<7>, NumHighPrioTasks<4>>)168 ->RangeMultiplier(2)169 ->Range(1 << 3, 1 << 5);170 171BENCHMARK_MAIN();172