200 lines · plain
1 2#undef NDEBUG3#include <cassert>4#include <vector>5 6#include "../src/check.h" // NOTE: check.h is for internal use only!7#include "benchmark/benchmark.h"8 9namespace {10 11class TestReporter : public benchmark::ConsoleReporter {12 public:13 bool ReportContext(const Context& context) override {14 return ConsoleReporter::ReportContext(context);15 };16 17 void ReportRuns(const std::vector<Run>& report) override {18 all_runs_.insert(all_runs_.end(), begin(report), end(report));19 ConsoleReporter::ReportRuns(report);20 }21 22 TestReporter() {}23 ~TestReporter() override {}24 25 mutable std::vector<Run> all_runs_;26};27 28struct TestCase {29 std::string name;30 bool error_occurred;31 std::string error_message;32 33 typedef benchmark::BenchmarkReporter::Run Run;34 35 void CheckRun(Run const& run) const {36 BM_CHECK(name == run.benchmark_name())37 << "expected " << name << " got " << run.benchmark_name();38 BM_CHECK_EQ(error_occurred,39 benchmark::internal::SkippedWithError == run.skipped);40 BM_CHECK(error_message == run.skip_message);41 if (error_occurred) {42 // BM_CHECK(run.iterations == 0);43 } else {44 BM_CHECK(run.iterations != 0);45 }46 }47};48 49std::vector<TestCase> ExpectedResults;50 51int AddCases(const std::string& base_name,52 std::initializer_list<TestCase> const& v) {53 for (auto TC : v) {54 TC.name = base_name + TC.name;55 ExpectedResults.push_back(std::move(TC));56 }57 return 0;58}59 60#define CONCAT(x, y) CONCAT2(x, y)61#define CONCAT2(x, y) x##y62#define ADD_CASES(...) int CONCAT(dummy, __LINE__) = AddCases(__VA_ARGS__)63 64} // end namespace65 66void BM_error_no_running(benchmark::State& state) {67 state.SkipWithError("error message");68}69BENCHMARK(BM_error_no_running);70ADD_CASES("BM_error_no_running", {{"", true, "error message"}});71 72void BM_error_before_running(benchmark::State& state) {73 state.SkipWithError("error message");74 while (state.KeepRunning()) {75 assert(false);76 }77}78BENCHMARK(BM_error_before_running);79ADD_CASES("BM_error_before_running", {{"", true, "error message"}});80 81void BM_error_before_running_batch(benchmark::State& state) {82 state.SkipWithError("error message");83 while (state.KeepRunningBatch(17)) {84 assert(false);85 }86}87BENCHMARK(BM_error_before_running_batch);88ADD_CASES("BM_error_before_running_batch", {{"", true, "error message"}});89 90void BM_error_before_running_range_for(benchmark::State& state) {91 state.SkipWithError("error message");92 for (auto _ : state) {93 assert(false);94 }95}96BENCHMARK(BM_error_before_running_range_for);97ADD_CASES("BM_error_before_running_range_for", {{"", true, "error message"}});98 99void BM_error_during_running(benchmark::State& state) {100 int first_iter = true;101 while (state.KeepRunning()) {102 if (state.range(0) == 1 && state.thread_index() <= (state.threads() / 2)) {103 assert(first_iter);104 first_iter = false;105 state.SkipWithError("error message");106 } else {107 state.PauseTiming();108 state.ResumeTiming();109 }110 }111}112BENCHMARK(BM_error_during_running)->Arg(1)->Arg(2)->ThreadRange(1, 8);113ADD_CASES("BM_error_during_running", {{"/1/threads:1", true, "error message"},114 {"/1/threads:2", true, "error message"},115 {"/1/threads:4", true, "error message"},116 {"/1/threads:8", true, "error message"},117 {"/2/threads:1", false, ""},118 {"/2/threads:2", false, ""},119 {"/2/threads:4", false, ""},120 {"/2/threads:8", false, ""}});121 122void BM_error_during_running_ranged_for(benchmark::State& state) {123 assert(state.max_iterations > 3 && "test requires at least a few iterations");124 bool first_iter = true;125 // NOTE: Users should not write the for loop explicitly.126 for (auto It = state.begin(), End = state.end(); It != End; ++It) {127 if (state.range(0) == 1) {128 assert(first_iter);129 first_iter = false;130 (void)first_iter;131 state.SkipWithError("error message");132 // Test the unfortunate but documented behavior that the ranged-for loop133 // doesn't automatically terminate when SkipWithError is set.134 assert(++It != End);135 break; // Required behavior136 }137 }138}139BENCHMARK(BM_error_during_running_ranged_for)->Arg(1)->Arg(2)->Iterations(5);140ADD_CASES("BM_error_during_running_ranged_for",141 {{"/1/iterations:5", true, "error message"},142 {"/2/iterations:5", false, ""}});143 144void BM_error_after_running(benchmark::State& state) {145 for (auto _ : state) {146 auto iterations = double(state.iterations()) * double(state.iterations());147 benchmark::DoNotOptimize(iterations);148 }149 if (state.thread_index() <= (state.threads() / 2))150 state.SkipWithError("error message");151}152BENCHMARK(BM_error_after_running)->ThreadRange(1, 8);153ADD_CASES("BM_error_after_running", {{"/threads:1", true, "error message"},154 {"/threads:2", true, "error message"},155 {"/threads:4", true, "error message"},156 {"/threads:8", true, "error message"}});157 158void BM_error_while_paused(benchmark::State& state) {159 bool first_iter = true;160 while (state.KeepRunning()) {161 if (state.range(0) == 1 && state.thread_index() <= (state.threads() / 2)) {162 assert(first_iter);163 first_iter = false;164 state.PauseTiming();165 state.SkipWithError("error message");166 } else {167 state.PauseTiming();168 state.ResumeTiming();169 }170 }171}172BENCHMARK(BM_error_while_paused)->Arg(1)->Arg(2)->ThreadRange(1, 8);173ADD_CASES("BM_error_while_paused", {{"/1/threads:1", true, "error message"},174 {"/1/threads:2", true, "error message"},175 {"/1/threads:4", true, "error message"},176 {"/1/threads:8", true, "error message"},177 {"/2/threads:1", false, ""},178 {"/2/threads:2", false, ""},179 {"/2/threads:4", false, ""},180 {"/2/threads:8", false, ""}});181 182int main(int argc, char* argv[]) {183 benchmark::Initialize(&argc, argv);184 185 TestReporter test_reporter;186 benchmark::RunSpecifiedBenchmarks(&test_reporter);187 188 typedef benchmark::BenchmarkReporter::Run Run;189 auto EB = ExpectedResults.begin();190 191 for (Run const& run : test_reporter.all_runs_) {192 assert(EB != ExpectedResults.end());193 EB->CheckRun(run);194 ++EB;195 }196 assert(EB == ExpectedResults.end());197 198 return 0;199}200