1134 lines · plain
1 2#undef NDEBUG3#include <numeric>4#include <utility>5 6#include "benchmark/benchmark.h"7#include "output_test.h"8 9// ========================================================================= //10// ---------------------- Testing Prologue Output -------------------------- //11// ========================================================================= //12 13ADD_CASES(TC_ConsoleOut, {{"^[-]+$", MR_Next},14 {"^Benchmark %s Time %s CPU %s Iterations$", MR_Next},15 {"^[-]+$", MR_Next}});16static int AddContextCases() {17 AddCases(TC_ConsoleErr,18 {19 {"^%int-%int-%intT%int:%int:%int[-+]%int:%int$", MR_Default},20 {"Running .*(/|\\\\)reporter_output_test(\\.exe)?$", MR_Next},21 {"Run on \\(%int X %float MHz CPU s?\\)", MR_Next},22 });23 AddCases(TC_JSONOut,24 {{"^\\{", MR_Default},25 {"\"context\":", MR_Next},26 {"\"date\": \"", MR_Next},27 {"\"host_name\":", MR_Next},28 {"\"executable\": \".*(/|\\\\)reporter_output_test(\\.exe)?\",",29 MR_Next},30 {"\"num_cpus\": %int,$", MR_Next},31 {"\"mhz_per_cpu\": %float,$", MR_Next},32 {"\"caches\": \\[$", MR_Default}});33 auto const& Info = benchmark::CPUInfo::Get();34 auto const& Caches = Info.caches;35 if (!Caches.empty()) {36 AddCases(TC_ConsoleErr, {{"CPU Caches:$", MR_Next}});37 }38 for (size_t I = 0; I < Caches.size(); ++I) {39 std::string num_caches_str =40 Caches[I].num_sharing != 0 ? " \\(x%int\\)$" : "$";41 AddCases(TC_ConsoleErr,42 {{"L%int (Data|Instruction|Unified) %int KiB" + num_caches_str,43 MR_Next}});44 AddCases(TC_JSONOut, {{"\\{$", MR_Next},45 {"\"type\": \"", MR_Next},46 {"\"level\": %int,$", MR_Next},47 {"\"size\": %int,$", MR_Next},48 {"\"num_sharing\": %int$", MR_Next},49 {"}[,]{0,1}$", MR_Next}});50 }51 AddCases(TC_JSONOut, {{"],$"}});52 auto const& LoadAvg = Info.load_avg;53 if (!LoadAvg.empty()) {54 AddCases(TC_ConsoleErr,55 {{"Load Average: (%float, ){0,2}%float$", MR_Next}});56 }57 AddCases(TC_JSONOut, {{"\"load_avg\": \\[(%float,?){0,3}],$", MR_Next}});58 AddCases(TC_JSONOut, {{"\"library_version\": \".*\",$", MR_Next}});59 AddCases(TC_JSONOut, {{"\"library_build_type\": \".*\",$", MR_Next}});60 AddCases(TC_JSONOut, {{"\"json_schema_version\": 1$", MR_Next}});61 return 0;62}63int dummy_register = AddContextCases();64ADD_CASES(TC_CSVOut, {{"%csv_header"}});65 66// ========================================================================= //67// ------------------------ Testing Basic Output --------------------------- //68// ========================================================================= //69 70void BM_basic(benchmark::State& state) {71 for (auto _ : state) {72 }73}74BENCHMARK(BM_basic);75 76ADD_CASES(TC_ConsoleOut, {{"^BM_basic %console_report$"}});77ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_basic\",$"},78 {"\"family_index\": 0,$", MR_Next},79 {"\"per_family_instance_index\": 0,$", MR_Next},80 {"\"run_name\": \"BM_basic\",$", MR_Next},81 {"\"run_type\": \"iteration\",$", MR_Next},82 {"\"repetitions\": 1,$", MR_Next},83 {"\"repetition_index\": 0,$", MR_Next},84 {"\"threads\": 1,$", MR_Next},85 {"\"iterations\": %int,$", MR_Next},86 {"\"real_time\": %float,$", MR_Next},87 {"\"cpu_time\": %float,$", MR_Next},88 {"\"time_unit\": \"ns\"$", MR_Next},89 {"}", MR_Next}});90ADD_CASES(TC_CSVOut, {{"^\"BM_basic\",%csv_report$"}});91 92// ========================================================================= //93// ------------------------ Testing Bytes per Second Output ---------------- //94// ========================================================================= //95 96void BM_bytes_per_second(benchmark::State& state) {97 for (auto _ : state) {98 // This test requires a non-zero CPU time to avoid divide-by-zero99 auto iterations = double(state.iterations()) * double(state.iterations());100 benchmark::DoNotOptimize(iterations);101 }102 state.SetBytesProcessed(1);103}104BENCHMARK(BM_bytes_per_second);105 106ADD_CASES(TC_ConsoleOut, {{"^BM_bytes_per_second %console_report "107 "bytes_per_second=%float[kM]{0,1}/s$"}});108ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_bytes_per_second\",$"},109 {"\"family_index\": 1,$", MR_Next},110 {"\"per_family_instance_index\": 0,$", MR_Next},111 {"\"run_name\": \"BM_bytes_per_second\",$", MR_Next},112 {"\"run_type\": \"iteration\",$", MR_Next},113 {"\"repetitions\": 1,$", MR_Next},114 {"\"repetition_index\": 0,$", MR_Next},115 {"\"threads\": 1,$", MR_Next},116 {"\"iterations\": %int,$", MR_Next},117 {"\"real_time\": %float,$", MR_Next},118 {"\"cpu_time\": %float,$", MR_Next},119 {"\"time_unit\": \"ns\",$", MR_Next},120 {"\"bytes_per_second\": %float$", MR_Next},121 {"}", MR_Next}});122ADD_CASES(TC_CSVOut, {{"^\"BM_bytes_per_second\",%csv_bytes_report$"}});123 124// ========================================================================= //125// ------------------------ Testing Items per Second Output ---------------- //126// ========================================================================= //127 128void BM_items_per_second(benchmark::State& state) {129 for (auto _ : state) {130 // This test requires a non-zero CPU time to avoid divide-by-zero131 auto iterations = double(state.iterations()) * double(state.iterations());132 benchmark::DoNotOptimize(iterations);133 }134 state.SetItemsProcessed(1);135}136BENCHMARK(BM_items_per_second);137 138ADD_CASES(TC_ConsoleOut, {{"^BM_items_per_second %console_report "139 "items_per_second=%float[kM]{0,1}/s$"}});140ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_items_per_second\",$"},141 {"\"family_index\": 2,$", MR_Next},142 {"\"per_family_instance_index\": 0,$", MR_Next},143 {"\"run_name\": \"BM_items_per_second\",$", MR_Next},144 {"\"run_type\": \"iteration\",$", MR_Next},145 {"\"repetitions\": 1,$", MR_Next},146 {"\"repetition_index\": 0,$", MR_Next},147 {"\"threads\": 1,$", MR_Next},148 {"\"iterations\": %int,$", MR_Next},149 {"\"real_time\": %float,$", MR_Next},150 {"\"cpu_time\": %float,$", MR_Next},151 {"\"time_unit\": \"ns\",$", MR_Next},152 {"\"items_per_second\": %float$", MR_Next},153 {"}", MR_Next}});154ADD_CASES(TC_CSVOut, {{"^\"BM_items_per_second\",%csv_items_report$"}});155 156// ========================================================================= //157// ------------------------ Testing Label Output --------------------------- //158// ========================================================================= //159 160void BM_label(benchmark::State& state) {161 for (auto _ : state) {162 }163 state.SetLabel("some label");164}165BENCHMARK(BM_label);166 167ADD_CASES(TC_ConsoleOut, {{"^BM_label %console_report some label$"}});168ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_label\",$"},169 {"\"family_index\": 3,$", MR_Next},170 {"\"per_family_instance_index\": 0,$", MR_Next},171 {"\"run_name\": \"BM_label\",$", MR_Next},172 {"\"run_type\": \"iteration\",$", MR_Next},173 {"\"repetitions\": 1,$", MR_Next},174 {"\"repetition_index\": 0,$", MR_Next},175 {"\"threads\": 1,$", MR_Next},176 {"\"iterations\": %int,$", MR_Next},177 {"\"real_time\": %float,$", MR_Next},178 {"\"cpu_time\": %float,$", MR_Next},179 {"\"time_unit\": \"ns\",$", MR_Next},180 {"\"label\": \"some label\"$", MR_Next},181 {"}", MR_Next}});182ADD_CASES(TC_CSVOut, {{"^\"BM_label\",%csv_label_report_begin\"some "183 "label\"%csv_label_report_end$"}});184 185// ========================================================================= //186// ------------------------ Testing Time Label Output ---------------------- //187// ========================================================================= //188 189void BM_time_label_nanosecond(benchmark::State& state) {190 for (auto _ : state) {191 }192}193BENCHMARK(BM_time_label_nanosecond)->Unit(benchmark::kNanosecond);194 195ADD_CASES(TC_ConsoleOut, {{"^BM_time_label_nanosecond %console_report$"}});196ADD_CASES(TC_JSONOut,197 {{"\"name\": \"BM_time_label_nanosecond\",$"},198 {"\"family_index\": 4,$", MR_Next},199 {"\"per_family_instance_index\": 0,$", MR_Next},200 {"\"run_name\": \"BM_time_label_nanosecond\",$", MR_Next},201 {"\"run_type\": \"iteration\",$", MR_Next},202 {"\"repetitions\": 1,$", MR_Next},203 {"\"repetition_index\": 0,$", MR_Next},204 {"\"threads\": 1,$", MR_Next},205 {"\"iterations\": %int,$", MR_Next},206 {"\"real_time\": %float,$", MR_Next},207 {"\"cpu_time\": %float,$", MR_Next},208 {"\"time_unit\": \"ns\"$", MR_Next},209 {"}", MR_Next}});210ADD_CASES(TC_CSVOut, {{"^\"BM_time_label_nanosecond\",%csv_report$"}});211 212void BM_time_label_microsecond(benchmark::State& state) {213 for (auto _ : state) {214 }215}216BENCHMARK(BM_time_label_microsecond)->Unit(benchmark::kMicrosecond);217 218ADD_CASES(TC_ConsoleOut, {{"^BM_time_label_microsecond %console_us_report$"}});219ADD_CASES(TC_JSONOut,220 {{"\"name\": \"BM_time_label_microsecond\",$"},221 {"\"family_index\": 5,$", MR_Next},222 {"\"per_family_instance_index\": 0,$", MR_Next},223 {"\"run_name\": \"BM_time_label_microsecond\",$", MR_Next},224 {"\"run_type\": \"iteration\",$", MR_Next},225 {"\"repetitions\": 1,$", MR_Next},226 {"\"repetition_index\": 0,$", MR_Next},227 {"\"threads\": 1,$", MR_Next},228 {"\"iterations\": %int,$", MR_Next},229 {"\"real_time\": %float,$", MR_Next},230 {"\"cpu_time\": %float,$", MR_Next},231 {"\"time_unit\": \"us\"$", MR_Next},232 {"}", MR_Next}});233ADD_CASES(TC_CSVOut, {{"^\"BM_time_label_microsecond\",%csv_us_report$"}});234 235void BM_time_label_millisecond(benchmark::State& state) {236 for (auto _ : state) {237 }238}239BENCHMARK(BM_time_label_millisecond)->Unit(benchmark::kMillisecond);240 241ADD_CASES(TC_ConsoleOut, {{"^BM_time_label_millisecond %console_ms_report$"}});242ADD_CASES(TC_JSONOut,243 {{"\"name\": \"BM_time_label_millisecond\",$"},244 {"\"family_index\": 6,$", MR_Next},245 {"\"per_family_instance_index\": 0,$", MR_Next},246 {"\"run_name\": \"BM_time_label_millisecond\",$", MR_Next},247 {"\"run_type\": \"iteration\",$", MR_Next},248 {"\"repetitions\": 1,$", MR_Next},249 {"\"repetition_index\": 0,$", MR_Next},250 {"\"threads\": 1,$", MR_Next},251 {"\"iterations\": %int,$", MR_Next},252 {"\"real_time\": %float,$", MR_Next},253 {"\"cpu_time\": %float,$", MR_Next},254 {"\"time_unit\": \"ms\"$", MR_Next},255 {"}", MR_Next}});256ADD_CASES(TC_CSVOut, {{"^\"BM_time_label_millisecond\",%csv_ms_report$"}});257 258void BM_time_label_second(benchmark::State& state) {259 for (auto _ : state) {260 }261}262BENCHMARK(BM_time_label_second)->Unit(benchmark::kSecond);263 264ADD_CASES(TC_ConsoleOut, {{"^BM_time_label_second %console_s_report$"}});265ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_time_label_second\",$"},266 {"\"family_index\": 7,$", MR_Next},267 {"\"per_family_instance_index\": 0,$", MR_Next},268 {"\"run_name\": \"BM_time_label_second\",$", MR_Next},269 {"\"run_type\": \"iteration\",$", MR_Next},270 {"\"repetitions\": 1,$", MR_Next},271 {"\"repetition_index\": 0,$", MR_Next},272 {"\"threads\": 1,$", MR_Next},273 {"\"iterations\": %int,$", MR_Next},274 {"\"real_time\": %float,$", MR_Next},275 {"\"cpu_time\": %float,$", MR_Next},276 {"\"time_unit\": \"s\"$", MR_Next},277 {"}", MR_Next}});278ADD_CASES(TC_CSVOut, {{"^\"BM_time_label_second\",%csv_s_report$"}});279 280// ========================================================================= //281// ------------------------ Testing Error Output --------------------------- //282// ========================================================================= //283 284void BM_error(benchmark::State& state) {285 state.SkipWithError("message");286 for (auto _ : state) {287 }288}289BENCHMARK(BM_error);290ADD_CASES(TC_ConsoleOut, {{"^BM_error[ ]+ERROR OCCURRED: 'message'$"}});291ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_error\",$"},292 {"\"family_index\": 8,$", MR_Next},293 {"\"per_family_instance_index\": 0,$", MR_Next},294 {"\"run_name\": \"BM_error\",$", MR_Next},295 {"\"run_type\": \"iteration\",$", MR_Next},296 {"\"repetitions\": 1,$", MR_Next},297 {"\"repetition_index\": 0,$", MR_Next},298 {"\"threads\": 1,$", MR_Next},299 {"\"error_occurred\": true,$", MR_Next},300 {"\"error_message\": \"message\",$", MR_Next}});301 302ADD_CASES(TC_CSVOut, {{"^\"BM_error\",,,,,,,,true,\"message\"$"}});303 304// ========================================================================= //305// ------------------------ Testing No Arg Name Output -----------------------306// //307// ========================================================================= //308 309void BM_no_arg_name(benchmark::State& state) {310 for (auto _ : state) {311 }312}313BENCHMARK(BM_no_arg_name)->Arg(3);314ADD_CASES(TC_ConsoleOut, {{"^BM_no_arg_name/3 %console_report$"}});315ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_no_arg_name/3\",$"},316 {"\"family_index\": 9,$", MR_Next},317 {"\"per_family_instance_index\": 0,$", MR_Next},318 {"\"run_name\": \"BM_no_arg_name/3\",$", MR_Next},319 {"\"run_type\": \"iteration\",$", MR_Next},320 {"\"repetitions\": 1,$", MR_Next},321 {"\"repetition_index\": 0,$", MR_Next},322 {"\"threads\": 1,$", MR_Next}});323ADD_CASES(TC_CSVOut, {{"^\"BM_no_arg_name/3\",%csv_report$"}});324 325// ========================================================================= //326// ------------------------ Testing Arg Name Output ------------------------ //327// ========================================================================= //328 329void BM_arg_name(benchmark::State& state) {330 for (auto _ : state) {331 }332}333BENCHMARK(BM_arg_name)->ArgName("first")->Arg(3);334ADD_CASES(TC_ConsoleOut, {{"^BM_arg_name/first:3 %console_report$"}});335ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_arg_name/first:3\",$"},336 {"\"family_index\": 10,$", MR_Next},337 {"\"per_family_instance_index\": 0,$", MR_Next},338 {"\"run_name\": \"BM_arg_name/first:3\",$", MR_Next},339 {"\"run_type\": \"iteration\",$", MR_Next},340 {"\"repetitions\": 1,$", MR_Next},341 {"\"repetition_index\": 0,$", MR_Next},342 {"\"threads\": 1,$", MR_Next}});343ADD_CASES(TC_CSVOut, {{"^\"BM_arg_name/first:3\",%csv_report$"}});344 345// ========================================================================= //346// ------------------------ Testing Arg Names Output ----------------------- //347// ========================================================================= //348 349void BM_arg_names(benchmark::State& state) {350 for (auto _ : state) {351 }352}353BENCHMARK(BM_arg_names)->Args({2, 5, 4})->ArgNames({"first", "", "third"});354ADD_CASES(TC_ConsoleOut,355 {{"^BM_arg_names/first:2/5/third:4 %console_report$"}});356ADD_CASES(TC_JSONOut,357 {{"\"name\": \"BM_arg_names/first:2/5/third:4\",$"},358 {"\"family_index\": 11,$", MR_Next},359 {"\"per_family_instance_index\": 0,$", MR_Next},360 {"\"run_name\": \"BM_arg_names/first:2/5/third:4\",$", MR_Next},361 {"\"run_type\": \"iteration\",$", MR_Next},362 {"\"repetitions\": 1,$", MR_Next},363 {"\"repetition_index\": 0,$", MR_Next},364 {"\"threads\": 1,$", MR_Next}});365ADD_CASES(TC_CSVOut, {{"^\"BM_arg_names/first:2/5/third:4\",%csv_report$"}});366 367// ========================================================================= //368// ------------------------ Testing Name Output ---------------------------- //369// ========================================================================= //370 371void BM_name(benchmark::State& state) {372 for (auto _ : state) {373 }374}375BENCHMARK(BM_name)->Name("BM_custom_name");376 377ADD_CASES(TC_ConsoleOut, {{"^BM_custom_name %console_report$"}});378ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_custom_name\",$"},379 {"\"family_index\": 12,$", MR_Next},380 {"\"per_family_instance_index\": 0,$", MR_Next},381 {"\"run_name\": \"BM_custom_name\",$", MR_Next},382 {"\"run_type\": \"iteration\",$", MR_Next},383 {"\"repetitions\": 1,$", MR_Next},384 {"\"repetition_index\": 0,$", MR_Next},385 {"\"threads\": 1,$", MR_Next},386 {"\"iterations\": %int,$", MR_Next},387 {"\"real_time\": %float,$", MR_Next},388 {"\"cpu_time\": %float,$", MR_Next},389 {"\"time_unit\": \"ns\"$", MR_Next},390 {"}", MR_Next}});391ADD_CASES(TC_CSVOut, {{"^\"BM_custom_name\",%csv_report$"}});392 393// ========================================================================= //394// ------------------------ Testing Big Args Output ------------------------ //395// ========================================================================= //396 397void BM_BigArgs(benchmark::State& state) {398 for (auto _ : state) {399 }400}401BENCHMARK(BM_BigArgs)->RangeMultiplier(2)->Range(1U << 30U, 1U << 31U);402ADD_CASES(TC_ConsoleOut, {{"^BM_BigArgs/1073741824 %console_report$"},403 {"^BM_BigArgs/2147483648 %console_report$"}});404 405// ========================================================================= //406// ----------------------- Testing Complexity Output ----------------------- //407// ========================================================================= //408 409void BM_Complexity_O1(benchmark::State& state) {410 for (auto _ : state) {411 // This test requires a non-zero CPU time to avoid divide-by-zero412 auto iterations = double(state.iterations()) * double(state.iterations());413 benchmark::DoNotOptimize(iterations);414 }415 state.SetComplexityN(state.range(0));416}417BENCHMARK(BM_Complexity_O1)->Range(1, 1 << 18)->Complexity(benchmark::o1);418SET_SUBSTITUTIONS({{"%bigOStr", "[ ]* %float \\([0-9]+\\)"},419 {"%RMS", "[ ]*[0-9]+ %"}});420ADD_CASES(TC_ConsoleOut, {{"^BM_Complexity_O1_BigO %bigOStr %bigOStr[ ]*$"},421 {"^BM_Complexity_O1_RMS %RMS %RMS[ ]*$"}});422 423// ========================================================================= //424// ----------------------- Testing Aggregate Output ------------------------ //425// ========================================================================= //426 427// Test that non-aggregate data is printed by default428void BM_Repeat(benchmark::State& state) {429 for (auto _ : state) {430 }431}432// need two repetitions min to be able to output any aggregate output433BENCHMARK(BM_Repeat)->Repetitions(2);434ADD_CASES(TC_ConsoleOut,435 {{"^BM_Repeat/repeats:2 %console_report$"},436 {"^BM_Repeat/repeats:2 %console_report$"},437 {"^BM_Repeat/repeats:2_mean %console_time_only_report [ ]*2$"},438 {"^BM_Repeat/repeats:2_median %console_time_only_report [ ]*2$"},439 {"^BM_Repeat/repeats:2_stddev %console_time_only_report [ ]*2$"}});440ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Repeat/repeats:2\",$"},441 {"\"family_index\": 15,$", MR_Next},442 {"\"per_family_instance_index\": 0,$", MR_Next},443 {"\"run_name\": \"BM_Repeat/repeats:2\"", MR_Next},444 {"\"run_type\": \"iteration\",$", MR_Next},445 {"\"repetitions\": 2,$", MR_Next},446 {"\"repetition_index\": 0,$", MR_Next},447 {"\"threads\": 1,$", MR_Next},448 {"\"name\": \"BM_Repeat/repeats:2\",$"},449 {"\"family_index\": 15,$", MR_Next},450 {"\"per_family_instance_index\": 0,$", MR_Next},451 {"\"run_name\": \"BM_Repeat/repeats:2\",$", MR_Next},452 {"\"run_type\": \"iteration\",$", MR_Next},453 {"\"repetitions\": 2,$", MR_Next},454 {"\"repetition_index\": 1,$", MR_Next},455 {"\"threads\": 1,$", MR_Next},456 {"\"name\": \"BM_Repeat/repeats:2_mean\",$"},457 {"\"family_index\": 15,$", MR_Next},458 {"\"per_family_instance_index\": 0,$", MR_Next},459 {"\"run_name\": \"BM_Repeat/repeats:2\",$", MR_Next},460 {"\"run_type\": \"aggregate\",$", MR_Next},461 {"\"repetitions\": 2,$", MR_Next},462 {"\"threads\": 1,$", MR_Next},463 {"\"aggregate_name\": \"mean\",$", MR_Next},464 {"\"aggregate_unit\": \"time\",$", MR_Next},465 {"\"iterations\": 2,$", MR_Next},466 {"\"name\": \"BM_Repeat/repeats:2_median\",$"},467 {"\"family_index\": 15,$", MR_Next},468 {"\"per_family_instance_index\": 0,$", MR_Next},469 {"\"run_name\": \"BM_Repeat/repeats:2\",$", MR_Next},470 {"\"run_type\": \"aggregate\",$", MR_Next},471 {"\"repetitions\": 2,$", MR_Next},472 {"\"threads\": 1,$", MR_Next},473 {"\"aggregate_name\": \"median\",$", MR_Next},474 {"\"aggregate_unit\": \"time\",$", MR_Next},475 {"\"iterations\": 2,$", MR_Next},476 {"\"name\": \"BM_Repeat/repeats:2_stddev\",$"},477 {"\"family_index\": 15,$", MR_Next},478 {"\"per_family_instance_index\": 0,$", MR_Next},479 {"\"run_name\": \"BM_Repeat/repeats:2\",$", MR_Next},480 {"\"run_type\": \"aggregate\",$", MR_Next},481 {"\"repetitions\": 2,$", MR_Next},482 {"\"threads\": 1,$", MR_Next},483 {"\"aggregate_name\": \"stddev\",$", MR_Next},484 {"\"aggregate_unit\": \"time\",$", MR_Next},485 {"\"iterations\": 2,$", MR_Next}});486ADD_CASES(TC_CSVOut, {{"^\"BM_Repeat/repeats:2\",%csv_report$"},487 {"^\"BM_Repeat/repeats:2\",%csv_report$"},488 {"^\"BM_Repeat/repeats:2_mean\",%csv_report$"},489 {"^\"BM_Repeat/repeats:2_median\",%csv_report$"},490 {"^\"BM_Repeat/repeats:2_stddev\",%csv_report$"}});491// but for two repetitions, mean and median is the same, so let's repeat..492BENCHMARK(BM_Repeat)->Repetitions(3);493ADD_CASES(TC_ConsoleOut,494 {{"^BM_Repeat/repeats:3 %console_report$"},495 {"^BM_Repeat/repeats:3 %console_report$"},496 {"^BM_Repeat/repeats:3 %console_report$"},497 {"^BM_Repeat/repeats:3_mean %console_time_only_report [ ]*3$"},498 {"^BM_Repeat/repeats:3_median %console_time_only_report [ ]*3$"},499 {"^BM_Repeat/repeats:3_stddev %console_time_only_report [ ]*3$"}});500ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Repeat/repeats:3\",$"},501 {"\"family_index\": 16,$", MR_Next},502 {"\"per_family_instance_index\": 0,$", MR_Next},503 {"\"run_name\": \"BM_Repeat/repeats:3\",$", MR_Next},504 {"\"run_type\": \"iteration\",$", MR_Next},505 {"\"repetitions\": 3,$", MR_Next},506 {"\"repetition_index\": 0,$", MR_Next},507 {"\"threads\": 1,$", MR_Next},508 {"\"name\": \"BM_Repeat/repeats:3\",$"},509 {"\"family_index\": 16,$", MR_Next},510 {"\"per_family_instance_index\": 0,$", MR_Next},511 {"\"run_name\": \"BM_Repeat/repeats:3\",$", MR_Next},512 {"\"run_type\": \"iteration\",$", MR_Next},513 {"\"repetitions\": 3,$", MR_Next},514 {"\"repetition_index\": 1,$", MR_Next},515 {"\"threads\": 1,$", MR_Next},516 {"\"name\": \"BM_Repeat/repeats:3\",$"},517 {"\"family_index\": 16,$", MR_Next},518 {"\"per_family_instance_index\": 0,$", MR_Next},519 {"\"run_name\": \"BM_Repeat/repeats:3\",$", MR_Next},520 {"\"run_type\": \"iteration\",$", MR_Next},521 {"\"repetitions\": 3,$", MR_Next},522 {"\"repetition_index\": 2,$", MR_Next},523 {"\"threads\": 1,$", MR_Next},524 {"\"name\": \"BM_Repeat/repeats:3_mean\",$"},525 {"\"family_index\": 16,$", MR_Next},526 {"\"per_family_instance_index\": 0,$", MR_Next},527 {"\"run_name\": \"BM_Repeat/repeats:3\",$", MR_Next},528 {"\"run_type\": \"aggregate\",$", MR_Next},529 {"\"repetitions\": 3,$", MR_Next},530 {"\"threads\": 1,$", MR_Next},531 {"\"aggregate_name\": \"mean\",$", MR_Next},532 {"\"aggregate_unit\": \"time\",$", MR_Next},533 {"\"iterations\": 3,$", MR_Next},534 {"\"name\": \"BM_Repeat/repeats:3_median\",$"},535 {"\"family_index\": 16,$", MR_Next},536 {"\"per_family_instance_index\": 0,$", MR_Next},537 {"\"run_name\": \"BM_Repeat/repeats:3\",$", MR_Next},538 {"\"run_type\": \"aggregate\",$", MR_Next},539 {"\"repetitions\": 3,$", MR_Next},540 {"\"threads\": 1,$", MR_Next},541 {"\"aggregate_name\": \"median\",$", MR_Next},542 {"\"aggregate_unit\": \"time\",$", MR_Next},543 {"\"iterations\": 3,$", MR_Next},544 {"\"name\": \"BM_Repeat/repeats:3_stddev\",$"},545 {"\"family_index\": 16,$", MR_Next},546 {"\"per_family_instance_index\": 0,$", MR_Next},547 {"\"run_name\": \"BM_Repeat/repeats:3\",$", MR_Next},548 {"\"run_type\": \"aggregate\",$", MR_Next},549 {"\"repetitions\": 3,$", MR_Next},550 {"\"threads\": 1,$", MR_Next},551 {"\"aggregate_name\": \"stddev\",$", MR_Next},552 {"\"aggregate_unit\": \"time\",$", MR_Next},553 {"\"iterations\": 3,$", MR_Next}});554ADD_CASES(TC_CSVOut, {{"^\"BM_Repeat/repeats:3\",%csv_report$"},555 {"^\"BM_Repeat/repeats:3\",%csv_report$"},556 {"^\"BM_Repeat/repeats:3\",%csv_report$"},557 {"^\"BM_Repeat/repeats:3_mean\",%csv_report$"},558 {"^\"BM_Repeat/repeats:3_median\",%csv_report$"},559 {"^\"BM_Repeat/repeats:3_stddev\",%csv_report$"}});560// median differs between even/odd number of repetitions, so just to be sure561BENCHMARK(BM_Repeat)->Repetitions(4);562ADD_CASES(TC_ConsoleOut,563 {{"^BM_Repeat/repeats:4 %console_report$"},564 {"^BM_Repeat/repeats:4 %console_report$"},565 {"^BM_Repeat/repeats:4 %console_report$"},566 {"^BM_Repeat/repeats:4 %console_report$"},567 {"^BM_Repeat/repeats:4_mean %console_time_only_report [ ]*4$"},568 {"^BM_Repeat/repeats:4_median %console_time_only_report [ ]*4$"},569 {"^BM_Repeat/repeats:4_stddev %console_time_only_report [ ]*4$"}});570ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Repeat/repeats:4\",$"},571 {"\"family_index\": 17,$", MR_Next},572 {"\"per_family_instance_index\": 0,$", MR_Next},573 {"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next},574 {"\"run_type\": \"iteration\",$", MR_Next},575 {"\"repetitions\": 4,$", MR_Next},576 {"\"repetition_index\": 0,$", MR_Next},577 {"\"threads\": 1,$", MR_Next},578 {"\"name\": \"BM_Repeat/repeats:4\",$"},579 {"\"family_index\": 17,$", MR_Next},580 {"\"per_family_instance_index\": 0,$", MR_Next},581 {"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next},582 {"\"run_type\": \"iteration\",$", MR_Next},583 {"\"repetitions\": 4,$", MR_Next},584 {"\"repetition_index\": 1,$", MR_Next},585 {"\"threads\": 1,$", MR_Next},586 {"\"name\": \"BM_Repeat/repeats:4\",$"},587 {"\"family_index\": 17,$", MR_Next},588 {"\"per_family_instance_index\": 0,$", MR_Next},589 {"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next},590 {"\"run_type\": \"iteration\",$", MR_Next},591 {"\"repetitions\": 4,$", MR_Next},592 {"\"repetition_index\": 2,$", MR_Next},593 {"\"threads\": 1,$", MR_Next},594 {"\"name\": \"BM_Repeat/repeats:4\",$"},595 {"\"family_index\": 17,$", MR_Next},596 {"\"per_family_instance_index\": 0,$", MR_Next},597 {"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next},598 {"\"run_type\": \"iteration\",$", MR_Next},599 {"\"repetitions\": 4,$", MR_Next},600 {"\"repetition_index\": 3,$", MR_Next},601 {"\"threads\": 1,$", MR_Next},602 {"\"name\": \"BM_Repeat/repeats:4_mean\",$"},603 {"\"family_index\": 17,$", MR_Next},604 {"\"per_family_instance_index\": 0,$", MR_Next},605 {"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next},606 {"\"run_type\": \"aggregate\",$", MR_Next},607 {"\"repetitions\": 4,$", MR_Next},608 {"\"threads\": 1,$", MR_Next},609 {"\"aggregate_name\": \"mean\",$", MR_Next},610 {"\"aggregate_unit\": \"time\",$", MR_Next},611 {"\"iterations\": 4,$", MR_Next},612 {"\"name\": \"BM_Repeat/repeats:4_median\",$"},613 {"\"family_index\": 17,$", MR_Next},614 {"\"per_family_instance_index\": 0,$", MR_Next},615 {"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next},616 {"\"run_type\": \"aggregate\",$", MR_Next},617 {"\"repetitions\": 4,$", MR_Next},618 {"\"threads\": 1,$", MR_Next},619 {"\"aggregate_name\": \"median\",$", MR_Next},620 {"\"aggregate_unit\": \"time\",$", MR_Next},621 {"\"iterations\": 4,$", MR_Next},622 {"\"name\": \"BM_Repeat/repeats:4_stddev\",$"},623 {"\"family_index\": 17,$", MR_Next},624 {"\"per_family_instance_index\": 0,$", MR_Next},625 {"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next},626 {"\"run_type\": \"aggregate\",$", MR_Next},627 {"\"repetitions\": 4,$", MR_Next},628 {"\"threads\": 1,$", MR_Next},629 {"\"aggregate_name\": \"stddev\",$", MR_Next},630 {"\"aggregate_unit\": \"time\",$", MR_Next},631 {"\"iterations\": 4,$", MR_Next}});632ADD_CASES(TC_CSVOut, {{"^\"BM_Repeat/repeats:4\",%csv_report$"},633 {"^\"BM_Repeat/repeats:4\",%csv_report$"},634 {"^\"BM_Repeat/repeats:4\",%csv_report$"},635 {"^\"BM_Repeat/repeats:4\",%csv_report$"},636 {"^\"BM_Repeat/repeats:4_mean\",%csv_report$"},637 {"^\"BM_Repeat/repeats:4_median\",%csv_report$"},638 {"^\"BM_Repeat/repeats:4_stddev\",%csv_report$"}});639 640// Test that a non-repeated test still prints non-aggregate results even when641// only-aggregate reports have been requested642void BM_RepeatOnce(benchmark::State& state) {643 for (auto _ : state) {644 }645}646BENCHMARK(BM_RepeatOnce)->Repetitions(1)->ReportAggregatesOnly();647ADD_CASES(TC_ConsoleOut, {{"^BM_RepeatOnce/repeats:1 %console_report$"}});648ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_RepeatOnce/repeats:1\",$"},649 {"\"family_index\": 18,$", MR_Next},650 {"\"per_family_instance_index\": 0,$", MR_Next},651 {"\"run_name\": \"BM_RepeatOnce/repeats:1\",$", MR_Next},652 {"\"run_type\": \"iteration\",$", MR_Next},653 {"\"repetitions\": 1,$", MR_Next},654 {"\"repetition_index\": 0,$", MR_Next},655 {"\"threads\": 1,$", MR_Next}});656ADD_CASES(TC_CSVOut, {{"^\"BM_RepeatOnce/repeats:1\",%csv_report$"}});657 658// Test that non-aggregate data is not reported659void BM_SummaryRepeat(benchmark::State& state) {660 for (auto _ : state) {661 }662}663BENCHMARK(BM_SummaryRepeat)->Repetitions(3)->ReportAggregatesOnly();664ADD_CASES(665 TC_ConsoleOut,666 {{".*BM_SummaryRepeat/repeats:3 ", MR_Not},667 {"^BM_SummaryRepeat/repeats:3_mean %console_time_only_report [ ]*3$"},668 {"^BM_SummaryRepeat/repeats:3_median %console_time_only_report [ ]*3$"},669 {"^BM_SummaryRepeat/repeats:3_stddev %console_time_only_report [ ]*3$"}});670ADD_CASES(TC_JSONOut,671 {{".*BM_SummaryRepeat/repeats:3 ", MR_Not},672 {"\"name\": \"BM_SummaryRepeat/repeats:3_mean\",$"},673 {"\"family_index\": 19,$", MR_Next},674 {"\"per_family_instance_index\": 0,$", MR_Next},675 {"\"run_name\": \"BM_SummaryRepeat/repeats:3\",$", MR_Next},676 {"\"run_type\": \"aggregate\",$", MR_Next},677 {"\"repetitions\": 3,$", MR_Next},678 {"\"threads\": 1,$", MR_Next},679 {"\"aggregate_name\": \"mean\",$", MR_Next},680 {"\"aggregate_unit\": \"time\",$", MR_Next},681 {"\"iterations\": 3,$", MR_Next},682 {"\"name\": \"BM_SummaryRepeat/repeats:3_median\",$"},683 {"\"family_index\": 19,$", MR_Next},684 {"\"per_family_instance_index\": 0,$", MR_Next},685 {"\"run_name\": \"BM_SummaryRepeat/repeats:3\",$", MR_Next},686 {"\"run_type\": \"aggregate\",$", MR_Next},687 {"\"repetitions\": 3,$", MR_Next},688 {"\"threads\": 1,$", MR_Next},689 {"\"aggregate_name\": \"median\",$", MR_Next},690 {"\"aggregate_unit\": \"time\",$", MR_Next},691 {"\"iterations\": 3,$", MR_Next},692 {"\"name\": \"BM_SummaryRepeat/repeats:3_stddev\",$"},693 {"\"family_index\": 19,$", MR_Next},694 {"\"per_family_instance_index\": 0,$", MR_Next},695 {"\"run_name\": \"BM_SummaryRepeat/repeats:3\",$", MR_Next},696 {"\"run_type\": \"aggregate\",$", MR_Next},697 {"\"repetitions\": 3,$", MR_Next},698 {"\"threads\": 1,$", MR_Next},699 {"\"aggregate_name\": \"stddev\",$", MR_Next},700 {"\"aggregate_unit\": \"time\",$", MR_Next},701 {"\"iterations\": 3,$", MR_Next}});702ADD_CASES(TC_CSVOut, {{".*BM_SummaryRepeat/repeats:3 ", MR_Not},703 {"^\"BM_SummaryRepeat/repeats:3_mean\",%csv_report$"},704 {"^\"BM_SummaryRepeat/repeats:3_median\",%csv_report$"},705 {"^\"BM_SummaryRepeat/repeats:3_stddev\",%csv_report$"}});706 707// Test that non-aggregate data is not displayed.708// NOTE: this test is kinda bad. we are only testing the display output.709// But we don't check that the file output still contains everything...710void BM_SummaryDisplay(benchmark::State& state) {711 for (auto _ : state) {712 }713}714BENCHMARK(BM_SummaryDisplay)->Repetitions(2)->DisplayAggregatesOnly();715ADD_CASES(716 TC_ConsoleOut,717 {{".*BM_SummaryDisplay/repeats:2 ", MR_Not},718 {"^BM_SummaryDisplay/repeats:2_mean %console_time_only_report [ ]*2$"},719 {"^BM_SummaryDisplay/repeats:2_median %console_time_only_report [ ]*2$"},720 {"^BM_SummaryDisplay/repeats:2_stddev %console_time_only_report [ ]*2$"}});721ADD_CASES(TC_JSONOut,722 {{".*BM_SummaryDisplay/repeats:2 ", MR_Not},723 {"\"name\": \"BM_SummaryDisplay/repeats:2_mean\",$"},724 {"\"family_index\": 20,$", MR_Next},725 {"\"per_family_instance_index\": 0,$", MR_Next},726 {"\"run_name\": \"BM_SummaryDisplay/repeats:2\",$", MR_Next},727 {"\"run_type\": \"aggregate\",$", MR_Next},728 {"\"repetitions\": 2,$", MR_Next},729 {"\"threads\": 1,$", MR_Next},730 {"\"aggregate_name\": \"mean\",$", MR_Next},731 {"\"aggregate_unit\": \"time\",$", MR_Next},732 {"\"iterations\": 2,$", MR_Next},733 {"\"name\": \"BM_SummaryDisplay/repeats:2_median\",$"},734 {"\"family_index\": 20,$", MR_Next},735 {"\"per_family_instance_index\": 0,$", MR_Next},736 {"\"run_name\": \"BM_SummaryDisplay/repeats:2\",$", MR_Next},737 {"\"run_type\": \"aggregate\",$", MR_Next},738 {"\"repetitions\": 2,$", MR_Next},739 {"\"threads\": 1,$", MR_Next},740 {"\"aggregate_name\": \"median\",$", MR_Next},741 {"\"aggregate_unit\": \"time\",$", MR_Next},742 {"\"iterations\": 2,$", MR_Next},743 {"\"name\": \"BM_SummaryDisplay/repeats:2_stddev\",$"},744 {"\"family_index\": 20,$", MR_Next},745 {"\"per_family_instance_index\": 0,$", MR_Next},746 {"\"run_name\": \"BM_SummaryDisplay/repeats:2\",$", MR_Next},747 {"\"run_type\": \"aggregate\",$", MR_Next},748 {"\"repetitions\": 2,$", MR_Next},749 {"\"threads\": 1,$", MR_Next},750 {"\"aggregate_name\": \"stddev\",$", MR_Next},751 {"\"aggregate_unit\": \"time\",$", MR_Next},752 {"\"iterations\": 2,$", MR_Next}});753ADD_CASES(TC_CSVOut,754 {{".*BM_SummaryDisplay/repeats:2 ", MR_Not},755 {"^\"BM_SummaryDisplay/repeats:2_mean\",%csv_report$"},756 {"^\"BM_SummaryDisplay/repeats:2_median\",%csv_report$"},757 {"^\"BM_SummaryDisplay/repeats:2_stddev\",%csv_report$"}});758 759// Test repeats with custom time unit.760void BM_RepeatTimeUnit(benchmark::State& state) {761 for (auto _ : state) {762 }763}764BENCHMARK(BM_RepeatTimeUnit)765 ->Repetitions(3)766 ->ReportAggregatesOnly()767 ->Unit(benchmark::kMicrosecond);768ADD_CASES(769 TC_ConsoleOut,770 {{".*BM_RepeatTimeUnit/repeats:3 ", MR_Not},771 {"^BM_RepeatTimeUnit/repeats:3_mean %console_us_time_only_report [ ]*3$"},772 {"^BM_RepeatTimeUnit/repeats:3_median %console_us_time_only_report [ "773 "]*3$"},774 {"^BM_RepeatTimeUnit/repeats:3_stddev %console_us_time_only_report [ "775 "]*3$"}});776ADD_CASES(TC_JSONOut,777 {{".*BM_RepeatTimeUnit/repeats:3 ", MR_Not},778 {"\"name\": \"BM_RepeatTimeUnit/repeats:3_mean\",$"},779 {"\"family_index\": 21,$", MR_Next},780 {"\"per_family_instance_index\": 0,$", MR_Next},781 {"\"run_name\": \"BM_RepeatTimeUnit/repeats:3\",$", MR_Next},782 {"\"run_type\": \"aggregate\",$", MR_Next},783 {"\"repetitions\": 3,$", MR_Next},784 {"\"threads\": 1,$", MR_Next},785 {"\"aggregate_name\": \"mean\",$", MR_Next},786 {"\"aggregate_unit\": \"time\",$", MR_Next},787 {"\"iterations\": 3,$", MR_Next},788 {"\"time_unit\": \"us\",?$"},789 {"\"name\": \"BM_RepeatTimeUnit/repeats:3_median\",$"},790 {"\"family_index\": 21,$", MR_Next},791 {"\"per_family_instance_index\": 0,$", MR_Next},792 {"\"run_name\": \"BM_RepeatTimeUnit/repeats:3\",$", MR_Next},793 {"\"run_type\": \"aggregate\",$", MR_Next},794 {"\"repetitions\": 3,$", MR_Next},795 {"\"threads\": 1,$", MR_Next},796 {"\"aggregate_name\": \"median\",$", MR_Next},797 {"\"aggregate_unit\": \"time\",$", MR_Next},798 {"\"iterations\": 3,$", MR_Next},799 {"\"time_unit\": \"us\",?$"},800 {"\"name\": \"BM_RepeatTimeUnit/repeats:3_stddev\",$"},801 {"\"family_index\": 21,$", MR_Next},802 {"\"per_family_instance_index\": 0,$", MR_Next},803 {"\"run_name\": \"BM_RepeatTimeUnit/repeats:3\",$", MR_Next},804 {"\"run_type\": \"aggregate\",$", MR_Next},805 {"\"repetitions\": 3,$", MR_Next},806 {"\"threads\": 1,$", MR_Next},807 {"\"aggregate_name\": \"stddev\",$", MR_Next},808 {"\"aggregate_unit\": \"time\",$", MR_Next},809 {"\"iterations\": 3,$", MR_Next},810 {"\"time_unit\": \"us\",?$"}});811ADD_CASES(TC_CSVOut,812 {{".*BM_RepeatTimeUnit/repeats:3 ", MR_Not},813 {"^\"BM_RepeatTimeUnit/repeats:3_mean\",%csv_us_report$"},814 {"^\"BM_RepeatTimeUnit/repeats:3_median\",%csv_us_report$"},815 {"^\"BM_RepeatTimeUnit/repeats:3_stddev\",%csv_us_report$"}});816 817// ========================================================================= //818// -------------------- Testing user-provided statistics ------------------- //819// ========================================================================= //820 821const auto UserStatistics = [](const std::vector<double>& v) {822 return v.back();823};824void BM_UserStats(benchmark::State& state) {825 for (auto _ : state) {826 state.SetIterationTime(150 / 10e8);827 }828}829// clang-format off830BENCHMARK(BM_UserStats)831 ->Repetitions(3)832 ->Iterations(5)833 ->UseManualTime()834 ->ComputeStatistics("", UserStatistics);835// clang-format on836 837// check that user-provided stats is calculated, and is after the default-ones838// empty string as name is intentional, it would sort before anything else839ADD_CASES(TC_ConsoleOut, {{"^BM_UserStats/iterations:5/repeats:3/manual_time [ "840 "]* 150 ns %time [ ]*5$"},841 {"^BM_UserStats/iterations:5/repeats:3/manual_time [ "842 "]* 150 ns %time [ ]*5$"},843 {"^BM_UserStats/iterations:5/repeats:3/manual_time [ "844 "]* 150 ns %time [ ]*5$"},845 {"^BM_UserStats/iterations:5/repeats:3/"846 "manual_time_mean [ ]* 150 ns %time [ ]*3$"},847 {"^BM_UserStats/iterations:5/repeats:3/"848 "manual_time_median [ ]* 150 ns %time [ ]*3$"},849 {"^BM_UserStats/iterations:5/repeats:3/"850 "manual_time_stddev [ ]* 0.000 ns %time [ ]*3$"},851 {"^BM_UserStats/iterations:5/repeats:3/manual_time_ "852 "[ ]* 150 ns %time [ ]*3$"}});853ADD_CASES(854 TC_JSONOut,855 {{"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$"},856 {"\"family_index\": 22,$", MR_Next},857 {"\"per_family_instance_index\": 0,$", MR_Next},858 {"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$",859 MR_Next},860 {"\"run_type\": \"iteration\",$", MR_Next},861 {"\"repetitions\": 3,$", MR_Next},862 {"\"repetition_index\": 0,$", MR_Next},863 {"\"threads\": 1,$", MR_Next},864 {"\"iterations\": 5,$", MR_Next},865 {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next},866 {"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$"},867 {"\"family_index\": 22,$", MR_Next},868 {"\"per_family_instance_index\": 0,$", MR_Next},869 {"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$",870 MR_Next},871 {"\"run_type\": \"iteration\",$", MR_Next},872 {"\"repetitions\": 3,$", MR_Next},873 {"\"repetition_index\": 1,$", MR_Next},874 {"\"threads\": 1,$", MR_Next},875 {"\"iterations\": 5,$", MR_Next},876 {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next},877 {"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$"},878 {"\"family_index\": 22,$", MR_Next},879 {"\"per_family_instance_index\": 0,$", MR_Next},880 {"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$",881 MR_Next},882 {"\"run_type\": \"iteration\",$", MR_Next},883 {"\"repetitions\": 3,$", MR_Next},884 {"\"repetition_index\": 2,$", MR_Next},885 {"\"threads\": 1,$", MR_Next},886 {"\"iterations\": 5,$", MR_Next},887 {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next},888 {"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time_mean\",$"},889 {"\"family_index\": 22,$", MR_Next},890 {"\"per_family_instance_index\": 0,$", MR_Next},891 {"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$",892 MR_Next},893 {"\"run_type\": \"aggregate\",$", MR_Next},894 {"\"repetitions\": 3,$", MR_Next},895 {"\"threads\": 1,$", MR_Next},896 {"\"aggregate_name\": \"mean\",$", MR_Next},897 {"\"aggregate_unit\": \"time\",$", MR_Next},898 {"\"iterations\": 3,$", MR_Next},899 {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next},900 {"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time_median\",$"},901 {"\"family_index\": 22,$", MR_Next},902 {"\"per_family_instance_index\": 0,$", MR_Next},903 {"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$",904 MR_Next},905 {"\"run_type\": \"aggregate\",$", MR_Next},906 {"\"repetitions\": 3,$", MR_Next},907 {"\"threads\": 1,$", MR_Next},908 {"\"aggregate_name\": \"median\",$", MR_Next},909 {"\"aggregate_unit\": \"time\",$", MR_Next},910 {"\"iterations\": 3,$", MR_Next},911 {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next},912 {"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time_stddev\",$"},913 {"\"family_index\": 22,$", MR_Next},914 {"\"per_family_instance_index\": 0,$", MR_Next},915 {"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$",916 MR_Next},917 {"\"run_type\": \"aggregate\",$", MR_Next},918 {"\"repetitions\": 3,$", MR_Next},919 {"\"threads\": 1,$", MR_Next},920 {"\"aggregate_name\": \"stddev\",$", MR_Next},921 {"\"aggregate_unit\": \"time\",$", MR_Next},922 {"\"iterations\": 3,$", MR_Next},923 {"\"real_time\": %float,$", MR_Next},924 {"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time_\",$"},925 {"\"family_index\": 22,$", MR_Next},926 {"\"per_family_instance_index\": 0,$", MR_Next},927 {"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$",928 MR_Next},929 {"\"run_type\": \"aggregate\",$", MR_Next},930 {"\"repetitions\": 3,$", MR_Next},931 {"\"threads\": 1,$", MR_Next},932 {"\"aggregate_name\": \"\",$", MR_Next},933 {"\"aggregate_unit\": \"time\",$", MR_Next},934 {"\"iterations\": 3,$", MR_Next},935 {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next}});936ADD_CASES(937 TC_CSVOut,938 {{"^\"BM_UserStats/iterations:5/repeats:3/manual_time\",%csv_report$"},939 {"^\"BM_UserStats/iterations:5/repeats:3/manual_time\",%csv_report$"},940 {"^\"BM_UserStats/iterations:5/repeats:3/manual_time\",%csv_report$"},941 {"^\"BM_UserStats/iterations:5/repeats:3/manual_time_mean\",%csv_report$"},942 {"^\"BM_UserStats/iterations:5/repeats:3/"943 "manual_time_median\",%csv_report$"},944 {"^\"BM_UserStats/iterations:5/repeats:3/"945 "manual_time_stddev\",%csv_report$"},946 {"^\"BM_UserStats/iterations:5/repeats:3/manual_time_\",%csv_report$"}});947 948// ========================================================================= //949// ------------- Testing relative standard deviation statistics ------------ //950// ========================================================================= //951 952const auto UserPercentStatistics = [](const std::vector<double>&) {953 return 1. / 100.;954};955void BM_UserPercentStats(benchmark::State& state) {956 for (auto _ : state) {957 state.SetIterationTime(150 / 10e8);958 }959}960// clang-format off961BENCHMARK(BM_UserPercentStats)962 ->Repetitions(3)963 ->Iterations(5)964 ->UseManualTime()965 ->Unit(benchmark::TimeUnit::kNanosecond)966 ->ComputeStatistics("", UserPercentStatistics, benchmark::StatisticUnit::kPercentage);967// clang-format on968 969// check that UserPercent-provided stats is calculated, and is after the970// default-ones empty string as name is intentional, it would sort before971// anything else972ADD_CASES(TC_ConsoleOut,973 {{"^BM_UserPercentStats/iterations:5/repeats:3/manual_time [ "974 "]* 150 ns %time [ ]*5$"},975 {"^BM_UserPercentStats/iterations:5/repeats:3/manual_time [ "976 "]* 150 ns %time [ ]*5$"},977 {"^BM_UserPercentStats/iterations:5/repeats:3/manual_time [ "978 "]* 150 ns %time [ ]*5$"},979 {"^BM_UserPercentStats/iterations:5/repeats:3/"980 "manual_time_mean [ ]* 150 ns %time [ ]*3$"},981 {"^BM_UserPercentStats/iterations:5/repeats:3/"982 "manual_time_median [ ]* 150 ns %time [ ]*3$"},983 {"^BM_UserPercentStats/iterations:5/repeats:3/"984 "manual_time_stddev [ ]* 0.000 ns %time [ ]*3$"},985 {"^BM_UserPercentStats/iterations:5/repeats:3/manual_time_ "986 "[ ]* 1.00 % [ ]* 1.00 %[ ]*3$"}});987ADD_CASES(988 TC_JSONOut,989 {{"\"name\": \"BM_UserPercentStats/iterations:5/repeats:3/manual_time\",$"},990 {"\"family_index\": 23,$", MR_Next},991 {"\"per_family_instance_index\": 0,$", MR_Next},992 {"\"run_name\": "993 "\"BM_UserPercentStats/iterations:5/repeats:3/manual_time\",$",994 MR_Next},995 {"\"run_type\": \"iteration\",$", MR_Next},996 {"\"repetitions\": 3,$", MR_Next},997 {"\"repetition_index\": 0,$", MR_Next},998 {"\"threads\": 1,$", MR_Next},999 {"\"iterations\": 5,$", MR_Next},1000 {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next},1001 {"\"name\": \"BM_UserPercentStats/iterations:5/repeats:3/manual_time\",$"},1002 {"\"family_index\": 23,$", MR_Next},1003 {"\"per_family_instance_index\": 0,$", MR_Next},1004 {"\"run_name\": "1005 "\"BM_UserPercentStats/iterations:5/repeats:3/manual_time\",$",1006 MR_Next},1007 {"\"run_type\": \"iteration\",$", MR_Next},1008 {"\"repetitions\": 3,$", MR_Next},1009 {"\"repetition_index\": 1,$", MR_Next},1010 {"\"threads\": 1,$", MR_Next},1011 {"\"iterations\": 5,$", MR_Next},1012 {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next},1013 {"\"name\": \"BM_UserPercentStats/iterations:5/repeats:3/manual_time\",$"},1014 {"\"family_index\": 23,$", MR_Next},1015 {"\"per_family_instance_index\": 0,$", MR_Next},1016 {"\"run_name\": "1017 "\"BM_UserPercentStats/iterations:5/repeats:3/manual_time\",$",1018 MR_Next},1019 {"\"run_type\": \"iteration\",$", MR_Next},1020 {"\"repetitions\": 3,$", MR_Next},1021 {"\"repetition_index\": 2,$", MR_Next},1022 {"\"threads\": 1,$", MR_Next},1023 {"\"iterations\": 5,$", MR_Next},1024 {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next},1025 {"\"name\": "1026 "\"BM_UserPercentStats/iterations:5/repeats:3/manual_time_mean\",$"},1027 {"\"family_index\": 23,$", MR_Next},1028 {"\"per_family_instance_index\": 0,$", MR_Next},1029 {"\"run_name\": "1030 "\"BM_UserPercentStats/iterations:5/repeats:3/manual_time\",$",1031 MR_Next},1032 {"\"run_type\": \"aggregate\",$", MR_Next},1033 {"\"repetitions\": 3,$", MR_Next},1034 {"\"threads\": 1,$", MR_Next},1035 {"\"aggregate_name\": \"mean\",$", MR_Next},1036 {"\"aggregate_unit\": \"time\",$", MR_Next},1037 {"\"iterations\": 3,$", MR_Next},1038 {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next},1039 {"\"name\": "1040 "\"BM_UserPercentStats/iterations:5/repeats:3/manual_time_median\",$"},1041 {"\"family_index\": 23,$", MR_Next},1042 {"\"per_family_instance_index\": 0,$", MR_Next},1043 {"\"run_name\": "1044 "\"BM_UserPercentStats/iterations:5/repeats:3/manual_time\",$",1045 MR_Next},1046 {"\"run_type\": \"aggregate\",$", MR_Next},1047 {"\"repetitions\": 3,$", MR_Next},1048 {"\"threads\": 1,$", MR_Next},1049 {"\"aggregate_name\": \"median\",$", MR_Next},1050 {"\"aggregate_unit\": \"time\",$", MR_Next},1051 {"\"iterations\": 3,$", MR_Next},1052 {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next},1053 {"\"name\": "1054 "\"BM_UserPercentStats/iterations:5/repeats:3/manual_time_stddev\",$"},1055 {"\"family_index\": 23,$", MR_Next},1056 {"\"per_family_instance_index\": 0,$", MR_Next},1057 {"\"run_name\": "1058 "\"BM_UserPercentStats/iterations:5/repeats:3/manual_time\",$",1059 MR_Next},1060 {"\"run_type\": \"aggregate\",$", MR_Next},1061 {"\"repetitions\": 3,$", MR_Next},1062 {"\"threads\": 1,$", MR_Next},1063 {"\"aggregate_name\": \"stddev\",$", MR_Next},1064 {"\"aggregate_unit\": \"time\",$", MR_Next},1065 {"\"iterations\": 3,$", MR_Next},1066 {"\"real_time\": %float,$", MR_Next},1067 {"\"name\": "1068 "\"BM_UserPercentStats/iterations:5/repeats:3/manual_time_\",$"},1069 {"\"family_index\": 23,$", MR_Next},1070 {"\"per_family_instance_index\": 0,$", MR_Next},1071 {"\"run_name\": "1072 "\"BM_UserPercentStats/iterations:5/repeats:3/manual_time\",$",1073 MR_Next},1074 {"\"run_type\": \"aggregate\",$", MR_Next},1075 {"\"repetitions\": 3,$", MR_Next},1076 {"\"threads\": 1,$", MR_Next},1077 {"\"aggregate_name\": \"\",$", MR_Next},1078 {"\"aggregate_unit\": \"percentage\",$", MR_Next},1079 {"\"iterations\": 3,$", MR_Next},1080 {"\"real_time\": 1\\.(0)*e-(0)*2,$", MR_Next}});1081ADD_CASES(TC_CSVOut, {{"^\"BM_UserPercentStats/iterations:5/repeats:3/"1082 "manual_time\",%csv_report$"},1083 {"^\"BM_UserPercentStats/iterations:5/repeats:3/"1084 "manual_time\",%csv_report$"},1085 {"^\"BM_UserPercentStats/iterations:5/repeats:3/"1086 "manual_time\",%csv_report$"},1087 {"^\"BM_UserPercentStats/iterations:5/repeats:3/"1088 "manual_time_mean\",%csv_report$"},1089 {"^\"BM_UserPercentStats/iterations:5/repeats:3/"1090 "manual_time_median\",%csv_report$"},1091 {"^\"BM_UserPercentStats/iterations:5/repeats:3/"1092 "manual_time_stddev\",%csv_report$"},1093 {"^\"BM_UserPercentStats/iterations:5/repeats:3/"1094 "manual_time_\",%csv_cv_report$"}});1095 1096// ========================================================================= //1097// ------------------------- Testing StrEscape JSON ------------------------ //1098// ========================================================================= //1099#if 0 // enable when csv testing code correctly handles multi-line fields1100void BM_JSON_Format(benchmark::State& state) {1101 state.SkipWithError("val\b\f\n\r\t\\\"with\"es,capes");1102 for (auto _ : state) {1103 }1104}1105BENCHMARK(BM_JSON_Format);1106ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_JSON_Format\",$"},1107 {"\"family_index\": 23,$", MR_Next},1108{"\"per_family_instance_index\": 0,$", MR_Next},1109 {"\"run_name\": \"BM_JSON_Format\",$", MR_Next},1110 {"\"run_type\": \"iteration\",$", MR_Next},1111 {"\"repetitions\": 1,$", MR_Next},1112 {"\"repetition_index\": 0,$", MR_Next},1113 {"\"threads\": 1,$", MR_Next},1114 {"\"error_occurred\": true,$", MR_Next},1115 {R"("error_message": "val\\b\\f\\n\\r\\t\\\\\\"with\\"es,capes",$)", MR_Next}});1116#endif1117// ========================================================================= //1118// -------------------------- Testing CsvEscape ---------------------------- //1119// ========================================================================= //1120 1121void BM_CSV_Format(benchmark::State& state) {1122 state.SkipWithError("\"freedom\"");1123 for (auto _ : state) {1124 }1125}1126BENCHMARK(BM_CSV_Format);1127ADD_CASES(TC_CSVOut, {{"^\"BM_CSV_Format\",,,,,,,,true,\"\"\"freedom\"\"\"$"}});1128 1129// ========================================================================= //1130// --------------------------- TEST CASES END ------------------------------ //1131// ========================================================================= //1132 1133int main(int argc, char* argv[]) { RunOutputTests(argc, argv); }1134