521 lines · plain
1#include <cstdio>2#include <cstring>3#include <fstream>4#include <iostream>5#include <map>6#include <memory>7#include <random>8#include <sstream>9#include <streambuf>10 11#include "../src/benchmark_api_internal.h"12#include "../src/check.h" // NOTE: check.h is for internal use only!13#include "../src/log.h" // NOTE: log.h is for internal use only14#include "../src/re.h" // NOTE: re.h is for internal use only15#include "output_test.h"16 17// ========================================================================= //18// ------------------------------ Internals -------------------------------- //19// ========================================================================= //20namespace internal {21namespace {22 23using TestCaseList = std::vector<TestCase>;24 25// Use a vector because the order elements are added matters during iteration.26// std::map/unordered_map don't guarantee that.27// For example:28// SetSubstitutions({{"%HelloWorld", "Hello"}, {"%Hello", "Hi"}});29// Substitute("%HelloWorld") // Always expands to Hello.30using SubMap = std::vector<std::pair<std::string, std::string>>;31 32TestCaseList& GetTestCaseList(TestCaseID ID) {33 // Uses function-local statics to ensure initialization occurs34 // before first use.35 static TestCaseList lists[TC_NumID];36 return lists[ID];37}38 39SubMap& GetSubstitutions() {40 // Don't use 'dec_re' from header because it may not yet be initialized.41 // clang-format off42 static std::string safe_dec_re = "[0-9]*[.]?[0-9]+([eE][-+][0-9]+)?";43 static std::string time_re = "([0-9]+[.])?[0-9]+";44 static std::string percentage_re = "[0-9]+[.][0-9]{2}";45 static SubMap map = {46 {"%float", "[0-9]*[.]?[0-9]+([eE][-+][0-9]+)?"},47 // human-readable float48 {"%hrfloat", "[0-9]*[.]?[0-9]+([eE][-+][0-9]+)?[kKMGTPEZYmunpfazy]?i?"},49 {"%percentage", percentage_re},50 {"%int", "[ ]*[0-9]+"},51 {" %s ", "[ ]+"},52 {"%time", "[ ]*" + time_re + "[ ]+ns"},53 {"%console_report", "[ ]*" + time_re + "[ ]+ns [ ]*" + time_re + "[ ]+ns [ ]*[0-9]+"},54 {"%console_percentage_report", "[ ]*" + percentage_re + "[ ]+% [ ]*" + percentage_re + "[ ]+% [ ]*[0-9]+"},55 {"%console_us_report", "[ ]*" + time_re + "[ ]+us [ ]*" + time_re + "[ ]+us [ ]*[0-9]+"},56 {"%console_ms_report", "[ ]*" + time_re + "[ ]+ms [ ]*" + time_re + "[ ]+ms [ ]*[0-9]+"},57 {"%console_s_report", "[ ]*" + time_re + "[ ]+s [ ]*" + time_re + "[ ]+s [ ]*[0-9]+"},58 {"%console_time_only_report", "[ ]*" + time_re + "[ ]+ns [ ]*" + time_re + "[ ]+ns"},59 {"%console_us_report", "[ ]*" + time_re + "[ ]+us [ ]*" + time_re + "[ ]+us [ ]*[0-9]+"},60 {"%console_us_time_only_report", "[ ]*" + time_re + "[ ]+us [ ]*" + time_re + "[ ]+us"},61 {"%csv_header",62 "name,iterations,real_time,cpu_time,time_unit,bytes_per_second,"63 "items_per_second,label,error_occurred,error_message"},64 {"%csv_report", "[0-9]+," + safe_dec_re + "," + safe_dec_re + ",ns,,,,,"},65 {"%csv_us_report", "[0-9]+," + safe_dec_re + "," + safe_dec_re + ",us,,,,,"},66 {"%csv_ms_report", "[0-9]+," + safe_dec_re + "," + safe_dec_re + ",ms,,,,,"},67 {"%csv_s_report", "[0-9]+," + safe_dec_re + "," + safe_dec_re + ",s,,,,,"},68 {"%csv_cv_report", "[0-9]+," + safe_dec_re + "," + safe_dec_re + ",,,,,,"},69 {"%csv_bytes_report",70 "[0-9]+," + safe_dec_re + "," + safe_dec_re + ",ns," + safe_dec_re + ",,,,"},71 {"%csv_items_report",72 "[0-9]+," + safe_dec_re + "," + safe_dec_re + ",ns,," + safe_dec_re + ",,,"},73 {"%csv_bytes_items_report",74 "[0-9]+," + safe_dec_re + "," + safe_dec_re + ",ns," + safe_dec_re +75 "," + safe_dec_re + ",,,"},76 {"%csv_label_report_begin", "[0-9]+," + safe_dec_re + "," + safe_dec_re + ",ns,,,"},77 {"%csv_label_report_end", ",,"}};78 // clang-format on79 return map;80}81 82std::string PerformSubstitutions(std::string source) {83 SubMap const& subs = GetSubstitutions();84 using SizeT = std::string::size_type;85 for (auto const& KV : subs) {86 SizeT pos;87 SizeT next_start = 0;88 while ((pos = source.find(KV.first, next_start)) != std::string::npos) {89 next_start = pos + KV.second.size();90 source.replace(pos, KV.first.size(), KV.second);91 }92 }93 return source;94}95 96void CheckCase(std::stringstream& remaining_output, TestCase const& TC,97 TestCaseList const& not_checks) {98 std::string first_line;99 bool on_first = true;100 std::string line;101 while (remaining_output.eof() == false) {102 BM_CHECK(remaining_output.good());103 std::getline(remaining_output, line);104 if (on_first) {105 first_line = line;106 on_first = false;107 }108 for (const auto& NC : not_checks) {109 BM_CHECK(!NC.regex->Match(line))110 << "Unexpected match for line \"" << line << "\" for MR_Not regex \""111 << NC.regex_str << "\""112 << "\n actual regex string \"" << TC.substituted_regex << "\""113 << "\n started matching near: " << first_line;114 }115 if (TC.regex->Match(line)) return;116 BM_CHECK(TC.match_rule != MR_Next)117 << "Expected line \"" << line << "\" to match regex \"" << TC.regex_str118 << "\""119 << "\n actual regex string \"" << TC.substituted_regex << "\""120 << "\n started matching near: " << first_line;121 }122 BM_CHECK(remaining_output.eof() == false)123 << "End of output reached before match for regex \"" << TC.regex_str124 << "\" was found"125 << "\n actual regex string \"" << TC.substituted_regex << "\""126 << "\n started matching near: " << first_line;127}128 129void CheckCases(TestCaseList const& checks, std::stringstream& output) {130 std::vector<TestCase> not_checks;131 for (size_t i = 0; i < checks.size(); ++i) {132 const auto& TC = checks[i];133 if (TC.match_rule == MR_Not) {134 not_checks.push_back(TC);135 continue;136 }137 CheckCase(output, TC, not_checks);138 not_checks.clear();139 }140}141 142class TestReporter : public benchmark::BenchmarkReporter {143 public:144 TestReporter(std::vector<benchmark::BenchmarkReporter*> reps)145 : reporters_(std::move(reps)) {}146 147 bool ReportContext(const Context& context) override {148 bool last_ret = false;149 bool first = true;150 for (auto rep : reporters_) {151 bool new_ret = rep->ReportContext(context);152 BM_CHECK(first || new_ret == last_ret)153 << "Reports return different values for ReportContext";154 first = false;155 last_ret = new_ret;156 }157 (void)first;158 return last_ret;159 }160 161 void ReportRuns(const std::vector<Run>& report) override {162 for (auto rep : reporters_) rep->ReportRuns(report);163 }164 void Finalize() override {165 for (auto rep : reporters_) rep->Finalize();166 }167 168 private:169 std::vector<benchmark::BenchmarkReporter*> reporters_;170};171} // namespace172 173} // end namespace internal174 175// ========================================================================= //176// -------------------------- Results checking ----------------------------- //177// ========================================================================= //178 179namespace internal {180 181// Utility class to manage subscribers for checking benchmark results.182// It works by parsing the CSV output to read the results.183class ResultsChecker {184 public:185 struct PatternAndFn : public TestCase { // reusing TestCase for its regexes186 PatternAndFn(const std::string& rx, ResultsCheckFn fn_)187 : TestCase(rx), fn(std::move(fn_)) {}188 ResultsCheckFn fn;189 };190 191 std::vector<PatternAndFn> check_patterns;192 std::vector<Results> results;193 std::vector<std::string> field_names;194 195 void Add(const std::string& entry_pattern, const ResultsCheckFn& fn);196 197 void CheckResults(std::stringstream& output);198 199 private:200 void SetHeader_(const std::string& csv_header);201 void SetValues_(const std::string& entry_csv_line);202 203 std::vector<std::string> SplitCsv_(const std::string& line);204};205 206// store the static ResultsChecker in a function to prevent initialization207// order problems208ResultsChecker& GetResultsChecker() {209 static ResultsChecker rc;210 return rc;211}212 213// add a results checker for a benchmark214void ResultsChecker::Add(const std::string& entry_pattern,215 const ResultsCheckFn& fn) {216 check_patterns.emplace_back(entry_pattern, fn);217}218 219// check the results of all subscribed benchmarks220void ResultsChecker::CheckResults(std::stringstream& output) {221 // first reset the stream to the start222 {223 auto start = std::stringstream::pos_type(0);224 // clear before calling tellg()225 output.clear();226 // seek to zero only when needed227 if (output.tellg() > start) output.seekg(start);228 // and just in case229 output.clear();230 }231 // now go over every line and publish it to the ResultsChecker232 std::string line;233 bool on_first = true;234 while (output.eof() == false) {235 BM_CHECK(output.good());236 std::getline(output, line);237 if (on_first) {238 SetHeader_(line); // this is important239 on_first = false;240 continue;241 }242 SetValues_(line);243 }244 // finally we can call the subscribed check functions245 for (const auto& p : check_patterns) {246 BM_VLOG(2) << "--------------------------------\n";247 BM_VLOG(2) << "checking for benchmarks matching " << p.regex_str << "...\n";248 for (const auto& r : results) {249 if (!p.regex->Match(r.name)) {250 BM_VLOG(2) << p.regex_str << " is not matched by " << r.name << "\n";251 continue;252 }253 BM_VLOG(2) << p.regex_str << " is matched by " << r.name << "\n";254 BM_VLOG(1) << "Checking results of " << r.name << ": ... \n";255 p.fn(r);256 BM_VLOG(1) << "Checking results of " << r.name << ": OK.\n";257 }258 }259}260 261// prepare for the names in this header262void ResultsChecker::SetHeader_(const std::string& csv_header) {263 field_names = SplitCsv_(csv_header);264}265 266// set the values for a benchmark267void ResultsChecker::SetValues_(const std::string& entry_csv_line) {268 if (entry_csv_line.empty()) return; // some lines are empty269 BM_CHECK(!field_names.empty());270 auto vals = SplitCsv_(entry_csv_line);271 BM_CHECK_EQ(vals.size(), field_names.size());272 results.emplace_back(vals[0]); // vals[0] is the benchmark name273 auto& entry = results.back();274 for (size_t i = 1, e = vals.size(); i < e; ++i) {275 entry.values[field_names[i]] = vals[i];276 }277}278 279// a quick'n'dirty csv splitter (eliminating quotes)280std::vector<std::string> ResultsChecker::SplitCsv_(const std::string& line) {281 std::vector<std::string> out;282 if (line.empty()) return out;283 if (!field_names.empty()) out.reserve(field_names.size());284 size_t prev = 0, pos = line.find_first_of(','), curr = pos;285 while (pos != line.npos) {286 BM_CHECK(curr > 0);287 if (line[prev] == '"') ++prev;288 if (line[curr - 1] == '"') --curr;289 out.push_back(line.substr(prev, curr - prev));290 prev = pos + 1;291 pos = line.find_first_of(',', pos + 1);292 curr = pos;293 }294 curr = line.size();295 if (line[prev] == '"') ++prev;296 if (line[curr - 1] == '"') --curr;297 out.push_back(line.substr(prev, curr - prev));298 return out;299}300 301} // end namespace internal302 303size_t AddChecker(const std::string& bm_name, const ResultsCheckFn& fn) {304 auto& rc = internal::GetResultsChecker();305 rc.Add(bm_name, fn);306 return rc.results.size();307}308 309int Results::NumThreads() const {310 auto pos = name.find("/threads:");311 if (pos == name.npos) return 1;312 auto end = name.find('/', pos + 9);313 std::stringstream ss;314 ss << name.substr(pos + 9, end);315 int num = 1;316 ss >> num;317 BM_CHECK(!ss.fail());318 return num;319}320 321double Results::NumIterations() const { return GetAs<double>("iterations"); }322 323double Results::GetTime(BenchmarkTime which) const {324 BM_CHECK(which == kCpuTime || which == kRealTime);325 const char* which_str = which == kCpuTime ? "cpu_time" : "real_time";326 double val = GetAs<double>(which_str);327 auto unit = Get("time_unit");328 BM_CHECK(unit);329 if (*unit == "ns") {330 return val * 1.e-9;331 }332 if (*unit == "us") {333 return val * 1.e-6;334 }335 if (*unit == "ms") {336 return val * 1.e-3;337 }338 if (*unit == "s") {339 return val;340 }341 BM_CHECK(1 == 0) << "unknown time unit: " << *unit;342 return 0;343}344 345// ========================================================================= //346// -------------------------- Public API Definitions------------------------ //347// ========================================================================= //348 349TestCase::TestCase(std::string re, int rule)350 : regex_str(std::move(re)),351 match_rule(rule),352 substituted_regex(internal::PerformSubstitutions(regex_str)),353 regex(std::make_shared<benchmark::Regex>()) {354 std::string err_str;355 regex->Init(substituted_regex, &err_str);356 BM_CHECK(err_str.empty())357 << "Could not construct regex \"" << substituted_regex << "\""358 << "\n originally \"" << regex_str << "\""359 << "\n got error: " << err_str;360}361 362int AddCases(TestCaseID ID, std::initializer_list<TestCase> il) {363 auto& L = internal::GetTestCaseList(ID);364 L.insert(L.end(), il);365 return 0;366}367 368int SetSubstitutions(369 std::initializer_list<std::pair<std::string, std::string>> il) {370 auto& subs = internal::GetSubstitutions();371 for (auto KV : il) {372 bool exists = false;373 KV.second = internal::PerformSubstitutions(KV.second);374 for (auto& EKV : subs) {375 if (EKV.first == KV.first) {376 EKV.second = std::move(KV.second);377 exists = true;378 break;379 }380 }381 if (!exists) subs.push_back(std::move(KV));382 }383 return 0;384}385 386// Disable deprecated warnings temporarily because we need to reference387// CSVReporter but don't want to trigger -Werror=-Wdeprecated-declarations388BENCHMARK_DISABLE_DEPRECATED_WARNING389 390void RunOutputTests(int argc, char* argv[]) {391 using internal::GetTestCaseList;392 benchmark::Initialize(&argc, argv);393 auto options = benchmark::internal::GetOutputOptions(/*force_no_color*/ true);394 benchmark::ConsoleReporter CR(options);395 benchmark::JSONReporter JR;396 benchmark::CSVReporter CSVR;397 struct ReporterTest {398 std::string name;399 std::vector<TestCase>& output_cases;400 std::vector<TestCase>& error_cases;401 benchmark::BenchmarkReporter& reporter;402 std::stringstream out_stream;403 std::stringstream err_stream;404 405 ReporterTest(const std::string& n, std::vector<TestCase>& out_tc,406 std::vector<TestCase>& err_tc,407 benchmark::BenchmarkReporter& br)408 : name(n), output_cases(out_tc), error_cases(err_tc), reporter(br) {409 reporter.SetOutputStream(&out_stream);410 reporter.SetErrorStream(&err_stream);411 }412 } TestCases[] = {413 {std::string("ConsoleReporter"), GetTestCaseList(TC_ConsoleOut),414 GetTestCaseList(TC_ConsoleErr), CR},415 {std::string("JSONReporter"), GetTestCaseList(TC_JSONOut),416 GetTestCaseList(TC_JSONErr), JR},417 {std::string("CSVReporter"), GetTestCaseList(TC_CSVOut),418 GetTestCaseList(TC_CSVErr), CSVR},419 };420 421 // Create the test reporter and run the benchmarks.422 std::cout << "Running benchmarks...\n";423 internal::TestReporter test_rep({&CR, &JR, &CSVR});424 benchmark::RunSpecifiedBenchmarks(&test_rep);425 426 for (auto& rep_test : TestCases) {427 std::string msg =428 std::string("\nTesting ") + rep_test.name + std::string(" Output\n");429 std::string banner(msg.size() - 1, '-');430 std::cout << banner << msg << banner << "\n";431 432 std::cerr << rep_test.err_stream.str();433 std::cout << rep_test.out_stream.str();434 435 internal::CheckCases(rep_test.error_cases, rep_test.err_stream);436 internal::CheckCases(rep_test.output_cases, rep_test.out_stream);437 438 std::cout << "\n";439 }440 441 // now that we know the output is as expected, we can dispatch442 // the checks to subscribees.443 auto& csv = TestCases[2];444 // would use == but gcc spits a warning445 BM_CHECK(csv.name == std::string("CSVReporter"));446 internal::GetResultsChecker().CheckResults(csv.out_stream);447}448 449BENCHMARK_RESTORE_DEPRECATED_WARNING450 451int SubstrCnt(const std::string& haystack, const std::string& pat) {452 if (pat.length() == 0) return 0;453 int count = 0;454 for (size_t offset = haystack.find(pat); offset != std::string::npos;455 offset = haystack.find(pat, offset + pat.length()))456 ++count;457 return count;458}459 460static char ToHex(int ch) {461 return ch < 10 ? static_cast<char>('0' + ch)462 : static_cast<char>('a' + (ch - 10));463}464 465static char RandomHexChar() {466 static std::mt19937 rd{std::random_device{}()};467 static std::uniform_int_distribution<int> mrand{0, 15};468 return ToHex(mrand(rd));469}470 471static std::string GetRandomFileName() {472 std::string model = "test.%%%%%%";473 for (auto& ch : model) {474 if (ch == '%') ch = RandomHexChar();475 }476 return model;477}478 479static bool FileExists(std::string const& name) {480 std::ifstream in(name.c_str());481 return in.good();482}483 484static std::string GetTempFileName() {485 // This function attempts to avoid race conditions where two tests486 // create the same file at the same time. However, it still introduces races487 // similar to tmpnam.488 int retries = 3;489 while (--retries) {490 std::string name = GetRandomFileName();491 if (!FileExists(name)) return name;492 }493 std::cerr << "Failed to create unique temporary file name" << std::endl;494 std::abort();495}496 497std::string GetFileReporterOutput(int argc, char* argv[]) {498 std::vector<char*> new_argv(argv, argv + argc);499 assert(static_cast<decltype(new_argv)::size_type>(argc) == new_argv.size());500 501 std::string tmp_file_name = GetTempFileName();502 std::cout << "Will be using this as the tmp file: " << tmp_file_name << '\n';503 504 std::string tmp = "--benchmark_out=";505 tmp += tmp_file_name;506 new_argv.emplace_back(const_cast<char*>(tmp.c_str()));507 508 argc = int(new_argv.size());509 510 benchmark::Initialize(&argc, new_argv.data());511 benchmark::RunSpecifiedBenchmarks();512 513 // Read the output back from the file, and delete the file.514 std::ifstream tmp_stream(tmp_file_name);515 std::string output = std::string((std::istreambuf_iterator<char>(tmp_stream)),516 std::istreambuf_iterator<char>());517 std::remove(tmp_file_name.c_str());518 519 return output;520}521