212 lines · c
1#ifndef TEST_OUTPUT_TEST_H2#define TEST_OUTPUT_TEST_H3 4#undef NDEBUG5#include <functional>6#include <initializer_list>7#include <memory>8#include <sstream>9#include <string>10#include <utility>11#include <vector>12 13#include "../src/re.h"14#include "benchmark/benchmark.h"15 16#define CONCAT2(x, y) x##y17#define CONCAT(x, y) CONCAT2(x, y)18 19#define ADD_CASES(...) int CONCAT(dummy, __LINE__) = ::AddCases(__VA_ARGS__)20 21#define SET_SUBSTITUTIONS(...) \22 int CONCAT(dummy, __LINE__) = ::SetSubstitutions(__VA_ARGS__)23 24enum MatchRules {25 MR_Default, // Skip non-matching lines until a match is found.26 MR_Next, // Match must occur on the next line.27 MR_Not // No line between the current position and the next match matches28 // the regex29};30 31struct TestCase {32 TestCase(std::string re, int rule = MR_Default);33 34 std::string regex_str;35 int match_rule;36 std::string substituted_regex;37 std::shared_ptr<benchmark::Regex> regex;38};39 40enum TestCaseID {41 TC_ConsoleOut,42 TC_ConsoleErr,43 TC_JSONOut,44 TC_JSONErr,45 TC_CSVOut,46 TC_CSVErr,47 48 TC_NumID // PRIVATE49};50 51// Add a list of test cases to be run against the output specified by52// 'ID'53int AddCases(TestCaseID ID, std::initializer_list<TestCase> il);54 55// Add or set a list of substitutions to be performed on constructed regex's56// See 'output_test_helper.cc' for a list of default substitutions.57int SetSubstitutions(58 std::initializer_list<std::pair<std::string, std::string>> il);59 60// Run all output tests.61void RunOutputTests(int argc, char* argv[]);62 63// Count the number of 'pat' substrings in the 'haystack' string.64int SubstrCnt(const std::string& haystack, const std::string& pat);65 66// Run registered benchmarks with file reporter enabled, and return the content67// outputted by the file reporter.68std::string GetFileReporterOutput(int argc, char* argv[]);69 70// ========================================================================= //71// ------------------------- Results checking ------------------------------ //72// ========================================================================= //73 74// Call this macro to register a benchmark for checking its results. This75// should be all that's needed. It subscribes a function to check the (CSV)76// results of a benchmark. This is done only after verifying that the output77// strings are really as expected.78// bm_name_pattern: a name or a regex pattern which will be matched against79// all the benchmark names. Matching benchmarks80// will be the subject of a call to checker_function81// checker_function: should be of type ResultsCheckFn (see below)82#define CHECK_BENCHMARK_RESULTS(bm_name_pattern, checker_function) \83 size_t CONCAT(dummy, __LINE__) = AddChecker(bm_name_pattern, checker_function)84 85struct Results;86typedef std::function<void(Results const&)> ResultsCheckFn;87 88size_t AddChecker(const std::string& bm_name_pattern, const ResultsCheckFn& fn);89 90// Class holding the results of a benchmark.91// It is passed in calls to checker functions.92struct Results {93 // the benchmark name94 std::string name;95 // the benchmark fields96 std::map<std::string, std::string> values;97 98 Results(const std::string& n) : name(n) {}99 100 int NumThreads() const;101 102 double NumIterations() const;103 104 typedef enum { kCpuTime, kRealTime } BenchmarkTime;105 106 // get cpu_time or real_time in seconds107 double GetTime(BenchmarkTime which) const;108 109 // get the real_time duration of the benchmark in seconds.110 // it is better to use fuzzy float checks for this, as the float111 // ASCII formatting is lossy.112 double DurationRealTime() const {113 return NumIterations() * GetTime(kRealTime);114 }115 // get the cpu_time duration of the benchmark in seconds116 double DurationCPUTime() const { return NumIterations() * GetTime(kCpuTime); }117 118 // get the string for a result by name, or nullptr if the name119 // is not found120 const std::string* Get(const std::string& entry_name) const {121 auto it = values.find(entry_name);122 if (it == values.end()) return nullptr;123 return &it->second;124 }125 126 // get a result by name, parsed as a specific type.127 // NOTE: for counters, use GetCounterAs instead.128 template <class T>129 T GetAs(const std::string& entry_name) const;130 131 // counters are written as doubles, so they have to be read first132 // as a double, and only then converted to the asked type.133 template <class T>134 T GetCounterAs(const std::string& entry_name) const {135 double dval = GetAs<double>(entry_name);136 T tval = static_cast<T>(dval);137 return tval;138 }139};140 141template <class T>142T Results::GetAs(const std::string& entry_name) const {143 auto* sv = Get(entry_name);144 BM_CHECK(sv != nullptr && !sv->empty());145 std::stringstream ss;146 ss << *sv;147 T out;148 ss >> out;149 BM_CHECK(!ss.fail());150 return out;151}152 153//----------------------------------154// Macros to help in result checking. Do not use them with arguments causing155// side-effects.156 157// clang-format off158 159#define CHECK_RESULT_VALUE_IMPL(entry, getfn, var_type, var_name, relationship, value) \160 CONCAT(BM_CHECK_, relationship) \161 (entry.getfn< var_type >(var_name), (value)) << "\n" \162 << __FILE__ << ":" << __LINE__ << ": " << (entry).name << ":\n" \163 << __FILE__ << ":" << __LINE__ << ": " \164 << "expected (" << #var_type << ")" << (var_name) \165 << "=" << (entry).getfn< var_type >(var_name) \166 << " to be " #relationship " to " << (value) << "\n"167 168// check with tolerance. eps_factor is the tolerance window, which is169// interpreted relative to value (eg, 0.1 means 10% of value).170#define CHECK_FLOAT_RESULT_VALUE_IMPL(entry, getfn, var_type, var_name, relationship, value, eps_factor) \171 CONCAT(BM_CHECK_FLOAT_, relationship) \172 (entry.getfn< var_type >(var_name), (value), (eps_factor) * (value)) << "\n" \173 << __FILE__ << ":" << __LINE__ << ": " << (entry).name << ":\n" \174 << __FILE__ << ":" << __LINE__ << ": " \175 << "expected (" << #var_type << ")" << (var_name) \176 << "=" << (entry).getfn< var_type >(var_name) \177 << " to be " #relationship " to " << (value) << "\n" \178 << __FILE__ << ":" << __LINE__ << ": " \179 << "with tolerance of " << (eps_factor) * (value) \180 << " (" << (eps_factor)*100. << "%), " \181 << "but delta was " << ((entry).getfn< var_type >(var_name) - (value)) \182 << " (" << (((entry).getfn< var_type >(var_name) - (value)) \183 / \184 ((value) > 1.e-5 || value < -1.e-5 ? value : 1.e-5)*100.) \185 << "%)"186 187#define CHECK_RESULT_VALUE(entry, var_type, var_name, relationship, value) \188 CHECK_RESULT_VALUE_IMPL(entry, GetAs, var_type, var_name, relationship, value)189 190#define CHECK_COUNTER_VALUE(entry, var_type, var_name, relationship, value) \191 CHECK_RESULT_VALUE_IMPL(entry, GetCounterAs, var_type, var_name, relationship, value)192 193#define CHECK_FLOAT_RESULT_VALUE(entry, var_name, relationship, value, eps_factor) \194 CHECK_FLOAT_RESULT_VALUE_IMPL(entry, GetAs, double, var_name, relationship, value, eps_factor)195 196#define CHECK_FLOAT_COUNTER_VALUE(entry, var_name, relationship, value, eps_factor) \197 CHECK_FLOAT_RESULT_VALUE_IMPL(entry, GetCounterAs, double, var_name, relationship, value, eps_factor)198 199// clang-format on200 201// ========================================================================= //202// --------------------------- Misc Utilities ------------------------------ //203// ========================================================================= //204 205namespace {206 207const char* const dec_re = "[0-9]*[.]?[0-9]+([eE][-+][0-9]+)?";208 209} // end namespace210 211#endif // TEST_OUTPUT_TEST_H212