#include <iostream> #include <map> #include <memory> #include <sstream> #include <cstring> #include "../src/check.h" // NOTE: check.h is for internal use only! #include "../src/re.h" // NOTE: re.h is for internal use only #include "output_test.h" #include "../src/benchmark_api_internal.h" // ========================================================================= // // ------------------------------ Internals -------------------------------- // // ========================================================================= // namespace internal { namespace { using TestCaseList = std::vector<TestCase>; // Use a vector because the order elements are added matters during iteration. // std::map/unordered_map don't guarantee that. // For example: // SetSubstitutions({{"%HelloWorld", "Hello"}, {"%Hello", "Hi"}}); // Substitute("%HelloWorld") // Always expands to Hello. using SubMap = std::vector<std::pair<std::string, std::string>>; TestCaseList& GetTestCaseList(TestCaseID ID) { // Uses function-local statics to ensure initialization occurs // before first use. static TestCaseList lists[TC_NumID]; return lists[ID]; } SubMap& GetSubstitutions() { // Don't use 'dec_re' from header because it may not yet be initialized. static std::string safe_dec_re = "[0-9]*[.]?[0-9]+([eE][-+][0-9]+)?"; static SubMap map = { {"%float", "[0-9]*[.]?[0-9]+([eE][-+][0-9]+)?"}, // human-readable float {"%hrfloat", "[0-9]*[.]?[0-9]+([eE][-+][0-9]+)?[kMGTPEZYmunpfazy]?"}, {"%int", "[ ]*[0-9]+"}, {" %s ", "[ ]+"}, {"%time", "[ ]*[0-9]{1,5} ns"}, {"%console_report", "[ ]*[0-9]{1,5} ns [ ]*[0-9]{1,5} ns [ ]*[0-9]+"}, {"%console_us_report", "[ ]*[0-9] us [ ]*[0-9] us [ ]*[0-9]+"}, {"%csv_header", "name,iterations,real_time,cpu_time,time_unit,bytes_per_second," "items_per_second,label,error_occurred,error_message"}, {"%csv_report", "[0-9]+," + safe_dec_re + "," + safe_dec_re + ",ns,,,,,"}, {"%csv_us_report", "[0-9]+," + safe_dec_re + "," + safe_dec_re + ",us,,,,,"}, {"%csv_bytes_report", "[0-9]+," + safe_dec_re + "," + safe_dec_re + ",ns," + safe_dec_re + ",,,,"}, {"%csv_items_report", "[0-9]+," + safe_dec_re + "," + safe_dec_re + ",ns,," + safe_dec_re + ",,,"}, {"%csv_bytes_items_report", "[0-9]+," + safe_dec_re + "," + safe_dec_re + ",ns," + safe_dec_re + "," + safe_dec_re + ",,,"}, {"%csv_label_report_begin", "[0-9]+," + safe_dec_re + "," + safe_dec_re + ",ns,,,"}, {"%csv_label_report_end", ",,"}}; return map; } std::string PerformSubstitutions(std::string source) { SubMap const& subs = GetSubstitutions(); using SizeT = std::string::size_type; for (auto const& KV : subs) { SizeT pos; SizeT next_start = 0; while ((pos = source.find(KV.first, next_start)) != std::string::npos) { next_start = pos + KV.second.size(); source.replace(pos, KV.first.size(), KV.second); } } return source; } void CheckCase(std::stringstream& remaining_output, TestCase const& TC, TestCaseList const& not_checks) { std::string first_line; bool on_first = true; std::string line; while (remaining_output.eof() == false) { CHECK(remaining_output.good()); std::getline(remaining_output, line); if (on_first) { first_line = line; on_first = false; } for (const auto& NC : not_checks) { CHECK(!NC.regex->Match(line)) << "Unexpected match for line \"" << line << "\" for MR_Not regex \"" << NC.regex_str << "\"" << "\n actual regex string \"" << TC.substituted_regex << "\"" << "\n started matching near: " << first_line; } if (TC.regex->Match(line)) return; CHECK(TC.match_rule != MR_Next) << "Expected line \"" << line << "\" to match regex \"" << TC.regex_str << "\"" << "\n actual regex string \"" << TC.substituted_regex << "\"" << "\n started matching near: " << first_line; } CHECK(remaining_output.eof() == false) << "End of output reached before match for regex \"" << TC.regex_str << "\" was found" << "\n actual regex string \"" << TC.substituted_regex << "\"" << "\n started matching near: " << first_line; } void CheckCases(TestCaseList const& checks, std::stringstream& output) { std::vector<TestCase> not_checks; for (size_t i = 0; i < checks.size(); ++i) { const auto& TC = checks[i]; if (TC.match_rule == MR_Not) { not_checks.push_back(TC); continue; } CheckCase(output, TC, not_checks); not_checks.clear(); } } class TestReporter : public benchmark::BenchmarkReporter { public: TestReporter(std::vector<benchmark::BenchmarkReporter*> reps) : reporters_(reps) {} virtual bool ReportContext(const Context& context) { bool last_ret = false; bool first = true; for (auto rep : reporters_) { bool new_ret = rep->ReportContext(context); CHECK(first || new_ret == last_ret) << "Reports return different values for ReportContext"; first = false; last_ret = new_ret; } (void)first; return last_ret; } void ReportRuns(const std::vector<Run>& report) { for (auto rep : reporters_) rep->ReportRuns(report); } void Finalize() { for (auto rep : reporters_) rep->Finalize(); } private: std::vector<benchmark::BenchmarkReporter *> reporters_; }; } } // end namespace internal // ========================================================================= // // -------------------------- Results checking ----------------------------- // // ========================================================================= // namespace internal { // Utility class to manage subscribers for checking benchmark results. // It works by parsing the CSV output to read the results. class ResultsChecker { public: struct PatternAndFn : public TestCase { // reusing TestCase for its regexes PatternAndFn(const std::string& rx, ResultsCheckFn fn_) : TestCase(rx), fn(fn_) {} ResultsCheckFn fn; }; std::vector< PatternAndFn > check_patterns; std::vector< Results > results; std::vector< std::string > field_names; void Add(const std::string& entry_pattern, ResultsCheckFn fn); void CheckResults(std::stringstream& output); private: void SetHeader_(const std::string& csv_header); void SetValues_(const std::string& entry_csv_line); std::vector< std::string > SplitCsv_(const std::string& line); }; // store the static ResultsChecker in a function to prevent initialization // order problems ResultsChecker& GetResultsChecker() { static ResultsChecker rc; return rc; } // add a results checker for a benchmark void ResultsChecker::Add(const std::string& entry_pattern, ResultsCheckFn fn) { check_patterns.emplace_back(entry_pattern, fn); } // check the results of all subscribed benchmarks void ResultsChecker::CheckResults(std::stringstream& output) { // first reset the stream to the start { auto start = std::ios::streampos(0); // clear before calling tellg() output.clear(); // seek to zero only when needed if(output.tellg() > start) output.seekg(start); // and just in case output.clear(); } // now go over every line and publish it to the ResultsChecker std::string line; bool on_first = true; while (output.eof() == false) { CHECK(output.good()); std::getline(output, line); if (on_first) { SetHeader_(line); // this is important on_first = false; continue; } SetValues_(line); } // finally we can call the subscribed check functions for(const auto& p : check_patterns) { VLOG(2) << "--------------------------------\n"; VLOG(2) << "checking for benchmarks matching " << p.regex_str << "...\n"; for(const auto& r : results) { if(!p.regex->Match(r.name)) { VLOG(2) << p.regex_str << " is not matched by " << r.name << "\n"; continue; } else { VLOG(2) << p.regex_str << " is matched by " << r.name << "\n"; } VLOG(1) << "Checking results of " << r.name << ": ... \n"; p.fn(r); VLOG(1) << "Checking results of " << r.name << ": OK.\n"; } } } // prepare for the names in this header void ResultsChecker::SetHeader_(const std::string& csv_header) { field_names = SplitCsv_(csv_header); } // set the values for a benchmark void ResultsChecker::SetValues_(const std::string& entry_csv_line) { if(entry_csv_line.empty()) return; // some lines are empty CHECK(!field_names.empty()); auto vals = SplitCsv_(entry_csv_line); CHECK_EQ(vals.size(), field_names.size()); results.emplace_back(vals[0]); // vals[0] is the benchmark name auto &entry = results.back(); for (size_t i = 1, e = vals.size(); i < e; ++i) { entry.values[field_names[i]] = vals[i]; } } // a quick'n'dirty csv splitter (eliminating quotes) std::vector< std::string > ResultsChecker::SplitCsv_(const std::string& line) { std::vector< std::string > out; if(line.empty()) return out; if(!field_names.empty()) out.reserve(field_names.size()); size_t prev = 0, pos = line.find_first_of(','), curr = pos; while(pos != line.npos) { CHECK(curr > 0); if(line[prev] == '"') ++prev; if(line[curr-1] == '"') --curr; out.push_back(line.substr(prev, curr-prev)); prev = pos + 1; pos = line.find_first_of(',', pos + 1); curr = pos; } curr = line.size(); if(line[prev] == '"') ++prev; if(line[curr-1] == '"') --curr; out.push_back(line.substr(prev, curr-prev)); return out; } } // end namespace internal size_t AddChecker(const char* bm_name, ResultsCheckFn fn) { auto &rc = internal::GetResultsChecker(); rc.Add(bm_name, fn); return rc.results.size(); } int Results::NumThreads() const { auto pos = name.find("/threads:"); if(pos == name.npos) return 1; auto end = name.find('/', pos + 9); std::stringstream ss; ss << name.substr(pos + 9, end); int num = 1; ss >> num; CHECK(!ss.fail()); return num; } double Results::GetTime(BenchmarkTime which) const { CHECK(which == kCpuTime || which == kRealTime); const char *which_str = which == kCpuTime ? "cpu_time" : "real_time"; double val = GetAs< double >(which_str); auto unit = Get("time_unit"); CHECK(unit); if(*unit == "ns") { return val * 1.e-9; } else if(*unit == "us") { return val * 1.e-6; } else if(*unit == "ms") { return val * 1.e-3; } else if(*unit == "s") { return val; } else { CHECK(1 == 0) << "unknown time unit: " << *unit; return 0; } } // ========================================================================= // // -------------------------- Public API Definitions------------------------ // // ========================================================================= // TestCase::TestCase(std::string re, int rule) : regex_str(std::move(re)), match_rule(rule), substituted_regex(internal::PerformSubstitutions(regex_str)), regex(std::make_shared<benchmark::Regex>()) { std::string err_str; regex->Init(substituted_regex,& err_str); CHECK(err_str.empty()) << "Could not construct regex \"" << substituted_regex << "\"" << "\n originally \"" << regex_str << "\"" << "\n got error: " << err_str; } int AddCases(TestCaseID ID, std::initializer_list<TestCase> il) { auto& L = internal::GetTestCaseList(ID); L.insert(L.end(), il); return 0; } int SetSubstitutions( std::initializer_list<std::pair<std::string, std::string>> il) { auto& subs = internal::GetSubstitutions(); for (auto KV : il) { bool exists = false; KV.second = internal::PerformSubstitutions(KV.second); for (auto& EKV : subs) { if (EKV.first == KV.first) { EKV.second = std::move(KV.second); exists = true; break; } } if (!exists) subs.push_back(std::move(KV)); } return 0; } void RunOutputTests(int argc, char* argv[]) { using internal::GetTestCaseList; benchmark::Initialize(&argc, argv); auto options = benchmark::internal::GetOutputOptions(/*force_no_color*/true); benchmark::ConsoleReporter CR(options); benchmark::JSONReporter JR; benchmark::CSVReporter CSVR; struct ReporterTest { const char* name; std::vector<TestCase>& output_cases; std::vector<TestCase>& error_cases; benchmark::BenchmarkReporter& reporter; std::stringstream out_stream; std::stringstream err_stream; ReporterTest(const char* n, std::vector<TestCase>& out_tc, std::vector<TestCase>& err_tc, benchmark::BenchmarkReporter& br) : name(n), output_cases(out_tc), error_cases(err_tc), reporter(br) { reporter.SetOutputStream(&out_stream); reporter.SetErrorStream(&err_stream); } } TestCases[] = { {"ConsoleReporter", GetTestCaseList(TC_ConsoleOut), GetTestCaseList(TC_ConsoleErr), CR}, {"JSONReporter", GetTestCaseList(TC_JSONOut), GetTestCaseList(TC_JSONErr), JR}, {"CSVReporter", GetTestCaseList(TC_CSVOut), GetTestCaseList(TC_CSVErr), CSVR}, }; // Create the test reporter and run the benchmarks. std::cout << "Running benchmarks...\n"; internal::TestReporter test_rep({&CR, &JR, &CSVR}); benchmark::RunSpecifiedBenchmarks(&test_rep); for (auto& rep_test : TestCases) { std::string msg = std::string("\nTesting ") + rep_test.name + " Output\n"; std::string banner(msg.size() - 1, '-'); std::cout << banner << msg << banner << "\n"; std::cerr << rep_test.err_stream.str(); std::cout << rep_test.out_stream.str(); internal::CheckCases(rep_test.error_cases, rep_test.err_stream); internal::CheckCases(rep_test.output_cases, rep_test.out_stream); std::cout << "\n"; } // now that we know the output is as expected, we can dispatch // the checks to subscribees. auto &csv = TestCases[2]; // would use == but gcc spits a warning CHECK(std::strcmp(csv.name, "CSVReporter") == 0); internal::GetResultsChecker().CheckResults(csv.out_stream); }