From 47ce92d9201ba802fad40ab0029dc9e61eb86985 Mon Sep 17 00:00:00 2001 From: Diego Krupitza Date: Thu, 8 Jun 2023 01:02:33 +0200 Subject: [PATCH] Add `--benchmark_human_readable` flag Allows used to add a command line flag called `--benchmark_human_readable`. By adding this flag the arguments passed to benchmarks are formatted in a human friendly format. This means that numbers that are the power of 2 are formatted as `2^x` (e.g., 64 will be `2^6`). For numbers that are the power of 10 a different formatting style is used. Numbers 0-999 no formatting is used. For numbers 1000-999999 the format `k` is used (e.g., `32000` -> `32k`). This also works for millions, billions, trillions, ... For numbers greater than septillions no special formatting is used. The design is rather simple allowing to by easily extendable. Closes: #1006 --- AUTHORS | 3 +- CONTRIBUTORS | 1 + docs/user_guide.md | 51 +++++++++ src/benchmark.cc | 8 +- src/benchmark_api_internal.cc | 20 +++- src/benchmark_api_internal.h | 3 + src/string_util.cc | 43 +++++++- src/string_util.h | 29 +++++ test/BUILD | 1 + test/CMakeLists.txt | 7 ++ test/human_readable_formatting_test.cc | 122 +++++++++++++++++++++ test/human_readable_gtest.cc | 70 ++++++++++++ test/non_human_readable_formatting_test.cc | 113 +++++++++++++++++++ 13 files changed, 466 insertions(+), 5 deletions(-) create mode 100644 test/human_readable_formatting_test.cc create mode 100644 test/human_readable_gtest.cc create mode 100644 test/non_human_readable_formatting_test.cc diff --git a/AUTHORS b/AUTHORS index bafecaddb5..95ed28608b 100644 --- a/AUTHORS +++ b/AUTHORS @@ -20,7 +20,8 @@ Colin Braley Daniel Harvey David Coeurjolly Deniz Evrenci -Dirac Research +Dirac Research +Diego Krupitza Dominik Czarnota Dominik Korman Donald Aingworth diff --git a/CONTRIBUTORS b/CONTRIBUTORS index 56f03e2d62..8c1b4ae043 100644 --- a/CONTRIBUTORS +++ b/CONTRIBUTORS @@ -38,6 +38,7 @@ Cyrille Faucheux Daniel Harvey David Coeurjolly Deniz Evrenci +Diego Krupitza Dominic Hamon Dominik Czarnota Dominik Korman diff --git a/docs/user_guide.md b/docs/user_guide.md index 2ceb13eb59..59b7733cfa 100644 --- a/docs/user_guide.md +++ b/docs/user_guide.md @@ -185,6 +185,57 @@ BM_memcpy/32 12 ns 12 ns 54687500 BM_memcpy/32k 1834 ns 1837 ns 357143 ``` + + +## Human Readable Format + +The `--benchmark_human_readable={true|false}` option can be used to display +args in a more human friendly format. Meaning numbers that are power of 2 +will be formatted as `2^x`. Furthermore, for numbers that are the power of +10 special formatting abbreviations are used. For instance, `1000` will be +formatted to `1k`, `32000` to `32k`, `1000000` to `1m` and so on. + +By default `benchmark_human_readable` is disabled. + + +```bash +$ ./run_benchmarks.x +Run on (1 X 2300 MHz CPU ) +2016-06-25 19:34:24 +BM_base_two_args/1 2.22 ns 2.22 ns 6280843 +BM_base_two_args/2 2.20 ns 2.20 ns 6278027 +BM_base_two_args/64 2.14 ns 2.14 ns 6263982 +BM_base_two_args/128 2.22 ns 2.22 ns 6286484 +BM_base_ten_args/1 2.25 ns 2.25 ns 6349206 +BM_base_ten_args/10 2.18 ns 2.18 ns 6241641 +BM_base_ten_args/100 2.24 ns 2.24 ns 6167401 +BM_base_ten_args/1000 2.25 ns 2.25 ns 6137659 +BM_base_ten_args/10000 2.24 ns 2.24 ns 6068487 +BM_base_ten_args/32000 2.26 ns 2.26 ns 6063231 +BM_base_ten_args/100000 2.25 ns 2.25 ns 6105539 +BM_base_ten_args/1000000 2.21 ns 2.21 ns 6766554 +BM_base_ten_args/1000000000 2.23 ns 2.23 ns 6233304 +``` + +```bash +$ ./run_benchmarks.x --benchmark_human_readable +Run on (1 X 2300 MHz CPU ) +2016-06-25 19:34:24 +BM_base_two_args/1 2.18 ns 2.18 ns 6222222 +BM_base_two_args/2^1 2.24 ns 2.24 ns 6208426 +BM_base_two_args/2^6 2.24 ns 2.24 ns 6159261 +BM_base_two_args/2^7 2.25 ns 2.25 ns 6337709 +BM_base_ten_args/1 2.27 ns 2.27 ns 6071119 +BM_base_ten_args/10 2.32 ns 2.32 ns 6278027 +BM_base_ten_args/100 2.28 ns 2.27 ns 6029285 +BM_base_ten_args/1k 2.27 ns 2.26 ns 5845511 +BM_base_ten_args/10k 2.27 ns 2.26 ns 6252791 +BM_base_ten_args/32k 2.27 ns 2.26 ns 6208426 +BM_base_ten_args/100k 2.27 ns 2.26 ns 6129597 +BM_base_ten_args/1m 2.28 ns 2.28 ns 5988024 +BM_base_ten_args/1bn 2.25 ns 2.25 ns 6211180 +``` + ## Disabling Benchmarks It is possible to temporarily disable benchmarks by renaming the benchmark diff --git a/src/benchmark.cc b/src/benchmark.cc index f1633b703f..839d40547e 100644 --- a/src/benchmark.cc +++ b/src/benchmark.cc @@ -144,6 +144,9 @@ BM_DEFINE_string(benchmark_time_unit, ""); // The level of verbose logging to output BM_DEFINE_int32(v, 0); +// defines whether to use human-readable format or not +BM_DEFINE_bool(benchmark_human_readable, false); + namespace internal { std::map* global_context = nullptr; @@ -691,7 +694,9 @@ void ParseCommandLineFlags(int* argc, char** argv) { &FLAGS_benchmark_context) || ParseStringFlag(argv[i], "benchmark_time_unit", &FLAGS_benchmark_time_unit) || - ParseInt32Flag(argv[i], "v", &FLAGS_v)) { + ParseInt32Flag(argv[i], "v", &FLAGS_v) || + ParseBoolFlag(argv[i], "benchmark_human_readable", + &FLAGS_benchmark_human_readable)) { for (int j = i; j != *argc - 1; ++j) argv[j] = argv[j + 1]; --(*argc); @@ -743,6 +748,7 @@ void PrintDefaultHelp() { #endif " [--benchmark_context==,...]\n" " [--benchmark_time_unit={ns|us|ms|s}]\n" + " [--benchmark_human_readable={true|false}]\n" " [--v=]\n"); } diff --git a/src/benchmark_api_internal.cc b/src/benchmark_api_internal.cc index 286f986530..55c03b2861 100644 --- a/src/benchmark_api_internal.cc +++ b/src/benchmark_api_internal.cc @@ -5,6 +5,9 @@ #include "string_util.h" namespace benchmark { + +BM_DECLARE_bool(benchmark_human_readable); + namespace internal { BenchmarkInstance::BenchmarkInstance(Benchmark* benchmark, int family_idx, @@ -43,7 +46,13 @@ BenchmarkInstance::BenchmarkInstance(Benchmark* benchmark, int family_idx, } } - name_.args += StrFormat("%" PRId64, arg); + // formatting args either in default mode or in human-readable + if (FLAGS_benchmark_human_readable) { + name_.args += FormatHumanReadable(arg); + } else { + name_.args += StrFormat("%" PRId64, arg); + } + ++arg_i; } @@ -114,5 +123,14 @@ void BenchmarkInstance::Teardown() const { teardown_(st); } } + +/** + * Check whether the given value is a power of two or not. + * @param val the value to check + */ +bool IsPowerOfTwo(const int64_t& val) { + return (val & (val - 1)) == 0 && (val > 1); +} + } // namespace internal } // namespace benchmark diff --git a/src/benchmark_api_internal.h b/src/benchmark_api_internal.h index 94f516531b..8af8004742 100644 --- a/src/benchmark_api_internal.h +++ b/src/benchmark_api_internal.h @@ -1,6 +1,8 @@ #ifndef BENCHMARK_API_INTERNAL_H #define BENCHMARK_API_INTERNAL_H +#include +#include #include #include #include @@ -10,6 +12,7 @@ #include "benchmark/benchmark.h" #include "commandlineflags.h" +#include "string_util.h" namespace benchmark { namespace internal { diff --git a/src/string_util.cc b/src/string_util.cc index 5e2d24a3cd..973b42a89a 100644 --- a/src/string_util.cc +++ b/src/string_util.cc @@ -1,6 +1,7 @@ #include "string_util.h" #include +#include #ifdef BENCHMARK_STL_ANDROID_GNUSTL #include #endif @@ -15,6 +16,11 @@ namespace benchmark { namespace { +// Thousands, Millions, Billions, Trillions, Quadrillions, Quintillions, +// Sextillions, Septillions. +const std::array base10Units = {"k", "M", "B", "T", + "Q", "Qi", "Sx", "Sp"}; + // kilo, Mega, Giga, Tera, Peta, Exa, Zetta, Yotta. const char kBigSIUnits[] = "kMGTPEZY"; // Kibi, Mebi, Gibi, Tebi, Pebi, Exbi, Zebi, Yobi. @@ -32,7 +38,8 @@ static const int64_t kUnitsSize = arraysize(kBigSIUnits); void ToExponentAndMantissa(double val, double thresh, int precision, double one_k, std::string* mantissa, - int64_t* exponent) { + int64_t* exponent, + bool inclusiveBigThreshhold = false) { std::stringstream mantissa_stream; if (val < 0) { @@ -44,7 +51,8 @@ void ToExponentAndMantissa(double val, double thresh, int precision, // in 'precision' digits. const double adjusted_threshold = std::max(thresh, 1.0 / std::pow(10.0, precision)); - const double big_threshold = adjusted_threshold * one_k; + const double big_threshold = + (adjusted_threshold * one_k) - inclusiveBigThreshhold; const double small_threshold = adjusted_threshold; // Values in ]simple_threshold,small_threshold[ will be printed as-is const double simple_threshold = 0.01; @@ -262,4 +270,35 @@ double stod(const std::string& str, size_t* pos) { } #endif +std::string ExponentToBase10Prefix(int64_t exponent) { + if (exponent == 0) return ""; + + const int64_t index = (exponent > 0 ? exponent - 1 : -exponent - 1); + if (index >= kUnitsSize) return ""; + + return base10Units[index]; +} + +std::string Base10HumanReadableFormat(const int64_t& arg) { + std::string mantissa; + int64_t exponent; + ToExponentAndMantissa(arg, 1, 1, 1000, &mantissa, &exponent, true); + return mantissa + ExponentToBase10Prefix(exponent); +} + +bool IsPowerOfTwo(const int64_t& val) { + return (val & (val - 1)) == 0 && (val > 1); +} + +std::string Base2HumanReadableFormat(const int64_t& arg) { + return StrFormat("2^%.0f", std::log2(arg)); +} + +std::string FormatHumanReadable(const int64_t& arg) { + if (IsPowerOfTwo(arg)) { + return Base2HumanReadableFormat(arg); + } + return Base10HumanReadableFormat(arg); +} + } // end namespace benchmark diff --git a/src/string_util.h b/src/string_util.h index 37bdd2e980..c03bdb5345 100644 --- a/src/string_util.h +++ b/src/string_util.h @@ -65,6 +65,35 @@ using std::stoul; // NOLINT(misc-unused-using-decls) #endif // NOLINTEND +/** + * Check whether the given value is a power of two or not. + * @param val the value to check + */ +bool IsPowerOfTwo(const int64_t& val); + +/** + * Gets the human readable format for a given base10 value. + * In other words converts 1_000 to 1k, 40_000_000 to 40m etc + * @param arg the positive value to convert + * @return human readable formatted string + */ +std::string Base10HumanReadableFormat(const int64_t& arg); + +/** + * Gets the human readable format for a given base2 value. + * In other words converts 64 to 2^6, 1024 to 2^10 etc + * @param arg the positive value to convert + * @return human readable formatted string + */ +std::string Base2HumanReadableFormat(const int64_t& arg); + +/** + * Formats an argument into a human readable format. + * @param arg the argument to format + * @return the argument formatted as human readable + */ +std::string FormatHumanReadable(const int64_t& arg); + } // end namespace benchmark #endif // BENCHMARK_STRING_UTIL_H_ diff --git a/test/BUILD b/test/BUILD index 8262d080fa..4931571a34 100644 --- a/test/BUILD +++ b/test/BUILD @@ -32,6 +32,7 @@ PER_SRC_TEST_ARGS = { "repetitions_test.cc": [" --benchmark_repetitions=3"], "spec_arg_test.cc": ["--benchmark_filter=BM_NotChosen"], "spec_arg_verbosity_test.cc": ["--v=42"], + "human_readable_formatting_test.cc": ["--benchmark_human_readable"] } cc_library( diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 78d6d51750..37a3e74077 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -177,6 +177,12 @@ add_test(NAME user_counters_thousands_test COMMAND user_counters_thousands_test compile_output_test(memory_manager_test) add_test(NAME memory_manager_test COMMAND memory_manager_test --benchmark_min_time=0.01s) +compile_benchmark_test(human_readable_formatting_test) +add_test(NAME human_readable_formatting_test COMMAND human_readable_formatting_test --benchmark_min_time=0.01s --benchmark_human_readable) + +compile_benchmark_test(non_human_readable_formatting_test) +add_test(NAME non_human_readable_formatting_test COMMAND non_human_readable_formatting_test --benchmark_min_time=0.01s --benchmark_human_readable=false) + # MSVC does not allow to set the language standard to C++98/03. if(NOT CMAKE_CXX_COMPILER_ID STREQUAL "MSVC") compile_benchmark_test(cxx03_test) @@ -237,6 +243,7 @@ if (BENCHMARK_ENABLE_GTEST_TESTS) add_gtest(perf_counters_gtest) add_gtest(time_unit_gtest) add_gtest(min_time_parse_gtest) + add_gtest(human_readable_gtest) endif(BENCHMARK_ENABLE_GTEST_TESTS) ############################################################################### diff --git a/test/human_readable_formatting_test.cc b/test/human_readable_formatting_test.cc new file mode 100644 index 0000000000..455637bc1b --- /dev/null +++ b/test/human_readable_formatting_test.cc @@ -0,0 +1,122 @@ +#undef NDEBUG + +#include +#include + +#include "../src/check.h" // NOTE: check.h is for internal use only! +#include "benchmark/benchmark.h" + +namespace { + +class TestReporter : public benchmark::ConsoleReporter { + public: + bool ReportContext(const Context& context) override { + return ConsoleReporter::ReportContext(context); + }; + + void ReportRuns(const std::vector& report) override { + all_runs_.insert(all_runs_.end(), begin(report), end(report)); + ConsoleReporter::ReportRuns(report); + } + + TestReporter() {} + ~TestReporter() override {} + + mutable std::vector all_runs_; +}; + +struct TestCase { + std::string name; + + typedef benchmark::BenchmarkReporter::Run Run; + + void CheckRun(Run const& run) const { + BM_CHECK(name == run.benchmark_name()) + << "expected " << name << " got " << run.benchmark_name(); + } +}; + +std::vector ExpectedResults; + +int AddCases(const std::string& base_name, + std::initializer_list const& v) { + for (auto TC : v) { + TC.name = base_name + TC.name; + ExpectedResults.push_back(std::move(TC)); + } + return 0; +} + +#define CONCAT(x, y) CONCAT2(x, y) +#define CONCAT2(x, y) x##y +#define ADD_CASES(...) int CONCAT(dummy, __LINE__) = AddCases(__VA_ARGS__) + +} // end namespace + +// ============== test case we non base 2 args ============== // +void BM_non_base_two_args(benchmark::State& state) { + for (auto _ : state) { + } +} +BENCHMARK(BM_non_base_two_args)->Arg(9)->Arg(19)->Arg(24)->Arg(65); +ADD_CASES("BM_non_base_two_args", {{"/9"}, {"/19"}, {"/24"}, {"/65"}}); + +// ============== test case we base 2 args ============== // +void BM_base_two_args(benchmark::State& state) { + for (auto _ : state) { + } +} +BENCHMARK(BM_base_two_args)->RangeMultiplier(2)->Range(1, 2 << 7 - 1); +ADD_CASES("BM_base_two_args", {{"/1"}, + {"/2^1"}, + {"/2^2"}, + {"/2^3"}, + {"/2^4"}, + {"/2^5"}, + {"/2^6"}, + {"/2^7"}}); + +// ============== test case we base 10 args ============== // +void BM_base_ten_args(benchmark::State& state) { + for (auto _ : state) { + } +} +BENCHMARK(BM_base_ten_args) + ->Arg(1) + ->Arg(10) + ->Arg(100) + ->Arg(1000) + ->Arg(10000) + ->Arg(32000) + ->Arg(100000) + ->Arg(1000000) + ->Arg(1000000000); +ADD_CASES("BM_base_ten_args", {{"/1"}, + {"/10"}, + {"/100"}, + {"/1k"}, + {"/10k"}, + {"/32k"}, + {"/100k"}, + {"/1M"}, + {"/1B"}}); + +int main(int argc, char* argv[]) { + benchmark::Initialize(&argc, argv); + + TestReporter test_reporter; + benchmark::RunSpecifiedBenchmarks(&test_reporter); + + typedef benchmark::BenchmarkReporter::Run Run; + + auto EB = ExpectedResults.begin(); + + for (Run const& run : test_reporter.all_runs_) { + assert(EB != ExpectedResults.end()); + EB->CheckRun(run); + ++EB; + } + assert(EB == ExpectedResults.end()); + + return 0; +} diff --git a/test/human_readable_gtest.cc b/test/human_readable_gtest.cc new file mode 100644 index 0000000000..0866c3a7f1 --- /dev/null +++ b/test/human_readable_gtest.cc @@ -0,0 +1,70 @@ +//===---------------------------------------------------------------------===// +// human_readable_test - Unit tests for human readable converters +//===---------------------------------------------------------------------===// + +#include "../src/string_util.h" +#include "gtest/gtest.h" + +namespace { + +TEST(HumanReadableTest, base2) { + for (int i = 2; i < 10; ++i) { + const auto res = benchmark::Base2HumanReadableFormat(1 << i); + + const std::string expected = "2^" + std::to_string(i); + EXPECT_STREQ(res.c_str(), expected.c_str()); + } +} + +TEST(HumanReadableTest, base10) { + { + const auto res = benchmark::Base10HumanReadableFormat(100); + EXPECT_STREQ(res.c_str(), "100"); + } + { + const auto res = benchmark::Base10HumanReadableFormat(1000); + EXPECT_STREQ(res.c_str(), "1k"); + } + { + const auto res = benchmark::Base10HumanReadableFormat(10000); + EXPECT_STREQ(res.c_str(), "10k"); + } + { + const auto res = benchmark::Base10HumanReadableFormat(20000); + EXPECT_STREQ(res.c_str(), "20k"); + } + { + const auto res = benchmark::Base10HumanReadableFormat(32000); + EXPECT_STREQ(res.c_str(), "32k"); + } + { + const auto res = benchmark::Base10HumanReadableFormat(1000000); + EXPECT_STREQ(res.c_str(), "1M"); + } + { + const auto res = benchmark::Base10HumanReadableFormat(42000000); + EXPECT_STREQ(res.c_str(), "42M"); + } + { + const auto res = benchmark::Base10HumanReadableFormat(1000000000); + EXPECT_STREQ(res.c_str(), "1B"); + } + { + const auto res = benchmark::Base10HumanReadableFormat(4000000000); + EXPECT_STREQ(res.c_str(), "4B"); + } + { + const auto res = benchmark::Base10HumanReadableFormat(4200000000); + EXPECT_STREQ(res.c_str(), "4.2B"); + } + { + const auto res = benchmark::Base10HumanReadableFormat(40200000); + EXPECT_STREQ(res.c_str(), "40.2M"); + } + { + const auto res = benchmark::Base10HumanReadableFormat(4200000000000000000); + EXPECT_STREQ(res.c_str(), "4.2Qi"); + } +} + +} // end namespace diff --git a/test/non_human_readable_formatting_test.cc b/test/non_human_readable_formatting_test.cc new file mode 100644 index 0000000000..e8723ad2a0 --- /dev/null +++ b/test/non_human_readable_formatting_test.cc @@ -0,0 +1,113 @@ +#undef NDEBUG + +#include +#include + +#include "../src/check.h" // NOTE: check.h is for internal use only! +#include "benchmark/benchmark.h" + +namespace { + +class TestReporter : public benchmark::ConsoleReporter { + public: + bool ReportContext(const Context& context) override { + return ConsoleReporter::ReportContext(context); + }; + + void ReportRuns(const std::vector& report) override { + all_runs_.insert(all_runs_.end(), begin(report), end(report)); + ConsoleReporter::ReportRuns(report); + } + + TestReporter() {} + ~TestReporter() override {} + + mutable std::vector all_runs_; +}; + +struct TestCase { + std::string name; + + typedef benchmark::BenchmarkReporter::Run Run; + + void CheckRun(Run const& run) const { + BM_CHECK(name == run.benchmark_name()) + << "expected " << name << " got " << run.benchmark_name(); + } +}; + +std::vector ExpectedResults; + +int AddCases(const std::string& base_name, + std::initializer_list const& v) { + for (auto TC : v) { + TC.name = base_name + TC.name; + ExpectedResults.push_back(std::move(TC)); + } + return 0; +} + +#define CONCAT(x, y) CONCAT2(x, y) +#define CONCAT2(x, y) x##y +#define ADD_CASES(...) int CONCAT(dummy, __LINE__) = AddCases(__VA_ARGS__) + +} // end namespace + +// ============== test case we non base 2 args ============== // +void BM_non_base_two_args(benchmark::State& state) { + for (auto _ : state) { + } +} +BENCHMARK(BM_non_base_two_args)->Arg(9)->Arg(19)->Arg(24)->Arg(1023); +ADD_CASES("BM_non_base_two_args", {{"/9"}, {"/19"}, {"/24"}, {"/1023"}}); + +// ============== test case we base 2 args ============== // +void BM_base_two_args(benchmark::State& state) { + for (auto _ : state) { + } +} +BENCHMARK(BM_base_two_args)->RangeMultiplier(2)->Range(1, 2 << 7 - 1); +ADD_CASES("BM_base_two_args", {{"/1"}, + {"/2"}, + {"/4"}, + {"/8"}, + {"/16"}, + {"/32"}, + {"/64"}, + {"/128"}}); + +// ============== test case we base 10 args ============== // +void BM_base_ten_args(benchmark::State& state) { + for (auto _ : state) { + } +} +BENCHMARK(BM_base_ten_args)->Arg(1)->Arg(10)->Arg(100)->Arg(1000)->Arg(10000)->Arg(32000)->Arg(100000)->Arg(1000000)->Arg(1000000000); +ADD_CASES("BM_base_ten_args", {{"/1"}, + {"/10"}, + {"/100"}, + {"/1000"}, + {"/10000"}, + {"/32000"}, + {"/100000"}, + {"/1000000"}, + {"/1000000000"}}); + +int main(int argc, char* argv[]) { + benchmark::Initialize(&argc, argv); + + TestReporter test_reporter; + benchmark::RunSpecifiedBenchmarks(&test_reporter); + + typedef benchmark::BenchmarkReporter::Run Run; + + auto EB = ExpectedResults.begin(); + + for (Run const& run : test_reporter.all_runs_) { + assert(EB != ExpectedResults.end()); + EB->CheckRun(run); + ++EB; + } + assert(EB == ExpectedResults.end()); + + return 0; +}