diff options
| author | android-build-team Robot <android-build-team-robot@google.com> | 2020-04-28 20:24:45 +0000 |
|---|---|---|
| committer | android-build-team Robot <android-build-team-robot@google.com> | 2020-04-28 20:24:45 +0000 |
| commit | b916ff7fef5985b373972c35b6b970edfcec38e5 (patch) | |
| tree | e223e4f0faa8917ed3bb4aa33382949455ef3f17 | |
| parent | 8d86176442fcc5b121db9f505c58b44683d62191 (diff) | |
| parent | 20f5b67313dbf0ddafb792a5a2cc298b9cc58063 (diff) | |
| download | google-benchmark-android10-android13-mainline-tzdata-release.tar.gz | |
Snap for 6439596 from 20f5b67313dbf0ddafb792a5a2cc298b9cc58063 to qt-aml-tzdata-releaseq_tzdata_aml_297100400q_tzdata_aml_297100300q_tzdata_aml_297100000q_tzdata_aml_296200000q_tzdata_aml_295600118q_tzdata_aml_295600110q_tzdata_aml_295500002q_tzdata_aml_295500001q_tzdata_aml_294400310android-mainline-12.0.0_r54android-mainline-12.0.0_r111android-mainline-10.0.0_r13android-mainline-10.0.0_r12android-mainline-10.0.0_r11q_tzdata_aml_297100000android12-mainline-tzdata-releaseandroid10-mainline-tzdata-releaseandroid10-android13-mainline-tzdata-release
Change-Id: Icfa12102074483c104af8f55acad903495be6e3f
66 files changed, 1011 insertions, 2595 deletions
diff --git a/.clang-format b/.clang-format index e7d00fe..4b3f13f 100644 --- a/.clang-format +++ b/.clang-format @@ -1,5 +1,5 @@ --- Language: Cpp BasedOnStyle: Google -PointerAlignment: Left ... + @@ -8,7 +8,6 @@ !/cmake/*.cmake !/test/AssemblyTests.cmake *~ -*.swp *.pyc __pycache__ @@ -57,6 +56,3 @@ build*/ # Visual Studio 2015/2017 cache/options directory .vs/ CMakeSettings.json - -# Visual Studio Code cache/options directory -.vscode/ diff --git a/.travis.yml b/.travis.yml index 6b6cfc7..4625dfb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -23,25 +23,13 @@ matrix: apt: packages: - g++-multilib - - libc6:i386 - env: - - COMPILER=g++ - - C_COMPILER=gcc - - BUILD_TYPE=Debug - - BUILD_32_BITS=ON - - EXTRA_FLAGS="-m32" + env: COMPILER=g++ C_COMPILER=gcc BUILD_TYPE=Debug BUILD_32_BITS=ON - compiler: gcc addons: apt: packages: - g++-multilib - - libc6:i386 - env: - - COMPILER=g++ - - C_COMPILER=gcc - - BUILD_TYPE=Release - - BUILD_32_BITS=ON - - EXTRA_FLAGS="-m32" + env: COMPILER=g++ C_COMPILER=gcc BUILD_TYPE=Release BUILD_32_BITS=ON - compiler: gcc env: - INSTALL_GCC6_FROM_PPA=1 @@ -54,102 +42,81 @@ matrix: env: COMPILER=clang++ C_COMPILER=clang BUILD_TYPE=Release # Clang w/ libc++ - compiler: clang - dist: xenial addons: apt: packages: clang-3.8 env: - - INSTALL_GCC6_FROM_PPA=1 - COMPILER=clang++-3.8 C_COMPILER=clang-3.8 BUILD_TYPE=Debug - LIBCXX_BUILD=1 - - EXTRA_CXX_FLAGS="-stdlib=libc++" + - EXTRA_FLAGS="-stdlib=libc++" - compiler: clang - dist: xenial addons: apt: packages: clang-3.8 env: - - INSTALL_GCC6_FROM_PPA=1 - COMPILER=clang++-3.8 C_COMPILER=clang-3.8 BUILD_TYPE=Release - LIBCXX_BUILD=1 - - EXTRA_CXX_FLAGS="-stdlib=libc++" + - EXTRA_FLAGS="-stdlib=libc++" # Clang w/ 32bit libc++ - compiler: clang - dist: xenial addons: apt: packages: - clang-3.8 - g++-multilib - - libc6:i386 env: - - INSTALL_GCC6_FROM_PPA=1 - COMPILER=clang++-3.8 C_COMPILER=clang-3.8 BUILD_TYPE=Debug - LIBCXX_BUILD=1 - BUILD_32_BITS=ON - - EXTRA_FLAGS="-m32" - - EXTRA_CXX_FLAGS="-stdlib=libc++" + - EXTRA_FLAGS="-stdlib=libc++ -m32" # Clang w/ 32bit libc++ - compiler: clang - dist: xenial addons: apt: packages: - clang-3.8 - g++-multilib - - libc6:i386 env: - - INSTALL_GCC6_FROM_PPA=1 - COMPILER=clang++-3.8 C_COMPILER=clang-3.8 BUILD_TYPE=Release - LIBCXX_BUILD=1 - BUILD_32_BITS=ON - - EXTRA_FLAGS="-m32" - - EXTRA_CXX_FLAGS="-stdlib=libc++" + - EXTRA_FLAGS="-stdlib=libc++ -m32" # Clang w/ libc++, ASAN, UBSAN - compiler: clang - dist: xenial addons: apt: packages: clang-3.8 env: - - INSTALL_GCC6_FROM_PPA=1 - COMPILER=clang++-3.8 C_COMPILER=clang-3.8 BUILD_TYPE=Debug - LIBCXX_BUILD=1 LIBCXX_SANITIZER="Undefined;Address" - ENABLE_SANITIZER=1 - - EXTRA_FLAGS="-g -O2 -fno-omit-frame-pointer -fsanitize=undefined,address -fno-sanitize-recover=all" - - EXTRA_CXX_FLAGS="-stdlib=libc++" + - EXTRA_FLAGS="-stdlib=libc++ -g -O2 -fno-omit-frame-pointer -fsanitize=undefined,address -fno-sanitize-recover=all" - UBSAN_OPTIONS=print_stacktrace=1 # Clang w/ libc++ and MSAN - compiler: clang - dist: xenial addons: apt: packages: clang-3.8 env: - - INSTALL_GCC6_FROM_PPA=1 - COMPILER=clang++-3.8 C_COMPILER=clang-3.8 BUILD_TYPE=Debug - LIBCXX_BUILD=1 LIBCXX_SANITIZER=MemoryWithOrigins - ENABLE_SANITIZER=1 - - EXTRA_FLAGS="-g -O2 -fno-omit-frame-pointer -fsanitize=memory -fsanitize-memory-track-origins" - - EXTRA_CXX_FLAGS="-stdlib=libc++" + - EXTRA_FLAGS="-stdlib=libc++ -g -O2 -fno-omit-frame-pointer -fsanitize=memory -fsanitize-memory-track-origins" # Clang w/ libc++ and MSAN - compiler: clang - dist: xenial addons: apt: packages: clang-3.8 env: - - INSTALL_GCC6_FROM_PPA=1 - COMPILER=clang++-3.8 C_COMPILER=clang-3.8 BUILD_TYPE=RelWithDebInfo - LIBCXX_BUILD=1 LIBCXX_SANITIZER=Thread - ENABLE_SANITIZER=1 - - EXTRA_FLAGS="-g -O2 -fno-omit-frame-pointer -fsanitize=thread -fno-sanitize-recover=all" - - EXTRA_CXX_FLAGS="-stdlib=libc++" + - EXTRA_FLAGS="-stdlib=libc++ -g -O2 -fno-omit-frame-pointer -fsanitize=thread -fno-sanitize-recover=all" - os: osx osx_image: xcode8.3 compiler: clang @@ -164,10 +131,7 @@ matrix: osx_image: xcode8.3 compiler: clang env: - - COMPILER=clang++ - - BUILD_TYPE=Release - - BUILD_32_BITS=ON - - EXTRA_FLAGS="-m32" + - COMPILER=clang++ BUILD_TYPE=Release BUILD_32_BITS=ON - os: osx osx_image: xcode8.3 compiler: gcc @@ -214,7 +178,7 @@ install: fi - if [ "${TRAVIS_OS_NAME}" == "linux" ]; then sudo apt-get update -qq; - sudo apt-get install -qq unzip cmake3; + sudo apt-get install -qq unzip; wget https://github.com/bazelbuild/bazel/releases/download/0.10.1/bazel-0.10.1-installer-linux-x86_64.sh --output-document bazel-installer.sh; travis_wait sudo bash bazel-installer.sh; fi @@ -224,7 +188,7 @@ install: fi script: - - cmake -DCMAKE_C_COMPILER=${C_COMPILER} -DCMAKE_CXX_COMPILER=${COMPILER} -DCMAKE_BUILD_TYPE=${BUILD_TYPE} -DCMAKE_C_FLAGS="${EXTRA_FLAGS}" -DCMAKE_CXX_FLAGS="${EXTRA_FLAGS} ${EXTRA_CXX_FLAGS}" -DBENCHMARK_DOWNLOAD_DEPENDENCIES=ON -DBENCHMARK_BUILD_32_BITS=${BUILD_32_BITS} ${EXTRA_OPTIONS} .. + - cmake -DCMAKE_C_COMPILER=${C_COMPILER} -DCMAKE_CXX_COMPILER=${COMPILER} -DCMAKE_BUILD_TYPE=${BUILD_TYPE} -DCMAKE_CXX_FLAGS="${EXTRA_FLAGS}" -DBENCHMARK_DOWNLOAD_DEPENDENCIES=ON -DBENCHMARK_BUILD_32_BITS=${BUILD_32_BITS} ${EXTRA_OPTIONS} .. - make - ctest -C ${BUILD_TYPE} --output-on-failure - bazel test -c dbg --define google_benchmark.have_regex=posix --announce_rc --verbose_failures --test_output=errors --keep_going //test/... @@ -9,18 +9,14 @@ # Please keep the list sorted. Albert Pretorius <pretoalb@gmail.com> -Alex Steele <steeleal123@gmail.com> Andriy Berestovskyy <berestovskyy@gmail.com> Arne Beer <arne@twobeer.de> Carto Christopher Seymour <chris.j.seymour@hotmail.com> -Colin Braley <braley.colin@gmail.com> -Daniel Harvey <danielharvey458@gmail.com> David Coeurjolly <david.coeurjolly@liris.cnrs.fr> Deniz Evrenci <denizevrenci@gmail.com> Dirac Research Dominik Czarnota <dominik.b.czarnota@gmail.com> -Eric Backus <eric_backus@alum.mit.edu> Eric Fiselier <eric@efcs.ca> Eugene Zhuk <eugene.zhuk@gmail.com> Evgeny Safronov <division494@gmail.com> @@ -45,7 +41,6 @@ Ori Livneh <ori.livneh@gmail.com> Paul Redmond <paul.redmond@gmail.com> Radoslav Yovchev <radoslav.tm@gmail.com> Roman Lebedev <lebedev.ri@gmail.com> -Sayan Bhattacharjee <aero.sayan@gmail.com> Shuo Chen <chenshuo@chenshuo.com> Steinar H. Gunderson <sgunderson@bigfoot.com> Stripe, Inc. @@ -18,6 +18,7 @@ cc_library_static { name: "libgoogle-benchmark", host_supported: true, local_include_dirs: ["include"], + vendor_available: true, cflags: [ "-DBENCHMARK_ANDROID", "-DHAVE_POSIX_REGEX", @@ -25,11 +26,24 @@ cc_library_static { "-Wno-deprecated-declarations", ], - exclude_srcs: [ - "src/benchmark_main.cc", - ], srcs: [ - "src/*.cc", + "src/benchmark_api_internal.cc", + "src/benchmark.cc", + "src/benchmark_register.cc", + "src/benchmark_runner.cc", + "src/colorprint.cc", + "src/commandlineflags.cc", + "src/complexity.cc", + "src/console_reporter.cc", + "src/counter.cc", + "src/csv_reporter.cc", + "src/json_reporter.cc", + "src/reporter.cc", + "src/sleep.cc", + "src/statistics.cc", + "src/string_util.cc", + "src/sysinfo.cc", + "src/timers.cc", ], export_include_dirs: ["include"], } diff --git a/BUILD.bazel b/BUILD.bazel index d97a019..6ee69f2 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -8,8 +8,6 @@ config_setting( visibility = [":__subpackages__"], ) -load("@rules_cc//cc:defs.bzl", "cc_library") - cc_library( name = "benchmark", srcs = glob( diff --git a/CMakeLists.txt b/CMakeLists.txt index 8cfe125..d7ed57e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,18 +1,17 @@ -cmake_minimum_required (VERSION 3.5.1) +cmake_minimum_required (VERSION 2.8.12) foreach(p CMP0048 # OK to clear PROJECT_VERSION on project() CMP0054 # CMake 3.1 CMP0056 # export EXE_LINKER_FLAGS to try_run CMP0057 # Support no if() IN_LIST operator - CMP0063 # Honor visibility properties for all targets ) if(POLICY ${p}) cmake_policy(SET ${p} NEW) endif() endforeach() -project (benchmark CXX) +project (benchmark) option(BENCHMARK_ENABLE_TESTING "Enable testing of the benchmark library." ON) option(BENCHMARK_ENABLE_EXCEPTIONS "Enable the use of exceptions in the benchmark library." ON) @@ -33,7 +32,6 @@ option(BENCHMARK_DOWNLOAD_DEPENDENCIES "Allow the downloading and in-tree buildi # in cases where it is not possible to build or find a valid version of gtest. option(BENCHMARK_ENABLE_GTEST_TESTS "Enable building the unit tests which depend on gtest" ON) -set(CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS ON) set(ENABLE_ASSEMBLY_TESTS_DEFAULT OFF) function(should_enable_assembly_tests) if(CMAKE_BUILD_TYPE) @@ -192,7 +190,7 @@ else() # Link time optimisation if (BENCHMARK_ENABLE_LTO) add_cxx_compiler_flag(-flto) - if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU") + if ("${CMAKE_C_COMPILER_ID}" STREQUAL "GNU") find_program(GCC_AR gcc-ar) if (GCC_AR) set(CMAKE_AR ${GCC_AR}) @@ -201,7 +199,7 @@ else() if (GCC_RANLIB) set(CMAKE_RANLIB ${GCC_RANLIB}) endif() - elseif("${CMAKE_CXX_COMPILER_ID}" MATCHES "Clang") + elseif("${CMAKE_C_COMPILER_ID}" MATCHES "Clang") include(llvm-toolchain) endif() endif() @@ -268,10 +266,8 @@ add_subdirectory(src) if (BENCHMARK_ENABLE_TESTING) enable_testing() - if (BENCHMARK_ENABLE_GTEST_TESTS AND - NOT (TARGET gtest AND TARGET gtest_main AND - TARGET gmock AND TARGET gmock_main)) - include(GoogleTest) + if (BENCHMARK_ENABLE_GTEST_TESTS) + include(HandleGTest) endif() add_subdirectory(test) endif() diff --git a/CONTRIBUTORS b/CONTRIBUTORS index 6b64a00..d0c31df 100644 --- a/CONTRIBUTORS +++ b/CONTRIBUTORS @@ -23,27 +23,21 @@ # Please keep the list sorted. Albert Pretorius <pretoalb@gmail.com> -Alex Steele <steelal123@gmail.com> Andriy Berestovskyy <berestovskyy@gmail.com> Arne Beer <arne@twobeer.de> Billy Robert O'Neal III <billy.oneal@gmail.com> <bion@microsoft.com> Chris Kennelly <ckennelly@google.com> <ckennelly@ckennelly.com> Christopher Seymour <chris.j.seymour@hotmail.com> -Colin Braley <braley.colin@gmail.com> Cyrille Faucheux <cyrille.faucheux@gmail.com> -Daniel Harvey <danielharvey458@gmail.com> David Coeurjolly <david.coeurjolly@liris.cnrs.fr> Deniz Evrenci <denizevrenci@gmail.com> Dominic Hamon <dma@stripysock.com> <dominic@google.com> Dominik Czarnota <dominik.b.czarnota@gmail.com> -Eric Backus <eric_backus@alum.mit.edu> Eric Fiselier <eric@efcs.ca> Eugene Zhuk <eugene.zhuk@gmail.com> Evgeny Safronov <division494@gmail.com> Federico Ficarelli <federico.ficarelli@gmail.com> Felix Homann <linuxaudio@showlabor.de> -Geoffrey Martin-Noble <gcmn@google.com> <gmngeoffrey@gmail.com> -Hannes Hauswedell <h2@fsfe.org> Ismael Jimenez Martinez <ismael.jimenez.martinez@gmail.com> Jern-Kuan Leong <jernkuan@gmail.com> JianXiong Zhou <zhoujianxiong2@gmail.com> @@ -51,8 +45,8 @@ Joao Paulo Magalhaes <joaoppmagalhaes@gmail.com> John Millikin <jmillikin@stripe.com> Jussi Knuuttila <jussi.knuuttila@gmail.com> Kai Wolf <kai.wolf@gmail.com> -Kaito Udagawa <umireon@gmail.com> Kishan Kumar <kumar.kishan@outlook.com> +Kaito Udagawa <umireon@gmail.com> Lei Xu <eddyxu@gmail.com> Matt Clarkson <mattyclarkson@gmail.com> Maxim Vafin <maxvafin@gmail.com> @@ -67,7 +61,6 @@ Raul Marin <rmrodriguez@cartodb.com> Ray Glover <ray.glover@uk.ibm.com> Robert Guo <robert.guo@mongodb.com> Roman Lebedev <lebedev.ri@gmail.com> -Sayan Bhattacharjee <aero.sayan@gmail.com> Shuo Chen <chenshuo@chenshuo.com> Tobias Ulvgård <tobias.ulvgard@dirac.se> Tom Madams <tom.ej.madams@gmail.com> <tmadams@google.com> @@ -9,10 +9,10 @@ third_party { type: GIT value: "https://github.com/google/benchmark.git" } - version: "c50ac68c50ff8da3827cd6720792117910d85666" + version: "d205ead299c7cddd5e1bc3478d57ad4320a4a53c" last_upgrade_date { year: 2019 - month: 11 - day: 6 + month: 3 + day: 4 } } @@ -1,36 +1,10 @@ -# Benchmark +# benchmark [](https://travis-ci.org/google/benchmark) [](https://ci.appveyor.com/project/google/benchmark/branch/master) [](https://coveralls.io/r/google/benchmark) [](https://slackin-iqtfqnpzxd.now.sh/) - -A library to benchmark code snippets, similar to unit tests. Example: - -```c++ -#include <benchmark/benchmark.h> - -static void BM_SomeFunction(benchmark::State& state) { - // Perform setup here - for (auto _ : state) { - // This code gets timed - SomeFunction(); - } -} -// Register the function as a benchmark -BENCHMARK(BM_SomeFunction); -// Run the benchmark -BENCHMARK_MAIN(); -``` - -To get started, see [Requirements](#requirements) and -[Installation](#installation). See [Usage](#usage) for a full example and the -[User Guide](#user-guide) for a more comprehensive feature overview. - -It may also help to read the [Google Test documentation](https://github.com/google/googletest/blob/master/googletest/docs/primer.md) -as some of the structural aspects of the APIs are similar. - -### Resources +A library to support the benchmarking of functions, similar to unit-tests. [Discussion group](https://groups.google.com/d/forum/benchmark-discuss) @@ -40,68 +14,20 @@ IRC channel: [freenode](https://freenode.net) #googlebenchmark [Assembly Testing Documentation](docs/AssemblyTests.md) -## Requirements - -The library can be used with C++03. However, it requires C++11 to build, -including compiler and standard library support. - -The following minimum versions are required to build the library: - -* GCC 4.8 -* Clang 3.4 -* Visual Studio 14 2015 -* Intel 2015 Update 1 - -See [Platform-Specific Build Instructions](#platform-specific-build-instructions). - -## Installation -This describes the installation process using cmake. As pre-requisites, you'll -need git and cmake installed. +## Building -_See [dependencies.md](dependencies.md) for more details regarding supported -versions of build tools._ +The basic steps for configuring and building the library look like this: ```bash -# Check out the library. $ git clone https://github.com/google/benchmark.git # Benchmark requires Google Test as a dependency. Add the source tree as a subdirectory. $ git clone https://github.com/google/googletest.git benchmark/googletest -# Go to the library root directory -$ cd benchmark -# Make a build directory to place the build output. $ mkdir build && cd build -# Generate a Makefile with cmake. -# Use cmake -G <generator> to generate a different file type. -$ cmake ../ -# Build the library. -# Use make -j<number_of_parallel_jobs> to speed up the build process, e.g. make -j8 . +$ cmake -G <generator> [options] ../benchmark +# Assuming a makefile generator was used $ make ``` -This builds the `benchmark` and `benchmark_main` libraries and tests. -On a unix system, the build directory should now look something like this: - -``` -/benchmark - /build - /src - /libbenchmark.a - /libbenchmark_main.a - /test - ... -``` - -Next, you can run the tests to check the build. - -```bash -$ make test -``` - -If you want to install the library globally, also run: - -``` -sudo make install -``` Note that Google Benchmark requires Google Test to build and run the tests. This dependency can be provided two ways: @@ -114,29 +40,37 @@ dependency can be provided two ways: If you do not wish to build and run the tests, add `-DBENCHMARK_ENABLE_GTEST_TESTS=OFF` to `CMAKE_ARGS`. -### Debug vs Release -By default, benchmark builds as a debug library. You will see a warning in the -output when this is the case. To build it as a release library instead, use: +## Installation Guide + +For Ubuntu and Debian Based System + +First make sure you have git and cmake installed (If not please install them) ``` -cmake -DCMAKE_BUILD_TYPE=Release +sudo apt-get install git cmake ``` -To enable link-time optimisation, use +Now, let's clone the repository and build it ``` -cmake -DCMAKE_BUILD_TYPE=Release -DBENCHMARK_ENABLE_LTO=true +git clone https://github.com/google/benchmark.git +cd benchmark +# If you want to build tests and don't use BENCHMARK_DOWNLOAD_DEPENDENCIES, then +# git clone https://github.com/google/googletest.git +mkdir build +cd build +cmake .. -DCMAKE_BUILD_TYPE=RELEASE +make ``` -If you are using gcc, you might need to set `GCC_AR` and `GCC_RANLIB` cmake -cache variables, if autodetection fails. - -If you are using clang, you may need to set `LLVMAR_EXECUTABLE`, -`LLVMNM_EXECUTABLE` and `LLVMRANLIB_EXECUTABLE` cmake cache variables. +If you need to install the library globally +``` +sudo make install +``` -### Stable and Experimental Library Versions +## Stable and Experimental Library Versions The main branch contains the latest stable version of the benchmarking library; the API of which can be considered largely stable, with source breaking changes @@ -148,11 +82,16 @@ to use, test, and provide feedback on the new features are encouraged to try this branch. However, this branch provides no stability guarantees and reserves the right to change and break the API at any time. -## Usage +## Further knowledge + +It may help to read the [Google Test documentation](https://github.com/google/googletest/blob/master/googletest/docs/primer.md) +as some of the structural aspects of the APIs are similar. + +## Example usage ### Basic usage -Define a function that executes the code to measure, register it as a benchmark -function using the `BENCHMARK` macro, and ensure an appropriate `main` function -is available: +Define a function that executes the code to be measured, register it as a +benchmark function using the `BENCHMARK` macro, and ensure an appropriate `main` +function is available: ```c++ #include <benchmark/benchmark.h> @@ -175,27 +114,15 @@ BENCHMARK(BM_StringCopy); BENCHMARK_MAIN(); ``` -To run the benchmark, compile and link against the `benchmark` library -(libbenchmark.a/.so). If you followed the build steps above, this -library will be under the build directory you created. +Don't forget to inform your linker to add benchmark library e.g. through +`-lbenchmark` compilation flag. Alternatively, you may leave out the +`BENCHMARK_MAIN();` at the end of the source file and link against +`-lbenchmark_main` to get the same default behavior. -```bash -# Example on linux after running the build steps above. Assumes the -# `benchmark` and `build` directories are under the current directory. -$ g++ mybenchmark.cc -std=c++11 -isystem benchmark/include \ - -Lbenchmark/build/src -lbenchmark -lpthread -o mybenchmark -``` - -Alternatively, link against the `benchmark_main` library and remove -`BENCHMARK_MAIN();` above to get the same behavior. - -The compiled executable will run all benchmarks by default. Pass the `--help` -flag for option information or see the guide below. - -## Platform Specific Build Instructions - -### Building with GCC +The benchmark library will measure and report the timing for code within the +`for(...)` loop. +#### Platform-specific libraries When the library is built using GCC it is necessary to link with the pthread library due to how GCC implements `std::thread`. Failing to link to pthread will lead to runtime exceptions (unless you're using libc++), not linker errors. See @@ -204,217 +131,13 @@ can link to pthread by adding `-pthread` to your linker command. Note, you can also use `-lpthread`, but there are potential issues with ordering of command line parameters if you use that. -### Building with Visual Studio 2015 or 2017 - -The `shlwapi` library (`-lshlwapi`) is required to support a call to `CPUInfo` which reads the registry. Either add `shlwapi.lib` under `[ Configuration Properties > Linker > Input ]`, or use the following: - -``` -// Alternatively, can add libraries using linker options. -#ifdef _WIN32 -#pragma comment ( lib, "Shlwapi.lib" ) -#ifdef _DEBUG -#pragma comment ( lib, "benchmarkd.lib" ) -#else -#pragma comment ( lib, "benchmark.lib" ) -#endif -#endif -``` - -Can also use the graphical version of CMake: -* Open `CMake GUI`. -* Under `Where to build the binaries`, same path as source plus `build`. -* Under `CMAKE_INSTALL_PREFIX`, same path as source plus `install`. -* Click `Configure`, `Generate`, `Open Project`. -* If build fails, try deleting entire directory and starting again, or unticking options to build less. - -### Building with Intel 2015 Update 1 or Intel System Studio Update 4 - -See instructions for building with Visual Studio. Once built, right click on the solution and change the build to Intel. - -### Building on Solaris +If you're running benchmarks on Windows, the shlwapi library (`-lshlwapi`) is +also required. If you're running benchmarks on solaris, you'll want the kstat library linked in too (`-lkstat`). -## User Guide - -### Command Line -[Output Formats](#output-formats) - -[Output Files](#output-files) - -[Running a Subset of Benchmarks](#running-a-subset-of-benchmarks) - -[Result Comparison](#result-comparison) - -### Library -[Runtime and Reporting Considerations](#runtime-and-reporting-considerations) - -[Passing Arguments](#passing-arguments) - -[Calculating Asymptotic Complexity](#asymptotic-complexity) - -[Templated Benchmarks](#templated-benchmarks) - -[Fixtures](#fixtures) - -[Custom Counters](#custom-counters) - -[Multithreaded Benchmarks](#multithreaded-benchmarks) - -[CPU Timers](#cpu-timers) - -[Manual Timing](#manual-timing) - -[Setting the Time Unit](#setting-the-time-unit) - -[Preventing Optimization](#preventing-optimization) - -[Reporting Statistics](#reporting-statistics) - -[Custom Statistics](#custom-statistics) - -[Using RegisterBenchmark](#using-register-benchmark) - -[Exiting with an Error](#exiting-with-an-error) - -[A Faster KeepRunning Loop](#a-faster-keep-running-loop) - -[Disabling CPU Frequency Scaling](#disabling-cpu-frequency-scaling) - -<a name="output-formats" /> - -### Output Formats - -The library supports multiple output formats. Use the -`--benchmark_format=<console|json|csv>` flag to set the format type. `console` -is the default format. - -The Console format is intended to be a human readable format. By default -the format generates color output. Context is output on stderr and the -tabular data on stdout. Example tabular output looks like: -``` -Benchmark Time(ns) CPU(ns) Iterations ----------------------------------------------------------------------- -BM_SetInsert/1024/1 28928 29349 23853 133.097kB/s 33.2742k items/s -BM_SetInsert/1024/8 32065 32913 21375 949.487kB/s 237.372k items/s -BM_SetInsert/1024/10 33157 33648 21431 1.13369MB/s 290.225k items/s -``` - -The JSON format outputs human readable json split into two top level attributes. -The `context` attribute contains information about the run in general, including -information about the CPU and the date. -The `benchmarks` attribute contains a list of every benchmark run. Example json -output looks like: -```json -{ - "context": { - "date": "2015/03/17-18:40:25", - "num_cpus": 40, - "mhz_per_cpu": 2801, - "cpu_scaling_enabled": false, - "build_type": "debug" - }, - "benchmarks": [ - { - "name": "BM_SetInsert/1024/1", - "iterations": 94877, - "real_time": 29275, - "cpu_time": 29836, - "bytes_per_second": 134066, - "items_per_second": 33516 - }, - { - "name": "BM_SetInsert/1024/8", - "iterations": 21609, - "real_time": 32317, - "cpu_time": 32429, - "bytes_per_second": 986770, - "items_per_second": 246693 - }, - { - "name": "BM_SetInsert/1024/10", - "iterations": 21393, - "real_time": 32724, - "cpu_time": 33355, - "bytes_per_second": 1199226, - "items_per_second": 299807 - } - ] -} -``` - -The CSV format outputs comma-separated values. The `context` is output on stderr -and the CSV itself on stdout. Example CSV output looks like: -``` -name,iterations,real_time,cpu_time,bytes_per_second,items_per_second,label -"BM_SetInsert/1024/1",65465,17890.7,8407.45,475768,118942, -"BM_SetInsert/1024/8",116606,18810.1,9766.64,3.27646e+06,819115, -"BM_SetInsert/1024/10",106365,17238.4,8421.53,4.74973e+06,1.18743e+06, -``` - -<a name="output-files" /> - -### Output Files - -Write benchmark results to a file with the `--benchmark_out=<filename>` option. -Specify the output format with `--benchmark_out_format={json|console|csv}`. Note that Specifying -`--benchmark_out` does not suppress the console output. - -<a name="running-a-subset-of-benchmarks" /> - -### Running a Subset of Benchmarks - -The `--benchmark_filter=<regex>` option can be used to only run the benchmarks -which match the specified `<regex>`. For example: - -```bash -$ ./run_benchmarks.x --benchmark_filter=BM_memcpy/32 -Run on (1 X 2300 MHz CPU ) -2016-06-25 19:34:24 -Benchmark Time CPU Iterations ----------------------------------------------------- -BM_memcpy/32 11 ns 11 ns 79545455 -BM_memcpy/32k 2181 ns 2185 ns 324074 -BM_memcpy/32 12 ns 12 ns 54687500 -BM_memcpy/32k 1834 ns 1837 ns 357143 -``` - -<a name="result-comparison" /> - -### Result comparison - -It is possible to compare the benchmarking results. See [Additional Tooling Documentation](docs/tools.md) - -<a name="runtime-and-reporting-considerations" /> - -### Runtime and Reporting Considerations - -When the benchmark binary is executed, each benchmark function is run serially. -The number of iterations to run is determined dynamically by running the -benchmark a few times and measuring the time taken and ensuring that the -ultimate result will be statistically stable. As such, faster benchmark -functions will be run for more iterations than slower benchmark functions, and -the number of iterations is thus reported. - -In all cases, the number of iterations for which the benchmark is run is -governed by the amount of time the benchmark takes. Concretely, the number of -iterations is at least one, not more than 1e9, until CPU time is greater than -the minimum time, or the wallclock time is 5x minimum time. The minimum time is -set per benchmark by calling `MinTime` on the registered benchmark object. - -Average timings are then reported over the iterations run. If multiple -repetitions are requested using the `--benchmark_repetitions` command-line -option, or at registration time, the benchmark function will be run several -times and statistical results across these repetitions will also be reported. - -As well as the per-benchmark entries, a preamble in the report will include -information about the machine on which the benchmarks are run. - -<a name="passing-arguments" /> - -### Passing Arguments - +### Passing arguments Sometimes a family of benchmarks can be implemented with just one routine that takes an extra argument to specify which one of the family of benchmarks to run. For example, the following code defines a family of benchmarks for @@ -452,22 +175,6 @@ BENCHMARK(BM_memcpy)->RangeMultiplier(2)->Range(8, 8<<10); ``` Now arguments generated are [ 8, 16, 32, 64, 128, 256, 512, 1024, 2k, 4k, 8k ]. -The preceding code shows a method of defining a sparse range. The following -example shows a method of defining a dense range. It is then used to benchmark -the performance of `std::vector` initialization for uniformly increasing sizes. - -```c++ -static void BM_DenseRange(benchmark::State& state) { - for(auto _ : state) { - std::vector<int> v(state.range(0), state.range(0)); - benchmark::DoNotOptimize(v.data()); - benchmark::ClobberMemory(); - } -} -BENCHMARK(BM_DenseRange)->DenseRange(0, 1024, 128); -``` -Now arguments generated are [ 0, 128, 256, 384, 512, 640, 768, 896, 1024 ]. - You might have a benchmark that depends on two or more inputs. For example, the following code defines a family of benchmarks for measuring the speed of set insertion. @@ -517,31 +224,7 @@ static void CustomArguments(benchmark::internal::Benchmark* b) { BENCHMARK(BM_SetInsert)->Apply(CustomArguments); ``` -#### Passing Arbitrary Arguments to a Benchmark - -In C++11 it is possible to define a benchmark that takes an arbitrary number -of extra arguments. The `BENCHMARK_CAPTURE(func, test_case_name, ...args)` -macro creates a benchmark that invokes `func` with the `benchmark::State` as -the first argument followed by the specified `args...`. -The `test_case_name` is appended to the name of the benchmark and -should describe the values passed. - -```c++ -template <class ...ExtraArgs> -void BM_takes_args(benchmark::State& state, ExtraArgs&&... extra_args) { - [...] -} -// Registers a benchmark named "BM_takes_args/int_string_test" that passes -// the specified values to `extra_args`. -BENCHMARK_CAPTURE(BM_takes_args, int_string_test, 42, std::string("abc")); -``` -Note that elements of `...args` may refer to global variables. Users should -avoid modifying global state inside of a benchmark. - -<a name="asymptotic-complexity" /> - -### Calculating Asymptotic Complexity (Big O) - +### Calculate asymptotic complexity (Big O) Asymptotic complexity might be calculated for a family of benchmarks. The following code will calculate the coefficient for the high-order term in the running time and the normalized root-mean square error of string comparison. @@ -575,12 +258,10 @@ BENCHMARK(BM_StringCompare)->RangeMultiplier(2) ->Range(1<<10, 1<<18)->Complexity([](int64_t n)->double{return n; }); ``` -<a name="templated-benchmarks" /> - -### Templated Benchmarks - -This example produces and consumes messages of size `sizeof(v)` `range_x` -times. It also outputs throughput in the absence of multiprogramming. +### Templated benchmarks +Templated benchmarks work the same way: This example produces and consumes +messages of size `sizeof(v)` `range_x` times. It also outputs throughput in the +absence of multiprogramming. ```c++ template <class Q> void BM_Sequential(benchmark::State& state) { @@ -611,216 +292,110 @@ Three macros are provided for adding benchmark templates. #define BENCHMARK_TEMPLATE2(func, arg1, arg2) ``` -<a name="fixtures" /> +### A Faster KeepRunning loop -### Fixtures - -Fixture tests are created by first defining a type that derives from -`::benchmark::Fixture` and then creating/registering the tests using the -following macros: - -* `BENCHMARK_F(ClassName, Method)` -* `BENCHMARK_DEFINE_F(ClassName, Method)` -* `BENCHMARK_REGISTER_F(ClassName, Method)` - -For Example: +In C++11 mode, a ranged-based for loop should be used in preference to +the `KeepRunning` loop for running the benchmarks. For example: ```c++ -class MyFixture : public benchmark::Fixture { -public: - void SetUp(const ::benchmark::State& state) { - } - - void TearDown(const ::benchmark::State& state) { - } -}; - -BENCHMARK_F(MyFixture, FooTest)(benchmark::State& st) { - for (auto _ : st) { - ... - } -} - -BENCHMARK_DEFINE_F(MyFixture, BarTest)(benchmark::State& st) { - for (auto _ : st) { - ... +static void BM_Fast(benchmark::State &state) { + for (auto _ : state) { + FastOperation(); } } -/* BarTest is NOT registered */ -BENCHMARK_REGISTER_F(MyFixture, BarTest)->Threads(2); -/* BarTest is now registered */ +BENCHMARK(BM_Fast); ``` -#### Templated Fixtures - -Also you can create templated fixture by using the following macros: - -* `BENCHMARK_TEMPLATE_F(ClassName, Method, ...)` -* `BENCHMARK_TEMPLATE_DEFINE_F(ClassName, Method, ...)` +The reason the ranged-for loop is faster than using `KeepRunning`, is +because `KeepRunning` requires a memory load and store of the iteration count +ever iteration, whereas the ranged-for variant is able to keep the iteration count +in a register. -For example: -```c++ -template<typename T> -class MyFixture : public benchmark::Fixture {}; +For example, an empty inner loop of using the ranged-based for method looks like: -BENCHMARK_TEMPLATE_F(MyFixture, IntTest, int)(benchmark::State& st) { - for (auto _ : st) { - ... - } -} +```asm +# Loop Init + mov rbx, qword ptr [r14 + 104] + call benchmark::State::StartKeepRunning() + test rbx, rbx + je .LoopEnd +.LoopHeader: # =>This Inner Loop Header: Depth=1 + add rbx, -1 + jne .LoopHeader +.LoopEnd: +``` -BENCHMARK_TEMPLATE_DEFINE_F(MyFixture, DoubleTest, double)(benchmark::State& st) { - for (auto _ : st) { - ... - } -} +Compared to an empty `KeepRunning` loop, which looks like: -BENCHMARK_REGISTER_F(MyFixture, DoubleTest)->Threads(2); +```asm +.LoopHeader: # in Loop: Header=BB0_3 Depth=1 + cmp byte ptr [rbx], 1 + jne .LoopInit +.LoopBody: # =>This Inner Loop Header: Depth=1 + mov rax, qword ptr [rbx + 8] + lea rcx, [rax + 1] + mov qword ptr [rbx + 8], rcx + cmp rax, qword ptr [rbx + 104] + jb .LoopHeader + jmp .LoopEnd +.LoopInit: + mov rdi, rbx + call benchmark::State::StartKeepRunning() + jmp .LoopBody +.LoopEnd: ``` -<a name="custom-counters" /> - -### Custom Counters +Unless C++03 compatibility is required, the ranged-for variant of writing +the benchmark loop should be preferred. -You can add your own counters with user-defined names. The example below -will add columns "Foo", "Bar" and "Baz" in its output: +## Passing arbitrary arguments to a benchmark +In C++11 it is possible to define a benchmark that takes an arbitrary number +of extra arguments. The `BENCHMARK_CAPTURE(func, test_case_name, ...args)` +macro creates a benchmark that invokes `func` with the `benchmark::State` as +the first argument followed by the specified `args...`. +The `test_case_name` is appended to the name of the benchmark and +should describe the values passed. ```c++ -static void UserCountersExample1(benchmark::State& state) { - double numFoos = 0, numBars = 0, numBazs = 0; - for (auto _ : state) { - // ... count Foo,Bar,Baz events - } - state.counters["Foo"] = numFoos; - state.counters["Bar"] = numBars; - state.counters["Baz"] = numBazs; +template <class ...ExtraArgs> +void BM_takes_args(benchmark::State& state, ExtraArgs&&... extra_args) { + [...] } +// Registers a benchmark named "BM_takes_args/int_string_test" that passes +// the specified values to `extra_args`. +BENCHMARK_CAPTURE(BM_takes_args, int_string_test, 42, std::string("abc")); ``` +Note that elements of `...args` may refer to global variables. Users should +avoid modifying global state inside of a benchmark. -The `state.counters` object is a `std::map` with `std::string` keys -and `Counter` values. The latter is a `double`-like class, via an implicit -conversion to `double&`. Thus you can use all of the standard arithmetic -assignment operators (`=,+=,-=,*=,/=`) to change the value of each counter. - -In multithreaded benchmarks, each counter is set on the calling thread only. -When the benchmark finishes, the counters from each thread will be summed; -the resulting sum is the value which will be shown for the benchmark. - -The `Counter` constructor accepts three parameters: the value as a `double` -; a bit flag which allows you to show counters as rates, and/or as per-thread -iteration, and/or as per-thread averages, and/or iteration invariants, -and/or finally inverting the result; and a flag specifying the 'unit' - i.e. -is 1k a 1000 (default, `benchmark::Counter::OneK::kIs1000`), or 1024 -(`benchmark::Counter::OneK::kIs1024`)? - -```c++ - // sets a simple counter - state.counters["Foo"] = numFoos; - - // Set the counter as a rate. It will be presented divided - // by the duration of the benchmark. - // Meaning: per one second, how many 'foo's are processed? - state.counters["FooRate"] = Counter(numFoos, benchmark::Counter::kIsRate); - - // Set the counter as a rate. It will be presented divided - // by the duration of the benchmark, and the result inverted. - // Meaning: how many seconds it takes to process one 'foo'? - state.counters["FooInvRate"] = Counter(numFoos, benchmark::Counter::kIsRate | benchmark::Counter::kInvert); - - // Set the counter as a thread-average quantity. It will - // be presented divided by the number of threads. - state.counters["FooAvg"] = Counter(numFoos, benchmark::Counter::kAvgThreads); +## Using RegisterBenchmark(name, fn, args...) - // There's also a combined flag: - state.counters["FooAvgRate"] = Counter(numFoos,benchmark::Counter::kAvgThreadsRate); +The `RegisterBenchmark(name, func, args...)` function provides an alternative +way to create and register benchmarks. +`RegisterBenchmark(name, func, args...)` creates, registers, and returns a +pointer to a new benchmark with the specified `name` that invokes +`func(st, args...)` where `st` is a `benchmark::State` object. - // This says that we process with the rate of state.range(0) bytes every iteration: - state.counters["BytesProcessed"] = Counter(state.range(0), benchmark::Counter::kIsIterationInvariantRate, benchmark::Counter::OneK::kIs1024); -``` +Unlike the `BENCHMARK` registration macros, which can only be used at the global +scope, the `RegisterBenchmark` can be called anywhere. This allows for +benchmark tests to be registered programmatically. -When you're compiling in C++11 mode or later you can use `insert()` with -`std::initializer_list`: +Additionally `RegisterBenchmark` allows any callable object to be registered +as a benchmark. Including capturing lambdas and function objects. +For Example: ```c++ - // With C++11, this can be done: - state.counters.insert({{"Foo", numFoos}, {"Bar", numBars}, {"Baz", numBazs}}); - // ... instead of: - state.counters["Foo"] = numFoos; - state.counters["Bar"] = numBars; - state.counters["Baz"] = numBazs; -``` - -#### Counter Reporting - -When using the console reporter, by default, user counters are printed at -the end after the table, the same way as ``bytes_processed`` and -``items_processed``. This is best for cases in which there are few counters, -or where there are only a couple of lines per benchmark. Here's an example of -the default output: - -``` ------------------------------------------------------------------------------- -Benchmark Time CPU Iterations UserCounters... ------------------------------------------------------------------------------- -BM_UserCounter/threads:8 2248 ns 10277 ns 68808 Bar=16 Bat=40 Baz=24 Foo=8 -BM_UserCounter/threads:1 9797 ns 9788 ns 71523 Bar=2 Bat=5 Baz=3 Foo=1024m -BM_UserCounter/threads:2 4924 ns 9842 ns 71036 Bar=4 Bat=10 Baz=6 Foo=2 -BM_UserCounter/threads:4 2589 ns 10284 ns 68012 Bar=8 Bat=20 Baz=12 Foo=4 -BM_UserCounter/threads:8 2212 ns 10287 ns 68040 Bar=16 Bat=40 Baz=24 Foo=8 -BM_UserCounter/threads:16 1782 ns 10278 ns 68144 Bar=32 Bat=80 Baz=48 Foo=16 -BM_UserCounter/threads:32 1291 ns 10296 ns 68256 Bar=64 Bat=160 Baz=96 Foo=32 -BM_UserCounter/threads:4 2615 ns 10307 ns 68040 Bar=8 Bat=20 Baz=12 Foo=4 -BM_Factorial 26 ns 26 ns 26608979 40320 -BM_Factorial/real_time 26 ns 26 ns 26587936 40320 -BM_CalculatePiRange/1 16 ns 16 ns 45704255 0 -BM_CalculatePiRange/8 73 ns 73 ns 9520927 3.28374 -BM_CalculatePiRange/64 609 ns 609 ns 1140647 3.15746 -BM_CalculatePiRange/512 4900 ns 4901 ns 142696 3.14355 -``` - -If this doesn't suit you, you can print each counter as a table column by -passing the flag `--benchmark_counters_tabular=true` to the benchmark -application. This is best for cases in which there are a lot of counters, or -a lot of lines per individual benchmark. Note that this will trigger a -reprinting of the table header any time the counter set changes between -individual benchmarks. Here's an example of corresponding output when -`--benchmark_counters_tabular=true` is passed: +auto BM_test = [](benchmark::State& st, auto Inputs) { /* ... */ }; +int main(int argc, char** argv) { + for (auto& test_input : { /* ... */ }) + benchmark::RegisterBenchmark(test_input.name(), BM_test, test_input); + benchmark::Initialize(&argc, argv); + benchmark::RunSpecifiedBenchmarks(); +} ``` ---------------------------------------------------------------------------------------- -Benchmark Time CPU Iterations Bar Bat Baz Foo ---------------------------------------------------------------------------------------- -BM_UserCounter/threads:8 2198 ns 9953 ns 70688 16 40 24 8 -BM_UserCounter/threads:1 9504 ns 9504 ns 73787 2 5 3 1 -BM_UserCounter/threads:2 4775 ns 9550 ns 72606 4 10 6 2 -BM_UserCounter/threads:4 2508 ns 9951 ns 70332 8 20 12 4 -BM_UserCounter/threads:8 2055 ns 9933 ns 70344 16 40 24 8 -BM_UserCounter/threads:16 1610 ns 9946 ns 70720 32 80 48 16 -BM_UserCounter/threads:32 1192 ns 9948 ns 70496 64 160 96 32 -BM_UserCounter/threads:4 2506 ns 9949 ns 70332 8 20 12 4 --------------------------------------------------------------- -Benchmark Time CPU Iterations --------------------------------------------------------------- -BM_Factorial 26 ns 26 ns 26392245 40320 -BM_Factorial/real_time 26 ns 26 ns 26494107 40320 -BM_CalculatePiRange/1 15 ns 15 ns 45571597 0 -BM_CalculatePiRange/8 74 ns 74 ns 9450212 3.28374 -BM_CalculatePiRange/64 595 ns 595 ns 1173901 3.15746 -BM_CalculatePiRange/512 4752 ns 4752 ns 147380 3.14355 -BM_CalculatePiRange/4k 37970 ns 37972 ns 18453 3.14184 -BM_CalculatePiRange/32k 303733 ns 303744 ns 2305 3.14162 -BM_CalculatePiRange/256k 2434095 ns 2434186 ns 288 3.1416 -BM_CalculatePiRange/1024k 9721140 ns 9721413 ns 71 3.14159 -BM_CalculatePi/threads:8 2255 ns 9943 ns 70936 -``` -Note above the additional header printed when the benchmark changes from -``BM_UserCounter`` to ``BM_Factorial``. This is because ``BM_Factorial`` does -not have the same counter set as ``BM_UserCounter``. - -<a name="multithreaded-benchmarks"/> - -### Multithreaded Benchmarks +### Multithreaded benchmarks In a multithreaded test (benchmark invoked by multiple threads simultaneously), it is guaranteed that none of the threads will start until all have reached the start of the benchmark loop, and all will have finished before any thread @@ -853,58 +428,11 @@ BENCHMARK(BM_test)->Range(8, 8<<10)->UseRealTime(); Without `UseRealTime`, CPU time is used by default. -<a name="cpu-timers" /> - -### CPU Timers - -By default, the CPU timer only measures the time spent by the main thread. -If the benchmark itself uses threads internally, this measurement may not -be what you are looking for. Instead, there is a way to measure the total -CPU usage of the process, by all the threads. - -```c++ -void callee(int i); - -static void MyMain(int size) { -#pragma omp parallel for - for(int i = 0; i < size; i++) - callee(i); -} - -static void BM_OpenMP(benchmark::State& state) { - for (auto _ : state) - MyMain(state.range(0)); -} - -// Measure the time spent by the main thread, use it to decide for how long to -// run the benchmark loop. Depending on the internal implementation detail may -// measure to anywhere from near-zero (the overhead spent before/after work -// handoff to worker thread[s]) to the whole single-thread time. -BENCHMARK(BM_OpenMP)->Range(8, 8<<10); - -// Measure the user-visible time, the wall clock (literally, the time that -// has passed on the clock on the wall), use it to decide for how long to -// run the benchmark loop. This will always be meaningful, an will match the -// time spent by the main thread in single-threaded case, in general decreasing -// with the number of internal threads doing the work. -BENCHMARK(BM_OpenMP)->Range(8, 8<<10)->UseRealTime(); - -// Measure the total CPU consumption, use it to decide for how long to -// run the benchmark loop. This will always measure to no less than the -// time spent by the main thread in single-threaded case. -BENCHMARK(BM_OpenMP)->Range(8, 8<<10)->MeasureProcessCPUTime(); - -// A mixture of the last two. Measure the total CPU consumption, but use the -// wall clock to decide for how long to run the benchmark loop. -BENCHMARK(BM_OpenMP)->Range(8, 8<<10)->MeasureProcessCPUTime()->UseRealTime(); -``` - -#### Controlling Timers - +## Controlling timers Normally, the entire duration of the work loop (`for (auto _ : state) {}`) -is measured. But sometimes, it is necessary to do some work inside of +is measured. But sometimes, it is nessesary to do some work inside of that loop, every iteration, but without counting that time to the benchmark time. -That is possible, although it is not recommended, since it has high overhead. +That is possible, althought it is not recommended, since it has high overhead. ```c++ static void BM_SetInsert_With_Timer_Control(benchmark::State& state) { @@ -921,10 +449,7 @@ static void BM_SetInsert_With_Timer_Control(benchmark::State& state) { BENCHMARK(BM_SetInsert_With_Timer_Control)->Ranges({{1<<10, 8<<10}, {128, 512}}); ``` -<a name="manual-timing" /> - -### Manual Timing - +## Manual timing For benchmarking something for which neither CPU time nor real-time are correct or accurate enough, completely manual timing is supported using the `UseManualTime` function. @@ -950,7 +475,7 @@ static void BM_ManualTiming(benchmark::State& state) { auto start = std::chrono::high_resolution_clock::now(); // Simulate some useful workload with a sleep std::this_thread::sleep_for(sleep_duration); - auto end = std::chrono::high_resolution_clock::now(); + auto end = std::chrono::high_resolution_clock::now(); auto elapsed_seconds = std::chrono::duration_cast<std::chrono::duration<double>>( @@ -962,22 +487,7 @@ static void BM_ManualTiming(benchmark::State& state) { BENCHMARK(BM_ManualTiming)->Range(1, 1<<17)->UseManualTime(); ``` -<a name="setting-the-time-unit" /> - -### Setting the Time Unit - -If a benchmark runs a few milliseconds it may be hard to visually compare the -measured times, since the output data is given in nanoseconds per default. In -order to manually set the time unit, you can specify it manually: - -```c++ -BENCHMARK(BM_test)->Unit(benchmark::kMillisecond); -``` - -<a name="preventing-optimization" /> - -### Preventing Optimization - +### Preventing optimisation To prevent a value or expression from being optimized away by the compiler the `benchmark::DoNotOptimize(...)` and `benchmark::ClobberMemory()` functions can be used. @@ -1035,10 +545,16 @@ static void BM_vector_push_back(benchmark::State& state) { Note that `ClobberMemory()` is only available for GNU or MSVC based compilers. -<a name="reporting-statistics" /> +### Set time unit manually +If a benchmark runs a few milliseconds it may be hard to visually compare the +measured times, since the output data is given in nanoseconds per default. In +order to manually set the time unit, you can specify it manually: -### Statistics: Reporting the Mean, Median and Standard Deviation of Repeated Benchmarks +```c++ +BENCHMARK(BM_test)->Unit(benchmark::kMillisecond); +``` +### Reporting the mean, median and standard deviation by repeated benchmarks By default each benchmark is run once and that single result is reported. However benchmarks are often noisy and a single result may not be representative of the overall behavior. For this reason it's possible to repeatedly rerun the @@ -1064,13 +580,10 @@ Calling `ReportAggregatesOnly(bool)` / `DisplayAggregatesOnly(bool)` on a registered benchmark object overrides the value of the appropriate flag for that benchmark. -<a name="custom-statistics" /> - -### Custom Statistics - +## User-defined statistics for repeated benchmarks While having mean, median and standard deviation is nice, this may not be -enough for everyone. For example you may want to know what the largest -observation is, e.g. because you have some real-time constraints. This is easy. +enough for everyone. For example you may want to know what is the largest +observation, e.g. because you have some real-time constraints. This is easy. The following code will specify a custom statistic to be calculated, defined by a lambda function. @@ -1090,38 +603,201 @@ BENCHMARK(BM_spin_empty) ->Arg(512); ``` -<a name="using-register-benchmark" /> +## Fixtures +Fixture tests are created by +first defining a type that derives from `::benchmark::Fixture` and then +creating/registering the tests using the following macros: + +* `BENCHMARK_F(ClassName, Method)` +* `BENCHMARK_DEFINE_F(ClassName, Method)` +* `BENCHMARK_REGISTER_F(ClassName, Method)` -### Using RegisterBenchmark(name, fn, args...) +For Example: -The `RegisterBenchmark(name, func, args...)` function provides an alternative -way to create and register benchmarks. -`RegisterBenchmark(name, func, args...)` creates, registers, and returns a -pointer to a new benchmark with the specified `name` that invokes -`func(st, args...)` where `st` is a `benchmark::State` object. +```c++ +class MyFixture : public benchmark::Fixture { +public: + void SetUp(const ::benchmark::State& state) { + } -Unlike the `BENCHMARK` registration macros, which can only be used at the global -scope, the `RegisterBenchmark` can be called anywhere. This allows for -benchmark tests to be registered programmatically. + void TearDown(const ::benchmark::State& state) { + } +}; -Additionally `RegisterBenchmark` allows any callable object to be registered -as a benchmark. Including capturing lambdas and function objects. +BENCHMARK_F(MyFixture, FooTest)(benchmark::State& st) { + for (auto _ : st) { + ... + } +} -For Example: +BENCHMARK_DEFINE_F(MyFixture, BarTest)(benchmark::State& st) { + for (auto _ : st) { + ... + } +} +/* BarTest is NOT registered */ +BENCHMARK_REGISTER_F(MyFixture, BarTest)->Threads(2); +/* BarTest is now registered */ +``` + +### Templated fixtures +Also you can create templated fixture by using the following macros: + +* `BENCHMARK_TEMPLATE_F(ClassName, Method, ...)` +* `BENCHMARK_TEMPLATE_DEFINE_F(ClassName, Method, ...)` + +For example: ```c++ -auto BM_test = [](benchmark::State& st, auto Inputs) { /* ... */ }; +template<typename T> +class MyFixture : public benchmark::Fixture {}; -int main(int argc, char** argv) { - for (auto& test_input : { /* ... */ }) - benchmark::RegisterBenchmark(test_input.name(), BM_test, test_input); - benchmark::Initialize(&argc, argv); - benchmark::RunSpecifiedBenchmarks(); +BENCHMARK_TEMPLATE_F(MyFixture, IntTest, int)(benchmark::State& st) { + for (auto _ : st) { + ... + } } + +BENCHMARK_TEMPLATE_DEFINE_F(MyFixture, DoubleTest, double)(benchmark::State& st) { + for (auto _ : st) { + ... + } +} + +BENCHMARK_REGISTER_F(MyFixture, DoubleTest)->Threads(2); +``` + +## User-defined counters + +You can add your own counters with user-defined names. The example below +will add columns "Foo", "Bar" and "Baz" in its output: + +```c++ +static void UserCountersExample1(benchmark::State& state) { + double numFoos = 0, numBars = 0, numBazs = 0; + for (auto _ : state) { + // ... count Foo,Bar,Baz events + } + state.counters["Foo"] = numFoos; + state.counters["Bar"] = numBars; + state.counters["Baz"] = numBazs; +} +``` + +The `state.counters` object is a `std::map` with `std::string` keys +and `Counter` values. The latter is a `double`-like class, via an implicit +conversion to `double&`. Thus you can use all of the standard arithmetic +assignment operators (`=,+=,-=,*=,/=`) to change the value of each counter. + +In multithreaded benchmarks, each counter is set on the calling thread only. +When the benchmark finishes, the counters from each thread will be summed; +the resulting sum is the value which will be shown for the benchmark. + +The `Counter` constructor accepts three parameters: the value as a `double` +; a bit flag which allows you to show counters as rates, and/or as per-thread +iteration, and/or as per-thread averages, and/or iteration invariants; +and a flag specifying the 'unit' - i.e. is 1k a 1000 (default, +`benchmark::Counter::OneK::kIs1000`), or 1024 +(`benchmark::Counter::OneK::kIs1024`)? + +```c++ + // sets a simple counter + state.counters["Foo"] = numFoos; + + // Set the counter as a rate. It will be presented divided + // by the duration of the benchmark. + state.counters["FooRate"] = Counter(numFoos, benchmark::Counter::kIsRate); + + // Set the counter as a thread-average quantity. It will + // be presented divided by the number of threads. + state.counters["FooAvg"] = Counter(numFoos, benchmark::Counter::kAvgThreads); + + // There's also a combined flag: + state.counters["FooAvgRate"] = Counter(numFoos,benchmark::Counter::kAvgThreadsRate); + + // This says that we process with the rate of state.range(0) bytes every iteration: + state.counters["BytesProcessed"] = Counter(state.range(0), benchmark::Counter::kIsIterationInvariantRate, benchmark::Counter::OneK::kIs1024); +``` + +When you're compiling in C++11 mode or later you can use `insert()` with +`std::initializer_list`: + +```c++ + // With C++11, this can be done: + state.counters.insert({{"Foo", numFoos}, {"Bar", numBars}, {"Baz", numBazs}}); + // ... instead of: + state.counters["Foo"] = numFoos; + state.counters["Bar"] = numBars; + state.counters["Baz"] = numBazs; +``` + +### Counter reporting + +When using the console reporter, by default, user counters are are printed at +the end after the table, the same way as ``bytes_processed`` and +``items_processed``. This is best for cases in which there are few counters, +or where there are only a couple of lines per benchmark. Here's an example of +the default output: + +``` +------------------------------------------------------------------------------ +Benchmark Time CPU Iterations UserCounters... +------------------------------------------------------------------------------ +BM_UserCounter/threads:8 2248 ns 10277 ns 68808 Bar=16 Bat=40 Baz=24 Foo=8 +BM_UserCounter/threads:1 9797 ns 9788 ns 71523 Bar=2 Bat=5 Baz=3 Foo=1024m +BM_UserCounter/threads:2 4924 ns 9842 ns 71036 Bar=4 Bat=10 Baz=6 Foo=2 +BM_UserCounter/threads:4 2589 ns 10284 ns 68012 Bar=8 Bat=20 Baz=12 Foo=4 +BM_UserCounter/threads:8 2212 ns 10287 ns 68040 Bar=16 Bat=40 Baz=24 Foo=8 +BM_UserCounter/threads:16 1782 ns 10278 ns 68144 Bar=32 Bat=80 Baz=48 Foo=16 +BM_UserCounter/threads:32 1291 ns 10296 ns 68256 Bar=64 Bat=160 Baz=96 Foo=32 +BM_UserCounter/threads:4 2615 ns 10307 ns 68040 Bar=8 Bat=20 Baz=12 Foo=4 +BM_Factorial 26 ns 26 ns 26608979 40320 +BM_Factorial/real_time 26 ns 26 ns 26587936 40320 +BM_CalculatePiRange/1 16 ns 16 ns 45704255 0 +BM_CalculatePiRange/8 73 ns 73 ns 9520927 3.28374 +BM_CalculatePiRange/64 609 ns 609 ns 1140647 3.15746 +BM_CalculatePiRange/512 4900 ns 4901 ns 142696 3.14355 ``` -<a name="exiting-with-an-error" /> +If this doesn't suit you, you can print each counter as a table column by +passing the flag `--benchmark_counters_tabular=true` to the benchmark +application. This is best for cases in which there are a lot of counters, or +a lot of lines per individual benchmark. Note that this will trigger a +reprinting of the table header any time the counter set changes between +individual benchmarks. Here's an example of corresponding output when +`--benchmark_counters_tabular=true` is passed: -### Exiting with an Error +``` +--------------------------------------------------------------------------------------- +Benchmark Time CPU Iterations Bar Bat Baz Foo +--------------------------------------------------------------------------------------- +BM_UserCounter/threads:8 2198 ns 9953 ns 70688 16 40 24 8 +BM_UserCounter/threads:1 9504 ns 9504 ns 73787 2 5 3 1 +BM_UserCounter/threads:2 4775 ns 9550 ns 72606 4 10 6 2 +BM_UserCounter/threads:4 2508 ns 9951 ns 70332 8 20 12 4 +BM_UserCounter/threads:8 2055 ns 9933 ns 70344 16 40 24 8 +BM_UserCounter/threads:16 1610 ns 9946 ns 70720 32 80 48 16 +BM_UserCounter/threads:32 1192 ns 9948 ns 70496 64 160 96 32 +BM_UserCounter/threads:4 2506 ns 9949 ns 70332 8 20 12 4 +-------------------------------------------------------------- +Benchmark Time CPU Iterations +-------------------------------------------------------------- +BM_Factorial 26 ns 26 ns 26392245 40320 +BM_Factorial/real_time 26 ns 26 ns 26494107 40320 +BM_CalculatePiRange/1 15 ns 15 ns 45571597 0 +BM_CalculatePiRange/8 74 ns 74 ns 9450212 3.28374 +BM_CalculatePiRange/64 595 ns 595 ns 1173901 3.15746 +BM_CalculatePiRange/512 4752 ns 4752 ns 147380 3.14355 +BM_CalculatePiRange/4k 37970 ns 37972 ns 18453 3.14184 +BM_CalculatePiRange/32k 303733 ns 303744 ns 2305 3.14162 +BM_CalculatePiRange/256k 2434095 ns 2434186 ns 288 3.1416 +BM_CalculatePiRange/1024k 9721140 ns 9721413 ns 71 3.14159 +BM_CalculatePi/threads:8 2255 ns 9943 ns 70936 +``` +Note above the additional header printed when the benchmark changes from +``BM_UserCounter`` to ``BM_Factorial``. This is because ``BM_Factorial`` does +not have the same counter set as ``BM_UserCounter``. + +## Exiting Benchmarks in Error When errors caused by external influences, such as file I/O and network communication, occur within a benchmark the @@ -1143,7 +819,7 @@ static void BM_test(benchmark::State& state) { state.SkipWithError("Resource is not good!"); // KeepRunning() loop will not be entered. } - while (state.KeepRunning()) { + for (state.KeepRunning()) { auto data = resource.read_data(); if (!resource.good()) { state.SkipWithError("Failed to read data!"); @@ -1161,67 +837,162 @@ static void BM_test_ranged_fo(benchmark::State & state) { } } ``` -<a name="a-faster-keep-running-loop" /> -### A Faster KeepRunning Loop +## Running a subset of the benchmarks -In C++11 mode, a ranged-based for loop should be used in preference to -the `KeepRunning` loop for running the benchmarks. For example: +The `--benchmark_filter=<regex>` option can be used to only run the benchmarks +which match the specified `<regex>`. For example: -```c++ -static void BM_Fast(benchmark::State &state) { - for (auto _ : state) { - FastOperation(); - } +```bash +$ ./run_benchmarks.x --benchmark_filter=BM_memcpy/32 +Run on (1 X 2300 MHz CPU ) +2016-06-25 19:34:24 +Benchmark Time CPU Iterations +---------------------------------------------------- +BM_memcpy/32 11 ns 11 ns 79545455 +BM_memcpy/32k 2181 ns 2185 ns 324074 +BM_memcpy/32 12 ns 12 ns 54687500 +BM_memcpy/32k 1834 ns 1837 ns 357143 +``` + +## Runtime and reporting considerations +When the benchmark binary is executed, each benchmark function is run serially. +The number of iterations to run is determined dynamically by running the +benchmark a few times and measuring the time taken and ensuring that the +ultimate result will be statistically stable. As such, faster benchmark +functions will be run for more iterations than slower benchmark functions, and +the number of iterations is thus reported. + +In all cases, the number of iterations for which the benchmark is run is +governed by the amount of time the benchmark takes. Concretely, the number of +iterations is at least one, not more than 1e9, until CPU time is greater than +the minimum time, or the wallclock time is 5x minimum time. The minimum time is +set per benchmark by calling `MinTime` on the registered benchmark object. + +Average timings are then reported over the iterations run. If multiple +repetitions are requested using the `--benchmark_repetitions` command-line +option, or at registration time, the benchmark function will be run several +times and statistical results across these repetitions will also be reported. + +As well as the per-benchmark entries, a preamble in the report will include +information about the machine on which the benchmarks are run. + +### Output Formats +The library supports multiple output formats. Use the +`--benchmark_format=<console|json|csv>` flag to set the format type. `console` +is the default format. + +The Console format is intended to be a human readable format. By default +the format generates color output. Context is output on stderr and the +tabular data on stdout. Example tabular output looks like: +``` +Benchmark Time(ns) CPU(ns) Iterations +---------------------------------------------------------------------- +BM_SetInsert/1024/1 28928 29349 23853 133.097kB/s 33.2742k items/s +BM_SetInsert/1024/8 32065 32913 21375 949.487kB/s 237.372k items/s +BM_SetInsert/1024/10 33157 33648 21431 1.13369MB/s 290.225k items/s +``` + +The JSON format outputs human readable json split into two top level attributes. +The `context` attribute contains information about the run in general, including +information about the CPU and the date. +The `benchmarks` attribute contains a list of every benchmark run. Example json +output looks like: +```json +{ + "context": { + "date": "2015/03/17-18:40:25", + "num_cpus": 40, + "mhz_per_cpu": 2801, + "cpu_scaling_enabled": false, + "build_type": "debug" + }, + "benchmarks": [ + { + "name": "BM_SetInsert/1024/1", + "iterations": 94877, + "real_time": 29275, + "cpu_time": 29836, + "bytes_per_second": 134066, + "items_per_second": 33516 + }, + { + "name": "BM_SetInsert/1024/8", + "iterations": 21609, + "real_time": 32317, + "cpu_time": 32429, + "bytes_per_second": 986770, + "items_per_second": 246693 + }, + { + "name": "BM_SetInsert/1024/10", + "iterations": 21393, + "real_time": 32724, + "cpu_time": 33355, + "bytes_per_second": 1199226, + "items_per_second": 299807 + } + ] } -BENCHMARK(BM_Fast); ``` -The reason the ranged-for loop is faster than using `KeepRunning`, is -because `KeepRunning` requires a memory load and store of the iteration count -ever iteration, whereas the ranged-for variant is able to keep the iteration count -in a register. +The CSV format outputs comma-separated values. The `context` is output on stderr +and the CSV itself on stdout. Example CSV output looks like: +``` +name,iterations,real_time,cpu_time,bytes_per_second,items_per_second,label +"BM_SetInsert/1024/1",65465,17890.7,8407.45,475768,118942, +"BM_SetInsert/1024/8",116606,18810.1,9766.64,3.27646e+06,819115, +"BM_SetInsert/1024/10",106365,17238.4,8421.53,4.74973e+06,1.18743e+06, +``` -For example, an empty inner loop of using the ranged-based for method looks like: +### Output Files +The library supports writing the output of the benchmark to a file specified +by `--benchmark_out=<filename>`. The format of the output can be specified +using `--benchmark_out_format={json|console|csv}`. Specifying +`--benchmark_out` does not suppress the console output. + +## Result comparison + +It is possible to compare the benchmarking results. See [Additional Tooling Documentation](docs/tools.md) + +## Debug vs Release +By default, benchmark builds as a debug library. You will see a warning in the +output when this is the case. To build it as a release library instead, use: -```asm -# Loop Init - mov rbx, qword ptr [r14 + 104] - call benchmark::State::StartKeepRunning() - test rbx, rbx - je .LoopEnd -.LoopHeader: # =>This Inner Loop Header: Depth=1 - add rbx, -1 - jne .LoopHeader -.LoopEnd: +``` +cmake -DCMAKE_BUILD_TYPE=Release ``` -Compared to an empty `KeepRunning` loop, which looks like: +To enable link-time optimisation, use -```asm -.LoopHeader: # in Loop: Header=BB0_3 Depth=1 - cmp byte ptr [rbx], 1 - jne .LoopInit -.LoopBody: # =>This Inner Loop Header: Depth=1 - mov rax, qword ptr [rbx + 8] - lea rcx, [rax + 1] - mov qword ptr [rbx + 8], rcx - cmp rax, qword ptr [rbx + 104] - jb .LoopHeader - jmp .LoopEnd -.LoopInit: - mov rdi, rbx - call benchmark::State::StartKeepRunning() - jmp .LoopBody -.LoopEnd: +``` +cmake -DCMAKE_BUILD_TYPE=Release -DBENCHMARK_ENABLE_LTO=true ``` -Unless C++03 compatibility is required, the ranged-for variant of writing -the benchmark loop should be preferred. +If you are using gcc, you might need to set `GCC_AR` and `GCC_RANLIB` cmake +cache variables, if autodetection fails. + +If you are using clang, you may need to set `LLVMAR_EXECUTABLE`, +`LLVMNM_EXECUTABLE` and `LLVMRANLIB_EXECUTABLE` cmake cache variables. + +## Compiler Support + +Google Benchmark uses C++11 when building the library. As such we require +a modern C++ toolchain, both compiler and standard library. + +The following minimum versions are strongly recommended build the library: + +* GCC 4.8 +* Clang 3.4 +* Visual Studio 2013 +* Intel 2015 Update 1 + +Anything older *may* work. -<a name="disabling-cpu-frequency-scaling" /> +Note: Using the library and its headers in C++03 is supported. C++11 is only +required to build the library. -### Disabling CPU Frequency Scaling +## Disable CPU frequency scaling If you see this error: ``` ***WARNING*** CPU scaling is enabled, the benchmark real time measurements may be noisy and will incur extra overhead. @@ -1,15 +1,7 @@ workspace(name = "com_github_google_benchmark") -load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") - -http_archive( - name = "rules_cc", - strip_prefix = "rules_cc-a508235df92e71d537fcbae0c7c952ea6957a912", - urls = ["https://github.com/bazelbuild/rules_cc/archive/a508235df92e71d537fcbae0c7c952ea6957a912.zip"], -) - http_archive( - name = "com_google_googletest", - strip_prefix = "googletest-3f0cf6b62ad1eb50d8736538363d3580dd640c3e", - urls = ["https://github.com/google/googletest/archive/3f0cf6b62ad1eb50d8736538363d3580dd640c3e.zip"], + name = "com_google_googletest", + urls = ["https://github.com/google/googletest/archive/3f0cf6b62ad1eb50d8736538363d3580dd640c3e.zip"], + strip_prefix = "googletest-3f0cf6b62ad1eb50d8736538363d3580dd640c3e", ) diff --git a/_config.yml b/_config.yml deleted file mode 100644 index 1885487..0000000 --- a/_config.yml +++ /dev/null @@ -1 +0,0 @@ -theme: jekyll-theme-midnight
\ No newline at end of file diff --git a/appveyor.yml b/appveyor.yml index 81da955..cf24019 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -41,7 +41,7 @@ build_script: - cmake --build . --config %configuration% test_script: - - ctest --build-config %configuration% --timeout 300 --output-on-failure + - ctest -c %configuration% --timeout 300 --output-on-failure artifacts: - path: '_build/CMakeFiles/*.log' diff --git a/cmake/CXXFeatureCheck.cmake b/cmake/CXXFeatureCheck.cmake index 059d510..99b56dd 100644 --- a/cmake/CXXFeatureCheck.cmake +++ b/cmake/CXXFeatureCheck.cmake @@ -37,9 +37,9 @@ function(cxx_feature_check FILE) if(COMPILE_${FEATURE}) message(WARNING "If you see build failures due to cross compilation, try setting HAVE_${VAR} to 0") - set(RUN_${FEATURE} 0 CACHE INTERNAL "") + set(RUN_${FEATURE} 0) else() - set(RUN_${FEATURE} 1 CACHE INTERNAL "") + set(RUN_${FEATURE} 1) endif() else() message(STATUS "Performing Test ${FEATURE}") diff --git a/cmake/GoogleTest.cmake b/cmake/GoogleTest.cmake deleted file mode 100644 index dd611fc..0000000 --- a/cmake/GoogleTest.cmake +++ /dev/null @@ -1,41 +0,0 @@ -# Download and unpack googletest at configure time -set(GOOGLETEST_PREFIX "${benchmark_BINARY_DIR}/third_party/googletest") -configure_file(${benchmark_SOURCE_DIR}/cmake/GoogleTest.cmake.in ${GOOGLETEST_PREFIX}/CMakeLists.txt @ONLY) - -set(GOOGLETEST_PATH "${CMAKE_CURRENT_SOURCE_DIR}/googletest" CACHE PATH "") # Mind the quotes -execute_process(COMMAND ${CMAKE_COMMAND} -G "${CMAKE_GENERATOR}" - -DALLOW_DOWNLOADING_GOOGLETEST=${BENCHMARK_DOWNLOAD_DEPENDENCIES} -DGOOGLETEST_PATH:PATH=${GOOGLETEST_PATH} . - RESULT_VARIABLE result - WORKING_DIRECTORY ${GOOGLETEST_PREFIX} -) - -if(result) - message(FATAL_ERROR "CMake step for googletest failed: ${result}") -endif() - -execute_process( - COMMAND ${CMAKE_COMMAND} --build . - RESULT_VARIABLE result - WORKING_DIRECTORY ${GOOGLETEST_PREFIX} -) - -if(result) - message(FATAL_ERROR "Build step for googletest failed: ${result}") -endif() - -# Prevent overriding the parent project's compiler/linker -# settings on Windows -set(gtest_force_shared_crt ON CACHE BOOL "" FORCE) - -include(${GOOGLETEST_PREFIX}/googletest-paths.cmake) - -# Add googletest directly to our build. This defines -# the gtest and gtest_main targets. -add_subdirectory(${GOOGLETEST_SOURCE_DIR} - ${GOOGLETEST_BINARY_DIR} - EXCLUDE_FROM_ALL) - -set_target_properties(gtest PROPERTIES INTERFACE_SYSTEM_INCLUDE_DIRECTORIES $<TARGET_PROPERTY:gtest,INTERFACE_INCLUDE_DIRECTORIES>) -set_target_properties(gtest_main PROPERTIES INTERFACE_SYSTEM_INCLUDE_DIRECTORIES $<TARGET_PROPERTY:gtest_main,INTERFACE_INCLUDE_DIRECTORIES>) -set_target_properties(gmock PROPERTIES INTERFACE_SYSTEM_INCLUDE_DIRECTORIES $<TARGET_PROPERTY:gmock,INTERFACE_INCLUDE_DIRECTORIES>) -set_target_properties(gmock_main PROPERTIES INTERFACE_SYSTEM_INCLUDE_DIRECTORIES $<TARGET_PROPERTY:gmock_main,INTERFACE_INCLUDE_DIRECTORIES>) diff --git a/cmake/GoogleTest.cmake.in b/cmake/GoogleTest.cmake.in deleted file mode 100644 index 28818ee..0000000 --- a/cmake/GoogleTest.cmake.in +++ /dev/null @@ -1,58 +0,0 @@ -cmake_minimum_required(VERSION 2.8.12) - -project(googletest-download NONE) - -# Enable ExternalProject CMake module -include(ExternalProject) - -option(ALLOW_DOWNLOADING_GOOGLETEST "If googletest src tree is not found in location specified by GOOGLETEST_PATH, do fetch the archive from internet" OFF) -set(GOOGLETEST_PATH "/usr/src/googletest" CACHE PATH - "Path to the googletest root tree. Should contain googletest and googlemock subdirs. And CMakeLists.txt in root, and in both of these subdirs") - -# Download and install GoogleTest - -message(STATUS "Looking for Google Test sources") -message(STATUS "Looking for Google Test sources in ${GOOGLETEST_PATH}") -if(EXISTS "${GOOGLETEST_PATH}" AND IS_DIRECTORY "${GOOGLETEST_PATH}" AND EXISTS "${GOOGLETEST_PATH}/CMakeLists.txt" AND - EXISTS "${GOOGLETEST_PATH}/googletest" AND IS_DIRECTORY "${GOOGLETEST_PATH}/googletest" AND EXISTS "${GOOGLETEST_PATH}/googletest/CMakeLists.txt" AND - EXISTS "${GOOGLETEST_PATH}/googlemock" AND IS_DIRECTORY "${GOOGLETEST_PATH}/googlemock" AND EXISTS "${GOOGLETEST_PATH}/googlemock/CMakeLists.txt") - message(STATUS "Found Google Test in ${GOOGLETEST_PATH}") - - ExternalProject_Add( - googletest - PREFIX "${CMAKE_BINARY_DIR}" - DOWNLOAD_DIR "${CMAKE_BINARY_DIR}/download" - SOURCE_DIR "${GOOGLETEST_PATH}" # use existing src dir. - BINARY_DIR "${CMAKE_BINARY_DIR}/build" - CONFIGURE_COMMAND "" - BUILD_COMMAND "" - INSTALL_COMMAND "" - TEST_COMMAND "" - ) -else() - if(NOT ALLOW_DOWNLOADING_GOOGLETEST) - message(SEND_ERROR "Did not find Google Test sources! Either pass correct path in GOOGLETEST_PATH, or enable ALLOW_DOWNLOADING_GOOGLETEST, or disable BENCHMARK_ENABLE_GTEST_TESTS / BENCHMARK_ENABLE_TESTING.") - else() - message(WARNING "Did not find Google Test sources! Fetching from web...") - ExternalProject_Add( - googletest - GIT_REPOSITORY https://github.com/google/googletest.git - GIT_TAG master - PREFIX "${CMAKE_BINARY_DIR}" - STAMP_DIR "${CMAKE_BINARY_DIR}/stamp" - DOWNLOAD_DIR "${CMAKE_BINARY_DIR}/download" - SOURCE_DIR "${CMAKE_BINARY_DIR}/src" - BINARY_DIR "${CMAKE_BINARY_DIR}/build" - CONFIGURE_COMMAND "" - BUILD_COMMAND "" - INSTALL_COMMAND "" - TEST_COMMAND "" - ) - endif() -endif() - -ExternalProject_Get_Property(googletest SOURCE_DIR BINARY_DIR) -file(WRITE googletest-paths.cmake -"set(GOOGLETEST_SOURCE_DIR \"${SOURCE_DIR}\") -set(GOOGLETEST_BINARY_DIR \"${BINARY_DIR}\") -") diff --git a/cmake/HandleGTest.cmake b/cmake/HandleGTest.cmake new file mode 100644 index 0000000..b9c1443 --- /dev/null +++ b/cmake/HandleGTest.cmake @@ -0,0 +1,113 @@ + +include(split_list) + +macro(build_external_gtest) + include(ExternalProject) + set(GTEST_FLAGS "") + if (BENCHMARK_USE_LIBCXX) + if ("${CMAKE_CXX_COMPILER_ID}" MATCHES "Clang") + list(APPEND GTEST_FLAGS -stdlib=libc++) + else() + message(WARNING "Unsupported compiler (${CMAKE_CXX_COMPILER}) when using libc++") + endif() + endif() + if (BENCHMARK_BUILD_32_BITS) + list(APPEND GTEST_FLAGS -m32) + endif() + if (NOT "${CMAKE_CXX_FLAGS}" STREQUAL "") + list(APPEND GTEST_FLAGS ${CMAKE_CXX_FLAGS}) + endif() + string(TOUPPER "${CMAKE_BUILD_TYPE}" GTEST_BUILD_TYPE) + if ("${GTEST_BUILD_TYPE}" STREQUAL "COVERAGE") + set(GTEST_BUILD_TYPE "DEBUG") + endif() + # FIXME: Since 10/Feb/2017 the googletest trunk has had a bug where + # -Werror=unused-function fires during the build on OS X. This is a temporary + # workaround to keep our travis bots from failing. It should be removed + # once gtest is fixed. + if (NOT "${CMAKE_CXX_COMPILER_ID}" STREQUAL "MSVC") + list(APPEND GTEST_FLAGS "-Wno-unused-function") + endif() + split_list(GTEST_FLAGS) + set(EXCLUDE_FROM_ALL_OPT "") + set(EXCLUDE_FROM_ALL_VALUE "") + if (${CMAKE_VERSION} VERSION_GREATER "3.0.99") + set(EXCLUDE_FROM_ALL_OPT "EXCLUDE_FROM_ALL") + set(EXCLUDE_FROM_ALL_VALUE "ON") + endif() + ExternalProject_Add(googletest + ${EXCLUDE_FROM_ALL_OPT} ${EXCLUDE_FROM_ALL_VALUE} + GIT_REPOSITORY https://github.com/google/googletest.git + GIT_TAG master + PREFIX "${CMAKE_BINARY_DIR}/googletest" + INSTALL_DIR "${CMAKE_BINARY_DIR}/googletest" + CMAKE_CACHE_ARGS + -DCMAKE_BUILD_TYPE:STRING=${GTEST_BUILD_TYPE} + -DCMAKE_C_COMPILER:STRING=${CMAKE_C_COMPILER} + -DCMAKE_CXX_COMPILER:STRING=${CMAKE_CXX_COMPILER} + -DCMAKE_INSTALL_PREFIX:PATH=<INSTALL_DIR> + -DCMAKE_INSTALL_LIBDIR:PATH=<INSTALL_DIR>/lib + -DCMAKE_CXX_FLAGS:STRING=${GTEST_FLAGS} + -Dgtest_force_shared_crt:BOOL=ON + ) + + ExternalProject_Get_Property(googletest install_dir) + set(GTEST_INCLUDE_DIRS ${install_dir}/include) + file(MAKE_DIRECTORY ${GTEST_INCLUDE_DIRS}) + + set(LIB_SUFFIX "${CMAKE_STATIC_LIBRARY_SUFFIX}") + set(LIB_PREFIX "${CMAKE_STATIC_LIBRARY_PREFIX}") + if("${GTEST_BUILD_TYPE}" STREQUAL "DEBUG") + set(LIB_SUFFIX "d${CMAKE_STATIC_LIBRARY_SUFFIX}") + endif() + + # Use gmock_main instead of gtest_main because it initializes gtest as well. + # Note: The libraries are listed in reverse order of their dependancies. + foreach(LIB gtest gmock gmock_main) + add_library(${LIB} UNKNOWN IMPORTED) + set_target_properties(${LIB} PROPERTIES + IMPORTED_LOCATION ${install_dir}/lib/${LIB_PREFIX}${LIB}${LIB_SUFFIX} + INTERFACE_INCLUDE_DIRECTORIES ${GTEST_INCLUDE_DIRS} + INTERFACE_LINK_LIBRARIES "${GTEST_BOTH_LIBRARIES}" + ) + add_dependencies(${LIB} googletest) + list(APPEND GTEST_BOTH_LIBRARIES ${LIB}) + endforeach() +endmacro(build_external_gtest) + +if (BENCHMARK_ENABLE_GTEST_TESTS) + if (IS_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/googletest) + set(GTEST_ROOT "${CMAKE_CURRENT_SOURCE_DIR}/googletest") + set(INSTALL_GTEST OFF CACHE INTERNAL "") + set(INSTALL_GMOCK OFF CACHE INTERNAL "") + add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/googletest) + set(GTEST_BOTH_LIBRARIES gtest gmock gmock_main) + foreach(HEADER test mock) + # CMake 2.8 and older don't respect INTERFACE_INCLUDE_DIRECTORIES, so we + # have to add the paths ourselves. + set(HFILE g${HEADER}/g${HEADER}.h) + set(HPATH ${GTEST_ROOT}/google${HEADER}/include) + find_path(HEADER_PATH_${HEADER} ${HFILE} + NO_DEFAULT_PATHS + HINTS ${HPATH} + ) + if (NOT HEADER_PATH_${HEADER}) + message(FATAL_ERROR "Failed to find header ${HFILE} in ${HPATH}") + endif() + list(APPEND GTEST_INCLUDE_DIRS ${HEADER_PATH_${HEADER}}) + endforeach() + elseif(BENCHMARK_DOWNLOAD_DEPENDENCIES) + build_external_gtest() + else() + find_package(GTest REQUIRED) + find_path(GMOCK_INCLUDE_DIRS gmock/gmock.h + HINTS ${GTEST_INCLUDE_DIRS}) + if (NOT GMOCK_INCLUDE_DIRS) + message(FATAL_ERROR "Failed to find header gmock/gmock.h with hint ${GTEST_INCLUDE_DIRS}") + endif() + set(GTEST_INCLUDE_DIRS ${GTEST_INCLUDE_DIRS} ${GMOCK_INCLUDE_DIRS}) + # FIXME: We don't currently require the gmock library to build the tests, + # and it's likely we won't find it, so we don't try. As long as we've + # found the gmock/gmock.h header and gtest_main that should be good enough. + endif() +endif() diff --git a/dependencies.md b/dependencies.md deleted file mode 100644 index 6289b4e..0000000 --- a/dependencies.md +++ /dev/null @@ -1,18 +0,0 @@ -# Build tool dependency policy - -To ensure the broadest compatibility when building the benchmark library, but -still allow forward progress, we require any build tooling to be available for: - -* Debian stable AND -* The last two Ubuntu LTS releases AND - -Currently, this means using build tool versions that are available for Ubuntu -16.04 (Xenial), Ubuntu 18.04 (Bionic), and Debian stretch. - -_Note, [travis](.travis.yml) runs under Ubuntu 14.04 (Trusty) for linux builds._ - -## cmake -The current supported version is cmake 3.5.1 as of 2018-06-06. - -_Note, this version is also available for Ubuntu 14.04, the previous Ubuntu LTS -release, as `cmake3`._ diff --git a/docs/_config.yml b/docs/_config.yml deleted file mode 100644 index 1885487..0000000 --- a/docs/_config.yml +++ /dev/null @@ -1 +0,0 @@ -theme: jekyll-theme-midnight
\ No newline at end of file diff --git a/include/benchmark/benchmark.h b/include/benchmark/benchmark.h index 144e212..e4f3921 100644 --- a/include/benchmark/benchmark.h +++ b/include/benchmark/benchmark.h @@ -56,7 +56,8 @@ static void BM_memcpy(benchmark::State& state) { memset(src, 'x', state.range(0)); for (auto _ : state) memcpy(dst, src, state.range(0)); - state.SetBytesProcessed(state.iterations() * state.range(0)); + state.SetBytesProcessed(int64_t(state.iterations()) * + int64_t(state.range(0))); delete[] src; delete[] dst; } BENCHMARK(BM_memcpy)->Arg(8)->Arg(64)->Arg(512)->Arg(1<<10)->Arg(8<<10); @@ -121,7 +122,8 @@ template <class Q> int BM_Sequential(benchmark::State& state) { q.Wait(&v); } // actually messages, not bytes: - state.SetBytesProcessed(state.iterations() * state.range(0)); + state.SetBytesProcessed( + static_cast<int64_t>(state.iterations())*state.range(0)); } BENCHMARK_TEMPLATE(BM_Sequential, WaitQueue<int>)->Range(1<<0, 1<<10); @@ -244,11 +246,11 @@ BENCHMARK(BM_test)->Unit(benchmark::kMillisecond); #endif #if defined(__GNUC__) || __has_builtin(__builtin_unreachable) -#define BENCHMARK_UNREACHABLE() __builtin_unreachable() + #define BENCHMARK_UNREACHABLE() __builtin_unreachable() #elif defined(_MSC_VER) -#define BENCHMARK_UNREACHABLE() __assume(false) + #define BENCHMARK_UNREACHABLE() __assume(false) #else -#define BENCHMARK_UNREACHABLE() ((void)0) + #define BENCHMARK_UNREACHABLE() ((void)0) #endif namespace benchmark { @@ -368,10 +370,7 @@ class Counter { // It will be presented divided by the number of iterations. kAvgIterations = 1U << 3U, // Mark the counter as a iteration-average rate. See above. - kAvgIterationsRate = kIsRate | kAvgIterations, - - // In the end, invert the result. This is always done last! - kInvert = 1U << 31U + kAvgIterationsRate = kIsRate | kAvgIterations }; enum OneK { @@ -414,11 +413,9 @@ enum TimeUnit { kNanosecond, kMicrosecond, kMillisecond }; // calculated automatically to the best fit. enum BigO { oNone, o1, oN, oNSquared, oNCubed, oLogN, oNLogN, oAuto, oLambda }; -typedef uint64_t IterationCount; - // BigOFunc is passed to a benchmark in order to specify the asymptotic // computational complexity for the benchmark. -typedef double(BigOFunc)(IterationCount); +typedef double(BigOFunc)(int64_t); // StatisticsFunc is passed to a benchmark in order to compute some descriptive // statistics over all the measurements of some type @@ -491,7 +488,7 @@ class State { // while (state.KeepRunningBatch(1000)) { // // process 1000 elements // } - bool KeepRunningBatch(IterationCount n); + bool KeepRunningBatch(size_t n); // REQUIRES: timer is running and 'SkipWithError(...)' has not been called // by the current thread. @@ -577,7 +574,7 @@ class State { void SetComplexityN(int64_t complexity_n) { complexity_n_ = complexity_n; } BENCHMARK_ALWAYS_INLINE - int64_t complexity_length_n() const { return complexity_n_; } + int64_t complexity_length_n() { return complexity_n_; } // If this routine is called with items > 0, then an items/s // label is printed on the benchmark report line for the currently @@ -630,7 +627,7 @@ class State { int64_t range_y() const { return range(1); } BENCHMARK_ALWAYS_INLINE - IterationCount iterations() const { + size_t iterations() const { if (BENCHMARK_BUILTIN_EXPECT(!started_, false)) { return 0; } @@ -641,15 +638,15 @@ class State { : // items we expect on the first cache line (ie 64 bytes of the struct) // When total_iterations_ is 0, KeepRunning() and friends will return false. // May be larger than max_iterations. - IterationCount total_iterations_; + size_t total_iterations_; // When using KeepRunningBatch(), batch_leftover_ holds the number of // iterations beyond max_iters that were run. Used to track // completed_iterations_ accurately. - IterationCount batch_leftover_; + size_t batch_leftover_; public: - const IterationCount max_iterations; + const size_t max_iterations; private: bool started_; @@ -670,14 +667,14 @@ class State { const int threads; private: - State(IterationCount max_iters, const std::vector<int64_t>& ranges, - int thread_i, int n_threads, internal::ThreadTimer* timer, + State(size_t max_iters, const std::vector<int64_t>& ranges, int thread_i, + int n_threads, internal::ThreadTimer* timer, internal::ThreadManager* manager); void StartKeepRunning(); // Implementation of KeepRunning() and KeepRunningBatch(). // is_batch must be true unless n is 1. - bool KeepRunningInternal(IterationCount n, bool is_batch); + bool KeepRunningInternal(size_t n, bool is_batch); void FinishKeepRunning(); internal::ThreadTimer* timer_; internal::ThreadManager* manager_; @@ -689,11 +686,11 @@ inline BENCHMARK_ALWAYS_INLINE bool State::KeepRunning() { return KeepRunningInternal(1, /*is_batch=*/false); } -inline BENCHMARK_ALWAYS_INLINE bool State::KeepRunningBatch(IterationCount n) { +inline BENCHMARK_ALWAYS_INLINE bool State::KeepRunningBatch(size_t n) { return KeepRunningInternal(n, /*is_batch=*/true); } -inline BENCHMARK_ALWAYS_INLINE bool State::KeepRunningInternal(IterationCount n, +inline BENCHMARK_ALWAYS_INLINE bool State::KeepRunningInternal(size_t n, bool is_batch) { // total_iterations_ is set to 0 by the constructor, and always set to a // nonzero value by StartKepRunning(). @@ -757,7 +754,7 @@ struct State::StateIterator { } private: - IterationCount cached_; + size_t cached_; State* const parent_; }; @@ -861,7 +858,7 @@ class Benchmark { // NOTE: This function should only be used when *exact* iteration control is // needed and never to control or limit how long a benchmark runs, where // `--benchmark_min_time=N` or `MinTime(...)` should be used instead. - Benchmark* Iterations(IterationCount n); + Benchmark* Iterations(size_t n); // Specify the amount of times to repeat this benchmark. This option overrides // the `benchmark_repetitions` flag. @@ -877,18 +874,11 @@ class Benchmark { // Same as ReportAggregatesOnly(), but applies to display reporter only. Benchmark* DisplayAggregatesOnly(bool value = true); - // By default, the CPU time is measured only for the main thread, which may - // be unrepresentative if the benchmark uses threads internally. If called, - // the total CPU time spent by all the threads will be measured instead. - // By default, the only the main thread CPU time will be measured. - Benchmark* MeasureProcessCPUTime(); - - // If a particular benchmark should use the Wall clock instead of the CPU time - // (be it either the CPU time of the main thread only (default), or the - // total CPU usage of the benchmark), call this method. If called, the elapsed - // (wall) time will be used to control how many iterations are run, and in the - // printing of items/second or MB/seconds values. - // If not called, the CPU time used by the benchmark will be used. + // If a particular benchmark is I/O bound, runs multiple threads internally or + // if for some reason CPU timings are not representative, call this method. If + // called, the elapsed time will be used to control how many iterations are + // run, and in the printing of items/second or MB/seconds values. If not + // called, the cpu time used by the benchmark will be used. Benchmark* UseRealTime(); // If a benchmark must measure time manually (e.g. if GPU execution time is @@ -960,9 +950,8 @@ class Benchmark { TimeUnit time_unit_; int range_multiplier_; double min_time_; - IterationCount iterations_; + size_t iterations_; int repetitions_; - bool measure_process_cpu_time_; bool use_real_time_; bool use_manual_time_; BigO complexity_; @@ -1304,33 +1293,15 @@ struct CPUInfo { BENCHMARK_DISALLOW_COPY_AND_ASSIGN(CPUInfo); }; -// Adding Struct for System Information +//Adding Struct for System Information struct SystemInfo { std::string name; static const SystemInfo& Get(); - private: SystemInfo(); BENCHMARK_DISALLOW_COPY_AND_ASSIGN(SystemInfo); }; -// BenchmarkName contains the components of the Benchmark's name -// which allows individual fields to be modified or cleared before -// building the final name using 'str()'. -struct BenchmarkName { - std::string function_name; - std::string args; - std::string min_time; - std::string iterations; - std::string repetitions; - std::string time_type; - std::string threads; - - // Return the full name of the benchmark with each non-empty - // field separated by a '/' - std::string str() const; -}; - // Interface for custom benchmark result printers. // By default, benchmark reports are printed to stdout. However an application // can control the destination of the reports by calling @@ -1348,14 +1319,12 @@ class BenchmarkReporter { }; struct Run { - static const int64_t no_repetition_index = -1; enum RunType { RT_Iteration, RT_Aggregate }; Run() : run_type(RT_Iteration), error_occurred(false), iterations(1), - threads(1), time_unit(kNanosecond), real_accumulated_time(0), cpu_accumulated_time(0), @@ -1371,17 +1340,14 @@ class BenchmarkReporter { max_bytes_used(0) {} std::string benchmark_name() const; - BenchmarkName run_name; - RunType run_type; + std::string run_name; + RunType run_type; // is this a measurement, or an aggregate? std::string aggregate_name; std::string report_label; // Empty if not set by benchmark. bool error_occurred; std::string error_message; - IterationCount iterations; - int64_t threads; - int64_t repetition_index; - int64_t repetitions; + int64_t iterations; TimeUnit time_unit; double real_accumulated_time; double cpu_accumulated_time; @@ -1519,9 +1485,8 @@ class JSONReporter : public BenchmarkReporter { bool first_report_; }; -class BENCHMARK_DEPRECATED_MSG( - "The CSV Reporter will be removed in a future release") CSVReporter - : public BenchmarkReporter { +class BENCHMARK_DEPRECATED_MSG("The CSV Reporter will be removed in a future release") + CSVReporter : public BenchmarkReporter { public: CSVReporter() : printed_header_(false) {} virtual bool ReportContext(const Context& context); diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index eab1428..977474f 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1,5 +1,4 @@ # Allow the source files to find headers in src/ -include(GNUInstallDirs) include_directories(${PROJECT_SOURCE_DIR}/src) if (DEFINED BENCHMARK_CXX_LINKER_FLAGS) @@ -34,17 +33,9 @@ if(LIBRT) target_link_libraries(benchmark ${LIBRT}) endif() -if(CMAKE_BUILD_TYPE) - string(TOUPPER ${CMAKE_BUILD_TYPE} CMAKE_BUILD_TYPE_UPPER) -endif() -if(NOT CMAKE_THREAD_LIBS_INIT AND "${CMAKE_CXX_FLAGS} ${CMAKE_CXX_FLAGS_${CMAKE_BUILD_TYPE_UPPER}}" MATCHES ".*-fsanitize=[^ ]*address.*") - message(WARNING "CMake's FindThreads.cmake did not fail, but CMAKE_THREAD_LIBS_INIT ended up being empty. This was fixed in https://github.com/Kitware/CMake/commit/d53317130e84898c5328c237186dbd995aaf1c12 Let's guess that -pthread is sufficient.") - target_link_libraries(benchmark -pthread) -endif() - # We need extra libraries on Windows if(${CMAKE_SYSTEM_NAME} MATCHES "Windows") - target_link_libraries(benchmark shlwapi) + target_link_libraries(benchmark Shlwapi) endif() # We need extra libraries on Solaris @@ -64,6 +55,11 @@ target_include_directories(benchmark PUBLIC ) target_link_libraries(benchmark_main benchmark) +set(include_install_dir "include") +set(lib_install_dir "lib/") +set(bin_install_dir "bin/") +set(config_install_dir "lib/cmake/${PROJECT_NAME}") +set(pkgconfig_install_dir "lib/pkgconfig") set(generated_dir "${CMAKE_CURRENT_BINARY_DIR}/generated") @@ -87,26 +83,26 @@ if (BENCHMARK_ENABLE_INSTALL) install( TARGETS benchmark benchmark_main EXPORT ${targets_export_name} - ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} - LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} - RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} - INCLUDES DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}) + ARCHIVE DESTINATION ${lib_install_dir} + LIBRARY DESTINATION ${lib_install_dir} + RUNTIME DESTINATION ${bin_install_dir} + INCLUDES DESTINATION ${include_install_dir}) install( DIRECTORY "${PROJECT_SOURCE_DIR}/include/benchmark" - DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} + DESTINATION ${include_install_dir} FILES_MATCHING PATTERN "*.*h") install( FILES "${project_config}" "${version_config}" - DESTINATION "${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME}") + DESTINATION "${config_install_dir}") install( FILES "${pkg_config}" - DESTINATION "${CMAKE_INSTALL_LIBDIR}/pkgconfig") + DESTINATION "${pkgconfig_install_dir}") install( EXPORT "${targets_export_name}" NAMESPACE "${namespace}" - DESTINATION "${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME}") + DESTINATION "${config_install_dir}") endif() diff --git a/src/benchmark.cc b/src/benchmark.cc index 07942eb..aab0750 100644 --- a/src/benchmark.cc +++ b/src/benchmark.cc @@ -51,60 +51,66 @@ #include "thread_manager.h" #include "thread_timer.h" -// Print a list of benchmarks. This option overrides all other options. -DEFINE_bool(benchmark_list_tests, false); - -// A regular expression that specifies the set of benchmarks to execute. If -// this flag is empty, or if this flag is the string \"all\", all benchmarks -// linked into the binary are run. -DEFINE_string(benchmark_filter, "."); - -// Minimum number of seconds we should run benchmark before results are -// considered significant. For cpu-time based tests, this is the lower bound -// on the total cpu time used by all threads that make up the test. For -// real-time based tests, this is the lower bound on the elapsed time of the -// benchmark execution, regardless of number of threads. -DEFINE_double(benchmark_min_time, 0.5); - -// The number of runs of each benchmark. If greater than 1, the mean and -// standard deviation of the runs will be reported. -DEFINE_int32(benchmark_repetitions, 1); - -// Report the result of each benchmark repetitions. When 'true' is specified -// only the mean, standard deviation, and other statistics are reported for -// repeated benchmarks. Affects all reporters. -DEFINE_bool(benchmark_report_aggregates_only, false); - -// Display the result of each benchmark repetitions. When 'true' is specified -// only the mean, standard deviation, and other statistics are displayed for -// repeated benchmarks. Unlike benchmark_report_aggregates_only, only affects -// the display reporter, but *NOT* file reporter, which will still contain -// all the output. -DEFINE_bool(benchmark_display_aggregates_only, false); - -// The format to use for console output. -// Valid values are 'console', 'json', or 'csv'. -DEFINE_string(benchmark_format, "console"); - -// The format to use for file output. -// Valid values are 'console', 'json', or 'csv'. -DEFINE_string(benchmark_out_format, "json"); - -// The file to write additional output to. -DEFINE_string(benchmark_out, ""); - -// Whether to use colors in the output. Valid values: -// 'true'/'yes'/1, 'false'/'no'/0, and 'auto'. 'auto' means to use colors if -// the output is being sent to a terminal and the TERM environment variable is -// set to a terminal type that supports colors. -DEFINE_string(benchmark_color, "auto"); - -// Whether to use tabular format when printing user counters to the console. -// Valid values: 'true'/'yes'/1, 'false'/'no'/0. Defaults to false. -DEFINE_bool(benchmark_counters_tabular, false); - -// The level of verbose logging to output -DEFINE_int32(v, 0); +DEFINE_bool(benchmark_list_tests, false, + "Print a list of benchmarks. This option overrides all other " + "options."); + +DEFINE_string(benchmark_filter, ".", + "A regular expression that specifies the set of benchmarks " + "to execute. If this flag is empty, or if this flag is the " + "string \"all\", all benchmarks linked into the binary are " + "run."); + +DEFINE_double(benchmark_min_time, 0.5, + "Minimum number of seconds we should run benchmark before " + "results are considered significant. For cpu-time based " + "tests, this is the lower bound on the total cpu time " + "used by all threads that make up the test. For real-time " + "based tests, this is the lower bound on the elapsed time " + "of the benchmark execution, regardless of number of " + "threads."); + +DEFINE_int32(benchmark_repetitions, 1, + "The number of runs of each benchmark. If greater than 1, the " + "mean and standard deviation of the runs will be reported."); + +DEFINE_bool( + benchmark_report_aggregates_only, false, + "Report the result of each benchmark repetitions. When 'true' is specified " + "only the mean, standard deviation, and other statistics are reported for " + "repeated benchmarks. Affects all reporters."); + +DEFINE_bool( + benchmark_display_aggregates_only, false, + "Display the result of each benchmark repetitions. When 'true' is " + "specified only the mean, standard deviation, and other statistics are " + "displayed for repeated benchmarks. Unlike " + "benchmark_report_aggregates_only, only affects the display reporter, but " + "*NOT* file reporter, which will still contain all the output."); + +DEFINE_string(benchmark_format, "console", + "The format to use for console output. Valid values are " + "'console', 'json', or 'csv'."); + +DEFINE_string(benchmark_out_format, "json", + "The format to use for file output. Valid values are " + "'console', 'json', or 'csv'."); + +DEFINE_string(benchmark_out, "", "The file to write additional output to"); + +DEFINE_string(benchmark_color, "auto", + "Whether to use colors in the output. Valid values: " + "'true'/'yes'/1, 'false'/'no'/0, and 'auto'. 'auto' means to use " + "colors if the output is being sent to a terminal and the TERM " + "environment variable is set to a terminal type that supports " + "colors."); + +DEFINE_bool(benchmark_counters_tabular, false, + "Whether to use tabular format when printing user counters to " + "the console. Valid values: 'true'/'yes'/1, 'false'/'no'/0." + "Defaults to false."); + +DEFINE_int32(v, 0, "The level of verbose logging to output"); namespace benchmark { @@ -115,8 +121,8 @@ void UseCharPointer(char const volatile*) {} } // namespace internal -State::State(IterationCount max_iters, const std::vector<int64_t>& ranges, - int thread_i, int n_threads, internal::ThreadTimer* timer, +State::State(size_t max_iters, const std::vector<int64_t>& ranges, int thread_i, + int n_threads, internal::ThreadTimer* timer, internal::ThreadManager* manager) : total_iterations_(0), batch_leftover_(0), @@ -142,7 +148,7 @@ State::State(IterationCount max_iters, const std::vector<int64_t>& ranges, // which must be suppressed. #if defined(__INTEL_COMPILER) #pragma warning push -#pragma warning(disable : 1875) +#pragma warning(disable:1875) #elif defined(__GNUC__) #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Winvalid-offsetof" @@ -227,7 +233,7 @@ void RunBenchmarks(const std::vector<BenchmarkInstance>& benchmarks, size_t stat_field_width = 0; for (const BenchmarkInstance& benchmark : benchmarks) { name_field_width = - std::max<size_t>(name_field_width, benchmark.name.str().size()); + std::max<size_t>(name_field_width, benchmark.name.size()); might_have_aggregates |= benchmark.repetitions > 1; for (const auto& Stat : *benchmark.statistics) @@ -283,13 +289,6 @@ void RunBenchmarks(const std::vector<BenchmarkInstance>& benchmarks, flushStreams(file_reporter); } -// Disable deprecated warnings temporarily because we need to reference -// CSVReporter but don't want to trigger -Werror=-Wdeprecated -#ifdef __GNUC__ -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wdeprecated" -#endif - std::unique_ptr<BenchmarkReporter> CreateReporter( std::string const& name, ConsoleReporter::OutputOptions output_opts) { typedef std::unique_ptr<BenchmarkReporter> PtrType; @@ -305,10 +304,6 @@ std::unique_ptr<BenchmarkReporter> CreateReporter( } } -#ifdef __GNUC__ -#pragma GCC diagnostic pop -#endif - } // end namespace bool IsZero(double n) { @@ -317,7 +312,7 @@ bool IsZero(double n) { ConsoleReporter::OutputOptions GetOutputOptions(bool force_no_color) { int output_opts = ConsoleReporter::OO_Defaults; - auto is_benchmark_color = [force_no_color]() -> bool { + auto is_benchmark_color = [force_no_color] () -> bool { if (force_no_color) { return false; } @@ -398,8 +393,7 @@ size_t RunSpecifiedBenchmarks(BenchmarkReporter* display_reporter, } if (FLAGS_benchmark_list_tests) { - for (auto const& benchmark : benchmarks) - Out << benchmark.name.str() << "\n"; + for (auto const& benchmark : benchmarks) Out << benchmark.name << "\n"; } else { internal::RunBenchmarks(benchmarks, display_reporter, file_reporter); } diff --git a/src/benchmark_api_internal.cc b/src/benchmark_api_internal.cc index d468a25..8d31083 100644 --- a/src/benchmark_api_internal.cc +++ b/src/benchmark_api_internal.cc @@ -3,9 +3,9 @@ namespace benchmark { namespace internal { -State BenchmarkInstance::Run(IterationCount iters, int thread_id, - internal::ThreadTimer* timer, - internal::ThreadManager* manager) const { +State BenchmarkInstance::Run( + size_t iters, int thread_id, internal::ThreadTimer* timer, + internal::ThreadManager* manager) const { State st(iters, arg, thread_id, threads, timer, manager); benchmark->Run(st); return st; diff --git a/src/benchmark_api_internal.h b/src/benchmark_api_internal.h index 264eff9..0524a85 100644 --- a/src/benchmark_api_internal.h +++ b/src/benchmark_api_internal.h @@ -16,13 +16,12 @@ namespace internal { // Information kept per benchmark we may want to run struct BenchmarkInstance { - BenchmarkName name; + std::string name; Benchmark* benchmark; AggregationReportMode aggregation_report_mode; std::vector<int64_t> arg; TimeUnit time_unit; int range_multiplier; - bool measure_process_cpu_time; bool use_real_time; bool use_manual_time; BigO complexity; @@ -32,10 +31,10 @@ struct BenchmarkInstance { bool last_benchmark_instance; int repetitions; double min_time; - IterationCount iterations; + size_t iterations; int threads; // Number of concurrent threads to us - State Run(IterationCount iters, int thread_id, internal::ThreadTimer* timer, + State Run(size_t iters, int thread_id, internal::ThreadTimer* timer, internal::ThreadManager* manager) const; }; diff --git a/src/benchmark_name.cc b/src/benchmark_name.cc deleted file mode 100644 index 2a17ebc..0000000 --- a/src/benchmark_name.cc +++ /dev/null @@ -1,58 +0,0 @@ -// Copyright 2015 Google Inc. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include <benchmark/benchmark.h> - -namespace benchmark { - -namespace { - -// Compute the total size of a pack of std::strings -size_t size_impl() { return 0; } - -template <typename Head, typename... Tail> -size_t size_impl(const Head& head, const Tail&... tail) { - return head.size() + size_impl(tail...); -} - -// Join a pack of std::strings using a delimiter -// TODO: use absl::StrJoin -void join_impl(std::string&, char) {} - -template <typename Head, typename... Tail> -void join_impl(std::string& s, const char delimiter, const Head& head, - const Tail&... tail) { - if (!s.empty() && !head.empty()) { - s += delimiter; - } - - s += head; - - join_impl(s, delimiter, tail...); -} - -template <typename... Ts> -std::string join(char delimiter, const Ts&... ts) { - std::string s; - s.reserve(sizeof...(Ts) + size_impl(ts...)); - join_impl(s, delimiter, ts...); - return s; -} -} // namespace - -std::string BenchmarkName::str() const { - return join('/', function_name, args, min_time, iterations, repetitions, - time_type, threads); -} -} // namespace benchmark diff --git a/src/benchmark_register.cc b/src/benchmark_register.cc index cca39b2..f17f5b2 100644 --- a/src/benchmark_register.cc +++ b/src/benchmark_register.cc @@ -34,11 +34,6 @@ #include <sstream> #include <thread> -#ifndef __STDC_FORMAT_MACROS -#define __STDC_FORMAT_MACROS -#endif -#include <inttypes.h> - #include "benchmark/benchmark.h" #include "benchmark_api_internal.h" #include "check.h" @@ -158,7 +153,7 @@ bool BenchmarkFamilies::FindBenchmarks( for (auto const& args : family->args_) { for (int num_threads : *thread_counts) { BenchmarkInstance instance; - instance.name.function_name = family->name_; + instance.name = family->name_; instance.benchmark = family.get(); instance.aggregation_report_mode = family->aggregation_report_mode_; instance.arg = args; @@ -167,7 +162,6 @@ bool BenchmarkFamilies::FindBenchmarks( instance.min_time = family->min_time_; instance.iterations = family->iterations_; instance.repetitions = family->repetitions_; - instance.measure_process_cpu_time = family->measure_process_cpu_time_; instance.use_real_time = family->use_real_time_; instance.use_manual_time = family->use_manual_time_; instance.complexity = family->complexity_; @@ -178,57 +172,45 @@ bool BenchmarkFamilies::FindBenchmarks( // Add arguments to instance name size_t arg_i = 0; for (auto const& arg : args) { - if (!instance.name.args.empty()) { - instance.name.args += '/'; - } + instance.name += "/"; if (arg_i < family->arg_names_.size()) { const auto& arg_name = family->arg_names_[arg_i]; if (!arg_name.empty()) { - instance.name.args += StrFormat("%s:", arg_name.c_str()); + instance.name += + StrFormat("%s:", family->arg_names_[arg_i].c_str()); } } - instance.name.args += StrFormat("%" PRId64, arg); + // we know that the args are always non-negative (see 'AddRange()'), + // thus print as 'unsigned'. BUT, do a cast due to the 32-bit builds. + instance.name += StrFormat("%lu", static_cast<unsigned long>(arg)); ++arg_i; } if (!IsZero(family->min_time_)) - instance.name.min_time = - StrFormat("min_time:%0.3f", family->min_time_); + instance.name += StrFormat("/min_time:%0.3f", family->min_time_); if (family->iterations_ != 0) { - instance.name.iterations = - StrFormat("iterations:%lu", + instance.name += + StrFormat("/iterations:%lu", static_cast<unsigned long>(family->iterations_)); } if (family->repetitions_ != 0) - instance.name.repetitions = - StrFormat("repeats:%d", family->repetitions_); - - if (family->measure_process_cpu_time_) { - instance.name.time_type = "process_time"; - } + instance.name += StrFormat("/repeats:%d", family->repetitions_); if (family->use_manual_time_) { - if (!instance.name.time_type.empty()) { - instance.name.time_type += '/'; - } - instance.name.time_type += "manual_time"; + instance.name += "/manual_time"; } else if (family->use_real_time_) { - if (!instance.name.time_type.empty()) { - instance.name.time_type += '/'; - } - instance.name.time_type += "real_time"; + instance.name += "/real_time"; } // Add the number of threads used to the name if (!family->thread_counts_.empty()) { - instance.name.threads = StrFormat("threads:%d", instance.threads); + instance.name += StrFormat("/threads:%d", instance.threads); } - const auto full_name = instance.name.str(); - if ((re.Match(full_name) && !isNegativeFilter) || - (!re.Match(full_name) && isNegativeFilter)) { + if ((re.Match(instance.name) && !isNegativeFilter) || + (!re.Match(instance.name) && isNegativeFilter)) { instance.last_benchmark_instance = (&args == &family->args_.back()); benchmarks->push_back(std::move(instance)); } @@ -265,7 +247,6 @@ Benchmark::Benchmark(const char* name) min_time_(0), iterations_(0), repetitions_(0), - measure_process_cpu_time_(false), use_real_time_(false), use_manual_time_(false), complexity_(oNone), @@ -347,6 +328,7 @@ Benchmark* Benchmark::ArgNames(const std::vector<std::string>& names) { Benchmark* Benchmark::DenseRange(int64_t start, int64_t limit, int step) { CHECK(ArgsCnt() == -1 || ArgsCnt() == 1); + CHECK_GE(start, 0); CHECK_LE(start, limit); for (int64_t arg = start; arg <= limit; arg += step) { args_.push_back({arg}); @@ -378,7 +360,7 @@ Benchmark* Benchmark::MinTime(double t) { return this; } -Benchmark* Benchmark::Iterations(IterationCount n) { +Benchmark* Benchmark::Iterations(size_t n) { CHECK(n > 0); CHECK(IsZero(min_time_)); iterations_ = n; @@ -412,12 +394,6 @@ Benchmark* Benchmark::DisplayAggregatesOnly(bool value) { return this; } -Benchmark* Benchmark::MeasureProcessCPUTime() { - // Can be used together with UseRealTime() / UseManualTime(). - measure_process_cpu_time_ = true; - return this; -} - Benchmark* Benchmark::UseRealTime() { CHECK(!use_manual_time_) << "Cannot set UseRealTime and UseManualTime simultaneously."; diff --git a/src/benchmark_register.h b/src/benchmark_register.h index 61377d7..0705e21 100644 --- a/src/benchmark_register.h +++ b/src/benchmark_register.h @@ -5,103 +5,29 @@ #include "check.h" -namespace benchmark { -namespace internal { - -// Append the powers of 'mult' in the closed interval [lo, hi]. -// Returns iterator to the start of the inserted range. template <typename T> -typename std::vector<T>::iterator -AddPowers(std::vector<T>* dst, T lo, T hi, int mult) { +void AddRange(std::vector<T>* dst, T lo, T hi, int mult) { CHECK_GE(lo, 0); CHECK_GE(hi, lo); CHECK_GE(mult, 2); - const size_t start_offset = dst->size(); + // Add "lo" + dst->push_back(lo); static const T kmax = std::numeric_limits<T>::max(); - // Space out the values in multiples of "mult" - for (T i = 1; i <= hi; i *= mult) { - if (i >= lo) { + // Now space out the benchmarks in multiples of "mult" + for (T i = 1; i < kmax / mult; i *= mult) { + if (i >= hi) break; + if (i > lo) { dst->push_back(i); } - // Break the loop here since multiplying by - // 'mult' would move outside of the range of T - if (i > kmax / mult) break; - } - - return dst->begin() + start_offset; -} - -template <typename T> -void AddNegatedPowers(std::vector<T>* dst, T lo, T hi, int mult) { - // We negate lo and hi so we require that they cannot be equal to 'min'. - CHECK_GT(lo, std::numeric_limits<T>::min()); - CHECK_GT(hi, std::numeric_limits<T>::min()); - CHECK_GE(hi, lo); - CHECK_LE(hi, 0); - - // Add positive powers, then negate and reverse. - // Casts necessary since small integers get promoted - // to 'int' when negating. - const auto lo_complement = static_cast<T>(-lo); - const auto hi_complement = static_cast<T>(-hi); - - const auto it = AddPowers(dst, hi_complement, lo_complement, mult); - - std::for_each(it, dst->end(), [](T& t) { t *= -1; }); - std::reverse(it, dst->end()); -} - -template <typename T> -void AddRange(std::vector<T>* dst, T lo, T hi, int mult) { - static_assert(std::is_integral<T>::value && std::is_signed<T>::value, - "Args type must be a signed integer"); - - CHECK_GE(hi, lo); - CHECK_GE(mult, 2); - - // Add "lo" - dst->push_back(lo); - - // Handle lo == hi as a special case, so we then know - // lo < hi and so it is safe to add 1 to lo and subtract 1 - // from hi without falling outside of the range of T. - if (lo == hi) return; - - // Ensure that lo_inner <= hi_inner below. - if (lo + 1 == hi) { - dst->push_back(hi); - return; } - // Add all powers of 'mult' in the range [lo+1, hi-1] (inclusive). - const auto lo_inner = static_cast<T>(lo + 1); - const auto hi_inner = static_cast<T>(hi - 1); - - // Insert negative values - if (lo_inner < 0) { - AddNegatedPowers(dst, lo_inner, std::min(hi_inner, T{-1}), mult); - } - - // Treat 0 as a special case (see discussion on #762). - if (lo <= 0 && hi >= 0) { - dst->push_back(0); - } - - // Insert positive values - if (hi_inner > 0) { - AddPowers(dst, std::max(lo_inner, T{1}), hi_inner, mult); - } - - // Add "hi" (if different from last value). - if (hi != dst->back()) { + // Add "hi" (if different from "lo") + if (hi != lo) { dst->push_back(hi); } } -} // namespace internal -} // namespace benchmark - #endif // BENCHMARK_REGISTER_H diff --git a/src/benchmark_runner.cc b/src/benchmark_runner.cc index 0bae6a5..38faeec 100644 --- a/src/benchmark_runner.cc +++ b/src/benchmark_runner.cc @@ -59,14 +59,12 @@ MemoryManager* memory_manager = nullptr; namespace { -static constexpr IterationCount kMaxIterations = 1000000000; +static const size_t kMaxIterations = 1000000000; BenchmarkReporter::Run CreateRunReport( const benchmark::internal::BenchmarkInstance& b, - const internal::ThreadManager::Result& results, - IterationCount memory_iterations, - const MemoryManager::Result& memory_result, double seconds, - int64_t repetition_index) { + const internal::ThreadManager::Result& results, size_t memory_iterations, + const MemoryManager::Result& memory_result, double seconds) { // Create report about this benchmark run. BenchmarkReporter::Run report; @@ -77,9 +75,6 @@ BenchmarkReporter::Run CreateRunReport( // This is the total iterations across all threads. report.iterations = results.iterations; report.time_unit = b.time_unit; - report.threads = b.threads; - report.repetition_index = repetition_index; - report.repetitions = b.repetitions; if (!report.error_occurred) { if (b.use_manual_time) { @@ -110,12 +105,9 @@ BenchmarkReporter::Run CreateRunReport( // Execute one thread of benchmark b for the specified number of iterations. // Adds the stats collected for the thread into *total. -void RunInThread(const BenchmarkInstance* b, IterationCount iters, - int thread_id, ThreadManager* manager) { - internal::ThreadTimer timer( - b->measure_process_cpu_time - ? internal::ThreadTimer::CreateProcessCpuTime() - : internal::ThreadTimer::Create()); +void RunInThread(const BenchmarkInstance* b, size_t iters, int thread_id, + ThreadManager* manager) { + internal::ThreadTimer timer; State st = b->Run(iters, thread_id, &timer, manager); CHECK(st.iterations() >= st.max_iterations) << "Benchmark returned before State::KeepRunning() returned false!"; @@ -158,7 +150,8 @@ class BenchmarkRunner { } for (int repetition_num = 0; repetition_num < repeats; repetition_num++) { - DoOneRepetition(repetition_num); + const bool is_the_first_repetition = repetition_num == 0; + DoOneRepetition(is_the_first_repetition); } // Calculate additional statistics @@ -188,17 +181,17 @@ class BenchmarkRunner { std::vector<std::thread> pool; - IterationCount iters; // preserved between repetitions! + size_t iters; // preserved between repetitions! // So only the first repetition has to find/calculate it, // the other repetitions will just use that precomputed iteration count. struct IterationResults { internal::ThreadManager::Result results; - IterationCount iters; + size_t iters; double seconds; }; IterationResults DoNIterations() { - VLOG(2) << "Running " << b.name.str() << " for " << iters << "\n"; + VLOG(2) << "Running " << b.name << " for " << iters << "\n"; std::unique_ptr<internal::ThreadManager> manager; manager.reset(new internal::ThreadManager(b.threads)); @@ -230,8 +223,6 @@ class BenchmarkRunner { // Adjust real/manual time stats since they were reported per thread. i.results.real_time_used /= b.threads; i.results.manual_time_used /= b.threads; - // If we were measuring whole-process CPU usage, adjust the CPU time too. - if (b.measure_process_cpu_time) i.results.cpu_time_used /= b.threads; VLOG(2) << "Ran in " << i.results.cpu_time_used << "/" << i.results.real_time_used << "\n"; @@ -249,7 +240,7 @@ class BenchmarkRunner { return i; } - IterationCount PredictNumItersNeeded(const IterationResults& i) const { + size_t PredictNumItersNeeded(const IterationResults& i) const { // See how much iterations should be increased by. // Note: Avoid division by zero with max(seconds, 1ns). double multiplier = min_time * 1.4 / std::max(i.seconds, 1e-9); @@ -263,10 +254,10 @@ class BenchmarkRunner { if (multiplier <= 1.0) multiplier = 2.0; // So what seems to be the sufficiently-large iteration count? Round up. - const IterationCount max_next_iters = + const size_t max_next_iters = 0.5 + std::max(multiplier * i.iters, i.iters + 1.0); // But we do have *some* sanity limits though.. - const IterationCount next_iters = std::min(max_next_iters, kMaxIterations); + const size_t next_iters = std::min(max_next_iters, kMaxIterations); VLOG(3) << "Next iters: " << next_iters << ", " << multiplier << "\n"; return next_iters; // round up before conversion to integer. @@ -285,8 +276,7 @@ class BenchmarkRunner { ((i.results.real_time_used >= 5 * min_time) && !b.use_manual_time); } - void DoOneRepetition(int64_t repetition_index) { - const bool is_the_first_repetition = repetition_index == 0; + void DoOneRepetition(bool is_the_first_repetition) { IterationResults i; // We *may* be gradually increasing the length (iteration count) @@ -320,11 +310,11 @@ class BenchmarkRunner { // Oh, one last thing, we need to also produce the 'memory measurements'.. MemoryManager::Result memory_result; - IterationCount memory_iterations = 0; + size_t memory_iterations = 0; if (memory_manager != nullptr) { // Only run a few iterations to reduce the impact of one-time // allocations in benchmarks that are not properly managed. - memory_iterations = std::min<IterationCount>(16, iters); + memory_iterations = std::min<size_t>(16, iters); memory_manager->Start(); std::unique_ptr<internal::ThreadManager> manager; manager.reset(new internal::ThreadManager(1)); @@ -336,9 +326,8 @@ class BenchmarkRunner { } // Ok, now actualy report. - BenchmarkReporter::Run report = - CreateRunReport(b, i.results, memory_iterations, memory_result, - i.seconds, repetition_index); + BenchmarkReporter::Run report = CreateRunReport( + b, i.results, memory_iterations, memory_result, i.seconds); if (!report.error_occurred && b.complexity != oNone) complexity_reports.push_back(report); diff --git a/src/commandlineflags.cc b/src/commandlineflags.cc index 4e60f0b..734e88b 100644 --- a/src/commandlineflags.cc +++ b/src/commandlineflags.cc @@ -14,7 +14,6 @@ #include "commandlineflags.h" -#include <algorithm> #include <cctype> #include <cstdlib> #include <cstring> @@ -22,8 +21,6 @@ #include <limits> namespace benchmark { -namespace { - // Parses 'str' for a 32-bit signed integer. If successful, writes // the result to *value and returns true; otherwise leaves *value // unchanged and returns false. @@ -91,42 +88,44 @@ static std::string FlagToEnvVar(const char* flag) { return "BENCHMARK_" + env_var; } -} // namespace - -bool BoolFromEnv(const char* flag, bool default_val) { +// Reads and returns the Boolean environment variable corresponding to +// the given flag; if it's not set, returns default_value. +// +// The value is considered true iff it's not "0". +bool BoolFromEnv(const char* flag, bool default_value) { const std::string env_var = FlagToEnvVar(flag); - const char* const value_str = getenv(env_var.c_str()); - return value_str == nullptr ? default_val : IsTruthyFlagValue(value_str); + const char* const string_value = getenv(env_var.c_str()); + return string_value == nullptr ? default_value + : strcmp(string_value, "0") != 0; } -int32_t Int32FromEnv(const char* flag, int32_t default_val) { +// Reads and returns a 32-bit integer stored in the environment +// variable corresponding to the given flag; if it isn't set or +// doesn't represent a valid 32-bit integer, returns default_value. +int32_t Int32FromEnv(const char* flag, int32_t default_value) { const std::string env_var = FlagToEnvVar(flag); - const char* const value_str = getenv(env_var.c_str()); - int32_t value = default_val; - if (value_str == nullptr || - !ParseInt32(std::string("Environment variable ") + env_var, value_str, - &value)) { - return default_val; + const char* const string_value = getenv(env_var.c_str()); + if (string_value == nullptr) { + // The environment variable is not set. + return default_value; } - return value; -} -double DoubleFromEnv(const char* flag, double default_val) { - const std::string env_var = FlagToEnvVar(flag); - const char* const value_str = getenv(env_var.c_str()); - double value = default_val; - if (value_str == nullptr || - !ParseDouble(std::string("Environment variable ") + env_var, value_str, - &value)) { - return default_val; + int32_t result = default_value; + if (!ParseInt32(std::string("Environment variable ") + env_var, string_value, + &result)) { + std::cout << "The default value " << default_value << " is used.\n"; + return default_value; } - return value; + + return result; } -const char* StringFromEnv(const char* flag, const char* default_val) { +// Reads and returns the string environment variable corresponding to +// the given flag; if it's not set, returns default_value. +const char* StringFromEnv(const char* flag, const char* default_value) { const std::string env_var = FlagToEnvVar(flag); const char* const value = getenv(env_var.c_str()); - return value == nullptr ? default_val : value; + return value == nullptr ? default_value : value; } // Parses a string as a command line flag. The string should have @@ -211,18 +210,9 @@ bool IsFlag(const char* str, const char* flag) { } bool IsTruthyFlagValue(const std::string& value) { - if (value.size() == 1) { - char v = value[0]; - return isalnum(v) && - !(v == '0' || v == 'f' || v == 'F' || v == 'n' || v == 'N'); - } else if (!value.empty()) { - std::string value_lower(value); - std::transform(value_lower.begin(), value_lower.end(), - value_lower.begin(), ::tolower); - return !(value_lower == "false" || value_lower == "no" || - value_lower == "off"); - } else - return true; + if (value.empty()) return true; + char ch = value[0]; + return isalnum(ch) && + !(ch == '0' || ch == 'f' || ch == 'F' || ch == 'n' || ch == 'N'); } - } // end namespace benchmark diff --git a/src/commandlineflags.h b/src/commandlineflags.h index 3a1f6a8..945c9a9 100644 --- a/src/commandlineflags.h +++ b/src/commandlineflags.h @@ -10,51 +10,29 @@ // Macros for declaring flags. #define DECLARE_bool(name) extern bool FLAG(name) #define DECLARE_int32(name) extern int32_t FLAG(name) +#define DECLARE_int64(name) extern int64_t FLAG(name) #define DECLARE_double(name) extern double FLAG(name) #define DECLARE_string(name) extern std::string FLAG(name) // Macros for defining flags. -#define DEFINE_bool(name, default_val) \ - bool FLAG(name) = \ - benchmark::BoolFromEnv(#name, default_val) -#define DEFINE_int32(name, default_val) \ - int32_t FLAG(name) = \ - benchmark::Int32FromEnv(#name, default_val) -#define DEFINE_double(name, default_val) \ - double FLAG(name) = \ - benchmark::DoubleFromEnv(#name, default_val) -#define DEFINE_string(name, default_val) \ - std::string FLAG(name) = \ - benchmark::StringFromEnv(#name, default_val) +#define DEFINE_bool(name, default_val, doc) bool FLAG(name) = (default_val) +#define DEFINE_int32(name, default_val, doc) int32_t FLAG(name) = (default_val) +#define DEFINE_int64(name, default_val, doc) int64_t FLAG(name) = (default_val) +#define DEFINE_double(name, default_val, doc) double FLAG(name) = (default_val) +#define DEFINE_string(name, default_val, doc) \ + std::string FLAG(name) = (default_val) namespace benchmark { +// Parses 'str' for a 32-bit signed integer. If successful, writes the result +// to *value and returns true; otherwise leaves *value unchanged and returns +// false. +bool ParseInt32(const std::string& src_text, const char* str, int32_t* value); -// Parses a bool from the environment variable -// corresponding to the given flag. -// -// If the variable exists, returns IsTruthyFlagValue() value; if not, -// returns the given default value. +// Parses a bool/Int32/string from the environment variable +// corresponding to the given Google Test flag. bool BoolFromEnv(const char* flag, bool default_val); - -// Parses an Int32 from the environment variable -// corresponding to the given flag. -// -// If the variable exists, returns ParseInt32() value; if not, returns -// the given default value. int32_t Int32FromEnv(const char* flag, int32_t default_val); - -// Parses an Double from the environment variable -// corresponding to the given flag. -// -// If the variable exists, returns ParseDouble(); if not, returns -// the given default value. double DoubleFromEnv(const char* flag, double default_val); - -// Parses a string from the environment variable -// corresponding to the given flag. -// -// If variable exists, returns its value; if not, returns -// the given default value. const char* StringFromEnv(const char* flag, const char* default_val); // Parses a string for a bool flag, in the form of either @@ -93,11 +71,9 @@ bool ParseStringFlag(const char* str, const char* flag, std::string* value); bool IsFlag(const char* str, const char* flag); // Returns true unless value starts with one of: '0', 'f', 'F', 'n' or 'N', or -// some non-alphanumeric character. Also returns false if the value matches -// one of 'no', 'false', 'off' (case-insensitive). As a special case, also -// returns true if value is the empty string. +// some non-alphanumeric character. As a special case, also returns true if +// value is the empty string. bool IsTruthyFlagValue(const std::string& value); - } // end namespace benchmark #endif // BENCHMARK_COMMANDLINEFLAGS_H_ diff --git a/src/complexity.cc b/src/complexity.cc index aeed67f..6ef1766 100644 --- a/src/complexity.cc +++ b/src/complexity.cc @@ -29,23 +29,20 @@ BigOFunc* FittingCurve(BigO complexity) { static const double kLog2E = 1.44269504088896340736; switch (complexity) { case oN: - return [](IterationCount n) -> double { return static_cast<double>(n); }; + return [](int64_t n) -> double { return static_cast<double>(n); }; case oNSquared: - return [](IterationCount n) -> double { return std::pow(n, 2); }; + return [](int64_t n) -> double { return std::pow(n, 2); }; case oNCubed: - return [](IterationCount n) -> double { return std::pow(n, 3); }; + return [](int64_t n) -> double { return std::pow(n, 3); }; case oLogN: /* Note: can't use log2 because Android's GNU STL lacks it */ - return - [](IterationCount n) { return kLog2E * log(static_cast<double>(n)); }; + return [](int64_t n) { return kLog2E * log(static_cast<double>(n)); }; case oNLogN: /* Note: can't use log2 because Android's GNU STL lacks it */ - return [](IterationCount n) { - return kLog2E * n * log(static_cast<double>(n)); - }; + return [](int64_t n) { return kLog2E * n * log(static_cast<double>(n)); }; case o1: default: - return [](IterationCount) { return 1.0; }; + return [](int64_t) { return 1.0; }; } } @@ -186,19 +183,14 @@ std::vector<BenchmarkReporter::Run> ComputeBigO( result_real = MinimalLeastSq(n, real_time, result_cpu.complexity); } - // Drop the 'args' when reporting complexity. - auto run_name = reports[0].run_name; - run_name.args.clear(); + std::string run_name = reports[0].benchmark_name().substr( + 0, reports[0].benchmark_name().find('/')); // Get the data from the accumulator to BenchmarkReporter::Run's. Run big_o; big_o.run_name = run_name; big_o.run_type = BenchmarkReporter::Run::RT_Aggregate; - big_o.repetitions = reports[0].repetitions; - big_o.repetition_index = Run::no_repetition_index; - big_o.threads = reports[0].threads; big_o.aggregate_name = "BigO"; - big_o.report_label = reports[0].report_label; big_o.iterations = 0; big_o.real_accumulated_time = result_real.coef; big_o.cpu_accumulated_time = result_cpu.coef; @@ -215,13 +207,11 @@ std::vector<BenchmarkReporter::Run> ComputeBigO( // Only add label to mean/stddev if it is same for all runs Run rms; rms.run_name = run_name; + big_o.report_label = reports[0].report_label; rms.run_type = BenchmarkReporter::Run::RT_Aggregate; rms.aggregate_name = "RMS"; rms.report_label = big_o.report_label; rms.iterations = 0; - rms.repetition_index = Run::no_repetition_index; - rms.repetitions = reports[0].repetitions; - rms.threads = reports[0].threads; rms.real_accumulated_time = result_real.rms / multiplier; rms.cpu_accumulated_time = result_cpu.rms / multiplier; rms.report_rms = true; diff --git a/src/console_reporter.cc b/src/console_reporter.cc index 6fd7645..cc8ae27 100644 --- a/src/console_reporter.cc +++ b/src/console_reporter.cc @@ -12,21 +12,21 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include "benchmark/benchmark.h" +#include "complexity.h" +#include "counter.h" + #include <algorithm> #include <cstdint> #include <cstdio> -#include <cstring> #include <iostream> #include <string> #include <tuple> #include <vector> -#include "benchmark/benchmark.h" #include "check.h" #include "colorprint.h" #include "commandlineflags.h" -#include "complexity.h" -#include "counter.h" #include "internal_macros.h" #include "string_util.h" #include "timers.h" @@ -156,14 +156,16 @@ void ConsoleReporter::PrintRunData(const Run& result) { const std::size_t cNameLen = std::max(std::string::size_type(10), c.first.length()); auto const& s = HumanReadableNumber(c.second.value, c.second.oneK); - const char* unit = ""; - if (c.second.flags & Counter::kIsRate) - unit = (c.second.flags & Counter::kInvert) ? "s" : "/s"; if (output_options_ & OO_Tabular) { - printer(Out, COLOR_DEFAULT, " %*s%s", cNameLen - strlen(unit), s.c_str(), - unit); + if (c.second.flags & Counter::kIsRate) { + printer(Out, COLOR_DEFAULT, " %*s/s", cNameLen - 2, s.c_str()); + } else { + printer(Out, COLOR_DEFAULT, " %*s", cNameLen, s.c_str()); + } } else { - printer(Out, COLOR_DEFAULT, " %s=%s%s", c.first.c_str(), s.c_str(), unit); + const char* unit = (c.second.flags & Counter::kIsRate) ? "/s" : ""; + printer(Out, COLOR_DEFAULT, " %s=%s%s", c.first.c_str(), s.c_str(), + unit); } } diff --git a/src/counter.cc b/src/counter.cc index cf5b78e..cb604e0 100644 --- a/src/counter.cc +++ b/src/counter.cc @@ -17,7 +17,7 @@ namespace benchmark { namespace internal { -double Finish(Counter const& c, IterationCount iterations, double cpu_time, +double Finish(Counter const& c, int64_t iterations, double cpu_time, double num_threads) { double v = c.value; if (c.flags & Counter::kIsRate) { @@ -32,15 +32,10 @@ double Finish(Counter const& c, IterationCount iterations, double cpu_time, if (c.flags & Counter::kAvgIterations) { v /= iterations; } - - if (c.flags & Counter::kInvert) { // Invert is *always* last. - v = 1.0 / v; - } return v; } -void Finish(UserCounters* l, IterationCount iterations, double cpu_time, - double num_threads) { +void Finish(UserCounters* l, int64_t iterations, double cpu_time, double num_threads) { for (auto& c : *l) { c.second.value = Finish(c.second, iterations, cpu_time, num_threads); } diff --git a/src/counter.h b/src/counter.h index 1ad46d4..d884e50 100644 --- a/src/counter.h +++ b/src/counter.h @@ -18,8 +18,7 @@ namespace benchmark { // these counter-related functions are hidden to reduce API surface. namespace internal { -void Finish(UserCounters* l, IterationCount iterations, double time, - double num_threads); +void Finish(UserCounters* l, int64_t iterations, double time, double num_threads); void Increment(UserCounters* l, UserCounters const& r); bool SameNames(UserCounters const& l, UserCounters const& r); } // end namespace internal diff --git a/src/csv_reporter.cc b/src/csv_reporter.cc index af2c18f..d2f1d27 100644 --- a/src/csv_reporter.cc +++ b/src/csv_reporter.cc @@ -37,18 +37,6 @@ std::vector<std::string> elements = { "error_occurred", "error_message"}; } // namespace -std::string CsvEscape(const std::string & s) { - std::string tmp; - tmp.reserve(s.size() + 2); - for (char c : s) { - switch (c) { - case '"' : tmp += "\"\""; break; - default : tmp += c; break; - } - } - return '"' + tmp + '"'; -} - bool CSVReporter::ReportContext(const Context& context) { PrintBasicContext(&GetErrorStream(), context); return true; @@ -101,11 +89,18 @@ void CSVReporter::ReportRuns(const std::vector<Run>& reports) { void CSVReporter::PrintRunData(const Run& run) { std::ostream& Out = GetOutputStream(); - Out << CsvEscape(run.benchmark_name()) << ","; + + // Field with embedded double-quote characters must be doubled and the field + // delimited with double-quotes. + std::string name = run.benchmark_name(); + ReplaceAll(&name, "\"", "\"\""); + Out << '"' << name << "\","; if (run.error_occurred) { Out << std::string(elements.size() - 3, ','); Out << "true,"; - Out << CsvEscape(run.error_message) << "\n"; + std::string msg = run.error_message; + ReplaceAll(&msg, "\"", "\"\""); + Out << '"' << msg << "\"\n"; return; } @@ -135,7 +130,11 @@ void CSVReporter::PrintRunData(const Run& run) { } Out << ","; if (!run.report_label.empty()) { - Out << CsvEscape(run.report_label); + // Field with embedded double-quote characters must be doubled and the field + // delimited with double-quotes. + std::string label = run.report_label; + ReplaceAll(&label, "\"", "\"\""); + Out << "\"" << label << "\""; } Out << ",,"; // for error_occurred and error_message diff --git a/src/cycleclock.h b/src/cycleclock.h index d5d62c4..f5e37b0 100644 --- a/src/cycleclock.h +++ b/src/cycleclock.h @@ -164,21 +164,6 @@ inline BENCHMARK_ALWAYS_INLINE int64_t Now() { uint64_t tsc; asm("stck %0" : "=Q"(tsc) : : "cc"); return tsc; -#elif defined(__riscv) // RISC-V - // Use RDCYCLE (and RDCYCLEH on riscv32) -#if __riscv_xlen == 32 - uint64_t cycles_low, cycles_hi0, cycles_hi1; - asm("rdcycleh %0" : "=r"(cycles_hi0)); - asm("rdcycle %0" : "=r"(cycles_lo)); - asm("rdcycleh %0" : "=r"(cycles_hi1)); - // This matches the PowerPC overflow detection, above - cycles_lo &= -static_cast<int64_t>(cycles_hi0 == cycles_hi1); - return (cycles_hi1 << 32) | cycles_lo; -#else - uint64_t cycles; - asm("rdcycle %0" : "=r"(cycles)); - return cycles; -#endif #else // The soft failover to a generic implementation is automatic only for ARM. // For other platforms the developer is expected to make an attempt to create diff --git a/src/json_reporter.cc b/src/json_reporter.cc index fe7b1be..7d01e8e 100644 --- a/src/json_reporter.cc +++ b/src/json_reporter.cc @@ -16,7 +16,6 @@ #include "complexity.h" #include <algorithm> -#include <cmath> #include <cstdint> #include <iomanip> // for setprecision #include <iostream> @@ -32,63 +31,32 @@ namespace benchmark { namespace { -std::string StrEscape(const std::string & s) { - std::string tmp; - tmp.reserve(s.size()); - for (char c : s) { - switch (c) { - case '\b': tmp += "\\b"; break; - case '\f': tmp += "\\f"; break; - case '\n': tmp += "\\n"; break; - case '\r': tmp += "\\r"; break; - case '\t': tmp += "\\t"; break; - case '\\': tmp += "\\\\"; break; - case '"' : tmp += "\\\""; break; - default : tmp += c; break; - } - } - return tmp; -} - std::string FormatKV(std::string const& key, std::string const& value) { - return StrFormat("\"%s\": \"%s\"", StrEscape(key).c_str(), StrEscape(value).c_str()); + return StrFormat("\"%s\": \"%s\"", key.c_str(), value.c_str()); } std::string FormatKV(std::string const& key, const char* value) { - return StrFormat("\"%s\": \"%s\"", StrEscape(key).c_str(), StrEscape(value).c_str()); + return StrFormat("\"%s\": \"%s\"", key.c_str(), value); } std::string FormatKV(std::string const& key, bool value) { - return StrFormat("\"%s\": %s", StrEscape(key).c_str(), value ? "true" : "false"); + return StrFormat("\"%s\": %s", key.c_str(), value ? "true" : "false"); } std::string FormatKV(std::string const& key, int64_t value) { std::stringstream ss; - ss << '"' << StrEscape(key) << "\": " << value; - return ss.str(); -} - -std::string FormatKV(std::string const& key, IterationCount value) { - std::stringstream ss; - ss << '"' << StrEscape(key) << "\": " << value; + ss << '"' << key << "\": " << value; return ss.str(); } std::string FormatKV(std::string const& key, double value) { std::stringstream ss; - ss << '"' << StrEscape(key) << "\": "; - - if (std::isnan(value)) - ss << (value < 0 ? "-" : "") << "NaN"; - else if (std::isinf(value)) - ss << (value < 0 ? "-" : "") << "Infinity"; - else { - const auto max_digits10 = - std::numeric_limits<decltype(value)>::max_digits10; - const auto max_fractional_digits10 = max_digits10 - 1; - ss << std::scientific << std::setprecision(max_fractional_digits10) - << value; - } + ss << '"' << key << "\": "; + + const auto max_digits10 = std::numeric_limits<decltype(value)>::max_digits10; + const auto max_fractional_digits10 = max_digits10 - 1; + + ss << std::scientific << std::setprecision(max_fractional_digits10) << value; return ss.str(); } @@ -112,7 +80,12 @@ bool JSONReporter::ReportContext(const Context& context) { out << indent << FormatKV("host_name", context.sys_info.name) << ",\n"; if (Context::executable_name) { - out << indent << FormatKV("executable", Context::executable_name) << ",\n"; + // windows uses backslash for its path separator, + // which must be escaped in JSON otherwise it blows up conforming JSON + // decoders + std::string executable_name = Context::executable_name; + ReplaceAll(&executable_name, "\\", "\\\\"); + out << indent << FormatKV("executable", executable_name) << ",\n"; } CPUInfo const& info = context.cpu_info; @@ -135,7 +108,7 @@ bool JSONReporter::ReportContext(const Context& context) { out << cache_indent << FormatKV("level", static_cast<int64_t>(CI.level)) << ",\n"; out << cache_indent - << FormatKV("size", static_cast<int64_t>(CI.size)) << ",\n"; + << FormatKV("size", static_cast<int64_t>(CI.size) * 1000u) << ",\n"; out << cache_indent << FormatKV("num_sharing", static_cast<int64_t>(CI.num_sharing)) << "\n"; @@ -195,7 +168,7 @@ void JSONReporter::PrintRunData(Run const& run) { std::string indent(6, ' '); std::ostream& out = GetOutputStream(); out << indent << FormatKV("name", run.benchmark_name()) << ",\n"; - out << indent << FormatKV("run_name", run.run_name.str()) << ",\n"; + out << indent << FormatKV("run_name", run.run_name) << ",\n"; out << indent << FormatKV("run_type", [&run]() -> const char* { switch (run.run_type) { case BenchmarkReporter::Run::RT_Iteration: @@ -205,12 +178,6 @@ void JSONReporter::PrintRunData(Run const& run) { } BENCHMARK_UNREACHABLE(); }()) << ",\n"; - out << indent << FormatKV("repetitions", run.repetitions) << ",\n"; - if (run.run_type != BenchmarkReporter::Run::RT_Aggregate) { - out << indent << FormatKV("repetition_index", run.repetition_index) - << ",\n"; - } - out << indent << FormatKV("threads", run.threads) << ",\n"; if (run.run_type == BenchmarkReporter::Run::RT_Aggregate) { out << indent << FormatKV("aggregate_name", run.aggregate_name) << ",\n"; } diff --git a/src/reporter.cc b/src/reporter.cc index 4d3e477..59bc5f7 100644 --- a/src/reporter.cc +++ b/src/reporter.cc @@ -83,7 +83,7 @@ BenchmarkReporter::Context::Context() : cpu_info(CPUInfo::Get()), sys_info(SystemInfo::Get()) {} std::string BenchmarkReporter::Run::benchmark_name() const { - std::string name = run_name.str(); + std::string name = run_name; if (run_type == RT_Aggregate) { name += "_" + aggregate_name; } diff --git a/src/statistics.cc b/src/statistics.cc index bd5a3d6..e821aec 100644 --- a/src/statistics.cc +++ b/src/statistics.cc @@ -97,7 +97,7 @@ std::vector<BenchmarkReporter::Run> ComputeStats( // All repetitions should be run with the same number of iterations so we // can take this information from the first benchmark. - const IterationCount run_iterations = reports.front().iterations; + int64_t const run_iterations = reports.front().iterations; // create stats for user counters struct CounterStat { Counter c; @@ -147,11 +147,8 @@ std::vector<BenchmarkReporter::Run> ComputeStats( for (const auto& Stat : *reports[0].statistics) { // Get the data from the accumulator to BenchmarkReporter::Run's. Run data; - data.run_name = reports[0].run_name; + data.run_name = reports[0].benchmark_name(); data.run_type = BenchmarkReporter::Run::RT_Aggregate; - data.threads = reports[0].threads; - data.repetitions = reports[0].repetitions; - data.repetition_index = Run::no_repetition_index; data.aggregate_name = Stat.name_; data.report_label = report_label; diff --git a/src/string_util.cc b/src/string_util.cc index 39b01a1..05ac5b4 100644 --- a/src/string_util.cc +++ b/src/string_util.cc @@ -160,6 +160,15 @@ std::string StrFormat(const char* format, ...) { return tmp; } +void ReplaceAll(std::string* str, const std::string& from, + const std::string& to) { + std::size_t start = 0; + while ((start = str->find(from, start)) != std::string::npos) { + str->replace(start, from.length(), to); + start += to.length(); + } +} + #ifdef BENCHMARK_STL_ANDROID_GNUSTL /* * GNU STL in Android NDK lacks support for some C++11 functions, including diff --git a/src/string_util.h b/src/string_util.h index 09d7b4b..fc5f8b0 100644 --- a/src/string_util.h +++ b/src/string_util.h @@ -12,9 +12,7 @@ void AppendHumanReadable(int n, std::string* str); std::string HumanReadableNumber(double n, double one_k = 1024.0); -#if defined(__MINGW32__) -__attribute__((format(__MINGW_PRINTF_FORMAT, 1, 2))) -#elif defined(__GNUC__) +#ifdef __GNUC__ __attribute__((format(printf, 1, 2))) #endif std::string @@ -37,6 +35,9 @@ inline std::string StrCat(Args&&... args) { return ss.str(); } +void ReplaceAll(std::string* str, const std::string& from, + const std::string& to); + #ifdef BENCHMARK_STL_ANDROID_GNUSTL /* * GNU STL in Android NDK lacks support for some C++11 functions, including diff --git a/src/sysinfo.cc b/src/sysinfo.cc index b5f99c7..953c170 100644 --- a/src/sysinfo.cc +++ b/src/sysinfo.cc @@ -429,20 +429,11 @@ std::string GetSystemName() { #endif return str; #else // defined(BENCHMARK_OS_WINDOWS) -#ifndef HOST_NAME_MAX -#ifdef BENCHMARK_HAS_SYSCTL // BSD/Mac Doesnt have HOST_NAME_MAX defined -#define HOST_NAME_MAX 64 -#elif defined(BENCHMARK_OS_NACL) +#ifdef BENCHMARK_OS_MACOSX //Mac Doesnt have HOST_NAME_MAX defined #define HOST_NAME_MAX 64 #elif defined(BENCHMARK_OS_QNX) #define HOST_NAME_MAX 154 -#elif defined(BENCHMARK_OS_RTEMS) -#define HOST_NAME_MAX 256 -#else -#warning "HOST_NAME_MAX not defined. using 64" -#define HOST_NAME_MAX 64 #endif -#endif // def HOST_NAME_MAX char hostname[HOST_NAME_MAX]; int retVal = gethostname(hostname, HOST_NAME_MAX); if (retVal != 0) return std::string(""); @@ -667,9 +658,9 @@ double GetCPUCyclesPerSecond() { } std::vector<double> GetLoadAvg() { -#if (defined BENCHMARK_OS_FREEBSD || defined(BENCHMARK_OS_LINUX) || \ +#if defined BENCHMARK_OS_FREEBSD || defined(BENCHMARK_OS_LINUX) || \ defined BENCHMARK_OS_MACOSX || defined BENCHMARK_OS_NETBSD || \ - defined BENCHMARK_OS_OPENBSD) && !defined(__ANDROID__) + defined BENCHMARK_OS_OPENBSD constexpr int kMaxSamples = 3; std::vector<double> res(kMaxSamples, 0.0); const int nelem = getloadavg(res.data(), kMaxSamples); diff --git a/src/thread_manager.h b/src/thread_manager.h index 28e2dd5..6e274c7 100644 --- a/src/thread_manager.h +++ b/src/thread_manager.h @@ -11,7 +11,7 @@ namespace internal { class ThreadManager { public: - explicit ThreadManager(int num_threads) + ThreadManager(int num_threads) : alive_threads_(num_threads), start_stop_barrier_(num_threads) {} Mutex& GetBenchmarkMutex() const RETURN_CAPABILITY(benchmark_mutex_) { @@ -38,7 +38,7 @@ class ThreadManager { public: struct Result { - IterationCount iterations = 0; + int64_t iterations = 0; double real_time_used = 0; double cpu_time_used = 0; double manual_time_used = 0; diff --git a/src/thread_timer.h b/src/thread_timer.h index 1703ca0..eaf108e 100644 --- a/src/thread_timer.h +++ b/src/thread_timer.h @@ -8,22 +8,14 @@ namespace benchmark { namespace internal { class ThreadTimer { - explicit ThreadTimer(bool measure_process_cpu_time_) - : measure_process_cpu_time(measure_process_cpu_time_) {} - public: - static ThreadTimer Create() { - return ThreadTimer(/*measure_process_cpu_time_=*/false); - } - static ThreadTimer CreateProcessCpuTime() { - return ThreadTimer(/*measure_process_cpu_time_=*/true); - } + ThreadTimer() = default; // Called by each thread void StartTimer() { running_ = true; start_real_time_ = ChronoClockNow(); - start_cpu_time_ = ReadCpuTimerOfChoice(); + start_cpu_time_ = ThreadCPUUsage(); } // Called by each thread @@ -33,8 +25,7 @@ class ThreadTimer { real_time_used_ += ChronoClockNow() - start_real_time_; // Floating point error can result in the subtraction producing a negative // time. Guard against that. - cpu_time_used_ += - std::max<double>(ReadCpuTimerOfChoice() - start_cpu_time_, 0); + cpu_time_used_ += std::max<double>(ThreadCPUUsage() - start_cpu_time_, 0); } // Called by each thread @@ -43,32 +34,24 @@ class ThreadTimer { bool running() const { return running_; } // REQUIRES: timer is not running - double real_time_used() const { + double real_time_used() { CHECK(!running_); return real_time_used_; } // REQUIRES: timer is not running - double cpu_time_used() const { + double cpu_time_used() { CHECK(!running_); return cpu_time_used_; } // REQUIRES: timer is not running - double manual_time_used() const { + double manual_time_used() { CHECK(!running_); return manual_time_used_; } private: - double ReadCpuTimerOfChoice() const { - if (measure_process_cpu_time) return ProcessCPUUsage(); - return ThreadCPUUsage(); - } - - // should the thread, or the process, time be measured? - const bool measure_process_cpu_time; - bool running_ = false; // Is the timer running double start_real_time_ = 0; // If running_ double start_cpu_time_ = 0; // If running_ @@ -5,7 +5,7 @@ TEST_COPTS = [ "-Wall", "-Wextra", "-Wshadow", - # "-Wshorten-64-to-32", +# "-Wshorten-64-to-32", "-Wfloat-equal", "-fstrict-aliasing", ] @@ -16,14 +16,13 @@ PER_SRC_COPTS = ({ "donotoptimize_test.cc": ["-O3"], }) + TEST_ARGS = ["--benchmark_min_time=0.01"] PER_SRC_TEST_ARGS = ({ "user_counters_tabular_test.cc": ["--benchmark_counters_tabular=true"], }) -load("@rules_cc//cc:defs.bzl", "cc_library", "cc_test") - cc_library( name = "output_test_helper", testonly = 1, @@ -37,31 +36,24 @@ cc_library( ) [ - cc_test( - name = test_src[:-len(".cc")], - size = "small", - srcs = [test_src], - args = TEST_ARGS + PER_SRC_TEST_ARGS.get(test_src, []), - copts = TEST_COPTS + PER_SRC_COPTS.get(test_src, []), - deps = [ - ":output_test_helper", - "//:benchmark", - "//:benchmark_internal_headers", - "@com_google_googletest//:gtest", - ] + ( - ["@com_google_googletest//:gtest_main"] if (test_src[-len("gtest.cc"):] == "gtest.cc") else [] - ), - # FIXME: Add support for assembly tests to bazel. - # See Issue #556 - # https://github.com/google/benchmark/issues/556 - ) - for test_src in glob( - ["*test.cc"], - exclude = [ - "*_assembly_test.cc", - "link_main_test.cc", - ], - ) + cc_test( + name = test_src[:-len(".cc")], + size = "small", + srcs = [test_src], + args = TEST_ARGS + PER_SRC_TEST_ARGS.get(test_src, []), + copts = TEST_COPTS + PER_SRC_COPTS.get(test_src, []), + deps = [ + ":output_test_helper", + "//:benchmark", + "//:benchmark_internal_headers", + "@com_google_googletest//:gtest", + ] + ( + ["@com_google_googletest//:gtest_main"] if (test_src[-len("gtest.cc"):] == "gtest.cc") else [] + ), + # FIXME: Add support for assembly tests to bazel. + # See Issue #556 + # https://github.com/google/benchmark/issues/556 + ) for test_src in glob(["*test.cc"], exclude = ["*_assembly_test.cc", "link_main_test.cc"]) ] cc_test( diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index ddcb1a1..f15ce20 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -54,12 +54,12 @@ endmacro(compile_output_test) # Demonstration executable compile_benchmark_test(benchmark_test) -add_test(NAME benchmark COMMAND benchmark_test --benchmark_min_time=0.01) +add_test(benchmark benchmark_test --benchmark_min_time=0.01) compile_benchmark_test(filter_test) macro(add_filter_test name filter expect) - add_test(NAME ${name} COMMAND filter_test --benchmark_min_time=0.01 --benchmark_filter=${filter} ${expect}) - add_test(NAME ${name}_list_only COMMAND filter_test --benchmark_list_tests --benchmark_filter=${filter} ${expect}) + add_test(${name} filter_test --benchmark_min_time=0.01 --benchmark_filter=${filter} ${expect}) + add_test(${name}_list_only filter_test --benchmark_list_tests --benchmark_filter=${filter} ${expect}) endmacro(add_filter_test) add_filter_test(filter_simple "Foo" 3) @@ -82,16 +82,16 @@ add_filter_test(filter_regex_end ".*Ba$" 1) add_filter_test(filter_regex_end_negative "-.*Ba$" 4) compile_benchmark_test(options_test) -add_test(NAME options_benchmarks COMMAND options_test --benchmark_min_time=0.01) +add_test(options_benchmarks options_test --benchmark_min_time=0.01) compile_benchmark_test(basic_test) -add_test(NAME basic_benchmark COMMAND basic_test --benchmark_min_time=0.01) +add_test(basic_benchmark basic_test --benchmark_min_time=0.01) compile_benchmark_test(diagnostics_test) -add_test(NAME diagnostics_test COMMAND diagnostics_test --benchmark_min_time=0.01) +add_test(diagnostics_test diagnostics_test --benchmark_min_time=0.01) compile_benchmark_test(skip_with_error_test) -add_test(NAME skip_with_error_test COMMAND skip_with_error_test --benchmark_min_time=0.01) +add_test(skip_with_error_test skip_with_error_test --benchmark_min_time=0.01) compile_benchmark_test(donotoptimize_test) # Some of the issues with DoNotOptimize only occur when optimization is enabled @@ -99,49 +99,46 @@ check_cxx_compiler_flag(-O3 BENCHMARK_HAS_O3_FLAG) if (BENCHMARK_HAS_O3_FLAG) set_target_properties(donotoptimize_test PROPERTIES COMPILE_FLAGS "-O3") endif() -add_test(NAME donotoptimize_test COMMAND donotoptimize_test --benchmark_min_time=0.01) +add_test(donotoptimize_test donotoptimize_test --benchmark_min_time=0.01) compile_benchmark_test(fixture_test) -add_test(NAME fixture_test COMMAND fixture_test --benchmark_min_time=0.01) +add_test(fixture_test fixture_test --benchmark_min_time=0.01) compile_benchmark_test(register_benchmark_test) -add_test(NAME register_benchmark_test COMMAND register_benchmark_test --benchmark_min_time=0.01) +add_test(register_benchmark_test register_benchmark_test --benchmark_min_time=0.01) compile_benchmark_test(map_test) -add_test(NAME map_test COMMAND map_test --benchmark_min_time=0.01) +add_test(map_test map_test --benchmark_min_time=0.01) compile_benchmark_test(multiple_ranges_test) -add_test(NAME multiple_ranges_test COMMAND multiple_ranges_test --benchmark_min_time=0.01) +add_test(multiple_ranges_test multiple_ranges_test --benchmark_min_time=0.01) compile_benchmark_test_with_main(link_main_test) -add_test(NAME link_main_test COMMAND link_main_test --benchmark_min_time=0.01) +add_test(link_main_test link_main_test --benchmark_min_time=0.01) compile_output_test(reporter_output_test) -add_test(NAME reporter_output_test COMMAND reporter_output_test --benchmark_min_time=0.01) +add_test(reporter_output_test reporter_output_test --benchmark_min_time=0.01) compile_output_test(templated_fixture_test) -add_test(NAME templated_fixture_test COMMAND templated_fixture_test --benchmark_min_time=0.01) +add_test(templated_fixture_test templated_fixture_test --benchmark_min_time=0.01) compile_output_test(user_counters_test) -add_test(NAME user_counters_test COMMAND user_counters_test --benchmark_min_time=0.01) - -compile_output_test(internal_threading_test) -add_test(NAME internal_threading_test COMMAND internal_threading_test --benchmark_min_time=0.01) +add_test(user_counters_test user_counters_test --benchmark_min_time=0.01) compile_output_test(report_aggregates_only_test) -add_test(NAME report_aggregates_only_test COMMAND report_aggregates_only_test --benchmark_min_time=0.01) +add_test(report_aggregates_only_test report_aggregates_only_test --benchmark_min_time=0.01) compile_output_test(display_aggregates_only_test) -add_test(NAME display_aggregates_only_test COMMAND display_aggregates_only_test --benchmark_min_time=0.01) +add_test(display_aggregates_only_test display_aggregates_only_test --benchmark_min_time=0.01) compile_output_test(user_counters_tabular_test) -add_test(NAME user_counters_tabular_test COMMAND user_counters_tabular_test --benchmark_counters_tabular=true --benchmark_min_time=0.01) +add_test(user_counters_tabular_test user_counters_tabular_test --benchmark_counters_tabular=true --benchmark_min_time=0.01) compile_output_test(user_counters_thousands_test) -add_test(NAME user_counters_thousands_test COMMAND user_counters_thousands_test --benchmark_min_time=0.01) +add_test(user_counters_thousands_test user_counters_thousands_test --benchmark_min_time=0.01) compile_output_test(memory_manager_test) -add_test(NAME memory_manager_test COMMAND memory_manager_test --benchmark_min_time=0.01) +add_test(memory_manager_test memory_manager_test --benchmark_min_time=0.01) check_cxx_compiler_flag(-std=c++03 BENCHMARK_HAS_CXX03_FLAG) if (BENCHMARK_HAS_CXX03_FLAG) @@ -159,7 +156,7 @@ if (BENCHMARK_HAS_CXX03_FLAG) PROPERTIES LINK_FLAGS "-Wno-odr") endif() - add_test(NAME cxx03 COMMAND cxx03_test --benchmark_min_time=0.01) + add_test(cxx03 cxx03_test --benchmark_min_time=0.01) endif() # Attempt to work around flaky test failures when running on Appveyor servers. @@ -169,7 +166,7 @@ else() set(COMPLEXITY_MIN_TIME "0.01") endif() compile_output_test(complexity_test) -add_test(NAME complexity_benchmark COMMAND complexity_test --benchmark_min_time=${COMPLEXITY_MIN_TIME}) +add_test(complexity_benchmark complexity_test --benchmark_min_time=${COMPLEXITY_MIN_TIME}) ############################################################################### # GoogleTest Unit Tests @@ -178,18 +175,22 @@ add_test(NAME complexity_benchmark COMMAND complexity_test --benchmark_min_time= if (BENCHMARK_ENABLE_GTEST_TESTS) macro(compile_gtest name) add_executable(${name} "${name}.cc") + if (TARGET googletest) + add_dependencies(${name} googletest) + endif() + if (GTEST_INCLUDE_DIRS) + target_include_directories(${name} PRIVATE ${GTEST_INCLUDE_DIRS}) + endif() target_link_libraries(${name} benchmark - gmock_main ${CMAKE_THREAD_LIBS_INIT}) + ${GTEST_BOTH_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT}) endmacro(compile_gtest) macro(add_gtest name) compile_gtest(${name}) - add_test(NAME ${name} COMMAND ${name}) + add_test(${name} ${name}) endmacro() add_gtest(benchmark_gtest) - add_gtest(benchmark_name_gtest) - add_gtest(commandlineflags_gtest) add_gtest(statistics_gtest) add_gtest(string_util_gtest) endif(BENCHMARK_ENABLE_GTEST_TESTS) diff --git a/test/basic_test.cc b/test/basic_test.cc index 5f3dd1a..d07fbc0 100644 --- a/test/basic_test.cc +++ b/test/basic_test.cc @@ -98,7 +98,7 @@ BENCHMARK(BM_empty_stop_start)->ThreadPerCpu(); void BM_KeepRunning(benchmark::State& state) { - benchmark::IterationCount iter_count = 0; + size_t iter_count = 0; assert(iter_count == state.iterations()); while (state.KeepRunning()) { ++iter_count; @@ -109,8 +109,8 @@ BENCHMARK(BM_KeepRunning); void BM_KeepRunningBatch(benchmark::State& state) { // Choose a prime batch size to avoid evenly dividing max_iterations. - const benchmark::IterationCount batch_size = 101; - benchmark::IterationCount iter_count = 0; + const size_t batch_size = 101; + size_t iter_count = 0; while (state.KeepRunningBatch(batch_size)) { iter_count += batch_size; } @@ -119,7 +119,7 @@ void BM_KeepRunningBatch(benchmark::State& state) { BENCHMARK(BM_KeepRunningBatch); void BM_RangedFor(benchmark::State& state) { - benchmark::IterationCount iter_count = 0; + size_t iter_count = 0; for (auto _ : state) { ++iter_count; } diff --git a/test/benchmark_gtest.cc b/test/benchmark_gtest.cc index 9557b20..10683b4 100644 --- a/test/benchmark_gtest.cc +++ b/test/benchmark_gtest.cc @@ -4,8 +4,6 @@ #include "gmock/gmock.h" #include "gtest/gtest.h" -namespace benchmark { -namespace internal { namespace { TEST(AddRangeTest, Simple) { @@ -32,97 +30,4 @@ TEST(AddRangeTest, Advanced64) { EXPECT_THAT(dst, testing::ElementsAre(5, 8, 15)); } -TEST(AddRangeTest, FullRange8) { - std::vector<int8_t> dst; - AddRange(&dst, int8_t{1}, std::numeric_limits<int8_t>::max(), 8); - EXPECT_THAT(dst, testing::ElementsAre(1, 8, 64, 127)); -} - -TEST(AddRangeTest, FullRange64) { - std::vector<int64_t> dst; - AddRange(&dst, int64_t{1}, std::numeric_limits<int64_t>::max(), 1024); - EXPECT_THAT( - dst, testing::ElementsAre(1LL, 1024LL, 1048576LL, 1073741824LL, - 1099511627776LL, 1125899906842624LL, - 1152921504606846976LL, 9223372036854775807LL)); -} - -TEST(AddRangeTest, NegativeRanges) { - std::vector<int> dst; - AddRange(&dst, -8, 0, 2); - EXPECT_THAT(dst, testing::ElementsAre(-8, -4, -2, -1, 0)); -} - -TEST(AddRangeTest, StrictlyNegative) { - std::vector<int> dst; - AddRange(&dst, -8, -1, 2); - EXPECT_THAT(dst, testing::ElementsAre(-8, -4, -2, -1)); -} - -TEST(AddRangeTest, SymmetricNegativeRanges) { - std::vector<int> dst; - AddRange(&dst, -8, 8, 2); - EXPECT_THAT(dst, testing::ElementsAre(-8, -4, -2, -1, 0, 1, 2, 4, 8)); -} - -TEST(AddRangeTest, SymmetricNegativeRangesOddMult) { - std::vector<int> dst; - AddRange(&dst, -30, 32, 5); - EXPECT_THAT(dst, testing::ElementsAre(-30, -25, -5, -1, 0, 1, 5, 25, 32)); -} - -TEST(AddRangeTest, NegativeRangesAsymmetric) { - std::vector<int> dst; - AddRange(&dst, -3, 5, 2); - EXPECT_THAT(dst, testing::ElementsAre(-3, -2, -1, 0, 1, 2, 4, 5)); -} - -TEST(AddRangeTest, NegativeRangesLargeStep) { - // Always include -1, 0, 1 when crossing zero. - std::vector<int> dst; - AddRange(&dst, -8, 8, 10); - EXPECT_THAT(dst, testing::ElementsAre(-8, -1, 0, 1, 8)); -} - -TEST(AddRangeTest, ZeroOnlyRange) { - std::vector<int> dst; - AddRange(&dst, 0, 0, 2); - EXPECT_THAT(dst, testing::ElementsAre(0)); -} - -TEST(AddRangeTest, NegativeRange64) { - std::vector<int64_t> dst; - AddRange<int64_t>(&dst, -4, 4, 2); - EXPECT_THAT(dst, testing::ElementsAre(-4, -2, -1, 0, 1, 2, 4)); -} - -TEST(AddRangeTest, NegativeRangePreservesExistingOrder) { - // If elements already exist in the range, ensure we don't change - // their ordering by adding negative values. - std::vector<int64_t> dst = {1, 2, 3}; - AddRange<int64_t>(&dst, -2, 2, 2); - EXPECT_THAT(dst, testing::ElementsAre(1, 2, 3, -2, -1, 0, 1, 2)); -} - -TEST(AddRangeTest, FullNegativeRange64) { - std::vector<int64_t> dst; - const auto min = std::numeric_limits<int64_t>::min(); - const auto max = std::numeric_limits<int64_t>::max(); - AddRange(&dst, min, max, 1024); - EXPECT_THAT( - dst, testing::ElementsAreArray(std::vector<int64_t>{ - min, -1152921504606846976LL, -1125899906842624LL, - -1099511627776LL, -1073741824LL, -1048576LL, -1024LL, -1LL, 0LL, - 1LL, 1024LL, 1048576LL, 1073741824LL, 1099511627776LL, - 1125899906842624LL, 1152921504606846976LL, max})); -} - -TEST(AddRangeTest, Simple8) { - std::vector<int8_t> dst; - AddRange<int8_t>(&dst, 1, 8, 2); - EXPECT_THAT(dst, testing::ElementsAre(1, 2, 4, 8)); -} - -} // namespace -} // namespace internal -} // namespace benchmark +} // end namespace diff --git a/test/benchmark_name_gtest.cc b/test/benchmark_name_gtest.cc deleted file mode 100644 index afb401c..0000000 --- a/test/benchmark_name_gtest.cc +++ /dev/null @@ -1,74 +0,0 @@ -#include "benchmark/benchmark.h" -#include "gtest/gtest.h" - -namespace { - -using namespace benchmark; -using namespace benchmark::internal; - -TEST(BenchmarkNameTest, Empty) { - const auto name = BenchmarkName(); - EXPECT_EQ(name.str(), std::string()); -} - -TEST(BenchmarkNameTest, FunctionName) { - auto name = BenchmarkName(); - name.function_name = "function_name"; - EXPECT_EQ(name.str(), "function_name"); -} - -TEST(BenchmarkNameTest, FunctionNameAndArgs) { - auto name = BenchmarkName(); - name.function_name = "function_name"; - name.args = "some_args:3/4/5"; - EXPECT_EQ(name.str(), "function_name/some_args:3/4/5"); -} - -TEST(BenchmarkNameTest, MinTime) { - auto name = BenchmarkName(); - name.function_name = "function_name"; - name.args = "some_args:3/4"; - name.min_time = "min_time:3.4s"; - EXPECT_EQ(name.str(), "function_name/some_args:3/4/min_time:3.4s"); -} - -TEST(BenchmarkNameTest, Iterations) { - auto name = BenchmarkName(); - name.function_name = "function_name"; - name.min_time = "min_time:3.4s"; - name.iterations = "iterations:42"; - EXPECT_EQ(name.str(), "function_name/min_time:3.4s/iterations:42"); -} - -TEST(BenchmarkNameTest, Repetitions) { - auto name = BenchmarkName(); - name.function_name = "function_name"; - name.min_time = "min_time:3.4s"; - name.repetitions = "repetitions:24"; - EXPECT_EQ(name.str(), "function_name/min_time:3.4s/repetitions:24"); -} - -TEST(BenchmarkNameTest, TimeType) { - auto name = BenchmarkName(); - name.function_name = "function_name"; - name.min_time = "min_time:3.4s"; - name.time_type = "hammer_time"; - EXPECT_EQ(name.str(), "function_name/min_time:3.4s/hammer_time"); -} - -TEST(BenchmarkNameTest, Threads) { - auto name = BenchmarkName(); - name.function_name = "function_name"; - name.min_time = "min_time:3.4s"; - name.threads = "threads:256"; - EXPECT_EQ(name.str(), "function_name/min_time:3.4s/threads:256"); -} - -TEST(BenchmarkNameTest, TestEmptyFunctionName) { - auto name = BenchmarkName(); - name.args = "first:3/second:4"; - name.threads = "threads:22"; - EXPECT_EQ(name.str(), "first:3/second:4/threads:22"); -} - -} // end namespace diff --git a/test/commandlineflags_gtest.cc b/test/commandlineflags_gtest.cc deleted file mode 100644 index 36bdb44..0000000 --- a/test/commandlineflags_gtest.cc +++ /dev/null @@ -1,201 +0,0 @@ -#include <cstdlib> - -#include "../src/commandlineflags.h" -#include "../src/internal_macros.h" -#include "gtest/gtest.h" - -namespace benchmark { -namespace { - -#if defined(BENCHMARK_OS_WINDOWS) -int setenv(const char* name, const char* value, int overwrite) { - if (!overwrite) { - // NOTE: getenv_s is far superior but not available under mingw. - char* env_value = getenv(name); - if (env_value == nullptr) { - return -1; - } - } - return _putenv_s(name, value); -} - -int unsetenv(const char* name) { - return _putenv_s(name, ""); -} - -#endif // BENCHMARK_OS_WINDOWS - -TEST(BoolFromEnv, Default) { - ASSERT_EQ(unsetenv("BENCHMARK_NOT_IN_ENV"), 0); - EXPECT_EQ(BoolFromEnv("not_in_env", true), true); -} - -TEST(BoolFromEnv, False) { - ASSERT_EQ(setenv("BENCHMARK_IN_ENV", "0", 1), 0); - EXPECT_EQ(BoolFromEnv("in_env", true), false); - unsetenv("BENCHMARK_IN_ENV"); - - ASSERT_EQ(setenv("BENCHMARK_IN_ENV", "N", 1), 0); - EXPECT_EQ(BoolFromEnv("in_env", true), false); - unsetenv("BENCHMARK_IN_ENV"); - - ASSERT_EQ(setenv("BENCHMARK_IN_ENV", "n", 1), 0); - EXPECT_EQ(BoolFromEnv("in_env", true), false); - unsetenv("BENCHMARK_IN_ENV"); - - ASSERT_EQ(setenv("BENCHMARK_IN_ENV", "NO", 1), 0); - EXPECT_EQ(BoolFromEnv("in_env", true), false); - unsetenv("BENCHMARK_IN_ENV"); - - ASSERT_EQ(setenv("BENCHMARK_IN_ENV", "No", 1), 0); - EXPECT_EQ(BoolFromEnv("in_env", true), false); - unsetenv("BENCHMARK_IN_ENV"); - - ASSERT_EQ(setenv("BENCHMARK_IN_ENV", "no", 1), 0); - EXPECT_EQ(BoolFromEnv("in_env", true), false); - unsetenv("BENCHMARK_IN_ENV"); - - ASSERT_EQ(setenv("BENCHMARK_IN_ENV", "F", 1), 0); - EXPECT_EQ(BoolFromEnv("in_env", true), false); - unsetenv("BENCHMARK_IN_ENV"); - - ASSERT_EQ(setenv("BENCHMARK_IN_ENV", "f", 1), 0); - EXPECT_EQ(BoolFromEnv("in_env", true), false); - unsetenv("BENCHMARK_IN_ENV"); - - ASSERT_EQ(setenv("BENCHMARK_IN_ENV", "FALSE", 1), 0); - EXPECT_EQ(BoolFromEnv("in_env", true), false); - unsetenv("BENCHMARK_IN_ENV"); - - ASSERT_EQ(setenv("BENCHMARK_IN_ENV", "False", 1), 0); - EXPECT_EQ(BoolFromEnv("in_env", true), false); - unsetenv("BENCHMARK_IN_ENV"); - - ASSERT_EQ(setenv("BENCHMARK_IN_ENV", "false", 1), 0); - EXPECT_EQ(BoolFromEnv("in_env", true), false); - unsetenv("BENCHMARK_IN_ENV"); - - ASSERT_EQ(setenv("BENCHMARK_IN_ENV", "OFF", 1), 0); - EXPECT_EQ(BoolFromEnv("in_env", true), false); - unsetenv("BENCHMARK_IN_ENV"); - - ASSERT_EQ(setenv("BENCHMARK_IN_ENV", "Off", 1), 0); - EXPECT_EQ(BoolFromEnv("in_env", true), false); - unsetenv("BENCHMARK_IN_ENV"); - - ASSERT_EQ(setenv("BENCHMARK_IN_ENV", "off", 1), 0); - EXPECT_EQ(BoolFromEnv("in_env", true), false); - unsetenv("BENCHMARK_IN_ENV"); -} - -TEST(BoolFromEnv, True) { - ASSERT_EQ(setenv("BENCHMARK_IN_ENV", "1", 1), 0); - EXPECT_EQ(BoolFromEnv("in_env", false), true); - unsetenv("BENCHMARK_IN_ENV"); - - ASSERT_EQ(setenv("BENCHMARK_IN_ENV", "Y", 1), 0); - EXPECT_EQ(BoolFromEnv("in_env", false), true); - unsetenv("BENCHMARK_IN_ENV"); - - ASSERT_EQ(setenv("BENCHMARK_IN_ENV", "y", 1), 0); - EXPECT_EQ(BoolFromEnv("in_env", false), true); - unsetenv("BENCHMARK_IN_ENV"); - - ASSERT_EQ(setenv("BENCHMARK_IN_ENV", "YES", 1), 0); - EXPECT_EQ(BoolFromEnv("in_env", false), true); - unsetenv("BENCHMARK_IN_ENV"); - - ASSERT_EQ(setenv("BENCHMARK_IN_ENV", "Yes", 1), 0); - EXPECT_EQ(BoolFromEnv("in_env", false), true); - unsetenv("BENCHMARK_IN_ENV"); - - ASSERT_EQ(setenv("BENCHMARK_IN_ENV", "yes", 1), 0); - EXPECT_EQ(BoolFromEnv("in_env", false), true); - unsetenv("BENCHMARK_IN_ENV"); - - ASSERT_EQ(setenv("BENCHMARK_IN_ENV", "T", 1), 0); - EXPECT_EQ(BoolFromEnv("in_env", false), true); - unsetenv("BENCHMARK_IN_ENV"); - - ASSERT_EQ(setenv("BENCHMARK_IN_ENV", "t", 1), 0); - EXPECT_EQ(BoolFromEnv("in_env", false), true); - unsetenv("BENCHMARK_IN_ENV"); - - ASSERT_EQ(setenv("BENCHMARK_IN_ENV", "TRUE", 1), 0); - EXPECT_EQ(BoolFromEnv("in_env", false), true); - unsetenv("BENCHMARK_IN_ENV"); - - ASSERT_EQ(setenv("BENCHMARK_IN_ENV", "True", 1), 0); - EXPECT_EQ(BoolFromEnv("in_env", false), true); - unsetenv("BENCHMARK_IN_ENV"); - - ASSERT_EQ(setenv("BENCHMARK_IN_ENV", "true", 1), 0); - EXPECT_EQ(BoolFromEnv("in_env", false), true); - unsetenv("BENCHMARK_IN_ENV"); - - ASSERT_EQ(setenv("BENCHMARK_IN_ENV", "ON", 1), 0); - EXPECT_EQ(BoolFromEnv("in_env", false), true); - unsetenv("BENCHMARK_IN_ENV"); - - ASSERT_EQ(setenv("BENCHMARK_IN_ENV", "On", 1), 0); - EXPECT_EQ(BoolFromEnv("in_env", false), true); - unsetenv("BENCHMARK_IN_ENV"); - - ASSERT_EQ(setenv("BENCHMARK_IN_ENV", "on", 1), 0); - EXPECT_EQ(BoolFromEnv("in_env", false), true); - unsetenv("BENCHMARK_IN_ENV"); - -#ifndef BENCHMARK_OS_WINDOWS - ASSERT_EQ(setenv("BENCHMARK_IN_ENV", "", 1), 0); - EXPECT_EQ(BoolFromEnv("in_env", false), true); - unsetenv("BENCHMARK_IN_ENV"); -#endif -} - -TEST(Int32FromEnv, NotInEnv) { - ASSERT_EQ(unsetenv("BENCHMARK_NOT_IN_ENV"), 0); - EXPECT_EQ(Int32FromEnv("not_in_env", 42), 42); -} - -TEST(Int32FromEnv, InvalidInteger) { - ASSERT_EQ(setenv("BENCHMARK_IN_ENV", "foo", 1), 0); - EXPECT_EQ(Int32FromEnv("in_env", 42), 42); - unsetenv("BENCHMARK_IN_ENV"); -} - -TEST(Int32FromEnv, ValidInteger) { - ASSERT_EQ(setenv("BENCHMARK_IN_ENV", "42", 1), 0); - EXPECT_EQ(Int32FromEnv("in_env", 64), 42); - unsetenv("BENCHMARK_IN_ENV"); -} - -TEST(DoubleFromEnv, NotInEnv) { - ASSERT_EQ(unsetenv("BENCHMARK_NOT_IN_ENV"), 0); - EXPECT_EQ(DoubleFromEnv("not_in_env", 0.51), 0.51); -} - -TEST(DoubleFromEnv, InvalidReal) { - ASSERT_EQ(setenv("BENCHMARK_IN_ENV", "foo", 1), 0); - EXPECT_EQ(DoubleFromEnv("in_env", 0.51), 0.51); - unsetenv("BENCHMARK_IN_ENV"); -} - -TEST(DoubleFromEnv, ValidReal) { - ASSERT_EQ(setenv("BENCHMARK_IN_ENV", "0.51", 1), 0); - EXPECT_EQ(DoubleFromEnv("in_env", 0.71), 0.51); - unsetenv("BENCHMARK_IN_ENV"); -} - -TEST(StringFromEnv, Default) { - ASSERT_EQ(unsetenv("BENCHMARK_NOT_IN_ENV"), 0); - EXPECT_STREQ(StringFromEnv("not_in_env", "foo"), "foo"); -} - -TEST(StringFromEnv, Valid) { - ASSERT_EQ(setenv("BENCHMARK_IN_ENV", "foo", 1), 0); - EXPECT_STREQ(StringFromEnv("in_env", "bar"), "foo"); - unsetenv("BENCHMARK_IN_ENV"); -} - -} // namespace -} // namespace benchmark diff --git a/test/complexity_test.cc b/test/complexity_test.cc index 5681fdc..323ddfe 100644 --- a/test/complexity_test.cc +++ b/test/complexity_test.cc @@ -28,8 +28,6 @@ int AddComplexityTest(std::string test_name, std::string big_o_test_name, AddCases(TC_JSONOut, {{"\"name\": \"%bigo_name\",$"}, {"\"run_name\": \"%name\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, - {"\"repetitions\": %int,$", MR_Next}, - {"\"threads\": 1,$", MR_Next}, {"\"aggregate_name\": \"BigO\",$", MR_Next}, {"\"cpu_coefficient\": %float,$", MR_Next}, {"\"real_coefficient\": %float,$", MR_Next}, @@ -39,8 +37,6 @@ int AddComplexityTest(std::string test_name, std::string big_o_test_name, {"\"name\": \"%rms_name\",$"}, {"\"run_name\": \"%name\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, - {"\"repetitions\": %int,$", MR_Next}, - {"\"threads\": 1,$", MR_Next}, {"\"aggregate_name\": \"RMS\",$", MR_Next}, {"\"rms\": %float$", MR_Next}, {"}", MR_Next}}); @@ -66,9 +62,9 @@ void BM_Complexity_O1(benchmark::State& state) { } BENCHMARK(BM_Complexity_O1)->Range(1, 1 << 18)->Complexity(benchmark::o1); BENCHMARK(BM_Complexity_O1)->Range(1, 1 << 18)->Complexity(); -BENCHMARK(BM_Complexity_O1) - ->Range(1, 1 << 18) - ->Complexity([](benchmark::IterationCount) { return 1.0; }); +BENCHMARK(BM_Complexity_O1)->Range(1, 1 << 18)->Complexity([](int64_t) { + return 1.0; +}); const char *one_test_name = "BM_Complexity_O1"; const char *big_o_1_test_name = "BM_Complexity_O1_BigO"; @@ -121,9 +117,7 @@ BENCHMARK(BM_Complexity_O_N) BENCHMARK(BM_Complexity_O_N) ->RangeMultiplier(2) ->Range(1 << 10, 1 << 16) - ->Complexity([](benchmark::IterationCount n) -> double { - return static_cast<double>(n); - }); + ->Complexity([](int64_t n) -> double { return static_cast<double>(n); }); BENCHMARK(BM_Complexity_O_N) ->RangeMultiplier(2) ->Range(1 << 10, 1 << 16) @@ -162,9 +156,7 @@ BENCHMARK(BM_Complexity_O_N_log_N) BENCHMARK(BM_Complexity_O_N_log_N) ->RangeMultiplier(2) ->Range(1 << 10, 1 << 16) - ->Complexity([](benchmark::IterationCount n) { - return kLog2E * n * log(static_cast<double>(n)); - }); + ->Complexity([](int64_t n) { return kLog2E * n * log(static_cast<double>(n)); }); BENCHMARK(BM_Complexity_O_N_log_N) ->RangeMultiplier(2) ->Range(1 << 10, 1 << 16) @@ -185,28 +177,6 @@ ADD_COMPLEXITY_CASES(n_lg_n_test_name, big_o_n_lg_n_test_name, rms_o_n_lg_n_test_name, lambda_big_o_n_lg_n); // ========================================================================= // -// -------- Testing formatting of Complexity with captured args ------------ // -// ========================================================================= // - -void BM_ComplexityCaptureArgs(benchmark::State& state, int n) { - for (auto _ : state) { - // This test requires a non-zero CPU time to avoid divide-by-zero - benchmark::DoNotOptimize(state.iterations()); - } - state.SetComplexityN(n); -} - -BENCHMARK_CAPTURE(BM_ComplexityCaptureArgs, capture_test, 100) - ->Complexity(benchmark::oN) - ->Ranges({{1, 2}, {3, 4}}); - -const std::string complexity_capture_name = - "BM_ComplexityCaptureArgs/capture_test"; - -ADD_COMPLEXITY_CASES(complexity_capture_name, complexity_capture_name + "_BigO", - complexity_capture_name + "_RMS", "N"); - -// ========================================================================= // // --------------------------- TEST CASES END ------------------------------ // // ========================================================================= // diff --git a/test/cxx03_test.cc b/test/cxx03_test.cc index c4c9a52..baa9ed9 100644 --- a/test/cxx03_test.cc +++ b/test/cxx03_test.cc @@ -14,7 +14,7 @@ void BM_empty(benchmark::State& state) { while (state.KeepRunning()) { - volatile benchmark::IterationCount x = state.iterations(); + volatile std::size_t x = state.iterations(); ((void)x); } } diff --git a/test/internal_threading_test.cc b/test/internal_threading_test.cc deleted file mode 100644 index 039d7c1..0000000 --- a/test/internal_threading_test.cc +++ /dev/null @@ -1,184 +0,0 @@ - -#undef NDEBUG - -#include <chrono> -#include <thread> -#include "../src/timers.h" -#include "benchmark/benchmark.h" -#include "output_test.h" - -static const std::chrono::duration<double, std::milli> time_frame(50); -static const double time_frame_in_sec( - std::chrono::duration_cast<std::chrono::duration<double, std::ratio<1, 1>>>( - time_frame) - .count()); - -void MyBusySpinwait() { - const auto start = benchmark::ChronoClockNow(); - - while (true) { - const auto now = benchmark::ChronoClockNow(); - const auto elapsed = now - start; - - if (std::chrono::duration<double, std::chrono::seconds::period>(elapsed) >= - time_frame) - return; - } -} - -// ========================================================================= // -// --------------------------- TEST CASES BEGIN ---------------------------- // -// ========================================================================= // - -// ========================================================================= // -// BM_MainThread - -void BM_MainThread(benchmark::State& state) { - for (auto _ : state) { - MyBusySpinwait(); - state.SetIterationTime(time_frame_in_sec); - } - state.counters["invtime"] = - benchmark::Counter{1, benchmark::Counter::kIsRate}; -} - -BENCHMARK(BM_MainThread)->Iterations(1)->Threads(1); -BENCHMARK(BM_MainThread)->Iterations(1)->Threads(1)->UseRealTime(); -BENCHMARK(BM_MainThread)->Iterations(1)->Threads(1)->UseManualTime(); -BENCHMARK(BM_MainThread)->Iterations(1)->Threads(1)->MeasureProcessCPUTime(); -BENCHMARK(BM_MainThread) - ->Iterations(1) - ->Threads(1) - ->MeasureProcessCPUTime() - ->UseRealTime(); -BENCHMARK(BM_MainThread) - ->Iterations(1) - ->Threads(1) - ->MeasureProcessCPUTime() - ->UseManualTime(); - -BENCHMARK(BM_MainThread)->Iterations(1)->Threads(2); -BENCHMARK(BM_MainThread)->Iterations(1)->Threads(2)->UseRealTime(); -BENCHMARK(BM_MainThread)->Iterations(1)->Threads(2)->UseManualTime(); -BENCHMARK(BM_MainThread)->Iterations(1)->Threads(2)->MeasureProcessCPUTime(); -BENCHMARK(BM_MainThread) - ->Iterations(1) - ->Threads(2) - ->MeasureProcessCPUTime() - ->UseRealTime(); -BENCHMARK(BM_MainThread) - ->Iterations(1) - ->Threads(2) - ->MeasureProcessCPUTime() - ->UseManualTime(); - -// ========================================================================= // -// BM_WorkerThread - -void BM_WorkerThread(benchmark::State& state) { - for (auto _ : state) { - std::thread Worker(&MyBusySpinwait); - Worker.join(); - state.SetIterationTime(time_frame_in_sec); - } - state.counters["invtime"] = - benchmark::Counter{1, benchmark::Counter::kIsRate}; -} - -BENCHMARK(BM_WorkerThread)->Iterations(1)->Threads(1); -BENCHMARK(BM_WorkerThread)->Iterations(1)->Threads(1)->UseRealTime(); -BENCHMARK(BM_WorkerThread)->Iterations(1)->Threads(1)->UseManualTime(); -BENCHMARK(BM_WorkerThread)->Iterations(1)->Threads(1)->MeasureProcessCPUTime(); -BENCHMARK(BM_WorkerThread) - ->Iterations(1) - ->Threads(1) - ->MeasureProcessCPUTime() - ->UseRealTime(); -BENCHMARK(BM_WorkerThread) - ->Iterations(1) - ->Threads(1) - ->MeasureProcessCPUTime() - ->UseManualTime(); - -BENCHMARK(BM_WorkerThread)->Iterations(1)->Threads(2); -BENCHMARK(BM_WorkerThread)->Iterations(1)->Threads(2)->UseRealTime(); -BENCHMARK(BM_WorkerThread)->Iterations(1)->Threads(2)->UseManualTime(); -BENCHMARK(BM_WorkerThread)->Iterations(1)->Threads(2)->MeasureProcessCPUTime(); -BENCHMARK(BM_WorkerThread) - ->Iterations(1) - ->Threads(2) - ->MeasureProcessCPUTime() - ->UseRealTime(); -BENCHMARK(BM_WorkerThread) - ->Iterations(1) - ->Threads(2) - ->MeasureProcessCPUTime() - ->UseManualTime(); - -// ========================================================================= // -// BM_MainThreadAndWorkerThread - -void BM_MainThreadAndWorkerThread(benchmark::State& state) { - for (auto _ : state) { - std::thread Worker(&MyBusySpinwait); - MyBusySpinwait(); - Worker.join(); - state.SetIterationTime(time_frame_in_sec); - } - state.counters["invtime"] = - benchmark::Counter{1, benchmark::Counter::kIsRate}; -} - -BENCHMARK(BM_MainThreadAndWorkerThread)->Iterations(1)->Threads(1); -BENCHMARK(BM_MainThreadAndWorkerThread) - ->Iterations(1) - ->Threads(1) - ->UseRealTime(); -BENCHMARK(BM_MainThreadAndWorkerThread) - ->Iterations(1) - ->Threads(1) - ->UseManualTime(); -BENCHMARK(BM_MainThreadAndWorkerThread) - ->Iterations(1) - ->Threads(1) - ->MeasureProcessCPUTime(); -BENCHMARK(BM_MainThreadAndWorkerThread) - ->Iterations(1) - ->Threads(1) - ->MeasureProcessCPUTime() - ->UseRealTime(); -BENCHMARK(BM_MainThreadAndWorkerThread) - ->Iterations(1) - ->Threads(1) - ->MeasureProcessCPUTime() - ->UseManualTime(); - -BENCHMARK(BM_MainThreadAndWorkerThread)->Iterations(1)->Threads(2); -BENCHMARK(BM_MainThreadAndWorkerThread) - ->Iterations(1) - ->Threads(2) - ->UseRealTime(); -BENCHMARK(BM_MainThreadAndWorkerThread) - ->Iterations(1) - ->Threads(2) - ->UseManualTime(); -BENCHMARK(BM_MainThreadAndWorkerThread) - ->Iterations(1) - ->Threads(2) - ->MeasureProcessCPUTime(); -BENCHMARK(BM_MainThreadAndWorkerThread) - ->Iterations(1) - ->Threads(2) - ->MeasureProcessCPUTime() - ->UseRealTime(); -BENCHMARK(BM_MainThreadAndWorkerThread) - ->Iterations(1) - ->Threads(2) - ->MeasureProcessCPUTime() - ->UseManualTime(); - -// ========================================================================= // -// ---------------------------- TEST CASES END ----------------------------- // -// ========================================================================= // - -int main(int argc, char* argv[]) { RunOutputTests(argc, argv); } diff --git a/test/memory_manager_test.cc b/test/memory_manager_test.cc index 90bed16..94be608 100644 --- a/test/memory_manager_test.cc +++ b/test/memory_manager_test.cc @@ -23,9 +23,6 @@ ADD_CASES(TC_ConsoleOut, {{"^BM_empty %console_report$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_empty\",$"}, {"\"run_name\": \"BM_empty\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, - {"\"repetitions\": 0,$", MR_Next}, - {"\"repetition_index\": 0,$", MR_Next}, - {"\"threads\": 1,$", MR_Next}, {"\"iterations\": %int,$", MR_Next}, {"\"real_time\": %float,$", MR_Next}, {"\"cpu_time\": %float,$", MR_Next}, @@ -35,7 +32,8 @@ ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_empty\",$"}, {"}", MR_Next}}); ADD_CASES(TC_CSVOut, {{"^\"BM_empty\",%csv_report$"}}); -int main(int argc, char* argv[]) { + +int main(int argc, char *argv[]) { std::unique_ptr<benchmark::MemoryManager> mm(new TestMemoryManager()); benchmark::RegisterMemoryManager(mm.get()); diff --git a/test/multiple_ranges_test.cc b/test/multiple_ranges_test.cc index b25f40e..c64acab 100644 --- a/test/multiple_ranges_test.cc +++ b/test/multiple_ranges_test.cc @@ -40,7 +40,8 @@ class MultipleRangesFixture : public ::benchmark::Fixture { // NOTE: This is not TearDown as we want to check after _all_ runs are // complete. virtual ~MultipleRangesFixture() { - if (actualValues != expectedValues) { + assert(actualValues.size() == expectedValues.size()); + if (actualValues.size() != expectedValues.size()) { std::cout << "EXPECTED\n"; for (auto v : expectedValues) { std::cout << "{"; diff --git a/test/options_test.cc b/test/options_test.cc index 7bfc235..fdec691 100644 --- a/test/options_test.cc +++ b/test/options_test.cc @@ -35,16 +35,6 @@ BENCHMARK(BM_basic)->UseRealTime(); BENCHMARK(BM_basic)->ThreadRange(2, 4); BENCHMARK(BM_basic)->ThreadPerCpu(); BENCHMARK(BM_basic)->Repetitions(3); -BENCHMARK(BM_basic) - ->RangeMultiplier(std::numeric_limits<int>::max()) - ->Range(std::numeric_limits<int64_t>::min(), - std::numeric_limits<int64_t>::max()); - -// Negative ranges -BENCHMARK(BM_basic)->Range(-64, -1); -BENCHMARK(BM_basic)->RangeMultiplier(4)->Range(-8, 8); -BENCHMARK(BM_basic)->DenseRange(-2, 2, 1); -BENCHMARK(BM_basic)->Ranges({{-64, 1}, {-8, -1}}); void CustomArgs(benchmark::internal::Benchmark* b) { for (int i = 0; i < 10; ++i) { diff --git a/test/output_test_helper.cc b/test/output_test_helper.cc index bdb34c8..5dc951d 100644 --- a/test/output_test_helper.cc +++ b/test/output_test_helper.cc @@ -373,12 +373,6 @@ int SetSubstitutions( return 0; } -// Disable deprecated warnings temporarily because we need to reference -// CSVReporter but don't want to trigger -Werror=-Wdeprecated -#ifdef __GNUC__ -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wdeprecated" -#endif void RunOutputTests(int argc, char* argv[]) { using internal::GetTestCaseList; benchmark::Initialize(&argc, argv); @@ -437,10 +431,6 @@ void RunOutputTests(int argc, char* argv[]) { internal::GetResultsChecker().CheckResults(csv.out_stream); } -#ifdef __GNUC__ -#pragma GCC diagnostic pop -#endif - int SubstrCnt(const std::string& haystack, const std::string& pat) { if (pat.length() == 0) return 0; int count = 0; diff --git a/test/reporter_output_test.cc b/test/reporter_output_test.cc index 8486d59..ec6d51b 100644 --- a/test/reporter_output_test.cc +++ b/test/reporter_output_test.cc @@ -74,9 +74,6 @@ ADD_CASES(TC_ConsoleOut, {{"^BM_basic %console_report$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_basic\",$"}, {"\"run_name\": \"BM_basic\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, - {"\"repetitions\": 0,$", MR_Next}, - {"\"repetition_index\": 0,$", MR_Next}, - {"\"threads\": 1,$", MR_Next}, {"\"iterations\": %int,$", MR_Next}, {"\"real_time\": %float,$", MR_Next}, {"\"cpu_time\": %float,$", MR_Next}, @@ -90,8 +87,6 @@ ADD_CASES(TC_CSVOut, {{"^\"BM_basic\",%csv_report$"}}); void BM_bytes_per_second(benchmark::State& state) { for (auto _ : state) { - // This test requires a non-zero CPU time to avoid divide-by-zero - benchmark::DoNotOptimize(state.iterations()); } state.SetBytesProcessed(1); } @@ -102,9 +97,6 @@ ADD_CASES(TC_ConsoleOut, {{"^BM_bytes_per_second %console_report " ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_bytes_per_second\",$"}, {"\"run_name\": \"BM_bytes_per_second\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, - {"\"repetitions\": 0,$", MR_Next}, - {"\"repetition_index\": 0,$", MR_Next}, - {"\"threads\": 1,$", MR_Next}, {"\"iterations\": %int,$", MR_Next}, {"\"real_time\": %float,$", MR_Next}, {"\"cpu_time\": %float,$", MR_Next}, @@ -119,8 +111,6 @@ ADD_CASES(TC_CSVOut, {{"^\"BM_bytes_per_second\",%csv_bytes_report$"}}); void BM_items_per_second(benchmark::State& state) { for (auto _ : state) { - // This test requires a non-zero CPU time to avoid divide-by-zero - benchmark::DoNotOptimize(state.iterations()); } state.SetItemsProcessed(1); } @@ -131,9 +121,6 @@ ADD_CASES(TC_ConsoleOut, {{"^BM_items_per_second %console_report " ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_items_per_second\",$"}, {"\"run_name\": \"BM_items_per_second\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, - {"\"repetitions\": 0,$", MR_Next}, - {"\"repetition_index\": 0,$", MR_Next}, - {"\"threads\": 1,$", MR_Next}, {"\"iterations\": %int,$", MR_Next}, {"\"real_time\": %float,$", MR_Next}, {"\"cpu_time\": %float,$", MR_Next}, @@ -157,9 +144,6 @@ ADD_CASES(TC_ConsoleOut, {{"^BM_label %console_report some label$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_label\",$"}, {"\"run_name\": \"BM_label\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, - {"\"repetitions\": 0,$", MR_Next}, - {"\"repetition_index\": 0,$", MR_Next}, - {"\"threads\": 1,$", MR_Next}, {"\"iterations\": %int,$", MR_Next}, {"\"real_time\": %float,$", MR_Next}, {"\"cpu_time\": %float,$", MR_Next}, @@ -183,9 +167,6 @@ ADD_CASES(TC_ConsoleOut, {{"^BM_error[ ]+ERROR OCCURRED: 'message'$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_error\",$"}, {"\"run_name\": \"BM_error\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, - {"\"repetitions\": 0,$", MR_Next}, - {"\"repetition_index\": 0,$", MR_Next}, - {"\"threads\": 1,$", MR_Next}, {"\"error_occurred\": true,$", MR_Next}, {"\"error_message\": \"message\",$", MR_Next}}); @@ -204,10 +185,7 @@ BENCHMARK(BM_no_arg_name)->Arg(3); ADD_CASES(TC_ConsoleOut, {{"^BM_no_arg_name/3 %console_report$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_no_arg_name/3\",$"}, {"\"run_name\": \"BM_no_arg_name/3\",$", MR_Next}, - {"\"run_type\": \"iteration\",$", MR_Next}, - {"\"repetitions\": 0,$", MR_Next}, - {"\"repetition_index\": 0,$", MR_Next}, - {"\"threads\": 1,$", MR_Next}}); + {"\"run_type\": \"iteration\",$", MR_Next}}); ADD_CASES(TC_CSVOut, {{"^\"BM_no_arg_name/3\",%csv_report$"}}); // ========================================================================= // @@ -222,10 +200,7 @@ BENCHMARK(BM_arg_name)->ArgName("first")->Arg(3); ADD_CASES(TC_ConsoleOut, {{"^BM_arg_name/first:3 %console_report$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_arg_name/first:3\",$"}, {"\"run_name\": \"BM_arg_name/first:3\",$", MR_Next}, - {"\"run_type\": \"iteration\",$", MR_Next}, - {"\"repetitions\": 0,$", MR_Next}, - {"\"repetition_index\": 0,$", MR_Next}, - {"\"threads\": 1,$", MR_Next}}); + {"\"run_type\": \"iteration\",$", MR_Next}}); ADD_CASES(TC_CSVOut, {{"^\"BM_arg_name/first:3\",%csv_report$"}}); // ========================================================================= // @@ -242,10 +217,7 @@ ADD_CASES(TC_ConsoleOut, ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_arg_names/first:2/5/third:4\",$"}, {"\"run_name\": \"BM_arg_names/first:2/5/third:4\",$", MR_Next}, - {"\"run_type\": \"iteration\",$", MR_Next}, - {"\"repetitions\": 0,$", MR_Next}, - {"\"repetition_index\": 0,$", MR_Next}, - {"\"threads\": 1,$", MR_Next}}); + {"\"run_type\": \"iteration\",$", MR_Next}}); ADD_CASES(TC_CSVOut, {{"^\"BM_arg_names/first:2/5/third:4\",%csv_report$"}}); // ========================================================================= // @@ -266,8 +238,6 @@ ADD_CASES(TC_ConsoleOut, {{"^BM_BigArgs/1073741824 %console_report$"}, void BM_Complexity_O1(benchmark::State& state) { for (auto _ : state) { - // This test requires a non-zero CPU time to avoid divide-by-zero - benchmark::DoNotOptimize(state.iterations()); } state.SetComplexityN(state.range(0)); } @@ -297,34 +267,22 @@ ADD_CASES(TC_ConsoleOut, ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Repeat/repeats:2\",$"}, {"\"run_name\": \"BM_Repeat/repeats:2\"", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, - {"\"repetitions\": 2,$", MR_Next}, - {"\"repetition_index\": 0,$", MR_Next}, - {"\"threads\": 1,$", MR_Next}, {"\"name\": \"BM_Repeat/repeats:2\",$"}, {"\"run_name\": \"BM_Repeat/repeats:2\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, - {"\"repetitions\": 2,$", MR_Next}, - {"\"repetition_index\": 1,$", MR_Next}, - {"\"threads\": 1,$", MR_Next}, {"\"name\": \"BM_Repeat/repeats:2_mean\",$"}, {"\"run_name\": \"BM_Repeat/repeats:2\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, - {"\"repetitions\": 2,$", MR_Next}, - {"\"threads\": 1,$", MR_Next}, {"\"aggregate_name\": \"mean\",$", MR_Next}, {"\"iterations\": 2,$", MR_Next}, {"\"name\": \"BM_Repeat/repeats:2_median\",$"}, {"\"run_name\": \"BM_Repeat/repeats:2\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, - {"\"repetitions\": 2,$", MR_Next}, - {"\"threads\": 1,$", MR_Next}, {"\"aggregate_name\": \"median\",$", MR_Next}, {"\"iterations\": 2,$", MR_Next}, {"\"name\": \"BM_Repeat/repeats:2_stddev\",$"}, {"\"run_name\": \"BM_Repeat/repeats:2\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, - {"\"repetitions\": 2,$", MR_Next}, - {"\"threads\": 1,$", MR_Next}, {"\"aggregate_name\": \"stddev\",$", MR_Next}, {"\"iterations\": 2,$", MR_Next}}); ADD_CASES(TC_CSVOut, {{"^\"BM_Repeat/repeats:2\",%csv_report$"}, @@ -344,40 +302,25 @@ ADD_CASES(TC_ConsoleOut, ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Repeat/repeats:3\",$"}, {"\"run_name\": \"BM_Repeat/repeats:3\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, - {"\"repetitions\": 3,$", MR_Next}, - {"\"repetition_index\": 0,$", MR_Next}, - {"\"threads\": 1,$", MR_Next}, {"\"name\": \"BM_Repeat/repeats:3\",$"}, {"\"run_name\": \"BM_Repeat/repeats:3\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, - {"\"repetitions\": 3,$", MR_Next}, - {"\"repetition_index\": 1,$", MR_Next}, - {"\"threads\": 1,$", MR_Next}, {"\"name\": \"BM_Repeat/repeats:3\",$"}, {"\"run_name\": \"BM_Repeat/repeats:3\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, - {"\"repetitions\": 3,$", MR_Next}, - {"\"repetition_index\": 2,$", MR_Next}, - {"\"threads\": 1,$", MR_Next}, {"\"name\": \"BM_Repeat/repeats:3_mean\",$"}, {"\"run_name\": \"BM_Repeat/repeats:3\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, - {"\"repetitions\": 3,$", MR_Next}, - {"\"threads\": 1,$", MR_Next}, {"\"aggregate_name\": \"mean\",$", MR_Next}, {"\"iterations\": 3,$", MR_Next}, {"\"name\": \"BM_Repeat/repeats:3_median\",$"}, {"\"run_name\": \"BM_Repeat/repeats:3\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, - {"\"repetitions\": 3,$", MR_Next}, - {"\"threads\": 1,$", MR_Next}, {"\"aggregate_name\": \"median\",$", MR_Next}, {"\"iterations\": 3,$", MR_Next}, {"\"name\": \"BM_Repeat/repeats:3_stddev\",$"}, {"\"run_name\": \"BM_Repeat/repeats:3\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, - {"\"repetitions\": 3,$", MR_Next}, - {"\"threads\": 1,$", MR_Next}, {"\"aggregate_name\": \"stddev\",$", MR_Next}, {"\"iterations\": 3,$", MR_Next}}); ADD_CASES(TC_CSVOut, {{"^\"BM_Repeat/repeats:3\",%csv_report$"}, @@ -399,46 +342,28 @@ ADD_CASES(TC_ConsoleOut, ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Repeat/repeats:4\",$"}, {"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, - {"\"repetitions\": 4,$", MR_Next}, - {"\"repetition_index\": 0,$", MR_Next}, - {"\"threads\": 1,$", MR_Next}, {"\"name\": \"BM_Repeat/repeats:4\",$"}, {"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, - {"\"repetitions\": 4,$", MR_Next}, - {"\"repetition_index\": 1,$", MR_Next}, - {"\"threads\": 1,$", MR_Next}, {"\"name\": \"BM_Repeat/repeats:4\",$"}, {"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, - {"\"repetitions\": 4,$", MR_Next}, - {"\"repetition_index\": 2,$", MR_Next}, - {"\"threads\": 1,$", MR_Next}, {"\"name\": \"BM_Repeat/repeats:4\",$"}, {"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, - {"\"repetitions\": 4,$", MR_Next}, - {"\"repetition_index\": 3,$", MR_Next}, - {"\"threads\": 1,$", MR_Next}, {"\"name\": \"BM_Repeat/repeats:4_mean\",$"}, {"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, - {"\"repetitions\": 4,$", MR_Next}, - {"\"threads\": 1,$", MR_Next}, {"\"aggregate_name\": \"mean\",$", MR_Next}, {"\"iterations\": 4,$", MR_Next}, {"\"name\": \"BM_Repeat/repeats:4_median\",$"}, {"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, - {"\"repetitions\": 4,$", MR_Next}, - {"\"threads\": 1,$", MR_Next}, {"\"aggregate_name\": \"median\",$", MR_Next}, {"\"iterations\": 4,$", MR_Next}, {"\"name\": \"BM_Repeat/repeats:4_stddev\",$"}, {"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, - {"\"repetitions\": 4,$", MR_Next}, - {"\"threads\": 1,$", MR_Next}, {"\"aggregate_name\": \"stddev\",$", MR_Next}, {"\"iterations\": 4,$", MR_Next}}); ADD_CASES(TC_CSVOut, {{"^\"BM_Repeat/repeats:4\",%csv_report$"}, @@ -459,10 +384,7 @@ BENCHMARK(BM_RepeatOnce)->Repetitions(1)->ReportAggregatesOnly(); ADD_CASES(TC_ConsoleOut, {{"^BM_RepeatOnce/repeats:1 %console_report$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_RepeatOnce/repeats:1\",$"}, {"\"run_name\": \"BM_RepeatOnce/repeats:1\",$", MR_Next}, - {"\"run_type\": \"iteration\",$", MR_Next}, - {"\"repetitions\": 1,$", MR_Next}, - {"\"repetition_index\": 0,$", MR_Next}, - {"\"threads\": 1,$", MR_Next}}); + {"\"run_type\": \"iteration\",$", MR_Next}}); ADD_CASES(TC_CSVOut, {{"^\"BM_RepeatOnce/repeats:1\",%csv_report$"}}); // Test that non-aggregate data is not reported @@ -482,22 +404,16 @@ ADD_CASES(TC_JSONOut, {"\"name\": \"BM_SummaryRepeat/repeats:3_mean\",$"}, {"\"run_name\": \"BM_SummaryRepeat/repeats:3\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, - {"\"repetitions\": 3,$", MR_Next}, - {"\"threads\": 1,$", MR_Next}, {"\"aggregate_name\": \"mean\",$", MR_Next}, {"\"iterations\": 3,$", MR_Next}, {"\"name\": \"BM_SummaryRepeat/repeats:3_median\",$"}, {"\"run_name\": \"BM_SummaryRepeat/repeats:3\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, - {"\"repetitions\": 3,$", MR_Next}, - {"\"threads\": 1,$", MR_Next}, {"\"aggregate_name\": \"median\",$", MR_Next}, {"\"iterations\": 3,$", MR_Next}, {"\"name\": \"BM_SummaryRepeat/repeats:3_stddev\",$"}, {"\"run_name\": \"BM_SummaryRepeat/repeats:3\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, - {"\"repetitions\": 3,$", MR_Next}, - {"\"threads\": 1,$", MR_Next}, {"\"aggregate_name\": \"stddev\",$", MR_Next}, {"\"iterations\": 3,$", MR_Next}}); ADD_CASES(TC_CSVOut, {{".*BM_SummaryRepeat/repeats:3 ", MR_Not}, @@ -524,22 +440,16 @@ ADD_CASES(TC_JSONOut, {"\"name\": \"BM_SummaryDisplay/repeats:2_mean\",$"}, {"\"run_name\": \"BM_SummaryDisplay/repeats:2\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, - {"\"repetitions\": 2,$", MR_Next}, - {"\"threads\": 1,$", MR_Next}, {"\"aggregate_name\": \"mean\",$", MR_Next}, {"\"iterations\": 2,$", MR_Next}, {"\"name\": \"BM_SummaryDisplay/repeats:2_median\",$"}, {"\"run_name\": \"BM_SummaryDisplay/repeats:2\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, - {"\"repetitions\": 2,$", MR_Next}, - {"\"threads\": 1,$", MR_Next}, {"\"aggregate_name\": \"median\",$", MR_Next}, {"\"iterations\": 2,$", MR_Next}, {"\"name\": \"BM_SummaryDisplay/repeats:2_stddev\",$"}, {"\"run_name\": \"BM_SummaryDisplay/repeats:2\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, - {"\"repetitions\": 2,$", MR_Next}, - {"\"threads\": 1,$", MR_Next}, {"\"aggregate_name\": \"stddev\",$", MR_Next}, {"\"iterations\": 2,$", MR_Next}}); ADD_CASES(TC_CSVOut, @@ -570,24 +480,18 @@ ADD_CASES(TC_JSONOut, {"\"name\": \"BM_RepeatTimeUnit/repeats:3_mean\",$"}, {"\"run_name\": \"BM_RepeatTimeUnit/repeats:3\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, - {"\"repetitions\": 3,$", MR_Next}, - {"\"threads\": 1,$", MR_Next}, {"\"aggregate_name\": \"mean\",$", MR_Next}, {"\"iterations\": 3,$", MR_Next}, {"\"time_unit\": \"us\",?$"}, {"\"name\": \"BM_RepeatTimeUnit/repeats:3_median\",$"}, {"\"run_name\": \"BM_RepeatTimeUnit/repeats:3\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, - {"\"repetitions\": 3,$", MR_Next}, - {"\"threads\": 1,$", MR_Next}, {"\"aggregate_name\": \"median\",$", MR_Next}, {"\"iterations\": 3,$", MR_Next}, {"\"time_unit\": \"us\",?$"}, {"\"name\": \"BM_RepeatTimeUnit/repeats:3_stddev\",$"}, {"\"run_name\": \"BM_RepeatTimeUnit/repeats:3\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, - {"\"repetitions\": 3,$", MR_Next}, - {"\"threads\": 1,$", MR_Next}, {"\"aggregate_name\": \"stddev\",$", MR_Next}, {"\"iterations\": 3,$", MR_Next}, {"\"time_unit\": \"us\",?$"}}); @@ -639,35 +543,24 @@ ADD_CASES( {"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, - {"\"repetitions\": 3,$", MR_Next}, - {"\"repetition_index\": 0,$", MR_Next}, - {"\"threads\": 1,$", MR_Next}, {"\"iterations\": 5,$", MR_Next}, {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next}, {"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$"}, {"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, - {"\"repetitions\": 3,$", MR_Next}, - {"\"repetition_index\": 1,$", MR_Next}, - {"\"threads\": 1,$", MR_Next}, {"\"iterations\": 5,$", MR_Next}, {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next}, {"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$"}, {"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, - {"\"repetitions\": 3,$", MR_Next}, - {"\"repetition_index\": 2,$", MR_Next}, - {"\"threads\": 1,$", MR_Next}, {"\"iterations\": 5,$", MR_Next}, {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next}, {"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time_mean\",$"}, {"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, - {"\"repetitions\": 3,$", MR_Next}, - {"\"threads\": 1,$", MR_Next}, {"\"aggregate_name\": \"mean\",$", MR_Next}, {"\"iterations\": 3,$", MR_Next}, {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next}, @@ -675,8 +568,6 @@ ADD_CASES( {"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, - {"\"repetitions\": 3,$", MR_Next}, - {"\"threads\": 1,$", MR_Next}, {"\"aggregate_name\": \"median\",$", MR_Next}, {"\"iterations\": 3,$", MR_Next}, {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next}, @@ -684,8 +575,6 @@ ADD_CASES( {"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, - {"\"repetitions\": 3,$", MR_Next}, - {"\"threads\": 1,$", MR_Next}, {"\"aggregate_name\": \"stddev\",$", MR_Next}, {"\"iterations\": 3,$", MR_Next}, {"\"real_time\": %float,$", MR_Next}, @@ -693,8 +582,6 @@ ADD_CASES( {"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, - {"\"repetitions\": 3,$", MR_Next}, - {"\"threads\": 1,$", MR_Next}, {"\"aggregate_name\": \"\",$", MR_Next}, {"\"iterations\": 3,$", MR_Next}, {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next}}); @@ -711,37 +598,6 @@ ADD_CASES( {"^\"BM_UserStats/iterations:5/repeats:3/manual_time_\",%csv_report$"}}); // ========================================================================= // -// ------------------------- Testing StrEscape JSON ------------------------ // -// ========================================================================= // -#if 0 // enable when csv testing code correctly handles multi-line fields -void BM_JSON_Format(benchmark::State& state) { - state.SkipWithError("val\b\f\n\r\t\\\"with\"es,capes"); - for (auto _ : state) { - } -} -BENCHMARK(BM_JSON_Format); -ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_JSON_Format\",$"}, - {"\"run_name\": \"BM_JSON_Format\",$", MR_Next}, - {"\"run_type\": \"iteration\",$", MR_Next}, - {"\"repetitions\": 0,$", MR_Next}, - {"\"repetition_index\": 0,$", MR_Next}, - {"\"threads\": 1,$", MR_Next}, - {"\"error_occurred\": true,$", MR_Next}, - {R"("error_message": "val\\b\\f\\n\\r\\t\\\\\\"with\\"es,capes",$)", MR_Next}}); -#endif -// ========================================================================= // -// -------------------------- Testing CsvEscape ---------------------------- // -// ========================================================================= // - -void BM_CSV_Format(benchmark::State& state) { - state.SkipWithError("\"freedom\""); - for (auto _ : state) { - } -} -BENCHMARK(BM_CSV_Format); -ADD_CASES(TC_CSVOut, {{"^\"BM_CSV_Format\",,,,,,,,true,\"\"\"freedom\"\"\"$"}}); - -// ========================================================================= // // --------------------------- TEST CASES END ------------------------------ // // ========================================================================= // diff --git a/test/state_assembly_test.cc b/test/state_assembly_test.cc index 7ddbb3b..abe9a4d 100644 --- a/test/state_assembly_test.cc +++ b/test/state_assembly_test.cc @@ -25,7 +25,7 @@ extern "C" int test_for_auto_loop() { for (auto _ : S) { // CHECK: .L[[LOOP_HEAD:[a-zA-Z0-9_]+]]: // CHECK-GNU-NEXT: subq $1, %rbx - // CHECK-CLANG-NEXT: {{(addq \$1, %rax|incq %rax|addq \$-1, %rbx)}} + // CHECK-CLANG-NEXT: {{(addq \$1,|incq)}} %rax // CHECK-NEXT: jne .L[[LOOP_HEAD]] benchmark::DoNotOptimize(x); } diff --git a/test/string_util_gtest.cc b/test/string_util_gtest.cc index 01bf155..2c5d073 100644 --- a/test/string_util_gtest.cc +++ b/test/string_util_gtest.cc @@ -3,7 +3,6 @@ //===---------------------------------------------------------------------===// #include "../src/string_util.h" -#include "../src/internal_macros.h" #include "gtest/gtest.h" namespace { @@ -61,11 +60,9 @@ TEST(StringUtilTest, stoul) { EXPECT_EQ(0xBEEFul, benchmark::stoul("BEEF", &pos, 16)); EXPECT_EQ(4ul, pos); } -#ifndef BENCHMARK_HAS_NO_EXCEPTIONS { ASSERT_THROW(benchmark::stoul("this is a test"), std::invalid_argument); } -#endif } TEST(StringUtilTest, stoi) { @@ -109,11 +106,9 @@ TEST(StringUtilTest, stoi) { EXPECT_EQ(0xBEEF, benchmark::stoi("BEEF", &pos, 16)); EXPECT_EQ(4ul, pos); } -#ifndef BENCHMARK_HAS_NO_EXCEPTIONS { ASSERT_THROW(benchmark::stoi("this is a test"), std::invalid_argument); } -#endif } TEST(StringUtilTest, stod) { @@ -143,11 +138,9 @@ TEST(StringUtilTest, stod) { EXPECT_EQ(-1.25e+9, benchmark::stod("-1.25e+9", &pos)); EXPECT_EQ(8ul, pos); } -#ifndef BENCHMARK_HAS_NO_EXCEPTIONS { ASSERT_THROW(benchmark::stod("this is a test"), std::invalid_argument); } -#endif } } // end namespace diff --git a/test/user_counters_tabular_test.cc b/test/user_counters_tabular_test.cc index 18373c0..030e989 100644 --- a/test/user_counters_tabular_test.cc +++ b/test/user_counters_tabular_test.cc @@ -73,9 +73,6 @@ ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_Tabular/threads:%int\",$"}, {"\"run_name\": \"BM_Counters_Tabular/threads:%int\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, - {"\"repetitions\": 0,$", MR_Next}, - {"\"repetition_index\": 0,$", MR_Next}, - {"\"threads\": 1,$", MR_Next}, {"\"iterations\": %int,$", MR_Next}, {"\"real_time\": %float,$", MR_Next}, {"\"cpu_time\": %float,$", MR_Next}, @@ -107,8 +104,6 @@ CHECK_BENCHMARK_RESULTS("BM_Counters_Tabular/threads:%int", &CheckTabular); void BM_CounterRates_Tabular(benchmark::State& state) { for (auto _ : state) { - // This test requires a non-zero CPU time to avoid divide-by-zero - benchmark::DoNotOptimize(state.iterations()); } namespace bm = benchmark; state.counters.insert({ @@ -126,9 +121,6 @@ ADD_CASES(TC_JSONOut, {"\"run_name\": \"BM_CounterRates_Tabular/threads:%int\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, - {"\"repetitions\": 0,$", MR_Next}, - {"\"repetition_index\": 0,$", MR_Next}, - {"\"threads\": 1,$", MR_Next}, {"\"iterations\": %int,$", MR_Next}, {"\"real_time\": %float,$", MR_Next}, {"\"cpu_time\": %float,$", MR_Next}, @@ -176,9 +168,6 @@ ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_CounterSet0_Tabular/threads:%int\",$"}, {"\"run_name\": \"BM_CounterSet0_Tabular/threads:%int\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, - {"\"repetitions\": 0,$", MR_Next}, - {"\"repetition_index\": 0,$", MR_Next}, - {"\"threads\": 1,$", MR_Next}, {"\"iterations\": %int,$", MR_Next}, {"\"real_time\": %float,$", MR_Next}, {"\"cpu_time\": %float,$", MR_Next}, @@ -214,9 +203,6 @@ ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_CounterSet1_Tabular/threads:%int\",$"}, {"\"run_name\": \"BM_CounterSet1_Tabular/threads:%int\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, - {"\"repetitions\": 0,$", MR_Next}, - {"\"repetition_index\": 0,$", MR_Next}, - {"\"threads\": 1,$", MR_Next}, {"\"iterations\": %int,$", MR_Next}, {"\"real_time\": %float,$", MR_Next}, {"\"cpu_time\": %float,$", MR_Next}, @@ -256,9 +242,6 @@ ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_CounterSet2_Tabular/threads:%int\",$"}, {"\"run_name\": \"BM_CounterSet2_Tabular/threads:%int\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, - {"\"repetitions\": 0,$", MR_Next}, - {"\"repetition_index\": 0,$", MR_Next}, - {"\"threads\": 1,$", MR_Next}, {"\"iterations\": %int,$", MR_Next}, {"\"real_time\": %float,$", MR_Next}, {"\"cpu_time\": %float,$", MR_Next}, diff --git a/test/user_counters_test.cc b/test/user_counters_test.cc index 5699f4f..bb0d6b4 100644 --- a/test/user_counters_test.cc +++ b/test/user_counters_test.cc @@ -34,9 +34,6 @@ ADD_CASES(TC_ConsoleOut, ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_Simple\",$"}, {"\"run_name\": \"BM_Counters_Simple\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, - {"\"repetitions\": 0,$", MR_Next}, - {"\"repetition_index\": 0,$", MR_Next}, - {"\"threads\": 1,$", MR_Next}, {"\"iterations\": %int,$", MR_Next}, {"\"real_time\": %float,$", MR_Next}, {"\"cpu_time\": %float,$", MR_Next}, @@ -64,8 +61,6 @@ int num_calls1 = 0; } void BM_Counters_WithBytesAndItemsPSec(benchmark::State& state) { for (auto _ : state) { - // This test requires a non-zero CPU time to avoid divide-by-zero - benchmark::DoNotOptimize(state.iterations()); } state.counters["foo"] = 1; state.counters["bar"] = ++num_calls1; @@ -80,9 +75,6 @@ ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_WithBytesAndItemsPSec\",$"}, {"\"run_name\": \"BM_Counters_WithBytesAndItemsPSec\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, - {"\"repetitions\": 0,$", MR_Next}, - {"\"repetition_index\": 0,$", MR_Next}, - {"\"threads\": 1,$", MR_Next}, {"\"iterations\": %int,$", MR_Next}, {"\"real_time\": %float,$", MR_Next}, {"\"cpu_time\": %float,$", MR_Next}, @@ -113,8 +105,6 @@ CHECK_BENCHMARK_RESULTS("BM_Counters_WithBytesAndItemsPSec", void BM_Counters_Rate(benchmark::State& state) { for (auto _ : state) { - // This test requires a non-zero CPU time to avoid divide-by-zero - benchmark::DoNotOptimize(state.iterations()); } namespace bm = benchmark; state.counters["foo"] = bm::Counter{1, bm::Counter::kIsRate}; @@ -127,9 +117,6 @@ ADD_CASES( ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_Rate\",$"}, {"\"run_name\": \"BM_Counters_Rate\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, - {"\"repetitions\": 0,$", MR_Next}, - {"\"repetition_index\": 0,$", MR_Next}, - {"\"threads\": 1,$", MR_Next}, {"\"iterations\": %int,$", MR_Next}, {"\"real_time\": %float,$", MR_Next}, {"\"cpu_time\": %float,$", MR_Next}, @@ -149,89 +136,6 @@ void CheckRate(Results const& e) { CHECK_BENCHMARK_RESULTS("BM_Counters_Rate", &CheckRate); // ========================================================================= // -// ----------------------- Inverted Counters Output ------------------------ // -// ========================================================================= // - -void BM_Invert(benchmark::State& state) { - for (auto _ : state) { - // This test requires a non-zero CPU time to avoid divide-by-zero - benchmark::DoNotOptimize(state.iterations()); - } - namespace bm = benchmark; - state.counters["foo"] = bm::Counter{0.0001, bm::Counter::kInvert}; - state.counters["bar"] = bm::Counter{10000, bm::Counter::kInvert}; -} -BENCHMARK(BM_Invert); -ADD_CASES(TC_ConsoleOut, - {{"^BM_Invert %console_report bar=%hrfloatu foo=%hrfloatk$"}}); -ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Invert\",$"}, - {"\"run_name\": \"BM_Invert\",$", MR_Next}, - {"\"run_type\": \"iteration\",$", MR_Next}, - {"\"repetitions\": 0,$", MR_Next}, - {"\"repetition_index\": 0,$", MR_Next}, - {"\"threads\": 1,$", MR_Next}, - {"\"iterations\": %int,$", MR_Next}, - {"\"real_time\": %float,$", MR_Next}, - {"\"cpu_time\": %float,$", MR_Next}, - {"\"time_unit\": \"ns\",$", MR_Next}, - {"\"bar\": %float,$", MR_Next}, - {"\"foo\": %float$", MR_Next}, - {"}", MR_Next}}); -ADD_CASES(TC_CSVOut, {{"^\"BM_Invert\",%csv_report,%float,%float$"}}); -// VS2013 does not allow this function to be passed as a lambda argument -// to CHECK_BENCHMARK_RESULTS() -void CheckInvert(Results const& e) { - CHECK_FLOAT_COUNTER_VALUE(e, "foo", EQ, 10000, 0.0001); - CHECK_FLOAT_COUNTER_VALUE(e, "bar", EQ, 0.0001, 0.0001); -} -CHECK_BENCHMARK_RESULTS("BM_Invert", &CheckInvert); - -// ========================================================================= // -// ------------------------- InvertedRate Counters Output -// -------------------------- // -// ========================================================================= // - -void BM_Counters_InvertedRate(benchmark::State& state) { - for (auto _ : state) { - // This test requires a non-zero CPU time to avoid divide-by-zero - benchmark::DoNotOptimize(state.iterations()); - } - namespace bm = benchmark; - state.counters["foo"] = - bm::Counter{1, bm::Counter::kIsRate | bm::Counter::kInvert}; - state.counters["bar"] = - bm::Counter{8192, bm::Counter::kIsRate | bm::Counter::kInvert}; -} -BENCHMARK(BM_Counters_InvertedRate); -ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_InvertedRate %console_report " - "bar=%hrfloats foo=%hrfloats$"}}); -ADD_CASES(TC_JSONOut, - {{"\"name\": \"BM_Counters_InvertedRate\",$"}, - {"\"run_name\": \"BM_Counters_InvertedRate\",$", MR_Next}, - {"\"run_type\": \"iteration\",$", MR_Next}, - {"\"repetitions\": 0,$", MR_Next}, - {"\"repetition_index\": 0,$", MR_Next}, - {"\"threads\": 1,$", MR_Next}, - {"\"iterations\": %int,$", MR_Next}, - {"\"real_time\": %float,$", MR_Next}, - {"\"cpu_time\": %float,$", MR_Next}, - {"\"time_unit\": \"ns\",$", MR_Next}, - {"\"bar\": %float,$", MR_Next}, - {"\"foo\": %float$", MR_Next}, - {"}", MR_Next}}); -ADD_CASES(TC_CSVOut, - {{"^\"BM_Counters_InvertedRate\",%csv_report,%float,%float$"}}); -// VS2013 does not allow this function to be passed as a lambda argument -// to CHECK_BENCHMARK_RESULTS() -void CheckInvertedRate(Results const& e) { - double t = e.DurationCPUTime(); // this (and not real time) is the time used - // check that the values are within 0.1% of the expected values - CHECK_FLOAT_COUNTER_VALUE(e, "foo", EQ, t, 0.001); - CHECK_FLOAT_COUNTER_VALUE(e, "bar", EQ, t / 8192.0, 0.001); -} -CHECK_BENCHMARK_RESULTS("BM_Counters_InvertedRate", &CheckInvertedRate); - -// ========================================================================= // // ------------------------- Thread Counters Output ------------------------ // // ========================================================================= // @@ -248,9 +152,6 @@ ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_Threads/threads:%int\",$"}, {"\"run_name\": \"BM_Counters_Threads/threads:%int\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, - {"\"repetitions\": 0,$", MR_Next}, - {"\"repetition_index\": 0,$", MR_Next}, - {"\"threads\": 1,$", MR_Next}, {"\"iterations\": %int,$", MR_Next}, {"\"real_time\": %float,$", MR_Next}, {"\"cpu_time\": %float,$", MR_Next}, @@ -287,9 +188,6 @@ ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_AvgThreads/threads:%int\",$"}, {"\"run_name\": \"BM_Counters_AvgThreads/threads:%int\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, - {"\"repetitions\": 0,$", MR_Next}, - {"\"repetition_index\": 0,$", MR_Next}, - {"\"threads\": 1,$", MR_Next}, {"\"iterations\": %int,$", MR_Next}, {"\"real_time\": %float,$", MR_Next}, {"\"cpu_time\": %float,$", MR_Next}, @@ -315,8 +213,6 @@ CHECK_BENCHMARK_RESULTS("BM_Counters_AvgThreads/threads:%int", void BM_Counters_AvgThreadsRate(benchmark::State& state) { for (auto _ : state) { - // This test requires a non-zero CPU time to avoid divide-by-zero - benchmark::DoNotOptimize(state.iterations()); } namespace bm = benchmark; state.counters["foo"] = bm::Counter{1, bm::Counter::kAvgThreadsRate}; @@ -330,9 +226,6 @@ ADD_CASES(TC_JSONOut, {"\"run_name\": \"BM_Counters_AvgThreadsRate/threads:%int\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, - {"\"repetitions\": 0,$", MR_Next}, - {"\"repetition_index\": 0,$", MR_Next}, - {"\"threads\": 1,$", MR_Next}, {"\"iterations\": %int,$", MR_Next}, {"\"real_time\": %float,$", MR_Next}, {"\"cpu_time\": %float,$", MR_Next}, @@ -369,9 +262,6 @@ ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_IterationInvariant\",$"}, {"\"run_name\": \"BM_Counters_IterationInvariant\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, - {"\"repetitions\": 0,$", MR_Next}, - {"\"repetition_index\": 0,$", MR_Next}, - {"\"threads\": 1,$", MR_Next}, {"\"iterations\": %int,$", MR_Next}, {"\"real_time\": %float,$", MR_Next}, {"\"cpu_time\": %float,$", MR_Next}, @@ -398,8 +288,6 @@ CHECK_BENCHMARK_RESULTS("BM_Counters_IterationInvariant", void BM_Counters_kIsIterationInvariantRate(benchmark::State& state) { for (auto _ : state) { - // This test requires a non-zero CPU time to avoid divide-by-zero - benchmark::DoNotOptimize(state.iterations()); } namespace bm = benchmark; state.counters["foo"] = @@ -415,9 +303,6 @@ ADD_CASES(TC_JSONOut, {"\"run_name\": \"BM_Counters_kIsIterationInvariantRate\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, - {"\"repetitions\": 0,$", MR_Next}, - {"\"repetition_index\": 0,$", MR_Next}, - {"\"threads\": 1,$", MR_Next}, {"\"iterations\": %int,$", MR_Next}, {"\"real_time\": %float,$", MR_Next}, {"\"cpu_time\": %float,$", MR_Next}, @@ -457,9 +342,6 @@ ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_AvgIterations\",$"}, {"\"run_name\": \"BM_Counters_AvgIterations\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, - {"\"repetitions\": 0,$", MR_Next}, - {"\"repetition_index\": 0,$", MR_Next}, - {"\"threads\": 1,$", MR_Next}, {"\"iterations\": %int,$", MR_Next}, {"\"real_time\": %float,$", MR_Next}, {"\"cpu_time\": %float,$", MR_Next}, @@ -485,8 +367,6 @@ CHECK_BENCHMARK_RESULTS("BM_Counters_AvgIterations", &CheckAvgIterations); void BM_Counters_kAvgIterationsRate(benchmark::State& state) { for (auto _ : state) { - // This test requires a non-zero CPU time to avoid divide-by-zero - benchmark::DoNotOptimize(state.iterations()); } namespace bm = benchmark; state.counters["foo"] = bm::Counter{1, bm::Counter::kAvgIterationsRate}; @@ -500,9 +380,6 @@ ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_kAvgIterationsRate\",$"}, {"\"run_name\": \"BM_Counters_kAvgIterationsRate\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, - {"\"repetitions\": 0,$", MR_Next}, - {"\"repetition_index\": 0,$", MR_Next}, - {"\"threads\": 1,$", MR_Next}, {"\"iterations\": %int,$", MR_Next}, {"\"real_time\": %float,$", MR_Next}, {"\"cpu_time\": %float,$", MR_Next}, diff --git a/test/user_counters_thousands_test.cc b/test/user_counters_thousands_test.cc index 21d8285..fa0ef97 100644 --- a/test/user_counters_thousands_test.cc +++ b/test/user_counters_thousands_test.cc @@ -53,9 +53,6 @@ ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_Thousands/repeats:2\",$"}, {"\"run_name\": \"BM_Counters_Thousands/repeats:2\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, - {"\"repetitions\": 2,$", MR_Next}, - {"\"repetition_index\": 0,$", MR_Next}, - {"\"threads\": 1,$", MR_Next}, {"\"iterations\": %int,$", MR_Next}, {"\"real_time\": %float,$", MR_Next}, {"\"cpu_time\": %float,$", MR_Next}, @@ -70,9 +67,6 @@ ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_Thousands/repeats:2\",$"}, {"\"run_name\": \"BM_Counters_Thousands/repeats:2\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, - {"\"repetitions\": 2,$", MR_Next}, - {"\"repetition_index\": 1,$", MR_Next}, - {"\"threads\": 1,$", MR_Next}, {"\"iterations\": %int,$", MR_Next}, {"\"real_time\": %float,$", MR_Next}, {"\"cpu_time\": %float,$", MR_Next}, @@ -87,8 +81,6 @@ ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_Thousands/repeats:2_mean\",$"}, {"\"run_name\": \"BM_Counters_Thousands/repeats:2\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, - {"\"repetitions\": 2,$", MR_Next}, - {"\"threads\": 1,$", MR_Next}, {"\"aggregate_name\": \"mean\",$", MR_Next}, {"\"iterations\": 2,$", MR_Next}, {"\"real_time\": %float,$", MR_Next}, @@ -104,8 +96,6 @@ ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_Thousands/repeats:2_median\",$"}, {"\"run_name\": \"BM_Counters_Thousands/repeats:2\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, - {"\"repetitions\": 2,$", MR_Next}, - {"\"threads\": 1,$", MR_Next}, {"\"aggregate_name\": \"median\",$", MR_Next}, {"\"iterations\": 2,$", MR_Next}, {"\"real_time\": %float,$", MR_Next}, @@ -121,8 +111,6 @@ ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_Thousands/repeats:2_stddev\",$"}, {"\"run_name\": \"BM_Counters_Thousands/repeats:2\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, - {"\"repetitions\": 2,$", MR_Next}, - {"\"threads\": 1,$", MR_Next}, {"\"aggregate_name\": \"stddev\",$", MR_Next}, {"\"iterations\": 2,$", MR_Next}, {"\"real_time\": %float,$", MR_Next}, diff --git a/tools/gbench/Inputs/test1_run1.json b/tools/gbench/Inputs/test1_run1.json index 601e327..d7ec6a9 100644 --- a/tools/gbench/Inputs/test1_run1.json +++ b/tools/gbench/Inputs/test1_run1.json @@ -85,24 +85,7 @@ "time_unit": "ns" }, { - "name": "MyComplexityTest_BigO", - "run_name": "MyComplexityTest", - "run_type": "aggregate", - "aggregate_name": "BigO", - "cpu_coefficient": 4.2749856294592886e+00, - "real_coefficient": 6.4789275289789780e+00, - "big_o": "N", - "time_unit": "ns" - }, - { - "name": "MyComplexityTest_RMS", - "run_name": "MyComplexityTest", - "run_type": "aggregate", - "aggregate_name": "RMS", - "rms": 4.5097802512472874e-03 - }, - { - "name": "BM_NotBadTimeUnit", + "name": "BM_BadTimeUnit", "iterations": 1000, "real_time": 0.4, "cpu_time": 0.5, diff --git a/tools/gbench/Inputs/test1_run2.json b/tools/gbench/Inputs/test1_run2.json index 3cbcf39..59a5ffa 100644 --- a/tools/gbench/Inputs/test1_run2.json +++ b/tools/gbench/Inputs/test1_run2.json @@ -85,24 +85,7 @@ "time_unit": "ns" }, { - "name": "MyComplexityTest_BigO", - "run_name": "MyComplexityTest", - "run_type": "aggregate", - "aggregate_name": "BigO", - "cpu_coefficient": 5.6215779594361486e+00, - "real_coefficient": 5.6288314793554610e+00, - "big_o": "N", - "time_unit": "ns" - }, - { - "name": "MyComplexityTest_RMS", - "run_name": "MyComplexityTest", - "run_type": "aggregate", - "aggregate_name": "RMS", - "rms": 3.3128901852342174e-03 - }, - { - "name": "BM_NotBadTimeUnit", + "name": "BM_BadTimeUnit", "iterations": 1000, "real_time": 0.04, "cpu_time": 0.6, diff --git a/tools/gbench/report.py b/tools/gbench/report.py index 5bd3a8d..5085b93 100644 --- a/tools/gbench/report.py +++ b/tools/gbench/report.py @@ -114,10 +114,6 @@ def intersect(list1, list2): return [x for x in list1 if x in list2] -def is_potentially_comparable_benchmark(x): - return ('time_unit' in x and 'real_time' in x and 'cpu_time' in x) - - def partition_benchmarks(json1, json2): """ While preserving the ordering, find benchmarks with the same names in @@ -129,17 +125,10 @@ def partition_benchmarks(json1, json2): names = intersect(json1_unique_names, json2_unique_names) partitions = [] for name in names: - time_unit = None # Pick the time unit from the first entry of the lhs benchmark. - # We should be careful not to crash with unexpected input. - for x in json1['benchmarks']: - if (x['name'] == name and is_potentially_comparable_benchmark(x)): - time_unit = x['time_unit'] - break - if time_unit is None: - continue + time_unit = (x['time_unit'] + for x in json1['benchmarks'] if x['name'] == name).next() # Filter by name and time unit. - # All the repetitions are assumed to be comparable. lhs = [x for x in json1['benchmarks'] if x['name'] == name and x['time_unit'] == time_unit] rhs = [x for x in json2['benchmarks'] if x['name'] == name and @@ -154,7 +143,11 @@ def extract_field(partition, field_name): rhs = [x[field_name] for x in partition[1]] return [lhs, rhs] -def calc_utest(timings_cpu, timings_time): + +def print_utest(partition, utest_alpha, first_col_width, use_color=True): + timings_time = extract_field(partition, 'real_time') + timings_cpu = extract_field(partition, 'cpu_time') + min_rep_cnt = min(len(timings_time[0]), len(timings_time[1]), len(timings_cpu[0]), @@ -162,33 +155,21 @@ def calc_utest(timings_cpu, timings_time): # Does *everything* has at least UTEST_MIN_REPETITIONS repetitions? if min_rep_cnt < UTEST_MIN_REPETITIONS: - return False, None, None + return [] + + def get_utest_color(pval): + return BC_FAIL if pval >= utest_alpha else BC_OKGREEN time_pvalue = mannwhitneyu( timings_time[0], timings_time[1], alternative='two-sided').pvalue cpu_pvalue = mannwhitneyu( timings_cpu[0], timings_cpu[1], alternative='two-sided').pvalue - return (min_rep_cnt >= UTEST_OPTIMAL_REPETITIONS), cpu_pvalue, time_pvalue - -def print_utest(partition, utest_alpha, first_col_width, use_color=True): - def get_utest_color(pval): - return BC_FAIL if pval >= utest_alpha else BC_OKGREEN - - timings_time = extract_field(partition, 'real_time') - timings_cpu = extract_field(partition, 'cpu_time') - have_optimal_repetitions, cpu_pvalue, time_pvalue = calc_utest(timings_cpu, timings_time) - - # Check if we failed miserably with minimum required repetitions for utest - if not have_optimal_repetitions and cpu_pvalue is None and time_pvalue is None: - return [] - dsc = "U Test, Repetitions: {} vs {}".format( len(timings_cpu[0]), len(timings_cpu[1])) dsc_color = BC_OKGREEN - # We still got some results to show but issue a warning about it. - if not have_optimal_repetitions: + if min_rep_cnt < UTEST_OPTIMAL_REPETITIONS: dsc_color = BC_WARNING dsc += ". WARNING: Results unreliable! {}+ repetitions recommended.".format( UTEST_OPTIMAL_REPETITIONS) @@ -349,7 +330,7 @@ class TestReportDifference(unittest.TestCase): ['BM_10PercentCPUToTime', '+0.1000', '-0.1000', '100', '110', '100', '90'], ['BM_ThirdFaster', '-0.3333', '-0.3334', '100', '67', '100', '67'], - ['BM_NotBadTimeUnit', '-0.9000', '+0.2000', '0', '0', '0', '1'], + ['BM_BadTimeUnit', '-0.9000', '+0.2000', '0', '0', '0', '1'], ] json1, json2 = self.load_results() output_lines_with_header = generate_difference_report( |