Merge branch 'release/v2.5.0'

DigitalInBlue · Jun 9, 2019 · 6f24a1d · 6f24a1d
2 parents a2ffd07 + b9db3d9
commit 6f24a1d
Show file tree

Hide file tree

Showing 53 changed files with 253 additions and 97 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -22,6 +22,7 @@ addons:
     packages:
       - g++-6
       - g++-7
+      - g++-8
 
     sources: &sources
       - ubuntu-toolchain-r-test
@@ -30,6 +31,7 @@ cache:
   directories:
     - ${TRAVIS_BUILD_DIR}/deps/llvm-3.9.0
     - ${TRAVIS_BUILD_DIR}/deps/llvm-5.0.1
+    - ${TRAVIS_BUILD_DIR}/deps/llvm-7.0.0
 
 matrix:
   exclude:
@@ -43,16 +45,21 @@ matrix:
     # speed up Travis builds.
     ##########################################################################
 
-    # Clang 3.8
+    # Clang 3.9
     - os: linux
       env: LLVM_VERSION=3.9.0
       compiler: clang
 
-    # Clang 3.9
+    # Clang 5.0.1
     - os: linux
       env: LLVM_VERSION=5.0.1
       compiler: clang
 
+    # Clang 7.0.0
+    - os: linux
+      env: LLVM_VERSION=7.0.0
+      compiler: clang
+
     # GCC 6
     - os: linux
       env: COMPILER=g++-6
@@ -63,6 +70,11 @@ matrix:
       env: COMPILER=g++-7
       compiler: gcc
 
+    # GCC 8
+    - os: linux
+      env: COMPILER=g++-8
+      compiler: gcc
+
     ##########################################################################
     # Build with variations in the configuration
     ##########################################################################
@@ -71,6 +83,9 @@ matrix:
       env: LLVM_VERSION=default CMAKE_OPTIONS="-DCMAKE_BUILD_TYPE=DEBUG"
       compiler: clang
 
+    - os: osx
+      osx_image: xcode10.1
+
 install:
   ############################################################################
   # All the dependencies are installed in ${TRAVIS_BUILD_DIR}/deps/

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -30,10 +30,6 @@ include(CheckFunctionExists)
 include(CheckCXXSourceCompiles)
 include(CheckIncludeFile)
 
-if(POLICY CMP0042)
-	cmake_policy(SET CMP0042 OLD) # MACOSX_RPATH migration
-endif()
-
 #
 # User Options
 #
@@ -231,23 +227,18 @@ export(EXPORT ${PROJECT_NAME}-target
 # ---------------------------------------------------------------------------
 
 if(CELERO_ENABLE_TESTS)
-	# Pull in Google Test
-	# https://github.com/google/googletest/tree/master/googletest#incorporating-into-an-existing-cmake-project
-
-	# Download and unpack googletest at configure time
+	set(gtest_repository "https://github.com/google/googletest.git" CACHE STRING "")
 	configure_file(CMakeLists.txt.in googletest-download/CMakeLists.txt)
-
-	execute_process(COMMAND ${CMAKE_COMMAND} -G "${CMAKE_GENERATOR}" . 
+	execute_process(COMMAND ${CMAKE_COMMAND} -G "${CMAKE_GENERATOR}" .
 		RESULT_VARIABLE result
-		WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/googletest-download )
-
+		WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/googletest-download )
 	if(result)
 		message(FATAL_ERROR "CMake step for googletest failed: ${result}")
 	endif()
-
+	
 	execute_process(COMMAND ${CMAKE_COMMAND} --build .
 		RESULT_VARIABLE result
-		WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/googletest-download )
+		WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/googletest-download )
 
 	if(result)
 		message(FATAL_ERROR "Build step for googletest failed: ${result}")
@@ -259,11 +250,16 @@ if(CELERO_ENABLE_TESTS)
 
 	# Add googletest directly to our build. This defines
 	# the gtest and gtest_main targets.
-	add_subdirectory(${CMAKE_BINARY_DIR}/googletest-src
-					 ${CMAKE_BINARY_DIR}/googletest-build
-					 EXCLUDE_FROM_ALL)
-
-	include_directories("${gtest_SOURCE_DIR}/include")
+	add_subdirectory(${CMAKE_CURRENT_BINARY_DIR}/googletest-src
+	                 ${CMAKE_CURRENT_BINARY_DIR}/googletest-build
+	                 EXCLUDE_FROM_ALL)
+
+	# The gtest/gtest_main targets carry header search path
+	# dependencies automatically when using CMake 2.8.11 or
+	# later. Otherwise we have to add them here ourselves.
+	if (CMAKE_VERSION VERSION_LESS 2.8.11)
+		include_directories("${gtest_SOURCE_DIR}/include")
+	endif()
 
 	set(PROJECT_NAME CeleroTest)
 

diff --git a/CMakeLists.txt.in b/CMakeLists.txt.in
@@ -4,7 +4,7 @@ project(googletest-download NONE)
 
 include(ExternalProject)
 ExternalProject_Add(googletest
-  GIT_REPOSITORY    https://github.com/google/googletest.git
+  GIT_REPOSITORY    ${gtest_repository}
   GIT_TAG           master
   SOURCE_DIR        "${CMAKE_BINARY_DIR}/googletest-src"
   BINARY_DIR        "${CMAKE_BINARY_DIR}/googletest-build"

diff --git a/README.md b/README.md
@@ -6,13 +6,33 @@ Copyright 2017-2019 John Farrier
 
 Apache 2.0 License
 
+#### Community Support
+
+A Special Thanks to the following corporations for their support:
+- [Hellebore Consulting Group](http://www.hellebore.com)
+- [Araxis](https://www.araxis.com/)
+- [Axosoft](https://www.gitkraken.com/)
+- [Microsoft](https://www.microsoft.com)
+- [Travis.CI](https://travis-ci.org)
+
 #### Builds and Testing
 
 Branch                 | Status
 ---------------------- | --------------------------------------------------------------------------------------------------------------------------------------
 ```origin/master: ```  | [![Build Status (Master)](https://travis-ci.org/DigitalInBlue/Celero.svg?branch=master)](https://travis-ci.org/DigitalInBlue/Celero)
 ```origin/develop: ``` | [![Build Status (Develop)](https://travis-ci.org/DigitalInBlue/Celero.svg?branch=develop)](https://travis-ci.org/DigitalInBlue/Celero)
 
+Celero has been successfully built on the following platforms during development.  See [Travis.CI](https://travis-ci.org/DigitalInBlue/Celero) for more details.
+- GCC v6.0.0
+- GCC v7.0.0
+- GCC v8.0.0
+- LLVM v3.9.0
+- LLVM v5.0.1
+- LLVM v7.0.0
+- Visual Studio 2017
+- Visual Studio 2019
+- XCode v10.1
+
 ### Overview
 
 Developing consistent and meaningful benchmark results for code is a complex task. Measurement tools exist (Intel® VTune™ Amplifier, SmartBear AQTime, Valgrind, etc.) external to applications, but they are sometimes expensive for small teams or cumbersome to utilize. This project, Celero, aims to be a small library which can be added to a C++ project and perform benchmarks on code in a way which is easy to reproduce, share, and compare among individual runs, developers, or projects. Celero uses a framework similar to that of GoogleTest to make its API easier to use and integrate into a project. Make automated benchmarking as much a part of your development process as automated testing.
@@ -214,21 +234,49 @@ Running Celero's simple example experiment (`celeroDemoSimple.exe`) benchmark ga
 
 ```
 Celero
-Timer resolution: 0.069841 us
------------------------------------------------------------------------------------------------------------------------------------------------
-     Group      |   Experiment    |   Prob. Space   |     Samples     |   Iterations    |    Baseline     |  us/Iteration   | Iterations/sec  |
------------------------------------------------------------------------------------------------------------------------------------------------
-DemoSimple      | Baseline        |               0 |              10 |         1000000 |         1.00000 |         0.28789 |      3473512.73 |
-DemoSimple      | Complex1        |               0 |               1 |          710000 |         1.11028 |         0.31964 |      3128497.53 |
-DemoSimple      | Complex2        |               0 |              30 |          710000 |         1.10749 |         0.31884 |      3136388.74 |
-DemoSimple      | Complex3        |               0 |              60 |          710000 |         1.10678 |         0.31863 |      3138398.97 |
-Complete.
+Timer resolution: 0.277056 us
+|     Group      |   Experiment    |   Prob. Space   |     Samples     |   Iterations    |    Baseline     |  us/Iteration   | Iterations/sec  |
+|:--------------:|:---------------:|:---------------:|:---------------:|:---------------:|:---------------:|:---------------:|:---------------:|
+|DemoSimple      | Baseline        |            Null |              30 |         1000000 |         1.00000 |         0.09320 |     10729498.61 |
+|DemoSimple      | Complex1        |            Null |               1 |          710000 |         0.99833 |         0.09305 |     10747479.64 |
+|DemoSimple      | Complex2        |            Null |              30 |          710000 |         0.97898 |         0.09124 |     10959834.52 |
+|DemoSimple      | Complex3        |            Null |              60 |          710000 |         0.98547 |         0.09185 |     10887733.66 |
+Completed in 00:00:10.315012
 ```
 
-The first test that executes will be the group's baseline.  Celero took 10 samples of 1000000 iterations of the code in our test.  (Each set of 1000000 iterations was measured, and this was done 10 times and the smallest time was taken.)  The "Baseline" value for the baseline measurement itself will always be 1.0.
+The first test that executes will be the group's baseline.  Celero took 30 samples of 1000000 iterations of the code in our test.  (Each set of 1000000 iterations was measured, and this was done 10 times and the smallest time was taken.)  The "Baseline" value for the baseline measurement itself will always be 1.0.
 
 After the baseline is complete, each individual test is ran.  Each test is executed and measured in the same way, however, there is an additional metric reported: Baseline.  This compares the time it takes to compute the benchmark to the baseline.  The data here shows that `CeleroBenchTest.Complex1` takes 1.007949 times longer to execute than the baseline.
 
+### Automatically computing the number of Iterations and Samples
+
+If you do want Celero to figure out a reasonable number of iterations to run, you can set the iteration value to ```0``` for your experiment.  You can also set the number of samples to ```0``` to have it compute a statistically valid number of samples.  (Note that the current implementation uses ```30``` as the default number of samples, but does compute a reasonable number of iterations.) 
+
+Update the previous "DemoSimple" code's ```Complex1``` case to use this feature as follows:
+
+```cpp
+/// Run a test consisting of 0 samples of 0 iterations per measurement.
+/// Since the sample size is equal to 0, celero will compute a number to use for both samples and iterations.
+BENCHMARK(DemoSimple, Complex1, 0, 0)
+{
+	celero::DoNotOptimizeAway(static_cast<float>(sin(fmod(UniformDistribution(RandomDevice), 3.14159265))));
+}
+```
+
+Now, when this executes, you will see a different number automatically computed for the number of iterations and the sample size has been increased.
+
+```
+Celero
+Timer resolution: 0.277056 us
+|     Group      |   Experiment    |   Prob. Space   |     Samples     |   Iterations    |    Baseline     |  us/Iteration   | Iterations/sec  |
+|:--------------:|:---------------:|:---------------:|:---------------:|:---------------:|:---------------:|:---------------:|:---------------:|
+|DemoSimple      | Baseline        |            Null |              30 |         1000000 |         1.00000 |         0.09177 |     10897044.72 |
+|DemoSimple      | Complex1        |            Null |              30 |         8388608 |         1.01211 |         0.09288 |     10766703.67 |
+|DemoSimple      | Complex2        |            Null |              30 |          710000 |         0.99559 |         0.09136 |     10945304.31 |
+|DemoSimple      | Complex3        |            Null |              60 |          710000 |         0.99671 |         0.09147 |     10933000.72 |
+Completed in 00:00:37.583872
+```
+
 #### Statistically Sound Results
 
 In order to use Celero for real science, there are three primary factors to consider when reviewing results.  Firstly, you MUST check the generated assembly for your test.  There are different paths to viewing the assembly for different compilers, but essentially this must be done to ensure that you did not optimize out critical code.  You must also verify, via assembly, that you are comparing apples to apples.  
@@ -384,6 +432,16 @@ You will now be reporting statistics on the number of page faults that occurred
 
 A note on User Defined Measurements: This capability was introduced well after the creation of Celero.  While it is a great enhancement to the library, it was not designed-in to the library.  As such, the next major release of the library (v3.x) may change the way this is implemented and exposed to the library's users.  
 
+### Frequency Scaling
+
+CPU Frequency Scaling should be disabled if possible when executing benchmarks.  While there is code in Celero to attempt to do this, it may not have sufficient privileges to be effective.  On Linux systems, this can be accomplished as follows:
+
+```bash
+sudo cpupower frequency-set --governor performance
+./celeroBenchmarkExecutable
+sudo cpupower frequency-set --governor powersave
+```
+
 ### Notes
 
 - Benchmarks should always be performed on Release builds.  Never measure the performance of a Debug build and make changes based on the results.  The (optimizing) compiler is your friend with respect to code performance.
@@ -462,7 +520,7 @@ public:
 };
 ```
 
-Before the test fixture is utilized by a benchmark, Celero will create an instanciation of the class and call its "getExperimentValues()" function.  The test fixture can then build a vector of TestFixture::ExperimentValue values.  For each value added to this array, benchmarks will be executed following calls to the "setUp" virtual function.  A new test fixture is created for each measurement.
+Before the test fixture is utilized by a benchmark, Celero will create an instantiation of the class and call its "getExperimentValues()" function.  The test fixture can then build a vector of TestFixture::ExperimentValue values.  For each value added to this array, benchmarks will be executed following calls to the "setUp" virtual function.  A new test fixture is created for each measurement.
 
 The `SetUp()` virtual function is called before each benchmark test is executed. When using a problem space values vector, the function will be given a value that was previously pushed into the array within the constructor. The function's code can then decide what to do with it. Here, we are using the value to indicate how many elements should be in the array that we intend to sort. For each of the array elements, we simply add a pseudo-random integer.
 

diff --git a/cmake/FindCelero.cmake b/cmake/FindCelero.cmake
@@ -0,0 +1,52 @@
+# Try to find Celero headers and libraries.
+#
+# Usage of this module as follows:
+#
+#     find_package(Celero)
+#
+# Variables used by this module, they can change the default behaviour and need
+# to be set before calling find_package:
+#
+#  CELERO_PREFIX         Set this variable to the root installation of
+#                        celero if the module has problems finding the
+#                        proper installation path.
+#
+# Note that you can set the CELERO_PREFIX variable as environment variable.
+#
+# Variables defined by this module:
+#
+#  CELERO_FOUND              System has the Celero library and headers
+#  CELERO_LIBRARIES          The Celero library
+#  CELERO_INCLUDE_DIRS       The location of Celero headers
+
+if(NOT CELRO_PREFIX)
+	set(CELERO_PREFIX $ENV{CELERO_PREFIX})
+endif()
+
+set(CELERO_HINTS ${CELERO_PREFIX} $ENV{CELERO_PREFIX})
+find_path(CELERO_ROOT
+  NAMES include/celero/Celero.h
+	HINTS "${CELERO_HINTS}"
+)
+
+find_library(CELERO_LIBRARIES
+    NAMES libcelero.so
+    HINTS ${CELERO_ROOT}/lib ${CELERO_ROOT}/lib64 
+)
+
+find_path(CELERO_INCLUDE_DIRS
+    NAMES celero/Celero.h
+    HINTS ${CELERO_ROOT}/include
+)
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(CELERO DEFAULT_MSG
+    CELERO_LIBRARIES
+    CELERO_INCLUDE_DIRS
+)
+
+mark_as_advanced(
+    CELERO_LIBRARIES
+    CELERO_INCLUDE_DIRS
+    CELERO_ROOT
+)
diff --git a/include/celero/Archive.h b/include/celero/Archive.h
@@ -4,7 +4,7 @@
 ///
 /// \author	John Farrier
 ///
-/// \copyright Copyright 2015, 2016, 2017, 2018 John Farrier
+/// \copyright Copyright 2015, 2016, 2017, 2018. 2019 John Farrier
 ///
 /// Licensed under the Apache License, Version 2.0 (the "License");
 /// you may not use this file except in compliance with the License.

diff --git a/include/celero/Benchmark.h b/include/celero/Benchmark.h
@@ -4,7 +4,7 @@
 ///
 /// \author	John Farrier
 ///
-/// \copyright Copyright 2015, 2016, 2017, 2018 John Farrier
+/// \copyright Copyright 2015, 2016, 2017, 2018. 2019 John Farrier
 ///
 /// Licensed under the Apache License, Version 2.0 (the "License");
 /// you may not use this file except in compliance with the License.

diff --git a/include/celero/Callbacks.h b/include/celero/Callbacks.h
@@ -6,7 +6,7 @@
 ///
 /// \author	John Farrier
 ///
-/// \copyright Copyright 2015, 2016, 2017, 2018 John Farrier
+/// \copyright Copyright 2015, 2016, 2017, 2018. 2019 John Farrier
 ///
 /// Licensed under the Apache License, Version 2.0 (the "License");
 /// you may not use this file except in compliance with the License.

diff --git a/include/celero/Celero.h b/include/celero/Celero.h
@@ -6,7 +6,7 @@
 ///
 /// \author	John Farrier
 ///
-/// \copyright Copyright 2015, 2016, 2017, 2018 John Farrier
+/// \copyright Copyright 2015, 2016, 2017, 2018. 2019 John Farrier
 ///
 /// Licensed under the Apache License, Version 2.0 (the "License");
 /// you may not use this file except in compliance with the License.

diff --git a/include/celero/Console.h b/include/celero/Console.h
@@ -4,7 +4,7 @@
 ///
 /// \author	John Farrier
 ///
-/// \copyright Copyright 2015, 2016, 2017, 2018 John Farrier
+/// \copyright Copyright 2015, 2016, 2017, 2018. 2019 John Farrier
 ///
 /// Licensed under the Apache License, Version 2.0 (the "License");
 /// you may not use this file except in compliance with the License.

diff --git a/include/celero/Distribution.h b/include/celero/Distribution.h
@@ -4,7 +4,7 @@
 ///
 /// \author	John Farrier
 ///
-/// \copyright Copyright 2015, 2016, 2017, 2018 John Farrier
+/// \copyright Copyright 2015, 2016, 2017, 2018. 2019 John Farrier
 ///
 /// Licensed under the Apache License, Version 2.0 (the "License");
 /// you may not use this file except in compliance with the License.

diff --git a/include/celero/Exceptions.h b/include/celero/Exceptions.h
@@ -4,7 +4,7 @@
 ///
 /// \author	Peter Azmanov
 ///
-/// \copyright Copyright 2015, 2016, 2017, 2018 John Farrier
+/// \copyright Copyright 2015, 2016, 2017, 2018. 2019 John Farrier
 ///
 /// Licensed under the Apache License, Version 2.0 (the "License");
 /// you may not use this file except in compliance with the License.

diff --git a/include/celero/Executor.h b/include/celero/Executor.h
@@ -4,7 +4,7 @@
 ///
 /// \author	John Farrier
 ///
-/// \copyright Copyright 2015, 2016, 2017, 2018 John Farrier
+/// \copyright Copyright 2015, 2016, 2017, 2018. 2019 John Farrier
 ///
 /// Licensed under the Apache License, Version 2.0 (the "License");
 /// you may not use this file except in compliance with the License.

diff --git a/include/celero/Experiment.h b/include/celero/Experiment.h
@@ -4,7 +4,7 @@
 ///
 /// \author	John Farrier
 ///
-/// \copyright Copyright 2015, 2016, 2017, 2018 John Farrier
+/// \copyright Copyright 2015, 2016, 2017, 2018. 2019 John Farrier
 ///
 /// Licensed under the Apache License, Version 2.0 (the "License");
 /// you may not use this file except in compliance with the License.

diff --git a/include/celero/ExperimentResult.h b/include/celero/ExperimentResult.h
@@ -4,7 +4,7 @@
 ///
 /// \author	John Farrier
 ///
-/// \copyright Copyright 2015, 2016, 2017, 2018 John Farrier
+/// \copyright Copyright 2015, 2016, 2017, 2018. 2019 John Farrier
 ///
 /// Licensed under the Apache License, Version 2.0 (the "License");
 /// you may not use this file except in compliance with the License.