Skip to content
Permalink
Browse files

Various improvements to the benchmarking support

* Units from <ratio> are no longer redeclared in our own namespace
* The default clock is `steady_clock`, not `high_resolution_clock`,
because, as HH says "high_resolution_clock is useless. If you want
measure the passing of time, use steady_clock. If you want user
friendly time, use system_clock".
* Benchmarking support is opt-in, not opt-out, to avoid the large
(~10%) compile time penalty.
* Benchmarking-related options in CLI are always present, to decrease
the amount of code that is only compiled conditionally and making
the whole shebang more maintainble.
  • Loading branch information...
horenmar committed May 19, 2019
1 parent ce2560c commit e340ab8db6dbe07a58b6e72a9364d67d2178293c
@@ -149,7 +149,7 @@ by using `_NO_` in the macro, e.g. `CATCH_CONFIG_NO_CPP17_UNCAUGHT_EXCEPTIONS`.
CATCH_CONFIG_DISABLE // Disables assertions and test case registration
CATCH_CONFIG_WCHAR // Enables use of wchart_t
CATCH_CONFIG_EXPERIMENTAL_REDIRECT // Enables the new (experimental) way of capturing stdout/stderr
CATCH_CONFIG_DISABLE_BENCHMARKING // Disables the compile-time heavy benchmarking features
CATCH_CONFIG_ENABLE_BENCHMARKING // Enables the integrated benchmarking features (has a significant effect on compilation speed)

Currently Catch enables `CATCH_CONFIG_WINDOWS_SEH` only when compiled with MSVC, because some versions of MinGW do not have the necessary Win32 API support.

@@ -33,9 +33,6 @@
# if defined(CATCH_CONFIG_DISABLE_MATCHERS)
# undef CATCH_CONFIG_DISABLE_MATCHERS
# endif
# if defined(CATCH_CONFIG_DISABLE_BENCHMARKING)
# undef CATCH_CONFIG_DISABLE_BENCHMARKING
# endif
# if !defined(CATCH_CONFIG_ENABLE_CHRONO_STRINGMAKER)
# define CATCH_CONFIG_ENABLE_CHRONO_STRINGMAKER
# endif
@@ -77,14 +74,14 @@
#include "internal/catch_objc.hpp"
#endif

#ifndef CATCH_CONFIG_DISABLE_BENCHMARKING
#include "internal/benchmark/catch_benchmark.hpp"
#endif

#ifdef CATCH_CONFIG_EXTERNAL_INTERFACES
#include "internal/catch_external_interfaces.h"
#endif

#if defined(CATCH_CONFIG_ENABLE_BENCHMARKING)
#include "internal/benchmark/catch_benchmark.hpp"
#endif

#endif // ! CATCH_CONFIG_IMPL_ONLY

#ifdef CATCH_IMPL
@@ -195,12 +192,12 @@
#define CATCH_THEN( desc ) INTERNAL_CATCH_DYNAMIC_SECTION( " Then: " << desc )
#define CATCH_AND_THEN( desc ) INTERNAL_CATCH_DYNAMIC_SECTION( " And: " << desc )

#ifndef CATCH_CONFIG_DISABLE_BENCHMARKING
#if defined(CATCH_CONFIG_ENABLE_BENCHMARKING)
#define CATCH_BENCHMARK(...) \
INTERNAL_CATCH_BENCHMARK(INTERNAL_CATCH_UNIQUE_NAME(____C_A_T_C_H____B_E_N_C_H____), INTERNAL_CATCH_GET_1_ARG(__VA_ARGS__,,), INTERNAL_CATCH_GET_2_ARG(__VA_ARGS__,,))
#define CATCH_BENCHMARK_ADVANCED(name) \
INTERNAL_CATCH_BENCHMARK_ADVANCED(INTERNAL_CATCH_UNIQUE_NAME(____C_A_T_C_H____B_E_N_C_H____), name)
#endif // CATCH_CONFIG_DISABLE_BENCHMARKING
#endif // CATCH_CONFIG_ENABLE_BENCHMARKING

// If CATCH_CONFIG_PREFIX_ALL is not defined then the CATCH_ prefix is not required
#else
@@ -297,12 +294,12 @@
#define THEN( desc ) INTERNAL_CATCH_DYNAMIC_SECTION( " Then: " << desc )
#define AND_THEN( desc ) INTERNAL_CATCH_DYNAMIC_SECTION( " And: " << desc )

#ifndef CATCH_CONFIG_DISABLE_BENCHMARKING
#if defined(CATCH_CONFIG_ENABLE_BENCHMARKING)
#define BENCHMARK(...) \
INTERNAL_CATCH_BENCHMARK(INTERNAL_CATCH_UNIQUE_NAME(____C_A_T_C_H____B_E_N_C_H____), INTERNAL_CATCH_GET_1_ARG(__VA_ARGS__,,), INTERNAL_CATCH_GET_2_ARG(__VA_ARGS__,,))
#define BENCHMARK_ADVANCED(name) \
INTERNAL_CATCH_BENCHMARK_ADVANCED(INTERNAL_CATCH_UNIQUE_NAME(____C_A_T_C_H____B_E_N_C_H____), name)
#endif // CATCH_CONFIG_DISABLE_BENCHMARKING
#endif // CATCH_CONFIG_ENABLE_BENCHMARKING

using Catch::Detail::Approx;

@@ -16,12 +16,6 @@

namespace Catch {
namespace Benchmark {
template <unsigned Num, unsigned Den = 1>
using ratio = std::ratio<Num, Den>;
using milli = ratio<1, 1000>;
using micro = ratio<1, 1000000>;
using nano = ratio<1, 1000000000>;

template <typename Clock>
using ClockDuration = typename Clock::duration;
template <typename Clock>
@@ -30,7 +24,7 @@ namespace Catch {
template <typename Clock>
using TimePoint = typename Clock::time_point;

using default_clock = std::chrono::high_resolution_clock;
using default_clock = std::chrono::steady_clock;

template <typename Clock>
struct now {
@@ -39,7 +33,7 @@ namespace Catch {
}
};

using fp_seconds = std::chrono::duration<double, ratio<1>>;
using fp_seconds = std::chrono::duration<double, std::ratio<1>>;
} // namespace Benchmark
} // namespace Catch

@@ -196,7 +196,6 @@ namespace Catch {
| Opt( setWaitForKeypress, "start|exit|both" )
["--wait-for-keypress"]
( "waits for a keypress before exiting" )
#ifndef CATCH_CONFIG_DISABLE_BENCHMARKING
| Opt( config.benchmarkSamples, "samples" )
["--benchmark-samples"]
( "number of samples to collect (default: 100)" )
@@ -209,7 +208,6 @@ namespace Catch {
| Opt( config.benchmarkNoAnalysis )
["--benchmark-no-analysis"]
( "perform only measurements; do not perform any analysis" )
#endif // CATCH_CONFIG_DISABLE_BENCHMARKING
| Arg( config.testsOrTags, "test name|pattern|tags" )
( "which test or tests to use" );

@@ -60,12 +60,10 @@ namespace Catch {
bool Config::showInvisibles() const { return m_data.showInvisibles; }
Verbosity Config::verbosity() const { return m_data.verbosity; }

#ifndef CATCH_CONFIG_DISABLE_BENCHMARKING
bool Config::benchmarkNoAnalysis() const { return m_data.benchmarkNoAnalysis; }
int Config::benchmarkSamples() const { return m_data.benchmarkSamples; }
double Config::benchmarkConfidenceInterval() const { return m_data.benchmarkConfidenceInterval; }
unsigned int Config::benchmarkResamples() const { return m_data.benchmarkResamples; }
#endif // CATCH_CONFIG_DISABLE_BENCHMARKING

IStream const* Config::openStream() {
return Catch::makeStream(m_data.outputFilename);
@@ -43,12 +43,10 @@ namespace Catch {
int abortAfter = -1;
unsigned int rngSeed = 0;

#ifndef CATCH_CONFIG_DISABLE_BENCHMARKING
bool benchmarkNoAnalysis = false;
unsigned int benchmarkSamples = 100;
double benchmarkConfidenceInterval = 0.95;
unsigned int benchmarkResamples = 100000;
#endif // CATCH_CONFIG_DISABLE_BENCHMARKING

Verbosity verbosity = Verbosity::Normal;
WarnAbout::What warnings = WarnAbout::Nothing;
@@ -111,12 +109,10 @@ namespace Catch {
int abortAfter() const override;
bool showInvisibles() const override;
Verbosity verbosity() const override;
#ifndef CATCH_CONFIG_DISABLE_BENCHMARKING
bool benchmarkNoAnalysis() const override;
int benchmarkSamples() const override;
double benchmarkConfidenceInterval() const override;
unsigned int benchmarkResamples() const override;
#endif // CATCH_CONFIG_DISABLE_BENCHMARKING

private:

@@ -29,11 +29,11 @@ namespace Catch {
struct ITransientExpression;
struct IGeneratorTracker;

#ifndef CATCH_CONFIG_DISABLE_BENCHMARKING
#if defined(CATCH_CONFIG_ENABLE_BENCHMARKING)
struct BenchmarkInfo;
template <typename Duration = std::chrono::duration<double, std::nano>>
struct BenchmarkStats;
#endif // CATCH_CONFIG_DISABLE_BENCHMARKING
#endif // CATCH_CONFIG_ENABLE_BENCHMARKING

struct IResultCapture {

@@ -46,12 +46,12 @@ namespace Catch {

virtual auto acquireGeneratorTracker( SourceLineInfo const& lineInfo ) -> IGeneratorTracker& = 0;

#ifndef CATCH_CONFIG_DISABLE_BENCHMARKING
#if defined(CATCH_CONFIG_ENABLE_BENCHMARKING)
virtual void benchmarkPreparing( std::string const& name ) = 0;
virtual void benchmarkStarting( BenchmarkInfo const& info ) = 0;
virtual void benchmarkEnded( BenchmarkStats<> const& stats ) = 0;
virtual void benchmarkFailed( std::string const& error ) = 0;
#endif // CATCH_CONFIG_DISABLE_BENCHMARKING
#endif // CATCH_CONFIG_ENABLE_BENCHMARKING

virtual void pushScopedMessage( MessageInfo const& message ) = 0;
virtual void popScopedMessage( MessageInfo const& message ) = 0;
@@ -77,12 +77,10 @@ namespace Catch {
virtual std::vector<std::string> const& getSectionsToRun() const = 0;
virtual Verbosity verbosity() const = 0;

#ifndef CATCH_CONFIG_DISABLE_BENCHMARKING
virtual bool benchmarkNoAnalysis() const = 0;
virtual int benchmarkSamples() const = 0;
virtual double benchmarkConfidenceInterval() const = 0;
virtual unsigned int benchmarkResamples() const = 0;
#endif // CATCH_CONFIG_DISABLE_BENCHMARKING
};

using IConfigPtr = std::shared_ptr<IConfig const>;
@@ -18,10 +18,10 @@
#include "catch_option.hpp"
#include "catch_stringref.h"

#ifndef CATCH_CONFIG_DISABLE_BENCHMARKING
#if defined(CATCH_CONFIG_ENABLE_BENCHMARKING)
#include "benchmark/catch_estimate.hpp"
#include "benchmark/catch_outlier_classification.hpp"
#endif // CATCH_CONFIG_DISABLE_BENCHMARKING
#endif // CATCH_CONFIG_ENABLE_BENCHMARKING


#include <string>
@@ -165,7 +165,7 @@ namespace Catch {
bool aborting;
};

#ifndef CATCH_CONFIG_DISABLE_BENCHMARKING
#if defined(CATCH_CONFIG_ENABLE_BENCHMARKING)
struct BenchmarkInfo {
std::string name;
double estimatedDuration;
@@ -201,7 +201,7 @@ namespace Catch {
};
}
};
#endif // CATCH_CONFIG_DISABLE_BENCHMARKING
#endif // CATCH_CONFIG_ENABLE_BENCHMARKING

struct IStreamingReporter {
virtual ~IStreamingReporter() = default;
@@ -220,12 +220,12 @@ namespace Catch {
virtual void testCaseStarting( TestCaseInfo const& testInfo ) = 0;
virtual void sectionStarting( SectionInfo const& sectionInfo ) = 0;

#ifndef CATCH_CONFIG_DISABLE_BENCHMARKING
#if defined(CATCH_CONFIG_ENABLE_BENCHMARKING)
virtual void benchmarkPreparing( std::string const& ) {}
virtual void benchmarkStarting( BenchmarkInfo const& ) {}
virtual void benchmarkEnded( BenchmarkStats<> const& ) {}
virtual void benchmarkFailed( std::string const& ) {}
#endif // CATCH_CONFIG_DISABLE_BENCHMARKING
#endif // CATCH_CONFIG_ENABLE_BENCHMARKING

virtual void assertionStarting( AssertionInfo const& assertionInfo ) = 0;

@@ -231,7 +231,7 @@ namespace Catch {
m_unfinishedSections.push_back(endInfo);
}

#ifndef CATCH_CONFIG_DISABLE_BENCHMARKING
#if defined(CATCH_CONFIG_ENABLE_BENCHMARKING)
void RunContext::benchmarkPreparing(std::string const& name) {
m_reporter->benchmarkPreparing(name);
}
@@ -244,7 +244,7 @@ namespace Catch {
void RunContext::benchmarkFailed(std::string const & error) {
m_reporter->benchmarkFailed(error);
}
#endif // CATCH_CONFIG_DISABLE_BENCHMARKING
#endif // CATCH_CONFIG_ENABLE_BENCHMARKING

void RunContext::pushScopedMessage(MessageInfo const & message) {
m_messages.push_back(message);
@@ -82,12 +82,12 @@ namespace Catch {

auto acquireGeneratorTracker( SourceLineInfo const& lineInfo ) -> IGeneratorTracker& override;

#ifndef CATCH_CONFIG_DISABLE_BENCHMARKING
#if defined(CATCH_CONFIG_ENABLE_BENCHMARKING)
void benchmarkPreparing( std::string const& name ) override;
void benchmarkStarting( BenchmarkInfo const& info ) override;
void benchmarkEnded( BenchmarkStats<> const& stats ) override;
void benchmarkFailed( std::string const& error ) override;
#endif // CATCH_CONFIG_DISABLE_BENCHMARKING
#endif // CATCH_CONFIG_ENABLE_BENCHMARKING

void pushScopedMessage( MessageInfo const& message ) override;
void popScopedMessage( MessageInfo const& message ) override;
@@ -20,10 +20,16 @@
#if defined(_MSC_VER)
#pragma warning(push)
#pragma warning(disable:4061) // Not all labels are EXPLICITLY handled in switch
// Note that 4062 (not all labels are handled
// and default is missing) is enabled
// Note that 4062 (not all labels are handled and default is missing) is enabled
#endif

#if defined(__clang__)
# pragma clang diagnostic push
// For simplicity, benchmarking-only helpers are always enabled
# pragma clang diagnostic ignored "-Wunused-function"
#endif



namespace Catch {

@@ -287,7 +293,7 @@ class TablePrinter {
if (!m_isOpen) {
m_isOpen = true;
*this << RowBreak();

Columns headerCols;
Spacer spacer(2);
for (auto const& info : m_columnInfos) {
@@ -408,7 +414,7 @@ void ConsoleReporter::sectionEnded(SectionStats const& _sectionStats) {
StreamingReporterBase::sectionEnded(_sectionStats);
}

#ifndef CATCH_CONFIG_DISABLE_BENCHMARKING
#if defined(CATCH_CONFIG_ENABLE_BENCHMARKING)
void ConsoleReporter::benchmarkPreparing(std::string const& name) {
lazyPrintWithoutClosingBenchmarkTable();

@@ -446,7 +452,7 @@ void ConsoleReporter::benchmarkFailed(std::string const& error) {
<< "Benchmark failed (" << error << ")"
<< ColumnBreak() << RowBreak();
}
#endif // CATCH_CONFIG_DISABLE_BENCHMARKING
#endif // CATCH_CONFIG_ENABLE_BENCHMARKING

void ConsoleReporter::testCaseEnded(TestCaseStats const& _testCaseStats) {
m_tablePrinter->close();
@@ -665,3 +671,7 @@ CATCH_REGISTER_REPORTER("console", ConsoleReporter)
#if defined(_MSC_VER)
#pragma warning(pop)
#endif

#if defined(__clang__)
# pragma clang diagnostic pop
#endif
@@ -39,12 +39,12 @@ namespace Catch {
void sectionStarting(SectionInfo const& _sectionInfo) override;
void sectionEnded(SectionStats const& _sectionStats) override;

#ifndef CATCH_CONFIG_DISABLE_BENCHMARKING
#if defined(CATCH_CONFIG_ENABLE_BENCHMARKING)
void benchmarkPreparing(std::string const& name) override;
void benchmarkStarting(BenchmarkInfo const& info) override;
void benchmarkEnded(BenchmarkStats<> const& stats) override;
void benchmarkFailed(std::string const& error) override;
#endif // CATCH_CONFIG_DISABLE_BENCHMARKING
#endif // CATCH_CONFIG_ENABLE_BENCHMARKING

void testCaseEnded(TestCaseStats const& _testCaseStats) override;
void testGroupEnded(TestGroupStats const& _testGroupStats) override;
@@ -42,7 +42,7 @@ namespace Catch {
m_reporter->noMatchingTestCases( spec );
}

#ifndef CATCH_CONFIG_DISABLE_BENCHMARKING
#if defined(CATCH_CONFIG_ENABLE_BENCHMARKING)
void ListeningReporter::benchmarkPreparing( std::string const& name ) {
for (auto const& listener : m_listeners) {
listener->benchmarkPreparing(name);
@@ -68,7 +68,7 @@ namespace Catch {
}
m_reporter->benchmarkFailed(error);
}
#endif // CATCH_CONFIG_DISABLE_BENCHMARKING
#endif // CATCH_CONFIG_ENABLE_BENCHMARKING

void ListeningReporter::testRunStarting( TestRunInfo const& testRunInfo ) {
for ( auto const& listener : m_listeners ) {
@@ -31,12 +31,12 @@ namespace Catch {

static std::set<Verbosity> getSupportedVerbosities();

#ifndef CATCH_CONFIG_DISABLE_BENCHMARKING
#if defined(CATCH_CONFIG_ENABLE_BENCHMARKING)
void benchmarkPreparing(std::string const& name) override;
void benchmarkStarting( BenchmarkInfo const& benchmarkInfo ) override;
void benchmarkEnded( BenchmarkStats<> const& benchmarkStats ) override;
void benchmarkFailed(std::string const&) override;
#endif // CATCH_CONFIG_DISABLE_BENCHMARKING
#endif // CATCH_CONFIG_ENABLE_BENCHMARKING

void testRunStarting( TestRunInfo const& testRunInfo ) override;
void testGroupStarting( GroupInfo const& groupInfo ) override;
@@ -219,7 +219,7 @@ namespace Catch {
m_xml.endElement();
}

#ifndef CATCH_CONFIG_DISABLE_BENCHMARKING
#if defined(CATCH_CONFIG_ENABLE_BENCHMARKING)
void XmlReporter::benchmarkStarting(BenchmarkInfo const &info) {
m_xml.startElement("BenchmarkResults")
.writeAttribute("name", info.name)
@@ -259,7 +259,7 @@ namespace Catch {
writeAttribute("message", error);
m_xml.endElement();
}
#endif // CATCH_CONFIG_DISABLE_BENCHMARKING
#endif // CATCH_CONFIG_ENABLE_BENCHMARKING

CATCH_REGISTER_REPORTER( "xml", XmlReporter )

0 comments on commit e340ab8

Please sign in to comment.
You can’t perform that action at this time.