Merge pull request #5 from borasoftware/features/test-runner-improvem…

…ents

Test runner report generation.

CMakeLists.txt

@@ -20,7 +20,7 @@ cmake_minimum_required(VERSION 3.10.2)
 #  - BalauConfigVersion.cmake
 #  - title-page.bdml
 #
-project(Balau VERSION 2019.5.3)
+project(Balau VERSION 2019.6.1)
 
 configure_file(src/doc/api/Doxyfile.in Doxyfile)
 
@@ -598,6 +598,7 @@ set(BALAU_SOURCE_FILES
 	src/main/cpp/Balau/Testing/Impl/TestMethodBase.hpp
 	src/main/cpp/Balau/Testing/Impl/TestResult.hpp
 	src/main/cpp/Balau/Testing/Impl/TestResultQueue.hpp
+	src/main/cpp/Balau/Testing/Impl/TestReportGenerator.hpp
 	src/main/cpp/Balau/Testing/Impl/TestRunnerExecutor.hpp
 	src/main/cpp/Balau/Testing/Impl/TestRunnerLimits.hpp
 	src/main/cpp/Balau/Testing/Impl/WorkerProcessesTestRunnerExecutor.hpp
@@ -614,6 +615,8 @@ set(BALAU_SOURCE_FILES
 	src/main/cpp/Balau/Testing/Renderers/HexSignedLongRenderer.hpp
 	src/main/cpp/Balau/Testing/Renderers/HexUnsignedLongLongRenderer.hpp
 	src/main/cpp/Balau/Testing/Renderers/HexSignedLongLongRenderer.hpp
+	src/main/cpp/Balau/Testing/Reporters/SurefireTestReportGenerator.cpp
+	src/main/cpp/Balau/Testing/Reporters/SurefireTestReportGenerator.hpp
 	src/main/cpp/Balau/Testing/Util/NetworkTesting.cpp
 	src/main/cpp/Balau/Testing/Util/NetworkTesting.hpp
 	src/main/cpp/Balau/Type/Character.hpp

src/doc/manual/NonCode/Releases.bdml

@@ -33,6 +33,16 @@
 	<chapter title="Release history">
 		<para>Balau uses a continuous integration methodology, with a Jenkins build running on Bora's servers. Use of the head of the master branch should thus provide a stable library. Release tags are also created periodically in order to provide reference versions to link against.</para>
 
+		<h1>V2019.6.1</h1>
+
+		<para>New features in this release:</para>
+
+		<bullets>
+			<entry>XML based test report generation;</entry>
+			<entry>test runner now uses SSV style command line options;</entry>
+			<entry>command line parser supports multiple final values in SSV mode.</entry>
+		</bullets>
+
 		<h1>V2019.5.3</h1>
 
 		<para>New features in this release:</para>

src/doc/manual/Testing/TestRunner.bdml

@@ -67,52 +67,55 @@
 		<code lang="C++">
 			// Example test group from the Balau unit tests.
 
-			struct CommandLineTest : public Testing::TestGroup&lt;CommandLineTest&gt; {
-				CommandLineTest() {
-					registerTest(&amp;CommandLineTest::basicTest,        "basicTest");
-					registerTest(&amp;CommandLineTest::failingTest,      "failingTest");
-					registerTest(&amp;CommandLineTest::finalValueTest,   "finalValueTest");
-					registerTest(&amp;CommandLineTest::numericValueTest, "numericValueTest");
+			struct ObjectTrieTest : public Testing::TestGroup&lt;ObjectTrieTest&gt; {
+				ObjectTrieTest() {
+					registerTest(&amp;ObjectTrieTest::uIntTrieBuild,               "uIntTrieBuild");
+					registerTest(&amp;ObjectTrieTest::uIntTrieCopy,                "uIntTrieCopy");
+					registerTest(&amp;ObjectTrieTest::uIntTreeDepthIterate,        "uIntTreeDepthIterate");
+					registerTest(&amp;ObjectTrieTest::uIntTreeDepthIterateForLoop, "uIntTreeDepthIterateForLoop");
+					registerTest(&amp;ObjectTrieTest::uIntTreeBreadthIterate,      "uIntTreeBreadthIterate");
+					registerTest(&amp;ObjectTrieTest::fluentBuild,                 "fluentBuild");
 				}
 
-				void basicTest();
-				void failingTest();
-				void finalValueTest();
-				void numericValueTest();
+				void uIntTrieBuild();
+				void uIntTrieCopy();
+				void uIntTreeDepthIterate();
+				void uIntTreeDepthIterateForLoop();
+				void uIntTreeBreadthIterate();
+				void fluentBuild();
 			};
 		</code>
 
 		<h2>Test application</h2>
 
 		<para>The test application executes the test runner by calling the test runner's <emph>run</emph> method. Within the main function, the test runner is initialised via one of the runner's initialisation methods. The most common <emph>run</emph> methods to use are the ones that take <emph>argc</emph> and <emph>argv</emph> arguments. The <emph>argc</emph> and <emph>argv</emph> arguments of the test application's main function are passed to the test runner's <emph>run</emph> method, in order to parse the command line arguments.</para>
 
-		<para>The first <emph>run</emph> method that takes <emph>argc</emph> and <emph>argv</emph> arguments also accepts an execution model:</para>
-
 		<code lang="C++">
 			#include &lt;Balau/Testing/TestRunner.hpp&gt;
 
 			using namespace Balau::Testing;
 
 			int main(int argc, char * argv[]) {
-				// Run the tests with the worker processes execution model.
-				return TestRunner::run(WorkerProcesses, argc, argv);
+				return TestRunner::run(argc, argv);
 			}
 		</code>
 
-		<para>The test runner will interpret the command line arguments as a space/comma delimited list of globbed test names to run.</para>
-
-		<para>The second <emph>run</emph> method that takes <emph>argc</emph> and <emph>argv</emph> arguments does not have an execution model input parameter:</para>
+		<para>The test runner uses the command line parser to parse a space separated command line which can also contain globbed test name pattners to run.</para>
 
-		<code lang="C++">
-			#include &lt;Balau/Testing/TestRunner.hpp&gt;
+		<para>The following command line options are available.</para>
 
-			using namespace Balau::Testing;
+		<table class="bdml-table161612L56">
+			<head> <cell>Short option</cell> <cell>Long option</cell> <cell>Has value</cell> <cell>Description</cell> </head>
 
-			int main(int argc, char * argv[]) {
-				// Run the tests with the execution model specified as the first element of argv.
-				return TestRunner::run(argc, argv);
-			}
-		</code>
+			<body>
+				<row> <cell>-e</cell> <cell>--execution-model</cell> <cell>yes</cell> <cell>The execution model (default = SingleThreaded).</cell> </row>
+				<row> <cell>-n</cell> <cell>--namespaces</cell>      <cell>no</cell>  <cell>Use namespaces in test group names (default is not to use).</cell> </row>
+				<row> <cell>-p</cell> <cell>--pause</cell>           <cell>no</cell>  <cell>Pause at exit (default is not to pause).</cell> </row>
+				<row> <cell>-c</cell> <cell>--concurrency</cell>     <cell>yes</cell> <cell>The number of threads or processes to use to run the tests (default = detect).</cell> </row>
+				<row> <cell>-r</cell> <cell>--report-folder</cell>   <cell>yes</cell> <cell>Generate test reports in the specified folder.</cell> </row>
+				<row> <cell>-h</cell> <cell>--help</cell>            <cell>no</cell>  <cell>Displays this help message.</cell> </row>
+			</body>
+		</table>
 
 		<para>The test runner will interpret the first element of <emph>argv</emph> as the execution model (case insensitive) if it is a valid execution model, and the remainder of the command line arguments as a space/comma delimited list of globbed test names to run. If the first element of <emph>argv</emph> is not a valid execution model, it will form the head of the globbed test name list and the <emph>SingleThreaded</emph> execution model will be used by default.</para>
 
@@ -138,11 +141,11 @@
 		<code lang="C++">
 			# Run the Balau test application with the worker processes execution model
 			# and specifying a subset of tests via a comma delimited list of patterns.
-			BalauTests WorkerProcesses Injector::*,Environment::*
+			BalauTests -e WorkerProcesses Injector::*,Environment::*
 
 			# Run the Balau test application with the worker processes execution model
 			# and specifying a subset of tests via a space delimited list of patterns.
-			BalauTests WorkerProcesses Injector::* Environment::*
+			BalauTests -e WorkerProcesses Injector::* Environment::*
 		</code>
 
 		<h3>Execution models</h3>
@@ -463,6 +466,12 @@
 
 		<para>More information on logging system configuration is available in the <ref url="Logging/Logger">Logging</ref> documentation.</para>
 
+		<h2>Test reports</h2>
+
+		<para>In addition to test result logging, the test runner can be configured to generate XML based test run report files.</para>
+
+		<para>The default reporter provides Maven Surefire plugin schema based XML reports. Alternative report generators may be defined by deriving from the <emph>TestReportGenerator</emph> base class and providing an instance of the reporter class to the test runner <emph>run</emph> function.</para>
+
 		<h1>Test utilities</h1>
 
 		<para>The test framework utilities are designed to provide domain specific help in accomplishing certain management tasks required during testing. The test framework utilities are in an early stage of development, and currently only a pair of network related test utility functions are defined.</para>
@@ -548,15 +557,17 @@
 		<code lang="C++">
 			# Run the Balau test application with the worker processes execution model
 			# and specifying a subset of tests via a comma delimited list of patterns.
-			BalauTests WorkerProcesses Injector*,Environment*
+			BalauTests -e WorkerProcesses Injector*,Environment*
 
 			# Run the Balau test application with the worker processes execution model
 			# and specifying a subset of tests via a space delimited list of patterns.
-			BalauTests WorkerProcesses Injector* Environment*
+			BalauTests -e WorkerProcesses Injector* Environment*
 		</code>
 
 		<h2>Model selection</h2>
 
+		<para>TODO update documentation to reflect command line parsing</para>
+
 		<para>The test execution model to run can be specified either as the first argument of the command line by calling the <emph>TestRunner::run(argc, argv)</emph> method, or explicitly by using other <emph>run</emph> method overloads, most commonly the <emph>TestRunner::run(model, argc, argv)</emph> overload.</para>
 
 		<code lang="C++">

src/doc/manual/index.bdml

@@ -236,7 +236,7 @@
 			</body>
 		</table>
 
-		<para>It is anticipated that these items will be completed by May 2019.</para>
+		<para>It is anticipated that these items will be completed by July 2019.</para>
 
 		<section class="notices">
 			<para class="notice-body">* Configuration of the browser's security settings may be required. See the documentation on <ref type="raw" new="true" url="https://borasoftware.com/doc/boradoc/latest/manual/direct-loading.html">direct loading</ref> for more information.</para>

src/doc/single-page/pdf/title-page.bdml

@@ -13,7 +13,7 @@
 <document xmlns="http://boradoc.org/1.0">
 	<chapter title="Balau core C++ library" id="bdml-title-page">
 		<para><strong>User manual</strong></para>
-		<para>Version 2019.5.3</para>
+		<para>Version 2019.6.1</para>
 		<para class="bdml-strong">Bora Software</para>
 	</chapter>
 </document>

src/main/cmake/BalauConfig.cmake

@@ -1,4 +1,4 @@
 
 get_filename_component(Balau_INSTALL_PREFIX "${CMAKE_CURRENT_LIST_DIR}/../../" ABSOLUTE)
-set(Balau_INCLUDE_DIRS ${Balau_INSTALL_PREFIX}/include/Balau-2019.5.3)
-set(Balau_LIBRARY ${Balau_INSTALL_PREFIX}/lib/Balau-2019.5.3/libBalau.a)
+set(Balau_INCLUDE_DIRS ${Balau_INSTALL_PREFIX}/include/Balau-2019.6.1)
+set(Balau_LIBRARY ${Balau_INSTALL_PREFIX}/lib/Balau-2019.6.1/libBalau.a)

src/main/cmake/BalauConfigVersion.cmake

@@ -1,14 +1,14 @@
 
-set(PACKAGE_VERSION 2019.5.3)
+set(PACKAGE_VERSION 2019.6.1)
 
 if(NOT "${PACKAGE_FIND_VERSION_COUNT}" EQUAL 3)
 	set(PACKAGE_VERSION_UNSUITABLE 1)
 	set(PACKAGE_VERSION_COMPATIBLE 0)
 elseif("${PACKAGE_FIND_VERSION_MAJOR}" EQUAL 2019)
-	if("${PACKAGE_FIND_VERSION_MINOR}" EQUAL 5)
+	if("${PACKAGE_FIND_VERSION_MINOR}" EQUAL 6)
 		set(PACKAGE_VERSION_COMPATIBLE 1)
 
-		if("${PACKAGE_FIND_VERSION_PATCH}" EQUAL 3)
+		if("${PACKAGE_FIND_VERSION_PATCH}" EQUAL 1)
 			set(PACKAGE_VERSION_EXACT 1)
 		endif()
 	endif()