Add support for additional configuration parameters

README.adoc

@@ -125,19 +125,38 @@ The engine requires at least JUnit Platform 1.5.x.
 
 === Configuration Parameters
 
-The following JUnit Platform https://junit.org/junit5/docs/current/user-guide/#running-tests-config-params[configuration parameters] are supported:
+The following JUnit Platform https://junit.org/junit5/docs/current/user-guide/#running-tests-config-params[configuration parameters] are supported.
 
-`testng.outputDirectory` (file path)::
-the output directory for reports (default: "test-output")
+==== Execution
+
+`testng.allowReturnValues` (boolean)::
+whether methods annotated with `@Test` that have return values should be considered test methods (default: `false`; see https://testng.org/doc/documentation-main.html#test-methods[documentation])
 +
-`testng.useDefaultListeners` (boolean)::
-whether TestNG's default report generating listeners should be used (default: `false`)
+`testng.dataProviderThreadCount` (integer)::
+maximum number of threads to use for running data providers in parallel, if enabled via `@DataProvider(parallel = true)` (default: `10`; see https://testng.org/doc/documentation-main.html#parameters-dataproviders[documentation])
 +
+`testng.parallel` (methods|tests|classes|instances|none)::
+TestNG's parallel execution mode for running tests in separate threads (default: `"none"`; see https://testng.org/doc/documentation-main.html#parallel-tests[documentation])
++
+`testng.preserveOrder` (boolean)::
+whether classes and methods should be run in a predictable order (default: `true`; see https://testng.org/doc/documentation-main.html#testng-xml[documentation])
++
+`testng.threadCount` (integer)::
+maximum number of threads for running tests in parallel, if enabled via `testng.parallel` (default: `5`; see https://testng.org/doc/documentation-main.html#parallel-tests[documentation])
+
+==== Reporting
+
 `testng.listeners` (comma-separated list of fully-qualified class names)::
-custom listeners that should be registered when executing tests (default: none)
+custom listeners that should be registered when executing tests (default: `""`; see https://testng.org/doc/documentation-main.html#testng-listeners[documentation])
++
+`testng.outputDirectory` (file path)::
+the output directory for reports (default: `"test-output"`; see https://testng.org/doc/documentation-main.html#running-testng[documentation])
++
+`testng.useDefaultListeners` (boolean)::
+whether TestNG's default report generating listeners should be used (default: `false`; see https://testng.org/doc/documentation-main.html#running-testng[documentation])
 +
 `testng.verbose` (integer)::
-TestNG's level of verbosity (default: 0)
+TestNG's level of verbosity for console output (default: `0`)
 
 === Generating TestNG reports
 

build.gradle.kts

@@ -153,7 +153,7 @@ tasks {
             classpath = configuration + sourceSets.testFixtures.get().output
             testClassesDirs = sourceSets.testFixtures.get().output
             useTestNG {
-                listeners.add("example.listeners.SystemPropertyProvidingListener")
+                listeners.add("example.configparams.SystemPropertyProvidingListener")
             }
         }
         register<Test>("testFixturesJUnitPlatform_${version.suffix}") {
@@ -163,7 +163,7 @@ tasks {
             useJUnitPlatform {
                 includeEngines("testng")
             }
-            systemProperty("testng.listeners", "example.listeners.SystemPropertyProvidingListener")
+            systemProperty("testng.listeners", "example.configparams.SystemPropertyProvidingListener")
             testLogging {
                 events = EnumSet.allOf(TestLogEvent::class.java)
             }

src/main/java/org/junit/support/testng/engine/ConfiguringListener.java

@@ -0,0 +1,31 @@
+/*
+ * Copyright 2021 the original author or authors.
+ *
+ * All rights reserved. This program and the accompanying materials are
+ * made available under the terms of the Eclipse Public License v2.0 which
+ * accompanies this distribution and is available at
+ *
+ * https://www.eclipse.org/legal/epl-v20.html
+ */
+
+package org.junit.support.testng.engine;
+
+import java.util.List;
+
+import org.junit.platform.engine.ConfigurationParameters;
+import org.testng.xml.XmlSuite;
+
+class ConfiguringListener extends DefaultListener {
+
+	private final ConfigurationParameters configurationParameters;
+
+	ConfiguringListener(ConfigurationParameters configurationParameters) {
+		this.configurationParameters = configurationParameters;
+	}
+
+	@Override
+	public void alter(List<XmlSuite> suites) {
+		configurationParameters.getBoolean("testng.allowReturnValues") //
+				.ifPresent(allowReturnValues -> suites.forEach(it -> it.setAllowReturnValues(allowReturnValues)));
+	}
+}

src/main/java/org/junit/support/testng/engine/DefaultListener.java

@@ -10,15 +10,23 @@
 
 package org.junit.support.testng.engine;
 
+import java.util.List;
+
+import org.testng.IAlterSuiteListener;
 import org.testng.IClassListener;
 import org.testng.IConfigurationListener;
 import org.testng.ITestClass;
 import org.testng.ITestContext;
 import org.testng.ITestListener;
 import org.testng.ITestNGMethod;
 import org.testng.ITestResult;
+import org.testng.xml.XmlSuite;
+
+abstract class DefaultListener implements IClassListener, ITestListener, IConfigurationListener, IAlterSuiteListener {
 
-abstract class DefaultListener implements IClassListener, ITestListener, IConfigurationListener {
+	@Override
+	public void alter(List<XmlSuite> suites) {
+	}
 
 	@Override
 	public void onBeforeClass(ITestClass testClass) {

src/main/java/org/junit/support/testng/engine/LoggingListener.java

@@ -10,12 +10,14 @@
 
 package org.junit.support.testng.engine;
 
+import java.util.List;
 import java.util.logging.Logger;
 
 import org.testng.ITestClass;
 import org.testng.ITestContext;
 import org.testng.ITestNGMethod;
 import org.testng.ITestResult;
+import org.testng.xml.XmlSuite;
 
 class LoggingListener extends DefaultListener {
 
@@ -25,6 +27,11 @@ class LoggingListener extends DefaultListener {
 	private LoggingListener() {
 	}
 
+	@Override
+	public void alter(List<XmlSuite> suites) {
+		LOGGER.fine(() -> "alter: " + suites);
+	}
+
 	@Override
 	public void onBeforeClass(ITestClass testClass) {
 		LOGGER.fine(() -> "onBeforeClass: " + testClass);

src/main/java/org/junit/support/testng/engine/TestNGTestEngine.java

@@ -10,13 +10,12 @@
 
 package org.junit.support.testng.engine;
 
+import static org.junit.support.testng.engine.TestNGTestEngine.Configurer.testClasses;
+import static org.junit.support.testng.engine.TestNGTestEngine.Configurer.testMethods;
 import static org.testng.internal.RuntimeBehavior.TESTNG_MODE_DRYRUN;
 
-import java.util.Arrays;
 import java.util.List;
 
-import org.junit.platform.commons.JUnitException;
-import org.junit.platform.commons.support.ReflectionSupport;
 import org.junit.platform.engine.ConfigurationParameters;
 import org.junit.platform.engine.EngineDiscoveryRequest;
 import org.junit.platform.engine.EngineExecutionListener;
@@ -28,6 +27,8 @@
 import org.testng.CommandLineArgs;
 import org.testng.ITestNGListener;
 import org.testng.TestNG;
+import org.testng.annotations.DataProvider;
+import org.testng.xml.XmlSuite.ParallelMode;
 
 /**
  * The TestNG {@link TestEngine} for running TestNG tests on the JUnit Platform.
@@ -86,15 +87,13 @@ public TestDescriptor discover(EngineDiscoveryRequest request, UniqueId uniqueId
 		DiscoveryListener listener = new DiscoveryListener(engineDescriptor);
 
 		if (testClasses.length > 0) {
-			TestNG testNG = createTestNGForTestClasses(testClasses);
 			withTemporarySystemProperty(TESTNG_MODE_DRYRUN, "true",
-				() -> configureAndRun(testNG, Phase.DISCOVERY, configurationParameters, listener));
+				() -> configureAndRun(configurationParameters, listener, testClasses(testClasses), Phase.DISCOVERY));
 		}
 
 		if (!methodNames.isEmpty()) {
-			TestNG testNG = createTestNGForTestMethods(methodNames);
 			withTemporarySystemProperty(TESTNG_MODE_DRYRUN, "true",
-				() -> configureAndRun(testNG, Phase.DISCOVERY, configurationParameters, listener));
+				() -> configureAndRun(configurationParameters, listener, testMethods(methodNames), Phase.DISCOVERY));
 		}
 
 		listener.finalizeDiscovery();
@@ -106,15 +105,36 @@ public TestDescriptor discover(EngineDiscoveryRequest request, UniqueId uniqueId
 	 * Execute the previously discovered TestNG tests in the supplied {@linkplain ExecutionRequest request}.
 	 * <p>
 	 * Supports the following configuration parameters:
+	 * <h4>Execution</h4>
 	 * <dl>
+	 *     <dt>{@code testng.allowReturnValues} (file path)</dt>
+	 *     <dd>whether methods annotated with {@code @Test} that have return values should be considered test methods (default: {@code false})</dd>
+	 *
+	 *     <dt>{@code testng.dataProviderThreadCount} (file path)</dt>
+	 *     <dd>maximum number of threads to use for running data providers in parallel, if enabled via {@link DataProvider#parallel()} (default: {@code 10})</dd>
+	 *
+	 *     <dt>{@code testng.parallel} (methods|tests|classes|instances|none)</dt>
+	 *     <dd>TestNG's parallel execution mode for running tests in separate threads (default: {@code "none"})</dd>
+	 *
+	 *     <dt>{@code testng.preserveOrder} (boolean)</dt>
+	 *     <dd>whether classes and methods should be run in a predictable order (default: {@code true})</dd>
+	 *
+	 *     <dt>{@code testng.threadCount} (boolean)</dt>
+	 *     <dd>maximum number of threads for running tests in parallel, if enabled via {@code testng.parallel} (default: {@code 5})</dd>
+	 * </dl>
+	 * <h4>Reporting</h4>
+	 * <dl>
+	 *     <dt>{@code testng.listeners} (comma-separated list of fully-qualified class names)</dt>
+	 *     <dd>custom listeners that should be registered when executing tests (default: {@code ""})</dd>
+	 *
 	 *     <dt>{@code testng.outputDirectory} (file path)</dt>
-	 *     <dd>the output directory for reports (default: "test-output")</dd>
+	 *     <dd>the output directory for reports (default: {@code "test-output"})</dd>
+	 *
 	 *     <dt>{@code testng.useDefaultListeners} (boolean)</dt>
 	 *     <dd>whether TestNG's default report generating listeners should be used (default: {@code false})</dd>
-	 *     <dt>{@code testng.listeners} (comma-separated list of fully-qualified class names)</dt>
-	 *     <dd>custom listeners that should be registered when executing tests (default: none)</dd>
+	 *
 	 *     <dt>{@code testng.verbose} (integer)</dt>
-	 *     <dd>TestNG's level of verbosity (default: 0)</dd>
+	 *     <dd>TestNG's level of verbosity (default: {@code 0})</dd>
 	 * </dl>
 	 * <p>
 	 * The implementation configures TestNG as if the discovered methods were specified on the
@@ -136,46 +156,30 @@ public void execute(ExecutionRequest request) {
 		TestNGEngineDescriptor engineDescriptor = (TestNGEngineDescriptor) request.getRootTestDescriptor();
 		listener.executionStarted(engineDescriptor);
 		engineDescriptor.prepareExecution();
-		ExecutionListener executionListener = createExecutionListener(listener, engineDescriptor);
+		ExecutionListener executionListener = new ExecutionListener(listener, engineDescriptor);
 		List<String> methodNames = engineDescriptor.getQualifiedMethodNames();
 		if (!methodNames.isEmpty()) {
-			configureAndRun(createTestNGForTestMethods(methodNames), Phase.EXECUTION,
-				request.getConfigurationParameters(), executionListener);
+			configureAndRun(request.getConfigurationParameters(), executionListener, testMethods(methodNames),
+				Phase.EXECUTION);
 		}
 		listener.executionFinished(engineDescriptor, executionListener.toEngineResult());
 	}
 
-	private ExecutionListener createExecutionListener(EngineExecutionListener listener,
-			TestNGEngineDescriptor engineDescriptor) {
-		return new ExecutionListener(listener, engineDescriptor);
-	}
-
-	private void configureAndRun(TestNG testNG, Phase phase, ConfigurationParameters configurationParameters,
-			ITestNGListener listener) {
-		phase.configure(testNG, configurationParameters);
-		testNG.addListener(listener);
-		testNG.run();
-	}
-
-	private TestNG createTestNGForTestClasses(Class<?>[] testClasses) {
-		ConfigurableTestNG testNG = new ConfigurableTestNG();
-		testNG.setTestClasses(testClasses);
-		return testNG;
-	}
-
-	private TestNG createTestNGForTestMethods(List<String> methodNames) {
+	private static void configureAndRun(ConfigurationParameters configurationParameters, ITestNGListener listener,
+			Configurer... configurers) {
+		CommandLineArgs commandLineArgs = new CommandLineArgs();
+		for (Configurer configurer : configurers) {
+			configurer.configure(commandLineArgs, configurationParameters);
+		}
 		ConfigurableTestNG testNG = new ConfigurableTestNG();
-		if (!methodNames.isEmpty()) {
-			testNG.configure(createCommandLineArgs(methodNames));
+		testNG.configure(commandLineArgs);
+		for (Configurer configurer : configurers) {
+			configurer.configure(testNG, configurationParameters);
 		}
-		return testNG;
-	}
-
-	private static CommandLineArgs createCommandLineArgs(List<String> methodNames) {
-		CommandLineArgs commandLineArgs = new CommandLineArgs();
-		commandLineArgs.useDefaultListeners = String.valueOf(false);
-		commandLineArgs.commandLineMethods = methodNames;
-		return commandLineArgs;
+		testNG.addListener(LoggingListener.INSTANCE);
+		testNG.addListener(new ConfiguringListener(configurationParameters));
+		testNG.addListener(listener);
+		testNG.run();
 	}
 
 	@SuppressWarnings("SameParameterValue")
@@ -195,40 +199,67 @@ private static void withTemporarySystemProperty(String key, String value, Runnab
 		}
 	}
 
-	enum Phase {
+	interface Configurer {
+
+		static Configurer testClasses(Class<?>[] testClasses) {
+			return new Configurer() {
+				@Override
+				public void configure(TestNG testNG, ConfigurationParameters config) {
+					testNG.setTestClasses(testClasses);
+				}
+			};
+		}
+
+		static Configurer testMethods(List<String> methodNames) {
+			return new Configurer() {
+				@Override
+				public void configure(CommandLineArgs commandLineArgs, ConfigurationParameters config) {
+					commandLineArgs.commandLineMethods = methodNames;
+				}
+			};
+		}
+
+		default void configure(TestNG testNG, ConfigurationParameters config) {
+		}
+
+		default void configure(CommandLineArgs commandLineArgs, ConfigurationParameters config) {
+		}
+
+	}
+
+	enum Phase implements Configurer {
 
 		DISCOVERY {
 			@Override
-			void configure(TestNG testNG, ConfigurationParameters config) {
-				testNG.addListener(LoggingListener.INSTANCE);
+			public void configure(TestNG testNG, ConfigurationParameters config) {
 				testNG.setVerbose(0);
 				testNG.setUseDefaultListeners(false);
 			}
 		},
 
 		EXECUTION {
 			@Override
-			void configure(TestNG testNG, ConfigurationParameters config) {
-				testNG.addListener(LoggingListener.INSTANCE);
+			public void configure(TestNG testNG, ConfigurationParameters config) {
 				testNG.setVerbose(config.get("testng.verbose", Integer::valueOf).orElse(0));
 				testNG.setUseDefaultListeners(config.getBoolean("testng.useDefaultListeners").orElse(false));
-				config.get("testng.outputDirectory").ifPresent(testNG::setOutputDirectory);
-				config.get("testng.listeners").ifPresent(listeners -> Arrays.stream(listeners.split(",")) //
-						.map(ReflectionSupport::tryToLoadClass) //
-						.map(result -> result.getOrThrow(
-							cause -> new JUnitException("Failed to load custom listener class", cause))) //
-						.map(listenerClass -> {
-							if (!ITestNGListener.class.isAssignableFrom(listenerClass)) {
-								throw new JUnitException("Custom listener class must implement "
-										+ ITestNGListener.class.getName() + ": " + listenerClass.getName());
-							}
-							return (ITestNGListener) ReflectionSupport.newInstance(listenerClass);
-						}) //
-						.forEach(testNG::addListener));
+				config.get("testng.outputDirectory") //
+						.ifPresent(testNG::setOutputDirectory);
+				config.getBoolean("testng.preserveOrder") //
+						.ifPresent(testNG::setPreserveOrder);
+				config.get("testng.parallel", ParallelMode::getValidParallel) //
+						.ifPresent(testNG::setParallel);
+				config.get("testng.threadCount", Integer::parseInt) //
+						.ifPresent(testNG::setThreadCount);
+				config.get("testng.dataProviderThreadCount", Integer::parseInt) //
+						.ifPresent(testNG::setDataProviderThreadCount);
 			}
-		};
 
-		abstract void configure(TestNG testNG, ConfigurationParameters config);
+			@Override
+			public void configure(CommandLineArgs commandLineArgs, ConfigurationParameters config) {
+				config.get("testng.listeners") //
+						.ifPresent(listeners -> commandLineArgs.listener = listeners);
+			}
+		};
 	}
 
 	/**