Skip to content

Commit

Permalink
[sinttest] Allow custom processing of test run result
Browse files Browse the repository at this point in the history 
This adds a new configuration option, `testRunResultProcessors`, that allows a user to customize the way the results of a test run is processed.

By default, the pre-exising printing-to-stderr is used.
  • Loading branch information
@guusdk
guusdk committed Apr 11, 2024
1 parent 2e94599 commit ae11892
Show file tree
Hide file tree
Showing 3 changed files with 102 additions and 38 deletions.
31 changes: 31 additions & 0 deletions smack-integration-test/src/main/java/org/igniterealtime/smack/inttest/Configuration.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
import java.lang.reflect.Method;
import java.security.KeyManagementException;
import java.security.NoSuchAlgorithmException;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
Expand All @@ -31,6 +32,7 @@
import java.util.Properties;
import java.util.Set;
import java.util.logging.Logger;
import java.util.stream.Collectors;

import javax.net.ssl.SSLContext;

Expand Down Expand Up @@ -131,6 +133,8 @@ public enum CompatibilityMode {

public final CompatibilityMode compatibilityMode;

public final List<? extends SmackIntegrationTestFramework.TestRunResultProcessor> testRunResultProcessors;

private Configuration(Configuration.Builder builder) throws KeyManagementException, NoSuchAlgorithmException {
service = Objects.requireNonNull(builder.service,
"'service' must be set. Either via 'properties' files or via system property 'sinttest.service'.");
Expand Down Expand Up @@ -203,6 +207,7 @@ else if (StringUtils.isNotEmpty(builder.accountOneUsername, builder.accountOnePa

this.dnsResolver = builder.dnsResolver;
this.compatibilityMode = builder.compatibilityMode;
this.testRunResultProcessors = builder.testRunResultProcessors;
}

public boolean isAccountRegistrationPossible() {
Expand Down Expand Up @@ -263,6 +268,8 @@ public static final class Builder {

private CompatibilityMode compatibilityMode = CompatibilityMode.standardsCompliant;

private List<? extends SmackIntegrationTestFramework.TestRunResultProcessor> testRunResultProcessors;

private Builder() {
}

Expand Down Expand Up @@ -473,6 +480,16 @@ public Builder setCompatibilityMode(String compatibilityModeString) {
return setCompatibilityMode(compatibilityMode);
}

public Builder setTestRunResultProcessors(String testRunResultProcessorsString) {
if (testRunResultProcessorsString == null) {
testRunResultProcessors = List.of(new SmackIntegrationTestFramework.ConsoleTestRunResultProcessor());
return this;
}

testRunResultProcessors = getTestRunProcessorListFrom(testRunResultProcessorsString);
return this;
}

public Configuration build() throws KeyManagementException, NoSuchAlgorithmException {
return new Configuration(this);
}
Expand Down Expand Up @@ -550,6 +567,7 @@ public static Configuration newConfiguration(String[] testPackages)

builder.setCompatibilityMode(properties.getProperty("compatibilityMode"));

builder.setTestRunResultProcessors(properties.getProperty("testRunResultProcessors"));
return builder.build();
}

Expand Down Expand Up @@ -605,6 +623,19 @@ private static Set<String> getSpecificationSetFrom(String input) {
return split(input, Configuration::normalizeSpecification);
}

private static List<? extends SmackIntegrationTestFramework.TestRunResultProcessor> getTestRunProcessorListFrom(String input) {
return Arrays.stream(input.split(","))
.map(element -> {
try {
final Class<? extends SmackIntegrationTestFramework.TestRunResultProcessor> aClass = Class.forName(element).asSubclass(SmackIntegrationTestFramework.TestRunResultProcessor.class);
return aClass.getConstructor().newInstance();
} catch (Exception e) {
throw new IllegalArgumentException("Unable to construct TestRunResultProcessor from value: " + element, e);
}
})
.collect(Collectors.toList());
}

private static Map<String, Set<String>> convertTestsToMap(Set<String> tests) {
Map<String, Set<String>> res = new HashMap<>();
for (String test : tests) {
Expand Down
92 changes: 54 additions & 38 deletions ...on-test/src/main/java/org/igniterealtime/smack/inttest/SmackIntegrationTestFramework.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -114,54 +114,65 @@ public static void main(String[] args) throws IOException, KeyManagementExceptio
SmackIntegrationTestFramework sinttest = new SmackIntegrationTestFramework(config);
TestRunResult testRunResult = sinttest.run();

for (Map.Entry<Class<? extends AbstractSmackIntTest>, Throwable> entry : testRunResult.impossibleTestClasses.entrySet()) {
LOGGER.info("Could not run " + entry.getKey().getName() + " because: "
+ entry.getValue().getLocalizedMessage());
}
for (TestNotPossible testNotPossible : testRunResult.impossibleIntegrationTests) {
LOGGER.info("Could not run " + testNotPossible.concreteTest + " because: "
+ testNotPossible.testNotPossibleException.getMessage());
}
for (SuccessfulTest successfulTest : testRunResult.successfulIntegrationTests) {
LOGGER.info(successfulTest.concreteTest + " ✔");
}
final int successfulTests = testRunResult.successfulIntegrationTests.size();
final int failedTests = testRunResult.failedIntegrationTests.size();
final int availableTests = testRunResult.getNumberOfAvailableTests();
LOGGER.info("SmackIntegrationTestFramework[" + testRunResult.testRunId + ']' + " finished: "
+ successfulTests + '/' + availableTests + " [" + failedTests + " failed]");

final int exitStatus;
if (failedTests > 0) {
LOGGER.warning("💀 The following " + failedTests + " tests failed! 💀");
final SortedSet<String> bySpecification = new TreeSet<>();
for (FailedTest failedTest : testRunResult.failedIntegrationTests) {
final Throwable cause = failedTest.failureReason;
LOGGER.log(Level.SEVERE, failedTest.concreteTest + " failed: " + cause, cause);
if (failedTest.concreteTest.method.getDeclaringClass().isAnnotationPresent(SpecificationReference.class)) {
final String specificationReference = getSpecificationReference(failedTest.concreteTest.method);
if (specificationReference != null) {
bySpecification.add("- " + specificationReference + " (as tested by '" + failedTest.concreteTest + "')");
}
}
}
if (!bySpecification.isEmpty()) {
LOGGER.log(Level.SEVERE, "The failed tests correspond to the following specifications:" + System.lineSeparator() + String.join(System.lineSeparator(), bySpecification));
for (final TestRunResultProcessor testRunResultProcessor : config.testRunResultProcessors) {
try {
testRunResultProcessor.process(testRunResult);
} catch (Throwable t) {
LOGGER.log(Level.WARNING, "Invocation of TestRunResultProcessor " + testRunResultProcessor + " failed.", t);
}

exitStatus = 2;
} else {
LOGGER.info("All possible Smack Integration Tests completed successfully. \\o/");
exitStatus = 0;
}

if (config.debuggerFactory instanceof EnhancedDebugger) {
EnhancedDebuggerWindow.getInstance().waitUntilClosed();
}

final int exitStatus = testRunResult.failedIntegrationTests.isEmpty() ? 0 : 2;
System.exit(exitStatus);
}

public static class ConsoleTestRunResultProcessor implements TestRunResultProcessor {

@Override
public void process(final TestRunResult testRunResult) {
for (Map.Entry<Class<? extends AbstractSmackIntTest>, Throwable> entry : testRunResult.impossibleTestClasses.entrySet()) {
LOGGER.info("Could not run " + entry.getKey().getName() + " because: "
+ entry.getValue().getLocalizedMessage());
}
for (TestNotPossible testNotPossible : testRunResult.impossibleIntegrationTests) {
LOGGER.info("Could not run " + testNotPossible.concreteTest + " because: "
+ testNotPossible.testNotPossibleException.getMessage());
}
for (SuccessfulTest successfulTest : testRunResult.successfulIntegrationTests) {
LOGGER.info(successfulTest.concreteTest + " ✔");
}
final int successfulTests = testRunResult.successfulIntegrationTests.size();
final int failedTests = testRunResult.failedIntegrationTests.size();
final int availableTests = testRunResult.getNumberOfAvailableTests();
LOGGER.info("SmackIntegrationTestFramework[" + testRunResult.testRunId + ']' + " finished: "
+ successfulTests + '/' + availableTests + " [" + failedTests + " failed]");

if (failedTests > 0) {
LOGGER.warning("💀 The following " + failedTests + " tests failed! 💀");
final SortedSet<String> bySpecification = new TreeSet<>();
for (FailedTest failedTest : testRunResult.failedIntegrationTests) {
final Throwable cause = failedTest.failureReason;
LOGGER.log(Level.SEVERE, failedTest.concreteTest + " failed: " + cause, cause);
if (failedTest.concreteTest.method.getDeclaringClass().isAnnotationPresent(SpecificationReference.class)) {
final String specificationReference = getSpecificationReference(failedTest.concreteTest.method);
if (specificationReference != null) {
bySpecification.add("- " + specificationReference + " (as tested by '" + failedTest.concreteTest + "')");
}
}
}
if (!bySpecification.isEmpty()) {
LOGGER.log(Level.SEVERE, "The failed tests correspond to the following specifications:" + System.lineSeparator() + String.join(System.lineSeparator(), bySpecification));
}
} else {
LOGGER.info("All possible Smack Integration Tests completed successfully. \\o/");
}
}
}

private static String getSpecificationReference(Method method) {
final SpecificationReference spec = method.getDeclaringClass().getAnnotation(SpecificationReference.class);
if (spec == null || spec.document().isBlank()) {
Expand Down Expand Up @@ -665,6 +676,11 @@ private static Exception throwFatalException(Throwable e) throws Error, NoRespon
return (Exception) e;
}

public interface TestRunResultProcessor
{
void process(final SmackIntegrationTestFramework.TestRunResult testRunResult);
}

public static final class TestRunResult {

/**
Expand Down
17 changes: 17 additions & 0 deletions smack-integration-test/src/main/java/org/igniterealtime/smack/inttest/package-info.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -167,6 +167,10 @@
* <td>dnsResolver</td>
* <td>One of ‘minidns’, ‘javax’ or ‘dnsjava’. Defaults to ‘minidns’.</td>
* </tr>
* <tr>
* <td>testRunResultProcessors</td>
* <td>List of class names for generating test run output. Defaults to 'org.igniterealtime.smack.inttest.SmackIntegrationTestFramework$ConsoleTestRunResultProcessor'</td>
* </tr>
* </table>
* <h3>Where to place the properties file</h3>
* <p>
Expand Down Expand Up @@ -338,5 +342,18 @@
* <pre>{@code
* $ gradle integrationTest -Dsinttest.service=my.xmppservice.org -Dsinttest.testPackages=org.mypackage,org.otherpackage
* }</pre>
* <h2>Generating test run reports</h2>
* <p>
* By default, the results of the test run is printed to standard-error. You can, however, provide your own processing
* for the test run results. To do so, create an implementation of
* {@link org.igniterealtime.smack.inttest.SmackIntegrationTestFramework.TestRunResultProcessor} and provide its class
* name to the <code>testRunResultProcessor</code> property. This property takes a comma separated list of class names.
* </p>
* <p>
* Example:
* </p>
* <pre>{@code
* $ gradle integrationTest -Dsinttest.service=my.xmppservice.org -Dsinttest.testRunResultProcessor=org.igniterealtime.smack.inttest.SmackIntegrationTestFramework$ConsoleTestRunResultProcessor
* }</pre>
*/
package org.igniterealtime.smack.inttest;

0 comments on commit ae11892

Please sign in to comment.