Skip to content
Permalink
Browse files
GEODE-7554: Add retry mechanism for failed tests (#120)
* write failed tests to a file
* enable all benchmarks
* update help message for --ci option in run_against_baseline.sh
  • Loading branch information
nonbinaryprogrammer committed Dec 11, 2019
1 parent 477d381 commit bd2e0008335859ae012f383707a321299006fd02
Showing 3 changed files with 10 additions and 7 deletions.
@@ -63,11 +63,6 @@ task benchmark(type: Test) {
useJUnitPlatform()
testLogging { exceptionFormat = 'full' }

exclude "**/*NonIndexedQueryBenchmark.class"
exclude "**/PartitionedFunctionExecutionBenchmark.class"
exclude "**/NoopBenchmark.class"
exclude "**/*LongBenchmark.class"

forkEvery 1

systemProperty 'TEST_HOSTS', project.findProperty('hosts')
@@ -16,7 +16,9 @@

import static java.lang.Double.isNaN;

import java.io.BufferedWriter;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.io.PrintWriter;

@@ -70,6 +72,9 @@ public static void main(String[] args) throws IOException {
benchmarkRunResult.writeResult(new PrintWriter(System.out));
/* throw exc if failed? */

String errorFilePath = testResultArg + "/../../../failedTests";
BufferedWriter writer = new BufferedWriter(new FileWriter(errorFilePath, true));

boolean isSignificantlyBetter = false;
boolean isHighWaterCandidate = true;
StringBuilder errorMessage = new StringBuilder();
@@ -79,19 +84,22 @@ public static void main(String[] args) throws IOException {
if (isNaN(probeResult.baseline) || isNaN(probeResult.test)) {
errorMessage.append("BENCHMARK FAILED: ").append(benchmarkResult.name)
.append(" missing result file.\n");
writer.append(benchmarkResult.name + "\n");
} else if (probeResult.description.equals("average latency")) {
if (probeResult.getDifference() > 0) {
isHighWaterCandidate = false;
if (probeResult.getDifference() >= 0.05) {
errorMessage.append("BENCHMARK FAILED: ").append(benchmarkResult.name)
.append(" average latency is 5% worse than baseline.\n");
writer.append(benchmarkResult.name + "\n");
}
} else if (probeResult.getDifference() <= -0.5) {
isSignificantlyBetter = true;
}
}
}
}
writer.close();

if (isCI && isHighWaterCandidate && isSignificantlyBetter) {
System.out.println(
@@ -129,7 +129,7 @@ while (( "$#" )); do
echo "-R|--baseline-geode-repo : Geode baseline repo (default: ${DEFAULT_BASELINE_REPO})"
echo "-B|--baseline-geode-branch : Geode baseline branch"
echo "-m|--metadata : Test metadata to output to file, comma-delimited"
echo "--ci : Set if starting instances for Continuous Integration"
echo "--ci : Set if starting instances for Continuous Integration - used to retry failed tests and to name/target AWS instances"
echo "-- : All subsequent arguments are passed to the benchmark task as arguments."
echo "-h|--help|-? : This help message"
exit 1
@@ -208,4 +208,4 @@ if [[ -z "${CI}" ]]; then
./analyze_tests.sh -o ${OUTPUT}
else
./analyze_tests.sh -o ${OUTPUT} --ci
fi
fi

0 comments on commit bd2e000

Please sign in to comment.