GEODE-7554: Add retry mechanism for failed tests (#120)

* write failed tests to a file
* enable all benchmarks
* update help message for --ci option in run_against_baseline.sh

geode-benchmarks/build.gradle

@@ -63,11 +63,6 @@ task benchmark(type: Test) {
   useJUnitPlatform()
   testLogging { exceptionFormat = 'full' }
 
-  exclude "**/*NonIndexedQueryBenchmark.class"
-  exclude "**/PartitionedFunctionExecutionBenchmark.class"
-  exclude "**/NoopBenchmark.class"
-  exclude "**/*LongBenchmark.class"
-
   forkEvery 1
 
   systemProperty 'TEST_HOSTS', project.findProperty('hosts')

harness/src/main/java/org/apache/geode/perftest/analysis/Analyzer.java

@@ -16,7 +16,9 @@
 
 import static java.lang.Double.isNaN;
 
+import java.io.BufferedWriter;
 import java.io.File;
+import java.io.FileWriter;
 import java.io.IOException;
 import java.io.PrintWriter;
 
@@ -70,6 +72,9 @@ public static void main(String[] args) throws IOException {
     benchmarkRunResult.writeResult(new PrintWriter(System.out));
     /* throw exc if failed? */
 
+    String errorFilePath = testResultArg + "/../../../failedTests";
+    BufferedWriter writer = new BufferedWriter(new FileWriter(errorFilePath, true));
+
     boolean isSignificantlyBetter = false;
     boolean isHighWaterCandidate = true;
     StringBuilder errorMessage = new StringBuilder();
@@ -79,19 +84,22 @@ public static void main(String[] args) throws IOException {
         if (isNaN(probeResult.baseline) || isNaN(probeResult.test)) {
           errorMessage.append("BENCHMARK FAILED: ").append(benchmarkResult.name)
               .append(" missing result file.\n");
+          writer.append(benchmarkResult.name + "\n");
         } else if (probeResult.description.equals("average latency")) {
           if (probeResult.getDifference() > 0) {
             isHighWaterCandidate = false;
             if (probeResult.getDifference() >= 0.05) {
               errorMessage.append("BENCHMARK FAILED: ").append(benchmarkResult.name)
                   .append(" average latency is 5% worse than baseline.\n");
+              writer.append(benchmarkResult.name + "\n");
             }
           } else if (probeResult.getDifference() <= -0.5) {
             isSignificantlyBetter = true;
           }
         }
       }
     }
+    writer.close();
 
     if (isCI && isHighWaterCandidate && isSignificantlyBetter) {
       System.out.println(

infrastructure/scripts/aws/run_against_baseline.sh

@@ -129,7 +129,7 @@ while (( "$#" )); do
       echo "-R|--baseline-geode-repo : Geode baseline repo (default: ${DEFAULT_BASELINE_REPO})"
       echo "-B|--baseline-geode-branch : Geode baseline branch"
       echo "-m|--metadata : Test metadata to output to file, comma-delimited"
-      echo "--ci : Set if starting instances for Continuous Integration"
+      echo "--ci : Set if starting instances for Continuous Integration - used to retry failed tests and to name/target AWS instances"
       echo "-- : All subsequent arguments are passed to the benchmark task as arguments."
       echo "-h|--help|-? : This help message"
       exit 1
@@ -208,4 +208,4 @@ if [[ -z "${CI}" ]]; then
     ./analyze_tests.sh -o ${OUTPUT}
 else
     ./analyze_tests.sh -o ${OUTPUT} --ci
-fi
+fi