chore(ci): rerun failed tests, scan log, report flaky tests

.ci/scripts/distribution/it-java.sh

@@ -1,6 +1,32 @@
-#!/bin/sh -eux
+#!/bin/bash -eux
 
 
 export JAVA_TOOL_OPTIONS="$JAVA_TOOL_OPTIONS -XX:MaxRAMFraction=$((LIMITS_CPU))"
 
-mvn -o -B -T$LIMITS_CPU -s ${MAVEN_SETTINGS_XML} verify -P skip-unstable-ci,parallel-tests -pl qa/integration-tests -pl upgrade-tests -DtestMavenId=2
+tmpfile=$(mktemp)
+
+mvn -o -B --fail-never -T$LIMITS_CPU -s ${MAVEN_SETTINGS_XML} verify -P skip-unstable-ci,parallel-tests -pl qa/integration-tests -pl upgrade-tests -DtestMavenId=2 -Dsurefire.rerunFailingTestsCount=5 | tee ${tmpfile}
+
+status=${PIPESTATUS[0]}
+
+if grep -q "\[WARNING\] Flakes:" ${tmpfile}; then
+
+  tmpfile2=$(mktemp)
+
+  awk '/^\[WARNING\] Flakes:.*$/{flag=1}/^\[ERROR\] Tests run:.*Flakes: [0-9]*$/{print;flag=0}flag' ${tmpfile} > ${tmpfile2}
+
+  grep "\[ERROR\]   Run 1: " ${tmpfile} | awk '{print $4}' >> ./target/FlakyTests.txt
+
+  echo ERROR: Flaky Tests detected>&2
+  rm test-it.txt
+  rm flaky-tests-log.txt
+  exit 1
+fi
+
+if [[ $status != 0 ]]; then
+  exit $status;
+fi
+
+if grep -q "\[INFO\] Build failures were ignored\." ${tmpfile}; then
+  exit 1
+fi

.ci/scripts/distribution/test-java.sh

@@ -1,6 +1,30 @@
-#!/bin/sh -eux
-
+#!/bin/bash -eux
 
 export JAVA_TOOL_OPTIONS="$JAVA_TOOL_OPTIONS -XX:MaxRAMFraction=$((LIMITS_CPU))"
 
-mvn -o -B -T$LIMITS_CPU -s ${MAVEN_SETTINGS_XML} verify -P skip-unstable-ci,parallel-tests -Dzeebe.it.skip -DtestMavenId=1
+tmpfile=$(mktemp)
+
+mvn -o -B --fail-never -T$LIMITS_CPU -s ${MAVEN_SETTINGS_XML} verify -P skip-unstable-ci,parallel-tests -Dzeebe.it.skip -DtestMavenId=1 -Dsurefire.rerunFailingTestsCount=5 | tee ${tmpfile}
+
+status=${PIPESTATUS[0]}
+
+if grep -q "\[WARNING\] Flakes:" ${tmpfile}; then
+
+  tmpfile2=$(mktemp)
+
+  awk '/^\[WARNING\] Flakes:.*$/{flag=1}/^\[ERROR\] Tests run:.*Flakes: [0-9]*$/{print;flag=0}flag' ${tmpfile} > ${tmpfile2}
+
+  grep "\[ERROR\]   Run 1: " ${tmpfile2} | awk '{print $4}' >> ./target/FlakyTests.txt
+
+  echo ERROR: Flaky Tests detected>&2
+
+  exit 1
+fi
+
+if [[ $status != 0 ]]; then
+  exit $status;
+fi
+
+if grep -q "\[INFO\] Build failures were ignored\." ${tmpfile}; then
+  exit 1
+fi

.ci/scripts/distribution/test-java8.sh

@@ -1,8 +1,34 @@
-#!/bin/sh -eux
-
+#!/bin/bash -eux
 
 export JAVA_TOOL_OPTIONS="$JAVA_TOOL_OPTIONS -XX:MaxRAMFraction=$((LIMITS_CPU))"
 
 mvn -v
 
-mvn -o -B -T$LIMITS_CPU -s ${MAVEN_SETTINGS_XML} verify -pl clients/java -DtestMavenId=3
+tmpfile=$(mktemp)
+
+mvn -o -B --fail-never -T$LIMITS_CPU -s ${MAVEN_SETTINGS_XML} verify -pl clients/java -DtestMavenId=3 -Dsurefire.rerunFailingTestsCount=5 | tee ${tmpfile}
+
+status=${PIPESTATUS[0]}
+
+
+if grep -q "\[WARNING\] Flakes:" ${tmpfile}; then
+
+  tmpfile2=$(mktemp)
+
+  awk '/^\[WARNING\] Flakes:.*$/{flag=1}/^\[ERROR\] Tests run:.*Flakes: [0-9]*$/{print;flag=0}flag' ${tmpfile} > ${tmpfile2}
+
+  grep "\[ERROR\]   Run 1: " ${tmpfile2} | awk '{print $4}' >> ./target/FlakyTests.txt
+
+  echo ERROR: Flaky Tests detected>&2
+
+  exit 1
+fi
+
+
+if [[ $status != 0 ]]; then
+  exit $status;
+fi
+
+if grep -q "\[INFO\] Build failures were ignored\." ${tmpfile}; then
+  exit 1
+fi

Jenkinsfile

@@ -189,6 +189,11 @@ pipeline {
             post {
                 failure {
                     archive "**/*/surefire-reports/*-output.txt"
+                    script {
+                      if (fileExists('./target/FlakyTests.txt')) {
+                          currentBuild.description = "Flaky Tests: <br>" + readFile('./target/FlakyTests.txt').split('\n').join('<br>')
+                      }
+                    }
                 }
             }
         }