Merge #4625

4625: chore(ci): rerun failed tests, scan log, report flaky tests r=menski a=pihme

## Description

* rerun failing tests
* extract flaky tests from log 
* collect flaky tests at the end and add them to description

## Related issues

closes #4347

#

Co-authored-by: pihme <pihme@users.noreply.github.com>

.ci/scripts/distribution/it-java.sh

@@ -5,10 +5,24 @@ export JAVA_TOOL_OPTIONS="$JAVA_TOOL_OPTIONS -XX:MaxRAMFraction=$((LIMITS_CPU))"
 
 tmpfile=$(mktemp)
 
-mvn -o -B --fail-never -T$LIMITS_CPU -s ${MAVEN_SETTINGS_XML} verify -P skip-unstable-ci,parallel-tests -pl qa/integration-tests -pl upgrade-tests -DtestMavenId=2 | tee ${tmpfile}
+mvn -o -B --fail-never -T$LIMITS_CPU -s ${MAVEN_SETTINGS_XML} verify -P skip-unstable-ci,parallel-tests -pl qa/integration-tests -pl upgrade-tests -DtestMavenId=2 -Dsurefire.rerunFailingTestsCount=5 | tee ${tmpfile}
 
 status=${PIPESTATUS[0]}
 
+if grep -q "\[WARNING\] Flakes:" ${tmpfile}; then
+
+  tmpfile2=$(mktemp)
+
+  awk '/^\[WARNING\] Flakes:.*$/{flag=1}/^\[ERROR\] Tests run:.*Flakes: [0-9]*$/{print;flag=0}flag' ${tmpfile} > ${tmpfile2}
+
+  grep "\[ERROR\]   Run 1: " ${tmpfile} | awk '{print $4}' >> ./target/FlakyTests.txt
+
+  echo ERROR: Flaky Tests detected>&2
+  rm test-it.txt
+  rm flaky-tests-log.txt
+  exit 1
+fi
+
 if [[ $status != 0 ]]; then
   exit $status;
 fi

.ci/scripts/distribution/test-java.sh

@@ -4,10 +4,23 @@ export JAVA_TOOL_OPTIONS="$JAVA_TOOL_OPTIONS -XX:MaxRAMFraction=$((LIMITS_CPU))"
 
 tmpfile=$(mktemp)
 
-mvn -o -B --fail-never -T$LIMITS_CPU -s ${MAVEN_SETTINGS_XML} verify -P skip-unstable-ci,parallel-tests -Dzeebe.it.skip -DtestMavenId=1 | tee ${tmpfile}
+mvn -o -B --fail-never -T$LIMITS_CPU -s ${MAVEN_SETTINGS_XML} verify -P skip-unstable-ci,parallel-tests -Dzeebe.it.skip -DtestMavenId=1 -Dsurefire.rerunFailingTestsCount=5 | tee ${tmpfile}
 
 status=${PIPESTATUS[0]}
 
+if grep -q "\[WARNING\] Flakes:" ${tmpfile}; then
+
+  tmpfile2=$(mktemp)
+
+  awk '/^\[WARNING\] Flakes:.*$/{flag=1}/^\[ERROR\] Tests run:.*Flakes: [0-9]*$/{print;flag=0}flag' ${tmpfile} > ${tmpfile2}
+
+  grep "\[ERROR\]   Run 1: " ${tmpfile2} | awk '{print $4}' >> ./target/FlakyTests.txt
+
+  echo ERROR: Flaky Tests detected>&2
+
+  exit 1
+fi
+
 if [[ $status != 0 ]]; then
   exit $status;
 fi

.ci/scripts/distribution/test-java8.sh

@@ -1,16 +1,30 @@
 #!/bin/bash -eux
 
-
 export JAVA_TOOL_OPTIONS="$JAVA_TOOL_OPTIONS -XX:MaxRAMFraction=$((LIMITS_CPU))"
 
 mvn -v
 
 tmpfile=$(mktemp)
 
-mvn -o -B --fail-never -T$LIMITS_CPU -s ${MAVEN_SETTINGS_XML} verify -pl clients/java -DtestMavenId=3 | tee ${tmpfile}
+mvn -o -B --fail-never -T$LIMITS_CPU -s ${MAVEN_SETTINGS_XML} verify -pl clients/java -DtestMavenId=3 -Dsurefire.rerunFailingTestsCount=5 | tee ${tmpfile}
 
 status=${PIPESTATUS[0]}
 
+
+if grep -q "\[WARNING\] Flakes:" ${tmpfile}; then
+
+  tmpfile2=$(mktemp)
+
+  awk '/^\[WARNING\] Flakes:.*$/{flag=1}/^\[ERROR\] Tests run:.*Flakes: [0-9]*$/{print;flag=0}flag' ${tmpfile} > ${tmpfile2}
+
+  grep "\[ERROR\]   Run 1: " ${tmpfile2} | awk '{print $4}' >> ./target/FlakyTests.txt
+
+  echo ERROR: Flaky Tests detected>&2
+
+  exit 1
+fi
+
+
 if [[ $status != 0 ]]; then
   exit $status;
 fi

Jenkinsfile

@@ -190,6 +190,11 @@ pipeline {
             post {
                 failure {
                     archive "**/*/surefire-reports/*-output.txt"
+                    script {
+                      if (fileExists('./target/FlakyTests.txt')) {
+                          currentBuild.description = "Flaky Tests: <br>" + readFile('./target/FlakyTests.txt').split('\n').join('<br>')
+                      }
+                    }
                 }
             }
         }