introduce JUnit tag "flaky", to exclude tests from running.

Jenkinsfile

@@ -158,7 +158,7 @@ pipeline {
             }
         }
 
-        stage('Integration Tests') {
+        stage('Integration Tests (without flaky tests)') {
             steps {
                 sh './gradlew --console=plain integrationTest'
             }
@@ -176,5 +176,26 @@ pipeline {
                 }
             }
         }
+
+        stage('Integration Tests (flaky tests only)') {
+            steps {
+                warnError("Integration Tests Failed") {  // if this errs, mark the build unstable, not failed.
+                    sh './gradlew --console=plain integrationTestFlaky'
+                }
+            }
+            post {
+                always {
+                    // Gradle generates both a HTML report of the unit tests to `build/reports/tests/*`
+                    // and XML reports to `build/test-results/*`.
+                    // We need to upload the XML reports for visualization in Jenkins.
+                    //
+                    // See https://docs.gradle.org/current/userguide/java_testing.html#test_reporting
+                    junit testResults: '**/build/test-results/integrationTestFlaky/*.xml', allowEmptyResults: true
+                    // Jenkins truncates large test outputs, so archive it as well.
+                    tar file: 'build/integrationTestFlaky-results.tgz', archive: true, compress: true, overwrite: true,
+                        glob: '**/build/test-results/integrationTestFlaky/*.xml'
+                }
+            }
+        }
     }
 }

engine-tests/build.gradle.kts

@@ -2,6 +2,13 @@
 // SPDX-License-Identifier: Apache-2.0
 
 // Engine tests are split out due to otherwise quirky project dependency issues with module tests extending engine tests
+// while locally all tests should be run, when building via github, tests can be run separated in the github pipeline
+// file. For integration tests, a tag "flaky" was introduced to mark tests which frequently fail pipelines
+//      gradle test
+//      gradle --console=plain unitTest
+//      gradle --console=plain integrationTest
+//      gradle --console=plain integrationTestFlaky
+
 plugins {
     id("java-library")
     id("org.jetbrains.gradle.plugin.idea-ext")
@@ -105,18 +112,30 @@ tasks.register<Test>("unitTest") {
     group =  "Verification"
     description = "Runs unit tests (fast)"
     useJUnitPlatform {
-        excludeTags = setOf("MteTest", "TteTest")
+        excludeTags("MteTest", "TteTest")
     }
     systemProperty("junit.jupiter.execution.timeout.default", "1m")
 }
 
 tasks.register<Test>("integrationTest") {
     dependsOn(tasks.getByPath(":extractNatives"))
     group = "Verification"
-    description = "Runs integration tests (slow) tagged with 'MteTest' or 'TteTest'"
+    description = "Runs integration tests (slow) tagged with 'MteTest' or 'TteTest', exclude tests tagged 'flaky'."
+
+    useJUnitPlatform {
+        excludeTags("flaky")
+        includeTags("MteTest", "TteTest")
+    }
+    systemProperty("junit.jupiter.execution.timeout.default", "5m")
+}
+
+tasks.register<Test>("integrationTestFlaky") {
+    dependsOn(tasks.getByPath(":extractNatives"))
+    group = "Verification"
+    description = "Runs integration tests tagged with 'flaky' and either 'MteTest' or 'TteTest'."
 
     useJUnitPlatform {
-        includeTags = setOf("MteTest", "TteTest")
+        includeTags("MteTest & flaky", "TteTest & flaky")
     }
     systemProperty("junit.jupiter.execution.timeout.default", "5m")
 }

engine-tests/src/test/java/org/terasology/engine/integrationenvironment/ExampleTest.java

@@ -5,6 +5,7 @@
 import com.google.common.collect.Lists;
 import org.joml.Vector3i;
 import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Tag;
 import org.junit.jupiter.api.Test;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -38,13 +39,15 @@ public class ExampleTest {
     private ModuleTestingHelper helper;
 
     @Test
+    @Tag("flaky")
     public void testClientCreation() {
         logger.info("Starting test 'testClientCreation'");
         Assertions.assertDoesNotThrow(helper::createClient);
         logger.info("Done with test 'testClientCreation'");
     }
 
     @Test
+    @Tag("flaky")
     public void testClientConnection() throws IOException {
         int currentClients = Lists.newArrayList(entityManager.getEntitiesWith(ClientComponent.class)).size();