From b9754f67ee8122c1ddd246619a5277d1dabb087b Mon Sep 17 00:00:00 2001
From: Jochen Roth <jochen.roth@b13.com>
Date: Mon, 24 Jan 2022 10:55:49 +0100
Subject: [PATCH] [TASK] Add acceptance test reports to Gitlab
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Gitlab is able to give a nice overview of tests
and how often a test failed recently which
makes it easier to investigate flaky/failed
tests. Also it takes away the pain of searching
for the actual failing test in a job because it only
displays information of the failed test
(no bootstrap log details and such)

Codeception AC test reports have been added to the
Gitlab GUI using the --xml flag to generate a Gitlab
readable report format and it has been added to the
jobs artifacts:reports:junit.

Resolves: #96628
Releases: main, 11.5
Change-Id: Id5ab5adfdc318c0c97143b0f05311f47fd1d37f0
Reviewed-on: https://review.typo3.org/c/Packages/TYPO3.CMS/+/73136
Tested-by: core-ci <typo3@b13.com>
Tested-by: Oliver Bartsch <bo@cedev.de>
Tested-by: Stefan Bürk <stefan@buerk.tech>
Tested-by: Christian Kuhn <lolli@schwarzbu.ch>
Reviewed-by: Oliver Bartsch <bo@cedev.de>
Reviewed-by: Stefan Bürk <stefan@buerk.tech>
Reviewed-by: Christian Kuhn <lolli@schwarzbu.ch>
---
 .../nightly/acceptance-application.yml        | 27 +++++++++++++++
 .../gitlab-ci/nightly/acceptance-install.yml  | 33 +++++++++++++++++++
 .../pre-merge/acceptance-application.yml      |  3 ++
 .../pre-merge/acceptance-install.yml          | 12 +++++++
 Build/testing-docker/local/docker-compose.yml | 32 +++++++++---------
 5 files changed, 91 insertions(+), 16 deletions(-)

diff --git a/Build/gitlab-ci/nightly/acceptance-application.yml b/Build/gitlab-ci/nightly/acceptance-application.yml
index f0eca9e51080..9c2cef38bbb8 100644
--- a/Build/gitlab-ci/nightly/acceptance-application.yml
+++ b/Build/gitlab-ci/nightly/acceptance-application.yml
@@ -8,6 +8,9 @@ acceptance application mariadb php 8.1 locked:
     paths:
       - typo3temp/var/tests/acceptance/typo3temp/var/log
       - typo3temp/var/tests/AcceptanceReports
+    reports:
+      junit:
+        - typo3temp/var/tests/AcceptanceReports/reports.xml
   parallel: 8
   script:
     - Build/Scripts/runTests.sh -s composerInstall -p 8.1
@@ -26,6 +29,9 @@ acceptance application mariadb php 8.1 max:
     paths:
       - typo3temp/var/tests/acceptance/typo3temp/var/log
       - typo3temp/var/tests/AcceptanceReports
+    reports:
+      junit:
+        - typo3temp/var/tests/AcceptanceReports/reports.xml
   parallel: 8
   script:
     - Build/Scripts/runTests.sh -s composerInstallMax -p 8.1
@@ -44,6 +50,9 @@ acceptance application mariadb php 8.1 min:
     paths:
       - typo3temp/var/tests/acceptance/typo3temp/var/log
       - typo3temp/var/tests/AcceptanceReports
+    reports:
+      junit:
+        - typo3temp/var/tests/AcceptanceReports/reports.xml
   parallel: 8
   script:
     - Build/Scripts/runTests.sh -s composerInstallMin -p 8.1
@@ -59,6 +68,9 @@ acceptance application mysql php 8.1 locked:
     paths:
       - typo3temp/var/tests/acceptance/typo3temp/var/log
       - typo3temp/var/tests/AcceptanceReports
+    reports:
+      junit:
+        - typo3temp/var/tests/AcceptanceReports/reports.xml
   parallel: 8
   script:
     - Build/Scripts/runTests.sh -s composerInstall -p 8.1
@@ -77,6 +89,9 @@ acceptance application mysql php 8.1 max:
     paths:
       - typo3temp/var/tests/acceptance/typo3temp/var/log
       - typo3temp/var/tests/AcceptanceReports
+    reports:
+      junit:
+        - typo3temp/var/tests/AcceptanceReports/reports.xml
   parallel: 8
   script:
     - Build/Scripts/runTests.sh -s composerInstallMax -p 8.1
@@ -95,6 +110,9 @@ acceptance application mysql php 8.1 min:
     paths:
       - typo3temp/var/tests/acceptance/typo3temp/var/log
       - typo3temp/var/tests/AcceptanceReports
+    reports:
+      junit:
+        - typo3temp/var/tests/AcceptanceReports/reports.xml
   parallel: 8
   script:
     - Build/Scripts/runTests.sh -s composerInstallMin -p 8.1
@@ -110,6 +128,9 @@ acceptance application sqlite php 8.1 locked:
     paths:
       - typo3temp/var/tests/acceptance/typo3temp/var/log
       - typo3temp/var/tests/AcceptanceReports
+    reports:
+      junit:
+        - typo3temp/var/tests/AcceptanceReports/reports.xml
   parallel: 8
   script:
     - Build/Scripts/runTests.sh -s composerInstall -p 8.1
@@ -128,6 +149,9 @@ acceptance application sqlite php 8.1 max:
     paths:
       - typo3temp/var/tests/acceptance/typo3temp/var/log
       - typo3temp/var/tests/AcceptanceReports
+    reports:
+      junit:
+        - typo3temp/var/tests/AcceptanceReports/reports.xml
   parallel: 8
   script:
     - Build/Scripts/runTests.sh -s composerInstallMax -p 8.1
@@ -146,6 +170,9 @@ acceptance application sqlite php 8.1 min:
     paths:
       - typo3temp/var/tests/acceptance/typo3temp/var/log
       - typo3temp/var/tests/AcceptanceReports
+    reports:
+      junit:
+        - typo3temp/var/tests/AcceptanceReports/reports.xml
   parallel: 8
   script:
     - Build/Scripts/runTests.sh -s composerInstallMin -p 8.1
diff --git a/Build/gitlab-ci/nightly/acceptance-install.yml b/Build/gitlab-ci/nightly/acceptance-install.yml
index e4d17d309701..9f76cd0ae6a6 100644
--- a/Build/gitlab-ci/nightly/acceptance-install.yml
+++ b/Build/gitlab-ci/nightly/acceptance-install.yml
@@ -7,6 +7,9 @@ acceptance install mariadb php 8.1 locked:
     when: on_failure
     paths:
       - typo3temp/var/tests/AcceptanceReports
+    reports:
+      junit:
+        - typo3temp/var/tests/AcceptanceReports/reports.xml
   script:
     - Build/Scripts/runTests.sh -s composerInstall -p 8.1
     - Build/Scripts/runTests.sh -s acceptanceInstall -d mariadb -i 10.3 -p 8.1
@@ -39,6 +42,9 @@ acceptance install mariadb php 8.1 min:
     when: on_failure
     paths:
       - typo3temp/var/tests/AcceptanceReports
+    reports:
+      junit:
+        - typo3temp/var/tests/AcceptanceReports/reports.xml
   script:
     - Build/Scripts/runTests.sh -s composerInstallMin -p 8.1
     - Build/Scripts/runTests.sh -s acceptanceInstall -d mariadb -i 10.3 -p 8.1
@@ -52,6 +58,9 @@ acceptance install mysql php 8.1 locked:
     when: on_failure
     paths:
       - typo3temp/var/tests/AcceptanceReports
+    reports:
+      junit:
+        - typo3temp/var/tests/AcceptanceReports/reports.xml
   script:
     - Build/Scripts/runTests.sh -s composerInstall -p 8.1
     - Build/Scripts/runTests.sh -s acceptanceInstall -d mysql -j 8.0 -p 8.1
@@ -68,6 +77,9 @@ acceptance install mysql php 8.1 max:
     when: on_failure
     paths:
       - typo3temp/var/tests/AcceptanceReports
+    reports:
+      junit:
+        - typo3temp/var/tests/AcceptanceReports/reports.xml
   script:
     - Build/Scripts/runTests.sh -s composerInstallMax -p 8.1
     - Build/Scripts/runTests.sh -s acceptanceInstall -d mysql -j 8.0 -p 8.1
@@ -84,6 +96,9 @@ acceptance install mysql php 8.1 min:
     when: on_failure
     paths:
       - typo3temp/var/tests/AcceptanceReports
+    reports:
+      junit:
+        - typo3temp/var/tests/AcceptanceReports/reports.xml
   script:
     - Build/Scripts/runTests.sh -s composerInstallMin -p 8.1
     - Build/Scripts/runTests.sh -s acceptanceInstall -d mysql -j 8.0 -p 8.1
@@ -97,6 +112,9 @@ acceptance install postgres php 8.1 locked:
     when: on_failure
     paths:
       - typo3temp/var/tests/AcceptanceReports
+    reports:
+      junit:
+        - typo3temp/var/tests/AcceptanceReports/reports.xml
   script:
     - Build/Scripts/runTests.sh -s composerInstall -p 8.1
     - Build/Scripts/runTests.sh -s acceptanceInstall -d postgres -k 10 -p 8.1
@@ -113,6 +131,9 @@ acceptance install postgres php 8.1 max:
     when: on_failure
     paths:
       - typo3temp/var/tests/AcceptanceReports
+    reports:
+      junit:
+        - typo3temp/var/tests/AcceptanceReports/reports.xml
   script:
     - Build/Scripts/runTests.sh -s composerInstallMax -p 8.1
     - Build/Scripts/runTests.sh -s acceptanceInstall -d postgres -k 10 -p 8.1
@@ -129,6 +150,9 @@ acceptance install postgres php 8.1 min:
     when: on_failure
     paths:
       - typo3temp/var/tests/AcceptanceReports
+    reports:
+      junit:
+        - typo3temp/var/tests/AcceptanceReports/reports.xml
   script:
     - Build/Scripts/runTests.sh -s composerInstallMin -p 8.1
     - Build/Scripts/runTests.sh -s acceptanceInstall -d postgres -k 10 -p 8.1
@@ -142,6 +166,9 @@ acceptance install sqlite php 8.1 locked:
     when: on_failure
     paths:
       - typo3temp/var/tests/AcceptanceReports
+    reports:
+      junit:
+        - typo3temp/var/tests/AcceptanceReports/reports.xml
   script:
     - Build/Scripts/runTests.sh -s composerInstall -p 8.1
     - Build/Scripts/runTests.sh -s acceptanceInstall -d sqlite -p 8.1
@@ -158,6 +185,9 @@ acceptance install sqlite php 8.1 max:
     when: on_failure
     paths:
       - typo3temp/var/tests/AcceptanceReports
+    reports:
+      junit:
+        - typo3temp/var/tests/AcceptanceReports/reports.xml
   script:
     - Build/Scripts/runTests.sh -s composerInstallMax -p 8.1
     - Build/Scripts/runTests.sh -s acceptanceInstall -d sqlite -p 8.1
@@ -174,6 +204,9 @@ acceptance install sqlite php 8.1 min:
     when: on_failure
     paths:
       - typo3temp/var/tests/AcceptanceReports
+    reports:
+      junit:
+        - typo3temp/var/tests/AcceptanceReports/reports.xml
   script:
     - Build/Scripts/runTests.sh -s composerInstallMin -p 8.1
     - Build/Scripts/runTests.sh -s acceptanceInstall -d sqlite -p 8.1
diff --git a/Build/gitlab-ci/pre-merge/acceptance-application.yml b/Build/gitlab-ci/pre-merge/acceptance-application.yml
index eb288841e666..c9fd759ee053 100644
--- a/Build/gitlab-ci/pre-merge/acceptance-application.yml
+++ b/Build/gitlab-ci/pre-merge/acceptance-application.yml
@@ -9,6 +9,9 @@ acceptance application mariadb php 8.1 pre-merge:
     paths:
       - typo3temp/var/tests/acceptance/typo3temp/var/log
       - typo3temp/var/tests/AcceptanceReports
+    reports:
+      junit:
+        - typo3temp/var/tests/AcceptanceReports/reports.xml
   parallel: 13
   script:
     - Build/Scripts/runTests.sh -s composerInstall -p 8.1
diff --git a/Build/gitlab-ci/pre-merge/acceptance-install.yml b/Build/gitlab-ci/pre-merge/acceptance-install.yml
index 1073354c5cc6..80c585e6f402 100644
--- a/Build/gitlab-ci/pre-merge/acceptance-install.yml
+++ b/Build/gitlab-ci/pre-merge/acceptance-install.yml
@@ -8,6 +8,9 @@ acceptance install mariadb php 8.1 pre-merge:
     when: on_failure
     paths:
       - typo3temp/var/tests/AcceptanceReports
+    reports:
+      junit:
+        - typo3temp/var/tests/AcceptanceReports/reports.xml
   script:
     - Build/Scripts/runTests.sh -s composerInstall -p 8.1
     - Build/Scripts/runTests.sh -s acceptanceInstall -d mariadb -i 10.3 -p 8.1
@@ -22,6 +25,9 @@ acceptance install mysql php 8.1 pre-merge:
     when: on_failure
     paths:
       - typo3temp/var/tests/AcceptanceReports
+    reports:
+      junit:
+        - typo3temp/var/tests/AcceptanceReports/reports.xml
   script:
     - Build/Scripts/runTests.sh -s composerInstall -p 8.1
     - Build/Scripts/runTests.sh -s acceptanceInstall -d mysql -j 8.0 -p 8.1
@@ -36,6 +42,9 @@ acceptance install postgres php 8.1 pre-merge:
     when: on_failure
     paths:
       - typo3temp/var/tests/AcceptanceReports
+    reports:
+      junit:
+        - typo3temp/var/tests/AcceptanceReports/reports.xml
   script:
     - Build/Scripts/runTests.sh -s composerInstall -p 8.1
     - Build/Scripts/runTests.sh -s acceptanceInstall -d postgres -k 10 -p 8.1
@@ -50,6 +59,9 @@ acceptance install sqlite php 8.1 pre-merge:
     when: on_failure
     paths:
       - typo3temp/var/tests/AcceptanceReports
+    reports:
+      junit:
+        - typo3temp/var/tests/AcceptanceReports/reports.xml
   script:
     - Build/Scripts/runTests.sh -s composerInstall -p 8.1
     - Build/Scripts/runTests.sh -s acceptanceInstall -d sqlite -p 8.1
diff --git a/Build/testing-docker/local/docker-compose.yml b/Build/testing-docker/local/docker-compose.yml
index 6e4bec6dd1c1..9f630a76fa6a 100644
--- a/Build/testing-docker/local/docker-compose.yml
+++ b/Build/testing-docker/local/docker-compose.yml
@@ -100,9 +100,9 @@ services:
         php -v | grep '^PHP'
         if [ ${CHUNKS} -gt 0 ]; then
           echo \"Running chunk ${THISCHUNK}\"
-          COMMAND=\"bin/codecept run Application -d -g AcceptanceTests-Job-${THISCHUNK} -c typo3/sysext/core/Tests/codeception.yml ${EXTRA_TEST_OPTIONS} ${TEST_FILE} --html reports.html\"
+          COMMAND=\"bin/codecept run Application -d -g AcceptanceTests-Job-${THISCHUNK} -c typo3/sysext/core/Tests/codeception.yml ${EXTRA_TEST_OPTIONS} ${TEST_FILE} --xml reports.xml --html reports.html\"
         else
-          COMMAND=\"bin/codecept run Application -d -c typo3/sysext/core/Tests/codeception.yml ${EXTRA_TEST_OPTIONS} ${TEST_FILE} --html reports.html\"
+          COMMAND=\"bin/codecept run Application -d -c typo3/sysext/core/Tests/codeception.yml ${EXTRA_TEST_OPTIONS} ${TEST_FILE} --xml reports.xml --html reports.html\"
         fi
         mkdir -p typo3temp/var/tests/
         if [ ${PHP_XDEBUG_ON} -eq 0 ]; then
@@ -154,9 +154,9 @@ services:
         php -v | grep '^PHP'
         if [ ${CHUNKS} -gt 0 ]; then
           echo \"Running chunk ${THISCHUNK}\"
-          COMMAND=\"bin/codecept run Application -d -g AcceptanceTests-Job-${THISCHUNK} -c typo3/sysext/core/Tests/codeception.yml ${EXTRA_TEST_OPTIONS} ${TEST_FILE} --html reports.html\"
+          COMMAND=\"bin/codecept run Application -d -g AcceptanceTests-Job-${THISCHUNK} -c typo3/sysext/core/Tests/codeception.yml ${EXTRA_TEST_OPTIONS} ${TEST_FILE} --xml reports.xml --html reports.html\"
         else
-          COMMAND=\"bin/codecept run Application -d -c typo3/sysext/core/Tests/codeception.yml ${EXTRA_TEST_OPTIONS} ${TEST_FILE} --html reports.html\"
+          COMMAND=\"bin/codecept run Application -d -c typo3/sysext/core/Tests/codeception.yml ${EXTRA_TEST_OPTIONS} ${TEST_FILE} --xml reports.xml --html reports.html\"
         fi
         mkdir -p typo3temp/var/tests/
         if [ ${PHP_XDEBUG_ON} -eq 0 ]; then
@@ -211,9 +211,9 @@ services:
         php -v | grep '^PHP'
         if [ ${CHUNKS} -gt 0 ]; then
           echo \"Running chunk ${THISCHUNK}\"
-          COMMAND=\"bin/codecept run Application -d -g AcceptanceTests-Job-${THISCHUNK} -c typo3/sysext/core/Tests/codeception.yml ${EXTRA_TEST_OPTIONS} ${TEST_FILE} --html reports.html\"
+          COMMAND=\"bin/codecept run Application -d -g AcceptanceTests-Job-${THISCHUNK} -c typo3/sysext/core/Tests/codeception.yml ${EXTRA_TEST_OPTIONS} ${TEST_FILE} --xml reports.xml --html reports.html\"
         else
-          COMMAND=\"bin/codecept run Application -d -c typo3/sysext/core/Tests/codeception.yml ${EXTRA_TEST_OPTIONS} ${TEST_FILE} --html reports.html\"
+          COMMAND=\"bin/codecept run Application -d -c typo3/sysext/core/Tests/codeception.yml ${EXTRA_TEST_OPTIONS} ${TEST_FILE} --xml reports.xml --html reports.html\"
         fi
         mkdir -p typo3temp/var/tests/
         if [ ${PHP_XDEBUG_ON} -eq 0 ]; then
@@ -257,9 +257,9 @@ services:
         php -v | grep '^PHP'
         if [ ${CHUNKS} -gt 0 ]; then
           echo \"Running chunk ${THISCHUNK}\"
-          COMMAND=\"bin/codecept run Application -d -g AcceptanceTests-Job-${THISCHUNK} -c typo3/sysext/core/Tests/codeception.yml ${EXTRA_TEST_OPTIONS} ${TEST_FILE} --html reports.html\"
+          COMMAND=\"bin/codecept run Application -d -g AcceptanceTests-Job-${THISCHUNK} -c typo3/sysext/core/Tests/codeception.yml ${EXTRA_TEST_OPTIONS} ${TEST_FILE} --xml reports.xml --html reports.html\"
         else
-          COMMAND=\"bin/codecept run Application -d -c typo3/sysext/core/Tests/codeception.yml ${EXTRA_TEST_OPTIONS} ${TEST_FILE} --html reports.html\"
+          COMMAND=\"bin/codecept run Application -d -c typo3/sysext/core/Tests/codeception.yml ${EXTRA_TEST_OPTIONS} ${TEST_FILE} --xml reports.xml --html reports.html\"
         fi
         mkdir -p typo3temp/var/tests/
         if [ ${PHP_XDEBUG_ON} -eq 0 ]; then
@@ -311,12 +311,12 @@ services:
         mkdir -p typo3temp/var/tests/
         if [ ${PHP_XDEBUG_ON} -eq 0 ]; then
           XDEBUG_MODE=\"off\" \
-          bin/codecept run Install -d -c typo3/sysext/core/Tests/codeception.yml --env=mysql --html reports.html
+          bin/codecept run Install -d -c typo3/sysext/core/Tests/codeception.yml --env=mysql --xml reports.xml --html reports.html
         else
           XDEBUG_MODE=\"debug,develop\" \
           XDEBUG_TRIGGER=\"foo\" \
           XDEBUG_CONFIG=\"client_port=${PHP_XDEBUG_PORT} client_host=host.docker.internal\" \
-          bin/codecept run Install -d -c typo3/sysext/core/Tests/codeception.yml --env=mysql --html reports.html
+          bin/codecept run Install -d -c typo3/sysext/core/Tests/codeception.yml --env=mysql --xml reports.xml --html reports.html
         fi
       "
 
@@ -358,12 +358,12 @@ services:
         mkdir -p typo3temp/var/tests/
         if [ ${PHP_XDEBUG_ON} -eq 0 ]; then
           XDEBUG_MODE=\"off\" \
-          bin/codecept run Install -d -c typo3/sysext/core/Tests/codeception.yml ${EXTRA_TEST_OPTIONS} --env=mysql --html reports.html
+          bin/codecept run Install -d -c typo3/sysext/core/Tests/codeception.yml ${EXTRA_TEST_OPTIONS} --env=mysql --xml reports.xml --html reports.html
         else
           XDEBUG_MODE=\"debug,develop\" \
           XDEBUG_TRIGGER=\"foo\" \
           XDEBUG_CONFIG=\"client_port=${PHP_XDEBUG_PORT} client_host=host.docker.internal\" \
-          bin/codecept run Install -d -c typo3/sysext/core/Tests/codeception.yml ${EXTRA_TEST_OPTIONS} --env=mysql --html reports.html
+          bin/codecept run Install -d -c typo3/sysext/core/Tests/codeception.yml ${EXTRA_TEST_OPTIONS} --env=mysql --xml reports.xml --html reports.html
         fi
       "
 
@@ -405,12 +405,12 @@ services:
         mkdir -p typo3temp/var/tests/
         if [ ${PHP_XDEBUG_ON} -eq 0 ]; then
           XDEBUG_MODE=\"off\" \
-          bin/codecept run Install -d -c typo3/sysext/core/Tests/codeception.yml ${EXTRA_TEST_OPTIONS} --env=postgresql --html reports.html
+          bin/codecept run Install -d -c typo3/sysext/core/Tests/codeception.yml ${EXTRA_TEST_OPTIONS} --env=postgresql --xml reports.xml --html reports.html
         else
           XDEBUG_MODE=\"debug,develop\" \
           XDEBUG_TRIGGER=\"foo\" \
           XDEBUG_CONFIG=\"client_port=${PHP_XDEBUG_PORT} client_host=host.docker.internal\" \
-          bin/codecept run Install -d -c typo3/sysext/core/Tests/codeception.yml ${EXTRA_TEST_OPTIONS} --env=postgresql --html reports.html
+          bin/codecept run Install -d -c typo3/sysext/core/Tests/codeception.yml ${EXTRA_TEST_OPTIONS} --env=postgresql --xml reports.xml --html reports.html
         fi
       "
 
@@ -439,12 +439,12 @@ services:
         mkdir -p typo3temp/var/tests/
         if [ ${PHP_XDEBUG_ON} -eq 0 ]; then
           XDEBUG_MODE=\"off\" \
-          bin/codecept run Install -d -c typo3/sysext/core/Tests/codeception.yml ${EXTRA_TEST_OPTIONS} --env=sqlite --html reports.html
+          bin/codecept run Install -d -c typo3/sysext/core/Tests/codeception.yml ${EXTRA_TEST_OPTIONS} --env=sqlite --xml reports.xml --html reports.html
         else
           XDEBUG_MODE=\"debug,develop\" \
           XDEBUG_TRIGGER=\"foo\" \
           XDEBUG_CONFIG=\"client_port=${PHP_XDEBUG_PORT} client_host=host.docker.internal\" \
-          bin/codecept run Install -d -c typo3/sysext/core/Tests/codeception.yml ${EXTRA_TEST_OPTIONS} --env=sqlite --html reports.html
+          bin/codecept run Install -d -c typo3/sysext/core/Tests/codeception.yml ${EXTRA_TEST_OPTIONS} --env=sqlite --xml reports.xml --html reports.html
         fi
       "