[TASK] runTests.sh: Allow chunking of acceptance and functional tasks

The '-c x/y' option for acceptance and functional jobs now allows to split the set of acceptance/functional tests into chuncks just like the CI does to share resources between multiple parallel jobs. Adding this to runTests.sh allows to replay the exact same list of tests that CI does. This example will execute tests of chunk 3 from 10 chunks with mariadb and php 7.4: Build/Scripts/runTests.sh -s functional -d mariadb -p 7.4 -c 3/10 Resolves: #93361 Releases: master, 10.4, 9.5 Change-Id: I5753a8f83a3ebec6823cb0b44d4ca35f3a64055f Reviewed-on: https://review.typo3.org/c/Packages/TYPO3.CMS/+/67527 Tested-by: TYPO3com <noreply@typo3.com> Tested-by: Christian Kuhn <lolli@schwarzbu.ch> Reviewed-by: Christian Kuhn <lolli@schwarzbu.ch>
TYPO3 · Jan 25, 2021 · 5dcd532 · 5dcd532
1 parent 4762a7f
commit 5dcd532
Show file tree

Hide file tree

Showing 3 changed files with 106 additions and 38 deletions.
diff --git a/.gitignore b/.gitignore
@@ -40,6 +40,7 @@ nbproject
 /Build/testing-docker/local/.env
 /typo3/sysext/*/Resources/Private/TypeScript/*.js
 /typo3/sysext/*/Resources/Public/JavaScript/*.js.map
+typo3/sysext/core/Tests/Acceptance/AcceptanceTests-Job-*
 #
 # TypeScript stuff
 .baseDir.ts

diff --git a/Build/Scripts/runTests.sh b/Build/Scripts/runTests.sh
@@ -35,6 +35,9 @@ setUpDockerComposeDotEnv() {
     echo "MYSQL_VERSION=${MYSQL_VERSION}" >> .env
     echo "POSTGRES_VERSION=${POSTGRES_VERSION}" >> .env
     echo "PHP_VERSION=${PHP_VERSION}" >> .env
+    echo "CHUNKS=${CHUNKS}" >> .env
+    echo "THISCHUNK=${THISCHUNK}" >> .env
+
     # Set a custom database driver provided by option: -a
     [[ ! -z "$DATABASE_DRIVER" ]] && echo "DATABASE_DRIVER=${DATABASE_DRIVER}" >> .env
 }
@@ -52,19 +55,6 @@ Usage: $0 [options] [file]
 No arguments: Run all unit tests with PHP 7.2
 
 Options:
-    -a <mysqli|pdo_mysql|sqlsrv|pdo_sqlsrv>
-        Only with -s functional
-        Specifies to use another driver, following combinations are available:
-            - mysql
-                - mysqli (default)
-                - pdo_mysql
-            - mariadb
-                - mysqli (default)
-                - pdo_mysql
-            - mssql
-                - sqlsrv (default)
-                - pdo_sqlsrv
-
     -s <...>
         Specifies which test suite to run
             - acceptance: backend acceptance tests
@@ -101,6 +91,19 @@ Options:
             - unitJavascript: JavaScript unit tests
             - unitRandom: PHP unit tests in random order, add -o <number> to use specific seed
 
+    -a <mysqli|pdo_mysql|sqlsrv|pdo_sqlsrv>
+        Only with -s functional
+        Specifies to use another driver, following combinations are available:
+            - mysql
+                - mysqli (default)
+                - pdo_mysql
+            - mariadb
+                - mysqli (default)
+                - pdo_mysql
+            - mssql
+                - sqlsrv (default)
+                - pdo_sqlsrv
+
     -d <mariadb|mysql|mssql|postgres|sqlite>
         Only with -s install|functional
         Specifies on which DBMS tests are performed
@@ -136,6 +139,11 @@ Options:
             - 12
              -13
 
+    -c <chunk/numberOfChunks>
+        Only with -s functional|acceptance
+        Hack functional or acceptance tests into #numberOfChunks pieces and run tests of #chunk.
+        Example -c 3/13
+
     -p <7.2|7.3|7.4|8.0>
         Specifies the PHP minor version to be used
             - 7.2 (default): use PHP 7.2
@@ -248,20 +256,31 @@ DOCKER_JS_IMAGE="js"
 MARIADB_VERSION="10.3"
 MYSQL_VERSION="5.5"
 POSTGRES_VERSION="10"
+CHUNKS=0
+THISCHUNK=0
 
 # Option parsing
 # Reset in case getopts has been used previously in the shell
 OPTIND=1
 # Array for invalid options
 INVALID_OPTIONS=();
 # Simple option parsing based on getopts (! not getopt)
-while getopts ":a:s:d:i:j:k:p:e:xy:o:nhuv" OPT; do
+while getopts ":a:s:c:d:i:j:k:p:e:xy:o:nhuv" OPT; do
     case ${OPT} in
+        s)
+            TEST_SUITE=${OPTARG}
+            ;;
         a)
             DATABASE_DRIVER=${OPTARG}
             ;;
-        s)
-            TEST_SUITE=${OPTARG}
+        c)
+            if ! [[ ${OPTARG} =~ ^([0-9]+\/[0-9]+)$ ]]; then
+                INVALID_OPTIONS+=(${OPTARG})
+            else
+                # Split "2/13" - run chunk 2 of 13 chunks
+                THISCHUNK=`echo ${OPTARG} | cut -d '/' -f1`
+                CHUNKS=`echo ${OPTARG} | cut -d '/' -f2`
+            fi
             ;;
         d)
             DBMS=${OPTARG}
@@ -350,6 +369,9 @@ fi
 case ${TEST_SUITE} in
     acceptance)
         setUpDockerComposeDotEnv
+        if [ ${CHUNKS} -gt 1 ]; then
+            docker-compose run acceptance_split
+        fi
         docker-compose run prepare_acceptance_backend_mariadb
         docker-compose run acceptance_backend_mariadb
         SUITE_EXIT_CODE=$?
@@ -495,6 +517,9 @@ case ${TEST_SUITE} in
         ;;
     functional)
         setUpDockerComposeDotEnv
+        if [ ${CHUNKS} -gt 1 ]; then
+            docker-compose run functional_split
+        fi
         case ${DBMS} in
             mariadb)
                 [[ ! -z "$DATABASE_DRIVER" ]] && echo "Using driver: ${DATABASE_DRIVER}"

diff --git a/Build/testing-docker/local/docker-compose.yml b/Build/testing-docker/local/docker-compose.yml
@@ -48,6 +48,13 @@ services:
   memcached1-5:
     image: memcached:1.5-alpine
 
+  acceptance_split:
+    image: typo3gmbh/${DOCKER_PHP_IMAGE}:latest
+    user: ${HOST_UID}
+    volumes:
+      - ${CORE_ROOT}:${CORE_ROOT}
+    working_dir: ${CORE_ROOT}
+    command: php vendor/typo3/testing-framework/Resources/Core/Build/Scripts/splitAcceptanceTests.php -v ${CHUNKS}
 
   prepare_acceptance_backend_mariadb:
     image: alpine:3.8
@@ -85,8 +92,15 @@ services:
         if [ ${SCRIPT_VERBOSE} -eq 1 ]; then
           set -x
         fi
-        mkdir -p typo3temp/var/tests/ \
-          && ./bin/codecept run Backend -d -c typo3/sysext/core/Tests/codeception.yml ${TEST_FILE} --html reports.html
+        php -v | grep '^PHP'
+        if [ ${CHUNKS} -gt 0 ]; then
+          echo \"Running chunk ${THISCHUNK}\"
+          COMMAND=\"bin/codecept run Backend -d -g AcceptanceTests-Job-${THISCHUNK} -c typo3/sysext/core/Tests/codeception.yml ${TEST_FILE} --xml reports.xml --html reports.html\"
+        else
+          COMMAND=\"bin/codecept run Backend -d -c typo3/sysext/core/Tests/codeception.yml ${TEST_FILE} --xml reports.xml --html reports.html\"
+        fi
+        mkdir -p typo3temp/var/tests/
+        $${COMMAND}
       "
 
   prepare_acceptance_pagetree_mariadb:
@@ -626,6 +640,14 @@ services:
         composer validate;
       "
 
+  functional_split:
+    image: typo3gmbh/${DOCKER_PHP_IMAGE}:latest
+    user: ${HOST_UID}
+    volumes:
+      - ${CORE_ROOT}:${CORE_ROOT}
+    working_dir: ${CORE_ROOT}
+    command: php vendor/typo3/testing-framework/Resources/Core/Build/Scripts/splitFunctionalTests.php -v ${CHUNKS}
+
   prepare_functional_mariadb:
     image: alpine:3.8
     links:
@@ -666,13 +688,17 @@ services:
           set -x
         fi
         php -v | grep '^PHP'
+        if [ ${CHUNKS} -gt 0 ]; then
+          echo \"Running chunk ${THISCHUNK}\"
+          COMMAND=\"vendor/phpunit/phpunit/phpunit -c vendor/typo3/testing-framework/Resources/Core/Build/FunctionalTests-Job-${THISCHUNK}.xml ${EXTRA_TEST_OPTIONS} ${TEST_FILE}\"
+        else
+          COMMAND=\"vendor/phpunit/phpunit/phpunit -c vendor/typo3/testing-framework/Resources/Core/Build/FunctionalTests.xml ${EXTRA_TEST_OPTIONS} ${TEST_FILE}\"
+        fi
         if [ ${PHP_XDEBUG_ON} -eq 0 ]; then
-          php -n -c /etc/php/cli-no-xdebug/php.ini \
-            vendor/phpunit/phpunit/phpunit -c vendor/typo3/testing-framework/Resources/Core/Build/FunctionalTests.xml ${EXTRA_TEST_OPTIONS} ${TEST_FILE};
+          php -n -c /etc/php/cli-no-xdebug/php.ini $${COMMAND};
         else
           DOCKER_HOST=`route -n | awk '/^0.0.0.0/ { print $$2 }'`
-          XDEBUG_CONFIG=\"remote_port=${PHP_XDEBUG_PORT} remote_enable=1 remote_host=$${DOCKER_HOST}\" \
-            vendor/phpunit/phpunit/phpunit -c vendor/typo3/testing-framework/Resources/Core/Build/FunctionalTests.xml ${EXTRA_TEST_OPTIONS} ${TEST_FILE};
+          XDEBUG_CONFIG=\"remote_port=${PHP_XDEBUG_PORT} remote_enable=1 remote_host=$${DOCKER_HOST}\" $${COMMAND};
         fi
       "
 
@@ -716,13 +742,17 @@ services:
           set -x
         fi
         php -v | grep '^PHP'
+        if [ ${CHUNKS} -gt 0 ]; then
+          echo \"Running chunk ${THISCHUNK}\"
+          COMMAND=\"vendor/phpunit/phpunit/phpunit -c vendor/typo3/testing-framework/Resources/Core/Build/FunctionalTests-Job-${THISCHUNK}.xml ${EXTRA_TEST_OPTIONS} ${TEST_FILE}\"
+        else
+          COMMAND=\"vendor/phpunit/phpunit/phpunit -c vendor/typo3/testing-framework/Resources/Core/Build/FunctionalTests.xml ${EXTRA_TEST_OPTIONS} ${TEST_FILE}\"
+        fi
         if [ ${PHP_XDEBUG_ON} -eq 0 ]; then
-          php -n -c /etc/php/cli-no-xdebug/php.ini \
-            vendor/phpunit/phpunit/phpunit -c vendor/typo3/testing-framework/Resources/Core/Build/FunctionalTests.xml ${EXTRA_TEST_OPTIONS} ${TEST_FILE};
+          php -n -c /etc/php/cli-no-xdebug/php.ini $${COMMAND};
         else
           DOCKER_HOST=`route -n | awk '/^0.0.0.0/ { print $$2 }'`
-          XDEBUG_CONFIG=\"remote_port=${PHP_XDEBUG_PORT} remote_enable=1 remote_host=$${DOCKER_HOST}\" \
-            vendor/phpunit/phpunit/phpunit -c vendor/typo3/testing-framework/Resources/Core/Build/FunctionalTests.xml ${EXTRA_TEST_OPTIONS} ${TEST_FILE};
+          XDEBUG_CONFIG=\"remote_port=${PHP_XDEBUG_PORT} remote_enable=1 remote_host=$${DOCKER_HOST}\" $${COMMAND};
         fi
       "
 
@@ -769,13 +799,17 @@ services:
           set -x
         fi
         php -v | grep '^PHP'
+        if [ ${CHUNKS} -gt 0 ]; then
+          echo \"Running chunk ${THISCHUNK}\"
+          COMMAND=\"vendor/phpunit/phpunit/phpunit -c vendor/typo3/testing-framework/Resources/Core/Build/FunctionalTests-Job-${THISCHUNK}.xml ${EXTRA_TEST_OPTIONS} --exclude-group not-mssql ${TEST_FILE}\"
+        else
+          COMMAND=\"vendor/phpunit/phpunit/phpunit -c vendor/typo3/testing-framework/Resources/Core/Build/FunctionalTests.xml ${EXTRA_TEST_OPTIONS} --exclude-group not-mssql ${TEST_FILE}\"
+        fi
         if [ ${PHP_XDEBUG_ON} -eq 0 ]; then
-          php -n -c /etc/php/cli-no-xdebug/php.ini \
-            vendor/phpunit/phpunit/phpunit -c vendor/typo3/testing-framework/Resources/Core/Build/FunctionalTests.xml ${EXTRA_TEST_OPTIONS} --exclude-group not-mssql ${TEST_FILE};
+          php -n -c /etc/php/cli-no-xdebug/php.ini $${COMMAND};
         else
           DOCKER_HOST=`route -n | awk '/^0.0.0.0/ { print $$2 }'`
-          XDEBUG_CONFIG=\"remote_port=${PHP_XDEBUG_PORT} remote_enable=1 remote_host=$${DOCKER_HOST}\" \
-            vendor/phpunit/phpunit/phpunit -c vendor/typo3/testing-framework/Resources/Core/Build/FunctionalTests.xml ${EXTRA_TEST_OPTIONS} --exclude-group not-mssql ${TEST_FILE};
+          XDEBUG_CONFIG=\"remote_port=${PHP_XDEBUG_PORT} remote_enable=1 remote_host=$${DOCKER_HOST}\" $${COMMAND};
         fi
       "
 
@@ -819,13 +853,17 @@ services:
           set -x
         fi
         php -v | grep '^PHP'
+        if [ ${CHUNKS} -gt 0 ]; then
+          echo \"Running chunk ${THISCHUNK}\"
+          COMMAND=\"vendor/phpunit/phpunit/phpunit -c vendor/typo3/testing-framework/Resources/Core/Build/FunctionalTests-Job-${THISCHUNK}.xml ${EXTRA_TEST_OPTIONS} --exclude-group not-postgres ${TEST_FILE}\"
+        else
+          COMMAND=\"vendor/phpunit/phpunit/phpunit -c vendor/typo3/testing-framework/Resources/Core/Build/FunctionalTests.xml ${EXTRA_TEST_OPTIONS} --exclude-group not-postgres ${TEST_FILE}\"
+        fi
         if [ ${PHP_XDEBUG_ON} -eq 0 ]; then
-          php -n -c /etc/php/cli-no-xdebug/php.ini \
-            vendor/phpunit/phpunit/phpunit -c vendor/typo3/testing-framework/Resources/Core/Build/FunctionalTests.xml ${EXTRA_TEST_OPTIONS} --exclude-group not-postgres ${TEST_FILE};
+          php -n -c /etc/php/cli-no-xdebug/php.ini $${COMMAND};
         else
           DOCKER_HOST=`route -n | awk '/^0.0.0.0/ { print $$2 }'`
-          XDEBUG_CONFIG=\"remote_port=${PHP_XDEBUG_PORT} remote_enable=1 remote_host=$${DOCKER_HOST}\" \
-            vendor/phpunit/phpunit/phpunit -c vendor/typo3/testing-framework/Resources/Core/Build/FunctionalTests.xml ${EXTRA_TEST_OPTIONS} --exclude-group not-postgres ${TEST_FILE};
+          XDEBUG_CONFIG=\"remote_port=${PHP_XDEBUG_PORT} remote_enable=1 remote_host=$${DOCKER_HOST}\" $${COMMAND};
         fi
       "
 
@@ -861,13 +899,17 @@ services:
           set -x
         fi
         php -v | grep '^PHP'
+        if [ ${CHUNKS} -gt 0 ]; then
+          echo \"Running chunk ${THISCHUNK}\"
+          COMMAND=\"vendor/phpunit/phpunit/phpunit -c vendor/typo3/testing-framework/Resources/Core/Build/FunctionalTests-Job-${THISCHUNK}.xml ${EXTRA_TEST_OPTIONS} --exclude-group not-sqlite ${TEST_FILE}\"
+        else
+          COMMAND=\"vendor/phpunit/phpunit/phpunit -c vendor/typo3/testing-framework/Resources/Core/Build/FunctionalTests.xml ${EXTRA_TEST_OPTIONS} --exclude-group not-sqlite ${TEST_FILE}\"
+        fi
         if [ ${PHP_XDEBUG_ON} -eq 0 ]; then
-          php -n -c /etc/php/cli-no-xdebug/php.ini \
-            vendor/phpunit/phpunit/phpunit -c vendor/typo3/testing-framework/Resources/Core/Build/FunctionalTests.xml ${EXTRA_TEST_OPTIONS} --exclude-group not-sqlite ${TEST_FILE};
+          php -n -c /etc/php/cli-no-xdebug/php.ini $${COMMAND};
         else
           DOCKER_HOST=`route -n | awk '/^0.0.0.0/ { print $$2 }'`
-          XDEBUG_CONFIG=\"remote_port=${PHP_XDEBUG_PORT} remote_enable=1 remote_host=$${DOCKER_HOST}\" \
-            vendor/phpunit/phpunit/phpunit -c vendor/typo3/testing-framework/Resources/Core/Build/FunctionalTests.xml ${EXTRA_TEST_OPTIONS} --exclude-group not-sqlite ${TEST_FILE};
+          XDEBUG_CONFIG=\"remote_port=${PHP_XDEBUG_PORT} remote_enable=1 remote_host=$${DOCKER_HOST}\" $${COMMAND};
         fi
       "