Merge remote-tracking branch 'mainstream/master' into distribute-equi…

…spaced2

.ci/travis/linux/before_script.sh

@@ -26,7 +26,7 @@ echo "travis_fold:start:docker_build"
 echo "${bold}Docker build deps${endbold}"
 docker --version
 docker-compose --version
-docker-compose -f ${TRAVIS_BUILD_DIR}/.docker/docker-compose.travis.yml config
+docker-compose -f ${TRAVIS_BUILD_DIR}/.ci/travis/linux/docker-compose.travis.yml config
 docker pull "qgis/qgis3-build-deps:${DOCKER_TAG}" || true
 docker build --cache-from "qgis/qgis3-build-deps:${DOCKER_TAG}" -t "qgis/qgis3-build-deps:${DOCKER_TAG}" -f ${DOCKER_BUILD_DEPS_FILE} .
 echo "travis_fold:end:docker_build"

.ci/travis/linux/docker-compose.travis.yml

@@ -0,0 +1,24 @@
+version: '3'
+services:
+  postgres:
+    image: kartoza/postgis:9.5-2.2
+    environment:
+      - ALLOW_IP_RANGE="172.18.0.0/16"
+
+  mssql:
+    image: microsoft/mssql-server-linux:2017-latest
+    environment:
+      ACCEPT_EULA: Y
+      SA_PASSWORD: <YourStrong!Passw0rd>
+
+  qgis-deps:
+    tty: true
+    image: qgis_image
+    volumes:
+      - ${TRAVIS_BUILD_DIR}:/root/QGIS
+    links:
+      - postgres
+      - mssql
+    env_file:
+      - docker-variables.env
+

.ci/travis/linux/docker-variables.env

@@ -0,0 +1,23 @@
+
+# Env variables for Docker
+# These without assignment are taken from the host env variabbles
+
+# TRAVIS variables
+TRAVIS_AVAILABLE_TIME
+TRAVIS_TIMESTAMP
+TRAVIS_UPLOAD_TIME
+TRAVIS_BRANCH
+TRAVIS_PULL_REQUEST
+TRAVIS_OS_NAME
+TRAVIS_CONFIG
+TRAVIS
+
+# CTEST
+LD_PRELOAD=/lib/x86_64-linux-gnu/libSegFault.so
+SEGFAULT_SIGNALS="abrt segv"
+CTEST_BUILD_COMMAND=/usr/bin/ninja
+CTEST_PARALLEL_LEVEL=1
+CTEST_BUILD_DIR=/root/QGIS
+
+# Other var
+QGIS_NO_OVERRIDE_IMPORT=1

.ci/travis/linux/script.sh

@@ -16,21 +16,63 @@
 
 set -e
 
-# running QGIS tests
-docker-compose -f ${TRAVIS_BUILD_DIR}/.docker/docker-compose.travis.yml run --rm qgis-deps
+# build QGIS in docker
+echo "travis_fold:start:docker_build_qgis"
+echo "${bold}Docker build QGIS${endbold}"
+docker run -t --name qgis_container \
+           -v ${TRAVIS_BUILD_DIR}:/root/QGIS \
+           -v ${CCACHE_DIR}:/root/.ccache \
+           --env-file ${TRAVIS_BUILD_DIR}/.ci/travis/linux/docker-variables.env \
+           qgis/qgis3-build-deps:${DOCKER_TAG} \
+           /root/QGIS/.ci/travis/linux/scripts/docker-qgis-build.sh
+
+# commit container
+docker commit qgis_container qgis_image
+echo "travis_fold:end:docker_build_qgis"
+
+# running QGIS tests in commited image
+echo "travis_fold:start:docker_test_qgis"
+echo "${bold}Docker run tests${endbold}"
+docker-compose -f ${TRAVIS_BUILD_DIR}/.ci/travis/linux/docker-compose.travis.yml run qgis-deps /root/QGIS/.ci/travis/linux/scripts/docker-qgis-test.sh
+echo "travis_fold:end:docker_test_qgis"
 
 # running tests for the python test runner
-docker run -d --name qgis-testing-environment -v ${TRAVIS_BUILD_DIR}/tests/src/python:/tests_directory -e DISPLAY=:99 "qgis/qgis:${DOCKER_TAG}"
-sleep 10  # Wait for xvfb to finish starting
-# Temporary workaround until docker images are built
-docker cp ${TRAVIS_BUILD_DIR}/.docker/qgis_resources/test_runner/qgis_testrunner.sh qgis-testing-environment:/usr/bin/qgis_testrunner.sh
-# Run tests in the docker
+echo "travis_fold:start:docker_test_runners"
+echo "${bold}Docker test QGIS runners${endbold}"
+docker run -d --name qgis-testing-environment \
+           -v ${TRAVIS_BUILD_DIR}:/root/QGIS \
+           -v ${TRAVIS_BUILD_DIR}/tests/src/python:/tests_directory \
+           -v ${TRAVIS_BUILD_DIR}/.docker/qgis_resources/test_runner:/usr/bin/test_runner \
+           -v ${TRAVIS_BUILD_DIR}/.docker/qgis_resources/supervisor:/etc/supervisor \
+           -e QGIS_BUILD_PATH=/root/QGIS/build/output/bin/qgis \
+           -e TEST_RUNNER_PATH=/usr/bin/test_runner/qgis_testrunner.py \
+           -e DISPLAY=:99 \
+           qgis_image \
+           /usr/bin/supervisord -c /etc/supervisor/supervisord.conf
+
+# Wait for xvfb to finish starting
+printf "Waiting for the docker...🐳..."
+sleep 10
+echo " done 🥩"
+
+declare -A testrunners
 # Passing cases:
-TEST_SCRIPT_PATH=${TRAVIS_BUILD_DIR}/.ci/travis/linux/docker_test.sh
-[[ $(${TEST_SCRIPT_PATH} test_testrunner.run_passing) -eq '0' ]]
-[[ $(${TEST_SCRIPT_PATH} test_testrunner.run_skipped_and_passing) -eq '0' ]]
+testrunners["test_testrunner.run_passing"]=0
+testrunners["test_testrunner.run_skipped_and_passing"]=0
 # Failing cases:
-[[ $(${TEST_SCRIPT_PATH} test_testrunner) -eq '1' ]]
-[[ $(${TEST_SCRIPT_PATH} test_testrunner.run_all) -eq '1' ]]
-[[ $(${TEST_SCRIPT_PATH} test_testrunner.run_failing) -eq '1' ]]
-
+testrunners["test_testrunner"]=1
+testrunners["test_testrunner.run_all"]=1
+testrunners["test_testrunner.run_failing"]=1
+set +e # do not exit on error
+# Run tests in the docker
+for i in "${!testrunners[@]}"
+do
+  echo "travis_fold:start:docker_test_runner_${i}"
+  echo "test ${i}..."
+  docker exec -it qgis-testing-environment sh -c "cd /tests_directory && /usr/bin/test_runner/qgis_testrunner.sh ${i}"
+  [[ $? -eq "${testrunners[$i]}" ]] && echo "success" || exit 1
+  echo "travis_fold:end:docker_test_runner_${i}"
+done
+set -e # switch back
+docker stop qgis-testing-environment
+echo "travis_fold:end:docker_test_runners"

.ci/travis/linux/docker-build-test.sh → .ci/travis/linux/scripts/docker-qgis-build.sh

@@ -12,14 +12,6 @@ ccache -M 1G
 # export CCACHE_LOGFILE=/tmp/cache.debug
 ccache -z
 
-############################
-# Setup the (c)test environment
-############################
-export LD_PRELOAD=/lib/x86_64-linux-gnu/libSegFault.so
-export SEGFAULT_SIGNALS="abrt segv"
-export CTEST_BUILD_COMMAND="/usr/bin/ninja"
-export CTEST_PARALLEL_LEVEL=1
-
 ##############################
 # Variables for output styling
 ##############################
@@ -75,84 +67,17 @@ echo "travis_fold:end:cmake"
 # Travis will kill the job after approx 150 minutes, we subtract 5 minutes for
 # uploading and subtract the bootstrapping time from that.
 # Hopefully clocks are in sync :)
-TRAVIS_TIME=150
-UPLOAD_TIME=5
+
 CURRENT_TIME=$(date +%s)
-TIMEOUT=$((( TRAVIS_TIME - UPLOAD_TIME ) * 60 - CURRENT_TIME + TRAVIS_TIMESTAMP))
+TIMEOUT=$((( TRAVIS_AVAILABLE_TIME - TRAVIS_UPLOAD_TIME ) * 60 - CURRENT_TIME + TRAVIS_TIMESTAMP))
 TIMEOUT=$(( TIMEOUT < 300 ? 300 : TIMEOUT ))
 echo "Timeout: ${TIMEOUT}s (started at ${TRAVIS_TIMESTAMP}, current: ${CURRENT_TIME})"
 
 # echo "travis_fold:start:ninja-build.1"
 echo "${bold}Building QGIS...${endbold}"
 timeout ${TIMEOUT}s ${CTEST_BUILD_COMMAND}
 # echo "travis_fold:end:ninja-build.1"
-
-rv=$?
-if [ $rv -eq 124 ] ; then
-    printf '\n\n${bold}Build and test timeout. Please restart the build for meaningful results.${endbold}\n'
-    exit #$rv
-fi
-
-# Temporarily uncomment to debug ccache issues
-# echo "travis_fold:start:ccache-debug"
-# cat /tmp/cache.debug
-# echo "travis_fold:end:ccache-debug"
-
-############################
-# Restore postgres test data
-############################
-printf "[qgis_test]\nhost=postgres\nport=5432\ndbname=qgis_test\nuser=docker\npassword=docker" > ~/.pg_service.conf
-export PGUSER=docker
-export PGHOST=postgres
-export PGPASSWORD=docker
-export PGDATABASE=qgis_test
-
-pushd /root/QGIS > /dev/null
-/root/QGIS/tests/testdata/provider/testdata_pg.sh
-popd > /dev/null # /root/QGIS
-
-##############################
-# Restore SQL Server test data
-##############################
-
-echo "Importing SQL Server test data..."
-
-export SQLUSER=sa
-export SQLHOST=mssql
-export SQLPORT=1433
-export SQLPASSWORD='<YourStrong!Passw0rd>'
-export SQLDATABASE=qgis_test
-
-export PATH=$PATH:/opt/mssql-tools/bin
-
-pushd /root/QGIS > /dev/null
-/root/QGIS/tests/testdata/provider/testdata_mssql.sh
-popd > /dev/null # /root/QGIS
-
-echo "Setting up DSN for test SQL Server"
-
-cat <<EOT > /etc/odbc.ini
-[ODBC Data Sources]
-testsqlserver = ODBC Driver 17 for SQL Server
-
-[testsqlserver]
-Driver       = ODBC Driver 17 for SQL Server
-Description  = Test SQL Server
-Server       = mssql
-EOT
-
-###########
-# Run tests
-###########
-CURRENT_TIME=$(date +%s)
-TIMEOUT=$((( TRAVIS_TIME - UPLOAD_TIME) * 60 - CURRENT_TIME + TRAVIS_TIMESTAMP))
-echo "Timeout: ${TIMEOUT}s (started at ${TRAVIS_TIMESTAMP}, current: ${CURRENT_TIME})"
-timeout ${TIMEOUT}s python3 /root/QGIS/.ci/travis/scripts/ctest2travis.py xvfb-run ctest -V -E "$(cat /root/QGIS/.ci/travis/linux/blacklist.txt | sed -r '/^(#.*?)?$/d' | paste -sd '|' -)" -S /root/QGIS/.ci/travis/travis.ctest --output-on-failure
 rv=$?
-if [ $rv -eq 124 ] ; then
-    printf '\n\n${bold}Build and test timeout. Please restart the build for meaningful results.${endbold}\n'
-    exit #$rv
-fi
 
 ########################
 # Show ccache statistics
@@ -166,3 +91,11 @@ popd > /dev/null # build
 popd > /dev/null # /root/QGIS
 
 [ -r /tmp/ctest-important.log ] && cat /tmp/ctest-important.log || true
+
+############################
+# Exit with error if timeout
+############################
+if [ $rv -eq 124 ] ; then
+    printf '\n\n${bold}Build and test timeout. Please restart the build for meaningful results.${endbold}\n'
+    exit #$rv
+fi

.ci/travis/linux/scripts/docker-qgis-test.sh

@@ -0,0 +1,76 @@
+#!/usr/bin/env bash
+
+set -e
+
+# Temporarily uncomment to debug ccache issues
+# echo "travis_fold:start:ccache-debug"
+# cat /tmp/cache.debug
+# echo "travis_fold:end:ccache-debug"
+
+############################
+# Restore postgres test data
+############################
+printf "[qgis_test]\nhost=postgres\nport=5432\ndbname=qgis_test\nuser=docker\npassword=docker" > ~/.pg_service.conf
+export PGUSER=docker
+export PGHOST=postgres
+export PGPASSWORD=docker
+export PGDATABASE=qgis_test
+
+# wait for the DB to be available
+echo "Wait a moment while loading the database."
+while ! PGPASSWORD='docker' psql -h postgres -U docker -p 5432 -l &> /dev/null
+do
+  printf "🐘"
+  sleep 1
+done
+echo " done 🥩"
+
+pushd /root/QGIS > /dev/null
+/root/QGIS/tests/testdata/provider/testdata_pg.sh
+popd > /dev/null # /root/QGIS
+
+##############################
+# Restore SQL Server test data
+##############################
+
+echo "Importing SQL Server test data..."
+
+export SQLUSER=sa
+export SQLHOST=mssql
+export SQLPORT=1433
+export SQLPASSWORD='<YourStrong!Passw0rd>'
+export SQLDATABASE=qgis_test
+
+export PATH=$PATH:/opt/mssql-tools/bin
+
+pushd /root/QGIS > /dev/null
+/root/QGIS/tests/testdata/provider/testdata_mssql.sh
+popd > /dev/null # /root/QGIS
+
+echo "Setting up DSN for test SQL Server"
+
+cat <<EOT > /etc/odbc.ini
+[ODBC Data Sources]
+testsqlserver = ODBC Driver 17 for SQL Server
+
+[testsqlserver]
+Driver       = ODBC Driver 17 for SQL Server
+Description  = Test SQL Server
+Server       = mssql
+EOT
+
+###########
+# Run tests
+###########
+CURRENT_TIME=$(date +%s)
+TIMEOUT=$((( TRAVIS_AVAILABLE_TIME - TRAVIS_UPLOAD_TIME) * 60 - CURRENT_TIME + TRAVIS_TIMESTAMP))
+echo "Timeout: ${TIMEOUT}s (started at ${TRAVIS_TIMESTAMP}, current: ${CURRENT_TIME})"
+timeout ${TIMEOUT}s python3 /root/QGIS/.ci/travis/scripts/ctest2travis.py xvfb-run ctest -V -E "$(cat /root/QGIS/.ci/travis/linux/scripts/test_blacklist.txt | sed -r '/^(#.*?)?$/d' | paste -sd '|' -)" -S /root/QGIS/.ci/travis/travis.ctest --output-on-failure
+rv=$?
+if [ $rv -eq 124 ] ; then
+    printf '\n\n${bold}Build and test timeout. Please restart the build for meaningful results.${endbold}\n'
+    exit #$rv
+fi
+
+
+

.ci/travis/scripts/echo_travis_var.sh

@@ -6,7 +6,6 @@ echo "TRAVIS_BRANCH: $TRAVIS_BRANCH"
 echo "TRAVIS_EVENT_TYPE: $TRAVIS_EVENT_TYPE"
 echo "DOCKER_TAG: $DOCKER_TAG"
 echo "TRAVIS_COMMIT_MESSAGE: $TRAVIS_COMMIT_MESSAGE"
-echo "DOCKER_DEPS_PUSH: $DOCKER_DEPS_PUSH"
 echo "DOCKER_BUILD_DEPS_FILE: $DOCKER_BUILD_DEPS_FILE"
 echo "TRAVIS_TIMESTAMP: $TRAVIS_TIMESTAMP"
 echo "travis_fold:end:travis_env"