Sibling of 9ddbae6

.ci/travis/linux/before_script.sh

@@ -26,7 +26,7 @@ echo "travis_fold:start:docker_build"
 echo "${bold}Docker build deps${endbold}"
 docker --version
 docker-compose --version
-docker-compose -f ${TRAVIS_BUILD_DIR}/.ci/travis/linux/docker-compose.travis.yml config
+docker-compose -f ${TRAVIS_BUILD_DIR}/.docker/docker-compose.travis.yml config
 docker pull "qgis/qgis3-build-deps:${DOCKER_TAG}" || true
 docker build --cache-from "qgis/qgis3-build-deps:${DOCKER_TAG}" -t "qgis/qgis3-build-deps:${DOCKER_TAG}" -f ${DOCKER_BUILD_DEPS_FILE} .
 echo "travis_fold:end:docker_build"

.ci/travis/linux/scripts/docker-qgis-build.sh → .ci/travis/linux/docker-build-test.sh

@@ -12,6 +12,14 @@ ccache -M 1G
 # export CCACHE_LOGFILE=/tmp/cache.debug
 ccache -z
 
+############################
+# Setup the (c)test environment
+############################
+export LD_PRELOAD=/lib/x86_64-linux-gnu/libSegFault.so
+export SEGFAULT_SIGNALS="abrt segv"
+export CTEST_BUILD_COMMAND="/usr/bin/ninja"
+export CTEST_PARALLEL_LEVEL=1
+
 ##############################
 # Variables for output styling
 ##############################
@@ -67,17 +75,84 @@ echo "travis_fold:end:cmake"
 # Travis will kill the job after approx 150 minutes, we subtract 5 minutes for
 # uploading and subtract the bootstrapping time from that.
 # Hopefully clocks are in sync :)
-
+TRAVIS_TIME=150
+UPLOAD_TIME=5
 CURRENT_TIME=$(date +%s)
-TIMEOUT=$((( TRAVIS_AVAILABLE_TIME - TRAVIS_UPLOAD_TIME ) * 60 - CURRENT_TIME + TRAVIS_TIMESTAMP))
+TIMEOUT=$((( TRAVIS_TIME - UPLOAD_TIME ) * 60 - CURRENT_TIME + TRAVIS_TIMESTAMP))
 TIMEOUT=$(( TIMEOUT < 300 ? 300 : TIMEOUT ))
 echo "Timeout: ${TIMEOUT}s (started at ${TRAVIS_TIMESTAMP}, current: ${CURRENT_TIME})"
 
 # echo "travis_fold:start:ninja-build.1"
 echo "${bold}Building QGIS...${endbold}"
 timeout ${TIMEOUT}s ${CTEST_BUILD_COMMAND}
 # echo "travis_fold:end:ninja-build.1"
+
+rv=$?
+if [ $rv -eq 124 ] ; then
+    printf '\n\n${bold}Build and test timeout. Please restart the build for meaningful results.${endbold}\n'
+    exit #$rv
+fi
+
+# Temporarily uncomment to debug ccache issues
+# echo "travis_fold:start:ccache-debug"
+# cat /tmp/cache.debug
+# echo "travis_fold:end:ccache-debug"
+
+############################
+# Restore postgres test data
+############################
+printf "[qgis_test]\nhost=postgres\nport=5432\ndbname=qgis_test\nuser=docker\npassword=docker" > ~/.pg_service.conf
+export PGUSER=docker
+export PGHOST=postgres
+export PGPASSWORD=docker
+export PGDATABASE=qgis_test
+
+pushd /root/QGIS > /dev/null
+/root/QGIS/tests/testdata/provider/testdata_pg.sh
+popd > /dev/null # /root/QGIS
+
+##############################
+# Restore SQL Server test data
+##############################
+
+echo "Importing SQL Server test data..."
+
+export SQLUSER=sa
+export SQLHOST=mssql
+export SQLPORT=1433
+export SQLPASSWORD='<YourStrong!Passw0rd>'
+export SQLDATABASE=qgis_test
+
+export PATH=$PATH:/opt/mssql-tools/bin
+
+pushd /root/QGIS > /dev/null
+/root/QGIS/tests/testdata/provider/testdata_mssql.sh
+popd > /dev/null # /root/QGIS
+
+echo "Setting up DSN for test SQL Server"
+
+cat <<EOT > /etc/odbc.ini
+[ODBC Data Sources]
+testsqlserver = ODBC Driver 17 for SQL Server
+
+[testsqlserver]
+Driver       = ODBC Driver 17 for SQL Server
+Description  = Test SQL Server
+Server       = mssql
+EOT
+
+###########
+# Run tests
+###########
+CURRENT_TIME=$(date +%s)
+TIMEOUT=$((( TRAVIS_TIME - UPLOAD_TIME) * 60 - CURRENT_TIME + TRAVIS_TIMESTAMP))
+echo "Timeout: ${TIMEOUT}s (started at ${TRAVIS_TIMESTAMP}, current: ${CURRENT_TIME})"
+timeout ${TIMEOUT}s python3 /root/QGIS/.ci/travis/scripts/ctest2travis.py xvfb-run ctest -V -E "$(cat /root/QGIS/.ci/travis/linux/blacklist.txt | sed -r '/^(#.*?)?$/d' | paste -sd '|' -)" -S /root/QGIS/.ci/travis/travis.ctest --output-on-failure
 rv=$?
+if [ $rv -eq 124 ] ; then
+    printf '\n\n${bold}Build and test timeout. Please restart the build for meaningful results.${endbold}\n'
+    exit #$rv
+fi
 
 ########################
 # Show ccache statistics
@@ -91,11 +166,3 @@ popd > /dev/null # build
 popd > /dev/null # /root/QGIS
 
 [ -r /tmp/ctest-important.log ] && cat /tmp/ctest-important.log || true
-
-############################
-# Exit with error if timeout
-############################
-if [ $rv -eq 124 ] ; then
-    printf '\n\n${bold}Build and test timeout. Please restart the build for meaningful results.${endbold}\n'
-    exit #$rv
-fi

.ci/travis/linux/script.sh

@@ -16,63 +16,6 @@
 
 set -e
 
-# build QGIS in docker
-echo "travis_fold:start:docker_build_qgis"
-echo "${bold}Docker build QGIS${endbold}"
-docker run -t --name qgis_container \
-           -v ${TRAVIS_BUILD_DIR}:/root/QGIS \
-           -v ${CCACHE_DIR}:/root/.ccache \
-           --env-file ${TRAVIS_BUILD_DIR}/.ci/travis/linux/docker-variables.env \
-           qgis/qgis3-build-deps:${DOCKER_TAG} \
-           /root/QGIS/.ci/travis/linux/scripts/docker-qgis-build.sh
+# running QGIS tests
+docker-compose -f ${TRAVIS_BUILD_DIR}/.docker/docker-compose.travis.yml run --rm qgis-deps
 
-# commit container
-docker commit qgis_container qgis_image
-echo "travis_fold:end:docker_build_qgis"
-
-# running QGIS tests in commited image
-echo "travis_fold:start:docker_test_qgis"
-echo "${bold}Docker run tests${endbold}"
-docker-compose -f ${TRAVIS_BUILD_DIR}/.ci/travis/linux/docker-compose.travis.yml run qgis-deps /root/QGIS/.ci/travis/linux/scripts/docker-qgis-test.sh
-echo "travis_fold:end:docker_test_qgis"
-
-# running tests for the python test runner
-echo "travis_fold:start:docker_test_runners"
-echo "${bold}Docker test QGIS runners${endbold}"
-docker run -d --name qgis-testing-environment \
-           -v ${TRAVIS_BUILD_DIR}:/root/QGIS \
-           -v ${TRAVIS_BUILD_DIR}/tests/src/python:/tests_directory \
-           -v ${TRAVIS_BUILD_DIR}/.docker/qgis_resources/test_runner:/usr/bin/test_runner \
-           -v ${TRAVIS_BUILD_DIR}/.docker/qgis_resources/supervisor:/etc/supervisor \
-           -e QGIS_BUILD_PATH=/root/QGIS/build/output/bin/qgis \
-           -e TEST_RUNNER_PATH=/usr/bin/test_runner/qgis_testrunner.py \
-           -e DISPLAY=:99 \
-           qgis_image \
-           /usr/bin/supervisord -c /etc/supervisor/supervisord.conf
-
-# Wait for xvfb to finish starting
-printf "Waiting for the docker...🐳..."
-sleep 10
-echo " done 🥩"
-
-declare -A testrunners
-# Passing cases:
-testrunners["test_testrunner.run_passing"]=0
-testrunners["test_testrunner.run_skipped_and_passing"]=0
-# Failing cases:
-testrunners["test_testrunner"]=1
-testrunners["test_testrunner.run_all"]=1
-testrunners["test_testrunner.run_failing"]=1
-set +e # do not exit on error
-# Run tests in the docker
-for i in "${!testrunners[@]}"
-do
-  echo "travis_fold:start:docker_test_runner_${i}"
-  echo "test ${i}..."
-  docker exec -it qgis-testing-environment sh -c "cd /tests_directory && /usr/bin/test_runner/qgis_testrunner.sh ${i}"
-  [[ $? -eq "${testrunners[$i]}" ]] && echo "success" || exit 1
-  echo "travis_fold:end:docker_test_runner_${i}"
-done
-set -e # switch back
-docker stop qgis-testing-environment
-echo "travis_fold:end:docker_test_runners"

.ci/travis/scripts/echo_travis_var.sh

@@ -6,6 +6,7 @@ echo "TRAVIS_BRANCH: $TRAVIS_BRANCH"
 echo "TRAVIS_EVENT_TYPE: $TRAVIS_EVENT_TYPE"
 echo "DOCKER_TAG: $DOCKER_TAG"
 echo "TRAVIS_COMMIT_MESSAGE: $TRAVIS_COMMIT_MESSAGE"
+echo "DOCKER_DEPS_PUSH: $DOCKER_DEPS_PUSH"
 echo "DOCKER_BUILD_DEPS_FILE: $DOCKER_BUILD_DEPS_FILE"
 echo "TRAVIS_TIMESTAMP: $TRAVIS_TIMESTAMP"
 echo "travis_fold:end:travis_env"

.docker/docker-compose.travis.yml

@@ -0,0 +1,35 @@
+version: '3'
+services:
+  postgres:
+    image: kartoza/postgis:9.5-2.2
+    environment:
+      - ALLOW_IP_RANGE="172.18.0.0/16"
+
+  mssql:
+    image: microsoft/mssql-server-linux:2017-latest
+    environment:
+      ACCEPT_EULA: Y
+      SA_PASSWORD: <YourStrong!Passw0rd>
+
+  qgis-deps:
+    tty: true
+    image: qgis/qgis3-build-deps:${DOCKER_TAG}
+    volumes:
+      - ${TRAVIS_BUILD_DIR}:/root/QGIS
+      - ${CCACHE_DIR}:/root/.ccache  # if changed, also change env var
+    links:
+      - postgres
+      - mssql
+    environment:
+      - CCACHE_DIR=/root/.ccache
+      - CTEST_BUILD_DIR=/root/QGIS
+      - TRAVIS_BRANCH=${TRAVIS_BRANCH}
+      - TRAVIS_PULL_REQUEST=${TRAVIS_PULL_REQUEST}
+      - TRAVIS_PULL_REQUEST_BRANCH=${TRAVIS_PULL_REQUEST_BRANCH}
+      - TRAVIS_COMMIT_RANGE=${TRAVIS_COMMIT_RANGE}
+      - TRAVIS_OS_NAME=${TRAVIS_OS_NAME}
+      - TRAVIS_CONFIG=${TRAVIS_CONFIG}
+      - TRAVIS=${TRAVIS}
+      - TRAVIS_TIMESTAMP=${TRAVIS_TIMESTAMP}
+      - QGIS_DISABLE_MESSAGE_HOOKS=1
+      - QGIS_NO_OVERRIDE_IMPORT=1

.docker/qgis.dockerfile

@@ -59,7 +59,8 @@ COPY .docker/qgis_resources/test_runner/qgis_* /usr/bin/
 RUN chmod +x /usr/bin/qgis_*
 
 # Add supervisor service configuration script
-COPY .docker/qgis_resources/supervisor/ /etc/supervisor
+COPY .docker/qgis_resources/supervisor/supervisord.conf /etc/supervisor/
+COPY .docker/qgis_resources/supervisor/supervisor.xvfb.conf /etc/supervisor/supervisor.d/
 
 # Python paths are for
 # - kartoza images (compiled)

.docker/qgis3-build-deps-cosmic.dockerfile

@@ -49,6 +49,7 @@ RUN  apt-get update \
     libqt5svg5-dev \
     libqt5webkit5-dev \
     libqt5xml5 \
+    libqt5xmlpatterns5-dev \
     libqt5serialport5-dev \
     libqwt-qt5-dev \
     libspatialindex-dev \
@@ -135,3 +136,5 @@ ENV CXX=/usr/lib/ccache/clang++
 ENV QT_SELECT=5
 ENV LANG=C.UTF-8
 ENV PATH="/usr/local/bin:${PATH}"
+
+CMD /root/QGIS/.ci/travis/linux/docker-build-test.sh

.docker/qgis3-build-deps.dockerfile

@@ -49,6 +49,7 @@ RUN  apt-get update \
     libqt5svg5-dev \
     libqt5webkit5-dev \
     libqt5xml5 \
+    libqt5xmlpatterns5-dev \
     libqt5serialport5-dev \
     libqwt-qt5-dev \
     libspatialindex-dev \
@@ -149,3 +150,4 @@ ENV QT_SELECT=5
 ENV LANG=C.UTF-8
 ENV PATH="/usr/local/bin:${PATH}"
 
+CMD /root/QGIS/.ci/travis/linux/docker-build-test.sh