simplify CI scripts

by separating the testing and docker image building scripts
there was almost no redundany in the scripts, it is much clearer now

.ci/travis/docker_image/before_install.sh

@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+###########################################################################
+#    before_install.sh
+#    ---------------------
+#    Date                 : March 2016
+#    Copyright            : (C) 2016 by Matthias Kuhn
+#    Email                : matthias at opengis dot ch
+###########################################################################
+#                                                                         #
+#   This program is free software; you can redistribute it and/or modify  #
+#   it under the terms of the GNU General Public License as published by  #
+#   the Free Software Foundation; either version 2 of the License, or     #
+#   (at your option) any later version.                                   #
+#                                                                         #
+###########################################################################
+

.ci/travis/docker_image/before_script.sh

@@ -0,0 +1,35 @@
+#!/usr/bin/env bash
+###########################################################################
+#    before_script.sh
+#    ---------------------
+#    Date                 : August 2015
+#    Copyright            : (C) 2015 by Nyall Dawson
+#    Email                : nyall dot dawson at gmail dot com
+###########################################################################
+#                                                                         #
+#   This program is free software; you can redistribute it and/or modify  #
+#   it under the terms of the GNU General Public License as published by  #
+#   the Free Software Foundation; either version 2 of the License, or     #
+#   (at your option) any later version.                                   #
+#                                                                         #
+###########################################################################
+
+set -e
+
+.ci/travis/scripts/echo_travis_var.sh
+
+pushd .docker
+
+
+
+echo "travis_fold:start:docker_build"
+echo "${bold}Docker build deps${endbold}"
+docker --version
+
+docker pull "qgis/qgis3-build-deps:${DOCKER_TAG}" || true
+docker build --cache-from "qgis/qgis3-build-deps:${DOCKER_TAG}" -t "qgis/qgis3-build-deps:${DOCKER_TAG}" -f ${DOCKER_BUILD_DEPS_FILE} .
+
+echo "travis_fold:end:docker_build"
+
+
+popd

.ci/travis/docker_image/install.sh

@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+###########################################################################
+#    install.sh
+#    ---------------------
+#    Date                 : March 2016
+#    Copyright            : (C) 2016 by Matthias Kuhn
+#    Email                : matthias at opengis dot ch
+###########################################################################
+#                                                                         #
+#   This program is free software; you can redistribute it and/or modify  #
+#   it under the terms of the GNU General Public License as published by  #
+#   the Free Software Foundation; either version 2 of the License, or     #
+#   (at your option) any later version.                                   #
+#                                                                         #
+###########################################################################
+

.ci/travis/docker_image/script.sh

@@ -0,0 +1,57 @@
+#!/usr/bin/env bash
+###########################################################################
+#    script.sh
+#    ---------------------
+#    Date                 : March 2016
+#    Copyright            : (C) 2016 by Matthias Kuhn
+#    Email                : matthias at opengis dot ch
+###########################################################################
+#                                                                         #
+#   This program is free software; you can redistribute it and/or modify  #
+#   it under the terms of the GNU General Public License as published by  #
+#   the Free Software Foundation; either version 2 of the License, or     #
+#   (at your option) any later version.                                   #
+#                                                                         #
+###########################################################################
+
+set -e
+
+mkdir -p "CCACHE_DIR_IMAGE_BUILD"
+
+# copy ccache dir within QGIS source so it can be accessed from docker
+cp -r ${CCACHE_DIR_IMAGE_BUILD} ${TRAVIS_BUILD_DIR}/.ccache_image_build
+# building docker images
+DIR=$(git rev-parse --show-toplevel)/.docker
+pushd "${DIR}"
+echo "${bold}Building QGIS Docker image '${DOCKER_TAG}'...${endbold}"
+docker build --build-arg DOCKER_TAG="${DOCKER_TAG}" \
+             --cache-from "qgis/qgis:${DOCKER_TAG}" \
+             -t "qgis/qgis:${DOCKER_TAG}" \
+             -f qgis.dockerfile ..
+echo "${bold}Pushing image to docker hub...${endbold}"
+docker login -u="$DOCKER_USERNAME" -p="$DOCKER_PASSWORD"
+docker push "qgis/qgis:${DOCKER_TAG}"
+echo "Copy build cache from Docker container to Travis cache directory"
+rm -rf "${CCACHE_DIR_IMAGE_BUILD:?}/"*
+container_id=$(docker images -q "qgis/qgis:${DOCKER_TAG}")
+docker cp ${container_id}:/usr/src/QGIS/.ccache_image_build ${CCACHE_DIR_IMAGE_BUILD}
+popd
+echo "Trigger build of PyQGIS Documentation"
+body='{
+  "request": {
+    "branch":"master",
+    "message": "Trigger PyQGIS doc build after release of new Docker image as __DOCKER_TAG__",
+    "config": {
+      "merge_mode": "deep_merge",
+      "matrix": {
+        "include": {
+          "env": ["QGIS_VERSION_BRANCH=__QGIS_VERSION_BRANCH__"]
+        }
+      }
+    }
+  }
+}'
+body=$(sed "s/__QGIS_VERSION_BRANCH__/${TRAVIS_BRANCH}/; s/__DOCKER_TAG__/${DOCKER_TAG}/" <<< $body)
+curl -s -X POST -H "Content-Type: application/json" -H "Accept: application/json" \
+  -H "Travis-API-Version: 3" -H "Authorization: token $TRAVIS_TOKEN" -d "$body" \
+  https://api.travis-ci.org/repo/qgis%2Fpyqgis/requests

.ci/travis/linux/before_install.sh

@@ -14,4 +14,3 @@
 #                                                                         #
 ###########################################################################
 
-#pip3 install termcolor

.ci/travis/linux/before_script.sh

@@ -19,36 +19,16 @@ set -e
 pushd .docker
 
 DOCKER_DEPS_PUSH=$( [[ $TRAVIS_REPO_SLUG =~ qgis/QGIS ]] && [[ "${TRAVIS_EVENT_TYPE}" != "pull_request" ]] && echo "true" || echo "false" )
-DOCKER_DEPS_IMAGE_REBUILD=$( [[ $TRAVIS_COMMIT_MESSAGE =~ '[docker] update dependencies' ]] && echo "true" || echo "false" )
-
-echo "travis_fold:start:travis_env"
-echo "${bold}Travis environment variables${endbold}"
-echo "TRAVIS_BRANCH: $TRAVIS_BRANCH"
-echo "TRAVIS_EVENT_TYPE: $TRAVIS_EVENT_TYPE"
-echo "DOCKER_TAG: $DOCKER_TAG"
-echo "TRAVIS_COMMIT_MESSAGE: $TRAVIS_COMMIT_MESSAGE"
-echo "DOCKER_DEPS_PUSH: $DOCKER_DEPS_PUSH"
-echo "DOCKER_DEPS_IMAGE_REBUILD: $DOCKER_DEPS_IMAGE_REBUILD"
-echo "DOCKER_BUILD_QGIS_IMAGE: $DOCKER_BUILD_QGIS_IMAGE"
-echo "DOCKER_BUILD_DEPS_FILE: $DOCKER_BUILD_DEPS_FILE"
-echo "TRAVIS_TIMESTAMP: $TRAVIS_TIMESTAMP"
-echo "travis_fold:end:travis_env"
 
+.ci/travis/scripts/echo_travis_var.sh
 
 echo "travis_fold:start:docker_build"
 echo "${bold}Docker build deps${endbold}"
 docker --version
-if [[ $DOCKER_BUILD_QGIS_IMAGE =~ false ]]; then
-  docker-compose --version
-  docker-compose -f ${TRAVIS_BUILD_DIR}/.docker/docker-compose.travis.yml config
-fi
-
-if [[ $DOCKER_DEPS_IMAGE_REBUILD =~ true ]]; then
-  docker build --no-cache -t "qgis/qgis3-build-deps:${DOCKER_TAG}" -f ${DOCKER_BUILD_DEPS_FILE} .
-else
-  docker pull "qgis/qgis3-build-deps:${DOCKER_TAG}" || true
-  docker build --cache-from "qgis/qgis3-build-deps:${DOCKER_TAG}" -t "qgis/qgis3-build-deps:${DOCKER_TAG}" -f ${DOCKER_BUILD_DEPS_FILE} .
-fi
+docker-compose --version
+docker-compose -f ${TRAVIS_BUILD_DIR}/.docker/docker-compose.travis.yml config
+docker pull "qgis/qgis3-build-deps:${DOCKER_TAG}" || true
+docker build --cache-from "qgis/qgis3-build-deps:${DOCKER_TAG}" -t "qgis/qgis3-build-deps:${DOCKER_TAG}" -f ${DOCKER_BUILD_DEPS_FILE} .
 echo "travis_fold:end:docker_build"
 
 echo "travis_fold:start:docker_push"

.ci/travis/linux/script.sh

@@ -16,62 +16,21 @@
 
 set -e
 
-mkdir -p "CCACHE_DIR_IMAGE_BUILD"
+# running QGIS tests
+docker-compose -f ${TRAVIS_BUILD_DIR}/.docker/docker-compose.travis.yml run --rm qgis-deps
 
-if [[ ${DOCKER_BUILD_QGIS_IMAGE} =~ true ]]; then
-  # copy ccache dir within QGIS source so it can be accessed from docker
-  cp -r ${CCACHE_DIR_IMAGE_BUILD} ${TRAVIS_BUILD_DIR}/.ccache_image_build
-  # building docker images
-  DIR=$(git rev-parse --show-toplevel)/.docker
-  pushd "${DIR}"
-  echo "${bold}Building QGIS Docker image '${DOCKER_TAG}'...${endbold}"
-  docker build --build-arg DOCKER_TAG="${DOCKER_TAG}" \
-               --cache-from "qgis/qgis:${DOCKER_TAG}" \
-               -t "qgis/qgis:${DOCKER_TAG}" \
-               -f qgis.dockerfile ..
-  echo "${bold}Pushing image to docker hub...${endbold}"
-  docker login -u="$DOCKER_USERNAME" -p="$DOCKER_PASSWORD"
-  docker push "qgis/qgis:${DOCKER_TAG}"
-  echo "Copy build cache from Docker container to Travis cache directory"
-  rm -rf "${CCACHE_DIR_IMAGE_BUILD:?}/"*
-  container_id=$(docker images -q "qgis/qgis:${DOCKER_TAG}")
-  docker cp ${container_id}:/usr/src/QGIS/.ccache_image_build ${CCACHE_DIR_IMAGE_BUILD}
-  popd
-  echo "Trigger build of PyQGIS Documentation"
-  body='{
-    "request": {
-      "branch":"master",
-      "message": "Trigger PyQGIS doc build after release of new Docker image as __DOCKER_TAG__",
-      "config": {
-        "merge_mode": "deep_merge",
-        "matrix": {
-          "include": {
-            "env": ["QGIS_VERSION_BRANCH=__QGIS_VERSION_BRANCH__"]
-          }
-        }
-      }
-    }
-  }'
-  body=$(sed "s/__QGIS_VERSION_BRANCH__/${TRAVIS_BRANCH}/; s/__DOCKER_TAG__/${DOCKER_TAG}/" <<< $body)
-  curl -s -X POST -H "Content-Type: application/json" -H "Accept: application/json" \
-    -H "Travis-API-Version: 3" -H "Authorization: token $TRAVIS_TOKEN" -d "$body" \
-    https://api.travis-ci.org/repo/qgis%2Fpyqgis/requests
-else
-  # running QGIS tests
-  docker-compose -f ${TRAVIS_BUILD_DIR}/.docker/docker-compose.travis.yml run --rm qgis-deps
+# running tests for the python test runner
+docker run -d --name qgis-testing-environment -v ${TRAVIS_BUILD_DIR}/tests/src/python:/tests_directory -e DISPLAY=:99 "qgis/qgis:${DOCKER_TAG}"
+sleep 10  # Wait for xvfb to finish starting
+# Temporary workaround until docker images are built
+docker cp ${TRAVIS_BUILD_DIR}/.docker/qgis_resources/test_runner/qgis_testrunner.sh qgis-testing-environment:/usr/bin/qgis_testrunner.sh
+# Run tests in the docker
+# Passing cases:
+TEST_SCRIPT_PATH=${TRAVIS_BUILD_DIR}/.ci/travis/linux/docker_test.sh
+[[ $(${TEST_SCRIPT_PATH} test_testrunner.run_passing) -eq '0' ]]
+[[ $(${TEST_SCRIPT_PATH} test_testrunner.run_skipped_and_passing) -eq '0' ]]
+# Failing cases:
+[[ $(${TEST_SCRIPT_PATH} test_testrunner) -eq '1' ]]
+[[ $(${TEST_SCRIPT_PATH} test_testrunner.run_all) -eq '1' ]]
+[[ $(${TEST_SCRIPT_PATH} test_testrunner.run_failing) -eq '1' ]]
 
-  # running tests for the python test runner
-  docker run -d --name qgis-testing-environment -v ${TRAVIS_BUILD_DIR}/tests/src/python:/tests_directory -e DISPLAY=:99 "qgis/qgis:${DOCKER_TAG}"
-  sleep 10  # Wait for xvfb to finish starting
-  # Temporary workaround until docker images are built
-  docker cp ${TRAVIS_BUILD_DIR}/.docker/qgis_resources/test_runner/qgis_testrunner.sh qgis-testing-environment:/usr/bin/qgis_testrunner.sh
-  # Run tests in the docker
-  # Passing cases:
-  TEST_SCRIPT_PATH=${TRAVIS_BUILD_DIR}/.ci/travis/linux/docker_test.sh
-  [[ $(${TEST_SCRIPT_PATH} test_testrunner.run_passing) -eq '0' ]]
-  [[ $(${TEST_SCRIPT_PATH} test_testrunner.run_skipped_and_passing) -eq '0' ]]
-  # Failing cases:
-  [[ $(${TEST_SCRIPT_PATH} test_testrunner) -eq '1' ]]
-  [[ $(${TEST_SCRIPT_PATH} test_testrunner.run_all) -eq '1' ]]
-  [[ $(${TEST_SCRIPT_PATH} test_testrunner.run_failing) -eq '1' ]]
-fi

.ci/travis/scripts/echo_travis_var.sh

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+
+echo "travis_fold:start:travis_env"
+echo "${bold}Travis environment variables${endbold}"
+echo "TRAVIS_BRANCH: $TRAVIS_BRANCH"
+echo "TRAVIS_EVENT_TYPE: $TRAVIS_EVENT_TYPE"
+echo "DOCKER_TAG: $DOCKER_TAG"
+echo "TRAVIS_COMMIT_MESSAGE: $TRAVIS_COMMIT_MESSAGE"
+echo "DOCKER_DEPS_PUSH: $DOCKER_DEPS_PUSH"
+echo "DOCKER_BUILD_DEPS_FILE: $DOCKER_BUILD_DEPS_FILE"
+echo "TRAVIS_TIMESTAMP: $TRAVIS_TIMESTAMP"
+echo "travis_fold:end:travis_env"

.travis.yml

@@ -43,7 +43,6 @@ matrix:
       env:
         - TRAVIS_CONFIG=linux
         - DOCKER_TAG=$( [[ $TRAVIS_REPO_SLUG =~ qgis/QGIS ]] && echo $TRAVIS_BRANCH | sed 's/master/latest/' || echo "latest" )
-        - DOCKER_BUILD_QGIS_IMAGE=false
         - DOCKER_BUILD_DEPS_FILE=qgis3-build-deps.dockerfile
 
       ##########################################################
@@ -104,9 +103,8 @@ matrix:
       if: repo = qgis/QGIS AND (tag IS PRESENT OR type = cron)
       services: docker
       env:
-        - TRAVIS_CONFIG=linux
+        - TRAVIS_CONFIG=docker_image
         - DOCKER_TAG=$( echo $TRAVIS_BRANCH | sed 's/master/latest/' )
-        - DOCKER_BUILD_QGIS_IMAGE=true
         - DOCKER_BUILD_DEPS_FILE=qgis3-build-deps.dockerfile
 
       ##########################################################
@@ -117,9 +115,8 @@ matrix:
       if: repo = qgis/QGIS AND (tag IS PRESENT OR type = cron)
       services: docker
       env:
-        - TRAVIS_CONFIG=linux
+        - TRAVIS_CONFIG=docker_image
         - DOCKER_TAG="$( echo $TRAVIS_BRANCH | sed 's/master/latest/' )_cosmic"
-        - DOCKER_BUILD_QGIS_IMAGE=true
         - DOCKER_BUILD_DEPS_FILE=qgis3-build-deps-cosmic.dockerfile