Navigation Menu

Skip to content

Commit

Permalink
simplify CI scripts
Browse files Browse the repository at this point in the history
by separating the testing and docker image building scripts
there was almost no redundany in the scripts, it is much clearer now
  • Loading branch information
3nids committed Mar 5, 2019
1 parent 03fcded commit 742cee1
Show file tree
Hide file tree
Showing 9 changed files with 159 additions and 88 deletions.
16 changes: 16 additions & 0 deletions .ci/travis/docker_image/before_install.sh
@@ -0,0 +1,16 @@
#!/usr/bin/env bash
###########################################################################
# before_install.sh
# ---------------------
# Date : March 2016
# Copyright : (C) 2016 by Matthias Kuhn
# Email : matthias at opengis dot ch
###########################################################################
# #
# This program is free software; you can redistribute it and/or modify #
# it under the terms of the GNU General Public License as published by #
# the Free Software Foundation; either version 2 of the License, or #
# (at your option) any later version. #
# #
###########################################################################

35 changes: 35 additions & 0 deletions .ci/travis/docker_image/before_script.sh
@@ -0,0 +1,35 @@
#!/usr/bin/env bash
###########################################################################
# before_script.sh
# ---------------------
# Date : August 2015
# Copyright : (C) 2015 by Nyall Dawson
# Email : nyall dot dawson at gmail dot com
###########################################################################
# #
# This program is free software; you can redistribute it and/or modify #
# it under the terms of the GNU General Public License as published by #
# the Free Software Foundation; either version 2 of the License, or #
# (at your option) any later version. #
# #
###########################################################################

set -e

.ci/travis/scripts/echo_travis_var.sh

pushd .docker



echo "travis_fold:start:docker_build"
echo "${bold}Docker build deps${endbold}"
docker --version

docker pull "qgis/qgis3-build-deps:${DOCKER_TAG}" || true
docker build --cache-from "qgis/qgis3-build-deps:${DOCKER_TAG}" -t "qgis/qgis3-build-deps:${DOCKER_TAG}" -f ${DOCKER_BUILD_DEPS_FILE} .

echo "travis_fold:end:docker_build"


popd
16 changes: 16 additions & 0 deletions .ci/travis/docker_image/install.sh
@@ -0,0 +1,16 @@
#!/usr/bin/env bash
###########################################################################
# install.sh
# ---------------------
# Date : March 2016
# Copyright : (C) 2016 by Matthias Kuhn
# Email : matthias at opengis dot ch
###########################################################################
# #
# This program is free software; you can redistribute it and/or modify #
# it under the terms of the GNU General Public License as published by #
# the Free Software Foundation; either version 2 of the License, or #
# (at your option) any later version. #
# #
###########################################################################

57 changes: 57 additions & 0 deletions .ci/travis/docker_image/script.sh
@@ -0,0 +1,57 @@
#!/usr/bin/env bash
###########################################################################
# script.sh
# ---------------------
# Date : March 2016
# Copyright : (C) 2016 by Matthias Kuhn
# Email : matthias at opengis dot ch
###########################################################################
# #
# This program is free software; you can redistribute it and/or modify #
# it under the terms of the GNU General Public License as published by #
# the Free Software Foundation; either version 2 of the License, or #
# (at your option) any later version. #
# #
###########################################################################

set -e

mkdir -p "CCACHE_DIR_IMAGE_BUILD"

# copy ccache dir within QGIS source so it can be accessed from docker
cp -r ${CCACHE_DIR_IMAGE_BUILD} ${TRAVIS_BUILD_DIR}/.ccache_image_build
# building docker images
DIR=$(git rev-parse --show-toplevel)/.docker
pushd "${DIR}"
echo "${bold}Building QGIS Docker image '${DOCKER_TAG}'...${endbold}"
docker build --build-arg DOCKER_TAG="${DOCKER_TAG}" \
--cache-from "qgis/qgis:${DOCKER_TAG}" \
-t "qgis/qgis:${DOCKER_TAG}" \
-f qgis.dockerfile ..
echo "${bold}Pushing image to docker hub...${endbold}"
docker login -u="$DOCKER_USERNAME" -p="$DOCKER_PASSWORD"
docker push "qgis/qgis:${DOCKER_TAG}"
echo "Copy build cache from Docker container to Travis cache directory"
rm -rf "${CCACHE_DIR_IMAGE_BUILD:?}/"*
container_id=$(docker images -q "qgis/qgis:${DOCKER_TAG}")
docker cp ${container_id}:/usr/src/QGIS/.ccache_image_build ${CCACHE_DIR_IMAGE_BUILD}
popd
echo "Trigger build of PyQGIS Documentation"
body='{
"request": {
"branch":"master",
"message": "Trigger PyQGIS doc build after release of new Docker image as __DOCKER_TAG__",
"config": {
"merge_mode": "deep_merge",
"matrix": {
"include": {
"env": ["QGIS_VERSION_BRANCH=__QGIS_VERSION_BRANCH__"]
}
}
}
}
}'
body=$(sed "s/__QGIS_VERSION_BRANCH__/${TRAVIS_BRANCH}/; s/__DOCKER_TAG__/${DOCKER_TAG}/" <<< $body)
curl -s -X POST -H "Content-Type: application/json" -H "Accept: application/json" \
-H "Travis-API-Version: 3" -H "Authorization: token $TRAVIS_TOKEN" -d "$body" \
https://api.travis-ci.org/repo/qgis%2Fpyqgis/requests
1 change: 0 additions & 1 deletion .ci/travis/linux/before_install.sh
Expand Up @@ -14,4 +14,3 @@
# #
###########################################################################

#pip3 install termcolor
30 changes: 5 additions & 25 deletions .ci/travis/linux/before_script.sh
Expand Up @@ -19,36 +19,16 @@ set -e
pushd .docker

DOCKER_DEPS_PUSH=$( [[ $TRAVIS_REPO_SLUG =~ qgis/QGIS ]] && [[ "${TRAVIS_EVENT_TYPE}" != "pull_request" ]] && echo "true" || echo "false" )
DOCKER_DEPS_IMAGE_REBUILD=$( [[ $TRAVIS_COMMIT_MESSAGE =~ '[docker] update dependencies' ]] && echo "true" || echo "false" )

echo "travis_fold:start:travis_env"
echo "${bold}Travis environment variables${endbold}"
echo "TRAVIS_BRANCH: $TRAVIS_BRANCH"
echo "TRAVIS_EVENT_TYPE: $TRAVIS_EVENT_TYPE"
echo "DOCKER_TAG: $DOCKER_TAG"
echo "TRAVIS_COMMIT_MESSAGE: $TRAVIS_COMMIT_MESSAGE"
echo "DOCKER_DEPS_PUSH: $DOCKER_DEPS_PUSH"
echo "DOCKER_DEPS_IMAGE_REBUILD: $DOCKER_DEPS_IMAGE_REBUILD"
echo "DOCKER_BUILD_QGIS_IMAGE: $DOCKER_BUILD_QGIS_IMAGE"
echo "DOCKER_BUILD_DEPS_FILE: $DOCKER_BUILD_DEPS_FILE"
echo "TRAVIS_TIMESTAMP: $TRAVIS_TIMESTAMP"
echo "travis_fold:end:travis_env"

.ci/travis/scripts/echo_travis_var.sh

echo "travis_fold:start:docker_build"
echo "${bold}Docker build deps${endbold}"
docker --version
if [[ $DOCKER_BUILD_QGIS_IMAGE =~ false ]]; then
docker-compose --version
docker-compose -f ${TRAVIS_BUILD_DIR}/.docker/docker-compose.travis.yml config
fi

if [[ $DOCKER_DEPS_IMAGE_REBUILD =~ true ]]; then
docker build --no-cache -t "qgis/qgis3-build-deps:${DOCKER_TAG}" -f ${DOCKER_BUILD_DEPS_FILE} .
else
docker pull "qgis/qgis3-build-deps:${DOCKER_TAG}" || true
docker build --cache-from "qgis/qgis3-build-deps:${DOCKER_TAG}" -t "qgis/qgis3-build-deps:${DOCKER_TAG}" -f ${DOCKER_BUILD_DEPS_FILE} .
fi
docker-compose --version
docker-compose -f ${TRAVIS_BUILD_DIR}/.docker/docker-compose.travis.yml config
docker pull "qgis/qgis3-build-deps:${DOCKER_TAG}" || true
docker build --cache-from "qgis/qgis3-build-deps:${DOCKER_TAG}" -t "qgis/qgis3-build-deps:${DOCKER_TAG}" -f ${DOCKER_BUILD_DEPS_FILE} .
echo "travis_fold:end:docker_build"

echo "travis_fold:start:docker_push"
Expand Down
73 changes: 16 additions & 57 deletions .ci/travis/linux/script.sh
Expand Up @@ -16,62 +16,21 @@

set -e

mkdir -p "CCACHE_DIR_IMAGE_BUILD"
# running QGIS tests
docker-compose -f ${TRAVIS_BUILD_DIR}/.docker/docker-compose.travis.yml run --rm qgis-deps

if [[ ${DOCKER_BUILD_QGIS_IMAGE} =~ true ]]; then
# copy ccache dir within QGIS source so it can be accessed from docker
cp -r ${CCACHE_DIR_IMAGE_BUILD} ${TRAVIS_BUILD_DIR}/.ccache_image_build
# building docker images
DIR=$(git rev-parse --show-toplevel)/.docker
pushd "${DIR}"
echo "${bold}Building QGIS Docker image '${DOCKER_TAG}'...${endbold}"
docker build --build-arg DOCKER_TAG="${DOCKER_TAG}" \
--cache-from "qgis/qgis:${DOCKER_TAG}" \
-t "qgis/qgis:${DOCKER_TAG}" \
-f qgis.dockerfile ..
echo "${bold}Pushing image to docker hub...${endbold}"
docker login -u="$DOCKER_USERNAME" -p="$DOCKER_PASSWORD"
docker push "qgis/qgis:${DOCKER_TAG}"
echo "Copy build cache from Docker container to Travis cache directory"
rm -rf "${CCACHE_DIR_IMAGE_BUILD:?}/"*
container_id=$(docker images -q "qgis/qgis:${DOCKER_TAG}")
docker cp ${container_id}:/usr/src/QGIS/.ccache_image_build ${CCACHE_DIR_IMAGE_BUILD}
popd
echo "Trigger build of PyQGIS Documentation"
body='{
"request": {
"branch":"master",
"message": "Trigger PyQGIS doc build after release of new Docker image as __DOCKER_TAG__",
"config": {
"merge_mode": "deep_merge",
"matrix": {
"include": {
"env": ["QGIS_VERSION_BRANCH=__QGIS_VERSION_BRANCH__"]
}
}
}
}
}'
body=$(sed "s/__QGIS_VERSION_BRANCH__/${TRAVIS_BRANCH}/; s/__DOCKER_TAG__/${DOCKER_TAG}/" <<< $body)
curl -s -X POST -H "Content-Type: application/json" -H "Accept: application/json" \
-H "Travis-API-Version: 3" -H "Authorization: token $TRAVIS_TOKEN" -d "$body" \
https://api.travis-ci.org/repo/qgis%2Fpyqgis/requests
else
# running QGIS tests
docker-compose -f ${TRAVIS_BUILD_DIR}/.docker/docker-compose.travis.yml run --rm qgis-deps
# running tests for the python test runner
docker run -d --name qgis-testing-environment -v ${TRAVIS_BUILD_DIR}/tests/src/python:/tests_directory -e DISPLAY=:99 "qgis/qgis:${DOCKER_TAG}"
sleep 10 # Wait for xvfb to finish starting
# Temporary workaround until docker images are built
docker cp ${TRAVIS_BUILD_DIR}/.docker/qgis_resources/test_runner/qgis_testrunner.sh qgis-testing-environment:/usr/bin/qgis_testrunner.sh
# Run tests in the docker
# Passing cases:
TEST_SCRIPT_PATH=${TRAVIS_BUILD_DIR}/.ci/travis/linux/docker_test.sh
[[ $(${TEST_SCRIPT_PATH} test_testrunner.run_passing) -eq '0' ]]
[[ $(${TEST_SCRIPT_PATH} test_testrunner.run_skipped_and_passing) -eq '0' ]]
# Failing cases:
[[ $(${TEST_SCRIPT_PATH} test_testrunner) -eq '1' ]]
[[ $(${TEST_SCRIPT_PATH} test_testrunner.run_all) -eq '1' ]]
[[ $(${TEST_SCRIPT_PATH} test_testrunner.run_failing) -eq '1' ]]

# running tests for the python test runner
docker run -d --name qgis-testing-environment -v ${TRAVIS_BUILD_DIR}/tests/src/python:/tests_directory -e DISPLAY=:99 "qgis/qgis:${DOCKER_TAG}"
sleep 10 # Wait for xvfb to finish starting
# Temporary workaround until docker images are built
docker cp ${TRAVIS_BUILD_DIR}/.docker/qgis_resources/test_runner/qgis_testrunner.sh qgis-testing-environment:/usr/bin/qgis_testrunner.sh
# Run tests in the docker
# Passing cases:
TEST_SCRIPT_PATH=${TRAVIS_BUILD_DIR}/.ci/travis/linux/docker_test.sh
[[ $(${TEST_SCRIPT_PATH} test_testrunner.run_passing) -eq '0' ]]
[[ $(${TEST_SCRIPT_PATH} test_testrunner.run_skipped_and_passing) -eq '0' ]]
# Failing cases:
[[ $(${TEST_SCRIPT_PATH} test_testrunner) -eq '1' ]]
[[ $(${TEST_SCRIPT_PATH} test_testrunner.run_all) -eq '1' ]]
[[ $(${TEST_SCRIPT_PATH} test_testrunner.run_failing) -eq '1' ]]
fi
12 changes: 12 additions & 0 deletions .ci/travis/scripts/echo_travis_var.sh
@@ -0,0 +1,12 @@
#!/usr/bin/env bash

echo "travis_fold:start:travis_env"
echo "${bold}Travis environment variables${endbold}"
echo "TRAVIS_BRANCH: $TRAVIS_BRANCH"
echo "TRAVIS_EVENT_TYPE: $TRAVIS_EVENT_TYPE"
echo "DOCKER_TAG: $DOCKER_TAG"
echo "TRAVIS_COMMIT_MESSAGE: $TRAVIS_COMMIT_MESSAGE"
echo "DOCKER_DEPS_PUSH: $DOCKER_DEPS_PUSH"
echo "DOCKER_BUILD_DEPS_FILE: $DOCKER_BUILD_DEPS_FILE"
echo "TRAVIS_TIMESTAMP: $TRAVIS_TIMESTAMP"
echo "travis_fold:end:travis_env"
7 changes: 2 additions & 5 deletions .travis.yml
Expand Up @@ -43,7 +43,6 @@ matrix:
env:
- TRAVIS_CONFIG=linux
- DOCKER_TAG=$( [[ $TRAVIS_REPO_SLUG =~ qgis/QGIS ]] && echo $TRAVIS_BRANCH | sed 's/master/latest/' || echo "latest" )
- DOCKER_BUILD_QGIS_IMAGE=false
- DOCKER_BUILD_DEPS_FILE=qgis3-build-deps.dockerfile

##########################################################
Expand Down Expand Up @@ -104,9 +103,8 @@ matrix:
if: repo = qgis/QGIS AND (tag IS PRESENT OR type = cron)
services: docker
env:
- TRAVIS_CONFIG=linux
- TRAVIS_CONFIG=docker_image
- DOCKER_TAG=$( echo $TRAVIS_BRANCH | sed 's/master/latest/' )
- DOCKER_BUILD_QGIS_IMAGE=true
- DOCKER_BUILD_DEPS_FILE=qgis3-build-deps.dockerfile

##########################################################
Expand All @@ -117,9 +115,8 @@ matrix:
if: repo = qgis/QGIS AND (tag IS PRESENT OR type = cron)
services: docker
env:
- TRAVIS_CONFIG=linux
- TRAVIS_CONFIG=docker_image
- DOCKER_TAG="$( echo $TRAVIS_BRANCH | sed 's/master/latest/' )_cosmic"
- DOCKER_BUILD_QGIS_IMAGE=true
- DOCKER_BUILD_DEPS_FILE=qgis3-build-deps-cosmic.dockerfile


Expand Down

0 comments on commit 742cee1

Please sign in to comment.