Skip to content
Permalink
Browse files

simplify CI scripts (#9387)

by separating the testing and docker image building scripts
there was almost no redundany in the scripts, it is much clearer now
  • Loading branch information
3nids committed Mar 5, 2019
1 parent 47f1f4c commit 1e11e1157a8f5ec645c2e5267d6f31ab810ad110
@@ -0,0 +1,16 @@
#!/usr/bin/env bash
###########################################################################
# before_install.sh
# ---------------------
# Date : March 2019
# Copyright : (C) 2019 by Denis Rouzaud
# Email : denis@opengis.ch
###########################################################################
# #
# This program is free software; you can redistribute it and/or modify #
# it under the terms of the GNU General Public License as published by #
# the Free Software Foundation; either version 2 of the License, or #
# (at your option) any later version. #
# #
###########################################################################

@@ -0,0 +1,36 @@
#!/usr/bin/env bash
###########################################################################
# before_install.sh
# ---------------------
# Date : March 2019
# Copyright : (C) 2019 by Denis Rouzaud
# Email : denis@opengis.ch
###########################################################################
# #
# This program is free software; you can redistribute it and/or modify #
# it under the terms of the GNU General Public License as published by #
# the Free Software Foundation; either version 2 of the License, or #
# (at your option) any later version. #
# #
###########################################################################


set -e

.ci/travis/scripts/echo_travis_var.sh

pushd .docker



echo "travis_fold:start:docker_build"
echo "${bold}Docker build deps${endbold}"
docker --version

docker pull "qgis/qgis3-build-deps:${DOCKER_TAG}" || true
docker build --cache-from "qgis/qgis3-build-deps:${DOCKER_TAG}" -t "qgis/qgis3-build-deps:${DOCKER_TAG}" -f ${DOCKER_BUILD_DEPS_FILE} .

echo "travis_fold:end:docker_build"


popd
@@ -0,0 +1,17 @@
#!/usr/bin/env bash
###########################################################################
# before_install.sh
# ---------------------
# Date : March 2019
# Copyright : (C) 2019 by Denis Rouzaud
# Email : denis@opengis.ch
###########################################################################
# #
# This program is free software; you can redistribute it and/or modify #
# it under the terms of the GNU General Public License as published by #
# the Free Software Foundation; either version 2 of the License, or #
# (at your option) any later version. #
# #
###########################################################################


@@ -0,0 +1,59 @@
#!/usr/bin/env bash
###########################################################################
# before_install.sh
# ---------------------
# Date : March 2019
# Copyright : (C) 2019 by Denis Rouzaud
# Email : denis@opengis.ch
###########################################################################
# #
# This program is free software; you can redistribute it and/or modify #
# it under the terms of the GNU General Public License as published by #
# the Free Software Foundation; either version 2 of the License, or #
# (at your option) any later version. #
# #
###########################################################################


set -e

mkdir -p "CCACHE_DIR_IMAGE_BUILD"

# copy ccache dir within QGIS source so it can be accessed from docker
cp -r ${CCACHE_DIR_IMAGE_BUILD} ${TRAVIS_BUILD_DIR}/.ccache_image_build
# building docker images
DIR=$(git rev-parse --show-toplevel)/.docker
pushd "${DIR}"
echo "${bold}Building QGIS Docker image '${DOCKER_TAG}'...${endbold}"
docker build --build-arg DOCKER_TAG="${DOCKER_TAG}" \
--cache-from "qgis/qgis:${DOCKER_TAG}" \
-t "qgis/qgis:${DOCKER_TAG}" \
-f qgis.dockerfile ..
echo "${bold}Pushing image to docker hub...${endbold}"
docker login -u="$DOCKER_USERNAME" -p="$DOCKER_PASSWORD"
docker push "qgis/qgis:${DOCKER_TAG}"
echo "Copy build cache from Docker container to Travis cache directory"
rm -rf "${CCACHE_DIR_IMAGE_BUILD:?}/"*
container_id=$(docker images -q "qgis/qgis:${DOCKER_TAG}")
docker start ${container_id}
docker cp ${container_id}:/usr/src/QGIS/.ccache_image_build ${CCACHE_DIR_IMAGE_BUILD}
popd
echo "Trigger build of PyQGIS Documentation"
body='{
"request": {
"branch":"master",
"message": "Trigger PyQGIS doc build after release of new Docker image as __DOCKER_TAG__",
"config": {
"merge_mode": "deep_merge",
"matrix": {
"include": {
"env": ["QGIS_VERSION_BRANCH=__QGIS_VERSION_BRANCH__"]
}
}
}
}
}'
body=$(sed "s/__QGIS_VERSION_BRANCH__/${TRAVIS_BRANCH}/; s/__DOCKER_TAG__/${DOCKER_TAG}/" <<< $body)
curl -s -X POST -H "Content-Type: application/json" -H "Accept: application/json" \
-H "Travis-API-Version: 3" -H "Authorization: token $TRAVIS_TOKEN" -d "$body" \
https://api.travis-ci.org/repo/qgis%2Fpyqgis/requests
@@ -14,4 +14,3 @@
# #
###########################################################################

#pip3 install termcolor
@@ -16,39 +16,19 @@

set -e

pushd .docker

DOCKER_DEPS_PUSH=$( [[ $TRAVIS_REPO_SLUG =~ qgis/QGIS ]] && [[ "${TRAVIS_EVENT_TYPE}" != "pull_request" ]] && echo "true" || echo "false" )
DOCKER_DEPS_IMAGE_REBUILD=$( [[ $TRAVIS_COMMIT_MESSAGE =~ '[docker] update dependencies' ]] && echo "true" || echo "false" )

echo "travis_fold:start:travis_env"
echo "${bold}Travis environment variables${endbold}"
echo "TRAVIS_BRANCH: $TRAVIS_BRANCH"
echo "TRAVIS_EVENT_TYPE: $TRAVIS_EVENT_TYPE"
echo "DOCKER_TAG: $DOCKER_TAG"
echo "TRAVIS_COMMIT_MESSAGE: $TRAVIS_COMMIT_MESSAGE"
echo "DOCKER_DEPS_PUSH: $DOCKER_DEPS_PUSH"
echo "DOCKER_DEPS_IMAGE_REBUILD: $DOCKER_DEPS_IMAGE_REBUILD"
echo "DOCKER_BUILD_QGIS_IMAGE: $DOCKER_BUILD_QGIS_IMAGE"
echo "DOCKER_BUILD_DEPS_FILE: $DOCKER_BUILD_DEPS_FILE"
echo "TRAVIS_TIMESTAMP: $TRAVIS_TIMESTAMP"
echo "travis_fold:end:travis_env"
.ci/travis/scripts/echo_travis_var.sh

pushd .docker

echo "travis_fold:start:docker_build"
echo "${bold}Docker build deps${endbold}"
docker --version
if [[ $DOCKER_BUILD_QGIS_IMAGE =~ false ]]; then
docker-compose --version
docker-compose -f ${TRAVIS_BUILD_DIR}/.docker/docker-compose.travis.yml config
fi

if [[ $DOCKER_DEPS_IMAGE_REBUILD =~ true ]]; then
docker build --no-cache -t "qgis/qgis3-build-deps:${DOCKER_TAG}" -f ${DOCKER_BUILD_DEPS_FILE} .
else
docker pull "qgis/qgis3-build-deps:${DOCKER_TAG}" || true
docker build --cache-from "qgis/qgis3-build-deps:${DOCKER_TAG}" -t "qgis/qgis3-build-deps:${DOCKER_TAG}" -f ${DOCKER_BUILD_DEPS_FILE} .
fi
docker-compose --version
docker-compose -f ${TRAVIS_BUILD_DIR}/.docker/docker-compose.travis.yml config
docker pull "qgis/qgis3-build-deps:${DOCKER_TAG}" || true
docker build --cache-from "qgis/qgis3-build-deps:${DOCKER_TAG}" -t "qgis/qgis3-build-deps:${DOCKER_TAG}" -f ${DOCKER_BUILD_DEPS_FILE} .
echo "travis_fold:end:docker_build"

echo "travis_fold:start:docker_push"
@@ -16,62 +16,21 @@

set -e

mkdir -p "CCACHE_DIR_IMAGE_BUILD"
# running QGIS tests
docker-compose -f ${TRAVIS_BUILD_DIR}/.docker/docker-compose.travis.yml run --rm qgis-deps

if [[ ${DOCKER_BUILD_QGIS_IMAGE} =~ true ]]; then
# copy ccache dir within QGIS source so it can be accessed from docker
cp -r ${CCACHE_DIR_IMAGE_BUILD} ${TRAVIS_BUILD_DIR}/.ccache_image_build
# building docker images
DIR=$(git rev-parse --show-toplevel)/.docker
pushd "${DIR}"
echo "${bold}Building QGIS Docker image '${DOCKER_TAG}'...${endbold}"
docker build --build-arg DOCKER_TAG="${DOCKER_TAG}" \
--cache-from "qgis/qgis:${DOCKER_TAG}" \
-t "qgis/qgis:${DOCKER_TAG}" \
-f qgis.dockerfile ..
echo "${bold}Pushing image to docker hub...${endbold}"
docker login -u="$DOCKER_USERNAME" -p="$DOCKER_PASSWORD"
docker push "qgis/qgis:${DOCKER_TAG}"
echo "Copy build cache from Docker container to Travis cache directory"
rm -rf "${CCACHE_DIR_IMAGE_BUILD:?}/"*
container_id=$(docker images -q "qgis/qgis:${DOCKER_TAG}")
docker cp ${container_id}:/usr/src/QGIS/.ccache_image_build ${CCACHE_DIR_IMAGE_BUILD}
popd
echo "Trigger build of PyQGIS Documentation"
body='{
"request": {
"branch":"master",
"message": "Trigger PyQGIS doc build after release of new Docker image as __DOCKER_TAG__",
"config": {
"merge_mode": "deep_merge",
"matrix": {
"include": {
"env": ["QGIS_VERSION_BRANCH=__QGIS_VERSION_BRANCH__"]
}
}
}
}
}'
body=$(sed "s/__QGIS_VERSION_BRANCH__/${TRAVIS_BRANCH}/; s/__DOCKER_TAG__/${DOCKER_TAG}/" <<< $body)
curl -s -X POST -H "Content-Type: application/json" -H "Accept: application/json" \
-H "Travis-API-Version: 3" -H "Authorization: token $TRAVIS_TOKEN" -d "$body" \
https://api.travis-ci.org/repo/qgis%2Fpyqgis/requests
else
# running QGIS tests
docker-compose -f ${TRAVIS_BUILD_DIR}/.docker/docker-compose.travis.yml run --rm qgis-deps
# running tests for the python test runner
docker run -d --name qgis-testing-environment -v ${TRAVIS_BUILD_DIR}/tests/src/python:/tests_directory -e DISPLAY=:99 "qgis/qgis:${DOCKER_TAG}"
sleep 10 # Wait for xvfb to finish starting
# Temporary workaround until docker images are built
docker cp ${TRAVIS_BUILD_DIR}/.docker/qgis_resources/test_runner/qgis_testrunner.sh qgis-testing-environment:/usr/bin/qgis_testrunner.sh
# Run tests in the docker
# Passing cases:
TEST_SCRIPT_PATH=${TRAVIS_BUILD_DIR}/.ci/travis/linux/docker_test.sh
[[ $(${TEST_SCRIPT_PATH} test_testrunner.run_passing) -eq '0' ]]
[[ $(${TEST_SCRIPT_PATH} test_testrunner.run_skipped_and_passing) -eq '0' ]]
# Failing cases:
[[ $(${TEST_SCRIPT_PATH} test_testrunner) -eq '1' ]]
[[ $(${TEST_SCRIPT_PATH} test_testrunner.run_all) -eq '1' ]]
[[ $(${TEST_SCRIPT_PATH} test_testrunner.run_failing) -eq '1' ]]

# running tests for the python test runner
docker run -d --name qgis-testing-environment -v ${TRAVIS_BUILD_DIR}/tests/src/python:/tests_directory -e DISPLAY=:99 "qgis/qgis:${DOCKER_TAG}"
sleep 10 # Wait for xvfb to finish starting
# Temporary workaround until docker images are built
docker cp ${TRAVIS_BUILD_DIR}/.docker/qgis_resources/test_runner/qgis_testrunner.sh qgis-testing-environment:/usr/bin/qgis_testrunner.sh
# Run tests in the docker
# Passing cases:
TEST_SCRIPT_PATH=${TRAVIS_BUILD_DIR}/.ci/travis/linux/docker_test.sh
[[ $(${TEST_SCRIPT_PATH} test_testrunner.run_passing) -eq '0' ]]
[[ $(${TEST_SCRIPT_PATH} test_testrunner.run_skipped_and_passing) -eq '0' ]]
# Failing cases:
[[ $(${TEST_SCRIPT_PATH} test_testrunner) -eq '1' ]]
[[ $(${TEST_SCRIPT_PATH} test_testrunner.run_all) -eq '1' ]]
[[ $(${TEST_SCRIPT_PATH} test_testrunner.run_failing) -eq '1' ]]
fi
@@ -0,0 +1,12 @@
#!/usr/bin/env bash

echo "travis_fold:start:travis_env"
echo "${bold}Travis environment variables${endbold}"
echo "TRAVIS_BRANCH: $TRAVIS_BRANCH"
echo "TRAVIS_EVENT_TYPE: $TRAVIS_EVENT_TYPE"
echo "DOCKER_TAG: $DOCKER_TAG"
echo "TRAVIS_COMMIT_MESSAGE: $TRAVIS_COMMIT_MESSAGE"
echo "DOCKER_DEPS_PUSH: $DOCKER_DEPS_PUSH"
echo "DOCKER_BUILD_DEPS_FILE: $DOCKER_BUILD_DEPS_FILE"
echo "TRAVIS_TIMESTAMP: $TRAVIS_TIMESTAMP"
echo "travis_fold:end:travis_env"
@@ -43,7 +43,6 @@ matrix:
env:
- TRAVIS_CONFIG=linux
- DOCKER_TAG=$( [[ $TRAVIS_REPO_SLUG =~ qgis/QGIS ]] && echo $TRAVIS_BRANCH | sed 's/master/latest/' || echo "latest" )
- DOCKER_BUILD_QGIS_IMAGE=false
- DOCKER_BUILD_DEPS_FILE=qgis3-build-deps.dockerfile

##########################################################
@@ -104,9 +103,8 @@ matrix:
if: repo = qgis/QGIS AND (tag IS PRESENT OR type = cron)
services: docker
env:
- TRAVIS_CONFIG=linux
- TRAVIS_CONFIG=docker_image
- DOCKER_TAG=$( echo $TRAVIS_BRANCH | sed 's/master/latest/' )
- DOCKER_BUILD_QGIS_IMAGE=true
- DOCKER_BUILD_DEPS_FILE=qgis3-build-deps.dockerfile

##########################################################
@@ -117,9 +115,8 @@ matrix:
if: repo = qgis/QGIS AND (tag IS PRESENT OR type = cron)
services: docker
env:
- TRAVIS_CONFIG=linux
- TRAVIS_CONFIG=docker_image
- DOCKER_TAG="$( echo $TRAVIS_BRANCH | sed 's/master/latest/' )_cosmic"
- DOCKER_BUILD_QGIS_IMAGE=true
- DOCKER_BUILD_DEPS_FILE=qgis3-build-deps-cosmic.dockerfile


0 comments on commit 1e11e11

Please sign in to comment.
You can’t perform that action at this time.