Skip to content

Commit 1e11e11

Browse files
3nids3nids
authored
simplify CI scripts (#9387)
by separating the testing and docker image building scripts there was almost no redundany in the scripts, it is much clearer now
1 parent 47f1f4c commit 1e11e11

File tree

9 files changed

+164
-89
lines changed

9 files changed

+164
-89
lines changed

‎.ci/travis/docker_image/before_install.sh

+16
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
#!/usr/bin/env bash
2+
###########################################################################
3+
# before_install.sh
4+
# ---------------------
5+
# Date : March 2019
6+
# Copyright : (C) 2019 by Denis Rouzaud
7+
# Email : denis@opengis.ch
8+
###########################################################################
9+
# #
10+
# This program is free software; you can redistribute it and/or modify #
11+
# it under the terms of the GNU General Public License as published by #
12+
# the Free Software Foundation; either version 2 of the License, or #
13+
# (at your option) any later version. #
14+
# #
15+
###########################################################################
16+

‎.ci/travis/docker_image/before_script.sh

+36
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
#!/usr/bin/env bash
2+
###########################################################################
3+
# before_install.sh
4+
# ---------------------
5+
# Date : March 2019
6+
# Copyright : (C) 2019 by Denis Rouzaud
7+
# Email : denis@opengis.ch
8+
###########################################################################
9+
# #
10+
# This program is free software; you can redistribute it and/or modify #
11+
# it under the terms of the GNU General Public License as published by #
12+
# the Free Software Foundation; either version 2 of the License, or #
13+
# (at your option) any later version. #
14+
# #
15+
###########################################################################
16+
17+
18+
set -e
19+
20+
.ci/travis/scripts/echo_travis_var.sh
21+
22+
pushd .docker
23+
24+
25+
26+
echo "travis_fold:start:docker_build"
27+
echo "${bold}Docker build deps${endbold}"
28+
docker --version
29+
30+
docker pull "qgis/qgis3-build-deps:${DOCKER_TAG}" || true
31+
docker build --cache-from "qgis/qgis3-build-deps:${DOCKER_TAG}" -t "qgis/qgis3-build-deps:${DOCKER_TAG}" -f ${DOCKER_BUILD_DEPS_FILE} .
32+
33+
echo "travis_fold:end:docker_build"
34+
35+
36+
popd

‎.ci/travis/docker_image/install.sh

+17
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
#!/usr/bin/env bash
2+
###########################################################################
3+
# before_install.sh
4+
# ---------------------
5+
# Date : March 2019
6+
# Copyright : (C) 2019 by Denis Rouzaud
7+
# Email : denis@opengis.ch
8+
###########################################################################
9+
# #
10+
# This program is free software; you can redistribute it and/or modify #
11+
# it under the terms of the GNU General Public License as published by #
12+
# the Free Software Foundation; either version 2 of the License, or #
13+
# (at your option) any later version. #
14+
# #
15+
###########################################################################
16+
17+

‎.ci/travis/docker_image/script.sh

+59
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
#!/usr/bin/env bash
2+
###########################################################################
3+
# before_install.sh
4+
# ---------------------
5+
# Date : March 2019
6+
# Copyright : (C) 2019 by Denis Rouzaud
7+
# Email : denis@opengis.ch
8+
###########################################################################
9+
# #
10+
# This program is free software; you can redistribute it and/or modify #
11+
# it under the terms of the GNU General Public License as published by #
12+
# the Free Software Foundation; either version 2 of the License, or #
13+
# (at your option) any later version. #
14+
# #
15+
###########################################################################
16+
17+
18+
set -e
19+
20+
mkdir -p "CCACHE_DIR_IMAGE_BUILD"
21+
22+
# copy ccache dir within QGIS source so it can be accessed from docker
23+
cp -r ${CCACHE_DIR_IMAGE_BUILD} ${TRAVIS_BUILD_DIR}/.ccache_image_build
24+
# building docker images
25+
DIR=$(git rev-parse --show-toplevel)/.docker
26+
pushd "${DIR}"
27+
echo "${bold}Building QGIS Docker image '${DOCKER_TAG}'...${endbold}"
28+
docker build --build-arg DOCKER_TAG="${DOCKER_TAG}" \
29+
--cache-from "qgis/qgis:${DOCKER_TAG}" \
30+
-t "qgis/qgis:${DOCKER_TAG}" \
31+
-f qgis.dockerfile ..
32+
echo "${bold}Pushing image to docker hub...${endbold}"
33+
docker login -u="$DOCKER_USERNAME" -p="$DOCKER_PASSWORD"
34+
docker push "qgis/qgis:${DOCKER_TAG}"
35+
echo "Copy build cache from Docker container to Travis cache directory"
36+
rm -rf "${CCACHE_DIR_IMAGE_BUILD:?}/"*
37+
container_id=$(docker images -q "qgis/qgis:${DOCKER_TAG}")
38+
docker start ${container_id}
39+
docker cp ${container_id}:/usr/src/QGIS/.ccache_image_build ${CCACHE_DIR_IMAGE_BUILD}
40+
popd
41+
echo "Trigger build of PyQGIS Documentation"
42+
body='{
43+
"request": {
44+
"branch":"master",
45+
"message": "Trigger PyQGIS doc build after release of new Docker image as __DOCKER_TAG__",
46+
"config": {
47+
"merge_mode": "deep_merge",
48+
"matrix": {
49+
"include": {
50+
"env": ["QGIS_VERSION_BRANCH=__QGIS_VERSION_BRANCH__"]
51+
}
52+
}
53+
}
54+
}
55+
}'
56+
body=$(sed "s/__QGIS_VERSION_BRANCH__/${TRAVIS_BRANCH}/; s/__DOCKER_TAG__/${DOCKER_TAG}/" <<< $body)
57+
curl -s -X POST -H "Content-Type: application/json" -H "Accept: application/json" \
58+
-H "Travis-API-Version: 3" -H "Authorization: token $TRAVIS_TOKEN" -d "$body" \
59+
https://api.travis-ci.org/repo/qgis%2Fpyqgis/requests

‎.ci/travis/linux/before_install.sh

-1
Original file line numberDiff line numberDiff line change
@@ -14,4 +14,3 @@
1414
# #
1515
###########################################################################
1616

17-
#pip3 install termcolor

‎.ci/travis/linux/before_script.sh

+6-26
Original file line numberDiff line numberDiff line change
@@ -16,39 +16,19 @@
1616

1717
set -e
1818

19-
pushd .docker
20-
2119
DOCKER_DEPS_PUSH=$( [[ $TRAVIS_REPO_SLUG =~ qgis/QGIS ]] && [[ "${TRAVIS_EVENT_TYPE}" != "pull_request" ]] && echo "true" || echo "false" )
22-
DOCKER_DEPS_IMAGE_REBUILD=$( [[ $TRAVIS_COMMIT_MESSAGE =~ '[docker] update dependencies' ]] && echo "true" || echo "false" )
2320

24-
echo "travis_fold:start:travis_env"
25-
echo "${bold}Travis environment variables${endbold}"
26-
echo "TRAVIS_BRANCH: $TRAVIS_BRANCH"
27-
echo "TRAVIS_EVENT_TYPE: $TRAVIS_EVENT_TYPE"
28-
echo "DOCKER_TAG: $DOCKER_TAG"
29-
echo "TRAVIS_COMMIT_MESSAGE: $TRAVIS_COMMIT_MESSAGE"
30-
echo "DOCKER_DEPS_PUSH: $DOCKER_DEPS_PUSH"
31-
echo "DOCKER_DEPS_IMAGE_REBUILD: $DOCKER_DEPS_IMAGE_REBUILD"
32-
echo "DOCKER_BUILD_QGIS_IMAGE: $DOCKER_BUILD_QGIS_IMAGE"
33-
echo "DOCKER_BUILD_DEPS_FILE: $DOCKER_BUILD_DEPS_FILE"
34-
echo "TRAVIS_TIMESTAMP: $TRAVIS_TIMESTAMP"
35-
echo "travis_fold:end:travis_env"
21+
.ci/travis/scripts/echo_travis_var.sh
3622

23+
pushd .docker
3724

3825
echo "travis_fold:start:docker_build"
3926
echo "${bold}Docker build deps${endbold}"
4027
docker --version
41-
if [[ $DOCKER_BUILD_QGIS_IMAGE =~ false ]]; then
42-
docker-compose --version
43-
docker-compose -f ${TRAVIS_BUILD_DIR}/.docker/docker-compose.travis.yml config
44-
fi
45-
46-
if [[ $DOCKER_DEPS_IMAGE_REBUILD =~ true ]]; then
47-
docker build --no-cache -t "qgis/qgis3-build-deps:${DOCKER_TAG}" -f ${DOCKER_BUILD_DEPS_FILE} .
48-
else
49-
docker pull "qgis/qgis3-build-deps:${DOCKER_TAG}" || true
50-
docker build --cache-from "qgis/qgis3-build-deps:${DOCKER_TAG}" -t "qgis/qgis3-build-deps:${DOCKER_TAG}" -f ${DOCKER_BUILD_DEPS_FILE} .
51-
fi
28+
docker-compose --version
29+
docker-compose -f ${TRAVIS_BUILD_DIR}/.docker/docker-compose.travis.yml config
30+
docker pull "qgis/qgis3-build-deps:${DOCKER_TAG}" || true
31+
docker build --cache-from "qgis/qgis3-build-deps:${DOCKER_TAG}" -t "qgis/qgis3-build-deps:${DOCKER_TAG}" -f ${DOCKER_BUILD_DEPS_FILE} .
5232
echo "travis_fold:end:docker_build"
5333

5434
echo "travis_fold:start:docker_push"

‎.ci/travis/linux/script.sh

+16-57
Original file line numberDiff line numberDiff line change
@@ -16,62 +16,21 @@
1616

1717
set -e
1818

19-
mkdir -p "CCACHE_DIR_IMAGE_BUILD"
19+
# running QGIS tests
20+
docker-compose -f ${TRAVIS_BUILD_DIR}/.docker/docker-compose.travis.yml run --rm qgis-deps
2021

21-
if [[ ${DOCKER_BUILD_QGIS_IMAGE} =~ true ]]; then
22-
# copy ccache dir within QGIS source so it can be accessed from docker
23-
cp -r ${CCACHE_DIR_IMAGE_BUILD} ${TRAVIS_BUILD_DIR}/.ccache_image_build
24-
# building docker images
25-
DIR=$(git rev-parse --show-toplevel)/.docker
26-
pushd "${DIR}"
27-
echo "${bold}Building QGIS Docker image '${DOCKER_TAG}'...${endbold}"
28-
docker build --build-arg DOCKER_TAG="${DOCKER_TAG}" \
29-
--cache-from "qgis/qgis:${DOCKER_TAG}" \
30-
-t "qgis/qgis:${DOCKER_TAG}" \
31-
-f qgis.dockerfile ..
32-
echo "${bold}Pushing image to docker hub...${endbold}"
33-
docker login -u="$DOCKER_USERNAME" -p="$DOCKER_PASSWORD"
34-
docker push "qgis/qgis:${DOCKER_TAG}"
35-
echo "Copy build cache from Docker container to Travis cache directory"
36-
rm -rf "${CCACHE_DIR_IMAGE_BUILD:?}/"*
37-
container_id=$(docker images -q "qgis/qgis:${DOCKER_TAG}")
38-
docker cp ${container_id}:/usr/src/QGIS/.ccache_image_build ${CCACHE_DIR_IMAGE_BUILD}
39-
popd
40-
echo "Trigger build of PyQGIS Documentation"
41-
body='{
42-
"request": {
43-
"branch":"master",
44-
"message": "Trigger PyQGIS doc build after release of new Docker image as __DOCKER_TAG__",
45-
"config": {
46-
"merge_mode": "deep_merge",
47-
"matrix": {
48-
"include": {
49-
"env": ["QGIS_VERSION_BRANCH=__QGIS_VERSION_BRANCH__"]
50-
}
51-
}
52-
}
53-
}
54-
}'
55-
body=$(sed "s/__QGIS_VERSION_BRANCH__/${TRAVIS_BRANCH}/; s/__DOCKER_TAG__/${DOCKER_TAG}/" <<< $body)
56-
curl -s -X POST -H "Content-Type: application/json" -H "Accept: application/json" \
57-
-H "Travis-API-Version: 3" -H "Authorization: token $TRAVIS_TOKEN" -d "$body" \
58-
https://api.travis-ci.org/repo/qgis%2Fpyqgis/requests
59-
else
60-
# running QGIS tests
61-
docker-compose -f ${TRAVIS_BUILD_DIR}/.docker/docker-compose.travis.yml run --rm qgis-deps
22+
# running tests for the python test runner
23+
docker run -d --name qgis-testing-environment -v ${TRAVIS_BUILD_DIR}/tests/src/python:/tests_directory -e DISPLAY=:99 "qgis/qgis:${DOCKER_TAG}"
24+
sleep 10 # Wait for xvfb to finish starting
25+
# Temporary workaround until docker images are built
26+
docker cp ${TRAVIS_BUILD_DIR}/.docker/qgis_resources/test_runner/qgis_testrunner.sh qgis-testing-environment:/usr/bin/qgis_testrunner.sh
27+
# Run tests in the docker
28+
# Passing cases:
29+
TEST_SCRIPT_PATH=${TRAVIS_BUILD_DIR}/.ci/travis/linux/docker_test.sh
30+
[[ $(${TEST_SCRIPT_PATH} test_testrunner.run_passing) -eq '0' ]]
31+
[[ $(${TEST_SCRIPT_PATH} test_testrunner.run_skipped_and_passing) -eq '0' ]]
32+
# Failing cases:
33+
[[ $(${TEST_SCRIPT_PATH} test_testrunner) -eq '1' ]]
34+
[[ $(${TEST_SCRIPT_PATH} test_testrunner.run_all) -eq '1' ]]
35+
[[ $(${TEST_SCRIPT_PATH} test_testrunner.run_failing) -eq '1' ]]
6236

63-
# running tests for the python test runner
64-
docker run -d --name qgis-testing-environment -v ${TRAVIS_BUILD_DIR}/tests/src/python:/tests_directory -e DISPLAY=:99 "qgis/qgis:${DOCKER_TAG}"
65-
sleep 10 # Wait for xvfb to finish starting
66-
# Temporary workaround until docker images are built
67-
docker cp ${TRAVIS_BUILD_DIR}/.docker/qgis_resources/test_runner/qgis_testrunner.sh qgis-testing-environment:/usr/bin/qgis_testrunner.sh
68-
# Run tests in the docker
69-
# Passing cases:
70-
TEST_SCRIPT_PATH=${TRAVIS_BUILD_DIR}/.ci/travis/linux/docker_test.sh
71-
[[ $(${TEST_SCRIPT_PATH} test_testrunner.run_passing) -eq '0' ]]
72-
[[ $(${TEST_SCRIPT_PATH} test_testrunner.run_skipped_and_passing) -eq '0' ]]
73-
# Failing cases:
74-
[[ $(${TEST_SCRIPT_PATH} test_testrunner) -eq '1' ]]
75-
[[ $(${TEST_SCRIPT_PATH} test_testrunner.run_all) -eq '1' ]]
76-
[[ $(${TEST_SCRIPT_PATH} test_testrunner.run_failing) -eq '1' ]]
77-
fi

‎.ci/travis/scripts/echo_travis_var.sh

+12
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
#!/usr/bin/env bash
2+
3+
echo "travis_fold:start:travis_env"
4+
echo "${bold}Travis environment variables${endbold}"
5+
echo "TRAVIS_BRANCH: $TRAVIS_BRANCH"
6+
echo "TRAVIS_EVENT_TYPE: $TRAVIS_EVENT_TYPE"
7+
echo "DOCKER_TAG: $DOCKER_TAG"
8+
echo "TRAVIS_COMMIT_MESSAGE: $TRAVIS_COMMIT_MESSAGE"
9+
echo "DOCKER_DEPS_PUSH: $DOCKER_DEPS_PUSH"
10+
echo "DOCKER_BUILD_DEPS_FILE: $DOCKER_BUILD_DEPS_FILE"
11+
echo "TRAVIS_TIMESTAMP: $TRAVIS_TIMESTAMP"
12+
echo "travis_fold:end:travis_env"

‎.travis.yml

+2-5
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,6 @@ matrix:
4343
env:
4444
- TRAVIS_CONFIG=linux
4545
- DOCKER_TAG=$( [[ $TRAVIS_REPO_SLUG =~ qgis/QGIS ]] && echo $TRAVIS_BRANCH | sed 's/master/latest/' || echo "latest" )
46-
- DOCKER_BUILD_QGIS_IMAGE=false
4746
- DOCKER_BUILD_DEPS_FILE=qgis3-build-deps.dockerfile
4847

4948
##########################################################
@@ -104,9 +103,8 @@ matrix:
104103
if: repo = qgis/QGIS AND (tag IS PRESENT OR type = cron)
105104
services: docker
106105
env:
107-
- TRAVIS_CONFIG=linux
106+
- TRAVIS_CONFIG=docker_image
108107
- DOCKER_TAG=$( echo $TRAVIS_BRANCH | sed 's/master/latest/' )
109-
- DOCKER_BUILD_QGIS_IMAGE=true
110108
- DOCKER_BUILD_DEPS_FILE=qgis3-build-deps.dockerfile
111109

112110
##########################################################
@@ -117,9 +115,8 @@ matrix:
117115
if: repo = qgis/QGIS AND (tag IS PRESENT OR type = cron)
118116
services: docker
119117
env:
120-
- TRAVIS_CONFIG=linux
118+
- TRAVIS_CONFIG=docker_image
121119
- DOCKER_TAG="$( echo $TRAVIS_BRANCH | sed 's/master/latest/' )_cosmic"
122-
- DOCKER_BUILD_QGIS_IMAGE=true
123120
- DOCKER_BUILD_DEPS_FILE=qgis3-build-deps-cosmic.dockerfile
124121

125122

0 commit comments

Comments
 (0)