Merge pull request #5256 from qgis/docker2

[travis] Use docker to run test

.ci/travis/code_layout/script.sh

@@ -16,9 +16,9 @@ set -e
 
 pushd build
 
+export CTEST_BUILD_DIR=${TRAVIS_BUILD_DIR}
 export CTEST_BUILD_COMMAND="/usr/bin/make -j3 -i -k"
-python3 ${TRAVIS_BUILD_DIR}/.ci/travis/scripts/ctest2travis.py xvfb-run ctest -V --output-on-failure -S ${TRAVIS_BUILD_DIR}/.ci/travis/travis.ctest
 
-ctest -V -R PyQgsDocCoverage
+python3 ${TRAVIS_BUILD_DIR}/.ci/travis/scripts/ctest2travis.py xvfb-run ctest -V --output-on-failure -S ${TRAVIS_BUILD_DIR}/.ci/travis/travis.ctest
 
 popd

.ci/travis/linux/after_script.sh

@@ -12,7 +12,3 @@
 #   (at your option) any later version.                                   #
 #                                                                         #
 ###########################################################################
-
-ccache -s
-
-[ -r /tmp/ctest-important.log ] && cat /tmp/ctest-important.log

.ci/travis/linux/before_install.sh

@@ -13,33 +13,5 @@
 #                                                                         #
 ###########################################################################
 
-##################################################
-#
-# Get precompiled dependencies
-#
-##################################################
 
-pushd ${HOME}
-
-# fetching data from github should be just as fast as S3
-curl -s -S -L https://github.com/opengisch/osgeo4travis/archive/qt55bin.tar.gz | tar --strip-components=1 -xz -C /home/travis &
-SETUP_OSGEO4W_PID=$!
-
-mkdir /home/travis/osgeo4travis
-
-# other dependencies live in a cached folder
-pushd depcache
-# Download newer version of cmake than in the repository
-[[ -f cmake-3.5.0-Linux-x86_64.tar.gz ]] || curl -s -S -O https://cmake.org/files/v3.5/cmake-3.5.0-Linux-x86_64.tar.gz
-tar --strip-components=1 -zx -f cmake-3.5.0-Linux-x86_64.tar.gz -C /home/travis/osgeo4travis
-
-# Download OTB package for Processing tests
-[[ -f OTB-5.6.0-Linux64.run ]] || curl -s -S -O https://www.orfeo-toolbox.org/packages/archives/OTB/OTB-5.6.0-Linux64.run
-sh ./OTB-5.6.0-Linux64.run
-
-wait $SETUP_OSGEO4W_PID
-
-popd
-popd
-
-pip install psycopg2 numpy nose2 pyyaml mock future termcolor
+#pip3 install termcolor

.ci/travis/linux/before_script.sh

@@ -13,7 +13,22 @@
 #                                                                         #
 ###########################################################################
 
-printf "[qgis_test]\nhost=localhost\ndbname=qgis_test\nuser=postgres" > ~/.pg_service.conf
+set -e
 
-export PGUSER=postgres
-$TRAVIS_BUILD_DIR/tests/testdata/provider/testdata_pg.sh
+pushd .docker
+
+docker --version
+docker-compose --version
+docker-compose -f $DOCKER_COMPOSE config
+#docker pull ubuntu:16.04
+docker pull "qgis/qgis3-build-deps:${DOCKER_TAG}" || true
+docker build --cache-from "qgis/qgis3-build-deps:${DOCKER_TAG}" -t "qgis/qgis3-build-deps:${DOCKER_TAG}" .
+# image should be pushed even if QGIS build fails
+# but push is achieved only on branches (not for PRs)
+if [[ $DOCKER_PUSH =~ true ]]; then
+  docker login -u="$DOCKER_USERNAME" -p="$DOCKER_PASSWORD"
+  #docker tag "qgis/qgis3-build-deps:${DOCKER_TAG}" "qgis/qgis3-build-deps:latest"
+  docker push "qgis/qgis3-build-deps:${DOCKER_TAG}"
+fi
+
+popd

.ci/travis/linux/docker-build-test.sh

@@ -0,0 +1,76 @@
+#!/usr/bin/env bash
+
+set -e
+
+##############
+# Setup ccache
+##############
+export CCACHE_TEMPDIR=/tmp
+ccache -M 500M
+ccache -z
+
+############################
+# Setup the (c)test environment
+############################
+export LD_PRELOAD=/lib/x86_64-linux-gnu/libSegFault.so
+export CTEST_BUILD_COMMAND="/usr/bin/ninja"
+export CTEST_PARALLEL_LEVEL=1
+
+###########
+# Configure
+###########
+pushd /root/QGIS
+mkdir -p build
+
+pushd build
+
+cmake \
+ -GNinja \
+ -DWITH_STAGED_PLUGINS=ON \
+ -DWITH_GRASS=OFF \
+ -DSUPPRESS_QT_WARNINGS=ON \
+ -DENABLE_MODELTEST=ON \
+ -DENABLE_PGTEST=ON \
+ -DWITH_QSPATIALITE=ON \
+ -DWITH_QWTPOLAR=OFF \
+ -DWITH_APIDOC=OFF \
+ -DWITH_ASTYLE=OFF \
+ -DWITH_DESKTOP=ON \
+ -DWITH_BINDINGS=ON \
+ -DDISABLE_DEPRECATED=ON \
+ -DCXX_EXTRA_FLAGS=${CLANG_WARNINGS} ..
+
+#######
+# Build
+#######
+echo "travis_fold:start:ninja-build.1"
+ninja
+echo "travis_fold:end:ninja-build.1"
+
+############################
+# Restore postgres test data
+############################
+printf "[qgis_test]\nhost=postgres\nport=5432\ndbname=qgis_test\nuser=docker\npassword=docker" > ~/.pg_service.conf
+export PGUSER=docker
+export PGHOST=postgres
+export PGPASSWORD=docker
+export PGDATABASE=qgis_test
+
+pushd /root/QGIS
+/root/QGIS/tests/testdata/provider/testdata_pg.sh
+popd # /root/QGIS
+
+###########
+# Run tests
+###########
+python3 /root/QGIS/.ci/travis/scripts/ctest2travis.py xvfb-run ctest -V -E "$(cat /root/QGIS/.ci/travis/linux/blacklist.txt | sed -r '/^(#.*?)?$/d' | paste -sd '|' -)" -S /root/QGIS/.ci/travis/travis.ctest --output-on-failure
+
+########################
+# Show ccache statistics
+########################
+ccache -s
+
+popd # build
+popd # /root/QGIS
+
+[ -r /tmp/ctest-important.log ] && cat /tmp/ctest-important.log || true

.ci/travis/linux/install.sh

@@ -13,67 +13,5 @@
 #                                                                         #
 ###########################################################################
 
-mkdir build
-cd build
-
-ln -s ${HOME}/osgeo4travis/bin/ccache ${HOME}/osgeo4travis/bin/clang++-${LLVM_VERSION}
-ln -s ${HOME}/osgeo4travis/bin/ccache ${HOME}/osgeo4travis/bin/clang-${LLVM_VERSION}
-
-ccache -s
-ccache -z
-
-export CXX="clang++-${LLVM_VERSION}"
-export CC="clang-${LLVM_VERSION}"
-#export CXX="g++-6"
-#export CC="gcc-6"
-export PATH=${HOME}/osgeo4travis/bin:${PATH}
-export PYTHONPATH=${HOME}/osgeo4travis/lib/python3.3/site-packages/
-
-cmake --version
-${CC} --version
-${CXX} --version
-
-# CLANG_WARNINGS="-Wimplicit-fallthrough"
-CLANG_WARNINGS=""
-
-# Include this line for debug reasons
-#      -DCMAKE_BUILD_TYPE=RelWithDebInfo \
-#
-CMAKE_FLAGS="
-      -DCMAKE_PREFIX_PATH=/home/travis/osgeo4travis
-      -DWITH_STAGED_PLUGINS=ON
-      -DWITH_GRASS=ON
-      -DWITH_GRASS7=ON
-      -DGRASS_PREFIX7=/home/travis/osgeo4travis/grass-7.0.4
-      -DSUPPRESS_QT_WARNINGS=ON
-      -DENABLE_MODELTEST=ON
-      -DENABLE_PGTEST=ON
-      -DWITH_QSPATIALITE=ON
-      -DWITH_QWTPOLAR=OFF
-      -DWITH_APIDOC=OFF
-      -DWITH_ASTYLE=OFF
-      -DDISABLE_DEPRECATED=ON
-      -DCXX_EXTRA_FLAGS=${CLANG_WARNINGS}
-      -DPYTHON_TEST_WRAPPER="timeout -sSIGSEGV 55s"
-      "
-
-# The following options trigger a minimalized build to
-# reduce the travis build time so we don't time out and
-# have a chance of slowly filling the ccache.
-if [ "$CACHE_WARMING" = true ] ; then
-  CMAKE_FLAGS="
-    ${CMAKE_FLAGS}
-    -DWITH_DESKTOP=OFF
-    -DWITH_SERVER=OFF
-    -DWITH_CUSTOM_WIDGETS=OFF
-  "
-else
-  CMAKE_FLAGS="
-    ${CMAKE_FLAGS}
-    -DWITH_DESKTOP=ON
-    -DWITH_SERVER=ON
-    -DWITH_CUSTOM_WIDGETS=ON
-  "
-fi
-
-cmake $CMAKE_FLAGS ..
+shopt -s expand_aliases
+alias python=python3

.ci/travis/linux/script.sh

@@ -15,31 +15,8 @@
 
 set -e
 
-export PYTHONPATH=${HOME}/osgeo4travis/lib/python3.3/site-packages/
-export PATH=${HOME}/osgeo4travis/bin:${HOME}/osgeo4travis/sbin:${HOME}/OTB-5.6.0-Linux64/bin:${PATH}
-export LD_LIBRARY_PATH=${HOME}/osgeo4travis/lib
-export CTEST_PARALLEL_LEVEL=1
-export CCACHE_TEMPDIR=/tmp
-ccache -M 500M
-ccache -z
+DIR=$(git rev-parse --show-toplevel)/.docker
 
-DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+mkdir -p $CCACHE_DIR
 
-# Set OTB application path (installed in before_install.sh script)
-export OTB_APPLICATION_PATH=${HOME}/OTB-5.6.0-Linux64/lib/otb/applications
-export LD_PRELOAD=/lib/x86_64-linux-gnu/libSegFault.so
-
-export CTEST_BUILD_COMMAND="/usr/bin/make -j3 -i"
-
-# This works around an issue where travis would timeout because
-# when make is run inside ctest no output is generated. At the current time
-# nobody know why, but at least this workaround gets travis results
-# back. Better approaches VERY welcome.
-pushd build
-echo "travis_fold:start:qgis_build"
-$CTEST_BUILD_COMMAND
-echo "travis_fold:end:qgis_build"
-popd
-
-python ${TRAVIS_BUILD_DIR}/.ci/travis/scripts/ctest2travis.py \
-  xvfb-run ctest -V -E "$(cat ${DIR}/blacklist.txt | sed -r '/^(#.*?)?$/d' | paste -sd '|' -)" -S ${DIR}/../travis.ctest --output-on-failure
+docker-compose -f $DOCKER_COMPOSE run --rm qgis-deps

.ci/travis/macos/script.sh

@@ -30,6 +30,7 @@ ccache -z
 TIMEOUT=$(expr 40 \* 60 - `date +%s` + `cat /tmp/travis_timestamp`)
 
 export CTEST_BUILD_COMMAND=/usr/local/bin/ninja
+export CTEST_BUILD_DIR=${TRAVIS_BUILD_DIR}
 
 gtimeout ${TIMEOUT}s ctest -V -E "$(cat ${DIR}/blacklist.txt | gsed -r '/^(#.*?)?$/d' | gpaste -sd '|' -)" -S ${DIR}/../travis.ctest --output-on-failure
 

.ci/travis/travis.ctest

@@ -1,5 +1,5 @@
-SET (CTEST_SOURCE_DIRECTORY $ENV{TRAVIS_BUILD_DIR})
-SET (CTEST_BINARY_DIRECTORY "$ENV{TRAVIS_BUILD_DIR}/build")
+SET (CTEST_SOURCE_DIRECTORY $ENV{CTEST_BUILD_DIR})
+SET (CTEST_BINARY_DIRECTORY $ENV{CTEST_BUILD_DIR}/build)
 #SET (CTEST_SOURCE_DIRECTORY /usr/src/QGIS)
 #SET (CTEST_BINARY_DIRECTORY /usr/src/qgis-build)
 

.docker/Dockerfile

@@ -0,0 +1,101 @@
+FROM      ubuntu:16.04
+MAINTAINER Denis Rouzaud <denis.rouzaud@gmail.com>
+
+LABEL Description="Docker container with QGIS dependencies" Vendor="QGIS.org" Version="1.0"
+
+RUN  apt-get update \
+  && apt-get install -y software-properties-common \
+  && add-apt-repository ppa:ubuntugis/ubuntugis-unstable \
+  && apt-get update \
+  && apt-get install -y \
+    bison \
+    ca-certificates \
+    ccache \
+    clang \
+    cmake \
+    curl \
+    dh-python \
+    flex \
+    gdal-bin \
+    git \
+    graphviz \
+    grass-dev \
+    libexpat1-dev \
+    libfcgi-dev \
+    libgdal-dev \
+    libgeos-dev \
+    libgsl-dev \
+    libpq-dev \
+    libproj-dev \
+    libqca-qt5-2-dev \
+    libqca-qt5-2-plugins \
+    libqt5opengl5-dev \
+    libqt5scintilla2-dev \
+    libqt5sql5-sqlite \
+    libqt5svg5-dev \
+    libqt5webkit5-dev \
+    libqt5xmlpatterns5-dev \
+    libqwt-qt5-dev \
+    libspatialindex-dev \
+    libspatialite-dev \
+    libsqlite3-dev \
+    libsqlite3-mod-spatialite \
+    libzip-dev \
+    lighttpd \
+    locales \
+    ninja-build \
+    pkg-config \
+    poppler-utils \
+    postgresql-client \
+    pyqt5-dev \
+    pyqt5-dev-tools \
+    pyqt5.qsci-dev \
+    python3-all-dev \
+    python3-dev \
+    python3-future \
+    python3-gdal \
+    python3-mock \
+    python3-nose2 \
+    python3-pip \
+    python3-psycopg2 \
+    python3-pyqt5 \
+    python3-pyqt5.qsci \
+    python3-pyqt5.qtsql \
+    python3-pyqt5.qtsvg \
+    python3-sip \
+    python3-sip-dev \
+    python3-termcolor \
+    python3-yaml \
+    qt5keychain-dev \
+    qtbase5-dev \
+    qtpositioning5-dev \
+    qtscript5-dev \
+    qttools5-dev \
+    qttools5-dev-tools \
+    spawn-fcgi \
+    txt2tags \
+    xauth \
+    xfonts-100dpi \
+    xfonts-75dpi \
+    xfonts-base \
+    xfonts-scalable \
+    xvfb \
+  && pip3 install \
+    psycopg2 \
+    numpy \
+    nose2 \
+    pyyaml \
+    mock \
+    future \
+    termcolor \
+  && apt-get autoremove -y python3-pip python2.7 \
+  && apt-get clean
+
+RUN echo "alias python=python3" >> ~/.bash_aliases
+
+ENV CC=/usr/lib/ccache/clang
+ENV CXX=/usr/lib/ccache/clang++
+ENV QT_SELECT=5
+ENV LANG=C.UTF-8
+
+CMD /root/QGIS/.ci/travis/linux/docker-build-test.sh

.docker/Dockerfile-build-test

@@ -0,0 +1,10 @@
+
+ARG DOCKER_TAG=latest
+
+FROM      qgis/qgis3-build-deps:$DOCKER_TAG
+MAINTAINER Denis Rouzaud <denis.rouzaud@gmail.com>
+
+RUN ls /root/.ccache
+
+RUN echo "DDDD"
+RUN /root/QGIS/.ci/travis/linux/docker-build-test.sh