Merge pull request #8475 from elpaso/docker-testing-env

Added QGIS testing environment to dockers

.ci/travis/linux/blacklist.txt

@@ -31,3 +31,7 @@ PyQgsAuthManagerPKIPostgresTest
 PyQgsAuthManagerPasswordPostgresTest
 PyQgsAuthManagerOgrPostgresTest
 PyQgsDbManagerPostgis
+
+# Needs an OpenCL device, the library is not enough
+qgis_openclutilstest
+

.docker/qgis.dockerfile

@@ -8,6 +8,9 @@ ARG CACHE_DIR
 FROM  qgis/qgis3-build-deps:${DOCKER_TAG}
 MAINTAINER Denis Rouzaud <denis@opengis.ch>
 
+LABEL Description="Docker container with QGIS" Vendor="QGIS.org" Version="1.1"
+
+
 ENV CC=/usr/lib/ccache/clang
 ENV CXX=/usr/lib/ccache/clang++
 ENV QT_SELECT=5
@@ -25,7 +28,7 @@ RUN cmake \
   -GNinja \
   -DCMAKE_BUILD_TYPE=Release \
   -DCMAKE_INSTALL_PREFIX=/usr \
-  -DWITH_DESKTOP=OFF \
+  -DWITH_DESKTOP=ON \
   -DWITH_SERVER=ON \
   -DWITH_3D=ON \
   -DWITH_BINDINGS=ON \
@@ -42,4 +45,28 @@ RUN cmake \
  && ninja install \
  && rm -rf /usr/src/QGIS
 
+################################################################################
+# Python testing environment setup
+
+# Add QGIS test runner
+COPY .docker/qgis_resources/test_runner/qgis_* /usr/bin/
+
+# Make all scripts executable
+RUN chmod +x /usr/bin/qgis_*
+
+# Add supervisor service configuration script
+COPY .docker/qgis_resources/supervisor/supervisord.conf /etc/supervisor/
+COPY .docker/qgis_resources/supervisor/supervisor.xvfb.conf /etc/supervisor/supervisor.d/
+
+# Python paths are for
+# - kartoza images (compiled)
+# - deb installed
+# - built from git
+# needed to find PyQt wrapper provided by QGIS
+ENV PYTHONPATH=/usr/share/qgis/python/:/usr/lib/python3/dist-packages/qgis:/usr/share/qgis/python/qgis
+
+
 WORKDIR /
+
+# Run supervisor
+CMD ["/usr/bin/supervisord", "-c", "/etc/supervisor/supervisord.conf"]

.docker/qgis3-build-deps.dockerfile

@@ -1,7 +1,7 @@
 FROM      ubuntu:18.04
 MAINTAINER Denis Rouzaud <denis@opengis.ch>
 
-LABEL Description="Docker container with QGIS dependencies" Vendor="QGIS.org" Version="1.0"
+LABEL Description="Docker container with QGIS dependencies" Vendor="QGIS.org" Version="1.1"
 
 # && echo "deb http://ppa.launchpad.net/ubuntugis/ubuntugis-unstable/ubuntu xenial main" >> /etc/apt/sources.list \
 # && echo "deb-src http://ppa.launchpad.net/ubuntugis/ubuntugis-unstable/ubuntu xenial main" >> /etc/apt/sources.list \
@@ -11,7 +11,8 @@ LABEL Description="Docker container with QGIS dependencies" Vendor="QGIS.org" Ve
 RUN  apt-get update \
   && apt-get install -y software-properties-common \
   && apt-get update \
-  && apt-get install -y \
+  && DEBIAN_FRONTEND=noninteractive \
+  apt-get install -y \
     apt-transport-https \
     bison \
     ca-certificates \
@@ -77,6 +78,7 @@ RUN  apt-get update \
     python3-pyqt5.qsci \
     python3-pyqt5.qtsql \
     python3-pyqt5.qtsvg \
+    python3-pyqt5.qtwebkit \
     python3-sip \
     python3-sip-dev \
     python3-termcolor \
@@ -103,6 +105,11 @@ RUN  apt-get update \
     xfonts-base \
     xfonts-scalable \
     xvfb \
+    opencl-headers \
+    ocl-icd-libopencl1 \
+    ocl-icd-opencl-dev \
+    supervisor \
+    expect \
   && pip3 install \
     psycopg2 \
     numpy \
@@ -114,6 +121,12 @@ RUN  apt-get update \
     owslib \
     oauthlib \
     pyopenssl \
+    pep8 \
+    pexpect \
+    capturer \
+    sphinx \
+    requests \
+    six \
   && apt-get clean
 
 

.docker/qgis_resources/requirements.txt

@@ -0,0 +1,7 @@
+pep8
+pexpect
+capturer
+sphinx
+requests
+future
+six

.docker/qgis_resources/supervisor/supervisor.xvfb.conf

@@ -0,0 +1,4 @@
+; Supervisor config file for Xvfb
+
+[program:Xvfb]
+command=/usr/bin/Xvfb :99 -screen 0 1024x768x24 -ac +extension GLX +render -noreset -nolisten tcp

.docker/qgis_resources/supervisor/supervisord.conf

@@ -0,0 +1,13 @@
+; Supervisor config file.
+
+[supervisord]
+nodaemon=true
+logfile=/var/log/supervisor/supervisord.log
+logfile_maxbytes=50MB
+logfile_backups=10
+loglevel=info
+pidfile=/var/run/supervisord.pid
+childlogdir=/var/log
+
+[include]
+files = /etc/supervisor/supervisor.d/*.conf

.docker/qgis_resources/test_runner/qgis_setup.sh

@@ -0,0 +1,61 @@
+#!/bin/bash
+# Setup QGIS for the automated tests
+# This is normally called from Travis or rundockertests.sh
+# before running the tests for a particular plugin
+#
+# Note: on QGIS3 assumes the default profile for root user
+#
+# - create the folders
+# - install startup.py monkey patches
+# - disable tips
+# - enable the plugin
+
+PLUGIN_NAME=$1
+CONF_MASTER_FOLDER="/root/.local/share/QGIS/QGIS3/profiles/default/QGIS/"
+CONF_MASTER_FILE="${CONF_MASTER_FOLDER}/QGIS3.ini"
+
+QGIS_MASTER_FOLDER="/root/.local/share/QGIS/QGIS3/profiles/default"
+PLUGIN_MASTER_FOLDER="${QGIS_MASTER_FOLDER}/python/plugins"
+
+STARTUP_MASTER_FOLDER="/root/.local/share/QGIS/QGIS3/"
+
+# Creates the config file
+mkdir -p $CONF_MASTER_FOLDER
+if [ -e "$CONF_MASTER_FILE" ]; then
+    rm -f $CONF_MASTER_FILE
+fi
+touch $CONF_MASTER_FILE
+
+# Creates plugin folder
+mkdir -p $PLUGIN_MASTER_FOLDER
+mkdir -p $STARTUP_MASTER_FOLDER
+
+# Install the monkey patches to prevent modal stacktrace on python errors
+cp /usr/bin/qgis_startup.py ${STARTUP_MASTER_FOLDER}/startup.py
+
+# Disable tips
+printf "[qgis]\n" >> $CONF_MASTER_FILE
+SHOW_TIPS=$(qgis --help 2>&1 | head -2 | grep 'QGIS - ' | perl -npe 'chomp; s/QGIS - (\d+)\.(\d+).*/showTips\1\2=false/')
+printf "%s\n\n" "$SHOW_TIPS" >> $CONF_MASTER_FILE
+
+if [ -n "$PLUGIN_NAME" ]; then
+    # Enable plugin
+    printf '[PythonPlugins]\n' >> $CONF_MASTER_FILE
+    printf "%s=true\n\n" "$PLUGIN_NAME" >> $CONF_MASTER_FILE
+fi
+
+# Disable firstRunVersionFlag for master
+{
+    printf
+    "\n[migration]\n"
+    "fileVersion=2\n"
+    "firstRunVersionFlag=30500\n"
+    "settings=true\n\n"
+} >> $CONF_MASTER_FILE
+
+
+# Install the plugin
+if [ ! -d "${PLUGIN_MASTER_FOLDER}/${PLUGIN_NAME}" ]; then
+    ln -s "/tests_directory/${PLUGIN_NAME}" "${PLUGIN_MASTER_FOLDER}"
+    echo "Plugin master folder linked in ${PLUGIN_MASTER_FOLDER}/${PLUGIN_NAME}"
+fi

.docker/qgis_resources/test_runner/qgis_startup.py

@@ -0,0 +1,26 @@
+"""
+Disable QGIS modal error dialog.
+
+This script is meant to be run automatically when QGIS starts.
+Is should be renamed to `startup.py` and placed into
+~/.qgis3/python/startup.py
+
+"""
+from qgis import utils
+import traceback
+
+
+def _showException(type, value, tb, msg, messagebar=False):
+    print(msg)
+    logmessage = ''
+    for s in traceback.format_exception(type, value, tb):
+        logmessage += s.decode('utf-8', 'replace') if hasattr(s, 'decode') else s
+    print(logmessage)
+
+
+def _open_stack_dialog(type, value, tb, msg, pop_error=True):
+    print(msg)
+
+
+utils.showException = _showException
+utils.open_stack_dialog = _open_stack_dialog

.docker/qgis_resources/test_runner/qgis_testrunner.py

@@ -0,0 +1,170 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+"""
+***************************************************************************
+    Launches a unit test inside QGIS and exit the application.
+
+    Arguments:
+
+    accepts a single argument with the package name in python dotted notation,
+    the program tries first to load the module and launch the `run_all`
+    function of the module, if that fails it considers the last part of
+    the dotted path to be the function name and the previous part to be the
+    module.
+
+    Extra options for QGIS command line can be passed in the env var
+    QGIS_EXTRA_OPTIONS
+
+    Example run:
+
+    # Will load geoserverexplorer.test.catalogtests and run `run_all`
+    QGIS_EXTRA_OPTIONS='--optionspath .' \
+        GSHOSTNAME=localhost \
+        python qgis_testrunner.py geoserverexplorer.test.catalogtests
+
+
+    GSHOSTNAME=localhost \
+         python qgis_testrunner.py geoserverexplorer.test.catalogtests.run_my
+
+
+    ---------------------
+    Date                 : May 2016
+    Copyright            : (C) 2016 by Alessandro Pasotti
+    Email                : apasotti at boundlessgeo dot com
+***************************************************************************
+*                                                                         *
+*   This program is free software; you can redistribute it and/or modify  *
+*   it under the terms of the GNU General Public License as published by  *
+*   the Free Software Foundation; either version 2 of the License, or     *
+*   (at your option) any later version.                                   *
+*                                                                         *
+***************************************************************************
+"""
+
+__author__ = 'Alessandro Pasotti'
+__date__ = 'May 2016'
+
+import os
+import re
+import sys
+import traceback
+import signal
+import importlib
+from pexpect import run
+from pipes import quote
+
+from qgis.utils import iface
+
+
+def eprint(text):
+    sys.__stderr__.write(text + "\n")
+
+
+def __get_test_function(test_module_name):
+    """
+    Load the test module and return the test function
+    """
+    print("QGIS Test Runner - Trying to import %s" % test_module_name)
+    try:
+        test_module = importlib.import_module(test_module_name)
+        function_name = 'run_all'
+    except ImportError as e:
+        traceback.print_exc(file=sys.stdout)
+        # Strip latest name
+        pos = test_module_name.rfind('.')
+        if pos <= 0:
+            raise e
+        test_module_name, function_name = test_module_name[:pos], test_module_name[pos + 1:]
+        print("QGIS Test Runner - Trying to import %s" % test_module_name)
+        sys.stdout.flush()
+        try:
+            test_module = importlib.import_module(test_module_name)
+        except ImportError as e:
+            traceback.print_exc(file=sys.stdout)
+            raise e
+    return getattr(test_module, function_name, None)
+
+
+if iface is None:
+    """
+    Launch QGIS and passes itself as an init script
+    """
+    sys.path.append(os.getcwd())
+    test_module_name = sys.argv[-1]
+    if __get_test_function(test_module_name) is None:
+        print("QGIS Test Runner - [ERROR] cannot load test function from %s" % test_module_name)
+        sys.exit(1)
+    try:
+        me = __file__
+    except NameError:
+        me = sys.argv[0]
+    os.environ['QGIS_DEBUG'] = '1'
+    args = [
+        'qgis',
+        os.environ.get('QGIS_EXTRA_OPTIONS', ''),
+        '--nologo',
+        '--noversioncheck',
+        '--code',
+        me,
+        test_module_name, # Must be the last one!
+    ]
+    command_line = ' '.join(args)
+    print("QGIS Test Runner - launching QGIS as %s ..." % command_line)
+    out, returncode = run("sh -c " + quote(command_line), withexitstatus=1)
+    assert returncode is not None
+    print("QGIS Test Runner - QGIS exited.")
+    ok = out.find('(failures=') < 0 and \
+        len(re.findall(r'Ran \d+ tests in\s',
+                       out, re.MULTILINE)) > 0
+    print('=' * 60)
+    if not ok:
+        print(out)
+    else:
+        eprint(out)
+    if len(out) == 0:
+        print("QGIS Test Runner - [WARNING] subprocess returned no output")
+    print('=' * 60)
+
+    print("QGIS Test Runner - %s bytes returned and finished with exit code: %s" % (len(out), 0 if ok else 1))
+    sys.exit(0 if ok else 1)
+
+else: # We are inside QGIS!
+    # Start as soon as the initializationCompleted signal is fired
+    from qgis.core import QgsApplication, QgsProjectBadLayerHandler, QgsProject
+    from PyQt.QtCore import QDir
+    from qgis.utils import iface
+
+    class QgsProjectBadLayerDefaultHandler(QgsProjectBadLayerHandler):
+        def handleBadLayers(self, layers, dom):
+            pass
+
+    # Monkey patch QGIS Python console
+    from console.console_output import writeOut
+
+    def _write(self, m):
+        sys.__stdout__.write(m)
+    writeOut.write = _write
+
+    # Add current working dir to the python path
+    sys.path.append(QDir.current().path())
+
+    def __run_test():
+        """
+        Run the test specified as last argument in the command line.
+        """
+        # Disable modal handler for bad layers
+        QgsProject.instance().setBadLayerHandler(QgsProjectBadLayerDefaultHandler())
+        eprint("QGIS Test Runner Inside - starting the tests ...")
+        try:
+            test_module_name = QgsApplication.instance().arguments()[-1]
+            function_name = __get_test_function(test_module_name)
+            eprint("QGIS Test Runner Inside - executing function %s" % function_name)
+            function_name()
+        except Exception as e:
+            eprint("QGIS Test Runner Inside - [FAILED] Exception: %s" % e)
+            # Print tb
+            traceback.print_exc(file=sys.stdout)
+        app = QgsApplication.instance()
+        os.kill(app.applicationPid(), signal.SIGTERM)
+    iface.initializationCompleted.connect(__run_test)