Merge pull request #1054 from nejch/chore/cleanup-test-env

chore: improve test environment for upcoming features

.travis.yml

@@ -39,6 +39,22 @@ jobs:
       script:
         - pip3 install tox
         - tox -e py_func_v4
+    - stage: test
+      name: cli_func_nightly
+      dist: bionic
+      python: 3.8
+      env: GITLAB_TAG=nightly
+      script:
+        - pip3 install tox
+        - tox -e cli_func_v4
+    - stage: test
+      name: py_func_nightly
+      dist: bionic
+      python: 3.8
+      env: GITLAB_TAG=nightly
+      script:
+        - pip3 install tox
+        - tox -e py_func_v4
     - stage: test
       name: docs
       dist: bionic
@@ -67,3 +83,5 @@ jobs:
       script:
         - pip3 install tox
         - tox -e py38
+  allow_failures:
+    - env: GITLAB_TAG=nightly

README.rst

@@ -129,11 +129,11 @@ You need to install ``tox`` to run unit tests and documentation builds locally:
 
 .. code-block:: bash
 
-   # run the unit tests for python 2/3, and the pep8 tests:
+   # run the unit tests for all supported python3 versions, and the pep8 tests:
    tox
 
    # run tests in one environment only:
-   tox -epy35
+   tox -epy36
 
    # build the documentation, the result will be generated in
    # build/sphinx/html/
@@ -156,6 +156,26 @@ To run these tests:
    # run the python API tests:
    ./tools/py_functional_tests.sh
 
+By default, the tests run against the ``gitlab/gitlab-ce:latest`` image. You can
+override both the image and tag with the ``-i`` and ``-t`` options, or by providing
+either the ``GITLAB_IMAGE`` or ``GITLAB_TAG`` environment variables.
+
+This way you can run tests against different versions, such as ``nightly`` for
+features in an upcoming release, or an older release (e.g. ``12.8.0-ce.0``).
+The tag must match an exact tag on Docker Hub:
+
+.. code-block:: bash
+
+   # run tests against `nightly` or specific tag
+   ./tools/py_functional_tests.sh -t nightly
+   ./tools/py_functional_tests.sh -t 12.8.0-ce.0
+
+   # run tests against the latest gitlab EE image
+   ./tools/py_functional_tests.sh -i gitlab/gitlab-ee
+
+   # override tags with environment variables
+   GITLAB_TAG=nightly ./tools/py_functional_tests.sh
+
 You can also build a test environment using the following command:
 
 .. code-block:: bash

tools/build_test_env.sh

@@ -27,29 +27,23 @@ try() { "$@" || fatal "'$@' failed"; }
 
 REUSE_CONTAINER=
 NOVENV=
-PY_VER=3
 API_VER=4
-GITLAB_IMAGE="gitlab/gitlab-ce"
-GITLAB_TAG="latest"
-while getopts :knp:a: opt "$@"; do
+GITLAB_IMAGE="${GITLAB_IMAGE:-gitlab/gitlab-ce}"
+GITLAB_TAG="${GITLAB_TAG:-latest}"
+VENV_CMD="python3 -m venv"
+while getopts :knp:a:i:t: opt "$@"; do
     case $opt in
         k) REUSE_CONTAINER=1;;
         n) NOVENV=1;;
-        p) PY_VER=$OPTARG;;
         a) API_VER=$OPTARG;;
+        i) GITLAB_IMAGE=$OPTARG;;
         t) GITLAB_TAG=$OPTARG;;
         :) fatal "Option -${OPTARG} requires a value";;
         '?') fatal "Unknown option: -${OPTARG}";;
         *) fatal "Internal error: opt=${opt}";;
     esac
 done
 
-case $PY_VER in
-    2) VENV_CMD=virtualenv;;
-    3) VENV_CMD="python3 -m venv";;
-    *) fatal "Wrong python version (2 or 3)";;
-esac
-
 case $API_VER in
     4) ;;
     *) fatal "Wrong API version (4 only)";;

tox.ini

@@ -4,6 +4,7 @@ skipsdist = True
 envlist = py38,py37,py36,pep8,black
 
 [testenv]
+passenv = GITLAB_IMAGE GITLAB_TAG
 setenv = VIRTUAL_ENV={envdir}
 whitelist_externals = true
 usedevelop = True
@@ -44,7 +45,7 @@ commands =
    coverage html --omit=*tests*
 
 [testenv:cli_func_v4]
-commands = {toxinidir}/tools/functional_tests.sh -a 4 -p 2
+commands = {toxinidir}/tools/functional_tests.sh -a 4
 
 [testenv:py_func_v4]
-commands = {toxinidir}/tools/py_functional_tests.sh -a 4 -p 2
+commands = {toxinidir}/tools/py_functional_tests.sh -a 4