Merge pull request #4035 from GeotrekCE/improve_test_environment

Improve test environment

.env.dist

@@ -1,5 +1,5 @@
 ENV=dev
-SERVER_NAME=geotrek.localhost
+SERVER_NAME=geotrek.local
 POSTGRES_LOCAL_PORT=5432
 POSTGRES_HOST=postgres
 POSTGRES_USER=geotrek

.github/workflows/dependencies.yml

@@ -35,9 +35,9 @@ jobs:
 
       - name: Check dependency graph
         run: |
-          pip-compile -q
-          pip-compile -q dev-requirements.in
-          pip-compile -q docs/requirements.in
+          pip-compile -q --strip-extras
+          pip-compile -q --strip-extras dev-requirements.in
+          pip-compile -q --strip-extras docs/requirements.in
 
       - name: Verify dependency graph is ok
         uses: tj-actions/verify-changed-files@v17

.github/workflows/test.yml

@@ -113,14 +113,20 @@ jobs:
     - name: Install python dependencies
       run: |
         python3.8 -m venv ./venv
-        ./venv/bin/pip3 install --upgrade pip wheel setuptools
+        ./venv/bin/pip3 install --upgrade pip wheel setuptools -c requirements.txt -c dev-requirements.txt
         ./venv/bin/pip3 install -r requirements.txt -U
         ./venv/bin/pip3 install -r dev-requirements.txt -U
 
     - name: Create test required directories
       run: |
         mkdir -p ./var/log ./var/cache/sessions
 
+    - name: Prepare environment
+      run: |
+        mkdir -p $VAR_DIR/conf/extra_static $VAR_DIR/tmp
+        ./venv/bin/python3 ./manage.py collectstatic --clear --noinput --verbosity=0
+        ./venv/bin/python3 ./manage.py compilemessages
+
     - name: Check missing migrations
       run: |
         ENV=dev ./venv/bin/python3 ./manage.py makemigrations --check
@@ -129,20 +135,14 @@ jobs:
       run: |
         ENV=dev ./venv/bin/python3 ./manage.py migrate
 
-    - name: Prepare environment
-      run: |
-        mkdir -p $VAR_DIR
-        mkdir -p $VAR_DIR/conf/extra_static
-        ./venv/bin/python3 ./manage.py collectstatic --clear --noinput --verbosity=0
-        ./venv/bin/python3 ./manage.py compilemessages
-
     - name: Launch Celery
       run: |
         ./venv/bin/celery -A geotrek worker -c 1 &
 
     - name: Unit test
       run: |
-        ./venv/bin/coverage run ./manage.py test -v 3
+        ./venv/bin/coverage run --parallel-mode --concurrency=multiprocessing ./manage.py test --parallel -v 3
+        ./venv/bin/coverage combine
         ./venv/bin/coverage xml -o coverage.xml
 
     # codecov-action version 3.1.4 is the last version supported by bionic (3.1.5 requires node 20)

.gitignore

@@ -60,3 +60,5 @@ docs/_build/
 /debian/geotrek-admin.postrm.debhelper
 /cypress/node_modules/
 /dpkg/
+/coverage.lcov
+/htmlcov/

Makefile

@@ -32,30 +32,36 @@ serve:
 	$(docker_compose) up
 
 deps:
-	$(docker_compose) run --rm web bash -c "pip-compile -q && pip-compile -q dev-requirements.in && pip-compile -q docs/requirements.in"
+	$(docker_compose) run --rm web bash -c "pip-compile -q --strip-extras && pip-compile -q --strip-extras dev-requirements.in && pip-compile -q --strip-extras docs/requirements.in"
 
 flake8:
 	$(docker_compose) run --rm web flake8 geotrek
 
 messages:
 	$(docker_compose) run --rm web ./manage.py makemessages -a --no-location --no-obsolete
 
+compilemessages:
+	$(docker_compose) run --rm web ./manage.py compilemessages
+
 ###########################
 #        coverage         #
 ###########################
 verbose_level ?= 1
+report ?= report -m
 .PHONY: coverage
 coverage:
+	rm ./var/.coverage* || true
 	@$(PRINT_COLOR) "$(COLOR_SUCCESS) ### Start coverage ### $(COLOR_RESET)\n"
-	$(docker_compose) run -e ENV=tests web coverage run ./manage.py test $(test_name) -v $(verbose_level)
-	$(docker_compose) run -e ENV=tests_nds web coverage run -a ./manage.py test $(test_name) -v $(verbose_level)
-	$(docker_compose) run -e ENV=tests web coverage lcov
+	$(docker_compose) run -e ENV=tests web coverage run --parallel-mode --concurrency=multiprocessing ./manage.py test $(test_name) --noinput --parallel -v $(verbose_level)
+	$(docker_compose) run -e ENV=tests_nds web coverage run --parallel-mode --concurrency=multiprocessing ./manage.py test $(test_name) --noinput --parallel -v $(verbose_level)
+	$(docker_compose) run -e ENV=tests web bash -c "coverage combine && coverage $(report)"
+	rm ./var/.coverage*
 
 test:
-	$(docker_compose) run -e ENV=tests --rm web ./manage.py test
+	$(docker_compose) run -e ENV=tests --rm web ./manage.py test --noinput --parallel
 
 test_nds:
-	$(docker_compose) run -e ENV=tests_nds --rm web ./manage.py test
+	$(docker_compose) run -e ENV=tests_nds --rm web ./manage.py test --noinput --parallel
 
 test_nav:
 	casperjs test --baseurl=$(baseurl) geotrek/jstests/nav-*.js
@@ -69,7 +75,7 @@ node_modules:
 test_js: node_modules
 	./node_modules/.bin/mocha-phantomjs geotrek/jstests/index.html
 
-tests: test test_js test_nav
+tests: test test_nds test_js test_nav
 
 update:
 	$(docker_compose) run web update.sh

dev-requirements.in

@@ -2,6 +2,7 @@
 
 pip-tools
 lxml
+tblib
 flake8
 freezegun
 coverage

dev-requirements.txt

@@ -2,7 +2,7 @@
 # This file is autogenerated by pip-compile with Python 3.8
 # by the following command:
 #
-#    pip-compile dev-requirements.in
+#    pip-compile --strip-extras dev-requirements.in
 #
 asgiref==3.7.2
     # via
@@ -77,6 +77,8 @@ sqlparse==0.4.4
     #   -c requirements.txt
     #   django
     #   django-debug-toolbar
+tblib==3.0.0
+    # via -r dev-requirements.in
 tomli==2.0.1
     # via
     #   build

docker/scripts/entrypoint.sh

@@ -5,7 +5,6 @@ set -e
 mkdir -p /opt/geotrek-admin/var/static \
          /opt/geotrek-admin/var/conf/extra_static \
          /opt/geotrek-admin/var/media/upload \
-         /opt/geotrek-admin/var/data \
          /opt/geotrek-admin/var/cache/sessions \
          /opt/geotrek-admin/var/cache/api_v2 \
          /opt/geotrek-admin/var/cache/fat \

docs/contribute/development.rst

@@ -12,16 +12,17 @@ Quickstart
 
 ::
 
-    cp .env-dev.dist .env
+    cp .env.dist .env
     # Edit .env if need be
-    cp docker-compose-dev.yml docker-compose.yml
-    docker-compose build
-    docker-compose run --rm web update.sh
-    docker-compose run --rm web load_data.sh
-    docker-compose run --rm web ./manage.py createsuperuser
-    docker-compose up -d
+    docker compose build
+    docker compose run --rm web update.sh
+    docker compose run --rm web load_data.sh
+    docker compose run --rm web ./manage.py createsuperuser
+    docker compose up
 
-Go to ``http://geotrek.localhost:8000``.
+Edit `/etc/hosts` file to add `geotrek.local` alias to `127.0.0.1`
+
+Go to ``http://geotrek.local:8000`` in your browser
 
 PDF generation might not work unless you use this domain and is correctly set to SERVER_NAME variable in your .env file.
 
@@ -46,23 +47,23 @@ Consider using pip-tools to manage dependencies.
 
 ::
 
-   docker-compose run --rm web pip-compile
-   docker-compose run --rm web pip-compile dev-requirements.in
+   make deps
 
 or
 
 ::
 
-   make deps
+    docker compose run --rm web pip-compile
+    docker compose run --rm web pip-compile dev-requirements.in
 
 
 Model modification
 ==================
 
 ::
 
-   docker-compose run --rm web ./manage.py makemigrations <appName>
-   docker-compose run --rm web ./manage.py migrate
+   docker compose run --rm web ./manage.py makemigrations <appName>
+   docker compose run --rm web ./manage.py migrate
 
 .. note::
 
@@ -137,32 +138,57 @@ run:
 
 ::
 
-   docker-compose run --rm web flake8 geotrek
+   make flake8
 
 
 or
 
 ::
 
-   make flake8
+   docker compose run --rm web flake8 geotrek
 
 
 Run tests
 =========
 
 **Django tests :**
 
-``ENV`` variable must be set to run tests:
+run:
+
+::
+
+   make coverage
+
+
+To run all test suites and report global coverage
+
+
+To run a specific test suite, run:
 
 ::
 
-   docker-compose run --rm -e ENV=tests web ./manage.py test
+    make tests
+
+or
+
+::
+
+   docker compose run --rm -e ENV=tests web ./manage.py test
+
+
+And
 
-Test without dynamic segmentation:
 
 ::
 
-   docker-compose run --rm -e ENV=tests_nds web ./manage.py test
+    make tests_nds
+
+or
+
+::
+
+   docker compose run --rm -e ENV=tests_nds web ./manage.py test
+
 
 
 **Cypress tests :**
@@ -171,8 +197,8 @@ Create an empty project with Docker :
 
 ::
 
-    docker-compose down
-    docker-compose up -d
+    docker compose down
+    docker compose up -d
 
 
 Install elements for the cypress tests
@@ -205,7 +231,7 @@ Pictures of the problem and videos are generated in ``cypress/videos`` and ``cyp
 Setup to use screamshotter-related features locally
 ===================================================
 
-Use the domain defined in ``SERVER_NAME`` in your ``.env`` to reach your local Geotrek-admin web instance. By default the address is ``http://geotrek.localhost:8000``.
+Use the domain defined in ``SERVER_NAME`` in your ``.env`` to reach your local Geotrek-admin web instance. By default the address is ``http://geotrek.local:8000``.
 
 
 Database reset
@@ -215,7 +241,7 @@ Data only:
 
 ::
 
-   docker-compose run --rm web ./manage.py flush
+   docker compose run --rm web ./manage.py flush
 
 Restore existing Database
 =========================
@@ -224,7 +250,7 @@ Assuming a dump of your database is located in your project directory:
 
 ::
 
-   docker-compose run --rm web pg_restore --clean --no-owner --no-acl -h $POSTGRES_HOST -U $POSTGRES_USER -d $POSTGRES_DB /opt/geotrek-admin/<path_to_backup>.dump
+   docker compose run --rm web pg_restore --clean --no-owner --no-acl -h $POSTGRES_HOST -U $POSTGRES_USER -d $POSTGRES_DB /opt/geotrek-admin/<path_to_backup>.dump
 
 Restore your ``./var/conf/`` project files, and data files into ``./var/media``.
 

docs/contribute/documentation.rst

@@ -13,7 +13,7 @@ To compile and test documentation on local environment, run:
 
 .. code-block:: bash
 
-    docker-compose up -d sphinx
+    docker compose up -d sphinx
 
 
 Access to documentation built in html : http://0.0.0.0:8800
@@ -26,10 +26,10 @@ Translate documentation
 
 .. code-block:: python
 
-    docker-compose run --rm sphinx make gettext
+    docker compose run --rm sphinx make gettext
 
 - Generate ``.po`` files
 
 .. code-block:: python
 
-    docker-compose run --rm sphinx sphinx-intl update -p _build/locale -l fr
+    docker compose run --rm sphinx sphinx-intl update -p _build/locale -l fr

docs/install/installation.rst

@@ -181,18 +181,18 @@ and to install it on other distributions than Ubuntu Linux 18.04.
 5. Edit ``docker-compose.yml`` to feed your needs if necessary
 6. Copy ``.env.dist`` to ``.env`` and edit to feed your needs if necessary
 7. Create user and database, enable PostGIS extension
-8. Run ``docker-compose run --rm web update.sh``
-9. Run ``docker-compose up``
+8. Run ``docker compose run --rm web update.sh``
+9. Run ``docker compose up``
 10. Install NGINX (or equivalent) and add a configuration file (taking inspiration from `nginx.conf.in`)
 
 Management commands
 -------------------
 
-Replace ``sudo geotrek …`` commands by ``cd <install directory>; docker-compose run --rm web ./manage.py …``
+Replace ``sudo geotrek …`` commands by ``cd <install directory>; docker compose run --rm web ./manage.py …``
 
 To load minimal data and create an application superuser, run:
 
 ::
 
-   docker-compose run --rm web load_data.sh
-   docker-compose run --rm web ./manage.py createsuperuser
+   docker compose run --rm web load_data.sh
+   docker compose run --rm web ./manage.py createsuperuser

docs/locales/fr/LC_MESSAGES/contribute.po

@@ -195,7 +195,7 @@ msgid "Quickstart"
 msgstr ""
 
 #: ../../contribute/development.rst:21
-msgid "Go to http://geotrek.localhost:8000"
+msgid "Go to http://geotrek.local:8000"
 msgstr ""
 
 #: ../../contribute/development.rst:23
@@ -473,7 +473,7 @@ msgid ""
 msgstr ""
 
 #: ../../contribute/development.rst:208
-msgid "For instance with SERVER_NAME=geotrek.localhost (default value)"
+msgid "For instance with SERVER_NAME=geotrek.local (default value)"
 msgstr ""
 
 #: ../../contribute/development.rst:217
@@ -484,7 +484,7 @@ msgstr ""
 msgid ""
 "Use the domain defined in SERVER_NAME in your .env to reach your local "
 "geotrek admin web instance. By default the address is "
-"`http://geotrek.localhost:8000`."
+"`http://geotrek.local:8000`."
 msgstr ""
 
 #: ../../contribute/development.rst:223