Merge pull request #560 from EOxServer/ci-speedup

Trying to help bring down CI pipeline execution time

.github/workflows/ci.yml

@@ -16,13 +16,19 @@ jobs:
       - uses: actions/checkout@v3
       - name: Build the eoxserver docker image
         run: |
-          docker build -t eoxserver .
+          docker pull eoxa/eoxserver:latest || true
+          docker build --cache-from eoxa/eoxserver:latest -t eoxserver .
           docker save eoxserver | gzip > eoxserver.tar.gz
-      - uses: actions/upload-artifact@v3
+      # - uses: actions/upload-artifact@v3
+      #   with:
+      #     name: eoxserver
+      #     path: eoxserver.tar.gz
+      #     retention-days: 2
+      - name: Save image to cache
+        uses: actions/cache/save@v3
         with:
-          name: eoxserver
           path: eoxserver.tar.gz
-          retention-days: 2
+          key: eoxserver.tar.gz-${{ github.run_id }}-${{ github.run_number }}
       - name: Slack Notify
         uses: 8398a7/action-slack@v3.8.0
         with:
@@ -37,45 +43,50 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        include:
-        - db: postgis
-          python_bin: python3
-        - db: spatialite
-          python_bin: python3
+        command:
+          - "-m eoxserver.services.ows.wps.test_data_types"
+          - "-m eoxserver.services.ows.wps.test_allowed_values"
+          - "manage.py test --pythonpath=./eoxserver/ eoxserver.core -v2"
+          - "manage.py test --pythonpath=./eoxserver/ eoxserver.backends -v2"
+          - "manage.py test --pythonpath=./eoxserver/ eoxserver.services -v2"
+          - "manage.py test --pythonpath=./eoxserver/ eoxserver.resources.coverages -v2"
+          - "manage.py test autotest_services --tag wcs20 -v2"
+          - "manage.py test autotest_services --tag wcs11 -v2"
+          - "manage.py test autotest_services --tag wcs10 -v2"
+          - "manage.py test autotest_services --tag wms -v2"
+          - "manage.py test autotest_services --tag wps -v2"
+          - "manage.py test autotest_services --tag opensearch -v2"
     needs: build-docker
     steps:
       - uses: actions/checkout@v3
-      - uses: actions/download-artifact@v3.0.1
+      - uses: actions/cache/restore@v3
+        id: cache
         with:
-          name: eoxserver
-      - name: Import docker image
+          path: eoxserver.tar.gz
+          key: eoxserver.tar.gz-${{ github.run_id }}-${{ github.run_number }}
+      # - uses: actions/download-artifact@v3.0.1
+      #   with:
+      #     name: eoxserver
+      - name: Import docker image and name it autotest
         run: |
           docker load --input eoxserver.tar.gz
-      - name: Run the tests
-        env:
-          COMPOSE_INTERACTIVE_NO_CLI: 1
+          docker tag eoxserver:latest eoxserver:autotest
+      - name: Start the services and install test dependencies
         run: |
-          echo "DB=${{ matrix.db }}" >> sample.env
+          echo "DB=spatialite" >> sample.env
           docker-compose config
           docker-compose up -d
           docker-compose ps
           docker exec -i eoxserver_autotest_1 pip3 install scipy
-          docker exec -i eoxserver_autotest_1 ${{ matrix.python_bin }} -m eoxserver.services.ows.wps.test_data_types
-          docker exec -i eoxserver_autotest_1 ${{ matrix.python_bin }} -m eoxserver.services.ows.wps.test_allowed_values
-          docker exec -i eoxserver_autotest_1 ${{ matrix.python_bin }} manage.py test --pythonpath=./eoxserver/ eoxserver.core -v2
-          docker exec -i eoxserver_autotest_1 ${{ matrix.python_bin }} manage.py test --pythonpath=./eoxserver/ eoxserver.backends -v2
-          docker exec -i eoxserver_autotest_1 ${{ matrix.python_bin }} manage.py test --pythonpath=./eoxserver/ eoxserver.services -v2
-          docker exec -i eoxserver_autotest_1 ${{ matrix.python_bin }} manage.py test --pythonpath=./eoxserver/ eoxserver.resources.coverages -v2
-          docker exec -i eoxserver_autotest_1 ${{ matrix.python_bin }} manage.py test autotest_services --tag wcs20 -v2
-          docker exec -i eoxserver_autotest_1 ${{ matrix.python_bin }} manage.py test autotest_services --tag wcs11 -v2
-          docker exec -i eoxserver_autotest_1 ${{ matrix.python_bin }} manage.py test autotest_services --tag wcs10 -v2
-          docker exec -i eoxserver_autotest_1 ${{ matrix.python_bin }} manage.py test autotest_services --tag wms -v2
-          docker exec -i eoxserver_autotest_1 ${{ matrix.python_bin }} manage.py test autotest_services --tag wps -v2
-          docker exec -i eoxserver_autotest_1 ${{ matrix.python_bin }} manage.py test autotest_services --tag opensearch -v2
+      - name: Run the tests
+        env:
+          COMPOSE_INTERACTIVE_NO_CLI: 1
+        run: |
+          docker exec -i eoxserver_autotest_1 python3 ${{ matrix.command }}
       - name: Upload logs and outputs of failed tests
         uses: 'actions/upload-artifact@v2'
         with:
-          name: logs ${{ matrix.python }} ${{ matrix.db }} ${{ matrix.django }}
+          name: logs ${{ matrix.command }}
           path: |
             autotest/autotest/logs/*.log
             autotest/autotest/responses/*
@@ -170,8 +181,8 @@ jobs:
           fields: repo,message,commit,author,action,eventName,ref,workflow,job,took
         env:
           SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
-        if: failure()  
-  
+        if: failure()
+
   publish-github:
     runs-on: ubuntu-latest
     needs: publish-pypi
@@ -224,7 +235,7 @@ jobs:
         env:
           SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
         if: failure()
-  
+
   notify:
     runs-on: ubuntu-20.04
     needs: [publish-github, publish-docker]

Dockerfile

@@ -2,6 +2,8 @@ FROM ubuntu:22.04
 
 ENV INSTANCE_NAME=instance
 ENV TZ=UTC
+ENV PYTHONPATH='/opt/eoxserver'
+ENV PYTHONUNBUFFERED="1"
 
 # possible values are "postgis" and "spatialite"
 ENV DB=spatialite
@@ -42,9 +44,14 @@ RUN mkdir $PROMETHEUS_MULTIPROC_DIR  # make sure this is writable by webserver u
 
 RUN mkdir /opt/eoxserver/
 WORKDIR /opt/eoxserver
+
+# install dependencies
+COPY requirements.txt .
+RUN python3 -m pip install -U pip \
+  && python3 -m pip install --no-cache-dir -r requirements.txt
+
+# install EOxServer
 COPY . .
-RUN python3 -m pip install --no-cache-dir . \
-  && chmod +x entrypoint.sh
 
 EXPOSE 8000
 

docker-compose.yml

@@ -10,6 +10,7 @@ services:
       POSTGRES_DB: "dbms"
 
   autotest:
+    image: eoxserver:autotest
     build: .
     env_file:
       - ./sample.env

entrypoint.sh

@@ -27,8 +27,8 @@ if [ ! -d "${INSTANCE_DIR}" ]; then
       source $f
     done
   fi
-
-  eoxserver-instance.py "${INSTANCE_NAME}" "${INSTANCE_DIR}"
+  
+  ./eoxserver/scripts/eoxserver-instance.py "${INSTANCE_NAME}" "${INSTANCE_DIR}"
   cd "${INSTANCE_DIR}"
 
   # create the database schema

eoxserver/instance_template/manage.py

@@ -19,4 +19,4 @@
                 "forget to activate a virtual environment?"
             )
         raise
-    execute_from_command_line(sys.argv)
+    execute_from_command_line(sys.argv)

eoxserver/scripts/eoxserver-instance.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
 #-------------------------------------------------------------------------------
 #
 # Project: EOxServer <http://eoxserver.org>
@@ -42,7 +42,6 @@
 import eoxserver
 from eoxserver.core.instance import create_instance
 
-
 def main():
     parser = OptionParser(
         usage=textwrap.dedent("""\

requirements.txt

@@ -0,0 +1,17 @@
+django<4
+python-dateutil
+django-model-utils<5.0.0
+django-utils-six==2.0
+django-prometheus
+tzdata
+zipstream
+psycopg2
+lxml
+pycql==0.0.8
+matplotlib
+pyows>=0.2.6
+python-keystoneclient<6.0.0
+python-swiftclient<5.0.0
+jsonfield
+gunicorn
+mapscript

setup.cfg

@@ -1,3 +1,66 @@
+[metadata]
+name = EOxServer
+version = attr: eoxserver.__version__
+author = EOX IT Services GmbH
+author_email = office@eox.at
+maintainer = EOX IT Services GmbH
+maintainer_email = packages@eox.at
+description = EOxServer is a server for Earth Observation (EO) data
+long_description = file: README.md
+long_description_content_type = text/markdown
+keywords = Earth Observation, EO, OGC, WCS, WMS
+license = EOxServer Open License (MIT-style)
+url = http://eoxserver.org/
+classifiers =
+    Development Status :: 5 - Production/Stable
+    Environment :: Console
+    Environment :: Web Environment
+    Framework :: Django
+    Intended Audience :: End Users/Desktop
+    Intended Audience :: Other Audience
+    Intended Audience :: System Administrators
+    Intended Audience :: Science/Research
+    License :: OSI Approved :: MIT License
+    Natural Language :: English
+    Operating System :: OS Independent
+    Programming Language :: Python
+    Programming Language :: Python :: 3.8
+    Programming Language :: Python :: 3.9
+    Programming Language :: Python :: 3.10
+    Topic :: Database
+    Topic :: Internet
+    Topic :: Internet :: WWW/HTTP
+    Topic :: Internet :: WWW/HTTP :: Dynamic Content
+    Topic :: Multimedia :: Graphics
+    Topic :: Scientific/Engineering :: GIS
+    Topic :: Scientific/Engineering :: Information Analysis
+    Topic :: Scientific/Engineering :: Visualization
+
+[options]
+zip_safe = False
+include_package_data = True
+packages = find:
+scripts =
+    eoxserver/scripts/eoxserver-instance.py
+    tools/eoxserver-atpd.py
+    tools/eoxserver-validate_xml.py
+    tools/eoxserver-preprocess.py
+
+[options.extras_require]
+dev =
+    scipy
+    flake8
+    mypy
+    black
+docs =
+    sphinx
+    sphinx_rtd_theme
+    myst-parser
+
+
+[options.packages.find]
+include = eoxserver
+
 [bdist_rpm]
 requires = Django >= 1.4
            gdal-python