Add Workflow Matrix (#1052)

* update install script to take args for python and django versions * add matrix to tethys workflow yml * fixes in orders for versions * fix sed command for mac * fix docker build * get right tag for docker build * update default version for django * don't need matrix for conda build * address feedback from code review. * only run coveralls on one set of tests. * add brackets to if * fix tethys release yml * force build * fix docker tag for startup tests * fixed missing a new tag reference. * fix version number and remove conda pin * missed a version definition
tethysplatform · May 28, 2024 · 2011158 · 2011158
1 parent 28e80b7
commit 2011158
Show file tree

Hide file tree

Showing 3 changed files with 54 additions and 27 deletions.
diff --git a/.github/workflows/tethys-release.yml b/.github/workflows/tethys-release.yml
@@ -12,13 +12,14 @@ env:
 
 jobs:
   docker-build:
-    name: Docker Build (${{ matrix.platform }})
+    name: Docker Build (${{ matrix.platform }}, ${{ matrix.django-version }}, ${{ matrix.python-version }})
     runs-on: ${{ matrix.platform }}
     strategy:
       fail-fast: false
       matrix:
         platform: [ubuntu-latest]
-        python-version: [3.10]
+        python-version: ["3.10", "3.11", "3.12"]
+        django-version: ["3.2", "4.2", "5.0"]
     steps:
       # Checkout the source
       - name: Checkout Source
@@ -32,7 +33,7 @@ jobs:
       - name: Safe Tag
         id: safetag
         run: |
-          full_tag="${{ steps.version.outputs.full }}";
+          full_tag="${{ steps.version.outputs.full }}-py${{ matrix.python-version }}-dj${{ matrix.django-version }}";
           # no "+" characters allowed in Docker tags
           safe_tag="${full_tag//+/-}";
           echo "safetag=$safe_tag" >> $GITHUB_OUTPUT;
@@ -68,12 +69,6 @@ jobs:
         run: |
           echo "Pushing to docker registry";
           docker push ${{ secrets.DOCKER_UPLOAD_URL }}:${{ steps.safetag.outputs.safetag }};
-      - name: Upload Docker With Latest Tag
-        if: ${{ steps.version.outputs.prerelease == '' }}
-        run: |
-          echo "Updating latest on the docker registry";
-          docker tag ${{ secrets.DOCKER_UPLOAD_URL }}:${{ steps.safetag.outputs.safetag }} ${{ secrets.DOCKER_UPLOAD_URL }}:latest;
-          docker push ${{ secrets.DOCKER_UPLOAD_URL }}:latest;
 
   conda-build:
     name: Conda Build (${{ matrix.platform }})
@@ -82,9 +77,8 @@ jobs:
       fail-fast: false
       matrix:
         platform: [ubuntu-latest]
-        python-version: [3.10]
-    steps:
       # Checkout the source
+    steps:
       - name: Checkout Source
         uses: actions/checkout@v2
         with:

diff --git a/.github/workflows/tethys.yml b/.github/workflows/tethys.yml
@@ -16,7 +16,7 @@ on:
 env:
   CONDA_BUILD_PIN_LEVEL: minor
   GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-  TEST_IMAGE: tethys:dev
+  TEST_IMAGE: tethys
   POSTGRES_DB: tethys_postgis
   POSTGRES_PASSWORD: please_dont_use_default_passwords
   POSTGRES_PORT: 5432
@@ -49,13 +49,14 @@ jobs:
       - uses: psf/black@stable
 
   tests:
-    name: Tests (${{ matrix.platform }})
+    name: Tests (${{ matrix.platform }}, ${{ matrix.django-version }}, ${{ matrix.python-version }})
     runs-on: ${{ matrix.platform }}
     strategy:
       fail-fast: false
       matrix:
         platform: [ubuntu-latest, macos-latest]
-        python-version: [3.7]
+        python-version: ["3.10", "3.11", "3.12"]
+        django-version: ["3.2", "4.2", "5.0"]
     steps:
       # Checkout the source
       - name: Checkout Source
@@ -67,7 +68,7 @@ jobs:
         run: |
           cd ..
           bash ./tethys/scripts/install_tethys.sh -h
-          bash ./tethys/scripts/install_tethys.sh --partial-tethys-install meds -n tethys -s $PWD/tethys -x
+          bash ./tethys/scripts/install_tethys.sh --partial-tethys-install meds -n tethys -s $PWD/tethys -x -d ${{ matrix.django-version }} -p ${{ matrix.python-version }}
       # Setup Tethys and Conda
       - name: Setup Tethys and Conda
         run: |
@@ -84,20 +85,21 @@ jobs:
           tethys test -c -u -v 2
       # Generate Coverage Report
       - name: Generate Coverage Report
-        if: matrix.platform == 'ubuntu-latest'
+        if: ${{ matrix.platform == 'ubuntu-latest' && matrix.python-version == '3.10' && matrix.django-version == '4.2' }}
         run: |
           . ~/miniconda/etc/profile.d/conda.sh
           conda activate tethys
           coveralls --service=github
 
   docker-build:
-    name: Docker Build
+    name: Docker Build (${{ matrix.platform }}, ${{ matrix.django-version }}, ${{ matrix.python-version }})
     runs-on: ${{ matrix.platform }}
     strategy:
       fail-fast: false
       matrix:
         platform: [ubuntu-latest]
-        python-version: [3.7]
+        python-version: ["3.10", "3.11", "3.12"]
+        django-version: ["3.2", "4.2", "5.0"]
     steps:
       # Checkout the source
       - name: Checkout Source
@@ -107,15 +109,15 @@ jobs:
       # Build the docker for no tag
       - name: Build Without Tag
         run: |
-          docker build -t ${{ secrets.DOCKER_UPLOAD_URL }}:dev .;
-          docker tag ${{ secrets.DOCKER_UPLOAD_URL }}:dev ${{ env.TEST_IMAGE }};
+          docker build --build-arg DJANGO_VERSION=${{ matrix.django-version }} --build-arg PYTHON_VERSION=${{ matrix.python-version }} -t ${{ secrets.DOCKER_UPLOAD_URL }}:dev-py${{ matrix.python-version }}-dj${{ matrix.django-version }} .;
+          docker tag ${{ secrets.DOCKER_UPLOAD_URL }}:dev-py${{ matrix.python-version }}-dj${{ matrix.django-version }} ${{ env.TEST_IMAGE }}:dev-py${{ matrix.python-version }}-dj${{ matrix.django-version }};
       # Upload docker if pull request no tag
       - name: Upload Docker No Tag
         if: ${{ github.event_name != 'pull_request' }}
         run: |
           echo "Pushing to docker registry";
           echo "${{ secrets.DOCKER_PASSWORD }}" | docker login -u "${{ secrets.DOCKER_USERNAME }}" --password-stdin;
-          docker push ${{ secrets.DOCKER_UPLOAD_URL }}:dev;
+          docker push ${{ secrets.DOCKER_UPLOAD_URL }}:dev-py${{ matrix.python-version }}-dj${{ matrix.django-version }};
       # No Upload if Pull Request
       - name: No Upload
         if: ${{ github.event_name == 'pull_request' }}
@@ -125,13 +127,19 @@ jobs:
       - name: Upload Docker Artifact
         uses: ishworkh/docker-image-artifact-upload@v1
         with:
-          image: ${{ env.TEST_IMAGE }}
+          image: ${{ env.TEST_IMAGE }}:dev-py${{ matrix.python-version }}-dj${{ matrix.django-version }}
           retention_days: "1"
 
   startup_test:
-    name: Docker Start-up Test
+    name: Docker Start-up Test (${{ matrix.platform }}, ${{ matrix.django-version }}, ${{ matrix.python-version }})
     needs: [docker-build]
     runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        platform: [ubuntu-latest]
+        python-version: ["3.10", "3.11", "3.12"]
+        django-version: ["3.2", "4.2", "5.0"]
     services:
       tethys-postgis:
         image: postgis/postgis:14-3.3
@@ -151,7 +159,7 @@ jobs:
       - name: Download Docker Artifact
         uses: ishworkh/docker-image-artifact-download@v1
         with:
-          image: ${{ env.TEST_IMAGE }}
+          image: ${{ env.TEST_IMAGE }}:dev-py${{ matrix.python-version }}-dj${{ matrix.django-version }}
       - name: Run Salt Test
         run: |
           docker run --rm \
@@ -164,7 +172,7 @@ jobs:
             -e TETHYS_DB_PASSWORD=${{ env.TETHYS_DB_PASSWORD }} \
             -e TETHYS_DB_SUPERUSER=${{ env.TETHYS_DB_SUPERUSER }} \
             -e TETHYS_DB_SUPERUSER_PASS=${{ env.TETHYS_DB_SUPERUSER_PASS }} \
-            ${{ env.TEST_IMAGE }} \
+            ${{ env.TEST_IMAGE }}:dev-py${{ matrix.python-version }}-dj${{ matrix.django-version }} \
             /bin/bash -c "cd /usr/lib/tethys && source ./run.sh --test"
 
   conda-build:
@@ -174,7 +182,6 @@ jobs:
       fail-fast: false
       matrix:
         platform: [ubuntu-latest]
-        python-version: [3.7]
     steps:
       # Checkout the source
       - name: Checkout Source
@@ -270,4 +277,4 @@ jobs:
       - name: No Upload - micro
         if: ${{ github.event_name == 'pull_request' }}
         run: |
-          echo "Uploading is skipped for pull requests."
+          echo "Uploading is skipped for pull requests."
diff --git a/scripts/install_tethys.sh b/scripts/install_tethys.sh
@@ -15,6 +15,8 @@ OPTIONS:\n
 \t    -p, --port <PORT>                   \t\t\t Port on which to serve tethys. Default is 8000.\n
 \t    -b, --branch <BRANCH_NAME>          \t\t Branch to checkout from version control. Default is 'main'.\n
 \t    -c, --conda-home <PATH>             \t\t Path where Miniconda will be installed, or to an existing installation of Miniconda. Default is ~/miniconda.\n
+\t    -d, --django-version <VERSION>      \t\t Version of Django to install. Default is '4.2'.\n
+\t    -p, --python-version <VERSION>      \t\t Version of Python to install. Default is '3.12'.\n
 \t    --db-username <USERNAME>            \t\t Username that the tethys database server will use. Default is 'tethys_default'.\n
 \t    --db-password <PASSWORD>            \t\t Password that the tethys database server will use. Default is 'pass'.\n
 \t    --db-super-username <USERNAME>      \t Username for super user on the tethys database server. Default is 'tethys_super'.\n
@@ -88,6 +90,8 @@ TETHYS_DB_SUPER_PASSWORD='pass'
 TETHYS_DB_PORT=5436
 TETHYS_DB_DIR='psql'
 CONDA_HOME=~/miniconda
+PYTHON_VERSION='3.12'
+DJANGO_VERSION='4.2'
 CONDA_ENV_NAME='tethys-dev'
 BRANCH='main'
 
@@ -150,6 +154,14 @@ case $key in
     set_option_value CONDA_ENV_NAME "$2"
     shift # past argument
     ;;
+    -d|--django-version)
+    set_option_value DJANGO_VERSION "$2"
+    shift # past argument
+    ;;
+    -v|--python-version)
+    set_option_value PYTHON_VERSION "$2"
+    shift # past argument
+    ;;
     --db-username)
     set_option_value TETHYS_DB_USERNAME "$2"
     shift # past argument
@@ -351,6 +363,20 @@ then
         git checkout ${BRANCH}
     fi
 
+    if [ -n "${DJANGO_VERSION}" ]
+    then
+        echo "Updating environment.yml Django version ${DJANGO_VERSION}..."
+        sudo sed -i.bak "s/django>=.*/django>=${DJANGO_VERSION}/" "${TETHYS_SRC}/environment.yml"
+        sudo sed -i.bak "s/django>=.*/django>=${DJANGO_VERSION}/" "${TETHYS_SRC}/micro_environment.yml"
+    fi
+
+    if [ -n "${PYTHON_VERSION}" ]
+    then
+        echo "Updating environment.yml Python version ${PYTHON_VERSION}..."
+        sudo sed -i.bak "s/django>=.*/python>=${PYTHON_VERSION}/" "${TETHYS_SRC}/environment.yml"
+        sudo sed -i.bak "s/django>=.*/python>=${PYTHON_VERSION}/" "${TETHYS_SRC}/micro_environment.yml"
+    fi
+
     if [ -n "${CREATE_ENV}" ]
     then
         # create conda env and install Tethys