[MAINTENANCE] Update ci to check version is not bumped and then auto …

…bump pre-release on merge (#203)
great-expectations · Apr 16, 2024 · 34f47ff · 34f47ff
1 parent bf8f403
commit 34f47ff
Show file tree

Hide file tree

Showing 3 changed files with 133 additions and 61 deletions.
diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
@@ -6,6 +6,81 @@ on:
     - cron: "0 */2 * * *"
 
 jobs:
+  static-analysis:
+    strategy:
+      matrix:
+        task: ["lint", "fmt", "type-check", "python-build", "docker"]
+      fail-fast: false
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.8"
+          cache-dependency-path: poetry.lock
+
+      - name: Load cached Poetry installation
+        id: cached-poetry
+        uses: actions/cache@v4
+        with:
+          path: ~/.local # the path depends on the OS
+          key: poetry-3.8 # increment to reset cache
+
+      - name: Install Poetry
+        if: steps.cached-poetry.outputs.cache-hit != 'true'
+        uses: snok/install-poetry@v1
+
+      - name: Install dependencies
+        run: poetry install --sync
+
+      - run: poetry run invoke ${{ matrix.task }} --check
+
+  tests:
+    strategy:
+      matrix:
+        python-version: ["3.8", "3.9", "3.10", "3.11"]
+
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+          cache-dependency-path: poetry.lock
+
+      - name: Load cached Poetry installation
+        id: cached-poetry
+        uses: actions/cache@v4
+        with:
+          path: ~/.local # the path depends on the OS
+          key: poetry-${{ matrix.python-version  }} # increment to reset cache
+
+      - name: Install Poetry
+        if: steps.cached-poetry.outputs.cache-hit != 'true'
+        uses: snok/install-poetry@v1
+
+      - name: Install dependencies
+        run: poetry install --sync
+
+      - name: Run tests
+        run: poetry run pytest -m "not agentjobs" --cov-report xml
+
+      # upload coverage report to codecov
+      - name: Upload coverage reports to Codecov
+        uses: codecov/codecov-action@v4.1.0
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          slug: great-expectations/cloud
+          flags: ${{ matrix.python-version }}
+
   check-if-agent-changed:
     runs-on: ubuntu-latest
     if: ${{ github.event_name != 'schedule' }}
@@ -27,6 +102,8 @@ jobs:
     needs: check-if-agent-changed
     runs-on: ubuntu-latest
     if: ${{ needs.check-if-agent-changed.outputs.agent_changed == 'true' && github.event_name != 'schedule' }}
+    outputs:
+      is_version_bumped: ${{ steps.is_version_bumped.outputs.IS_VERSION_BUMPED }}
 
     steps:
       - name: Checkout
@@ -43,7 +120,7 @@ jobs:
         uses: actions/cache@v4
         with:
           path: ~/.local # the path depends on the OS
-          key: poetry-0 # increment to reset cache
+          key: poetry-3.8 # increment to reset cache
 
       - name: Install Poetry
         if: steps.cached-poetry.outputs.cache-hit != 'true'
@@ -62,19 +139,21 @@ jobs:
         id: main_version
         run: echo "VERSION=$(poetry version -s)" >> $GITHUB_OUTPUT
 
-      - name: Check version in PR is not the same as in main
-        if: ${{ steps.pr_version.outputs.VERSION == steps.main_version.outputs.VERSION }}
+      - name: Get is_version_bumped
+        id: is_version_bumped
         run: |
-          echo "Version in PR is the same as in main. Please bump the version in pyproject.toml."
-          exit 1
-
-  static-analysis:
-    strategy:
-      matrix:
-        task: ["lint", "fmt", "type-check", "python-build", "docker"]
-      fail-fast: false
+          if [ "${{ steps.pr_version.outputs.VERSION }}" == "${{ steps.main_version.outputs.VERSION }}" ]; then
+            echo "IS_VERSION_BUMPED=false" >> $GITHUB_OUTPUT
+          else
+            echo "IS_VERSION_BUMPED=true" >> $GITHUB_OUTPUT
+          fi
+
+  bump-pre-release-version:
+    permissions:
+      contents: write # Allow us to write to the repository
+    needs: check-version-is-bumped
     runs-on: ubuntu-latest
-
+    if: ${{ needs.check-version-is-bumped.outputs.is_version_bumped == 'false' && github.event_name != 'schedule' }}
     steps:
       - name: Checkout
         uses: actions/checkout@v4
@@ -97,48 +176,20 @@ jobs:
         uses: snok/install-poetry@v1
 
       - name: Install dependencies
-        run: poetry install --sync
-
-      - run: poetry run invoke ${{ matrix.task }} --check
-
-  tests:
-    strategy:
-      matrix:
-        python-version: ["3.8", "3.9", "3.10", "3.11"]
+        run: poetry install --sync # Needed for calling invoke
 
-    runs-on: ubuntu-latest
+      - name: Checkout PR
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: gh pr checkout ${{ github.event.pull_request.number }}
 
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
+      - name: Bump pre-release version
+        run: invoke pre-release
 
-      - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v5
-        with:
-          python-version: ${{ matrix.python-version }}
-          cache-dependency-path: poetry.lock
-
-      - name: Load cached Poetry installation
-        id: cached-poetry
-        uses: actions/cache@v4
-        with:
-          path: ~/.local # the path depends on the OS
-          key: poetry-${{ matrix.python-version  }} # increment to reset cache
-
-      - name: Install Poetry
-        if: steps.cached-poetry.outputs.cache-hit != 'true'
-        uses: snok/install-poetry@v1
-
-      - name: Install dependencies
-        run: poetry install --sync
-
-      - name: Run tests
-        run: poetry run pytest -m "not agentjobs" --cov-report xml
-
-      # upload coverage report to codecov
-      - name: Upload coverage reports to Codecov
-        uses: codecov/codecov-action@v4.1.0
-        with:
-          token: ${{ secrets.CODECOV_TOKEN }}
-          slug: great-expectations/cloud
-          flags: ${{ matrix.python-version }}
+      - name: Push changes to PR
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          git add .
+          git commit -m "Pre-release version bump"
+          git push
diff --git a/.github/workflows/containerize-agent.yaml b/.github/workflows/containerize-agent.yaml
@@ -103,8 +103,24 @@ jobs:
         id: get_version
         run: echo "POETRY_VERSION=$(poetry version -s)" >> $GITHUB_OUTPUT
 
+      - name: Get is_release from pyproject.toml
+        id: get_is_release
+        run: echo "IS_RELEASE=$(poetry version -s | grep -q 'dev' && echo false || echo true)" >> $GITHUB_OUTPUT
+
       # Uses the version from pyproject.toml for the tag
-      - name: Push to Docker Hub and Internal ECR
+      - name: Push Pre-Release to Docker Hub and Internal ECR
+        if: ${{ steps.get_is_release.outputs.IS_RELEASE == 'false' }}
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          platforms: linux/amd64,linux/arm64
+          push: true
+          tags: |
+            greatexpectations/agent:${{ steps.get_version.outputs.POETRY_VERSION }}
+            258143015559.dkr.ecr.us-east-1.amazonaws.com/gx/agent:${{ steps.get_version.outputs.POETRY_VERSION }}
+
+      - name: Push Release to Docker Hub and Internal ECR
+        if: ${{ steps.get_is_release.outputs.IS_RELEASE == 'true' }}
         uses: docker/build-push-action@v5
         with:
           context: .

diff --git a/README.md b/README.md
@@ -174,16 +174,21 @@ There can be days with no standard releases, only pre-releases or days with no p
 
 #### Pre-releases
 
-Pre-releases will be completed automatically with each merge to the main branch
-in the future. For now, to create a pre-release, run the following command to update the version in `pyproject.toml`
-and then merge along with your changes. This should be done in each PR that is merged to `main` that includes
-changes to the agent or it's dependencies:
+Pre-releases are completed automatically with each merge to the `main` branch.
+The version is updated in `pyproject.toml` and a pre-release is created on PyPi.
+A new docker tag will also be generated and pushed to [Docker Hub](https://hub.docker.com/r/greatexpectations/agent)
+
+**Manual Pre-releases**
+
+NOTE: CI will automatically create pre-releases on merges to `main`. Instead of manually creating pre-releases, consider using the CI process. This is only for exceptional cases.
+
+To manually create a pre-release, run the following command to update the version in `pyproject.toml` and then merge it to `main` in a standalone PR:
 
 ```console
 invoke pre-release
 ```
 
-This will create a new pre-release version and push it to PyPi.
+This will create a new pre-release version. On the next merge to `main`, the release will be uploaded to PyPi.
 A new docker tag will also be generated and pushed to [Docker Hub](https://hub.docker.com/r/greatexpectations/agent)
 
 #### Releases
@@ -197,5 +202,5 @@ merge it to `main` in a standalone PR:
 invoke release
 ```
 
-This will create a new release version and push it to PyPi.
+This will create a new release version. On the next merge to `main`, the release will be uploaded to PyPi.
 A new docker tag will also be generated and pushed to [Docker Hub](https://hub.docker.com/r/greatexpectations/agent)