great-expectations 0.18.16 (#307) #332
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: containerize-agent | |
| on: | |
| push: | |
| branches: | |
| - main | |
| jobs: | |
| containerize-agent: | |
| runs-on: ubuntu-latest | |
| permissions: | |
| id-token: write | |
| contents: read | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.8" | |
| cache-dependency-path: poetry.lock | |
| - name: Load cached Poetry installation | |
| id: cached-poetry | |
| uses: actions/cache@v4 | |
| with: | |
| path: ~/.local # the path depends on the OS | |
| key: poetry-0 # increment to reset cache | |
| - name: Install Poetry | |
| if: steps.cached-poetry.outputs.cache-hit != 'true' | |
| uses: snok/install-poetry@v1 | |
| - name: Install dependencies | |
| run: poetry install --with dev,sql --sync | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Login to Docker Hub | |
| uses: docker/login-action@v3 | |
| with: | |
| username: ${{ secrets.DOCKERHUB_USERNAME }} | |
| password: ${{ secrets.DOCKERHUB_TOKEN }} | |
| - name: Configure ECR AWS Credentials | |
| uses: aws-actions/configure-aws-credentials@v4 | |
| with: | |
| aws-region: us-east-1 | |
| role-to-assume: arn:aws:iam::258143015559:role/github-amazonec2containerregistrypoweruser | |
| - name: Login to Amazon ECR | |
| id: login-ecr | |
| uses: aws-actions/amazon-ecr-login@v2 | |
| # Cannot multi arch build AND load | |
| # https://github.com/docker/buildx/issues/59 | |
| - name: Build arm64, No Load, No Push | |
| uses: docker/build-push-action@v5 | |
| with: | |
| context: . | |
| platforms: linux/arm64 | |
| push: false | |
| load: false | |
| tags: | | |
| greatexpectations/agent:latest | |
| greatexpectations/agent:stable | |
| # https://docs.docker.com/build/ci/github-actions/test-before-push/ | |
| - name: Build amd64, With Load, No Push | |
| uses: docker/build-push-action@v5 | |
| with: | |
| context: . | |
| platforms: linux/amd64 | |
| push: false | |
| load: true | |
| tags: | | |
| greatexpectations/agent:latest | |
| greatexpectations/agent:stable | |
| - name: Smoke Test the Image | |
| run: docker run --rm greatexpectations/agent:stable poetry run gx-agent -h | |
| # Uses local image built in previous step | |
| - name: Test New amd64 Agent Image | |
| run: | | |
| docker compose up db mq -d | |
| docker compose run db-provisioner | |
| docker compose run db-seeder | |
| docker compose run mq-wait | |
| docker compose up mercury-service-api -d | |
| docker compose up gx-agent -d | |
| poetry run pytest -m "agentjobs" | |
| env: | |
| GX_CLOUD_BASE_URL: ${{ vars.GX_CLOUD_BASE_URL }} | |
| GX_CLOUD_ORGANIZATION_ID: ${{ secrets.GX_CLOUD_ORGANIZATION_ID }} | |
| GX_CLOUD_ACCESS_TOKEN: ${{ secrets.GX_CLOUD_ACCESS_TOKEN }} | |
| AUTH0_API_AUDIENCE: ${{ secrets.AUTH0_API_AUDIENCE }} | |
| AUTH0_MERCURY_API_CLIENT_ID: ${{ secrets.AUTH0_MERCURY_API_CLIENT_ID }} | |
| AUTH0_MERCURY_API_CLIENT_SECRET: ${{ secrets.AUTH0_MERCURY_API_CLIENT_SECRET }} | |
| AUTH0_DOMAIN: ${{ secrets.AUTH0_DOMAIN }} | |
| GE_USAGE_STATISTICS_URL: ${{ secrets.GE_USAGE_STATISTICS_URL }} | |
| - name: Get version from pyproject.toml | |
| id: get_version | |
| run: echo "POETRY_VERSION=$(poetry version -s)" >> $GITHUB_OUTPUT | |
| - name: Get is_release from pyproject.toml | |
| id: get_is_release | |
| run: echo "IS_RELEASE=$(poetry version -s | grep -q 'dev' && echo false || echo true)" >> $GITHUB_OUTPUT | |
| # Uses the version from pyproject.toml for the tag | |
| - name: Push Pre-Release to Docker Hub and Internal ECR | |
| if: ${{ steps.get_is_release.outputs.IS_RELEASE == 'false' }} | |
| uses: docker/build-push-action@v5 | |
| with: | |
| context: . | |
| platforms: linux/amd64,linux/arm64 | |
| push: true | |
| tags: | | |
| greatexpectations/agent:dev | |
| greatexpectations/agent:${{ steps.get_version.outputs.POETRY_VERSION }} | |
| 258143015559.dkr.ecr.us-east-1.amazonaws.com/gx/agent:${{ steps.get_version.outputs.POETRY_VERSION }} | |
| - name: Push Release to Docker Hub and Internal ECR | |
| if: ${{ steps.get_is_release.outputs.IS_RELEASE == 'true' }} | |
| uses: docker/build-push-action@v5 | |
| with: | |
| context: . | |
| platforms: linux/amd64,linux/arm64 | |
| push: true | |
| tags: | | |
| greatexpectations/agent:dev | |
| greatexpectations/agent:${{ steps.get_version.outputs.POETRY_VERSION }} | |
| greatexpectations/agent:latest | |
| greatexpectations/agent:stable | |
| 258143015559.dkr.ecr.us-east-1.amazonaws.com/gx/agent:${{ steps.get_version.outputs.POETRY_VERSION }} | |
| - name: Show logs, if failure | |
| if: failure() | |
| run: docker compose logs | |
| # NOTE: can't use coveralls here without refactoring workflows | |
| # upload coverage report to codecov | |
| - name: Upload coverage reports to Codecov | |
| uses: codecov/codecov-action@v4.1.0 | |
| with: | |
| token: ${{ secrets.CODECOV_TOKEN }} | |
| slug: great-expectations/cloud | |
| # use the same flags as the test markers | |
| flags: agentjobs |