Skip to content
ci

ci #5562

Sign in to view logs

ci

ci #5562

Workflow file for this run

.github/workflows/ci.yaml at d366b02
name: ci
on:
pull_request:
schedule:
- cron: "0 */2 * * *"
jobs:
static-analysis:
strategy:
matrix:
task: ["lint", "fmt", "type-check", "python-build", "docker"]
fail-fast: false
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: "3.8"
cache-dependency-path: poetry.lock
- name: Load cached Poetry installation
id: cached-poetry
uses: actions/cache@v4
with:
path: ~/.local # the path depends on the OS
key: poetry-3.8 # increment to reset cache
- name: Install Poetry
if: steps.cached-poetry.outputs.cache-hit != 'true'
uses: snok/install-poetry@v1
- name: Install dependencies
run: poetry install --sync
- run: poetry run invoke ${{ matrix.task }} --check
tests:
strategy:
matrix:
python-version: ["3.8", "3.9", "3.10", "3.11"]
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python-version }}
cache-dependency-path: poetry.lock
- name: Load cached Poetry installation
id: cached-poetry
uses: actions/cache@v4
with:
path: ~/.local # the path depends on the OS
key: poetry-${{ matrix.python-version }} # increment to reset cache
- name: Install Poetry
if: steps.cached-poetry.outputs.cache-hit != 'true'
uses: snok/install-poetry@v1
- name: Install dependencies
run: poetry install --sync
- name: Run tests
run: poetry run pytest -m "not agentjobs and not integration" --cov-report xml
# upload coverage report to codecov
- name: Upload coverage reports to Codecov
uses: codecov/codecov-action@v4.1.0
with:
token: ${{ secrets.CODECOV_TOKEN }}
slug: great-expectations/cloud
flags: ${{ matrix.python-version }}
integration-tests:
strategy:
matrix:
python-version: ["3.10"]
permissions:
id-token: write
contents: read
actions: read
runs-on: ubuntu-latest
env:
# GX Agent
GX_CLOUD_ACCESS_TOKEN: ${{ secrets.GX_CLOUD_ACCESS_TOKEN }}
GX_CLOUD_ORGANIZATION_ID: ${{ secrets.GX_CLOUD_ORGANIZATION_ID }}
GX_CLOUD_BASE_URL: "http://localhost:5000"
# GX Cloud
AUTH0_DOMAIN: ${{secrets.AUTH0_DOMAIN}}
AUTH0_API_AUDIENCE: ${{secrets.AUTH0_API_AUDIENCE}}
AUTH0_MERCURY_API_CLIENT_ID: ${{secrets.AUTH0_MERCURY_API_CLIENT_ID}}
AUTH0_MERCURY_API_CLIENT_SECRET: ${{secrets.AUTH0_MERCURY_API_CLIENT_SECRET}}
LOGGING_LEVEL: DEBUG
ENVIRONMENT: local
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python-version }}
cache-dependency-path: poetry.lock
- name: Load cached Poetry installation
id: cached-poetry
uses: actions/cache@v4
with:
path: ~/.local # the path depends on the OS
key: poetry-${{ matrix.python-version }} # increment to reset cache
- name: Install Poetry
if: steps.cached-poetry.outputs.cache-hit != 'true'
uses: snok/install-poetry@v1
- name: Install dependencies
run: poetry install --sync
- name: Configure ECR AWS Credentials
uses: aws-actions/configure-aws-credentials@v4
with:
aws-region: us-east-1
role-to-assume: arn:aws:iam::258143015559:role/github-amazonec2containerregistryreadonly
- name: Login to Amazon ECR
id: login-ecr
uses: aws-actions/amazon-ecr-login@v2
- name: Start services
run: |
docker compose up db mq -d
docker compose run db-provisioner
docker compose run db-seeder
docker compose run mq-wait
docker compose up mercury-service-api -d
- name: Show logs for debugging
if: failure()
run: |
docker compose -f docker-compose.yml logs
- name: Run tests
run: poetry run pytest -vv -m integration -W default::great_expectations.datasource.fluent.GxInvalidDatasourceWarning --cov-report xml
# upload coverage report to codecov
- name: Upload coverage reports to Codecov
uses: codecov/codecov-action@v4.1.0
with:
token: ${{ secrets.CODECOV_TOKEN }}
slug: great-expectations/cloud
flags: integration
test-docker-build:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: "3.10"
cache-dependency-path: poetry.lock
- name: Load cached Poetry installation
id: cached-poetry
uses: actions/cache@v4
with:
path: ~/.local # the path depends on the OS
key: poetry-3.10 # increment to reset cache
- name: Install Poetry
if: steps.cached-poetry.outputs.cache-hit != 'true'
uses: snok/install-poetry@v1
- name: Install dependencies
run: poetry install --sync --only dev
- run: poetry run invoke docker --no-pty
check-if-agent-changed:
runs-on: ubuntu-latest
if: ${{ github.event_name != 'schedule' && !contains(github.event.pull_request.labels.*.name, 'no version bump') }}
outputs:
agent_changed: ${{ steps.filter.outputs.agent }}
steps:
- name: Checkout
uses: actions/checkout@v4
- uses: dorny/paths-filter@v2
id: filter
with:
filters: |
agent:
- 'great_expectations_cloud/agent/**'
- 'poetry.lock'
- 'pyproject.toml'
check-version-is-bumped:
# This job checks that the version in the PR is different from the version in main if the agent has changed.
needs: check-if-agent-changed
runs-on: ubuntu-latest
if: ${{ needs.check-if-agent-changed.outputs.agent_changed == 'true' && !contains(github.event.pull_request.labels.*.name, 'no version bump') && github.event_name != 'schedule' }}
outputs:
is_version_bumped: ${{ steps.is_version_bumped.outputs.IS_VERSION_BUMPED }}
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: "3.8"
cache-dependency-path: poetry.lock
- name: Load cached Poetry installation
id: cached-poetry
uses: actions/cache@v4
with:
path: ~/.local # the path depends on the OS
key: poetry-3.8 # increment to reset cache
- name: Install Poetry
if: steps.cached-poetry.outputs.cache-hit != 'true'
uses: snok/install-poetry@v1
- name: Get version from pyproject.toml for PR
id: pr_version
run: echo "VERSION=$(poetry version -s)" >> $GITHUB_OUTPUT
- name: Checkout `main` branch
uses: actions/checkout@v4
with:
ref: main
- name: Get version from pyproject.toml for main
id: main_version
run: echo "VERSION=$(poetry version -s)" >> $GITHUB_OUTPUT
- name: Get is_version_bumped
id: is_version_bumped
run: |
if [ "${{ steps.pr_version.outputs.VERSION }}" == "${{ steps.main_version.outputs.VERSION }}" ]; then
echo "IS_VERSION_BUMPED=false" >> $GITHUB_OUTPUT
else
echo "IS_VERSION_BUMPED=true" >> $GITHUB_OUTPUT
fi
bump-pre-release-version:
permissions:
contents: write # Allow us to write to the repository
needs: check-version-is-bumped
runs-on: ubuntu-latest
if: ${{ needs.check-version-is-bumped.outputs.is_version_bumped == 'false' && !contains(github.event.pull_request.labels.*.name, 'no version bump') && !contains(github.event.pull_request.labels.*.name, 'dependencies') && github.event_name != 'schedule' }}
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: "3.8"
cache-dependency-path: poetry.lock
- name: Load cached Poetry installation
id: cached-poetry
uses: actions/cache@v4
with:
path: ~/.local # the path depends on the OS
key: poetry-3.8 # increment to reset cache
- name: Install Poetry
if: steps.cached-poetry.outputs.cache-hit != 'true'
uses: snok/install-poetry@v1
- name: Install dependencies
run: poetry install --sync # Needed for calling invoke
- name: Checkout PR
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: gh pr checkout ${{ github.event.pull_request.number }}
- name: Bump pre-release version
run: poetry run invoke pre-release
- name: Push changes to PR
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
git config --global user.email "team@greatexpectations.io"
git config --global user.name "Great Expectations"
git add .
git commit -m "Pre-release version bump"
git push