Skip to content

CI pipeline

CI pipeline #689

Workflow file for this run

name: CI pipeline
on:
push:
branches:
- main
tags:
- "*"
schedule:
- cron: "0 0 * * *"
env:
telemetry_local_debug_mode: true
jobs:
pre-commit:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: actions/setup-python@v3
with:
python-version: '3.11.x'
- uses: pre-commit/action@v3.0.0
with:
extra_args: --all-files
run-tests:
runs-on: ubuntu-22.04
strategy:
fail-fast: false
matrix:
python-version:
- "3.8"
- "3.9"
- "3.10"
data-source:
- "postgres"
- "snowflake"
- "bigquery"
- "redshift"
- "athena"
# - "db2"
# - "spark"
- "spark_df"
- "sqlserver"
- "mysql"
- "duckdb"
- "dask"
# - "vertica"
env:
DATA_SOURCE: ${{ matrix.data-source }}
PYTHON_VERSION: ${{ matrix.python-version }}
SNOWFLAKE_HOST: ${{ secrets.SNOWFLAKE_HOST }}
SNOWFLAKE_ACCOUNT: ${{ secrets.SNOWFLAKE_ACCOUNT }}
SNOWFLAKE_USERNAME: ${{ secrets.SNOWFLAKE_USERNAME }}
SNOWFLAKE_PASSWORD: ${{ secrets.SNOWFLAKE_PASSWORD }}
SNOWFLAKE_DATABASE: ${{ secrets.SNOWFLAKE_DATABASE }}
SNOWFLAKE_SCHEMA: "public"
BIGQUERY_ACCOUNT_INFO_JSON: ${{ secrets.BIGQUERY_ACCOUNT_INFO_JSON }}
BIGQUERY_DATASET: "test"
REDSHIFT_HOST: ${{ secrets.REDSHIFT_HOST }}
REDSHIFT_USERNAME: ${{ secrets.REDSHIFT_USERNAME }}
REDSHIFT_PASSWORD: ${{ secrets.REDSHIFT_PASSWORD }}
REDSHIFT_DATABASE: "soda_test"
REDSHIFT_PORT: "5439"
ATHENA_ACCESS_KEY_ID: ${{ secrets.ATHENA_ACCESS_KEY_ID }}
ATHENA_SECRET_ACCESS_KEY: ${{ secrets.ATHENA_SECRET_ACCESS_KEY }}
ATHENA_S3_TEST_DIR: ${{ secrets.ATHENA_STAGING_DIR }}
ATHENA_SCHEMA: ${{ secrets.ATHENA_SCHEMA }}
SPARK_HOST: ${{ secrets.SPARK_HOST }}
SPARK_USERNAME: ${{ secrets.SPARK_USERNAME }}
SPARK_PASSWORD: ${{ secrets.SPARK_PASSWORD }}
SPARK_DATABASE: ${{ secrets.SPARK_DATABASE }}
SQLSERVER_USERNAME: SA
SQLSERVER_PASSWORD: Password1!
SQLSERVER_DATABASE: master
SQLSERVER_SCHEMA: dbo
MYSQL_USER: sodacore
MYSQL_DATABASE: sodacore
MYSQL_PASSWORD: sodacore
MYSQL_ROOT_PASSWORD: sodacore
# TODO: add appropriate vars
SPARK_DF_HOST: ${{ secrets.SPARK_DF_HOST }}
steps:
- uses: actions/checkout@v3
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
- name: Install dependencies
run: |
curl https://packages.microsoft.com/keys/microsoft.asc | sudo apt-key add -
curl https://packages.microsoft.com/config/ubuntu/21.04/prod.list | sudo tee /etc/apt/sources.list.d/mssql-release.list > /dev/null
sudo apt-get update
ACCEPT_EULA=Y sudo apt-get install -y libsasl2-dev msodbcsql18
python -m pip install --upgrade pip
cat dev-requirements.in | grep tox | xargs pip install
- name: Test with tox
run: |
tox -- soda -k soda/core
tox -- soda -k soda/${{ matrix.data-source }}
env:
test_data_source: ${{ matrix.data-source }}
test-cloud-integration:
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
python-version:
- "3.8"
- "3.9"
- "3.10"
env:
PYTHON_VERSION: ${{ matrix.python-version }}
steps:
- uses: actions/checkout@v3
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
- name: Install dependencies
run: |
sudo apt-get update
sudo apt-get install -y libsasl2-dev
python -m pip install --upgrade pip
cat dev-requirements.in | grep tox | xargs pip install
- name: Test with tox
run: |
tox -- soda -k soda/core
env:
test_data_source: postgres
WESTMALLE: BETTER_THAN_LA_TRAPPE
test-scientific:
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
python-version:
- "3.8"
- "3.9"
- "3.10"
env:
PYTHON_VERSION: ${{ matrix.python-version }}
steps:
- uses: actions/checkout@v3
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
- name: Install dependencies
run: |
sudo apt-get update
sudo apt-get install -y libsasl2-dev
python -m pip install --upgrade pip
cat dev-requirements.in | grep tox | xargs pip install
- name: Test with tox
run: |
tox -- soda -k soda/scientific
test-contracts:
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
python-version:
- "3.9"
env:
PYTHON_VERSION: ${{ matrix.python-version }}
steps:
- uses: actions/checkout@v3
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
- name: Install dependencies
run: |
sudo apt-get update
sudo apt-get install -y libsasl2-dev
python -m pip install --upgrade pip
cat dev-requirements.in | grep tox | xargs pip install
- name: Test with tox
run: |
tox -- soda -k soda/contracts
publish-pypi:
name: Build & Publish Package
if: contains(github.ref, 'refs/tags/')
timeout-minutes: 10
runs-on: ubuntu-20.04
needs: [run-tests]
env:
REPORTS_URL: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}
TWINE_USERNAME: __token__
TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }}
steps:
- uses: actions/checkout@v3
- name: Setup Python 3.8
uses: actions/setup-python@v4
with:
python-version: 3.8
- name: Setup dependencies
run: pip install --upgrade setuptools wheel twine
- name: Get version
id: get_version
run: echo ::set-output name=VERSION::${GITHUB_REF/refs\/tags\//}
- name: Build and publish packages
# Hacky version for now :)
run: |
FAILED=false
cd soda
for pack in *
do
pushd .
cd $pack
echo "| Building $pack"
python3 setup.py sdist bdist_wheel
echo "| Uploading $pack to pypi"
UPLOAD_FAILED=false
twine upload dist/* || UPLOAD_FAILED=true
if [ $UPLOAD_FAILED = true ]; then
FAILED=true
echo "Failed to upload $pack"
fi
popd
done
if [ $FAILED = true ]; then
echo "There was an error, check the logs please."
exit 1
fi
trigger-docker-build:
name: Trigger Docker build workflow
needs: [publish-pypi]
timeout-minutes: 10
runs-on: ubuntu-20.04
if: contains(github.ref, 'refs/tags/')
steps:
- name: check if a version tag
id: check-version-tag
run: |
if [[ ${{ github.event.ref }} =~ ^refs/tags/v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
echo ::set-output name=match::true
echo ::set-output name=versiontag::$(echo $GITHUB_REF | cut -d / -f 3)
fi
- name: Repository Dispatch
if: steps.check-version-tag.outputs.match == 'true'
uses: peter-evans/repository-dispatch@v2
with:
token: ${{ secrets.GITHUB_TOKEN }}
repository: sodadata/soda-core
event-type: started-from-version-tag
client-payload: '{"tag": "${{ steps.check-version-tag.outputs.versiontag }}", "ref": "${{ github.ref }}"}'