main.workflow.yml

name: CI pipeline

on:
  push:
    branches:
      - main
    tags:
      - "*"
  schedule:
    - cron: "0 0 * * *"
env:
  telemetry_local_debug_mode: true

jobs:
  pre-commit:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3
      - uses: actions/setup-python@v3
        with:
            python-version: '3.11.x'
      - uses: pre-commit/action@v3.0.0
        with:
          extra_args: --all-files

  run-tests:
    runs-on: ubuntu-22.04
    strategy:
      fail-fast: false
      matrix:
        python-version:
          - "3.8"
          - "3.9"
          - "3.10"

        data-source:
          - "postgres"
          - "snowflake"
          - "bigquery"
          - "redshift"
          - "athena"
          #  - "db2"
          # - "spark"
          - "spark_df"
          - "sqlserver"
          - "mysql"
          - "duckdb"
          - "dask"
          # - "vertica"

    env:
      DATA_SOURCE: ${{ matrix.data-source }}
      PYTHON_VERSION: ${{ matrix.python-version }}
      SNOWFLAKE_HOST: ${{ secrets.SNOWFLAKE_HOST }}
      SNOWFLAKE_ACCOUNT: ${{ secrets.SNOWFLAKE_ACCOUNT }}
      SNOWFLAKE_USERNAME: ${{ secrets.SNOWFLAKE_USERNAME }}
      SNOWFLAKE_PASSWORD: ${{ secrets.SNOWFLAKE_PASSWORD }}
      SNOWFLAKE_DATABASE: ${{ secrets.SNOWFLAKE_DATABASE }}
      SNOWFLAKE_SCHEMA: "public"
      BIGQUERY_ACCOUNT_INFO_JSON: ${{ secrets.BIGQUERY_ACCOUNT_INFO_JSON }}
      BIGQUERY_DATASET: "test"
      REDSHIFT_HOST: ${{ secrets.REDSHIFT_HOST }}
      REDSHIFT_USERNAME: ${{ secrets.REDSHIFT_USERNAME }}
      REDSHIFT_PASSWORD: ${{ secrets.REDSHIFT_PASSWORD }}
      REDSHIFT_DATABASE: "soda_test"
      REDSHIFT_PORT: "5439"
      ATHENA_ACCESS_KEY_ID: ${{ secrets.ATHENA_ACCESS_KEY_ID }}
      ATHENA_SECRET_ACCESS_KEY: ${{ secrets.ATHENA_SECRET_ACCESS_KEY }}
      ATHENA_S3_TEST_DIR: ${{ secrets.ATHENA_STAGING_DIR }}
      ATHENA_SCHEMA: ${{ secrets.ATHENA_SCHEMA }}
      SPARK_HOST: ${{ secrets.SPARK_HOST }}
      SPARK_USERNAME: ${{ secrets.SPARK_USERNAME }}
      SPARK_PASSWORD: ${{ secrets.SPARK_PASSWORD }}
      SPARK_DATABASE: ${{ secrets.SPARK_DATABASE }}
      SQLSERVER_USERNAME: SA
      SQLSERVER_PASSWORD: Password1!
      SQLSERVER_DATABASE: master
      SQLSERVER_SCHEMA: dbo
      MYSQL_USER: sodacore
      MYSQL_DATABASE: sodacore
      MYSQL_PASSWORD: sodacore
      MYSQL_ROOT_PASSWORD: sodacore
      # TODO: add appropriate vars
      SPARK_DF_HOST: ${{ secrets.SPARK_DF_HOST }}

    steps:
      - uses: actions/checkout@v3

      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python-version }}

      - name: Install dependencies
        run: |
          curl https://packages.microsoft.com/keys/microsoft.asc | sudo apt-key add -
          curl https://packages.microsoft.com/config/ubuntu/21.04/prod.list | sudo tee /etc/apt/sources.list.d/mssql-release.list > /dev/null
          sudo apt-get update
          ACCEPT_EULA=Y sudo apt-get install -y libsasl2-dev msodbcsql18
          python -m pip install --upgrade pip
          cat dev-requirements.in | grep tox | xargs pip install

      - name: Test with tox
        run: |
          tox -- soda -k soda/core
          tox -- soda -k soda/${{ matrix.data-source }}
        env:
          test_data_source: ${{ matrix.data-source }}

  test-cloud-integration:
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        python-version:
          - "3.8"
          - "3.9"
          - "3.10"

    env:
      PYTHON_VERSION: ${{ matrix.python-version }}
    steps:
      - uses: actions/checkout@v3

      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python-version }}

      - name: Install dependencies
        run: |
          sudo apt-get update
          sudo apt-get install -y libsasl2-dev
          python -m pip install --upgrade pip
          cat dev-requirements.in | grep tox | xargs pip install

      - name: Test with tox
        run: |
          tox -- soda -k soda/core
        env:
          test_data_source: postgres
          WESTMALLE: BETTER_THAN_LA_TRAPPE

  test-scientific:
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        python-version:
          - "3.8"
          - "3.9"
          - "3.10"

    env:
      PYTHON_VERSION: ${{ matrix.python-version }}
    steps:
      - uses: actions/checkout@v3

      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python-version }}

      - name: Install dependencies
        run: |
          sudo apt-get update
          sudo apt-get install -y libsasl2-dev
          python -m pip install --upgrade pip
          cat dev-requirements.in | grep tox | xargs pip install

      - name: Test with tox
        run: |
          tox -- soda -k soda/scientific
  test-contracts:
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        python-version:
          - "3.9"

    env:
      PYTHON_VERSION: ${{ matrix.python-version }}
    steps:
      - uses: actions/checkout@v3

      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python-version }}

      - name: Install dependencies
        run: |
          sudo apt-get update
          sudo apt-get install -y libsasl2-dev
          python -m pip install --upgrade pip
          cat dev-requirements.in | grep tox | xargs pip install

      - name: Test with tox
        run: |
          tox -- soda -k soda/contracts

  publish-pypi:
    name: Build & Publish Package
    if: contains(github.ref, 'refs/tags/')
    timeout-minutes: 10
    runs-on: ubuntu-20.04
    needs: [run-tests]
    env:
      REPORTS_URL: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}
      TWINE_USERNAME: __token__
      TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }}
    steps:
      - uses: actions/checkout@v3
      - name: Setup Python 3.8
        uses: actions/setup-python@v4
        with:
          python-version: 3.8
      - name: Setup dependencies
        run: pip install --upgrade setuptools wheel twine
      - name: Get version
        id: get_version
        run: echo ::set-output name=VERSION::${GITHUB_REF/refs\/tags\//}
      - name: Build and publish packages
        # Hacky version for now :)
        run: |
          FAILED=false
          cd soda
          for pack in *
          do
            pushd .
            cd $pack

            echo "| Building $pack"
            python3 setup.py sdist bdist_wheel

            echo "| Uploading $pack to pypi"
            UPLOAD_FAILED=false
            twine upload dist/* || UPLOAD_FAILED=true
            if [ $UPLOAD_FAILED = true ]; then
              FAILED=true
              echo "Failed to upload $pack"
            fi

            popd
          done

          if [ $FAILED = true ]; then
            echo "There was an error, check the logs please."
            exit 1
          fi
  trigger-docker-build:
    name: Trigger Docker build workflow
    needs: [publish-pypi]
    timeout-minutes: 10
    runs-on: ubuntu-20.04
    if: contains(github.ref, 'refs/tags/')
    steps:
      - name: check if a version tag
        id: check-version-tag
        run: |
          if [[ ${{ github.event.ref }} =~ ^refs/tags/v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
              echo ::set-output name=match::true
              echo ::set-output name=versiontag::$(echo $GITHUB_REF | cut -d / -f 3)
          fi
      - name: Repository Dispatch
        if: steps.check-version-tag.outputs.match == 'true'
        uses: peter-evans/repository-dispatch@v2
        with:
          token: ${{ secrets.GITHUB_TOKEN }}
          repository: sodadata/soda-core
          event-type: started-from-version-tag
          client-payload: '{"tag": "${{ steps.check-version-tag.outputs.versiontag }}", "ref": "${{ github.ref }}"}'