ci.yaml

name: Build and test astro
on:
  push:
    branches: [ 'main' ]
  pull_request:
    branches: [ 'main' ]
  # Run on PRs from forks
  pull_request_target:
    branches: [ 'main' ]
    types: ['labeled']
jobs:
  Pre-Commit:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3
      - uses: actions/setup-python@v3
        with:
          python-version: '3.9'
          architecture: 'x64'
      - uses: actions/cache@v3
        with:
          path: ~/.cache/pip
          key: ${{ hashFiles('pyproject.toml') }}
      - uses: pre-commit/action@v2.0.3

  Markdown-link-check:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3
      - uses: gaurav-nelson/github-action-markdown-link-check@v1
        with:
          config-file: '.github/workflows/mlc_config.json'

  Type-Check:
    runs-on: ubuntu-latest
    env:
      MYPY_FORCE_COLOR: 1
      TERM: xterm-color
      SETUPTOOLS_USE_DISTUTILS: stdlib
    steps:
      - uses: actions/checkout@v3
      - uses: actions/setup-python@v3
        with:
          python-version: '3.8'
          architecture: 'x64'
      - uses: actions/cache@v3
        with:
           path: |
             ~/.cache/pip
             .nox
           key: ${{ runner.os }}-${{ hashFiles('pyproject.toml') }}
      - run: pip3 install nox
      - run: nox -s type_check

  Build-Docs:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3
      - uses: actions/setup-python@v3
        with:
          python-version: '3.9'
          architecture: 'x64'
      - uses: actions/cache@v3
        with:
          path: |
            ~/.cache/pip
            .nox
          key: ${{ runner.os }}-${{ hashFiles('pyproject.toml') }}
      - run: pip3 install nox
      - run: nox -s build_docs

  Run-Optional-Packages-tests:
    if: >-
      github.event_name == 'push' ||
      (
        github.event_name == 'pull_request' &&
        github.event.pull_request.head.repo.fork == false
      ) ||
      (
        github.event_name == 'pull_request_target' &&
        contains(github.event.pull_request.labels.*.name, 'safe to test')
      )
    runs-on: ubuntu-latest
    services:
      postgres:
        image: dimberman/pagila-test
        env:
          POSTGRES_PASSWORD: postgres
        options: >-
          --health-cmd pg_isready
          --health-interval 10s
          --health-timeout 5s
          --health-retries 5
          --name postgres
        ports:
          - 5432:5432
    steps:
      - uses: actions/checkout@v3
        if: github.event_name != 'pull_request_target'

      - name: Checkout pull/${{ github.event.number }}
        uses: actions/checkout@v3
        with:
          ref: ${{ github.event.pull_request.head.sha }}
        if: github.event_name == 'pull_request_target'

      - uses: actions/setup-python@v3
        with:
          python-version: '3.8'
          architecture: 'x64'
      - uses: actions/cache@v3
        with:
          path: |
            ~/.cache/pip
            .nox
          key: ${{ runner.os }}-${{ hashFiles('pyproject.toml') }}
      - run: cat .github/ci-test-connections.yaml > test-connections.yaml
      - run: python -c 'import os; print(os.getenv("GOOGLE_APPLICATION_CREDENTIALS_JSON", "").strip())' > ${{ env.GOOGLE_APPLICATION_CREDENTIALS }}
      - run: sqlite3 /tmp/sqlite_default.db "VACUUM;"
      - run: pip3 install nox
      - run: nox -s test_examples_by_dependency -- --cov=src --cov-report=xml --cov-branch
      - name: Upload coverage
        uses: actions/upload-artifact@v2
        with:
          name: coverage${{ matrix.group }}
          path: .coverage
    env:
      SETUPTOOLS_USE_DISTUTILS: stdlib
      GOOGLE_APPLICATION_CREDENTIALS_JSON: ${{ secrets.GOOGLE_APPLICATION_CREDENTIALS_JSON }}
      GOOGLE_APPLICATION_CREDENTIALS: /tmp/google_credentials.json
      POSTGRES_HOST: postgres
      POSTGRES_PORT: 5432
      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
      AIRFLOW__ASTRO_SDK__SQL_SCHEMA: astroflow_ci
      SNOWFLAKE_ACCOUNT_NAME: ${{ secrets.SNOWFLAKE_UNAME }}
      SNOWFLAKE_PASSWORD: ${{ secrets.SNOWFLAKE_PASSWORD }}
      SNOWFLAKE_SCHEMA: ASTROFLOW_CI
      SNOWFLAKE_DATABASE: SANDBOX
      SNOWFLAKE_WAREHOUSE: DEMO
      SNOWFLAKE_HOST: https://gp21411.us-east-1.snowflakecomputing.com
      SNOWFLAKE_ACCOUNT: gp21411
      SNOWFLAKE_REGION: us-east-1
      SNOWFLAKE_ROLE: AIRFLOW_TEST_USER
      AIRFLOW__CORE__LOAD_DEFAULT_CONNECTIONS: True
      AIRFLOW__CORE__ENABLE_XCOM_PICKLING: True
      AIRFLOW_VAR_FOO: templated_file_name
      AIRFLOW__ASTRO_SDK__DATAFRAME_ALLOW_UNSAFE_STORAGE: True
      FORCE_COLOR: "true"
  Run-Unit-tests-Airflow-2-3:
    if: >-
      github.event_name == 'push' ||
      (
        github.event_name == 'pull_request' &&
        github.event.pull_request.head.repo.fork == false
      ) ||
      (
        github.event_name == 'pull_request_target' &&
        contains(github.event.pull_request.labels.*.name, 'safe to test')
      )
    strategy:
      fail-fast: false
      matrix:
        group: [ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12 ]
    runs-on: ubuntu-latest
    services:
      postgres:
        # Docker Hub image
        image: dimberman/pagila-test
        env:
          POSTGRES_PASSWORD: postgres
        # Set health checks to wait until postgres has started
        options: >-
          --health-cmd pg_isready
          --health-interval 10s
          --health-timeout 5s
          --health-retries 5
        ports:
          - 5432:5432
    steps:
      - uses: actions/checkout@v3
        if: github.event_name != 'pull_request_target'

      - name: Checkout pull/${{ github.event.number }}
        uses: actions/checkout@v3
        with:
          ref: ${{ github.event.pull_request.head.sha }}
        if: github.event_name == 'pull_request_target'
      - uses: actions/setup-python@v3
        with:
          python-version: '3.8'
          architecture: 'x64'
      - uses: actions/cache@v3
        with:
          path: |
            ~/.cache/pip
            .nox
          key: ${{ runner.os }}-2.3-${{ hashFiles('pyproject.toml') }}
      - run: cat .github/ci-test-connections.yaml > test-connections.yaml
      - run: python -c 'import os; print(os.getenv("GOOGLE_APPLICATION_CREDENTIALS_JSON", "").strip())' > ${{ env.GOOGLE_APPLICATION_CREDENTIALS }}
      - run: sqlite3 /tmp/sqlite_default.db "VACUUM;"
      - run: pip3 install nox
      - run: nox -s "test-3.8(airflow='2.3')" -- --splits 12 --group ${{ matrix.group }} --cov=src --cov-report=xml --cov-branch
      - name: Upload coverage
        uses: actions/upload-artifact@v2
        with:
          name: coverage${{ matrix.group }}
          path: .coverage
    env:
      AWS_BUCKET: tmp9
      GOOGLE_APPLICATION_CREDENTIALS_JSON: ${{ secrets.GOOGLE_APPLICATION_CREDENTIALS_JSON }}
      GOOGLE_APPLICATION_CREDENTIALS: /tmp/google_credentials.json
      GOOGLE_BUCKET: dag-authoring
      POSTGRES_HOST: postgres
      POSTGRES_PORT: 5432
      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
      AIRFLOW__ASTRO_SDK__SQL_SCHEMA: astroflow_ci
      SNOWFLAKE_ACCOUNT_NAME: ${{ secrets.SNOWFLAKE_UNAME }}
      SNOWFLAKE_PASSWORD: ${{ secrets.SNOWFLAKE_PASSWORD }}
      SNOWFLAKE_SCHEMA: ASTROFLOW_CI
      SNOWFLAKE_DATABASE: SANDBOX
      SNOWFLAKE_WAREHOUSE: DEMO
      SNOWFLAKE_HOST: https://gp21411.us-east-1.snowflakecomputing.com
      SNOWFLAKE_ACCOUNT: gp21411
      SNOWFLAKE_REGION: us-east-1
      SNOWFLAKE_ROLE: AIRFLOW_TEST_USER
      AIRFLOW__CORE__LOAD_DEFAULT_CONNECTIONS: True
      AIRFLOW__CORE__ENABLE_XCOM_PICKLING: True
      AIRFLOW_VAR_FOO: templated_file_name
      AIRFLOW__ASTRO_SDK__DATAFRAME_ALLOW_UNSAFE_STORAGE: True


  Run-Unit-tests-Airflow-2-2-5:
    if: >-
      github.event_name == 'push' ||
      (
        github.event_name == 'pull_request' &&
        github.event.pull_request.head.repo.fork == false
      ) ||
      (
        github.event_name == 'pull_request_target' &&
        contains(github.event.pull_request.labels.*.name, 'safe to test')
      )
    runs-on: ubuntu-latest
    needs:
      - Run-Unit-tests-Airflow-2-3
    services:
      postgres:
        # Docker Hub image
        image: dimberman/pagila-test
        env:
          POSTGRES_PASSWORD: postgres
        # Set health checks to wait until postgres has started
        options: >-
          --health-cmd pg_isready
          --health-interval 10s
          --health-timeout 5s
          --health-retries 5
        ports:
          - 5432:5432
    steps:
      - uses: actions/checkout@v3
        if: github.event_name != 'pull_request_target'

      - name: Checkout pull/${{ github.event.number }}
        uses: actions/checkout@v3
        with:
          ref: ${{ github.event.pull_request.head.sha }}
        if: github.event_name == 'pull_request_target'

      - uses: actions/setup-python@v3
        with:
          python-version: '3.8'
          architecture: 'x64'
      - uses: actions/cache@v3
        with:
          path: |
            ~/.cache/pip
            .nox
          key: ${{ runner.os }}-2.2.5-${{ hashFiles('pyproject.toml') }}
      - run: cat .github/ci-test-connections.yaml > test-connections.yaml
      - run: python -c 'import os; print(os.getenv("GOOGLE_APPLICATION_CREDENTIALS_JSON", "").strip())' > ${{ env.GOOGLE_APPLICATION_CREDENTIALS }}
      - run: sqlite3 /tmp/sqlite_default.db "VACUUM;"
      - run: pip3 install nox
      - run: nox -s "test-3.8(airflow='2.2.5')" -- "tests/test_example_dags.py" "tests/integration_test_dag.py"
    env:
      AWS_BUCKET: tmp9
      GOOGLE_APPLICATION_CREDENTIALS_JSON: ${{ secrets.GOOGLE_APPLICATION_CREDENTIALS_JSON }}
      GOOGLE_APPLICATION_CREDENTIALS: /tmp/google_credentials.json
      GOOGLE_BUCKET: dag-authoring
      POSTGRES_HOST: postgres
      POSTGRES_PORT: 5432
      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
      AIRFLOW__ASTRO_SDK__SQL_SCHEMA: astroflow_ci
      SNOWFLAKE_ACCOUNT_NAME: ${{ secrets.SNOWFLAKE_UNAME }}
      SNOWFLAKE_PASSWORD: ${{ secrets.SNOWFLAKE_PASSWORD }}
      SNOWFLAKE_SCHEMA: ASTROFLOW_CI
      SNOWFLAKE_DATABASE: SANDBOX
      SNOWFLAKE_WAREHOUSE: DEMO
      SNOWFLAKE_HOST: https://gp21411.us-east-1.snowflakecomputing.com
      SNOWFLAKE_ACCOUNT: gp21411
      SNOWFLAKE_REGION: us-east-1
      SNOWFLAKE_ROLE: AIRFLOW_TEST_USER
      AIRFLOW__CORE__LOAD_DEFAULT_CONNECTIONS: True
      AIRFLOW__CORE__ENABLE_XCOM_PICKLING: True
      AIRFLOW_VAR_FOO: templated_file_name
      AIRFLOW__ASTRO_SDK__DATAFRAME_ALLOW_UNSAFE_STORAGE: True

  Code-Coverage:
    needs:
      - Run-Unit-tests-Airflow-2-3
      - Run-Optional-Packages-tests
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3
      - name: Set up Python 3.8
        uses: actions/setup-python@v3
        with:
          python-version: 3.8
      - name: Install coverage
        run: |
          pip3 install coverage
      - name: Download all artifacts
        uses: actions/download-artifact@v2
      - name: Run coverage
        run: |
          coverage combine coverage*/.coverage*
          coverage report
          coverage xml
      - name: Upload coverage
        uses: codecov/codecov-action@v2
        with:
          fail_ci_if_error: true
          token: ${{ secrets.CODECOV_TOKEN }}
          files: ./coverage.xml