This repository has been archived by the owner on Mar 11, 2024. It is now read-only.
feat: add lazyframe support for parquet and delta #170
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} | |
| cancel-in-progress: true | |
| on: | |
| workflow_dispatch: | |
| push: | |
| branches: | |
| - main | |
| - master | |
| pull_request: | |
| branches: | |
| - main | |
| - master | |
| release: | |
| types: | |
| - created | |
| # Sequence of patterns matched against refs/tags | |
| tags: | |
| - 'v*' # Push events to matching v*, i.e. v1.0, v20.15.10 | |
| jobs: | |
| test: | |
| name: test polars=${{ matrix.polars_version }} dagster=${{ matrix.dagster_version }} py=${{ matrix.py }} ${{ matrix.os }} | |
| runs-on: ${{ matrix.os }}-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: | |
| - Ubuntu | |
| # MacOs is commented out because: 1. For some reason it's very slow. 2. It never had OS-specific issues | |
| # - MacOs | |
| py: | |
| - "3.11" | |
| - "3.10" | |
| - "3.9" | |
| - "3.8" | |
| dagster_version: | |
| - "1.5.1" | |
| - "1.6.0" | |
| polars_version: | |
| - "0.18.0" | |
| - "0.19.0" | |
| - "0.20.6" | |
| steps: | |
| - name: Setup python for test ${{ matrix.py }} | |
| uses: actions/setup-python@v2 | |
| with: | |
| python-version: ${{ matrix.py }} | |
| - uses: actions/checkout@v2 | |
| - name: Install and configure Poetry | |
| uses: snok/install-poetry@v1 | |
| with: | |
| version: 1.4.2 | |
| virtualenvs-create: false | |
| virtualenvs-in-project: false | |
| installer-parallel: true | |
| - name: Install dependencies | |
| run: poetry install --all-extras --sync | |
| - name: Install matrix overrides | |
| run: poetry add polars=~${{ matrix.polars_version }} dagster=~${{ matrix.dagster_version }} && poetry lock && poetry install --sync --all-extras | |
| - name: Print polars info | |
| run: python -c 'import polars; print(polars.show_versions())' | |
| - name: Print dagster info | |
| run: python -c 'import dagster; print(dagster.__version__)' | |
| - name: Run tests | |
| run: pytest -v . | |
| lint: | |
| name: lint polars=${{ matrix.polars_version }} dagster=${{ matrix.dagster_version }} py=${{ matrix.py }} ${{ matrix.os }} | |
| runs-on: ${{ matrix.os }}-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: | |
| - Ubuntu | |
| py: | |
| - "3.11" | |
| - "3.10" | |
| - "3.9" | |
| - "3.8" | |
| dagster_version: | |
| - "1.5.1" | |
| - "1.6.0" | |
| polars_version: | |
| - "0.18.0" | |
| - "0.19.0" | |
| - "0.20.6" | |
| steps: | |
| - name: Setup python for test ${{ matrix.py }} | |
| uses: actions/setup-python@v2 | |
| with: | |
| python-version: ${{ matrix.py }} | |
| - uses: actions/checkout@v2 | |
| - name: Install and configure Poetry | |
| uses: snok/install-poetry@v1 | |
| with: | |
| version: 1.4.2 | |
| virtualenvs-create: false | |
| virtualenvs-in-project: false | |
| installer-parallel: true | |
| - name: Install dependencies | |
| run: poetry install --all-extras --sync | |
| - name: Install matrix overrides | |
| run: poetry add polars=~${{ matrix.polars_version }} dagster=~${{ matrix.dagster_version }} && poetry lock && poetry install --sync --all-extras | |
| - name: Run pre-commit hooks | |
| run: pre-commit run --all-files | |
| qa_success: | |
| name: QA Success | |
| needs: [test, lint] | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Success | |
| run: echo "QA Succeeded!" | |
| publish: | |
| needs: qa_success | |
| if: contains(github.ref, 'refs/tags/') | |
| name: Publish to PyPI | |
| strategy: | |
| fail-fast: false | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Setup Python | |
| uses: actions/setup-python@v2 | |
| with: | |
| python-version: 3.9 | |
| - uses: actions/checkout@v2 | |
| - name: Install and configure Poetry | |
| uses: snok/install-poetry@v1 | |
| with: | |
| version: 1.4.2 | |
| virtualenvs-create: false | |
| virtualenvs-in-project: false | |
| installer-parallel: true | |
| - name: Install poetry-dynamic-versioning | |
| run: poetry self add poetry-dynamic-versioning@0.21.4 | |
| - name: Publish to PyPI | |
| env: | |
| PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }} | |
| run: | | |
| poetry config pypi-token.pypi $PYPI_TOKEN | |
| poetry publish --build |