remove sparksession code that is not compliant with spark connect (#186) #138
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Unit tests | |
| on: | |
| push: | |
| branches: | |
| - main | |
| workflow_dispatch: | |
| jobs: | |
| test: | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - pyspark-version: 2.4.8 # latest published 2.x version | |
| pip-packages: "pypandoc==1.7 pyspark==2.4.8" # downgrade of pypandoc necessary | |
| - pyspark-version: 3.0.3 | |
| pip-packages: "pyspark==3.0.3" | |
| - pyspark-version: 3.1.3 | |
| pip-packages: "pyspark==3.1.3" | |
| - pyspark-version: 3.2.4 | |
| pip-packages: "pyspark==3.2.4" | |
| - pyspark-version: 3.3.2 | |
| pip-packages: "pyspark==3.3.2" | |
| - pyspark-version: 3.4.0 | |
| pip-packages: "pyspark==3.4.0" | |
| steps: | |
| - uses: actions/checkout@v1 | |
| with: | |
| fetch-depth: 1 | |
| - name: Setup Java | |
| uses: actions/setup-java@v3 | |
| with: | |
| distribution: 'zulu' | |
| java-version: '8' # Supported by Spark 2.x & 3.x | |
| - name: Get supported Python Version depending on PySpark | |
| uses: haya14busa/action-cond@v1 | |
| id: python_version | |
| with: | |
| cond: ${{ startsWith(matrix.pyspark-version, '2.') }} | |
| if_true: '3.7' # latest supported version for PySpark 2.x | |
| if_false: '3.9' # PySpark 3+ | |
| - name: Set up Python ${{ steps.python_version.outputs.value }} | |
| uses: actions/setup-python@v2 | |
| with: | |
| python-version: ${{ steps.python_version.outputs.value }} | |
| - name: Get supported Poetry version | |
| uses: haya14busa/action-cond@v1 | |
| id: poetry_version | |
| with: | |
| cond: ${{ startsWith(matrix.pyspark-version, '2.') }} | |
| if_true: '1.5.1' # latest supported version for PySpark 2.x | |
| if_false: '1.6.1' # PySpark 3+ | |
| - name: Install Poetry | |
| uses: snok/install-poetry@v1 | |
| with: | |
| version: ${{ steps.poetry_version.outputs.value }} | |
| - name: Cache Poetry virtualenv | |
| uses: actions/cache@v1 | |
| id: cache | |
| with: | |
| path: ~/.virtualenvs | |
| key: poetry-${{ hashFiles('**/poetry.lock') }} | |
| restore-keys: | | |
| poetry-${{ hashFiles('**/poetry.lock') }} | |
| - name: Install dependencies | |
| run: make install_test | |
| if: steps.cache.outputs.cache-hit != 'true' | |
| - name: Change PySpark to version ${{ matrix.pyspark-version }} | |
| env: | |
| PIP_PACKAGES: ${{ matrix.pip-packages }} | |
| run: poetry run pip install $PIP_PACKAGES # Using pip shouldn't mess up poetry cache | |
| - name: Run tests with pytest against PySpark ${{ matrix.pyspark-version }} | |
| run: make test |