Merge remote-tracking branch 'upstream/main' into dask-dataframe-over…

…rides
dask-contrib · Apr 10, 2024 · a25fcb9 · a25fcb9
2 parents 32f6881 + 7600f60
commit a25fcb9
Show file tree

Hide file tree

Showing 82 changed files with 1,849 additions and 1,409 deletions.
diff --git a/.github/workflows/conda.yml b/.github/workflows/conda.yml
@@ -31,7 +31,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python: ["3.8", "3.9", "3.10"]
+        python: ["3.9", "3.10", "3.11", "3.12"]
         arch: ["linux-64", "linux-aarch64"]
     steps:
       - name: Manage disk space
@@ -64,15 +64,15 @@ jobs:
           sudo chmod 600 /swapfile || true
           sudo mkswap /swapfile || true
           sudo swapon /swapfile || true
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0
       - name: Set up Python
-        uses: conda-incubator/setup-miniconda@v2.2.0
+        uses: conda-incubator/setup-miniconda@v2.3.0
         with:
           miniforge-variant: Mambaforge
           use-mamba: true
-          python-version: "3.8"
+          python-version: "3.9"
           channel-priority: strict
       - name: Install dependencies
         run: |

diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
@@ -11,7 +11,7 @@ on:
       - Cargo.toml
       - Cargo.lock
       - pyproject.toml
-      - docker/**
+      - continuous_integration/docker/**
       - .github/workflows/docker.yml
 
 # When this workflow is queued, automatically cancel any previous running
@@ -25,29 +25,29 @@ jobs:
     name: Push Docker image to Docker Hub
     runs-on: ubuntu-latest
     env:
-      DOCKER_PUSH: ${{ contains(['push', 'release'], github.event_name) && github.repository == 'dask-contrib/dask-sql' }}
+      DOCKER_PUSH: ${{ contains(fromJSON('["push", "release"]'), github.event_name) && github.repository == 'dask-contrib/dask-sql' }}
     strategy:
       fail-fast: false
       matrix:
         platform: ["linux/amd64", "linux/arm64", "linux/386"]
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Login to DockerHub
         if: ${{ fromJSON(env.DOCKER_PUSH) }}
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
       - name: Docker meta for main image
         id: docker_meta_main
-        uses: crazy-max/ghaction-docker-meta@v4
+        uses: crazy-max/ghaction-docker-meta@v5
         with:
           images: nbraun/dask-sql
       - name: Build and push main image
-        uses: docker/build-push-action@v4
+        uses: docker/build-push-action@v5
         with:
           context: .
-          file: ./docker/main.dockerfile
+          file: ./continuous_integration/docker/main.dockerfile
           build-args: DOCKER_META_VERSION=${{ steps.docker_meta_main.outputs.version }}
           platforms: ${{ matrix.platform }}
           tags: ${{ steps.docker_meta_main.outputs.tags }}
@@ -61,14 +61,14 @@ jobs:
           docker image inspect ${{ steps.docker_meta_main.outputs.tags }}
       - name: Docker meta for cloud image
         id: docker_meta_cloud
-        uses: crazy-max/ghaction-docker-meta@v4
+        uses: crazy-max/ghaction-docker-meta@v5
         with:
           images: nbraun/dask-sql-cloud
       - name: Build and push cloud image
-        uses: docker/build-push-action@v4
+        uses: docker/build-push-action@v5
         with:
           context: .
-          file: ./docker/cloud.dockerfile
+          file: ./continuous_integration/docker/cloud.dockerfile
           build-args: DOCKER_META_VERSION=${{ steps.docker_meta_main.outputs.version }}
           platforms: ${{ matrix.platform }}
           tags: ${{ steps.docker_meta_cloud.outputs.tags }}

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -25,13 +25,7 @@ jobs:
       matrix:
         target: [x86_64, aarch64]
     steps:
-      - uses: actions/checkout@v3
-      - name: Install Protoc
-        uses: arduino/setup-protoc@v1
-        if: matrix.target == 'aarch64'
-        with:
-          version: '3.x'
-          repo-token: ${{ secrets.GITHUB_TOKEN }}
+      - uses: actions/checkout@v4
       - uses: actions/setup-python@v4
         with:
           python-version: '3.10'
@@ -43,11 +37,6 @@ jobs:
           args: --release --out dist
           sccache: 'true'
           manylinux: '2_17'
-          before-script-linux: >
-            DOWNLOAD_URL=$(curl --retry 6 --retry-delay 10 -s https://api.github.com/repos/protocolbuffers/protobuf/releases/latest | grep -o '"browser_download_url": "[^"]*' | cut -d'"' -f4 | grep "\linux-x86_64.zip$") &&
-            curl --retry 6 --retry-delay 10 -LO $DOWNLOAD_URL &&
-            unzip protoc-*-linux-x86_64.zip -d $HOME/.local
-          docker-options: --env PROTOC=/root/.local/bin/protoc
       - name: Build wheels for aarch64
         if: matrix.target == 'aarch64'
         uses: PyO3/maturin-action@v1
@@ -78,12 +67,7 @@ jobs:
     name: Build and publish wheels for windows
     runs-on: windows-latest
     steps:
-      - uses: actions/checkout@v3
-      - name: Install Protoc
-        uses: arduino/setup-protoc@v1
-        with:
-          version: '3.x'
-          repo-token: ${{ secrets.GITHUB_TOKEN }}
+      - uses: actions/checkout@v4
       - uses: actions/setup-python@v4
         with:
           python-version: '3.10'
@@ -120,12 +104,7 @@ jobs:
       matrix:
         target: [x86_64, aarch64]
     steps:
-      - uses: actions/checkout@v3
-      - name: Install Protoc
-        uses: arduino/setup-protoc@v1
-        with:
-          version: '3.x'
-          repo-token: ${{ secrets.GITHUB_TOKEN }}
+      - uses: actions/checkout@v4
       - uses: actions/setup-python@v4
         with:
           python-version: '3.10'
@@ -156,7 +135,7 @@ jobs:
   sdist:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Build sdist
         uses: PyO3/maturin-action@v1
         with:

diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml
@@ -23,26 +23,21 @@ jobs:
     outputs:
       triggered: ${{ steps.detect-trigger.outputs.trigger-found }}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 2
       - uses: xarray-contrib/ci-trigger@v1.2
         id: detect-trigger
         with:
           keyword: "[test-df-upstream]"
-      - name: Install Protoc
-        uses: arduino/setup-protoc@v1
-        with:
-          version: '3.x'
-          repo-token: ${{ secrets.GITHUB_TOKEN }}
 
   # Check crate compiles
   linux-build-lib:
     name: cargo check
     needs: [detect-ci-trigger]
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Cache Cargo
         uses: actions/cache@v3
         with:
@@ -52,11 +47,6 @@ jobs:
         if: needs.detect-ci-trigger.outputs.triggered == 'true'
         run: |
           bash continuous_integration/scripts/update-dependencies.sh
-      - name: Install Protoc
-        uses: arduino/setup-protoc@v1
-        with:
-          version: '3.x'
-          repo-token: ${{ secrets.GITHUB_TOKEN }}
       - name: Check workspace in debug mode
         run: |
           cargo check
@@ -70,7 +60,7 @@ jobs:
     needs: [detect-ci-trigger]
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           submodules: true
       - name: Cache Cargo
@@ -82,11 +72,6 @@ jobs:
         if: needs.detect-ci-trigger.outputs.triggered == 'true'
         run: |
           bash continuous_integration/scripts/update-dependencies.sh
-      - name: Install Protoc
-        uses: arduino/setup-protoc@v1
-        with:
-          version: '3.x'
-          repo-token: ${{ secrets.GITHUB_TOKEN }}
       - name: Run tests
         run: |
           cargo test
diff --git a/.github/workflows/style.yml b/.github/workflows/style.yml
@@ -13,13 +13,8 @@ jobs:
     name: Run pre-commit hooks
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: actions/setup-python@v4
-      - name: Install Protoc
-        uses: arduino/setup-protoc@v1
-        with:
-          version: '3.x'
-          repo-token: ${{ secrets.GITHUB_TOKEN }}
       - uses: actions-rs/toolchain@v1
         with:
             toolchain: nightly

diff --git a/.github/workflows/test-upstream.yml b/.github/workflows/test-upstream.yml
@@ -1,85 +1,57 @@
 name: Nightly upstream testing
 on:
   schedule:
-    - cron: "0 0 * * *" # Daily “At 00:00” UTC for upstream dask testing
-    - cron: "0 3 * * *" # Daily "At 03:00" UTC for upstream datafusion testing
+    - cron: "0 0 * * *" # Daily “At 00:00” UTC
   workflow_dispatch: # allows you to trigger the workflow run manually
-    inputs:
-      which_upstream:
-        type: choice
-        description: 'Library to update for upstream testing'
-        required: false
-        default: 'Dask'
-        options:
-          - Dask
-          - DataFusion
 
 # Required shell entrypoint to have properly activated conda environments
 defaults:
   run:
     shell: bash -l {0}
 
-env:
-  which_upstream: >-
-    ${{
-      github.event_name == 'workflow_dispatch'
-      && inputs.which_upstream
-      || (
-        github.event.schedule == '0 0 * * *'
-        && 'Dask'
-        || 'DataFusion'
-      )
-    }}
-
 jobs:
   test-dev:
     name: "Test upstream dev (${{ matrix.os }}, python: ${{ matrix.python }}, distributed: ${{ matrix.distributed }})"
     runs-on: ${{ matrix.os }}
     env:
-      CONDA_FILE: continuous_integration/environment-${{ matrix.python }}-dev.yaml
+      CONDA_FILE: continuous_integration/environment-${{ matrix.python }}.yaml
       DASK_SQL_DISTRIBUTED_TESTS: ${{ matrix.distributed }}
     strategy:
       fail-fast: false
       matrix:
         os: [ubuntu-latest, windows-latest, macos-latest]
-        python: ["3.8", "3.9", "3.10"]
+        python: ["3.9", "3.10", "3.11", "3.12"]
         distributed: [false]
         include:
           # run tests on a distributed client
           - os: "ubuntu-latest"
-            python: "3.8"
+            python: "3.9"
             distributed: true
           - os: "ubuntu-latest"
-            python: "3.10"
+            python: "3.11"
             distributed: true
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0 # Fetch all history for all branches and tags.
       - name: Set up Python
-        uses: conda-incubator/setup-miniconda@v2.2.0
+        uses: conda-incubator/setup-miniconda@v2.3.0
         with:
           miniforge-variant: Mambaforge
           use-mamba: true
           python-version: ${{ matrix.python }}
           channel-priority: strict
           activate-environment: dask-sql
           environment-file: ${{ env.CONDA_FILE }}
-      - name: Optionally update upstream cargo dependencies
-        if: env.which_upstream == 'DataFusion'
-        run: |
-          bash continuous_integration/scripts/update-dependencies.sh
       - name: Build the Rust DataFusion bindings
         run: |
           maturin develop
       - name: Install hive testing dependencies
         if: matrix.os == 'ubuntu-latest'
         run: |
-          mamba install -c conda-forge "sasl>=0.3.1"
           docker pull bde2020/hive:2.3.2-postgresql-metastore
           docker pull bde2020/hive-metastore-postgresql:2.3.0
       - name: Install upstream dev Dask
-        if: env.which_upstream == 'Dask'
         run: |
           mamba install --no-channel-priority dask/label/dev::dask
       - name: Install pytest-reportlog
@@ -103,27 +75,14 @@ jobs:
     name: "Test importing with bare requirements and upstream dev"
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Set up Python
-        uses: conda-incubator/setup-miniconda@v2.2.0
+        uses: conda-incubator/setup-miniconda@v2.3.0
         with:
           miniforge-variant: Mambaforge
           use-mamba: true
-          # TODO: drop support for python 3.8, add support for python 3.11
-          # https://github.com/dask-contrib/dask-sql/pull/1143
           python-version: "3.9"
           channel-priority: strict
-      - name: Install Protoc
-        uses: arduino/setup-protoc@v1
-        with:
-          version: '3.x'
-          repo-token: ${{ secrets.GITHUB_TOKEN }}
-      - name: Optionally update upstream cargo dependencies
-        if: env.which_upstream == 'DataFusion'
-        env:
-          UPDATE_ALL_CARGO_DEPS: false
-        run: |
-          bash continuous_integration/scripts/update-dependencies.sh
       - name: Install dependencies and nothing else
         run: |
           pip install -e . -vv
@@ -132,7 +91,6 @@ jobs:
           pip list
           mamba list
       - name: Install upstream dev Dask
-        if: env.which_upstream == 'Dask'
         run: |
           python -m pip install git+https://github.com/dask/dask
           python -m pip install git+https://github.com/dask/distributed
@@ -152,7 +110,7 @@ jobs:
       && github.repository == 'dask-contrib/dask-sql'
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: actions/download-artifact@v3
       - name: Prepare logs & issue label
         run: |
@@ -162,12 +120,9 @@ jobs:
           else
               touch results.jsonl
           fi
-
-          # convert which_upstream to lowercase
-          echo "which_upstream_lower=${which_upstream,,}" >> $GITHUB_ENV
       - name: Open or update issue on failure
         uses: xarray-contrib/issue-from-pytest-log@v1.2.6
         with:
           log-path: results.jsonl
-          issue-title: ⚠️ Upstream CI ${{ env.which_upstream }} failed ⚠️
-          issue-label: upstream-${{ env.which_upstream_lower }}
+          issue-title: ⚠️ Upstream CI failed ⚠️
+          issue-label: upstream