.github/workflows/check-generated-files.yml

@@ -15,6 +15,7 @@ on:
       - pyproject.toml
       - requirements.txt
       - .github/workflows/check-requirements-txt.yml
+  merge_group:
 
 concurrency:
   group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }}
@@ -31,7 +32,7 @@ jobs:
         uses: actions/setup-python@v4
         id: install_python
         with:
-          python-version: "3.10"
+          python-version: "3.11"
 
       - name: update apt-get
         run: sudo apt-get update -y -q
@@ -40,11 +41,26 @@ jobs:
         run: sudo apt-get install -y -q build-essential graphviz libgeos-dev
 
       - name: install poetry
-        run: pip install 'poetry>=1.2'
+        run: pip install 'poetry<1.4'
 
       - name: check consistency with pyproject.toml
         run: poetry lock --check
 
+      - name: check that the `all` extra matches the current lock file
+        run: |
+          set -euo pipefail
+
+          js="$(mktemp --suffix=.pyproject.json)"
+          docker run --rm -i sclevine/yj -tj < pyproject.toml > "$js"
+
+          old="$(mktemp --suffix=.old)"
+          jq -rM '.tool.poetry.extras.all | unique | sort' < "$js" > "$old"
+
+          new="$(mktemp --suffix=.new)"
+          jq -rM '.tool.poetry.extras | with_entries(select(.key != "all")) | [.[]] | add | unique | sort' < "$js" > "$new"
+
+          diff --unified "$old" "$new"
+
       - name: generate requirements.txt
         run: poetry export --extras all --with dev --with test --with docs --without-hashes --no-ansi > requirements.txt
 
@@ -54,7 +70,7 @@ jobs:
       - uses: syphar/restore-virtualenv@v1
         with:
           requirement_files: requirements.txt
-          custom_cache_key_element: check-setuptools-install-${{ steps.install_python.outputs.python-version }}
+          custom_cache_key_element: check-setuptools-install
 
       - uses: syphar/restore-pip-download-cache@v1
         with:

.github/workflows/conda-lock.yml

@@ -15,6 +15,7 @@ jobs:
           - "3.8"
           - "3.9"
           - "3.10"
+          - "3.11"
     defaults:
       run:
         shell: bash -l {0}
@@ -91,7 +92,6 @@ jobs:
           body: "Relock conda-lock environment files"
           labels: |
             dependencies
-            autorebase:opt-in
 
       - uses: juliangruber/approve-pull-request-action@v2.0.3
         if: steps.create_pr.outputs.pull-request-operation == 'created'

.github/workflows/ibis-backends-cloud.yml

@@ -18,6 +18,9 @@ concurrency:
   group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }}
   cancel-in-progress: true
 
+env:
+  FORCE_COLOR: "1"
+
 jobs:
   test_backends:
     name: ${{ matrix.backend.title }} ${{ matrix.os }} python-${{ matrix.python-version }}
@@ -28,12 +31,25 @@ jobs:
         os:
           - ubuntu-latest
         python-version:
-          - "3.10"
+          - "3.11"
         backend:
-          - name: snowflake
-            title: Snowflake
           - name: bigquery
             title: BigQuery
+            randomly-seed: true
+
+          - name: snowflake
+            title: Snowflake
+            randomly-seed: false
+
+          - name: snowflake
+            key: snowflake-pyarrow
+            title: Snowflake + PyArrow
+            randomly-seed: false
+            deps:
+              - "snowflake-connector-python[pandas]"
+              - "--optional"
+              - "snowflake-sqlalchemy[pandas]"
+              - "--optional"
     steps:
       - name: checkout
         uses: actions/checkout@v3
@@ -44,7 +60,21 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
 
-      - run: python -m pip install --upgrade pip 'poetry>=1.2'
+      - uses: syphar/restore-virtualenv@v1
+        with:
+          requirement_files: poetry.lock
+          custom_cache_key_element: ${{ matrix.backend.name }}
+
+      - run: python -m pip install --upgrade pip 'poetry<1.4'
+
+      - name: modify deps
+        if: matrix.backend.deps != null
+        run: poetry add ${{ join(matrix.backend.deps, ' ') }}
+
+      - uses: syphar/restore-pip-download-cache@v1
+        with:
+          requirement_files: poetry.lock
+          custom_cache_key_element: ${{ matrix.backend.name }}-${{ steps.install_python.outputs.python-version }}
 
       - name: install ibis
         run: poetry install --without dev --without docs --extras ${{ matrix.backend.name }}
@@ -67,18 +97,23 @@ jobs:
         env:
           SNOWFLAKE_URL: ${{ secrets.SNOWFLAKE_URL }}
 
-      - name: "run parallel tests: ${{ matrix.backend.name }}"
+      - name: "run parallel tests: ${{ matrix.backend.name }}; reset random seed"
+        if: matrix.backend.randomly-seed
         run: just ci-check -m ${{ matrix.backend.name }} --numprocesses auto --dist=loadgroup
 
+      - name: "run parallel tests: ${{ matrix.backend.name }}; don't reset random seed"
+        if: ${{ !matrix.backend.randomly-seed }}
+        run: just ci-check -m ${{ matrix.backend.name }} --numprocesses auto --dist=loadgroup --randomly-dont-reset-seed
+
       - name: upload code coverage
         if: success()
         uses: codecov/codecov-action@v3
         with:
-          flags: backend,${{ matrix.backend.name }},${{ runner.os }},python-${{ steps.install_python.outputs.python-version }}
+          flags: backend,${{ matrix.backend.key || matrix.backend.name }},${{ runner.os }},python-${{ steps.install_python.outputs.python-version }}
 
       - name: publish test report
         uses: actions/upload-artifact@v3
         if: success() || failure()
         with:
-          name: ${{ matrix.backend.name }}-${{ matrix.os }}-${{ matrix.python-version }}
+          name: ${{ matrix.backend.key || matrix.backend.name }}-${{ matrix.os }}-${{ matrix.python-version }}
           path: junit.xml

.github/workflows/ibis-backends-skip-helper.yml

@@ -21,8 +21,21 @@ on:
     branches:
       - master
       - "*.x.x"
+  merge_group:
 jobs:
-  backends:
+  test_backends:
+    runs-on: ubuntu-latest
+    steps:
+      - run: echo "No build required"
+  test_backends_min_version:
     runs-on: ubuntu-latest
     steps:
       - run: echo "No build required"
+  backends:
+    # this job exists so that we can use a single job from this workflow to gate merging
+    runs-on: ubuntu-latest
+    needs:
+      - test_backends_min_version
+      - test_backends
+    steps:
+      - run: exit 0