.commitlintrc.json

@@ -21,6 +21,7 @@
         "build",
         "chore",
         "ci",
+        "depr",
         "docs",
         "feat",
         "fix",
@@ -52,6 +53,11 @@
             "title": "Documentation",
             "emoji": "📚"
           },
+          "depr": {
+            "description": "Deprecated APIs and behaviors",
+            "title": "Deprecations",
+            "emoji": "⛔"
+          },
           "style": {
             "description": "Changes that do not affect the meaning of the code (whitespace, formatting, missing semi-colons, etc)",
             "title": "Styles",

.conventionalcommits.js

@@ -0,0 +1,20 @@
+"use strict";
+
+const releaseConfig = require("./.releaserc.js");
+
+function extractConventionalCommitsConfig(config) {
+  return config.plugins
+    .filter(
+      ([plugin, _]) => plugin == "@semantic-release/release-notes-generator"
+    )
+    .map(([_, config]) => config)[0].presetConfig.types;
+}
+
+module.exports = {
+  options: {
+    preset: {
+      name: "conventionalcommits",
+      types: extractConventionalCommitsConfig(releaseConfig),
+    },
+  },
+};

.github/workflows/assign.yml

@@ -8,6 +8,6 @@ jobs:
     runs-on: ubuntu-latest
     if: ${{ github.event.comment.body == '/take' }}
     steps:
-      - uses: pozil/auto-assign-issue@v1.8.0
+      - uses: pozil/auto-assign-issue@v1.10.0
         with:
           assignees: ${{ github.event.comment.user.login }}

.github/workflows/auto-rebase.yml

@@ -12,6 +12,7 @@ on:
       - labeled
 jobs:
   auto-rebase:
+    if: github.repository == 'ibis-project/ibis'
     runs-on: ubuntu-latest
     steps:
       - uses: tibdex/github-app-token@v1

.github/workflows/check-generated-files.yml

@@ -8,6 +8,7 @@ on:
       - README.md
       - poetry.lock
       - pyproject.toml
+      - requirements.txt
       - setup.py
       - .github/workflows/check-setup-py.yml
   pull_request:
@@ -17,6 +18,7 @@ on:
       - README.md
       - poetry.lock
       - pyproject.toml
+      - requirements.txt
       - setup.py
       - .github/workflows/check-setup-py.yml
 
@@ -50,8 +52,6 @@ jobs:
 
   check_setuptools_install:
     runs-on: ubuntu-latest
-    needs:
-      - check_setup_py
     steps:
       - name: checkout
         uses: actions/checkout@v3
@@ -69,7 +69,7 @@ jobs:
         run: sudo apt-get install -y -q build-essential graphviz krb5-config libkrb5-dev libgeos-dev
 
       - name: install poetry
-        run: pip install poetry
+        run: pip install 'poetry<1.2'
 
       - name: generate requirements.txt
         run: poetry export --dev --extras all --without-hashes --no-ansi > requirements.txt
@@ -108,18 +108,16 @@ jobs:
       - name: checkout
         uses: actions/checkout@v3
 
-      - name: install python
-        uses: actions/setup-python@v4
+      - name: install nix
+        uses: cachix/install-nix-action@v17
         with:
-          python-version: "3.10"
-
-      - name: install poetry
-        run: pip install poetry
+          nix_path: nixpkgs=channel:nixos-unstable-small
 
-      - name: run poetry lock
-        run: poetry lock --no-update
-        env:
-          PYTHONHASHSEED: "42"
+      - name: setup cachix
+        uses: cachix/cachix-action@v10
+        with:
+          name: ibis
+          extraPullNames: nix-community,poetry2nix
 
-      - name: check whether poetry lockfile needs to be updated
-        run: git diff --exit-code poetry.lock
+      - name: verify lockfile
+        run: ./dev/lockfile_diff.sh

.github/workflows/ci-data.yml

@@ -56,4 +56,9 @@ jobs:
           for table in workflows jobs; do
             gsuri="gs://ibis-workflow-data/${yesterday}/${table}.json"
             gsutil cp -Z "${table}.json" "${gsuri}"
+            bq load \
+              --autodetect=true \
+              --source_format=NEWLINE_DELIMITED_JSON \
+              "workflows_native.${table}" \
+              "${gsuri}"
           done

.github/workflows/ibis-backends-skip-helper.yml

@@ -6,12 +6,14 @@ on:
   push:
     paths:
       - "docs/**"
+      - "mkdocs.yml"
     branches:
       - master
       - "*.x.x"
   pull_request:
     paths:
       - "docs/**"
+      - "mkdocs.yml"
     branches:
       - master
       - "*.x.x"

.github/workflows/ibis-backends.yml

@@ -6,13 +6,15 @@ on:
     # Skip the backend suite if all changes are in the docs directory
     paths-ignore:
       - "docs/**"
+      - "mkdocs.yml"
     branches:
       - master
       - "*.x.x"
   pull_request:
     # Skip the backend suite if all changes are in the docs directory
     paths-ignore:
       - "docs/**"
+      - "mkdocs.yml"
     branches:
       - master
       - "*.x.x"
@@ -143,7 +145,7 @@ jobs:
           requirement_files: poetry.lock
           custom_cache_key_element: ${{ matrix.backend.name }}-${{ steps.install_python.outputs.python-version }}
 
-      - run: python -m pip install --upgrade pip poetry
+      - run: python -m pip install --upgrade pip 'poetry<1.2'
 
       - name: install ibis
         if: ${{ matrix.backend.name != 'postgres' }}
@@ -158,11 +160,11 @@ jobs:
       - name: download backend data
         run: just download-data
 
-      - name: run parallel ${{ matrix.backend.name }} tests
+      - name: "run parallel tests: ${{ matrix.backend.name }}"
         if: ${{ matrix.backend.name != 'pyspark' && matrix.backend.name != 'impala' }}
         run: just ci-check -m ${{ matrix.backend.name }} --numprocesses auto --dist=loadgroup
 
-      - name: run non-parallel ${{ matrix.backend.name }} tests
+      - name: "run serial tests: ${{ matrix.backend.name }}"
         if: ${{ matrix.backend.name == 'pyspark' || matrix.backend.name == 'impala' }}
         run: just ci-check -m ${{ matrix.backend.name }}
         env:
@@ -236,7 +238,7 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
 
-      - run: python -m pip install --upgrade pip poetry
+      - run: python -m pip install --upgrade pip 'poetry<1.2'
 
       - name: install minimum versions
         run: poetry add --lock --optional ${{ join(matrix.backend.deps, ' ') }}

.github/workflows/ibis-docs-lint.yml

@@ -63,6 +63,7 @@ jobs:
 
   benchmarks:
     runs-on: ubuntu-latest
+    if: ${{ github.event_name == 'push' }}
     steps:
       - name: checkout
         uses: actions/checkout@v3
@@ -86,25 +87,24 @@ jobs:
           requirement_files: poetry.lock
           custom_cache_key_element: benchmarks-${{ steps.install_python.outputs.python-version }}
 
-      - run: python -m pip install --upgrade pip poetry
+      - run: python -m pip install --upgrade pip 'poetry<1.2'
 
       - name: install ibis
         run: poetry install --extras all
 
-      - run: mkdir .benchmarks
+      - name: make benchmark output dir
+        run: mkdir .benchmarks
 
       - name: benchmark
-        run: poetry run pytest --benchmark-only --benchmark-json .benchmarks/output.json ibis/tests/benchmarks
+        run: poetry run pytest --benchmark-enable --benchmark-json .benchmarks/output.json ibis/tests/benchmarks
 
       - uses: tibdex/github-app-token@v1
-        if: ${{ github.event_name == 'push' }}
         id: generate-token
         with:
           app_id: ${{ secrets.SQUAWK_BOT_APP_ID }}
           private_key: ${{ secrets.SQUAWK_BOT_APP_PRIVATE_KEY }}
 
       - uses: benchmark-action/github-action-benchmark@v1
-        if: ${{ github.event_name == 'push' }}
         with:
           tool: pytest
           github-token: ${{ steps.generate-token.outputs.token }}
@@ -114,13 +114,10 @@ jobs:
           comment-on-alert: true
           alert-threshold: "300%"
 
-  docs:
+  docs_pr:
     runs-on: ubuntu-latest
+    if: ${{ github.event_name != 'push' }}
     concurrency: docs-${{ github.repository }}-${{ github.head_ref || github.sha }}
-    needs:
-      # wait on benchmarks to prevent a race condition when pushing to the
-      # gh-pages branch
-      - benchmarks
     steps:
       - name: install nix
         uses: cachix/install-nix-action@v17
@@ -136,43 +133,57 @@ jobs:
 
       - name: checkout
         uses: actions/checkout@v3
-        if: ${{ github.event_name != 'push' }}
         with:
           fetch-depth: 0
 
+      - name: build docs
+        run: nix run -f nix ibisDevEnv310 -- -m mkdocs build
+
+      - name: verify internal links
+        run: nix shell -f nix --ignore-environment bash findutils just lychee -c just checklinks --offline --no-progress
+
+  docs_push:
+    runs-on: ubuntu-latest
+    if: ${{ github.event_name == 'push' }}
+    concurrency: docs-${{ github.repository }}-${{ github.head_ref || github.sha }}
+    needs:
+      # wait on benchmarks to prevent a race condition when pushing to the
+      # gh-pages branch
+      - benchmarks
+    steps:
+      - name: install nix
+        uses: cachix/install-nix-action@v17
+        with:
+          nix_path: nixpkgs=channel:nixos-unstable-small
+
+      - name: setup cachix
+        uses: cachix/cachix-action@v10
+        with:
+          name: ibis
+          authToken: ${{ secrets.CACHIX_AUTH_TOKEN }}
+          extraPullNames: nix-community,poetry2nix
+
       - name: Generate a GitHub token
         uses: tibdex/github-app-token@v1
-        if: ${{ github.event_name == 'push' }}
         id: generate_token
         with:
           app_id: ${{ secrets.DOCS_BOT_APP_ID }}
           private_key: ${{ secrets.DOCS_BOT_APP_PRIVATE_KEY }}
 
       - name: checkout
         uses: actions/checkout@v3
-        if: ${{ github.event_name == 'push' }}
         with:
           fetch-depth: 0
           token: ${{ steps.generate_token.outputs.token }}
 
-      - name: build docs
-        if: ${{ github.event_name != 'push' }}
-        run: nix run -f nix ibisDevEnv310 -- -m mkdocs build
-
-      - name: verify internal links
-        if: ${{ github.event_name != 'push' }}
-        run: nix shell -f nix --ignore-environment bash findutils just lychee -c just checklinks --offline --no-progress
-
       - name: Configure git info
-        if: ${{ github.event_name == 'push' }}
         run: |
           set -euo pipefail
 
           git config user.name 'ibis-docs-bot[bot]'
           git config user.email 'ibis-docs-bot[bot]@users.noreply.github.com'
 
       - name: build and push dev docs
-        if: ${{ github.event_name == 'push' }}
         run: |
           set -euo pipefail
 
@@ -203,5 +214,18 @@ jobs:
           authToken: ${{ secrets.CACHIX_AUTH_TOKEN }}
           extraPullNames: nix-community,poetry2nix
 
+      - name: Configure git info
+        run: |
+          set -euo pipefail
+
+          # not incredibly important what user we use here
+          #
+          # we're making a commit in a temporary worktree that is thrown away
+          # if the process exits successfully
+          #
+          # git requires user information to make commits
+          git config user.name 'ibis-squawk-bot[bot]'
+          git config user.email 'ibis-squawk-bot[bot]@users.noreply.github.com'
+
       - name: run semantic-release
         run: ./ci/release/dry_run.sh

.github/workflows/ibis-main-skip-helper.yml

@@ -6,12 +6,14 @@ on:
   push:
     paths:
       - "docs/**"
+      - "mkdocs.yml"
     branches:
       - master
       - "*.x.x"
   pull_request:
     paths:
       - "docs/**"
+      - "mkdocs.yml"
     branches:
       - master
       - "*.x.x"