.coveragerc

@@ -8,6 +8,7 @@ exclude_lines =
     pragma: no cover
     raise NotImplementedError
     if __name__ == .__main__.:
+    if TYPE_CHECKING:
 ignore_errors = True
 omit =
     *_version.py

.envrc

@@ -1,3 +1,3 @@
-use nix --keep-going
+use nix
 watch_file poetry.lock
 eval "$shellHook"

.git-blame-ignore-revs

@@ -0,0 +1,3 @@
+# convert imports to absolute
+5332737fefbef3eac1063d93034d7ce87ba1c01b
+bb025103420741836cf594aa8163962d03b71b7b

.gitattributes

@@ -3,3 +3,4 @@ nix/sources.json linguist-generated=true
 nix/sources.nix linguist-generated=true
 poetry.lock linguist-generated=true
 setup.py linguist-generated=true
+docs/**/*.ipynb linguist-generated=true

.gitconfig

@@ -0,0 +1,2 @@
+[blame]
+    ignoreRevsFile = .git-blame-ignore-revs

.github/renovate.json

@@ -4,7 +4,7 @@
   "schedule": ["after 10pm and before 5am every weekday", "every weekend"],
   "semanticCommits": "enabled",
   "lockFileMaintenance": { "enabled": true },
-  "enabledManagers": ["github-actions", "poetry"],
+  "enabledManagers": ["docker-compose", "github-actions", "poetry"],
   "automerge": false,
   "labels": ["dependencies"]
 }

.github/workflows/assign.yml

@@ -8,6 +8,6 @@ jobs:
     runs-on: ubuntu-latest
     if: ${{ github.event.comment.body == '/take' }}
     steps:
-      - uses: pozil/auto-assign-issue@v1.1.0
+      - uses: pozil/auto-assign-issue@v1.4.0
         with:
           assignees: ${{ github.event.comment.user.login }}

.github/workflows/check-setup-py.yml → .github/workflows/check-generated-files.yml

@@ -1,4 +1,4 @@
-name: Check setup.py
+name: Check Generated Files
 
 on:
   push:
@@ -29,10 +29,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
 
       - name: install nix
-        uses: cachix/install-nix-action@v16
+        uses: cachix/install-nix-action@v17
         with:
           nix_path: nixpkgs=channel:nixos-unstable-small
 
@@ -54,16 +54,19 @@ jobs:
       - check_setup_py
     steps:
       - name: checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
 
       - name: install python
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v3
         id: install_python
         with:
           python-version: "3.9"
 
+      - name: update apt-get
+        run: sudo apt-get update -y -q
+
       - name: install system dependencies
-        run: sudo apt-get install -y -q build-essential cmake krb5-config python-dev libkrb5-dev libboost-all-dev graphviz
+        run: sudo apt-get install -y -q build-essential graphviz krb5-config libkrb5-dev
 
       - name: install poetry
         run: pip install poetry
@@ -87,17 +90,23 @@ jobs:
       - name: install ibis in development mode
         run: pip install -e .
 
-      - name: run simple tests
-        run: pytest ibis/tests --numprocesses auto
+      - name: run tests
+        run: pytest -m core --cov-report=xml:coverage.xml --cov=ibis
+
+      - name: upload code coverage
+        if: success()
+        uses: codecov/codecov-action@v3
+        with:
+          flags: core,${{ runner.os }},python-${{ steps.install_python.outputs.python-version }}
 
   check_poetry_lock:
     runs-on: ubuntu-latest
     steps:
       - name: checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
 
       - name: install python
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v3
         with:
           python-version: "3.x"
 

.github/workflows/ci-data.yml

@@ -53,5 +53,7 @@ jobs:
 
           yesterday="${{ steps.set_date.outputs.yesterday }}"
 
-          gsutil cp -Z workflows.json "gs://ibis-workflow-data/${yesterday}/workflows.json"
-          gsutil cp -Z jobs.json "gs://ibis-workflow-data/${yesterday}/jobs.json"
+          for table in workflows jobs; do
+            gsuri="gs://ibis-workflow-data/${yesterday}/${table}.json"
+            gsutil cp -Z "${table}.json" "${gsuri}"
+          done

.github/workflows/codeql-analysis.yml

@@ -23,7 +23,7 @@ jobs:
           - python
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - uses: github/codeql-action/init@v1
         with:
           languages: ${{ matrix.language }}

.github/workflows/conda-lock-dispatch.yml

@@ -17,7 +17,7 @@ jobs:
           app_id: ${{ secrets.SQUAWK_BOT_APP_ID }}
           private_key: ${{ secrets.SQUAWK_BOT_APP_PRIVATE_KEY }}
 
-      - uses: peter-evans/slash-command-dispatch@v2
+      - uses: peter-evans/slash-command-dispatch@v3
         with:
           token: ${{ steps.generate_token.outputs.token }}
           reaction-token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/conda-lock.yml

@@ -1,7 +1,11 @@
 # vim: filetype=yaml
-name: Relock conda environment files
+name: Generate Conda Lockfiles
 
 on:
+  schedule:
+    # At minute 30 past every 24th hour
+    - cron: "30 */24 * * *"
+  workflow_dispatch:
   repository_dispatch:
     types:
       - condalock-command
@@ -12,15 +16,15 @@ jobs:
     strategy:
       matrix:
         python-version:
-          - "3.7"
           - "3.8"
           - "3.9"
+          - "3.10"
     defaults:
       run:
         shell: bash -l {0}
     steps:
       - name: checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
         with:
           repository: ${{ github.event.client_payload.pull_request.head.repo.full_name }}
           ref: ${{ github.event.client_payload.pull_request.head.ref }}
@@ -35,9 +39,10 @@ jobs:
           condarc-file: ci/condarc
 
       - name: install conda-lock
-        run: mamba install conda-lock
+        run: mamba install 'conda-lock <1.0'
 
       - name: generate lock file
+        continue-on-error: ${{ matrix.python-version == '3.10' }}
         run: |
           set -euo pipefail
 
@@ -49,55 +54,125 @@ jobs:
             echo '  - python=${{ matrix.python-version }}'
           } > "${python_version_file}"
 
+          template='conda-lock/{platform}-${{ matrix.python-version }}.lock'
           conda lock \
+            --kind explicit \
             --file pyproject.toml \
             --file "${python_version_file}" \
             --platform linux-64 \
             --platform osx-64 \
+            --filename-template "${template}" \
+            --extras all \
+            --mamba
+
+          # not great, but conda-forge is missing packages for duckdb and
+          # clickhouse-cityhash for windows
+          conda lock \
+            --kind explicit \
+            --file pyproject.toml \
+            --file "${python_version_file}" \
             --platform win-64 \
-            --filename-template 'conda-lock/{platform}-${{ matrix.python-version }}.lock' \
-            --extras clickhouse \
-            --extras dask \
-            --extras geospatial \
-            --extras hdf5 \
-            --extras impala \
-            --extras mysql \
-            --extras parquet \
-            --extras postgres \
-            --extras pyspark \
-            --extras sqlite \
+            --filename-template "${template}" \
+            -e dask \
+            -e datafusion \
+            -e geospatial \
+            -e impala \
+            -e mysql \
+            -e pandas \
+            -e postgres \
+            -e pyspark \
+            -e sqlite \
+            -e visualization \
             --mamba
 
       - name: create conda environment
+        continue-on-error: ${{ matrix.python-version == '3.10' }}
         run: mamba create --name ibis${{ matrix.python-version }} --file conda-lock/linux-64-${{ matrix.python-version }}.lock
 
       - name: upload conda lock files
-        uses: actions/upload-artifact@v2
+        continue-on-error: ${{ matrix.python-version == '3.10' }}
+        uses: actions/upload-artifact@v3
         with:
           name: conda-lock-files-${{ github.run_attempt }}
           path: conda-lock/*-${{ matrix.python-version }}.lock
 
+  condalock_pr:
+    if: ${{ github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' }}
+    runs-on: ubuntu-latest
+    needs:
+      - condalock
+    steps:
+      - uses: tibdex/github-app-token@v1
+        id: generate_pr_token
+        with:
+          app_id: ${{ secrets.SQUAWK_BOT_APP_ID }}
+          private_key: ${{ secrets.SQUAWK_BOT_APP_PRIVATE_KEY }}
+
+      - uses: tibdex/github-app-token@v1
+        id: generate_pr_approval_token
+        with:
+          app_id: ${{ secrets.PR_APPROVAL_BOT_APP_ID }}
+          private_key: ${{ secrets.PR_APPROVAL_BOT_APP_PRIVATE_KEY }}
+
+      - uses: actions/checkout@v3
+        with:
+          token: ${{ steps.generate_pr_token.outputs.token }}
+
+      - name: download conda lock files
+        uses: actions/download-artifact@v3
+        with:
+          name: conda-lock-files-${{ github.run_attempt }}
+          path: conda-lock
+
+      - uses: peter-evans/create-pull-request@v4
+        id: create_pr
+        with:
+          token: ${{ steps.generate_pr_token.outputs.token }}
+          commit-message: "chore(conda-lock): relock"
+          branch: "create-pull-request/conda-relock"
+          delete-branch: true
+          add-paths: conda-lock/*.lock
+          committer: "ibis-squawk-bot[bot] <ibis-squawk-bot[bot]@users.noreply.github.com>"
+          author: "ibis-squawk-bot[bot] <ibis-squawk-bot[bot]@users.noreply.github.com>"
+          title: "chore(conda-lock): relock"
+          body: "Relock conda-lock environment files"
+          labels: |
+            dependencies
+            autorebase:opt-in
+
+      - uses: juliangruber/approve-pull-request-action@v1.1.1
+        if: ${{ steps.create_pr.outputs.pull-request-operation == 'created' }}
+        with:
+          github-token: ${{ steps.generate_pr_approval_token.outputs.token }}
+          number: ${{ steps.create_pr.outputs.pull-request-number }}
+
+      - uses: peter-evans/enable-pull-request-automerge@v2
+        if: ${{ steps.create_pr.outputs.pull-request-operation == 'created' }}
+        with:
+          token: ${{ steps.generate_pr_token.outputs.token }}
+          pull-request-number: ${{ steps.create_pr.outputs.pull-request-number }}
+          merge-method: rebase
+
   condalock_push:
+    if: ${{ github.event_name == 'repository_dispatch' }}
     runs-on: ubuntu-latest
     needs:
       - condalock
     steps:
-      - name: Generate a GitHub token
-        uses: tibdex/github-app-token@v1
+      - uses: tibdex/github-app-token@v1
         id: generate_token
         with:
           app_id: ${{ secrets.SQUAWK_BOT_APP_ID }}
           private_key: ${{ secrets.SQUAWK_BOT_APP_PRIVATE_KEY }}
 
-      - name: checkout
-        uses: actions/checkout@v2
+      - uses: actions/checkout@v3
         with:
           token: ${{ steps.generate_token.outputs.token }}
           repository: ${{ github.event.client_payload.pull_request.head.repo.full_name }}
           ref: ${{ github.event.client_payload.pull_request.head.ref }}
 
       - name: download conda lock files
-        uses: actions/download-artifact@v2
+        uses: actions/download-artifact@v3
         with:
           name: conda-lock-files-${{ github.run_attempt }}
           path: conda-lock
@@ -125,7 +200,7 @@ jobs:
           fi
 
       - name: react on success
-        uses: peter-evans/create-or-update-comment@v1
+        uses: peter-evans/create-or-update-comment@v2
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
           repository: ${{ github.event.client_payload.github.payload.repository.full_name }}

.github/workflows/create-rotate-key-issue.yml

@@ -17,13 +17,13 @@ jobs:
           private_key: ${{ secrets.SQUAWK_BOT_APP_PRIVATE_KEY }}
 
       - name: checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
 
       - name: create issue to rotate key
-        uses: peter-evans/create-issue-from-file@v3
+        uses: peter-evans/create-issue-from-file@v4
         with:
-           token: ${{ steps.generate_token.outputs.token }}
-           title: "chore: rotate ibis bot keys"
-           content-filepath: .github/rotate-key-template.md
-           labels: ci
-           assignees: cpcloud,jreback
+          token: ${{ steps.generate_token.outputs.token }}
+          title: "chore: rotate ibis bot keys"
+          content-filepath: .github/rotate-key-template.md
+          labels: ci
+          assignees: cpcloud,jreback