31 changes: 31 additions & 0 deletions .github/workflows/codeql-analysis.yml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
name: CodeQL
on:
schedule:
# every day at midnight
- cron: "0 0 * * *"

concurrency:
group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }}
cancel-in-progress: true

jobs:
analyze:
runs-on: ubuntu-latest
permissions:
actions: read
contents: read
security-events: write

strategy:
fail-fast: false
matrix:
language:
- python

steps:
- uses: actions/checkout@v2
- uses: github/codeql-action/init@v1
with:
languages: ${{ matrix.language }}
- uses: github/codeql-action/autobuild@v1
- uses: github/codeql-action/analyze@v1
26 changes: 26 additions & 0 deletions .github/workflows/conda-lock-dispatch.yml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
# vim: filetype=yaml
name: Dispatch conda lock

on:
issue_comment:
types:
- created

jobs:
condalock_dispatch:
runs-on: ubuntu-latest
steps:
- name: Generate a GitHub token
uses: tibdex/github-app-token@v1
id: generate_token
with:
app_id: ${{ secrets.SQUAWK_BOT_APP_ID }}
private_key: ${{ secrets.SQUAWK_BOT_APP_PRIVATE_KEY }}

- uses: peter-evans/slash-command-dispatch@v2
with:
token: ${{ steps.generate_token.outputs.token }}
reaction-token: ${{ secrets.GITHUB_TOKEN }}
permission: none
issue-type: pull-request
commands: condalock
32 changes: 32 additions & 0 deletions .github/workflows/conda-lock-pr.yml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
name: Conda lock PR
on:
pull_request_target:
types:
- opened
- reopened
paths:
- 'conda-lock/*.lock'
- pyproject.toml
- poetry.lock

jobs:
conda_lock_comment:
runs-on: ubuntu-latest
if: ${{ github.repository == 'ibis-project/ibis' }}
steps:
- name: Generate a GitHub token
uses: tibdex/github-app-token@v1
id: generate_token
with:
app_id: ${{ secrets.SQUAWK_BOT_APP_ID }}
private_key: ${{ secrets.SQUAWK_BOT_APP_PRIVATE_KEY }}

- name: checkout
uses: actions/checkout@v2

- name: add condalock comment
uses: peter-evans/create-or-update-comment@v1
with:
token: ${{ steps.generate_token.outputs.token }}
issue-number: ${{ github.event.number }}
body: "/condalock"
133 changes: 133 additions & 0 deletions .github/workflows/conda-lock.yml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,133 @@
# vim: filetype=yaml
name: Relock conda environment files

on:
repository_dispatch:
types:
- condalock-command

jobs:
condalock:
runs-on: ubuntu-latest
strategy:
matrix:
python-version:
- "3.7"
- "3.8"
- "3.9"
defaults:
run:
shell: bash -l {0}
steps:
- name: checkout
uses: actions/checkout@v2
with:
repository: ${{ github.event.client_payload.pull_request.head.repo.full_name }}
ref: ${{ github.event.client_payload.pull_request.head.ref }}

- uses: conda-incubator/setup-miniconda@v2
with:
mamba-version: "*"
miniforge-version: latest
miniforge-variant: Mambaforge
activate-environment: conda-lock
python-version: ${{ matrix.python-version }}
condarc-file: ci/condarc

- name: install conda-lock
run: mamba install conda-lock

- name: generate lock file
run: |
set -euo pipefail
python_version_file="$(mktemp --suffix=.yml)"
{
echo 'name: conda-lock'
echo 'dependencies:'
echo ' - python=${{ matrix.python-version }}'
} > "${python_version_file}"
conda lock \
--file pyproject.toml \
--file "${python_version_file}" \
--platform linux-64 \
--platform osx-64 \
--platform win-64 \
--filename-template 'conda-lock/{platform}-${{ matrix.python-version }}.lock' \
--extras clickhouse \
--extras dask \
--extras geospatial \
--extras hdf5 \
--extras impala \
--extras mysql \
--extras parquet \
--extras postgres \
--extras pyspark \
--extras sqlite \
--mamba
- name: create conda environment
run: mamba create --name ibis${{ matrix.python-version }} --file conda-lock/linux-64-${{ matrix.python-version }}.lock

- name: upload conda lock files
uses: actions/upload-artifact@v2
with:
name: conda-lock-files-${{ github.run_attempt }}
path: conda-lock/*-${{ matrix.python-version }}.lock

condalock_push:
runs-on: ubuntu-latest
needs:
- condalock
steps:
- name: Generate a GitHub token
uses: tibdex/github-app-token@v1
id: generate_token
with:
app_id: ${{ secrets.SQUAWK_BOT_APP_ID }}
private_key: ${{ secrets.SQUAWK_BOT_APP_PRIVATE_KEY }}

- name: checkout
uses: actions/checkout@v2
with:
token: ${{ steps.generate_token.outputs.token }}
repository: ${{ github.event.client_payload.pull_request.head.repo.full_name }}
ref: ${{ github.event.client_payload.pull_request.head.ref }}

- name: download conda lock files
uses: actions/download-artifact@v2
with:
name: conda-lock-files-${{ github.run_attempt }}
path: conda-lock

- name: Configure git info
run: |
set -euo pipefail
git config --global user.name 'ibis-squawk-bot[bot]'
git config --global user.email 'ibis-squawk-bot[bot]@users.noreply.github.com'
- name: commit lock files and push to PR
run: |
set -euo pipefail
git add conda-lock/*.lock
if git commit -m 'chore(conda-lock): relock'; then
# pull in case another commit happened in the meantime
#
# `ours` is actually the *other* changeset, not the current branch, per
# https://stackoverflow.com/a/3443225/564538
git pull --rebase -s recursive -X ours
git push
fi
- name: react on success
uses: peter-evans/create-or-update-comment@v1
with:
token: ${{ secrets.GITHUB_TOKEN }}
repository: ${{ github.event.client_payload.github.payload.repository.full_name }}
comment-id: ${{ github.event.client_payload.github.payload.comment.id }}
reaction-type: hooray
29 changes: 29 additions & 0 deletions .github/workflows/create-rotate-key-issue.yml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
name: Rotate bot keys
on:
schedule:
# https://crontab.guru/#0_0_1_*/3_*
# "At 00:00 on day-of-month 1 in every 3rd month."
- cron: "0 0 1 */3 *"

jobs:
rotate_private_key:
runs-on: ubuntu-latest
steps:
- name: Generate a GitHub token
uses: tibdex/github-app-token@v1
id: generate_token
with:
app_id: ${{ secrets.SQUAWK_BOT_APP_ID }}
private_key: ${{ secrets.SQUAWK_BOT_APP_PRIVATE_KEY }}

- name: checkout
uses: actions/checkout@v2

- name: create issue to rotate key
uses: peter-evans/create-issue-from-file@v3
with:
token: ${{ steps.generate_token.outputs.token }}
title: "chore: rotate ibis bot keys"
content-filepath: .github/rotate-key-template.md
labels: ci
assignees: cpcloud,jreback
557 changes: 557 additions & 0 deletions .github/workflows/ibis-backends.yml
View file

Large diffs are not rendered by default.

172 changes: 172 additions & 0 deletions .github/workflows/ibis-docs-lint.yml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,172 @@
# vim: filetype=yaml
name: Ibis Docs and Linting

on:
push:
branches:
- master
pull_request:
branches:
- master

concurrency:
group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }}
cancel-in-progress: true

jobs:
commitlint:
runs-on: ubuntu-latest
if: ${{ github.event_name == 'pull_request' }}
steps:
- name: checkout
uses: actions/checkout@v2
with:
fetch-depth: 0

- name: install nix
uses: cachix/install-nix-action@v16
with:
nix_path: nixpkgs=channel:nixos-unstable-small

- name: lint commits
run: nix shell -L --keep-going -f '<nixpkgs>' commitlint -c commitlint --from=${{ github.event.pull_request.base.sha }} --to=${{ github.sha }} --verbose

lint:
runs-on: ubuntu-latest
steps:
- name: checkout
uses: actions/checkout@v2

- name: install nix
uses: cachix/install-nix-action@v16
with:
nix_path: nixpkgs=channel:nixos-unstable-small

- name: setup cachix
uses: cachix/cachix-action@v10
with:
name: ibis
authToken: ${{ secrets.CACHIX_AUTH_TOKEN }}
extraPullNames: nix-community,poetry2nix

- name: pre-commit checks
run: nix-shell --pure --keep-going --run 'pre-commit run --all-files'

docs:
name: Docs
runs-on: ubuntu-latest
steps:
- name: checkout
uses: actions/checkout@v2
with:
path: ibis

- name: install nix
uses: cachix/install-nix-action@v16
with:
nix_path: nixpkgs=channel:nixos-unstable-small

- name: setup cachix
uses: cachix/cachix-action@v10
with:
name: ibis
authToken: ${{ secrets.CACHIX_AUTH_TOKEN }}
extraPullNames: nix-community,poetry2nix

- name: build docs
working-directory: ibis
run: nix-shell --pure --run 'make -C docs html BUILDDIR=web/docs'

- name: build website
working-directory: ibis
run: nix-shell --pure --run 'mkdocs build -f docs/mkdocs.yml'

- name: Add config to docs
working-directory: ibis
run: |
set -euo pipefail
touch docs/site/.nojekyll
echo "ibis-project.org" > docs/site/CNAME
- name: Generate a GitHub token
if: ${{ github.event_name == 'push' }}
uses: tibdex/github-app-token@v1
id: generate_token
with:
app_id: ${{ secrets.DOCS_BOT_APP_ID }}
private_key: ${{ secrets.DOCS_BOT_APP_PRIVATE_KEY }}
repository: ibis-project/ibis-project.org

- name: checkout
uses: actions/checkout@v2
if: ${{ github.event_name == 'push' }}
with:
repository: ibis-project/ibis-project.org
token: ${{ steps.generate_token.outputs.token }}
path: ibis-project.org

- name: checkout
uses: actions/checkout@v2
if: ${{ github.event_name != 'push' }}
with:
repository: ibis-project/ibis-project.org
path: ibis-project.org

- name: Copy docbuild to checkout
working-directory: ibis
run: |
set -euo pipefail
# the trailing slash matters here; it means "everything underneath
# docbuild, but not docbuild itself"
rsync --delete --exclude=.git -avz docs/site/ ../ibis-project.org
- name: Configure git info
working-directory: ibis-project.org
run: |
set -euo pipefail
git config user.name 'ibis-docs-bot[bot]'
git config user.email 'ibis-docs-bot[bot]@users.noreply.github.com'
- name: Commit docs
working-directory: ibis-project.org
run: |
set -euo pipefail
git add .
git commit -am 'docs: ibis-project/ibis@${{ github.sha }}' || true
- name: tag docs if this is a release
if: ${{ startsWith(github.ref, 'refs/tags') }}
working-directory: ibis-project.org
run: |
set -euo pipefail
git tag "${GITHUB_REF##*/}"
- name: Push docs
if: ${{ github.event_name == 'push' }}
working-directory: ibis-project.org
run: git push --tags -f origin master

simulate_release:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
with:
fetch-depth: 0

- uses: cachix/install-nix-action@v16
with:
nix_path: nixpkgs=channel:nixos-unstable-small

- uses: cachix/cachix-action@v10
with:
name: ibis
authToken: ${{ secrets.CACHIX_AUTH_TOKEN }}
extraPullNames: nix-community,poetry2nix

- name: run semantic-release
run: ./ci/release/dry_run.sh
145 changes: 145 additions & 0 deletions .github/workflows/ibis-main.yml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,145 @@
# vim: filetype=yaml
name: Ibis

on:
push:
# Skip the test suite if all changes are in the docs directory
paths-ignore:
- 'docs/**'
branches:
- master
pull_request:
# Skip the test suite if all changes are in the docs directory
paths-ignore:
- 'docs/**'
branches:
- master

concurrency:
group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }}
cancel-in-progress: true

jobs:
nix:
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
matrix:
os:
- ubuntu-latest
python-version:
- "3.7"
- "3.8"
- "3.9"
- "3.10"
steps:
- name: checkout
uses: actions/checkout@v2

- name: install nix
uses: cachix/install-nix-action@v16
with:
nix_path: nixpkgs=channel:nixos-unstable-small

- name: setup cachix
uses: cachix/cachix-action@v10
with:
name: ibis
authToken: ${{ secrets.CACHIX_AUTH_TOKEN }}
extraPullNames: nix-community,poetry2nix

- name: nix build and run tests
run: nix build --keep-going --print-build-logs --file . --argstr python ${{ matrix.python-version }}

test_no_backends:
name: Test ${{ matrix.os }} python-${{ matrix.python-version }}
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
matrix:
os:
- ubuntu-latest
- windows-latest
python-version:
- "3.7"
- "3.8"
- "3.9"
- "3.10"
steps:
- name: checkout
uses: actions/checkout@v2

- name: install python
uses: actions/setup-python@v2
id: install_python
with:
python-version: ${{ matrix.python-version }}

- uses: syphar/restore-virtualenv@v1
with:
requirement_files: poetry.lock
custom_cache_key_element: no-backends-${{ steps.install_python.outputs.python-version }}

- uses: syphar/restore-pip-download-cache@v1
with:
requirement_files: poetry.lock
custom_cache_key_element: no-backends-${{ steps.install_python.outputs.python-version }}

- run: python -m pip install --upgrade pip poetry

- name: install ibis
run: poetry install

- name: run tests
shell: bash
run: ./ci/run_tests.sh ibis/tests

- name: publish test report
uses: actions/upload-artifact@v2
if: success() || failure()
with:
name: no-backends-${{ matrix.os }}-${{ matrix.python-version }}
path: junit.xml

benchmarks:
name: Benchmark ${{ matrix.os }} python-${{ matrix.python-version }}
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
python-version:
- "3.10"
steps:
- name: checkout
uses: actions/checkout@v2

- name: install python
uses: actions/setup-python@v2
id: install_python
with:
python-version: ${{ matrix.python-version }}

- name: install system dependencies
run: sudo apt-get install -qq -y build-essential cmake krb5-config python-dev libkrb5-dev libboost-all-dev

- uses: syphar/restore-virtualenv@v1
with:
requirement_files: poetry.lock
custom_cache_key_element: benchmarks-${{ steps.install_python.outputs.python-version }}

- uses: syphar/restore-pip-download-cache@v1
with:
requirement_files: poetry.lock
custom_cache_key_element: benchmarks-${{ steps.install_python.outputs.python-version }}

- run: python -m pip install --upgrade pip poetry

- name: install ibis
run: poetry install --extras impala

- name: benchmark
run: |
set -euo pipefail
poetry run asv machine --yes
poetry run asv dev
524 changes: 0 additions & 524 deletions .github/workflows/main.yml
View file

This file was deleted.

43 changes: 43 additions & 0 deletions .github/workflows/release.yml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
name: Release

on:
workflow_dispatch:

# we do not want more than one release workflow executing at the same time, ever
concurrency:
group: release
# cancelling in the middle of a release would create incomplete releases
# so cancel-in-progress is false
cancel-in-progress: false

jobs:
release:
runs-on: ubuntu-latest
steps:
- uses: tibdex/github-app-token@v1
id: generate_token
with:
app_id: ${{ secrets.APP_ID }}
private_key: ${{ secrets.APP_PRIVATE_KEY }}

- uses: actions/checkout@v2
with:
fetch-depth: 0
token: ${{ steps.generate_token.outputs.token }}

- uses: cachix/install-nix-action@v16
with:
nix_path: nixpkgs=channel:nixos-unstable-small
extra_nix_config: |
access-tokens = github.com=${{ secrets.GITHUB_TOKEN }}
- uses: cachix/cachix-action@v10
with:
name: ibis
extraPullNames: nix-community,poetry2nix

- name: run semantic-release
run: ./ci/release/run.sh
env:
POETRY_PYPI_TOKEN_PYPI: ${{ secrets.PYPI_TOKEN }}
GITHUB_TOKEN: ${{ steps.generate_token.outputs.token }}
12 changes: 8 additions & 4 deletions .github/workflows/test-report.yml
View file
Original file line number Diff line number Diff line change
@@ -1,24 +1,28 @@
name: Test Report
on:
workflow_run:
workflows: ['CI']
workflows: ['Ibis', 'Backends']
types:
- completed
branches-ignore:
- master

concurrency: report

jobs:
report:
if: ${{ github.event.workflow_run.conclusion == 'success' || github.event.workflow_run.conclusion == 'failure' }}
runs-on: ubuntu-latest
concurrency: report
steps:
- name: Download artifact
uses: dawidd6/action-download-artifact@v2
with:
workflow: ${{ github.event.workflow_run.workflow_id }}
workflow_conclusion: completed
pr: ${{ github.event.pull_request.number }}
path: artifacts

- name: publish test report
uses: EnricoMi/publish-unit-test-result-action@v1
with:
commit: ${{ github.event.workflow_run.head_sha }}
commit: ${{ github.event.pull_request.head_sha }}
files: artifacts/**/junit.xml
111 changes: 111 additions & 0 deletions .github/workflows/update-deps.yml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,111 @@
name: Update Dependencies
on:
schedule:
# run every 24 hours at midnight
- cron: "0 */24 * * *"
workflow_dispatch:

jobs:
generate_updates:
runs-on: ubuntu-latest
outputs:
matrix: ${{ steps.set-matrix.outputs.matrix }}
steps:
- uses: actions/checkout@v2

- name: output dependency list
id: set-matrix
run: |
set -euo pipefail
deps="$(jq -rcM '{dep: keys}' < nix/sources.json)"
echo "::set-output name=matrix::$deps"
niv_update:
runs-on: ubuntu-latest
needs:
- generate_updates
strategy:
matrix: ${{ fromJSON(needs.generate_updates.outputs.matrix) }}
steps:
- uses: actions/checkout@v2

- uses: cachix/install-nix-action@v16
with:
nix_path: nixpkgs=channel:nixos-unstable-small

- name: setup cachix
uses: cachix/cachix-action@v10
with:
name: ibis
extraPullNames: nix-community,poetry2nix

- uses: cpcloud/niv-dep-info-action@main
id: get_current_commit
with:
dependency: ${{ matrix.dep }}

- name: update ${{ matrix.dep }}
run: nix shell -f '<nixpkgs>' niv -c niv update ${{ matrix.dep }}

- uses: cpcloud/niv-dep-info-action@main
id: get_new_commit
with:
dependency: ${{ matrix.dep }}

- name: create an output indicating whether a PR is needed
id: needs_pr
run: |
set -euo pipefail
echo "::set-output name=did_change::${{ steps.get_current_commit.outputs.rev != steps.get_new_commit.outputs.rev }}"
- uses: tibdex/github-app-token@v1
if: ${{ fromJSON(steps.needs_pr.outputs.did_change) }}
id: generate_pr_token
with:
app_id: ${{ secrets.SQUAWK_BOT_APP_ID }}
private_key: ${{ secrets.SQUAWK_BOT_APP_PRIVATE_KEY }}

- uses: tibdex/github-app-token@v1
if: ${{ fromJSON(steps.needs_pr.outputs.did_change) }}
id: generate_pr_approval_token
with:
app_id: ${{ secrets.PR_APPROVAL_BOT_APP_ID }}
private_key: ${{ secrets.PR_APPROVAL_BOT_APP_PRIVATE_KEY }}

- uses: cpcloud/compare-commits-action@v5.0.13
if: ${{ fromJSON(steps.needs_pr.outputs.did_change) }}
id: compare_commits
with:
token: ${{ steps.generate_pr_token.outputs.token }}
owner: ${{ steps.get_new_commit.outputs.owner }}
repo: ${{ steps.get_new_commit.outputs.repo }}
basehead: ${{ steps.get_current_commit.outputs.rev }}...${{ steps.get_new_commit.outputs.rev }}
include-merge-commits: false

- uses: peter-evans/create-pull-request@v3
if: ${{ fromJSON(steps.needs_pr.outputs.did_change) }}
id: create_pr
with:
token: ${{ steps.generate_pr_token.outputs.token }}
commit-message: "chore(deps/${{ matrix.dep }}): update"
branch: "create-pull-request/update-${{ matrix.dep }}"
delete-branch: true
author: "ibis-squawk-bot[bot] <ibis-squawk-bot[bot]@users.noreply.github.com>"
title: "chore(deps/${{ matrix.dep }}): update"
body: ${{ steps.compare_commits.outputs.differences }}
labels: dependencies,autorebase:opt-in

- uses: juliangruber/approve-pull-request-action@v1.1.0
if: ${{ fromJSON(steps.needs_pr.outputs.did_change) }}
with:
github-token: ${{ steps.generate_pr_approval_token.outputs.token }}
number: ${{ steps.create_pr.outputs.pull-request-number }}

- uses: peter-evans/enable-pull-request-automerge@v1
if: ${{ fromJSON(steps.needs_pr.outputs.did_change) }}
with:
token: ${{ steps.generate_pr_token.outputs.token }}
pull-request-number: ${{ steps.create_pr.outputs.pull-request-number }}
merge-method: rebase
63 changes: 63 additions & 0 deletions .github/workflows/update-setup-py.yml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,63 @@
name: Update setup.py

on:
pull_request:
paths:
- ./dev/poetry2setup
- ./dev/poetry2setup.py
- pyproject.toml
- poetry.lock

jobs:
generate_setup_py:
# this can only run on pull requests made from branches in the main ibis
# repository ano not forks, since the add-and-commit action requires the
# ability to push commits to the PR branch
#
# this condition checks whether the PR is coming from a branch in the main
# repo
if: ${{ github.event.pull_request.head.repo.full_name == github.repository }}
runs-on: ubuntu-latest
steps:
- name: Generate a GitHub token
uses: tibdex/github-app-token@v1
id: generate_token
with:
app_id: ${{ secrets.SQUAWK_BOT_APP_ID }}
private_key: ${{ secrets.SQUAWK_BOT_APP_PRIVATE_KEY }}

- name: checkout
uses: actions/checkout@v2
with:
token: ${{ steps.generate_token.outputs.token }}

- name: install nix
uses: cachix/install-nix-action@v16
with:
nix_path: nixpkgs=channel:nixos-unstable-small

- name: setup cachix
uses: cachix/cachix-action@v10
with:
name: ibis
extraPullNames: nix-community,poetry2nix

- name: generate setup.py
run: ./dev/poetry2setup -o setup.py

- name: setup git credentials
uses: OleksiyRudenko/gha-git-credentials@v2.1
with:
token: ${{ steps.generate_token.outputs.token }}
global: true
name: ibis-squawk-bot[bot]
email: ibis-squawk-bot[bot]@users.noreply.github.com

- name: commit setup.py and push to pull request
uses: EndBug/add-and-commit@v7
with:
add: setup.py
author_name: ibis-squawk-bot[bot]
author_email: ibis-squawk-bot[bot]@users.noreply.github.com
message: "chore(setup.py): regenerate"
push: true
18 changes: 18 additions & 0 deletions .gitignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,24 @@ ibis_testing*
.ipynb_checkpoints/
.pytest_cache
.mypy_cache
docs/source/backends/generated

# temporary doc build
docbuild

# nix generated files
.direnv
.pre-commit-config.yaml
result
result-*

# generate this with:
#
# poetry export --dev --without-hashes --no-ansi > requirements.txt
#
# if you need it
requirements.txt

# generated mkdocs website
docs/site
docs/web/docs
4 changes: 4 additions & 0 deletions .pep8speaks.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,3 +3,7 @@
scanner:
diff_only: True # If True, errors caused by only the patch are shown
linter: flake8

flake8:
exclude:
- setup.py
24 changes: 0 additions & 24 deletions .pre-commit-config.yaml
View file

This file was deleted.

36 changes: 36 additions & 0 deletions .releaserc.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
{
"branches": ["master"],
"tagFormat": "${version}",
"plugins": [
"@semantic-release/commit-analyzer",
"@semantic-release/release-notes-generator",
[
"@semantic-release/changelog",
{
"changelogTitle": "Release Notes\n---",
"changelogFile": "docs/web/release_notes.md"
}
],
[
"@semantic-release/exec",
{
"verifyConditionsCmd": "ci/release/verify.sh",
"prepareCmd": "ci/release/prepare.sh ${nextRelease.version}",
"publishCmd": "ci/release/publish.sh"
}
],
[
"@semantic-release/github",
{
"assets": ["dist/*.whl"]
}
],
[
"@semantic-release/git",
{
"assets": ["pyproject.toml", "docs/web/release_notes.md"],
"message": "chore(release): ${nextRelease.version}"
}
]
]
}
1 change: 1 addition & 0 deletions .rgignore
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
!.github
24 changes: 0 additions & 24 deletions MANIFEST.in
View file

This file was deleted.

3 changes: 2 additions & 1 deletion README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,8 @@
| Documentation | [![Documentation Status](https://img.shields.io/badge/docs-docs.ibis--project.org-blue.svg)](http://ibis-project.org) |
| Conda packages | [![Anaconda-Server Badge](https://anaconda.org/conda-forge/ibis-framework/badges/version.svg)](https://anaconda.org/conda-forge/ibis-framework) |
| PyPI | [![PyPI](https://img.shields.io/pypi/v/ibis-framework.svg)](https://pypi.org/project/ibis-framework) |
| GitHub Actions | [![Build status](https://github.com/ibis-project/ibis/actions/workflows/main.yml/badge.svg)](https://github.com/ibis-project/ibis/actions/workflows/main.yml) |
| Ibis CI | [![Build status](https://github.com/ibis-project/ibis/actions/workflows/ibis-main.yml/badge.svg)](https://github.com/ibis-project/ibis/actions/workflows/ibis-main.yml?query=branch%3Amaster) |
| Backend CI | [![Build status](https://github.com/ibis-project/ibis/actions/workflows/ibis-backends.yml/badge.svg)](https://github.com/ibis-project/ibis/actions/workflows/ibis-backends.yml?query=branch%3Amaster) |
| Coverage | [![Codecov branch](https://img.shields.io/codecov/c/github/ibis-project/ibis/master.svg)](https://codecov.io/gh/ibis-project/ibis) |


Expand Down
54 changes: 25 additions & 29 deletions ci/datamgr.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@
import click
import pandas as pd
import sqlalchemy as sa
from plumbum import local
from toolz import dissoc

SCRIPT_DIR = Path(__file__).parent.absolute()
Expand Down Expand Up @@ -242,20 +241,12 @@ def parquet(tables, data_directory, ignore_missing_dependency, **params):
path_type=Path,
),
)
@click.option(
'-l',
'--psql-path',
type=click.Path(exists=True),
required=os.name == 'nt',
default=None if os.name == 'nt' else '/usr/bin/psql',
)
@click.option(
'--plpython/--no-plpython',
help='Create PL/Python extension in database',
default=True,
)
def postgres(schema, tables, data_directory, psql_path, plpython, **params):
psql = local[psql_path]
def postgres(schema, tables, data_directory, plpython, **params):
logger.info('Initializing PostgreSQL...')
engine = init_database(
'postgresql', params, schema, isolation_level='AUTOCOMMIT'
Expand All @@ -270,8 +261,6 @@ def postgres(schema, tables, data_directory, psql_path, plpython, **params):
if plpython:
engine.execute("CREATE EXTENSION IF NOT EXISTS PLPYTHONU")

query = "COPY {} FROM STDIN WITH (FORMAT CSV, HEADER TRUE, DELIMITER ',')"
database = params['database']
for table in tables:
src = data_directory / f'{table}.csv'

Expand All @@ -298,23 +287,21 @@ def postgres(schema, tables, data_directory, psql_path, plpython, **params):
"geo_multipolygon": Geometry("MULTIPOLYGON", srid=srid),
},
)
continue

load = psql[
'--host',
params['host'],
'--port',
params['port'],
'--username',
params['user'],
'--dbname',
database,
'--command',
query.format(table),
]
with local.env(PGPASSWORD=params['password']):
with src.open('r') as f:
load(stdin=f)
else:
# Here we insert rows using COPY table FROM STDIN, by way of
# psycopg2's `copy_expert` API.
#
# We could use DataFrame.to_sql(method=callable), but that incurs
# an unnecessary round trip and requires more code: the `data_iter`
# argument would have to be turned back into a CSV before being
# passed to `copy_expert`.
sql = (
f"COPY {table} FROM STDIN "
"WITH (FORMAT CSV, HEADER TRUE, DELIMITER ',')"
)
with src.open('r') as file:
with engine.begin() as con, con.connection.cursor() as cur:
cur.copy_expert(sql=sql, file=file)

engine.execute('VACUUM FULL ANALYZE')

Expand Down Expand Up @@ -461,6 +448,15 @@ def dask(**params):
"""


@cli.command()
def datafusion(**params):
"""
The datafusion backend does not need test data, but we still
have an option for the backend for consistency, and to not
have to avoid calling `./datamgr.py datafusion` in the CI.
"""


@cli.command()
def csv(**params):
"""
Expand Down
7 changes: 0 additions & 7 deletions ci/deps/clickhouse.yml
View file

This file was deleted.

4 changes: 0 additions & 4 deletions ci/deps/dask-min.yml
View file

This file was deleted.

6 changes: 0 additions & 6 deletions ci/deps/dask.yml
View file

This file was deleted.

3 changes: 0 additions & 3 deletions ci/deps/hdf5.yml
View file

This file was deleted.

8 changes: 0 additions & 8 deletions ci/deps/impala.yml
View file

This file was deleted.

4 changes: 0 additions & 4 deletions ci/deps/mysql.yml
View file

This file was deleted.

3 changes: 0 additions & 3 deletions ci/deps/parquet.yml
View file

This file was deleted.

7 changes: 0 additions & 7 deletions ci/deps/postgres-min.yml
View file

This file was deleted.

7 changes: 0 additions & 7 deletions ci/deps/postgres.yml
View file

This file was deleted.

6 changes: 0 additions & 6 deletions ci/deps/pyspark-min.yml
View file

This file was deleted.

3 changes: 0 additions & 3 deletions ci/deps/pyspark.yml
View file

This file was deleted.

28 changes: 0 additions & 28 deletions ci/merge_and_update_env.sh
View file

This file was deleted.

3 changes: 2 additions & 1 deletion ci/recipe/meta.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,9 +24,10 @@ requirements:
host:
- pip
- python
- setuptools
- poetry

run:
- atpublic
- cached_property
- clickhouse-driver >=0.1.3
- clickhouse-cityhash # [not win]
Expand Down
40 changes: 40 additions & 0 deletions ci/release/dry_run.sh
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
#!/usr/bin/env nix-shell
#!nix-shell -I nixpkgs=channel:nixos-unstable-small --pure -p git nodejs nix -i bash
# shellcheck shell=bash

set -euo pipefail

curdir="$PWD"
worktree="$(mktemp -d)"
branch="$(basename "$worktree")"

git worktree add "$worktree"

function cleanup() {
cd "$curdir" || exit 1
git worktree remove "$worktree"
git worktree prune
git branch -D "$branch"
}

trap cleanup EXIT ERR

cd "$worktree" || exit 1

npx --yes \
-p semantic-release \
-p "@semantic-release/commit-analyzer" \
-p "@semantic-release/release-notes-generator" \
-p "@semantic-release/changelog" \
-p "@semantic-release/exec" \
-p "@semantic-release/git" \
semantic-release \
--ci \
--dry-run \
--plugins \
--analyze-commits "@semantic-release/commit-analyzer" \
--generate-notes "@semantic-release/release-notes-generator" \
--verify-conditions "@semantic-release/changelog,@semantic-release/exec,@semantic-release/git" \
--prepare "@semantic-release/changelog,@semantic-release/exec" \
--branches "$branch" \
--repository-url "file://$PWD"
11 changes: 11 additions & 0 deletions ci/release/prepare.sh
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
#!/usr/bin/env nix-shell
#!nix-shell --pure -p poetry -i bash
# shellcheck shell=bash

set -euo pipefail

# set version
poetry version "$1"

# build artifacts
poetry build
7 changes: 7 additions & 0 deletions ci/release/publish.sh
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
#!/usr/bin/env nix-shell
#!nix-shell --pure --keep POETRY_PYPI_TOKEN_PYPI -p poetry -i bash
# shellcheck shell=bash

set -euo pipefail

poetry publish
15 changes: 15 additions & 0 deletions ci/release/run.sh
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
#!/usr/bin/env nix-shell
#!nix-shell -p cacert poetry git nodejs nix -i bash
# shellcheck shell=bash

set -euo pipefail

npx --yes \
-p semantic-release \
-p "@semantic-release/commit-analyzer" \
-p "@semantic-release/release-notes-generator" \
-p "@semantic-release/changelog" \
-p "@semantic-release/github" \
-p "@semantic-release/exec" \
-p "@semantic-release/git" \
semantic-release --ci
15 changes: 15 additions & 0 deletions ci/release/verify.sh
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
#!/usr/bin/env nix-shell
#!nix-shell --pure --keep POETRY_PYPI_TOKEN_PYPI -p poetry -p git -i bash
# shellcheck shell=bash

set -euo pipefail

# verify TOML is sane
poetry check

# verify that the lock file is up to date
poetry lock --no-update
git diff --exit-code poetry.lock

# verify that we have a token available to push to pypi using set -u
: "${POETRY_PYPI_TOKEN_PYPI}"
26 changes: 15 additions & 11 deletions ci/run_tests.sh
View file
Original file line number Diff line number Diff line change
@@ -1,26 +1,30 @@
#!/usr/bin/env bash
# Run the Ibis tests. Two environment variables are considered:
#
# Run the Ibis test suite.
#
# One environment variable is considered:
# - PYTEST_BACKENDS: Space-separated list of backends to run

set -eo pipefail

TESTS_DIRS=()

if [ -n "$PYTEST_BACKENDS" ]; then
TESTS_DIRS+=("ibis/backends/tests")
TESTS_DIRS+=("ibis/backends/tests")
fi

for backend in $PYTEST_BACKENDS; do
backend_test_dir="ibis/backends/$backend/tests"
if [ -d "$backend_test_dir" ]; then
TESTS_DIRS+=("$backend_test_dir")
fi
backend_test_dir="ibis/backends/$backend/tests"
if [ -d "$backend_test_dir" ]; then
TESTS_DIRS+=("$backend_test_dir")
fi
done

set -x

pytest "${TESTS_DIRS[@]}" \
-ra \
--junitxml=junit.xml \
--cov=ibis \
--cov-report=xml:coverage.xml "$@"
poetry run pytest "${TESTS_DIRS[@]}" \
--durations=25 \
-ra \
--junitxml=junit.xml \
--cov=ibis \
--cov-report=xml:coverage.xml "$@"
351 changes: 351 additions & 0 deletions conda-lock/linux-64-3.7.lock
View file

Large diffs are not rendered by default.

351 changes: 351 additions & 0 deletions conda-lock/linux-64-3.8.lock
View file

Large diffs are not rendered by default.

349 changes: 349 additions & 0 deletions conda-lock/linux-64-3.9.lock
View file

Large diffs are not rendered by default.

330 changes: 330 additions & 0 deletions conda-lock/osx-64-3.7.lock
View file

Large diffs are not rendered by default.

330 changes: 330 additions & 0 deletions conda-lock/osx-64-3.8.lock
View file

Large diffs are not rendered by default.

328 changes: 328 additions & 0 deletions conda-lock/osx-64-3.9.lock
View file

Large diffs are not rendered by default.

326 changes: 326 additions & 0 deletions conda-lock/win-64-3.7.lock
View file

Large diffs are not rendered by default.

326 changes: 326 additions & 0 deletions conda-lock/win-64-3.8.lock
View file

Large diffs are not rendered by default.

324 changes: 324 additions & 0 deletions conda-lock/win-64-3.9.lock
View file

Large diffs are not rendered by default.

42 changes: 42 additions & 0 deletions default.nix
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
{ python ? "3.9"
, doCheck ? true
}:
let
pkgs = import ./nix;
drv =
{ poetry2nix, python }:

poetry2nix.mkPoetryApplication {
inherit python;

projectDir = ./.;
src = pkgs.gitignoreSource ./.;

overrides = pkgs.poetry2nix.overrides.withDefaults (
import ./poetry-overrides.nix {
inherit pkgs;
inherit (pkgs) lib stdenv;
}
);

preConfigure = ''
rm -f setup.py
'';

buildInputs = with pkgs; [ graphviz-nox ];
checkInputs = with pkgs; [ graphviz-nox ];

checkPhase = ''
runHook preCheck
pytest ibis/tests --numprocesses auto
runHook postCheck
'';

inherit doCheck;

pythonImportsCheck = [ "ibis" ];
};
in
pkgs.callPackage drv {
python = pkgs."python${builtins.replaceStrings [ "." ] [ "" ] python}";
}
14 changes: 14 additions & 0 deletions dev/poetry2setup
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
#!/usr/bin/env nix-shell
#!nix-shell --pure -p python3Packages.black -p python3Packages.tomli -p python3Packages.poetry-core -p bash -i bash
# vim: filetype=sh

set -euo pipefail

dir="$(readlink -f "$(dirname "$0")")"

# PYTHONHASHSEED is set is to ensure reproducible setup.py generation
#
# Because the `extras` data structure in poetry is a frozenset and therefore
# arbitrarily ordered, regenerating setup.py without a fixed hash seed can
# cause unnecessary reordering of extras.
PYTHONHASHSEED=42 python "$dir/poetry2setup.py" "$@"
56 changes: 56 additions & 0 deletions dev/poetry2setup.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@
import argparse
import re
import sys
from pathlib import Path

import black
import tomli
from poetry.core.factory import Factory
from poetry.core.masonry.builders.sdist import SdistBuilder

# Poetry inserts a double pipe for "OR" version constraints.
# We use this regular expression to turn those into a single pipe.
DOUBLE_PIPE_REGEX = re.compile(r"\s+\|\|\s+")


def main(args: argparse.Namespace) -> None:
input_dir = args.input_directory
# create poetry things
poetry = Factory().create_poetry(input_dir)
sdist_builder = SdistBuilder(poetry)

# generate setup.py code
code = sdist_builder.build_setup().decode("UTF-8")

# pull out black config
config = tomli.loads(input_dir.joinpath("pyproject.toml").read_text())
black_config = config["tool"]["black"]
black_config["string_normalization"] = black_config.pop(
"skip_string_normalization", False
)
black_config.pop("exclude", None)
out = black.format_file_contents(
code, fast=False, mode=black.Mode(**black_config)
)
print(DOUBLE_PIPE_REGEX.sub("|", out), file=args.output_file, end="")


if __name__ == "__main__":
p = argparse.ArgumentParser(
description="Generate a setup.py file from pyproject.toml"
)
p.add_argument(
"-i",
"--input-directory",
type=Path,
default=Path(__file__).parent.parent.resolve(),
help="The input directory to use for poetry setup",
)
p.add_argument(
"-o",
"--output-file",
type=argparse.FileType(mode="w"),
default=sys.stdout,
help="The file to which to write the generated setup.py output",
)
main(p.parse_args())
7 changes: 4 additions & 3 deletions docs/Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,11 @@
#

# You can set these variables from the command line.
SPHINXOPTS =
SPHINXOPTS = -W -T -j auto
SPHINXBUILD = sphinx-build
PAPER =
BUILDDIR = build
SOURCE_DATE_EPOCH = $(shell git log -1 --format=%ct)

# User-friendly check for sphinx-build
ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1)
Expand Down Expand Up @@ -51,9 +52,9 @@ clean:
rm -rf source/generated

html:
$(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html
$(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)
@echo
@echo "Build finished. The HTML pages are in $(BUILDDIR)/html."
@echo "Build finished. The HTML pages are in $(BUILDDIR)."

dirhtml:
$(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml
Expand Down
1 change: 0 additions & 1 deletion docs/_config.yml
View file

This file was deleted.

82 changes: 82 additions & 0 deletions docs/mkdocs.yml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,82 @@
site_name: Ibis Project
project_name: "ibis"
site_url: https://ibis-project.org
repo_url: https://github.com/ibis-project/ibis
docs_dir: web
theme:
name: material
features:
- search.suggest
- search.highlight
- search.share
- content.tabs.link
- navigation.instant
- header.autohide
icon:
repo: fontawesome/brands/github
logo: static/img/logo_ibis.svg
favicon: static/img/favicon.ico
palette:
- scheme: default
media: "(prefers-color-scheme: light)"
toggle:
icon: material/toggle-switch-off-outline
name: Switch to dark mode
- scheme: slate
media: "(prefers-color-scheme: dark)"
toggle:
icon: material/toggle-switch
name: Switch to light mode
plugins:
- search
- macros
markdown_extensions:
- admonition
- meta
- toc
- tables
- attr_list
- md_in_html
- pymdownx.superfences
- pymdownx.highlight
- pymdownx.inlinehilite
- pymdownx.details
- pymdownx.tabbed:
alternate_style: true
nav:
- Home: index.md
- About:
- Introduction: about/index.md
- Team: about/team.md
- Roadmap: about/roadmap.md
- License: about/license.md
- Getting Started: getting_started.md
- Documentation: /docs
- Community:
- Ask a question (StackOverflow): https://stackoverflow.com/questions/tagged/ibis
- Chat (Gitter): https://gitter.im/ibis-dev/Lobby
- Code of Conduct: community/coc.md
- Ecosystem: community/ecosystem.md
- Contribute: contribute.md
- Release Notes: release_notes.md
team:
- name: "Active maintainers"
kind: github
members:
- jreback
- datapythonista
- cpcloud
- kszucs
- name: "Former maintainers"
kind: github
members:
- wesm

extra:
social:
- icon: fontawesome/brands/twitter
link: https://twitter.com/IbisData
- icon: fontawesome/brands/github
link: https://github.com/ibis-project/ibis

copyright: "Copyright &copy; 2014-2022, Ibis developers"
1 change: 0 additions & 1 deletion docs/source/api.rst
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,6 @@ These methods are available directly in the ``ibis`` module namespace.
now
NA
null
expr_list
row_number
window
range_window
Expand Down
16 changes: 14 additions & 2 deletions docs/source/backends/clickhouse.rst
View file
Original file line number Diff line number Diff line change
@@ -1,14 +1,26 @@
.. _install.clickhouse:

`Clickhouse <https://clickhouse.yandex/>`_
------------------------------------------
==========================================

Install
-------

Install dependencies for Ibis's Clickhouse dialect(minimal supported version is `0.1.3`):

::

pip install 'ibis-framework[clickhouse]'

or

::

conda install -c conda-forge ibis-clickhouse

Connect
-------

Create a client by passing in database connection parameters such as ``host``,
``port``, ``database``, and ``user`` to :func:`ibis.clickhouse.connect`:

Expand All @@ -20,7 +32,7 @@ Create a client by passing in database connection parameters such as ``host``,
.. _api.clickhouse:

API
===
---
.. currentmodule:: ibis.backends.clickhouse

The ClickHouse client is accessible through the ``ibis.clickhouse`` namespace.
Expand Down
56 changes: 56 additions & 0 deletions docs/source/backends/datafusion.rst
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@
.. _install.datafusion:

`Datafusion <https://arrow.apache.org/datafusion/>`_
====================================================

.. note::

The Datafusion backend is experimental

Install
-------

Install ibis along with its dependencies for the datafusion backend:

::

pip install 'ibis-framework[datafusion]'

or

::

conda install -c conda-forge ibis-datafusion

Connect
-------

Create a client by passing a dictionary that maps table names to paths to
:func:`ibis.datafusion.connect`:

.. code-block:: python
>>> import ibis
>>> data_sources = {"t": "path/to/file.parquet", "s": "path/to/file.csv"}
>>> client = ibis.datafusion.connect(data_sources)
>>> t = clien.table("t")
.. _api.datafusion:

API
---
.. currentmodule:: ibis.backends.datafusion

The Datafusion client is accessible through the ``ibis.datafusion`` namespace.

Use ``ibis.datafusion.connect`` to create a Datafusion client.

.. autosummary::
:toctree: ../generated/

Backend.connect
Backend.database
Backend.list_tables
Backend.table
Backend.register_csv
Backend.register_parquet
21 changes: 15 additions & 6 deletions docs/source/backends/impala.rst
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,32 +2,41 @@

.. _backends.impala:

******
Impala
******
`Impala <https://impala.apache.org/>`_
=======================================

One goal of Ibis is to provide an integrated Python API for an Impala cluster
without requiring you to switch back and forth between Python code and the
Impala shell (where one would be using a mix of DDL and SQL statements).

If you find an Impala task that you cannot perform with Ibis, please get in
touch on the `GitHub issue tracker <http://github.com/pandas-dev/ibis>`_.
touch on the `GitHub issue tracker <http://github.com/ibis-project/ibis>`_.

While interoperability between the Hadoop / Spark ecosystems and pandas / the
PyData stack is overall poor (but improving), we also show some ways that you
can use pandas with Ibis and Impala.

.. _install.impala:

`Impala <https://impala.apache.org/>`_ Quickstart
-------------------------------------------------
Install
-------

Install dependencies for Ibis's Impala dialect:

::

pip install 'ibis-framework[impala]'

or

::

conda install -c conda-forge ibis-impala


Connect
-------

To create an Ibis client, you must first connect your services and assemble the
client using :func:`ibis.impala.connect`:

Expand Down
1 change: 1 addition & 0 deletions docs/source/backends/index.rst
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ For more information on a specific backend, check the next backend pages:
pyspark
pandas
dask
datafusion


.. _classes_of_backends:
Expand Down
13 changes: 13 additions & 0 deletions docs/source/backends/mysql.rst
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,12 +3,25 @@
`MySQL <https://www.mysql.com/>`_
=================================

Install
-------

Install dependencies for Ibis's MySQL dialect:

::

pip install 'ibis-framework[mysql]'


or

::

conda install -c conda-forge ibis-mysql

Connect
-------

Create a client by passing a connection string or individual parameters to
:func:`ibis.mysql.connect`:

Expand Down
2 changes: 1 addition & 1 deletion docs/source/backends/pandas.rst
View file
Original file line number Diff line number Diff line change
Expand Up @@ -112,7 +112,7 @@ Using ``add_one`` from above as an example, the following call will receive a
import ibis
import pandas as pd
df = pd.DataFrame({'a': [1, 2, 3]})
con = ibis.backends.pandas.connect({'df': df})
con = ibis.pandas.connect({'df': df})
t = con.table('df')
expr = add_one(t.a)
expr
Expand Down
12 changes: 12 additions & 0 deletions docs/source/backends/postgres.rst
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,12 +3,24 @@
`PostgreSQL <https://www.postgresql.org/>`_
===========================================

Install
-------

Install dependencies for Ibis's PostgreSQL dialect:

::

pip install 'ibis-framework[postgres]'

or

::

conda install -c conda-forge ibis-postgres

Connect
-------

Create a client by passing a connection string to the ``url`` parameter or
individual parameters to :func:`ibis.postgres.connect`:

Expand Down
21 changes: 15 additions & 6 deletions docs/source/backends/pyspark.rst
View file
Original file line number Diff line number Diff line change
@@ -1,25 +1,34 @@
.. _install.pyspark:

`PySpark <https://spark.apache.org/sql/>`_
====================================================
==========================================

Install
-------

Install dependencies for Ibis's PySpark dialect:

::

pip install 'ibis-framework[pyspark]'

or

::

conda install -c conda-forge ibis-pyspark

.. note::

When using the PySpark backend with PySpark 2.4.x and pyarrow >= 0.15.0, you
need to set ``ARROW_PRE_0_15_IPC_FORMAT=1``. See `here
<https://spark.apache.org/docs/latest/api/python/user_guide/arrow_pandas.html#compatibility-setting-for-pyarrow-0-15-0-and-spark-2-3-x-2-4-x>`_
When using the PySpark backend with PySpark 2.3.x, 2.4.x and pyarrow >= 0.15.0, you
need to set ``ARROW_PRE_0_15_IPC_FORMAT=1``. See `here <https://spark.apache.org/docs/3.0.1/sql-pyspark-pandas-with-arrow.html#compatibility-setting-for-pyarrow--0150-and-spark-23x-24x>`_
for details

.. _api.pyspark:

PySpark client
~~~~~~~~~~~~~~
Connect
-------

.. currentmodule:: ibis.backends.pyspark

The PySpark client is accessible through the ``ibis.pyspark`` namespace.
Expand Down
12 changes: 12 additions & 0 deletions docs/source/backends/sqlite.rst
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,12 +3,24 @@
`SQLite <https://www.sqlite.org/>`_
===================================

Install
-------

Install dependencies for Ibis's SQLite dialect:

::

pip install 'ibis-framework[sqlite]'

or

::

conda install -c conda-forge ibis-sqlite

Connect
-------

Create a client by passing a path to a SQLite database to
:func:`ibis.sqlite.connect`:

Expand Down
33 changes: 16 additions & 17 deletions docs/source/conf.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,14 +11,9 @@
# All configuration values have a default; values that are commented out
# serve to show the default.

import datetime
import glob
import os

import sphinx_rtd_theme # noqa: E402

from ibis import __version__ as version # noqa: E402

# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
Expand All @@ -41,13 +36,9 @@
'nbsphinx',
'IPython.sphinxext.ipython_directive',
'IPython.sphinxext.ipython_console_highlighting',
'releases',
]
napoleon_google_docstring = False
napoleon_numpy_docstring = True
releases_github_path = "ibis-project/ibis"
releases_unstable_prehistory = True
releases_document_name = [os.path.join("release", "index")]
ipython_warning_is_error = True
autosummary_generate = glob.glob("*.rst") + glob.glob(
os.path.join("backends", "*.rst")
Expand All @@ -67,15 +58,14 @@

# General information about the project.
project = 'Ibis'
copyright = f'{datetime.date.today().year}, Ibis Developers'
copyright = '2014-2022, Ibis Developers'

# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
# built documents.
#
# The short X.Y version.
# version = '0.2'

version = '2.0.0'

# The full version, including alpha/beta/rc tags.
release = version
Expand Down Expand Up @@ -110,7 +100,7 @@
# show_authors = False

# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'sphinx'
pygments_style = 'material'

# A list of ignored prefixes for module index sorting.
# modindex_common_prefix = []
Expand All @@ -133,13 +123,15 @@
# a list of builtin themes.


html_theme = "sphinx_rtd_theme"
html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
html_theme = "sphinx_material"

# Theme options are theme-specific and customize the look and feel of a theme
# further. For a list of options available for each theme, see the
# documentation.
# html_theme_options = {}
html_theme_options = {
'color_primary': 'indigo',
'color_accent': 'blue',
}

# Add any paths that contain custom themes here, relative to this directory.
# html_theme_path = []
Expand Down Expand Up @@ -179,7 +171,14 @@
# html_use_smartypants = True

# Custom sidebar templates, maps document names to template names.
# html_sidebars = {}
html_sidebars = {
"**": [
"logo-text.html",
"globaltoc.html",
"localtoc.html",
"searchbox.html",
]
}

# Additional templates that should be rendered to pages, maps page names to
# template names.
Expand Down
7 changes: 3 additions & 4 deletions docs/source/index.rst
View file
Original file line number Diff line number Diff line change
Expand Up @@ -105,17 +105,16 @@ SQL engine support needing code contributors:
- `Presto <https://prestosql.io/>`_
- `Hive <https://hive.apache.org/>`_

Learning Resources
------------------

.. toctree::
:maxdepth: 1

tutorial/index
user_guide/index
api
backends/index
release/index

Learning Resources
------------------

We collect Jupyter notebooks for learning how to use ibis here:
https://github.com/ibis-project/ibis/tree/master/docs/source/notebooks/tutorial.
Expand Down
384 changes: 0 additions & 384 deletions docs/source/release/index.rst
View file

This file was deleted.

665 changes: 0 additions & 665 deletions docs/source/release/release-pre-1.0.rst
View file

This file was deleted.

2 changes: 1 addition & 1 deletion docs/source/tutorial/01-Introduction-to-Ibis.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@
"source": [
"### Getting started\n",
"\n",
"To start using Ibis, you need a Python environment with Ibis installed. If you don't know how to create an environment, we recommend following the [setup instructions](https://ibis-project.org/getting_started.html) in the Ibis website.\n",
"To start using Ibis, you need a Python environment with Ibis installed. If you don't know how to create an environment, we recommend following the [setup instructions](https://ibis-project.org/getting_started/) in the Ibis website.\n",
"\n",
"Once you have your environment ready, to start using Ibis simply import the `ibis` module:"
]
Expand Down
2 changes: 1 addition & 1 deletion docs/source/tutorial/03-Expressions-Lazy-Mode-Logging.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -138,7 +138,7 @@
"- We select the `name`, `continent` and `population` columns\n",
"- We limit the results to only the first `3` rows\n",
"\n",
"Now consider that the data is in a database, possibly in a diferent host than the one executing Ibis.\n",
"Now consider that the data is in a database, possibly in a different host than the one executing Ibis.\n",
"Also consider that the results returned to the user need to be moved to the memory of the host executing Ibis.\n",
"\n",
"When using interactive (or eager) mode, if we perform one operation at a time, we would do:\n",
Expand Down
3 changes: 2 additions & 1 deletion docs/source/user_guide/extending/extending_elementwise_expr.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,8 @@
"import ibis.expr.datatypes as dt\n",
"import ibis.expr.rules as rlz\n",
"\n",
"from ibis.expr.operations import ValueOp, Arg\n",
"from ibis.expr.operations import ValueOp\n",
"from ibis.expr.signature import Argument as Arg\n",
"\n",
"\n",
"class JulianDay(ValueOp):\n",
Expand Down
3 changes: 2 additions & 1 deletion docs/source/user_guide/extending/extending_reduce_expr.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,8 @@
"import ibis.expr.datatypes as dt\n",
"import ibis.expr.rules as rlz\n",
"\n",
"from ibis.expr.operations import Reduction, Arg\n",
"from ibis.expr.operations import Reduction\n",
"from ibis.expr.signature import Argument as Arg\n",
"\n",
"\n",
"class LastDate(Reduction):\n",
Expand Down
39 changes: 7 additions & 32 deletions docs/web/about/team.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,45 +2,20 @@

## Contributors

{{ ibis.project_name }} is developed and maintained by a
[community of volunteer contributors](https://github.com/{{ ibis.github_repo_url }}/graphs/contributors).
{{ config.project_name }} is developed and maintained by a
[community of volunteer contributors](https://github.com/{{ config.repo_url }}/graphs/contributors).


{% for group in team %}
{% for group in config.team %}

## {{ group.name }}

<div class="row maintainers">
{% for row in group.members | batch(6, "") %}
<div class="card-group maintainers">
{% for person in row %}
{% if person %}
<div class="card">
<img class="card-img-top" alt="" src="{{ person.avatar_url }}"/>
<div class="card-body">
<h6 class="card-title">
{% if person.blog %}
<a href="{{ person.blog }}">
{{ person.name or person.login }}
</a>
{% else %}
{{ person.name or person.login }}
{% endif %}
</h6>
<p class="card-text small"><a href="{{ person.html_url }}">{{ person.login }}</a></p>
</div>
</div>
{% else %}
<div class="card border-0"></div>
{% endif %}
{% endfor %}
</div>
{% endfor %}
</div>
{% for person in group.members %}
* [{{ person }}](https://github.com/{{ person }})
{% endfor %}

{% endfor %}

{{ ibis.project_name }} aims to be a welcoming, friendly, diverse and inclusive community.
{{ config.project_name }} aims to be a welcoming, friendly, diverse and inclusive community.
Everybody is welcome, regardless of gender, sexual orientation, gender identity,
and expression, disability, physical appearance, body size, race, or religion.
We do not tolerate harassment of community members in any form.
Expand Down
29 changes: 15 additions & 14 deletions docs/web/community/coc.md
View file
Original file line number Diff line number Diff line change
@@ -1,20 +1,21 @@
# Code of Conduct

Ibis is governed by the
[NumFOCUS code of conduct](https://numfocus.org/code-of-conduct),
which in a short version is:
{{ config.project_name | title }} is governed by the
[NumFOCUS code of conduct](https://numfocus.org/code-of-conduct):

Be kind to others. Do not insult or put down others. Behave professionally.
Remember that harassment and sexist, racist, or exclusionary jokes are not
appropriate for {{ ibis.project_name }}.
!!! quote

All communication should be appropriate for a professional audience
including people of many different backgrounds. Sexual language and
imagery is not appropriate.
Be kind to others. Do not insult or put down others. Behave professionally.
Remember that harassment and sexist, racist, or exclusionary jokes are not
appropriate for {{ config.project_name | upper }}.

{{ ibis.project_name }} is dedicated to providing a harassment-free community for everyone,
regardless of gender, sexual orientation, gender identity, and expression,
disability, physical appearance, body size, race, or religion. We do not
tolerate harassment of community members in any form.
All communication should be appropriate for a professional audience including
people of many different backgrounds. Sexual language and imagery is not
appropriate.

Thank you for helping make this a welcoming, friendly community for all.
{{ config.project_name | title }} is dedicated to providing a harassment-free
community for everyone, regardless of gender, sexual orientation, gender
identity, and expression, disability, physical appearance, body size, race,
or religion. We do not tolerate harassment of community members in any form.

Thank you for helping make this a welcoming, friendly community for all.
2 changes: 0 additions & 2 deletions docs/web/community/ecosystem.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,10 +51,8 @@ sql_to_ibis.query(
```
This would output a dataframe that looks like:

```
| column1 | my_col2 |
|---------|---------|
| 1 | 5 |
| 2 | 6 |
| 3 | 7 |
```
68 changes: 0 additions & 68 deletions docs/web/config.yml
View file

This file was deleted.

456 changes: 339 additions & 117 deletions docs/web/contribute.md
View file

Large diffs are not rendered by default.

35 changes: 13 additions & 22 deletions docs/web/getting_started.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,38 +2,29 @@

## Installation instructions

The next steps provides the easiest and recommended way to set up your
environment to use {{ ibis.project_name }}. Other installation options can be found in
the [advanced installation page]({{ base_url}}/docs/index.html#installation).
The next steps show the recommended way of installing {{ config.project_name }}. Other installation
options can be found in the [advanced installation
page](/docs/index.html#installation).

1. Download [Anaconda](https://www.anaconda.com/distribution/) for your operating system and
the latest Python version, run the installer, and follow the steps. Detailed instructions
on how to install Anaconda can be found in the
[Anaconda documentation](https://docs.anaconda.com/anaconda/install/)).

2. In the Anaconda prompt (or terminal in Linux or MacOS), install {{ ibis.project_name }}:
2. In a shell prompt install `ibis-framework`:

:::sh
conda install -c conda-forge ibis-framework
```sh
conda install -c conda-forge ibis-framework
```

3. In the Anaconda prompt (or terminal in Linux or MacOS), start JupyterLab:
5. In the same shell prompt, import ibis and print its version

<img class="img-fluid" alt="" src="{{ base_url }}/static/img/install/anaconda_prompt.png"/>
```sh
python -c 'import ibis; print(ibis.__version__)'
```

4. In JupyterLab, create a new (Python 3) notebook:

<img class="img-fluid" alt="" src="{{ base_url }}/static/img/install/jupyterlab_home.png"/>

5. In the first cell of the notebook, you can import {{ ibis.project_name }} and check the version with:

:::python
import ibis
ibis.__version__

6. Now you are ready to use {{ ibis.project_name }}, and you can write your code in the next cells.
6. You're ready to start using {{ config.project_name }}!
## Tutorials
You can learn more about {{ ibis.project_name }} in the
[tutorials](https://ibis-project.org/docs/tutorial/index.html),
and more about JupyterLab in the [JupyterLab documentation](https://jupyterlab.readthedocs.io/en/stable/user/interface.html).
Learn more about ibis in the [tutorials](https://ibis-project.org/docs/tutorial/index.html)!
13 changes: 4 additions & 9 deletions docs/web/index.md
View file
Original file line number Diff line number Diff line change
@@ -1,12 +1,7 @@
<div class="row">
<div class="col">
<section class="jumbotron text-center home-jumbotron">
<p>
Write your analytics code once, run it everywhere.
</p>
</section>
</div>
</div>
<figure markdown>
![Image title](/static/img/ibis_sky.png){ width="300" }
<figcaption>Write your analytics code once, run it everywhere.</figcaption>
</figure>

## Main features

Expand Down
1,121 changes: 1,121 additions & 0 deletions docs/web/release_notes.md
View file

Large diffs are not rendered by default.

69 changes: 0 additions & 69 deletions docs/web/static/css/codehilite.css
View file

This file was deleted.

104 changes: 0 additions & 104 deletions docs/web/static/css/ibis.css
View file

This file was deleted.

Binary file removed docs/web/static/img/install/anaconda_prompt.png
View file
Binary file not shown.
Binary file removed docs/web/static/img/install/jupyterlab_home.png
View file
Binary file not shown.
59 changes: 0 additions & 59 deletions environment.yml
View file

This file was deleted.

35 changes: 23 additions & 12 deletions ibis/__init__.py
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
"""Initialize Ibis module."""
import pkg_resources

# Converting an Ibis schema to a pandas DataFrame requires registering
# some type conversions that are currently registered in the pandas backend
Expand All @@ -13,12 +12,15 @@
from ibis.expr import api
from ibis.expr.api import * # noqa: F401,F403

from ._version import get_versions # noqa: E402
try:
import importlib.metadata as importlib_metadata
except ImportError:
# TODO: remove this when Python 3.7 support is dropped
import importlib_metadata

__all__ = ['api', 'ir', 'util', 'IbisError', 'options']
__all__ += api.__all__


ibis.config.register_option(
'interactive', False, validator=ibis.config.is_bool
)
Expand Down Expand Up @@ -49,8 +51,10 @@
'Number of rows to be retrieved for an unlimited table expression',
)

__version__ = get_versions()['version']
del get_versions
try:
__version__ = importlib_metadata.version(__name__)
except Exception:
__version__ = importlib_metadata.version("ibis-framework")


def __getattr__(name: str) -> BaseBackend:
Expand All @@ -68,23 +72,30 @@ def __getattr__(name: str) -> BaseBackend:
the `ibis.backends` entrypoints. If successful, the `ibis.sqlite`
attribute is "cached", so this function is only called the first time.
"""
entry_points = list(
pkg_resources.iter_entry_points(group='ibis.backends', name=name)
)
if len(entry_points) == 0:
entry_points = {
entry_point
for entry_point in importlib_metadata.entry_points()["ibis.backends"]
if name == entry_point.name
}

if not entry_points:
raise AttributeError(
f"module 'ibis' has no attribute '{name}'. "
f"If you are trying to access the '{name}' backend, "
f"try installing it first with `pip install ibis-{name}`"
)
elif len(entry_points) > 1:

if len(entry_points) > 1:
raise RuntimeError(
f"{len(entry_points)} packages found for backend '{name}'. "
f"{len(entry_points)} packages found for backend '{name}': "
f"{entry_points}\n"
"There should be only one, please uninstall the unused packages "
"and just leave the one that needs to be used."
)

backend = entry_points[0].resolve().Backend()
(entry_point,) = entry_points
module = entry_point.load()
backend = module.Backend()

# The first time a backend is loaded, we register its options, and we set
# it as an attribute of `ibis`, so `__getattr__` is not called again for it
Expand Down
520 changes: 0 additions & 520 deletions ibis/_version.py
View file

This file was deleted.

108 changes: 62 additions & 46 deletions ibis/backends/base/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,14 +2,16 @@

import abc
import re
import warnings
from typing import Any, Callable

from cached_property import cached_property

import ibis
import ibis.expr.operations as ops
import ibis.expr.schema as sch
import ibis.expr.types as ir
from ibis.common.exceptions import TranslationError
from ibis.util import deprecated

__all__ = ('BaseBackend', 'Database')

Expand Down Expand Up @@ -136,19 +138,64 @@ class BaseBackend(abc.ABC):
database_class = Database
table_class: type[ops.DatabaseTable] = ops.DatabaseTable

def __init__(self, *args, **kwargs):
self._con_args: tuple[Any] = args
self._con_kwargs: dict[str, Any] = kwargs

def __getstate__(self):
return dict(
database_class=self.database_class,
table_class=self.table_class,
_con_args=self._con_args,
_con_kwargs=self._con_kwargs,
)

def __hash__(self):
return hash(self.db_identity)

def __eq__(self, other):
return self.db_identity == other.db_identity

@property
@abc.abstractmethod
def name(self) -> str:
"""
Name of the backend, for example 'sqlite'.
"""

@abc.abstractmethod
def connect(connection_string, **options):
@cached_property
def db_identity(self) -> str:
"""
Identity of the database. Multiple connections to the same
database will have the same db_identity. Default implementation
assumes connection parameters uniquely specify the database.
"""
parts = [self.table_class.__name__]
parts.extend(self._con_args)
parts.extend(f'{k}={v}' for k, v in self._con_kwargs.items())
return '_'.join(map(str, parts))

def connect(self, *args, **kwargs) -> BaseBackend:
"""
Return new client object with saved args/kwargs, having called
.reconnect() on it.
"""
new_backend = self.__class__(*args, **kwargs)
new_backend.reconnect()
return new_backend

def reconnect(self) -> None:
"""
Connect to the underlying database and return a client object.
Reconnect to the target database already configured with connect().
"""
self.do_connect(*self._con_args, **self._con_kwargs)

def do_connect(self, *args, **kwargs) -> None:
"""
Connect to database specified by args and kwargs.
"""

@deprecated(instead='equivalent methods in the backend')
def database(self, name: str = None) -> Database:
"""
Return a Database object for the `name` database.
Expand All @@ -163,12 +210,6 @@ def database(self, name: str = None) -> Database:
Database
A database object for the specified database.
"""
warnings.warn(
'The `database` method and the `Database` object are '
'deprecated and will be removed in a future version of Ibis. '
'Use the equivalent methods in the backend instead.',
FutureWarning,
)
return self.database_class(
name=name or self.current_database, client=self
)
Expand Down Expand Up @@ -205,18 +246,11 @@ def list_databases(self, like: str = None) -> list[str]:
the `like` pattern if provided.
"""

@deprecated(version='2.0', instead='`name in client.list_databases()`')
def exists_database(self, name: str) -> bool:
"""
Return whether a database name exists in the current connection.

Deprecated in Ibis 2.0. Use `name in client.list_databases()` instead.
"""
warnings.warn(
'`client.exists_database(name)` is deprecated, and will be '
'removed in a future version of Ibis. Use '
'`name in client.list_databases()` instead.',
FutureWarning,
)
return name in self.list_databases()

@staticmethod
Expand Down Expand Up @@ -258,42 +292,25 @@ def list_tables(self, like: str = None, database: str = None) -> list[str]:
The list of the table names that match the pattern `like`.
"""

@deprecated(version='2.0', instead='`name in client.list_tables()`')
def exists_table(self, name: str, database: str = None) -> bool:
"""
Return whether a table name exists in the database.

Deprecated in Ibis 2.0. Use `name in client.list_tables()` instead.
"""
warnings.warn(
'`client.exists_table(name)` is deprecated, and will be '
'removed in a future version of Ibis. Use '
'`name in client.list_tables()` instead.',
FutureWarning,
)
return len(self.list_tables(like=name, database=database)) > 0

# @abc.abstractmethod
@deprecated(
version='2.0',
instead='change the current database before calling `.table()`',
)
def table(self, name: str, database: str = None) -> ir.TableExpr:
""" """
warnings.warn(
'`database` argument of `.table()` is deprecated and '
'will be removed in a future version of Ibis. Change '
'the current database before calling `.table()` instead',
FutureWarning,
)

@deprecated(version='2.0', instead='`.table(name).schema()`')
def get_schema(self, table_name: str, database: str = None) -> sch.Schema:
"""
Return the schema of `table_name`.

Deprecated in Ibis 2.0. Use `.table(name).schema()` instead.
"""
warnings.warn(
'`.get_schema(name)` is deprecated, and will be '
'removed in a future version of Ibis. Use '
'`.table(name).schema()` instead',
FutureWarning,
)
return self.table(name=table_name, database=database).schema()

@property
Expand Down Expand Up @@ -322,15 +339,14 @@ def compile(self, expr: ir.Expr, params=None) -> Any:
def execute(self, expr: ir.Expr) -> Any: # XXX DataFrame for now?
""" """

@deprecated(
version='2.0',
instead='`compile` and capture `TranslationError` instead',
)
def verify(self, expr: ir.Expr, params=None) -> bool:
"""
Verify `expr` is an expression that can be compiled.
"""
warnings.warn(
'`verify` is deprecated, use `compile` and capture the '
'`TranslationError` exception instead',
FutureWarning,
)
try:
self.compile(expr, params=params)
return True
Expand Down
19 changes: 8 additions & 11 deletions ibis/backends/base/file/__init__.py
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
import abc
import warnings
from pathlib import Path

import pandas as pd
Expand All @@ -8,6 +7,7 @@
import ibis.expr.types as ir
from ibis.backends.base import BaseBackend, Database
from ibis.backends.pandas.core import execute_and_reset
from ibis.util import warn_deprecated

# Load options of pandas backend
ibis.pandas
Expand Down Expand Up @@ -56,7 +56,7 @@ class BaseFileBackend(BaseBackend):

database_class = FileDatabase

def connect(self, path):
def do_connect(self, path):
"""Create a Client for use with Ibis

Parameters
Expand All @@ -67,10 +67,8 @@ def connect(self, path):
-------
Backend
"""
new_backend = self.__class__()
new_backend.path = new_backend.root = Path(path)
new_backend.dictionary = {}
return new_backend
self.path = self.root = Path(path)
self.dictionary = {}

@property
def version(self) -> str:
Expand Down Expand Up @@ -135,11 +133,10 @@ def list_databases(self, path=None, like=None):
if path is None:
path = self.path
else:
warnings.warn(
'The `path` argument of `list_databases` is deprecated and '
'will be removed in a future version of Ibis. Connect to a '
'different path with the `connect()` method instead.',
FutureWarning,
warn_deprecated(
'The `path` argument of `list_databases`',
version='2.0',
instead='`connect()` with a different path',
)
databases = ['.'] + self._list_databases_dirs(path)
return self._filter_with_like(databases, like)
Expand Down
2 changes: 1 addition & 1 deletion ibis/backends/base/sql/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -149,7 +149,7 @@ def ast_schema(self, query_ast):
dml = getattr(query_ast, 'dml', query_ast)
expr = getattr(dml, 'parent_expr', getattr(dml, 'table_set', None))

if isinstance(expr, (ir.TableExpr, ir.ExprList, sch.HasSchema)):
if isinstance(expr, (ir.TableExpr, sch.HasSchema)):
return expr.schema()
elif isinstance(expr, ir.ValueExpr):
return sch.schema([(expr.get_name(), expr.type())])
Expand Down
18 changes: 6 additions & 12 deletions ibis/backends/base/sql/alchemy/__init__.py
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
import contextlib
import getpass
import warnings
from typing import Dict, List, Optional, Union

import pandas as pd
Expand Down Expand Up @@ -101,13 +100,11 @@ def _build_alchemy_url(
database=database,
)

def connect(self, con: sqlalchemy.engine.Engine):
new_backend = self.__class__()
new_backend.con = con
new_backend.meta = sqlalchemy.MetaData(bind=con)
new_backend._inspector = sqlalchemy.inspect(con)
new_backend._schemas: Dict[str, sch.Schema] = {}
return new_backend
def do_connect(self, con: sqlalchemy.engine.Engine) -> None:
self.con = con
self._inspector = sqlalchemy.inspect(self.con)
self.meta = sqlalchemy.MetaData(bind=self.con)
self._schemas: Dict[str, sch.Schema] = {}

@property
def version(self):
Expand Down Expand Up @@ -325,11 +322,8 @@ def current_database(self):
"""The name of the current database this client is connected to."""
return self.database_name

@util.deprecated(version='2.0', instead='`list_databases`')
def list_schemas(self):
warnings.warn(
'`list_schemas` is deprecated, use `list_databases` instead',
FutureWarning,
)
return self.list_databases()

def raw_sql(self, query: str, results=False):
Expand Down
5 changes: 5 additions & 0 deletions ibis/backends/base/sql/alchemy/database.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,3 +21,8 @@ def __init__(self, table, source, schema=None):
schema = sch.infer(table, schema=schema)
super().__init__(table.name, schema, source)
self.sqla_table = table

def __getstate__(self):
d = super().__getstate__()
d['sqla_table'] = self.sqla_table
return d
30 changes: 14 additions & 16 deletions ibis/backends/base/sql/alchemy/query_builder.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -81,11 +81,8 @@ def _format_table(self, expr):
# use SQLAlchemy's TableClause and ColumnClause for unbound tables
schema = ref_op.schema
result = sa.table(
ref_op.name if ref_op.name is not None else ctx.get_ref(expr),
*(
sa.column(n, to_sqla_type(t))
for n, t in zip(schema.names, schema.types)
),
ref_op.name,
*(sa.column(n, to_sqla_type(t)) for n, t in schema.items()),
)
else:
# A subquery
Expand All @@ -96,18 +93,19 @@ def _format_table(self, expr):

# hack
if isinstance(op, ops.SelfReference):
table = ctx.get_table(ref_expr)
self_ref = table.alias(alias)
ctx.set_table(expr, self_ref)
table = ctx.get_ref(ref_expr)
self_ref = (
alias if hasattr(alias, "name") else table.alias(alias)
)
ctx.set_ref(expr, self_ref)
return self_ref
else:
return ctx.get_table(expr)
return alias

result = ctx.get_compiled_expr(expr)
alias = ctx.get_ref(expr)
result = ctx.get_compiled_expr(expr)

result = result.alias(alias)
ctx.set_table(expr, result)
result = alias if hasattr(alias, "name") else result.alias(alias)
ctx.set_ref(expr, result)
return result


Expand Down Expand Up @@ -169,7 +167,7 @@ def _compile_subqueries(self):
result = self.context.get_compiled_expr(expr)
alias = self.context.get_ref(expr)
result = result.cte(alias)
self.context.set_table(expr, result)
self.context.set_ref(expr, result)

def _compile_table_set(self):
if self.table_set is not None:
Expand All @@ -192,15 +190,15 @@ def _add_select(self, table_set):
arg = self._translate(expr, named=True)
elif isinstance(expr, ir.TableExpr):
if expr.equals(self.table_set):
cached_table = self.context.get_table(expr)
cached_table = self.context.get_ref(expr)
if cached_table is None:
# the select * case from materialized join
has_select_star = True
continue
else:
arg = table_set
else:
arg = self.context.get_table(expr)
arg = self.context.get_ref(expr)
if arg is None:
raise ValueError(expr)

Expand Down
6 changes: 3 additions & 3 deletions ibis/backends/base/sql/alchemy/registry.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -77,12 +77,12 @@ def _varargs_call(sa_func, t, expr):


def get_sqla_table(ctx, table):
if ctx.has_ref(table):
if ctx.has_ref(table, parent_contexts=True):
ctx_level = ctx
sa_table = ctx_level.get_table(table)
sa_table = ctx_level.get_ref(table)
while sa_table is None and ctx_level.parent is not ctx_level:
ctx_level = ctx_level.parent
sa_table = ctx_level.get_table(table)
sa_table = ctx_level.get_ref(table)
else:
op = table.op()
if isinstance(op, AlchemyTable):
Expand Down
30 changes: 4 additions & 26 deletions ibis/backends/base/sql/alchemy/translator.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,10 +9,6 @@


class AlchemyContext(QueryContext):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self._table_objects = {}

def collapse(self, queries):
if isinstance(queries, str):
return queries
Expand All @@ -24,30 +20,12 @@ def collapse(self, queries):
return queries[0]

def subcontext(self):
return type(self)(
compiler=self.compiler, parent=self, params=self.params
)

def _compile_subquery(self, expr):
sub_ctx = self.subcontext()
return self._to_sql(expr, sub_ctx)

def has_table(self, expr, parent_contexts=False):
key = self._get_table_key(expr)
return self._key_in(
key, '_table_objects', parent_contexts=parent_contexts
return self.__class__(
compiler=self.compiler,
parent=self,
params=self.params,
)

def set_table(self, expr, obj):
key = self._get_table_key(expr)
self._table_objects[key] = obj

def get_table(self, expr):
"""
Get the memoized SQLAlchemy expression object
"""
return self._get_table_item('_table_objects', expr)


class AlchemyExprTranslator(ExprTranslator):

Expand Down
26 changes: 1 addition & 25 deletions ibis/backends/base/sql/compiler/select_builder.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -301,26 +301,6 @@ def scalar_handler(results):
elif isinstance(expr, ir.AnalyticExpr):
return expr.to_aggregation(), toolz.identity

elif isinstance(expr, ir.ExprList):
exprs = expr.exprs()

is_aggregation = True
any_aggregation = False

for x in exprs:
if not L.is_scalar_reduction(x):
is_aggregation = False
else:
any_aggregation = True

if is_aggregation:
table = ir.find_base_table(exprs[0])
return table.aggregate(exprs), toolz.identity
elif not any_aggregation:
return expr, toolz.identity
else:
raise NotImplementedError(expr._repr())

elif isinstance(expr, ir.ColumnExpr):
op = expr.op()

Expand Down Expand Up @@ -663,11 +643,7 @@ def _collect_elements(self):
if self.table_set is None:
raise com.InternalError('no table set')
else:
# Expressions not depending on any table
if isinstance(root_op, ops.ExpressionList):
self.select_set = source_expr.exprs()
else:
self.select_set = [source_expr]
self.select_set = [source_expr]

def _collect(self, expr, toplevel=False):
op = expr.op()
Expand Down
Loading